usr/src/lib/trousers/Patches/tspps.c.patch
author Cyril Plisko <cyril.plisko@grigale.com>
Tue, 16 Nov 2010 22:41:08 -0800
changeset 54 c8df60226931
parent 10 612517e396e0
child 58 2605957be88f
permissions -rw-r--r--
Import sfw build 153 Bugs Fixed ---------- 6917181 svc-net-snmp does not work on OpenSolaris 6917302 Apache HTTPD 1.3 removal 6921017 RFE: update tcpdump to version 4.1.1 after release 6921019 RFE: update libpcap to version 1.1.1 6932608 core found from snmpd SIGSEGV at netsnmp_access_systemstats_entry_update 6934478 snmpd dumps core in var_extensible_vmstat 6956251 snmpd cores with SIGFPE in update_stats 6981644 Upgrade bash to 4.1 6983060 BIND Packages Description needs updating when version changes. 6984919 SMA snmpd takes a long time to service ifOperStatus and ifAdminStatus requests 6985355 trousers should use /var/user/$USERNAME subdirectories to store data 6992874 Upgrade CURL to 7.21.2 6993289 Please migrate rdiff-backup to Python 2.6 or otherwise remove Python 2.4 dependency 6993295 Please migrate grails to Python 2.6. 6993306 antlr-2 packages depend on Python 2.4, but do not seem to require Python 2.4 at all. 6993308 pywbem seems to depend on Python 2.4 but actually uses Python 2.6 6993310 Please remove python 2.4 pycups bindings 6993610 Wireshark update to version 1.2.12 6995326 switch sfw to sunstudio12.1

--- src/tspi/ps/tspps.c	2010-01-15 03:48:29.000000000 -0800
+++ src/tspi/ps/tspps.c.new	2010-09-16 08:36:51.294495115 -0700
@@ -19,6 +19,21 @@
 #include <sys/file.h>
 #include <sys/stat.h>
 #include <assert.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <netdb.h>
+#if defined (HAVE_BYTEORDER_H)
+#include <sys/byteorder.h>
+#elif defined(HAVE_ENDIAN_H)
+#include <endian.h>
+#define LE_16 htole16
+#define LE_32 htole32
+#define LE_64 htole64
+#else
+#define LE_16(x) (x)
+#define LE_32(x) (x)
+#define LE_64(x) (x)
+#endif
 
 #include "trousers/tss.h"
 #include "trousers/trousers.h"
@@ -33,6 +48,17 @@
 #if (defined (__FreeBSD__) || defined (__OpenBSD__))
 static MUTEX_DECLARE_INIT(user_ps_path);
 #endif
+#if defined (SOLARIS)
+static struct flock fl = {
+       0,
+       0,
+       0,
+       0,
+       0,
+       0,
+       {0, 0, 0, 0}
+};
+#endif
 
 
 /*
@@ -44,11 +70,12 @@
 	TSS_RESULT result;
 	char *file_name = NULL, *home_dir = NULL;
 	struct passwd *pwp;
-#if (defined (__linux) || defined (linux))
+#if (defined (__linux) || defined (linux) || defined (SOLARIS))
 	struct passwd pw;
 #endif
 	struct stat stat_buf;
 	char buf[PASSWD_BUFSIZE];
+	char pwbuf[PASSWD_BUFSIZE];
 	uid_t euid;
 	int rc;
 
@@ -62,6 +89,23 @@
 
 	euid = geteuid();
 
+#if defined (SOLARIS)
+	/*
+         * Solaris keeps user PS in a local directory instead of
+         * in the user's home directory, which may be shared
+         * by multiple systems.
+         *
+         * The directory path on Solaris is /var/user/[USERNAME]/tpm/userps
+         */
+	pwp = NULL;
+	(void) getpwuid_r(euid, &pw, pwbuf, sizeof (pwbuf), &pwp);
+	if (pwp != NULL) {
+        	rc = snprintf(buf, sizeof (buf), "/var/user/%s/tpm/userps",
+		    pwp->pw_name);
+	} else {
+		return TSPERR(TSS_E_INTERNAL_ERROR);
+	}
+#else
 	setpwent();
 	while (1) {
 #if (defined (__linux) || defined (linux))
@@ -93,8 +137,9 @@
 		return TSPERR(TSS_E_OUTOFMEMORY);
 
 	/* Tack on TSS_USER_PS_DIR and see if it exists */
-	rc = snprintf(buf, PASSWD_BUFSIZE, "%s/%s", home_dir, TSS_USER_PS_DIR);
-	if (rc == PASSWD_BUFSIZE) {
+	rc = snprintf(buf, sizeof (buf), "%s/%s", home_dir, TSS_USER_PS_DIR);
+#endif /* SOLARIS */
+	if (rc == sizeof (buf)) {
 		LogDebugFn("USER PS: Path to file too long! (> %d bytes)", PASSWD_BUFSIZE);
 		result = TSPERR(TSS_E_INTERNAL_ERROR);
 		goto done;
@@ -104,7 +149,7 @@
 	if ((rc = stat(buf, &stat_buf)) == -1) {
 		if (errno == ENOENT) {
 			errno = 0;
-			/* Create the base directory, $HOME/.trousers */
+			/* Create the user's ps directory if it is not there. */
 			if ((rc = mkdir(buf, 0700)) == -1) {
 				LogDebugFn("USER PS: Error creating dir: %s: %s", buf,
 					   strerror(errno));
@@ -119,10 +164,15 @@
 	}
 
 	/* Directory exists or has been created, return the path to the file */
-	rc = snprintf(buf, PASSWD_BUFSIZE, "%s/%s/%s", home_dir, TSS_USER_PS_DIR,
+#if defined (SOLARIS)
+        rc = snprintf(buf, sizeof (buf), "/var/user/%s/tpm/userps/%s",
+	    pwp->pw_name, TSS_USER_PS_FILE);
+#else
+	rc = snprintf(buf, sizeof (buf), "%s/%s/%s", home_dir, TSS_USER_PS_DIR,
 		      TSS_USER_PS_FILE);
-	if (rc == PASSWD_BUFSIZE) {
-		LogDebugFn("USER PS: Path to file too long! (> %d bytes)", PASSWD_BUFSIZE);
+#endif
+	if (rc == sizeof (buf)) {
+		LogDebugFn("USER PS: Path to file too long! (> %d bytes)", sizeof (buf));
 	} else
 		*file = strdup(buf);
 
@@ -143,12 +193,16 @@
 
 	/* check the global file handle first.  If it exists, lock it and return */
 	if (user_ps_fd != -1) {
+#if defined (SOLARIS)
+		fl.l_type = F_WRLCK;
+		if ((rc = fcntl(user_ps_fd, F_SETLKW, &fl))) {
+#else
 		if ((rc = flock(user_ps_fd, LOCK_EX))) {
+#endif /* SOLARIS */
 			LogDebug("USER PS: failed to lock file: %s", strerror(errno));
 			MUTEX_UNLOCK(user_ps_lock);
 			return TSPERR(TSS_E_INTERNAL_ERROR);
 		}
-
 		*fd = user_ps_fd;
 		return TSS_SUCCESS;
 	}
@@ -167,8 +221,12 @@
 		MUTEX_UNLOCK(user_ps_lock);
 		return TSPERR(TSS_E_INTERNAL_ERROR);
 	}
-
+#if defined (SOLARIS)
+	fl.l_type = F_WRLCK;
+	if ((rc = fcntl(user_ps_fd, F_SETLKW, &fl))) {
+#else
 	if ((rc = flock(user_ps_fd, LOCK_EX))) {
+#endif /* SOLARIS */
 		LogDebug("USER PS: failed to get lock of %s: %s", file_name, strerror(errno));
 		free(file_name);
 		close(user_ps_fd);
@@ -190,7 +248,12 @@
 	fsync(fd);
 
 	/* release the file lock */
+#if defined (SOLARIS)
+	fl.l_type = F_UNLCK;
+	if ((rc = fcntl(fd, F_SETLKW, &fl))) {
+#else
 	if ((rc = flock(fd, LOCK_UN))) {
+#endif /* SOLARIS */
 		LogDebug("USER PS: failed to unlock file: %s", strerror(errno));
 		rc = -1;
 	}
@@ -365,6 +428,7 @@
 		LogDebug("read of %zd bytes: %s", sizeof(UINT32), strerror(errno));
 		return TSPERR(TSS_E_INTERNAL_ERROR);
 	}
+	num_keys = LE_32(num_keys);
 
 	if (increment)
 		num_keys++;
@@ -377,6 +441,7 @@
 		return TSPERR(TSS_E_INTERNAL_ERROR);
 	}
 
+	num_keys = LE_32(num_keys);
 	if ((result = write_data(fd, (void *)&num_keys, sizeof(UINT32)))) {
 		LogDebug("%s", __FUNCTION__);
 		return result;
@@ -498,16 +563,20 @@
 	}
 
 	/* [UINT16   pub_data_size0  ] yes */
+	pub_key_size = LE_16(pub_key_size);
         if ((result = write_data(fd, &pub_key_size, sizeof(UINT16)))) {
 		LogDebug("%s", __FUNCTION__);
 		goto done;
 	}
+	pub_key_size = LE_16(pub_key_size);
 
 	/* [UINT16   blob_size0      ] yes */
+	key_blob_size = LE_16(key_blob_size);
         if ((result = write_data(fd, &key_blob_size, sizeof(UINT16)))) {
 		LogDebug("%s", __FUNCTION__);
 		goto done;
 	}
+	key_blob_size = LE_16(key_blob_size);
 
 	/* [UINT32   vendor_data_size0 ] yes */
         if ((result = write_data(fd, &zero, sizeof(UINT32)))) {
@@ -516,10 +585,12 @@
 	}
 
 	/* [UINT16   cache_flags0    ] yes */
+	cache_flags = LE_16(cache_flags);
         if ((result = write_data(fd, &cache_flags, sizeof(UINT16)))) {
 		LogDebug("%s", __FUNCTION__);
 		goto done;
 	}
+	cache_flags = LE_16(cache_flags);
 
 	/* [BYTE[]   pub_data0       ] no */
         if ((result = write_data(fd, (void *)key.pubKey.key, pub_key_size))) {
@@ -685,6 +756,7 @@
 			LogDebug("%s", __FUNCTION__);
 			goto err_exit;
 		}
+		tmp[i].pub_data_size = LE_16(tmp[i].pub_data_size);
 
 		DBG_ASSERT(tmp[i].pub_data_size <= 2048);
 
@@ -693,6 +765,7 @@
 			LogDebug("%s", __FUNCTION__);
 			goto err_exit;
 		}
+		tmp[i].blob_size = LE_16(tmp[i].blob_size);
 
 		DBG_ASSERT(tmp[i].blob_size <= 4096);
 
@@ -701,12 +774,14 @@
 			LogDebug("%s", __FUNCTION__);
 			goto err_exit;
 		}
+		tmp[i].vendor_data_size = LE_32(tmp[i].vendor_data_size);
 
 		/* cache flags */
 		if ((result = read_data(fd, &tmp[i].flags, sizeof(UINT16)))) {
 			LogDebug("%s", __FUNCTION__);
 			goto err_exit;
 		}
+		tmp[i].flags = LE_16(tmp[i].flags);
 
 		/* fast forward over the pub key */
 		offset = lseek(fd, tmp[i].pub_data_size, SEEK_CUR);
@@ -1031,6 +1106,8 @@
 		num_keys = 0;
 	}
 
+	/* The system PS file is written in little-endian */
+	num_keys = LE_32(num_keys);
 	return num_keys;
 }
 
@@ -1109,7 +1186,7 @@
 			LogDebug("%s", __FUNCTION__);
 			return result;
 		}
-
+		c->pub_data_size = LE_16(c->pub_data_size);
 		DBG_ASSERT(c->pub_data_size <= 2048 && c->pub_data_size > 0);
 
 		/* blob size */
@@ -1117,7 +1194,7 @@
 			LogDebug("%s", __FUNCTION__);
 			return result;
 		}
-
+		c->blob_size = LE_16(c->blob_size); 
 		DBG_ASSERT(c->blob_size <= 4096 && c->blob_size > 0);
 
 		/* vendor data size */
@@ -1125,12 +1202,14 @@
 			LogDebug("%s", __FUNCTION__);
 			return result;
 		}
+		c->vendor_data_size = LE_32(c->vendor_data_size); 
 
 		/* cache flags */
 		if ((result = read_data(fd, &c->flags, sizeof(UINT16)))) {
 			LogDebug("%s", __FUNCTION__);
 			return result;
 		}
+		c->flags = LE_16(c->flags); 
 
 		/* fast forward over the pub key */
 		offset = lseek(fd, c->pub_data_size, SEEK_CUR);
@@ -1198,6 +1277,7 @@
 			return result;
 		}
 
+		c->pub_data_size = LE_16(c->pub_data_size);
 		DBG_ASSERT(c->pub_data_size <= 2048 && c->pub_data_size > 0);
 
 		/* blob size */
@@ -1206,6 +1286,7 @@
 			return result;
 		}
 
+		c->blob_size = LE_16(c->blob_size);
 		DBG_ASSERT(c->blob_size <= 4096 && c->blob_size > 0);
 
 		/* vendor data size */
@@ -1213,12 +1294,14 @@
 			LogDebug("%s", __FUNCTION__);
 			return result;
 		}
+		c->vendor_data_size = LE_32(c->vendor_data_size);
 
 		/* cache flags */
 		if ((result = read_data(fd, &c->flags, sizeof(UINT16)))) {
 			LogDebug("%s", __FUNCTION__);
 			return result;
 		}
+		c->flags = LE_16(c->flags);
 
 		if (c->pub_data_size == pub_size) {
 			/* read in the pub key */