PSARC 2007/607 ZFS cachefile property
authoreschrock
Mon, 29 Oct 2007 14:37:26 -0700
changeset 5363 36eeffc5336d
parent 5362 be320d63c97a
child 5364 ab6a3ee7392a
PSARC 2007/607 ZFS cachefile property 6620436 zpool import can dump core with multiple '-o' arguments 6620452 zfs should support placing pool configuration in alternate cachefiles
usr/src/cmd/zpool/zpool_main.c
usr/src/common/zfs/zpool_prop.c
usr/src/lib/libzfs/common/libzfs.h
usr/src/lib/libzfs/common/libzfs_import.c
usr/src/lib/libzfs/common/libzfs_pool.c
usr/src/lib/libzfs/common/libzfs_util.c
usr/src/lib/libzfs/common/mapfile-vers
usr/src/uts/common/fs/zfs/spa.c
usr/src/uts/common/fs/zfs/spa_config.c
usr/src/uts/common/fs/zfs/spa_misc.c
usr/src/uts/common/fs/zfs/sys/spa.h
usr/src/uts/common/fs/zfs/sys/spa_impl.h
usr/src/uts/common/sys/fs/zfs.h
--- a/usr/src/cmd/zpool/zpool_main.c	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/cmd/zpool/zpool_main.c	Mon Oct 29 14:37:26 2007 -0700
@@ -194,10 +194,11 @@
 		return (gettext("\thistory [-il] [<pool>] ...\n"));
 	case HELP_IMPORT:
 		return (gettext("\timport [-d dir] [-D]\n"
-		    "\timport [-o mntopts] [-o property=value] ... [-d dir]\n"
-		    "\t    [-D] [-f] [-R root] -a\n"
-		    "\timport [-o mntopts] [-o property=value] ... [-d dir]\n"
-		    "\t    [-D] [-f] [-R root] <pool | id> [newpool]\n"));
+		    "\timport [-o mntopts] [-o property=value] ... \n"
+		    "\t    [-d dir | -c cachefile] [-D] [-f] [-R root] -a\n"
+		    "\timport [-o mntopts] [-o property=value] ... \n"
+		    "\t    [-d dir | -c cachefile] [-D] [-f] [-R root] "
+		    "<pool | id> [newpool]\n"));
 	case HELP_IOSTAT:
 		return (gettext("\tiostat [-v] [pool] ... [interval "
 		    "[count]]\n"));
@@ -369,7 +370,7 @@
 
 	/* Use normalized property name for nvlist operations */
 	if (nvlist_lookup_string(proplist, zpool_prop_to_name(prop),
-	    &strval) == 0) {
+	    &strval) == 0 && prop != ZPOOL_PROP_CACHEFILE) {
 		(void) fprintf(stderr, gettext("property '%s' "
 		    "specified multiple times\n"), propname);
 		return (2);
@@ -559,7 +560,7 @@
 	nvlist_t **child;
 	uint_t children;
 	nvlist_t *props = NULL;
-	char *propval = NULL;
+	char *propval;
 
 	/* check options */
 	while ((c = getopt(argc, argv, ":fnR:m:o:")) != -1) {
@@ -575,8 +576,12 @@
 			if (add_prop_list(zpool_prop_to_name(
 			    ZPOOL_PROP_ALTROOT), optarg, &props))
 				goto errout;
+			if (nvlist_lookup_string(props,
+			    zpool_prop_to_name(ZPOOL_PROP_CACHEFILE),
+			    &propval) == 0)
+				break;
 			if (add_prop_list(zpool_prop_to_name(
-			    ZPOOL_PROP_TEMPORARY), "on", &props))
+			    ZPOOL_PROP_CACHEFILE), "none", &props))
 				goto errout;
 			break;
 		case 'm':
@@ -741,10 +746,8 @@
 	}
 
 errout:
-	if (nvroot)
-		nvlist_free(nvroot);
-	if (props)
-		nvlist_free(props);
+	nvlist_free(nvroot);
+	nvlist_free(props);
 	return (ret);
 badusage:
 	nvlist_free(props);
@@ -1257,9 +1260,13 @@
 
 /*
  * zpool import [-d dir] [-D]
- *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-d dir] [-f] -a
- *       import [-o mntopts] [-o prop=value] ... [-R root] [-D] [-d dir] [-f]
- *		<pool | id> [newpool]
+ *       import [-o mntopts] [-o prop=value] ... [-R root] [-D]
+ *              [-d dir | -c cachefile] [-f] -a
+ *       import [-o mntopts] [-o prop=value] ... [-R root] [-D]
+ *              [-d dir | -c cachefile] [-f] <pool | id> [newpool]
+ *
+ *	 -c	Read pool information from a cachefile instead of searching
+ *		devices.
  *
  *       -d	Scan in a specific directory, other than /dev/dsk.  More than
  *		one directory can be specified using multiple '-d' options.
@@ -1287,7 +1294,7 @@
 	int nsearch = 0;
 	int c;
 	int err;
-	nvlist_t *pools;
+	nvlist_t *pools = NULL;
 	boolean_t do_all = B_FALSE;
 	boolean_t do_destroyed = B_FALSE;
 	char *mntopts = NULL;
@@ -1301,13 +1308,17 @@
 	nvlist_t *props = NULL;
 	boolean_t first;
 	uint64_t pool_state;
+	char *cachefile = NULL;
 
 	/* check options */
-	while ((c = getopt(argc, argv, ":Dfd:R:ao:p:")) != -1) {
+	while ((c = getopt(argc, argv, ":afc:d:Do:p:R:")) != -1) {
 		switch (c) {
 		case 'a':
 			do_all = B_TRUE;
 			break;
+		case 'c':
+			cachefile = optarg;
+			break;
 		case 'd':
 			if (searchdirs == NULL) {
 				searchdirs = safe_malloc(sizeof (char *));
@@ -1341,8 +1352,12 @@
 			if (add_prop_list(zpool_prop_to_name(
 			    ZPOOL_PROP_ALTROOT), optarg, &props))
 				goto error;
+			if (nvlist_lookup_string(props,
+			    zpool_prop_to_name(ZPOOL_PROP_CACHEFILE),
+			    &propval) == 0)
+				break;
 			if (add_prop_list(zpool_prop_to_name(
-			    ZPOOL_PROP_TEMPORARY), "on", &props))
+			    ZPOOL_PROP_CACHEFILE), "none", &props))
 				goto error;
 			break;
 		case ':':
@@ -1360,6 +1375,11 @@
 	argc -= optind;
 	argv += optind;
 
+	if (cachefile && nsearch != 0) {
+		(void) fprintf(stderr, gettext("-c is incompatible with -d\n"));
+		usage(B_FALSE);
+	}
+
 	if (searchdirs == NULL) {
 		searchdirs = safe_malloc(sizeof (char *));
 		searchdirs[0] = "/dev/dsk";
@@ -1391,7 +1411,12 @@
 		}
 	}
 
-	if ((pools = zpool_find_import(g_zfs, nsearch, searchdirs)) == NULL) {
+	if (cachefile)
+		pools = zpool_find_import_cached(g_zfs, cachefile);
+	else
+		pools = zpool_find_import(g_zfs, nsearch, searchdirs);
+
+	if (pools == NULL) {
 		free(searchdirs);
 		return (1);
 	}
@@ -1505,8 +1530,7 @@
 		    gettext("no pools available to import\n"));
 
 error:
-	if (props)
-		nvlist_free(props);
+	nvlist_free(props);
 	nvlist_free(pools);
 	free(searchdirs);
 
--- a/usr/src/common/zfs/zpool_prop.c	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/common/zfs/zpool_prop.c	Mon Oct 29 14:37:26 2007 -0700
@@ -70,6 +70,8 @@
 	    ZFS_TYPE_POOL, "<path>", "ALTROOT");
 	register_string(ZPOOL_PROP_BOOTFS, "bootfs", NULL, PROP_DEFAULT,
 	    ZFS_TYPE_POOL, "<filesystem>", "BOOTFS");
+	register_string(ZPOOL_PROP_CACHEFILE, "cachefile", NULL, PROP_DEFAULT,
+	    ZFS_TYPE_POOL, "<file>", "CACHEFILE");
 
 	/* readonly number properties */
 	register_number(ZPOOL_PROP_SIZE, "size", 0, PROP_READONLY,
@@ -94,8 +96,6 @@
 	    ZFS_TYPE_POOL, "on | off", "DELEGATION", boolean_table);
 	register_index(ZPOOL_PROP_AUTOREPLACE, "autoreplace", 0, PROP_DEFAULT,
 	    ZFS_TYPE_POOL, "on | off", "REPLACE", boolean_table);
-	register_index(ZPOOL_PROP_TEMPORARY, "temporary", 0, PROP_DEFAULT,
-	    ZFS_TYPE_POOL, "on | off", "TEMP", boolean_table);
 
 	/* default index properties */
 	register_index(ZPOOL_PROP_FAILUREMODE, "failmode",
--- a/usr/src/lib/libzfs/common/libzfs.h	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/lib/libzfs/common/libzfs.h	Mon Oct 29 14:37:26 2007 -0700
@@ -112,6 +112,7 @@
 	EZFS_PERMRDONLY,	/* pemissions are readonly */
 	EZFS_UNSHARESMBFAILED,	/* failed to unshare over smb */
 	EZFS_SHARESMBFAILED,	/* failed to share over smb */
+	EZFS_BADCACHE,		/* bad cache file */
 	EZFS_UNKNOWN
 };
 
@@ -290,6 +291,7 @@
  * Search for pools to import
  */
 extern nvlist_t *zpool_find_import(libzfs_handle_t *, int, char **);
+extern nvlist_t *zpool_find_import_cached(libzfs_handle_t *, const char *);
 
 /*
  * Miscellaneous pool functions
--- a/usr/src/lib/libzfs/common/libzfs_import.c	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/lib/libzfs/common/libzfs_import.c	Mon Oct 29 14:37:26 2007 -0700
@@ -361,6 +361,46 @@
 	return (0);
 }
 
+static nvlist_t *
+refresh_config(libzfs_handle_t *hdl, nvlist_t *config)
+{
+	nvlist_t *nvl;
+	zfs_cmd_t zc = { 0 };
+	int err;
+
+	if (zcmd_write_conf_nvlist(hdl, &zc, config) != 0)
+		return (NULL);
+
+	if (zcmd_alloc_dst_nvlist(hdl, &zc,
+	    zc.zc_nvlist_conf_size * 2) != 0) {
+		zcmd_free_nvlists(&zc);
+		return (NULL);
+	}
+
+	while ((err = ioctl(hdl->libzfs_fd, ZFS_IOC_POOL_TRYIMPORT,
+	    &zc)) != 0 && errno == ENOMEM) {
+		if (zcmd_expand_dst_nvlist(hdl, &zc) != 0) {
+			zcmd_free_nvlists(&zc);
+			return (NULL);
+		}
+	}
+
+	if (err) {
+		(void) zpool_standard_error(hdl, errno,
+		    dgettext(TEXT_DOMAIN, "cannot discover pools"));
+		zcmd_free_nvlists(&zc);
+		return (NULL);
+	}
+
+	if (zcmd_read_dst_nvlist(hdl, &zc, &nvl) != 0) {
+		zcmd_free_nvlists(&zc);
+		return (NULL);
+	}
+
+	zcmd_free_nvlists(&zc);
+	return (nvl);
+}
+
 /*
  * Convert our list of pools into the definitive set of configurations.  We
  * start by picking the best config for each toplevel vdev.  Once that's done,
@@ -380,14 +420,13 @@
 	boolean_t config_seen;
 	uint64_t best_txg;
 	char *name, *hostname;
-	zfs_cmd_t zc = { 0 };
 	uint64_t version, guid;
-	int err;
 	uint_t children = 0;
 	nvlist_t **child = NULL;
 	uint_t c;
 	boolean_t isactive;
 	uint64_t hostid;
+	nvlist_t *nvl;
 
 	if (nvlist_alloc(&ret, 0, 0) != 0)
 		goto nomem;
@@ -587,42 +626,11 @@
 			continue;
 		}
 
-		/*
-		 * Try to do the import in order to get vdev state.
-		 */
-		if (zcmd_write_conf_nvlist(hdl, &zc, config) != 0)
+		if ((nvl = refresh_config(hdl, config)) == NULL)
 			goto error;
 
 		nvlist_free(config);
-		config = NULL;
-
-		if (zcmd_alloc_dst_nvlist(hdl, &zc,
-		    zc.zc_nvlist_conf_size * 2) != 0) {
-			zcmd_free_nvlists(&zc);
-			goto error;
-		}
-
-		while ((err = ioctl(hdl->libzfs_fd, ZFS_IOC_POOL_TRYIMPORT,
-		    &zc)) != 0 && errno == ENOMEM) {
-			if (zcmd_expand_dst_nvlist(hdl, &zc) != 0) {
-				zcmd_free_nvlists(&zc);
-				goto error;
-			}
-		}
-
-		if (err) {
-			(void) zpool_standard_error(hdl, errno,
-			    dgettext(TEXT_DOMAIN, "cannot discover pools"));
-			zcmd_free_nvlists(&zc);
-			goto error;
-		}
-
-		if (zcmd_read_dst_nvlist(hdl, &zc, &config) != 0) {
-			zcmd_free_nvlists(&zc);
-			goto error;
-		}
-
-		zcmd_free_nvlists(&zc);
+		config = nvl;
 
 		/*
 		 * Go through and update the paths for spares, now that we have
@@ -865,6 +873,113 @@
 	return (ret);
 }
 
+/*
+ * Given a cache file, return the contents as a list of importable pools.
+ */
+nvlist_t *
+zpool_find_import_cached(libzfs_handle_t *hdl, const char *cachefile)
+{
+	char *buf;
+	int fd;
+	struct stat64 statbuf;
+	nvlist_t *raw, *src, *dst;
+	nvlist_t *pools;
+	nvpair_t *elem;
+	char *name;
+	uint64_t guid;
+	boolean_t active;
+
+	if ((fd = open(cachefile, O_RDONLY)) < 0) {
+		zfs_error_aux(hdl, "%s", strerror(errno));
+		(void) zfs_error(hdl, EZFS_BADCACHE,
+		    dgettext(TEXT_DOMAIN, "failed to open cache file"));
+		return (NULL);
+	}
+
+	if (fstat64(fd, &statbuf) != 0) {
+		zfs_error_aux(hdl, "%s", strerror(errno));
+		(void) close(fd);
+		(void) zfs_error(hdl, EZFS_BADCACHE,
+		    dgettext(TEXT_DOMAIN, "failed to get size of cache file"));
+		return (NULL);
+	}
+
+	if ((buf = zfs_alloc(hdl, statbuf.st_size)) == NULL) {
+		(void) close(fd);
+		return (NULL);
+	}
+
+	if (read(fd, buf, statbuf.st_size) != statbuf.st_size) {
+		(void) close(fd);
+		free(buf);
+		(void) zfs_error(hdl, EZFS_BADCACHE,
+		    dgettext(TEXT_DOMAIN,
+		    "failed to read cache file contents"));
+		return (NULL);
+	}
+
+	(void) close(fd);
+
+	if (nvlist_unpack(buf, statbuf.st_size, &raw, 0) != 0) {
+		free(buf);
+		(void) zfs_error(hdl, EZFS_BADCACHE,
+		    dgettext(TEXT_DOMAIN,
+		    "invalid or corrupt cache file contents"));
+		return (NULL);
+	}
+
+	free(buf);
+
+	/*
+	 * Go through and get the current state of the pools and refresh their
+	 * state.
+	 */
+	if (nvlist_alloc(&pools, 0, 0) != 0) {
+		(void) no_memory(hdl);
+		nvlist_free(raw);
+		return (NULL);
+	}
+
+	elem = NULL;
+	while ((elem = nvlist_next_nvpair(raw, elem)) != NULL) {
+		verify(nvpair_value_nvlist(elem, &src) == 0);
+
+		verify(nvlist_lookup_string(src, ZPOOL_CONFIG_POOL_NAME,
+		    &name) == 0);
+		verify(nvlist_lookup_uint64(src, ZPOOL_CONFIG_POOL_GUID,
+		    &guid) == 0);
+
+		if (pool_active(hdl, name, guid, &active) != 0) {
+			nvlist_free(raw);
+			nvlist_free(pools);
+			return (NULL);
+		}
+
+		if (active)
+			continue;
+
+		if ((dst = refresh_config(hdl, src)) == NULL) {
+			nvlist_free(raw);
+			nvlist_free(pools);
+			return (NULL);
+		}
+
+		if (nvlist_add_nvlist(pools, nvpair_name(elem), dst) != 0) {
+			(void) no_memory(hdl);
+			nvlist_free(dst);
+			nvlist_free(raw);
+			nvlist_free(pools);
+			return (NULL);
+		}
+
+		nvlist_free(dst);
+	}
+
+	nvlist_free(raw);
+	return (pools);
+}
+
+
 boolean_t
 find_guid(nvlist_t *nv, uint64_t guid)
 {
--- a/usr/src/lib/libzfs/common/libzfs_pool.c	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/lib/libzfs/common/libzfs_pool.c	Mon Oct 29 14:37:26 2007 -0700
@@ -297,8 +297,8 @@
 	zpool_prop_t prop;
 	char *strval;
 	uint64_t intval;
-	int temp = -1;
-	boolean_t has_altroot = B_FALSE;
+	char *slash;
+	struct stat64 statbuf;
 
 	if (nvlist_alloc(&retprops, NV_UNIQUE_NAME, 0) != 0) {
 		(void) no_memory(hdl);
@@ -374,17 +374,6 @@
 			}
 			break;
 
-		case ZPOOL_PROP_TEMPORARY:
-			if (!create_or_import) {
-				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-				    "property '%s' can only be set during pool "
-				    "creation or import"), propname);
-				(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
-				goto error;
-			}
-			temp = intval;
-			break;
-
 		case ZPOOL_PROP_ALTROOT:
 			if (!create_or_import) {
 				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
@@ -400,25 +389,46 @@
 				(void) zfs_error(hdl, EZFS_BADPATH, errbuf);
 				goto error;
 			}
-
-			has_altroot = B_TRUE;
 			break;
-		}
-	}
-
-	if (has_altroot) {
-		if (temp == 0) {
-			zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
-			    "temporary property must be set to 'on' when "
-			    "altroot is set"));
-			(void) zfs_error(hdl, EZFS_BADPROP, errbuf);
-			goto error;
-
-		} else if (temp == -1 &&
-		    nvlist_add_uint64(retprops,
-		    zpool_prop_to_name(ZPOOL_PROP_TEMPORARY), 1) != 0) {
-			(void) no_memory(hdl);
-			goto error;
+
+		case ZPOOL_PROP_CACHEFILE:
+			if (strval[0] == '\0')
+				break;
+
+			if (strcmp(strval, "none") == 0)
+				break;
+
+			if (strval[0] != '/') {
+				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+				    "property '%s' must be empty, an "
+				    "absolute path, or 'none'"), propname);
+				(void) zfs_error(hdl, EZFS_BADPATH, errbuf);
+				goto error;
+			}
+
+			slash = strrchr(strval, '/');
+
+			if (slash[1] == '\0' || strcmp(slash, "/.") == 0 ||
+			    strcmp(slash, "/..") == 0) {
+				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+				    "'%s' is not a valid file"), strval);
+				(void) zfs_error(hdl, EZFS_BADPATH, errbuf);
+				goto error;
+			}
+
+			*slash = '\0';
+
+			if (stat64(strval, &statbuf) != 0 ||
+			    !S_ISDIR(statbuf.st_mode)) {
+				zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+				    "'%s' is not a valid directory"),
+				    strval);
+				(void) zfs_error(hdl, EZFS_BADPATH, errbuf);
+				goto error;
+			}
+
+			*slash = '/';
+			break;
 		}
 	}
 
--- a/usr/src/lib/libzfs/common/libzfs_util.c	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/lib/libzfs/common/libzfs_util.c	Mon Oct 29 14:37:26 2007 -0700
@@ -199,6 +199,8 @@
 	case EZFS_PERMRDONLY:
 		return (dgettext(TEXT_DOMAIN, "snapshot permissions cannot be"
 		    " modified"));
+	case EZFS_BADCACHE:
+		return (dgettext(TEXT_DOMAIN, "invalid or missing cache file"));
 	case EZFS_UNKNOWN:
 		return (dgettext(TEXT_DOMAIN, "unknown error"));
 	default:
--- a/usr/src/lib/libzfs/common/mapfile-vers	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/lib/libzfs/common/mapfile-vers	Mon Oct 29 14:37:26 2007 -0700
@@ -120,6 +120,7 @@
 	zpool_expand_proplist;
 	zpool_export;
 	zpool_find_import;
+	zpool_find_import_cached;
 	zpool_find_vdev;
 	zpool_get_config;
 	zpool_get_errlog;
--- a/usr/src/uts/common/fs/zfs/spa.c	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/uts/common/fs/zfs/spa.c	Mon Oct 29 14:37:26 2007 -0700
@@ -114,6 +114,8 @@
 	uint64_t cap, version;
 	zprop_source_t src = ZPROP_SRC_NONE;
 	int err;
+	char *cachefile;
+	size_t len;
 
 	/*
 	 * readonly properties
@@ -163,14 +165,24 @@
 			return (err);
 	}
 
-	if (spa->spa_temporary ==
-	    zpool_prop_default_numeric(ZPOOL_PROP_TEMPORARY))
-		src = ZPROP_SRC_DEFAULT;
-	else
-		src = ZPROP_SRC_LOCAL;
-	if (err = spa_prop_add_list(*nvp, ZPOOL_PROP_TEMPORARY, NULL,
-	    spa->spa_temporary, src))
-		return (err);
+	if (spa->spa_config_dir != NULL) {
+		if (strcmp(spa->spa_config_dir, "none") == 0) {
+			err = spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
+			    spa->spa_config_dir, 0, ZPROP_SRC_LOCAL);
+		} else {
+			len = strlen(spa->spa_config_dir) +
+			    strlen(spa->spa_config_file) + 2;
+			cachefile = kmem_alloc(len, KM_SLEEP);
+			(void) snprintf(cachefile, len, "%s/%s",
+			    spa->spa_config_dir, spa->spa_config_file);
+			err = spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE,
+			    cachefile, 0, ZPROP_SRC_LOCAL);
+			kmem_free(cachefile, len);
+		}
+
+		if (err)
+			return (err);
+	}
 
 	return (0);
 }
@@ -303,6 +315,7 @@
 		vdev_t *rvdev;
 		char *vdev_type;
 		objset_t *os;
+		char *slash;
 
 		propname = nvpair_name(elem);
 
@@ -383,6 +396,29 @@
 				error = EIO;
 			}
 			break;
+
+		case ZPOOL_PROP_CACHEFILE:
+			if ((error = nvpair_value_string(elem, &strval)) != 0)
+				break;
+
+			if (strval[0] == '\0')
+				break;
+
+			if (strcmp(strval, "none") == 0)
+				break;
+
+			if (strval[0] != '/') {
+				error = EINVAL;
+				break;
+			}
+
+			slash = strrchr(strval, '/');
+			ASSERT(slash != NULL);
+
+			if (slash[1] == '\0' || strcmp(slash, "/.") == 0 ||
+			    strcmp(slash, "/..") == 0)
+				error = EINVAL;
+			break;
 		}
 
 		if (error)
@@ -1645,7 +1681,6 @@
 	 */
 	spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS);
 	spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION);
-	spa->spa_temporary = zpool_prop_default_numeric(ZPOOL_PROP_TEMPORARY);
 	spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE);
 	if (props)
 		spa_sync_props(spa, props, CRED(), tx);
@@ -1924,6 +1959,8 @@
 		VERIFY(nvlist_dup(spa->spa_config, oldconfig, 0) == 0);
 
 	if (new_state != POOL_STATE_UNINITIALIZED) {
+		spa_config_check(spa->spa_config_dir,
+		    spa->spa_config_file);
 		spa_remove(spa);
 		spa_config_sync();
 	}
@@ -3375,7 +3412,7 @@
 	nvlist_t *nvp = arg2;
 	nvpair_t *elem;
 	uint64_t intval;
-	char *strval;
+	char *strval, *slash;
 	zpool_prop_t prop;
 	const char *propname;
 	zprop_type_t proptype;
@@ -3407,12 +3444,32 @@
 			ASSERT(spa->spa_root != NULL);
 			break;
 
-		case ZPOOL_PROP_TEMPORARY:
+		case ZPOOL_PROP_CACHEFILE:
 			/*
-			 * 'temporary' is a non-persistant property.
+			 * 'cachefile' is a non-persistent property, but note
+			 * an async request that the config cache needs to be
+			 * udpated.
 			 */
-			VERIFY(nvpair_value_uint64(elem, &intval) == 0);
-			spa->spa_temporary = intval;
+			VERIFY(nvpair_value_string(elem, &strval) == 0);
+			if (spa->spa_config_dir)
+				spa_strfree(spa->spa_config_dir);
+			if (spa->spa_config_file)
+				spa_strfree(spa->spa_config_file);
+
+			if (strval[0] == '\0') {
+				spa->spa_config_dir = NULL;
+				spa->spa_config_file = NULL;
+			} else if (strcmp(strval, "none") == 0) {
+				spa->spa_config_dir = spa_strdup(strval);
+				spa->spa_config_file = NULL;
+			} else {
+				slash = strrchr(strval, '/');
+				ASSERT(slash != NULL);
+				*slash = '\0';
+				spa->spa_config_dir = spa_strdup(strval);
+				spa->spa_config_file = spa_strdup(slash + 1);
+			}
+			spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
 			break;
 		default:
 			/*
--- a/usr/src/uts/common/fs/zfs/spa_config.c	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/uts/common/fs/zfs/spa_config.c	Mon Oct 29 14:37:26 2007 -0700
@@ -43,16 +43,18 @@
 /*
  * Pool configuration repository.
  *
- * The configuration for all pools, in addition to being stored on disk, is
- * stored in /etc/zfs/zpool.cache as a packed nvlist.  The kernel maintains
- * this list as pools are created, destroyed, or modified.
+ * Pool configuration is stored as a packed nvlist on the filesystem.  By
+ * default, all pools are stored in /etc/zfs/zpool.cache and loaded on boot
+ * (when the ZFS module is loaded).  Pools can also have the 'cachefile'
+ * property set that allows them to be stored in an alternate location until
+ * the control of external software.
  *
- * We have a single nvlist which holds all the configuration information.  When
- * the module loads, we read this information from the cache and populate the
- * SPA namespace.  This namespace is maintained independently in spa.c.
- * Whenever the namespace is modified, or the configuration of a pool is
- * changed, we call spa_config_sync(), which walks through all the active pools
- * and writes the configuration to disk.
+ * For each cache file, we have a single nvlist which holds all the
+ * configuration information.  When the module loads, we read this information
+ * from /etc/zfs/zpool.cache and populate the SPA namespace.  This namespace is
+ * maintained independently in spa.c.  Whenever the namespace is modified, or
+ * the configuration of a pool is changed, we call spa_config_sync(), which
+ * walks through all the active pools and writes the configuration to disk.
  */
 
 static uint64_t spa_config_generation = 1;
@@ -141,14 +143,104 @@
 }
 
 /*
- * Synchronize all pools to disk.  This must be called with the namespace lock
- * held.
+ * This function is called when destroying or exporting a pool.  It walks the
+ * list of active pools, and searches for any that match the given cache file.
+ * If there is only one cachefile, then the file is removed immediately,
+ * because we won't see the pool when iterating in spa_config_sync().
  */
 void
-spa_config_sync(void)
+spa_config_check(const char *dir, const char *file)
+{
+	size_t count = 0;
+	char pathname[128];
+	spa_t *spa;
+
+	if (dir != NULL && strcmp(dir, "none") == 0)
+		return;
+
+	ASSERT(MUTEX_HELD(&spa_namespace_lock));
+	spa = NULL;
+	while ((spa = spa_next(spa)) != NULL) {
+		if (dir == NULL) {
+			if (spa->spa_config_dir == NULL)
+				count++;
+		} else {
+			if (spa->spa_config_dir &&
+			    strcmp(spa->spa_config_dir, dir) == 0 &&
+			    strcmp(spa->spa_config_file, file) == 0)
+				count++;
+		}
+	}
+
+	if (count == 1) {
+		if (dir == NULL) {
+			dir = spa_config_dir;
+			file = ZPOOL_CACHE_FILE;
+		}
+
+		(void) snprintf(pathname, sizeof (pathname),
+		    "%s/%s", dir, file);
+		(void) vn_remove(pathname, UIO_SYSSPACE, RMFILE);
+	}
+}
+
+typedef struct spa_config_entry {
+	list_t		sc_link;
+	const char	*sc_dir;
+	const char	*sc_file;
+	nvlist_t	*sc_nvl;
+} spa_config_entry_t;
+
+static void
+spa_config_entry_add(list_t *listp, spa_t *spa)
 {
-	spa_t *spa = NULL;
-	nvlist_t *config;
+	spa_config_entry_t *entry;
+	const char *dir, *file;
+
+	mutex_enter(&spa->spa_config_cache_lock);
+	if (!spa->spa_config || !spa->spa_name) {
+		mutex_exit(&spa->spa_config_cache_lock);
+		return;
+	}
+
+	if (spa->spa_config_dir) {
+		dir = spa->spa_config_dir;
+		file = spa->spa_config_file;
+	} else {
+		dir = spa_config_dir;
+		file = ZPOOL_CACHE_FILE;
+	}
+
+	if (strcmp(dir, "none") == 0) {
+		mutex_exit(&spa->spa_config_cache_lock);
+		return;
+	}
+
+	for (entry = list_head(listp); entry != NULL;
+	    entry = list_next(listp, entry)) {
+		if (strcmp(entry->sc_dir, dir) == 0 &&
+		    strcmp(entry->sc_file, file) == 0)
+			break;
+	}
+
+	if (entry == NULL) {
+		entry = kmem_alloc(sizeof (spa_config_entry_t), KM_SLEEP);
+		entry->sc_dir = dir;
+		entry->sc_file = file;
+		VERIFY(nvlist_alloc(&entry->sc_nvl, NV_UNIQUE_NAME,
+		    KM_SLEEP) == 0);
+		list_insert_tail(listp, entry);
+	}
+
+	VERIFY(nvlist_add_nvlist(entry->sc_nvl, spa->spa_name,
+	    spa->spa_config) == 0);
+	mutex_exit(&spa->spa_config_cache_lock);
+}
+
+static void
+spa_config_entry_write(spa_config_entry_t *entry)
+{
+	nvlist_t *config = entry->sc_nvl;
 	size_t buflen;
 	char *buf;
 	vnode_t *vp;
@@ -156,24 +248,6 @@
 	char pathname[128];
 	char pathname2[128];
 
-	ASSERT(MUTEX_HELD(&spa_namespace_lock));
-
-	VERIFY(nvlist_alloc(&config, NV_UNIQUE_NAME, KM_SLEEP) == 0);
-
-	/*
-	 * Add all known pools to the configuration list, ignoring those with
-	 * alternate root paths.
-	 */
-	spa = NULL;
-	while ((spa = spa_next(spa)) != NULL) {
-		mutex_enter(&spa->spa_config_cache_lock);
-		if (spa->spa_config && spa->spa_name && !spa->spa_temporary) {
-			VERIFY(nvlist_add_nvlist(config, spa->spa_name,
-			    spa->spa_config) == 0);
-		}
-		mutex_exit(&spa->spa_config_cache_lock);
-	}
-
 	/*
 	 * Pack the configuration into a buffer.
 	 */
@@ -189,8 +263,8 @@
 	 * 'write to temporary file, sync, move over original' to make sure we
 	 * always have a consistent view of the data.
 	 */
-	(void) snprintf(pathname, sizeof (pathname), "%s/%s", spa_config_dir,
-	    ZPOOL_CACHE_TMP);
+	(void) snprintf(pathname, sizeof (pathname), "%s/.%s", entry->sc_dir,
+	    entry->sc_file);
 
 	if (vn_open(pathname, UIO_SYSSPACE, oflags, 0644, &vp, CRCREAT, 0) != 0)
 		goto out;
@@ -199,7 +273,7 @@
 	    0, RLIM64_INFINITY, kcred, NULL) == 0 &&
 	    VOP_FSYNC(vp, FSYNC, kcred, NULL) == 0) {
 		(void) snprintf(pathname2, sizeof (pathname2), "%s/%s",
-		    spa_config_dir, ZPOOL_CACHE_FILE);
+		    entry->sc_dir, entry->sc_file);
 		(void) vn_rename(pathname, pathname2, UIO_SYSSPACE);
 	}
 
@@ -208,10 +282,41 @@
 
 out:
 	(void) vn_remove(pathname, UIO_SYSSPACE, RMFILE);
-	spa_config_generation++;
+	kmem_free(buf, buflen);
+}
+
+/*
+ * Synchronize all pools to disk.  This must be called with the namespace lock
+ * held.
+ */
+void
+spa_config_sync(void)
+{
+	spa_t *spa = NULL;
+	list_t files = { 0 };
+	spa_config_entry_t *entry;
+
+	ASSERT(MUTEX_HELD(&spa_namespace_lock));
 
-	kmem_free(buf, buflen);
-	nvlist_free(config);
+	list_create(&files, sizeof (spa_config_entry_t),
+	    offsetof(spa_config_entry_t, sc_link));
+
+	/*
+	 * Add all known pools to the configuration list, ignoring those with
+	 * alternate root paths.
+	 */
+	spa = NULL;
+	while ((spa = spa_next(spa)) != NULL)
+		spa_config_entry_add(&files, spa);
+
+	while ((entry = list_head(&files)) != NULL) {
+		spa_config_entry_write(entry);
+		list_remove(&files, entry);
+		nvlist_free(entry->sc_nvl);
+		kmem_free(entry, sizeof (spa_config_entry_t));
+	}
+
+	spa_config_generation++;
 }
 
 /*
--- a/usr/src/uts/common/fs/zfs/spa_misc.c	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/uts/common/fs/zfs/spa_misc.c	Mon Oct 29 14:37:26 2007 -0700
@@ -313,6 +313,11 @@
 	if (spa->spa_name)
 		spa_strfree(spa->spa_name);
 
+	if (spa->spa_config_dir)
+		spa_strfree(spa->spa_config_dir);
+	if (spa->spa_config_file)
+		spa_strfree(spa->spa_config_file);
+
 	spa_config_set(spa, NULL);
 
 	refcount_destroy(&spa->spa_refcount);
--- a/usr/src/uts/common/fs/zfs/sys/spa.h	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/uts/common/fs/zfs/sys/spa.h	Mon Oct 29 14:37:26 2007 -0700
@@ -374,6 +374,7 @@
 #define	SPA_CONFIG_UPDATE_VDEVS	1
 
 extern void spa_config_sync(void);
+extern void spa_config_check(const char *, const char *);
 extern void spa_config_load(void);
 extern nvlist_t *spa_all_configs(uint64_t *);
 extern void spa_config_set(spa_t *spa, nvlist_t *config);
--- a/usr/src/uts/common/fs/zfs/sys/spa_impl.h	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/uts/common/fs/zfs/sys/spa_impl.h	Mon Oct 29 14:37:26 2007 -0700
@@ -140,7 +140,8 @@
 	uint64_t	spa_pool_props_object;	/* object for properties */
 	uint64_t	spa_bootfs;		/* default boot filesystem */
 	boolean_t	spa_delegation;		/* delegation on/off */
-	boolean_t	spa_temporary;		/* temporary on/off */
+	char		*spa_config_dir;	/* cache file directory */
+	char		*spa_config_file;	/* cache file name */
 	list_t		spa_zio_list;		/* zio error list */
 	kcondvar_t	spa_zio_cv;		/* resume I/O pipeline */
 	kmutex_t	spa_zio_lock;		/* zio error lock */
--- a/usr/src/uts/common/sys/fs/zfs.h	Mon Oct 29 14:31:40 2007 -0700
+++ b/usr/src/uts/common/sys/fs/zfs.h	Mon Oct 29 14:37:26 2007 -0700
@@ -120,7 +120,7 @@
 	ZPOOL_PROP_BOOTFS,
 	ZPOOL_PROP_DELEGATION,
 	ZPOOL_PROP_AUTOREPLACE,
-	ZPOOL_PROP_TEMPORARY,
+	ZPOOL_PROP_CACHEFILE,
 	ZPOOL_PROP_FAILUREMODE,
 	ZPOOL_NUM_PROPS
 } zpool_prop_t;