PSARC 2008/393 zfs primarycache and secondarycache properties
authorek110237
Thu, 31 Jul 2008 10:57:04 -0700
changeset 7237 f47d41541b14
parent 7236 a4434ad6aa68
child 7238 af0f33320dfb
PSARC 2008/393 zfs primarycache and secondarycache properties 6429855 Need a way to tell ZFS that caching is a lost cause 6652525 want to limit L2ARC to only cache particular datasets, and only metadata
usr/src/common/zfs/zfs_prop.c
usr/src/uts/common/fs/zfs/arc.c
usr/src/uts/common/fs/zfs/dbuf.c
usr/src/uts/common/fs/zfs/dmu.c
usr/src/uts/common/fs/zfs/dmu_objset.c
usr/src/uts/common/fs/zfs/dsl_dataset.c
usr/src/uts/common/fs/zfs/sys/arc.h
usr/src/uts/common/fs/zfs/sys/dbuf.h
usr/src/uts/common/fs/zfs/sys/dmu_objset.h
usr/src/uts/common/sys/fs/zfs.h
--- a/usr/src/common/zfs/zfs_prop.c	Thu Jul 31 10:29:01 2008 -0700
+++ b/usr/src/common/zfs/zfs_prop.c	Thu Jul 31 10:57:04 2008 -0700
@@ -151,6 +151,13 @@
 		{ NULL }
 	};
 
+	static zprop_index_t cache_table[] = {
+		{ "none",	ZFS_CACHE_NONE },
+		{ "metadata",	ZFS_CACHE_METADATA },
+		{ "all",	ZFS_CACHE_ALL },
+		{ NULL }
+	};
+
 	/* inherit index properties */
 	register_index(ZFS_PROP_CHECKSUM, "checksum", ZIO_CHECKSUM_DEFAULT,
 	    PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME,
@@ -173,6 +180,14 @@
 	register_index(ZFS_PROP_COPIES, "copies", 1,
 	    PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME,
 	    "1 | 2 | 3", "COPIES", copies_table);
+	register_index(ZFS_PROP_PRIMARYCACHE, "primarycache",
+	    ZFS_CACHE_ALL, PROP_INHERIT,
+	    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_SNAPSHOT | ZFS_TYPE_VOLUME,
+	    "all | none | metadata", "PRIMARYCACHE", cache_table);
+	register_index(ZFS_PROP_SECONDARYCACHE, "secondarycache",
+	    ZFS_CACHE_ALL, PROP_INHERIT,
+	    ZFS_TYPE_FILESYSTEM | ZFS_TYPE_SNAPSHOT | ZFS_TYPE_VOLUME,
+	    "all | none | metadata", "SECONDARYCACHE", cache_table);
 
 	/* inherit index (boolean) properties */
 	register_index(ZFS_PROP_ATIME, "atime", 1, PROP_INHERIT,
--- a/usr/src/uts/common/fs/zfs/arc.c	Thu Jul 31 10:29:01 2008 -0700
+++ b/usr/src/uts/common/fs/zfs/arc.c	Thu Jul 31 10:57:04 2008 -0700
@@ -475,11 +475,10 @@
 #define	ARC_BUF_AVAILABLE	(1 << 13)	/* block not in active use */
 #define	ARC_INDIRECT		(1 << 14)	/* this is an indirect block */
 #define	ARC_FREE_IN_PROGRESS	(1 << 15)	/* hdr about to be freed */
-#define	ARC_DONT_L2CACHE	(1 << 16)	/* originated by prefetch */
-#define	ARC_L2_WRITING		(1 << 17)	/* L2ARC write in progress */
-#define	ARC_L2_EVICTED		(1 << 18)	/* evicted during I/O */
-#define	ARC_L2_WRITE_HEAD	(1 << 19)	/* head of write list */
-#define	ARC_STORED		(1 << 20)	/* has been store()d to */
+#define	ARC_L2_WRITING		(1 << 16)	/* L2ARC write in progress */
+#define	ARC_L2_EVICTED		(1 << 17)	/* evicted during I/O */
+#define	ARC_L2_WRITE_HEAD	(1 << 18)	/* head of write list */
+#define	ARC_STORED		(1 << 19)	/* has been store()d to */
 
 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_IN_HASH_TABLE)
 #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS)
@@ -487,7 +486,7 @@
 #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FREED_IN_READ)
 #define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_BUF_AVAILABLE)
 #define	HDR_FREE_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FREE_IN_PROGRESS)
-#define	HDR_DONT_L2CACHE(hdr)	((hdr)->b_flags & ARC_DONT_L2CACHE)
+#define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_L2CACHE)
 #define	HDR_L2_READING(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS &&	\
 				    (hdr)->b_l2hdr != NULL)
 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_L2_WRITING)
@@ -2300,7 +2299,7 @@
 
 	hdr->b_flags &= ~ARC_L2_EVICTED;
 	if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
-		hdr->b_flags |= ARC_DONT_L2CACHE;
+		hdr->b_flags &= ~ARC_L2CACHE;
 
 	/* byteswap if necessary */
 	callback_list = hdr->b_acb;
@@ -2414,7 +2413,7 @@
  */
 int
 arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_buf_t *pbuf,
-    arc_done_func_t *done, void *private, int priority, int flags,
+    arc_done_func_t *done, void *private, int priority, int zio_flags,
     uint32_t *arc_flags, const zbookmark_t *zb)
 {
 	int err;
@@ -2424,7 +2423,7 @@
 	rw_enter(&pbuf->b_hdr->b_datalock, RW_READER);
 
 	err = arc_read_nolock(pio, spa, bp, done, private, priority,
-	    flags, arc_flags, zb);
+	    zio_flags, arc_flags, zb);
 
 	rw_exit(&pbuf->b_hdr->b_datalock);
 	return (err);
@@ -2432,7 +2431,7 @@
 
 int
 arc_read_nolock(zio_t *pio, spa_t *spa, blkptr_t *bp,
-    arc_done_func_t *done, void *private, int priority, int flags,
+    arc_done_func_t *done, void *private, int priority, int zio_flags,
     uint32_t *arc_flags, const zbookmark_t *zb)
 {
 	arc_buf_hdr_t *hdr;
@@ -2464,7 +2463,7 @@
 				acb->acb_private = private;
 				if (pio != NULL)
 					acb->acb_zio_dummy = zio_null(pio,
-					    spa, NULL, NULL, flags);
+					    spa, NULL, NULL, zio_flags);
 
 				ASSERT(acb->acb_done != NULL);
 				acb->acb_next = hdr->b_acb;
@@ -2501,6 +2500,8 @@
 		}
 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
 		arc_access(hdr, hash_lock);
+		if (*arc_flags & ARC_L2CACHE)
+			hdr->b_flags |= ARC_L2CACHE;
 		mutex_exit(hash_lock);
 		ARCSTAT_BUMP(arcstat_hits);
 		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
@@ -2540,6 +2541,8 @@
 				    private);
 				hdr->b_flags |= ARC_PREFETCH;
 			}
+			if (*arc_flags & ARC_L2CACHE)
+				hdr->b_flags |= ARC_L2CACHE;
 			if (BP_GET_LEVEL(bp) > 0)
 				hdr->b_flags |= ARC_INDIRECT;
 		} else {
@@ -2554,6 +2557,8 @@
 				hdr->b_flags |= ARC_PREFETCH;
 			else
 				add_reference(hdr, hash_lock, private);
+			if (*arc_flags & ARC_L2CACHE)
+				hdr->b_flags |= ARC_L2CACHE;
 			buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
 			buf->b_hdr = hdr;
 			buf->b_data = NULL;
@@ -2601,7 +2606,7 @@
 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
 		    data, metadata, misses);
 
-		if (l2arc_ndev != 0) {
+		if (l2arc_ndev != 0 && HDR_L2CACHE(hdr)) {
 			/*
 			 * Lock out device removal.
 			 */
@@ -2631,14 +2636,14 @@
 				cb->l2rcb_spa = spa;
 				cb->l2rcb_bp = *bp;
 				cb->l2rcb_zb = *zb;
-				cb->l2rcb_flags = flags;
+				cb->l2rcb_flags = zio_flags;
 
 				/*
 				 * l2arc read.
 				 */
 				rzio = zio_read_phys(pio, vd, addr, size,
 				    buf->b_data, ZIO_CHECKSUM_OFF,
-				    l2arc_read_done, cb, priority, flags |
+				    l2arc_read_done, cb, priority, zio_flags |
 				    ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL,
 				    B_FALSE);
 				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
@@ -2667,7 +2672,7 @@
 		}
 
 		rzio = zio_read(pio, spa, bp, buf->b_data, size,
-		    arc_read_done, buf, priority, flags, zb);
+		    arc_read_done, buf, priority, zio_flags, zb);
 
 		if (*arc_flags & ARC_WAIT)
 			return (zio_wait(rzio));
@@ -3109,9 +3114,9 @@
 
 zio_t *
 arc_write(zio_t *pio, spa_t *spa, const writeprops_t *wp,
-    uint64_t txg, blkptr_t *bp, arc_buf_t *buf,
+    boolean_t l2arc, uint64_t txg, blkptr_t *bp, arc_buf_t *buf,
     arc_done_func_t *ready, arc_done_func_t *done, void *private, int priority,
-    int flags, const zbookmark_t *zb)
+    int zio_flags, const zbookmark_t *zb)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 	arc_write_callback_t *callback;
@@ -3121,6 +3126,8 @@
 	ASSERT(!HDR_IO_ERROR(hdr));
 	ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
 	ASSERT(hdr->b_acb == 0);
+	if (l2arc)
+		hdr->b_flags |= ARC_L2CACHE;
 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
 	callback->awcb_ready = ready;
 	callback->awcb_done = done;
@@ -3130,7 +3137,7 @@
 	write_policy(spa, wp, &cksum, &comp, &copies);
 	zio = zio_write(pio, spa, cksum, comp, copies, txg, bp,
 	    buf->b_data, hdr->b_size, arc_write_ready, arc_write_done,
-	    callback, priority, flags, zb);
+	    callback, priority, zio_flags, zb);
 
 	return (zio);
 }
@@ -4101,7 +4108,7 @@
 				continue;
 			}
 
-			if (HDR_IO_IN_PROGRESS(ab) || HDR_DONT_L2CACHE(ab)) {
+			if (HDR_IO_IN_PROGRESS(ab) || !HDR_L2CACHE(ab)) {
 				mutex_exit(hash_lock);
 				continue;
 			}
--- a/usr/src/uts/common/fs/zfs/dbuf.c	Thu Jul 31 10:29:01 2008 -0700
+++ b/usr/src/uts/common/fs/zfs/dbuf.c	Thu Jul 31 10:57:04 2008 -0700
@@ -497,6 +497,9 @@
 	db->db_state = DB_READ;
 	mutex_exit(&db->db_mtx);
 
+	if (DBUF_IS_L2CACHEABLE(db))
+		aflags |= ARC_L2CACHE;
+
 	zb.zb_objset = db->db_objset->os_dsl_dataset ?
 	    db->db_objset->os_dsl_dataset->ds_object : 0;
 	zb.zb_object = db->db.db_object;
@@ -536,7 +539,8 @@
 		rw_enter(&db->db_dnode->dn_struct_rwlock, RW_READER);
 
 	prefetch = db->db_level == 0 && db->db_blkid != DB_BONUS_BLKID &&
-	    (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL;
+	    (flags & DB_RF_NOPREFETCH) == 0 && db->db_dnode != NULL &&
+	    DBUF_IS_CACHEABLE(db);
 
 	mutex_enter(&db->db_mtx);
 	if (db->db_state == DB_CACHED) {
@@ -1746,7 +1750,10 @@
 			dbuf_evict(db);
 		} else {
 			VERIFY(arc_buf_remove_ref(db->db_buf, db) == 0);
-			mutex_exit(&db->db_mtx);
+			if (!DBUF_IS_CACHEABLE(db))
+				dbuf_clear(db);
+			else
+				mutex_exit(&db->db_mtx);
 		}
 	} else {
 		mutex_exit(&db->db_mtx);
@@ -2126,7 +2133,8 @@
 		    os->os_dsl_dataset, db->db_blkptr, zio, tx);
 
 	dr->dr_zio = arc_write(zio, os->os_spa, &wp,
-	    txg, db->db_blkptr, data, dbuf_write_ready, dbuf_write_done, db,
+	    DBUF_IS_L2CACHEABLE(db), txg, db->db_blkptr,
+	    data, dbuf_write_ready, dbuf_write_done, db,
 	    ZIO_PRIORITY_ASYNC_WRITE, zio_flags, &zb);
 }
 
--- a/usr/src/uts/common/fs/zfs/dmu.c	Thu Jul 31 10:29:01 2008 -0700
+++ b/usr/src/uts/common/fs/zfs/dmu.c	Thu Jul 31 10:57:04 2008 -0700
@@ -969,8 +969,8 @@
 	wp.wp_dncompress = db->db_dnode->dn_compress;
 	wp.wp_oscompress = os->os_compress;
 	zio = arc_write(pio, os->os_spa, &wp,
-	    txg, bp, dr->dt.dl.dr_data, NULL, dmu_sync_done, in,
-	    ZIO_PRIORITY_SYNC_WRITE, zio_flags, &zb);
+	    DBUF_IS_L2CACHEABLE(db), txg, bp, dr->dt.dl.dr_data, NULL,
+	    dmu_sync_done, in, ZIO_PRIORITY_SYNC_WRITE, zio_flags, &zb);
 
 	if (pio) {
 		zio_nowait(zio);
--- a/usr/src/uts/common/fs/zfs/dmu_objset.c	Thu Jul 31 10:29:01 2008 -0700
+++ b/usr/src/uts/common/fs/zfs/dmu_objset.c	Thu Jul 31 10:57:04 2008 -0700
@@ -133,6 +133,34 @@
 	osi->os_copies = newval;
 }
 
+static void
+primary_cache_changed_cb(void *arg, uint64_t newval)
+{
+	objset_impl_t *osi = arg;
+
+	/*
+	 * Inheritance and range checking should have been done by now.
+	 */
+	ASSERT(newval == ZFS_CACHE_ALL || newval == ZFS_CACHE_NONE ||
+	    newval == ZFS_CACHE_METADATA);
+
+	osi->os_primary_cache = newval;
+}
+
+static void
+secondary_cache_changed_cb(void *arg, uint64_t newval)
+{
+	objset_impl_t *osi = arg;
+
+	/*
+	 * Inheritance and range checking should have been done by now.
+	 */
+	ASSERT(newval == ZFS_CACHE_ALL || newval == ZFS_CACHE_NONE ||
+	    newval == ZFS_CACHE_METADATA);
+
+	osi->os_secondary_cache = newval;
+}
+
 void
 dmu_objset_byteswap(void *buf, size_t size)
 {
@@ -165,6 +193,8 @@
 		zb.zb_object = 0;
 		zb.zb_level = -1;
 		zb.zb_blkid = 0;
+		if (DMU_OS_IS_L2CACHEABLE(osi))
+			aflags |= ARC_L2CACHE;
 
 		dprintf_bp(osi->os_rootbp, "reading %s", "");
 		/*
@@ -190,18 +220,26 @@
 	/*
 	 * Note: the changed_cb will be called once before the register
 	 * func returns, thus changing the checksum/compression from the
-	 * default (fletcher2/off).  Snapshots don't need to know, and
-	 * registering would complicate clone promotion.
+	 * default (fletcher2/off).  Snapshots don't need to know about
+	 * checksum/compression/copies.
 	 */
-	if (ds && ds->ds_phys->ds_num_children == 0) {
-		err = dsl_prop_register(ds, "checksum",
-		    checksum_changed_cb, osi);
+	if (ds) {
+		err = dsl_prop_register(ds, "primarycache",
+		    primary_cache_changed_cb, osi);
 		if (err == 0)
-			err = dsl_prop_register(ds, "compression",
-			    compression_changed_cb, osi);
-		if (err == 0)
-			err = dsl_prop_register(ds, "copies",
-			    copies_changed_cb, osi);
+			err = dsl_prop_register(ds, "secondarycache",
+			    secondary_cache_changed_cb, osi);
+		if (!dsl_dataset_is_snapshot(ds)) {
+			if (err == 0)
+				err = dsl_prop_register(ds, "checksum",
+				    checksum_changed_cb, osi);
+			if (err == 0)
+				err = dsl_prop_register(ds, "compression",
+				    compression_changed_cb, osi);
+			if (err == 0)
+				err = dsl_prop_register(ds, "copies",
+				    copies_changed_cb, osi);
+		}
 		if (err) {
 			VERIFY(arc_buf_remove_ref(osi->os_phys_buf,
 			    &osi->os_phys_buf) == 1);
@@ -213,6 +251,8 @@
 		osi->os_checksum = ZIO_CHECKSUM_FLETCHER_4;
 		osi->os_compress = ZIO_COMPRESS_LZJB;
 		osi->os_copies = spa_max_replication(spa);
+		osi->os_primary_cache = ZFS_CACHE_ALL;
+		osi->os_secondary_cache = ZFS_CACHE_ALL;
 	}
 
 	osi->os_zil_header = osi->os_phys->os_zil_header;
@@ -393,13 +433,19 @@
 		ASSERT(list_head(&osi->os_free_dnodes[i]) == NULL);
 	}
 
-	if (ds && ds->ds_phys && ds->ds_phys->ds_num_children == 0) {
-		VERIFY(0 == dsl_prop_unregister(ds, "checksum",
-		    checksum_changed_cb, osi));
-		VERIFY(0 == dsl_prop_unregister(ds, "compression",
-		    compression_changed_cb, osi));
-		VERIFY(0 == dsl_prop_unregister(ds, "copies",
-		    copies_changed_cb, osi));
+	if (ds) {
+		if (!dsl_dataset_is_snapshot(ds)) {
+			VERIFY(0 == dsl_prop_unregister(ds, "checksum",
+			    checksum_changed_cb, osi));
+			VERIFY(0 == dsl_prop_unregister(ds, "compression",
+			    compression_changed_cb, osi));
+			VERIFY(0 == dsl_prop_unregister(ds, "copies",
+			    copies_changed_cb, osi));
+		}
+		VERIFY(0 == dsl_prop_unregister(ds, "primarycache",
+		    primary_cache_changed_cb, osi));
+		VERIFY(0 == dsl_prop_unregister(ds, "secondarycache",
+		    secondary_cache_changed_cb, osi));
 	}
 
 	/*
@@ -867,9 +913,9 @@
 	wp.wp_oscompress = os->os_compress;
 	arc_release(os->os_phys_buf, &os->os_phys_buf);
 	zio = arc_write(pio, os->os_spa, &wp,
-	    tx->tx_txg, os->os_rootbp, os->os_phys_buf, ready, NULL, os,
-	    ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_METADATA,
-	    &zb);
+	    DMU_OS_IS_L2CACHEABLE(os), tx->tx_txg, os->os_rootbp,
+	    os->os_phys_buf, ready, NULL, os, ZIO_PRIORITY_ASYNC_WRITE,
+	    ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_METADATA, &zb);
 
 	/*
 	 * Sync meta-dnode - the parent IO for the sync is the root block
--- a/usr/src/uts/common/fs/zfs/dsl_dataset.c	Thu Jul 31 10:29:01 2008 -0700
+++ b/usr/src/uts/common/fs/zfs/dsl_dataset.c	Thu Jul 31 10:57:04 2008 -0700
@@ -836,6 +836,10 @@
 	*cp = '\0';
 	if (err == 0) {
 		dsl_dataset_make_exclusive(ds, da->dstg);
+		if (ds->ds_user_ptr) {
+			ds->ds_user_evict_func(ds, ds->ds_user_ptr);
+			ds->ds_user_ptr = NULL;
+		}
 		dsl_sync_task_create(da->dstg, dsl_dataset_destroy_check,
 		    dsl_dataset_destroy_sync, ds, da->dstg, 0);
 	} else if (err == ENOENT) {
@@ -905,6 +909,11 @@
 	if (dsl_dataset_is_snapshot(ds)) {
 		/* Destroying a snapshot is simpler */
 		dsl_dataset_make_exclusive(ds, tag);
+
+		if (ds->ds_user_ptr) {
+			ds->ds_user_evict_func(ds, ds->ds_user_ptr);
+			ds->ds_user_ptr = NULL;
+		}
 		err = dsl_sync_task_do(ds->ds_dir->dd_pool,
 		    dsl_dataset_destroy_check, dsl_dataset_destroy_sync,
 		    ds, tag, 0);
@@ -2376,6 +2385,11 @@
 	do {
 		dsl_dataset_t *ds = snap->ds;
 
+		/* unregister props as dsl_dir is changing */
+		if (ds->ds_user_ptr) {
+			ds->ds_user_evict_func(ds, ds->ds_user_ptr);
+			ds->ds_user_ptr = NULL;
+		}
 		/* move snap name entry */
 		dsl_dataset_name(ds, name);
 		VERIFY(0 == dsl_dataset_snap_remove(pa->old_head,
@@ -2383,7 +2397,6 @@
 		VERIFY(0 == zap_add(dp->dp_meta_objset,
 		    hds->ds_phys->ds_snapnames_zapobj, ds->ds_snapname,
 		    8, 1, &ds->ds_object, tx));
-
 		/* change containing dsl_dir */
 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
 		ASSERT3U(ds->ds_phys->ds_dir_obj, ==, odd->dd_object);
--- a/usr/src/uts/common/fs/zfs/sys/arc.h	Thu Jul 31 10:29:01 2008 -0700
+++ b/usr/src/uts/common/fs/zfs/sys/arc.h	Thu Jul 31 10:57:04 2008 -0700
@@ -67,6 +67,7 @@
 #define	ARC_NOWAIT	(1 << 2)	/* perform I/O asynchronously */
 #define	ARC_PREFETCH	(1 << 3)	/* I/O is a prefetch */
 #define	ARC_CACHED	(1 << 4)	/* I/O was already in cache */
+#define	ARC_L2CACHE	(1 << 5)	/* cache in L2ARC */
 
 void arc_space_consume(uint64_t space);
 void arc_space_return(uint64_t space);
@@ -95,15 +96,15 @@
 } writeprops_t;
 
 int arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_buf_t *pbuf,
-    arc_done_func_t *done, void *private, int priority, int flags,
+    arc_done_func_t *done, void *private, int priority, int zio_flags,
     uint32_t *arc_flags, const zbookmark_t *zb);
 int arc_read_nolock(zio_t *pio, spa_t *spa, blkptr_t *bp,
     arc_done_func_t *done, void *private, int priority, int flags,
     uint32_t *arc_flags, const zbookmark_t *zb);
 zio_t *arc_write(zio_t *pio, spa_t *spa, const writeprops_t *wp,
-    uint64_t txg, blkptr_t *bp, arc_buf_t *buf,
+    boolean_t l2arc, uint64_t txg, blkptr_t *bp, arc_buf_t *buf,
     arc_done_func_t *ready, arc_done_func_t *done, void *private, int priority,
-    int flags, const zbookmark_t *zb);
+    int zio_flags, const zbookmark_t *zb);
 int arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
     zio_done_func_t *done, void *private, uint32_t arc_flags);
 int arc_tryread(spa_t *spa, blkptr_t *bp, void *data);
--- a/usr/src/uts/common/fs/zfs/sys/dbuf.h	Thu Jul 31 10:29:01 2008 -0700
+++ b/usr/src/uts/common/fs/zfs/sys/dbuf.h	Thu Jul 31 10:57:04 2008 -0700
@@ -279,10 +279,21 @@
 void dbuf_init(void);
 void dbuf_fini(void);
 
-#define	DBUF_GET_BUFC_TYPE(db)					\
-	((((db)->db_level > 0) ||				\
-	    (dmu_ot[(db)->db_dnode->dn_type].ot_metadata)) ?	\
-	    ARC_BUFC_METADATA : ARC_BUFC_DATA);
+#define	DBUF_IS_METADATA(db)	\
+	((db)->db_level > 0 || dmu_ot[(db)->db_dnode->dn_type].ot_metadata)
+
+#define	DBUF_GET_BUFC_TYPE(db)	\
+	(DBUF_IS_METADATA(db) ? ARC_BUFC_METADATA : ARC_BUFC_DATA)
+
+#define	DBUF_IS_CACHEABLE(db)						\
+	((db)->db_objset->os_primary_cache == ZFS_CACHE_ALL ||		\
+	(DBUF_IS_METADATA(db) &&					\
+	((db)->db_objset->os_primary_cache == ZFS_CACHE_METADATA)))
+
+#define	DBUF_IS_L2CACHEABLE(db)						\
+	((db)->db_objset->os_secondary_cache == ZFS_CACHE_ALL ||	\
+	(DBUF_IS_METADATA(db) &&					\
+	((db)->db_objset->os_secondary_cache == ZFS_CACHE_METADATA)))
 
 #ifdef ZFS_DEBUG
 
--- a/usr/src/uts/common/fs/zfs/sys/dmu_objset.h	Thu Jul 31 10:29:01 2008 -0700
+++ b/usr/src/uts/common/fs/zfs/sys/dmu_objset.h	Thu Jul 31 10:57:04 2008 -0700
@@ -69,6 +69,8 @@
 	uint8_t os_checksum;	/* can change, under dsl_dir's locks */
 	uint8_t os_compress;	/* can change, under dsl_dir's locks */
 	uint8_t os_copies;	/* can change, under dsl_dir's locks */
+	uint8_t os_primary_cache;	/* can change, under dsl_dir's locks */
+	uint8_t os_secondary_cache;	/* can change, under dsl_dir's locks */
 
 	/* no lock needed: */
 	struct dmu_tx *os_synctx; /* XXX sketchy */
@@ -93,6 +95,10 @@
 
 #define	DMU_META_DNODE_OBJECT	0
 
+#define	DMU_OS_IS_L2CACHEABLE(os)				\
+	((os)->os_secondary_cache == ZFS_CACHE_ALL ||		\
+	(os)->os_secondary_cache == ZFS_CACHE_METADATA)
+
 /* called from zpl */
 int dmu_objset_open(const char *name, dmu_objset_type_t type, int mode,
     objset_t **osp);
--- a/usr/src/uts/common/sys/fs/zfs.h	Thu Jul 31 10:29:01 2008 -0700
+++ b/usr/src/uts/common/sys/fs/zfs.h	Thu Jul 31 10:57:04 2008 -0700
@@ -101,6 +101,8 @@
 	ZFS_PROP_REFQUOTA,
 	ZFS_PROP_REFRESERVATION,
 	ZFS_PROP_GUID,
+	ZFS_PROP_PRIMARYCACHE,
+	ZFS_PROP_SECONDARYCACHE,
 	ZFS_NUM_PROPS
 } zfs_prop_t;
 
@@ -221,6 +223,13 @@
 	ZFS_UNSHARE_SMB = 3
 } zfs_share_op_t;
 
+typedef enum zfs_cache_type {
+	ZFS_CACHE_NONE = 0,
+	ZFS_CACHE_METADATA = 1,
+	ZFS_CACHE_ALL = 2
+} zfs_cache_type_t;
+
+
 /*
  * On-disk version number.
  */