upstream/illumos/illumos-gate: comparison usr/src/uts/common/fs/zfs/arc.c

equal deleted inserted replaced

-:d4292813278d
+:361307ae060d
 * These tunables are for performance analysis.
 */
 uint64_t zfs_arc_max;
 uint64_t zfs_arc_min;
 uint64_t zfs_arc_meta_limit = 0;
+int zfs_mdcomp_disable = 0;
 /*
 * Note that buffers can be in one of 6 states:
 *	ARC_anon	- anonymous (discussed below)
 *	ARC_mru		- recently used, currently cached
 typedef struct arc_callback arc_callback_t;
 struct arc_callback {
 	void			*acb_private;
 	arc_done_func_t		*acb_done;
-	arc_byteswap_func_t	*acb_byteswap;
 	arc_buf_t		*acb_buf;
 	zio_t			*acb_zio_dummy;
 	arc_callback_t		*acb_next;
 };
 	/* self protecting */
 	refcount_t		b_refcnt;
 	l2arc_buf_hdr_t		*b_l2hdr;
 	list_node_t		b_l2node;
+	/*
+	 * scrub code can lockout access to the buf while it changes
+	 * bp's contained within it.
+	 */
+	krwlock_t		b_datalock;
 };
 static arc_buf_t *arc_eviction_list;
 static kmutex_t arc_eviction_mtx;
 static arc_buf_hdr_t arc_eviction_hdr;
 #define	ARC_FREE_IN_PROGRESS	(1 << 15)	/* hdr about to be freed */
 #define	ARC_DONT_L2CACHE	(1 << 16)	/* originated by prefetch */
 #define	ARC_L2_WRITING		(1 << 17)	/* L2ARC write in progress */
 #define	ARC_L2_EVICTED		(1 << 18)	/* evicted during I/O */
 #define	ARC_L2_WRITE_HEAD	(1 << 19)	/* head of write list */
+#define	ARC_STORED		(1 << 20)	/* has been store()d to */
 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_IN_HASH_TABLE)
 #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS)
 #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_IO_ERROR)
 #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FREED_IN_READ)
 static void l2arc_read_done(zio_t *zio);
 static void l2arc_hdr_stat_add(void);
 static void l2arc_hdr_stat_remove(void);
 static uint64_t
-buf_hash(spa_t *spa, dva_t *dva, uint64_t birth)
+buf_hash(spa_t *spa, const dva_t *dva, uint64_t birth)
 {
 	uintptr_t spav = (uintptr_t)spa;
 	uint8_t *vdva = (uint8_t *)dva;
 	uint64_t crc = -1ULL;
 	int i;
 	((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
 	((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
 	((buf)->b_birth == birth) && ((buf)->b_spa == spa)
 static arc_buf_hdr_t *
-buf_hash_find(spa_t *spa, dva_t *dva, uint64_t birth, kmutex_t **lockp)
+buf_hash_find(spa_t *spa, const dva_t *dva, uint64_t birth, kmutex_t **lockp)
 {
 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
 	arc_buf_hdr_t *buf;
 	bzero(buf, sizeof (arc_buf_hdr_t));
 	refcount_create(&buf->b_refcnt);
 	cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL);
 	mutex_init(&buf->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
+	rw_init(&buf->b_datalock, NULL, RW_DEFAULT, NULL);
 	ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE);
 	return (0);
 }
 	arc_buf_hdr_t *buf = vbuf;
 	refcount_destroy(&buf->b_refcnt);
 	cv_destroy(&buf->b_cv);
 	mutex_destroy(&buf->b_freeze_lock);
+	rw_destroy(&buf->b_datalock);
 	ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE);
 }
 /*
 		}
 		ASSERT(delta > 0);
 		ASSERT3U(*size, >=, delta);
 		atomic_add_64(size, -delta);
 		mutex_exit(&ab->b_state->arcs_mtx);
-		/* remove the prefetch flag is we get a reference */
+		/* remove the prefetch flag if we get a reference */
 		if (ab->b_flags & ARC_PREFETCH)
 			ab->b_flags &= ~ARC_PREFETCH;
 	}
 }
 arc_hdr_destroy(arc_buf_hdr_t *hdr)
 {
 	ASSERT(refcount_is_zero(&hdr->b_refcnt));
 	ASSERT3P(hdr->b_state, ==, arc_anon);
 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
+	ASSERT(!(hdr->b_flags & ARC_STORED));
 	if (hdr->b_l2hdr != NULL) {
 		if (!MUTEX_HELD(&l2arc_buflist_mtx)) {
 			/*
 			 * To prevent arc_free() and l2arc_evict() from
 		hdr->b_flags |= ARC_DONT_L2CACHE;
 	/* byteswap if necessary */
 	callback_list = hdr->b_acb;
 	ASSERT(callback_list != NULL);
-	if (BP_SHOULD_BYTESWAP(zio->io_bp) && callback_list->acb_byteswap)
+	if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
-		callback_list->acb_byteswap(buf->b_data, hdr->b_size);
+		arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
+		    byteswap_uint64_array :
+		    dmu_ot[BP_GET_TYPE(zio->io_bp)].ot_byteswap;
+		func(buf->b_data, hdr->b_size);
+	}
 	arc_cksum_compute(buf, B_FALSE);
 	/* create copies of the data buffer for the callers */
 	abuf = buf;
 * to the read to invoke the "done" func when the read completes,
 * and return; or just return.
 *
 * arc_read_done() will invoke all the requested "done" functions
 * for readers of this block.
+*
+* Normal callers should use arc_read and pass the arc buffer and offset
+* for the bp.  But if you know you don't need locking, you can use
+* arc_read_bp.
 */
 int
-arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_byteswap_func_t *swap,
+arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_buf_t *pbuf,
 arc_done_func_t *done, void *private, int priority, int flags,
-uint32_t *arc_flags, zbookmark_t *zb)
+uint32_t *arc_flags, const zbookmark_t *zb)
+{
+	int err;
+	ASSERT(!refcount_is_zero(&pbuf->b_hdr->b_refcnt));
+	ASSERT3U((char *)bp - (char *)pbuf->b_data, <, pbuf->b_hdr->b_size);
+	rw_enter(&pbuf->b_hdr->b_datalock, RW_READER);
+	err = arc_read_nolock(pio, spa, bp, done, private, priority,
+	    flags, arc_flags, zb);
+	rw_exit(&pbuf->b_hdr->b_datalock);
+	return (err);
+}
+int
+arc_read_nolock(zio_t *pio, spa_t *spa, blkptr_t *bp,
+arc_done_func_t *done, void *private, int priority, int flags,
+uint32_t *arc_flags, const zbookmark_t *zb)
 {
 	arc_buf_hdr_t *hdr;
 	arc_buf_t *buf;
 	kmutex_t *hash_lock;
 	zio_t *rzio;
 				acb = kmem_zalloc(sizeof (arc_callback_t),
 				    KM_SLEEP);
 				acb->acb_done = done;
 				acb->acb_private = private;
-				acb->acb_byteswap = swap;
 				if (pio != NULL)
 					acb->acb_zio_dummy = zio_null(pio,
 					    spa, NULL, NULL, flags);
 				ASSERT(acb->acb_done != NULL);
 		}
 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
 		acb->acb_done = done;
 		acb->acb_private = private;
-		acb->acb_byteswap = swap;
 		ASSERT(hdr->b_acb == NULL);
 		hdr->b_acb = acb;
 		hdr->b_flags |= ARC_IO_IN_PROGRESS;
 /*
 * Release this buffer from the cache.  This must be done
 * after a read and prior to modifying the buffer contents.
 * If the buffer has more than one reference, we must make
-* make a new hdr for the buffer.
+* a new hdr for the buffer.
 */
 void
 arc_release(arc_buf_t *buf, void *tag)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 	l2arc_buf_hdr_t *l2hdr = NULL;
 	uint64_t buf_size;
 	/* this buffer is not on any list */
 	ASSERT(refcount_count(&hdr->b_refcnt) > 0);
+	ASSERT(!(hdr->b_flags & ARC_STORED));
 	if (hdr->b_state == arc_anon) {
 		/* this buffer is already released */
 		ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1);
 		ASSERT(BUF_EMPTY(hdr));
 	arc_buf_t *buf = callback->awcb_buf;
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 	hdr->b_acb = NULL;
-	/* this buffer is on no lists and is not in the hash table */
-	ASSERT3P(hdr->b_state, ==, arc_anon);
 	hdr->b_dva = *BP_IDENTITY(zio->io_bp);
 	hdr->b_birth = zio->io_bp->blk_birth;
 	hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
 	/*
 	 * If the block to be written was all-zero, we may have
 			arc_hdr_destroy(exists);
 			exists = buf_hash_insert(hdr, &hash_lock);
 			ASSERT3P(exists, ==, NULL);
 		}
 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
-		arc_access(hdr, hash_lock);
+		/* if it's not anon, we are doing a scrub */
+		if (hdr->b_state == arc_anon)
+			arc_access(hdr, hash_lock);
 		mutex_exit(hash_lock);
 	} else if (callback->awcb_done == NULL) {
 		int destroy_hdr;
 		/*
 		 * This is an anonymous buffer with no user callback,
 		if (destroy_hdr)
 			arc_hdr_destroy(hdr);
 	} else {
 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
 	}
+	hdr->b_flags &= ~ARC_STORED;
 	if (callback->awcb_done) {
 		ASSERT(!refcount_is_zero(&hdr->b_refcnt));
 		callback->awcb_done(zio, buf, callback->awcb_private);
 	}
 	kmem_free(callback, sizeof (arc_write_callback_t));
 }
+static void
+write_policy(spa_t *spa, const writeprops_t *wp,
+int *cksump, int *compp, int *copiesp)
+{
+	int copies = wp->wp_copies;
+	boolean_t ismd = (wp->wp_level > 0 || dmu_ot[wp->wp_type].ot_metadata);
+	/* Determine copies setting */
+	if (ismd)
+		copies++;
+	*copiesp = MIN(copies, spa_max_replication(spa));
+	/* Determine checksum setting */
+	if (ismd) {
+		/*
+		 * Metadata always gets checksummed.  If the data
+		 * checksum is multi-bit correctable, and it's not a
+		 * ZBT-style checksum, then it's suitable for metadata
+		 * as well.  Otherwise, the metadata checksum defaults
+		 * to fletcher4.
+		 */
+		if (zio_checksum_table[wp->wp_oschecksum].ci_correctable &&
+		    !zio_checksum_table[wp->wp_oschecksum].ci_zbt)
+			*cksump = wp->wp_oschecksum;
+		else
+			*cksump = ZIO_CHECKSUM_FLETCHER_4;
+	} else {
+		*cksump = zio_checksum_select(wp->wp_dnchecksum,
+		    wp->wp_oschecksum);
+	}
+	/* Determine compression setting */
+	if (ismd) {
+		/*
+		 * XXX -- we should design a compression algorithm
+		 * that specializes in arrays of bps.
+		 */
+		*compp = zfs_mdcomp_disable ? ZIO_COMPRESS_EMPTY :
+		    ZIO_COMPRESS_LZJB;
+	} else {
+		*compp = zio_compress_select(wp->wp_dncompress,
+		    wp->wp_oscompress);
+	}
+}
 zio_t *
-arc_write(zio_t *pio, spa_t *spa, int checksum, int compress, int ncopies,
+arc_write(zio_t *pio, spa_t *spa, const writeprops_t *wp,
 uint64_t txg, blkptr_t *bp, arc_buf_t *buf,
 arc_done_func_t *ready, arc_done_func_t *done, void *private, int priority,
-int flags, zbookmark_t *zb)
+int flags, const zbookmark_t *zb)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 	arc_write_callback_t *callback;
 	zio_t	*zio;
+	int cksum, comp, copies;
-	/* this is a private buffer - no locking required */
-	ASSERT3P(hdr->b_state, ==, arc_anon);
-	ASSERT(BUF_EMPTY(hdr));
 	ASSERT(!HDR_IO_ERROR(hdr));
 	ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
 	ASSERT(hdr->b_acb == 0);
 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
 	callback->awcb_ready = ready;
 	callback->awcb_done = done;
 	callback->awcb_private = private;
 	callback->awcb_buf = buf;
-	zio = zio_write(pio, spa, checksum, compress, ncopies, txg, bp,
-	    buf->b_data, hdr->b_size, arc_write_ready, arc_write_done, callback,
+	write_policy(spa, wp, &cksum, &comp, &copies);
-	    priority, flags, zb);
+	zio = zio_write(pio, spa, cksum, comp, copies, txg, bp,
+	    buf->b_data, hdr->b_size, arc_write_ready, arc_write_done,
+	    callback, priority, flags, zb);
 	return (zio);
 }
 int

changeset 7046	361307ae060d
parent 6987	877c018eb06c
child 7237	f47d41541b14