upstream/illumos/illumos-gate: comparison usr/src/uts/common/fs/zfs/arc.c

equal deleted inserted replaced

-:1e7638a44ce6
+:877c018eb06c
 * (initialized in arc_init())
 */
 static int		arc_min_prefetch_lifespan;
 static int arc_dead;
+/*
+* The arc has filled available memory and has now warmed up.
+*/
+static boolean_t arc_warm;
 /*
 * These tunables are for performance analysis.
 */
 uint64_t zfs_arc_max;
 #define	ARC_FREED_IN_READ	(1 << 12)	/* buf freed while in read */
 #define	ARC_BUF_AVAILABLE	(1 << 13)	/* block not in active use */
 #define	ARC_INDIRECT		(1 << 14)	/* this is an indirect block */
 #define	ARC_FREE_IN_PROGRESS	(1 << 15)	/* hdr about to be freed */
 #define	ARC_DONT_L2CACHE	(1 << 16)	/* originated by prefetch */
-#define	ARC_L2_READING		(1 << 17)	/* L2ARC read in progress */
+#define	ARC_L2_WRITING		(1 << 17)	/* L2ARC write in progress */
-#define	ARC_L2_WRITING		(1 << 18)	/* L2ARC write in progress */
+#define	ARC_L2_EVICTED		(1 << 18)	/* evicted during I/O */
-#define	ARC_L2_EVICTED		(1 << 19)	/* evicted during I/O */
+#define	ARC_L2_WRITE_HEAD	(1 << 19)	/* head of write list */
-#define	ARC_L2_WRITE_HEAD	(1 << 20)	/* head of write list */
 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_IN_HASH_TABLE)
 #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS)
 #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_IO_ERROR)
 #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FREED_IN_READ)
 #define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_BUF_AVAILABLE)
 #define	HDR_FREE_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FREE_IN_PROGRESS)
 #define	HDR_DONT_L2CACHE(hdr)	((hdr)->b_flags & ARC_DONT_L2CACHE)
-#define	HDR_L2_READING(hdr)	((hdr)->b_flags & ARC_L2_READING)
+#define	HDR_L2_READING(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS &&	\
+				    (hdr)->b_l2hdr != NULL)
 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_L2_WRITING)
 #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_L2_EVICTED)
 #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_L2_WRITE_HEAD)
 /*
 * Level 2 ARC
 */
 #define	L2ARC_WRITE_SIZE	(8 * 1024 * 1024)	/* initial write max */
 #define	L2ARC_HEADROOM		4		/* num of writes */
-#define	L2ARC_FEED_DELAY	180		/* starting grace */
 #define	L2ARC_FEED_SECS		1		/* caching interval */
 #define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
 #define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
 /*
 * L2ARC Performance Tunables
 */
 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* default max write size */
+uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra write during warmup */
 uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* number of dev writes */
 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
 boolean_t l2arc_noprefetch = B_TRUE;		/* don't cache prefetch bufs */
 /*
 typedef struct l2arc_dev {
 	vdev_t			*l2ad_vdev;	/* vdev */
 	spa_t			*l2ad_spa;	/* spa */
 	uint64_t		l2ad_hand;	/* next write location */
 	uint64_t		l2ad_write;	/* desired write size, bytes */
+	uint64_t		l2ad_boost;	/* warmup write boost, bytes */
 	uint64_t		l2ad_start;	/* first addr on device */
 	uint64_t		l2ad_end;	/* last addr on device */
 	uint64_t		l2ad_evict;	/* last addr eviction reached */
 	boolean_t		l2ad_first;	/* first sweep through */
 	list_t			*l2ad_buflist;	/* buffer list */
 			/* reset the growth delay for every reclaim */
 			growtime = lbolt + (arc_grow_retry * hz);
 			arc_kmem_reap_now(last_reclaim);
+			arc_warm = B_TRUE;
 		} else if (arc_no_grow && lbolt >= growtime) {
 			arc_no_grow = FALSE;
 		}
 	ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
 	    (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
 	    (found == hdr && HDR_L2_READING(hdr)));
-	hdr->b_flags &= ~(ARC_L2_READING|ARC_L2_EVICTED);
+	hdr->b_flags &= ~ARC_L2_EVICTED;
 	if (l2arc_noprefetch && (hdr->b_flags & ARC_PREFETCH))
 		hdr->b_flags |= ARC_DONT_L2CACHE;
 	/* byteswap if necessary */
 	callback_list = hdr->b_acb;
 		if (done)
 			done(NULL, buf, private);
 	} else {
 		uint64_t size = BP_GET_LSIZE(bp);
 		arc_callback_t	*acb;
+		vdev_t *vd = NULL;
+		daddr_t addr;
 		if (hdr == NULL) {
 			/* this block is not in the cache */
 			arc_buf_hdr_t	*exists;
 			arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
 		 */
 		if (GHOST_STATE(hdr->b_state))
 			arc_access(hdr, hash_lock);
+		if (hdr->b_l2hdr != NULL) {
+			vd = hdr->b_l2hdr->b_dev->l2ad_vdev;
+			addr = hdr->b_l2hdr->b_daddr;
+		}
+		mutex_exit(hash_lock);
 		ASSERT3U(hdr->b_size, ==, size);
 		DTRACE_PROBE3(arc__miss, blkptr_t *, bp, uint64_t, size,
 		    zbookmark_t *, zb);
 		ARCSTAT_BUMP(arcstat_misses);
 		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
 		    data, metadata, misses);
 		if (l2arc_ndev != 0) {
 			/*
+			 * Lock out device removal.
+			 */
+			spa_config_enter(spa, RW_READER, FTAG);
+			/*
 			 * Read from the L2ARC if the following are true:
-			 * 1. This buffer has L2ARC metadata.
+			 * 1. The L2ARC vdev was previously cached.
-			 * 2. This buffer isn't currently writing to the L2ARC.
+			 * 2. This buffer still has L2ARC metadata.
+			 * 3. This buffer isn't currently writing to the L2ARC.
+			 * 4. The L2ARC entry wasn't evicted, which may
+			 *    also have invalidated the vdev.
 			 */
-			if (hdr->b_l2hdr != NULL && !HDR_L2_WRITING(hdr)) {
+			if (vd != NULL && hdr->b_l2hdr != NULL &&
-				vdev_t *vd = hdr->b_l2hdr->b_dev->l2ad_vdev;
+			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr)) {
-				daddr_t addr = hdr->b_l2hdr->b_daddr;
 				l2arc_read_callback_t *cb;
 				if (vdev_is_dead(vd))
-					goto skip_l2arc;
+					goto l2skip;
-				hdr->b_flags |= ARC_L2_READING;
-				mutex_exit(hash_lock);
 				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
 				ARCSTAT_BUMP(arcstat_l2_hits);
 				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
 				/*
 				 * l2arc read.
 				 */
 				rzio = zio_read_phys(pio, vd, addr, size,
 				    buf->b_data, ZIO_CHECKSUM_OFF,
-				    l2arc_read_done, cb, priority,
+				    l2arc_read_done, cb, priority, flags |
-				    flags | ZIO_FLAG_DONT_CACHE, B_FALSE);
+				    ZIO_FLAG_DONT_CACHE | ZIO_FLAG_CANFAIL,
+				    B_FALSE);
 				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
 				    zio_t *, rzio);
+				spa_config_exit(spa, FTAG);
-				if (*arc_flags & ARC_WAIT)
-					return (zio_wait(rzio));
+				if (*arc_flags & ARC_NOWAIT) {
+					zio_nowait(rzio);
-				ASSERT(*arc_flags & ARC_NOWAIT);
+					return (0);
-				zio_nowait(rzio);
+				}
-				return (0);
+				ASSERT(*arc_flags & ARC_WAIT);
+				if (zio_wait(rzio) == 0)
+					return (0);
+				/* l2arc read error; goto zio_read() */
 			} else {
 				DTRACE_PROBE1(l2arc__miss,
 				    arc_buf_hdr_t *, hdr);
 				ARCSTAT_BUMP(arcstat_l2_misses);
 				if (HDR_L2_WRITING(hdr))
 					ARCSTAT_BUMP(arcstat_l2_rw_clash);
+l2skip:
+				spa_config_exit(spa, FTAG);
 			}
 		}
-skip_l2arc:
-		mutex_exit(hash_lock);
 		rzio = zio_read(pio, spa, bp, buf->b_data, size,
 		    arc_read_done, buf, priority, flags, zb);
 		if (*arc_flags & ARC_WAIT)
 	(void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
 	    TS_RUN, minclsyspri);
 	arc_dead = FALSE;
+	arc_warm = B_FALSE;
 	if (zfs_write_limit_max == 0)
 		zfs_write_limit_max = physmem * PAGESIZE >>
 		    zfs_write_limit_shift;
 	else
 * pressure valve to prevent heavy read workloads from both stalling the ARC
 * with waits and clogging the L2ARC with writes.  This also helps prevent
 * the potential for the L2ARC to churn if it attempts to cache content too
 * quickly, such as during backups of the entire pool.
 *
-* 5. Writes to the L2ARC devices are grouped and sent in-sequence, so that
+* 5. After system boot and before the ARC has filled main memory, there are
+* no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
+* lists can remain mostly static.  Instead of searching from tail of these
+* lists as pictured, the l2arc_feed_thread() will search from the list heads
+* for eligible buffers, greatly increasing its chance of finding them.
+*
+* The L2ARC device write speed is also boosted during this time so that
+* the L2ARC warms up faster.  Since there have been no ARC evictions yet,
+* there are no L2ARC reads, and no fear of degrading read performance
+* through increased writes.
+*
+* 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
 * the vdev queue can aggregate them into larger and fewer writes.  Each
 * device is written to in a rotor fashion, sweeping writes through
 * available space then repeating.
 *
-* 6. The L2ARC does not store dirty content.  It never needs to flush
+* 7. The L2ARC does not store dirty content.  It never needs to flush
 * write buffers back to disk based storage.
 *
-* 7. If an ARC buffer is written (and dirtied) which also exists in the
+* 8. If an ARC buffer is written (and dirtied) which also exists in the
 * L2ARC, the now stale L2ARC buffer is immediately dropped.
 *
 * The performance of the L2ARC can be tweaked by a number of tunables, which
 * may be necessary for different workloads:
 *
 *	l2arc_write_max		max write bytes per interval
+*	l2arc_write_boost	extra write bytes during device warmup
 *	l2arc_noprefetch	skip caching prefetched buffers
 *	l2arc_headroom		number of max device writes to precache
 *	l2arc_feed_secs		seconds between L2ARC writing
 *
 * Tunables may be removed or added as future performance improvements are
 	ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE);
 }
 /*
 * Cycle through L2ARC devices.  This is how L2ARC load balances.
-* This is called with l2arc_dev_mtx held, which also locks out spa removal.
+* If a device is returned, this also returns holding the spa config lock.
 */
 static l2arc_dev_t *
 l2arc_dev_get_next(void)
 {
-	l2arc_dev_t *next, *first;
+	l2arc_dev_t *first, *next = NULL;
+	/*
+	 * Lock out the removal of spas (spa_namespace_lock), then removal
+	 * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
+	 * both locks will be dropped and a spa config lock held instead.
+	 */
+	mutex_enter(&spa_namespace_lock);
+	mutex_enter(&l2arc_dev_mtx);
 	/* if there are no vdevs, there is nothing to do */
 	if (l2arc_ndev == 0)
-		return (NULL);
+		goto out;
 	first = NULL;
 	next = l2arc_dev_last;
 	do {
 		/* loop around the list looking for a non-faulted vdev */
 	} while (vdev_is_dead(next->l2ad_vdev));
 	/* if we were unable to find any usable vdevs, return NULL */
 	if (vdev_is_dead(next->l2ad_vdev))
-		return (NULL);
+		next = NULL;
 	l2arc_dev_last = next;
+out:
+	mutex_exit(&l2arc_dev_mtx);
+	/*
+	 * Grab the config lock to prevent the 'next' device from being
+	 * removed while we are writing to it.
+	 */
+	if (next != NULL)
+		spa_config_enter(next->l2ad_spa, RW_READER, next);
+	mutex_exit(&spa_namespace_lock);
 	return (next);
+}
+/*
+* Free buffers that were tagged for destruction.
+*/
+static void
+l2arc_do_free_on_write()
+{
+	list_t *buflist;
+	l2arc_data_free_t *df, *df_prev;
+	mutex_enter(&l2arc_free_on_write_mtx);
+	buflist = l2arc_free_on_write;
+	for (df = list_tail(buflist); df; df = df_prev) {
+		df_prev = list_prev(buflist, df);
+		ASSERT(df->l2df_data != NULL);
+		ASSERT(df->l2df_func != NULL);
+		df->l2df_func(df->l2df_data, df->l2df_size);
+		list_remove(buflist, df);
+		kmem_free(df, sizeof (l2arc_data_free_t));
+	}
+	mutex_exit(&l2arc_free_on_write_mtx);
 }
 /*
 * A write to a cache device has completed.  Update all headers to allow
 * reads from these buffers to begin.
 l2arc_write_done(zio_t *zio)
 {
 	l2arc_write_callback_t *cb;
 	l2arc_dev_t *dev;
 	list_t *buflist;
-	l2arc_data_free_t *df, *df_prev;
 	arc_buf_hdr_t *head, *ab, *ab_prev;
+	l2arc_buf_hdr_t *abl2;
 	kmutex_t *hash_lock;
 	cb = zio->io_private;
 	ASSERT(cb != NULL);
 	dev = cb->l2wcb_dev;
 			continue;
 		}
 		if (zio->io_error != 0) {
 			/*
-			 * Error - invalidate L2ARC entry.
+			 * Error - drop L2ARC entry.
 			 */
+			list_remove(buflist, ab);
+			abl2 = ab->b_l2hdr;
 			ab->b_l2hdr = NULL;
+			kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
+			ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
 		}
 		/*
 		 * Allow ARC to begin reads to this L2ARC entry.
 		 */
 	atomic_inc_64(&l2arc_writes_done);
 	list_remove(buflist, head);
 	kmem_cache_free(hdr_cache, head);
 	mutex_exit(&l2arc_buflist_mtx);
-	/*
+	l2arc_do_free_on_write();
-	 * Free buffers that were tagged for destruction.
-	 */
-	mutex_enter(&l2arc_free_on_write_mtx);
-	buflist = l2arc_free_on_write;
-	for (df = list_tail(buflist); df; df = df_prev) {
-		df_prev = list_prev(buflist, df);
-		ASSERT(df->l2df_data != NULL);
-		ASSERT(df->l2df_func != NULL);
-		df->l2df_func(df->l2df_data, df->l2df_size);
-		list_remove(buflist, df);
-		kmem_free(df, sizeof (l2arc_data_free_t));
-	}
-	mutex_exit(&l2arc_free_on_write_mtx);
 	kmem_free(cb, sizeof (l2arc_write_callback_t));
 }
 /*
 	l2arc_read_callback_t *cb;
 	arc_buf_hdr_t *hdr;
 	arc_buf_t *buf;
 	zio_t *rzio;
 	kmutex_t *hash_lock;
-	int equal, err = 0;
+	int equal;
 	cb = zio->io_private;
 	ASSERT(cb != NULL);
 	buf = cb->l2rcb_buf;
 	ASSERT(buf != NULL);
 		mutex_exit(hash_lock);
 		/*
 		 * Buffer didn't survive caching.  Increment stats and
 		 * reissue to the original storage device.
 		 */
-		if (zio->io_error != 0)
+		if (zio->io_error != 0) {
 			ARCSTAT_BUMP(arcstat_l2_io_error);
+		} else {
+			zio->io_error = EIO;
+		}
 		if (!equal)
 			ARCSTAT_BUMP(arcstat_l2_cksum_bad);
-		zio->io_flags &= ~ZIO_FLAG_DONT_CACHE;
+		if (zio->io_waiter == NULL) {
-		rzio = zio_read(NULL, cb->l2rcb_spa, &cb->l2rcb_bp,
+			/*
-		    buf->b_data, zio->io_size, arc_read_done, buf,
+			 * Let the resent I/O call arc_read_done() instead.
-		    zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb);
+			 */
+			zio->io_done = NULL;
-		/*
+			zio->io_flags &= ~ZIO_FLAG_DONT_CACHE;
-		 * Since this is a seperate thread, we can wait on this
-		 * I/O whether there is an io_waiter or not.
+			rzio = zio_read(NULL, cb->l2rcb_spa, &cb->l2rcb_bp,
-		 */
+			    buf->b_data, zio->io_size, arc_read_done, buf,
-		err = zio_wait(rzio);
+			    zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb);
-		/*
+			(void) zio_nowait(rzio);
-		 * Let the resent I/O call arc_read_done() instead.
+		}
-		 * io_error is set to the reissued I/O error status.
-		 */
-		zio->io_done = NULL;
-		zio->io_waiter = NULL;
-		zio->io_error = err;
 	}
 	kmem_free(cb, sizeof (l2arc_read_callback_t));
 }
 	l2arc_buf_hdr_t *abl2;
 	arc_buf_hdr_t *ab, *ab_prev;
 	kmutex_t *hash_lock;
 	uint64_t taddr;
-	ASSERT(MUTEX_HELD(&l2arc_dev_mtx));
 	buflist = dev->l2ad_buflist;
 	if (buflist == NULL)
 		return;
 		 * nothing to evict.
 		 */
 		return;
 	}
-	if (dev->l2ad_hand >= (dev->l2ad_end - (2 * dev->l2ad_write))) {
+	if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) {
 		/*
 		 * When nearing the end of the device, evict to the end
 		 * before the device write hand jumps to the start.
 		 */
 		taddr = dev->l2ad_end;
 			 */
 			arc_change_state(arc_anon, ab, hash_lock);
 			arc_hdr_destroy(ab);
 		} else {
 			/*
+			 * Invalidate issued or about to be issued
+			 * reads, since we may be about to write
+			 * over this location.
+			 */
+			if (HDR_L2_READING(ab)) {
+				ARCSTAT_BUMP(arcstat_l2_evict_reading);
+				ab->b_flags |= ARC_L2_EVICTED;
+			}
+			/*
 			 * Tell ARC this no longer exists in L2ARC.
 			 */
 			if (ab->b_l2hdr != NULL) {
 				abl2 = ab->b_l2hdr;
 				ab->b_l2hdr = NULL;
 			/*
 			 * This may have been leftover after a
 			 * failed write.
 			 */
 			ab->b_flags &= ~ARC_L2_WRITING;
-			/*
-			 * Invalidate issued or about to be issued
-			 * reads, since we may be about to write
-			 * over this location.
-			 */
-			if (HDR_L2_READING(ab)) {
-				ARCSTAT_BUMP(arcstat_l2_evict_reading);
-				ab->b_flags |= ARC_L2_EVICTED;
-			}
 		}
 		mutex_exit(hash_lock);
 	}
 	mutex_exit(&l2arc_buflist_mtx);
 *
 * An ARC_L2_WRITING flag is set so that the L2ARC buffers are not valid
 * for reading until they have completed writing.
 */
 static void
-l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev)
+l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
 {
 	arc_buf_hdr_t *ab, *ab_prev, *head;
 	l2arc_buf_hdr_t *hdrl2;
 	list_t *list;
-	uint64_t passed_sz, write_sz, buf_sz;
+	uint64_t passed_sz, write_sz, buf_sz, headroom;
-	uint64_t target_sz = dev->l2ad_write;
-	uint64_t headroom = dev->l2ad_write * l2arc_headroom;
 	void *buf_data;
 	kmutex_t *hash_lock, *list_lock;
 	boolean_t have_lock, full;
 	l2arc_write_callback_t *cb;
 	zio_t *pio, *wzio;
-	ASSERT(MUTEX_HELD(&l2arc_dev_mtx));
 	ASSERT(dev->l2ad_vdev != NULL);
 	pio = NULL;
 	write_sz = 0;
 	full = B_FALSE;
 	mutex_enter(&l2arc_buflist_mtx);
 	for (int try = 0; try <= 3; try++) {
 		list = l2arc_list_locked(try, &list_lock);
 		passed_sz = 0;
-		for (ab = list_tail(list); ab; ab = ab_prev) {
+		/*
-			ab_prev = list_prev(list, ab);
+		 * L2ARC fast warmup.
+		 *
+		 * Until the ARC is warm and starts to evict, read from the
+		 * head of the ARC lists rather than the tail.
+		 */
+		headroom = target_sz * l2arc_headroom;
+		if (arc_warm == B_FALSE)
+			ab = list_head(list);
+		else
+			ab = list_tail(list);
+		for (; ab; ab = ab_prev) {
+			if (arc_warm == B_FALSE)
+				ab_prev = list_next(list, ab);
+			else
+				ab_prev = list_prev(list, ab);
 			hash_lock = HDR_LOCK(ab);
 			have_lock = MUTEX_HELD(hash_lock);
 			if (!have_lock && !mutex_tryenter(hash_lock)) {
 				/*
 	/*
 	 * Bump device hand to the device start if it is approaching the end.
 	 * l2arc_evict() will already have evicted ahead for this case.
 	 */
-	if (dev->l2ad_hand >= (dev->l2ad_end - dev->l2ad_write)) {
+	if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
 		spa_l2cache_space_update(dev->l2ad_vdev, 0,
 		    dev->l2ad_end - dev->l2ad_hand);
 		dev->l2ad_hand = dev->l2ad_start;
 		dev->l2ad_evict = dev->l2ad_start;
 		dev->l2ad_first = B_FALSE;
 l2arc_feed_thread(void)
 {
 	callb_cpr_t cpr;
 	l2arc_dev_t *dev;
 	spa_t *spa;
-	int interval;
+	uint64_t size;
-	boolean_t startup = B_TRUE;
 	CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
 	mutex_enter(&l2arc_feed_thr_lock);
 	while (l2arc_thread_exit == 0) {
 		/*
-		 * Initially pause for L2ARC_FEED_DELAY seconds as a grace
+		 * Pause for l2arc_feed_secs seconds between writes.
-		 * interval during boot, followed by l2arc_feed_secs seconds
-		 * thereafter.
 		 */
 		CALLB_CPR_SAFE_BEGIN(&cpr);
-		if (startup) {
-			interval = L2ARC_FEED_DELAY;
-			startup = B_FALSE;
-		} else {
-			interval = l2arc_feed_secs;
-		}
 		(void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock,
-		    lbolt + (hz * interval));
+		    lbolt + (hz * l2arc_feed_secs));
 		CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
+		/*
+		 * Quick check for L2ARC devices.
+		 */
 		mutex_enter(&l2arc_dev_mtx);
+		if (l2arc_ndev == 0) {
+			mutex_exit(&l2arc_dev_mtx);
+			continue;
+		}
+		mutex_exit(&l2arc_dev_mtx);
 		/*
 		 * This selects the next l2arc device to write to, and in
 		 * doing so the next spa to feed from: dev->l2ad_spa.   This
-		 * will return NULL if there are no l2arc devices or if they
+		 * will return NULL if there are now no l2arc devices or if
-		 * are all faulted.
+		 * they are all faulted.
+		 *
+		 * If a device is returned, its spa's config lock is also
+		 * held to prevent device removal.  l2arc_dev_get_next()
+		 * will grab and release l2arc_dev_mtx.
 		 */
-		if ((dev = l2arc_dev_get_next()) == NULL) {
+		if ((dev = l2arc_dev_get_next()) == NULL)
-			mutex_exit(&l2arc_dev_mtx);
 			continue;
-		}
+		spa = dev->l2ad_spa;
+		ASSERT(spa != NULL);
 		/*
 		 * Avoid contributing to memory pressure.
 		 */
 		if (arc_reclaim_needed()) {
 			ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
-			mutex_exit(&l2arc_dev_mtx);
+			spa_config_exit(spa, dev);
 			continue;
 		}
-		spa = dev->l2ad_spa;
-		ASSERT(spa != NULL);
 		ARCSTAT_BUMP(arcstat_l2_feeds);
+		size = dev->l2ad_write;
+		if (arc_warm == B_FALSE)
+			size += dev->l2ad_boost;
 		/*
 		 * Evict L2ARC buffers that will be overwritten.
 		 */
-		l2arc_evict(dev, dev->l2ad_write, B_FALSE);
+		l2arc_evict(dev, size, B_FALSE);
 		/*
 		 * Write ARC buffers.
 		 */
-		l2arc_write_buffers(spa, dev);
+		l2arc_write_buffers(spa, dev, size);
-		mutex_exit(&l2arc_dev_mtx);
+		spa_config_exit(spa, dev);
 	}
 	l2arc_thread_exit = 0;
 	cv_broadcast(&l2arc_feed_thr_cv);
 	CALLB_CPR_EXIT(&cpr);		/* drops l2arc_feed_thr_lock */
 	 */
 	adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
 	adddev->l2ad_spa = spa;
 	adddev->l2ad_vdev = vd;
 	adddev->l2ad_write = l2arc_write_max;
+	adddev->l2ad_boost = l2arc_write_boost;
 	adddev->l2ad_start = start;
 	adddev->l2ad_end = end;
 	adddev->l2ad_hand = adddev->l2ad_start;
 	adddev->l2ad_evict = adddev->l2ad_start;
 	adddev->l2ad_first = B_TRUE;
 */
 void
 l2arc_remove_vdev(vdev_t *vd)
 {
 	l2arc_dev_t *dev, *nextdev, *remdev = NULL;
-	/*
-	 * We can only grab the spa config lock when cache device writes
-	 * complete.
-	 */
-	ASSERT3U(l2arc_writes_sent, ==, l2arc_writes_done);
 	/*
 	 * Find the device by vdev
 	 */
 	mutex_enter(&l2arc_dev_mtx);
 	/*
 	 * Remove device from global list
 	 */
 	list_remove(l2arc_dev_list, remdev);
 	l2arc_dev_last = NULL;		/* may have been invalidated */
+	atomic_dec_64(&l2arc_ndev);
+	mutex_exit(&l2arc_dev_mtx);
 	/*
 	 * Clear all buflists and ARC references.  L2ARC device flush.
 	 */
 	l2arc_evict(remdev, 0, B_TRUE);
 	list_destroy(remdev->l2ad_buflist);
 	kmem_free(remdev->l2ad_buflist, sizeof (list_t));
 	kmem_free(remdev, sizeof (l2arc_dev_t));
-	atomic_dec_64(&l2arc_ndev);
-	mutex_exit(&l2arc_dev_mtx);
 }
 void
 l2arc_init()
 {
 }
 void
 l2arc_fini()
 {
+	/*
+	 * This is called from dmu_fini(), which is called from spa_fini();
+	 * Because of this, we can assume that all l2arc devices have
+	 * already been removed when the pools themselves were removed.
+	 */
 	mutex_enter(&l2arc_feed_thr_lock);
 	cv_signal(&l2arc_feed_thr_cv);	/* kick thread out of startup */
 	l2arc_thread_exit = 1;
 	while (l2arc_thread_exit != 0)
 		cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
 	mutex_exit(&l2arc_feed_thr_lock);
+	l2arc_do_free_on_write();
 	mutex_destroy(&l2arc_feed_thr_lock);
 	cv_destroy(&l2arc_feed_thr_cv);
 	mutex_destroy(&l2arc_dev_mtx);
 	mutex_destroy(&l2arc_buflist_mtx);
 	mutex_destroy(&l2arc_free_on_write_mtx);

changeset 6987	877c018eb06c
parent 6643	3a34b0dbb107
child 7046	361307ae060d