/*

 * CDDL HEADER START

 *

 * The contents of this file are subject to the terms of the

 * Common Development and Distribution License (the "License").

 * You may not use this file except in compliance with the License.

 *

 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

 * or http://www.opensolaris.org/os/licensing.

 * See the License for the specific language governing permissions

 * and limitations under the License.

 *

 * When distributing Covered Code, include this CDDL HEADER in each

 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.

 * If applicable, add the following below this CDDL HEADER, with the

 * fields enclosed by brackets "[]" replaced with your own identifying

 * information: Portions Copyright [yyyy] [name of copyright owner]

 *

 * CDDL HEADER END

 */

/*

 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.

 * Use is subject to license terms.

 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#include <sys/zfs_context.h>

#include <sys/fm/fs/zfs.h>

#include <sys/spa.h>

#include <sys/txg.h>

#include <sys/spa_impl.h>

#include <sys/vdev_impl.h>

#include <sys/zio_impl.h>

#include <sys/zio_compress.h>

#include <sys/zio_crypt.h>

#include <sys/zio_checksum.h>

#include <sys/sdt.h>

/*

 * ==========================================================================

 * I/O priority table

 * ==========================================================================

 */

uint8_t zio_priority_table[ZIO_PRIORITY_TABLE_SIZE] = {

	0,	/* ZIO_PRIORITY_NOW		*/

	0,	/* ZIO_PRIORITY_SYNC_READ	*/

	0,	/* ZIO_PRIORITY_SYNC_WRITE	*/

	6,	/* ZIO_PRIORITY_ASYNC_READ	*/

	4,	/* ZIO_PRIORITY_ASYNC_WRITE	*/

	4,	/* ZIO_PRIORITY_FREE		*/

	0,	/* ZIO_PRIORITY_CACHE_FILL	*/

	0,	/* ZIO_PRIORITY_LOG_WRITE	*/

	10,	/* ZIO_PRIORITY_RESILVER	*/

	20,	/* ZIO_PRIORITY_SCRUB		*/

};

/*

 * ==========================================================================

 * I/O type descriptions

 * ==========================================================================

 */

char *zio_type_name[ZIO_TYPES] = {

	"null", "read", "write", "free", "claim", "ioctl" };

/* Force an allocation failure when non-zero */

uint16_t zio_zil_fail_shift = 0;

uint16_t zio_io_fail_shift = 0;

/* Enable/disable the write-retry logic */

int zio_write_retry = 1;

/* Taskq to handle reissuing of I/Os */

taskq_t *zio_taskq;

int zio_resume_threads = 4;

typedef struct zio_sync_pass {

	int	zp_defer_free;		/* defer frees after this pass */

	int	zp_dontcompress;	/* don't compress after this pass */

	int	zp_rewrite;		/* rewrite new bps after this pass */

} zio_sync_pass_t;

zio_sync_pass_t zio_sync_pass = {

	1,	/* zp_defer_free */

	4,	/* zp_dontcompress */

	1,	/* zp_rewrite */

};

static boolean_t zio_io_should_fail(uint16_t);

/*

 * ==========================================================================

 * I/O kmem caches

 * ==========================================================================

 */

kmem_cache_t *zio_cache;

kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];

kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT];

#ifdef _KERNEL

extern vmem_t *zio_alloc_arena;

#endif

/*

 * Determine if we are allowed to issue the IO based on the

 * pool state. If we must wait then block until we are told

 * that we may continue.

 */

#define	ZIO_ENTER(spa) {						\

	if (spa->spa_state == POOL_STATE_IO_FAILURE) {			\

		mutex_enter(&spa->spa_zio_lock);			\

		while (spa->spa_state == POOL_STATE_IO_FAILURE)		\

			cv_wait(&spa->spa_zio_cv, &spa->spa_zio_lock);	\

		mutex_exit(&spa->spa_zio_lock);				\

	}								\

}

/*

 * An allocation zio is one that either currently has the DVA allocate

 * stage set or will have it later in it's lifetime.

 */

#define	IO_IS_ALLOCATING(zio) \

	((zio)->io_orig_pipeline & (1U << ZIO_STAGE_DVA_ALLOCATE))

void

zio_init(void)

{

	size_t c;

	vmem_t *data_alloc_arena = NULL;

#ifdef _KERNEL

	data_alloc_arena = zio_alloc_arena;

#endif

	zio_cache = kmem_cache_create("zio_cache", sizeof (zio_t), 0,

	    NULL, NULL, NULL, NULL, NULL, 0);

	/*

	 * For small buffers, we want a cache for each multiple of

	 * SPA_MINBLOCKSIZE.  For medium-size buffers, we want a cache

	 * for each quarter-power of 2.  For large buffers, we want

	 * a cache for each multiple of PAGESIZE.

	 */

	for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) {

		size_t size = (c + 1) << SPA_MINBLOCKSHIFT;

		size_t p2 = size;

		size_t align = 0;

		while (p2 & (p2 - 1))

			p2 &= p2 - 1;

		if (size <= 4 * SPA_MINBLOCKSIZE) {

			align = SPA_MINBLOCKSIZE;

		} else if (P2PHASE(size, PAGESIZE) == 0) {

			align = PAGESIZE;

		} else if (P2PHASE(size, p2 >> 2) == 0) {

			align = p2 >> 2;

		}

		if (align != 0) {

			char name[36];

			(void) sprintf(name, "zio_buf_%lu", (ulong_t)size);

			zio_buf_cache[c] = kmem_cache_create(name, size,

			    align, NULL, NULL, NULL, NULL, NULL, KMC_NODEBUG);

			(void) sprintf(name, "zio_data_buf_%lu", (ulong_t)size);

			zio_data_buf_cache[c] = kmem_cache_create(name, size,

			    align, NULL, NULL, NULL, NULL, data_alloc_arena,

			    KMC_NODEBUG);

		}

	}

	while (--c != 0) {

		ASSERT(zio_buf_cache[c] != NULL);

		if (zio_buf_cache[c - 1] == NULL)

			zio_buf_cache[c - 1] = zio_buf_cache[c];

		ASSERT(zio_data_buf_cache[c] != NULL);

		if (zio_data_buf_cache[c - 1] == NULL)

			zio_data_buf_cache[c - 1] = zio_data_buf_cache[c];

	}

	zio_taskq = taskq_create("zio_taskq", zio_resume_threads,

	    maxclsyspri, 50, INT_MAX, TASKQ_PREPOPULATE);

	zio_inject_init();

}

void

zio_fini(void)

{

	size_t c;

	kmem_cache_t *last_cache = NULL;

	kmem_cache_t *last_data_cache = NULL;

	for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) {

		if (zio_buf_cache[c] != last_cache) {

			last_cache = zio_buf_cache[c];

			kmem_cache_destroy(zio_buf_cache[c]);

		}

		zio_buf_cache[c] = NULL;

		if (zio_data_buf_cache[c] != last_data_cache) {

			last_data_cache = zio_data_buf_cache[c];

			kmem_cache_destroy(zio_data_buf_cache[c]);

		}

		zio_data_buf_cache[c] = NULL;

	}

	taskq_destroy(zio_taskq);

	kmem_cache_destroy(zio_cache);

	zio_inject_fini();

}

/*

 * ==========================================================================

 * Allocate and free I/O buffers

 * ==========================================================================

 */

/*

 * Use zio_buf_alloc to allocate ZFS metadata.  This data will appear in a

 * crashdump if the kernel panics, so use it judiciously.  Obviously, it's

 * useful to inspect ZFS metadata, but if possible, we should avoid keeping

 * excess / transient data in-core during a crashdump.

 */

void *

zio_buf_alloc(size_t size)

{

	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;

	ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);

	return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE));

}

/*

 * Use zio_data_buf_alloc to allocate data.  The data will not appear in a

 * crashdump if the kernel panics.  This exists so that we will limit the amount

 * of ZFS data that shows up in a kernel crashdump.  (Thus reducing the amount

 * of kernel heap dumped to disk when the kernel panics)

 */

void *

zio_data_buf_alloc(size_t size)

{

	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;

	ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);

	return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE));

}

void

zio_buf_free(void *buf, size_t size)

{

	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;

	ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);

	kmem_cache_free(zio_buf_cache[c], buf);

}

void

zio_data_buf_free(void *buf, size_t size)

{

	size_t c = (size - 1) >> SPA_MINBLOCKSHIFT;

	ASSERT(c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT);

	kmem_cache_free(zio_data_buf_cache[c], buf);

}

/*

 * ==========================================================================

 * Push and pop I/O transform buffers

 * ==========================================================================

 */

static void

zio_push_transform(zio_t *zio, void *data, uint64_t size, uint64_t bufsize)

{

	zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP);

	zt->zt_data = data;

	zt->zt_size = size;

	zt->zt_bufsize = bufsize;

	zt->zt_next = zio->io_transform_stack;

	zio->io_transform_stack = zt;

	zio->io_data = data;

	zio->io_size = size;

}

static void

zio_pop_transform(zio_t *zio, void **data, uint64_t *size, uint64_t *bufsize)

{

	zio_transform_t *zt = zio->io_transform_stack;

	*data = zt->zt_data;

	*size = zt->zt_size;

	*bufsize = zt->zt_bufsize;

	zio->io_transform_stack = zt->zt_next;

	kmem_free(zt, sizeof (zio_transform_t));

	if ((zt = zio->io_transform_stack) != NULL) {

		zio->io_data = zt->zt_data;

		zio->io_size = zt->zt_size;

	}

}

static void

zio_clear_transform_stack(zio_t *zio)

{

	void *data;

	uint64_t size, bufsize;

	ASSERT(zio->io_transform_stack != NULL);

	zio_pop_transform(zio, &data, &size, &bufsize);

	while (zio->io_transform_stack != NULL) {

		zio_buf_free(data, bufsize);

		zio_pop_transform(zio, &data, &size, &bufsize);

	}

}

/*

 * ==========================================================================

 * Create the various types of I/O (read, write, free)

 * ==========================================================================

 */

static zio_t *

zio_create(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,

    void *data, uint64_t size, zio_done_func_t *done, void *private,

    zio_type_t type, int priority, int flags, uint8_t stage, uint32_t pipeline)

{

	zio_t *zio;

	ASSERT3U(size, <=, SPA_MAXBLOCKSIZE);

	ASSERT(P2PHASE(size, SPA_MINBLOCKSIZE) == 0);

	zio = kmem_cache_alloc(zio_cache, KM_SLEEP);

	bzero(zio, sizeof (zio_t));

	zio->io_parent = pio;

	zio->io_spa = spa;

	zio->io_txg = txg;

	zio->io_flags = flags;

	if (bp != NULL) {

		zio->io_bp = bp;

		zio->io_bp_copy = *bp;

		zio->io_bp_orig = *bp;

	}

	zio->io_done = done;

	zio->io_private = private;

	zio->io_type = type;

	zio->io_priority = priority;

	zio->io_stage = stage;

	zio->io_pipeline = pipeline;

	zio->io_timestamp = lbolt64;

	mutex_init(&zio->io_lock, NULL, MUTEX_DEFAULT, NULL);

	cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL);

	zio_push_transform(zio, data, size, size);

	/*

	 * Note on config lock:

	 *

	 * If CONFIG_HELD is set, then the caller already has the config

	 * lock, so we don't need it for this io.

	 *

	 * We set CONFIG_GRABBED to indicate that we have grabbed the

	 * config lock on behalf of this io, so it should be released

	 * in zio_done.

	 *

	 * Unless CONFIG_HELD is set, we will grab the config lock for

	 * any top-level (parent-less) io, *except* NULL top-level ios.

	 * The NULL top-level ios rarely have any children, so we delay

	 * grabbing the lock until the first child is added (but it is

	 * still grabbed on behalf of the top-level i/o, so additional

	 * children don't need to also grab it).  This greatly reduces

	 * contention on the config lock.

	 */

	if (pio == NULL) {

		if (type != ZIO_TYPE_NULL &&

		    !(flags & ZIO_FLAG_CONFIG_HELD)) {

			spa_config_enter(spa, RW_READER, zio);

			zio->io_flags |= ZIO_FLAG_CONFIG_GRABBED;

		}

		zio->io_root = zio;

	} else {

		zio->io_root = pio->io_root;

		if (!(flags & ZIO_FLAG_NOBOOKMARK))

			zio->io_logical = pio->io_logical;

		mutex_enter(&pio->io_lock);

		if (pio->io_parent == NULL &&

		    pio->io_type == ZIO_TYPE_NULL &&

		    !(pio->io_flags & ZIO_FLAG_CONFIG_GRABBED) &&

		    !(pio->io_flags & ZIO_FLAG_CONFIG_HELD)) {

			pio->io_flags |= ZIO_FLAG_CONFIG_GRABBED;

			spa_config_enter(spa, RW_READER, pio);

		}

		if (stage < ZIO_STAGE_READY)

			pio->io_children_notready++;

		pio->io_children_notdone++;

		zio->io_sibling_next = pio->io_child;

		zio->io_sibling_prev = NULL;

		if (pio->io_child != NULL)

			pio->io_child->io_sibling_prev = zio;

		pio->io_child = zio;

		zio->io_ndvas = pio->io_ndvas;

		mutex_exit(&pio->io_lock);

	}

	/*

	 * Save off the original state incase we need to retry later.

	 */

	zio->io_orig_stage = zio->io_stage;

	zio->io_orig_pipeline = zio->io_pipeline;

	zio->io_orig_flags = zio->io_flags;

	return (zio);

}

static void

zio_reset(zio_t *zio)

{

	zio_clear_transform_stack(zio);

	zio->io_flags = zio->io_orig_flags;

	zio->io_stage = zio->io_orig_stage;

	zio->io_pipeline = zio->io_orig_pipeline;

	zio_push_transform(zio, zio->io_data, zio->io_size, zio->io_size);

}

zio_t *

zio_null(zio_t *pio, spa_t *spa, zio_done_func_t *done, void *private,

	int flags)

{

	zio_t *zio;

	zio = zio_create(pio, spa, 0, NULL, NULL, 0, done, private,

	    ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, ZIO_STAGE_OPEN,

	    ZIO_WAIT_FOR_CHILDREN_PIPELINE);

	return (zio);

}

zio_t *

zio_root(spa_t *spa, zio_done_func_t *done, void *private, int flags)

{

	return (zio_null(NULL, spa, done, private, flags));

}

zio_t *

zio_read(zio_t *pio, spa_t *spa, blkptr_t *bp, void *data,

    uint64_t size, zio_done_func_t *done, void *private,

    int priority, int flags, zbookmark_t *zb)

{

	zio_t *zio;

	ASSERT3U(size, ==, BP_GET_LSIZE(bp));

	/*

	 * If the user has specified that we allow I/Os to continue

	 * then attempt to satisfy the read.

	 */

	if (spa_get_failmode(spa) != ZIO_FAILURE_MODE_CONTINUE)

		ZIO_ENTER(spa);

	zio = zio_create(pio, spa, bp->blk_birth, bp, data, size, done, private,

	    ZIO_TYPE_READ, priority, flags | ZIO_FLAG_USER,

	    ZIO_STAGE_OPEN, ZIO_READ_PIPELINE);

	zio->io_bookmark = *zb;

	zio->io_logical = zio;

	/*

	 * Work off our copy of the bp so the caller can free it.

	 */

	zio->io_bp = &zio->io_bp_copy;

	return (zio);

}

zio_t *

zio_write(zio_t *pio, spa_t *spa, int checksum, int compress, int crypt,

    int ncopies, uint64_t txg, blkptr_t *bp, void *data, uint64_t size,

    zio_done_func_t *ready, zio_done_func_t *done, void *private, int priority,

    int flags, zbookmark_t *zb)

{

	zio_t *zio;

	ASSERT(checksum >= ZIO_CHECKSUM_OFF &&

	    checksum < ZIO_CHECKSUM_FUNCTIONS);

	ASSERT(compress >= ZIO_COMPRESS_OFF &&

	    compress < ZIO_COMPRESS_FUNCTIONS);

	ASSERT(crypt >= ZIO_CRYPT_OFF &&

	    crypt < ZIO_CRYPT_FUNCTIONS);

	ZIO_ENTER(spa);

	zio = zio_create(pio, spa, txg, bp, data, size, done, private,

	    ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_USER,