/*

 * CDDL HEADER START

 *

 * The contents of this file are subject to the terms of the

 * Common Development and Distribution License (the "License").

 * You may not use this file except in compliance with the License.

 *

 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

 * or http://www.opensolaris.org/os/licensing.

 * See the License for the specific language governing permissions

 * and limitations under the License.

 *

 * When distributing Covered Code, include this CDDL HEADER in each

 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.

 * If applicable, add the following below this CDDL HEADER, with the

 * fields enclosed by brackets "[]" replaced with your own identifying

 * information: Portions Copyright [yyyy] [name of copyright owner]

 *

 * CDDL HEADER END

 */

/*

 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.

 * Use is subject to license terms.

 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#include <sys/zfs_context.h>

#include <sys/dmu.h>

#include <sys/dmu_impl.h>

#include <sys/dbuf.h>

#include <sys/dmu_objset.h>

#include <sys/dsl_dataset.h>

#include <sys/dsl_dir.h>

#include <sys/dmu_tx.h>

#include <sys/spa.h>

#include <sys/zio.h>

#include <sys/dmu_zfetch.h>

static void dbuf_destroy(dmu_buf_impl_t *db);

static int dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);

static arc_done_func_t dbuf_write_done;

/*

 * Global data structures and functions for the dbuf cache.

 */

taskq_t *dbuf_tq;

static kmem_cache_t *dbuf_cache;

/* ARGSUSED */

static int

dbuf_cons(void *vdb, void *unused, int kmflag)

{

	dmu_buf_impl_t *db = vdb;

	bzero(db, sizeof (dmu_buf_impl_t));

	mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL);

	cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);

	refcount_create(&db->db_holds);

	return (0);

}

/* ARGSUSED */

static void

dbuf_dest(void *vdb, void *unused)

{

	dmu_buf_impl_t *db = vdb;

	mutex_destroy(&db->db_mtx);

	cv_destroy(&db->db_changed);

	refcount_destroy(&db->db_holds);

}

/*

 * dbuf hash table routines

 */

static dbuf_hash_table_t dbuf_hash_table;

static uint64_t dbuf_hash_count;

static uint64_t

dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid)

{

	uintptr_t osv = (uintptr_t)os;

	uint64_t crc = -1ULL;

	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);

	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF];

	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF];

	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF];

	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF];

	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF];

	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF];

	crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16);

	return (crc);

}

#define	DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid);

#define	DBUF_EQUAL(dbuf, os, obj, level, blkid)		\

	((dbuf)->db.db_object == (obj) &&		\

	(dbuf)->db_objset == (os) &&			\

	(dbuf)->db_level == (level) &&			\

	(dbuf)->db_blkid == (blkid))

dmu_buf_impl_t *

dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid)

{

	dbuf_hash_table_t *h = &dbuf_hash_table;

	objset_impl_t *os = dn->dn_objset;

	uint64_t obj = dn->dn_object;

	uint64_t hv = DBUF_HASH(os, obj, level, blkid);

	uint64_t idx = hv & h->hash_table_mask;

	dmu_buf_impl_t *db;

	mutex_enter(DBUF_HASH_MUTEX(h, idx));

	for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {

		if (DBUF_EQUAL(db, os, obj, level, blkid)) {

			mutex_enter(&db->db_mtx);

			if (db->db_state != DB_EVICTING) {

				mutex_exit(DBUF_HASH_MUTEX(h, idx));

				return (db);

			}

			mutex_exit(&db->db_mtx);

		}

	}

	mutex_exit(DBUF_HASH_MUTEX(h, idx));

	return (NULL);

}

/*

 * Insert an entry into the hash table.  If there is already an element

 * equal to elem in the hash table, then the already existing element

 * will be returned and the new element will not be inserted.

 * Otherwise returns NULL.

 */

static dmu_buf_impl_t *

dbuf_hash_insert(dmu_buf_impl_t *db)

{

	dbuf_hash_table_t *h = &dbuf_hash_table;

	objset_impl_t *os = db->db_objset;

	uint64_t obj = db->db.db_object;

	int level = db->db_level;

	uint64_t blkid = db->db_blkid;

	uint64_t hv = DBUF_HASH(os, obj, level, blkid);

	uint64_t idx = hv & h->hash_table_mask;

	dmu_buf_impl_t *dbf;

	mutex_enter(DBUF_HASH_MUTEX(h, idx));

	for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) {

		if (DBUF_EQUAL(dbf, os, obj, level, blkid)) {

			mutex_enter(&dbf->db_mtx);

			if (dbf->db_state != DB_EVICTING) {

				mutex_exit(DBUF_HASH_MUTEX(h, idx));

				return (dbf);

			}

			mutex_exit(&dbf->db_mtx);

		}

	}

	mutex_enter(&db->db_mtx);

	db->db_hash_next = h->hash_table[idx];

	h->hash_table[idx] = db;

	mutex_exit(DBUF_HASH_MUTEX(h, idx));

	atomic_add_64(&dbuf_hash_count, 1);

	return (NULL);

}

/*

 * Remove an entry from the hash table.  This operation will

 * fail if there are any existing holds on the db.

 */

static void

dbuf_hash_remove(dmu_buf_impl_t *db)

{

	dbuf_hash_table_t *h = &dbuf_hash_table;

	uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object,

	    db->db_level, db->db_blkid);

	uint64_t idx = hv & h->hash_table_mask;

	dmu_buf_impl_t *dbf, **dbp;

	/*

	 * We musn't hold db_mtx to maintin lock ordering:

	 * DBUF_HASH_MUTEX > db_mtx.

	 */

	ASSERT(refcount_is_zero(&db->db_holds));

	ASSERT(db->db_state == DB_EVICTING);

	ASSERT(!MUTEX_HELD(&db->db_mtx));

	mutex_enter(DBUF_HASH_MUTEX(h, idx));

	dbp = &h->hash_table[idx];

	while ((dbf = *dbp) != db) {

		dbp = &dbf->db_hash_next;

		ASSERT(dbf != NULL);

	}

	*dbp = db->db_hash_next;

	db->db_hash_next = NULL;

	mutex_exit(DBUF_HASH_MUTEX(h, idx));

	atomic_add_64(&dbuf_hash_count, -1);

}

static arc_evict_func_t dbuf_do_evict;

static void

dbuf_evict_user(dmu_buf_impl_t *db)

{

	ASSERT(MUTEX_HELD(&db->db_mtx));

	if (db->db_level != 0 || db->db_d.db_evict_func == NULL)

		return;

	if (db->db_d.db_user_data_ptr_ptr)

		*db->db_d.db_user_data_ptr_ptr = db->db.db_data;

	db->db_d.db_evict_func(&db->db, db->db_d.db_user_ptr);

	db->db_d.db_user_ptr = NULL;

	db->db_d.db_user_data_ptr_ptr = NULL;

	db->db_d.db_evict_func = NULL;

}

void

dbuf_evict(dmu_buf_impl_t *db)

{

	int i;

	ASSERT(MUTEX_HELD(&db->db_mtx));

	ASSERT(db->db_buf == NULL);

#ifdef ZFS_DEBUG

	for (i = 0; i < TXG_SIZE; i++) {

		ASSERT(!list_link_active(&db->db_dirty_node[i]));

		ASSERT(db->db_level != 0 || db->db_d.db_data_old[i] == NULL);

	}

#endif

	dbuf_clear(db);

	dbuf_destroy(db);

}

void

dbuf_init(void)

{

	uint64_t hsize = 1ULL << 16;

	dbuf_hash_table_t *h = &dbuf_hash_table;

	int i;

	/*

	 * The hash table is big enough to fill all of physical memory

	 * with an average 4K block size.  The table will take up

	 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers).

	 */

	while (hsize * 4096 < physmem * PAGESIZE)

		hsize <<= 1;

retry:

	h->hash_table_mask = hsize - 1;

	h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP);

	if (h->hash_table == NULL) {

		/* XXX - we should really return an error instead of assert */

		ASSERT(hsize > (1ULL << 10));

		hsize >>= 1;

		goto retry;

	}

	dbuf_cache = kmem_cache_create("dmu_buf_impl_t",

	    sizeof (dmu_buf_impl_t),

	    0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0);

	dbuf_tq = taskq_create("dbuf_tq", 8, maxclsyspri, 50, INT_MAX,

	    TASKQ_PREPOPULATE);

	for (i = 0; i < DBUF_MUTEXES; i++)

		mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);

}

void

dbuf_fini(void)

{

	dbuf_hash_table_t *h = &dbuf_hash_table;

	int i;

	taskq_destroy(dbuf_tq);

	dbuf_tq = NULL;

	for (i = 0; i < DBUF_MUTEXES; i++)

		mutex_destroy(&h->hash_mutexes[i]);

	kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));

	kmem_cache_destroy(dbuf_cache);

}

/*

 * Other stuff.

 */

#ifdef ZFS_DEBUG

static void

dbuf_verify(dmu_buf_impl_t *db)

{

	int i;

	dnode_t *dn = db->db_dnode;

	ASSERT(MUTEX_HELD(&db->db_mtx));

	if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY))

		return;

	ASSERT(db->db_objset != NULL);

	if (dn == NULL) {

		ASSERT(db->db_parent == NULL);

		ASSERT(db->db_blkptr == NULL);

	} else {

		ASSERT3U(db->db.db_object, ==, dn->dn_object);

		ASSERT3P(db->db_objset, ==, dn->dn_objset);

		ASSERT3U(db->db_level, <, dn->dn_nlevels);

		ASSERT(db->db_blkid == DB_BONUS_BLKID ||

		    list_head(&dn->dn_dbufs));

	}

	if (db->db_blkid == DB_BONUS_BLKID) {

		ASSERT(dn != NULL);

		ASSERT3U(db->db.db_size, ==, dn->dn_bonuslen);

		ASSERT3U(db->db.db_offset, ==, DB_BONUS_BLKID);

	} else {

		ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);

	}

	if (db->db_level == 0) {

		/* we can be momentarily larger in dnode_set_blksz() */

		if (db->db_blkid != DB_BONUS_BLKID && dn) {

			ASSERT3U(db->db.db_size, >=, dn->dn_datablksz);

		}

		if (db->db.db_object == DMU_META_DNODE_OBJECT) {

			for (i = 0; i < TXG_SIZE; i++) {

				/*

				 * it should only be modified in syncing

				 * context, so make sure we only have

				 * one copy of the data.

				 */

				ASSERT(db->db_d.db_data_old[i] == NULL ||

				    db->db_d.db_data_old[i] == db->db_buf);

			}

		}

	}

	/* verify db->db_blkptr */

	if (db->db_blkptr) {

		if (db->db_parent == dn->dn_dbuf) {

			/* db is pointed to by the dnode */

			/* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */

			if (db->db.db_object == DMU_META_DNODE_OBJECT)

				ASSERT(db->db_parent == NULL);

			else

				ASSERT(db->db_parent != NULL);

			ASSERT3P(db->db_blkptr, ==,

			    &dn->dn_phys->dn_blkptr[db->db_blkid]);

		} else {

			/* db is pointed to by an indirect block */

			int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT;

			ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);

			ASSERT3U(db->db_parent->db.db_object, ==,

			    db->db.db_object);

			/*

			 * dnode_grow_indblksz() can make this fail if we don't

			 * have the struct_rwlock.  XXX indblksz no longer

			 * grows.  safe to do this now?

			 */

			if (RW_WRITE_HELD(&db->db_dnode->dn_struct_rwlock)) {

				ASSERT3P(db->db_blkptr, ==,

				    ((blkptr_t *)db->db_parent->db.db_data +

				    db->db_blkid % epb));

			}

		}

	}

	if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&

	    db->db.db_data && db->db_blkid != DB_BONUS_BLKID &&

	    db->db_state != DB_FILL && !dn->dn_free_txg) {

		/*

		 * If the blkptr isn't set but they have nonzero data,

		 * it had better be dirty, otherwise we'll lose that

		 * data when we evict this buffer.

		 */

		if (db->db_dirtycnt == 0) {

			uint64_t *buf = db->db.db_data;

			int i;

			for (i = 0; i < db->db.db_size >> 3; i++) {

				ASSERT(buf[i] == 0);

			}

		}

	}

}

#endif

static void

dbuf_update_data(dmu_buf_impl_t *db)

{

	ASSERT(MUTEX_HELD(&db->db_mtx));

	if (db->db_level == 0 && db->db_d.db_user_data_ptr_ptr) {

		ASSERT(!refcount_is_zero(&db->db_holds));

		*db->db_d.db_user_data_ptr_ptr = db->db.db_data;

	}

}

static void

dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)

{

	ASSERT(MUTEX_HELD(&db->db_mtx));

	ASSERT(db->db_buf == NULL || !arc_has_callback(db->db_buf));

	db->db_buf = buf;

	if (buf != NULL) {

		ASSERT(buf->b_data != NULL);

		db->db.db_data = buf->b_data;

		if (!arc_released(buf))

			arc_set_callback(buf, dbuf_do_evict, db);

		dbuf_update_data(db);

	} else {

		dbuf_evict_user(db);

		db->db.db_data = NULL;

		db->db_state = DB_UNCACHED;

	}

}

uint64_t

dbuf_whichblock(dnode_t *dn, uint64_t offset)

{

	if (dn->dn_datablkshift) {

		return (offset >> dn->dn_datablkshift);

	} else {

		ASSERT3U(offset, <, dn->dn_datablksz);

		return (0);

	}

}

static void

dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb)

{

	dmu_buf_impl_t *db = vdb;

	mutex_enter(&db->db_mtx);

	ASSERT3U(db->db_state, ==, DB_READ);

	/*

	 * All reads are synchronous, so we must have a hold on the dbuf

	 */

	ASSERT(refcount_count(&db->db_holds) > 0);

	ASSERT(db->db_buf == NULL);

	ASSERT(db->db.db_data == NULL);

	if (db->db_level == 0 && db->db_d.db_freed_in_flight) {

		/* we were freed in flight; disregard any error */

		arc_release(buf, db);

		bzero(buf->b_data, db->db.db_size);

		db->db_d.db_freed_in_flight = FALSE;

		dbuf_set_data(db, buf);

		db->db_state = DB_CACHED;

	} else if (zio == NULL || zio->io_error == 0) {

		dbuf_set_data(db, buf);

		db->db_state = DB_CACHED;

	} else {

		ASSERT(db->db_blkid != DB_BONUS_BLKID);

		ASSERT3P(db->db_buf, ==, NULL);

		VERIFY(arc_buf_remove_ref(buf, db) == 1);

		db->db_state = DB_UNCACHED;

	}

	cv_broadcast(&db->db_changed);

	mutex_exit(&db->db_mtx);

	dbuf_rele(db, NULL);

}

static void

dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags)

{

	blkptr_t *bp;

	zbookmark_t zb;

	uint32_t aflags = ARC_NOWAIT;

	ASSERT(!refcount_is_zero(&db->db_holds));

	/* We need the struct_rwlock to prevent db_blkptr from changing. */

	ASSERT(RW_LOCK_HELD(&db->db_dnode->dn_struct_rwlock));

	ASSERT(MUTEX_HELD(&db->db_mtx));

	ASSERT(db->db_state == DB_UNCACHED);

	ASSERT(db->db_buf == NULL);

	if (db->db_blkid == DB_BONUS_BLKID) {

		ASSERT3U(db->db_dnode->dn_bonuslen, ==, db->db.db_size);

		db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN);

		if (db->db.db_size < DN_MAX_BONUSLEN)

			bzero(db->db.db_data, DN_MAX_BONUSLEN);

		bcopy(DN_BONUS(db->db_dnode->dn_phys), db->db.db_data,

		    db->db.db_size);

		dbuf_update_data(db);

		db->db_state = DB_CACHED;

		mutex_exit(&db->db_mtx);

		return;

	}

	if (db->db_level == 0 && dnode_block_freed(db->db_dnode, db->db_blkid))

		bp = NULL;

	else

		bp = db->db_blkptr;

	if (bp == NULL)

		dprintf_dbuf(db, "blkptr: %s\n", "NULL");

	else

		dprintf_dbuf_bp(db, bp, "%s", "blkptr:");

	if (bp == NULL || BP_IS_HOLE(bp)) {

		ASSERT(bp == NULL || BP_IS_HOLE(bp));

		dbuf_set_data(db, arc_buf_alloc(db->db_dnode->dn_objset->os_spa,

		    db->db.db_size, db));

		bzero(db->db.db_data, db->db.db_size);

		db->db_state = DB_CACHED;

		*flags |= DB_RF_CACHED;

		mutex_exit(&db->db_mtx);

		return;

	}

	db->db_state = DB_READ;

	mutex_exit(&db->db_mtx);

	zb.zb_objset = db->db_objset->os_dsl_dataset ?