/*

 * CDDL HEADER START

 *

 * The contents of this file are subject to the terms of the

 * Common Development and Distribution License, Version 1.0 only

 * (the "License").  You may not use this file except in compliance

 * with the License.

 *

 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

 * or http://www.opensolaris.org/os/licensing.

 * See the License for the specific language governing permissions

 * and limitations under the License.

 *

 * When distributing Covered Code, include this CDDL HEADER in each

 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.

 * If applicable, add the following below this CDDL HEADER, with the

 * fields enclosed by brackets "[]" replaced with your own identifying

 * information: Portions Copyright [yyyy] [name of copyright owner]

 *

 * CDDL HEADER END

 */

/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/

/*	  All Rights Reserved  	*/

/*

 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.

 * Use is subject to license terms.

 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#include <sys/types.h>

#include <sys/param.h>

#include <sys/thread.h>

#include <sys/sysmacros.h>

#include <sys/stropts.h>

#include <sys/stream.h>

#include <sys/strsubr.h>

#include <sys/strsun.h>

#include <sys/conf.h>

#include <sys/debug.h>

#include <sys/cmn_err.h>

#include <sys/kmem.h>

#include <sys/atomic.h>

#include <sys/errno.h>

#include <sys/vtrace.h>

#include <sys/ftrace.h>

#include <sys/ontrap.h>

#include <sys/multidata.h>

#include <sys/multidata_impl.h>

#include <sys/sdt.h>

#ifdef DEBUG

#include <sys/kmem_impl.h>

#endif

/*

 * This file contains all the STREAMS utility routines that may

 * be used by modules and drivers.

 */

/*

 * STREAMS message allocator: principles of operation

 *

 * The streams message allocator consists of all the routines that

 * allocate, dup and free streams messages: allocb(), [d]esballoc[a],

 * dupb(), freeb() and freemsg().  What follows is a high-level view

 * of how the allocator works.

 *

 * Every streams message consists of one or more mblks, a dblk, and data.

 * All mblks for all types of messages come from a common mblk_cache.

 * The dblk and data come in several flavors, depending on how the

 * message is allocated:

 *

 * (1) mblks up to DBLK_MAX_CACHE size are allocated from a collection of

 *     fixed-size dblk/data caches. For message sizes that are multiples of

 *     PAGESIZE, dblks are allocated separately from the buffer.

 *     The associated buffer is allocated by the constructor using kmem_alloc().

 *     For all other message sizes, dblk and its associated data is allocated

 *     as a single contiguous chunk of memory.

 *     Objects in these caches consist of a dblk plus its associated data.

 *     allocb() determines the nearest-size cache by table lookup:

 *     the dblk_cache[] array provides the mapping from size to dblk cache.

 *

 * (2) Large messages (size > DBLK_MAX_CACHE) are constructed by

 *     kmem_alloc()'ing a buffer for the data and supplying that

 *     buffer to gesballoc(), described below.

 *

 * (3) The four flavors of [d]esballoc[a] are all implemented by a

 *     common routine, gesballoc() ("generic esballoc").  gesballoc()

 *     allocates a dblk from the global dblk_esb_cache and sets db_base,

 *     db_lim and db_frtnp to describe the caller-supplied buffer.

 *

 * While there are several routines to allocate messages, there is only

 * one routine to free messages: freeb().  freeb() simply invokes the

 * dblk's free method, dbp->db_free(), which is set at allocation time.

 *

 * dupb() creates a new reference to a message by allocating a new mblk,

 * incrementing the dblk reference count and setting the dblk's free

 * method to dblk_decref().  The dblk's original free method is retained

 * in db_lastfree.  dblk_decref() decrements the reference count on each

 * freeb().  If this is not the last reference it just frees the mblk;

 * if this *is* the last reference, it restores db_free to db_lastfree,

 * sets db_mblk to the current mblk (see below), and invokes db_lastfree.

 *

 * The implementation makes aggressive use of kmem object caching for

 * maximum performance.  This makes the code simple and compact, but

 * also a bit abstruse in some places.  The invariants that constitute a

 * message's constructed state, described below, are more subtle than usual.

 *

 * Every dblk has an "attached mblk" as part of its constructed state.

 * The mblk is allocated by the dblk's constructor and remains attached

 * until the message is either dup'ed or pulled up.  In the dupb() case

 * the mblk association doesn't matter until the last free, at which time

 * dblk_decref() attaches the last mblk to the dblk.  pullupmsg() affects

 * the mblk association because it swaps the leading mblks of two messages,

 * so it is responsible for swapping their db_mblk pointers accordingly.

 * From a constructed-state viewpoint it doesn't matter that a dblk's

 * attached mblk can change while the message is allocated; all that

 * matters is that the dblk has *some* attached mblk when it's freed.

 *

 * The sizes of the allocb() small-message caches are not magical.

 * They represent a good trade-off between internal and external

 * fragmentation for current workloads.  They should be reevaluated

 * periodically, especially if allocations larger than DBLK_MAX_CACHE

 * become common.  We use 64-byte alignment so that dblks don't

 * straddle cache lines unnecessarily.

 */

#define	DBLK_MAX_CACHE		73728

#define	DBLK_CACHE_ALIGN	64

#define	DBLK_MIN_SIZE		8

#define	DBLK_SIZE_SHIFT		3

#ifdef _BIG_ENDIAN

#define	DBLK_RTFU_SHIFT(field)	\

	(8 * (&((dblk_t *)0)->db_struioflag - &((dblk_t *)0)->field))

#else

#define	DBLK_RTFU_SHIFT(field)	\

	(8 * (&((dblk_t *)0)->field - &((dblk_t *)0)->db_ref))

#endif

#define	DBLK_RTFU(ref, type, flags, uioflag)	\

	(((ref) << DBLK_RTFU_SHIFT(db_ref)) | \

	((type) << DBLK_RTFU_SHIFT(db_type)) | \

	(((flags) | (ref - 1)) << DBLK_RTFU_SHIFT(db_flags)) | \

	((uioflag) << DBLK_RTFU_SHIFT(db_struioflag)))

#define	DBLK_RTFU_REF_MASK	(DBLK_REFMAX << DBLK_RTFU_SHIFT(db_ref))

#define	DBLK_RTFU_WORD(dbp)	(*((uint32_t *)&(dbp)->db_ref))

#define	MBLK_BAND_FLAG_WORD(mp)	(*((uint32_t *)&(mp)->b_band))

static size_t dblk_sizes[] = {

#ifdef _LP64

	16, 80, 144, 208, 272, 336, 528, 1040, 1488, 1936, 2576, 3920,

	8192, 12112, 16384, 20304, 24576, 28496, 32768, 36688,

	40960, 44880, 49152, 53072, 57344, 61264, 65536, 69456,

#else

	64, 128, 320, 576, 1088, 1536, 1984, 2624, 3968,

	8192, 12160, 16384, 20352, 24576, 28544, 32768, 36736,

	40960, 44928, 49152, 53120, 57344, 61312, 65536, 69504,

#endif

	DBLK_MAX_CACHE, 0

};

static struct kmem_cache *dblk_cache[DBLK_MAX_CACHE / DBLK_MIN_SIZE];

static struct kmem_cache *mblk_cache;

static struct kmem_cache *dblk_esb_cache;

static struct kmem_cache *fthdr_cache;

static struct kmem_cache *ftblk_cache;

static void dblk_lastfree(mblk_t *mp, dblk_t *dbp);

static mblk_t *allocb_oversize(size_t size, int flags);

static int allocb_tryhard_fails;

static void frnop_func(void *arg);

frtn_t frnop = { frnop_func };

static void bcache_dblk_lastfree(mblk_t *mp, dblk_t *dbp);

static boolean_t rwnext_enter(queue_t *qp);

static void rwnext_exit(queue_t *qp);

/*

 * Patchable mblk/dblk kmem_cache flags.

 */

int dblk_kmem_flags = 0;

int mblk_kmem_flags = 0;

static int

dblk_constructor(void *buf, void *cdrarg, int kmflags)

{

	dblk_t *dbp = buf;

	ssize_t msg_size = (ssize_t)cdrarg;

	size_t index;

	ASSERT(msg_size != 0);

	index = (msg_size - 1) >> DBLK_SIZE_SHIFT;

	ASSERT(index < (DBLK_MAX_CACHE >> DBLK_SIZE_SHIFT));

	if ((dbp->db_mblk = kmem_cache_alloc(mblk_cache, kmflags)) == NULL)

		return (-1);

	if ((msg_size & PAGEOFFSET) == 0) {

		dbp->db_base = kmem_alloc(msg_size, kmflags);

		if (dbp->db_base == NULL) {

			kmem_cache_free(mblk_cache, dbp->db_mblk);

			return (-1);

		}

	} else {

		dbp->db_base = (unsigned char *)&dbp[1];

	}

	dbp->db_mblk->b_datap = dbp;

	dbp->db_cache = dblk_cache[index];

	dbp->db_lim = dbp->db_base + msg_size;

	dbp->db_free = dbp->db_lastfree = dblk_lastfree;

	dbp->db_frtnp = NULL;

	dbp->db_fthdr = NULL;

	dbp->db_credp = NULL;

	dbp->db_cpid = -1;

	dbp->db_struioflag = 0;

	dbp->db_struioun.cksum.flags = 0;

	return (0);

}

/*ARGSUSED*/

static int

dblk_esb_constructor(void *buf, void *cdrarg, int kmflags)

{

	dblk_t *dbp = buf;

	if ((dbp->db_mblk = kmem_cache_alloc(mblk_cache, kmflags)) == NULL)

		return (-1);

	dbp->db_mblk->b_datap = dbp;

	dbp->db_cache = dblk_esb_cache;

	dbp->db_fthdr = NULL;

	dbp->db_credp = NULL;

	dbp->db_cpid = -1;

	dbp->db_struioflag = 0;

	dbp->db_struioun.cksum.flags = 0;

	return (0);

}

static int

bcache_dblk_constructor(void *buf, void *cdrarg, int kmflags)

{

	dblk_t *dbp = buf;

	bcache_t *bcp = (bcache_t *)cdrarg;

	if ((dbp->db_mblk = kmem_cache_alloc(mblk_cache, kmflags)) == NULL)

		return (-1);

	if ((dbp->db_base = (unsigned char *)kmem_cache_alloc(bcp->buffer_cache,

	    kmflags)) == NULL) {

		kmem_cache_free(mblk_cache, dbp->db_mblk);

		return (-1);

	}

	dbp->db_mblk->b_datap = dbp;

	dbp->db_cache = (void *)bcp;

	dbp->db_lim = dbp->db_base + bcp->size;

	dbp->db_free = dbp->db_lastfree = bcache_dblk_lastfree;

	dbp->db_frtnp = NULL;

	dbp->db_fthdr = NULL;

	dbp->db_credp = NULL;

	dbp->db_cpid = -1;

	dbp->db_struioflag = 0;

	dbp->db_struioun.cksum.flags = 0;

	return (0);

}

/*ARGSUSED*/

static void

dblk_destructor(void *buf, void *cdrarg)

{

	dblk_t *dbp = buf;

	ssize_t msg_size = (ssize_t)cdrarg;

	ASSERT(dbp->db_mblk->b_datap == dbp);

	ASSERT(msg_size != 0);

	ASSERT(dbp->db_struioflag == 0);

	ASSERT(dbp->db_struioun.cksum.flags == 0);

	if ((msg_size & PAGEOFFSET) == 0) {

		kmem_free(dbp->db_base, msg_size);

	}

	kmem_cache_free(mblk_cache, dbp->db_mblk);

}

static void

bcache_dblk_destructor(void *buf, void *cdrarg)

{

	dblk_t *dbp = buf;

	bcache_t *bcp = (bcache_t *)cdrarg;

	kmem_cache_free(bcp->buffer_cache, dbp->db_base);

	ASSERT(dbp->db_mblk->b_datap == dbp);

	ASSERT(dbp->db_struioflag == 0);

	ASSERT(dbp->db_struioun.cksum.flags == 0);

	kmem_cache_free(mblk_cache, dbp->db_mblk);

}

void

streams_msg_init(void)

{

	char name[40];

	size_t size;

	size_t lastsize = DBLK_MIN_SIZE;

	size_t *sizep;

	struct kmem_cache *cp;

	size_t tot_size;

	int offset;

	mblk_cache = kmem_cache_create("streams_mblk",

		sizeof (mblk_t), 32, NULL, NULL, NULL, NULL, NULL,

		mblk_kmem_flags);

	for (sizep = dblk_sizes; (size = *sizep) != 0; sizep++) {

		if ((offset = (size & PAGEOFFSET)) != 0) {

			/*

			 * We are in the middle of a page, dblk should

			 * be allocated on the same page

			 */

			tot_size = size + sizeof (dblk_t);

			ASSERT((offset + sizeof (dblk_t) + sizeof (kmem_slab_t))

								< PAGESIZE);

			ASSERT((tot_size & (DBLK_CACHE_ALIGN - 1)) == 0);

		} else {

			/*

			 * buf size is multiple of page size, dblk and

			 * buffer are allocated separately.

			 */

			ASSERT((size & (DBLK_CACHE_ALIGN - 1)) == 0);

			tot_size = sizeof (dblk_t);

		}

		(void) sprintf(name, "streams_dblk_%ld", size);

		cp = kmem_cache_create(name, tot_size,

			DBLK_CACHE_ALIGN, dblk_constructor,

			dblk_destructor, NULL,

			(void *)(size), NULL, dblk_kmem_flags);

		while (lastsize <= size) {

			dblk_cache[(lastsize - 1) >> DBLK_SIZE_SHIFT] = cp;

			lastsize += DBLK_MIN_SIZE;

		}

	}

	dblk_esb_cache = kmem_cache_create("streams_dblk_esb",

			sizeof (dblk_t), DBLK_CACHE_ALIGN,

			dblk_esb_constructor, dblk_destructor, NULL,

			(void *) sizeof (dblk_t), NULL, dblk_kmem_flags);

	fthdr_cache = kmem_cache_create("streams_fthdr",

		sizeof (fthdr_t), 32, NULL, NULL, NULL, NULL, NULL, 0);

	ftblk_cache = kmem_cache_create("streams_ftblk",

		sizeof (ftblk_t), 32, NULL, NULL, NULL, NULL, NULL, 0);

	/* Initialize Multidata caches */

	mmd_init();

}

/*ARGSUSED*/

mblk_t *

allocb(size_t size, uint_t pri)

{

	dblk_t *dbp;

	mblk_t *mp;

	size_t index;

	index =  (size - 1)  >> DBLK_SIZE_SHIFT;

	if (index >= (DBLK_MAX_CACHE >> DBLK_SIZE_SHIFT)) {

		if (size != 0) {

			mp = allocb_oversize(size, KM_NOSLEEP);

			goto out;

		}

		index = 0;

	}

	if ((dbp = kmem_cache_alloc(dblk_cache[index], KM_NOSLEEP)) == NULL) {

		mp = NULL;

		goto out;

	}

	mp = dbp->db_mblk;

	DBLK_RTFU_WORD(dbp) = DBLK_RTFU(1, M_DATA, 0, 0);

	mp->b_next = mp->b_prev = mp->b_cont = NULL;

	mp->b_rptr = mp->b_wptr = dbp->db_base;

	mp->b_queue = NULL;

	MBLK_BAND_FLAG_WORD(mp) = 0;

	STR_FTALLOC(&dbp->db_fthdr, FTEV_ALLOCB, size);

out:

	FTRACE_1("allocb(): mp=0x%p", (uintptr_t)mp);

	return (mp);

}

mblk_t *

allocb_tmpl(size_t size, const mblk_t *tmpl)

{

	mblk_t *mp = allocb(size, 0);

	if (mp != NULL) {

		cred_t *cr = DB_CRED(tmpl);

		if (cr != NULL)

			crhold(mp->b_datap->db_credp = cr);

		DB_CPID(mp) = DB_CPID(tmpl);

		DB_TYPE(mp) = DB_TYPE(tmpl);

	}

	return (mp);

}

mblk_t *

allocb_cred(size_t size, cred_t *cr)

{

	mblk_t *mp = allocb(size, 0);

	if (mp != NULL && cr != NULL)

		crhold(mp->b_datap->db_credp = cr);

	return (mp);

}

mblk_t *

allocb_cred_wait(size_t size, uint_t flags, int *error, cred_t *cr)

{

	mblk_t *mp = allocb_wait(size, 0, flags, error);

	if (mp != NULL && cr != NULL)

		crhold(mp->b_datap->db_credp = cr);

	return (mp);

}

void

freeb(mblk_t *mp)

{

	dblk_t *dbp = mp->b_datap;

	ASSERT(dbp->db_ref > 0);

	ASSERT(mp->b_next == NULL && mp->b_prev == NULL);

	FTRACE_1("freeb(): mp=0x%lx", (uintptr_t)mp);

	STR_FTEVENT_MBLK(mp, caller(), FTEV_FREEB, dbp->db_ref);

	dbp->db_free(mp, dbp);

}

void

freemsg(mblk_t *mp)

{

	FTRACE_1("freemsg(): mp=0x%lx", (uintptr_t)mp);

	while (mp) {

		dblk_t *dbp = mp->b_datap;

		mblk_t *mp_cont = mp->b_cont;

		ASSERT(dbp->db_ref > 0);

		ASSERT(mp->b_next == NULL && mp->b_prev == NULL);

		STR_FTEVENT_MBLK(mp, caller(), FTEV_FREEB, dbp->db_ref);

		dbp->db_free(mp, dbp);

		mp = mp_cont;

	}

}

/*

 * Reallocate a block for another use.  Try hard to use the old block.

 * If the old data is wanted (copy), leave b_wptr at the end of the data,

 * otherwise return b_wptr = b_rptr.

 *

 * This routine is private and unstable.

 */

mblk_t	*

reallocb(mblk_t *mp, size_t size, uint_t copy)

{

	mblk_t		*mp1;

	unsigned char	*old_rptr;

	ptrdiff_t	cur_size;

	if (mp == NULL)

		return (allocb(size, BPRI_HI));

	cur_size = mp->b_wptr - mp->b_rptr;

	old_rptr = mp->b_rptr;

	ASSERT(mp->b_datap->db_ref != 0);

	if (mp->b_datap->db_ref == 1 && MBLKSIZE(mp) >= size) {

		/*

		 * If the data is wanted and it will fit where it is, no

		 * work is required.

		 */

		if (copy && mp->b_datap->db_lim - mp->b_rptr >= size)

			return (mp);

		mp->b_wptr = mp->b_rptr = mp->b_datap->db_base;

		mp1 = mp;

	} else if ((mp1 = allocb_tmpl(size, mp)) != NULL) {

		/* XXX other mp state could be copied too, db_flags ... ? */

		mp1->b_cont = mp->b_cont;

	} else {

		return (NULL);

	}

	if (copy) {

		bcopy(old_rptr, mp1->b_rptr, cur_size);

		mp1->b_wptr = mp1->b_rptr + cur_size;

	}

	if (mp != mp1)

		freeb(mp);

	return (mp1);

}

static void

dblk_lastfree(mblk_t *mp, dblk_t *dbp)