/*

 * CDDL HEADER START

 *

 * The contents of this file are subject to the terms of the

 * Common Development and Distribution License (the "License").

 * You may not use this file except in compliance with the License.

 *

 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

 * or http://www.opensolaris.org/os/licensing.

 * See the License for the specific language governing permissions

 * and limitations under the License.

 *

 * When distributing Covered Code, include this CDDL HEADER in each

 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.

 * If applicable, add the following below this CDDL HEADER, with the

 * fields enclosed by brackets "[]" replaced with your own identifying

 * information: Portions Copyright [yyyy] [name of copyright owner]

 *

 * CDDL HEADER END

 */

/*

 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.

 */

#include <sys/types.h>

#include <sys/param.h>

#include <sys/sysmacros.h>

#include <sys/signal.h>

#include <sys/stack.h>

#include <sys/pcb.h>

#include <sys/user.h>

#include <sys/systm.h>

#include <sys/sysinfo.h>

#include <sys/errno.h>

#include <sys/cmn_err.h>

#include <sys/cred.h>

#include <sys/resource.h>

#include <sys/task.h>

#include <sys/project.h>

#include <sys/proc.h>

#include <sys/debug.h>

#include <sys/disp.h>

#include <sys/class.h>

#include <vm/seg_kmem.h>

#include <vm/seg_kp.h>

#include <sys/machlock.h>

#include <sys/kmem.h>

#include <sys/varargs.h>

#include <sys/turnstile.h>

#include <sys/poll.h>

#include <sys/vtrace.h>

#include <sys/callb.h>

#include <c2/audit.h>

#include <sys/tnf.h>

#include <sys/sobject.h>

#include <sys/cpupart.h>

#include <sys/pset.h>

#include <sys/door.h>

#include <sys/spl.h>

#include <sys/copyops.h>

#include <sys/rctl.h>

#include <sys/brand.h>

#include <sys/pool.h>

#include <sys/zone.h>

#include <sys/tsol/label.h>

#include <sys/tsol/tndb.h>

#include <sys/cpc_impl.h>

#include <sys/sdt.h>

#include <sys/reboot.h>

#include <sys/kdi.h>

#include <sys/schedctl.h>

#include <sys/waitq.h>

#include <sys/cpucaps.h>

#include <sys/kiconv.h>

struct kmem_cache *thread_cache;	/* cache of free threads */

struct kmem_cache *lwp_cache;		/* cache of free lwps */

struct kmem_cache *turnstile_cache;	/* cache of free turnstiles */

/*

 * allthreads is only for use by kmem_readers.  All kernel loops can use

 * the current thread as a start/end point.

 */

static kthread_t *allthreads = &t0;	/* circular list of all threads */

static kcondvar_t reaper_cv;		/* synchronization var */

kthread_t	*thread_deathrow;	/* circular list of reapable threads */

kthread_t	*lwp_deathrow;		/* circular list of reapable threads */

kmutex_t	reaplock;		/* protects lwp and thread deathrows */

int	thread_reapcnt = 0;		/* number of threads on deathrow */

int	lwp_reapcnt = 0;		/* number of lwps on deathrow */

int	reaplimit = 16;			/* delay reaping until reaplimit */

thread_free_lock_t	*thread_free_lock;

					/* protects tick thread from reaper */

extern int nthread;

/* System Scheduling classes. */

id_t	syscid;				/* system scheduling class ID */

id_t	sysdccid = CLASS_UNUSED;	/* reset when SDC loads */

void	*segkp_thread;			/* cookie for segkp pool */

int lwp_cache_sz = 32;

int t_cache_sz = 8;

static kt_did_t next_t_id = 1;

/* Default mode for thread binding to CPUs and processor sets */

int default_binding_mode = TB_ALLHARD;

/*

 * Min/Max stack sizes for stack size parameters

 */

#define	MAX_STKSIZE	(32 * DEFAULTSTKSZ)

#define	MIN_STKSIZE	DEFAULTSTKSZ

/*

 * default_stksize overrides lwp_default_stksize if it is set.

 */

int	default_stksize;

int	lwp_default_stksize;

static zone_key_t zone_thread_key;

unsigned int kmem_stackinfo;		/* stackinfo feature on-off */

kmem_stkinfo_t *kmem_stkinfo_log;	/* stackinfo circular log */

static kmutex_t kmem_stkinfo_lock;	/* protects kmem_stkinfo_log */

/*

 * forward declarations for internal thread specific data (tsd)

 */

static void *tsd_realloc(void *, size_t, size_t);

void thread_reaper(void);

/* forward declarations for stackinfo feature */

static void stkinfo_begin(kthread_t *);

static void stkinfo_end(kthread_t *);

static size_t stkinfo_percent(caddr_t, caddr_t, caddr_t);

/*ARGSUSED*/

static int

turnstile_constructor(void *buf, void *cdrarg, int kmflags)

{

	bzero(buf, sizeof (turnstile_t));

	return (0);

}

/*ARGSUSED*/

static void

turnstile_destructor(void *buf, void *cdrarg)

{

	turnstile_t *ts = buf;

	ASSERT(ts->ts_free == NULL);

	ASSERT(ts->ts_waiters == 0);

	ASSERT(ts->ts_inheritor == NULL);

	ASSERT(ts->ts_sleepq[0].sq_first == NULL);

	ASSERT(ts->ts_sleepq[1].sq_first == NULL);

}

void

thread_init(void)

{

	kthread_t *tp;

	extern char sys_name[];

	extern void idle();

	struct cpu *cpu = CPU;

	int i;

	kmutex_t *lp;

	mutex_init(&reaplock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL));

	thread_free_lock =

	    kmem_alloc(sizeof (thread_free_lock_t) * THREAD_FREE_NUM, KM_SLEEP);

	for (i = 0; i < THREAD_FREE_NUM; i++) {

		lp = &thread_free_lock[i].tf_lock;

		mutex_init(lp, NULL, MUTEX_DEFAULT, NULL);

	}

#if defined(__i386) || defined(__amd64)

	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),

	    PTR24_ALIGN, NULL, NULL, NULL, NULL, NULL, 0);

	/*

	 * "struct _klwp" includes a "struct pcb", which includes a

	 * "struct fpu", which needs to be 64-byte aligned on amd64

	 * (and even on i386) for xsave/xrstor.

	 */

	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),

	    64, NULL, NULL, NULL, NULL, NULL, 0);

#else

	/*

	 * Allocate thread structures from static_arena.  This prevents

	 * issues where a thread tries to relocate its own thread

	 * structure and touches it after the mapping has been suspended.

	 */

	thread_cache = kmem_cache_create("thread_cache", sizeof (kthread_t),

	    PTR24_ALIGN, NULL, NULL, NULL, NULL, static_arena, 0);

	lwp_stk_cache_init();

	lwp_cache = kmem_cache_create("lwp_cache", sizeof (klwp_t),

	    0, NULL, NULL, NULL, NULL, NULL, 0);

#endif

	turnstile_cache = kmem_cache_create("turnstile_cache",

	    sizeof (turnstile_t), 0,

	    turnstile_constructor, turnstile_destructor, NULL, NULL, NULL, 0);

	label_init();

	cred_init();

	/*

	 * Initialize various resource management facilities.

	 */

	rctl_init();

	cpucaps_init();

	/*

	 * Zone_init() should be called before project_init() so that project ID

	 * for the first project is initialized correctly.

	 */

	zone_init();

	project_init();

	brand_init();

	kiconv_init();

	task_init();

	tcache_init();

	pool_init();

	curthread->t_ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);

	/*

	 * Originally, we had two parameters to set default stack

	 * size: one for lwp's (lwp_default_stksize), and one for

	 * kernel-only threads (DEFAULTSTKSZ, a.k.a. _defaultstksz).

	 * Now we have a third parameter that overrides both if it is

	 * set to a legal stack size, called default_stksize.

	 */

	if (default_stksize == 0) {

		default_stksize = DEFAULTSTKSZ;

	} else if (default_stksize % PAGESIZE != 0 ||

	    default_stksize > MAX_STKSIZE ||

	    default_stksize < MIN_STKSIZE) {

		cmn_err(CE_WARN, "Illegal stack size. Using %d",

		    (int)DEFAULTSTKSZ);

		default_stksize = DEFAULTSTKSZ;

	} else {

		lwp_default_stksize = default_stksize;

	}

	if (lwp_default_stksize == 0) {

		lwp_default_stksize = default_stksize;

	} else if (lwp_default_stksize % PAGESIZE != 0 ||

	    lwp_default_stksize > MAX_STKSIZE ||

	    lwp_default_stksize < MIN_STKSIZE) {

		cmn_err(CE_WARN, "Illegal stack size. Using %d",

		    default_stksize);

		lwp_default_stksize = default_stksize;

	}

	segkp_lwp = segkp_cache_init(segkp, lwp_cache_sz,

	    lwp_default_stksize,

	    (KPD_NOWAIT | KPD_HASREDZONE | KPD_LOCKED));

	segkp_thread = segkp_cache_init(segkp, t_cache_sz,

	    default_stksize, KPD_HASREDZONE | KPD_LOCKED | KPD_NO_ANON);

	(void) getcid(sys_name, &syscid);

	curthread->t_cid = syscid;	/* current thread is t0 */

	/*

	 * Set up the first CPU's idle thread.

	 * It runs whenever the CPU has nothing worthwhile to do.

	 */

	tp = thread_create(NULL, 0, idle, NULL, 0, &p0, TS_STOPPED, -1);

	cpu->cpu_idle_thread = tp;

	tp->t_preempt = 1;

	tp->t_disp_queue = cpu->cpu_disp;

	ASSERT(tp->t_disp_queue != NULL);

	tp->t_bound_cpu = cpu;

	tp->t_affinitycnt = 1;

	/*

	 * Registering a thread in the callback table is usually

	 * done in the initialization code of the thread. In this

	 * case, we do it right after thread creation to avoid

	 * blocking idle thread while registering itself. It also

	 * avoids the possibility of reregistration in case a CPU

	 * restarts its idle thread.

	 */

	CALLB_CPR_INIT_SAFE(tp, "idle");

	/*

	 * Create the thread_reaper daemon. From this point on, exited

	 * threads will get reaped.

	 */

	(void) thread_create(NULL, 0, (void (*)())thread_reaper,

	    NULL, 0, &p0, TS_RUN, minclsyspri);

	/*

	 * Finish initializing the kernel memory allocator now that

	 * thread_create() is available.

	 */

	kmem_thread_init();

	if (boothowto & RB_DEBUG)

		kdi_dvec_thravail();

}

/*

 * Create a thread.

 *

 * thread_create() blocks for memory if necessary.  It never fails.

 *

 * If stk is NULL, the thread is created at the base of the stack

 * and cannot be swapped.

 */

kthread_t *

thread_create(

	caddr_t	stk,

	size_t	stksize,

	void	(*proc)(),

	void	*arg,

	size_t	len,

	proc_t	 *pp,

	int	state,

	pri_t	pri)

{

	kthread_t *t;

	extern struct classfuncs sys_classfuncs;

	turnstile_t *ts;

	/*

	 * Every thread keeps a turnstile around in case it needs to block.

	 * The only reason the turnstile is not simply part of the thread

	 * structure is that we may have to break the association whenever

	 * more than one thread blocks on a given synchronization object.

	 * From a memory-management standpoint, turnstiles are like the

	 * "attached mblks" that hang off dblks in the streams allocator.

	 */

	ts = kmem_cache_alloc(turnstile_cache, KM_SLEEP);

	if (stk == NULL) {

		/*

		 * alloc both thread and stack in segkp chunk

		 */

		if (stksize < default_stksize)

			stksize = default_stksize;

		if (stksize == default_stksize) {

			stk = (caddr_t)segkp_cache_get(segkp_thread);

		} else {

			stksize = roundup(stksize, PAGESIZE);

			stk = (caddr_t)segkp_get(segkp, stksize,

			    (KPD_HASREDZONE | KPD_NO_ANON | KPD_LOCKED));

		}

		ASSERT(stk != NULL);

		/*

		 * The machine-dependent mutex code may require that

		 * thread pointers (since they may be used for mutex owner

		 * fields) have certain alignment requirements.

		 * PTR24_ALIGN is the size of the alignment quanta.

		 * XXX - assumes stack grows toward low addresses.

		 */

		if (stksize <= sizeof (kthread_t) + PTR24_ALIGN)

			cmn_err(CE_PANIC, "thread_create: proposed stack size"

			    " too small to hold thread.");

#ifdef STACK_GROWTH_DOWN

		stksize -= SA(sizeof (kthread_t) + PTR24_ALIGN - 1);

		stksize &= -PTR24_ALIGN;	/* make thread aligned */

		t = (kthread_t *)(stk + stksize);

		bzero(t, sizeof (kthread_t));

		if (audit_active)

			audit_thread_create(t);

		t->t_stk = stk + stksize;

		t->t_stkbase = stk;

#else	/* stack grows to larger addresses */

		stksize -= SA(sizeof (kthread_t));

		t = (kthread_t *)(stk);

		bzero(t, sizeof (kthread_t));

		t->t_stk = stk + sizeof (kthread_t);

		t->t_stkbase = stk + stksize + sizeof (kthread_t);

#endif	/* STACK_GROWTH_DOWN */

		t->t_flag |= T_TALLOCSTK;

		t->t_swap = stk;

	} else {

		t = kmem_cache_alloc(thread_cache, KM_SLEEP);

		bzero(t, sizeof (kthread_t));

		ASSERT(((uintptr_t)t & (PTR24_ALIGN - 1)) == 0);

		if (audit_active)

			audit_thread_create(t);

		/*

		 * Initialize t_stk to the kernel stack pointer to use

		 * upon entry to the kernel

		 */

#ifdef STACK_GROWTH_DOWN

		t->t_stk = stk + stksize;

		t->t_stkbase = stk;

#else

		t->t_stk = stk;			/* 3b2-like */

		t->t_stkbase = stk + stksize;

#endif /* STACK_GROWTH_DOWN */

	}

	if (kmem_stackinfo != 0) {

		stkinfo_begin(t);

	}

	t->t_ts = ts;

	/*

	 * p_cred could be NULL if it thread_create is called before cred_init

	 * is called in main.

	 */

	mutex_enter(&pp->p_crlock);

	if (pp->p_cred)

		crhold(t->t_cred = pp->p_cred);

	mutex_exit(&pp->p_crlock);

	t->t_start = gethrestime_sec();

	t->t_startpc = proc;

	t->t_procp = pp;

	t->t_clfuncs = &sys_classfuncs.thread;

	t->t_cid = syscid;

	t->t_pri = pri;

	t->t_stime = ddi_get_lbolt();

	t->t_schedflag = TS_LOAD | TS_DONT_SWAP;

	t->t_bind_cpu = PBIND_NONE;

	t->t_bindflag = (uchar_t)default_binding_mode;

	t->t_bind_pset = PS_NONE;

	t->t_plockp = &pp->p_lock;

	t->t_copyops = NULL;

	t->t_taskq = NULL;

	t->t_anttime = 0;

	t->t_hatdepth = 0;

	t->t_dtrace_vtime = 1;	/* assure vtimestamp is always non-zero */

	CPU_STATS_ADDQ(CPU, sys, nthreads, 1);

#ifndef NPROBE

	/* Kernel probe */

	tnf_thread_create(t);

#endif /* NPROBE */

	LOCK_INIT_CLEAR(&t->t_lock);

	/*

	 * Callers who give us a NULL proc must do their own

	 * stack initialization.  e.g. lwp_create()

	 */

	if (proc != NULL) {

		t->t_stk = thread_stk_init(t->t_stk);

		thread_load(t, proc, arg, len);

	}

	/*

	 * Put a hold on project0. If this thread is actually in a

	 * different project, then t_proj will be changed later in

	 * lwp_create().  All kernel-only threads must be in project 0.

	 */

	t->t_proj = project_hold(proj0p);

	lgrp_affinity_init(&t->t_lgrp_affinity);

	mutex_enter(&pidlock);

	nthread++;

	t->t_did = next_t_id++;

	t->t_prev = curthread->t_prev;

	t->t_next = curthread;

	/*

	 * Add the thread to the list of all threads, and initialize

	 * its t_cpu pointer.  We need to block preemption since

	 * cpu_offline walks the thread list looking for threads

	 * with t_cpu pointing to the CPU being offlined.  We want

	 * to make sure that the list is consistent and that if t_cpu

	 * is set, the thread is on the list.

	 */

	kpreempt_disable();

	curthread->t_prev->t_next = t;

	curthread->t_prev = t;

	/*

	 * Threads should never have a NULL t_cpu pointer so assign it

	 * here.  If the thread is being created with state TS_RUN a

	 * better CPU may be chosen when it is placed on the run queue.

	 *

	 * We need to keep kernel preemption disabled when setting all

	 * three fields to keep them in sync.  Also, always create in

	 * the default partition since that's where kernel threads go

	 * (if this isn't a kernel thread, t_cpupart will be changed

	 * in lwp_create before setting the thread runnable).

	 */

	t->t_cpupart = &cp_default;

	/*

	 * For now, affiliate this thread with the root lgroup.

	 * Since the kernel does not (presently) allocate its memory

	 * in a locality aware fashion, the root is an appropriate home.

	 * If this thread is later associated with an lwp, it will have

	 * it's lgroup re-assigned at that time.

	 */

	lgrp_move_thread(t, &cp_default.cp_lgrploads[LGRP_ROOTID], 1);

	/*

	 * Inherit the current cpu.  If this cpu isn't part of the chosen

	 * lgroup, a new cpu will be chosen by cpu_choose when the thread

	 * is ready to run.

	 */

	if (CPU->cpu_part == &cp_default)

		t->t_cpu = CPU;

	else

		t->t_cpu = disp_lowpri_cpu(cp_default.cp_cpulist, t->t_lpl,

		    t->t_pri, NULL);

	t->t_disp_queue = t->t_cpu->cpu_disp;