/*

 * CDDL HEADER START

 *

 * The contents of this file are subject to the terms of the

 * Common Development and Distribution License, Version 1.0 only

 * (the "License").  You may not use this file except in compliance

 * with the License.

 *

 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

 * or http://www.opensolaris.org/os/licensing.

 * See the License for the specific language governing permissions

 * and limitations under the License.

 *

 * When distributing Covered Code, include this CDDL HEADER in each

 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.

 * If applicable, add the following below this CDDL HEADER, with the

 * fields enclosed by brackets "[]" replaced with your own identifying

 * information: Portions Copyright [yyyy] [name of copyright owner]

 *

 * CDDL HEADER END

 */

/*

 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.

 * Use is subject to license terms.

 */

/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/

/*	  All Rights Reserved  	*/

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#include <sys/types.h>

#include <sys/param.h>

#include <sys/sysmacros.h>

#include <sys/signal.h>

#include <sys/cred.h>

#include <sys/policy.h>

#include <sys/user.h>

#include <sys/systm.h>

#include <sys/cpuvar.h>

#include <sys/vfs.h>

#include <sys/vnode.h>

#include <sys/file.h>

#include <sys/errno.h>

#include <sys/time.h>

#include <sys/proc.h>

#include <sys/cmn_err.h>

#include <sys/acct.h>

#include <sys/tuneable.h>

#include <sys/class.h>

#include <sys/kmem.h>

#include <sys/session.h>

#include <sys/ucontext.h>

#include <sys/stack.h>

#include <sys/procfs.h>

#include <sys/prsystm.h>

#include <sys/vmsystm.h>

#include <sys/vtrace.h>

#include <sys/debug.h>

#include <sys/shm_impl.h>

#include <sys/door_data.h>

#include <vm/as.h>

#include <vm/rm.h>

#include <c2/audit.h>

#include <sys/var.h>

#include <sys/schedctl.h>

#include <sys/utrap.h>

#include <sys/task.h>

#include <sys/resource.h>

#include <sys/cyclic.h>

#include <sys/lgrp.h>

#include <sys/rctl.h>

#include <sys/contract_impl.h>

#include <sys/contract/process_impl.h>

#include <sys/list.h>

#include <sys/dtrace.h>

#include <sys/pool.h>

#include <sys/zone.h>

#include <sys/sdt.h>

#include <sys/class.h>

#include <sys/corectl.h>

static int64_t cfork(int, int);

static int getproc(proc_t **, int);

static void fork_fail(proc_t *);

static void forklwp_fail(proc_t *);

int fork_fail_pending;

extern struct kmem_cache *process_cache;

/*

 * forkall system call.

 */

int64_t

forkall(void)

{

	return (cfork(0, 0));

}

/*

 * The parent is stopped until the child invokes relvm().

 */

int64_t

vfork(void)

{

	curthread->t_post_sys = 1;	/* so vfwait() will be called */

	return (cfork(1, 1));

}

/*

 * fork1 system call

 */

int64_t

fork1(void)

{

	return (cfork(0, 1));

}

/* ARGSUSED */

static int64_t

cfork(int isvfork, int isfork1)

{

	proc_t *p = ttoproc(curthread);

	struct as *as;

	proc_t *cp, **orphpp;

	klwp_t *clone;

	kthread_t *t;

	task_t *tk;

	rval_t	r;

	int error;

	int i;

	rctl_set_t *dup_set;

	rctl_alloc_gp_t *dup_gp;

	rctl_entity_p_t e;

	lwpdir_t *ldp;

	lwpent_t *lep;

	lwpent_t *clep;

	/*

	 * fork is not supported for the /proc agent lwp.

	 */

	if (curthread == p->p_agenttp) {

		error = ENOTSUP;

		goto forkerr;

	}

	if ((error = secpolicy_basic_fork(CRED())) != 0)

		goto forkerr;

	/*

	 * If the calling lwp is doing a fork1() then the

	 * other lwps in this process are not duplicated and

	 * don't need to be held where their kernel stacks can be

	 * cloned.  If doing forkall(), the process is held with

	 * SHOLDFORK, so that the lwps are at a point where their

	 * stacks can be copied which is on entry or exit from

	 * the kernel.

	 */

	if (!holdlwps(isfork1 ? SHOLDFORK1 : SHOLDFORK)) {

		aston(curthread);

		error = EINTR;

		goto forkerr;

	}

#if defined(__sparc)

	/*

	 * Ensure that the user stack is fully constructed

	 * before creating the child process structure.

	 */

	(void) flush_user_windows_to_stack(NULL);

#endif

	mutex_enter(&p->p_lock);

	/*

	 * If this is vfork(), cancel any suspend request we might

	 * have gotten from some other thread via lwp_suspend().

	 * Otherwise we could end up with a deadlock on return

	 * from the vfork() in both the parent and the child.

	 */

	if (isvfork)

		curthread->t_proc_flag &= ~TP_HOLDLWP;

	/*

	 * Prevent our resource set associations from being changed during fork.

	 */

	pool_barrier_enter();

	mutex_exit(&p->p_lock);

	/*

	 * Create a child proc struct. Place a VN_HOLD on appropriate vnodes.

	 */

	if (getproc(&cp, 0) < 0) {

		mutex_enter(&p->p_lock);

		pool_barrier_exit();

		continuelwps(p);

		mutex_exit(&p->p_lock);

		error = EAGAIN;

		goto forkerr;

	}

	TRACE_2(TR_FAC_PROC, TR_PROC_FORK, "proc_fork:cp %p p %p", cp, p);

	/*

	 * Assign an address space to child

	 */

	if (isvfork) {

		/*

		 * Clear any watched areas and remember the

		 * watched pages for restoring in vfwait().

		 */

		as = p->p_as;

		if (avl_numnodes(&as->a_wpage) != 0) {

			AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);

			as_clearwatch(as);

			p->p_wpage = as->a_wpage;

			avl_create(&as->a_wpage, wp_compare,

			    sizeof (struct watched_page),

			    offsetof(struct watched_page, wp_link));

			AS_LOCK_EXIT(as, &as->a_lock);

		}

		cp->p_as = as;

		cp->p_flag |= SVFORK;

	} else {

		/*

		 * We need to hold P_PR_LOCK until the address space has

		 * been duplicated and we've had a chance to remove from the

		 * child any DTrace probes that were in the parent. Holding

		 * P_PR_LOCK prevents any new probes from being added and any

		 * extant probes from being removed.

		 */

		mutex_enter(&p->p_lock);

		sprlock_proc(p);

		mutex_exit(&p->p_lock);

		error = as_dup(p->p_as, &cp->p_as);

		if (error != 0) {

			fork_fail(cp);

			mutex_enter(&pidlock);

			orphpp = &p->p_orphan;

			while (*orphpp != cp)

				orphpp = &(*orphpp)->p_nextorph;

			*orphpp = cp->p_nextorph;

			ASSERT(p->p_child == cp);

			p->p_child = cp->p_sibling;

			if (p->p_child) {

				p->p_child->p_psibling = NULL;

			}

			mutex_enter(&cp->p_lock);

			tk = cp->p_task;

			task_detach(cp);

			ASSERT(cp->p_pool->pool_ref > 0);

			atomic_add_32(&cp->p_pool->pool_ref, -1);

			mutex_exit(&cp->p_lock);

			pid_exit(cp);

			mutex_exit(&pidlock);

			task_rele(tk);

			mutex_enter(&p->p_lock);

			pool_barrier_exit();

			continuelwps(p);

			sprunlock(p);

			/*

			 * Preserve ENOMEM error condition but

			 * map all others to EAGAIN.

			 */

			error = (error == ENOMEM) ? ENOMEM : EAGAIN;

			goto forkerr;

		}

		/* Duplicate parent's shared memory */

		if (p->p_segacct)

			shmfork(p, cp);

		if (p->p_dtrace_helpers != NULL) {

			ASSERT(dtrace_helpers_fork != NULL);

			(*dtrace_helpers_fork)(p, cp);

		}

		/*

		 * Remove all DTrace tracepoints from the child process.

		 */

		mutex_enter(&p->p_lock);

		if (p->p_dtrace_count > 0)

			dtrace_fasttrap_fork(p, cp);

		sprunlock(p);

	}

	/*

	 * Duplicate parent's resource controls.

	 */

	dup_set = rctl_set_create();

	for (;;) {

		dup_gp = rctl_set_dup_prealloc(p->p_rctls);

		mutex_enter(&p->p_rctls->rcs_lock);

		if (rctl_set_dup_ready(p->p_rctls, dup_gp))

			break;

		mutex_exit(&p->p_rctls->rcs_lock);

		rctl_prealloc_destroy(dup_gp);

	}

	e.rcep_p.proc = cp;

	e.rcep_t = RCENTITY_PROCESS;

	cp->p_rctls = rctl_set_dup(p->p_rctls, p, cp, &e, dup_set, dup_gp,

	    RCD_DUP | RCD_CALLBACK);

	mutex_exit(&p->p_rctls->rcs_lock);

	rctl_prealloc_destroy(dup_gp);

	/*

	 * Allocate the child's lwp directory and lwpid hash table.

	 */

	if (isfork1)

		cp->p_lwpdir_sz = 2;

	else

		cp->p_lwpdir_sz = p->p_lwpdir_sz;

	cp->p_lwpdir = cp->p_lwpfree = ldp =

		kmem_zalloc(cp->p_lwpdir_sz * sizeof (lwpdir_t), KM_SLEEP);

	for (i = 1; i < cp->p_lwpdir_sz; i++, ldp++)

		ldp->ld_next = ldp + 1;

	cp->p_tidhash_sz = (cp->p_lwpdir_sz + 2) / 2;

	cp->p_tidhash =

		kmem_zalloc(cp->p_tidhash_sz * sizeof (lwpdir_t *), KM_SLEEP);

	/*

	 * Duplicate parent's lwps.

	 * Mutual exclusion is not needed because the process is

	 * in the hold state and only the current lwp is running.

	 */

	klgrpset_clear(cp->p_lgrpset);

	if (isfork1) {

		clone = forklwp(ttolwp(curthread), cp, curthread->t_tid);

		if (clone == NULL)

			goto forklwperr;

		/*

		 * Inherit only the lwp_wait()able flag,

		 * Daemon threads should not call fork1(), but oh well...

		 */

		lwptot(clone)->t_proc_flag |=

			(curthread->t_proc_flag & TP_TWAIT);

	} else {

		/* this is forkall(), no one can be in lwp_wait() */

		ASSERT(p->p_lwpwait == 0 && p->p_lwpdwait == 0);

		/* for each entry in the parent's lwp directory... */

		for (i = 0, ldp = p->p_lwpdir; i < p->p_lwpdir_sz; i++, ldp++) {

			klwp_t *clwp;

			kthread_t *ct;

			if ((lep = ldp->ld_entry) == NULL)

				continue;

			if ((t = lep->le_thread) != NULL) {

				clwp = forklwp(ttolwp(t), cp, t->t_tid);

				if (clwp == NULL)

					goto forklwperr;

				ct = lwptot(clwp);

				/*

				 * Inherit lwp_wait()able and daemon flags.

				 */

				ct->t_proc_flag |=

				    (t->t_proc_flag & (TP_TWAIT|TP_DAEMON));

				/*

				 * Keep track of the clone of curthread to

				 * post return values through lwp_setrval().

				 * Mark other threads for special treatment

				 * by lwp_rtt() / post_syscall().

				 */

				if (t == curthread)

					clone = clwp;

				else

					ct->t_flag |= T_FORKALL;

			} else {

				/*

				 * Replicate zombie lwps in the child.

				 */

				clep = kmem_zalloc(sizeof (*clep), KM_SLEEP);

				clep->le_lwpid = lep->le_lwpid;

				clep->le_start = lep->le_start;

				lwp_hash_in(cp, clep);

			}

		}

	}

	/*

	 * Put new process in the parent's process contract, or put it

	 * in a new one if there is an active process template.  Send a

	 * fork event (if requested) to whatever contract the child is

	 * a member of.  Fails if the parent has been SIGKILLed.

	 */

	if (contract_process_fork(NULL, cp, p, B_TRUE) == NULL)

		goto forklwperr;

	/*

	 * No fork failures occur beyond this point.

	 */

	cp->p_lwpid = p->p_lwpid;

	if (!isfork1) {

		cp->p_lwpdaemon = p->p_lwpdaemon;

		cp->p_zombcnt = p->p_zombcnt;

		/*

		 * If the parent's lwp ids have wrapped around, so have the

		 * child's.

		 */

		cp->p_flag |= p->p_flag & SLWPWRAP;

	}

	corectl_path_hold(cp->p_corefile = p->p_corefile);

	corectl_content_hold(cp->p_content = p->p_content);

	/*

	 * Duplicate process context ops, if any.

	 */

	if (p->p_pctx)

		forkpctx(p, cp);

#ifdef __sparc

	utrap_dup(p, cp);

#endif

	/*

	 * If the child process has been marked to stop on exit

	 * from this fork, arrange for all other lwps to stop in

	 * sympathy with the active lwp.

	 */

	if (PTOU(cp)->u_systrap &&

	    prismember(&PTOU(cp)->u_exitmask, curthread->t_sysnum)) {

		mutex_enter(&cp->p_lock);

		t = cp->p_tlist;

		do {

			t->t_proc_flag |= TP_PRSTOP;

			aston(t);	/* so TP_PRSTOP will be seen */

		} while ((t = t->t_forw) != cp->p_tlist);

		mutex_exit(&cp->p_lock);

	}

	/*

	 * If the parent process has been marked to stop on exit

	 * from this fork, and its asynchronous-stop flag has not

	 * been set, arrange for all other lwps to stop before

	 * they return back to user level.

	 */

	if (!(p->p_proc_flag & P_PR_ASYNC) && PTOU(p)->u_systrap &&

	    prismember(&PTOU(p)->u_exitmask, curthread->t_sysnum)) {

		mutex_enter(&p->p_lock);

		t = p->p_tlist;

		do {

			t->t_proc_flag |= TP_PRSTOP;

			aston(t);	/* so TP_PRSTOP will be seen */

		} while ((t = t->t_forw) != p->p_tlist);

		mutex_exit(&p->p_lock);

	}

	/* set return values for child */

	lwp_setrval(clone, p->p_pid, 1);

	/* set return values for parent */

	r.r_val1 = (int)cp->p_pid;

	r.r_val2 = 0;

	/*

	 * pool_barrier_exit() can now be called because the child process has:

	 * - all identifying features cloned or set (p_pid, p_task, p_pool)

	 * - all resource sets associated (p_tlist->*->t_cpupart, p_as->a_mset)

	 * - any other fields set which are used in resource set binding.

	 */

	mutex_enter(&p->p_lock);

	pool_barrier_exit();

	mutex_exit(&p->p_lock);

	mutex_enter(&pidlock);

	mutex_enter(&cp->p_lock);

	/*

	 * Now that there are lwps and threads attached, add the new

	 * process to the process group.

	 */

	pgjoin(cp, p->p_pgidp);

	cp->p_stat = SRUN;

	/*

	 * We are now done with all the lwps in the child process.

	 */

	t = cp->p_tlist;

	do {

		/*

		 * Set the lwp_suspend()ed lwps running.

		 * They will suspend properly at syscall exit.

		 */

		if (t->t_proc_flag & TP_HOLDLWP)

			lwp_create_done(t);

		else {

			/* set TS_CREATE to allow continuelwps() to work */

			thread_lock(t);

			ASSERT(t->t_state == TS_STOPPED &&

			    !(t->t_schedflag & (TS_CREATE|TS_CSTART)));

			t->t_schedflag |= TS_CREATE;

			thread_unlock(t);

		}

	} while ((t = t->t_forw) != cp->p_tlist);

	mutex_exit(&cp->p_lock);

	if (isvfork) {

		CPU_STATS_ADDQ(CPU, sys, sysvfork, 1);

		mutex_enter(&p->p_lock);

		p->p_flag |= SVFWAIT;

		DTRACE_PROC1(create, proc_t *, cp);

		cv_broadcast(&pr_pid_cv[p->p_slot]);	/* inform /proc */

		mutex_exit(&p->p_lock);

		/*

		 * Grab child's p_lock before dropping pidlock to ensure

		 * the process will not disappear before we set it running.

		 */

		mutex_enter(&cp->p_lock);

		mutex_exit(&pidlock);

		sigdefault(cp);

		continuelwps(cp);

		mutex_exit(&cp->p_lock);

	} else {

		CPU_STATS_ADDQ(CPU, sys, sysfork, 1);

		DTRACE_PROC1(create, proc_t *, cp);

		/*

		 * It is CL_FORKRET's job to drop pidlock.

		 * If we do it here, the process could be set running

		 * and disappear before CL_FORKRET() is called.

		 */

		CL_FORKRET(curthread, cp->p_tlist);

		ASSERT(MUTEX_NOT_HELD(&pidlock));

	}