/*

 * CDDL HEADER START

 *

 * The contents of this file are subject to the terms of the

 * Common Development and Distribution License, Version 1.0 only

 * (the "License").  You may not use this file except in compliance

 * with the License.

 *

 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

 * or http://www.opensolaris.org/os/licensing.

 * See the License for the specific language governing permissions

 * and limitations under the License.

 *

 * When distributing Covered Code, include this CDDL HEADER in each

 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.

 * If applicable, add the following below this CDDL HEADER, with the

 * fields enclosed by brackets "[]" replaced with your own identifying

 * information: Portions Copyright [yyyy] [name of copyright owner]

 *

 * CDDL HEADER END

 */

/*

 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.

 * Use is subject to license terms.

 */

/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/

/*	  All Rights Reserved  	*/

/*

 * University Copyright- Copyright (c) 1982, 1986, 1988

 * The Regents of the University of California

 * All Rights Reserved

 *

 * University Acknowledgment- Portions of this document are derived from

 * software developed by the University of California, Berkeley, and its

 * contributors.

 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

/*

 * VM - shared or copy-on-write from a vnode/anonymous memory.

 */

#include <sys/types.h>

#include <sys/param.h>

#include <sys/t_lock.h>

#include <sys/errno.h>

#include <sys/systm.h>

#include <sys/mman.h>

#include <sys/debug.h>

#include <sys/cred.h>

#include <sys/vmsystm.h>

#include <sys/tuneable.h>

#include <sys/bitmap.h>

#include <sys/swap.h>

#include <sys/kmem.h>

#include <sys/sysmacros.h>

#include <sys/vtrace.h>

#include <sys/cmn_err.h>

#include <sys/vm.h>

#include <sys/dumphdr.h>

#include <sys/lgrp.h>

#include <vm/hat.h>

#include <vm/as.h>

#include <vm/seg.h>

#include <vm/seg_vn.h>

#include <vm/pvn.h>

#include <vm/anon.h>

#include <vm/page.h>

#include <vm/vpage.h>

/*

 * Private seg op routines.

 */

static int	segvn_dup(struct seg *seg, struct seg *newseg);

static int	segvn_unmap(struct seg *seg, caddr_t addr, size_t len);

static void	segvn_free(struct seg *seg);

static faultcode_t segvn_fault(struct hat *hat, struct seg *seg,

		    caddr_t addr, size_t len, enum fault_type type,

		    enum seg_rw rw);

static faultcode_t segvn_faulta(struct seg *seg, caddr_t addr);

static int	segvn_setprot(struct seg *seg, caddr_t addr,

		    size_t len, uint_t prot);

static int	segvn_checkprot(struct seg *seg, caddr_t addr,

		    size_t len, uint_t prot);

static int	segvn_kluster(struct seg *seg, caddr_t addr, ssize_t delta);

static size_t	segvn_swapout(struct seg *seg);

static int	segvn_sync(struct seg *seg, caddr_t addr, size_t len,

		    int attr, uint_t flags);

static size_t	segvn_incore(struct seg *seg, caddr_t addr, size_t len,

		    char *vec);

static int	segvn_lockop(struct seg *seg, caddr_t addr, size_t len,

		    int attr, int op, ulong_t *lockmap, size_t pos);

static int	segvn_getprot(struct seg *seg, caddr_t addr, size_t len,

		    uint_t *protv);

static u_offset_t	segvn_getoffset(struct seg *seg, caddr_t addr);

static int	segvn_gettype(struct seg *seg, caddr_t addr);

static int	segvn_getvp(struct seg *seg, caddr_t addr,

		    struct vnode **vpp);

static int	segvn_advise(struct seg *seg, caddr_t addr, size_t len,

		    uint_t behav);

static void	segvn_dump(struct seg *seg);

static int	segvn_pagelock(struct seg *seg, caddr_t addr, size_t len,

		    struct page ***ppp, enum lock_type type, enum seg_rw rw);

static int	segvn_setpagesize(struct seg *seg, caddr_t addr, size_t len,

		    uint_t szc);

static int	segvn_getmemid(struct seg *seg, caddr_t addr,

		    memid_t *memidp);

static lgrp_mem_policy_info_t	*segvn_getpolicy(struct seg *, caddr_t);

struct	seg_ops segvn_ops = {

	segvn_dup,

	segvn_unmap,

	segvn_free,

	segvn_fault,

	segvn_faulta,

	segvn_setprot,

	segvn_checkprot,

	segvn_kluster,

	segvn_swapout,

	segvn_sync,

	segvn_incore,

	segvn_lockop,

	segvn_getprot,

	segvn_getoffset,

	segvn_gettype,

	segvn_getvp,

	segvn_advise,

	segvn_dump,

	segvn_pagelock,

	segvn_setpagesize,

	segvn_getmemid,

	segvn_getpolicy,

};

/*

 * Common zfod structures, provided as a shorthand for others to use.

 */

static segvn_crargs_t zfod_segvn_crargs =

	SEGVN_ZFOD_ARGS(PROT_ZFOD, PROT_ALL);

static segvn_crargs_t kzfod_segvn_crargs =

	SEGVN_ZFOD_ARGS(PROT_ZFOD & ~PROT_USER,

	PROT_ALL & ~PROT_USER);

static segvn_crargs_t stack_noexec_crargs =

	SEGVN_ZFOD_ARGS(PROT_ZFOD & ~PROT_EXEC, PROT_ALL);

caddr_t	zfod_argsp = (caddr_t)&zfod_segvn_crargs;	/* user zfod argsp */

caddr_t	kzfod_argsp = (caddr_t)&kzfod_segvn_crargs;	/* kernel zfod argsp */

caddr_t	stack_exec_argsp = (caddr_t)&zfod_segvn_crargs;	/* executable stack */

caddr_t	stack_noexec_argsp = (caddr_t)&stack_noexec_crargs; /* noexec stack */

#define	vpgtob(n)	((n) * sizeof (struct vpage))	/* For brevity */

size_t	segvn_comb_thrshld = UINT_MAX;	/* patchable -- see 1196681 */

static int	segvn_concat(struct seg *, struct seg *, int);

static int	segvn_extend_prev(struct seg *, struct seg *,

		    struct segvn_crargs *, size_t);

static int	segvn_extend_next(struct seg *, struct seg *,

		    struct segvn_crargs *, size_t);

static void	segvn_softunlock(struct seg *, caddr_t, size_t, enum seg_rw);

static void	segvn_pagelist_rele(page_t **);

static void	segvn_setvnode_mpss(vnode_t *);

static void	segvn_relocate_pages(page_t **, page_t *);

static int	segvn_full_szcpages(page_t **, uint_t, int *, uint_t *);

static int	segvn_fill_vp_pages(struct segvn_data *, vnode_t *, u_offset_t,

    uint_t, page_t **, page_t **, uint_t *, int *);

static faultcode_t segvn_fault_vnodepages(struct hat *, struct seg *, caddr_t,

    caddr_t, enum fault_type, enum seg_rw, caddr_t, caddr_t, int);

static faultcode_t segvn_fault_anonpages(struct hat *, struct seg *, caddr_t,

    caddr_t, enum fault_type, enum seg_rw, caddr_t, caddr_t, int);

static faultcode_t segvn_faultpage(struct hat *, struct seg *, caddr_t,

    u_offset_t, struct vpage *, page_t **, uint_t,

    enum fault_type, enum seg_rw, int);

static void	segvn_vpage(struct seg *);

static void segvn_purge(struct seg *seg);

static int segvn_reclaim(struct seg *, caddr_t, size_t, struct page **,

    enum seg_rw);

static int sameprot(struct seg *, caddr_t, size_t);

static int segvn_demote_range(struct seg *, caddr_t, size_t, int);

static int segvn_clrszc(struct seg *);

static struct seg *segvn_split_seg(struct seg *, caddr_t);

static int segvn_claim_pages(struct seg *, struct vpage *, u_offset_t,

    ulong_t, uint_t);

static struct kmem_cache *segvn_cache;

#ifdef VM_STATS

static struct segvnvmstats_str {

	ulong_t	fill_vp_pages[31];

	ulong_t fltvnpages[49];

	ulong_t	fullszcpages[10];

	ulong_t	relocatepages[3];

	ulong_t	fltanpages[17];

	ulong_t pagelock[3];

	ulong_t	demoterange[3];

} segvnvmstats;

#endif /* VM_STATS */

#define	SDR_RANGE	1		/* demote entire range */

#define	SDR_END		2		/* demote non aligned ends only */

#define	CALC_LPG_REGION(pgsz, seg, addr, len, lpgaddr, lpgeaddr) {            \

		if ((len) != 0) { 		      	      		      \

			lpgaddr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);  \

			ASSERT(lpgaddr >= (seg)->s_base);	      	      \

			lpgeaddr = (caddr_t)P2ROUNDUP((uintptr_t)((addr) +    \

			    (len)), pgsz);				      \

			ASSERT(lpgeaddr > lpgaddr);		      	      \

			ASSERT(lpgeaddr <= (seg)->s_base + (seg)->s_size);    \

		} else {					      	      \

			lpgeaddr = lpgaddr = (addr);	      		      \

		}							      \

	}

/*ARGSUSED*/

static int

segvn_cache_constructor(void *buf, void *cdrarg, int kmflags)

{

	struct segvn_data *svd = buf;

	rw_init(&svd->lock, NULL, RW_DEFAULT, NULL);

	mutex_init(&svd->segp_slock, NULL, MUTEX_DEFAULT, NULL);

	return (0);

}

/*ARGSUSED1*/

static void

segvn_cache_destructor(void *buf, void *cdrarg)

{

	struct segvn_data *svd = buf;

	rw_destroy(&svd->lock);

	mutex_destroy(&svd->segp_slock);

}

/*

 * Patching this variable to non-zero allows the system to run with

 * stacks marked as "not executable".  It's a bit of a kludge, but is

 * provided as a tweakable for platforms that export those ABIs

 * (e.g. sparc V8) that have executable stacks enabled by default.

 * There are also some restrictions for platforms that don't actually

 * implement 'noexec' protections.

 *

 * Once enabled, the system is (therefore) unable to provide a fully

 * ABI-compliant execution environment, though practically speaking,

 * most everything works.  The exceptions are generally some interpreters

 * and debuggers that create executable code on the stack and jump

 * into it (without explicitly mprotecting the address range to include

 * PROT_EXEC).

 *

 * One important class of applications that are disabled are those

 * that have been transformed into malicious agents using one of the

 * numerous "buffer overflow" attacks.  See 4007890.

 */

int noexec_user_stack = 0;

int noexec_user_stack_log = 1;

int segvn_lpg_disable = 0;

uint_t segvn_maxpgszc = 0;

ulong_t segvn_vmpss_clrszc_cnt;

ulong_t segvn_vmpss_clrszc_err;

ulong_t segvn_fltvnpages_clrszc_cnt;

ulong_t segvn_fltvnpages_clrszc_err;

ulong_t segvn_setpgsz_align_err;

ulong_t segvn_setpgsz_getattr_err;

ulong_t segvn_setpgsz_eof_err;

ulong_t segvn_faultvnmpss_align_err1;

ulong_t segvn_faultvnmpss_align_err2;

ulong_t segvn_faultvnmpss_align_err3;

ulong_t segvn_faultvnmpss_align_err4;

ulong_t segvn_faultvnmpss_align_err5;

ulong_t	segvn_vmpss_pageio_deadlk_err;

/*

 * Initialize segvn data structures

 */

void

segvn_init(void)

{

	uint_t maxszc;

	uint_t szc;

	size_t pgsz;

	segvn_cache = kmem_cache_create("segvn_cache",

		sizeof (struct segvn_data), 0,

		segvn_cache_constructor, segvn_cache_destructor, NULL,

		NULL, NULL, 0);

	if (segvn_lpg_disable != 0)

		return;

	szc = maxszc = page_num_pagesizes() - 1;

	if (szc == 0) {

		segvn_lpg_disable = 1;

		return;

	}

	if (page_get_pagesize(0) != PAGESIZE) {

		panic("segvn_init: bad szc 0");

		/*NOTREACHED*/

	}

	while (szc != 0) {

		pgsz = page_get_pagesize(szc);

		if (pgsz <= PAGESIZE || !IS_P2ALIGNED(pgsz, pgsz)) {

			panic("segvn_init: bad szc %d", szc);

			/*NOTREACHED*/

		}

		szc--;

	}

	if (segvn_maxpgszc == 0 || segvn_maxpgszc > maxszc)

		segvn_maxpgszc = maxszc;

}

#define	SEGVN_PAGEIO	((void *)0x1)

#define	SEGVN_NOPAGEIO	((void *)0x2)

static void

segvn_setvnode_mpss(vnode_t *vp)

{

	int err;

	ASSERT(vp->v_mpssdata == NULL ||

	    vp->v_mpssdata == SEGVN_PAGEIO ||

	    vp->v_mpssdata == SEGVN_NOPAGEIO);

	if (vp->v_mpssdata == NULL) {

		if (vn_vmpss_usepageio(vp)) {

			err = VOP_PAGEIO(vp, (page_t *)NULL,

			    (u_offset_t)0, 0, 0, CRED());

		} else {

			err = ENOSYS;

		}

		/*

		 * set v_mpssdata just once per vnode life

		 * so that it never changes.

		 */

		mutex_enter(&vp->v_lock);

		if (vp->v_mpssdata == NULL) {

			if (err == EINVAL) {

				vp->v_mpssdata = SEGVN_PAGEIO;

			} else {

				vp->v_mpssdata = SEGVN_NOPAGEIO;

			}

		}

		mutex_exit(&vp->v_lock);

	}

}

int

segvn_create(struct seg *seg, void *argsp)

{

	struct segvn_crargs *a = (struct segvn_crargs *)argsp;

	struct segvn_data *svd;

	size_t swresv = 0;

	struct cred *cred;

	struct anon_map *amp;

	int error = 0;

	size_t pgsz;

	lgrp_mem_policy_t mpolicy = LGRP_MEM_POLICY_DEFAULT;

	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));

	if (a->type != MAP_PRIVATE && a->type != MAP_SHARED) {

		panic("segvn_create type");

		/*NOTREACHED*/

	}

	/*

	 * Check arguments.  If a shared anon structure is given then

	 * it is illegal to also specify a vp.

	 */

	if (a->amp != NULL && a->vp != NULL) {

		panic("segvn_create anon_map");

		/*NOTREACHED*/

	}

	/* MAP_NORESERVE on a MAP_SHARED segment is meaningless. */

	if (a->type == MAP_SHARED)

		a->flags &= ~MAP_NORESERVE;

	if (a->szc != 0) {

		if (segvn_lpg_disable != 0 || a->amp != NULL ||

		    (a->type == MAP_SHARED && a->vp == NULL) ||

		    (a->flags & MAP_NORESERVE) || seg->s_as == &kas) {

			a->szc = 0;

		} else {

			if (a->szc > segvn_maxpgszc)

				a->szc = segvn_maxpgszc;

			pgsz = page_get_pagesize(a->szc);

			if (!IS_P2ALIGNED(seg->s_base, pgsz) ||

			    !IS_P2ALIGNED(seg->s_size, pgsz)) {

				a->szc = 0;

			} else if (a->vp != NULL) {

				extern struct vnode kvp;

				if (IS_SWAPFSVP(a->vp) || a->vp == &kvp) {

					/*

					 * paranoid check.

					 * hat_page_demote() is not supported

					 * on swapfs pages.

					 */

					a->szc = 0;

				} else if (map_addr_vacalign_check(seg->s_base,

				    a->offset & PAGEMASK)) {

					a->szc = 0;

				}

			}

		}

	}

	/*

	 * If segment may need private pages, reserve them now.

	 */

	if (!(a->flags & MAP_NORESERVE) && ((a->vp == NULL && a->amp == NULL) ||

	    (a->type == MAP_PRIVATE && (a->prot & PROT_WRITE)))) {

		if (anon_resv(seg->s_size) == 0)

			return (EAGAIN);

		swresv = seg->s_size;

		TRACE_3(TR_FAC_VM, TR_ANON_PROC, "anon proc:%p %lu %u",

			seg, swresv, 1);

	}

	/*

	 * Reserve any mapping structures that may be required.

	 */

	hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);

	if (a->cred) {

		cred = a->cred;

		crhold(cred);

	} else {

		crhold(cred = CRED());

	}

	/* Inform the vnode of the new mapping */

	if (a->vp) {

		error = VOP_ADDMAP(a->vp, a->offset & PAGEMASK,

		    seg->s_as, seg->s_base, seg->s_size, a->prot,

		    a->maxprot, a->type, cred);

		if (error) {

			if (swresv != 0) {

				anon_unresv(swresv);

				TRACE_3(TR_FAC_VM, TR_ANON_PROC,

					"anon proc:%p %lu %u",

					seg, swresv, 0);

			}

			crfree(cred);

			hat_unload(seg->s_as->a_hat, seg->s_base,

				seg->s_size, HAT_UNLOAD_UNMAP);

			return (error);

		}

	}

	/*

	 * If more than one segment in the address space, and

	 * they're adjacent virtually, try to concatenate them.

	 * Don't concatenate if an explicit anon_map structure

	 * was supplied (e.g., SystemV shared memory).

	 */

	if (a->amp == NULL) {

		struct seg *pseg, *nseg;

		struct segvn_data *psvd, *nsvd;

		lgrp_mem_policy_t ppolicy, npolicy;

		uint_t	lgrp_mem_policy_flags = 0;

		extern lgrp_mem_policy_t lgrp_mem_default_policy;

		/*

		 * Memory policy flags (lgrp_mem_policy_flags) is valid when

		 * extending stack/heap segments.

		 */

		if ((a->vp == NULL) && (a->type == MAP_PRIVATE) &&

			!(a->flags & MAP_NORESERVE) && (seg->s_as != &kas)) {

			lgrp_mem_policy_flags = a->lgrp_mem_policy_flags;

		} else {

			/*

			 * Get policy when not extending it from another segment

			 */

			mpolicy = lgrp_mem_policy_default(seg->s_size, a->type);

		}

		/*

		 * First, try to concatenate the previous and new segments

		 */

		pseg = AS_SEGPREV(seg->s_as, seg);

		if (pseg != NULL &&

		    pseg->s_base + pseg->s_size == seg->s_base &&

		    pseg->s_ops == &segvn_ops) {

			/*

			 * Get memory allocation policy from previous segment.

			 * When extension is specified (e.g. for heap) apply

			 * this policy to the new segment regardless of the

			 * outcome of segment concatenation.  Extension occurs

			 * for non-default policy otherwise default policy is

			 * used and is based on extended segment size.

			 */

			psvd = (struct segvn_data *)pseg->s_data;

			ppolicy = psvd->policy_info.mem_policy;

			if (lgrp_mem_policy_flags ==

			    LGRP_MP_FLAG_EXTEND_UP) {

				if (ppolicy != lgrp_mem_default_policy) {

					mpolicy = ppolicy;

				} else {

					mpolicy = lgrp_mem_policy_default(

					    pseg->s_size + seg->s_size,

					    a->type);

				}

			}

			if (mpolicy == ppolicy &&

			    (pseg->s_size + seg->s_size <=

			    segvn_comb_thrshld || psvd->amp == NULL) &&

			    segvn_extend_prev(pseg, seg, a, swresv) == 0) {

				/*

				 * success! now try to concatenate

				 * with following seg

				 */

				crfree(cred);

				nseg = AS_SEGNEXT(pseg->s_as, pseg);

				if (nseg != NULL &&

				    nseg != pseg &&