/*

 * CDDL HEADER START

 *

 * The contents of this file are subject to the terms of the

 * Common Development and Distribution License (the "License").

 * You may not use this file except in compliance with the License.

 *

 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

 * or http://www.opensolaris.org/os/licensing.

 * See the License for the specific language governing permissions

 * and limitations under the License.

 *

 * When distributing Covered Code, include this CDDL HEADER in each

 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.

 * If applicable, add the following below this CDDL HEADER, with the

 * fields enclosed by brackets "[]" replaced with your own identifying

 * information: Portions Copyright [yyyy] [name of copyright owner]

 *

 * CDDL HEADER END

 */

/*

 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.

 * Use is subject to license terms.

 */

#pragma ident	"%Z%%M%	%I%	%E% SMI"

#include <sys/types.h>

#include <sys/t_lock.h>

#include <sys/param.h>

#include <sys/sysmacros.h>

#include <sys/signal.h>

#include <sys/systm.h>

#include <sys/user.h>

#include <sys/mman.h>

#include <sys/vm.h>

#include <sys/conf.h>

#include <sys/avintr.h>

#include <sys/autoconf.h>

#include <sys/disp.h>

#include <sys/class.h>

#include <sys/bitmap.h>

#include <sys/privregs.h>

#include <sys/proc.h>

#include <sys/buf.h>

#include <sys/kmem.h>

#include <sys/mem.h>

#include <sys/kstat.h>

#include <sys/reboot.h>

#include <sys/cred.h>

#include <sys/vnode.h>

#include <sys/file.h>

#include <sys/procfs.h>

#include <sys/vfs.h>

#include <sys/cmn_err.h>

#include <sys/utsname.h>

#include <sys/debug.h>

#include <sys/kdi.h>

#include <sys/dumphdr.h>

#include <sys/bootconf.h>

#include <sys/varargs.h>

#include <sys/promif.h>

#include <sys/modctl.h>

#include <sys/sunddi.h>

#include <sys/sunndi.h>

#include <sys/ndi_impldefs.h>

#include <sys/ddidmareq.h>

#include <sys/psw.h>

#include <sys/regset.h>

#include <sys/clock.h>

#include <sys/pte.h>

#include <sys/tss.h>

#include <sys/stack.h>

#include <sys/trap.h>

#include <sys/fp.h>

#include <vm/anon.h>

#include <vm/as.h>

#include <vm/page.h>

#include <vm/seg.h>

#include <vm/seg_dev.h>

#include <vm/seg_kmem.h>

#include <vm/seg_kpm.h>

#include <vm/seg_map.h>

#include <vm/seg_vn.h>

#include <vm/seg_kp.h>

#include <sys/memnode.h>

#include <vm/vm_dep.h>

#include <sys/thread.h>

#include <sys/sysconf.h>

#include <sys/vm_machparam.h>

#include <sys/archsystm.h>

#include <sys/machsystm.h>

#include <vm/hat.h>

#include <vm/hat_i86.h>

#include <sys/pmem.h>

#include <sys/smp_impldefs.h>

#include <sys/x86_archext.h>

#include <sys/segments.h>

#include <sys/clconf.h>

#include <sys/kobj.h>

#include <sys/kobj_lex.h>

#include <sys/cpc_impl.h>

#include <sys/x86_archext.h>

#include <sys/cpu_module.h>

#include <sys/smbios.h>

#include <sys/debug_info.h>

#include <sys/ddi_timer.h>

#ifdef __xpv

#include <sys/hypervisor.h>

#include <sys/xen_mmu.h>

#include <sys/evtchn_impl.h>

#include <sys/gnttab.h>

#include <sys/xpv_panic.h>

#include <xen/sys/xenbus_comms.h>

#include <xen/public/physdev.h>

extern void xen_late_startup(void);

extern struct xen_evt_data cpu0_evt_data;

#endif

#include <sys/bootinfo.h>

#include <vm/kboot_mmu.h>

extern void progressbar_init(void);

extern void progressbar_start(void);

extern void brand_init(void);

extern int size_pse_array(pgcnt_t, int);

/*

 * XXX make declaration below "static" when drivers no longer use this

 * interface.

 */

extern caddr_t p0_va;	/* Virtual address for accessing physical page 0 */

/*

 * segkp

 */

extern int segkp_fromheap;

static void kvm_init(void);

static void startup_init(void);

static void startup_memlist(void);

static void startup_kmem(void);

static void startup_modules(void);

static void startup_vm(void);

static void startup_end(void);

static void layout_kernel_va(void);

/*

 * Declare these as initialized data so we can patch them.

 */

#ifdef __i386

/*

 * Due to virtual address space limitations running in 32 bit mode, restrict

 * the amount of physical memory configured to a max of PHYSMEM pages (16g).

 *

 * If the physical max memory size of 64g were allowed to be configured, the

 * size of user virtual address space will be less than 1g. A limited user

 * address space greatly reduces the range of applications that can run.

 *

 * If more physical memory than PHYSMEM is required, users should preferably

 * run in 64 bit mode which has far looser virtual address space limitations.

 *

 * If 64 bit mode is not available (as in IA32) and/or more physical memory

 * than PHYSMEM is required in 32 bit mode, physmem can be set to the desired

 * value or to 0 (to configure all available memory) via eeprom(1M). kernelbase

 * should also be carefully tuned to balance out the need of the user

 * application while minimizing the risk of kernel heap exhaustion due to

 * kernelbase being set too high.

 */

#define	PHYSMEM	0x400000

#else /* __amd64 */

/*

 * For now we can handle memory with physical addresses up to about

 * 64 Terabytes. This keeps the kernel above the VA hole, leaving roughly

 * half the VA space for seg_kpm. When systems get bigger than 64TB this

 * code will need revisiting. There is an implicit assumption that there

 * are no *huge* holes in the physical address space too.

 */

#define	TERABYTE		(1ul << 40)

#define	PHYSMEM_MAX64		mmu_btop(64 * TERABYTE)

#define	PHYSMEM			PHYSMEM_MAX64

#define	AMD64_VA_HOLE_END	0xFFFF800000000000ul

#endif /* __amd64 */

pgcnt_t physmem = PHYSMEM;

pgcnt_t obp_pages;	/* Memory used by PROM for its text and data */

char *kobj_file_buf;

int kobj_file_bufsize;	/* set in /etc/system */

/* Global variables for MP support. Used in mp_startup */

caddr_t	rm_platter_va;

uint32_t rm_platter_pa;

int	auto_lpg_disable = 1;

/*

 * Some CPUs have holes in the middle of the 64-bit virtual address range.

 */

uintptr_t hole_start, hole_end;

/*

 * kpm mapping window

 */

caddr_t kpm_vbase;

size_t  kpm_size;

static int kpm_desired;

#ifdef __amd64

static uintptr_t segkpm_base = (uintptr_t)SEGKPM_BASE;

#endif

/*

 * Configuration parameters set at boot time.

 */

caddr_t econtig;		/* end of first block of contiguous kernel */

struct bootops		*bootops = 0;	/* passed in from boot */

struct bootops		**bootopsp;

struct boot_syscalls	*sysp;		/* passed in from boot */

char bootblock_fstype[16];

char kern_bootargs[OBP_MAXPATHLEN];

/*

 * ZFS zio segment.  This allows us to exclude large portions of ZFS data that

 * gets cached in kmem caches on the heap.  If this is set to zero, we allocate

 * zio buffers from their own segment, otherwise they are allocated from the

 * heap.  The optimization of allocating zio buffers from their own segment is

 * only valid on 64-bit kernels.

 */

#if defined(__amd64)

int segzio_fromheap = 0;

#else

int segzio_fromheap = 1;

#endif

/*

 * new memory fragmentations are possible in startup() due to BOP_ALLOCs. this

 * depends on number of BOP_ALLOC calls made and requested size, memory size

 * combination and whether boot.bin memory needs to be freed.

 */

#define	POSS_NEW_FRAGMENTS	12

/*

 * VM data structures

 */

long page_hashsz;		/* Size of page hash table (power of two) */

struct page *pp_base;		/* Base of initial system page struct array */

struct page **page_hash;	/* Page hash table */

pad_mutex_t *pse_mutex;		/* Locks protecting pp->p_selock */

size_t pse_table_size;		/* Number of mutexes in pse_mutex[] */

int pse_shift;			/* log2(pse_table_size) */

struct seg ktextseg;		/* Segment used for kernel executable image */

struct seg kvalloc;		/* Segment used for "valloc" mapping */

struct seg kpseg;		/* Segment used for pageable kernel virt mem */

struct seg kmapseg;		/* Segment used for generic kernel mappings */

struct seg kdebugseg;		/* Segment used for the kernel debugger */

struct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */

static struct seg *segmap = &kmapseg;	/* easier to use name for in here */

struct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */

#if defined(__amd64)

struct seg kvseg_core;		/* Segment used for the core heap */

struct seg kpmseg;		/* Segment used for physical mapping */

struct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */

#else

struct seg *segkpm = NULL;	/* Unused on IA32 */

#endif

caddr_t segkp_base;		/* Base address of segkp */

caddr_t segzio_base;		/* Base address of segzio */

#if defined(__amd64)

pgcnt_t segkpsize = btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */

#else

pgcnt_t segkpsize = 0;

#endif

pgcnt_t segziosize = 0;		/* size of zio segment in pages */

/*

 * VA range available to the debugger

 */

const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE;

const size_t kdi_segdebugsize = SEGDEBUGSIZE;

struct memseg *memseg_base;

struct vnode unused_pages_vp;

#define	FOURGB	0x100000000LL

struct memlist *memlist;

caddr_t s_text;		/* start of kernel text segment */

caddr_t e_text;		/* end of kernel text segment */

caddr_t s_data;		/* start of kernel data segment */

caddr_t e_data;		/* end of kernel data segment */

caddr_t modtext;	/* start of loadable module text reserved */

caddr_t e_modtext;	/* end of loadable module text reserved */

caddr_t moddata;	/* start of loadable module data reserved */

caddr_t e_moddata;	/* end of loadable module data reserved */

struct memlist *phys_install;	/* Total installed physical memory */

struct memlist *phys_avail;	/* Total available physical memory */

/*

 * kphysm_init returns the number of pages that were processed

 */

static pgcnt_t kphysm_init(page_t *, pgcnt_t);

#define	IO_PROP_SIZE	64	/* device property size */

/*

 * a couple useful roundup macros

 */

#define	ROUND_UP_PAGE(x)	\

	((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)MMU_PAGESIZE))

#define	ROUND_UP_LPAGE(x)	\

	((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[1]))

#define	ROUND_UP_4MEG(x)	\

	((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)FOUR_MEG))

#define	ROUND_UP_TOPLEVEL(x)	\

	((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[mmu.max_level]))

/*

 *	32-bit Kernel's Virtual memory layout.

 *		+-----------------------+

 *		|			|

 * 0xFFC00000  -|-----------------------|- ARGSBASE

 *		|	debugger	|

 * 0xFF800000  -|-----------------------|- SEGDEBUGBASE

 *		|      Kernel Data	|

 * 0xFEC00000  -|-----------------------|

 *              |      Kernel Text	|

 * 0xFE800000  -|-----------------------|- KERNEL_TEXT (0xFB400000 on Xen)

 *		|---       GDT       ---|- GDT page (GDT_VA)

 *		|---    debug info   ---|- debug info (DEBUG_INFO_VA)

 *		|			|

 * 		|   page_t structures	|

 * 		|   memsegs, memlists, 	|

 * 		|   page hash, etc.	|

 * ---	       -|-----------------------|- ekernelheap, valloc_base (floating)

 *		|			|  (segkp is just an arena in the heap)

 *		|			|

 *		|	kvseg		|

 *		|			|

 *		|			|

 * ---         -|-----------------------|- kernelheap (floating)

 * 		|        Segkmap	|

 * 0xC3002000  -|-----------------------|- segmap_start (floating)

 *		|	Red Zone	|

 * 0xC3000000  -|-----------------------|- kernelbase / userlimit (floating)

 *		|			|			||

 *		|     Shared objects	|			\/

 *		|			|

 *		:			:

 *		|	user data	|

 *		|-----------------------|

 *		|	user text	|

 * 0x08048000  -|-----------------------|

 *		|	user stack	|

 *		:			:

 *		|	invalid		|

 * 0x00000000	+-----------------------+

 *

 *

 *		64-bit Kernel's Virtual memory layout. (assuming 64 bit app)

 *			+-----------------------+

 *			|			|

 * 0xFFFFFFFF.FFC00000  |-----------------------|- ARGSBASE

 *			|	debugger (?)	|

 * 0xFFFFFFFF.FF800000  |-----------------------|- SEGDEBUGBASE

 *			|      unused    	|

 *			+-----------------------+

 *			|      Kernel Data	|

 * 0xFFFFFFFF.FBC00000  |-----------------------|

 *			|      Kernel Text	|

 * 0xFFFFFFFF.FB800000  |-----------------------|- KERNEL_TEXT

 *			|---       GDT       ---|- GDT page (GDT_VA)

 *			|---    debug info   ---|- debug info (DEBUG_INFO_VA)

 *			|			|

 * 			|      Core heap	| (used for loadable modules)

 * 0xFFFFFFFF.C0000000  |-----------------------|- core_base / ekernelheap

 *			|	 Kernel		|

 *			|	  heap		|

 * 0xFFFFFXXX.XXX00000  |-----------------------|- kernelheap (floating)

 *			|	 segmap		|

 * 0xFFFFFXXX.XXX00000  |-----------------------|- segmap_start (floating)

 *			|    device mappings	|

 * 0xFFFFFXXX.XXX00000  |-----------------------|- toxic_addr (floating)

 *			|	  segzio	|

 * 0xFFFFFXXX.XXX00000  |-----------------------|- segzio_base (floating)

 *			|	  segkp		|

 * ---                  |-----------------------|- segkp_base (floating)

 * 			|   page_t structures	|  valloc_base + valloc_sz

 * 			|   memsegs, memlists, 	|

 * 			|   page hash, etc.	|

 * 0xFFFFFF00.00000000  |-----------------------|- valloc_base (lower if > 1TB)

 *			|	 segkpm		|

 * 0xFFFFFE00.00000000  |-----------------------|

 *			|	Red Zone	|

 * 0xFFFFFD80.00000000  |-----------------------|- KERNELBASE (lower if > 1TB)

 *			|     User stack	|- User space memory

 * 			|			|

 * 			| shared objects, etc	|	(grows downwards)

 *			:			:

 * 			|			|

 * 0xFFFF8000.00000000  |-----------------------|

 * 			|			|

 * 			| VA Hole / unused	|

 * 			|			|

 * 0x00008000.00000000  |-----------------------|

 *			|			|

 *			|			|

 *			:			:

 *			|	user heap	|	(grows upwards)

 *			|			|

 *			|	user data	|

 *			|-----------------------|

 *			|	user text	|

 * 0x00000000.04000000  |-----------------------|

 *			|	invalid		|

 * 0x00000000.00000000	+-----------------------+

 *

 * A 32 bit app on the 64 bit kernel sees the same layout as on the 32 bit

 * kernel, except that userlimit is raised to 0xfe000000

 *

 * Floating values:

 *

 * valloc_base: start of the kernel's memory management/tracking data

 * structures.  This region contains page_t structures for

 * physical memory, memsegs, memlists, and the page hash.

 *

 * core_base: start of the kernel's "core" heap area on 64-bit systems.

 * This area is intended to be used for global data as well as for module

 * text/data that does not fit into the nucleus pages.  The core heap is

 * restricted to a 2GB range, allowing every address within it to be

 * accessed using rip-relative addressing

 *

 * ekernelheap: end of kernelheap and start of segmap.

 *

 * kernelheap: start of kernel heap.  On 32-bit systems, this starts right

 * above a red zone that separates the user's address space from the

 * kernel's.  On 64-bit systems, it sits above segkp and segkpm.

 *

 * segmap_start: start of segmap. The length of segmap can be modified

 * by changing segmapsize in /etc/system (preferred) or eeprom (deprecated).

 * The default length is 16MB on 32-bit systems and 64MB on 64-bit systems.

 *

 * kernelbase: On a 32-bit kernel the default value of 0xd4000000 will be

 * decreased by 2X the size required for page_t.  This allows the kernel

 * heap to grow in size with physical memory.  With sizeof(page_t) == 80

 * bytes, the following shows the values of kernelbase and kernel heap

 * sizes for different memory configurations (assuming default segmap and

 * segkp sizes).

 *

 *	mem	size for	kernelbase	kernel heap

 *	size	page_t's			size

 *	----	---------	----------	-----------

 *	1gb	0x01400000	0xd1800000	684MB

 *	2gb	0x02800000	0xcf000000	704MB

 *	4gb	0x05000000	0xca000000	744MB

 *	6gb	0x07800000	0xc5000000	784MB

 *	8gb	0x0a000000	0xc0000000	824MB

 *	16gb	0x14000000	0xac000000	984MB

 *	32gb	0x28000000	0x84000000	1304MB

 *	64gb	0x50000000	0x34000000	1944MB (*)

 *

 * kernelbase is less than the abi minimum of 0xc0000000 for memory

 * configurations above 8gb.

 *

 * (*) support for memory configurations above 32gb will require manual tuning

 * of kernelbase to balance out the need of user applications.

 */

/* real-time-clock initialization parameters */

extern time_t process_rtc_config_file(void);

uintptr_t	kernelbase;

uintptr_t	postbootkernelbase;	/* not set till boot loader is gone */

uintptr_t	eprom_kernelbase;

size_t		segmapsize;

uintptr_t	segmap_start;

int		segmapfreelists;

pgcnt_t		npages;

pgcnt_t		orig_npages;

size_t		core_size;		/* size of "core" heap */

uintptr_t	core_base;		/* base address of "core" heap */

/*

 * List of bootstrap pages. We mark these as allocated in startup.

 * release_bootstrap() will free them when we're completely done with

 * the bootstrap.

 */

static page_t *bootpages;

/*