1 /*

     2  * CDDL HEADER START

     3  *

     4  * The contents of this file are subject to the terms of the

     5  * Common Development and Distribution License, Version 1.0 only

     6  * (the "License").  You may not use this file except in compliance

     7  * with the License.

     8  *

     9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

    10  * or http://www.opensolaris.org/os/licensing.

    11  * See the License for the specific language governing permissions

    12  * and limitations under the License.

    13  *

    14  * When distributing Covered Code, include this CDDL HEADER in each

    15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.

    16  * If applicable, add the following below this CDDL HEADER, with the

    17  * fields enclosed by brackets "[]" replaced with your own identifying

    18  * information: Portions Copyright [yyyy] [name of copyright owner]

    19  *

    20  * CDDL HEADER END

    21  */

    22 /*

    23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.

    24  * Use is subject to license terms.

    25  */

    26 

    27 #pragma ident	"%Z%%M%	%I%	%E% SMI"

    28 

    29 #include <sys/types.h>

    30 #include <sys/t_lock.h>

    31 #include <sys/param.h>

    32 #include <sys/sysmacros.h>

    33 #include <sys/signal.h>

    34 #include <sys/systm.h>

    35 #include <sys/user.h>

    36 #include <sys/mman.h>

    37 #include <sys/vm.h>

    38 #include <sys/conf.h>

    39 #include <sys/avintr.h>

    40 #include <sys/autoconf.h>

    41 #include <sys/disp.h>

    42 #include <sys/class.h>

    43 #include <sys/bitmap.h>

    44 

    45 #include <sys/privregs.h>

    46 

    47 #include <sys/proc.h>

    48 #include <sys/buf.h>

    49 #include <sys/kmem.h>

    50 #include <sys/kstat.h>

    51 

    52 #include <sys/reboot.h>

    53 #include <sys/uadmin.h>

    54 

    55 #include <sys/cred.h>

    56 #include <sys/vnode.h>

    57 #include <sys/file.h>

    58 

    59 #include <sys/procfs.h>

    60 #include <sys/acct.h>

    61 

    62 #include <sys/vfs.h>

    63 #include <sys/dnlc.h>

    64 #include <sys/var.h>

    65 #include <sys/cmn_err.h>

    66 #include <sys/utsname.h>

    67 #include <sys/debug.h>

    68 #include <sys/kdi.h>

    69 

    70 #include <sys/dumphdr.h>

    71 #include <sys/bootconf.h>

    72 #include <sys/varargs.h>

    73 #include <sys/promif.h>

    74 #include <sys/prom_emul.h>	/* for create_prom_prop */

    75 #include <sys/modctl.h>		/* for "procfs" hack */

    76 

    77 #include <sys/consdev.h>

    78 #include <sys/frame.h>

    79 

    80 #include <sys/sunddi.h>

    81 #include <sys/sunndi.h>

    82 #include <sys/ndi_impldefs.h>

    83 #include <sys/ddidmareq.h>

    84 #include <sys/psw.h>

    85 #include <sys/regset.h>

    86 #include <sys/clock.h>

    87 #include <sys/pte.h>

    88 #include <sys/mmu.h>

    89 #include <sys/tss.h>

    90 #include <sys/stack.h>

    91 #include <sys/trap.h>

    92 #include <sys/pic.h>

    93 #include <sys/fp.h>

    94 #include <vm/anon.h>

    95 #include <vm/as.h>

    96 #include <vm/page.h>

    97 #include <vm/seg.h>

    98 #include <vm/seg_dev.h>

    99 #include <vm/seg_kmem.h>

   100 #include <vm/seg_kpm.h>

   101 #include <vm/seg_map.h>

   102 #include <vm/seg_vn.h>

   103 #include <vm/seg_kp.h>

   104 #include <sys/memnode.h>

   105 #include <vm/vm_dep.h>

   106 #include <sys/swap.h>

   107 #include <sys/thread.h>

   108 #include <sys/sysconf.h>

   109 #include <sys/vm_machparam.h>

   110 #include <sys/archsystm.h>

   111 #include <sys/machsystm.h>

   112 #include <vm/hat.h>

   113 #include <vm/hat_i86.h>

   114 #include <sys/pmem.h>

   115 #include <sys/instance.h>

   116 #include <sys/smp_impldefs.h>

   117 #include <sys/x86_archext.h>

   118 #include <sys/segments.h>

   119 #include <sys/clconf.h>

   120 #include <sys/kobj.h>

   121 #include <sys/kobj_lex.h>

   122 #include <sys/prom_emul.h>

   123 #include <sys/cpc_impl.h>

   124 #include <sys/chip.h>

   125 #include <sys/x86_archext.h>

   126 

   127 extern void debug_enter(char *);

   128 extern void progressbar_init(void);

   129 extern void progressbar_start(void);

   130 

   131 /*

   132  * XXX make declaration below "static" when drivers no longer use this

   133  * interface.

   134  */

   135 extern caddr_t p0_va;	/* Virtual address for accessing physical page 0 */

   136 

   137 /*

   138  * segkp

   139  */

   140 extern int segkp_fromheap;

   141 

   142 static void kvm_init(void);

   143 static void startup_init(void);

   144 static void startup_memlist(void);

   145 static void startup_modules(void);

   146 static void startup_bop_gone(void);

   147 static void startup_vm(void);

   148 static void startup_end(void);

   149 

   150 /*

   151  * Declare these as initialized data so we can patch them.

   152  */

   153 pgcnt_t physmem = 0;	/* memory size in pages, patch if you want less */

   154 pgcnt_t obp_pages;	/* Memory used by PROM for its text and data */

   155 

   156 char *kobj_file_buf;

   157 int kobj_file_bufsize;	/* set in /etc/system */

   158 

   159 /* Global variables for MP support. Used in mp_startup */

   160 caddr_t	rm_platter_va;

   161 uint32_t rm_platter_pa;

   162 

   163 /*

   164  * Some CPUs have holes in the middle of the 64-bit virtual address range.

   165  */

   166 uintptr_t hole_start, hole_end;

   167 

   168 /*

   169  * kpm mapping window

   170  */

   171 caddr_t kpm_vbase;

   172 size_t  kpm_size;

   173 static int kpm_desired = 0;		/* Do we want to try to use segkpm? */

   174 

   175 /*

   176  * VA range that must be preserved for boot until we release all of its

   177  * mappings.

   178  */

   179 #if defined(__amd64)

   180 static void *kmem_setaside;

   181 #endif

   182 

   183 /*

   184  * Configuration parameters set at boot time.

   185  */

   186 

   187 caddr_t econtig;		/* end of first block of contiguous kernel */

   188 

   189 struct bootops		*bootops = 0;	/* passed in from boot */

   190 struct bootops		**bootopsp;

   191 struct boot_syscalls	*sysp;		/* passed in from boot */

   192 

   193 char bootblock_fstype[16];

   194 

   195 char kern_bootargs[OBP_MAXPATHLEN];

   196 

   197 /*

   198  * new memory fragmentations are possible in startup() due to BOP_ALLOCs. this

   199  * depends on number of BOP_ALLOC calls made and requested size, memory size

   200  * combination and whether boot.bin memory needs to be freed.

   201  */

   202 #define	POSS_NEW_FRAGMENTS	12

   203 

   204 /*

   205  * VM data structures

   206  */

   207 long page_hashsz;		/* Size of page hash table (power of two) */

   208 struct page *pp_base;		/* Base of initial system page struct array */

   209 struct page **page_hash;	/* Page hash table */

   210 struct seg ktextseg;		/* Segment used for kernel executable image */

   211 struct seg kvalloc;		/* Segment used for "valloc" mapping */

   212 struct seg kpseg;		/* Segment used for pageable kernel virt mem */

   213 struct seg kmapseg;		/* Segment used for generic kernel mappings */

   214 struct seg kdebugseg;		/* Segment used for the kernel debugger */

   215 

   216 struct seg *segkmap = &kmapseg;	/* Kernel generic mapping segment */

   217 struct seg *segkp = &kpseg;	/* Pageable kernel virtual memory segment */

   218 

   219 #if defined(__amd64)

   220 struct seg kvseg_core;		/* Segment used for the core heap */

   221 struct seg kpmseg;		/* Segment used for physical mapping */

   222 struct seg *segkpm = &kpmseg;	/* 64bit kernel physical mapping segment */

   223 #else

   224 struct seg *segkpm = NULL;	/* Unused on IA32 */

   225 #endif

   226 

   227 caddr_t segkp_base;		/* Base address of segkp */

   228 #if defined(__amd64)

   229 pgcnt_t segkpsize = btop(SEGKPDEFSIZE);	/* size of segkp segment in pages */

   230 #else

   231 pgcnt_t segkpsize = 0;

   232 #endif

   233 

   234 struct memseg *memseg_base;

   235 struct vnode unused_pages_vp;

   236 

   237 #define	FOURGB	0x100000000LL

   238 

   239 struct memlist *memlist;

   240 

   241 caddr_t s_text;		/* start of kernel text segment */

   242 caddr_t e_text;		/* end of kernel text segment */

   243 caddr_t s_data;		/* start of kernel data segment */

   244 caddr_t e_data;		/* end of kernel data segment */

   245 caddr_t modtext;	/* start of loadable module text reserved */

   246 caddr_t e_modtext;	/* end of loadable module text reserved */

   247 caddr_t moddata;	/* start of loadable module data reserved */

   248 caddr_t e_moddata;	/* end of loadable module data reserved */

   249 

   250 struct memlist *phys_install;	/* Total installed physical memory */

   251 struct memlist *phys_avail;	/* Total available physical memory */

   252 

   253 static void memlist_add(uint64_t, uint64_t, struct memlist *,

   254 	struct memlist **);

   255 

   256 /*

   257  * kphysm_init returns the number of pages that were processed

   258  */

   259 static pgcnt_t kphysm_init(page_t *, struct memseg *, pgcnt_t, pgcnt_t);

   260 

   261 #define	IO_PROP_SIZE	64	/* device property size */

   262 

   263 /*

   264  * a couple useful roundup macros

   265  */

   266 #define	ROUND_UP_PAGE(x)	\

   267 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)MMU_PAGESIZE))

   268 #define	ROUND_UP_LPAGE(x)	\

   269 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[1]))

   270 #define	ROUND_UP_4MEG(x)	\

   271 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)FOURMB_PAGESIZE))

   272 #define	ROUND_UP_TOPLEVEL(x)	\

   273 	((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[mmu.max_level]))

   274 

   275 /*

   276  *	32-bit Kernel's Virtual memory layout.

   277  *		+-----------------------+

   278  *		|	psm 1-1 map	|

   279  *		|	exec args area	|

   280  * 0xFFC00000  -|-----------------------|- ARGSBASE

   281  *		|	debugger	|

   282  * 0xFF800000  -|-----------------------|- SEGDEBUGBASE

   283  *		|      Kernel Data	|

   284  * 0xFEC00000  -|-----------------------|

   285  *              |      Kernel Text	|

   286  * 0xFE800000  -|-----------------------|- KERNEL_TEXT

   287  * 		|     LUFS sinkhole	|

   288  * 0xFE000000  -|-----------------------|- lufs_addr

   289  * ---         -|-----------------------|- valloc_base + valloc_sz

   290  * 		|   early pp structures	|

   291  * 		|   memsegs, memlists, 	|

   292  * 		|   page hash, etc.	|

   293  * ---	       -|-----------------------|- valloc_base (floating)

   294  * 		|     ptable_va    	|

   295  * 0xFDFFE000  -|-----------------------|- ekernelheap, ptable_va

   296  *		|			|  (segkp is an arena under the heap)

   297  *		|			|

   298  *		|	kvseg		|

   299  *		|			|

   300  *		|			|

   301  * ---         -|-----------------------|- kernelheap (floating)

   302  * 		|        Segkmap	|

   303  * 0xC3002000  -|-----------------------|- segkmap_start (floating)

   304  *		|	Red Zone	|

   305  * 0xC3000000  -|-----------------------|- kernelbase / userlimit (floating)

   306  *		|			|			||

   307  *		|     Shared objects	|			\/

   308  *		|			|

   309  *		:			:

   310  *		|	user data	|

   311  *		|-----------------------|

   312  *		|	user text	|

   313  * 0x08048000  -|-----------------------|

   314  *		|	user stack	|

   315  *		:			:

   316  *		|	invalid		|

   317  * 0x00000000	+-----------------------+

   318  *

   319  *

   320  *		64-bit Kernel's Virtual memory layout. (assuming 64 bit app)

   321  *			+-----------------------+

   322  *			|	psm 1-1 map	|

   323  *			|	exec args area	|

   324  * 0xFFFFFFFF.FFC00000  |-----------------------|- ARGSBASE

   325  *			|	debugger (?)	|

   326  * 0xFFFFFFFF.FF800000  |-----------------------|- SEGDEBUGBASE

   327  *			|      unused    	|

   328  *			+-----------------------+

   329  *			|      Kernel Data	|

   330  * 0xFFFFFFFF.FBC00000  |-----------------------|

   331  *			|      Kernel Text	|

   332  * 0xFFFFFFFF.FB800000  |-----------------------|- KERNEL_TEXT

   333  * 			|     LUFS sinkhole	|

   334  * 0xFFFFFFFF.FB000000 -|-----------------------|- lufs_addr

   335  * ---                  |-----------------------|- valloc_base + valloc_sz

   336  * 			|   early pp structures	|

   337  * 			|   memsegs, memlists, 	|

   338  * 			|   page hash, etc.	|

   339  * ---                  |-----------------------|- valloc_base

   340  * 			|     ptable_va    	|

   341  * ---                  |-----------------------|- ptable_va

   342  * 			|      Core heap	| (used for loadable modules)

   343  * 0xFFFFFFFF.C0000000  |-----------------------|- core_base / ekernelheap

   344  *			|	 Kernel		|

   345  *			|	  heap		|

   346  * 0xFFFFFXXX.XXX00000  |-----------------------|- kernelheap (floating)

   347  *			|	 segkmap	|

   348  * 0xFFFFFXXX.XXX00000  |-----------------------|- segkmap_start (floating)

   349  *			|    device mappings	|

   350  * 0xFFFFFXXX.XXX00000  |-----------------------|- toxic_addr (floating)

   351  *			|	  segkp		|

   352  * ---                  |-----------------------|- segkp_base

   353  *			|	 segkpm		|

   354  * 0xFFFFFE00.00000000  |-----------------------|

   355  *			|	Red Zone	|

   356  * 0xFFFFFD80.00000000  |-----------------------|- KERNELBASE

   357  *			|     User stack	|- User space memory

   358  * 			|			|

   359  * 			| shared objects, etc	|	(grows downwards)

   360  *			:			:

   361  * 			|			|

   362  * 0xFFFF8000.00000000  |-----------------------|

   363  * 			|			|

   364  * 			| VA Hole / unused	|

   365  * 			|			|

   366  * 0x00008000.00000000  |-----------------------|

   367  *			|			|

   368  *			|			|

   369  *			:			:

   370  *			|	user heap	|	(grows upwards)

   371  *			|			|

   372  *			|	user data	|

   373  *			|-----------------------|

   374  *			|	user text	|

   375  * 0x00000000.04000000  |-----------------------|

   376  *			|	invalid		|

   377  * 0x00000000.00000000	+-----------------------+

   378  *

   379  * A 32 bit app on the 64 bit kernel sees the same layout as on the 32 bit

   380  * kernel, except that userlimit is raised to 0xfe000000

   381  *

   382  * Floating values:

   383  *

   384  * valloc_base: start of the kernel's memory management/tracking data

   385  * structures.  This region contains page_t structures for the lowest 4GB

   386  * of physical memory, memsegs, memlists, and the page hash.

   387  *

   388  * core_base: start of the kernel's "core" heap area on 64-bit systems.

   389  * This area is intended to be used for global data as well as for module

   390  * text/data that does not fit into the nucleus pages.  The core heap is

   391  * restricted to a 2GB range, allowing every address within it to be

   392  * accessed using rip-relative addressing

   393  *

   394  * ekernelheap: end of kernelheap and start of segmap.

   395  *

   396  * kernelheap: start of kernel heap.  On 32-bit systems, this starts right

   397  * above a red zone that separates the user's address space from the

   398  * kernel's.  On 64-bit systems, it sits above segkp and segkpm.

   399  *

   400  * segkmap_start: start of segmap. The length of segmap can be modified

   401  * by changing segmapsize in /etc/system (preferred) or eeprom (deprecated).

   402  * The default length is 16MB on 32-bit systems and 64MB on 64-bit systems.

   403  *

   404  * kernelbase: On a 32-bit kernel the default value of 0xd4000000 will be

   405  * decreased by 2X the size required for page_t.  This allows the kernel

   406  * heap to grow in size with physical memory.  With sizeof(page_t) == 80

   407  * bytes, the following shows the values of kernelbase and kernel heap

   408  * sizes for different memory configurations (assuming default segmap and

   409  * segkp sizes).

   410  *

   411  *	mem	size for	kernelbase	kernel heap

   412  *	size	page_t's			size

   413  *	----	---------	----------	-----------

   414  *	1gb	0x01400000	0xd1800000	684MB

   415  *	2gb	0x02800000	0xcf000000	704MB

   416  *	4gb	0x05000000	0xca000000	744MB

   417  *	6gb	0x07800000	0xc5000000	784MB

   418  *	8gb	0x0a000000	0xc0000000	824MB

   419  *	16gb	0x14000000	0xac000000	984MB

   420  *	32gb	0x28000000	0x84000000	1304MB

   421  *	64gb	0x50000000	0x34000000	1944MB (*)

   422  *

   423  * kernelbase is less than the abi minimum of 0xc0000000 for memory

   424  * configurations above 8gb.

   425  *

   426  * (*) support for memory configurations above 32gb will require manual tuning

   427  * of kernelbase to balance out the need of user applications.

   428  */

   429 

   430 void init_intr_threads(struct cpu *);

   431 

   432 /*

   433  * Dummy spl priority masks

   434  */

   435 static unsigned char	dummy_cpu_pri[MAXIPL + 1] = {

   436 	0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf,

   437 	0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf, 0xf

   438 };

   439 

   440 /* real-time-clock initialization parameters */

   441 long gmt_lag;		/* offset in seconds of gmt to local time */

   442 extern long process_rtc_config_file(void);

   443 

   444 char		*final_kernelheap;

   445 char		*boot_kernelheap;

   446 uintptr_t	kernelbase;

   447 uintptr_t	eprom_kernelbase;

   448 size_t		segmapsize;

   449 static uintptr_t segmap_reserved;

   450 uintptr_t	segkmap_start;

   451 int		segmapfreelists;

   452 pgcnt_t		boot_npages;

   453 pgcnt_t		npages;

   454 size_t		core_size;		/* size of "core" heap */

   455 uintptr_t	core_base;		/* base address of "core" heap */

   456 

   457 /*

   458  * List of bootstrap pages. We mark these as allocated in startup.

   459  * release_bootstrap() will free them when we're completely done with

   460  * the bootstrap.

   461  */

   462 static page_t *bootpages, *rd_pages;

   463 

   464 struct system_hardware system_hardware;

   465 

   466 /*

   467  * Enable some debugging messages concerning memory usage...

   468  *

   469  * XX64 There should only be one print routine once memlist usage between

   470  * vmx and the kernel is cleaned up and there is a single memlist structure

   471  * shared between kernel and boot.

   472  */

   473 static void

   474 print_boot_memlist(char *title, struct memlist *mp)

   475 {

   476 	prom_printf("MEMLIST: %s:\n", title);

   477 	while (mp != NULL)  {

   478 		prom_printf("\tAddress 0x%" PRIx64 ", size 0x%" PRIx64 "\n",

   479 		    mp->address, mp->size);

   480 		mp = mp->next;

   481 	}

   482 }

   483 

   484 static void

   485 print_kernel_memlist(char *title, struct memlist *mp)

   486 {

   487 	prom_printf("MEMLIST: %s:\n", title);

   488 	while (mp != NULL)  {

   489 		prom_printf("\tAddress 0x%" PRIx64 ", size 0x%" PRIx64 "\n",

   490 		    mp->address, mp->size);

   491 		mp = mp->next;

   492 	}

   493 }

   494 

   495 /*

   496  * XX64 need a comment here.. are these just default values, surely

   497  * we read the "cpuid" type information to figure this out.

   498  */

   499 int	l2cache_sz = 0x80000;

   500 int	l2cache_linesz = 0x40;

   501 int	l2cache_assoc = 1;

   502 

   503 /*

   504  * on 64 bit we use a predifined VA range for mapping devices in the kernel

   505  * on 32 bit the mappings are intermixed in the heap, so we use a bit map

   506  */

   507 #ifdef __amd64

   508 

   509 vmem_t		*device_arena;

   510 uintptr_t	toxic_addr = (uintptr_t)NULL;

   511 size_t		toxic_size = 1 * 1024 * 1024 * 1024; /* Sparc uses 1 gig too */

   512 

   513 #else	/* __i386 */

   514 

   515 ulong_t		*toxic_bit_map;	/* one bit for each 4k of VA in heap_arena */

   516 size_t		toxic_bit_map_len = 0;	/* in bits */

   517 

   518 #endif	/* __i386 */

   519 

   520 /*

   521  * Simple boot time debug facilities

   522  */

   523 static char *prm_dbg_str[] = {

   524 	"%s:%d: '%s' is 0x%x\n",

   525 	"%s:%d: '%s' is 0x%llx\n"

   526 };

   527 

   528 int prom_debug;

   529 

   530 #define	PRM_DEBUG(q)	if (prom_debug) 	\

   531 	prom_printf(prm_dbg_str[sizeof (q) >> 3], "startup.c", __LINE__, #q, q);

   532 #define	PRM_POINT(q)	if (prom_debug) 	\

   533 	prom_printf("%s:%d: %s\n", "startup.c", __LINE__, q);

   534 

   535 /*

   536  * This structure is used to keep track of the intial allocations

   537  * done in startup_memlist(). The value of NUM_ALLOCATIONS needs to

   538  * be >= the number of ADD_TO_ALLOCATIONS() executed in the code.

   539  */

   540 #define	NUM_ALLOCATIONS 7

   541 int num_allocations = 0;

   542 struct {

   543 	void **al_ptr;

   544 	size_t al_size;

   545 } allocations[NUM_ALLOCATIONS];

   546 size_t valloc_sz = 0;

   547 uintptr_t valloc_base;

   548 extern uintptr_t ptable_va;

   549 extern size_t ptable_sz;

   550 

   551 #define	ADD_TO_ALLOCATIONS(ptr, size) {					\

   552 		size = ROUND_UP_PAGE(size);		 		\

   553 		if (num_allocations == NUM_ALLOCATIONS)			\

   554 			panic("too many ADD_TO_ALLOCATIONS()");		\

   555 		allocations[num_allocations].al_ptr = (void**)&ptr;	\

   556 		allocations[num_allocations].al_size = size;		\

   557 		valloc_sz += size;					\

   558 		++num_allocations;				 	\

   559 	}

   560 

   561 static void

   562 perform_allocations(void)

   563 {

   564 	caddr_t mem;

   565 	int i;

   566 

   567 	mem = BOP_ALLOC(bootops, (caddr_t)valloc_base, valloc_sz, BO_NO_ALIGN);

   568 	if (mem != (caddr_t)valloc_base)

   569 		panic("BOP_ALLOC() failed");

   570 	bzero(mem, valloc_sz);

   571 	for (i = 0; i < num_allocations; ++i) {

   572 		*allocations[i].al_ptr = (void *)mem;

   573 		mem += allocations[i].al_size;

   574 	}

   575 }

   576 

   577 /*

   578  * Our world looks like this at startup time.

   579  *

   580  * In a 32-bit OS, boot loads the kernel text at 0xfe800000 and kernel data

   581  * at 0xfec00000.  On a 64-bit OS, kernel text and data are loaded at

   582  * 0xffffffff.fe800000 and 0xffffffff.fec00000 respectively.  Those

   583  * addresses are fixed in the binary at link time.

   584  *

   585  * On the text page:

   586  * unix/genunix/krtld/module text loads.

   587  *

   588  * On the data page:

   589  * unix/genunix/krtld/module data loads and space for page_t's.

   590  */

   591 /*

   592  * Machine-dependent startup code

   593  */

   594 void

   595 startup(void)

   596 {

   597 	extern void startup_bios_disk();

   598 	/*

   599 	 * Make sure that nobody tries to use sekpm until we have

   600 	 * initialized it properly.

   601 	 */

   602 #if defined(__amd64)

   603 	kpm_desired = kpm_enable;

   604 #endif

   605 	kpm_enable = 0;

   606 

   607 	progressbar_init();

   608 	startup_init();

   609 	startup_memlist();

   610 	startup_modules();

   611 	startup_bios_disk();

   612 	startup_bop_gone();

   613 	startup_vm();

   614 	startup_end();

   615 	progressbar_start();

   616 }

   617 

   618 static void

   619 startup_init()

   620 {

   621 	PRM_POINT("startup_init() starting...");

   622 

   623 	/*

   624 	 * Complete the extraction of cpuid data

   625 	 */

   626 	cpuid_pass2(CPU);

   627 

   628 	(void) check_boot_version(BOP_GETVERSION(bootops));

   629 

   630 	/*

   631 	 * Check for prom_debug in boot environment

   632 	 */

   633 	if (BOP_GETPROPLEN(bootops, "prom_debug") >= 0) {

   634 		++prom_debug;

   635 		PRM_POINT("prom_debug found in boot enviroment");

   636 	}

   637 

   638 	/*

   639 	 * Collect node, cpu and memory configuration information.

   640 	 */

   641 	get_system_configuration();

   642 

   643 	/*

   644 	 * Halt if this is an unsupported processor.

   645 	 */

   646 	if (x86_type == X86_TYPE_486 || x86_type == X86_TYPE_CYRIX_486) {

   647 		printf("\n486 processor (\"%s\") detected.\n",

   648 		    CPU->cpu_brandstr);

   649 		halt("This processor is not supported by this release "

   650 		    "of Solaris.");

   651 	}

   652 

   653 	/*

   654 	 * Set up dummy values till psm spl code installed

   655 	 */

   656 	CPU->cpu_pri_data = dummy_cpu_pri;

   657 

   658 	PRM_POINT("startup_init() done");

   659 }

   660 

   661 /*

   662  * Callback for copy_memlist_filter() to filter nucleus, kadb/kmdb, (ie.

   663  * everything mapped above KERNEL_TEXT) pages from phys_avail. Note it

   664  * also filters out physical page zero.  There is some reliance on the

   665  * boot loader allocating only a few contiguous physical memory chunks.

   666  */

   667 static void

   668 avail_filter(uint64_t *addr, uint64_t *size)

   669 {

   670 	uintptr_t va;

   671 	uintptr_t next_va;

   672 	pfn_t pfn;

   673 	uint64_t pfn_addr;

   674 	uint64_t pfn_eaddr;

   675 	uint_t prot;

   676 	size_t len;

   677 	uint_t change;

   678 

   679 	if (prom_debug)

   680 		prom_printf("\tFilter: in: a=%" PRIx64 ", s=%" PRIx64 "\n",

   681 		    *addr, *size);

   682 

   683 	/*

   684 	 * page zero is required for BIOS.. never make it available

   685 	 */

   686 	if (*addr == 0) {

   687 		*addr += MMU_PAGESIZE;

   688 		*size -= MMU_PAGESIZE;

   689 	}

   690 

   691 	/*

   692 	 * First we trim from the front of the range. Since hat_boot_probe()

   693 	 * walks ranges in virtual order, but addr/size are physical, we need

   694 	 * to the list until no changes are seen.  This deals with the case

   695 	 * where page "p" is mapped at v, page "p + PAGESIZE" is mapped at w

   696 	 * but w < v.

   697 	 */

   698 	do {

   699 		change = 0;

   700 		for (va = KERNEL_TEXT;

   701 		    *size > 0 && hat_boot_probe(&va, &len, &pfn, &prot) != 0;

   702 		    va = next_va) {

   703 

   704 			next_va = va + len;

   705 			pfn_addr = ptob((uint64_t)pfn);

   706 			pfn_eaddr = pfn_addr + len;

   707 

   708 			if (pfn_addr <= *addr && pfn_eaddr > *addr) {

   709 				change = 1;

   710 				while (*size > 0 && len > 0) {

   711 					*addr += MMU_PAGESIZE;

   712 					*size -= MMU_PAGESIZE;

   713 					len -= MMU_PAGESIZE;

   714 				}

   715 			}

   716 		}

   717 		if (change && prom_debug)

   718 			prom_printf("\t\ttrim: a=%" PRIx64 ", s=%" PRIx64 "\n",

   719 			    *addr, *size);

   720 	} while (change);

   721 

   722 	/*

   723 	 * Trim pages from the end of the range.

   724 	 */

   725 	for (va = KERNEL_TEXT;

   726 	    *size > 0 && hat_boot_probe(&va, &len, &pfn, &prot) != 0;

   727 	    va = next_va) {

   728 

   729 		next_va = va + len;

   730 		pfn_addr = ptob((uint64_t)pfn);

   731 

   732 		if (pfn_addr >= *addr && pfn_addr < *addr + *size)

   733 			*size = pfn_addr - *addr;

   734 	}

   735 

   736 	if (prom_debug)

   737 		prom_printf("\tFilter out: a=%" PRIx64 ", s=%" PRIx64 "\n",

   738 		    *addr, *size);

   739 }

   740 

   741 static void

   742 kpm_init()

   743 {

   744 	struct segkpm_crargs b;

   745 	uintptr_t start, end;

   746 	struct memlist	*pmem;

   747 

   748 	/*

   749 	 * These variables were all designed for sfmmu in which segkpm is

   750 	 * mapped using a single pagesize - either 8KB or 4MB.  On x86, we

   751 	 * might use 2+ page sizes on a single machine, so none of these

   752 	 * variables have a single correct value.  They are set up as if we

   753 	 * always use a 4KB pagesize, which should do no harm.  In the long

   754 	 * run, we should get rid of KPM's assumption that only a single

   755 	 * pagesize is used.

   756 	 */

   757 	kpm_pgshft = MMU_PAGESHIFT;

   758 	kpm_pgsz =  MMU_PAGESIZE;

   759 	kpm_pgoff = MMU_PAGEOFFSET;

   760 	kpmp2pshft = 0;

   761 	kpmpnpgs = 1;

   762 	ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0);

   763 

   764 	PRM_POINT("about to create segkpm");

   765 	rw_enter(&kas.a_lock, RW_WRITER);

   766 

   767 	if (seg_attach(&kas, kpm_vbase, kpm_size, segkpm) < 0)

   768 		panic("cannot attach segkpm");

   769 

   770 	b.prot = PROT_READ | PROT_WRITE;

   771 	b.nvcolors = 1;

   772 

   773 	if (segkpm_create(segkpm, (caddr_t)&b) != 0)

   774 		panic("segkpm_create segkpm");

   775 

   776 	rw_exit(&kas.a_lock);

   777 

   778 	/*

   779 	 * Map each of the memsegs into the kpm segment, coalesing adjacent

   780 	 * memsegs to allow mapping with the largest possible pages.

   781 	 */

   782 	pmem = phys_install;

   783 	start = pmem->address;

   784 	end = start + pmem->size;

   785 	for (;;) {

   786 		if (pmem == NULL || pmem->address > end) {

   787 			hat_devload(kas.a_hat, kpm_vbase + start,

   788 			    end - start, mmu_btop(start),

   789 			    PROT_READ | PROT_WRITE,

   790 			    HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);

   791 			if (pmem == NULL)

   792 				break;

   793 			start = pmem->address;

   794 		}

   795 		end = pmem->address + pmem->size;

   796 		pmem = pmem->next;

   797 	}

   798 }

   799 

   800 /*

   801  * The purpose of startup memlist is to get the system to the

   802  * point where it can use kmem_alloc()'s that operate correctly

   803  * relying on BOP_ALLOC(). This includes allocating page_ts,

   804  * page hash table, vmem initialized, etc.

   805  *

   806  * Boot's versions of physinstalled and physavail are insufficient for

   807  * the kernel's purposes. Specifically we don't know which pages that

   808  * are not in physavail can be reclaimed after boot is gone.

   809  *

   810  * This code solves the problem by dividing the address space

   811  * into 3 regions as it takes over the MMU from the booter.

   812  *

   813  * 1) Any (non-nucleus) pages that are mapped at addresses above KERNEL_TEXT

   814  * can not be used by the kernel.

   815  *

   816  * 2) Any free page that happens to be mapped below kernelbase

   817  * is protected until the boot loader is released, but will then be reclaimed.

   818  *

   819  * 3) Boot shouldn't use any address in the remaining area between kernelbase

   820  * and KERNEL_TEXT.

   821  *

   822  * In the case of multiple mappings to the same page, region 1 has precedence

   823  * over region 2.

   824  */

   825 static void

   826 startup_memlist(void)

   827 {

   828 	size_t memlist_sz;

   829 	size_t memseg_sz;

   830 	size_t pagehash_sz;

   831 	size_t pp_sz;

   832 	uintptr_t va;

   833 	size_t len;

   834 	uint_t prot;

   835 	pfn_t pfn;

   836 	int memblocks;

   837 	caddr_t pagecolor_mem;

   838 	size_t pagecolor_memsz;

   839 	caddr_t page_ctrs_mem;

   840 	size_t page_ctrs_size;

   841 	struct memlist *current;

   842 	extern void startup_build_mem_nodes(struct memlist *);

   843 

   844 	/* XX64 fix these - they should be in include files */

   845 	extern ulong_t cr4_value;

   846 	extern size_t page_coloring_init(uint_t, int, int);

   847 	extern void page_coloring_setup(caddr_t);

   848 

   849 	PRM_POINT("startup_memlist() starting...");

   850 

   851 	/*

   852 	 * Take the most current snapshot we can by calling mem-update.

   853 	 * For this to work properly, we first have to ask boot for its

   854 	 * end address.

   855 	 */

   856 	if (BOP_GETPROPLEN(bootops, "memory-update") == 0)

   857 		(void) BOP_GETPROP(bootops, "memory-update", NULL);

   858 

   859 	/*

   860 	 * find if the kernel is mapped on a large page

   861 	 */

   862 	va = KERNEL_TEXT;

   863 	if (hat_boot_probe(&va, &len, &pfn, &prot) == 0)

   864 		panic("Couldn't find kernel text boot mapping");

   865 

   866 	/*

   867 	 * Use leftover large page nucleus text/data space for loadable modules.

   868 	 * Use at most MODTEXT/MODDATA.

   869 	 */

   870 	if (len > MMU_PAGESIZE) {

   871 

   872 		moddata = (caddr_t)ROUND_UP_PAGE(e_data);

   873 		e_moddata = (caddr_t)ROUND_UP_4MEG(e_data);

   874 		if (e_moddata - moddata > MODDATA)

   875 			e_moddata = moddata + MODDATA;

   876 

   877 		modtext = (caddr_t)ROUND_UP_PAGE(e_text);

   878 		e_modtext = (caddr_t)ROUND_UP_4MEG(e_text);

   879 		if (e_modtext - modtext > MODTEXT)

   880 			e_modtext = modtext + MODTEXT;

   881 

   882 

   883 	} else {

   884 

   885 		PRM_POINT("Kernel NOT loaded on Large Page!");

   886 		e_moddata = moddata = (caddr_t)ROUND_UP_PAGE(e_data);

   887 		e_modtext = modtext = (caddr_t)ROUND_UP_PAGE(e_text);

   888 

   889 	}

   890 	econtig = e_moddata;

   891 

   892 	PRM_DEBUG(modtext);

   893 	PRM_DEBUG(e_modtext);

   894 	PRM_DEBUG(moddata);

   895 	PRM_DEBUG(e_moddata);

   896 	PRM_DEBUG(econtig);

   897 

   898 	/*

   899 	 * For MP machines cr4_value must be set or the non-boot

   900 	 * CPUs will not be able to start.

   901 	 */

   902 	if (x86_feature & X86_LARGEPAGE)

   903 		cr4_value = getcr4();

   904 	PRM_DEBUG(cr4_value);

   905 

   906 	/*

   907 	 * Examine the boot loaders physical memory map to find out:

   908 	 * - total memory in system - physinstalled

   909 	 * - the max physical address - physmax

   910 	 * - the number of segments the intsalled memory comes in

   911 	 */

   912 	if (prom_debug)

   913 		print_boot_memlist("boot physinstalled",

   914 		    bootops->boot_mem->physinstalled);

   915 	installed_top_size(bootops->boot_mem->physinstalled, &physmax,

   916 	    &physinstalled, &memblocks);

   917 	PRM_DEBUG(physmax);

   918 	PRM_DEBUG(physinstalled);

   919 	PRM_DEBUG(memblocks);

   920 

   921 	if (prom_debug)

   922 		print_boot_memlist("boot physavail",

   923 		    bootops->boot_mem->physavail);

   924 

   925 	/*

   926 	 * Initialize hat's mmu parameters.

   927 	 * Check for enforce-prot-exec in boot environment. It's used to

   928 	 * enable/disable support for the page table entry NX bit.

   929 	 * The default is to enforce PROT_EXEC on processors that support NX.

   930 	 * Boot seems to round up the "len", but 8 seems to be big enough.

   931 	 */

   932 	mmu_init();

   933 

   934 #ifdef	__i386

   935 	/*

   936 	 * physmax is lowered if there is more memory than can be

   937 	 * physically addressed in 32 bit (PAE/non-PAE) modes.

   938 	 */

   939 	if (mmu.pae_hat) {

   940 		if (PFN_ABOVE64G(physmax)) {

   941 			physinstalled -= (physmax - (PFN_64G - 1));

   942 			physmax = PFN_64G - 1;

   943 		}

   944 	} else {

   945 		if (PFN_ABOVE4G(physmax)) {

   946 			physinstalled -= (physmax - (PFN_4G - 1));

   947 			physmax = PFN_4G - 1;

   948 		}

   949 	}

   950 #endif

   951 

   952 	startup_build_mem_nodes(bootops->boot_mem->physinstalled);

   953 

   954 	if (BOP_GETPROPLEN(bootops, "enforce-prot-exec") >= 0) {

   955 		int len = BOP_GETPROPLEN(bootops, "enforce-prot-exec");

   956 		char value[8];

   957 

   958 		if (len < 8)

   959 			(void) BOP_GETPROP(bootops, "enforce-prot-exec", value);

   960 		else

   961 			(void) strcpy(value, "");

   962 		if (strcmp(value, "off") == 0)

   963 			mmu.pt_nx = 0;

   964 	}

   965 	PRM_DEBUG(mmu.pt_nx);

   966 

   967 	/*

   968 	 * We will need page_t's for every page in the system, except for

   969 	 * memory mapped at or above above the start of the kernel text segment.

   970 	 *

   971 	 * pages above e_modtext are attributed to kernel debugger (obp_pages)

   972 	 */

   973 	npages = physinstalled - 1; /* avail_filter() skips page 0, so "- 1" */

   974 	obp_pages = 0;

   975 	va = KERNEL_TEXT;

   976 	while (hat_boot_probe(&va, &len, &pfn, &prot) != 0) {

   977 		npages -= len >> MMU_PAGESHIFT;

   978 		if (va >= (uintptr_t)e_moddata)

   979 			obp_pages += len >> MMU_PAGESHIFT;

   980 		va += len;

   981 	}

   982 	PRM_DEBUG(npages);

   983 	PRM_DEBUG(obp_pages);

   984 

   985 	/*

   986 	 * If physmem is patched to be non-zero, use it instead of

   987 	 * the computed value unless it is larger than the real

   988 	 * amount of memory on hand.

   989 	 */

   990 	if (physmem == 0 || physmem > npages)

   991 		physmem = npages;

   992 	else

   993 		npages = physmem;

   994 	PRM_DEBUG(physmem);

   995 

   996 	/*

   997 	 * We now compute the sizes of all the  initial allocations for

   998 	 * structures the kernel needs in order do kmem_alloc(). These

   999 	 * include:

  1000 	 *	memsegs

  1001 	 *	memlists

  1002 	 *	page hash table

  1003 	 *	page_t's

  1004 	 *	page coloring data structs

  1005 	 */

  1006 	memseg_sz = sizeof (struct memseg) * (memblocks + POSS_NEW_FRAGMENTS);

  1007 	ADD_TO_ALLOCATIONS(memseg_base, memseg_sz);

  1008 	PRM_DEBUG(memseg_sz);

  1009 

  1010 	/*

  1011 	 * Reserve space for phys_avail/phys_install memlists.

  1012 	 * There's no real good way to know exactly how much room we'll need,

  1013 	 * but this should be a good upper bound.

  1014 	 */

  1015 	memlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) *

  1016 	    (memblocks + POSS_NEW_FRAGMENTS));

  1017 	ADD_TO_ALLOCATIONS(memlist, memlist_sz);

  1018 	PRM_DEBUG(memlist_sz);

  1019 

  1020 	/*

  1021 	 * The page structure hash table size is a power of 2

  1022 	 * such that the average hash chain length is PAGE_HASHAVELEN.

  1023 	 */

  1024 	page_hashsz = npages / PAGE_HASHAVELEN;

  1025 	page_hashsz = 1 << highbit(page_hashsz);

  1026 	pagehash_sz = sizeof (struct page *) * page_hashsz;

  1027 	ADD_TO_ALLOCATIONS(page_hash, pagehash_sz);

  1028 	PRM_DEBUG(pagehash_sz);

  1029 

  1030 	/*

  1031 	 * Set aside room for the page structures themselves.  Note: on

  1032 	 * 64-bit systems we don't allocate page_t's for every page here.

  1033 	 * We just allocate enough to map the lowest 4GB of physical

  1034 	 * memory, minus those pages that are used for the "nucleus" kernel

  1035 	 * text and data.  The remaining pages are allocated once we can

  1036 	 * map around boot.

  1037 	 *

  1038 	 * boot_npages is used to allocate an area big enough for our

  1039 	 * initial page_t's. kphym_init may use less than that.

  1040 	 */

  1041 	boot_npages = npages;

  1042 #if defined(__amd64)

  1043 	if (npages > mmu_btop(FOURGB - (econtig - s_text)))

  1044 		boot_npages = mmu_btop(FOURGB - (econtig - s_text));

  1045 #endif

  1046 	PRM_DEBUG(boot_npages);

  1047 	pp_sz = sizeof (struct page) * boot_npages;

  1048 	ADD_TO_ALLOCATIONS(pp_base, pp_sz);

  1049 	PRM_DEBUG(pp_sz);

  1050 

  1051 	/*

  1052 	 * determine l2 cache info and memory size for page coloring

  1053 	 */

  1054 	(void) getl2cacheinfo(CPU,

  1055 	    &l2cache_sz, &l2cache_linesz, &l2cache_assoc);

  1056 	pagecolor_memsz =

  1057 	    page_coloring_init(l2cache_sz, l2cache_linesz, l2cache_assoc);

  1058 	ADD_TO_ALLOCATIONS(pagecolor_mem, pagecolor_memsz);

  1059 	PRM_DEBUG(pagecolor_memsz);

  1060 

  1061 	page_ctrs_size = page_ctrs_sz();

  1062 	ADD_TO_ALLOCATIONS(page_ctrs_mem, page_ctrs_size);

  1063 	PRM_DEBUG(page_ctrs_size);

  1064 

  1065 	/*

  1066 	 * valloc_base will be below kernel text

  1067 	 * The extra pages are for the HAT and kmdb to map page tables.

  1068 	 */

  1069 	valloc_sz = ROUND_UP_LPAGE(valloc_sz);

  1070 	valloc_base = KERNEL_TEXT - valloc_sz;

  1071 	PRM_DEBUG(valloc_base);

  1072 	ptable_va = valloc_base - ptable_sz;

  1073 

  1074 #if defined(__amd64)

  1075 	if (eprom_kernelbase && eprom_kernelbase != KERNELBASE)

  1076 		cmn_err(CE_NOTE, "!kernelbase cannot be changed on 64-bit "

  1077 		    "systems.");

  1078 	kernelbase = (uintptr_t)KERNELBASE;

  1079 	core_base = (uintptr_t)COREHEAP_BASE;

  1080 	core_size = ptable_va - core_base;

  1081 #else	/* __i386 */

  1082 	/*

  1083 	 * We configure kernelbase based on:

  1084 	 *

  1085 	 * 1. user specified kernelbase via eeprom command. Value cannot exceed

  1086 	 *    KERNELBASE_MAX. we large page align eprom_kernelbase

  1087 	 *

  1088 	 * 2. Default to KERNELBASE and adjust to 2X less the size for page_t.

  1089 	 *    On large memory systems we must lower kernelbase to allow

  1090 	 *    enough room for page_t's for all of memory.

  1091 	 *

  1092 	 * The value set here, might be changed a little later.

  1093 	 */

  1094 	if (eprom_kernelbase) {

  1095 		kernelbase = eprom_kernelbase & mmu.level_mask[1];

  1096 		if (kernelbase > KERNELBASE_MAX)

  1097 			kernelbase = KERNELBASE_MAX;

  1098 	} else {

  1099 		kernelbase = (uintptr_t)KERNELBASE;

  1100 		kernelbase -= ROUND_UP_4MEG(2 * valloc_sz);

  1101 	}

  1102 	ASSERT((kernelbase & mmu.level_offset[1]) == 0);

  1103 	core_base = ptable_va;

  1104 	core_size = 0;

  1105 #endif

  1106 

  1107 	PRM_DEBUG(kernelbase);

  1108 	PRM_DEBUG(core_base);

  1109 	PRM_DEBUG(core_size);

  1110 

  1111 	/*

  1112 	 * At this point, we can only use a portion of the kernelheap that

  1113 	 * will be available after we boot.  Both 32-bit and 64-bit systems

  1114 	 * have this limitation, although the reasons are completely

  1115 	 * different.

  1116 	 *

  1117 	 * On 64-bit systems, the booter only supports allocations in the

  1118 	 * upper 4GB of memory, so we have to work with a reduced kernel

  1119 	 * heap until we take over all allocations.  The booter also sits

  1120 	 * in the lower portion of that 4GB range, so we have to raise the

  1121 	 * bottom of the heap even further.

  1122 	 *

  1123 	 * On 32-bit systems we have to leave room to place segmap below

  1124 	 * the heap.  We don't yet know how large segmap will be, so we

  1125 	 * have to be very conservative.

  1126 	 */

  1127 #if defined(__amd64)

  1128 	/*

  1129 	 * XX64: For now, we let boot have the lower 2GB of the top 4GB

  1130 	 * address range.  In the long run, that should be fixed.  It's

  1131 	 * insane for a booter to need 2 2GB address ranges.

  1132 	 */

  1133 	boot_kernelheap = (caddr_t)(BOOT_DOUBLEMAP_BASE + BOOT_DOUBLEMAP_SIZE);

  1134 	segmap_reserved = 0;

  1135