/*

 * CDDL HEADER START

 *

 * The contents of this file are subject to the terms of the

 * Common Development and Distribution License (the "License").

 * You may not use this file except in compliance with the License.

 *

 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

 * or http://www.opensolaris.org/os/licensing.

 * See the License for the specific language governing permissions

 * and limitations under the License.

 *

 * When distributing Covered Code, include this CDDL HEADER in each

 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.

 * If applicable, add the following below this CDDL HEADER, with the

 * fields enclosed by brackets "[]" replaced with your own identifying

 * information: Portions Copyright [yyyy] [name of copyright owner]

 *

 * CDDL HEADER END

 */

/*

 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.

 * Use is subject to license terms.

 */

#include <sys/types.h>

#include <sys/thread.h>

#include <sys/cpuvar.h>

#include <sys/t_lock.h>

#include <sys/param.h>

#include <sys/proc.h>

#include <sys/disp.h>

#include <sys/class.h>

#include <sys/cmn_err.h>

#include <sys/debug.h>

#include <sys/asm_linkage.h>

#include <sys/x_call.h>

#include <sys/systm.h>

#include <sys/var.h>

#include <sys/vtrace.h>

#include <vm/hat.h>

#include <vm/as.h>

#include <vm/seg_kmem.h>

#include <vm/seg_kp.h>

#include <sys/segments.h>

#include <sys/kmem.h>

#include <sys/stack.h>

#include <sys/smp_impldefs.h>

#include <sys/x86_archext.h>

#include <sys/machsystm.h>

#include <sys/traptrace.h>

#include <sys/clock.h>

#include <sys/cpc_impl.h>

#include <sys/pg.h>

#include <sys/cmt.h>

#include <sys/dtrace.h>

#include <sys/archsystm.h>

#include <sys/fp.h>

#include <sys/reboot.h>

#include <sys/kdi_machimpl.h>

#include <vm/hat_i86.h>

#include <sys/memnode.h>

#include <sys/pci_cfgspace.h>

#include <sys/mach_mmu.h>

#include <sys/sysmacros.h>

#if defined(__xpv)

#include <sys/hypervisor.h>

#endif

#include <sys/cpu_module.h>

struct cpu	cpus[1];			/* CPU data */

struct cpu	*cpu[NCPU] = {&cpus[0]};	/* pointers to all CPUs */

cpu_core_t	cpu_core[NCPU];			/* cpu_core structures */

/*

 * Useful for disabling MP bring-up on a MP capable system.

 */

int use_mp = 1;

/*

 * to be set by a PSM to indicate what cpus

 * are sitting around on the system.

 */

cpuset_t mp_cpus;

/*

 * This variable is used by the hat layer to decide whether or not

 * critical sections are needed to prevent race conditions.  For sun4m,

 * this variable is set once enough MP initialization has been done in

 * order to allow cross calls.

 */

int flushes_require_xcalls;

cpuset_t cpu_ready_set;		/* initialized in startup() */

static 	void	mp_startup(void);

static void cpu_sep_enable(void);

static void cpu_sep_disable(void);

static void cpu_asysc_enable(void);

static void cpu_asysc_disable(void);

/*

 * Init CPU info - get CPU type info for processor_info system call.

 */

void

init_cpu_info(struct cpu *cp)

{

	processor_info_t *pi = &cp->cpu_type_info;

	char buf[CPU_IDSTRLEN];

	/*

	 * Get clock-frequency property for the CPU.

	 */

	pi->pi_clock = cpu_freq;

	/*

	 * Current frequency in Hz.

	 */

	cp->cpu_curr_clock = cpu_freq_hz;

	/*

	 * Supported frequencies.

	 */

	cpu_set_supp_freqs(cp, NULL);

	(void) strcpy(pi->pi_processor_type, "i386");

	if (fpu_exists)

		(void) strcpy(pi->pi_fputypes, "i387 compatible");

	(void) cpuid_getidstr(cp, buf, sizeof (buf));

	cp->cpu_idstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP);

	(void) strcpy(cp->cpu_idstr, buf);

	cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr);

	(void) cpuid_getbrandstr(cp, buf, sizeof (buf));

	cp->cpu_brandstr = kmem_alloc(strlen(buf) + 1, KM_SLEEP);

	(void) strcpy(cp->cpu_brandstr, buf);

	cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr);

}

/*

 * Configure syscall support on this CPU.

 */

/*ARGSUSED*/

void

init_cpu_syscall(struct cpu *cp)

{

	kpreempt_disable();

#if defined(__amd64)

	if ((x86_feature & (X86_MSR | X86_ASYSC)) == (X86_MSR | X86_ASYSC)) {

#if !defined(__lint)

		/*

		 * The syscall instruction imposes a certain ordering on

		 * segment selectors, so we double-check that ordering

		 * here.

		 */

		ASSERT(KDS_SEL == KCS_SEL + 8);

		ASSERT(UDS_SEL == U32CS_SEL + 8);

		ASSERT(UCS_SEL == U32CS_SEL + 16);

#endif

		/*

		 * Turn syscall/sysret extensions on.

		 */

		cpu_asysc_enable();

		/*

		 * Program the magic registers ..

		 */

		wrmsr(MSR_AMD_STAR,

		    ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32);

		wrmsr(MSR_AMD_LSTAR, (uint64_t)(uintptr_t)sys_syscall);

		wrmsr(MSR_AMD_CSTAR, (uint64_t)(uintptr_t)sys_syscall32);

		/*

		 * This list of flags is masked off the incoming

		 * %rfl when we enter the kernel.

		 */

		wrmsr(MSR_AMD_SFMASK, (uint64_t)(uintptr_t)(PS_IE | PS_T));

	}

#endif

	/*

	 * On 32-bit kernels, we use sysenter/sysexit because it's too

	 * hard to use syscall/sysret, and it is more portable anyway.

	 *

	 * On 64-bit kernels on Nocona machines, the 32-bit syscall

	 * variant isn't available to 32-bit applications, but sysenter is.

	 */

	if ((x86_feature & (X86_MSR | X86_SEP)) == (X86_MSR | X86_SEP)) {

#if !defined(__lint)

		/*

		 * The sysenter instruction imposes a certain ordering on

		 * segment selectors, so we double-check that ordering

		 * here. See "sysenter" in Intel document 245471-012, "IA-32

		 * Intel Architecture Software Developer's Manual Volume 2:

		 * Instruction Set Reference"

		 */

		ASSERT(KDS_SEL == KCS_SEL + 8);

		ASSERT32(UCS_SEL == ((KCS_SEL + 16) | 3));

		ASSERT32(UDS_SEL == UCS_SEL + 8);

		ASSERT64(U32CS_SEL == ((KCS_SEL + 16) | 3));

		ASSERT64(UDS_SEL == U32CS_SEL + 8);

#endif

		cpu_sep_enable();

		/*

		 * resume() sets this value to the base of the threads stack

		 * via a context handler.

		 */

		wrmsr(MSR_INTC_SEP_ESP, 0);

		wrmsr(MSR_INTC_SEP_EIP, (uint64_t)(uintptr_t)sys_sysenter);

	}

	kpreempt_enable();

}

/*

 * Multiprocessor initialization.

 *

 * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the

 * startup and idle threads for the specified CPU.

 */

struct cpu *

mp_startup_init(int cpun)

{

	struct cpu *cp;

	kthread_id_t tp;

	caddr_t	sp;

	proc_t *procp;

#if !defined(__xpv)

	extern int idle_cpu_prefer_mwait;

#endif

	extern void idle();

#ifdef TRAPTRACE

	trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun];

#endif

	ASSERT(cpun < NCPU && cpu[cpun] == NULL);

	cp = kmem_zalloc(sizeof (*cp), KM_SLEEP);

#if !defined(__xpv)

	if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait)

		cp->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU);

#endif

	procp = curthread->t_procp;

	mutex_enter(&cpu_lock);

	/*

	 * Initialize the dispatcher first.

	 */

	disp_cpu_init(cp);

	mutex_exit(&cpu_lock);

	cpu_vm_data_init(cp);

	/*

	 * Allocate and initialize the startup thread for this CPU.

	 * Interrupt and process switch stacks get allocated later

	 * when the CPU starts running.

	 */

	tp = thread_create(NULL, 0, NULL, NULL, 0, procp,

	    TS_STOPPED, maxclsyspri);

	/*

	 * Set state to TS_ONPROC since this thread will start running

	 * as soon as the CPU comes online.

	 *

	 * All the other fields of the thread structure are setup by

	 * thread_create().

	 */

	THREAD_ONPROC(tp, cp);

	tp->t_preempt = 1;

	tp->t_bound_cpu = cp;

	tp->t_affinitycnt = 1;

	tp->t_cpu = cp;

	tp->t_disp_queue = cp->cpu_disp;

	/*

	 * Setup thread to start in mp_startup.

	 */

	sp = tp->t_stk;

	tp->t_pc = (uintptr_t)mp_startup;

	tp->t_sp = (uintptr_t)(sp - MINFRAME);

#if defined(__amd64)

	tp->t_sp -= STACK_ENTRY_ALIGN;		/* fake a call */

#endif

	cp->cpu_id = cpun;

	cp->cpu_self = cp;

	cp->cpu_thread = tp;

	cp->cpu_lwp = NULL;

	cp->cpu_dispthread = tp;

	cp->cpu_dispatch_pri = DISP_PRIO(tp);

	/*

	 * cpu_base_spl must be set explicitly here to prevent any blocking

	 * operations in mp_startup from causing the spl of the cpu to drop

	 * to 0 (allowing device interrupts before we're ready) in resume().

	 * cpu_base_spl MUST remain at LOCK_LEVEL until the cpu is CPU_READY.

	 * As an extra bit of security on DEBUG kernels, this is enforced with

	 * an assertion in mp_startup() -- before cpu_base_spl is set to its

	 * proper value.

	 */

	cp->cpu_base_spl = ipltospl(LOCK_LEVEL);

	/*

	 * Now, initialize per-CPU idle thread for this CPU.

	 */

	tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1);

	cp->cpu_idle_thread = tp;

	tp->t_preempt = 1;

	tp->t_bound_cpu = cp;

	tp->t_affinitycnt = 1;

	tp->t_cpu = cp;

	tp->t_disp_queue = cp->cpu_disp;

	/*

	 * Bootstrap the CPU's PG data

	 */

	pg_cpu_bootstrap(cp);

	/*

	 * Perform CPC initialization on the new CPU.

	 */

	kcpc_hw_init(cp);

	/*

	 * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2

	 * for each CPU.

	 */

	setup_vaddr_for_ppcopy(cp);

	/*

	 * Allocate page for new GDT and initialize from current GDT.

	 */

#if !defined(__lint)

	ASSERT((sizeof (*cp->cpu_gdt) * NGDT) <= PAGESIZE);

#endif

	cp->cpu_gdt = kmem_zalloc(PAGESIZE, KM_SLEEP);

	bcopy(CPU->cpu_gdt, cp->cpu_gdt, (sizeof (*cp->cpu_gdt) * NGDT));

#if defined(__i386)

	/*

	 * setup kernel %gs.

	 */

	set_usegd(&cp->cpu_gdt[GDT_GS], cp, sizeof (struct cpu) -1, SDT_MEMRWA,

	    SEL_KPL, 0, 1);

#endif

	/*

	 * If we have more than one node, each cpu gets a copy of IDT

	 * local to its node. If this is a Pentium box, we use cpu 0's

	 * IDT. cpu 0's IDT has been made read-only to workaround the

	 * cmpxchgl register bug

	 */

	if (system_hardware.hd_nodes && x86_type != X86_TYPE_P5) {

#if !defined(__lint)

		ASSERT((sizeof (*CPU->cpu_idt) * NIDT) <= PAGESIZE);

#endif

		cp->cpu_idt = kmem_zalloc(PAGESIZE, KM_SLEEP);

		bcopy(CPU->cpu_idt, cp->cpu_idt, PAGESIZE);

	} else {

		cp->cpu_idt = CPU->cpu_idt;

	}

	/*

	 * Get interrupt priority data from cpu 0.

	 */

	cp->cpu_pri_data = CPU->cpu_pri_data;

	/*

	 * alloc space for cpuid info

	 */

	cpuid_alloc_space(cp);

	/*

	 * alloc space for ucode_info

	 */

	ucode_alloc_space(cp);

	hat_cpu_online(cp);

#ifdef TRAPTRACE

	/*

	 * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers

	 */

	ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP);

	ttc->ttc_next = ttc->ttc_first;

	ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize;

#endif

	/*

	 * Record that we have another CPU.

	 */

	mutex_enter(&cpu_lock);

	/*

	 * Initialize the interrupt threads for this CPU

	 */

	cpu_intr_alloc(cp, NINTR_THREADS);

	/*

	 * Add CPU to list of available CPUs.  It'll be on the active list

	 * after mp_startup().

	 */

	cpu_add_unit(cp);

	mutex_exit(&cpu_lock);

	return (cp);

}

/*

 * Undo what was done in mp_startup_init

 */

static void

mp_startup_fini(struct cpu *cp, int error)

{

	mutex_enter(&cpu_lock);

	/*

	 * Remove the CPU from the list of available CPUs.

	 */

	cpu_del_unit(cp->cpu_id);

	if (error == ETIMEDOUT) {

		/*

		 * The cpu was started, but never *seemed* to run any

		 * code in the kernel; it's probably off spinning in its

		 * own private world, though with potential references to

		 * our kmem-allocated IDTs and GDTs (for example).

		 *

		 * Worse still, it may actually wake up some time later,

		 * so rather than guess what it might or might not do, we

		 * leave the fundamental data structures intact.

		 */

		cp->cpu_flags = 0;

		mutex_exit(&cpu_lock);

		return;

	}

	/*

	 * At this point, the only threads bound to this CPU should

	 * special per-cpu threads: it's idle thread, it's pause threads,

	 * and it's interrupt threads.  Clean these up.

	 */

	cpu_destroy_bound_threads(cp);

	cp->cpu_idle_thread = NULL;

	/*

	 * Free the interrupt stack.

	 */

	segkp_release(segkp,

	    cp->cpu_intr_stack - (INTR_STACK_SIZE - SA(MINFRAME)));

	mutex_exit(&cpu_lock);

#ifdef TRAPTRACE

	/*

	 * Discard the trap trace buffer

	 */

	{

		trap_trace_ctl_t *ttc = &trap_trace_ctl[cp->cpu_id];

		kmem_free((void *)ttc->ttc_first, trap_trace_bufsize);

		ttc->ttc_first = NULL;

	}

#endif

	hat_cpu_offline(cp);

	cpuid_free_space(cp);

	ucode_free_space(cp);

	if (cp->cpu_idt != CPU->cpu_idt)

		kmem_free(cp->cpu_idt, PAGESIZE);

	cp->cpu_idt = NULL;

	kmem_free(cp->cpu_gdt, PAGESIZE);

	cp->cpu_gdt = NULL;

	teardown_vaddr_for_ppcopy(cp);

	kcpc_hw_fini(cp);

	cp->cpu_dispthread = NULL;

	cp->cpu_thread = NULL;	/* discarded by cpu_destroy_bound_threads() */

	cpu_vm_data_destroy(cp);

	mutex_enter(&cpu_lock);

	disp_cpu_fini(cp);

	mutex_exit(&cpu_lock);

#if !defined(__xpv)

	if (cp->cpu_m.mcpu_mwait != NULL)

		cpuid_mwait_free(cp);