PSARC/2005/471 BrandZ: Support for non-native zones
6374606 ::nm -D without an object may not work on processes in zones
6409350 BrandZ project integration into Solaris
6455289 pthread_setschedparam() should return EPERM rather than panic libc
6455591 setpriority(3C) gets errno wrong for deficient privileges failure
6458178 fifofs doesn't support lofs mounts of fifos
6460380 Attempted open() of a symlink with the O_NOFOLLOW flag set returns EINVAL, not ELOOP
6463857 renice(1) errors erroneously
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*/
/*
* Copyright 2006 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
/* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
/* All Rights Reserved */
/* Copyright (c) 1987, 1988 Microsoft Corporation */
/* All Rights Reserved */
#pragma ident "%Z%%M% %I% %E% SMI"
#include <sys/errno.h>
#include <sys/asm_linkage.h>
#if defined(__lint)
#include <sys/types.h>
#include <sys/systm.h>
#else /* __lint */
#include "assym.h"
#endif /* __lint */
#define KCOPY_MIN_SIZE 128 /* Must be >= 16 bytes */
#define XCOPY_MIN_SIZE 128 /* Must be >= 16 bytes */
/*
* Non-temopral access (NTA) alignment requirement
*/
#define NTA_ALIGN_SIZE 4 /* Must be at least 4-byte aligned */
#define NTA_ALIGN_MASK _CONST(NTA_ALIGN_SIZE-1)
#define COUNT_ALIGN_SIZE 16 /* Must be at least 16-byte aligned */
#define COUNT_ALIGN_MASK _CONST(COUNT_ALIGN_SIZE-1)
/*
* Copy a block of storage, returning an error code if `from' or
* `to' takes a kernel pagefault which cannot be resolved.
* Returns errno value on pagefault error, 0 if all ok
*/
#if defined(__lint)
/* ARGSUSED */
int
kcopy(const void *from, void *to, size_t count)
{ return (0); }
#else /* __lint */
.globl kernelbase
#if defined(__amd64)
ENTRY(kcopy)
pushq %rbp
movq %rsp, %rbp
#ifdef DEBUG
movq kernelbase(%rip), %rax
cmpq %rax, %rdi /* %rdi = from */
jb 0f
cmpq %rax, %rsi /* %rsi = to */
jnb 1f
0: leaq .kcopy_panic_msg(%rip), %rdi
xorl %eax, %eax
call panic
1:
#endif
/*
* pass lofault value as 4th argument to do_copy_fault
*/
leaq _kcopy_copyerr(%rip), %rcx
movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
do_copy_fault:
movq T_LOFAULT(%r9), %r11 /* save the current lofault */
movq %rcx, T_LOFAULT(%r9) /* new lofault */
xchgq %rdi, %rsi /* %rsi = source, %rdi = destination */
movq %rdx, %rcx /* %rcx = count */
shrq $3, %rcx /* 8-byte word count */
rep
smovq
movq %rdx, %rcx
andq $7, %rcx /* bytes left over */
rep
smovb
xorl %eax, %eax /* return 0 (success) */
/*
* A fault during do_copy_fault is indicated through an errno value
* in %rax and we iretq from the trap handler to here.
*/
_kcopy_copyerr:
movq %r11, T_LOFAULT(%r9) /* restore original lofault */
leave
ret
SET_SIZE(kcopy)
#elif defined(__i386)
#define ARG_FROM 8
#define ARG_TO 12
#define ARG_COUNT 16
ENTRY(kcopy)
#ifdef DEBUG
pushl %ebp
movl %esp, %ebp
movl kernelbase, %eax
cmpl %eax, ARG_FROM(%ebp)
jb 0f
cmpl %eax, ARG_TO(%ebp)
jnb 1f
0: pushl $.kcopy_panic_msg
call panic
1: popl %ebp
#endif
lea _kcopy_copyerr, %eax /* lofault value */
movl %gs:CPU_THREAD, %edx
do_copy_fault:
pushl %ebp
movl %esp, %ebp /* setup stack frame */
pushl %esi
pushl %edi /* save registers */
movl T_LOFAULT(%edx), %edi
pushl %edi /* save the current lofault */
movl %eax, T_LOFAULT(%edx) /* new lofault */
movl ARG_COUNT(%ebp), %ecx
movl ARG_FROM(%ebp), %esi
movl ARG_TO(%ebp), %edi
shrl $2, %ecx /* word count */
rep
smovl
movl ARG_COUNT(%ebp), %ecx
andl $3, %ecx /* bytes left over */
rep
smovb
xorl %eax, %eax
/*
* A fault during do_copy_fault is indicated through an errno value
* in %eax and we iret from the trap handler to here.
*/
_kcopy_copyerr:
popl %ecx
popl %edi
movl %ecx, T_LOFAULT(%edx) /* restore the original lofault */
popl %esi
popl %ebp
ret
SET_SIZE(kcopy)
#undef ARG_FROM
#undef ARG_TO
#undef ARG_COUNT
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/*
* Copy a block of storage. Similar to kcopy but uses non-temporal
* instructions.
*/
/* ARGSUSED */
int
kcopy_nta(const void *from, void *to, size_t count, int copy_cached)
{ return (0); }
#else /* __lint */
#if defined(__amd64)
#define COPY_LOOP_INIT(src, dst, cnt) \
addq cnt, src; \
addq cnt, dst; \
shrq $3, cnt; \
neg cnt
/* Copy 16 bytes per loop. Uses %rax and %r8 */
#define COPY_LOOP_BODY(src, dst, cnt) \
prefetchnta 0x100(src, cnt, 8); \
movq (src, cnt, 8), %rax; \
movq 0x8(src, cnt, 8), %r8; \
movnti %rax, (dst, cnt, 8); \
movnti %r8, 0x8(dst, cnt, 8); \
addq $2, cnt
ENTRY(kcopy_nta)
pushq %rbp
movq %rsp, %rbp
#ifdef DEBUG
movq kernelbase(%rip), %rax
cmpq %rax, %rdi /* %rdi = from */
jb 0f
cmpq %rax, %rsi /* %rsi = to */
jnb 1f
0: leaq .kcopy_panic_msg(%rip), %rdi
xorl %eax, %eax
call panic
1:
#endif
movq %gs:CPU_THREAD, %r9
cmpq $0, %rcx /* No non-temporal access? */
/*
* pass lofault value as 4th argument to do_copy_fault
*/
leaq _kcopy_nta_copyerr(%rip), %rcx /* doesn't set rflags */
jnz do_copy_fault /* use regular access */
/*
* Make sure cnt is >= KCOPY_MIN_SIZE
*/
cmpq $KCOPY_MIN_SIZE, %rdx
jb do_copy_fault
/*
* Make sure src and dst are NTA_ALIGN_SIZE aligned,
* count is COUNT_ALIGN_SIZE aligned.
*/
movq %rdi, %r10
orq %rsi, %r10
andq $NTA_ALIGN_MASK, %r10
orq %rdx, %r10
andq $COUNT_ALIGN_MASK, %r10
jnz do_copy_fault
ALTENTRY(do_copy_fault_nta)
movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
movq T_LOFAULT(%r9), %r11 /* save the current lofault */
movq %rcx, T_LOFAULT(%r9) /* new lofault */
/*
* COPY_LOOP_BODY uses %rax and %r8
*/
COPY_LOOP_INIT(%rdi, %rsi, %rdx)
2: COPY_LOOP_BODY(%rdi, %rsi, %rdx)
jnz 2b
mfence
xorl %eax, %eax /* return 0 (success) */
_kcopy_nta_copyerr:
movq %r11, T_LOFAULT(%r9) /* restore original lofault */
leave
ret
SET_SIZE(do_copy_fault_nta)
SET_SIZE(kcopy_nta)
#elif defined(__i386)
#define ARG_FROM 8
#define ARG_TO 12
#define ARG_COUNT 16
#define COPY_LOOP_INIT(src, dst, cnt) \
addl cnt, src; \
addl cnt, dst; \
shrl $3, cnt; \
neg cnt
#define COPY_LOOP_BODY(src, dst, cnt) \
prefetchnta 0x100(src, cnt, 8); \
movl (src, cnt, 8), %esi; \
movnti %esi, (dst, cnt, 8); \
movl 0x4(src, cnt, 8), %esi; \
movnti %esi, 0x4(dst, cnt, 8); \
movl 0x8(src, cnt, 8), %esi; \
movnti %esi, 0x8(dst, cnt, 8); \
movl 0xc(src, cnt, 8), %esi; \
movnti %esi, 0xc(dst, cnt, 8); \
addl $2, cnt
/*
* kcopy_nta is not implemented for 32-bit as no performance
* improvement was shown. We simply jump directly to kcopy
* and discard the 4 arguments.
*/
ENTRY(kcopy_nta)
jmp kcopy
lea _kcopy_nta_copyerr, %eax /* lofault value */
ALTENTRY(do_copy_fault_nta)
pushl %ebp
movl %esp, %ebp /* setup stack frame */
pushl %esi
pushl %edi
movl %gs:CPU_THREAD, %edx
movl T_LOFAULT(%edx), %edi
pushl %edi /* save the current lofault */
movl %eax, T_LOFAULT(%edx) /* new lofault */
/* COPY_LOOP_BODY needs to use %esi */
movl ARG_COUNT(%ebp), %ecx
movl ARG_FROM(%ebp), %edi
movl ARG_TO(%ebp), %eax
COPY_LOOP_INIT(%edi, %eax, %ecx)
1: COPY_LOOP_BODY(%edi, %eax, %ecx)
jnz 1b
mfence
xorl %eax, %eax
_kcopy_nta_copyerr:
popl %ecx
popl %edi
movl %ecx, T_LOFAULT(%edx) /* restore the original lofault */
popl %esi
leave
ret
SET_SIZE(do_copy_fault_nta)
SET_SIZE(kcopy_nta)
#undef ARG_FROM
#undef ARG_TO
#undef ARG_COUNT
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/* ARGSUSED */
void
bcopy(const void *from, void *to, size_t count)
{}
#else /* __lint */
#if defined(__amd64)
ENTRY(bcopy)
#ifdef DEBUG
orq %rdx, %rdx /* %rdx = count */
jz 1f
movq kernelbase(%rip), %rax
cmpq %rax, %rdi /* %rdi = from */
jb 0f
cmpq %rax, %rsi /* %rsi = to */
jnb 1f
0: leaq .bcopy_panic_msg(%rip), %rdi
jmp call_panic /* setup stack and call panic */
1:
#endif
do_copy:
xchgq %rdi, %rsi /* %rsi = source, %rdi = destination */
movq %rdx, %rcx /* %rcx = count */
shrq $3, %rcx /* 8-byte word count */
rep
smovq
movq %rdx, %rcx
andq $7, %rcx /* bytes left over */
rep
smovb
ret
#ifdef DEBUG
/*
* Setup frame on the run-time stack. The end of the input argument
* area must be aligned on a 16 byte boundary. The stack pointer %rsp,
* always points to the end of the latest allocated stack frame.
* panic(const char *format, ...) is a varargs function. When a
* function taking variable arguments is called, %rax must be set
* to eight times the number of floating point parameters passed
* to the function in SSE registers.
*/
call_panic:
pushq %rbp /* align stack properly */
movq %rsp, %rbp
xorl %eax, %eax /* no variable arguments */
call panic /* %rdi = format string */
#endif
SET_SIZE(bcopy)
#elif defined(__i386)
#define ARG_FROM 4
#define ARG_TO 8
#define ARG_COUNT 12
ENTRY(bcopy)
#ifdef DEBUG
movl ARG_COUNT(%esp), %eax
orl %eax, %eax
jz 1f
movl kernelbase, %eax
cmpl %eax, ARG_FROM(%esp)
jb 0f
cmpl %eax, ARG_TO(%esp)
jnb 1f
0: pushl %ebp
movl %esp, %ebp
pushl $.bcopy_panic_msg
call panic
1:
#endif
do_copy:
movl %esi, %eax /* save registers */
movl %edi, %edx
movl ARG_COUNT(%esp), %ecx
movl ARG_FROM(%esp), %esi
movl ARG_TO(%esp), %edi
shrl $2, %ecx /* word count */
rep
smovl
movl ARG_COUNT(%esp), %ecx
andl $3, %ecx /* bytes left over */
rep
smovb
movl %eax, %esi /* restore registers */
movl %edx, %edi
ret
SET_SIZE(bcopy)
#undef ARG_COUNT
#undef ARG_FROM
#undef ARG_TO
#endif /* __i386 */
#endif /* __lint */
/*
* Zero a block of storage, returning an error code if we
* take a kernel pagefault which cannot be resolved.
* Returns errno value on pagefault error, 0 if all ok
*/
#if defined(__lint)
/* ARGSUSED */
int
kzero(void *addr, size_t count)
{ return (0); }
#else /* __lint */
#if defined(__amd64)
ENTRY(kzero)
#ifdef DEBUG
cmpq kernelbase(%rip), %rdi /* %rdi = addr */
jnb 0f
leaq .kzero_panic_msg(%rip), %rdi
jmp call_panic /* setup stack and call panic */
0:
#endif
/*
* pass lofault value as 3rd argument to do_zero_fault
*/
leaq _kzeroerr(%rip), %rdx
do_zero_fault:
movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
movq T_LOFAULT(%r9), %r11 /* save the current lofault */
movq %rdx, T_LOFAULT(%r9) /* new lofault */
movq %rsi, %rcx /* get size in bytes */
shrq $3, %rcx /* count of 8-byte words to zero */
xorl %eax, %eax /* clear %rax; used in sstoq / sstob */
rep
sstoq /* %rcx = words to clear (%rax=0) */
movq %rsi, %rcx
andq $7, %rcx /* bytes left over */
rep
sstob /* %rcx = residual bytes to clear */
/*
* A fault during do_zero_fault is indicated through an errno value
* in %rax when we iretq to here.
*/
_kzeroerr:
movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
ret
SET_SIZE(kzero)
#elif defined(__i386)
#define ARG_ADDR 8
#define ARG_COUNT 12
ENTRY(kzero)
#ifdef DEBUG
pushl %ebp
movl %esp, %ebp
movl kernelbase, %eax
cmpl %eax, ARG_ADDR(%ebp)
jnb 0f
pushl $.kzero_panic_msg
call panic
0: popl %ebp
#endif
lea _kzeroerr, %eax /* kzeroerr is lofault value */
do_zero_fault:
pushl %ebp /* save stack base */
movl %esp, %ebp /* set new stack base */
pushl %edi /* save %edi */
mov %gs:CPU_THREAD, %edx
movl T_LOFAULT(%edx), %edi
pushl %edi /* save the current lofault */
movl %eax, T_LOFAULT(%edx) /* new lofault */
movl ARG_COUNT(%ebp), %ecx /* get size in bytes */
movl ARG_ADDR(%ebp), %edi /* %edi <- address of bytes to clear */
shrl $2, %ecx /* Count of double words to zero */
xorl %eax, %eax /* sstol val */
rep
sstol /* %ecx contains words to clear (%eax=0) */
movl ARG_COUNT(%ebp), %ecx /* get size in bytes */
andl $3, %ecx /* do mod 4 */
rep
sstob /* %ecx contains residual bytes to clear */
/*
* A fault during do_zero_fault is indicated through an errno value
* in %eax when we iret to here.
*/
_kzeroerr:
popl %edi
movl %edi, T_LOFAULT(%edx) /* restore the original lofault */
popl %edi
popl %ebp
ret
SET_SIZE(kzero)
#undef ARG_ADDR
#undef ARG_COUNT
#endif /* __i386 */
#endif /* __lint */
/*
* Zero a block of storage.
*/
#if defined(__lint)
/* ARGSUSED */
void
bzero(void *addr, size_t count)
{}
#else /* __lint */
#if defined(__amd64)
ENTRY(bzero)
#ifdef DEBUG
cmpq kernelbase(%rip), %rdi /* %rdi = addr */
jnb 0f
leaq .bzero_panic_msg(%rip), %rdi
jmp call_panic /* setup stack and call panic */
0:
#endif
do_zero:
movq %rsi, %rcx /* get size in bytes */
shrq $3, %rcx /* count of 8-byte words to zero */
xorl %eax, %eax /* clear %rax; used in sstoq / sstob */
rep
sstoq /* %rcx = words to clear (%rax=0) */
movq %rsi, %rcx
andq $7, %rcx /* bytes left over */
rep
sstob /* %rcx = residual bytes to clear */
ret
SET_SIZE(bzero)
#elif defined(__i386)
#define ARG_ADDR 4
#define ARG_COUNT 8
ENTRY(bzero)
#ifdef DEBUG
movl kernelbase, %eax
cmpl %eax, ARG_ADDR(%esp)
jnb 0f
pushl %ebp
movl %esp, %ebp
pushl $.bzero_panic_msg
call panic
0:
#endif
do_zero:
movl %edi, %edx
movl ARG_COUNT(%esp), %ecx
movl ARG_ADDR(%esp), %edi
shrl $2, %ecx
xorl %eax, %eax
rep
sstol
movl ARG_COUNT(%esp), %ecx
andl $3, %ecx
rep
sstob
movl %edx, %edi
ret
SET_SIZE(bzero)
#undef ARG_ADDR
#undef ARG_COUNT
#endif /* __i386 */
#endif /* __lint */
/*
* Transfer data to and from user space -
* Note that these routines can cause faults
* It is assumed that the kernel has nothing at
* less than KERNELBASE in the virtual address space.
*
* Note that copyin(9F) and copyout(9F) are part of the
* DDI/DKI which specifies that they return '-1' on "errors."
*
* Sigh.
*
* So there's two extremely similar routines - xcopyin_nta() and
* xcopyout_nta() which return the errno that we've faithfully computed.
* This allows other callers (e.g. uiomove(9F)) to work correctly.
* Given that these are used pretty heavily, we expand the calling
* sequences inline for all flavours (rather than making wrappers).
*/
/*
* Copy user data to kernel space.
*/
#if defined(__lint)
/* ARGSUSED */
int
copyin(const void *uaddr, void *kaddr, size_t count)
{ return (0); }
#else /* lint */
#if defined(__amd64)
ENTRY(copyin)
pushq %rbp
movq %rsp, %rbp
subq $32, %rsp
/*
* save args in case we trap and need to rerun as a copyop
*/
movq %rdi, (%rsp)
movq %rsi, 0x8(%rsp)
movq %rdx, 0x10(%rsp)
movq kernelbase(%rip), %rax
#ifdef DEBUG
cmpq %rax, %rsi /* %rsi = kaddr */
jnb 1f
leaq .copyin_panic_msg(%rip), %rdi
xorl %eax, %eax
call panic
1:
#endif
/*
* pass lofault value as 4th argument to do_copy_fault
*/
leaq _copyin_err(%rip), %rcx
movq %gs:CPU_THREAD, %r9
cmpq %rax, %rdi /* test uaddr < kernelbase */
jb do_copy_fault
jmp 3f
_copyin_err:
movq %r11, T_LOFAULT(%r9) /* restore original lofault */
3:
movq T_COPYOPS(%r9), %rax
cmpq $0, %rax
jz 2f
/*
* reload args for the copyop
*/
movq (%rsp), %rdi
movq 0x8(%rsp), %rsi
movq 0x10(%rsp), %rdx
leave
jmp *CP_COPYIN(%rax)
2: movl $-1, %eax
leave
ret
SET_SIZE(copyin)
#elif defined(__i386)
#define ARG_UADDR 4
#define ARG_KADDR 8
ENTRY(copyin)
movl kernelbase, %ecx
#ifdef DEBUG
cmpl %ecx, ARG_KADDR(%esp)
jnb 1f
pushl %ebp
movl %esp, %ebp
pushl $.copyin_panic_msg
call panic
1:
#endif
lea _copyin_err, %eax
movl %gs:CPU_THREAD, %edx
cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
jb do_copy_fault
jmp 3f
_copyin_err:
popl %ecx
popl %edi
movl %ecx, T_LOFAULT(%edx) /* restore original lofault */
popl %esi
popl %ebp
3:
movl T_COPYOPS(%edx), %eax
cmpl $0, %eax
jz 2f
jmp *CP_COPYIN(%eax)
2: movl $-1, %eax
ret
SET_SIZE(copyin)
#undef ARG_UADDR
#undef ARG_KADDR
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/* ARGSUSED */
int
xcopyin_nta(const void *uaddr, void *kaddr, size_t count, int copy_cached)
{ return (0); }
#else /* __lint */
#if defined(__amd64)
ENTRY(xcopyin_nta)
pushq %rbp
movq %rsp, %rbp
subq $32, %rsp
/*
* save args in case we trap and need to rerun as a copyop
* %rcx is consumed in this routine so we don't need to save
* it.
*/
movq %rdi, (%rsp)
movq %rsi, 0x8(%rsp)
movq %rdx, 0x10(%rsp)
movq kernelbase(%rip), %rax
#ifdef DEBUG
cmpq %rax, %rsi /* %rsi = kaddr */
jnb 1f
leaq .xcopyin_panic_msg(%rip), %rdi
xorl %eax, %eax
call panic
1:
#endif
movq %gs:CPU_THREAD, %r9
cmpq %rax, %rdi /* test uaddr < kernelbase */
jae 4f
cmpq $0, %rcx /* No non-temporal access? */
/*
* pass lofault value as 4th argument to do_copy_fault
*/
leaq _xcopyin_err(%rip), %rcx /* doesn't set rflags */
jnz do_copy_fault /* use regular access */
/*
* Make sure cnt is >= XCOPY_MIN_SIZE bytes
*/
cmpq $XCOPY_MIN_SIZE, %rdx
jb do_copy_fault
/*
* Make sure src and dst are NTA_ALIGN_SIZE aligned,
* count is COUNT_ALIGN_SIZE aligned.
*/
movq %rdi, %r10
orq %rsi, %r10
andq $NTA_ALIGN_MASK, %r10
orq %rdx, %r10
andq $COUNT_ALIGN_MASK, %r10
jnz do_copy_fault
jmp do_copy_fault_nta /* use non-temporal access */
4:
movl $EFAULT, %eax
jmp 3f
/*
* A fault during do_copy_fault or do_copy_fault_nta is
* indicated through an errno value in %rax and we iret from the
* trap handler to here.
*/
_xcopyin_err:
movq %r11, T_LOFAULT(%r9) /* restore original lofault */
3:
movq T_COPYOPS(%r9), %r8
cmpq $0, %r8
jz 2f
/*
* reload args for the copyop
*/
movq (%rsp), %rdi
movq 0x8(%rsp), %rsi
movq 0x10(%rsp), %rdx
leave
jmp *CP_XCOPYIN(%r8)
2: leave
ret
SET_SIZE(xcopyin_nta)
#elif defined(__i386)
#define ARG_UADDR 4
#define ARG_KADDR 8
#define ARG_COUNT 12
#define ARG_CACHED 16
.globl use_sse_copy
ENTRY(xcopyin_nta)
movl kernelbase, %ecx
lea _xcopyin_err, %eax
movl %gs:CPU_THREAD, %edx
cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
jae 4f
cmpl $0, use_sse_copy /* no sse support */
jz do_copy_fault
cmpl $0, ARG_CACHED(%esp) /* copy_cached hint set? */
jnz do_copy_fault
/*
* Make sure cnt is >= XCOPY_MIN_SIZE bytes
*/
cmpl $XCOPY_MIN_SIZE, ARG_COUNT(%esp)
jb do_copy_fault
/*
* Make sure src and dst are NTA_ALIGN_SIZE aligned,
* count is COUNT_ALIGN_SIZE aligned.
*/
movl ARG_UADDR(%esp), %ecx
orl ARG_KADDR(%esp), %ecx
andl $NTA_ALIGN_MASK, %ecx
orl ARG_COUNT(%esp), %ecx
andl $COUNT_ALIGN_MASK, %ecx
jnz do_copy_fault
jmp do_copy_fault_nta /* use regular access */
4:
movl $EFAULT, %eax
jmp 3f
/*
* A fault during do_copy_fault or do_copy_fault_nta is
* indicated through an errno value in %eax and we iret from the
* trap handler to here.
*/
_xcopyin_err:
popl %ecx
popl %edi
movl %ecx, T_LOFAULT(%edx) /* restore original lofault */
popl %esi
popl %ebp
3:
cmpl $0, T_COPYOPS(%edx)
jz 2f
movl T_COPYOPS(%edx), %eax
jmp *CP_XCOPYIN(%eax)
2: rep; ret /* use 2 byte return instruction when branch target */
/* AMD Software Optimization Guide - Section 6.2 */
SET_SIZE(xcopyin_nta)
#undef ARG_UADDR
#undef ARG_KADDR
#undef ARG_COUNT
#undef ARG_CACHED
#endif /* __i386 */
#endif /* __lint */
/*
* Copy kernel data to user space.
*/
#if defined(__lint)
/* ARGSUSED */
int
copyout(const void *kaddr, void *uaddr, size_t count)
{ return (0); }
#else /* __lint */
#if defined(__amd64)
ENTRY(copyout)
pushq %rbp
movq %rsp, %rbp
subq $32, %rsp
/*
* save args in case we trap and need to rerun as a copyop
*/
movq %rdi, (%rsp)
movq %rsi, 0x8(%rsp)
movq %rdx, 0x10(%rsp)
movq kernelbase(%rip), %rax
#ifdef DEBUG
cmpq %rax, %rdi /* %rdi = kaddr */
jnb 1f
leaq .copyout_panic_msg(%rip), %rdi
xorl %eax, %eax
call panic
1:
#endif
/*
* pass lofault value as 4th argument to do_copy_fault
*/
leaq _copyout_err(%rip), %rcx
movq %gs:CPU_THREAD, %r9
cmpq %rax, %rsi /* test uaddr < kernelbase */
jb do_copy_fault
jmp 3f
_copyout_err:
movq %r11, T_LOFAULT(%r9) /* restore original lofault */
3:
movq T_COPYOPS(%r9), %rax
cmpq $0, %rax
jz 2f
/*
* reload args for the copyop
*/
movq (%rsp), %rdi
movq 0x8(%rsp), %rsi
movq 0x10(%rsp), %rdx
leave
jmp *CP_COPYOUT(%rax)
2: movl $-1, %eax
leave
ret
SET_SIZE(copyout)
#elif defined(__i386)
#define ARG_KADDR 4
#define ARG_UADDR 8
ENTRY(copyout)
movl kernelbase, %ecx
#ifdef DEBUG
cmpl %ecx, ARG_KADDR(%esp)
jnb 1f
pushl %ebp
movl %esp, %ebp
pushl $.copyout_panic_msg
call panic
1:
#endif
lea _copyout_err, %eax
movl %gs:CPU_THREAD, %edx
cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
jb do_copy_fault
jmp 3f
_copyout_err:
popl %ecx
popl %edi
movl %ecx, T_LOFAULT(%edx) /* restore original lofault */
popl %esi
popl %ebp
3:
movl T_COPYOPS(%edx), %eax
cmpl $0, %eax
jz 2f
jmp *CP_COPYOUT(%eax)
2: movl $-1, %eax
ret
SET_SIZE(copyout)
#undef ARG_UADDR
#undef ARG_KADDR
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/* ARGSUSED */
int
xcopyout_nta(const void *kaddr, void *uaddr, size_t count, int copy_cached)
{ return (0); }
#else /* __lint */
#if defined(__amd64)
ENTRY(xcopyout_nta)
pushq %rbp
movq %rsp, %rbp
subq $32, %rsp
/*
* save args in case we trap and need to rerun as a copyop
*/
movq %rdi, (%rsp)
movq %rsi, 0x8(%rsp)
movq %rdx, 0x10(%rsp)
movq kernelbase(%rip), %rax
#ifdef DEBUG
cmpq %rax, %rdi /* %rdi = kaddr */
jnb 1f
leaq .xcopyout_panic_msg(%rip), %rdi
xorl %eax, %eax
call panic
1:
#endif
movq %gs:CPU_THREAD, %r9
cmpq %rax, %rsi /* test uaddr < kernelbase */
jae 4f
cmpq $0, %rcx /* No non-temporal access? */
/*
* pass lofault value as 4th argument to do_copy_fault
*/
leaq _xcopyout_err(%rip), %rcx
jnz do_copy_fault
/*
* Make sure cnt is >= XCOPY_MIN_SIZE bytes
*/
cmpq $XCOPY_MIN_SIZE, %rdx
jb do_copy_fault
/*
* Make sure src and dst are NTA_ALIGN_SIZE aligned,
* count is COUNT_ALIGN_SIZE aligned.
*/
movq %rdi, %r10
orq %rsi, %r10
andq $NTA_ALIGN_MASK, %r10
orq %rdx, %r10
andq $COUNT_ALIGN_MASK, %r10
jnz do_copy_fault
jmp do_copy_fault_nta
4:
movl $EFAULT, %eax
jmp 3f
/*
* A fault during do_copy_fault or do_copy_fault_nta is
* indicated through an errno value in %rax and we iret from the
* trap handler to here.
*/
_xcopyout_err:
movq %r11, T_LOFAULT(%r9) /* restore original lofault */
3:
movq T_COPYOPS(%r9), %r8
cmpq $0, %r8
jz 2f
/*
* reload args for the copyop
*/
movq (%rsp), %rdi
movq 0x8(%rsp), %rsi
movq 0x10(%rsp), %rdx
leave
jmp *CP_XCOPYOUT(%r8)
2: leave
ret
SET_SIZE(xcopyout_nta)
#elif defined(__i386)
#define ARG_KADDR 4
#define ARG_UADDR 8
#define ARG_COUNT 12
#define ARG_CACHED 16
ENTRY(xcopyout_nta)
movl kernelbase, %ecx
lea _xcopyout_err, %eax
movl %gs:CPU_THREAD, %edx
cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
jae 4f
cmpl $0, use_sse_copy /* no sse support */
jz do_copy_fault
cmpl $0, ARG_CACHED(%esp) /* copy_cached hint set? */
jnz do_copy_fault
/*
* Make sure cnt is >= XCOPY_MIN_SIZE bytes
*/
cmpl $XCOPY_MIN_SIZE, %edx
jb do_copy_fault
/*
* Make sure src and dst are NTA_ALIGN_SIZE aligned,
* count is COUNT_ALIGN_SIZE aligned.
*/
movl ARG_UADDR(%esp), %ecx
orl ARG_KADDR(%esp), %ecx
andl $NTA_ALIGN_MASK, %ecx
orl ARG_COUNT(%esp), %ecx
andl $COUNT_ALIGN_MASK, %ecx
jnz do_copy_fault
jmp do_copy_fault_nta
4:
movl $EFAULT, %eax
jmp 3f
/*
* A fault during do_copy_fault or do_copy_fault_nta is
* indicated through an errno value in %eax and we iret from the
* trap handler to here.
*/
_xcopyout_err:
/ restore the original lofault
popl %ecx
popl %edi
movl %ecx, T_LOFAULT(%edx) / original lofault
popl %esi
popl %ebp
3:
cmpl $0, T_COPYOPS(%edx)
jz 2f
movl T_COPYOPS(%edx), %eax
jmp *CP_XCOPYOUT(%eax)
2: rep; ret /* use 2 byte return instruction when branch target */
/* AMD Software Optimization Guide - Section 6.2 */
SET_SIZE(xcopyout_nta)
#undef ARG_UADDR
#undef ARG_KADDR
#undef ARG_COUNT
#undef ARG_CACHED
#endif /* __i386 */
#endif /* __lint */
/*
* Copy a null terminated string from one point to another in
* the kernel address space.
*/
#if defined(__lint)
/* ARGSUSED */
int
copystr(const char *from, char *to, size_t maxlength, size_t *lencopied)
{ return (0); }
#else /* __lint */
#if defined(__amd64)
ENTRY(copystr)
pushq %rbp
movq %rsp, %rbp
#ifdef DEBUG
movq kernelbase(%rip), %rax
cmpq %rax, %rdi /* %rdi = from */
jb 0f
cmpq %rax, %rsi /* %rsi = to */
jnb 1f
0: leaq .copystr_panic_msg(%rip), %rdi
xorl %eax, %eax
call panic
1:
#endif
movq %gs:CPU_THREAD, %r9
movq T_LOFAULT(%r9), %r8 /* pass current lofault value as */
/* 5th argument to do_copystr */
do_copystr:
movq %gs:CPU_THREAD, %r9 /* %r9 = thread addr */
movq T_LOFAULT(%r9), %r11 /* save the current lofault */
movq %r8, T_LOFAULT(%r9) /* new lofault */
movq %rdx, %r8 /* save maxlength */
cmpq $0, %rdx /* %rdx = maxlength */
je copystr_enametoolong /* maxlength == 0 */
copystr_loop:
decq %r8
movb (%rdi), %al
incq %rdi
movb %al, (%rsi)
incq %rsi
cmpb $0, %al
je copystr_null /* null char */
cmpq $0, %r8
jne copystr_loop
copystr_enametoolong:
movl $ENAMETOOLONG, %eax
jmp copystr_out
copystr_null:
xorl %eax, %eax /* no error */
copystr_out:
cmpq $0, %rcx /* want length? */
je copystr_done /* no */
subq %r8, %rdx /* compute length and store it */
movq %rdx, (%rcx)
copystr_done:
movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
leave
ret
SET_SIZE(copystr)
#elif defined(__i386)
#define ARG_FROM 8
#define ARG_TO 12
#define ARG_MAXLEN 16
#define ARG_LENCOPIED 20
ENTRY(copystr)
#ifdef DEBUG
pushl %ebp
movl %esp, %ebp
movl kernelbase, %eax
cmpl %eax, ARG_FROM(%esp)
jb 0f
cmpl %eax, ARG_TO(%esp)
jnb 1f
0: pushl $.copystr_panic_msg
call panic
1: popl %ebp
#endif
/* get the current lofault address */
movl %gs:CPU_THREAD, %eax
movl T_LOFAULT(%eax), %eax
do_copystr:
pushl %ebp /* setup stack frame */
movl %esp, %ebp
pushl %ebx /* save registers */
pushl %edi
movl %gs:CPU_THREAD, %ebx
movl T_LOFAULT(%ebx), %edi
pushl %edi /* save the current lofault */
movl %eax, T_LOFAULT(%ebx) /* new lofault */
movl ARG_MAXLEN(%ebp), %ecx
cmpl $0, %ecx
je copystr_enametoolong /* maxlength == 0 */
movl ARG_FROM(%ebp), %ebx /* source address */
movl ARG_TO(%ebp), %edx /* destination address */
copystr_loop:
decl %ecx
movb (%ebx), %al
incl %ebx
movb %al, (%edx)
incl %edx
cmpb $0, %al
je copystr_null /* null char */
cmpl $0, %ecx
jne copystr_loop
copystr_enametoolong:
movl $ENAMETOOLONG, %eax
jmp copystr_out
copystr_null:
xorl %eax, %eax /* no error */
copystr_out:
cmpl $0, ARG_LENCOPIED(%ebp) /* want length? */
je copystr_done /* no */
movl ARG_MAXLEN(%ebp), %edx
subl %ecx, %edx /* compute length and store it */
movl ARG_LENCOPIED(%ebp), %ecx
movl %edx, (%ecx)
copystr_done:
popl %edi
movl %gs:CPU_THREAD, %ebx
movl %edi, T_LOFAULT(%ebx) /* restore the original lofault */
popl %edi
popl %ebx
popl %ebp
ret
SET_SIZE(copystr)
#undef ARG_FROM
#undef ARG_TO
#undef ARG_MAXLEN
#undef ARG_LENCOPIED
#endif /* __i386 */
#endif /* __lint */
/*
* Copy a null terminated string from the user address space into
* the kernel address space.
*/
#if defined(__lint)
/* ARGSUSED */
int
copyinstr(const char *uaddr, char *kaddr, size_t maxlength,
size_t *lencopied)
{ return (0); }
#else /* __lint */
#if defined(__amd64)
ENTRY(copyinstr)
pushq %rbp
movq %rsp, %rbp
subq $32, %rsp
/*
* save args in case we trap and need to rerun as a copyop
*/
movq %rdi, (%rsp)
movq %rsi, 0x8(%rsp)
movq %rdx, 0x10(%rsp)
movq %rcx, 0x18(%rsp)
movq kernelbase(%rip), %rax
#ifdef DEBUG
cmpq %rax, %rsi /* %rsi = kaddr */
jnb 1f
leaq .copyinstr_panic_msg(%rip), %rdi
xorl %eax, %eax
call panic
1:
#endif
/*
* pass lofault value as 5th argument to do_copystr
*/
leaq _copyinstr_error(%rip), %r8
cmpq %rax, %rdi /* test uaddr < kernelbase */
jb do_copystr
movq %gs:CPU_THREAD, %r9
jmp 3f
_copyinstr_error:
movq %r11, T_LOFAULT(%r9) /* restore original lofault */
3:
movq T_COPYOPS(%r9), %rax
cmpq $0, %rax
jz 2f
/*
* reload args for the copyop
*/
movq (%rsp), %rdi
movq 0x8(%rsp), %rsi
movq 0x10(%rsp), %rdx
movq 0x18(%rsp), %rcx
leave
jmp *CP_COPYINSTR(%rax)
2: movl $EFAULT, %eax /* return EFAULT */
leave
ret
SET_SIZE(copyinstr)
#elif defined(__i386)
#define ARG_UADDR 4
#define ARG_KADDR 8
ENTRY(copyinstr)
movl kernelbase, %ecx
#ifdef DEBUG
cmpl %ecx, ARG_KADDR(%esp)
jnb 1f
pushl %ebp
movl %esp, %ebp
pushl $.copyinstr_panic_msg
call panic
1:
#endif
lea _copyinstr_error, %eax
cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
jb do_copystr
movl %gs:CPU_THREAD, %edx
jmp 3f
_copyinstr_error:
popl %edi
movl %gs:CPU_THREAD, %edx
movl %edi, T_LOFAULT(%edx) /* original lofault */
popl %edi
popl %ebx
popl %ebp
3:
movl T_COPYOPS(%edx), %eax
cmpl $0, %eax
jz 2f
jmp *CP_COPYINSTR(%eax)
2: movl $EFAULT, %eax /* return EFAULT */
ret
SET_SIZE(copyinstr)
#undef ARG_UADDR
#undef ARG_KADDR
#endif /* __i386 */
#endif /* __lint */
/*
* Copy a null terminated string from the kernel
* address space to the user address space.
*/
#if defined(__lint)
/* ARGSUSED */
int
copyoutstr(const char *kaddr, char *uaddr, size_t maxlength,
size_t *lencopied)
{ return (0); }
#else /* __lint */
#if defined(__amd64)
ENTRY(copyoutstr)
pushq %rbp
movq %rsp, %rbp
subq $32, %rsp
/*
* save args in case we trap and need to rerun as a copyop
*/
movq %rdi, (%rsp)
movq %rsi, 0x8(%rsp)
movq %rdx, 0x10(%rsp)
movq %rcx, 0x18(%rsp)
movq kernelbase(%rip), %rax
#ifdef DEBUG
cmpq %rax, %rdi /* %rdi = kaddr */
jnb 1f
leaq .copyoutstr_panic_msg(%rip), %rdi
jmp call_panic /* setup stack and call panic */
1:
#endif
/*
* pass lofault value as 5th argument to do_copystr
*/
leaq _copyoutstr_error(%rip), %r8
cmpq %rax, %rsi /* test uaddr < kernelbase */
jb do_copystr
movq %gs:CPU_THREAD, %r9
jmp 3f
_copyoutstr_error:
movq %r11, T_LOFAULT(%r9) /* restore the original lofault */
3:
movq T_COPYOPS(%r9), %rax
cmpq $0, %rax
jz 2f
/*
* reload args for the copyop
*/
movq (%rsp), %rdi
movq 0x8(%rsp), %rsi
movq 0x10(%rsp), %rdx
movq 0x18(%rsp), %rcx
leave
jmp *CP_COPYOUTSTR(%rax)
2: movl $EFAULT, %eax /* return EFAULT */
leave
ret
SET_SIZE(copyoutstr)
#elif defined(__i386)
#define ARG_KADDR 4
#define ARG_UADDR 8
ENTRY(copyoutstr)
movl kernelbase, %ecx
#ifdef DEBUG
cmpl %ecx, ARG_KADDR(%esp)
jnb 1f
pushl %ebp
movl %esp, %ebp
pushl $.copyoutstr_panic_msg
call panic
1:
#endif
lea _copyoutstr_error, %eax
cmpl %ecx, ARG_UADDR(%esp) /* test uaddr < kernelbase */
jb do_copystr
movl %gs:CPU_THREAD, %edx
jmp 3f
_copyoutstr_error:
popl %edi
movl %gs:CPU_THREAD, %edx
movl %edi, T_LOFAULT(%edx) /* restore the original lofault */
popl %edi
popl %ebx
popl %ebp
3:
movl T_COPYOPS(%edx), %eax
cmpl $0, %eax
jz 2f
jmp *CP_COPYOUTSTR(%eax)
2: movl $EFAULT, %eax /* return EFAULT */
ret
SET_SIZE(copyoutstr)
#undef ARG_KADDR
#undef ARG_UADDR
#endif /* __i386 */
#endif /* __lint */
/*
* Since all of the fuword() variants are so similar, we have a macro to spit
* them out. This allows us to create DTrace-unobservable functions easily.
*/
#if defined(__lint)
#if defined(__amd64)
/* ARGSUSED */
int
fuword64(const void *addr, uint64_t *dst)
{ return (0); }
#endif
/* ARGSUSED */
int
fuword32(const void *addr, uint32_t *dst)
{ return (0); }
/* ARGSUSED */
int
fuword16(const void *addr, uint16_t *dst)
{ return (0); }
/* ARGSUSED */
int
fuword8(const void *addr, uint8_t *dst)
{ return (0); }
#else /* __lint */
#if defined(__amd64)
/*
* (Note that we don't save and reload the arguments here
* because their values are not altered in the copy path)
*/
#define FUWORD(NAME, INSTR, REG, COPYOP) \
ENTRY(NAME) \
movq %gs:CPU_THREAD, %r9; \
cmpq kernelbase(%rip), %rdi; \
jae 1f; \
leaq _flt_/**/NAME, %rdx; \
movq %rdx, T_LOFAULT(%r9); \
INSTR (%rdi), REG; \
movq $0, T_LOFAULT(%r9); \
INSTR REG, (%rsi); \
xorl %eax, %eax; \
ret; \
_flt_/**/NAME: \
movq $0, T_LOFAULT(%r9); \
1: \
movq T_COPYOPS(%r9), %rax; \
cmpq $0, %rax; \
jz 2f; \
jmp *COPYOP(%rax); \
2: \
movl $-1, %eax; \
ret; \
SET_SIZE(NAME)
FUWORD(fuword64, movq, %rax, CP_FUWORD64)
FUWORD(fuword32, movl, %eax, CP_FUWORD32)
FUWORD(fuword16, movw, %ax, CP_FUWORD16)
FUWORD(fuword8, movb, %al, CP_FUWORD8)
#elif defined(__i386)
#define FUWORD(NAME, INSTR, REG, COPYOP) \
ENTRY(NAME) \
movl %gs:CPU_THREAD, %ecx; \
movl kernelbase, %eax; \
cmpl %eax, 4(%esp); \
jae 1f; \
lea _flt_/**/NAME, %edx; \
movl %edx, T_LOFAULT(%ecx); \
movl 4(%esp), %eax; \
movl 8(%esp), %edx; \
INSTR (%eax), REG; \
movl $0, T_LOFAULT(%ecx); \
INSTR REG, (%edx); \
xorl %eax, %eax; \
ret; \
_flt_/**/NAME: \
movl $0, T_LOFAULT(%ecx); \
1: \
movl T_COPYOPS(%ecx), %eax; \
cmpl $0, %eax; \
jz 2f; \
jmp *COPYOP(%eax); \
2: \
movl $-1, %eax; \
ret; \
SET_SIZE(NAME)
FUWORD(fuword32, movl, %eax, CP_FUWORD32)
FUWORD(fuword16, movw, %ax, CP_FUWORD16)
FUWORD(fuword8, movb, %al, CP_FUWORD8)
#endif /* __i386 */
#undef FUWORD
#endif /* __lint */
/*
* Set user word.
*/
#if defined(__lint)
#if defined(__amd64)
/* ARGSUSED */
int
suword64(void *addr, uint64_t value)
{ return (0); }
#endif
/* ARGSUSED */
int
suword32(void *addr, uint32_t value)
{ return (0); }
/* ARGSUSED */
int
suword16(void *addr, uint16_t value)
{ return (0); }
/* ARGSUSED */
int
suword8(void *addr, uint8_t value)
{ return (0); }
#else /* lint */
#if defined(__amd64)
/*
* (Note that we don't save and reload the arguments here
* because their values are not altered in the copy path)
*/
#define SUWORD(NAME, INSTR, REG, COPYOP) \
ENTRY(NAME) \
movq %gs:CPU_THREAD, %r9; \
cmpq kernelbase(%rip), %rdi; \
jae 1f; \
leaq _flt_/**/NAME, %rdx; \
movq %rdx, T_LOFAULT(%r9); \
INSTR REG, (%rdi); \
movq $0, T_LOFAULT(%r9); \
xorl %eax, %eax; \
ret; \
_flt_/**/NAME: \
movq $0, T_LOFAULT(%r9); \
1: \
movq T_COPYOPS(%r9), %rax; \
cmpq $0, %rax; \
jz 3f; \
jmp *COPYOP(%rax); \
3: \
movl $-1, %eax; \
ret; \
SET_SIZE(NAME)
SUWORD(suword64, movq, %rsi, CP_SUWORD64)
SUWORD(suword32, movl, %esi, CP_SUWORD32)
SUWORD(suword16, movw, %si, CP_SUWORD16)
SUWORD(suword8, movb, %sil, CP_SUWORD8)
#elif defined(__i386)
#define SUWORD(NAME, INSTR, REG, COPYOP) \
ENTRY(NAME) \
movl %gs:CPU_THREAD, %ecx; \
movl kernelbase, %eax; \
cmpl %eax, 4(%esp); \
jae 1f; \
lea _flt_/**/NAME, %edx; \
movl %edx, T_LOFAULT(%ecx); \
movl 4(%esp), %eax; \
movl 8(%esp), %edx; \
INSTR REG, (%eax); \
movl $0, T_LOFAULT(%ecx); \
xorl %eax, %eax; \
ret; \
_flt_/**/NAME: \
movl $0, T_LOFAULT(%ecx); \
1: \
movl T_COPYOPS(%ecx), %eax; \
cmpl $0, %eax; \
jz 3f; \
movl COPYOP(%eax), %ecx; \
jmp *%ecx; \
3: \
movl $-1, %eax; \
ret; \
SET_SIZE(NAME)
SUWORD(suword32, movl, %edx, CP_SUWORD32)
SUWORD(suword16, movw, %dx, CP_SUWORD16)
SUWORD(suword8, movb, %dl, CP_SUWORD8)
#endif /* __i386 */
#undef SUWORD
#endif /* __lint */
#if defined(__lint)
#if defined(__amd64)
/*ARGSUSED*/
void
fuword64_noerr(const void *addr, uint64_t *dst)
{}
#endif
/*ARGSUSED*/
void
fuword32_noerr(const void *addr, uint32_t *dst)
{}
/*ARGSUSED*/
void
fuword8_noerr(const void *addr, uint8_t *dst)
{}
/*ARGSUSED*/
void
fuword16_noerr(const void *addr, uint16_t *dst)
{}
#else /* __lint */
#if defined(__amd64)
#define FUWORD_NOERR(NAME, INSTR, REG) \
ENTRY(NAME) \
cmpq kernelbase(%rip), %rdi; \
cmovnbq kernelbase(%rip), %rdi; \
INSTR (%rdi), REG; \
INSTR REG, (%rsi); \
ret; \
SET_SIZE(NAME)
FUWORD_NOERR(fuword64_noerr, movq, %rax)
FUWORD_NOERR(fuword32_noerr, movl, %eax)
FUWORD_NOERR(fuword16_noerr, movw, %ax)
FUWORD_NOERR(fuword8_noerr, movb, %al)
#elif defined(__i386)
#define FUWORD_NOERR(NAME, INSTR, REG) \
ENTRY(NAME) \
movl 4(%esp), %eax; \
cmpl kernelbase, %eax; \
jb 1f; \
movl kernelbase, %eax; \
1: movl 8(%esp), %edx; \
INSTR (%eax), REG; \
INSTR REG, (%edx); \
ret; \
SET_SIZE(NAME)
FUWORD_NOERR(fuword32_noerr, movl, %ecx)
FUWORD_NOERR(fuword16_noerr, movw, %cx)
FUWORD_NOERR(fuword8_noerr, movb, %cl)
#endif /* __i386 */
#undef FUWORD_NOERR
#endif /* __lint */
#if defined(__lint)
#if defined(__amd64)
/*ARGSUSED*/
void
suword64_noerr(void *addr, uint64_t value)
{}
#endif
/*ARGSUSED*/
void
suword32_noerr(void *addr, uint32_t value)
{}
/*ARGSUSED*/
void
suword16_noerr(void *addr, uint16_t value)
{}
/*ARGSUSED*/
void
suword8_noerr(void *addr, uint8_t value)
{}
#else /* lint */
#if defined(__amd64)
#define SUWORD_NOERR(NAME, INSTR, REG) \
ENTRY(NAME) \
cmpq kernelbase(%rip), %rdi; \
cmovnbq kernelbase(%rip), %rdi; \
INSTR REG, (%rdi); \
ret; \
SET_SIZE(NAME)
SUWORD_NOERR(suword64_noerr, movq, %rsi)
SUWORD_NOERR(suword32_noerr, movl, %esi)
SUWORD_NOERR(suword16_noerr, movw, %si)
SUWORD_NOERR(suword8_noerr, movb, %sil)
#elif defined(__i386)
#define SUWORD_NOERR(NAME, INSTR, REG) \
ENTRY(NAME) \
movl 4(%esp), %eax; \
cmpl kernelbase, %eax; \
jb 1f; \
movl kernelbase, %eax; \
1: \
movl 8(%esp), %edx; \
INSTR REG, (%eax); \
ret; \
SET_SIZE(NAME)
SUWORD_NOERR(suword32_noerr, movl, %edx)
SUWORD_NOERR(suword16_noerr, movw, %dx)
SUWORD_NOERR(suword8_noerr, movb, %dl)
#endif /* __i386 */
#undef SUWORD_NOERR
#endif /* lint */
#if defined(__lint)
/*ARGSUSED*/
int
subyte(void *addr, uchar_t value)
{ return (0); }
/*ARGSUSED*/
void
subyte_noerr(void *addr, uchar_t value)
{}
/*ARGSUSED*/
int
fulword(const void *addr, ulong_t *valuep)
{ return (0); }
/*ARGSUSED*/
void
fulword_noerr(const void *addr, ulong_t *valuep)
{}
/*ARGSUSED*/
int
sulword(void *addr, ulong_t valuep)
{ return (0); }
/*ARGSUSED*/
void
sulword_noerr(void *addr, ulong_t valuep)
{}
#else
.weak subyte
subyte=suword8
.weak subyte_noerr
subyte_noerr=suword8_noerr
#if defined(__amd64)
.weak fulword
fulword=fuword64
.weak fulword_noerr
fulword_noerr=fuword64_noerr
.weak sulword
sulword=suword64
.weak sulword_noerr
sulword_noerr=suword64_noerr
#elif defined(__i386)
.weak fulword
fulword=fuword32
.weak fulword_noerr
fulword_noerr=fuword32_noerr
.weak sulword
sulword=suword32
.weak sulword_noerr
sulword_noerr=suword32_noerr
#endif /* __i386 */
#endif /* __lint */
#if defined(__lint)
/*
* Copy a block of storage - must not overlap (from + len <= to).
* No fault handler installed (to be called under on_fault())
*/
/* ARGSUSED */
void
copyout_noerr(const void *kfrom, void *uto, size_t count)
{}
/* ARGSUSED */
void
copyin_noerr(const void *ufrom, void *kto, size_t count)
{}
/*
* Zero a block of storage in user space
*/
/* ARGSUSED */
void
uzero(void *addr, size_t count)
{}
/*
* copy a block of storage in user space
*/
/* ARGSUSED */
void
ucopy(const void *ufrom, void *uto, size_t ulength)
{}
/*
* copy a string in user space
*/
/* ARGSUSED */
void
ucopystr(const char *ufrom, char *uto, size_t umaxlength, size_t *lencopied)
{}
#else /* __lint */
#if defined(__amd64)
ENTRY(copyin_noerr)
movq kernelbase(%rip), %rax
#ifdef DEBUG
cmpq %rax, %rsi /* %rsi = kto */
jae 1f
leaq .cpyin_ne_pmsg(%rip), %rdi
jmp call_panic /* setup stack and call panic */
1:
#endif
cmpq %rax, %rdi /* ufrom < kernelbase */
jb do_copy
movq %rax, %rdi /* force fault at kernelbase */
jmp do_copy
SET_SIZE(copyin_noerr)
ENTRY(copyout_noerr)
movq kernelbase(%rip), %rax
#ifdef DEBUG
cmpq %rax, %rdi /* %rdi = kfrom */
jae 1f
leaq .cpyout_ne_pmsg(%rip), %rdi
jmp call_panic /* setup stack and call panic */
1:
#endif
cmpq %rax, %rsi /* uto < kernelbase */
jb do_copy
movq %rax, %rsi /* force fault at kernelbase */
jmp do_copy
SET_SIZE(copyout_noerr)
ENTRY(uzero)
movq kernelbase(%rip), %rax
cmpq %rax, %rdi
jb do_zero
movq %rax, %rdi /* force fault at kernelbase */
jmp do_zero
SET_SIZE(uzero)
ENTRY(ucopy)
movq kernelbase(%rip), %rax
cmpq %rax, %rdi
cmovaeq %rax, %rdi /* force fault at kernelbase */
cmpq %rax, %rsi
cmovaeq %rax, %rsi /* force fault at kernelbase */
jmp do_copy
SET_SIZE(ucopy)
ENTRY(ucopystr)
movq kernelbase(%rip), %rax
cmpq %rax, %rdi
cmovaeq %rax, %rdi /* force fault at kernelbase */
cmpq %rax, %rsi
cmovaeq %rax, %rsi /* force fault at kernelbase */
/* do_copystr expects lofault address in %r8 */
movq %gs:CPU_THREAD, %r8
movq T_LOFAULT(%r8), %r8
jmp do_copystr
SET_SIZE(ucopystr)
#elif defined(__i386)
ENTRY(copyin_noerr)
movl kernelbase, %eax
#ifdef DEBUG
cmpl %eax, 8(%esp)
jae 1f
pushl $.cpyin_ne_pmsg
call panic
1:
#endif
cmpl %eax, 4(%esp)
jb do_copy
movl %eax, 4(%esp) /* force fault at kernelbase */
jmp do_copy
SET_SIZE(copyin_noerr)
ENTRY(copyout_noerr)
movl kernelbase, %eax
#ifdef DEBUG
cmpl %eax, 4(%esp)
jae 1f
pushl $.cpyout_ne_pmsg
call panic
1:
#endif
cmpl %eax, 8(%esp)
jb do_copy
movl %eax, 8(%esp) /* force fault at kernelbase */
jmp do_copy
SET_SIZE(copyout_noerr)
ENTRY(uzero)
movl kernelbase, %eax
cmpl %eax, 4(%esp)
jb do_zero
movl %eax, 4(%esp) /* force fault at kernelbase */
jmp do_zero
SET_SIZE(uzero)
ENTRY(ucopy)
movl kernelbase, %eax
cmpl %eax, 4(%esp)
jb 1f
movl %eax, 4(%esp) /* force fault at kernelbase */
1:
cmpl %eax, 8(%esp)
jb do_copy
movl %eax, 8(%esp) /* force fault at kernelbase */
jmp do_copy
SET_SIZE(ucopy)
ENTRY(ucopystr)
movl kernelbase, %eax
cmpl %eax, 4(%esp)
jb 1f
movl %eax, 4(%esp) /* force fault at kernelbase */
1:
cmpl %eax, 8(%esp)
jb 2f
movl %eax, 8(%esp) /* force fault at kernelbase */
2:
/* do_copystr expects the lofault address in %eax */
movl %gs:CPU_THREAD, %eax
movl T_LOFAULT(%eax), %eax
jmp do_copystr
SET_SIZE(ucopystr)
#endif /* __i386 */
#ifdef DEBUG
.data
.kcopy_panic_msg:
.string "kcopy: arguments below kernelbase"
.bcopy_panic_msg:
.string "bcopy: arguments below kernelbase"
.kzero_panic_msg:
.string "kzero: arguments below kernelbase"
.bzero_panic_msg:
.string "bzero: arguments below kernelbase"
.copyin_panic_msg:
.string "copyin: kaddr argument below kernelbase"
.xcopyin_panic_msg:
.string "xcopyin: kaddr argument below kernelbase"
.copyout_panic_msg:
.string "copyout: kaddr argument below kernelbase"
.xcopyout_panic_msg:
.string "xcopyout: kaddr argument below kernelbase"
.copystr_panic_msg:
.string "copystr: arguments in user space"
.copyinstr_panic_msg:
.string "copyinstr: kaddr argument not in kernel address space"
.copyoutstr_panic_msg:
.string "copyoutstr: kaddr argument not in kernel address space"
.cpyin_ne_pmsg:
.string "copyin_noerr: argument not in kernel address space"
.cpyout_ne_pmsg:
.string "copyout_noerr: argument not in kernel address space"
#endif
#endif /* __lint */