d0f7acf9eb
An x86-64 operating system built from xv6, now largely rewritten: SMP, a copy-on-write root filesystem (SchFS), PCI/USB/virtio drivers, a TTF rasteriser, and a Linux-numbered syscall ABI. No history before this point -- the tree was never under version control. This commit exists so the rename that follows has something to undo to.
257 lines
9.2 KiB
ArmAsm
257 lines
9.2 KiB
ArmAsm
# trapasm.S — kernel/user entry/exit with FS_BASE swapping for TLS
|
|
#
|
|
# The kernel uses __thread struct cpu *cpu and __thread struct proc *proc,
|
|
# accessed via %fs (IA32_FS_BASE MSR, 0xC0000100). seginit() sets this to
|
|
# kernel_fs_base. User-space TLS (musl) uses the same MSR via arch_prctl
|
|
# ARCH_SET_FS (saved in proc->fs_base).
|
|
#
|
|
# Rule: on ENTRY from user (syscall or interrupt), restore kernel FS_BASE.
|
|
# on EXIT to user (sysretq or iretq), write proc->fs_base to MSR.
|
|
#
|
|
# proc->fs_base is at byte offset 0x18 (24) inside struct proc.
|
|
# proc pointer is at %gs:24 (PERCPU_PROC in include/proc.h); cpu is at %gs:16.
|
|
|
|
# -----------------------------------------------------------------------
|
|
# Helper macro: write kernel_fs_base to IA32_FS_BASE.
|
|
# Clobbers %rax, %rcx, %rdx. Caller must save them if needed.
|
|
# -----------------------------------------------------------------------
|
|
# Read from %gs, not from a global.
|
|
#
|
|
# This used to be `movq kernel_fs_base(%rip), %rax`, and kernel_fs_base was a
|
|
# single global that every CPU wrote during seginit(). With one processor that
|
|
# is merely untidy. With two it is fatal: whichever CPU ran seginit() last wins,
|
|
# and from the first trap onward *every* CPU restores that one CPU's per-CPU
|
|
# block -- so `cpu` and `proc` are shared, pushcli/popcli nesting counts collide,
|
|
# and the kernel dies in panic("sched locks") shortly after the APs start.
|
|
#
|
|
# GS base is set per CPU by seginit() to the same value as FS base, and %gs:0
|
|
# holds the block's self-pointer. So this reads *this* CPU's kernel FS base with
|
|
# no global involved.
|
|
#
|
|
# No swapgs, deliberately. The usual reason for it is that userspace owns GS;
|
|
# here it does not -- sys_arch_prctl() implements only ARCH_SET_FS/ARCH_GET_FS
|
|
# and rejects everything else, musl uses FS for TLS on x86-64, and the only
|
|
# other writes to %gs are selector loads in entry.S and entryother.S that happen
|
|
# before seginit(). GS is the kernel's alone, so it can simply stay pointing at
|
|
# the per-CPU block. If ARCH_SET_GS is ever implemented, this must become a
|
|
# proper swapgs pair.
|
|
.macro RESTORE_KERNEL_FS
|
|
movq %gs:0, %rax
|
|
movq %rax, %rdx
|
|
shrq $32, %rdx
|
|
movl $0xC0000100, %ecx
|
|
wrmsr
|
|
.endm
|
|
|
|
# -----------------------------------------------------------------------
|
|
# Helper macro: write proc->fs_base to IA32_FS_BASE.
|
|
# Requires %fs to still hold the kernel base.
|
|
# Clobbers %rax, %rcx, %rdx. Caller must save them if needed.
|
|
# -----------------------------------------------------------------------
|
|
.macro WRITE_USER_FS
|
|
# proc, from the per-CPU block at %gs:24 (PERCPU_PROC in include/proc.h).
|
|
# This used to be %fs:(-8), the __thread slot, which no longer exists --
|
|
# see the comment in include/proc.h for why those stopped being __thread.
|
|
movq %gs:24, %rax # proc pointer
|
|
movq 0x18(%rax), %rax # proc->fs_base (offset 24 = 0x18)
|
|
movq %rax, %rdx
|
|
shrq $32, %rdx
|
|
movl $0xC0000100, %ecx
|
|
wrmsr
|
|
.endm
|
|
|
|
# -----------------------------------------------------------------------
|
|
# alltraps — IDT interrupt/fault entry
|
|
# Stack on entry (from user mode, CPU has done privilege switch):
|
|
# [rsp+0] trapno (pushed by vectors.S)
|
|
# [rsp+8] errorcode
|
|
# [rsp+16] rip
|
|
# [rsp+24] cs ← bit 1:0 == 3 means user mode
|
|
# [rsp+32] rflags
|
|
# [rsp+40] user_rsp
|
|
# [rsp+48] user_ss
|
|
# -----------------------------------------------------------------------
|
|
.global alltraps
|
|
alltraps:
|
|
# Always restore kernel FS_BASE before touching %fs:, even when the
|
|
# saved cs says we were already in kernel mode. A fault/interrupt can
|
|
# land in that narrow kernel-mode window after syscall_trapret/trapret
|
|
# have already done WRITE_USER_FS (cs is still 0x8 there, but FS_BASE
|
|
# is the *user* value) — the old cs-based skip trusted that combination
|
|
# never happens and read %fs:(-8) as garbage (proc/cpu == NULL) in that
|
|
# case. RESTORE_KERNEL_FS is just a wrmsr of an already-known value when
|
|
# we truly were in the kernel, so doing it unconditionally is cheap and
|
|
# always correct.
|
|
pushq %rax
|
|
pushq %rcx
|
|
pushq %rdx
|
|
RESTORE_KERNEL_FS
|
|
popq %rdx
|
|
popq %rcx
|
|
popq %rax
|
|
|
|
# Build trap frame.
|
|
pushq %r15
|
|
pushq %r14
|
|
pushq %r13
|
|
pushq %r12
|
|
pushq %r11
|
|
pushq %r10
|
|
pushq %r9
|
|
pushq %r8
|
|
pushq %rdi
|
|
pushq %rsi
|
|
pushq %rbp
|
|
pushq %rdx
|
|
pushq %rcx
|
|
pushq %rbx
|
|
pushq %rax
|
|
|
|
movq %rsp, %rdi # frame in arg1
|
|
callq trap
|
|
# Return falls through to trapret...
|
|
|
|
.global trapret
|
|
trapret:
|
|
popq %rax
|
|
popq %rbx
|
|
popq %rcx
|
|
popq %rdx
|
|
popq %rbp
|
|
popq %rsi
|
|
popq %rdi
|
|
popq %r8
|
|
popq %r9
|
|
popq %r10
|
|
popq %r11
|
|
popq %r12
|
|
popq %r13
|
|
popq %r14
|
|
popq %r15
|
|
|
|
addq $16, %rsp # discard trapno and errorcode
|
|
|
|
# If returning to user mode, write proc->fs_base to FS_BASE MSR.
|
|
# Stack now: [rsp+0]=rip, [rsp+8]=cs, [rsp+16]=rflags, [rsp+24]=rsp, [rsp+32]=ss
|
|
testb $3, 8(%rsp)
|
|
jz .Ltrapret_kret
|
|
|
|
# cli before WRITE_USER_FS: close the same race as in syscall_trapret.
|
|
# iretq will atomically restore RFLAGS (IF=1) when switching to user mode.
|
|
pushq %rax
|
|
pushq %rcx
|
|
pushq %rdx
|
|
cli
|
|
WRITE_USER_FS
|
|
popq %rdx
|
|
popq %rcx
|
|
popq %rax
|
|
|
|
.Ltrapret_kret:
|
|
iretq
|
|
|
|
# -----------------------------------------------------------------------
|
|
# syscall_entry — SYSCALL instruction entry point
|
|
# On entry: rcx=user rip, r11=user rflags, rsp=user stack, IF cleared.
|
|
# FS_BASE may be the user's TLS value (set at last sysretq).
|
|
# -----------------------------------------------------------------------
|
|
.global syscall_entry
|
|
syscall_entry:
|
|
# Swap FS_BASE to kernel value BEFORE any %fs: access.
|
|
# Save rax/rcx/rdx to single-CPU scratch globals (IF is off, no race).
|
|
movq %rax, _sc_rax(%rip)
|
|
movq %rcx, _sc_rcx(%rip) # user rip (SYSCALL stores it here)
|
|
movq %rdx, _sc_rdx(%rip)
|
|
RESTORE_KERNEL_FS # clobbers rax, rcx, rdx — that's OK
|
|
movq _sc_rax(%rip), %rax
|
|
movq _sc_rcx(%rip), %rcx
|
|
movq _sc_rdx(%rip), %rdx
|
|
|
|
# Switch to kernel stack: store user rsp at top of proc->kstack.
|
|
#
|
|
# The scratch slot is %fs:(8), NOT %fs:(0). %fs:(0) is the TCB
|
|
# self-pointer -- the x86-64 TLS ABI reserves the word at the thread
|
|
# pointer to hold the thread pointer itself, and a compiler is entitled to
|
|
# read it to materialise the TP before applying a variable's offset.
|
|
#
|
|
# It used to be %fs:(0), and that survived for years only because GCC
|
|
# compiles `proc`/`cpu` into direct `%fs:-8` segment references and never
|
|
# touches the word. clang at -O0 does `movq %fs:0, %rax; addq $-8, %rax;
|
|
# (%rax)` instead -- so it read the syscall number this line had just
|
|
# stored (7, SYS_exec, from initcode), computed 7-8 = -1, and page-faulted
|
|
# on the very first syscall of the very first process.
|
|
#
|
|
# Positive offsets are safe: the TLS block occupies negative offsets
|
|
# (-16 = cpu, -8 = proc) and seginit puts the thread pointer 2048 bytes
|
|
# into a 4096-byte page, so %fs:(8) onward is 2040 bytes we own outright.
|
|
movq %rax, %fs:(8) # save rax in per-CPU scratch slot
|
|
movq %gs:24, %rax # proc pointer (PERCPU_PROC)
|
|
movq 0x10(%rax), %rax # proc->kstack (offset 16)
|
|
addq $(4096-16), %rax # point to tf->rsp slot
|
|
movq %rsp, (%rax) # save user rsp
|
|
movq %rax, %rsp # switch to kernel stack
|
|
movq %fs:(8), %rax # restore rax
|
|
|
|
pushq %r11 # rflags
|
|
pushq $0 # cs (ignored)
|
|
pushq %rcx # rip (next user instruction)
|
|
|
|
pushq $0 # err
|
|
pushq $0 # trapno (ignored)
|
|
|
|
pushq %r15
|
|
pushq %r14
|
|
pushq %r13
|
|
pushq %r12
|
|
pushq %r11
|
|
pushq %r10
|
|
pushq %r9
|
|
pushq %r8
|
|
pushq %rdi
|
|
pushq %rsi
|
|
pushq %rbp
|
|
pushq %rdx
|
|
pushq %rcx
|
|
pushq %rbx
|
|
pushq %rax
|
|
|
|
movq %rsp, %rdi # frame in arg1
|
|
callq syscall
|
|
# Return falls through to syscall_trapret...
|
|
|
|
# -----------------------------------------------------------------------
|
|
# syscall_trapret — return path for SYSCALL-based syscalls
|
|
# -----------------------------------------------------------------------
|
|
.global syscall_trapret
|
|
syscall_trapret:
|
|
# Disable interrupts BEFORE writing user FS_BASE to the MSR.
|
|
# Without cli here, a timer IRQ between WRITE_USER_FS and the cli below
|
|
# would arrive in ring-0 with user FS_BASE set, causing %fs:(-8) to return
|
|
# garbage (proc=NULL) → panic in the interrupt handler.
|
|
# rax/rcx/rdx are clobbered by WRITE_USER_FS then restored by the pops.
|
|
cli
|
|
WRITE_USER_FS
|
|
|
|
popq %rax
|
|
popq %rbx
|
|
popq %rcx
|
|
popq %rdx
|
|
popq %rbp
|
|
popq %rsi
|
|
popq %rdi
|
|
popq %r8
|
|
popq %r9
|
|
popq %r10
|
|
popq %r11
|
|
popq %r12
|
|
popq %r13
|
|
popq %r14
|
|
popq %r15
|
|
|
|
addq $40, %rsp # discard trapno, errorcode, rip, cs, rflags
|
|
|
|
# Mask interrupts: sysretq is unsafe with IF set on non-canonical RSP.
|
|
# See https://www.felixcloutier.com/x86/sysret
|
|
cli
|
|
movq (%rsp), %rsp # restore the user stack
|
|
sysretq
|