1ba93dde2c
syscall_entry put the frame at kstack + 4096 - 16, from when a kernel stack was one page. It has been two since, so every syscall ran in the bottom 4 KB while traps used the top, and a deep enough read -- startx exec'ing through the new compressed ramdisk -- ran over the canary at kstack[0]. The frame now sits at the top of KSTACKSIZE, where allocproc and the TSS already put it, and exec's user-rsp slot moves with it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4
286 lines
11 KiB
ArmAsm
286 lines
11 KiB
ArmAsm
#include "core/param.h" /* KSTACKSIZE */
|
|
# trapasm.S — kernel/user entry/exit with FS_BASE swapping for TLS
|
|
#
|
|
# The kernel uses __thread struct cpu *cpu and __thread struct proc *proc,
|
|
# accessed via %fs (IA32_FS_BASE MSR, 0xC0000100). seginit() sets this to
|
|
# kernel_fs_base. User-space TLS (musl) uses the same MSR via arch_prctl
|
|
# ARCH_SET_FS (saved in proc->fs_base).
|
|
#
|
|
# Rule: on ENTRY from user (syscall or interrupt), restore kernel FS_BASE.
|
|
# on EXIT to user (sysretq or iretq), write proc->fs_base to MSR.
|
|
#
|
|
# proc->fs_base is at byte offset 0x18 (24) inside struct proc.
|
|
# proc pointer is at %gs:24 (PERCPU_PROC in include/core/proc.h); cpu is at %gs:16.
|
|
|
|
# -----------------------------------------------------------------------
|
|
# Helper macro: write kernel_fs_base to IA32_FS_BASE.
|
|
# Clobbers %rax, %rcx, %rdx. Caller must save them if needed.
|
|
# -----------------------------------------------------------------------
|
|
# Read from %gs, not from a global.
|
|
#
|
|
# This used to be `movq kernel_fs_base(%rip), %rax`, and kernel_fs_base was a
|
|
# single global that every CPU wrote during seginit(). With one processor that
|
|
# is merely untidy. With two it is fatal: whichever CPU ran seginit() last wins,
|
|
# and from the first trap onward *every* CPU restores that one CPU's per-CPU
|
|
# block -- so `cpu` and `proc` are shared, pushcli/popcli nesting counts collide,
|
|
# and the kernel dies in panic("sched locks") shortly after the APs start.
|
|
#
|
|
# GS base is set per CPU by seginit() to the same value as FS base, and %gs:0
|
|
# holds the block's self-pointer. So this reads *this* CPU's kernel FS base with
|
|
# no global involved.
|
|
#
|
|
# No swapgs, deliberately. The usual reason for it is that userspace owns GS;
|
|
# here it does not -- sys_arch_prctl() implements only ARCH_SET_FS/ARCH_GET_FS
|
|
# and rejects everything else, musl uses FS for TLS on x86-64, and the only
|
|
# other writes to %gs are selector loads in entry.S and entryother.S that happen
|
|
# before seginit(). GS is the kernel's alone, so it can simply stay pointing at
|
|
# the per-CPU block. If ARCH_SET_GS is ever implemented, this must become a
|
|
# proper swapgs pair.
|
|
.macro RESTORE_KERNEL_FS
|
|
movq %gs:0, %rax
|
|
movq %rax, %rdx
|
|
shrq $32, %rdx
|
|
movl $0xC0000100, %ecx
|
|
wrmsr
|
|
.endm
|
|
|
|
# -----------------------------------------------------------------------
|
|
# Helper macro: write proc->fs_base to IA32_FS_BASE.
|
|
# Requires %fs to still hold the kernel base.
|
|
# Clobbers %rax, %rcx, %rdx. Caller must save them if needed.
|
|
# -----------------------------------------------------------------------
|
|
.macro WRITE_USER_FS
|
|
# proc, from the per-CPU block at %gs:24 (PERCPU_PROC in include/core/proc.h).
|
|
# This used to be %fs:(-8), the __thread slot, which no longer exists --
|
|
# see the comment in include/core/proc.h for why those stopped being __thread.
|
|
movq %gs:24, %rax # proc pointer
|
|
movq 0x18(%rax), %rax # proc->fs_base (offset 24 = 0x18)
|
|
movq %rax, %rdx
|
|
shrq $32, %rdx
|
|
movl $0xC0000100, %ecx
|
|
wrmsr
|
|
.endm
|
|
|
|
# -----------------------------------------------------------------------
|
|
# alltraps — IDT interrupt/fault entry
|
|
# Stack on entry (from user mode, CPU has done privilege switch):
|
|
# [rsp+0] trapno (pushed by vectors.S)
|
|
# [rsp+8] errorcode
|
|
# [rsp+16] rip
|
|
# [rsp+24] cs ← bit 1:0 == 3 means user mode
|
|
# [rsp+32] rflags
|
|
# [rsp+40] user_rsp
|
|
# [rsp+48] user_ss
|
|
# -----------------------------------------------------------------------
|
|
.global alltraps
|
|
alltraps:
|
|
# Always restore kernel FS_BASE before touching %fs:, even when the
|
|
# saved cs says we were already in kernel mode. A fault/interrupt can
|
|
# land in that narrow kernel-mode window after syscall_trapret/trapret
|
|
# have already done WRITE_USER_FS (cs is still 0x8 there, but FS_BASE
|
|
# is the *user* value) — the old cs-based skip trusted that combination
|
|
# never happens and read %fs:(-8) as garbage (proc/cpu == NULL) in that
|
|
# case. RESTORE_KERNEL_FS is just a wrmsr of an already-known value when
|
|
# we truly were in the kernel, so doing it unconditionally is cheap and
|
|
# always correct.
|
|
pushq %rax
|
|
pushq %rcx
|
|
pushq %rdx
|
|
RESTORE_KERNEL_FS
|
|
popq %rdx
|
|
popq %rcx
|
|
popq %rax
|
|
|
|
# Build trap frame.
|
|
pushq %r15
|
|
pushq %r14
|
|
pushq %r13
|
|
pushq %r12
|
|
pushq %r11
|
|
pushq %r10
|
|
pushq %r9
|
|
pushq %r8
|
|
pushq %rdi
|
|
pushq %rsi
|
|
pushq %rbp
|
|
pushq %rdx
|
|
pushq %rcx
|
|
pushq %rbx
|
|
pushq %rax
|
|
|
|
movq %rsp, %rdi # frame in arg1
|
|
callq trap
|
|
# Return falls through to trapret...
|
|
|
|
.global trapret
|
|
trapret:
|
|
popq %rax
|
|
popq %rbx
|
|
popq %rcx
|
|
popq %rdx
|
|
popq %rbp
|
|
popq %rsi
|
|
popq %rdi
|
|
popq %r8
|
|
popq %r9
|
|
popq %r10
|
|
popq %r11
|
|
popq %r12
|
|
popq %r13
|
|
popq %r14
|
|
popq %r15
|
|
|
|
addq $16, %rsp # discard trapno and errorcode
|
|
|
|
# If returning to user mode, write proc->fs_base to FS_BASE MSR.
|
|
# Stack now: [rsp+0]=rip, [rsp+8]=cs, [rsp+16]=rflags, [rsp+24]=rsp, [rsp+32]=ss
|
|
testb $3, 8(%rsp)
|
|
jz .Ltrapret_kret
|
|
|
|
# cli before WRITE_USER_FS: close the same race as in syscall_trapret.
|
|
# iretq will atomically restore RFLAGS (IF=1) when switching to user mode.
|
|
pushq %rax
|
|
pushq %rcx
|
|
pushq %rdx
|
|
cli
|
|
WRITE_USER_FS
|
|
popq %rdx
|
|
popq %rcx
|
|
popq %rax
|
|
|
|
.Ltrapret_kret:
|
|
iretq
|
|
|
|
# -----------------------------------------------------------------------
|
|
# syscall_entry — SYSCALL instruction entry point
|
|
# On entry: rcx=user rip, r11=user rflags, rsp=user stack, IF cleared.
|
|
# FS_BASE may be the user's TLS value (set at last sysretq).
|
|
# -----------------------------------------------------------------------
|
|
.global syscall_entry
|
|
syscall_entry:
|
|
# Swap FS_BASE to the kernel value BEFORE any %fs: access. wrmsr destroys
|
|
# rax, rcx and rdx, so they have to be parked somewhere first -- not on the
|
|
# user stack, which is untrusted and has not been switched away from yet.
|
|
#
|
|
# **Per CPU, at %gs:32..48.** These were three globals, `_sc_rax`,
|
|
# `_sc_rcx` and `_sc_rdx`, under a comment reading "single-CPU scratch for
|
|
# syscall_entry FS swap (safe while IF cleared by SYSCALL/SFMASK)". IF
|
|
# being clear answers for an interrupt arriving on *this* CPU. It says
|
|
# nothing whatever about a second CPU executing these same three stores at
|
|
# the same moment -- and with two processors entering syscalls
|
|
# concurrently, that is not a rare interleaving, it is the common one.
|
|
#
|
|
# %rcx is the user return address: SYSCALL puts the user's rip there and
|
|
# SYSRET takes it back from there. So the CPU that lost the race returned
|
|
# its process to *the other process's* instruction pointer. The symptom was
|
|
# one to three processes per boot dying at `rip 0x0` or dereferencing
|
|
# 0xffffffffffffffea (-EINVAL, a syscall return value used as a pointer,
|
|
# because %rax was swapped too) -- never on -smp 1, one to three times on
|
|
# -smp 2, and in any process rather than one.
|
|
#
|
|
# %gs is usable here, before the FS swap, for the same reason
|
|
# RESTORE_KERNEL_FS can read %gs:0: GS base is per-CPU in hardware, is set
|
|
# once by seginit(), and userspace cannot change it -- sys_arch_prctl()
|
|
# implements ARCH_SET_FS and ARCH_GET_FS and rejects ARCH_SET_GS.
|
|
movq %rax, %gs:32
|
|
movq %rcx, %gs:40 # user rip (SYSCALL stores it here)
|
|
movq %rdx, %gs:48
|
|
RESTORE_KERNEL_FS # clobbers rax, rcx, rdx — that's OK
|
|
movq %gs:32, %rax
|
|
movq %gs:40, %rcx
|
|
movq %gs:48, %rdx
|
|
|
|
# Switch to kernel stack: store user rsp at top of proc->kstack.
|
|
#
|
|
# The scratch slot is %fs:(8), NOT %fs:(0). %fs:(0) is the TCB
|
|
# self-pointer -- the x86-64 TLS ABI reserves the word at the thread
|
|
# pointer to hold the thread pointer itself, and a compiler is entitled to
|
|
# read it to materialise the TP before applying a variable's offset.
|
|
#
|
|
# It used to be %fs:(0), and that survived for years only because GCC
|
|
# compiles `proc`/`cpu` into direct `%fs:-8` segment references and never
|
|
# touches the word. clang at -O0 does `movq %fs:0, %rax; addq $-8, %rax;
|
|
# (%rax)` instead -- so it read the syscall number this line had just
|
|
# stored (7, SYS_exec, from initcode), computed 7-8 = -1, and page-faulted
|
|
# on the very first syscall of the very first process.
|
|
#
|
|
# Positive offsets are safe: the TLS block occupies negative offsets
|
|
# (-16 = cpu, -8 = proc) and seginit puts the thread pointer 2048 bytes
|
|
# into a 4096-byte page, so %fs:(8) onward is 2040 bytes we own outright.
|
|
movq %rax, %fs:(8) # save rax in per-CPU scratch slot
|
|
movq %gs:24, %rax # proc pointer (PERCPU_PROC)
|
|
movq 0x10(%rax), %rax # proc->kstack (offset 16)
|
|
# The top of the whole stack, where allocproc() put the trapframe and
|
|
# where a trap from user mode lands (TSS rsp0). This was $(4096-16) from
|
|
# when the stack was one page, and stayed that when it became two: every
|
|
# syscall ran in the bottom 4 KB with the top 4 KB unused, and a deep
|
|
# enough read -- startx exec'ing through a SCFF chunk unpack -- ran
|
|
# through the canary at kstack[0].
|
|
addq $(KSTACKSIZE-16), %rax # point to tf->rsp slot
|
|
movq %rsp, (%rax) # save user rsp
|
|
movq %rax, %rsp # switch to kernel stack
|
|
movq %fs:(8), %rax # restore rax
|
|
|
|
pushq %r11 # rflags
|
|
pushq $0 # cs (ignored)
|
|
pushq %rcx # rip (next user instruction)
|
|
|
|
pushq $0 # err
|
|
pushq $0 # trapno (ignored)
|
|
|
|
pushq %r15
|
|
pushq %r14
|
|
pushq %r13
|
|
pushq %r12
|
|
pushq %r11
|
|
pushq %r10
|
|
pushq %r9
|
|
pushq %r8
|
|
pushq %rdi
|
|
pushq %rsi
|
|
pushq %rbp
|
|
pushq %rdx
|
|
pushq %rcx
|
|
pushq %rbx
|
|
pushq %rax
|
|
|
|
movq %rsp, %rdi # frame in arg1
|
|
callq syscall
|
|
# Return falls through to syscall_trapret...
|
|
|
|
# -----------------------------------------------------------------------
|
|
# syscall_trapret — return path for SYSCALL-based syscalls
|
|
# -----------------------------------------------------------------------
|
|
.global syscall_trapret
|
|
syscall_trapret:
|
|
# Disable interrupts BEFORE writing user FS_BASE to the MSR.
|
|
# Without cli here, a timer IRQ between WRITE_USER_FS and the cli below
|
|
# would arrive in ring-0 with user FS_BASE set, causing %fs:(-8) to return
|
|
# garbage (proc=NULL) → panic in the interrupt handler.
|
|
# rax/rcx/rdx are clobbered by WRITE_USER_FS then restored by the pops.
|
|
cli
|
|
WRITE_USER_FS
|
|
|
|
popq %rax
|
|
popq %rbx
|
|
popq %rcx
|
|
popq %rdx
|
|
popq %rbp
|
|
popq %rsi
|
|
popq %rdi
|
|
popq %r8
|
|
popq %r9
|
|
popq %r10
|
|
popq %r11
|
|
popq %r12
|
|
popq %r13
|
|
popq %r14
|
|
popq %r15
|
|
|
|
addq $40, %rsp # discard trapno, errorcode, rip, cs, rflags
|
|
|
|
# Mask interrupts: sysretq is unsafe with IF set on non-canonical RSP.
|
|
# See https://www.felixcloutier.com/x86/sysret
|
|
cli
|
|
movq (%rsp), %rsp # restore the user stack
|
|
sysretq
|