# trapasm.S — kernel/user entry/exit with FS_BASE swapping for TLS # # The kernel uses __thread struct cpu *cpu and __thread struct proc *proc, # accessed via %fs (IA32_FS_BASE MSR, 0xC0000100). seginit() sets this to # kernel_fs_base. User-space TLS (musl) uses the same MSR via arch_prctl # ARCH_SET_FS (saved in proc->fs_base). # # Rule: on ENTRY from user (syscall or interrupt), restore kernel FS_BASE. # on EXIT to user (sysretq or iretq), write proc->fs_base to MSR. # # proc->fs_base is at byte offset 0x18 (24) inside struct proc. # proc pointer is at %gs:24 (PERCPU_PROC in include/proc.h); cpu is at %gs:16. # ----------------------------------------------------------------------- # Helper macro: write kernel_fs_base to IA32_FS_BASE. # Clobbers %rax, %rcx, %rdx. Caller must save them if needed. # ----------------------------------------------------------------------- # Read from %gs, not from a global. # # This used to be `movq kernel_fs_base(%rip), %rax`, and kernel_fs_base was a # single global that every CPU wrote during seginit(). With one processor that # is merely untidy. With two it is fatal: whichever CPU ran seginit() last wins, # and from the first trap onward *every* CPU restores that one CPU's per-CPU # block -- so `cpu` and `proc` are shared, pushcli/popcli nesting counts collide, # and the kernel dies in panic("sched locks") shortly after the APs start. # # GS base is set per CPU by seginit() to the same value as FS base, and %gs:0 # holds the block's self-pointer. So this reads *this* CPU's kernel FS base with # no global involved. # # No swapgs, deliberately. The usual reason for it is that userspace owns GS; # here it does not -- sys_arch_prctl() implements only ARCH_SET_FS/ARCH_GET_FS # and rejects everything else, musl uses FS for TLS on x86-64, and the only # other writes to %gs are selector loads in entry.S and entryother.S that happen # before seginit(). GS is the kernel's alone, so it can simply stay pointing at # the per-CPU block. If ARCH_SET_GS is ever implemented, this must become a # proper swapgs pair. .macro RESTORE_KERNEL_FS movq %gs:0, %rax movq %rax, %rdx shrq $32, %rdx movl $0xC0000100, %ecx wrmsr .endm # ----------------------------------------------------------------------- # Helper macro: write proc->fs_base to IA32_FS_BASE. # Requires %fs to still hold the kernel base. # Clobbers %rax, %rcx, %rdx. Caller must save them if needed. # ----------------------------------------------------------------------- .macro WRITE_USER_FS # proc, from the per-CPU block at %gs:24 (PERCPU_PROC in include/proc.h). # This used to be %fs:(-8), the __thread slot, which no longer exists -- # see the comment in include/proc.h for why those stopped being __thread. movq %gs:24, %rax # proc pointer movq 0x18(%rax), %rax # proc->fs_base (offset 24 = 0x18) movq %rax, %rdx shrq $32, %rdx movl $0xC0000100, %ecx wrmsr .endm # ----------------------------------------------------------------------- # alltraps — IDT interrupt/fault entry # Stack on entry (from user mode, CPU has done privilege switch): # [rsp+0] trapno (pushed by vectors.S) # [rsp+8] errorcode # [rsp+16] rip # [rsp+24] cs ← bit 1:0 == 3 means user mode # [rsp+32] rflags # [rsp+40] user_rsp # [rsp+48] user_ss # ----------------------------------------------------------------------- .global alltraps alltraps: # Always restore kernel FS_BASE before touching %fs:, even when the # saved cs says we were already in kernel mode. A fault/interrupt can # land in that narrow kernel-mode window after syscall_trapret/trapret # have already done WRITE_USER_FS (cs is still 0x8 there, but FS_BASE # is the *user* value) — the old cs-based skip trusted that combination # never happens and read %fs:(-8) as garbage (proc/cpu == NULL) in that # case. RESTORE_KERNEL_FS is just a wrmsr of an already-known value when # we truly were in the kernel, so doing it unconditionally is cheap and # always correct. pushq %rax pushq %rcx pushq %rdx RESTORE_KERNEL_FS popq %rdx popq %rcx popq %rax # Build trap frame. pushq %r15 pushq %r14 pushq %r13 pushq %r12 pushq %r11 pushq %r10 pushq %r9 pushq %r8 pushq %rdi pushq %rsi pushq %rbp pushq %rdx pushq %rcx pushq %rbx pushq %rax movq %rsp, %rdi # frame in arg1 callq trap # Return falls through to trapret... .global trapret trapret: popq %rax popq %rbx popq %rcx popq %rdx popq %rbp popq %rsi popq %rdi popq %r8 popq %r9 popq %r10 popq %r11 popq %r12 popq %r13 popq %r14 popq %r15 addq $16, %rsp # discard trapno and errorcode # If returning to user mode, write proc->fs_base to FS_BASE MSR. # Stack now: [rsp+0]=rip, [rsp+8]=cs, [rsp+16]=rflags, [rsp+24]=rsp, [rsp+32]=ss testb $3, 8(%rsp) jz .Ltrapret_kret # cli before WRITE_USER_FS: close the same race as in syscall_trapret. # iretq will atomically restore RFLAGS (IF=1) when switching to user mode. pushq %rax pushq %rcx pushq %rdx cli WRITE_USER_FS popq %rdx popq %rcx popq %rax .Ltrapret_kret: iretq # ----------------------------------------------------------------------- # syscall_entry — SYSCALL instruction entry point # On entry: rcx=user rip, r11=user rflags, rsp=user stack, IF cleared. # FS_BASE may be the user's TLS value (set at last sysretq). # ----------------------------------------------------------------------- .global syscall_entry syscall_entry: # Swap FS_BASE to kernel value BEFORE any %fs: access. # Save rax/rcx/rdx to single-CPU scratch globals (IF is off, no race). movq %rax, _sc_rax(%rip) movq %rcx, _sc_rcx(%rip) # user rip (SYSCALL stores it here) movq %rdx, _sc_rdx(%rip) RESTORE_KERNEL_FS # clobbers rax, rcx, rdx — that's OK movq _sc_rax(%rip), %rax movq _sc_rcx(%rip), %rcx movq _sc_rdx(%rip), %rdx # Switch to kernel stack: store user rsp at top of proc->kstack. # # The scratch slot is %fs:(8), NOT %fs:(0). %fs:(0) is the TCB # self-pointer -- the x86-64 TLS ABI reserves the word at the thread # pointer to hold the thread pointer itself, and a compiler is entitled to # read it to materialise the TP before applying a variable's offset. # # It used to be %fs:(0), and that survived for years only because GCC # compiles `proc`/`cpu` into direct `%fs:-8` segment references and never # touches the word. clang at -O0 does `movq %fs:0, %rax; addq $-8, %rax; # (%rax)` instead -- so it read the syscall number this line had just # stored (7, SYS_exec, from initcode), computed 7-8 = -1, and page-faulted # on the very first syscall of the very first process. # # Positive offsets are safe: the TLS block occupies negative offsets # (-16 = cpu, -8 = proc) and seginit puts the thread pointer 2048 bytes # into a 4096-byte page, so %fs:(8) onward is 2040 bytes we own outright. movq %rax, %fs:(8) # save rax in per-CPU scratch slot movq %gs:24, %rax # proc pointer (PERCPU_PROC) movq 0x10(%rax), %rax # proc->kstack (offset 16) addq $(4096-16), %rax # point to tf->rsp slot movq %rsp, (%rax) # save user rsp movq %rax, %rsp # switch to kernel stack movq %fs:(8), %rax # restore rax pushq %r11 # rflags pushq $0 # cs (ignored) pushq %rcx # rip (next user instruction) pushq $0 # err pushq $0 # trapno (ignored) pushq %r15 pushq %r14 pushq %r13 pushq %r12 pushq %r11 pushq %r10 pushq %r9 pushq %r8 pushq %rdi pushq %rsi pushq %rbp pushq %rdx pushq %rcx pushq %rbx pushq %rax movq %rsp, %rdi # frame in arg1 callq syscall # Return falls through to syscall_trapret... # ----------------------------------------------------------------------- # syscall_trapret — return path for SYSCALL-based syscalls # ----------------------------------------------------------------------- .global syscall_trapret syscall_trapret: # Disable interrupts BEFORE writing user FS_BASE to the MSR. # Without cli here, a timer IRQ between WRITE_USER_FS and the cli below # would arrive in ring-0 with user FS_BASE set, causing %fs:(-8) to return # garbage (proc=NULL) → panic in the interrupt handler. # rax/rcx/rdx are clobbered by WRITE_USER_FS then restored by the pops. cli WRITE_USER_FS popq %rax popq %rbx popq %rcx popq %rdx popq %rbp popq %rsi popq %rdi popq %r8 popq %r9 popq %r10 popq %r11 popq %r12 popq %r13 popq %r14 popq %r15 addq $40, %rsp # discard trapno, errorcode, rip, cs, rflags # Mask interrupts: sysretq is unsafe with IF set on non-canonical RSP. # See https://www.felixcloutier.com/x86/sysret cli movq (%rsp), %rsp # restore the user stack sysretq