a1c965da1e
Sixty-five headers sat side by side in include/, so `#include "irq.h"` said nothing about where irq.h was or what it belonged to, and errno.h, stat.h and fcntl.h at the top shadowed the system's for any host tool built with -I./include. They are in folders now -- arch/ core/ fs/ dev/ gfx/ net/ uapi/ (and ldls/ for the Linux driver layer) -- and every include names the folder, as Linux's do: "core/proc.h". Done mechanically and checked the hard way: the kernel built from this tree disassembles identically to the one built from the tree before it. The full image boots, and userland is the same set of programs. Paths in docs and comments follow; docs/guides/layout.md has a table of what is where, and tools/fix-includes.py rewrites old-style includes from the folders in the tree, for code written before this. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4
279 lines
11 KiB
ArmAsm
279 lines
11 KiB
ArmAsm
# trapasm.S — kernel/user entry/exit with FS_BASE swapping for TLS
|
|
#
|
|
# The kernel uses __thread struct cpu *cpu and __thread struct proc *proc,
|
|
# accessed via %fs (IA32_FS_BASE MSR, 0xC0000100). seginit() sets this to
|
|
# kernel_fs_base. User-space TLS (musl) uses the same MSR via arch_prctl
|
|
# ARCH_SET_FS (saved in proc->fs_base).
|
|
#
|
|
# Rule: on ENTRY from user (syscall or interrupt), restore kernel FS_BASE.
|
|
# on EXIT to user (sysretq or iretq), write proc->fs_base to MSR.
|
|
#
|
|
# proc->fs_base is at byte offset 0x18 (24) inside struct proc.
|
|
# proc pointer is at %gs:24 (PERCPU_PROC in include/core/proc.h); cpu is at %gs:16.
|
|
|
|
# -----------------------------------------------------------------------
|
|
# Helper macro: write kernel_fs_base to IA32_FS_BASE.
|
|
# Clobbers %rax, %rcx, %rdx. Caller must save them if needed.
|
|
# -----------------------------------------------------------------------
|
|
# Read from %gs, not from a global.
|
|
#
|
|
# This used to be `movq kernel_fs_base(%rip), %rax`, and kernel_fs_base was a
|
|
# single global that every CPU wrote during seginit(). With one processor that
|
|
# is merely untidy. With two it is fatal: whichever CPU ran seginit() last wins,
|
|
# and from the first trap onward *every* CPU restores that one CPU's per-CPU
|
|
# block -- so `cpu` and `proc` are shared, pushcli/popcli nesting counts collide,
|
|
# and the kernel dies in panic("sched locks") shortly after the APs start.
|
|
#
|
|
# GS base is set per CPU by seginit() to the same value as FS base, and %gs:0
|
|
# holds the block's self-pointer. So this reads *this* CPU's kernel FS base with
|
|
# no global involved.
|
|
#
|
|
# No swapgs, deliberately. The usual reason for it is that userspace owns GS;
|
|
# here it does not -- sys_arch_prctl() implements only ARCH_SET_FS/ARCH_GET_FS
|
|
# and rejects everything else, musl uses FS for TLS on x86-64, and the only
|
|
# other writes to %gs are selector loads in entry.S and entryother.S that happen
|
|
# before seginit(). GS is the kernel's alone, so it can simply stay pointing at
|
|
# the per-CPU block. If ARCH_SET_GS is ever implemented, this must become a
|
|
# proper swapgs pair.
|
|
.macro RESTORE_KERNEL_FS
|
|
movq %gs:0, %rax
|
|
movq %rax, %rdx
|
|
shrq $32, %rdx
|
|
movl $0xC0000100, %ecx
|
|
wrmsr
|
|
.endm
|
|
|
|
# -----------------------------------------------------------------------
|
|
# Helper macro: write proc->fs_base to IA32_FS_BASE.
|
|
# Requires %fs to still hold the kernel base.
|
|
# Clobbers %rax, %rcx, %rdx. Caller must save them if needed.
|
|
# -----------------------------------------------------------------------
|
|
.macro WRITE_USER_FS
|
|
# proc, from the per-CPU block at %gs:24 (PERCPU_PROC in include/core/proc.h).
|
|
# This used to be %fs:(-8), the __thread slot, which no longer exists --
|
|
# see the comment in include/core/proc.h for why those stopped being __thread.
|
|
movq %gs:24, %rax # proc pointer
|
|
movq 0x18(%rax), %rax # proc->fs_base (offset 24 = 0x18)
|
|
movq %rax, %rdx
|
|
shrq $32, %rdx
|
|
movl $0xC0000100, %ecx
|
|
wrmsr
|
|
.endm
|
|
|
|
# -----------------------------------------------------------------------
|
|
# alltraps — IDT interrupt/fault entry
|
|
# Stack on entry (from user mode, CPU has done privilege switch):
|
|
# [rsp+0] trapno (pushed by vectors.S)
|
|
# [rsp+8] errorcode
|
|
# [rsp+16] rip
|
|
# [rsp+24] cs ← bit 1:0 == 3 means user mode
|
|
# [rsp+32] rflags
|
|
# [rsp+40] user_rsp
|
|
# [rsp+48] user_ss
|
|
# -----------------------------------------------------------------------
|
|
.global alltraps
|
|
alltraps:
|
|
# Always restore kernel FS_BASE before touching %fs:, even when the
|
|
# saved cs says we were already in kernel mode. A fault/interrupt can
|
|
# land in that narrow kernel-mode window after syscall_trapret/trapret
|
|
# have already done WRITE_USER_FS (cs is still 0x8 there, but FS_BASE
|
|
# is the *user* value) — the old cs-based skip trusted that combination
|
|
# never happens and read %fs:(-8) as garbage (proc/cpu == NULL) in that
|
|
# case. RESTORE_KERNEL_FS is just a wrmsr of an already-known value when
|
|
# we truly were in the kernel, so doing it unconditionally is cheap and
|
|
# always correct.
|
|
pushq %rax
|
|
pushq %rcx
|
|
pushq %rdx
|
|
RESTORE_KERNEL_FS
|
|
popq %rdx
|
|
popq %rcx
|
|
popq %rax
|
|
|
|
# Build trap frame.
|
|
pushq %r15
|
|
pushq %r14
|
|
pushq %r13
|
|
pushq %r12
|
|
pushq %r11
|
|
pushq %r10
|
|
pushq %r9
|
|
pushq %r8
|
|
pushq %rdi
|
|
pushq %rsi
|
|
pushq %rbp
|
|
pushq %rdx
|
|
pushq %rcx
|
|
pushq %rbx
|
|
pushq %rax
|
|
|
|
movq %rsp, %rdi # frame in arg1
|
|
callq trap
|
|
# Return falls through to trapret...
|
|
|
|
.global trapret
|
|
trapret:
|
|
popq %rax
|
|
popq %rbx
|
|
popq %rcx
|
|
popq %rdx
|
|
popq %rbp
|
|
popq %rsi
|
|
popq %rdi
|
|
popq %r8
|
|
popq %r9
|
|
popq %r10
|
|
popq %r11
|
|
popq %r12
|
|
popq %r13
|
|
popq %r14
|
|
popq %r15
|
|
|
|
addq $16, %rsp # discard trapno and errorcode
|
|
|
|
# If returning to user mode, write proc->fs_base to FS_BASE MSR.
|
|
# Stack now: [rsp+0]=rip, [rsp+8]=cs, [rsp+16]=rflags, [rsp+24]=rsp, [rsp+32]=ss
|
|
testb $3, 8(%rsp)
|
|
jz .Ltrapret_kret
|
|
|
|
# cli before WRITE_USER_FS: close the same race as in syscall_trapret.
|
|
# iretq will atomically restore RFLAGS (IF=1) when switching to user mode.
|
|
pushq %rax
|
|
pushq %rcx
|
|
pushq %rdx
|
|
cli
|
|
WRITE_USER_FS
|
|
popq %rdx
|
|
popq %rcx
|
|
popq %rax
|
|
|
|
.Ltrapret_kret:
|
|
iretq
|
|
|
|
# -----------------------------------------------------------------------
|
|
# syscall_entry — SYSCALL instruction entry point
|
|
# On entry: rcx=user rip, r11=user rflags, rsp=user stack, IF cleared.
|
|
# FS_BASE may be the user's TLS value (set at last sysretq).
|
|
# -----------------------------------------------------------------------
|
|
.global syscall_entry
|
|
syscall_entry:
|
|
# Swap FS_BASE to the kernel value BEFORE any %fs: access. wrmsr destroys
|
|
# rax, rcx and rdx, so they have to be parked somewhere first -- not on the
|
|
# user stack, which is untrusted and has not been switched away from yet.
|
|
#
|
|
# **Per CPU, at %gs:32..48.** These were three globals, `_sc_rax`,
|
|
# `_sc_rcx` and `_sc_rdx`, under a comment reading "single-CPU scratch for
|
|
# syscall_entry FS swap (safe while IF cleared by SYSCALL/SFMASK)". IF
|
|
# being clear answers for an interrupt arriving on *this* CPU. It says
|
|
# nothing whatever about a second CPU executing these same three stores at
|
|
# the same moment -- and with two processors entering syscalls
|
|
# concurrently, that is not a rare interleaving, it is the common one.
|
|
#
|
|
# %rcx is the user return address: SYSCALL puts the user's rip there and
|
|
# SYSRET takes it back from there. So the CPU that lost the race returned
|
|
# its process to *the other process's* instruction pointer. The symptom was
|
|
# one to three processes per boot dying at `rip 0x0` or dereferencing
|
|
# 0xffffffffffffffea (-EINVAL, a syscall return value used as a pointer,
|
|
# because %rax was swapped too) -- never on -smp 1, one to three times on
|
|
# -smp 2, and in any process rather than one.
|
|
#
|
|
# %gs is usable here, before the FS swap, for the same reason
|
|
# RESTORE_KERNEL_FS can read %gs:0: GS base is per-CPU in hardware, is set
|
|
# once by seginit(), and userspace cannot change it -- sys_arch_prctl()
|
|
# implements ARCH_SET_FS and ARCH_GET_FS and rejects ARCH_SET_GS.
|
|
movq %rax, %gs:32
|
|
movq %rcx, %gs:40 # user rip (SYSCALL stores it here)
|
|
movq %rdx, %gs:48
|
|
RESTORE_KERNEL_FS # clobbers rax, rcx, rdx — that's OK
|
|
movq %gs:32, %rax
|
|
movq %gs:40, %rcx
|
|
movq %gs:48, %rdx
|
|
|
|
# Switch to kernel stack: store user rsp at top of proc->kstack.
|
|
#
|
|
# The scratch slot is %fs:(8), NOT %fs:(0). %fs:(0) is the TCB
|
|
# self-pointer -- the x86-64 TLS ABI reserves the word at the thread
|
|
# pointer to hold the thread pointer itself, and a compiler is entitled to
|
|
# read it to materialise the TP before applying a variable's offset.
|
|
#
|
|
# It used to be %fs:(0), and that survived for years only because GCC
|
|
# compiles `proc`/`cpu` into direct `%fs:-8` segment references and never
|
|
# touches the word. clang at -O0 does `movq %fs:0, %rax; addq $-8, %rax;
|
|
# (%rax)` instead -- so it read the syscall number this line had just
|
|
# stored (7, SYS_exec, from initcode), computed 7-8 = -1, and page-faulted
|
|
# on the very first syscall of the very first process.
|
|
#
|
|
# Positive offsets are safe: the TLS block occupies negative offsets
|
|
# (-16 = cpu, -8 = proc) and seginit puts the thread pointer 2048 bytes
|
|
# into a 4096-byte page, so %fs:(8) onward is 2040 bytes we own outright.
|
|
movq %rax, %fs:(8) # save rax in per-CPU scratch slot
|
|
movq %gs:24, %rax # proc pointer (PERCPU_PROC)
|
|
movq 0x10(%rax), %rax # proc->kstack (offset 16)
|
|
addq $(4096-16), %rax # point to tf->rsp slot
|
|
movq %rsp, (%rax) # save user rsp
|
|
movq %rax, %rsp # switch to kernel stack
|
|
movq %fs:(8), %rax # restore rax
|
|
|
|
pushq %r11 # rflags
|
|
pushq $0 # cs (ignored)
|
|
pushq %rcx # rip (next user instruction)
|
|
|
|
pushq $0 # err
|
|
pushq $0 # trapno (ignored)
|
|
|
|
pushq %r15
|
|
pushq %r14
|
|
pushq %r13
|
|
pushq %r12
|
|
pushq %r11
|
|
pushq %r10
|
|
pushq %r9
|
|
pushq %r8
|
|
pushq %rdi
|
|
pushq %rsi
|
|
pushq %rbp
|
|
pushq %rdx
|
|
pushq %rcx
|
|
pushq %rbx
|
|
pushq %rax
|
|
|
|
movq %rsp, %rdi # frame in arg1
|
|
callq syscall
|
|
# Return falls through to syscall_trapret...
|
|
|
|
# -----------------------------------------------------------------------
|
|
# syscall_trapret — return path for SYSCALL-based syscalls
|
|
# -----------------------------------------------------------------------
|
|
.global syscall_trapret
|
|
syscall_trapret:
|
|
# Disable interrupts BEFORE writing user FS_BASE to the MSR.
|
|
# Without cli here, a timer IRQ between WRITE_USER_FS and the cli below
|
|
# would arrive in ring-0 with user FS_BASE set, causing %fs:(-8) to return
|
|
# garbage (proc=NULL) → panic in the interrupt handler.
|
|
# rax/rcx/rdx are clobbered by WRITE_USER_FS then restored by the pops.
|
|
cli
|
|
WRITE_USER_FS
|
|
|
|
popq %rax
|
|
popq %rbx
|
|
popq %rcx
|
|
popq %rdx
|
|
popq %rbp
|
|
popq %rsi
|
|
popq %rdi
|
|
popq %r8
|
|
popq %r9
|
|
popq %r10
|
|
popq %r11
|
|
popq %r12
|
|
popq %r13
|
|
popq %r14
|
|
popq %r15
|
|
|
|
addq $40, %rsp # discard trapno, errorcode, rip, cs, rflags
|
|
|
|
# Mask interrupts: sysretq is unsafe with IF set on non-canonical RSP.
|
|
# See https://www.felixcloutier.com/x86/sysret
|
|
cli
|
|
movq (%rsp), %rsp # restore the user stack
|
|
sysretq
|