#pragma once #include "types.h" #include "traps.h" // Simultaneous mmap() regions per process. // // 64 was enough for every first-party program and for toybox, and not enough // for zsh: musl's allocator takes a fresh mapping per size class and does not // release them eagerly, so compinit -- which reads twelve hundred completion // functions -- ran out of slots and reported "fatal error: out of memory" on // a machine with a gigabyte free. The slots were the limit, not the memory. // // The table is inline in struct proc, so this is NPROC * NVMA * sizeof(struct // vma) of kernel memory, always resident. At 256 that is about 1 MB total, // which is worth it to stop a shell dying on a machine that has 1000 MB spare. #define NVMA 256 struct vma { uint64 addr; uint64 length; int used; // The pages under this mapping are not ours to free on unmap: either they // are device memory (the framebuffer) or they belong to something with a // longer life than this process (a shared memory segment). This was // `is_fb`, which was the only such case until shared memory existed -- // and freeing another process's shared pages on exit is exactly the kind // of bug that shows up somewhere else entirely, hours later. int no_free; int shmid; // shared segment this came from, or -1 }; // Per-CPU state struct cpu { uchar id; uchar apicid; // Local APIC ID struct context *scheduler; // swtch() here to enter scheduler volatile uint64 started; // Has the CPU started? int ncli; // Depth of pushcli nesting. int intena; // Were interrupts enabled before pushcli? void *local; // CPU-local storage (GS base) }; extern struct cpu cpus[NCPU]; extern int ncpu; // processors present extern int ncpu_online; // processors actually running // --- per-CPU variables ------------------------------------------------------ // // These were `__thread struct cpu *cpu` and `__thread struct proc *proc`, and // that was wrong in a way that only appears once a second processor runs. // // `__thread` means *per thread*, and the compiler is entitled to assume a // thread's pointer never changes: it reads %fs:0 once and keeps the value in a // register or spilled on the stack across calls. clang did exactly that in // sleep(): // // movq %fs:0x0, %rax # read the thread pointer // movq %rax, -0x18(%rbp) # spill it // callq sched # <-- the CPU can change here // movq -0x18(%rbp), %rax # reload the OLD pointer // movq (%rax,%rcx), %rax # proc, from the previous CPU's block // movq $0x0, 0x40(%rax) # NULL, because that CPU set proc = 0 // // But %fs here is *per CPU*, not per thread, and a kernel thread changes CPU at // every sched(). So the assumption is false for exactly the variables that // most need to be right, and the failure is a null dereference deep in the // scheduler on a machine with more than one core. // // The fix is to read the value at each use, with no cached base: an explicit // %gs-relative access that the compiler cannot hoist across a call. `volatile` // is what forbids that, and the memory clobber on the setters keeps stores in // order with the rest of the scheduler's bookkeeping. // // Layout of the per-CPU block, whose base is in both FS_BASE and GS_BASE, set // by seginit() in kernel/arch/x86_64/vm.c: // // +0 self-pointer required by the x86-64 TLS ABI (%fs:0) // +8 syscall scratch trapasm.S parks %rsp here across the stack switch // +16 struct cpu * // +24 struct proc * // // Offset 8 is NOT free. syscall_entry uses %fs:(8) as its scratch slot, and // because seginit() sets GS base and FS base to the same address, %fs:8 and // %gs:8 are the same memory. Putting `cpu` there made every syscall overwrite // it with %rsp -- a #GP in syscall_entry itself, several layers below anything // that could report it. // // trapasm.S reads %gs:24 for proc on the way out to user mode, so these three // offsets are shared with assembly and must not be reordered. #define PERCPU_SELF 0 #define PERCPU_SCRATCH 8 #define PERCPU_CPU 16 #define PERCPU_PROC 24 static inline struct cpu * cpu_get(void) { struct cpu *c; asm volatile("movq %%gs:16, %0" : "=r" (c)); return c; } static inline void cpu_set(struct cpu *c) { asm volatile("movq %0, %%gs:16" :: "r" (c) : "memory"); } static inline struct proc * proc_get(void) { struct proc *p; asm volatile("movq %%gs:24, %0" : "=r" (p)); return p; } static inline void proc_set(struct proc *p) { asm volatile("movq %0, %%gs:24" :: "r" (p) : "memory"); } // Named mycpu()/myproc(), as xv6-riscv does, and *not* spelled `cpu`/`proc`. // A macro by those names would also rewrite every `struct cpu` and // `struct proc`, because macros have no namespace and tags do -- which is // exactly why the variables could carry those names in the first place. #define mycpu() cpu_get() #define myproc() proc_get() // Saved registers for kernel context switches (x86_64 ABI) struct context { uint64 r15; uint64 r14; uint64 r13; uint64 r12; uint64 rbx; uint64 rbp; uint64 rip; }; enum procstate { UNUSED, EMBRYO, SLEEPING, RUNNABLE, RUNNING, ZOMBIE }; // Per-process state // A copy of the interesting fields of one process, taken under ptable.lock. // procfs reads these rather than struct proc, so nothing outside proc.c needs // the lock or the layout. struct procinfo { int pid, ppid, pgid, sid; uint uid; int state; // enum procstate uint64 sz; int killed; char name[16]; char exe[64]; // full path, for /proc//exe }; struct proc { uint64 sz; // Size of process memory (bytes) [+0x00] pml4e_t* pgdir; // Page table (PML4 on x86_64) [+0x08] char *kstack; // Bottom of kernel stack [+0x10] uint64 fs_base; // User TLS base (IA32_FS_BASE MSR) [+0x18] ← trapasm.S depends on this offset enum procstate state; // Process state [+0x20] int pid; // [+0x24] struct proc *parent; // [+0x28] struct trapframe *tf; // [+0x30] struct context *context; // [+0x38] void *chan; // [+0x40] int killed; // [+0x48] int _pad; // [+0x4c] struct file *ofile[NOFILE]; // Open files [+0x50] struct inode *cwd; // follows char name[16]; // // The path exec() was given. `name` is only the last component and only 15 // characters of it, which is what /proc//stat wants; this is the whole // path, reported by /proc//exe. // // Note it is exposed as a regular *file*, not the symlink Linux uses, so // `cat /proc/1/exe` works but readlink() does not. That is why `ps -o comm` // still prints "[init]": toybox readlink()s exe and brackets the name when // that fails. Default ps and top output is unaffected. Fixing it properly // means symlink support in procfs, which is worth doing with the rest of the // VFS work rather than on its own. char exe[64]; // A kernel thread's entry point, or 0 for an ordinary process. // // Set by kthread_create and dispatched by forkret. A kernel thread has a // page directory with only the kernel mapped and never returns to user mode, // so it has no trapframe worth speaking of and no user stack -- it simply // runs `kfn` forever inside the scheduler's ordinary rotation, which is what // makes it able to sleep and to be preempted like anything else. void (*kfn)(void); uint64 signal_mask; uint64 pending_signals; int alarm_ticks; char ofile_flags[NOFILE]; // Real signal delivery (added for SIGCHLD — see kernel/sys/syscall.c's // deliver_signals() and kernel/proc/proc.c's exit()). sig_handler[sig]/ // sig_restorer[sig] come straight from the user's struct sigaction // (sys_rt_sigaction); 0 means no handler installed. saved_sigtf stashes // the pre-signal trapframe so sys_rt_sigreturn() can restore it; in_signal // prevents re-entering a handler while one is already running (we don't // support nested signal delivery). uint64 sig_handler[32]; uint64 sig_restorer[32]; struct trapframe saved_sigtf; int in_signal; // rt_sigsuspend() support: the pre-suspend mask to restore once the // handler that woke us finishes (sigsuspend swaps the mask atomically // for the duration of the wait -- POSIX). sigsusp_active is set while a // restore is owed; sys_rt_sigreturn() performs it. uint64 sigsusp_oldmask; int sigsusp_active; struct vma vma[NVMA]; // VMA tracking for mmap uint64 mmap_next; // next free mmap VA (bump allocator, see sys_mmap) // ABI translation (Linux compat syscall number map) int *syscall_map; int map_size; // Current working directory path; maintained by sys_chdir / sys_fchdir. char cwd_path[256]; // --- POSIX process identity ------------------------------------------- // Appended at the end of the struct deliberately: trapasm.S hard-codes the // offsets of the fields at the top (see the _Static_assert on fs_base in // kernel/proc/proc.c), so nothing may be inserted above them. // Exit status, already encoded the way wait(2) reports it -- see // W_EXITCODE() below. Read by waitpid() before the slot is recycled. int xstate; // Signal that killed us, if any. Set by sigsend() alongside ->killed so the // eventual exit() can report *which* signal, instead of a blanket SIGKILL. int killsig; // Process group and session. Job control is defined entirely in terms of // these: a shell puts each pipeline in its own process group and signals the // group, not individual pids. Without them a shell's wait loop cannot work, // which is why mksh's job control never made progress. int pgid; int sid; // Controlling terminal, 1-based (1 = /dev/tty1), 0 = none. TIOCSCTTY sets // it; /dev/tty resolves through it. Was impossible to have before there was // more than one terminal to choose between. int ctty; // Real/effective/saved user and group ids. Nothing enforces permissions with // them yet (the on-disk format has no owner field until SchFS), but they // are tracked honestly so getuid()/setuid() stop lying to userland. uint uid, euid, suid; uint gid, egid, sgid; }; #define FD_CLOEXEC 1 // wait(2) status encoding, as understood by musl's WIFEXITED/WEXITSTATUS/ // WIFSIGNALED/WTERMSIG macros: // normal exit -> (code & 0xff) << 8 // killed by signal -> sig & 0x7f #define W_EXITCODE(code, sig) ((((code) & 0xff) << 8) | ((sig) & 0x7f))