#pragma once #include "core/types.h" #include "arch/traps.h" #include "arch/ioperm.h" // Simultaneous mmap() regions per process. // // 64 was enough for every first-party program and for toybox, and not enough // for zsh: musl's allocator takes a fresh mapping per size class and does not // release them eagerly, so compinit -- which reads twelve hundred completion // functions -- ran out of slots and reported "fatal error: out of memory" on // a machine with a gigabyte free. The slots were the limit, not the memory. // // The table is inline in struct proc, so this is NPROC * NVMA * sizeof(struct // vma) of kernel memory, always resident. At 256 that is about 1 MB total, // which is worth it to stop a shell dying on a machine that has 1000 MB spare. #define NVMA 256 struct vma { uint64 addr; uint64 length; int used; // The pages under this mapping are not ours to free on unmap: either they // are device memory (the framebuffer) or they belong to something with a // longer life than this process (a shared memory segment). This was // `is_fb`, which was the only such case until shared memory existed -- // and freeing another process's shared pages on exit is exactly the kind // of bug that shows up somewhere else entirely, hours later. int no_free; int shmid; // shared segment this came from, or -1 }; // --- What threads share --------------------------------------------------- // // A thread is a struct proc like any other, scheduled like any other. What // makes several of them one program is that they point at the same three // objects below, each counted, each freed with its last user: clone() with // CLONE_VM, CLONE_FILES and CLONE_SIGHAND shares them, fork() copies them. // The address space. pgdir is mirrored in every struct proc that uses it, // because the scheduler, trapasm and half the kernel read it from there; it // only changes at exec, which makes a new mm. struct mm { int ref; int locked; // mm_lock(): one mmap/munmap/brk at a time uint64 sz; // top of the heap; p->sz mirrors it uint64 mmap_next; // next free mmap VA (bump allocator, see sys_mmap) pml4e_t *pgdir; struct vma vma[NVMA]; // VMA tracking for mmap struct mapname *names; // which file each span came from; mapname.c }; // The descriptor table, and FD_CLOEXEC beside it. struct files { int ref; struct file *ofile[NOFILE]; char flags[NOFILE]; }; // Signal dispositions: what sigaction() sets. struct sighand { int ref; uint64 handler[32]; // 0 SIG_DFL, 1 SIG_IGN, else an address uint64 restorer[32]; uint64 flags[32]; // sa_flags; SA_RESTART is the one read }; // Per-CPU state struct cpu { uchar id; uchar apicid; // Local APIC ID struct context *scheduler; // swtch() here to enter scheduler volatile uint64 started; // Has the CPU started? int ncli; // Depth of pushcli nesting. int intena; // Were interrupts enabled before pushcli? void *local; // CPU-local storage (GS base) }; extern struct cpu cpus[NCPU]; extern int ncpu; // processors present extern int ncpu_online; // processors actually running // --- per-CPU variables ------------------------------------------------------ // // These were `__thread struct cpu *cpu` and `__thread struct proc *proc`, and // that was wrong in a way that only appears once a second processor runs. // // `__thread` means *per thread*, and the compiler is entitled to assume a // thread's pointer never changes: it reads %fs:0 once and keeps the value in a // register or spilled on the stack across calls. clang did exactly that in // sleep(): // // movq %fs:0x0, %rax # read the thread pointer // movq %rax, -0x18(%rbp) # spill it // callq sched # <-- the CPU can change here // movq -0x18(%rbp), %rax # reload the OLD pointer // movq (%rax,%rcx), %rax # proc, from the previous CPU's block // movq $0x0, 0x40(%rax) # NULL, because that CPU set proc = 0 // // But %fs here is *per CPU*, not per thread, and a kernel thread changes CPU at // every sched(). So the assumption is false for exactly the variables that // most need to be right, and the failure is a null dereference deep in the // scheduler on a machine with more than one core. // // The fix is to read the value at each use, with no cached base: an explicit // %gs-relative access that the compiler cannot hoist across a call. `volatile` // is what forbids that, and the memory clobber on the setters keeps stores in // order with the rest of the scheduler's bookkeeping. // // Layout of the per-CPU block, whose base is in both FS_BASE and GS_BASE, set // by seginit() in kernel/arch/x86_64/vm.c: // // +0 self-pointer required by the x86-64 TLS ABI (%fs:0) // +8 syscall scratch trapasm.S parks %rsp here across the stack switch // +16 struct cpu * // +24 struct proc * // +32 syscall rax the three registers wrmsr destroys, parked across // +40 syscall rcx the FS_BASE swap at the top of syscall_entry // +48 syscall rdx // // Offset 8 is NOT free. syscall_entry uses %fs:(8) as its scratch slot, and // because seginit() sets GS base and FS base to the same address, %fs:8 and // %gs:8 are the same memory. Putting `cpu` there made every syscall overwrite // it with %rsp -- a #GP in syscall_entry itself, several layers below anything // that could report it. // // **Offsets 32..48 were three globals**, `_sc_rax`/`_sc_rcx`/`_sc_rdx` in // vm.c, under a comment saying they were safe because SYSCALL clears IF. IF // answers for interrupts on *this* CPU and says nothing about another CPU // executing the same three stores. Two CPUs entering a syscall together // overwrote each other's parked registers, and %rcx is the user return address // that sysretq uses -- so one process returned into another's instruction // stream. See the note on syscall_entry in trapasm.S. // // trapasm.S reads all of these, so the offsets are shared with assembly and // must not be reordered. #define PERCPU_SELF 0 #define PERCPU_SCRATCH 8 #define PERCPU_CPU 16 #define PERCPU_PROC 24 #define PERCPU_SC_RAX 32 #define PERCPU_SC_RCX 40 #define PERCPU_SC_RDX 48 static inline struct cpu * cpu_get(void) { struct cpu *c; asm volatile("movq %%gs:16, %0" : "=r" (c)); return c; } static inline void cpu_set(struct cpu *c) { asm volatile("movq %0, %%gs:16" :: "r" (c) : "memory"); } static inline struct proc * proc_get(void) { struct proc *p; asm volatile("movq %%gs:24, %0" : "=r" (p)); return p; } static inline void proc_set(struct proc *p) { asm volatile("movq %0, %%gs:24" :: "r" (p) : "memory"); } // Named mycpu()/myproc(), as xv6-riscv does, and *not* spelled `cpu`/`proc`. // A macro by those names would also rewrite every `struct cpu` and // `struct proc`, because macros have no namespace and tags do -- which is // exactly why the variables could carry those names in the first place. #define mycpu() cpu_get() #define myproc() proc_get() // Saved registers for kernel context switches (x86_64 ABI) struct context { uint64 r15; uint64 r14; uint64 r13; uint64 r12; uint64 rbx; uint64 rbp; uint64 rip; }; enum procstate { UNUSED, EMBRYO, SLEEPING, RUNNABLE, RUNNING, ZOMBIE }; // Per-process state // A copy of the interesting fields of one process, taken under ptable.lock. // procfs reads these rather than struct proc, so nothing outside proc.c needs // the lock or the layout. struct procinfo { int pid, ppid, pgid, sid; uint uid; int state; // enum procstate uint64 sz; int killed; int ctty; // controlling terminal, 1-based; 0 = none uint64 utime, stime; // CPU time in ticks, this process uint64 cutime, cstime; // and its reaped children int nice; // scheduling priority, -20..19 uint64 vruntime; // weighted CPU nanoseconds, for `top` and debugging uint64 nvcsw, nivcsw; // voluntary / involuntary context switches char name[16]; char exe[64]; // full path, for /proc//exe int kthread; // a kernel thread, not a program int stopped; // stopped by a signal (ps shows T) int tgid; // the process this thread belongs to int nthreads; // threads in that process }; #define NGROUPS_MAX 32 struct proc { uint64 sz; // Size of process memory (bytes) [+0x00] pml4e_t* pgdir; // Page table (PML4 on x86_64) [+0x08] char *kstack; // Bottom of kernel stack [+0x10] uint64 fs_base; // User TLS base (IA32_FS_BASE MSR) [+0x18] ← trapasm.S depends on this offset enum procstate state; // Process state [+0x20] int pid; // [+0x24] struct proc *parent; // [+0x28] struct trapframe *tf; // [+0x30] struct context *context; // [+0x38] void *chan; // [+0x40] int killed; // [+0x48] int _pad; // [+0x4c] struct files *files; // Open files, shared by threads [+0x50] struct inode *cwd; // follows char name[16]; // // The path exec() was given. `name` is only the last component and only 15 // characters of it, which is what /proc//stat wants; this is the whole // path, reported by /proc//exe. // // Note it is exposed as a regular *file*, not the symlink Linux uses, so // `cat /proc/1/exe` works but readlink() does not. That is why `ps -o comm` // still prints "[init]": toybox readlink()s exe and brackets the name when // that fails. Default ps and top output is unaffected. Fixing it properly // means symlink support in procfs, which is worth doing with the rest of the // VFS work rather than on its own. char exe[64]; // A kernel thread's entry point, or 0 for an ordinary process. // // Set by kthread_create and dispatched by forkret. A kernel thread has a // page directory with only the kernel mapped and never returns to user mode, // so it has no trapframe worth speaking of and no user stack -- it simply // runs `kfn` forever inside the scheduler's ordinary rotation, which is what // makes it able to sleep and to be preempted like anything else. void (*kfn)(void); // Ports this process may use directly, from the I/O BARs of the PCI devices // it has claimed. Empty for everything that is not hosting a driver, which is // everything. See include/arch/ioperm.h. struct ioport_range ioport[NIOPORT]; int nioport; // CPU time, in timer ticks, charged by trap() at each tick to whoever was // running. utime/stime split on the interrupted privilege level -- cs & 3 -- // which is the only information the tick carries about what it interrupted. // // cutime/cstime accumulate *reaped* children, and only reaped ones: that is // what RUSAGE_CHILDREN means, and it is why they are rolled up in wait() // rather than in exit(). A child still running has not contributed yet. uint64 utime, stime; uint64 cutime, cstime; uint64 signal_mask; uint64 pending_signals; // ITIMER_REAL, which alarm() is the whole-second form of: SIGALRM is due at // tick alarm_at (0 = not armed) and then every alarm_every ticks after. // alarmd, in kernel/proc/proc.c, is what posts it. uint alarm_at; uint alarm_every; // Real signal delivery (added for SIGCHLD — see kernel/sys/syscall.c's // deliver_signals() and kernel/proc/proc.c's exit()). sig_handler[sig]/ // sig_restorer[sig] come straight from the user's struct sigaction // (sys_rt_sigaction); 0 means no handler installed. saved_sigtf stashes // the pre-signal trapframe so sys_rt_sigreturn() can restore it; in_signal // prevents re-entering a handler while one is already running (we don't // support nested signal delivery). struct sighand *sighand; // dispositions, shared by threads struct trapframe saved_sigtf; int in_signal; // rt_sigsuspend() support: the pre-suspend mask to restore once the // handler that woke us finishes (sigsuspend swaps the mask atomically // for the duration of the wait -- POSIX). sigsusp_active is set while a // restore is owed; sys_rt_sigreturn() performs it. uint64 sigsusp_oldmask; int sigsusp_active; struct mm *mm; // address space, shared by threads // ABI translation (Linux compat syscall number map) int *syscall_map; int map_size; // Current working directory path; maintained by sys_chdir / sys_fchdir. char cwd_path[256]; // --- POSIX process identity ------------------------------------------- // Appended at the end of the struct deliberately: trapasm.S hard-codes the // offsets of the fields at the top (see the _Static_assert on fs_base in // kernel/proc/proc.c), so nothing may be inserted above them. // Exit status, already encoded the way wait(2) reports it -- see // W_EXITCODE() below. Read by waitpid() before the slot is recycled. int xstate; // Signal that killed us, if any. Set by sigsend() alongside ->killed so the // eventual exit() can report *which* signal, instead of a blanket SIGKILL. int killsig; // Process group and session. Job control is defined entirely in terms of // these: a shell puts each pipeline in its own process group and signals the // group, not individual pids. Without them a shell's wait loop cannot work, // which is why mksh's job control never made progress. int pgid; int sid; // Controlling terminal, 1-based (1 = /dev/tty1), 0 = none. TIOCSCTTY sets // it; /dev/tty resolves through it. Was impossible to have before there was // more than one terminal to choose between. int ctty; // Never scheduled again. Set by sysrq 'w' to produce a hang on purpose -- // see proc_wedge() and docs/architecture/init.md, "The heartbeat". int wedged; // Real/effective/saved user and group ids. Nothing enforces permissions with // them yet (the on-disk format has no owner field until SchFS), but they // are tracked honestly so getuid()/setuid() stop lying to userland. uint uid, euid, suid; uint gid, egid, sgid; // Supplementary groups, as setgroups(2) set them: a file's group is // consulted against these as well as against egid. int ngroups; uint groups[NGROUPS_MAX]; // --- Fair-share scheduling -------------------------------------------- // The whole of the scheduler's per-process state. What each field means, // and why the scheduler is shaped this way, is in kernel/proc/proc.c above // scheduler(); this is deliberately only the declaration. int nice; // -20..19, UNIX convention: lower means more CPU uint32 weight; // nice_weight[nice + 20], cached uint64 vruntime; // weighted CPU time consumed, in nanoseconds uint64 ran_at; // TSC when this process last entered RUNNING uint64 slice_ns; // how long the current slice is allowed to be uint64 slice_used; // how much of it has been spent, in nanoseconds uint64 nivcsw, nvcsw; // preempted / voluntary context switches // The process's x87/SSE registers while it is off the CPU, as fxsave // writes them; and a copy taken when a signal handler starts, put back by // rt_sigreturn. See "The FPU" above scheduler() in kernel/proc/proc.c. uint8 fpu[512] __attribute__((aligned(16))); uint8 sigfpu[512] __attribute__((aligned(16))); // chroot(2): where an absolute path starts, and where ".." stops. 0 is the // system's root. Inherited across fork, held as a reference like cwd. struct inode *root; // Syscall tracing: every call and its result go to the log. Set by exec // when the program's name is the one written to /proc/strace, and kept // across fork so that what a traced program starts is traced too. int strace; // umask(2). Inherited across fork; 022 for the first process. uint umask; // Job control. stopsig is the signal that stopped the process, 0 while it // runs; stop_report and cont_report are what wait(WUNTRACED/WCONTINUED) // has yet to tell the parent. int stopsig; int stop_report; int cont_report; // Threads. tgid is the pid of the thread group's leader -- what getpid() // returns -- and equal to pid for every ordinary process. int tgid; uint64 clear_child_tid; // CLONE_CHILD_CLEARTID: zeroed and woken at exit void *futex_key; // the futex word this thread waits on, or 0 int futex_woken; int autoreap; // a thread: reapd frees it, nobody waits for it struct proc *vfork_parent; // CLONE_VFORK: released at exec or exit int vfork_wait; // the vfork child this process is waiting on int thread_dying; // killed with its group: exits as a thread int group_status; // a thread's exit_group status, for the leader int group_status_set; }; #define FD_CLOEXEC 1 // What a process's FPU starts as: x87 control word 0x37F, every exception // masked, MXCSR 0x1F80 -- the state FNINIT and a reset leave, and what the // System V ABI promises a new program. In proc.c. extern const uint8 fpu_default[512]; // wait(2) status encoding, as understood by musl's WIFEXITED/WEXITSTATUS/ // WIFSIGNALED/WTERMSIG macros: // normal exit -> (code & 0xff) << 8 // killed by signal -> sig & 0x7f #define W_EXITCODE(code, sig) ((((code) & 0xff) << 8) | ((sig) & 0x7f))