nexoniarz/SchOS
nexoniarz/SchOS
Watch
1
0
Fork
You've already forked SchOS
 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions
SchOS/kernel/proc/proc.c
Nexoniarz 4ad49ed5f2 Linux's own r8169 through LDLS, deep idle, and the GPU plan 
The RTL8168 is now driven by Linux's r8169, unmodified, in a process.
`make ldls-r8169` stages it from the running kernel together with libphy,
mdio-bus, mdio_devres and the Realtek PHY driver, the kernel's BTF and
the RTL PHY firmware, and etc/schinit/ldls-r8169.job starts it at boot
with its log in /var/log/ldls-r8169.log. The native driver is out of the
registry; its code, now with Linux's 8168H setup, stays as the fallback.

LDLS grows what that needed:
- ldlsd loads a comma-separated module set in dependency order, each
  linking against the ones before it, with BTF read before any
  init_module runs;
- a driver core (bus match, driver probe), devres, kernel printf, real
  udelay/msleep/ktime and 1000 Hz jiffies, phylib's bitmaps and
  link_mode_params, firmware from /lib/firmware, and the symbols whose
  stub zero was a lie (list validity, try_module_get, dev_err_probe,
  eth_platform_get_mac_address);
- pdev->bus is a real pci_bus, and cmovcc from per-CPU memory is
  rewritten like the other %gs forms.
In QEMU all five modules load, link and initialise, and r8169 registers
and finds no device. Probe and traffic are untested until real hardware.

The kernel turns ASPM off on Realtek NIC links at boot, as r8169 does
through pci_disable_link_state, which a process cannot.

Idle CPUs enter ACPI _CST's C2 (CC6 on Zen) by port read or MWAIT when
the APIC timer keeps running, instead of only ever halting at C1.

docs/plan/gpu.md: amdgpu and nouveau through LDLS, with measurements.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4
2026-10-25 14:26:16 +01:00

3070 lines
94 KiB
C
Raw Blame History

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "arch/memlayout.h"
#include "arch/mmu.h"
#include "arch/x86.h"
#include "arch/traps.h"
#include "core/proc.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "core/sbuf.h"
#include "core/dbg.h"
#include "fs/fs.h"
#include "fs/file.h"
#include "uapi/posix.h"
#include "uapi/errno.h"
#include "fs/blkdev.h"
#include "core/uaccess.h"
#include "dev/acpi.h"
// Verify myproc()->fs_base offset matches the hardcoded 0x688 in trapasm.S.
_Static_assert(__builtin_offsetof(struct proc, fs_base) == 0x18,
"myproc()->fs_base offset changed — update trapasm.S WRITE_USER_FS macro (must be 0x18)");
struct {
struct spinlock lock;
struct proc proc[NPROC];
} ptable;
// CPU time of every process already reaped, so that /proc/stat's busy time
// does not drop each time one is. Under ptable.lock, like the processes.
static uint64 reaped_utime, reaped_stime;
// Kernel stacks, one per ptable slot, statically.
//
// **They used to be a single kalloc() page**, and a measurement said that was
// not enough: painting the stack at allocation and scanning it on every trap
// put the high-water mark at 3736 of 4096 bytes -- 91%, in netd, inside lwIP's
// input path. 360 bytes of headroom on a kernel that takes interrupts on the
// current stack is not headroom. A timer landing at that depth pushes a
// trapframe and runs trap() straight off the bottom of the page and into
// whatever kalloc handed out next, which is very often another process's
// kernel stack.
//
// That is a precise match for the symptom that had been written off as a
// mystery: roughly one shell in four boots died with `trap 14 ... rip 0x0`, a
// user-mode instruction fetch at address zero, having "jumped" somewhere it
// never branched to. A process does not corrupt itself into that state; it
// gets corrupted.
//
// The canary at kstack[0] did not catch it, and could not have. It is checked
// on trap entry and exit, and a frame allocated in one `sub $N, %rsp` can step
// clean over the bottom eight bytes and write below them without ever touching
// the canary. It catches slow leaks past the bottom, not one deep call.
//
// Static and per-slot rather than a bigger kalloc, because kalloc() hands out
// exactly one page and there is no contiguous multi-page allocator to ask. A
// process's ptable slot is fixed for its lifetime, so the stack that belongs
// to that slot is the natural answer: no allocation to fail, no free to
// forget, and the one kalloc failure path in allocproc() disappears with it.
//
// The cost is NPROC * KSTACKSIZE = 1 MB of BSS, paid at link time. That is
// what a kernel stack per process costs; it was being paid out of kalloc
// before, one page at a time, and undercounted.
__attribute__((aligned(PGSIZE)))
static char kstacks[NPROC][KSTACKSIZE];
static struct proc *initproc;
int nextpid = 1;
extern void forkret(void);
extern void syscall_trapret(void);
static void wakeup1(void *chan);
void
kstack_check_canary(const char *where)
{
if(myproc() == 0 || myproc()->kstack == 0)
return;
if(*(uint64*)(myproc()->kstack) != KSTACK_CANARY){
cprintf("kstack canary corrupted for pid %d (%s) at %s: got 0x%p\n",
myproc()->pid, myproc()->name, where, *(uint64*)(myproc()->kstack));
panic("kstack overflow");
}
}
// --- what the debugger can be told about processes --------------------------
//
// docs/architecture/debugging.md, D2. These are here rather than in kernel/dev/dbg.c
// because this file owns ptable, and a view written anywhere else would need
// the lock and the struct exported to get at it. One line at the bottom of the
// file that owns the data is the whole discipline.
static const char *
state_name(enum procstate s)
{
switch (s) {
case UNUSED: return "unused";
case EMBRYO: return "embryo";
case SLEEPING: return "sleep";
case RUNNABLE: return "ready";
case RUNNING: return "run";
case ZOMBIE: return "zombie";
default: return "?";
}
}
// "4" means only pid 4. Not a general query language -- just the one filter
// that stops a table from being unreadable on a busy machine.
static int
want_pid(const char *arg)
{
int v = 0;
if (!arg || !*arg)
return 0; // no filter
while (*arg >= '0' && *arg <= '9')
v = v * 10 + (*arg++ - '0');
return v;
}
static void
proc_dump(struct sbuf *out, const char *arg)
{
struct proc *p;
int only = want_pid(arg);
sb_puts(out, " pid ppid pgid state sz(KB) mmap_next "
"name\n");
acquire(&ptable.lock);
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
if (p->state == UNUSED)
continue;
if (only && p->pid != only)
continue;
sb_putu_pad(out, p->pid, 5);
sb_putu_pad(out, p->parent ? p->parent->pid : 0, 6);
sb_putu_pad(out, p->pgid, 6);
sb_puts(out, " ");
sb_pad(out, state_name(p->state), 8);
sb_putu_pad(out, p->sz / 1024, 6);
sb_puts(out, " 0x");
sb_puthex(out, p->mm ? p->mm->mmap_next : 0, 16);
sb_puts(out, " ");
sb_puts(out, p->name);
if (p->killed)
sb_puts(out, " [killed]");
if (p->wedged)
sb_puts(out, " [wedged]");
sb_putc(out, '\n');
}
release(&ptable.lock);
}
// Every mapping every process holds that it does not own the pages of.
//
// This is the table that decides whether exit() frees a page or leaves it
// alone -- see proc_release_shared() in kernel/sys/sys_mem.c -- and the bug
// that made the debugger worth building was a DMA block that was never in it.
// A view of it is the cheapest possible check that the rule is being kept.
static void
vma_dump(struct sbuf *out, const char *arg)
{
struct proc *p;
int any = 0;
int only = want_pid(arg);
sb_puts(out, " pid slot va bytes "
"no_free shmid name\n");
acquire(&ptable.lock);
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
if (p->state == UNUSED)
continue;
if (only && p->pid != only)
continue;
if (!p->mm || (p->tgid && p->tgid != p->pid))
continue; // a thread: its leader lists the mappings
for (int i = 0; i < NVMA; i++) {
if (!p->mm->vma[i].used)
continue;
any = 1;
sb_putu_pad(out, p->pid, 5);
sb_putu_pad(out, i, 6);
sb_puts(out, " 0x");
sb_puthex(out, p->mm->vma[i].addr, 16);
sb_putu_pad(out, p->mm->vma[i].length, 12);
sb_puts(out, p->mm->vma[i].no_free ? " yes " : " no ");
if (p->mm->vma[i].shmid >= 0)
sb_putu_pad(out, p->mm->vma[i].shmid, 5);
else
sb_puts(out, " -");
sb_puts(out, " ");
sb_puts(out, p->name);
sb_putc(out, '\n');
}
}
release(&ptable.lock);
if (!any)
sb_puts(out, only ? " (that pid holds no mappings)\n"
: " (no process is holding a mapping)\n");
}
// Every page table that maps this physical page, and which VMA it falls in.
//
// This is the half of `whose` that no allocator can answer. An allocator knows
// whether it has handed a page out; it has no idea whether anyone still has it
// mapped, and "free, but three processes are still writing to it" is the exact
// shape of the bug that prompted all of this.
//
// The walk is forward -- there is no reverse map -- so it costs one pass over
// every mapped page of every live process. See vm_walk_user().
struct maphit {
struct sbuf *out;
struct whose *w;
struct proc *p;
};
static void
maphit(void *ctx, uint64 va, uint64 pa, uint64 pte)
{
struct maphit *m = (struct maphit *)ctx;
if (pa != m->w->phys)
return;
m->w->mappings++;
sb_puts(m->out, " mapped pid ");
sb_putu(m->out, m->p->pid);
sb_puts(m->out, " (");
sb_puts(m->out, m->p->name);
sb_puts(m->out, ") at 0x");
sb_puthex(m->out, va, 16);
sb_puts(m->out, (pte & PTE_W) ? " rw" : " r-");
sb_puts(m->out, (pte & PTE_U) ? "u" : "-");
// And whether the process admits to holding it. A mapping with no VMA is
// how the DMA blocks used to look, which is why exit() freed them twice.
{
int i, found = 0;
for (i = 0; i < NVMA; i++) {
if (!m->p->mm->vma[i].used)
continue;
if (va < m->p->mm->vma[i].addr ||
va >= m->p->mm->vma[i].addr + m->p->mm->vma[i].length)
continue;
found = 1;
sb_puts(m->out, " vma[");
sb_putu(m->out, i);
sb_puts(m->out, m->p->mm->vma[i].no_free ? "] no_free" : "]");
break;
}
if (!found) {
if (va < m->p->sz)
sb_puts(m->out, " heap");
else
sb_puts(m->out, " NO VMA -- exit() will free this page");
}
}
sb_putc(m->out, '\n');
}
static void
proc_whose(struct sbuf *out, struct whose *w)
{
struct proc *p;
acquire(&ptable.lock);
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
struct maphit m;
if (p->state == UNUSED || p->pgdir == 0)
continue;
m.out = out; m.w = w; m.p = p;
vm_walk_user(p->pgdir, maphit, &m);
}
release(&ptable.lock);
}
// --- Shared objects --------------------------------------------------------
//
// One of each per slot is enough by construction: every struct proc holds at
// most one mm, one files and one sighand, so NPROC of each can never run out.
// ref 0 is free; -1 is being torn down and not yet free, so that a slot whose
// last user is still freeing it cannot be handed out from under it.
//
// objlock covers the counts and nothing else, and is never held while taking
// ptable.lock. mm_lock() sleeps on mmwait, a separate lock for the same reason.
static struct mm mmtab[NPROC];
static struct files filestab[NPROC];
static struct sighand sightab[NPROC];
static struct spinlock objlock, mmwait;
struct mm *
mm_alloc(void)
{
acquire(&objlock);
for (struct mm *m = mmtab; m < &mmtab[NPROC]; m++)
if (m->ref == 0) {
memset(m, 0, sizeof(*m));
for (int i = 0; i < NVMA; i++)
m->vma[i].shmid = -1;
m->ref = 1;
release(&objlock);
return m;
}
release(&objlock);
return 0;
}
void
mm_get(struct mm *m)
{
acquire(&objlock);
m->ref++;
release(&objlock);
}
// Drop a reference. The last one releases shared segments and frees the page
// tables, which nobody can be running on any more: a thread's reference is
// only dropped once it is off the CPU for good.
void
mm_put(struct mm *m)
{
if (m == 0)
return;
acquire(&objlock);
int last = --m->ref == 0;
if (last)
m->ref = -1;
release(&objlock);
if (!last)
return;
mm_release_shared(m);
mapname_free(m);
if (m->pgdir)
freevm(m->pgdir);
m->pgdir = 0;
acquire(&objlock);
m->ref = 0;
release(&objlock);
}
int
mm_shared(struct mm *m)
{
return m && m->ref > 1;
}
// One mmap, munmap or brk at a time per address space. Sleeps: an mmap of a
// file reads it.
void
mm_lock(struct mm *m)
{
acquire(&mmwait);
while (m->locked)
sleep(m, &mmwait);
m->locked = 1;
release(&mmwait);
}
void
mm_unlock(struct mm *m)
{
acquire(&mmwait);
m->locked = 0;
wakeup(m);
release(&mmwait);
}
struct files *
files_alloc(void)
{
acquire(&objlock);
for (struct files *f = filestab; f < &filestab[NPROC]; f++)
if (f->ref == 0) {
memset(f, 0, sizeof(*f));
f->ref = 1;
release(&objlock);
return f;
}
release(&objlock);
return 0;
}
void
files_get(struct files *f)
{
acquire(&objlock);
f->ref++;
release(&objlock);
}
// The last reference closes every descriptor. fileclose can sleep, so this is
// for process context only -- exit(), not the reaper.
void
files_put(struct files *f)
{
if (f == 0)
return;
acquire(&objlock);
int last = --f->ref == 0;
if (last)
f->ref = -1;
release(&objlock);
if (!last)
return;
for (int fd = 0; fd < NOFILE; fd++)
if (f->ofile[fd]) {
struct file *x = f->ofile[fd];
f->ofile[fd] = 0;
f->flags[fd] = 0;
fileclose(x);
}
acquire(&objlock);
f->ref = 0;
release(&objlock);
}
// Slot allocation in a table two threads may be filling at once.
void
files_lock(void)
{
acquire(&mmwait);
}
void
files_unlock(void)
{
release(&mmwait);
}
struct sighand *
sighand_alloc(void)
{
acquire(&objlock);
for (struct sighand *s = sightab; s < &sightab[NPROC]; s++)
if (s->ref == 0) {
memset(s, 0, sizeof(*s));
s->ref = 1;
release(&objlock);
return s;
}
release(&objlock);
return 0;
}
void
sighand_get(struct sighand *s)
{
acquire(&objlock);
s->ref++;
release(&objlock);
}
void
sighand_put(struct sighand *s)
{
if (s == 0)
return;
acquire(&objlock);
s->ref--;
release(&objlock);
}
void
pinit(void)
{
initlock(&objlock, "procobj");
initlock(&mmwait, "mmwait");
initlock(&ptable.lock, "ptable");
dbg_register("proc", proc_dump, "every process, with its state and size");
dbg_register("vma", vma_dump, "mappings a process holds but does not own");
dbg_register_whose("mapped", proc_whose);
}
static struct proc*
allocproc(void)
{
struct proc *p;
char *sp;
acquire(&ptable.lock);
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++)
if(p->state == UNUSED)
goto found;
release(&ptable.lock);
return 0;
found:
p->state = EMBRYO;
p->pid = nextpid++;
// ptable slots get reused once a previous occupant is reaped, but
// nothing previously cleared the signal-delivery fields added for
// SIGCHLD (sig_handler/sig_restorer/pending_signals/signal_mask/
// in_signal/saved_sigtf) — a freshly allocated process would silently
// inherit whatever a totally unrelated earlier process left behind,
// including stale handler addresses. deliver_signals() (syscall.c)
// would then jump to garbage on this new process's very first syscall
// return, faulting at whatever bogus rip happened to be sitting there.
p->pending_signals = 0;
p->alarm_at = p->alarm_every = 0; // a child's timers start disarmed
p->utime = p->stime = p->cutime = p->cstime = 0;
p->strace = 0;
p->stopsig = p->stop_report = p->cont_report = 0;
p->umask = 022;
// A recycled slot must not inherit a dead process's place in the queue --
// in either direction. sched_init_proc starts it at min_vruntime, which is
// level with everything currently runnable: neither owed CPU nor in debt.
// fork() then copies the parent's vruntime over this, which is what stops a
// process from getting a fresh share of the CPU simply by forking.
sched_init_proc(p, 0);
memmove(p->fpu, fpu_default, sizeof(p->fpu));
p->signal_mask = 0;
p->in_signal = 0;
p->sigsusp_oldmask = 0;
p->sigsusp_active = 0;
// Same reasoning as the signal fields above: a recycled slot must not
// inherit a dead process's identity or exit status.
p->xstate = 0;
p->killsig = 0;
p->pgid = 0;
p->sid = 0;
p->uid = p->euid = p->suid = 0;
p->gid = p->egid = p->sgid = 0;
release(&ptable.lock);
// The stack that belongs to this slot. Cannot fail, which is why there is no
// failure branch here any more -- see kstacks[] above.
p->kstack = kstacks[p - ptable.proc];
// Canary at the very bottom: a slow walk past the end clobbers this first,
// and it is checked on every trap()/syscall() entry. It does *not* catch a
// single frame large enough to step over it, which is the case kstacks[]
// being two pages is there to prevent.
*(uint64*)(p->kstack) = KSTACK_CANARY;
sp = p->kstack + KSTACKSIZE;
sp -= sizeof(struct trapframe);
p->tf = (struct trapframe*)sp;
sp -= sizeof(addr_t);
*(addr_t*)sp = (addr_t)syscall_trapret;
sp -= sizeof(struct context);
p->context = (struct context*)sp;
memset(p->context, 0, sizeof *p->context);
p->context->rip = (addr_t)forkret;
p->syscall_map = 0;
p->map_size = 0;
p->kfn = 0;
p->tgid = p->pid;
p->clear_child_tid = 0;
p->futex_key = 0;
p->futex_woken = 0;
p->autoreap = 0;
p->vfork_parent = 0;
p->vfork_wait = 0;
p->thread_dying = 0;
p->group_status_set = 0;
// Every process starts with its own of each; clone() swaps in the parent's
// to make a thread. The pools cannot run dry -- see mm_alloc().
p->mm = mm_alloc();
p->files = files_alloc();
p->sighand = sighand_alloc();
if (!p->mm || !p->files || !p->sighand)
panic("allocproc: object pools");
return p;
}
void
sinit(void)
{
struct proc *p;
extern char _binary_initcode_start[], _binary_initcode_size[];
p = allocproc();
initproc = p;
if((p->pgdir = setupkvm()) == 0)
panic("sinit: setupkvm failed");
inituvm(p->pgdir, _binary_initcode_start, (uint64)_binary_initcode_size);
if(allocuvm(p->pgdir, PGSIZE, 32 * PGSIZE) == 0)
panic("sinit: allocuvm failed");
p->sz = 32 * PGSIZE;
p->mm->sz = p->sz;
p->mm->pgdir = p->pgdir;
memset(p->tf, 0, sizeof(*p->tf));
p->tf->cs = (SEG_UCODE << 3) | DPL_USER;
p->tf->ss = (SEG_UDATA << 3) | DPL_USER;
p->tf->rflags = FL_IF;
p->tf->rsp = p->sz;
p->tf->rip = 0;
// PID 1 leads its own session and process group, and runs as root.
p->pgid = p->pid;
p->sid = p->pid;
p->uid = p->euid = p->suid = 0;
p->gid = p->egid = p->sgid = 0;
safestrcpy(p->name, "sinit", sizeof(p->name));
p->cwd = namei("/");
p->cwd_path[0] = '/'; p->cwd_path[1] = 0;
p->fs_base = 0;
p->state = RUNNABLE;
}
int
growproc(int64 n)
{
addr_t sz;
struct mm *m = myproc()->mm;
mm_lock(m);
sz = m->sz;
if(n > 0){
if((sz = allocuvm(myproc()->pgdir, sz, sz + n)) == 0){
mm_unlock(m);
return -1;
}
} else if(n < 0){
if((sz = deallocuvm(myproc()->pgdir, sz, sz + n)) == 0){
mm_unlock(m);
return -1;
}
}
m->sz = sz;
myproc()->sz = sz;
mm_unlock(m);
switchuvm(myproc());
return 0;
}
// The leader of p's thread group: the process, as its parent sees it.
struct proc *
proc_leader(struct proc *p)
{
if (p->tgid == p->pid)
return p;
for (struct proc *q = ptable.proc; q < &ptable.proc[NPROC]; q++)
if (q->pid == p->tgid && q->state != UNUSED)
return q;
return p;
}
// Give back a slot that fork() or clone() could not finish.
static void
fork_undo(struct proc *np)
{
mm_put(np->mm); // frees the half-copied pgdir with it
files_put(np->files);
sighand_put(np->sighand);
np->mm = 0;
np->files = 0;
np->sighand = 0;
np->pgdir = 0;
np->kstack = 0; // the slot keeps its stack; see kstacks[]
np->state = UNUSED;
}
int
fork(void)
{
int i, pid;
struct proc *np;
if((np = allocproc()) == 0)
return -1;
// The parent's registers are live in the CPU -- the kernel never touches
// them -- so the child's copy is taken straight from there.
fxsave(np->fpu);
// Another thread must not be reshaping the address space mid-copy.
struct mm *pm = myproc()->mm;
mm_lock(pm);
if((np->pgdir = copyuvm(myproc()->pgdir, pm->sz)) == 0){
mm_unlock(pm);
fork_undo(np);
return -1;
}
np->sz = pm->sz;
np->mm->sz = pm->sz;
np->mm->pgdir = np->pgdir;
// A child of any thread is a child of the process.
np->parent = proc_leader(myproc());
// copyuvm() only duplicates [0, sz) — mmap()'d regions live above that
// and were never copied, leaving the child's vma[] bookkeeping (stale
// from whichever process last occupied this ptable slot) pointing at
// memory that isn't actually mapped in np->pgdir. Reset and rebuild it
// from the parent's real, current mappings.
memset(np->mm->vma, 0, sizeof(np->mm->vma));
for(i = 0; i < NVMA; i++){
if(!myproc()->mm->vma[i].used)
continue;
if(copy_vma_range(np->pgdir, myproc()->pgdir, myproc()->mm->vma[i].addr,
myproc()->mm->vma[i].length, myproc()->mm->vma[i].no_free) < 0){
mm_unlock(pm);
fork_undo(np);
return -1;
}
np->mm->vma[i] = myproc()->mm->vma[i];
if(np->mm->vma[i].shmid >= 0)
shm_ref(np->mm->vma[i].shmid);
}
np->mm->mmap_next = myproc()->mm->mmap_next;
mapname_copy(np->mm, pm);
mm_unlock(pm);
memmove(np->tf, myproc()->tf, sizeof(struct trapframe));
np->syscall_map = myproc()->syscall_map;
np->map_size = myproc()->map_size;
// fork() preserves the parent's signal handlers and mask (POSIX); the
// child starts with no pending signals of its own, though — those
// aren't inherited. allocproc() already zeroed pending_signals/
// in_signal/saved_sigtf for this fresh slot.
memmove(np->sighand->handler, myproc()->sighand->handler, sizeof(np->sighand->handler));
memmove(np->sighand->flags, myproc()->sighand->flags, sizeof(np->sighand->flags));
memmove(np->sighand->restorer, myproc()->sighand->restorer, sizeof(np->sighand->restorer));
np->signal_mask = myproc()->signal_mask;
np->tf->rax = 0;
files_lock();
for(i = 0; i < NOFILE; i++)
if(myproc()->files->ofile[i]) {
np->files->ofile[i] = filedup(myproc()->files->ofile[i]);
np->files->flags[i] = myproc()->files->flags[i];
}
files_unlock();
// POSIX: the child inherits process group, session and all four ids.
np->pgid = myproc()->pgid;
np->sid = myproc()->sid;
// The controlling terminal is a property of the *session*, so it crosses
// fork() with the session id rather than being re-established. It was set
// only by TIOCSCTTY and never inherited, so every child of a shell had
// ctty 0 -- which is why plain `ps` listed the shell and nothing it had
// started, background jobs included. A job that is running on your terminal
// is on your terminal whoever forked it.
np->ctty = myproc()->ctty;
np->uid = myproc()->uid; np->euid = myproc()->euid; np->suid = myproc()->suid;
np->gid = myproc()->gid; np->egid = myproc()->egid; np->sgid = myproc()->sgid;
np->cwd = idup(myproc()->cwd);
np->root = myproc()->root ? idup(myproc()->root) : 0;
memmove(np->cwd_path, myproc()->cwd_path, sizeof(myproc()->cwd_path));
np->fs_base = myproc()->fs_base;
safestrcpy(np->name, myproc()->name, sizeof(myproc()->name));
np->strace = myproc()->strace;
np->umask = myproc()->umask;
// The child inherits the parent's place in the queue, priority and all.
//
// Both halves matter. Inheriting *nice* is what POSIX says and what makes
// `nice make -j8` mean anything at all -- the point of renicing a build is
// that the compilers it forks are reniced too. Inheriting *vruntime* is the
// part with teeth: were the child to start level with everyone, a process
// could reset its own CPU debt by forking, and something that forks per
// request would collect a fresh full share every time while a long-running
// process kept paying for everything it had ever used.
sched_set_nice(np, myproc()->nice);
np->vruntime = myproc()->vruntime;
pid = np->pid;
__sync_synchronize();
np->state = RUNNABLE;
return pid;
}
// status is already wait(2)-encoded -- see W_EXITCODE() in proc.h. Callers
// that die from a fault or signal pass W_EXITCODE(0, sig); a normal _exit(n)
// passes W_EXITCODE(n, 0).
// PID 1 is a role, and the kernel guarantees the role is filled.
//
// This is the rule change docs/architecture/init.md exists for. Unix's rule is "PID 1
// is special and must never die", honoured since 1975 by being careful --
// which is not a mechanism, it is a hope, and the thing it hopes about is the
// one program on the machine with no supervisor.
//
// So instead: tear nothing down, keep pid 1, and re-exec in place.
//
// The reason this is so short is the reason it is the right design. exec()
// already replaces the address space, and everything else is *deliberately*
// left alone:
//
// - the pid stays 1, so every orphan the kernel reparented to initproc is
// still correctly parented and initproc is still the pointer it was
// - the children survive, because nothing killed them
// - the open files survive, so the new image still has a console to
// complain to -- this runs before exit() closes anything, which is why
// the call site is the first line of exit() and not the last
// - the state survives, in /proc/schinit, which is the whole reason the
// job table lives outside the process holding it
//
// A sibling init resurrecting a dead one would have to solve all four of those
// and could not solve the first at all. This solves them by not creating them.
//
// The escalation ladder is what makes it safe to have. A machine that cannot
// hold PID 1 up is not a machine that should keep trying for ever, and PSOGM
// renders a stop code, a call chain and a QR code of the whole report -- which
// is a far better end than a silent loop.
//
// generation 1 normal boot
// generation 2-3 re-exec, keep the jobs, carry on
// generation 4 re-exec, but ignore /etc/schinit entirely and start only
// a console shell: a manifest that kills init must not
// kill it for ever
// generation 5 stop
uint init_generation = 1;
int
proc_reincarnate(int status)
{
static char gen[12];
char *argv[3];
int n = 0, v;
init_generation++;
if (init_generation > 5)
panic("init resurrection failed: five generations is enough");
// Decimal, by hand: this runs in a process that is dying and should call as
// little as possible.
v = (int)init_generation;
if (v == 0) {
gen[n++] = '0';
} else {
char tmp[12];
int i = 0;
while (v > 0) { tmp[i++] = '0' + (v % 10); v /= 10; }
while (i-- > 0) gen[n++] = tmp[i];
}
gen[n] = 0;
cprintf("init: pid 1 exited with status %d -- reincarnating, generation %d\n",
status, init_generation);
if (init_generation >= 4)
cprintf("init: generation %d starts a console shell only, "
"ignoring /etc/schinit\n", init_generation);
argv[0] = "schinit";
argv[1] = gen;
argv[2] = 0;
// Clear `killed` first.
//
// exit() was reached from the trap-return path *because* killed was set, and
// that path checks it again on the way back out. Leaving it set would exit
// the brand-new image before its first instruction, and the generation
// counter would climb to five in a few microseconds -- a resurrection loop
// that looks exactly like the crash it was meant to survive.
myproc()->killed = 0;
// No lock is held here and none may be taken: this is the first line of
// exit(), before the file table is walked and before ptable.lock. exec()
// does its own begin_op/end_op and is safe from process context.
if (exec("/bin/schinit", argv, 0) < 0)
return -1;
return 0;
}
// --- Threads ---------------------------------------------------------------
static void sched_enqueue(struct proc *p);
static int reap_chan;
// Every other live thread of self's group: killed, and woken so it notices.
// Non-leaders leave as threads; the leader, if it is one of them, leaves as
// the process. With ptable.lock.
static void
group_kill(struct proc *self)
{
for (struct proc *q = ptable.proc; q < &ptable.proc[NPROC]; q++) {
if (q == self || q->tgid != self->tgid)
continue;
if (q->state == UNUSED || q->state == ZOMBIE || q->state == EMBRYO)
continue;
q->killed = 1;
if (q->tgid != q->pid)
q->thread_dying = 1;
if (q->state == SLEEPING) {
q->state = RUNNABLE;
sched_enqueue(q);
}
}
}
// A non-leader thread ending the program: the leader exits with its status.
static void
group_exit(struct proc *p, int status)
{
acquire(&ptable.lock);
struct proc *l = proc_leader(p);
if (l != p && !l->group_status_set) {
l->group_status = status;
l->group_status_set = 1;
}
group_kill(p);
release(&ptable.lock);
}
// Live threads in group tgid, the caller included.
int
proc_thread_count(int tgid)
{
int n = 0;
acquire(&ptable.lock);
for (struct proc *q = ptable.proc; q < &ptable.proc[NPROC]; q++)
if (q->tgid == tgid && q->state != UNUSED && q->state != ZOMBIE)
n++;
release(&ptable.lock);
return n;
}
// A vfork parent waits until its child has its own address space or is gone.
static void
vfork_release(struct proc *p)
{
struct proc *vp = p->vfork_parent;
if (vp == 0)
return;
acquire(&ptable.lock);
if (vp->vfork_wait == p->pid) {
vp->vfork_wait = 0;
wakeup1(&vp->vfork_wait);
}
p->vfork_parent = 0;
release(&ptable.lock);
}
void
proc_exec_done(void)
{
vfork_release(myproc());
}
// An exec leaves one thread: the others die, and the caller becomes the
// process if it was not already. Dispositions are unshared, ignoring stays
// ignored and handlers reset -- exec does the resetting on its own copy.
void
proc_exec_threads(void)
{
struct proc *p = myproc();
acquire(&ptable.lock);
group_kill(p);
if (p->tgid != p->pid) {
struct proc *l = proc_leader(p);
p->parent = l->parent;
p->tgid = p->pid;
p->autoreap = 0;
}
release(&ptable.lock);
if (p->sighand->ref > 1) {
struct sighand *s = sighand_alloc();
if (s) {
memmove(s->handler, p->sighand->handler, sizeof(s->handler));
memmove(s->flags, p->sighand->flags, sizeof(s->flags));
sighand_put(p->sighand);
p->sighand = s;
}
}
}
// CLONE_CHILD_CLEARTID: the word pthread_join waits on.
static void
clear_tid(struct proc *p)
{
if (p->clear_child_tid == 0 || p->pgdir == 0)
return;
int *k = (int*)uva2ka(p->pgdir, (char*)p->clear_child_tid);
if (k && ((addr_t)k & 3) == 0) {
*k = 0;
futex_wake_key(k, 1);
}
p->clear_child_tid = 0;
}
// The end of one thread. Its descriptors and directories go now; its slot and
// its hold on the address space go when reapd sees it is off the CPU.
void
thread_exit(void)
{
struct proc *p = myproc();
clear_tid(p);
vfork_release(p);
files_put(p->files);
p->files = 0;
begin_op();
if (p->cwd)
iput(p->cwd);
if (p->root)
iput(p->root);
end_op();
p->cwd = 0;
p->root = 0;
acquire(&ptable.lock);
p->autoreap = 1;
p->state = ZOMBIE;
wakeup1(&reap_chan);
sched();
panic("zombie thread exit");
}
// Frees threads that have exited. A kernel thread because the last reference
// to an address space may be dropped here, and freeing one is not something
// to do inside the scheduler or under ptable.lock.
static void
reapd(void)
{
acquire(&ptable.lock);
for (;;) {
struct proc *p, *found = 0;
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++)
if (p->state == ZOMBIE && p->autoreap) {
found = p;
break;
}
if (!found) {
sleep(&reap_chan, &ptable.lock);
continue;
}
p = found;
struct mm *m = p->mm;
struct sighand *s = p->sighand;
struct proc *l = proc_leader(p);
if (l != p) {
l->utime += p->utime; // a program's time is all of its threads'
l->stime += p->stime;
} else {
reaped_utime += p->utime;
reaped_stime += p->stime;
}
p->mm = 0;
p->sighand = 0;
p->pgdir = 0;
p->kstack = 0;
p->pid = 0;
p->tgid = 0;
p->parent = 0;
p->name[0] = 0;
p->killed = 0;
p->thread_dying = 0;
p->autoreap = 0;
p->xstate = 0;
p->killsig = 0;
p->pgid = 0;
p->sid = 0;
p->state = UNUSED;
// The leader may have been waiting on its threads to be reapable.
if (l != p && l->state == ZOMBIE && l->parent)
wakeup1(l->parent);
release(&ptable.lock);
mm_put(m);
sighand_put(s);
acquire(&ptable.lock);
}
}
void
thread_reaper_start(void)
{
if (kthread_create(reapd, "reapd") < 0)
cprintf("proc: no reaper; exited threads will not be freed\n");
}
#define CLONE_FILES_ 0x00000400
#define CLONE_VFORK_ 0x00004000
#define CLONE_PARENT_ 0x00008000
#define CLONE_SETTLS_ 0x00080000
#define CLONE_PARENT_SETTID_ 0x00100000
#define CLONE_CHILD_CLEARTID_ 0x00200000
#define CLONE_CHILD_SETTID_ 0x01000000
// clone(2), Linux's argument order: flags, stack, parent_tid, child_tid, tls.
//
// What musl's pthread_create asks for -- VM, FS, FILES, SIGHAND, THREAD,
// SYSVSEM, SETTLS, PARENT_SETTID, CHILD_CLEARTID -- and what its posix_spawn
// asks for -- VM, VFORK and a stack -- are the two shapes that matter.
int
clone_proc(uint64 flags, uint64 stack, uint64 ptid, uint64 ctid, uint64 tls)
{
struct proc *cur = myproc(), *np;
int i, pid;
if (!(flags & CLONE_VM)) {
if (stack || (flags & (CLONE_THREAD | CLONE_SIGHAND)))
return -EINVAL;
return fork(); // plain fork, which is all that is left
}
if ((flags & CLONE_THREAD) && !(flags & CLONE_SIGHAND))
return -EINVAL;
if ((np = allocproc()) == 0)
return -EAGAIN;
mm_put(np->mm);
np->mm = cur->mm;
mm_get(cur->mm);
np->pgdir = cur->pgdir;
np->sz = cur->mm->sz;
if (flags & CLONE_FILES_) {
files_put(np->files);
np->files = cur->files;
files_get(cur->files);
} else {
files_lock();
for (i = 0; i < NOFILE; i++)
if (cur->files->ofile[i]) {
np->files->ofile[i] = filedup(cur->files->ofile[i]);
np->files->flags[i] = cur->files->flags[i];
}
files_unlock();
}
if (flags & CLONE_SIGHAND) {
sighand_put(np->sighand);
np->sighand = cur->sighand;
sighand_get(cur->sighand);
} else {
memmove(np->sighand->handler, cur->sighand->handler, sizeof(np->sighand->handler));
memmove(np->sighand->flags, cur->sighand->flags, sizeof(np->sighand->flags));
memmove(np->sighand->restorer, cur->sighand->restorer, sizeof(np->sighand->restorer));
}
fxsave(np->fpu);
memmove(np->tf, cur->tf, sizeof(struct trapframe));
np->tf->rax = 0;
if (stack)
np->tf->rsp = stack; // the top of the trapframe is the sysret RSP
np->fs_base = (flags & CLONE_SETTLS_) ? tls : cur->fs_base;
np->syscall_map = cur->syscall_map;
np->map_size = cur->map_size;
np->signal_mask = cur->signal_mask;
np->pgid = cur->pgid;
np->sid = cur->sid;
np->ctty = cur->ctty;
np->uid = cur->uid; np->euid = cur->euid; np->suid = cur->suid;
np->gid = cur->gid; np->egid = cur->egid; np->sgid = cur->sgid;
np->cwd = idup(cur->cwd);
np->root = cur->root ? idup(cur->root) : 0;
memmove(np->cwd_path, cur->cwd_path, sizeof(cur->cwd_path));
safestrcpy(np->name, cur->name, sizeof(cur->name));
safestrcpy(np->exe, cur->exe, sizeof(cur->exe));
np->strace = cur->strace;
np->umask = cur->umask;
sched_set_nice(np, cur->nice);
np->vruntime = cur->vruntime;
acquire(&ptable.lock);
struct proc *leader = proc_leader(cur);
if (flags & CLONE_THREAD) {
np->tgid = cur->tgid;
np->parent = leader->parent;
np->autoreap = 1;
} else {
np->tgid = np->pid;
np->parent = (flags & CLONE_PARENT_) ? leader->parent : leader;
}
release(&ptable.lock);
pid = np->pid;
if (flags & CLONE_CHILD_CLEARTID_)
np->clear_child_tid = ctid;
if (flags & CLONE_PARENT_SETTID_)
put_user(pid, ptid);
if (flags & CLONE_CHILD_SETTID_)
put_user(pid, ctid); // the same address space, so the same word
if (flags & CLONE_VFORK_) {
np->vfork_parent = cur;
cur->vfork_wait = pid;
}
__sync_synchronize();
acquire(&ptable.lock);
np->state = RUNNABLE;
sched_enqueue(np);
// vfork: the parent sleeps until the child has execed or exited, because
// until then they share one stack's worth of memory.
if (flags & CLONE_VFORK_)
while (cur->vfork_wait == pid && !cur->killed)
sleep(&cur->vfork_wait, &ptable.lock);
release(&ptable.lock);
return pid;
}
// --- Futexes ---------------------------------------------------------------
//
// futex() used to say EAGAIN to every wait and succeed at every wake, which
// was enough while there was one thread per process. A waiter is keyed by the
// kernel address of the word -- the physical page, in effect -- so two
// mappings of one shared page meet on the same key, as they must.
int
futex_wake_key(void *k, int n)
{
int woke = 0;
acquire(&ptable.lock);
for (struct proc *q = ptable.proc; q < &ptable.proc[NPROC] && woke < n; q++) {
if (q->futex_key != k || q->futex_woken)
continue;
q->futex_woken = 1;
woke++;
if (q->state == SLEEPING) {
q->state = RUNNABLE;
sched_enqueue(q);
}
}
release(&ptable.lock);
return woke;
}
static int *
futex_word(addr_t uaddr)
{
if (uaddr & 3)
return 0;
return (int*)uva2ka(myproc()->pgdir, (char*)uaddr);
}
// Wait while *uaddr == val, for at most `timeout` ticks (0: no limit).
static int
futex_wait(addr_t uaddr, int val, uint64 timeout)
{
struct proc *p = myproc();
int *k = futex_word(uaddr);
int r;
if (k == 0)
return -EFAULT;
acquire(&ptable.lock);
// Checked under the lock every waker takes, so a wake between the check
// and the sleep cannot be missed.
if (*(volatile int*)k != val) {
release(&ptable.lock);
return -EAGAIN;
}
p->futex_key = k;
p->futex_woken = 0;
uint t0 = ticks;
for (;;) {
if (p->futex_woken) { r = 0; break; }
if (p->killed || sig_attention(p)) { r = -EINTR; break; }
if (timeout && (uint64)(uint)(ticks - t0) >= timeout) { r = -ETIMEDOUT; break; }
sleep(timeout ? (void*)&ticks : (void*)k, &ptable.lock);
}
p->futex_key = 0;
release(&ptable.lock);
return r;
}
// Move up to n waiters from one word to another, still waiting.
static int
futex_requeue(int *from, int *to, int n)
{
int moved = 0;
acquire(&ptable.lock);
for (struct proc *q = ptable.proc; q < &ptable.proc[NPROC] && moved < n; q++)
if (q->futex_key == from && !q->futex_woken) {
q->futex_key = to;
moved++;
}
release(&ptable.lock);
return moved;
}
static uint64
ts_ticks(const struct timespec *ts)
{
if (ts->tv_sec < 0 || ts->tv_nsec < 0)
return 1;
uint64 t = (uint64)ts->tv_sec * HZ +
((uint64)ts->tv_nsec * HZ + 999999999) / 1000000000;
return t ? t : 1;
}
int
futex_op(addr_t uaddr, int op, int val, addr_t utime, addr_t uaddr2, int val3)
{
int cmd = op & 0x7f; // without PRIVATE and CLOCK_REALTIME
struct timespec ts;
uint64 timeout = 0;
switch (cmd) {
case 0: // FUTEX_WAIT: relative timeout
if (utime) {
if (copy_from_user(&ts, utime, sizeof(ts)) < 0)
return -EFAULT;
timeout = ts_ticks(&ts);
}
return futex_wait(uaddr, val, timeout);
case 9: { // FUTEX_WAIT_BITSET: absolute timeout
if (utime) {
if (copy_from_user(&ts, utime, sizeof(ts)) < 0)
return -EFAULT;
uint64 sec, usec;
wallclock(&sec, &usec);
if (!(op & 256))
sec -= boot_epoch; // CLOCK_MONOTONIC unless asked otherwise
long ds = ts.tv_sec - (long)sec, dn = ts.tv_nsec - (long)(usec * 1000);
if (dn < 0) { dn += 1000000000; ds--; }
if (ds < 0)
return -ETIMEDOUT;
ts.tv_sec = ds;
ts.tv_nsec = dn;
timeout = ts_ticks(&ts);
}
return futex_wait(uaddr, val, timeout);
}
case 1: // FUTEX_WAKE
case 10: { // FUTEX_WAKE_BITSET
int *k = futex_word(uaddr);
return k ? futex_wake_key(k, val) : -EFAULT;
}
case 3: // FUTEX_REQUEUE
case 4: { // FUTEX_CMP_REQUEUE
int *k = futex_word(uaddr), *k2 = futex_word(uaddr2);
if (!k || !k2)
return -EFAULT;
if (cmd == 4 && *(volatile int*)k != val3)
return -EAGAIN;
int n = futex_wake_key(k, val);
return n + futex_requeue(k, k2, (int)utime); // val2 rides in utime
}
}
return -ENOSYS;
}
void
exit(int status)
{
struct proc *p;
int fd;
// Killed by a fault of its own -- the trapframe holds an exception, not a
// syscall -- so say which file it was executing. The register dump from
// trap() gives addresses; this gives the library they are in.
{
int sig = status & 0x7f;
struct trapframe *tf = myproc()->tf;
if ((sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGFPE) &&
tf && (tf->cs & 3) == DPL_USER && tf->trapno < 32)
mapname_report(myproc());
}
if(myproc() == initproc) {
// Returning from exit() is the *success* path here. exec() does not fail
// to return -- it returns 0 having replaced the address space, and the new
// image begins running when the kernel returns to userspace through the
// trapframe it rewrote. So there is nothing more to do but go back.
if (proc_reincarnate(status) == 0)
return;
// Only reached if the re-exec itself failed, which is the one case where
// there is genuinely nothing left to try.
panic("init resurrection failed: could not exec /bin/schinit");
}
// A thread that is not the leader ends the whole program -- that is what
// exit_group, a fatal signal and a fault mean -- unless it is already dying
// with it, and then leaves as a thread.
if (myproc()->tgid != myproc()->pid) {
if (!myproc()->thread_dying)
group_exit(myproc(), status);
thread_exit();
}
// The leader: any other thread goes too, and if one of them asked for this
// with exit_group, its status is the program's.
acquire(&ptable.lock);
if (myproc()->group_status_set)
status = myproc()->group_status;
group_kill(myproc());
release(&ptable.lock);
vfork_release(myproc());
// Give back any device this process had claimed. A driver that crashed must
// not keep its hardware locked away from the process that replaces it --
// that is the entire point of running drivers outside the kernel.
pciuser_release_all(myproc());
// And any interface it was acting as. Same reason: a driver that crashed
// must not leave a link that claims to be up. See kernel/dev/netuser.c.
netuser_release_all(myproc());
myproc()->xstate = status;
// Shared with other threads, the table lives on; the last one closes it.
(void)fd;
files_put(myproc()->files);
myproc()->files = 0;
// Record locks belong to the process, not to any one descriptor.
lock_exit(myproc()->pid);
begin_op();
iput(myproc()->cwd);
if (myproc()->root)
iput(myproc()->root);
end_op();
myproc()->cwd = 0;
myproc()->root = 0;
acquire(&ptable.lock);
// Tell the parent a child exited — needed for mksh's sigsuspend()-based
// job-control wait loop to have any chance of making progress (without
// this, sigsuspend() can never return for the right reason and the
// parent hangs unconditionally). Signal delivery itself is verified
// working (handler invocation + sigreturn round-trip is clean); mksh's
// own job-control bookkeeping had further gaps (see docs/history/kernel.md
// §7) that this alone did not resolve, but
// disabling it guarantees no progress, whereas leaving it enabled at
// least allows the cases that don't hit those gaps to work.
if (myproc()->parent)
myproc()->parent->pending_signals |= (1ULL << SIGCHLD);
wakeup1(myproc()->parent);
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
if(p->parent == myproc()){
p->parent = initproc;
if(p->state == ZOMBIE)
wakeup1(initproc);
}
}
myproc()->state = ZOMBIE;
sched();
panic("zombie exit");
}
int
wait(void)
{
return waitpid(-1, 0, 0, 0, 0);
}
// waitpid(2)/wait4(2) proper: a specific pid (-1 = any child, matching
// wait()'s old behavior), an optional status-out pointer (NULL is valid —
// POSIX callers commonly do waitpid(pid, NULL, WNOHANG)), and WNOHANG.
//
// This used to be missing: sys_waitpid()/sys_wait4() both ignored pid and
// options and called the any-child, always-blocking wait() above. A NULL
// status pointer additionally got rejected by argptr() (page 0 is
// unmapped) before ever reaching here. Together, a caller doing
// waitpid(child, NULL, WNOHANG) right after a child exits — exactly what
// mksh's job control does — always failed instead of immediately reaping
// the already-zombie child. mksh then fell back to sigsuspend(), which
// this kernel can't wake for a real SIGCHLD (no signal delivery), so it
// silently retried forever: a real, low-CPU hang with no error printed.
// Whether any thread of p's group but p still has a slot. With ptable.lock.
static int
group_has_others(struct proc *p)
{
for (struct proc *q = ptable.proc; q < &ptable.proc[NPROC]; q++)
if (q != p && q->tgid == p->pid && q->state != UNUSED)
return 1;
return 0;
}
int
waitpid(int pid, int *status, int options, uint64 *ut, uint64 *st)
{
struct proc *p;
int havekids;
// Children belong to the process, not to the thread that made them: any
// thread may wait for them, and their exit wakes the leader's channel.
struct proc *me = proc_leader(myproc());
acquire(&ptable.lock);
for(;;){
havekids = 0;
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
if(p->parent != me)
continue;
// POSIX pid selection: >0 one child, 0 = caller's group,
// -1 = any child, < -1 = the process group with id -pid.
if(pid > 0 && p->pid != pid)
continue;
if(pid == 0 && p->pgid != me->pgid)
continue;
if(pid < -1 && p->pgid != -pid)
continue;
if(p->tgid != p->pid)
continue; // a thread, which nobody waits for
havekids = 1;
if(p->state == ZOMBIE && group_has_others(p))
continue; // reaped once its last thread is
if(p->state == ZOMBIE){
int rpid = p->pid;
int rstatus = p->xstate; // copy out before the slot is recycled
// RUSAGE_CHILDREN: the child's own time *and* whatever it had already
// reaped, which is what makes the total transitive down a whole tree
// of processes rather than one generation deep.
me->cutime += p->utime + p->cutime;
me->cstime += p->stime + p->cstime;
reaped_utime += p->utime;
reaped_stime += p->stime;
// wait4()'s fourth argument wants the *child's* usage, not the
// parent's running total, so it has to be read here -- three lines
// further down the slot is UNUSED and these are zero.
if (ut) *ut = p->utime + p->cutime;
if (st) *st = p->stime + p->cstime;
p->kstack = 0; // the slot keeps its stack; see kstacks[]
// The address space goes with its last user. See mm_put().
mm_put(p->mm);
p->mm = 0;
p->pgdir = 0;
sighand_put(p->sighand);
p->sighand = 0;
p->pid = 0;
p->parent = 0;
p->name[0] = 0;
p->killed = 0;
p->xstate = 0;
p->killsig = 0;
p->pgid = 0;
p->sid = 0;
p->tgid = 0;
p->group_status_set = 0;
p->state = UNUSED;
release(&ptable.lock);
if (status) *status = rstatus;
return rpid;
}
// WUNTRACED and WCONTINUED: a stop or a resume, each reported once.
if((options & 2) && p->stop_report){
int rpid = p->pid, rstatus = (p->stop_report << 8) | 0x7f;
p->stop_report = 0;
if (ut) *ut = p->utime;
if (st) *st = p->stime;
release(&ptable.lock);
if (status) *status = rstatus;
return rpid;
}
if((options & 8) && p->cont_report){
int rpid = p->pid;
p->cont_report = 0;
if (ut) *ut = p->utime;
if (st) *st = p->stime;
release(&ptable.lock);
if (status) *status = 0xffff;
return rpid;
}
}
// Two different failures, and they used to share one bare -1.
//
// musl reads a return in [-4095, -1] as -errno, so -1 arrived in
// userspace as EPERM: "operation not permitted". zsh's compinit calls
// wait, got that, printed "wait failed: operation not permitted", and
// abandoned the rest of /etc/zsh/zshrc -- which looked like a broken
// config file rather than a wait() that could not say what went wrong.
if(!havekids){
release(&ptable.lock);
return -ECHILD; // nothing to wait for, which is not an error
}
if(myproc()->killed){
release(&ptable.lock);
return -EINTR; // a signal arrived while waiting
}
if (options & 1) { // WNOHANG
release(&ptable.lock);
return 0;
}
sleep(me, &ptable.lock);
}
}
// Stop a process running, by name, and never start it again.
//
// This exists for one reason: docs/architecture/init.md says the heartbeat's
// timeout path should be "exercised rather than assumed", and a hang is the
// one failure that cannot be produced by killing something. A wedged process
// is still alive, still has its descriptors open, and still shows up in every
// listing -- which is exactly what makes it hard to notice and exactly what
// the heartbeat is for.
//
// Deliberately one-way. There is no unwedge: this is a test instrument for a
// machine that is about to be rebooted, not a job-control mechanism. Real
// SIGSTOP/SIGCONT is a separate thing and is not implemented.
int
proc_wedge(const char *name)
{
struct proc *p;
int found = 0;
acquire(&ptable.lock);
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
if (p->state == UNUSED || p->pid <= 1)
continue;
if (strncmp(p->name, name, sizeof(p->name)) != 0)
continue;
p->wedged = 1;
found = p->pid;
break;
}
release(&ptable.lock);
return found;
}
// --- The FPU ------------------------------------------------------------
//
// Every process has its own x87 and SSE registers, and they were never saved.
// musl's memcpy, the compiler's struct copies and anything doing arithmetic on
// doubles keep live values in %xmm across a context switch, so two processes
// sharing a CPU wrote over each other's: two shells counting in a loop on one
// processor took a NULL+8 fault within seconds, at a different rip each time.
// It had always been there, and every crash with no cause anyone could find
// was a candidate.
//
// The kernel is built with -mgeneral-regs-only, so it never touches these
// registers itself and they stay the user's from trap entry to return. That
// makes saving them a switch-time job and nothing more: sched() saves the
// process leaving the CPU, scheduler() restores the one arriving. No lazy
// switching (CR0.TS and a #NM handler), which would save the copy for a
// process that never uses SSE -- there is no such process; musl's string
// functions use it.
__attribute__((aligned(16)))
const uint8 fpu_default[512] = {
[0] = 0x7F, [1] = 0x03, // FCW 0x037F
[24] = 0x80, [25] = 0x1F, // MXCSR 0x1F80
};
// --- The scheduler ------------------------------------------------------
//
// It was round-robin: walk ptable from end to end, run everything RUNNABLE in
// slot order, and let the timer tick take the CPU back after 10 ms. That is
// xv6's scheduler, and three things are wrong with it on a machine that now
// runs a shell, a network stack and a display server at the same time.
//
// **Slot order is not fairness.** A process that blocks on a read after one
// microsecond and a process that burns its whole slice are treated as equals,
// because nothing counts what anyone actually used. The compile and the shell
// you are typing into get alternate turns; the shell feels it.
//
// **Every tick was a context switch.** `yield()` unconditionally on the timer
// means a process alone on a CPU still pays a full switch -- `switchuvm`, a
// `%cr3` reload, a flushed TLB -- a hundred times a second, to be handed
// straight back to itself.
//
// **There is no way to say one thing matters more than another.** No nice, no
// priority, nothing for a driver thread that must answer a device promptly.
//
// ## What replaces it
//
// Fair share by *virtual runtime*, which is the idea behind both Linux's CFS
// and Zircon's fair scheduler, and is much simpler than either implementation
// of it. Each process carries `vruntime`: the CPU time it has consumed,
// divided by its weight. The scheduler always runs the RUNNABLE process with
// the smallest vruntime.
//
// That single rule produces everything the old one lacked:
//
// - **Fairness falls out.** Running advances your clock; the process that
// has had least CPU is by definition the one whose clock is furthest
// behind, so it runs next. Equal weights converge on equal shares without
// anything having to enforce it.
//
// - **Interactivity falls out too, with no heuristic.** A process that
// spends its life blocked on a keystroke barely advances its vruntime, so
// when the keystroke arrives it is already at the front. No "is this
// interactive?" guess, which is the part of older schedulers that was
// always either gameable or wrong.
//
// - **Priority is one multiplication.** Weight scales how fast your clock
// runs. A high-weight process's vruntime advances slowly, so it is chosen
// more often, in exact proportion.
//
// ## Deliberately not a red-black tree
//
// CFS keeps the runnable set in a red-black tree and Zircon in a WAVL tree,
// so "smallest vruntime" is O(log n) and the leftmost node is cached. Here it
// is a linear scan of ptable, O(NPROC), the same scan the old scheduler did.
//
// That is not laziness. NPROC is 128 slots of a few words each; the scan is a
// handful of cache lines and runs with the lock already held, whereas a
// balanced tree is three hundred lines of rotations that must stay correct
// through every exit path in this file. The scan is the right trade until
// something measures it as the problem, and the rule this kernel keeps is
// that the measurement comes first. Swapping in a tree later changes
// pick_next() and nothing else, which is the point of it being one function.
//
// ## Still one lock, still one queue
//
// Every CPU picks out of the same table under `ptable.lock`. Per-CPU run
// queues with work stealing are the other half of a real SMP scheduler and
// they are a separate change -- they need a load balancer, and a load
// balancer that is wrong is worse than a global lock that is merely
// contended. With the number of cores this actually runs on, the lock is not
// yet what hurts.
// Weight per nice level, indexed by nice + 20.
//
// The ratio between adjacent levels is 1.25, so one step of nice is about a
// 10% change in CPU share when two processes compete -- the convention UNIX
// has used for long enough that programs and people both expect it. The table
// is 1024 / 1.25^nice, rounded, with nice 0 at 1024 so the common case
// multiplies by one.
static const uint32 nice_weight[40] = {
88818, 71054, 56843, 45475, 36380, 29104, 23283, 18626,
14901, 11921, 9537, 7629, 6104, 4883, 3906, 3125,
2500, 2000, 1600, 1280, 1024, 819, 655, 524,
419, 336, 268, 215, 172, 137, 110, 88,
70, 56, 45, 36, 29, 23, 18, 15,
};
#define WEIGHT_NICE0 1024u
// The period in which every runnable process should get a turn, and the floor
// on how short a turn may be.
//
// A slice is SCHED_LATENCY_NS divided between the runnable processes, so with
// few of them each gets a long one and switches are rare; as the count grows
// the slice shrinks until it hits SCHED_MIN_GRAN_NS, below which the switch
// costs more than the work done in it and the period is allowed to stretch
// instead. 20 ms and 2 ms on a kernel whose timer runs at 100 Hz: the minimum
// is deliberately shorter than a tick, because the tick is when the decision
// is *checked*, not the granularity at which time is *measured*.
#define SCHED_LATENCY_NS 20000000ull
#define SCHED_MIN_GRAN_NS 2000000ull
// The floor of every runnable vruntime, which is what keeps a process that
// slept from being owed the world.
//
// Without it a process blocked for ten seconds comes back ten seconds of
// vruntime behind everything else and monopolises the CPU until it catches
// up -- the classic way a naive fair scheduler turns one sleeper into a
// system-wide stall. A waking process is pulled forward to no worse than
// min_vruntime - SCHED_LATENCY_NS: far enough ahead in the queue to be
// scheduled promptly, not so far as to be owed a debt.
//
// It only ever moves forward, and it is guarded by ptable.lock like
// everything else here.
static uint64 min_vruntime;
// TSC cycles to nanoseconds.
//
// tsc_hz is measured against the PIT at boot (kernel/arch/x86_64/lapic.c), so
// this is real time rather than a cycle count that means something different
// on every machine. The multiply is done in 64 bits after dividing the clock
// down to MHz: cycles * 1000 / mhz overflows only after some centuries of
// uptime, where cycles * 1000000000 would overflow in seconds.
static uint64
tsc_to_ns(uint64 cycles)
{
uint64 mhz = tsc_hz / 1000000;
if (mhz == 0)
return 0; // before lapicinit; nothing to charge yet
return (cycles * 1000) / mhz;
}
// Give a process its scheduling state. Called from allocproc, before it is
// ever RUNNABLE.
void
sched_init_proc(struct proc *p, int nice)
{
if (nice < -20) nice = -20;
if (nice > 19) nice = 19;
p->nice = nice;
p->weight = nice_weight[nice + 20];
p->vruntime = min_vruntime; // start level with everyone, owed nothing
p->ran_at = 0;
p->slice_ns = SCHED_MIN_GRAN_NS;
p->slice_used = 0;
p->nvcsw = p->nivcsw = 0;
}
// setpriority(2)/getpriority(2), the only way nice is ever set.
//
// PRIO_PROCESS only. PRIO_PGRP and PRIO_USER are real and are not hard, but
// answering them requires deciding what "the priority of a user" returns when
// their processes disagree, and a wrong answer is worse than -EINVAL.
//
// Lowering nice -- asking for *more* CPU -- is root's alone, which is the one
// permission check in this file that is actually enforced. Without it nice is
// not a courtesy, it is a way for any process to outrank the rest of the
// system by asking.
int
proc_setnice(int pid, int nice)
{
struct proc *p;
int r = -ESRCH;
if (nice < -20) nice = -20;
if (nice > 19) nice = 19;
acquire(&ptable.lock);
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
if (p->state == UNUSED || (pid != 0 && p->pid != pid))
continue;
if (pid == 0 && p != myproc())
continue;
// Two separate rules, and POSIX gives them two separate errnos.
//
// EPERM: you may only renice something you own. Without it any user can
// push every root process to nice 19 and have the run of the machine.
// EACCES: *lowering* nice -- asking for more CPU -- is root's alone.
//
// Both are inert today, because everything runs as root and euid is 0
// everywhere. They are written now because the alternative is a scheduler
// that grows a priority knob and a permission model separately, and the
// window between the two is exactly the bug.
if (myproc()->euid != 0 && myproc()->euid != p->uid) {
r = -EPERM;
break;
}
if (nice < p->nice && myproc()->euid != 0) {
r = -EACCES;
break;
}
sched_set_nice(p, nice);
r = 0;
break;
}
release(&ptable.lock);
return r;
}
int
proc_getnice(int pid, int *out)
{
struct proc *p;
int r = -ESRCH;
acquire(&ptable.lock);
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
if (p->state == UNUSED || (pid != 0 && p->pid != pid))
continue;
if (pid == 0 && p != myproc())
continue;
*out = p->nice;
r = 0;
break;
}
release(&ptable.lock);
return r;
}
// Change a process's priority. ptable.lock must be held.
//
// vruntime is left alone rather than rescaled: it is a record of time already
// spent, and rewriting history so that renicing hands out a retroactive
// refund (or bill) is how a nice() call becomes a way to jump the queue.
void
sched_set_nice(struct proc *p, int nice)
{
if (nice < -20) nice = -20;
if (nice > 19) nice = 19;
p->nice = nice;
p->weight = nice_weight[nice + 20];
}
// Charge a process for the time it has just run, and advance its vruntime.
// ptable.lock must be held, and p must be the process leaving the CPU.
static void
sched_charge(struct proc *p)
{
uint64 now = rdtsc(), ns;
if (p->ran_at == 0 || now <= p->ran_at) {
p->ran_at = 0;
return;
}
ns = tsc_to_ns(now - p->ran_at);
p->ran_at = 0;
p->slice_used += ns;
// The weighting, and the only line in the scheduler where nice does
// anything: a process of twice the weight is charged half the vruntime for
// the same real time, so its clock falls behind and pick_next() returns to
// it sooner.
//
// The guard is for the one way weight can be zero: a ptable slot is BSS,
// and BSS is zeros until allocproc() runs sched_init_proc() on it. Nothing
// reaches here on such a slot today -- only a process that has run gets
// charged, and a process that has run went through allocproc -- but the
// failure if that ever stops being true is a divide-by-zero #GP inside the
// scheduler, which is a fault with no good place to report itself.
if (p->weight == 0)
p->weight = WEIGHT_NICE0;
p->vruntime += (ns * WEIGHT_NICE0) / p->weight;
}
// How long a slice should be, given how many processes want one.
static uint64
sched_slice(int nrunnable)
{
uint64 slice;
if (nrunnable <= 1)
return SCHED_LATENCY_NS;
slice = SCHED_LATENCY_NS / (uint64)nrunnable;
return slice < SCHED_MIN_GRAN_NS ? SCHED_MIN_GRAN_NS : slice;
}
// Place a process that has just become runnable, and keep min_vruntime
// honest. ptable.lock must be held.
//
// Called from wakeup1() and from anywhere else that moves a process into
// RUNNABLE from a state where it was not accruing vruntime.
static void
sched_enqueue(struct proc *p)
{
uint64 floor = min_vruntime > SCHED_LATENCY_NS
? min_vruntime - SCHED_LATENCY_NS : 0;
if (p->vruntime < floor)
p->vruntime = floor;
p->slice_used = 0;
}
// The runnable process with the smallest vruntime, and a count of how many
// were in the running. ptable.lock must be held.
static struct proc *
pick_next(int *nrunnable)
{
struct proc *p, *best = 0;
int n = 0;
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
// A wedged process is passed over for ever. It is still RUNNABLE, still
// owns its descriptors and still appears in every listing -- which is
// the point: it looks exactly like a process that is merely busy.
if (p->state != RUNNABLE || p->wedged)
continue;
n++;
if (best == 0 || p->vruntime < best->vruntime)
best = p;
}
*nrunnable = n;
return best;
}
// Whether the running process has used up its slice, so trap() knows whether
// the timer tick should become a yield(). No lock: it reads only the current
// process's own fields, and the only writer of them is the current process.
//
// This is the line that removed a hundred context switches a second from an
// idle machine. The old code yielded on *every* tick unconditionally.
int
sched_slice_expired(void)
{
struct proc *p = myproc();
uint64 now, ns;
if (p == 0 || p->ran_at == 0)
return 0;
now = rdtsc();
if (now <= p->ran_at)
return 0;
ns = tsc_to_ns(now - p->ran_at);
return (p->slice_used + ns) >= p->slice_ns;
}
// Wait for an interrupt as deeply as the firmware allows. See acpi_idle in
// kernel/acpi/aml.c. Interrupts are off on the way down so that one arriving
// between the decision and the sleep is not lost: a pending interrupt still
// ends the sleep (MWAIT's ECX bit 0 says so explicitly; an I/O-entered C2
// wakes on it by design), and is taken at the sti after.
static void
cpu_idle(void)
{
static volatile int monitor_line[16];
switch (acpi_idle.kind) {
case ACPI_IDLE_IO:
cli();
(void)inb((ushort)acpi_idle.addr);
sti();
break;
case ACPI_IDLE_MWAIT:
cli();
asm volatile("monitor" : : "a"(monitor_line), "c"(0), "d"(0));
asm volatile("mwait" : : "a"((uint32)acpi_idle.addr), "c"(1));
sti();
break;
default:
hlt();
}
}
void
scheduler(void)
{
struct proc *p;
int nrunnable;
for(;;){
sti();
acquire(&ptable.lock);
p = pick_next(&nrunnable);
if (p == 0) {
// Nothing to run. Drop the lock before halting -- holding it through a
// hlt() would stop every other CPU from scheduling until an interrupt
// happened to arrive here.
release(&ptable.lock);
cpu_idle();
continue;
}
// min_vruntime tracks the front of the queue and never moves backwards,
// so a process that sleeps through a busy period cannot return with a
// claim on all the CPU it missed. See the comment on the variable.
if (p->vruntime > min_vruntime)
min_vruntime = p->vruntime;
p->slice_ns = sched_slice(nrunnable);
p->slice_used = 0;
p->ran_at = rdtsc();
proc_set(p);
switchuvm(p);
fxrstor(p->fpu);
p->state = RUNNING;
swtch(&mycpu()->scheduler, p->context);
switchkvm();
proc_set(0);
// Back from swtch: p has given the CPU up, through sched(). It has
// already been charged there, where the reason for the switch was known.
release(&ptable.lock);
}
}
void
sched(void)
{
int intena;
if(!holding(&ptable.lock))
panic("sched ptable.lock");
if(mycpu()->ncli != 1)
panic("sched locks");
if(myproc()->state == RUNNING)
panic("sched running");
if(readeflags()&FL_IF)
panic("sched interruptible");
intena = mycpu()->intena;
// Charge for the time just run before handing the CPU back. Here rather
// than in scheduler(), because this is the one point every path off a CPU
// goes through -- yield(), sleep() and exit() alike -- and doing it on the
// way out means the vruntime the scheduler then compares is already
// up to date rather than a tick stale.
sched_charge(myproc());
fxsave(myproc()->fpu);
swtch(&myproc()->context, mycpu()->scheduler);
mycpu()->intena = intena;
}
void
yield(void)
{
acquire(&ptable.lock);
myproc()->state = RUNNABLE;
myproc()->nivcsw++; // involuntary: the slice ran out
sched();
release(&ptable.lock);
}
// Which device the root filesystem is on.
//
// It was `#define ROOTDEV 1` -- IDE disk 1 -- so a kernel could only ever root
// on the disk it was *built* expecting. Booting the same image from a USB
// stick was impossible by construction, and that is the one thing standing
// between this kernel and its own acceptance test, which begins "boot on real
// hardware from a USB stick".
//
// A probe rather than a boot argument, because the answer is discoverable and
// asking someone to name a device number is a worse interface than looking.
// schfs_kmount() is already written to be used this way -- its own comment
// says "the uberblock scan doubles as the probe: no valid uberblock means this
// is simply not a SchFS volume, which is not an error worth reporting loudly"
// -- so each candidate costs one scan and answers yes or no. A failed probe
// leaves the slot unmounted and reusable, which is what makes a loop over
// candidates legal at all.
//
// **The order is preference, not confidence.** A ramdisk root is an explicit
// build choice and wins outright. After it, USB comes before the fixed disks:
// a stick that is plugged in was plugged in on purpose, and a machine with
// both should boot the thing the person is holding.
uint rootdev;
static int
rootdev_usable(uint dev)
{
if (dev == RAMDISK_DEV)
return ramdisk_present();
if (dev == USBDISK_DEV)
return usb_storage_present();
if (dev >= BLKDEV_FIRST)
return blkdev_present(dev);
// iderw() panics on an absent drive rather than failing, so a probe has to
// ask first. That is the whole reason ide_disk_present() exists.
return ide_disk_present((int)dev);
}
static void
rootdev_mount(void)
{
// The fixed devices, with the registered ones -- SATA, NVMe, and every
// partition on them -- between USB and IDE. A partition GPT types as SchFS
// goes ahead of the rest of the registry, because it was labelled that way
// on purpose; the others are probed too, so a volume in an untyped or MBR
// partition, or on a whole unpartitioned disk, still boots.
struct { uint dev; const char *what; } cand[4 + 2 * NBLKDEV];
int ncand = 0;
#ifdef RAMDISK_ROOT
cand[ncand].dev = RAMDISK_DEV; cand[ncand++].what = "ramdisk";
#endif
cand[ncand].dev = USBDISK_DEV; cand[ncand++].what = "USB disk";
for (int pass = 0; pass < 2; pass++)
for (int i = 0, d; (d = blkdev_nth(i)) >= 0; i++)
if (blkdev_is_schfs(d) == !pass) {
cand[ncand].dev = d;
cand[ncand++].what = blkdev_name(d);
}
cand[ncand].dev = 1; cand[ncand++].what = "IDE disk 1";
cand[ncand].dev = 0; cand[ncand++].what = "IDE disk 0";
for (int i = 0; i < ncand; i++) {
if (!rootdev_usable(cand[i].dev))
continue;
schfs_kmount(cand[i].dev);
if (schfs_kmounted()) {
rootdev = cand[i].dev;
kinfo("root: %s (device %d)\n", cand[i].what, cand[i].dev);
return;
}
kinfo("root: %s (device %d) carries no SchFS volume\n",
cand[i].what, cand[i].dev);
}
// Name what was looked at. "no root filesystem" with no list is the least
// useful thing a kernel can say at the one moment somebody needs to know
// which disk it expected.
// Plain %s and no width. cprintf() has no field-width specifier -- it
// prints "%-11s" literally and then reads the *next* argument for the %d,
// which is how the first version of this message came out as
// `%-11s device 1596400`. See print_int() in kernel/core/console.c.
cprintf("root: no SchFS volume on any candidate device. Looked at:\n");
for (int i = 0; i < ncand; i++)
cprintf(" device %d, %s: %s\n", cand[i].dev, cand[i].what,
rootdev_usable(cand[i].dev) ? "present, but no SchFS on it"
: "not present");
panic("no root filesystem");
}
void
forkret(void) {
static int first = 1;
release(&ptable.lock);
// A kernel thread starts here and never leaves. It shares the scheduler's
// rotation with everything else, so it can hold sleeplocks and block on the
// disk exactly as a syscall would -- which is the whole reason it is a
// process rather than a timer callback.
//
// Ahead of the first-boot block below on purpose: a kernel thread is only
// ever created long after the root filesystem is mounted, so it must not be
// the one that runs that setup.
if (myproc()->kfn) {
myproc()->kfn();
panic("kthread returned");
}
if (first) {
first = 0;
iinit();
mountinit();
// SchFS is the root filesystem, on whichever device turns out to have
// one. Here rather than in main() because mounting reads blocks, bread()
// sleeps, and sleeping needs a process context.
rootdev_mount();
// Re-point pid 1's cwd at the real root, now that there is one.
//
// sinit() sets cwd = namei("/") back in main(), which is before this runs.
// The old v1 root survived that because iget(ROOTDEV, ROOTINO) is two
// compile-time constants and touches no disk -- the inode is filled in
// lazily at the first ilock(). SchFS has no constant to offer: the root
// object id comes off the uberblock, so namei("/") that early returns an
// inode built from whatever schfs_kroot() reads out of an unmounted
// volume.
//
// Nothing complains, because absolute paths never consult cwd -- exec
// "/bin/init" works fine. It is the first *relative* lookup that fails,
// and in init that is open("console"): no console means no fd 0/1/2, so
// every shell it spawns exits immediately and init respawns it forever.
// The symptom is a silent boot with no init: ready, which reads like a
// hang and is not one.
{
struct inode *stale = myproc()->cwd;
myproc()->cwd = iget(SCHFS_DEV, schfs_kroot(SCHFS_DEV));
if (stale)
iput(stale);
}
}
}
// Start a kernel thread: a process that runs `fn` in kernel mode and never
// returns to userland.
//
// It gets a page directory of its own with only the kernel mapped, because the
// scheduler calls switchuvm() for every process it runs and that panics on a
// null pgdir. Sharing kpgdir directly would work equally well; a private one
// costs a page and keeps the "every proc owns its pgdir" invariant intact.
//
// No files, no cwd, no parent: it is not in the process tree, cannot be waited
// for and must never exit. Returns the pid, or -1.
int
kthread_create(void (*fn)(void), const char *name)
{
struct proc *p;
if ((p = allocproc()) == 0)
return -1;
if ((p->pgdir = setupkvm()) == 0) {
fork_undo(p);
return -1;
}
p->sz = PGSIZE; // never used; 0 upsets code that treats it as a size
p->mm->sz = p->sz;
p->mm->pgdir = p->pgdir;
p->parent = 0;
p->cwd = 0;
p->kfn = fn;
safestrcpy(p->name, name, sizeof(p->name));
acquire(&ptable.lock);
p->state = RUNNABLE;
release(&ptable.lock);
return p->pid;
}
void
sleep(void *chan, struct spinlock *lk)
{
if(myproc() == 0)
panic("sleep");
if(lk == 0)
panic("sleep without lk");
if(lk != &ptable.lock){
acquire(&ptable.lock);
release(lk);
}
myproc()->chan = chan;
myproc()->state = SLEEPING;
myproc()->nvcsw++; // voluntary: it asked to wait
sched();
myproc()->chan = 0;
if(lk != &ptable.lock){
release(&ptable.lock);
acquire(lk);
}
}
static void
wakeup1(void *chan)
{
struct proc *p;
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++)
if(p->state == SLEEPING && p->chan == chan){
p->state = RUNNABLE;
// Pull it forward to the front of the timeline if it has been asleep
// long enough to have fallen off the back of it -- see sched_enqueue.
sched_enqueue(p);
}
}
void
wakeup(void *chan)
{
acquire(&ptable.lock);
wakeup1(chan);
release(&ptable.lock);
}
// Raise a signal on a process: set the pending bit and wake it if it's
// sleeping so blocking syscalls (sigsuspend, pause, read) re-check their
// conditions promptly. Signals with no handler installed fall back to the
// default action — termination for the fatal set, dropped otherwise.
// mksh depends on kill(getpid(), SIGCHLD) genuinely re-raising SIGCHLD
// (its held_sigchld recovery in j_startjob); the previous sys_kill
// silently discarded every non-fatal signal.
// PID 1 does not die from a signal it never asked to handle.
//
// POSIX, and Linux in sig_task_ignored(): a signal whose disposition is the
// default action is discarded for init rather than delivered. Without this the
// system has a single keystroke that kills it -- Ctrl-C sends SIGINT to the
// foreground process group, init is pgid 1, and any moment where a terminal's fg_pgrp
// is 1 (before a shell claims the terminal, after one exits, or in the window
// between the two while the key is held down) lands SIGINT on init. init has
// no handler, so killed is set, exit() runs, and exit() panics outright on
// initproc: "init exiting". Spamming Ctrl-C at a frozen program is exactly the
// input that finds that window.
//
// Applies to SIGKILL too, which is the point: init is not killable from
// userspace on any Unix. A handler it did install is still delivered.
static int
init_ignores(struct proc *p, int sig)
{
return p == initproc && p->sighand->handler[sig] == 0;
}
// Whether a signal's default action is to end the process.
int
sig_default_kills(int sig)
{
return sig == SIGKILL || sig == SIGTERM || sig == SIGABRT ||
sig == SIGSEGV || sig == SIGFPE || sig == SIGBUS ||
sig == SIGILL || sig == SIGINT || sig == SIGQUIT ||
sig == SIGHUP || sig == SIGPIPE || sig == SIGUSR1 ||
sig == SIGUSR2 || sig == SIGALRM;
}
// Whether sending `sig` ends `p` now: no handler, a default action that
// kills, and **not blocked**. A blocked signal stays pending, which is the
// whole of what blocking means -- and the contract signalfd is built on: a
// program blocks SIGUSR1 and reads it from a descriptor instead. This used to
// kill on arrival whatever the mask said. deliver_signals() applies the
// default action if the signal is unblocked later, still pending.
// SIGKILL cannot be blocked.
static int
sig_kills_now(struct proc *p, int sig)
{
if(sig == SIGKILL)
return 1;
if(p->signal_mask & (1ULL << sig))
return 0;
return p->sighand->handler[sig] == 0 && sig_default_kills(sig);
}
// SIGALRM, on time.
//
// alarm() and setitimer() stored a countdown that nothing ever counted down,
// so no timer anyone set has ever fired. Most programs never noticed, because
// what they used it for was a timeout; busybox's ping sends every request but
// the first from its SIGALRM handler, and on the Alpine root it is /bin/ping,
// so ping printed one reply and then nothing.
//
// A kernel thread rather than the timer interrupt, which is in trap.c, and
// which would have to walk the process table with interrupts off: this walks
// it once a tick, and only while some process has a timer armed.
static int alarms_armed; // processes with alarm_at set; a hint, not a lock
void
alarm_arm(struct proc *p, uint after, uint every)
{
acquire(&ptable.lock);
if (p->alarm_at == 0 && after)
alarms_armed++;
else if (p->alarm_at && after == 0)
alarms_armed--;
p->alarm_at = after ? ticks + after : 0;
if (p->alarm_at == 0 && after)
p->alarm_at = 1; // 0 means disarmed; a tick late is fine
p->alarm_every = after ? every : 0;
release(&ptable.lock);
}
// Ticks until p's timer is due, 0 if it is not armed.
uint
alarm_left(struct proc *p)
{
uint at = p->alarm_at;
if (at == 0)
return 0;
int left = (int)(at - ticks);
return left > 0 ? (uint)left : 1;
}
static void
alarm_thread(void)
{
for (;;) {
acquire(&tickslock);
sleep(&ticks, &tickslock);
uint now = ticks;
release(&tickslock);
vm_defer_tick(); // pages waiting out other CPUs' TLBs
if (alarms_armed == 0)
continue;
acquire(&ptable.lock);
for (struct proc *p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
if (p->alarm_at == 0 || (int)(now - p->alarm_at) < 0)
continue;
if (p->state == UNUSED || p->state == ZOMBIE) {
p->alarm_at = 0;
alarms_armed--;
continue;
}
if (p->alarm_every) {
p->alarm_at = now + p->alarm_every;
} else {
p->alarm_at = 0;
alarms_armed--;
}
// sigsend takes ptable.lock itself, and with this many lines it is
// clearer to repeat its three effects than to split it.
if (!init_ignores(p, SIGALRM)) {
p->pending_signals |= 1ULL << SIGALRM;
if (sig_kills_now(p, SIGALRM)) {
p->killed = 1;
if (p->killsig == 0)
p->killsig = SIGALRM;
}
if (p->state == SLEEPING) {
p->state = RUNNABLE;
sched_enqueue(p);
}
}
}
release(&ptable.lock);
}
}
void
alarm_thread_start(void)
{
if (kthread_create(alarm_thread, "alarmd") < 0)
cprintf("alarm: no kernel thread; alarm() and setitimer() will not fire\n");
}
// --- Stopping and continuing --------------------------------------------
//
// SIGSTOP, and SIGTSTP/SIGTTIN/SIGTTOU with no handler, stop a process; SIGCONT
// resumes it. Neither existed: Ctrl-Z did nothing, and mc -- which stops its
// subshell and waits to be told it has stopped -- hung before drawing.
//
// The stop itself happens on the way back to user mode, in deliver_signals(),
// where the process can sleep. SIGCONT acts when it is sent, because a
// stopped process will not reach that point by itself.
int
sig_default_stops(int sig)
{
return sig == SIGSTOP || sig == SIGTSTP || sig == SIGTTIN || sig == SIGTTOU;
}
#define STOP_SIGS ((1ULL << SIGSTOP) | (1ULL << SIGTSTP) | \
(1ULL << SIGTTIN) | (1ULL << SIGTTOU))
// Tell the parent about a stop or a resume: SIGCHLD, and a wake for a wait()
// or a sigsuspend() that is waiting to hear it. Called with ptable.lock.
static void
notify_parent(struct proc *p)
{
struct proc *pp = p->parent;
if(pp == 0)
return;
pp->pending_signals |= 1ULL << SIGCHLD;
if(pp->state == SLEEPING){
pp->state = RUNNABLE;
sched_enqueue(pp);
}
}
// The side effects of sending sig to p, beyond setting it pending. With
// ptable.lock held.
static void
sig_job_control(struct proc *p, int sig)
{
if(sig == SIGCONT){
// A resume cancels any stop still on its way, and the other way round.
p->pending_signals &= ~STOP_SIGS;
if(p->stopsig){
p->stopsig = 0;
p->stop_report = 0;
p->cont_report = 1;
notify_parent(p);
}
} else if(sig_default_stops(sig)){
p->pending_signals &= ~(1ULL << SIGCONT);
}
}
// Stop the calling process with sig until SIGCONT or SIGKILL.
void
proc_stop(int sig)
{
struct proc *p = myproc();
acquire(&ptable.lock);
p->stopsig = sig;
p->stop_report = sig;
p->cont_report = 0;
notify_parent(p);
// Other signals wake it too; only a resume or a kill ends the stop.
while(p->stopsig && !p->killed)
sleep(&p->stopsig, &ptable.lock);
p->stopsig = 0;
release(&ptable.lock);
}
// A handler that ends a blocking call. Installed with SA_RESTART -- which
// musl's signal() always asks for -- the call goes on waiting instead, and
// the handler runs when it returns: Linux restarts the call around the
// handler, and a program that asked for that does not check for EINTR.
#define SA_RESTART 0x10000000
static int
sig_interrupts(struct proc *p, int sig)
{
return p->sighand->handler[sig] > 1 && !(p->sighand->flags[sig] & SA_RESTART);
}
// Whether a pending signal should end a blocking wait: one that will stop the
// process, or one with a handler to run. A lockless look, for the inside of a
// sleep loop that holds some other lock; sig_blocking_check() does the work.
int
sig_attention(struct proc *p)
{
uint64 d = p->pending_signals & ~p->signal_mask;
if(d == 0)
return 0;
for(int sig = 1; sig < 32; sig++)
if((d & (1ULL << sig)) &&
(sig == SIGSTOP || sig_interrupts(p, sig) ||
(sig_default_stops(sig) && p->sighand->handler[sig] == 0)))
return 1;
return 0;
}
// For a blocking call that saw sig_attention(), with no locks held. A stop is
// taken here, and the call carries on waiting once continued -- which is what
// Linux's restart does, and what lets Ctrl-Z stop a `sleep` or a `cat`. A
// signal with a handler ends the call with -EINTR, as POSIX has it.
int
sig_blocking_check(void)
{
struct proc *p = myproc();
for(;;){
int stop = 0, caught = 0;
acquire(&ptable.lock);
uint64 d = p->pending_signals & ~p->signal_mask;
for(int sig = 1; sig < 32 && !stop; sig++){
if(!(d & (1ULL << sig)))
continue;
if(sig == SIGSTOP || (sig_default_stops(sig) && p->sighand->handler[sig] == 0)){
p->pending_signals &= ~(1ULL << sig);
stop = sig;
} else if(sig_interrupts(p, sig)){
caught = 1;
}
}
release(&ptable.lock);
if(stop){
proc_stop(stop);
if(p->killed)
return -EINTR;
continue;
}
return (caught || p->killed) ? -EINTR : 0;
}
}
// The other threads of p's group, for a stop or a resume. With ptable.lock.
static void
sig_to_threads(struct proc *p, int sig)
{
for (struct proc *q = ptable.proc; q < &ptable.proc[NPROC]; q++) {
if (q == p || q->tgid != p->tgid || q->state == UNUSED || q->state == ZOMBIE)
continue;
q->pending_signals |= 1ULL << sig;
sig_job_control(q, sig);
if (q->state == SLEEPING) {
q->state = RUNNABLE;
sched_enqueue(q);
}
}
}
int
sigsend(int pid, int sig)
{
struct proc *p;
if(sig <= 0 || sig >= 32)
return -1;
acquire(&ptable.lock);
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
if(p->pid == pid && p->state != UNUSED && p->state != ZOMBIE){
if(init_ignores(p, sig)){
release(&ptable.lock);
return 0; // silently dropped, as POSIX requires
}
p->pending_signals |= 1ULL << sig;
sig_job_control(p, sig);
// A stop or a resume is the whole program's, so every thread gets it.
if(sig == SIGCONT || sig_default_stops(sig))
sig_to_threads(p, sig);
if(sig_kills_now(p, sig))
p->killed = 1;
if(sig == SIGKILL)
p->killed = 1;
if(p->killed && p->killsig == 0)
p->killsig = sig; // so exit() can report the real cause
if(p->state == SLEEPING){
p->state = RUNNABLE;
sched_enqueue(p); // a signal is a wakeup too; same floor
}
release(&ptable.lock);
return 0;
}
}
release(&ptable.lock);
return -1;
}
// Deliver a signal to every member of a process group. This is what job
// control actually needs: a shell sends SIGINT/SIGTSTP/SIGHUP to a *group*,
// never to individual pids. sys_kill() with pid <= 0 routes here.
//
// Returns 0 if at least one process was signalled, -1 if the group is empty.
int
sigsend_pgrp(int pgid, int sig)
{
struct proc *p;
int found = 0;
if(sig < 0 || sig >= 32)
return -1;
acquire(&ptable.lock);
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
if(p->state == UNUSED || p->state == ZOMBIE)
continue;
if(p->pgid != pgid)
continue;
found = 1;
if(sig == 0)
continue; // signal 0 is an existence probe only
if(init_ignores(p, sig))
continue; // see init_ignores(): this is the Ctrl-C case
// A group signal is the process's, and goes to the leader -- once, not
// once per thread, or a SIGWINCH handler would run in each of them. A
// stop or a resume is the exception: every thread has to stop.
if(p->tgid != p->pid && sig != SIGCONT && !sig_default_stops(sig))
continue;
p->pending_signals |= 1ULL << sig;
sig_job_control(p, sig);
if(sig_kills_now(p, sig)){
p->killed = 1;
if(p->killsig == 0)
p->killsig = sig;
}
if(sig == SIGKILL){
p->killed = 1;
if(p->killsig == 0)
p->killsig = sig;
}
if(p->state == SLEEPING){
p->state = RUNNABLE;
sched_enqueue(p); // a signal is a wakeup too; same floor
}
}
release(&ptable.lock);
return found ? 0 : -1;
}
// Look up a process's process-group id. Returns -1 if no such live process.
int
proc_pgid(int pid)
{
struct proc *p;
int r = -1;
acquire(&ptable.lock);
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
if(p->pid == pid && p->state != UNUSED && p->state != ZOMBIE){
r = p->pgid;
break;
}
}
release(&ptable.lock);
return r;
}
// setpgid(pid, pgid) with POSIX's argument shorthands: pid 0 means the caller,
// pgid 0 means "use pid as the group id" (i.e. become a group leader).
int
proc_setpgid(int pid, int pgid)
{
struct proc *p;
int r = -ESRCH;
if(pid < 0 || pgid < 0)
return -EINVAL;
if(pid == 0)
pid = myproc()->pid;
if(pgid == 0)
pgid = pid;
acquire(&ptable.lock);
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
if(p->pid != pid || p->state == UNUSED || p->state == ZOMBIE)
continue;
// A caller may only move itself or one of its own children, and never
// across a session boundary.
if(p != myproc() && p->parent != proc_leader(myproc()))
r = -ESRCH;
else if(p->sid != myproc()->sid)
r = -EPERM;
else {
p->pgid = pgid;
r = 0;
}
break;
}
release(&ptable.lock);
return r;
}
// setsid(): become leader of a brand-new session and process group. Fails if
// the caller already leads a process group, since that group would otherwise
// end up split across two sessions.
int
proc_setsid(void)
{
struct proc *p;
int r;
acquire(&ptable.lock);
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
if(p->state != UNUSED && p->state != ZOMBIE && p->pgid == myproc()->pid){
release(&ptable.lock);
return -EPERM;
}
}
myproc()->sid = myproc()->pid;
myproc()->pgid = myproc()->pid;
// A new session starts with no controlling terminal; that is what lets the
// first terminal it opens become one. Keeping the old one meant a tmux pane
// still had tty1 as its /dev/tty, and zsh and htop drew straight onto the
// console underneath tmux instead of into their pane.
myproc()->ctty = 0;
r = myproc()->sid;
release(&ptable.lock);
return r;
}
int
kill(int pid)
{
struct proc *p;
acquire(&ptable.lock);
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
if(p->pid == pid){
p->killed = 1;
if(p->state == SLEEPING){
p->state = RUNNABLE;
sched_enqueue(p); // a signal is a wakeup too; same floor
}
release(&ptable.lock);
return 0;
}
}
release(&ptable.lock);
return -1;
}
void
procdump(void)
{
static char *states[] = {
[UNUSED] "unused",
[EMBRYO] "embryo",
[SLEEPING] "sleep ",
[RUNNABLE] "runble",
[RUNNING] "run ",
[ZOMBIE] "zombie"
};
int i;
struct proc *p;
char *state;
addr_t pc[10];
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
if(p->state == UNUSED)
continue;
if(p->state >= 0 && p->state < NELEM(states) && states[p->state])
state = states[p->state];
else
state = "???";
cprintf("%d %s %s", p->pid, state, p->name);
if(p->state == SLEEPING){
getstackpcs((addr_t*)p->context->rbp, pc);
for(i=0; i<10 && pc[i] != 0; i++)
cprintf(" %p", pc[i]);
}
// Where a thread that is not asleep last was in user space: the question
// when a program spins without making a system call.
if((p->state == RUNNABLE || p->state == RUNNING) && p->tf && p->pgdir)
cprintf(" user rip %p rsp %p", p->tf->rip, p->tf->rsp);
cprintf("\n");
}
}
// ---------------------------------------------------------------------------
// Snapshots, for /proc
//
// procfs must not walk ptable itself: it would need the lock, the struct
// layout, and a rule about what happens when a process exits between the
// readdir that listed it and the open that reads it. These two copy what is
// needed out from under the lock and hand back a value, so a process that dies
// mid-listing produces a stale line rather than a use-after-free.
// ---------------------------------------------------------------------------
// Fill `out` with the `idx`-th live process (0-based, in table order).
// Returns 0 on success, -1 when idx is past the end.
// proc_snapshot without the lock, and only the fields a crash screen lists.
// For GMMOS: the lock may be held by a processor the panic stopped, and a torn
// read of one line is a far smaller loss than a crash screen that hangs.
int
proc_peek(int idx, struct procinfo *out)
{
struct proc *p;
int n = 0;
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
if (p->state == UNUSED)
continue;
if (n++ != idx)
continue;
out->pid = p->pid;
out->ppid = p->parent ? p->parent->tgid : 0;
out->tgid = p->tgid;
out->nthreads = 0;
for (struct proc *q = ptable.proc; q < &ptable.proc[NPROC]; q++)
if (q->tgid == p->tgid && q->state != UNUSED && q->state != ZOMBIE)
out->nthreads++;
if (out->nthreads == 0)
out->nthreads = 1;
out->state = p->state;
out->sz = p->sz;
out->killed = p->killed;
safestrcpy(out->name, p->name, sizeof(out->name));
return 0;
}
return -1;
}
int
proc_snapshot(int idx, struct procinfo *out)
{
struct proc *p;
int n = 0;
acquire(&ptable.lock);
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
if (p->state == UNUSED)
continue;
if (n++ != idx)
continue;
out->pid = p->pid;
out->ppid = p->parent ? p->parent->tgid : 0;
out->tgid = p->tgid;
out->nthreads = 0;
for (struct proc *q = ptable.proc; q < &ptable.proc[NPROC]; q++)
if (q->tgid == p->tgid && q->state != UNUSED && q->state != ZOMBIE)
out->nthreads++;
if (out->nthreads == 0)
out->nthreads = 1;
out->pgid = p->pgid;
out->sid = p->sid;
out->uid = p->uid;
out->state = p->state;
out->sz = p->sz;
out->killed = p->killed;
out->ctty = p->ctty;
out->utime = p->utime;
out->stime = p->stime;
out->cutime = p->cutime;
out->cstime = p->cstime;
out->nice = p->nice;
out->vruntime = p->vruntime;
out->nvcsw = p->nvcsw;
out->nivcsw = p->nivcsw;
safestrcpy(out->name, p->name, sizeof(out->name));
safestrcpy(out->exe, p->exe, sizeof(out->exe));
out->kthread = p->kfn != 0;
out->stopped = p->stopsig != 0;
release(&ptable.lock);
return 0;
}
release(&ptable.lock);
return -1;
}
// The same, by pid rather than index.
int
proc_snapshot_pid(int pid, struct procinfo *out)
{
int i;
struct procinfo pi;
for (i = 0; proc_snapshot(i, &pi) == 0; i++)
if (pi.pid == pid) { *out = pi; return 0; }
return -1;
}
// /proc/<pid>/fd: which descriptors are open, and what one points at. The
// ptable lock keeps the process from being reaped -- exit() closes its files
// first -- and file_describe() holds the file steady itself.
int
proc_fd_mask(int pid, uint64 mask[2])
{
struct proc *p;
int r = -1;
mask[0] = mask[1] = 0;
acquire(&ptable.lock);
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
if (p->state == UNUSED || p->pid != pid)
continue;
for (int fd = 0; fd < NOFILE && fd < 128 && p->files; fd++)
if (p->files->ofile[fd])
mask[fd / 64] |= 1ULL << (fd % 64);
r = 0;
break;
}
release(&ptable.lock);
return r;
}
int
proc_fd_link(int pid, int fd, char *buf, int cap)
{
struct proc *p;
int r = -1;
if (fd < 0 || fd >= NOFILE)
return -1;
acquire(&ptable.lock);
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
if (p->state == UNUSED || p->pid != pid)
continue;
if (p->files && p->files->ofile[fd])
r = file_describe(p->files->ofile[fd], buf, cap);
break;
}
release(&ptable.lock);
return r;
}
int proc_num(void)
{
struct proc *p;
int num = 0;
acquire(&ptable.lock);
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++)
if (p->state != UNUSED) num++;
release(&ptable.lock);
return num;
}
int proc_exists(int pid)
{
struct proc *p;
acquire(&ptable.lock);
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
if (p->pid == pid && p->state != UNUSED && p->state != ZOMBIE) {
release(&ptable.lock);
return 1;
}
}
release(&ptable.lock);
return 0;
}
// Make PID 1 exit at its next trap return, so proc_reincarnate() runs.
//
// Not a signal: init_ignores() drops signals to initproc and should, because
// init is not killable from userspace on any Unix. This is the kernel
// deciding on its own behalf, which is a different thing, and it exists so
// the resurrection path can be exercised rather than reasoned about.
void
proc_kill_init(void)
{
if (initproc)
initproc->killed = 1;
}
// Ticks spent in user and kernel mode by every process, living or reaped.
// What /proc/stat's busy columns are made of; the rest of the time was idle.
void
proc_cputime(uint64 *user, uint64 *sys)
{
struct proc *p;
uint64 u, s;
acquire(&ptable.lock);
u = reaped_utime;
s = reaped_stime;
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++)
if (p->state != UNUSED) {
u += p->utime;
s += p->stime;
}
release(&ptable.lock);
*user = u;
*sys = s;
}
// /proc/<pid>/maps: see mapname_gen(). 0 bytes for a pid that is gone.
int
proc_maps(int pid, char *buf, int cap)
{
struct mm *m = 0;
acquire(&ptable.lock);
for (struct proc *p = ptable.proc; p < &ptable.proc[NPROC]; p++)
if (p->pid == pid && p->state != UNUSED && p->state != ZOMBIE && p->mm) {
m = p->mm;
mm_get(m);
break;
}
release(&ptable.lock);
if (!m)
return 0;
int n = mapname_gen(m, buf, cap);
mm_put(m);
return n;
}
Reference in a new issue View git blame Copy permalink