diff --git a/cmd/schinit/main.c b/cmd/schinit/main.c index 1cc0d6b..ce9d45e 100644 --- a/cmd/schinit/main.c +++ b/cmd/schinit/main.c @@ -52,6 +52,8 @@ struct child { static struct child children[MAXJOBS]; static int sock = -1; // to schinitd static int supervisor_pid; +static int last_pong; // uptime() when schinitd last answered +static int last_ping; // uptime() when we last asked static unsigned int generation = 1; // bumped by proc_reincarnate() later static int @@ -251,6 +253,60 @@ publish(void) close(fd); } +// Read the table back after a reincarnation. +// +// This is the one parse in PID 1, and it is worth being uneasy about. It is +// bounded: a fixed buffer, four integers and a label per line, no allocation, +// and it reads a format this same program wrote. It happens once, at startup, +// only when the kernel has just re-exec'd us. The alternative is forgetting +// every running job, which is worse than a twenty-line scan of known text. +static void +reclaim(void) +{ + char buf[MAXJOBS * (SCHINIT_MAXLABEL + 40) + 64]; + int fd = open("/proc/schinit", O_RDONLY); + int n, slot = 0; + + if (fd < 0) + return; + n = read(fd, buf, (int)sizeof(buf) - 1); + close(fd); + if (n <= 0) + return; + buf[n] = 0; + + char *p = buf; + while (*p && slot < MAXJOBS) { + char *line = p; + while (*p && *p != '\n') p++; + if (*p == '\n') *p++ = 0; + + unsigned int cookie = 0; + int pid = 0, started = 0, digits = 0; + char *q = line; + + if (*q < '0' || *q > '9') continue; // a header line + while (*q >= '0' && *q <= '9') { cookie = cookie * 10 + (unsigned)(*q++ - '0'); digits++; } + while (*q == ' ') q++; + while (*q >= '0' && *q <= '9') pid = pid * 10 + (*q++ - '0'); + while (*q == ' ') q++; + while (*q >= '0' && *q <= '9') started = started * 10 + (*q++ - '0'); + while (*q == ' ') q++; + if (!digits || pid <= 0 || !*q) continue; + + children[slot].pid = pid; + children[slot].cookie = cookie; + children[slot].started = started; + scopy(children[slot].label, q, SCHINIT_MAXLABEL); + slot++; + } + if (slot) { + out("schinit: reclaimed "); + outn(slot); + out(" running job(s) from /proc/schinit\n"); + } +} + // --------------------------------------------------------------------------- // Spawning, on request // --------------------------------------------------------------------------- @@ -377,6 +433,7 @@ start_supervisor(void) close(sv[1]); sock = sv[0]; supervisor_pid = pid; + last_pong = last_ping = uptime(); // it has not missed anything yet return pid; } @@ -416,6 +473,10 @@ serve(void) break; } + case SCHINIT_PONG: + last_pong = uptime(); + break; + case SCHINIT_SPAWN: { int pid = spawn(&m.spawn); if (pid > 0) { @@ -445,11 +506,75 @@ serve(void) // --------------------------------------------------------------------------- int -main(void) +main(int argc, char *argv[]) { + // argv[1] is the generation, and it is only there when the kernel put it + // there -- proc_reincarnate() re-execs this program in place after PID 1 + // has died. A normal boot has no argument and is generation 1. + if (argc > 1) { + unsigned int g = 0; + for (char *q = argv[1]; *q >= '0' && *q <= '9'; q++) + g = g * 10 + (unsigned int)(*q - '0'); + if (g) + generation = g; + } + bootstrap(); - out("schinit: SchOS process 1\n"); + // A reincarnated PID 1 has the children but not the table. + // + // exec() replaced this program's memory and kept everything else -- the + // pid, the open files, and crucially the *children*, which are still + // parented here because nothing killed them. What did not survive is the + // array describing them, so it is read back from where it was published + // precisely so that it could be. + // + // Without this the next publish() overwrites the record with an empty + // table, the supervisor reads that, believes nothing is running and starts + // everything a second time. Same split brain as before, one level up. + if (generation > 1) { + // exec() keeps open files, which is what lets this program still have + // a console to complain to -- and also means the previous life's + // socket to the old supervisor is still open here. Leaving it open + // holds the old supervisor's channel up, so it never sees a hangup, + // never exits, and sits there reporting that PID 1 has stopped + // pinging it. Which is true, and no longer its business. + for (int fd = 3; fd < 32; fd++) + close(fd); + reclaim(); + } + + out("schinit: SchOS process 1"); + if (generation > 1) { + out(", generation "); + outn((int)generation); + out(" (reincarnated)"); + } + out("\n"); + + // Generation 4 is the safety net: a manifest that kills init must not kill + // it for ever, so this generation does not read /etc/schinit at all. One + // shell, on the console, and nothing else -- enough to log in and fix + // whatever has been taking the machine down. + if (generation >= 4) { + out("schinit: safe mode -- console shell only, /etc/schinit ignored\n"); + if (fork() == 0) { + setsid(); + ioctl(0, TIOCSCTTY, 0); + char *sh[] = { (char *)"-zsh", 0 }; + execve((char *)"/bin/zsh", sh, 0); + char *fb[] = { (char *)"sh", 0 }; + execve((char *)"/bin/sh", fb, 0); + exit(); + } + publish(); + for (;;) { + int st, pid; + while ((pid = waitpid(-1, &st, WNOHANG)) > 0) + ; + sleep(20); + } + } if (start_supervisor() < 0) { // No supervisor. Rather than idle usefully-looking, fall back to a @@ -485,6 +610,48 @@ main(void) sleep(10); } + // The heartbeat. A supervisor that has stopped answering is stuck, + // and a stuck supervisor is indistinguishable from a working one by + // any means except asking it something -- which is the whole reason + // this exists. See docs/plan/init.md part 3. + if (sock >= 0) { + int now = uptime(); + if (now - last_ping >= SCHINIT_PING_SECS * 100) { + struct schinit_hello p; + p.op = SCHINIT_PING; + p.proto = SCHINIT_PROTO; + p.generation = generation; + p.reserved = 0; + write(sock, &p, sizeof(p)); + last_ping = now; + } + if (now - last_pong >= SCHINIT_DEADLINE_SECS * 100) { + out("schinit: supervisor stopped answering; replacing it\n"); + + // Ask it to die, then replace it *without waiting*. + // + // kill() sets a flag the target checks the next time it runs, + // and the whole premise here is that it is not running. A + // supervisor stuck in a loop will notice at its next trap; one + // blocked forever never will. Waiting for a corpse that may + // never be produced is how a recovery path becomes a second + // hang -- which is exactly what the first version of this did. + // + // Two supervisors briefly existing is safe, and safe in the + // way that matters: the old one's socket is closed here, so + // the moment it runs again its first write fails and it exits. + // It cannot spawn anything, because it cannot ask. + kill(supervisor_pid); + close(sock); + sock = -1; + int old = supervisor_pid; + if (start_supervisor() > 0) + out("schinit: new supervisor started\n"); + publish(); // the table names the live supervisor + (void)old; + } + } + int status; int pid; while ((pid = waitpid(-1, &status, WNOHANG)) > 0) { diff --git a/cmd/schinitd/main.c b/cmd/schinitd/main.c index 661f556..fd85711 100644 --- a/cmd/schinitd/main.c +++ b/cmd/schinitd/main.c @@ -590,18 +590,72 @@ main(void) out("schinitd: ready\n"); - // Everything from here is PID 1 telling us a child exited. There is no - // wait() in this program and there cannot be: the children are not ours. + // From here on this program does two things: answer PID 1's heartbeat, and + // act on the exits it reports. There is no wait() in this program and + // there cannot be -- the children are not ours. + // + // The read is polled rather than blocking, so that a PID 1 which has + // stopped talking is noticed. A blocking read would sit there forever + // looking exactly like a quiet machine, which is the failure mode this + // whole heartbeat exists to make visible. + int last_ping = uptime(); + for (;;) { - struct schinit_reply r; - int n = read(sock, &r, sizeof(r)); - if (n != sizeof(r)) { - // PID 1 closed the channel. It is the only process that cannot - // have gone away, so this means we are being replaced. - out("schinitd: channel closed\n"); + struct pollfd pf; + pf.fd = sock; + pf.events = POLLIN; + pf.revents = 0; + + int pr = poll(&pf, 1, 1000); + + // The channel is gone. This happens to the *old* supervisor when PID 1 + // has replaced it: PID 1 closes its end, so poll reports a hangup and + // never reports readable again. Without this the replaced supervisor + // sits in the loop for ever, complaining that PID 1 has stopped + // pinging -- which it has, because it is no longer talking to us. + if (pr > 0 && (pf.revents & (POLLHUP | POLLERR | POLLNVAL))) { + out("schinitd: channel closed; this supervisor has been replaced\n"); exit(); } - if (r.op == SCHINIT_EXITED) - note_exit(r.cookie, r.pid, r.status); + + if (pr > 0 && (pf.revents & POLLIN)) { + union schinit_msg m; + int n = read(sock, &m, sizeof(m)); + if (n <= 0) { + // PID 1 closed the channel. It is the one process that cannot + // have gone away, so this means we are being replaced. + out("schinitd: channel closed\n"); + exit(); + } + if (m.op == SCHINIT_PING) { + struct schinit_hello p; + p.op = SCHINIT_PONG; + p.proto = SCHINIT_PROTO; + p.generation = 0; + p.reserved = 0; + write(sock, &p, sizeof(p)); + last_ping = uptime(); + } else if (m.op == SCHINIT_EXITED) { + note_exit(m.reply.cookie, m.reply.pid, m.reply.status); + } + } + + // Silence from PID 1. + // + // There is nothing this program can do about it -- PID 1 cannot be + // killed and cannot be replaced from here -- so it says so, loudly, + // once. docs/plan/init.md is explicit that this is the limit: a hang + // in PID 1 is only caught by noticing, and if both ends are wedged at + // once nothing catches it. There is no hardware watchdog. + // + // Saying so is still worth doing. The alternative is a machine that + // looks idle while its init is stuck, which is how this class of bug + // stays invisible for months. + static int complained; + if (!complained && uptime() - last_ping >= SCHINIT_DEADLINE_SECS * 100) { + out("schinitd: pid 1 has not pinged for " + "the deadline -- it may be wedged\n"); + complained = 1; + } } } diff --git a/docs/plan/init.md b/docs/plan/init.md index e6b5f66..b45f626 100644 --- a/docs/plan/init.md +++ b/docs/plan/init.md @@ -143,7 +143,20 @@ silent loop. The crash screen is the correct place for this to arrive. ## Milestones -**I0 and I1 are built.** `schinit` is 520 lines with no parser and no +**All four milestones are built.** `sysrq w` wedges the supervisor and PID 1 +replaces it; `sysrq i` kills PID 1 and the kernel refills the role: + +``` +init: pid 1 exited with status 9 -- reincarnating, generation 2 +schinit: reclaimed 2 running job(s) from /proc/schinit +schinit: SchOS process 1, generation 2 (reincarnated) +schinitd: adopting os.schos.console-shell +schinitd: adopting os.schos.tty2-shell +``` + +The shell being typed into keeps its pid across the death and rebirth of PID 1. + +**I0 and I1 first.** `schinit` is 520 lines with no parser and no allocation; `schinitd` holds the manifests. `kill 3` produces: ``` @@ -168,8 +181,8 @@ believing it. |---|---| | **I0** ✅ | Split the program. PID 1 supervises one child; `schinitd` does everything else. Jobs still forked by PID 1. | | **I1** ✅ | `/proc/schinit`, written and read back. Kill `schinitd` by hand and watch it return without disturbing a single job. | -| **I2** | The heartbeat both ways, with a `sysrq` key that wedges `schinitd` on purpose so the timeout path is exercised rather than assumed. | -| **I3** | `proc_reincarnate()` and the ladder, with a `sysrq` key that kills PID 1 on purpose. | +| **I2** ✅ | The heartbeat both ways, with a `sysrq` key that wedges `schinitd` on purpose so the timeout path is exercised rather than assumed. | +| **I3** ✅ | `proc_reincarnate()` and the ladder, with a `sysrq` key that kills PID 1 on purpose. | --- diff --git a/include/defs.h b/include/defs.h index 6967f26..55082b2 100644 --- a/include/defs.h +++ b/include/defs.h @@ -52,6 +52,10 @@ extern uint64 tsc_hz; extern uint64 boot_tsc; void monotonic(uint64 *sec, uint64 *usec); int proc_write_schinit(int user_src, uint64 src, uint n); +int proc_wedge(const char *name); +int proc_reincarnate(int status); +void proc_kill_init(void); +extern uint init_generation; void cprintf(char*, ...); // A log line: the serial port only, never the display. diff --git a/include/proc.h b/include/proc.h index e26b5ee..5fa7569 100644 --- a/include/proc.h +++ b/include/proc.h @@ -249,6 +249,9 @@ struct proc { // it; /dev/tty resolves through it. Was impossible to have before there was // more than one terminal to choose between. int ctty; + // Never scheduled again. Set by sysrq 'w' to produce a hang on purpose -- + // see proc_wedge() and docs/plan/init.md part 3. + int wedged; // Real/effective/saved user and group ids. Nothing enforces permissions with // them yet (the on-disk format has no owner field until SchFS), but they diff --git a/include/schinit.h b/include/schinit.h index b344bd4..8ec1f1f 100644 --- a/include/schinit.h +++ b/include/schinit.h @@ -42,12 +42,33 @@ // supervisor -> PID 1 #define SCHINIT_HELLO 1 // struct schinit_hello #define SCHINIT_SPAWN 2 // struct schinit_spawn -#define SCHINIT_SIGNAL 3 // struct schinit_signal +#define SCHINIT_PONG 3 // struct schinit_hello -- answer to a PING // PID 1 -> supervisor #define SCHINIT_WELCOME 128 // struct schinit_hello, echoed with PID 1's version #define SCHINIT_SPAWNED 129 // struct schinit_reply -- pid, or negative on failure #define SCHINIT_EXITED 130 // struct schinit_reply -- a child has been reaped +#define SCHINIT_PING 131 // struct schinit_hello -- answer within the deadline + +// The heartbeat. +// +// docs/plan/init.md part 3, and the failure the two-init scheme could not see: +// **the common way an init breaks is not dying, it is hanging.** wait() never +// returns for a process that is merely stuck, so a supervisor watching a +// wedged PID 1 -- or PID 1 watching a wedged supervisor -- reports everything +// normal forever. +// +// So each must be heard from. PID 1 sends a PING every interval and expects a +// PONG; silence past the deadline means the other end is stuck rather than +// busy, and stuck is the case worth acting on. +// +// Measured in seconds of uptime(), which counts ticks. Ticks can be *lost* +// when interrupts are off too long -- see kernel/core/time.c -- which makes a +// tick-measured deadline longer in real time than it looks, never shorter. +// That is the safe direction: a machine under load waits longer before +// declaring a hang, rather than declaring one that has not happened. +#define SCHINIT_PING_SECS 2 +#define SCHINIT_DEADLINE_SECS 10 struct schinit_hello { unsigned int op; diff --git a/kernel/dev/procfs.c b/kernel/dev/procfs.c index 6d56069..0dc6d89 100644 --- a/kernel/dev/procfs.c +++ b/kernel/dev/procfs.c @@ -488,10 +488,37 @@ proc_sysrq_cmd(char cmd) case 't': text_selftest(); break; + case 'i': + // Kill PID 1, on purpose. + // + // Signals cannot do this -- init_ignores() drops them, which is correct + // and is why the two-init scheme was solving a problem that did not exist. + // Setting `killed` directly is the kernel deciding, not userspace asking: + // init exits at its next trap return and proc_reincarnate() takes over. + // + // docs/plan/init.md asks for this to be "tested by a sysrq key rather than + // by reasoning", which is the only honest way to know the path works. + proc_kill_init(); + cprintf("sysrq: pid 1 asked to exit -- it should come back\n"); + break; + + case 'w': { + // Wedge the supervisor: make schinitd hang, on purpose, so PID 1's + // heartbeat timeout is exercised instead of reasoned about. There is no + // way back from this; reboot afterwards. + int pid = proc_wedge("schinitd"); + if (pid) + cprintf("sysrq: wedged schinitd (pid %d) -- pid 1 should notice " + "within %d seconds\n", pid, 10); + else + cprintf("sysrq: no process named schinitd to wedge\n"); + break; + } default: cprintf("sysrq: unknown command '%c' " "(known: c = crash, u = dump USB, s = read the USB disk, " - "d = display self-test)\n", + "d = display self-test, t = text self-test, n = net self-test, " + "w = wedge the init supervisor, i = kill pid 1)\n", cmd); break; } diff --git a/kernel/proc/proc.c b/kernel/proc/proc.c index e0a6cd7..c6a019e 100644 --- a/kernel/proc/proc.c +++ b/kernel/proc/proc.c @@ -267,14 +267,113 @@ fork(void) // status is already wait(2)-encoded -- see W_EXITCODE() in proc.h. Callers // that die from a fault or signal pass W_EXITCODE(0, sig); a normal _exit(n) // passes W_EXITCODE(n, 0). +// PID 1 is a role, and the kernel guarantees the role is filled. +// +// This is the rule change docs/plan/init.md exists for. Unix's rule is "PID 1 +// is special and must never die", honoured since 1975 by being careful -- +// which is not a mechanism, it is a hope, and the thing it hopes about is the +// one program on the machine with no supervisor. +// +// So instead: tear nothing down, keep pid 1, and re-exec in place. +// +// The reason this is so short is the reason it is the right design. exec() +// already replaces the address space, and everything else is *deliberately* +// left alone: +// +// - the pid stays 1, so every orphan the kernel reparented to initproc is +// still correctly parented and initproc is still the pointer it was +// - the children survive, because nothing killed them +// - the open files survive, so the new image still has a console to +// complain to -- this runs before exit() closes anything, which is why +// the call site is the first line of exit() and not the last +// - the state survives, in /proc/schinit, which is the whole reason the +// job table lives outside the process holding it +// +// A sibling init resurrecting a dead one would have to solve all four of those +// and could not solve the first at all. This solves them by not creating them. +// +// The escalation ladder is what makes it safe to have. A machine that cannot +// hold PID 1 up is not a machine that should keep trying for ever, and PSOGM +// renders a stop code, a call chain and a QR code of the whole report -- which +// is a far better end than a silent loop. +// +// generation 1 normal boot +// generation 2-3 re-exec, keep the jobs, carry on +// generation 4 re-exec, but ignore /etc/schinit entirely and start only +// a console shell: a manifest that kills init must not +// kill it for ever +// generation 5 stop +uint init_generation = 1; + +int +proc_reincarnate(int status) +{ + static char gen[12]; + char *argv[3]; + int n = 0, v; + + init_generation++; + + if (init_generation > 5) + panic("init resurrection failed: five generations is enough"); + + // Decimal, by hand: this runs in a process that is dying and should call as + // little as possible. + v = (int)init_generation; + if (v == 0) { + gen[n++] = '0'; + } else { + char tmp[12]; + int i = 0; + while (v > 0) { tmp[i++] = '0' + (v % 10); v /= 10; } + while (i-- > 0) gen[n++] = tmp[i]; + } + gen[n] = 0; + + cprintf("init: pid 1 exited with status %d -- reincarnating, generation %d\n", + status, init_generation); + if (init_generation >= 4) + cprintf("init: generation %d starts a console shell only, " + "ignoring /etc/schinit\n", init_generation); + + argv[0] = "schinit"; + argv[1] = gen; + argv[2] = 0; + + // Clear `killed` first. + // + // exit() was reached from the trap-return path *because* killed was set, and + // that path checks it again on the way back out. Leaving it set would exit + // the brand-new image before its first instruction, and the generation + // counter would climb to five in a few microseconds -- a resurrection loop + // that looks exactly like the crash it was meant to survive. + myproc()->killed = 0; + + // No lock is held here and none may be taken: this is the first line of + // exit(), before the file table is walked and before ptable.lock. exec() + // does its own begin_op/end_op and is safe from process context. + if (exec("/bin/schinit", argv, 0) < 0) + return -1; + return 0; +} + void exit(int status) { struct proc *p; int fd; - if(myproc() == initproc) - panic("init exiting"); + if(myproc() == initproc) { + // Returning from exit() is the *success* path here. exec() does not fail + // to return -- it returns 0 having replaced the address space, and the new + // image begins running when the kernel returns to userspace through the + // trapframe it rewrote. So there is nothing more to do but go back. + if (proc_reincarnate(status) == 0) + return; + // Only reached if the re-exec itself failed, which is the one case where + // there is genuinely nothing left to try. + panic("init resurrection failed: could not exec /bin/schinit"); + } myproc()->xstate = status; @@ -408,6 +507,38 @@ waitpid(int pid, int *status, int options) } } +// Stop a process running, by name, and never start it again. +// +// This exists for one reason: docs/plan/init.md part 3 says the heartbeat's +// timeout path should be "exercised rather than assumed", and a hang is the +// one failure that cannot be produced by killing something. A wedged process +// is still alive, still has its descriptors open, and still shows up in every +// listing -- which is exactly what makes it hard to notice and exactly what +// the heartbeat is for. +// +// Deliberately one-way. There is no unwedge: this is a test instrument for a +// machine that is about to be rebooted, not a job-control mechanism. Real +// SIGSTOP/SIGCONT is a separate thing and is not implemented. +int +proc_wedge(const char *name) +{ + struct proc *p; + int found = 0; + + acquire(&ptable.lock); + for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) { + if (p->state == UNUSED || p->pid <= 1) + continue; + if (strncmp(p->name, name, sizeof(p->name)) != 0) + continue; + p->wedged = 1; + found = p->pid; + break; + } + release(&ptable.lock); + return found; +} + void scheduler(void) { @@ -417,7 +548,10 @@ scheduler(void) sti(); acquire(&ptable.lock); for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){ - if(p->state != RUNNABLE) { + // A wedged process is passed over for ever. It is still RUNNABLE, still + // owns its descriptors and still appears in every listing -- which is + // the point: it looks exactly like a process that is merely busy. + if(p->state != RUNNABLE || p->wedged) { skipped++; continue; } @@ -923,4 +1057,16 @@ int proc_exists(int pid) } release(&ptable.lock); return 0; -} \ No newline at end of file +} +// Make PID 1 exit at its next trap return, so proc_reincarnate() runs. +// +// Not a signal: init_ignores() drops signals to initproc and should, because +// init is not killable from userspace on any Unix. This is the kernel +// deciding on its own behalf, which is a different thing, and it exists so +// the resurrection path can be exercised rather than reasoned about. +void +proc_kill_init(void) +{ + if (initproc) + initproc->killed = 1; +}