init: a hang is as detectable as a death, and PID 1 comes back
docs/plan/init.md milestones I2 and I3. The rule change.
**I2 -- the heartbeat.** The failure the two-init scheme could not see is that
an init usually does not die, it *hangs*: wait() never returns for a process
that is merely stuck, so a supervisor watching a wedged init reports
everything normal for ever. PID 1 now pings every two seconds and expects an
answer within ten; silence means stuck, and stuck is worth acting on.
Exercised rather than reasoned about, as the plan asks: `sysrq w` wedges
schinitd by making the scheduler pass over it for ever. It stays alive, keeps
its descriptors and appears in every listing -- which is the point.
Two things that only showed up by running it:
- kill() sets a flag the target checks when it next runs, and the premise
here is that it is not running. Waiting for a corpse that may never be
produced turns the recovery path into a second hang. PID 1 replaces the
supervisor *without waiting*; the old one's socket is closed, so the moment
it runs again its first write fails and it exits.
- the replaced supervisor must notice the hangup. poll() reports POLLHUP and
never POLLIN again, so a loop that only looks for POLLIN sits there
complaining that PID 1 has stopped pinging it.
**I3 -- proc_reincarnate().** exit() on initproc panicked. Now the kernel
tears down nothing, keeps pid 1, and re-execs /bin/schinit in place with a
generation counter. The pid stays 1, so every orphan the kernel reparented is
still correctly parented; the children survive; the open files survive, which
is why this is the first line of exit() and not the last; and the job table
survives in /proc/schinit, which is what it is there for.
The escalation ladder: generations 2-3 carry on, generation 4 ignores
/etc/schinit and starts a console shell only -- a manifest that kills init
must not kill it for ever -- and generation 5 stops, with PSOGM rendering
INIT_RESURRECTION_FAILED, a call chain and a QR code.
Three bugs found by running it, each a lesson about exec():
- exec() returns 0 on success; it does not fail to return. The new image
starts when the kernel returns to userspace through the rewritten
trapframe. Returning from exit() is the success path.
- exit() was reached because `killed` was set, and the trap path checks it
again on the way out. Leaving it set exits the new image before its first
instruction: five generations in a few microseconds, a resurrection loop
that looks exactly like the crash it was meant to survive.
- exec() keeps open files, which is what gives the new image a console --
and also keeps the previous life's socket, holding the old supervisor's
channel up so it never sees a hangup.
Verified: `sysrq i`, and the shell being typed into keeps its pid across the
death and rebirth of PID 1, with /proc/schinit byte-identical either side.
schfs-check passes.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VQ7Qx8E1q1M1bbCq8QLHS2
This commit is contained in:
parent
c80b7bf3b2
commit
0def474272
8 changed files with 456 additions and 21 deletions
|
|
@ -52,6 +52,8 @@ struct child {
|
||||||
static struct child children[MAXJOBS];
|
static struct child children[MAXJOBS];
|
||||||
static int sock = -1; // to schinitd
|
static int sock = -1; // to schinitd
|
||||||
static int supervisor_pid;
|
static int supervisor_pid;
|
||||||
|
static int last_pong; // uptime() when schinitd last answered
|
||||||
|
static int last_ping; // uptime() when we last asked
|
||||||
static unsigned int generation = 1; // bumped by proc_reincarnate() later
|
static unsigned int generation = 1; // bumped by proc_reincarnate() later
|
||||||
|
|
||||||
static int
|
static int
|
||||||
|
|
@ -251,6 +253,60 @@ publish(void)
|
||||||
close(fd);
|
close(fd);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Read the table back after a reincarnation.
|
||||||
|
//
|
||||||
|
// This is the one parse in PID 1, and it is worth being uneasy about. It is
|
||||||
|
// bounded: a fixed buffer, four integers and a label per line, no allocation,
|
||||||
|
// and it reads a format this same program wrote. It happens once, at startup,
|
||||||
|
// only when the kernel has just re-exec'd us. The alternative is forgetting
|
||||||
|
// every running job, which is worse than a twenty-line scan of known text.
|
||||||
|
static void
|
||||||
|
reclaim(void)
|
||||||
|
{
|
||||||
|
char buf[MAXJOBS * (SCHINIT_MAXLABEL + 40) + 64];
|
||||||
|
int fd = open("/proc/schinit", O_RDONLY);
|
||||||
|
int n, slot = 0;
|
||||||
|
|
||||||
|
if (fd < 0)
|
||||||
|
return;
|
||||||
|
n = read(fd, buf, (int)sizeof(buf) - 1);
|
||||||
|
close(fd);
|
||||||
|
if (n <= 0)
|
||||||
|
return;
|
||||||
|
buf[n] = 0;
|
||||||
|
|
||||||
|
char *p = buf;
|
||||||
|
while (*p && slot < MAXJOBS) {
|
||||||
|
char *line = p;
|
||||||
|
while (*p && *p != '\n') p++;
|
||||||
|
if (*p == '\n') *p++ = 0;
|
||||||
|
|
||||||
|
unsigned int cookie = 0;
|
||||||
|
int pid = 0, started = 0, digits = 0;
|
||||||
|
char *q = line;
|
||||||
|
|
||||||
|
if (*q < '0' || *q > '9') continue; // a header line
|
||||||
|
while (*q >= '0' && *q <= '9') { cookie = cookie * 10 + (unsigned)(*q++ - '0'); digits++; }
|
||||||
|
while (*q == ' ') q++;
|
||||||
|
while (*q >= '0' && *q <= '9') pid = pid * 10 + (*q++ - '0');
|
||||||
|
while (*q == ' ') q++;
|
||||||
|
while (*q >= '0' && *q <= '9') started = started * 10 + (*q++ - '0');
|
||||||
|
while (*q == ' ') q++;
|
||||||
|
if (!digits || pid <= 0 || !*q) continue;
|
||||||
|
|
||||||
|
children[slot].pid = pid;
|
||||||
|
children[slot].cookie = cookie;
|
||||||
|
children[slot].started = started;
|
||||||
|
scopy(children[slot].label, q, SCHINIT_MAXLABEL);
|
||||||
|
slot++;
|
||||||
|
}
|
||||||
|
if (slot) {
|
||||||
|
out("schinit: reclaimed ");
|
||||||
|
outn(slot);
|
||||||
|
out(" running job(s) from /proc/schinit\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Spawning, on request
|
// Spawning, on request
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
@ -377,6 +433,7 @@ start_supervisor(void)
|
||||||
close(sv[1]);
|
close(sv[1]);
|
||||||
sock = sv[0];
|
sock = sv[0];
|
||||||
supervisor_pid = pid;
|
supervisor_pid = pid;
|
||||||
|
last_pong = last_ping = uptime(); // it has not missed anything yet
|
||||||
return pid;
|
return pid;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -416,6 +473,10 @@ serve(void)
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
case SCHINIT_PONG:
|
||||||
|
last_pong = uptime();
|
||||||
|
break;
|
||||||
|
|
||||||
case SCHINIT_SPAWN: {
|
case SCHINIT_SPAWN: {
|
||||||
int pid = spawn(&m.spawn);
|
int pid = spawn(&m.spawn);
|
||||||
if (pid > 0) {
|
if (pid > 0) {
|
||||||
|
|
@ -445,11 +506,75 @@ serve(void)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
int
|
int
|
||||||
main(void)
|
main(int argc, char *argv[])
|
||||||
{
|
{
|
||||||
|
// argv[1] is the generation, and it is only there when the kernel put it
|
||||||
|
// there -- proc_reincarnate() re-execs this program in place after PID 1
|
||||||
|
// has died. A normal boot has no argument and is generation 1.
|
||||||
|
if (argc > 1) {
|
||||||
|
unsigned int g = 0;
|
||||||
|
for (char *q = argv[1]; *q >= '0' && *q <= '9'; q++)
|
||||||
|
g = g * 10 + (unsigned int)(*q - '0');
|
||||||
|
if (g)
|
||||||
|
generation = g;
|
||||||
|
}
|
||||||
|
|
||||||
bootstrap();
|
bootstrap();
|
||||||
|
|
||||||
out("schinit: SchOS process 1\n");
|
// A reincarnated PID 1 has the children but not the table.
|
||||||
|
//
|
||||||
|
// exec() replaced this program's memory and kept everything else -- the
|
||||||
|
// pid, the open files, and crucially the *children*, which are still
|
||||||
|
// parented here because nothing killed them. What did not survive is the
|
||||||
|
// array describing them, so it is read back from where it was published
|
||||||
|
// precisely so that it could be.
|
||||||
|
//
|
||||||
|
// Without this the next publish() overwrites the record with an empty
|
||||||
|
// table, the supervisor reads that, believes nothing is running and starts
|
||||||
|
// everything a second time. Same split brain as before, one level up.
|
||||||
|
if (generation > 1) {
|
||||||
|
// exec() keeps open files, which is what lets this program still have
|
||||||
|
// a console to complain to -- and also means the previous life's
|
||||||
|
// socket to the old supervisor is still open here. Leaving it open
|
||||||
|
// holds the old supervisor's channel up, so it never sees a hangup,
|
||||||
|
// never exits, and sits there reporting that PID 1 has stopped
|
||||||
|
// pinging it. Which is true, and no longer its business.
|
||||||
|
for (int fd = 3; fd < 32; fd++)
|
||||||
|
close(fd);
|
||||||
|
reclaim();
|
||||||
|
}
|
||||||
|
|
||||||
|
out("schinit: SchOS process 1");
|
||||||
|
if (generation > 1) {
|
||||||
|
out(", generation ");
|
||||||
|
outn((int)generation);
|
||||||
|
out(" (reincarnated)");
|
||||||
|
}
|
||||||
|
out("\n");
|
||||||
|
|
||||||
|
// Generation 4 is the safety net: a manifest that kills init must not kill
|
||||||
|
// it for ever, so this generation does not read /etc/schinit at all. One
|
||||||
|
// shell, on the console, and nothing else -- enough to log in and fix
|
||||||
|
// whatever has been taking the machine down.
|
||||||
|
if (generation >= 4) {
|
||||||
|
out("schinit: safe mode -- console shell only, /etc/schinit ignored\n");
|
||||||
|
if (fork() == 0) {
|
||||||
|
setsid();
|
||||||
|
ioctl(0, TIOCSCTTY, 0);
|
||||||
|
char *sh[] = { (char *)"-zsh", 0 };
|
||||||
|
execve((char *)"/bin/zsh", sh, 0);
|
||||||
|
char *fb[] = { (char *)"sh", 0 };
|
||||||
|
execve((char *)"/bin/sh", fb, 0);
|
||||||
|
exit();
|
||||||
|
}
|
||||||
|
publish();
|
||||||
|
for (;;) {
|
||||||
|
int st, pid;
|
||||||
|
while ((pid = waitpid(-1, &st, WNOHANG)) > 0)
|
||||||
|
;
|
||||||
|
sleep(20);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (start_supervisor() < 0) {
|
if (start_supervisor() < 0) {
|
||||||
// No supervisor. Rather than idle usefully-looking, fall back to a
|
// No supervisor. Rather than idle usefully-looking, fall back to a
|
||||||
|
|
@ -485,6 +610,48 @@ main(void)
|
||||||
sleep(10);
|
sleep(10);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The heartbeat. A supervisor that has stopped answering is stuck,
|
||||||
|
// and a stuck supervisor is indistinguishable from a working one by
|
||||||
|
// any means except asking it something -- which is the whole reason
|
||||||
|
// this exists. See docs/plan/init.md part 3.
|
||||||
|
if (sock >= 0) {
|
||||||
|
int now = uptime();
|
||||||
|
if (now - last_ping >= SCHINIT_PING_SECS * 100) {
|
||||||
|
struct schinit_hello p;
|
||||||
|
p.op = SCHINIT_PING;
|
||||||
|
p.proto = SCHINIT_PROTO;
|
||||||
|
p.generation = generation;
|
||||||
|
p.reserved = 0;
|
||||||
|
write(sock, &p, sizeof(p));
|
||||||
|
last_ping = now;
|
||||||
|
}
|
||||||
|
if (now - last_pong >= SCHINIT_DEADLINE_SECS * 100) {
|
||||||
|
out("schinit: supervisor stopped answering; replacing it\n");
|
||||||
|
|
||||||
|
// Ask it to die, then replace it *without waiting*.
|
||||||
|
//
|
||||||
|
// kill() sets a flag the target checks the next time it runs,
|
||||||
|
// and the whole premise here is that it is not running. A
|
||||||
|
// supervisor stuck in a loop will notice at its next trap; one
|
||||||
|
// blocked forever never will. Waiting for a corpse that may
|
||||||
|
// never be produced is how a recovery path becomes a second
|
||||||
|
// hang -- which is exactly what the first version of this did.
|
||||||
|
//
|
||||||
|
// Two supervisors briefly existing is safe, and safe in the
|
||||||
|
// way that matters: the old one's socket is closed here, so
|
||||||
|
// the moment it runs again its first write fails and it exits.
|
||||||
|
// It cannot spawn anything, because it cannot ask.
|
||||||
|
kill(supervisor_pid);
|
||||||
|
close(sock);
|
||||||
|
sock = -1;
|
||||||
|
int old = supervisor_pid;
|
||||||
|
if (start_supervisor() > 0)
|
||||||
|
out("schinit: new supervisor started\n");
|
||||||
|
publish(); // the table names the live supervisor
|
||||||
|
(void)old;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
int status;
|
int status;
|
||||||
int pid;
|
int pid;
|
||||||
while ((pid = waitpid(-1, &status, WNOHANG)) > 0) {
|
while ((pid = waitpid(-1, &status, WNOHANG)) > 0) {
|
||||||
|
|
|
||||||
|
|
@ -590,18 +590,72 @@ main(void)
|
||||||
|
|
||||||
out("schinitd: ready\n");
|
out("schinitd: ready\n");
|
||||||
|
|
||||||
// Everything from here is PID 1 telling us a child exited. There is no
|
// From here on this program does two things: answer PID 1's heartbeat, and
|
||||||
// wait() in this program and there cannot be: the children are not ours.
|
// act on the exits it reports. There is no wait() in this program and
|
||||||
|
// there cannot be -- the children are not ours.
|
||||||
|
//
|
||||||
|
// The read is polled rather than blocking, so that a PID 1 which has
|
||||||
|
// stopped talking is noticed. A blocking read would sit there forever
|
||||||
|
// looking exactly like a quiet machine, which is the failure mode this
|
||||||
|
// whole heartbeat exists to make visible.
|
||||||
|
int last_ping = uptime();
|
||||||
|
|
||||||
for (;;) {
|
for (;;) {
|
||||||
struct schinit_reply r;
|
struct pollfd pf;
|
||||||
int n = read(sock, &r, sizeof(r));
|
pf.fd = sock;
|
||||||
if (n != sizeof(r)) {
|
pf.events = POLLIN;
|
||||||
// PID 1 closed the channel. It is the only process that cannot
|
pf.revents = 0;
|
||||||
// have gone away, so this means we are being replaced.
|
|
||||||
out("schinitd: channel closed\n");
|
int pr = poll(&pf, 1, 1000);
|
||||||
|
|
||||||
|
// The channel is gone. This happens to the *old* supervisor when PID 1
|
||||||
|
// has replaced it: PID 1 closes its end, so poll reports a hangup and
|
||||||
|
// never reports readable again. Without this the replaced supervisor
|
||||||
|
// sits in the loop for ever, complaining that PID 1 has stopped
|
||||||
|
// pinging -- which it has, because it is no longer talking to us.
|
||||||
|
if (pr > 0 && (pf.revents & (POLLHUP | POLLERR | POLLNVAL))) {
|
||||||
|
out("schinitd: channel closed; this supervisor has been replaced\n");
|
||||||
exit();
|
exit();
|
||||||
}
|
}
|
||||||
if (r.op == SCHINIT_EXITED)
|
|
||||||
note_exit(r.cookie, r.pid, r.status);
|
if (pr > 0 && (pf.revents & POLLIN)) {
|
||||||
|
union schinit_msg m;
|
||||||
|
int n = read(sock, &m, sizeof(m));
|
||||||
|
if (n <= 0) {
|
||||||
|
// PID 1 closed the channel. It is the one process that cannot
|
||||||
|
// have gone away, so this means we are being replaced.
|
||||||
|
out("schinitd: channel closed\n");
|
||||||
|
exit();
|
||||||
|
}
|
||||||
|
if (m.op == SCHINIT_PING) {
|
||||||
|
struct schinit_hello p;
|
||||||
|
p.op = SCHINIT_PONG;
|
||||||
|
p.proto = SCHINIT_PROTO;
|
||||||
|
p.generation = 0;
|
||||||
|
p.reserved = 0;
|
||||||
|
write(sock, &p, sizeof(p));
|
||||||
|
last_ping = uptime();
|
||||||
|
} else if (m.op == SCHINIT_EXITED) {
|
||||||
|
note_exit(m.reply.cookie, m.reply.pid, m.reply.status);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Silence from PID 1.
|
||||||
|
//
|
||||||
|
// There is nothing this program can do about it -- PID 1 cannot be
|
||||||
|
// killed and cannot be replaced from here -- so it says so, loudly,
|
||||||
|
// once. docs/plan/init.md is explicit that this is the limit: a hang
|
||||||
|
// in PID 1 is only caught by noticing, and if both ends are wedged at
|
||||||
|
// once nothing catches it. There is no hardware watchdog.
|
||||||
|
//
|
||||||
|
// Saying so is still worth doing. The alternative is a machine that
|
||||||
|
// looks idle while its init is stuck, which is how this class of bug
|
||||||
|
// stays invisible for months.
|
||||||
|
static int complained;
|
||||||
|
if (!complained && uptime() - last_ping >= SCHINIT_DEADLINE_SECS * 100) {
|
||||||
|
out("schinitd: pid 1 has not pinged for "
|
||||||
|
"the deadline -- it may be wedged\n");
|
||||||
|
complained = 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -143,7 +143,20 @@ silent loop. The crash screen is the correct place for this to arrive.
|
||||||
|
|
||||||
## Milestones
|
## Milestones
|
||||||
|
|
||||||
**I0 and I1 are built.** `schinit` is 520 lines with no parser and no
|
**All four milestones are built.** `sysrq w` wedges the supervisor and PID 1
|
||||||
|
replaces it; `sysrq i` kills PID 1 and the kernel refills the role:
|
||||||
|
|
||||||
|
```
|
||||||
|
init: pid 1 exited with status 9 -- reincarnating, generation 2
|
||||||
|
schinit: reclaimed 2 running job(s) from /proc/schinit
|
||||||
|
schinit: SchOS process 1, generation 2 (reincarnated)
|
||||||
|
schinitd: adopting os.schos.console-shell
|
||||||
|
schinitd: adopting os.schos.tty2-shell
|
||||||
|
```
|
||||||
|
|
||||||
|
The shell being typed into keeps its pid across the death and rebirth of PID 1.
|
||||||
|
|
||||||
|
**I0 and I1 first.** `schinit` is 520 lines with no parser and no
|
||||||
allocation; `schinitd` holds the manifests. `kill 3` produces:
|
allocation; `schinitd` holds the manifests. `kill 3` produces:
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
@ -168,8 +181,8 @@ believing it.
|
||||||
|---|---|
|
|---|---|
|
||||||
| **I0** ✅ | Split the program. PID 1 supervises one child; `schinitd` does everything else. Jobs still forked by PID 1. |
|
| **I0** ✅ | Split the program. PID 1 supervises one child; `schinitd` does everything else. Jobs still forked by PID 1. |
|
||||||
| **I1** ✅ | `/proc/schinit`, written and read back. Kill `schinitd` by hand and watch it return without disturbing a single job. |
|
| **I1** ✅ | `/proc/schinit`, written and read back. Kill `schinitd` by hand and watch it return without disturbing a single job. |
|
||||||
| **I2** | The heartbeat both ways, with a `sysrq` key that wedges `schinitd` on purpose so the timeout path is exercised rather than assumed. |
|
| **I2** ✅ | The heartbeat both ways, with a `sysrq` key that wedges `schinitd` on purpose so the timeout path is exercised rather than assumed. |
|
||||||
| **I3** | `proc_reincarnate()` and the ladder, with a `sysrq` key that kills PID 1 on purpose. |
|
| **I3** ✅ | `proc_reincarnate()` and the ladder, with a `sysrq` key that kills PID 1 on purpose. |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,10 @@ extern uint64 tsc_hz;
|
||||||
extern uint64 boot_tsc;
|
extern uint64 boot_tsc;
|
||||||
void monotonic(uint64 *sec, uint64 *usec);
|
void monotonic(uint64 *sec, uint64 *usec);
|
||||||
int proc_write_schinit(int user_src, uint64 src, uint n);
|
int proc_write_schinit(int user_src, uint64 src, uint n);
|
||||||
|
int proc_wedge(const char *name);
|
||||||
|
int proc_reincarnate(int status);
|
||||||
|
void proc_kill_init(void);
|
||||||
|
extern uint init_generation;
|
||||||
void cprintf(char*, ...);
|
void cprintf(char*, ...);
|
||||||
|
|
||||||
// A log line: the serial port only, never the display.
|
// A log line: the serial port only, never the display.
|
||||||
|
|
|
||||||
|
|
@ -249,6 +249,9 @@ struct proc {
|
||||||
// it; /dev/tty resolves through it. Was impossible to have before there was
|
// it; /dev/tty resolves through it. Was impossible to have before there was
|
||||||
// more than one terminal to choose between.
|
// more than one terminal to choose between.
|
||||||
int ctty;
|
int ctty;
|
||||||
|
// Never scheduled again. Set by sysrq 'w' to produce a hang on purpose --
|
||||||
|
// see proc_wedge() and docs/plan/init.md part 3.
|
||||||
|
int wedged;
|
||||||
|
|
||||||
// Real/effective/saved user and group ids. Nothing enforces permissions with
|
// Real/effective/saved user and group ids. Nothing enforces permissions with
|
||||||
// them yet (the on-disk format has no owner field until SchFS), but they
|
// them yet (the on-disk format has no owner field until SchFS), but they
|
||||||
|
|
|
||||||
|
|
@ -42,12 +42,33 @@
|
||||||
// supervisor -> PID 1
|
// supervisor -> PID 1
|
||||||
#define SCHINIT_HELLO 1 // struct schinit_hello
|
#define SCHINIT_HELLO 1 // struct schinit_hello
|
||||||
#define SCHINIT_SPAWN 2 // struct schinit_spawn
|
#define SCHINIT_SPAWN 2 // struct schinit_spawn
|
||||||
#define SCHINIT_SIGNAL 3 // struct schinit_signal
|
#define SCHINIT_PONG 3 // struct schinit_hello -- answer to a PING
|
||||||
|
|
||||||
// PID 1 -> supervisor
|
// PID 1 -> supervisor
|
||||||
#define SCHINIT_WELCOME 128 // struct schinit_hello, echoed with PID 1's version
|
#define SCHINIT_WELCOME 128 // struct schinit_hello, echoed with PID 1's version
|
||||||
#define SCHINIT_SPAWNED 129 // struct schinit_reply -- pid, or negative on failure
|
#define SCHINIT_SPAWNED 129 // struct schinit_reply -- pid, or negative on failure
|
||||||
#define SCHINIT_EXITED 130 // struct schinit_reply -- a child has been reaped
|
#define SCHINIT_EXITED 130 // struct schinit_reply -- a child has been reaped
|
||||||
|
#define SCHINIT_PING 131 // struct schinit_hello -- answer within the deadline
|
||||||
|
|
||||||
|
// The heartbeat.
|
||||||
|
//
|
||||||
|
// docs/plan/init.md part 3, and the failure the two-init scheme could not see:
|
||||||
|
// **the common way an init breaks is not dying, it is hanging.** wait() never
|
||||||
|
// returns for a process that is merely stuck, so a supervisor watching a
|
||||||
|
// wedged PID 1 -- or PID 1 watching a wedged supervisor -- reports everything
|
||||||
|
// normal forever.
|
||||||
|
//
|
||||||
|
// So each must be heard from. PID 1 sends a PING every interval and expects a
|
||||||
|
// PONG; silence past the deadline means the other end is stuck rather than
|
||||||
|
// busy, and stuck is the case worth acting on.
|
||||||
|
//
|
||||||
|
// Measured in seconds of uptime(), which counts ticks. Ticks can be *lost*
|
||||||
|
// when interrupts are off too long -- see kernel/core/time.c -- which makes a
|
||||||
|
// tick-measured deadline longer in real time than it looks, never shorter.
|
||||||
|
// That is the safe direction: a machine under load waits longer before
|
||||||
|
// declaring a hang, rather than declaring one that has not happened.
|
||||||
|
#define SCHINIT_PING_SECS 2
|
||||||
|
#define SCHINIT_DEADLINE_SECS 10
|
||||||
|
|
||||||
struct schinit_hello {
|
struct schinit_hello {
|
||||||
unsigned int op;
|
unsigned int op;
|
||||||
|
|
|
||||||
|
|
@ -488,10 +488,37 @@ proc_sysrq_cmd(char cmd)
|
||||||
case 't':
|
case 't':
|
||||||
text_selftest();
|
text_selftest();
|
||||||
break;
|
break;
|
||||||
|
case 'i':
|
||||||
|
// Kill PID 1, on purpose.
|
||||||
|
//
|
||||||
|
// Signals cannot do this -- init_ignores() drops them, which is correct
|
||||||
|
// and is why the two-init scheme was solving a problem that did not exist.
|
||||||
|
// Setting `killed` directly is the kernel deciding, not userspace asking:
|
||||||
|
// init exits at its next trap return and proc_reincarnate() takes over.
|
||||||
|
//
|
||||||
|
// docs/plan/init.md asks for this to be "tested by a sysrq key rather than
|
||||||
|
// by reasoning", which is the only honest way to know the path works.
|
||||||
|
proc_kill_init();
|
||||||
|
cprintf("sysrq: pid 1 asked to exit -- it should come back\n");
|
||||||
|
break;
|
||||||
|
|
||||||
|
case 'w': {
|
||||||
|
// Wedge the supervisor: make schinitd hang, on purpose, so PID 1's
|
||||||
|
// heartbeat timeout is exercised instead of reasoned about. There is no
|
||||||
|
// way back from this; reboot afterwards.
|
||||||
|
int pid = proc_wedge("schinitd");
|
||||||
|
if (pid)
|
||||||
|
cprintf("sysrq: wedged schinitd (pid %d) -- pid 1 should notice "
|
||||||
|
"within %d seconds\n", pid, 10);
|
||||||
|
else
|
||||||
|
cprintf("sysrq: no process named schinitd to wedge\n");
|
||||||
|
break;
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
cprintf("sysrq: unknown command '%c' "
|
cprintf("sysrq: unknown command '%c' "
|
||||||
"(known: c = crash, u = dump USB, s = read the USB disk, "
|
"(known: c = crash, u = dump USB, s = read the USB disk, "
|
||||||
"d = display self-test)\n",
|
"d = display self-test, t = text self-test, n = net self-test, "
|
||||||
|
"w = wedge the init supervisor, i = kill pid 1)\n",
|
||||||
cmd);
|
cmd);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -267,14 +267,113 @@ fork(void)
|
||||||
// status is already wait(2)-encoded -- see W_EXITCODE() in proc.h. Callers
|
// status is already wait(2)-encoded -- see W_EXITCODE() in proc.h. Callers
|
||||||
// that die from a fault or signal pass W_EXITCODE(0, sig); a normal _exit(n)
|
// that die from a fault or signal pass W_EXITCODE(0, sig); a normal _exit(n)
|
||||||
// passes W_EXITCODE(n, 0).
|
// passes W_EXITCODE(n, 0).
|
||||||
|
// PID 1 is a role, and the kernel guarantees the role is filled.
|
||||||
|
//
|
||||||
|
// This is the rule change docs/plan/init.md exists for. Unix's rule is "PID 1
|
||||||
|
// is special and must never die", honoured since 1975 by being careful --
|
||||||
|
// which is not a mechanism, it is a hope, and the thing it hopes about is the
|
||||||
|
// one program on the machine with no supervisor.
|
||||||
|
//
|
||||||
|
// So instead: tear nothing down, keep pid 1, and re-exec in place.
|
||||||
|
//
|
||||||
|
// The reason this is so short is the reason it is the right design. exec()
|
||||||
|
// already replaces the address space, and everything else is *deliberately*
|
||||||
|
// left alone:
|
||||||
|
//
|
||||||
|
// - the pid stays 1, so every orphan the kernel reparented to initproc is
|
||||||
|
// still correctly parented and initproc is still the pointer it was
|
||||||
|
// - the children survive, because nothing killed them
|
||||||
|
// - the open files survive, so the new image still has a console to
|
||||||
|
// complain to -- this runs before exit() closes anything, which is why
|
||||||
|
// the call site is the first line of exit() and not the last
|
||||||
|
// - the state survives, in /proc/schinit, which is the whole reason the
|
||||||
|
// job table lives outside the process holding it
|
||||||
|
//
|
||||||
|
// A sibling init resurrecting a dead one would have to solve all four of those
|
||||||
|
// and could not solve the first at all. This solves them by not creating them.
|
||||||
|
//
|
||||||
|
// The escalation ladder is what makes it safe to have. A machine that cannot
|
||||||
|
// hold PID 1 up is not a machine that should keep trying for ever, and PSOGM
|
||||||
|
// renders a stop code, a call chain and a QR code of the whole report -- which
|
||||||
|
// is a far better end than a silent loop.
|
||||||
|
//
|
||||||
|
// generation 1 normal boot
|
||||||
|
// generation 2-3 re-exec, keep the jobs, carry on
|
||||||
|
// generation 4 re-exec, but ignore /etc/schinit entirely and start only
|
||||||
|
// a console shell: a manifest that kills init must not
|
||||||
|
// kill it for ever
|
||||||
|
// generation 5 stop
|
||||||
|
uint init_generation = 1;
|
||||||
|
|
||||||
|
int
|
||||||
|
proc_reincarnate(int status)
|
||||||
|
{
|
||||||
|
static char gen[12];
|
||||||
|
char *argv[3];
|
||||||
|
int n = 0, v;
|
||||||
|
|
||||||
|
init_generation++;
|
||||||
|
|
||||||
|
if (init_generation > 5)
|
||||||
|
panic("init resurrection failed: five generations is enough");
|
||||||
|
|
||||||
|
// Decimal, by hand: this runs in a process that is dying and should call as
|
||||||
|
// little as possible.
|
||||||
|
v = (int)init_generation;
|
||||||
|
if (v == 0) {
|
||||||
|
gen[n++] = '0';
|
||||||
|
} else {
|
||||||
|
char tmp[12];
|
||||||
|
int i = 0;
|
||||||
|
while (v > 0) { tmp[i++] = '0' + (v % 10); v /= 10; }
|
||||||
|
while (i-- > 0) gen[n++] = tmp[i];
|
||||||
|
}
|
||||||
|
gen[n] = 0;
|
||||||
|
|
||||||
|
cprintf("init: pid 1 exited with status %d -- reincarnating, generation %d\n",
|
||||||
|
status, init_generation);
|
||||||
|
if (init_generation >= 4)
|
||||||
|
cprintf("init: generation %d starts a console shell only, "
|
||||||
|
"ignoring /etc/schinit\n", init_generation);
|
||||||
|
|
||||||
|
argv[0] = "schinit";
|
||||||
|
argv[1] = gen;
|
||||||
|
argv[2] = 0;
|
||||||
|
|
||||||
|
// Clear `killed` first.
|
||||||
|
//
|
||||||
|
// exit() was reached from the trap-return path *because* killed was set, and
|
||||||
|
// that path checks it again on the way back out. Leaving it set would exit
|
||||||
|
// the brand-new image before its first instruction, and the generation
|
||||||
|
// counter would climb to five in a few microseconds -- a resurrection loop
|
||||||
|
// that looks exactly like the crash it was meant to survive.
|
||||||
|
myproc()->killed = 0;
|
||||||
|
|
||||||
|
// No lock is held here and none may be taken: this is the first line of
|
||||||
|
// exit(), before the file table is walked and before ptable.lock. exec()
|
||||||
|
// does its own begin_op/end_op and is safe from process context.
|
||||||
|
if (exec("/bin/schinit", argv, 0) < 0)
|
||||||
|
return -1;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
exit(int status)
|
exit(int status)
|
||||||
{
|
{
|
||||||
struct proc *p;
|
struct proc *p;
|
||||||
int fd;
|
int fd;
|
||||||
|
|
||||||
if(myproc() == initproc)
|
if(myproc() == initproc) {
|
||||||
panic("init exiting");
|
// Returning from exit() is the *success* path here. exec() does not fail
|
||||||
|
// to return -- it returns 0 having replaced the address space, and the new
|
||||||
|
// image begins running when the kernel returns to userspace through the
|
||||||
|
// trapframe it rewrote. So there is nothing more to do but go back.
|
||||||
|
if (proc_reincarnate(status) == 0)
|
||||||
|
return;
|
||||||
|
// Only reached if the re-exec itself failed, which is the one case where
|
||||||
|
// there is genuinely nothing left to try.
|
||||||
|
panic("init resurrection failed: could not exec /bin/schinit");
|
||||||
|
}
|
||||||
|
|
||||||
myproc()->xstate = status;
|
myproc()->xstate = status;
|
||||||
|
|
||||||
|
|
@ -408,6 +507,38 @@ waitpid(int pid, int *status, int options)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stop a process running, by name, and never start it again.
|
||||||
|
//
|
||||||
|
// This exists for one reason: docs/plan/init.md part 3 says the heartbeat's
|
||||||
|
// timeout path should be "exercised rather than assumed", and a hang is the
|
||||||
|
// one failure that cannot be produced by killing something. A wedged process
|
||||||
|
// is still alive, still has its descriptors open, and still shows up in every
|
||||||
|
// listing -- which is exactly what makes it hard to notice and exactly what
|
||||||
|
// the heartbeat is for.
|
||||||
|
//
|
||||||
|
// Deliberately one-way. There is no unwedge: this is a test instrument for a
|
||||||
|
// machine that is about to be rebooted, not a job-control mechanism. Real
|
||||||
|
// SIGSTOP/SIGCONT is a separate thing and is not implemented.
|
||||||
|
int
|
||||||
|
proc_wedge(const char *name)
|
||||||
|
{
|
||||||
|
struct proc *p;
|
||||||
|
int found = 0;
|
||||||
|
|
||||||
|
acquire(&ptable.lock);
|
||||||
|
for (p = ptable.proc; p < &ptable.proc[NPROC]; p++) {
|
||||||
|
if (p->state == UNUSED || p->pid <= 1)
|
||||||
|
continue;
|
||||||
|
if (strncmp(p->name, name, sizeof(p->name)) != 0)
|
||||||
|
continue;
|
||||||
|
p->wedged = 1;
|
||||||
|
found = p->pid;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
release(&ptable.lock);
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
scheduler(void)
|
scheduler(void)
|
||||||
{
|
{
|
||||||
|
|
@ -417,7 +548,10 @@ scheduler(void)
|
||||||
sti();
|
sti();
|
||||||
acquire(&ptable.lock);
|
acquire(&ptable.lock);
|
||||||
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
|
for(p = ptable.proc; p < &ptable.proc[NPROC]; p++){
|
||||||
if(p->state != RUNNABLE) {
|
// A wedged process is passed over for ever. It is still RUNNABLE, still
|
||||||
|
// owns its descriptors and still appears in every listing -- which is
|
||||||
|
// the point: it looks exactly like a process that is merely busy.
|
||||||
|
if(p->state != RUNNABLE || p->wedged) {
|
||||||
skipped++;
|
skipped++;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
@ -923,4 +1057,16 @@ int proc_exists(int pid)
|
||||||
}
|
}
|
||||||
release(&ptable.lock);
|
release(&ptable.lock);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
// Make PID 1 exit at its next trap return, so proc_reincarnate() runs.
|
||||||
|
//
|
||||||
|
// Not a signal: init_ignores() drops signals to initproc and should, because
|
||||||
|
// init is not killable from userspace on any Unix. This is the kernel
|
||||||
|
// deciding on its own behalf, which is a different thing, and it exists so
|
||||||
|
// the resurrection path can be exercised rather than reasoned about.
|
||||||
|
void
|
||||||
|
proc_kill_init(void)
|
||||||
|
{
|
||||||
|
if (initproc)
|
||||||
|
initproc->killed = 1;
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue