ldls: a second driver, and everything it found

e1000 -- Linux's, unmodified -- probes, opens, gets an address by DHCP and carries ping traffic in a SchOS process. The kernel was driving that NIC and gave it up, which makes it the first real handover as well as the second driver. None of what it needed was a missing feature. Every one was a call that succeeded, returned zero and did nothing: _find_next_bit "bit 0 is set" -- for_each_cpu never ended netif_carrier_off carrier already up -- TCTL.EN never set __x86_indirect_thunk_r8 the jump discarded -- 255 rx buffers, none made dma_map_page_attrs bus address 0 -- NIC aimed at physical 0 so the rule from memcpy is now general: a stub returning zero is unsafe wherever zero is a meaningful answer. The loader grew ko_fixup_percpu(). The stack guard turned out to be one per-CPU variable among several; `current` cannot be defused because the driver dereferences it. Taking the %gs off instead leaves the same opcode reading the same width from storage the host supplies -- a per-CPU variable in a single-threaded host is just a variable. Eight bytes become seven and a nop, and the displacement is not always last, so there is now just enough of an x86 decoder in ko.c to know how long the instruction is. ldls/kpi/thunk.c is fifteen lines of assembly and the highest-value file here: retpolines are control flow, and a stub cannot stand in for a jump. The kernel side: - ioperm.c, the TSS I/O permission bitmap. A driver's outl is an instruction, not a call, and e1000's reset path picks it deliberately. IOPL=3 would hand over the PIC and the disk with it, so it is per-port. Writing it found that the I/O map base had been written at offset 64 rather than 102, leaving the CPU checking ports 0-831 against the TSS's own RSP0. - claim masks the device's IRQ line: 450,000 a second, otherwise. - net_detach(), the missing half of net_attach_late(). An interface left in the stack keeps its route and sends every frame to a driver that refuses it, which looks exactly like a userspace driver that does not work. - e1000_release() gives up the interface, not just the hardware. 8139too still passes 4/4; ldls-check still passes 27; four CPUs, no panic. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VQ7Qx8E1q1M1bbCq8QLHS2
2026-09-14 15:20:07 +02:00 · 2026-09-14 15:20:07 +02:00 · 071499a378
commit 071499a378
parent f38eb8a7fd
26 changed files with 2009 additions and 46 deletions
--- a/4
+++ b/4
@ -17,7 +17,7 @@ MAKEFLAGS += -j$(NPROC)
 # Kernel object files — organized by subsystem
 KERN_ARCH = $(addprefix $(BUILD)/kernel/arch/x86_64/, \
              acpi.o e820.o entry.o trapasm.o swtch.o \
-              trap.o vm.o lapic.o ioapic.o mp.o watch.o)
+              trap.o vm.o lapic.o ioapic.o mp.o watch.o ioperm.o)
 KERN_CORE = $(addprefix $(BUILD)/kernel/core/, \
              main.o kalloc.o dma.o spinlock.o sleeplock.o string.o console.o \
@ -803,7 +803,7 @@ $(BUILD)/lib/%.o: lib/%.S
 # stub.c are freestanding and build unchanged for either side, which is the
 # whole point of having written them that way.
 LDLS_GUEST = $(addprefix $(BUILD)/ldls/guest/, ko.o btf.o stub.o) \
-             $(addprefix $(BUILD)/ldls/guest/kpi/, kpi.o pci.o io.o net.o skb.o string.o)
+             $(addprefix $(BUILD)/ldls/guest/kpi/, kpi.o pci.o io.o net.o skb.o string.o work.o thunk.o)
 $(BUILD)/ldls/guest/%.o: ldls/%.c ldls/ko.h ldls/btf.h ldls/stub.h
 	@mkdir -p $(BUILD)/ldls/guest
--- a/cmd/ldlsd/main.c
+++ b/cmd/ldlsd/main.c
@ -216,6 +216,57 @@ resolve(void *ctx, const char *name, unsigned long long *out)
    return -1;
 }
 // --- per-CPU variables ---------------------------------------------------
 //
 // The stack guard was one of these; `current` is another, and every driver
 // touches it. ko_fixup_percpu() turns each `%gs:` reference into a `lea` of
 // memory we supply here, so a per-CPU variable becomes an ordinary one --
 // which is what it is, on a host with one thread pretending to be one CPU.
 //
 // The blobs are zeroed and generously sized, because we do not know what the
 // driver will read through them: `current` is a task_struct of some kilobytes,
 // and a driver that walks into it should get zeroes rather than a fault. Same
 // bargain as the logging stubs, and the same caveat -- see the memcpy note in
 // docs/architecture/ldls.md.
 #define PERCPU_SLOTS 8
 #define PERCPU_SIZE  (16 * 1024)
 static char percpu_mem[PERCPU_SLOTS][PERCPU_SIZE];
 static const char *percpu_name[PERCPU_SLOTS];
 static int percpu_n;
 static unsigned long long
 percpu_for(void *ctx, const char *name)
 {
    int i;
    (void)ctx;
    for (i = 0; i < percpu_n; i++)
        if (streq(percpu_name[i], name))
            return (unsigned long long)(void *)percpu_mem[i];
    if (percpu_n == PERCPU_SLOTS)
        return 0;
    i = percpu_n++;
    percpu_name[i] = name;
    // Zero is the right contents for a counter and for `cpu_number`, and the
    // wrong contents for `current`: a driver reads that one and dereferences
    // what it got. So the pointer-valued ones point at the rest of their own
    // slot, which is zeroes all the way down -- every field the driver reads
    // out of the task it thinks it has comes back 0, and anything it writes
    // lands somewhere harmless.
    if (streq(name, "const_current_task") || streq(name, "current_task")) {
        *(unsigned long long *)percpu_mem[i] =
            (unsigned long long)(void *)(percpu_mem[i] + 64);
        puts_("  per-CPU "); puts_(name); puts_(" -- a task of zeroes\n");
    } else {
        puts_("  per-CPU "); puts_(name); puts_(" -- 16K of zeroes\n");
    }
    return (unsigned long long)(void *)percpu_mem[i];
 }
 static unsigned long long
 find_symbol(struct ko *k, const char *want)
 {
@ -263,6 +314,15 @@ wr64(unsigned long base, unsigned int off, unsigned long v)
 // match is made against that rather than against anything LDLS knows -- which
 // is the property that keeps this general: nothing here has heard of an
 // rtl8139.
 // Which device, when more than one matches.
 //
 // The first driver tested was an rtl8139 in a machine whose only other NIC was
 // an e1000 the kernel was driving, so "the first that matches" was the same as
 // "the right one". With two e1000s it is not: taking the first means taking
 // the one the console's own network is on. Written as `bus:slot.func`, or -1
 // for "any", which is still the default.
 static int sel_bus = -1, sel_slot = -1, sel_func = -1;
 static int
 claim_for(unsigned long idtab, unsigned int id_size, unsigned int id_vendor,
          unsigned int id_device, struct pciuser_info *out)
@ -279,6 +339,11 @@ claim_for(unsigned long idtab, unsigned int id_size, unsigned int id_vendor,
        if (ioctl(kpi.pcifd, PCIIOC_INFO, &in) < 0)
            continue;
        if (sel_bus >= 0 && ((int)in.bus != sel_bus ||
                            (int)in.slot != sel_slot ||
                            (int)in.func != sel_func))
            continue;
        for (n = 0; n < 4096; n++) {
            unsigned long e = idtab + (unsigned long)n * id_size;
            unsigned int v = rd32(e, id_vendor), d = rd32(e, id_device);
@ -372,12 +437,35 @@ kpi_layout_from_btf(struct btf *b)
    GET(nd,  "real_num_tx_queues", kpi_l.nd_real_num_tx);
    GET(nd,  "state",          kpi_l.nd_state);
    GET(nd,  "flags",          kpi_l.nd_flags);
    {
        int dv = btf_find(b, "device", BTF_KIND_STRUCT);
        int ko = btf_find(b, "kobject", BTF_KIND_STRUCT);
        int pd = btf_find(b, "pci_dev", BTF_KIND_STRUCT);
        if (dv && ko && pd) {
            GET(pd, "dev",  kpi_l.pd_dev);
            GET(dv, "kobj", kpi_l.dev_kobj);
            GET(ko, "name", kpi_l.kobj_name);
        }
    }
    {
        int nq = btf_find(b, "netdev_queue", BTF_KIND_STRUCT);
        if (!nq) { puts_("ldlsd: no struct netdev_queue\n"); return -1; }
        kpi_l.nq_size = btf_type_size(b, nq);
        GET(nq, "dev", kpi_l.nq_dev);
    }
    {
        long long v = 0;
        if (btf_enum_value(b, "__LINK_STATE_NOCARRIER", &v) < 0) {
            puts_("ldlsd: this BTF has no __LINK_STATE_NOCARRIER\n");
            return -1;
        }
        kpi_l.link_nocarrier = (unsigned int)v;
    }
    {
        int wk = btf_find(b, "work_struct", BTF_KIND_STRUCT);
        if (!wk) { puts_("ldlsd: no struct work_struct\n"); return -1; }
        GET(wk, "func", kpi_l.work_func);
    }
    GET(ops, "ndo_open",       kpi_l.ndo_open);
    GET(ops, "ndo_stop",       kpi_l.ndo_stop);
    GET(ops, "ndo_start_xmit", kpi_l.ndo_start_xmit);
@ -439,6 +527,10 @@ run_interface(struct btf *b, void *ndev)
        d[kpi_l.nd_flags] |= 1;                 /* IFF_UP             */
    }
    // Each phase gets its own first forty. A driver makes thousands of stub
    // calls during probe, so without this the interesting ones -- the last
    // thing it did before it stopped -- are always past the cap.
    g_calls = 0;
    puts_("\n  calling ndo_open\n\n");
    {
        int (*op)(void *) = (int (*)(void *))open_fn;
@ -472,6 +564,18 @@ run_interface(struct btf *b, void *ndev)
            }
        }
        // The driver's own handler, as if the line had been asserted. Its
        // first act is to read the device's cause register, so calling it
        // with nothing pending costs one register read and returns IRQ_NONE.
        if (kpi_irq_poll())
            moved = 1;
        // Anything the driver asked to have done later. A watchdog belongs
        // here and not in a timer interrupt: it wants to read registers and
        // touch the rings, which is exactly what the rest of this loop does.
        if (kpi_work_run() > 0)
            moved = 1;
        // And anything the chip has. The driver's own poll does the work; all
        // this decides is when it gets to run.
        if (kpi_napi_poll) {
@ -483,7 +587,9 @@ run_interface(struct btf *b, void *ndev)
            if (++idle > 200) { sleep(1); idle = 0; }
        } else {
            idle = 0;
-            if (((tx + rx) % 64) == 0) {
+            // The first few, then every sixty-fourth. Bringing a driver up,
            // the question is always whether the *first* frame moved.
            if (tx + rx <= 4 || ((tx + rx) % 64) == 0) {
                puts_("  ldls0: tx "); putn(tx);
                puts_("  rx "); putn(rx); puts_("\n");
            }
@ -582,6 +688,32 @@ drive_probe(const char *btfpath)
        pdev = (unsigned long)getmem(dev_size);
        if (!pdev) { puts_("ldlsd: no memory for a pci_dev\n"); return; }
        // A name for the device.
        //
        // `pci_name(pdev)` is `pdev->dev.kobj.name`, and a driver copies it
        // into its interface's name during probe. Left null it is a fault
        // inside strnlen with nothing to say which field was missing.
        if (kpi_l.pd_dev && kpi_l.kobj_name) {
            static char nm[16];
            unsigned long q;
            int j = 0;
            nm[j++] = '0'; nm[j++] = '0'; nm[j++] = '0'; nm[j++] = '0';
            nm[j++] = ':';
            nm[j++] = "0123456789abcdef"[(in.bus >> 4) & 0xf];
            nm[j++] = "0123456789abcdef"[in.bus & 0xf];
            nm[j++] = ':';
            nm[j++] = "0123456789abcdef"[(in.slot >> 4) & 0xf];
            nm[j++] = "0123456789abcdef"[in.slot & 0xf];
            nm[j++] = '.';
            nm[j++] = "0123456789abcdef"[in.func & 0xf];
            nm[j]   = 0;
            q = (unsigned long)nm;
            for (int i = 0; i < 8; i++)
                ((unsigned char *)pdev)[kpi_l.pd_dev + kpi_l.dev_kobj +
                                        kpi_l.kobj_name + i] =
                    (unsigned char)(q >> (i * 8));
        }
        wr16(pdev, dev_vendor, in.vendor);
        wr16(pdev, dev_device, in.device);
        wr32(pdev, dev_devfn, (unsigned int)((in.slot << 3) | in.func));
@ -677,11 +809,34 @@ main(int argc, char *argv[])
    int nsec, i;
    if (argc < 2) {
-        puts_("usage: ldlsd <module.ko> [vmlinux-btf]\n");
+        puts_("usage: ldlsd <module.ko> [vmlinux-btf] [bus:slot.func]\n");
        puts_("  neither file ships with SchOS; both are yours.\n");
        exit();
    }
    // An optional bus address, to pick between devices that all match.
    if (argc > 3) {
        const char *a = argv[3];
        int f[3] = {0, 0, 0}, got = 0, v = 0, any = 0;
        for (;; a++) {
            int d = -1;
            if (*a >= '0' && *a <= '9')      d = *a - '0';
            else if (*a >= 'a' && *a <= 'f') d = *a - 'a' + 10;
            else if (*a >= 'A' && *a <= 'F') d = *a - 'A' + 10;
            if (d >= 0) { v = v * 16 + d; any = 1; continue; }
            if (any && got < 3) f[got++] = v;
            v = 0; any = 0;
            if (*a == 0) break;
        }
        if (got != 3) {
            puts_("ldlsd: a device is written bus:slot.func, as /dev/pci prints it\n");
            exit();
        }
        sel_bus = f[0]; sel_slot = f[1]; sel_func = f[2];
        puts_("  device  only "); puthexn(sel_bus, 2); puts_(":");
        puthexn(sel_slot, 2); puts_("."); puthexn(sel_func, 1); puts_("\n");
    }
    img = slurp(argv[1], &len);
    if (!img) { puts_("ldlsd: cannot read "); puts_(argv[1]); puts_("\n"); exit(); }
@ -738,6 +893,16 @@ main(int argc, char *argv[])
        if (g == 0) g = ko_defuse_gs(&k, "__stack_chk_guard");
        if (g > 0) { puts_("  defused "); putn(g); puts_(" %gs stack-guard load(s)\n"); }
    }
    // Whatever %gs references are left name real per-CPU variables, and those
    // cannot be defused -- the driver dereferences what it reads.
    {
        int p = ko_fixup_percpu(&k, percpu_for, 0);
        if (p < 0) {
            puts_("ldlsd: per-CPU: "); puts_(ko_err(&k)); puts_("\n");
            exit();
        }
        if (p > 0) { puts_("  rehomed "); putn(p); puts_(" %gs per-CPU reference(s)\n"); }
    }
    puts_("  loaded  "); putn(span); puts_(" bytes at "); puthex((unsigned long long)mem);
    puts_("\n");
--- a/docs/architecture/devices.md
+++ b/docs/architecture/devices.md
@ -73,8 +73,9 @@ What a driver needs from a kernel, and all it needs:
 | reach its registers | `mmap`, where the offset is a BAR's physical address |
 | memory it can hand to the device | `PCIIOC_DMA` |
 | be told when the device wants attention | `PCIIOC_IRQ`, then `read()` |
 | use its ports, if it has an I/O BAR | nothing — `CLAIM` grants them |
-Three of those are worth expanding on, because each encodes a decision:
+Four of those are worth expanding on, because each encodes a decision:
 **`mmap` is checked against the caller's claims.** An offset that is not inside
 a BAR of a device this process holds is refused. That is the whole difference
@ -105,6 +106,15 @@ happened here at 72,000 interrupts a second, and what it looked like from
 userspace was a clock running seven times slow. See §13 of
 [../history/kernel.md](../history/kernel.md).
 **Ports come with the claim, and only that device's.** There is no ioctl for
 this because there is no decision left to make: a process holding a device with
 an I/O BAR gets exactly the ports in that BAR, through the TSS I/O permission
 bitmap, and loses them when it releases or exits. Granting it is not optional —
 a driver's `outl` is an instruction compiled into it, not a call anything can
 intercept — and granting it *coarsely* is not acceptable, because `IOPL=3` is
 one flag that hands over the interrupt controller and the disk along with the
 NIC. See [../../include/ioperm.h](../../include/ioperm.h).
 ### Handing a device over
 `struct pci_driver` has a `release` hook. A driver that does not define one keeps
@ -122,6 +132,18 @@ console falls back to the VBE framebuffer.
 **No syscall on that path knows it is talking about a GPU.** The kernel hands
 over a PCI device; the process decides what it is.
 Two things a release has to do beyond quiescing the hardware, both found by
 handing over a NIC the kernel was actively using:
 - **Mask the line.** The kernel's handler is gone and the incoming driver is
  about to enable the device, so for a while nothing will acknowledge the
  cause. Left unmasked that is an interrupt storm — 450,000 a second, measured.
 - **Give up the class device too.** A NIC's `release` calls `net_detach()`, the
  missing half of `net_attach_late()`. An interface left in the stack keeps its
  address and the route to its subnet, so the stack goes on choosing it and
  every frame reaches a `transmit` that now correctly refuses. Nothing reports
  an error. See [ldls.md](ldls.md).
 It also **changes the display mode**, on a control socket, with `schmode` as the
 client — and that is worth a sentence here because it is the thing the device
 model makes possible rather than the thing it does. A modeset is five commands
--- a/docs/architecture/ldls.md
+++ b/docs/architecture/ldls.md
@ -4,8 +4,9 @@ SchOS cannot write a driver for every device that exists. No project can. The
 only route to the hardware support an established system has is to run Linux's
 drivers, and LDLS is what does that.
-It works. `8139too` — an unmodified GPL kernel module — loads into a SchOS
+It works, and on more than one driver. `8139too` and `e1000` — unmodified GPL
-process, drives QEMU's rtl8139 through `/dev/pci`, and carries packets:
+kernel modules — load into a SchOS process, drive their hardware through
 `/dev/pci`, and carry packets:
 ```
 # ldlsd /lib/modules/8139too.ko /lib/modules/btf &
@ -22,6 +23,23 @@ process, drives QEMU's rtl8139 through `/dev/pci`, and carries packets:
 `10.9.0.0/24` is reachable only through `ldls0`. The address on it came from
 DHCP, over the same driver.
 The e1000 run is the more interesting of the two, because it is a **handover**:
 the kernel was driving that NIC, and gave it up.
 ```
 net: eth1 is gone; interface 1 removed
 e1000: released to userspace; the interface is down
  rehomed 23 %gs per-CPU reference(s)
  calling probe, with 82 of 155 symbols implemented
  kpi: the driver read a MAC off the hardware: 52:54:00:12:34:57
 net: ldls0 attached as interface 2, asking for an address
 net: ldls0 10.9.0.15/255.255.255.0 via 10.9.0.2
 ```
 Everything the second driver needed that the first did not is below, and the
 pattern in it is worth naming up front: **none of it was a missing feature.**
 Every one was a call that succeeded, returned zero, and did nothing.
 ---
 ## The rules it is built on
@ -112,7 +130,7 @@ It is **freestanding**: no libc, no allocation, no I/O. The same source builds
 as a host tool, where a format failure costs a second, and as the SchOS-side
 host. That is the arrangement `mkschfs` and the kernel already share.
-#### Two things it has to fix in the object
+#### Three things it has to fix in the object
 **The `%gs` stack guard.** Modules are built `-mstack-protector-guard-reg=gs`,
 so the canary is fetched through the GS segment — an ABI between the kernel and
@ -124,6 +142,44 @@ sets ZF, so the branch falls through as a passing check does. It checks every
 site before patching any and refuses the whole operation on anything it does
 not recognise.
 **Every other per-CPU variable.** The guard turned out to be one instance of a
 general problem. `current` is another — `mov %gs:const_current_task(%rip), %r8`
 — and e1000 has 23 such references across four variables. Defusing does not
 work here: the guard's *value* is arbitrary, and `current`'s is not, because
 the driver dereferences what it reads.
 What does work is that **a per-CPU variable in a single-threaded host is just a
 variable**. `ko_fixup_percpu()` takes the `%gs` off, leaving the same opcode
 reading the same width in the same direction, from storage the host supplies
 per name. Eight bytes become seven and a `nop`. The displacement is measured
 from the end of the *shortened* instruction, and the displacement is not always
 last — `cmpb $0x0, %gs:softnet_data+0x75(%rip)` has an immediate after it — so
 there is just enough of an x86 decoder in there to know how long the
 instruction is. Anything it does not recognise is refused with the symbol
 named, which is rule 5 again.
 `ldlsd` hands out zeroed blobs, except for `current`, which points at the rest
 of its own: a driver that walks into the task it thinks it has reads zeroes.
 **The compiler's thunks.** `__x86_indirect_thunk_rN` and `__x86_return_thunk`
 are retpolines, not API, and they are pure control flow — `jmp *%r8` and `ret`.
 A stub cannot stand in for one, because a stub's whole behaviour is to log a
 name and return, and what a thunk is *for* is the jump. `ldls/kpi/thunk.c` is
 fifteen lines of assembly and is the single highest-value file in the layer.
 That one cost a day. e1000 refills its receive ring through a function pointer
 and the compiler made it a *tail* call:
 ```
 mov  0x2a0(%rbx),%r8          ; adapter->alloc_rx_buf
 ...                           ; compute E1000_DESC_UNUSED(ring) == 255
 jmp  __x86_indirect_thunk_r8
 ```
 so the driver asked for 255 receive buffers, the shim answered "yes", and the
 symptom was a NIC that transmitted perfectly and received nothing with `RDT`
 sitting at zero.
 **Where it is loaded.** `-mcmodel=kernel` means `R_X86_64_32S` relocations, so
 the object has to live where a signed 32-bit absolute address reaches. A
 process's mmap area starts at `0x40000000`, which is inside that window; on the
@ -141,6 +197,38 @@ returned. A stray access would otherwise be a page fault with no context, and
 with no IOMMU a hosted driver's wild writes have to be caught early or not at
 all.
 Three pieces of it are not register access and are worth naming, because each
 one is something a driver does that a poll loop does not obviously provide:
 - **The interrupt handler runs.** Not to receive packets — LDLS was already
  calling the NAPI poll by hand — but because a driver's handler is where it
  does everything *else*. e1000 queues its watchdog from there, and the
  watchdog is what sets `TCTL.EN`. A driver whose handler never ran was a
  driver whose transmitter was never switched on. Calling it from the loop is
  safe: a handler's first act is to read its cause register, and with nothing
  pending it returns `IRQ_NONE`.
 - **Deferred work runs.** `queue_delayed_work_on` keeps the item and
  `kpi_work_run()` calls what is due from the same loop. `jiffies` had to start
  moving for that to mean anything; it comes from `uptime()`, at 100 Hz.
 - **`netif_carrier_on`/`off` write the bit.** `netif_carrier_ok()` is an inline
  that tests `__LINK_STATE_NOCARRIER` in `net_device.state`, so a no-op means
  the driver reads "the carrier is already up" and skips its whole link-up
  path. The bit number comes from the kernel's BTF (`btf_enum_value()`), not
  from us — rule 3½ applied to a constant rather than an offset.
 And **DMA in small pieces**. `dma_alloc_coherent` maps onto `PCIIOC_DMA`
 exactly; `dma_map_single` does not. 8139too never needed it, because it copies
 each frame into a ring it allocated coherently up front; e1000 hands the device
 individual buffers. Since a process cannot know the physical address of an
 ordinary page, those buffers come from the arena too, a chunk at a time behind
 a bump pointer.
 The address arithmetic for that is not a call at all. `dma_map_single` is a
 static inline, so what reaches the shim is already expanded: the module
 computes `pa = va - page_offset_base`, then `page = vmemmap_base + (pa >> 12) *
 64`. Both bases are undefined *data* symbols — which means LDLS supplies them,
 and setting both to zero makes the whole thing reversible with no lookup table.
 ### The back end — `/dev/pci`
 Already built, for `vgpud`, before any of this existed.
@ -150,6 +238,51 @@ consumers are native drivers is a general mechanism rather than a hole shaped
 like something foreign. A Linux driver's `pci_iomap`, `dma_alloc_coherent` and
 `request_irq` land on it unaltered, which is that argument being collected.
 ### Giving a device up — `release`
 A driver in the kernel now has a `release` hook, and `/dev/pci` calls it before
 handing its device to a process. `docs/plan/hybrid.md` had said the honest
 answer to "may I have the one you are using" is *no* until a driver could be
 told to let go; e1000 is the first that can.
 Letting go is three things, and the third was missed at first:
 1. quiesce the hardware — mask interrupts, clear `RCTL` and `TCTL`;
 2. **mask the interrupt line.** The kernel's handler is gone and the device is
   about to be brought up by a driver in a process, so nothing will acknowledge
   the cause. The console said what that looks like: `irq 11: 449905
   interrupts/second -- a device on this line is not acknowledging it`. It
   comes back when the process asks for it with `PCIIOC_IRQ`.
 3. **take the interface out of the stack.** `net_detach()` is the missing half
   of `net_attach_late()`, and it went missing because nothing had ever taken a
   device back. An interface left behind keeps its address and the route to its
   subnet, so the stack goes on choosing it, and every frame reaches a
   `transmit` that now correctly refuses. Packets stop and nothing reports an
   error — which is indistinguishable from a driver in a process that does not
   work, and was mistaken for exactly that.
 ### Ports — the TSS I/O permission bitmap
 A Linux driver does not always ask. `ioread32` is a call the KPI can implement;
 `outl` is an *instruction* compiled into the module, and e1000's reset path
 picks it deliberately — "these controllers can't ack the 64-bit write when
 issuing the reset, so use IO-mapping as a workaround". On QEMU's 82540EM there
 is no route to a working NIC that avoids it.
 So a process holding a device gets that device's I/O BARs, **and no other
 ports**. x86 offers two ways to grant this and they are far apart: `IOPL=3` is
 one flag and hands over all 65536 ports, including the interrupt controller,
 the timer and the disk; the TSS I/O permission bitmap is per port. A foreign
 driver already has DMA with no IOMMU, and the answer to "this is not a security
 boundary" is not to move it further — so it is the bitmap, at 8 KB per CPU of
 resident memory. See [../../include/ioperm.h](../../include/ioperm.h).
 Writing it found a latent hole. The TSS's I/O map base is at offset 102 and the
 old code wrote offset 64; the field it meant to set stayed 0, and a base of 0
 with a limit of `0x67` does not mean "no bitmap" — it means the bitmap starts at
 the top of the TSS. The CPU had been checking ports 0–831 against the TSS's own
 bytes, which are `RSP0`: a kernel stack pointer whose zero bits are permission.
 ### The class bridge — `/dev/netdev`
 The one piece the kernel had to grow. `netdev_ops` is four entries —
@ -166,12 +299,16 @@ lwIP never learns that the thing behind the vtable is not a driver.
 **It polls.** A Linux driver expects its interrupt handler to be called from an
 interrupt. `/dev/pci` does deliver them — by blocking `read()`, which needs a
-thread `ldlsd` does not have — so the driver's NAPI poll is called from a loop
+thread `ldlsd` does not have — so the handler, the NAPI poll and the driver's
-instead. The same work, in a context LDLS can provide, at the cost of latency
+deferred work are all called from one loop instead. The same work, in a context
-and a spinning process.
+LDLS can provide, at the cost of latency and a spinning process.
-**The stack protector is gone**, per `ko_defuse_gs()` above. Foreign code, no
+**The stack protector is gone**, per `ko_defuse_gs()` above, and so are the
-IOMMU, one hardening measure fewer.
+retpolines, per `thunk.c`. Foreign code, no IOMMU, three hardening measures
 fewer.
 **Per-CPU means per-process.** Correct with one thread, and it is the same
 assumption as the no-op locks below, arrived at from the other side.
 **A lock is a no-op.** In one process with no preemption inside the driver
 there is nobody to race with. That is correct right up until a driver has an
@ -198,11 +335,20 @@ point of having built it against our own:
 ## What it does not do
- **One driver, one kernel version, tried once.** `8139too` on 6.18.46 against
+- **Two drivers, one kernel version, one machine.** `8139too` and `e1000` on
-  QEMU's rtl8139. Nothing here has been tried on a second driver, a second
+  6.18.46 against QEMU. The second driver was worth more than the first: it
-  kernel, or real silicon, and until it has, "LDLS runs Linux drivers" is a
+  found the per-CPU problem, the retpoline problem, the port-I/O problem, the
-  claim rather than a description.
+  interrupt storm and a seven-year-old hole in the TSS. Nothing here has been
  tried on a second kernel or on real silicon, and "LDLS runs Linux drivers" is
  still a claim about a category from a sample of two.
 - **No interrupts**, per the polling note above.
 - **`jiffies` is not `HZ`.** It comes from a 100 Hz `uptime()`, which is a
  plausible `HZ` and not necessarily the one the module was built for. A driver
  measuring a timeout against it gets a longer timeout, which is the safe
  direction and is still wrong.
 - **One work item at a time, on the main loop.** A driver whose deferred work
  blocks stops everything, including the transmit path it is probably waiting
  for.
 - **It is not a security boundary, and here that is worse than usual.** There
  is no IOMMU, so a hosted driver can point its device at any physical address.
  For SchOS's own drivers that isolates bugs and not intent; for *foreign*
@ -224,9 +370,19 @@ point of having built it against our own:
 - **No module-to-module dependencies, no firmware loading, no sysfs, no
  unload.** `request_firmware` fails, `module_exit` is never called, and the
  way to stop a driver is to kill the process.
- **Some symbols are still logging stubs.** Deliberate, per rule 5 — but a
+- **Some symbols are still logging stubs.** Deliberate, per rule 5 — and the
-  driver reaching one gets zero and carries on, and `memcpy` proved how quiet
+  single most expensive thing in this layer. **A stub that returns zero is not
-  that can be: it was a stub while the receive path was being brought up, so
+  safe when zero is a meaningful answer**, and the list of ways that bit is
-  every frame arrived as the right *length* of zeroes and nothing reported an
+  now long enough to be a rule rather than an anecdote:
-  error. **A stub that returns zero is not safe for a function whose whole job
+
-  is a side effect.**
+  | stub | what zero meant | what it looked like |
  |---|---|---|
  | `memcpy` | the copy did not happen | frames arrived as the right *length* of zeroes |
  | `_find_next_bit` | "bit 0 is set" | `for_each_cpu` never terminated; `ndo_open` hung |
  | `netif_carrier_off` | the carrier is already up | driver skipped its link-up path; `TCTL.EN` never set |
  | `__x86_indirect_thunk_r8` | the jump was discarded | 255 receive buffers requested, none allocated, `RDT` at 0 |
  | `dma_map_page_attrs` | bus address 0 | the NIC pointed at the bottom of physical memory |
  Not one of these reported an error, and only the last is a fault waiting to
  happen. A search function, a predicate, a jump and an address are all things
  for which zero is a specific answer rather than an absent one.
--- a/include/defs.h
+++ b/include/defs.h
@ -625,6 +625,8 @@ void            lwip_sys_init(void);
 void            lwip_sys_lock(void);
 void            lwip_sys_unlock(void);
 int             net_eth_attach(struct device *);
 int             net_eth_detach(struct device *);
 int             net_detach(struct device *);
 int             net_attach_late(struct device *);
 int             net_eth_poll(void);
 int             net_eth_count(void);
--- a/include/ioperm.h
+++ b/include/ioperm.h
@ -0,0 +1,61 @@
 #pragma once
 // Port I/O, from a process.
 //
 // A Linux driver does not always ask. `ioread32` is a function call LDLS can
 // implement, but `outl` is an *instruction*, compiled into the module, and the
 // first foreign driver to need one found this the hard way:
 //
 //   e1000_io_write+0xd:  out %eax,(%dx)      -> trap 13 err 0, rdx=0xc000
 //
 // That is not a driver doing something exotic. e1000's own reset path picks it
 // deliberately -- "these controllers can't ack the 64-bit write when issuing
 // the reset, so use IO-mapping as a workaround" -- so on QEMU's 82540EM there
 // is no route to a working NIC that avoids it. Any layer that runs somebody
 // else's drivers has to let a process reach the ports of the device it holds.
 //
 // **Which ports, and no others.** x86 offers two ways to grant this and they
 // are very far apart. IOPL=3 is one flag and hands over all 65536 ports -- the
 // interrupt controller, the timer, the disk. The TSS I/O permission bitmap is
 // per port, and the ports a process gets are exactly the I/O BARs of the PCI
 // devices it has claimed. The second costs a few hundred lines and is the only
 // one worth having: a foreign driver already has DMA with no IOMMU, and the
 // answer to "this is not a security boundary" is not to move it further.
 //
 // It is the same bargain as kpi_io_ok() in ldls/kpi/io.c, one level down and
 // enforced by hardware rather than by the driver's host being polite.
 #include "types.h"
 struct proc;
 // Ranges per process. A PCI device has at most six BARs and a process may hold
 // several devices; eight is past anything seen and small enough to scan.
 #define NIOPORT 8
 struct ioport_range {
  uint16 base;
  uint16 len;
 };
 // Grant / revoke. Called from /dev/pci when a process claims or releases a
 // device, and from exit() by way of pciuser_release_all().
 //
 // Granting a range twice is not an error -- a driver may claim, release and
 // claim again -- and revoking one that was never granted is not either.
 void ioperm_grant(struct proc *p, uint16 base, uint16 len);
 void ioperm_revoke_all(struct proc *p);
 // Install this process's ranges in the running CPU's TSS. Called from
 // switchuvm() with interrupts off.
 void ioperm_switch(struct proc *p);
 // The per-CPU TSS, which now has a bitmap attached to it. seginit() asks for
 // this CPU's and points the GDT descriptor at it.
 void *ioperm_tss(int cpu, uint64 *limit);
 // The same block, for everyone after seginit().
 void *ioperm_tss_of(int cpu);
 // Where RSP0 lives, so switchuvm() does not have to know the layout.
 void ioperm_set_rsp0(void *tss, addr_t stktop);
--- a/include/proc.h
+++ b/include/proc.h
@ -2,6 +2,7 @@
 #include "types.h"
 #include "traps.h"
 #include "ioperm.h"
 // Simultaneous mmap() regions per process.
 //
@ -192,6 +193,12 @@ struct proc {
  // makes it able to sleep and to be preempted like anything else.
  void (*kfn)(void);
  // Ports this process may use directly, from the I/O BARs of the PCI devices
  // it has claimed. Empty for everything that is not hosting a driver, which is
  // everything. See include/ioperm.h.
  struct ioport_range ioport[NIOPORT];
  int nioport;
  uint64 signal_mask;
  uint64 pending_signals;
  int alarm_ticks;
--- a/kernel/arch/x86_64/ioperm.c
+++ b/kernel/arch/x86_64/ioperm.c
@ -0,0 +1,165 @@
 // The TSS I/O permission bitmap. See include/ioperm.h for why.
 #include "types.h"
 #include "defs.h"
 #include "param.h"
 #include "mmu.h"
 #include "proc.h"
 #include "ioperm.h"
 // The 64-bit TSS, written out rather than indexed.
 //
 // The old code reached it as `uint *tss` and set `tss[16]` to put 0x68 in the
 // I/O map base -- but the I/O map base is at offset 102, which is the high half
 // of the dword at index 25, not 16. Index 16 is offset 64, in the middle of the
 // interrupt stack table.
 //
 // That was not harmless. The field it meant to write stayed 0, and a base of 0
 // with a limit of 0x67 does not mean "no bitmap" -- it means the bitmap starts
 // at the top of the TSS, so the CPU checked ports 0-831 against the TSS's own
 // bytes. Those bytes are RSP0: a kernel stack pointer, whose zero bits are
 // *permission*. Some low ports were reachable from user mode and had been
 // since the kernel was 64-bit, which nothing had any reason to notice.
 //
 // Naming the fields is what makes that visible, so they are named.
 struct taskstate64 {
  uint32 res0;
  uint64 rsp0;
  uint64 rsp1;
  uint64 rsp2;
  uint64 res1;
  uint64 ist[7];
  uint64 res2;
  uint16 res3;
  uint16 iomb;              // offset from the base of the TSS
 } __attribute__((packed));
 #define IOMAP_BITS  65536
 #define IOMAP_BYTES (IOMAP_BITS / 8)
 // The TSS and its bitmap have to be one object: `iomb` is a 16-bit offset from
 // the TSS base, and the TSS descriptor's limit is what cuts the bitmap off.
 // So they are declared together and there is no way for them to drift apart.
 //
 // 8 KB per CPU of permanently resident kernel memory is the cost of being able
 // to say "these ports and no others", and it is charged whether or not any
 // process ever uses it. A bitmap sized to the highest port actually granted
 // would be smaller and would have to be resized at runtime, under a descriptor
 // the CPU is using; this is the version that is obviously correct.
 struct tssblk {
  struct taskstate64 ts;
  uint8  map[IOMAP_BYTES];
  // Intel requires a byte of all-ones past the end, so an access that straddles
  // the final port still reads a deny bit rather than whatever follows.
  uint8  tail;
 } __attribute__((packed, aligned(16)));
 static struct tssblk tssblk[NCPU];
 // What each CPU currently has open. Kept per CPU rather than read back from the
 // process, because switching *away* has to close what was opened and the
 // process that opened it may be part-way through exit() by then.
 static struct {
  struct ioport_range r[NIOPORT];
  int n;
 } installed[NCPU];
 static void
 map_set(uint8 *map, uint16 base, uint16 len, int allow)
 {
  uint32 p, end = (uint32)base + len;
  if (end > IOMAP_BITS)
    end = IOMAP_BITS;
  for (p = base; p < end; p++) {
    if (allow)
      map[p >> 3] &= ~(1 << (p & 7));
    else
      map[p >> 3] |= (1 << (p & 7));
  }
 }
 void *
 ioperm_tss(int cpu, uint64 *limit)
 {
  struct tssblk *b = &tssblk[cpu];
  // Deny everything to start with. A zero bit is permission, so a zeroed
  // bitmap is an open machine -- this memset is not initialisation tidiness.
  memset(b->map, 0xFF, sizeof(b->map));
  b->tail = 0xFF;
  b->ts.iomb = (uint16)__builtin_offsetof(struct tssblk, map);
  installed[cpu].n = 0;
  // The limit is the offset of the last valid byte, and it has to include the
  // trailing 0xFF -- stopping one byte short is the difference between a
  // straddling access being denied and it reading the next object.
  *limit = (uint64)__builtin_offsetof(struct tssblk, tail);
  return b;
 }
 // The same block, without the reset. seginit() calls ioperm_tss() once per CPU
 // to build it; everything after that just wants the pointer.
 void *
 ioperm_tss_of(int cpu)
 {
  return &tssblk[cpu];
 }
 void
 ioperm_set_rsp0(void *tss, addr_t stktop)
 {
  ((struct taskstate64 *)tss)->rsp0 = (uint64)stktop;
 }
 void
 ioperm_grant(struct proc *p, uint16 base, uint16 len)
 {
  int i;
  if (len == 0)
    return;
  for (i = 0; i < p->nioport; i++)
    if (p->ioport[i].base == base && p->ioport[i].len == len)
      return;
  if (p->nioport >= NIOPORT)
    return;                     // refused quietly; the driver will fault loudly
  p->ioport[p->nioport].base = base;
  p->ioport[p->nioport].len  = len;
  p->nioport++;
 }
 void
 ioperm_revoke_all(struct proc *p)
 {
  p->nioport = 0;
  // The bitmap is not touched here. This can run on a CPU that is not the one
  // holding the process's ranges open, and the next switchuvm() on each CPU
  // closes what that CPU opened -- which is the only place the two can be made
  // to agree without an IPI.
 }
 void
 ioperm_switch(struct proc *p)
 {
  int c = cpunum();
  struct tssblk *b = &tssblk[c];
  int i;
  // Close whatever was open. Doing this unconditionally rather than diffing
  // against the incoming process costs a few hundred bit clears on a machine
  // where nobody uses ports at all -- which is `installed[c].n == 0`, and then
  // this loop does nothing.
  for (i = 0; i < installed[c].n; i++)
    map_set(b->map, installed[c].r[i].base, installed[c].r[i].len, 0);
  installed[c].n = 0;
  if (!p || p->nioport == 0)
    return;
  for (i = 0; i < p->nioport && i < NIOPORT; i++) {
    map_set(b->map, p->ioport[i].base, p->ioport[i].len, 1);
    installed[c].r[i] = p->ioport[i];
  }
  installed[c].n = i;
 }
--- a/kernel/arch/x86_64/vm.c
+++ b/kernel/arch/x86_64/vm.c
@ -6,6 +6,7 @@
 #include "memlayout.h"
 #include "mmu.h"
 #include "proc.h"
 #include "ioperm.h"
 int mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm);
 static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc);
@ -68,7 +69,7 @@ seginit(void)
 {
  struct segdesc *gdt;
  uint *tss;
-  uint64 addr;
+  uint64 addr, tsslim;
  void *local;
  struct cpu *c;
@ -91,8 +92,12 @@ seginit(void)
  }
  gdt = (struct segdesc*) local;
-  tss = (uint*) (((char*) local) + 1024);
+
-  tss[16] = 0x00680000;
+  // The TSS moved out of this page. It now carries an I/O permission bitmap
  // (include/ioperm.h) which does not fit here, and it has to be one object
  // with it. ioperm_tss() returns it and the descriptor limit that reaches the
  // end of the bitmap.
  tss = (uint*) ioperm_tss(cpunum(), &tsslim);
  kernel_fs_base = ((uint64) local) + 2048;
  wrmsr(0xC0000100, kernel_fs_base);
@ -157,14 +162,14 @@ seginit(void)
  gdt[SEG_KCPU]  = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
  struct tassdesc *td = (struct tassdesc*)(gdt + SEG_TSS);
-  td->lim_15_0 = 0x67;
+  td->lim_15_0 = tsslim & 0xffff;
  td->base_15_0 = addr & 0xffff;
  td->base_23_16 = (addr >> 16) & 0xff;
  td->type = STS_T64A;
  td->s = 0;
  td->dpl = 0;
  td->p = 1;
-  td->lim_19_16 = 0;
+  td->lim_19_16 = (tsslim >> 16) & 0xf;
  td->avl = 0;
  td->l = 0;
  td->d = 0;
@ -300,10 +305,14 @@ switchuvm(struct proc *p)
  pushcli();
  if(p->pgdir == 0)
    panic("switchuvm: no pgdir");
-  uint *tss = (uint*) (((char*) mycpu()->local) + 1024);
+  void *tss = ioperm_tss_of(cpunum());
  const addr_t stktop = (addr_t)p->kstack + KSTACKSIZE;
-  tss[1] = (uint)stktop;
+  ioperm_set_rsp0(tss, stktop);
-  tss[2] = (uint)(stktop >> 32);
+
  // Which ports this process may reach. Cheap when nobody uses any, which is
  // every process that is not hosting a driver.
  ioperm_switch(p);
  lcr3(v2p(p->pgdir));
  popcli();
 }
--- a/kernel/dev/pciuser.c
+++ b/kernel/dev/pciuser.c
@ -24,6 +24,7 @@
 #include "memlayout.h"
 #include "mmu.h"
 #include "proc.h"
 #include "ioperm.h"
 #include "pci.h"
 #include "errno.h"
 #include "uaccess.h"
@ -342,6 +343,10 @@ pciuser_release_all(struct proc *p)
      pciuser_stop(i);
  }
  // And its ports. The bitmap itself is closed by the next switchuvm() on each
  // CPU; see ioperm_revoke_all().
  ioperm_revoke_all(p);
  acquire(&pciuser_lock);
  // And its interrupt line -- unregistered, and left MASKED.
@ -485,6 +490,56 @@ do_claim(uint64 argp)
  // it back. The two are a pair, and the second driver to run is the one that
  // finds out if they are not.
  pci_enable((struct pci_dev *)d, PCI_CMD_MEMORY | PCI_CMD_BUS_MASTER);
  // And its interrupt line: masked.
  //
  // The kernel driver that just let go had a handler on this line and no
  // longer does, but the device is about to be brought up by a driver in a
  // process and *will* assert it. With nothing to acknowledge the cause at the
  // device, every EOI re-enters an empty dispatch immediately:
  //
  //   irq 11: 449905 interrupts/second -- a device on this line is not
  //           acknowledging it.
  //
  // which is the machine spending all of its time noticing. The line comes
  // back when the process asks for it with PCIIOC_IRQ, which is the same
  // arrangement pciuser_release_all() leaves behind and for the same reason.
  if (d->irq != 0xFF && d->irq < 24) {
    ioapicmask(d->irq);
    acquire(&pciuser_lock);
    if (irqdev.proc == 0)
      irqdev.masked = 1;
    release(&pciuser_lock);
  }
  // And the ports, if it has any.
  //
  // A Linux driver does not always go through a function we can intercept:
  // e1000's reset path issues `out` directly, because on an 82540 that is the
  // only way the chip will acknowledge the write. So a process holding a
  // device gets that device's I/O BARs and nothing else -- see
  // include/ioperm.h. PCI_CMD_IO goes with them for the same reason memory
  // space goes with the mapping above.
  {
    int granted = 0;
    for (int b = 0; b < PCIUSER_MAXBAR; b++) {
      if (!d->bar[b].is_io || !d->bar[b].base || !d->bar[b].size)
        continue;
      if (d->bar[b].base > 0xffff || d->bar[b].base + d->bar[b].size > 0x10000)
        continue;               // above the port space; nothing to grant
      ioperm_grant(myproc(), (uint16)d->bar[b].base, (uint16)d->bar[b].size);
      granted = 1;
    }
    if (granted) {
      pci_enable((struct pci_dev *)d, PCI_CMD_IO);
      // Install them on this CPU now, rather than at the next switchuvm().
      // The process asked for them in a system call and returns to user mode
      // from here, which need not go through the scheduler at all.
      pushcli();
      ioperm_switch(myproc());
      popcli();
    }
  }
  return 0;
 }
@ -507,8 +562,17 @@ do_release(uint64 argp)
  // politely has usually quiesced its device already, but "usually" is not a
  // property the kernel can check, and the cost of being sure is one config
  // write.
-  if (mine)
+  if (mine) {
    pciuser_stop(idx);
    // Ports go back too. Coarse on purpose: this drops *every* range the
    // process had rather than only this device's, and the next claim puts
    // them back. A process releasing one of several devices is not a case
    // that exists yet, and a revoke that is too wide fails safe.
    ioperm_revoke_all(myproc());
    pushcli();
    ioperm_switch(myproc());
    popcli();
  }
  return 0;
 }
--- a/kernel/drivers/net/e1000/e1000.c
+++ b/kernel/drivers/net/e1000/e1000.c
@ -49,6 +49,11 @@ struct e1000 {
  // the two calls are separate, and nothing else connects them.
  struct pci_dev *pdev;
  // And the network device the stack knows it by, so that letting go of the
  // hardware can also take the interface out of the stack. Set by attached(),
  // which is where the two are first introduced.
  struct device *ndev;
  struct spinlock lock;
 };
@ -387,6 +392,62 @@ e1000_link_up(struct device *dev)
  return e && e->ready && (rd(e, E1000_STATUS) & E1000_STATUS_LU) != 0;
 }
 // Let go of the device, so a userspace driver can have it.
 //
 // The same hook virtio-gpu grew for H1, on a NIC. docs/plan/hybrid.md's rule
 // is that the kernel keeps a minimal driver for everything on the path to a
 // shell -- and a network interface is not on that path, so this one may leave.
 //
 // "Letting go" is more than saying yes. The device is still bus-mastering into
 // descriptor rings this driver allocated, and the process about to claim it
 // will be handed those pages back by the arena. So the receiver and the
 // transmitter are switched off and the interrupt is masked *at the device*
 // before the handler is unregistered: masking after unregistering leaves a
 // window where a level-triggered line is asserted with nobody to clear it,
 // which is the failure docs/history/kernel.md section 13 is about.
 //
 // `ready` going false is what makes the class vtable safe afterwards --
 // transmit, receive and link_up all check it, so the stack sees an interface
 // whose link is down rather than one that swallows packets.
 static int
 e1000_release(struct pci_dev *d)
 {
  struct e1000 *e = 0;
  // Found by the device it is bound to. struct pci_dev carries no private
  // pointer -- this driver keeps its instances in a table and matches on
  // `pdev`, the same way e1000_attached() does.
  for (int i = 0; i < E1000_MAX_NICS; i++)
    if (nics[i].pdev == d) { e = &nics[i]; break; }
  if (!e || !e->ready)
    return 0;                     // never came up, or already let go
  e->ready = 0;
  wr(e, E1000_IMC, 0xFFFFFFFF);            // no more interrupts from it
  wr(e, E1000_RCTL, 0);                    // stop receiving
  wr(e, E1000_TCTL, 0);                    // stop transmitting
  rd(e, E1000_STATUS);                     // post the writes
  if (d->irq != 0xFF)
    irq_unregister(d->irq, e1000_intr, e);
  // And take the interface out of the stack.
  //
  // Leaving it there is worse than it sounds. The netif keeps its address and
  // the route to its subnet, so the stack goes on choosing it -- and every
  // frame it sends reaches e1000_transmit, which now correctly refuses. The
  // packets stop and nothing reports an error, which is indistinguishable
  // from a driver in a process that does not work. It was mistaken for
  // exactly that for an afternoon.
  if (e->ndev)
    net_detach(e->ndev);
  cprintf("e1000: released to userspace; the interface is down\n");
  return 0;
 }
 // Hang the instance on the device the bus has just published. See the comment
 // on `attached` in include/pci.h for why this cannot happen inside probe().
 static void
@ -395,6 +456,7 @@ e1000_attached(struct pci_dev *d, struct device *dev)
  for (int i = 0; i < E1000_MAX_NICS; i++)
    if (nics[i].ready && nics[i].pdev == d) {
      dev->priv = &nics[i];
      nics[i].ndev = dev;
      return;
    }
 }
@ -414,4 +476,5 @@ const struct pci_driver e1000_driver = {
  .ops   = &e1000_netdev_ops,
  .probe = e1000_probe,
  .attached = e1000_attached,
  .release = e1000_release,
 };
--- a/kernel/net/lwip/netif_eth.c
+++ b/kernel/net/lwip/netif_eth.c
@ -138,6 +138,52 @@ net_eth_attach(struct device *dev)
    return nifs++;
 }
 // An interface whose device has gone away.
 //
 // The missing half of net_eth_attach, and it went missing because nothing had
 // ever taken a device back. /dev/pci does: the kernel's e1000 driver hands the
 // NIC to a process, and until this existed the interface stayed in lwIP --
 // up, holding an address, holding the route to its subnet, and attached to a
 // driver that now refuses every frame. Packets went to it and stopped, which
 // looks exactly like a driver that is not working.
 //
 // The slot is left occupied rather than compacted. Interface indices are
 // visible in /proc/netif and in the names the stack has already handed out,
 // and renumbering them because a device left would be worse than a gap.
 int
 net_eth_detach(struct device *dev)
 {
    int i;
    for (i = 0; i < nifs; i++) {
        if (!ifs[i].used || ifs[i].dev != dev)
            continue;
        // Down first, then out. netif_remove() does stop DHCP and send the
        // release, but a netif removed while still up leaves lwIP's default
        // route pointing at freed storage if this was interface 0.
        netif_set_link_down(&ifs[i].netif);
        netif_set_down(&ifs[i].netif);
        if (netif_default == &ifs[i].netif)
            netif_set_default(NULL);
        netif_remove(&ifs[i].netif);
        ifs[i].used = 0;
        ifs[i].dev  = 0;
        // Something else becomes the default, if anything can. A machine with
        // a second interface should not lose its route because the first one
        // was handed to a driver in a process.
        if (netif_default == NULL) {
            int j;
            for (j = 0; j < nifs; j++)
                if (ifs[j].used) { netif_set_default(&ifs[j].netif); break; }
        }
        return i;
    }
    return -1;
 }
 int
 net_eth_count(void)
 {
--- a/kernel/net/net.c
+++ b/kernel/net/net.c
@ -208,6 +208,42 @@ net_attach_late(struct device *d)
    return idx;
 }
 // Which interfaces have had their address reported. File scope rather than
 // local to net_thread(), because an interface that goes away has to be able to
 // be announced again if one comes back in its slot.
 static int announced[NETIF_MAX];
 static void
 net_unannounce(int idx)
 {
    if (idx >= 0 && idx < NETIF_MAX)
        announced[idx] = 0;
 }
 // The other direction: a device the kernel no longer drives.
 //
 // Called when /dev/pci hands a NIC to a process. Same rule about lwIP headers
 // as net_attach_late above -- the caller is a device file and must not see
 // them.
 int
 net_detach(struct device *d)
 {
    int idx;
    if (!net_ready)
        return -1;
    lwip_sys_lock();
    idx = net_eth_detach(d);
    lwip_sys_unlock();
    if (idx >= 0) {
        net_unannounce(idx);
        cprintf("net: %s is gone; interface %d removed\n", d->name, idx);
    }
    return idx;
 }
 static void
 net_thread(void)
 {
@ -253,7 +289,7 @@ net_thread(void)
            attached, attached == 1 ? "" : "s");
    // One bit per interface, so a second one binding later is announced too.
-    int announced[NETIF_MAX] = { 0 };
+
    for (;;) {
        int moved;
@ -269,8 +305,14 @@ net_thread(void)
        // Report each address once, when it first arrives. Anyone who has spent
        // an evening wondering whether the guest has an address at all wants
        // this on the console rather than in a file they have to know about.
-        for (int i = 0; i < attached; i++) {
+        // Over every interface there is now, not over the ones there were at
        // boot: an interface can arrive later. /dev/netdev makes one whenever a
        // process becomes a network device, and until this counted them an
        // address on `ldls0` was never reported at all.
        for (int i = 0; i < net_eth_count() && i < NETIF_MAX; i++) {
            struct netif *nif = net_eth_netif(i);
            if (nif == 0 || net_eth_device(i) == 0)
                continue;
            if (announced[i] || ip4_addr_isany_val(*netif_ip4_addr(nif)))
                continue;
--- a/ldls/btf.c
+++ b/ldls/btf.c
@ -170,6 +170,55 @@ btf_find(struct btf *b, const char *name, int kind)
  return 0;
 }
 // The value of a named enumerator, wherever it lives.
 //
 // Not every constant a driver depends on is a struct offset. `netif_carrier_ok`
 // is an inline that tests bit `__LINK_STATE_NOCARRIER` of `net_device.state`,
 // so a host that wants to say "the link is down" has to set the right bit --
 // and getting it wrong means a driver that believes the carrier is already up
 // and skips everything it would have done on link-up. e1000 skips enabling its
 // transmitter, which is not an error anywhere.
 //
 // The enum is anonymous, so there is no type to look up by name; the search is
 // over enumerators rather than over types. Which is the same rule 3½ bargain as
 // the offsets: the number is the installed kernel's, not ours.
 //
 // Returns 0 and sets *val on success, -1 if no enumerator has that name.
 int
 btf_enum_value(struct btf *b, const char *name, long long *val)
 {
  unsigned int id;
  for (id = 1; id <= b->ntypes; id++) {
    const struct btf_type *t = (const struct btf_type *)(b->types + b->off[id]);
    int kind = (int)BTF_INFO_KIND(t->info);
    unsigned int n = BTF_INFO_VLEN(t->info), i;
    if (kind == BTF_KIND_ENUM) {
      const unsigned int *e = (const unsigned int *)(t + 1);
      for (i = 0; i < n; i++) {
        unsigned int noff = e[i * 2];
        if (noff < b->str_len && btf_streq(b->str + noff, name)) {
          *val = (int)e[i * 2 + 1];
          return 0;
        }
      }
    } else if (kind == BTF_KIND_ENUM64) {
      const unsigned int *e = (const unsigned int *)(t + 1);
      for (i = 0; i < n; i++) {
        unsigned int noff = e[i * 3];
        if (noff < b->str_len && btf_streq(b->str + noff, name)) {
          *val = (long long)e[i * 3 + 1] |
                 ((long long)e[i * 3 + 2] << 32);
          return 0;
        }
      }
    }
  }
  btf_fail(b, "no enumerator named ", name);
  return -1;
 }
 // Follow the things that wrap a type without changing its layout.
 static int
 btf_strip(struct btf *b, int id)
--- a/ldls/btf.h
+++ b/ldls/btf.h
@ -127,6 +127,10 @@ int btf_find(struct btf *b, const char *name, int kind);
 int btf_member_off(struct btf *b, int struct_id, const char *member,
                   unsigned int *off_bytes, unsigned int *size_bytes);
 // The value of a named enumerator, searched across every enum in the file.
 // Constants a driver depends on are not all struct offsets -- see btf.c.
 int btf_enum_value(struct btf *b, const char *name, long long *val);
 // Size in bytes of a type, following typedefs and qualifiers. 0 if unknown.
 unsigned int btf_type_size(struct btf *b, int id);
--- a/ldls/host/run.c
+++ b/ldls/host/run.c
@ -310,6 +310,40 @@ resolve(void *ctx, const char *name, unsigned long long *out)
 // x, so it is an ordinary defined global and needs no struct layout to find --
 // which matters, because the struct that *would* hold it is one of the things
 // rule 4 says not to assume.
 // See cmd/ldlsd/main.c for what this is for; the harness needs it too, because
 // a driver that reads `current` reads it here as well.
 #define PERCPU_SLOTS 8
 #define PERCPU_SIZE  (16 * 1024)
 static char percpu_mem[PERCPU_SLOTS][PERCPU_SIZE];
 static const char *percpu_name[PERCPU_SLOTS];
 static int percpu_n;
 static unsigned long long
 percpu_for(void *ctx, const char *name)
 {
  int i;
  (void)ctx;
  for (i = 0; i < percpu_n; i++)
    if (strcmp(percpu_name[i], name) == 0)
      return (unsigned long long)(void *)percpu_mem[i];
  if (percpu_n == PERCPU_SLOTS)
    return 0;
  i = percpu_n++;
  percpu_name[i] = name;
  // See cmd/ldlsd/main.c: `current` is read and then dereferenced.
  if (strcmp(name, "const_current_task") == 0 ||
      strcmp(name, "current_task") == 0) {
    *(unsigned long long *)percpu_mem[i] =
        (unsigned long long)(void *)(percpu_mem[i] + 64);
    printf("  per-CPU %s -- a task of zeroes\n", name);
  } else {
    printf("  per-CPU %s -- 16K of zeroes\n", name);
  }
  return (unsigned long long)(void *)percpu_mem[i];
 }
 static unsigned long long
 find_symbol(struct ko *k, const char *want)
 {
@ -431,6 +465,15 @@ main(int argc, char **argv)
    if (g > 0)
      printf("  defused %d %%gs stack-guard reference(s)\n", g);
  }
  {
    int p = ko_fixup_percpu(&k, percpu_for, 0);
    if (p < 0) {
      fprintf(stderr, "ldls-run: per-CPU: %s\n", ko_err(&k));
      return 1;
    }
    if (p > 0)
      printf("  rehomed %d %%gs per-CPU reference(s)\n", p);
  }
  printf("  loaded  %llu bytes at %p, %d stubs at %p\n\n",
         span, (void *)mem, k.nundef, (void *)code);
--- a/ldls/ko.c
+++ b/ldls/ko.c
@ -597,6 +597,250 @@ ko_defuse_gs(struct ko *k, const char *sym)
  return n;
 }
 // Could this symbol name a per-CPU variable at all?
 //
 // Imports can: `const_current_task` and `softnet_data` live in vmlinux. A
 // module's own per-CPU variables live in a section called .data..percpu.
 // Anything else -- a section symbol for .text, an ordinary global -- cannot,
 // and saying so is what keeps a coincidental 0x65 from being read as a prefix.
 static int
 percpu_symbol(struct ko *k, unsigned int si)
 {
  unsigned short shndx = k->sym[si].st_shndx;
  const struct ko_shdr *sh;
  const char *nm;
  int i;
  if (shndx == 0)                       /* undefined: an import */
    return 1;
  if (shndx >= (unsigned short)k->nsh || !k->shstr)
    return 0;
  sh = ko_sh(k, shndx);
  if (!sh)
    return 0;
  nm = k->shstr + sh->sh_name;
  for (i = 0; ".data..percpu"[i]; i++)
    if (nm[i] != ".data..percpu"[i])
      return 0;
  return 1;
 }
 // Take the `%gs` off.
 //
 // The first idea here was to rewrite the load into a `lea` of memory we
 // supply, which works for `current` -- a pointer -- and is wrong for
 // `cpu_number`, which is a value and needs to read as 0. Dropping the segment
 // prefix instead serves both: the instruction keeps its opcode, its width and
 // its direction, and simply addresses memory we own rather than memory only a
 // kernel has. A per-CPU variable becomes a variable, which on a host with one
 // thread pretending to be one CPU is what it was all along.
 //
 // It is one byte shorter, so the last byte becomes a nop and nothing after it
 // moves. The displacement is measured from the end of the *shortened*
 // instruction, which is the subtlety worth stating: taking it from the end of
 // the original puts every access one byte off, and reads plausible data from
 // the wrong place.
 // How long the instruction is, and how much of it comes before the
 // displacement.
 //
 // Enough of an x86 decoder to know that and no more. The displacement is not
 // always last -- `cmpb $0x0, %gs:softnet_data+0x75(%rip)` has an immediate
 // after it, and assuming otherwise moved the wrong four bytes. So the opcode
 // has to be read far enough to know how big the immediate is.
 //
 // Returns the total length, or 0 for a shape we have not established the
 // meaning of. Refusing is the useful answer there: rule 5 says the driver tells
 // us what to implement next, and a silently mis-decoded instruction tells us
 // nothing at all.
 static int
 gs_form(const unsigned char *p, int want_pre)
 {
  int j = 1, op, modrm, imm = 0, pre;
  if (p[0] != 0x65)
    return 0;
  if ((p[1] & 0xF0) == 0x40)            /* REX */
    j = 2;
  if (p[j] == 0x0f) {                   /* two-byte opcode */
    op = 0x0f00 | p[j + 1];
    modrm = p[j + 2];
    pre = j + 3;
  } else {
    op = p[j];
    modrm = p[j + 1];
    pre = j + 2;
  }
  if (pre != want_pre)
    return 0;
  if ((modrm & 0xC7) != 0x05)           /* rip-relative, no SIB */
    return 0;
  switch (op) {
  /* mov, arithmetic, test, exchange -- register or implicit operand */
  case 0x88: case 0x89: case 0x8a: case 0x8b: case 0x8d:
  case 0x00: case 0x01: case 0x02: case 0x03:
  case 0x08: case 0x09: case 0x0a: case 0x0b:
  case 0x20: case 0x21: case 0x22: case 0x23:
  case 0x28: case 0x29: case 0x2a: case 0x2b:
  case 0x30: case 0x31: case 0x32: case 0x33:
  case 0x38: case 0x39: case 0x3a: case 0x3b:
  case 0x84: case 0x85: case 0x86: case 0x87:
  case 0xff: case 0xfe:
    imm = 0; break;
  /* the same, with an immediate */
  case 0x80: case 0x83: case 0xc0: case 0xc1: case 0xc6:
    imm = 1; break;
  case 0x81: case 0xc7:
    imm = 4; break;
  /* group 3: only /0 (test) carries an immediate */
  case 0xf6:
    imm = ((modrm >> 3) & 7) == 0 ? 1 : 0; break;
  case 0xf7:
    imm = ((modrm >> 3) & 7) == 0 ? 4 : 0; break;
  /* movzx, movsx, setcc, bit tests, cmpxchg, xadd */
  case 0x0fb6: case 0x0fb7: case 0x0fbe: case 0x0fbf:
  case 0x0fa3: case 0x0fab: case 0x0fb3: case 0x0fbb:
  case 0x0fb0: case 0x0fb1: case 0x0fc0: case 0x0fc1:
    imm = 0; break;
  case 0x0fba:
    imm = 1; break;
  default:
    if (op >= 0x0f90 && op <= 0x0f9f)   /* setcc */
      { imm = 0; break; }
    return 0;
  }
  return pre + 4 + imm;
 }
 // A null `insn` measures without writing, which is how the first pass asks
 // whether every site is one this can do before the second pass does any of it.
 static int
 percpu_one(unsigned char *insn, int pre, int len, unsigned long long at,
           unsigned long long tgt)
 {
  long long rel;
  int i;
  rel = (long long)tgt - (long long)(at + len - 1);
  if (rel < -0x80000000LL || rel > 0x7fffffffLL)
    return -2;                         /* too far for a rip-relative access */
  if (!insn)
    return 0;
  for (i = 0; i < len - 1; i++)
    insn[i] = insn[i + 1];
  insn[len - 1] = 0x90;
  insn[pre - 1] = (unsigned char)rel;
  insn[pre + 0] = (unsigned char)(rel >> 8);
  insn[pre + 1] = (unsigned char)(rel >> 16);
  insn[pre + 2] = (unsigned char)(rel >> 24);
  return 0;
 }
 // Two passes, for the reason ko_defuse_gs() has two: a half-patched object is
 // worse than an unpatched one, because it runs.
 int
 ko_fixup_percpu(struct ko *k,
                unsigned long long (*percpu)(void *ctx, const char *name),
                void *ctx)
 {
  int pass, i, n = 0;
  if (!k->laid_out)
    { ko_fail(k, "percpu fixup before layout", 0); return -1; }
  for (pass = 0; pass < 2; pass++) {
    n = 0;
    for (i = 0; i < k->nsh; i++) {
      const struct ko_shdr *sh = ko_sh(k, i);
      struct ko_section *tgt;
      unsigned long long cnt, j;
      if (!sh || sh->sh_type != KO_SHT_RELA || !k->sec[i].data)
        continue;
      if (sh->sh_info >= (unsigned int)k->nsh)
        continue;
      tgt = &k->sec[sh->sh_info];
      if (!tgt->alloc || tgt->size == 0 || !(tgt->flags & KO_SHF_EXECINSTR))
        continue;
      cnt = sh->sh_size / sizeof(struct ko_rela);
      for (j = 0; j < cnt; j++) {
        const struct ko_rela *rl =
            (const struct ko_rela *)(k->sec[i].data) + j;
        unsigned int si = KO_R_SYM(rl->r_info);
        unsigned char *insn = 0;
        unsigned long long addr, t;
        const char *nm;
        int pre, len = 0, r;
        if ((int)si >= k->nsym || rl->r_offset < 5 ||
            rl->r_offset + 4 > tgt->size)
          continue;
        // Only PC32. PLT32 is the displacement of a call or a jump, and the
        // bytes in front of one are whatever the previous instruction ended
        // with -- which can be 0x65, and was: a call to `eth_type_trans` read
        // as a %gs access until this line existed.
        if (KO_R_TYPE(rl->r_info) != KO_R_X86_64_PC32)
          continue;
        // The prefix sits three to five bytes before the displacement,
        // depending on REX and on whether the opcode is two bytes. A REX byte
        // is never 0x65 and neither is 0x0f, so the longest match that decodes
        // is the right one.
        for (pre = 5; pre >= 3; pre--) {
          unsigned char *c = tgt->where + rl->r_offset - pre;
          len = gs_form(c, pre);
          if (len) { insn = c; break; }
        }
        if (!insn) {
          // Nothing decoded. Most of the time that is because this is an
          // ordinary rip-relative reference ko_relocate already resolved and
          // there is no %gs anywhere near it.
          const unsigned char *q = tgt->where + rl->r_offset;
          if (q[-3] != 0x65 && q[-4] != 0x65 && q[-5] != 0x65)
            continue;
          // A stray 0x65 in the tail of the previous instruction looks exactly
          // like a prefix from here, so the byte alone is not evidence. Only a
          // symbol that could actually *be* per-CPU is: an import, or
          // something the module defines in .data..percpu.
          if (!percpu_symbol(k, si))
            continue;
          // Then it is one, in a shape we have not established the meaning of.
          // Naming it is the point: it is the next thing to implement.
          ko_fail(k, "unrecognised %gs instruction for ",
                  k->str + k->sym[si].st_name);
          return -1;
        }
        if (rl->r_offset - pre + len > tgt->size)
          { ko_fail(k, "%gs instruction runs off the section for ",
                    k->str + k->sym[si].st_name); return -1; }
        nm = k->str + k->sym[si].st_name;
        addr = tgt->addr + (unsigned long long)(insn - tgt->where);
        t = percpu ? percpu(ctx, nm) : 0;
        if (!t)
          { ko_fail(k, "no per-CPU storage for ", nm); return -1; }
        r = percpu_one(pass ? insn : 0, pre, len, addr, t);
        if (r == -2)
          { ko_fail(k, "per-CPU storage is too far to reach for ", nm);
            return -1; }
        if (r < 0)
          { ko_fail(k, "cannot rewrite the per-CPU access to ", nm);
            return -1; }
        n++;
      }
    }
  }
  return n;
 }
 // --- what the object says about itself ---------------------------------------
 static const struct ko_section *
--- a/ldls/ko.h
+++ b/ldls/ko.h
@ -225,6 +225,35 @@ int ko_relocate(struct ko *k, ko_resolver r, void *ctx);
 // should not be editing it.
 int ko_defuse_gs(struct ko *k, const char *sym);
 // Turn every *other* `%gs:`-relative reference into an ordinary one.
 //
 // The stack guard is one per-CPU variable; there are others, and `current` is
 // the one every driver touches. `mov %gs:const_current_task(%rip), %r8` reads
 // a pointer out of the kernel's per-CPU area, and a host that is not that
 // kernel has no such area -- so on SchOS it reads a kernel address from user
 // mode and faults.
 //
 // Defusing is not available here: the guard's *value* is arbitrary, so zeroing
 // a register is a correct answer, and `current`'s is not -- the driver will
 // dereference it.
 //
 // What works instead is that **a per-CPU variable in a single-threaded host is
 // just a variable**. Taking the `%gs` off leaves the same instruction reading
 // the same width in the same direction, from storage the embedder supplies for
 // that name. Eight bytes become seven and a nop, and the driver never learns
 // the difference.
 //
 // `percpu` is asked for an address for each name and should return the same
 // one every time it is asked for that name -- and should initialise it, since
 // the driver now reads its *contents*: zero is right for a counter and wrong
 // for `current`. Returning 0 refuses the object.
 //
 // Call after ko_relocate() and after ko_defuse_gs(), for the same reason.
 // Returns how many it rewrote, or -1.
 int ko_fixup_percpu(struct ko *k,
                    unsigned long long (*percpu)(void *ctx, const char *name),
                    void *ctx);
 // A value from the `.modinfo` section -- "license", "description", "author",
 // "alias". Returns 0 if the object has no such key.
 //
--- a/ldls/kpi/kpi.c
+++ b/ldls/kpi/kpi.c
@ -25,6 +25,57 @@ int               kpi_verbose;
 int  (*kpi_napi_poll)(void *napi, int budget);
 void *kpi_napi;
 // --- the driver's interrupt handler ------------------------------------------
 //
 // LDLS has no interrupt to run this from, and for a while that was taken to
 // mean the handler could simply never run: request_irq returned 0, the NAPI
 // poll was called from a loop, and 8139too was perfectly happy.
 //
 // e1000 is not, and the reason is worth writing down because it is not about
 // receiving packets. Its handler does two things -- schedule the NAPI poll,
 // which LDLS was already doing by hand, and, on a link-status change, queue
 // the watchdog. The watchdog is what enables the transmitter. So a driver
 // whose handler never runs was a driver whose transmitter was never switched
 // on, and the symptom was frames accepted, mapped, and silently not sent.
 //
 // Calling the handler from the loop is safe in a way that it would not be in a
 // kernel: a handler's first act is to read its device's cause register, and a
 // device with no cause pending returns IRQ_NONE and does nothing.
 static int (*kpi_irq_fn)(int irq, void *dev);
 static void *kpi_irq_dev;
 static int   kpi_irq_no;
 // request_threaded_irq(irq, handler, thread_fn, flags, name, dev)
 int
 kpi_request_irq(unsigned int irq, void *handler, void *thread_fn,
                unsigned long flags, const char *name, void *dev)
 {
  (void)thread_fn; (void)flags; (void)name;
  kpi_irq_no = (int)irq;
  kpi_irq_fn = (int (*)(int, void *))handler;
  kpi_irq_dev = dev;
  kpi_log("kpi: the driver asked for irq "); kpi_lognum((int)irq);
  kpi_log("; it will be called from the poll loop\n");
  return 0;
 }
 int
 kpi_free_irq(unsigned int irq, void *dev)
 {
  (void)irq; (void)dev;
  kpi_irq_fn = 0;
  return 0;
 }
 // Returns 1 if the handler said it was ours, which the loop counts as work.
 int
 kpi_irq_poll(void)
 {
  if (!kpi_irq_fn)
    return 0;
  return kpi_irq_fn(kpi_irq_no, kpi_irq_dev) == 1;
 }
 // netif_napi_add_weight_locked(dev, napi, poll, weight)
 //
 // Recorded rather than ignored, because the poll function is the receive path:
@ -110,6 +161,24 @@ kpi_alloc(unsigned long n)
 void *kpi_kmalloc(unsigned long n, unsigned int flags)
 { (void)flags; return kpi_alloc(n); }
 // vmalloc's promise is that the result is virtually contiguous and need not be
 // physically so, which is what mmap already gives. The distinction only matters
 // to a kernel that has both kinds of address, and a process has one kind.
 //
 // This is not a detail: e1000 allocates its per-descriptor bookkeeping with
 // vzalloc, so without it ndo_open fails at the first Tx ring with "Allocation
 // for Tx Queue 0 failed" and -ENOMEM -- an honest error from the driver about
 // a function that was quietly returning zero.
 void *kpi_vmalloc(unsigned long n)          { return kpi_alloc(n); }
 void  kpi_vfree(void *p)                    { (void)p; }
 int   kpi_is_vmalloc_addr(const void *p)    { (void)p; return 0; }
 // __kmalloc_cache_noprof(cache, flags, size) -- the slab is somebody else's
 // bookkeeping, and the size is the third argument.
 void *
 kpi_kmalloc_cache(void *cache, unsigned int flags, unsigned long n)
 { (void)cache; (void)flags; return kpi_alloc(n); }
 void kpi_kfree(void *p) { (void)p; }
 void *
@ -156,6 +225,180 @@ kpi_dma_alloc_attrs(void *dev, unsigned long size, unsigned long long *handle,
  return (void *)d.uaddr;
 }
 // --- memory a device can be pointed at, in small pieces -----------------------
 //
 // dma_alloc_coherent is one big block per call and PCIIOC_DMA matches it
 // exactly. dma_map_single is the other shape: the driver allocates an ordinary
 // buffer, hands it to the device and expects a bus address back.
 //
 // 8139too never needed this -- it copies each frame into a ring it allocated
 // coherently up front -- so it went unimplemented and the stub returned 0. On
 // e1000 that is a NIC pointed at physical address zero, which is not a fault,
 // not an error, and not something any log would have shown. The first driver
 // to need it is the reason this exists.
 //
 // A process cannot know the physical address of an ordinary page, so the
 // buffers have to come from the arena too: a chunk at a time, handed out by a
 // bump pointer, with the pairing remembered so a virtual address can be turned
 // back into a bus address. Nothing is freed, like everything else here.
 #define KPI_DMA_CHUNK   (1024 * 1024)
 #define KPI_DMA_CHUNKS  8
 static struct {
  unsigned long long uaddr, paddr, size, used;
 } kpi_chunk[KPI_DMA_CHUNKS];
 static int kpi_nchunk;
 static int
 kpi_chunk_more(void)
 {
  struct pciuser_dma d;
  if (kpi_nchunk >= KPI_DMA_CHUNKS)
    return 0;
  d.size = KPI_DMA_CHUNK;
  if (ioctl(kpi.pcifd, PCIIOC_DMA, &d) < 0)
    return 0;
  kpi_chunk[kpi_nchunk].uaddr = d.uaddr;
  kpi_chunk[kpi_nchunk].paddr = d.paddr;
  kpi_chunk[kpi_nchunk].size  = KPI_DMA_CHUNK;
  kpi_chunk[kpi_nchunk].used  = 0;
  kpi_nchunk++;
  return 1;
 }
 void *
 kpi_dma_heap(unsigned long n, unsigned long align)
 {
  int i;
  if (align == 0)
    align = 64;
  for (i = 0; i < 2; i++) {
    int c;
    for (c = 0; c < kpi_nchunk; c++) {
      unsigned long long at = kpi_chunk[c].uaddr + kpi_chunk[c].used;
      unsigned long long pad = (align - (at % align)) % align;
      if (kpi_chunk[c].used + pad + n > kpi_chunk[c].size)
        continue;
      kpi_chunk[c].used += pad + n;
      return (void *)(unsigned long)(at + pad);
    }
    if (!kpi_chunk_more())
      break;
  }
  kpi_log("kpi: out of DMA heap for "); kpi_lognum(n); kpi_log(" bytes\n");
  return 0;
 }
 // The bus address of something the heap handed out, or 0.
 unsigned long long
 kpi_phys_of(const void *va)
 {
  unsigned long long a = (unsigned long long)(unsigned long)va;
  int c;
  for (c = 0; c < kpi_nchunk; c++)
    if (a >= kpi_chunk[c].uaddr && a < kpi_chunk[c].uaddr + kpi_chunk[c].size)
      return kpi_chunk[c].paddr + (a - kpi_chunk[c].uaddr);
  return 0;
 }
 // The two globals that make the module's own address arithmetic ours.
 //
 // `dma_map_single` is a static inline, so what reaches the shim is already
 // expanded: the module computes `pa = va - page_offset_base`, then
 // `page = vmemmap_base + (pa >> 12) * 64`, and calls dma_map_page_attrs with
 // that. Both bases are undefined *data* symbols, which means the shim supplies
 // them -- and setting both to zero makes the whole computation reversible with
 // no lookup table at all:
 //
 //     page  = (va >> 12) * 64          on the way in
 //     va    = (page / 64) << 12 | off  on the way back
 //
 // The 64 is sizeof(struct page) and is compiled into the module as `shl $0x6`,
 // so it is not a choice either of us gets to make.
 unsigned long kpi_page_offset_base;
 unsigned long kpi_vmemmap_base;
 #define KPI_PAGE_TO_VA(p) ((unsigned long)(((unsigned long)(p) >> 6) << 12))
 // dma_map_page_attrs(dev, page, offset, size, dir, attrs)
 unsigned long long
 kpi_dma_map_page(void *dev, void *page, unsigned long offset,
                 unsigned long size, int dir, unsigned long attrs)
 {
  unsigned long va = KPI_PAGE_TO_VA(page) + offset;
  unsigned long long pa = kpi_phys_of((const void *)va);
  static int said;
  (void)dev; (void)size; (void)dir; (void)attrs;
  if (!said) {
    said = 1;
    kpi_log("kpi: the driver mapped its first buffer for the device\n");
  }
  if (!pa) {
    // DMA_MAPPING_ERROR. Saying so is the whole point: a driver checks this
    // with dma_mapping_error() and drops the buffer, where a zero would have
    // aimed the device at the bottom of memory.
    kpi_log("kpi: dma_map of memory that is not in the arena\n");
    return ~0ULL;
  }
  return pa;
 }
 void
 kpi_dma_unmap_page(void *dev, unsigned long long h, unsigned long size,
                   int dir, unsigned long attrs)
 { (void)dev; (void)h; (void)size; (void)dir; (void)attrs; }
 // The arena is uncached-coherent as far as this host is concerned, and x86 is
 // cache-coherent for DMA anyway, so a sync is genuinely nothing.
 void kpi_dma_sync(void *dev, unsigned long long h, unsigned long size, int dir)
 { (void)dev; (void)h; (void)size; (void)dir; }
 // alloc_pages(gfp, order) -> struct page *, in the same fictional page space.
 void *
 kpi_alloc_pages(unsigned int gfp, unsigned int order)
 {
  unsigned long n = 4096UL << order;
  void *p;
  static int said;
  (void)gfp;
  p = kpi_dma_heap(n, 4096);
  if (!p)
    return 0;
  if (!said) {
    said = 1;
    kpi_log("kpi: the driver is allocating pages, order ");
    kpi_lognum(order); kpi_log("\n");
  }
  return (void *)((((unsigned long)p) >> 12) << 6);
 }
 void kpi_page_frag_free(void *p) { (void)p; }
 // __netdev_alloc_frag_align(fragsz, align_mask)
 void *
 kpi_netdev_alloc_frag(unsigned long fragsz, unsigned long align_mask)
 {
  static int said;
  unsigned long align = (~align_mask) + 1;
  void *p;
  if (align == 0 || align > 4096)
    align = 64;
  p = kpi_dma_heap(fragsz, align);
  if (!said) {
    said = 1;
    kpi_log("kpi: the driver is allocating receive buffers, ");
    kpi_lognum((long long)fragsz); kpi_log(" bytes each\n");
  }
  return p;
 }
 void
 kpi_dma_free_attrs(void *dev, unsigned long size, void *va,
                   unsigned long long handle, unsigned long attrs)
@ -165,6 +408,33 @@ kpi_dma_free_attrs(void *dev, unsigned long size, void *va,
  (void)dev; (void)size; (void)va; (void)handle; (void)attrs;
 }
 // --- bitmaps ------------------------------------------------------------------
 // _find_next_bit(addr, nbits, start) -- the first set bit at or after `start`,
 // or `nbits` if there is none.
 //
 // A pure function on the caller's own memory, so there is nothing to emulate:
 // this is simply it, written from the signature. It is here because the stub
 // version cost an hour. `for_each_cpu` is a loop whose body is
 // `i = _find_next_bit(mask, nbits, i + 1)` and whose exit test is `i < nbits`,
 // so a stub returning 0 does not mean "nothing found" -- it means "bit zero is
 // set", for ever. ndo_open hung with no error and no fault.
 //
 // The general lesson is the one memcpy taught in docs/architecture/ldls.md,
 // pointed a different way: for a function that *searches*, zero is not a
 // neutral answer, it is the most specific answer there is.
 unsigned long
 kpi_find_next_bit(const unsigned long *addr, unsigned long nbits,
                  unsigned long start)
 {
  unsigned long i;
  for (i = start; i < nbits; i++)
    if (addr[i / 64] & (1UL << (i % 64)))
      return i;
  return nbits;
 }
 // --- things that only have to not crash ---------------------------------------
 //
 // A hosted driver takes locks, starts timers and waits for things. In one
@ -235,15 +505,41 @@ static const unsigned char kpi_byte_rev_table[256] = {
  0x0f,0x8f,0x4f,0xcf,0x2f,0xaf,0x6f,0xef,0x1f,0x9f,0x5f,0xdf,0x3f,0xbf,0x7f,0xff,
 };
-// jiffies, also data. Nothing advances it: the drivers hosted so far read it
+// jiffies, also data -- and now it has to move.
-// only to compare against a timeout they will never reach, because nothing
+//
-// here runs their watchdog either.
+// It was a static zero for as long as nothing ran a driver's deferred work,
-static unsigned long kpi_jiffies;
+// and that was honest: a clock nobody reads need not tick. e1000's watchdog
 // reads it, re-arms itself against it, and enables the transmitter from
 // inside it, so a stopped clock is a NIC that never transmits.
 //
 // SchOS's uptime() counts 100 Hz ticks, which is a plausible HZ and is the
 // only monotonic source a process has. What matters to a driver is that it
 // advances and never goes backwards; the exact rate matters only to timeouts,
 // and a timeout measured against a slower clock is a longer timeout.
 unsigned long kpi_jiffies;
 unsigned long
 kpi_jiffies_now(void)
 {
  int t = uptime();
  if (t > 0)
    kpi_jiffies = (unsigned long)t;
  return kpi_jiffies;
 }
 // --- the table -----------------------------------------------------------------
 //
 // Names on the left are Linux's and are facts. Everything on the right is ours.
 extern void kpi_thunk_rax(void), kpi_thunk_rbx(void), kpi_thunk_rcx(void);
 extern void kpi_thunk_rdx(void), kpi_thunk_rsi(void), kpi_thunk_rdi(void);
 extern void kpi_thunk_rbp(void);
 extern void kpi_thunk_r8(void),  kpi_thunk_r9(void),  kpi_thunk_r10(void);
 extern void kpi_thunk_r11(void), kpi_thunk_r12(void), kpi_thunk_r13(void);
 extern void kpi_thunk_r14(void), kpi_thunk_r15(void);
 extern void kpi_return_thunk(void), kpi_fentry(void);
 extern void kpi_netif_carrier_off(void *dev);
 extern void kpi_netif_carrier_on(void *dev);
 extern void *kpi_pci_iomap(void *pdev, int bar, unsigned long max);
 extern void  kpi_pci_iounmap(void *pdev, void *addr);
 extern int   kpi_pci_enable_device(void *pdev);
@ -252,6 +548,12 @@ extern void  kpi_pci_release_regions(void *pdev);
 extern void  kpi_pci_set_master(void *pdev);
 extern int   kpi_pci_read_config_word(void *pdev, int where, unsigned short *v);
 extern int   kpi_pci_write_config_word(void *pdev, int where, unsigned short v);
 extern void *kpi_pci_ioremap_bar(void *pdev, int bar);
 extern int   kpi_pci_select_bars(void *pdev, unsigned long flags);
 extern int   kpi_pci_request_selected_regions(void *pdev, int bars, const char *n);
 extern void  kpi_pci_release_selected_regions(void *pdev, int bars);
 extern int   kpi_pci_save_state(void *pdev);
 extern void  kpi_pci_restore_state(void *pdev);
 extern unsigned char  kpi_ioread8(void *addr);
 extern unsigned short kpi_ioread16(void *addr);
@ -268,6 +570,8 @@ extern void  kpi_dev_addr_mod(void *dev, unsigned int off,
                              const unsigned char *addr, unsigned int len);
 extern void *kpi_napi_alloc_skb(void *napi, unsigned int len);
 extern void *kpi_build_skb(void *data, unsigned int frag_len);
 extern int kpi_gro_receive_skb(void *gro, void *skb);
 extern void *kpi_netdev_alloc_skb(void *dev, unsigned int len);
 extern void *kpi_skb_put(void *skb, unsigned int n);
 extern void *kpi_skb_reserve(void *skb, int n);
@ -297,6 +601,12 @@ const struct kpi_sym kpi_table[] = {
  { "pci_set_master",          (void *)kpi_pci_set_master },
  { "pci_iomap",               (void *)kpi_pci_iomap },
  { "pci_iounmap",             (void *)kpi_pci_iounmap },
  { "pci_ioremap_bar",         (void *)kpi_pci_ioremap_bar },
  { "pci_select_bars",         (void *)kpi_pci_select_bars },
  { "pci_request_selected_regions", (void *)kpi_pci_request_selected_regions },
  { "pci_release_selected_regions", (void *)kpi_pci_release_selected_regions },
  { "pci_save_state",          (void *)kpi_pci_save_state },
  { "pci_restore_state",       (void *)kpi_pci_restore_state },
  { "pci_read_config_word",    (void *)kpi_pci_read_config_word },
  { "pci_write_config_word",   (void *)kpi_pci_write_config_word },
@ -309,7 +619,50 @@ const struct kpi_sym kpi_table[] = {
  { "iowrite32",               (void *)kpi_iowrite32 },
  // Memory
  { "netif_carrier_off",       (void *)kpi_netif_carrier_off },
  { "netif_carrier_on",        (void *)kpi_netif_carrier_on },
  // The compiler's, not Linux's -- see thunk.c. A stub here is not a missing
  // feature, it is a discarded jump.
  { "__x86_indirect_thunk_rax", (void *)kpi_thunk_rax },
  { "__x86_indirect_thunk_rbx", (void *)kpi_thunk_rbx },
  { "__x86_indirect_thunk_rcx", (void *)kpi_thunk_rcx },
  { "__x86_indirect_thunk_rdx", (void *)kpi_thunk_rdx },
  { "__x86_indirect_thunk_rsi", (void *)kpi_thunk_rsi },
  { "__x86_indirect_thunk_rdi", (void *)kpi_thunk_rdi },
  { "__x86_indirect_thunk_rbp", (void *)kpi_thunk_rbp },
  { "__x86_indirect_thunk_r8",  (void *)kpi_thunk_r8 },
  { "__x86_indirect_thunk_r9",  (void *)kpi_thunk_r9 },
  { "__x86_indirect_thunk_r10", (void *)kpi_thunk_r10 },
  { "__x86_indirect_thunk_r11", (void *)kpi_thunk_r11 },
  { "__x86_indirect_thunk_r12", (void *)kpi_thunk_r12 },
  { "__x86_indirect_thunk_r13", (void *)kpi_thunk_r13 },
  { "__x86_indirect_thunk_r14", (void *)kpi_thunk_r14 },
  { "__x86_indirect_thunk_r15", (void *)kpi_thunk_r15 },
  { "__x86_return_thunk",       (void *)kpi_return_thunk },
  { "__fentry__",               (void *)kpi_fentry },
  { "_find_next_bit",          (void *)kpi_find_next_bit },
  { "queue_work_on",           (void *)kpi_queue_work_on },
  { "queue_delayed_work_on",   (void *)kpi_queue_delayed_work_on },
  { "cancel_work_sync",        (void *)kpi_cancel_work_sync },
  { "cancel_delayed_work_sync",(void *)kpi_cancel_delayed_work_sync },
  { "cancel_delayed_work",     (void *)kpi_cancel_delayed_work },
  { "flush_work",              (void *)kpi_flush_work },
  { "dma_map_page_attrs",      (void *)kpi_dma_map_page },
  { "dma_unmap_page_attrs",    (void *)kpi_dma_unmap_page },
  { "__dma_sync_single_for_cpu",    (void *)kpi_dma_sync },
  { "__dma_sync_single_for_device", (void *)kpi_dma_sync },
  { "alloc_pages_noprof",      (void *)kpi_alloc_pages },
  { "page_frag_free",          (void *)kpi_page_frag_free },
  { "__netdev_alloc_frag_align", (void *)kpi_netdev_alloc_frag },
  { "page_offset_base",        (void *)&kpi_page_offset_base },
  { "vmemmap_base",            (void *)&kpi_vmemmap_base },
  { "__kmalloc_noprof",        (void *)kpi_kmalloc },
  { "vzalloc_noprof",          (void *)kpi_vmalloc },
  { "vmalloc_noprof",          (void *)kpi_vmalloc },
  { "vfree",                   (void *)kpi_vfree },
  { "is_vmalloc_addr",         (void *)kpi_is_vmalloc_addr },
  { "__kmalloc_cache_noprof",  (void *)kpi_kmalloc_cache },
  { "kmalloc_noprof",          (void *)kpi_kmalloc },
  { "kfree",                   (void *)kpi_kfree },
  { "krealloc_noprof",         (void *)kpi_krealloc },
@ -349,6 +702,8 @@ const struct kpi_sym kpi_table[] = {
  { "_dev_err",                (void *)kpi_netdev_print },
  // Packets
  { "napi_build_skb",          (void *)kpi_build_skb },
  { "build_skb",               (void *)kpi_build_skb },
  { "napi_alloc_skb",          (void *)kpi_napi_alloc_skb },
  { "__napi_alloc_skb",        (void *)kpi_napi_alloc_skb },
  { "__netdev_alloc_skb",      (void *)kpi_netdev_alloc_skb },
@ -358,15 +713,16 @@ const struct kpi_sym kpi_table[] = {
  { "eth_type_trans",          (void *)kpi_eth_type_trans },
  { "netif_receive_skb",       (void *)kpi_netif_receive_skb },
  { "napi_gro_receive",        (void *)kpi_netif_receive_skb },
  { "gro_receive_skb",         (void *)kpi_gro_receive_skb },
  { "dev_kfree_skb_any_reason",(void *)kpi_kfree_skb_reason },
  { "consume_skb",             (void *)kpi_kfree_skb },
  { "kfree_skb_reason",        (void *)kpi_kfree_skb_reason },
-  // Bringing the interface up. LDLS polls rather than taking interrupts, so
+  // Bringing the interface up. LDLS polls rather than taking interrupts, but
-  // request_irq succeeds and nothing is ever delivered through it -- see the
+  // the handler is kept and called from the loop -- see kpi_request_irq.
-  // note in cmd/ldlsd on why a poll loop is the honest shape here.
+  { "request_threaded_irq",    (void *)kpi_request_irq },
-  { "request_threaded_irq",    (void *)kpi_zero },
+  { "free_irq",                (void *)kpi_free_irq },
-  { "free_irq",                (void *)kpi_nop },
+  { "disable_hardirq",         (void *)kpi_nop },
  { "enable_irq",              (void *)kpi_nop },
  { "disable_irq_nosync",      (void *)kpi_nop },
  { "napi_enable",             (void *)kpi_nop },
--- a/ldls/kpi/kpi.h
+++ b/ldls/kpi/kpi.h
@ -78,8 +78,18 @@ struct kpi_layout {
  unsigned int nd_tx, nd_num_tx, nd_real_num_tx;
  unsigned int nq_size, nq_dev;
  unsigned int nd_state, nd_flags;
  unsigned int pd_dev, dev_kobj, kobj_name;
  unsigned int ndo_open, ndo_stop, ndo_start_xmit;
  // work_struct.func -- where a deferred work item keeps the function to call.
  // `struct delayed_work` starts with a work_struct, so one offset serves both.
  unsigned int work_func;
  // __LINK_STATE_NOCARRIER, from BTF. `netif_carrier_ok` is an inline that
  // tests this bit of net_device.state, so it is the one number that decides
  // whether a driver believes its link came up.
  unsigned int link_nocarrier;
  unsigned int skb_size;
  unsigned int skb_len, skb_data, skb_head, skb_tail, skb_end, skb_dev;
 };
@ -95,6 +105,29 @@ extern struct kpi_layout kpi_l;
 extern int  (*kpi_napi_poll)(void *napi, int budget);
 extern void *kpi_napi;
 // The driver's own interrupt handler, kept and called from the loop. See the
 // note in kpi.c: this is not about receiving packets, it is about everything a
 // driver only ever does from inside its handler.
 int kpi_request_irq(unsigned int irq, void *handler, void *thread_fn,
                    unsigned long flags, const char *name, void *dev);
 int kpi_free_irq(unsigned int irq, void *dev);
 int kpi_irq_poll(void);
 // A driver's deferred work, run from the same loop that polls. See work.c --
 // e1000 enables its transmitter from a watchdog, so this is not optional.
 int kpi_queue_work_on(int cpu, void *q, void *work);
 int kpi_queue_delayed_work_on(int cpu, void *q, void *dwork, unsigned long delay);
 int kpi_cancel_work_sync(void *work);
 int kpi_cancel_delayed_work_sync(void *dwork);
 int kpi_cancel_delayed_work(void *dwork);
 int kpi_flush_work(void *work);
 int kpi_work_run(void);
 // The clock those deadlines are measured against, and the `jiffies` a driver
 // reads directly.
 extern unsigned long kpi_jiffies;
 unsigned long kpi_jiffies_now(void);
 // Output. The KPI is inside a process with no libc, and every implementation
 // wants to be able to say what it did.
 void kpi_log(const char *s);
@ -105,6 +138,12 @@ void kpi_loghex(unsigned long long v, int digits);
 // default -- 538 of 8139too's 552 probe calls are ioread8/iowrite8.
 extern int kpi_verbose;
 // Memory a device can be pointed at, in pieces smaller than a PCIIOC_DMA call.
 // The skb path uses it, because a driver that maps an skb for its device needs
 // the bus address of the data. See the long note in kpi.c.
 void *kpi_dma_heap(unsigned long n, unsigned long align);
 unsigned long long kpi_phys_of(const void *va);
 // Is this a register address we handed out? Reports the first few that are
 // not, then goes quiet -- a driver that is off the rails produces thousands.
 int kpi_io_ok(void *addr, int width, int write);
--- a/ldls/kpi/net.c
+++ b/ldls/kpi/net.c
@ -241,3 +241,28 @@ kpi_unregister_netdev(void *dev)
    kpi.netfd = -1;
  }
 }
 // netif_carrier_off / netif_carrier_on.
 //
 // These were no-ops, and it cost a day. `netif_carrier_ok()` is an inline that
 // tests __LINK_STATE_NOCARRIER in `dev->state`, so with the bit never set a
 // driver reads "the carrier is already up" and skips its entire link-up path.
 // e1000's link-up path is where it enables the transmitter, so the visible
 // symptom was frames accepted and mapped and never sent -- and TCTL read back
 // from the chip with every field set but the enable.
 //
 // The bit number comes from the kernel's own BTF; see btf_enum_value().
 void
 kpi_netif_carrier_off(void *dev)
 {
  unsigned long *st = (unsigned long *)((char *)dev + kpi_l.nd_state);
  *st |= 1UL << kpi_l.link_nocarrier;
 }
 void
 kpi_netif_carrier_on(void *dev)
 {
  unsigned long *st = (unsigned long *)((char *)dev + kpi_l.nd_state);
  *st &= ~(1UL << kpi_l.link_nocarrier);
  kpi_log("kpi: the driver says the link is up\n");
 }
--- a/ldls/kpi/pci.c
+++ b/ldls/kpi/pci.c
@ -20,6 +20,11 @@
 #include "pciuser.h"
 #include "kpi.h"
 // The PCI resource flag bits, which are ABI between the kernel and every
 // driver written against it. Not layout, so BTF does not carry them.
 #define LDLS_IORESOURCE_IO  0x00000100
 #define LDLS_IORESOURCE_MEM 0x00000200
 // The device is claimed by the host before the driver runs, because claiming
 // is what enables memory space and bus mastering -- and a BAR read before that
 // can legitimately be zero. So by the time a driver calls this, it has already
@ -145,6 +150,57 @@ kpi_pci_iomap(void *pdev, int bar, unsigned long max)
  return p;
 }
 // pci_ioremap_bar(pdev, bar) -- the same thing as pci_iomap with no cap. A
 // second spelling, which is the sort of thing a second driver turns up.
 void *
 kpi_pci_ioremap_bar(void *pdev, int bar)
 {
  return kpi_pci_iomap(pdev, bar, 0);
 }
 // pci_select_bars(pdev, flags) -- which BARs match a resource kind.
 //
 // A driver asks this before requesting regions, so the answer decides which
 // BARs it thinks exist. Reporting one that is not there gets it refused later;
 // omitting one it needs gets it to give up here.
 int
 kpi_pci_select_bars(void *pdev, unsigned long flags)
 {
  int bars = 0, b;
  (void)pdev;
  for (b = 0; b < 6; b++) {
    if (!kpi.bar_base[b])
      continue;
    if ((kpi.bar_flags[b] & PCIUSER_BAR_IO)  && (flags & LDLS_IORESOURCE_IO))
      bars |= 1 << b;
    if ((kpi.bar_flags[b] & PCIUSER_BAR_MMIO) && (flags & LDLS_IORESOURCE_MEM))
      bars |= 1 << b;
  }
  return bars;
 }
 int
 kpi_pci_request_selected_regions(void *pdev, int bars, const char *name)
 {
  (void)pdev; (void)bars; (void)name;
  return 0;                        // nobody to arbitrate with; see above
 }
 void
 kpi_pci_release_selected_regions(void *pdev, int bars)
 {
  (void)pdev; (void)bars;
 }
 // Saving and restoring config space is what a driver does around a reset it
 // expects to clear the BARs. LDLS never resets the device at that level and
 // has no config write to restore with, so there is nothing to save -- and
 // saying so by succeeding is right, because a driver that cannot save state
 // refuses to probe at all.
 int  kpi_pci_save_state(void *pdev)    { (void)pdev; return 0; }
 void kpi_pci_restore_state(void *pdev) { (void)pdev; }
 void
 kpi_pci_iounmap(void *pdev, void *addr)
 {
--- a/ldls/kpi/skb.c
+++ b/ldls/kpi/skb.c
@ -23,6 +23,19 @@
 // somewhere to go.
 #define KPI_HEADROOM 64
 // Room past `end` for `struct skb_shared_info`.
 //
 // `skb_shinfo(skb)` is `(struct skb_shared_info *)(skb->head + skb->end)` --
 // it is not a field, it lives immediately after the data, and every allocator
 // in Linux reserves space for it. 8139too never looked, so an skb that ended
 // exactly at `end` worked; e1000 reads `nr_frags` on every single transmit,
 // and read it from whatever followed the allocation.
 //
 // The struct is 320 bytes on x86-64 and this is deliberately more. It is not
 // read from BTF only because getting it wrong in the small direction is
 // silent, and a fixed size that is comfortably too large cannot be.
 #define KPI_SHINFO 512
 static unsigned int
 rd32(void *base, unsigned int off)
 {
@ -67,15 +80,26 @@ void *
 kpi_skb_alloc(unsigned int len)
 {
  unsigned char *p;
-  unsigned int total = kpi_l.skb_size + KPI_HEADROOM + len + 64;
+  unsigned int total = kpi_l.skb_size + KPI_HEADROOM + len + 64 + KPI_SHINFO;
  unsigned char *buf;
  if (kpi_l.skb_size == 0)
    return 0;
-  p = mmap(0, total, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  // From the DMA arena, not from mmap.
-  if (p == 0 || (long)p == -1)
+  //
  // A driver that maps an skb for the device -- e1000 does, 8139too did not --
  // needs the bus address of its data, and a process can only know that for
  // memory the kernel handed it through PCIIOC_DMA. Putting the control block
  // in there too wastes a little arena and keeps one buffer one allocation.
  p = kpi_dma_heap(total, 64);
  if (p == 0)
    return 0;
  {
    unsigned int i;
    for (i = 0; i < total; i++)
      p[i] = 0;                  // the arena is recycled; mmap's zeroes are not
  }
  buf = p + kpi_l.skb_size;
  wrptr(p, kpi_l.skb_head, buf);
@ -86,6 +110,48 @@ kpi_skb_alloc(unsigned int len)
  return p;
 }
 // napi_build_skb(data, frag_len) -- an skb wrapped around a buffer that
 // already exists.
 //
 // The other half of the receive path, and the one that carries anything
 // larger than `copybreak` (256 bytes by default). e1000 hands the device a
 // raw fragment, and when a frame arrives it wraps that same fragment rather
 // than copying it -- so the control block is allocated here and the data is
 // not. A DHCP offer is 590 bytes, which is why nothing larger than a ping
 // reply worked until this existed.
 //
 // `skb_shared_info` lives at the *end of the buffer* in this arrangement,
 // which is what makes `end` smaller than the fragment: the driver picked a
 // fragment size that already includes room for it.
 void *
 kpi_build_skb(void *data, unsigned int frag_len)
 {
  unsigned char *p;
  unsigned int i, room;
  if (kpi_l.skb_size == 0 || data == 0 || frag_len <= KPI_SHINFO)
    return 0;
  p = kpi_dma_heap(kpi_l.skb_size, 64);
  if (!p)
    return 0;
  for (i = 0; i < kpi_l.skb_size; i++)
    p[i] = 0;
  room = frag_len - KPI_SHINFO;
  // And zero the shared info, because this buffer is recycled arena and the
  // driver reads nr_frags out of it without writing it first.
  for (i = 0; i < KPI_SHINFO; i++)
    ((unsigned char *)data)[room + i] = 0;
  wrptr(p, kpi_l.skb_head, data);
  wrptr(p, kpi_l.skb_data, data);
  wr32(p, kpi_l.skb_tail, 0);
  wr32(p, kpi_l.skb_end,  room);
  wr32(p, kpi_l.skb_len,  0);
  return p;
 }
 void *kpi_napi_alloc_skb(void *napi, unsigned int len)
 { (void)napi; return kpi_skb_alloc(len); }
@ -146,6 +212,18 @@ kpi_netif_receive_skb(void *skb)
  return 0;
 }
 // gro_receive_skb(gro, skb) -- the same thing, with the GRO node in front.
 //
 // `napi_gro_receive` is an inline wrapper in this kernel, so what a module
 // actually calls is this, and the skb is the *second* argument. Taking the
 // first would hand the stack a `struct gro_node` and call it a frame.
 int
 kpi_gro_receive_skb(void *gro, void *skb)
 {
  (void)gro;
  return kpi_netif_receive_skb(skb);
 }
 // eth_type_trans(skb, dev) -- consume the Ethernet header and report the
 // protocol. The stack below wants the whole frame including that header, so
 // unlike Linux's this does not pull it off; what it returns is the type field,
--- a/ldls/kpi/string.c
+++ b/ldls/kpi/string.c
@ -58,10 +58,33 @@ kpi_memcmp(const void *x, const void *y, unsigned long n)
  return 0;
 }
 // A null string is survivable, and reported.
 //
 // A hosted driver reads names out of structures this host only partly fills
 // in, and the first one it found was `pci_name(pdev)` -- a pointer into a
 // `struct device` that had never been given one. Faulting inside strnlen tells
 // you nothing about which field was missing; answering zero and saying so
 // leaves the driver running and names the problem.
 static long kpi_nullstr;
 static int
 str_ok(const char *s, const char *who)
 {
  if (s)
    return 1;
  if (++kpi_nullstr <= 4) {
    kpi_log("  kpi: "); kpi_log(who);
    kpi_log(" on a null string -- some field of a struct this host fills in "
            "only partly\n");
  }
  return 0;
 }
 unsigned long
 kpi_strlen(const char *s)
 {
  unsigned long n = 0;
  if (!str_ok(s, "strlen")) return 0;
  while (s[n]) n++;
  return n;
 }
@ -70,6 +93,7 @@ unsigned long
 kpi_strnlen(const char *s, unsigned long max)
 {
  unsigned long n = 0;
  if (!str_ok(s, "strnlen")) return 0;
  while (n < max && s[n]) n++;
  return n;
 }
@ -83,6 +107,7 @@ kpi_sized_strscpy(char *d, const char *s, unsigned long n)
  if (n == 0)
    return -7;                   /* -E2BIG */
  if (!str_ok(s, "strscpy")) { d[0] = 0; return 0; }
  while (i + 1 < n && s[i]) { d[i] = s[i]; i++; }
  d[i] = 0;
  return s[i] ? -7 : (long)i;
--- a/ldls/kpi/thunk.c
+++ b/ldls/kpi/thunk.c
@ -0,0 +1,71 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 // Retpoline thunks. See ldls/kpi/kpi.h for the provenance rule.
 //
 // These are not API. They are the compiler's, emitted by `-mretpoline` /
 // `-mfunction-return`, and a module built for a mitigated kernel is full of
 // them: every indirect call becomes `jmp __x86_indirect_thunk_rN` and every
 // return becomes `jmp __x86_return_thunk`.
 //
 // They have to be assembly, because what they are *is* control flow. A C
 // function cannot express "jump to whatever is in %r8 without touching any
 // other register", and the stub the loader supplies for an unimplemented
 // symbol -- which logs its name and returns zero -- is exactly the wrong
 // answer: it discards the jump.
 //
 // That cost a day, and the shape of the failure is the reason this file has a
 // long comment. e1000 refills its receive ring through a function pointer, and
 // the compiler made the call a *tail* call:
 //
 //     mov  0x2a0(%rbx),%r8        ; adapter->alloc_rx_buf
 //     ...                         ; compute E1000_DESC_UNUSED(ring)
 //     jmp  __x86_indirect_thunk_r8
 //
 // so the one thing in the log was `__x86_indirect_thunk_r8(...)` among four
 // thousand other stub calls, and the visible symptom was a NIC that
 // transmitted perfectly and received nothing, with RDT sitting at zero. The
 // driver had asked for 255 receive buffers and the shim had answered "yes".
 //
 // **These are the plain versions, with no mitigation in them.** A real
 // retpoline traps speculation in a loop; this is `jmp *%reg`. Spectre-v2
 // hardening for foreign code in a process is not something LDLS can offer
 // honestly -- the kernel it runs under is not mitigated either -- and a thunk
 // that pretends otherwise would be worse than one that says so here.
 // One per register, because the register is in the name -- that is the whole
 // point of the encoding. `endbr64` heads each one because a module built with
 // IBT reaches them by indirect jump.
 #define THUNK(reg) \
  ".globl kpi_thunk_" #reg "\n" \
  ".type kpi_thunk_" #reg ", @function\n" \
  "kpi_thunk_" #reg ":\n" \
  "  endbr64\n" \
  "  jmp *%" #reg "\n" \
  ".size kpi_thunk_" #reg ", .-kpi_thunk_" #reg "\n"
 __asm__(
  ".text\n"
  THUNK(rax) THUNK(rbx) THUNK(rcx) THUNK(rdx)
  THUNK(rsi) THUNK(rdi) THUNK(rbp)
  THUNK(r8)  THUNK(r9)  THUNK(r10) THUNK(r11)
  THUNK(r12) THUNK(r13) THUNK(r14) THUNK(r15)
  // __x86_return_thunk is just a return. It had been resolving to a stub,
  // which happened to work -- the stub is reached by `jmp`, so its own `ret`
  // returned to the right place -- while clobbering %rax with zero on the way
  // out. Every function whose return value came back as 0 for no reason was
  // this.
  ".globl kpi_return_thunk\n"
  ".type kpi_return_thunk, @function\n"
  "kpi_return_thunk:\n"
  "  ret\n"
  ".size kpi_return_thunk, .-kpi_return_thunk\n"
  // __fentry__ is the ftrace hook: on an untraced kernel it is a bare return,
  // and it is called at the top of essentially every function in the module.
  ".globl kpi_fentry\n"
  ".type kpi_fentry, @function\n"
  "kpi_fentry:\n"
  "  ret\n"
  ".size kpi_fentry, .-kpi_fentry\n"
 );
--- a/ldls/kpi/work.c
+++ b/ldls/kpi/work.c
@ -0,0 +1,142 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 // A driver's periodic work. See ldls/kpi/kpi.h for the provenance rule.
 //
 // e1000 is the reason this exists, and the way it failed is worth keeping.
 // The driver probed, opened, accepted frames from the stack, mapped them and
 // wrote the transmit tail register -- and the wire stayed silent. Reading the
 // chip's own registers said why:
 //
 //     TCTL=0x010000f8   PSP, collision threshold, retry-on-late-collision
 //                       -- and bit 1, the enable, clear
 //     RDT=0             no receive descriptors offered to the device
 //
 // Neither is set by `ndo_open`. e1000 enables its transmitter in
 // `e1000_watchdog`, once it has seen the link come up, and refills the receive
 // ring from the same place. The watchdog is a `delayed_work`, and LDLS was
 // dropping every one on the floor: `queue_delayed_work_on` resolved to a stub
 // that returned zero, which is the correct return value and does nothing.
 //
 // So a hosted driver needs somewhere for deferred work to actually run. There
 // is no thread to run it on, which is the same constraint that made the NAPI
 // poll a loop -- and the same answer: keep the function, and call it from the
 // loop that was going to spin anyway.
 //
 // **A work item here runs on the main loop, not concurrently.** That is why
 // there are no locks, and it is also the limitation: a driver whose work item
 // blocks stops everything, including the transmit path it is probably waiting
 // for.
 #include "types.h"
 #include "user.h"
 #include "posix.h"
 #include "kpi.h"
 #define KPI_NWORK 16
 static struct {
  void         *work;                  // struct work_struct *, our identity
  unsigned long due;                   // in jiffies
  int           armed;
 } wq[KPI_NWORK];
 static void
 rdfn(void *work, void (**fn)(void *))
 {
  unsigned char *p = (unsigned char *)work + kpi_l.work_func;
  int i;
  for (i = 0; i < 8; i++) ((unsigned char *)fn)[i] = p[i];
 }
 static int
 slot_for(void *work)
 {
  int i, free = -1;
  for (i = 0; i < KPI_NWORK; i++) {
    if (wq[i].armed && wq[i].work == work)
      return i;
    if (!wq[i].armed && free < 0)
      free = i;
  }
  return free;
 }
 static int
 arm(void *work, unsigned long delay)
 {
  int i;
  if (kpi_l.work_func == 0) {
    kpi_log("kpi: deferred work with no work_struct.func offset\n");
    return 0;
  }
  i = slot_for(work);
  if (i < 0) {
    kpi_log("kpi: too many deferred work items\n");
    return 0;
  }
  // Re-arming an item that is already queued moves its deadline, which is what
  // queue_delayed_work does *not* do -- it leaves the first deadline standing.
  // The difference only shows up as work running sooner than a driver asked,
  // and running a watchdog early is harmless in a way that running it late is
  // not.
  wq[i].work  = work;
  wq[i].due   = kpi_jiffies_now() + delay;
  wq[i].armed = 1;
  return 1;
 }
 // queue_work_on(cpu, wq, work)
 int kpi_queue_work_on(int cpu, void *q, void *work)
 { (void)cpu; (void)q; return arm(work, 0); }
 // queue_delayed_work_on(cpu, wq, dwork, delay)
 //
 // `struct delayed_work` begins with its `work_struct`, so the pointer is
 // already the right one -- which is the only reason this does not need a
 // second BTF offset.
 int
 kpi_queue_delayed_work_on(int cpu, void *q, void *dwork, unsigned long delay)
 { (void)cpu; (void)q; return arm(dwork, delay); }
 int
 kpi_cancel_work_sync(void *work)
 {
  int i = slot_for(work);
  if (i >= 0 && wq[i].armed && wq[i].work == work) { wq[i].armed = 0; return 1; }
  return 0;
 }
 int kpi_cancel_delayed_work_sync(void *dwork) { return kpi_cancel_work_sync(dwork); }
 int kpi_cancel_delayed_work(void *dwork)      { return kpi_cancel_work_sync(dwork); }
 int kpi_flush_work(void *work)                { (void)work; return 0; }
 void kpi_flush_workqueue(void *q)             { (void)q; }
 // Run whatever is due. Returns how many ran, so the caller's loop can tell
 // whether anything happened.
 //
 // The slot is disarmed *before* the call, because the overwhelming majority of
 // periodic work in a driver ends by queueing itself again -- and disarming
 // after would throw that away and stop the clock.
 int
 kpi_work_run(void)
 {
  unsigned long now = kpi_jiffies_now();
  int i, n = 0;
  for (i = 0; i < KPI_NWORK; i++) {
    void (*fn)(void *) = 0;
    void *w;
    if (!wq[i].armed || (long)(now - wq[i].due) < 0)
      continue;
    w = wq[i].work;
    wq[i].armed = 0;
    rdfn(w, &fn);
    if (!fn)
      continue;
    fn(w);
    n++;
  }
  return n;
 }