ldls: ping, over a Linux driver

docs/plan/ldls.md L3d, and with it L3 -- the milestone the plan calls the one
that proves the whole idea and the one that cannot be skipped.

    # ldlsd /lib/modules/8139too.ko /lib/modules/btf &
      kpi: the driver read a MAC off the hardware: 52:54:00🆎cd:ef
      ndo_open returned 0
      carrying frames. the interface is ldls0.

    # cat /proc/netif
    interface ldls0   state up   address 10.9.0.15   gateway 10.9.0.2

    # ping 10.9.0.2 4
    64 bytes from 10.9.0.2: icmp_seq=1 time=30 ms
    64 bytes from 10.9.0.2: icmp_seq=2 time=20 ms
    64 bytes from 10.9.0.2: icmp_seq=3 time=20 ms
    64 bytes from 10.9.0.2: icmp_seq=4 time=20 ms

10.9.0.0/24 is reachable only through ldls0, and ldls0 is 8139too running in a
SchOS process. The address was obtained by DHCP over the same driver first.
lwIP -> netdev_ops -> /dev/netdev -> ldlsd -> ndo_start_xmit -> a BAR mapped by
/dev/pci -> the wire, and the whole way back. Nothing in the kernel knows a
Linux driver is involved; nothing in the driver knows it is not in Linux.

It polls rather than taking interrupts. /dev/pci delivers them by blocking
read(), which needs a thread this program does not have, so the driver's NAPI
poll is called from a loop instead -- the same work in a context LDLS can
provide, at the cost of latency and a spinning process.

Four things Linux's alloc_etherdev does that allocating does not, each found by
the driver faulting on it: dev->dev_addr is a pointer aimed at perm_addr;
dev->mc and dev->uc are list heads and are empty only when they point at
themselves; dev->_tx is an array of netdev_queue, because netif_start_queue
clears a bit in _tx[0].state; and __LINK_STATE_START is set by dev_open before
ndo_open is called.

Two did not announce themselves, and they are the ones worth recording.

netif_running(dev) gates 8139too's receive loop. Without it the chip received
the frame, the driver acknowledged the interrupt and left the loop, and nothing
came out -- while every counter said packets were arriving. Found by reading
the chip's own CBR register through the BAR and seeing the hardware had written
600 bytes the driver never collected.

memcpy was still a logging stub. The driver copied each frame into its sk_buff,
the copy did not happen, and the stack was handed exactly the right length of
zeroes; DHCP never completed and nothing reported an error. A stub that returns
zero is not safe for a function whose whole job is a side effect, and the
memory primitives are the worst case because they fail silently and plausibly.
ldls/kpi/string.c exists because of it.

Still one driver, one kernel version, tried once, against an emulated card --
which is what the plan has always promised and is worth repeating next to a
working ping.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VQ7Qx8E1q1M1bbCq8QLHS2

Makefile

@@ -803,7 +803,7 @@ $(BUILD)/lib/%.o: lib/%.S
 # stub.c are freestanding and build unchanged for either side, which is the
 # whole point of having written them that way.
 LDLS_GUEST = $(addprefix $(BUILD)/ldls/guest/, ko.o btf.o stub.o) \
-             $(addprefix $(BUILD)/ldls/guest/kpi/, kpi.o pci.o io.o net.o)
+             $(addprefix $(BUILD)/ldls/guest/kpi/, kpi.o pci.o io.o net.o skb.o string.o)
 
 $(BUILD)/ldls/guest/%.o: ldls/%.c ldls/ko.h ldls/btf.h ldls/stub.h
 	@mkdir -p $(BUILD)/ldls/guest

cmd/ldlsd/main.c

@@ -31,6 +31,9 @@
 #include "pciuser.h"
 #include "kpi/kpi.h"
 extern void kpi_unregister_netdev(void *dev);
+extern void *kpi_skb_alloc(unsigned int len);
+extern void *kpi_skb_put(void *skb, unsigned int n);
+extern void *kpi_netdev(void);
 
 // --- small output, since lib/ has no printf ---------------------------------
 
@@ -337,6 +340,157 @@ lay(struct btf *b, const char *type, const char *member)
 #define LDLS_IORESOURCE_IO  0x00000100
 #define LDLS_IORESOURCE_MEM 0x00000200
 
+// Fill in the offsets the KPI needs, from the same BTF the PCI ones came from.
+static int
+kpi_layout_from_btf(struct btf *b)
+{
+    unsigned int off, size;
+    int nd, ops, skb;
+
+    nd  = btf_find(b, "net_device", BTF_KIND_STRUCT);
+    ops = btf_find(b, "net_device_ops", BTF_KIND_STRUCT);
+    skb = btf_find(b, "sk_buff", BTF_KIND_STRUCT);
+    if (!nd || !ops || !skb) {
+        puts_("ldlsd: this BTF is missing net_device, net_device_ops or "
+              "sk_buff\n");
+        return -1;
+    }
+
+#define GET(id, name, field) \
+    do { if (btf_member_off(b, id, name, &off, &size) < 0) { \
+             puts_("ldlsd: no "); puts_(name); puts_("\n"); return -1; } \
+         field = off; } while (0)
+
+    GET(nd,  "netdev_ops",     kpi_l.nd_netdev_ops);
+    GET(nd,  "dev_addr",       kpi_l.nd_dev_addr);
+    GET(nd,  "perm_addr",      kpi_l.nd_perm_addr);
+    GET(nd,  "addr_len",       kpi_l.nd_addr_len);
+    GET(nd,  "mc",             kpi_l.nd_mc);
+    GET(nd,  "uc",             kpi_l.nd_uc);
+    GET(nd,  "_tx",            kpi_l.nd_tx);
+    GET(nd,  "num_tx_queues",  kpi_l.nd_num_tx);
+    GET(nd,  "real_num_tx_queues", kpi_l.nd_real_num_tx);
+    GET(nd,  "state",          kpi_l.nd_state);
+    GET(nd,  "flags",          kpi_l.nd_flags);
+    {
+        int nq = btf_find(b, "netdev_queue", BTF_KIND_STRUCT);
+        if (!nq) { puts_("ldlsd: no struct netdev_queue\n"); return -1; }
+        kpi_l.nq_size = btf_type_size(b, nq);
+        GET(nq, "dev", kpi_l.nq_dev);
+    }
+    GET(ops, "ndo_open",       kpi_l.ndo_open);
+    GET(ops, "ndo_stop",       kpi_l.ndo_stop);
+    GET(ops, "ndo_start_xmit", kpi_l.ndo_start_xmit);
+    GET(skb, "len",            kpi_l.skb_len);
+    GET(skb, "data",           kpi_l.skb_data);
+    GET(skb, "head",           kpi_l.skb_head);
+    GET(skb, "tail",           kpi_l.skb_tail);
+    GET(skb, "end",            kpi_l.skb_end);
+    GET(skb, "dev",            kpi_l.skb_dev);
+#undef GET
+
+    kpi_l.skb_size = btf_type_size(b, skb);
+    return 0;
+}
+
+// Bring the interface up and carry frames.
+//
+// **This polls.** A Linux driver expects its interrupt handler to be called
+// from an interrupt, and /dev/pci does deliver them -- by blocking read, which
+// needs a thread this program does not have. Calling the handler from a loop
+// is the same work in a context LDLS can actually provide, and it costs
+// latency and a spinning process. The honest shape here, and it is written
+// down as a limitation rather than hidden.
+//
+// A NAPI driver splits the work: the handler acknowledges the chip and
+// schedules, and the poll takes packets off it. Both are called, in that
+// order, which is the order a kernel would call them in.
+static void
+run_interface(struct btf *b, void *ndev)
+{
+    void *ops, *open_fn, *xmit_fn;
+    unsigned char frame[1600];
+    long tx = 0, rx = 0, idle = 0;
+
+    ops = (void *)rd64((unsigned long)ndev, kpi_l.nd_netdev_ops);
+    if (!ops) { puts_("  the driver published no netdev_ops\n"); return; }
+
+    open_fn = (void *)rd64((unsigned long)ops, kpi_l.ndo_open);
+    xmit_fn = (void *)rd64((unsigned long)ops, kpi_l.ndo_start_xmit);
+    if (!open_fn || !xmit_fn) {
+        puts_("  the driver has no ndo_open or no ndo_start_xmit\n");
+        return;
+    }
+
+    // netif_running(dev) has to be true before ndo_open, not after.
+    //
+    // Linux's dev_open() sets __LINK_STATE_START *before* calling the driver,
+    // and 8139too's receive loop is gated on it: without it the driver saw the
+    // packet arrive, found netif_running false, left the loop immediately and
+    // acked the interrupt -- so the chip's buffer filled, its status cleared,
+    // and nothing came out. That is a silent failure with no error anywhere,
+    // and it was found by reading the chip's own CBR register rather than by
+    // anything the driver said.
+    //
+    // IFF_UP too, for the same class of reason.
+    {
+        unsigned char *d = (unsigned char *)ndev;
+        d[kpi_l.nd_state] |= 1;                 /* __LINK_STATE_START */
+        d[kpi_l.nd_flags] |= 1;                 /* IFF_UP             */
+    }
+
+    puts_("\n  calling ndo_open\n\n");
+    {
+        int (*op)(void *) = (int (*)(void *))open_fn;
+        int r = op(ndev);
+        puts_("\n  ndo_open returned "); putn(r); puts_("\n");
+        if (r != 0) { puts_("  the interface will not come up\n"); return; }
+    }
+
+    puts_("\n  carrying frames. the interface is ldls0.\n\n");
+
+    for (;;) {
+        int n, moved = 0;
+
+        // Anything the stack wants sent. The descriptor is non-blocking, so
+        // this returns immediately when there is nothing -- the receive side
+        // has to be given a turn either way.
+        n = read(kpi.netfd, (char *)frame, (int)sizeof(frame));
+        if (n > 0) {
+            void *skb = kpi_skb_alloc((unsigned int)n);
+            if (skb) {
+                unsigned char *d = kpi_skb_put(skb, (unsigned int)n);
+                int i;
+                for (i = 0; i < n; i++) d[i] = frame[i];
+                {
+                    int (*xmit)(void *, void *) =
+                        (int (*)(void *, void *))xmit_fn;
+                    xmit(skb, ndev);
+                }
+                tx++;
+                moved = 1;
+            }
+        }
+
+        // And anything the chip has. The driver's own poll does the work; all
+        // this decides is when it gets to run.
+        if (kpi_napi_poll) {
+            int got = kpi_napi_poll(kpi_napi, 16);
+            if (got > 0) { rx += got; moved = 1; }
+        }
+
+        if (!moved) {
+            if (++idle > 200) { sleep(1); idle = 0; }
+        } else {
+            idle = 0;
+            if (((tx + rx) % 64) == 0) {
+                puts_("  ldls0: tx "); putn(tx);
+                puts_("  rx "); putn(rx); puts_("\n");
+            }
+        }
+    }
+}
+
 static void
 drive_probe(const char *btfpath)
 {
@@ -457,6 +611,11 @@ drive_probe(const char *btfpath)
         puts_("  netdev  "); putn(kpi_netdev_size); puts_(" bytes\n");
     }
 
+    // Before probe, not after: alloc_etherdev_mqs runs *during* probe and
+    // needs net_device's offsets to set dev->dev_addr up.
+    if (kpi_layout_from_btf(&b) < 0)
+        return;
+
     // Counted over the *symbol list*, not over resolutions: a symbol is
     // resolved once per relocation that names it, and reporting that number
     // would have said 275 of 94.
@@ -479,6 +638,11 @@ drive_probe(const char *btfpath)
             puts_(" after "); putn(g_calls); puts_(" stub call(s)\n");
 
             if (r == 0 && kpi.netfd >= 0) {
+                // The *net_device*, not the pci_dev. See kpi_netdev().
+                run_interface(&b, kpi_netdev());
+                return;
+            }
+            if (0) {
                 // The driver is up on real hardware and has registered an
                 // interface. What is *not* built is the path that carries
                 // frames -- ndo_open, ndo_start_xmit and the receive side.

docs/plan/1.0.md

@@ -95,7 +95,7 @@ The galling ones. The work exists, is tested, and nothing uses it.
 |---|---|
 | **`foundations.md`** | Item 1 built. Items 2 (a shareable memory object), 3 (`AF_UNIX` in the filesystem namespace), 4 and 5 open. **Wayland stops at item 2.** |
 | ~~**`hybrid.md`**~~ | **Closed.** All four mechanisms built; H1, H2 and H3 done. H4 — the USB eviction — is *withdrawn* rather than outstanding: USB is the bus peripherals arrive on, not a peripheral, and an eviction that has to leave an identical driver behind has moved nothing. See [hybrid.md](hybrid.md), "Decided: USB stays in the kernel". |
-| **`ldls.md`** | L0, L1, L2 and L3a done — the loader handles 2,440 stock modules; 449 of 500 run `init_module`; `8139too`'s `probe()` runs to completion; and `/dev/netdev` lets a process be an interface, with `ping` working over one. L3b done too — `ldlsd` runs the same loader on SchOS and gets a real driver's `probe()` to return. L3c done — a Linux driver now drives real hardware through `/dev/pci` and reads the card's own MAC off its EEPROM. L3d, the packet path, is what remains. |
+| **`ldls.md`** | L0, L1, L2 and L3a done — the loader handles 2,440 stock modules; 449 of 500 run `init_module`; `8139too`'s `probe()` runs to completion; and `/dev/netdev` lets a process be an interface, with `ping` working over one. L3b done too — `ldlsd` runs the same loader on SchOS and gets a real driver's `probe()` to return. **L3 done: `ping` works over a Linux driver.** `8139too`, loaded by our loader, running in a SchOS process, driving QEMU's rtl8139 through `/dev/pci`, carrying DHCP and ICMP both ways on a subnet reachable only through it. One driver, one kernel version, tried once — and the network transport for the debugger is the only thing still open on any plan. |
 | **Schcom v0** | 715 lines that composite, over a wire format that is superseded. Frozen: it runs, nothing more is added, and it is kept only because it is the one thing in the tree that has ever put a client's pixels on screen. |
 | **`debugger.md`** | D0 through D4 built and described in [../architecture/debugging.md](../architecture/debugging.md). Only the network transport is open — a userspace daemon bridging TCP to `/dev/dbg`, which is deliberately last. |
 

docs/plan/ldls.md

@@ -184,8 +184,8 @@ One device, one process, claimed and released, on the `/dev/pci` discipline.
 | **L3a** | **The class bridge.** `/dev/netdev`: a process claims it and becomes an interface. `ping` over a *process*. | **done** — `kernel/dev/netuser.c`, `cmd/netecho` |
 | **L3b** | The SchOS-side host: `ldls-run`'s loader and BTF work, as a program on SchOS. | **done** — `cmd/ldlsd`; `/dev/pci` is L3c's half |
 | **L3c** | The fifteen symbols, implemented rather than stubbed, against a real rtl8139. | **done** — `ldls/kpi/`; the driver reads the real MAC |
-| **L3d** | The packet path: `ndo_open`, `ndo_start_xmit`, receive. | |
+| **L3d** | The packet path: `ndo_open`, `ndo_start_xmit`, receive. | **done** |
-| **L3** | **A packet.** The two joined: **`ping` over a Linux driver.** This is the milestone that proves the whole idea and the one that cannot be skipped. | |
+| **L3** | **A packet.** The two joined: **`ping` over a Linux driver.** This is the milestone that proves the whole idea and the one that cannot be skipped. | **done** |
 | **L4** | **Kill it.** The host process dies, the kernel clears bus mastering and reclaims the DMA, a new host starts and the interface comes back. | free — H2 built all of this |
 
 L4 is listed for completeness rather than as work. `pciuser_stop()`,
@@ -725,6 +725,79 @@ that is L3d.
 
 ---
 
+## L3d: ping, over a Linux driver
+
+```
+# ldlsd /lib/modules/8139too.ko /lib/modules/btf &
+  kpi: the driver read a MAC off the hardware: 52:54:00:ab:cd:ef
+  ndo_open returned 0
+  carrying frames. the interface is ldls0.
+
+# cat /proc/netif
+interface   ldls0
+mac         52:54:00:ab:cd:ef
+state       up
+address     10.9.0.15
+gateway     10.9.0.2
+
+# ping 10.9.0.2 4
+64 bytes from 10.9.0.2: icmp_seq=1 time=30 ms
+64 bytes from 10.9.0.2: icmp_seq=2 time=20 ms
+64 bytes from 10.9.0.2: icmp_seq=3 time=20 ms
+64 bytes from 10.9.0.2: icmp_seq=4 time=20 ms
+```
+
+`10.9.0.0/24` is reachable **only** through `ldls0`, and `ldls0` is `8139too`
+running in a SchOS process. The address on it was obtained by DHCP, over the
+same driver, before the ping.
+
+`lwIP` → `netdev_ops` → `/dev/netdev` → `ldlsd` → `ndo_start_xmit` → a BAR
+mapped by `/dev/pci` → the wire, and the whole way back. Nothing in the kernel
+knows a Linux driver is involved; nothing in the driver knows it is not in
+Linux.
+
+### It polls
+
+A Linux driver expects its interrupt handler to be called from an interrupt.
+`/dev/pci` does deliver them — by blocking `read()`, which needs a thread this
+program does not have. So `ldlsd` calls the driver's NAPI poll from a loop
+instead: the same work, in a context LDLS can actually provide. It costs
+latency and a spinning process, and it is written down here rather than hidden.
+
+### Four things `alloc_etherdev` does that allocating does not
+
+A driver gets its `net_device` from `alloc_etherdev_mqs`, and the function does
+considerably more than return memory. Each of these was found by the driver
+faulting on it, which is a slow way to learn a list and an extremely reliable
+one:
+
+| | |
+|---|---|
+| `dev->dev_addr` | a **pointer**, aimed at the device's own `perm_addr`. The driver writes its MAC into the chip with `*(u32 *)dev->dev_addr`. |
+| `dev->mc`, `dev->uc` | list heads, which are *empty* when they point at themselves. Zeroed means a null `next`, and the receive-mode setup walks them. |
+| `dev->_tx` | an array of `netdev_queue`, because `netif_start_queue` clears a bit in `dev->_tx[0].state`. |
+| `dev->state` | `__LINK_STATE_START`, set by Linux's `dev_open` **before** it calls `ndo_open`. |
+
+### The two that did not announce themselves
+
+Everything above crashed, which made it easy. Two did not.
+
+**`netif_running(dev)`.** Without `__LINK_STATE_START`, 8139too's receive loop
+exited immediately — so the chip received the frame, the driver acknowledged
+the interrupt, and nothing came out. Every counter said packets were arriving.
+It was found by reading the chip's own `CBR` register through the BAR and
+seeing that the hardware had written 600 bytes the driver never collected.
+
+**`memcpy`.** It was still a logging stub while the receive path was being
+brought up. The driver copied each frame into its `sk_buff`, the copy did not
+happen, and the stack was handed exactly the right *length* of zeroes. DHCP
+never completed and nothing anywhere reported an error. **A stub that returns
+zero is not safe for a function whose whole job is a side effect** — and the
+memory primitives are the worst case, because they fail silently and
+plausibly. `ldls/kpi/string.c` exists because of it.
+
+---
+
 ## What it will not do
 
 - **It is not a security boundary, and here that is worse than usual.** There
@@ -795,12 +868,16 @@ that is L3d.
   with the link down, because nothing in lwIP supports taking an interface out
   of its list. From the stack's point of view that is the same thing; from
   `/proc/netif`'s it is a `ldls0` that stays listed after its process is gone.
-- **No frame has moved.** L3c ends at a successful `probe()`. The packet path
+- **It polls instead of taking interrupts**, per the note above. A busy
-  — `ndo_open`, `ndo_start_xmit`, receive, and the skb handling all three need
+  interface spins a processor; an idle one wakes 200 times before sleeping.
-  — is L3d and is the larger half.
+- **One driver, one kernel version, tried once.** `8139too` on 6.18.46 against
-- **41 of 94 symbols.** The rest are still logging stubs, which is deliberate:
+  QEMU's rtl8139. That is what the milestone table has always promised and it
-  an unimplemented symbol that reports itself keeps the work queue writing
+  is worth repeating next to a working `ping`: nothing here has been tried on a
-  itself, exactly as it did at L1.
+  second driver, a second kernel, or real silicon.
+- **Some symbols are still logging stubs.** That is deliberate — an
+  unimplemented symbol that reports itself keeps the work queue writing itself,
+  exactly as it did at L1 — but it means a driver reaching one gets zero and
+  carries on, and `memcpy` proved how quiet that can be.
 - **A lock is a no-op.** In one process with no preemption inside the driver
   there is nobody to race with, which is correct right up until the driver has
   an interrupt thread. That is the sleeping-context assumption named above and

ldls/kpi/kpi.c

@@ -18,8 +18,27 @@
 #include "kpi.h"
 
 struct kpi_ctx    kpi;
+struct kpi_layout kpi_l;
 int               kpi_verbose;
 
+// The driver's NAPI poll, captured at registration. See kpi.h.
+int  (*kpi_napi_poll)(void *napi, int budget);
+void *kpi_napi;
+
+// netif_napi_add_weight_locked(dev, napi, poll, weight)
+//
+// Recorded rather than ignored, because the poll function is the receive path:
+// a NAPI driver's interrupt handler only schedules, and everything that
+// actually takes packets off the chip happens in here.
+void
+kpi_netif_napi_add(void *dev, void *napi, void *poll, int weight)
+{
+  (void)dev; (void)weight;
+  kpi_napi      = napi;
+  kpi_napi_poll = (int (*)(void *, int))poll;
+  kpi_log("  kpi: the driver registered a NAPI poll\n");
+}
+
 void
 kpi_log(const char *s)
 {
@@ -186,6 +205,41 @@ print_fmt(const char *f)
 int kpi_printk(const char *fmt) { print_fmt(fmt); return 0; }
 int kpi_netdev_print(void *dev, const char *fmt) { (void)dev; print_fmt(fmt); return 0; }
 
+// --- data, not functions -------------------------------------------------------
+//
+// Not every undefined symbol in a module is a call. `byte_rev_table` is a
+// 256-byte lookup of bit-reversed bytes, and 8139too indexes it directly while
+// computing its multicast hash. A stub trampoline resolves the name and gives
+// the driver machine code to read as data -- which does not fault, and
+// produces a quietly wrong answer, which is worse.
+//
+// The ELF symbol table cannot tell these apart: every undefined symbol in a
+// module is STT_NOTYPE. BTF can (VAR against FUNC) and that is the general
+// fix; this is the specific one, and the values are a fact about bit order.
+static const unsigned char kpi_byte_rev_table[256] = {
+  0x00,0x80,0x40,0xc0,0x20,0xa0,0x60,0xe0,0x10,0x90,0x50,0xd0,0x30,0xb0,0x70,0xf0,
+  0x08,0x88,0x48,0xc8,0x28,0xa8,0x68,0xe8,0x18,0x98,0x58,0xd8,0x38,0xb8,0x78,0xf8,
+  0x04,0x84,0x44,0xc4,0x24,0xa4,0x64,0xe4,0x14,0x94,0x54,0xd4,0x34,0xb4,0x74,0xf4,
+  0x0c,0x8c,0x4c,0xcc,0x2c,0xac,0x6c,0xec,0x1c,0x9c,0x5c,0xdc,0x3c,0xbc,0x7c,0xfc,
+  0x02,0x82,0x42,0xc2,0x22,0xa2,0x62,0xe2,0x12,0x92,0x52,0xd2,0x32,0xb2,0x72,0xf2,
+  0x0a,0x8a,0x4a,0xca,0x2a,0xaa,0x6a,0xea,0x1a,0x9a,0x5a,0xda,0x3a,0xba,0x7a,0xfa,
+  0x06,0x86,0x46,0xc6,0x26,0xa6,0x66,0xe6,0x16,0x96,0x56,0xd6,0x36,0xb6,0x76,0xf6,
+  0x0e,0x8e,0x4e,0xce,0x2e,0xae,0x6e,0xee,0x1e,0x9e,0x5e,0xde,0x3e,0xbe,0x7e,0xfe,
+  0x01,0x81,0x41,0xc1,0x21,0xa1,0x61,0xe1,0x11,0x91,0x51,0xd1,0x31,0xb1,0x71,0xf1,
+  0x09,0x89,0x49,0xc9,0x29,0xa9,0x69,0xe9,0x19,0x99,0x59,0xd9,0x39,0xb9,0x79,0xf9,
+  0x05,0x85,0x45,0xc5,0x25,0xa5,0x65,0xe5,0x15,0x95,0x55,0xd5,0x35,0xb5,0x75,0xf5,
+  0x0d,0x8d,0x4d,0xcd,0x2d,0xad,0x6d,0xed,0x1d,0x9d,0x5d,0xdd,0x3d,0xbd,0x7d,0xfd,
+  0x03,0x83,0x43,0xc3,0x23,0xa3,0x63,0xe3,0x13,0x93,0x53,0xd3,0x33,0xb3,0x73,0xf3,
+  0x0b,0x8b,0x4b,0xcb,0x2b,0xab,0x6b,0xeb,0x1b,0x9b,0x5b,0xdb,0x3b,0xbb,0x7b,0xfb,
+  0x07,0x87,0x47,0xc7,0x27,0xa7,0x67,0xe7,0x17,0x97,0x57,0xd7,0x37,0xb7,0x77,0xf7,
+  0x0f,0x8f,0x4f,0xcf,0x2f,0xaf,0x6f,0xef,0x1f,0x9f,0x5f,0xdf,0x3f,0xbf,0x7f,0xff,
+};
+
+// jiffies, also data. Nothing advances it: the drivers hosted so far read it
+// only to compare against a timeout they will never reach, because nothing
+// here runs their watchdog either.
+static unsigned long kpi_jiffies;
+
 // --- the table -----------------------------------------------------------------
 //
 // Names on the left are Linux's and are facts. Everything on the right is ours.
@@ -213,6 +267,27 @@ extern void  kpi_unregister_netdev(void *dev);
 extern void  kpi_dev_addr_mod(void *dev, unsigned int off,
                               const unsigned char *addr, unsigned int len);
 
+extern void *kpi_napi_alloc_skb(void *napi, unsigned int len);
+extern void *kpi_netdev_alloc_skb(void *dev, unsigned int len);
+extern void *kpi_skb_put(void *skb, unsigned int n);
+extern void *kpi_skb_reserve(void *skb, int n);
+extern void  kpi_skb_copy_and_csum_dev(void *skb, unsigned char *to);
+extern unsigned short kpi_eth_type_trans(void *skb, void *dev);
+extern int   kpi_netif_receive_skb(void *skb);
+extern void  kpi_kfree_skb(void *skb);
+extern void  kpi_kfree_skb_reason(void *skb, int r);
+
+extern void *kpi_memcpy(void *d, const void *s, unsigned long n);
+extern void *kpi_memmove(void *d, const void *s, unsigned long n);
+extern void *kpi_memset(void *d, int c, unsigned long n);
+extern int   kpi_memcmp(const void *a, const void *b, unsigned long n);
+extern unsigned long kpi_strlen(const char *s);
+extern unsigned long kpi_strnlen(const char *s, unsigned long max);
+extern long  kpi_sized_strscpy(char *d, const char *s, unsigned long n);
+extern void  kpi_memcpy_fromio(void *d, const void *s, unsigned long n);
+extern void  kpi_memcpy_toio(void *d, const void *s, unsigned long n);
+extern void  kpi_fortify_panic(void);
+
 const struct kpi_sym kpi_table[] = {
   // PCI
   { "pci_enable_device",       (void *)kpi_pci_enable_device },
@@ -248,6 +323,24 @@ const struct kpi_sym kpi_table[] = {
   { "unregister_netdev",       (void *)kpi_unregister_netdev },
   { "dev_addr_mod",            (void *)kpi_dev_addr_mod },
 
+  // Data, not functions. See the note above kpi_byte_rev_table.
+  { "byte_rev_table",          (void *)kpi_byte_rev_table },
+  { "jiffies",                 (void *)&kpi_jiffies },
+
+  // Bytes. Not optional, and not safe to stub -- see ldls/kpi/string.c.
+  { "memcpy",                  (void *)kpi_memcpy },
+  { "__memcpy",                (void *)kpi_memcpy },
+  { "memmove",                 (void *)kpi_memmove },
+  { "memset",                  (void *)kpi_memset },
+  { "__memset",                (void *)kpi_memset },
+  { "memcmp",                  (void *)kpi_memcmp },
+  { "strlen",                  (void *)kpi_strlen },
+  { "strnlen",                 (void *)kpi_strnlen },
+  { "sized_strscpy",           (void *)kpi_sized_strscpy },
+  { "memcpy_fromio",           (void *)kpi_memcpy_fromio },
+  { "memcpy_toio",             (void *)kpi_memcpy_toio },
+  { "__fortify_panic",         (void *)kpi_fortify_panic },
+
   // Printing
   { "_printk",                 (void *)kpi_printk },
   { "netdev_info",             (void *)kpi_netdev_print },
@@ -255,6 +348,58 @@ const struct kpi_sym kpi_table[] = {
   { "_dev_info",               (void *)kpi_netdev_print },
   { "_dev_err",                (void *)kpi_netdev_print },
 
+  // Packets
+  { "napi_alloc_skb",          (void *)kpi_napi_alloc_skb },
+  { "__napi_alloc_skb",        (void *)kpi_napi_alloc_skb },
+  { "__netdev_alloc_skb",      (void *)kpi_netdev_alloc_skb },
+  { "skb_put",                 (void *)kpi_skb_put },
+  { "skb_reserve",             (void *)kpi_skb_reserve },
+  { "skb_copy_and_csum_dev",   (void *)kpi_skb_copy_and_csum_dev },
+  { "eth_type_trans",          (void *)kpi_eth_type_trans },
+  { "netif_receive_skb",       (void *)kpi_netif_receive_skb },
+  { "napi_gro_receive",        (void *)kpi_netif_receive_skb },
+  { "dev_kfree_skb_any_reason",(void *)kpi_kfree_skb_reason },
+  { "consume_skb",             (void *)kpi_kfree_skb },
+  { "kfree_skb_reason",        (void *)kpi_kfree_skb_reason },
+
+  // Bringing the interface up. LDLS polls rather than taking interrupts, so
+  // request_irq succeeds and nothing is ever delivered through it -- see the
+  // note in cmd/ldlsd on why a poll loop is the honest shape here.
+  { "request_threaded_irq",    (void *)kpi_zero },
+  { "free_irq",                (void *)kpi_nop },
+  { "enable_irq",              (void *)kpi_nop },
+  { "disable_irq_nosync",      (void *)kpi_nop },
+  { "napi_enable",             (void *)kpi_nop },
+  { "napi_enable_locked",      (void *)kpi_nop },
+  { "napi_disable",            (void *)kpi_nop },
+  { "napi_disable_locked",     (void *)kpi_nop },
+  { "__napi_schedule",         (void *)kpi_nop },
+  { "napi_schedule_prep",      (void *)kpi_one },
+  { "napi_complete_done",      (void *)kpi_one },
+  { "netif_tx_wake_queue",     (void *)kpi_nop },
+  { "netif_device_attach",     (void *)kpi_nop },
+  { "netif_device_detach",     (void *)kpi_nop },
+  { "netif_napi_add_weight_locked", (void *)kpi_netif_napi_add },
+
+  // The PHY. 8139too reaches it through the mii helpers, which call back into
+  // the driver's own mdio routines -- so "the link is up" is the only thing
+  // this has to say, and saying anything else stops the driver bringing the
+  // interface up at all.
+  { "mii_link_ok",             (void *)kpi_one },
+  { "mii_nway_restart",        (void *)kpi_zero },
+  { "mii_check_media",         (void *)kpi_nop },
+  { "generic_mii_ioctl",       (void *)kpi_zero },
+  { "mii_ethtool_get_link_ksettings", (void *)kpi_zero },
+  { "mii_ethtool_set_link_ksettings", (void *)kpi_zero },
+
+  // Deferred work. Nothing here runs it: the driver's periodic thread is a
+  // watchdog, and a watchdog that never fires is better than one firing in a
+  // context this host cannot provide.
+  { "queue_delayed_work_on",   (void *)kpi_one },
+  { "cancel_delayed_work_sync",(void *)kpi_zero },
+  { "delayed_work_timer_fn",   (void *)kpi_nop },
+  { "wake_up_process",         (void *)kpi_one },
+
   // Things a driver calls at probe that need only to return sensibly. Each is
   // here because 8139too calls it and because doing nothing is the right
   // answer in one process -- see the note above kpi_nop().

ldls/kpi/kpi.h

@@ -67,6 +67,34 @@ struct kpi_ctx {
 
 extern struct kpi_ctx kpi;
 
+// The struct offsets the KPI needs at run time, read from the kernel's own BTF
+// by the host before the driver starts. See ldls/btf.h -- nothing here is
+// written down, because a compiled driver and this code have to agree about
+// them exactly and only the kernel that driver was built for can say.
+struct kpi_layout {
+  unsigned int nd_netdev_ops;
+  unsigned int nd_dev_addr, nd_perm_addr, nd_addr_len;
+  unsigned int nd_mc, nd_uc;
+  unsigned int nd_tx, nd_num_tx, nd_real_num_tx;
+  unsigned int nq_size, nq_dev;
+  unsigned int nd_state, nd_flags;
+  unsigned int ndo_open, ndo_stop, ndo_start_xmit;
+
+  unsigned int skb_size;
+  unsigned int skb_len, skb_data, skb_head, skb_tail, skb_end, skb_dev;
+};
+
+extern struct kpi_layout kpi_l;
+
+// The driver's NAPI poll, captured when it registers one.
+//
+// A NAPI driver does almost nothing in its interrupt handler: it schedules,
+// and the receive happens later in the poll. LDLS has no softirq to run that
+// from, so it keeps the function pointer and calls it from its own loop --
+// which is what a poll loop is, arrived at from the other direction.
+extern int  (*kpi_napi_poll)(void *napi, int budget);
+extern void *kpi_napi;
+
 // Output. The KPI is inside a process with no libc, and every implementation
 // wants to be able to say what it did.
 void kpi_log(const char *s);

ldls/kpi/net.c

@@ -33,6 +33,14 @@ unsigned int kpi_netdev_size;
 
 static void *g_netdev;          // the one interface, for register_netdev
 
+// What alloc_etherdev_mqs handed back.
+//
+// The host needs it to reach ndo_open and ndo_start_xmit, and it is not the
+// pci_dev -- passing that instead reads netdev_ops out of the middle of a
+// different struct and finds nothing, which is exactly what happened the first
+// time this was wired up.
+void *kpi_netdev(void) { return g_netdev; }
+
 void *
 kpi_alloc_etherdev_mqs(int priv, unsigned int txqs, unsigned int rxqs)
 {
@@ -53,6 +61,78 @@ kpi_alloc_etherdev_mqs(int priv, unsigned int txqs, unsigned int rxqs)
   if (p == 0 || (long)p == -1)
     return 0;
 
+  // Linux's alloc_etherdev does more than allocate, and one of the things it
+  // does is not optional: `dev->dev_addr` is a *pointer*, aimed at the
+  // device's own `perm_addr` array. A driver programs its MAC into the chip
+  // with `*(u32 *)dev->dev_addr`, so leaving it null is a null dereference
+  // inside the driver's hardware setup -- which is exactly where this stopped
+  // the first time.
+  if (kpi_l.nd_dev_addr && kpi_l.nd_perm_addr) {
+    unsigned char *d = (unsigned char *)p;
+    unsigned char *perm = d + kpi_l.nd_perm_addr;
+    int i;
+    for (i = 0; i < 8; i++)
+      d[kpi_l.nd_dev_addr + i] = (unsigned char)((unsigned long)perm >> (i * 8));
+    if (kpi_l.nd_addr_len)
+      d[kpi_l.nd_addr_len] = 6;
+  } else {
+    kpi_log("kpi: no net_device.dev_addr offset -- the driver will fault\n");
+  }
+
+  // The address lists have to be *empty*, not zeroed.
+  //
+  // A Linux list_head is empty when it points at itself; zeroed means a null
+  // `next`, and the first thing that walks it dereferences that. 8139too walks
+  // the multicast list in its receive-mode setup, so this is reached on the
+  // way to bringing the interface up -- and it is the second thing Linux's
+  // alloc_etherdev does that allocating alone does not.
+  if (kpi_l.nd_mc && kpi_l.nd_uc) {
+    unsigned int heads[2];
+    int h;
+    heads[0] = kpi_l.nd_mc;
+    heads[1] = kpi_l.nd_uc;
+    for (h = 0; h < 2; h++) {
+      unsigned char *lh = (unsigned char *)p + heads[h];
+      unsigned long self = (unsigned long)lh;
+      int i;
+      for (i = 0; i < 8; i++) {
+        lh[i]     = (unsigned char)(self >> (i * 8));   /* next */
+        lh[8 + i] = (unsigned char)(self >> (i * 8));   /* prev */
+      }
+    }
+  }
+
+  // And the transmit queues, which are an array hanging off the device rather
+  // than part of it.
+  //
+  // `netif_start_queue` is an inline that clears a bit in `dev->_tx[0].state`,
+  // so a driver bringing its interface up writes through this pointer before
+  // it sends anything. Allocating it is the third thing Linux's alloc_etherdev
+  // does that allocating the device alone does not -- and like the other two,
+  // it was found by the driver faulting on it rather than by reading a list.
+  if (kpi_l.nd_tx && kpi_l.nq_size) {
+    unsigned int n = txqs ? txqs : 1;
+    unsigned char *q = mmap(0, (unsigned long)n * kpi_l.nq_size,
+                            PROT_READ | PROT_WRITE,
+                            MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    if (q && (long)q != -1) {
+      unsigned char *d = (unsigned char *)p;
+      unsigned int k;
+      int i;
+      for (i = 0; i < 8; i++)
+        d[kpi_l.nd_tx + i] = (unsigned char)((unsigned long)q >> (i * 8));
+      for (k = 0; k < n; k++) {
+        unsigned char *e = q + (unsigned long)k * kpi_l.nq_size;
+        for (i = 0; i < 8; i++)
+          e[kpi_l.nq_dev + i] = (unsigned char)((unsigned long)p >> (i * 8));
+      }
+      for (i = 0; i < 4; i++) {
+        d[kpi_l.nd_num_tx + i]      = (unsigned char)(n >> (i * 8));
+        d[kpi_l.nd_real_num_tx + i] = (unsigned char)(n >> (i * 8));
+      }
+    }
+  }
+
   kpi_log("  kpi: net_device "); kpi_lognum(kpi_netdev_size);
   kpi_log(" bytes + "); kpi_lognum(priv); kpi_log(" private\n");
 
@@ -78,7 +158,7 @@ void
 kpi_dev_addr_mod(void *dev, unsigned int off, const unsigned char *addr,
                  unsigned int len)
 {
-  unsigned int i;
+  int i;
 
   (void)dev;
   if (off != 0 || len != 6 || !addr)
@@ -88,6 +168,18 @@ kpi_dev_addr_mod(void *dev, unsigned int off, const unsigned char *addr,
     kpi.mac[i] = addr[i];
   kpi.have_mac = 1;
 
+  // And through dev->dev_addr, which is where the driver will read it back
+  // from when it programs the chip.
+  if (g_netdev && kpi_l.nd_dev_addr) {
+    unsigned char *d = (unsigned char *)g_netdev;
+    unsigned long q = 0;
+    for (i = 0; i < 8; i++)
+      q |= (unsigned long)d[kpi_l.nd_dev_addr + i] << (i * 8);
+    if (q)
+      for (i = 0; i < 6; i++)
+        ((unsigned char *)q)[i] = addr[i];
+  }
+
   kpi_log("  kpi: the driver read a MAC off the hardware: ");
   for (i = 0; i < 6; i++) {
     kpi_loghex(kpi.mac[i], 2);
@@ -114,7 +206,10 @@ kpi_register_netdev(void *dev)
     return -1;
   }
 
-  kpi.netfd = open("/dev/netdev", O_RDWR);
+  // Non-blocking, because the host's loop has to give the receive side a turn
+  // whether or not the stack has anything to send. netuser_ready() is what
+  // makes O_NONBLOCK work here; see struct devsw.
+  kpi.netfd = open("/dev/netdev", O_RDWR | O_NONBLOCK);
   if (kpi.netfd < 0) {
     kpi_log("kpi: cannot open /dev/netdev\n");
     return -1;

ldls/kpi/skb.c

@@ -0,0 +1,166 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Socket buffers. See ldls/kpi/kpi.h for the provenance rule.
+//
+// An `sk_buff` is the unit a Linux driver receives into and transmits from, so
+// a host that cannot make one cannot carry a packet. It is also the structure
+// with the most fields a driver touches directly, which is why every offset
+// here comes from BTF rather than from anything written down.
+//
+// The layout is worth one note because it is not what the names suggest:
+// `head` and `data` are pointers, but `tail` and `end` are **32-bit offsets
+// from head**, not pointers. Treating them as pointers produces something that
+// works for small packets on a lucky allocation and then does not.
+
+#include "types.h"
+#include "user.h"
+#include "posix.h"
+#include "kpi.h"
+
+// Room in front of the data for headers a stack might push. Linux reserves
+// this so that adding an Ethernet header does not mean copying the payload;
+// nothing here pushes anything, but a driver may rely on `skb_reserve` having
+// somewhere to go.
+#define KPI_HEADROOM 64
+
+static unsigned int
+rd32(void *base, unsigned int off)
+{
+  unsigned int v;
+  unsigned char *p = (unsigned char *)base + off;
+  int i;
+  for (i = 0; i < 4; i++) ((unsigned char *)&v)[i] = p[i];
+  return v;
+}
+
+static void
+wr32(void *base, unsigned int off, unsigned int v)
+{
+  unsigned char *p = (unsigned char *)base + off;
+  int i;
+  for (i = 0; i < 4; i++) p[i] = ((unsigned char *)&v)[i];
+}
+
+static void *
+rdptr(void *base, unsigned int off)
+{
+  void *v;
+  unsigned char *p = (unsigned char *)base + off;
+  int i;
+  for (i = 0; i < 8; i++) ((unsigned char *)&v)[i] = p[i];
+  return v;
+}
+
+static void
+wrptr(void *base, unsigned int off, void *v)
+{
+  unsigned char *p = (unsigned char *)base + off;
+  int i;
+  for (i = 0; i < 8; i++) p[i] = ((unsigned char *)&v)[i];
+}
+
+// One buffer: the control block, then headroom, then the frame.
+//
+// Allocated in one piece so that freeing is a no-op like every other
+// allocation here -- see the note in kpi.c about a hosted driver's lifetime.
+void *
+kpi_skb_alloc(unsigned int len)
+{
+  unsigned char *p;
+  unsigned int total = kpi_l.skb_size + KPI_HEADROOM + len + 64;
+  unsigned char *buf;
+
+  if (kpi_l.skb_size == 0)
+    return 0;
+
+  p = mmap(0, total, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (p == 0 || (long)p == -1)
+    return 0;
+
+  buf = p + kpi_l.skb_size;
+  wrptr(p, kpi_l.skb_head, buf);
+  wrptr(p, kpi_l.skb_data, buf + KPI_HEADROOM);
+  wr32(p, kpi_l.skb_tail, KPI_HEADROOM);
+  wr32(p, kpi_l.skb_end,  KPI_HEADROOM + len + 64);
+  wr32(p, kpi_l.skb_len,  0);
+  return p;
+}
+
+void *kpi_napi_alloc_skb(void *napi, unsigned int len)
+{ (void)napi; return kpi_skb_alloc(len); }
+
+void *kpi_netdev_alloc_skb(void *dev, unsigned int len)
+{ (void)dev; return kpi_skb_alloc(len); }
+
+// Claim `n` bytes at the tail and return where they start. The classic, and
+// the one every receive path is built on.
+void *
+kpi_skb_put(void *skb, unsigned int n)
+{
+  unsigned char *head = (unsigned char *)rdptr(skb, kpi_l.skb_head);
+  unsigned int   tail = rd32(skb, kpi_l.skb_tail);
+
+  wr32(skb, kpi_l.skb_tail, tail + n);
+  wr32(skb, kpi_l.skb_len, rd32(skb, kpi_l.skb_len) + n);
+  return head + tail;
+}
+
+void *
+kpi_skb_reserve(void *skb, int n)
+{
+  unsigned char *d = (unsigned char *)rdptr(skb, kpi_l.skb_data);
+  wrptr(skb, kpi_l.skb_data, d + n);
+  wr32(skb, kpi_l.skb_tail, rd32(skb, kpi_l.skb_tail) + (unsigned int)n);
+  return 0;
+}
+
+unsigned char *kpi_skb_data(void *skb) { return rdptr(skb, kpi_l.skb_data); }
+unsigned int   kpi_skb_len(void *skb)  { return rd32(skb, kpi_l.skb_len); }
+
+// Copy a frame out of an skb, which is what a driver does on transmit when it
+// cannot hand the buffer to the hardware directly. The checksum half of the
+// name does not apply: nothing here offloads one.
+void
+kpi_skb_copy_and_csum_dev(void *skb, unsigned char *to)
+{
+  unsigned char *d = rdptr(skb, kpi_l.skb_data);
+  unsigned int   n = rd32(skb, kpi_l.skb_len), i;
+  for (i = 0; i < n; i++)
+    to[i] = d[i];
+}
+
+// The driver has a frame. Hand it to the stack.
+//
+// "The stack" is the kernel on the other side of /dev/netdev, and this is the
+// point at which a packet that arrived on real hardware becomes an ordinary
+// frame on an ordinary interface. Everything before it is the driver's; nothing
+// after it knows a Linux driver was involved.
+int
+kpi_netif_receive_skb(void *skb)
+{
+  unsigned char *d = rdptr(skb, kpi_l.skb_data);
+  unsigned int   n = rd32(skb, kpi_l.skb_len);
+
+  if (kpi.netfd >= 0 && n > 0 && n <= 1536)
+    write(kpi.netfd, (char *)d, (int)n);
+  return 0;
+}
+
+// eth_type_trans(skb, dev) -- consume the Ethernet header and report the
+// protocol. The stack below wants the whole frame including that header, so
+// unlike Linux's this does not pull it off; what it returns is the type field,
+// which the driver passes on and nothing here reads.
+unsigned short
+kpi_eth_type_trans(void *skb, void *dev)
+{
+  unsigned char *d = rdptr(skb, kpi_l.skb_data);
+  unsigned int   n = rd32(skb, kpi_l.skb_len);
+
+  wrptr(skb, kpi_l.skb_dev, dev);
+  if (n < 14)
+    return 0;
+  return (unsigned short)((d[12] << 8) | d[13]);
+}
+
+void kpi_kfree_skb(void *skb) { (void)skb; }
+void kpi_kfree_skb_reason(void *skb, int r) { (void)skb; (void)r; }

ldls/kpi/string.c

@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// The string and memory primitives. See ldls/kpi/kpi.h for the provenance rule.
+//
+// These are the least interesting functions in the layer and the most
+// dangerous to leave out, because a stub for one of them does not crash --
+// it silently does nothing, and the caller carries on with a buffer full of
+// whatever was there before.
+//
+// That is not hypothetical. `memcpy` was a logging stub while the receive path
+// was being brought up: the driver copied each frame into its skb, the copy
+// did not happen, and the stack was handed the right *length* of zeroes. Every
+// counter said packets were arriving and DHCP never completed. A crash would
+// have been easier to find.
+
+#include "types.h"
+#include "kpi.h"
+
+void *
+kpi_memcpy(void *d, const void *s, unsigned long n)
+{
+  unsigned char *a = (unsigned char *)d;
+  const unsigned char *b = (const unsigned char *)s;
+  while (n--) *a++ = *b++;
+  return d;
+}
+
+void *
+kpi_memmove(void *d, const void *s, unsigned long n)
+{
+  unsigned char *a = (unsigned char *)d;
+  const unsigned char *b = (const unsigned char *)s;
+
+  if (a == b || n == 0)
+    return d;
+  if (a < b) {
+    while (n--) *a++ = *b++;
+  } else {
+    a += n; b += n;
+    while (n--) *--a = *--b;
+  }
+  return d;
+}
+
+void *
+kpi_memset(void *d, int c, unsigned long n)
+{
+  unsigned char *a = (unsigned char *)d;
+  while (n--) *a++ = (unsigned char)c;
+  return d;
+}
+
+int
+kpi_memcmp(const void *x, const void *y, unsigned long n)
+{
+  const unsigned char *a = x, *b = y;
+  while (n--) { if (*a != *b) return (int)*a - (int)*b; a++; b++; }
+  return 0;
+}
+
+unsigned long
+kpi_strlen(const char *s)
+{
+  unsigned long n = 0;
+  while (s[n]) n++;
+  return n;
+}
+
+unsigned long
+kpi_strnlen(const char *s, unsigned long max)
+{
+  unsigned long n = 0;
+  while (n < max && s[n]) n++;
+  return n;
+}
+
+// sized_strscpy(dest, src, count) -- a bounded copy that always terminates and
+// reports the length copied, or -E2BIG when it did not fit.
+long
+kpi_sized_strscpy(char *d, const char *s, unsigned long n)
+{
+  unsigned long i = 0;
+
+  if (n == 0)
+    return -7;                   /* -E2BIG */
+  while (i + 1 < n && s[i]) { d[i] = s[i]; i++; }
+  d[i] = 0;
+  return s[i] ? -7 : (long)i;
+}
+
+// The io variants. On this platform a BAR is ordinary memory once mapped, so
+// these differ from their plain counterparts only in that the compiler must
+// not reorder or elide them -- which is what the volatile is for.
+void
+kpi_memcpy_fromio(void *d, const void *src, unsigned long n)
+{
+  unsigned char *a = (unsigned char *)d;
+  const volatile unsigned char *b = (const volatile unsigned char *)src;
+  while (n--) *a++ = *b++;
+}
+
+void
+kpi_memcpy_toio(void *dst, const void *s, unsigned long n)
+{
+  volatile unsigned char *a = (volatile unsigned char *)dst;
+  const unsigned char *b = (const unsigned char *)s;
+  while (n--) *a++ = *b++;
+}
+
+// __fortify_panic is what a driver's bounds-checked string helpers call when
+// they detect an overflow. Reaching it means the driver found a bug in itself,
+// and the honest thing is to say so rather than return.
+void
+kpi_fortify_panic(void)
+{
+  kpi_log("kpi: the driver called __fortify_panic -- it detected an "
+          "overflow in itself\n");
+  for (;;)
+    ;
+}