diff --git a/Makefile b/Makefile index 59d25b0..03490f9 100644 --- a/Makefile +++ b/Makefile @@ -803,7 +803,7 @@ $(BUILD)/lib/%.o: lib/%.S # stub.c are freestanding and build unchanged for either side, which is the # whole point of having written them that way. LDLS_GUEST = $(addprefix $(BUILD)/ldls/guest/, ko.o btf.o stub.o) \ - $(addprefix $(BUILD)/ldls/guest/kpi/, kpi.o pci.o io.o net.o) + $(addprefix $(BUILD)/ldls/guest/kpi/, kpi.o pci.o io.o net.o skb.o string.o) $(BUILD)/ldls/guest/%.o: ldls/%.c ldls/ko.h ldls/btf.h ldls/stub.h @mkdir -p $(BUILD)/ldls/guest diff --git a/cmd/ldlsd/main.c b/cmd/ldlsd/main.c index f3ca7f7..73897c1 100644 --- a/cmd/ldlsd/main.c +++ b/cmd/ldlsd/main.c @@ -31,6 +31,9 @@ #include "pciuser.h" #include "kpi/kpi.h" extern void kpi_unregister_netdev(void *dev); +extern void *kpi_skb_alloc(unsigned int len); +extern void *kpi_skb_put(void *skb, unsigned int n); +extern void *kpi_netdev(void); // --- small output, since lib/ has no printf --------------------------------- @@ -337,6 +340,157 @@ lay(struct btf *b, const char *type, const char *member) #define LDLS_IORESOURCE_IO 0x00000100 #define LDLS_IORESOURCE_MEM 0x00000200 +// Fill in the offsets the KPI needs, from the same BTF the PCI ones came from. +static int +kpi_layout_from_btf(struct btf *b) +{ + unsigned int off, size; + int nd, ops, skb; + + nd = btf_find(b, "net_device", BTF_KIND_STRUCT); + ops = btf_find(b, "net_device_ops", BTF_KIND_STRUCT); + skb = btf_find(b, "sk_buff", BTF_KIND_STRUCT); + if (!nd || !ops || !skb) { + puts_("ldlsd: this BTF is missing net_device, net_device_ops or " + "sk_buff\n"); + return -1; + } + +#define GET(id, name, field) \ + do { if (btf_member_off(b, id, name, &off, &size) < 0) { \ + puts_("ldlsd: no "); puts_(name); puts_("\n"); return -1; } \ + field = off; } while (0) + + GET(nd, "netdev_ops", kpi_l.nd_netdev_ops); + GET(nd, "dev_addr", kpi_l.nd_dev_addr); + GET(nd, "perm_addr", kpi_l.nd_perm_addr); + GET(nd, "addr_len", kpi_l.nd_addr_len); + GET(nd, "mc", kpi_l.nd_mc); + GET(nd, "uc", kpi_l.nd_uc); + GET(nd, "_tx", kpi_l.nd_tx); + GET(nd, "num_tx_queues", kpi_l.nd_num_tx); + GET(nd, "real_num_tx_queues", kpi_l.nd_real_num_tx); + GET(nd, "state", kpi_l.nd_state); + GET(nd, "flags", kpi_l.nd_flags); + { + int nq = btf_find(b, "netdev_queue", BTF_KIND_STRUCT); + if (!nq) { puts_("ldlsd: no struct netdev_queue\n"); return -1; } + kpi_l.nq_size = btf_type_size(b, nq); + GET(nq, "dev", kpi_l.nq_dev); + } + GET(ops, "ndo_open", kpi_l.ndo_open); + GET(ops, "ndo_stop", kpi_l.ndo_stop); + GET(ops, "ndo_start_xmit", kpi_l.ndo_start_xmit); + GET(skb, "len", kpi_l.skb_len); + GET(skb, "data", kpi_l.skb_data); + GET(skb, "head", kpi_l.skb_head); + GET(skb, "tail", kpi_l.skb_tail); + GET(skb, "end", kpi_l.skb_end); + GET(skb, "dev", kpi_l.skb_dev); +#undef GET + + kpi_l.skb_size = btf_type_size(b, skb); + return 0; +} + +// Bring the interface up and carry frames. +// +// **This polls.** A Linux driver expects its interrupt handler to be called +// from an interrupt, and /dev/pci does deliver them -- by blocking read, which +// needs a thread this program does not have. Calling the handler from a loop +// is the same work in a context LDLS can actually provide, and it costs +// latency and a spinning process. The honest shape here, and it is written +// down as a limitation rather than hidden. +// +// A NAPI driver splits the work: the handler acknowledges the chip and +// schedules, and the poll takes packets off it. Both are called, in that +// order, which is the order a kernel would call them in. +static void +run_interface(struct btf *b, void *ndev) +{ + void *ops, *open_fn, *xmit_fn; + unsigned char frame[1600]; + long tx = 0, rx = 0, idle = 0; + + ops = (void *)rd64((unsigned long)ndev, kpi_l.nd_netdev_ops); + if (!ops) { puts_(" the driver published no netdev_ops\n"); return; } + + open_fn = (void *)rd64((unsigned long)ops, kpi_l.ndo_open); + xmit_fn = (void *)rd64((unsigned long)ops, kpi_l.ndo_start_xmit); + if (!open_fn || !xmit_fn) { + puts_(" the driver has no ndo_open or no ndo_start_xmit\n"); + return; + } + + // netif_running(dev) has to be true before ndo_open, not after. + // + // Linux's dev_open() sets __LINK_STATE_START *before* calling the driver, + // and 8139too's receive loop is gated on it: without it the driver saw the + // packet arrive, found netif_running false, left the loop immediately and + // acked the interrupt -- so the chip's buffer filled, its status cleared, + // and nothing came out. That is a silent failure with no error anywhere, + // and it was found by reading the chip's own CBR register rather than by + // anything the driver said. + // + // IFF_UP too, for the same class of reason. + { + unsigned char *d = (unsigned char *)ndev; + d[kpi_l.nd_state] |= 1; /* __LINK_STATE_START */ + d[kpi_l.nd_flags] |= 1; /* IFF_UP */ + } + + puts_("\n calling ndo_open\n\n"); + { + int (*op)(void *) = (int (*)(void *))open_fn; + int r = op(ndev); + puts_("\n ndo_open returned "); putn(r); puts_("\n"); + if (r != 0) { puts_(" the interface will not come up\n"); return; } + } + + puts_("\n carrying frames. the interface is ldls0.\n\n"); + + for (;;) { + int n, moved = 0; + + // Anything the stack wants sent. The descriptor is non-blocking, so + // this returns immediately when there is nothing -- the receive side + // has to be given a turn either way. + n = read(kpi.netfd, (char *)frame, (int)sizeof(frame)); + if (n > 0) { + void *skb = kpi_skb_alloc((unsigned int)n); + if (skb) { + unsigned char *d = kpi_skb_put(skb, (unsigned int)n); + int i; + for (i = 0; i < n; i++) d[i] = frame[i]; + { + int (*xmit)(void *, void *) = + (int (*)(void *, void *))xmit_fn; + xmit(skb, ndev); + } + tx++; + moved = 1; + } + } + + // And anything the chip has. The driver's own poll does the work; all + // this decides is when it gets to run. + if (kpi_napi_poll) { + int got = kpi_napi_poll(kpi_napi, 16); + if (got > 0) { rx += got; moved = 1; } + } + + if (!moved) { + if (++idle > 200) { sleep(1); idle = 0; } + } else { + idle = 0; + if (((tx + rx) % 64) == 0) { + puts_(" ldls0: tx "); putn(tx); + puts_(" rx "); putn(rx); puts_("\n"); + } + } + } +} + static void drive_probe(const char *btfpath) { @@ -457,6 +611,11 @@ drive_probe(const char *btfpath) puts_(" netdev "); putn(kpi_netdev_size); puts_(" bytes\n"); } + // Before probe, not after: alloc_etherdev_mqs runs *during* probe and + // needs net_device's offsets to set dev->dev_addr up. + if (kpi_layout_from_btf(&b) < 0) + return; + // Counted over the *symbol list*, not over resolutions: a symbol is // resolved once per relocation that names it, and reporting that number // would have said 275 of 94. @@ -479,6 +638,11 @@ drive_probe(const char *btfpath) puts_(" after "); putn(g_calls); puts_(" stub call(s)\n"); if (r == 0 && kpi.netfd >= 0) { + // The *net_device*, not the pci_dev. See kpi_netdev(). + run_interface(&b, kpi_netdev()); + return; + } + if (0) { // The driver is up on real hardware and has registered an // interface. What is *not* built is the path that carries // frames -- ndo_open, ndo_start_xmit and the receive side. diff --git a/docs/plan/1.0.md b/docs/plan/1.0.md index 84eba1c..a554346 100644 --- a/docs/plan/1.0.md +++ b/docs/plan/1.0.md @@ -95,7 +95,7 @@ The galling ones. The work exists, is tested, and nothing uses it. |---|---| | **`foundations.md`** | Item 1 built. Items 2 (a shareable memory object), 3 (`AF_UNIX` in the filesystem namespace), 4 and 5 open. **Wayland stops at item 2.** | | ~~**`hybrid.md`**~~ | **Closed.** All four mechanisms built; H1, H2 and H3 done. H4 — the USB eviction — is *withdrawn* rather than outstanding: USB is the bus peripherals arrive on, not a peripheral, and an eviction that has to leave an identical driver behind has moved nothing. See [hybrid.md](hybrid.md), "Decided: USB stays in the kernel". | -| **`ldls.md`** | L0, L1, L2 and L3a done — the loader handles 2,440 stock modules; 449 of 500 run `init_module`; `8139too`'s `probe()` runs to completion; and `/dev/netdev` lets a process be an interface, with `ping` working over one. L3b done too — `ldlsd` runs the same loader on SchOS and gets a real driver's `probe()` to return. L3c done — a Linux driver now drives real hardware through `/dev/pci` and reads the card's own MAC off its EEPROM. L3d, the packet path, is what remains. | +| **`ldls.md`** | L0, L1, L2 and L3a done — the loader handles 2,440 stock modules; 449 of 500 run `init_module`; `8139too`'s `probe()` runs to completion; and `/dev/netdev` lets a process be an interface, with `ping` working over one. L3b done too — `ldlsd` runs the same loader on SchOS and gets a real driver's `probe()` to return. **L3 done: `ping` works over a Linux driver.** `8139too`, loaded by our loader, running in a SchOS process, driving QEMU's rtl8139 through `/dev/pci`, carrying DHCP and ICMP both ways on a subnet reachable only through it. One driver, one kernel version, tried once — and the network transport for the debugger is the only thing still open on any plan. | | **Schcom v0** | 715 lines that composite, over a wire format that is superseded. Frozen: it runs, nothing more is added, and it is kept only because it is the one thing in the tree that has ever put a client's pixels on screen. | | **`debugger.md`** | D0 through D4 built and described in [../architecture/debugging.md](../architecture/debugging.md). Only the network transport is open — a userspace daemon bridging TCP to `/dev/dbg`, which is deliberately last. | diff --git a/docs/plan/ldls.md b/docs/plan/ldls.md index baf1401..9692eda 100644 --- a/docs/plan/ldls.md +++ b/docs/plan/ldls.md @@ -184,8 +184,8 @@ One device, one process, claimed and released, on the `/dev/pci` discipline. | **L3a** | **The class bridge.** `/dev/netdev`: a process claims it and becomes an interface. `ping` over a *process*. | **done** — `kernel/dev/netuser.c`, `cmd/netecho` | | **L3b** | The SchOS-side host: `ldls-run`'s loader and BTF work, as a program on SchOS. | **done** — `cmd/ldlsd`; `/dev/pci` is L3c's half | | **L3c** | The fifteen symbols, implemented rather than stubbed, against a real rtl8139. | **done** — `ldls/kpi/`; the driver reads the real MAC | -| **L3d** | The packet path: `ndo_open`, `ndo_start_xmit`, receive. | | -| **L3** | **A packet.** The two joined: **`ping` over a Linux driver.** This is the milestone that proves the whole idea and the one that cannot be skipped. | | +| **L3d** | The packet path: `ndo_open`, `ndo_start_xmit`, receive. | **done** | +| **L3** | **A packet.** The two joined: **`ping` over a Linux driver.** This is the milestone that proves the whole idea and the one that cannot be skipped. | **done** | | **L4** | **Kill it.** The host process dies, the kernel clears bus mastering and reclaims the DMA, a new host starts and the interface comes back. | free — H2 built all of this | L4 is listed for completeness rather than as work. `pciuser_stop()`, @@ -725,6 +725,79 @@ that is L3d. --- +## L3d: ping, over a Linux driver + +``` +# ldlsd /lib/modules/8139too.ko /lib/modules/btf & + kpi: the driver read a MAC off the hardware: 52:54:00:ab:cd:ef + ndo_open returned 0 + carrying frames. the interface is ldls0. + +# cat /proc/netif +interface ldls0 +mac 52:54:00:ab:cd:ef +state up +address 10.9.0.15 +gateway 10.9.0.2 + +# ping 10.9.0.2 4 +64 bytes from 10.9.0.2: icmp_seq=1 time=30 ms +64 bytes from 10.9.0.2: icmp_seq=2 time=20 ms +64 bytes from 10.9.0.2: icmp_seq=3 time=20 ms +64 bytes from 10.9.0.2: icmp_seq=4 time=20 ms +``` + +`10.9.0.0/24` is reachable **only** through `ldls0`, and `ldls0` is `8139too` +running in a SchOS process. The address on it was obtained by DHCP, over the +same driver, before the ping. + +`lwIP` → `netdev_ops` → `/dev/netdev` → `ldlsd` → `ndo_start_xmit` → a BAR +mapped by `/dev/pci` → the wire, and the whole way back. Nothing in the kernel +knows a Linux driver is involved; nothing in the driver knows it is not in +Linux. + +### It polls + +A Linux driver expects its interrupt handler to be called from an interrupt. +`/dev/pci` does deliver them — by blocking `read()`, which needs a thread this +program does not have. So `ldlsd` calls the driver's NAPI poll from a loop +instead: the same work, in a context LDLS can actually provide. It costs +latency and a spinning process, and it is written down here rather than hidden. + +### Four things `alloc_etherdev` does that allocating does not + +A driver gets its `net_device` from `alloc_etherdev_mqs`, and the function does +considerably more than return memory. Each of these was found by the driver +faulting on it, which is a slow way to learn a list and an extremely reliable +one: + +| | | +|---|---| +| `dev->dev_addr` | a **pointer**, aimed at the device's own `perm_addr`. The driver writes its MAC into the chip with `*(u32 *)dev->dev_addr`. | +| `dev->mc`, `dev->uc` | list heads, which are *empty* when they point at themselves. Zeroed means a null `next`, and the receive-mode setup walks them. | +| `dev->_tx` | an array of `netdev_queue`, because `netif_start_queue` clears a bit in `dev->_tx[0].state`. | +| `dev->state` | `__LINK_STATE_START`, set by Linux's `dev_open` **before** it calls `ndo_open`. | + +### The two that did not announce themselves + +Everything above crashed, which made it easy. Two did not. + +**`netif_running(dev)`.** Without `__LINK_STATE_START`, 8139too's receive loop +exited immediately — so the chip received the frame, the driver acknowledged +the interrupt, and nothing came out. Every counter said packets were arriving. +It was found by reading the chip's own `CBR` register through the BAR and +seeing that the hardware had written 600 bytes the driver never collected. + +**`memcpy`.** It was still a logging stub while the receive path was being +brought up. The driver copied each frame into its `sk_buff`, the copy did not +happen, and the stack was handed exactly the right *length* of zeroes. DHCP +never completed and nothing anywhere reported an error. **A stub that returns +zero is not safe for a function whose whole job is a side effect** — and the +memory primitives are the worst case, because they fail silently and +plausibly. `ldls/kpi/string.c` exists because of it. + +--- + ## What it will not do - **It is not a security boundary, and here that is worse than usual.** There @@ -795,12 +868,16 @@ that is L3d. with the link down, because nothing in lwIP supports taking an interface out of its list. From the stack's point of view that is the same thing; from `/proc/netif`'s it is a `ldls0` that stays listed after its process is gone. -- **No frame has moved.** L3c ends at a successful `probe()`. The packet path - — `ndo_open`, `ndo_start_xmit`, receive, and the skb handling all three need - — is L3d and is the larger half. -- **41 of 94 symbols.** The rest are still logging stubs, which is deliberate: - an unimplemented symbol that reports itself keeps the work queue writing - itself, exactly as it did at L1. +- **It polls instead of taking interrupts**, per the note above. A busy + interface spins a processor; an idle one wakes 200 times before sleeping. +- **One driver, one kernel version, tried once.** `8139too` on 6.18.46 against + QEMU's rtl8139. That is what the milestone table has always promised and it + is worth repeating next to a working `ping`: nothing here has been tried on a + second driver, a second kernel, or real silicon. +- **Some symbols are still logging stubs.** That is deliberate — an + unimplemented symbol that reports itself keeps the work queue writing itself, + exactly as it did at L1 — but it means a driver reaching one gets zero and + carries on, and `memcpy` proved how quiet that can be. - **A lock is a no-op.** In one process with no preemption inside the driver there is nobody to race with, which is correct right up until the driver has an interrupt thread. That is the sleeping-context assumption named above and diff --git a/ldls/kpi/kpi.c b/ldls/kpi/kpi.c index 67969d7..f01cbb4 100644 --- a/ldls/kpi/kpi.c +++ b/ldls/kpi/kpi.c @@ -17,8 +17,27 @@ #include "netuser.h" #include "kpi.h" -struct kpi_ctx kpi; -int kpi_verbose; +struct kpi_ctx kpi; +struct kpi_layout kpi_l; +int kpi_verbose; + +// The driver's NAPI poll, captured at registration. See kpi.h. +int (*kpi_napi_poll)(void *napi, int budget); +void *kpi_napi; + +// netif_napi_add_weight_locked(dev, napi, poll, weight) +// +// Recorded rather than ignored, because the poll function is the receive path: +// a NAPI driver's interrupt handler only schedules, and everything that +// actually takes packets off the chip happens in here. +void +kpi_netif_napi_add(void *dev, void *napi, void *poll, int weight) +{ + (void)dev; (void)weight; + kpi_napi = napi; + kpi_napi_poll = (int (*)(void *, int))poll; + kpi_log(" kpi: the driver registered a NAPI poll\n"); +} void kpi_log(const char *s) @@ -186,6 +205,41 @@ print_fmt(const char *f) int kpi_printk(const char *fmt) { print_fmt(fmt); return 0; } int kpi_netdev_print(void *dev, const char *fmt) { (void)dev; print_fmt(fmt); return 0; } +// --- data, not functions ------------------------------------------------------- +// +// Not every undefined symbol in a module is a call. `byte_rev_table` is a +// 256-byte lookup of bit-reversed bytes, and 8139too indexes it directly while +// computing its multicast hash. A stub trampoline resolves the name and gives +// the driver machine code to read as data -- which does not fault, and +// produces a quietly wrong answer, which is worse. +// +// The ELF symbol table cannot tell these apart: every undefined symbol in a +// module is STT_NOTYPE. BTF can (VAR against FUNC) and that is the general +// fix; this is the specific one, and the values are a fact about bit order. +static const unsigned char kpi_byte_rev_table[256] = { + 0x00,0x80,0x40,0xc0,0x20,0xa0,0x60,0xe0,0x10,0x90,0x50,0xd0,0x30,0xb0,0x70,0xf0, + 0x08,0x88,0x48,0xc8,0x28,0xa8,0x68,0xe8,0x18,0x98,0x58,0xd8,0x38,0xb8,0x78,0xf8, + 0x04,0x84,0x44,0xc4,0x24,0xa4,0x64,0xe4,0x14,0x94,0x54,0xd4,0x34,0xb4,0x74,0xf4, + 0x0c,0x8c,0x4c,0xcc,0x2c,0xac,0x6c,0xec,0x1c,0x9c,0x5c,0xdc,0x3c,0xbc,0x7c,0xfc, + 0x02,0x82,0x42,0xc2,0x22,0xa2,0x62,0xe2,0x12,0x92,0x52,0xd2,0x32,0xb2,0x72,0xf2, + 0x0a,0x8a,0x4a,0xca,0x2a,0xaa,0x6a,0xea,0x1a,0x9a,0x5a,0xda,0x3a,0xba,0x7a,0xfa, + 0x06,0x86,0x46,0xc6,0x26,0xa6,0x66,0xe6,0x16,0x96,0x56,0xd6,0x36,0xb6,0x76,0xf6, + 0x0e,0x8e,0x4e,0xce,0x2e,0xae,0x6e,0xee,0x1e,0x9e,0x5e,0xde,0x3e,0xbe,0x7e,0xfe, + 0x01,0x81,0x41,0xc1,0x21,0xa1,0x61,0xe1,0x11,0x91,0x51,0xd1,0x31,0xb1,0x71,0xf1, + 0x09,0x89,0x49,0xc9,0x29,0xa9,0x69,0xe9,0x19,0x99,0x59,0xd9,0x39,0xb9,0x79,0xf9, + 0x05,0x85,0x45,0xc5,0x25,0xa5,0x65,0xe5,0x15,0x95,0x55,0xd5,0x35,0xb5,0x75,0xf5, + 0x0d,0x8d,0x4d,0xcd,0x2d,0xad,0x6d,0xed,0x1d,0x9d,0x5d,0xdd,0x3d,0xbd,0x7d,0xfd, + 0x03,0x83,0x43,0xc3,0x23,0xa3,0x63,0xe3,0x13,0x93,0x53,0xd3,0x33,0xb3,0x73,0xf3, + 0x0b,0x8b,0x4b,0xcb,0x2b,0xab,0x6b,0xeb,0x1b,0x9b,0x5b,0xdb,0x3b,0xbb,0x7b,0xfb, + 0x07,0x87,0x47,0xc7,0x27,0xa7,0x67,0xe7,0x17,0x97,0x57,0xd7,0x37,0xb7,0x77,0xf7, + 0x0f,0x8f,0x4f,0xcf,0x2f,0xaf,0x6f,0xef,0x1f,0x9f,0x5f,0xdf,0x3f,0xbf,0x7f,0xff, +}; + +// jiffies, also data. Nothing advances it: the drivers hosted so far read it +// only to compare against a timeout they will never reach, because nothing +// here runs their watchdog either. +static unsigned long kpi_jiffies; + // --- the table ----------------------------------------------------------------- // // Names on the left are Linux's and are facts. Everything on the right is ours. @@ -213,6 +267,27 @@ extern void kpi_unregister_netdev(void *dev); extern void kpi_dev_addr_mod(void *dev, unsigned int off, const unsigned char *addr, unsigned int len); +extern void *kpi_napi_alloc_skb(void *napi, unsigned int len); +extern void *kpi_netdev_alloc_skb(void *dev, unsigned int len); +extern void *kpi_skb_put(void *skb, unsigned int n); +extern void *kpi_skb_reserve(void *skb, int n); +extern void kpi_skb_copy_and_csum_dev(void *skb, unsigned char *to); +extern unsigned short kpi_eth_type_trans(void *skb, void *dev); +extern int kpi_netif_receive_skb(void *skb); +extern void kpi_kfree_skb(void *skb); +extern void kpi_kfree_skb_reason(void *skb, int r); + +extern void *kpi_memcpy(void *d, const void *s, unsigned long n); +extern void *kpi_memmove(void *d, const void *s, unsigned long n); +extern void *kpi_memset(void *d, int c, unsigned long n); +extern int kpi_memcmp(const void *a, const void *b, unsigned long n); +extern unsigned long kpi_strlen(const char *s); +extern unsigned long kpi_strnlen(const char *s, unsigned long max); +extern long kpi_sized_strscpy(char *d, const char *s, unsigned long n); +extern void kpi_memcpy_fromio(void *d, const void *s, unsigned long n); +extern void kpi_memcpy_toio(void *d, const void *s, unsigned long n); +extern void kpi_fortify_panic(void); + const struct kpi_sym kpi_table[] = { // PCI { "pci_enable_device", (void *)kpi_pci_enable_device }, @@ -248,6 +323,24 @@ const struct kpi_sym kpi_table[] = { { "unregister_netdev", (void *)kpi_unregister_netdev }, { "dev_addr_mod", (void *)kpi_dev_addr_mod }, + // Data, not functions. See the note above kpi_byte_rev_table. + { "byte_rev_table", (void *)kpi_byte_rev_table }, + { "jiffies", (void *)&kpi_jiffies }, + + // Bytes. Not optional, and not safe to stub -- see ldls/kpi/string.c. + { "memcpy", (void *)kpi_memcpy }, + { "__memcpy", (void *)kpi_memcpy }, + { "memmove", (void *)kpi_memmove }, + { "memset", (void *)kpi_memset }, + { "__memset", (void *)kpi_memset }, + { "memcmp", (void *)kpi_memcmp }, + { "strlen", (void *)kpi_strlen }, + { "strnlen", (void *)kpi_strnlen }, + { "sized_strscpy", (void *)kpi_sized_strscpy }, + { "memcpy_fromio", (void *)kpi_memcpy_fromio }, + { "memcpy_toio", (void *)kpi_memcpy_toio }, + { "__fortify_panic", (void *)kpi_fortify_panic }, + // Printing { "_printk", (void *)kpi_printk }, { "netdev_info", (void *)kpi_netdev_print }, @@ -255,6 +348,58 @@ const struct kpi_sym kpi_table[] = { { "_dev_info", (void *)kpi_netdev_print }, { "_dev_err", (void *)kpi_netdev_print }, + // Packets + { "napi_alloc_skb", (void *)kpi_napi_alloc_skb }, + { "__napi_alloc_skb", (void *)kpi_napi_alloc_skb }, + { "__netdev_alloc_skb", (void *)kpi_netdev_alloc_skb }, + { "skb_put", (void *)kpi_skb_put }, + { "skb_reserve", (void *)kpi_skb_reserve }, + { "skb_copy_and_csum_dev", (void *)kpi_skb_copy_and_csum_dev }, + { "eth_type_trans", (void *)kpi_eth_type_trans }, + { "netif_receive_skb", (void *)kpi_netif_receive_skb }, + { "napi_gro_receive", (void *)kpi_netif_receive_skb }, + { "dev_kfree_skb_any_reason",(void *)kpi_kfree_skb_reason }, + { "consume_skb", (void *)kpi_kfree_skb }, + { "kfree_skb_reason", (void *)kpi_kfree_skb_reason }, + + // Bringing the interface up. LDLS polls rather than taking interrupts, so + // request_irq succeeds and nothing is ever delivered through it -- see the + // note in cmd/ldlsd on why a poll loop is the honest shape here. + { "request_threaded_irq", (void *)kpi_zero }, + { "free_irq", (void *)kpi_nop }, + { "enable_irq", (void *)kpi_nop }, + { "disable_irq_nosync", (void *)kpi_nop }, + { "napi_enable", (void *)kpi_nop }, + { "napi_enable_locked", (void *)kpi_nop }, + { "napi_disable", (void *)kpi_nop }, + { "napi_disable_locked", (void *)kpi_nop }, + { "__napi_schedule", (void *)kpi_nop }, + { "napi_schedule_prep", (void *)kpi_one }, + { "napi_complete_done", (void *)kpi_one }, + { "netif_tx_wake_queue", (void *)kpi_nop }, + { "netif_device_attach", (void *)kpi_nop }, + { "netif_device_detach", (void *)kpi_nop }, + { "netif_napi_add_weight_locked", (void *)kpi_netif_napi_add }, + + // The PHY. 8139too reaches it through the mii helpers, which call back into + // the driver's own mdio routines -- so "the link is up" is the only thing + // this has to say, and saying anything else stops the driver bringing the + // interface up at all. + { "mii_link_ok", (void *)kpi_one }, + { "mii_nway_restart", (void *)kpi_zero }, + { "mii_check_media", (void *)kpi_nop }, + { "generic_mii_ioctl", (void *)kpi_zero }, + { "mii_ethtool_get_link_ksettings", (void *)kpi_zero }, + { "mii_ethtool_set_link_ksettings", (void *)kpi_zero }, + + // Deferred work. Nothing here runs it: the driver's periodic thread is a + // watchdog, and a watchdog that never fires is better than one firing in a + // context this host cannot provide. + { "queue_delayed_work_on", (void *)kpi_one }, + { "cancel_delayed_work_sync",(void *)kpi_zero }, + { "delayed_work_timer_fn", (void *)kpi_nop }, + { "wake_up_process", (void *)kpi_one }, + // Things a driver calls at probe that need only to return sensibly. Each is // here because 8139too calls it and because doing nothing is the right // answer in one process -- see the note above kpi_nop(). diff --git a/ldls/kpi/kpi.h b/ldls/kpi/kpi.h index 56cf872..34a70fc 100644 --- a/ldls/kpi/kpi.h +++ b/ldls/kpi/kpi.h @@ -67,6 +67,34 @@ struct kpi_ctx { extern struct kpi_ctx kpi; +// The struct offsets the KPI needs at run time, read from the kernel's own BTF +// by the host before the driver starts. See ldls/btf.h -- nothing here is +// written down, because a compiled driver and this code have to agree about +// them exactly and only the kernel that driver was built for can say. +struct kpi_layout { + unsigned int nd_netdev_ops; + unsigned int nd_dev_addr, nd_perm_addr, nd_addr_len; + unsigned int nd_mc, nd_uc; + unsigned int nd_tx, nd_num_tx, nd_real_num_tx; + unsigned int nq_size, nq_dev; + unsigned int nd_state, nd_flags; + unsigned int ndo_open, ndo_stop, ndo_start_xmit; + + unsigned int skb_size; + unsigned int skb_len, skb_data, skb_head, skb_tail, skb_end, skb_dev; +}; + +extern struct kpi_layout kpi_l; + +// The driver's NAPI poll, captured when it registers one. +// +// A NAPI driver does almost nothing in its interrupt handler: it schedules, +// and the receive happens later in the poll. LDLS has no softirq to run that +// from, so it keeps the function pointer and calls it from its own loop -- +// which is what a poll loop is, arrived at from the other direction. +extern int (*kpi_napi_poll)(void *napi, int budget); +extern void *kpi_napi; + // Output. The KPI is inside a process with no libc, and every implementation // wants to be able to say what it did. void kpi_log(const char *s); diff --git a/ldls/kpi/net.c b/ldls/kpi/net.c index f08d300..becac02 100644 --- a/ldls/kpi/net.c +++ b/ldls/kpi/net.c @@ -33,6 +33,14 @@ unsigned int kpi_netdev_size; static void *g_netdev; // the one interface, for register_netdev +// What alloc_etherdev_mqs handed back. +// +// The host needs it to reach ndo_open and ndo_start_xmit, and it is not the +// pci_dev -- passing that instead reads netdev_ops out of the middle of a +// different struct and finds nothing, which is exactly what happened the first +// time this was wired up. +void *kpi_netdev(void) { return g_netdev; } + void * kpi_alloc_etherdev_mqs(int priv, unsigned int txqs, unsigned int rxqs) { @@ -53,6 +61,78 @@ kpi_alloc_etherdev_mqs(int priv, unsigned int txqs, unsigned int rxqs) if (p == 0 || (long)p == -1) return 0; + // Linux's alloc_etherdev does more than allocate, and one of the things it + // does is not optional: `dev->dev_addr` is a *pointer*, aimed at the + // device's own `perm_addr` array. A driver programs its MAC into the chip + // with `*(u32 *)dev->dev_addr`, so leaving it null is a null dereference + // inside the driver's hardware setup -- which is exactly where this stopped + // the first time. + if (kpi_l.nd_dev_addr && kpi_l.nd_perm_addr) { + unsigned char *d = (unsigned char *)p; + unsigned char *perm = d + kpi_l.nd_perm_addr; + int i; + for (i = 0; i < 8; i++) + d[kpi_l.nd_dev_addr + i] = (unsigned char)((unsigned long)perm >> (i * 8)); + if (kpi_l.nd_addr_len) + d[kpi_l.nd_addr_len] = 6; + } else { + kpi_log("kpi: no net_device.dev_addr offset -- the driver will fault\n"); + } + + // The address lists have to be *empty*, not zeroed. + // + // A Linux list_head is empty when it points at itself; zeroed means a null + // `next`, and the first thing that walks it dereferences that. 8139too walks + // the multicast list in its receive-mode setup, so this is reached on the + // way to bringing the interface up -- and it is the second thing Linux's + // alloc_etherdev does that allocating alone does not. + if (kpi_l.nd_mc && kpi_l.nd_uc) { + unsigned int heads[2]; + int h; + heads[0] = kpi_l.nd_mc; + heads[1] = kpi_l.nd_uc; + for (h = 0; h < 2; h++) { + unsigned char *lh = (unsigned char *)p + heads[h]; + unsigned long self = (unsigned long)lh; + int i; + for (i = 0; i < 8; i++) { + lh[i] = (unsigned char)(self >> (i * 8)); /* next */ + lh[8 + i] = (unsigned char)(self >> (i * 8)); /* prev */ + } + } + } + + // And the transmit queues, which are an array hanging off the device rather + // than part of it. + // + // `netif_start_queue` is an inline that clears a bit in `dev->_tx[0].state`, + // so a driver bringing its interface up writes through this pointer before + // it sends anything. Allocating it is the third thing Linux's alloc_etherdev + // does that allocating the device alone does not -- and like the other two, + // it was found by the driver faulting on it rather than by reading a list. + if (kpi_l.nd_tx && kpi_l.nq_size) { + unsigned int n = txqs ? txqs : 1; + unsigned char *q = mmap(0, (unsigned long)n * kpi_l.nq_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q && (long)q != -1) { + unsigned char *d = (unsigned char *)p; + unsigned int k; + int i; + for (i = 0; i < 8; i++) + d[kpi_l.nd_tx + i] = (unsigned char)((unsigned long)q >> (i * 8)); + for (k = 0; k < n; k++) { + unsigned char *e = q + (unsigned long)k * kpi_l.nq_size; + for (i = 0; i < 8; i++) + e[kpi_l.nq_dev + i] = (unsigned char)((unsigned long)p >> (i * 8)); + } + for (i = 0; i < 4; i++) { + d[kpi_l.nd_num_tx + i] = (unsigned char)(n >> (i * 8)); + d[kpi_l.nd_real_num_tx + i] = (unsigned char)(n >> (i * 8)); + } + } + } + kpi_log(" kpi: net_device "); kpi_lognum(kpi_netdev_size); kpi_log(" bytes + "); kpi_lognum(priv); kpi_log(" private\n"); @@ -78,7 +158,7 @@ void kpi_dev_addr_mod(void *dev, unsigned int off, const unsigned char *addr, unsigned int len) { - unsigned int i; + int i; (void)dev; if (off != 0 || len != 6 || !addr) @@ -88,6 +168,18 @@ kpi_dev_addr_mod(void *dev, unsigned int off, const unsigned char *addr, kpi.mac[i] = addr[i]; kpi.have_mac = 1; + // And through dev->dev_addr, which is where the driver will read it back + // from when it programs the chip. + if (g_netdev && kpi_l.nd_dev_addr) { + unsigned char *d = (unsigned char *)g_netdev; + unsigned long q = 0; + for (i = 0; i < 8; i++) + q |= (unsigned long)d[kpi_l.nd_dev_addr + i] << (i * 8); + if (q) + for (i = 0; i < 6; i++) + ((unsigned char *)q)[i] = addr[i]; + } + kpi_log(" kpi: the driver read a MAC off the hardware: "); for (i = 0; i < 6; i++) { kpi_loghex(kpi.mac[i], 2); @@ -114,7 +206,10 @@ kpi_register_netdev(void *dev) return -1; } - kpi.netfd = open("/dev/netdev", O_RDWR); + // Non-blocking, because the host's loop has to give the receive side a turn + // whether or not the stack has anything to send. netuser_ready() is what + // makes O_NONBLOCK work here; see struct devsw. + kpi.netfd = open("/dev/netdev", O_RDWR | O_NONBLOCK); if (kpi.netfd < 0) { kpi_log("kpi: cannot open /dev/netdev\n"); return -1; diff --git a/ldls/kpi/skb.c b/ldls/kpi/skb.c new file mode 100644 index 0000000..5dcf0c5 --- /dev/null +++ b/ldls/kpi/skb.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Socket buffers. See ldls/kpi/kpi.h for the provenance rule. +// +// An `sk_buff` is the unit a Linux driver receives into and transmits from, so +// a host that cannot make one cannot carry a packet. It is also the structure +// with the most fields a driver touches directly, which is why every offset +// here comes from BTF rather than from anything written down. +// +// The layout is worth one note because it is not what the names suggest: +// `head` and `data` are pointers, but `tail` and `end` are **32-bit offsets +// from head**, not pointers. Treating them as pointers produces something that +// works for small packets on a lucky allocation and then does not. + +#include "types.h" +#include "user.h" +#include "posix.h" +#include "kpi.h" + +// Room in front of the data for headers a stack might push. Linux reserves +// this so that adding an Ethernet header does not mean copying the payload; +// nothing here pushes anything, but a driver may rely on `skb_reserve` having +// somewhere to go. +#define KPI_HEADROOM 64 + +static unsigned int +rd32(void *base, unsigned int off) +{ + unsigned int v; + unsigned char *p = (unsigned char *)base + off; + int i; + for (i = 0; i < 4; i++) ((unsigned char *)&v)[i] = p[i]; + return v; +} + +static void +wr32(void *base, unsigned int off, unsigned int v) +{ + unsigned char *p = (unsigned char *)base + off; + int i; + for (i = 0; i < 4; i++) p[i] = ((unsigned char *)&v)[i]; +} + +static void * +rdptr(void *base, unsigned int off) +{ + void *v; + unsigned char *p = (unsigned char *)base + off; + int i; + for (i = 0; i < 8; i++) ((unsigned char *)&v)[i] = p[i]; + return v; +} + +static void +wrptr(void *base, unsigned int off, void *v) +{ + unsigned char *p = (unsigned char *)base + off; + int i; + for (i = 0; i < 8; i++) p[i] = ((unsigned char *)&v)[i]; +} + +// One buffer: the control block, then headroom, then the frame. +// +// Allocated in one piece so that freeing is a no-op like every other +// allocation here -- see the note in kpi.c about a hosted driver's lifetime. +void * +kpi_skb_alloc(unsigned int len) +{ + unsigned char *p; + unsigned int total = kpi_l.skb_size + KPI_HEADROOM + len + 64; + unsigned char *buf; + + if (kpi_l.skb_size == 0) + return 0; + + p = mmap(0, total, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == 0 || (long)p == -1) + return 0; + + buf = p + kpi_l.skb_size; + wrptr(p, kpi_l.skb_head, buf); + wrptr(p, kpi_l.skb_data, buf + KPI_HEADROOM); + wr32(p, kpi_l.skb_tail, KPI_HEADROOM); + wr32(p, kpi_l.skb_end, KPI_HEADROOM + len + 64); + wr32(p, kpi_l.skb_len, 0); + return p; +} + +void *kpi_napi_alloc_skb(void *napi, unsigned int len) +{ (void)napi; return kpi_skb_alloc(len); } + +void *kpi_netdev_alloc_skb(void *dev, unsigned int len) +{ (void)dev; return kpi_skb_alloc(len); } + +// Claim `n` bytes at the tail and return where they start. The classic, and +// the one every receive path is built on. +void * +kpi_skb_put(void *skb, unsigned int n) +{ + unsigned char *head = (unsigned char *)rdptr(skb, kpi_l.skb_head); + unsigned int tail = rd32(skb, kpi_l.skb_tail); + + wr32(skb, kpi_l.skb_tail, tail + n); + wr32(skb, kpi_l.skb_len, rd32(skb, kpi_l.skb_len) + n); + return head + tail; +} + +void * +kpi_skb_reserve(void *skb, int n) +{ + unsigned char *d = (unsigned char *)rdptr(skb, kpi_l.skb_data); + wrptr(skb, kpi_l.skb_data, d + n); + wr32(skb, kpi_l.skb_tail, rd32(skb, kpi_l.skb_tail) + (unsigned int)n); + return 0; +} + +unsigned char *kpi_skb_data(void *skb) { return rdptr(skb, kpi_l.skb_data); } +unsigned int kpi_skb_len(void *skb) { return rd32(skb, kpi_l.skb_len); } + +// Copy a frame out of an skb, which is what a driver does on transmit when it +// cannot hand the buffer to the hardware directly. The checksum half of the +// name does not apply: nothing here offloads one. +void +kpi_skb_copy_and_csum_dev(void *skb, unsigned char *to) +{ + unsigned char *d = rdptr(skb, kpi_l.skb_data); + unsigned int n = rd32(skb, kpi_l.skb_len), i; + for (i = 0; i < n; i++) + to[i] = d[i]; +} + +// The driver has a frame. Hand it to the stack. +// +// "The stack" is the kernel on the other side of /dev/netdev, and this is the +// point at which a packet that arrived on real hardware becomes an ordinary +// frame on an ordinary interface. Everything before it is the driver's; nothing +// after it knows a Linux driver was involved. +int +kpi_netif_receive_skb(void *skb) +{ + unsigned char *d = rdptr(skb, kpi_l.skb_data); + unsigned int n = rd32(skb, kpi_l.skb_len); + + if (kpi.netfd >= 0 && n > 0 && n <= 1536) + write(kpi.netfd, (char *)d, (int)n); + return 0; +} + +// eth_type_trans(skb, dev) -- consume the Ethernet header and report the +// protocol. The stack below wants the whole frame including that header, so +// unlike Linux's this does not pull it off; what it returns is the type field, +// which the driver passes on and nothing here reads. +unsigned short +kpi_eth_type_trans(void *skb, void *dev) +{ + unsigned char *d = rdptr(skb, kpi_l.skb_data); + unsigned int n = rd32(skb, kpi_l.skb_len); + + wrptr(skb, kpi_l.skb_dev, dev); + if (n < 14) + return 0; + return (unsigned short)((d[12] << 8) | d[13]); +} + +void kpi_kfree_skb(void *skb) { (void)skb; } +void kpi_kfree_skb_reason(void *skb, int r) { (void)skb; (void)r; } diff --git a/ldls/kpi/string.c b/ldls/kpi/string.c new file mode 100644 index 0000000..1fd8624 --- /dev/null +++ b/ldls/kpi/string.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// The string and memory primitives. See ldls/kpi/kpi.h for the provenance rule. +// +// These are the least interesting functions in the layer and the most +// dangerous to leave out, because a stub for one of them does not crash -- +// it silently does nothing, and the caller carries on with a buffer full of +// whatever was there before. +// +// That is not hypothetical. `memcpy` was a logging stub while the receive path +// was being brought up: the driver copied each frame into its skb, the copy +// did not happen, and the stack was handed the right *length* of zeroes. Every +// counter said packets were arriving and DHCP never completed. A crash would +// have been easier to find. + +#include "types.h" +#include "kpi.h" + +void * +kpi_memcpy(void *d, const void *s, unsigned long n) +{ + unsigned char *a = (unsigned char *)d; + const unsigned char *b = (const unsigned char *)s; + while (n--) *a++ = *b++; + return d; +} + +void * +kpi_memmove(void *d, const void *s, unsigned long n) +{ + unsigned char *a = (unsigned char *)d; + const unsigned char *b = (const unsigned char *)s; + + if (a == b || n == 0) + return d; + if (a < b) { + while (n--) *a++ = *b++; + } else { + a += n; b += n; + while (n--) *--a = *--b; + } + return d; +} + +void * +kpi_memset(void *d, int c, unsigned long n) +{ + unsigned char *a = (unsigned char *)d; + while (n--) *a++ = (unsigned char)c; + return d; +} + +int +kpi_memcmp(const void *x, const void *y, unsigned long n) +{ + const unsigned char *a = x, *b = y; + while (n--) { if (*a != *b) return (int)*a - (int)*b; a++; b++; } + return 0; +} + +unsigned long +kpi_strlen(const char *s) +{ + unsigned long n = 0; + while (s[n]) n++; + return n; +} + +unsigned long +kpi_strnlen(const char *s, unsigned long max) +{ + unsigned long n = 0; + while (n < max && s[n]) n++; + return n; +} + +// sized_strscpy(dest, src, count) -- a bounded copy that always terminates and +// reports the length copied, or -E2BIG when it did not fit. +long +kpi_sized_strscpy(char *d, const char *s, unsigned long n) +{ + unsigned long i = 0; + + if (n == 0) + return -7; /* -E2BIG */ + while (i + 1 < n && s[i]) { d[i] = s[i]; i++; } + d[i] = 0; + return s[i] ? -7 : (long)i; +} + +// The io variants. On this platform a BAR is ordinary memory once mapped, so +// these differ from their plain counterparts only in that the compiler must +// not reorder or elide them -- which is what the volatile is for. +void +kpi_memcpy_fromio(void *d, const void *src, unsigned long n) +{ + unsigned char *a = (unsigned char *)d; + const volatile unsigned char *b = (const volatile unsigned char *)src; + while (n--) *a++ = *b++; +} + +void +kpi_memcpy_toio(void *dst, const void *s, unsigned long n) +{ + volatile unsigned char *a = (volatile unsigned char *)dst; + const unsigned char *b = (const unsigned char *)s; + while (n--) *a++ = *b++; +} + +// __fortify_panic is what a driver's bounds-checked string helpers call when +// they detect an overflow. Reaching it means the driver found a bug in itself, +// and the honest thing is to say so rather than return. +void +kpi_fortify_panic(void) +{ + kpi_log("kpi: the driver called __fortify_panic -- it detected an " + "overflow in itself\n"); + for (;;) + ; +}