SchOS/cmd/ldlsd/main.c

// SPDX-License-Identifier: GPL-2.0-or-later
//
// ldlsd -- LDLS, on SchOS.
//
// docs/architecture/ldls.md. `ldls-run` does this on the build machine so
// that loader failures cost a second instead of a boot; this is the same
// loader, the same BTF reader and the same trampolines, in a process on the
// system they were written for.
//
//     ldlsd <module.ko> [vmlinux-btf]
//
// **Neither file ships with SchOS.** Both are the user's: a module they
// obtained, and the BTF of the kernel it was built for. docs/licensing.md §6
// is the whole argument, and `.gitignore` refusing `*.ko` is how it is kept
// mechanically rather than by intention.
//
// What it does *not* yet do is implement any of the symbols the driver asks
// for -- every one is still a stub that reports itself and returns zero, which
// is L3c. So the useful output here is the same as `ldls-run`'s: what the
// driver asks for, in what order, and how far it gets.

#include "core/types.h"
#include "uapi/user.h"
#include "uapi/fcntl.h"
#include "uapi/posix.h"
#include "uapi/stat.h"

#include "ko.h"
#include "stub.h"
#include "btf.h"
#include "dev/pciuser.h"
#include "kpi/kpi.h"
extern void kpi_unregister_netdev(void *dev);
extern void *kpi_skb_alloc(unsigned int len);
extern void *kpi_skb_put(void *skb, unsigned int n);
extern void *kpi_netdev(void);

// --- small output, since lib/ has no printf ---------------------------------

static void
puts_(const char *s)
{
    int n = 0;
    while (s[n]) n++;
    write(1, (void *)s, n);
}

static void
putn(long long v)
{
    char b[24];
    int i = 0;
    if (v < 0) { write(1, "-", 1); v = -v; }
    if (v == 0) { write(1, "0", 1); return; }
    while (v > 0) { b[i++] = (char)('0' + (v % 10)); v /= 10; }
    while (i-- > 0) write(1, &b[i], 1);
}

static void
puthexn(unsigned long long v, int digits)
{
    static const char h[] = "0123456789abcdef";
    char b[16];
    int i;
    if (digits > 16) digits = 16;
    for (i = digits - 1; i >= 0; i--) { b[i] = h[v & 0xF]; v >>= 4; }
    write(1, b, digits);
}

static void
puthex(unsigned long long v)
{
    puts_("0x");
    puthexn(v, 16);
}

// lib/ has no string.h either, so the two byte helpers this needs are here.
static void
bcopy_(void *d, const void *s, int n)
{
    unsigned char *a = (unsigned char *)d;
    const unsigned char *b = (const unsigned char *)s;
    while (n-- > 0) *a++ = *b++;
}

static int
streq(const char *a, const char *b)
{
    while (*a && *a == *b) { a++; b++; }
    return *a == 0 && *b == 0;
}

// --- allocation -------------------------------------------------------------
//
// mmap and nothing else. lib/ has no malloc, and this needs both large
// buffers (a kernel's BTF is several megabytes) and memory it can execute --
// which on SchOS is every mapping, because the kernel never sets EFER.NXE.
// That is convenient here and is a gap rather than a feature; see the note in
// docs/architecture/ldls.md.
//
// The module lands in the low 2 GB automatically: a process's mmap area starts
// at 0x40000000 and bumps upward, which is exactly the window R_X86_64_32S
// relocations need. L1 predicted that and this is where it is collected.
static void *
getmem(unsigned long long n)
{
    void *p = mmap(0, n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS,
                   -1, 0);
    if (p == 0 || (long)p == -1)
        return 0;
    return p;
}

static unsigned char *
slurp(const char *path, unsigned long long *len)
{
    struct stat st;
    unsigned char *buf;
    int fd, got = 0;

    fd = open((char *)path, O_RDONLY);
    if (fd < 0)
        return 0;
    if (fstat(fd, &st) < 0 || st.size == 0) { close(fd); return 0; }

    buf = getmem(st.size);
    if (!buf) { close(fd); return 0; }

    while (got < (int)st.size) {
        int n = read(fd, (char *)buf + got, (int)st.size - got);
        if (n <= 0)
            break;
        got += n;
    }
    close(fd);
    if (got != (int)st.size)
        return 0;
    *len = st.size;
    return buf;
}

// --- the stubs ---------------------------------------------------------------

static const char **g_names;
static int          g_count;
static int          g_calls;
static unsigned long g_pci_driver;

unsigned long
ldls_stub(long idx, unsigned long *args)
{
    const char *name = (idx >= 0 && idx < g_count) ? g_names[idx] : "(?)";

    g_calls++;
    if (g_calls <= 40) {
        puts_("  "); putn(g_calls); puts_("  "); puts_(name);
        puts_("("); puthex(args[0]); puts_(", "); puthex(args[1]);
        puts_(")\n");
    } else if (g_calls == 41) {
        puts_("  ... (further calls not printed)\n");
    }

    // The driver's own words, which is the clearest evidence a load worked.
    // Linux format strings carry a leading level byte and %p extensions that
    // nothing here can expand, so the string is printed raw.
    if (streq(name, "_printk") || streq(name, "printk")) {
        const char *f = (const char *)args[0];
        if (f) {
            if ((unsigned char)f[0] == 0x01 && f[1] >= '0' && f[1] <= '7')
                f += 2;
            puts_("      | "); puts_(f);
            {
                int n = 0;
                while (f[n]) n++;
                if (n == 0 || f[n - 1] != '\n') puts_("\n");
            }
        }
    }

    if (streq(name, "__pci_register_driver") && !g_pci_driver)
        g_pci_driver = args[0];

    return 0;
}

struct rctx { struct ko *k; struct stubs *s; };

// Modules loaded before the one being relocated. A module set is loaded in
// dependency order -- libphy before r8169, ttm before amdgpu -- and what an
// earlier one defines is what a later one imports from it.
#define MAXMOD 32
static struct ko *g_mods[MAXMOD];
static int        g_nmods;

static unsigned long long
find_global(struct ko *k, const char *want)
{
    int i;
    for (i = 0; i < k->nsym; i++) {
        if ((k->sym[i].st_info >> 4) == 0)      // STB_LOCAL: not for others
            continue;
        if (k->sym[i].st_shndx == KO_SHN_UNDEF ||
            k->sym[i].st_shndx >= (unsigned short)k->nsh)
            continue;
        if (!k->sec[k->sym[i].st_shndx].alloc)
            continue;
        if (!streq(k->str + k->sym[i].st_name, want))
            continue;
        return k->sec[k->sym[i].st_shndx].addr + k->sym[i].st_value;
    }
    return 0;
}

// The KPI first, a logging stub second.
//
// An unimplemented symbol still resolves -- to something that reports itself
// and returns zero -- so the driver carries on and the *next* thing it wants
// becomes visible. That is what made L1's measurement possible and it stays
// useful for exactly the same reason: the work queue keeps writing itself.
static int g_kpi_used;
static int k_nundef_global;

static int
resolve(void *ctx, const char *name, unsigned long long *out)
{
    struct rctx *r = (struct rctx *)ctx;
    void *fn;
    int i;

    fn = kpi_find(name);
    if (fn) {
        *out = (unsigned long long)fn;
        g_kpi_used++;
        return 0;
    }

    for (i = 0; i < g_nmods; i++) {
        unsigned long long a = find_global(g_mods[i], name);
        if (a) {
            *out = a;
            return 0;
        }
    }

    for (i = 0; i < r->k->nundef; i++)
        if (streq(r->k->undef[i], name)) {
            *out = stub_addr(r->s, i);
            return 0;
        }
    return -1;
}


// --- per-CPU variables ---------------------------------------------------
//
// The stack guard was one of these; `current` is another, and every driver
// touches it. ko_fixup_percpu() turns each `%gs:` reference into a `lea` of
// memory we supply here, so a per-CPU variable becomes an ordinary one --
// which is what it is, on a host with one thread pretending to be one CPU.
//
// The blobs are zeroed and generously sized, because we do not know what the
// driver will read through them: `current` is a task_struct of some kilobytes,
// and a driver that walks into it should get zeroes rather than a fault. Same
// bargain as the logging stubs, and the same caveat -- see the memcpy note in
// docs/architecture/ldls.md.

#define PERCPU_SLOTS 16
#define PERCPU_SIZE  (16 * 1024)

static char percpu_mem[PERCPU_SLOTS][PERCPU_SIZE];
static const char *percpu_name[PERCPU_SLOTS];
static int percpu_n;

static unsigned long long
percpu_for(void *ctx, const char *name)
{
    int i;
    (void)ctx;
    for (i = 0; i < percpu_n; i++)
        if (streq(percpu_name[i], name))
            return (unsigned long long)(void *)percpu_mem[i];
    if (percpu_n == PERCPU_SLOTS)
        return 0;
    i = percpu_n++;
    percpu_name[i] = name;

    // Zero is the right contents for a counter and for `cpu_number`, and the
    // wrong contents for `current`: a driver reads that one and dereferences
    // what it got. So the pointer-valued ones point at the rest of their own
    // slot, which is zeroes all the way down -- every field the driver reads
    // out of the task it thinks it has comes back 0, and anything it writes
    // lands somewhere harmless.
    if (streq(name, "const_current_task") || streq(name, "current_task")) {
        *(unsigned long long *)percpu_mem[i] =
            (unsigned long long)(void *)(percpu_mem[i] + 64);
        puts_("  per-CPU "); puts_(name); puts_(" -- a task of zeroes\n");
    } else if (streq(name, "__preempt_count")) {
        // x86 keeps PREEMPT_NEED_RESCHED inverted in this word, so that
        // `decl %gs:__preempt_count; je resched` is one instruction. Zero
        // means "count zero *and* a reschedule wanted", and every
        // preempt_enable() then called preempt_schedule -- see the note on
        // __SCT__preempt_schedule in ldls/kpi/base.c for what that cost.
        *(unsigned int *)percpu_mem[i] = 0x80000000u;
        puts_("  per-CPU "); puts_(name); puts_(" -- PREEMPT_ENABLED\n");
    } else {
        puts_("  per-CPU "); puts_(name); puts_(" -- 16K of zeroes\n");
    }
    return (unsigned long long)(void *)percpu_mem[i];
}


static unsigned long long
find_symbol(struct ko *k, const char *want)
{
    int i;
    for (i = 0; i < k->nsym; i++) {
        if (!streq(k->str + k->sym[i].st_name, want))
            continue;
        if (k->sym[i].st_shndx == KO_SHN_UNDEF ||
            k->sym[i].st_shndx >= (unsigned short)k->nsh)
            continue;
        if (!k->sec[k->sym[i].st_shndx].alloc)
            continue;
        return k->sec[k->sym[i].st_shndx].addr + k->sym[i].st_value;
    }
    return 0;
}

// "r8169: rtl_open.cold+0x4f" for an address in a loaded module, so that a
// photograph of the screen is enough to find the line. Modules are named by
// the file they were loaded from, with the directory and ".ko" dropped.
static const char *g_modname[MAXMOD];

static void
where(unsigned long addr)
{
    for (int m = 0; m < g_nmods; m++) {
        struct ko *k = g_mods[m];
        for (int i = 0; i < k->nsym; i++) {
            const struct ko_sym *s = &k->sym[i];
            if (s->st_shndx == KO_SHN_UNDEF || s->st_shndx >= (unsigned short)k->nsh ||
                !k->sec[s->st_shndx].alloc || s->st_size == 0 || (s->st_info & 0xf) != 2)
                continue;
            unsigned long long a = k->sec[s->st_shndx].addr + s->st_value;
            if (addr >= a && addr < a + s->st_size) {
                puts_(g_modname[m]); puts_(": ");
                puts_(k->str + s->st_name); puts_("+");
                puthex(addr - a);
                return;
            }
        }
    }
    puthex(addr);
}

// --- reading and writing a struct at an offset BTF gave us --------------------

static unsigned int
rd32(unsigned long base, unsigned int off)
{ unsigned int v; bcopy_(&v, (void *)(base + off), 4); return v; }

static unsigned long
rd64(unsigned long base, unsigned int off)
{ unsigned long v; bcopy_(&v, (void *)(base + off), 8); return v; }

static void
wr16(unsigned long base, unsigned int off, unsigned short v)
{ bcopy_((void *)(base + off), &v, 2); }

static void
wr32(unsigned long base, unsigned int off, unsigned int v)
{ bcopy_((void *)(base + off), &v, 4); }

static void
wr64(unsigned long base, unsigned int off, unsigned long v)
{ bcopy_((void *)(base + off), &v, 8); }

// --- finding the hardware ------------------------------------------------------

// Claim the first device the driver says it supports.
//
// The driver's own id_table is the authority on what it can drive, so the
// match is made against that rather than against anything LDLS knows -- which
// is the property that keeps this general: nothing here has heard of an
// rtl8139.
// Which device, when more than one matches.
//
// The first driver tested was an rtl8139 in a machine whose only other NIC was
// an e1000 the kernel was driving, so "the first that matches" was the same as
// "the right one". With two e1000s it is not: taking the first means taking
// the one the console's own network is on. Written as `bus:slot.func`, or -1
// for "any", which is still the default.
static int sel_bus = -1, sel_slot = -1, sel_func = -1;

static int
claim_for(unsigned long idtab, unsigned int id_size, unsigned int id_vendor,
          unsigned int id_device, struct pciuser_info *out)
{
    int count = 0, i, n;

    kpi.pcifd = open("/dev/pci", O_RDWR);
    if (kpi.pcifd < 0) { puts_("ldlsd: cannot open /dev/pci\n"); return -1; }
    if (ioctl(kpi.pcifd, PCIIOC_COUNT, &count) < 0) return -1;

    for (i = 0; i < count; i++) {
        struct pciuser_info in;
        in.index = (unsigned int)i;
        if (ioctl(kpi.pcifd, PCIIOC_INFO, &in) < 0)
            continue;

        if (sel_bus >= 0 && ((int)in.bus != sel_bus ||
                            (int)in.slot != sel_slot ||
                            (int)in.func != sel_func))
            continue;

        for (n = 0; n < 4096; n++) {
            unsigned long e = idtab + (unsigned long)n * id_size;
            unsigned int v = rd32(e, id_vendor), d = rd32(e, id_device);
            if (!v && !d) break;
            if (v != in.vendor || d != in.device) continue;

            puts_("  found   "); puthexn(in.vendor, 4); puts_(":");
            puthexn(in.device, 4); puts_(" at ");
            puthexn(in.bus, 2); puts_(":"); puthexn(in.slot, 2);
            puts_("."); puthexn(in.func, 1);
            if (in.driver[0]) { puts_(", held by "); puts_(in.driver); }
            puts_("\n");

            if (ioctl(kpi.pcifd, PCIIOC_CLAIM, &i) < 0) {
                puts_("  the kernel would not release it\n");
                return -1;
            }
            // Re-read: claiming is what enables memory space and bus
            // mastering, and a BAR read before that can legitimately be zero.
            in.index = (unsigned int)i;
            if (ioctl(kpi.pcifd, PCIIOC_INFO, &in) < 0) return -1;

            kpi.index = i;
            for (int b = 0; b < 6; b++) {
                kpi.bar_base[b]  = in.bar[b].base;
                kpi.bar_size[b]  = in.bar[b].size;
                kpi.bar_flags[b] = in.bar[b].flags;
            }
            *out = in;
            return 0;
        }
    }
    puts_("  no device on this machine matches the driver's id_table\n");
    return -1;
}

// --- being the PCI core -------------------------------------------------------

static unsigned int
lay(struct btf *b, const char *type, const char *member)
{
    unsigned int off, size;
    int id = btf_find(b, type, BTF_KIND_STRUCT);

    if (!id) {
        puts_("ldlsd: this BTF has no struct "); puts_(type); puts_("\n");
        return 0xffffffffu;
    }
    if (!member)
        return btf_type_size(b, id);
    if (btf_member_off(b, id, member, &off, &size) < 0) {
        puts_("ldlsd: "); puts_(type); puts_("."); puts_(member);
        puts_(": "); puts_(btf_err(b)); puts_("\n");
        return 0xffffffffu;
    }
    return off;
}

#define LDLS_IORESOURCE_IO  0x00000100
#define LDLS_IORESOURCE_MEM 0x00000200

// Fill in the offsets the KPI needs, from the same BTF the PCI ones came from.
static int
kpi_layout_from_btf(struct btf *b)
{
    unsigned int off, size;
    int nd, ops, skb;

    nd  = btf_find(b, "net_device", BTF_KIND_STRUCT);
    ops = btf_find(b, "net_device_ops", BTF_KIND_STRUCT);
    skb = btf_find(b, "sk_buff", BTF_KIND_STRUCT);
    if (!nd || !ops || !skb) {
        puts_("ldlsd: this BTF is missing net_device, net_device_ops or "
              "sk_buff\n");
        return -1;
    }

#define GET(id, name, field) \
    do { if (btf_member_off(b, id, name, &off, &size) < 0) { \
             puts_("ldlsd: no "); puts_(name); puts_("\n"); return -1; } \
         field = off; } while (0)

    GET(nd,  "netdev_ops",     kpi_l.nd_netdev_ops);
    GET(nd,  "dev_addr",       kpi_l.nd_dev_addr);
    GET(nd,  "perm_addr",      kpi_l.nd_perm_addr);
    GET(nd,  "addr_len",       kpi_l.nd_addr_len);
    // Optional: older kernels do not have it, and 0 is how that is said.
    if (btf_member_off(b, nd, "priv_len", &off, &size) == 0)
        kpi_l.nd_priv_len = off;
    GET(nd,  "mc",             kpi_l.nd_mc);
    GET(nd,  "uc",             kpi_l.nd_uc);
    GET(nd,  "_tx",            kpi_l.nd_tx);
    GET(nd,  "num_tx_queues",  kpi_l.nd_num_tx);
    GET(nd,  "real_num_tx_queues", kpi_l.nd_real_num_tx);
    GET(nd,  "state",          kpi_l.nd_state);
    GET(nd,  "flags",          kpi_l.nd_flags);
    {
        int dv = btf_find(b, "device", BTF_KIND_STRUCT);
        int ko = btf_find(b, "kobject", BTF_KIND_STRUCT);
        int pd = btf_find(b, "pci_dev", BTF_KIND_STRUCT);
        if (dv && ko && pd) {
            GET(pd, "dev",  kpi_l.pd_dev);
            GET(dv, "kobj", kpi_l.dev_kobj);
            GET(ko, "name", kpi_l.kobj_name);
            kpi_l.dev_kobj_ok = 1;
        }
    }
    // The driver core and struct firmware. Optional: a driver that never
    // registers a bus or loads firmware does not need them, so a BTF without
    // one of these turns the feature off rather than refusing to run.
    {
        int dv = btf_find(b, "device", BTF_KIND_STRUCT);
        int dd = btf_find(b, "device_driver", BTF_KIND_STRUCT);
        int bt = btf_find(b, "bus_type", BTF_KIND_STRUCT);
        int fw = btf_find(b, "firmware", BTF_KIND_STRUCT);
        int ok = dv && dd && bt;
#define OPT(id, name, field) \
        do { if (btf_member_off(b, id, name, &off, &size) < 0) ok = 0; \
             else field = off; } while (0)
        if (ok) {
            OPT(dv, "bus",    kpi_l.dev_bus);
            OPT(dv, "driver", kpi_l.dev_driver);
            OPT(dd, "bus",    kpi_l.drv_bus);
            OPT(dd, "probe",  kpi_l.drv_probe);
            OPT(dd, "name",   kpi_l.drv_name);
            OPT(bt, "match",  kpi_l.bus_match);
            OPT(bt, "probe",  kpi_l.bus_probe);
        }
        kpi_l.core_ok = ok;
        if (fw) {
            ok = 1;
            OPT(fw, "size", kpi_l.fw_size);
            OPT(fw, "data", kpi_l.fw_data);
            kpi_l.fw_size_of = btf_type_size(b, fw);
            kpi_l.fw_ok = ok;
        }
#undef OPT
        puts_("  kpi     driver core "); puts_(kpi_l.core_ok ? "on" : "off");
        puts_(", firmware "); puts_(kpi_l.fw_ok ? "on" : "off"); puts_("\n");
    }
    {
        int nq = btf_find(b, "netdev_queue", BTF_KIND_STRUCT);
        if (!nq) { puts_("ldlsd: no struct netdev_queue\n"); return -1; }
        kpi_l.nq_size = btf_type_size(b, nq);
        GET(nq, "dev", kpi_l.nq_dev);
    }
    {
        long long v = 0;
        if (btf_enum_value(b, "__LINK_STATE_NOCARRIER", &v) < 0) {
            puts_("ldlsd: this BTF has no __LINK_STATE_NOCARRIER\n");
            return -1;
        }
        kpi_l.link_nocarrier = (unsigned int)v;
    }
    {
        int wk = btf_find(b, "work_struct", BTF_KIND_STRUCT);
        if (!wk) { puts_("ldlsd: no struct work_struct\n"); return -1; }
        GET(wk, "func", kpi_l.work_func);
    }
    GET(ops, "ndo_open",       kpi_l.ndo_open);
    GET(ops, "ndo_stop",       kpi_l.ndo_stop);
    GET(ops, "ndo_start_xmit", kpi_l.ndo_start_xmit);
    GET(skb, "len",            kpi_l.skb_len);
    GET(skb, "data",           kpi_l.skb_data);
    GET(skb, "head",           kpi_l.skb_head);
    GET(skb, "tail",           kpi_l.skb_tail);
    GET(skb, "end",            kpi_l.skb_end);
    GET(skb, "dev",            kpi_l.skb_dev);
#undef GET

    kpi_l.skb_size = btf_type_size(b, skb);
    return 0;
}

// Bring the interface up and carry frames.
//
// **This polls.** A Linux driver expects its interrupt handler to be called
// from an interrupt, and /dev/pci does deliver them -- by blocking read, which
// needs a thread this program does not have. Calling the handler from a loop
// is the same work in a context LDLS can actually provide, and it costs
// latency and a spinning process. The honest shape here, and it is written
// down as a limitation rather than hidden.
//
// A NAPI driver splits the work: the handler acknowledges the chip and
// schedules, and the poll takes packets off it. Both are called, in that
// order, which is the order a kernel would call them in.
static void
run_interface(struct btf *b, void *ndev)
{
    void *ops, *open_fn, *xmit_fn;
    unsigned char frame[1600];
    long tx = 0, rx = 0, idle = 0;

    ops = (void *)rd64((unsigned long)ndev, kpi_l.nd_netdev_ops);
    if (!ops) { puts_("  the driver published no netdev_ops\n"); return; }

    open_fn = (void *)rd64((unsigned long)ops, kpi_l.ndo_open);
    xmit_fn = (void *)rd64((unsigned long)ops, kpi_l.ndo_start_xmit);
    if (!open_fn || !xmit_fn) {
        puts_("  the driver has no ndo_open or no ndo_start_xmit\n");
        return;
    }

    // netif_running(dev) has to be true before ndo_open, not after.
    //
    // Linux's dev_open() sets __LINK_STATE_START *before* calling the driver,
    // and 8139too's receive loop is gated on it: without it the driver saw the
    // packet arrive, found netif_running false, left the loop immediately and
    // acked the interrupt -- so the chip's buffer filled, its status cleared,
    // and nothing came out. That is a silent failure with no error anywhere,
    // and it was found by reading the chip's own CBR register rather than by
    // anything the driver said.
    //
    // IFF_UP too, for the same class of reason.
    {
        unsigned char *d = (unsigned char *)ndev;
        d[kpi_l.nd_state] |= 1;                 /* __LINK_STATE_START */
        d[kpi_l.nd_flags] |= 1;                 /* IFF_UP             */
    }

    // Each phase gets its own first forty. A driver makes thousands of stub
    // calls during probe, so without this the interesting ones -- the last
    // thing it did before it stopped -- are always past the cap.
    g_calls = 0;
    puts_("\n  calling ndo_open\n\n");
    {
        int (*op)(void *) = (int (*)(void *))open_fn;
        int r = op(ndev);
        puts_("\n  ndo_open returned "); putn(r); puts_("\n");
        if (r != 0) { puts_("  the interface will not come up\n"); return; }
    }

    puts_("\n  carrying frames. the interface is ldls0.\n\n");

    for (;;) {
        int n, moved = 0;

        // Anything the stack wants sent. The descriptor is non-blocking, so
        // this returns immediately when there is nothing -- the receive side
        // has to be given a turn either way.
        n = read(kpi.netfd, (char *)frame, (int)sizeof(frame));
        if (n > 0) {
            void *skb = kpi_skb_alloc((unsigned int)n);
            if (skb) {
                unsigned char *d = kpi_skb_put(skb, (unsigned int)n);
                int i;
                for (i = 0; i < n; i++) d[i] = frame[i];
                {
                    int (*xmit)(void *, void *) =
                        (int (*)(void *, void *))xmit_fn;
                    xmit(skb, ndev);
                }
                tx++;
                moved = 1;
            }
        }

        // The driver's own handler, as if the line had been asserted. Its
        // first act is to read the device's cause register, so calling it
        // with nothing pending costs one register read and returns IRQ_NONE.
        if (kpi_irq_poll())
            moved = 1;

        // Anything the driver asked to have done later. A watchdog belongs
        // here and not in a timer interrupt: it wants to read registers and
        // touch the rings, which is exactly what the rest of this loop does.
        if (kpi_work_run() > 0)
            moved = 1;

        // And anything the chip has. The driver's own poll does the work; all
        // this decides is when it gets to run.
        if (kpi_napi_poll) {
            int got = kpi_napi_poll(kpi_napi, 16);
            if (got > 0) { rx += got; moved = 1; }
        }

        if (!moved) {
            if (++idle > 200) { sleep(1); idle = 0; }
        } else {
            idle = 0;
            // The first few, then every sixty-fourth. Bringing a driver up,
            // the question is always whether the *first* frame moved.
            if (tx + rx <= 4 || ((tx + rx) % 64) == 0) {
                puts_("  ldls0: tx "); putn(tx);
                puts_("  rx "); putn(rx); puts_("\n");
            }
        }
    }
}

static void
drive_probe(const char *btfpath)
{
    struct btf b;
    unsigned char *bimg;
    unsigned long long blen;
    unsigned int *boff;
    long nt;
    unsigned int drv_probe, drv_idtable, drv_name;
    unsigned int id_size, id_vendor, id_device;
    unsigned int dev_size, dev_vendor, dev_device, dev_devfn, dev_irq, dev_res;
    unsigned int res_size, res_start, res_end, res_flags;
    unsigned long idtab, pdev;
    int nid = 0;

    bimg = slurp(btfpath, &blen);
    if (!bimg) { puts_("ldlsd: cannot read the BTF\n"); return; }

    nt = btf_count(bimg, blen);
    if (nt < 0) { puts_("ldlsd: that file is not BTF\n"); return; }

    boff = getmem(((unsigned long long)nt + 2) * 4);
    if (!boff) { puts_("ldlsd: no memory for the BTF index\n"); return; }
    if (btf_open(&b, bimg, blen, boff, (unsigned int)nt + 2) < 0) {
        puts_("ldlsd: "); puts_(btf_err(&b)); puts_("\n");
        return;
    }
    puts_("  btf     "); putn(nt); puts_(" types\n");

    drv_probe   = lay(&b, "pci_driver", "probe");
    drv_idtable = lay(&b, "pci_driver", "id_table");
    drv_name    = lay(&b, "pci_driver", "name");
    id_size     = lay(&b, "pci_device_id", 0);
    id_vendor   = lay(&b, "pci_device_id", "vendor");
    id_device   = lay(&b, "pci_device_id", "device");
    dev_size    = lay(&b, "pci_dev", 0);
    dev_vendor  = lay(&b, "pci_dev", "vendor");
    dev_device  = lay(&b, "pci_dev", "device");
    dev_devfn   = lay(&b, "pci_dev", "devfn");
    dev_irq     = lay(&b, "pci_dev", "irq");
    dev_res     = lay(&b, "pci_dev", "resource");
    res_size    = lay(&b, "resource", 0);
    res_start   = lay(&b, "resource", "start");
    res_end     = lay(&b, "resource", "end");
    res_flags   = lay(&b, "resource", "flags");

    if (drv_probe == 0xffffffffu || dev_size == 0xffffffffu ||
        res_size == 0xffffffffu) {
        puts_("ldlsd: this kernel's BTF cannot drive probe()\n");
        return;
    }

    puts_("  layout  pci_driver.probe +"); putn(drv_probe);
    puts_("  pci_dev "); putn(dev_size); puts_(" bytes\n");

    {
        const char *dn = (const char *)rd64(g_pci_driver, drv_name);
        if (dn) { puts_("  driver  \""); puts_(dn); puts_("\"\n"); }
    }

    idtab = rd64(g_pci_driver, drv_idtable);
    if (!idtab) { puts_("  it has no id_table\n"); return; }

    puts_("\n  devices it claims:\n");
    for (;;) {
        unsigned long e = idtab + (unsigned long)nid * id_size;
        unsigned int v = rd32(e, id_vendor), d = rd32(e, id_device);
        if (!v && !d) break;
        if (nid < 6) {
            puts_("    "); puthexn(v, 4); puts_(":"); puthexn(d, 4);
            puts_("\n");
        }
        nid++;
        if (nid > 4096) break;
    }
    if (nid > 6) { puts_("    ... and "); putn(nid - 6); puts_(" more\n"); }
    if (!nid) return;

    // Find the real device, claim it, and describe *it* rather than something
    // made up. This is the line between L2 and L3c: probe used to be handed a
    // pci_dev whose shape was right and whose contents were invented, and now
    // every field in it came off the bus.
    {
        struct pciuser_info in;

        // No device: not a failure worth restarting for. schinit's
        // KeepAlive would otherwise start this again every few seconds on
        // every machine without the NIC. See SCHINIT_EXIT_NORESTART.
        if (claim_for(idtab, id_size, id_vendor, id_device, &in) < 0)
            exit_group(78);

        pdev = (unsigned long)getmem(dev_size);
        if (!pdev) { puts_("ldlsd: no memory for a pci_dev\n"); return; }

        // A name for the device.
        //
        // `pci_name(pdev)` is `pdev->dev.kobj.name`, and a driver copies it
        // into its interface's name during probe. Left null it is a fault
        // inside strnlen with nothing to say which field was missing.
        if (kpi_l.pd_dev && kpi_l.kobj_name) {
            static char nm[16];
            unsigned long q;
            int j = 0;
            nm[j++] = '0'; nm[j++] = '0'; nm[j++] = '0'; nm[j++] = '0';
            nm[j++] = ':';
            nm[j++] = "0123456789abcdef"[(in.bus >> 4) & 0xf];
            nm[j++] = "0123456789abcdef"[in.bus & 0xf];
            nm[j++] = ':';
            nm[j++] = "0123456789abcdef"[(in.slot >> 4) & 0xf];
            nm[j++] = "0123456789abcdef"[in.slot & 0xf];
            nm[j++] = '.';
            nm[j++] = "0123456789abcdef"[in.func & 0xf];
            nm[j]   = 0;
            q = (unsigned long)nm;
            for (int i = 0; i < 8; i++)
                ((unsigned char *)pdev)[kpi_l.pd_dev + kpi_l.dev_kobj +
                                        kpi_l.kobj_name + i] =
                    (unsigned char)(q >> (i * 8));
        }

        // pdev->bus. pci_domain_nr() and pci_dev_id() are inlines that
        // dereference it -- r8169 names its MDIO bus with both -- and a null
        // one is a fault at the first line of MDIO setup.
        {
            unsigned int pd_bus = lay(&b, "pci_dev", "bus");
            unsigned int pb_num = lay(&b, "pci_bus", "number");
            unsigned int pb_sys = lay(&b, "pci_bus", "sysdata");
            unsigned int pb_size = lay(&b, "pci_bus", 0);
            if (pd_bus != 0xffffffffu && pb_num != 0xffffffffu &&
                pb_sys != 0xffffffffu && pb_size != 0xffffffffu) {
                unsigned long bus = (unsigned long)getmem(pb_size);
                unsigned long sys = (unsigned long)getmem(4096);
                if (bus && sys) {
                    ((unsigned char *)bus)[pb_num] = (unsigned char)in.bus;
                    bcopy_((void *)(bus + pb_sys), &sys, 8);
                    bcopy_((void *)(pdev + pd_bus), &bus, 8);
                }
            }
        }

        wr16(pdev, dev_vendor, in.vendor);
        wr16(pdev, dev_device, in.device);
        wr32(pdev, dev_devfn, (unsigned int)((in.slot << 3) | in.func));
        wr32(pdev, dev_irq, in.irq);

        for (int b = 0; b < 6; b++) {
            unsigned long r = pdev + dev_res + (unsigned long)b * res_size;
            if (!in.bar[b].base)
                continue;
            wr64(r, res_start, in.bar[b].base);
            wr64(r, res_end,   in.bar[b].base + in.bar[b].size - 1);
            wr64(r, res_flags, (in.bar[b].flags & PCIUSER_BAR_IO)
                                   ? LDLS_IORESOURCE_IO : LDLS_IORESOURCE_MEM);
            puts_("  bar "); putn(b); puts_("    0x");
            puthexn(in.bar[b].base, 16); puts_("  ");
            putn((long long)in.bar[b].size);
            puts_((in.bar[b].flags & PCIUSER_BAR_IO) ? " io\n" : " mem\n");
        }
    }

    // sizeof(struct net_device), for alloc_etherdev_mqs -- a driver reaches
    // its private area past it and gets silent corruption if this is wrong.
    {
        extern unsigned int kpi_netdev_size;
        int nd = btf_find(&b, "net_device", BTF_KIND_STRUCT);
        kpi_netdev_size = nd ? btf_type_size(&b, nd) : 0;
        puts_("  netdev  "); putn(kpi_netdev_size); puts_(" bytes\n");
    }

    // Before probe, not after: alloc_etherdev_mqs runs *during* probe and
    // needs net_device's offsets to set dev->dev_addr up.
    if (kpi_layout_from_btf(&b) < 0)
        return;

    // Counted over the *symbol list*, not over resolutions: a symbol is
    // resolved once per relocation that names it, and reporting that number
    // would have said 275 of 94.
    {
        int impl = 0, i;
        for (i = 0; i < k_nundef_global; i++)
            if (kpi_find(g_names[i])) impl++;
        puts_("\n  calling probe, with "); putn(impl);
        puts_(" of "); putn(k_nundef_global);
        puts_(" symbols implemented\n\n");
    }
    g_calls = 0;
    {
        int (*probe)(void *, void *) =
            (int (*)(void *, void *))(void *)rd64(g_pci_driver, drv_probe);
        if (!probe) { puts_("  the driver has no probe\n"); return; }
        {
            int r = probe((void *)pdev, (void *)idtab);
            puts_("\n  probe returned "); putn(r);
            puts_(" after "); putn(g_calls); puts_(" stub call(s)\n");

            if (r == 0 && kpi.netfd >= 0) {
                // The *net_device*, not the pci_dev. See kpi_netdev().
                run_interface(&b, kpi_netdev());
                return;
            }
            if (0) {
                // The driver is up on real hardware and has registered an
                // interface. What is *not* built is the path that carries
                // frames -- ndo_open, ndo_start_xmit and the receive side.
                //
                // So the interface is released rather than left up. A link
                // that reports itself up with nothing draining its queue is a
                // black hole: the stack would send DHCP into it for ever and
                // every layer above would believe the network was merely
                // quiet. Down is the true answer until packets move.
                puts_("\n  the driver is up on real hardware and has an "
                      "interface.\n");
                puts_("  carrying frames is the half of L3 that is not built, "
                      "so the\n  interface is released rather than left up "
                      "and silent.\n");
                kpi_unregister_netdev(0);
            }
        }
    }
}

// Read the BTF and fill in the KPI's layouts, before any module runs.
static void
early_layout(const char *btfpath)
{
    static struct btf b;
    unsigned long long blen;
    unsigned char *bimg = slurp(btfpath, &blen);
    long nt;
    unsigned int *boff;

    if (!bimg) return;
    nt = btf_count(bimg, blen);
    if (nt < 0) return;
    boff = getmem(((unsigned long long)nt + 2) * 4);
    if (!boff || btf_open(&b, bimg, blen, boff, (unsigned int)nt + 2) < 0)
        return;
    {
        extern unsigned int kpi_netdev_size;
        int nd = btf_find(&b, "net_device", BTF_KIND_STRUCT);
        kpi_netdev_size = nd ? btf_type_size(&b, nd) : 0;
    }
    kpi_layout_from_btf(&b);
}

// One module: read, laid out, relocated, its %gs references fixed, and its
// init_module called. Kept loaded for the ones after it to link against.
static void
load_module(const char *path)
{
    struct ko *k;
    struct ko_section *sec;
    struct stubs *stubs;
    struct rctx rc;
    unsigned char *img, *mem, *code;
    unsigned long long len, span, entry;
    int nsec, i;

    k = getmem(sizeof(*k));
    stubs = getmem(sizeof(*stubs));
    if (!k || !stubs) { puts_("ldlsd: out of memory\n"); exit(); }

    img = slurp(path, &len);
    if (!img) { puts_("ldlsd: cannot read "); puts_(path); puts_("\n"); exit(); }

    nsec = ko_sections_needed(img, len);
    if (nsec < 0) {
        puts_("ldlsd: that is not an ELF64 relocatable object\n");
        exit();
    }
    sec = getmem((unsigned long long)(nsec + 1) * sizeof(struct ko_section));
    if (!sec) { puts_("ldlsd: out of memory\n"); exit(); }

    if (ko_open(k, img, len, sec, nsec) < 0) {
        puts_("ldlsd: "); puts_(ko_err(k)); puts_("\n");
        exit();
    }

    puts_(path); puts_("\n");
    puts_("  module  ");
    puts_(ko_name(k) ? ko_name(k) : "(not a kernel module)"); puts_("\n");
    puts_("  license ");
    puts_(ko_modinfo(k, "license") ? ko_modinfo(k, "license")
                                    : "(none declared)"); puts_("\n");
    puts_("  wants   "); putn(k->nundef); puts_(" symbols\n");

    span = ko_span(k);
    mem  = getmem(span ? span : 4096);
    code = getmem((unsigned long long)(k->nundef + 1) * STUB_SIZE);
    if (!mem || !code) { puts_("ldlsd: out of memory\n"); exit(); }

    stub_build_from(stubs, code, k->nundef, g_count);
    for (i = 0; i < k->nundef; i++) {
        g_names[g_count + i] = k->undef[i];
        if (stub_is_bare(k->undef[i]))
            stub_make_bare(code + (unsigned long long)i * STUB_SIZE);
    }
    g_count += k->nundef;
    k_nundef_global = g_count;

    if (ko_layout(k, mem, span, (unsigned long long)mem) < 0) {
        puts_("ldlsd: layout: "); puts_(ko_err(k)); puts_("\n");
        exit();
    }

    rc.k = k; rc.s = stubs;
    if (ko_relocate(k, resolve, &rc) < 0) {
        puts_("ldlsd: relocate: "); puts_(ko_err(k)); puts_("\n");
        exit();
    }
    // The stack canary is fetched through %gs, which belongs to the kernel
    // here. See ko_defuse_gs().
    {
        int g = ko_defuse_gs(k, "__ref_stack_chk_guard");
        if (g < 0) g = 0;
        if (g == 0) g = ko_defuse_gs(k, "__stack_chk_guard");
        if (g > 0) { puts_("  defused "); putn(g); puts_(" %gs stack-guard load(s)\n"); }
    }
    // Whatever %gs references are left name real per-CPU variables, and those
    // cannot be defused -- the driver dereferences what it reads.
    {
        int p = ko_fixup_percpu(k, percpu_for, 0);
        if (p < 0) {
            puts_("ldlsd: per-CPU: "); puts_(ko_err(k)); puts_("\n");
            exit();
        }
        if (p > 0) { puts_("  rehomed "); putn(p); puts_(" %gs per-CPU reference(s)\n"); }
    }

    puts_("  loaded  "); putn(span); puts_(" bytes at "); puthex((unsigned long long)mem);
    puts_("\n");

    entry = find_symbol(k, "init_module");
    if (g_nmods < MAXMOD) {
        const char *b = path, *q;
        for (q = path; *q; q++)
            if (*q == '/') b = q + 1;
        g_modname[g_nmods] = b;
        g_mods[g_nmods++] = k;
        kpi_where = where;
    }
    if (!entry) { puts_("  no init_module -- nothing to call\n"); return; }

    puts_("\n  calling init_module\n\n");
    {
        int (*init)(void) = (int (*)(void))(void *)entry;
        int r = init();
        puts_("\n  init_module returned "); putn(r);
        puts_(" after "); putn(g_calls); puts_(" stub call(s)\n");
    }

}

// --- main ----------------------------------------------------------------------

int
main(int argc, char *argv[])
{
    if (argc < 2) {
        puts_("usage: ldlsd <module.ko>[,<module.ko>...] [vmlinux-btf] [bus:slot.func]\n");
        puts_("  neither file ships with SchOS; both are yours.\n");
        exit();
    }

    // An optional bus address, to pick between devices that all match.
    if (argc > 3) {
        const char *a = argv[3];
        int f[3] = {0, 0, 0}, got = 0, v = 0, any = 0;
        for (;; a++) {
            int d = -1;
            if (*a >= '0' && *a <= '9')      d = *a - '0';
            else if (*a >= 'a' && *a <= 'f') d = *a - 'a' + 10;
            else if (*a >= 'A' && *a <= 'F') d = *a - 'A' + 10;
            if (d >= 0) { v = v * 16 + d; any = 1; continue; }
            if (any && got < 3) f[got++] = v;
            v = 0; any = 0;
            if (*a == 0) break;
        }
        if (got != 3) {
            puts_("ldlsd: a device is written bus:slot.func, as /dev/pci prints it\n");
            exit();
        }
        sel_bus = f[0]; sel_slot = f[1]; sel_func = f[2];
        puts_("  device  only "); puthexn(sel_bus, 2); puts_(":");
        puthexn(sel_slot, 2); puts_("."); puthexn(sel_func, 1); puts_("\n");
    }

    // argv[1] is one module, or a set in dependency order separated by
    // commas: libphy.ko,realtek.ko,r8169.ko. Each is loaded, relocated
    // against the KPI and the ones before it, and initialised before the next.
    {
        static char list[4096];
        char *p = list, *name;
        int n = 0;
        while (argv[1][n] && n < (int)sizeof(list) - 1) { list[n] = argv[1][n]; n++; }
        list[n] = 0;
        // The layouts first: init_module is where a module registers its
        // drivers, and the driver core needs struct offsets to do that.
        if (argc > 2)
            early_layout(argv[2]);
        g_names = getmem(sizeof(char *) * (unsigned long long)MAXMOD * KO_MAXUNDEF);
        if (!g_names) { puts_("ldlsd: out of memory\n"); exit(); }
        kpi.netfd = -1;
        for (;;) {
            name = p;
            while (*p && *p != ',') p++;
            int last = *p == 0;
            *p = 0;
            if (*name)
                load_module(name);
            if (last) break;
            p++;
        }
    }

    if (argc > 2 && g_pci_driver)
        drive_probe(argv[2]);
    else if (argc > 2)
        puts_("\n  no pci_driver was registered -- nothing to probe\n");

    exit();
}