ldls: the same loader, on SchOS

docs/plan/ldls.md milestone L3b. ldls/ko.c, btf.c and stub.c were written
freestanding so they would build unchanged on both sides; cmd/ldlsd collects
that promise -- the same three files, linked against lib/ instead of a libc,
in an ordinary SchOS process.

    # ldlsd /lib/modules/8139too.ko /lib/modules/btf
      module  8139too      license GPL      wants 94 symbols
      defused 6 %gs stack-guard reference(s)
      calling init_module
        1  _printk(...)
            | 8139too: 8139too Fast Ethernet driver 0.9.28
        2  __pci_register_driver(...)
      btf     122279 types
      layout  pci_driver.probe +16   pci_dev 2680 bytes
      driver  "8139too"    devices it claims: 10ec:8139 ... and 18 more
      calling probe
        1  alloc_etherdev_mqs(0x368, 0x1)
      probe returned -12 after 1 stub call(s)

Byte for byte the host harness's behaviour, which is what the shared source
was for. Two earlier predictions came good with no work: a process's mmap area
starts at 0x40000000, inside the window R_X86_64_32S needs, and the kernel
never sets EFER.NXE so every mapping is executable. The second is a gap rather
than a feature and is now recorded as one -- LDLS is relying on the absence of
a protection an OS should have.

One thing did not just work. The first run faulted before probe's first call,
err 5 -- present, read, user -- at a kernel address, with the rip resolving to

    mov %gs:0x0(%rip),%rax      # rtl8139_init_one+0x26

Modules are built -mstack-protector-guard-reg=gs, so the canary comes through
the GS segment, and that is an ABI between the kernel and its own per-CPU area
which a host that is not that kernel cannot provide. ko_defuse_gs() rewrites
those references to xor %reg,%reg.

There are two forms and the second is easy to miss: the prologue loads the
guard with mov (8b) and the epilogue compares it with sub (2b) before a jne to
__stack_chk_fail. xor is right for both -- it zeroes the saved canary and sets
ZF, so the epilogue's branch falls through exactly as a passing check does. Six
sites in 8139too.

It checks every site before patching any and refuses the whole operation on
anything it does not recognise. That refusal is what found the sub form, by
declining to guess -- a loader that half-understands an instruction should not
be editing it.

make ldls-stage/ldls-unstage put a module and its kernel's BTF into the local
sysroot and take them out again. Not part of `all`, and .gitignore refuses
*.ko, so an image only carries somebody else's GPL code between two commands
you typed on purpose.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VQ7Qx8E1q1M1bbCq8QLHS2

Makefile

@@ -108,7 +108,7 @@ KERN_NOENTRY = $(filter-out $(BUILD)/kernel/arch/x86_64/entry.o, $(KERNOBJS))
 # Third-party software does NOT go here — see ports/.
 # -----------------------------------------------------------------------
 ULIB    = $(addprefix $(BUILD)/lib/, start.o glue.o usys.o)
-CMDS    = schinit schinitd fbinfo shmtest socktest inputtest nettest reflink schattr ping chvt pcitest vgpud schdbg schmode netecho
+CMDS    = schinit schinitd fbinfo shmtest socktest inputtest nettest reflink schattr ping chvt pcitest vgpud schdbg schmode netecho ldlsd
 CMDBINS = $(addprefix $(BUILD)/cmd/, $(CMDS))
 
 # -----------------------------------------------------------------------
@@ -798,6 +798,23 @@ $(BUILD)/lib/%.o: lib/%.S
 	@mkdir -p $(BUILD)/lib
 	$(CC) $(CFLAGS) -c $< -o $@
 
+# ldlsd links the LDLS core as well as its own main, so it needs a rule of its
+# own -- a specific rule beats the pattern one below. ldls/ko.c, btf.c and
+# stub.c are freestanding and build unchanged for either side, which is the
+# whole point of having written them that way.
+LDLS_GUEST = $(addprefix $(BUILD)/ldls/guest/, ko.o btf.o stub.o)
+
+$(BUILD)/ldls/guest/%.o: ldls/%.c ldls/ko.h ldls/btf.h ldls/stub.h
+	@mkdir -p $(BUILD)/ldls/guest
+	$(CC) $(CFLAGS) -c $< -o $@
+
+$(BUILD)/cmd/ldlsd: cmd/ldlsd/main.c $(ULIB) $(LDLS_GUEST)
+	@mkdir -p $(BUILD)/cmd
+	$(CC) $(CFLAGS) -I./ldls -c $< -o $(BUILD)/cmd/ldlsd.o
+	$(LD) $(LDFLAGS) $(LD_LOWBASE) -e _start -Ttext 0x4000 -o $@ \
+	    $(BUILD)/cmd/ldlsd.o $(LDLS_GUEST) $(ULIB)
+	$(OBJCOPY) -S $@
+
 $(BUILD)/cmd/%: cmd/%/main.c $(ULIB)
 	@mkdir -p $(BUILD)/cmd
 	$(CC) $(CFLAGS) -c $< -o $(BUILD)/cmd/$*.o
@@ -960,6 +977,29 @@ $(LDLS_TYPES): ldls/host/types.c ldls/btf.c ldls/btf.h
 
 ldls: $(LDLS_INSPECT) $(LDLS_RUN) $(LDLS_TYPES)
 
+# Put a module and its kernel's BTF into the *local* sysroot, for testing ldlsd
+# in QEMU.
+#
+#     make ldls-stage KO=/path/to/8139too.ko BTF=/sys/kernel/btf/vmlinux
+#
+# This is the model docs/plan/ldls.md describes, carried out on the machine
+# that is doing the testing: the user obtains a module and loads it on their
+# own machine. Deliberately **not** a dependency of `all`, so an ordinary build
+# never picks it up, and `.gitignore` refuses `*.ko` so it cannot be committed
+# by accident. An image built after this has somebody else's GPL code in it and
+# must not be redistributed -- which is exactly what licensing.md §6 says about
+# the combination, and why it is one command you have to type on purpose.
+ldls-stage:
+	@test -n "$(KO)" || { 	  echo "usage: make ldls-stage KO=<module.ko> [BTF=<vmlinux-btf>]"; exit 1; }
+	@mkdir -p $(SYSROOT)/lib/modules
+	cp "$(KO)" $(SYSROOT)/lib/modules/
+	@test -z "$(BTF)" || cp "$(BTF)" $(SYSROOT)/lib/modules/btf
+	@echo "staged into $(SYSROOT)/lib/modules -- this image is now yours alone"
+
+ldls-unstage:
+	rm -rf $(SYSROOT)/lib/modules
+	@echo "removed"
+
 # Load our own object and check the loader got it right. No network, no QEMU,
 # no module from anywhere.
 ldls-check: $(LDLS_INSPECT) $(LDLS_RUN) $(LDLS_TYPES) $(LDLS_REFMOD)

cmd/ldlsd/main.c

@@ -0,0 +1,462 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// ldlsd -- LDLS, on SchOS.
+//
+// docs/plan/ldls.md milestone L3b. `ldls-run` did this on the build machine so
+// that loader failures cost a second instead of a boot; this is the same
+// loader, the same BTF reader and the same trampolines, in a process on the
+// system they were written for.
+//
+//     ldlsd <module.ko> [vmlinux-btf]
+//
+// **Neither file ships with SchOS.** Both are the user's: a module they
+// obtained, and the BTF of the kernel it was built for. docs/licensing.md §6
+// is the whole argument, and `.gitignore` refusing `*.ko` is how it is kept
+// mechanically rather than by intention.
+//
+// What it does *not* yet do is implement any of the symbols the driver asks
+// for -- every one is still a stub that reports itself and returns zero, which
+// is L3c. So the useful output here is the same as `ldls-run`'s: what the
+// driver asks for, in what order, and how far it gets.
+
+#include "types.h"
+#include "user.h"
+#include "fcntl.h"
+#include "posix.h"
+#include "stat.h"
+
+#include "ko.h"
+#include "stub.h"
+#include "btf.h"
+
+// --- small output, since lib/ has no printf ---------------------------------
+
+static void
+puts_(const char *s)
+{
+    int n = 0;
+    while (s[n]) n++;
+    write(1, (void *)s, n);
+}
+
+static void
+putn(long long v)
+{
+    char b[24];
+    int i = 0;
+    if (v < 0) { write(1, "-", 1); v = -v; }
+    if (v == 0) { write(1, "0", 1); return; }
+    while (v > 0) { b[i++] = (char)('0' + (v % 10)); v /= 10; }
+    while (i-- > 0) write(1, &b[i], 1);
+}
+
+static void
+puthexn(unsigned long long v, int digits)
+{
+    static const char h[] = "0123456789abcdef";
+    char b[16];
+    int i;
+    if (digits > 16) digits = 16;
+    for (i = digits - 1; i >= 0; i--) { b[i] = h[v & 0xF]; v >>= 4; }
+    write(1, b, digits);
+}
+
+static void
+puthex(unsigned long long v)
+{
+    puts_("0x");
+    puthexn(v, 16);
+}
+
+// lib/ has no string.h either, so the two byte helpers this needs are here.
+static void
+bcopy_(void *d, const void *s, int n)
+{
+    unsigned char *a = (unsigned char *)d;
+    const unsigned char *b = (const unsigned char *)s;
+    while (n-- > 0) *a++ = *b++;
+}
+
+static int
+streq(const char *a, const char *b)
+{
+    while (*a && *a == *b) { a++; b++; }
+    return *a == 0 && *b == 0;
+}
+
+// --- allocation -------------------------------------------------------------
+//
+// mmap and nothing else. lib/ has no malloc, and this needs both large
+// buffers (a kernel's BTF is several megabytes) and memory it can execute --
+// which on SchOS is every mapping, because the kernel never sets EFER.NXE.
+// That is convenient here and is a gap rather than a feature; see the note in
+// docs/plan/ldls.md.
+//
+// The module lands in the low 2 GB automatically: a process's mmap area starts
+// at 0x40000000 and bumps upward, which is exactly the window R_X86_64_32S
+// relocations need. L1 predicted that and this is where it is collected.
+static void *
+getmem(unsigned long long n)
+{
+    void *p = mmap(0, n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS,
+                   -1, 0);
+    if (p == 0 || (long)p == -1)
+        return 0;
+    return p;
+}
+
+static unsigned char *
+slurp(const char *path, unsigned long long *len)
+{
+    struct stat st;
+    unsigned char *buf;
+    int fd, got = 0;
+
+    fd = open((char *)path, O_RDONLY);
+    if (fd < 0)
+        return 0;
+    if (fstat(fd, &st) < 0 || st.size == 0) { close(fd); return 0; }
+
+    buf = getmem(st.size);
+    if (!buf) { close(fd); return 0; }
+
+    while (got < (int)st.size) {
+        int n = read(fd, (char *)buf + got, (int)st.size - got);
+        if (n <= 0)
+            break;
+        got += n;
+    }
+    close(fd);
+    if (got != (int)st.size)
+        return 0;
+    *len = st.size;
+    return buf;
+}
+
+// --- the stubs ---------------------------------------------------------------
+
+static const char **g_names;
+static int          g_count;
+static int          g_calls;
+static unsigned long g_pci_driver;
+
+unsigned long
+ldls_stub(long idx, unsigned long *args)
+{
+    const char *name = (idx >= 0 && idx < g_count) ? g_names[idx] : "(?)";
+
+    g_calls++;
+    if (g_calls <= 40) {
+        puts_("  "); putn(g_calls); puts_("  "); puts_(name);
+        puts_("("); puthex(args[0]); puts_(", "); puthex(args[1]);
+        puts_(")\n");
+    } else if (g_calls == 41) {
+        puts_("  ... (further calls not printed)\n");
+    }
+
+    // The driver's own words, which is the clearest evidence a load worked.
+    // Linux format strings carry a leading level byte and %p extensions that
+    // nothing here can expand, so the string is printed raw.
+    if (streq(name, "_printk") || streq(name, "printk")) {
+        const char *f = (const char *)args[0];
+        if (f) {
+            if ((unsigned char)f[0] == 0x01 && f[1] >= '0' && f[1] <= '7')
+                f += 2;
+            puts_("      | "); puts_(f);
+            {
+                int n = 0;
+                while (f[n]) n++;
+                if (n == 0 || f[n - 1] != '\n') puts_("\n");
+            }
+        }
+    }
+
+    if (streq(name, "__pci_register_driver") && !g_pci_driver)
+        g_pci_driver = args[0];
+
+    return 0;
+}
+
+struct rctx { struct ko *k; struct stubs *s; };
+
+static int
+resolve(void *ctx, const char *name, unsigned long long *out)
+{
+    struct rctx *r = (struct rctx *)ctx;
+    int i;
+
+    for (i = 0; i < r->k->nundef; i++)
+        if (streq(r->k->undef[i], name)) {
+            *out = stub_addr(r->s, i);
+            return 0;
+        }
+    return -1;
+}
+
+static unsigned long long
+find_symbol(struct ko *k, const char *want)
+{
+    int i;
+    for (i = 0; i < k->nsym; i++) {
+        if (!streq(k->str + k->sym[i].st_name, want))
+            continue;
+        if (k->sym[i].st_shndx == KO_SHN_UNDEF ||
+            k->sym[i].st_shndx >= (unsigned short)k->nsh)
+            continue;
+        if (!k->sec[k->sym[i].st_shndx].alloc)
+            continue;
+        return k->sec[k->sym[i].st_shndx].addr + k->sym[i].st_value;
+    }
+    return 0;
+}
+
+// --- being the PCI core -------------------------------------------------------
+
+static unsigned int
+lay(struct btf *b, const char *type, const char *member)
+{
+    unsigned int off, size;
+    int id = btf_find(b, type, BTF_KIND_STRUCT);
+
+    if (!id) {
+        puts_("ldlsd: this BTF has no struct "); puts_(type); puts_("\n");
+        return 0xffffffffu;
+    }
+    if (!member)
+        return btf_type_size(b, id);
+    if (btf_member_off(b, id, member, &off, &size) < 0) {
+        puts_("ldlsd: "); puts_(type); puts_("."); puts_(member);
+        puts_(": "); puts_(btf_err(b)); puts_("\n");
+        return 0xffffffffu;
+    }
+    return off;
+}
+
+static unsigned int
+rd32(unsigned long base, unsigned int off)
+{ unsigned int v; bcopy_(&v, (void *)(base + off), 4); return v; }
+
+static unsigned long
+rd64(unsigned long base, unsigned int off)
+{ unsigned long v; bcopy_(&v, (void *)(base + off), 8); return v; }
+
+static void
+wr16(unsigned long base, unsigned int off, unsigned short v)
+{ bcopy_((void *)(base + off), &v, 2); }
+
+static void
+wr32(unsigned long base, unsigned int off, unsigned int v)
+{ bcopy_((void *)(base + off), &v, 4); }
+
+static void
+wr64(unsigned long base, unsigned int off, unsigned long v)
+{ bcopy_((void *)(base + off), &v, 8); }
+
+#define LDLS_IORESOURCE_IO  0x00000100
+#define LDLS_IORESOURCE_MEM 0x00000200
+
+static void
+drive_probe(const char *btfpath)
+{
+    struct btf b;
+    unsigned char *bimg;
+    unsigned long long blen;
+    unsigned int *boff;
+    long nt;
+    unsigned int drv_probe, drv_idtable, drv_name;
+    unsigned int id_size, id_vendor, id_device;
+    unsigned int dev_size, dev_vendor, dev_device, dev_devfn, dev_irq, dev_res;
+    unsigned int res_size, res_start, res_end, res_flags;
+    unsigned long idtab, pdev;
+    int nid = 0;
+
+    bimg = slurp(btfpath, &blen);
+    if (!bimg) { puts_("ldlsd: cannot read the BTF\n"); return; }
+
+    nt = btf_count(bimg, blen);
+    if (nt < 0) { puts_("ldlsd: that file is not BTF\n"); return; }
+
+    boff = getmem(((unsigned long long)nt + 2) * 4);
+    if (!boff) { puts_("ldlsd: no memory for the BTF index\n"); return; }
+    if (btf_open(&b, bimg, blen, boff, (unsigned int)nt + 2) < 0) {
+        puts_("ldlsd: "); puts_(btf_err(&b)); puts_("\n");
+        return;
+    }
+    puts_("  btf     "); putn(nt); puts_(" types\n");
+
+    drv_probe   = lay(&b, "pci_driver", "probe");
+    drv_idtable = lay(&b, "pci_driver", "id_table");
+    drv_name    = lay(&b, "pci_driver", "name");
+    id_size     = lay(&b, "pci_device_id", 0);
+    id_vendor   = lay(&b, "pci_device_id", "vendor");
+    id_device   = lay(&b, "pci_device_id", "device");
+    dev_size    = lay(&b, "pci_dev", 0);
+    dev_vendor  = lay(&b, "pci_dev", "vendor");
+    dev_device  = lay(&b, "pci_dev", "device");
+    dev_devfn   = lay(&b, "pci_dev", "devfn");
+    dev_irq     = lay(&b, "pci_dev", "irq");
+    dev_res     = lay(&b, "pci_dev", "resource");
+    res_size    = lay(&b, "resource", 0);
+    res_start   = lay(&b, "resource", "start");
+    res_end     = lay(&b, "resource", "end");
+    res_flags   = lay(&b, "resource", "flags");
+
+    if (drv_probe == 0xffffffffu || dev_size == 0xffffffffu ||
+        res_size == 0xffffffffu) {
+        puts_("ldlsd: this kernel's BTF cannot drive probe()\n");
+        return;
+    }
+
+    puts_("  layout  pci_driver.probe +"); putn(drv_probe);
+    puts_("  pci_dev "); putn(dev_size); puts_(" bytes\n");
+
+    {
+        const char *dn = (const char *)rd64(g_pci_driver, drv_name);
+        if (dn) { puts_("  driver  \""); puts_(dn); puts_("\"\n"); }
+    }
+
+    idtab = rd64(g_pci_driver, drv_idtable);
+    if (!idtab) { puts_("  it has no id_table\n"); return; }
+
+    puts_("\n  devices it claims:\n");
+    for (;;) {
+        unsigned long e = idtab + (unsigned long)nid * id_size;
+        unsigned int v = rd32(e, id_vendor), d = rd32(e, id_device);
+        if (!v && !d) break;
+        if (nid < 6) {
+            puts_("    "); puthexn(v, 4); puts_(":"); puthexn(d, 4);
+            puts_("\n");
+        }
+        nid++;
+        if (nid > 4096) break;
+    }
+    if (nid > 6) { puts_("    ... and "); putn(nid - 6); puts_(" more\n"); }
+    if (!nid) return;
+
+    pdev = (unsigned long)getmem(dev_size);
+    if (!pdev) { puts_("ldlsd: no memory for a pci_dev\n"); return; }
+    {
+        unsigned long r0 = pdev + dev_res;
+        unsigned long r1 = r0 + res_size;
+
+        wr16(pdev, dev_vendor, (unsigned short)rd32(idtab, id_vendor));
+        wr16(pdev, dev_device, (unsigned short)rd32(idtab, id_device));
+        wr32(pdev, dev_devfn, 0x18);
+        wr32(pdev, dev_irq, 11);
+        wr64(r0, res_start, 0xc000);
+        wr64(r0, res_end,   0xc0ff);
+        wr64(r0, res_flags, LDLS_IORESOURCE_IO);
+        wr64(r1, res_start, 0xfebc0000);
+        wr64(r1, res_end,   0xfebc00ff);
+        wr64(r1, res_flags, LDLS_IORESOURCE_MEM);
+    }
+
+    puts_("\n  calling probe\n\n");
+    g_calls = 0;
+    {
+        int (*probe)(void *, void *) =
+            (int (*)(void *, void *))(void *)rd64(g_pci_driver, drv_probe);
+        if (!probe) { puts_("  the driver has no probe\n"); return; }
+        {
+            int r = probe((void *)pdev, (void *)idtab);
+            puts_("\n  probe returned "); putn(r);
+            puts_(" after "); putn(g_calls); puts_(" stub call(s)\n");
+        }
+    }
+}
+
+// --- main ----------------------------------------------------------------------
+
+int
+main(int argc, char *argv[])
+{
+    struct ko k;
+    struct ko_section *sec;
+    struct stubs stubs;
+    struct rctx rc;
+    unsigned char *img, *mem, *code;
+    unsigned long long len, span, entry;
+    int nsec, i;
+
+    if (argc < 2) {
+        puts_("usage: ldlsd <module.ko> [vmlinux-btf]\n");
+        puts_("  neither file ships with SchOS; both are yours.\n");
+        exit();
+    }
+
+    img = slurp(argv[1], &len);
+    if (!img) { puts_("ldlsd: cannot read "); puts_(argv[1]); puts_("\n"); exit(); }
+
+    nsec = ko_sections_needed(img, len);
+    if (nsec < 0) {
+        puts_("ldlsd: that is not an ELF64 relocatable object\n");
+        exit();
+    }
+    sec = getmem((unsigned long long)(nsec + 1) * sizeof(struct ko_section));
+    if (!sec) { puts_("ldlsd: out of memory\n"); exit(); }
+
+    if (ko_open(&k, img, len, sec, nsec) < 0) {
+        puts_("ldlsd: "); puts_(ko_err(&k)); puts_("\n");
+        exit();
+    }
+
+    puts_(argv[1]); puts_("\n");
+    puts_("  module  ");
+    puts_(ko_name(&k) ? ko_name(&k) : "(not a kernel module)"); puts_("\n");
+    puts_("  license ");
+    puts_(ko_modinfo(&k, "license") ? ko_modinfo(&k, "license")
+                                    : "(none declared)"); puts_("\n");
+    puts_("  wants   "); putn(k.nundef); puts_(" symbols\n");
+
+    span = ko_span(&k);
+    mem  = getmem(span ? span : 4096);
+    code = getmem((unsigned long long)(k.nundef + 1) * STUB_SIZE);
+    if (!mem || !code) { puts_("ldlsd: out of memory\n"); exit(); }
+
+    stub_build(&stubs, code, k.nundef);
+    g_names = k.undef;
+    g_count = k.nundef;
+    for (i = 0; i < k.nundef; i++)
+        if (stub_is_bare(k.undef[i]))
+            stub_make_bare(code + (unsigned long long)i * STUB_SIZE);
+
+    if (ko_layout(&k, mem, span, (unsigned long long)mem) < 0) {
+        puts_("ldlsd: layout: "); puts_(ko_err(&k)); puts_("\n");
+        exit();
+    }
+
+    rc.k = &k; rc.s = &stubs;
+    if (ko_relocate(&k, resolve, &rc) < 0) {
+        puts_("ldlsd: relocate: "); puts_(ko_err(&k)); puts_("\n");
+        exit();
+    }
+    // The stack canary is fetched through %gs, which belongs to the kernel
+    // here. See ko_defuse_gs().
+    {
+        int g = ko_defuse_gs(&k, "__ref_stack_chk_guard");
+        if (g < 0) g = 0;
+        if (g == 0) g = ko_defuse_gs(&k, "__stack_chk_guard");
+        if (g > 0) { puts_("  defused "); putn(g); puts_(" %gs stack-guard load(s)\n"); }
+    }
+
+    puts_("  loaded  "); putn(span); puts_(" bytes at "); puthex((unsigned long long)mem);
+    puts_("\n");
+
+    entry = find_symbol(&k, "init_module");
+    if (!entry) { puts_("  no init_module -- nothing to call\n"); exit(); }
+
+    puts_("\n  calling init_module\n\n");
+    {
+        int (*init)(void) = (int (*)(void))(void *)entry;
+        int r = init();
+        puts_("\n  init_module returned "); putn(r);
+        puts_(" after "); putn(g_calls); puts_(" stub call(s)\n");
+    }
+
+    if (argc > 2 && g_pci_driver)
+        drive_probe(argv[2]);
+    else if (argc > 2)
+        puts_("\n  no pci_driver was registered -- nothing to probe\n");
+
+    exit();
+}

docs/plan/1.0.md

@@ -95,7 +95,7 @@ The galling ones. The work exists, is tested, and nothing uses it.
 |---|---|
 | **`foundations.md`** | Item 1 built. Items 2 (a shareable memory object), 3 (`AF_UNIX` in the filesystem namespace), 4 and 5 open. **Wayland stops at item 2.** |
 | ~~**`hybrid.md`**~~ | **Closed.** All four mechanisms built; H1, H2 and H3 done. H4 — the USB eviction — is *withdrawn* rather than outstanding: USB is the bus peripherals arrive on, not a peripheral, and an eviction that has to leave an identical driver behind has moved nothing. See [hybrid.md](hybrid.md), "Decided: USB stays in the kernel". |
-| **`ldls.md`** | L0, L1, L2 and L3a done — the loader handles 2,440 stock modules; 449 of 500 run `init_module`; `8139too`'s `probe()` runs to completion; and `/dev/netdev` lets a process be an interface, with `ping` working over one. L3b (the SchOS-side host) and L3c (the fifteen symbols, for real) open. No Linux driver has yet touched hardware. |
+| **`ldls.md`** | L0, L1, L2 and L3a done — the loader handles 2,440 stock modules; 449 of 500 run `init_module`; `8139too`'s `probe()` runs to completion; and `/dev/netdev` lets a process be an interface, with `ping` working over one. L3b done too — `ldlsd` runs the same loader on SchOS and gets a real driver's `probe()` to return. L3c (the fifteen symbols, against `/dev/pci`) open. No Linux driver has yet touched hardware. |
 | **Schcom v0** | 715 lines that composite, over a wire format that is superseded. Frozen: it runs, nothing more is added, and it is kept only because it is the one thing in the tree that has ever put a client's pixels on screen. |
 | **`debugger.md`** | D0 through D4 built and described in [../architecture/debugging.md](../architecture/debugging.md). Only the network transport is open — a userspace daemon bridging TCP to `/dev/dbg`, which is deliberately last. |
 

docs/plan/ldls.md

@@ -182,7 +182,7 @@ One device, one process, claimed and released, on the `/dev/pci` discipline.
 | **L1** | **Stubs.** Every symbol resolves, to a function that logs its name. `init_module` runs, which yields the *call order* — the second thing you need and the other thing you cannot guess. | **done** — `ldls/stub.c`, `ldls-run` |
 | **L2** | **The hardware.** `probe()` receives a working `struct pci_dev`, maps its BARs, allocates coherent DMA and registers an interrupt, all through `/dev/pci`. The driver reaches its device. | **done**, except `/dev/pci` — see below |
 | **L3a** | **The class bridge.** `/dev/netdev`: a process claims it and becomes an interface. `ping` over a *process*. | **done** — `kernel/dev/netuser.c`, `cmd/netecho` |
-| **L3b** | The SchOS-side host: `ldls-run`'s loader and BTF work, as a program on SchOS, reaching hardware through `/dev/pci`. | |
+| **L3b** | The SchOS-side host: `ldls-run`'s loader and BTF work, as a program on SchOS. | **done** — `cmd/ldlsd`; `/dev/pci` is L3c's half |
 | **L3c** | The fifteen symbols, implemented rather than stubbed, against a real rtl8139. | |
 | **L3** | **A packet.** The two joined: **`ping` over a Linux driver.** This is the milestone that proves the whole idea and the one that cannot be skipped. | |
 | **L4** | **Kill it.** The host process dies, the kernel clears bus mastering and reclaims the DMA, a new host starts and the interface comes back. | free — H2 built all of this |
@@ -547,6 +547,94 @@ allowed to include lwIP's headers.
 
 ---
 
+## L3b: the same loader, on SchOS
+
+`ldls/ko.c`, `ldls/btf.c` and `ldls/stub.c` were written freestanding so they
+would build unchanged on both sides. `cmd/ldlsd` is the collection of that
+promise: the same three files, linked against `lib/` instead of a libc, in an
+ordinary SchOS process.
+
+```
+# ldlsd /lib/modules/8139too.ko /lib/modules/btf
+/lib/modules/8139too.ko
+  module  8139too
+  license GPL
+  wants   94 symbols
+  defused 6 %gs stack-guard reference(s)
+  loaded  33988 bytes at 0x000000004001f000
+
+  calling init_module
+  1  _printk(...)
+      | 8139too: 8139too Fast Ethernet driver 0.9.28
+  2  __pci_register_driver(...)
+  init_module returned 0 after 2 stub call(s)
+
+  btf     122279 types
+  layout  pci_driver.probe +16  pci_dev 2680 bytes
+  driver  "8139too"
+  devices it claims:
+    10ec:8139
+    ... and 18 more
+
+  calling probe
+  1  alloc_etherdev_mqs(0x368, 0x1)
+  probe returned -12 after 1 stub call(s)
+```
+
+Byte for byte the behaviour of the host harness, which is what the shared
+source was for. Two predictions from earlier milestones came good without any
+work: a process's mmap area starts at `0x40000000`, so the module lands inside
+the window `R_X86_64_32S` needs, and the kernel never sets `EFER.NXE`, so every
+mapping is executable.
+
+That second one is **a gap, not a feature**. A process being able to execute
+its own heap is normally a thing an operating system prevents, and LDLS is
+quietly relying on its absence. It belongs on Gate 0's list.
+
+### The one thing that did not just work
+
+The first run on SchOS faulted before probe's first call:
+
+```
+pid 8 ldlsd: trap 14 err 5 on cpu 0 rip 0x4002138e addr 0xffff80007de5fc60
+```
+
+`err 5` is present, read, user mode, and the address is a *kernel* one. The rip
+resolved to `.text+0x206e`, which is:
+
+```
+mov %gs:0x0(%rip),%rax      # rtl8139_init_one+0x26
+```
+
+Linux builds modules with `-mstack-protector-guard-reg=gs`, so the stack canary
+is fetched through the GS segment. That is an ABI between the kernel and its
+own per-CPU area, and **a host that is not that kernel cannot provide it** — on
+SchOS the GS base belongs to the kernel, so a user-mode read of it faults on a
+kernel address.
+
+`ko_defuse_gs()` rewrites those references to `xor %reg, %reg`. There are two
+forms and the second is easy to miss: the prologue *loads* the guard with `mov`
+(`8b`), and the epilogue *compares* it with `sub` (`2b`) followed by a `jne` to
+`__stack_chk_fail`. `xor` is correct for both — it makes the saved canary zero,
+and it sets ZF so the epilogue's branch falls through exactly as a passing
+check does. Six sites in `8139too`.
+
+It checks every site before patching any, and refuses the whole operation on
+anything it does not recognise. A loader that half-understands an instruction
+should not be editing it — and that refusal is what found the `sub` form, by
+declining to guess.
+
+### Getting the files there
+
+`make ldls-stage KO=… BTF=…` copies a module and its kernel's BTF into the
+local sysroot. It is deliberately not part of `all`, `.gitignore` refuses
+`*.ko`, and `make ldls-unstage` removes them again. An image built between
+those two commands has somebody else's GPL code in it and must not be
+redistributed — which is exactly what [../licensing.md](../licensing.md) §6
+says about the combination, and why it is a thing you type on purpose.
+
+---
+
 ## What it will not do
 
 - **It is not a security boundary, and here that is worse than usual.** There
@@ -592,6 +680,14 @@ allowed to include lwIP's headers.
 - **`ldls-run` is a development harness, not the deliverable.** It runs on the
   build machine so that loader failures cost a second instead of a boot. The
   thing that ships is the SchOS-side host.
+- **The stack protector is gone from a hosted driver.** `ko_defuse_gs()`
+  removes it because the ABI it depends on cannot be supplied here. That is a
+  hardening measure lost, on foreign code, in a host with no IOMMU — worth
+  saying plainly rather than filing as a detail.
+- **LDLS relies on SchOS having no NX.** Every mapping is executable because
+  `EFER.NXE` is never set. If that is ever fixed — and it should be — `ldlsd`
+  needs a way to ask for executable memory, and nothing in `mmap` currently
+  offers one.
 - **BTF has to come from somewhere.** On Linux it is `/sys/kernel/btf/vmlinux`.
   SchOS has no such file and never will — so the user brings the BTF along with
   the module, from the kernel the module was built for. Both are theirs,

ldls/host/run.c

@@ -424,6 +424,14 @@ main(int argc, char **argv)
     fprintf(stderr, "ldls-run: relocate: %s\n", ko_err(&k));
     return 1;
   }
+  {
+    int g = ko_defuse_gs(&k, "__ref_stack_chk_guard");
+    if (g < 0) g = 0;
+    if (g == 0) g = ko_defuse_gs(&k, "__stack_chk_guard");
+    if (g > 0)
+      printf("  defused %d %%gs stack-guard reference(s)\n", g);
+  }
+
   printf("  loaded  %llu bytes at %p, %d stubs at %p\n\n",
          span, (void *)mem, k.nundef, (void *)code);
 

ldls/ko.c

@@ -494,6 +494,109 @@ ko_relocate(struct ko *k, ko_resolver r, void *ctx)
   return 0;
 }
 
+// --- the %gs stack guard ------------------------------------------------------
+
+// Rewrite one `mov %gs:disp32(%rip), %reg` into `xor %reg, %reg` plus padding.
+//
+// The encoding is fixed and short: 65 (the GS prefix), a REX byte, 8b (MOV
+// r64, r/m64), a ModRM of mod=00 reg=DEST rm=101 (RIP-relative), then the
+// displacement. Eight bytes, of which the last four are what the relocation
+// was going to fill in.
+//
+// The replacement has to clear the *same* register, so the destination is
+// decoded rather than assumed: REX.R is the high bit and ModRM.reg the low
+// three. `xor %rN, %rN` is three bytes, and the rest is padded with nops.
+static int
+defuse_one(unsigned char *insn)
+{
+  unsigned int rex, modrm, reg;
+
+  if (insn[0] != 0x65)                 return -1;   /* not GS-prefixed   */
+  if ((insn[1] & 0xF0) != 0x40)        return -1;   /* not a REX byte    */
+  // Two forms, and both have to go. The prologue *loads* the guard with
+  // MOV (8b); the epilogue *compares* it with SUB (2b), followed by a jne to
+  // __stack_chk_fail. Replacing either with `xor %reg,%reg` is right: the
+  // first makes the saved canary zero, and the second sets the register to
+  // zero *and* sets ZF, so the jne falls through exactly as a passing check
+  // does.
+  if (insn[2] != 0x8b && insn[2] != 0x2b) return -1;
+  rex   = insn[1];
+  modrm = insn[3];
+  if ((modrm & 0xC7) != 0x05)          return -1;   /* not mod=00 rm=101 */
+
+  reg = ((rex & 0x4) << 1) | ((modrm >> 3) & 0x7);
+
+  insn[0] = (unsigned char)(0x48 | ((reg >= 8) ? 0x05 : 0x00));  /* REX.W(+R+B) */
+  insn[1] = 0x31;                                                /* XOR r/m64,r64 */
+  insn[2] = (unsigned char)(0xC0 | ((reg & 7) << 3) | (reg & 7));
+  insn[3] = 0x90;
+  insn[4] = 0x90; insn[5] = 0x90; insn[6] = 0x90; insn[7] = 0x90;
+  return 0;
+}
+
+int
+ko_defuse_gs(struct ko *k, const char *sym)
+{
+  int i, n = 0;
+
+  if (!k->laid_out)
+    { ko_fail(k, "defuse before layout", 0); return -1; }
+
+  // Two passes: check everything first, then patch. A loader that rewrites
+  // half the instructions and then refuses has left the object in a state
+  // nobody can reason about.
+  for (int pass = 0; pass < 2; pass++) {
+    n = 0;
+    for (i = 0; i < k->nsh; i++) {
+      const struct ko_shdr *sh = ko_sh(k, i);
+      struct ko_section *tgt;
+      unsigned long long cnt, j;
+
+      if (!sh || sh->sh_type != KO_SHT_RELA || !k->sec[i].data)
+        continue;
+      if (sh->sh_info >= (unsigned int)k->nsh)
+        continue;
+      tgt = &k->sec[sh->sh_info];
+      if (!tgt->alloc || tgt->size == 0)
+        continue;
+
+      cnt = sh->sh_size / sizeof(struct ko_rela);
+      for (j = 0; j < cnt; j++) {
+        const struct ko_rela *rl =
+            (const struct ko_rela *)(k->sec[i].data) + j;
+        unsigned int si = KO_R_SYM(rl->r_info);
+        const char *nm;
+
+        if ((int)si >= k->nsym)
+          continue;
+        nm = k->str + k->sym[si].st_name;
+        if (!ko_streq(nm, sym))
+          continue;
+        if (rl->r_offset < 4 || rl->r_offset + 4 > tgt->size)
+          { ko_fail(k, "a guard relocation is at an impossible offset", 0);
+            return -1; }
+
+        {
+          unsigned char *insn = tgt->where + rl->r_offset - 4;
+          if (pass == 0) {
+            // Check only.
+            if (insn[0] != 0x65 || (insn[1] & 0xF0) != 0x40 ||
+                (insn[2] != 0x8b && insn[2] != 0x2b) ||
+                (insn[3] & 0xC7) != 0x05)
+              { ko_fail(k, "unrecognised instruction at a guard reference: ",
+                        sym); return -1; }
+          } else if (defuse_one(insn) < 0) {
+            ko_fail(k, "failed to rewrite a guard reference", 0);
+            return -1;
+          }
+          n++;
+        }
+      }
+    }
+  }
+  return n;
+}
+
 // --- what the object says about itself ---------------------------------------
 
 static const struct ko_section *

ldls/ko.h

@@ -199,6 +199,32 @@ void ko_reltypes(struct ko *k);
 // makes this fail, naming the first one.
 int ko_relocate(struct ko *k, ko_resolver r, void *ctx);
 
+// Neutralise every `%gs:`-relative load of a named symbol.
+//
+// Linux builds modules with `-mstack-protector-guard-reg=gs`, so the stack
+// canary is fetched through the GS segment:
+//
+//     mov %gs:__ref_stack_chk_guard(%rip), %rax     (prologue)
+//     sub %gs:__ref_stack_chk_guard(%rip), %rax     (epilogue, then jne)
+//
+// That is an ABI between the kernel and its own per-CPU area, and a host which
+// is not that kernel cannot provide it. On SchOS the GS base belongs to the
+// kernel, so a user-mode read of it faults on a kernel address -- which is
+// exactly where 8139too's probe stopped the first time it ran there.
+//
+// So the instruction is rewritten to zero the destination register. The canary
+// becomes zero, the check at the end of the function compares zero against
+// zero, and `__stack_chk_fail` is never reached. What is lost is a hardening
+// measure that was never going to work here; what is gained is a driver that
+// runs.
+//
+// Call it **after** ko_relocate(), which will have written a displacement into
+// bytes this then overwrites entirely. Returns how many it patched, or -1 if
+// it found something at that offset it did not recognise -- in which case
+// nothing is modified, because a loader that half-understands an instruction
+// should not be editing it.
+int ko_defuse_gs(struct ko *k, const char *sym);
+
 // A value from the `.modinfo` section -- "license", "description", "author",
 // "alias". Returns 0 if the object has no such key.
 //