ldls: the top-2 GB window Linux modules link against

A Linux module is built with -mcmodel=kernel and can only reach addresses
in the top 2 GB, where this kernel is not linked. So the kernel image is
mapped a second time at LDLS_KALIAS, with a vmemmap area and a module area
beside it, in one PDPT shared by every address space. Nothing loads a
module yet; this is the address-space groundwork, set up at every boot,
and include/ldls/ is the interface the loader will use.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4

include/arch/memlayout.h

@@ -91,6 +91,22 @@ extern uint64 lowtop;
 #define KERNPHYS 0x1000000
 #define KERNLINK (KERNBASE+KERNPHYS) // Address where kernel is linked
 
+// The LDLS window, in the top 2 GB, where a Linux module expects to live. See
+// ldls_vm_init() in kernel/arch/x86_64/vm.c.
+//
+//   LDLS_VMEMMAP  1 GB   Linux's `struct page` array, 64 bytes per frame --
+//                        enough for 64 GB, not for all of PHYSTOP_MAX; a
+//                        frame above that has no struct page. Backed on demand.
+//   LDLS_KALIAS   512 MB the kernel image again, from physical zero
+//   LDLS_MODBASE  512 MB modules, their stacks and their vmalloc
+#define LDLS_VMEMMAP 0xFFFFFFFF40000000ULL
+#define LDLS_KALIAS  0xFFFFFFFF80000000ULL
+#define LDLS_MODBASE 0xFFFFFFFFA0000000ULL
+#define LDLS_MODEND  0xFFFFFFFFC0000000ULL
+
+// A kernel address, as a module sees it.
+#define LDLS_ALIAS(a) ((uint64)(a) - KERNBASE + LDLS_KALIAS)
+
 #ifndef __ASSEMBLER__
 static inline addr_t v2p(void *a) {
   return ((addr_t)(a)) - ((addr_t)KERNBASE);

include/ldls/kldls.h

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#pragma once
+
+// The kernel's side of LDLS, as the rest of the kernel sees it. The layer
+// itself is kernel/ldls/; this is the handful of places it is called from.
+
+struct trapframe;
+
+// kernel/arch/x86_64/vm.c -- the window in the top 2 GB.
+void    ldls_vm_init(void);
+int     kmap_page(uint64 va, addr_t pa);
+addr_t  kmap_phys(uint64 va);
+addr_t  kunmap_page(uint64 va);
+
+// kernel/ldls/core.c
+void    ldls_init(void);
+int     ldls_ioctl(uint64 request, uint64 argp);
+
+// A trap in kernel mode. Returns 1 if LDLS dealt with it -- a WARN() that was
+// reported and stepped over, or a hosted driver that faulted and has been
+// stopped -- and 0 if it is the kernel's own and should panic as before.
+int     ldls_trap(struct trapframe *tf);
+
+// The name of an address in a hosted module, for a backtrace. 0 if `a` is not
+// in one.
+int     ldls_symbolize(uint64 a, char *out, int cap);

include/ldls/ldls.h

@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#pragma once
+
+// /dev/ldls -- Linux drivers, in the kernel. docs/architecture/ldls.md.
+//
+// The device is a control channel and nothing else: a module is handed over
+// whole, as bytes, together with the BTF of the kernel it was built for, and
+// the kernel does the rest -- loading, linking, init_module, probe, and
+// bringing the interface up -- on a thread of its own. What happened comes
+// back as text, because the useful answer to "did it work" is the driver's
+// own account of what it found.
+//
+//     ldls load /lib/modules/e1000.ko
+//     ldls list
+//     ldls unload e1000
+//
+// Shared with userland (cmd/ldls), so it is spelled in plain C types.
+
+#define LDLS_MAJOR 13
+
+// Take the device even if a kernel driver holds it, provided that driver can
+// be asked to let go. Without this LDLS only takes devices nothing else
+// drives, which is the right default at boot: the kernel's own driver is on
+// the path to a shell, and the hosted one is the better one only when someone
+// says so.
+#define LDLS_TAKE     0x1
+// Load and link, run init_module, and stop: no device is probed. For a
+// library module (mii, libphy) whose job is to export symbols to the next.
+#define LDLS_NOPROBE  0x2
+
+struct ldls_load {
+  unsigned long long ko;        // in: the module image
+  unsigned long long kolen;
+  unsigned long long btf;       // in: the kernel's BTF, /sys/kernel/btf/vmlinux
+  unsigned long long btflen;
+  unsigned long long msg;       // out: what happened, NUL-terminated
+  unsigned int       msgcap;
+  unsigned int       flags;     // LDLS_*
+  int                bus, slot, func;   // -1 for "whichever matches"
+  unsigned int       hz;        // CONFIG_HZ of that kernel; 0 means 1000
+  int                id;        // out: the module's id, for unload
+};
+
+struct ldls_text {
+  unsigned long long buf;       // out
+  unsigned int       cap;
+  unsigned int       pad;
+};
+
+#define LDLSIOC_LOAD    0xD000  // struct ldls_load
+#define LDLSIOC_UNLOAD  0xD001  // in: int id
+#define LDLSIOC_LIST    0xD002  // struct ldls_text
+#define LDLSIOC_LOG     0xD003  // struct ldls_text, in: pad = id

kernel/arch/x86_64/vm.c

@@ -7,7 +7,9 @@
 #include "arch/mmu.h"
 #include "core/proc.h"
 #include "arch/ioperm.h"
+#include "core/spinlock.h"
 #include "arch/vmap.h"
+#include "ldls/kldls.h"
 
 int mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm);
 static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc);
@@ -15,7 +17,7 @@ static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc);
 
 // `cpu` and `proc` are no longer __thread variables; they live at fixed offsets
 // in the per-CPU block and are read through %gs at each use. See the long
-// comment in include/core/proc.h for why -- the short version is that __thread means
+// comment in include/proc.h for why -- the short version is that __thread means
 // per *thread*, the compiler caches the thread pointer across calls on that
 // basis, and a kernel thread changes CPU at every sched().
 
@@ -156,7 +158,7 @@ seginit(void)
   gdt = (struct segdesc*) local;
 
   // The TSS moved out of this page. It now carries an I/O permission bitmap
-  // (include/arch/ioperm.h) which does not fit here, and it has to be one object
+  // (include/ioperm.h) which does not fit here, and it has to be one object
   // with it. ioperm_tss() returns it and the descriptor limit that reaches the
   // end of the bitmap.
   tss = (uint*) ioperm_tss(cpunum(), &tsslim);
@@ -179,7 +181,7 @@ seginit(void)
   *(uint64 *) kernel_fs_base = kernel_fs_base;
 
   // Clear the per-CPU slots before anything can read them. Their offsets are
-  // the PERCPU_* constants in include/core/proc.h and are shared with trapasm.S,
+  // the PERCPU_* constants in include/proc.h and are shared with trapasm.S,
   // which reads %gs:24 for proc on the way out to user mode and parks three
   // registers at %gs:32..48 across the FS_BASE swap on the way in.
   ((uint64 *) kernel_fs_base)[2] = 0;      // struct cpu *   (PERCPU_CPU)
@@ -256,6 +258,10 @@ setupkvm(void)
   memset(pml4, 0, PGSIZE);
 
   pml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W;
+  // The LDLS window: the kernel's second mapping and the modules above it.
+  // A module's interrupt and its kernel thread can run in any address space,
+  // so it has to be in all of them. See ldls_vm_init().
+  pml4[PMX(LDLS_KALIAS)] = kpml4[PMX(LDLS_KALIAS)];
 
   // The framebuffer is deliberately NOT mapped here.
   //
@@ -318,9 +324,157 @@ kvmalloc(void)
   // correct only while the direct map was exactly four gigabytes.
   ioremap_next = (int)gigs;
 
+  ldls_vm_init();
+
   switchkvm();
 }
 
+// --- the top two gigabytes, for LDLS -----------------------------------------
+//
+// A Linux module is compiled with -mcmodel=kernel, which is a promise that the
+// module and everything it calls live in the top 2 GB of the address space:
+// R_X86_64_32S relocations sign-extend a 32-bit field, so the only addresses
+// they can name are 0xffffffff80000000 and above. This kernel is linked at
+// KERNBASE, nowhere near there, so a module cannot be relocated against it.
+//
+// The fix is the layout Linux itself uses. The kernel image is mapped a second
+// time at LDLS_KALIAS -- same physical pages, so the same code and the same
+// variables -- and modules load at LDLS_MODBASE, a few hundred megabytes above
+// it. Every symbol a module imports then resolves to its alias address, and a
+// call or a data reference reaches it with a 32-bit displacement.
+//
+// Running kernel code at the alias is sound because the whole image moves
+// together: an absolute reference still names the KERNBASE copy, and a
+// rip-relative one names the alias of the same byte. Either way it is the same
+// memory.
+//
+// One PDPT under PML4 slot 511, shared by every address space exactly as
+// kpdpt is, so a mapping made here is visible everywhere at once. Nothing in
+// it is user-accessible: the leaves carry no PTE_U, and the U/S bit is the AND
+// of every level.
+static pdpe_t *hipdpt;
+static struct spinlock kmaplock;
+
+void
+ldls_vm_init(void)
+{
+  extern char end[];
+  pde_t *alias, *vmm;
+  uint64 top, i;
+
+  initlock(&kmaplock, "kmap");
+
+  hipdpt = (pdpe_t *)kalloc();
+  alias  = (pde_t *)kalloc();
+  vmm    = (pde_t *)kalloc();
+  if (!hipdpt || !alias || !vmm)
+    panic("ldls_vm_init");
+  memset(hipdpt, 0, PGSIZE);
+  memset(alias, 0, PGSIZE);
+  memset(vmm, 0, PGSIZE);
+
+  // The alias, in 2 MB pages from physical zero to past the end of the image.
+  // It covers some memory that is not the kernel, which is harmless: it is a
+  // second name for pages the direct map already names.
+  top = ((uint64)V2P(end) + (2ULL << 20) - 1) >> 21;
+  if (top > 256)
+    panic("ldls_vm_init: kernel image larger than 512 MB");
+  for (i = 0; i < top; i++)
+    alias[i] = (i << 21) | PTE_PS | PTE_P | PTE_W;
+
+  hipdpt[PDPX(LDLS_KALIAS)]  = v2p(alias) | PTE_P | PTE_W;
+  hipdpt[PDPX(LDLS_VMEMMAP)] = v2p(vmm) | PTE_P | PTE_W;
+  kpml4[PMX(LDLS_KALIAS)]    = v2p(hipdpt) | PTE_P | PTE_W;
+}
+
+// Map one page in the LDLS window. Returns 0, or -1 if the slot is taken or a
+// page table could not be had.
+int
+kmap_page(uint64 va, addr_t pa)
+{
+  pde_t *pd, *pde;
+  pte_t *pt, *pte;
+
+  if (PMX(va) != PMX(LDLS_KALIAS) || !(hipdpt[PDPX(va)] & PTE_P))
+    return -1;
+
+  acquire(&kmaplock);
+  pd  = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)]));
+  pde = &pd[PDX(va)];
+  if (*pde & PTE_PS) {
+    release(&kmaplock);
+    return -1;                      // the alias; never mapped over
+  }
+  if (!(*pde & PTE_P)) {
+    if ((pt = (pte_t *)kalloc()) == 0) {
+      release(&kmaplock);
+      return -1;
+    }
+    memset(pt, 0, PGSIZE);
+    *pde = v2p(pt) | PTE_P | PTE_W;
+  }
+  pt  = (pte_t *)P2V(PTE_ADDR(*pde));
+  pte = &pt[PTX(va)];
+  if (*pte & PTE_P) {
+    release(&kmaplock);
+    return -1;
+  }
+  *pte = pa | PTE_P | PTE_W;
+  release(&kmaplock);
+  invlpg((void *)va);
+  return 0;
+}
+
+// What is mapped at `va` in the window, or 0.
+addr_t
+kmap_phys(uint64 va)
+{
+  pde_t *pd, *pde;
+  pte_t *pt;
+
+  if (PMX(va) != PMX(LDLS_KALIAS) || !(hipdpt[PDPX(va)] & PTE_P))
+    return 0;
+  pd  = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)]));
+  pde = &pd[PDX(va)];
+  if (!(*pde & PTE_P))
+    return 0;
+  if (*pde & PTE_PS)
+    return PTE_ADDR(*pde) + (va & ((2ULL << 20) - 1));
+  pt = (pte_t *)P2V(PTE_ADDR(*pde));
+  if (!(pt[PTX(va)] & PTE_P))
+    return 0;
+  return PTE_ADDR(pt[PTX(va)]) + (va & (PGSIZE - 1));
+}
+
+// Take a page out of the window and say which frame it was, or 0.
+//
+// Only this CPU's TLB is flushed. There is no shootdown IPI in this kernel, so
+// another CPU may keep a stale translation for a while -- which is why the
+// LDLS window never hands the same virtual address out twice. A stale entry
+// for an address nobody will use again is harmless.
+addr_t
+kunmap_page(uint64 va)
+{
+  addr_t pa = kmap_phys(va);
+  pde_t *pd, *pde;
+  pte_t *pt;
+
+  if (!pa)
+    return 0;
+  acquire(&kmaplock);
+  pd  = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)]));
+  pde = &pd[PDX(va)];
+  if (!(*pde & PTE_PS)) {
+    pt = (pte_t *)P2V(PTE_ADDR(*pde));
+    pt[PTX(va)] = 0;
+  } else {
+    pa = 0;                         // the alias is not ours to unmap
+  }
+  release(&kmaplock);
+  invlpg((void *)va);
+  return pa ? PGROUNDDOWN(pa) : 0;
+}
+
 // Map a device's registers so the kernel can reach them.
 //
 // Every driver used to say P2V(bar.base), which quietly assumes the BAR lands
@@ -390,7 +544,7 @@ ioremap(uint64 pa, uint64 size)
 
 // --- write-combining, and big buffers -----------------------------------------
 //
-// See include/arch/vmap.h. The console used to draw straight into the framebuffer
+// See include/vmap.h. The console used to draw straight into the framebuffer
 // through the direct map, which is uncached across the whole PCI hole: every
 // store was its own bus transaction and every read -- a scroll is a memmove,
 // so a read for every byte on screen -- waited on the card. In QEMU that is