From 65617710fa08f4abb8a2e9584d2c3987b6ed8570 Mon Sep 17 00:00:00 2001 From: Nexoniarz Date: Sat, 24 Oct 2026 22:12:40 +0200 Subject: [PATCH] ldls: the top-2 GB window Linux modules link against A Linux module is built with -mcmodel=kernel and can only reach addresses in the top 2 GB, where this kernel is not linked. So the kernel image is mapped a second time at LDLS_KALIAS, with a vmemmap area and a module area beside it, in one PDPT shared by every address space. Nothing loads a module yet; this is the address-space groundwork, set up at every boot, and include/ldls/ is the interface the loader will use. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4 --- include/arch/memlayout.h | 16 ++++ include/ldls/kldls.h | 26 +++++++ include/ldls/ldls.h | 53 +++++++++++++ kernel/arch/x86_64/vm.c | 162 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 253 insertions(+), 4 deletions(-) create mode 100644 include/ldls/kldls.h create mode 100644 include/ldls/ldls.h diff --git a/include/arch/memlayout.h b/include/arch/memlayout.h index 22ad979..37d5f03 100644 --- a/include/arch/memlayout.h +++ b/include/arch/memlayout.h @@ -91,6 +91,22 @@ extern uint64 lowtop; #define KERNPHYS 0x1000000 #define KERNLINK (KERNBASE+KERNPHYS) // Address where kernel is linked +// The LDLS window, in the top 2 GB, where a Linux module expects to live. See +// ldls_vm_init() in kernel/arch/x86_64/vm.c. +// +// LDLS_VMEMMAP 1 GB Linux's `struct page` array, 64 bytes per frame -- +// enough for 64 GB, not for all of PHYSTOP_MAX; a +// frame above that has no struct page. Backed on demand. +// LDLS_KALIAS 512 MB the kernel image again, from physical zero +// LDLS_MODBASE 512 MB modules, their stacks and their vmalloc +#define LDLS_VMEMMAP 0xFFFFFFFF40000000ULL +#define LDLS_KALIAS 0xFFFFFFFF80000000ULL +#define LDLS_MODBASE 0xFFFFFFFFA0000000ULL +#define LDLS_MODEND 0xFFFFFFFFC0000000ULL + +// A kernel address, as a module sees it. +#define LDLS_ALIAS(a) ((uint64)(a) - KERNBASE + LDLS_KALIAS) + #ifndef __ASSEMBLER__ static inline addr_t v2p(void *a) { return ((addr_t)(a)) - ((addr_t)KERNBASE); diff --git a/include/ldls/kldls.h b/include/ldls/kldls.h new file mode 100644 index 0000000..5bdf4e4 --- /dev/null +++ b/include/ldls/kldls.h @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#pragma once + +// The kernel's side of LDLS, as the rest of the kernel sees it. The layer +// itself is kernel/ldls/; this is the handful of places it is called from. + +struct trapframe; + +// kernel/arch/x86_64/vm.c -- the window in the top 2 GB. +void ldls_vm_init(void); +int kmap_page(uint64 va, addr_t pa); +addr_t kmap_phys(uint64 va); +addr_t kunmap_page(uint64 va); + +// kernel/ldls/core.c +void ldls_init(void); +int ldls_ioctl(uint64 request, uint64 argp); + +// A trap in kernel mode. Returns 1 if LDLS dealt with it -- a WARN() that was +// reported and stepped over, or a hosted driver that faulted and has been +// stopped -- and 0 if it is the kernel's own and should panic as before. +int ldls_trap(struct trapframe *tf); + +// The name of an address in a hosted module, for a backtrace. 0 if `a` is not +// in one. +int ldls_symbolize(uint64 a, char *out, int cap); diff --git a/include/ldls/ldls.h b/include/ldls/ldls.h new file mode 100644 index 0000000..58802ec --- /dev/null +++ b/include/ldls/ldls.h @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#pragma once + +// /dev/ldls -- Linux drivers, in the kernel. docs/architecture/ldls.md. +// +// The device is a control channel and nothing else: a module is handed over +// whole, as bytes, together with the BTF of the kernel it was built for, and +// the kernel does the rest -- loading, linking, init_module, probe, and +// bringing the interface up -- on a thread of its own. What happened comes +// back as text, because the useful answer to "did it work" is the driver's +// own account of what it found. +// +// ldls load /lib/modules/e1000.ko +// ldls list +// ldls unload e1000 +// +// Shared with userland (cmd/ldls), so it is spelled in plain C types. + +#define LDLS_MAJOR 13 + +// Take the device even if a kernel driver holds it, provided that driver can +// be asked to let go. Without this LDLS only takes devices nothing else +// drives, which is the right default at boot: the kernel's own driver is on +// the path to a shell, and the hosted one is the better one only when someone +// says so. +#define LDLS_TAKE 0x1 +// Load and link, run init_module, and stop: no device is probed. For a +// library module (mii, libphy) whose job is to export symbols to the next. +#define LDLS_NOPROBE 0x2 + +struct ldls_load { + unsigned long long ko; // in: the module image + unsigned long long kolen; + unsigned long long btf; // in: the kernel's BTF, /sys/kernel/btf/vmlinux + unsigned long long btflen; + unsigned long long msg; // out: what happened, NUL-terminated + unsigned int msgcap; + unsigned int flags; // LDLS_* + int bus, slot, func; // -1 for "whichever matches" + unsigned int hz; // CONFIG_HZ of that kernel; 0 means 1000 + int id; // out: the module's id, for unload +}; + +struct ldls_text { + unsigned long long buf; // out + unsigned int cap; + unsigned int pad; +}; + +#define LDLSIOC_LOAD 0xD000 // struct ldls_load +#define LDLSIOC_UNLOAD 0xD001 // in: int id +#define LDLSIOC_LIST 0xD002 // struct ldls_text +#define LDLSIOC_LOG 0xD003 // struct ldls_text, in: pad = id diff --git a/kernel/arch/x86_64/vm.c b/kernel/arch/x86_64/vm.c index f8fb93c..04e61ea 100644 --- a/kernel/arch/x86_64/vm.c +++ b/kernel/arch/x86_64/vm.c @@ -7,7 +7,9 @@ #include "arch/mmu.h" #include "core/proc.h" #include "arch/ioperm.h" +#include "core/spinlock.h" #include "arch/vmap.h" +#include "ldls/kldls.h" int mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm); static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc); @@ -15,7 +17,7 @@ static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc); // `cpu` and `proc` are no longer __thread variables; they live at fixed offsets // in the per-CPU block and are read through %gs at each use. See the long -// comment in include/core/proc.h for why -- the short version is that __thread means +// comment in include/proc.h for why -- the short version is that __thread means // per *thread*, the compiler caches the thread pointer across calls on that // basis, and a kernel thread changes CPU at every sched(). @@ -156,7 +158,7 @@ seginit(void) gdt = (struct segdesc*) local; // The TSS moved out of this page. It now carries an I/O permission bitmap - // (include/arch/ioperm.h) which does not fit here, and it has to be one object + // (include/ioperm.h) which does not fit here, and it has to be one object // with it. ioperm_tss() returns it and the descriptor limit that reaches the // end of the bitmap. tss = (uint*) ioperm_tss(cpunum(), &tsslim); @@ -179,7 +181,7 @@ seginit(void) *(uint64 *) kernel_fs_base = kernel_fs_base; // Clear the per-CPU slots before anything can read them. Their offsets are - // the PERCPU_* constants in include/core/proc.h and are shared with trapasm.S, + // the PERCPU_* constants in include/proc.h and are shared with trapasm.S, // which reads %gs:24 for proc on the way out to user mode and parks three // registers at %gs:32..48 across the FS_BASE swap on the way in. ((uint64 *) kernel_fs_base)[2] = 0; // struct cpu * (PERCPU_CPU) @@ -256,6 +258,10 @@ setupkvm(void) memset(pml4, 0, PGSIZE); pml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W; + // The LDLS window: the kernel's second mapping and the modules above it. + // A module's interrupt and its kernel thread can run in any address space, + // so it has to be in all of them. See ldls_vm_init(). + pml4[PMX(LDLS_KALIAS)] = kpml4[PMX(LDLS_KALIAS)]; // The framebuffer is deliberately NOT mapped here. // @@ -318,9 +324,157 @@ kvmalloc(void) // correct only while the direct map was exactly four gigabytes. ioremap_next = (int)gigs; + ldls_vm_init(); + switchkvm(); } +// --- the top two gigabytes, for LDLS ----------------------------------------- +// +// A Linux module is compiled with -mcmodel=kernel, which is a promise that the +// module and everything it calls live in the top 2 GB of the address space: +// R_X86_64_32S relocations sign-extend a 32-bit field, so the only addresses +// they can name are 0xffffffff80000000 and above. This kernel is linked at +// KERNBASE, nowhere near there, so a module cannot be relocated against it. +// +// The fix is the layout Linux itself uses. The kernel image is mapped a second +// time at LDLS_KALIAS -- same physical pages, so the same code and the same +// variables -- and modules load at LDLS_MODBASE, a few hundred megabytes above +// it. Every symbol a module imports then resolves to its alias address, and a +// call or a data reference reaches it with a 32-bit displacement. +// +// Running kernel code at the alias is sound because the whole image moves +// together: an absolute reference still names the KERNBASE copy, and a +// rip-relative one names the alias of the same byte. Either way it is the same +// memory. +// +// One PDPT under PML4 slot 511, shared by every address space exactly as +// kpdpt is, so a mapping made here is visible everywhere at once. Nothing in +// it is user-accessible: the leaves carry no PTE_U, and the U/S bit is the AND +// of every level. +static pdpe_t *hipdpt; +static struct spinlock kmaplock; + +void +ldls_vm_init(void) +{ + extern char end[]; + pde_t *alias, *vmm; + uint64 top, i; + + initlock(&kmaplock, "kmap"); + + hipdpt = (pdpe_t *)kalloc(); + alias = (pde_t *)kalloc(); + vmm = (pde_t *)kalloc(); + if (!hipdpt || !alias || !vmm) + panic("ldls_vm_init"); + memset(hipdpt, 0, PGSIZE); + memset(alias, 0, PGSIZE); + memset(vmm, 0, PGSIZE); + + // The alias, in 2 MB pages from physical zero to past the end of the image. + // It covers some memory that is not the kernel, which is harmless: it is a + // second name for pages the direct map already names. + top = ((uint64)V2P(end) + (2ULL << 20) - 1) >> 21; + if (top > 256) + panic("ldls_vm_init: kernel image larger than 512 MB"); + for (i = 0; i < top; i++) + alias[i] = (i << 21) | PTE_PS | PTE_P | PTE_W; + + hipdpt[PDPX(LDLS_KALIAS)] = v2p(alias) | PTE_P | PTE_W; + hipdpt[PDPX(LDLS_VMEMMAP)] = v2p(vmm) | PTE_P | PTE_W; + kpml4[PMX(LDLS_KALIAS)] = v2p(hipdpt) | PTE_P | PTE_W; +} + +// Map one page in the LDLS window. Returns 0, or -1 if the slot is taken or a +// page table could not be had. +int +kmap_page(uint64 va, addr_t pa) +{ + pde_t *pd, *pde; + pte_t *pt, *pte; + + if (PMX(va) != PMX(LDLS_KALIAS) || !(hipdpt[PDPX(va)] & PTE_P)) + return -1; + + acquire(&kmaplock); + pd = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)])); + pde = &pd[PDX(va)]; + if (*pde & PTE_PS) { + release(&kmaplock); + return -1; // the alias; never mapped over + } + if (!(*pde & PTE_P)) { + if ((pt = (pte_t *)kalloc()) == 0) { + release(&kmaplock); + return -1; + } + memset(pt, 0, PGSIZE); + *pde = v2p(pt) | PTE_P | PTE_W; + } + pt = (pte_t *)P2V(PTE_ADDR(*pde)); + pte = &pt[PTX(va)]; + if (*pte & PTE_P) { + release(&kmaplock); + return -1; + } + *pte = pa | PTE_P | PTE_W; + release(&kmaplock); + invlpg((void *)va); + return 0; +} + +// What is mapped at `va` in the window, or 0. +addr_t +kmap_phys(uint64 va) +{ + pde_t *pd, *pde; + pte_t *pt; + + if (PMX(va) != PMX(LDLS_KALIAS) || !(hipdpt[PDPX(va)] & PTE_P)) + return 0; + pd = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)])); + pde = &pd[PDX(va)]; + if (!(*pde & PTE_P)) + return 0; + if (*pde & PTE_PS) + return PTE_ADDR(*pde) + (va & ((2ULL << 20) - 1)); + pt = (pte_t *)P2V(PTE_ADDR(*pde)); + if (!(pt[PTX(va)] & PTE_P)) + return 0; + return PTE_ADDR(pt[PTX(va)]) + (va & (PGSIZE - 1)); +} + +// Take a page out of the window and say which frame it was, or 0. +// +// Only this CPU's TLB is flushed. There is no shootdown IPI in this kernel, so +// another CPU may keep a stale translation for a while -- which is why the +// LDLS window never hands the same virtual address out twice. A stale entry +// for an address nobody will use again is harmless. +addr_t +kunmap_page(uint64 va) +{ + addr_t pa = kmap_phys(va); + pde_t *pd, *pde; + pte_t *pt; + + if (!pa) + return 0; + acquire(&kmaplock); + pd = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)])); + pde = &pd[PDX(va)]; + if (!(*pde & PTE_PS)) { + pt = (pte_t *)P2V(PTE_ADDR(*pde)); + pt[PTX(va)] = 0; + } else { + pa = 0; // the alias is not ours to unmap + } + release(&kmaplock); + invlpg((void *)va); + return pa ? PGROUNDDOWN(pa) : 0; +} + // Map a device's registers so the kernel can reach them. // // Every driver used to say P2V(bar.base), which quietly assumes the BAR lands @@ -390,7 +544,7 @@ ioremap(uint64 pa, uint64 size) // --- write-combining, and big buffers ----------------------------------------- // -// See include/arch/vmap.h. The console used to draw straight into the framebuffer +// See include/vmap.h. The console used to draw straight into the framebuffer // through the direct map, which is uncached across the whole PCI hole: every // store was its own bus transaction and every read -- a scroll is a memmove, // so a read for every byte on screen -- waited on the card. In QEMU that is