ldls: the top-2 GB window Linux modules link against
A Linux module is built with -mcmodel=kernel and can only reach addresses in the top 2 GB, where this kernel is not linked. So the kernel image is mapped a second time at LDLS_KALIAS, with a vmemmap area and a module area beside it, in one PDPT shared by every address space. Nothing loads a module yet; this is the address-space groundwork, set up at every boot, and include/ldls/ is the interface the loader will use. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4
This commit is contained in:
Nexoniarz
parent
b524a73522
commit
65617710fa
4 changed files with 253 additions and 4 deletions
|
|
@ -7,7 +7,9 @@
|
|||
#include "arch/mmu.h"
|
||||
#include "core/proc.h"
|
||||
#include "arch/ioperm.h"
|
||||
#include "core/spinlock.h"
|
||||
#include "arch/vmap.h"
|
||||
#include "ldls/kldls.h"
|
||||
|
||||
int mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm);
|
||||
static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc);
|
||||
|
|
@ -15,7 +17,7 @@ static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc);
|
|||
|
||||
// `cpu` and `proc` are no longer __thread variables; they live at fixed offsets
|
||||
// in the per-CPU block and are read through %gs at each use. See the long
|
||||
// comment in include/core/proc.h for why -- the short version is that __thread means
|
||||
// comment in include/proc.h for why -- the short version is that __thread means
|
||||
// per *thread*, the compiler caches the thread pointer across calls on that
|
||||
// basis, and a kernel thread changes CPU at every sched().
|
||||
|
||||
|
|
@ -156,7 +158,7 @@ seginit(void)
|
|||
gdt = (struct segdesc*) local;
|
||||
|
||||
// The TSS moved out of this page. It now carries an I/O permission bitmap
|
||||
// (include/arch/ioperm.h) which does not fit here, and it has to be one object
|
||||
// (include/ioperm.h) which does not fit here, and it has to be one object
|
||||
// with it. ioperm_tss() returns it and the descriptor limit that reaches the
|
||||
// end of the bitmap.
|
||||
tss = (uint*) ioperm_tss(cpunum(), &tsslim);
|
||||
|
|
@ -179,7 +181,7 @@ seginit(void)
|
|||
*(uint64 *) kernel_fs_base = kernel_fs_base;
|
||||
|
||||
// Clear the per-CPU slots before anything can read them. Their offsets are
|
||||
// the PERCPU_* constants in include/core/proc.h and are shared with trapasm.S,
|
||||
// the PERCPU_* constants in include/proc.h and are shared with trapasm.S,
|
||||
// which reads %gs:24 for proc on the way out to user mode and parks three
|
||||
// registers at %gs:32..48 across the FS_BASE swap on the way in.
|
||||
((uint64 *) kernel_fs_base)[2] = 0; // struct cpu * (PERCPU_CPU)
|
||||
|
|
@ -256,6 +258,10 @@ setupkvm(void)
|
|||
memset(pml4, 0, PGSIZE);
|
||||
|
||||
pml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W;
|
||||
// The LDLS window: the kernel's second mapping and the modules above it.
|
||||
// A module's interrupt and its kernel thread can run in any address space,
|
||||
// so it has to be in all of them. See ldls_vm_init().
|
||||
pml4[PMX(LDLS_KALIAS)] = kpml4[PMX(LDLS_KALIAS)];
|
||||
|
||||
// The framebuffer is deliberately NOT mapped here.
|
||||
//
|
||||
|
|
@ -318,9 +324,157 @@ kvmalloc(void)
|
|||
// correct only while the direct map was exactly four gigabytes.
|
||||
ioremap_next = (int)gigs;
|
||||
|
||||
ldls_vm_init();
|
||||
|
||||
switchkvm();
|
||||
}
|
||||
|
||||
// --- the top two gigabytes, for LDLS -----------------------------------------
|
||||
//
|
||||
// A Linux module is compiled with -mcmodel=kernel, which is a promise that the
|
||||
// module and everything it calls live in the top 2 GB of the address space:
|
||||
// R_X86_64_32S relocations sign-extend a 32-bit field, so the only addresses
|
||||
// they can name are 0xffffffff80000000 and above. This kernel is linked at
|
||||
// KERNBASE, nowhere near there, so a module cannot be relocated against it.
|
||||
//
|
||||
// The fix is the layout Linux itself uses. The kernel image is mapped a second
|
||||
// time at LDLS_KALIAS -- same physical pages, so the same code and the same
|
||||
// variables -- and modules load at LDLS_MODBASE, a few hundred megabytes above
|
||||
// it. Every symbol a module imports then resolves to its alias address, and a
|
||||
// call or a data reference reaches it with a 32-bit displacement.
|
||||
//
|
||||
// Running kernel code at the alias is sound because the whole image moves
|
||||
// together: an absolute reference still names the KERNBASE copy, and a
|
||||
// rip-relative one names the alias of the same byte. Either way it is the same
|
||||
// memory.
|
||||
//
|
||||
// One PDPT under PML4 slot 511, shared by every address space exactly as
|
||||
// kpdpt is, so a mapping made here is visible everywhere at once. Nothing in
|
||||
// it is user-accessible: the leaves carry no PTE_U, and the U/S bit is the AND
|
||||
// of every level.
|
||||
static pdpe_t *hipdpt;
|
||||
static struct spinlock kmaplock;
|
||||
|
||||
void
|
||||
ldls_vm_init(void)
|
||||
{
|
||||
extern char end[];
|
||||
pde_t *alias, *vmm;
|
||||
uint64 top, i;
|
||||
|
||||
initlock(&kmaplock, "kmap");
|
||||
|
||||
hipdpt = (pdpe_t *)kalloc();
|
||||
alias = (pde_t *)kalloc();
|
||||
vmm = (pde_t *)kalloc();
|
||||
if (!hipdpt || !alias || !vmm)
|
||||
panic("ldls_vm_init");
|
||||
memset(hipdpt, 0, PGSIZE);
|
||||
memset(alias, 0, PGSIZE);
|
||||
memset(vmm, 0, PGSIZE);
|
||||
|
||||
// The alias, in 2 MB pages from physical zero to past the end of the image.
|
||||
// It covers some memory that is not the kernel, which is harmless: it is a
|
||||
// second name for pages the direct map already names.
|
||||
top = ((uint64)V2P(end) + (2ULL << 20) - 1) >> 21;
|
||||
if (top > 256)
|
||||
panic("ldls_vm_init: kernel image larger than 512 MB");
|
||||
for (i = 0; i < top; i++)
|
||||
alias[i] = (i << 21) | PTE_PS | PTE_P | PTE_W;
|
||||
|
||||
hipdpt[PDPX(LDLS_KALIAS)] = v2p(alias) | PTE_P | PTE_W;
|
||||
hipdpt[PDPX(LDLS_VMEMMAP)] = v2p(vmm) | PTE_P | PTE_W;
|
||||
kpml4[PMX(LDLS_KALIAS)] = v2p(hipdpt) | PTE_P | PTE_W;
|
||||
}
|
||||
|
||||
// Map one page in the LDLS window. Returns 0, or -1 if the slot is taken or a
|
||||
// page table could not be had.
|
||||
int
|
||||
kmap_page(uint64 va, addr_t pa)
|
||||
{
|
||||
pde_t *pd, *pde;
|
||||
pte_t *pt, *pte;
|
||||
|
||||
if (PMX(va) != PMX(LDLS_KALIAS) || !(hipdpt[PDPX(va)] & PTE_P))
|
||||
return -1;
|
||||
|
||||
acquire(&kmaplock);
|
||||
pd = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)]));
|
||||
pde = &pd[PDX(va)];
|
||||
if (*pde & PTE_PS) {
|
||||
release(&kmaplock);
|
||||
return -1; // the alias; never mapped over
|
||||
}
|
||||
if (!(*pde & PTE_P)) {
|
||||
if ((pt = (pte_t *)kalloc()) == 0) {
|
||||
release(&kmaplock);
|
||||
return -1;
|
||||
}
|
||||
memset(pt, 0, PGSIZE);
|
||||
*pde = v2p(pt) | PTE_P | PTE_W;
|
||||
}
|
||||
pt = (pte_t *)P2V(PTE_ADDR(*pde));
|
||||
pte = &pt[PTX(va)];
|
||||
if (*pte & PTE_P) {
|
||||
release(&kmaplock);
|
||||
return -1;
|
||||
}
|
||||
*pte = pa | PTE_P | PTE_W;
|
||||
release(&kmaplock);
|
||||
invlpg((void *)va);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// What is mapped at `va` in the window, or 0.
|
||||
addr_t
|
||||
kmap_phys(uint64 va)
|
||||
{
|
||||
pde_t *pd, *pde;
|
||||
pte_t *pt;
|
||||
|
||||
if (PMX(va) != PMX(LDLS_KALIAS) || !(hipdpt[PDPX(va)] & PTE_P))
|
||||
return 0;
|
||||
pd = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)]));
|
||||
pde = &pd[PDX(va)];
|
||||
if (!(*pde & PTE_P))
|
||||
return 0;
|
||||
if (*pde & PTE_PS)
|
||||
return PTE_ADDR(*pde) + (va & ((2ULL << 20) - 1));
|
||||
pt = (pte_t *)P2V(PTE_ADDR(*pde));
|
||||
if (!(pt[PTX(va)] & PTE_P))
|
||||
return 0;
|
||||
return PTE_ADDR(pt[PTX(va)]) + (va & (PGSIZE - 1));
|
||||
}
|
||||
|
||||
// Take a page out of the window and say which frame it was, or 0.
|
||||
//
|
||||
// Only this CPU's TLB is flushed. There is no shootdown IPI in this kernel, so
|
||||
// another CPU may keep a stale translation for a while -- which is why the
|
||||
// LDLS window never hands the same virtual address out twice. A stale entry
|
||||
// for an address nobody will use again is harmless.
|
||||
addr_t
|
||||
kunmap_page(uint64 va)
|
||||
{
|
||||
addr_t pa = kmap_phys(va);
|
||||
pde_t *pd, *pde;
|
||||
pte_t *pt;
|
||||
|
||||
if (!pa)
|
||||
return 0;
|
||||
acquire(&kmaplock);
|
||||
pd = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)]));
|
||||
pde = &pd[PDX(va)];
|
||||
if (!(*pde & PTE_PS)) {
|
||||
pt = (pte_t *)P2V(PTE_ADDR(*pde));
|
||||
pt[PTX(va)] = 0;
|
||||
} else {
|
||||
pa = 0; // the alias is not ours to unmap
|
||||
}
|
||||
release(&kmaplock);
|
||||
invlpg((void *)va);
|
||||
return pa ? PGROUNDDOWN(pa) : 0;
|
||||
}
|
||||
|
||||
// Map a device's registers so the kernel can reach them.
|
||||
//
|
||||
// Every driver used to say P2V(bar.base), which quietly assumes the BAR lands
|
||||
|
|
@ -390,7 +544,7 @@ ioremap(uint64 pa, uint64 size)
|
|||
|
||||
// --- write-combining, and big buffers -----------------------------------------
|
||||
//
|
||||
// See include/arch/vmap.h. The console used to draw straight into the framebuffer
|
||||
// See include/vmap.h. The console used to draw straight into the framebuffer
|
||||
// through the direct map, which is uncached across the whole PCI hole: every
|
||||
// store was its own bus transaction and every read -- a scroll is a memmove,
|
||||
// so a read for every byte on screen -- waited on the card. In QEMU that is
|
||||
|
|
|
|||
Loading…
Reference in a new issue