diff --git a/.gitignore b/.gitignore index 3a26800..ca58d66 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,15 @@ docs/Website/releases/ # remembering. core core.* + +# Reference material: somebody else's source tree, read and never shipped. +# +# help/ is where a third-party kernel gets cloned when it is worth reading -- +# Zircon, at the time of writing. It is BSD-licensed and this project is +# Apache-2.0, so the licences are compatible and that is *not* the reason it is +# ignored: it is ignored because a copy of another project's source inside this +# one is a copy that will rot, that nobody will update, and that makes the +# provenance of every file around it a question. Clone it, read it, learn the +# facts, write our own expression -- the rule docs/architecture/ldls.md section +# 3 already states for Linux drivers. +help/ diff --git a/boot/uefi/main.c b/boot/uefi/main.c index c7a0021..8828eb3 100644 --- a/boot/uefi/main.c +++ b/boot/uefi/main.c @@ -942,21 +942,50 @@ cpu_brand(char *out, int cap) * from ExitBootServices long after the mistake. The count is deliberately of * the types the kernel will treat as usable, so it matches what the system * later reports rather than what the firmware owns. */ +// The firmware's memory map, read into the one buffer there is. +// +// There used to be two of these, each with its own guess at how big a memory +// map gets -- 32 KB where it mattered and 16 KB here -- and both guesses were +// wrong on real hardware. The one here was the worse of the two, because it +// did not fail loudly: GetMemoryMap answered EFI_BUFFER_TOO_SMALL, this +// returned zero, and the System Information screen reported **0 MB usable** on +// a machine with sixteen gigabytes in it. +// +// So: one buffer, one function, and the size comes from the firmware rather +// than from an estimate. A call with a zero size fails on purpose and fills in +// what is needed; it allocates nothing, so a caller that needs the map key for +// ExitBootServices still has a valid one. +#define MAPBUF_BYTES 262144 +static UINT8 mapbuf[MAPBUF_BYTES]; + +static EFI_STATUS +map_read(UINTN *size, UINTN *key, UINTN *dsize, UINT32 *dver) +{ + EFI_STATUS st; + + *size = 0; + st = BS->GetMemoryMap(size, (EFI_MEMORY_DESCRIPTOR *)mapbuf, + key, dsize, dver); + if (st == EFI_BUFFER_TOO_SMALL && *size > sizeof mapbuf) + return st; /* the caller decides how loudly to complain */ + + *size = sizeof mapbuf; + return BS->GetMemoryMap(size, (EFI_MEMORY_DESCRIPTOR *)mapbuf, + key, dsize, dver); +} + static UINT64 memory_mb(void) { UINTN size = 0, mapkey = 0, dsize = 0; UINT32 dver = 0; - static char buf[16384]; + UINT64 pages = 0; - size = sizeof buf; - if (EFI_ERROR(BS->GetMemoryMap(&size, (EFI_MEMORY_DESCRIPTOR *)buf, - &mapkey, &dsize, &dver))) + if (EFI_ERROR(map_read(&size, &mapkey, &dsize, &dver))) return 0; - UINT64 pages = 0; for (UINTN off = 0; off + dsize <= size; off += dsize) { - EFI_MEMORY_DESCRIPTOR *d = (EFI_MEMORY_DESCRIPTOR *)(buf + off); + EFI_MEMORY_DESCRIPTOR *d = (EFI_MEMORY_DESCRIPTOR *)(mapbuf + off); if (efi_type_to_e820(d->Type) == E820_USABLE) pages += d->NumberOfPages; } @@ -1760,13 +1789,11 @@ EfiMain(EFI_HANDLE image, EFI_SYSTEM_TABLE *systab) // actually needed. It allocates nothing, so the key from the real call // below is still valid -- which is the whole constraint this code is // written around. - static UINT8 mapbuf[262144]; UINTN mapsize = 0, mapkey = 0, descsize = 0; UINT32 descver = 0; EFI_STATUS st; - st = BS->GetMemoryMap(&mapsize, (EFI_MEMORY_DESCRIPTOR *)mapbuf, - &mapkey, &descsize, &descver); + st = map_read(&mapsize, &mapkey, &descsize, &descver); if (st == EFI_BUFFER_TOO_SMALL && mapsize > sizeof mapbuf) { // Say the number. A loader that dies on "too small" without saying how // small is a loader that costs somebody a rebuild to find out. @@ -1778,11 +1805,6 @@ EfiMain(EFI_HANDLE image, EFI_SYSTEM_TABLE *systab) die(L"memory map larger than the loader's buffer", st); } - // Two descriptors of slack, as the specification advises: the firmware is - // allowed to need a little more between the question and the answer. - mapsize = sizeof mapbuf; - st = BS->GetMemoryMap(&mapsize, (EFI_MEMORY_DESCRIPTOR *)mapbuf, - &mapkey, &descsize, &descver); if (EFI_ERROR(st)) die(L"GetMemoryMap failed", st); // Translate before leaving, while the map is still readable and while @@ -1816,9 +1838,7 @@ EfiMain(EFI_HANDLE image, EFI_SYSTEM_TABLE *systab) // The map changed underneath us between GetMemoryMap and here. Take it // again and retry once; this is normal and the specification expects // a loader to handle it. - mapsize = sizeof mapbuf; - st = BS->GetMemoryMap(&mapsize, (EFI_MEMORY_DESCRIPTOR *)mapbuf, - &mapkey, &descsize, &descver); + st = map_read(&mapsize, &mapkey, &descsize, &descver); if (EFI_ERROR(st)) die(L"GetMemoryMap failed on retry", st); // Translate again. The retry exists because the map *changed*, so // handing the kernel the table built from the previous one describes @@ -1855,7 +1875,35 @@ EfiMain(EFI_HANDLE image, EFI_SYSTEM_TABLE *systab) zero(pdpt, 0x1000); pml4[0] = PDPT_ADDR | PTE_P | PTE_W; // identity pml4[256] = PDPT_ADDR | PTE_P | PTE_W; // 0xffff800000000000 - for (int i = 0; i < 4; i++) // 4 GB in 1 GB pages + + // **All 512 entries, not four.** + // + // Four gigabytes was enough for as long as everything that mattered lived + // below four gigabytes -- and on a machine with a lot of RAM, one thing + // does not: *this loader*. The firmware places the image where it likes, + // and on an 8 GB machine it lands above 4 GB. The instruction after + // + // mov %rax, %cr3 + // + // is then an instruction fetch from an address these tables do not map, + // and the machine dies between one instruction and the next, with the + // splash still on screen and nothing said: + // + // X64 Exception Type - 0E(#PF - Page-Fault) + // ExceptionData - 0x10 I:1 P:0 + // RIP - 0000000140004DEE <- the loader, at 5.4 GB + // RAX - 0000000000001000 <- the page tables it just installed + // RCX - 00000000001000F0 <- the kernel it was about to jump to + // + // That is the whole of "it freezes at Starting the kernel". It reproduces + // on QEMU with `-m 8G` and never on `-m 1G`, which is why every test here + // passed. + // + // The fix costs nothing: the PDPT is one page whether four entries are + // filled or all of them, and 512 GB of identity map covers any machine + // whose firmware is going to load us somewhere surprising. The kernel + // replaces all of this in kvmalloc() a few hundred instructions later. + for (int i = 0; i < 512; i++) pdpt[i] = ((UINT64)i << 30) | PTE_P | PTE_PS | PTE_W; ui_progress_label("Starting the kernel"); diff --git a/docs/architecture/boot.md b/docs/architecture/boot.md index 829fe3a..b76b26a 100644 --- a/docs/architecture/boot.md +++ b/docs/architecture/boot.md @@ -122,6 +122,34 @@ when the early console is set up. So a low framebuffer is attached exactly as before, and a high one is attached a few dozen instructions later, immediately after `kvmalloc()` — still thousands of lines before anything else happens. +### The loader jumped out from under itself + +A machine with a lot of RAM froze at "Starting the kernel" — the splash still +up, no panic, no reset, nothing. It reproduces exactly on QEMU with `-m 8G` and +never on `-m 1G`, which is why every test here passed for months. + +The firmware's own exception handler had the answer all along, once a machine +was found that printed it: + +``` +X64 Exception Type - 0E(#PF - Page-Fault) +ExceptionData - 0x10 I:1 P:0 +RIP - 0000000140004DEE <- the loader's own code, at 5.4 GB +RAX - 0000000000001000 <- the page tables it had just installed +RCX - 00000000001000F0 <- the kernel it was about to jump to +``` + +`I:1 P:0` is an instruction fetch from a page that is not present. The loader +built page tables covering the low four gigabytes, loaded `cr3` with them — and +**the firmware had loaded the loader image above four gigabytes**, so the very +next instruction fetch was unmapped. It jumped out from under itself, between +one instruction and the next. + +The fix costs nothing. The PDPT is one page whether four of its entries are +filled or all 512, so all 512 are: a 512 GB identity map, which covers any +machine whose firmware puts us somewhere surprising. The kernel replaces the +whole thing in `kvmalloc()` a few hundred instructions later. + ### The memory map does not fit in a guess `GetMemoryMap` answered **`EFI_BUFFER_TOO_SMALL` (0x5)** on real hardware, and diff --git a/include/defs.h b/include/defs.h index e6826f2..12663df 100644 --- a/include/defs.h +++ b/include/defs.h @@ -420,6 +420,7 @@ char* kalloc(void); uint64 kalloc_free_bytes(void); void kfree(char*); void kinit1(void*, void*); +void kinit_high(void); void kinit2(void); // kbd.c diff --git a/include/memlayout.h b/include/memlayout.h index 0a69064..41d492e 100644 --- a/include/memlayout.h +++ b/include/memlayout.h @@ -37,16 +37,39 @@ struct memregion { extern struct memregion mem_usable[MEMREGION_MAX]; extern int nmem_usable; + +// The top of the low region: where the DMA arena is carved from, and the +// ceiling for kinit1(). Always below DIRECTMAP_BOOT, and separate from +// `phystop` because a device is not obliged to be able to reach the top of +// memory -- an RTL8139's buffer address register is thirty-two bits wide. +extern uint64 lowtop; #endif // What to assume when the firmware gives us nothing usable: the old constant, // so that path behaves exactly as the kernel always did. #define PHYSTOP_FALLBACK 0xE000000ULL -// kvmalloc() maps physical 0-4 GB as four 1 GB pages and marks the top one -// uncached, because that is where PCI BARs and the framebuffer sit. 3 GB is -// therefore the most that can be treated as RAM. -#define PHYSTOP_MAX 0xC0000000ULL +// The most RAM the kernel will use. +// +// This was 0xC0000000 -- three gigabytes -- because kvmalloc() mapped exactly +// four 1 GB pages and the fourth was marked uncached for the PCI hole. On a +// laptop with sixteen gigabytes that threw away thirteen of them, silently. +// +// kvmalloc() now maps as many 1 GB pages as the machine has RAM, so the limit +// is the page table rather than a constant: the kernel PDPT has 512 entries +// and ioremap takes what the direct map does not, so 64 GB of RAM still leaves +// 448 GB of device window. A machine with more than this gets the rest +// ignored, with a line saying so. +#define PHYSTOP_MAX (64ULL << 30) + +// What the *loader's* page tables cover, and therefore the ceiling on anything +// that runs before kvmalloc() builds the kernel's own. +// +// kinit1() frees pages by writing a free-list pointer into each one, so every +// page it touches has to be mapped at the time. Before kvmalloc that means the +// low four gigabytes and nothing else, which is why freeing memory happens in +// two passes -- see kinit_high(). +#define DIRECTMAP_BOOT (4ULL << 30) // Below this the kernel image, its page tables and the DMA arena do not fit. #define PHYSTOP_MIN (24ULL * 1024 * 1024) diff --git a/kernel/arch/x86_64/e820.c b/kernel/arch/x86_64/e820.c index 9c55781..c22ea10 100644 --- a/kernel/arch/x86_64/e820.c +++ b/kernel/arch/x86_64/e820.c @@ -23,6 +23,11 @@ // than in some new way. uint64 phystop = PHYSTOP_FALLBACK; +// The top of the low region. See memlayout.h: the DMA arena and everything +// kinit1() frees live below this, because that is all the loader's page tables +// map and all a 32-bit device can reach. +uint64 lowtop = PHYSTOP_FALLBACK; + #include "schboot.h" /* the hand-off block */ #define E820_USABLE 1 @@ -44,6 +49,8 @@ assume_fallback(void) mem_usable[0].base = EXTMEM; mem_usable[0].end = PHYSTOP_FALLBACK; nmem_usable = 1; + phystop = PHYSTOP_FALLBACK; + lowtop = PHYSTOP_FALLBACK; } struct e820entry { @@ -81,7 +88,7 @@ e820_init(void) // 8 MB, and every byte above it was lost. kalloc now frees region by region, // so the holes are skipped instead of being either freed or fatal. uint64 largest = 0; - uint64 top = 0; + uint64 phys_top = 0, low_top = 0; nmem_usable = 0; for (uint32 i = 0; i < count; i++) { if (e[i].type != E820_USABLE) @@ -91,7 +98,8 @@ e820_init(void) uint64 base = (e[i].base + PGSIZE - 1) & ~(uint64)(PGSIZE - 1); uint64 end = (e[i].base + e[i].length) & ~(uint64)(PGSIZE - 1); - // Above PHYSTOP_MAX the direct map is device memory, not RAM (see below). + // More than the direct map will ever cover. Clipped rather than refused: + // a machine with more RAM than this should use what it can. if (end > PHYSTOP_MAX) end = PHYSTOP_MAX; if (base >= end) @@ -101,16 +109,23 @@ e820_init(void) mem_usable[nmem_usable].end = end; nmem_usable++; - // phystop comes from the largest region, not the highest. It has to be a - // region that can hold the DMA arena, which is carved off the top of - // PHYSTOP and must be physically contiguous; the largest is the only one - // that is certain to have room. A small usable region above it is left - // unfreed rather than raising the bound past a hole -- kfree() rejects - // anything at or above PHYSTOP, and that check is worth more than the - // stray page it costs. - if (end - base > largest) { - largest = end - base; - top = end; + // Two different questions, which used to be one and should not have been. + // + // `phystop` is how far RAM goes, and it is simply the highest end: kfree() + // uses it as a bound and kvmalloc() maps up to it. + if (end > phys_top) + phys_top = end; + + // `lowtop` is where the DMA arena goes, and it has to be the largest + // region *below the boot direct map* -- contiguous, reachable by a 32-bit + // device, and mapped before kvmalloc has run. The largest such region is + // the only one certain to have room. + { + uint64 lo_end = end > DIRECTMAP_BOOT ? DIRECTMAP_BOOT : end; + if (base < lo_end && lo_end - base > largest) { + largest = lo_end - base; + low_top = lo_end; + } } } @@ -121,18 +136,12 @@ e820_init(void) return; } - // The PHYSTOP_MAX ceiling is applied per region above, when each one is - // clipped. kvmalloc() maps physical 0-4 GB with four 1 GB pages and marks the - // top one uncached because that is where PCI BARs and the framebuffer live, - // so 3 GB is the ceiling for anything the kernel will treat as RAM: above it - // the direct map either does not exist or is device memory that would be - // catastrophically slow to allocate from and is not ours to hand out. - // Below this the kernel image, its page tables and the DMA arena do not fit, // and continuing produces a failure much harder to read than this line. - if (top < PHYSTOP_MIN) { - cprintf("e820: only %d MB usable; SchOS needs at least %d MB\n", - (int)(top / (1024 * 1024)), (int)(PHYSTOP_MIN / (1024 * 1024))); + if (low_top < PHYSTOP_MIN) { + cprintf("e820: only %d MB usable below 4 GB; SchOS needs at least %d MB " + "there\n", + (int)(low_top / (1024 * 1024)), (int)(PHYSTOP_MIN / (1024 * 1024))); // Left at the fallback deliberately rather than lowered: a machine this // small is going to fail, and it should fail having said why. assume_fallback(); @@ -154,7 +163,8 @@ e820_init(void) return; } - phystop = top; + phystop = phys_top; + lowtop = low_top; // Report what is actually usable, not phystop: with a fragmented map those // are different numbers, and printing the top address as though it were the diff --git a/kernel/arch/x86_64/vm.c b/kernel/arch/x86_64/vm.c index d1b2448..014df5b 100644 --- a/kernel/arch/x86_64/vm.c +++ b/kernel/arch/x86_64/vm.c @@ -32,6 +32,16 @@ _Static_assert(PERCPU_SELF == 0 && PERCPU_SCRATCH == 8 && PERCPU_CPU == 16 && "per-CPU block layout changed -- update kernel/arch/x86_64/trapasm.S"); static pml4e_t *kpml4; +// Where the device windows begin is now decided by kvmalloc(), which knows how +// much of the address space the direct map took. The constant that used to be +// here said 4, and would have handed out windows on top of RAM the moment the +// direct map grew past four gigabytes. +#define IOREMAP_LAST_PDPT 511 +#define GIGABYTE (1ULL << 30) + +// The first PDPT entry the direct map did not take. Set by kvmalloc(). +static int ioremap_next = 4; + static pdpe_t *kpdpt; struct tassdesc { @@ -231,14 +241,34 @@ kvmalloc(void) kpml4[0] = v2p(kpdpt) | PTE_P | PTE_W; kpml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W; - for(uint64 i = 0; i < 4; i++) { + // As many gigabytes as the machine has RAM, not four. + // + // Four was the number for as long as PHYSTOP was capped at 3 GB, and the cap + // existed because this loop stopped at four -- each holding the other up. A + // laptop with sixteen gigabytes lost thirteen of them to that circle. + // + // The 3-4 GB entry stays uncached because that is where the PCI hole is and + // always has been. Everything above 4 GB is RAM -- the hole is below it on + // every machine this runs on -- so those are cached, and a device BAR up + // there still gets its own uncached window from ioremap(). + uint64 gigs = (phystop + GIGABYTE - 1) / GIGABYTE; + if (gigs < 4) + gigs = 4; // always map the low four + if (gigs > IOREMAP_LAST_PDPT - 8) + gigs = IOREMAP_LAST_PDPT - 8; // leave room for device windows + + for(uint64 i = 0; i < gigs; i++) { uint64 flags = PTE_PS | PTE_P | PTE_W; - if(i >= 3) { + if(i == 3) { flags |= PTE_PCD | PTE_PWT; } kpdpt[i] = (i << 30) | flags; } + // ioremap takes what is left. It used to start at a constant 4, which was + // correct only while the direct map was exactly four gigabytes. + ioremap_next = (int)gigs; + switchkvm(); } @@ -261,11 +291,6 @@ kvmalloc(void) // Cache-disabled and write-through, for the same reason the top GB of the // direct map is: these are registers, and a write that sits in a cache line // waiting for company is a write the device never sees. -#define IOREMAP_FIRST_PDPT 4 -#define IOREMAP_LAST_PDPT 511 -#define GIGABYTE (1ULL << 30) - -static int ioremap_next = IOREMAP_FIRST_PDPT; void * ioremap(uint64 pa, uint64 size) @@ -285,7 +310,7 @@ ioremap(uint64 pa, uint64 size) // Already mapped? A second BAR in the same gigabyte should not consume a // second window, and drivers do re-map the same device. - for (int i = IOREMAP_FIRST_PDPT; i < ioremap_next; i++) { + for (int i = 4; i < ioremap_next; i++) { // Clearing the low 30 bits drops every flag as well, leaving the frame. if ((kpdpt[i] & PTE_P) && (kpdpt[i] & ~(GIGABYTE - 1)) == base) return (void *)(KERNBASE + ((uint64)i << 30) + (pa - base)); diff --git a/kernel/core/kalloc.c b/kernel/core/kalloc.c index d4807c3..f9557e1 100644 --- a/kernel/core/kalloc.c +++ b/kernel/core/kalloc.c @@ -163,6 +163,41 @@ mem_whose(struct sbuf *out, struct whose *w) } } +// The memory the loader's page tables could not reach. +// +// kinit1() runs before kvmalloc(), on page tables that map four gigabytes, and +// freeing a page means writing a free-list pointer *into* it -- so anything +// higher would fault. kvmalloc() has now mapped as much of physical memory as +// the machine has, so the rest can be handed over. +// +// Two passes rather than one because of that ordering, and the ordering is +// forced: kvmalloc() calls kalloc() for its own page tables, so the allocator +// has to be working before the map it needs exists. +void +kinit_high(void) +{ + uint64 added = 0; + + for (int i = 0; i < nmem_usable; i++) { + uint64 lo = mem_usable[i].base; + uint64 hi = mem_usable[i].end; + + if (lo < DIRECTMAP_BOOT) + lo = DIRECTMAP_BOOT; + if (hi > phystop) + hi = phystop; + if (lo >= hi) + continue; + + freerange(P2V(lo), P2V(hi)); + added += hi - lo; + } + + if (added) + kinfo("mem: %d MB above 4 GB added to the allocator\n", + (int)(added / (1024 * 1024))); +} + void kinit2() { diff --git a/kernel/core/main.c b/kernel/core/main.c index ddf766e..41035fe 100644 --- a/kernel/core/main.c +++ b/kernel/core/main.c @@ -188,13 +188,22 @@ kmain(uint32 fb_addr) // is a parameter, so moving it up by KLOG_BYTES is the whole of it. klog_base = (void *)PGROUNDUP((addr_t)end); klog_init(klog_base, KLOG_BYTES); - kinit1((char *)klog_base + KLOG_BYTES, P2V(PHYSTOP - DMA_ARENA_BYTES)); + // `lowtop`, not PHYSTOP: this runs on the loader's page tables, which map + // four gigabytes, and freeing a page writes into it. Everything above that + // is added by kinit_high() once kvmalloc has mapped it. + kinit1((char *)klog_base + KLOG_BYTES, P2V(lowtop - DMA_ARENA_BYTES)); if (boot_verbose) cprintf("verbose boot: kinit1 done\n"); - dma_init(P2V(PHYSTOP - DMA_ARENA_BYTES), P2V(PHYSTOP)); + // The arena stays low for the same reason it is contiguous: a device is not + // obliged to be able to reach the top of memory. An RTL8139's buffer address + // register is thirty-two bits wide, and it is not the only one. + dma_init(P2V(lowtop - DMA_ARENA_BYTES), P2V(lowtop)); if (boot_verbose) cprintf("verbose boot: dma_init done\n"); kvmalloc(); if (boot_verbose) cprintf("verbose boot: kvmalloc done\n"); + // Now that the direct map reaches it, the rest of RAM can be allocated from. + kinit_high(); + // The other kind of framebuffer: one the direct map does not reach. // // Any machine with a discrete GPU is entitled to put its aperture above