diff --git a/boot/uefi/main.c b/boot/uefi/main.c index 0579cef..1bf7932 100644 --- a/boot/uefi/main.c +++ b/boot/uefi/main.c @@ -640,6 +640,34 @@ extern const UINT8 schos_tramp[], schos_tramp_end[]; #define TRAMP_LIST 256 /* the copy list's offset in the page */ static EFI_PHYSICAL_ADDRESS tramp_page; static UINT64 kern_lo; /* the kernel's lowest address; set by load_elf() */ +static EFI_PHYSICAL_ADDRESS low_pds; /* see gb_pages() */ + +/* Whether the CPU can map a gigabyte with one PDPT entry: CPUID 0x80000001, + * EDX bit 26. Every AMD CPU this runs on can, and so can every Intel one + * since Haswell -- but Intel's desktop parts before that cannot, and on them + * the PS bit in a PDPT entry is reserved. The first access through such a + * table is a page fault whose handler faults the same way: "Placing and + * starting the kernel" on an Ivy Bridge i3, then a reset. + * + * Without them the low four gigabytes are mapped in 2 MB pages instead, + * through four page directories below the kernel. Below it because the + * kernel never allocates from there, and the other CPUs start through these + * same tables (entry32mp loads 0x1000) long after this loader is gone. Four + * gigabytes because that is all that is touched before kvmalloc() builds the + * kernel's own map: the trampoline page is below the kernel, and the staged + * segments are kept below 4 GB on such a CPU. */ +static int +gb_pages(void) +{ + UINT32 eax, ebx, ecx, edx; + __asm__ volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) + : "a"(0x80000000u), "c"(0)); + if (eax < 0x80000001u) + return 0; + __asm__ volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) + : "a"(0x80000001u), "c"(0)); + return (edx >> 26) & 1; +} /* Before ExitBootServices: the page, below the kernel and above the first * megabyte -- the hand-off block and the page tables are written down there @@ -653,6 +681,15 @@ tramp_reserve(void) a < 0x100000) die(L"no page below 16 MB for the final jump", 0); tramp_page = a; + + if (!gb_pages()) { + a = kern_lo - 1; + if (EFI_ERROR(BS->AllocatePages(AllocateMaxAddress, EfiLoaderData, 4, &a)) || + a < 0x100000) + die(L"no room below the kernel for its page tables", 0); + low_pds = a; + vprint(L"schos: no 1 GB pages; mapping with 2 MB pages\r\n"); + } } /* After ExitBootServices, and the last thing this loader does. */ @@ -745,7 +782,14 @@ load_elf(EFI_FILE_PROTOCOL *file) UINTN pages = (UINTN)((ph.p_memsz + 0xfff) / 0x1000); if (nlater == NLATER) die(L"too many kernel segments to stage", 0); - st = BS->AllocatePages(AllocateAnyPages, EfiLoaderData, pages, &buf); + // Below 4 GB if possible, and always without 1 GB pages, where + // that is all the final jump has mapped; see gb_pages(). + buf = 0xFFFFFFFFULL; + st = BS->AllocatePages(AllocateMaxAddress, EfiLoaderData, pages, &buf); + if (EFI_ERROR(st) && gb_pages()) { + buf = 0; + st = BS->AllocatePages(AllocateAnyPages, EfiLoaderData, pages, &buf); + } if (EFI_ERROR(st)) die(L"no memory to stage a kernel segment in", st); vprint(L"schos: staging "); vprint_hex(ph.p_paddr); @@ -2296,8 +2340,18 @@ EfiMain(EFI_HANDLE image, EFI_SYSTEM_TABLE *systab) // filled or all of them, and 512 GB of identity map covers any machine // whose firmware is going to load us somewhere surprising. The kernel // replaces all of this in kvmalloc() a few hundred instructions later. - for (int i = 0; i < 512; i++) - pdpt[i] = ((UINT64)i << 30) | PTE_P | PTE_PS | PTE_W; + // + // Unless the CPU has no 1 GB pages; see gb_pages(). + if (!low_pds) { + for (int i = 0; i < 512; i++) + pdpt[i] = ((UINT64)i << 30) | PTE_P | PTE_PS | PTE_W; + } else { + UINT64 *pd = (UINT64 *)(UINTN)low_pds; + for (int i = 0; i < 4 * 512; i++) + pd[i] = ((UINT64)i << 21) | PTE_P | PTE_PS | PTE_W; + for (int i = 0; i < 4; i++) + pdpt[i] = (low_pds + (UINT64)i * 0x1000) | PTE_P | PTE_W; + } // The kernel segments that could not be read straight home are copied // there now, by tramp_go(), from a page they cannot overwrite -- and the diff --git a/kernel/arch/x86_64/vm.c b/kernel/arch/x86_64/vm.c index 2a77b4c..f8fb93c 100644 --- a/kernel/arch/x86_64/vm.c +++ b/kernel/arch/x86_64/vm.c @@ -45,6 +45,49 @@ static int ioremap_next = 4; static pdpe_t *kpdpt; +// Whether a PDPT entry can map a gigabyte (CPUID 0x80000001, EDX bit 26). +// Intel's desktop CPUs before Haswell cannot: the bit is reserved there, and +// a 1 GB entry is a page fault on first use. The direct map and the device +// window then use a page directory of 2 MB pages per gigabyte instead -- the +// same reach, one page of table per gigabyte. See gb_pages() in the loader, +// which has the same fallback for the tables this kernel starts on. +static int gbpages; + +static int +cpu_gb_pages(void) +{ + uint32 max, d; + + cpuid_leaf(0x80000000, &max, 0, 0, 0); + if (max < 0x80000001) + return 0; + cpuid_leaf(0x80000001, 0, 0, 0, &d); + return (d >> 26) & 1; +} + +// One gigabyte at `pa` with `flags`, in kpdpt[slot]: a single entry if the +// CPU allows, otherwise a page directory of 2 MB pages. 0 if that page +// directory could not be had. +static int +map_gigabyte(int slot, uint64 pa, uint64 flags) +{ + if (gbpages) { + kpdpt[slot] = pa | flags | PTE_PS; + return 1; + } + pde_t *pd = (pde_t *)kalloc(); + if (pd == 0) + return 0; + for (int i = 0; i < 512; i++) + pd[i] = (pa + ((uint64)i << 21)) | flags | PTE_PS; + kpdpt[slot] = v2p(pd) | PTE_P | PTE_W; + return 1; +} + +// What each device-window slot maps, since kpdpt[] holds a page directory +// rather than the frame when there are no 1 GB pages. +static uint64 window_base[512]; + struct tassdesc { uint lim_15_0 : 16; uint base_15_0 : 16; @@ -258,12 +301,17 @@ kvmalloc(void) if (gigs > IOREMAP_LAST_PDPT - 8) gigs = IOREMAP_LAST_PDPT - 8; // leave room for device windows + gbpages = cpu_gb_pages(); + if (!gbpages) + kinfo("vm: no 1 GB pages; the direct map uses 2 MB ones\n"); + for(uint64 i = 0; i < gigs; i++) { - uint64 flags = PTE_PS | PTE_P | PTE_W; + uint64 flags = PTE_P | PTE_W; if(i == 3) { flags |= PTE_PCD | PTE_PWT; } - kpdpt[i] = (i << 30) | flags; + if (!map_gigabyte((int)i, i << 30, flags)) + panic("kvmalloc: no memory for the direct map"); } // ioremap takes what is left. It used to start at a constant 4, which was @@ -312,8 +360,7 @@ ioremap(uint64 pa, uint64 size) // Already mapped? A second BAR in the same gigabyte should not consume a // second window, and drivers do re-map the same device. for (int i = 4; i < ioremap_next; i++) { - // Clearing the low 30 bits drops every flag as well, leaving the frame. - if ((kpdpt[i] & PTE_P) && (kpdpt[i] & ~(GIGABYTE - 1)) == base) + if ((kpdpt[i] & PTE_P) && window_base[i] == base) return (void *)(KERNBASE + ((uint64)i << 30) + (pa - base)); } @@ -324,8 +371,12 @@ ioremap(uint64 pa, uint64 size) int first = ioremap_next; for (int i = 0; i < span; i++) { - kpdpt[ioremap_next] = (base + (uint64)i * GIGABYTE) - | PTE_PS | PTE_P | PTE_W | PTE_PCD | PTE_PWT; + if (!map_gigabyte(ioremap_next, base + (uint64)i * GIGABYTE, + PTE_P | PTE_W | PTE_PCD | PTE_PWT)) { + cprintf("ioremap: no memory to map %p\n", (void *)pa); + return 0; + } + window_base[ioremap_next] = base + (uint64)i * GIGABYTE; ioremap_next++; }