SchOS/boot/uefi/main.c

// SPDX-License-Identifier: Apache-2.0
//
// SchOS UEFI loader.
//
// The BIOS path and this one meet at a contract, not at shared code: see
// boot/README.md. This loader's whole job is to arrive at that contract from
// the other direction.
//
//   1. find the kernel on the volume this image was loaded from
//   2. load its ELF PT_LOAD segments to their physical addresses
//   3. record the framebuffer where kernel/dev/vga.c looks for it
//   4. translate the UEFI memory map into the E820 entries
//      kernel/arch/x86_64/e820.c already parses
//   5. ExitBootServices
//   6. build the page tables the kernel's 64-bit entry expects, and jump
//
// Step 6 is why kernel/arch/x86_64/entry.S grew a `sch64_header`. The
// multiboot entry is 32-bit code that climbs into long mode itself; the
// firmware has already put us there, and dropping back down only to climb out
// again is a mode switch whose failure mode is a triple fault with no console.
// The kernel publishes its 64-bit entry instead, and this jumps straight to it.
//
// Nothing here allocates after ExitBootServices, and nothing calls a firmware
// service after it either -- there are none. Everything the jump needs is
// computed before.

#include "efi.h"
#include "../../include/schboot.h"
#include "../common/ui.h"

// Where the kernel expects to find things comes from schboot.h, included
// above: the hand-off is one contract and both loaders write the same one.
#define E820_MAXENTRIES  SCHBOOT_E820_MAX
#define E820_USABLE      1
#define E820_RESERVED    2
#define E820_ACPI        3
#define E820_NVS         4

#define KERNEL_PATH      L"\\schos\\kernel"
#define KERNEL_PATH_ALT  L"\\kernel"

// Page tables are built here, exactly where mboot_entry builds them, so the
// kernel's entry sees the state it was written for.
#define PML4_ADDR  0x1000
#define PDPT_ADDR  0x2000

#define PTE_P   0x001
#define PTE_W   0x002
#define PTE_PS  0x080

struct e820entry {
    UINT64 base;
    UINT64 length;
    UINT32 type;
    UINT32 acpi_ext;
} __attribute__((packed));

// ELF64, only what is needed to place segments.
struct elf64_ehdr {
    UINT8  e_ident[16];
    UINT16 e_type, e_machine;
    UINT32 e_version;
    UINT64 e_entry, e_phoff, e_shoff;
    UINT32 e_flags;
    UINT16 e_ehsize, e_phentsize, e_phnum, e_shentsize, e_shnum, e_shstrndx;
};

struct elf64_phdr {
    UINT32 p_type, p_flags;
    UINT64 p_offset, p_vaddr, p_paddr, p_filesz, p_memsz, p_align;
};

#define PT_LOAD 1

static EFI_SYSTEM_TABLE  *ST;
static EFI_BOOT_SERVICES *BS;

// ---------------------------------------------------------------------------
// Console. Enough to say what went wrong, and no more.
// ---------------------------------------------------------------------------

static void
print(const CHAR16 *s)
{
    if (ST && ST->ConOut)
        ST->ConOut->OutputString(ST->ConOut, (CHAR16 *)s);
}

static void
print_hex(UINT64 v)
{
    CHAR16 buf[19];
    const CHAR16 *digits = L"0123456789abcdef";
    buf[0] = L'0'; buf[1] = L'x';
    for (int i = 0; i < 16; i++)
        buf[2 + i] = digits[(v >> ((15 - i) * 4)) & 0xf];
    buf[18] = 0;
    print(buf);
}

// A failure here cannot be recovered from and must not be silent: without this
// the firmware simply returns to its boot menu and the user is told nothing.
static EFI_STATUS
die(const CHAR16 *msg, EFI_STATUS st)
{
    print(L"schos: "); print(msg);
    print(L" (status "); print_hex(st); print(L")\r\n");
    print(L"schos: halted.\r\n");
    for (;;)
        __asm__ volatile("hlt");
}

static void
zero(void *p, UINTN n)
{
    UINT8 *b = (UINT8 *)p;
    while (n--) *b++ = 0;
}

static void
copy(void *dst, const void *src, UINTN n)
{
    UINT8 *d = (UINT8 *)dst;
    const UINT8 *s = (const UINT8 *)src;
    while (n--) *d++ = *s++;
}

// ---------------------------------------------------------------------------
// Reading the kernel off the ESP
// ---------------------------------------------------------------------------

static EFI_FILE_PROTOCOL *
open_kernel(EFI_HANDLE image)
{
    EFI_GUID li_guid = EFI_LOADED_IMAGE_PROTOCOL_GUID;
    EFI_GUID fs_guid = EFI_SIMPLE_FILE_SYSTEM_PROTOCOL_GUID;
    EFI_LOADED_IMAGE_PROTOCOL *li = 0;
    EFI_SIMPLE_FILE_SYSTEM_PROTOCOL *fs = 0;
    EFI_FILE_PROTOCOL *root = 0, *file = 0;
    EFI_STATUS st;

    // The volume this loader was itself loaded from, rather than a search of
    // every filesystem: booting one installation's kernel from another's ESP
    // is not a thing to do by accident.
    st = BS->HandleProtocol(image, &li_guid, (void **)&li);
    if (EFI_ERROR(st)) die(L"cannot get LOADED_IMAGE", st);

    st = BS->HandleProtocol(li->DeviceHandle, &fs_guid, (void **)&fs);
    if (EFI_ERROR(st)) die(L"boot volume has no filesystem", st);

    st = fs->OpenVolume(fs, &root);
    if (EFI_ERROR(st)) die(L"cannot open the boot volume", st);

    st = root->Open(root, &file, (CHAR16 *)KERNEL_PATH, EFI_FILE_MODE_READ, 0);
    if (EFI_ERROR(st))
        st = root->Open(root, &file, (CHAR16 *)KERNEL_PATH_ALT,
                        EFI_FILE_MODE_READ, 0);
    if (EFI_ERROR(st)) die(L"no kernel at \\schos\\kernel or \\kernel", st);

    return file;
}

// Load PT_LOAD segments to p_paddr. Physical addresses, not virtual: the
// kernel is linked at 0xffff800000100000 but must be *placed* at 0x100000, and
// p_paddr is where the linker script says each segment belongs.
static UINT64
load_elf(EFI_FILE_PROTOCOL *file)
{
    struct elf64_ehdr eh;
    UINTN n = sizeof eh;
    EFI_STATUS st;

    st = file->SetPosition(file, 0);
    if (EFI_ERROR(st)) die(L"cannot seek the kernel", st);
    st = file->Read(file, &n, &eh);
    if (EFI_ERROR(st) || n != sizeof eh) die(L"cannot read the ELF header", st);

    if (eh.e_ident[0] != 0x7f || eh.e_ident[1] != 'E' ||
        eh.e_ident[2] != 'L'  || eh.e_ident[3] != 'F')
        die(L"kernel is not an ELF file", 0);
    if (eh.e_ident[4] != 2)                 // ELFCLASS64
        die(L"kernel is not 64-bit", 0);

    UINT64 lowest = ~0ULL, highest = 0;

    // Two passes' worth of information from one: the loop below both loads and
    // reports, so the bar tracks bytes actually written rather than a segment
    // count that says nothing about how long each takes.
    UINT64 total = 0, done = 0;
    for (UINT16 i = 0; i < eh.e_phnum; i++) {
        struct elf64_phdr ph;
        UINTN m = sizeof ph;
        if (EFI_ERROR(file->SetPosition(file, eh.e_phoff + (UINT64)i * eh.e_phentsize)))
            break;
        if (EFI_ERROR(file->Read(file, &m, &ph)) || m != sizeof ph)
            break;
        if (ph.p_type == PT_LOAD)
            total += ph.p_memsz;
    }
    ui_progress_begin("Loading SchOS", (ui_u32)(total ? total : 1));

    for (UINT16 i = 0; i < eh.e_phnum; i++) {
        struct elf64_phdr ph;
        n = sizeof ph;
        st = file->SetPosition(file, eh.e_phoff + (UINT64)i * eh.e_phentsize);
        if (EFI_ERROR(st)) die(L"cannot seek a program header", st);
        st = file->Read(file, &n, &ph);
        if (EFI_ERROR(st) || n != sizeof ph) die(L"cannot read a program header", st);

        if (ph.p_type != PT_LOAD || ph.p_memsz == 0)
            continue;

        // Claim the pages before writing them. The firmware is still running
        // and this is its memory until it says otherwise; writing to
        // conventional memory it has not handed over is how a loader corrupts
        // the very services it is about to call.
        EFI_PHYSICAL_ADDRESS at = ph.p_paddr & ~0xfffULL;
        UINTN pages = (UINTN)((ph.p_paddr + ph.p_memsz - at + 0xfff) / 0x1000);
        st = BS->AllocatePages(AllocateAddress, EfiLoaderData, pages, &at);
        if (EFI_ERROR(st)) {
            // Already allocated by the firmware for something of its own, or
            // simply not conventional memory. Not fatal on its own -- the
            // range may still be usable -- but worth saying, because a kernel
            // that boots and then corrupts itself starts here.
            print(L"schos: warning: could not reserve "); print_hex(at);
            print(L"\r\n");
        }

        if (ph.p_filesz) {
            n = (UINTN)ph.p_filesz;
            st = file->SetPosition(file, ph.p_offset);
            if (EFI_ERROR(st)) die(L"cannot seek a segment", st);
            st = file->Read(file, &n, (void *)(UINTN)ph.p_paddr);
            if (EFI_ERROR(st) || n != ph.p_filesz)
                die(L"short read loading a segment", st);
        }
        // .bss: present in memsz, absent from filesz, and the kernel assumes
        // it is zeroed.
        if (ph.p_memsz > ph.p_filesz)
            zero((void *)(UINTN)(ph.p_paddr + ph.p_filesz),
                 (UINTN)(ph.p_memsz - ph.p_filesz));

        done += ph.p_memsz;
        ui_progress_tick((ui_u32)done);

        if (ph.p_paddr < lowest)  lowest = ph.p_paddr;
        if (ph.p_paddr + ph.p_memsz > highest) highest = ph.p_paddr + ph.p_memsz;
    }

    if (highest == 0) die(L"kernel has no loadable segments", 0);

    // Find the 64-bit entry the kernel publishes, by scanning the first 16 KB
    // of what was just loaded -- the same way the BIOS loader finds the
    // multiboot header. A magic in the image beats a symbol table: no ELF
    // symbol parsing, and no fixed offset to drift.
    UINT8 *img = (UINT8 *)(UINTN)lowest;
    for (UINTN off = 0; off + sizeof(struct sch64_header) <= 16384; off += 8) {
        struct sch64_header *h = (struct sch64_header *)(img + off);
        if (h->magic0 == SCH64_MAGIC0 && h->magic1 == SCH64_MAGIC1)
            return h->entry;
    }

    die(L"kernel has no SCH64 header -- too old for this loader", 0);
    return 0;
}

// ---------------------------------------------------------------------------
// Framebuffer
// ---------------------------------------------------------------------------


// Offer a resolution choice, briefly. The UEFI half of the same feature the
// BIOS loader has -- see boot/bios/stage2.S.
//
// Different mechanism, same reasoning. GOP has no mode numbers worth
// hardcoding: the firmware publishes however many modes the hardware can do,
// and which index means what varies by machine. So this walks them, and the
// menu is built from what is actually there rather than from a table.
//
// A second and a half, then it boots on its own. This was four seconds and that
// was too long: it is paid on every unattended boot, and adding four seconds to
// every start to offer a menu nobody usually wants makes the common case worse
// to serve the rare one.

static void
print_dec(UINT32 v)
{
    CHAR16 buf[12];
    int i = 11;
    buf[i--] = 0;
    if (v == 0)
        buf[i--] = L'0';
    while (v && i >= 0) {
        buf[i--] = (CHAR16)(L'0' + (v % 10));
        v /= 10;
    }
    print(&buf[i + 1]);
}

// A key, or 0 if nothing was typed within `ms` milliseconds. Stall is the
// simplest clock available before boot services go away, and polling in 10 ms
// steps is far below anything a person can notice.
static CHAR16
wait_key(UINTN ms)
{
    EFI_INPUT_KEY key;
    for (UINTN waited = 0; waited < ms; waited += 10) {
        if (ST->ConIn && !EFI_ERROR(ST->ConIn->ReadKeyStroke(ST->ConIn, &key)))
            return key.UnicodeChar;
        BS->Stall(10000);
    }
    return 0;
}

static CHAR16
wait_key_forever(void)
{
    EFI_INPUT_KEY key;
    for (;;) {
        if (ST->ConIn && !EFI_ERROR(ST->ConIn->ReadKeyStroke(ST->ConIn, &key)))
            return key.UnicodeChar;
        BS->Stall(10000);
    }
}

// Read a decimal number, echoing it. Ends on Enter (CR, or LF over a serial
// console). Non-digits are ignored rather than aborting, so a stray keystroke
// does not silently truncate the number.
static UINT32
read_dec(void)
{
    UINT32 v = 0;
    for (;;) {
        CHAR16 c = wait_key_forever();
        if (c == L'\r' || c == L'\n')
            return v;
        if (c < L'0' || c > L'9')
            continue;
        CHAR16 echo[2]; echo[0] = c; echo[1] = 0;
        print(echo);
        v = v * 10 + (UINT32)(c - L'0');
    }
}

// Pick a GOP mode. Returns the mode number to set, or the current one.
static UINT32
choose_mode(EFI_GRAPHICS_OUTPUT_PROTOCOL *gop)
{
    UINT32 current = gop->Mode->Mode;

    print(L"Press R for resolution... ");
    CHAR16 c = wait_key(1500);
    if (c != L'r' && c != L'R') {
        print(L"keeping ");
        print_dec(gop->Mode->Info->HorizontalResolution); print(L"x");
        print_dec(gop->Mode->Info->VerticalResolution);   print(L"\r\n");
        return current;
    }

    // List what the firmware actually offers, 32bpp only: the console, the
    // fbdev ioctls and Schcom all assume it, and PixelFormat 2 is a packed
    // format this loader does not describe to the kernel.
    print(L"\r\n");
    for (UINT32 i = 0; i < gop->Mode->MaxMode; i++) {
        EFI_GRAPHICS_OUTPUT_MODE_INFORMATION *dbg = 0;
        UINTN dsz = 0;
        EFI_STATUS dst = gop->QueryMode(gop, i, &dsz, &dbg);
        print(L"   raw "); print_dec(i); print(L" st="); print_hex(dst);
        if (!EFI_ERROR(dst) && dbg) {
            print(L" "); print_dec(dbg->HorizontalResolution);
            print(L"x"); print_dec(dbg->VerticalResolution);
            print(L" fmt="); print_dec(dbg->PixelFormat);
        }
        print(L"\r\n");
    }
    for (UINT32 i = 0; i < gop->Mode->MaxMode; i++) {
        EFI_GRAPHICS_OUTPUT_MODE_INFORMATION *info = 0;
        UINTN size = 0;
        if (EFI_ERROR(gop->QueryMode(gop, i, &size, &info)) || !info)
            continue;
        if (info->PixelFormat > 1)
            continue;
        if (info->HorizontalResolution < 800 || info->VerticalResolution < 600)
            continue;
        print(L"  ");
        print_dec(i); print(L") ");
        print_dec(info->HorizontalResolution); print(L"x");
        print_dec(info->VerticalResolution);   print(L"\r\n");
    }
    print(L"  mode number, or Enter to type a size: ");

    UINT32 want = read_dec();
    print(L"\r\n");

    // Entering nothing means "let me type it instead". Zero is a valid mode
    // number, so the two are told apart by whether anything was typed at all;
    // read_dec returns 0 for an empty line, and mode 0 is reachable from the
    // typed path anyway.
    if (want != 0 || gop->Mode->MaxMode == 0) {
        if (want < gop->Mode->MaxMode)
            return want;
        print(L"schos: no such mode, keeping the current one\r\n");
        return current;
    }

    print(L"  width:  ");
    UINT32 w = read_dec();
    print(L"\r\n  height: ");
    UINT32 h = read_dec();
    print(L"\r\n");

    // 800x600 is the floor the system is built around.
    if (w < 800 || h < 600) {
        print(L"schos: below the 800x600 minimum, keeping the current mode\r\n");
        return current;
    }

    for (UINT32 i = 0; i < gop->Mode->MaxMode; i++) {
        EFI_GRAPHICS_OUTPUT_MODE_INFORMATION *info = 0;
        UINTN size = 0;
        if (EFI_ERROR(gop->QueryMode(gop, i, &size, &info)) || !info)
            continue;
        if (info->PixelFormat > 1)
            continue;
        if (info->HorizontalResolution == w && info->VerticalResolution == h)
            return i;
    }

    print(L"schos: no such 32bpp mode, keeping the current one\r\n");
    return current;
}

static UINT32
setup_framebuffer(void)
{
    EFI_GUID gop_guid = EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID;
    EFI_GRAPHICS_OUTPUT_PROTOCOL *gop = 0;

    if (EFI_ERROR(BS->LocateProtocol(&gop_guid, 0, (void **)&gop)) || !gop) {
        print(L"schos: no graphics output; console only\r\n");
        return 0;
    }

    UINT32 pick = choose_mode(gop);
    if (pick != gop->Mode->Mode)
        gop->SetMode(gop, pick);

    // Re-read after SetMode: Info and FrameBufferBase both describe the mode
    // that is current, and the framebuffer can move when the mode changes.
    EFI_GRAPHICS_OUTPUT_MODE_INFORMATION *info = gop->Mode->Info;
    UINT32 fb = (UINT32)gop->Mode->FrameBufferBase;

    // The layout kernel/dev/vga.c reads, byte for byte. It is a VBE mode-info
    // block as the BIOS path leaves it, and the kernel takes exactly four
    // fields out of it -- so those four are what get written, at the offsets
    // it uses.
    UINT8 *vbe = (UINT8 *)SCHBOOT_VBE_INFO;
    zero(vbe, 512);

    UINT32 bpp = 32;                       // GOP's 8-bit formats are 32bpp
    UINT16 stride = (UINT16)(info->PixelsPerScanLine * (bpp / 8));

    *(UINT16 *)(vbe + 16) = stride;                             // bytes/line
    *(UINT16 *)(vbe + 18) = (UINT16)info->HorizontalResolution;
    *(UINT16 *)(vbe + 20) = (UINT16)info->VerticalResolution;
    *(UINT8  *)(vbe + 25) = (UINT8)bpp;
    *(UINT32 *)(vbe + 40) = fb;

    // The same UI the BIOS loader draws, on the framebuffer this one just
    // obtained. Everything above this point is firmware-specific; nothing below
    // it is, which is the whole reason it can be shared.
    struct ui_fb uifb;
    uifb.base   = (ui_u8 *)(UINTN)fb;
    uifb.width  = info->HorizontalResolution;
    uifb.height = info->VerticalResolution;
    uifb.stride = stride;
    ui_init(&uifb);
    ui_splash("SchOS", "Starting up");

    print(L"schos: framebuffer "); print_hex(fb);
    print(L" "); print_hex(info->HorizontalResolution);
    print(L"x"); print_hex(info->VerticalResolution);
    print(L"\r\n");

    return fb;
}

// ---------------------------------------------------------------------------
// Memory map
//
// UEFI hands out EFI_MEMORY_DESCRIPTORs; the kernel parses E820 entries. The
// translation happens here rather than in the kernel, so that
// kernel/arch/x86_64/e820.c stays the one place that understands a memory map.
// ---------------------------------------------------------------------------

static UINT32
efi_type_to_e820(UINT32 t)
{
    switch (t) {
    // Boot services memory is ours the moment ExitBootServices returns, and
    // the loader's own pages likewise -- the kernel is what runs next.
    case EfiConventionalMemory:
    case EfiBootServicesCode:
    case EfiBootServicesData:
    case EfiLoaderCode:
    case EfiLoaderData:
        return E820_USABLE;
    case EfiACPIReclaimMemory:
        return E820_ACPI;
    case EfiACPIMemoryNVS:
        return E820_NVS;
    default:
        // Runtime services, MMIO, unusable, persistent, reserved: all of it is
        // memory the kernel must not allocate from.
        return E820_RESERVED;
    }
}

// ---------------------------------------------------------------------------

static int
guid_eq(const EFI_GUID *a, const EFI_GUID *b)
{
    const UINT8 *p = (const UINT8 *)a, *q = (const UINT8 *)b;
    for (UINTN i = 0; i < sizeof(EFI_GUID); i++)
        if (p[i] != q[i])
            return 0;
    return 1;
}

// The ACPI RSDP, from the UEFI configuration table.
//
// The kernel's own search scans the EBDA and 0xE0000-0x100000. That is a BIOS
// convention, and UEFI firmware is under no obligation to leave a copy there --
// OVMF does not. The kernel therefore found no RSDP, so no MADT, so no IOAPIC,
// so nothing routed IRQ 4: it booted to a shell prompt on polled serial output
// and then ignored every key, which reads as a broken console rather than a
// missing table.
//
// ACPI 2.0 is preferred over 1.0 because its RSDP carries the XSDT, whose
// entries are 64-bit; the 1.0 RSDT is kept only as a fallback.
static UINT64
find_rsdp(void)
{
    EFI_GUID acpi20 = EFI_ACPI_20_TABLE_GUID;
    EFI_GUID acpi10 = EFI_ACPI_10_TABLE_GUID;
    EFI_CONFIGURATION_TABLE *t =
        (EFI_CONFIGURATION_TABLE *)ST->ConfigurationTable;
    UINT64 fallback = 0;

    for (UINTN i = 0; i < ST->NumberOfTableEntries; i++) {
        if (guid_eq(&t[i].VendorGuid, &acpi20))
            return (UINT64)(UINTN)t[i].VendorTable;
        if (guid_eq(&t[i].VendorGuid, &acpi10))
            fallback = (UINT64)(UINTN)t[i].VendorTable;
    }
    return fallback;
}

EFI_STATUS
EfiMain(EFI_HANDLE image, EFI_SYSTEM_TABLE *systab)
{
    ST = systab;
    BS = systab->BootServices;

    if (ST->ConOut)
        ST->ConOut->ClearScreen(ST->ConOut);
    print(L"SchOS UEFI loader\r\n");

    // The display comes first, before the kernel is read.
    //
    // It used to be the other way round, which meant the resolution prompt
    // appeared after the slowest step and the progress bar was drawn into a
    // framebuffer that did not exist yet -- every tick was silently discarded
    // and then the splash painted over the result. Setting the mode first also
    // puts the menu where a person expects it: at the start.
    UINT32 fb = setup_framebuffer();

    EFI_FILE_PROTOCOL *file = open_kernel(image);
    UINT64 entry = load_elf(file);
    ui_progress_end();
    file->Close(file);

    print(L"schos: kernel entry "); print_hex(entry); print(L"\r\n");

    // Read while boot services still exist and the system table is guaranteed
    // valid; written to low memory further down, after the firmware has let go
    // of it, for the same reason the e820 map is.
    UINT64 rsdp = find_rsdp();
    print(L"schos: acpi rsdp "); print_hex(rsdp); print(L"\r\n");

    // The memory map, and then out. GetMemoryMap must be the last thing before
    // ExitBootServices: the key it returns is invalidated by any allocation,
    // and AllocatePool for the map buffer is itself an allocation -- hence the
    // static buffer and the retry.
    static UINT8 mapbuf[32768];
    UINTN mapsize = sizeof mapbuf, mapkey = 0, descsize = 0;
    UINT32 descver = 0;
    EFI_STATUS st;

    st = BS->GetMemoryMap(&mapsize, (EFI_MEMORY_DESCRIPTOR *)mapbuf,
                          &mapkey, &descsize, &descver);
    if (EFI_ERROR(st)) die(L"GetMemoryMap failed", st);

    // Translate before leaving, while the map is still readable and while
    // printing still works, but write it to low memory only after the firmware
    // has let go -- 0x30000 may well be firmware data until then.
    static struct e820entry e820[E820_MAXENTRIES];
    UINT32 nent = 0;
    for (UINTN off = 0; off + descsize <= mapsize && nent < E820_MAXENTRIES;
         off += descsize) {
        EFI_MEMORY_DESCRIPTOR *d = (EFI_MEMORY_DESCRIPTOR *)(mapbuf + off);
        UINT32 type = efi_type_to_e820(d->Type);

        // Coalesce adjacent runs of the same type. A UEFI map is routinely
        // over a hundred descriptors and the kernel accepts 32; without this
        // it would see a truncated map and believe the machine has less
        // memory than it does.
        if (nent > 0 && e820[nent-1].type == type &&
            e820[nent-1].base + e820[nent-1].length == d->PhysicalStart) {
            e820[nent-1].length += d->NumberOfPages * 0x1000;
            continue;
        }
        e820[nent].base     = d->PhysicalStart;
        e820[nent].length   = d->NumberOfPages * 0x1000;
        e820[nent].type     = type;
        e820[nent].acpi_ext = 1;
        nent++;
    }

    print(L"schos: exiting boot services\r\n");

    st = BS->ExitBootServices(image, mapkey);
    if (EFI_ERROR(st)) {
        // The map changed underneath us between GetMemoryMap and here. Take it
        // again and retry once; this is normal and the specification expects
        // a loader to handle it.
        mapsize = sizeof mapbuf;
        st = BS->GetMemoryMap(&mapsize, (EFI_MEMORY_DESCRIPTOR *)mapbuf,
                              &mapkey, &descsize, &descver);
        if (EFI_ERROR(st)) die(L"GetMemoryMap failed on retry", st);
        st = BS->ExitBootServices(image, mapkey);
        if (EFI_ERROR(st)) die(L"ExitBootServices failed", st);
    }

    // ---- no firmware services exist past this line ----

    __asm__ volatile("cli");

    copy((void *)SCHBOOT_E820_MAP, e820, nent * sizeof e820[0]);
    *(UINT32 *)SCHBOOT_E820_COUNT = nent;
    *(UINT64 *)SCHBOOT_RSDP_ADDR = rsdp;

    // The page tables mboot_entry would have built. entry64low's first act is
    // a movabs to 0xffff8000...., so the high half must be mapped before the
    // jump, not after: identity alone triple faults on the first instruction.
    UINT64 *pml4 = (UINT64 *)PML4_ADDR;
    UINT64 *pdpt = (UINT64 *)PDPT_ADDR;
    zero(pml4, 0x1000);
    zero(pdpt, 0x1000);
    pml4[0]   = PDPT_ADDR | PTE_P | PTE_W;      // identity
    pml4[256] = PDPT_ADDR | PTE_P | PTE_W;      // 0xffff800000000000
    for (int i = 0; i < 4; i++)                  // 4 GB in 1 GB pages
        pdpt[i] = ((UINT64)i << 30) | PTE_P | PTE_PS | PTE_W;

    // RBX carries the framebuffer address into kmain, and must never be 1 --
    // that value marks an application processor in entry64high. A physical
    // MMIO address or zero, so it cannot collide.
    __asm__ volatile(
        "mov %0, %%cr3\n\t"
        "mov %1, %%rbx\n\t"
        "jmp *%2\n\t"
        :
        : "r"((UINT64)PML4_ADDR), "r"((UINT64)fb), "r"(entry)
        : "memory");

    for (;;)
        __asm__ volatile("hlt");
}