SchOS/kernel/arch/x86_64/vm.c

#include "core/param.h"
#include "core/types.h"
#include "core/defs.h"
#include "gfx/gfx.h"
#include "arch/x86.h"
#include "arch/memlayout.h"
#include "arch/mmu.h"
#include "core/proc.h"
#include "arch/ioperm.h"
#include "core/spinlock.h"
#include "arch/vmap.h"
#include "ldls/kldls.h"

int mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm);
static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc);


// `cpu` and `proc` are no longer __thread variables; they live at fixed offsets
// in the per-CPU block and are read through %gs at each use. See the long
// comment in include/proc.h for why -- the short version is that __thread means
// per *thread*, the compiler caches the thread pointer across calls on that
// basis, and a kernel thread changes CPU at every sched().

// Kernel FS_BASE set by seginit(); read by trapasm.S to restore after user TLS
uint64 kernel_fs_base;

// The per-CPU block's layout is shared with trapasm.S, which spells the offsets
// as literals because there is no way to include a C header from there. These
// pin the C side: if a slot moves, the assembly keeps reading the old address
// and the failure is a register quietly holding somebody else's value, which is
// the hardest possible thing to find. Ask the question at compile time instead.
_Static_assert(PERCPU_SELF == 0 && PERCPU_SCRATCH == 8 && PERCPU_CPU == 16 &&
               PERCPU_PROC == 24 && PERCPU_SC_RAX == 32 &&
               PERCPU_SC_RCX == 40 && PERCPU_SC_RDX == 48,
               "per-CPU block layout changed -- update kernel/arch/x86_64/trapasm.S");

static pml4e_t *kpml4;
// Where the device windows begin is now decided by kvmalloc(), which knows how
// much of the address space the direct map took. The constant that used to be
// here said 4, and would have handed out windows on top of RAM the moment the
// direct map grew past four gigabytes.
#define IOREMAP_LAST_PDPT  511
#define GIGABYTE           (1ULL << 30)

// The first PDPT entry the direct map did not take. Set by kvmalloc().
static int ioremap_next = 4;

static pdpe_t *kpdpt;

// Whether a PDPT entry can map a gigabyte (CPUID 0x80000001, EDX bit 26).
// Intel's desktop CPUs before Haswell cannot: the bit is reserved there, and
// a 1 GB entry is a page fault on first use. The direct map and the device
// window then use a page directory of 2 MB pages per gigabyte instead -- the
// same reach, one page of table per gigabyte. See gb_pages() in the loader,
// which has the same fallback for the tables this kernel starts on.
static int gbpages;

static int
cpu_gb_pages(void)
{
  uint32 max, d;

  cpuid_leaf(0x80000000, &max, 0, 0, 0);
  if (max < 0x80000001)
    return 0;
  cpuid_leaf(0x80000001, 0, 0, 0, &d);
  return (d >> 26) & 1;
}

// One gigabyte at `pa` with `flags`, in kpdpt[slot]: a single entry if the
// CPU allows, otherwise a page directory of 2 MB pages. 0 if that page
// directory could not be had.
static int
map_gigabyte(int slot, uint64 pa, uint64 flags)
{
  if (gbpages) {
    kpdpt[slot] = pa | flags | PTE_PS;
    return 1;
  }
  pde_t *pd = (pde_t *)kalloc();
  if (pd == 0)
    return 0;
  for (int i = 0; i < 512; i++)
    pd[i] = (pa + ((uint64)i << 21)) | flags | PTE_PS;
  kpdpt[slot] = v2p(pd) | PTE_P | PTE_W;
  return 1;
}

// What each device-window slot maps, since kpdpt[] holds a page directory
// rather than the frame when there are no 1 GB pages.
static uint64 window_base[512];

struct tassdesc {
  uint lim_15_0 : 16;  
  uint base_15_0 : 16; 
  uint base_23_16 : 8; 
  uint type : 4;       
  uint s : 1;          
  uint dpl : 2;        
  uint p : 1;          
  uint lim_19_16 : 4;  
  uint avl : 1;        
  uint l : 1;          
  uint d : 1;          
  uint g : 1;          
  uint base_31_24 : 8; 
  uint32 base_63_32;   
  uint32 res;          
};

void
syscallinit(void)
{
  // SYSRETQ does NOT use USER_CS/USER_DS directly. Per the AMD64/Intel
  // convention, it computes CS = STAR[63:48]+16 and SS = STAR[63:48]+8
  // (RPL forced to 3), so STAR[63:48] must hold the *compat* 32-bit user
  // code selector (SEG_UCODE32), relying on the GDT's consecutive
  // [UCODE32, UDATA, UCODE] layout — NOT USER_CS itself. Using USER_CS
  // here made every sysretq load CS=USER_CS+16 (an invalid selector,
  // pointing past UCODE into the TSS descriptor) and SS=USER_CS+8. QEMU's
  // TCG apparently doesn't validate segment descriptors on SYSRET, so this
  // went unnoticed until an interrupt forced a strictly-validated iretq
  // with that same bogus CS still loaded, which correctly #GP'd.
  wrmsr(MSR_STAR,
    ((((uint64)((SEG_UCODE32 << 3) | DPL_USER) << 16) | (uint64)KERNEL_CS) << 32));
  wrmsr(MSR_LSTAR, (addr_t)syscall_entry);
  wrmsr(MSR_CSTAR, (addr_t)ignore_sysret);
  wrmsr(MSR_SFMASK, FL_TF|FL_DF|FL_IF|FL_IOPL_MASK|FL_AC|FL_NT);
}

void
seginit(void)
{
  struct segdesc *gdt;
  uint *tss;
  uint64 addr, tsslim;
  void *local;
  struct cpu *c;

  // The per-CPU page, preallocated by the boot processor for every AP.
  //
  // It must not be kalloc()'d here on an application processor. kalloc takes a
  // spinlock; acquire() calls holding(); holding() reads `cpu`, which is a
  // __thread variable reached through %fs -- and %fs is set up four lines
  // below. So an AP allocating its own page reads the per-CPU pointer before
  // the per-CPU pointer exists, and the kernel dies in panic("acquire") with
  // every lock appearing to be held.
  //
  // startothers() fills c->local in before starting each AP, on the boot
  // processor, where locking works.
  c = &cpus[cpunum()];
  local = c->local;
  if (local == 0) {
    local = kalloc();          // the boot processor's own, taken normally
    memset(local, 0, PGSIZE);
  }

  gdt = (struct segdesc*) local;

  // The TSS moved out of this page. It now carries an I/O permission bitmap
  // (include/ioperm.h) which does not fit here, and it has to be one object
  // with it. ioperm_tss() returns it and the descriptor limit that reaches the
  // end of the bitmap.
  tss = (uint*) ioperm_tss(cpunum(), &tsslim);

  kernel_fs_base = ((uint64) local) + 2048;
  wrmsr(0xC0000100, kernel_fs_base);

  // The TCB self-pointer: the word at %fs:0 must hold the thread pointer.
  //
  // This is part of the x86-64 TLS ABI and it was missing, which worked only
  // by accident. GCC with -mtls-direct-seg-refs compiles `cpu = c` into a
  // single `movq %rcx, %fs:-0x10` and never needs the thread pointer as a
  // value, so nothing ever read this word. clang materialises it instead --
  // `movq %fs:0x0, %rax; addq $-0x10, %rax; movq %rcx, (%rax)` -- which read
  // the zero left by the memset above and stored through address
  // 0xfffffffffffffff0, three instructions later.
  //
  // Writing it costs one store per CPU and removes the dependency on a
  // particular compiler's TLS optimisation.
  *(uint64 *) kernel_fs_base = kernel_fs_base;

  // Clear the per-CPU slots before anything can read them. Their offsets are
  // the PERCPU_* constants in include/proc.h and are shared with trapasm.S,
  // which reads %gs:24 for proc on the way out to user mode and parks three
  // registers at %gs:32..48 across the FS_BASE swap on the way in.
  ((uint64 *) kernel_fs_base)[2] = 0;      // struct cpu *   (PERCPU_CPU)
  ((uint64 *) kernel_fs_base)[3] = 0;      // struct proc *  (PERCPU_PROC)
  ((uint64 *) kernel_fs_base)[4] = 0;      // syscall rax    (PERCPU_SC_RAX)
  ((uint64 *) kernel_fs_base)[5] = 0;      // syscall rcx    (PERCPU_SC_RCX)
  ((uint64 *) kernel_fs_base)[6] = 0;      // syscall rdx    (PERCPU_SC_RDX)

  // The same address in GS base, per CPU.
  //
  // trapasm.S needs to find *this* CPU's kernel FS base on every entry from
  // user mode, and it has no register to work from at that instant -- which is
  // why it used to read the kernel_fs_base global. That global is written by
  // every CPU during its own seginit(), so with more than one processor the
  // last writer wins and every CPU restores one CPU's per-CPU block. GS base is
  // per-CPU in hardware, so %gs:0 answers the question with no global at all.
  //
  // Safe because GS belongs entirely to the kernel here: sys_arch_prctl()
  // implements ARCH_SET_FS and ARCH_GET_FS only, and musl uses FS for TLS on
  // x86-64. See the comment on RESTORE_KERNEL_FS.
  wrmsr(0xC0000101, kernel_fs_base);      // IA32_GS_BASE

  c->local = local;

  cpu_set(c);
  proc_set(0);

  // Two of the entries below are deliberately null descriptors, and neither is
  // dead code even though both look it:
  //
  //   SEG_UCODE32  is never loaded, but SYSRETQ computes CS from STAR[63:48]+16
  //                and SS from +8, so the *slot* must exist to keep the
  //                [UCODE32, UDATA, UCODE] triple consecutive. See syscallinit.
  //   SEG_KCPU     is a leftover slot number from 32-bit xv6, where %gs held the
  //                per-CPU pointer. SchOS uses FS_BASE/TLS instead, but the
  //                index has to stay occupied or SEG_TSS stops landing at 7.
  //
  // Deleting either moves every descriptor after it and breaks sysret.
  addr = (uint64) tss;
  gdt[0] = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
  gdt[SEG_KCODE] = SEG(STA_X|STA_R, 0, 0, 0);
  gdt[SEG_KDATA] = SEG(STA_W, 0, 0, 0);
  gdt[SEG_UCODE32] = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
  gdt[SEG_UDATA] = SEG(STA_W, 0, 0, DPL_USER);
  gdt[SEG_UCODE] = SEG(STA_X|STA_R, 0, 0, DPL_USER);
  gdt[SEG_KCPU]  = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
  
  struct tassdesc *td = (struct tassdesc*)(gdt + SEG_TSS);
  td->lim_15_0 = tsslim & 0xffff;
  td->base_15_0 = addr & 0xffff;
  td->base_23_16 = (addr >> 16) & 0xff;
  td->type = STS_T64A;
  td->s = 0;
  td->dpl = 0;
  td->p = 1;
  td->lim_19_16 = (tsslim >> 16) & 0xf;
  td->avl = 0;
  td->l = 0;
  td->d = 0;
  td->g = 0;
  td->base_31_24 = (addr >> 24) & 0xff;
  td->base_63_32 = (addr >> 32) & 0xffffffff;
  td->res = 0;

  lgdt((void*) gdt, (NSEGS+1) * sizeof(struct segdesc));
  ltr(SEG_TSS << 3);
}

pml4e_t*
setupkvm(void)
{
  pml4e_t *pml4 = (pml4e_t*) kalloc();
  if(pml4 == 0) return 0;
  memset(pml4, 0, PGSIZE);

  pml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W;
  // The LDLS window: the kernel's second mapping and the modules above it.
  // A module's interrupt and its kernel thread can run in any address space,
  // so it has to be in all of them. See ldls_vm_init().
  pml4[PMX(LDLS_KALIAS)] = kpml4[PMX(LDLS_KALIAS)];

  // The framebuffer is deliberately NOT mapped here.
  //
  // This used to identity-map 0xfd000000 for 800*600*4 bytes, user-writable,
  // into every process that ever existed. Three things were wrong with that.
  // The address and the size were both guesses that only hold on QEMU's
  // stdvga at one resolution; setupkvm runs before vga_init, so on the first
  // call nothing had even read the VBE block yet; and every process could
  // scribble on the screen without asking anyone.
  //
  // Nothing used it -- userspace reaches the framebuffer through
  // mmap("/dev/fb0"), which maps the real base at the real size. So it is
  // gone, and a process gets the framebuffer exactly when it asks for it.

  return pml4;
}

void
kvmalloc(void)
{
  kpml4 = (pml4e_t*) kalloc();
  memset(kpml4, 0, PGSIZE);

  kpdpt = (pdpe_t*) kalloc();
  memset(kpdpt, 0, PGSIZE);

  kpml4[0] = v2p(kpdpt) | PTE_P | PTE_W;
  kpml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W;
  
  // As many gigabytes as the machine has RAM, not four.
  //
  // Four was the number for as long as PHYSTOP was capped at 3 GB, and the cap
  // existed because this loop stopped at four -- each holding the other up. A
  // laptop with sixteen gigabytes lost thirteen of them to that circle.
  //
  // The 3-4 GB entry stays uncached because that is where the PCI hole is and
  // always has been. Everything above 4 GB is RAM -- the hole is below it on
  // every machine this runs on -- so those are cached, and a device BAR up
  // there still gets its own uncached window from ioremap().
  uint64 gigs = (phystop + GIGABYTE - 1) / GIGABYTE;
  if (gigs < 4)
    gigs = 4;                       // always map the low four
  if (gigs > IOREMAP_LAST_PDPT - 8)
    gigs = IOREMAP_LAST_PDPT - 8;   // leave room for device windows

  gbpages = cpu_gb_pages();
  if (!gbpages)
    kinfo("vm: no 1 GB pages; the direct map uses 2 MB ones\n");

  for(uint64 i = 0; i < gigs; i++) {
    uint64 flags = PTE_P | PTE_W;
    if(i == 3) {
      flags |= PTE_PCD | PTE_PWT;
    }
    if (!map_gigabyte((int)i, i << 30, flags))
      panic("kvmalloc: no memory for the direct map");
  }

  // ioremap takes what is left. It used to start at a constant 4, which was
  // correct only while the direct map was exactly four gigabytes.
  ioremap_next = (int)gigs;

  ldls_vm_init();

  switchkvm();
}

// --- the top two gigabytes, for LDLS -----------------------------------------
//
// A Linux module is compiled with -mcmodel=kernel, which is a promise that the
// module and everything it calls live in the top 2 GB of the address space:
// R_X86_64_32S relocations sign-extend a 32-bit field, so the only addresses
// they can name are 0xffffffff80000000 and above. This kernel is linked at
// KERNBASE, nowhere near there, so a module cannot be relocated against it.
//
// The fix is the layout Linux itself uses. The kernel image is mapped a second
// time at LDLS_KALIAS -- same physical pages, so the same code and the same
// variables -- and modules load at LDLS_MODBASE, a few hundred megabytes above
// it. Every symbol a module imports then resolves to its alias address, and a
// call or a data reference reaches it with a 32-bit displacement.
//
// Running kernel code at the alias is sound because the whole image moves
// together: an absolute reference still names the KERNBASE copy, and a
// rip-relative one names the alias of the same byte. Either way it is the same
// memory.
//
// One PDPT under PML4 slot 511, shared by every address space exactly as
// kpdpt is, so a mapping made here is visible everywhere at once. Nothing in
// it is user-accessible: the leaves carry no PTE_U, and the U/S bit is the AND
// of every level.
static pdpe_t *hipdpt;
static struct spinlock kmaplock;

void
ldls_vm_init(void)
{
  extern char end[];
  pde_t *alias, *vmm;
  uint64 top, i;

  initlock(&kmaplock, "kmap");

  hipdpt = (pdpe_t *)kalloc();
  alias  = (pde_t *)kalloc();
  vmm    = (pde_t *)kalloc();
  if (!hipdpt || !alias || !vmm)
    panic("ldls_vm_init");
  memset(hipdpt, 0, PGSIZE);
  memset(alias, 0, PGSIZE);
  memset(vmm, 0, PGSIZE);

  // The alias, in 2 MB pages from physical zero to past the end of the image.
  // It covers some memory that is not the kernel, which is harmless: it is a
  // second name for pages the direct map already names.
  top = ((uint64)V2P(end) + (2ULL << 20) - 1) >> 21;
  if (top > 256)
    panic("ldls_vm_init: kernel image larger than 512 MB");
  for (i = 0; i < top; i++)
    alias[i] = (i << 21) | PTE_PS | PTE_P | PTE_W;

  hipdpt[PDPX(LDLS_KALIAS)]  = v2p(alias) | PTE_P | PTE_W;
  hipdpt[PDPX(LDLS_VMEMMAP)] = v2p(vmm) | PTE_P | PTE_W;
  kpml4[PMX(LDLS_KALIAS)]    = v2p(hipdpt) | PTE_P | PTE_W;
}

// Map one page in the LDLS window. Returns 0, or -1 if the slot is taken or a
// page table could not be had.
int
kmap_page(uint64 va, addr_t pa)
{
  pde_t *pd, *pde;
  pte_t *pt, *pte;

  if (PMX(va) != PMX(LDLS_KALIAS) || !(hipdpt[PDPX(va)] & PTE_P))
    return -1;

  acquire(&kmaplock);
  pd  = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)]));
  pde = &pd[PDX(va)];
  if (*pde & PTE_PS) {
    release(&kmaplock);
    return -1;                      // the alias; never mapped over
  }
  if (!(*pde & PTE_P)) {
    if ((pt = (pte_t *)kalloc()) == 0) {
      release(&kmaplock);
      return -1;
    }
    memset(pt, 0, PGSIZE);
    *pde = v2p(pt) | PTE_P | PTE_W;
  }
  pt  = (pte_t *)P2V(PTE_ADDR(*pde));
  pte = &pt[PTX(va)];
  if (*pte & PTE_P) {
    release(&kmaplock);
    return -1;
  }
  *pte = pa | PTE_P | PTE_W;
  release(&kmaplock);
  invlpg((void *)va);
  return 0;
}

// What is mapped at `va` in the window, or 0.
addr_t
kmap_phys(uint64 va)
{
  pde_t *pd, *pde;
  pte_t *pt;

  if (PMX(va) != PMX(LDLS_KALIAS) || !(hipdpt[PDPX(va)] & PTE_P))
    return 0;
  pd  = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)]));
  pde = &pd[PDX(va)];
  if (!(*pde & PTE_P))
    return 0;
  if (*pde & PTE_PS)
    return PTE_ADDR(*pde) + (va & ((2ULL << 20) - 1));
  pt = (pte_t *)P2V(PTE_ADDR(*pde));
  if (!(pt[PTX(va)] & PTE_P))
    return 0;
  return PTE_ADDR(pt[PTX(va)]) + (va & (PGSIZE - 1));
}

// Take a page out of the window and say which frame it was, or 0.
//
// Only this CPU's TLB is flushed. There is no shootdown IPI in this kernel, so
// another CPU may keep a stale translation for a while -- which is why the
// LDLS window never hands the same virtual address out twice. A stale entry
// for an address nobody will use again is harmless.
addr_t
kunmap_page(uint64 va)
{
  addr_t pa = kmap_phys(va);
  pde_t *pd, *pde;
  pte_t *pt;

  if (!pa)
    return 0;
  acquire(&kmaplock);
  pd  = (pde_t *)P2V(PTE_ADDR(hipdpt[PDPX(va)]));
  pde = &pd[PDX(va)];
  if (!(*pde & PTE_PS)) {
    pt = (pte_t *)P2V(PTE_ADDR(*pde));
    pt[PTX(va)] = 0;
  } else {
    pa = 0;                         // the alias is not ours to unmap
  }
  release(&kmaplock);
  invlpg((void *)va);
  return pa ? PGROUNDDOWN(pa) : 0;
}

// Map a device's registers so the kernel can reach them.
//
// Every driver used to say P2V(bar.base), which quietly assumes the BAR lands
// inside the 0-4 GB the loop above maps. Under SeaBIOS it always does. Under
// OVMF it does not: firmware is free to put a 64-bit BAR anywhere, and it puts
// virtio-gpu's at about 768 GB. P2V then produces an address with no
// translation behind it, and the driver's first register write takes a page
// fault -- which, coming from a driver bind, looks like broken hardware rather
// than a missing mapping.
//
// The direct map uses PDPT entries 0-3 of a page directory pointer table that
// kpml4[0], kpml4[KERNBASE] and every process's PML4 (see setupkvm) all share.
// Entries 4-511 are free, so each one can carry a 1 GB page of device memory:
// no page table to allocate, nothing to free, and the mapping appears in every
// address space at once because they all point at this one table.
//
// Cache-disabled and write-through, for the same reason the top GB of the
// direct map is: these are registers, and a write that sits in a cache line
// waiting for company is a write the device never sees.

void *
ioremap(uint64 pa, uint64 size)
{
  if (size == 0)
    return 0;

  // Below 4 GB the direct map already covers it, so hand back the same address
  // the old code would have produced. This is what keeps the BIOS path -- where
  // every BAR is low -- byte for byte what it was.
  if (pa + size <= 4 * GIGABYTE)
    return P2V(pa);

  uint64 base = pa & ~(GIGABYTE - 1);          // 1 GB pages, so align to one
  uint64 last = (pa + size - 1) & ~(GIGABYTE - 1);
  int    span = (int)((last - base) / GIGABYTE) + 1;

  // Already mapped? A second BAR in the same gigabyte should not consume a
  // second window, and drivers do re-map the same device.
  for (int i = 4; i < ioremap_next; i++) {
    if ((kpdpt[i] & PTE_P) && window_base[i] == base)
      return (void *)(KERNBASE + ((uint64)i << 30) + (pa - base));
  }

  if (ioremap_next + span > IOREMAP_LAST_PDPT) {
    cprintf("ioremap: out of device window for %p\n", (void *)pa);
    return 0;
  }

  int first = ioremap_next;
  for (int i = 0; i < span; i++) {
    if (!map_gigabyte(ioremap_next, base + (uint64)i * GIGABYTE,
                      PTE_P | PTE_W | PTE_PCD | PTE_PWT)) {
      cprintf("ioremap: no memory to map %p\n", (void *)pa);
      return 0;
    }
    window_base[ioremap_next] = base + (uint64)i * GIGABYTE;
    ioremap_next++;
  }

  // The entries were not present a moment ago. Reloading CR3 is heavier than
  // invlpg but this runs a handful of times at boot, and it cannot be the thing
  // that is subtly wrong.
  switchkvm();

  return (void *)(KERNBASE + ((uint64)first << 30) + (pa - base));
}

// --- write-combining, and big buffers -----------------------------------------
//
// See include/vmap.h. The console used to draw straight into the framebuffer
// through the direct map, which is uncached across the whole PCI hole: every
// store was its own bus transaction and every read -- a scroll is a memmove,
// so a read for every byte on screen -- waited on the card. In QEMU that is
// free. On a real graphics card a single scroll took seconds, long enough
// that a second CPU waiting to print gave up and called it a deadlock.

#define MSR_PAT    0x277
#define PAT_WC     0x01
#define PDE_PAT    (1ULL << 12)    // in a 2 MB entry; PAT index 4 with PCD=PWT=0
#define PGSIZE_2M  (2ULL << 20)

void
pat_init(void)
{
  // Entry 4 is write-back at reset and nothing here sets the PAT bit, so
  // repurposing it changes no existing mapping. Entries 0-3, which every
  // other mapping selects through PCD and PWT, keep their reset values.
  uint64 pat = rdmsr(MSR_PAT);
  pat = (pat & ~(0xFFULL << 32)) | ((uint64)PAT_WC << 32);
  wrmsr(MSR_PAT, pat);
}

// A slot of the device window pointing at a fresh page directory.
static pde_t *
window_pd(int *slot)
{
  pde_t *pd;

  if (ioremap_next >= IOREMAP_LAST_PDPT)
    return 0;
  if ((pd = (pde_t *)kalloc()) == 0)
    return 0;
  memset(pd, 0, PGSIZE);
  *slot = ioremap_next++;
  kpdpt[*slot] = v2p(pd) | PTE_P | PTE_W;
  return pd;
}

void *
ioremap_wc(uint64 pa, uint64 size)
{
  uint64 base = pa & ~(uint64)(PGSIZE_2M - 1);
  uint64 n = (pa + size - base + PGSIZE_2M - 1) / PGSIZE_2M;
  pde_t *pd;
  int slot;

  if (size == 0 || n > 512 || (pd = window_pd(&slot)) == 0)
    return 0;
  // An uncached alias of the same frames is the direct map's 3-4 GB page.
  // Two uncached types for one device page is allowed; what the manuals
  // warn against is a cached alias, and there is none.
  for (uint64 i = 0; i < n; i++)
    pd[i] = (base + i * PGSIZE_2M) | PTE_P | PTE_W | PTE_PS | PDE_PAT;
  switchkvm();
  return (void *)(KERNBASE + ((uint64)slot << 30) + (pa - base));
}

void *
vmap_alloc(uint64 size)
{
  uint64 pages = (size + PGSIZE - 1) / PGSIZE;
  pde_t *pd;
  int slot;

  if (size == 0 || pages > 512 * 512 || (pd = window_pd(&slot)) == 0)
    return 0;
  for (uint64 i = 0; i < pages; i++) {
    pte_t *pt;
    char *page;
    if (pd[i / 512] == 0) {
      if ((pt = (pte_t *)kalloc()) == 0)
        goto oom;
      memset(pt, 0, PGSIZE);
      pd[i / 512] = v2p(pt) | PTE_P | PTE_W;
    }
    pt = (pte_t *)P2V(PTE_ADDR(pd[i / 512]));
    if ((page = kalloc()) == 0)
      goto oom;
    memset(page, 0, PGSIZE);
    pt[i % 512] = v2p(page) | PTE_P | PTE_W;
  }
  switchkvm();
  return (void *)(KERNBASE + ((uint64)slot << 30));

oom:
  // Whatever was mapped stays mapped and is lost. This runs once at boot,
  // and a machine out of memory by then has bigger problems.
  cprintf("vmap: out of memory for %d KB\n", (int)(size / 1024));
  return 0;
}

// CR3 loads per processor, and what each has loaded; see "Pages another
// processor may still have in its TLB" below.
static volatile uint64 cr3_loads[NCPU];
static pml4e_t *volatile cpu_pgdir[NCPU];

void
switchuvm(struct proc *p)
{
  pushcli();
  if(p->pgdir == 0)
    panic("switchuvm: no pgdir");
  void *tss = ioperm_tss_of(cpunum());
  const addr_t stktop = (addr_t)p->kstack + KSTACKSIZE;
  ioperm_set_rsp0(tss, stktop);

  // Which ports this process may reach. Cheap when nobody uses any, which is
  // every process that is not hosting a driver.
  ioperm_switch(p);

  cr3_loads[cpunum()]++;
  cpu_pgdir[cpunum()] = p->pgdir;
  lcr3(v2p(p->pgdir));
  popcli();
}

static pte_t *
walkpgdir(pml4e_t *pml4, const void *va, int alloc)
{
  pml4e_t *pml4e;
  pdpe_t *pdp, *pdpe;
  pde_t *pd, *pde;
  pte_t *pgtab;

  pml4e = &pml4[PMX(va)];
  if(*pml4e & PTE_P)
    pdp = (pdpe_t*)P2V(PTE_ADDR(*pml4e));
  else {
    if(!alloc || (pdp = (pdpe_t*)kalloc()) == 0)
      return 0;
    memset(pdp, 0, PGSIZE);
    *pml4e = V2P(pdp) | PTE_P | PTE_W | PTE_U;
  }

  pdpe = &pdp[PDPX(va)];
  if(*pdpe & PTE_P)
    pd = (pde_t*)P2V(PTE_ADDR(*pdpe));
  else {
    if(!alloc || (pd = (pde_t*)kalloc()) == 0)
      return 0;
    memset(pd, 0, PGSIZE);
    *pdpe = V2P(pd) | PTE_P | PTE_W | PTE_U;
  }

  pde = &pd[PDX(va)];
  if(*pde & PTE_P)
    pgtab = (pte_t*)P2V(PTE_ADDR(*pde));
  else {
    if(!alloc || (pgtab = (pte_t*)kalloc()) == 0)
      return 0;
    memset(pgtab, 0, PGSIZE);
    *pde = V2P(pgtab) | PTE_P | PTE_W | PTE_U;
  }

  return (pte_t*)&pgtab[PTX(va)];
}

// --- Pages another processor may still have in its TLB -----------------
//
// Threads put one address space on several processors at once, and there is
// no shootdown IPI: unmapping a page here invalidates it in this TLB only.
// Freed at once, it could be handed to someone else while a thread on another
// CPU still reaches it through a stale entry -- a write into a stranger's
// memory with no fault to show for it.
//
// So a page unmapped from a shared address space waits. Every CR3 load is
// counted per processor, and a batch of pages is freed once every processor
// has reloaded CR3 since the batch was closed, or is not running a user
// address space at all. A processor reloads at every switch, so a batch waits
// a scheduling slice or two.
#define DEFER_MAX 2048
struct defer_batch {
  int n;
  uint64 snap[NCPU];
  uint64 pa[DEFER_MAX];
};
static struct defer_batch defer_q[2];
static int defer_open;              // which of the two is filling
static int defer_closed_valid;      // the other one is waiting to be freed
static struct spinlock deferlock;
static int defer_init;

static int
defer_ready(struct defer_batch *b)
{
  for (int c = 0; c < ncpu; c++)
    if (cr3_loads[c] == b->snap[c] && cpu_pgdir[c] != 0)
      return 0;
  return 1;
}

// With deferlock. Frees the closed batch if it can, then closes the open one.
static void
defer_reclaim(void)
{
  struct defer_batch *closed = &defer_q[defer_open ^ 1];
  if (defer_closed_valid && defer_ready(closed)) {
    for (int i = 0; i < closed->n; i++)
      kfree((char*)P2V(closed->pa[i]));
    closed->n = 0;
    defer_closed_valid = 0;
  }
  struct defer_batch *open = &defer_q[defer_open];
  if (!defer_closed_valid && open->n > 0) {
    for (int c = 0; c < ncpu; c++)
      open->snap[c] = cr3_loads[c];
    defer_closed_valid = 1;
    defer_open ^= 1;
  }
}

void
vm_defer_tick(void)
{
  if (!defer_init)
    return;
  acquire(&deferlock);
  defer_reclaim();
  release(&deferlock);
}

static void
defer_free(uint64 pa)
{
  if (!defer_init) {
    initlock(&deferlock, "vmdefer");
    defer_init = 1;
  }
  for (;;) {
    acquire(&deferlock);
    struct defer_batch *open = &defer_q[defer_open];
    if (open->n == DEFER_MAX)
      defer_reclaim();
    open = &defer_q[defer_open];
    if (open->n < DEFER_MAX) {
      open->pa[open->n++] = pa;
      release(&deferlock);
      return;
    }
    release(&deferlock);
    yield();                        // both batches full: let the CPUs switch
  }
}

// Free a user page just unmapped from pgdir: now, or once no TLB can hold it.
static void
user_page_free(pml4e_t *pgdir, uint64 pa)
{
  struct proc *p = myproc();
  if (p && p->pgdir == pgdir && mm_shared(p->mm))
    defer_free(pa);
  else
    kfree((char*)P2V(pa));
}

void
switchkvm(void)
{
  cr3_loads[cpunum()]++;
  cpu_pgdir[cpunum()] = 0;
  lcr3(v2p(kpml4));
}

int
mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm)
{
  char *a, *last;
  pte_t *pte;

  a = (char*)PGROUNDDOWN((addr_t)va);
  last = (char*)PGROUNDDOWN(((addr_t)va) + size - 1);
  for(;;){
    if((pte = walkpgdir(pgdir, a, 1)) == 0)
      return -1;
    if(*pte & PTE_P)
      panic("remap");
    *pte = pa | perm | PTE_P | PTE_U;
    invlpg(a);
    if(a == last)
      break;
    a += PGSIZE;
    pa += PGSIZE;
  }
  return 0;
}

void
vm_unmap_range(pml4e_t *pgdir, uint64 va, uint64 length)
{
  uint64 a   = PGROUNDDOWN(va);
  uint64 end = PGROUNDUP(va + length);
  for (; a < end; a += PGSIZE) {
    pte_t *pte = walkpgdir(pgdir, (void*)a, 0);
    if (pte && (*pte & PTE_P)) {
      uint64 pa = PTE_ADDR(*pte);
      *pte = 0;
      invlpg((void*)a);
      user_page_free(pgdir, pa);
    }
  }
}

// Tear down a mapping without freeing what is underneath it.
//
// vm_unmap_range() kfree()s every page it unmaps, which is right for an
// anonymous mapping and catastrophic for anything shared: the framebuffer is
// device memory that was never allocated, and a shared memory segment's pages
// belong to the segment, not to whichever process happens to be detaching.
// Handing either of those to kfree puts them on the free list while another
// process is still reading and writing them.
void
vm_unmap_norelease(pml4e_t *pgdir, uint64 va, uint64 length)
{
  uint64 a   = PGROUNDDOWN(va);
  uint64 end = PGROUNDUP(va + length);
  for (; a < end; a += PGSIZE) {
    pte_t *pte = walkpgdir(pgdir, (void*)a, 0);
    if (pte && (*pte & PTE_P)) {
      *pte = 0;
      invlpg((void*)a);
    }
  }
}

// Duplicate an mmap()'d VMA region from src into dst for fork().
// Framebuffer mappings are re-pointed at the same physical MMIO range
// (shared, never copied/freed); anonymous mappings get fresh pages with
// the parent's contents copied in — mirrors what copyuvm() does for the
// [0, sz) heap range, which never covers mmap()'d addresses.
int
copy_vma_range(pml4e_t *dstpgdir, pml4e_t *srcpgdir, uint64 va, uint64 length, int share)
{
  uint64 a   = PGROUNDDOWN(va);
  uint64 end = PGROUNDUP(va + length);
  for (; a < end; a += PGSIZE) {
    if (share) {
      // Point the child at the very same physical pages, whatever they are.
      //
      // This used to special-case the framebuffer and map gfx_fb_phys()
      // explicitly. Reading the parent's own PTE instead covers device memory
      // and shared memory segments with one rule, and cannot disagree with
      // what the parent actually has mapped -- which the explicit version
      // could, and did, when the address was hardcoded.
      pte_t *spte = walkpgdir(srcpgdir, (void*)a, 0);
      if (!spte || !(*spte & PTE_P))
        continue;
      if (mappages(dstpgdir, (void*)a, PGSIZE, PTE_ADDR(*spte), PTE_FLAGS(*spte)) < 0)
        return -1;
      continue;
    }
    pte_t *spte = walkpgdir(srcpgdir, (void*)a, 0);
    if (!spte || !(*spte & PTE_P))
      continue;
    char *mem = kalloc();
    if (!mem)
      return -1;
    memmove(mem, (char*)P2V(PTE_ADDR(*spte)), PGSIZE);
    if (mappages(dstpgdir, (void*)a, PGSIZE, V2P(mem), PTE_FLAGS(*spte)) < 0) {
      kfree(mem);
      return -1;
    }
  }
  return 0;
}

void
inituvm(pde_t *pgdir, char *sinit, uint64 sz)
{
  char *mem;

  if(sz >= PGSIZE)
    panic("inituvm: more than a page");
  mem = kalloc(); 
  memset(mem, 0, PGSIZE);
  mappages(pgdir, 0, PGSIZE, v2p(mem), PTE_W|PTE_U);
  memmove(mem, sinit, sz); 
}

int
loaduvm(pml4e_t *pgdir, char *addr, struct inode *ip, uint64 offset, uint64 sz)
{
  uint64 done, n, off_in_page;
  addr_t pa;
  pte_t *pte;
  char *cur_page;

  for(done = 0; done < sz; done += n){
    cur_page   = (char*)PGROUNDDOWN((uint64)(addr + done));
    off_in_page = (uint64)(addr + done) - (uint64)cur_page;
    if((pte = walkpgdir(pgdir, cur_page, 0)) == 0)
      panic("loaduvm: address should exist");
    pa = PTE_ADDR(*pte);
    n = PGSIZE - off_in_page;
    if(n > sz - done)
      n = sz - done;
    if(readi(ip, 0, (uint64)(P2V(pa) + off_in_page), offset + done, n) != (int)n)
      return -1;
  }
  return 0;
}

uint64
allocuvm(pml4e_t *pgdir, uint64 oldsz, uint64 newsz)
{
  char *mem;
  addr_t a;

  if(newsz >= KERNBASE)
    return 0;
  if(newsz < oldsz)
    return oldsz;

  a = PGROUNDUP(oldsz);
  for(; a < newsz; a += PGSIZE){
    mem = kalloc();
    if(mem == 0){
      deallocuvm(pgdir, newsz, oldsz);
      return 0;
    }
    memset(mem, 0, PGSIZE);
    if(mappages(pgdir, (char*)a, PGSIZE, V2P(mem), PTE_W|PTE_U) < 0){
      deallocuvm(pgdir, newsz, oldsz);
      kfree(mem);
      return 0;
    }
  }
  return newsz;
}

uint64
deallocuvm(pml4e_t *pgdir, uint64 oldsz, uint64 newsz)
{
  pte_t *pte;
  addr_t a, pa;

  if(newsz >= oldsz)
    return oldsz;

  a = PGROUNDUP(newsz);
  for(; a  < oldsz; a += PGSIZE){
    pte = walkpgdir(pgdir, (char*)a, 0);
    if(pte && (*pte & PTE_P) != 0){
      pa = PTE_ADDR(*pte);
      if(pa == 0)
        panic("kfree");
      *pte = 0;
      invlpg((void*)a);
      if(pa < PHYSTOP)
        user_page_free(pgdir, pa);
    }
  }
  return newsz;
}

// Walk every page a process has mapped in the low half, calling back for each.
//
// Written for `whose` (docs/architecture/debugging.md): answering "which
// process has this physical page mapped" means a reverse lookup, and there is
// no reverse map in this kernel -- so the only way is forward, over every
// entry. That is O(pages mapped), a few thousand iterations for a real
// process, which is nothing for a command somebody typed and would be
// unacceptable anywhere else.
//
// The low half only. The kernel half is the direct map and is identical in
// every address space, so reporting it per process would be noise: every
// physical page is mapped there, always, and saying so once is the useful
// form of that fact.
void
vm_walk_user(pml4e_t *pml4, void (*fn)(void *ctx, uint64 va, uint64 pa, uint64 pte),
             void *ctx)
{
  uint i, j, k, l;
  pdpe_t *pdp;
  pde_t *pd;
  pte_t *pt;

  if (pml4 == 0)
    return;

  for (i = 0; i < (512/2); i++) {
    if (!(pml4[i] & PTE_P))
      continue;
    pdp = (pdpe_t*)P2V(PTE_ADDR(pml4[i]));
    for (j = 0; j < 512; j++) {
      if (!(pdp[j] & PTE_P))
        continue;
      pd = (pde_t*)P2V(PTE_ADDR(pdp[j]));
      for (k = 0; k < 512; k++) {
        if (!(pd[k] & PTE_P))
          continue;
        pt = (pte_t*)P2V(PTE_ADDR(pd[k]));
        for (l = 0; l < 512; l++) {
          if (!(pt[l] & PTE_P))
            continue;
          fn(ctx,
             ((uint64)i << 39) | ((uint64)j << 30) | ((uint64)k << 21) |
                 ((uint64)l << 12),
             PTE_ADDR(pt[l]), pt[l]);
        }
      }
    }
  }
}

void
freevm(pml4e_t *pml4)
{
  uint i, j, k, l;
  pdpe_t *pdp;
  pde_t *pd;
  pte_t *pt;

  if(pml4 == 0)
    panic("freevm: no pgdir");

  for(i = 0; i < (512/2); i++){
    if(pml4[i] & PTE_P){
      pdp = (pdpe_t*)P2V(PTE_ADDR(pml4[i]));
      for(j = 0; j < 512; j++){
        if(pdp[j] & PTE_P){
          pd = (pde_t*)P2V(PTE_ADDR(pdp[j]));
          for(k = 0; k < 512; k++){
            if(pd[k] & PTE_P) {
              pt = (pte_t*)P2V(PTE_ADDR(pd[k]));
              for(l = 0; l < 512; l++){
                if(pt[l] & PTE_P) {
                  addr_t pa = PTE_ADDR(pt[l]);
                  if(pa < PHYSTOP && pa != 0) {
                    kfree(P2V(pa));
                  }
                }
              }
              kfree((char*)pt);
            }
          }
          kfree((char*)pd);
        }
      }
      kfree((char*)pdp);
    }
  }
  kfree((char*)pml4);
}

void
clearpteu(pml4e_t *pgdir, char *uva)
{
  pte_t *pte;
  pte = walkpgdir(pgdir, uva, 0);
  if(pte == 0)
    panic("clearpteu");
  *pte &= ~PTE_U;
}

pml4e_t*
copyuvm(pml4e_t *pgdir, uint64 sz)
{
  pml4e_t *d;
  pte_t *pte;
  addr_t pa, i, flags;
  char *mem;

  if((d = setupkvm()) == 0)
    return 0;
  for(i = 0; i < sz; i += PGSIZE){
    if((pte = walkpgdir(pgdir, (void*)i, 0)) == 0)
      continue;
    if(!(*pte & PTE_P))
      continue;
    pa = PTE_ADDR(*pte);
    flags = PTE_FLAGS(*pte);

    if(pa < PHYSTOP) {
      if((mem = kalloc()) == 0)
        goto bad;
      memmove(mem, (char*)P2V(pa), PGSIZE);
      if(mappages(d, (void*)i, PGSIZE, V2P(mem), flags) < 0) {
        kfree(mem);
        goto bad;
      }
    }
  }
  return d;

  bad:
  freevm(d);
  return 0;
}

char*
uva2ka(pml4e_t *pgdir, char *uva)
{
  if((addr_t)uva >= KERNBASE)
    return 0;

  pte_t *pte;
  pte = walkpgdir(pgdir, uva, 0);
  if(pte == 0 || (*pte & PTE_P) == 0)
    return 0;
  if((*pte & PTE_U) == 0)
    return 0;
  addr_t pa = (*pte) & 0x000FFFFFFFFFF000ull;
  // Include uva's in-page offset — callers dereference the returned pointer
  // directly at uva's address, not just somewhere within its page.
  return (char*)P2V(pa) + ((addr_t)uva & 0xFFF);
}

int
copyout(pml4e_t *pgdir, addr_t va, void *p, uint64 len)
{
  char *buf, *pa0;
  addr_t n, va0;

  buf = (char*)p;
  while(len > 0){
    va0 = PGROUNDDOWN(va);
    pa0 = uva2ka(pgdir, (char*)va0);

    if(pa0 == 0) {
      return -1;
    }

    n = PGSIZE - (va - va0);
    if(n > len)
      n = len;
    memmove(pa0 + (va - va0), buf, n);

    len -= n;
    buf += n;
    va += n;
  }
  return 0;
}

int
copyin(pml4e_t *pgdir, addr_t srcva, void *dst, uint64 len)
{
  uint64 n, va0;
  char *pa0;
  char *dst_buf = (char*)dst;

  while(len > 0){
    va0 = PGROUNDDOWN(srcva);
    pa0 = uva2ka(pgdir, (char*)va0);
    if(pa0 == 0)
      return -1;
    n = PGSIZE - (srcva - va0);
    if(n > len)
      n = len;
    memmove(dst_buf, pa0 + (srcva - va0), n);
    len -= n;
    dst_buf += n;
    srcva += n;
  }
  return 0;
}