SchOS/kernel/arch/x86_64/vm.c

#include "param.h"
#include "types.h"
#include "defs.h"
#include "gfx.h"
#include "x86.h"
#include "memlayout.h"
#include "mmu.h"
#include "proc.h"
#include "ioperm.h"

int mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm);
static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc);


// `cpu` and `proc` are no longer __thread variables; they live at fixed offsets
// in the per-CPU block and are read through %gs at each use. See the long
// comment in include/proc.h for why -- the short version is that __thread means
// per *thread*, the compiler caches the thread pointer across calls on that
// basis, and a kernel thread changes CPU at every sched().

// Kernel FS_BASE set by seginit(); read by trapasm.S to restore after user TLS
uint64 kernel_fs_base;

// The per-CPU block's layout is shared with trapasm.S, which spells the offsets
// as literals because there is no way to include a C header from there. These
// pin the C side: if a slot moves, the assembly keeps reading the old address
// and the failure is a register quietly holding somebody else's value, which is
// the hardest possible thing to find. Ask the question at compile time instead.
_Static_assert(PERCPU_SELF == 0 && PERCPU_SCRATCH == 8 && PERCPU_CPU == 16 &&
               PERCPU_PROC == 24 && PERCPU_SC_RAX == 32 &&
               PERCPU_SC_RCX == 40 && PERCPU_SC_RDX == 48,
               "per-CPU block layout changed -- update kernel/arch/x86_64/trapasm.S");

static pml4e_t *kpml4;
// Where the device windows begin is now decided by kvmalloc(), which knows how
// much of the address space the direct map took. The constant that used to be
// here said 4, and would have handed out windows on top of RAM the moment the
// direct map grew past four gigabytes.
#define IOREMAP_LAST_PDPT  511
#define GIGABYTE           (1ULL << 30)

// The first PDPT entry the direct map did not take. Set by kvmalloc().
static int ioremap_next = 4;

static pdpe_t *kpdpt;

struct tassdesc {
  uint lim_15_0 : 16;  
  uint base_15_0 : 16; 
  uint base_23_16 : 8; 
  uint type : 4;       
  uint s : 1;          
  uint dpl : 2;        
  uint p : 1;          
  uint lim_19_16 : 4;  
  uint avl : 1;        
  uint l : 1;          
  uint d : 1;          
  uint g : 1;          
  uint base_31_24 : 8; 
  uint32 base_63_32;   
  uint32 res;          
};

void
syscallinit(void)
{
  // SYSRETQ does NOT use USER_CS/USER_DS directly. Per the AMD64/Intel
  // convention, it computes CS = STAR[63:48]+16 and SS = STAR[63:48]+8
  // (RPL forced to 3), so STAR[63:48] must hold the *compat* 32-bit user
  // code selector (SEG_UCODE32), relying on the GDT's consecutive
  // [UCODE32, UDATA, UCODE] layout — NOT USER_CS itself. Using USER_CS
  // here made every sysretq load CS=USER_CS+16 (an invalid selector,
  // pointing past UCODE into the TSS descriptor) and SS=USER_CS+8. QEMU's
  // TCG apparently doesn't validate segment descriptors on SYSRET, so this
  // went unnoticed until an interrupt forced a strictly-validated iretq
  // with that same bogus CS still loaded, which correctly #GP'd.
  wrmsr(MSR_STAR,
    ((((uint64)((SEG_UCODE32 << 3) | DPL_USER) << 16) | (uint64)KERNEL_CS) << 32));
  wrmsr(MSR_LSTAR, (addr_t)syscall_entry);
  wrmsr(MSR_CSTAR, (addr_t)ignore_sysret);
  wrmsr(MSR_SFMASK, FL_TF|FL_DF|FL_IF|FL_IOPL_MASK|FL_AC|FL_NT);
}

void
seginit(void)
{
  struct segdesc *gdt;
  uint *tss;
  uint64 addr, tsslim;
  void *local;
  struct cpu *c;

  // The per-CPU page, preallocated by the boot processor for every AP.
  //
  // It must not be kalloc()'d here on an application processor. kalloc takes a
  // spinlock; acquire() calls holding(); holding() reads `cpu`, which is a
  // __thread variable reached through %fs -- and %fs is set up four lines
  // below. So an AP allocating its own page reads the per-CPU pointer before
  // the per-CPU pointer exists, and the kernel dies in panic("acquire") with
  // every lock appearing to be held.
  //
  // startothers() fills c->local in before starting each AP, on the boot
  // processor, where locking works.
  c = &cpus[cpunum()];
  local = c->local;
  if (local == 0) {
    local = kalloc();          // the boot processor's own, taken normally
    memset(local, 0, PGSIZE);
  }

  gdt = (struct segdesc*) local;

  // The TSS moved out of this page. It now carries an I/O permission bitmap
  // (include/ioperm.h) which does not fit here, and it has to be one object
  // with it. ioperm_tss() returns it and the descriptor limit that reaches the
  // end of the bitmap.
  tss = (uint*) ioperm_tss(cpunum(), &tsslim);

  kernel_fs_base = ((uint64) local) + 2048;
  wrmsr(0xC0000100, kernel_fs_base);

  // The TCB self-pointer: the word at %fs:0 must hold the thread pointer.
  //
  // This is part of the x86-64 TLS ABI and it was missing, which worked only
  // by accident. GCC with -mtls-direct-seg-refs compiles `cpu = c` into a
  // single `movq %rcx, %fs:-0x10` and never needs the thread pointer as a
  // value, so nothing ever read this word. clang materialises it instead --
  // `movq %fs:0x0, %rax; addq $-0x10, %rax; movq %rcx, (%rax)` -- which read
  // the zero left by the memset above and stored through address
  // 0xfffffffffffffff0, three instructions later.
  //
  // Writing it costs one store per CPU and removes the dependency on a
  // particular compiler's TLS optimisation.
  *(uint64 *) kernel_fs_base = kernel_fs_base;

  // Clear the per-CPU slots before anything can read them. Their offsets are
  // the PERCPU_* constants in include/proc.h and are shared with trapasm.S,
  // which reads %gs:24 for proc on the way out to user mode and parks three
  // registers at %gs:32..48 across the FS_BASE swap on the way in.
  ((uint64 *) kernel_fs_base)[2] = 0;      // struct cpu *   (PERCPU_CPU)
  ((uint64 *) kernel_fs_base)[3] = 0;      // struct proc *  (PERCPU_PROC)
  ((uint64 *) kernel_fs_base)[4] = 0;      // syscall rax    (PERCPU_SC_RAX)
  ((uint64 *) kernel_fs_base)[5] = 0;      // syscall rcx    (PERCPU_SC_RCX)
  ((uint64 *) kernel_fs_base)[6] = 0;      // syscall rdx    (PERCPU_SC_RDX)

  // The same address in GS base, per CPU.
  //
  // trapasm.S needs to find *this* CPU's kernel FS base on every entry from
  // user mode, and it has no register to work from at that instant -- which is
  // why it used to read the kernel_fs_base global. That global is written by
  // every CPU during its own seginit(), so with more than one processor the
  // last writer wins and every CPU restores one CPU's per-CPU block. GS base is
  // per-CPU in hardware, so %gs:0 answers the question with no global at all.
  //
  // Safe because GS belongs entirely to the kernel here: sys_arch_prctl()
  // implements ARCH_SET_FS and ARCH_GET_FS only, and musl uses FS for TLS on
  // x86-64. See the comment on RESTORE_KERNEL_FS.
  wrmsr(0xC0000101, kernel_fs_base);      // IA32_GS_BASE

  c->local = local;

  cpu_set(c);
  proc_set(0);

  // Two of the entries below are deliberately null descriptors, and neither is
  // dead code even though both look it:
  //
  //   SEG_UCODE32  is never loaded, but SYSRETQ computes CS from STAR[63:48]+16
  //                and SS from +8, so the *slot* must exist to keep the
  //                [UCODE32, UDATA, UCODE] triple consecutive. See syscallinit.
  //   SEG_KCPU     is a leftover slot number from 32-bit xv6, where %gs held the
  //                per-CPU pointer. SchOS uses FS_BASE/TLS instead, but the
  //                index has to stay occupied or SEG_TSS stops landing at 7.
  //
  // Deleting either moves every descriptor after it and breaks sysret.
  addr = (uint64) tss;
  gdt[0] = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
  gdt[SEG_KCODE] = SEG(STA_X|STA_R, 0, 0, 0);
  gdt[SEG_KDATA] = SEG(STA_W, 0, 0, 0);
  gdt[SEG_UCODE32] = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
  gdt[SEG_UDATA] = SEG(STA_W, 0, 0, DPL_USER);
  gdt[SEG_UCODE] = SEG(STA_X|STA_R, 0, 0, DPL_USER);
  gdt[SEG_KCPU]  = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
  
  struct tassdesc *td = (struct tassdesc*)(gdt + SEG_TSS);
  td->lim_15_0 = tsslim & 0xffff;
  td->base_15_0 = addr & 0xffff;
  td->base_23_16 = (addr >> 16) & 0xff;
  td->type = STS_T64A;
  td->s = 0;
  td->dpl = 0;
  td->p = 1;
  td->lim_19_16 = (tsslim >> 16) & 0xf;
  td->avl = 0;
  td->l = 0;
  td->d = 0;
  td->g = 0;
  td->base_31_24 = (addr >> 24) & 0xff;
  td->base_63_32 = (addr >> 32) & 0xffffffff;
  td->res = 0;

  lgdt((void*) gdt, (NSEGS+1) * sizeof(struct segdesc));
  ltr(SEG_TSS << 3);
}

pml4e_t*
setupkvm(void)
{
  pml4e_t *pml4 = (pml4e_t*) kalloc();
  if(pml4 == 0) return 0;
  memset(pml4, 0, PGSIZE);

  pml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W;

  // The framebuffer is deliberately NOT mapped here.
  //
  // This used to identity-map 0xfd000000 for 800*600*4 bytes, user-writable,
  // into every process that ever existed. Three things were wrong with that.
  // The address and the size were both guesses that only hold on QEMU's
  // stdvga at one resolution; setupkvm runs before vga_init, so on the first
  // call nothing had even read the VBE block yet; and every process could
  // scribble on the screen without asking anyone.
  //
  // Nothing used it -- userspace reaches the framebuffer through
  // mmap("/dev/fb0"), which maps the real base at the real size. So it is
  // gone, and a process gets the framebuffer exactly when it asks for it.

  return pml4;
}

void
kvmalloc(void)
{
  kpml4 = (pml4e_t*) kalloc();
  memset(kpml4, 0, PGSIZE);

  kpdpt = (pdpe_t*) kalloc();
  memset(kpdpt, 0, PGSIZE);

  kpml4[0] = v2p(kpdpt) | PTE_P | PTE_W;
  kpml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W;
  
  // As many gigabytes as the machine has RAM, not four.
  //
  // Four was the number for as long as PHYSTOP was capped at 3 GB, and the cap
  // existed because this loop stopped at four -- each holding the other up. A
  // laptop with sixteen gigabytes lost thirteen of them to that circle.
  //
  // The 3-4 GB entry stays uncached because that is where the PCI hole is and
  // always has been. Everything above 4 GB is RAM -- the hole is below it on
  // every machine this runs on -- so those are cached, and a device BAR up
  // there still gets its own uncached window from ioremap().
  uint64 gigs = (phystop + GIGABYTE - 1) / GIGABYTE;
  if (gigs < 4)
    gigs = 4;                       // always map the low four
  if (gigs > IOREMAP_LAST_PDPT - 8)
    gigs = IOREMAP_LAST_PDPT - 8;   // leave room for device windows

  for(uint64 i = 0; i < gigs; i++) {
    uint64 flags = PTE_PS | PTE_P | PTE_W;
    if(i == 3) {
      flags |= PTE_PCD | PTE_PWT;
    }
    kpdpt[i] = (i << 30) | flags;
  }

  // ioremap takes what is left. It used to start at a constant 4, which was
  // correct only while the direct map was exactly four gigabytes.
  ioremap_next = (int)gigs;

  switchkvm();
}

// Map a device's registers so the kernel can reach them.
//
// Every driver used to say P2V(bar.base), which quietly assumes the BAR lands
// inside the 0-4 GB the loop above maps. Under SeaBIOS it always does. Under
// OVMF it does not: firmware is free to put a 64-bit BAR anywhere, and it puts
// virtio-gpu's at about 768 GB. P2V then produces an address with no
// translation behind it, and the driver's first register write takes a page
// fault -- which, coming from a driver bind, looks like broken hardware rather
// than a missing mapping.
//
// The direct map uses PDPT entries 0-3 of a page directory pointer table that
// kpml4[0], kpml4[KERNBASE] and every process's PML4 (see setupkvm) all share.
// Entries 4-511 are free, so each one can carry a 1 GB page of device memory:
// no page table to allocate, nothing to free, and the mapping appears in every
// address space at once because they all point at this one table.
//
// Cache-disabled and write-through, for the same reason the top GB of the
// direct map is: these are registers, and a write that sits in a cache line
// waiting for company is a write the device never sees.

void *
ioremap(uint64 pa, uint64 size)
{
  if (size == 0)
    return 0;

  // Below 4 GB the direct map already covers it, so hand back the same address
  // the old code would have produced. This is what keeps the BIOS path -- where
  // every BAR is low -- byte for byte what it was.
  if (pa + size <= 4 * GIGABYTE)
    return P2V(pa);

  uint64 base = pa & ~(GIGABYTE - 1);          // 1 GB pages, so align to one
  uint64 last = (pa + size - 1) & ~(GIGABYTE - 1);
  int    span = (int)((last - base) / GIGABYTE) + 1;

  // Already mapped? A second BAR in the same gigabyte should not consume a
  // second window, and drivers do re-map the same device.
  for (int i = 4; i < ioremap_next; i++) {
    // Clearing the low 30 bits drops every flag as well, leaving the frame.
    if ((kpdpt[i] & PTE_P) && (kpdpt[i] & ~(GIGABYTE - 1)) == base)
      return (void *)(KERNBASE + ((uint64)i << 30) + (pa - base));
  }

  if (ioremap_next + span > IOREMAP_LAST_PDPT) {
    cprintf("ioremap: out of device window for %p\n", (void *)pa);
    return 0;
  }

  int first = ioremap_next;
  for (int i = 0; i < span; i++) {
    kpdpt[ioremap_next] = (base + (uint64)i * GIGABYTE)
                        | PTE_PS | PTE_P | PTE_W | PTE_PCD | PTE_PWT;
    ioremap_next++;
  }

  // The entries were not present a moment ago. Reloading CR3 is heavier than
  // invlpg but this runs a handful of times at boot, and it cannot be the thing
  // that is subtly wrong.
  switchkvm();

  return (void *)(KERNBASE + ((uint64)first << 30) + (pa - base));
}

void
switchuvm(struct proc *p)
{
  pushcli();
  if(p->pgdir == 0)
    panic("switchuvm: no pgdir");
  void *tss = ioperm_tss_of(cpunum());
  const addr_t stktop = (addr_t)p->kstack + KSTACKSIZE;
  ioperm_set_rsp0(tss, stktop);

  // Which ports this process may reach. Cheap when nobody uses any, which is
  // every process that is not hosting a driver.
  ioperm_switch(p);

  lcr3(v2p(p->pgdir));
  popcli();
}

static pte_t *
walkpgdir(pml4e_t *pml4, const void *va, int alloc)
{
  pml4e_t *pml4e;
  pdpe_t *pdp, *pdpe;
  pde_t *pd, *pde;
  pte_t *pgtab;

  pml4e = &pml4[PMX(va)];
  if(*pml4e & PTE_P)
    pdp = (pdpe_t*)P2V(PTE_ADDR(*pml4e));
  else {
    if(!alloc || (pdp = (pdpe_t*)kalloc()) == 0)
      return 0;
    memset(pdp, 0, PGSIZE);
    *pml4e = V2P(pdp) | PTE_P | PTE_W | PTE_U;
  }

  pdpe = &pdp[PDPX(va)];
  if(*pdpe & PTE_P)
    pd = (pde_t*)P2V(PTE_ADDR(*pdpe));
  else {
    if(!alloc || (pd = (pde_t*)kalloc()) == 0)
      return 0;
    memset(pd, 0, PGSIZE);
    *pdpe = V2P(pd) | PTE_P | PTE_W | PTE_U;
  }

  pde = &pd[PDX(va)];
  if(*pde & PTE_P)
    pgtab = (pte_t*)P2V(PTE_ADDR(*pde));
  else {
    if(!alloc || (pgtab = (pte_t*)kalloc()) == 0)
      return 0;
    memset(pgtab, 0, PGSIZE);
    *pde = V2P(pgtab) | PTE_P | PTE_W | PTE_U;
  }

  return (pte_t*)&pgtab[PTX(va)];
}

void
switchkvm(void)
{
  lcr3(v2p(kpml4));
}

int
mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm)
{
  char *a, *last;
  pte_t *pte;

  a = (char*)PGROUNDDOWN((addr_t)va);
  last = (char*)PGROUNDDOWN(((addr_t)va) + size - 1);
  for(;;){
    if((pte = walkpgdir(pgdir, a, 1)) == 0)
      return -1;
    if(*pte & PTE_P)
      panic("remap");
    *pte = pa | perm | PTE_P | PTE_U;
    invlpg(a);
    if(a == last)
      break;
    a += PGSIZE;
    pa += PGSIZE;
  }
  return 0;
}

void
vm_unmap_range(pml4e_t *pgdir, uint64 va, uint64 length)
{
  uint64 a   = PGROUNDDOWN(va);
  uint64 end = PGROUNDUP(va + length);
  for (; a < end; a += PGSIZE) {
    pte_t *pte = walkpgdir(pgdir, (void*)a, 0);
    if (pte && (*pte & PTE_P)) {
      uint64 pa = PTE_ADDR(*pte);
      kfree((char*)P2V(pa));
      *pte = 0;
      invlpg((void*)a);
    }
  }
}

// Tear down a mapping without freeing what is underneath it.
//
// vm_unmap_range() kfree()s every page it unmaps, which is right for an
// anonymous mapping and catastrophic for anything shared: the framebuffer is
// device memory that was never allocated, and a shared memory segment's pages
// belong to the segment, not to whichever process happens to be detaching.
// Handing either of those to kfree puts them on the free list while another
// process is still reading and writing them.
void
vm_unmap_norelease(pml4e_t *pgdir, uint64 va, uint64 length)
{
  uint64 a   = PGROUNDDOWN(va);
  uint64 end = PGROUNDUP(va + length);
  for (; a < end; a += PGSIZE) {
    pte_t *pte = walkpgdir(pgdir, (void*)a, 0);
    if (pte && (*pte & PTE_P)) {
      *pte = 0;
      invlpg((void*)a);
    }
  }
}

// Duplicate an mmap()'d VMA region from src into dst for fork().
// Framebuffer mappings are re-pointed at the same physical MMIO range
// (shared, never copied/freed); anonymous mappings get fresh pages with
// the parent's contents copied in — mirrors what copyuvm() does for the
// [0, sz) heap range, which never covers mmap()'d addresses.
int
copy_vma_range(pml4e_t *dstpgdir, pml4e_t *srcpgdir, uint64 va, uint64 length, int share)
{
  uint64 a   = PGROUNDDOWN(va);
  uint64 end = PGROUNDUP(va + length);
  for (; a < end; a += PGSIZE) {
    if (share) {
      // Point the child at the very same physical pages, whatever they are.
      //
      // This used to special-case the framebuffer and map gfx_fb_phys()
      // explicitly. Reading the parent's own PTE instead covers device memory
      // and shared memory segments with one rule, and cannot disagree with
      // what the parent actually has mapped -- which the explicit version
      // could, and did, when the address was hardcoded.
      pte_t *spte = walkpgdir(srcpgdir, (void*)a, 0);
      if (!spte || !(*spte & PTE_P))
        continue;
      if (mappages(dstpgdir, (void*)a, PGSIZE, PTE_ADDR(*spte), PTE_FLAGS(*spte)) < 0)
        return -1;
      continue;
    }
    pte_t *spte = walkpgdir(srcpgdir, (void*)a, 0);
    if (!spte || !(*spte & PTE_P))
      continue;
    char *mem = kalloc();
    if (!mem)
      return -1;
    memmove(mem, (char*)P2V(PTE_ADDR(*spte)), PGSIZE);
    if (mappages(dstpgdir, (void*)a, PGSIZE, V2P(mem), PTE_FLAGS(*spte)) < 0) {
      kfree(mem);
      return -1;
    }
  }
  return 0;
}

void
inituvm(pde_t *pgdir, char *sinit, uint64 sz)
{
  char *mem;

  if(sz >= PGSIZE)
    panic("inituvm: more than a page");
  mem = kalloc(); 
  memset(mem, 0, PGSIZE);
  mappages(pgdir, 0, PGSIZE, v2p(mem), PTE_W|PTE_U);
  memmove(mem, sinit, sz); 
}

int
loaduvm(pml4e_t *pgdir, char *addr, struct inode *ip, uint64 offset, uint64 sz)
{
  uint64 done, n, off_in_page;
  addr_t pa;
  pte_t *pte;
  char *cur_page;

  for(done = 0; done < sz; done += n){
    cur_page   = (char*)PGROUNDDOWN((uint64)(addr + done));
    off_in_page = (uint64)(addr + done) - (uint64)cur_page;
    if((pte = walkpgdir(pgdir, cur_page, 0)) == 0)
      panic("loaduvm: address should exist");
    pa = PTE_ADDR(*pte);
    n = PGSIZE - off_in_page;
    if(n > sz - done)
      n = sz - done;
    if(readi(ip, 0, (uint64)(P2V(pa) + off_in_page), offset + done, n) != (int)n)
      return -1;
  }
  return 0;
}

uint64
allocuvm(pml4e_t *pgdir, uint64 oldsz, uint64 newsz)
{
  char *mem;
  addr_t a;

  if(newsz >= KERNBASE)
    return 0;
  if(newsz < oldsz)
    return oldsz;

  a = PGROUNDUP(oldsz);
  for(; a < newsz; a += PGSIZE){
    mem = kalloc();
    if(mem == 0){
      deallocuvm(pgdir, newsz, oldsz);
      return 0;
    }
    memset(mem, 0, PGSIZE);
    if(mappages(pgdir, (char*)a, PGSIZE, V2P(mem), PTE_W|PTE_U) < 0){
      deallocuvm(pgdir, newsz, oldsz);
      kfree(mem);
      return 0;
    }
  }
  return newsz;
}

uint64
deallocuvm(pml4e_t *pgdir, uint64 oldsz, uint64 newsz)
{
  pte_t *pte;
  addr_t a, pa;

  if(newsz >= oldsz)
    return oldsz;

  a = PGROUNDUP(newsz);
  for(; a  < oldsz; a += PGSIZE){
    pte = walkpgdir(pgdir, (char*)a, 0);
    if(pte && (*pte & PTE_P) != 0){
      pa = PTE_ADDR(*pte);
      if(pa == 0)
        panic("kfree");
      if(pa < PHYSTOP) {
        char *v = P2V(pa);
        kfree(v);
      }
      *pte = 0;
    }
  }
  return newsz;
}

// Walk every page a process has mapped in the low half, calling back for each.
//
// Written for `whose` (docs/architecture/debugging.md): answering "which
// process has this physical page mapped" means a reverse lookup, and there is
// no reverse map in this kernel -- so the only way is forward, over every
// entry. That is O(pages mapped), a few thousand iterations for a real
// process, which is nothing for a command somebody typed and would be
// unacceptable anywhere else.
//
// The low half only. The kernel half is the direct map and is identical in
// every address space, so reporting it per process would be noise: every
// physical page is mapped there, always, and saying so once is the useful
// form of that fact.
void
vm_walk_user(pml4e_t *pml4, void (*fn)(void *ctx, uint64 va, uint64 pa, uint64 pte),
             void *ctx)
{
  uint i, j, k, l;
  pdpe_t *pdp;
  pde_t *pd;
  pte_t *pt;

  if (pml4 == 0)
    return;

  for (i = 0; i < (512/2); i++) {
    if (!(pml4[i] & PTE_P))
      continue;
    pdp = (pdpe_t*)P2V(PTE_ADDR(pml4[i]));
    for (j = 0; j < 512; j++) {
      if (!(pdp[j] & PTE_P))
        continue;
      pd = (pde_t*)P2V(PTE_ADDR(pdp[j]));
      for (k = 0; k < 512; k++) {
        if (!(pd[k] & PTE_P))
          continue;
        pt = (pte_t*)P2V(PTE_ADDR(pd[k]));
        for (l = 0; l < 512; l++) {
          if (!(pt[l] & PTE_P))
            continue;
          fn(ctx,
             ((uint64)i << 39) | ((uint64)j << 30) | ((uint64)k << 21) |
                 ((uint64)l << 12),
             PTE_ADDR(pt[l]), pt[l]);
        }
      }
    }
  }
}

void
freevm(pml4e_t *pml4)
{
  uint i, j, k, l;
  pdpe_t *pdp;
  pde_t *pd;
  pte_t *pt;

  if(pml4 == 0)
    panic("freevm: no pgdir");

  for(i = 0; i < (512/2); i++){
    if(pml4[i] & PTE_P){
      pdp = (pdpe_t*)P2V(PTE_ADDR(pml4[i]));
      for(j = 0; j < 512; j++){
        if(pdp[j] & PTE_P){
          pd = (pde_t*)P2V(PTE_ADDR(pdp[j]));
          for(k = 0; k < 512; k++){
            if(pd[k] & PTE_P) {
              pt = (pte_t*)P2V(PTE_ADDR(pd[k]));
              for(l = 0; l < 512; l++){
                if(pt[l] & PTE_P) {
                  addr_t pa = PTE_ADDR(pt[l]);
                  if(pa < PHYSTOP && pa != 0) {
                    kfree(P2V(pa));
                  }
                }
              }
              kfree((char*)pt);
            }
          }
          kfree((char*)pd);
        }
      }
      kfree((char*)pdp);
    }
  }
  kfree((char*)pml4);
}

void
clearpteu(pml4e_t *pgdir, char *uva)
{
  pte_t *pte;
  pte = walkpgdir(pgdir, uva, 0);
  if(pte == 0)
    panic("clearpteu");
  *pte &= ~PTE_U;
}

pml4e_t*
copyuvm(pml4e_t *pgdir, uint64 sz)
{
  pml4e_t *d;
  pte_t *pte;
  addr_t pa, i, flags;
  char *mem;

  if((d = setupkvm()) == 0)
    return 0;
  for(i = 0; i < sz; i += PGSIZE){
    if((pte = walkpgdir(pgdir, (void*)i, 0)) == 0)
      continue;
    if(!(*pte & PTE_P))
      continue;
    pa = PTE_ADDR(*pte);
    flags = PTE_FLAGS(*pte);

    if(pa < PHYSTOP) {
      if((mem = kalloc()) == 0)
        goto bad;
      memmove(mem, (char*)P2V(pa), PGSIZE);
      if(mappages(d, (void*)i, PGSIZE, V2P(mem), flags) < 0) {
        kfree(mem);
        goto bad;
      }
    }
  }
  return d;

  bad:
  freevm(d);
  return 0;
}

char*
uva2ka(pml4e_t *pgdir, char *uva)
{
  if((addr_t)uva >= KERNBASE)
    return 0;

  pte_t *pte;
  pte = walkpgdir(pgdir, uva, 0);
  if(pte == 0 || (*pte & PTE_P) == 0)
    return 0;
  if((*pte & PTE_U) == 0)
    return 0;
  addr_t pa = (*pte) & 0x000FFFFFFFFFF000ull;
  // Include uva's in-page offset — callers dereference the returned pointer
  // directly at uva's address, not just somewhere within its page.
  return (char*)P2V(pa) + ((addr_t)uva & 0xFFF);
}

int
copyout(pml4e_t *pgdir, addr_t va, void *p, uint64 len)
{
  char *buf, *pa0;
  addr_t n, va0;

  buf = (char*)p;
  while(len > 0){
    va0 = PGROUNDDOWN(va);
    pa0 = uva2ka(pgdir, (char*)va0);

    if(pa0 == 0) {
      return -1;
    }

    n = PGSIZE - (va - va0);
    if(n > len)
      n = len;
    memmove(pa0 + (va - va0), buf, n);

    len -= n;
    buf += n;
    va += n;
  }
  return 0;
}

int
copyin(pml4e_t *pgdir, addr_t srcva, void *dst, uint64 len)
{
  uint64 n, va0;
  char *pa0;
  char *dst_buf = (char*)dst;

  while(len > 0){
    va0 = PGROUNDDOWN(srcva);
    pa0 = uva2ka(pgdir, (char*)va0);
    if(pa0 == 0)
      return -1;
    n = PGSIZE - (srcva - va0);
    if(n > len)
      n = len;
    memmove(dst_buf, pa0 + (srcva - va0), n);
    len -= n;
    dst_buf += n;
    srcva += n;
  }
  return 0;
}