SchOS/kernel/core/kalloc.c

// Physical memory allocator, intended to allocate
// memory for user processes, kernel stacks, page table pages,
// and pipe buffers. Allocates 4096-byte pages.

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "arch/memlayout.h"
#include "arch/mmu.h"
#include "core/spinlock.h"
#include "core/sbuf.h"
#include "core/dbg.h"

void freerange(void *vstart, void *vend);
extern char end[]; // first address after kernel loaded from ELF file


struct run {
  struct run *next;
};

// Free memory is two things: pages that have been allocated and given back,
// on a list through their first word, and ranges that have never been handed
// out at all, as a cursor and an end. kalloc() takes from the list first.
//
// Everything used to go on the list at boot, which means writing every page
// of RAM -- a pointer and, from kfree(), 4 KB of junk -- before the first
// process ran. On a 64 GB machine that is 64 GB of memset to say "this is
// free". A range costs sixteen bytes however large it is, and a page is first
// touched by whoever asked for it.
//
// `nfree` counts both, so the free total is a read rather than a walk of
// millions of links under the lock every time top refreshes.
#define NFRESH (2 * MEMREGION_MAX)

struct fresh {
  addr_t next, end;             // physical, page aligned
};

struct {
  struct spinlock lock;
  int use_lock;
  struct run *freelist;
  struct fresh fresh[NFRESH];
  int nfresh, cur;              // ranges, and the first that may have pages
  uint64 nfree;                 // pages, list and ranges together
} kmem;

// Before the allocator has a lock, or under it.
static void
add_fresh(addr_t lo, addr_t hi)
{
  lo = PGROUNDUP(lo);
  hi &= ~(addr_t)(PGSIZE - 1);
  if (lo >= hi)
    return;
  if (kmem.nfresh == NFRESH) {
    // Not reachable with MEMREGION_MAX regions per pass; but if it were, the
    // old way still works.
    freerange(P2V(lo), P2V(hi));
    return;
  }
  kmem.fresh[kmem.nfresh].next = lo;
  kmem.fresh[kmem.nfresh].end = hi;
  kmem.nfresh++;
  kmem.nfree += (hi - lo) / PGSIZE;
}

// Catch kmem.freelist corruption at the point it's touched, rather than
// letting a bad pointer be silently returned by kalloc() and crash whatever
// code dereferences it minutes later on an unrelated call stack.
static void
check_run(struct run *r, char *where)
{
  if(r == 0)
    return;
  if((addr_t)r % PGSIZE || (char*)r < end || V2P(r) >= PHYSTOP){
    cprintf("kalloc: corrupt freelist entry %p at %s\n", r, where);
    panic("kalloc: corrupt freelist");
  }
}

// vstart..vend is the window the kernel is willing to give away: from the end
// of its own image to the bottom of the DMA arena. Inside that window the
// firmware's map decides what is real -- free each usable region clipped to the
// window, rather than walking the whole span and handing out the holes.
//
// The old code walked it as one range, which is what a BIOS map looks like.
// Under UEFI it is not: OVMF puts an ACPI NVS region at 8 MB, so the run
// containing the kernel stopped there and everything above it was lost. The
// window still bounds things at both ends, so nothing below the kernel image
// and nothing inside the DMA arena is ever freed, whatever the map says.
void
kinit1(void *vstart, void *vend)
{
  initlock(&kmem.lock, "kmem");
  kmem.use_lock = 0;
  kmem.freelist = 0; // empty

  // No map at all means e820_init() could not parse one; it leaves a single
  // synthetic region behind for exactly this reason, so this is belt and
  // braces rather than a path anything takes.
  if(nmem_usable == 0){
    add_fresh(V2P((addr_t)vstart), V2P((addr_t)vend));
    return;
  }

  addr_t lo_limit = V2P((addr_t)vstart);
  addr_t hi_limit = V2P((addr_t)vend);

  for(int i = 0; i < nmem_usable; i++){
    addr_t lo = mem_usable[i].base;
    addr_t hi = mem_usable[i].end;

    if(lo < lo_limit)
      lo = lo_limit;
    if(hi > hi_limit)
      hi = hi_limit;
    if(lo >= hi)
      continue;

    add_fresh(lo, hi);
  }
}

// The page allocator, from the outside.
//
// Counting the free list is O(the free list), which on a gigabyte is a quarter
// of a million links. That is fine for a command someone typed and would not
// be fine anywhere else, so it lives here and nowhere near a hot path -- and
// the walk is bounded, because the one time you most want to run this is when
// you suspect the list is a loop.
static void
mem_dump(struct sbuf *out, const char *arg)
{
  (void)arg;
  struct run *r;
  uint64 n = 0;
  int    looped = 0;

  if (kmem.use_lock)
    acquire(&kmem.lock);
  for (r = kmem.freelist; r; r = r->next) {
    if (++n > (PHYSTOP / PGSIZE) + 16) { looped = 1; break; }
  }
  if (kmem.use_lock)
    release(&kmem.lock);

  sb_puts(out, "window 0x");
  sb_puthex(out, (addr_t)end, 16);
  sb_puts(out, " upward; PHYSTOP is 0x");
  sb_puthex(out, (addr_t)PHYSTOP, 16);
  sb_putc(out, '\n');
  sb_kv(out, "listed ", n, "pages");
  sb_kv(out, "free   ", kmem.nfree, "pages");
  sb_kv(out, "       ", kmem.nfree * PGSIZE / 1024, "KB");
  for (int i = 0; i < kmem.nfresh; i++) {
    sb_puts(out, "  never used 0x");
    sb_puthex(out, kmem.fresh[i].next, 12);
    sb_puts(out, "-0x");
    sb_puthex(out, kmem.fresh[i].end, 12);
    sb_putc(out, '\n');
  }
  if (looped)
    sb_puts(out, "\n  the free list did not end -- it has a CYCLE in it\n");
}

// The page allocator, and the kernel's own image.
//
// Both live here because both are answered from the same two facts -- where
// the kernel ends and what is on the free list -- and because a page inside
// the kernel image being on the free list is precisely the corruption this
// query exists to catch.
static void
mem_whose(struct sbuf *out, struct whose *w)
{
  addr_t kva = (addr_t)P2V(w->phys);
  struct run *r;
  uint64 n = 0;
  int found = 0;

  if (w->phys >= (addr_t)PHYSTOP)
    return;

  if (kva < (addr_t)end) {
    // Below `end` is the kernel image itself: text, rodata, data and bss, plus
    // everything the loader put under it. It is owned, permanently, by nothing
    // that can be asked -- so it is reported here.
    w->owners++;
    sb_puts(out, "  kernel  image (below `end` at 0x");
    sb_puthex(out, (addr_t)end, 16);
    sb_puts(out, ")\n");
    return;
  }

  if (kmem.use_lock)
    acquire(&kmem.lock);
  for (r = kmem.freelist; r; r = r->next) {
    if (++n > (PHYSTOP / PGSIZE) + 16)
      break;                                   // a cycle; mem_dump says so
    if ((addr_t)r == kva) { found = 1; break; }
  }
  if (kmem.use_lock)
    release(&kmem.lock);

  for (int i = 0; !found && i < kmem.nfresh; i++)
    if (w->phys >= kmem.fresh[i].next && w->phys < kmem.fresh[i].end) {
      w->owners++;
      w->free_owners++;
      sb_puts(out, "  kalloc  FREE -- never handed out yet\n");
      return;
    }

  if (found) {
    w->owners++;
    w->free_owners++;
    sb_puts(out, "  kalloc  FREE -- position ");
    sb_putu(out, n);
    sb_puts(out, " on the free list\n");
  }
}

// The memory the loader's page tables could not reach.
//
// kinit1() runs before kvmalloc(), on page tables that map four gigabytes, and
// freeing a page means writing a free-list pointer *into* it -- so anything
// higher would fault. kvmalloc() has now mapped as much of physical memory as
// the machine has, so the rest can be handed over.
//
// Two passes rather than one because of that ordering, and the ordering is
// forced: kvmalloc() calls kalloc() for its own page tables, so the allocator
// has to be working before the map it needs exists.
void
kinit_high(void)
{
  uint64 added = 0;

  for (int i = 0; i < nmem_usable; i++) {
    uint64 lo = mem_usable[i].base;
    uint64 hi = mem_usable[i].end;

    if (lo < DIRECTMAP_BOOT)
      lo = DIRECTMAP_BOOT;
    if (hi > phystop)
      hi = phystop;
    if (lo >= hi)
      continue;

    if (kmem.use_lock)
      acquire(&kmem.lock);
    add_fresh(lo, hi);
    if (kmem.use_lock)
      release(&kmem.lock);
    added += hi - lo;
  }

  if (added)
    kinfo("mem: %d MB above 4 GB added to the allocator\n",
          (int)(added / (1024 * 1024)));
}

void
kinit2()
{
  kmem.use_lock = 1;
  dbg_register("mem", mem_dump, "the page allocator's free list");
  dbg_register_whose("mem", mem_whose);
}

void
freerange(void *vstart, void *vend)
{
  char *p;
  p = (char*)PGROUNDUP((addr_t)vstart);
  for(; p + PGSIZE <= (char*)vend; p += PGSIZE)
    kfree(p);
}

// Free the page of physical memory pointed at by v,
// which normally should have been returned by a
// call to kalloc().  (The exception is when
// initializing the allocator; see kinit above.)
void
kfree(char *v)
{
  struct run *r;

  if((addr_t)v % PGSIZE || v < end || V2P(v) >= PHYSTOP)
    panic("kfree");

  // Fill with junk to catch dangling refs.
  memset(v, 1, PGSIZE);

  if(kmem.use_lock)
    acquire(&kmem.lock);
  check_run(kmem.freelist, "kfree entry");
  r = (struct run*)v;
  r->next = kmem.freelist;
  kmem.freelist = r;
  kmem.nfree++;
  if(kmem.use_lock)
    release(&kmem.lock);
}

// Allocate one 4096-byte page of physical memory.
// Returns a pointer that the kernel can use.
// Returns 0 if the memory cannot be allocated.
char*
kalloc(void)
{
  struct run *r;

  if(kmem.use_lock)
    acquire(&kmem.lock);
  check_run(kmem.freelist, "kalloc entry");
  r = kmem.freelist;
  if(r){
    kmem.freelist = r->next;
  } else {
    while(kmem.cur < kmem.nfresh &&
          kmem.fresh[kmem.cur].next >= kmem.fresh[kmem.cur].end)
      kmem.cur++;
    if(kmem.cur < kmem.nfresh){
      r = (struct run*)P2V(kmem.fresh[kmem.cur].next);
      kmem.fresh[kmem.cur].next += PGSIZE;
    }
  }
  if(r)
    kmem.nfree--;

  if(kmem.use_lock)
    release(&kmem.lock);
  return (char*)r;
}

// Total free physical memory, in bytes.
//
// This walked the free list under the allocator's lock: two million links on
// 8 GB, cache misses all, every second that top or a status bar was running --
// and every CPU that wanted a page in that time waited for it. One word now.
uint64
kalloc_free_bytes(void)
{
  return kmem.nfree * PGSIZE;
}