SchOS/kernel/core/kalloc.c

// Physical memory allocator, intended to allocate
// memory for user processes, kernel stacks, page table pages,
// and pipe buffers. Allocates 4096-byte pages.

#include "types.h"
#include "defs.h"
#include "param.h"
#include "memlayout.h"
#include "mmu.h"
#include "spinlock.h"
#include "sbuf.h"
#include "dbg.h"

void freerange(void *vstart, void *vend);
extern char end[]; // first address after kernel loaded from ELF file


struct run {
  struct run *next;
};

struct {
  struct spinlock lock;
  int use_lock;
  struct run *freelist;
} kmem;

// Catch kmem.freelist corruption at the point it's touched, rather than
// letting a bad pointer be silently returned by kalloc() and crash whatever
// code dereferences it minutes later on an unrelated call stack.
static void
check_run(struct run *r, char *where)
{
  if(r == 0)
    return;
  if((addr_t)r % PGSIZE || (char*)r < end || V2P(r) >= PHYSTOP){
    cprintf("kalloc: corrupt freelist entry %p at %s\n", r, where);
    panic("kalloc: corrupt freelist");
  }
}

// vstart..vend is the window the kernel is willing to give away: from the end
// of its own image to the bottom of the DMA arena. Inside that window the
// firmware's map decides what is real -- free each usable region clipped to the
// window, rather than walking the whole span and handing out the holes.
//
// The old code walked it as one range, which is what a BIOS map looks like.
// Under UEFI it is not: OVMF puts an ACPI NVS region at 8 MB, so the run
// containing the kernel stopped there and everything above it was lost. The
// window still bounds things at both ends, so nothing below the kernel image
// and nothing inside the DMA arena is ever freed, whatever the map says.
void
kinit1(void *vstart, void *vend)
{
  initlock(&kmem.lock, "kmem");
  kmem.use_lock = 0;
  kmem.freelist = 0; // empty

  // No map at all means e820_init() could not parse one; it leaves a single
  // synthetic region behind for exactly this reason, so this is belt and
  // braces rather than a path anything takes.
  if(nmem_usable == 0){
    freerange(vstart, vend);
    return;
  }

  addr_t lo_limit = V2P((addr_t)vstart);
  addr_t hi_limit = V2P((addr_t)vend);

  for(int i = 0; i < nmem_usable; i++){
    addr_t lo = mem_usable[i].base;
    addr_t hi = mem_usable[i].end;

    if(lo < lo_limit)
      lo = lo_limit;
    if(hi > hi_limit)
      hi = hi_limit;
    if(lo >= hi)
      continue;

    freerange(P2V(lo), P2V(hi));
  }
}

// The page allocator, from the outside.
//
// Counting the free list is O(the free list), which on a gigabyte is a quarter
// of a million links. That is fine for a command someone typed and would not
// be fine anywhere else, so it lives here and nowhere near a hot path -- and
// the walk is bounded, because the one time you most want to run this is when
// you suspect the list is a loop.
static void
mem_dump(struct sbuf *out, const char *arg)
{
  (void)arg;
  struct run *r;
  uint64 n = 0;
  int    looped = 0;

  if (kmem.use_lock)
    acquire(&kmem.lock);
  for (r = kmem.freelist; r; r = r->next) {
    if (++n > (PHYSTOP / PGSIZE) + 16) { looped = 1; break; }
  }
  if (kmem.use_lock)
    release(&kmem.lock);

  sb_puts(out, "window 0x");
  sb_puthex(out, (addr_t)end, 16);
  sb_puts(out, " upward; PHYSTOP is 0x");
  sb_puthex(out, (addr_t)PHYSTOP, 16);
  sb_putc(out, '\n');
  sb_kv(out, "free   ", n, "pages");
  sb_kv(out, "       ", n * PGSIZE / 1024, "KB");
  if (looped)
    sb_puts(out, "\n  the free list did not end -- it has a CYCLE in it\n");
}

// The page allocator, and the kernel's own image.
//
// Both live here because both are answered from the same two facts -- where
// the kernel ends and what is on the free list -- and because a page inside
// the kernel image being on the free list is precisely the corruption this
// query exists to catch.
static void
mem_whose(struct sbuf *out, struct whose *w)
{
  addr_t kva = (addr_t)P2V(w->phys);
  struct run *r;
  uint64 n = 0;
  int found = 0;

  if (w->phys >= (addr_t)PHYSTOP)
    return;

  if (kva < (addr_t)end) {
    // Below `end` is the kernel image itself: text, rodata, data and bss, plus
    // everything the loader put under it. It is owned, permanently, by nothing
    // that can be asked -- so it is reported here.
    w->owners++;
    sb_puts(out, "  kernel  image (below `end` at 0x");
    sb_puthex(out, (addr_t)end, 16);
    sb_puts(out, ")\n");
    return;
  }

  if (kmem.use_lock)
    acquire(&kmem.lock);
  for (r = kmem.freelist; r; r = r->next) {
    if (++n > (PHYSTOP / PGSIZE) + 16)
      break;                                   // a cycle; mem_dump says so
    if ((addr_t)r == kva) { found = 1; break; }
  }
  if (kmem.use_lock)
    release(&kmem.lock);

  if (found) {
    w->owners++;
    w->free_owners++;
    sb_puts(out, "  kalloc  FREE -- position ");
    sb_putu(out, n);
    sb_puts(out, " on the free list\n");
  }
}

// The memory the loader's page tables could not reach.
//
// kinit1() runs before kvmalloc(), on page tables that map four gigabytes, and
// freeing a page means writing a free-list pointer *into* it -- so anything
// higher would fault. kvmalloc() has now mapped as much of physical memory as
// the machine has, so the rest can be handed over.
//
// Two passes rather than one because of that ordering, and the ordering is
// forced: kvmalloc() calls kalloc() for its own page tables, so the allocator
// has to be working before the map it needs exists.
void
kinit_high(void)
{
  uint64 added = 0;

  for (int i = 0; i < nmem_usable; i++) {
    uint64 lo = mem_usable[i].base;
    uint64 hi = mem_usable[i].end;

    if (lo < DIRECTMAP_BOOT)
      lo = DIRECTMAP_BOOT;
    if (hi > phystop)
      hi = phystop;
    if (lo >= hi)
      continue;

    freerange(P2V(lo), P2V(hi));
    added += hi - lo;
  }

  if (added)
    kinfo("mem: %d MB above 4 GB added to the allocator\n",
          (int)(added / (1024 * 1024)));
}

void
kinit2()
{
  kmem.use_lock = 1;
  dbg_register("mem", mem_dump, "the page allocator's free list");
  dbg_register_whose("mem", mem_whose);
}

void
freerange(void *vstart, void *vend)
{
  char *p;
  p = (char*)PGROUNDUP((addr_t)vstart);
  for(; p + PGSIZE <= (char*)vend; p += PGSIZE)
    kfree(p);
}

// Free the page of physical memory pointed at by v,
// which normally should have been returned by a
// call to kalloc().  (The exception is when
// initializing the allocator; see kinit above.)
void
kfree(char *v)
{
  struct run *r;

  if((addr_t)v % PGSIZE || v < end || V2P(v) >= PHYSTOP)
    panic("kfree");

  // Fill with junk to catch dangling refs.
  memset(v, 1, PGSIZE);

  if(kmem.use_lock)
    acquire(&kmem.lock);
  check_run(kmem.freelist, "kfree entry");
  r = (struct run*)v;
  r->next = kmem.freelist;
  kmem.freelist = r;
  if(kmem.use_lock)
    release(&kmem.lock);
}

// Allocate one 4096-byte page of physical memory.
// Returns a pointer that the kernel can use.
// Returns 0 if the memory cannot be allocated.
char*
kalloc(void)
{
  struct run *r;

  if(kmem.use_lock)
    acquire(&kmem.lock);
  check_run(kmem.freelist, "kalloc entry");
  r = kmem.freelist;
  if(r)
    kmem.freelist = r->next;

  if(kmem.use_lock)
    release(&kmem.lock);
  return (char*)r;
}

// Total free physical memory, in bytes. Walks the free list; only /proc
// readers call it, so the O(n) scan is not on any hot path.
uint64
kalloc_free_bytes(void)
{
  struct run *r;
  uint64 n = 0;

  if(kmem.use_lock)
    acquire(&kmem.lock);
  for(r = kmem.freelist; r; r = r->next)
    n += PGSIZE;
  if(kmem.use_lock)
    release(&kmem.lock);
  return n;
}