SchOS/kernel/core/dma.c

// Physically contiguous memory for devices. See include/core/dma.h.

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "arch/memlayout.h"
#include "arch/mmu.h"
#include "core/spinlock.h"
#include "core/dma.h"
#include "core/sbuf.h"
#include "core/dbg.h"

#define DMA_GRAIN  64          // one cache line; also xHCI's ring alignment
#define NEXTENT    64          // free extents we can track

struct extent {
  addr_t base;                 // kernel virtual address
  uint64 len;
};

static struct {
  struct spinlock lock;
  struct extent   free[NEXTENT];
  int             n;           // free extents, kept sorted by base
  addr_t          lo, hi;
  uint64          total;
} arena;

static uint64
roundup(uint64 v, uint64 to)
{
  return (v + to - 1) & ~(to - 1);
}

// What the arena is holding, extent by extent.
//
// This is the view that would have ended the H2 investigation in one line. The
// bug was that a 4 MB block was on this free list *and* on kalloc's at the same
// time, and the only way to see it at the time was to read two allocators and
// reason about PHYSTOP. Printing the list is not clever; not being able to
// print it was the problem.
static void
dma_dump(struct sbuf *out, const char *arg)
{
  (void)arg;
  acquire(&arena.lock);

  sb_puts(out, "arena  0x");
  sb_puthex(out, arena.lo, 16);
  sb_puts(out, "-0x");
  sb_puthex(out, arena.hi, 16);
  sb_puts(out, "  (virtual; physical is this minus KERNBASE)\n");

  sb_puts(out, "       ");
  sb_putu(out, arena.total / 1024);
  sb_puts(out, " KB total, ");
  {
    uint64 free = 0;
    for (int i = 0; i < arena.n; i++)
      free += arena.free[i].len;
    sb_putu(out, free / 1024);
    sb_puts(out, " KB free in ");
    sb_putu(out, arena.n);
    sb_puts(out, " extent(s), ");
    sb_putu(out, (arena.total - free) / 1024);
    sb_puts(out, " KB out\n");
  }

  sb_puts(out, "\n  #   base                 phys                 bytes\n");
  for (int i = 0; i < arena.n; i++) {
    sb_puts(out, "  ");
    sb_putu_pad(out, i, 2);
    sb_puts(out, "  0x");
    sb_puthex(out, arena.free[i].base, 16);
    sb_puts(out, "   0x");
    sb_puthex(out, arena.free[i].base - (addr_t)KERNBASE, 16);
    sb_puts(out, "   ");
    sb_putu_pad(out, arena.free[i].len, 10);
    sb_putc(out, '\n');
  }
  if (arena.n == NEXTENT)
    sb_puts(out, "\n  the extent table is FULL -- a free here leaks rather "
                 "than corrupting the list\n");

  release(&arena.lock);
}

// Does the arena own this page, and does it think it is free?
//
// The "free" half is what matters. A page the arena has handed out and a page
// the arena is holding look identical from outside, and the difference is
// whether someone else may be given it next.
static void
dma_whose(struct sbuf *out, struct whose *w)
{
  addr_t kva = (addr_t)P2V(w->phys);
  int    i, freed = 0;

  if (kva < arena.lo || kva >= arena.hi)
    return;

  acquire(&arena.lock);
  for (i = 0; i < arena.n; i++) {
    if (kva >= arena.free[i].base &&
        kva <  arena.free[i].base + arena.free[i].len) {
      freed = 1;
      break;
    }
  }
  release(&arena.lock);

  w->owners++;
  sb_puts(out, "  dma     arena");
  if (freed) {
    w->free_owners++;
    sb_puts(out, ", FREE -- in extent ");
    sb_putu(out, i);
    sb_puts(out, " (0x");
    sb_puthex(out, arena.free[i].base, 16);
    sb_puts(out, ", ");
    sb_putu(out, arena.free[i].len);
    sb_puts(out, " bytes)\n");
  } else {
    sb_puts(out, ", handed out\n");
  }
}

void
dma_init(void *vstart, void *vend)
{
  initlock(&arena.lock, "dma");
  arena.lo = roundup((addr_t)vstart, PGSIZE);
  arena.hi = (addr_t)vend & ~(addr_t)(PGSIZE - 1);
  if(arena.hi <= arena.lo)
    panic("dma_init: empty arena");
  arena.free[0].base = arena.lo;
  arena.free[0].len  = arena.hi - arena.lo;
  arena.n = 1;
  arena.total = arena.free[0].len;

  dbg_register("dma", dma_dump, "the DMA arena's free extents");
  dbg_register_whose("dma", dma_whose);
}

// Remove free[i], shifting the tail down.
static void
drop(int i)
{
  int j;
  for(j = i; j + 1 < arena.n; j++)
    arena.free[j] = arena.free[j+1];
  arena.n--;
}

// Insert a free extent at index i, shifting the tail up. Returns 0 if the
// table is full, in which case the memory is leaked rather than corrupting
// the list -- a leak is recoverable by rebooting, a corrupt free list hands
// two drivers the same ring.
static int
insert(int i, addr_t base, uint64 len)
{
  int j;

  if(arena.n >= NEXTENT)
    return 0;
  for(j = arena.n; j > i; j--)
    arena.free[j] = arena.free[j-1];
  arena.free[i].base = base;
  arena.free[i].len  = len;
  arena.n++;
  return 1;
}

void *
dma_alloc(uint64 size, uint64 align)
{
  int i;
  addr_t start;

  if(size == 0)
    return 0;
  if(align < DMA_GRAIN)
    align = DMA_GRAIN;
  if(align & (align - 1))
    panic("dma_alloc: alignment not a power of two");
  size = roundup(size, DMA_GRAIN);

  acquire(&arena.lock);

  // Best fit, not first fit.
  //
  // Everything in this arena has to be *contiguous* -- that is the entire
  // reason it exists -- so the number that matters is not how much is free but
  // how big the largest piece is. First fit destroys that number cheaply: a
  // 64 KB ring, allocated at boot from the front of a 16 MB arena, leaves one
  // 16 MB extent alone; allocated from the middle it leaves two of 8 MB, and a
  // framebuffer that needed 9 MB now cannot be had on an arena three quarters
  // empty.
  //
  // That is not hypothetical -- it is what the arena looked like in practice:
  //
  //   16384 KB total, 12295 KB free in 3 extent(s), 4088 KB out
  //     0  ...  4091904        <- too small for a 4000 KB framebuffer, by 4 KB
  //     1  ...     3584
  //     2  ...  8495104
  //
  // Best fit puts each request in the tightest extent that will hold it, which
  // leaves the large ones whole for the requests that have no alternative. It
  // is O(n) over a table that holds a handful of entries, so the cost is
  // nothing, and it cannot fragment *worse* than first fit for any request.
  {
    int best = -1;
    uint64 best_waste = 0;

    for(i = 0; i < arena.n; i++){
      addr_t base = arena.free[i].base;
      uint64 len  = arena.free[i].len;
      uint64 lead = roundup(base, align) - base;
      uint64 waste;

      if(lead + size > len)
        continue;
      waste = len - size - lead;
      if(best < 0 || waste < best_waste){
        best = i;
        best_waste = waste;
        if(waste == 0)
          break;                   // an exact fit cannot be improved on
      }
    }
    i = best;
  }

  if(i >= 0){
    addr_t base = arena.free[i].base;
    uint64 len  = arena.free[i].len;
    uint64 lead;

    start = roundup(base, align);
    lead  = start - base;

    // Three cases: the request consumes the extent, sits at its head, sits
    // at its tail, or sits in the middle and splits it in two.
    if(lead == 0 && size == len){
      drop(i);
    } else if(lead == 0){
      arena.free[i].base += size;
      arena.free[i].len  -= size;
    } else if(lead + size == len){
      arena.free[i].len = lead;
    } else {
      uint64 tail_len = len - lead - size;
      arena.free[i].len = lead;
      if(!insert(i + 1, start + size, tail_len)){
        // No room to record the tail. Give the whole extent to the caller
        // rather than lose track of where the tail was.
        arena.free[i].len = len;
        release(&arena.lock);
        cprintf("dma_alloc: extent table full; %d bytes handed out with it\n",
                (int)(len - size));
        memset((void*)start, 0, size);
        return (void*)start;
      }
    }

    release(&arena.lock);
    memset((void*)start, 0, size);
    return (void*)start;
  }

  release(&arena.lock);

  // Both numbers, because they say different things and only one of them is
  // usually the problem. "12 MB free" and "the biggest piece is 3 MB" is not a
  // full arena, it is a fragmented one, and the fix for the two is not the
  // same: a fragmented arena is cured by whoever is holding the block in the
  // middle letting go, and a full one is cured by DMA_ARENA_BYTES.
  //
  // This distinction is invisible from the old message, which reported only
  // the total and so read as "out of memory" on a machine with three quarters
  // of the arena free.
  cprintf("dma_alloc: no room for %d bytes -- %d free, "
          "largest contiguous piece %d (`schdbg dma` for the map)\n",
          (int)size, (int)dma_free_bytes(), (int)dma_largest_bytes());
  return 0;
}

void
dma_free(void *p, uint64 size)
{
  addr_t base = (addr_t)p;
  int i;

  if(p == 0)
    return;
  size = roundup(size, DMA_GRAIN);
  if(base < arena.lo || base + size > arena.hi)
    panic("dma_free: not DMA memory");

  acquire(&arena.lock);

  // Find the first extent that starts after this block, so the list stays
  // sorted and neighbours end up adjacent for coalescing.
  for(i = 0; i < arena.n && arena.free[i].base < base; i++)
    ;

  if(i > 0 && arena.free[i-1].base + arena.free[i-1].len == base){
    arena.free[i-1].len += size;          // merge with the block before
    i--;
  } else if(!insert(i, base, size)){
    release(&arena.lock);
    cprintf("dma_free: extent table full, leaking %d bytes\n", (int)size);
    return;
  }

  if(i + 1 < arena.n &&
     arena.free[i].base + arena.free[i].len == arena.free[i+1].base){
    arena.free[i].len += arena.free[i+1].len;   // and with the block after
    drop(i + 1);
  }

  release(&arena.lock);
}

addr_t
dma_phys(const void *p)
{
  return V2P((addr_t)p);
}

uint64
dma_free_bytes(void)
{
  uint64 n = 0;
  int i;

  for(i = 0; i < arena.n; i++)
    n += arena.free[i].len;
  return n;
}

// The largest single allocation this arena could still satisfy.
//
// Not the same as dma_free_bytes(), and the gap between them is the whole
// story when an allocation fails on an arena that is mostly free.
uint64
dma_largest_bytes(void)
{
  uint64 best = 0;
  int i;

  acquire(&arena.lock);
  for(i = 0; i < arena.n; i++)
    if(arena.free[i].len > best)
      best = arena.free[i].len;
  release(&arena.lock);
  return best;
}

uint64
dma_total_bytes(void)
{
  return arena.total;
}