SchOS/kernel/core/dma.c

// Physically contiguous memory for devices. See include/dma.h.

#include "types.h"
#include "defs.h"
#include "param.h"
#include "memlayout.h"
#include "mmu.h"
#include "spinlock.h"
#include "dma.h"
#include "sbuf.h"
#include "dbg.h"

#define DMA_GRAIN  64          // one cache line; also xHCI's ring alignment
#define NEXTENT    64          // free extents we can track

struct extent {
  addr_t base;                 // kernel virtual address
  uint64 len;
};

static struct {
  struct spinlock lock;
  struct extent   free[NEXTENT];
  int             n;           // free extents, kept sorted by base
  addr_t          lo, hi;
  uint64          total;
} arena;

static uint64
roundup(uint64 v, uint64 to)
{
  return (v + to - 1) & ~(to - 1);
}

// What the arena is holding, extent by extent.
//
// This is the view that would have ended the H2 investigation in one line. The
// bug was that a 4 MB block was on this free list *and* on kalloc's at the same
// time, and the only way to see it at the time was to read two allocators and
// reason about PHYSTOP. Printing the list is not clever; not being able to
// print it was the problem.
static void
dma_dump(struct sbuf *out, const char *arg)
{
  (void)arg;
  acquire(&arena.lock);

  sb_puts(out, "arena  0x");
  sb_puthex(out, arena.lo, 16);
  sb_puts(out, "-0x");
  sb_puthex(out, arena.hi, 16);
  sb_puts(out, "  (virtual; physical is this minus KERNBASE)\n");

  sb_puts(out, "       ");
  sb_putu(out, arena.total / 1024);
  sb_puts(out, " KB total, ");
  {
    uint64 free = 0;
    for (int i = 0; i < arena.n; i++)
      free += arena.free[i].len;
    sb_putu(out, free / 1024);
    sb_puts(out, " KB free in ");
    sb_putu(out, arena.n);
    sb_puts(out, " extent(s), ");
    sb_putu(out, (arena.total - free) / 1024);
    sb_puts(out, " KB out\n");
  }

  sb_puts(out, "\n  #   base                 phys                 bytes\n");
  for (int i = 0; i < arena.n; i++) {
    sb_puts(out, "  ");
    sb_putu_pad(out, i, 2);
    sb_puts(out, "  0x");
    sb_puthex(out, arena.free[i].base, 16);
    sb_puts(out, "   0x");
    sb_puthex(out, arena.free[i].base - (addr_t)KERNBASE, 16);
    sb_puts(out, "   ");
    sb_putu_pad(out, arena.free[i].len, 10);
    sb_putc(out, '\n');
  }
  if (arena.n == NEXTENT)
    sb_puts(out, "\n  the extent table is FULL -- a free here leaks rather "
                 "than corrupting the list\n");

  release(&arena.lock);
}

// Does the arena own this page, and does it think it is free?
//
// The "free" half is what matters. A page the arena has handed out and a page
// the arena is holding look identical from outside, and the difference is
// whether someone else may be given it next.
static void
dma_whose(struct sbuf *out, struct whose *w)
{
  addr_t kva = (addr_t)P2V(w->phys);
  int    i, freed = 0;

  if (kva < arena.lo || kva >= arena.hi)
    return;

  acquire(&arena.lock);
  for (i = 0; i < arena.n; i++) {
    if (kva >= arena.free[i].base &&
        kva <  arena.free[i].base + arena.free[i].len) {
      freed = 1;
      break;
    }
  }
  release(&arena.lock);

  w->owners++;
  sb_puts(out, "  dma     arena");
  if (freed) {
    w->free_owners++;
    sb_puts(out, ", FREE -- in extent ");
    sb_putu(out, i);
    sb_puts(out, " (0x");
    sb_puthex(out, arena.free[i].base, 16);
    sb_puts(out, ", ");
    sb_putu(out, arena.free[i].len);
    sb_puts(out, " bytes)\n");
  } else {
    sb_puts(out, ", handed out\n");
  }
}

void
dma_init(void *vstart, void *vend)
{
  initlock(&arena.lock, "dma");
  arena.lo = roundup((addr_t)vstart, PGSIZE);
  arena.hi = (addr_t)vend & ~(addr_t)(PGSIZE - 1);
  if(arena.hi <= arena.lo)
    panic("dma_init: empty arena");
  arena.free[0].base = arena.lo;
  arena.free[0].len  = arena.hi - arena.lo;
  arena.n = 1;
  arena.total = arena.free[0].len;

  dbg_register("dma", dma_dump, "the DMA arena's free extents");
  dbg_register_whose("dma", dma_whose);
}

// Remove free[i], shifting the tail down.
static void
drop(int i)
{
  int j;
  for(j = i; j + 1 < arena.n; j++)
    arena.free[j] = arena.free[j+1];
  arena.n--;
}

// Insert a free extent at index i, shifting the tail up. Returns 0 if the
// table is full, in which case the memory is leaked rather than corrupting
// the list -- a leak is recoverable by rebooting, a corrupt free list hands
// two drivers the same ring.
static int
insert(int i, addr_t base, uint64 len)
{
  int j;

  if(arena.n >= NEXTENT)
    return 0;
  for(j = arena.n; j > i; j--)
    arena.free[j] = arena.free[j-1];
  arena.free[i].base = base;
  arena.free[i].len  = len;
  arena.n++;
  return 1;
}

void *
dma_alloc(uint64 size, uint64 align)
{
  int i;
  addr_t start;

  if(size == 0)
    return 0;
  if(align < DMA_GRAIN)
    align = DMA_GRAIN;
  if(align & (align - 1))
    panic("dma_alloc: alignment not a power of two");
  size = roundup(size, DMA_GRAIN);

  acquire(&arena.lock);

  for(i = 0; i < arena.n; i++){
    addr_t base = arena.free[i].base;
    uint64 len  = arena.free[i].len;
    uint64 lead;

    start = roundup(base, align);
    lead  = start - base;
    if(lead + size > len)
      continue;

    // Three cases: the request consumes the extent, sits at its head, sits
    // at its tail, or sits in the middle and splits it in two.
    if(lead == 0 && size == len){
      drop(i);
    } else if(lead == 0){
      arena.free[i].base += size;
      arena.free[i].len  -= size;
    } else if(lead + size == len){
      arena.free[i].len = lead;
    } else {
      uint64 tail_len = len - lead - size;
      arena.free[i].len = lead;
      if(!insert(i + 1, start + size, tail_len)){
        // No room to record the tail. Give the whole extent to the caller
        // rather than lose track of where the tail was.
        arena.free[i].len = len;
        continue;
      }
    }

    release(&arena.lock);
    memset((void*)start, 0, size);
    return (void*)start;
  }

  release(&arena.lock);
  cprintf("dma_alloc: out of DMA memory (wanted %d bytes, %d free)\n",
          (int)size, (int)dma_free_bytes());
  return 0;
}

void
dma_free(void *p, uint64 size)
{
  addr_t base = (addr_t)p;
  int i;

  if(p == 0)
    return;
  size = roundup(size, DMA_GRAIN);
  if(base < arena.lo || base + size > arena.hi)
    panic("dma_free: not DMA memory");

  acquire(&arena.lock);

  // Find the first extent that starts after this block, so the list stays
  // sorted and neighbours end up adjacent for coalescing.
  for(i = 0; i < arena.n && arena.free[i].base < base; i++)
    ;

  if(i > 0 && arena.free[i-1].base + arena.free[i-1].len == base){
    arena.free[i-1].len += size;          // merge with the block before
    i--;
  } else if(!insert(i, base, size)){
    release(&arena.lock);
    cprintf("dma_free: extent table full, leaking %d bytes\n", (int)size);
    return;
  }

  if(i + 1 < arena.n &&
     arena.free[i].base + arena.free[i].len == arena.free[i+1].base){
    arena.free[i].len += arena.free[i+1].len;   // and with the block after
    drop(i + 1);
  }

  release(&arena.lock);
}

addr_t
dma_phys(const void *p)
{
  return V2P((addr_t)p);
}

uint64
dma_free_bytes(void)
{
  uint64 n = 0;
  int i;

  for(i = 0; i < arena.n; i++)
    n += arena.free[i].len;
  return n;
}

uint64
dma_total_bytes(void)
{
  return arena.total;
}