SchOS/kernel/core/dma.c

// Physically contiguous memory for devices. See include/dma.h.

#include "types.h"
#include "defs.h"
#include "param.h"
#include "memlayout.h"
#include "mmu.h"
#include "spinlock.h"
#include "dma.h"

#define DMA_GRAIN  64          // one cache line; also xHCI's ring alignment
#define NEXTENT    64          // free extents we can track

struct extent {
  addr_t base;                 // kernel virtual address
  uint64 len;
};

static struct {
  struct spinlock lock;
  struct extent   free[NEXTENT];
  int             n;           // free extents, kept sorted by base
  addr_t          lo, hi;
  uint64          total;
} arena;

static uint64
roundup(uint64 v, uint64 to)
{
  return (v + to - 1) & ~(to - 1);
}

void
dma_init(void *vstart, void *vend)
{
  initlock(&arena.lock, "dma");
  arena.lo = roundup((addr_t)vstart, PGSIZE);
  arena.hi = (addr_t)vend & ~(addr_t)(PGSIZE - 1);
  if(arena.hi <= arena.lo)
    panic("dma_init: empty arena");
  arena.free[0].base = arena.lo;
  arena.free[0].len  = arena.hi - arena.lo;
  arena.n = 1;
  arena.total = arena.free[0].len;
}

// Remove free[i], shifting the tail down.
static void
drop(int i)
{
  int j;
  for(j = i; j + 1 < arena.n; j++)
    arena.free[j] = arena.free[j+1];
  arena.n--;
}

// Insert a free extent at index i, shifting the tail up. Returns 0 if the
// table is full, in which case the memory is leaked rather than corrupting
// the list -- a leak is recoverable by rebooting, a corrupt free list hands
// two drivers the same ring.
static int
insert(int i, addr_t base, uint64 len)
{
  int j;

  if(arena.n >= NEXTENT)
    return 0;
  for(j = arena.n; j > i; j--)
    arena.free[j] = arena.free[j-1];
  arena.free[i].base = base;
  arena.free[i].len  = len;
  arena.n++;
  return 1;
}

void *
dma_alloc(uint64 size, uint64 align)
{
  int i;
  addr_t start;

  if(size == 0)
    return 0;
  if(align < DMA_GRAIN)
    align = DMA_GRAIN;
  if(align & (align - 1))
    panic("dma_alloc: alignment not a power of two");
  size = roundup(size, DMA_GRAIN);

  acquire(&arena.lock);

  for(i = 0; i < arena.n; i++){
    addr_t base = arena.free[i].base;
    uint64 len  = arena.free[i].len;
    uint64 lead;

    start = roundup(base, align);
    lead  = start - base;
    if(lead + size > len)
      continue;

    // Three cases: the request consumes the extent, sits at its head, sits
    // at its tail, or sits in the middle and splits it in two.
    if(lead == 0 && size == len){
      drop(i);
    } else if(lead == 0){
      arena.free[i].base += size;
      arena.free[i].len  -= size;
    } else if(lead + size == len){
      arena.free[i].len = lead;
    } else {
      uint64 tail_len = len - lead - size;
      arena.free[i].len = lead;
      if(!insert(i + 1, start + size, tail_len)){
        // No room to record the tail. Give the whole extent to the caller
        // rather than lose track of where the tail was.
        arena.free[i].len = len;
        continue;
      }
    }

    release(&arena.lock);
    memset((void*)start, 0, size);
    return (void*)start;
  }

  release(&arena.lock);
  cprintf("dma_alloc: out of DMA memory (wanted %d bytes, %d free)\n",
          (int)size, (int)dma_free_bytes());
  return 0;
}

void
dma_free(void *p, uint64 size)
{
  addr_t base = (addr_t)p;
  int i;

  if(p == 0)
    return;
  size = roundup(size, DMA_GRAIN);
  if(base < arena.lo || base + size > arena.hi)
    panic("dma_free: not DMA memory");

  acquire(&arena.lock);

  // Find the first extent that starts after this block, so the list stays
  // sorted and neighbours end up adjacent for coalescing.
  for(i = 0; i < arena.n && arena.free[i].base < base; i++)
    ;

  if(i > 0 && arena.free[i-1].base + arena.free[i-1].len == base){
    arena.free[i-1].len += size;          // merge with the block before
    i--;
  } else if(!insert(i, base, size)){
    release(&arena.lock);
    cprintf("dma_free: extent table full, leaking %d bytes\n", (int)size);
    return;
  }

  if(i + 1 < arena.n &&
     arena.free[i].base + arena.free[i].len == arena.free[i+1].base){
    arena.free[i].len += arena.free[i+1].len;   // and with the block after
    drop(i + 1);
  }

  release(&arena.lock);
}

addr_t
dma_phys(const void *p)
{
  return V2P((addr_t)p);
}

uint64
dma_free_bytes(void)
{
  uint64 n = 0;
  int i;

  for(i = 0; i < arena.n; i++)
    n += arena.free[i].len;
  return n;
}

uint64
dma_total_bytes(void)
{
  return arena.total;
}