SchOS/kernel/dev/bio.c

// Buffer cache.
//
// The buffer cache is a linked list of buf structures holding
// cached copies of disk block contents.  Caching disk blocks
// in memory reduces the number of disk reads and also provides
// a synchronization point for disk blocks used by multiple processes.
//
// Interface:
// * To get a buffer for a particular disk block, call bread.
// * After changing buffer data, call bwrite to write it to disk.
// * When done with the buffer, call brelse.
// * Do not use the buffer after calling brelse.
// * Only one process at a time can use a buffer,
//     so do not keep them longer than necessary.
//
// The implementation uses two state flags internally:
// * B_VALID: the buffer data has been read from the disk.
// * B_DIRTY: the buffer data has been modified
//     and needs to be written to disk.

#include "types.h"
#include "defs.h"
#include "param.h"
#include "spinlock.h"
#include "sleeplock.h"
#include "fs.h"
#include "buf.h"
#include "x86.h"
#include "fsstat.h"
#include "blkdev.h"

#include "kmem.h"
#include "hwinfo.h"

struct fsstat fsstat;

// The cache starts as NBUF buffers in the kernel image, 1 MB, and grows on
// demand: a miss with every buffer in use for something makes a new one
// rather than evicting, until the cache is 1/32 of RAM (at most 256 MB).
//
// It was NBUF and nothing else, on every machine. 1 MB is a fine floor for
// a machine with tens of megabytes and nothing at all for one with sixteen
// gigabytes, where it was re-reading the same directory blocks off the disk
// for every command. Growing rather than sizing at boot means a small machine,
// or a large one doing little, spends nothing it does not use -- and that
// binit() needs nothing from the page allocator that early.
//
// Lookups go through a hash on (dev, blockno). The LRU list was also the
// lookup structure, a linear walk per bread(), which is fine at 256 and
// hopeless at 65536.
#define BUF_MAX_BYTES  (256ULL * 1024 * 1024)
#define NHASH          8192            // a power of two

static uchar pooldata[NBUF][BSIZE];

struct {
  struct spinlock lock;
  struct buf buf[NBUF];

  // Linked list of all buffers, through prev/next.
  // head.next is most recently used.
  struct buf head;

  struct buf *hash[NHASH];
  struct buf *all;            // through anext
  struct kmem_cache *hdrs;
  uint  n;                    // buffers that exist
  uint  max;                  // how many there may be; 0 until first needed
} bcache;

static inline uint
bhash(uint dev, uint64 blockno)
{
  uint64 h = (blockno ^ ((uint64)dev << 40)) * 0x9E3779B97F4A7C15ULL;
  return (uint)(h >> 51) & (NHASH - 1);
}

// Both under bcache.lock.
static void
hash_del(struct buf *b)
{
  struct buf **pp = &bcache.hash[bhash(b->dev, b->blockno)];
  while (*pp && *pp != b)
    pp = &(*pp)->hnext;
  if (*pp)
    *pp = b->hnext;
  b->hnext = 0;
}

static void
hash_add(struct buf *b)
{
  struct buf **pp = &bcache.hash[bhash(b->dev, b->blockno)];
  b->hnext = *pp;
  *pp = b;
}

// Give b a new identity, moving it between chains. Under bcache.lock.
static void
rename_buf(struct buf *b, uint dev, uint64 blockno)
{
  hash_del(b);
  b->dev = dev;
  b->blockno = blockno;
  hash_add(b);
}

static void
link_new(struct buf *b)
{
  b->next = bcache.head.next;
  b->prev = &bcache.head;
  initsleeplock(&b->lock, "buffer");
  bcache.head.next->prev = b;
  bcache.head.next = b;
  b->anext = bcache.all;
  bcache.all = b;
  // An identity no block has, so the first rename has a chain to leave.
  b->dev = (uint)-1;
  b->blockno = (uint64)bcache.n;
  hash_add(b);
  bcache.n++;
}

void
binit(void)
{
  initlock(&bcache.lock, "bcache");

  bcache.head.prev = &bcache.head;
  bcache.head.next = &bcache.head;
  for (int i = 0; i < NBUF; i++) {
    bcache.buf[i].data = pooldata[i];
    link_new(&bcache.buf[i]);
  }
}

// One more buffer, or 0 at the cap or when memory is short. Under
// bcache.lock; the allocators take only their own locks.
static struct buf *
grow(void)
{
  if (bcache.max == 0) {
    uint64 want = ram_total_bytes() / 32;
    if (want > BUF_MAX_BYTES)
      want = BUF_MAX_BYTES;
    bcache.max = (uint)(want / BSIZE);
    if (bcache.max < NBUF)
      bcache.max = NBUF;
    bcache.hdrs = kmem_cache_create("buf", sizeof(struct buf));
  }
  if (bcache.n >= bcache.max || bcache.hdrs == 0)
    return 0;
  struct buf *b = kmem_cache_alloc(bcache.hdrs);
  if (b == 0)
    return 0;
  memset(b, 0, sizeof *b);
  if ((b->data = (uchar *)kalloc()) == 0) {
    kmem_cache_free(bcache.hdrs, b);
    return 0;
  }
  link_new(b);
  return b;
}

// How many buffers there are, and may be. For /proc/fsstat.
void
bcache_size(uint *n, uint *max)
{
  *n = bcache.n;
  *max = bcache.max ? bcache.max : NBUF;
}

// Route a request to the driver that owns b->dev. Everything above this line
// is device-agnostic; everything below picks a backend. RAMDISK_DEV is served
// out of memory (kernel/dev/ramdisk.c), USBDISK_DEV by bulk transfers to a
// USB stick, everything else by the IDE driver.
static void bdevrw_inner(struct buf *b);

static void
bdevrw(struct buf *b)
{
  uint64 t0 = rdtsc();
  if(b->flags & B_DIRTY) fsstat.disk_writes++; else fsstat.disk_reads++;

  bdevrw_inner(b);
  fsstat.disk_cycles += rdtsc() - t0;
}

static void
bdevrw_inner(struct buf *b)
{
  if(b->dev == RAMDISK_DEV){
    ramdiskrw(b);
    return;
  }
  if(b->dev == USBDISK_DEV){
    usb_storage_rw(b);
    return;
  }
  // SATA, NVMe and every partition: registered at probe time rather than
  // known at build time. See include/blkdev.h.
  if(b->dev >= BLKDEV_FIRST){
    blkdev_rw(b);
    return;
  }
  iderw(b);
}

// Tell the device to commit whatever it has accepted but not yet made durable.
// Same dispatch as bdevrw, and the same rule: everything above this line is
// device-agnostic.
static int
bdevflush(uint dev)
{
  if(dev == RAMDISK_DEV)
    return 0;                   // memory has no volatile stage below it
  if(dev == USBDISK_DEV)
    return usb_storage_flush();
  if(dev >= BLKDEV_FIRST)
    return blkdev_flush(dev);
  return ideflush(dev);
}

// Look through buffer cache for block on device dev.
// If not found, allocate a buffer.
// In either case, return locked buffer.
// Forget every cached block of a device that is no longer there: clean and
// invalid, so it is neither served to a reader nor written back on eviction.
// A buffer somebody is holding is left to them -- they are about to find out.
void
binval(uint dev)
{
  struct buf *b;

  acquire(&bcache.lock);
  for(b = bcache.all; b; b = b->anext)
    if(b->dev == dev && b->refcnt == 0)
      b->flags = 0;
  release(&bcache.lock);
}

static struct buf*
bget(uint dev, uint64 blockno)
{
  struct buf *b;

 retry:
  acquire(&bcache.lock);

  // Is the block already cached?
  for(b = bcache.hash[bhash(dev, blockno)]; b; b = b->hnext){
    if(b->dev == dev && b->blockno == blockno){
      if(b->refcnt++ == 0)
        FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak);
      release(&bcache.lock);
      acquiresleep(&b->lock);
      return b;
    }
  }

  // Not cached. A new buffer while the cache may still grow, so nothing
  // cached is thrown away for it.
  if((b = grow()) != 0){
    rename_buf(b, dev, blockno);
    b->flags = 0;
    b->refcnt = 1;
    FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak);
    release(&bcache.lock);
    acquiresleep(&b->lock);
    return b;
  }

  // Otherwise recycle some unused buffer and clean buffer
  // "clean" because B_DIRTY and not locked means log.c
  // hasn't yet committed the changes to the buffer.
  for(b = bcache.head.prev; b != &bcache.head; b = b->prev){
    if(b->refcnt == 0 && (b->flags & B_DIRTY) == 0) {
      rename_buf(b, dev, blockno);
      b->flags = 0;
      b->refcnt = 1;
      FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak);
      release(&bcache.lock);
      acquiresleep(&b->lock);
      return b;
    }
  }

  // Every buffer is dirty and none is clean enough to take.
  //
  // This used to be a straight panic, which was right while every writer went
  // through bwrite and a dirty buffer therefore meant "the log has not
  // committed yet". SchFS marks tree blocks dirty and leaves them there (see
  // bdirty), so an all-dirty cache is now an ordinary state: a transaction that
  // touches more than NBUF blocks reaches it long before it commits.
  //
  // Writing one out early is always safe. The only ordering the format needs is
  // that every block of the new tree precede the uberblock that names it, and
  // the uberblock goes down after an explicit bflush -- so nothing here can
  // land on the wrong side of that barrier.
  for(b = bcache.head.prev; b != &bcache.head; b = b->prev){
    if(b->refcnt == 0){
      fsstat.bget_evict_dirty++;
      b->refcnt++;              // claim it, so nobody else recycles it either
      FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak);
      release(&bcache.lock);
      acquiresleep(&b->lock);

      if(b->flags & B_DIRTY)
        bdevrw(b);

      // Between dropping bcache.lock and taking b's, another process may have
      // found this buffer under its *old* identity and be waiting on it. It is
      // clean now either way, so hand it back rather than renaming a block out
      // from under them, and look for another victim.
      acquire(&bcache.lock);
      if(b->refcnt != 1){
        b->refcnt--;
        FSSTAT_DROP(fsstat.buf_pinned);
        release(&bcache.lock);
        releasesleep(&b->lock);
        goto retry;
      }
      rename_buf(b, dev, blockno);
      b->flags = 0;
      release(&bcache.lock);
      return b;                 // b->lock still held, refcnt 1
    }
  }
  panic("bget: no buffers");
}

// Return a locked buf with the contents of the indicated block.
struct buf*
bread(uint dev, uint64 blockno)
{
  struct buf *b;

  fsstat.bread_calls++;
  b = bget(dev, blockno);
  if(!(b->flags & B_VALID)) {
    fsstat.bread_miss++;
    bdevrw(b);
  }
  return b;
}

// A locked buf for a block the caller is about to overwrite whole, without
// reading what is there first: Linux's getblk(). The caller must fill every
// byte and then bdirty() it.
//
// A copy-on-write filesystem writes almost nothing but blocks it has just
// allocated, and reading each one off the disk only to replace it was every
// fourth disk operation of a SchFS write.
struct buf*
bgetblk(uint dev, uint64 blockno)
{
  fsstat.bread_calls++;
  return bget(dev, blockno);
}

// Write b's contents to disk.  Must be locked.
void
bwrite(struct buf *b)
{
  if(!holdingsleep(&b->lock))
    panic("bwrite");
  b->flags |= B_DIRTY;
  bdevrw(b);
}

// Mark b modified but do not write it. The buffer stays dirty in the cache
// until something evicts it or bflush() comes past.
//
// This is what makes a copy-on-write filesystem affordable. SchFS's B-tree
// seals and writes a node on every single record insert, so a hot leaf -- the
// one collecting a file's data checksums, say -- was written to the disk once
// per 4 KB of file. Measured over 1 MB: 1288 block writes for 256 blocks of
// data. Deferring collapses every one of those repeats into the last one.
//
// It is only safe because the filesystem asks for the ordering it needs
// explicitly, at commit, instead of assuming that write order is medium order.
// A filesystem that did assume that would be broken here -- and was already
// broken on any drive with a write cache, which is all of them.
void
bdirty(struct buf *b)
{
  if(!holdingsleep(&b->lock))
    panic("bdirty");
  b->flags |= B_DIRTY;
  b->flags |= B_VALID;
}

// Push every dirty buffer for `dev` out, then make the device itself commit
// what it has taken. Returns 0, or -1 if the device could not confirm it.
//
// Walks the list of every buffer rather than the MRU list: brelse relinks
// buffers as it goes, and creation order is the one order that cannot change
// underneath the walk. The unlocked look first is only a filter -- the lock is
// taken before anything is believed -- and it is what keeps a flush cheap when
// tens of thousands of buffers are clean.
int
bflush(uint dev)
{
  struct buf *b;

  for(b = bcache.all; b; b = b->anext){
    if(b->dev != dev || !(b->flags & B_DIRTY))
      continue;
    acquire(&bcache.lock);
    if(b->dev != dev || !(b->flags & B_DIRTY)){
      release(&bcache.lock);
      continue;
    }
    b->refcnt++;
    release(&bcache.lock);

    acquiresleep(&b->lock);
    if(b->flags & B_DIRTY)      // recheck: an eviction may have beaten us to it
      bdevrw(b);
    releasesleep(&b->lock);

    acquire(&bcache.lock);
    b->refcnt--;
    release(&bcache.lock);
  }

  return bdevflush(dev);
}

// Release a locked buffer.
// Move to the head of the MRU list.
void
brelse(struct buf *b)
{
  if(!holdingsleep(&b->lock))
    panic("brelse");

  releasesleep(&b->lock);

  acquire(&bcache.lock);
  b->refcnt--;
  if (b->refcnt == 0) {
    FSSTAT_DROP(fsstat.buf_pinned);
    // no one is waiting for it.
    b->next->prev = b->prev;
    b->prev->next = b->next;
    b->next = bcache.head.next;
    b->prev = &bcache.head;
    bcache.head.next->prev = b;
    bcache.head.next = b;
  }

  release(&bcache.lock);
}
//PAGEBREAK!