SchOS/kernel/dev/bio.c

// Buffer cache.
//
// The buffer cache is a linked list of buf structures holding
// cached copies of disk block contents.  Caching disk blocks
// in memory reduces the number of disk reads and also provides
// a synchronization point for disk blocks used by multiple processes.
//
// Interface:
// * To get a buffer for a particular disk block, call bread.
// * After changing buffer data, call bwrite to write it to disk.
// * When done with the buffer, call brelse.
// * Do not use the buffer after calling brelse.
// * Only one process at a time can use a buffer,
//     so do not keep them longer than necessary.
//
// The implementation uses two state flags internally:
// * B_VALID: the buffer data has been read from the disk.
// * B_DIRTY: the buffer data has been modified
//     and needs to be written to disk.

#include "types.h"
#include "defs.h"
#include "param.h"
#include "spinlock.h"
#include "sleeplock.h"
#include "fs.h"
#include "buf.h"
#include "x86.h"
#include "fsstat.h"
#include "blkdev.h"

struct fsstat fsstat;

struct {
  struct spinlock lock;
  struct buf buf[NBUF];

  // Linked list of all buffers, through prev/next.
  // head.next is most recently used.
  struct buf head;
} bcache;

void
binit(void)
{
  struct buf *b;

  initlock(&bcache.lock, "bcache");
//PAGEBREAK!

  // Create linked list of buffers
  bcache.head.prev = &bcache.head;
  bcache.head.next = &bcache.head;
  for(b = bcache.buf; b < bcache.buf+NBUF; b++){
    b->next = bcache.head.next;
    b->prev = &bcache.head;
    initsleeplock(&b->lock, "buffer");
    bcache.head.next->prev = b;
    bcache.head.next = b;
  }
}

// Route a request to the driver that owns b->dev. Everything above this line
// is device-agnostic; everything below picks a backend. RAMDISK_DEV is served
// out of memory (kernel/dev/ramdisk.c), USBDISK_DEV by bulk transfers to a
// USB stick, everything else by the IDE driver.
static void bdevrw_inner(struct buf *b);

static void
bdevrw(struct buf *b)
{
  uint64 t0 = rdtsc();
  if(b->flags & B_DIRTY) fsstat.disk_writes++; else fsstat.disk_reads++;

  bdevrw_inner(b);
  fsstat.disk_cycles += rdtsc() - t0;
}

static void
bdevrw_inner(struct buf *b)
{
  if(b->dev == RAMDISK_DEV){
    ramdiskrw(b);
    return;
  }
  if(b->dev == USBDISK_DEV){
    usb_storage_rw(b);
    return;
  }
  // SATA, NVMe and every partition: registered at probe time rather than
  // known at build time. See include/blkdev.h.
  if(b->dev >= BLKDEV_FIRST){
    blkdev_rw(b);
    return;
  }
  iderw(b);
}

// Tell the device to commit whatever it has accepted but not yet made durable.
// Same dispatch as bdevrw, and the same rule: everything above this line is
// device-agnostic.
static int
bdevflush(uint dev)
{
  if(dev == RAMDISK_DEV)
    return 0;                   // memory has no volatile stage below it
  if(dev == USBDISK_DEV)
    return usb_storage_flush();
  if(dev >= BLKDEV_FIRST)
    return blkdev_flush(dev);
  return ideflush(dev);
}

// Look through buffer cache for block on device dev.
// If not found, allocate a buffer.
// In either case, return locked buffer.
// Forget every cached block of a device that is no longer there: clean and
// invalid, so it is neither served to a reader nor written back on eviction.
// A buffer somebody is holding is left to them -- they are about to find out.
void
binval(uint dev)
{
  struct buf *b;

  acquire(&bcache.lock);
  for(b = bcache.head.next; b != &bcache.head; b = b->next)
    if(b->dev == dev && b->refcnt == 0)
      b->flags = 0;
  release(&bcache.lock);
}

static struct buf*
bget(uint dev, uint64 blockno)
{
  struct buf *b;

 retry:
  acquire(&bcache.lock);

  // Is the block already cached?
  for(b = bcache.head.next; b != &bcache.head; b = b->next){
    if(b->dev == dev && b->blockno == blockno){
      if(b->refcnt++ == 0)
        FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak);
      release(&bcache.lock);
      acquiresleep(&b->lock);
      return b;
    }
  }

  // Not cached; recycle some unused buffer and clean buffer
  // "clean" because B_DIRTY and not locked means log.c
  // hasn't yet committed the changes to the buffer.
  for(b = bcache.head.prev; b != &bcache.head; b = b->prev){
    if(b->refcnt == 0 && (b->flags & B_DIRTY) == 0) {
      b->dev = dev;
      b->blockno = blockno;
      b->flags = 0;
      b->refcnt = 1;
      FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak);
      release(&bcache.lock);
      acquiresleep(&b->lock);
      return b;
    }
  }

  // Every buffer is dirty and none is clean enough to take.
  //
  // This used to be a straight panic, which was right while every writer went
  // through bwrite and a dirty buffer therefore meant "the log has not
  // committed yet". SchFS marks tree blocks dirty and leaves them there (see
  // bdirty), so an all-dirty cache is now an ordinary state: a transaction that
  // touches more than NBUF blocks reaches it long before it commits.
  //
  // Writing one out early is always safe. The only ordering the format needs is
  // that every block of the new tree precede the uberblock that names it, and
  // the uberblock goes down after an explicit bflush -- so nothing here can
  // land on the wrong side of that barrier.
  for(b = bcache.head.prev; b != &bcache.head; b = b->prev){
    if(b->refcnt == 0){
      fsstat.bget_evict_dirty++;
      b->refcnt++;              // claim it, so nobody else recycles it either
      FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak);
      release(&bcache.lock);
      acquiresleep(&b->lock);

      if(b->flags & B_DIRTY)
        bdevrw(b);

      // Between dropping bcache.lock and taking b's, another process may have
      // found this buffer under its *old* identity and be waiting on it. It is
      // clean now either way, so hand it back rather than renaming a block out
      // from under them, and look for another victim.
      acquire(&bcache.lock);
      if(b->refcnt != 1){
        b->refcnt--;
        FSSTAT_DROP(fsstat.buf_pinned);
        release(&bcache.lock);
        releasesleep(&b->lock);
        goto retry;
      }
      b->dev = dev;
      b->blockno = blockno;
      b->flags = 0;
      release(&bcache.lock);
      return b;                 // b->lock still held, refcnt 1
    }
  }
  panic("bget: no buffers");
}

// Return a locked buf with the contents of the indicated block.
struct buf*
bread(uint dev, uint64 blockno)
{
  struct buf *b;

  fsstat.bread_calls++;
  b = bget(dev, blockno);
  if(!(b->flags & B_VALID)) {
    fsstat.bread_miss++;
    bdevrw(b);
  }
  return b;
}

// A locked buf for a block the caller is about to overwrite whole, without
// reading what is there first: Linux's getblk(). The caller must fill every
// byte and then bdirty() it.
//
// A copy-on-write filesystem writes almost nothing but blocks it has just
// allocated, and reading each one off the disk only to replace it was every
// fourth disk operation of a SchFS write.
struct buf*
bgetblk(uint dev, uint64 blockno)
{
  fsstat.bread_calls++;
  return bget(dev, blockno);
}

// Write b's contents to disk.  Must be locked.
void
bwrite(struct buf *b)
{
  if(!holdingsleep(&b->lock))
    panic("bwrite");
  b->flags |= B_DIRTY;
  bdevrw(b);
}

// Mark b modified but do not write it. The buffer stays dirty in the cache
// until something evicts it or bflush() comes past.
//
// This is what makes a copy-on-write filesystem affordable. SchFS's B-tree
// seals and writes a node on every single record insert, so a hot leaf -- the
// one collecting a file's data checksums, say -- was written to the disk once
// per 4 KB of file. Measured over 1 MB: 1288 block writes for 256 blocks of
// data. Deferring collapses every one of those repeats into the last one.
//
// It is only safe because the filesystem asks for the ordering it needs
// explicitly, at commit, instead of assuming that write order is medium order.
// A filesystem that did assume that would be broken here -- and was already
// broken on any drive with a write cache, which is all of them.
void
bdirty(struct buf *b)
{
  if(!holdingsleep(&b->lock))
    panic("bdirty");
  b->flags |= B_DIRTY;
  b->flags |= B_VALID;
}

// Push every dirty buffer for `dev` out, then make the device itself commit
// what it has taken. Returns 0, or -1 if the device could not confirm it.
//
// Walks the array rather than the MRU list: brelse relinks buffers as it goes,
// and the array is the one order that cannot change underneath the walk.
int
bflush(uint dev)
{
  struct buf *b;

  for(b = bcache.buf; b < bcache.buf+NBUF; b++){
    acquire(&bcache.lock);
    if(b->dev != dev || !(b->flags & B_DIRTY)){
      release(&bcache.lock);
      continue;
    }
    b->refcnt++;
    release(&bcache.lock);

    acquiresleep(&b->lock);
    if(b->flags & B_DIRTY)      // recheck: an eviction may have beaten us to it
      bdevrw(b);
    releasesleep(&b->lock);

    acquire(&bcache.lock);
    b->refcnt--;
    release(&bcache.lock);
  }

  return bdevflush(dev);
}

// Release a locked buffer.
// Move to the head of the MRU list.
void
brelse(struct buf *b)
{
  if(!holdingsleep(&b->lock))
    panic("brelse");

  releasesleep(&b->lock);

  acquire(&bcache.lock);
  b->refcnt--;
  if (b->refcnt == 0) {
    FSSTAT_DROP(fsstat.buf_pinned);
    // no one is waiting for it.
    b->next->prev = b->prev;
    b->prev->next = b->next;
    b->next = bcache.head.next;
    b->prev = &bcache.head;
    bcache.head.next->prev = b;
    bcache.head.next = b;
  }

  release(&bcache.lock);
}
//PAGEBREAK!