bio: a buffer cache that grows to 1/32 of RAM, looked up by hash

The cache was NBUF buffers, 1 MB, on every machine: enough for one with
tens of megabytes and nothing on one with gigabytes, which re-read the
same directory and binary blocks off the disk for every command. It
still starts as those 256 in the kernel image, and a miss with every
buffer busy now makes a new one rather than evicting, up to 1/32 of RAM
or 256 MB. Growing on demand means nothing is spent that is not used,
and binit() needs nothing from the page allocator.

Lookups go through a hash on (dev, blockno) instead of walking the LRU
list, and bflush() walks a creation-order list with an unlocked filter
first, so neither costs more at 65536 buffers than it did at 256.
struct buf's data is a pointer to its own page now; the drivers copy
through their own DMA buffers, so where it sits does not matter.

On a 2 GB QEMU guest: a 40 MB file read back in 0.89 s instead of 1.83,
the big binaries in 0.17 s instead of 0.26. /proc/fsstat shows the
cache as buf_cached now / most.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4

docs/architecture/filesystem.md

@@ -102,9 +102,21 @@ every `*at()` call uses them.
 
 ## The buffer cache
 
-`kernel/dev/bio.c`, xv6's design unchanged: a fixed pool of buffers, an LRU
+`kernel/dev/bio.c`, xv6's interface: `bread()` returns a locked buffer,
-list, one sleeplock per buffer. `bread()` returns a locked buffer; `bwrite()`
+`bwrite()` writes it, `bdirty()` marks it for later, `brelse()` releases. One
-flushes it; `brelse()` releases.
+sleeplock per buffer and an LRU list for eviction.
+
+**It grows with the machine.** It starts as the 256 buffers in the kernel
+image (1 MB), and a miss with every buffer busy makes a new one instead of
+evicting, until the cache is 1/32 of RAM, at most 256 MB: 32 MB on 1 GB, 256 MB
+from 8 GB up. It never shrinks, so the fraction is kept small. Lookups go
+through a hash on `(dev, blockno)`, not the LRU walk, so a 64k-buffer cache
+costs no more per `bread()` than a 256-buffer one. `/proc/fsstat` shows
+`buf_cached` as now / most.
+
+On a 2 GB machine under QEMU, reading a 40 MB file back went from 1.83 s to
+0.89 s, and `ls -R /` and the big binaries are each served from the cache on the
+second run instead of from the disk.
 
 **Sleeping is the point.** A buffer lock is held across a disk read, which is
 why every path that touches a filesystem must be able to sleep — and why the

include/buf.h

@@ -11,8 +11,16 @@ struct buf {
   struct buf *prev; // LRU cache list
   struct buf *next;
   struct buf *qnext; // disk queue
-  uchar data[BSIZE];
+  struct buf *hnext; // hash chain, by (dev, blockno)
+  struct buf *anext; // every buffer, in the order made; never reordered
+  // BSIZE bytes, a page of its own. A pointer rather than an array since the
+  // cache grows at run time (see bio.c); drivers copy through their own DMA
+  // buffers, so where the page sits does not matter to them.
+  uchar *data;
 };
 #define B_VALID 0x2  // buffer has been read from disk
 #define B_DIRTY 0x4  // buffer needs to be written to disk
 
+
+// How many buffers the cache holds now, and the most it will grow to. bio.c.
+void bcache_size(uint *n, uint *max);

kernel/dev/bio.c

@@ -29,8 +29,30 @@
 #include "fsstat.h"
 #include "blkdev.h"
 
+#include "kmem.h"
+#include "hwinfo.h"
+
 struct fsstat fsstat;
 
+// The cache starts as NBUF buffers in the kernel image, 1 MB, and grows on
+// demand: a miss with every buffer in use for something makes a new one
+// rather than evicting, until the cache is 1/32 of RAM (at most 256 MB).
+//
+// It was NBUF and nothing else, on every machine. 1 MB is a fine floor for
+// a machine with tens of megabytes and nothing at all for one with sixteen
+// gigabytes, where it was re-reading the same directory blocks off the disk
+// for every command. Growing rather than sizing at boot means a small machine,
+// or a large one doing little, spends nothing it does not use -- and that
+// binit() needs nothing from the page allocator that early.
+//
+// Lookups go through a hash on (dev, blockno). The LRU list was also the
+// lookup structure, a linear walk per bread(), which is fine at 256 and
+// hopeless at 65536.
+#define BUF_MAX_BYTES  (256ULL * 1024 * 1024)
+#define NHASH          8192            // a power of two
+
+static uchar pooldata[NBUF][BSIZE];
+
 struct {
   struct spinlock lock;
   struct buf buf[NBUF];
@@ -38,28 +60,117 @@ struct {
   // Linked list of all buffers, through prev/next.
   // head.next is most recently used.
   struct buf head;
+
+  struct buf *hash[NHASH];
+  struct buf *all;            // through anext
+  struct kmem_cache *hdrs;
+  uint  n;                    // buffers that exist
+  uint  max;                  // how many there may be; 0 until first needed
 } bcache;
 
-void
+static inline uint
-binit(void)
+bhash(uint dev, uint64 blockno)
 {
-  struct buf *b;
+  uint64 h = (blockno ^ ((uint64)dev << 40)) * 0x9E3779B97F4A7C15ULL;
+  return (uint)(h >> 51) & (NHASH - 1);
+}
 
-  initlock(&bcache.lock, "bcache");
+// Both under bcache.lock.
-//PAGEBREAK!
+static void
+hash_del(struct buf *b)
+{
+  struct buf **pp = &bcache.hash[bhash(b->dev, b->blockno)];
+  while (*pp && *pp != b)
+    pp = &(*pp)->hnext;
+  if (*pp)
+    *pp = b->hnext;
+  b->hnext = 0;
+}
 
-  // Create linked list of buffers
+static void
-  bcache.head.prev = &bcache.head;
+hash_add(struct buf *b)
-  bcache.head.next = &bcache.head;
+{
-  for(b = bcache.buf; b < bcache.buf+NBUF; b++){
+  struct buf **pp = &bcache.hash[bhash(b->dev, b->blockno)];
+  b->hnext = *pp;
+  *pp = b;
+}
+
+// Give b a new identity, moving it between chains. Under bcache.lock.
+static void
+rename_buf(struct buf *b, uint dev, uint64 blockno)
+{
+  hash_del(b);
+  b->dev = dev;
+  b->blockno = blockno;
+  hash_add(b);
+}
+
+static void
+link_new(struct buf *b)
+{
   b->next = bcache.head.next;
   b->prev = &bcache.head;
   initsleeplock(&b->lock, "buffer");
   bcache.head.next->prev = b;
   bcache.head.next = b;
+  b->anext = bcache.all;
+  bcache.all = b;
+  // An identity no block has, so the first rename has a chain to leave.
+  b->dev = (uint)-1;
+  b->blockno = (uint64)bcache.n;
+  hash_add(b);
+  bcache.n++;
+}
+
+void
+binit(void)
+{
+  initlock(&bcache.lock, "bcache");
+
+  bcache.head.prev = &bcache.head;
+  bcache.head.next = &bcache.head;
+  for (int i = 0; i < NBUF; i++) {
+    bcache.buf[i].data = pooldata[i];
+    link_new(&bcache.buf[i]);
   }
 }
 
+// One more buffer, or 0 at the cap or when memory is short. Under
+// bcache.lock; the allocators take only their own locks.
+static struct buf *
+grow(void)
+{
+  if (bcache.max == 0) {
+    uint64 want = ram_total_bytes() / 32;
+    if (want > BUF_MAX_BYTES)
+      want = BUF_MAX_BYTES;
+    bcache.max = (uint)(want / BSIZE);
+    if (bcache.max < NBUF)
+      bcache.max = NBUF;
+    bcache.hdrs = kmem_cache_create("buf", sizeof(struct buf));
+  }
+  if (bcache.n >= bcache.max || bcache.hdrs == 0)
+    return 0;
+  struct buf *b = kmem_cache_alloc(bcache.hdrs);
+  if (b == 0)
+    return 0;
+  memset(b, 0, sizeof *b);
+  if ((b->data = (uchar *)kalloc()) == 0) {
+    kmem_cache_free(bcache.hdrs, b);
+    return 0;
+  }
+  link_new(b);
+  return b;
+}
+
+// How many buffers there are, and may be. For /proc/fsstat.
+void
+bcache_size(uint *n, uint *max)
+{
+  *n = bcache.n;
+  *max = bcache.max ? bcache.max : NBUF;
+}
+
 // Route a request to the driver that owns b->dev. Everything above this line
 // is device-agnostic; everything below picks a backend. RAMDISK_DEV is served
 // out of memory (kernel/dev/ramdisk.c), USBDISK_DEV by bulk transfers to a
@@ -123,7 +234,7 @@ binval(uint dev)
   struct buf *b;
 
   acquire(&bcache.lock);
-  for(b = bcache.head.next; b != &bcache.head; b = b->next)
+  for(b = bcache.all; b; b = b->anext)
     if(b->dev == dev && b->refcnt == 0)
       b->flags = 0;
   release(&bcache.lock);
@@ -138,7 +249,7 @@ bget(uint dev, uint64 blockno)
   acquire(&bcache.lock);
 
   // Is the block already cached?
-  for(b = bcache.head.next; b != &bcache.head; b = b->next){
+  for(b = bcache.hash[bhash(dev, blockno)]; b; b = b->hnext){
     if(b->dev == dev && b->blockno == blockno){
       if(b->refcnt++ == 0)
         FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak);
@@ -148,13 +259,24 @@ bget(uint dev, uint64 blockno)
     }
   }
 
-  // Not cached; recycle some unused buffer and clean buffer
+  // Not cached. A new buffer while the cache may still grow, so nothing
+  // cached is thrown away for it.
+  if((b = grow()) != 0){
+    rename_buf(b, dev, blockno);
+    b->flags = 0;
+    b->refcnt = 1;
+    FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak);
+    release(&bcache.lock);
+    acquiresleep(&b->lock);
+    return b;
+  }
+
+  // Otherwise recycle some unused buffer and clean buffer
   // "clean" because B_DIRTY and not locked means log.c
   // hasn't yet committed the changes to the buffer.
   for(b = bcache.head.prev; b != &bcache.head; b = b->prev){
     if(b->refcnt == 0 && (b->flags & B_DIRTY) == 0) {
-      b->dev = dev;
+      rename_buf(b, dev, blockno);
-      b->blockno = blockno;
       b->flags = 0;
       b->refcnt = 1;
       FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak);
@@ -199,8 +321,7 @@ bget(uint dev, uint64 blockno)
         releasesleep(&b->lock);
         goto retry;
       }
-      b->dev = dev;
+      rename_buf(b, dev, blockno);
-      b->blockno = blockno;
       b->flags = 0;
       release(&bcache.lock);
       return b;                 // b->lock still held, refcnt 1
@@ -273,14 +394,19 @@ bdirty(struct buf *b)
 // Push every dirty buffer for `dev` out, then make the device itself commit
 // what it has taken. Returns 0, or -1 if the device could not confirm it.
 //
-// Walks the array rather than the MRU list: brelse relinks buffers as it goes,
+// Walks the list of every buffer rather than the MRU list: brelse relinks
-// and the array is the one order that cannot change underneath the walk.
+// buffers as it goes, and creation order is the one order that cannot change
+// underneath the walk. The unlocked look first is only a filter -- the lock is
+// taken before anything is believed -- and it is what keeps a flush cheap when
+// tens of thousands of buffers are clean.
 int
 bflush(uint dev)
 {
   struct buf *b;
 
-  for(b = bcache.buf; b < bcache.buf+NBUF; b++){
+  for(b = bcache.all; b; b = b->anext){
+    if(b->dev != dev || !(b->flags & B_DIRTY))
+      continue;
     acquire(&bcache.lock);
     if(b->dev != dev || !(b->flags & B_DIRTY)){
       release(&bcache.lock);

kernel/dev/procfs.c

@@ -30,6 +30,7 @@
 #include "spinlock.h"
 #include "sleeplock.h"
 #include "fs.h"
+#include "buf.h"
 #include "file.h"
 #include "proc.h"
 #include "uaccess.h"
@@ -299,8 +300,14 @@ gen_fsstat(struct sbuf *sb)
   sb_puts(sb, "\nfile_peak        "); sb_putu(sb, fsstat.file_peak);
   sb_puts(sb, " / "); sb_putu(sb, NFILE);
   sb_puts(sb, "\nbuf_pinned       "); sb_putu(sb, fsstat.buf_pinned);
+  {
+    uint nb, maxb;
+    bcache_size(&nb, &maxb);
     sb_puts(sb, "\nbuf_pinned_peak  "); sb_putu(sb, fsstat.buf_pinned_peak);
-  sb_puts(sb, " / "); sb_putu(sb, NBUF);
+    sb_puts(sb, " / "); sb_putu(sb, nb);
+    sb_puts(sb, "\nbuf_cached       "); sb_putu(sb, nb);
+    sb_puts(sb, " / "); sb_putu(sb, maxb);
+  }
 
   // The SchFS bracket. `locked 1` with `owner 0` is the wedge: the volume lock
   // is held by something that is not in a transaction, so nothing will ever