From 62d2a85ac8fdaa4dbb8c692e854a430994aecc4e Mon Sep 17 00:00:00 2001 From: Nexoniarz Date: Fri, 23 Oct 2026 13:28:12 +0200 Subject: [PATCH] bio: a buffer cache that grows to 1/32 of RAM, looked up by hash The cache was NBUF buffers, 1 MB, on every machine: enough for one with tens of megabytes and nothing on one with gigabytes, which re-read the same directory and binary blocks off the disk for every command. It still starts as those 256 in the kernel image, and a miss with every buffer busy now makes a new one rather than evicting, up to 1/32 of RAM or 256 MB. Growing on demand means nothing is spent that is not used, and binit() needs nothing from the page allocator. Lookups go through a hash on (dev, blockno) instead of walking the LRU list, and bflush() walks a creation-order list with an unlocked filter first, so neither costs more at 65536 buffers than it did at 256. struct buf's data is a pointer to its own page now; the drivers copy through their own DMA buffers, so where it sits does not matter. On a 2 GB QEMU guest: a 40 MB file read back in 0.89 s instead of 1.83, the big binaries in 0.17 s instead of 0.26. /proc/fsstat shows the cache as buf_cached now / most. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4 --- docs/architecture/filesystem.md | 18 +++- include/buf.h | 10 +- kernel/dev/bio.c | 166 ++++++++++++++++++++++++++++---- kernel/dev/procfs.c | 11 ++- 4 files changed, 179 insertions(+), 26 deletions(-) diff --git a/docs/architecture/filesystem.md b/docs/architecture/filesystem.md index a4aa433..4700982 100644 --- a/docs/architecture/filesystem.md +++ b/docs/architecture/filesystem.md @@ -102,9 +102,21 @@ every `*at()` call uses them. ## The buffer cache -`kernel/dev/bio.c`, xv6's design unchanged: a fixed pool of buffers, an LRU -list, one sleeplock per buffer. `bread()` returns a locked buffer; `bwrite()` -flushes it; `brelse()` releases. +`kernel/dev/bio.c`, xv6's interface: `bread()` returns a locked buffer, +`bwrite()` writes it, `bdirty()` marks it for later, `brelse()` releases. One +sleeplock per buffer and an LRU list for eviction. + +**It grows with the machine.** It starts as the 256 buffers in the kernel +image (1 MB), and a miss with every buffer busy makes a new one instead of +evicting, until the cache is 1/32 of RAM, at most 256 MB: 32 MB on 1 GB, 256 MB +from 8 GB up. It never shrinks, so the fraction is kept small. Lookups go +through a hash on `(dev, blockno)`, not the LRU walk, so a 64k-buffer cache +costs no more per `bread()` than a 256-buffer one. `/proc/fsstat` shows +`buf_cached` as now / most. + +On a 2 GB machine under QEMU, reading a 40 MB file back went from 1.83 s to +0.89 s, and `ls -R /` and the big binaries are each served from the cache on the +second run instead of from the disk. **Sleeping is the point.** A buffer lock is held across a disk read, which is why every path that touches a filesystem must be able to sleep — and why the diff --git a/include/buf.h b/include/buf.h index 3e094e6..4b81b4a 100644 --- a/include/buf.h +++ b/include/buf.h @@ -11,8 +11,16 @@ struct buf { struct buf *prev; // LRU cache list struct buf *next; struct buf *qnext; // disk queue - uchar data[BSIZE]; + struct buf *hnext; // hash chain, by (dev, blockno) + struct buf *anext; // every buffer, in the order made; never reordered + // BSIZE bytes, a page of its own. A pointer rather than an array since the + // cache grows at run time (see bio.c); drivers copy through their own DMA + // buffers, so where the page sits does not matter to them. + uchar *data; }; #define B_VALID 0x2 // buffer has been read from disk #define B_DIRTY 0x4 // buffer needs to be written to disk + +// How many buffers the cache holds now, and the most it will grow to. bio.c. +void bcache_size(uint *n, uint *max); diff --git a/kernel/dev/bio.c b/kernel/dev/bio.c index 7b6d68d..3777a35 100644 --- a/kernel/dev/bio.c +++ b/kernel/dev/bio.c @@ -29,8 +29,30 @@ #include "fsstat.h" #include "blkdev.h" +#include "kmem.h" +#include "hwinfo.h" + struct fsstat fsstat; +// The cache starts as NBUF buffers in the kernel image, 1 MB, and grows on +// demand: a miss with every buffer in use for something makes a new one +// rather than evicting, until the cache is 1/32 of RAM (at most 256 MB). +// +// It was NBUF and nothing else, on every machine. 1 MB is a fine floor for +// a machine with tens of megabytes and nothing at all for one with sixteen +// gigabytes, where it was re-reading the same directory blocks off the disk +// for every command. Growing rather than sizing at boot means a small machine, +// or a large one doing little, spends nothing it does not use -- and that +// binit() needs nothing from the page allocator that early. +// +// Lookups go through a hash on (dev, blockno). The LRU list was also the +// lookup structure, a linear walk per bread(), which is fine at 256 and +// hopeless at 65536. +#define BUF_MAX_BYTES (256ULL * 1024 * 1024) +#define NHASH 8192 // a power of two + +static uchar pooldata[NBUF][BSIZE]; + struct { struct spinlock lock; struct buf buf[NBUF]; @@ -38,28 +60,117 @@ struct { // Linked list of all buffers, through prev/next. // head.next is most recently used. struct buf head; + + struct buf *hash[NHASH]; + struct buf *all; // through anext + struct kmem_cache *hdrs; + uint n; // buffers that exist + uint max; // how many there may be; 0 until first needed } bcache; +static inline uint +bhash(uint dev, uint64 blockno) +{ + uint64 h = (blockno ^ ((uint64)dev << 40)) * 0x9E3779B97F4A7C15ULL; + return (uint)(h >> 51) & (NHASH - 1); +} + +// Both under bcache.lock. +static void +hash_del(struct buf *b) +{ + struct buf **pp = &bcache.hash[bhash(b->dev, b->blockno)]; + while (*pp && *pp != b) + pp = &(*pp)->hnext; + if (*pp) + *pp = b->hnext; + b->hnext = 0; +} + +static void +hash_add(struct buf *b) +{ + struct buf **pp = &bcache.hash[bhash(b->dev, b->blockno)]; + b->hnext = *pp; + *pp = b; +} + +// Give b a new identity, moving it between chains. Under bcache.lock. +static void +rename_buf(struct buf *b, uint dev, uint64 blockno) +{ + hash_del(b); + b->dev = dev; + b->blockno = blockno; + hash_add(b); +} + +static void +link_new(struct buf *b) +{ + b->next = bcache.head.next; + b->prev = &bcache.head; + initsleeplock(&b->lock, "buffer"); + bcache.head.next->prev = b; + bcache.head.next = b; + b->anext = bcache.all; + bcache.all = b; + // An identity no block has, so the first rename has a chain to leave. + b->dev = (uint)-1; + b->blockno = (uint64)bcache.n; + hash_add(b); + bcache.n++; +} + void binit(void) { - struct buf *b; - initlock(&bcache.lock, "bcache"); -//PAGEBREAK! - // Create linked list of buffers bcache.head.prev = &bcache.head; bcache.head.next = &bcache.head; - for(b = bcache.buf; b < bcache.buf+NBUF; b++){ - b->next = bcache.head.next; - b->prev = &bcache.head; - initsleeplock(&b->lock, "buffer"); - bcache.head.next->prev = b; - bcache.head.next = b; + for (int i = 0; i < NBUF; i++) { + bcache.buf[i].data = pooldata[i]; + link_new(&bcache.buf[i]); } } +// One more buffer, or 0 at the cap or when memory is short. Under +// bcache.lock; the allocators take only their own locks. +static struct buf * +grow(void) +{ + if (bcache.max == 0) { + uint64 want = ram_total_bytes() / 32; + if (want > BUF_MAX_BYTES) + want = BUF_MAX_BYTES; + bcache.max = (uint)(want / BSIZE); + if (bcache.max < NBUF) + bcache.max = NBUF; + bcache.hdrs = kmem_cache_create("buf", sizeof(struct buf)); + } + if (bcache.n >= bcache.max || bcache.hdrs == 0) + return 0; + struct buf *b = kmem_cache_alloc(bcache.hdrs); + if (b == 0) + return 0; + memset(b, 0, sizeof *b); + if ((b->data = (uchar *)kalloc()) == 0) { + kmem_cache_free(bcache.hdrs, b); + return 0; + } + link_new(b); + return b; +} + +// How many buffers there are, and may be. For /proc/fsstat. +void +bcache_size(uint *n, uint *max) +{ + *n = bcache.n; + *max = bcache.max ? bcache.max : NBUF; +} + // Route a request to the driver that owns b->dev. Everything above this line // is device-agnostic; everything below picks a backend. RAMDISK_DEV is served // out of memory (kernel/dev/ramdisk.c), USBDISK_DEV by bulk transfers to a @@ -123,7 +234,7 @@ binval(uint dev) struct buf *b; acquire(&bcache.lock); - for(b = bcache.head.next; b != &bcache.head; b = b->next) + for(b = bcache.all; b; b = b->anext) if(b->dev == dev && b->refcnt == 0) b->flags = 0; release(&bcache.lock); @@ -138,7 +249,7 @@ bget(uint dev, uint64 blockno) acquire(&bcache.lock); // Is the block already cached? - for(b = bcache.head.next; b != &bcache.head; b = b->next){ + for(b = bcache.hash[bhash(dev, blockno)]; b; b = b->hnext){ if(b->dev == dev && b->blockno == blockno){ if(b->refcnt++ == 0) FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak); @@ -148,13 +259,24 @@ bget(uint dev, uint64 blockno) } } - // Not cached; recycle some unused buffer and clean buffer + // Not cached. A new buffer while the cache may still grow, so nothing + // cached is thrown away for it. + if((b = grow()) != 0){ + rename_buf(b, dev, blockno); + b->flags = 0; + b->refcnt = 1; + FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak); + release(&bcache.lock); + acquiresleep(&b->lock); + return b; + } + + // Otherwise recycle some unused buffer and clean buffer // "clean" because B_DIRTY and not locked means log.c // hasn't yet committed the changes to the buffer. for(b = bcache.head.prev; b != &bcache.head; b = b->prev){ if(b->refcnt == 0 && (b->flags & B_DIRTY) == 0) { - b->dev = dev; - b->blockno = blockno; + rename_buf(b, dev, blockno); b->flags = 0; b->refcnt = 1; FSSTAT_TAKE(fsstat.buf_pinned, fsstat.buf_pinned_peak); @@ -199,8 +321,7 @@ bget(uint dev, uint64 blockno) releasesleep(&b->lock); goto retry; } - b->dev = dev; - b->blockno = blockno; + rename_buf(b, dev, blockno); b->flags = 0; release(&bcache.lock); return b; // b->lock still held, refcnt 1 @@ -273,14 +394,19 @@ bdirty(struct buf *b) // Push every dirty buffer for `dev` out, then make the device itself commit // what it has taken. Returns 0, or -1 if the device could not confirm it. // -// Walks the array rather than the MRU list: brelse relinks buffers as it goes, -// and the array is the one order that cannot change underneath the walk. +// Walks the list of every buffer rather than the MRU list: brelse relinks +// buffers as it goes, and creation order is the one order that cannot change +// underneath the walk. The unlocked look first is only a filter -- the lock is +// taken before anything is believed -- and it is what keeps a flush cheap when +// tens of thousands of buffers are clean. int bflush(uint dev) { struct buf *b; - for(b = bcache.buf; b < bcache.buf+NBUF; b++){ + for(b = bcache.all; b; b = b->anext){ + if(b->dev != dev || !(b->flags & B_DIRTY)) + continue; acquire(&bcache.lock); if(b->dev != dev || !(b->flags & B_DIRTY)){ release(&bcache.lock); diff --git a/kernel/dev/procfs.c b/kernel/dev/procfs.c index 0745f7c..c47f28d 100644 --- a/kernel/dev/procfs.c +++ b/kernel/dev/procfs.c @@ -30,6 +30,7 @@ #include "spinlock.h" #include "sleeplock.h" #include "fs.h" +#include "buf.h" #include "file.h" #include "proc.h" #include "uaccess.h" @@ -299,8 +300,14 @@ gen_fsstat(struct sbuf *sb) sb_puts(sb, "\nfile_peak "); sb_putu(sb, fsstat.file_peak); sb_puts(sb, " / "); sb_putu(sb, NFILE); sb_puts(sb, "\nbuf_pinned "); sb_putu(sb, fsstat.buf_pinned); - sb_puts(sb, "\nbuf_pinned_peak "); sb_putu(sb, fsstat.buf_pinned_peak); - sb_puts(sb, " / "); sb_putu(sb, NBUF); + { + uint nb, maxb; + bcache_size(&nb, &maxb); + sb_puts(sb, "\nbuf_pinned_peak "); sb_putu(sb, fsstat.buf_pinned_peak); + sb_puts(sb, " / "); sb_putu(sb, nb); + sb_puts(sb, "\nbuf_cached "); sb_putu(sb, nb); + sb_puts(sb, " / "); sb_putu(sb, maxb); + } // The SchFS bracket. `locked 1` with `owner 0` is the wedge: the volume lock // is held by something that is not in a transaction, so nothing will ever