ramdisk: the live root grows into half of memory

The live root was a fixed 96 MB image in the kernel: 60 MB of zeros on
the ISO, twice, and room for 60 MB of packages whatever the machine had.

Now the image is only what the root holds, plus slack, measured by
building it once large. Past its end the ramdisk goes on to half of the
memory free at mount, as pages taken the first time each block is
written; an unwritten block reads as zeros and costs nothing. SchFS
keeps no free-space map on disk, so growing a volume is a larger total
before open builds the map: grow_to, set for the ramdisk root.

8 GB machine: a 4 GB root, Xorg installed into it in 77 s with 7.6 GB
still free. The ISO is 172 MB instead of 344.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4

Makefile

@@ -461,12 +461,14 @@ CFLAGS_SYS = $(CFLAGS) -I./kernel/sys
 # loads that by PIO one sector at a time -- a kernel with a userland inside it
 # takes visibly longer to boot than a 350 KB one.
 #
-# 24576 blocks of 4 KB is 96 MB: the apk root's base system is 35 MB, and the rest is
-# room for `apk add` on the live system. The ramdisk is part of the kernel file
-# and so of the ISO, empty space included; a system that installs a desktop
-# wants a disk.
+# Sized to fit, not fixed. The image is what the root uses plus RAMDISK_SLACK
+# blocks and an eighth, measured by building it once large; the kernel then
+# grows the volume into half of the free memory at boot (kernel/dev/ramdisk.c),
+# which is where `apk add` on the live system writes. It was a fixed 96 MB,
+# 60 of it zeros carried in the kernel and twice over on the ISO, and room for
+# 60 MB of packages whatever the machine had.
 # -----------------------------------------------------------------------
-RAMDISK_BLOCKS ?= 24576
+RAMDISK_SLACK ?= 1024
 
 # RAMDISK_NINODES used to live here, sizing mkfs's preallocated inode table.
 # The ramdisk is a SchFS image now (see $(BUILD)/fs-ram.img) and SchFS allocates
@@ -825,7 +827,7 @@ $(BUILD)/kernel/kernel: $(BUILD)/kernel/kernel.pass1 $(BUILD)/kernel/ksym.o
 	$(OBJDUMP) -S $@ > $(BUILD)/kernel/kernel.asm
 
 # The ramdisk root image, and the object wrapping it for the linker. Built from
-# the same staged sysroot as the main image, just capped at RAMDISK_BLOCKS --
+# the same staged sysroot as the main image, sized to fit (see RAMDISK_SLACK) --
 # mkfs's content dedup means the whole userland still fits in a few MB.
 # Built with mkschfs, not mkfs.
 #
@@ -837,7 +839,12 @@ $(BUILD)/kernel/kernel: $(BUILD)/kernel/kernel.pass1 $(BUILD)/kernel/ksym.o
 $(BUILD)/fs-ram.img: $(MKSCHFS) $(ROOT_STAMP)
 	@mkdir -p $(BUILD)
 	@rm -f $@
-	$(MKSCHFS) $@ --size $(RAMDISK_BLOCKS) --label schos \
+	@used=$$($(MKSCHFS) $@ --size 262144 --label schos \
+	    $(RAMDISK_EXCLUDE_ARGS) --sysroot $(ROOT_TREE) | \
+	    sed -n 's/^ *\([0-9][0-9]*\) of [0-9][0-9]* blocks used.*/\1/p'); \
+	test -n "$$used" || { echo "fs-ram.img: mkschfs did not say how much it used" >&2; exit 1; }; \
+	rm -f $@; \
+	$(MKSCHFS) $@ --size $$((used + used / 8 + $(RAMDISK_SLACK))) --label schos \
 	    $(RAMDISK_EXCLUDE_ARGS) --sysroot $(ROOT_TREE)
 
 $(BUILD)/kernel/ramdisk_obj: $(BUILD)/fs-ram.img

docs/plan/alpine.md

@@ -39,7 +39,7 @@ only what is its own (see `tools/posix/mkaroot.sh`):
 ## Building
 
     make qemu           boots build/schfs.img, the root as a 1 GB SchFS disk
-    make iso            the same root as the live system's 96 MB ramdisk
+    make iso            the same root as the live system's ramdisk
     make alpine-lock    re-resolve world against Alpine (needs the network)
 
 It is the only root. sysroot/ is what schos-base is packaged from.
@@ -68,8 +68,10 @@ All of it found by running apk:
 
 ## Next
 
-- The live ISO runs the apk root from RAM: `apk add` works, into 59 MB, and
-  is gone at reboot. Keeping packages needs an installer that writes the root
+- The live ISO runs the apk root from RAM. The image in the kernel is only
+  what the root holds; at boot the volume grows into half of the free memory
+  (4 GB of root on an 8 GB machine), backed by pages as they are written.
+  Xorg installs into it in about a minute. All of it is gone at reboot. Keeping packages needs an installer that writes the root
   to a SATA or NVMe disk, and a kernel that finds its root there by the
   SchFS GPT type rather than as IDE disk 1.
 - Xorg from apk (`xorg-server xf86-video-fbdev xf86-input-evdev`) instead of

include/core/defs.h

@@ -368,6 +368,7 @@ void            input_mouse(int dx, int dy, int buttons);
 void            ramdiskinit(void);
 void            ramdiskrw(struct buf*);
 int             ramdisk_present(void);
+uint64          ramdisk_blocks(void);
 
 // usb_storage.c
 void            usb_storage_rw(struct buf*);

kernel/dev/ramdisk.c

@@ -14,6 +14,19 @@
 //
 // Enabled by building with RAMROOT=1. Without it this file still compiles, the
 // device reports zero size, and the kernel boots from the IDE disk as before.
+//
+// The device is larger than the image. The image is only what the root holds,
+// with a little slack; past its end the device goes on to half of the memory
+// free at mount time, and those blocks are pages taken from kalloc the first
+// time each is written -- a block nobody has written reads as zeros and costs
+// nothing. SchFS grows the volume to the device's size when it mounts (see
+// grow_to in kernel/fs/schfs/vol.c), so `apk add` on a live system has as
+// much room as the machine has memory to give, rather than whatever empty
+// space was built into the ISO. The ISO used to carry 60 MB of zeros for it.
+//
+// Pages are not given back when SchFS frees a block: there is no discard.
+// A freed block is reused by the next write, so the cost is the high-water
+// mark, not a leak.
 
 #include "core/types.h"
 #include "core/defs.h"
@@ -33,11 +46,23 @@ extern uchar _binary_ramdisk_size[];
 
 static uchar *rd_base;
 static uint64 rd_size;     // bytes
-static uint   rd_blocks;   // rd_size / BSIZE
+static uint   rd_blocks;   // rd_size / BSIZE: the image, served in place
+
+// Past the image: rd_dir[i] is a page of pointers, entry j the page holding
+// block rd_blocks + i*RD_PTRS + j, or 0 for one never written. 32768 pages of
+// 512 pointers reach 64 GB, which is as far as half of any machine this runs
+// on is likely to go, and costs 256 KB of pointers here.
+#define RD_PTRS   (PGSIZE / sizeof(uchar *))
+#define RD_DIRS   32768
+static uchar **rd_dir[RD_DIRS];
+static uint64 rd_total;     // blocks, image and the rest; 0 until sized
+static uint64 rd_pages;     // pages taken for blocks past the image
+static struct spinlock rd_lock;
 
 void
 ramdiskinit(void)
 {
+  initlock(&rd_lock, "ramdisk");
 #ifdef RAMDISK_ROOT
   rd_base  = _binary_ramdisk_start;
   rd_size  = (uint64)_binary_ramdisk_size;
@@ -66,6 +91,53 @@ ramdisk_present(void)
   return rd_blocks != 0;
 }
 
+// The device's size in blocks, fixed the first time anyone asks -- which is
+// SchFS mounting the root, after every page has reached the allocator. Asked
+// earlier, during ramdiskinit, the free count would be the first few
+// megabytes kinit1 handed over and nothing more.
+uint64
+ramdisk_blocks(void)
+{
+  if (rd_blocks == 0)
+    return 0;
+  acquire(&rd_lock);
+  if (rd_total == 0) {
+    uint64 extra = kalloc_free_bytes() / 2 / BSIZE;
+    if (extra > (uint64)RD_DIRS * RD_PTRS)
+      extra = (uint64)RD_DIRS * RD_PTRS;
+    rd_total = rd_blocks + extra;
+    cprintf("ramdisk: %u MB image, %u MB of memory to grow into\n",
+            (uint)(rd_size >> 20), (uint)(extra * BSIZE >> 20));
+  }
+  release(&rd_lock);
+  return rd_total;
+}
+
+// The page behind a block past the image, made when `make` says so.
+static uchar *
+rd_page(uint64 blk, int make)
+{
+  uint64 i = (blk - rd_blocks) / RD_PTRS, j = (blk - rd_blocks) % RD_PTRS;
+  uchar *pg = 0;
+
+  acquire(&rd_lock);
+  if (rd_dir[i] == 0 && make) {
+    if ((rd_dir[i] = (uchar **)kalloc()) != 0)
+      memset(rd_dir[i], 0, PGSIZE);
+  }
+  if (rd_dir[i]) {
+    pg = rd_dir[i][j];
+    if (pg == 0 && make && (pg = (uchar *)kalloc()) != 0) {
+      rd_dir[i][j] = pg;
+      rd_pages++;
+    }
+  }
+  release(&rd_lock);
+  if (make && pg == 0)
+    panic("ramdisk: out of memory for the live root");
+  return pg;
+}
+
 // Sync buf with the in-memory image. Mirrors iderw()'s contract: on a read,
 // fill b->data and set B_VALID; on a write, copy out and clear B_DIRTY.
 void
@@ -77,8 +149,21 @@ ramdiskrw(struct buf *b)
     panic("ramdiskrw: nothing to do");
   if(rd_base == 0)
     panic("ramdiskrw: no ramdisk");
-  if(b->blockno >= rd_blocks)
+  if(b->blockno >= rd_blocks){
+    if(b->blockno >= ramdisk_blocks())
       panic("ramdiskrw: block out of range");
+    uchar *pg = rd_page(b->blockno, (b->flags & B_DIRTY) != 0);
+    if(b->flags & B_DIRTY){
+      memmove(pg, b->data, BSIZE);
+      b->flags &= ~B_DIRTY;
+    } else if(pg){
+      memmove(b->data, pg, BSIZE);
+    } else {
+      memset(b->data, 0, BSIZE);    // never written
+    }
+    b->flags |= B_VALID;
+    return;
+  }
 
   uchar *p = rd_base + (uint64)b->blockno * BSIZE;
 

kernel/fs/schfs/kmount.c

@@ -641,6 +641,9 @@ kmount_slot(int slot, uint bdev)
   // Opening with a page allocator also builds the allocation bitmap, by walking
   // the tree — the only mount-time cost SchFS has, and what buys it a mount
   // that cannot disagree with the filesystem it just adopted.
+  // The live root fills its ramdisk, however much memory that turned out to
+  // be. See kernel/dev/ramdisk.c.
+  m->vol.grow_to = (bdev == RAMDISK_DEV) ? ramdisk_blocks() : 0;
   int r = schfs_vol_open(&m->vol, &bd, &cx, kd_palloc, kd_pfree);
   if (r == -1)
     return -EINVAL;              /* not a SchFS volume at all */

kernel/fs/schfs/schfs_int.h

@@ -104,6 +104,13 @@ struct schfs_vol {
 
     uint64_t total_blocks;
     uint64_t free_blocks;
+    // Set by the opener before schfs_vol_open: the device's size, when the
+    // volume should grow to fill it. Nothing on disk records which blocks are
+    // free -- open rebuilds that from the tree -- so growing is only a larger
+    // total before the map is built, and the next commit writes it down. The
+    // live root uses it to fill a ramdisk sized from memory at boot. 0, and
+    // anything not larger than the volume, leaves it alone.
+    uint64_t grow_to;
     uint64_t next_objectid;
     uint64_t root_objectid;
     uint64_t generation;        // last *committed* transaction group

kernel/fs/schfs/vol.c

@@ -215,6 +215,8 @@ schfs_vol_open(struct schfs_vol *v, struct schfs_bdev *dev,
         }
 
         vol_from_uber(v, u, best_slot);
+        if (v->grow_to > v->total_blocks)
+            v->total_blocks = v->grow_to;
         v->readonly = 1;            // until an allocation map exists, below
 
         if (v->dev.bread(v->dev.ctx, v->tree.root, x->a) != 0)