From d0941780f57183f7848e40a945b474b2a3c833c3 Mon Sep 17 00:00:00 2001 From: Nexoniarz Date: Sat, 24 Oct 2026 17:20:43 +0200 Subject: [PATCH] ramdisk: the live root grows into half of memory The live root was a fixed 96 MB image in the kernel: 60 MB of zeros on the ISO, twice, and room for 60 MB of packages whatever the machine had. Now the image is only what the root holds, plus slack, measured by building it once large. Past its end the ramdisk goes on to half of the memory free at mount, as pages taken the first time each block is written; an unwritten block reads as zeros and costs nothing. SchFS keeps no free-space map on disk, so growing a volume is a larger total before open builds the map: grow_to, set for the ramdisk root. 8 GB machine: a 4 GB root, Xorg installed into it in 77 s with 7.6 GB still free. The ISO is 172 MB instead of 344. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4 --- Makefile | 21 ++++++--- docs/plan/alpine.md | 8 ++-- include/core/defs.h | 1 + kernel/dev/ramdisk.c | 91 +++++++++++++++++++++++++++++++++++-- kernel/fs/schfs/kmount.c | 3 ++ kernel/fs/schfs/schfs_int.h | 7 +++ kernel/fs/schfs/vol.c | 2 + 7 files changed, 120 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index d23f6c3..791c654 100644 --- a/Makefile +++ b/Makefile @@ -461,12 +461,14 @@ CFLAGS_SYS = $(CFLAGS) -I./kernel/sys # loads that by PIO one sector at a time -- a kernel with a userland inside it # takes visibly longer to boot than a 350 KB one. # -# 24576 blocks of 4 KB is 96 MB: the apk root's base system is 35 MB, and the rest is -# room for `apk add` on the live system. The ramdisk is part of the kernel file -# and so of the ISO, empty space included; a system that installs a desktop -# wants a disk. +# Sized to fit, not fixed. The image is what the root uses plus RAMDISK_SLACK +# blocks and an eighth, measured by building it once large; the kernel then +# grows the volume into half of the free memory at boot (kernel/dev/ramdisk.c), +# which is where `apk add` on the live system writes. It was a fixed 96 MB, +# 60 of it zeros carried in the kernel and twice over on the ISO, and room for +# 60 MB of packages whatever the machine had. # ----------------------------------------------------------------------- -RAMDISK_BLOCKS ?= 24576 +RAMDISK_SLACK ?= 1024 # RAMDISK_NINODES used to live here, sizing mkfs's preallocated inode table. # The ramdisk is a SchFS image now (see $(BUILD)/fs-ram.img) and SchFS allocates @@ -825,7 +827,7 @@ $(BUILD)/kernel/kernel: $(BUILD)/kernel/kernel.pass1 $(BUILD)/kernel/ksym.o $(OBJDUMP) -S $@ > $(BUILD)/kernel/kernel.asm # The ramdisk root image, and the object wrapping it for the linker. Built from -# the same staged sysroot as the main image, just capped at RAMDISK_BLOCKS -- +# the same staged sysroot as the main image, sized to fit (see RAMDISK_SLACK) -- # mkfs's content dedup means the whole userland still fits in a few MB. # Built with mkschfs, not mkfs. # @@ -837,7 +839,12 @@ $(BUILD)/kernel/kernel: $(BUILD)/kernel/kernel.pass1 $(BUILD)/kernel/ksym.o $(BUILD)/fs-ram.img: $(MKSCHFS) $(ROOT_STAMP) @mkdir -p $(BUILD) @rm -f $@ - $(MKSCHFS) $@ --size $(RAMDISK_BLOCKS) --label schos \ + @used=$$($(MKSCHFS) $@ --size 262144 --label schos \ + $(RAMDISK_EXCLUDE_ARGS) --sysroot $(ROOT_TREE) | \ + sed -n 's/^ *\([0-9][0-9]*\) of [0-9][0-9]* blocks used.*/\1/p'); \ + test -n "$$used" || { echo "fs-ram.img: mkschfs did not say how much it used" >&2; exit 1; }; \ + rm -f $@; \ + $(MKSCHFS) $@ --size $$((used + used / 8 + $(RAMDISK_SLACK))) --label schos \ $(RAMDISK_EXCLUDE_ARGS) --sysroot $(ROOT_TREE) $(BUILD)/kernel/ramdisk_obj: $(BUILD)/fs-ram.img diff --git a/docs/plan/alpine.md b/docs/plan/alpine.md index 824889c..6cf01d6 100644 --- a/docs/plan/alpine.md +++ b/docs/plan/alpine.md @@ -39,7 +39,7 @@ only what is its own (see `tools/posix/mkaroot.sh`): ## Building make qemu boots build/schfs.img, the root as a 1 GB SchFS disk - make iso the same root as the live system's 96 MB ramdisk + make iso the same root as the live system's ramdisk make alpine-lock re-resolve world against Alpine (needs the network) It is the only root. sysroot/ is what schos-base is packaged from. @@ -68,8 +68,10 @@ All of it found by running apk: ## Next -- The live ISO runs the apk root from RAM: `apk add` works, into 59 MB, and - is gone at reboot. Keeping packages needs an installer that writes the root +- The live ISO runs the apk root from RAM. The image in the kernel is only + what the root holds; at boot the volume grows into half of the free memory + (4 GB of root on an 8 GB machine), backed by pages as they are written. + Xorg installs into it in about a minute. All of it is gone at reboot. Keeping packages needs an installer that writes the root to a SATA or NVMe disk, and a kernel that finds its root there by the SchFS GPT type rather than as IDE disk 1. - Xorg from apk (`xorg-server xf86-video-fbdev xf86-input-evdev`) instead of diff --git a/include/core/defs.h b/include/core/defs.h index 94ce346..feaa099 100644 --- a/include/core/defs.h +++ b/include/core/defs.h @@ -368,6 +368,7 @@ void input_mouse(int dx, int dy, int buttons); void ramdiskinit(void); void ramdiskrw(struct buf*); int ramdisk_present(void); +uint64 ramdisk_blocks(void); // usb_storage.c void usb_storage_rw(struct buf*); diff --git a/kernel/dev/ramdisk.c b/kernel/dev/ramdisk.c index 055ad99..8736a75 100644 --- a/kernel/dev/ramdisk.c +++ b/kernel/dev/ramdisk.c @@ -14,6 +14,19 @@ // // Enabled by building with RAMROOT=1. Without it this file still compiles, the // device reports zero size, and the kernel boots from the IDE disk as before. +// +// The device is larger than the image. The image is only what the root holds, +// with a little slack; past its end the device goes on to half of the memory +// free at mount time, and those blocks are pages taken from kalloc the first +// time each is written -- a block nobody has written reads as zeros and costs +// nothing. SchFS grows the volume to the device's size when it mounts (see +// grow_to in kernel/fs/schfs/vol.c), so `apk add` on a live system has as +// much room as the machine has memory to give, rather than whatever empty +// space was built into the ISO. The ISO used to carry 60 MB of zeros for it. +// +// Pages are not given back when SchFS frees a block: there is no discard. +// A freed block is reused by the next write, so the cost is the high-water +// mark, not a leak. #include "core/types.h" #include "core/defs.h" @@ -33,11 +46,23 @@ extern uchar _binary_ramdisk_size[]; static uchar *rd_base; static uint64 rd_size; // bytes -static uint rd_blocks; // rd_size / BSIZE +static uint rd_blocks; // rd_size / BSIZE: the image, served in place + +// Past the image: rd_dir[i] is a page of pointers, entry j the page holding +// block rd_blocks + i*RD_PTRS + j, or 0 for one never written. 32768 pages of +// 512 pointers reach 64 GB, which is as far as half of any machine this runs +// on is likely to go, and costs 256 KB of pointers here. +#define RD_PTRS (PGSIZE / sizeof(uchar *)) +#define RD_DIRS 32768 +static uchar **rd_dir[RD_DIRS]; +static uint64 rd_total; // blocks, image and the rest; 0 until sized +static uint64 rd_pages; // pages taken for blocks past the image +static struct spinlock rd_lock; void ramdiskinit(void) { + initlock(&rd_lock, "ramdisk"); #ifdef RAMDISK_ROOT rd_base = _binary_ramdisk_start; rd_size = (uint64)_binary_ramdisk_size; @@ -66,6 +91,53 @@ ramdisk_present(void) return rd_blocks != 0; } +// The device's size in blocks, fixed the first time anyone asks -- which is +// SchFS mounting the root, after every page has reached the allocator. Asked +// earlier, during ramdiskinit, the free count would be the first few +// megabytes kinit1 handed over and nothing more. +uint64 +ramdisk_blocks(void) +{ + if (rd_blocks == 0) + return 0; + acquire(&rd_lock); + if (rd_total == 0) { + uint64 extra = kalloc_free_bytes() / 2 / BSIZE; + if (extra > (uint64)RD_DIRS * RD_PTRS) + extra = (uint64)RD_DIRS * RD_PTRS; + rd_total = rd_blocks + extra; + cprintf("ramdisk: %u MB image, %u MB of memory to grow into\n", + (uint)(rd_size >> 20), (uint)(extra * BSIZE >> 20)); + } + release(&rd_lock); + return rd_total; +} + +// The page behind a block past the image, made when `make` says so. +static uchar * +rd_page(uint64 blk, int make) +{ + uint64 i = (blk - rd_blocks) / RD_PTRS, j = (blk - rd_blocks) % RD_PTRS; + uchar *pg = 0; + + acquire(&rd_lock); + if (rd_dir[i] == 0 && make) { + if ((rd_dir[i] = (uchar **)kalloc()) != 0) + memset(rd_dir[i], 0, PGSIZE); + } + if (rd_dir[i]) { + pg = rd_dir[i][j]; + if (pg == 0 && make && (pg = (uchar *)kalloc()) != 0) { + rd_dir[i][j] = pg; + rd_pages++; + } + } + release(&rd_lock); + if (make && pg == 0) + panic("ramdisk: out of memory for the live root"); + return pg; +} + // Sync buf with the in-memory image. Mirrors iderw()'s contract: on a read, // fill b->data and set B_VALID; on a write, copy out and clear B_DIRTY. void @@ -77,8 +149,21 @@ ramdiskrw(struct buf *b) panic("ramdiskrw: nothing to do"); if(rd_base == 0) panic("ramdiskrw: no ramdisk"); - if(b->blockno >= rd_blocks) - panic("ramdiskrw: block out of range"); + if(b->blockno >= rd_blocks){ + if(b->blockno >= ramdisk_blocks()) + panic("ramdiskrw: block out of range"); + uchar *pg = rd_page(b->blockno, (b->flags & B_DIRTY) != 0); + if(b->flags & B_DIRTY){ + memmove(pg, b->data, BSIZE); + b->flags &= ~B_DIRTY; + } else if(pg){ + memmove(b->data, pg, BSIZE); + } else { + memset(b->data, 0, BSIZE); // never written + } + b->flags |= B_VALID; + return; + } uchar *p = rd_base + (uint64)b->blockno * BSIZE; diff --git a/kernel/fs/schfs/kmount.c b/kernel/fs/schfs/kmount.c index a16528e..cbee97d 100644 --- a/kernel/fs/schfs/kmount.c +++ b/kernel/fs/schfs/kmount.c @@ -641,6 +641,9 @@ kmount_slot(int slot, uint bdev) // Opening with a page allocator also builds the allocation bitmap, by walking // the tree — the only mount-time cost SchFS has, and what buys it a mount // that cannot disagree with the filesystem it just adopted. + // The live root fills its ramdisk, however much memory that turned out to + // be. See kernel/dev/ramdisk.c. + m->vol.grow_to = (bdev == RAMDISK_DEV) ? ramdisk_blocks() : 0; int r = schfs_vol_open(&m->vol, &bd, &cx, kd_palloc, kd_pfree); if (r == -1) return -EINVAL; /* not a SchFS volume at all */ diff --git a/kernel/fs/schfs/schfs_int.h b/kernel/fs/schfs/schfs_int.h index 0a0d302..991bcfc 100644 --- a/kernel/fs/schfs/schfs_int.h +++ b/kernel/fs/schfs/schfs_int.h @@ -104,6 +104,13 @@ struct schfs_vol { uint64_t total_blocks; uint64_t free_blocks; + // Set by the opener before schfs_vol_open: the device's size, when the + // volume should grow to fill it. Nothing on disk records which blocks are + // free -- open rebuilds that from the tree -- so growing is only a larger + // total before the map is built, and the next commit writes it down. The + // live root uses it to fill a ramdisk sized from memory at boot. 0, and + // anything not larger than the volume, leaves it alone. + uint64_t grow_to; uint64_t next_objectid; uint64_t root_objectid; uint64_t generation; // last *committed* transaction group diff --git a/kernel/fs/schfs/vol.c b/kernel/fs/schfs/vol.c index b7b2254..4d3e1e2 100644 --- a/kernel/fs/schfs/vol.c +++ b/kernel/fs/schfs/vol.c @@ -215,6 +215,8 @@ schfs_vol_open(struct schfs_vol *v, struct schfs_bdev *dev, } vol_from_uber(v, u, best_slot); + if (v->grow_to > v->total_blocks) + v->total_blocks = v->grow_to; v->readonly = 1; // until an allocation map exists, below if (v->dev.bread(v->dev.ctx, v->tree.root, x->a) != 0)