diff --git a/Makefile b/Makefile index d23f6c3..791c654 100644 --- a/Makefile +++ b/Makefile @@ -461,12 +461,14 @@ CFLAGS_SYS = $(CFLAGS) -I./kernel/sys # loads that by PIO one sector at a time -- a kernel with a userland inside it # takes visibly longer to boot than a 350 KB one. # -# 24576 blocks of 4 KB is 96 MB: the apk root's base system is 35 MB, and the rest is -# room for `apk add` on the live system. The ramdisk is part of the kernel file -# and so of the ISO, empty space included; a system that installs a desktop -# wants a disk. +# Sized to fit, not fixed. The image is what the root uses plus RAMDISK_SLACK +# blocks and an eighth, measured by building it once large; the kernel then +# grows the volume into half of the free memory at boot (kernel/dev/ramdisk.c), +# which is where `apk add` on the live system writes. It was a fixed 96 MB, +# 60 of it zeros carried in the kernel and twice over on the ISO, and room for +# 60 MB of packages whatever the machine had. # ----------------------------------------------------------------------- -RAMDISK_BLOCKS ?= 24576 +RAMDISK_SLACK ?= 1024 # RAMDISK_NINODES used to live here, sizing mkfs's preallocated inode table. # The ramdisk is a SchFS image now (see $(BUILD)/fs-ram.img) and SchFS allocates @@ -825,7 +827,7 @@ $(BUILD)/kernel/kernel: $(BUILD)/kernel/kernel.pass1 $(BUILD)/kernel/ksym.o $(OBJDUMP) -S $@ > $(BUILD)/kernel/kernel.asm # The ramdisk root image, and the object wrapping it for the linker. Built from -# the same staged sysroot as the main image, just capped at RAMDISK_BLOCKS -- +# the same staged sysroot as the main image, sized to fit (see RAMDISK_SLACK) -- # mkfs's content dedup means the whole userland still fits in a few MB. # Built with mkschfs, not mkfs. # @@ -837,7 +839,12 @@ $(BUILD)/kernel/kernel: $(BUILD)/kernel/kernel.pass1 $(BUILD)/kernel/ksym.o $(BUILD)/fs-ram.img: $(MKSCHFS) $(ROOT_STAMP) @mkdir -p $(BUILD) @rm -f $@ - $(MKSCHFS) $@ --size $(RAMDISK_BLOCKS) --label schos \ + @used=$$($(MKSCHFS) $@ --size 262144 --label schos \ + $(RAMDISK_EXCLUDE_ARGS) --sysroot $(ROOT_TREE) | \ + sed -n 's/^ *\([0-9][0-9]*\) of [0-9][0-9]* blocks used.*/\1/p'); \ + test -n "$$used" || { echo "fs-ram.img: mkschfs did not say how much it used" >&2; exit 1; }; \ + rm -f $@; \ + $(MKSCHFS) $@ --size $$((used + used / 8 + $(RAMDISK_SLACK))) --label schos \ $(RAMDISK_EXCLUDE_ARGS) --sysroot $(ROOT_TREE) $(BUILD)/kernel/ramdisk_obj: $(BUILD)/fs-ram.img diff --git a/docs/plan/alpine.md b/docs/plan/alpine.md index 824889c..6cf01d6 100644 --- a/docs/plan/alpine.md +++ b/docs/plan/alpine.md @@ -39,7 +39,7 @@ only what is its own (see `tools/posix/mkaroot.sh`): ## Building make qemu boots build/schfs.img, the root as a 1 GB SchFS disk - make iso the same root as the live system's 96 MB ramdisk + make iso the same root as the live system's ramdisk make alpine-lock re-resolve world against Alpine (needs the network) It is the only root. sysroot/ is what schos-base is packaged from. @@ -68,8 +68,10 @@ All of it found by running apk: ## Next -- The live ISO runs the apk root from RAM: `apk add` works, into 59 MB, and - is gone at reboot. Keeping packages needs an installer that writes the root +- The live ISO runs the apk root from RAM. The image in the kernel is only + what the root holds; at boot the volume grows into half of the free memory + (4 GB of root on an 8 GB machine), backed by pages as they are written. + Xorg installs into it in about a minute. All of it is gone at reboot. Keeping packages needs an installer that writes the root to a SATA or NVMe disk, and a kernel that finds its root there by the SchFS GPT type rather than as IDE disk 1. - Xorg from apk (`xorg-server xf86-video-fbdev xf86-input-evdev`) instead of diff --git a/include/core/defs.h b/include/core/defs.h index 94ce346..feaa099 100644 --- a/include/core/defs.h +++ b/include/core/defs.h @@ -368,6 +368,7 @@ void input_mouse(int dx, int dy, int buttons); void ramdiskinit(void); void ramdiskrw(struct buf*); int ramdisk_present(void); +uint64 ramdisk_blocks(void); // usb_storage.c void usb_storage_rw(struct buf*); diff --git a/kernel/dev/ramdisk.c b/kernel/dev/ramdisk.c index 055ad99..8736a75 100644 --- a/kernel/dev/ramdisk.c +++ b/kernel/dev/ramdisk.c @@ -14,6 +14,19 @@ // // Enabled by building with RAMROOT=1. Without it this file still compiles, the // device reports zero size, and the kernel boots from the IDE disk as before. +// +// The device is larger than the image. The image is only what the root holds, +// with a little slack; past its end the device goes on to half of the memory +// free at mount time, and those blocks are pages taken from kalloc the first +// time each is written -- a block nobody has written reads as zeros and costs +// nothing. SchFS grows the volume to the device's size when it mounts (see +// grow_to in kernel/fs/schfs/vol.c), so `apk add` on a live system has as +// much room as the machine has memory to give, rather than whatever empty +// space was built into the ISO. The ISO used to carry 60 MB of zeros for it. +// +// Pages are not given back when SchFS frees a block: there is no discard. +// A freed block is reused by the next write, so the cost is the high-water +// mark, not a leak. #include "core/types.h" #include "core/defs.h" @@ -33,11 +46,23 @@ extern uchar _binary_ramdisk_size[]; static uchar *rd_base; static uint64 rd_size; // bytes -static uint rd_blocks; // rd_size / BSIZE +static uint rd_blocks; // rd_size / BSIZE: the image, served in place + +// Past the image: rd_dir[i] is a page of pointers, entry j the page holding +// block rd_blocks + i*RD_PTRS + j, or 0 for one never written. 32768 pages of +// 512 pointers reach 64 GB, which is as far as half of any machine this runs +// on is likely to go, and costs 256 KB of pointers here. +#define RD_PTRS (PGSIZE / sizeof(uchar *)) +#define RD_DIRS 32768 +static uchar **rd_dir[RD_DIRS]; +static uint64 rd_total; // blocks, image and the rest; 0 until sized +static uint64 rd_pages; // pages taken for blocks past the image +static struct spinlock rd_lock; void ramdiskinit(void) { + initlock(&rd_lock, "ramdisk"); #ifdef RAMDISK_ROOT rd_base = _binary_ramdisk_start; rd_size = (uint64)_binary_ramdisk_size; @@ -66,6 +91,53 @@ ramdisk_present(void) return rd_blocks != 0; } +// The device's size in blocks, fixed the first time anyone asks -- which is +// SchFS mounting the root, after every page has reached the allocator. Asked +// earlier, during ramdiskinit, the free count would be the first few +// megabytes kinit1 handed over and nothing more. +uint64 +ramdisk_blocks(void) +{ + if (rd_blocks == 0) + return 0; + acquire(&rd_lock); + if (rd_total == 0) { + uint64 extra = kalloc_free_bytes() / 2 / BSIZE; + if (extra > (uint64)RD_DIRS * RD_PTRS) + extra = (uint64)RD_DIRS * RD_PTRS; + rd_total = rd_blocks + extra; + cprintf("ramdisk: %u MB image, %u MB of memory to grow into\n", + (uint)(rd_size >> 20), (uint)(extra * BSIZE >> 20)); + } + release(&rd_lock); + return rd_total; +} + +// The page behind a block past the image, made when `make` says so. +static uchar * +rd_page(uint64 blk, int make) +{ + uint64 i = (blk - rd_blocks) / RD_PTRS, j = (blk - rd_blocks) % RD_PTRS; + uchar *pg = 0; + + acquire(&rd_lock); + if (rd_dir[i] == 0 && make) { + if ((rd_dir[i] = (uchar **)kalloc()) != 0) + memset(rd_dir[i], 0, PGSIZE); + } + if (rd_dir[i]) { + pg = rd_dir[i][j]; + if (pg == 0 && make && (pg = (uchar *)kalloc()) != 0) { + rd_dir[i][j] = pg; + rd_pages++; + } + } + release(&rd_lock); + if (make && pg == 0) + panic("ramdisk: out of memory for the live root"); + return pg; +} + // Sync buf with the in-memory image. Mirrors iderw()'s contract: on a read, // fill b->data and set B_VALID; on a write, copy out and clear B_DIRTY. void @@ -77,8 +149,21 @@ ramdiskrw(struct buf *b) panic("ramdiskrw: nothing to do"); if(rd_base == 0) panic("ramdiskrw: no ramdisk"); - if(b->blockno >= rd_blocks) - panic("ramdiskrw: block out of range"); + if(b->blockno >= rd_blocks){ + if(b->blockno >= ramdisk_blocks()) + panic("ramdiskrw: block out of range"); + uchar *pg = rd_page(b->blockno, (b->flags & B_DIRTY) != 0); + if(b->flags & B_DIRTY){ + memmove(pg, b->data, BSIZE); + b->flags &= ~B_DIRTY; + } else if(pg){ + memmove(b->data, pg, BSIZE); + } else { + memset(b->data, 0, BSIZE); // never written + } + b->flags |= B_VALID; + return; + } uchar *p = rd_base + (uint64)b->blockno * BSIZE; diff --git a/kernel/fs/schfs/kmount.c b/kernel/fs/schfs/kmount.c index a16528e..cbee97d 100644 --- a/kernel/fs/schfs/kmount.c +++ b/kernel/fs/schfs/kmount.c @@ -641,6 +641,9 @@ kmount_slot(int slot, uint bdev) // Opening with a page allocator also builds the allocation bitmap, by walking // the tree — the only mount-time cost SchFS has, and what buys it a mount // that cannot disagree with the filesystem it just adopted. + // The live root fills its ramdisk, however much memory that turned out to + // be. See kernel/dev/ramdisk.c. + m->vol.grow_to = (bdev == RAMDISK_DEV) ? ramdisk_blocks() : 0; int r = schfs_vol_open(&m->vol, &bd, &cx, kd_palloc, kd_pfree); if (r == -1) return -EINVAL; /* not a SchFS volume at all */ diff --git a/kernel/fs/schfs/schfs_int.h b/kernel/fs/schfs/schfs_int.h index 0a0d302..991bcfc 100644 --- a/kernel/fs/schfs/schfs_int.h +++ b/kernel/fs/schfs/schfs_int.h @@ -104,6 +104,13 @@ struct schfs_vol { uint64_t total_blocks; uint64_t free_blocks; + // Set by the opener before schfs_vol_open: the device's size, when the + // volume should grow to fill it. Nothing on disk records which blocks are + // free -- open rebuilds that from the tree -- so growing is only a larger + // total before the map is built, and the next commit writes it down. The + // live root uses it to fill a ramdisk sized from memory at boot. 0, and + // anything not larger than the volume, leaves it alone. + uint64_t grow_to; uint64_t next_objectid; uint64_t root_objectid; uint64_t generation; // last *committed* transaction group diff --git a/kernel/fs/schfs/vol.c b/kernel/fs/schfs/vol.c index b7b2254..4d3e1e2 100644 --- a/kernel/fs/schfs/vol.c +++ b/kernel/fs/schfs/vol.c @@ -215,6 +215,8 @@ schfs_vol_open(struct schfs_vol *v, struct schfs_bdev *dev, } vol_from_uber(v, u, best_slot); + if (v->grow_to > v->total_blocks) + v->total_blocks = v->grow_to; v->readonly = 1; // until an allocation map exists, below if (v->dev.bread(v->dev.ctx, v->tree.root, x->a) != 0)