SCFF: the live root ships compressed, with Xorg and Window Maker
SCFF, the SchOS Compressed File Format (include/fs/scff.h), is a block image cut into 128 KB chunks, each its own zstd frame, behind an index. tools/mkscff writes it from fs-ram.img on every core (-19, 11 s) and checks each chunk decompresses to itself. The kernel carries the .scff as its ramdisk and unpacks a chunk the first time a block in it is read; zero blocks share one page. The kernel and the ISO are the compressed size, and the root costs memory only for what is touched. See docs/architecture/scff.md. zstd 1.5.7 comes from ports/zstd like uACPI: the decoder is compiled into the kernel at -O2, with a static context and a 64 KB stack of its own, and mkscff builds the whole library from the same release. world now has xorg-server, the fbdev and evdev drivers, xinit, Window Maker, xterm and DejaVu, and /root/.xinitrc runs wmaker. That root is 358 MB, 102 MB as SCFF. The ISO no longer carries the root a third time as an ISO 9660 tree, which nothing read. schos.iso: 1389 MB -> 318 MB. El Torito keeps its own copy of the EFI image, the layout real machines have booted. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4
This commit is contained in:
Nexoniarz
parent
1ba93dde2c
commit
df1394eaa3
18 changed files with 832 additions and 38 deletions
|
|
@ -27,6 +27,15 @@
|
|||
// Pages are not given back when SchFS frees a block: there is no discard.
|
||||
// A freed block is reused by the next write, so the cost is the high-water
|
||||
// mark, not a leak.
|
||||
//
|
||||
// **A compressed image.** The blob can also be SCFF (include/fs/scff.h): the
|
||||
// image cut into chunks, each compressed with zstd. Then no block is served in
|
||||
// place. The page directory covers the image too, and a block with no page
|
||||
// has not been unpacked yet: reading it decompresses its whole chunk, and
|
||||
// every block of that chunk nobody has written gets a page of its own. The
|
||||
// kernel and the ISO are the compressed size -- the 360 MB root with Xorg in
|
||||
// it is 104 MB -- and the root costs memory only for what has been read or
|
||||
// written. A block of zeros points at one shared zero page and costs nothing.
|
||||
|
||||
#include "core/types.h"
|
||||
#include "core/defs.h"
|
||||
|
|
@ -37,6 +46,8 @@
|
|||
#include "core/sleeplock.h"
|
||||
#include "fs/fs.h"
|
||||
#include "fs/buf.h"
|
||||
#include "fs/scff.h"
|
||||
#include "../scff/scff_zstd.h"
|
||||
|
||||
#ifdef RAMDISK_ROOT
|
||||
// Supplied by `ld -r -b binary` over build/fs-ram.img, renamed by objcopy.
|
||||
|
|
@ -46,10 +57,20 @@ extern uchar _binary_ramdisk_size[];
|
|||
|
||||
static uchar *rd_base;
|
||||
static uint64 rd_size; // bytes
|
||||
static uint rd_blocks; // rd_size / BSIZE: the image, served in place
|
||||
static uint rd_blocks; // blocks in the image
|
||||
static uint rd_first; // the first block with a page: rd_blocks, or 0 for SCFF
|
||||
|
||||
// Past the image: rd_dir[i] is a page of pointers, entry j the page holding
|
||||
// block rd_blocks + i*RD_PTRS + j, or 0 for one never written. 32768 pages of
|
||||
// SCFF only. rd_zlock is held across a chunk's decompression, and makes
|
||||
// rd_chunk -- the one chunk-sized buffer -- its holder's.
|
||||
static struct scff_header *rd_scff;
|
||||
static uint64 *rd_index;
|
||||
static struct sleeplock rd_zlock;
|
||||
static uchar rd_chunk[SCFF_CHUNK_MAX] __attribute__((aligned(PGSIZE)));
|
||||
static uchar rd_zero[BSIZE] __attribute__((aligned(PGSIZE)));
|
||||
static uint64 rd_unpacked; // chunks decompressed
|
||||
|
||||
// rd_dir[i] is a page of pointers, entry j the page holding block
|
||||
// rd_first + i*RD_PTRS + j, or 0 for one never written. 32768 pages of
|
||||
// 512 pointers reach 64 GB, which is as far as half of any machine this runs
|
||||
// on is likely to go, and costs 256 KB of pointers here.
|
||||
#define RD_PTRS (PGSIZE / sizeof(uchar *))
|
||||
|
|
@ -67,6 +88,32 @@ ramdiskinit(void)
|
|||
rd_base = _binary_ramdisk_start;
|
||||
rd_size = (uint64)_binary_ramdisk_size;
|
||||
rd_blocks = (uint)(rd_size / BSIZE);
|
||||
rd_first = rd_blocks;
|
||||
|
||||
if(rd_size >= sizeof(struct scff_header) && memcmp(rd_base, SCFF_MAGIC, 4) == 0){
|
||||
struct scff_header *h = (struct scff_header *)rd_base;
|
||||
uint64 chunks = h->chunk_bytes ? (h->image_bytes + h->chunk_bytes - 1) / h->chunk_bytes : 0;
|
||||
if(h->version != SCFF_VERSION || h->codec != SCFF_CODEC_ZSTD ||
|
||||
h->chunk_bytes < BSIZE || h->chunk_bytes > SCFF_CHUNK_MAX ||
|
||||
(h->chunk_bytes & (h->chunk_bytes - 1)) || h->image_bytes % BSIZE ||
|
||||
chunks != h->nchunks || h->stored_bytes > rd_size ||
|
||||
h->index_off + (h->nchunks + 1) * sizeof(uint64) > rd_size)
|
||||
panic("ramdiskinit: the SCFF header does not describe this image");
|
||||
rd_index = (uint64 *)(rd_base + h->index_off);
|
||||
for(uint i = 0; i < h->nchunks; i++)
|
||||
if(rd_index[i] > rd_index[i + 1] || rd_index[i + 1] > h->stored_bytes ||
|
||||
rd_index[i + 1] - rd_index[i] > h->chunk_bytes)
|
||||
panic("ramdiskinit: the SCFF index is damaged");
|
||||
if(scff_zstd_init() < 0)
|
||||
panic("ramdiskinit: no room for the zstd decoder");
|
||||
initsleeplock(&rd_zlock, "scff");
|
||||
rd_scff = h;
|
||||
rd_blocks = (uint)(h->image_bytes / BSIZE);
|
||||
rd_first = 0;
|
||||
cprintf("ramdisk: SCFF, %u MB in %u MB, %u chunks of %u KB\n",
|
||||
(uint)(h->image_bytes >> 20), (uint)(rd_size >> 20),
|
||||
h->nchunks, h->chunk_bytes >> 10);
|
||||
}
|
||||
|
||||
if(rd_blocks == 0)
|
||||
panic("ramdiskinit: embedded image is empty");
|
||||
|
|
@ -76,8 +123,9 @@ ramdiskinit(void)
|
|||
if(rd_blocks < 2)
|
||||
panic("ramdiskinit: embedded image smaller than a superblock");
|
||||
|
||||
cprintf("ramdisk: %u blocks (%u KB) at %p\n",
|
||||
rd_blocks, (uint)(rd_size / 1024), rd_base);
|
||||
if(!rd_scff)
|
||||
cprintf("ramdisk: %u blocks (%u KB) at %p\n",
|
||||
rd_blocks, (uint)(rd_size / 1024), rd_base);
|
||||
#else
|
||||
rd_base = 0;
|
||||
rd_size = 0;
|
||||
|
|
@ -103,8 +151,9 @@ ramdisk_blocks(void)
|
|||
acquire(&rd_lock);
|
||||
if (rd_total == 0) {
|
||||
uint64 extra = kalloc_free_bytes() / 2 / BSIZE;
|
||||
if (extra > (uint64)RD_DIRS * RD_PTRS)
|
||||
extra = (uint64)RD_DIRS * RD_PTRS;
|
||||
uint64 room = (uint64)RD_DIRS * RD_PTRS - (rd_blocks - rd_first);
|
||||
if (extra > room)
|
||||
extra = room;
|
||||
rd_total = rd_blocks + extra;
|
||||
cprintf("ramdisk: %u MB image, %u MB of memory to grow into\n",
|
||||
(uint)(rd_size >> 20), (uint)(extra * BSIZE >> 20));
|
||||
|
|
@ -113,11 +162,12 @@ ramdisk_blocks(void)
|
|||
return rd_total;
|
||||
}
|
||||
|
||||
// The page behind a block past the image, made when `make` says so.
|
||||
// The page behind a block, made when `make` says so. One that is only the
|
||||
// shared zero page gets a page of its own before anything writes to it.
|
||||
static uchar *
|
||||
rd_page(uint64 blk, int make)
|
||||
{
|
||||
uint64 i = (blk - rd_blocks) / RD_PTRS, j = (blk - rd_blocks) % RD_PTRS;
|
||||
uint64 i = (blk - rd_first) / RD_PTRS, j = (blk - rd_first) % RD_PTRS;
|
||||
uchar *pg = 0;
|
||||
|
||||
acquire(&rd_lock);
|
||||
|
|
@ -127,7 +177,9 @@ rd_page(uint64 blk, int make)
|
|||
}
|
||||
if (rd_dir[i]) {
|
||||
pg = rd_dir[i][j];
|
||||
if (pg == 0 && make && (pg = (uchar *)kalloc()) != 0) {
|
||||
if ((pg == 0 || pg == rd_zero) && make && (pg = (uchar *)kalloc()) != 0) {
|
||||
if (rd_dir[i][j] == rd_zero)
|
||||
memset(pg, 0, BSIZE);
|
||||
rd_dir[i][j] = pg;
|
||||
rd_pages++;
|
||||
}
|
||||
|
|
@ -138,6 +190,64 @@ rd_page(uint64 blk, int make)
|
|||
return pg;
|
||||
}
|
||||
|
||||
// Give every block of SCFF chunk c that has no page one: its contents, or the
|
||||
// zero page. Blocks already there -- written since boot, or unpacked by
|
||||
// whoever held the lock before us -- are left as they are.
|
||||
static void
|
||||
rd_unpack(uint c)
|
||||
{
|
||||
uint per = rd_scff->chunk_bytes / BSIZE;
|
||||
uint64 first = (uint64)c * per;
|
||||
uint n = rd_blocks - first < per ? (uint)(rd_blocks - first) : per;
|
||||
uint64 len = rd_index[c + 1] - rd_index[c];
|
||||
uchar *src = rd_base + rd_index[c];
|
||||
uint64 want = (uint64)n * BSIZE;
|
||||
uchar *data = rd_chunk;
|
||||
|
||||
acquiresleep(&rd_zlock);
|
||||
if (len == 0) {
|
||||
data = 0; // all zeros
|
||||
} else if (len == want) {
|
||||
data = src; // stored as it is
|
||||
} else {
|
||||
long got = scff_zstd_decompress(rd_chunk, sizeof(rd_chunk), src, len);
|
||||
if (got != (long)want)
|
||||
panic("ramdisk: an SCFF chunk does not decompress");
|
||||
rd_unpacked++;
|
||||
}
|
||||
|
||||
for (uint k = 0; k < n; k++) {
|
||||
uint64 blk = first + k, i = blk / RD_PTRS, j = blk % RD_PTRS;
|
||||
uchar *b = data ? data + (uint64)k * BSIZE : 0;
|
||||
int zero = 1;
|
||||
if (b)
|
||||
for (uint w = 0; w < BSIZE / 8 && zero; w++)
|
||||
zero = ((uint64 *)b)[w] == 0;
|
||||
|
||||
uchar *pg = zero ? rd_zero : (uchar *)kalloc();
|
||||
if (pg == 0)
|
||||
panic("ramdisk: out of memory for the live root");
|
||||
if (!zero)
|
||||
memmove(pg, b, BSIZE);
|
||||
|
||||
acquire(&rd_lock);
|
||||
if (rd_dir[i] == 0 && (rd_dir[i] = (uchar **)kalloc()) != 0)
|
||||
memset(rd_dir[i], 0, PGSIZE);
|
||||
if (rd_dir[i] == 0)
|
||||
panic("ramdisk: out of memory for the live root");
|
||||
if (rd_dir[i][j] == 0) {
|
||||
rd_dir[i][j] = pg;
|
||||
if (!zero)
|
||||
rd_pages++;
|
||||
pg = 0;
|
||||
}
|
||||
release(&rd_lock);
|
||||
if (pg && pg != rd_zero)
|
||||
kfree((char *)pg); // someone was there first
|
||||
}
|
||||
releasesleep(&rd_zlock);
|
||||
}
|
||||
|
||||
// Sync buf with the in-memory image. Mirrors iderw()'s contract: on a read,
|
||||
// fill b->data and set B_VALID; on a write, copy out and clear B_DIRTY.
|
||||
void
|
||||
|
|
@ -149,10 +259,14 @@ ramdiskrw(struct buf *b)
|
|||
panic("ramdiskrw: nothing to do");
|
||||
if(rd_base == 0)
|
||||
panic("ramdiskrw: no ramdisk");
|
||||
if(b->blockno >= rd_blocks){
|
||||
if(b->blockno >= rd_first){
|
||||
if(b->blockno >= ramdisk_blocks())
|
||||
panic("ramdiskrw: block out of range");
|
||||
uchar *pg = rd_page(b->blockno, (b->flags & B_DIRTY) != 0);
|
||||
if(pg == 0 && rd_scff && b->blockno < rd_blocks && !(b->flags & B_DIRTY)){
|
||||
rd_unpack((uint)(b->blockno / (rd_scff->chunk_bytes / BSIZE)));
|
||||
pg = rd_page(b->blockno, 0);
|
||||
}
|
||||
if(b->flags & B_DIRTY){
|
||||
memmove(pg, b->data, BSIZE);
|
||||
b->flags &= ~B_DIRTY;
|
||||
|
|
|
|||
Loading…
Reference in a new issue