SCFF: the live root ships compressed, with Xorg and Window Maker

SCFF, the SchOS Compressed File Format (include/fs/scff.h), is a block
image cut into 128 KB chunks, each its own zstd frame, behind an index.
tools/mkscff writes it from fs-ram.img on every core (-19, 11 s) and
checks each chunk decompresses to itself. The kernel carries the .scff
as its ramdisk and unpacks a chunk the first time a block in it is
read; zero blocks share one page. The kernel and the ISO are the
compressed size, and the root costs memory only for what is touched.
See docs/architecture/scff.md.

zstd 1.5.7 comes from ports/zstd like uACPI: the decoder is compiled
into the kernel at -O2, with a static context and a 64 KB stack of its
own, and mkscff builds the whole library from the same release.

world now has xorg-server, the fbdev and evdev drivers, xinit, Window
Maker, xterm and DejaVu, and /root/.xinitrc runs wmaker. That root is
358 MB, 102 MB as SCFF.

The ISO no longer carries the root a third time as an ISO 9660 tree,
which nothing read. schos.iso: 1389 MB -> 318 MB. El Torito keeps its
own copy of the EFI image, the layout real machines have booted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4

Makefile

@@ -103,6 +103,18 @@ KERN_ACPI = $(addprefix $(BUILD)/uacpi/, \
               uacpi.o utilities.o)
 KERN_ACPI += $(addprefix $(BUILD)/kernel/acpi/, host.o aml.o)
 
+# zstd's decoder, for SCFF -- the compressed live root (include/fs/scff.h).
+# Unpacked by ports/zstd into build/ports/zstd/src and compiled into the kernel
+# like uACPI; kernel/scff/ is the host side of it.
+ZSTDDIR   = $(BUILD)/ports/zstd/src/lib
+KERN_ZSTD = $(addprefix $(BUILD)/zstd/, \
+              common/entropy_common.o common/error_private.o \
+              common/fse_decompress.o common/zstd_common.o common/xxhash.o \
+              common/debug.o decompress/huf_decompress.o \
+              decompress/zstd_ddict.o decompress/zstd_decompress.o \
+              decompress/zstd_decompress_block.o)
+KERN_ZSTD += $(BUILD)/kernel/scff/scff.o
+
 KERN_PROC = $(addprefix $(BUILD)/kernel/proc/, \
               proc.o)
 
@@ -114,7 +126,7 @@ KERN_SYS  = $(addprefix $(BUILD)/kernel/sys/, \
 KERN_ARCH += $(BUILD)/kernel/arch/x86_64/vectors.o
 
 KERNOBJS = $(KERN_ARCH) $(KERN_CORE) $(KERN_DEV) $(KERN_DRIVERS) \
-           $(KERN_FS) $(KERN_NET) $(KERN_ACPI) $(KERN_PROC) $(KERN_SYS)
+           $(KERN_FS) $(KERN_NET) $(KERN_ACPI) $(KERN_ZSTD) $(KERN_PROC) $(KERN_SYS)
 # entry.o must be first in link order; remove it from KERNOBJS for the rule
 KERN_NOENTRY = $(filter-out $(BUILD)/kernel/arch/x86_64/entry.o, $(KERNOBJS))
 
@@ -392,6 +404,7 @@ endif
 # looks satisfied.
 MKFS       = $(BUILD)/tools/mkfs$(EXE)
 MKSCHFS    = $(BUILD)/tools/mkschfs$(EXE)
+MKSCFF     = $(BUILD)/tools/mkscff$(EXE)
 SCHFS_TEST = $(BUILD)/tools/schfs_test$(EXE)
 SCHFS_OBJT = $(BUILD)/tools/schfs_objtest$(EXE)
 SCHFS_FSCK = $(BUILD)/tools/schfs_fsck$(EXE)
@@ -431,7 +444,7 @@ CFLAGS += $(NO_SSP) $(CFLAGS_EXTRA)
 # copies and memset into %xmm, and an interrupt then scribbles over the
 # registers of the program it interrupted. Kernel objects only -- cmd/ is
 # built with these CFLAGS too and is welcome to use SSE.
-$(BUILD)/kernel/%.o $(BUILD)/lwip/%.o $(BUILD)/uacpi/%.o: CFLAGS += -mgeneral-regs-only
+$(BUILD)/kernel/%.o $(BUILD)/lwip/%.o $(BUILD)/uacpi/%.o $(BUILD)/zstd/%.o: CFLAGS += -mgeneral-regs-only
 
 # lwIP's own headers, plus ours (cc.h and lwipopts.h live in kernel/net/lwip).
 # Only the network files are compiled with these: putting them in the global
@@ -768,9 +781,9 @@ iso:
 	$(MAKE) RAMROOT=1 schos.iso
 
 # One El Torito entry, pointing at a FAT image holding the loader and kernel.
-# sysroot-install because the disc also carries the tree as a real hierarchy.
-schos.iso: $(BUILD)/kernel/kernel $(BUILD)/uefi/BOOTX64.EFI tools/posix/mkiso.sh sysroot-install $(ROOT_STAMP)
-	SYSROOT=$(ROOT_TREE) sh tools/posix/mkiso.sh $@
+# The root is in the kernel, as its ramdisk; the disc carries no other copy.
+schos.iso: $(BUILD)/kernel/kernel $(BUILD)/uefi/BOOTX64.EFI tools/posix/mkiso.sh $(ROOT_STAMP)
+	sh tools/posix/mkiso.sh $@
 
 # A partitioned disk -- GPT, an ESP and a SchFS partition -- which is what a
 # real machine boots from: dd it to a USB stick or a disk. See
@@ -847,13 +860,22 @@ $(BUILD)/fs-ram.img: $(MKSCHFS) $(ROOT_STAMP)
 	$(MKSCHFS) $@ --size $$((used + used / 8 + $(RAMDISK_SLACK))) --label schos \
 	    $(RAMDISK_EXCLUDE_ARGS) --sysroot $(ROOT_TREE)
 
-$(BUILD)/kernel/ramdisk_obj: $(BUILD)/fs-ram.img
+# The image goes into the kernel as SCFF (include/fs/scff.h): 128 KB chunks,
+# each its own zstd frame, unpacked by the kernel the first time a block in it
+# is read. -19 is slow to write and costs nothing to read -- zstd decodes at
+# the same speed whatever level made the frame -- and mkscff uses every core.
+SCFF_LEVEL ?= 19
+SCFF_CHUNK ?= 128
+$(BUILD)/fs-ram.scff: $(BUILD)/fs-ram.img $(MKSCFF)
+	$(MKSCFF) $< $@ --level $(SCFF_LEVEL) --chunk $(SCFF_CHUNK)
+
+$(BUILD)/kernel/ramdisk_obj: $(BUILD)/fs-ram.scff
 	@mkdir -p $(BUILD)/kernel
 	$(LD) -r -b binary -m elf_x86_64 -o $@ $<
 	$(OBJCOPY) \
-	    --redefine-sym _binary_build_fs_ram_img_start=_binary_ramdisk_start \
-	    --redefine-sym _binary_build_fs_ram_img_end=_binary_ramdisk_end     \
-	    --redefine-sym _binary_build_fs_ram_img_size=_binary_ramdisk_size   \
+	    --redefine-sym _binary_build_fs_ram_scff_start=_binary_ramdisk_start \
+	    --redefine-sym _binary_build_fs_ram_scff_end=_binary_ramdisk_end     \
+	    --redefine-sym _binary_build_fs_ram_scff_size=_binary_ramdisk_size   \
 	    $@
 
 # The console font, linked into the kernel.
@@ -1019,6 +1041,30 @@ $(UACPIDIR):
 $(patsubst $(BUILD)/uacpi/%.o,$(UACPIDIR)/source/%.c,$(filter $(BUILD)/uacpi/%,$(KERN_ACPI))): | $(UACPIDIR)
 	@:
 
+# zstd at -O2 whatever OPT says: the decoder runs on every first read of a
+# root block, and at -O0 it is several times slower. No assembly, no BMI2
+# dispatch, no legacy formats, no tracing hooks -- the kernel has none of
+# what those need. compat/ stands in for the libc headers it includes.
+ZSTD_CFLAGS = -Ikernel/scff/compat -I$(ZSTDDIR) -I$(ZSTDDIR)/common -O2 \
+              -DZSTD_DISABLE_ASM -DDYNAMIC_BMI2=0 -DZSTD_LEGACY_SUPPORT=0 \
+              -DZSTD_TRACE=0 -DDEBUGLEVEL=0 -DZSTD_STRIP_ERROR_STRINGS \
+              -DXXH_NAMESPACE=ZSTD_
+ZSTD_CC = $(CC) $(filter-out -I./include,$(CFLAGS)) $(ZSTD_CFLAGS)
+
+$(BUILD)/zstd/%.o: $(ZSTDDIR)/%.c | $(ZSTDDIR)
+	@mkdir -p $(dir $@)
+	$(ZSTD_CC) -Wno-unused-function -c $< -o $@
+
+$(BUILD)/kernel/scff/%.o: kernel/scff/%.c | $(ZSTDDIR)
+	@mkdir -p $(dir $@)
+	$(ZSTD_CC) -c $< -o $@
+
+$(ZSTDDIR):
+	$(MAKE) -C ports zstd
+
+$(patsubst $(BUILD)/zstd/%.o,$(ZSTDDIR)/%.c,$(filter $(BUILD)/zstd/%,$(KERN_ZSTD))): | $(ZSTDDIR)
+	@:
+
 $(BUILD)/kernel/proc/%.o: kernel/proc/%.c
 	@mkdir -p $(BUILD)/kernel/proc
 	$(CC) $(CFLAGS) -c $< -o $@
@@ -1222,6 +1268,14 @@ $(SCHFS_FSCK): tools/schfs_fsck.c $(SCHFS_SRC)
 	$(HOSTCC) $(HOSTCFLAGS) -o $@ tools/schfs_fsck.c
 
 mkschfs: $(MKSCHFS)
+
+# The SCFF writer, built from the same zstd release as the kernel's decoder.
+MKSCFF_ZSTD = $(wildcard $(ZSTDDIR)/common/*.c $(ZSTDDIR)/compress/*.c $(ZSTDDIR)/decompress/*.c)
+$(MKSCFF): tools/mkscff.c include/fs/scff.h | $(ZSTDDIR)
+	@mkdir -p $(BUILD)/tools
+	$(HOSTCC) -O2 -pthread -iquote ./include -I$(ZSTDDIR) -DZSTD_DISABLE_ASM \
+	    -DZSTD_LEGACY_SUPPORT=0 -o $@ tools/mkscff.c \
+	    $(ZSTDDIR)/common/*.c $(ZSTDDIR)/compress/*.c $(ZSTDDIR)/decompress/*.c
 schfs-fsck: $(SCHFS_FSCK)
 
 # End-to-end filesystem test: build an image from a directory tree, read it