From df1394eaa3316eedca5e72dfc130151ae843a056 Mon Sep 17 00:00:00 2001 From: Nexoniarz Date: Sat, 24 Oct 2026 21:17:31 +0200 Subject: [PATCH] SCFF: the live root ships compressed, with Xorg and Window Maker SCFF, the SchOS Compressed File Format (include/fs/scff.h), is a block image cut into 128 KB chunks, each its own zstd frame, behind an index. tools/mkscff writes it from fs-ram.img on every core (-19, 11 s) and checks each chunk decompresses to itself. The kernel carries the .scff as its ramdisk and unpacks a chunk the first time a block in it is read; zero blocks share one page. The kernel and the ISO are the compressed size, and the root costs memory only for what is touched. See docs/architecture/scff.md. zstd 1.5.7 comes from ports/zstd like uACPI: the decoder is compiled into the kernel at -O2, with a static context and a 64 KB stack of its own, and mkscff builds the whole library from the same release. world now has xorg-server, the fbdev and evdev drivers, xinit, Window Maker, xterm and DejaVu, and /root/.xinitrc runs wmaker. That root is 358 MB, 102 MB as SCFF. The ISO no longer carries the root a third time as an ISO 9660 tree, which nothing read. schos.iso: 1389 MB -> 318 MB. El Torito keeps its own copy of the EFI image, the layout real machines have booted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4 --- Makefile | 72 ++++++++++++-- NOTICE | 40 ++++++++ docs/architecture/scff.md | 79 +++++++++++++++ docs/plan/alpine.md | 7 +- include/fs/scff.h | 54 ++++++++++ kernel/dev/ramdisk.c | 136 +++++++++++++++++++++++--- kernel/scff/compat/assert.h | 6 ++ kernel/scff/compat/stdio.h | 5 + kernel/scff/compat/stdlib.h | 11 +++ kernel/scff/compat/string.h | 11 +++ kernel/scff/scff.c | 80 +++++++++++++++ kernel/scff/scff_zstd.h | 13 +++ ports/alpine/world | 10 ++ ports/alpine/world.lock | 79 +++++++++++++++ ports/zstd/Makefile | 43 ++++++++ tools/mkscff.c | 190 ++++++++++++++++++++++++++++++++++++ tools/posix/mkaroot.sh | 7 ++ tools/posix/mkiso.sh | 27 +++-- 18 files changed, 832 insertions(+), 38 deletions(-) create mode 100644 docs/architecture/scff.md create mode 100644 include/fs/scff.h create mode 100644 kernel/scff/compat/assert.h create mode 100644 kernel/scff/compat/stdio.h create mode 100644 kernel/scff/compat/stdlib.h create mode 100644 kernel/scff/compat/string.h create mode 100644 kernel/scff/scff.c create mode 100644 kernel/scff/scff_zstd.h create mode 100644 ports/zstd/Makefile create mode 100644 tools/mkscff.c diff --git a/Makefile b/Makefile index 791c654..5e4e48a 100644 --- a/Makefile +++ b/Makefile @@ -103,6 +103,18 @@ KERN_ACPI = $(addprefix $(BUILD)/uacpi/, \ uacpi.o utilities.o) KERN_ACPI += $(addprefix $(BUILD)/kernel/acpi/, host.o aml.o) +# zstd's decoder, for SCFF -- the compressed live root (include/fs/scff.h). +# Unpacked by ports/zstd into build/ports/zstd/src and compiled into the kernel +# like uACPI; kernel/scff/ is the host side of it. +ZSTDDIR = $(BUILD)/ports/zstd/src/lib +KERN_ZSTD = $(addprefix $(BUILD)/zstd/, \ + common/entropy_common.o common/error_private.o \ + common/fse_decompress.o common/zstd_common.o common/xxhash.o \ + common/debug.o decompress/huf_decompress.o \ + decompress/zstd_ddict.o decompress/zstd_decompress.o \ + decompress/zstd_decompress_block.o) +KERN_ZSTD += $(BUILD)/kernel/scff/scff.o + KERN_PROC = $(addprefix $(BUILD)/kernel/proc/, \ proc.o) @@ -114,7 +126,7 @@ KERN_SYS = $(addprefix $(BUILD)/kernel/sys/, \ KERN_ARCH += $(BUILD)/kernel/arch/x86_64/vectors.o KERNOBJS = $(KERN_ARCH) $(KERN_CORE) $(KERN_DEV) $(KERN_DRIVERS) \ - $(KERN_FS) $(KERN_NET) $(KERN_ACPI) $(KERN_PROC) $(KERN_SYS) + $(KERN_FS) $(KERN_NET) $(KERN_ACPI) $(KERN_ZSTD) $(KERN_PROC) $(KERN_SYS) # entry.o must be first in link order; remove it from KERNOBJS for the rule KERN_NOENTRY = $(filter-out $(BUILD)/kernel/arch/x86_64/entry.o, $(KERNOBJS)) @@ -392,6 +404,7 @@ endif # looks satisfied. MKFS = $(BUILD)/tools/mkfs$(EXE) MKSCHFS = $(BUILD)/tools/mkschfs$(EXE) +MKSCFF = $(BUILD)/tools/mkscff$(EXE) SCHFS_TEST = $(BUILD)/tools/schfs_test$(EXE) SCHFS_OBJT = $(BUILD)/tools/schfs_objtest$(EXE) SCHFS_FSCK = $(BUILD)/tools/schfs_fsck$(EXE) @@ -431,7 +444,7 @@ CFLAGS += $(NO_SSP) $(CFLAGS_EXTRA) # copies and memset into %xmm, and an interrupt then scribbles over the # registers of the program it interrupted. Kernel objects only -- cmd/ is # built with these CFLAGS too and is welcome to use SSE. -$(BUILD)/kernel/%.o $(BUILD)/lwip/%.o $(BUILD)/uacpi/%.o: CFLAGS += -mgeneral-regs-only +$(BUILD)/kernel/%.o $(BUILD)/lwip/%.o $(BUILD)/uacpi/%.o $(BUILD)/zstd/%.o: CFLAGS += -mgeneral-regs-only # lwIP's own headers, plus ours (cc.h and lwipopts.h live in kernel/net/lwip). # Only the network files are compiled with these: putting them in the global @@ -768,9 +781,9 @@ iso: $(MAKE) RAMROOT=1 schos.iso # One El Torito entry, pointing at a FAT image holding the loader and kernel. -# sysroot-install because the disc also carries the tree as a real hierarchy. -schos.iso: $(BUILD)/kernel/kernel $(BUILD)/uefi/BOOTX64.EFI tools/posix/mkiso.sh sysroot-install $(ROOT_STAMP) - SYSROOT=$(ROOT_TREE) sh tools/posix/mkiso.sh $@ +# The root is in the kernel, as its ramdisk; the disc carries no other copy. +schos.iso: $(BUILD)/kernel/kernel $(BUILD)/uefi/BOOTX64.EFI tools/posix/mkiso.sh $(ROOT_STAMP) + sh tools/posix/mkiso.sh $@ # A partitioned disk -- GPT, an ESP and a SchFS partition -- which is what a # real machine boots from: dd it to a USB stick or a disk. See @@ -847,13 +860,22 @@ $(BUILD)/fs-ram.img: $(MKSCHFS) $(ROOT_STAMP) $(MKSCHFS) $@ --size $$((used + used / 8 + $(RAMDISK_SLACK))) --label schos \ $(RAMDISK_EXCLUDE_ARGS) --sysroot $(ROOT_TREE) -$(BUILD)/kernel/ramdisk_obj: $(BUILD)/fs-ram.img +# The image goes into the kernel as SCFF (include/fs/scff.h): 128 KB chunks, +# each its own zstd frame, unpacked by the kernel the first time a block in it +# is read. -19 is slow to write and costs nothing to read -- zstd decodes at +# the same speed whatever level made the frame -- and mkscff uses every core. +SCFF_LEVEL ?= 19 +SCFF_CHUNK ?= 128 +$(BUILD)/fs-ram.scff: $(BUILD)/fs-ram.img $(MKSCFF) + $(MKSCFF) $< $@ --level $(SCFF_LEVEL) --chunk $(SCFF_CHUNK) + +$(BUILD)/kernel/ramdisk_obj: $(BUILD)/fs-ram.scff @mkdir -p $(BUILD)/kernel $(LD) -r -b binary -m elf_x86_64 -o $@ $< $(OBJCOPY) \ - --redefine-sym _binary_build_fs_ram_img_start=_binary_ramdisk_start \ - --redefine-sym _binary_build_fs_ram_img_end=_binary_ramdisk_end \ - --redefine-sym _binary_build_fs_ram_img_size=_binary_ramdisk_size \ + --redefine-sym _binary_build_fs_ram_scff_start=_binary_ramdisk_start \ + --redefine-sym _binary_build_fs_ram_scff_end=_binary_ramdisk_end \ + --redefine-sym _binary_build_fs_ram_scff_size=_binary_ramdisk_size \ $@ # The console font, linked into the kernel. @@ -1019,6 +1041,30 @@ $(UACPIDIR): $(patsubst $(BUILD)/uacpi/%.o,$(UACPIDIR)/source/%.c,$(filter $(BUILD)/uacpi/%,$(KERN_ACPI))): | $(UACPIDIR) @: +# zstd at -O2 whatever OPT says: the decoder runs on every first read of a +# root block, and at -O0 it is several times slower. No assembly, no BMI2 +# dispatch, no legacy formats, no tracing hooks -- the kernel has none of +# what those need. compat/ stands in for the libc headers it includes. +ZSTD_CFLAGS = -Ikernel/scff/compat -I$(ZSTDDIR) -I$(ZSTDDIR)/common -O2 \ + -DZSTD_DISABLE_ASM -DDYNAMIC_BMI2=0 -DZSTD_LEGACY_SUPPORT=0 \ + -DZSTD_TRACE=0 -DDEBUGLEVEL=0 -DZSTD_STRIP_ERROR_STRINGS \ + -DXXH_NAMESPACE=ZSTD_ +ZSTD_CC = $(CC) $(filter-out -I./include,$(CFLAGS)) $(ZSTD_CFLAGS) + +$(BUILD)/zstd/%.o: $(ZSTDDIR)/%.c | $(ZSTDDIR) + @mkdir -p $(dir $@) + $(ZSTD_CC) -Wno-unused-function -c $< -o $@ + +$(BUILD)/kernel/scff/%.o: kernel/scff/%.c | $(ZSTDDIR) + @mkdir -p $(dir $@) + $(ZSTD_CC) -c $< -o $@ + +$(ZSTDDIR): + $(MAKE) -C ports zstd + +$(patsubst $(BUILD)/zstd/%.o,$(ZSTDDIR)/%.c,$(filter $(BUILD)/zstd/%,$(KERN_ZSTD))): | $(ZSTDDIR) + @: + $(BUILD)/kernel/proc/%.o: kernel/proc/%.c @mkdir -p $(BUILD)/kernel/proc $(CC) $(CFLAGS) -c $< -o $@ @@ -1222,6 +1268,14 @@ $(SCHFS_FSCK): tools/schfs_fsck.c $(SCHFS_SRC) $(HOSTCC) $(HOSTCFLAGS) -o $@ tools/schfs_fsck.c mkschfs: $(MKSCHFS) + +# The SCFF writer, built from the same zstd release as the kernel's decoder. +MKSCFF_ZSTD = $(wildcard $(ZSTDDIR)/common/*.c $(ZSTDDIR)/compress/*.c $(ZSTDDIR)/decompress/*.c) +$(MKSCFF): tools/mkscff.c include/fs/scff.h | $(ZSTDDIR) + @mkdir -p $(BUILD)/tools + $(HOSTCC) -O2 -pthread -iquote ./include -I$(ZSTDDIR) -DZSTD_DISABLE_ASM \ + -DZSTD_LEGACY_SUPPORT=0 -o $@ tools/mkscff.c \ + $(ZSTDDIR)/common/*.c $(ZSTDDIR)/compress/*.c $(ZSTDDIR)/decompress/*.c schfs-fsck: $(SCHFS_FSCK) # End-to-end filesystem test: build an image from a directory tree, read it diff --git a/NOTICE b/NOTICE index 333b4f6..4578dce 100644 --- a/NOTICE +++ b/NOTICE @@ -151,6 +151,46 @@ interpreter, used for PCI interrupt routing, the power button and power-off. The host layer written for SchOS (kernel/acpi/) is GPL-2.0-or-later and is not part of uACPI; uACPI's own sources are unmodified. +-------------------------------------------------------------------------------- +zstd 1.5.7 BSD 3-Clause (or GPL-2.0) +-------------------------------------------------------------------------------- +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Fetched by ports/zstd. Its decoder is compiled into the kernel to read SCFF, +the compressed live root (docs/architecture/scff.md), and tools/mkscff builds +the whole library on the host to write it. zstd is dual-licensed; SchOS takes +it under the BSD licence above. The glue written for SchOS (kernel/scff/) is +GPL-2.0-or-later and is not part of zstd; zstd's own sources are unmodified. + -------------------------------------------------------------------------------- Intel 8254x device identifiers BSD 3-Clause -------------------------------------------------------------------------------- diff --git a/docs/architecture/scff.md b/docs/architecture/scff.md new file mode 100644 index 0000000..5c1439e --- /dev/null +++ b/docs/architecture/scff.md @@ -0,0 +1,79 @@ +# SCFF: the SchOS Compressed File Format + +The live root is shipped compressed. `make iso` builds the root as a SchFS +image, `tools/mkscff` writes that image as SCFF, and the kernel carries the +`.scff` file as its ramdisk. Blocks are decompressed the first time something +reads them. + +| | raw image | SCFF | +|---|---|---| +| root with Xorg and Window Maker | 358 MB | 102 MB | +| kernel and ISO | carry all 358 MB | carry 102 MB | +| memory the root takes at boot | 358 MB | what has been read | + +## Layout + +The definition is `include/fs/scff.h`, which the kernel and `mkscff` both +use. + + offset 0 struct scff_header (64 bytes, "SCFF", version 1) + index_off uint64 off[nchunks + 1] + off[i]..off[i+1] chunk i + +The image is cut into chunks of `chunk_bytes`, 128 KB by default. Each chunk +is stored in one of three ways, and its stored length says which: + +- **Zero bytes:** every byte of the chunk is zero. The slack at the end of the + image costs nothing. +- **The chunk's full length:** stored as it is, because it did not compress. +- **Anything else:** one zstd frame, with a content checksum. + +## Why it is shaped like this + +**Chunks, not one stream.** Each chunk can be decompressed without the others, +so reading one file does not mean decompressing the root. This costs a little +ratio: the 358 MB root is 104 MB in 128 KB chunks and 97 MB in 1 MB chunks. +128 KB is squashfs's default too. + +**A block image, not a compressed filesystem.** squashfs compresses files and +is read-only, so a writable root needs an overlay filesystem above it. SCFF +compresses a SchFS image instead, and the ramdisk is still a block device that +takes writes. A block that is written gets a page of its own, exactly as it +did before SCFF. SchFS does not know the image was ever compressed. + +**zstd.** It compresses close to xz at the levels a build can afford (`-19`). +It decompresses at memory speed whatever level made the frame. Its decoder is +dual BSD and GPL-2.0, and it is a plain C library. `ports/zstd` pins the +release. The kernel compiles only the decoder half. `mkscff` compiles all of +it from the same release, so neither side depends on the host's libzstd. + +## In the kernel + +`kernel/dev/ramdisk.c` reads the header at boot, checks it and the index, and +panics if either is wrong. A damaged image is found before anything mounts. + +The ramdisk's page directory covers every block. A block with no page has not +been unpacked yet. Reading one decompresses its whole chunk into `rd_chunk`, +and every block in that chunk without a page gets one. A block of zeros points +at one shared zero page and gets a real page only when it is written. + +`kernel/scff/scff.c` holds the decoder: one static zstd context in a 256 KB +workspace, so nothing allocates. **Decompression runs on a 64 KB stack of its +own.** A kernel stack is 8 KB, and a chunk is decompressed at the bottom of a +read that has already used most of it. zstd's sequence decoder needs several +KB of locals, and Linux gives it a 16 KB stack. The chunk lock (`rd_zlock`) +makes that stack its holder's alone. `kernel/scff/compat/` stands in for the +few libc headers zstd includes. + +zstd is built at `-O2` whatever `OPT` says, because it runs on every first +read of a block. + +## Building + + make iso builds build/fs-ram.scff and links it in + make iso SCFF_LEVEL=3 much faster to build, about a third bigger + make iso SCFF_CHUNK=256 bigger chunks: a little smaller, more work per miss + +`mkscff` uses every core. At `-19` the root takes about 11 seconds on 16 +threads. It decompresses every chunk again before writing it, so an image +that would not boot fails the build instead. diff --git a/docs/plan/alpine.md b/docs/plan/alpine.md index 6cf01d6..5e141ab 100644 --- a/docs/plan/alpine.md +++ b/docs/plan/alpine.md @@ -74,6 +74,7 @@ All of it found by running apk: Xorg installs into it in about a minute. All of it is gone at reboot. Keeping packages needs an installer that writes the root to a SATA or NVMe disk, and a kernel that finds its root there by the SchFS GPT type rather than as IDE disk 1. -- Xorg from apk (`xorg-server xf86-video-fbdev xf86-input-evdev`) instead of - the hand port in [xorg.md](xorg.md). The packages install; running them is - the next test. +- Xorg and Window Maker are in `world`, so the image starts with them: + `startx` runs `wmaker` from `/root/.xinitrc`. Mesa brings LLVM with it, + about 200 MB of the 358 MB root; the root ships compressed + ([SCFF](../architecture/scff.md)), so that costs about 60 MB of ISO. diff --git a/include/fs/scff.h b/include/fs/scff.h new file mode 100644 index 0000000..19e0c44 --- /dev/null +++ b/include/fs/scff.h @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// SCFF, the SchOS Compressed File Format: a block image cut into chunks, each +// compressed on its own, with an index in front. +// +// It is what the live root is shipped as. The kernel carries the compressed +// image and decompresses a chunk the first time any block in it is read, so +// the kernel and the ISO are the compressed size, and the memory the root +// takes is what has been touched, not the whole of it. See kernel/dev/ramdisk.c +// for the reader and tools/mkscff.c for the writer; this header is the one +// definition of the layout both of them use. +// +// offset 0 struct scff_header +// index_off uint64_t off[nchunks + 1], from the start of the file +// off[i]..off[i+1] chunk i +// +// A chunk is chunk_bytes of the image (the last one may be shorter). Its +// stored length says how it is kept: +// +// 0 every byte is zero, and nothing is stored +// its full length stored as it is: it did not compress +// anything else one zstd frame +// +// Little-endian throughout, which is every machine SchOS runs on. + +#ifndef SCHOS_FS_SCFF_H +#define SCHOS_FS_SCFF_H + +#include + +#define SCFF_MAGIC "SCFF" +#define SCFF_VERSION 1 +#define SCFF_CODEC_ZSTD 1 + +// The kernel decompresses one chunk at a time into a buffer this big, so it +// is the largest chunk a reader accepts. 128 KB is squashfs's default and +// the point past which a bigger chunk buys little ratio for a lot of +// decompression on every miss. +#define SCFF_CHUNK_MAX (1u << 20) + +struct scff_header { + char magic[4]; // "SCFF" + uint16_t version; // SCFF_VERSION + uint16_t codec; // SCFF_CODEC_ZSTD + uint32_t chunk_bytes; // a power of two, a multiple of 4096, <= SCFF_CHUNK_MAX + uint32_t nchunks; + uint64_t image_bytes; // the uncompressed image + uint64_t index_off; + uint64_t stored_bytes; // the whole file, header and index included + uint8_t reserved[24]; +}; + +_Static_assert(sizeof(struct scff_header) == 64, "scff_header is 64 bytes"); + +#endif diff --git a/kernel/dev/ramdisk.c b/kernel/dev/ramdisk.c index 8736a75..e988716 100644 --- a/kernel/dev/ramdisk.c +++ b/kernel/dev/ramdisk.c @@ -27,6 +27,15 @@ // Pages are not given back when SchFS frees a block: there is no discard. // A freed block is reused by the next write, so the cost is the high-water // mark, not a leak. +// +// **A compressed image.** The blob can also be SCFF (include/fs/scff.h): the +// image cut into chunks, each compressed with zstd. Then no block is served in +// place. The page directory covers the image too, and a block with no page +// has not been unpacked yet: reading it decompresses its whole chunk, and +// every block of that chunk nobody has written gets a page of its own. The +// kernel and the ISO are the compressed size -- the 360 MB root with Xorg in +// it is 104 MB -- and the root costs memory only for what has been read or +// written. A block of zeros points at one shared zero page and costs nothing. #include "core/types.h" #include "core/defs.h" @@ -37,6 +46,8 @@ #include "core/sleeplock.h" #include "fs/fs.h" #include "fs/buf.h" +#include "fs/scff.h" +#include "../scff/scff_zstd.h" #ifdef RAMDISK_ROOT // Supplied by `ld -r -b binary` over build/fs-ram.img, renamed by objcopy. @@ -46,10 +57,20 @@ extern uchar _binary_ramdisk_size[]; static uchar *rd_base; static uint64 rd_size; // bytes -static uint rd_blocks; // rd_size / BSIZE: the image, served in place +static uint rd_blocks; // blocks in the image +static uint rd_first; // the first block with a page: rd_blocks, or 0 for SCFF -// Past the image: rd_dir[i] is a page of pointers, entry j the page holding -// block rd_blocks + i*RD_PTRS + j, or 0 for one never written. 32768 pages of +// SCFF only. rd_zlock is held across a chunk's decompression, and makes +// rd_chunk -- the one chunk-sized buffer -- its holder's. +static struct scff_header *rd_scff; +static uint64 *rd_index; +static struct sleeplock rd_zlock; +static uchar rd_chunk[SCFF_CHUNK_MAX] __attribute__((aligned(PGSIZE))); +static uchar rd_zero[BSIZE] __attribute__((aligned(PGSIZE))); +static uint64 rd_unpacked; // chunks decompressed + +// rd_dir[i] is a page of pointers, entry j the page holding block +// rd_first + i*RD_PTRS + j, or 0 for one never written. 32768 pages of // 512 pointers reach 64 GB, which is as far as half of any machine this runs // on is likely to go, and costs 256 KB of pointers here. #define RD_PTRS (PGSIZE / sizeof(uchar *)) @@ -67,6 +88,32 @@ ramdiskinit(void) rd_base = _binary_ramdisk_start; rd_size = (uint64)_binary_ramdisk_size; rd_blocks = (uint)(rd_size / BSIZE); + rd_first = rd_blocks; + + if(rd_size >= sizeof(struct scff_header) && memcmp(rd_base, SCFF_MAGIC, 4) == 0){ + struct scff_header *h = (struct scff_header *)rd_base; + uint64 chunks = h->chunk_bytes ? (h->image_bytes + h->chunk_bytes - 1) / h->chunk_bytes : 0; + if(h->version != SCFF_VERSION || h->codec != SCFF_CODEC_ZSTD || + h->chunk_bytes < BSIZE || h->chunk_bytes > SCFF_CHUNK_MAX || + (h->chunk_bytes & (h->chunk_bytes - 1)) || h->image_bytes % BSIZE || + chunks != h->nchunks || h->stored_bytes > rd_size || + h->index_off + (h->nchunks + 1) * sizeof(uint64) > rd_size) + panic("ramdiskinit: the SCFF header does not describe this image"); + rd_index = (uint64 *)(rd_base + h->index_off); + for(uint i = 0; i < h->nchunks; i++) + if(rd_index[i] > rd_index[i + 1] || rd_index[i + 1] > h->stored_bytes || + rd_index[i + 1] - rd_index[i] > h->chunk_bytes) + panic("ramdiskinit: the SCFF index is damaged"); + if(scff_zstd_init() < 0) + panic("ramdiskinit: no room for the zstd decoder"); + initsleeplock(&rd_zlock, "scff"); + rd_scff = h; + rd_blocks = (uint)(h->image_bytes / BSIZE); + rd_first = 0; + cprintf("ramdisk: SCFF, %u MB in %u MB, %u chunks of %u KB\n", + (uint)(h->image_bytes >> 20), (uint)(rd_size >> 20), + h->nchunks, h->chunk_bytes >> 10); + } if(rd_blocks == 0) panic("ramdiskinit: embedded image is empty"); @@ -76,8 +123,9 @@ ramdiskinit(void) if(rd_blocks < 2) panic("ramdiskinit: embedded image smaller than a superblock"); - cprintf("ramdisk: %u blocks (%u KB) at %p\n", - rd_blocks, (uint)(rd_size / 1024), rd_base); + if(!rd_scff) + cprintf("ramdisk: %u blocks (%u KB) at %p\n", + rd_blocks, (uint)(rd_size / 1024), rd_base); #else rd_base = 0; rd_size = 0; @@ -103,8 +151,9 @@ ramdisk_blocks(void) acquire(&rd_lock); if (rd_total == 0) { uint64 extra = kalloc_free_bytes() / 2 / BSIZE; - if (extra > (uint64)RD_DIRS * RD_PTRS) - extra = (uint64)RD_DIRS * RD_PTRS; + uint64 room = (uint64)RD_DIRS * RD_PTRS - (rd_blocks - rd_first); + if (extra > room) + extra = room; rd_total = rd_blocks + extra; cprintf("ramdisk: %u MB image, %u MB of memory to grow into\n", (uint)(rd_size >> 20), (uint)(extra * BSIZE >> 20)); @@ -113,11 +162,12 @@ ramdisk_blocks(void) return rd_total; } -// The page behind a block past the image, made when `make` says so. +// The page behind a block, made when `make` says so. One that is only the +// shared zero page gets a page of its own before anything writes to it. static uchar * rd_page(uint64 blk, int make) { - uint64 i = (blk - rd_blocks) / RD_PTRS, j = (blk - rd_blocks) % RD_PTRS; + uint64 i = (blk - rd_first) / RD_PTRS, j = (blk - rd_first) % RD_PTRS; uchar *pg = 0; acquire(&rd_lock); @@ -127,7 +177,9 @@ rd_page(uint64 blk, int make) } if (rd_dir[i]) { pg = rd_dir[i][j]; - if (pg == 0 && make && (pg = (uchar *)kalloc()) != 0) { + if ((pg == 0 || pg == rd_zero) && make && (pg = (uchar *)kalloc()) != 0) { + if (rd_dir[i][j] == rd_zero) + memset(pg, 0, BSIZE); rd_dir[i][j] = pg; rd_pages++; } @@ -138,6 +190,64 @@ rd_page(uint64 blk, int make) return pg; } +// Give every block of SCFF chunk c that has no page one: its contents, or the +// zero page. Blocks already there -- written since boot, or unpacked by +// whoever held the lock before us -- are left as they are. +static void +rd_unpack(uint c) +{ + uint per = rd_scff->chunk_bytes / BSIZE; + uint64 first = (uint64)c * per; + uint n = rd_blocks - first < per ? (uint)(rd_blocks - first) : per; + uint64 len = rd_index[c + 1] - rd_index[c]; + uchar *src = rd_base + rd_index[c]; + uint64 want = (uint64)n * BSIZE; + uchar *data = rd_chunk; + + acquiresleep(&rd_zlock); + if (len == 0) { + data = 0; // all zeros + } else if (len == want) { + data = src; // stored as it is + } else { + long got = scff_zstd_decompress(rd_chunk, sizeof(rd_chunk), src, len); + if (got != (long)want) + panic("ramdisk: an SCFF chunk does not decompress"); + rd_unpacked++; + } + + for (uint k = 0; k < n; k++) { + uint64 blk = first + k, i = blk / RD_PTRS, j = blk % RD_PTRS; + uchar *b = data ? data + (uint64)k * BSIZE : 0; + int zero = 1; + if (b) + for (uint w = 0; w < BSIZE / 8 && zero; w++) + zero = ((uint64 *)b)[w] == 0; + + uchar *pg = zero ? rd_zero : (uchar *)kalloc(); + if (pg == 0) + panic("ramdisk: out of memory for the live root"); + if (!zero) + memmove(pg, b, BSIZE); + + acquire(&rd_lock); + if (rd_dir[i] == 0 && (rd_dir[i] = (uchar **)kalloc()) != 0) + memset(rd_dir[i], 0, PGSIZE); + if (rd_dir[i] == 0) + panic("ramdisk: out of memory for the live root"); + if (rd_dir[i][j] == 0) { + rd_dir[i][j] = pg; + if (!zero) + rd_pages++; + pg = 0; + } + release(&rd_lock); + if (pg && pg != rd_zero) + kfree((char *)pg); // someone was there first + } + releasesleep(&rd_zlock); +} + // Sync buf with the in-memory image. Mirrors iderw()'s contract: on a read, // fill b->data and set B_VALID; on a write, copy out and clear B_DIRTY. void @@ -149,10 +259,14 @@ ramdiskrw(struct buf *b) panic("ramdiskrw: nothing to do"); if(rd_base == 0) panic("ramdiskrw: no ramdisk"); - if(b->blockno >= rd_blocks){ + if(b->blockno >= rd_first){ if(b->blockno >= ramdisk_blocks()) panic("ramdiskrw: block out of range"); uchar *pg = rd_page(b->blockno, (b->flags & B_DIRTY) != 0); + if(pg == 0 && rd_scff && b->blockno < rd_blocks && !(b->flags & B_DIRTY)){ + rd_unpack((uint)(b->blockno / (rd_scff->chunk_bytes / BSIZE))); + pg = rd_page(b->blockno, 0); + } if(b->flags & B_DIRTY){ memmove(pg, b->data, BSIZE); b->flags &= ~B_DIRTY; diff --git a/kernel/scff/compat/assert.h b/kernel/scff/compat/assert.h new file mode 100644 index 0000000..bb6e6fd --- /dev/null +++ b/kernel/scff/compat/assert.h @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// zstd builds with DEBUGLEVEL 0, where assert() is never expanded. +#ifndef SCFF_COMPAT_ASSERT_H +#define SCFF_COMPAT_ASSERT_H +#define assert(x) ((void)0) +#endif diff --git a/kernel/scff/compat/stdio.h b/kernel/scff/compat/stdio.h new file mode 100644 index 0000000..454095e --- /dev/null +++ b/kernel/scff/compat/stdio.h @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Included by zstd's debug printing, which DEBUGLEVEL 0 compiles out. +#ifndef SCFF_COMPAT_STDIO_H +#define SCFF_COMPAT_STDIO_H +#endif diff --git a/kernel/scff/compat/stdlib.h b/kernel/scff/compat/stdlib.h new file mode 100644 index 0000000..be9b1d1 --- /dev/null +++ b/kernel/scff/compat/stdlib.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// zstd's allocator hooks. The kernel decoder is a static context in a fixed +// workspace (kernel/scff/scff.c), so nothing allocates: these exist only for +// code paths that are compiled in and never taken. +#ifndef SCFF_COMPAT_STDLIB_H +#define SCFF_COMPAT_STDLIB_H +#include +static inline void *malloc(size_t n) { (void)n; return 0; } +static inline void *calloc(size_t n, size_t s) { (void)n; (void)s; return 0; } +static inline void free(void *p) { (void)p; } +#endif diff --git a/kernel/scff/compat/string.h b/kernel/scff/compat/string.h new file mode 100644 index 0000000..e238d0d --- /dev/null +++ b/kernel/scff/compat/string.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// What zstd asks of , from the kernel's own (kernel/core/string.c). +// zstd calls __builtin_memcpy and friends where it can, so these are the +// fallbacks for the sizes the compiler does not open-code. +#ifndef SCFF_COMPAT_STRING_H +#define SCFF_COMPAT_STRING_H +#include +void *memcpy(void *, const void *, unsigned int); +void *memmove(void *, const void *, unsigned long); +void *memset(void *, int, unsigned long); +#endif diff --git a/kernel/scff/scff.c b/kernel/scff/scff.c new file mode 100644 index 0000000..5c2acfc --- /dev/null +++ b/kernel/scff/scff.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// The zstd decoder for SCFF chunks (include/fs/scff.h). +// +// One static decompression context in a fixed workspace, so nothing here +// allocates, and one caller at a time: kernel/dev/ramdisk.c holds its chunk +// lock across every call. +// +// **The decoder runs on a stack of its own.** A kernel stack is 8 KB, and a +// chunk is decompressed at the bottom of a read -- syscall, namei, SchFS, +// bread, ramdiskrw -- with most of it already used. zstd's sequence decoder +// keeps several KB of state in locals, which Linux gives it on a 16 KB stack. +// Here that would run off the end of the process's stack into whatever is +// below it. So the call is made on scff_stack, which the chunk lock makes +// this caller's alone. An interrupt taken meanwhile lands on it too, which +// is fine: it is a kernel stack like any other, and a big one. +// +// This file sees zstd's headers and none of the kernel's, which keeps the two +// sets of type definitions from meeting. + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + +#include "scff_zstd.h" + +static unsigned char scff_work[256 * 1024] __attribute__((aligned(64))); +static unsigned char scff_stack[64 * 1024] __attribute__((aligned(16))); +static ZSTD_DCtx *dctx; + +// call_on_stack(fn, arg, top): fn(arg) with %rsp at top, then back. +// %rbp keeps the old stack pointer across the call; it is callee-saved. +long call_on_stack(long (*fn)(void *), void *arg, void *top); +__asm__( + ".text\n" + ".globl call_on_stack\n" + "call_on_stack:\n" + " pushq %rbp\n" + " movq %rsp, %rbp\n" + " movq %rdx, %rsp\n" + " movq %rdi, %rax\n" + " movq %rsi, %rdi\n" + " callq *%rax\n" + " movq %rbp, %rsp\n" + " popq %rbp\n" + " retq\n"); + +struct job { + void *dst; + unsigned long cap; + const void *src; + unsigned long n; +}; + +static long +run(void *arg) +{ + struct job *j = arg; + size_t r = ZSTD_decompressDCtx(dctx, j->dst, j->cap, j->src, j->n); + return ZSTD_isError(r) ? -1 : (long)r; +} + +int +scff_zstd_init(void) +{ + if (dctx) + return 0; + if (ZSTD_estimateDCtxSize() > sizeof(scff_work)) + return -1; + dctx = ZSTD_initStaticDCtx(scff_work, sizeof(scff_work)); + return dctx ? 0 : -1; +} + +long +scff_zstd_decompress(void *dst, unsigned long cap, const void *src, unsigned long n) +{ + struct job j = { dst, cap, src, n }; + + if (!dctx) + return -1; + return call_on_stack(run, &j, scff_stack + sizeof(scff_stack)); +} diff --git a/kernel/scff/scff_zstd.h b/kernel/scff/scff_zstd.h new file mode 100644 index 0000000..8badcf6 --- /dev/null +++ b/kernel/scff/scff_zstd.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// The SCFF decoder's interface, in types both the kernel's headers and zstd's +// agree on. See kernel/scff/scff.c. +#ifndef SCHOS_SCFF_ZSTD_H +#define SCHOS_SCFF_ZSTD_H + +// 0, or -1 if the workspace is too small for this zstd. +int scff_zstd_init(void); +// Decompress one zstd frame. The length written, or -1 for a bad frame. +// One caller at a time. +long scff_zstd_decompress(void *dst, unsigned long cap, const void *src, unsigned long n); + +#endif diff --git a/ports/alpine/world b/ports/alpine/world index fab5aa1..bb9a788 100644 --- a/ports/alpine/world +++ b/ports/alpine/world @@ -16,3 +16,13 @@ ncurses-terminfo-base # The PCI and USB ID databases: lspci, fastfetch and Xorg read them. hwdata-pci hwdata-usb +# X on the framebuffer, with startx. /root/.xserverrc and .xinitrc (from +# mkaroot.sh) start it with -dumbSched and run Window Maker in it. +xorg-server +xf86-video-fbdev +xf86-input-evdev +xinit +# The desktop: Window Maker, a terminal for it, and a font for both. +windowmaker +xterm +font-dejavu diff --git a/ports/alpine/world.lock b/ports/alpine/world.lock index 90005b1..fd28a61 100644 --- a/ports/alpine/world.lock +++ b/ports/alpine/world.lock @@ -1,12 +1,91 @@ # Generated by tools/posix/alpine-lock.sh from world. Do not edit; # change world and run `make alpine-lock`. # file sha256 repository +brotli-libs-1.2.1-r0.apk 62a7e37e43bbfc1f8525a00ad2125fd5897135dc93b849733884cb78ffd8e26c main busybox-1.37.0-r31.apk 0e626fa97cf937fda7bba4ed4dde8e2f2dbb5083ac34e02180cec0c4eba9351b main busybox-binsh-1.37.0-r31.apk 93f803ae5db6b729c0f66f4afba6e907ccc17d7d5f8b8c3d25bdb4636de2c48a main +encodings-1.1.0-r0.apk f7c0eb762638d0249b4fbbe07a8e4ed6aab8e86b74c239c40e47a36219777d52 main +eudev-libs-3.2.14-r6.apk 45fdbcf37c5c0e82067e98a818d3c1fd16d0261c62128dcb6b935b86b4ec6c86 main +font-alias-1.0.6-r0.apk 754e5c1d19c2c69d5a2c3c02c7340cad7df3c26e63789516372ea164b564b927 main +fontconfig-2.17.1-r1.apk 7634cf2fb1066ec97c8f41ec90ec9f036540e056276b7bf9a10dc4ec9e44d238 main +font-cursor-misc-1.0.4-r1.apk a1a0c0659a9a48e24b4cc4b35583f4e57da9d2a0ae2cc8adb9be1143b66911b4 main +font-dejavu-2.37-r6.apk 6e7609e7a1aaed947afe4ddcc02bbdeec571dac5ff4ea1c04af9b8480b7b9dcf main +font-misc-misc-1.1.3-r1.apk 2c48f2a8529ac63243a0bdd5e7ab2329e0a8b8f996ffe95faed8b79fb05b4a52 main +freetype-2.14.3-r0.apk 09d76fde675a9ca3cfeb83d72fb09cb2e9ccf79b2937daa05a1af9a37a6b7815 main +gmp-6.3.0-r4.apk 85323ef5bce8ff234736dc2d6592ea37aebdc6348a531e27430bdfe38364d256 main hwdata-pci-0.408-r0.apk fe61404e6d2bd1e04c021d53641c0ebce97911084d77994e43a288a1278686c1 main hwdata-usb-0.408-r0.apk 272d365cb05dd085dd8022154cf71e3f3b0aaddd41c1269eeaf939c9d38675c8 main +libbsd-0.12.2-r0.apk 407597c04b41dbfe8a05b35d45a3453e32fae7df84aed3a06ff48dd4df1338b6 main +libbz2-1.0.8-r6.apk 2737e2b23ed08d4911ee62cd0163b5f4bf0379cacf4dc35b23a00209584c98fd main libcap2-2.78-r0.apk 42886a7b4bcde26347c712925c8dc7db202b6029ef9b8c7ee81577ab7bdd8fad main +libdrm-2.4.134-r0.apk 584fd84ad5030040e613fa1816097cd1706ddd542100070852b0eab6e09e12ca main +libelf-0.195-r0.apk 78c5a1993342ab1bdcdce30639fd0937e8000f6d0d85cea171f573d31decd036 main +libepoxy-1.5.10-r1.apk 0be13b4b524fe64315e0663ab615a244d75d7da73130ac1c630cd6096841444e main +libevdev-1.13.6-r0.apk b38a27a4ac55701fd2eb0032977af2664b6fc23b2f9c2cc6e893ad4452fd82d5 community +libexpat-2.9.0-r0.apk 32b127f31e0a919110a18a7f0bb3780d8fb66ba4d8d61b2d301d633f0e319d0e main +libffi-3.5.2-r1.apk 6da1a19313a40c56cac756fb11e2c4d5267831e1c22fc8323f24c943c94da6e0 main +libfontenc-1.1.9-r0.apk ea7f6d097653572991da749ac0764c2568d8902a01eea4236e56b10b90a7b0e3 main +libgcc-15.2.0-r5.apk 393dcd32629f06d7d85409c272d142d0c082772d10b87ef55ee82f47de3be637 main +libice-1.1.2-r0.apk 08daa589d57149b52b00c666b4009212aa3af95021ebdb2a359555c9bcaa76e1 main +libjpeg-turbo-3.1.3-r0.apk 6299627dd0a9156a7463511e4915a6294dfb592d8b4bdf85e55265adf1b8ea04 main +libmd-1.2.0-r0.apk f6d68d072fcefc8edd4ffe3afc341ca391f8a9a209130628fb065d98b18a2792 main libncursesw-6.6_p20260516-r0.apk cf8caa8a88bc4ce9d9e395567fefaf9ef7fbd55c50ae6155b35c1b58f3755023 main +libpciaccess-0.19-r0.apk 03fdccd1e2a0825e7f7fabbac67212e05d271d31d2eff44cc92f707e69908ad4 main +libpng-1.6.59-r0.apk d16aeb0597f2bc43e0583ac9ff779388ec160c3e22568b640122e94ba55be6d7 main +libsharpyuv-1.6.0-r0.apk 34621ff53f43030d24d1224fe9cb53b1590ab614cbce6080005cde703e6c874a main +libsm-1.2.6-r0.apk dd7eff58a46fd60e4afee9c7c1c3508920b0e1d6560129e59aee82329f68afc8 main +libstdc++-15.2.0-r5.apk 14c987b556f5385a5db18376e788c75f37d85321b8dc1920d926ea7daac1d6f6 main +libuuid-2.42.3-r1.apk 8306e5bb577696c9069fe1dfd9e1dcc39d2d481c6a1b0e707fd03c3e21aa6aa2 main +libwebp-1.6.0-r0.apk dc50c0280348330bb113079331c4a35d85e28351c7baa9268ee346aad8398288 main +libx11-1.8.14-r0.apk 016efc189c6d566061ffe206ed0780e32686c1a8c832a3917d35d2c66d61a40f main +libxau-1.0.12-r0.apk 3506cdba7ed916a6f814a010fbb127dbfdae4147107f64a15f5fa2651c5d8510 main +libxaw-1.0.16-r1.apk 8ac469cddeac40ff2e91c84cb02ba526d7c62e30a79a25c2e3b6e5269236801d main +libxcb-1.17.0-r2.apk 7935d5ad9c7137b7b3f1292d62f55f7e0cc5c919155218c3060ea46503b3f211 main +libxcvt-0.1.3-r0.apk 809b454981887c1d1d835c909c46a80650ca72a73a92268a2cf8a666ad82b6b7 community +libxdmcp-1.1.5-r1.apk 2a93f44f3f85c1eda9ed53691c35d0742b5fcd8e0057cd7380934025936cbf6f main +libxext-1.3.7-r0.apk 8afe1ee6ab8cb656e12f20dc5b7fb2d4103faee4a38298b94f26ab52e597cbc8 main +libxfont2-2.0.8-r0.apk f8bf87b9ce835cb307eb8a3d270e226c4437757478c52ba3ad384cecc2ac6fef community +libxft-2.3.9-r0.apk eba6fd85599b4fedf3acdd3ad3e2eec84ea97a5ba8f1ac570fa1ee37a7d81420 main +libxinerama-1.1.6-r0.apk 9f780efb3cf675b5253b73b84f278b40757fa7869c6fe70a9790a73f4f7e810f main +libxkbfile-1.2.0-r0.apk e575c6c314ad87e74887526aca3b1429f5c1d6290e66f2945d4053f387beea57 main +libxml2-2.13.9-r2.apk 4b2f986159c659f014b942fe8a5d70c67af475153ca826c3dd53389eea46a300 main +libxmu-1.3.1-r0.apk 5d0b06d1a4b38ac0e070e0dad5e61583e5e147e378ba50418c83e1be4b19986c main +libxpm-3.5.20-r0.apk 1ed312a00abac5a5840c1f3721921ea8183e664566c19763ed00aa9f35bb2178 main +libxrender-0.9.13-r0.apk 724aa6c6353e70869e0eb5aa5461e8fc0753b99db765de47a3ae238573dbdcf8 main +libxshmfence-1.3.3-r0.apk 53790552de7eea17a47aa4c082b06b735c545a0947c41128f97c7be95a7d3d92 main +libxt-1.3.1-r0.apk 08b167b05ac7dfb892421876cc2467214208b83b2960a9f49e84f9dbcf36e812 main +libxxf86vm-1.1.7-r0.apk 887a3c5438e1db624d07ac23d16ec08d21fb16de438d75ad2dd5637c958b4108 main +llvm22-libs-22.1.3-r1.apk dec15bdd29b6ec7fc22aee43bac153a22e6b132c599e5e5d9ccd0d159eae764f main +mcookie-2.42.3-r1.apk 64f86340991e2f283ce95b1bae21157da6849118cd9f1a2931de5c73d0f9fc3d main +mesa-26.1.6-r0.apk 8fb0e3d386f09df4f3e3643108b38197558465b2558e6d66d6a9c055adc8d1a9 main +mesa-egl-26.1.6-r0.apk ccdd25c8f3a069b018a39a5641e4f13ab7ceb511d5991e9f8fb18bd551172ffa main +mesa-gbm-26.1.6-r0.apk db6b00543e7e686f377a5c250036067742f914112839255e2e5edbbc241bfb0d main +mesa-gl-26.1.6-r0.apk 4d2d502a3938bd5dc4affec13545271571bb8bec40e7078ef01f34222bfb5d5d main +mesa-gles-26.1.6-r0.apk 2544a75e74778ab9ddcecfc8d9dc35375cb3a4ac3661e94a1156e2c528d7b59b main +mkfontscale-1.2.4-r0.apk 2385a7aad3971fcad9d862cb90937e643e633b14bf4b91f8b8ce3040a1468d20 main +mtdev-1.1.7-r0.apk 004f42bc67bc0eb573d64d802c5f0325f49013cf4aa7576d4b571399df46f3f7 community musl-1.2.6-r3.apk e142255d0fa397f14318996bc10378093dcc069dc3dd1b6818c8f28a789c21da main ncurses-terminfo-base-6.6_p20260516-r0.apk ce83be6d0bd10584a53c3a5868cc1595ce5f0f8e0b621f8f7f70b6144e521fc9 main +nettle-3.10.2-r0.apk 693c1a99b781cd17149916df30113ccd6bdddac130298c17b1286fd2ae438386 main +pixman-0.46.4-r0.apk 378550cbe348dda9b0733014375dba87a89f0077c5f3dbeba7dce8dbcdda707e main +pkgconf-2.5.1-r0.apk 20cc47b76b82e9424a35ecc4fdd7bd904ef49ada1684214910fae31bc0cab737 main +spirv-tools-1.4.341.0-r0.apk 6034c22c4f61fb76e517c1d5b11ef5c26c7154d2baeacec8b0a9dcca94ff9a93 main +tiff-4.7.2-r0.apk 757ce87ebe4923a6be9869d026e250958ca09696529c464781e8a91d2b81670b main +util-macros-1.20.2-r0.apk 7c129d085b646b13ccef3738e93038221aec840c1a6764279c5adc9950ba1bdc main +wayland-libs-client-1.25.0-r0.apk a728f7103217cafdc83ff6ad0d679c197dbbe280bd5788c8336259f7a39cb303 main +windowmaker-0.96.0-r0.apk 15b500d9afccf94a385e042b01ed9b1d709b1627d97597cea59bf63d0bab4f6b community +xauth-1.1.5-r0.apk 94e80e9687a1829a6fe95fb28d4b08a5531dab182cde8a58ff1b370b2cd293fb community +xf86-input-evdev-2.11.0-r0.apk 1f3fcd3e837b43ad5e32005e03095b547ddbfc25973f0511e9ff74bf342d2fe0 community +xf86-video-fbdev-0.5.0-r6.apk 6a8a1b16f82d8810896f00c00b356c8b8d923e9aa0ce64737ae8274ca82f507a community +xinit-1.4.4-r0.apk efb5dc1e7929824f5add7d5d51d0ecb34ecfd0a773dfca4f061adc8a353b234f community +xkbcomp-1.5.0-r0.apk 8bfbd3c8ce7180e8cf599d5a0b791993b61fa98f3a19c47aaf5fcbe30c12d0ba main +xkeyboard-config-2.47-r0.apk 8c8f9af2b78f8e955480aa88103e495d2637edac659a417075e877c88a41fed0 main +xmodmap-1.0.11-r1.apk 42489c8865d401e894b8da210f5c8e27f58c1d72c83250a505a65bdb8c45bec0 community +xorg-server-21.1.25-r0.apk 8130c341a5f047069cb35ac77a44d8bab6a2e3cb5a7d0745e3c5096b1909ecfb community +xorg-server-common-21.1.25-r0.apk bf5994707b4030f249ad44a123ad2686501c82d2002d73d0a641d9001caa37c3 community +xrdb-1.2.2-r0.apk 29649088693b5974dec3f928a5303e354f7b91e381b778006fe3011d003b1369 community +xterm-410-r0.apk eff11da325ffb1dcfa4c9902ef27d34cc27b76457e1fc8dad90f10944ecb33b8 community +xz-libs-5.8.4-r0.apk ab362dd515ead04c31a6c302163248da7aa4813e283c971e6dd3f789b453a1d9 main +zlib-1.3.2-r1.apk 63aeea03c15a2f9018f81805cfc8aa926bdf5cd68921f22149c2fbb5d0ee9f47 main zsh-5.9-r7.apk 22ec3c4de69e38437e351ba5a881f5d88e9b518a248c8aa06d36b148cf16ce32 main +zstd-libs-1.5.7-r2.apk 23c6065b0049b2406441564bcf0032515a43f78e80d76fcb85535a3803ef5d4e main diff --git a/ports/zstd/Makefile b/ports/zstd/Makefile new file mode 100644 index 0000000..33fca6b --- /dev/null +++ b/ports/zstd/Makefile @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# zstd -- the decompressor behind SCFF, the compressed live root. Dual +# BSD-3-Clause / GPL-2.0; see LICENSE and COPYING in the tarball and +# docs/licensing.md. +# +# Kernel code, like uACPI: nothing is installed into the sysroot. This +# directory fetches and unpacks a pinned release, and the top-level Makefile +# compiles the decoder half of lib/ straight into the kernel against +# kernel/scff/compat, which stands in for the libc headers it includes. The +# host side, tools/mkscff, links the system's libzstd. +# +# Chosen for what SCFF asks of a codec: compress as hard as the build likes, +# once, and decompress at memory speed on every first read of a block. + +include ../mk/port.mk + +ZSTD_VERSION := 1.5.7 +ZSTD_URL := https://github.com/facebook/zstd/releases/download/v$(ZSTD_VERSION)/zstd-$(ZSTD_VERSION).tar.gz +ZSTD_SHA256 := eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 + +TARBALL := $(DISTDIR)/zstd-$(ZSTD_VERSION).tar.gz + +$(TARBALL): | $(DISTDIR) + curl -fsSL -o $@ $(ZSTD_URL) + sh $(SHA256CHECK) "$(ZSTD_SHA256)" "$@" + +download: $(TARBALL) + +$(SRCDIR): $(TARBALL) + mkdir -p $(PORTDIR) + cd $(PORTDIR) && tar xzf $(abspath $<) + mv $(PORTDIR)/zstd-$(ZSTD_VERSION) $@ + +# The kernel build depends on this directory existing, nothing more. +all: | $(SRCDIR) + +install: + +clean: + rm -rf $(PORTDIR) + +distclean: clean + rm -f $(TARBALL) diff --git a/tools/mkscff.c b/tools/mkscff.c new file mode 100644 index 0000000..64d34d1 --- /dev/null +++ b/tools/mkscff.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// mkscff: write a block image as SCFF, the SchOS Compressed File Format. +// +// mkscff IN.img OUT.scff [--chunk KB] [--level N] [--threads N] +// +// The layout is include/fs/scff.h. Chunks are compressed on every core, since +// zstd's high levels are slow and the image is hundreds of megabytes, and each +// one is decompressed again and compared before it is written: an image the +// kernel cannot read is found here rather than at boot. +// +// Built against the zstd release the kernel's decoder comes from (ports/zstd), +// not the host's libzstd, so both sides are one version and nothing needs to +// be installed for it. + +#include +#include +#include +#include +#include +#include + +#include "fs/scff.h" +#include "zstd.h" + +static unsigned char *img; +static uint64_t img_bytes; +static uint32_t chunk_bytes = 128 * 1024; +static int level = 19; +static uint32_t nchunks; + +static unsigned char **out; // chunk i as stored, or 0 for an all-zero one +static size_t *outlen; +static uint32_t next_chunk; +static pthread_mutex_t next_lock = PTHREAD_MUTEX_INITIALIZER; + +static void +die(const char *what) +{ + fprintf(stderr, "mkscff: %s\n", what); + exit(1); +} + +static int +all_zero(const unsigned char *p, size_t n) +{ + for (size_t i = 0; i < n; i++) + if (p[i]) + return 0; + return 1; +} + +static void * +worker(void *arg) +{ + (void)arg; + ZSTD_CCtx *cc = ZSTD_createCCtx(); + ZSTD_DCtx *dc = ZSTD_createDCtx(); + size_t cap = ZSTD_compressBound(chunk_bytes); + unsigned char *buf = malloc(cap), *back = malloc(chunk_bytes); + + if (!cc || !dc || !buf || !back) + die("out of memory"); + ZSTD_CCtx_setParameter(cc, ZSTD_c_compressionLevel, level); + ZSTD_CCtx_setParameter(cc, ZSTD_c_checksumFlag, 1); + + for (;;) { + pthread_mutex_lock(&next_lock); + uint32_t i = next_chunk++; + pthread_mutex_unlock(&next_lock); + if (i >= nchunks) + break; + + const unsigned char *src = img + (uint64_t)i * chunk_bytes; + size_t n = img_bytes - (uint64_t)i * chunk_bytes; + if (n > chunk_bytes) + n = chunk_bytes; + + if (all_zero(src, n)) { + outlen[i] = 0; + continue; + } + size_t c = ZSTD_compress2(cc, buf, cap, src, n); + if (ZSTD_isError(c)) + die(ZSTD_getErrorName(c)); + if (c >= n) { // did not compress: stored as it is + out[i] = malloc(n); + memcpy(out[i], src, n); + outlen[i] = n; + continue; + } + size_t d = ZSTD_decompressDCtx(dc, back, chunk_bytes, buf, c); + if (ZSTD_isError(d) || d != n || memcmp(back, src, n) != 0) + die("a chunk did not decompress to itself"); + out[i] = malloc(c); + memcpy(out[i], buf, c); + outlen[i] = c; + } + ZSTD_freeCCtx(cc); + ZSTD_freeDCtx(dc); + free(buf); + free(back); + return 0; +} + +int +main(int argc, char **argv) +{ + const char *in = 0, *outpath = 0; + long threads = sysconf(_SC_NPROCESSORS_ONLN); + + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--chunk") && i + 1 < argc) + chunk_bytes = (uint32_t)atoi(argv[++i]) * 1024; + else if (!strcmp(argv[i], "--level") && i + 1 < argc) + level = atoi(argv[++i]); + else if (!strcmp(argv[i], "--threads") && i + 1 < argc) + threads = atol(argv[++i]); + else if (!in) + in = argv[i]; + else if (!outpath) + outpath = argv[i]; + else + die("usage: mkscff IN.img OUT.scff [--chunk KB] [--level N] [--threads N]"); + } + if (!in || !outpath) + die("usage: mkscff IN.img OUT.scff [--chunk KB] [--level N] [--threads N]"); + if (chunk_bytes < 4096 || chunk_bytes > SCFF_CHUNK_MAX || + (chunk_bytes & (chunk_bytes - 1))) + die("--chunk must be a power of two from 4 to 1024 KB"); + if (threads < 1) + threads = 1; + + FILE *f = fopen(in, "rb"); + if (!f) + die(strerror(errno)); + fseek(f, 0, SEEK_END); + img_bytes = (uint64_t)ftell(f); + fseek(f, 0, SEEK_SET); + if (img_bytes == 0 || img_bytes % 4096) + die("the image is not a whole number of 4 KB blocks"); + if (!(img = malloc(img_bytes)) || fread(img, 1, img_bytes, f) != img_bytes) + die("cannot read the image"); + fclose(f); + + nchunks = (uint32_t)((img_bytes + chunk_bytes - 1) / chunk_bytes); + out = calloc(nchunks, sizeof(*out)); + outlen = calloc(nchunks, sizeof(*outlen)); + if (!out || !outlen) + die("out of memory"); + + pthread_t *t = calloc((size_t)threads, sizeof(*t)); + for (long i = 0; i < threads; i++) + pthread_create(&t[i], 0, worker, 0); + for (long i = 0; i < threads; i++) + pthread_join(t[i], 0); + + struct scff_header h; + memset(&h, 0, sizeof(h)); + memcpy(h.magic, SCFF_MAGIC, 4); + h.version = SCFF_VERSION; + h.codec = SCFF_CODEC_ZSTD; + h.chunk_bytes = chunk_bytes; + h.nchunks = nchunks; + h.image_bytes = img_bytes; + h.index_off = sizeof(h); + + uint64_t *index = malloc(((size_t)nchunks + 1) * sizeof(uint64_t)); + uint64_t pos = sizeof(h) + ((uint64_t)nchunks + 1) * sizeof(uint64_t); + for (uint32_t i = 0; i < nchunks; i++) { + index[i] = pos; + pos += outlen[i]; + } + index[nchunks] = pos; + h.stored_bytes = pos; + + if (!(f = fopen(outpath, "wb"))) + die(strerror(errno)); + fwrite(&h, sizeof(h), 1, f); + fwrite(index, sizeof(uint64_t), (size_t)nchunks + 1, f); + for (uint32_t i = 0; i < nchunks; i++) + if (outlen[i]) + fwrite(out[i], 1, outlen[i], f); + if (fclose(f) != 0) + die("cannot write the output"); + + printf("mkscff: %llu KB -> %llu KB (%.1f%%), %u chunks of %u KB, zstd -%d\n", + (unsigned long long)(img_bytes >> 10), (unsigned long long)(pos >> 10), + 100.0 * (double)pos / (double)img_bytes, nchunks, chunk_bytes >> 10, level); + return 0; +} diff --git a/tools/posix/mkaroot.sh b/tools/posix/mkaroot.sh index 3494745..bd233f4 100644 --- a/tools/posix/mkaroot.sh +++ b/tools/posix/mkaroot.sh @@ -109,6 +109,13 @@ cat > "$P/root/.xserverrc" <<'XEOF' exec /usr/bin/X -nolisten tcp -dumbSched "$@" XEOF +# What startx runs inside it: Window Maker, which world installs. Without this +# xinit falls back to twm and an xterm, neither of which is in the image. +cat > "$P/root/.xinitrc" <<'XEOF' +#!/bin/sh +exec wmaker +XEOF + # Mount points and runtime directories are alpine-baselayout's; an empty # directory here only duplicates it. for d in dev proc sys run tmp srv; do diff --git a/tools/posix/mkiso.sh b/tools/posix/mkiso.sh index 4cf408a..09c120a 100644 --- a/tools/posix/mkiso.sh +++ b/tools/posix/mkiso.sh @@ -3,11 +3,16 @@ # # tools/posix/mkiso.sh [output.iso] # -# The disc carries one El Torito boot entry and a real directory tree: +# The disc carries one El Torito boot entry and the loader and kernel as files: # # /boot/efiboot.img a FAT image holding the loader and the kernel # /EFI/BOOT/BOOTX64.EFI the same loader, visible as a file on the disc -# /bin /etc /usr ... the staged sysroot, as an ordinary hierarchy +# /schos/kernel the same kernel, for chainloaders (below) +# +# The root filesystem is inside the kernel, as its ramdisk. The disc used to +# carry it a third time as an ISO 9660 tree, which nothing reads -- SchOS has +# no ISO 9660 driver -- and once Xorg was in the root that copy alone was +# 360 MB. # # --- Why only one entry ----------------------------------------------------- # @@ -69,7 +74,6 @@ cd "$repo" BUILD=${BUILD:-build} STAGING="$BUILD/iso" -SYSROOT=${SYSROOT:-sysroot} KERNEL="$BUILD/kernel/kernel" EFIAPP="$BUILD/uefi/BOOTX64.EFI" @@ -113,9 +117,6 @@ cp "$KERNEL" "$STAGING/schos/kernel" # this FAT image -- not the ISO 9660 tree around it. Without the kernel in here # the loader comes up and halts with EFI_NOT_FOUND, which is what it did. # -# The ISO 9660 tree still carries the sysroot for a running system to read; the -# loader's own two files live in here because that is the volume it is handed. -# # Sized from what actually goes in plus slack, rather than fixed: a hardcoded # size is a thing that breaks quietly the first time the kernel grows. EFI_IMG="$STAGING/boot/efiboot.img" @@ -145,15 +146,6 @@ if true; then mcopy -i "$EFI_IMG" "$KERNEL" ::/schos/kernel fi -# The sysroot, as an ordinary hierarchy. Copied rather than linked so that -# whatever ends up on the disc is a snapshot, not a view of a tree that may be -# rebuilt underneath it. -if [ -d "$SYSROOT" ]; then - for d in "$SYSROOT"/*; do - [ -e "$d" ] || continue - cp -a "$d" "$STAGING/" - done -fi # xorriso if it is here, mkisofs otherwise. Both accept the El Torito flags # below unchanged -- xorriso's -as mkisofs mode exists precisely to be @@ -173,6 +165,11 @@ if command -v xorriso >/dev/null 2>&1; then # of type 0xEF is what every UEFI firmware looks for on a removable # disk, MBR rather than GPT because the oldest ones read that best. HYBRID="-append_partition 2 0xef $EFI_IMG" + # El Torito keeps its own copy of the image in the tree, the layout real + # machines have booted. Pointing it into the partition instead + # (--interval:appended_partition_2:all::, as Arch does) saves a copy, and + # an ASUS laptop stopped listing the stick the day that was tried. With + # the root compressed (SCFF) the copy is only the kernel's size. elif command -v mkisofs >/dev/null 2>&1; then MKISOFS="mkisofs" HYBRID=""