SCFF: the live root ships compressed, with Xorg and Window Maker

SCFF, the SchOS Compressed File Format (include/fs/scff.h), is a block
image cut into 128 KB chunks, each its own zstd frame, behind an index.
tools/mkscff writes it from fs-ram.img on every core (-19, 11 s) and
checks each chunk decompresses to itself. The kernel carries the .scff
as its ramdisk and unpacks a chunk the first time a block in it is
read; zero blocks share one page. The kernel and the ISO are the
compressed size, and the root costs memory only for what is touched.
See docs/architecture/scff.md.

zstd 1.5.7 comes from ports/zstd like uACPI: the decoder is compiled
into the kernel at -O2, with a static context and a 64 KB stack of its
own, and mkscff builds the whole library from the same release.

world now has xorg-server, the fbdev and evdev drivers, xinit, Window
Maker, xterm and DejaVu, and /root/.xinitrc runs wmaker. That root is
358 MB, 102 MB as SCFF.

The ISO no longer carries the root a third time as an ISO 9660 tree,
which nothing read. schos.iso: 1389 MB -> 318 MB. El Torito keeps its
own copy of the EFI image, the layout real machines have booted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4
This commit is contained in:
Nexoniarz 2026-10-24 21:17:31 +02:00
commit df1394eaa3
18 changed files with 832 additions and 38 deletions

View file

@ -103,6 +103,18 @@ KERN_ACPI = $(addprefix $(BUILD)/uacpi/, \
uacpi.o utilities.o)
KERN_ACPI += $(addprefix $(BUILD)/kernel/acpi/, host.o aml.o)
# zstd's decoder, for SCFF -- the compressed live root (include/fs/scff.h).
# Unpacked by ports/zstd into build/ports/zstd/src and compiled into the kernel
# like uACPI; kernel/scff/ is the host side of it.
ZSTDDIR = $(BUILD)/ports/zstd/src/lib
KERN_ZSTD = $(addprefix $(BUILD)/zstd/, \
common/entropy_common.o common/error_private.o \
common/fse_decompress.o common/zstd_common.o common/xxhash.o \
common/debug.o decompress/huf_decompress.o \
decompress/zstd_ddict.o decompress/zstd_decompress.o \
decompress/zstd_decompress_block.o)
KERN_ZSTD += $(BUILD)/kernel/scff/scff.o
KERN_PROC = $(addprefix $(BUILD)/kernel/proc/, \
proc.o)
@ -114,7 +126,7 @@ KERN_SYS = $(addprefix $(BUILD)/kernel/sys/, \
KERN_ARCH += $(BUILD)/kernel/arch/x86_64/vectors.o
KERNOBJS = $(KERN_ARCH) $(KERN_CORE) $(KERN_DEV) $(KERN_DRIVERS) \
$(KERN_FS) $(KERN_NET) $(KERN_ACPI) $(KERN_PROC) $(KERN_SYS)
$(KERN_FS) $(KERN_NET) $(KERN_ACPI) $(KERN_ZSTD) $(KERN_PROC) $(KERN_SYS)
# entry.o must be first in link order; remove it from KERNOBJS for the rule
KERN_NOENTRY = $(filter-out $(BUILD)/kernel/arch/x86_64/entry.o, $(KERNOBJS))
@ -392,6 +404,7 @@ endif
# looks satisfied.
MKFS = $(BUILD)/tools/mkfs$(EXE)
MKSCHFS = $(BUILD)/tools/mkschfs$(EXE)
MKSCFF = $(BUILD)/tools/mkscff$(EXE)
SCHFS_TEST = $(BUILD)/tools/schfs_test$(EXE)
SCHFS_OBJT = $(BUILD)/tools/schfs_objtest$(EXE)
SCHFS_FSCK = $(BUILD)/tools/schfs_fsck$(EXE)
@ -431,7 +444,7 @@ CFLAGS += $(NO_SSP) $(CFLAGS_EXTRA)
# copies and memset into %xmm, and an interrupt then scribbles over the
# registers of the program it interrupted. Kernel objects only -- cmd/ is
# built with these CFLAGS too and is welcome to use SSE.
$(BUILD)/kernel/%.o $(BUILD)/lwip/%.o $(BUILD)/uacpi/%.o: CFLAGS += -mgeneral-regs-only
$(BUILD)/kernel/%.o $(BUILD)/lwip/%.o $(BUILD)/uacpi/%.o $(BUILD)/zstd/%.o: CFLAGS += -mgeneral-regs-only
# lwIP's own headers, plus ours (cc.h and lwipopts.h live in kernel/net/lwip).
# Only the network files are compiled with these: putting them in the global
@ -768,9 +781,9 @@ iso:
$(MAKE) RAMROOT=1 schos.iso
# One El Torito entry, pointing at a FAT image holding the loader and kernel.
# sysroot-install because the disc also carries the tree as a real hierarchy.
schos.iso: $(BUILD)/kernel/kernel $(BUILD)/uefi/BOOTX64.EFI tools/posix/mkiso.sh sysroot-install $(ROOT_STAMP)
SYSROOT=$(ROOT_TREE) sh tools/posix/mkiso.sh $@
# The root is in the kernel, as its ramdisk; the disc carries no other copy.
schos.iso: $(BUILD)/kernel/kernel $(BUILD)/uefi/BOOTX64.EFI tools/posix/mkiso.sh $(ROOT_STAMP)
sh tools/posix/mkiso.sh $@
# A partitioned disk -- GPT, an ESP and a SchFS partition -- which is what a
# real machine boots from: dd it to a USB stick or a disk. See
@ -847,13 +860,22 @@ $(BUILD)/fs-ram.img: $(MKSCHFS) $(ROOT_STAMP)
$(MKSCHFS) $@ --size $$((used + used / 8 + $(RAMDISK_SLACK))) --label schos \
$(RAMDISK_EXCLUDE_ARGS) --sysroot $(ROOT_TREE)
$(BUILD)/kernel/ramdisk_obj: $(BUILD)/fs-ram.img
# The image goes into the kernel as SCFF (include/fs/scff.h): 128 KB chunks,
# each its own zstd frame, unpacked by the kernel the first time a block in it
# is read. -19 is slow to write and costs nothing to read -- zstd decodes at
# the same speed whatever level made the frame -- and mkscff uses every core.
SCFF_LEVEL ?= 19
SCFF_CHUNK ?= 128
$(BUILD)/fs-ram.scff: $(BUILD)/fs-ram.img $(MKSCFF)
$(MKSCFF) $< $@ --level $(SCFF_LEVEL) --chunk $(SCFF_CHUNK)
$(BUILD)/kernel/ramdisk_obj: $(BUILD)/fs-ram.scff
@mkdir -p $(BUILD)/kernel
$(LD) -r -b binary -m elf_x86_64 -o $@ $<
$(OBJCOPY) \
--redefine-sym _binary_build_fs_ram_img_start=_binary_ramdisk_start \
--redefine-sym _binary_build_fs_ram_img_end=_binary_ramdisk_end \
--redefine-sym _binary_build_fs_ram_img_size=_binary_ramdisk_size \
--redefine-sym _binary_build_fs_ram_scff_start=_binary_ramdisk_start \
--redefine-sym _binary_build_fs_ram_scff_end=_binary_ramdisk_end \
--redefine-sym _binary_build_fs_ram_scff_size=_binary_ramdisk_size \
$@
# The console font, linked into the kernel.
@ -1019,6 +1041,30 @@ $(UACPIDIR):
$(patsubst $(BUILD)/uacpi/%.o,$(UACPIDIR)/source/%.c,$(filter $(BUILD)/uacpi/%,$(KERN_ACPI))): | $(UACPIDIR)
@:
# zstd at -O2 whatever OPT says: the decoder runs on every first read of a
# root block, and at -O0 it is several times slower. No assembly, no BMI2
# dispatch, no legacy formats, no tracing hooks -- the kernel has none of
# what those need. compat/ stands in for the libc headers it includes.
ZSTD_CFLAGS = -Ikernel/scff/compat -I$(ZSTDDIR) -I$(ZSTDDIR)/common -O2 \
-DZSTD_DISABLE_ASM -DDYNAMIC_BMI2=0 -DZSTD_LEGACY_SUPPORT=0 \
-DZSTD_TRACE=0 -DDEBUGLEVEL=0 -DZSTD_STRIP_ERROR_STRINGS \
-DXXH_NAMESPACE=ZSTD_
ZSTD_CC = $(CC) $(filter-out -I./include,$(CFLAGS)) $(ZSTD_CFLAGS)
$(BUILD)/zstd/%.o: $(ZSTDDIR)/%.c | $(ZSTDDIR)
@mkdir -p $(dir $@)
$(ZSTD_CC) -Wno-unused-function -c $< -o $@
$(BUILD)/kernel/scff/%.o: kernel/scff/%.c | $(ZSTDDIR)
@mkdir -p $(dir $@)
$(ZSTD_CC) -c $< -o $@
$(ZSTDDIR):
$(MAKE) -C ports zstd
$(patsubst $(BUILD)/zstd/%.o,$(ZSTDDIR)/%.c,$(filter $(BUILD)/zstd/%,$(KERN_ZSTD))): | $(ZSTDDIR)
@:
$(BUILD)/kernel/proc/%.o: kernel/proc/%.c
@mkdir -p $(BUILD)/kernel/proc
$(CC) $(CFLAGS) -c $< -o $@
@ -1222,6 +1268,14 @@ $(SCHFS_FSCK): tools/schfs_fsck.c $(SCHFS_SRC)
$(HOSTCC) $(HOSTCFLAGS) -o $@ tools/schfs_fsck.c
mkschfs: $(MKSCHFS)
# The SCFF writer, built from the same zstd release as the kernel's decoder.
MKSCFF_ZSTD = $(wildcard $(ZSTDDIR)/common/*.c $(ZSTDDIR)/compress/*.c $(ZSTDDIR)/decompress/*.c)
$(MKSCFF): tools/mkscff.c include/fs/scff.h | $(ZSTDDIR)
@mkdir -p $(BUILD)/tools
$(HOSTCC) -O2 -pthread -iquote ./include -I$(ZSTDDIR) -DZSTD_DISABLE_ASM \
-DZSTD_LEGACY_SUPPORT=0 -o $@ tools/mkscff.c \
$(ZSTDDIR)/common/*.c $(ZSTDDIR)/compress/*.c $(ZSTDDIR)/decompress/*.c
schfs-fsck: $(SCHFS_FSCK)
# End-to-end filesystem test: build an image from a directory tree, read it

40
NOTICE
View file

@ -151,6 +151,46 @@ interpreter, used for PCI interrupt routing, the power button and power-off.
The host layer written for SchOS (kernel/acpi/) is GPL-2.0-or-later and is not
part of uACPI; uACPI's own sources are unmodified.
--------------------------------------------------------------------------------
zstd 1.5.7 BSD 3-Clause (or GPL-2.0)
--------------------------------------------------------------------------------
BSD License
For Zstandard software
Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name Facebook, nor Meta, nor the names of its contributors may
be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Fetched by ports/zstd. Its decoder is compiled into the kernel to read SCFF,
the compressed live root (docs/architecture/scff.md), and tools/mkscff builds
the whole library on the host to write it. zstd is dual-licensed; SchOS takes
it under the BSD licence above. The glue written for SchOS (kernel/scff/) is
GPL-2.0-or-later and is not part of zstd; zstd's own sources are unmodified.
--------------------------------------------------------------------------------
Intel 8254x device identifiers BSD 3-Clause
--------------------------------------------------------------------------------

79
docs/architecture/scff.md Normal file
View file

@ -0,0 +1,79 @@
# SCFF: the SchOS Compressed File Format
The live root is shipped compressed. `make iso` builds the root as a SchFS
image, `tools/mkscff` writes that image as SCFF, and the kernel carries the
`.scff` file as its ramdisk. Blocks are decompressed the first time something
reads them.
| | raw image | SCFF |
|---|---|---|
| root with Xorg and Window Maker | 358 MB | 102 MB |
| kernel and ISO | carry all 358 MB | carry 102 MB |
| memory the root takes at boot | 358 MB | what has been read |
## Layout
The definition is `include/fs/scff.h`, which the kernel and `mkscff` both
use.
offset 0 struct scff_header (64 bytes, "SCFF", version 1)
index_off uint64 off[nchunks + 1]
off[i]..off[i+1] chunk i
The image is cut into chunks of `chunk_bytes`, 128 KB by default. Each chunk
is stored in one of three ways, and its stored length says which:
- **Zero bytes:** every byte of the chunk is zero. The slack at the end of the
image costs nothing.
- **The chunk's full length:** stored as it is, because it did not compress.
- **Anything else:** one zstd frame, with a content checksum.
## Why it is shaped like this
**Chunks, not one stream.** Each chunk can be decompressed without the others,
so reading one file does not mean decompressing the root. This costs a little
ratio: the 358 MB root is 104 MB in 128 KB chunks and 97 MB in 1 MB chunks.
128 KB is squashfs's default too.
**A block image, not a compressed filesystem.** squashfs compresses files and
is read-only, so a writable root needs an overlay filesystem above it. SCFF
compresses a SchFS image instead, and the ramdisk is still a block device that
takes writes. A block that is written gets a page of its own, exactly as it
did before SCFF. SchFS does not know the image was ever compressed.
**zstd.** It compresses close to xz at the levels a build can afford (`-19`).
It decompresses at memory speed whatever level made the frame. Its decoder is
dual BSD and GPL-2.0, and it is a plain C library. `ports/zstd` pins the
release. The kernel compiles only the decoder half. `mkscff` compiles all of
it from the same release, so neither side depends on the host's libzstd.
## In the kernel
`kernel/dev/ramdisk.c` reads the header at boot, checks it and the index, and
panics if either is wrong. A damaged image is found before anything mounts.
The ramdisk's page directory covers every block. A block with no page has not
been unpacked yet. Reading one decompresses its whole chunk into `rd_chunk`,
and every block in that chunk without a page gets one. A block of zeros points
at one shared zero page and gets a real page only when it is written.
`kernel/scff/scff.c` holds the decoder: one static zstd context in a 256 KB
workspace, so nothing allocates. **Decompression runs on a 64 KB stack of its
own.** A kernel stack is 8 KB, and a chunk is decompressed at the bottom of a
read that has already used most of it. zstd's sequence decoder needs several
KB of locals, and Linux gives it a 16 KB stack. The chunk lock (`rd_zlock`)
makes that stack its holder's alone. `kernel/scff/compat/` stands in for the
few libc headers zstd includes.
zstd is built at `-O2` whatever `OPT` says, because it runs on every first
read of a block.
## Building
make iso builds build/fs-ram.scff and links it in
make iso SCFF_LEVEL=3 much faster to build, about a third bigger
make iso SCFF_CHUNK=256 bigger chunks: a little smaller, more work per miss
`mkscff` uses every core. At `-19` the root takes about 11 seconds on 16
threads. It decompresses every chunk again before writing it, so an image
that would not boot fails the build instead.

View file

@ -74,6 +74,7 @@ All of it found by running apk:
Xorg installs into it in about a minute. All of it is gone at reboot. Keeping packages needs an installer that writes the root
to a SATA or NVMe disk, and a kernel that finds its root there by the
SchFS GPT type rather than as IDE disk 1.
- Xorg from apk (`xorg-server xf86-video-fbdev xf86-input-evdev`) instead of
the hand port in [xorg.md](xorg.md). The packages install; running them is
the next test.
- Xorg and Window Maker are in `world`, so the image starts with them:
`startx` runs `wmaker` from `/root/.xinitrc`. Mesa brings LLVM with it,
about 200 MB of the 358 MB root; the root ships compressed
([SCFF](../architecture/scff.md)), so that costs about 60 MB of ISO.

54
include/fs/scff.h Normal file
View file

@ -0,0 +1,54 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// SCFF, the SchOS Compressed File Format: a block image cut into chunks, each
// compressed on its own, with an index in front.
//
// It is what the live root is shipped as. The kernel carries the compressed
// image and decompresses a chunk the first time any block in it is read, so
// the kernel and the ISO are the compressed size, and the memory the root
// takes is what has been touched, not the whole of it. See kernel/dev/ramdisk.c
// for the reader and tools/mkscff.c for the writer; this header is the one
// definition of the layout both of them use.
//
// offset 0 struct scff_header
// index_off uint64_t off[nchunks + 1], from the start of the file
// off[i]..off[i+1] chunk i
//
// A chunk is chunk_bytes of the image (the last one may be shorter). Its
// stored length says how it is kept:
//
// 0 every byte is zero, and nothing is stored
// its full length stored as it is: it did not compress
// anything else one zstd frame
//
// Little-endian throughout, which is every machine SchOS runs on.
#ifndef SCHOS_FS_SCFF_H
#define SCHOS_FS_SCFF_H
#include <stdint.h>
#define SCFF_MAGIC "SCFF"
#define SCFF_VERSION 1
#define SCFF_CODEC_ZSTD 1
// The kernel decompresses one chunk at a time into a buffer this big, so it
// is the largest chunk a reader accepts. 128 KB is squashfs's default and
// the point past which a bigger chunk buys little ratio for a lot of
// decompression on every miss.
#define SCFF_CHUNK_MAX (1u << 20)
struct scff_header {
char magic[4]; // "SCFF"
uint16_t version; // SCFF_VERSION
uint16_t codec; // SCFF_CODEC_ZSTD
uint32_t chunk_bytes; // a power of two, a multiple of 4096, <= SCFF_CHUNK_MAX
uint32_t nchunks;
uint64_t image_bytes; // the uncompressed image
uint64_t index_off;
uint64_t stored_bytes; // the whole file, header and index included
uint8_t reserved[24];
};
_Static_assert(sizeof(struct scff_header) == 64, "scff_header is 64 bytes");
#endif

View file

@ -27,6 +27,15 @@
// Pages are not given back when SchFS frees a block: there is no discard.
// A freed block is reused by the next write, so the cost is the high-water
// mark, not a leak.
//
// **A compressed image.** The blob can also be SCFF (include/fs/scff.h): the
// image cut into chunks, each compressed with zstd. Then no block is served in
// place. The page directory covers the image too, and a block with no page
// has not been unpacked yet: reading it decompresses its whole chunk, and
// every block of that chunk nobody has written gets a page of its own. The
// kernel and the ISO are the compressed size -- the 360 MB root with Xorg in
// it is 104 MB -- and the root costs memory only for what has been read or
// written. A block of zeros points at one shared zero page and costs nothing.
#include "core/types.h"
#include "core/defs.h"
@ -37,6 +46,8 @@
#include "core/sleeplock.h"
#include "fs/fs.h"
#include "fs/buf.h"
#include "fs/scff.h"
#include "../scff/scff_zstd.h"
#ifdef RAMDISK_ROOT
// Supplied by `ld -r -b binary` over build/fs-ram.img, renamed by objcopy.
@ -46,10 +57,20 @@ extern uchar _binary_ramdisk_size[];
static uchar *rd_base;
static uint64 rd_size; // bytes
static uint rd_blocks; // rd_size / BSIZE: the image, served in place
static uint rd_blocks; // blocks in the image
static uint rd_first; // the first block with a page: rd_blocks, or 0 for SCFF
// Past the image: rd_dir[i] is a page of pointers, entry j the page holding
// block rd_blocks + i*RD_PTRS + j, or 0 for one never written. 32768 pages of
// SCFF only. rd_zlock is held across a chunk's decompression, and makes
// rd_chunk -- the one chunk-sized buffer -- its holder's.
static struct scff_header *rd_scff;
static uint64 *rd_index;
static struct sleeplock rd_zlock;
static uchar rd_chunk[SCFF_CHUNK_MAX] __attribute__((aligned(PGSIZE)));
static uchar rd_zero[BSIZE] __attribute__((aligned(PGSIZE)));
static uint64 rd_unpacked; // chunks decompressed
// rd_dir[i] is a page of pointers, entry j the page holding block
// rd_first + i*RD_PTRS + j, or 0 for one never written. 32768 pages of
// 512 pointers reach 64 GB, which is as far as half of any machine this runs
// on is likely to go, and costs 256 KB of pointers here.
#define RD_PTRS (PGSIZE / sizeof(uchar *))
@ -67,6 +88,32 @@ ramdiskinit(void)
rd_base = _binary_ramdisk_start;
rd_size = (uint64)_binary_ramdisk_size;
rd_blocks = (uint)(rd_size / BSIZE);
rd_first = rd_blocks;
if(rd_size >= sizeof(struct scff_header) && memcmp(rd_base, SCFF_MAGIC, 4) == 0){
struct scff_header *h = (struct scff_header *)rd_base;
uint64 chunks = h->chunk_bytes ? (h->image_bytes + h->chunk_bytes - 1) / h->chunk_bytes : 0;
if(h->version != SCFF_VERSION || h->codec != SCFF_CODEC_ZSTD ||
h->chunk_bytes < BSIZE || h->chunk_bytes > SCFF_CHUNK_MAX ||
(h->chunk_bytes & (h->chunk_bytes - 1)) || h->image_bytes % BSIZE ||
chunks != h->nchunks || h->stored_bytes > rd_size ||
h->index_off + (h->nchunks + 1) * sizeof(uint64) > rd_size)
panic("ramdiskinit: the SCFF header does not describe this image");
rd_index = (uint64 *)(rd_base + h->index_off);
for(uint i = 0; i < h->nchunks; i++)
if(rd_index[i] > rd_index[i + 1] || rd_index[i + 1] > h->stored_bytes ||
rd_index[i + 1] - rd_index[i] > h->chunk_bytes)
panic("ramdiskinit: the SCFF index is damaged");
if(scff_zstd_init() < 0)
panic("ramdiskinit: no room for the zstd decoder");
initsleeplock(&rd_zlock, "scff");
rd_scff = h;
rd_blocks = (uint)(h->image_bytes / BSIZE);
rd_first = 0;
cprintf("ramdisk: SCFF, %u MB in %u MB, %u chunks of %u KB\n",
(uint)(h->image_bytes >> 20), (uint)(rd_size >> 20),
h->nchunks, h->chunk_bytes >> 10);
}
if(rd_blocks == 0)
panic("ramdiskinit: embedded image is empty");
@ -76,8 +123,9 @@ ramdiskinit(void)
if(rd_blocks < 2)
panic("ramdiskinit: embedded image smaller than a superblock");
cprintf("ramdisk: %u blocks (%u KB) at %p\n",
rd_blocks, (uint)(rd_size / 1024), rd_base);
if(!rd_scff)
cprintf("ramdisk: %u blocks (%u KB) at %p\n",
rd_blocks, (uint)(rd_size / 1024), rd_base);
#else
rd_base = 0;
rd_size = 0;
@ -103,8 +151,9 @@ ramdisk_blocks(void)
acquire(&rd_lock);
if (rd_total == 0) {
uint64 extra = kalloc_free_bytes() / 2 / BSIZE;
if (extra > (uint64)RD_DIRS * RD_PTRS)
extra = (uint64)RD_DIRS * RD_PTRS;
uint64 room = (uint64)RD_DIRS * RD_PTRS - (rd_blocks - rd_first);
if (extra > room)
extra = room;
rd_total = rd_blocks + extra;
cprintf("ramdisk: %u MB image, %u MB of memory to grow into\n",
(uint)(rd_size >> 20), (uint)(extra * BSIZE >> 20));
@ -113,11 +162,12 @@ ramdisk_blocks(void)
return rd_total;
}
// The page behind a block past the image, made when `make` says so.
// The page behind a block, made when `make` says so. One that is only the
// shared zero page gets a page of its own before anything writes to it.
static uchar *
rd_page(uint64 blk, int make)
{
uint64 i = (blk - rd_blocks) / RD_PTRS, j = (blk - rd_blocks) % RD_PTRS;
uint64 i = (blk - rd_first) / RD_PTRS, j = (blk - rd_first) % RD_PTRS;
uchar *pg = 0;
acquire(&rd_lock);
@ -127,7 +177,9 @@ rd_page(uint64 blk, int make)
}
if (rd_dir[i]) {
pg = rd_dir[i][j];
if (pg == 0 && make && (pg = (uchar *)kalloc()) != 0) {
if ((pg == 0 || pg == rd_zero) && make && (pg = (uchar *)kalloc()) != 0) {
if (rd_dir[i][j] == rd_zero)
memset(pg, 0, BSIZE);
rd_dir[i][j] = pg;
rd_pages++;
}
@ -138,6 +190,64 @@ rd_page(uint64 blk, int make)
return pg;
}
// Give every block of SCFF chunk c that has no page one: its contents, or the
// zero page. Blocks already there -- written since boot, or unpacked by
// whoever held the lock before us -- are left as they are.
static void
rd_unpack(uint c)
{
uint per = rd_scff->chunk_bytes / BSIZE;
uint64 first = (uint64)c * per;
uint n = rd_blocks - first < per ? (uint)(rd_blocks - first) : per;
uint64 len = rd_index[c + 1] - rd_index[c];
uchar *src = rd_base + rd_index[c];
uint64 want = (uint64)n * BSIZE;
uchar *data = rd_chunk;
acquiresleep(&rd_zlock);
if (len == 0) {
data = 0; // all zeros
} else if (len == want) {
data = src; // stored as it is
} else {
long got = scff_zstd_decompress(rd_chunk, sizeof(rd_chunk), src, len);
if (got != (long)want)
panic("ramdisk: an SCFF chunk does not decompress");
rd_unpacked++;
}
for (uint k = 0; k < n; k++) {
uint64 blk = first + k, i = blk / RD_PTRS, j = blk % RD_PTRS;
uchar *b = data ? data + (uint64)k * BSIZE : 0;
int zero = 1;
if (b)
for (uint w = 0; w < BSIZE / 8 && zero; w++)
zero = ((uint64 *)b)[w] == 0;
uchar *pg = zero ? rd_zero : (uchar *)kalloc();
if (pg == 0)
panic("ramdisk: out of memory for the live root");
if (!zero)
memmove(pg, b, BSIZE);
acquire(&rd_lock);
if (rd_dir[i] == 0 && (rd_dir[i] = (uchar **)kalloc()) != 0)
memset(rd_dir[i], 0, PGSIZE);
if (rd_dir[i] == 0)
panic("ramdisk: out of memory for the live root");
if (rd_dir[i][j] == 0) {
rd_dir[i][j] = pg;
if (!zero)
rd_pages++;
pg = 0;
}
release(&rd_lock);
if (pg && pg != rd_zero)
kfree((char *)pg); // someone was there first
}
releasesleep(&rd_zlock);
}
// Sync buf with the in-memory image. Mirrors iderw()'s contract: on a read,
// fill b->data and set B_VALID; on a write, copy out and clear B_DIRTY.
void
@ -149,10 +259,14 @@ ramdiskrw(struct buf *b)
panic("ramdiskrw: nothing to do");
if(rd_base == 0)
panic("ramdiskrw: no ramdisk");
if(b->blockno >= rd_blocks){
if(b->blockno >= rd_first){
if(b->blockno >= ramdisk_blocks())
panic("ramdiskrw: block out of range");
uchar *pg = rd_page(b->blockno, (b->flags & B_DIRTY) != 0);
if(pg == 0 && rd_scff && b->blockno < rd_blocks && !(b->flags & B_DIRTY)){
rd_unpack((uint)(b->blockno / (rd_scff->chunk_bytes / BSIZE)));
pg = rd_page(b->blockno, 0);
}
if(b->flags & B_DIRTY){
memmove(pg, b->data, BSIZE);
b->flags &= ~B_DIRTY;

View file

@ -0,0 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// zstd builds with DEBUGLEVEL 0, where assert() is never expanded.
#ifndef SCFF_COMPAT_ASSERT_H
#define SCFF_COMPAT_ASSERT_H
#define assert(x) ((void)0)
#endif

View file

@ -0,0 +1,5 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// Included by zstd's debug printing, which DEBUGLEVEL 0 compiles out.
#ifndef SCFF_COMPAT_STDIO_H
#define SCFF_COMPAT_STDIO_H
#endif

View file

@ -0,0 +1,11 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// zstd's allocator hooks. The kernel decoder is a static context in a fixed
// workspace (kernel/scff/scff.c), so nothing allocates: these exist only for
// code paths that are compiled in and never taken.
#ifndef SCFF_COMPAT_STDLIB_H
#define SCFF_COMPAT_STDLIB_H
#include <stddef.h>
static inline void *malloc(size_t n) { (void)n; return 0; }
static inline void *calloc(size_t n, size_t s) { (void)n; (void)s; return 0; }
static inline void free(void *p) { (void)p; }
#endif

View file

@ -0,0 +1,11 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// What zstd asks of <string.h>, from the kernel's own (kernel/core/string.c).
// zstd calls __builtin_memcpy and friends where it can, so these are the
// fallbacks for the sizes the compiler does not open-code.
#ifndef SCFF_COMPAT_STRING_H
#define SCFF_COMPAT_STRING_H
#include <stddef.h>
void *memcpy(void *, const void *, unsigned int);
void *memmove(void *, const void *, unsigned long);
void *memset(void *, int, unsigned long);
#endif

80
kernel/scff/scff.c Normal file
View file

@ -0,0 +1,80 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// The zstd decoder for SCFF chunks (include/fs/scff.h).
//
// One static decompression context in a fixed workspace, so nothing here
// allocates, and one caller at a time: kernel/dev/ramdisk.c holds its chunk
// lock across every call.
//
// **The decoder runs on a stack of its own.** A kernel stack is 8 KB, and a
// chunk is decompressed at the bottom of a read -- syscall, namei, SchFS,
// bread, ramdiskrw -- with most of it already used. zstd's sequence decoder
// keeps several KB of state in locals, which Linux gives it on a 16 KB stack.
// Here that would run off the end of the process's stack into whatever is
// below it. So the call is made on scff_stack, which the chunk lock makes
// this caller's alone. An interrupt taken meanwhile lands on it too, which
// is fine: it is a kernel stack like any other, and a big one.
//
// This file sees zstd's headers and none of the kernel's, which keeps the two
// sets of type definitions from meeting.
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "scff_zstd.h"
static unsigned char scff_work[256 * 1024] __attribute__((aligned(64)));
static unsigned char scff_stack[64 * 1024] __attribute__((aligned(16)));
static ZSTD_DCtx *dctx;
// call_on_stack(fn, arg, top): fn(arg) with %rsp at top, then back.
// %rbp keeps the old stack pointer across the call; it is callee-saved.
long call_on_stack(long (*fn)(void *), void *arg, void *top);
__asm__(
".text\n"
".globl call_on_stack\n"
"call_on_stack:\n"
" pushq %rbp\n"
" movq %rsp, %rbp\n"
" movq %rdx, %rsp\n"
" movq %rdi, %rax\n"
" movq %rsi, %rdi\n"
" callq *%rax\n"
" movq %rbp, %rsp\n"
" popq %rbp\n"
" retq\n");
struct job {
void *dst;
unsigned long cap;
const void *src;
unsigned long n;
};
static long
run(void *arg)
{
struct job *j = arg;
size_t r = ZSTD_decompressDCtx(dctx, j->dst, j->cap, j->src, j->n);
return ZSTD_isError(r) ? -1 : (long)r;
}
int
scff_zstd_init(void)
{
if (dctx)
return 0;
if (ZSTD_estimateDCtxSize() > sizeof(scff_work))
return -1;
dctx = ZSTD_initStaticDCtx(scff_work, sizeof(scff_work));
return dctx ? 0 : -1;
}
long
scff_zstd_decompress(void *dst, unsigned long cap, const void *src, unsigned long n)
{
struct job j = { dst, cap, src, n };
if (!dctx)
return -1;
return call_on_stack(run, &j, scff_stack + sizeof(scff_stack));
}

13
kernel/scff/scff_zstd.h Normal file
View file

@ -0,0 +1,13 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// The SCFF decoder's interface, in types both the kernel's headers and zstd's
// agree on. See kernel/scff/scff.c.
#ifndef SCHOS_SCFF_ZSTD_H
#define SCHOS_SCFF_ZSTD_H
// 0, or -1 if the workspace is too small for this zstd.
int scff_zstd_init(void);
// Decompress one zstd frame. The length written, or -1 for a bad frame.
// One caller at a time.
long scff_zstd_decompress(void *dst, unsigned long cap, const void *src, unsigned long n);
#endif

View file

@ -16,3 +16,13 @@ ncurses-terminfo-base
# The PCI and USB ID databases: lspci, fastfetch and Xorg read them.
hwdata-pci
hwdata-usb
# X on the framebuffer, with startx. /root/.xserverrc and .xinitrc (from
# mkaroot.sh) start it with -dumbSched and run Window Maker in it.
xorg-server
xf86-video-fbdev
xf86-input-evdev
xinit
# The desktop: Window Maker, a terminal for it, and a font for both.
windowmaker
xterm
font-dejavu

View file

@ -1,12 +1,91 @@
# Generated by tools/posix/alpine-lock.sh from world. Do not edit;
# change world and run `make alpine-lock`.
# file sha256 repository
brotli-libs-1.2.1-r0.apk 62a7e37e43bbfc1f8525a00ad2125fd5897135dc93b849733884cb78ffd8e26c main
busybox-1.37.0-r31.apk 0e626fa97cf937fda7bba4ed4dde8e2f2dbb5083ac34e02180cec0c4eba9351b main
busybox-binsh-1.37.0-r31.apk 93f803ae5db6b729c0f66f4afba6e907ccc17d7d5f8b8c3d25bdb4636de2c48a main
encodings-1.1.0-r0.apk f7c0eb762638d0249b4fbbe07a8e4ed6aab8e86b74c239c40e47a36219777d52 main
eudev-libs-3.2.14-r6.apk 45fdbcf37c5c0e82067e98a818d3c1fd16d0261c62128dcb6b935b86b4ec6c86 main
font-alias-1.0.6-r0.apk 754e5c1d19c2c69d5a2c3c02c7340cad7df3c26e63789516372ea164b564b927 main
fontconfig-2.17.1-r1.apk 7634cf2fb1066ec97c8f41ec90ec9f036540e056276b7bf9a10dc4ec9e44d238 main
font-cursor-misc-1.0.4-r1.apk a1a0c0659a9a48e24b4cc4b35583f4e57da9d2a0ae2cc8adb9be1143b66911b4 main
font-dejavu-2.37-r6.apk 6e7609e7a1aaed947afe4ddcc02bbdeec571dac5ff4ea1c04af9b8480b7b9dcf main
font-misc-misc-1.1.3-r1.apk 2c48f2a8529ac63243a0bdd5e7ab2329e0a8b8f996ffe95faed8b79fb05b4a52 main
freetype-2.14.3-r0.apk 09d76fde675a9ca3cfeb83d72fb09cb2e9ccf79b2937daa05a1af9a37a6b7815 main
gmp-6.3.0-r4.apk 85323ef5bce8ff234736dc2d6592ea37aebdc6348a531e27430bdfe38364d256 main
hwdata-pci-0.408-r0.apk fe61404e6d2bd1e04c021d53641c0ebce97911084d77994e43a288a1278686c1 main
hwdata-usb-0.408-r0.apk 272d365cb05dd085dd8022154cf71e3f3b0aaddd41c1269eeaf939c9d38675c8 main
libbsd-0.12.2-r0.apk 407597c04b41dbfe8a05b35d45a3453e32fae7df84aed3a06ff48dd4df1338b6 main
libbz2-1.0.8-r6.apk 2737e2b23ed08d4911ee62cd0163b5f4bf0379cacf4dc35b23a00209584c98fd main
libcap2-2.78-r0.apk 42886a7b4bcde26347c712925c8dc7db202b6029ef9b8c7ee81577ab7bdd8fad main
libdrm-2.4.134-r0.apk 584fd84ad5030040e613fa1816097cd1706ddd542100070852b0eab6e09e12ca main
libelf-0.195-r0.apk 78c5a1993342ab1bdcdce30639fd0937e8000f6d0d85cea171f573d31decd036 main
libepoxy-1.5.10-r1.apk 0be13b4b524fe64315e0663ab615a244d75d7da73130ac1c630cd6096841444e main
libevdev-1.13.6-r0.apk b38a27a4ac55701fd2eb0032977af2664b6fc23b2f9c2cc6e893ad4452fd82d5 community
libexpat-2.9.0-r0.apk 32b127f31e0a919110a18a7f0bb3780d8fb66ba4d8d61b2d301d633f0e319d0e main
libffi-3.5.2-r1.apk 6da1a19313a40c56cac756fb11e2c4d5267831e1c22fc8323f24c943c94da6e0 main
libfontenc-1.1.9-r0.apk ea7f6d097653572991da749ac0764c2568d8902a01eea4236e56b10b90a7b0e3 main
libgcc-15.2.0-r5.apk 393dcd32629f06d7d85409c272d142d0c082772d10b87ef55ee82f47de3be637 main
libice-1.1.2-r0.apk 08daa589d57149b52b00c666b4009212aa3af95021ebdb2a359555c9bcaa76e1 main
libjpeg-turbo-3.1.3-r0.apk 6299627dd0a9156a7463511e4915a6294dfb592d8b4bdf85e55265adf1b8ea04 main
libmd-1.2.0-r0.apk f6d68d072fcefc8edd4ffe3afc341ca391f8a9a209130628fb065d98b18a2792 main
libncursesw-6.6_p20260516-r0.apk cf8caa8a88bc4ce9d9e395567fefaf9ef7fbd55c50ae6155b35c1b58f3755023 main
libpciaccess-0.19-r0.apk 03fdccd1e2a0825e7f7fabbac67212e05d271d31d2eff44cc92f707e69908ad4 main
libpng-1.6.59-r0.apk d16aeb0597f2bc43e0583ac9ff779388ec160c3e22568b640122e94ba55be6d7 main
libsharpyuv-1.6.0-r0.apk 34621ff53f43030d24d1224fe9cb53b1590ab614cbce6080005cde703e6c874a main
libsm-1.2.6-r0.apk dd7eff58a46fd60e4afee9c7c1c3508920b0e1d6560129e59aee82329f68afc8 main
libstdc++-15.2.0-r5.apk 14c987b556f5385a5db18376e788c75f37d85321b8dc1920d926ea7daac1d6f6 main
libuuid-2.42.3-r1.apk 8306e5bb577696c9069fe1dfd9e1dcc39d2d481c6a1b0e707fd03c3e21aa6aa2 main
libwebp-1.6.0-r0.apk dc50c0280348330bb113079331c4a35d85e28351c7baa9268ee346aad8398288 main
libx11-1.8.14-r0.apk 016efc189c6d566061ffe206ed0780e32686c1a8c832a3917d35d2c66d61a40f main
libxau-1.0.12-r0.apk 3506cdba7ed916a6f814a010fbb127dbfdae4147107f64a15f5fa2651c5d8510 main
libxaw-1.0.16-r1.apk 8ac469cddeac40ff2e91c84cb02ba526d7c62e30a79a25c2e3b6e5269236801d main
libxcb-1.17.0-r2.apk 7935d5ad9c7137b7b3f1292d62f55f7e0cc5c919155218c3060ea46503b3f211 main
libxcvt-0.1.3-r0.apk 809b454981887c1d1d835c909c46a80650ca72a73a92268a2cf8a666ad82b6b7 community
libxdmcp-1.1.5-r1.apk 2a93f44f3f85c1eda9ed53691c35d0742b5fcd8e0057cd7380934025936cbf6f main
libxext-1.3.7-r0.apk 8afe1ee6ab8cb656e12f20dc5b7fb2d4103faee4a38298b94f26ab52e597cbc8 main
libxfont2-2.0.8-r0.apk f8bf87b9ce835cb307eb8a3d270e226c4437757478c52ba3ad384cecc2ac6fef community
libxft-2.3.9-r0.apk eba6fd85599b4fedf3acdd3ad3e2eec84ea97a5ba8f1ac570fa1ee37a7d81420 main
libxinerama-1.1.6-r0.apk 9f780efb3cf675b5253b73b84f278b40757fa7869c6fe70a9790a73f4f7e810f main
libxkbfile-1.2.0-r0.apk e575c6c314ad87e74887526aca3b1429f5c1d6290e66f2945d4053f387beea57 main
libxml2-2.13.9-r2.apk 4b2f986159c659f014b942fe8a5d70c67af475153ca826c3dd53389eea46a300 main
libxmu-1.3.1-r0.apk 5d0b06d1a4b38ac0e070e0dad5e61583e5e147e378ba50418c83e1be4b19986c main
libxpm-3.5.20-r0.apk 1ed312a00abac5a5840c1f3721921ea8183e664566c19763ed00aa9f35bb2178 main
libxrender-0.9.13-r0.apk 724aa6c6353e70869e0eb5aa5461e8fc0753b99db765de47a3ae238573dbdcf8 main
libxshmfence-1.3.3-r0.apk 53790552de7eea17a47aa4c082b06b735c545a0947c41128f97c7be95a7d3d92 main
libxt-1.3.1-r0.apk 08b167b05ac7dfb892421876cc2467214208b83b2960a9f49e84f9dbcf36e812 main
libxxf86vm-1.1.7-r0.apk 887a3c5438e1db624d07ac23d16ec08d21fb16de438d75ad2dd5637c958b4108 main
llvm22-libs-22.1.3-r1.apk dec15bdd29b6ec7fc22aee43bac153a22e6b132c599e5e5d9ccd0d159eae764f main
mcookie-2.42.3-r1.apk 64f86340991e2f283ce95b1bae21157da6849118cd9f1a2931de5c73d0f9fc3d main
mesa-26.1.6-r0.apk 8fb0e3d386f09df4f3e3643108b38197558465b2558e6d66d6a9c055adc8d1a9 main
mesa-egl-26.1.6-r0.apk ccdd25c8f3a069b018a39a5641e4f13ab7ceb511d5991e9f8fb18bd551172ffa main
mesa-gbm-26.1.6-r0.apk db6b00543e7e686f377a5c250036067742f914112839255e2e5edbbc241bfb0d main
mesa-gl-26.1.6-r0.apk 4d2d502a3938bd5dc4affec13545271571bb8bec40e7078ef01f34222bfb5d5d main
mesa-gles-26.1.6-r0.apk 2544a75e74778ab9ddcecfc8d9dc35375cb3a4ac3661e94a1156e2c528d7b59b main
mkfontscale-1.2.4-r0.apk 2385a7aad3971fcad9d862cb90937e643e633b14bf4b91f8b8ce3040a1468d20 main
mtdev-1.1.7-r0.apk 004f42bc67bc0eb573d64d802c5f0325f49013cf4aa7576d4b571399df46f3f7 community
musl-1.2.6-r3.apk e142255d0fa397f14318996bc10378093dcc069dc3dd1b6818c8f28a789c21da main
ncurses-terminfo-base-6.6_p20260516-r0.apk ce83be6d0bd10584a53c3a5868cc1595ce5f0f8e0b621f8f7f70b6144e521fc9 main
nettle-3.10.2-r0.apk 693c1a99b781cd17149916df30113ccd6bdddac130298c17b1286fd2ae438386 main
pixman-0.46.4-r0.apk 378550cbe348dda9b0733014375dba87a89f0077c5f3dbeba7dce8dbcdda707e main
pkgconf-2.5.1-r0.apk 20cc47b76b82e9424a35ecc4fdd7bd904ef49ada1684214910fae31bc0cab737 main
spirv-tools-1.4.341.0-r0.apk 6034c22c4f61fb76e517c1d5b11ef5c26c7154d2baeacec8b0a9dcca94ff9a93 main
tiff-4.7.2-r0.apk 757ce87ebe4923a6be9869d026e250958ca09696529c464781e8a91d2b81670b main
util-macros-1.20.2-r0.apk 7c129d085b646b13ccef3738e93038221aec840c1a6764279c5adc9950ba1bdc main
wayland-libs-client-1.25.0-r0.apk a728f7103217cafdc83ff6ad0d679c197dbbe280bd5788c8336259f7a39cb303 main
windowmaker-0.96.0-r0.apk 15b500d9afccf94a385e042b01ed9b1d709b1627d97597cea59bf63d0bab4f6b community
xauth-1.1.5-r0.apk 94e80e9687a1829a6fe95fb28d4b08a5531dab182cde8a58ff1b370b2cd293fb community
xf86-input-evdev-2.11.0-r0.apk 1f3fcd3e837b43ad5e32005e03095b547ddbfc25973f0511e9ff74bf342d2fe0 community
xf86-video-fbdev-0.5.0-r6.apk 6a8a1b16f82d8810896f00c00b356c8b8d923e9aa0ce64737ae8274ca82f507a community
xinit-1.4.4-r0.apk efb5dc1e7929824f5add7d5d51d0ecb34ecfd0a773dfca4f061adc8a353b234f community
xkbcomp-1.5.0-r0.apk 8bfbd3c8ce7180e8cf599d5a0b791993b61fa98f3a19c47aaf5fcbe30c12d0ba main
xkeyboard-config-2.47-r0.apk 8c8f9af2b78f8e955480aa88103e495d2637edac659a417075e877c88a41fed0 main
xmodmap-1.0.11-r1.apk 42489c8865d401e894b8da210f5c8e27f58c1d72c83250a505a65bdb8c45bec0 community
xorg-server-21.1.25-r0.apk 8130c341a5f047069cb35ac77a44d8bab6a2e3cb5a7d0745e3c5096b1909ecfb community
xorg-server-common-21.1.25-r0.apk bf5994707b4030f249ad44a123ad2686501c82d2002d73d0a641d9001caa37c3 community
xrdb-1.2.2-r0.apk 29649088693b5974dec3f928a5303e354f7b91e381b778006fe3011d003b1369 community
xterm-410-r0.apk eff11da325ffb1dcfa4c9902ef27d34cc27b76457e1fc8dad90f10944ecb33b8 community
xz-libs-5.8.4-r0.apk ab362dd515ead04c31a6c302163248da7aa4813e283c971e6dd3f789b453a1d9 main
zlib-1.3.2-r1.apk 63aeea03c15a2f9018f81805cfc8aa926bdf5cd68921f22149c2fbb5d0ee9f47 main
zsh-5.9-r7.apk 22ec3c4de69e38437e351ba5a881f5d88e9b518a248c8aa06d36b148cf16ce32 main
zstd-libs-1.5.7-r2.apk 23c6065b0049b2406441564bcf0032515a43f78e80d76fcb85535a3803ef5d4e main

43
ports/zstd/Makefile Normal file
View file

@ -0,0 +1,43 @@
# SPDX-License-Identifier: GPL-2.0-or-later
# zstd -- the decompressor behind SCFF, the compressed live root. Dual
# BSD-3-Clause / GPL-2.0; see LICENSE and COPYING in the tarball and
# docs/licensing.md.
#
# Kernel code, like uACPI: nothing is installed into the sysroot. This
# directory fetches and unpacks a pinned release, and the top-level Makefile
# compiles the decoder half of lib/ straight into the kernel against
# kernel/scff/compat, which stands in for the libc headers it includes. The
# host side, tools/mkscff, links the system's libzstd.
#
# Chosen for what SCFF asks of a codec: compress as hard as the build likes,
# once, and decompress at memory speed on every first read of a block.
include ../mk/port.mk
ZSTD_VERSION := 1.5.7
ZSTD_URL := https://github.com/facebook/zstd/releases/download/v$(ZSTD_VERSION)/zstd-$(ZSTD_VERSION).tar.gz
ZSTD_SHA256 := eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3
TARBALL := $(DISTDIR)/zstd-$(ZSTD_VERSION).tar.gz
$(TARBALL): | $(DISTDIR)
curl -fsSL -o $@ $(ZSTD_URL)
sh $(SHA256CHECK) "$(ZSTD_SHA256)" "$@"
download: $(TARBALL)
$(SRCDIR): $(TARBALL)
mkdir -p $(PORTDIR)
cd $(PORTDIR) && tar xzf $(abspath $<)
mv $(PORTDIR)/zstd-$(ZSTD_VERSION) $@
# The kernel build depends on this directory existing, nothing more.
all: | $(SRCDIR)
install:
clean:
rm -rf $(PORTDIR)
distclean: clean
rm -f $(TARBALL)

190
tools/mkscff.c Normal file
View file

@ -0,0 +1,190 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// mkscff: write a block image as SCFF, the SchOS Compressed File Format.
//
// mkscff IN.img OUT.scff [--chunk KB] [--level N] [--threads N]
//
// The layout is include/fs/scff.h. Chunks are compressed on every core, since
// zstd's high levels are slow and the image is hundreds of megabytes, and each
// one is decompressed again and compared before it is written: an image the
// kernel cannot read is found here rather than at boot.
//
// Built against the zstd release the kernel's decoder comes from (ports/zstd),
// not the host's libzstd, so both sides are one version and nothing needs to
// be installed for it.
#include <errno.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "fs/scff.h"
#include "zstd.h"
static unsigned char *img;
static uint64_t img_bytes;
static uint32_t chunk_bytes = 128 * 1024;
static int level = 19;
static uint32_t nchunks;
static unsigned char **out; // chunk i as stored, or 0 for an all-zero one
static size_t *outlen;
static uint32_t next_chunk;
static pthread_mutex_t next_lock = PTHREAD_MUTEX_INITIALIZER;
static void
die(const char *what)
{
fprintf(stderr, "mkscff: %s\n", what);
exit(1);
}
static int
all_zero(const unsigned char *p, size_t n)
{
for (size_t i = 0; i < n; i++)
if (p[i])
return 0;
return 1;
}
static void *
worker(void *arg)
{
(void)arg;
ZSTD_CCtx *cc = ZSTD_createCCtx();
ZSTD_DCtx *dc = ZSTD_createDCtx();
size_t cap = ZSTD_compressBound(chunk_bytes);
unsigned char *buf = malloc(cap), *back = malloc(chunk_bytes);
if (!cc || !dc || !buf || !back)
die("out of memory");
ZSTD_CCtx_setParameter(cc, ZSTD_c_compressionLevel, level);
ZSTD_CCtx_setParameter(cc, ZSTD_c_checksumFlag, 1);
for (;;) {
pthread_mutex_lock(&next_lock);
uint32_t i = next_chunk++;
pthread_mutex_unlock(&next_lock);
if (i >= nchunks)
break;
const unsigned char *src = img + (uint64_t)i * chunk_bytes;
size_t n = img_bytes - (uint64_t)i * chunk_bytes;
if (n > chunk_bytes)
n = chunk_bytes;
if (all_zero(src, n)) {
outlen[i] = 0;
continue;
}
size_t c = ZSTD_compress2(cc, buf, cap, src, n);
if (ZSTD_isError(c))
die(ZSTD_getErrorName(c));
if (c >= n) { // did not compress: stored as it is
out[i] = malloc(n);
memcpy(out[i], src, n);
outlen[i] = n;
continue;
}
size_t d = ZSTD_decompressDCtx(dc, back, chunk_bytes, buf, c);
if (ZSTD_isError(d) || d != n || memcmp(back, src, n) != 0)
die("a chunk did not decompress to itself");
out[i] = malloc(c);
memcpy(out[i], buf, c);
outlen[i] = c;
}
ZSTD_freeCCtx(cc);
ZSTD_freeDCtx(dc);
free(buf);
free(back);
return 0;
}
int
main(int argc, char **argv)
{
const char *in = 0, *outpath = 0;
long threads = sysconf(_SC_NPROCESSORS_ONLN);
for (int i = 1; i < argc; i++) {
if (!strcmp(argv[i], "--chunk") && i + 1 < argc)
chunk_bytes = (uint32_t)atoi(argv[++i]) * 1024;
else if (!strcmp(argv[i], "--level") && i + 1 < argc)
level = atoi(argv[++i]);
else if (!strcmp(argv[i], "--threads") && i + 1 < argc)
threads = atol(argv[++i]);
else if (!in)
in = argv[i];
else if (!outpath)
outpath = argv[i];
else
die("usage: mkscff IN.img OUT.scff [--chunk KB] [--level N] [--threads N]");
}
if (!in || !outpath)
die("usage: mkscff IN.img OUT.scff [--chunk KB] [--level N] [--threads N]");
if (chunk_bytes < 4096 || chunk_bytes > SCFF_CHUNK_MAX ||
(chunk_bytes & (chunk_bytes - 1)))
die("--chunk must be a power of two from 4 to 1024 KB");
if (threads < 1)
threads = 1;
FILE *f = fopen(in, "rb");
if (!f)
die(strerror(errno));
fseek(f, 0, SEEK_END);
img_bytes = (uint64_t)ftell(f);
fseek(f, 0, SEEK_SET);
if (img_bytes == 0 || img_bytes % 4096)
die("the image is not a whole number of 4 KB blocks");
if (!(img = malloc(img_bytes)) || fread(img, 1, img_bytes, f) != img_bytes)
die("cannot read the image");
fclose(f);
nchunks = (uint32_t)((img_bytes + chunk_bytes - 1) / chunk_bytes);
out = calloc(nchunks, sizeof(*out));
outlen = calloc(nchunks, sizeof(*outlen));
if (!out || !outlen)
die("out of memory");
pthread_t *t = calloc((size_t)threads, sizeof(*t));
for (long i = 0; i < threads; i++)
pthread_create(&t[i], 0, worker, 0);
for (long i = 0; i < threads; i++)
pthread_join(t[i], 0);
struct scff_header h;
memset(&h, 0, sizeof(h));
memcpy(h.magic, SCFF_MAGIC, 4);
h.version = SCFF_VERSION;
h.codec = SCFF_CODEC_ZSTD;
h.chunk_bytes = chunk_bytes;
h.nchunks = nchunks;
h.image_bytes = img_bytes;
h.index_off = sizeof(h);
uint64_t *index = malloc(((size_t)nchunks + 1) * sizeof(uint64_t));
uint64_t pos = sizeof(h) + ((uint64_t)nchunks + 1) * sizeof(uint64_t);
for (uint32_t i = 0; i < nchunks; i++) {
index[i] = pos;
pos += outlen[i];
}
index[nchunks] = pos;
h.stored_bytes = pos;
if (!(f = fopen(outpath, "wb")))
die(strerror(errno));
fwrite(&h, sizeof(h), 1, f);
fwrite(index, sizeof(uint64_t), (size_t)nchunks + 1, f);
for (uint32_t i = 0; i < nchunks; i++)
if (outlen[i])
fwrite(out[i], 1, outlen[i], f);
if (fclose(f) != 0)
die("cannot write the output");
printf("mkscff: %llu KB -> %llu KB (%.1f%%), %u chunks of %u KB, zstd -%d\n",
(unsigned long long)(img_bytes >> 10), (unsigned long long)(pos >> 10),
100.0 * (double)pos / (double)img_bytes, nchunks, chunk_bytes >> 10, level);
return 0;
}

View file

@ -109,6 +109,13 @@ cat > "$P/root/.xserverrc" <<'XEOF'
exec /usr/bin/X -nolisten tcp -dumbSched "$@"
XEOF
# What startx runs inside it: Window Maker, which world installs. Without this
# xinit falls back to twm and an xterm, neither of which is in the image.
cat > "$P/root/.xinitrc" <<'XEOF'
#!/bin/sh
exec wmaker
XEOF
# Mount points and runtime directories are alpine-baselayout's; an empty
# directory here only duplicates it.
for d in dev proc sys run tmp srv; do

View file

@ -3,11 +3,16 @@
#
# tools/posix/mkiso.sh [output.iso]
#
# The disc carries one El Torito boot entry and a real directory tree:
# The disc carries one El Torito boot entry and the loader and kernel as files:
#
# /boot/efiboot.img a FAT image holding the loader and the kernel
# /EFI/BOOT/BOOTX64.EFI the same loader, visible as a file on the disc
# /bin /etc /usr ... the staged sysroot, as an ordinary hierarchy
# /schos/kernel the same kernel, for chainloaders (below)
#
# The root filesystem is inside the kernel, as its ramdisk. The disc used to
# carry it a third time as an ISO 9660 tree, which nothing reads -- SchOS has
# no ISO 9660 driver -- and once Xorg was in the root that copy alone was
# 360 MB.
#
# --- Why only one entry -----------------------------------------------------
#
@ -69,7 +74,6 @@ cd "$repo"
BUILD=${BUILD:-build}
STAGING="$BUILD/iso"
SYSROOT=${SYSROOT:-sysroot}
KERNEL="$BUILD/kernel/kernel"
EFIAPP="$BUILD/uefi/BOOTX64.EFI"
@ -113,9 +117,6 @@ cp "$KERNEL" "$STAGING/schos/kernel"
# this FAT image -- not the ISO 9660 tree around it. Without the kernel in here
# the loader comes up and halts with EFI_NOT_FOUND, which is what it did.
#
# The ISO 9660 tree still carries the sysroot for a running system to read; the
# loader's own two files live in here because that is the volume it is handed.
#
# Sized from what actually goes in plus slack, rather than fixed: a hardcoded
# size is a thing that breaks quietly the first time the kernel grows.
EFI_IMG="$STAGING/boot/efiboot.img"
@ -145,15 +146,6 @@ if true; then
mcopy -i "$EFI_IMG" "$KERNEL" ::/schos/kernel
fi
# The sysroot, as an ordinary hierarchy. Copied rather than linked so that
# whatever ends up on the disc is a snapshot, not a view of a tree that may be
# rebuilt underneath it.
if [ -d "$SYSROOT" ]; then
for d in "$SYSROOT"/*; do
[ -e "$d" ] || continue
cp -a "$d" "$STAGING/"
done
fi
# xorriso if it is here, mkisofs otherwise. Both accept the El Torito flags
# below unchanged -- xorriso's -as mkisofs mode exists precisely to be
@ -173,6 +165,11 @@ if command -v xorriso >/dev/null 2>&1; then
# of type 0xEF is what every UEFI firmware looks for on a removable
# disk, MBR rather than GPT because the oldest ones read that best.
HYBRID="-append_partition 2 0xef $EFI_IMG"
# El Torito keeps its own copy of the image in the tree, the layout real
# machines have booted. Pointing it into the partition instead
# (--interval:appended_partition_2:all::, as Arch does) saves a copy, and
# an ASUS laptop stopped listing the stick the day that was tried. With
# the root compressed (SCFF) the copy is only the kernel's size.
elif command -v mkisofs >/dev/null 2>&1; then
MKISOFS="mkisofs"
HYBRID=""