SCFF: the live root ships compressed, with Xorg and Window Maker
SCFF, the SchOS Compressed File Format (include/fs/scff.h), is a block image cut into 128 KB chunks, each its own zstd frame, behind an index. tools/mkscff writes it from fs-ram.img on every core (-19, 11 s) and checks each chunk decompresses to itself. The kernel carries the .scff as its ramdisk and unpacks a chunk the first time a block in it is read; zero blocks share one page. The kernel and the ISO are the compressed size, and the root costs memory only for what is touched. See docs/architecture/scff.md. zstd 1.5.7 comes from ports/zstd like uACPI: the decoder is compiled into the kernel at -O2, with a static context and a 64 KB stack of its own, and mkscff builds the whole library from the same release. world now has xorg-server, the fbdev and evdev drivers, xinit, Window Maker, xterm and DejaVu, and /root/.xinitrc runs wmaker. That root is 358 MB, 102 MB as SCFF. The ISO no longer carries the root a third time as an ISO 9660 tree, which nothing read. schos.iso: 1389 MB -> 318 MB. El Torito keeps its own copy of the EFI image, the layout real machines have booted. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4
This commit is contained in:
Nexoniarz
parent
1ba93dde2c
commit
df1394eaa3
18 changed files with 832 additions and 38 deletions
190
tools/mkscff.c
Normal file
190
tools/mkscff.c
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
// mkscff: write a block image as SCFF, the SchOS Compressed File Format.
|
||||
//
|
||||
// mkscff IN.img OUT.scff [--chunk KB] [--level N] [--threads N]
|
||||
//
|
||||
// The layout is include/fs/scff.h. Chunks are compressed on every core, since
|
||||
// zstd's high levels are slow and the image is hundreds of megabytes, and each
|
||||
// one is decompressed again and compared before it is written: an image the
|
||||
// kernel cannot read is found here rather than at boot.
|
||||
//
|
||||
// Built against the zstd release the kernel's decoder comes from (ports/zstd),
|
||||
// not the host's libzstd, so both sides are one version and nothing needs to
|
||||
// be installed for it.
|
||||
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "fs/scff.h"
|
||||
#include "zstd.h"
|
||||
|
||||
static unsigned char *img;
|
||||
static uint64_t img_bytes;
|
||||
static uint32_t chunk_bytes = 128 * 1024;
|
||||
static int level = 19;
|
||||
static uint32_t nchunks;
|
||||
|
||||
static unsigned char **out; // chunk i as stored, or 0 for an all-zero one
|
||||
static size_t *outlen;
|
||||
static uint32_t next_chunk;
|
||||
static pthread_mutex_t next_lock = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
static void
|
||||
die(const char *what)
|
||||
{
|
||||
fprintf(stderr, "mkscff: %s\n", what);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static int
|
||||
all_zero(const unsigned char *p, size_t n)
|
||||
{
|
||||
for (size_t i = 0; i < n; i++)
|
||||
if (p[i])
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void *
|
||||
worker(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
ZSTD_CCtx *cc = ZSTD_createCCtx();
|
||||
ZSTD_DCtx *dc = ZSTD_createDCtx();
|
||||
size_t cap = ZSTD_compressBound(chunk_bytes);
|
||||
unsigned char *buf = malloc(cap), *back = malloc(chunk_bytes);
|
||||
|
||||
if (!cc || !dc || !buf || !back)
|
||||
die("out of memory");
|
||||
ZSTD_CCtx_setParameter(cc, ZSTD_c_compressionLevel, level);
|
||||
ZSTD_CCtx_setParameter(cc, ZSTD_c_checksumFlag, 1);
|
||||
|
||||
for (;;) {
|
||||
pthread_mutex_lock(&next_lock);
|
||||
uint32_t i = next_chunk++;
|
||||
pthread_mutex_unlock(&next_lock);
|
||||
if (i >= nchunks)
|
||||
break;
|
||||
|
||||
const unsigned char *src = img + (uint64_t)i * chunk_bytes;
|
||||
size_t n = img_bytes - (uint64_t)i * chunk_bytes;
|
||||
if (n > chunk_bytes)
|
||||
n = chunk_bytes;
|
||||
|
||||
if (all_zero(src, n)) {
|
||||
outlen[i] = 0;
|
||||
continue;
|
||||
}
|
||||
size_t c = ZSTD_compress2(cc, buf, cap, src, n);
|
||||
if (ZSTD_isError(c))
|
||||
die(ZSTD_getErrorName(c));
|
||||
if (c >= n) { // did not compress: stored as it is
|
||||
out[i] = malloc(n);
|
||||
memcpy(out[i], src, n);
|
||||
outlen[i] = n;
|
||||
continue;
|
||||
}
|
||||
size_t d = ZSTD_decompressDCtx(dc, back, chunk_bytes, buf, c);
|
||||
if (ZSTD_isError(d) || d != n || memcmp(back, src, n) != 0)
|
||||
die("a chunk did not decompress to itself");
|
||||
out[i] = malloc(c);
|
||||
memcpy(out[i], buf, c);
|
||||
outlen[i] = c;
|
||||
}
|
||||
ZSTD_freeCCtx(cc);
|
||||
ZSTD_freeDCtx(dc);
|
||||
free(buf);
|
||||
free(back);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
main(int argc, char **argv)
|
||||
{
|
||||
const char *in = 0, *outpath = 0;
|
||||
long threads = sysconf(_SC_NPROCESSORS_ONLN);
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
if (!strcmp(argv[i], "--chunk") && i + 1 < argc)
|
||||
chunk_bytes = (uint32_t)atoi(argv[++i]) * 1024;
|
||||
else if (!strcmp(argv[i], "--level") && i + 1 < argc)
|
||||
level = atoi(argv[++i]);
|
||||
else if (!strcmp(argv[i], "--threads") && i + 1 < argc)
|
||||
threads = atol(argv[++i]);
|
||||
else if (!in)
|
||||
in = argv[i];
|
||||
else if (!outpath)
|
||||
outpath = argv[i];
|
||||
else
|
||||
die("usage: mkscff IN.img OUT.scff [--chunk KB] [--level N] [--threads N]");
|
||||
}
|
||||
if (!in || !outpath)
|
||||
die("usage: mkscff IN.img OUT.scff [--chunk KB] [--level N] [--threads N]");
|
||||
if (chunk_bytes < 4096 || chunk_bytes > SCFF_CHUNK_MAX ||
|
||||
(chunk_bytes & (chunk_bytes - 1)))
|
||||
die("--chunk must be a power of two from 4 to 1024 KB");
|
||||
if (threads < 1)
|
||||
threads = 1;
|
||||
|
||||
FILE *f = fopen(in, "rb");
|
||||
if (!f)
|
||||
die(strerror(errno));
|
||||
fseek(f, 0, SEEK_END);
|
||||
img_bytes = (uint64_t)ftell(f);
|
||||
fseek(f, 0, SEEK_SET);
|
||||
if (img_bytes == 0 || img_bytes % 4096)
|
||||
die("the image is not a whole number of 4 KB blocks");
|
||||
if (!(img = malloc(img_bytes)) || fread(img, 1, img_bytes, f) != img_bytes)
|
||||
die("cannot read the image");
|
||||
fclose(f);
|
||||
|
||||
nchunks = (uint32_t)((img_bytes + chunk_bytes - 1) / chunk_bytes);
|
||||
out = calloc(nchunks, sizeof(*out));
|
||||
outlen = calloc(nchunks, sizeof(*outlen));
|
||||
if (!out || !outlen)
|
||||
die("out of memory");
|
||||
|
||||
pthread_t *t = calloc((size_t)threads, sizeof(*t));
|
||||
for (long i = 0; i < threads; i++)
|
||||
pthread_create(&t[i], 0, worker, 0);
|
||||
for (long i = 0; i < threads; i++)
|
||||
pthread_join(t[i], 0);
|
||||
|
||||
struct scff_header h;
|
||||
memset(&h, 0, sizeof(h));
|
||||
memcpy(h.magic, SCFF_MAGIC, 4);
|
||||
h.version = SCFF_VERSION;
|
||||
h.codec = SCFF_CODEC_ZSTD;
|
||||
h.chunk_bytes = chunk_bytes;
|
||||
h.nchunks = nchunks;
|
||||
h.image_bytes = img_bytes;
|
||||
h.index_off = sizeof(h);
|
||||
|
||||
uint64_t *index = malloc(((size_t)nchunks + 1) * sizeof(uint64_t));
|
||||
uint64_t pos = sizeof(h) + ((uint64_t)nchunks + 1) * sizeof(uint64_t);
|
||||
for (uint32_t i = 0; i < nchunks; i++) {
|
||||
index[i] = pos;
|
||||
pos += outlen[i];
|
||||
}
|
||||
index[nchunks] = pos;
|
||||
h.stored_bytes = pos;
|
||||
|
||||
if (!(f = fopen(outpath, "wb")))
|
||||
die(strerror(errno));
|
||||
fwrite(&h, sizeof(h), 1, f);
|
||||
fwrite(index, sizeof(uint64_t), (size_t)nchunks + 1, f);
|
||||
for (uint32_t i = 0; i < nchunks; i++)
|
||||
if (outlen[i])
|
||||
fwrite(out[i], 1, outlen[i], f);
|
||||
if (fclose(f) != 0)
|
||||
die("cannot write the output");
|
||||
|
||||
printf("mkscff: %llu KB -> %llu KB (%.1f%%), %u chunks of %u KB, zstd -%d\n",
|
||||
(unsigned long long)(img_bytes >> 10), (unsigned long long)(pos >> 10),
|
||||
100.0 * (double)pos / (double)img_bytes, nchunks, chunk_bytes >> 10, level);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -109,6 +109,13 @@ cat > "$P/root/.xserverrc" <<'XEOF'
|
|||
exec /usr/bin/X -nolisten tcp -dumbSched "$@"
|
||||
XEOF
|
||||
|
||||
# What startx runs inside it: Window Maker, which world installs. Without this
|
||||
# xinit falls back to twm and an xterm, neither of which is in the image.
|
||||
cat > "$P/root/.xinitrc" <<'XEOF'
|
||||
#!/bin/sh
|
||||
exec wmaker
|
||||
XEOF
|
||||
|
||||
# Mount points and runtime directories are alpine-baselayout's; an empty
|
||||
# directory here only duplicates it.
|
||||
for d in dev proc sys run tmp srv; do
|
||||
|
|
|
|||
|
|
@ -3,11 +3,16 @@
|
|||
#
|
||||
# tools/posix/mkiso.sh [output.iso]
|
||||
#
|
||||
# The disc carries one El Torito boot entry and a real directory tree:
|
||||
# The disc carries one El Torito boot entry and the loader and kernel as files:
|
||||
#
|
||||
# /boot/efiboot.img a FAT image holding the loader and the kernel
|
||||
# /EFI/BOOT/BOOTX64.EFI the same loader, visible as a file on the disc
|
||||
# /bin /etc /usr ... the staged sysroot, as an ordinary hierarchy
|
||||
# /schos/kernel the same kernel, for chainloaders (below)
|
||||
#
|
||||
# The root filesystem is inside the kernel, as its ramdisk. The disc used to
|
||||
# carry it a third time as an ISO 9660 tree, which nothing reads -- SchOS has
|
||||
# no ISO 9660 driver -- and once Xorg was in the root that copy alone was
|
||||
# 360 MB.
|
||||
#
|
||||
# --- Why only one entry -----------------------------------------------------
|
||||
#
|
||||
|
|
@ -69,7 +74,6 @@ cd "$repo"
|
|||
|
||||
BUILD=${BUILD:-build}
|
||||
STAGING="$BUILD/iso"
|
||||
SYSROOT=${SYSROOT:-sysroot}
|
||||
|
||||
KERNEL="$BUILD/kernel/kernel"
|
||||
EFIAPP="$BUILD/uefi/BOOTX64.EFI"
|
||||
|
|
@ -113,9 +117,6 @@ cp "$KERNEL" "$STAGING/schos/kernel"
|
|||
# this FAT image -- not the ISO 9660 tree around it. Without the kernel in here
|
||||
# the loader comes up and halts with EFI_NOT_FOUND, which is what it did.
|
||||
#
|
||||
# The ISO 9660 tree still carries the sysroot for a running system to read; the
|
||||
# loader's own two files live in here because that is the volume it is handed.
|
||||
#
|
||||
# Sized from what actually goes in plus slack, rather than fixed: a hardcoded
|
||||
# size is a thing that breaks quietly the first time the kernel grows.
|
||||
EFI_IMG="$STAGING/boot/efiboot.img"
|
||||
|
|
@ -145,15 +146,6 @@ if true; then
|
|||
mcopy -i "$EFI_IMG" "$KERNEL" ::/schos/kernel
|
||||
fi
|
||||
|
||||
# The sysroot, as an ordinary hierarchy. Copied rather than linked so that
|
||||
# whatever ends up on the disc is a snapshot, not a view of a tree that may be
|
||||
# rebuilt underneath it.
|
||||
if [ -d "$SYSROOT" ]; then
|
||||
for d in "$SYSROOT"/*; do
|
||||
[ -e "$d" ] || continue
|
||||
cp -a "$d" "$STAGING/"
|
||||
done
|
||||
fi
|
||||
|
||||
# xorriso if it is here, mkisofs otherwise. Both accept the El Torito flags
|
||||
# below unchanged -- xorriso's -as mkisofs mode exists precisely to be
|
||||
|
|
@ -173,6 +165,11 @@ if command -v xorriso >/dev/null 2>&1; then
|
|||
# of type 0xEF is what every UEFI firmware looks for on a removable
|
||||
# disk, MBR rather than GPT because the oldest ones read that best.
|
||||
HYBRID="-append_partition 2 0xef $EFI_IMG"
|
||||
# El Torito keeps its own copy of the image in the tree, the layout real
|
||||
# machines have booted. Pointing it into the partition instead
|
||||
# (--interval:appended_partition_2:all::, as Arch does) saves a copy, and
|
||||
# an ASUS laptop stopped listing the stick the day that was tried. With
|
||||
# the root compressed (SCFF) the copy is only the kernel's size.
|
||||
elif command -v mkisofs >/dev/null 2>&1; then
|
||||
MKISOFS="mkisofs"
|
||||
HYBRID=""
|
||||
|
|
|
|||
Loading…
Reference in a new issue