nexoniarz/SchOS
nexoniarz/SchOS
Watch
1
0
Fork
You've already forked SchOS
 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Initial commit: Schunix as it stands before the SchOS rename

Browse source 
An x86-64 operating system built from xv6, now largely rewritten: SMP,
a copy-on-write root filesystem (SchFS), PCI/USB/virtio drivers, a TTF
rasteriser, and a Linux-numbered syscall ABI.

No history before this point -- the tree was never under version control.
This commit exists so the rename that follows has something to undo to.
This commit is contained in:
Nexoniarz 2026-08-09 13:47:01 +02:00
commit d0f7acf9eb
275 changed files with 51417 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

650
tools/mkschfs.c Normal file
View file

@ -0,0 +1,650 @@
// SPDX-License-Identifier: Apache-2.0
//
// mkschfs — build a SchFS image from a host directory tree.
//
// This is the tool that makes the on-disk format real: everything the kernel
// mounts in phase 2 is produced here. It deliberately shares the actual
// filesystem code (btree.c, vol.c, obj.c, objw.c) rather than reimplementing
// the layout, so a disagreement between writer and reader is impossible by
// construction — there is only one implementation, exercised on the host where
// a mistake costs a second instead of a boot.
//
// mkschfs out.img --sysroot DIR [--size BLOCKS] [--label NAME]
//
// Two behaviours worth knowing about:
//
// * Identical file contents are stored once and hard-linked. sysroot/bin
// holds a full copy of the toybox binary per applet, so this is the
// difference between a 190 MB image and a 6 MB one.
// * Output is reproducible. Directory entries are sorted, the UUID is
// derived rather than random, and timestamps come from the source files,
// so building the same tree twice gives byte-identical images and a diff
// means something actually changed.
#define _FILE_OFFSET_BITS 64
#define SCHFS_HOSTED 1
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <unistd.h>
#include <dirent.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/types.h>
/* major()/minor() for device nodes. glibc puts them here; mingw has no such
header and no device numbers at all, because Windows has no mknod. On that
host the macros are stubbed: a device node cannot exist in the source tree
being imaged, so the values are never anything but zero, and stubbing is
honest where pretending to read st_rdev would not be. */
#ifdef __MINGW32__
#define major(x) 0
#define minor(x) 0
/* Symlinks, likewise. Windows has no lstat/readlink and mingw exposes neither,
so on this host a symlink cannot be encountered while walking the tree --
lstat degrades to stat, nothing is ever S_ISLNK, and readlink can only be
reached through a branch that is now unreachable. Verified rather than
assumed: `find sysroot -type l` returns nothing.
If symlinks are ever staged (procfs wants /proc/<pid>/exe to be one), this
host stops being able to build a faithful image and the stub must become a
real Win32 reparse-point read. Failing loudly then is the point of the
abort() below. */
#define lstat(p, b) stat((p), (b))
#define S_ISLNK(m) 0
static long readlink(const char *p, char *b, unsigned long n)
{ (void)p; (void)b; (void)n; return -1; }
#else
#include <sys/sysmacros.h>
#endif
#include "../kernel/fs/schfs/btree.c"
#include "../kernel/fs/schfs/space.c"
#include "../kernel/fs/schfs/vol.c"
#include "../kernel/fs/schfs/obj.c"
#include "../kernel/fs/schfs/objw.c"
/* ------------------------------------------------------------------ */
/* File-backed block device */
/* ------------------------------------------------------------------ */
struct imgdev {
FILE *f;
struct schfs_vol *vol;
uint64_t nblocks;
long reads, writes;
};
static struct imgdev IMG;
static int
img_read(void *ctx, uint64_t b, void *buf)
{
struct imgdev *d = ctx;
if (b >= d->nblocks)
return -1;
if (fseeko(d->f, (off_t)b * SCHFS_BLKSIZE, SEEK_SET) != 0)
return -1;
if (fread(buf, 1, SCHFS_BLKSIZE, d->f) != SCHFS_BLKSIZE)
return -1;
d->reads++;
return 0;
}
static int
img_write(void *ctx, uint64_t b, const void *buf)
{
struct imgdev *d = ctx;
if (b >= d->nblocks)
return -1;
if (fseeko(d->f, (off_t)b * SCHFS_BLKSIZE, SEEK_SET) != 0)
return -1;
if (fwrite(buf, 1, SCHFS_BLKSIZE, d->f) != SCHFS_BLKSIZE)
return -1;
d->writes++;
return 0;
}
/* ------------------------------------------------------------------ */
/* Allocation */
/* */
/* Straight through to the volume's space manager. An earlier version */
/* kept a build-time free list of its own, because the bump allocator it */
/* replaced could only ever take back the block it handed out last and */
/* a build otherwise leaked one root-to-leaf path per insert. Two things */
/* made that unnecessary: the space manager reuses any freed block, and */
/* the tree now rewrites its own nodes in place within a transaction */
/* group, so a build barely frees anything in the first place. */
/* ------------------------------------------------------------------ */
static uint64_t
img_alloc(void *ctx)
{
return schfs_vol_alloc(((struct imgdev *)ctx)->vol);
}
static uint64_t
img_alloc_run(void *ctx, uint64_t n)
{
return schfs_vol_alloc_run(((struct imgdev *)ctx)->vol, n);
}
static void
img_free(void *ctx, uint64_t b)
{
schfs_vol_free(((struct imgdev *)ctx)->vol, b);
}
// Pages for the allocation bitmap. calloc rather than malloc so a bug that
// forgets to initialise one shows up as "everything is free" consistently
// rather than as whatever the heap happened to contain.
static void *
img_palloc(void)
{
return calloc(1, SCHFS_BLKSIZE);
}
/* ------------------------------------------------------------------ */
/* Content dedup */
/* */
/* sysroot/bin contains one independent ~800 KB copy of toybox per */
/* applet name. Storing each separately is the difference between an */
/* image that fits in RAM and one that does not, so identical content */
/* becomes one object with several names -- a plain hard link, which */
/* the format already supports through nlink. */
/* ------------------------------------------------------------------ */
struct dedup_ent {
char *path;
dev_t hdev; /* host device+inode: when these match, */
ino_t hino; /* it is literally the same file */
off_t size;
uint32_t crc;
uint64_t ino;
};
static struct dedup_ent *dedup_tab;
static int dedup_n, dedup_cap;
/* A file the host already told us is the same file.
*
* ports/toybox/Makefile hard-links its 236 applet names to one binary, so this
* settles all but the first of them without reading anything: the content path
* below reads the candidate to CRC it and then re-reads both files to confirm,
* which is half a gigabyte of I/O for an answer fstat already has. */
static uint64_t
dedup_lookup_ino(dev_t hdev, ino_t hino)
{
if (hino == 0)
return 0;
for (int i = 0; i < dedup_n; i++)
if (dedup_tab[i].hdev == hdev && dedup_tab[i].hino == hino)
return dedup_tab[i].ino;
return 0;
}
static int
files_identical(const char *a, const char *b)
{
FILE *fa = fopen(a, "rb"), *fb = fopen(b, "rb");
if (!fa || !fb) { if (fa) fclose(fa); if (fb) fclose(fb); return 0; }
int same = 1;
for (;;) {
char ba[8192], bb[8192];
size_t na = fread(ba, 1, sizeof(ba), fa);
size_t nb = fread(bb, 1, sizeof(bb), fb);
if (na != nb || memcmp(ba, bb, na) != 0) { same = 0; break; }
if (na == 0) break;
}
fclose(fa); fclose(fb);
return same;
}
// Size and CRC agreeing is not proof, so a candidate is confirmed byte by byte.
// Silently hard-linking two files that merely collide would corrupt the image
// in a way nothing downstream could detect.
static uint64_t
dedup_lookup(const char *path, off_t size, uint32_t crc)
{
for (int i = 0; i < dedup_n; i++)
if (dedup_tab[i].size == size && dedup_tab[i].crc == crc &&
files_identical(dedup_tab[i].path, path))
return dedup_tab[i].ino;
return 0;
}
static void
dedup_remember(const char *path, const struct stat *st,
off_t size, uint32_t crc, uint64_t ino)
{
dev_t dedup_hdev = st->st_nlink > 1 ? st->st_dev : 0;
ino_t dedup_hino = st->st_nlink > 1 ? st->st_ino : 0;
if (dedup_n == dedup_cap) {
dedup_cap = dedup_cap ? dedup_cap * 2 : 256;
dedup_tab = realloc(dedup_tab, (size_t)dedup_cap * sizeof(*dedup_tab));
if (!dedup_tab) { perror("realloc"); exit(1); }
}
dedup_tab[dedup_n].path = strdup(path);
dedup_tab[dedup_n].hdev = dedup_hdev;
dedup_tab[dedup_n].hino = dedup_hino;
dedup_tab[dedup_n].size = size;
dedup_tab[dedup_n].crc = crc;
dedup_tab[dedup_n].ino = ino;
dedup_n++;
}
/* ------------------------------------------------------------------ */
static struct schfs_vol VOL;
static char DATABLK[SCHFS_BLKSIZE];
static char CURBLK[SCHFS_BLKSIZE];
// The volume's three scratch blocks. Statics rather than locals because the
// same code runs in the kernel, where a 4096-byte local is the whole stack —
// keeping the shapes identical is what makes the host the place bugs are found.
static char CTX_A[SCHFS_BLKSIZE];
static char CTX_B[SCHFS_BLKSIZE];
static char CTX_C[SCHFS_BLKSIZE];
static struct schfs_ctx CTX = { CTX_A, CTX_B, CTX_C };
static long nfiles, ndirs, nlinks, nsymlinks, ndevs;
static long long nbytes, ndeduped;
static void
die(const char *what, int r)
{
fprintf(stderr, "mkschfs: %s failed (%d)%s\n", what, r,
r == -3 ? " -- image is full, try a larger --size" : "");
exit(1);
}
static void
inode_init(struct schfs_inode *in, const struct stat *st, uint16_t type,
uint64_t parent)
{
memset(in, 0, sizeof(*in));
in->type = type;
in->parent = parent;
in->nlink = 1;
in->mode = st ? (uint32_t)(st->st_mode & 07777)
: (type == SCHFS_T_DIR ? 0755u : 0644u);
in->uid = 0;
in->gid = 0;
if (st) {
// All three come from mtime, deliberately.
//
// Copying the host's atime would make the output depend on when the
// source files were last *read* -- and mkschfs reads every one of them,
// so building the same tree twice produced two different images. ctime
// is similar: it tracks host inode changes that mean nothing inside the
// image. mtime is the only one that describes the content, which is the
// only thing being copied.
in->atime = (uint64_t)st->st_mtime;
in->mtime = (uint64_t)st->st_mtime;
in->ctime = (uint64_t)st->st_mtime;
}
}
static void
bump_nlink(uint64_t ino)
{
struct schfs_cursor c;
schfs_cursor_init(&c, &VOL.tree, CURBLK);
struct schfs_inode in;
int r = schfs_inode_get(&VOL, &c, ino, &in);
if (r != 0)
die("re-reading an inode for a hard link", r);
in.nlink++;
// Under v1 this was the bug that shipped broken images: nlink was bumped
// but the inode's checksum was not recomputed, so every deduped file failed
// its own integrity check on first open. Here the checksum belongs to the
// tree node and is recomputed by the insert, so it cannot get out of step.
r = schfs_inode_put(&VOL, ino, &in);
if (r != 0)
die("writing back a hard link count", r);
}
static uint64_t
add_file(uint64_t parent, const char *host, const char *name,
const struct stat *st)
{
// A hard link is the same inode by definition, so ask the host before
// reading anything.
if (st->st_nlink > 1) {
uint64_t hit = dedup_lookup_ino(st->st_dev, st->st_ino);
if (hit) {
int r = schfs_dir_add(&VOL, parent, name, (unsigned)strlen(name),
hit, SCHFS_T_FILE);
if (r != 0)
die("linking a deduplicated file", r);
bump_nlink(hit);
nlinks++;
ndeduped += st->st_size;
return hit;
}
}
// CRC the content: it is both the dedup key and a cheap way to avoid
// reading candidates that cannot possibly match.
FILE *f = fopen(host, "rb");
if (!f) { perror(host); exit(1); }
uint32_t crc = 0;
for (;;) {
char buf[8192];
size_t n = fread(buf, 1, sizeof(buf), f);
if (n == 0) break;
crc = schfs_crc32c(crc, buf, (unsigned)n);
}
uint64_t existing = dedup_lookup(host, st->st_size, crc);
if (existing) {
fclose(f);
int r = schfs_dir_add(&VOL, parent, name, (unsigned)strlen(name),
existing, SCHFS_T_FILE);
if (r != 0)
die("linking a deduplicated file", r);
bump_nlink(existing);
nlinks++;
ndeduped += st->st_size;
return existing;
}
// The final size is known before a byte is written, so the block count goes
// into the inode on its first and only write, and the whole run of data
// blocks is claimed up front. That is what collapses a multi-megabyte file
// into a single extent record instead of one per 4 KB block.
uint64_t need = ((uint64_t)st->st_size + SCHFS_BLKSIZE - 1) / SCHFS_BLKSIZE;
uint64_t ino = schfs_alloc_objectid(&VOL);
struct schfs_inode in;
inode_init(&in, st, SCHFS_T_FILE, parent);
in.blocks = need;
int r = schfs_inode_put(&VOL, ino, &in);
if (r != 0) die("writing a file inode", r);
r = schfs_file_reserve(&VOL, ino, need);
if (r != 0) die("reserving data blocks", r);
rewind(f);
for (;;) {
// Large chunks on purpose: each append call re-reads and rewrites the
// inode, and every tree write copies its path, so appending a big file
// 8 KB at a time spends more space on metadata churn than on data.
static char buf[256 * 1024];
size_t n = fread(buf, 1, sizeof(buf), f);
if (n == 0) break;
r = schfs_file_append(&VOL, ino, buf, (unsigned)n, DATABLK);
if (r != 0) die("writing file data", r);
nbytes += (long long)n;
}
fclose(f);
r = schfs_dir_add(&VOL, parent, name, (unsigned)strlen(name),
ino, SCHFS_T_FILE);
if (r != 0) die("adding a directory entry", r);
dedup_remember(host, st, st->st_size, crc, ino);
nfiles++;
return ino;
}
static void
add_symlink(uint64_t parent, const char *host, const char *name,
const struct stat *st)
{
char target[4096];
ssize_t n = readlink(host, target, sizeof(target) - 1);
if (n < 0) { perror(host); exit(1); }
target[n] = '\0';
uint64_t need = ((uint64_t)n + SCHFS_BLKSIZE - 1) / SCHFS_BLKSIZE;
uint64_t ino = schfs_alloc_objectid(&VOL);
struct schfs_inode in;
inode_init(&in, st, SCHFS_T_SYMLINK, parent);
in.blocks = need;
int r = schfs_inode_put(&VOL, ino, &in);
if (r != 0) die("writing a symlink inode", r);
// The target is stored as the object's data, exactly like a file's
// contents, so nothing special is needed to read it back.
r = schfs_file_reserve(&VOL, ino, need);
if (r != 0) die("reserving a symlink block", r);
r = schfs_file_append(&VOL, ino, target, (unsigned)n, DATABLK);
if (r != 0) die("writing a symlink target", r);
r = schfs_dir_add(&VOL, parent, name, (unsigned)strlen(name),
ino, SCHFS_T_SYMLINK);
if (r != 0) die("adding a symlink entry", r);
nsymlinks++;
}
static void
add_dev(uint64_t parent, const char *name, const struct stat *st)
{
uint64_t ino = schfs_alloc_objectid(&VOL);
struct schfs_inode in;
inode_init(&in, st, SCHFS_T_DEV, parent);
in.rdev_major = (uint16_t)major(st->st_rdev);
in.rdev_minor = (uint16_t)minor(st->st_rdev);
int r = schfs_inode_put(&VOL, ino, &in);
if (r != 0) die("writing a device inode", r);
r = schfs_dir_add(&VOL, parent, name, (unsigned)strlen(name),
ino, SCHFS_T_DEV);
if (r != 0) die("adding a device entry", r);
ndevs++;
}
/* Sorted directory listing, so images are reproducible. */
struct namelist { char **v; int n, cap; };
static void
nl_add(struct namelist *l, const char *s)
{
if (l->n == l->cap) {
l->cap = l->cap ? l->cap * 2 : 64;
l->v = realloc(l->v, (size_t)l->cap * sizeof(*l->v));
if (!l->v) { perror("realloc"); exit(1); }
}
l->v[l->n++] = strdup(s);
}
static int
nl_cmp(const void *a, const void *b)
{
return strcmp(*(char *const *)a, *(char *const *)b);
}
static void add_tree(uint64_t dirino, const char *hostdir);
// Returns 1 if the entry was a directory, so the caller can total up the link
// count without walking the directory a second time.
static int
add_entry(uint64_t dirino, const char *hostdir, const char *name)
{
char path[8192];
snprintf(path, sizeof(path), "%s/%s", hostdir, name);
struct stat st;
// lstat, not stat: a symlink must be stored as a symlink rather than
// silently turned into a second copy of whatever it points at.
if (lstat(path, &st) != 0) { perror(path); exit(1); }
if (strlen(name) > SCHFS_NAME_MAX) {
fprintf(stderr, "mkschfs: name too long (max %d): %s\n",
SCHFS_NAME_MAX, path);
exit(1);
}
if (S_ISDIR(st.st_mode)) {
uint64_t ino = schfs_alloc_objectid(&VOL);
struct schfs_inode in;
inode_init(&in, &st, SCHFS_T_DIR, dirino);
in.nlink = 2; /* itself and its own "." */
int r = schfs_inode_put(&VOL, ino, &in);
if (r != 0) die("writing a directory inode", r);
r = schfs_dir_add(&VOL, dirino, name, (unsigned)strlen(name),
ino, SCHFS_T_DIR);
if (r != 0) die("adding a subdirectory entry", r);
ndirs++;
add_tree(ino, path);
return 1;
}
if (S_ISLNK(st.st_mode))
add_symlink(dirino, path, name, &st);
else if (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode))
add_dev(dirino, name, &st);
else if (S_ISREG(st.st_mode))
add_file(dirino, path, name, &st);
else
fprintf(stderr, "mkschfs: skipping %s (unsupported file type)\n", path);
return 0;
}
static void
add_tree(uint64_t dirino, const char *hostdir)
{
DIR *d = opendir(hostdir);
if (!d) { perror(hostdir); exit(1); }
struct namelist l = { 0, 0, 0 };
struct dirent *e;
while ((e = readdir(d)) != NULL) {
if (strcmp(e->d_name, ".") == 0 || strcmp(e->d_name, "..") == 0)
continue;
nl_add(&l, e->d_name);
}
closedir(d);
qsort(l.v, l.n, sizeof(*l.v), nl_cmp);
long subdirs = 0;
for (int i = 0; i < l.n; i++) {
subdirs += add_entry(dirino, hostdir, l.v[i]);
free(l.v[i]);
}
free(l.v);
// A directory's link count is 2 (itself, and its own ".") plus one per
// subdirectory, because each subdirectory's ".." points back here. ".." is
// synthetic in SchFS, but the count still has to match what POSIX tools
// expect, or "find -type d" prunes real directories and du walks the tree
// wrongly. Written last because the count is not known until the children
// have been added.
struct schfs_cursor c;
schfs_cursor_init(&c, &VOL.tree, CURBLK);
struct schfs_inode in;
if (schfs_inode_get(&VOL, &c, dirino, &in) != 0)
die("re-reading a directory inode", -2);
in.nlink = (uint32_t)(2 + subdirs);
if (schfs_inode_put(&VOL, dirino, &in) != 0)
die("writing a directory link count", -2);
}
/* ------------------------------------------------------------------ */
int
main(int argc, char **argv)
{
const char *out = NULL, *sysroot = NULL, *label = "schunix";
uint64_t nblocks = 6000;
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--sysroot") == 0 && i + 1 < argc)
sysroot = argv[++i];
else if (strcmp(argv[i], "--size") == 0 && i + 1 < argc)
nblocks = strtoull(argv[++i], NULL, 0);
else if (strcmp(argv[i], "--label") == 0 && i + 1 < argc)
label = argv[++i];
else if (strcmp(argv[i], "--hash-bits") == 0 && i + 1 < argc) {
// Testing only: shrink the name hash so collisions are the norm
// rather than a once-in-a-lifetime event. schfs_fsck takes the
// same flag and must be given the same value to read the image.
int bits = atoi(argv[++i]);
schfs_hash_mask = bits >= 64 ? ~0ULL : ((1ULL << bits) - 1);
}
else if (argv[i][0] == '-') {
fprintf(stderr, "mkschfs: unknown option %s\n", argv[i]);
return 1;
} else if (!out)
out = argv[i];
else {
fprintf(stderr, "mkschfs: too many arguments\n");
return 1;
}
}
if (!out || !sysroot) {
fprintf(stderr,
"usage: mkschfs out.img --sysroot DIR [--size BLOCKS] [--label NAME]\n"
" BLOCKS are %d-byte SchFS blocks (default 6000 = %d MB)\n",
SCHFS_BLKSIZE, (int)(6000 * SCHFS_BLKSIZE / (1024 * 1024)));
return 1;
}
if (nblocks <= SCHFS_FIRST_BLOCK) {
fprintf(stderr, "mkschfs: --size must exceed %d blocks (boot block "
"plus the uberblock ring)\n", SCHFS_FIRST_BLOCK);
return 1;
}
FILE *f = fopen(out, "w+b");
if (!f) { perror(out); return 1; }
// Pre-extend so every block is readable. The bump allocator hands out
// blocks in order, but the tree reads a block back the moment it is
// written, and a short file would fail that read rather than the write.
if (ftruncate(fileno(f), (off_t)nblocks * SCHFS_BLKSIZE) != 0) {
perror("ftruncate");
return 1;
}
IMG.f = f;
IMG.vol = &VOL;
IMG.nblocks = nblocks;
struct schfs_bdev bd;
bd.ctx = &IMG;
bd.bread = img_read;
bd.bwrite = img_write;
bd.balloc = img_alloc;
bd.bfree = img_free;
bd.balloc_run = img_alloc_run;
int r = schfs_vol_format(&VOL, &bd, nblocks, label, &CTX, img_palloc);
if (r != 0) die("formatting", r);
// The root directory is an object like any other; nothing points at it
// except the uberblock's root_objectid.
struct schfs_inode root;
struct stat rst;
int have_rst = (lstat(sysroot, &rst) == 0);
inode_init(&root, have_rst ? &rst : NULL, SCHFS_T_DIR, 0);
root.mode = 0755;
root.nlink = 2;
r = schfs_inode_put(&VOL, VOL.root_objectid, &root);
if (r != 0) die("writing the root inode", r);
ndirs++;
add_tree(VOL.root_objectid, sysroot);
// Publish. Everything above this line is invisible to a reader; if the tool
// died half way through, the image would simply have no valid uberblock and
// be rejected as "not a SchFS volume" rather than mounted half-built.
r = schfs_vol_commit(&VOL);
if (r != 0) die("committing the uberblock", r);
if (fflush(f) != 0 || fclose(f) != 0) { perror(out); return 1; }
uint64_t used = nblocks - VOL.free_blocks - SCHFS_FIRST_BLOCK;
printf("mkschfs: %s\n", out);
printf(" %ld dirs, %ld files, %ld hard links, %ld symlinks, %ld devices\n",
ndirs, nfiles, nlinks, nsymlinks, ndevs);
printf(" %lld bytes of data written, %lld bytes saved by dedup\n",
nbytes, ndeduped);
printf(" %llu of %llu blocks used (%llu KB of %llu KB), generation %llu\n",
(unsigned long long)used, (unsigned long long)nblocks,
(unsigned long long)used * SCHFS_BLKSIZE / 1024,
(unsigned long long)nblocks * SCHFS_BLKSIZE / 1024,
(unsigned long long)VOL.generation);
return 0;
}