nexoniarz/SchOS
nexoniarz/SchOS
Watch
1
0
Fork
You've already forked SchOS
 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

schfs: the commit barrier, reflink, and attributes with an index over them

Browse source 
The last three phases of the filesystem, which were written together and
touch the same files throughout -- kmount.c, objw.c and vfs.c are each
rewritten by all three -- so they land as one commit rather than three
that could not be built separately.

Durability. A transaction that reaches the platter out of order is worse
than one that never reached it: the uberblock can name a tree whose
blocks are still in the drive's write cache. bdirty/bflush give the
buffer layer an explicit barrier, ideflush issues ATA FLUSH CACHE and
usb_storage_flush the SCSI SYNCHRONIZE CACHE that means the same thing,
and a commit now waits for the data before it writes the root that
points at it. The uberblock groups are read newest-first with fallback,
so a volume whose newest roots are torn mounts at an older one and says
which generations were lost.

Reflink. Extents carry a refcount, so two objects can name the same
blocks and neither frees them while the other holds them. Exposed as
FICLONE -- Linux's ioctl value, so `cp --reflink` works unmodified --
and as cmd/reflink for the case where there is no cp to hand.

Attributes, and the one genuinely new idea here. attr.c stores named
values on an object, reachable through setxattr/getxattr/removexattr at
Linux's syscall numbers. On top of that, schfs_index declares an index
over an attribute name and schfs_query answers "which objects carry this
value" from the B-tree rather than by walking. A query against an
unindexed attribute is refused with an error, never silently answered by
a full scan: a query that quietly becomes a scan is worse than one that
admits it cannot help. cmd/schattr drives both.

Three bugs fixed on the way, each found by the numbers being wrong
rather than by inspection:

  - statfs's f_fsid was declared int64[2]. Linux's fsid_t is two *ints*,
    so every field after it landed one slot late: userland read f_frsize
    out of f_namelen, got 255, and df reported a 32 MB root filesystem
    as 1992 KB. It hid for as long as it did because the fields used to
    be invented constants, so wrong numbers looked like the usual ones.
  - struct inode's nlink was a short against SchFS's uint32, so a file
    past 32767 names came back negative -- and iput frees an object
    whose count reaches zero. It is an int now, signed on purpose, so an
    underflow lands somewhere a check can see rather than wrapping.
  - filewrite split every write to fit v1's log. That limit belongs to
    the journal, not to writing, and SchFS makes one transaction of
    whatever it is handed: a 1 MB write was becoming 170 commits and
    170 flushes for nothing. fs_ops carries max_write now, and SchFS
    sets no limit.

Also here: a real mount table, so `mount -t schfs /dev/hdb /mnt` shows
up in /proc/mounts -- df looks a path up there before it calls statfs at
all, so a mount missing from that list was invisible to it -- several
volumes mounted at once, snapshots mounted by name, and /proc/fsstat
with an rdtsc-based counter, which is the only instrument fine enough to
see inside a single syscall.

Verified: `make schfs-check` passes in full, including the sweep that
crashes the volume at each of 119 points mid-transaction and requires
every one to mount with exactly its committed data, and the case where
both newest uberblocks are rubbish and the volume still comes back. The
headless boot check passes with 27 [ok] and no failures, on a SchFS root.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Nexoniarz 2026-08-15 16:01:22 +02:00
commit 03cc316001
39 changed files with 3942 additions and 550 deletions
Download patch file Download diff file Expand all files Collapse all files

219
kernel/fs/schfs/vfs.c
View file

@ -39,6 +39,7 @@
#include "buf.h"
#include "file.h"
#include "errno.h"
#include "fsstat.h"
#include "schfs_int.h"
@ -91,7 +92,10 @@ schfs_vfs_iload(struct inode *ip)
ip->type = type_to_vfs(in.type);
ip->major = (short)in.rdev_major;
ip->minor = (short)in.rdev_minor;
ip->nlink = (short)in.nlink;
// Clamped, not cast. The on-disk count is a uint32 and the VFS's is a signed
// int, so the top half of the range has nowhere to go; saturating keeps it
// positive, and a file with two billion names is not going to lose one here.
ip->nlink = in.nlink > 0x7fffffffu ? 0x7fffffff : (int)in.nlink;
ip->size = in.size;
ip->rwmode = RW;
ip->supermode = 0;
@ -123,7 +127,7 @@ schfs_vfs_istore(struct inode *ip)
// Read-modify-write rather than build-from-scratch: the on-disk record holds
// fields the VFS has no slot for (uid, gid, the real mode bits, the
// generation), and rebuilding it here would quietly zero every one of them.
if (schfs_kstat(ip->inum, &in) != 0) {
if (schfs_kstat_dev(ip->dev, ip->inum, &in) != 0) {
memset(&in, 0, sizeof(in));
in.mode = ip->type == T_DIR ? 0755 : 0644;
}
@ -137,7 +141,7 @@ schfs_vfs_istore(struct inode *ip)
in.mtime = ip->mtime;
in.ctime = ip->ctime;
schfs_ksetattr(ip->inum, &in);
schfs_ksetattr(ip->dev, ip->inum, &in);
}
static void
@ -145,7 +149,7 @@ schfs_vfs_ifree(struct inode *ip)
{
if (readonly(ip))
return;
schfs_kfree(ip->inum);
schfs_kfree(ip->dev, ip->inum);
}
/* ------------------------------------------------------------------ */
@ -173,6 +177,9 @@ schfs_vfs_ifree(struct inode *ip)
static int
schfs_vfs_read(struct inode *ip, int user_dst, uint64 dst, uint64 off, uint n)
{
uint64 t0 = rdtsc();
fsstat.read_calls++;
char *tmp = kalloc();
if (tmp == 0)
return -1;
@ -199,6 +206,8 @@ schfs_vfs_read(struct inode *ip, int user_dst, uint64 dst, uint64 off, uint n)
}
kfree(tmp);
fsstat.read_bytes += done;
fsstat.read_cycles += rdtsc() - t0;
return (int)done;
}
@ -212,6 +221,22 @@ schfs_vfs_write(struct inode *ip, int user_src, uint64 src, uint64 off, uint n)
if (tmp == 0)
return -1;
// One transaction for the whole call, not one per 4 KB chunk.
//
// Each schfs_kwrite is its own group otherwise, and a group ends in two cache
// flushes and an uberblock write. A flush is a barrier, so committing per
// chunk forbids the buffer cache to coalesce the tree writes that every chunk
// repeats — the same leaf, holding this file's data checksums, went to the
// disk once per chunk. Measured over 1 MB: 1410 block writes per chunk-commit
// against 267 for one group. It is also better semantics, since a short write
// now leaves the file at a length some prefix of this call actually reached
// rather than at whichever chunk happened to commit last.
//
// copyin under the bracket is safe: it walks the page table and returns -1 on
// an unmapped address rather than faulting, and a page fault here would be a
// SIGSEGV in any case — there is no demand paging to re-enter the filesystem.
schfs_ktx_open(ip->dev);
uint done = 0;
while (done < n) {
uint want = n - done;
@ -225,13 +250,19 @@ schfs_vfs_write(struct inode *ip, int user_src, uint64 src, uint64 off, uint n)
memmove(tmp, (char *)(src + done), want);
}
long put = schfs_kwrite(ip->inum, off + done, tmp, want);
long put = schfs_kwrite(ip->dev, ip->inum, off + done, tmp, want);
if (put <= 0) break;
done += (uint)put;
if ((uint)put < want) break;
}
int cr = schfs_ktx_close(ip->dev);
kfree(tmp);
// A failed commit means nothing durably happened, so report it that way
// rather than claiming bytes that are not on the disk.
if (cr != 0)
return -1;
if (off + done > ip->size)
ip->size = off + done;
return (int)done;
@ -242,7 +273,7 @@ schfs_vfs_truncate(struct inode *ip, uint64 newsize)
{
if (readonly(ip))
return;
if (schfs_ktruncate(ip->inum, newsize) == 0)
if (schfs_ktruncate(ip->dev, ip->inum, newsize) == 0)
ip->size = newsize;
}
@ -297,7 +328,7 @@ schfs_vfs_dirlink(struct inode *dp, char *name, uint64 inum)
if (schfs_kstat_dev(dp->dev, inum, &in) != 0)
return -1;
int r = schfs_kdiradd(dp->inum, name, namelen, inum, in.type);
int r = schfs_kdiradd(dp->dev, dp->inum, name, namelen, inum, in.type);
if (r != 0)
return -1;
@ -306,7 +337,7 @@ schfs_vfs_dirlink(struct inode *dp, char *name, uint64 inum)
// parent -- and every ".." walk out of it lands somewhere else entirely.
if (in.type == SCHFS_T_DIR && in.parent != dp->inum) {
in.parent = dp->inum;
schfs_ksetattr(inum, &in);
schfs_ksetattr(dp->dev, inum, &in);
}
return 0;
}
@ -320,21 +351,21 @@ schfs_vfs_dirunlink(struct inode *dp, char *name, uint off)
(void)off;
unsigned namelen = 0;
while (name[namelen]) namelen++;
return schfs_kunlink(dp->inum, name, namelen) == 0 ? 0 : -1;
return schfs_kunlink(dp->dev, dp->inum, name, namelen) == 0 ? 0 : -1;
}
static int
schfs_vfs_dirempty(struct inode *dp)
{
return schfs_kdirempty(dp->inum);
return schfs_kdirempty(dp->dev, dp->inum);
}
static struct inode *
schfs_vfs_ialloc(uint dev, short type)
{
uint64 ino = 0;
if (schfs_kalloc_object(type_from_vfs(type), type == T_DIR ? 0755 : 0644,
&ino) != 0)
if (schfs_kalloc_object(dev, type_from_vfs(type),
type == T_DIR ? 0755 : 0644, &ino) != 0)
return 0;
return iget(dev, ino);
}
@ -345,44 +376,60 @@ schfs_vfs_ialloc(uint dev, short type)
// One record's worth of state for the callback below.
//
// Entries come back in name-hash order, which has nothing to do with their
// names -- POSIX promises no order, and hash order is what makes a lookup one
// tree descent instead of a scan. The position handed back to userland is
// therefore an *index*: entry N of the walk. Resuming means walking to N again,
// which readdir already does linearly, so a full listing stays linear overall
// only because the common case is resuming exactly where the last call stopped.
// ---------------------------------------------------------------------------
// The resume cookie, and why it is a key rather than a count
//
// getdents is called repeatedly until it returns nothing, and each call has to
// carry on where the last stopped. `*poff` is the only state there is, so
// whatever goes in it must be enough to find that place again.
//
// It used to be an ordinal -- "entry N of the walk" -- and resuming meant
// walking from the first entry and skipping N. That is quadratic in the size of
// the directory, and measurably so: listing /bin, 243 entries, walked 914
// entries across five calls, because each call started over. A larger directory
// gets worse faster than it gets bigger.
//
// It is now the tree key to seek to. Entries are keyed (dir, DIRENT, hash), so
// the cookie is a hash and resuming is one descent to that hash. The three
// values below `SCHFS_DIRENT_FIRST` are reserved for the synthesised "." and
// "..", which are not records and would otherwise have no position at all.
//
// One consequence worth stating: a whole hash bucket is emitted or none of it
// is. Buckets hold colliding names, at most SCHFS_MAX_INLINE packed, which is
// under 3 KB of output records -- so any caller with a 4 KB buffer never
// notices. A caller whose buffer cannot hold one bucket gets EINVAL, which is
// what Linux does when a buffer cannot hold a single record.
#define GD_DOT 0 /* emit "." next */
#define GD_DOTDOT 1 /* emit ".." next */
#define GD_ENTRIES 2 /* emit real entries, from hash (cookie - 2) */
struct gd_ctx {
char *out;
int cap;
int written;
uint64 skip; // entries to step over before emitting
uint64 index; // entries seen so far
int full;
uint64 from; // hash to resume at
uint64 next; // hash the *following* call should resume at
int full; // buffer filled; stop the walk
int toobig; // a single bucket did not fit in an empty buffer
uint64 bucket; // hash of the bucket being emitted
int bucket_started;
};
static int
gd_entry(uint64 ino, unsigned short type, const char *name, unsigned namelen,
void *arg)
gd_emit(struct gd_ctx *g, uint64 ino, unsigned short type,
const char *name, unsigned namelen)
{
struct gd_ctx *g = arg;
if (g->index++ < g->skip)
return 0;
if (g->full)
return 1;
if (namelen > 255)
namelen = 255;
int reclen = (8 + 8 + 2 + 1 + (int)namelen + 1 + 7) & ~7;
if (g->written + reclen > g->cap) {
g->full = 1;
return 1; /* stop; the caller will come back */
}
if (g->written + reclen > g->cap)
return -1;
char *k = g->out + g->written;
uint64 *p64 = (uint64 *)k;
p64[0] = ino;
p64[1] = g->index; /* resume position: entries consumed */
p64[1] = g->next;
*(uint16 *)(k + 16) = (uint16)reclen;
*(uint8 *)(k + 18) = type == SCHFS_T_DIR ? 4 /* DT_DIR */
: type == SCHFS_T_DEV ? 2 /* DT_CHR */
@ -391,44 +438,98 @@ gd_entry(uint64 ino, unsigned short type, const char *name, unsigned namelen,
memmove(k + 19, name, namelen);
memset(k + 19 + namelen, 0, reclen - 19 - namelen);
g->written += reclen;
fsstat.getdents_walked++; // entries emitted; no longer entries re-walked
return 0;
}
static int
gd_entry(uint64 ino, unsigned short type, const char *name, unsigned namelen,
uint64 hash, void *arg)
{
struct gd_ctx *g = arg;
if (g->full)
return 1;
// A new bucket. Everything emitted so far is safe to keep, because the
// cookie already points past the previous bucket.
if (!g->bucket_started || hash != g->bucket) {
g->bucket = hash;
g->bucket_started = 1;
g->next = hash + GD_ENTRIES; // resume *at* this bucket if we stop here
}
// The cookie the emitted records carry has to be "past this bucket", or a
// caller that stops reading mid-buffer would come back and get the bucket
// again. Setting it before the emit is what makes that true.
uint64 saved = g->next;
g->next = hash + 1 + GD_ENTRIES;
if (gd_emit(g, ino, type, name, namelen) < 0) {
// Did not fit. Roll back to the start of this bucket so the whole thing is
// re-read next time, and stop.
g->next = saved;
g->full = 1;
if (g->written == 0)
g->toobig = 1;
return 1;
}
return 0;
}
static int
schfs_vfs_getdents(struct inode *dp, char *out, int cap, uint64 *poff)
{
uint64 t0 = rdtsc();
fsstat.getdents_calls++;
struct gd_ctx g;
g.out = out;
g.cap = cap;
g.written = 0;
g.skip = *poff;
g.index = 0;
g.full = 0;
g.out = out;
g.cap = cap;
g.written = 0;
g.from = *poff;
g.next = *poff;
g.full = 0;
g.toobig = 0;
g.bucket = 0;
g.bucket_started = 0;
// "." and ".." are synthesised: they are not records, so a walk would never
// produce them, and every readdir caller expects both.
static const char *dots[2] = { ".", ".." };
for (int i = 0; i < 2; i++) {
if (g.index++ < g.skip)
continue;
int reclen = (8 + 8 + 2 + 1 + (i + 1) + 1 + 7) & ~7;
if (g.written + reclen > cap)
return g.written;
char *k = out + g.written;
uint64 *p64 = (uint64 *)k;
p64[0] = dp->inum; /* close enough for ".."; nothing reads it */
p64[1] = g.index;
*(uint16 *)(k + 16) = (uint16)reclen;
*(uint8 *)(k + 18) = 4; /* DT_DIR */
memmove(k + 19, dots[i], i + 1);
memset(k + 19 + i + 1, 0, reclen - 19 - i - 1);
g.written += reclen;
if (g.from <= GD_DOTDOT) {
static const char *dots[2] = { ".", ".." };
for (int i = (int)g.from; i < 2; i++) {
g.next = (uint64)i + 1;
if (gd_emit(&g, dp->inum, SCHFS_T_DIR, dots[i], (unsigned)(i + 1)) < 0) {
if (g.written == 0)
return -EINVAL;
*poff = g.next;
return g.written;
}
}
g.next = GD_ENTRIES; // real entries start at hash 0
g.from = GD_ENTRIES;
}
if (schfs_kreaddir_dev(dp->dev, dp->inum, gd_entry, &g) != 0 && g.written == 0)
uint64 start_hash = g.from - GD_ENTRIES;
if (schfs_kreaddir_from_dev(dp->dev, dp->inum, start_hash, gd_entry, &g) != 0
&& g.written == 0)
return -1;
*poff = g.index;
if (g.toobig)
return -EINVAL;
// Nothing emitted and the walk ran to the end: this is the end of the
// directory, and the cookie must not move or the caller loops.
if (!g.full && g.written == 0) {
*poff = g.next;
fsstat.getdents_cycles += rdtsc() - t0;
return 0;
}
*poff = g.next;
fsstat.getdents_cycles += rdtsc() - t0;
return g.written;
}