SchOS/include/schfs.h

// SchFS — on-disk format.
//
// Design rationale lives in docs/architecture/schfs.md. In short: copy-on-write
// with the checksum of a block stored in the pointer that leads to it (ZFS),
// one B-tree reused for every kind of record (btrfs), and attribute indexes as
// just another record type (BeFS).
//
// This header is shared by the kernel and by host tools (tools/mkfs,
// tools/schfs_test), so it must stay free of kernel-only types.

#ifndef SCHFS_H
#define SCHFS_H

#include <stdint.h>

// ---------------------------------------------------------------------------
// Records that live at an arbitrary offset inside a block.
//
// A leaf's value area holds variable-length records packed end to end, so a
// record begins wherever the one before it ended -- which is a multiple of 8
// only by luck. The code reads them by casting a pointer into the block, and
// that is a lie to the compiler unless it is told: the type says "8-byte
// aligned" and the pointer is not.
//
// It works on x86, which is why it never showed as a bug, and it is undefined
// in C. A compiler entitled to assume alignment may emit an instruction that
// requires it, and the failure would arrive at a different optimisation level
// or on a different architecture looking like filesystem corruption rather than
// like codegen. UndefinedBehaviorSanitizer reported 31 of these across obj.c,
// objw.c, attr.c and space.c.
//
// This annotation changes NO on-disk layout. Every struct it is applied to
// already carries explicit `pad` fields and therefore has no implicit padding;
// sizes and member offsets are identical with and without it, which was
// measured rather than assumed before it was added. What it changes is the
// code generated to read them: byte-wise where the compiler cannot prove
// alignment, which is exactly the truth here.
//
// It is deliberately NOT applied to schfs_node_hdr, schfs_item, schfs_kptr or
// schfs_uberblock. Those sit at fixed offsets in a block -- the header is 32
// bytes, so everything after it is 8-aligned -- and annotating them would cost
// byte-wise access on the hottest paths in the tree to fix a problem they do
// not have.
#define SCHFS_PACKED __attribute__((packed))

#define SCHFS_MAGIC      0x53636846u   // "SchF"
#define SCHFS_NODE_MAGIC 0x536e6f64u   // "Snod"

// 4096 to match modern media and make extents worth having. The old format
// used 1024, which is also why its directories were so wasteful.
#define SCHFS_BLKSIZE    4096

// Deepest tree we will walk without recursing. 8 levels of a ~84-way internal
// node is far more than any plausible volume needs.
#define SCHFS_MAX_LEVEL  8

// ---------------------------------------------------------------------------
// Keys
//
// Every record in the filesystem is addressed by one 24-byte key, sorted by
// (objectid, type, offset). This is the whole reason there is only one tree:
// inodes, directory entries, extent maps and indexes are just different `type`
// values in the same keyspace.
// ---------------------------------------------------------------------------

#define SCHFS_KEY_INODE   1   // (ino,   INODE,  0)         -> struct schfs_inode
#define SCHFS_KEY_DIRENT  2   // (dirino,DIRENT, namehash)  -> packed dirent list
#define SCHFS_KEY_EXTENT  3   // (ino,   EXTENT, fileblk)   -> struct schfs_extent
#define SCHFS_KEY_ATTR    4   // (ino,   ATTR,   namehash)  -> attribute value
#define SCHFS_KEY_INDEX   5   // (idxid, INDEX,  valuehash) -> objectid
#define SCHFS_KEY_CSUM    6   // (CSUM,  CSUM,   diskblk)   -> uint32 CRC32c
#define SCHFS_KEY_SNAP    7   // (SNAP,  SNAP,    id)        -> struct schfs_snap
#define SCHFS_KEY_EXTREF  8   // (EXTREF,EXTREF, diskblk)   -> struct schfs_extref

// Data-block checksums live under one reserved object id, exactly as btrfs
// keeps a checksum tree. Tree nodes carry their checksum in their parent
// pointer, but file *data* blocks have no parent to hold it — without these
// records "end-to-end checksums" would only cover metadata, which is the part
// least likely to rot.
#define SCHFS_CSUM_OBJECTID  1ULL
// Snapshots live under their own reserved objectid, as ordinary records in the
// live tree. A snapshot is nothing more than a tree root that is kept: the
// B-tree is copy-on-write, so the blocks a root reaches are already immutable
// as long as nothing frees them. Keeping one costs a record.
#define SCHFS_SNAP_OBJECTID  2ULL
// Shared extents, keyed by the disk block a run starts at. btrfs calls this the
// extent tree and so, in effect, is this -- one reserved object id whose records
// say how many files point at a run.
//
// Only *shared* runs have a record. A run nothing has cloned has no entry at
// all, and absence means a reference count of one. That is not a space
// optimisation so much as a correctness one: the overwhelmingly common case
// stays exactly as it was, so the write path pays nothing for a feature it is
// not using, and there is no second structure to keep in step on every write.
#define SCHFS_EXTREF_OBJECTID 3ULL

// Object ids below this are reserved; the root directory is the first real one.
#define SCHFS_FIRST_OBJECTID 16ULL
#define SCHFS_ROOT_OBJECTID  16ULL

struct schfs_key {
    uint64_t objectid;
    uint64_t offset;
    uint8_t  type;
    uint8_t  pad[7];
};                                   /* 24 bytes */

// ---------------------------------------------------------------------------
// Node header — identical for internal nodes and leaves.
//
// `blocknr` is a self-pointer: a block that was written to the wrong place
// still checksums correctly, so the only way to catch a misdirected write is
// for the block to say where it thinks it lives.
// ---------------------------------------------------------------------------

struct schfs_node_hdr {
    uint32_t magic;         // SCHFS_NODE_MAGIC
    uint32_t crc;           // CRC32c of the whole block, this field zeroed
    uint64_t blocknr;       // where this block believes it lives
    uint64_t generation;    // transaction that wrote it
    uint16_t level;         // 0 = leaf, >0 = internal
    uint16_t nritems;
    uint32_t pad;
};                                   /* 32 bytes */

// Internal node entry: a key and the child it leads to, plus that child's
// checksum and generation. Storing the checksum *here* rather than in the
// child is what makes the tree self-validating.
struct schfs_kptr {
    struct schfs_key key;
    uint64_t blocknr;
    uint64_t generation;
    uint32_t crc;
    uint32_t pad;
};                                   /* 48 bytes */

// Leaf entry. Items grow forward from the end of the header; their data grows
// backward from the end of the block, exactly like btrfs. `offset` is measured
// from the start of the block.
struct schfs_item {
    struct schfs_key key;
    uint32_t offset;
    uint32_t size;
};                                   /* 32 bytes */

#define SCHFS_NODE_SPACE  (SCHFS_BLKSIZE - (int)sizeof(struct schfs_node_hdr))
#define SCHFS_MAX_KPTR    (SCHFS_NODE_SPACE / (int)sizeof(struct schfs_kptr))
#define SCHFS_MAX_ITEMS   (SCHFS_NODE_SPACE / (int)sizeof(struct schfs_item))

// Largest value that can live inline in a leaf. Anything bigger becomes an
// extent record pointing at data blocks.
#define SCHFS_MAX_INLINE  (SCHFS_NODE_SPACE / 4)

// ---------------------------------------------------------------------------
// Uberblock
//
// A ring of these is written round-robin. Mounting means reading them all and
// taking the highest generation that checksums — that is the entire crash
// recovery story, in place of a journal replay.
// ---------------------------------------------------------------------------

#define SCHFS_UBER_COUNT  128
#define SCHFS_UBER_START  1           // block 0 stays free for a boot sector

// First block available for the tree and for file data: everything below is
// the boot block and the uberblock ring.
#define SCHFS_FIRST_BLOCK (SCHFS_UBER_START + SCHFS_UBER_COUNT)

// ---------------------------------------------------------------------------
// The on-disk format version, and what a bump means
//
// The rule, so that "can this driver read this volume" has an answer that does
// not depend on reading the release notes:
//
//   * SCHFS_VERSION changes only when the *frame* changes -- the uberblock, the
//     node header, the key, the things every reader must understand before it
//     can parse anything at all. A driver refuses a version above its own,
//     loudly, and says so; it does not guess and it does not fall back to an
//     older uberblock, because an older uberblock is a stale filesystem.
//
//   * Everything else is a FEATURE FLAG, and this is what keeps the version
//     number from being a blunt instrument. Adding a record type, an index
//     kind or an optional field does not break a reader that ignores it, so it
//     must not cost every older reader the volume.
//
// Three classes, the arrangement ext4 and btrfs both arrived at because it is
// the one that works:
//
//   compat     Unknown bits may be ignored. Mount read-write regardless.
//   ro_compat  Unknown bits mean the volume may be MOUNTED READ-ONLY. The
//              writer would maintain a structure this driver does not know
//              about, so writing would silently invalidate it.
//   incompat   Unknown bits mean REFUSE. The volume cannot be parsed correctly
//              without understanding them.
//
// Choosing the class is the whole design decision, and the test is simple:
// what happens if a driver that does not know this feature writes to the
// volume? Nothing -> compat. Something on disk goes stale -> ro_compat.
// Something is unreadable -> incompat.
//
// All three words are zero on every volume written so far: schfs_vol_commit
// memsets the uberblock before filling it in, so v1 images already say "no
// features" without having been written by code that knew the field existed.
// ---------------------------------------------------------------------------

#define SCHFS_VERSION     1

// No features are defined yet. The masks exist so that the *checks* exist:
// adding a flag later is one bit here, and every driver already refuses
// correctly for the bits it does not have.
#define SCHFS_FEAT_COMPAT_SUPP     0u
#define SCHFS_FEAT_RO_COMPAT_SUPP  0u
#define SCHFS_FEAT_INCOMPAT_SUPP   0u

#define SCHFS_LABEL_MAX   32

// A retained tree root.
//
// The root a snapshot names is a *previous* version of the live tree, and it is
// reached exactly as the live one is -- same node format, same checksums. What
// makes it survive is schfs_space_scan walking it too, so its blocks are never
// handed out again.
//
// The record is stored in the live tree, which means creating one rewrites the
// path to it under copy-on-write and produces a new live root. That is correct
// and is why the snapshot names the root as it was *before* the record was
// added: a snapshot that included its own creation would have to refer to a
// block that does not exist yet.
#define SCHFS_SNAP_NAME_MAX 64

// Set on a volume that has at least one snapshot. See schfs_space_free.
#define SCHFS_SPACE_RETAIN_ALL 1

struct schfs_snap {
    uint64_t root;              // block number of the retained tree root
    uint64_t root_generation;
    uint32_t root_crc;          // checksum of that root block
    uint32_t pad0;
    uint64_t created;           // generation the snapshot was taken at
    char     name[SCHFS_SNAP_NAME_MAX];
} SCHFS_PACKED;

struct schfs_uberblock {
    uint32_t magic;             // SCHFS_MAGIC
    uint32_t crc;               // CRC32c of this struct, this field zeroed
    uint32_t version;           // SCHFS_VERSION
    uint32_t blocksize;         // SCHFS_BLKSIZE; a reader must refuse a mismatch
    uint64_t generation;        // transaction number; highest valid one wins
    uint64_t root;              // block number of the tree root
    uint32_t root_crc;          // checksum of that root block
    uint32_t pad0;
    uint64_t root_generation;
    uint64_t total_blocks;
    // Informational only. Which blocks are actually free is derived from the
    // tree at mount rather than stored, so this is what the volume believed at
    // commit time — useful for df before anything is mounted, never trusted.
    uint64_t free_blocks;
    // Feature flags. See the block comment above SCHFS_VERSION for what the
    // three classes mean and how to choose between them.
    //
    // These occupy what was one `uint64_t reserved0`, so the offset of every
    // field after them is unchanged and a v1 image -- which has zeros here --
    // reads identically. That is the only reason this could be added without a
    // version bump, and it is the last time that trick is available.
    uint32_t feature_incompat;
    uint32_t feature_ro_compat;
    uint64_t next_objectid;
    uint64_t root_objectid;     // normally SCHFS_ROOT_OBJECTID
    uint8_t  uuid[16];
    char     label[SCHFS_LABEL_MAX];
    uint32_t feature_compat;
    uint8_t  pad[SCHFS_BLKSIZE - 140];
};

// The uberblock is one block, and every field above is at a fixed offset in it.
// Both are on-disk facts rather than C conveniences, so the compiler checks
// them: a padding change or a reordered field is a format change that would
// otherwise be found by a volume that no longer mounts.
_Static_assert(sizeof(struct schfs_uberblock) == SCHFS_BLKSIZE,
               "uberblock must be exactly one block");
_Static_assert(__builtin_offsetof(struct schfs_uberblock, version) == 8,
               "uberblock version moved -- older drivers read this offset");
_Static_assert(__builtin_offsetof(struct schfs_uberblock, feature_incompat) == 64,
               "feature flags moved -- they must stay where reserved0 was");
_Static_assert(__builtin_offsetof(struct schfs_uberblock, next_objectid) == 72,
               "fields after the feature flags must not shift");

// ---------------------------------------------------------------------------
// Records
// ---------------------------------------------------------------------------

#define SCHFS_T_FILE     1
#define SCHFS_T_DIR      2
#define SCHFS_T_DEV      3
#define SCHFS_T_SYMLINK  4
// A bound AF_UNIX socket. A *compat* addition by the rule above: a driver that
// does not know this value maps it to SCHFS_T_FILE and sees a zero-length
// regular file, which is harmless -- nothing goes stale and nothing becomes
// unreadable, so it needs no feature flag.
#define SCHFS_T_SOCK     5

struct schfs_inode {
    uint64_t size;
    uint64_t blocks;
    uint64_t atime, mtime, ctime;
    // Containing directory. Meaningful for directories only, where it is what
    // ".." resolves to — neither "." nor ".." is stored on disk, so a directory
    // costs nothing until it has real entries. For files with several hard
    // links "parent" is simply the first directory that linked them and must
    // not be trusted.
    uint64_t parent;
    uint32_t mode;              // real POSIX mode bits, at last
    uint32_t uid, gid;
    uint32_t nlink;
    uint16_t type;              // SCHFS_T_*
    uint16_t rdev_major, rdev_minor;
    uint16_t pad;
    uint32_t flags;             // SCHFS_I_* — none defined yet
    uint32_t gen;               // transaction that last changed this inode;
                                // what an incremental send/snapshot diff needs
} SCHFS_PACKED;

// An extent maps a run of file blocks onto a run of disk blocks. refs is what
// makes snapshots and reflink copies nearly free: sharing a range is a refcount
// bump rather than a data copy.
struct schfs_extent {
    uint64_t disk_block;
    uint64_t nblocks;
    // Transaction group that allocated this run. A run stamped with the group
    // currently being built cannot be reached from any committed root, so its
    // blocks may be overwritten in place — which is what stops a program that
    // writes a hundred bytes at a time from copying a whole 4 KB block, and
    // then a whole tree path, on every call.
    //
    // A full 64 bits, matching the tree's generation counter, because a 32-bit
    // one would wrap after four billion commits and a wrapped comparison says
    // "same group" about a block belonging to a committed tree — the one
    // mistake this field exists to prevent. Only ever trusted together with
    // refs == 1; a shared run belongs to somebody else as well.
    uint64_t gen;
    // Whether this run is shared, as far as *this* file knows. A hint, not the
    // truth: the count that decides whether blocks may be released lives in the
    // extent-reference tree (SCHFS_KEY_EXTREF), because a per-file copy cannot
    // be decremented by the other file that shares the run.
    //
    // Kept because it makes the common case free: refs == 1 here means nothing
    // has ever cloned this run, so the write path can rewrite it in place
    // without consulting the tree at all.
    uint32_t refs;
    uint32_t pad;
} SCHFS_PACKED;

// How many files point at a run of blocks. Present only for runs with more than
// one reference; see SCHFS_EXTREF_OBJECTID.
struct schfs_extref {
    uint64_t nblocks;
    uint64_t refs;
} SCHFS_PACKED;

// Directory entry payload. The key's offset field holds the name hash, so a
// lookup is one tree descent rather than a directory scan.
//
// Two names in one directory can hash the same, and the key would then be
// identical — the second insert would silently replace the first. So the value
// is a *packed list*: one schfs_dirent_rec plus its name bytes, repeated. A
// 64-bit hash makes lists longer than one vanishingly rare, but "vanishingly
// rare" is not "impossible", and a filesystem that loses a file under a hash
// collision is not one to build on.
struct schfs_dirent_rec {
    uint64_t objectid;
    uint16_t type;
    uint16_t namelen;
    uint32_t pad;
    /* name bytes follow, namelen of them, not NUL-terminated */
} SCHFS_PACKED;

#define SCHFS_NAME_MAX 255

// ---------------------------------------------------------------------------
// Attributes, and the index over them
//
// An attribute is a named value on an object -- the same thing Linux calls an
// extended attribute, and the thing BeFS built its query interface on. Stored
// under (ino, ATTR, namehash), packed exactly as directory entries are, and for
// the same reason: two names can hash alike, and a filesystem that loses one of
// them under a collision is not one to build on.
//
// The point of having them is the *index*. An attribute may be indexed, and an
// indexed attribute gets a second set of records keyed by its value rather than
// by the object -- so "every object whose `type` is `photo`" is a range scan of
// a few keys rather than a walk of every object in the filesystem. That is the
// whole of what "a filesystem you can query" means here, and the reason the
// design is one B-tree with many key types: the feature is a few hundred lines
// on machinery that already exists, rather than a second engine.
struct schfs_attr_rec {
    uint32_t namelen;
    uint32_t valuelen;
    uint16_t type;              // SCHFS_ATTR_* below; what a query compares as
    uint16_t flags;
    uint32_t pad;
    /* name bytes, then value bytes; the pair padded to an 8-byte boundary */
} SCHFS_PACKED;

// What a value *is*, so that a query can compare two of them meaningfully. A
// string and an integer that happen to share bytes are not the same value, and
// ordering differs between them.
#define SCHFS_ATTR_RAW    0     // opaque bytes; only equality is meaningful
#define SCHFS_ATTR_STRING 1
#define SCHFS_ATTR_INT    2     // int64, little-endian
#define SCHFS_ATTR_TIME   3     // seconds since the epoch, int64

#define SCHFS_ATTR_NAME_MAX  63
#define SCHFS_ATTR_VALUE_MAX 255

// Step from one packed attribute record to the next. 8-byte aligned so the
// uint32s of the next one are naturally aligned.
#define SCHFS_ATTR_LEN(namelen, valuelen) \
    ((unsigned)((sizeof(struct schfs_attr_rec) + (namelen) + (valuelen) + 7) & ~7u))

// An index entry: one object that carries a given value for a given attribute.
//
// Keyed (indexid, INDEX, valuehash), where indexid is the hash of the attribute
// *name*. Several objects can share a value, so the payload is a packed list of
// these, exactly as dirents and attributes are.
struct schfs_index_rec {
    uint64_t objectid;
    uint32_t valuelen;
    uint32_t pad;
    /* value bytes follow, so a query can reject a hash collision */
} SCHFS_PACKED;

#define SCHFS_INDEX_LEN(valuelen) \
    ((unsigned)((sizeof(struct schfs_index_rec) + (valuelen) + 7) & ~7u))

// Which attributes are indexed. One record, under the index object id, whose
// value is a packed list of attribute names.
//
// A list rather than a flag on each attribute, because the question asked at
// every setxattr is "is *this name* indexed", and answering it from one small
// record is a single cached lookup rather than a search.
#define SCHFS_INDEX_OBJECTID 4ULL
#define SCHFS_INDEX_LIST_OFF 0ULL   /* the one record's offset in the key */

// Step from one packed dirent to the next. Records are 8-byte aligned so the
// uint64 objectid of the next one is naturally aligned.
#define SCHFS_DIRENT_LEN(namelen) \
    ((unsigned)((sizeof(struct schfs_dirent_rec) + (namelen) + 7) & ~7u))

// ---------------------------------------------------------------------------
// Compile-time layout checks.
//
// Every one of these is a number that ends up baked into images on disk. A
// silent change to any of them — adding a field, changing a type, a compiler
// padding a struct differently — makes every existing image unreadable in a way
// that looks like corruption rather than like a mistake. Failing the build is
// enormously cheaper than debugging that.
// ---------------------------------------------------------------------------

#define SCHFS_ASSERT_SIZE(t, n) \
    typedef char schfs_sizecheck_##t[(sizeof(struct t) == (n)) ? 1 : -1]

SCHFS_ASSERT_SIZE(schfs_key,         24);
SCHFS_ASSERT_SIZE(schfs_node_hdr,    32);
SCHFS_ASSERT_SIZE(schfs_kptr,        48);
SCHFS_ASSERT_SIZE(schfs_item,        32);
SCHFS_ASSERT_SIZE(schfs_uberblock,   SCHFS_BLKSIZE);
SCHFS_ASSERT_SIZE(schfs_inode,       80);
SCHFS_ASSERT_SIZE(schfs_extent,      32);
SCHFS_ASSERT_SIZE(schfs_dirent_rec,  16);
SCHFS_ASSERT_SIZE(schfs_extref,      16);
SCHFS_ASSERT_SIZE(schfs_attr_rec,    16);
SCHFS_ASSERT_SIZE(schfs_index_rec,   16);

#endif /* SCHFS_H */

// The annotation above must not have moved anything. These are on-disk sizes:
// if one changes, every existing volume is misread from that record onward, and
// the failure looks like corruption rather than like a struct that grew.
_Static_assert(sizeof(struct schfs_dirent_rec) == 16, "dirent record size changed");
_Static_assert(sizeof(struct schfs_extent)     == 32, "extent record size changed");
_Static_assert(sizeof(struct schfs_extref)     == 16, "extref record size changed");
_Static_assert(sizeof(struct schfs_index_rec)  == 16, "index record size changed");
_Static_assert(sizeof(struct schfs_attr_rec)   == 16, "attr record size changed");
_Static_assert(sizeof(struct schfs_snap)       == 96, "snapshot record size changed");
_Static_assert(sizeof(struct schfs_inode)      == 80, "inode record size changed");