SchOS/kernel/fs/schfs/kmount.c

// SPDX-License-Identifier: GPL-2.0-or-later
//
// SchFS in the kernel: block device, mount, and the transaction bracket.
//
// The filesystem code itself (btree.c, space.c, vol.c, obj.c, objw.c) is the
// same portable C the host tools compile, so nothing here re-implements the
// format. This file is only the three things that cannot be shared: a block
// device backed by the buffer cache, the boot-time mount, and the rule about
// when a group of changes becomes durable.
//
// SchFS is the root filesystem: proc.c mounts it before pid 1 gets a working
// directory and panics if it is not there. That was not true when this file was
// written -- v1 in kernel/fs/fs.c was still the root, and a SchFS volume that
// failed to mount cost nothing -- and it is worth knowing that a mount failure
// here now stops the machine.
//
// ---------------------------------------------------------------------------
// Transactions
//
// Every entry point below that changes anything is one transaction group: it
// takes the volume lock, does its work, and commits before returning. That
// makes an operation durable the moment it returns, and — because a commit is a
// single uberblock write that either lands or does not — indivisible against a
// crash. There is no journal to replay and nothing to undo, because none of the
// work is reachable until that one block is written.
//
// The cost is two cache flushes and an uberblock write per operation, and it is
// steep enough that callers doing many related changes should bracket them —
// see schfs_ktx_open below. Measured over 1 MB of file data: 1410 block writes
// committing every 4 KB against 267 for the same megabyte as one group. Not a
// tuning difference. A commit contains a flush, and a flush is a barrier, so
// committing often is exactly the same thing as forbidding the buffer cache to
// coalesce anything.
//
// Note that this is the opposite of what the numbers said before kd_write
// started deferring: batching used to buy about 8%, because every write went
// straight to the disk whether it was committed or not, so there was nothing
// for a bracket to save. Deferral is what makes batching worth anything, and
// batching is what makes deferral worth anything. Neither pays alone.

#include "types.h"
#include "defs.h"
#include "param.h"
#include "spinlock.h"
#include "sleeplock.h"
#include "proc.h"   /* myproc() — transaction brackets are per-process */
#include "fs.h"
#include "buf.h"

#include "file.h"   /* SCHFS_DEV, SCHFS_SNAP_DEV */
#include "errno.h"  /* mounting reports why it failed, not just that it did */
#include "x86.h"
#include "fsstat.h"
#include "schfs_int.h"

// How many buffer-cache blocks one filesystem block spans. BSIZE was 1024 when
// this was written, so it was four; BSIZE is a whole SchFS block now and it is
// one. The loops below stay written for the general case because the constant
// is the sort of thing that moves back.
#define SCHFS_BPB  (SCHFS_BLKSIZE / BSIZE)

struct schfs_kdev {
  uint dev;                  // the *block* device this volume lives on
};

// One mounted volume.
//
// This used to be a handful of file-scope singletons -- one kvol, one kmounted,
// one set of block buffers -- because there was one volume and no way to ask for
// another. Everything that mutated named kvol directly, so sys_mount's source
// argument had nothing it could mean and was ignored: `mount -t schfs /dev/hdb
// /mnt` mounted whatever the kernel had probed at boot, whatever you typed.
//
// Slot 0 is the root volume, so SCHFS_DEV on its own still names it.
struct schfs_mount {
  struct schfs_kdev kdev;
  struct schfs_vol  vol;
  int               mounted;

  // Block buffers, allocated once at mount and kept for its lifetime.
  //
  // Nothing in SchFS may put a block on the stack: a cursor needs a whole
  // SCHFS_BLKSIZE buffer and the kernel stack is one 4096-byte page
  // (KSTACKSIZE), so a single such local overflows it. kalloc returns exactly
  // one page, which is exactly one SchFS block, so each of these fits with
  // nothing to spare.
  //
  // curblk and datablk serve the read paths; ctx* are the three the volume
  // keeps for mutations. They are separate pages because a write path runs a
  // cursor of its own inside the volume's buffers while the caller's cursor is
  // still positioned in curblk.
  char *curblk, *datablk;
  char *ctxa, *ctxb, *ctxc;

  // Those buffers are shared by every caller of this volume, so every entry
  // point below holds this lock for the whole operation. A sleeplock rather
  // than a spinlock because bread() sleeps waiting on the disk.
  //
  // It also serialises transactions, which is not incidental: a group being
  // built rewrites its own blocks in place, so two operations interleaving
  // inside one group would each see the other's half-finished tree.
  //
  // Per volume rather than one for all of them: two mounted disks share no
  // state at all, and a single lock would have made a write to one wait behind
  // a write to the other for no reason.
  struct sleeplock lock;

  // The explicit transaction bracket, per volume for the same reason.
  //
  // The pid is carried alongside the pointer, and both must match, because a
  // `struct proc *` alone does not identify a process: ptable slots are
  // recycled, so an exited owner's pointer compares equal to whichever
  // unrelated process next lands in that slot. tx_mine() would then be true for
  // a process that holds nothing, klock() would skip the lock entirely, and two
  // callers would share one volume's scratch pages. A recycled slot always
  // carries a new pid, so the pair is unambiguous.
  struct proc      *tx_owner;
  int               tx_owner_pid;
  int               tx_depth;
  int               tx_dirty;
};

static struct schfs_mount kmnt[SCHFS_VOL_SLOTS];

// A mounted snapshot: the same block device with an older tree root, read-only.
//
// A snapshot borrows its live volume's lock and block device; only its tree
// root and scratch pages are its own. It is not a separate filesystem, it is a
// second way into one that is already mounted.
static struct schfs_vol  ksnapvol[SCHFS_SNAP_SLOTS];
static int               ksnap_mounted[SCHFS_SNAP_SLOTS];
static int               ksnap_home[SCHFS_SNAP_SLOTS];   // which volume slot
static char             *ksnapcurblk[SCHFS_SNAP_SLOTS];
static char             *ksnapdatablk[SCHFS_SNAP_SLOTS];

// Seconds since the epoch, for inode timestamps.
//
// This used to return uptime(), with a comment saying the kernel had no wall
// clock. It has had one since time_init() started reading the RTC at boot, and
// the stale comment outlived the condition it described -- which is how every
// file created after boot ended up stamped 1970.
//
// Two errors, not one, and the second is the less obvious: uptime() counts
// 10 ms *ticks*, and they were being used as *seconds*. A node created 60 ticks
// (0.6 s) into boot therefore read back as 00:01:00, so the timestamps were not
// merely offset, they ran a hundred times too fast.
//
// wallclock() is the one source (kernel/core/time.c), so this cannot disagree
// with gettimeofday about what time it is -- which was the whole reason that
// function exists.
static uint64_t
know(void)
{
  uint64 sec;
  wallclock(&sec, 0);
  return (uint64_t)sec;
}

// Explicit transaction brackets.
//
// One schfs_k* call is one transaction group, which is the right default and
// the wrong granularity for a caller making many related changes. A single
// write(2) of 1 MB arrives here as 256 separate 4 KB writes, and committing
// each one costs two cache flushes and an uberblock — measured at 1410 block
// writes to store 256 blocks of data, against 267 when the lot is one group.
// A flush is a barrier: nothing before it can be deferred past it, so the more
// often you commit the less the buffer cache can coalesce.
//
// So a caller may bracket a run of calls into one group. It is not only
// faster: the bracket is indivisible against a crash as a whole, which is
// closer to what a single write(2) is supposed to mean than 256 separately
// durable pieces ever were.
//
// `tx_owner` is read without the lock, which is safe in the one direction it
// is used: only the process inside a bracket ever writes its own proc pointer
// there, and it holds the lock the whole time it is set. Nobody else can
// observe their own pointer, so the test cannot be wrong for the caller asking.
// The null check matters — myproc() is 0 in some early contexts, and 0 == 0
// would otherwise read as "I hold the lock" for a process that does not exist.

/* ------------------------------------------------------------------ */
/* Which volume a device number names                                  */
/*                                                                     */
/* The VFS hands an inode to every fs_ops call and an inode carries its */
/* device, so this is all the dispatch either a second disk or a        */
/* read-only snapshot needs.                                            */
/* ------------------------------------------------------------------ */

// The mount a device number belongs to. A snapshot belongs to the live volume
// it was taken from: they share a block device, a lock and a space map.
static struct schfs_mount *
mnt_for(uint dev)
{
  if (SCHFS_IS_SNAP_DEV(dev))
    return &kmnt[ksnap_home[dev - SCHFS_SNAP_DEV]];
  if (SCHFS_IS_VOL_DEV(dev))
    return &kmnt[dev - SCHFS_DEV];
  return &kmnt[0];               // not ours; callers check mounted_for first
}

// The volume a device number reads through. For a snapshot this is the slot's
// own copy, holding an older tree root.
static struct schfs_vol *
vol_for(uint dev)
{
  return SCHFS_IS_SNAP_DEV(dev) ? &ksnapvol[dev - SCHFS_SNAP_DEV]
                                : &mnt_for(dev)->vol;
}

static char *
curblk_for(uint dev)
{
  return SCHFS_IS_SNAP_DEV(dev) ? ksnapcurblk[dev - SCHFS_SNAP_DEV]
                                : mnt_for(dev)->curblk;
}

static char *
datablk_for(uint dev)
{
  return SCHFS_IS_SNAP_DEV(dev) ? ksnapdatablk[dev - SCHFS_SNAP_DEV]
                                : mnt_for(dev)->datablk;
}

static int
mounted_for(uint dev)
{
  if (SCHFS_IS_SNAP_DEV(dev))
    return ksnap_mounted[dev - SCHFS_SNAP_DEV];
  if (SCHFS_IS_VOL_DEV(dev))
    return kmnt[dev - SCHFS_DEV].mounted;
  return 0;
}

// Is this volume writable? A snapshot never is: it is a view of a tree that has
// already been committed, and there is nowhere for a change to go.
static int
writable_for(uint dev)
{
  return SCHFS_IS_VOL_DEV(dev) && mounted_for(dev) &&
         !kmnt[dev - SCHFS_DEV].vol.readonly;
}

static int
tx_mine(struct schfs_mount *m)
{
  struct proc *p = myproc();
  return m->tx_owner != 0 && p != 0 &&
         m->tx_owner == p && m->tx_owner_pid == p->pid;
}

// Take a volume's lock unless this process already holds it through a bracket.
// Every entry point in this file goes through these two rather than touching a
// sleeplock directly: it is not recursive, so a bracketed caller reaching a
// bare acquiresleep would wait on itself forever.
static void
klock(uint dev)
{
  struct schfs_mount *m = mnt_for(dev);
  if (!tx_mine(m))
    acquiresleep(&m->lock);
}

static void
kunlock(uint dev)
{
  struct schfs_mount *m = mnt_for(dev);
  if (!tx_mine(m))
    releasesleep(&m->lock);
}

// The bracket's state, for /proc/fsstat. A wedged filesystem looks exactly like
// a slow one from userland -- every path hangs in acquiresleep with no way to
// tell whether anything owns the lock. This says so.
//
// Read without the lock on purpose: the one time it matters is when the lock
// cannot be taken, and a reporting path that blocks on the thing it is
// reporting on is useless.
void
schfs_lockinfo(int slot, int *locked, int *owner_pid, int *depth)
{
  if (slot < 0 || slot >= SCHFS_VOL_SLOTS) {
    *locked = *owner_pid = *depth = -1;
    return;
  }
  struct schfs_mount *m = &kmnt[slot];
  *locked    = m->mounted ? m->lock.locked : -1;
  *owner_pid = m->tx_owner_pid;
  *depth     = m->tx_depth;
}

void
schfs_ktx_open(uint dev)
{
  struct schfs_mount *m = mnt_for(dev);

  if (tx_mine(m)) {
    m->tx_depth++;               // already inside one: nest, do not relock
    return;
  }
  acquiresleep(&m->lock);
  m->tx_owner     = myproc();
  m->tx_owner_pid = myproc() ? myproc()->pid : 0;
  m->tx_depth     = 1;
  m->tx_dirty     = 0;
}

int
schfs_ktx_close(uint dev)
{
  struct schfs_mount *m = mnt_for(dev);
  int r = 0;

  if (!tx_mine(m))
    return -1;                   // not ours to close
  if (--m->tx_depth > 0)
    return 0;

  if (m->tx_dirty)
    r = schfs_vol_commit(&m->vol);
  m->tx_dirty     = 0;
  m->tx_owner     = 0;
  m->tx_owner_pid = 0;
  releasesleep(&m->lock);
  return r;
}

// Start and finish one transaction. `changed` says whether anything was
// written, so a read does not pay for an uberblock write.
static void
ktx_begin(uint dev)
{
  klock(dev);
}

static int
ktx_end(uint dev, int changed, int r)
{
  struct schfs_mount *m = mnt_for(dev);

  if (tx_mine(m)) {
    // Inside an explicit bracket: the commit belongs to whoever opened it.
    if (changed && r >= 0)
      m->tx_dirty = 1;
    return r;
  }
  if (changed && r >= 0) {
    int c = schfs_vol_commit(&m->vol);
    if (c != 0)
      r = c;
  }
  kunlock(dev);
  return r;
}

// Every hook below takes the mount as its ctx. It used to be a bare device
// number, which was enough while the allocator could reach the one volume
// there was by name; with slots, the block I/O and the space manager have to
// agree about *which* volume they are serving, and the mount is the one thing
// that carries both.
static int
kd_read(void *ctx, uint64_t blk, void *out)
{
  struct schfs_kdev *d = &((struct schfs_mount *)ctx)->kdev;

  if (blk == 0)
    return -1;
  // The buffer cache takes 64-bit block numbers now, so this no longer caps the
  // volume; the check remains as an overflow guard on the multiply below.
  if (blk > (~0ULL) / SCHFS_BPB)
    return -1;

  for (int i = 0; i < SCHFS_BPB; i++) {
    struct buf *b = bread(d->dev, (uint)(blk * SCHFS_BPB + i));
    memmove((char *)out + i * BSIZE, b->data, BSIZE);
    brelse(b);
  }
  return 0;
}

static int
kd_write(void *ctx, uint64_t blk, const void *in)
{
  struct schfs_kdev *d = &((struct schfs_mount *)ctx)->kdev;

  if (blk == 0)
    return -1;
  if (blk > (~0ULL) / SCHFS_BPB)
    return -1;

  // Straight through the buffer cache, with no log. The xv6 log exists to make
  // multi-block updates atomic; copy-on-write makes them atomic by never
  // modifying a block the committed tree can reach, so logging every write
  // would double the I/O to re-solve a problem the format has already solved.
  //
  // bread before the overwrite because the cache has no "give me this block
  // without reading it" entry point. It costs a read the first time a block is
  // touched and nothing afterwards.
  //
  // bdirty, not bwrite. The B-tree seals and writes a node on every record it
  // inserts, so a leaf that is collecting one file's checksums was written to
  // the disk once per 4 KB of that file -- 1288 block writes to store 256
  // blocks of data. Leaving them dirty in the cache collapses every repeat
  // into the last one, and costs nothing in safety because kd_flush below
  // gives the format the one ordering guarantee it actually asks for.
  for (int i = 0; i < SCHFS_BPB; i++) {
    struct buf *b = bread(d->dev, (uint)(blk * SCHFS_BPB + i));
    memmove(b->data, (const char *)in + i * BSIZE, BSIZE);
    bdirty(b);
    brelse(b);
  }
  return 0;
}

// The barrier. schfs_vol_commit calls this either side of the uberblock write,
// and those two calls are the entire reason deferring above is safe.
static int
kd_flush(void *ctx)
{
  struct schfs_kdev *d = &((struct schfs_mount *)ctx)->kdev;
  return bflush(d->dev);
}

// Allocation goes through this volume's own space manager.
static uint64_t
kd_alloc(void *ctx)
{
  return schfs_vol_alloc(&((struct schfs_mount *)ctx)->vol);
}

static uint64_t
kd_alloc_run(void *ctx, uint64_t n)
{
  return schfs_vol_alloc_run(&((struct schfs_mount *)ctx)->vol, n);
}

static void
kd_free(void *ctx, uint64_t b)
{
  schfs_vol_free(&((struct schfs_mount *)ctx)->vol, b);
}

// Pages for the allocation bitmap: one per 32768 blocks of volume, taken once
// at mount and never given back.
static void *
kd_palloc(void)
{
  return kalloc();
}

// The other half of kd_palloc, and the reason unmount stopped leaking: the
// allocation map is up to SCHFS_SPACE_PAGES pages of kalloc'd memory, and
// nothing ever gave it back. A mount/unmount cycle leaked the lot.
static void
kd_pfree(void *p)
{
  kfree((char *)p);
}

// Mount the SchFS volume on block device `bdev` into slot `slot`.
//
// Never panics: a disk that is not SchFS, or is an unreadable SchFS, simply
// leaves the slot unmounted. Returns the filesystem device number on success,
// or a negative error -- distinct codes, because "it did not work" is useless
// to a caller deciding between "wrong disk" and "out of memory".
static int
kmount_slot(int slot, uint bdev)
{
  struct schfs_mount *m = &kmnt[slot];

  if (m->mounted)
    return -EBUSY;

  initsleeplock(&m->lock, "schfs");

  if (m->curblk == 0)  m->curblk  = kalloc();
  if (m->datablk == 0) m->datablk = kalloc();
  if (m->ctxa == 0)    m->ctxa    = kalloc();
  if (m->ctxb == 0)    m->ctxb    = kalloc();
  if (m->ctxc == 0)    m->ctxc    = kalloc();
  if (m->curblk == 0 || m->datablk == 0 ||
      m->ctxa == 0 || m->ctxb == 0 || m->ctxc == 0) {
    cprintf("schfs: no memory for block buffers\n");
    return -ENOMEM;
  }

  m->kdev.dev = bdev;

  struct schfs_bdev bd;
  bd.ctx        = m;
  bd.bread      = kd_read;
  bd.bwrite     = kd_write;
  bd.balloc     = kd_alloc;
  bd.bfree      = kd_free;
  bd.balloc_run = kd_alloc_run;
  bd.bflush     = kd_flush;

  struct schfs_ctx cx;
  cx.a = m->ctxa;
  cx.b = m->ctxb;
  cx.c = m->ctxc;

  // The uberblock scan doubles as the probe: no valid uberblock means this is
  // simply not a SchFS volume, which is not an error worth reporting loudly.
  // Opening with a page allocator also builds the allocation bitmap, by walking
  // the tree — the only mount-time cost SchFS has, and what buys it a mount
  // that cannot disagree with the filesystem it just adopted.
  int r = schfs_vol_open(&m->vol, &bd, &cx, kd_palloc, kd_pfree);
  if (r == -1)
    return -EINVAL;              /* not a SchFS volume at all */
  if (r == -4) {
    cprintf("schfs: device %d is larger than %d GB and cannot be mapped\n",
            bdev,
            (int)(SCHFS_MAX_BLOCKS * SCHFS_BLKSIZE / (1024 * 1024 * 1024)));
    return -EFBIG;
  }
  if (r == -5) {
    cprintf("schfs: no memory for the allocation map of device %d\n", bdev);
    return -ENOMEM;
  }
  if (r == -3) {
    // Refused on purpose, and the wording matters: the volume is fine, this
    // driver is the thing that is out of date. Older uberblocks may well be
    // readable, and mounting one would quietly roll the filesystem back to
    // before the upgrade -- so it is not offered.
    cprintf("schfs: device %d was written by a newer SchFS than this one "
            "understands; refusing rather than mounting a stale root\n", bdev);
    return -ENOTSUP;
  }
  if (r != 0) {
    cprintf("schfs: device %d has a SchFS uberblock but an unreadable tree\n",
            bdev);
    return -EIO;
  }

  m->mounted = 1;
  kinfo("schfs: mounted '%s' on device %d, %d MB, generation %d\n",
          m->vol.label, bdev,
          (int)(m->vol.total_blocks * SCHFS_BLKSIZE / (1024 * 1024)),
          (int)m->vol.generation);

  // Say so, loudly, when the mount had to reach past newer uberblocks to find
  // a root it could read. The volume is consistent -- that is what the ring is
  // for -- but transactions were lost, and a filesystem that rolls back
  // silently is one that teaches you to trust it when you should not.
  if (m->vol.recovered)
    cprintf("schfs: WARNING: %d newer transaction group%s unreadable; "
            "rolled back to generation %d\n",
            m->vol.recovered, m->vol.recovered == 1 ? " was" : "s were",
            (int)m->vol.generation);

  return SCHFS_DEV + slot;
}

// The boot mount: slot 0, which is the root filesystem. Kept as its own entry
// point because proc.c calls it before there is anything to ask.
void
schfs_kmount(uint bdev)
{
  kmount_slot(0, bdev);
}

// Mount another SchFS disk into a free slot. Returns its filesystem device
// number, or a negative errno.
int
schfs_kmount_dev(uint bdev)
{
  // Already mounted? Hand back the slot it is in rather than mounting a second
  // view of the same disk: two slots over one device would each derive their
  // own allocation map from the same blocks, and both would then hand out the
  // same free block to different files.
  for (int i = 0; i < SCHFS_VOL_SLOTS; i++)
    if (kmnt[i].mounted && kmnt[i].kdev.dev == bdev)
      return -EBUSY;

  for (int i = 1; i < SCHFS_VOL_SLOTS; i++)
    if (!kmnt[i].mounted)
      return kmount_slot(i, bdev);

  return -EBUSY;                 /* every slot in use */
}

// Release a mounted volume. The blocks it holds in the buffer cache are pushed
// out first: a slot that is reused, or a disk that is unplugged, must not leave
// a transaction half on the medium.
int
schfs_kumount(uint dev)
{
  if (!SCHFS_IS_VOL_DEV(dev))
    return -EINVAL;
  int slot = dev - SCHFS_DEV;
  if (slot == 0)
    return -EBUSY;               /* the root filesystem; nothing to fall back to */

  struct schfs_mount *m = &kmnt[slot];
  if (!m->mounted)
    return -EINVAL;

  // A snapshot of this volume outliving it would read through a mount that no
  // longer exists.
  for (int i = 0; i < SCHFS_SNAP_SLOTS; i++)
    if (ksnap_mounted[i] && ksnap_home[i] == slot)
      return -EBUSY;

  acquiresleep(&m->lock);
  bflush(m->kdev.dev);
  // Give the allocation map back. It is rebuilt from the tree on every mount,
  // so there is nothing in it worth keeping -- and at 8 KiB per GiB of volume,
  // keeping it was the largest thing this kernel leaked.
  schfs_space_release(&m->vol.space);
  m->mounted = 0;
  releasesleep(&m->lock);
  return 0;
}

// Mount a snapshot of the volume on `dev` by name, read-only. Returns the
// device number of the slot it landed in, or a negative error.
//
// The snapshot's records live in the *live* tree, so the name is resolved
// there; what comes back is a root, and the slot's volume is a copy of the live
// one with its tree pointed at it. Nothing else differs -- same device, same
// node format, same checksums.
int
schfs_ksnap_mount(uint dev, const char *name, unsigned namelen)
{
  if (!SCHFS_IS_VOL_DEV(dev) || !mounted_for(dev))
    return -1;

  int home = dev - SCHFS_DEV;
  int slot = -1;
  for (int i = 0; i < SCHFS_SNAP_SLOTS; i++)
    if (!ksnap_mounted[i]) { slot = i; break; }
  if (slot < 0)
    return -2;                    /* every slot in use */

  if (ksnapcurblk[slot] == 0)  ksnapcurblk[slot]  = kalloc();
  if (ksnapdatablk[slot] == 0) ksnapdatablk[slot] = kalloc();
  if (ksnapcurblk[slot] == 0 || ksnapdatablk[slot] == 0)
    return -3;

  klock(dev);

  struct schfs_cursor c;
  fsstat.descents++;
  schfs_cursor_init(&c, &vol_for(dev)->tree, curblk_for(dev));

  struct schfs_key k;
  schfs_mkkey(&k, SCHFS_SNAP_OBJECTID, SCHFS_KEY_SNAP, 0);

  int found = 0;
  struct schfs_snap sn;
  int r = schfs_cursor_seek(&c, &k);
  while (r == 0) {
    const struct schfs_key *got = schfs_cursor_key(&c);
    if (!got || got->objectid != SCHFS_SNAP_OBJECTID ||
        got->type != SCHFS_KEY_SNAP)
      break;

    unsigned len;
    const struct schfs_snap *p = (const struct schfs_snap *)
                                  schfs_cursor_val(&c, &len);
    if (p && len == sizeof(*p)) {
      unsigned i = 0;
      while (i < namelen && i < SCHFS_SNAP_NAME_MAX && p->name[i] == name[i])
        i++;
      if (i == namelen &&
          (namelen == SCHFS_SNAP_NAME_MAX || p->name[namelen] == 0)) {
        sn = *p;
        found = 1;
        break;
      }
    }
    r = schfs_cursor_next(&c);
  }

  if (!found) {
    kunlock(dev);
    return -4;
  }

  // Same volume, older root. The device, the space map and the block callbacks
  // are all the live volume's -- a snapshot is not a separate filesystem, it is
  // a second way into this one. That is also why the slot records which mount
  // it came from: mnt_for() sends a snapshot's device number to its live
  // volume's lock, and unmounting that volume has to refuse while it is there.
  ksnapvol[slot]               = *vol_for(dev);
  ksnapvol[slot].tree.root     = sn.root;
  ksnapvol[slot].tree.root_crc = sn.root_crc;
  ksnapvol[slot].tree.root_gen = sn.root_generation;
  // Belt and braces. Nothing can reach a snapshot through a write path --
  // writable_for() requires a live volume device number -- but the copy above
  // inherits the live volume's writability, and a flag that says the wrong
  // thing is a trap for whoever adds the next entry point.
  ksnapvol[slot].readonly      = 1;
  // Its own scratch, or a read through the snapshot would tread on a cursor
  // the live volume still has positioned.
  ksnapvol[slot].ctx.a = ksnapcurblk[slot];

  ksnap_home[slot]    = home;
  ksnap_mounted[slot] = 1;
  kunlock(dev);
  return SCHFS_SNAP_DEV + slot;
}

void
schfs_ksnap_unmount(uint dev)
{
  if (!SCHFS_IS_SNAP_DEV(dev))
    return;
  klock(dev);
  ksnap_mounted[dev - SCHFS_SNAP_DEV] = 0;
  kunlock(dev);
}

uint64
schfs_ksnap_root(uint dev)
{
  if (!SCHFS_IS_SNAP_DEV(dev) || !ksnap_mounted[dev - SCHFS_SNAP_DEV])
    return 0;
  return ksnapvol[dev - SCHFS_SNAP_DEV].root_objectid;
}

// The read path, addressable per device so a snapshot can be read through the
// same VFS. Each is the existing call with the volume chosen rather than
// assumed; the originals below stay as the live-volume spelling, because most
// callers have no business knowing there is a choice.
long
schfs_kread_dev(uint dev, uint64_t ino, uint64_t off, void *buf, unsigned n)
{
  if (!mounted_for(dev))
    return -1;
  klock(dev);
  struct schfs_cursor c;
  fsstat.descents++;
  schfs_cursor_init(&c, &vol_for(dev)->tree, curblk_for(dev));
  long r = schfs_file_read(vol_for(dev), &c, ino, off, buf, n, datablk_for(dev));
  kunlock(dev);
  return r;
}

int
schfs_kstat_dev(uint dev, uint64_t ino, struct schfs_inode *out)
{
  if (!mounted_for(dev))
    return -1;
  klock(dev);
  struct schfs_cursor c;
  fsstat.descents++;
  schfs_cursor_init(&c, &vol_for(dev)->tree, curblk_for(dev));
  int r = schfs_inode_get(vol_for(dev), &c, ino, out);
  kunlock(dev);
  return r;
}

uint64
schfs_klookup_at_dev(uint dev, uint64_t dir, const char *name, unsigned len)
{
  if (!mounted_for(dev))
    return 0;
  klock(dev);
  struct schfs_cursor c;
  fsstat.descents++;
  schfs_cursor_init(&c, &vol_for(dev)->tree, curblk_for(dev));
  uint64_t ino = 0;
  uint16_t type = 0;
  int r = schfs_dir_lookup(vol_for(dev), &c, dir, name, len, &ino, &type);
  kunlock(dev);
  return r == 0 ? ino : 0;
}

// Walk a directory from a given hash. What getdents resumes with: seeking
// straight to where the last call stopped is the difference between a listing
// that costs one descent per call and one that re-walks everything before it.
int
schfs_kreaddir_from_dev(uint dev, uint64_t dir, uint64_t start_hash,
                        schfs_dir_fn fn, void *arg)
{
  if (!mounted_for(dev))
    return -1;
  klock(dev);
  struct schfs_cursor c;
  fsstat.descents++;
  schfs_cursor_init(&c, &vol_for(dev)->tree, curblk_for(dev));
  int r = schfs_dir_iterate_from(vol_for(dev), &c, dir, start_hash, fn, arg);
  kunlock(dev);
  return r;
}

int
schfs_kreaddir_dev(uint dev, uint64_t dir,
                   int (*fn)(uint64_t, uint16_t, const char *, unsigned, void *),
                   void *arg)
{
  if (!mounted_for(dev))
    return -1;
  klock(dev);
  struct schfs_cursor c;
  fsstat.descents++;
  schfs_cursor_init(&c, &vol_for(dev)->tree, curblk_for(dev));
  int r = schfs_dir_iterate(vol_for(dev), &c, dir, fn, arg);
  kunlock(dev);
  return r;
}

// Is any SchFS volume mounted at all? Asked at boot, before there is a device
// number to name.
int
schfs_kmounted(void)
{
  return kmnt[0].mounted;
}

// Resolve a path within one volume. Returns the object id, or 0.
uint64_t
schfs_klookup(uint dev, const char *path)
{
  if (!mounted_for(dev))
    return 0;
  klock(dev);
  struct schfs_cursor c;
  fsstat.descents++;
  schfs_cursor_init(&c, &vol_for(dev)->tree, curblk_for(dev));
  uint64_t r = schfs_path_lookup(vol_for(dev), &c, path);
  kunlock(dev);
  return r;
}

struct schfs_vol *
schfs_kvol(uint dev)
{
  return mounted_for(dev) ? vol_for(dev) : 0;
}

/* ------------------------------------------------------------------ */
/* Write side                                                          */
/*                                                                     */
/* Each of these is one transaction unless the caller has a bracket     */
/* open: everything it does is invisible until it commits, and a crash  */
/* part way through leaves the volume at the state before the call.     */
/* That is the entire failure story — there is no partial state to      */
/* detect and nothing to roll back.                                     */
/*                                                                     */
/* Every one takes the filesystem device number first. They used to     */
/* name the single volume directly, which is why sys_mount had nothing  */
/* it could do with its source argument.                                */
/* ------------------------------------------------------------------ */

int
schfs_kcreate(uint dev, uint64_t dir, const char *name, unsigned namelen,
              uint16_t type, uint32_t mode, uint16_t major, uint16_t minor,
              uint64_t *out)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_create(vol_for(dev), dir, name, namelen, type, mode, major,
                       minor, know(), out);
  return ktx_end(dev, 1, r);
}

long
schfs_kwrite(uint dev, uint64_t ino, uint64_t off, const void *buf, unsigned n)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  long r = schfs_file_write(vol_for(dev), ino, off, buf, n, datablk_for(dev));
  // ktx_end speaks in error codes, and a write speaks in byte counts. Passing
  // the count straight through told it the transaction had failed with status
  // 17, which it duly reported -- and a short write is a panic one layer up.
  int e = ktx_end(dev, 1, r < 0 ? (int)r : 0);
  return e == 0 ? r : (long)e;
}

int
schfs_ktruncate(uint dev, uint64_t ino, uint64_t size)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_file_truncate(vol_for(dev), ino, size, datablk_for(dev));
  return ktx_end(dev, 1, r);
}

int
schfs_kunlink(uint dev, uint64_t dir, const char *name, unsigned namelen)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_unlink(vol_for(dev), dir, name, namelen);
  return ktx_end(dev, 1, r);
}

int
schfs_klink(uint dev, uint64_t dir, const char *name, unsigned namelen,
            uint64_t ino)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_link(vol_for(dev), dir, name, namelen, ino);
  return ktx_end(dev, 1, r);
}

int
schfs_krename(uint dev, uint64_t olddir, const char *oldname, unsigned oldlen,
              uint64_t newdir, const char *newname, unsigned newlen)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_rename(vol_for(dev), olddir, oldname, oldlen,
                       newdir, newname, newlen);
  return ktx_end(dev, 1, r);
}

// Write back an inode record the caller has modified — the one way to change
// mode, ownership or timestamps. The caller reads it with schfs_kstat_dev,
// edits the struct, and hands it back.
int
schfs_ksetattr(uint dev, uint64_t ino, const struct schfs_inode *in)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_inode_put(vol_for(dev), ino, in);
  return ktx_end(dev, 1, r);
}

// Flush without changing anything. Every operation above commits already, but
// sync(2) has to mean something -- and with writes deferred into the buffer
// cache it now means rather more than it did: an empty commit drives the same
// barrier, so anything a bracket left dirty reaches the medium.
int
schfs_ksync(uint dev)
{
  if (!writable_for(dev))
    return 0;
  ktx_begin(dev);
  return ktx_end(dev, 1, 0);
}

// Sync every mounted volume. What sys_sync wants: it names no filesystem.
int
schfs_ksync_all(void)
{
  int r = 0;
  for (int i = 0; i < SCHFS_VOL_SLOTS; i++)
    if (kmnt[i].mounted) {
      int e = schfs_ksync(SCHFS_DEV + i);
      if (e != 0)
        r = e;
    }
  return r;
}

/* ------------------------------------------------------------------ */
/* Pieces the VFS needs                                                */
/*                                                                     */
/* The VFS keeps its own link count and decides when an object dies,   */
/* so it needs the directory entry and the object as separate           */
/* operations rather than the combined create/unlink above. Two layers  */
/* counting links independently is exactly how a filesystem loses a     */
/* file that something still has open.                                  */
/* ------------------------------------------------------------------ */

// 1 if the directory holds no entries, 0 if it does, -1 on error.
int
schfs_kdirempty(uint dev, uint64_t dir)
{
  if (!mounted_for(dev))
    return -1;
  klock(dev);
  int r = schfs_dir_empty(vol_for(dev), dir);
  kunlock(dev);
  return r == -2 ? -1 : r;
}

// An object with no name and no links yet. Nothing can reach it until the
// caller links it, and nothing reaches the disk until the transaction commits,
// so a caller that gives up in between leaves nothing behind.
int
schfs_kalloc_object(uint dev, uint16_t type, uint32_t mode, uint64_t *out)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);

  struct schfs_vol *v = vol_for(dev);
  uint64_t ino = schfs_alloc_objectid(v);
  struct schfs_inode in;
  memset(&in, 0, sizeof(in));
  in.type  = type;
  in.mode  = mode;
  in.nlink = 0;
  in.atime = in.mtime = in.ctime = know();
  in.gen   = v->tree.generation;

  int r = schfs_inode_put(v, ino, &in);
  if (r == 0 && out)
    *out = ino;
  return ktx_end(dev, 1, r);
}

int
schfs_kdiradd(uint dev, uint64_t dir, const char *name, unsigned namelen,
              uint64_t ino, uint16_t type)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_dir_add(vol_for(dev), dir, name, namelen, ino, type);
  return ktx_end(dev, 1, r);
}

// Remove one directory entry, leaving the object alone. The VFS decrements the
// link count itself and calls schfs_kfree when it reaches zero.
int
schfs_kdirdel(uint dev, uint64_t dir, const char *name, unsigned namelen)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_dir_del(vol_for(dev), dir, name, namelen);
  return ktx_end(dev, 1, r);
}

// Release an object's data, checksums and inode record. Called from iput once
// the last name is gone and nothing has it open.
int
schfs_kfree(uint dev, uint64_t ino)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_object_free(vol_for(dev), ino);
  return ktx_end(dev, 1, r);
}

// The root object of one volume. SCHFS_DEV on its own still means the root
// filesystem, which is slot 0, so callers that only ever wanted that are
// unchanged.
uint64_t
schfs_kroot(uint dev)
{
  return mounted_for(dev) ? vol_for(dev)->root_objectid : 0;
}

// Share one file's data blocks with another without copying them.
//
// One transaction, because a half-finished clone would be a file holding some
// of another's blocks and some of its own -- and no caller could tell.
int
schfs_kreflink(uint dev, uint64_t src, uint64_t dst)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_file_reflink(vol_for(dev), src, dst);
  return ktx_end(dev, 1, r);
}

/* ------------------------------------------------------------------ */
/* Attributes, and the index over them                                 */
/* ------------------------------------------------------------------ */

int
schfs_kattr_set(uint dev, uint64_t ino, const char *name, unsigned namelen,
                const void *val, unsigned vallen, uint16_t type)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_attr_set(vol_for(dev), ino, name, namelen, val, vallen, type);
  return ktx_end(dev, 1, r);
}

int
schfs_kattr_get(uint dev, uint64_t ino, const char *name, unsigned namelen,
                void *out, unsigned cap, unsigned *vallen)
{
  if (!mounted_for(dev))
    return -1;
  klock(dev);
  int r = schfs_attr_get(vol_for(dev), ino, name, namelen, out, cap, vallen, 0);
  kunlock(dev);
  return r;
}

int
schfs_kattr_del(uint dev, uint64_t ino, const char *name, unsigned namelen)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_attr_del(vol_for(dev), ino, name, namelen);
  return ktx_end(dev, 1, r);
}

// Start indexing an attribute. Covers values written after this only -- there
// is no reindexing pass, and the caller is told so rather than left to assume.
int
schfs_kindex(uint dev, const char *name, unsigned namelen)
{
  if (!writable_for(dev))
    return -1;
  ktx_begin(dev);
  int r = schfs_index_create(vol_for(dev), name, namelen);
  return ktx_end(dev, 1, r);
}

int
schfs_kquery(uint dev, const char *name, unsigned namelen,
             const void *val, unsigned vallen, uint64 *out, int cap)
{
  if (!mounted_for(dev))
    return -1;
  klock(dev);
  int r = schfs_query_equal(vol_for(dev), name, namelen, val, vallen, out, cap);
  kunlock(dev);
  return r;
}

// Volume-level numbers for statfs. Returns 0 if `dev` names a mounted SchFS
// volume and fills in the three counts, or -1 if it does not.
//
// The free count is the live one the space map maintains, not the figure in the
// uberblock: they agree at every commit and differ in between, and "in between"
// is exactly when someone runs df to find out whether their last write fitted.
//
// A snapshot reports its own total but the live volume's free space, which is
// the truth -- there is one pool of blocks and the snapshot is a second view of
// it, not a second filesystem with room of its own.
int
schfs_kstatfs(uint dev, uint64 *total, uint64 *freeblocks, uint64 *objects)
{
  if (!mounted_for(dev))
    return -1;

  klock(dev);
  struct schfs_vol *v = vol_for(dev);
  // Free space comes from the *live* volume even when `dev` is a snapshot:
  // there is one pool of blocks and a snapshot is a second view of it, not a
  // filesystem with room of its own.
  struct schfs_vol *live = &mnt_for(dev)->vol;
  if (total)      *total      = v->total_blocks;
  if (freeblocks) *freeblocks = live->space.nblocks ? live->space.free_blocks
                                                    : live->free_blocks;
  // Object ids are handed out and never reused, so this is "ids consumed",
  // which is the closest honest answer to "how many files". There is no inode
  // table to be full of, and no number of free ones to report.
  if (objects)    *objects    = v->next_objectid > SCHFS_ROOT_OBJECTID
                              ? v->next_objectid - SCHFS_ROOT_OBJECTID : 0;
  kunlock(dev);
  return 0;
}

/* ------------------------------------------------------------------ */
/* /proc/schfs                                                         */
/*                                                                     */
/* Phase 2 has no VFS integration -- SchFS files are not reachable      */
/* through open(2) yet -- so this is how the mount is observable from   */
/* userland at all. It reports the volume and lists the root directory, */
/* which is enough to confirm from inside a running SchOS that the    */
/* kernel really did parse an image mkschfs wrote.                      */
/* ------------------------------------------------------------------ */

static void
kp_puts(char *buf, int cap, int *len, const char *s)
{
  while (*s && *len < cap - 1)
    buf[(*len)++] = *s++;
}

static void
kp_putu(char *buf, int cap, int *len, uint64 v)
{
  char tmp[24];
  int  i = 0;
  if (v == 0) { if (*len < cap - 1) buf[(*len)++] = '0'; return; }
  while (v && i < (int)sizeof(tmp)) { tmp[i++] = '0' + (v % 10); v /= 10; }
  while (i--) { if (*len < cap - 1) buf[(*len)++] = tmp[i]; }
}

struct kp_ctx {
  char *buf;
  int   cap;
  int  *len;
  int   count;
};

static int
kp_entry(uint64_t ino, uint16_t type, const char *name, unsigned namelen,
         void *arg)
{
  struct kp_ctx *c = arg;
  c->count++;
  // Only the first handful, and no per-entry stat: schfs_kreaddir holds the
  // lock across the walk, so calling schfs_kstat from here would deadlock.
  if (c->count > 16)
    return 0;
  kp_puts(c->buf, c->cap, c->len, "  ");
  kp_puts(c->buf, c->cap, c->len,
          type == SCHFS_T_DIR ? "dir     " :
          type == SCHFS_T_SYMLINK ? "symlink " :
          type == SCHFS_T_SOCK    ? "socket  " :
          type == SCHFS_T_DEV ? "dev     " : "file    ");
  for (unsigned i = 0; i < namelen && *c->len < c->cap - 1; i++)
    c->buf[(*c->len)++] = name[i];
  kp_puts(c->buf, c->cap, c->len, "  #");
  kp_putu(c->buf, c->cap, c->len, ino);
  kp_puts(c->buf, c->cap, c->len, "\n");
  return 0;
}

// One volume's section of /proc/schfs.
static void
kinfo_one(char *buf, int cap, int *len, uint dev)
{
  struct schfs_mount *m = &kmnt[dev - SCHFS_DEV];

  kp_puts(buf, cap, len, "\nfsdev:         ");
  kp_putu(buf, cap, len, dev);
  kp_puts(buf, cap, len, "\nblockdev:      ");
  kp_putu(buf, cap, len, m->kdev.dev);
  kp_puts(buf, cap, len, "\nlabel:         ");
  kp_puts(buf, cap, len, m->vol.label);
  kp_puts(buf, cap, len, "\nblocksize:     ");
  kp_putu(buf, cap, len, SCHFS_BLKSIZE);
  kp_puts(buf, cap, len, "\ntotal_blocks:  ");
  kp_putu(buf, cap, len, m->vol.total_blocks);
  kp_puts(buf, cap, len, "\nfree_blocks:   ");
  kp_putu(buf, cap, len, m->vol.space.nblocks ? m->vol.space.free_blocks
                                              : m->vol.free_blocks);
  kp_puts(buf, cap, len, "\ngeneration:    ");
  kp_putu(buf, cap, len, m->vol.generation);
  kp_puts(buf, cap, len, "\ntree_root:     ");
  kp_putu(buf, cap, len, m->vol.tree.root);
  kp_puts(buf, cap, len, "\nroot_objectid: ");
  kp_putu(buf, cap, len, m->vol.root_objectid);
  if (m->vol.recovered) {
    // Reported here as well as on the console, because the console message
    // scrolled past hours ago and this is the thing anyone looks at afterwards.
    kp_puts(buf, cap, len, "\nrecovered:     ");
    kp_putu(buf, cap, len, (uint64)m->vol.recovered);
    kp_puts(buf, cap, len, " newer transaction group(s) were unreadable");
  }
  kp_puts(buf, cap, len, "\n\nroot directory:\n");

  struct kp_ctx c = { buf, cap, len, 0 };
  if (schfs_kreaddir_dev(dev, m->vol.root_objectid, kp_entry, &c) != 0) {
    kp_puts(buf, cap, len, "  (directory read failed)\n");
    return;
  }
  if (c.count > 16) {
    kp_puts(buf, cap, len, "  ... ");
    kp_putu(buf, cap, len, (uint64)(c.count - 16));
    kp_puts(buf, cap, len, " more\n");
  }
}

int
schfs_kinfo(char *buf, int cap)
{
  int len = 0;
  int any = 0;

  for (int i = 0; i < SCHFS_VOL_SLOTS; i++)
    if (kmnt[i].mounted) {
      kinfo_one(buf, cap, &len, SCHFS_DEV + i);
      any = 1;
    }

  for (int i = 0; i < SCHFS_SNAP_SLOTS; i++)
    if (ksnap_mounted[i]) {
      kp_puts(buf, cap, &len, "\nsnapshot fsdev ");
      kp_putu(buf, cap, &len, SCHFS_SNAP_DEV + i);
      kp_puts(buf, cap, &len, " of fsdev ");
      kp_putu(buf, cap, &len, SCHFS_DEV + ksnap_home[i]);
      kp_puts(buf, cap, &len, ", generation ");
      kp_putu(buf, cap, &len, ksnapvol[i].tree.root_gen);
      kp_puts(buf, cap, &len, "\n");
    }

  if (!any)
    kp_puts(buf, cap, &len, "no SchFS volume mounted\n");
  return len;
}