SchOS/kernel/fs/schfs/vfs.c

// SPDX-License-Identifier: Apache-2.0
//
// SchFS behind the VFS: the fs_ops table that makes SchFS files reachable
// through open(2), read(2), write(2) and the rest.
//
// Everything below is translation, not filesystem logic. The filesystem lives
// in btree.c/space.c/vol.c/obj.c/objw.c and is driven through the schfs_k*
// entry points in kmount.c, each of which is one transaction. This file's whole
// job is turning a `struct inode` into an object id and back.
//
// ---------------------------------------------------------------------------
// The two models, and where they disagree
//
// The VFS inherited xv6's shape: allocate an inode, fill it in, then link a
// name to it. SchFS has no free-standing inode — an object exists because a
// record with its id exists — so schfs_ialloc writes an inode record with
// nlink 0 and no name at all. Nothing can reach it until dirlink adds an entry.
// If the caller dies in between, the record is unreferenced but harmless, and
// it never reaches the disk anyway: nothing is durable until the transaction
// that created it commits.
//
// The other disagreement is `nlink`. The VFS adjusts it itself (create sets it
// to 1, link bumps it, unlink drops it) and then calls iupdate. So the object
// layer's own link accounting is deliberately bypassed here: schfs_dir_add and
// schfs_dir_del are used rather than schfs_link and schfs_unlink, and iput's
// ifree hook is what releases an object once the VFS has decided it has no
// names left. Two pieces of code counting links independently is precisely how
// a filesystem loses a file that is still referenced.

#include "types.h"
#include "x86.h"
#include "defs.h"
#include "param.h"
#include "stat.h"
#include "spinlock.h"
#include "proc.h"          /* `proc` — copyin/copyout need the current pgdir */
#include "sleeplock.h"
#include "fs.h"
#include "buf.h"
#include "file.h"
#include "errno.h"
#include "fsstat.h"

#include "schfs_int.h"

// The VFS talks in T_*; the disk format uses SCHFS_T_*. Neither is going to
// change to suit the other, so the mapping lives here, once, in both
// directions rather than as an assumption that the numbers happen to line up.
static short
type_to_vfs(uint16_t t)
{
  switch (t) {
  case SCHFS_T_DIR:     return T_DIR;
  case SCHFS_T_DEV:     return T_DEV;
  case SCHFS_T_SYMLINK: return T_SYMLINK;
  case SCHFS_T_SOCK:    return T_SOCK;
  default:              return T_FILE;
  }
}

static uint16_t
type_from_vfs(short t)
{
  switch (t) {
  case T_DIR:     return SCHFS_T_DIR;
  case T_DEV:     return SCHFS_T_DEV;
  case T_SYMLINK: return SCHFS_T_SYMLINK;
  case T_SOCK:    return SCHFS_T_SOCK;
  default:        return SCHFS_T_FILE;
  }
}

/* ------------------------------------------------------------------ */
/* Inode load / store / free                                           */
/* ------------------------------------------------------------------ */

static void
schfs_vfs_iload(struct inode *ip)
{
  struct schfs_inode in;

  if (schfs_kstat_dev(ip->dev, ip->inum, &in) != 0) {
    // An unreadable inode on the root filesystem is a panic in v1 because the
    // system cannot continue without it. Here it is a mounted volume that may
    // simply be damaged, so report an empty file and let the caller fail
    // normally rather than taking the machine down with it.
    ip->type  = T_FILE;
    ip->nlink = 0;
    ip->size  = 0;
    ip->valid = 1;
    return;
  }

  ip->type      = type_to_vfs(in.type);
  ip->major     = (short)in.rdev_major;
  ip->minor     = (short)in.rdev_minor;
  // Clamped, not cast. The on-disk count is a uint32 and the VFS's is a signed
  // int, so the top half of the range has nowhere to go; saturating keeps it
  // positive, and a file with two billion names is not going to lose one here.
  ip->nlink     = in.nlink > 0x7fffffffu ? 0x7fffffff : (int)in.nlink;
  ip->size      = in.size;
  ip->rwmode    = RW;
  ip->supermode = 0;
  ip->showmode  = 1;
  ip->atime     = (uint)in.atime;
  ip->mtime     = (uint)in.mtime;
  ip->ctime     = (uint)in.ctime;
  memset(ip->addrs, 0, sizeof(ip->addrs));
  ip->valid = 1;
}

// Everything below this point mutates, and a snapshot is a view of a tree that
// has already been committed -- there is nowhere for a change to go. Refusing
// here rather than in each schfs_k* call keeps the check next to the reason.
static int
readonly(const struct inode *ip)
{
  return SCHFS_IS_SNAP_DEV(ip->dev);
}

static void
schfs_vfs_istore(struct inode *ip)
{
  struct schfs_inode in;

  if (readonly(ip))
    return;

  // Read-modify-write rather than build-from-scratch: the on-disk record holds
  // fields the VFS has no slot for (uid, gid, the real mode bits, the
  // generation), and rebuilding it here would quietly zero every one of them.
  if (schfs_kstat_dev(ip->dev, ip->inum, &in) != 0) {
    memset(&in, 0, sizeof(in));
    in.mode = ip->type == T_DIR ? 0755 : 0644;
  }

  in.type       = type_from_vfs(ip->type);
  in.rdev_major = (uint16_t)ip->major;
  in.rdev_minor = (uint16_t)ip->minor;
  in.nlink      = (uint32_t)ip->nlink;
  in.size       = ip->size;
  in.atime      = ip->atime;
  in.mtime      = ip->mtime;
  in.ctime      = ip->ctime;

  schfs_ksetattr(ip->dev, ip->inum, &in);
}

static void
schfs_vfs_ifree(struct inode *ip)
{
  if (readonly(ip))
    return;
  schfs_kfree(ip->dev, ip->inum);
}

/* ------------------------------------------------------------------ */
/* Data                                                                */
/* ------------------------------------------------------------------ */

// Reads and writes are staged through a bounce buffer rather than going
// straight to user memory, because the filesystem takes a sleeplock and walks
// the tree while a copyout could fault. One page at a time keeps that buffer
// off the 4096-byte kernel stack -- it is kalloc'd -- and bounds how long the
// volume lock is held.
//
// One filesystem block, which is also one kalloc page: this said 2048 while
// the comment above it said one page, so half of every bounce buffer went
// unused and every read did twice the work it needed to.
//
// The chunk size is not a copy-size tuning knob. Each trip through the loop
// takes the volume sleeplock and builds a fresh B-tree cursor to find the
// extent for that offset, so this is really "how many tree descents per byte
// read". Measured on an 834 KB file: 417 descents at 2048, 209 at 4096.
//
// It must not exceed a page, because the buffer is a single kalloc().
#define BOUNCE SCHFS_BLKSIZE

static int
schfs_vfs_read(struct inode *ip, int user_dst, uint64 dst, uint64 off, uint n)
{
  uint64 t0 = rdtsc();
  fsstat.read_calls++;

  char *tmp = kalloc();
  if (tmp == 0)
    return -1;

  uint done = 0;
  while (done < n) {
    uint want = n - done;
    if (want > BOUNCE)
      want = BOUNCE;

    long got = schfs_kread_dev(ip->dev, ip->inum, off + done, tmp, want);
    if (got < 0) { kfree(tmp); return done ? (int)done : -1; }
    if (got == 0) break;                 /* end of file */

    if (user_dst) {
      if (copyout(myproc()->pgdir, dst + done, tmp, (uint)got) < 0)
        break;
    } else {
      memmove((char *)(dst + done), tmp, (uint)got);
    }
    done += (uint)got;
    if ((uint)got < want)
      break;                             /* short read: end of file */
  }

  kfree(tmp);
  fsstat.read_bytes  += done;
  fsstat.read_cycles += rdtsc() - t0;
  return (int)done;
}

static int
schfs_vfs_write(struct inode *ip, int user_src, uint64 src, uint64 off, uint n)
{
  if (readonly(ip))
    return -EROFS;

  char *tmp = kalloc();
  if (tmp == 0)
    return -1;

  // One transaction for the whole call, not one per 4 KB chunk.
  //
  // Each schfs_kwrite is its own group otherwise, and a group ends in two cache
  // flushes and an uberblock write. A flush is a barrier, so committing per
  // chunk forbids the buffer cache to coalesce the tree writes that every chunk
  // repeats — the same leaf, holding this file's data checksums, went to the
  // disk once per chunk. Measured over 1 MB: 1410 block writes per chunk-commit
  // against 267 for one group. It is also better semantics, since a short write
  // now leaves the file at a length some prefix of this call actually reached
  // rather than at whichever chunk happened to commit last.
  //
  // copyin under the bracket is safe: it walks the page table and returns -1 on
  // an unmapped address rather than faulting, and a page fault here would be a
  // SIGSEGV in any case — there is no demand paging to re-enter the filesystem.
  schfs_ktx_open(ip->dev);

  uint done = 0;
  while (done < n) {
    uint want = n - done;
    if (want > BOUNCE)
      want = BOUNCE;

    if (user_src) {
      if (copyin(myproc()->pgdir, (addr_t)(src + done), tmp, (uint64)want) < 0)
        break;
    } else {
      memmove(tmp, (char *)(src + done), want);
    }

    long put = schfs_kwrite(ip->dev, ip->inum, off + done, tmp, want);
    if (put <= 0) break;
    done += (uint)put;
    if ((uint)put < want) break;
  }

  int cr = schfs_ktx_close(ip->dev);

  kfree(tmp);
  // A failed commit means nothing durably happened, so report it that way
  // rather than claiming bytes that are not on the disk.
  if (cr != 0)
    return -1;
  if (off + done > ip->size)
    ip->size = off + done;
  return (int)done;
}

static void
schfs_vfs_truncate(struct inode *ip, uint64 newsize)
{
  if (readonly(ip))
    return;
  if (schfs_ktruncate(ip->dev, ip->inum, newsize) == 0)
    ip->size = newsize;
}

/* ------------------------------------------------------------------ */
/* Directories                                                         */
/* ------------------------------------------------------------------ */

static struct inode *
schfs_vfs_dirlookup(struct inode *dp, char *name, uint *poff)
{
  // "." and ".." are not stored: a directory costs nothing until it has real
  // entries. Both are answered from what the VFS already knows, except ".."
  // at the mount root, which the mount layer redirects before we get here.
  if (name[0] == '.' && name[1] == '\0') {
    if (poff) *poff = 0;
    return iget(dp->dev, dp->inum);
  }

  unsigned namelen = 0;
  while (name[namelen]) namelen++;

  if (namelen == 2 && name[0] == '.' && name[1] == '.') {
    struct schfs_inode in;
    if (schfs_kstat_dev(dp->dev, dp->inum, &in) != 0)
      return 0;
    if (poff) *poff = 0;
    return iget(dp->dev, in.parent ? in.parent : dp->inum);
  }

  uint64 ino = schfs_klookup_at_dev(dp->dev, dp->inum, name, namelen);
  if (ino == 0)
    return 0;
  // poff is v1's byte offset into the directory file. SchFS finds an entry by
  // name, so there is nothing meaningful to hand back; dirunlink below takes
  // the name and ignores it.
  if (poff) *poff = 0;
  return iget(dp->dev, ino);
}

static int
schfs_vfs_dirlink(struct inode *dp, char *name, uint64 inum)
{
  if (readonly(dp))
    return -EROFS;

  unsigned namelen = 0;
  while (name[namelen]) namelen++;
  if (namelen == 0 || namelen > SCHFS_NAME_MAX)
    return -1;

  struct schfs_inode in;
  if (schfs_kstat_dev(dp->dev, inum, &in) != 0)
    return -1;

  int r = schfs_kdiradd(dp->dev, dp->inum, name, namelen, inum, in.type);
  if (r != 0)
    return -1;

  // A subdirectory's ".." is synthesised from its parent field, so moving or
  // creating one has to set that field or the new directory reports the wrong
  // parent -- and every ".." walk out of it lands somewhere else entirely.
  if (in.type == SCHFS_T_DIR && in.parent != dp->inum) {
    in.parent = dp->inum;
    schfs_ksetattr(dp->dev, inum, &in);
  }
  return 0;
}

static int
schfs_vfs_dirunlink(struct inode *dp, char *name, uint off)
{
  if (readonly(dp))
    return -EROFS;

  (void)off;
  unsigned namelen = 0;
  while (name[namelen]) namelen++;
  return schfs_kunlink(dp->dev, dp->inum, name, namelen) == 0 ? 0 : -1;
}

static int
schfs_vfs_dirempty(struct inode *dp)
{
  return schfs_kdirempty(dp->dev, dp->inum);
}

static struct inode *
schfs_vfs_ialloc(uint dev, short type)
{
  uint64 ino = 0;
  if (schfs_kalloc_object(dev, type_from_vfs(type),
                          type == T_DIR ? 0755 : 0644, &ino) != 0)
    return 0;
  return iget(dev, ino);
}

/* ------------------------------------------------------------------ */
/* getdents64                                                          */
/* ------------------------------------------------------------------ */

// One record's worth of state for the callback below.
//
// ---------------------------------------------------------------------------
// The resume cookie, and why it is a key rather than a count
//
// getdents is called repeatedly until it returns nothing, and each call has to
// carry on where the last stopped. `*poff` is the only state there is, so
// whatever goes in it must be enough to find that place again.
//
// It used to be an ordinal -- "entry N of the walk" -- and resuming meant
// walking from the first entry and skipping N. That is quadratic in the size of
// the directory, and measurably so: listing /bin, 243 entries, walked 914
// entries across five calls, because each call started over. A larger directory
// gets worse faster than it gets bigger.
//
// It is now the tree key to seek to. Entries are keyed (dir, DIRENT, hash), so
// the cookie is a hash and resuming is one descent to that hash. The three
// values below `SCHFS_DIRENT_FIRST` are reserved for the synthesised "." and
// "..", which are not records and would otherwise have no position at all.
//
// One consequence worth stating: a whole hash bucket is emitted or none of it
// is. Buckets hold colliding names, at most SCHFS_MAX_INLINE packed, which is
// under 3 KB of output records -- so any caller with a 4 KB buffer never
// notices. A caller whose buffer cannot hold one bucket gets EINVAL, which is
// what Linux does when a buffer cannot hold a single record.
#define GD_DOT       0            /* emit "." next */
#define GD_DOTDOT    1            /* emit ".." next */
#define GD_ENTRIES   2            /* emit real entries, from hash (cookie - 2) */

struct gd_ctx {
  char   *out;
  int     cap;
  int     written;
  uint64  from;             // hash to resume at
  uint64  next;             // hash the *following* call should resume at
  int     full;             // buffer filled; stop the walk
  int     toobig;           // a single bucket did not fit in an empty buffer
  uint64  bucket;           // hash of the bucket being emitted
  int     bucket_started;
};

static int
gd_emit(struct gd_ctx *g, uint64 ino, unsigned short type,
        const char *name, unsigned namelen)
{
  if (namelen > 255)
    namelen = 255;
  int reclen = (8 + 8 + 2 + 1 + (int)namelen + 1 + 7) & ~7;

  if (g->written + reclen > g->cap)
    return -1;

  char *k = g->out + g->written;
  uint64 *p64 = (uint64 *)k;
  p64[0] = ino;
  p64[1] = g->next;
  *(uint16 *)(k + 16) = (uint16)reclen;
  *(uint8  *)(k + 18) = type == SCHFS_T_DIR     ? 4    /* DT_DIR  */
                      : type == SCHFS_T_DEV     ? 2    /* DT_CHR  */
                      : type == SCHFS_T_SYMLINK ? 10   /* DT_LNK  */
                      : type == SCHFS_T_SOCK    ? 12   /* DT_SOCK */
                      :                           8;   /* DT_REG  */
  memmove(k + 19, name, namelen);
  memset(k + 19 + namelen, 0, reclen - 19 - namelen);
  g->written += reclen;
  fsstat.getdents_walked++;   // entries emitted; no longer entries re-walked
  return 0;
}

static int
gd_entry(uint64 ino, unsigned short type, const char *name, unsigned namelen,
         uint64 hash, void *arg)
{
  struct gd_ctx *g = arg;

  if (g->full)
    return 1;

  // A new bucket. Everything emitted so far is safe to keep, because the
  // cookie already points past the previous bucket.
  if (!g->bucket_started || hash != g->bucket) {
    g->bucket = hash;
    g->bucket_started = 1;
    g->next = hash + GD_ENTRIES;      // resume *at* this bucket if we stop here
  }

  // The cookie the emitted records carry has to be "past this bucket", or a
  // caller that stops reading mid-buffer would come back and get the bucket
  // again. Setting it before the emit is what makes that true.
  uint64 saved = g->next;
  g->next = hash + 1 + GD_ENTRIES;

  if (gd_emit(g, ino, type, name, namelen) < 0) {
    // Did not fit. Roll back to the start of this bucket so the whole thing is
    // re-read next time, and stop.
    g->next = saved;
    g->full = 1;
    if (g->written == 0)
      g->toobig = 1;
    return 1;
  }
  return 0;
}

static int
schfs_vfs_getdents(struct inode *dp, char *out, int cap, uint64 *poff)
{
  uint64 t0 = rdtsc();
  fsstat.getdents_calls++;

  struct gd_ctx g;
  g.out            = out;
  g.cap            = cap;
  g.written        = 0;
  g.from           = *poff;
  g.next           = *poff;
  g.full           = 0;
  g.toobig         = 0;
  g.bucket         = 0;
  g.bucket_started = 0;

  // "." and ".." are synthesised: they are not records, so a walk would never
  // produce them, and every readdir caller expects both.
  if (g.from <= GD_DOTDOT) {
    static const char *dots[2] = { ".", ".." };
    for (int i = (int)g.from; i < 2; i++) {
      g.next = (uint64)i + 1;
      if (gd_emit(&g, dp->inum, SCHFS_T_DIR, dots[i], (unsigned)(i + 1)) < 0) {
        if (g.written == 0)
          return -EINVAL;
        *poff = g.next;
        return g.written;
      }
    }
    g.next = GD_ENTRIES;              // real entries start at hash 0
    g.from = GD_ENTRIES;
  }

  uint64 start_hash = g.from - GD_ENTRIES;

  if (schfs_kreaddir_from_dev(dp->dev, dp->inum, start_hash, gd_entry, &g) != 0
      && g.written == 0)
    return -1;

  if (g.toobig)
    return -EINVAL;

  // Nothing emitted and the walk ran to the end: this is the end of the
  // directory, and the cookie must not move or the caller loops.
  if (!g.full && g.written == 0) {
    *poff = g.next;
    fsstat.getdents_cycles += rdtsc() - t0;
    return 0;
  }

  *poff = g.next;
  fsstat.getdents_cycles += rdtsc() - t0;
  return g.written;
}

/* ------------------------------------------------------------------ */

// One table, not two. Every entry already takes an inode and every inode
// carries its device, so the same functions serve the live volume and a mounted
// snapshot -- the read path picks the volume, the write path refuses. A second
// table would be the same pointers with a different name on it.
const struct fs_ops schfs_ops = {
  .name      = "schfs",
  .iload     = schfs_vfs_iload,
  .istore    = schfs_vfs_istore,
  .ifree     = schfs_vfs_ifree,
  .read      = schfs_vfs_read,
  .write     = schfs_vfs_write,
  .truncate  = schfs_vfs_truncate,
  .dirlookup = schfs_vfs_dirlookup,
  .dirlink   = schfs_vfs_dirlink,
  .dirunlink = schfs_vfs_dirunlink,
  .dirempty  = schfs_vfs_dirempty,
  .ialloc    = schfs_vfs_ialloc,
  .getdents  = schfs_vfs_getdents,
};