// SchFS object-layer test: files, directories and space accounting. // // schfs_test covers the B-tree. This covers everything built on it — create, // write at an offset, truncate, unlink, rename, hard links — and, most // importantly, that space comes back. A filesystem that loses a block on every // delete passes every functional test ever written and is still useless. // // Everything runs against a memory-backed volume, so a full create/write/ // delete/remount cycle takes milliseconds. The kernel gets this code only once // it passes here, because the same failure inside a VM costs a rebuild and a // boot to see once. // // Build and run: make schfs-objtest #define SCHFS_HOSTED 1 #include #include #include #include "../kernel/fs/schfs/btree.c" #include "../kernel/fs/schfs/space.c" #include "../kernel/fs/schfs/vol.c" #include "../kernel/fs/schfs/obj.c" #include "../kernel/fs/schfs/objw.c" /* ------------------------------------------------------------------ */ /* Memory-backed device */ /* ------------------------------------------------------------------ */ #define NBLOCKS 4096 static unsigned char *DISK; static struct schfs_vol VOL; static char CTX_A[SCHFS_BLKSIZE]; static char CTX_B[SCHFS_BLKSIZE]; static char CTX_C[SCHFS_BLKSIZE]; static struct schfs_ctx CTX = { CTX_A, CTX_B, CTX_C }; static char CURBLK[SCHFS_BLKSIZE]; static char DATABLK[SCHFS_BLKSIZE]; static int md_read(void *ctx, uint64_t b, void *buf) { (void)ctx; if (b >= NBLOCKS) return -1; memcpy(buf, DISK + b * SCHFS_BLKSIZE, SCHFS_BLKSIZE); return 0; } static int md_write(void *ctx, uint64_t b, const void *buf) { (void)ctx; if (b >= NBLOCKS) return -1; memcpy(DISK + b * SCHFS_BLKSIZE, buf, SCHFS_BLKSIZE); return 0; } static uint64_t md_alloc(void *ctx) { (void)ctx; return schfs_vol_alloc(&VOL); } static uint64_t md_run(void *ctx, uint64_t n) { (void)ctx; return schfs_vol_alloc_run(&VOL, n); } static void md_free(void *ctx, uint64_t b) { (void)ctx; schfs_vol_free(&VOL, b); } static void *md_palloc(void) { return calloc(1, SCHFS_BLKSIZE); } static void mkbdev(struct schfs_bdev *bd) { bd->ctx = 0; bd->bread = md_read; bd->bwrite = md_write; bd->balloc = md_alloc; bd->bfree = md_free; bd->balloc_run = md_run; } /* ------------------------------------------------------------------ */ static int failures; static void ok(int cond, const char *what) { if (cond) { printf(" [ok] %s\n", what); } else { printf(" FAIL: %s\n", what); failures++; } } static uint32_t rng = 987654321; static uint32_t rnd(void) { rng ^= rng << 13; rng ^= rng >> 17; rng ^= rng << 5; return rng; } // Read a whole file into buf, returning its length or -1. static long readall(uint64_t ino, char *buf, unsigned cap) { struct schfs_cursor c; schfs_cursor_init(&c, &VOL.tree, CURBLK); struct schfs_inode in; if (schfs_inode_get(&VOL, &c, ino, &in) != 0) return -1; if (in.size > cap) return -1; unsigned off = 0; while (off < in.size) { long got = schfs_file_read(&VOL, &c, ino, off, buf + off, (unsigned)(in.size - off), DATABLK); if (got <= 0) return -1; off += (unsigned)got; } return (long)off; } static uint64_t lookup(const char *path) { struct schfs_cursor c; schfs_cursor_init(&c, &VOL.tree, CURBLK); return schfs_path_lookup(&VOL, &c, path); } static uint32_t nlink_of(uint64_t ino) { struct schfs_cursor c; schfs_cursor_init(&c, &VOL.tree, CURBLK); struct schfs_inode in; if (schfs_inode_get(&VOL, &c, ino, &in) != 0) return 0; return in.nlink; } static uint64_t size_of(uint64_t ino) { struct schfs_cursor c; schfs_cursor_init(&c, &VOL.tree, CURBLK); struct schfs_inode in; if (schfs_inode_get(&VOL, &c, ino, &in) != 0) return ~0ULL; return in.size; } static int count_extents_of(uint64_t ino) { struct schfs_cursor c; schfs_cursor_init(&c, &VOL.tree, CURBLK); struct schfs_key start; schfs_mkkey(&start, ino, SCHFS_KEY_EXTENT, 0); if (schfs_cursor_seek(&c, &start) != 0) return 0; int n = 0; for (;;) { const struct schfs_key *k = schfs_cursor_key(&c); if (!k || k->objectid != ino || k->type != SCHFS_KEY_EXTENT) break; n++; if (schfs_cursor_next(&c) != 0) break; } return n; } // Publish, then reopen the volume from its uberblock — which rebuilds the // allocation map by walking the tree. Returns the free count the writer had, so // a caller can hold it against the one the mount derived: a bitmap that has // drifted from its own counter shows up here and nowhere else. // // Reopening also ages every block into a previous transaction group, which is // what makes the copy-on-write paths run at all. Within one group the tree and // the data both rewrite in place, so a test that never commits exercises the // cheap half of the filesystem and none of the careful half. static uint64_t remount(void) { if (schfs_vol_commit(&VOL) != 0) { printf(" FAIL: commit before remount\n"); failures++; return ~0ULL; } uint64_t before = VOL.free_blocks; struct schfs_bdev bd2; mkbdev(&bd2); // Straight back into VOL, not into a local that is then copied over it: a // schfs_vol points at its own dev field, so a copy of one leaves the tree // talking to whichever struct it was built in. memset(&VOL, 0, sizeof(VOL)); if (schfs_vol_open(&VOL, &bd2, &CTX, md_palloc) != 0) { printf(" FAIL: remount\n"); failures++; return ~0ULL; } return before; } /* ------------------------------------------------------------------ */ #define ROOT (VOL.root_objectid) static uint64_t mkfile(uint64_t dir, const char *name) { uint64_t ino = 0; int r = schfs_create(&VOL, dir, name, (unsigned)strlen(name), SCHFS_T_FILE, 0644, 0, 0, 1000, &ino); if (r != 0) { printf(" FAIL: create %s -> %d\n", name, r); failures++; return 0; } return ino; } static uint64_t mkdirent(uint64_t dir, const char *name) { uint64_t ino = 0; int r = schfs_create(&VOL, dir, name, (unsigned)strlen(name), SCHFS_T_DIR, 0755, 0, 0, 1000, &ino); if (r != 0) { printf(" FAIL: mkdir %s -> %d\n", name, r); failures++; return 0; } return ino; } int main(void) { struct schfs_bdev bd; DISK = calloc(NBLOCKS, SCHFS_BLKSIZE); if (!DISK) { perror("calloc"); return 1; } mkbdev(&bd); printf("SchFS object-layer test\n\n"); if (schfs_vol_format(&VOL, &bd, NBLOCKS, "objtest", &CTX, md_palloc) != 0) { fprintf(stderr, "FAIL: format\n"); return 1; } // The root directory is an object like any other; nothing points at it // except the uberblock's root_objectid. { struct schfs_inode root; memset(&root, 0, sizeof(root)); root.type = SCHFS_T_DIR; root.mode = 0755; root.nlink = 2; if (schfs_inode_put(&VOL, ROOT, &root) != 0) { fprintf(stderr, "FAIL: root inode\n"); return 1; } } if (schfs_vol_commit(&VOL) != 0) { fprintf(stderr, "FAIL: commit\n"); return 1; } uint64_t baseline_free = VOL.free_blocks; /* ---- 1. create, write, read back ---- */ { uint64_t d = mkdirent(ROOT, "dir"); uint64_t f = mkfile(d, "hello"); const char *msg = "the quick brown fox"; long w = schfs_file_write(&VOL, f, 0, msg, (unsigned)strlen(msg), DATABLK); ok(w == (long)strlen(msg), "write returns the byte count"); char buf[128]; long got = readall(f, buf, sizeof(buf)); ok(got == (long)strlen(msg) && memcmp(buf, msg, (size_t)got) == 0, "a short file reads back exactly"); ok(lookup("/dir/hello") == f, "path lookup finds it"); ok(nlink_of(d) == 2, "a new directory has nlink 2"); ok(nlink_of(ROOT) == 3, "creating a subdirectory bumps the parent"); } /* ---- 2. a large file becomes few extents, not one per block ---- */ { uint64_t f = mkfile(ROOT, "big"); static char src[64 * 1024]; for (unsigned i = 0; i < sizeof(src); i++) src[i] = (char)(i * 7 + (i >> 8)); // Written in small pieces on purpose: this is the shape that used to // produce one extent record per block, and one copy-on-write per call. unsigned off = 0; while (off < sizeof(src)) { unsigned n = 100 + rnd() % 900; if (off + n > sizeof(src)) n = (unsigned)sizeof(src) - off; long w = schfs_file_write(&VOL, f, off, src + off, n, DATABLK); if (w != (long)n) { printf(" FAIL: partial write %ld\n", w); failures++; break; } off += n; } static char back[64 * 1024]; long got = readall(f, back, sizeof(back)); ok(got == (long)sizeof(src) && memcmp(back, src, sizeof(src)) == 0, "64 KB written in small pieces reads back identical"); int ne = count_extents_of(f); ok(ne <= 4, "sequential writes merge into a handful of extents"); if (ne > 4) printf(" (%d extents for 16 blocks)\n", ne); } /* ---- 3. overwriting the middle, and holes ---- */ { uint64_t f = mkfile(ROOT, "patch"); static char src[16 * 1024]; memset(src, 'a', sizeof(src)); schfs_file_write(&VOL, f, 0, src, sizeof(src), DATABLK); // Straddles a block boundary, so it exercises the split path on both // sides of the extent it lands in. char patch[600]; memset(patch, 'Z', sizeof(patch)); schfs_file_write(&VOL, f, 4000, patch, sizeof(patch), DATABLK); memcpy(src + 4000, patch, sizeof(patch)); static char back[16 * 1024]; long got = readall(f, back, sizeof(back)); ok(got == (long)sizeof(src) && memcmp(back, src, sizeof(src)) == 0, "an overwrite across a block boundary lands exactly"); // Now the same thing across a commit. Everything above happened inside // one transaction group, where a data block may be rewritten in place; // once it is committed it belongs to a tree a crash can still land on, // so an overwrite has to copy the block and split the extent record // around it. That is the path that actually risks losing data, and only // a commit makes it run. if (schfs_vol_commit(&VOL) != 0) { printf(" FAIL: commit\n"); failures++; } char patch2[3000]; memset(patch2, 'K', sizeof(patch2)); long w = schfs_file_write(&VOL, f, 9000, patch2, sizeof(patch2), DATABLK); ok(w == (long)sizeof(patch2), "an overwrite after a commit succeeds"); memcpy(src + 9000, patch2, sizeof(patch2)); got = readall(f, back, sizeof(back)); ok(got == (long)sizeof(src) && memcmp(back, src, sizeof(src)) == 0, "copy-on-write of a committed block keeps the rest of the run intact"); // And the block it replaced must really be gone, not still in the map. uint64_t was = remount(); ok(was == VOL.free_blocks, "free space after a copy-on-write overwrite survives a remount"); got = readall(lookup("/patch"), back, sizeof(back)); ok(got == (long)sizeof(src) && memcmp(back, src, sizeof(src)) == 0, "and the patched file still reads correctly after remounting"); uint64_t h = mkfile(ROOT, "sparse"); schfs_file_write(&VOL, h, 40000, "end", 3, DATABLK); ok(size_of(h) == 40003, "writing past the end extends the file"); static char hb[64 * 1024]; got = readall(h, hb, sizeof(hb)); int zeros = (got == 40003); for (long i = 0; i < 40000 && zeros; i++) if (hb[i] != 0) zeros = 0; ok(zeros && memcmp(hb + 40000, "end", 3) == 0, "the gap left behind is a hole and reads as zeros"); } /* ---- 4. truncate ---- */ { uint64_t f = mkfile(ROOT, "trunc"); static char src[20 * 1024]; memset(src, 'q', sizeof(src)); schfs_file_write(&VOL, f, 0, src, sizeof(src), DATABLK); ok(schfs_file_truncate(&VOL, f, 100, DATABLK) == 0, "truncate succeeds"); ok(size_of(f) == 100, "truncate sets the size"); char back[256]; long got = readall(f, back, sizeof(back)); int good = (got == 100); for (int i = 0; i < 100 && good; i++) if (back[i] != 'q') good = 0; ok(good, "what survives a truncate is the original head"); // Growing again must read as zeros, not as whatever the released // blocks used to hold. ok(schfs_file_truncate(&VOL, f, 8192, DATABLK) == 0, "truncate can grow"); static char big[8192]; got = readall(f, big, sizeof(big)); int zeroed = (got == 8192); for (int i = 100; i < 8192 && zeroed; i++) if (big[i] != 0) zeroed = 0; ok(zeroed, "growing past the old end reads as zeros, not stale data"); } /* ---- 5. links, unlink, rename, rmdir ---- */ { uint64_t f = mkfile(ROOT, "orig"); schfs_file_write(&VOL, f, 0, "content", 7, DATABLK); ok(schfs_link(&VOL, ROOT, "alias", 5, f) == 0, "hard link succeeds"); ok(nlink_of(f) == 2, "hard link bumps nlink"); ok(lookup("/alias") == f, "both names reach the same object"); ok(schfs_unlink(&VOL, ROOT, "orig", 4) == 0, "unlink one name"); ok(nlink_of(f) == 1, "nlink drops but the object survives"); char buf[32]; ok(readall(f, buf, sizeof(buf)) == 7, "the surviving name still reads"); ok(schfs_rename(&VOL, ROOT, "alias", 5, ROOT, "renamed", 7) == 0, "rename within a directory"); ok(lookup("/alias") == 0 && lookup("/renamed") == f, "the old name is gone and the new one resolves"); uint64_t d = lookup("/dir"); ok(schfs_unlink(&VOL, ROOT, "dir", 3) == -5, "rmdir refuses a directory that still has entries"); ok(schfs_unlink(&VOL, d, "hello", 5) == 0, "emptying it first"); ok(schfs_unlink(&VOL, ROOT, "dir", 3) == 0, "then rmdir succeeds"); ok(nlink_of(ROOT) == 2, "removing a subdirectory drops the parent's count"); // Moving a directory under itself would detach the whole subtree. uint64_t a = mkdirent(ROOT, "a"); uint64_t b = mkdirent(a, "b"); (void)b; ok(schfs_rename(&VOL, ROOT, "a", 1, lookup("/a/b"), "loop", 4) != 0, "renaming a directory into its own subtree is refused"); } /* ---- 6. it all survives a commit and a remount ---- */ { if (schfs_vol_commit(&VOL) != 0) { printf(" FAIL: commit\n"); failures++; } static char back_before[64 * 1024]; long before = readall(lookup("/big"), back_before, sizeof(back_before)); uint64_t was_free = remount(); ok(was_free != ~0ULL, "the volume reopens from its uberblock"); static char back_after[64 * 1024]; long after = readall(lookup("/big"), back_after, sizeof(back_after)); ok(before == after && before > 0 && memcmp(back_before, back_after, (size_t)before) == 0, "file contents are identical after a remount"); ok(lookup("/renamed") != 0 && lookup("/dir") == 0, "the namespace is exactly as it was left"); ok(VOL.free_blocks == was_free, "free space derived at mount matches what the writer had"); if (VOL.free_blocks != was_free) printf(" (mount says %llu, writer said %llu)\n", (unsigned long long)VOL.free_blocks, (unsigned long long)was_free); } /* ---- 7. deleting everything gives the space back ---- * * The one test that catches a leak. Every earlier check would still pass on * a filesystem that never reused a block; this one does not. */ { const char *names[] = { "big", "patch", "sparse", "trunc", "renamed" }; for (unsigned i = 0; i < sizeof(names) / sizeof(names[0]); i++) { int r = schfs_unlink(&VOL, ROOT, names[i], (unsigned)strlen(names[i])); if (r != 0) { printf(" FAIL: unlink %s -> %d\n", names[i], r); failures++; } } int r = schfs_unlink(&VOL, lookup("/a"), "b", 1); if (r == 0) r = schfs_unlink(&VOL, ROOT, "a", 1); if (r != 0) { printf(" FAIL: rmdir a/b -> %d\n", r); failures++; } if (schfs_vol_commit(&VOL) != 0) { printf(" FAIL: commit\n"); failures++; } ok(VOL.free_blocks == baseline_free, "an emptied filesystem is back to its starting free-space count"); if (VOL.free_blocks != baseline_free) printf(" (%llu free now, %llu when empty: %lld blocks lost)\n", (unsigned long long)VOL.free_blocks, (unsigned long long)baseline_free, (long long)baseline_free - (long long)VOL.free_blocks); ok(schfs_btree_check(&VOL.tree) == 0, "the tree is still structurally sound after all of that"); } /* ---- 8. fill, empty, repeat: the space has to be genuinely reusable ---- * * The free *count* coming back is not the same as the free *map* coming * back, and a remount hides the difference -- it throws the map away and * derives a correct one from the tree. Only spending the space again, on * the same mount, holds the two against each other. A map that leaks a * little on every delete stores less on each round until nothing fits. */ { static char chunk[32 * 1024]; memset(chunk, 'x', sizeof(chunk)); uint64_t first_round = 0; int good = 1; for (int round = 0; round < 3 && good; round++) { uint64_t f = mkfile(ROOT, "hog"); uint64_t off = 0; int saw_enospc = 0; for (int i = 0; i < 10000; i++) { long w = schfs_file_write(&VOL, f, off, chunk, sizeof(chunk), DATABLK); if (w == -3) { saw_enospc = 1; break; } if (w != (long)sizeof(chunk)) { printf(" FAIL: write returned %ld\n", w); good = 0; break; } off += sizeof(chunk); } if (round == 0) { ok(saw_enospc, "filling the volume reports out of space rather than corrupting"); first_round = off; } else if (off + 128 * 1024 < first_round) { printf(" FAIL: round %d stored %llu KB, round 0 stored %llu KB\n", round, (unsigned long long)(off / 1024), (unsigned long long)(first_round / 1024)); good = 0; } int r = schfs_unlink(&VOL, ROOT, "hog", 3); if (r != 0) { printf(" FAIL: could not delete on a full volume (%d)\n", r); good = 0; } if (schfs_vol_commit(&VOL) != 0) { printf(" FAIL: commit\n"); good = 0; } } ok(good, "filling and emptying three times reuses the same space"); ok(schfs_btree_check(&VOL.tree) == 0, "the tree survives being filled to the brim"); } /* ---- 8b. snapshots ---- */ { // A snapshot is only worth anything if the data it names survives the // live tree changing underneath it. So: write, snapshot, overwrite, // and then read *through the snapshot's root* and expect the old bytes. int r; uint64_t ino = mkfile(VOL.root_objectid, "snapfile"); ok(ino != 0, "created a file to snapshot"); const char before[] = "BEFORE-THE-SNAPSHOT"; schfs_file_write(&VOL, ino, 0, before, sizeof before, DATABLK); if (schfs_vol_commit(&VOL) != 0) { printf(" FAIL: commit\n"); failures++; } // The snapshot names the committed root, which is the state just // written -- not the root this call produces. r = schfs_snap_create(&VOL, 1, "first", 5); ok(r == 0, "took a snapshot"); if (schfs_vol_commit(&VOL) != 0) { printf(" FAIL: commit\n"); failures++; } struct schfs_cursor c; schfs_cursor_init(&c, &VOL.tree, CURBLK); struct schfs_snap sn; ok(schfs_snap_get(&VOL, &c, 1, &sn) == 0, "the snapshot record reads back"); // Change the live tree: overwrite the file completely. const char after[] = "AFTER-THE-SNAPSHOT!"; schfs_file_write(&VOL, ino, 0, after, sizeof after, DATABLK); if (schfs_vol_commit(&VOL) != 0) { printf(" FAIL: commit\n"); failures++; } char buf[64]; memset(buf, 0, sizeof buf); schfs_cursor_init(&c, &VOL.tree, CURBLK); schfs_file_read(&VOL, &c, ino, 0, buf, sizeof after, DATABLK); ok(memcmp(buf, after, sizeof after) == 0, "the live tree shows the new contents"); // Read through the retained root. Same volume, same block device, a // tree pointed at the old root -- which is all a snapshot is. struct schfs_vol snapvol = VOL; snapvol.tree.root = sn.root; snapvol.tree.root_crc = sn.root_crc; snapvol.tree.root_gen = sn.root_generation; memset(buf, 0, sizeof buf); schfs_cursor_init(&c, &snapvol.tree, CURBLK); long got = schfs_file_read(&snapvol, &c, ino, 0, buf, sizeof before, DATABLK); ok(got == (long)sizeof before && memcmp(buf, before, sizeof before) == 0, "the snapshot still shows the OLD contents"); if (memcmp(buf, before, sizeof before) != 0) printf(" (snapshot read back \"%s\")\n", buf); // And the blocks it holds must not be considered free, or they get // handed out and the snapshot rots silently. // Exact agreement is not the invariant once a snapshot exists, and the // number is the only thing that drifts. Allocation works from the // in-use bitmap, which the derivation filled by walking the snapshot // too, so a snapshot's blocks are never handed out. The running // counter is not corrected for them, so it and the derivation disagree // until the next mount. Cosmetic, and reported rather than asserted // away. uint64_t claimed = remount(); if (claimed != VOL.free_blocks) printf(" (note: counter %llu, derived %llu -- the counter " "drifts while a snapshot exists)\n", (unsigned long long)claimed, (unsigned long long)VOL.free_blocks); // Deleting the file from the live tree must not disturb the snapshot. schfs_dir_del(&VOL, VOL.root_objectid, "snapfile", 8); schfs_object_free(&VOL, ino); if (schfs_vol_commit(&VOL) != 0) { printf(" FAIL: commit\n"); failures++; } memset(buf, 0, sizeof buf); snapvol = VOL; snapvol.tree.root = sn.root; snapvol.tree.root_crc = sn.root_crc; snapvol.tree.root_gen = sn.root_generation; schfs_cursor_init(&c, &snapvol.tree, CURBLK); got = schfs_file_read(&snapvol, &c, ino, 0, buf, sizeof before, DATABLK); ok(got == (long)sizeof before && memcmp(buf, before, sizeof before) == 0, "a file deleted from the live tree is still in the snapshot"); // Drop it again, so what it was holding becomes reclaimable and the // volume goes back to exact accounting for the checks that follow. ok(schfs_snap_delete(&VOL, 1) == 0, "the snapshot can be deleted"); if (schfs_vol_commit(&VOL) != 0) { printf(" FAIL: commit\n"); failures++; } schfs_cursor_init(&c, &VOL.tree, CURBLK); ok(schfs_snap_get(&VOL, &c, 1, &sn) != 0, "a deleted snapshot is gone"); // Deleting it makes what it held unreachable, and the derivation at // the next mount is what reclaims it -- there is no refcount to // decrement. So the derived figure must be at least what the stale // counter believed; check 9 below then proves exact agreement is back. claimed = remount(); ok(VOL.free_blocks >= claimed, "space the snapshot held is reclaimed at the next mount"); } /* ---- 9. the map and its counter still agree ---- */ { uint64_t claimed = remount(); ok(claimed == VOL.free_blocks, "free space counted in memory matches free space derived from the tree"); if (claimed != VOL.free_blocks) printf(" (counter said %llu, the tree says %llu)\n", (unsigned long long)claimed, (unsigned long long)VOL.free_blocks); } printf("\n%s\n", failures ? "FAILED" : "all checks passed"); return failures ? 1 : 0; }