diff --git a/Makefile b/Makefile index d975eb5..36cabf4 100644 --- a/Makefile +++ b/Makefile @@ -545,7 +545,7 @@ $(shell mkdir -p $(BUILD); \ # ----------------------------------------------------------------------- # The root filesystem is an Alpine root that apk manages, with SchOS installed -# into it as the package schos-base -- see docs/plan/alpine.md and "The +# into it as the package schos-base -- see docs/architecture/packages.md and "The # apk-managed root" below. Both images are made from it: schfs.img, which # `make qemu` boots, and the ramdisk inside the kernel, which `make iso` # boots. sysroot/ is what schos-base is packaged from, not a root of its own. @@ -1321,7 +1321,7 @@ crashqr: $(PYTHON) tools/python/crashqr.py # ----------------------------------------------------------------------- -# LDLS -- the Linux Driver Layer. docs/plan/ldls.md. +# LDLS -- the Linux Driver Layer. docs/architecture/ldls.md. # # ldls/ko.c is freestanding and builds on both sides: here as part of a host # tool, and later as part of the SchOS-side host. Same discipline as SchFS, @@ -1346,7 +1346,7 @@ $(LDLS_RUN): ldls/host/run.c ldls/ko.c ldls/stub.c ldls/btf.c \ $(HOSTCC) $(HOSTWARN) -O1 -fno-strict-aliasing -o $@ \ ldls/host/run.c ldls/ko.c ldls/stub.c ldls/btf.c -# The test object, compiled and never linked. Ours, because docs/plan/ldls.md +# The test object, compiled and never linked. Ours, because docs/architecture/ldls.md # rule 1 means no .ko is committed here -- see the header of refmod.c. # # -fno-pic matters: a PIC object routes its data through the GOT, and a GOT is @@ -1369,7 +1369,7 @@ ldls: $(LDLS_INSPECT) $(LDLS_RUN) $(LDLS_TYPES) # # make ldls-stage KO=/path/to/8139too.ko BTF=/sys/kernel/btf/vmlinux # -# This is the model docs/plan/ldls.md describes, carried out on the machine +# This is the model docs/architecture/ldls.md describes, carried out on the machine # that is doing the testing: the user obtains a module and loads it on their # own machine. Deliberately **not** a dependency of `all`, so an ordinary build # never picks it up, and `.gitignore` refuses `*.ko` so it cannot be committed diff --git a/README.md b/README.md index 87f9dfc..d848a2d 100644 --- a/README.md +++ b/README.md @@ -4,48 +4,50 @@ # SchOS -**A 64-bit operating system, written from the metal up.** +**A 64-bit operating system, written from the metal up — and it runs Linux's software.** -SMP kernel · copy-on-write filesystem · Linux syscall ABI · Alpine packages · X11 on real hardware +SMP kernel · copy-on-write filesystem · Linux syscall ABI · Alpine packages · Xorg · Linux's own drivers --- -SchOS began as [xv6](https://github.com/mit-pdos/xv6-public) and is now mostly -something else. The scheduler, the memory manager and the trap path have been -rewritten for x86-64 and for more than one processor; the filesystem has been -replaced outright; the syscall table speaks Linux numbering, so real Linux -software runs on it unmodified — statically or dynamically linked, straight -out of Alpine's package repositories. +SchOS is a hobby operating system built by one person. It began as +[xv6](https://github.com/mit-pdos/xv6-public) and almost nothing of that is left: +the scheduler, the memory manager, the trap path and every driver have been +rewritten for x86-64 and for more than one processor, and the filesystem is a +different one. The syscall table speaks Linux's numbering, so **real Linux +programs run on it unmodified** — statically or dynamically linked, straight out +of Alpine's package repositories — and a desktop (Xorg and Window Maker) and a +web browser (NetSurf) come up on the framebuffer. -It is a hobby system built by one person, and it is honest about that: what -works below has been *run*, not inferred, and the gaps are listed as plainly as -the features. +It is honest about being a hobby system: what is listed below has been *run*, +not inferred, and [the gaps](#not-there-yet) are listed as plainly as the +features. It boots from a USB stick on three different real machines. ## In numbers | | | |---|---| -| **First-party code** | 61,367 lines, plus 21,128 lines of comments (26%) in 272 files | -| **Kernel** (`kernel/`, `include/`) | 45,207 lines of code, 16,111 of comments | -| **Syscalls** | 208, of which 184 on Linux numbering | +| **First-party code** | 72,766 lines, plus 23,668 lines of comments (25%) in 307 files | +| **Kernel** (`kernel/`, `include/`) | 51,893 lines of code, 17,213 of comments | +| **Syscalls** | 215, of which 193 on Linux numbering | | **Drivers** | 15 in the kernel | | **First-party programs** | 29 in `cmd/` | -| **Alpine packages in the root** | 88, from 11 asked for in `ports/alpine/world` | -| **History** | 253 commits since 2026-08-09 (76 days) | +| **Alpine packages in the root** | 179, from 29 asked for in `ports/alpine/world` | +| **History** | 289 commits since 2026-08-09 (78 days) |
Lines by language | Language | Code | Comments | |---|--:|--:| -| C | 56,580 | 19,427 | +| C | 67,615 | 21,904 | | Assembly | 1,750 | 336 | | PowerShell | 1,362 | 638 | -| Python | 759 | 147 | -| Shell | 716 | 469 | +| Python | 943 | 150 | +| Shell | 896 | 529 | | Perl | 129 | 45 | | Linker script | 71 | 66 | @@ -55,78 +57,74 @@ the features. ports/ and generated tables are not counted. -## The pieces +## What it is made of | Layer | Name | What it is | |---|---|---| | System | **SchOS** | The distribution — kernel, userland, tooling | -| Kernel | **SchIFT** | Multi-core x86-64 kernel with a modular VFS | +| Kernel | **SchIFT** | Multi-core x86-64 kernel with a modular VFS and Linux's syscall ABI | | Filesystem | **SchFS** | Transactional, copy-on-write, the root filesystem | | Image format | **SCFF** | The SchOS Compressed File Format the live root ships in | -| API | **SchAPI** | System programming interface — the stable surface | -| Compositor | **SchCom** | Display compositor *(planned)* | -| Window manager | **SchWM** | *(planned)* | -| Desktop | **SchDE** | *(planned)* | +| Packages | **apk** | Alpine's package manager manages the root; SchOS itself is the package `schos-base` | +| Desktop | **Xorg + Window Maker** | Alpine's, on the framebuffer | +| Crash screen | **PSOGM / GMMOS** | The Purple Screen of Guru Meditation, and the micro OS that runs under it | +| API | **SchAPI** | The long-term native interface — *designed, not built* | `uname -s` reports `SchIFT`, the way every Unix reports its kernel rather than its distribution. ## What works -**Kernel** — Long mode, 4-level paging, preemptive multitasking. **SMP** up to -64 processors, enumerated from the ACPI MADT. ACPI through -[uACPI](https://github.com/uACPI/uACPI): interrupt routing from `_PRT`, the -power button, clean power-off and reset. Memory sized from the E820/UEFI map. +**Kernel** — Long mode, 4-level paging, preemptive multitasking, a fair-share +scheduler with `nice`. **SMP** up to 64 processors, enumerated from the ACPI +MADT. **Threads** (`clone`, `futex`, TLB shootdowns), **demand paging** for the +huge reservations browsers make, and dynamic linking. ACPI through +[uACPI](https://github.com/uACPI/uACPI): interrupt routing, the power button, +clean power-off and reset, batteries and AC through a laptop's embedded +controller, and deep idle states. Memory sized from the firmware's map, up to 256 GB. -**Filesystem** — **SchFS** is the root: a copy-on-write B-tree with a -uberblock ring, so a crash leaves either the old tree or the new one and there -is nothing to repair. The live image ships as **[SCFF](docs/architecture/scff.md)**: -the 358 MB root compresses to 102 MB with zstd and is unpacked a chunk at a -time, the first time something reads it. `/proc` has `/proc//fd`, -`/proc/self` and the rest that `ps`, `top` and Xorg read. +**Filesystem** — **SchFS** is the root: a copy-on-write B-tree with an uberblock +ring, so a crash leaves either the old tree or the new one and there is nothing +to repair. The live image ships as **[SCFF](docs/architecture/scff.md)**: the +root compresses with zstd to about a third of its size and is unpacked a chunk at +a time, the first time something reads it. `/proc`, `/sys` and `/dev` are +synthesised, in Linux's layouts, because that is what the programs read. -**Processes** — fork, exec, wait, pipes, signals, sessions and job control, -`mmap` (anonymous and file-backed), POSIX shared memory, `memfd`, Unix domain -sockets with descriptor passing (`SCM_RIGHTS`), `poll`, `epoll`, `eventfd`, -`timerfd`, `signalfd`. **Dynamic linking**: `exec` loads `ld-musl`, so -binaries come from Alpine as they are. +**Linux software** — `fork`, `clone`, `exec`, pipes, signals, sessions and job +control, `mmap`, POSIX and `memfd` shared memory, Unix sockets with descriptor and +credential passing, `poll`, `epoll`, `eventfd`, `timerfd`, `signalfd`, `inotify`, +pseudo-terminals. The root is managed by Alpine's **apk**: `ports/alpine/world` +says what is installed, `world.lock` pins it, and `apk add` fetches more over HTTPS. -**Packages** — the root is managed by Alpine's **apk**. `ports/alpine/world` -says what is installed; the build resolves it once into `world.lock` and every -image after that is reproducible. SchOS itself is one more package in it. +**Desktop** — **Xorg with Window Maker**, `xterm`, the classic X clients, SDL games +and the NetSurf browser, on the framebuffer (`/dev/fb0`, Linux fbdev) with input +as Linux evdev. Twelve virtual terminals, switched with Ctrl+Alt+F1–F12, with X on +a terminal of its own. Sound through ALSA (over OSS and an AC'97 driver). -**Graphics** — **Xorg with Window Maker**, preinstalled, on the framebuffer -(`/dev/fb0`, Linux fbdev). Six virtual terminals, switched with -Ctrl+Alt+F1–F6 or Alt+F1–F6, with X on a terminal of its own; Ctrl+Alt+Backspace -ends the session. Input as Linux evdev on `/dev/input/event*`. - -**Devices** — PCI with a bus-agnostic device model at `/proc/devices`. +**Drivers** — written from the specifications, and **Linux's own, unmodified, +through [LDLS](docs/architecture/ldls.md)**: a `.ko` runs in an ordinary process +and reaches the hardware through `/dev/pci`, so a crashed driver costs an interface, +not the machine. The laptop's Realtek NIC is driven that way. | | | |---|---| | Storage | AHCI (SATA), NVMe, IDE, USB mass storage | -| Network | Intel e1000, Realtek r8169 and RTL8139 | +| Network | Intel e1000, Realtek RTL8139 and r8169 (the latter also as Linux's driver in a process) | | USB | xHCI, hubs, HID keyboards and mice | -| Input | PS/2, I2C-HID laptop touchpads (DesignWare I2C, found through ACPI) | +| Input | PS/2, I2C-HID laptop touchpads and keyboards (DesignWare I2C, found through ACPI) | | Display | UEFI GOP framebuffer, virtio-gpu, Bochs/stdvga | | Audio | AC'97 | -**Networking** — ARP, IPv4, ICMP, UDP, TCP and DHCP on lwIP. DNS, `wget` -over **HTTPS**, `ping`, and an `httpd` serving a directory back. +**Networking** — ARP, IPv4, ICMP, UDP, TCP and DHCP on lwIP; DNS, `wget` and `curl` +over **HTTPS**, `ping`, `httpd`, netlink for `getifaddrs`. -**Userland** — musl, toybox, mksh and zsh, plus whatever Alpine has: `ls`, -`ps`, `top`, `awk`, `diff`, `xterm`, `fastfetch` behave the way you expect, -because they are the real programs. +**When it dies** — PSOGM, the Purple Screen of Guru Meditation: stop code, fault +address, call chain, and the log as QR codes. Under it runs **GMMOS**, a micro OS with +its own keyboard drivers, so a dead machine can still show its log, take a few +commands, and restart or power off. -**Text** — a TrueType rasteriser (`glyf` outlines, composite glyphs, cmap -format 4). JetBrains Mono for the terminal, Roboto for the desktop to come. - -**When it dies** — PSOGM, the Purple Screen of Guru Meditation: stop code, -fault address, call chain, and a QR code containing a text copy of the whole -report. - -**On real hardware** — booted on an AMD B550 desktop (Ryzen 5 4600G), an ASUS -TUF A15 laptop and an Ivy Bridge HP all-in-one, from a USB stick. +**On real hardware** — booted from a USB stick on an AMD B550 desktop (Ryzen 5 +4600G), an ASUS TUF A15 laptop and an Ivy Bridge HP all-in-one. ## Quick start @@ -134,19 +132,19 @@ TUF A15 laptop and an Ivy Bridge HP all-in-one, from a USB stick. make # kernel + UEFI loader + root filesystem make qemu # build it and boot it make iso # a bootable ISO for VMware or real hardware +./write.sh sdX # write that ISO to a USB stick (Linux; it checks what it is overwriting) ``` -The ISO is a hybrid image: write it to a USB stick with -`dd if=schos.iso of=/dev/sdX bs=4M` (or Rufus/Etcher in DD mode) and boot it -on any UEFI machine. +The ISO is a hybrid image: it boots on any UEFI machine from a stick written with +`write.sh`, `dd if=schos.iso of=/dev/sdX bs=4M`, or Rufus/Etcher in DD mode. **It runs +from RAM and touches none of the machine's disks.** Log in on the console, and type +`startx` for the desktop. Builds on **Linux, macOS, FreeBSD/OpenBSD/NetBSD and Windows** (Git Bash). The toolchain is clang + lld everywhere — it is a cross-compiler by construction, so one LLVM install targets the bare-metal kernel, the 16-bit boot code and musl -userland from any host. See **[docs/guides/building.md](docs/guides/building.md)** for -prerequisites per platform, and -**[docs/history/moving-to-linux.md](docs/history/moving-to-linux.md)** for moving a checkout -between machines. +userland from any host; `nix-shell` supplies all of it. See +**[docs/guides/building.md](docs/guides/building.md)** for prerequisites per platform. ## Testing @@ -160,28 +158,35 @@ tools/windows/bootcheck.ps1 -Tty -Ps # terminal behaviour, process enumeration make schfs-check # SchFS B-tree, object and image suites ``` -`bootcheck` is PowerShell because `pwsh` runs on every supported host. - -The numbers at the top are measured, not typed: `python3 tools/python/stats.py` -rewrites them here and on the website. +`bootcheck` is PowerShell because `pwsh` runs on every supported host. Several +programs in `cmd/` (`socktest`, `memfdtest`, `xready`, …) exist to be run *on* the +system and report which of their own checks failed. See +**[docs/guides/testing.md](docs/guides/testing.md)** for what each proves. The numbers at +the top are measured, not typed: `python3 tools/python/stats.py` rewrites them here +and on the website. ## Layout ``` -kernel/ arch/ core/ dev/ drivers/ fs/ net/ proc/ sys/ +kernel/ arch/ core/ dev/ drivers/ fs/ net/ proc/ sys/ acpi/ gmmos/ scff/ fs/schfs/ the filesystem -- same source the host tools compile +ldls/ the Linux driver layer: the .ko loader, BTF reader, Linux symbols include/ kernel and userland headers, by subsystem (arch/ core/ fs/ dev/ ...) boot/ uefi/ (PE loader) common/ (shared loader UI) lib/ cmd/ first-party userland runtime and programs -ports/ third-party software built from source +etc/ what the image starts with: schinit's job manifests, motd, ... +ports/ third-party software built from source; ports/alpine is the apk root tools/ windows/ (.ps1) posix/ (.sh) python/ (.py) and .c/.pl +projects/ things that run ON SchOS (an installer, SchFS for Linux: both frozen) docs/ architecture/ (what it is) plan/ (what's next) guides/ (how to work on it) history/ (why it's like this) + Website/ (the site, and the docs reader) ``` **[docs/README.md](docs/README.md)** is the index. If you are reading the tree for the first time, [docs/architecture/README.md](docs/architecture/README.md) -is the system layer by layer. +is the system layer by layer, and [docs/plan/1.0.md](docs/plan/1.0.md) says +where it stands against its finish line. One rule worth knowing: **the kernel and the host tools compile the same SchFS source**. `mkschfs` cannot disagree with the kernel about the on-disk format, @@ -193,19 +198,24 @@ Another: **syscalls are declared in exactly one place**, ## Not there yet -- **No threads.** `clone` makes processes, not threads (`CLONE_VM` is - refused), so multi-threaded programs run on one thread or not at all. Xorg - runs with `-dumbSched` for this reason. +- **No installer, and nothing persists.** The live image runs from RAM and + writes nothing to the machine's disks, so what you install or save is gone at + reboot. This is the one thing between the system and its 1.0 definition. +- **No GPU acceleration.** Xorg draws in software; the plan is Linux's `amdgpu` + through LDLS ([docs/plan/gpu.md](docs/plan/gpu.md)). There is no IOMMU either, + so a driver in a process is isolated against its bugs, not against malice. - **No udev.** Xorg is told its input devices by a config file instead of - finding them; `/sys` and the device-event socket are what is missing. + finding them; `/sys` has no device tree and no uevent socket. +- **WebKit browsers do not start.** NetSurf does; Epiphany's web process dies at + start-up, and it has not been run to ground. - **USB 2.0 controllers (EHCI)** have no driver. On older machines the ports wired only to EHCI do nothing; xHCI ports work. - **Touchpads run in mouse mode**: pointer, click and tap, but no two-finger scroll yet. - **No page cache**, so writable `MAP_SHARED` of a file is refused rather than - faked. SysV shared memory is missing too. -- **No installer.** SchOS runs live from the USB stick; nothing is written to - the machine's disks. + faked, and `mprotect` is accepted and ignored (nothing has NX). +- **No Wi-Fi, no HDA audio**, and the Realtek NIC on the laptop has reached the + point of finding its PHY; whether it carries traffic there is not confirmed. - SchFS has no Linux driver yet; the plan is on-disk format version first, then FUSE, then a kernel module. - CFF/OpenType outlines and TrueType hinting are deliberately out of scope. @@ -215,14 +225,15 @@ Another: **syscalls are declared in exactly one place**, **GPL-2.0-or-later**, copyright Nexoniarz. See [LICENSE](LICENSE). The kernel derives from xv6, which is MIT — see [LICENSE.MIT](LICENSE.MIT). -Third-party code redistributed in a built image (musl, toybox, mksh, zsh, kilo, -the fonts) keeps its own licence; [NOTICE](NOTICE) carries the attributions. -See [docs/licensing.md](docs/licensing.md). +Third-party code redistributed in a built image (musl, toybox, mksh, zstd, the +fonts, and Alpine's packages, each under its own licence) keeps that licence; +[NOTICE](NOTICE) carries the attributions. See [docs/licensing.md](docs/licensing.md). Drivers may be adapted from BSD sources and, since the move to the GPL, from Linux — GPL-2.0-only code combines with GPL-2.0-or-later, and the result ships -under GPLv2. Best of all is still the public specification — xHCI, virtio, -AHCI, e1000 — which carries no licence on what you write from it. +under GPLv2; Linux's own drivers can also be run unmodified through LDLS. Best of +all is still the public specification — xHCI, virtio, AHCI, e1000 — which carries +no licence on what you write from it. --- diff --git a/cmd/memfdtest/main.c b/cmd/memfdtest/main.c index 75b721d..ebdd60b 100644 --- a/cmd/memfdtest/main.c +++ b/cmd/memfdtest/main.c @@ -2,7 +2,7 @@ // // memfdtest -- two processes map the same memfd, one writes, the other sees it. // -// That sentence is docs/plan/foundations.md's own acceptance criterion for +// That sentence is docs/architecture/kernel.md's own acceptance criterion for // item 2, and it is the check a permissive implementation cannot pass: an // mmap that quietly returns private zeroed pages satisfies every return code // here and fails the one assertion that matters, because the reader is a diff --git a/cmd/pcitest/main.c b/cmd/pcitest/main.c index 78a5387..aac0002 100644 --- a/cmd/pcitest/main.c +++ b/cmd/pcitest/main.c @@ -2,7 +2,7 @@ // // pcitest -- prove that a driver could live out here. // -// docs/plan/hybrid.md mechanisms 2 and 4, exercised end to end from a process: +// docs/architecture/devices.md mechanisms 2 and 4, exercised end to end from a process: // enumerate the bus, claim a device, map its registers, read them, and get // memory the device could be pointed at. // diff --git a/cmd/ping/main.c b/cmd/ping/main.c index 3d18ffa..4d74733 100644 --- a/cmd/ping/main.c +++ b/cmd/ping/main.c @@ -6,7 +6,7 @@ // worth recording because it is not "ours is better". toybox's ping reads its // replies with recvmsg(2) and asks for the TTL through an IP_RECVTTL control // message, and SchOS has neither: no recvmsg, no ancillary data. Both are owed -// -- recvmsg is item 1 of docs/plan/foundations.md, where it is wanted for +// -- recvmsg is item 1 of docs/architecture/kernel.md, where it is wanted for // SCM_RIGHTS -- and neither should be rushed to make one applet run. // // So this uses sendto/recvfrom, which the kernel does have, and proves the ICMP diff --git a/cmd/schmode/main.c b/cmd/schmode/main.c index b308ddc..508ae82 100644 --- a/cmd/schmode/main.c +++ b/cmd/schmode/main.c @@ -2,7 +2,7 @@ // // schmode -- change the display mode. // -// docs/plan/hybrid.md milestone H3. The interesting thing about this program +// docs/architecture/devices.md milestone H3. The interesting thing about this program // is how little it is: it opens a socket to `vgpud` and writes a line. No // syscall it makes has heard of a GPU, no kernel code runs that knows what a // scanout is, and if the driver on the other end gets the sequence wrong the diff --git a/cmd/socktest/main.c b/cmd/socktest/main.c index f026818..de5f777 100644 --- a/cmd/socktest/main.c +++ b/cmd/socktest/main.c @@ -447,7 +447,7 @@ strneq_(const char *a, const char *b, int n) // --------------------------------------------------------------------------- // 8. SCM_RIGHTS: a descriptor crossing between processes. // -// docs/plan/foundations.md item 1, docs/plan/hybrid.md mechanism 1. The thing +// docs/architecture/kernel.md item 1, docs/architecture/devices.md mechanism 1. The thing // being proved is not that recvmsg returns a number -- it is that the number // names the *same open file* the other process had, which is why every check // below moves data through the descriptor rather than inspecting it. @@ -763,7 +763,7 @@ test_fdpass_nodata(void) // --------------------------------------------------------------------------- // 15. A bound socket is a file. // -// docs/plan/foundations.md item 3, in its own words: bind() a socket, then +// docs/architecture/kernel.md item 3, in its own words: bind() a socket, then // stat() and `ls -l` the path and see an `s`, then unlink() it. // // The reason it matters is not tidiness. A Wayland client finds its compositor diff --git a/cmd/vgpud/main.c b/cmd/vgpud/main.c index 31a2f5e..99ee71a 100644 --- a/cmd/vgpud/main.c +++ b/cmd/vgpud/main.c @@ -2,7 +2,7 @@ // // vgpud -- the virtio-gpu driver, outside the kernel. // -// docs/plan/hybrid.md's first eviction. The same device the kernel drives in +// docs/architecture/devices.md's first eviction. The same device the kernel drives in // kernel/drivers/gpu/virtio_gpu/, driven instead by a process that asks the // kernel for four things and nothing else: // @@ -648,7 +648,7 @@ present(void) // // This used to be a test card: three colour bands, four gradients, a // checkerboard, and this instance's pid in binary along the bottom edge. That -// was scaffolding for hybrid.md's H2 and H3 -- a screenshot had to answer "did +// was scaffolding for the out-of-kernel driver work -- a screenshot had to answer "did // a *userspace process* put this here", and "did the driver come back" needed // a second run to look different from a first that never stopped. Both // milestones are closed, and what was left was a garish test pattern on a @@ -866,7 +866,7 @@ register_panic_fb(void) // --- the modeset -------------------------------------------------------------- // -// docs/plan/hybrid.md milestone H3: *"a modeset API, which is the thing an +// docs/architecture/devices.md milestone H3: *"a modeset API, which is the thing an // out-of-kernel GPU driver makes affordable and an in-kernel one never got."* // // The claim is worth testing rather than repeating, so here is the whole of @@ -944,7 +944,7 @@ set_mode(int w, int h) // // The name is a *name*, not a path. AF_UNIX sockets in this kernel live in a // table keyed by string and never touch the filesystem -- that is item 3 of -// docs/plan/foundations.md and it is not built. Nothing here depends on which +// docs/architecture/kernel.md and it is not built. Nothing here depends on which // it is, and the day it becomes a real path this keeps working. #define VGPUD_SOCKET "/run/vgpud" diff --git a/docs/README.md b/docs/README.md index 69dca39..411e591 100644 --- a/docs/README.md +++ b/docs/README.md @@ -72,29 +72,36 @@ good chance it is in there. | | | |---|---| | [README](architecture/README.md) | The system, layer by layer. Read first. | -| [boot.md](architecture/boot.md) | Firmware to the first userspace process. | -| [kernel.md](architecture/kernel.md) | SchIFT: memory, processes, scheduling, traps, signals, SMP. | +| [boot.md](architecture/boot.md) | Firmware to the first userspace process, and where the loader puts the kernel. | +| [kernel.md](architecture/kernel.md) | SchIFT: memory, processes and threads, traps, signals, SMP. | +| [scheduler.md](architecture/scheduler.md) | Fair share by virtual runtime, slices, and `nice`. | | [syscalls.md](architecture/syscalls.md) | The call table, and the Linux personality that lets real software run. | -| [filesystem.md](architecture/filesystem.md) | The VFS, the buffer cache, procfs, mounts. | +| [filesystem.md](architecture/filesystem.md) | The VFS, the buffer cache, procfs, sysfs, devtmpfs, mounts. | | [schfs.md](architecture/schfs.md) | SchFS: the copy-on-write root filesystem. | -| [devices.md](architecture/devices.md) | The device model, driver binding, the class vtables. | +| [scff.md](architecture/scff.md) | SCFF: the compressed image the live root ships as. | +| [fat.md](architecture/fat.md) | FAT12/16/32 with long names: USB sticks, SD cards, the ESP. | +| [devices.md](architecture/devices.md) | The device model, driver binding, the class vtables, and drivers that live in a process. | +| [ldls.md](architecture/ldls.md) | Running Linux's drivers: a `.ko` in a process, reaching hardware through `/dev/pci`. | | [usb.md](architecture/usb.md) | xHCI, the USB core, and the class drivers. | -| [graphics.md](architecture/graphics.md) | Framebuffer, text rendering, console, and the crash screen. | -| [terminals.md](architecture/terminals.md) | The six virtual terminals, and how one is switched to. | -| [networking.md](architecture/networking.md) | Drivers, lwIP, sockets — Unix and INET. | +| [graphics.md](architecture/graphics.md) | Framebuffer, `/dev/fb0`, text rendering, the console, PSOGM and GMMOS. | +| [terminals.md](architecture/terminals.md) | The twelve virtual terminals, ptys, and handing one to a display server. | +| [audio.md](architecture/audio.md) | `/dev/dsp`, AudioEngine, AC'97, and ALSA over OSS. | +| [networking.md](architecture/networking.md) | NIC drivers, lwIP, sockets — Unix, INET and netlink. | | [init.md](architecture/init.md) | `schinit`, PID 1 as a role, and the job manifests. | +| [packages.md](architecture/packages.md) | The apk root: Alpine's packages, and SchOS as one of them. | +| [desktop.md](architecture/desktop.md) | Xorg on the framebuffer, Window Maker, and what they stand on. | +| [debugging.md](architecture/debugging.md) | Symbols, a log that outlives the kernel, and asking the kernel a question. | ## plan/ — what SchOS will be | | | |---|---| -| [README](plan/README.md) | The three current decisions and the order they happen in. | -| [1.0.md](plan/1.0.md) | **The finish line, and the freeze.** What 1.0 contains, what is deferred, and the register of every half-built thing in the tree. | -| [foundations.md](plan/foundations.md) | The kernel work everything else is blocked on. | -| [wayland.md](plan/wayland.md) | The GUI stack, and what gets deleted to make room. | -| [hybrid.md](plan/hybrid.md) | Moving drivers out of the kernel. | +| [README](plan/README.md) | What is here, what used to be, and the order. | +| [1.0.md](plan/1.0.md) | **The finish line, and the freeze.** What 1.0 contains, where it stands, and the register of every half-built thing in the tree. | +| [roadmap.md](plan/roadmap.md) | Everything else scoped and still open. | +| [gpu.md](plan/gpu.md) | Linux's `amdgpu` through LDLS, and `/dev/dri` served by a process. | +| [finch.md](plan/finch.md) | The author's language for SchOS's own programs. | | [schapi.md](plan/schapi.md) | The long-term API layering, and the compatibility promise. | -| [roadmap.md](plan/roadmap.md) | Everything else still open. | ## guides/ — how to work on it diff --git a/docs/Website/README.md b/docs/Website/README.md index d9081c1..c4b5110 100644 --- a/docs/Website/README.md +++ b/docs/Website/README.md @@ -30,7 +30,7 @@ is a token. A hex value in that file means the system is missing something, and the answer is to add it to Niveus rather than to work around it here. What that replaced was a 21 KB `style.css` that transcribed -[`include/gfx/theme.h`](../../include/theme.h) by hand — palette, type scale, +[`include/gfx/theme.h`](../../include/gfx/theme.h) by hand — palette, type scale, shadows, the lot. The intent was right and the mechanism was a copy: two files that had to be kept in step by remembering to. Now the whole identity is one declaration, in the `` of each page: diff --git a/docs/Website/docs.js b/docs/Website/docs.js index dff0158..0243280 100644 --- a/docs/Website/docs.js +++ b/docs/Website/docs.js @@ -30,6 +30,7 @@ ["architecture/syscalls.md", "Syscalls"], ["architecture/filesystem.md", "The VFS"], ["architecture/schfs.md", "SchFS"], + ["architecture/scff.md", "SCFF"], ["architecture/fat.md", "FAT"], ["architecture/terminals.md", "Virtual terminals"], ["architecture/devices.md", "Device model"], @@ -38,6 +39,8 @@ ["architecture/audio.md", "Audio"], ["architecture/networking.md", "Networking"], ["architecture/init.md", "init"], + ["architecture/packages.md", "Packages (apk)"], + ["architecture/desktop.md", "The desktop (Xorg)"], ["architecture/ldls.md", "Linux drivers (LDLS)"], ["architecture/debugging.md", "Debugging"] ]}, @@ -51,9 +54,8 @@ { group: "Plan", items: [ ["plan/README.md", "What plan/ is for"], ["plan/roadmap.md", "Roadmap"], - ["plan/foundations.md", "Foundations"], - ["plan/wayland.md", "Wayland"], - ["plan/hybrid.md", "Hybrid kernel"], + ["plan/gpu.md", "The GPU"], + ["plan/finch.md", "Finch"], ["plan/schapi.md", "SchAPI"] ]}, { group: "History", items: [ diff --git a/docs/Website/download.html b/docs/Website/download.html index 54efd76..896ff72 100644 --- a/docs/Website/download.html +++ b/docs/Website/download.html @@ -66,33 +66,34 @@

SchOS ships as one bootable ISO. It boots on UEFI, runs entirely from RAM, and touches nothing on the machine unless you tell it to — there is - no installer yet, and that is the point of trying it this way. + no installer yet, and that is the point of trying it this way. It comes up with + Xorg, Window Maker and Alpine's package manager.

- Download the ISO + Download the ISO Build it yourself

- schos.iso · about 62 MB · UEFI x86-64 · - nothing is written to your disks + schos.iso · a few hundred MB · UEFI x86-64 · + nothing is written to your disks · only the newest release keeps its ISO

- +

Before you do

-

This has booted on exactly one real machine

+

This has booted on three real machines

Everything below QEMU is honest reporting rather than a compatibility list. SchOS drives real hardware, and it has been run on a very small - number of it. + number of it — from a USB stick, each time.
@@ -103,8 +104,16 @@ boots - Asus Chromebox Guado CN62 - under test + AMD B550 desktop Ryzen 5 4600G + boots + + + ASUS TUF A15 laptop Ryzen, RTL8168H + boots, network under test + + + HP all-in-one Ivy Bridge, 2012 + boots Anything else @@ -231,11 +240,10 @@ Step three

A shell

- schinit starts one on the console and a - second on tty2. - mksh is /bin/sh, - with zsh beside it and the real toybox - userland behind both. + schinit starts one on the console and one on + each of the next five terminals. zsh is the login shell, + /bin/sh is busybox's, and mksh is + the static one recovery uses. Type startx for the desktop.

diff --git a/docs/Website/index.html b/docs/Website/index.html index 31fecef..1b5b922 100644 --- a/docs/Website/index.html +++ b/docs/Website/index.html @@ -398,13 +398,13 @@
-

Schcom

- in progress +

The desktop

+ runs
- The compositor that will own the display. v0 runs, and descriptor - passing — the one thing a Wayland client cannot do without — now - exists. What a buffer still needs is a shareable memory object. + Xorg on the framebuffer with Window Maker, xterm and a few X clients; + NetSurf for the web. No acceleration yet — the GPU is a plan, and the plan is Linux's own + amdgpu through LDLS.
@@ -415,7 +415,7 @@
The native programming interface — specified, not built, and deliberately - so. It should be designed from the compositor's experience rather + so. It should be designed from the desktop's experience rather than before it.
@@ -549,19 +549,19 @@
lines of code in kernel/ and include/
-
45,207
+
51,893
lines of comments alongside them
-
16,111
+
17,213
first-party lines in all
-
61,367
+
72,766
syscalls, Linux numbering
-
184
+
193
drivers in the kernel
@@ -569,15 +569,15 @@
Alpine packages in the live root
-
88
+
179
commits
-
253
+
289
days since the first one
-
76
+
78
patches carried against fastfetch
diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 8f1687c..025fa1f 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -16,11 +16,10 @@ its distribution. | Kernel | **SchIFT** | Multi-core x86-64 kernel with a modular VFS | | Filesystem | **SchFS** | Transactional, copy-on-write, the root filesystem | | API | **SchAPI** | The native programming interface — *designed, not built* | -| Compositor | **Schcom** | Owns the display; v0 runs | +| Packages | **apk** | Alpine's package manager manages the root; SchOS itself is the package `schos-base` | Twelve virtual terminals sit under all of it -- `/dev/tty1`..`tty12`, Ctrl+Alt+F1..F12 -or `chvt N` -- -each with its own screen, line discipline and foreground process group. See +or `chvt N` -- each with its own screen, line discipline and foreground process group. See [terminals.md](terminals.md). --- @@ -29,9 +28,11 @@ each with its own screen, line discipline and foreground process group. See ``` ┌─────────────────────────────────────────────────┐ - │ toybox kilo mksh zsh musl first-party cmd/ │ userland + │ Xorg + Window Maker · Alpine's packages (apk) │ desktop, userland + │ toybox mksh musl first-party cmd/ │ ├─────────────────────────────────────────────────┤ - │ vgpud — a driver that is an ordinary process │ + │ vgpud, ldlsd — drivers that are ordinary │ + │ processes (and Linux's r8169 inside ldlsd) │ ├─────────────────────────────────────────────────┤ │ schinit — PID 1, job manifests, supervision │ ├─────────────────────────────────────────────────┤ @@ -45,7 +46,8 @@ each with its own screen, line discipline and foreground process group. See ├─────────────────────────────────────────────────┤ │ /dev/pci — MMIO, DMA and IRQs to a process │ ← the eviction seam ├─────────────────────────────────────────────────┤ - │ drivers: xHCI e1000 virtio-gpu VBE IDE PS/2│ + │ drivers: xHCI AHCI NVMe e1000 8139 r8169 AC97 │ + │ virtio-gpu VBE IDE PS/2 I2C-HID │ ├─────────────────────────────────────────────────┤ │ scheduler │ paging │ traps │ IRQ │ SMP │ DMA │ SchIFT core └─────────────────────────────────────────────────┘ @@ -59,7 +61,8 @@ never names its consumer and a consumer never names a driver. And at **`/dev/pci`**, a driver stops having to be in the kernel at all: a process is handed a device's registers, DMA memory and interrupts, and the in-kernel driver that held it releases it cleanly. `vgpud` is the virtio-gpu driver living out -there today. See [../plan/hybrid.md](../plan/hybrid.md). +there today, and `ldlsd` runs Linux's own drivers the same way. See +[devices.md](devices.md) and [ldls.md](ldls.md). --- @@ -91,9 +94,12 @@ Read in this order if you are new: | [fat.md](fat.md) | FAT12/16/32 with long names: USB sticks, SD cards, the ESP. | | [devices.md](devices.md) | Buses, classes, binding, and the vtables consumers talk to. | | [usb.md](usb.md) | xHCI, the USB core, and the three class drivers. | -| [graphics.md](graphics.md) | Framebuffer, text rendering, the console, and PSOGM. | -| [terminals.md](terminals.md) | The six virtual terminals, and the chord that switches them. | -| [networking.md](networking.md) | NIC drivers, lwIP, and both socket families. | +| [graphics.md](graphics.md) | Framebuffer, `/dev/fb0`, text rendering, the console, PSOGM and GMMOS. | +| [desktop.md](desktop.md) | Xorg on the framebuffer, Window Maker, and what they stand on. | +| [packages.md](packages.md) | The apk root: Alpine's packages, and SchOS as one of them. | +| [scff.md](scff.md) | The compressed image the live root ships as. | +| [terminals.md](terminals.md) | The twelve virtual terminals, ptys, and handing one to a display server. | +| [networking.md](networking.md) | NIC drivers, lwIP, and the socket families. | | [audio.md](audio.md) | `/dev/dsp`, AudioEngine, and the AC'97 driver under it. | | [init.md](init.md) | `schinit`, PID 1 as a role, and the job manifests. | | [debugging.md](debugging.md) | Symbols, a log that outlives the kernel, and asking the kernel a question. | diff --git a/docs/architecture/boot.md b/docs/architecture/boot.md index e66f2c8..98db777 100644 --- a/docs/architecture/boot.md +++ b/docs/architecture/boot.md @@ -44,7 +44,7 @@ because there are none.** Everything the jump needs is computed before. ### The hand-off block One page of low memory at `0x30000`, defined once in -[`include/arch/schboot.h`](../../include/schboot.h) and included by both the loader +[`include/arch/schboot.h`](../../include/arch/schboot.h) and included by both the loader and `entry.S` so the two cannot disagree. | Offset | What | diff --git a/docs/architecture/devices.md b/docs/architecture/devices.md index 7926450..173cb4e 100644 --- a/docs/architecture/devices.md +++ b/docs/architecture/devices.md @@ -191,7 +191,7 @@ bitmap, and loses them when it releases or exits. Granting it is not optional a driver's `outl` is an instruction compiled into it, not a call anything can intercept — and granting it *coarsely* is not acceptable, because `IOPL=3` is one flag that hands over the interrupt controller and the disk along with the -NIC. See [../../include/ioperm.h](../../include/ioperm.h). +NIC. See [../../include/arch/ioperm.h](../../include/arch/ioperm.h). ### Handing a device over diff --git a/docs/architecture/graphics.md b/docs/architecture/graphics.md index 4db9f44..6676a1f 100644 --- a/docs/architecture/graphics.md +++ b/docs/architecture/graphics.md @@ -11,7 +11,7 @@ layer, the crash screen — and `/dev/fb0`, which is how Xorg reaches the screen This distinction causes confusion every time somebody meets it, so it is the first thing here. -| | [`gfx.h`](../../include/gfx.h) | [`display.h`](../../include/display.h) | +| | [`gfx.h`](../../include/gfx/gfx.h) | [`display.h`](../../include/gfx/display.h) | |---|---|---| | **What** | The kernel's own drawing surface | A frame interface for a display server | | **About** | Pixels and text | *Frames* | @@ -281,7 +281,7 @@ beyond that. ## Text -[`include/gfx/text.h`](../../include/text.h) + `kernel/dev/text.c`. +[`include/gfx/text.h`](../../include/gfx/text.h) + `kernel/dev/text.c`. **The font was never the problem — the character-cell grid was.** The console draws text by indexing `font_data[256][16]` and stepping the cursor eight @@ -609,7 +609,7 @@ is the report and the keyboards the laptops have; the xHCI path is the least exe ## One palette -[`include/gfx/theme.h`](../../include/theme.h). There were three unrelated designs +[`include/gfx/theme.h`](../../include/gfx/theme.h). There were three unrelated designs and the seams showed: the splash was deep purple with gold text deliberately matching the *panic* screen, the console was cream with near-black text, and so a healthy boot was dressed as a crash and then abruptly became an amber diff --git a/docs/architecture/kernel.md b/docs/architecture/kernel.md index 80ced5b..a8c456c 100644 --- a/docs/architecture/kernel.md +++ b/docs/architecture/kernel.md @@ -438,6 +438,13 @@ allocation over 8 MB mapped straight into a neighbouring live mapping. - **Permissions are tracked but not enforced.** uid/gid are maintained honestly so `getuid()`/`setuid()` stop lying, and file access does not consult them; `setpriority` is the one place the uids are checked. +- **`mprotect` is accepted and not enforced** (`sys_mprotect` returns 0). RELRO is not read-only, + a `PROT_NONE` guard does not fault, and nothing has `NX` (`EFER.NXE` is never set), so every + mapping is executable. +- **`poll`, `select` and `epoll` wait a tick at a time** (10 ms at `HZ=100`) unless something + that can make a descriptor ready calls `poll_kick()` — which the pipe, socket and event paths + do. busybox's `wget` polls before every read and downloaded at 85 KB/s, a segment per tick, + until the network path started kicking. - **No swap**, and no memory pressure handling of any kind. `kalloc` failing is reported up as `-ENOMEM` and that is the whole policy. - **Nested signal delivery is not supported.** `in_signal` guards against diff --git a/docs/architecture/ldls.md b/docs/architecture/ldls.md index 1d34830..e01310a 100644 --- a/docs/architecture/ldls.md +++ b/docs/architecture/ldls.md @@ -330,7 +330,7 @@ one flag and hands over all 65536 ports, including the interrupt controller, the timer and the disk; the TSS I/O permission bitmap is per port. A foreign driver already has DMA with no IOMMU, and the answer to "this is not a security boundary" is not to move it further — so it is the bitmap, at 8 KB per CPU of -resident memory. See [../../include/ioperm.h](../../include/ioperm.h). +resident memory. See [../../include/arch/ioperm.h](../../include/arch/ioperm.h). Writing it found a latent hole. The TSS's I/O map base is at offset 102 and the old code wrote offset 64; the field it meant to set stayed 0, and a base of 0 diff --git a/docs/architecture/networking.md b/docs/architecture/networking.md index 8ed53b3..029e53a 100644 --- a/docs/architecture/networking.md +++ b/docs/architecture/networking.md @@ -168,7 +168,7 @@ Recorded because both would have worked, quietly: ## The netdev class -A driver publishes a [`struct netdev_ops`](../../include/netdev.h) and is then +A driver publishes a [`struct netdev_ops`](../../include/dev/netdev.h) and is then finished with the network stack forever: it never mentions lwIP, and the stack never mentions it. diff --git a/docs/architecture/terminals.md b/docs/architecture/terminals.md index 8489a8f..f3846bf 100644 --- a/docs/architecture/terminals.md +++ b/docs/architecture/terminals.md @@ -272,12 +272,10 @@ Three layers, in increasing order of how much a program is claiming: `VT_SETMODE` used to be accepted and ignored — a program could ask for the chord, be told yes, and still not get it. It works now. -What `VT_PROCESS` does **not** do yet is the release handshake: Linux signals -the owning process on every switch request and blocks until it answers -`VT_RELDISP`, which is how a display server releases the GPU before the screen -changes hands. Nothing here has anything to release, so the kernel simply -stops switching rather than negotiating. `struct vt_mode` already carries -`relsig`/`acqsig` for when Schcom needs it. +`VT_PROCESS` also does the release handshake Linux does: a switch away from a terminal in +that mode signals the owning process (`relsig`) and waits for `VT_RELDISP`, which is how a +display server stops drawing before the screen changes hands; switching back sends `acqsig`. +See *Handing a terminal to a program*, below. Input goes to whichever terminal is on screen. That is what makes a terminal a terminal: the keyboard follows the eyes. diff --git a/docs/architecture/usb.md b/docs/architecture/usb.md index d3bae1a..29f55e6 100644 --- a/docs/architecture/usb.md +++ b/docs/architecture/usb.md @@ -28,7 +28,7 @@ framework underneath, not the licence. ## What the spec gives you for free -Everything in [`include/dev/usb.h`](../../include/usb.h) comes from USB 2.0 +Everything in [`include/dev/usb.h`](../../include/dev/usb.h) comes from USB 2.0 chapter 9, which every device implements regardless of speed, class or vendor. **That is what makes enumeration possible at all:** before you know anything diff --git a/docs/branding/README.md b/docs/branding/README.md index 89b5de2..7e456e9 100644 --- a/docs/branding/README.md +++ b/docs/branding/README.md @@ -192,7 +192,7 @@ If either is edited, edit both. |---|---| | Loader splash | Shift, from `include/gfx/logo.h`, shaded per row by `metal()` in `boot/common/ui.c` — silver, not accent. | | Kernel console / early boot | Shift, same mask, composited flat in a theme colour by `gfx_draw_logo`. | -| A display with no compositor | Shift again, drawn by `vgpud` on `SCH_BG`. It replaced a test card — colour bands, gradients, a checkerboard and the driver's pid in binary — which was scaffolding for [../plan/hybrid.md](../plan/hybrid.md)'s H2 and H3 and outlived them. The mark answers the same question a black screen cannot, *something is driving this device*, and answers it as the system rather than as a test rig. | +| A display with no compositor | Shift again, drawn by `vgpud` on `SCH_BG`. It replaced a test card — colour bands, gradients, a checkerboard and the driver's pid in binary — which was scaffolding for [../architecture/devices.md](../architecture/devices.md)'s driver work and outlived them. The mark answers the same question a black screen cannot, *something is driving this device*, and answers it as the system rather than as a test rig. | | PSOGM | Shift, the same mask again, by `gfx_draw_logo()` from `kernel/core/panic.c`. At its native 128 px — there is no resampler in the crash path, and the mark was only ever drawn at one size there anyway. | | Serial crash report | `:(`, as text. The mark cannot be drawn in ASCII, and a mood marker in a log is not a mark. | | SchFS | **Nowhere yet.** `uberblock.svg` has no consumer; it is waiting for `projects/schfs-linux/` to have a README worth heading. | diff --git a/docs/guides/adding-a-driver.md b/docs/guides/adding-a-driver.md index a345c7a..1feac7b 100644 --- a/docs/guides/adding-a-driver.md +++ b/docs/guides/adding-a-driver.md @@ -12,12 +12,20 @@ virtio, NVMe and the Intel datasheets are public documents with no licence on what you write from them. `kernel/drivers/bus/xhci/xhci.c` came from the spec, and so did `net/e1000/` and `gpu/virtio_gpu/`. -**BSD: yes. Linux: no.** This is a licence incompatibility rather than a -preference, and it decides the whole strategy — see -[../licensing.md](../licensing.md). BSD-2 and BSD-3 code can be used directly -with a `NOTICE` entry; GPL-2.0-only code can never be, at any distance. Reading -Linux to understand *hardware* is fine and normal; translating a Linux driver -function by function produces a derivative work whatever the identifiers say. +**Three sources, in the order to try them:** + +1. **A native driver, written from the specification.** The default, and what every driver in the + kernel is. +2. **Linux's own driver, unmodified, through LDLS** ([ldls.md](../architecture/ldls.md)), when the + device is one Linux supports and SchOS cannot afford to write a driver for — a NIC with a PHY + behind it, a GPU, a Wi-Fi card. It runs in a process, so a crash costs the interface and not the + machine. This is how the laptop's Realtek NIC is driven. +3. **BSD or Linux code taken into the tree.** Licensing no longer forbids either: SchOS is + GPL-2.0-or-later, so GPL-2.0-only code combines with it, and BSD-2/3 code can be used directly with + a `NOTICE` entry — see [../licensing.md](../licensing.md). The reason to prefer 1 and 2 is + maintenance, not law: a driver copied in is a driver that has to be tracked. + +Reading another kernel to understand *hardware* is fine and normal. One caveat about "port it from BSD" as a plan, learned by looking: FreeBSD's `sys/dev/usb` is roughly 100k lines welded to newbus, devclass, `bus_dma`, @@ -45,7 +53,9 @@ ours. kernel/drivers/ registry.c gpu/vga/ gpu/virtio_gpu/ - net/e1000/ + net/e1000/ net/rtl8139/ net/r8169/ + storage/ahci/ storage/nvme/ + audio/ac97.c bus/xhci/ usb/core/ usb/hid/ usb/storage/ usb/hub/ @@ -56,11 +66,12 @@ ours. | class | vtable | consumer | |---|---|---| - | `DEVCLASS_DISPLAY` | `struct display_ops` ([display.h](../../include/display.h)) | the compositor | - | `DEVCLASS_NET` | `struct netdev_ops` ([netdev.h](../../include/netdev.h)) | the TCP/IP stack | + | `DEVCLASS_DISPLAY` | `struct display_ops` ([display.h](../../include/gfx/display.h)) | none today — Xorg draws through `/dev/fb0` | + | `DEVCLASS_NET` | `struct netdev_ops` ([netdev.h](../../include/dev/netdev.h)) | the TCP/IP stack | + | `DEVCLASS_AUDIO` | `struct audio_ops` | `/dev/dsp` | A display driver that owns a linear framebuffer also publishes it with - `gfx_attach_scanout()` ([gfx.h](../../include/gfx.h)). That is what the text + `gfx_attach_scanout()` ([gfx.h](../../include/gfx/gfx.h)). That is what the text console and the crash screen draw on, and it is separate from the display class on purpose: the class is about *frames* — acquire, present, vblank — and the crash screen has to work when nothing else in the system does. diff --git a/docs/guides/building.md b/docs/guides/building.md index a4b3a5f..e21c6b3 100644 --- a/docs/guides/building.md +++ b/docs/guides/building.md @@ -8,8 +8,15 @@ change that only works on one of them is a bug. ```sh make # kernel + UEFI loader + root filesystem make qemu # build and boot it +make iso # a bootable ISO (the live system) for VMware or real hardware +./write.sh sdX # write that ISO to a USB stick (Linux; it checks what it is writing over) ``` +The first build downloads what it needs (musl, toybox, Alpine's packages -- a few +hundred megabytes, kept in `dist/`) and compiles Xorg's dependencies not at all: the +packages are Alpine's binaries, pinned in `ports/alpine/world.lock`. See +[packages](../architecture/packages.md). + ## Prerequisites | Need | Why | Windows | Linux | macOS | BSD | @@ -17,7 +24,7 @@ make qemu # build and boot it | `clang`, `ld.lld`, `llvm-objcopy`, `llvm-objdump` | cross-compiles the kernel, the 16/32-bit boot code, and musl userland | LLVM installer | `clang lld llvm` | `brew install llvm` | `pkg install llvm` | | `make` | — | Git Bash | `make` | Xcode CLT | `gmake` | | `perl` | generates the interrupt vector table | Git Bash | usually present | present | `pkg install perl5` | -| `python3` | boot logo, crash-screen QR | python.org (`python`) | `python3` | `brew install python` | `pkg install python3` | +| `python3` | boot logo, crash-screen QR, the numbers in the README | python.org (`python`) | `python3` | `brew install python` | `pkg install python3` | | `qemu-system-x86_64` | running and testing | `scoop install qemu` | `qemu-system-x86` | `brew install qemu` | `pkg install qemu` | | `curl`, `tar` | fetching ports | Git Bash | present | present | present | | `xorriso` **or** `mkisofs` | `make iso` only | `scoop install cdrtools` | `xorriso` | `brew install xorriso` | `pkg install xorriso` | @@ -81,10 +88,11 @@ produce. ## Ports -`musl`, `toybox` and `mksh` are built from source into `sysroot/`. **This does -not work on a Windows host** — `make` prints `ports: skipped` and uses whatever -is already staged in `sysroot/`. Building them needs a Linux host (or any other -Unix); do it once and the staged result serves the Windows builds. +`musl`, `toybox`, `mksh`, lwIP, uACPI, zstd, ncurses and the rest of `ports/` are built from source +into `sysroot/`; Alpine's packages are unpacked into the root from `dist/alpine/`. **This does not work +on a Windows host** -- `make` prints `ports: skipped` and uses whatever is already staged in +`sysroot/`. Building them needs a Linux host (or any other Unix); do it once and the staged result +serves the Windows builds. ## Where the tools live diff --git a/docs/guides/layout.md b/docs/guides/layout.md index d33f509..1fb16cd 100644 --- a/docs/guides/layout.md +++ b/docs/guides/layout.md @@ -14,47 +14,46 @@ deliberately kept out of `clean`). schos/ ├─ boot/ uefi/ the PE/COFF loader (splash, F8 menu, kernel load), │ common/ the UI it shares. UEFI only -- there is no BIOS path. -├─ kernel/ The kernel, by subsystem: arch/ core/ dev/ fs/ proc/ sys/ +├─ kernel/ The kernel, by subsystem: arch/ core/ dev/ drivers/ fs/ net/ +│ proc/ sys/ acpi/ (uACPI glue, the EC, I2C-HID) gmmos/ (the +│ micro OS under the crash screen) scff/ (the decompressor). +│ drivers/ is the in-kernel drivers, by class: audio bus gpu +│ net storage usb. +├─ ldls/ The Linux driver layer: the .ko loader, the BTF reader, the +│ Linux symbols reimplemented (kpi/), and host-side test tools. +│ The process that runs it is cmd/ldlsd. ├─ include/ Headers shared by kernel, userland, and tools/mkfs, by │ subsystem -- see "include/" below ├─ lib/ First-party userland runtime: _start glue, syscall stubs, │ the initcode blob embedded in the kernel, user.ld ├─ cmd/ First-party programs — one directory per program: │ cmd//main.c → sysroot/bin/ -│ schinit and schinitd are PID 1 and its supervisor; vgpud is -│ the virtio-gpu driver, which runs out here rather than in -│ the kernel; socktest/shmtest/nettest/pcitest are suites. -├─ ports/ Third-party software — one directory per port. -│ Contains ONLY a Makefile and optional patches/. -├─ projects/ Things that run ON SchOS rather than things that ARE it: -│ the compositor, window manager, desktop, and installer. +│ schinit and schinitd are PID 1 and its supervisor; vgpud and +│ ldlsd are drivers that run out here rather than in the +│ kernel; socktest/shmtest/nettest/pcitest/xready are suites. +├─ etc/ The configuration the image starts with: schinit's job +│ manifests (etc/schinit/*.job), motd, os-release, the zsh files. +├─ ports/ Third-party software -- one directory per port, containing +│ ONLY a Makefile and optional patches/. musl, toybox, mksh, +│ lwIP, uACPI, zstd, ncurses, the fonts, fastfetch, kilo; and +│ alpine/, which is the apk root (world, world.lock). +├─ projects/ Things that run ON SchOS rather than things that ARE it. │ Not part of the kernel build. See projects/README.md. -├─ tools/ Host-side helpers, SOURCE ONLY: mkfs.c, the SchFS test -│ suites, sign.pl, vectors.pl, and the PowerShell harnesses. -│ Their binaries build into build/tools/ -- this directory -│ used to hold both, which is the one rule this layout has. -│ tools/windows/loc.ps1 counts how much of the tree is first-party. -├─ sysroot/ Staging for the disk image. etc/ is hand-maintained; -│ bin/ lib/ sbin/ are populated by `make sysroot-install`. -├─ dist/ Upstream tarball cache. Survives `make clean`. -├─ Refrence/ A FreeBSD checkout (Refrence/freebsd-src), kept as reference -│ material and a parts bin. Never built, never in an image. -│ Read it when a specification is ambiguous; copy from it only -│ after the per-file licence check in docs/licensing.md. -│ A full clone is 2.1 GB and a previous NetBSD one was deleted -│ for that reason. Only sys/dev, sys/net and sys/netinet have -│ ever been read, so a sparse checkout is ~100 MB: -│ git clone --filter=blob:none --sparse Refrence/freebsd-src -│ git -C Refrence/freebsd-src sparse-checkout set │ sys/dev sys/net sys/netinet -├─ src-trunk/ A NetBSD checkout, when present -- reference material and a -│ parts bin. NOT built, NOT installed, must never reach an -│ image. Currently absent; it was deleted for disk space and -│ can be re-fetched sparsely. Copying from it is governed by -│ the per-file licence check in docs/licensing.md §3 -- -│ NetBSD is not uniformly licensed. +├─ tools/ Host-side helpers, SOURCE ONLY: mkfs.c, mkschfs.c, mkscff.c, +│ the SchFS test suites, vectors.pl, and the harnesses, grouped +│ by what runs them (windows/ .ps1, posix/ .sh, python/ .py). +│ Their binaries build into build/tools/. tools/windows/loc.ps1 +│ and tools/python/stats.py count how much of the tree is +│ first-party. +├─ sysroot/ Staging for the disk image, GENERATED by the build -- edit +│ tools/posix/mkaroot.sh and etc/, not this. +├─ dist/ Upstream tarball and package cache. Survives `make clean`. ├─ shell.nix NixOS development shell: the whole toolchain in one file +├─ write.sh Writes schos.iso to a USB stick, safely (Linux). +├─ run.ps1 The Windows-side QEMU launcher. ├─ docs/ architecture/ what it is plan/ what's next │ guides/ how to work history/ why it's like this +│ Website/ the site, and the docs reader that renders this └─ build/ ALL generated output (see below) ``` @@ -92,7 +91,10 @@ build/ ├─ kernel/ object files, the linked kernel, kernel.asm ├─ lib/ libc glue objects, initcode blob ├─ cmd/ linked first-party programs -├─ fs.img the filesystem image +├─ aroot/ the apk root the image is made from +├─ schfs.img the root as a disk image (make qemu) +├─ fs-ram.img the same root as the live ramdisk, fs-ram.scff compressed +├─ iso/ … what make iso assembles └─ ports// ├─ src/ extracted upstream tarball ├─ obj/ out-of-tree build directory @@ -130,57 +132,43 @@ Local modifications to upstream go in `ports/foo/patches/*.patch` and are applied after extraction — never by editing files under `build/`, which are wiped on every clean. -## Ramdisk root +## The live root ```sh -make # normal: root on the IDE disk (build/fs.img as hdb) -make ramroot # root from a SCHFS image linked into the kernel +make # normal: the root is a SchFS disk image, build/schfs.img +make iso # the same root as a compressed ramdisk linked into the kernel +make ramroot # the ramdisk, uncompressed ``` -`make ramroot` builds `build/fs-ram.img` (default 6000 blocks, override with -`RAMDISK_BLOCKS=`) and links it into the kernel, so the system boots to a shell -with **no data disk attached at all**. That is what makes destructive work on -the real disk survivable — reformatting it, or bringing up a new on-disk format -— and it is the first piece of the installer. +`make iso` builds the root as a SchFS image, compresses it +([SCFF](../architecture/scff.md)) and links it into the kernel, so the system boots +to a shell with **no data disk attached at all** -- which is what the USB stick is. +It is also what makes destructive work on a real disk survivable. Nothing written +to a live root survives a reboot. -Use `make ramroot`, not `make RAMROOT=1`: the flag changes `CFLAGS`, which make -cannot see, so objects compiled for one setting would be silently reused for -the other and `ROOTDEV` would point at the wrong device. The target drops the -compiled objects first. Switching back is the same — plain `make` after a -`ramroot` build reuses stale objects, so run `rm -rf build/kernel build/lib -build/cmd` first (or just `make clean`). - -The image costs boot time: the loader reads the kernel through the firmware's -file protocol, so a 6 MB ramroot kernel is visibly slower to start than the -350 KB default. +Use the targets, not `make RAMROOT=1`: the flag changes `CFLAGS`, which make cannot +see, so objects compiled for one setting would be silently reused for the other and +`ROOTDEV` would point at the wrong device. The targets drop the compiled objects +first. Going back to the disk root is the same -- run `make clean` or remove +`build/kernel build/lib build/cmd` first. **What may go in it is a licensing question, not only a size one.** The image is linked into the kernel binary, so anything in `sysroot/` when it is built becomes -part of a distributable artifact. A file staged there under a licence that -is not ours to redistribute does not merely ride along — it makes the kernel -itself undistributable. `RAMDISK_EXCLUDE` in the Makefile is the list of -paths kept out, and enforcing it in the build rather than relying on someone -remembering to unstage something first is deliberate. It has been needed once -already: a 33 MB test payload landed in `sysroot/` and both overflowed the image -and would have gone into the kernel binary. +part of a distributable artifact. A file staged there under a licence that is not +ours to redistribute does not merely ride along -- it makes the kernel itself +undistributable. `RAMDISK_EXCLUDE` in the Makefile is the list of paths kept out, +and enforcing it in the build rather than relying on someone remembering to unstage +something first is deliberate. (Linux modules for LDLS are staged by `make +ldls-stage`/`ldls-r8169` only on request, and `.gitignore` refuses `*.ko`.) -Three knobs, all in the Makefile, none of them `FSSIZE`: +The knobs: `RAMDISK_BLOCKS` (image size), `RAMDISK_NINODES` (the inode table is +*preallocated*: the default of 12000 once cost a quarter of a small image to +describe about 260 files), `RAMDISK_EXCLUDE`. If `mkfs` reports the image is full, +read its message: it prints the metadata breakdown and names the right knob. -| | | -|---|---| -| `RAMDISK_BLOCKS` | image size, 6000 (× 1 KiB `BSIZE`) | -| `RAMDISK_NINODES` | inode table size, 2000 | -| `RAMDISK_EXCLUDE` | sysroot-relative paths to omit, with their subtrees | - -`RAMDISK_NINODES` is worth knowing about because the inode table is -*preallocated*: mkfs's default of 12000 inodes cost 1501 of the 6000 blocks — a -quarter of the image — to describe about 260 files. At 2000 it costs 251, and -metadata overall fell from 1596 blocks to 345. - -If `mkfs` reports the image is full, read its message: it now prints the -metadata breakdown and names the right knob for whichever image overflowed. It -used to say only "increase FSSIZE in param.h", which is a constant that has -nothing to do with the ramdisk root and sends you to the wrong file. +The ISO build is fast on purpose (a few seconds when little changed): it copies with +`cp -p` and `install -C` so unchanged files keep their time stamps, and toggling the +ramdisk does not wipe the tree. ## Two things that will bite you diff --git a/docs/guides/testing.md b/docs/guides/testing.md index 8b0f6b8..9e4651b 100644 --- a/docs/guides/testing.md +++ b/docs/guides/testing.md @@ -12,7 +12,7 @@ reporting that it started. | | What it runs | What it proves | |---|---|---| -| `make schfs-check` | Host-native unit suites | The filesystem's logic, in about a second | +| `make schfs-check`, `make kmem-test`, `ldls-inspect`/`ldls-run` | Host-native unit suites and the Linux-module loader run on the build machine | The filesystem's logic, the slab allocator's bookkeeping, and a `.ko`'s layout, in about a second | | `tools/windows/bootcheck.ps1` | A real boot, driven over the serial console | That the system answers correctly | | `make crashqr`, `make schcom-test` | A boot, then pixel assertions from the host | That the *screen* is right | @@ -77,6 +77,22 @@ dropped every byte would pass any check made from one side alone. Under QEMU's user-mode networking the host is `10.0.2.2` seen from inside, and SLIRP connects to the host's loopback — so the listener binds there. +## Test programs that run on SchOS + +Several first-party programs in `cmd/` exist to be run **on** the system and print which of their checks +failed, because only the system can say whether its own interfaces behave. `bootcheck` runs the ones it +can; the rest are run by hand or by a script: + +| | Checks | +|---|---| +| `socktest`, `nettest`, `netecho` | Unix and INET sockets, `SCM_RIGHTS` and credentials, `setsockopt`'s refusals | +| `shmtest`, `memfdtest` | System V shared memory; `memfd` mapped by two processes, and one sent over a socket | +| `pcitest` | `/dev/pci`: claim, map, DMA, interrupts, release | +| `evtest`, `locktest`, `sigtest` | the event descriptors; `flock` and record locks; signals | +| `dynhello` | dynamic linking: TLS, the heap, a forked child that still has its libraries | +| `xready` | every kernel call Xorg makes, before Xorg is started — the evdev queries, the fbdev ioctls, the console calls, the `/sys` files | +| `tools/python/ldls_check.py` | what LDLS needs from a kernel's BTF | + ## Pixel assertions ```sh @@ -148,10 +164,12 @@ leak. - **No automated test runs on real hardware.** Everything above is QEMU. - **SMP is exercised but not stressed** — boots on 1, 2 and 4 processors are checked; nothing hunts for races. +- **Nothing asserts the desktop by pixel.** `xready` checks the calls Xorg makes; whether Window Maker + and an `xterm` appear on a real screen is looked at in screenshots, not asserted. - **No fuzzing anywhere**, and no `fsx`/`xfstests` against SchFS. Getting SchFS under those is the main argument for the FUSE implementation in [../plan/roadmap.md](../plan/roadmap.md). - **The SchFS round-trip test needs symlinks and hard links**, so it skips on a Git Bash host and only runs on Unix. -- **`make ports` does not work on Windows.** musl, toybox and mksh must be built +- **`make ports` does not work on Windows.** musl, toybox, mksh and the rest must be built once on a Unix host and staged. diff --git a/docs/history/README.md b/docs/history/README.md index e20176b..904dfbb 100644 --- a/docs/history/README.md +++ b/docs/history/README.md @@ -5,7 +5,7 @@ and the few write-ups that have nowhere better to live. | | | |---|---| -| [kernel.md](kernel.md) | The expensive ones. Boot, SMP, signals, ABI assumptions, `/proc`, input. | +| [kernel.md](kernel.md) | The expensive ones. Boot, SMP, signals, ABI assumptions, `/proc`, input, TLBs, a stub that returned a flag word. | | [schfs.md](schfs.md) | How the filesystem was built, and what each phase cost. | | [moving-to-linux.md](moving-to-linux.md) | The Windows-to-Fedora move. | diff --git a/docs/history/kernel.md b/docs/history/kernel.md index ff72a21..dc35a20 100644 --- a/docs/history/kernel.md +++ b/docs/history/kernel.md @@ -298,17 +298,8 @@ exported entry points, `nameifrom()`/`nameiparentfrom()`. `sys_fs.c` gained Every `*at()` syscall now uses these instead of its own inline absolute-path/`AT_FDCWD`-only check. -**Known follow-on issue, not yet root-caused**: heavy `ls`/`ls -lR` use -across many commands in one long-running shell session can eventually make -a *later*, unrelated directory hang (reproduced with `ls -lR /var` after -~10 prior commands in the same session; the identical command succeeds -immediately on a fresh boot). Not an inode-table or file-table exhaustion -— both `iget()` and `filealloc()` fail cleanly (panic / return 0 -respectively) when full, and neither fired. Smells like a lock or -reference-count leak somewhere in the *at() paths above, or in toybox's -own fd handling, that only manifests after enough accumulated opens/closes. -Next step: instrument `itable`/`ftable` occupancy counts and watch them -across a long `ls`-heavy session to see what's actually climbing. +**Known follow-on issue** (heavy `ls -lR` in one long shell session eventually hanging a +*later* directory): hunted, and not reproducible — see section 24. ## 6. Boot speed @@ -1255,3 +1246,86 @@ is at least as long as the width, and that the aperture holds that many rows. If not, it goes back to the previous mode and says so in the menu. This proves nothing about a mode that passes, but it stops a lying one from reaching the kernel. + +--- + +## 24. The `ls -lR` hang that was hunted and not found + +**Symptom.** After about ten `ls`-heavy commands in one long shell session, a later and +unrelated `ls -lR /var` would hang; the same command succeeded at once on a fresh boot. It +was not inode-table or file-table exhaustion (both fail cleanly), and the first hypothesis +was a lock or reference-count leak in the `*at()` paths. + +**What was done.** `/proc/fsstat` was given the occupancy counters the hypothesis asked +for (inode, file and buffer peaks, the SchFS volume lock and transaction owner), and the +workload was made heavier than the report: ten rounds of `ls -lR /bin` interleaved with +writes and `cd` — 33 execs and 13,378 tree descents. Nothing climbed: + + inode_peak 9 / 200 file_peak 9 / 512 + buf_pinned_peak 1 / 256 schfs0 locked 0 tx_owner_pid 0 tx_depth 0 + +**Conclusion.** The hypothesis is eliminated, and the bug is very likely gone — fixed in passing +by the `*at()` dirfd rewrite, the `getdents` cookie rewrite, or SchFS becoming the root, all of +which landed after it was reported. It was closed rather than chased. The instrumentation +stayed: if it happens again, `/proc/fsstat` names the table, or shows `locked 1` with +`tx_owner_pid 0`, which is an orphaned volume lock. + +Two real defects turned up on the way and were fixed, neither proven to be the cause: +`holdingsleep()` never compared the owning pid, which disarmed all seven `panic()` assertions +built on it; and SchFS's `tx_mine()` compared a bare `struct proc *`, which a recycled ptable +slot makes ambiguous. + +**The transferable part.** A hang you cannot reproduce still earns its instrumentation. The +absence of a result *is* the result, and the right place to leave it is where the next person +with the same symptom will look. + +## 25. Epiphany's heap was corrupted by a page nobody had unmapped yet + +**Symptom.** Epiphany (WebKit: forty threads on four processors) died at a different address +on every run; musl's allocator was the first to notice, a group's metadata no longer pointing +back at its group. + +**Diagnosis.** Nothing was freed twice. One thread unmapped a range, `mmap` handed the same +addresses straight back, and a thread on *another* processor went on reading and writing the +**old** pages through its TLB: stores vanished, loads were stale. Deferring the free of the +unmapped pages had already kept a stale translation from reaching a page someone else owned; +it did not keep one address space *coherent*. + +**Fix.** Unmapping from a shared address space interrupts the processors running it and waits +until each has reloaded `CR3` (`tlb_shootdown()`, `kernel/arch/x86_64/vm.c`). + +**The transferable part.** Threads turned an invisible assumption — one CPU's TLB is the only +one that matters — into the loudest bug in the tree, in the one program that used the most +threads. Memory corruption that moves every run is a coherence bug until proven otherwise. + +## 26. Every MDIO read returned `0x02460246` + +**Symptom.** Linux's `r8169` under LDLS, on the laptop, read its PHY's ID as `0x02460246`. + +**Diagnosis.** `0x246` is a value of `RFLAGS`, not a register. `mdio-bus` keeps the word it +has just read in `r11` across `preempt_enable()`, which on x86 is `decl %gs:__preempt_count; je` +to a call that preserves every register. LDLS had given `__preempt_count` zeroes, so the count +reached zero after the decrement and the call was made — into a logging stub whose `syscall` +instruction leaves `RFLAGS` in `r11`. + +**Fix.** `__preempt_count` starts at `PREEMPT_ENABLED` (x86 keeps `NEED_RESCHED` inverted), and +`preempt_schedule` and its notrace twin are a bare `ret`. + +**The transferable part.** A stub that returns zero is not safe when zero changes which path +runs — the same family as the five in [../architecture/ldls.md](../architecture/ldls.md). The +tell here was a value that was a plausible *flag word* rather than a plausible *register*. + +## 27. GLib's network monitor spun at 100% + +**Symptom.** Epiphany's main thread consumed a full core in `poll()` returning at once. + +**Diagnosis.** GLib sizes each netlink datagram with `recvmsg(MSG_PEEK|MSG_TRUNC)` and no buffer. +The kernel answered 0, GLib read 0 bytes, the datagram stayed queued, and `poll()` said readable +again, for ever. A drained netlink socket also reported hangup, because `sock_hup` took "no peer" +for "closed". + +**Fix.** `recvmsg` on netlink peeks without consuming and with `MSG_TRUNC` returns what is waiting. + +**The transferable part.** A kernel that accepts a flag and does nothing with it turns a +well-formed program into a busy loop. Do it, or refuse it. + diff --git a/docs/plan/1.0.md b/docs/plan/1.0.md index ccd09b7..5b2eb50 100644 --- a/docs/plan/1.0.md +++ b/docs/plan/1.0.md @@ -7,17 +7,16 @@ A finish line, so that the list of open work stops being infinite. ## Why this document exists [schapi.md](schapi.md) §3 says the stability freeze starts at 1.0 and covers -whatever is marked `stable` at that moment. **1.0 was never defined**, which -means there has been no finish line — and with no finish line every gap is -equally mandatory, the list only grows, and the project feels larger every week -regardless of how much gets done. +whatever is marked `stable` at that moment. 1.0 was never defined, which means +there was no finish line — and with no finish line every gap is equally mandatory, +the list only grows, and the project feels larger every week regardless of how much +gets done. -That feeling is structural, not a judgement about progress. This document fixes -the structure by drawing a line and putting most of the open work on the far -side of it. +That feeling is structural, not a judgement about progress. This document fixes the +structure by drawing a line and putting most of the open work on the far side of it. -> **Anything not named in this document is not part of 1.0.** It is not -> cancelled, not forgotten, and not up for reconsideration until 1.0 ships. +> **Anything not named in this document is not part of 1.0.** It is not cancelled, +> not forgotten, and not up for reconsideration until 1.0 ships. --- @@ -26,31 +25,55 @@ side of it. > **A machine one person can sit down at and use for an afternoon without > falling off an edge.** -Concretely — boots on real hardware, roots on SchFS, runs a shell and the real -toybox userland, is on the network, and puts a windowed terminal on the screen. +Concretely — boots on real hardware, roots on SchOS's own filesystem, runs a shell +and a real userland, is on the network, and puts a windowed desktop on the screen. -It is deliberately a **small complete system** rather than a rich incomplete -one. SchOS 1.0 is not Linux, does not run third-party applications, and has no -package manager. It is a system that works, end to end, with nothing in it that -lies about what it did. +It started as a deliberately small system: *no third-party applications and no package +manager*. That has changed on its own, and is worth recording rather than tidying away: +the root is an **apk root** ([packages](../architecture/packages.md)), Alpine's +programs run on it unmodified, and the desktop is Xorg with Window Maker +([desktop](../architecture/desktop.md)). What did not change is the bar: nothing in the +system lies about what it did. ### The acceptance test -One unattended run, asserted from the host, the way everything else here is -tested. When this passes, it is 1.0: +One unattended run, asserted from the host, the way everything else here is tested. +When this passes, it is 1.0: ``` boot on real hardware from a USB stick → SchFS root mounts, generation counter advances - → schinit starts the compositor - → a terminal window appears and is asserted by pixel - → in it: ls, cat, a file written, a network fetch + → schinit starts shells on the virtual terminals + → X starts; a window appears and is asserted by pixel + → in it: ls, cat, a file written, a network fetch over HTTPS → reboot → the file is still there and the checksum matches ``` -Nothing in that sentence is a feature list. Every clause is something that is -currently either broken, missing, or unproven. +Where each clause stands: + +| Clause | State | +|---|---| +| boot from a USB stick on real hardware | **done** on an AMD B550 desktop, an ASUS TUF A15 laptop and an Ivy Bridge HP all-in-one | +| SchFS root, generation counter | **done** — the root is a SchFS image, in RAM when live | +| a terminal to type in | **done** — a shell on the first six of the twelve virtual terminals (`etc/schinit/*-shell.job`); the rest are there to be started | +| X starts, a window appears | **done** (`startx` → Window Maker + xterm); *asserted by pixel* has no automated test yet — `bootcheck` drives the serial console, and the screenshots are looked at | +| `ls`, `cat`, a file written | **done** | +| a network fetch over HTTPS | **done** (`wget`, `curl`, `apk`) | +| reboot, **the file is still there** | **not met on the live image.** The root lives in RAM and everything written is gone at reboot; SchFS is crash-safe on a disk, but nothing writes a disk root. This is the one clause that fails, and it is the open decision below | + +### The one open decision + +The acceptance test's last clause needs a root that survives a reboot. There are two honest ways to +close it, and which one is the owner's to choose: + +1. **Write the installer** (`projects/schinst/`, [roadmap.md](roadmap.md) §5): partition a disk, + write the root, install the loader. Large, and until now frozen. +2. **Narrow the clause**: 1.0 is a *live* system, and the last line becomes "a file written to a + SchFS disk or partition that was attached survives a reboot" — which already works + (`make disk`, `make qemu` with `schfs.img`). Smaller, and it says what the system is. + +Nothing else is on the path to 1.0. --- @@ -59,11 +82,10 @@ currently either broken, missing, or unproven. **No new capability is started until this register is empty.** The failure this exists to stop is not laziness and not scope creep — it is -*abandonment by promotion*. A thing gets built to the point where it works, the -next thing becomes more interesting, and the first one is left in a corner: -present in the tree, not finished, not deleted, and not visible in any list. -Enough of those and the tree stops matching its own documentation, which is the -state this register was written to end. +*abandonment by promotion*. A thing gets built to the point where it works, the next +thing becomes more interesting, and the first one is left in a corner: present in the tree, +not finished, not deleted, and not visible in any list. Enough of those and the tree stops +matching its own documentation. Two rules, and they are the same rule seen from either end: @@ -71,73 +93,47 @@ Two rules, and they are the same rule seen from either end: > > **Adding a row requires closing one.** Two things in flight. Never three. -A row leaves this table in one of exactly two ways. Either it is **finished** — -and finished means a description exists in [../architecture/](../architecture/), -not that the code works — or it is **deleted**, and deleted means gone from the -tree, not marked for deletion. A directory that has said "to be deleted" for -months is the corner, not the cure; `projects/schwm/` and `projects/schde/` -proved that and are now actually gone. +A row leaves this table in one of exactly two ways. Either it is **finished** — and finished +means a description exists in [../architecture/](../architecture/), not that the code works — +or it is **deleted**, and deleted means gone from the tree, not marked for deletion. A +directory that has said "to be deleted" for months is the corner, not the cure. ### Built, and not connected to anything -The galling ones. The work exists, is tested, and nothing uses it. - | | Where it stopped | |---|---| -| ~~**The text layer**~~ | **Half closed, and the half that closed is the one that mattered.** The console draws real TrueType through `font_ops` now: cells hold a codepoint, the cell size comes from the font's measured advance, and `kernel/dev/ttf.c` has a caller that is not a self-test. What is still unused is `text.c`'s *proportional* path — the console remains a fixed grid. See Gate 0. | -| **A font from disk** | Still true, and now deliberate rather than pending: the console typeface is linked into the kernel image, because a panic during boot has to be readable before there is a filesystem. Loading a *second* font from disk is the open part. | -| **Socket activation** | `schinitd` parses `[Socket name]` and remembers the path. Nothing holds the socket. **This is the entire reason for copying launchd**, and it is the half that was not built. | -| **`display_ops.wait_vblank`** | In the vtable, implemented by the VBE driver, left null by virtio-gpu. A compositor that needs it cannot rely on it. | +| **`projects/schcom/`** | A compositor v0 (about 700 lines) over a wire format that nothing speaks any more: the desktop is Xorg. It is the one thing in the tree that has ever put a client's pixels on screen by itself, and it has no consumer. **Delete it**, or say what it is for. | +| **The in-kernel LDLS window** | `vm.c` maps the kernel image a second time in the top 2 GB at every boot, and `include/ldls/ldls.h` describes a `/dev/ldls` loader — the start of loading Linux modules *into the kernel*, which [ldls.md](../architecture/ldls.md) rule 2 says is not how it is done. Nothing calls either. **Delete it**, or decide. | +| **`display_ops`** | `acquire` and `wait_vblank` are in the vtable, implemented by one driver at most, with no consumer: Xorg draws through `/dev/fb0`. Kept for the GPU work ([gpu.md](gpu.md)); that is the reason it stays. | +| **Socket activation** | `schinitd` parses `[Socket name]` and remembers the path. Nothing holds the socket. This is the entire reason for copying launchd, and it is the half that was not built. | +| **`sys_delete`, `sys_show`, `sys_showstat`** | Orphaned entries from the old v1 filesystem's recycle bin. They die with `fs.c`. | ### Started, and stopped part-way | | Where it stopped | |---|---| -| **`foundations.md`** | All but item 4 (`setsockopt`) built: descriptor passing, `memfd`, filesystem-named `AF_UNIX`, dynamic linking, and epoll/eventfd/timerfd/signalfd. | -| ~~**`hybrid.md`**~~ | **Closed.** All four mechanisms built; H1, H2 and H3 done. H4 — the USB eviction — is *withdrawn* rather than outstanding: USB is the bus peripherals arrive on, not a peripheral, and an eviction that has to leave an identical driver behind has moved nothing. See [hybrid.md](hybrid.md), "Decided: USB stays in the kernel". | -| ~~**`ldls.md`**~~ | **Closed.** L0 through L4 built: the loader handles 2,440 stock modules, and both `8139too` and Linux's `e1000` run in a SchOS process through `/dev/pci` with `ping` working over each. The e1000 run is a handover — the kernel was driving that NIC and gave it up. Described in [../architecture/ldls.md](../architecture/ldls.md). Two drivers, one kernel version, one machine. | -| **Schcom v0** | 715 lines that composite, over a wire format that is superseded. Frozen: it runs, nothing more is added, and it is kept only because it is the one thing in the tree that has ever put a client's pixels on screen. | -| ~~**`debugger.md`**~~ | **Closed.** D0 through D5 built, including the network transport: `schdbgd` bridges TCP to `/dev/dbgro`, ships disabled, binds loopback, and cannot reach a view that changes state. Described in [../architecture/debugging.md](../architecture/debugging.md). | - -**The row LDLS spent, and gave back.** `hybrid.md` closed — H3 built, H4 -withdrawn — and `ldls.md` took the slot it freed. It has since closed too, so -the slot is free again. Worth being honest that this is -the largest thing ever to enter this register, and that it was **explicitly not -part of 1.0**: nothing in the acceptance test needs a foreign driver. It was -taken on because the mechanisms it uses were finished and idle, and because -the alternative to it is writing a driver for every device that exists. - -**The row the debugger spent, and gave back.** Adding one requires closing one, -and *A released PCI device* closed when H2 landed. It is worth saying plainly -that the debugger is not a capability of the OS the way every other row here is: -nothing in it ships to a user or appears in the acceptance test. The case for it -is that **Gate 0 is a list of bugs to find in a kernel that could not be asked -anything** — so it is the instrument the rest of this register is surveyed with, -and it earned its slot by making every other row cheaper to close. - -It has closed. **Every plan in `docs/plan/` that was open is now closed**, which -means the next thing is Gate 0 itself: the survey, with the instrument built. - -**Closed since this register was written.** *A released PCI device* — release is -now symmetric with claim (bus mastering cleared before the DMA is freed, a -refusal when the device is the only display), a supervised job starts the -replacement, and it is described in -[../architecture/devices.md](../architecture/devices.md) under "Taking it back". +| **LDLS on real hardware** | `r8169` registers its interface on the laptop and reads its PHY; whether it carries traffic there is **not confirmed**. The native driver is out of the registry and stays as the fallback. | +| **The laptop's embedded controller** | The EC is found and served (battery and AC read through it); **SCI queries are not handled**, so events from it are not seen. | +| **Epiphany (WebKit)** | `apk add epiphany` installs it; its web-process child dies at start-up ("short read from the peer"). NetSurf runs. Not diagnosed. | ### Named in a "what it does not do" and still true -Not corners — these were written down honestly on the day and stayed written -down. They are here so the register is the whole picture rather than the -flattering part of it. +Not corners — these were written down honestly on the day and stayed written down. They are here +so the register is the whole picture rather than the flattering part of it. Each is described where +it lives. | | | |---|---| -| `/proc//exe` is a regular file | so `readlink()` fails and `ps -o comm` prints `[init]` | -| `getrusage` | does not distinguish `RUSAGE_CHILDREN` from `RUSAGE_SELF` | -| No slab allocator | every subsystem's fixed-size table is a consequence | -| No `getsockname`/`getpeername` | and `accept()` does not report an AF_INET peer | -| No routing table | first interface wins, which is a rule and not routing | +| No page cache | writable `MAP_SHARED` of a file is refused — [kernel.md](../architecture/kernel.md) | +| `mprotect` is a no-op, nothing has NX | [kernel.md](../architecture/kernel.md) | +| No routing table | first interface wins, which is a rule and not routing — [networking.md](../architecture/networking.md) | | No descriptor garbage collector | a socket passed over itself and never received leaks the pair | +| No udev / uevent socket | Xorg is told its input devices — [desktop.md](../architecture/desktop.md) | +| No EHCI driver | USB 2.0 ports wired only to EHCI do nothing — [usb.md](../architecture/usb.md) | +| Touchpads are mice | no two-finger scroll — [desktop.md](../architecture/desktop.md) | +| `ru_stime` is zero | [kernel.md](../architecture/kernel.md) | +| No IOMMU | a driver in a process is isolated against bugs, not malice — [devices.md](../architecture/devices.md) | +| One audio chip family | AC'97 only; laptops have HDA — [audio.md](../architecture/audio.md) | ### Frozen — scoped, unstarted, and not to be started @@ -145,433 +141,77 @@ Neither cancelled nor in progress. **Do not open these.** | | | |---|---| -| `projects/schinst/` | the installer, the installation environment, `.sip` | +| `projects/schinst/` | the installer, the installation environment, `.sip` (see the open decision above) | | `projects/schfs-linux/` | SchFS as a filesystem for Linux | -| [roadmap.md](roadmap.md) §4 | the interactive crash screen | -| [roadmap.md](roadmap.md) §6 | SchFS beyond SchOS | +| [gpu.md](gpu.md) | amdgpu and nouveau through LDLS — measured, with the work it needs written down | +| [finch.md](finch.md) | Finch inside SchOS (the language itself is finished and lives in its own repository) | | [schapi.md](schapi.md) | the API layering, designed and not built | -Each of those is well argued and none of them is on the path to the acceptance -test above. That is the whole test for whether something belongs in this section: -**not "is it worth doing" but "does 1.0 fail without it".** +Each of those is well argued and none of them is on the path to the acceptance test above. That is +the whole test for whether something belongs in this section: **not "is it worth doing" but "does 1.0 +fail without it".** --- -## The four gates +## The gates -In order. **Do not start a gate before the one before it closes**, and closing -means a description exists in [../architecture/](../architecture/) — see -[the rule](../README.md#the-rule-that-keeps-this-from-rotting). +In order. **Do not start a gate before the one before it closes**, and closing means a description exists in +[../architecture/](../architecture/). -### Gate 0 — Fix what is already wrong - -**This comes first, before any new capability.** The desktop work is the -temptation and it is the wrong thing to do next: a compositor built on a kernel -with an unexplained hang and a socket layer that reports success for things it -did not do will fail in ways indistinguishable from its own bugs. That is the -same argument that put `netdev_selftest()` before the TCP/IP stack, and it was -right then. - -**Things that are actively wrong:** - -- [x] ~~**The `ls -lR` hang.**~~ **Not reproducible.** Instrumented and hunted; - see below. Left here rather than deleted because the absence of a result - is the result. -- [x] ~~**`setsockopt` returns 0 without setting anything.**~~ Six options now - genuinely reach lwIP's pcb and everything else is `ENOPROTOOPT`. Round-trip - checks in `cmd/socktest`, which a permissive stub cannot pass: it has to - read back a value it was set *and* refuse an unknown option. -- [x] ~~**`accept4` silently drops its flags.**~~ It is its own table entry now - rather than an alias for `accept` — an alias would have had to read a - fourth argument that `accept` never passes, inventing a `SOCK_NONBLOCK` - out of whatever the caller last left in that register. -- [x] ~~**Timestamps on everything created after boot read 1970.**~~ Both - `know()` (SchFS) and `now_sec()` (v1) returned `uptime()` under comments - saying the kernel had no wall clock. It has had one since `time_init()` - started reading the RTC; the comments outlived the condition. Two errors, - not one — `uptime()` counts 10 ms ticks and they were used as seconds, so - the clocks also ran 100× fast. -- [x] ~~**User processes are corrupted on SMP.**~~ **Found and fixed.** - `syscall_entry` parked `%rax`, `%rcx` and `%rdx` in three *globals* - across the `FS_BASE` swap, under a comment arguing that `SYSCALL` - clearing `IF` made it safe. `IF` answers for an interrupt on *this* CPU - and says nothing about a second CPU running the same three stores — and - `%rcx` is the user return address `SYSRET` uses, so the CPU that lost the - race returned its process into another process's instruction stream. - - That is the whole of it: `rip 0x0` was a process handed a zero and - following it, and `0xffffffffffffffea` was `-EINVAL` in a swapped `%rax` - used as a pointer. Per-CPU words at `%gs:32..48` now. Measured at four - CPUs against the same pristine image: **10 faults in 10 boots before, 0 - in 18 after.** See [../architecture/kernel.md](../architecture/kernel.md), - "Per-CPU state". - -- [x] ~~**`$(command substitution)` hangs the shell.**~~ **Not a kernel bug.** - `ports/zsh/Makefile` answered configure's "does `kill(pid, 0)` return - ESRCH correctly?" with `no`, which makes zsh `#define ESRCH EINVAL`. - `waitforpid()` ends its wait on `errno != ESRCH`, so with the kernel - correctly returning ESRCH (3) and zsh comparing against EINVAL (22), the - shell waited for ever on a child it had already reaped. One word in a - cross-compile cache. Autoconf's own default for that check is `yes`; - omitting the line would have been right. See - [../history/kernel.md](../history/kernel.md), section 7. -- [x] ~~**`getrusage` does not distinguish `RUSAGE_CHILDREN`** from - `RUSAGE_SELF`.~~ It did not distinguish anything: it ignored `who` and - reported the machine's *uptime* as the caller's user time, so every - process of every age answered with the same number. Per-process CPU time - is sampled at the timer tick now, children's time is rolled up on reap, - and `wait4()`'s fourth argument — a `struct rusage *` that was read off - the stack and thrown away — is filled. That last one is why zsh's `time` - reported `16777216.01s user` for everything: 2^24 seconds is - uninitialised memory, and it never varied because it was never written. - `time` now says 97% cpu for a busy loop and 0% for `sleep 2`. - `ru_stime` is a known zero; see - [../architecture/kernel.md](../architecture/kernel.md). -- [x] ~~**`/proc//exe` is a regular file, not a symlink.**~~ - `lrwxrwxrwx ... exe -> /bin/schinit`, and `readlink` returns it. It also - needed a real `st_size`, which every other procfs file deliberately - reports as zero: `readlink(2)` bounds its copy by it, so a symlink of - size 0 resolves to the empty string — which looks like it worked and is - worse than the regular file was. -- [x] ~~**OVMF memory reporting.**~~ The region-list fix did close it. 1 GB - reports 1017 MB usable, 2 GB reports 2041 MB, which is the right shape: - the firmware keeps about 7 MB either way. -- [x] ~~**`ps` printed a header and nothing else.**~~ Found while checking the - `exe` fix. `ps` with no arguments shows only processes on *this* - terminal, and `/proc//stat` hardcoded `tty_nr` to 0, so nothing - could ever match — `ps -e` always worked, which is what hid it. Fields - 14-17 (utime/stime/cutime/cstime) were hardcoded zero too, so `TIME` was - `00:00:00` for everything however long it had run. - -**Work already done and not connected:** - -These are the galling ones — the replacement exists, is tested, and is unused. - -- [x] ~~**Point the console at a real typeface.**~~ It draws JetBrains Mono - through `font_ops` and `ttf.c`. The blocker was not the font: `vtcell.ch` - was **one byte** holding a CP437 index, so `Ą` (U+0104) could not be - *stored*, whatever was drawing it. The cell is a `uint32` codepoint now - and the cell size comes from the font's measured advance. -- [x] ~~**Delete the cell grid.**~~ Done as far as it should be done, and the - rest of it retired — which needs explaining, because the item as written - asked for something a terminal cannot be. - - `include/gfx/text.h` made the case against the grid in four charges: it - cannot do variable-width glyphs, cannot position text off a multiple of - eight, cannot anti-alias, and cannot hold a codepoint above 255. **All - four were about the drawing layer, and all four are now false** — glyphs - carry metrics, PSOGM positions text at arbitrary pixels, the TrueType - path blends 8-bit coverage, and `vtcell.ch` is a `uint32`. - - What is left is a grid in `vt.c`, and that one is not in the way: it is - what a terminal *is*. VT100 escapes address columns, `TIOCGWINSZ` - reports rows and columns, and every program that draws a box assumes - both. Deleting it would not modernise the console, it would stop it - being a terminal. - - So the real remainder was the duplicate: `gfx.c` walked `font_data` - itself and blitted glyphs its own way, while `text.c` did the same job - into a `struct surface` for nobody but a self-test. Two implementations - that had to agree about a cell size only one of them was told had - changed — and they did not agree, which is why every line of PSOGM once - drew the next character's bitmap underneath itself. There is one blitter - now: `gfx.c` hands `text_draw_glyph` the cell *as a surface of its own*, - so clipping a tall glyph to its line is the same bounds check that clips - to a screen. The bitmap is reached through `font_builtin()` like any - other provider, and `gfx.c` no longer includes `font.h`. -- [x] ~~**Load a font from disk**~~ and point the console at it. The built-in - one is linked in on purpose — an early panic has to be readable — so this - is about a *replaceable* font, not about having one. Drop a TrueType file - at `\schos\console.ttf` on the ESP and the console is drawn with it. - - **The loader reads it, not the kernel**, and that is the whole design - decision. The kernel could open the file once the root filesystem is - mounted — but the console has been drawing since long before that, so - switching then means re-measuring the cell and re-laying out every - terminal mid-boot, which wipes the kernel log off the screen at exactly - the moment a machine being brought up for the first time needs it. The - loader allocates the pages as `EfiReservedMemoryType`, so the E820 - handoff marks them reserved, the page allocator never offers them, and - the kernel uses the file *in place* — no copy, and no need for a - contiguous allocator the kernel does not have. -- [x] ~~**Interrupt-driven receive.**~~ The e1000's handler calls - `netdev_rx_signal()`, which ends the network thread's wait early; the frame - is still *pulled* by the thread, because an interrupt handler can neither - sleep on the buffer cache nor take the lwIP lock. The wait stays on the - tick so lwIP's timers keep their floor. Proven rather than assumed: - `/proc/netif` gained `wake_rx`/`wake_tick`, and across three pings - `rx_frames` and `wake_rx` both went 2 -> 6. - -**Structural, because everything after this compounds on it:** - -- [x] ~~**A slab or pool allocator.**~~ `kernel/core/kmem.c`. Slabs are pages - from `kalloc()` with a header at the front; free objects hold the free - list in their own first eight bytes, so an object needs no header and - freeing needs no search — the object's page base *is* its slab, and the - slab names its cache. Two interfaces: named `kmem_cache` for a subsystem - with one kind of thing, and `kmalloc`/`kmfree` over power-of-two classes - for everything else. Visible in `/proc/slab`, asserted by - `make kmem-test` on the host — the same arrangement as `schfs-objtest`, - so the bookkeeping is checked in a second rather than in a boot. - - **It does not do what this row said it would, and that is worth - recording.** The claim was that every fixed table — `NPROC`, `NSOCK`, - `NDEVICE`, `NFILE`, `NINODE`, `NBUF` — exists to work around `kalloc`. - Measured: `struct proc` is 10656 bytes, `struct socket` 4928, `struct - buf` 4272. **All three are larger than a page**, so a page-backed slab - cannot hold them whatever it does; `NINODE` is a cache and is supposed to - be bounded; and `NFILE`/`NDEVICE` are 48 and 120 bytes, where the table - costs 24 KB and 3.8 KB and the ceiling has never been reached. The tables - were not all working around `kalloc`. Freeing the big ones needs a - contiguous multi-page allocator, which is a different thing and is not - written. - - What it does fix is real and was measured rather than assumed. A pipe is - 640 bytes and came from a whole 4096-byte page, so every `|` in a shell - command burned 3.5 KB it never touched — **six pipes per page now**. And - `procread()` built its answer in a 2048-byte array on the kernel stack, - under a comment that had warned about exactly that since it was written; - its frame is 64 bytes now instead of ~2100. -- [x] ~~**SchFS on-disk format version policy.**~~ Done. `SCHFS_VERSION` for - frame changes, plus `compat`/`ro_compat`/`incompat` feature flags in what - was `reserved0` — so existing images keep every field offset and read - unchanged. `uber_check()` now distinguishes *damaged* (step over it; that - is what the ring is for) from *too new* (stop), because falling back to an - older uberblock mounts a stale filesystem and reports success. Proven on a - forged image: the old code mounted the stale root and exited 0. - Still owed: an actual upgrade step, which is only writable once there are - two formats. See [../architecture/schfs.md](../architecture/schfs.md). - -**Found while doing the above, and fixed:** - -- [x] ~~Every `AF_INET` socket failed with `ENFILE` until DHCP bound.~~ - `inet_new()` required an *address* before it would hand out a socket, and - reported the refusal as "file table overflow". Creating a socket is not - the same as having somewhere to send; a bind or connect with no route - fails on its own with an error that says so. `net_is_ready()` (lwIP up) is - now distinct from `net_is_up()` (DHCP bound). -- [x] ~~`socktest` asserted "AF_INET is refused, not faked".~~ True when written, - false since AF_INET landed — and **green the whole time**, because the - suite runs before DHCP binds and the `ENFILE` above made the refusal look - real. A test asserting a failure has to say *which* failure. - -**The suite itself:** - -Both of these were found by running `make schfs-check` rather than by reading -it, and both predate the current work — confirmed by stashing and re-running. -They are here rather than filed as nice-to-haves because a suite that is red for -known reasons stops being read, and then it is red for unknown reasons too. - -- [x] ~~**`make schfs-check` fails at baseline.**~~ Fixed, and it was not only a - test bug. The allocation map was taken through a `palloc` callback and - there was **no way to give it back** — no `schfs_space_release`, and a - comment in `schfs_space_init` reading *"caller frees what was taken"* for - an API the caller did not have. - - So the kernel leaked the whole map on **every unmount**, up to 2 MB a - time, and `schfs_space_init`'s own error path stranded a page it had - already taken because `npages` was only raised after *both* allocations - succeeded. The test's 518 leaked pages were the symptom that pointed at - it. - - The map now carries its own `pfree` so it can only be freed by the - allocator that filled it, `schfs_vol_open`/`schfs_vol_format` release any - map the volume already holds, and unmount releases. `schfs_objtest` - asserts the balance — 520 pages taken, 520 given back — so the next leak - fails the suite instead of being absorbed by it. - -- [x] ~~**31 UndefinedBehaviorSanitizer errors.**~~ Fixed, plus two the test - never reached. One root cause: a leaf's value area holds variable-length - records packed end to end, so a record begins wherever the last one ended - — a multiple of 8 only by luck — and the code read them by casting a - pointer into the block, which tells the compiler they are aligned when - they are not. - - The seven record types that live at arbitrary offsets now carry - `SCHFS_PACKED`. **No on-disk layout changed**, and that was measured - before it was applied rather than assumed: every one already had explicit - `pad` fields, so sizes and member offsets are identical with and without - the annotation. Seven `_Static_assert`s now pin those sizes, because a - record that silently grows misreads every existing volume from that point - on and looks like corruption rather than like a struct that changed. - - `schfs_node_hdr`, `schfs_item`, `schfs_kptr` and `schfs_uberblock` are - deliberately left alone — the node header is exactly 32 bytes, so - everything after it is already 8-aligned, and annotating them would cost - byte-wise access on the tree's hottest paths to fix a problem they do not - have. - - Verified by reading an image written *before* the change: full - `schfs_fsck --check` passes on structure, checksums, key ordering, data - blocks and derived free space. - - Also caught on the way: `mkschfs` passed a null pointer to `qsort` for an - empty directory. Every implementation ignores it when the count is zero; - the standard does not say they have to. - -**The identity item:** - -- [x] ~~**Boot from a USB disk.**~~ The root device is probed at boot instead of - compiled in: ramdisk (only in a `RAMDISK_ROOT` build), then USB, then the - IDE disks, first SchFS volume wins. Verified with **no IDE disk attached - at all** — boots, mounts the stick, writes a file, and the file is still - there on the next boot. `dmesg` says `root: USB disk (device 3)`, and a - machine with no root names every candidate it looked at instead of - panicking with "no root filesystem". See - [../architecture/boot.md](../architecture/boot.md). - -#### The `ls -lR` hang, and why it is closed - -Hunted with the instrumentation it asked for, under a workload heavier than the -one that originally triggered it — ten iterations of `ls -lR /bin`, interleaved -with writes and `cd`, 33 execs and 13378 tree descents. Measured across it: - -``` -inode_peak 9 / 200 unchanged -file_peak 9 / 512 unchanged -buf_pinned_peak 1 / 256 unchanged -schfs0 locked 0 tx_owner_pid 0 tx_depth 0 -``` - -Nothing climbed, nothing wedged, and the repaired `holdingsleep` assertion never -fired. **That eliminates this file's own stated hypothesis** — "a lock or -reference-count leak somewhere in the `*at()` paths" — and it is very likely the -bug was fixed in passing by the `*at()` dirfd rewrite, the getdents cookie -rewrite, or SchFS becoming the root, all of which landed after it was reported. - -It is closed rather than chased further, and that is a deliberate application of -the two-things-in-flight rule. The instrumentation is permanent: if it ever -happens again, `/proc/fsstat` names the table, or shows `locked 1` with -`tx_owner_pid 0` and the volume lock is orphaned. That is a much better position -than the one it was reported from. - -Two real defects were found on the way and fixed, neither proven to be the -cause: `holdingsleep()` never compared the owning pid, which disarmed all seven -`panic()` assertions built on it; and SchFS's `tx_mine()` compared a bare -`struct proc *`, which a recycled ptable slot makes ambiguous. - -### Gate 1 — The foundations - -[foundations.md](foundations.md), items 1–4. Descriptor passing, a shareable -memory object, `AF_UNIX` in the filesystem namespace. - -Nothing above needs these; everything below stops without them. - -- [x] ~~**Item 1 — `sendmsg`/`recvmsg` with `SCM_RIGHTS`.**~~ Built and tested; - fourteen checks in `cmd/socktest`, each moving data through the passed - descriptor rather than inspecting it. Described in - [../architecture/networking.md](../architecture/networking.md#descriptor-passing). - This was also mechanism 1 of [hybrid.md](hybrid.md) — one primitive for - both, which is the whole reason those are one plan. -- [x] ~~**Item 2 — a shareable memory object.**~~ `memfd_create`, `ftruncate`, - `mmap(MAP_SHARED)`. Built on the `shmget` segments exactly as - [foundations.md](foundations.md) proposed — the pages, refcounts, - `no_free` VMAs and the fork/exit accounting were all already right; what - memfd adds is a segment reached by *descriptor* rather than by name or - System V key, which is the whole reason it exists on Linux. - `cmd/memfdtest`: seventeen checks, ending with a memfd sent over a - socket to a receiver that closes every inherited copy first. -- [x] ~~**Item 3 — `AF_UNIX` sockets in the filesystem namespace.**~~ `bind()` - creates a `T_SOCK` node; `connect()` resolves the path and matches the - socket bound to *that inode*, not to a string. `ls -l` shows - `srwxr-xr-x`, `stat()` reports `S_IFSOCK`, `unlink()` removes it, and a - second bind to a live path is `EADDRINUSE` *from the directory* — which - is what makes a socket file left by a crashed server refuse a rebind, as - it must. `cmd/socktest` section 15. `flock()` is still missing and is - recorded as such. - -### Gate 2 — Init as a role — **done** - -Milestones I0–I3, all built. PID 1 forks a supervisor and spawns jobs on its -behalf; the job table lives in `/proc/schinit`; a heartbeat runs both ways; and -`proc_reincarnate()` re-execs PID 1 in place rather than panicking. Both failure -paths are exercised by `sysrq` keys rather than reasoned about. - -Before the desktop rather than after, because a compositor being brought up for -the first time crashes constantly, and something that restarts it and says why is -worth far more during that work than at the end of it. - -The description is now [../architecture/init.md](../architecture/init.md); the -plan document it replaced has been deleted, per -[the rule](../README.md#the-rule-that-keeps-this-from-rotting). - -### Gate 3 — The desktop - -[xorg.md](xorg.md): Xorg on the framebuffer, twm and xterm, then Window -Maker. Wayland ([wayland.md](wayland.md)) is not being worked on -- what it -needs, DRM/KMS above all, is far larger than what X needs. - -**W4 and W5 are not in 1.0.** Multi-window focus policy and a panel are -post-1.0. One terminal window that works is the bar. +| Gate | What | State | +|---|---|---| +| **0 — Fix what is already wrong** | the SMP register corruption, `setsockopt` that lied, timestamps that read 1970, the `ls -lR` hang (hunted, not reproducible — [history](../history/kernel.md) §24), … | **closed** | +| **1 — The foundations** | descriptor passing, a shareable memory object (`memfd`), `AF_UNIX` in the filesystem, dynamic linking, the event descriptors | **closed** — [kernel.md](../architecture/kernel.md), [networking.md](../architecture/networking.md) | +| **2 — Init as a role** | `schinit`, the supervisor, the heartbeat, PID 1 re-exec'd in place | **closed** — [init.md](../architecture/init.md) | +| **3 — The desktop** | Xorg on the framebuffer, Window Maker, a terminal | **closed** — [desktop.md](../architecture/desktop.md) | +| **4 — A root that survives a reboot** | the open decision above | **open** | --- ## Not in 1.0 -The point of this list is that it is *written down*, so it can stop occupying -attention. Nothing here is cancelled. +The point of this list is that it is *written down*, so it can stop occupying attention. Nothing here is cancelled. | | Why it waits | |---|---| -| **The rest of the hybrid kernel** ([hybrid.md](hybrid.md)) | Still not a completeness requirement — but the mechanisms and the first eviction happened early, because the alternative to a driver compatibility layer is writing a driver for every device that exists, and that is not a plan. What is deferred is the foreign driver host, not the interface it needs. | -| **Mesa, SDL3** | The kernel and musl do dynamic linking now (`cmd/dynhello`), but every shipped program is still static. Mesa is the single biggest post-1.0 unlock and it is a whole project. | -| **A page cache** | `memfd` covers the desktop's needs. General writable `MAP_SHARED` of a file is a real project with no 1.0 consumer. | -| **`.sip` and the installer** | 1.0 installs by writing a USB stick. That is how most systems start and it is honest. | -| **SchFS snapshots, FUSE, Linux module, sparse files, online fsck** | The format version (Gate 0) is the prerequisite for all of them, and it is the only part on a clock. | -| **The interactive crash screen** | PSOGM renders, prints a full report and emits a QR code. That is enough. | +| **GPU acceleration, Mesa** | [gpu.md](gpu.md). A whole project; software rendering is enough to use the machine. | +| **A page cache** | General writable `MAP_SHARED` of a file is a real project with no 1.0 consumer. | +| **SchFS snapshots, FUSE, a Linux module, sparse files, online fsck** | The format version is the prerequisite and it exists; the rest is [roadmap.md](roadmap.md) §6. | | **CFF/OTF fonts, kerning, text layout** | TrueType and fixed advances are enough for a terminal. | -| **Routing, `getsockname`/`getpeername`, IPv6** | One interface, one default route, works. | -| **Multiple outputs, damage tracking, subsurfaces, xkbcommon** | One screen, full repaints, one keymap. | -| **Audio** | Does not exist in any form and is not required to use a machine. | -| **Panel, notifications, settings, file manager** | A terminal is a desktop. | -| **SchAPI itself** | Deliberately. It should be designed *from* the compositor's experience, not before it. | +| **Routing, IPv6, Wi-Fi** | One interface, one default route, works. | +| **HDA audio, recording, mixing** | AC'97 plays; that is sound. | +| **Threads' loose ends, `ptrace`, `seccomp`** | Whatever a package needs is found by running it. | +| **SchAPI itself** | Deliberately. It should be designed from the desktop's experience, not before it. | --- ## The rule for this list -**Adding to Gate 0–3 requires deleting something else from it.** +**Adding to the gates requires deleting something else from them.** -Not a productivity trick — a forcing function. The reason this project feels -overwhelming is that ~20 fronts have been open simultaneously, and no amount of -discipline makes that workable. The fix is structural. +Not a productivity trick — a forcing function. The reason this project felt overwhelming is that ~20 fronts +were open simultaneously, and no amount of discipline makes that workable. The fix is structural. And one working limit, worth holding to literally: > **Two things in flight. Never three.** -One in Gate 0 and one being written up. Everything else is closed — in the -document, and out of your head. - ---- - ## How you will know it is going well -Not by velocity. By this: - -**Every closed item leaves a description behind.** When the `ls -lR` hang is -fixed, `history/kernel.md` gains an entry saying what it actually was. When the -console moves onto the text layer, `architecture/graphics.md` loses a line from -its *What it does not do* section. - -Those "does not do" sections shrinking is the real progress bar, and it is the -only one that does not lie. - ---- +Not by velocity. By this: **every closed item leaves a description behind.** When the `ls -lR` hang was +closed, `history/kernel.md` gained an entry. When a gap closed, a "what it does not do" section shrank. +Those sections shrinking is the real progress bar, and it is the only one that does not lie. ## A note worth keeping -This document was written because the work started to feel overwhelming, and -that is worth recording rather than tidying away — because the feeling was -about the *shape* of the work and not its quality, and the two are easy to -confuse when you are inside it. +This document was written because the work started to feel overwhelming, and that is worth recording rather +than tidying away — because the feeling was about the *shape* of the work and not its quality, and the two are +easy to confuse when you are inside it. -The evidence at the time: SMP working on real hardware after five bugs stacked -behind each other; a copy-on-write filesystem whose crash test cuts the power one -write later, 119 times, and demands a clean mount after each; a TCP/IP stack -asserted from both ends; an xHCI driver written from the specification; and a -documentation tree that records why things are strange so they do not get -un-fixed by accident. +The evidence at the time: SMP working on real hardware after five bugs stacked behind each other; a copy-on-write +filesystem whose crash test cuts the power one write later, 119 times, and demands a clean mount after each; a +TCP/IP stack asserted from both ends; an xHCI driver written from the specification; and a documentation tree that +records why things are strange so they do not get un-fixed by accident. Since then: a desktop, real Linux +programs and Linux's own drivers, running. -None of that is the output of a project in trouble. It was a project with too -many tabs open. +None of that is the output of a project in trouble. It was a project with too many tabs open. diff --git a/docs/plan/README.md b/docs/plan/README.md index 1d91491..6608be7 100644 --- a/docs/plan/README.md +++ b/docs/plan/README.md @@ -1,62 +1,47 @@ # What happens next -Everything in this directory is **unbuilt**. When a piece of it is finished it -leaves — deleted from here, rewritten as a description in -[../architecture/](../architecture/). See [the rule](../README.md#the-rule-that-keeps-this-from-rotting). +Everything in this directory is **unbuilt**. When a piece of it is finished it leaves — deleted from here, +rewritten as a description in [../architecture/](../architecture/). See +[the rule](../README.md#the-rule-that-keeps-this-from-rotting). > ## Read [1.0.md](1.0.md) first. > -> It draws a finish line and puts most of the work below on the far side of it. -> **Anything not named there is not being worked on**, and the documents in this -> directory describe more than 1.0 contains. -> -> It also holds **[the register of unfinished work](1.0.md#the-freeze-and-the-register-of-unfinished-work)**, -> which is the one place anything half-built is listed. No new capability starts -> until that register is empty, and a row leaves it only by being finished or by -> being deleted from the tree. +> It draws the finish line, says where the system stands against it, and holds +> **[the register of unfinished work](1.0.md#the-freeze-and-the-register-of-unfinished-work)** +> — the one place anything half-built is listed. No new capability starts until that register is +> empty, and a row leaves it only by being finished or by being deleted from the tree. --- -## The three current decisions - -Taken together, because they turn out to need the same foundation. +## What is here | | | |---|---| -| **[alpine.md](alpine.md)** | **Packages.** The root is managed by Alpine's apk: Alpine's binaries run unmodified, and SchOS itself is the apk package schos-base. | -| **[xorg.md](xorg.md)** | **The desktop.** Xorg on the framebuffer with the fbdev and evdev drivers; twm, then Window Maker. Replaces wayland.md for Gate 3. | -| **[wayland.md](wayland.md)** | *Not being worked on; see xorg.md.* The GUI stack becomes Wayland. SchWM and SchDE are **deleted** — actually gone from the tree, not marked; Schcom survives and changes its wire format. | -| ~~**hybrid.md**~~ | **Built.** All four mechanisms exist, the GPU driver runs outside the kernel, it survives being killed, and the display mode can be changed from a process. H4 (USB) was *withdrawn* rather than left open — USB is the bus peripherals arrive on, not a peripheral. | -| ~~init.md~~ | **Built.** PID 1 is now a role the kernel refills; the description moved to [../architecture/init.md](../architecture/init.md). | +| **[1.0.md](1.0.md)** | **The finish line.** What 1.0 is, the acceptance test and where each clause stands, the register, and everything explicitly deferred. | +| **[roadmap.md](roadmap.md)** | Everything else that is scoped and agreed: storage and the installer, graphics, networking, the hardware that makes a laptop a laptop, SchFS beyond SchOS, the kernel's own gaps. | +| **[gpu.md](gpu.md)** | Linux's own `amdgpu` through LDLS, and the `/dev/dri` a process has to serve. Measured, with the work it needs written down. | +| **[finch.md](finch.md)** | **Later.** The author's own LLVM language for SchOS's applications and tools in place of C. The language is finished and self-hosting in its own repository; the plan here is a target, bindings generated from `include/uapi/`, and the compiler in the image. | +| **[schapi.md](schapi.md)** | The long-term API layering and the compatibility promise. Direction agreed, nothing built. | -And the one they all wait on: +## What used to be here, and is built -| | | +Plans leave this directory when they are done. These did, and what replaced them is the description: + +| Was | Now | |---|---| -| **[foundations.md](foundations.md)** | Descriptor passing, shared memory, sockets in the filesystem. The critical path. **Item 1 is built** — `sendmsg`/`recvmsg` with `SCM_RIGHTS`. | +| Descriptor passing, shared memory, sockets in the filesystem, dynamic linking, the event descriptors (`foundations.md`) | [kernel.md](../architecture/kernel.md), [networking.md](../architecture/networking.md), [syscalls.md](../architecture/syscalls.md) | +| `init.md` | [init.md](../architecture/init.md) | +| `hybrid.md` — drivers in processes, `/dev/pci`, a modeset API, the panic path, why USB stays | [devices.md](../architecture/devices.md) | +| `ldls.md` — Linux drivers in a process | [ldls.md](../architecture/ldls.md) | +| `debugger.md` | [debugging.md](../architecture/debugging.md) | +| `alpine.md` — an apk-managed root | [packages.md](../architecture/packages.md) | +| `xorg.md` — the desktop | [desktop.md](../architecture/desktop.md) | +| The interactive crash screen (`roadmap.md` §4) | GMMOS, in [graphics.md](../architecture/graphics.md) | -And the one that says which parts of the above are actually being worked on: - -| | | -|---|---| -| **[1.0.md](1.0.md)** | The finish line. Four gates, and everything else explicitly deferred. | - -And the one the hybrid mechanisms were built for: - -| | | -|---|---| -| ~~ldls.md~~ | **Built.** The Linux Driver Layer for SchOS: someone else's drivers, in a process, reached through `/dev/pci`. `ping` works over `8139too` and over `e1000`. The description moved to [../architecture/ldls.md](../architecture/ldls.md). | -| ~~debugger.md~~ | **Built**, all of it, including the network transport. A debugger that observes rather than halts and answers in this kernel's own vocabulary: symbols, a log that outlives the kernel, `/dev/dbg`, `whose`, hardware watchpoints, and `schdbgd`. The description moved to [../architecture/debugging.md](../architecture/debugging.md). | - -And one for after 1.0, recorded so the decision is not lost: - -| | | -|---|---| -| **[finch.md](finch.md)** | **Later.** The author's own LLVM language, Finch, for SchOS's applications and tools in place of C. Already self-hosting; the plan is a target, bindings generated from `include/uapi/`, and the compiler in the image. | - -Two longer-horizon documents sit behind these: [schapi.md](schapi.md), the API -layering and the compatibility promise, and [roadmap.md](roadmap.md), which holds -everything still open that is not one of the three above. +And one that was **decided against**: `wayland.md`. Wayland needs DRM/KMS, libinput, udev and libseat, and DRM/KMS +alone is a kernel graphics subsystem. The desktop is X. The reasoning moved to [desktop.md](../architecture/desktop.md). +`SchWM` and `SchDE`, which were to be the window manager and the desktop on top of it, are gone from the tree: Window +Maker is the window manager, and a panel or launcher is an ordinary client. --- @@ -64,86 +49,17 @@ everything still open that is not one of the three above. Set by [1.0.md](1.0.md), which is the authority. In short: -### 0. Fix what is already wrong — **first** - -Not a document of its own; it is Gate 0 of [1.0.md](1.0.md), assembled from the -*What it does not do* sections in [../architecture/](../architecture/). - -**Before any new capability**, and the reason is the one that put -`netdev_selftest()` before the TCP/IP stack: a compositor built on a kernel with -an unexplained hang and a socket layer that reports success for things it did not -do will fail in ways indistinguishable from its own bugs. - -### 1. `foundations.md` — the critical path - -Both the Wayland work and the hybrid work stop here. Descriptor passing and -shared memory are the substrate for handing a buffer to a compositor *and* for -handing a device to a driver process. **Doing it once for both is the entire -reason these are one plan rather than three.** - -### 2. `init.md` — before the desktop, not after — **done** - -A compositor being brought up for the first time crashes constantly, and -something that restarts it and says why is worth far more during that work than -at the end of it. Built; see [../architecture/init.md](../architecture/init.md). - -### 3. `hybrid.md` — **moved up, and the reason is worth recording** - -This document used to put the hybrid work *after 1.0*, deliberately last, on this -argument: - -> designing a userspace driver interface with no driver consuming it produces an -> interface that is wrong in ways nobody can see - -The argument was sound and the conclusion drawn from it was wrong. What forced -the change was the driver situation: SchOS cannot write a driver for every device -that exists, no project can, and the only route to the hardware support an -established OS has is a compatibility layer with the foreign drivers kept -*outside* the kernel. That layer is this plan's mechanisms. It is not optional and -it is not late work. - -The original argument still holds, and it is answered rather than ignored: -**SchOS's own drivers are the consumers.** `vgpud` is a real driver, with real -requirements, and building the interface against it is what keeps it a general -mechanism rather than a hole shaped like something foreign. The compositor is -still a demanding consumer — it is no longer the *only* one available. - -Four mechanisms, all built. One eviction done, the second withdrawn with its -reasons recorded, and the foreign driver host they were for is -[../architecture/ldls.md](../architecture/ldls.md) — built, and carrying -packets. - -### 4. `wayland.md` — with the GPU driver either side of the seam - -W1–W3 and a terminal. W4 and W5 are post-1.0. Item 1 of `foundations.md` — -descriptor passing — was the thing it could not start without, and it now -exists. - ---- - -## In one sentence - -> Fix what is broken, build the descriptor-passing and shared-memory substrate, -> make init survivable, move the drivers out through an interface SchOS's own -> drivers have to live with first — and put a terminal on the screen. - ---- +1. **Close the one open gate**: decide what makes a root survive a reboot — the installer, or a narrower + definition of 1.0 (see *The one open decision* there). +2. **Empty the register.** Delete `projects/schcom/` and the unused in-kernel LDLS window, or say what they are for; build + socket activation or remove the key. +3. **Then**, and only then, whatever [roadmap.md](roadmap.md) says is next. GPU acceleration is the thing the laptop + wants most ([gpu.md](gpu.md)), and Finch inside SchOS is the thing the author wants next ([finch.md](finch.md)). ## What this plan does not address Stated here so it is not mistaken for covered: -- **No IOMMU**, so the hybrid work isolates against bugs and not against a - hostile driver. -- **No page cache**, so `foundations.md` delivers a `memfd` and not a general - answer to shared file mappings. The general answer is still owed. -- **No garbage collector for descriptors in flight**, so an `AF_UNIX` socket - passed over itself and never received leaks. Bounded at thirty-two sockets, and - nothing on this plan does it. -- **No dynamic linking**, so no Mesa, no accelerated clients, and no third party - shipping a `.so` for anything. -- **A new compatibility surface** between PID 1 and PID 2, created by a plan - whose own [schapi.md](schapi.md) §3 spends a section warning about those. -- **The three open networking items** — `getsockname`, routing, and - interrupt-driven receive — are untouched by any of this and still open. They - are in [roadmap.md](roadmap.md). +- **No IOMMU**, so a driver in a process is isolated against its own bugs and not against a hostile driver. +- **No page cache**, so a general writable `MAP_SHARED` of a file is refused; `memfd` is the narrow answer. +- **No installer**, so nothing a live system writes survives a reboot. diff --git a/docs/plan/alpine.md b/docs/plan/alpine.md deleted file mode 100644 index 5e141ab..0000000 --- a/docs/plan/alpine.md +++ /dev/null @@ -1,80 +0,0 @@ -# apk: Alpine's packages on SchOS - -Alpine is musl with a different init, and SchOS is musl with a different -kernel. Alpine's binaries run on SchOS unmodified: its dynamic linker, its -busybox, and apk itself, fetching over TLS through SchOS's network stack. -So instead of porting programs one at a time, SchOS can install them. - -## The model - -The root filesystem is an apk-managed root: - -| Part | Where it comes from | -|---|---| -| musl, busybox, apk-tools, alpine-keys, CA certificates | Alpine's minirootfs (`ports/alpine`) | -| what `ports/alpine/world` lists (zsh, terminfo, ...) | Alpine's packages, pinned in `world.lock` | -| schinit, its jobs, SchOS's tools, toybox, mksh, configuration | **schos-base**, an apk package the build makes from `sysroot/` | - -`apk add` installs anything from Alpine's main and community repositories. -`apk upgrade` updates Alpine's packages, and schos-base from the SchOS -repository at `/var/lib/schos/repo`. - -PID 1 is still schinit. Alpine's OpenRC is not installed, and packages that -ship OpenRC services install them where nothing reads them. - -## Who owns what - -Where Alpine and SchOS both have a program, Alpine's wins, and SchOS keeps -only what is its own (see `tools/posix/mkaroot.sh`): - -- `/bin/sh`, `cat`, `sed`, `find` and the rest are busybox. Alpine's package - scripts are written against busybox. -- zsh is Alpine's. SchOS keeps its `zshrc` and `zshenv` (`replaces=zsh`). -- `/etc/os-release` and `/etc/motd` are SchOS's (`replaces=`). -- `passwd`, `group` and `profile` are Alpine's. schos-base's post-install - sets root's shell to zsh, and SchOS's profile is `/etc/profile.d/schos.sh`. -- mksh is `/bin/mksh`. schinit's recovery shell looks for it first because it - is static: a broken musl cannot take recovery down with it. - -## Building - - make qemu boots build/schfs.img, the root as a 1 GB SchFS disk - make iso the same root as the live system's ramdisk - make alpine-lock re-resolve world against Alpine (needs the network) - -It is the only root. sysroot/ is what schos-base is packaged from. - -Only `alpine-lock` asks Alpine what is current. A normal build installs -exactly what `world.lock` names: files in `dist/alpine/`, fetched once and -checked against the lock's sha256, and against Alpine's signatures by apk. - -schos-base and its repository index are signed with a key made on the first -build (`build/apk-keys/`, never committed). The root trusts it beside -Alpine's keys. - -## Kernel work this needed - -All of it found by running apk: - -- **TCP receive window:** it reopened before the reader drained, so lwIP - refused data and the sender waited on retransmission timers (5 KB/s). It - now reopens on read, with a 32-segment window. -- **mmap address reuse:** freed ranges were never reused, so musl's malloc - used up the 256 mapping records with 780 MB of RAM free. -- **unlinkat / renameat:** bare `-1`, read as EPERM, where apk expects ENOENT; - renameat could not replace its target, and wrote v1 dirents into SchFS. -- **`#!` scripts in execve**, and ENOEXEC: apk runs triggers directly. -- **chroot**, and mounting devtmpfs and proc more than once. - -## Next - -- The live ISO runs the apk root from RAM. The image in the kernel is only - what the root holds; at boot the volume grows into half of the free memory - (4 GB of root on an 8 GB machine), backed by pages as they are written. - Xorg installs into it in about a minute. All of it is gone at reboot. Keeping packages needs an installer that writes the root - to a SATA or NVMe disk, and a kernel that finds its root there by the - SchFS GPT type rather than as IDE disk 1. -- Xorg and Window Maker are in `world`, so the image starts with them: - `startx` runs `wmaker` from `/root/.xinitrc`. Mesa brings LLVM with it, - about 200 MB of the 358 MB root; the root ships compressed - ([SCFF](../architecture/scff.md)), so that costs about 60 MB of ISO. diff --git a/docs/plan/finch.md b/docs/plan/finch.md index 219a66b..f6df53b 100644 --- a/docs/plan/finch.md +++ b/docs/plan/finch.md @@ -8,8 +8,8 @@ Finch repository and does not touch this tree. It can proceed now; see [the gap](#the-self-hosting-gap). -Finch is the author's own language: -(GitHub mirror: `Nexoniarz/Finch`). The plan is to write SchOS's own +Finch is the author's own language: (mirrored to +`pandor.pl/git/nexoniarz/Finch`, which is read-only). The plan is to write SchOS's own applications and tools in it, in place of C, and so leave behind what C does badly: silent narrowing, signed/unsigned mixing, undefined behaviour on an out-of-range index, a forgotten `free`. @@ -45,7 +45,7 @@ Measured from the repository, not from its README alone: 1. **A `schos` target.** `finch` picks a triple, a C compiler and a linker per target. SchOS runs unmodified Alpine binaries and Alpine's musl - (see [alpine.md](alpine.md)), so the first target is **the existing + (see [packages](../architecture/packages.md)), so the first target is **the existing `x86_64-unknown-linux-musl`**, and a Finch program built on the host should run on SchOS before anything in Finch is made SchOS-specific. Test that before writing a line of target code. diff --git a/docs/plan/foundations.md b/docs/plan/foundations.md deleted file mode 100644 index 1e0252d..0000000 --- a/docs/plan/foundations.md +++ /dev/null @@ -1,199 +0,0 @@ -# Foundations - -The kernel work that [wayland.md](wayland.md) and [hybrid.md](hybrid.md) both -stop at. **This is the critical path**, and it is the reason those two are one -plan rather than two. - -The insight is small and it decides the schedule: the substrate for passing a -buffer to a compositor and the substrate for handing a device to a driver process -are the same substrate. Descriptor passing and shared memory, built once. - ---- - -## What is actually in the tree - -Measured against the source, not against a README. Each of these was checked -before anything below was planned around it: - -| | | -|---|---| -| ~~`sendmsg`/`recvmsg`, and therefore **`SCM_RIGHTS`**~~ | **Built.** See item 1 below. | -| Writable `MAP_SHARED` of a file | **Refused** — `kernel/sys/sys_mem.c:131` returns `-ENODEV`. | -| `memfd_create` | Does not exist. | -| `AF_UNIX` names | In a table in `kernel/net/unix.c`, not in the filesystem. | -| `setsockopt` | `return 0;` unconditionally — `kernel/sys/sys_net.c:436`. | -| `exec` | `ET_EXEC` only. No `PT_INTERP`, no `ET_DYN`. | - -> **`projects/schcom/README.md` said `SCM_RIGHTS` was done while nothing in the -> kernel implemented it** — the most load-bearing wrong line in the tree, with an -> entire GUI plan drawn on top of it. The line is now true, which is not the same -> as it having been right, and the README says so. - -The `MAP_SHARED` refusal is worth defending rather than fixing carelessly. The -code returns `-ENODEV` *honestly*: the contract of a writable shared mapping is -that stores land in the file and other mappers see them, and there is no page -cache that could make that true. Quietly handing back a private copy would be the -same lie in a new place. - ---- - -## The six items - -In dependency order. The first three are what Wayland cannot start without. -All but item 4 are built. - -### 1. `sendmsg` / `recvmsg` with `SCM_RIGHTS` — **done** - -Not optional and not substitutable. `wl_shm_create_pool` passes a **file -descriptor**; there is no path in the Wayland protocol that passes an integer -instead. - -The `handle_kind` field in `projects/schcom/proto.h` was good foresight and it -cannot help here — *our* protocol is the flexible one, and Wayland's is not. - -Built as described: `struct msghdr`/`cmsghdr` in Linux's layout, ancillary data -queued alongside the byte stream in `kernel/net/unix.c`, descriptor installation -into the receiver's file table, and the three ways a message can end — received, -received without room, never received — each releasing the kernel's reference. -Fourteen checks in `cmd/socktest`, every one of them moving data through the -descriptor rather than inspecting it. Documented in -[../architecture/networking.md](../architecture/networking.md#descriptor-passing). - -The delivery rule is the part worth knowing before building on it: **descriptors -are pinned to the stream position of the first byte of the message that carried -them**, and a read that has not reached that position is cut short so it stops -before it. That is what keeps "which message did these belong to" answerable. - -**This was also item 1 of the hybrid work**, where it is how a driver process is -handed the device it is allowed to touch. - -Two bugs fell out of building it, neither in the new code: - -- `sockpair()` never set the socket's domain, so **every `socketpair()` socket - claimed domain 0**. Invisible for as long as the data paths asked only whether - a socket was `AF_INET` or `AF_NETLINK` — zero answers "neither" correctly. - `SCM_RIGHTS` asks "is this `AF_UNIX`", and zero is wrong for that one. -- `pipe_poll()` reports `POLLHUP` but not `POLLIN` for a pipe at end of file, so - a **non-blocking read of a pipe whose writers have all gone returned `EAGAIN` - forever** instead of 0. A program that polls and reads would spin waiting for - an end that had already happened. `sockreadable()` always had the equivalent - condition; the pipe path did not. - -### 2. A shareable memory object — **done** - -`memfd_create`, plus `mmap(MAP_SHARED)` on it genuinely working between two -processes. Described in -[../architecture/kernel.md](../architecture/kernel.md#shared-memory-objects); -asserted by `cmd/memfdtest`, seventeen checks including the one that matters — -a descriptor sent over a socket, mapped by a receiver that has closed every -inherited copy, with stores visible in both directions. - -Narrower than a general page cache, and sufficient. An anonymous shared object -needs no file behind it, which is exactly why `memfd` exists on Linux at all — -and it means this can be built on the shared-memory machinery already in -`kernel/sys/sys_mem.c` (`shmget`/`shmat`/`shmdt` work, and `struct vma` already -carries `shmid` and a `no_free` flag for pages that are not ours to release). - -The general answer — a page cache, and writable `MAP_SHARED` of a real file — -stays owed. It is a larger project and nothing on this plan needs it. - -### 3. `AF_UNIX` sockets in the filesystem namespace — **done** - -Clients find a Wayland compositor by `bind()`ing `$XDG_RUNTIME_DIR/wayland-0` as -a real path, and libwayland `flock()`s the lock file beside it. A name table can -express neither. - -`bind()` creates a `T_SOCK` node and `connect()` resolves the path and matches -the socket bound to **that inode** — not to a string, which is what made two -spellings of one path miss each other. `ls -l` shows `srwxr-xr-x`, `stat()` -reports `S_IFSOCK`, and `unlink()` removes it. Asserted by `cmd/socktest`, -section 15. `flock()` is still not implemented; see below. - -`kernel/net/unix.c`'s own header comment already predicted this: - -> *"What is lost is `ls -l /tmp/schcom` showing an `s`, and `unlink()` of a socket -> path failing with ENOENT. Both are worth fixing when the VFS next gets opened -> up; neither blocks anything."* - -They block something now. This is that moment, and it is a VFS change rather than -a socket change: a socket inode type threaded through `iget`, `dirlookup` and -`stat`, plus `flock`. - -### 4. `setsockopt` that does not lie - -At minimum `SO_PASSCRED` and `SO_SNDBUF`. Returning 0 for an option you ignored -is worse than `ENOPROTOOPT`, because the caller adjusts its behaviour on the -strength of the lie. - -Small, and grouped here because it is in the same file as item 3 and will be -found by the same testing. - -### 5. Dynamic linking — **done** - -`PT_INTERP` and `ET_DYN` in `kernel/fs/exec.c`. A program that names an -interpreter gets it loaded at the bottom of the mmap area and started instead, -with `AT_PHDR`, `AT_PHENT`, `AT_PHNUM`, `AT_BASE` and `AT_ENTRY` to find the -program by; a PIE program is placed where a fixed one would be. The interpreter -is musl's own: `ports/musl` now builds `libc.so` as well, shipped as -`/lib/ld-musl-x86_64.so.1`, which is libc and loader in one file. - -The kernel half that was missing besides `exec` was `mmap(MAP_FIXED)` of a -file. It handed back zeroed pages whatever the descriptor said, and kept no -record of them for `fork` — and a dynamic linker maps every library segment -after the first exactly that way. - -Every port still links `-static`, on purpose: this makes the other kind -possible, it does not convert anything. `cmd/dynhello` is the test — a -program against `libc.so` and a library of its own, checking TLS, the heap and -a forked child that still has its libraries. - -`mprotect` is accepted and not enforced, so RELRO is not read-only yet. - -### 6. The event descriptors — **done** - -`epoll`, `eventfd`, `timerfd` and `signalfd`, in `kernel/sys/sys_event.c`. -libwayland's event loop is all four in one epoll set, which is why they arrive -together; not in the original list because they were stubs that looked like -syscalls, returning `ENOSYS` from `table.inc` entries that read as done. - -They wait the way `poll()` does, a tick at a time, so the latency is 10 ms. -`cmd/evtest` checks each one, then all four in one set, level- and -edge-triggered. - -Building it found a signal bug: a default-fatal signal killed on arrival even -when it was blocked. A blocked signal now stays pending, which is the contract -`signalfd` is built on. - ---- - -## Verification - -Each item gets a test before the thing that needs it is built, because a -substrate that is subtly wrong fails inside the consumer and looks like the -consumer's bug. - -- **Item 1:** two processes, one passes an open descriptor, the other reads - through it. Plus the leak case: a sender that exits with descriptors in flight. -- **Item 2:** two processes map the same `memfd`, one writes, the other sees it. -- **Item 3:** `bind()` a socket, then `stat()` and `ls -l` the path and see an - `s`, then `unlink()` it. -- **Items 1–3 together:** a third test that does all of it at once, because that - combination is what Wayland actually performs and the three passing separately - does not prove it. - -These belong in `bootcheck` as a new switch, following the `-Sock` model. - ---- - -## What this does not deliver - -- **No page cache**, so writable `MAP_SHARED` of a real file stays refused. -- **No `dlopen`** until item 5 is done, which means no Mesa and no third-party - shared libraries. -- **`SO_*` coverage is the two options named**, not the full set. -- **No credential passing** (`SCM_CREDENTIALS`) — only descriptors. Nothing on - this plan needs it, and a compositor that wants to know who connected will. -- **No `flock()`.** libwayland locks `wayland-0.lock` beside the socket to stop - two compositors claiming one display. The socket itself is now a real path, - so the lock file can be created and opened; what is missing is the advisory - lock, which is a separate mechanism and has no other caller yet. diff --git a/docs/plan/gpu.md b/docs/plan/gpu.md index 481ce29..cd78da8 100644 --- a/docs/plan/gpu.md +++ b/docs/plan/gpu.md @@ -16,7 +16,7 @@ blobs may be shipped. That supersedes LDLS's rule 1. FreeBSD's drm-kmod compiles Linux's DRM source into kernel modules against a shim (linuxkpi). SchOS has already chosen the other route, and for good reasons -([hybrid.md](hybrid.md)): GPU drivers live outside the kernel, a crashed driver +([devices.md](../architecture/devices.md)): GPU drivers live outside the kernel, a crashed driver falls back to the framebuffer instead of taking the machine down, and LDLS loads unmodified `.ko` files against the running kernel's BTF instead of needing a source port per Linux release. vgpud already drives a GPU from a @@ -77,8 +77,8 @@ descriptors. The kernel has to forward each of those to the driver process: physical ranges (VRAM through the BAR, or GTT pages from the arena), and the kernel maps those into the client's address space. - **dma-buf** becomes a kernel file object naming a buffer the server owns. - `SCM_RIGHTS` passes it between processes (already built, foundations.md - item 1), and X's DRI3 needs exactly that. + `SCM_RIGHTS` passes it between processes (already built; see + [networking.md](../architecture/networking.md#descriptor-passing)), and X's DRI3 needs exactly that. - **sysfs**: `/sys/class/drm/card0`, `/sys/dev/char/226:*`, and the vendor/device files that libdrm and Mesa read to pick a driver. diff --git a/docs/plan/hybrid.md b/docs/plan/hybrid.md deleted file mode 100644 index a26c4c9..0000000 --- a/docs/plan/hybrid.md +++ /dev/null @@ -1,558 +0,0 @@ -# The kernel becomes hybrid - -"Hybrid" is used to mean everything from "a monolith with a nice device model" to -"a microkernel that gave up". **Drawing the line precisely is most of the work**, -so that is what this document is. - -The device model it builds on is -[../architecture/devices.md](../architecture/devices.md). It is last on the -plan, and the reason is in [README.md](README.md#the-order-and-why-it-is-not-the-order-they-are-listed-in). - ---- - -## The line - -### Stays in the kernel, permanently - -- scheduling, address spaces, the trap and syscall path -- IPC and shared memory — the mechanism everything else is built out of -- the VFS and the **root** filesystem -- IRQ demultiplexing -- the console and PSOGM - -**That last one is not a size compromise, it is an invariant.** The crash screen -must work when everything else in the system is dead, which means it cannot -depend on anything that could *be* dead. A crash screen served by a userspace -process is a crash screen that is absent in exactly the case it exists for. - -### May leave, and should - -- GPU drivers -- USB, and every USB class driver under it -- non-root filesystems -- NICs after the first -- input transformation — layouts, gestures, remapping - -### The rule that separates the columns - -> **Anything needed to reach a shell stays in. Anything discovered at runtime, -> replaceable, or plausibly written by someone else goes out.** - -That is NT's compromise and macOS's, arrived at from the same pressure, and it is -the one that survives contact with a real machine. - ---- - -## Built so far - -**Mechanisms 2 and 4 exist.** `/dev/pci` (include/dev/pciuser.h, kernel/dev/pciuser.c) -enumerates the bus, hands a device to a process exclusively, maps a BAR into it -and allocates memory a device can be pointed at. `pcitest` proves it end to end: - -``` - # bdf vendor:device class driver - 6 00:04.0 8086:100e network e1000 - 7 00:07.0 8086:10d3 network - - -device 7: 8086:10d3 (network) - claimed - BAR0 at 0x0000000081060000, 128 KB - regs[0..3] = 0x00140261 0x00000000 0x00080283 0x00000000 - MMIO reaches the device - DMA 8 KB at virtual 0x40001000, physical 0x3f8eb000 -``` - -Those register values are the e1000e's CTRL and STATUS, with full-duplex and -link-up set -- which is how you tell a working mapping from one that reads -0xFFFFFFFF off an unmapped address. Claiming a device the kernel drives is -refused, and everything a process held is released when it exits, because a -driver that crashed must not keep its hardware away from its replacement. - -**Mechanism 3 exists too.** A process asks for a device's interrupts and then -reads them, uio-style: - -``` - interrupts routed here (line 10) - interrupt delivered, count = 1 - ICR = 0x80000004 -``` - -`0x80000004` is the e1000e's cause register with the bit the test asserted and -INT_ASSERTED set, read back by userspace as its acknowledgement. - -The line is **masked by the kernel on arrival and unmasked by userspace's -write()**, and that is the entire design rather than a detail. A PCI interrupt -line is level-triggered: it stays asserted until the cause is cleared *at the -device*, which is a different register on every chip and therefore something -only the driver can do. Signalling EOI on a line that is still asserted -re-enters the handler immediately and for ever -- 72,000 times a second, when -this kernel last did it by accident (history/kernel.md §13). - -So a driver that dies mid-interrupt leaves the line **masked, not unmasked**. -The instinct is the other way round -- do not leave the device dead -- and it -is wrong: unmasking an asserted line with no handler attached is a crashed -driver taking the machine down with it. Verified by killing one on purpose: - -``` - interrupt asserted; exiting WITHOUT acking (simulated crash) - timer: 3431 ticks taken, 3436 expected -``` - -Five ticks lost in seven seconds, which is the ordinary background. The next -driver to claim the device re-arms the line, so masked is safe without being -permanent -- also verified. - -## The first eviction happened - -`cmd/vgpud` is the virtio-gpu driver, outside the kernel. It finds the device -by vendor and device id, asks for it, and the kernel driver hands it over: - -``` -vgpud: virtio-gpu at 00:03.0, currently held by virtio-gpu -virtio-gpu: released to userspace; screen is the VBE framebuffer -vgpud: claimed -vgpud: display is 1280x800 -vgpud: scanout is ours, framebuffer at physical 0x3eceb000 -vgpud: drew a test pattern from userspace -``` - -Full virtio 1.0 bring-up from a process: reset, feature negotiation, virtqueue -in DMA memory, GET_DISPLAY_INFO, RESOURCE_CREATE_2D, ATTACH_BACKING, -SET_SCANOUT, and TRANSFER/FLUSH to present. The pattern it draws is on the -virtio-gpu head while the console carries on in VBE on the other one -- which -is the fallback rule working rather than being described. - -**No syscall knows this is a GPU.** The kernel handed over a PCI device; the -process decided what it was. That is the property that makes the interface -general, and it is why the evictions come before anything foreign. - -### What the first real driver asked for that a test program had not - -- **PCI configuration space** (`PCIIOC_CFG`). A virtio device says where its - registers are in *vendor capabilities*, which live in config space -- so a - driver that cannot read it cannot find its own registers. Anything modern is - discovered the same way. -- **A way for the incumbent to let go.** `struct pci_driver` gained a - `release` hook. A driver without one keeps its device, which is always safe - and, for anything on the path to a shell, correct. -- **The outgoing driver freeing its DMA.** Releasing the device while keeping - its 4 MB framebuffer is half a handover: the incoming driver asks for one and - is told there is no room. Found by doing it. -- **A bigger arena.** 4 MB was exactly one framebuffer at this resolution, and - for the moment either side of a handover there are two. 16 MB now. - -### And one bug it reproduced immediately - -The first working run raised this: - -``` -irq 11: 77780 interrupts/second -- a device on this line is not acknowledging it. -irq 11: handler 'e1000' is attached -``` - -`vgpud` polls the used ring, so it registered no handler -- and a virtio device -interrupts on every completion. A level-triggered line nobody acknowledges -re-fires for ever, drowning the e1000 that shares it. The fix is one line, -`VRING_AVAIL_F_NO_INTERRUPT` before the queue is enabled, and the value of it -is that the storm detector built for history/kernel.md section 13 named the -line, the rate and the neighbour without anyone having to look. - -**All four mechanisms now exist**, and with H3 done and H4 withdrawn, nothing -remains on this plan. What the mechanisms were built for next is the foreign -driver host -- [ldls.md](ldls.md). - -### One thing the interface is deliberately not - -It is about **PCI devices**, not about any driver model, and its first users -will be SchOS's own drivers. That ordering is the design: an interface whose -first consumers are native drivers is a general mechanism that something -foreign can use later, rather than a hole shaped like something foreign. The -difference matters technically -- a Linux-shaped interface would be wrong for -the evictions this plan actually calls for -- and it happens to be the only -version that survives a licensing conversation. - ---- - -## The driver came back - -H2 is the milestone this plan calls the one that cannot be skipped, and the -reason is that H1 proves only half of the idea. A driver that runs in a process -is interesting; a driver that runs in a process, *dies*, and is replaced by -another process that picks the device up is the thing the whole hybrid argument -rests on. - -`etc/schinit/vgpud.job` makes it a supervised job -- `KeepAlive = yes` -- so -killing it is the whole test: - -``` -# cat /proc/schinit -1 4 11 os.schos.vgpud -# kill 4 -schinitd: restarting os.schos.vgpud -vgpud: virtio-gpu at 00:03.0 -vgpud: claimed -vgpud: display is 1280x800 -vgpud: scanout is ours, framebuffer at physical 0x3eceb000 -vgpud: pid 8 drew a test pattern from userspace -``` - -The test pattern ends in a twelve-block tally of the drawing process's pid, -because a second run that redraws an identical picture is indistinguishable -from a first run that never stopped. Read off the virtio-gpu scanout across -four generations of the driver it says 4, 8, 10, 12. The same physical address -comes back every time, which is the DMA arena reclaiming cleanly rather than -drifting. - -### What it took that H1 had not needed - -- **A halt the kernel can apply without knowing what the device is.** A driver - that dies leaves its device bus-mastering into memory the kernel is about to - hand to somebody else. The kernel cannot quiesce a device it does not - understand, but it can clear `PCI_CMD_BUS_MASTER`, and that is enough to stop - the writes. `pciuser_stop()` does it before the DMA is freed, not after, and - claim/release became a symmetric pair: claiming sets the bit, releasing - clears it. `pcitest` grew a `bm` column so the state is visible rather than - assumed. -- **A refusal, when there is nothing to fall back to.** `vgpu_release()` now - asks `vga_fb_restore()` whether there is a framebuffer to put the console on, - and declines to hand the device over if there is not. The rule that the - kernel keeps a minimal driver for everything on the path to a shell is only - worth writing down if something enforces it. -- **The DMA arena not being handed to two allocators at once.** This is the bug - that stopped H2 for a day, and it is worth writing out: `freevm()` frees - every mapped page below `PHYSTOP`, and the DMA arena is the top 16 MB of - exactly that range. So a DMA block mapped into a driver and left for - `freevm()` went onto the page allocator's free list when the driver died, - while `pciuser_release_all()` handed the same pages back to the arena. Both - allocators then believed they owned it. The restarted `vgpud` asked for its - 4 MB framebuffer, got pages `kalloc()` had already given out -- including its - own page tables -- and `dma_alloc()`'s zeroing wiped them, so the driver died - inside the ioctl, on the return from a syscall whose stack had stopped - existing. `do_dma()` now records a VMA with `no_free` set, the same way - shared memory does, so `proc_release_shared()` unmaps the range before the - page tables are torn down. - -### The clause that is not true yet - -The milestone says *"the compositor reattaches"*. It does not, because there is -no compositor running: Schcom is v0 and is not started at boot. What has been -shown is the two parts that do not depend on it -- the screen falls back to -VBE, and the driver restarts and takes the device back. During the gap the VBE -head carries a live console and the virtio-gpu head still shows the dead -driver's last frame, which is exactly what a device nobody is driving does. - -(That last sentence is still true of a *crashed driver*, and no longer true of -a crashed *kernel* -- see "The clause nobody wrote down" below.) - -Reattachment is a Schcom milestone, not a hybrid one, and it is recorded that -way rather than counted here. - ---- - -## A modeset, from a process - -H3 is one sentence in the milestone table -- *"a modeset API, which is the -thing an out-of-kernel GPU driver makes affordable and an in-kernel one never -got"* -- and the claim in it is worth testing rather than repeating. - -``` -# schmode -1280x800 current -1280x800 maximum, framebuffer 4000 KB -# schmode modes -1280x800 * -1280x720 -1024x768 -800x600 -640x480 -# schmode 1024 768 -1024x768 ok -``` - -The scanout really resizes: captured from the virtio-gpu head, the frames come -back 1280x800, then 1024x768, then 640x480, then 1280x800 again, with the -checkerboard band showing no shear at any of them. - -**Nothing in the kernel changed for this.** No syscall on the path has heard of -a scanout; `schmode` writes a line to a socket and `vgpud` sends five commands -to the device. That is the whole of the milestone, and the reason it is -interesting is the comparison: the same five commands in the kernel would have -been the same five commands *and a reason to be nervous*, because getting the -sequence wrong there takes the machine instead of one process. **That is why -the in-kernel driver never grew one** -- not difficulty, exposure. - -### What it costs, and what it refuses - -The framebuffer is allocated **once, at the largest mode**, and every smaller -mode reuses it. `PCIIOC_DMA` has no free -- a driver's blocks come back when it -exits and not before -- so a modeset that allocated would spend sixteen slots -and then the arena, and the failure would look like a GPU bug rather than like -counting. A mode larger than the initial allocation is refused, and says why: - -``` -# schmode 1920 1080 -refused: larger than the framebuffer, which is fixed at startup -``` - -That bound is 4 MB, `PCIIOC_DMA`'s cap, which at 32 bits per pixel is about -1280x800. Raising it is a one-line change to a constant and a question about -how much of a 16 MB arena one driver should be able to take, which is a better -question than it looks and is not answered here. - -### The socket, and the argument it settles - -`vgpud` used to end in `for(;;) sleep(100);` under a comment arguing that -building a control socket before there is a compositor to judge it is exactly -the mistake this document spends a section warning about. - -That argument was right about a *compositor protocol* and wrong as a reason to -have no socket at all. A modeset API has a consumer that exists -- somebody -changing the mode and looking at the screen -- and `schmode` is small enough -that it can judge an interface this small. So the socket is there, it speaks -lines of text for the same reasons -[../architecture/debugging.md](../architecture/debugging.md) gives, and what is -deliberately **not** on it is everything a compositor would need: no buffer -submission, no damage, no vblank. Those still wait, for the original reason. - ---- - -## The mechanisms - -Four, and only the fourth is large. - -### 1. A control channel — **built** - -`AF_UNIX` plus `SCM_RIGHTS` — item 1 of [foundations.md](foundations.md), which -the Wayland work also needed first. A descriptor is how a driver process is -handed the device it is allowed to touch. - -Built as `sendmsg`/`recvmsg` in Linux's shape, because the programs that will -use it are not ours: libwayland and toybox both call `recvmsg` with Linux's -structures and would not notice a friendlier door. Descriptors are pinned to the -stream position of the first byte of their message, and the kernel holds a -reference to each until it is received — or closes it when the socket is dropped -with the message still queued, which is the case that leaks if nobody writes it -down. See -[../architecture/networking.md](../architecture/networking.md#descriptor-passing). - -**Do not build Mach ports.** The port model is worth its cost when there is a -naming and capability story to hang on it; there is not one yet, and inventing it -before the first userspace driver exists is designing against an imaginary user. - -### 2. MMIO to userspace - -Map a PCI BAR into a process. - -`sys_mmap` already maps the framebuffer's real physical range at its real -address — the path exists. This is that path generalised, behind a check that the -caller was granted that device. - -### 3. Interrupts to userspace - -**The handler stays in the kernel and does one thing: write to an `eventfd`.** -The driver process reads. - -`eventfd` exists already, and that is why it is the right primitive. A signal -would be delivered to a process that cannot safely be interrupted mid-transaction. - -### 4. DMA to userspace - -Physically contiguous memory whose physical address the process is told. - -`kernel/core/dma.c` already owns an arena reserved for exactly this, before -`kalloc` gets anything. It needs a door, a per-device budget, and a way to hand -the region back. - ---- - -## The clause nobody wrote down: a panic has to reach the screen - -Handing a device away costs something that was not on the list. The kernel -panics, paints the report into the console framebuffer, and halts — and the -display it gave to a process keeps showing whatever that process last put -there. On a machine with two heads, one says the kernel is dead and the other -shows a shell prompt. - -It is not a race that can be won by being careful. There is no scheduling after -a panic, so the process holding the device is never going to run again, and the -kernel let go of the device deliberately. - -**So the userspace driver says in advance where it is scanning out**, with -`PCIIOC_PANICFB` (see `include/dev/pciuser.h`), and the kernel does the rest: - -1. paints the report into that memory, -2. asks the *kernel* driver the device was taken from to put it on screen — - `panic_present` in `struct pci_driver`, -3. which resets the device and rebuilds one queue from **static** memory: no - allocation, no locks, no sleeping, because a corrupted heap is one of the - ways a machine gets here in the first place. - -The registration is checked like the promise it is: the range must lie inside a -block that process got from `PCIIOC_DMA` on that device. Without that it would -be a way to nominate any physical address in the machine to be overwritten at -panic, which is a worse bug than the one it fixes. - -What this buys is a property worth stating plainly: **a device can be evicted -to userspace without the kernel losing its ability to tell you the machine -died.** That is the difference between a driver boundary and a blind spot, and -it is one more thing the in-kernel arrangement got for free and the hybrid one -has to earn. - -It costs 20 KB of static memory in the virtio-gpu driver, paid whether or not -the machine ever panics. Allocating it on demand would be cheaper and would not -work. - ---- - -## The honest limitation - -> **There is no IOMMU, so a userspace driver is isolated against its own bugs and -> not against malice.** - -A process that can program a device's DMA engine can write to any physical -address, whatever its page tables say. - -That is not a reason to stop — isolation against bugs is most of the value, and a -crashed driver that restarts instead of panicking the machine is the entire -point. But it must be written down, because *"userspace driver"* reads as -*"sandboxed"* and here it does not mean that yet. VT-d/AMD-Vi is its own project -and comes after. - ---- - -## Order, and one thing not to do - -### Do not build loadable kernel modules first - -It is the obvious first step and it is a generation behind. It means designing a -**stable in-kernel driver ABI**, which is the most expensive promise in this -entire plan, for a mechanism Apple is actively retiring in favour of the one -above. - -`../architecture/devices.md` already separates the two ideas. They are separable, -and the second one is the one to build. - -### First eviction: the GPU driver - -Four reasons it is the right one and not just the interesting one: - -- **The boundary already exists and is already correct** — `struct display_ops`. -- **It has exactly one consumer**, the compositor, so the interface has a real - user to be judged by. -- **There is a working fallback underneath.** The VBE framebuffer stays - in-kernel, so a GPU driver that fails to start degrades to a working screen - rather than to no screen. -- It directly serves the work that will actually be happening. - -### Second: USB — and why it turned out not to be - -`xhci.c` is 1364 lines, self-contained, already spoken to through a bus -abstraction, and it is where third-party drivers will show up first. This is what -DriverKit was built for and the resemblance is not accidental. - -#### It is not the same shape as the GPU, and that was missed - -Written before the GPU eviction had been done, H4 says *"USB, same shape"*. It -is not, and the difference is one this document's own rule already decides. - -The GPU could leave because **there was a working fallback underneath it**: the -VBE framebuffer stays in the kernel, so a GPU driver that fails to start -degrades to a working screen. That is one of the four reasons the GPU was -chosen to go first, listed a few lines above. - -USB has no fallback, and it carries two things that are on the path to a shell: - -- **The root disk.** `USBDISK_DEV` dispatches through `bio.c` to - `usb_storage_rw`, and `usbstick.img` is a real boot medium built by `make`. -- **The keyboard.** `usb_hid` feeds `input_report(INPUT_KBD, ...)`, which is how - anything without PS/2 is typed on — including the Chromebox this is meant to - boot. - -So evicting `xhci.c` wholesale breaks the rule three sections down: - -> **The kernel keeps a minimal driver for everything on the path to a shell, and -> userspace owns the good one.** -> -> Break this and you get the circular boot failure where the compositor cannot -> start because the display driver is a process that needs a compositor. - -The USB version of that sentence is *the root filesystem cannot mount because -the storage driver is a process that lives on the root filesystem*, and it is -the same failure wearing different clothes. - -There is a second difference, subtler and probably the harder one. `/dev/pci` -hands over a **PCI function** — one device, one owner, nothing shared. A USB -device is not that: it sits behind a controller that other devices also sit -behind, so the unit of ownership and the unit of hardware are no longer the -same thing. Handing a process "the xHCI controller" gives it the keyboard too; -handing it "one USB device" is an interface `/dev/pci` does not have and that -this plan has not designed. - -#### Decided: USB stays in the kernel - -**H4 is withdrawn.** Not deferred, not blocked on the interface above — -withdrawn, because the premise was wrong. - -> **USB is not a peripheral.** It is the bus the peripherals arrive on. - -Nearly every machine has a USB port in some form, down to a bad micro-B on a -board that has nothing else, and the ones that do not have one are not made -worse by the stack being present. That makes USB structurally the same kind of -thing as PCI, not the same kind of thing as a GPU — and this document has never -proposed evicting PCI. - -Reading the eviction rule again with that in mind, it already said so: - -> **Anything needed to reach a shell stays in. Anything discovered at runtime, -> replaceable, or plausibly written by someone else goes out.** - -USB is how things are *discovered at runtime*, which is not the same as being -one of them. The keyboard and the root disk that arrive over it are on the path -to a shell, and the stack that finds them is part of that path. - -The two obstacles in the section above are real and they are why the premise -got questioned, but they are not the reason for this decision. Even with a -minimal in-kernel fallback written and a per-device ownership interface -designed, the answer would be the same: the good USB driver and the minimal one -would be the same driver, and an eviction that leaves an identical copy behind -has moved nothing. - -What the eviction was *for* — third-party drivers, written by someone else, -living outside the kernel — is a real goal and it survives this. It is simply -not a USB goal; it belongs to the foreign driver host, which is -[ldls.md](ldls.md). - -### The rule that keeps the boot path honest - -> **The kernel keeps a minimal driver for everything on the path to a shell, and -> userspace owns the good one.** - -Break this and you get the circular boot failure where the compositor cannot -start because the display driver is a process that needs a compositor. - ---- - -## Milestones - -| | | | -|---|---|---| -| **H0** | Mechanisms 2, 3 and 4, each with a test that exercises it from a userspace program against a device nothing else is using. | done | -| **H1** | A userspace virtio-gpu driver, publishing `display_ops` across the channel. Schcom talks to it and does not know the difference. | done, except that Schcom does not talk to it -- `cmd/vgpud` holds the device and draws its own frame | -| **H2** | Kill the driver process on purpose. The screen falls back to VBE, the driver restarts, the compositor reattaches. **This is the milestone that proves the whole idea**, and it is the one that cannot be skipped. | done, less the compositor -- see "The driver came back" | -| **H3** | A modeset API, which is the thing an out-of-kernel GPU driver makes affordable and an in-kernel one never got. | done -- see "A modeset, from a process" | -| ~~**H4**~~ | ~~USB, same shape.~~ **Withdrawn** — USB is the bus peripherals arrive on, not a peripheral. See "Decided: USB stays in the kernel". | | - ---- - -## What it will still not do - -- **No isolation against a hostile driver**, per the IOMMU note above. -- **No stable driver ABI**, deliberately. Out-of-tree drivers remain a - recompile-with-the-tree affair until there is a reason to promise otherwise. -- **The root filesystem stays in the kernel**, so SchFS cannot be restarted and a - bug in it is still fatal. Moving it out needs an initramfs story that does not - exist. -- **The first NIC stays in the kernel**, because the network thread and lwIP are - in the kernel and moving them is a separate project with no current benefit. -- **No driver sandboxing beyond address-space separation** — no seccomp-shaped - syscall restriction, no resource limits per driver. diff --git a/docs/plan/roadmap.md b/docs/plan/roadmap.md index a024ebb..234a8c9 100644 --- a/docs/plan/roadmap.md +++ b/docs/plan/roadmap.md @@ -1,183 +1,120 @@ # Roadmap — everything else still open -The three current decisions have their own documents; see -[README.md](README.md). This file holds what is left: work that is scoped, -agreed and unstarted, with nothing in it that is finished. +[1.0.md](1.0.md) draws the finish line and holds the register of what is half-built. +This file holds what is left beyond it: work that is scoped, agreed and unstarted, with +nothing in it that is finished. -**Finished work is not here.** It moved to -[../architecture/](../architecture/) and was rewritten as a description of what -it produced — see [the rule](../README.md#the-rule-that-keeps-this-from-rotting). -What used to be phases 1 through 4 of this file, several hundred lines of ticked -boxes, is now `usb`, `networking.md`, `graphics.md` and `devices.md` over there. +**Finished work is not here.** It moved to [../architecture/](../architecture/) and was +rewritten as a description of what it produced — see +[the rule](../README.md#the-rule-that-keeps-this-from-rotting). --- ## 1. Storage and the boot path -**Done: a USB disk, a SATA disk and their partitions can be booted from.** The -root is chosen at boot from what is present (`rootdev_mount()` in -`kernel/proc/proc.c`), and `make disk` builds a GPT image with an ESP and a SchFS -partition that a PC boots from a stick or a disk. +A USB disk, a SATA disk, an NVMe SSD and their partitions boot, and the root is chosen at boot from +what is present ([boot.md](../architecture/boot.md)). -NVMe (`kernel/drivers/storage/nvme/`) registers with the same block registry, -so an NVMe SSD boots the same way. - -**Next:** an installer that writes that layout to a disk, and interrupt-driven, -queued I/O for AHCI and NVMe. +- **An installer** that writes the layout `make disk` builds (a GPT disk with an ESP and a SchFS + partition) to a machine's disk, so a root survives a reboot. See §5. +- **Interrupt-driven, queued I/O for AHCI and NVMe.** Both poll for completion, one command at a time + per disk: enough to boot and run, not to be fast. +- **An EHCI driver.** USB 2.0 ports wired only to EHCI do nothing; xHCI ports work. ## 2. Graphics -- **A `/dev/dri`-style device with mode setting**, so resolution stops being - whatever the loader asked for. This was to be done *with* - [hybrid.md](hybrid.md)'s GPU eviction, and that eviction has now happened — - `cmd/vgpud` drives virtio-gpu from userspace. The modeset API is the next thing - it makes affordable, and is milestone H3 over there rather than here. -- **Double buffering and vblank.** `display_ops` specifies both and no driver - implements either. Schcom needs both to not tear. -- **Port the console onto the text layer and delete the cell grid.** The layer - exists, the rasteriser exists, and the console still indexes `font_data[256][16]`. -- **Load a font from disk** and point the console at it. -- **CFF/Type 2 charstrings**, so OTF fonts work. A separate `font_ops` provider - and a bytecode interpreter, *not* an extension of the `glyf` path. -- **Text layout** — advance widths, kerning — as a library in `lib/`, so the - terminal and the file manager are not each solving it. - -The console keeps its fixed grid until the first item above; proportional text -belongs to the GUI. +- **A `/dev/dri` device with mode setting**, so a real panel's resolution stops being whatever the + loader's menu chose and Mesa has something to open. This is the GPU plan, [gpu.md](gpu.md). +- **Double buffering and vblank**, which `display_ops` specifies and no driver implements; the same plan. +- **CFF/Type 2 charstrings**, so OTF fonts work. A separate `font_ops` provider and a bytecode + interpreter, *not* an extension of the `glyf` path. +- **Text layout** — advance widths, kerning — as a library, so the terminal and the file manager are not + each solving it. ## 3. Networking -Two things the stack owed on the day it landed, both still owed: +- **A routing table** and `/proc/net/route`. With two interfaces up the first is the default route and the + second is reachable only by an address on its own subnet; that is a rule and not routing. +- **Wi-Fi.** The laptop has no other way to be wireless. The route is the same as for the NIC: a Linux + driver (`iwlwifi`, `mt76`…) through LDLS, which needs `nl80211`, a firmware loader that exists, and a + supplicant (`wpa_supplicant` is an Alpine package). +- **IPv6.** +- **A netlink event stream**, for anything that wants to be told a link changed. -- **`accept()` does not report the peer's address**, and there is no - `getsockname`/`getpeername`. Nothing has needed them yet; a server that wants - to log who connected will. -- **No routing table.** With two interfaces up, the first is the default route - and the second is reachable only by an address on its own subnet. That is a - rule rather than an accident, and it is not routing. +## 4. The hardware that makes a laptop a laptop -Two that were on this list are now done and described in -[../architecture/networking.md](../architecture/networking.md): `setsockopt` -honours six options and refuses the rest, and `sendmsg`/`recvmsg` carry -`SCM_RIGHTS`. A third is closer than it was — toybox's `ping` needed `recvmsg` -*and* `IP_RECVTTL`, and only the second is still missing. - -## 4. The interactive crash screen - -PSOGM renders and stops. The idea is *a small OS inside the crash*, and the -framing decides the design: - -> At panic time the kernel is the only thing running, which is exactly why it can -> offer what a live system cannot: walk the stack, read another process's memory, -> dump page tables, inspect the process table — with no scheduler and no locking -> to respect, because nothing else can race you. - -What makes it a *system* rather than a print statement is that it keeps the base -machine's drivers. Concretely, the ones that work with interrupts off: the -framebuffer (`gfx_draw_pixel` is a plain memory write), the PS/2 keyboard -(pollable via ports 0x60/0x64), and the serial port. **Those three are a complete -console.** - -Everything else — IDE, USB, the timer, anything that sleeps or takes a -sleeplock — is off limits, because the lock it needs may be precisely the one -held by the CPU that just died. - -So the shape is a **polled event loop over preallocated state**: no `kalloc`, no -`sleep`/`wakeup`, no `acquire` on anything a live path might hold, one static -command buffer, and that driver set. **That restriction is not a limitation to -work around later; it is the invariant that makes the mode trustworthy when -everything else is not.** Anything wanting more should be answered by rebooting -into a recovery environment. - -A useful precedent to read but not copy: DDB, the BSD in-kernel debugger, is -exactly this, and its hard-won rule is the same one — *a debugger that can -deadlock is worse than no debugger.* +- **HDA audio.** AC'97 is what QEMU and old boards have; a laptop built this century has Intel HDA, + which is codec enumeration, CORB/RIRB and stream descriptors. It arrives as a second `struct audio_ops`. +- **Embedded-controller events** (the ACPI SCI), so a lid, a power button on the EC and a battery change are seen. +- **CPU frequency control** (AMD CPPC; Intel HWP/EIST). Idle already reaches `_CST`'s C2; under load the cores run at + the firmware's default clock. +- **Touchpad gestures** — two-finger scroll, palm rejection — on top of I2C-HID's mouse mode. +- **An IOMMU** (VT-d / AMD-Vi), which turns "a driver in a process is isolated against bugs" into "against malice" — + the largest honest caveat on [ldls.md](../architecture/ldls.md). +- **udev's job** — a uevent socket and `/sys/devices` — so X and others are told about devices instead of being + given a list. ## 5. The installer and `.sip` -Scoped in [`projects/schinst/README.md`](../../projects/schinst/README.md), -which holds the detail. In short: +Scoped in [`projects/schinst/README.md`](../../projects/schinst/README.md), which holds the detail. In short: - [ ] `.sip` format specification, **written down before any code** - [ ] `tools/mksip` — capture a directory tree into a package - [ ] The apply path -- [ ] An installation environment that boots from the ISO, partitions, applies - the `.sip`, and installs the loader +- [ ] An installation environment that boots from the ISO, partitions, applies the `.sip`, and installs the loader -`.sip` is modelled on WIM, and the thing worth copying from WIM is that it is -**file-based, not block-based**: one file holds a whole filesystem, applies to a -partition of any size, and several images can share their common files. +`.sip` is modelled on WIM, and the thing worth copying from WIM is that it is **file-based, not block-based**: one +file holds a whole filesystem, applies to a partition of any size, and several images can share their common files. -**Most of this already exists.** `tools/mkschfs.c` and `tools/mkfs.c` both do -content dedup with an `(st_dev, st_ino)` fast path, and SchFS's B-tree is content -addressed. `mksip` should share that code rather than reimplement it — that is -the reason the dedup was factored the way it was. +**Most of this already exists.** `tools/mkschfs.c` and `tools/mkfs.c` both do content dedup with an `(st_dev, st_ino)` +fast path, and SchFS's B-tree is content addressed. `mksip` should share that code rather than reimplement it. And +the live image is already a compressed block image of a SchFS root ([scff.md](../architecture/scff.md)); an installer +may need nothing more than to write that, decompressed, to a partition. Open questions, all wanting a decision before code: -- **Compression.** Something with a small decompressor is worth more than - something with a good ratio, because the decompressor runs in the installation - environment. -- **Multiple images per `.sip`** — WIM's real trick, install/recovery/boot - sharing one blob store. Worth the complexity now, or not? -- **Can the kernel mount a `.sip` read-only directly?** It is close to a - filesystem already, and if it can, the installation environment's root can - simply *be* the `.sip`. +- **Compression.** Something with a small decompressor is worth more than something with a good ratio, because the + decompressor runs in the installation environment. (zstd is already in the kernel for SCFF.) +- **Multiple images per `.sip`** — WIM's real trick, install/recovery/boot sharing one blob store. Worth the complexity + now, or not? +- **Can the kernel mount a `.sip` read-only directly?** If it can, the installation environment's root can simply *be* the `.sip`. -The GUI for this is a Wayland client; see [wayland.md](wayland.md). +The GUI for this is an X client, like everything else on the desktop. ## 6. SchFS beyond SchOS In the order they should happen. -**~~An on-disk format version~~, and an upgrade path.** The version policy is -done — see [../architecture/schfs.md](../architecture/schfs.md). What remains is -the **upgrade step**, and it cannot be written until there are two formats to -convert between; the first bump is when that becomes real work rather than -speculation. +**An upgrade step.** The format version policy is done ([schfs.md](../architecture/schfs.md)); the upgrade step cannot be +written until there are two formats to convert between, and the first bump is when it becomes real work. -The format is now pinned, which is what the other two items were waiting on. +**A Linux implementation, FUSE first.** Three reasons in order of weight: it can be tested with `fsx` and `xfstests`, which +are far harsher than anything written alongside the code; it carries no licence entanglement, since libfuse is LGPL and used +across a process boundary; and it can reuse the read/write logic already in `tools/mkschfs` and `tools/schfs_fsck` rather than +being written twice and having the two disagree. A kernel module comes only after FUSE passes `xfstests`; SchOS is GPL-2.0-or-later +([../licensing.md](../licensing.md)), so a Linux module is a technical project now and not a licensing one. -**A Linux implementation, FUSE first.** Three reasons in order of weight: it can -be tested with `fsx` and `xfstests`, which are far harsher than anything written -alongside the code; it carries no licence entanglement, since libfuse is LGPL and -used across a process boundary; and it can reuse the read/write logic already in -`tools/mkschfs` and `tools/schfs_fsck` rather than being written twice and having -the two disagree. - -A kernel module is a different project, comes only after FUSE passes `xfstests`, -and carries the licence problem [../licensing.md](../licensing.md) describes. - -**The missing features**, listed honestly before adding any: no journal replay -across an unclean shutdown beyond what copy-on-write gives for free, no -snapshots, no sparse-file support, and no online `fsck`. **Snapshots are the one -where the existing design already pays for most of the cost** — the B-tree is -copy-on-write, which is most of the machinery; nothing takes or names a snapshot. +**The missing features**, listed honestly before adding any: no journal replay across an unclean shutdown beyond what +copy-on-write gives for free, no snapshots, no sparse-file support, and no online `fsck`. **Snapshots are the one where the +existing design already pays for most of the cost** — the B-tree is copy-on-write, which is most of the machinery; nothing takes +or names a snapshot. ## 7. The kernel's own gaps Not phases, and each one improves everything above it: -- **A slab or pool allocator.** `kalloc` hands out one page at a time, and every - subsystem written since has invented its own fixed-size table to work around - that. This is the single change that would most improve the rest of the kernel. -- **A page cache**, which is what a general writable `MAP_SHARED` needs. - [foundations.md](foundations.md) delivers a `memfd` instead, which is narrower - and sufficient for the GUI. -- **Symlinks in procfs**, so `/proc//exe` can be one and `ps -o comm` stops - printing `[init]`. Belongs with the rest of the VFS work rather than on its own. -- **The `ls -lR` hang** — **not reproducible**, and kept here because the absence - of a result is a result. Instrumented and hunted at length; see Gate 0 of - [1.0.md](1.0.md). Two real defects were found on the way and fixed, neither - proven to be the cause. -- **No garbage collector for descriptors in flight.** An `AF_UNIX` socket passed - over itself and never received leaks the pair. Bounded at thirty-two sockets, - and nothing in the tree does it — but a third-party program could. +- **A page cache**, which is what a general writable `MAP_SHARED` of a file needs. `memfd` is the narrow answer and has been enough. +- **A working `mprotect`, and NX.** `mprotect` is accepted and ignored and every mapping is executable; RELRO, guard pages + and a W^X JIT all want the real thing. +- **Per-CPU run queues** and work stealing, with a load balancer that is not worse than the lock it replaces. +- **A garbage collector for descriptors in flight.** An `AF_UNIX` socket passed over itself and never received leaks the pair; + bounded at thirty-two, and nothing in the tree does it. +- **Kernel-time accounting**, so `ru_stime` is not zero. +- **A contiguous multi-page allocator**, so `struct proc` and `struct socket` can come from somewhere other than fixed tables. --- ## Kept out of scope on purpose -- Anything that needs pushing or publishing. This work stays local. - TrueType hinting. -- XWayland. +- DRM/KMS as a *kernel* subsystem; the GPU plan serves `/dev/dri` from a process. diff --git a/docs/plan/schapi.md b/docs/plan/schapi.md index bbc386f..1ec4a0f 100644 --- a/docs/plan/schapi.md +++ b/docs/plan/schapi.md @@ -1,6 +1,9 @@ # SchAPI, and SchOS as a system -Status: **direction agreed, not yet implemented.** Nothing in this document +Status: **direction agreed, not yet implemented**, and frozen until 1.0 ships +([1.0.md](1.0.md)). Read it as a direction and not as a schedule: written when the plan was a +compositor of SchOS's own, it has not been revisited since the desktop became Xorg and the +packages became Alpine's, and §7 (what happens now) in particular predates both. Nothing in this document changes what is being built this week; it changes what everything is being built *towards*. See §7 for what actually happens now. diff --git a/docs/plan/wayland.md b/docs/plan/wayland.md deleted file mode 100644 index 1008c2b..0000000 --- a/docs/plan/wayland.md +++ /dev/null @@ -1,177 +0,0 @@ -# The GUI becomes Wayland - -Schcom survives and changes its wire format. SchWM and SchDE are deleted. What -follows is why that is not a retreat, and what it costs. - -Blocked on [foundations.md](foundations.md). Everything there is built except -item 4, `setsockopt` that does not lie: descriptor passing, shareable memory, -sockets in the filesystem, dynamic linking, and the epoll/eventfd/timerfd/ -signalfd set libwayland's event loop is made of. - ---- - -## What was deleted - -`projects/schwm/` and `projects/schde/`, both README-only; no code was lost. They -are **gone from the tree now** rather than marked for deletion — a directory that -has said "to be deleted" for months is a corner, not a decision. The arguments -each made are kept below, because they are the part worth keeping. - -### SchWM is not abandoned — Wayland dissolves it - -In X11 the window manager is a separate privileged client because X11 puts -compositing in the server and policy nowhere. **Wayland's central decision is -that the compositor *is* the window manager**: there is no protocol by which a -second process places another's windows, and every attempt to add one has been a -private extension. - -So SchWM's own argument — - -> *"Compositing is mechanical and must never stall: it runs every frame and a -> mistake there is a frozen screen. Window management is policy, full of special -> cases and personal taste, and it changes constantly."* - -— is a real and correct observation **about X11**, which Wayland answers a -different way. It stays worth honouring, but as a **module boundary inside -Schcom**, not a process boundary: policy in its own file behind its own -interface. Do not keep it in its own process, because the protocol to talk to it -does not exist. - -### SchDE is re-sited rather than abandoned - -A panel, a launcher and a wallpaper are ordinary Wayland clients using -`wlr-layer-shell` — MIT, an XML file, no dependency. They need no directory of -their own until one of them is written. SchDE's other observation stands on its -own and is recorded here so it does not go with the directory: **a GUI terminal -is not the kernel console with a window around it.** The console parses ANSI -because it draws straight to the framebuffer; a terminal client parses ANSI -itself and renders it with proportional-capable text. The two share no code. - -### `projects/schinst/` stays - -Its GUI was the part tied to SchWM, and that part becomes a Wayland client. -Everything else in it — the `.sip` format, `mksip`, the apply path, the -installation environment — is untouched by any of this, and remains the -best-specified unstarted work in the tree. - ---- - -## What survives - -**Schcom stays, and keeps its name.** It already owns the display, allocates and -owns client buffers, implements surface/attach/commit, routes input from -`/dev/input/eventN`, composites multiple clients bottom-to-top, and cleans up -after clients that die. - -Its own README says the protocol is deliberately Wayland-shaped — surfaces rather -than windows, buffers attached and made visible only on commit, the compositor -telling clients when to draw. That was the right call and this is where it is -cashed in: **what changes is the wire format and the buffer handle, not the -architecture.** - ---- - -## What has to exist first - -Items 1–4 of [foundations.md](foundations.md), and the first three are absolute — -libwayland does not start without them. - -The sharpest is descriptor passing. `wl_shm_create_pool` passes a **file -descriptor**, and there is no path in the protocol that passes an integer -instead. `struct schcom_surface`'s `handle_kind` field was written precisely so -that switching would be a new value rather than a protocol break, and it cannot -help: *our* protocol is the flexible one. - ---- - -## Port libwayland. Do not port Weston. - -**libwayland is the small piece**: a wire protocol, an object map, and an event -loop, needing `libffi` and — for the build-time scanner, on the host only — -`libexpat`. A real port, but a bounded one. - -**Weston is the trap.** It arrives welded to `libinput`, `udev`, `libdrm`, -`logind`-shaped seat management and a GL renderer, and porting it means porting -Linux's device management first and finding the compositor somewhere underneath. - -This is precisely the lesson already recorded twice in this tree — about -FreeBSD's `sys/dev/usb` (100k lines welded to newbus) and about FreeBSD's -`sys/netinet` (not a library; reaches into mbufs, epoch, netisr, VNET and uma). - -> The tree has learned this twice. It should not pay to learn it a third time. - -So **Schcom links `libwayland-server` and remains ours.** Input comes from -`/dev/input/eventN`, which already works, rather than from libinput. Output goes -through `display_ops`, which already exists. - -### Software rendering first, and probably for a long time - -`pixman` for composition, `wl_shm` for buffers. Mesa means EGL means `dlopen` -means a dynamic loader -- which now exists (see [foundations.md](foundations.md) -item 5), so it is no longer what stands in the way of Mesa; a GPU driver is. -Static binaries are still enough to reach a working desktop, and the desktop's -own programs can stay static until something concrete demands otherwise. - -### One sharp edge - -`wl_keyboard` hands the client **a keymap as an XKB text blob over a file -descriptor**. A compositor cannot avoid xkbcommon by having simpler opinions -about keyboards — the protocol requires the blob. - -The cheap route is one precompiled keymap string shipped in the binary, with -xkbcommon and `xkeyboard-config` deferred until more than one layout is wanted. - ---- - -## Licensing - -Checked before anything is fetched — the standing rule is that the licence goes -in `NOTICE` **before** the file is committed, not after. - -| | | -|---|---| -| wayland, wayland-protocols | MIT | -| `wlr-layer-shell` | MIT | -| pixman | MIT | -| libffi | MIT | -| libxkbcommon | MIT | -| xkeyboard-config | MIT | - -All GPL-2.0-or-later compatible, all needing only a `NOTICE` entry. The -Wayland ecosystem is permissively licensed almost throughout, so nothing here -raises a licensing question at all. - ---- - -## Milestones - -| | | -|---|---| -| **W0** | [foundations.md](foundations.md) items 1–4, with the combined test that passes a descriptor *and* shares a `memfd` *and* binds a socket at a real path. | -| **W1** | libwayland builds and installs into the sysroot. `wayland-scanner` runs on the host. | -| **W2** | Schcom serves `wl_compositor`, `wl_shm`, `wl_surface`, `xdg_shell`. A client draws one window. | -| **W3** | `wl_seat`: keyboard and pointer, from the evdev path that exists. | -| **W4** | Two clients, focus, stacking, move and resize. **This is where SchWM's policy lives, as a file.** | -| **W5** | `wlr-layer-shell`, and a panel. This is SchDE's first pixel. | - -Every milestone from W2 on is asserted **from the host, by pixel**, the way -`schcom-test` already does. The compositor's own report that it composited -something proves nothing. - ---- - -## What it will still not do - -- **No hardware acceleration.** Software rendering only, until Mesa, which is - not on this plan. Dynamic linking, which it also needs, is in. -- **No XWayland**, so no X11 clients, ever, unless someone ports it. -- **No mode setting.** Resolution stays whatever the loader asked for — see - [hybrid.md](hybrid.md), where the GPU driver's eviction is also where a modeset - API would arrive. -- **No damage tracking beyond "repaint everything"**, no multiple outputs, no - subsurfaces. Schcom's README already lists these; they are additions rather - than rewrites, which is the reason for committing to the surface/commit model - now. -- **One keyboard layout**, until xkbcommon is ported. -- **No clipboard, no drag and drop, no screen capture protocol**, none of which - W0–W5 touches. diff --git a/docs/plan/xorg.md b/docs/plan/xorg.md deleted file mode 100644 index db5d4e5..0000000 --- a/docs/plan/xorg.md +++ /dev/null @@ -1,44 +0,0 @@ -# Xorg - -The desktop is X11, on the framebuffer. This replaces [wayland.md](wayland.md) -as the plan for Gate 3 of [1.0.md](1.0.md); Wayland stays on file and is not -being worked on. - -## Why X and not Wayland - -Every Wayland compositor worth running -- labwc, and everything else built on -wlroots -- needs DRM/KMS, libinput, udev and libseat. DRM/KMS is the largest of -them by far: a kernel graphics subsystem, and in practice a great deal taken -from Linux. Xorg runs on a plain framebuffer through its `fbdev` driver and -reads input through `evdev`, both of which are interfaces SchOS already has. -It is the shorter road to a desktop by a long way, and it brings the window -managers with it: twm first, then Window Maker. - -DRM/KMS is not planned. libinput, udev and libseat are worth having for their -own sake and may come later; Xorg does not need them. - -## What is built - -- **The kernel calls** (`b1323e1`, checked by `cmd/xready`): evdev's queries, - fbdev's mode-set to the current mode, `KDSETMODE`, `KDSKBMODE`, the - `VT_PROCESS` switch handshake, and `/sys/bus/pci/devices/*/{boot_vga, - enable,graphics/fb0}` for libpciaccess. -- **Dynamic linking and `dlopen`** (`4330576`): Xorg loads its drivers as - modules. -- **``** in the build sysroot, as `ports/linux-headers`. - -## What is left, in order - -1. **The X client libraries.** xorgproto, xtrans, libXau, libXdmcp, - xcb-proto, libxcb, libX11, libXext, libXrender, libXfixes, libxkbfile, - libfontenc, libXfont2, freetype, zlib, pixman, libpciaccess. -2. **Keyboard and fonts.** xkbcomp and xkeyboard-config, the data X compiles - keymaps from; a misc-fixed font. -3. **The server.** xorg-server built with meson, GLX, glamor and DRI off; - xf86-video-fbdev; xf86-input-evdev with libevdev and mtdev. -4. **The first picture.** `twm` and `xterm`, started by hand from a VT. -5. **Window Maker**, and an `startx` that works. - -The kernel calls were written against what these programs are documented -to do. Each step past it is expected to find a call that was not -- that is -what running the real program is for. diff --git a/include/core/dma.h b/include/core/dma.h index a4b5e54..c4d751a 100644 --- a/include/core/dma.h +++ b/include/core/dma.h @@ -26,7 +26,7 @@ // // 4 MB was one framebuffer at 1280x800, which was fine while exactly one driver // wanted one. It stopped being fine the day a display driver could be handed to -// userspace (docs/plan/hybrid.md): either side of that handover **both drivers +// userspace (docs/architecture/devices.md): either side of that handover **both drivers // exist**, and the incoming one has to allocate its framebuffer before the // outgoing one lets go of its own. So the real requirement is two framebuffers // at once, not one. diff --git a/include/dev/pci.h b/include/dev/pci.h index 4d5770a..93b2048 100644 --- a/include/dev/pci.h +++ b/include/dev/pci.h @@ -151,7 +151,7 @@ struct pci_driver { void (*attached)(struct pci_dev *, struct device *); // Give the device up, so a userspace driver can take it. // - // docs/plan/hybrid.md is built on the kernel *handing over* a device + // docs/architecture/devices.md is built on the kernel *handing over* a device // rather than a process wrestling it away: two drivers writing one set of // registers is not a subtle failure, it is hardware that appears to have // started lying. So the transfer is the incumbent's decision, and this is diff --git a/include/dev/pciuser.h b/include/dev/pciuser.h index 49f5da8..43f6e7e 100644 --- a/include/dev/pciuser.h +++ b/include/dev/pciuser.h @@ -2,7 +2,7 @@ // // /dev/pci -- device access from userspace. // -// docs/plan/hybrid.md, mechanisms 2 and 4: MMIO and DMA to a process. This is +// docs/architecture/devices.md, mechanisms 2 and 4: MMIO and DMA to a process. This is // the interface a driver that does not live in the kernel talks to, and it is // deliberately about *PCI devices*, not about any particular driver model. // @@ -23,7 +23,7 @@ // // There is no IOMMU. A process holding a DMA buffer can point the device at // any physical address, so this isolates a driver against its own *bugs* and -// not against its own *intent*. docs/plan/hybrid.md says so at length and it +// not against its own *intent*. docs/architecture/devices.md says so at length and it // remains true here: a crashed driver that restarts instead of taking the // machine down is the whole benefit, and it is not a security boundary. diff --git a/include/dev/virtio.h b/include/dev/virtio.h index 3d1f61a..46fdbd1 100644 --- a/include/dev/virtio.h +++ b/include/dev/virtio.h @@ -4,7 +4,7 @@ // // It lived next to the driver while exactly one driver read it, with a note // saying it should move when a second reader appeared. One has: cmd/vgpud is -// the same device driven from userspace (docs/plan/hybrid.md's first +// the same device driven from userspace (docs/architecture/devices.md's first // eviction), and it needs the same register offsets and the same commands. // // Still one file rather than transport-and-protocol split in two. Both readers diff --git a/include/net/socket.h b/include/net/socket.h index ac17a2c..027ac1c 100644 --- a/include/net/socket.h +++ b/include/net/socket.h @@ -63,7 +63,7 @@ // is not a number: it names an object the kernel already knows, and handing it // over is the difference between "here is a buffer" and "here is a *copy* of a // buffer". Wayland's wl_shm_create_pool passes one and has no integer-shaped -// alternative; docs/plan/hybrid.md's mechanism 1 is this and nothing else. +// alternative; docs/architecture/devices.md's mechanism 1 is this and nothing else. // // Every structure below is Linux's, byte for byte, because a musl-linked // program builds them with Linux's headers and we do not get a vote. The diff --git a/kernel/dev/pciuser.c b/kernel/dev/pciuser.c index 30b0560..8d4eb20 100644 --- a/kernel/dev/pciuser.c +++ b/kernel/dev/pciuser.c @@ -302,7 +302,7 @@ pciuser_mmio_ok(uint64 phys, uint64 len) // // Nothing had noticed because the only consumer was virtio-gpu, whose // BARs are large. The first foreign driver found it in its first minute, - // which is the thing docs/plan/hybrid.md said a foreign driver would be + // which is the thing docs/architecture/devices.md said a foreign driver would be // good for. // // The widening is real and is worth naming: a sub-page BAR hands the @@ -477,13 +477,13 @@ do_claim(uint64 argp) // // Two drivers on one device is not a subtle failure -- it is two writers of // the same registers, and what it looks like from outside is hardware that - // has started lying. The eviction path (docs/plan/hybrid.md) is for the + // has started lying. The eviction path (docs/architecture/devices.md) is for the // kernel to *hand over* a device deliberately; until a driver can be told to // let go, the honest answer to "may I have the one you are using" is no. if (d->driver) { // Ask it to let go. A driver with no release hook keeps its device, which // is always safe and, for anything on the path to a shell, correct: the - // rule in docs/plan/hybrid.md is that the kernel keeps a minimal driver + // rule in docs/architecture/devices.md is that the kernel keeps a minimal driver // for everything needed to reach a prompt and userspace owns the good one. if (!d->drv || !d->drv->release) return -EBUSY; diff --git a/kernel/drivers/gpu/vga/vga_fb.c b/kernel/drivers/gpu/vga/vga_fb.c index 487013e..c0e3c0f 100644 --- a/kernel/drivers/gpu/vga/vga_fb.c +++ b/kernel/drivers/gpu/vga/vga_fb.c @@ -272,7 +272,7 @@ const struct display_ops vga_display_ops = { // driver that leaves has to name its replacement rather than simply detaching; // detaching alone leaves gfx pointing at a device nobody is driving. // -// This is the fallback rule from docs/plan/hybrid.md made real: the kernel +// This is the fallback rule from docs/architecture/devices.md made real: the kernel // keeps a minimal driver for everything on the path to a shell, so a fancy // driver leaving degrades to a working screen instead of to no screen. int diff --git a/kernel/drivers/gpu/virtio_gpu/virtio_gpu.c b/kernel/drivers/gpu/virtio_gpu/virtio_gpu.c index 7a1d961..0ae5f6c 100644 --- a/kernel/drivers/gpu/virtio_gpu/virtio_gpu.c +++ b/kernel/drivers/gpu/virtio_gpu/virtio_gpu.c @@ -557,7 +557,7 @@ static struct { // Hand the device to a userspace driver. // -// docs/plan/hybrid.md's first eviction, and the reason it is this device: the +// docs/architecture/devices.md's first eviction, and the reason it is this device: the // boundary already exists (struct display_ops), it has exactly one consumer, // and there is a working fallback underneath. So letting go is safe in a way // it would not be for the disk the root filesystem is on. @@ -580,7 +580,7 @@ vgpu_release(struct pci_dev *d) if (!g) return -1; - // The rule from docs/plan/hybrid.md, enforced rather than described: **the + // The rule from docs/architecture/devices.md, enforced rather than described: **the // kernel keeps a minimal driver for everything on the path to a shell.** If // there is no framebuffer to fall back to, this device is the only screen the // machine has, and handing it to a process that might crash would make a @@ -645,7 +645,7 @@ vgpu_release(struct pci_dev *d) // --- the crash screen, on a device this driver was told to let go ---------- // -// docs/plan/hybrid.md's bargain has a clause nobody wrote down: a device the +// docs/architecture/devices.md's bargain has a clause nobody wrote down: a device the // kernel hands away is a device the kernel cannot use to tell you the machine // died. The second monitor sat there showing a shell prompt from before the // panic, which is the most misleading thing a screen can do. diff --git a/kernel/drivers/net/e1000/e1000.c b/kernel/drivers/net/e1000/e1000.c index 4007bba..0481ca1 100644 --- a/kernel/drivers/net/e1000/e1000.c +++ b/kernel/drivers/net/e1000/e1000.c @@ -423,7 +423,7 @@ e1000_link_up(struct device *dev) // Let go of the device, so a userspace driver can have it. // -// The same hook virtio-gpu grew for H1, on a NIC. docs/plan/hybrid.md's rule +// The same hook virtio-gpu grew for H1, on a NIC. docs/architecture/devices.md's rule // is that the kernel keeps a minimal driver for everything on the path to a // shell -- and a network interface is not on that path, so this one may leave. // diff --git a/kernel/net/unix.c b/kernel/net/unix.c index 3b9c8c1..9383782 100644 --- a/kernel/net/unix.c +++ b/kernel/net/unix.c @@ -339,7 +339,7 @@ sockname(struct socket *s) // --------------------------------------------------------------------------- // Descriptor passing // -// docs/plan/hybrid.md mechanism 1, and item 1 of foundations.md. The kernel +// docs/architecture/devices.md mechanism 1, and item 1 of the descriptor-passing work (kernel.md). The kernel // holds a reference on every descriptor in flight, which is what makes the // sender free to close its own copy the instant sendmsg returns -- and what // makes dropping a socket with descriptors still queued a thing that has to be diff --git a/kernel/sys/sys_mem.c b/kernel/sys/sys_mem.c index 5f4e0db..76a3c31 100644 --- a/kernel/sys/sys_mem.c +++ b/kernel/sys/sys_mem.c @@ -686,7 +686,7 @@ shm_detach(int id) // or race to invent a key. // // It is the same segment machinery underneath, which is exactly what -// docs/plan/foundations.md proposed: shmget()'s pages, refcounts, no_free +// docs/architecture/kernel.md proposed: shmget()'s pages, refcounts, no_free // VMAs and the fork/exit accounting are all already right. What memfd adds is // a segment that starts empty and is sized by ftruncate(), and a file that // owns one reference to it. diff --git a/kernel/sys/sys_net.c b/kernel/sys/sys_net.c index 53d5280..a47d701 100644 --- a/kernel/sys/sys_net.c +++ b/kernel/sys/sys_net.c @@ -812,7 +812,7 @@ SYSCALL_DEFINE(getsockopt) // programs that need it are not ours: libwayland and toybox's ping both call // recvmsg with Linux's structures and would not notice a friendlier door. // -// docs/plan/foundations.md item 1, and docs/plan/hybrid.md mechanism 1. +// docs/architecture/kernel.md item 1, and docs/architecture/devices.md mechanism 1. // --------------------------------------------------------------------------- // Linux caps an iovec at UIO_MAXIOV = 1024, and so does this. It was 32, and diff --git a/kernel/sys/table.inc b/kernel/sys/table.inc index 3550cef..3e342aa 100644 --- a/kernel/sys/table.inc +++ b/kernel/sys/table.inc @@ -141,11 +141,11 @@ SYSCALL(192, 55, getsockopt) // reads whatever it last left there and invents a flag out of stack litter. SYSCALL(198, 288, accept4) // sendmsg/recvmsg exist for the one thing only they can do: carry a descriptor -// from one process to another (SCM_RIGHTS). docs/plan/foundations.md item 1. +// from one process to another (SCM_RIGHTS). docs/architecture/kernel.md item 1. SYSCALL(199, 46, sendmsg) SYSCALL(200, 47, recvmsg) // memfd_create: an anonymous shared memory object reached by descriptor. -// docs/plan/foundations.md item 2 -- the thing a Wayland buffer crosses on, +// docs/architecture/kernel.md item 2 -- the thing a Wayland buffer crosses on, // and the reason the descriptor passing above had to exist first. SYSCALL(201, 319, memfd_create) diff --git a/projects/README.md b/projects/README.md index 27e6887..6316bd7 100644 --- a/projects/README.md +++ b/projects/README.md @@ -27,18 +27,14 @@ Concretely, code in here: | Directory | What it is | |------------|-----------------------------------------------------------------| -| `schcom/` | SchOS Compositor. Owns the display, composites clients, routes input. | +| `schcom/` | SchOS Compositor v0. Superseded by Xorg; awaiting deletion (see Status). | | `schinst/` | The installer, the installation environment, and `.sip` tooling. | | `schfs-linux/` | SchFS as a filesystem for Linux. FUSE first, a kernel module later. | -`schwm/` and `schde/` used to be here and have been **deleted**. In Wayland the -compositor *is* the window manager — there is no protocol by which a second -process places another's windows — and a panel or a launcher is an ordinary -client that needs no home of its own until one is written. Both directories held -a README and no code, and both had said "to be deleted" for long enough to prove -that a directory marked for deletion is not deleted by being marked. What they -argued for survives as a module boundary inside Schcom; see -[`docs/plan/wayland.md`](../docs/plan/wayland.md). +`schwm/` and `schde/` used to be here and have been **deleted**: both held a README and no code, and both +had said "to be deleted" for long enough to prove that a directory marked for deletion is not deleted by being +marked. The desktop is Xorg with Window Maker ([`docs/architecture/desktop.md`](../docs/architecture/desktop.md)), +so the window manager is someone else's and a panel or launcher is an ordinary client. **`schfs-linux/` is written to stand on its own.** It does not mention SchOS anywhere, and that is deliberate rather than an oversight: it is intended to be @@ -55,17 +51,9 @@ by anything above it. ## Status -`schcom/` has a v0: 715 lines that run, over a wire format of its own. **That -wire format is superseded** — the compositor moves onto Wayland, keeping the -architecture and replacing the protocol. See -[`docs/plan/wayland.md`](../docs/plan/wayland.md). +`schcom/` is a compositor v0, 715 lines that run, over a wire format of its own. **It is superseded**: the desktop is +Xorg on the framebuffer, not a compositor of ours, and nothing speaks Schcom's protocol. It is on the register in +[`docs/plan/1.0.md`](../docs/plan/1.0.md) as *built and not connected*: delete it, or say what it is for. -Everything Schcom was blocked on in the kernel now exists: unix domain sockets, -raw input events, shared memory, framebuffer `mmap`, and — as of the descriptor -passing work — `SCM_RIGHTS`, which is the one thing a Wayland client cannot be -written without. What is still missing is a **shareable memory object**, item 2 -of [`docs/plan/foundations.md`](../docs/plan/foundations.md): a Wayland buffer -is a file descriptor for memory, and a System V segment id is not one. - -`schinst/` and `schfs-linux/` are scoped and unstarted, and are **frozen** — -see [`docs/plan/1.0.md`](../docs/plan/1.0.md). +`schinst/` and `schfs-linux/` are scoped and unstarted, and are **frozen** -- see +[`docs/plan/1.0.md`](../docs/plan/1.0.md). diff --git a/projects/schcom/schcom b/projects/schcom/schcom new file mode 100644 index 0000000..92d09d6 Binary files /dev/null and b/projects/schcom/schcom differ diff --git a/projects/schcom/schcom-demo b/projects/schcom/schcom-demo new file mode 100644 index 0000000..d6cdb49 Binary files /dev/null and b/projects/schcom/schcom-demo differ diff --git a/tools/python/stats.py b/tools/python/stats.py index 7b96b38..57e73d5 100644 --- a/tools/python/stats.py +++ b/tools/python/stats.py @@ -23,7 +23,7 @@ import sys ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -FIRST_PARTY = ["kernel", "include", "boot", "lib", "cmd", "tools"] +FIRST_PARTY = ["kernel", "include", "boot", "lib", "cmd", "tools", "ldls", "etc"] LANGS = { ".c": "C", ".h": "C", ".S": "Assembly", ".s": "Assembly", ".asm": "Assembly", ".py": "Python", ".sh": "Shell", ".ps1": "PowerShell", ".pl": "Perl",