SchOS/tools/windows/bootcheck.ps1

# Headless boot check: run SchOS under QEMU, type commands at the shell, and
# assert on what actually comes back.
#
# Booting to a prompt proves nothing on its own -- a filesystem can mount and
# still fail on the first real command, and exactly that has happened here
# before. So this drives the serial console and greps the transcript for the
# answers those commands should produce.
#
# The console is stdio with a chardev logfile: QEMU writes every byte of the
# session to a file itself, so this script only ever has to *write* to stdin.
# Reading QEMU's stdout from PowerShell needs a reader that cannot block the
# writer, and two attempts at that (a background task, then a duplex named
# pipe) both deadlocked. Letting QEMU do the logging removes the problem.
#
# QEMU's serial line has no flow control: a burst of characters gets dropped,
# and anything typed before the shell is listening is discarded outright. Hence
# the pacing below -- not superstition, just the difference between a reliable
# check and a flaky one.
#
#   tools/bootcheck.ps1
#   tools/bootcheck.ps1 -Tty -Ps

param(
    [switch]$Fb,
    # Boot schos.iso from an emulated optical drive with NO hard disk
    # attached, which is the only way to prove the ISO is self-contained.
    # Needs `make iso` first.
    [switch]$Iso,
    # Accepted and ignored. It used to select OVMF over the BIOS loader; there
    # is only the one path now, so it names a choice that no longer exists.
    [switch]$Uefi,
    # Terminal behaviour: pipes, isatty, SIGPIPE, and Ctrl-C. None of these
    # can be checked by looking at a screenshot, and all of them were broken
    # in ways that looked like broken *commands*.
    [switch]$Tty,
    # Attach an xHCI controller with a keyboard, a mouse and a mass storage
    # device on it. Nothing on the default machine has a USB port, so without
    # this the USB driver is never asked to do anything and a broken one looks
    # exactly like a working one.
    [switch]$Usb,
    # Unix domain sockets. Nothing in the default command set opens one, and a
    # socket layer that accepted every call and dropped every byte would look
    # identical from the outside -- socktest has one process write and another
    # read, which is the only check that can tell the difference.
    [switch]$Sock,
    # Raw input events. Needs the QEMU monitor for the same reason the USB run
    # does: a key has to be *pressed*, and characters typed at the serial port
    # never touch the keyboard controller at all.
    [switch]$Evdev,
    # Process enumeration. /proc is a synthetic filesystem now, and the thing
    # it exists for -- one directory per process -- is exactly what nothing
    # else in this suite touches.
    [switch]$Ps,
    # TCP/IP. A listener is started on this host and the guest is told to
    # connect out to it, which is the only arrangement that checks both
    # directions from both ends: the guest reports what it received, and the
    # listener reports what arrived. A stack that completed a handshake and
    # then dropped every byte would pass any check made from one side alone.
    #
    # Under QEMU's user-mode networking the host is 10.0.2.2 seen from inside,
    # and SLIRP connects to the host's loopback -- so the listener binds there.
    [switch]$Net,
    [int]$BootSeconds = 20,
    [string]$Qemu
)

# QEMU, on whichever host this is.
#
# PowerShell 7 runs on Linux, macOS and the BSDs as well as Windows, so these
# scripts are usable from all of them -- but the old default was a hardcoded
# scoop path under $env:USERPROFILE, which exists on exactly one of those. PATH
# first, because a package-managed qemu is on it everywhere; the scoop location
# is only a fallback for the Windows install that does not add itself to PATH.
function Find-Qemu {
    $exe = if ($IsWindows -or $env:OS -eq 'Windows_NT') { 'qemu-system-x86_64.exe' }
           else { 'qemu-system-x86_64' }
    $onPath = Get-Command $exe -ErrorAction SilentlyContinue
    if ($onPath) { return $onPath.Source }
    if ($env:USERPROFILE) {
        $scoop = Join-Path (Join-Path $env:USERPROFILE 'scoop/apps/qemu/current') $exe
        if (Test-Path $scoop) { return $scoop }
    }
    foreach ($p in @('/usr/bin', '/usr/local/bin', '/opt/homebrew/bin')) {
        $c = Join-Path $p $exe
        if (Test-Path $c) { return $c }
    }
    throw "qemu-system-x86_64 not found on PATH. Install QEMU, or pass -Qemu <path>."
}

if (-not $Qemu) { $Qemu = Find-Qemu }

$ErrorActionPreference = 'Stop'
# Repo root is two levels up: this script lives in tools/windows/.
$root = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent
# The root filesystem. SchFS is the only root there is, so this is not a
# choice -- a kernel handed anything else on device 1 panics in forkret().
$hdb  = Join-Path $root 'build/schfs.img'
$log  = Join-Path $root 'build/bootcheck.log'
# Where the running QEMU's pid is recorded, so the *next* run can clean up after
# one that never got to its `finally` -- see reap_stale_qemu below.
$pidfile = Join-Path $root 'build/bootcheck-qemu.pid'

Remove-Item -Force $log -ErrorAction SilentlyContinue

# Kill a QEMU left over from a previous run.
#
# The `finally` at the bottom stops QEMU on every path the script controls, but
# not when the script itself is killed -- a CI timeout, Ctrl-C at the wrong
# moment, `pkill pwsh`. QEMU then keeps running with no parent, and because
# these tests run under TCG rather than KVM, a spare emulator pinning a core
# makes every *later* run slower. Commands that normally answer in a second
# drift past Wait-Prompt's timeout, the next command is typed into a busy
# shell, and the transcript interleaves -- which surfaces as a failed check
# somewhere unrelated to whatever actually changed. A test environment that
# silently degrades is worse than one that fails loudly, so each run reaps the
# last one's leftovers before starting.
function Reap-StaleQemu {
    if (-not (Test-Path $pidfile)) { return }
    $stale = (Get-Content $pidfile -Raw -ErrorAction SilentlyContinue).Trim()
    Remove-Item -Force $pidfile -ErrorAction SilentlyContinue
    if (-not $stale) { return }
    $proc = Get-Process -Id ([int]$stale) -ErrorAction SilentlyContinue
    # Check the name too: pids are recycled, and killing an unrelated process
    # because it inherited the number would be a far worse bug than the leak.
    if ($proc -and $proc.ProcessName -like 'qemu*') {
        Write-Host "  reaping leaked QEMU from a previous run (pid $stale)" -ForegroundColor Yellow
        try { $proc.Kill() } catch { }
    }
}
Reap-StaleQemu

# The default run exercises the *root* filesystem: create a file, read it back,
# grow it, list a directory, seek around it. Booting to a prompt proves nothing
# on its own, and every one of these has broken at some point while the boot
# still looked perfectly healthy.
$commands = @(
    'echo hello-from-schos > /tmp/bc.txt'
    'cat /tmp/bc.txt'
    'echo second-line >> /tmp/bc.txt'
    'wc -c /tmp/bc.txt'
    'mkdir /tmp/bcdir'
    'cp /tmp/bc.txt /tmp/bcdir/copy.txt'
    'cat /tmp/bcdir/copy.txt'
    'ls /tmp/bcdir'
    'rm /tmp/bcdir/copy.txt'
    'rmdir /tmp/bcdir'
    'ls /bin | wc -l'
    'cat /proc/pci'
)
# Every command above already went to SchFS. What is left is to show that
# it is SchFS they went to, and that the volume noticed: the generation
# counter must advance across the writes, which is the one thing a
# read-only mount or a silently-discarded write could not fake.
$commands += 'cat /proc/schfs'
# Mounting the root device a second time has to fail. It is the check that
# tells "the volume is genuinely mounted as /" apart from "it happens to be
# readable", and getting EBUSY here is the correct answer, not a defect.
$commands += 'mkdir /mnt'
$commands += 'mount -t schfs /dev/hdb /mnt'
# A deep path built and torn down entirely on the root volume.
$commands += 'mkdir /tmp/rootdir'
$commands += 'mkdir /tmp/rootdir/sub'
$commands += 'echo schfs-is-root > /tmp/rootdir/sub/deep.txt'
$commands += 'cat /tmp/rootdir/sub/deep.txt'
$commands += 'ls /tmp/rootdir/sub'
$commands += 'rm /tmp/rootdir/sub/deep.txt'
$commands += 'rmdir /tmp/rootdir/sub'
$commands += 'rmdir /tmp/rootdir'
# A binary loaded off the root volume, not a shell builtin: exec() reading
# an ELF through schfs_ops is the path that carried init itself.
$commands += 'wc -l /etc/motd'
# The second dump. The generation counter is compared against the first
# one: on a copy-on-write volume it moves on every committed transaction,
# so a root that silently discarded its writes would still pass every read
# above and fail here.
$commands += 'cat /proc/schfs'

# init must survive signals it has no handler for.
#
# This is a crash-the-whole-machine bug, not a nicety: Ctrl-C sends SIGINT to
# the foreground process group, init is pgid 1, and any window where
# tty_fg_pgrp is 1 lands it on init -- which had no handler, so killed was set,
# exit() ran, and exit() panics outright on initproc. Spamming Ctrl-C at a
# frozen program is what finds that window.
#
# SIGKILL is in here deliberately: init is not killable from userspace on any
# Unix, and a guard that covers SIGINT but not SIGKILL is half a fix.
$commands += 'kill -INT 1'
$commands += 'kill -KILL 1'
$commands += 'kill -TERM 1'
$commands += 'echo INIT-SURVIVED'
$commands += 'ps | head -3'

# KeepAlive. Exiting the shell must bring a new one back -- that is the
# supervision loop doing its job, and it is the only part of schinit that
# cannot be checked by reading its startup output.
$commands += 'echo BEFORE-EXIT'
$commands += 'exit'
$commands += 'echo SHELL-RESPAWNED'

if ($Fb) {
    # /dev/fb0. The geometry has to come back from the mode the bootloader
    # actually programmed -- three places in the kernel used to assume
    # 0xfd000000 at 800x600x32, and nothing would have said when that stopped
    # being true. `bars` goes through the device's write(), `fill` through
    # mmap, so a failure names which of the two paths broke.
    $commands += 'fbinfo'
    $commands += 'fbinfo bars'
    $commands += 'fbinfo fill'
}
if ($Tty) {
    # `ls` piped into `wc -l` reported 48 rather than 240 because ioctl
    # answered TIOCGWINSZ on a *pipe* with success, so ls formatted in
    # columns. `yes` failed outright because writev capped iovcnt at 16 and
    # toybox builds 256 of them. `cat | head` killed the shell, because
    # SIG_IGN is handler value 1 and signal delivery jumped to address 1.
    # Three different-looking bugs, none of them in the commands.
    $commands += 'ls /bin | wc -l'
    $commands += 'yes | head -2'
    # 4 bytes, not 3: an ELF file starts with 0x7F 'E' 'L' 'F', so three bytes
    # stops at "EL" and the invisible leading byte makes that easy to misread.
    $commands += 'cat /bin/toybox | head -c 4; echo'
    $commands += 'printf ''caf\303\251-\303\261-\303\266\n'''
    $commands += 'echo TTY-SURVIVED'
    # Shared memory and file-backed mmap: the two things a display server
    # cannot be written without, and the two whose failure mode is a
    # successful-looking return value.
    $commands += 'shmtest'
}
if ($Usb) {
    # /proc/interrupts is how you tell a registered handler from one that
    # merely compiled. A line with a zero count is a device that was found and
    # never heard from.
    $commands += 'cat /proc/interrupts'
    # Read a real block off the USB stick, through the buffer cache. Answering
    # INQUIRY proves a device is there; only this proves it can read.
    $commands += 'echo s > /proc/sysrq'
}
if ($Sock) {
    $commands += 'socktest'
}
if ($Net) {
    # A free loopback port, found by binding one and letting go, so two runs on
    # one machine do not collide. It has to be chosen here rather than with the
    # rest of the network setup below: the command line that uses it is built
    # in this block, and an unset variable interpolates to nothing -- which is
    # how nettest first ran with no port at all and reported its usage.
    $l = [System.Net.Sockets.TcpListener]::new([System.Net.IPAddress]::Loopback, 0)
    $l.Start(); $script:netPort = $l.LocalEndpoint.Port; $l.Stop()

    # A second port for UDP. Bound the same way, and separately: a free TCP
    # port says nothing about the UDP port of the same number.
    $u = [System.Net.Sockets.UdpClient]::new(0)
    $script:netUdpPort = $u.Client.LocalEndPoint.Port; $u.Close()

    # The interface first, so a failure below reads as "the address never
    # arrived" rather than "TCP is broken" -- two very different bugs that look
    # identical from a failed connect.
    $commands += 'cat /proc/netif'
    # The raw driver path, underneath lwIP: an ARP request built by hand and
    # the reply read back off the receive ring. It lives on the netdev class
    # now rather than inside the e1000, so this is also the check that moving
    # it did not break it.
    $commands += 'echo n > /proc/sysrq'
    $commands += "nettest 10.0.2.2 $netPort"
    $commands += "nettest -u 10.0.2.2 $netUdpPort"
}
if ($Ps) {
    $commands += 'ls /proc'
    $commands += 'cat /proc/1/stat'
    $commands += 'ps'
    $commands += 'ps -A -o pid,ppid,comm'
    $commands += 'top -b -n 1 | head -6'
}
$commands += 'echo MARKER-DONE'

# The ISO run attaches no hard disk at all. That is the whole point: a kernel
# that still quietly needs one would boot happily here otherwise, and only
# fail on someone else's machine.
$isoPath = Join-Path $root 'schos.iso'
if ($Iso -and -not (Test-Path $isoPath)) { throw "no $isoPath -- run 'make iso' first" }

# OVMF is not optional any more: the BIOS loader is gone, so every run boots
# through firmware. -Uefi is still accepted and does nothing, because it named
# a choice that no longer exists.
$code = Join-Path $root 'build/OVMF_CODE.fd'
$vars = Join-Path $root 'build/OVMF_VARS.fd'
foreach ($f in @($code, $vars)) {
    if (-not (Test-Path $f)) { throw "no $f -- run 'make qemu-uefi' once to fetch OVMF" }
}
# The varstore is written by the firmware, so each run gets its own copy.
# Sharing one would let a run that changed a UEFI variable -- boot order,
# console mode -- decide how the next run behaves, which is exactly the kind of
# dependency between runs that makes a suite stop being evidence.
$runVars = Join-Path $root 'build/bootcheck-OVMF_VARS.fd'
Copy-Item -Force $vars $runVars

$firmware = @("-drive if=pflash,format=raw,unit=0,readonly=on,file=`"$code`""
              "-drive if=pflash,format=raw,unit=1,file=`"$runVars`"")

$drives = if ($Iso) {
    # No hard disk at all, so a kernel that still secretly depends on one fails
    # here rather than in someone's optical drive.
    $firmware + @("-cdrom `"$isoPath`"", '-boot d')
} else {
    $esp = Join-Path $root 'build/esp'
    if (-not (Test-Path (Join-Path $esp 'EFI/BOOT/BOOTX64.EFI'))) {
        throw "no staged ESP at $esp -- run 'make esp' first"
    }
    $firmware + @("-drive file=fat:rw:`"$esp`",index=0,media=disk,format=raw"
                  "-drive file=`"$hdb`",format=raw,index=1,media=disk")
}

$psi = New-Object System.Diagnostics.ProcessStartInfo
$psi.FileName = $Qemu

$usbDevices = if ($Usb) {
    @('-device qemu-xhci,id=xhci'
      '-device usb-kbd,bus=xhci.0'
      '-device usb-hub,id=hub,bus=xhci.0,port=2'
      '-device usb-mouse,bus=xhci.0,port=2.1'
      "-drive id=usbstick,file=`"$(Join-Path $root 'build/usbstick.img')`",format=raw,if=none"
      '-device usb-storage,bus=xhci.0,drive=usbstick')
} else { @() }

# A free loopback port, found by binding one and letting go. Hardcoding it made
# two runs on one machine collide.
$monPort = if ($Usb -or $Evdev) {
    $l = [System.Net.Sockets.TcpListener]::new([System.Net.IPAddress]::Loopback, 0)
    $l.Start(); $mp = $l.LocalEndpoint.Port; $l.Stop(); $mp
} else { 0 }

$monitor = if ($Usb -or $Evdev) { @("-monitor tcp:127.0.0.1:$monPort,server,nowait") } else { @() }

# The network run: an explicit NIC, and a listener on this host for the guest
# to talk to.
#
# The NIC is spelled out rather than left to QEMU's default. Every other run
# gets the default one and that is fine for "did a driver bind", but a check
# that asserts on an address wants to be reading its own configuration rather
# than whatever the machine type happens to supply this year.
$netDevices = if ($Net) {
    @('-netdev user,id=net0', '-device e1000,netdev=net0')
} else { @() }

# The listener runs in a background job -- a separate process -- rather than on
# a thread here. This script's one hard rule is that nothing may block the
# writer feeding QEMU's stdin, and a socket accept is exactly the kind of wait
# that has deadlocked this harness before.
#
# It answers the protocol cmd/nettest speaks: read what the guest sends, send a
# line back, close. The close matters -- nettest reads until end of file, so a
# listener that stayed open would hang the guest rather than fail it.
$netJob = if ($Net) {
    Start-Job -ArgumentList $netPort -ScriptBlock {
        param($port)
        $l = [System.Net.Sockets.TcpListener]::new([System.Net.IPAddress]::Loopback, $port)
        $l.Start()
        # Generous, because it has to cover the whole boot and every command
        # before nettest, under TCG. Nothing waits on this deadline: the job is
        # torn down by the caller either way.
        $deadline = (Get-Date).AddSeconds(900)
        while (-not $l.Pending()) {
            if ((Get-Date) -gt $deadline) { $l.Stop(); return '' }
            Start-Sleep -Milliseconds 200
        }
        $c = $l.AcceptTcpClient()
        $s = $c.GetStream()
        $s.ReadTimeout = 30000
        $buf = New-Object byte[] 512
        $got = ''
        try {
            $n = $s.Read($buf, 0, $buf.Length)
            if ($n -gt 0) { $got = [Text.Encoding]::ASCII.GetString($buf, 0, $n) }
        } catch { }
        $reply = [Text.Encoding]::ASCII.GetBytes("HELLO-FROM-HOST`n")
        try { $s.Write($reply, 0, $reply.Length); $s.Flush() } catch { }
        $c.Close()
        $l.Stop()
        return $got
    }
} else { $null }

# The UDP responder. A separate job because it is a separate exchange, and
# because a datagram socket has nothing to accept: it waits for one datagram
# and answers the address it came from, which is precisely what the guest is
# being asked to report back.
$netUdpJob = if ($Net) {
    Start-Job -ArgumentList $netUdpPort -ScriptBlock {
        param($port)
        $u = [System.Net.Sockets.UdpClient]::new($port)
        $u.Client.ReceiveTimeout = 900000
        $ep = [System.Net.IPEndPoint]::new([System.Net.IPAddress]::Any, 0)
        $got = ''
        try {
            $data = $u.Receive([ref]$ep)
            $got  = [Text.Encoding]::ASCII.GetString($data)
            $reply = [Text.Encoding]::ASCII.GetBytes("HELLO-UDP-FROM-HOST`n")
            $u.Send($reply, $reply.Length, $ep) | Out-Null
            # A second datagram, so the guest can show that it reads them one
            # at a time. Two sends rather than one long one: the boundary is
            # the property being tested, and it only exists if it was put there
            # by the sender.
            $second = [Text.Encoding]::ASCII.GetBytes("SECOND-DATAGRAM`n")
            $u.Send($second, $second.Length, $ep) | Out-Null
        } catch { }
        $u.Close()
        return $got
    }
} else { $null }

$psi.Arguments = ($drives + $usbDevices + $netDevices + $monitor + @(
    '-m 1G -smp 1 -cpu qemu64,+rdtscp -machine pc-i440fx-9.2'
    '-display none'
    "-chardev stdio,id=c0,logfile=`"$log`""
    '-serial chardev:c0'
)) -join ' '
$psi.RedirectStandardInput = $true
$psi.UseShellExecute       = $false

$what = if ($Iso) { 'schos.iso, no hard disk' } else { "hdb = $(Split-Path $hdb -Leaf)" }
Write-Host "==> booting ($what)" -ForegroundColor Cyan
$p = [System.Diagnostics.Process]::Start($psi)
# Recorded immediately, so that a run killed one instruction later is still
# cleaned up by the next one.
Set-Content -Path $pidfile -Value $p.Id

# Commands that never produced a prompt. Collected rather than just printed:
# once one command overruns, every later check is reading a transcript that may
# be interleaved, so this has to be visible in the summary and not scrolled past
# in the middle of a long run.
$promptMisses = @()

try {
    Start-Sleep -Seconds $BootSeconds     # wait for the shell prompt

    if ($Usb) {
        # Type a whole command on the *USB* keyboard. QEMU routes sendkey to
        # the most recently attached keyboard, so with usb-kbd present these
        # keystrokes go through xHCI, the interrupt endpoint, the HID report
        # decoder and the console -- none of which the serial path touches.
        # If any link in that chain is broken the characters simply vanish,
        # which is exactly how the missing interrupt showed up.
        Write-Host '  > (usb keyboard) echo usbworks' -ForegroundColor DarkGray
        $mon = New-Object System.Net.Sockets.TcpClient('127.0.0.1', $monPort)
        $monw = New-Object System.IO.StreamWriter($mon.GetStream())
        $monw.AutoFlush = $true
        Start-Sleep -Milliseconds 400
        foreach ($k in @('e','c','h','o','spc','u','s','b','w','o','r','k','s','ret')) {
            $monw.WriteLine("sendkey $k")
            Start-Sleep -Milliseconds 200
        }
        Start-Sleep -Seconds 2
        $mon.Close()
    }

    # Wait until the shell has finished the last command and printed a fresh
    # prompt, rather than sleeping a fixed interval and hoping.
    #
    # A flat 4-second wait was wrong in both directions: too long for the
    # dozens of commands that answer instantly, and too short for `ls /bin |
    # wc -l`, which reliably overran it. When it overran, the next command was
    # typed while the shell was still busy -- the tty echoed it but no reader
    # took it, and it ended up concatenated onto the following one. The
    # transcript showed `cat /proc/pcicat /proc/schfs` and three PCI checks
    # failed, which looks exactly like a broken PCI driver and is not one.
    #
    # This cannot read QEMU's stdout -- see the header; two attempts at that
    # deadlocked. It reads the logfile QEMU is writing instead, which is the
    # same bytes with none of the plumbing.
    function Read-Log {
        try { $t = Get-Content $log -Raw -ErrorAction Stop } catch { return $null }
        if ($null -eq $t) { return '' }
        return $t
    }

    # Wait for a prompt printed *after* $Since characters of transcript.
    #
    # Anchoring is not optional. Between writing a command to stdin and the
    # guest echoing it, the transcript still ends with the *previous* prompt.
    # Matching the whole file sees that stale prompt and returns immediately,
    # so the next command is typed on top of one that has not started yet --
    # a false pass that corrupts everything after it. Only text produced since
    # the command was sent counts.
    #
    # The timeout is 60 s rather than 20 s because these tests run under TCG,
    # not KVM. Measured on the Fedora laptop with the guest's own /proc/uptime,
    # a cold `ls /bin | wc -l` takes 23 s -- it was over the old wall every
    # time, and before the region-based allocator landed it took 51 s. A
    # timeout only costs time when something is genuinely wrong, whereas one
    # that is too short costs a false failure somewhere else entirely.
    function Wait-Prompt([int]$Since, [int]$TimeoutMs = 60000) {
        $deadline = (Get-Date).AddMilliseconds($TimeoutMs)
        while ((Get-Date) -lt $deadline) {
            Start-Sleep -Milliseconds 150
            $t = Read-Log
            if ($null -eq $t -or $t.Length -le $Since) { continue }
            # A prompt at the very end, with nothing typed after it.
            if ($t.Substring($Since) -match '(?s)#[ ]?$') { return $true }
        }
        return $false
    }

    # Wait for the guest to stop producing output, used only when Wait-Prompt
    # has given up.
    #
    # The old fallback slept two seconds and typed regardless, which is exactly
    # how a slow command becomes a failure attributed to a later one: the tty
    # echoes the new command, no reader takes it, and it arrives concatenated
    # onto the next. Waiting for the transcript to stop growing at least means
    # the shell is idle before anything else is sent.
    function Wait-Quiet([int]$StableMs = 1500, [int]$TimeoutMs = 45000) {
        $deadline   = (Get-Date).AddMilliseconds($TimeoutMs)
        $last       = -1
        $lastChange = Get-Date
        while ((Get-Date) -lt $deadline) {
            Start-Sleep -Milliseconds 250
            $t   = Read-Log
            $len = if ($null -eq $t) { -1 } else { $t.Length }
            if ($len -ne $last) { $last = $len; $lastChange = Get-Date; continue }
            if (((Get-Date) - $lastChange).TotalMilliseconds -ge $StableMs) { return $true }
        }
        return $false
    }

    foreach ($cmd in $commands) {
        Write-Host "  > $cmd" -ForegroundColor DarkGray
        $pre   = Read-Log
        $since = if ($null -eq $pre) { 0 } else { $pre.Length }
        foreach ($ch in ($cmd + "`n").ToCharArray()) {
            $p.StandardInput.Write($ch)
            $p.StandardInput.Flush()
            Start-Sleep -Milliseconds 30   # no flow control: pace the input
        }
        if (-not (Wait-Prompt $since)) {
            $promptMisses += $cmd
            Write-Host "  (no prompt after '$cmd' -- waiting for the guest to go quiet)" -ForegroundColor Yellow
            Wait-Quiet | Out-Null
        }
    }

    if ($Evdev) {
        # Start a reader on /dev/input/event0 and then press keys at it. The
        # program blocks in read() until events arrive, so if the keyboard
        # driver is not feeding the input layer this hangs and prints nothing
        # -- which is a failure that says exactly which half is broken.
        Write-Host '  > inputtest -n 6, then press keys' -ForegroundColor DarkGray
        foreach ($ch in "inputtest -n 6`n".ToCharArray()) {
            $p.StandardInput.Write($ch); $p.StandardInput.Flush()
            Start-Sleep -Milliseconds 30
        }
        Start-Sleep -Seconds 2

        $mon  = New-Object System.Net.Sockets.TcpClient('127.0.0.1', $monPort)
        $monw = New-Object System.IO.StreamWriter($mon.GetStream())
        $monw.AutoFlush = $true
        Start-Sleep -Milliseconds 400
        # 'a' is keycode 30 and shift is 42, chosen because both numbers are
        # unmistakable in the output and neither is a scancode by accident.
        # Each sendkey is a press *and* a release, which is the whole point:
        # the console would report one character and nothing else.
        foreach ($k in @('a', 'shift', 'a', 'b')) {
            $monw.WriteLine("sendkey $k")
            Start-Sleep -Milliseconds 400
        }
        $mon.Close()
        Start-Sleep -Seconds 3

        # Those keypresses reached the console as well as the input device --
        # that is the design, not a leak -- so the shell's input line now holds
        # "aab". Clear it before typing the next command, or the shell tries to
        # run `aabecho`.
        $p.StandardInput.Write("`n"); $p.StandardInput.Flush()
        Start-Sleep -Milliseconds 500

        foreach ($ch in "echo INPUT-DONE`n".ToCharArray()) {
            $p.StandardInput.Write($ch); $p.StandardInput.Flush()
            Start-Sleep -Milliseconds 30
        }
        Start-Sleep -Seconds 3
    }

    if ($Tty) {
        # Ctrl-C has to reach the *foreground process group*. The old handler
        # set killed=1 on whatever process happened to be running when the
        # keyboard interrupt fired, which is almost never the one on screen.
        Write-Host '  > sleep 30, then Ctrl-C' -ForegroundColor DarkGray
        foreach ($ch in "sleep 30`n".ToCharArray()) {
            $p.StandardInput.Write($ch); $p.StandardInput.Flush()
            Start-Sleep -Milliseconds 30
        }
        Start-Sleep -Seconds 2
        $p.StandardInput.Write([char]3)   # ^C
        $p.StandardInput.Flush()
        Start-Sleep -Seconds 2
        foreach ($ch in "echo INTERRUPTED-OK`n".ToCharArray()) {
            $p.StandardInput.Write($ch); $p.StandardInput.Flush()
            Start-Sleep -Milliseconds 30
        }
        Start-Sleep -Seconds 3
    }
} finally {
    if (-not $p.HasExited) { $p.Kill() }
    Start-Sleep -Milliseconds 800
    Remove-Item -Force $pidfile -ErrorAction SilentlyContinue
}

# What the listener on this host received, if there was one. Collected after
# QEMU is gone: the job either finished when the guest closed the connection or
# is still sitting in accept, and the second case is a failure to report rather
# than one to wait out.
$netFromGuest = ''
$netUdpFromGuest = ''
if ($Net -and $netJob) {
    Wait-Job $netJob -Timeout 20 | Out-Null
    $netFromGuest = (Receive-Job $netJob) -join ''
    Remove-Job $netJob -Force -ErrorAction SilentlyContinue
}
if ($Net -and $netUdpJob) {
    Wait-Job $netUdpJob -Timeout 20 | Out-Null
    $netUdpFromGuest = (Receive-Job $netUdpJob) -join ''
    Remove-Job $netUdpJob -Force -ErrorAction SilentlyContinue
}

if (-not (Test-Path $log)) { throw "no serial log produced at $log" }
$text = Get-Content $log -Raw
if ($null -eq $text) { $text = '' }

Write-Host "==> serial transcript ($($text.Length) bytes) -> $log" -ForegroundColor Cyan
Write-Host $text

$script:fail = $false
function Expect($pattern, $what) {
    if ($script:text -match $pattern) {
        Write-Host "  [ok] $what" -ForegroundColor Green
    } else {
        Write-Host "  FAIL: $what (no match for /$pattern/)" -ForegroundColor Red
        $script:fail = $true
    }
}

# Assert something the transcript does not state literally. Expect can only ask
# whether a string is present, which is why the framebuffer checks below used to
# hardcode 800x600: it was the only mode the BIOS loader ever set, so a literal
# was indistinguishable from an invariant. Under UEFI the firmware chooses, so
# the literal was testing the firmware's taste rather than our code.
function ExpectThat($cond, $what, $detail) {
    if ($cond) {
        Write-Host "  [ok] $what" -ForegroundColor Green
    } else {
        Write-Host "  FAIL: $what ($detail)" -ForegroundColor Red
        $script:fail = $true
    }
}

# Pull a single "name   value" field out of the fbset-style dump.
function FbField($name) {
    if ($script:text -match "(?m)^$name\s+(\d+)") { return [int]$Matches[1] }
    return -1
}

Expect 'MARKER-DONE'              'shell stayed alive through every command'
Expect 'hello-from-schos'       'a file written on the root filesystem reads back'
Expect 'second-line'              'appending to that file works'
# 17 bytes of "hello-from-schos\n" plus 12 of "second-line\n". The exact
# number is the point: an append that silently truncated still produced a file,
# and only the byte count said which one.
#
# It follows the string above, so changing that changes this -- the SchOS
# rename shortened "hello-from-schunix" by two characters and this check went
# red on a filesystem that was working perfectly.
Expect '(?m)^\s*29\s'             'append extended the file rather than replacing it'
Expect '(?m)^copy\.txt'           'a copied file appears in a new directory'
Expect '(?ms)copy\.txt.*rmdir'    'rmdir removed the directory without an error'

# PCI. The point of these is that the numbers come off the bus rather than out
# of a header: the host bridge identifies the chipset QEMU is emulating, and
# the display controller's aperture is the same address VBE reported, checked
# by two completely independent paths.
Expect 'pci: \d+ devices'         'the PCI bus enumerated'
Expect '00:00\.0 8086:1237'       'the host bridge is where a host bridge belongs'
Expect '00:01\.1 8086:7010 IDE'   'the IDE controller was identified by class'
Expect '\[vga\]'                  'a driver bound itself to the display controller'
Expect 'vga: display controller'  'the display driver found its aperture'

# If init had died the machine would have panicked and nothing below this
# point would appear at all -- but check explicitly, so a failure names the
# cause instead of showing up as "the shell stopped answering".
Expect 'INIT-SURVIVED'        'init survives SIGINT, SIGKILL and SIGTERM'
Expect 'schinit: ready \(pid 1\)' 'schinit came up as process 1'
Expect 'job.? loaded'         'schinit read its manifests from /etc/schinit'
Expect 'starting os\.schos\.' 'a job started by Label, not by hardcoded path'
Expect 'SHELL-RESPAWNED'      'KeepAlive restarted the shell after it exited'
Expect '(?m)^\s*1\s'          'pid 1 is still in the process table afterwards'

Expect 'schfs: mounted'       'the kernel mounted SchFS as the root filesystem'
Expect 'init: ready'          'init came up on a SchFS root'
Expect 'label:\s*schos'     '/proc/schfs reports the volume label'
Expect 'root_objectid:\s*16'  '/proc/schfs reports the root object id'
Expect 'dir\s+bin'            'the root directory really contains bin/'
# / is the mounted volume, so mounting that same device again must be
# refused. Without this a kernel that had quietly fallen back to some other
# root would still pass everything else here.
Expect 'Resource busy'        'mounting the root device twice is refused'
Expect 'schfs-is-root'        'a file three directories deep reads back'
Expect '(?m)^deep\.txt'       'that file is listed by its full name'
Expect '(?m)^\s*1\s+/etc/motd' 'a binary loaded off the SchFS root runs'
# Two /proc/schfs dumps, before and after the writes. Same volume, so the
# generation must have moved -- if it has not, nothing was committed.
$gens = [regex]::Matches($text, 'generation:\s*(\d+)')
if ($gens.Count -ge 2 -and
    [int]$gens[$gens.Count-1].Groups[1].Value -gt [int]$gens[0].Groups[1].Value) {
    Write-Host "  [ok] the volume generation advanced across the writes" -ForegroundColor Green
} else {
    Write-Host "  FAIL: the volume generation did not advance -- writes were not committed" -ForegroundColor Red
    $script:fail = $true
}

if ($Fb) {
    Expect 'id\s+schosfb'        'FBIOGET_FSCREENINFO identifies the device'
    # The bug this guards against is an ioctl that reports success having
    # touched nothing -- a 0x0 screen. What proves the numbers are real is not
    # any particular pair of them but that they agree with each other and with
    # a mode the hardware could actually scan out.
    $xres = FbField 'xres'
    $yres = FbField 'yres'
    $line = FbField 'line_length'
    ExpectThat ($xres -ge 640) 'the reported width is a mode actually set' "xres = $xres"
    ExpectThat ($yres -ge 480) 'the reported height is a mode actually set' "yres = $yres"
    # A width that is not a multiple of 8 is the mode that sheared under OVMF:
    # the firmware advertised 1366 and the hardware scanned out 1360.
    ExpectThat (($xres % 8) -eq 0) 'the width is a whole number of scanout units' "xres = $xres"
    Expect '(?m)^bits_per_pixel\s+32' 'the reported depth is the mode actually set'
    Expect '(?m)^smem_start\s+0x'  'the framebuffer reports a physical address'
    # The stride must come from the mode information, not be recomputed as
    # xres*bpp -- on a mode whose scanline is padded those two differ, and every
    # row after the first lands in the wrong place.
    ExpectThat ($line -ge $xres * 4) 'the stride covers a full row of pixels' "line_length = $line, xres = $xres"
    ExpectThat (($line % 4) -eq 0) 'the stride is a whole number of pixels' "line_length = $line"
    Expect 'transp.*absent'        'the absent alpha channel is reported as length 0'
    Expect 'colour bars drawn'     'writing pixels through write() reaches the screen'
    Expect 'gradient drawn'        'mmap of /dev/fb0 maps the real framebuffer'
}

if ($Tty) {
    # Each of these is a bug that once looked like a broken *command*.
    # `ls | wc -l` reported 48 rather than the real count because ioctl
    # answered TIOCGWINSZ on a pipe with success, so ls formatted in columns.
    # Anything in the hundreds means it printed one name per line.
    Expect '(?m)^\s*[2-9][0-9]{2}\s*$' 'ls into a pipe prints one name per line'
    # writev capped iovcnt at 16; toybox builds 256 of them, so `yes` failed
    # outright.
    Expect '(?ms)yes \| head -2\s*\n\s*y\s*\n\s*y' 'yes writes through a pipe'
    # 0x7F E L F. Four bytes, not three -- the leading byte is invisible.
    Expect 'ELF'                   'a binary reads back through a pipe intact'
    Expect 'caf.*-.*-'             'UTF-8 survives the terminal unmangled'
    # SIG_IGN is handler value 1, and signal delivery once jumped to address 1,
    # which killed the shell rather than the pipeline.
    Expect 'TTY-SURVIVED'          'the shell is still alive after all of that'
    Expect 'shmtest: all checks passed' 'shared memory and file-backed mmap work'
}

if ($Usb) {
    # A registered handler that never fires looks exactly like a working one
    # from userspace. A non-zero count on the xhci line is the difference.
    # Not anchored to the start of the line: xHCI shares IRQ 11 with the e1000
    # here, so the line reads "11:  28  e1000,  28  xhci level" and the xhci
    # entry is the second one on it.
    Expect '[1-9]\d*\s+xhci' 'the xHCI interrupt is actually delivered'
    # build/usbstick.img is 16 MB = 32768 sectors of 512 bytes. READ CAPACITY
    # returning the true geometry is what separates a driver that parsed the
    # reply from one that assumed a size.
    Expect 'usb-storage: 32768 sectors of 512 bytes' 'READ CAPACITY returned the true size'
    # And then an actual transfer: block 1 of a v1 image is its superblock.
    Expect 'block 1 magic 10203040' 'a real block came back off the USB disk'
    Expect 'size 4096'             'the superblock read over USB is the right one'
}

if ($Sock) {
    Expect 'socktest: ALL CHECKS PASSED' 'every socket check passed'
}

if ($Net) {
    # The stack came up at all. Cheap, and it separates "lwIP never started"
    # from every other network failure below.
    Expect 'net: lwIP \d+\.\d+\.\d+ up' 'the TCP/IP stack started'
    # 10.0.2.15 is the address QEMU's user-mode networking hands out, so this
    # is DHCP genuinely completing -- discover, offer, request, ack -- and not
    # a hardcoded guess: nothing in the guest knows that number in advance.
    Expect 'net: eth\d 10\.0\.2\.15/'   'DHCP obtained an address from SLIRP'
    # The interface is named for the device, and the device names the driver
    # that claimed it. Together they say the stack went through the netdev
    # class rather than calling a driver it was compiled to know about.
    Expect '(?m)^interface\s+eth\d'     '/proc/netif names the bound interface'
    Expect '(?m)^driver\s+e1000'        'and the driver behind it'
    Expect '(?m)^state\s+up'            '/proc/netif reports the interface up'
    Expect '(?m)^address\s+10\.0\.2\.15' '/proc/netif reports the address it was given'
    Expect '(?m)^gateway\s+10\.0\.2\.2' '/proc/netif reports the gateway'
    # Frames in both directions. DHCP alone guarantees these are non-zero, so a
    # zero here means the counters are wrong rather than the network.
    Expect '(?m)^rx_frames\s+[1-9]'     'frames arrived and reached the stack'
    Expect '(?m)^tx_frames\s+[1-9]'     'frames were transmitted'
    Expect 'netdev: ARP reply'          'the raw driver path carries a frame both ways'

    # And now real traffic. Each of these is a different part of the exchange,
    # so a failure says which one: the handshake, the guest's receive path, and
    # the guest's send path as seen from outside.
    Expect 'nettest: connected'         'a TCP connection out of the guest completed'
    Expect 'nettest: OK'                'the guest read a reply back off that connection'
    Expect 'HELLO-FROM-HOST'            'and the reply is the host''s bytes'
    ExpectThat ($netFromGuest -match 'HELLO-FROM-SCHOS') `
        'the listener on this host received what the guest sent' `
        "the listener read '$($netFromGuest -replace '\s+', ' ')'"

    # UDP, and specifically the source address. recvfrom filling that in is the
    # thing being checked: over a stream the source is known before any byte
    # arrives, so a kernel writing nothing at all would still look right there.
    Expect 'nettest: udp OK'            'a UDP datagram went out and one came back'
    Expect 'HELLO-UDP-FROM-HOST'        'the datagram carried the host''s bytes'
    Expect "nettest: udp source 10\.0\.2\.2:$netUdpPort" `
                                        'recvfrom reported the real sender, address and port'
    ExpectThat ($netUdpFromGuest -match 'HELLO-UDP-FROM-SCHOS') `
        'the responder on this host received the guest''s datagram' `
        "the responder read '$($netUdpFromGuest -replace '\s+', ' ')'"
    # Message boundaries. The responder sends two datagrams of 20 and 16 bytes;
    # a kernel that treated UDP as a byte stream would hand the first read all
    # 36 at once, which is why the counts are asserted and not just the text.
    Expect 'nettest: udp received 20 bytes'  'the first read returned exactly one datagram'
    Expect 'nettest: udp second 16 bytes'    'and the second read returned the next one, whole'
    Expect 'SECOND-DATAGRAM'                 'the second datagram arrived intact'
}

if ($Ps) {
    # /proc is synthetic, so a per-process directory is the thing to prove.
    # `ls /proc` prints in columns, so pid 1 is a token on a line rather than a
    # line of its own.
    Expect '(?m)^1\s+\d'          '/proc has a directory for pid 1'
    Expect '(?m)^1 \('             '/proc/1/stat starts with the pid and comm'
    Expect 'sinit|init'            'ps names pid 1'
    Expect '(?m)PID.*PPID.*COMM'   'ps honours -o with the requested columns'
    Expect 'load average|Mem|CPU'  'top produces a report rather than an error'
}

if ($Evdev) {
    # A key has to be *pressed* for this to say anything: characters typed at
    # the serial port never reach the keyboard controller, which is why this
    # run drives the QEMU monitor's sendkey instead.
    Expect 'inputtest: done|event limit reached' 'the input layer delivered events'
    Expect 'value'                 'events carry a value, not just a type'
}

# A command that never produced a prompt does not fail on its own, but every
# check after it read a transcript that may be interleaved -- so say so next to
# the result rather than only in the scrollback. A FAIL below this line is
# suspect until the transcript has been read.
if ($promptMisses.Count) {
    Write-Host "==> $($promptMisses.Count) command(s) never produced a prompt; later output may be interleaved:" -ForegroundColor Yellow
    $promptMisses | ForEach-Object { Write-Host "      $_" -ForegroundColor Yellow }
}

if ($script:fail) { exit 1 }
Write-Host 'boot check passed' -ForegroundColor Green