SchOS/kernel/core/main.c

#include "types.h"
#include "defs.h"
#include "vt.h"
#include "gfx.h"
#include "param.h"
#include "memlayout.h"
#include "mmu.h"
#include "proc.h"
#include "x86.h"
#include "irq.h"
#include "dma.h"
#include "klog.h"

static void *klog_base;   // the persistent log, just above the kernel image

static void startothers(void);
static void mpmain(void)  __attribute__((noreturn));
extern char end[]; // first address after kernel loaded from ELF file

// Set by stage2main.c (boot/bios/stage2main.c, SCHBOOT_VERBOSE_FLAG) when the
// user picks "Boot SchOS (verbose)" from the F8 boot menu; read once here,
// before anything else, so subsystem init can log its own steps.
int boot_verbose;

// Remote debugging, chosen at the boot menu. See SCHBOOT_DEBUG_FLAG.
//
// The kernel does nothing with it beyond publishing /proc/bootdebug: the thing
// that acts on it is schinitd, because what it turns on is a *job*. A kernel
// that changed its own behaviour on a debug flag would be a kernel whose bugs
// could depend on the flag.
int boot_debug;

// Must match SCHBOOT_VERBOSE_FLAG in boot/bios/stage2main.c exactly: just past
// the VBE info block, not 0x7000 (startothers() below uses that address for
// the AP boot trampoline).
#include "schboot.h"

// Bootstrap processor starts running C code here.
// Allocate a real stack and switch to it, first
// doing some setup required for memory allocator to work.
int
kmain(uint32 fb_addr)
{
  // Carried out of the block below, for the framebuffer that could not be
  // attached there because it needs page tables that do not exist yet.
  struct fb_info fb_info_early;
  uint64 fb_early = 0;
  int    fb_early_ok = 0, fb_attached = 0;

  boot_verbose = *(uchar*)P2V(SCHBOOT_VERBOSE_FLAG);
  boot_debug   = *(uchar*)P2V(SCHBOOT_DEBUG_FLAG);

  // The kernel's first act: put a mark on the screen.
  //
  // Eight lines, before a single subsystem exists, because "the loader
  // finished and the machine froze" has two completely different causes and
  // nothing else here can tell them apart. If this bar appears over the
  // loader's splash, the jump landed, the page tables are good and the kernel
  // is executing; if it does not, none of those are true and everything below
  // is the wrong place to look.
  //
  // It writes straight through the handoff block rather than through gfx,
  // because gfx needs a scanout attached and attaching one is itself a thing
  // that can fail. Skipped when the framebuffer is above the direct map -- a
  // fault here would be a triple fault with the machine even quieter than it
  // was. It costs nothing on a boot that works: the console clears the screen
  // a few lines later.
  {
    uint64 pa = *(uint64*)P2V(SCHBOOT_FB_BASE);
    uint32 w  = *(uint32*)P2V(SCHBOOT_FB_WIDTH);
    uint32 st = *(uint32*)P2V(SCHBOOT_FB_STRIDE);

    if (pa && w && st && pa + (uint64)st * 8 <= 4ULL << 30) {
      volatile uint32 *px = (volatile uint32 *)P2V(pa);
      for (uint32 y = 0; y < 8; y++)
        for (uint32 x = 0; x < w; x++)
          px[y * (st / 4) + x] = 0x7C5CFF;   /* SCH_ACCENT */
    }
  }

  // On a verbose boot, put the kernel log on the screen from the first line.
  //
  // Everything below talks to the serial port and nothing else until the first
  // exec calls console_takeover(). That is fine on a machine with a UART and
  // useless on one without -- and a laptop, a Chromebox, anything built this
  // century has no 16550 at 0x3f8. The screen showed the loader's progress bar
  // and then nothing at all, for the whole of boot, on exactly the machines
  // where something was going wrong.
  //
  // The loader has already put a framebuffer on the screen and described it in
  // the boot block, so there is a perfectly good console sitting there unused.
  // Attaching it here costs one struct and makes the difference between "it
  // froze" and knowing which line it froze after.
  //
  // Unconditional, not just on a verbose boot. F8 is no use to someone whose
  // machine resets before they can read anything, and "it froze after the
  // loading bar" is a bug report nobody can act on. This is what Linux does
  // too: the log is the default and `quiet splash` is the thing you ask for.
  //
  // The splash stays on screen until the kernel's first message, and a boot
  // that succeeds clears the whole lot when the first program starts -- so
  // the only time anyone sees this is the only time anyone needs it.
  {
    struct fb_info bi;
    // The *physical* address, tested before it is translated.
    //
    // This used to be `if (bi.ptr && ...)` with bi.ptr already run through
    // P2V -- and P2V(0) is KERNBASE, which is not zero, so the guard could
    // not fail. The comment below was right and the test could never enforce
    // it.
    //
    // It matters on any machine whose only display is a virtio-gpu. OVMF's
    // virtio GOP implements Blt and nothing else: there is no linear
    // aperture to report, so it publishes width and height and a
    // FrameBufferBase of **zero**. The kernel took that at face value, and
    // gfx_clear() then wrote four megabytes over physical address 0 -- the
    // real-mode IVT, the BIOS data area, the AP trampoline at 0x7000 and the
    // early page tables. The machine reset before printing anything and came
    // back round to the firmware, for ever.
    // The UEFI loader's own display block if it wrote one, and the VBE block
    // otherwise. See SCHBOOT_FB_BASE: the VBE structure is a BIOS one and its
    // framebuffer address is 32 bits, which is not enough to describe a
    // machine whose aperture is above 4 GB.
    uint64 fb_phys = *(uint64*)P2V(SCHBOOT_FB_BASE);

    if (fb_phys) {
      bi.width  = *(uint32*)P2V(SCHBOOT_FB_WIDTH);
      bi.height = *(uint32*)P2V(SCHBOOT_FB_HEIGHT);
      bi.stride = *(uint32*)P2V(SCHBOOT_FB_STRIDE);
      bi.bpp    = 32;
    } else {
      fb_phys   = *(uint32*)P2V(SCHBOOT_VBE_LFB);
      bi.width  = *(uint16*)P2V(SCHBOOT_VBE_WIDTH);
      bi.height = *(uint16*)P2V(SCHBOOT_VBE_HEIGHT);
      bi.stride = *(uint16*)P2V(SCHBOOT_VBE_STRIDE);
      bi.bpp    = *(uchar *)P2V(SCHBOOT_VBE_BPP);
    }
    bi.ptr = (volatile uchar *)P2V(fb_phys);

    // Only if the loader actually described something, and only if the direct
    // map reaches it. A framebuffer above 4 GB needs ioremap(), and ioremap()
    // needs kpdpt, which kvmalloc() has not allocated yet -- so that case is
    // picked up further down, a few dozen instructions later, rather than
    // here. Everything else about the early console is unchanged.
    //
    // A zero here means the BIOS path or a firmware that gave no GOP, and
    // drawing into address zero is a worse failure than staying quiet.
    fb_early = fb_phys;
    fb_early_ok = fb_phys && bi.width && bi.height && bi.stride && bi.bpp >= 16;

    if (fb_early_ok && fb_phys + (uint64)bi.stride * bi.height <= 4ULL << 30) {
      fb_attached = 1;
      gfx_attach_scanout(&bi);
      // Before vt_init, which reads the cell size to work out how many rows
      // and columns there are. The font is in the kernel image, so this needs
      // nothing that does not exist yet -- no allocator, no filesystem.
      vt_init();
      gfx_clear(0x0C0E13);          /* SCH_BG; theme.h is not included here */
      cons_serial_only = 0;
    }
    fb_info_early = bi;
  }

  uartearlyinit();
  if (boot_verbose) cprintf("verbose boot: uartearlyinit done\n");
  // The DMA arena is carved off the top of physical memory *before* kalloc is
  // given the rest, because a device needs physically contiguous memory and
  // kalloc's free list makes no such promise -- two pages from it are adjacent
  // only by chance. Taking it first is the only way to be sure the region
  // exists; asking for it later would mean asking kalloc for something it
  // fundamentally cannot provide. See include/dma.h.
  // Before anything touches PHYSTOP, which is now a variable read from the
  // firmware's memory map rather than a constant asserting 224 MB exists.
  e820_init();

  // The log region: the page after the kernel image, and the page allocator is
  // started above it rather than at `end`.
  //
  // Low memory, deliberately, and it took a measurement to learn why. The
  // obvious place is the top of RAM beside the DMA arena -- and on a reboot
  // the magic came back as ASCII, because that is exactly where the firmware
  // puts its own allocations before it hands the machine over. The bytes below
  // the kernel image are the ones the loader has already committed to leaving
  // alone, and `end` is the same address on every boot of the same build,
  // which is the property a region has to have to be found again.
  //
  // No hole has to be punched in the allocator for this: kinit1's lower bound
  // is a parameter, so moving it up by KLOG_BYTES is the whole of it.
  klog_base = (void *)PGROUNDUP((addr_t)end);
  klog_init(klog_base, KLOG_BYTES);
  // `lowtop`, not PHYSTOP: this runs on the loader's page tables, which map
  // four gigabytes, and freeing a page writes into it. Everything above that
  // is added by kinit_high() once kvmalloc has mapped it.
  kinit1((char *)klog_base + KLOG_BYTES, P2V(lowtop - DMA_ARENA_BYTES));
  if (boot_verbose) cprintf("verbose boot: kinit1 done\n");
  // The arena stays low for the same reason it is contiguous: a device is not
  // obliged to be able to reach the top of memory. An RTL8139's buffer address
  // register is thirty-two bits wide, and it is not the only one.
  dma_init(P2V(lowtop - DMA_ARENA_BYTES), P2V(lowtop));
  if (boot_verbose) cprintf("verbose boot: dma_init done\n");
  kvmalloc();
  if (boot_verbose) cprintf("verbose boot: kvmalloc done\n");

  // Now that the direct map reaches it, the rest of RAM can be allocated from.
  kinit_high();

  // The other kind of framebuffer: one the direct map does not reach.
  //
  // Any machine with a discrete GPU is entitled to put its aperture above
  // 4 GB, and plenty do. P2V() cannot describe it, so the early block above
  // deliberately left it alone and it is mapped here instead -- the first
  // point in the boot where there is a page table to map it into. The console
  // starts a few lines later than it would on a low framebuffer, which is
  // still thousands of lines earlier than anything else.
  if (fb_early_ok && !fb_attached) {
    uint64 bytes = (uint64)fb_info_early.stride * fb_info_early.height;
    void  *va    = ioremap(fb_early, bytes);

    if (va) {
      fb_info_early.ptr = (volatile uchar *)va;
      gfx_attach_scanout(&fb_info_early);
      vt_init();
      gfx_clear(0x0C0E13);        /* SCH_BG */
      cons_serial_only = 0;
      kinfo("gfx: framebuffer at %p is above the direct map; mapped at %p\n",
            (void *)fb_early, va);
    } else {
      cprintf("gfx: cannot map the framebuffer at %p; no graphical console\n",
              (void *)fb_early);
    }
  }
  // ACPI first, the legacy MP table only if it finds nothing. Modern firmware
  // publishes a complete MADT and an MP table with one processor in it, so
  // asking the older interface first means never seeing more than one CPU.
  // See kernel/arch/x86_64/acpi.c.
  if (!acpi_init())
    mpinit();
  if (boot_verbose) cprintf("verbose boot: cpu enumeration done\n");
  lapicinit();
  if (boot_verbose) cprintf("verbose boot: lapicinit done\n");
  tvinit();
  // Once, on the boot processor. mpmain() runs on every CPU, and registering
  // from there published the command as many times as there are processors.
  watch_init();
  if (boot_verbose) cprintf("verbose boot: tvinit done\n");
  seginit();
  if (boot_verbose) cprintf("verbose boot: seginit done\n");
  ioapicinit();
  if (boot_verbose) cprintf("verbose boot: ioapicinit done\n");
  irq_init();       // must precede any driver init that registers a handler
  if (boot_verbose) cprintf("verbose boot: irq_init done\n");
  // Before the keyboard and mouse, which start producing events the moment
  // their handlers are registered.
  inputinit();
  if (boot_verbose) cprintf("verbose boot: inputinit done\n");
  // Before consoleinit, because the console writes to a terminal and there
  // has to be one. vt_init reads the framebuffer geometry, so it also has to
  // come after whatever attached a scanout.
  gfx_font_init();                 // no-op if the early path already did it
  vt_init();
  if (boot_verbose) cprintf("verbose boot: vt_init done\n");
  consoleinit();
  if (boot_verbose) cprintf("verbose boot: consoleinit done\n");
  // Now that there is a console to say it on. gfx_font_init() ran before
  // vt_init() and deliberately printed nothing; see the note there.
  {
    int cw, chh;
    gfx_cell_size(&cw, &chh);
    if (gfx_font_ready())
      kinfo("console: JetBrains Mono, %dx%d cells, %d by %d\n",
            cw, chh, gfx_cols(), gfx_rows());
    else
      kinfo("console: built-in bitmap font (the typeface would not open)\n");
  }
  uartinit();
  if (boot_verbose) cprintf("verbose boot: uartinit done\n");
  mouseinit();
  if (boot_verbose) cprintf("verbose boot: mouseinit done\n");
  // The devices that are simply known to be on a PC rather than enumerated by
  // any bus, and the framebuffer the loader left running. Registered before
  // pci_init() so that /proc/devices lists them first, which puts the machine's
  // fixed furniture above whatever happened to be plugged into it -- and
  // because the display has to work long before any bus is walked.
  //
  // main() names no display driver. Whichever one owns the boot framebuffer
  // publishes it there as this kernel's scanout, and the console and crash
  // screen draw on that; see include/gfx.h.
  //
  // No splash drawing here on purpose. The loader owns the splash, logo and
  // all, and bringing the framebuffer up deliberately does not clear it -- so
  // it stays on the display right through to console_takeover(). Painting it
  // again from here would land in the last few hundred milliseconds of boot
  // and change nothing.
  platform_register_devices(fb_addr);
  pciuser_init();
  pci_init();
  if (boot_verbose) cprintf("verbose boot: pci_init done\n");

  // A second display has just appeared, with a black scanout of its own, and
  // the console it ought to be showing was drawn before it existed -- by the
  // loader, into a framebuffer this device knows nothing about. So it is
  // caught up here, once, rather than waiting for the next line of output:
  // on a quiet boot that is most of a second of one screen showing the splash
  // and the other showing nothing.
  //
  // After pci_init() rather than inside device_add(), because a driver's
  // attached() hook -- which is what tells the instance apart from its
  // siblings -- does not run until the device has been published.
  display_mirror_console();
  pinit();
  if (boot_verbose) cprintf("verbose boot: pinit done\n");
  binit();
  if (boot_verbose) cprintf("verbose boot: binit done\n");
  fileinit();
  if (boot_verbose) cprintf("verbose boot: fileinit done\n");
  sockinit();
  if (boot_verbose) cprintf("verbose boot: sockinit done\n");
  // After the timer is running, so boot_epoch can subtract the uptime so far,
  // and before userspace, so the very first timestamp any program takes is
  // already real. Reading the RTC is slow, which is exactly why it happens once
  // here rather than on every gettimeofday().
  time_init();
  ideinit();
  if (boot_verbose) cprintf("verbose boot: ideinit done\n");
  ramdiskinit();
  if (boot_verbose) cprintf("verbose boot: ramdiskinit done\n");
  // Start the application processors.
  //
  // This was broken from the beginning of the project and is fixed as of
  // 2026-08-08. Five distinct bugs, each hidden behind the one before it, and
  // all of them invisible under QEMU because it publishes only one processor in
  // the legacy MP table -- see kernel/arch/x86_64/acpi.c. VMware published the
  // real count, which is why the triple fault only ever appeared there.
  //
  // 1. entry64high had no AP branch at all; the check was a comment saying it
  //    had never been written. Every AP fell into the boot processor's path,
  //    took the BSP's stack, and ran kmain() a second time.
  // 2. seginit() called kalloc() for its per-CPU page. kalloc locks, locking
  //    reads `cpu`, and `cpu` is what seginit() was about to set up.
  // 3. kernel_fs_base was one global for all CPUs, and trapasm.S read it on
  //    every entry from user mode, so both CPUs shared one per-CPU block.
  // 4. `cpu` and `proc` were __thread. That means per *thread*, so the compiler
  //    cached the thread pointer across calls -- but %fs here is per *CPU*, and
  //    a kernel thread changes CPU at every sched(). See include/proc.h.
  // 5. The new per-CPU slots collided with syscall_entry's scratch word at
  //    %fs:(8), which is the same memory as %gs:8.
  //
  int smp_start_aps = 1;
  if (smp_start_aps) {
    startothers();
    if (boot_verbose) cprintf("verbose boot: startothers done\n");
  } else if (ncpu > 1) {
    cprintf("smp: %d cpus found, running on 1 (AP startup is broken)\n", ncpu);
  }
  kinit2();
  if (boot_verbose) cprintf("verbose boot: kinit2 done\n");
  sinit();
  if (boot_verbose) cprintf("verbose boot: sinit done\n");
  // The TCP/IP stack, after sinit because it starts a kernel thread and
  // kthread_create needs the process table to be usable. It returns
  // immediately: everything it does happens in that thread, which cannot run
  // until mpmain() below enters the scheduler.
  netinit();
  if (boot_verbose) cprintf("verbose boot: netinit done\n");
  // For the same reason: a kernel thread. See irq_poll_thread() for the
  // lines it is there for, which on real hardware is most PCI INTx.
  irq_poll_start();
  // The debugger's own thread, for the same reason netinit's comes after
  // sinit: kthread_create needs a usable process table. From here on Ctrl-G
  // on the console answers, with no filesystem mounted and nothing in
  // userspace started -- which is the whole point of putting it this early.
  // After timeinit(), which is what boot_epoch comes from, and before any
  // program can ask for a random byte.
  random_init();
  netuser_init();
  dbg_thread_start();
  if (boot_verbose) cprintf("verbose boot: dbg thread started\n");

  // The greeting is not printed here.
  //
  // It was, and it landed before userspace had said anything -- so on a quiet
  // boot it was followed by the last few lines of `schinit` and whatever the
  // jobs had to say, which is the wrong way round for a line whose whole job is
  // to be the last thing you read. It belongs at the end of the thing it is
  // announcing, and the end of the thing it is announcing is in userspace.
  //
  // schinitd prints it, once its RunAtLoad jobs are away. See cmd/schinitd.
  mpmain();
}

// Other CPUs jump here from entryother.S.
void lapic_xapic_mode(void);
void lapic_park(uchar apicid);

void
mpenter(void)
{
  // Before anything reads the APIC -- seginit() and lapicinit() both do. An
  // AP keeps whatever mode the firmware left its APIC in across INIT, so the
  // boot processor having switched itself back says nothing about this one.
  lapic_xapic_mode();
  switchkvm();
  seginit();
  lapicinit();
  mpmain();
}

// Enable SSE/FXSAVE so user binaries can use XMM registers.
static void
enable_sse(void)
{
  uint64 cr0, cr4;
  asm volatile("mov %%cr0, %0" : "=r"(cr0));
  cr0 &= ~(1UL << 2);  // clear CR0.EM (no FP emulation)
  cr0 |=  (1UL << 1);  // set   CR0.MP (monitor coprocessor)
  asm volatile("mov %0, %%cr0" :: "r"(cr0));
  asm volatile("mov %%cr4, %0" : "=r"(cr4));
  cr4 |= (1UL << 9);   // CR4.OSFXSR    — OS handles FXSAVE/FXRSTOR
  cr4 |= (1UL << 10);  // CR4.OSXMMEXCPT — OS handles SSE exceptions
  asm volatile("mov %0, %%cr4" :: "r"(cr4));
}

// Common CPU setup code.
static void
mpmain(void)
{
  // **Set the processor up before saying anything.** This used to announce
  // itself first, and that reset the machine on any verbose boot with more
  // than one CPU -- silently, with no panic and no last line, because the
  // failure is a triple fault and a triple fault prints nothing.
  //
  // Two of the three lines below have to come first, and for different
  // reasons:
  //
  //   enable_sse() clears CR0.EM. Until it runs, *any* SSE instruction on
  //   this CPU faults -- and the kernel is not built with -mno-sse, so the
  //   compiler emits them freely for struct copies and inline memset. A
  //   quiet boot got away with it because kinfo() is log-only then and the
  //   work is a few bytes into a ring; a verbose boot renders the line
  //   through the TrueType rasteriser, which is where the XMM lands.
  //
  //   idtinit() loads this CPU's IDT. Until it runs there is nowhere for a
  //   fault to go, so the #UD above is not a panic -- it is a double fault
  //   with no handler, then a triple fault, then the firmware comes back up.
  //   That is why the only symptom was "F8 verbose reboots into a normal
  //   boot": the machine really did reboot, and the second one was quiet.
  //
  // The general rule this is an instance of: a processor announces that it
  // is up after it is able to run, not before.
  enable_sse();    // allow user XMM/SSE instructions
  idtinit();       // load idt register
  syscallinit();   // syscall set up

  kinfo("cpu%d: starting\n", cpunum());
  xchg(&mycpu()->started, 1); // tell startothers() we're up
  scheduler();     // start running processes
}

void entry32mp(void);

// One scheduler stack per possible CPU. See the note at its use below.
__attribute__((aligned(PGSIZE)))
static char apstacks[NCPU][KSTACKSIZE];

// Start the non-boot (AP) processors.
static void
startothers(void)
{
  extern uchar _binary_entryother_start[], _binary_entryother_size[];
  uchar *code;
  struct cpu *c;
  char *stack;

  // Write entry code to unused memory at 0x7000.
  // The linker has placed the image of entryother.S in
  // _binary_entryother_start.
  code = P2V(0x7000);
  memmove(code, _binary_entryother_start,
          (addr_t)_binary_entryother_size);

  for(c = cpus; c < cpus+ncpu; c++){
    if(c == cpus+cpunum())  // We've started already.
      continue;

    // Tell entryother.S what stack to use, where to enter, and what
    // pgdir to use. We cannot use kpgdir yet, because the AP processor
    // is running in low  memory, so we use entrypgdir for the APs too.
    // The AP's per-CPU page, allocated here rather than by the AP itself.
    // seginit() explains why: an AP cannot call kalloc() before %fs is set up,
    // because kalloc locks and locking reads the per-CPU pointer.
    if (c->local == 0) {
      c->local = kalloc();
      memset(c->local, 0, PGSIZE);
    }

    // The AP's scheduler stack. Static rather than kalloc(), for the same
    // reason and by the same argument as kstacks[] in proc.c -- and here there
    // was a second, sharper problem: this said `kalloc()` for one page and
    // then handed the AP `stack + KSTACKSIZE` as its stack top. That was
    // exactly right for as long as KSTACKSIZE happened to equal PGSIZE, and
    // silently points a whole page past the allocation the moment it does not.
    //
    // This is not a throwaway boot stack. The AP runs mpmain() and then
    // scheduler() on it for the rest of the machine's life, and takes
    // interrupts on it whenever no process is current.
    stack = apstacks[c - cpus];
    *(uint32*)(code-4) = 0x8000; // enough stack to get us to entry64mp
    *(uint32*)(code-8) = v2p(entry32mp);
    *(uint64*)(code-16) = (uint64) (stack + KSTACKSIZE);

    lapicstartap(c->apicid, V2P(code));

    // Wait for it to finish mpmain() -- for a second, not for ever.
    //
    // This had no bound, so one processor that did not answer stopped the
    // machine here for good, after the last line of hardware detection and
    // before anything else. A processor the firmware listed that is not
    // there, one that is disabled in setup, or one that simply fails to start
    // is a machine running on fewer CPUs, not a machine that cannot boot.
    // ncpu_online below counts only the ones that answered, and procfs
    // already reports the difference.
    uint64 deadline = rdtsc() + tsc_hz;
    while(c->started == 0 && rdtsc() < deadline)
      ;
    if(c->started == 0){
      cprintf("smp: cpu with apic id %d did not start; continuing without it\n",
              c->apicid);
      lapic_park(c->apicid);
      continue;
    }

    // Counted here, on the boot processor, after the AP has actually reported
    // in -- not by the AP itself. This is serialised by the loop, so it needs
    // no atomic, and it cannot overcount a processor that started and died.
    ncpu_online++;
  }
}