#include "types.h" #include "defs.h" #include "param.h" #include "memlayout.h" #include "mmu.h" #include "proc.h" #include "x86.h" #include "irq.h" #include "dma.h" static void startothers(void); static void mpmain(void) __attribute__((noreturn)); extern char end[]; // first address after kernel loaded from ELF file // Set by stage2main.c (boot/bios/stage2main.c, BOOT_VERBOSE_FLAG_ADDR) when the // user picks "Boot SchOS (verbose)" from the F8 boot menu; read once here, // before anything else, so subsystem init can log its own steps. int boot_verbose; // Must match BOOT_VERBOSE_FLAG_ADDR in boot/bios/stage2main.c exactly: just past // the VBE info block, not 0x7000 (startothers() below uses that address for // the AP boot trampoline). #define BOOT_VERBOSE_FLAG_ADDR (0x30000 + 0x100) // Bootstrap processor starts running C code here. // Allocate a real stack and switch to it, first // doing some setup required for memory allocator to work. int kmain(uint32 fb_addr) { boot_verbose = *(uchar*)P2V(BOOT_VERBOSE_FLAG_ADDR); uartearlyinit(); if (boot_verbose) cprintf("verbose boot: uartearlyinit done\n"); // The DMA arena is carved off the top of physical memory *before* kalloc is // given the rest, because a device needs physically contiguous memory and // kalloc's free list makes no such promise -- two pages from it are adjacent // only by chance. Taking it first is the only way to be sure the region // exists; asking for it later would mean asking kalloc for something it // fundamentally cannot provide. See include/dma.h. // Before anything touches PHYSTOP, which is now a variable read from the // firmware's memory map rather than a constant asserting 224 MB exists. e820_init(); kinit1(end, P2V(PHYSTOP - DMA_ARENA_BYTES)); if (boot_verbose) cprintf("verbose boot: kinit1 done\n"); dma_init(P2V(PHYSTOP - DMA_ARENA_BYTES), P2V(PHYSTOP)); if (boot_verbose) cprintf("verbose boot: dma_init done\n"); kvmalloc(); if (boot_verbose) cprintf("verbose boot: kvmalloc done\n"); // ACPI first, the legacy MP table only if it finds nothing. Modern firmware // publishes a complete MADT and an MP table with one processor in it, so // asking the older interface first means never seeing more than one CPU. // See kernel/arch/x86_64/acpi.c. if (!acpi_init()) mpinit(); if (boot_verbose) cprintf("verbose boot: cpu enumeration done\n"); lapicinit(); if (boot_verbose) cprintf("verbose boot: lapicinit done\n"); tvinit(); if (boot_verbose) cprintf("verbose boot: tvinit done\n"); seginit(); if (boot_verbose) cprintf("verbose boot: seginit done\n"); ioapicinit(); if (boot_verbose) cprintf("verbose boot: ioapicinit done\n"); irq_init(); // must precede any driver init that registers a handler if (boot_verbose) cprintf("verbose boot: irq_init done\n"); // Before the keyboard and mouse, which start producing events the moment // their handlers are registered. inputinit(); if (boot_verbose) cprintf("verbose boot: inputinit done\n"); consoleinit(); if (boot_verbose) cprintf("verbose boot: consoleinit done\n"); uartinit(); if (boot_verbose) cprintf("verbose boot: uartinit done\n"); mouseinit(); if (boot_verbose) cprintf("verbose boot: mouseinit done\n"); vga_init(fb_addr); if (boot_verbose) cprintf("verbose boot: vga_init done (fb=0x%x)\n", fb_addr); // No splash drawing here on purpose. The loader owns the splash, logo and // all, and vga_init() deliberately does not clear -- so it is still on the // display right through to console_takeover(). Painting it again from here // would land in the last few hundred milliseconds of boot and change nothing. // After vga_init, because the display driver cross-checks the framebuffer // address it finds on the bus against the one VBE reported -- and it can // only do that once vga.c has read the VBE block. // The devices that are simply known to be on a PC rather than enumerated by // any bus. Registered before pci_init() so that /proc/devices lists them // first, which puts the machine's fixed furniture above whatever happened to // be plugged into it. platform_register_devices(); pci_init(); if (boot_verbose) cprintf("verbose boot: pci_init done\n"); pinit(); if (boot_verbose) cprintf("verbose boot: pinit done\n"); binit(); if (boot_verbose) cprintf("verbose boot: binit done\n"); fileinit(); if (boot_verbose) cprintf("verbose boot: fileinit done\n"); sockinit(); if (boot_verbose) cprintf("verbose boot: sockinit done\n"); // After the timer is running, so boot_epoch can subtract the uptime so far, // and before userspace, so the very first timestamp any program takes is // already real. Reading the RTC is slow, which is exactly why it happens once // here rather than on every gettimeofday(). time_init(); ideinit(); if (boot_verbose) cprintf("verbose boot: ideinit done\n"); ramdiskinit(); if (boot_verbose) cprintf("verbose boot: ramdiskinit done\n"); // Start the application processors. // // This was broken from the beginning of the project and is fixed as of // 2026-08-08. Five distinct bugs, each hidden behind the one before it, and // all of them invisible under QEMU because it publishes only one processor in // the legacy MP table -- see kernel/arch/x86_64/acpi.c. VMware published the // real count, which is why the triple fault only ever appeared there. // // 1. entry64high had no AP branch at all; the check was a comment saying it // had never been written. Every AP fell into the boot processor's path, // took the BSP's stack, and ran kmain() a second time. // 2. seginit() called kalloc() for its per-CPU page. kalloc locks, locking // reads `cpu`, and `cpu` is what seginit() was about to set up. // 3. kernel_fs_base was one global for all CPUs, and trapasm.S read it on // every entry from user mode, so both CPUs shared one per-CPU block. // 4. `cpu` and `proc` were __thread. That means per *thread*, so the compiler // cached the thread pointer across calls -- but %fs here is per *CPU*, and // a kernel thread changes CPU at every sched(). See include/proc.h. // 5. The new per-CPU slots collided with syscall_entry's scratch word at // %fs:(8), which is the same memory as %gs:8. // int smp_start_aps = 1; if (smp_start_aps) { startothers(); if (boot_verbose) cprintf("verbose boot: startothers done\n"); } else if (ncpu > 1) { cprintf("smp: %d cpus found, running on 1 (AP startup is broken)\n", ncpu); } kinit2(); if (boot_verbose) cprintf("verbose boot: kinit2 done\n"); sinit(); if (boot_verbose) cprintf("verbose boot: sinit done\n"); mpmain(); } // Other CPUs jump here from entryother.S. void mpenter(void) { switchkvm(); seginit(); lapicinit(); mpmain(); } // Enable SSE/FXSAVE so user binaries can use XMM registers. static void enable_sse(void) { uint64 cr0, cr4; asm volatile("mov %%cr0, %0" : "=r"(cr0)); cr0 &= ~(1UL << 2); // clear CR0.EM (no FP emulation) cr0 |= (1UL << 1); // set CR0.MP (monitor coprocessor) asm volatile("mov %0, %%cr0" :: "r"(cr0)); asm volatile("mov %%cr4, %0" : "=r"(cr4)); cr4 |= (1UL << 9); // CR4.OSFXSR — OS handles FXSAVE/FXRSTOR cr4 |= (1UL << 10); // CR4.OSXMMEXCPT — OS handles SSE exceptions asm volatile("mov %0, %%cr4" :: "r"(cr4)); } // Common CPU setup code. static void mpmain(void) { cprintf("cpu%d: starting\n", cpunum()); enable_sse(); // allow user XMM/SSE instructions idtinit(); // load idt register syscallinit(); // syscall set up xchg(&mycpu()->started, 1); // tell startothers() we're up scheduler(); // start running processes } void entry32mp(void); // Start the non-boot (AP) processors. static void startothers(void) { extern uchar _binary_entryother_start[], _binary_entryother_size[]; uchar *code; struct cpu *c; char *stack; // Write entry code to unused memory at 0x7000. // The linker has placed the image of entryother.S in // _binary_entryother_start. code = P2V(0x7000); memmove(code, _binary_entryother_start, (addr_t)_binary_entryother_size); for(c = cpus; c < cpus+ncpu; c++){ if(c == cpus+cpunum()) // We've started already. continue; // Tell entryother.S what stack to use, where to enter, and what // pgdir to use. We cannot use kpgdir yet, because the AP processor // is running in low memory, so we use entrypgdir for the APs too. // The AP's per-CPU page, allocated here rather than by the AP itself. // seginit() explains why: an AP cannot call kalloc() before %fs is set up, // because kalloc locks and locking reads the per-CPU pointer. if (c->local == 0) { c->local = kalloc(); memset(c->local, 0, PGSIZE); } stack = kalloc(); *(uint32*)(code-4) = 0x8000; // enough stack to get us to entry64mp *(uint32*)(code-8) = v2p(entry32mp); *(uint64*)(code-16) = (uint64) (stack + KSTACKSIZE); lapicstartap(c->apicid, V2P(code)); // wait for cpu to finish mpmain() while(c->started == 0) ; // Counted here, on the boot processor, after the AP has actually reported // in -- not by the AP itself. This is serialised by the loop, so it needs // no atomic, and it cannot overcount a processor that started and died. ncpu_online++; } }