nexoniarz/SchOS
nexoniarz/SchOS
Watch
1
0
Fork
You've already forked SchOS
 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Initial commit: Schunix as it stands before the SchOS rename

Browse source 
An x86-64 operating system built from xv6, now largely rewritten: SMP,
a copy-on-write root filesystem (SchFS), PCI/USB/virtio drivers, a TTF
rasteriser, and a Linux-numbered syscall ABI.

No history before this point -- the tree was never under version control.
This commit exists so the rename that follows has something to undo to.
This commit is contained in:
Nexoniarz 2026-08-09 13:47:01 +02:00
commit d0f7acf9eb
275 changed files with 51417 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

264
kernel/arch/x86_64/acpi.c Normal file
View file

@ -0,0 +1,264 @@
// Find the processors, the way firmware from this century describes them.
//
// mpinit() walks the Intel MP table, which is from 1994. Modern firmware fills
// it in minimally or omits it: measured 2026-08-08, QEMU/SeaBIOS with -smp 4
// publishes a 208-byte MP table containing exactly *one* processor entry, then
// buses and interrupt assignments. Everything since has expected the operating
// system to read the ACPI MADT instead.
//
// The consequence was not that SMP was slow, it was that ncpu was 1 and SMP was
// silently never exercised. Machines whose firmware does publish a full MP
// table -- VMware, older hardware -- took the other path, started their APs,
// and triple faulted. So this file is a prerequisite for debugging that: until
// the CPUs are visible under QEMU there is no way to reproduce the crash except
// on the hardware it happens on.
//
// Written from the ACPI specification. No table layout here is copied from
// another kernel; they are all in chapter 5 of the published standard.
#include "types.h"
#include "defs.h"
#include "param.h"
#include "memlayout.h"
#include "mmu.h"
#include "proc.h"
#include "x86.h"
extern struct cpu cpus[NCPU];
extern int ncpu;
extern uchar ioapicid;
extern volatile uint *lapic;
// The Root System Description Pointer. Found by signature, not by being told
// where it is -- it sits either in the Extended BIOS Data Area or in the BIOS
// read-only region, on a 16-byte boundary.
struct acpi_rsdp {
char sig[8]; // "RSD PTR "
uint8 checksum; // over the first 20 bytes
char oemid[6];
uint8 revision; // 0 = ACPI 1.0 (RSDT), 2+ = 2.0 (XSDT)
uint32 rsdt_addr;
// ACPI 2.0 and later only, and only valid when revision >= 2.
uint32 length;
uint64 xsdt_addr;
uint8 ext_checksum; // over `length` bytes
uint8 reserved[3];
} __attribute__((packed));
// Every ACPI table starts with this.
struct acpi_sdt {
char sig[4];
uint32 length;
uint8 revision;
uint8 checksum;
char oemid[6];
char oem_table_id[8];
uint32 oem_revision;
uint32 creator_id;
uint32 creator_revision;
} __attribute__((packed));
// The Multiple APIC Description Table: signature "APIC".
struct acpi_madt {
struct acpi_sdt h;
uint32 lapic_addr;
uint32 flags; // bit 0: legacy 8259 PICs are present
} __attribute__((packed));
#define MADT_PCAT_COMPAT 1
// MADT entry types. Only these three are read; the rest are interrupt
// routing overrides that ioapic.c does not consult yet.
#define MADT_LAPIC 0
#define MADT_IOAPIC 1
#define MADT_LAPIC_OVERRIDE 5
struct madt_entry {
uint8 type;
uint8 length;
} __attribute__((packed));
struct madt_lapic {
struct madt_entry e;
uint8 acpi_id;
uint8 apic_id;
uint32 flags; // bit 0: enabled, bit 1: online-capable
} __attribute__((packed));
#define LAPIC_ENABLED 1
#define LAPIC_ONLINE_CAPABLE 2
struct madt_ioapic {
struct madt_entry e;
uint8 id;
uint8 reserved;
uint32 addr;
uint32 gsi_base;
} __attribute__((packed));
struct madt_lapic_override {
struct madt_entry e;
uint16 reserved;
uint64 addr;
} __attribute__((packed));
static int
checksum_ok(const void *p, uint32 len)
{
const uint8 *b = p;
uint8 sum = 0;
for (uint32 i = 0; i < len; i++)
sum += b[i];
return sum == 0;
}
// Scan a physical range for the RSDP signature. It is 16-byte aligned by
// specification, which makes this cheap enough to do twice.
static struct acpi_rsdp *
scan_rsdp(uint64 base, uint64 len)
{
for (uint64 off = 0; off + sizeof(struct acpi_rsdp) <= len; off += 16) {
struct acpi_rsdp *r = (struct acpi_rsdp *)P2V(base + off);
if (memcmp(r->sig, "RSD PTR ", 8) != 0)
continue;
// Signature alone is not enough: the string can appear in unrelated data,
// and following a bad pointer here means faulting before there is anything
// to report the fault with.
if (!checksum_ok(r, 20))
continue;
return r;
}
return 0;
}
static struct acpi_rsdp *
find_rsdp(void)
{
// The EBDA, whose segment address the BIOS leaves at 0x40E.
uint16 ebda_seg = *(uint16 *)P2V(0x40E);
if (ebda_seg) {
struct acpi_rsdp *r = scan_rsdp((uint64)ebda_seg << 4, 1024);
if (r)
return r;
}
return scan_rsdp(0xE0000, 0x20000);
}
// Walk the RSDT or XSDT looking for a table by signature.
//
// Two nearly identical loops rather than one clever one: the only difference is
// the pointer width -- 32-bit entries in the RSDT, 64-bit in the XSDT -- and
// reading a 64-bit pointer out of a 32-bit array is exactly the sort of bug
// that produces a plausible address pointing at nothing.
static struct acpi_sdt *
find_table(struct acpi_rsdp *rsdp, const char *sig)
{
struct acpi_sdt *root;
int wide;
if (rsdp->revision >= 2 && rsdp->xsdt_addr) {
root = (struct acpi_sdt *)P2V(rsdp->xsdt_addr);
wide = 1;
} else {
root = (struct acpi_sdt *)P2V((uint64)rsdp->rsdt_addr);
wide = 0;
}
if (!checksum_ok(root, root->length))
return 0;
uint32 n = (root->length - sizeof(struct acpi_sdt)) / (wide ? 8 : 4);
for (uint32 i = 0; i < n; i++) {
uint64 pa;
if (wide) {
// Byte-wise, because the array is packed and 8-byte entries in it are
// not guaranteed to be 8-byte aligned.
uint8 *p = (uint8 *)(root + 1) + i * 8;
pa = 0;
for (int b = 7; b >= 0; b--)
pa = (pa << 8) | p[b];
} else {
pa = ((uint32 *)(root + 1))[i];
}
struct acpi_sdt *t = (struct acpi_sdt *)P2V(pa);
if (memcmp(t->sig, sig, 4) == 0)
return t;
}
return 0;
}
// Returns 1 if the processors were enumerated, 0 to fall back to mpinit().
int
acpi_init(void)
{
struct acpi_rsdp *rsdp = find_rsdp();
if (!rsdp)
return 0;
struct acpi_madt *madt = (struct acpi_madt *)find_table(rsdp, "APIC");
if (!madt || !checksum_ok(madt, madt->h.length))
return 0;
lapic = (uint *)P2V((uint64)madt->lapic_addr);
uint8 *p = (uint8 *)madt + sizeof(struct acpi_madt);
uint8 *e = (uint8 *)madt + madt->h.length;
int found_ioapic = 0;
while (p + sizeof(struct madt_entry) <= e) {
struct madt_entry *ent = (struct madt_entry *)p;
// A zero length would spin here forever. Firmware tables are not always
// well formed, and an infinite loop this early is indistinguishable from a
// hang with no output at all.
if (ent->length < sizeof(struct madt_entry))
break;
switch (ent->type) {
case MADT_LAPIC: {
struct madt_lapic *l = (struct madt_lapic *)p;
// Skip processors that are neither enabled nor online-capable: those are
// sockets the firmware is describing but not offering.
if (!(l->flags & (LAPIC_ENABLED | LAPIC_ONLINE_CAPABLE)))
break;
if (ncpu < NCPU) {
cpus[ncpu].apicid = l->apic_id;
ncpu++;
}
break;
}
case MADT_IOAPIC: {
struct madt_ioapic *io = (struct madt_ioapic *)p;
// The first one only. Multi-I/O-APIC routing needs the interrupt source
// overrides too, which nothing here reads yet, so claiming to support it
// would be worse than not.
if (!found_ioapic) {
ioapicid = io->id;
found_ioapic = 1;
}
break;
}
case MADT_LAPIC_OVERRIDE: {
struct madt_lapic_override *o = (struct madt_lapic_override *)p;
lapic = (uint *)P2V(o->addr);
break;
}
}
p += ent->length;
}
if (ncpu == 0)
return 0;
// The 8259s are still there and still wired to the CPU. On a system with an
// IMCR, this switches interrupt delivery from the PICs to the APIC; on one
// without, the ports are unclaimed and the writes go nowhere. mpinit() does
// the same thing guarded by the MP table's imcrp flag.
if (madt->flags & MADT_PCAT_COMPAT) {
outb(0x22, 0x70);
outb(0x23, inb(0x23) | 1);
}
cprintf("acpi: %d cpu%s, lapic at %p, ioapic id %d\n",
ncpu, ncpu == 1 ? "" : "s", lapic, ioapicid);
return 1;
}

105
kernel/arch/x86_64/e820.c Normal file
View file

@ -0,0 +1,105 @@
// Find out how much memory this machine actually has.
//
// PHYSTOP used to be a constant: 0xE000000, 224 MB, inherited from xv6 and
// never revisited. That is not a limit on how much memory the kernel *uses* --
// it is an assertion that exactly that much exists. Boot a machine with less
// and kinit1() walks off the end of RAM into nothing; boot one with more and
// every byte above 224 MB is invisible. Measured before this file existed:
// Schunix booted at 224 MB and triple faulted at 192 MB, for no reason at all.
//
// The firmware has known the answer the whole time. boot/stage2.S asks it with
// INT 0x15 / E820 in real mode -- the same interface Linux's boot stub and GRUB
// use -- and leaves the result in low memory. stage2main.c already parses it to
// print detected RAM on the splash. The kernel simply never read it.
#include "types.h"
#include "defs.h"
#include "memlayout.h"
#include "mmu.h"
// Set before kinit1() and never again. Starts at the old constant so that a
// crash between entry and e820_init() behaves the way it always did rather
// than in some new way.
uint64 phystop = PHYSTOP_FALLBACK;
// Where stage2 left the map. Must match E820_COUNT_ADDR / E820_MAP_ADDR in
// boot/stage2main.c, which are themselves relative to VBE_INFO_ADDR.
#define E820_COUNT_ADDR (0x30000 + 0x200)
#define E820_MAP_ADDR (0x30000 + 0x208)
#define E820_USABLE 1
#define E820_MAXENTRIES 32
struct e820entry {
uint64 base;
uint64 length;
uint32 type;
uint32 acpi_ext;
} __attribute__((packed));
void
e820_init(void)
{
uint32 count = *(uint32 *)P2V(E820_COUNT_ADDR);
struct e820entry *e = (struct e820entry *)P2V(E820_MAP_ADDR);
// A count of zero means the firmware declined; anything absurd means we are
// reading something that is not a memory map. Either way the constant is the
// only honest answer left.
if (count == 0 || count > E820_MAXENTRIES) {
cprintf("e820: no usable map (count %d), assuming %d MB\n",
count, (int)(PHYSTOP_FALLBACK / (1024 * 1024)));
return;
}
// The end of the usable region that *contains the kernel*, not the highest
// usable address anywhere.
//
// This distinction matters and is easy to get wrong. A memory map is not one
// run of RAM: there are reserved holes in it -- ACPI tables, firmware
// scratch, the legacy hole below 1 MB -- and kalloc's freerange() walks
// straight from the end of the kernel image to phystop handing out every page
// between. Taking the global maximum would step over a reserved region and
// put firmware tables on the free list, which fails later, elsewhere, and
// looks like anything but a memory map problem.
uint64 top = 0;
for (uint32 i = 0; i < count; i++) {
if (e[i].type != E820_USABLE)
continue;
uint64 base = e[i].base, end = base + e[i].length;
if (base <= EXTMEM && EXTMEM < end) {
top = end;
break;
}
}
if (top == 0) {
cprintf("e820: no usable region covers 0x%x, assuming %d MB\n",
EXTMEM, (int)(PHYSTOP_FALLBACK / (1024 * 1024)));
return;
}
// kvmalloc() maps physical 0-4 GB with four 1 GB pages, and marks the top one
// uncached because that is where PCI BARs and the framebuffer live. So 3 GB
// is the ceiling for anything the kernel will treat as RAM: above it the
// direct map either does not exist or is device memory that would be
// catastrophically slow to allocate from and is not ours to hand out.
if (top > PHYSTOP_MAX)
top = PHYSTOP_MAX;
top &= ~(uint64)(PGSIZE - 1);
// Below this the kernel image, its page tables and the DMA arena do not fit,
// and continuing produces a failure much harder to read than this line.
if (top < PHYSTOP_MIN) {
cprintf("e820: only %d MB usable; Schunix needs at least %d MB\n",
(int)(top / (1024 * 1024)), (int)(PHYSTOP_MIN / (1024 * 1024)));
// Left at the fallback deliberately rather than lowered: a machine this
// small is going to fail, and it should fail having said why.
return;
}
phystop = top;
cprintf("e820: %d MB usable (%d entries)\n",
(int)(phystop / (1024 * 1024)), count);
}

174
kernel/arch/x86_64/entry.S Normal file
View file

@ -0,0 +1,174 @@
/* entry.S
*
* Copyright (c) 2013 Brian Swetland
* Modified by Nexoniarz
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
* LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
* OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
* WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
*/
#define mboot_magic 0x1badb002
#define mboot_flags 0x00010000
#include "mmu.h"
.code64
.global mboot_header
.global mboot_entry
mboot_header:
.long mboot_magic
.long mboot_flags
.long (-mboot_magic -mboot_flags)
.long mboot_load_addr
.long mboot_load_addr
.long mboot_load_end
.long mboot_bss_end
.long mboot_entry_addr
.code32
mboot_entry:
mov %eax, %ebp
# bootstrap page tables (0x1000 - 0x3000)
xor %eax, %eax
mov $0x1000, %edi
mov $0x2000, %ecx
rep stosb
# Bootstrap page tables (Identity mapping 4GB)
# PML4T[0] i PML4T[256] -> PDPT pod 0x2000
mov $(0x2000 | PTE_P | PTE_W), %edx
mov %edx, 0x1000
mov %edx, 0x1800
# PDPT mapping 4GB using 1GB huge pages
mov $(0x00000000 | PTE_P | PTE_PS | PTE_W), %edx
mov %edx, 0x2000
mov $(0x40000000 | PTE_P | PTE_PS | PTE_W), %edx
mov %edx, 0x2008
mov $(0x80000000 | PTE_P | PTE_PS | PTE_W), %edx
mov %edx, 0x2010
mov $(0xC0000000 | PTE_P | PTE_PS | PTE_W), %edx
mov %edx, 0x2018
mov %ebp, %ebx
.global entry32mp
entry32mp:
mov $0x1000, %eax
mov %eax, %cr3
lgdt (gdtr64 - mboot_header + mboot_load_addr)
mov %cr4, %eax
bts $5, %eax
mov %eax, %cr4
mov $MSR_EFER, %ecx
rdmsr
bts $0, %eax
bts $8, %eax
wrmsr
mov %cr0, %eax
orl $(CR0_PG | CR0_WP | CR0_MP), %eax
mov %eax, %cr0
ljmp $8, $(entry64low - mboot_header + mboot_load_addr)
.align 16
gdtr64:
.word gdt64_end - gdt64_begin - 1;
.quad gdt64_begin - mboot_header + mboot_load_addr
.align 16
gdt64_begin:
.long 0x00000000
.long 0x00000000
.long 0x00000000 # 64-bit code
.long 0x00209800
.long 0x00000000 # 64-bit data
.long 0x00009000
gdt64_end:
.align 16
.code64
entry64low:
movabs $entry64high, %rax
jmp *%rax
.global _start
_start:
entry64high:
xor %rax, %rax
mov %ax, %ss
mov %ax, %ds
mov %ax, %es
mov %ax, %fs
mov %ax, %gs
# Is this an application processor, or the boot processor?
#
# This check used to be a comment saying it had not been written yet, and
# every AP fell straight through into the BSP path below: it set RSP to the
# *boot processor's* stack, passed its own flag to kmain() as a framebuffer
# address, and re-ran the entire bootstrap -- e820_init, kinit1, kvmalloc --
# on a second core sharing one stack with the first. That is the SMP triple
# fault. QEMU hid it for years by never reporting more than one CPU (see
# kernel/arch/x86_64/acpi.c); VMware did not.
#
# EBX is the discriminator, and entryother.S sets it to 1 immediately before
# entering long mode for exactly this purpose. It is safe: on the boot
# processor EBX carries the VESA framebuffer address from the loader, which is
# either zero or a physical MMIO address, and can never be 1.
cmp $1, %ebx
je entry64mp
# Setup initial stack
movabs $0xFFFF800000010000, %rax
mov %rax, %rsp
# Pass VESA address to main (1st argument in RDI)
mov %rbx, %rdi
call kmain
.global __deadloop
__deadloop:
jmp .
entry64mp:
mov $0x7000, %rax
mov -16(%rax), %rsp
jmp mpenter
.global wrmsr
wrmsr:
mov %rdi, %rcx
mov %rsi, %rax
shr $32, %rsi
mov %rsi, %rdx
wrmsr
retq
.global ignore_sysret
ignore_sysret:
mov $-38, %rax
sysretq

78
kernel/arch/x86_64/entryother.S Normal file
View file

@ -0,0 +1,78 @@
#include "asm.h"
#include "memlayout.h"
#include "mmu.h"
# Each non-boot CPU ("AP") is started up in response to a STARTUP
# IPI from the boot CPU. Section B.4.2 of the Multi-Processor
# Specification says that the AP will start in real mode with CS:IP
# set to XY00:0000, where XY is an 8-bit value sent with the
# STARTUP. Thus this code must start at a 4096-byte boundary.
#
# Because this code sets DS to zero, it must sit
# at an address in the low 2^16 bytes.
#
# Startothers (in main.c) sends the STARTUPs one at a time.
# It copies this code (start) at 0x7000. It puts the address of
# a newly allocated per-core stack in start-4,the address of the
# place to jump to (mpenter) in start-8, and the physical address
# of entrypgdir in start-12.
#
# This code is identical to bootasm.S except:
# - it does not need to enable A20
# - it uses the address at start-4, start-8, and start-12
.code16
.global start
start:
cli
xorw %ax,%ax
movw %ax,%ds
movw %ax,%es
movw %ax,%ss
lgdt gdtdesc
movl %cr0, %eax
orl $CR0_PE, %eax
movl %eax, %cr0
ljmpl $(SEG_KCODE<<3), $(start32)
//PAGEBREAK!
.code32
start32:
movw $(SEG_KDATA<<3), %ax
movw %ax, %ds
movw %ax, %es
movw %ax, %ss
movw $0, %ax
movw %ax, %fs
movw %ax, %gs
# defer paging until we switch to 64bit mode
# set ebx=1 so shared boot code knows we're booting a secondary core
mov $1, %ebx
# Switch to the stack allocated by startothers()
movl (start-4), %esp
# Call mpenter()
call *(start-8)
movw $0x8a00, %ax
movw %ax, %dx
outw %ax, %dx
movw $0x8ae0, %ax
outw %ax, %dx
spin:
jmp spin
.p2align 2
gdt:
SEG_NULLASM
SEG_ASM(STA_X|STA_R, 0, 0xffffffff)
SEG_ASM(STA_W, 0, 0xffffffff)
gdtdesc:
.word (gdtdesc - gdt - 1)
.long gdt

102
kernel/arch/x86_64/ioapic.c Normal file
View file

@ -0,0 +1,102 @@
// The I/O APIC manages hardware interrupts for an SMP system.
// http://www.intel.com/design/chipsets/datashts/29056601.pdf
// See also picirq.c.
#include "types.h"
#include "defs.h"
#include "traps.h"
#include "memlayout.h"
#define IOAPIC 0xFEC00000 // Default physical address of IO APIC
#define REG_ID 0x00 // Register index: ID
#define REG_VER 0x01 // Register index: version
#define REG_TABLE 0x10 // Redirection table base
// The redirection table starts at REG_TABLE and uses
// two registers to configure each interrupt.
// The first (low) register in a pair contains configuration bits.
// The second (high) register contains a bitmask telling which
// CPUs can serve that interrupt.
#define INT_DISABLED 0x00010000 // Interrupt disabled
#define INT_LEVEL 0x00008000 // Level-triggered (vs edge-)
#define INT_ACTIVELOW 0x00002000 // Active low (vs high)
#define INT_LOGICAL 0x00000800 // Destination is CPU id (vs APIC ID)
volatile struct ioapic *ioapic;
// IO APIC MMIO structure: write reg, then read or write data.
struct ioapic {
uint reg;
uint pad[3];
uint data;
};
static uint
ioapicread(int reg)
{
ioapic->reg = reg;
return ioapic->data;
}
static void
ioapicwrite(int reg, uint data)
{
ioapic->reg = reg;
ioapic->data = data;
}
void
ioapicinit(void)
{
int i, id, maxintr;
ioapic = P2V((volatile struct ioapic*)IOAPIC);
maxintr = (ioapicread(REG_VER) >> 16) & 0xFF;
id = ioapicread(REG_ID) >> 24;
if(id != ioapicid)
cprintf("ioapicinit: id isn't equal to ioapicid; not a MP\n");
// Mark all interrupts edge-triggered, active high, disabled,
// and not routed to any CPUs.
for(i = 0; i <= maxintr; i++){
ioapicwrite(REG_TABLE+2*i, INT_DISABLED | (T_IRQ0 + i));
ioapicwrite(REG_TABLE+2*i+1, 0);
}
}
// Route an interrupt to a CPU and unmask it.
//
// `level_low` picks PCI's signalling instead of ISA's. This distinction did
// not matter while every interrupt in the kernel came from an ISA-era device
// at a fixed pin, all of which are edge-triggered and active high -- which is
// why the original ioapicenable() hardcoded exactly that. A PCI device wired
// through an INTx pin is the opposite on both counts, and configuring it as
// edge/high does not fail cleanly: the device raises the line, we take one
// interrupt, and because the level never goes away there is no second edge
// and the device is never heard from again.
void
ioapicroute(int irq, int cpunum, int level_low)
{
uint lo = T_IRQ0 + irq;
if(level_low)
lo |= INT_LEVEL | INT_ACTIVELOW;
// Program the destination before unmasking, so the entry is never briefly
// live pointing at CPU 0 when it was meant for another.
ioapicwrite(REG_TABLE+2*irq+1, cpunum << 24);
ioapicwrite(REG_TABLE+2*irq, lo);
}
void
ioapicmask(int irq)
{
ioapicwrite(REG_TABLE+2*irq, INT_DISABLED | (T_IRQ0 + irq));
}
void
ioapicenable(int irq, int cpunum)
{
ioapicroute(irq, cpunum, 0);
}

245
kernel/arch/x86_64/lapic.c Normal file
View file

@ -0,0 +1,245 @@
// The local APIC manages internal (non-I/O) interrupts.
// See Chapter 8 & Appendix C of Intel processor manual volume 3.
#include "param.h"
#include "types.h"
#include "defs.h"
#include "date.h"
#include "memlayout.h"
#include "traps.h"
#include "mmu.h"
#include "x86.h"
#include "proc.h" // ncpu
// Local APIC registers, divided by 4 for use as uint[] indices.
#define ID (0x0020/4) // ID
#define VER (0x0030/4) // Version
#define TPR (0x0080/4) // Task Priority
#define EOI (0x00B0/4) // EOI
#define SVR (0x00F0/4) // Spurious Interrupt Vector
#define ENABLE 0x00000100 // Unit Enable
#define ESR (0x0280/4) // Error Status
#define ICRLO (0x0300/4) // Interrupt Command
#define SINIT 0x00000500 // INIT/RESET
#define STARTUP 0x00000600 // Startup IPI
#define DELIVS 0x00001000 // Delivery status
#define ASSERT 0x00004000 // Assert interrupt (vs deassert)
#define LEVEL 0x00008000 // Level triggered
#define BCAST 0x00080000 // Send to all APICs, including self.
#define BUSY 0x00001000
#define ICRHI (0x0310/4) // Interrupt Command [63:32]
#define TIMER (0x0320/4) // Local Vector Table 0 (TIMER)
#define X1 0x0000000B // divide counts by 1
#define PERIODIC 0x00020000 // Periodic
#define PCINT (0x0340/4) // Performance Counter LVT
#define LINT0 (0x0350/4) // Local Vector Table 1 (LINT0)
#define LINT1 (0x0360/4) // Local Vector Table 2 (LINT1)
#define ERROR (0x0370/4) // Local Vector Table 3 (ERROR)
#define MASKED 0x00010000 // Interrupt masked
#define TICR (0x0380/4) // Timer Initial Count
#define TCCR (0x0390/4) // Timer Current Count
#define TDCR (0x03E0/4) // Timer Divide Configuration
volatile uint *lapic; // Initialized in mp.c
static void
lapicw(int index, int value)
{
lapic[index] = value;
lapic[ID]; // wait for write to finish, by reading
}
void
lapicinit(void)
{
if(!lapic)
return;
// Enable local APIC; set spurious interrupt vector.
lapicw(SVR, ENABLE | (T_IRQ0 + IRQ_SPURIOUS));
// The timer repeatedly counts down at bus frequency
// from lapic[TICR] and then issues an interrupt.
// If xv6 cared more about precise timekeeping,
// TICR would be calibrated using an external time source.
lapicw(TDCR, X1);
lapicw(TIMER, PERIODIC | (T_IRQ0 + IRQ_TIMER));
lapicw(TICR, 10000000);
// Disable logical interrupt lines.
lapicw(LINT0, MASKED);
lapicw(LINT1, MASKED);
// Disable performance counter overflow interrupts
// on machines that provide that interrupt entry.
if(((lapic[VER]>>16) & 0xFF) >= 4)
lapicw(PCINT, MASKED);
// Map error interrupt to IRQ_ERROR.
lapicw(ERROR, T_IRQ0 + IRQ_ERROR);
// Clear error status register (requires back-to-back writes).
lapicw(ESR, 0);
lapicw(ESR, 0);
// Ack any outstanding interrupts.
lapicw(EOI, 0);
// Send an Init Level De-Assert to synchronise arbitration ID's.
lapicw(ICRHI, 0);
lapicw(ICRLO, BCAST | SINIT | LEVEL);
while(lapic[ICRLO] & DELIVS)
;
// Enable interrupts on the APIC (but not on the processor).
lapicw(TPR, 0);
}
int
cpunum(void)
{
int apicid, i;
// Cannot call cpu when interrupts are enabled:
// result not guaranteed to last long enough to be used!
// Would prefer to panic but even printing is chancy here:
// almost everything, including cprintf and panic, calls cpu,
// often indirectly through acquire and release.
if(readeflags()&FL_IF){
static int n;
if(n++ == 0)
cprintf("cpu called from %x with interrupts enabled\n",
__builtin_return_address(0));
}
if (!lapic)
return 0;
apicid = lapic[ID] >> 24;
for (i = 0; i < ncpu; ++i) {
if (cpus[i].apicid == apicid)
return i;
}
panic("unknown apicid\n");
}
// Acknowledge interrupt.
void
lapiceoi(void)
{
if(lapic)
lapicw(EOI, 0);
}
// Spin for a given number of microseconds.
// On real hardware would want to tune this dynamically.
void
microdelay(int us)
{
}
#define CMOS_PORT 0x70
#define CMOS_RETURN 0x71
// Start additional processor running entry code at addr.
// See Appendix B of MultiProcessor Specification.
void
lapicstartap(uchar apicid, uint addr)
{
int i;
ushort *wrv;
// "The BSP must initialize CMOS shutdown code to 0AH
// and the warm reset vector (DWORD based at 40:67) to point at
// the AP startup code prior to the [universal startup algorithm]."
outb(CMOS_PORT, 0xF); // offset 0xF is shutdown code
outb(CMOS_PORT+1, 0x0A);
wrv = (ushort*)P2V((0x40<<4 | 0x67)); // Warm reset vector
wrv[0] = 0;
wrv[1] = addr >> 4;
// "Universal startup algorithm."
// Send INIT (level-triggered) interrupt to reset other CPU.
lapicw(ICRHI, apicid<<24);
lapicw(ICRLO, SINIT | LEVEL | ASSERT);
microdelay(200);
lapicw(ICRLO, SINIT | LEVEL);
microdelay(100); // should be 10ms, but too slow in Bochs!
// Send startup IPI (twice!) to enter code.
// Regular hardware is supposed to only accept a STARTUP
// when it is in the halted state due to an INIT. So the second
// should be ignored, but it is part of the official Intel algorithm.
// Bochs complains about the second one. Too bad for Bochs.
for(i = 0; i < 2; i++){
lapicw(ICRHI, apicid<<24);
lapicw(ICRLO, STARTUP | (addr>>12));
microdelay(200);
}
}
#define CMOS_STATA 0x0a
#define CMOS_STATB 0x0b
#define CMOS_UIP (1 << 7) // RTC update in progress
#define SECS 0x00
#define MINS 0x02
#define HOURS 0x04
#define DAY 0x07
#define MONTH 0x08
#define YEAR 0x09
static uint cmos_read(uint reg)
{
outb(CMOS_PORT, reg);
microdelay(200);
return inb(CMOS_RETURN);
}
static void fill_rtcdate(struct rtcdate *r)
{
r->second = cmos_read(SECS);
r->minute = cmos_read(MINS);
r->hour = cmos_read(HOURS);
r->day = cmos_read(DAY);
r->month = cmos_read(MONTH);
r->year = cmos_read(YEAR);
}
//PAGEBREAK!
// qemu seems to use 24-hour GWT and the values are BCD encoded
void cmostime(struct rtcdate *r)
{
struct rtcdate t1, t2;
int sb, bcd;
sb = cmos_read(CMOS_STATB);
bcd = (sb & (1 << 2)) == 0;
// make sure CMOS doesn't modify time while we read it
for(;;) {
fill_rtcdate(&t1);
if(cmos_read(CMOS_STATA) & CMOS_UIP)
continue;
fill_rtcdate(&t2);
if(memcmp(&t1, &t2, sizeof(t1)) == 0)
break;
}
// convert
if(bcd) {
#define CONV(x) (t1.x = ((t1.x >> 4) * 10) + (t1.x & 0xf))
CONV(second);
CONV(minute);
CONV(hour );
CONV(day );
CONV(month );
CONV(year );
#undef CONV
}
*r = t1;
r->year += 2000;
}

159
kernel/arch/x86_64/mp.c Normal file
View file

@ -0,0 +1,159 @@
// Multiprocessor support
// Search memory for MP description structures.
// http://developer.intel.com/design/pentium/datashts/24201606.pdf
#include "types.h"
#include "defs.h"
#include "param.h"
#include "memlayout.h"
#include "mp.h"
#include "x86.h"
#include "mmu.h"
#include "proc.h"
struct cpu cpus[NCPU];
// Processors the firmware says exist.
int ncpu;
// Processors actually running, which is not the same number and must not be
// confused with it. Enumeration is cheap and startup is where the difficulty
// is: right now the APs are found and deliberately not started, so this stays
// at 1 while ncpu is 2, 4 or 16.
//
// /proc/cpuinfo reports *this* one, matching Linux, where cpuinfo lists online
// CPUs. Reporting ncpu there told a VM with four processors that it had four
// working ones, which was a comfortable thing to read and false.
int ncpu_online = 1;
uchar ioapicid;
static uchar
sum(uchar *addr, int len)
{
int i, sum;
sum = 0;
for(i=0; i<len; i++)
sum += addr[i];
return sum;
}
// Look for an MP structure in the len bytes at addr.
static struct mp*
mpsearch1(addr_t a, int len)
{
uchar *e, *p, *addr;
addr = P2V(a);
e = addr+len;
for(p = addr; p < e; p += sizeof(struct mp))
if(memcmp(p, "_MP_", 4) == 0 && sum(p, sizeof(struct mp)) == 0)
return (struct mp*)p;
return 0;
}
// Search for the MP Floating Pointer Structure, which according to the
// spec is in one of the following three locations:
// 1) in the first KB of the EBDA;
// 2) in the last KB of system base memory;
// 3) in the BIOS ROM between 0xE0000 and 0xFFFFF.
static struct mp*
mpsearch(void)
{
uchar *bda;
uint p;
struct mp *mp;
bda = (uchar *) P2V(0x400);
if((p = ((bda[0x0F]<<8)| bda[0x0E]) << 4)){
if((mp = mpsearch1(p, 1024)))
return mp;
} else {
p = ((bda[0x14]<<8)|bda[0x13])*1024;
if((mp = mpsearch1(p-1024, 1024)))
return mp;
}
return mpsearch1(0xF0000, 0x10000);
}
// Search for an MP configuration table. For now,
// don't accept the default configurations (physaddr == 0).
// Check for correct signature, calculate the checksum and,
// if correct, check the version.
// To do: check extended table checksum.
static struct mpconf*
mpconfig(struct mp **pmp)
{
struct mpconf *conf;
struct mp *mp;
if((mp = mpsearch()) == 0 || mp->physaddr == 0)
return 0;
conf = (struct mpconf*) P2V((addr_t) mp->physaddr);
if(memcmp(conf, "PCMP", 4) != 0)
return 0;
if(conf->version != 1 && conf->version != 4)
return 0;
if(sum((uchar*)conf, conf->length) != 0)
return 0;
*pmp = mp;
return conf;
}
void
mpinit(void)
{
uchar *p, *e;
struct mp *mp;
struct mpconf *conf;
struct mpproc *mproc;
struct mpioapic *ioapic;
if((conf = mpconfig(&mp)) == 0) {
cprintf("No other CPUs found.\n");
return;
}
lapic = (uint*)P2V(conf->lapicaddr_p);
for(p=(uchar*)(conf+1), e=(uchar*)conf+conf->length; p<e; ){
switch(*p){
case MPPROC:
mproc = (struct mpproc*)p;
if(ncpu < NCPU) {
cpus[ncpu].apicid = mproc->apicid; // apicid may differ from ncpu
ncpu++;
}
p += sizeof(struct mpproc);
continue;
case MPIOAPIC:
ioapic = (struct mpioapic*)p;
ioapicid = ioapic->apicno;
p += sizeof(struct mpioapic);
continue;
case MPBUS:
case MPIOINTR:
case MPLINTR:
p += 8;
continue;
default:
panic("Major problem parsing mp config.");
break;
}
}
// "Seems we are" is doing a lot of work here, and the number is usually
// wrong. This walks the *legacy Intel MP table* (1994), which modern firmware
// fills in minimally or not at all -- QEMU/SeaBIOS emits exactly one
// processor entry regardless of -smp, because it expects the OS to enumerate
// CPUs from the ACPI MADT instead. Verified 2026-08-08: with -smp 4 the table
// is 208 bytes and contains one type-0 entry, then buses, an I/O APIC and
// interrupt assignments. So ncpu is 1 on QEMU no matter what, and SMP is
// silently never exercised there.
//
// See docs/roadmap.md "Known broken" -- this is half of the SMP bug, and the
// reason the other half (AP startup triple-faulting) cannot be reproduced
// under QEMU until the MADT is parsed.
cprintf("Seems we are SMP, ncpu = %d\n",ncpu);
if(mp->imcrp){
// Bochs doesn't support IMCR, so this doesn't run on Bochs.
// But it would on real hardware.
outb(0x22, 0x70); // Select IMCR
outb(0x23, inb(0x23) | 1); // Mask external interrupts.
}
}

30
kernel/arch/x86_64/swtch.S Normal file
View file

@ -0,0 +1,30 @@
# Context switch
#
# void swtch(struct context **old, struct context *new);
#
# Save current register context in old
# and then load register context from new.
.global swtch
swtch:
# Save old callee-save registers
pushq %rbp
pushq %rbx
pushq %r12
pushq %r13
pushq %r14
pushq %r15
# Switch stacks
movq %rsp, (%rdi)
movq %rsi, %rsp
# Load new callee-save registers
popq %r15
popq %r14
popq %r13
popq %r12
popq %rbx
popq %rbp
retq #??

159
kernel/arch/x86_64/trap.c Normal file
View file

@ -0,0 +1,159 @@
#include "types.h"
#include "defs.h"
#include "irq.h"
#include "param.h"
#include "memlayout.h"
#include "mmu.h"
#include "proc.h"
#include "x86.h"
#include "traps.h"
#include "spinlock.h"
#include "posix.h"
uint *idt;
extern addr_t vectors[];
struct spinlock tickslock;
uint ticks;
static void
mkgate(uint *idt, uint n, addr_t kva, uint pl)
{
uint64 addr = (uint64) kva;
n *= 4;
idt[n+0] = (addr & 0xFFFF) | (KERNEL_CS << 16);
idt[n+1] = (addr & 0xFFFF0000) | 0x8E00 | ((pl & 3) << 13);
idt[n+2] = addr >> 32;
idt[n+3] = 0;
}
void idtinit(void)
{
struct {
ushort limit;
uint64 base;
} __attribute__((packed)) idtdesc;
idtdesc.limit = PGSIZE - 1;
idtdesc.base = (uint64)idt;
asm volatile("lidt %0" : : "m" (idtdesc));
}
void tvinit(void)
{
int n;
idt = (uint*) kalloc();
memset(idt, 0, PGSIZE);
for (n = 0; n < 256; n++)
mkgate(idt, n, vectors[n], 0);
mkgate(idt, T_SYSCALL, vectors[T_SYSCALL], 3);
}
uint
uptime(void)
{
uint xticks;
acquire(&tickslock);
xticks = ticks;
release(&tickslock);
return xticks;
}
// Which signal a CPU fault corresponds to, so a process killed by a fault is
// reported to wait() as died-by-signal with a sensible cause rather than a
// blanket SIGKILL.
static int
trapsig(uint64 trapno)
{
switch(trapno){
case T_DIVIDE: return SIGFPE;
case T_ILLOP: return SIGILL;
case T_GPFLT: return SIGSEGV;
case T_PGFLT: return SIGSEGV;
case T_BRKPT: return SIGTRAP;
case T_DEBUG: return SIGTRAP;
default: return SIGSEGV;
}
}
void
trap(struct trapframe *tf)
{
kstack_check_canary("trap entry");
switch(tf->trapno){
case T_IRQ0 + IRQ_TIMER:
if(cpunum() == 0){
acquire(&tickslock);
ticks++;
wakeup(&ticks);
release(&tickslock);
}
lapiceoi();
break;
case T_IRQ0 + IRQ_SPURIOUS:
cprintf("cpu%d: spurious interrupt at %p:%p\n",
cpunum(), tf->cs, tf->rip);
lapiceoi();
break;
default:
// Every other IOAPIC pin goes through the registration table. The device
// interrupts used to be listed here one case at a time, which cannot work
// for anything found on the PCI bus: a driver does not know its line
// until it reads the device's config space at probe time, so there is no
// constant to write a case label for. See include/irq.h.
if(tf->trapno >= T_IRQ0 && tf->trapno < T_IRQ0 + NIRQ){
irq_dispatch(tf->trapno - T_IRQ0);
lapiceoi();
break;
}
if(myproc() == 0 || (tf->cs&3) == 0){
cprintf("unexpected trap %d from cpu %d rip %p (cr2=0x%p) err=0x%p\n",
tf->trapno, cpunum(), tf->rip, rcr2(), tf->err);
if (myproc())
cprintf("proc id: %d\n", myproc()->pid);
if (tf->trapno == T_GPFLT) {
uint64 *iretframe = (uint64*)tf->rsp;
cprintf("iretq frame @ %p: rip=%p cs=%p rflags=%p rsp=%p ss=%p\n",
iretframe, iretframe[0], iretframe[1], iretframe[2],
iretframe[3], iretframe[4]);
if (myproc())
cprintf("this tf=%p myproc()->tf=%p myproc()->kstack=%p (tf-kstack=%p)\n",
tf, myproc()->tf, myproc()->kstack, (uint64)tf - (uint64)myproc()->kstack);
}
panic("trap");
}
cprintf("pid %d %s: trap %d err %d on cpu %d "
"rip 0x%p addr 0x%p--kill proc\n",
myproc()->pid, myproc()->name, tf->trapno, tf->err, cpunum(), tf->rip,
rcr2());
// General-purpose register dump. This used to be followed by ~25 lines
// that walked musl's mallocng group/meta structures out of rax/rdi and
// dumped the user stack -- scaffolding for the boot-crash investigation
// recorded in docs/architecture/kernel-history.md section 1. That
// investigation is closed and the underlying kernel bugs are fixed, so
// the allocator-specific part is gone; the registers stay, since they are
// useful for any fault.
if (tf->trapno == T_GPFLT) {
cprintf(" rax=%p rbx=%p rcx=%p rdx=%p rsi=%p rdi=%p rbp=%p rsp=%p\n",
tf->rax, tf->rbx, tf->rcx, tf->rdx, tf->rsi, tf->rdi, tf->rbp, tf->rsp);
}
myproc()->killed = 1;
if(myproc()->killsig == 0)
myproc()->killsig = trapsig(tf->trapno);
}
if(myproc() && myproc()->killed && (tf->cs&3) == DPL_USER)
exit(W_EXITCODE(0, myproc()->killsig ? myproc()->killsig : SIGKILL));
if(myproc() && myproc()->state == RUNNING && tf->trapno == T_IRQ0+IRQ_TIMER)
yield();
if(myproc() && myproc()->killed && (tf->cs&3) == DPL_USER)
exit(W_EXITCODE(0, myproc()->killsig ? myproc()->killsig : SIGKILL));
kstack_check_canary("trap exit");
}

257
kernel/arch/x86_64/trapasm.S Normal file
View file

@ -0,0 +1,257 @@
# trapasm.S — kernel/user entry/exit with FS_BASE swapping for TLS
#
# The kernel uses __thread struct cpu *cpu and __thread struct proc *proc,
# accessed via %fs (IA32_FS_BASE MSR, 0xC0000100). seginit() sets this to
# kernel_fs_base. User-space TLS (musl) uses the same MSR via arch_prctl
# ARCH_SET_FS (saved in proc->fs_base).
#
# Rule: on ENTRY from user (syscall or interrupt), restore kernel FS_BASE.
# on EXIT to user (sysretq or iretq), write proc->fs_base to MSR.
#
# proc->fs_base is at byte offset 0x18 (24) inside struct proc.
# proc pointer is at %gs:24 (PERCPU_PROC in include/proc.h); cpu is at %gs:16.
# -----------------------------------------------------------------------
# Helper macro: write kernel_fs_base to IA32_FS_BASE.
# Clobbers %rax, %rcx, %rdx. Caller must save them if needed.
# -----------------------------------------------------------------------
# Read from %gs, not from a global.
#
# This used to be `movq kernel_fs_base(%rip), %rax`, and kernel_fs_base was a
# single global that every CPU wrote during seginit(). With one processor that
# is merely untidy. With two it is fatal: whichever CPU ran seginit() last wins,
# and from the first trap onward *every* CPU restores that one CPU's per-CPU
# block -- so `cpu` and `proc` are shared, pushcli/popcli nesting counts collide,
# and the kernel dies in panic("sched locks") shortly after the APs start.
#
# GS base is set per CPU by seginit() to the same value as FS base, and %gs:0
# holds the block's self-pointer. So this reads *this* CPU's kernel FS base with
# no global involved.
#
# No swapgs, deliberately. The usual reason for it is that userspace owns GS;
# here it does not -- sys_arch_prctl() implements only ARCH_SET_FS/ARCH_GET_FS
# and rejects everything else, musl uses FS for TLS on x86-64, and the only
# other writes to %gs are selector loads in entry.S and entryother.S that happen
# before seginit(). GS is the kernel's alone, so it can simply stay pointing at
# the per-CPU block. If ARCH_SET_GS is ever implemented, this must become a
# proper swapgs pair.
.macro RESTORE_KERNEL_FS
movq %gs:0, %rax
movq %rax, %rdx
shrq $32, %rdx
movl $0xC0000100, %ecx
wrmsr
.endm
# -----------------------------------------------------------------------
# Helper macro: write proc->fs_base to IA32_FS_BASE.
# Requires %fs to still hold the kernel base.
# Clobbers %rax, %rcx, %rdx. Caller must save them if needed.
# -----------------------------------------------------------------------
.macro WRITE_USER_FS
# proc, from the per-CPU block at %gs:24 (PERCPU_PROC in include/proc.h).
# This used to be %fs:(-8), the __thread slot, which no longer exists --
# see the comment in include/proc.h for why those stopped being __thread.
movq %gs:24, %rax # proc pointer
movq 0x18(%rax), %rax # proc->fs_base (offset 24 = 0x18)
movq %rax, %rdx
shrq $32, %rdx
movl $0xC0000100, %ecx
wrmsr
.endm
# -----------------------------------------------------------------------
# alltraps — IDT interrupt/fault entry
# Stack on entry (from user mode, CPU has done privilege switch):
# [rsp+0] trapno (pushed by vectors.S)
# [rsp+8] errorcode
# [rsp+16] rip
# [rsp+24] cs ← bit 1:0 == 3 means user mode
# [rsp+32] rflags
# [rsp+40] user_rsp
# [rsp+48] user_ss
# -----------------------------------------------------------------------
.global alltraps
alltraps:
# Always restore kernel FS_BASE before touching %fs:, even when the
# saved cs says we were already in kernel mode. A fault/interrupt can
# land in that narrow kernel-mode window after syscall_trapret/trapret
# have already done WRITE_USER_FS (cs is still 0x8 there, but FS_BASE
# is the *user* value) — the old cs-based skip trusted that combination
# never happens and read %fs:(-8) as garbage (proc/cpu == NULL) in that
# case. RESTORE_KERNEL_FS is just a wrmsr of an already-known value when
# we truly were in the kernel, so doing it unconditionally is cheap and
# always correct.
pushq %rax
pushq %rcx
pushq %rdx
RESTORE_KERNEL_FS
popq %rdx
popq %rcx
popq %rax
# Build trap frame.
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %r11
pushq %r10
pushq %r9
pushq %r8
pushq %rdi
pushq %rsi
pushq %rbp
pushq %rdx
pushq %rcx
pushq %rbx
pushq %rax
movq %rsp, %rdi # frame in arg1
callq trap
# Return falls through to trapret...
.global trapret
trapret:
popq %rax
popq %rbx
popq %rcx
popq %rdx
popq %rbp
popq %rsi
popq %rdi
popq %r8
popq %r9
popq %r10
popq %r11
popq %r12
popq %r13
popq %r14
popq %r15
addq $16, %rsp # discard trapno and errorcode
# If returning to user mode, write proc->fs_base to FS_BASE MSR.
# Stack now: [rsp+0]=rip, [rsp+8]=cs, [rsp+16]=rflags, [rsp+24]=rsp, [rsp+32]=ss
testb $3, 8(%rsp)
jz .Ltrapret_kret
# cli before WRITE_USER_FS: close the same race as in syscall_trapret.
# iretq will atomically restore RFLAGS (IF=1) when switching to user mode.
pushq %rax
pushq %rcx
pushq %rdx
cli
WRITE_USER_FS
popq %rdx
popq %rcx
popq %rax
.Ltrapret_kret:
iretq
# -----------------------------------------------------------------------
# syscall_entry — SYSCALL instruction entry point
# On entry: rcx=user rip, r11=user rflags, rsp=user stack, IF cleared.
# FS_BASE may be the user's TLS value (set at last sysretq).
# -----------------------------------------------------------------------
.global syscall_entry
syscall_entry:
# Swap FS_BASE to kernel value BEFORE any %fs: access.
# Save rax/rcx/rdx to single-CPU scratch globals (IF is off, no race).
movq %rax, _sc_rax(%rip)
movq %rcx, _sc_rcx(%rip) # user rip (SYSCALL stores it here)
movq %rdx, _sc_rdx(%rip)
RESTORE_KERNEL_FS # clobbers rax, rcx, rdx — that's OK
movq _sc_rax(%rip), %rax
movq _sc_rcx(%rip), %rcx
movq _sc_rdx(%rip), %rdx
# Switch to kernel stack: store user rsp at top of proc->kstack.
#
# The scratch slot is %fs:(8), NOT %fs:(0). %fs:(0) is the TCB
# self-pointer -- the x86-64 TLS ABI reserves the word at the thread
# pointer to hold the thread pointer itself, and a compiler is entitled to
# read it to materialise the TP before applying a variable's offset.
#
# It used to be %fs:(0), and that survived for years only because GCC
# compiles `proc`/`cpu` into direct `%fs:-8` segment references and never
# touches the word. clang at -O0 does `movq %fs:0, %rax; addq $-8, %rax;
# (%rax)` instead -- so it read the syscall number this line had just
# stored (7, SYS_exec, from initcode), computed 7-8 = -1, and page-faulted
# on the very first syscall of the very first process.
#
# Positive offsets are safe: the TLS block occupies negative offsets
# (-16 = cpu, -8 = proc) and seginit puts the thread pointer 2048 bytes
# into a 4096-byte page, so %fs:(8) onward is 2040 bytes we own outright.
movq %rax, %fs:(8) # save rax in per-CPU scratch slot
movq %gs:24, %rax # proc pointer (PERCPU_PROC)
movq 0x10(%rax), %rax # proc->kstack (offset 16)
addq $(4096-16), %rax # point to tf->rsp slot
movq %rsp, (%rax) # save user rsp
movq %rax, %rsp # switch to kernel stack
movq %fs:(8), %rax # restore rax
pushq %r11 # rflags
pushq $0 # cs (ignored)
pushq %rcx # rip (next user instruction)
pushq $0 # err
pushq $0 # trapno (ignored)
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %r11
pushq %r10
pushq %r9
pushq %r8
pushq %rdi
pushq %rsi
pushq %rbp
pushq %rdx
pushq %rcx
pushq %rbx
pushq %rax
movq %rsp, %rdi # frame in arg1
callq syscall
# Return falls through to syscall_trapret...
# -----------------------------------------------------------------------
# syscall_trapret — return path for SYSCALL-based syscalls
# -----------------------------------------------------------------------
.global syscall_trapret
syscall_trapret:
# Disable interrupts BEFORE writing user FS_BASE to the MSR.
# Without cli here, a timer IRQ between WRITE_USER_FS and the cli below
# would arrive in ring-0 with user FS_BASE set, causing %fs:(-8) to return
# garbage (proc=NULL) → panic in the interrupt handler.
# rax/rcx/rdx are clobbered by WRITE_USER_FS then restored by the pops.
cli
WRITE_USER_FS
popq %rax
popq %rbx
popq %rcx
popq %rdx
popq %rbp
popq %rsi
popq %rdi
popq %r8
popq %r9
popq %r10
popq %r11
popq %r12
popq %r13
popq %r14
popq %r15
addq $40, %rsp # discard trapno, errorcode, rip, cs, rflags
# Mask interrupts: sysretq is unsafe with IF set on non-canonical RSP.
# See https://www.felixcloutier.com/x86/sysret
cli
movq (%rsp), %rsp # restore the user stack
sysretq

1281
kernel/arch/x86_64/vectors.S Normal file
View file

File diff suppressed because it is too large Load diff

631
kernel/arch/x86_64/vm.c Normal file
View file

@ -0,0 +1,631 @@
#include "param.h"
#include "types.h"
#include "defs.h"
#include "x86.h"
#include "memlayout.h"
#include "mmu.h"
#include "proc.h"
int mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm);
static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc);
// `cpu` and `proc` are no longer __thread variables; they live at fixed offsets
// in the per-CPU block and are read through %gs at each use. See the long
// comment in include/proc.h for why -- the short version is that __thread means
// per *thread*, the compiler caches the thread pointer across calls on that
// basis, and a kernel thread changes CPU at every sched().
// Kernel FS_BASE set by seginit(); read by trapasm.S to restore after user TLS
uint64 kernel_fs_base;
// Single-CPU scratch for syscall_entry FS swap (safe while IF cleared by SYSCALL/SFMASK)
uint64 _sc_rax, _sc_rcx, _sc_rdx;
static pml4e_t *kpml4;
static pdpe_t *kpdpt;
struct tassdesc {
uint lim_15_0 : 16;
uint base_15_0 : 16;
uint base_23_16 : 8;
uint type : 4;
uint s : 1;
uint dpl : 2;
uint p : 1;
uint lim_19_16 : 4;
uint avl : 1;
uint l : 1;
uint d : 1;
uint g : 1;
uint base_31_24 : 8;
uint32 base_63_32;
uint32 res;
};
void
syscallinit(void)
{
// SYSRETQ does NOT use USER_CS/USER_DS directly. Per the AMD64/Intel
// convention, it computes CS = STAR[63:48]+16 and SS = STAR[63:48]+8
// (RPL forced to 3), so STAR[63:48] must hold the *compat* 32-bit user
// code selector (SEG_UCODE32), relying on the GDT's consecutive
// [UCODE32, UDATA, UCODE] layout — NOT USER_CS itself. Using USER_CS
// here made every sysretq load CS=USER_CS+16 (an invalid selector,
// pointing past UCODE into the TSS descriptor) and SS=USER_CS+8. QEMU's
// TCG apparently doesn't validate segment descriptors on SYSRET, so this
// went unnoticed until an interrupt forced a strictly-validated iretq
// with that same bogus CS still loaded, which correctly #GP'd.
wrmsr(MSR_STAR,
((((uint64)((SEG_UCODE32 << 3) | DPL_USER) << 16) | (uint64)KERNEL_CS) << 32));
wrmsr(MSR_LSTAR, (addr_t)syscall_entry);
wrmsr(MSR_CSTAR, (addr_t)ignore_sysret);
wrmsr(MSR_SFMASK, FL_TF|FL_DF|FL_IF|FL_IOPL_MASK|FL_AC|FL_NT);
}
void
seginit(void)
{
struct segdesc *gdt;
uint *tss;
uint64 addr;
void *local;
struct cpu *c;
// The per-CPU page, preallocated by the boot processor for every AP.
//
// It must not be kalloc()'d here on an application processor. kalloc takes a
// spinlock; acquire() calls holding(); holding() reads `cpu`, which is a
// __thread variable reached through %fs -- and %fs is set up four lines
// below. So an AP allocating its own page reads the per-CPU pointer before
// the per-CPU pointer exists, and the kernel dies in panic("acquire") with
// every lock appearing to be held.
//
// startothers() fills c->local in before starting each AP, on the boot
// processor, where locking works.
c = &cpus[cpunum()];
local = c->local;
if (local == 0) {
local = kalloc(); // the boot processor's own, taken normally
memset(local, 0, PGSIZE);
}
gdt = (struct segdesc*) local;
tss = (uint*) (((char*) local) + 1024);
tss[16] = 0x00680000;
kernel_fs_base = ((uint64) local) + 2048;
wrmsr(0xC0000100, kernel_fs_base);
// The TCB self-pointer: the word at %fs:0 must hold the thread pointer.
//
// This is part of the x86-64 TLS ABI and it was missing, which worked only
// by accident. GCC with -mtls-direct-seg-refs compiles `cpu = c` into a
// single `movq %rcx, %fs:-0x10` and never needs the thread pointer as a
// value, so nothing ever read this word. clang materialises it instead --
// `movq %fs:0x0, %rax; addq $-0x10, %rax; movq %rcx, (%rax)` -- which read
// the zero left by the memset above and stored through address
// 0xfffffffffffffff0, three instructions later.
//
// Writing it costs one store per CPU and removes the dependency on a
// particular compiler's TLS optimisation.
*(uint64 *) kernel_fs_base = kernel_fs_base;
// Clear the two per-CPU slots before anything can read them. Their offsets
// are PERCPU_CPU and PERCPU_PROC in include/proc.h and are shared with
// trapasm.S, which reads %gs:16 for proc on the way out to user mode.
((uint64 *) kernel_fs_base)[2] = 0; // struct cpu * (PERCPU_CPU)
((uint64 *) kernel_fs_base)[3] = 0; // struct proc * (PERCPU_PROC)
// The same address in GS base, per CPU.
//
// trapasm.S needs to find *this* CPU's kernel FS base on every entry from
// user mode, and it has no register to work from at that instant -- which is
// why it used to read the kernel_fs_base global. That global is written by
// every CPU during its own seginit(), so with more than one processor the
// last writer wins and every CPU restores one CPU's per-CPU block. GS base is
// per-CPU in hardware, so %gs:0 answers the question with no global at all.
//
// Safe because GS belongs entirely to the kernel here: sys_arch_prctl()
// implements ARCH_SET_FS and ARCH_GET_FS only, and musl uses FS for TLS on
// x86-64. See the comment on RESTORE_KERNEL_FS.
wrmsr(0xC0000101, kernel_fs_base); // IA32_GS_BASE
c->local = local;
cpu_set(c);
proc_set(0);
// Two of the entries below are deliberately null descriptors, and neither is
// dead code even though both look it:
//
// SEG_UCODE32 is never loaded, but SYSRETQ computes CS from STAR[63:48]+16
// and SS from +8, so the *slot* must exist to keep the
// [UCODE32, UDATA, UCODE] triple consecutive. See syscallinit.
// SEG_KCPU is a leftover slot number from 32-bit xv6, where %gs held the
// per-CPU pointer. Schunix uses FS_BASE/TLS instead, but the
// index has to stay occupied or SEG_TSS stops landing at 7.
//
// Deleting either moves every descriptor after it and breaks sysret.
addr = (uint64) tss;
gdt[0] = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
gdt[SEG_KCODE] = SEG(STA_X|STA_R, 0, 0, 0);
gdt[SEG_KDATA] = SEG(STA_W, 0, 0, 0);
gdt[SEG_UCODE32] = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
gdt[SEG_UDATA] = SEG(STA_W, 0, 0, DPL_USER);
gdt[SEG_UCODE] = SEG(STA_X|STA_R, 0, 0, DPL_USER);
gdt[SEG_KCPU] = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
struct tassdesc *td = (struct tassdesc*)(gdt + SEG_TSS);
td->lim_15_0 = 0x67;
td->base_15_0 = addr & 0xffff;
td->base_23_16 = (addr >> 16) & 0xff;
td->type = STS_T64A;
td->s = 0;
td->dpl = 0;
td->p = 1;
td->lim_19_16 = 0;
td->avl = 0;
td->l = 0;
td->d = 0;
td->g = 0;
td->base_31_24 = (addr >> 24) & 0xff;
td->base_63_32 = (addr >> 32) & 0xffffffff;
td->res = 0;
lgdt((void*) gdt, (NSEGS+1) * sizeof(struct segdesc));
ltr(SEG_TSS << 3);
}
pml4e_t*
setupkvm(void)
{
pml4e_t *pml4 = (pml4e_t*) kalloc();
if(pml4 == 0) return 0;
memset(pml4, 0, PGSIZE);
pml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W;
// The framebuffer is deliberately NOT mapped here.
//
// This used to identity-map 0xfd000000 for 800*600*4 bytes, user-writable,
// into every process that ever existed. Three things were wrong with that.
// The address and the size were both guesses that only hold on QEMU's
// stdvga at one resolution; setupkvm runs before vga_init, so on the first
// call nothing had even read the VBE block yet; and every process could
// scribble on the screen without asking anyone.
//
// Nothing used it -- userspace reaches the framebuffer through
// mmap("/dev/fb0"), which maps the real base at the real size. So it is
// gone, and a process gets the framebuffer exactly when it asks for it.
return pml4;
}
void
kvmalloc(void)
{
kpml4 = (pml4e_t*) kalloc();
memset(kpml4, 0, PGSIZE);
kpdpt = (pdpe_t*) kalloc();
memset(kpdpt, 0, PGSIZE);
kpml4[0] = v2p(kpdpt) | PTE_P | PTE_W;
kpml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W;
for(uint64 i = 0; i < 4; i++) {
uint64 flags = PTE_PS | PTE_P | PTE_W;
if(i >= 3) {
flags |= PTE_PCD | PTE_PWT;
}
kpdpt[i] = (i << 30) | flags;
}
switchkvm();
}
void
switchuvm(struct proc *p)
{
pushcli();
if(p->pgdir == 0)
panic("switchuvm: no pgdir");
uint *tss = (uint*) (((char*) mycpu()->local) + 1024);
const addr_t stktop = (addr_t)p->kstack + KSTACKSIZE;
tss[1] = (uint)stktop;
tss[2] = (uint)(stktop >> 32);
lcr3(v2p(p->pgdir));
popcli();
}
static pte_t *
walkpgdir(pml4e_t *pml4, const void *va, int alloc)
{
pml4e_t *pml4e;
pdpe_t *pdp, *pdpe;
pde_t *pd, *pde;
pte_t *pgtab;
pml4e = &pml4[PMX(va)];
if(*pml4e & PTE_P)
pdp = (pdpe_t*)P2V(PTE_ADDR(*pml4e));
else {
if(!alloc || (pdp = (pdpe_t*)kalloc()) == 0)
return 0;
memset(pdp, 0, PGSIZE);
*pml4e = V2P(pdp) | PTE_P | PTE_W | PTE_U;
}
pdpe = &pdp[PDPX(va)];
if(*pdpe & PTE_P)
pd = (pde_t*)P2V(PTE_ADDR(*pdpe));
else {
if(!alloc || (pd = (pde_t*)kalloc()) == 0)
return 0;
memset(pd, 0, PGSIZE);
*pdpe = V2P(pd) | PTE_P | PTE_W | PTE_U;
}
pde = &pd[PDX(va)];
if(*pde & PTE_P)
pgtab = (pte_t*)P2V(PTE_ADDR(*pde));
else {
if(!alloc || (pgtab = (pte_t*)kalloc()) == 0)
return 0;
memset(pgtab, 0, PGSIZE);
*pde = V2P(pgtab) | PTE_P | PTE_W | PTE_U;
}
return (pte_t*)&pgtab[PTX(va)];
}
void
switchkvm(void)
{
lcr3(v2p(kpml4));
}
int
mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm)
{
char *a, *last;
pte_t *pte;
a = (char*)PGROUNDDOWN((addr_t)va);
last = (char*)PGROUNDDOWN(((addr_t)va) + size - 1);
for(;;){
if((pte = walkpgdir(pgdir, a, 1)) == 0)
return -1;
if(*pte & PTE_P)
panic("remap");
*pte = pa | perm | PTE_P | PTE_U;
invlpg(a);
if(a == last)
break;
a += PGSIZE;
pa += PGSIZE;
}
return 0;
}
void
vm_unmap_range(pml4e_t *pgdir, uint64 va, uint64 length)
{
uint64 a = PGROUNDDOWN(va);
uint64 end = PGROUNDUP(va + length);
for (; a < end; a += PGSIZE) {
pte_t *pte = walkpgdir(pgdir, (void*)a, 0);
if (pte && (*pte & PTE_P)) {
uint64 pa = PTE_ADDR(*pte);
kfree((char*)P2V(pa));
*pte = 0;
invlpg((void*)a);
}
}
}
// Tear down a mapping without freeing what is underneath it.
//
// vm_unmap_range() kfree()s every page it unmaps, which is right for an
// anonymous mapping and catastrophic for anything shared: the framebuffer is
// device memory that was never allocated, and a shared memory segment's pages
// belong to the segment, not to whichever process happens to be detaching.
// Handing either of those to kfree puts them on the free list while another
// process is still reading and writing them.
void
vm_unmap_norelease(pml4e_t *pgdir, uint64 va, uint64 length)
{
uint64 a = PGROUNDDOWN(va);
uint64 end = PGROUNDUP(va + length);
for (; a < end; a += PGSIZE) {
pte_t *pte = walkpgdir(pgdir, (void*)a, 0);
if (pte && (*pte & PTE_P)) {
*pte = 0;
invlpg((void*)a);
}
}
}
// Duplicate an mmap()'d VMA region from src into dst for fork().
// Framebuffer mappings are re-pointed at the same physical MMIO range
// (shared, never copied/freed); anonymous mappings get fresh pages with
// the parent's contents copied in — mirrors what copyuvm() does for the
// [0, sz) heap range, which never covers mmap()'d addresses.
int
copy_vma_range(pml4e_t *dstpgdir, pml4e_t *srcpgdir, uint64 va, uint64 length, int share)
{
uint64 a = PGROUNDDOWN(va);
uint64 end = PGROUNDUP(va + length);
for (; a < end; a += PGSIZE) {
if (share) {
// Point the child at the very same physical pages, whatever they are.
//
// This used to special-case the framebuffer and map vga_fb_phys()
// explicitly. Reading the parent's own PTE instead covers device memory
// and shared memory segments with one rule, and cannot disagree with
// what the parent actually has mapped -- which the explicit version
// could, and did, when the address was hardcoded.
pte_t *spte = walkpgdir(srcpgdir, (void*)a, 0);
if (!spte || !(*spte & PTE_P))
continue;
if (mappages(dstpgdir, (void*)a, PGSIZE, PTE_ADDR(*spte), PTE_FLAGS(*spte)) < 0)
return -1;
continue;
}
pte_t *spte = walkpgdir(srcpgdir, (void*)a, 0);
if (!spte || !(*spte & PTE_P))
continue;
char *mem = kalloc();
if (!mem)
return -1;
memmove(mem, (char*)P2V(PTE_ADDR(*spte)), PGSIZE);
if (mappages(dstpgdir, (void*)a, PGSIZE, V2P(mem), PTE_FLAGS(*spte)) < 0) {
kfree(mem);
return -1;
}
}
return 0;
}
void
inituvm(pde_t *pgdir, char *sinit, uint64 sz)
{
char *mem;
if(sz >= PGSIZE)
panic("inituvm: more than a page");
mem = kalloc();
memset(mem, 0, PGSIZE);
mappages(pgdir, 0, PGSIZE, v2p(mem), PTE_W|PTE_U);
memmove(mem, sinit, sz);
}
int
loaduvm(pml4e_t *pgdir, char *addr, struct inode *ip, uint64 offset, uint64 sz)
{
uint64 done, n, off_in_page;
addr_t pa;
pte_t *pte;
char *cur_page;
for(done = 0; done < sz; done += n){
cur_page = (char*)PGROUNDDOWN((uint64)(addr + done));
off_in_page = (uint64)(addr + done) - (uint64)cur_page;
if((pte = walkpgdir(pgdir, cur_page, 0)) == 0)
panic("loaduvm: address should exist");
pa = PTE_ADDR(*pte);
n = PGSIZE - off_in_page;
if(n > sz - done)
n = sz - done;
if(readi(ip, 0, (uint64)(P2V(pa) + off_in_page), offset + done, n) != (int)n)
return -1;
}
return 0;
}
uint64
allocuvm(pml4e_t *pgdir, uint64 oldsz, uint64 newsz)
{
char *mem;
addr_t a;
if(newsz >= KERNBASE)
return 0;
if(newsz < oldsz)
return oldsz;
a = PGROUNDUP(oldsz);
for(; a < newsz; a += PGSIZE){
mem = kalloc();
if(mem == 0){
deallocuvm(pgdir, newsz, oldsz);
return 0;
}
memset(mem, 0, PGSIZE);
if(mappages(pgdir, (char*)a, PGSIZE, V2P(mem), PTE_W|PTE_U) < 0){
deallocuvm(pgdir, newsz, oldsz);
kfree(mem);
return 0;
}
}
return newsz;
}
uint64
deallocuvm(pml4e_t *pgdir, uint64 oldsz, uint64 newsz)
{
pte_t *pte;
addr_t a, pa;
if(newsz >= oldsz)
return oldsz;
a = PGROUNDUP(newsz);
for(; a < oldsz; a += PGSIZE){
pte = walkpgdir(pgdir, (char*)a, 0);
if(pte && (*pte & PTE_P) != 0){
pa = PTE_ADDR(*pte);
if(pa == 0)
panic("kfree");
if(pa < PHYSTOP) {
char *v = P2V(pa);
kfree(v);
}
*pte = 0;
}
}
return newsz;
}
void
freevm(pml4e_t *pml4)
{
uint i, j, k, l;
pdpe_t *pdp;
pde_t *pd;
pte_t *pt;
if(pml4 == 0)
panic("freevm: no pgdir");
for(i = 0; i < (512/2); i++){
if(pml4[i] & PTE_P){
pdp = (pdpe_t*)P2V(PTE_ADDR(pml4[i]));
for(j = 0; j < 512; j++){
if(pdp[j] & PTE_P){
pd = (pde_t*)P2V(PTE_ADDR(pdp[j]));
for(k = 0; k < 512; k++){
if(pd[k] & PTE_P) {
pt = (pte_t*)P2V(PTE_ADDR(pd[k]));
for(l = 0; l < 512; l++){
if(pt[l] & PTE_P) {
addr_t pa = PTE_ADDR(pt[l]);
if(pa < PHYSTOP && pa != 0) {
kfree(P2V(pa));
}
}
}
kfree((char*)pt);
}
}
kfree((char*)pd);
}
}
kfree((char*)pdp);
}
}
kfree((char*)pml4);
}
void
clearpteu(pml4e_t *pgdir, char *uva)
{
pte_t *pte;
pte = walkpgdir(pgdir, uva, 0);
if(pte == 0)
panic("clearpteu");
*pte &= ~PTE_U;
}
pml4e_t*
copyuvm(pml4e_t *pgdir, uint64 sz)
{
pml4e_t *d;
pte_t *pte;
addr_t pa, i, flags;
char *mem;
if((d = setupkvm()) == 0)
return 0;
for(i = 0; i < sz; i += PGSIZE){
if((pte = walkpgdir(pgdir, (void*)i, 0)) == 0)
continue;
if(!(*pte & PTE_P))
continue;
pa = PTE_ADDR(*pte);
flags = PTE_FLAGS(*pte);
if(pa < PHYSTOP) {
if((mem = kalloc()) == 0)
goto bad;
memmove(mem, (char*)P2V(pa), PGSIZE);
if(mappages(d, (void*)i, PGSIZE, V2P(mem), flags) < 0) {
kfree(mem);
goto bad;
}
}
}
return d;
bad:
freevm(d);
return 0;
}
char*
uva2ka(pml4e_t *pgdir, char *uva)
{
if((addr_t)uva >= KERNBASE)
return 0;
pte_t *pte;
pte = walkpgdir(pgdir, uva, 0);
if(pte == 0 || (*pte & PTE_P) == 0)
return 0;
if((*pte & PTE_U) == 0)
return 0;
addr_t pa = (*pte) & 0x000FFFFFFFFFF000ull;
// Include uva's in-page offset — callers dereference the returned pointer
// directly at uva's address, not just somewhere within its page.
return (char*)P2V(pa) + ((addr_t)uva & 0xFFF);
}
int
copyout(pml4e_t *pgdir, addr_t va, void *p, uint64 len)
{
char *buf, *pa0;
addr_t n, va0;
buf = (char*)p;
while(len > 0){
va0 = PGROUNDDOWN(va);
pa0 = uva2ka(pgdir, (char*)va0);
if(pa0 == 0) {
return -1;
}
n = PGSIZE - (va - va0);
if(n > len)
n = len;
memmove(pa0 + (va - va0), buf, n);
len -= n;
buf += n;
va += n;
}
return 0;
}
int
copyin(pml4e_t *pgdir, addr_t srcva, void *dst, uint64 len)
{
uint64 n, va0;
char *pa0;
char *dst_buf = (char*)dst;
while(len > 0){
va0 = PGROUNDDOWN(srcva);
pa0 = uva2ka(pgdir, (char*)va0);
if(pa0 == 0)
return -1;
n = PGSIZE - (srcva - va0);
if(n > len)
n = len;
memmove(dst_buf, pa0 + (srcva - va0), n);
len -= n;
dst_buf += n;
srcva += n;
}
return 0;
}