diff --git a/include/mp.h b/include/mp.h index dea5705..350e2af 100644 --- a/include/mp.h +++ b/include/mp.h @@ -52,3 +52,8 @@ struct mpioapic { // I/O APIC table entry #define MPIOINTR 0x03 // One per bus interrupt source #define MPLINTR 0x04 // One per system interrupt source + +// The I/O APIC's registers, as the firmware described them. 0xFEC00000 is +// only where they usually are; see kernel/arch/x86_64/mp.c. +#define IOAPIC_DEFAULT_PHYS 0xFEC00000ULL +extern uint64 ioapic_phys; diff --git a/kernel/arch/x86_64/acpi.c b/kernel/arch/x86_64/acpi.c index d11b590..0204ee1 100644 --- a/kernel/arch/x86_64/acpi.c +++ b/kernel/arch/x86_64/acpi.c @@ -28,6 +28,13 @@ extern struct cpu cpus[NCPU]; extern int ncpu; extern uchar ioapicid; +extern uint64 ioapic_phys; + +// The ACPI power-management timer: a 3.579545 MHz counter at an I/O port the +// FADT names. 0 when there is none. lapic.c times the TSC against it when the +// PIT does not count -- see calibrate_without_pit(). +uint32 acpi_pmtmr_port; +int acpi_pmtmr_32; // 32 bits wide rather than 24 extern volatile uint *lapic; // The Root System Description Pointer. Found by signature, not by being told @@ -73,6 +80,7 @@ struct acpi_madt { #define MADT_LAPIC 0 #define MADT_IOAPIC 1 #define MADT_LAPIC_OVERRIDE 5 +#define MADT_X2APIC 9 struct madt_entry { uint8 type; @@ -97,6 +105,18 @@ struct madt_ioapic { uint32 gsi_base; } __attribute__((packed)); +// The same as MADT_LAPIC, for a processor whose APIC ID does not fit in a +// byte -- or whose firmware simply prefers this form. Firmware that hands over +// in x2APIC mode is entitled to describe every processor this way and none +// the old way, and then a parser that reads only type 0 finds no CPUs at all. +struct madt_x2apic { + struct madt_entry e; + uint16 reserved; + uint32 apic_id; + uint32 flags; + uint32 acpi_uid; +} __attribute__((packed)); + struct madt_lapic_override { struct madt_entry e; uint16 reserved; @@ -211,6 +231,36 @@ find_table(struct acpi_rsdp *rsdp, const char *sig) } // Returns 1 if the processors were enumerated, 0 to fall back to mpinit(). +// One processor from the MADT. +// +// Enabled processors only. "Online capable" means the processor is *not* +// running but could be brought up later by hot-plug, which this kernel does +// not do -- and AMD firmware routinely lists every core the socket could have +// that way, present or not. Counting those as CPUs sent startothers() a +// STARTUP to an APIC that does not exist, and then it waited for an answer +// for ever: a hang after "acpi: N cpus" on exactly the machines whose table +// was the most complete. +// +// An ID above 254 is refused too: xAPIC addresses a processor with eight bits +// (255 is broadcast), and that is the mode this kernel runs the APIC in. +static void +add_cpu(uint32 apic_id, uint32 flags) +{ + if (!(flags & LAPIC_ENABLED)) + return; + if (apic_id > 254) { + cprintf("acpi: cpu with apic id %u needs x2APIC; not using it\n", apic_id); + return; + } + for (int i = 0; i < ncpu; i++) + if (cpus[i].apicid == apic_id) + return; // listed as both type 0 and type 9 + if (ncpu < NCPU) { + cpus[ncpu].apicid = (uchar)apic_id; + ncpu++; + } +} + int acpi_init(void) { @@ -218,6 +268,17 @@ acpi_init(void) if (!rsdp) return 0; + // The FADT first, for the PM timer: PM_TMR_BLK at offset 76, and bit 8 of + // the flags at 112 says whether it counts 32 bits or 24. A hardware-reduced + // platform has neither and says so with a zero port. + struct acpi_sdt *fadt = find_table(rsdp, "FACP"); + if (fadt && fadt->length >= 116 && checksum_ok(fadt, fadt->length)) { + acpi_pmtmr_port = *(uint32 *)((uint8 *)fadt + 76); + acpi_pmtmr_32 = (*(uint32 *)((uint8 *)fadt + 112) >> 8) & 1; + if (acpi_pmtmr_port > 0xFFFF) + acpi_pmtmr_port = 0; + } + struct acpi_madt *madt = (struct acpi_madt *)find_table(rsdp, "APIC"); if (!madt || !checksum_ok(madt, madt->h.length)) return 0; @@ -239,23 +300,28 @@ acpi_init(void) switch (ent->type) { case MADT_LAPIC: { struct madt_lapic *l = (struct madt_lapic *)p; - // Skip processors that are neither enabled nor online-capable: those are - // sockets the firmware is describing but not offering. - if (!(l->flags & (LAPIC_ENABLED | LAPIC_ONLINE_CAPABLE))) - break; - if (ncpu < NCPU) { - cpus[ncpu].apicid = l->apic_id; - ncpu++; - } + add_cpu(l->apic_id, l->flags); + break; + } + case MADT_X2APIC: { + struct madt_x2apic *x = (struct madt_x2apic *)p; + add_cpu(x->apic_id, x->flags); break; } case MADT_IOAPIC: { struct madt_ioapic *io = (struct madt_ioapic *)p; - // The first one only. Multi-I/O-APIC routing needs the interrupt source - // overrides too, which nothing here reads yet, so claiming to support it - // would be worse than not. - if (!found_ioapic) { + // One only: the one whose inputs start at GSI 0, which is where the ISA + // lines and the chipset's PCI lines are. Multi-I/O-APIC routing needs + // the interrupt source overrides too, which nothing here reads yet, so + // claiming to support it would be worse than not. + // + // And its address from the table rather than 0xFEC00000. That is the + // usual place and not a promised one: a second I/O APIC listed first, + // or a board that moved the first, used to get its registers written + // at an address where there were none. + if (!found_ioapic || io->gsi_base == 0) { ioapicid = io->id; + ioapic_phys = io->addr; found_ioapic = 1; } break; diff --git a/kernel/arch/x86_64/ioapic.c b/kernel/arch/x86_64/ioapic.c index 3fc6525..17be31c 100644 --- a/kernel/arch/x86_64/ioapic.c +++ b/kernel/arch/x86_64/ioapic.c @@ -6,8 +6,11 @@ #include "defs.h" #include "traps.h" #include "memlayout.h" +#include "mmu.h" +#include "mp.h" +#include "param.h" +#include "proc.h" -#define IOAPIC 0xFEC00000 // Default physical address of IO APIC #define REG_ID 0x00 // Register index: ID #define REG_VER 0x01 // Register index: version @@ -51,7 +54,12 @@ ioapicinit(void) { int i, id, maxintr; - ioapic = P2V((volatile struct ioapic*)IOAPIC); + // The firmware's address for it, not a constant: see ioapic_phys in mp.c. + // ioremap() rather than P2V because nothing obliges it to be in the low + // four gigabytes the direct map is sure to cover. + ioapic = ioremap(ioapic_phys, PGSIZE); + if(!ioapic) + panic("ioapicinit: cannot map the I/O APIC"); maxintr = (ioapicread(REG_VER) >> 16) & 0xFF; id = ioapicread(REG_ID) >> 24; if(id != ioapicid) @@ -85,7 +93,12 @@ ioapicroute(int irq, int cpunum, int level_low) // Program the destination before unmasking, so the entry is never briefly // live pointing at CPU 0 when it was meant for another. - ioapicwrite(REG_TABLE+2*irq+1, cpunum << 24); + // + // The destination is an APIC *ID*, not a CPU index. This wrote the index, + // which is the same number only when the boot processor's APIC ID is 0 -- + // true under every emulator and on most boards, and not promised: an + // interrupt aimed at an ID nobody has is an interrupt nobody receives. + ioapicwrite(REG_TABLE+2*irq+1, (uint)cpus[cpunum].apicid << 24); ioapicwrite(REG_TABLE+2*irq, lo); } diff --git a/kernel/arch/x86_64/lapic.c b/kernel/arch/x86_64/lapic.c index 743ea6c..fb6867e 100644 --- a/kernel/arch/x86_64/lapic.c +++ b/kernel/arch/x86_64/lapic.c @@ -80,6 +80,116 @@ uint lapic_timer_hz; // measured LAPIC timer counts per second uint64 tsc_hz; // measured TSC cycles per second uint64 boot_tsc; // the TSC when calibration finished +// The TSC's frequency from the processor itself, for when there is no PIT to +// time it against. 0 if the processor does not say. +// +// Leaf 0x15 is the ratio of the TSC to the core crystal, and the crystal's +// frequency when the part reports it; leaf 0x16 is the base frequency in MHz, +// which is the TSC's on every part that has an invariant one. The first is +// exact and the second is a nominal figure, so the first is preferred. +static uint64 +tsc_hz_from_cpuid(void) +{ + uint32 max, a, b, c, d; + + cpuid_leaf(0, &max, 0, 0, 0); + if (max >= 0x15) { + cpuid_leaf(0x15, &a, &b, &c, &d); + if (a && b && c) + return (uint64)c * b / a; + } + if (max >= 0x16) { + cpuid_leaf(0x16, &a, 0, 0, 0); + if (a & 0xFFFF) + return (uint64)(a & 0xFFFF) * 1000000; + } + return 0; +} + +// The fallback when the PIT does not count. +// +// Something else has to be the stopwatch. In order: the ACPI PM timer, which +// is a real clock at a fixed 3.579545 MHz on every PC that is not +// hardware-reduced; then the TSC frequency the processor states in CPUID, +// with the TSC itself as the stopwatch; and only then a guess. +extern uint32 acpi_pmtmr_port; +extern int acpi_pmtmr_32; + +#define PMTMR_HZ 3579545 + +// 50 ms of the PM timer, with the LAPIC and the TSC counted across it. +// Returns 0 if the timer did not move, which is a port that is not one. +static int +calibrate_pmtimer(void) +{ + uint32 mask = acpi_pmtmr_32 ? 0xFFFFFFFFu : 0x00FFFFFFu; + uint32 want = PMTMR_HZ / 20, t0, t; + + lapicw(TDCR, X1); + lapicw(TIMER, MASKED); + lapicw(TICR, 0xFFFFFFFF); + + t0 = inl(acpi_pmtmr_port) & mask; + uint64 tsc0 = rdtsc(); + uint lap0 = lapic[TCCR]; + do { + t = inl(acpi_pmtmr_port) & mask; + if (rdtsc() - tsc0 > 4000000000ULL) { + lapicw(TICR, 0); + return 0; + } + } while (((t - t0) & mask) < want); + uint lap1 = lapic[TCCR]; + uint64 tsc1 = rdtsc(); + lapicw(TICR, 0); + + uint32 ticks = (t - t0) & mask; + tsc_hz = (tsc1 - tsc0) * PMTMR_HZ / ticks; + lapic_timer_hz = (uint)((uint64)(lap0 - lap1) * PMTMR_HZ / ticks); + boot_tsc = tsc1; + return tsc_hz >= 20000000ULL; +} + +static void +calibrate_without_pit(void) +{ + if (acpi_pmtmr_port && calibrate_pmtimer()) { + kinfo("cpu0: the PIT does not count; timed against the ACPI PM timer\n"); + } else { + tsc_hz = tsc_hz_from_cpuid(); + if (tsc_hz == 0) { + // Nothing left to measure against. Wrong-but-running, as below: time + // will run at the wrong speed and the line says that it may. + tsc_hz = 2000000000ULL; + cprintf("cpu0: no PIT, no PM timer and no TSC frequency from CPUID; " + "assuming 2 GHz\n"); + } else { + kinfo("cpu0: the PIT does not count; TSC frequency from CPUID\n"); + } + + lapicw(TDCR, X1); + lapicw(TIMER, MASKED); + lapicw(TICR, 0xFFFFFFFF); + uint64 tsc0 = rdtsc(); + uint lap0 = lapic[TCCR]; + uint64 until = tsc0 + tsc_hz / 20; // 50 ms + while (rdtsc() < until) + asm volatile("pause"); + uint lap1 = lapic[TCCR]; + uint64 tsc1 = rdtsc(); + lapicw(TICR, 0); + + lapic_timer_hz = (uint)((uint64)(lap0 - lap1) * tsc_hz / (tsc1 - tsc0)); + boot_tsc = tsc1; + } + + if (lapic_timer_hz < HZ * 1000u) { + cprintf("cpu0: lapic calibration implausible (%u Hz), assuming 1 GHz bus\n", + lapic_timer_hz); + lapic_timer_hz = 1000000000u; + } +} + static void calibrate_timers(void) { @@ -105,12 +215,41 @@ calibrate_timers(void) uint64 tsc0 = rdtsc(); uint lap0 = lapic[TCCR]; - while (!(inb(PIT_GATE) & 0x20)) // ch2 output rises at terminal count - ; + // Bounded. "Always present" stopped being true: recent Intel boards + // clock-gate the 8254 to save power, and then the output never rises and + // this loop was the whole of the boot -- a machine stopped at the line + // before "lapic timer", with nothing to say why. Four billion TSC cycles is + // over a second on anything that exists and twenty times the window on a + // 4 GHz part, so a working PIT never comes near it. + // + // And checked at the start: in mode 0 the output goes low the moment the + // count is written, so a bit that is already high is no PIT at all. An + // absent one floats -- port 0x61 reads 0xFF -- and then the loop ended at + // once, the TSC was "measured" at 0 MHz, and every clock in the system ran + // thousands of times fast. Under QEMU with pit=off uptime read four days + // thirty seconds after boot. + int pit_ok = !(inb(PIT_GATE) & 0x20); + while (pit_ok && !(inb(PIT_GATE) & 0x20)) { // ch2 output rises at terminal count + if (rdtsc() - tsc0 > 4000000000ULL) { + pit_ok = 0; + break; + } + } uint lap1 = lapic[TCCR]; uint64 tsc1 = rdtsc(); + // 50 ms of a TSC slower than 20 MHz is no processor this runs on: the + // window was not 50 ms, so the PIT was not counting it. + if (pit_ok && tsc1 - tsc0 < 1000000) + pit_ok = 0; + + if (!pit_ok) { + lapicw(TICR, 0); + calibrate_without_pit(); + return; + } + lapicw(TICR, 0); // stop counting // TCCR counts down, so elapsed counts are start minus end. @@ -134,12 +273,61 @@ calibrate_timers(void) } } +#define MSR_APIC_BASE 0x1B +#define APIC_BASE_EN (1ULL << 11) // globally enabled +#define APIC_BASE_EXTD (1ULL << 10) // x2APIC mode + +// Put this processor's local APIC in xAPIC mode, which is the mode everything +// here speaks: registers at an MMIO address, read and written as memory. +// +// Firmware is allowed to hand over in x2APIC mode, and on recent Intel +// laptops it usually does. In that mode the MMIO page is not the APIC any +// more -- the registers are MSRs -- so every lapic[] access read nothing +// useful: the ID came back wrong, cpunum() panicked "unknown apicid", and the +// timer was never started. A hang or a panic within the first second, on the +// newest machines and on no emulator, because none of them hand over that way. +// +// The architecture allows exactly one way back: x2APIC to disabled, disabled +// to xAPIC. Disabling discards the APIC's state, which costs nothing here +// because lapicinit() is about to set all of it. +// +// It cannot be done when the processor has locked xAPIC out (some server +// parts, which report it in IA32_XAPIC_DISABLE_STATUS), or when its APIC ID +// does not fit in eight bits. Both get said plainly rather than left to fail +// at the first register access. +void +lapic_xapic_mode(void) +{ + uint32 d; + uint64 base = rdmsr(MSR_APIC_BASE); + + if ((base & (APIC_BASE_EN | APIC_BASE_EXTD)) == APIC_BASE_EN) + return; // xAPIC already: the usual case + + if (base & APIC_BASE_EXTD) { + cpuid_leaf(7, 0, 0, 0, &d); + if (d & (1u << 29)) { // IA32_ARCH_CAPABILITIES exists + uint64 caps = rdmsr(0x10A); + if ((caps & (1ULL << 21)) && (rdmsr(0xBD) & 1)) + panic("lapic: firmware has locked the APIC in x2APIC mode"); + } + if (rdmsr(0x802) > 254) // x2APIC ID register + panic("lapic: APIC id too large for xAPIC mode"); + wrmsr(MSR_APIC_BASE, base & ~(APIC_BASE_EN | APIC_BASE_EXTD)); + } + // From disabled (whether firmware left it so or the line above did) to + // xAPIC. The base address bits are kept: they are where lapic[] points. + wrmsr(MSR_APIC_BASE, (base & ~APIC_BASE_EXTD) | APIC_BASE_EN); +} + void lapicinit(void) { if(!lapic) return; + lapic_xapic_mode(); + // Enable local APIC; set spurious interrupt vector. lapicw(SVR, ENABLE | (T_IRQ0 + IRQ_SPURIOUS)); @@ -281,7 +469,11 @@ lapicstartap(uchar apicid, uint addr) lapicw(ICRLO, SINIT | LEVEL | ASSERT); microdelay(200); lapicw(ICRLO, SINIT | LEVEL); - microdelay(100); // should be 10ms, but too slow in Bochs! + // The MP specification's 10 ms. This was 100 us, "because Bochs is slow", + // which is a figure for an emulator: older real processors want the full + // wait between INIT and STARTUP, and starting one too early is an AP that + // does not come up. + microdelay(10000); // Send startup IPI (twice!) to enter code. // Regular hardware is supposed to only accept a STARTUP @@ -295,6 +487,20 @@ lapicstartap(uchar apicid, uint addr) } } +// Hold a processor in wait-for-SIPI. For an AP that did not report in on +// time: if it is merely slow, this stops it from waking later and reading the +// trampoline parameters startothers() has since written for the next one -- +// which would be two processors on one stack. +void +lapic_park(uchar apicid) +{ + lapicw(ICRHI, apicid<<24); + lapicw(ICRLO, SINIT | LEVEL | ASSERT); + microdelay(200); + lapicw(ICRLO, SINIT | LEVEL); + microdelay(100); +} + #define CMOS_STATA 0x0a #define CMOS_STATB 0x0b #define CMOS_UIP (1 << 7) // RTC update in progress diff --git a/kernel/arch/x86_64/mp.c b/kernel/arch/x86_64/mp.c index fccf02d..db48be5 100644 --- a/kernel/arch/x86_64/mp.c +++ b/kernel/arch/x86_64/mp.c @@ -25,6 +25,9 @@ int ncpu; // working ones, which was a comfortable thing to read and false. int ncpu_online = 1; uchar ioapicid; +// Where the I/O APIC's registers are. Every source that describes it -- the +// MADT, or this table -- says so, and ioapicinit() maps whatever they said. +uint64 ioapic_phys = IOAPIC_DEFAULT_PHYS; static uchar sum(uchar *addr, int len) @@ -116,7 +119,9 @@ mpinit(void) switch(*p){ case MPPROC: mproc = (struct mpproc*)p; - if(ncpu < NCPU) { + // Bit 0 is "usable". A disabled entry is a processor that is not + // there, and starting it means waiting for an answer that never comes. + if((mproc->flags & 0x01) && ncpu < NCPU) { cpus[ncpu].apicid = mproc->apicid; // apicid may differ from ncpu ncpu++; } @@ -124,7 +129,10 @@ mpinit(void) continue; case MPIOAPIC: ioapic = (struct mpioapic*)p; - ioapicid = ioapic->apicno; + if(ioapic_phys == IOAPIC_DEFAULT_PHYS || ioapic->addr_p == IOAPIC_DEFAULT_PHYS){ + ioapicid = ioapic->apicno; + ioapic_phys = ioapic->addr_p; + } p += sizeof(struct mpioapic); continue; case MPBUS: diff --git a/kernel/core/main.c b/kernel/core/main.c index 41035fe..edfbce9 100644 --- a/kernel/core/main.c +++ b/kernel/core/main.c @@ -386,9 +386,16 @@ kmain(uint32 fb_addr) } // Other CPUs jump here from entryother.S. +void lapic_xapic_mode(void); +void lapic_park(uchar apicid); + void mpenter(void) { + // Before anything reads the APIC -- seginit() and lapicinit() both do. An + // AP keeps whatever mode the firmware left its APIC in across INIT, so the + // boot processor having switched itself back says nothing about this one. + lapic_xapic_mode(); switchkvm(); seginit(); lapicinit(); @@ -500,9 +507,24 @@ startothers(void) lapicstartap(c->apicid, V2P(code)); - // wait for cpu to finish mpmain() - while(c->started == 0) + // Wait for it to finish mpmain() -- for a second, not for ever. + // + // This had no bound, so one processor that did not answer stopped the + // machine here for good, after the last line of hardware detection and + // before anything else. A processor the firmware listed that is not + // there, one that is disabled in setup, or one that simply fails to start + // is a machine running on fewer CPUs, not a machine that cannot boot. + // ncpu_online below counts only the ones that answered, and procfs + // already reports the difference. + uint64 deadline = rdtsc() + tsc_hz; + while(c->started == 0 && rdtsc() < deadline) ; + if(c->started == 0){ + cprintf("smp: cpu with apic id %d did not start; continuing without it\n", + c->apicid); + lapic_park(c->apicid); + continue; + } // Counted here, on the boot processor, after the AP has actually reported // in -- not by the AP itself. This is serialised by the loop, so it needs diff --git a/kernel/dev/mouse.c b/kernel/dev/mouse.c index 5fdd320..650426a 100644 --- a/kernel/dev/mouse.c +++ b/kernel/dev/mouse.c @@ -223,7 +223,19 @@ mouseinit(void) mouse_state.x = 400; mouse_state.y = 300; - while(inb(KBD_S_STAT) & 0x01) + // No 8042 at all: the port floats and reads 0xFF. That is a machine with + // USB input only -- most laptops and desktops built in the last few years, + // whenever the firmware's legacy emulation is off -- and every bit set means + // "output full" for ever, so the flush below never ended and boot stopped + // here. There is no mouse to set up, so do not. + if(inb(KBD_S_STAT) == 0xFF){ + kinfo("ps2: no controller; mouse not set up\n"); + return; + } + + // Drain whatever the controller is holding. Bounded, because a controller + // that keeps producing bytes is broken and still not a reason to hang. + for(int i = 0; i < 1024 && (inb(KBD_S_STAT) & 0x01); i++) inb(KBD_S_DATA); mouse_wait(1);