SchOS/kernel/arch/x86_64/lapic.c

// The local APIC manages internal (non-I/O) interrupts.
// See Chapter 8 & Appendix C of Intel processor manual volume 3.

#include "core/param.h"
#include "core/types.h"
#include "core/defs.h"
#include "core/date.h"
#include "arch/memlayout.h"
#include "arch/traps.h"
#include "arch/mmu.h"
#include "arch/x86.h"
#include "core/proc.h"  // ncpu

// Local APIC registers, divided by 4 for use as uint[] indices.
#define ID      (0x0020/4)   // ID
#define VER     (0x0030/4)   // Version
#define TPR     (0x0080/4)   // Task Priority
#define EOI     (0x00B0/4)   // EOI
#define SVR     (0x00F0/4)   // Spurious Interrupt Vector
  #define ENABLE     0x00000100   // Unit Enable
#define ESR     (0x0280/4)   // Error Status
#define ICRLO   (0x0300/4)   // Interrupt Command
  #define SINIT       0x00000500   // INIT/RESET
  #define STARTUP    0x00000600   // Startup IPI
  #define DELIVS     0x00001000   // Delivery status
  #define ASSERT     0x00004000   // Assert interrupt (vs deassert)
  #define LEVEL      0x00008000   // Level triggered
  #define BCAST      0x00080000   // Send to all APICs, including self.
  #define BUSY       0x00001000
#define ICRHI   (0x0310/4)   // Interrupt Command [63:32]
#define TIMER   (0x0320/4)   // Local Vector Table 0 (TIMER)
  #define X1         0x0000000B   // divide counts by 1
  #define PERIODIC   0x00020000   // Periodic
#define PCINT   (0x0340/4)   // Performance Counter LVT
#define LINT0   (0x0350/4)   // Local Vector Table 1 (LINT0)
#define LINT1   (0x0360/4)   // Local Vector Table 2 (LINT1)
#define ERROR   (0x0370/4)   // Local Vector Table 3 (ERROR)
  #define MASKED     0x00010000   // Interrupt masked
#define TICR    (0x0380/4)   // Timer Initial Count
#define TCCR    (0x0390/4)   // Timer Current Count
#define TDCR    (0x03E0/4)   // Timer Divide Configuration

volatile uint *lapic;  // Initialized in mp.c

static void
lapicw(int index, int value)
{
  lapic[index] = value;
  lapic[ID];  // wait for write to finish, by reading
}

// ---------------------------------------------------------------------
// Timer calibration
// ---------------------------------------------------------------------
// The LAPIC timer counts down at the APIC bus frequency, and nothing in the
// architecture tells you what that frequency is. xv6 loaded TICR with
// 10000000 and hoped; its own comment admitted the gap -- "if xv6 cared more
// about precise timekeeping, TICR would be calibrated using an external time
// source." That value yields the intended 100 Hz only if the bus happens to
// run at exactly 1 GHz.
//
// Under KVM it does not, and the error is not subtle. The kernel believed
// ticks arrived 100 times a second while they were arriving roughly 18 times
// a second, so every number derived from ticks was wrong by that same factor,
// in the same direction, everywhere at once: uptime ran slow, `time` reported
// three seconds for twenty, ping's one-second interval stretched to six. It
// read as three unrelated bugs in three unrelated programs. It was this line.
//
// So measure it. The 8254 PIT is exactly the external time source the comment
// asked for: its input clock is fixed at 1193182 Hz by the hardware, it is
// always present, and channel 2 is the one whose gate software can raise and
// whose output software can poll -- a stopwatch that needs no interrupt, which
// matters because this runs before the IDT is ready to take one.
#define PIT_HZ    1193182   // 8254 input clock, fixed by definition
#define PIT_CH2   0x42
#define PIT_CMD   0x43
#define PIT_GATE  0x61      // bit 0: ch2 gate, bit 1: speaker, bit 5: ch2 output

uint   lapic_timer_hz;      // measured LAPIC timer counts per second
uint64 tsc_hz;              // measured TSC cycles per second
uint64 boot_tsc;            // the TSC when calibration finished

// The TSC's frequency from the processor itself, for when there is no PIT to
// time it against. 0 if the processor does not say.
//
// Leaf 0x15 is the ratio of the TSC to the core crystal, and the crystal's
// frequency when the part reports it; leaf 0x16 is the base frequency in MHz,
// which is the TSC's on every part that has an invariant one. The first is
// exact and the second is a nominal figure, so the first is preferred.
static uint64
tsc_hz_from_cpuid(void)
{
  uint32 max, a, b, c, d;

  cpuid_leaf(0, &max, 0, 0, 0);
  if (max >= 0x15) {
    cpuid_leaf(0x15, &a, &b, &c, &d);
    if (a && b && c)
      return (uint64)c * b / a;
  }
  if (max >= 0x16) {
    cpuid_leaf(0x16, &a, 0, 0, 0);
    if (a & 0xFFFF)
      return (uint64)(a & 0xFFFF) * 1000000;
  }
  return 0;
}

// The fallback when the PIT does not count.
//
// Something else has to be the stopwatch. In order: the ACPI PM timer, which
// is a real clock at a fixed 3.579545 MHz on every PC that is not
// hardware-reduced; then the TSC frequency the processor states in CPUID,
// with the TSC itself as the stopwatch; and only then a guess.
extern uint32 acpi_pmtmr_port;
extern int    acpi_pmtmr_32;

#define PMTMR_HZ 3579545

// 50 ms of the PM timer, with the LAPIC and the TSC counted across it.
// Returns 0 if the timer did not move, which is a port that is not one.
static int
calibrate_pmtimer(void)
{
  uint32 mask = acpi_pmtmr_32 ? 0xFFFFFFFFu : 0x00FFFFFFu;
  uint32 want = PMTMR_HZ / 20, t0, t;

  lapicw(TDCR, X1);
  lapicw(TIMER, MASKED);
  lapicw(TICR, 0xFFFFFFFF);

  t0 = inl(acpi_pmtmr_port) & mask;
  uint64 tsc0 = rdtsc();
  uint   lap0 = lapic[TCCR];
  do {
    t = inl(acpi_pmtmr_port) & mask;
    if (rdtsc() - tsc0 > 4000000000ULL) {
      lapicw(TICR, 0);
      return 0;
    }
  } while (((t - t0) & mask) < want);
  uint   lap1 = lapic[TCCR];
  uint64 tsc1 = rdtsc();
  lapicw(TICR, 0);

  uint32 ticks = (t - t0) & mask;
  tsc_hz         = (tsc1 - tsc0) * PMTMR_HZ / ticks;
  lapic_timer_hz = (uint)((uint64)(lap0 - lap1) * PMTMR_HZ / ticks);
  boot_tsc = tsc1;
  return tsc_hz >= 20000000ULL;
}

static void
calibrate_without_pit(void)
{
  if (acpi_pmtmr_port && calibrate_pmtimer()) {
    kinfo("cpu0: the PIT does not count; timed against the ACPI PM timer\n");
  } else {
    tsc_hz = tsc_hz_from_cpuid();
    if (tsc_hz == 0) {
      // Nothing left to measure against. Wrong-but-running, as below: time
      // will run at the wrong speed and the line says that it may.
      tsc_hz = 2000000000ULL;
      cprintf("cpu0: no PIT, no PM timer and no TSC frequency from CPUID; "
              "assuming 2 GHz\n");
    } else {
      kinfo("cpu0: the PIT does not count; TSC frequency from CPUID\n");
    }

    lapicw(TDCR, X1);
    lapicw(TIMER, MASKED);
    lapicw(TICR, 0xFFFFFFFF);
    uint64 tsc0 = rdtsc();
    uint   lap0 = lapic[TCCR];
    uint64 until = tsc0 + tsc_hz / 20;        // 50 ms
    while (rdtsc() < until)
      asm volatile("pause");
    uint   lap1 = lapic[TCCR];
    uint64 tsc1 = rdtsc();
    lapicw(TICR, 0);

    lapic_timer_hz = (uint)((uint64)(lap0 - lap1) * tsc_hz / (tsc1 - tsc0));
    boot_tsc = tsc1;
  }

  if (lapic_timer_hz < HZ * 1000u) {
    cprintf("cpu0: lapic calibration implausible (%u Hz), assuming 1 GHz bus\n",
            lapic_timer_hz);
    lapic_timer_hz = 1000000000u;
  }
}

static void
calibrate_timers(void)
{
  // 50 ms: long enough that the port I/O around the measurement is noise,
  // short enough that nobody notices it in boot time. The count fits in the
  // 16 bits the PIT has (59659), which is the real upper bound here.
  const uint us = 50000;
  uint count = (uint)(((uint64)PIT_HZ * us) / 1000000);

  // Channel 2, mode 0 (interrupt on terminal count), lobyte then hibyte.
  // Gate high so it counts; the speaker bit stays clear so the box is silent.
  outb(PIT_GATE, (inb(PIT_GATE) & ~0x02) | 0x01);
  outb(PIT_CMD, 0xB0);
  outb(PIT_CH2, count & 0xFF);
  outb(PIT_CH2, (count >> 8) & 0xFF);

  // A free-running LAPIC counter for the duration: one-shot, masked, started
  // at the largest count there is so it cannot wrap inside 50 ms.
  lapicw(TDCR, X1);
  lapicw(TIMER, MASKED);
  lapicw(TICR, 0xFFFFFFFF);

  uint64 tsc0 = rdtsc();
  uint   lap0 = lapic[TCCR];

  // Bounded. "Always present" stopped being true: recent Intel boards
  // clock-gate the 8254 to save power, and then the output never rises and
  // this loop was the whole of the boot -- a machine stopped at the line
  // before "lapic timer", with nothing to say why. Four billion TSC cycles is
  // over a second on anything that exists and twenty times the window on a
  // 4 GHz part, so a working PIT never comes near it.
  //
  // And checked at the start: in mode 0 the output goes low the moment the
  // count is written, so a bit that is already high is no PIT at all. An
  // absent one floats -- port 0x61 reads 0xFF -- and then the loop ended at
  // once, the TSC was "measured" at 0 MHz, and every clock in the system ran
  // thousands of times fast. Under QEMU with pit=off uptime read four days
  // thirty seconds after boot.
  int pit_ok = !(inb(PIT_GATE) & 0x20);
  while (pit_ok && !(inb(PIT_GATE) & 0x20)) { // ch2 output rises at terminal count
    if (rdtsc() - tsc0 > 4000000000ULL) {
      pit_ok = 0;
      break;
    }
  }

  uint   lap1 = lapic[TCCR];
  uint64 tsc1 = rdtsc();

  // 50 ms of a TSC slower than 20 MHz is no processor this runs on: the
  // window was not 50 ms, so the PIT was not counting it.
  if (pit_ok && tsc1 - tsc0 < 1000000)
    pit_ok = 0;

  if (!pit_ok) {
    lapicw(TICR, 0);
    calibrate_without_pit();
    return;
  }

  lapicw(TICR, 0);                  // stop counting

  // TCCR counts down, so elapsed counts are start minus end.
  lapic_timer_hz = (uint)(((uint64)(lap0 - lap1) * 1000000) / us);
  tsc_hz         = ((tsc1 - tsc0) * 1000000) / us;

  // Where the clock starts. Everything the system reports as elapsed time is
  // measured from here, against the TSC, rather than by counting timer
  // interrupts -- see kernel/core/time.c.
  boot_tsc = tsc1;

  // If the measurement came back implausible -- no LAPIC, a hypervisor that
  // stops the counter, a PIT that never raises its output -- fall back to the
  // old assumption rather than dividing by zero and taking the machine down.
  // Wrong-but-running beats a triple fault, and the printed value says which
  // case this is.
  if (lapic_timer_hz < HZ * 1000u) {
    cprintf("cpu0: lapic calibration implausible (%u Hz), assuming 1 GHz bus\n",
            lapic_timer_hz);
    lapic_timer_hz = 1000000000u;   // the 1 GHz that TICR=10000000 implied
  }
}

#define MSR_APIC_BASE      0x1B
#define APIC_BASE_EN       (1ULL << 11)   // globally enabled
#define APIC_BASE_EXTD     (1ULL << 10)   // x2APIC mode

// Put this processor's local APIC in xAPIC mode, which is the mode everything
// here speaks: registers at an MMIO address, read and written as memory.
//
// Firmware is allowed to hand over in x2APIC mode, and on recent Intel
// laptops it usually does. In that mode the MMIO page is not the APIC any
// more -- the registers are MSRs -- so every lapic[] access read nothing
// useful: the ID came back wrong, cpunum() panicked "unknown apicid", and the
// timer was never started. A hang or a panic within the first second, on the
// newest machines and on no emulator, because none of them hand over that way.
//
// The architecture allows exactly one way back: x2APIC to disabled, disabled
// to xAPIC. Disabling discards the APIC's state, which costs nothing here
// because lapicinit() is about to set all of it.
//
// It cannot be done when the processor has locked xAPIC out (some server
// parts, which report it in IA32_XAPIC_DISABLE_STATUS), or when its APIC ID
// does not fit in eight bits. Both get said plainly rather than left to fail
// at the first register access.
void
lapic_xapic_mode(void)
{
  uint32 d;
  uint64 base = rdmsr(MSR_APIC_BASE);

  if ((base & (APIC_BASE_EN | APIC_BASE_EXTD)) == APIC_BASE_EN)
    return;                                    // xAPIC already: the usual case

  if (base & APIC_BASE_EXTD) {
    cpuid_leaf(7, 0, 0, 0, &d);
    if (d & (1u << 29)) {                      // IA32_ARCH_CAPABILITIES exists
      uint64 caps = rdmsr(0x10A);
      if ((caps & (1ULL << 21)) && (rdmsr(0xBD) & 1))
        panic("lapic: firmware has locked the APIC in x2APIC mode");
    }
    if (rdmsr(0x802) > 254)                    // x2APIC ID register
      panic("lapic: APIC id too large for xAPIC mode");
    wrmsr(MSR_APIC_BASE, base & ~(APIC_BASE_EN | APIC_BASE_EXTD));
  }
  // From disabled (whether firmware left it so or the line above did) to
  // xAPIC. The base address bits are kept: they are where lapic[] points.
  wrmsr(MSR_APIC_BASE, (base & ~APIC_BASE_EXTD) | APIC_BASE_EN);
}

void
lapicinit(void)
{
  if(!lapic)
    return;

  lapic_xapic_mode();

  // Enable local APIC; set spurious interrupt vector.
  lapicw(SVR, ENABLE | (T_IRQ0 + IRQ_SPURIOUS));

  // The timer counts down at the APIC bus frequency and interrupts at zero.
  // What that frequency *is*, calibrate_timers() had to go and measure --
  // see the comment above it for why loading a constant here does not work.
  if (lapic_timer_hz == 0) {
    calibrate_timers();
    kinfo("cpu0: lapic timer %u Hz, tsc %u MHz, tick = %u counts at %d Hz\n",
            lapic_timer_hz, (uint)(tsc_hz / 1000000),
            lapic_timer_hz / HZ, HZ);
  }

  lapicw(TDCR, X1);
  lapicw(TIMER, PERIODIC | (T_IRQ0 + IRQ_TIMER));
  lapicw(TICR, lapic_timer_hz / HZ);

  // Disable logical interrupt lines.
  lapicw(LINT0, MASKED);
  lapicw(LINT1, MASKED);

  // Disable performance counter overflow interrupts
  // on machines that provide that interrupt entry.
  if(((lapic[VER]>>16) & 0xFF) >= 4)
    lapicw(PCINT, MASKED);

  // Map error interrupt to IRQ_ERROR.
  lapicw(ERROR, T_IRQ0 + IRQ_ERROR);

  // Clear error status register (requires back-to-back writes).
  lapicw(ESR, 0);
  lapicw(ESR, 0);

  // Ack any outstanding interrupts.
  lapicw(EOI, 0);

  // Send an Init Level De-Assert to synchronise arbitration ID's.
  lapicw(ICRHI, 0);
  lapicw(ICRLO, BCAST | SINIT | LEVEL);
  while(lapic[ICRLO] & DELIVS)
    ;

  // Enable interrupts on the APIC (but not on the processor).
  lapicw(TPR, 0);
}

int
cpunum(void)
{
  int apicid, i;

  // Cannot call cpu when interrupts are enabled:
  // result not guaranteed to last long enough to be used!
  // Would prefer to panic but even printing is chancy here:
  // almost everything, including cprintf and panic, calls cpu,
  // often indirectly through acquire and release.
  if(readeflags()&FL_IF){
    static int n;
    if(n++ == 0)
      cprintf("cpu called from %x with interrupts enabled\n",
        __builtin_return_address(0));
  }

  if (!lapic)
    return 0;

  apicid = lapic[ID] >> 24;
  for (i = 0; i < ncpu; ++i) {
    if (cpus[i].apicid == apicid)
      return i;
  }
  panic("unknown apicid\n");
}

// An NMI to every processor but this one: how a panic stops the others. An
// NMI is delivered with interrupts off too, so it reaches a processor spinning
// on a lock with cli as surely as one running a program; trap() halts any that
// takes one while panic_stop is set. Bounded, because the caller is dying.
void
lapic_nmi_others(void)
{
  if (!lapic)
    return;
  lapicw(ICRHI, 0);
  lapicw(ICRLO, 0x000C0000 | 0x00000400 | ASSERT);   // all but self, NMI
  for (int i = 0; i < 100000 && (lapic[ICRLO] & DELIVS); i++)
    microdelay(1);
}

// Acknowledge interrupt.
void
lapiceoi(void)
{
  if(lapic)
    lapicw(EOI, 0);
}

// Spin for a given number of microseconds.
//
// This was an empty function body -- xv6's stub, carrying a comment saying
// real hardware would want to tune it dynamically. Nothing ever did, so every
// microdelay() in the tree returned immediately: the xHCI reset sequence, the
// USB hub power-on wait, the UART drain. Those delays are written into the
// specifications because the hardware genuinely needs the time, and a port
// that "works anyway" is one that is racing and winning.
//
// The TSC is the right clock for this: calibrate_timers() measured it in the
// same pass as the LAPIC, and unlike the LAPIC timer it is not already busy
// driving the tick.
void
microdelay(int us)
{
  if (us <= 0)
    return;

  // Before calibration (early boot, and the AP trampoline) there is no
  // measured frequency to work from. Spin on a rough count instead: the
  // figure is wrong, but being approximately late is the entire point of the
  // call, and being instantly early is what broke it.
  if (tsc_hz == 0) {
    for (volatile int i = 0; i < us * 100; i++)
      ;
    return;
  }

  uint64 end = rdtsc() + ((uint64)us * tsc_hz) / 1000000;
  while (rdtsc() < end)
    asm volatile("pause");
}

#define CMOS_PORT    0x70
#define CMOS_RETURN  0x71

// Start additional processor running entry code at addr.
// See Appendix B of MultiProcessor Specification.
void
lapicstartap(uchar apicid, uint addr)
{
  int i;
  ushort *wrv;

  // "The BSP must initialize CMOS shutdown code to 0AH
  // and the warm reset vector (DWORD based at 40:67) to point at
  // the AP startup code prior to the [universal startup algorithm]."
  outb(CMOS_PORT, 0xF);  // offset 0xF is shutdown code
  outb(CMOS_PORT+1, 0x0A);
  wrv = (ushort*)P2V((0x40<<4 | 0x67));  // Warm reset vector
  wrv[0] = 0;
  wrv[1] = addr >> 4;

  // "Universal startup algorithm."
  // Send INIT (level-triggered) interrupt to reset other CPU.
  lapicw(ICRHI, apicid<<24);
  lapicw(ICRLO, SINIT | LEVEL | ASSERT);
  microdelay(200);
  lapicw(ICRLO, SINIT | LEVEL);
  // The MP specification's 10 ms. This was 100 us, "because Bochs is slow",
  // which is a figure for an emulator: older real processors want the full
  // wait between INIT and STARTUP, and starting one too early is an AP that
  // does not come up.
  microdelay(10000);

  // Send startup IPI (twice!) to enter code.
  // Regular hardware is supposed to only accept a STARTUP
  // when it is in the halted state due to an INIT.  So the second
  // should be ignored, but it is part of the official Intel algorithm.
  // Bochs complains about the second one.  Too bad for Bochs.
  for(i = 0; i < 2; i++){
    lapicw(ICRHI, apicid<<24);
    lapicw(ICRLO, STARTUP | (addr>>12));
    microdelay(200);
  }
}

// Hold a processor in wait-for-SIPI. For an AP that did not report in on
// time: if it is merely slow, this stops it from waking later and reading the
// trampoline parameters startothers() has since written for the next one --
// which would be two processors on one stack.
void
lapic_park(uchar apicid)
{
  lapicw(ICRHI, apicid<<24);
  lapicw(ICRLO, SINIT | LEVEL | ASSERT);
  microdelay(200);
  lapicw(ICRLO, SINIT | LEVEL);
  microdelay(100);
}

#define CMOS_STATA   0x0a
#define CMOS_STATB   0x0b
#define CMOS_UIP    (1 << 7)        // RTC update in progress

#define SECS    0x00
#define MINS    0x02
#define HOURS   0x04
#define DAY     0x07
#define MONTH   0x08
#define YEAR    0x09

static uint cmos_read(uint reg)
{
  outb(CMOS_PORT,  reg);
  microdelay(200);

  return inb(CMOS_RETURN);
}

static void fill_rtcdate(struct rtcdate *r)
{
  r->second = cmos_read(SECS);
  r->minute = cmos_read(MINS);
  r->hour   = cmos_read(HOURS);
  r->day    = cmos_read(DAY);
  r->month  = cmos_read(MONTH);
  r->year   = cmos_read(YEAR);
}
//PAGEBREAK!

// qemu seems to use 24-hour GWT and the values are BCD encoded
void cmostime(struct rtcdate *r)
{
  struct rtcdate t1, t2;
  int sb, bcd;

  sb = cmos_read(CMOS_STATB);

  bcd = (sb & (1 << 2)) == 0;

  // make sure CMOS doesn't modify time while we read it
  for(;;) {
    fill_rtcdate(&t1);
    if(cmos_read(CMOS_STATA) & CMOS_UIP)
        continue;
    fill_rtcdate(&t2);
    if(memcmp(&t1, &t2, sizeof(t1)) == 0)
      break;
  }

  // convert
  if(bcd) {
#define    CONV(x)     (t1.x = ((t1.x >> 4) * 10) + (t1.x & 0xf))
    CONV(second);
    CONV(minute);
    CONV(hour  );
    CONV(day   );
    CONV(month );
    CONV(year  );
#undef     CONV
  }

  *r = t1;
  r->year += 2000;
}

// A fixed interrupt to one processor, by APIC id. Waits for the local APIC
// to say it has sent the last one before writing the next: two ICR writes
// back to back lose the first.
void
lapic_send_ipi(int apicid, int vector)
{
  if (!lapic)
    return;
  while (lapic[ICRLO] & DELIVS)
    ;
  lapicw(ICRHI, apicid << 24);
  lapicw(ICRLO, vector);                // fixed, physical, edge, no shorthand
  while (lapic[ICRLO] & DELIVS)
    ;
}