SchOS/kernel/arch/x86_64/lapic.c

// The local APIC manages internal (non-I/O) interrupts.
// See Chapter 8 & Appendix C of Intel processor manual volume 3.

#include "param.h"
#include "types.h"
#include "defs.h"
#include "date.h"
#include "memlayout.h"
#include "traps.h"
#include "mmu.h"
#include "x86.h"
#include "proc.h"  // ncpu

// Local APIC registers, divided by 4 for use as uint[] indices.
#define ID      (0x0020/4)   // ID
#define VER     (0x0030/4)   // Version
#define TPR     (0x0080/4)   // Task Priority
#define EOI     (0x00B0/4)   // EOI
#define SVR     (0x00F0/4)   // Spurious Interrupt Vector
  #define ENABLE     0x00000100   // Unit Enable
#define ESR     (0x0280/4)   // Error Status
#define ICRLO   (0x0300/4)   // Interrupt Command
  #define SINIT       0x00000500   // INIT/RESET
  #define STARTUP    0x00000600   // Startup IPI
  #define DELIVS     0x00001000   // Delivery status
  #define ASSERT     0x00004000   // Assert interrupt (vs deassert)
  #define LEVEL      0x00008000   // Level triggered
  #define BCAST      0x00080000   // Send to all APICs, including self.
  #define BUSY       0x00001000
#define ICRHI   (0x0310/4)   // Interrupt Command [63:32]
#define TIMER   (0x0320/4)   // Local Vector Table 0 (TIMER)
  #define X1         0x0000000B   // divide counts by 1
  #define PERIODIC   0x00020000   // Periodic
#define PCINT   (0x0340/4)   // Performance Counter LVT
#define LINT0   (0x0350/4)   // Local Vector Table 1 (LINT0)
#define LINT1   (0x0360/4)   // Local Vector Table 2 (LINT1)
#define ERROR   (0x0370/4)   // Local Vector Table 3 (ERROR)
  #define MASKED     0x00010000   // Interrupt masked
#define TICR    (0x0380/4)   // Timer Initial Count
#define TCCR    (0x0390/4)   // Timer Current Count
#define TDCR    (0x03E0/4)   // Timer Divide Configuration

volatile uint *lapic;  // Initialized in mp.c

static void
lapicw(int index, int value)
{
  lapic[index] = value;
  lapic[ID];  // wait for write to finish, by reading
}

// ---------------------------------------------------------------------
// Timer calibration
// ---------------------------------------------------------------------
// The LAPIC timer counts down at the APIC bus frequency, and nothing in the
// architecture tells you what that frequency is. xv6 loaded TICR with
// 10000000 and hoped; its own comment admitted the gap -- "if xv6 cared more
// about precise timekeeping, TICR would be calibrated using an external time
// source." That value yields the intended 100 Hz only if the bus happens to
// run at exactly 1 GHz.
//
// Under KVM it does not, and the error is not subtle. The kernel believed
// ticks arrived 100 times a second while they were arriving roughly 18 times
// a second, so every number derived from ticks was wrong by that same factor,
// in the same direction, everywhere at once: uptime ran slow, `time` reported
// three seconds for twenty, ping's one-second interval stretched to six. It
// read as three unrelated bugs in three unrelated programs. It was this line.
//
// So measure it. The 8254 PIT is exactly the external time source the comment
// asked for: its input clock is fixed at 1193182 Hz by the hardware, it is
// always present, and channel 2 is the one whose gate software can raise and
// whose output software can poll -- a stopwatch that needs no interrupt, which
// matters because this runs before the IDT is ready to take one.
#define PIT_HZ    1193182   // 8254 input clock, fixed by definition
#define PIT_CH2   0x42
#define PIT_CMD   0x43
#define PIT_GATE  0x61      // bit 0: ch2 gate, bit 1: speaker, bit 5: ch2 output

uint   lapic_timer_hz;      // measured LAPIC timer counts per second
uint64 tsc_hz;              // measured TSC cycles per second

static void
calibrate_timers(void)
{
  // 50 ms: long enough that the port I/O around the measurement is noise,
  // short enough that nobody notices it in boot time. The count fits in the
  // 16 bits the PIT has (59659), which is the real upper bound here.
  const uint us = 50000;
  uint count = (uint)(((uint64)PIT_HZ * us) / 1000000);

  // Channel 2, mode 0 (interrupt on terminal count), lobyte then hibyte.
  // Gate high so it counts; the speaker bit stays clear so the box is silent.
  outb(PIT_GATE, (inb(PIT_GATE) & ~0x02) | 0x01);
  outb(PIT_CMD, 0xB0);
  outb(PIT_CH2, count & 0xFF);
  outb(PIT_CH2, (count >> 8) & 0xFF);

  // A free-running LAPIC counter for the duration: one-shot, masked, started
  // at the largest count there is so it cannot wrap inside 50 ms.
  lapicw(TDCR, X1);
  lapicw(TIMER, MASKED);
  lapicw(TICR, 0xFFFFFFFF);

  uint64 tsc0 = rdtsc();
  uint   lap0 = lapic[TCCR];

  while (!(inb(PIT_GATE) & 0x20))   // ch2 output rises at terminal count
    ;

  uint   lap1 = lapic[TCCR];
  uint64 tsc1 = rdtsc();

  lapicw(TICR, 0);                  // stop counting

  // TCCR counts down, so elapsed counts are start minus end.
  lapic_timer_hz = (uint)(((uint64)(lap0 - lap1) * 1000000) / us);
  tsc_hz         = ((tsc1 - tsc0) * 1000000) / us;

  // If the measurement came back implausible -- no LAPIC, a hypervisor that
  // stops the counter, a PIT that never raises its output -- fall back to the
  // old assumption rather than dividing by zero and taking the machine down.
  // Wrong-but-running beats a triple fault, and the printed value says which
  // case this is.
  if (lapic_timer_hz < HZ * 1000u) {
    cprintf("cpu0: lapic calibration implausible (%u Hz), assuming 1 GHz bus\n",
            lapic_timer_hz);
    lapic_timer_hz = 1000000000u;   // the 1 GHz that TICR=10000000 implied
  }
}

void
lapicinit(void)
{
  if(!lapic)
    return;

  // Enable local APIC; set spurious interrupt vector.
  lapicw(SVR, ENABLE | (T_IRQ0 + IRQ_SPURIOUS));

  // The timer counts down at the APIC bus frequency and interrupts at zero.
  // What that frequency *is*, calibrate_timers() had to go and measure --
  // see the comment above it for why loading a constant here does not work.
  if (lapic_timer_hz == 0) {
    calibrate_timers();
    cprintf("cpu0: lapic timer %u Hz, tsc %u MHz, tick = %u counts at %d Hz\n",
            lapic_timer_hz, (uint)(tsc_hz / 1000000),
            lapic_timer_hz / HZ, HZ);
  }

  lapicw(TDCR, X1);
  lapicw(TIMER, PERIODIC | (T_IRQ0 + IRQ_TIMER));
  lapicw(TICR, lapic_timer_hz / HZ);

  // Disable logical interrupt lines.
  lapicw(LINT0, MASKED);
  lapicw(LINT1, MASKED);

  // Disable performance counter overflow interrupts
  // on machines that provide that interrupt entry.
  if(((lapic[VER]>>16) & 0xFF) >= 4)
    lapicw(PCINT, MASKED);

  // Map error interrupt to IRQ_ERROR.
  lapicw(ERROR, T_IRQ0 + IRQ_ERROR);

  // Clear error status register (requires back-to-back writes).
  lapicw(ESR, 0);
  lapicw(ESR, 0);

  // Ack any outstanding interrupts.
  lapicw(EOI, 0);

  // Send an Init Level De-Assert to synchronise arbitration ID's.
  lapicw(ICRHI, 0);
  lapicw(ICRLO, BCAST | SINIT | LEVEL);
  while(lapic[ICRLO] & DELIVS)
    ;

  // Enable interrupts on the APIC (but not on the processor).
  lapicw(TPR, 0);
}

int
cpunum(void)
{
  int apicid, i;

  // Cannot call cpu when interrupts are enabled:
  // result not guaranteed to last long enough to be used!
  // Would prefer to panic but even printing is chancy here:
  // almost everything, including cprintf and panic, calls cpu,
  // often indirectly through acquire and release.
  if(readeflags()&FL_IF){
    static int n;
    if(n++ == 0)
      cprintf("cpu called from %x with interrupts enabled\n",
        __builtin_return_address(0));
  }

  if (!lapic)
    return 0;

  apicid = lapic[ID] >> 24;
  for (i = 0; i < ncpu; ++i) {
    if (cpus[i].apicid == apicid)
      return i;
  }
  panic("unknown apicid\n");
}

// Acknowledge interrupt.
void
lapiceoi(void)
{
  if(lapic)
    lapicw(EOI, 0);
}

// Spin for a given number of microseconds.
//
// This was an empty function body -- xv6's stub, carrying a comment saying
// real hardware would want to tune it dynamically. Nothing ever did, so every
// microdelay() in the tree returned immediately: the xHCI reset sequence, the
// USB hub power-on wait, the UART drain. Those delays are written into the
// specifications because the hardware genuinely needs the time, and a port
// that "works anyway" is one that is racing and winning.
//
// The TSC is the right clock for this: calibrate_timers() measured it in the
// same pass as the LAPIC, and unlike the LAPIC timer it is not already busy
// driving the tick.
void
microdelay(int us)
{
  if (us <= 0)
    return;

  // Before calibration (early boot, and the AP trampoline) there is no
  // measured frequency to work from. Spin on a rough count instead: the
  // figure is wrong, but being approximately late is the entire point of the
  // call, and being instantly early is what broke it.
  if (tsc_hz == 0) {
    for (volatile int i = 0; i < us * 100; i++)
      ;
    return;
  }

  uint64 end = rdtsc() + ((uint64)us * tsc_hz) / 1000000;
  while (rdtsc() < end)
    asm volatile("pause");
}

#define CMOS_PORT    0x70
#define CMOS_RETURN  0x71

// Start additional processor running entry code at addr.
// See Appendix B of MultiProcessor Specification.
void
lapicstartap(uchar apicid, uint addr)
{
  int i;
  ushort *wrv;

  // "The BSP must initialize CMOS shutdown code to 0AH
  // and the warm reset vector (DWORD based at 40:67) to point at
  // the AP startup code prior to the [universal startup algorithm]."
  outb(CMOS_PORT, 0xF);  // offset 0xF is shutdown code
  outb(CMOS_PORT+1, 0x0A);
  wrv = (ushort*)P2V((0x40<<4 | 0x67));  // Warm reset vector
  wrv[0] = 0;
  wrv[1] = addr >> 4;

  // "Universal startup algorithm."
  // Send INIT (level-triggered) interrupt to reset other CPU.
  lapicw(ICRHI, apicid<<24);
  lapicw(ICRLO, SINIT | LEVEL | ASSERT);
  microdelay(200);
  lapicw(ICRLO, SINIT | LEVEL);
  microdelay(100);    // should be 10ms, but too slow in Bochs!

  // Send startup IPI (twice!) to enter code.
  // Regular hardware is supposed to only accept a STARTUP
  // when it is in the halted state due to an INIT.  So the second
  // should be ignored, but it is part of the official Intel algorithm.
  // Bochs complains about the second one.  Too bad for Bochs.
  for(i = 0; i < 2; i++){
    lapicw(ICRHI, apicid<<24);
    lapicw(ICRLO, STARTUP | (addr>>12));
    microdelay(200);
  }
}

#define CMOS_STATA   0x0a
#define CMOS_STATB   0x0b
#define CMOS_UIP    (1 << 7)        // RTC update in progress

#define SECS    0x00
#define MINS    0x02
#define HOURS   0x04
#define DAY     0x07
#define MONTH   0x08
#define YEAR    0x09

static uint cmos_read(uint reg)
{
  outb(CMOS_PORT,  reg);
  microdelay(200);

  return inb(CMOS_RETURN);
}

static void fill_rtcdate(struct rtcdate *r)
{
  r->second = cmos_read(SECS);
  r->minute = cmos_read(MINS);
  r->hour   = cmos_read(HOURS);
  r->day    = cmos_read(DAY);
  r->month  = cmos_read(MONTH);
  r->year   = cmos_read(YEAR);
}
//PAGEBREAK!

// qemu seems to use 24-hour GWT and the values are BCD encoded
void cmostime(struct rtcdate *r)
{
  struct rtcdate t1, t2;
  int sb, bcd;

  sb = cmos_read(CMOS_STATB);

  bcd = (sb & (1 << 2)) == 0;

  // make sure CMOS doesn't modify time while we read it
  for(;;) {
    fill_rtcdate(&t1);
    if(cmos_read(CMOS_STATA) & CMOS_UIP)
        continue;
    fill_rtcdate(&t2);
    if(memcmp(&t1, &t2, sizeof(t1)) == 0)
      break;
  }

  // convert
  if(bcd) {
#define    CONV(x)     (t1.x = ((t1.x >> 4) * 10) + (t1.x & 0xf))
    CONV(second);
    CONV(minute);
    CONV(hour  );
    CONV(day   );
    CONV(month );
    CONV(year  );
#undef     CONV
  }

  *r = t1;
  r->year += 2000;
}