SchOS/kernel/dev/irq.c

// Hardware interrupt registration. See include/core/irq.h for why this exists.

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "arch/traps.h"
#include "core/spinlock.h"
#include "core/irq.h"
#include "arch/x86.h"
#include "dev/acpi.h"

struct irq_action {
  irq_handler_t fn;
  void         *arg;
  const char   *name;
  uint64        count;
  int           quiet;          // registered IRQ_QUIET
};

static struct {
  struct irq_action act[NCHAIN];
  int  n;               // handlers installed; only ever grows during boot
  int  enabled;
  int  level_low;
  int  msi;             // an MSI vector, not an IOAPIC pin: nothing to route
  int  polled;          // handlers run from irq_poll_thread as well
  int  nquiet;          // handlers registered IRQ_QUIET
  uint64 delivered;     // real interrupts, not polls
  uint64 since;         // ticks when the first handler was attached
  uint64 unhandled;
  int  (*waiting)(void);  // see irq_watch()
  uint64 seen;          // `delivered` when the poller last looked
  int  stuck;           // ticks the device has waited with nothing delivered
  int  flipped;         // polarity already turned over once
} line[NIRQ];

static int msi_used[NIRQ];

// Guards registration against itself. The *dispatch* path deliberately does
// not take it: an interrupt handler that blocked on a lock held by whoever
// was registering would deadlock the CPU it fired on. Instead registration
// publishes an entry by writing all of its fields, issuing a barrier, and
// only then incrementing n -- so a concurrent dispatcher either does not see
// the entry at all or sees it complete.
//
// Removal is the harder direction, because compacting the array *does* move
// entries a dispatcher may be walking. irq_unregister therefore masks the
// line first and only re-routes it once the chain has been rewritten: with
// the line masked no new interrupt can arrive, and any handler already
// running holds no reference past its own return.
static struct spinlock irqlock;

void
irq_init(void)
{
  initlock(&irqlock, "irq");
}

// IRQ_ERROR and IRQ_SPURIOUS are not IOAPIC pins -- they are local APIC
// vectors that xv6 numbered as though they were. The error vector used to be
// 19, which *is* a pin: one of the four an Intel chipset wires its PCI
// interrupts to, so a device there would have been refused. It is 30 now,
// above every pin; see include/arch/traps.h.
static int
reserved(int irq)
{
  return irq == IRQ_ERROR || irq == IRQ_SPURIOUS || irq == IRQ_TIMER;
}

int
irq_register(int irq, irq_handler_t fn, void *arg, const char *name,
             int level_low)
{
  struct irq_action *a;
  int quiet = (level_low & IRQ_QUIET) != 0;

  level_low &= ~IRQ_QUIET;
  if(irq < 0 || irq >= NIRQ || fn == 0)
    return -1;

  // An ISA line, as the MADT describes it. The override entries were parsed
  // and never read, so a line the firmware says is active low or
  // level-triggered was programmed edge, active high, regardless. Only the
  // mode: a line moved to another pin would need the pin numbering here to
  // stop being the IRQ numbering, and no machine seen so far does that for
  // anything but the timer.
  if(irq < 16 && level_low == IRQ_EDGE_HIGH){
    int mode;
    if(acpi_isa_gsi(irq, IRQ_EDGE_HIGH, &mode) == irq)
      level_low = mode;
  }
  if(reserved(irq)){
    cprintf("irq_register: %s wants irq %d, which is reserved\n", name, irq);
    return -1;
  }

  acquire(&irqlock);

  if(line[irq].n >= NCHAIN){
    release(&irqlock);
    cprintf("irq_register: irq %d chain full, %s not attached\n", irq, name);
    return -1;
  }

  // Two devices sharing a line must agree on how it is signalled; they
  // physically cannot be wired both ways.
  if(line[irq].n > 0 && line[irq].level_low != level_low){
    release(&irqlock);
    cprintf("irq_register: irq %d shared with conflicting trigger mode "
            "(%s)\n", irq, name);
    return -1;
  }

  a = &line[irq].act[line[irq].n];
  a->fn    = fn;
  a->arg   = arg;
  a->name  = name;
  a->count = 0;
  a->quiet = quiet;
  __sync_synchronize();         // publish the entry before it is reachable
  line[irq].n++;
  line[irq].nquiet += quiet;
  if(line[irq].n == 1)
    line[irq].since = ticks;

  if(!line[irq].enabled){
    line[irq].level_low = level_low;
    if(!line[irq].msi)
      ioapicroute(irq, 0, level_low);
    line[irq].enabled = 1;
  }

  release(&irqlock);
  return 0;
}

void
irq_unregister(int irq, irq_handler_t fn, void *arg)
{
  int i, j;

  if(irq < 0 || irq >= NIRQ)
    return;

  acquire(&irqlock);
  for(i = 0; i < line[irq].n; i++){
    if(line[irq].act[i].fn == fn && line[irq].act[i].arg == arg){
      // Mask before touching the chain, always -- not only when this is the
      // last handler. Compacting the array moves entries that a dispatcher on
      // another CPU may be walking, so the line has to be quiet for the
      // duration, and then brought back only if anything is left on it.
      //
      // An MSI vector has no pin to mask. Its caller, pci_irq_detach(), has
      // already switched the device's MSI off, which is the same thing.
      if(!line[irq].msi)
        ioapicmask(irq);
      line[irq].enabled = 0;

      line[irq].nquiet -= line[irq].act[i].quiet;
      for(j = i; j + 1 < line[irq].n; j++)
        line[irq].act[j] = line[irq].act[j+1];
      line[irq].n--;
      __sync_synchronize();

      if(line[irq].n > 0){
        if(!line[irq].msi && !line[irq].polled)
          ioapicroute(irq, 0, line[irq].level_low);
        line[irq].enabled = 1;
      } else {
        line[irq].polled = 0;
        line[irq].delivered = 0;
        line[irq].waiting = 0;
        line[irq].stuck = line[irq].flipped = 0;
      }
      break;
    }
  }
  release(&irqlock);
}

// Notice a line that is firing far faster than any real device would, and say
// so once.
//
// A level-triggered PCI interrupt that nobody acknowledges re-fires the instant
// EOI is signalled, so the line runs at whatever rate the CPU can take traps --
// tens of thousands per second. That starves everything else on the machine,
// and it does so silently: the symptom appears wherever the missing CPU time
// hurts first, which is nowhere near the device at fault.
//
// It cost a long afternoon to find one of these. virtio-gpu shared IRQ 11 with
// the e1000 and registered no handler, so the line storm at ~72 kHz and took
// 83% of CPU0. What that looked like from userspace was a clock running seven
// times slow -- uptime, `time`, and ping's interval all wrong together, which
// reads as a timekeeping bug and sends you into the timer code. The timer was
// fine the whole time. This is the message that would have said so.
//
// The rate is measured against the TSC, deliberately: `ticks` is the first
// casualty of a storm, so a storm detector that counted ticks would go quiet
// exactly when it was needed.
#define STORM_SAMPLE 4096          // dispatches between rate checks
#define STORM_HZ     20000         // per second; no real device sustains this

static void
storm_check(int irq)
{
  static uint64 sample_tsc[NIRQ];
  static uint   sample_n[NIRQ];
  static uchar  reported[NIRQ];

  if(reported[irq] || !tsc_hz)
    return;
  if(++sample_n[irq] < STORM_SAMPLE)
    return;

  uint64 now = rdtsc(), prev = sample_tsc[irq];
  sample_n[irq] = 0;
  sample_tsc[irq] = now;
  if(!prev)
    return;                        // first sample: no interval to divide by

  uint64 elapsed = now - prev;
  if(elapsed == 0)
    return;
  uint64 rate = (uint64)STORM_SAMPLE * tsc_hz / elapsed;
  if(rate < STORM_HZ)
    return;

  reported[irq] = 1;
  cprintf("irq %d: %d interrupts/second -- a device on this line is not "
          "acknowledging it.\n", irq, (int)rate);
  for(int i = 0; i < line[irq].n; i++)
    cprintf("irq %d:   handler '%s' is attached\n", irq, line[irq].act[i].name);
  if(line[irq].n == 0)
    cprintf("irq %d:   no handler is attached at all\n", irq);

  // And stop it. Reporting a storm and carrying on is a machine that spends
  // every cycle in this handler -- which is what a PCI line configured on the
  // wrong pin does on real hardware, where an idle ISA input read active-low
  // is asserted for ever. The handlers move to the poller, so a device that
  // was really on this line still works, ten milliseconds at a time.
  if(irq < NIRQ_IOAPIC && !line[irq].msi){
    ioapicmask(irq);
    line[irq].polled = 1;
    cprintf("irq %d: masked; its handlers are polled from now on\n", irq);
  }
}

void
irq_dispatch(int irq)
{
  int i, n;

  if(irq < 0 || irq >= NIRQ)
    return;

  n = line[irq].n;
  if(n == 0){
    line[irq].unhandled++;
    storm_check(irq);
    return;
  }

  line[irq].delivered++;

  // Every handler on a shared line is offered the interrupt; each one is
  // responsible for recognising that its device is idle and returning.
  for(i = 0; i < n; i++){
    line[irq].act[i].count++;
    line[irq].act[i].fn(line[irq].act[i].arg);
  }
  storm_check(irq);
}

// ---------------------------------------------------------------------------
// /proc/interrupts
// ---------------------------------------------------------------------------

static int
emit(char *buf, int cap, int off, const char *s)
{
  while(*s && off < cap)
    buf[off++] = *s++;
  return off;
}

static int
emit_dec(char *buf, int cap, int off, uint64 v, int width)
{
  char tmp[24];
  int n = 0, i;

  if(v == 0)
    tmp[n++] = '0';
  while(v > 0){
    tmp[n++] = '0' + (v % 10);
    v /= 10;
  }
  for(i = n; i < width && off < cap; i++)
    buf[off++] = ' ';
  while(n > 0 && off < cap)
    buf[off++] = tmp[--n];
  return off;
}

int
irq_info(char *buf, int cap)
{
  int irq, i, off = 0;

  for(irq = 0; irq < NIRQ; irq++){
    if(line[irq].n == 0 && line[irq].unhandled == 0)
      continue;
    off = emit_dec(buf, cap, off, irq, 4);
    off = emit(buf, cap, off, ": ");
    for(i = 0; i < line[irq].n; i++){
      off = emit_dec(buf, cap, off, line[irq].act[i].count, 10);
      off = emit(buf, cap, off, "  ");
      off = emit(buf, cap, off,
                 line[irq].act[i].name ? line[irq].act[i].name : "?");
      off = emit(buf, cap, off, i + 1 < line[irq].n ? ", " : "");
    }
    if(line[irq].n == 0)
      off = emit(buf, cap, off, "(unhandled)");
    if(line[irq].unhandled)
      off = emit(buf, cap, off, " [spurious]");
    if(line[irq].polled)
      off = emit(buf, cap, off, " [polled]");
    off = emit(buf, cap, off, line[irq].msi ? " msi\n" :
                              IRQ_IS_LEVEL(line[irq].level_low) ? " level\n" : " edge\n");
  }

  // How many timer interrupts were lost.
  //
  // The clock does not depend on this any more -- kernel/core/time.c reads the
  // TSC -- but that makes lost ticks *invisible* rather than harmless, and
  // they are not harmless: the scheduler preempts on them, and a machine
  // dropping them is a machine spending too long with interrupts disabled.
  //
  // Twice now that has been diagnosed from the outside, by noticing a clock
  // running slow, which is a terrible way to find out. This is the number to
  // look at instead: expected is elapsed time times HZ, and the gap is how
  // many the machine did not take.
  {
    uint64 sec, usec;
    monotonic(&sec, &usec);
    uint64 expect = sec * HZ + (usec * HZ) / 1000000;
    uint64 got    = ticks;
    off = emit(buf, cap, off, "\ntimer: ");
    off = emit_dec(buf, cap, off, got, 0);
    off = emit(buf, cap, off, " ticks taken, ");
    off = emit_dec(buf, cap, off, expect, 0);
    off = emit(buf, cap, off, " expected");
    if (expect > got + HZ) {
      off = emit(buf, cap, off, " -- LOSING ");
      off = emit_dec(buf, cap, off, expect - got, 0);
      off = emit(buf, cap, off, ", interrupts are off too long somewhere");
    }
    off = emit(buf, cap, off, "\n");
  }
  return off;
}

void
irq_watch(int irq, int (*waiting)(void))
{
  if(irq < 0 || irq >= NIRQ_IOAPIC)
    return;
  acquire(&irqlock);
  line[irq].seen = line[irq].delivered;
  line[irq].stuck = 0;
  line[irq].waiting = waiting;
  release(&irqlock);
}

// How many handlers are attached to a line.
//
// Asked by kernel/dev/pciuser.c before it gives a line to a userspace driver:
// that driver masks the line while it works, and a masked line is masked for
// everyone on it. Handing out a shared line would silence whichever kernel
// driver is also there, with no message and no obvious cause.
int
irq_handler_count(int irq)
{
  if (irq < 0 || irq >= NIRQ)
    return 0;
  return line[irq].n;
}

// ---------------------------------------------------------------------------
// MSI vectors
// ---------------------------------------------------------------------------

// A line that is an MSI vector rather than a pin. They are handed out from
// IRQ_MSI_FIRST..IRQ_MSI_LAST, which are vectors the IOAPIC never uses; see
// pci_irq_attach() in kernel/dev/pci.c, which is the only caller.
int
irq_alloc_msi(void)
{
  acquire(&irqlock);
  for(int irq = IRQ_MSI_FIRST; irq <= IRQ_MSI_LAST; irq++){
    if(!msi_used[irq] && !reserved(irq)){
      msi_used[irq] = 1;
      line[irq].msi = 1;
      release(&irqlock);
      return irq;
    }
  }
  release(&irqlock);
  return -1;
}

void
irq_free_msi(int irq)
{
  if(irq < IRQ_MSI_FIRST || irq > IRQ_MSI_LAST)
    return;
  acquire(&irqlock);
  if(line[irq].n == 0){
    msi_used[irq] = 0;
    line[irq].msi = 0;
  }
  release(&irqlock);
}

// ---------------------------------------------------------------------------
// Polling the lines that do not interrupt
// ---------------------------------------------------------------------------
//
// A PCI driver that cannot have MSI learns its line from the device's
// INTERRUPT_LINE register, and on real hardware that is the wrong answer.
// The register holds the number firmware gave the *8259* -- 10, 11 -- and in
// APIC mode the device is wired to an IOAPIC input the firmware only
// describes in ACPI's _PRT, which takes an AML interpreter to read. So the
// pin gets unmasked, the device's real pin stays masked, and nothing arrives.
// Under QEMU the two numbers happen to agree, which is why it never showed.
//
// Until there is an AML interpreter, a level-triggered line that has had a
// handler for two seconds and has not once fired is presumed to be that,
// and its handlers are called from here every tick. Every handler is already
// written to be called when its device has nothing to say -- a shared line
// needs that -- so being called spuriously costs a register read. A storm
// does the same from the other side; see storm_check().
//
// The moment a real interrupt arrives the line stops being polled, unless it
// was put here for storming.
//
// An *edge* line cannot be judged by silence -- a keyboard nobody types on is
// silent too -- so a driver that can tell whether its device has something
// waiting says so with irq_watch(), and the question becomes: has it waited
// with nothing delivered? That is a line programmed the wrong way up. The
// device asserts it, the IOAPIC is looking for the opposite edge, and the
// byte sits in the controller for ever, because only the interrupt that never
// comes would read it. An ASUS TUF A15's keyboard did exactly that -- the
// internal keyboard dead, a USB one fine -- and Linux keeps a list of AMD
// laptops whose firmware describes IRQ 1 wrongly one way or the other.
//
// So the polarity is turned over and the device drained, which re-arms the
// edge; if the line still says nothing, it is polled. Either way it says so.
#define POLL_AFTER (2 * HZ)
#define STUCK_TICKS 5                   // 50 ms waiting with nothing delivered

static void
watch_line(int irq)
{
  if(line[irq].polled || !line[irq].waiting)
    return;
  if(line[irq].delivered != line[irq].seen || !line[irq].waiting()){
    line[irq].seen = line[irq].delivered;
    line[irq].stuck = 0;
    return;
  }
  if(++line[irq].stuck < STUCK_TICKS)
    return;
  line[irq].stuck = 0;

  if(!line[irq].flipped){
    static const int other[4] = {
      [IRQ_EDGE_HIGH]  = IRQ_EDGE_LOW,  [IRQ_EDGE_LOW]  = IRQ_EDGE_HIGH,
      [IRQ_LEVEL_HIGH] = IRQ_LEVEL_LOW, [IRQ_LEVEL_LOW] = IRQ_LEVEL_HIGH,
    };
    int was = line[irq].level_low;
    line[irq].flipped = 1;
    line[irq].level_low = other[was & 3];
    ioapicroute(irq, 0, line[irq].level_low);
    cprintf("irq %d: '%s' had data waiting and no interrupt came; the line is "
            "active %s, not %s, whatever the firmware says\n", irq,
            line[irq].act[0].name,
            (line[irq].level_low == IRQ_EDGE_LOW ||
             line[irq].level_low == IRQ_LEVEL_LOW) ? "low" : "high",
            (was == IRQ_EDGE_LOW || was == IRQ_LEVEL_LOW) ? "low" : "high");
  } else {
    ioapicmask(irq);
    line[irq].polled = 1;
    cprintf("irq %d: '%s' still gets no interrupts either way up; polling "
            "it\n", irq, line[irq].act[0].name);
  }
}

static void
irq_poll_thread(void)
{
  static uchar said[NIRQ];

  for(;;){
    acquire(&tickslock);
    uint start = ticks;
    while(ticks == start)
      sleep(&ticks, &tickslock);
    uint now = ticks;
    release(&tickslock);

    for(int irq = 0; irq < NIRQ_IOAPIC; irq++){
      if(line[irq].n == 0 || line[irq].msi || reserved(irq))
        continue;
      // Silent by design, and only its own handlers on it.
      if(line[irq].nquiet == line[irq].n && !line[irq].polled)
        continue;

      int flipped = line[irq].flipped;
      watch_line(irq);
      if(line[irq].flipped != flipped){
        // Drain what was waiting: the new polarity sees the line already
        // asserted, and an edge needs it to go back first.
        pushcli();
        int n = line[irq].n;
        for(int i = 0; i < n; i++)
          line[irq].act[i].fn(line[irq].act[i].arg);
        popcli();
        continue;
      }

      int poll = line[irq].polled ||
                 (IRQ_IS_LEVEL(line[irq].level_low) && line[irq].delivered == 0 &&
                  now - line[irq].since > POLL_AFTER);
      if(!poll)
        continue;
      if(!said[irq] && !line[irq].polled){
        said[irq] = 1;
        kinfo("irq %d: nothing has arrived on it; polling its handlers\n",
              irq);
      }
      // As an interrupt would: on this CPU, with interrupts off. n is read
      // once, as irq_dispatch() reads it.
      pushcli();
      int n = line[irq].n;
      for(int i = 0; i < n; i++)
        line[irq].act[i].fn(line[irq].act[i].arg);
      popcli();
    }
  }
}

void
irq_poll_start(void)
{
  if(kthread_create(irq_poll_thread, "irqpoll") < 0)
    cprintf("irq: cannot start the poller; misrouted PCI lines stay silent\n");
}