irq: MSI first, and poll the INTx lines that never arrive
PCI drivers took their interrupt from INTERRUPT_LINE. On real hardware that is the 8259's number, not the IOAPIC pin the device is wired to (that is in ACPI _PRT, which needs AML), so nothing ever arrived. xHCI is entirely interrupt-driven, so the USB keyboard and a USB root would have hung even on a clean boot. Under QEMU the two numbers agree, which is why it never showed. - pci_irq_attach()/pci_irq_detach(): MSI, then MSI-X if the driver allows it, then INTx. The handler is registered before MSI is enabled, so no edge is lost. xHCI, e1000, r8169 and rtl8139 use it. - Lines 24-29 are MSI vectors. The LAPIC error vector moves from 19, which is a PCI pin on Intel chipsets, to 30. Nothing is placed at or above vector 64 (T_SYSCALL). - A storming line is masked and its handlers are polled instead of being left to eat the CPU. A level line that has had a handler for 2 s and never fired is polled every tick by a kernel thread. Tested in QEMU q35 with the root on a USB stick: xHCI and e1000e on MSI; then MSI disabled and INTx deliberately misrouted, where xHCI got zero interrupts and the root still mounted through the poller. i440fx INTx and the rtl8139 hand-over are unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HH7xukt4aaiGj1nQEAoQf1
This commit is contained in:
Nexoniarz
parent
94c936ed98
commit
7e8d5e2821
11 changed files with 407 additions and 35 deletions
152
kernel/dev/irq.c
152
kernel/dev/irq.c
|
|
@ -20,9 +20,15 @@ static struct {
|
|||
int n; // handlers installed; only ever grows during boot
|
||||
int enabled;
|
||||
int level_low;
|
||||
int msi; // an MSI vector, not an IOAPIC pin: nothing to route
|
||||
int polled; // handlers run from irq_poll_thread as well
|
||||
uint64 delivered; // real interrupts, not polls
|
||||
uint64 since; // ticks when the first handler was attached
|
||||
uint64 unhandled;
|
||||
} line[NIRQ];
|
||||
|
||||
static int msi_used[NIRQ];
|
||||
|
||||
// Guards registration against itself. The *dispatch* path deliberately does
|
||||
// not take it: an interrupt handler that blocked on a lock held by whoever
|
||||
// was registering would deadlock the CPU it fired on. Instead registration
|
||||
|
|
@ -44,10 +50,10 @@ irq_init(void)
|
|||
}
|
||||
|
||||
// IRQ_ERROR and IRQ_SPURIOUS are not IOAPIC pins -- they are local APIC
|
||||
// vectors that xv6 numbered as though they were, so T_IRQ0+19 and T_IRQ0+31
|
||||
// are already spoken for. A PCI device routed to pin 19 would deliver on the
|
||||
// same vector as the LAPIC error interrupt. Refuse it here rather than let it
|
||||
// be debugged later.
|
||||
// vectors that xv6 numbered as though they were. The error vector used to be
|
||||
// 19, which *is* a pin: one of the four an Intel chipset wires its PCI
|
||||
// interrupts to, so a device there would have been refused. It is 30 now,
|
||||
// above every pin; see include/traps.h.
|
||||
static int
|
||||
reserved(int irq)
|
||||
{
|
||||
|
|
@ -91,10 +97,13 @@ irq_register(int irq, irq_handler_t fn, void *arg, const char *name,
|
|||
a->count = 0;
|
||||
__sync_synchronize(); // publish the entry before it is reachable
|
||||
line[irq].n++;
|
||||
if(line[irq].n == 1)
|
||||
line[irq].since = ticks;
|
||||
|
||||
if(!line[irq].enabled){
|
||||
line[irq].level_low = level_low;
|
||||
ioapicroute(irq, 0, level_low);
|
||||
if(!line[irq].msi)
|
||||
ioapicroute(irq, 0, level_low);
|
||||
line[irq].enabled = 1;
|
||||
}
|
||||
|
||||
|
|
@ -117,7 +126,11 @@ irq_unregister(int irq, irq_handler_t fn, void *arg)
|
|||
// last handler. Compacting the array moves entries that a dispatcher on
|
||||
// another CPU may be walking, so the line has to be quiet for the
|
||||
// duration, and then brought back only if anything is left on it.
|
||||
ioapicmask(irq);
|
||||
//
|
||||
// An MSI vector has no pin to mask. Its caller, pci_irq_detach(), has
|
||||
// already switched the device's MSI off, which is the same thing.
|
||||
if(!line[irq].msi)
|
||||
ioapicmask(irq);
|
||||
line[irq].enabled = 0;
|
||||
|
||||
for(j = i; j + 1 < line[irq].n; j++)
|
||||
|
|
@ -126,8 +139,12 @@ irq_unregister(int irq, irq_handler_t fn, void *arg)
|
|||
__sync_synchronize();
|
||||
|
||||
if(line[irq].n > 0){
|
||||
ioapicroute(irq, 0, line[irq].level_low);
|
||||
if(!line[irq].msi && !line[irq].polled)
|
||||
ioapicroute(irq, 0, line[irq].level_low);
|
||||
line[irq].enabled = 1;
|
||||
} else {
|
||||
line[irq].polled = 0;
|
||||
line[irq].delivered = 0;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
|
@ -189,6 +206,17 @@ storm_check(int irq)
|
|||
cprintf("irq %d: handler '%s' is attached\n", irq, line[irq].act[i].name);
|
||||
if(line[irq].n == 0)
|
||||
cprintf("irq %d: no handler is attached at all\n", irq);
|
||||
|
||||
// And stop it. Reporting a storm and carrying on is a machine that spends
|
||||
// every cycle in this handler -- which is what a PCI line configured on the
|
||||
// wrong pin does on real hardware, where an idle ISA input read active-low
|
||||
// is asserted for ever. The handlers move to the poller, so a device that
|
||||
// was really on this line still works, ten milliseconds at a time.
|
||||
if(irq < NIRQ_IOAPIC && !line[irq].msi){
|
||||
ioapicmask(irq);
|
||||
line[irq].polled = 1;
|
||||
cprintf("irq %d: masked; its handlers are polled from now on\n", irq);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
|
|
@ -206,6 +234,8 @@ irq_dispatch(int irq)
|
|||
return;
|
||||
}
|
||||
|
||||
line[irq].delivered++;
|
||||
|
||||
// Every handler on a shared line is offered the interrupt; each one is
|
||||
// responsible for recognising that its device is idle and returning.
|
||||
for(i = 0; i < n; i++){
|
||||
|
|
@ -267,7 +297,10 @@ irq_info(char *buf, int cap)
|
|||
off = emit(buf, cap, off, "(unhandled)");
|
||||
if(line[irq].unhandled)
|
||||
off = emit(buf, cap, off, " [spurious]");
|
||||
off = emit(buf, cap, off, line[irq].level_low ? " level\n" : " edge\n");
|
||||
if(line[irq].polled)
|
||||
off = emit(buf, cap, off, " [polled]");
|
||||
off = emit(buf, cap, off, line[irq].msi ? " msi\n" :
|
||||
line[irq].level_low ? " level\n" : " edge\n");
|
||||
}
|
||||
|
||||
// How many timer interrupts were lost.
|
||||
|
|
@ -314,3 +347,106 @@ irq_handler_count(int irq)
|
|||
return 0;
|
||||
return line[irq].n;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MSI vectors
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// A line that is an MSI vector rather than a pin. They are handed out from
|
||||
// IRQ_MSI_FIRST..IRQ_MSI_LAST, which are vectors the IOAPIC never uses; see
|
||||
// pci_irq_attach() in kernel/dev/pci.c, which is the only caller.
|
||||
int
|
||||
irq_alloc_msi(void)
|
||||
{
|
||||
acquire(&irqlock);
|
||||
for(int irq = IRQ_MSI_FIRST; irq <= IRQ_MSI_LAST; irq++){
|
||||
if(!msi_used[irq] && !reserved(irq)){
|
||||
msi_used[irq] = 1;
|
||||
line[irq].msi = 1;
|
||||
release(&irqlock);
|
||||
return irq;
|
||||
}
|
||||
}
|
||||
release(&irqlock);
|
||||
return -1;
|
||||
}
|
||||
|
||||
void
|
||||
irq_free_msi(int irq)
|
||||
{
|
||||
if(irq < IRQ_MSI_FIRST || irq > IRQ_MSI_LAST)
|
||||
return;
|
||||
acquire(&irqlock);
|
||||
if(line[irq].n == 0){
|
||||
msi_used[irq] = 0;
|
||||
line[irq].msi = 0;
|
||||
}
|
||||
release(&irqlock);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Polling the lines that do not interrupt
|
||||
// ---------------------------------------------------------------------------
|
||||
//
|
||||
// A PCI driver that cannot have MSI learns its line from the device's
|
||||
// INTERRUPT_LINE register, and on real hardware that is the wrong answer.
|
||||
// The register holds the number firmware gave the *8259* -- 10, 11 -- and in
|
||||
// APIC mode the device is wired to an IOAPIC input the firmware only
|
||||
// describes in ACPI's _PRT, which takes an AML interpreter to read. So the
|
||||
// pin gets unmasked, the device's real pin stays masked, and nothing arrives.
|
||||
// Under QEMU the two numbers happen to agree, which is why it never showed.
|
||||
//
|
||||
// Until there is an AML interpreter, a level-triggered line that has had a
|
||||
// handler for two seconds and has not once fired is presumed to be that,
|
||||
// and its handlers are called from here every tick. Every handler is already
|
||||
// written to be called when its device has nothing to say -- a shared line
|
||||
// needs that -- so being called spuriously costs a register read. A storm
|
||||
// does the same from the other side; see storm_check().
|
||||
//
|
||||
// The moment a real interrupt arrives the line stops being polled, unless it
|
||||
// was put here for storming.
|
||||
#define POLL_AFTER (2 * HZ)
|
||||
|
||||
static void
|
||||
irq_poll_thread(void)
|
||||
{
|
||||
static uchar said[NIRQ];
|
||||
|
||||
for(;;){
|
||||
acquire(&tickslock);
|
||||
uint start = ticks;
|
||||
while(ticks == start)
|
||||
sleep(&ticks, &tickslock);
|
||||
uint now = ticks;
|
||||
release(&tickslock);
|
||||
|
||||
for(int irq = 0; irq < NIRQ_IOAPIC; irq++){
|
||||
if(line[irq].n == 0 || line[irq].msi || reserved(irq))
|
||||
continue;
|
||||
int poll = line[irq].polled ||
|
||||
(line[irq].level_low && line[irq].delivered == 0 &&
|
||||
now - line[irq].since > POLL_AFTER);
|
||||
if(!poll)
|
||||
continue;
|
||||
if(!said[irq] && !line[irq].polled){
|
||||
said[irq] = 1;
|
||||
kinfo("irq %d: nothing has arrived on it; polling its handlers\n",
|
||||
irq);
|
||||
}
|
||||
// As an interrupt would: on this CPU, with interrupts off. n is read
|
||||
// once, as irq_dispatch() reads it.
|
||||
pushcli();
|
||||
int n = line[irq].n;
|
||||
for(int i = 0; i < n; i++)
|
||||
line[irq].act[i].fn(line[irq].act[i].arg);
|
||||
popcli();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
irq_poll_start(void)
|
||||
{
|
||||
if(kthread_create(irq_poll_thread, "irqpoll") < 0)
|
||||
cprintf("irq: cannot start the poller; misrouted PCI lines stay silent\n");
|
||||
}
|
||||
|
|
|
|||
133
kernel/dev/pci.c
133
kernel/dev/pci.c
|
|
@ -24,6 +24,10 @@
|
|||
#include "sbuf.h"
|
||||
#include "dbg.h"
|
||||
#include "device.h"
|
||||
#include "traps.h"
|
||||
#include "irq.h"
|
||||
#include "memlayout.h"
|
||||
#include "proc.h"
|
||||
|
||||
static struct pci_dev devs[NPCI];
|
||||
static int ndevs;
|
||||
|
|
@ -178,6 +182,135 @@ pci_msi_enable(struct pci_dev *d, int vector, int cpu)
|
|||
return 0;
|
||||
}
|
||||
|
||||
// The same for MSI-X, entry 0 only.
|
||||
//
|
||||
// MSI-X keeps its address/data pairs in a table inside one of the device's
|
||||
// own BARs rather than in config space, which is the whole difference. Some
|
||||
// controllers -- several AMD xHCIs among them -- offer MSI-X and no MSI, and
|
||||
// without this they had only INTx, which on real hardware is the line that
|
||||
// does not arrive.
|
||||
static int
|
||||
pci_msix_enable(struct pci_dev *d, int vector, int apicid)
|
||||
{
|
||||
uint8 cap = pci_find_cap(d, PCI_CAP_MSIX);
|
||||
if(cap == 0)
|
||||
return -1;
|
||||
|
||||
uint32 tbl = pci_read32(d->bus, d->slot, d->func, cap + 4);
|
||||
int bir = tbl & 7;
|
||||
uint64 off = tbl & ~7u;
|
||||
if(bir > 5 || d->bar[bir].base == 0 || d->bar[bir].is_io)
|
||||
return -1;
|
||||
|
||||
volatile uint32 *t = ioremap(d->bar[bir].base + off, 16);
|
||||
if(t == 0)
|
||||
return -1;
|
||||
|
||||
// Enable with the whole function masked, program entry 0, unmask it, then
|
||||
// drop the function mask: at no point is an entry live with half an
|
||||
// address in it. The other entries come out of reset masked and stay so.
|
||||
uint32 dw = pci_read32(d->bus, d->slot, d->func, cap);
|
||||
pci_write32(d->bus, d->slot, d->func, cap, dw | (3u << 30)); // enable + fmask
|
||||
t[0] = 0xFEE00000u | ((uint32)apicid << 12);
|
||||
t[1] = 0;
|
||||
t[2] = (uint32)vector;
|
||||
t[3] = 0; // unmask
|
||||
dw = pci_read32(d->bus, d->slot, d->func, cap);
|
||||
pci_write32(d->bus, d->slot, d->func, cap, (dw | (1u << 31)) & ~(1u << 30));
|
||||
|
||||
pci_write32(d->bus, d->slot, d->func, PCI_COMMAND,
|
||||
pci_read32(d->bus, d->slot, d->func, PCI_COMMAND) | (1 << 10));
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Turn off whichever of MSI or MSI-X is on, and give the device back its INTx
|
||||
// -- the next driver to claim it may be one that only knows INTx.
|
||||
static void
|
||||
pci_msi_off(struct pci_dev *d)
|
||||
{
|
||||
uint8 cap;
|
||||
uint32 dw;
|
||||
|
||||
if((cap = pci_find_cap(d, PCI_CAP_MSI)) != 0){
|
||||
dw = pci_read32(d->bus, d->slot, d->func, cap);
|
||||
pci_write32(d->bus, d->slot, d->func, cap, dw & ~(1u << 16));
|
||||
}
|
||||
if((cap = pci_find_cap(d, PCI_CAP_MSIX)) != 0){
|
||||
dw = pci_read32(d->bus, d->slot, d->func, cap);
|
||||
pci_write32(d->bus, d->slot, d->func, cap, dw & ~(1u << 31));
|
||||
}
|
||||
pci_write32(d->bus, d->slot, d->func, PCI_COMMAND,
|
||||
pci_read32(d->bus, d->slot, d->func, PCI_COMMAND) & ~(1u << 10));
|
||||
}
|
||||
|
||||
int
|
||||
pci_irq_attach(struct pci_dev *d, irq_handler_t fn, void *arg,
|
||||
const char *name, int flags)
|
||||
{
|
||||
// The processor the interrupt goes to, by APIC ID: the boot processor's,
|
||||
// which is the one IOAPIC lines are routed to as well.
|
||||
int dest = cpus[0].apicid;
|
||||
int irq = irq_alloc_msi();
|
||||
|
||||
if(irq >= 0){
|
||||
// Handler first, then the device. An MSI is an edge: one that arrives
|
||||
// before anything is listening is gone, and an xHCI whose first event
|
||||
// interrupt is lost will not raise another until it is acknowledged.
|
||||
if(irq_register(irq, fn, arg, name, 0) == 0){
|
||||
int kind = PCI_IRQ_NONE;
|
||||
if(pci_msi_enable(d, T_IRQ0 + irq, dest) == 0)
|
||||
kind = PCI_IRQ_MSI;
|
||||
else if((flags & PCI_IRQF_MSIX) &&
|
||||
pci_msix_enable(d, T_IRQ0 + irq, dest) == 0)
|
||||
kind = PCI_IRQ_MSIX;
|
||||
if(kind != PCI_IRQ_NONE){
|
||||
d->irq_kind = kind;
|
||||
d->irq_line = irq;
|
||||
return irq;
|
||||
}
|
||||
irq_unregister(irq, fn, arg);
|
||||
}
|
||||
irq_free_msi(irq);
|
||||
}
|
||||
|
||||
if(d->irq == 0xFF || d->irq >= NIRQ_IOAPIC)
|
||||
return -1;
|
||||
if(irq_register(d->irq, fn, arg, name, 1) < 0)
|
||||
return -1;
|
||||
d->irq_kind = PCI_IRQ_INTX;
|
||||
d->irq_line = d->irq;
|
||||
return d->irq;
|
||||
}
|
||||
|
||||
void
|
||||
pci_irq_detach(struct pci_dev *d, irq_handler_t fn, void *arg)
|
||||
{
|
||||
switch(d->irq_kind){
|
||||
case PCI_IRQ_MSI:
|
||||
case PCI_IRQ_MSIX:
|
||||
// The device first, so nothing is written to a vector being taken away.
|
||||
pci_msi_off(d);
|
||||
irq_unregister(d->irq_line, fn, arg);
|
||||
irq_free_msi(d->irq_line);
|
||||
break;
|
||||
case PCI_IRQ_INTX:
|
||||
irq_unregister(d->irq_line, fn, arg);
|
||||
break;
|
||||
}
|
||||
d->irq_kind = PCI_IRQ_NONE;
|
||||
}
|
||||
|
||||
const char *
|
||||
pci_irq_kind(const struct pci_dev *d)
|
||||
{
|
||||
switch(d->irq_kind){
|
||||
case PCI_IRQ_MSI: return "msi";
|
||||
case PCI_IRQ_MSIX: return "msi-x";
|
||||
case PCI_IRQ_INTX: return "intx";
|
||||
default: return "none";
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Base address registers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Reference in a new issue