diff --git a/include/defs.h b/include/defs.h index 11bde89..ec70dc2 100644 --- a/include/defs.h +++ b/include/defs.h @@ -437,6 +437,11 @@ void console_takeover(void); // vm.c void seginit(void); void kvmalloc(void); +// Map device registers. Returns a kernel address for `size` bytes of physical +// MMIO at `pa`, or 0 if the window is exhausted. Use this rather than P2V for +// anything that came out of a BAR: firmware may place a 64-bit BAR far above +// the 4 GB the direct map covers. +void* ioremap(uint64 pa, uint64 size); pml4e_t* setupkvm(void); char* uva2ka(pml4e_t*, char*); addr_t allocuvm(pml4e_t*, uint64, uint64); diff --git a/kernel/arch/x86_64/vm.c b/kernel/arch/x86_64/vm.c index de1171a..1798450 100644 --- a/kernel/arch/x86_64/vm.c +++ b/kernel/arch/x86_64/vm.c @@ -224,6 +224,75 @@ kvmalloc(void) switchkvm(); } +// Map a device's registers so the kernel can reach them. +// +// Every driver used to say P2V(bar.base), which quietly assumes the BAR lands +// inside the 0-4 GB the loop above maps. Under SeaBIOS it always does. Under +// OVMF it does not: firmware is free to put a 64-bit BAR anywhere, and it puts +// virtio-gpu's at about 768 GB. P2V then produces an address with no +// translation behind it, and the driver's first register write takes a page +// fault -- which, coming from a driver bind, looks like broken hardware rather +// than a missing mapping. +// +// The direct map uses PDPT entries 0-3 of a page directory pointer table that +// kpml4[0], kpml4[KERNBASE] and every process's PML4 (see setupkvm) all share. +// Entries 4-511 are free, so each one can carry a 1 GB page of device memory: +// no page table to allocate, nothing to free, and the mapping appears in every +// address space at once because they all point at this one table. +// +// Cache-disabled and write-through, for the same reason the top GB of the +// direct map is: these are registers, and a write that sits in a cache line +// waiting for company is a write the device never sees. +#define IOREMAP_FIRST_PDPT 4 +#define IOREMAP_LAST_PDPT 511 +#define GIGABYTE (1ULL << 30) + +static int ioremap_next = IOREMAP_FIRST_PDPT; + +void * +ioremap(uint64 pa, uint64 size) +{ + if (size == 0) + return 0; + + // Below 4 GB the direct map already covers it, so hand back the same address + // the old code would have produced. This is what keeps the BIOS path -- where + // every BAR is low -- byte for byte what it was. + if (pa + size <= 4 * GIGABYTE) + return P2V(pa); + + uint64 base = pa & ~(GIGABYTE - 1); // 1 GB pages, so align to one + uint64 last = (pa + size - 1) & ~(GIGABYTE - 1); + int span = (int)((last - base) / GIGABYTE) + 1; + + // Already mapped? A second BAR in the same gigabyte should not consume a + // second window, and drivers do re-map the same device. + for (int i = IOREMAP_FIRST_PDPT; i < ioremap_next; i++) { + // Clearing the low 30 bits drops every flag as well, leaving the frame. + if ((kpdpt[i] & PTE_P) && (kpdpt[i] & ~(GIGABYTE - 1)) == base) + return (void *)(KERNBASE + ((uint64)i << 30) + (pa - base)); + } + + if (ioremap_next + span > IOREMAP_LAST_PDPT) { + cprintf("ioremap: out of device window for %p\n", (void *)pa); + return 0; + } + + int first = ioremap_next; + for (int i = 0; i < span; i++) { + kpdpt[ioremap_next] = (base + (uint64)i * GIGABYTE) + | PTE_PS | PTE_P | PTE_W | PTE_PCD | PTE_PWT; + ioremap_next++; + } + + // The entries were not present a moment ago. Reloading CR3 is heavier than + // invlpg but this runs a handful of times at boot, and it cannot be the thing + // that is subtly wrong. + switchkvm(); + + return (void *)(KERNBASE + ((uint64)first << 30) + (pa - base)); +} + void switchuvm(struct proc *p) { diff --git a/kernel/drivers/bus/xhci/xhci.c b/kernel/drivers/bus/xhci/xhci.c index 40100bf..c54f6bd 100644 --- a/kernel/drivers/bus/xhci/xhci.c +++ b/kernel/drivers/bus/xhci/xhci.c @@ -1219,18 +1219,17 @@ xhci_probe(struct pci_dev *d) if(d->bar[0].base == 0) return -1; - // The kernel's identity map only marks 3-4 GB uncacheable, and only covers - // the first 4 GB at all. A BAR above that needs its own mapping, which does - // not exist yet -- refuse rather than read garbage from a stale mapping. - if(d->bar[0].base + d->bar[0].size > 0x100000000ULL){ - cprintf("xhci: BAR0 at 0x%p is above 4 GB and cannot be mapped yet\n", - d->bar[0].base); - return -1; - } - x = &xhcis[nxhci]; x->pci = d; - x->cap = (volatile uchar*)P2V(d->bar[0].base); + // A BAR above 4 GB used to be refused here: the direct map covers only the + // first 4 GB, so there was nowhere to put it. ioremap() gives it somewhere, + // and returns the direct-map address unchanged when the BAR is low -- which + // every BAR is under SeaBIOS. + x->cap = (volatile uchar*)ioremap(d->bar[0].base, d->bar[0].size); + if(x->cap == 0){ + cprintf("xhci: cannot map BAR0 at 0x%p\n", d->bar[0].base); + return -1; + } initlock(&x->lock, "xhci"); // Bus mastering is not optional: every structure below is read by the diff --git a/kernel/drivers/gpu/virtio_gpu/virtio_gpu.c b/kernel/drivers/gpu/virtio_gpu/virtio_gpu.c index 4fe1bf0..6727a06 100644 --- a/kernel/drivers/gpu/virtio_gpu/virtio_gpu.c +++ b/kernel/drivers/gpu/virtio_gpu/virtio_gpu.c @@ -111,7 +111,23 @@ find_cfg(struct pci_dev *d, int want, uint32 *out_mult) if (type == want && bar < 6 && d->bar[bar].base) { if (out_mult) // only the notify capability has this trailing field *out_mult = pci_read32(d->bus, d->slot, d->func, off + 16); - return (volatile uint8 *)P2V(d->bar[bar].base + o); + + // ioremap, not P2V. OVMF puts this device's 64-bit BAR at about 768 GB, + // far outside the 4 GB the direct map covers, and P2V there produces an + // address with nothing behind it -- the first register write took a + // page fault, then a double fault, then a triple fault and a reboot + // loop. SeaBIOS places the BAR low, which is why the BIOS path never + // showed it. + // + // Map at least as far as the capability's own offset reaches: the BAR + // size is what the device reports, and a mapping shorter than the + // offset we are about to add would fault just as mysteriously. + uint64 need = (uint64)o + PGSIZE; + uint64 len = d->bar[bar].size > need ? d->bar[bar].size : need; + volatile uint8 *va = (volatile uint8 *)ioremap(d->bar[bar].base, len); + if (!va) + return 0; + return va + o; } } off = next; diff --git a/kernel/drivers/net/e1000/e1000.c b/kernel/drivers/net/e1000/e1000.c index 5350ae6..b150764 100644 --- a/kernel/drivers/net/e1000/e1000.c +++ b/kernel/drivers/net/e1000/e1000.c @@ -264,7 +264,12 @@ e1000_probe(struct pci_dev *d) initlock(&e->lock, "e1000"); pci_enable(d, 0x0006); // memory space + bus master - e->regs = (volatile uint8 *)P2V(d->bar[0].base); + // ioremap rather than P2V: firmware may place a 64-bit BAR above the 4 GB the + // direct map covers. It returns the direct-map address unchanged for a low + // BAR, which is every BAR under SeaBIOS. + e->regs = (volatile uint8 *)ioremap(d->bar[0].base, d->bar[0].size); + if (!e->regs) + return -1; // Reset, then wait for it to clear. The manual asks for a delay after reset // before any other register is touched; polling CTRL is that delay.