SchOS/kernel/dev/pciuser.c

// SPDX-License-Identifier: GPL-2.0-or-later
//
// /dev/pci -- PCI devices, reachable from userspace.
//
// See include/dev/pciuser.h for what this is and why the interface is about PCI
// rather than about drivers. This file is the kernel half: enumeration, the
// claim, the check that makes mmap safe, and DMA memory a device can be
// pointed at.
//
// The claim is the part worth reading carefully. A device that a process has
// claimed must no longer be touched by the kernel, or two drivers write the
// same registers and the failure looks like hardware that has gone mad. So
// claiming records the owner, and the kernel's own drivers are expected to
// have been told to let go first -- which for now means "do not claim a device
// the kernel is driving", enforced below rather than assumed.

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "fs/fs.h"
#include "fs/file.h"
#include "arch/memlayout.h"
#include "arch/mmu.h"
#include "core/proc.h"
#include "arch/ioperm.h"
#include "dev/pci.h"
#include "uapi/errno.h"
#include "core/uaccess.h"
#include "core/irq.h"
#include "dev/pciuser.h"
#include "core/sbuf.h"
#include "core/dbg.h"
int mappages(pml4e_t *pgdir, void *va, uint64 size, uint64 pa, int perm);

// Who holds each device, or 0. Indexed the same way pci_get() is, because the
// index a caller passes in is the one the bus scan produced and nothing
// renumbers it afterwards.
static struct proc *owner[NPCI];

// What used to drive each device, for the one thing a kernel driver is still
// needed for after it has been told to let go: the crash screen.
//
// d->drv is cleared when the claim succeeds, deliberately -- a driver that has
// handed a device over must not be reachable through the device it no longer
// drives. This is the exception, and it is narrow: nothing calls through here
// except panic(), the device is reset first, and by then the process that was
// driving it is not going to run again.
static const struct pci_driver *released_drv[NPCI];

// Where a userspace display driver is scanning out, so panic() can put the
// report there. See PCIIOC_PANICFB in include/dev/pciuser.h.
struct panicfb {
  uint64 paddr;
  uint32 width, height, stride;
};
static struct panicfb panicfb[NPCI];

static struct spinlock pciuser_lock;

// Interrupt delivery, one device per process.
//
// The count is what read() returns and what the reader compares against: a
// driver that was slow sees the count jump rather than missing an edge, which
// is the property that makes a count better than a flag.
struct irqdev {
  struct proc *proc;        // who reads, 0 when unused
  int    index;             // pci_get() index
  int    irq;               // the line
  uint64 count;             // interrupts taken
  uint64 seen;              // what the reader has consumed
  int    masked;
};
static struct irqdev irqdev;

// DMA buffers handed out, so they can be taken back.
//
// The arena is small and contiguous -- it has to be, a device cannot follow a
// free list -- so a driver that exits without returning its buffers exhausts
// it in a handful of restarts. Which is precisely the workload this interface
// invites: the whole promise is that a driver can crash and be restarted, and
// a promise that leaks a few kilobytes every time it is kept is not one.
#define PCIUSER_MAXDMA 16
struct dmablk {
  struct proc *proc;
  void  *kva;
  uint64 size;
};
static struct dmablk dmablk[PCIUSER_MAXDMA];

// The handler. Runs in interrupt context, so it does the two things that
// cannot wait and nothing else.
//
// Masking is the important one. The line is level-triggered and still asserted
// -- only the device's own driver can clear that, and its driver is a process
// that is not running yet. Signalling EOI on an asserted line re-enters here
// immediately and for ever; see docs/history/kernel.md section 13 for what
// that looks like from outside. So the line goes quiet until userspace says
// it has dealt with the device.
static void
pciuser_irq_handler(void *arg)
{
  (void)arg;
  irqdev.count++;
  if (!irqdev.masked) {
    ioapicmask(irqdev.irq);
    irqdev.masked = 1;
  }
  wakeup(&irqdev);
}

// The pid holding a device, or 0. For kernel/dev/pci.c's `pci` view, which
// otherwise prints "-" in the driver column for a device that is very much
// being driven -- just not from in here.
int
pciuser_owner_pid(int idx)
{
  int pid = 0;
  if (idx < 0 || idx >= NPCI)
    return 0;
  acquire(&pciuser_lock);
  if (owner[idx])
    pid = owner[idx]->pid;
  release(&pciuser_lock);
  return pid;
}

// A DMA block handed to a userspace driver.
//
// Informational rather than an ownership claim: the arena is the owner and
// says so separately. What this adds is *which driver was given it*, which is
// the question you actually have when a page turns up somewhere unexpected.
static void
pciuser_whose(struct sbuf *out, struct whose *w)
{
  acquire(&pciuser_lock);
  for (int i = 0; i < PCIUSER_MAXDMA; i++) {
    addr_t base;
    if (!dmablk[i].proc)
      continue;
    base = dma_phys(dmablk[i].kva);
    if (w->phys < base || w->phys >= base + dmablk[i].size)
      continue;
    sb_puts(out, "  pcidma  slot ");
    sb_putu(out, i);
    sb_puts(out, ", ");
    sb_putu(out, (w->phys - base) / 1024);
    sb_puts(out, " KB into a ");
    sb_putu(out, dmablk[i].size / 1024);
    sb_puts(out, " KB block given to pid ");
    sb_putu(out, dmablk[i].proc->pid);
    sb_puts(out, " (");
    sb_puts(out, dmablk[i].proc->name);
    sb_puts(out, ")\n");
  }
  release(&pciuser_lock);
}

// Who is holding what, and what memory they were given for it.
//
// The three things that go wrong with a userspace driver are all here: a
// device claimed by a process that no longer exists, DMA handed out and never
// returned, and an interrupt line left masked with nobody reading it. Each was
// found the hard way, and each used to need a rebuild to see.
static void
pciuser_dump(struct sbuf *out, const char *arg)
{
  (void)arg;
  int held = 0, blocks = 0;
  uint64 bytes = 0;

  acquire(&pciuser_lock);

  sb_puts(out, "claims\n");
  for (int i = 0; i < NPCI && i < pci_count(); i++) {
    const struct pci_dev *d;
    if (!owner[i])
      continue;
    held++;
    d = pci_get(i);
    sb_puts(out, "  index ");
    sb_putu_pad(out, i, 2);
    sb_puts(out, "  ");
    if (d) {
      sb_puthex(out, d->bus, 2);  sb_putc(out, ':');
      sb_puthex(out, d->slot, 2); sb_putc(out, '.');
      sb_puthex(out, d->func, 1);
      sb_puts(out, "  ");
      sb_puthex4(out, d->vendor); sb_putc(out, ':');
      sb_puthex4(out, d->device);
    }
    sb_puts(out, "  held by pid ");
    sb_putu(out, owner[i]->pid);
    sb_puts(out, " (");
    sb_puts(out, owner[i]->name);
    sb_puts(out, ")\n");
  }
  if (!held)
    sb_puts(out, "  (nothing is claimed)\n");

  sb_puts(out, "\ndma handed out\n");
  for (int i = 0; i < PCIUSER_MAXDMA; i++) {
    if (!dmablk[i].proc)
      continue;
    blocks++;
    bytes += dmablk[i].size;
    sb_puts(out, "  slot ");
    sb_putu_pad(out, i, 2);
    sb_puts(out, "  0x");
    sb_puthex(out, (addr_t)dmablk[i].kva, 16);
    sb_puts(out, "  phys 0x");
    sb_puthex(out, dma_phys(dmablk[i].kva), 16);
    sb_putu_pad(out, dmablk[i].size, 10);
    sb_puts(out, "  pid ");
    sb_putu(out, dmablk[i].proc->pid);
    sb_puts(out, " (");
    sb_puts(out, dmablk[i].proc->name);
    sb_puts(out, ")\n");
  }
  if (!blocks)
    sb_puts(out, "  (none)\n");
  else {
    sb_puts(out, "  ");
    sb_putu(out, blocks);
    sb_puts(out, " block(s), ");
    sb_putu(out, bytes / 1024);
    sb_puts(out, " KB of ");
    sb_putu(out, PCIUSER_MAXDMA);
    sb_puts(out, " slots\n");
  }

  sb_puts(out, "\ninterrupts\n");
  if (!irqdev.proc) {
    sb_puts(out, "  (no line is delivered to userspace)\n");
  } else {
    sb_puts(out, "  irq ");
    sb_putu(out, irqdev.irq);
    sb_puts(out, "  pid ");
    sb_putu(out, irqdev.proc->pid);
    sb_puts(out, "  taken ");
    sb_putu(out, irqdev.count);
    sb_puts(out, ", seen ");
    sb_putu(out, irqdev.seen);
    sb_puts(out, irqdev.masked ? ", MASKED" : ", unmasked");
    if (irqdev.count != irqdev.seen)
      sb_puts(out, "  -- the driver is behind");
    sb_putc(out, '\n');
  }

  release(&pciuser_lock);
}

void
pciuser_init(void)
{
  initlock(&pciuser_lock, "pciuser");
  dbg_register("claims", pciuser_dump,
               "devices, DMA and interrupts held by userspace drivers");
  dbg_register_whose("pcidma", pciuser_whose);
}

// Does `phys` fall inside a memory BAR of a device this process has claimed?
//
// This is the whole security story for mmap, such as it is. Without it the
// device is /dev/mem with extra steps: any process could map any physical
// address, including the kernel's own memory. With it, a process reaches
// exactly the registers of the devices it was given.
//
// Length matters as much as the base. A caller asking for one page at the end
// of a BAR must not be handed the page after it, which on a real machine is
// the next device's registers.
int
pciuser_mmio_ok(uint64 phys, uint64 len)
{
  struct proc *p = myproc();
  int n = pci_count();

  if (len == 0)
    return 0;

  for (int i = 0; i < n && i < NPCI; i++) {
    if (owner[i] != p)
      continue;
    const struct pci_dev *d = pci_get(i);
    if (!d)
      continue;
    for (int b = 0; b < PCIUSER_MAXBAR; b++) {
      uint64 base = d->bar[b].base, size = d->bar[b].size;
      uint64 lo, hi;

      if (!base || !size || d->bar[b].is_io)
        continue;

      // The *pages* the BAR lies in, not the BAR.
      //
      // sys_mmap rounds a length up to a page before asking, because that is
      // the only granularity an MMU has. Comparing that against the BAR's own
      // size meant a BAR smaller than a page could never be mapped at all:
      // rtl8139's registers are 256 bytes, the request became 4096, and the
      // answer was -EACCES for a device the process legitimately held.
      //
      // Nothing had noticed because the only consumer was virtio-gpu, whose
      // BARs are large. The first foreign driver found it in its first minute,
      // which is the thing docs/plan/hybrid.md said a foreign driver would be
      // good for.
      //
      // The widening is real and is worth naming: a sub-page BAR hands the
      // process the whole page it sits in. That is forced by the hardware
      // rather than chosen -- there is no way to map less -- and firmware
      // places BARs on their natural alignment, so in practice the rest of the
      // page is not another device's.
      lo = PGROUNDDOWN(base);
      hi = PGROUNDUP(base + size);
      if (phys >= lo && phys + len > phys && phys + len <= hi)
        return 1;
    }
  }
  return 0;
}

// Stop a device dead, without knowing anything about it.
//
// Clearing PCI_CMD_BUS_MASTER is the one halt available to a kernel that does
// not know what the device is: bus mastering lives in the standard config
// header of every PCI device, and with it clear the device cannot start another
// DMA cycle. Everything else -- quiescing a queue, parking a scanout, waiting
// for in-flight work -- is device-specific, which is exactly the knowledge that
// was moved out of the kernel on purpose.
//
// **This has to happen before the DMA buffers go back to the arena.** A driver
// that died left its device configured and running: virtio-gpu with a scanout
// set keeps reading its framebuffer for ever, and that framebuffer is about to
// be handed to whoever asks next. The failure is not a crash, which is why it
// is worth writing down -- it is a display showing another program's memory, or
// a NIC writing packets into a buffer a different driver is using.
static void
pciuser_stop(int idx)
{
  const struct pci_dev *d = pci_get(idx);
  if (d)
    pci_disable((struct pci_dev *)d, PCI_CMD_BUS_MASTER);
}

// Release everything a process held. Called when it exits, because a driver
// that crashed must not keep its device locked away from its replacement --
// that is the entire point of running drivers in processes.
void
pciuser_release_all(struct proc *p)
{
  // Devices first, and outside the lock, because stopping one is a config-space
  // write and the ordering against the DMA free below is the whole point.
  for (int i = 0; i < NPCI; i++) {
    int mine;
    acquire(&pciuser_lock);
    mine = (owner[i] == p);
    if (mine)
      owner[i] = 0;
    release(&pciuser_lock);
    if (mine) {
      panicfb[i].paddr = 0;
      pciuser_stop(i);
    }
  }

  // And its ports. The bitmap itself is closed by the next switchuvm() on each
  // CPU; see ioperm_revoke_all().
  ioperm_revoke_all(p);

  acquire(&pciuser_lock);

  // And its interrupt line -- unregistered, and left MASKED.
  //
  // The instinct is to unmask on the way out, so the device is not left dead.
  // That is exactly wrong, and for the reason running through this whole file:
  // the driver died, so nobody cleared the cause at the device, so the line is
  // very likely still asserted. Unmasking it with no handler attached means
  // every EOI re-enters an empty dispatch, for ever -- a crashed driver taking
  // the machine down with it, which is the precise opposite of what running
  // drivers in processes is for.
  //
  // Masked is safe and it is not permanent: PCIIOC_IRQ arms the line when the
  // next driver claims the device.
  // Its DMA buffers. The process's page tables are torn down by exit() either
  // way; what needs saying here is that the *arena* gets its pages back.
  for (int i = 0; i < PCIUSER_MAXDMA; i++) {
    if (dmablk[i].proc != p)
      continue;
    if (dmablk[i].kva)
      dma_free(dmablk[i].kva, dmablk[i].size);
    dmablk[i].proc = 0;
    dmablk[i].kva  = 0;
    dmablk[i].size = 0;
  }

  if (irqdev.proc == p) {
    irq_unregister(irqdev.irq, pciuser_irq_handler, 0);
    ioapicmask(irqdev.irq);
    irqdev.masked = 1;
    irqdev.proc = 0;
  }
  release(&pciuser_lock);
}

static void
copy_name(char *dst, const char *src, int cap)
{
  int i = 0;
  if (src)
    for (; i < cap - 1 && src[i]; i++)
      dst[i] = src[i];
  dst[i] = 0;
}

static int
do_info(uint64 argp)
{
  struct pciuser_info *ui = (struct pciuser_info *)user_ptr(argp, sizeof(*ui));
  if (!ui)
    return -EFAULT;

  unsigned int idx = ui->index;
  if (idx >= (unsigned int)pci_count())
    return -ENODEV;

  const struct pci_dev *d = pci_get((int)idx);
  if (!d)
    return -ENODEV;

  memset(ui, 0, sizeof(*ui));
  ui->index       = idx;
  ui->bus         = d->bus;
  ui->slot        = d->slot;
  ui->func        = d->func;
  ui->header_type = d->header_type;
  ui->vendor      = d->vendor;
  ui->device      = d->device;
  ui->class       = d->class;
  ui->subclass    = d->subclass;
  ui->prog_if     = d->prog_if;
  ui->revision    = d->revision;
  ui->irq         = d->irq;
  ui->claimed     = owner[idx] ? 1 : 0;

  if (owner[idx])
    copy_name(ui->driver, "userspace", sizeof(ui->driver));
  else
    copy_name(ui->driver, d->driver, sizeof(ui->driver));

  for (int b = 0; b < PCIUSER_MAXBAR; b++) {
    ui->bar[b].base = d->bar[b].base;
    ui->bar[b].size = d->bar[b].size;
    ui->bar[b].flags = 0;
    if (d->bar[b].base) {
      ui->bar[b].flags |= d->bar[b].is_io ? PCIUSER_BAR_IO : PCIUSER_BAR_MMIO;
      if (d->bar[b].prefetch)
        ui->bar[b].flags |= PCIUSER_BAR_PREFETCH;
    }
  }
  return 0;
}

static int
do_claim(uint64 argp)
{
  int idx;
  if (copy_from_user(&idx, argp, sizeof(idx)) < 0)
    return -EFAULT;
  if (idx < 0 || idx >= pci_count() || idx >= NPCI)
    return -ENODEV;

  const struct pci_dev *d = pci_get(idx);
  if (!d)
    return -ENODEV;

  // Refuse a device the kernel is already driving.
  //
  // Two drivers on one device is not a subtle failure -- it is two writers of
  // the same registers, and what it looks like from outside is hardware that
  // has started lying. The eviction path (docs/plan/hybrid.md) is for the
  // kernel to *hand over* a device deliberately; until a driver can be told to
  // let go, the honest answer to "may I have the one you are using" is no.
  if (d->driver) {
    // Ask it to let go. A driver with no release hook keeps its device, which
    // is always safe and, for anything on the path to a shell, correct: the
    // rule in docs/plan/hybrid.md is that the kernel keeps a minimal driver
    // for everything needed to reach a prompt and userspace owns the good one.
    if (!d->drv || !d->drv->release)
      return -EBUSY;
    if (d->drv->release((struct pci_dev *)d) != 0)
      return -EBUSY;
    released_drv[idx] = d->drv;
    ((struct pci_dev *)d)->driver = 0;
    ((struct pci_dev *)d)->drv    = 0;
  }

  acquire(&pciuser_lock);
  if (owner[idx] && owner[idx] != myproc()) {
    release(&pciuser_lock);
    return -EBUSY;
  }
  owner[idx] = myproc();
  release(&pciuser_lock);

  // Bus mastering and memory space, because a driver that has to ask for this
  // separately will forget, and the symptom -- DMA that silently does nothing
  // -- costs a day to find.
  //
  // This is also what makes a device usable again after the driver holding it
  // died: release clears bus mastering to stop the device, so claim has to put
  // it back. The two are a pair, and the second driver to run is the one that
  // finds out if they are not.
  pci_enable((struct pci_dev *)d, PCI_CMD_MEMORY | PCI_CMD_BUS_MASTER);

  // And its interrupt line: masked.
  //
  // The kernel driver that just let go had a handler on this line and no
  // longer does, but the device is about to be brought up by a driver in a
  // process and *will* assert it. With nothing to acknowledge the cause at the
  // device, every EOI re-enters an empty dispatch immediately:
  //
  //   irq 11: 449905 interrupts/second -- a device on this line is not
  //           acknowledging it.
  //
  // which is the machine spending all of its time noticing. The line comes
  // back when the process asks for it with PCIIOC_IRQ, which is the same
  // arrangement pciuser_release_all() leaves behind and for the same reason.
  if (d->irq != 0xFF && d->irq < 24) {
    ioapicmask(d->irq);
    acquire(&pciuser_lock);
    if (irqdev.proc == 0)
      irqdev.masked = 1;
    release(&pciuser_lock);
  }

  // And the ports, if it has any.
  //
  // A Linux driver does not always go through a function we can intercept:
  // e1000's reset path issues `out` directly, because on an 82540 that is the
  // only way the chip will acknowledge the write. So a process holding a
  // device gets that device's I/O BARs and nothing else -- see
  // include/arch/ioperm.h. PCI_CMD_IO goes with them for the same reason memory
  // space goes with the mapping above.
  {
    int granted = 0;
    for (int b = 0; b < PCIUSER_MAXBAR; b++) {
      if (!d->bar[b].is_io || !d->bar[b].base || !d->bar[b].size)
        continue;
      if (d->bar[b].base > 0xffff || d->bar[b].base + d->bar[b].size > 0x10000)
        continue;               // above the port space; nothing to grant
      ioperm_grant(myproc(), (uint16)d->bar[b].base, (uint16)d->bar[b].size);
      granted = 1;
    }
    if (granted) {
      pci_enable((struct pci_dev *)d, PCI_CMD_IO);
      // Install them on this CPU now, rather than at the next switchuvm().
      // The process asked for them in a system call and returns to user mode
      // from here, which need not go through the scheduler at all.
      pushcli();
      ioperm_switch(myproc());
      popcli();
    }
  }
  return 0;
}

static int
do_release(uint64 argp)
{
  int idx;
  if (copy_from_user(&idx, argp, sizeof(idx)) < 0)
    return -EFAULT;
  if (idx < 0 || idx >= NPCI)
    return -ENODEV;

  acquire(&pciuser_lock);
  int mine = (owner[idx] == myproc());
  if (mine) {
    owner[idx] = 0;
    panicfb[idx].paddr = 0;
  }
  release(&pciuser_lock);

  // Same halt as the death path, for the same reason. A driver that releases
  // politely has usually quiesced its device already, but "usually" is not a
  // property the kernel can check, and the cost of being sure is one config
  // write.
  if (mine) {
    pciuser_stop(idx);
    // Ports go back too. Coarse on purpose: this drops *every* range the
    // process had rather than only this device's, and the next claim puts
    // them back. A process releasing one of several devices is not a case
    // that exists yet, and a revoke that is too wide fails safe.
    ioperm_revoke_all(myproc());
    pushcli();
    ioperm_switch(myproc());
    popcli();
  }
  return 0;
}

// Memory a device can be pointed at.
//
// mmap cannot do this: a process can map memory, but nothing tells it the
// *physical* address, and a device only understands physical addresses. So
// the kernel allocates from the DMA arena -- which is contiguous, which
// kalloc's free list is not -- maps it in, and reports both.
static int
do_dma(uint64 argp)
{
  struct pciuser_dma *req = (struct pciuser_dma *)user_ptr(argp, sizeof(*req));
  if (!req)
    return -EFAULT;

  // The cap on one request, and it is the thing that actually said no.
  //
  // It was 4 MB, which is one framebuffer at 1280x800 and was chosen when that
  // was the only thing anybody asked for. virtio-gpu's display size is the size
  // of the host's *window*, though, and a user who drags that window out to
  // 1920x1080 is asking for 8100 KB in one piece. The refusal came from here
  // and not from the arena, which is why it kept happening on an arena with
  // 61 MB free:
  //
  //   vgpud: wanted 7365 KB of DMA memory for a 1920x982 framebuffer
  //   ...
  //   65536 KB total, 61847 KB free in 2 extent(s)
  //
  // 32 MB is a 2560x1600 framebuffer with room over. The cap is not there to
  // ration memory -- the arena does that, and says so properly when it runs
  // out -- it is there so that a wild `size` from a process cannot ask for the
  // whole arena by accident.
  uint64 size = PGROUNDUP(req->size);
  if (size == 0 || size > 32 * 1024 * 1024)
    return -EINVAL;

  struct proc *p = myproc();

  // Two slots, both found before anything is allocated: one to remember the
  // block by, one to remember the mapping by. Handing out memory that cannot
  // be recorded is how the leak this fixes got written in the first place.
  //
  // The VMA is not bookkeeping. It is the thing that keeps this memory out of
  // kfree()'s hands. freevm() frees every mapped page below PHYSTOP, and the
  // DMA arena is the top 16 MB of exactly that range -- so a DMA block left
  // for freevm() to find goes onto the page allocator's free list when the
  // process dies, while pciuser_release_all() hands the same pages back to
  // the arena. Both allocators then believe they own it.
  //
  // That is what broke H2. The restarted vgpud asked for a 4 MB framebuffer,
  // dma_alloc() returned pages kalloc() had already given to somebody --
  // including the page tables of the process doing the asking -- and zeroed
  // them, so the driver died inside this ioctl, on the return from a syscall
  // whose stack had just stopped existing.
  //
  // no_free says these pages are not freevm()'s to release;
  // proc_release_shared() unmaps the range before the page tables go away.
  struct dmablk *blk = 0;
  acquire(&pciuser_lock);
  for (int i = 0; i < PCIUSER_MAXDMA; i++)
    if (!dmablk[i].proc) { blk = &dmablk[i]; blk->proc = p; break; }
  release(&pciuser_lock);
  if (!blk)
    return -ENOMEM;

  struct vma *v = 0;
  for (int i = 0; i < NVMA; i++)
    if (!p->mm->vma[i].used) { v = &p->mm->vma[i]; break; }
  if (!v) {
    blk->proc = 0;
    return -ENOMEM;
  }

  // Its own mapping in the caller, taken from the same bump allocator mmap
  // uses so the two cannot collide.
  if (p->mm->mmap_next == 0)
    p->mm->mmap_next = 0x40000000;
  uint64 va = p->mm->mmap_next;
  if (va + size < va || va + size > KERNBASE) {
    blk->proc = 0;
    return -ENOMEM;
  }

  void *kva = dma_alloc(size, PGSIZE);
  if (!kva) {
    blk->proc = 0;
    return -ENOMEM;
  }
  p->mm->mmap_next += size;

  uint64 phys = dma_phys(kva);
  if (mappages(p->pgdir, (void *)va, size, phys, PTE_W | PTE_U) < 0) {
    vm_unmap_norelease(p->pgdir, va, size);   // whatever part of it took
    dma_free(kva, size);
    blk->proc = 0;
    return -ENOMEM;
  }
  blk->kva  = kva;
  blk->size = size;

  v->addr    = va;
  v->length  = size;
  v->used    = 1;
  v->no_free = 1;
  v->shmid   = -1;

  req->uaddr = va;
  req->paddr = phys;
  return 0;
}

// Register where this process is scanning out, so panic() can draw there.
//
// The check is the whole of the security of this call: the range has to lie
// inside a block this process got from PCIIOC_DMA. Without it, a claimed
// device would be a way to nominate any physical address in the machine as the
// place the kernel writes four megabytes of crash report -- page tables, the
// kernel's own text -- at the one moment nothing is left to notice.
static int
do_panicfb(uint64 argp)
{
  struct pciuser_panicfb req;
  struct proc *p = myproc();

  if (copy_from_user(&req, argp, sizeof(req)) < 0)
    return -EFAULT;
  if ((int)req.index >= pci_count() || (int)req.index >= NPCI)
    return -ENODEV;
  if (owner[req.index] != p)
    return -EACCES;               // claim it first

  if (req.paddr == 0) {           // unregister
    acquire(&pciuser_lock);
    panicfb[req.index].paddr = 0;
    release(&pciuser_lock);
    return 0;
  }

  if (req.width == 0 || req.height == 0 || req.stride < req.width * 4)
    return -EINVAL;

  uint64 need = (uint64)req.height * req.stride;
  uint64 end  = req.paddr + need;
  if (end < req.paddr)
    return -EINVAL;

  int ok = 0;
  acquire(&pciuser_lock);
  for (int i = 0; i < PCIUSER_MAXDMA; i++) {
    uint64 base, lim;
    if (dmablk[i].proc != p || !dmablk[i].kva)
      continue;
    base = dma_phys(dmablk[i].kva);
    lim  = base + dmablk[i].size;
    if (req.paddr >= base && end <= lim) { ok = 1; break; }
  }
  if (ok) {
    panicfb[req.index].paddr  = req.paddr;
    panicfb[req.index].width  = req.width;
    panicfb[req.index].height = req.height;
    panicfb[req.index].stride = req.stride;
  }
  release(&pciuser_lock);

  // Not EINVAL: the argument is fine, the memory is not this process's to
  // nominate. EACCES is the difference between "you asked wrong" and "no".
  return ok ? 0 : -EACCES;
}

// The panic path's view of the above. Called with interrupts off, from a
// machine that is already dead: no locks, because whoever held one is not
// going to release it, and a crash screen that deadlocks is worse than no
// crash screen.
int
pciuser_panic_surface(int idx, uint64 *paddr, int *w, int *h, uint32 *stride,
                      const struct pci_driver **drv)
{
  if (idx < 0 || idx >= NPCI || owner[idx] == 0 || panicfb[idx].paddr == 0)
    return 0;
  if (!released_drv[idx] || !released_drv[idx]->panic_present)
    return 0;
  *paddr  = panicfb[idx].paddr;
  *w      = (int)panicfb[idx].width;
  *h      = (int)panicfb[idx].height;
  *stride = panicfb[idx].stride;
  *drv    = released_drv[idx];
  return 1;
}

// Hand a device's interrupts to the calling process.
static int
do_irq(uint64 argp)
{
  int idx;
  if (copy_from_user(&idx, argp, sizeof(idx)) < 0)
    return -EFAULT;
  if (idx < 0 || idx >= pci_count() || idx >= NPCI)
    return -ENODEV;
  if (owner[idx] != myproc())
    return -EACCES;               // claim it first

  const struct pci_dev *d = pci_get(idx);
  if (!d || d->irq == 0xFF)
    return -ENODEV;               // the firmware routed it nowhere

  // Refuse a line the kernel is already listening on.
  //
  // This handler masks the line, and a masked line is masked for *everyone* on
  // it. Doing that to a shared line silences whichever kernel driver is also
  // there, and the symptom -- a device that simply stops, with nothing logged
  // -- is about as hard to find as bugs get. MSI is the real answer, since it
  // is per-device and cannot be shared; pci_msi_enable() is already in the
  // tree waiting for a caller.
  if (irq_handler_count(d->irq) > 0)
    return -EBUSY;

  if (irqdev.proc && irqdev.proc != myproc())
    return -EBUSY;                // one device per process, and one process

  irqdev.proc   = myproc();
  irqdev.index  = idx;
  irqdev.irq    = d->irq;
  irqdev.count  = 0;
  irqdev.seen   = 0;
  irqdev.masked = 0;

  // Level-triggered and active low, like every PCI INTx line.
  irq_register(d->irq, pciuser_irq_handler, 0, "userspace", 1);

  // Arm it. The line may have been left masked by whatever held this device
  // before -- see pciuser_release_all -- and a driver that has to write()
  // before its first read() to get going is an interface nobody will guess.
  ioapicroute(d->irq, 0, 1);
  irqdev.masked = 0;
  return 0;
}

// PCI configuration space.
//
// The first real driver asked for this and the test program never had to: a
// virtio device describes where its registers are in *vendor capabilities*,
// which live in config space, so a driver that cannot read config space cannot
// find its own registers. Anything modern is discovered the same way -- MSI-X
// tables, PCIe extended capabilities, power management.
//
// Reads are unrestricted for a claimed device; writes are not offered at all
// yet. Nothing needs them, and a write to the wrong offset re-programs a BAR
// or disables bus mastering under the kernel's feet.
static int
do_cfg(uint64 argp)
{
  struct pciuser_cfg *c = (struct pciuser_cfg *)user_ptr(argp, sizeof(*c));
  if (!c)
    return -EFAULT;

  unsigned int idx = c->index;
  if (idx >= (unsigned int)pci_count() || idx >= NPCI)
    return -ENODEV;
  if (c->offset > 0xFC)
    return -EINVAL;

  // The standard header -- the first 64 bytes -- is readable without holding
  // the device. Refusing it was inconsistent rather than careful: PCIIOC_INFO
  // already hands back the vendor, device, class, interrupt line and every BAR,
  // all of which are read out of exactly these bytes. Denying the same bytes in
  // a different shape protects nothing and makes the bus unenumerable by
  // anything that is not already driving something.
  //
  // Past 0x40 is the capability list and whatever the device puts after it.
  // That is device-specific, it is what a driver reads to find its own
  // registers, and it belongs to whoever holds the device.
  //
  // Reads only. There is no config *write* here at all, and that is the line
  // that matters -- a write is how you move a BAR out from under the kernel.
  if (c->offset >= 0x40 && owner[idx] != myproc())
    return -EACCES;

  const struct pci_dev *d = pci_get((int)idx);
  if (!d)
    return -ENODEV;
  c->value = pci_read32(d->bus, d->slot, d->func, (uint8)(c->offset & 0xFC));
  return 0;
}

int
pciuser_ioctl(uint64 request, uint64 argp)
{
  switch (request) {
  case PCIIOC_COUNT: {
    int n = pci_count();
    put_user(n, argp);
    return 0;
  }
  case PCIIOC_INFO:    return do_info(argp);
  case PCIIOC_CLAIM:   return do_claim(argp);
  case PCIIOC_RELEASE: return do_release(argp);
  case PCIIOC_DMA:     return do_dma(argp);
  case PCIIOC_IRQ:     return do_irq(argp);
  case PCIIOC_CFG:     return do_cfg(argp);
  case PCIIOC_PANICFB: return do_panicfb(argp);
  default:             return -ENOTTY;
  }
}

// Wait for the device to interrupt.
//
// Returns the total count rather than a flag, so a driver that was busy can
// see it missed three and act once -- which is what you want, because the
// alternative is either losing them or queueing them, and a device does not
// care how many times you noticed it.
int
pciuser_read(struct inode *ip, uint64 off, char *dst, int n)
{
  (void)off;

  if (irqdev.proc != myproc())
    return -1;
  if (n < (int)sizeof(uint64))
    return -1;

  iunlock(ip);
  acquire(&pciuser_lock);
  while (irqdev.count == irqdev.seen) {
    if (myproc()->killed) {
      release(&pciuser_lock);
      ilock(ip);
      return -1;
    }
    sleep(&irqdev, &pciuser_lock);
  }
  uint64 c = irqdev.count;
  irqdev.seen = c;
  release(&pciuser_lock);
  ilock(ip);

  memmove(dst, &c, sizeof(c));
  return (int)sizeof(c);
}

// "I have dealt with the device" -- unmask the line.
//
// The driver must have cleared the interrupt *at the device* before this, or
// the line is still asserted and unmasking simply re-enters the handler. That
// is the contract, it cannot be checked from here, and getting it wrong is the
// 72,000-interrupts-a-second failure in docs/history/kernel.md section 13.
int
pciuser_write(struct inode *ip, uint64 off, char *src, int n)
{
  (void)ip; (void)off; (void)src;

  if (irqdev.proc != myproc())
    return -1;

  acquire(&pciuser_lock);
  if (irqdev.masked) {
    ioapicroute(irqdev.irq, 0, 1);     // level, active low: a PCI INTx line
    irqdev.masked = 0;
  }
  release(&pciuser_lock);
  return n;
}