SchOS/kernel/dev/pciuser.c

// SPDX-License-Identifier: Apache-2.0
//
// /dev/pci -- PCI devices, reachable from userspace.
//
// See include/pciuser.h for what this is and why the interface is about PCI
// rather than about drivers. This file is the kernel half: enumeration, the
// claim, the check that makes mmap safe, and DMA memory a device can be
// pointed at.
//
// The claim is the part worth reading carefully. A device that a process has
// claimed must no longer be touched by the kernel, or two drivers write the
// same registers and the failure looks like hardware that has gone mad. So
// claiming records the owner, and the kernel's own drivers are expected to
// have been told to let go first -- which for now means "do not claim a device
// the kernel is driving", enforced below rather than assumed.

#include "types.h"
#include "defs.h"
#include "param.h"
#include "spinlock.h"
#include "sleeplock.h"
#include "fs.h"
#include "file.h"
#include "memlayout.h"
#include "mmu.h"
#include "proc.h"
#include "pci.h"
#include "errno.h"
#include "uaccess.h"
#include "irq.h"
#include "pciuser.h"
#include "sbuf.h"
#include "dbg.h"
int mappages(pml4e_t *pgdir, void *va, uint64 size, uint64 pa, int perm);

// Who holds each device, or 0. Indexed the same way pci_get() is, because the
// index a caller passes in is the one the bus scan produced and nothing
// renumbers it afterwards.
static struct proc *owner[NPCI];

static struct spinlock pciuser_lock;

// Interrupt delivery, one device per process.
//
// The count is what read() returns and what the reader compares against: a
// driver that was slow sees the count jump rather than missing an edge, which
// is the property that makes a count better than a flag.
struct irqdev {
  struct proc *proc;        // who reads, 0 when unused
  int    index;             // pci_get() index
  int    irq;               // the line
  uint64 count;             // interrupts taken
  uint64 seen;              // what the reader has consumed
  int    masked;
};
static struct irqdev irqdev;

// DMA buffers handed out, so they can be taken back.
//
// The arena is small and contiguous -- it has to be, a device cannot follow a
// free list -- so a driver that exits without returning its buffers exhausts
// it in a handful of restarts. Which is precisely the workload this interface
// invites: the whole promise is that a driver can crash and be restarted, and
// a promise that leaks a few kilobytes every time it is kept is not one.
#define PCIUSER_MAXDMA 16
struct dmablk {
  struct proc *proc;
  void  *kva;
  uint64 size;
};
static struct dmablk dmablk[PCIUSER_MAXDMA];

// The handler. Runs in interrupt context, so it does the two things that
// cannot wait and nothing else.
//
// Masking is the important one. The line is level-triggered and still asserted
// -- only the device's own driver can clear that, and its driver is a process
// that is not running yet. Signalling EOI on an asserted line re-enters here
// immediately and for ever; see docs/history/kernel.md section 13 for what
// that looks like from outside. So the line goes quiet until userspace says
// it has dealt with the device.
static void
pciuser_irq_handler(void *arg)
{
  (void)arg;
  irqdev.count++;
  if (!irqdev.masked) {
    ioapicmask(irqdev.irq);
    irqdev.masked = 1;
  }
  wakeup(&irqdev);
}

// The pid holding a device, or 0. For kernel/dev/pci.c's `pci` view, which
// otherwise prints "-" in the driver column for a device that is very much
// being driven -- just not from in here.
int
pciuser_owner_pid(int idx)
{
  int pid = 0;
  if (idx < 0 || idx >= NPCI)
    return 0;
  acquire(&pciuser_lock);
  if (owner[idx])
    pid = owner[idx]->pid;
  release(&pciuser_lock);
  return pid;
}

// Who is holding what, and what memory they were given for it.
//
// The three things that go wrong with a userspace driver are all here: a
// device claimed by a process that no longer exists, DMA handed out and never
// returned, and an interrupt line left masked with nobody reading it. Each was
// found the hard way, and each used to need a rebuild to see.
static void
pciuser_dump(struct sbuf *out, const char *arg)
{
  (void)arg;
  int held = 0, blocks = 0;
  uint64 bytes = 0;

  acquire(&pciuser_lock);

  sb_puts(out, "claims\n");
  for (int i = 0; i < NPCI && i < pci_count(); i++) {
    const struct pci_dev *d;
    if (!owner[i])
      continue;
    held++;
    d = pci_get(i);
    sb_puts(out, "  index ");
    sb_putu_pad(out, i, 2);
    sb_puts(out, "  ");
    if (d) {
      sb_puthex(out, d->bus, 2);  sb_putc(out, ':');
      sb_puthex(out, d->slot, 2); sb_putc(out, '.');
      sb_puthex(out, d->func, 1);
      sb_puts(out, "  ");
      sb_puthex4(out, d->vendor); sb_putc(out, ':');
      sb_puthex4(out, d->device);
    }
    sb_puts(out, "  held by pid ");
    sb_putu(out, owner[i]->pid);
    sb_puts(out, " (");
    sb_puts(out, owner[i]->name);
    sb_puts(out, ")\n");
  }
  if (!held)
    sb_puts(out, "  (nothing is claimed)\n");

  sb_puts(out, "\ndma handed out\n");
  for (int i = 0; i < PCIUSER_MAXDMA; i++) {
    if (!dmablk[i].proc)
      continue;
    blocks++;
    bytes += dmablk[i].size;
    sb_puts(out, "  slot ");
    sb_putu_pad(out, i, 2);
    sb_puts(out, "  0x");
    sb_puthex(out, (addr_t)dmablk[i].kva, 16);
    sb_puts(out, "  phys 0x");
    sb_puthex(out, dma_phys(dmablk[i].kva), 16);
    sb_putu_pad(out, dmablk[i].size, 10);
    sb_puts(out, "  pid ");
    sb_putu(out, dmablk[i].proc->pid);
    sb_puts(out, " (");
    sb_puts(out, dmablk[i].proc->name);
    sb_puts(out, ")\n");
  }
  if (!blocks)
    sb_puts(out, "  (none)\n");
  else {
    sb_puts(out, "  ");
    sb_putu(out, blocks);
    sb_puts(out, " block(s), ");
    sb_putu(out, bytes / 1024);
    sb_puts(out, " KB of ");
    sb_putu(out, PCIUSER_MAXDMA);
    sb_puts(out, " slots\n");
  }

  sb_puts(out, "\ninterrupts\n");
  if (!irqdev.proc) {
    sb_puts(out, "  (no line is delivered to userspace)\n");
  } else {
    sb_puts(out, "  irq ");
    sb_putu(out, irqdev.irq);
    sb_puts(out, "  pid ");
    sb_putu(out, irqdev.proc->pid);
    sb_puts(out, "  taken ");
    sb_putu(out, irqdev.count);
    sb_puts(out, ", seen ");
    sb_putu(out, irqdev.seen);
    sb_puts(out, irqdev.masked ? ", MASKED" : ", unmasked");
    if (irqdev.count != irqdev.seen)
      sb_puts(out, "  -- the driver is behind");
    sb_putc(out, '\n');
  }

  release(&pciuser_lock);
}

void
pciuser_init(void)
{
  initlock(&pciuser_lock, "pciuser");
  dbg_register("claims", pciuser_dump,
               "devices, DMA and interrupts held by userspace drivers");
}

// Does `phys` fall inside a memory BAR of a device this process has claimed?
//
// This is the whole security story for mmap, such as it is. Without it the
// device is /dev/mem with extra steps: any process could map any physical
// address, including the kernel's own memory. With it, a process reaches
// exactly the registers of the devices it was given.
//
// Length matters as much as the base. A caller asking for one page at the end
// of a BAR must not be handed the page after it, which on a real machine is
// the next device's registers.
int
pciuser_mmio_ok(uint64 phys, uint64 len)
{
  struct proc *p = myproc();
  int n = pci_count();

  if (len == 0)
    return 0;

  for (int i = 0; i < n && i < NPCI; i++) {
    if (owner[i] != p)
      continue;
    const struct pci_dev *d = pci_get(i);
    if (!d)
      continue;
    for (int b = 0; b < PCIUSER_MAXBAR; b++) {
      uint64 base = d->bar[b].base, size = d->bar[b].size;
      if (!base || !size || d->bar[b].is_io)
        continue;
      if (phys >= base && phys + len > phys && phys + len <= base + size)
        return 1;
    }
  }
  return 0;
}

// Stop a device dead, without knowing anything about it.
//
// Clearing PCI_CMD_BUS_MASTER is the one halt available to a kernel that does
// not know what the device is: bus mastering lives in the standard config
// header of every PCI device, and with it clear the device cannot start another
// DMA cycle. Everything else -- quiescing a queue, parking a scanout, waiting
// for in-flight work -- is device-specific, which is exactly the knowledge that
// was moved out of the kernel on purpose.
//
// **This has to happen before the DMA buffers go back to the arena.** A driver
// that died left its device configured and running: virtio-gpu with a scanout
// set keeps reading its framebuffer for ever, and that framebuffer is about to
// be handed to whoever asks next. The failure is not a crash, which is why it
// is worth writing down -- it is a display showing another program's memory, or
// a NIC writing packets into a buffer a different driver is using.
static void
pciuser_stop(int idx)
{
  const struct pci_dev *d = pci_get(idx);
  if (d)
    pci_disable((struct pci_dev *)d, PCI_CMD_BUS_MASTER);
}

// Release everything a process held. Called when it exits, because a driver
// that crashed must not keep its device locked away from its replacement --
// that is the entire point of running drivers in processes.
void
pciuser_release_all(struct proc *p)
{
  // Devices first, and outside the lock, because stopping one is a config-space
  // write and the ordering against the DMA free below is the whole point.
  for (int i = 0; i < NPCI; i++) {
    int mine;
    acquire(&pciuser_lock);
    mine = (owner[i] == p);
    if (mine)
      owner[i] = 0;
    release(&pciuser_lock);
    if (mine)
      pciuser_stop(i);
  }

  acquire(&pciuser_lock);

  // And its interrupt line -- unregistered, and left MASKED.
  //
  // The instinct is to unmask on the way out, so the device is not left dead.
  // That is exactly wrong, and for the reason running through this whole file:
  // the driver died, so nobody cleared the cause at the device, so the line is
  // very likely still asserted. Unmasking it with no handler attached means
  // every EOI re-enters an empty dispatch, for ever -- a crashed driver taking
  // the machine down with it, which is the precise opposite of what running
  // drivers in processes is for.
  //
  // Masked is safe and it is not permanent: PCIIOC_IRQ arms the line when the
  // next driver claims the device.
  // Its DMA buffers. The process's page tables are torn down by exit() either
  // way; what needs saying here is that the *arena* gets its pages back.
  for (int i = 0; i < PCIUSER_MAXDMA; i++) {
    if (dmablk[i].proc != p)
      continue;
    if (dmablk[i].kva)
      dma_free(dmablk[i].kva, dmablk[i].size);
    dmablk[i].proc = 0;
    dmablk[i].kva  = 0;
    dmablk[i].size = 0;
  }

  if (irqdev.proc == p) {
    irq_unregister(irqdev.irq, pciuser_irq_handler, 0);
    ioapicmask(irqdev.irq);
    irqdev.masked = 1;
    irqdev.proc = 0;
  }
  release(&pciuser_lock);
}

static void
copy_name(char *dst, const char *src, int cap)
{
  int i = 0;
  if (src)
    for (; i < cap - 1 && src[i]; i++)
      dst[i] = src[i];
  dst[i] = 0;
}

static int
do_info(uint64 argp)
{
  struct pciuser_info *ui = (struct pciuser_info *)user_ptr(argp, sizeof(*ui));
  if (!ui)
    return -EFAULT;

  unsigned int idx = ui->index;
  if (idx >= (unsigned int)pci_count())
    return -ENODEV;

  const struct pci_dev *d = pci_get((int)idx);
  if (!d)
    return -ENODEV;

  memset(ui, 0, sizeof(*ui));
  ui->index       = idx;
  ui->bus         = d->bus;
  ui->slot        = d->slot;
  ui->func        = d->func;
  ui->header_type = d->header_type;
  ui->vendor      = d->vendor;
  ui->device      = d->device;
  ui->class       = d->class;
  ui->subclass    = d->subclass;
  ui->prog_if     = d->prog_if;
  ui->revision    = d->revision;
  ui->irq         = d->irq;
  ui->claimed     = owner[idx] ? 1 : 0;

  if (owner[idx])
    copy_name(ui->driver, "userspace", sizeof(ui->driver));
  else
    copy_name(ui->driver, d->driver, sizeof(ui->driver));

  for (int b = 0; b < PCIUSER_MAXBAR; b++) {
    ui->bar[b].base = d->bar[b].base;
    ui->bar[b].size = d->bar[b].size;
    ui->bar[b].flags = 0;
    if (d->bar[b].base) {
      ui->bar[b].flags |= d->bar[b].is_io ? PCIUSER_BAR_IO : PCIUSER_BAR_MMIO;
      if (d->bar[b].prefetch)
        ui->bar[b].flags |= PCIUSER_BAR_PREFETCH;
    }
  }
  return 0;
}

static int
do_claim(uint64 argp)
{
  int idx;
  if (copy_from_user(&idx, argp, sizeof(idx)) < 0)
    return -EFAULT;
  if (idx < 0 || idx >= pci_count() || idx >= NPCI)
    return -ENODEV;

  const struct pci_dev *d = pci_get(idx);
  if (!d)
    return -ENODEV;

  // Refuse a device the kernel is already driving.
  //
  // Two drivers on one device is not a subtle failure -- it is two writers of
  // the same registers, and what it looks like from outside is hardware that
  // has started lying. The eviction path (docs/plan/hybrid.md) is for the
  // kernel to *hand over* a device deliberately; until a driver can be told to
  // let go, the honest answer to "may I have the one you are using" is no.
  if (d->driver) {
    // Ask it to let go. A driver with no release hook keeps its device, which
    // is always safe and, for anything on the path to a shell, correct: the
    // rule in docs/plan/hybrid.md is that the kernel keeps a minimal driver
    // for everything needed to reach a prompt and userspace owns the good one.
    if (!d->drv || !d->drv->release)
      return -EBUSY;
    if (d->drv->release((struct pci_dev *)d) != 0)
      return -EBUSY;
    ((struct pci_dev *)d)->driver = 0;
    ((struct pci_dev *)d)->drv    = 0;
  }

  acquire(&pciuser_lock);
  if (owner[idx] && owner[idx] != myproc()) {
    release(&pciuser_lock);
    return -EBUSY;
  }
  owner[idx] = myproc();
  release(&pciuser_lock);

  // Bus mastering and memory space, because a driver that has to ask for this
  // separately will forget, and the symptom -- DMA that silently does nothing
  // -- costs a day to find.
  //
  // This is also what makes a device usable again after the driver holding it
  // died: release clears bus mastering to stop the device, so claim has to put
  // it back. The two are a pair, and the second driver to run is the one that
  // finds out if they are not.
  pci_enable((struct pci_dev *)d, PCI_CMD_MEMORY | PCI_CMD_BUS_MASTER);
  return 0;
}

static int
do_release(uint64 argp)
{
  int idx;
  if (copy_from_user(&idx, argp, sizeof(idx)) < 0)
    return -EFAULT;
  if (idx < 0 || idx >= NPCI)
    return -ENODEV;

  acquire(&pciuser_lock);
  int mine = (owner[idx] == myproc());
  if (mine)
    owner[idx] = 0;
  release(&pciuser_lock);

  // Same halt as the death path, for the same reason. A driver that releases
  // politely has usually quiesced its device already, but "usually" is not a
  // property the kernel can check, and the cost of being sure is one config
  // write.
  if (mine)
    pciuser_stop(idx);
  return 0;
}

// Memory a device can be pointed at.
//
// mmap cannot do this: a process can map memory, but nothing tells it the
// *physical* address, and a device only understands physical addresses. So
// the kernel allocates from the DMA arena -- which is contiguous, which
// kalloc's free list is not -- maps it in, and reports both.
static int
do_dma(uint64 argp)
{
  struct pciuser_dma *req = (struct pciuser_dma *)user_ptr(argp, sizeof(*req));
  if (!req)
    return -EFAULT;

  uint64 size = PGROUNDUP(req->size);
  if (size == 0 || size > 4 * 1024 * 1024)
    return -EINVAL;

  struct proc *p = myproc();

  // Two slots, both found before anything is allocated: one to remember the
  // block by, one to remember the mapping by. Handing out memory that cannot
  // be recorded is how the leak this fixes got written in the first place.
  //
  // The VMA is not bookkeeping. It is the thing that keeps this memory out of
  // kfree()'s hands. freevm() frees every mapped page below PHYSTOP, and the
  // DMA arena is the top 16 MB of exactly that range -- so a DMA block left
  // for freevm() to find goes onto the page allocator's free list when the
  // process dies, while pciuser_release_all() hands the same pages back to
  // the arena. Both allocators then believe they own it.
  //
  // That is what broke H2. The restarted vgpud asked for a 4 MB framebuffer,
  // dma_alloc() returned pages kalloc() had already given to somebody --
  // including the page tables of the process doing the asking -- and zeroed
  // them, so the driver died inside this ioctl, on the return from a syscall
  // whose stack had just stopped existing.
  //
  // no_free says these pages are not freevm()'s to release;
  // proc_release_shared() unmaps the range before the page tables go away.
  struct dmablk *blk = 0;
  acquire(&pciuser_lock);
  for (int i = 0; i < PCIUSER_MAXDMA; i++)
    if (!dmablk[i].proc) { blk = &dmablk[i]; blk->proc = p; break; }
  release(&pciuser_lock);
  if (!blk)
    return -ENOMEM;

  struct vma *v = 0;
  for (int i = 0; i < NVMA; i++)
    if (!p->vma[i].used) { v = &p->vma[i]; break; }
  if (!v) {
    blk->proc = 0;
    return -ENOMEM;
  }

  // Its own mapping in the caller, taken from the same bump allocator mmap
  // uses so the two cannot collide.
  if (p->mmap_next == 0)
    p->mmap_next = 0x40000000;
  uint64 va = p->mmap_next;
  if (va + size < va || va + size > KERNBASE) {
    blk->proc = 0;
    return -ENOMEM;
  }

  void *kva = dma_alloc(size, PGSIZE);
  if (!kva) {
    blk->proc = 0;
    return -ENOMEM;
  }
  p->mmap_next += size;

  uint64 phys = dma_phys(kva);
  if (mappages(p->pgdir, (void *)va, size, phys, PTE_W | PTE_U) < 0) {
    vm_unmap_norelease(p->pgdir, va, size);   // whatever part of it took
    dma_free(kva, size);
    blk->proc = 0;
    return -ENOMEM;
  }
  blk->kva  = kva;
  blk->size = size;

  v->addr    = va;
  v->length  = size;
  v->used    = 1;
  v->no_free = 1;
  v->shmid   = -1;

  req->uaddr = va;
  req->paddr = phys;
  return 0;
}

// Hand a device's interrupts to the calling process.
static int
do_irq(uint64 argp)
{
  int idx;
  if (copy_from_user(&idx, argp, sizeof(idx)) < 0)
    return -EFAULT;
  if (idx < 0 || idx >= pci_count() || idx >= NPCI)
    return -ENODEV;
  if (owner[idx] != myproc())
    return -EACCES;               // claim it first

  const struct pci_dev *d = pci_get(idx);
  if (!d || d->irq == 0xFF)
    return -ENODEV;               // the firmware routed it nowhere

  // Refuse a line the kernel is already listening on.
  //
  // This handler masks the line, and a masked line is masked for *everyone* on
  // it. Doing that to a shared line silences whichever kernel driver is also
  // there, and the symptom -- a device that simply stops, with nothing logged
  // -- is about as hard to find as bugs get. MSI is the real answer, since it
  // is per-device and cannot be shared; pci_msi_enable() is already in the
  // tree waiting for a caller.
  if (irq_handler_count(d->irq) > 0)
    return -EBUSY;

  if (irqdev.proc && irqdev.proc != myproc())
    return -EBUSY;                // one device per process, and one process

  irqdev.proc   = myproc();
  irqdev.index  = idx;
  irqdev.irq    = d->irq;
  irqdev.count  = 0;
  irqdev.seen   = 0;
  irqdev.masked = 0;

  // Level-triggered and active low, like every PCI INTx line.
  irq_register(d->irq, pciuser_irq_handler, 0, "userspace", 1);

  // Arm it. The line may have been left masked by whatever held this device
  // before -- see pciuser_release_all -- and a driver that has to write()
  // before its first read() to get going is an interface nobody will guess.
  ioapicroute(d->irq, 0, 1);
  irqdev.masked = 0;
  return 0;
}

// PCI configuration space.
//
// The first real driver asked for this and the test program never had to: a
// virtio device describes where its registers are in *vendor capabilities*,
// which live in config space, so a driver that cannot read config space cannot
// find its own registers. Anything modern is discovered the same way -- MSI-X
// tables, PCIe extended capabilities, power management.
//
// Reads are unrestricted for a claimed device; writes are not offered at all
// yet. Nothing needs them, and a write to the wrong offset re-programs a BAR
// or disables bus mastering under the kernel's feet.
static int
do_cfg(uint64 argp)
{
  struct pciuser_cfg *c = (struct pciuser_cfg *)user_ptr(argp, sizeof(*c));
  if (!c)
    return -EFAULT;

  unsigned int idx = c->index;
  if (idx >= (unsigned int)pci_count() || idx >= NPCI)
    return -ENODEV;
  if (c->offset > 0xFC)
    return -EINVAL;

  // The standard header -- the first 64 bytes -- is readable without holding
  // the device. Refusing it was inconsistent rather than careful: PCIIOC_INFO
  // already hands back the vendor, device, class, interrupt line and every BAR,
  // all of which are read out of exactly these bytes. Denying the same bytes in
  // a different shape protects nothing and makes the bus unenumerable by
  // anything that is not already driving something.
  //
  // Past 0x40 is the capability list and whatever the device puts after it.
  // That is device-specific, it is what a driver reads to find its own
  // registers, and it belongs to whoever holds the device.
  //
  // Reads only. There is no config *write* here at all, and that is the line
  // that matters -- a write is how you move a BAR out from under the kernel.
  if (c->offset >= 0x40 && owner[idx] != myproc())
    return -EACCES;

  const struct pci_dev *d = pci_get((int)idx);
  if (!d)
    return -ENODEV;
  c->value = pci_read32(d->bus, d->slot, d->func, (uint8)(c->offset & 0xFC));
  return 0;
}

int
pciuser_ioctl(uint64 request, uint64 argp)
{
  switch (request) {
  case PCIIOC_COUNT: {
    int n = pci_count();
    put_user(n, argp);
    return 0;
  }
  case PCIIOC_INFO:    return do_info(argp);
  case PCIIOC_CLAIM:   return do_claim(argp);
  case PCIIOC_RELEASE: return do_release(argp);
  case PCIIOC_DMA:     return do_dma(argp);
  case PCIIOC_IRQ:     return do_irq(argp);
  case PCIIOC_CFG:     return do_cfg(argp);
  default:             return -ENOTTY;
  }
}

// Wait for the device to interrupt.
//
// Returns the total count rather than a flag, so a driver that was busy can
// see it missed three and act once -- which is what you want, because the
// alternative is either losing them or queueing them, and a device does not
// care how many times you noticed it.
int
pciuser_read(struct inode *ip, uint64 off, char *dst, int n)
{
  (void)off;

  if (irqdev.proc != myproc())
    return -1;
  if (n < (int)sizeof(uint64))
    return -1;

  iunlock(ip);
  acquire(&pciuser_lock);
  while (irqdev.count == irqdev.seen) {
    if (myproc()->killed) {
      release(&pciuser_lock);
      ilock(ip);
      return -1;
    }
    sleep(&irqdev, &pciuser_lock);
  }
  uint64 c = irqdev.count;
  irqdev.seen = c;
  release(&pciuser_lock);
  ilock(ip);

  memmove(dst, &c, sizeof(c));
  return (int)sizeof(c);
}

// "I have dealt with the device" -- unmask the line.
//
// The driver must have cleared the interrupt *at the device* before this, or
// the line is still asserted and unmasking simply re-enters the handler. That
// is the contract, it cannot be checked from here, and getting it wrong is the
// 72,000-interrupts-a-second failure in docs/history/kernel.md section 13.
int
pciuser_write(struct inode *ip, uint64 off, char *src, int n)
{
  (void)ip; (void)off; (void)src;

  if (irqdev.proc != myproc())
    return -1;

  acquire(&pciuser_lock);
  if (irqdev.masked) {
    ioapicroute(irqdev.irq, 0, 1);     // level, active low: a PCI INTx line
    irqdev.masked = 0;
  }
  release(&pciuser_lock);
  return n;
}