SchOS/kernel/drivers/gpu/virtio_gpu/virtio_gpu.c

// virtio-gpu, 2D. Written from the OASIS virtio 1.1 specification.
//
// This is the second implementation of the display class (include/gfx/display.h),
// and the first one that is a real GPU driver rather than a flat framebuffer.
// The difference that matters for a compositor: presenting is an explicit
// operation. With VBE you write pixels and hope the scanout is not looking;
// here you transfer a rectangle to the host and then flush it, and the host
// does not show a half-drawn frame.
//
// Bring-up is the standard virtio 1.0 sequence and the order is not negotiable:
// reset, ACKNOWLEDGE, DRIVER, read features, write features, FEATURES_OK, check
// FEATURES_OK stuck, set up queues, DRIVER_OK. A device that finds the sequence
// wrong is entitled to simply refuse, and it will not say why.

#include "core/types.h"
#include "core/defs.h"
#include "core/irq.h"
#include "core/param.h"
#include "arch/x86.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "fs/fs.h"
#include "fs/file.h"
#include "arch/mmu.h"
#include "arch/memlayout.h"
#include "dev/pci.h"
#include "dev/device.h"
#include "gfx/display.h"
#include "core/dma.h"
#include "dev/virtio.h"
#include "gfx/gfx.h"

// One control queue is enough for 2D: there is also a cursor queue, which is
// only worth having once there is a hardware cursor to move.
#define CTRL_QUEUE 0

struct vgpu {
  volatile uint8 *common;      // common configuration structure
  volatile uint8 *notify;      // notification area
  // The interrupt status byte. Reading it is what tells the device we have
  // noticed, and therefore what makes it drop the PCI interrupt line.
  volatile uint8 *isr;
  uint32 notify_mult;          // queue_notify_off multiplier

  // The queue. Allocated as one contiguous block from the DMA arena, because
  // the device reads it by physical address and kalloc's pages are adjacent
  // only by chance.
  struct vring_desc  *desc;
  struct vring_avail *avail;
  struct vring_used  *used;
  uint16 qsize;
  uint16 notify_off;
  uint16 last_used;
  uint16 next_desc;

  // A command buffer and a response buffer, both in DMA memory, reused for
  // every request. Only one command is ever in flight -- everything here is
  // called from process context during setup or from present(), and a GPU
  // command queue with one entry is not the bottleneck in a 2D compositor.
  void *cmd;
  void *resp;
  void *ring;                  // the three vrings, one block, for dma_free

  int   width, height;
  void *fb;                    // the framebuffer we scan out from
  uint32 fb_bytes;
  int    ready;
  // Which PCI device this instance drives, so attached() can match the device
  // the bus publishes against the instance probe() set up.
  struct pci_dev *pdev;
};

// More than one, because two display adapters in a machine is ordinary --
// integrated plus discrete on real hardware, two -device virtio-gpu-pci under
// QEMU -- and this driver used to refuse the second outright.
#define VGPU_MAX 2

static struct vgpu gpus[VGPU_MAX];

// The instance behind a device, or 0. Every op goes through this rather than a
// file-static, which is what makes the second adapter possible at all. See
// include/dev/netdev.h and the e1000 for the same arrangement on the network side.
static struct vgpu *
vgpu_of(struct device *d)
{
  return d ? (struct vgpu *)d->priv : 0;
}

// --- MMIO accessors --------------------------------------------------------
//
// Explicit widths. The common configuration structure has 8-, 16- and 32-bit
// fields side by side, and a device is entitled to ignore an access made at the
// wrong width -- silently, with no error anywhere.

static inline uint8  r8 (volatile uint8 *p, int o) { return *(volatile uint8  *)(p + o); }
static inline uint16 r16(volatile uint8 *p, int o) { return *(volatile uint16 *)(p + o); }
static inline uint32 r32(volatile uint8 *p, int o) { return *(volatile uint32 *)(p + o); }
static inline void   w8 (volatile uint8 *p, int o, uint8 v)  { *(volatile uint8  *)(p + o) = v; }
static inline void   w16(volatile uint8 *p, int o, uint16 v) { *(volatile uint16 *)(p + o) = v; }
static inline void   w32(volatile uint8 *p, int o, uint32 v) { *(volatile uint32 *)(p + o) = v; }

static void
w64(volatile uint8 *p, int o, uint64 v)
{
  // Two 32-bit halves, low first. The specification defines these fields as a
  // pair of 32-bit registers precisely so that a 32-bit driver can write them,
  // and some implementations latch on the high write.
  w32(p, o, (uint32)v);
  w32(p, o + 4, (uint32)(v >> 32));
}

// --- capability walk -------------------------------------------------------

// Find a virtio configuration structure and map it.
//
// Each lives at an offset inside one of the device's BARs, and which BAR is not
// fixed -- the device says so in a vendor-specific PCI capability. Assuming BAR
// 0 works on QEMU today and is exactly the kind of assumption that stops
// working on the next device.
static volatile uint8 *
find_cfg(struct pci_dev *d, int want, uint32 *out_mult)
{
  uint8 off = pci_read8(d->bus, d->slot, d->func, 0x34) & 0xFC;

  while (off) {
    uint8 id   = pci_read8(d->bus, d->slot, d->func, off);
    uint8 next = pci_read8(d->bus, d->slot, d->func, off + 1);

    if (id == PCI_CAP_ID_VNDR) {
      uint8 type = pci_read8(d->bus, d->slot, d->func, off + 3);
      uint8 bar  = pci_read8(d->bus, d->slot, d->func, off + 4);
      uint32 o   = pci_read32(d->bus, d->slot, d->func, off + 8);

      if (type == want && bar < 6 && d->bar[bar].base) {
        if (out_mult)   // only the notify capability has this trailing field
          *out_mult = pci_read32(d->bus, d->slot, d->func, off + 16);

        // ioremap, not P2V. OVMF puts this device's 64-bit BAR at about 768 GB,
        // far outside the 4 GB the direct map covers, and P2V there produces an
        // address with nothing behind it -- the first register write took a
        // page fault, then a double fault, then a triple fault and a reboot
        // loop. SeaBIOS places the BAR low, which is why the BIOS path never
        // showed it.
        //
        // Map at least as far as the capability's own offset reaches: the BAR
        // size is what the device reports, and a mapping shorter than the
        // offset we are about to add would fault just as mysteriously.
        uint64 need = (uint64)o + PGSIZE;
        uint64 len  = d->bar[bar].size > need ? d->bar[bar].size : need;
        volatile uint8 *va = (volatile uint8 *)ioremap(d->bar[bar].base, len);
        if (!va)
          return 0;
        return va + o;
      }
    }
    off = next;
  }
  return 0;
}

// --- virtqueue -------------------------------------------------------------

static int
queue_setup_mem(struct vgpu *g, uint8 *mem)
{
  w16(g->common, VCOMMON_Q_SELECT, CTRL_QUEUE);

  uint16 qsize = r16(g->common, VCOMMON_Q_SIZE);
  if (qsize == 0)
    return -1;
  if (qsize > 64)
    qsize = 64;          // more than we will ever have outstanding
  g->qsize = qsize;
  g->notify_off = r16(g->common, VCOMMON_Q_NOTIFY_OFF);

  // The three rings, each aligned as the specification requires: 16 for the
  // descriptor table, 2 for available, 4 for used. Allocated as one block and
  // carved up, so there is one physical address to reason about.
  uint32 desc_sz  = qsize * sizeof(struct vring_desc);
  uint32 avail_sz = 6 + 2 * qsize;
  uint32 used_sz  = 6 + 8 * qsize;

  if (!mem)
    return -1;
  memset(mem, 0, 4096 * 3);

  g->desc  = (struct vring_desc  *) mem;
  g->avail = (struct vring_avail *)(mem + ((desc_sz + 1) & ~1U));
  g->used  = (struct vring_used  *)(mem + 4096);      // its own page, 4-aligned
  (void)avail_sz; (void)used_sz;

  w64(g->common, VCOMMON_Q_DESC,   dma_phys(g->desc));
  w64(g->common, VCOMMON_Q_DRIVER, dma_phys(g->avail));
  w64(g->common, VCOMMON_Q_DEVICE, dma_phys(g->used));
  w16(g->common, VCOMMON_Q_ENABLE, 1);
  return 0;
}

static int
queue_setup(struct vgpu *g)
{
  g->ring = dma_alloc(4096 * 3, 4096);
  return queue_setup_mem(g, g->ring);
}

// Submit one command and wait for the reply.
//
// Two descriptors: the request, which the device reads, and the response, which
// it writes. Polled rather than interrupt-driven -- every caller is either boot
// setup or present(), both of which want the answer before continuing, and an
// interrupt would only add a sleep and a wakeup to a wait of a few microseconds.
static int
cmd(struct vgpu *g, uint32 cmdlen, uint32 resplen)
{
  uint16 head = g->next_desc;
  uint16 second = (uint16)((head + 1) % g->qsize);

  g->desc[head].addr  = dma_phys(g->cmd);
  g->desc[head].len   = cmdlen;
  g->desc[head].flags = VRING_DESC_F_NEXT;
  g->desc[head].next  = second;

  g->desc[second].addr  = dma_phys(g->resp);
  g->desc[second].len   = resplen;
  g->desc[second].flags = VRING_DESC_F_WRITE;
  g->desc[second].next  = 0;

  g->next_desc = (uint16)((second + 1) % g->qsize);

  g->avail->ring[g->avail->idx % g->qsize] = head;
  __sync_synchronize();                 // ring entry visible before idx
  g->avail->idx++;
  __sync_synchronize();

  // Notify. The address is the notify region plus this queue's offset scaled by
  // the multiplier the device advertised; the value written is the queue index.
  *(volatile uint16 *)(g->notify + g->notify_off * g->notify_mult) = CTRL_QUEUE;

  // Bounded, because a device that never answers must not hang the boot. At
  // QEMU's speed the reply is there almost immediately; on a wedged device this
  // returns an error that a caller can report.
  for (int i = 0; i < 10000000; i++) {
    if (g->used->idx != g->last_used) {
      g->last_used = g->used->idx;
      __sync_synchronize();
      struct virtio_gpu_ctrl_hdr *h = g->resp;
      return (h->type == VIRTIO_GPU_RESP_OK_NODATA ||
              h->type == VIRTIO_GPU_RESP_OK_DISPLAY_INFO) ? 0 : -1;
    }
  }
  cprintf("virtio-gpu: device did not answer\n");
  return -1;
}

static void
hdr_init(void *p, uint32 type)
{
  struct virtio_gpu_ctrl_hdr *h = p;
  memset(h, 0, sizeof(*h));
  h->type = type;
}

// --- the 2D sequence -------------------------------------------------------

static int
setup_scanout(struct vgpu *g)
{
  // What resolution is the display actually at? Asking is the point of
  // GET_DISPLAY_INFO; guessing 1024x768 is how a driver ends up working on
  // exactly one host configuration.
  hdr_init(g->cmd, VIRTIO_GPU_CMD_GET_DISPLAY_INFO);
  if (cmd(g, sizeof(struct virtio_gpu_ctrl_hdr),
          sizeof(struct virtio_gpu_resp_display_info)) != 0)
    return -1;

  struct virtio_gpu_resp_display_info *info = g->resp;
  if (!info->pmodes[0].enabled || info->pmodes[0].r.width == 0) {
    cprintf("virtio-gpu: no enabled scanout\n");
    return -1;
  }
  g->width  = (int)info->pmodes[0].r.width;
  g->height = (int)info->pmodes[0].r.height;

  // But the console's mode wins, when there is a console.
  //
  // GET_DISPLAY_INFO answers with the size of the *host's window*, which says
  // nothing about the machine -- somebody dragged a corner. The resolution
  // chosen at the F8 menu is the console's, and a second head that shows the
  // console in a different mode shows it letterboxed for no reason. Same
  // argument, and the same fix, as in cmd/vgpud/main.c's bring_up().
  if (gfx_ready() && gfx_width() && gfx_height()) {
    g->width  = (int)gfx_width();
    g->height = (int)gfx_height();
  }

  // A host resource to scan out from.
  {
    struct virtio_gpu_resource_create_2d *c = g->cmd;
    hdr_init(c, VIRTIO_GPU_CMD_RESOURCE_CREATE_2D);
    c->resource_id = 1;
    c->format      = VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM;
    c->width       = g->width;
    c->height      = g->height;
    if (cmd(g, sizeof(*c), sizeof(struct virtio_gpu_ctrl_hdr)) != 0)
      return -1;
  }

  // Guest memory to back it. This is the buffer the rest of the kernel draws
  // into; the device never reads it directly -- TRANSFER_TO_HOST_2D copies from
  // here into the host resource, which is what makes presenting explicit.
  g->fb_bytes = (uint32)g->width * g->height * 4;
  g->fb = dma_alloc(g->fb_bytes, 4096);
  if (!g->fb) {
    cprintf("virtio-gpu: no room for a %d KB framebuffer\n", g->fb_bytes / 1024);
    return -1;
  }
  memset(g->fb, 0, g->fb_bytes);

  {
    struct virtio_gpu_resource_attach_backing *a = g->cmd;
    hdr_init(a, VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING);
    a->resource_id = 1;
    a->nr_entries  = 1;
    struct virtio_gpu_mem_entry *e = (struct virtio_gpu_mem_entry *)(a + 1);
    e->addr    = dma_phys(g->fb);
    e->length  = g->fb_bytes;
    e->padding = 0;
    if (cmd(g, sizeof(*a) + sizeof(*e), sizeof(struct virtio_gpu_ctrl_hdr)) != 0)
      return -1;
  }

  // Point scanout 0 at it.
  {
    struct virtio_gpu_set_scanout *s = g->cmd;
    hdr_init(s, VIRTIO_GPU_CMD_SET_SCANOUT);
    s->r.x = s->r.y = 0;
    s->r.width  = g->width;
    s->r.height = g->height;
    s->scanout_id  = 0;
    s->resource_id = 1;
    if (cmd(g, sizeof(*s), sizeof(struct virtio_gpu_ctrl_hdr)) != 0)
      return -1;
  }

  return 0;
}

// --- display class ---------------------------------------------------------

static void
vgpu_mode(struct device *d, struct display_mode *m)
{
  struct vgpu *g = vgpu_of(d);

  if (!g)
    return;
  m->width  = g->width;
  m->height = g->height;
  m->bpp    = 32;
  m->stride = (uint32)g->width * 4;
}

static void *
vgpu_acquire(struct device *d, uint32 *stride_out)
{
  struct vgpu *g = vgpu_of(d);

  if (!g)
    return 0;
  if (stride_out)
    *stride_out = (uint32)g->width * 4;
  // No separate back buffer: the guest-side framebuffer already *is* one. The
  // host cannot see it until TRANSFER_TO_HOST_2D, so drawing into it is
  // inherently off-screen -- which is the whole reason this device is better to
  // composite against than a raw VBE aperture.
  return g->ready ? g->fb : 0;
}

static void
present_g(struct vgpu *g, int x, int y, int w, int h)
{
  if (!g || !g->ready)
    return;
  if (x < 0) { w += x; x = 0; }
  if (y < 0) { h += y; y = 0; }
  if (x + w > g->width)  w = g->width - x;
  if (y + h > g->height) h = g->height - y;
  if (w <= 0 || h <= 0)
    return;

  {
    struct virtio_gpu_transfer_to_host_2d *t = g->cmd;
    hdr_init(t, VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D);
    t->r.x = x; t->r.y = y; t->r.width = w; t->r.height = h;
    t->offset      = (uint64)y * g->width * 4 + (uint64)x * 4;
    t->resource_id = 1;
    if (cmd(g, sizeof(*t), sizeof(struct virtio_gpu_ctrl_hdr)) != 0)
      return;
  }
  {
    struct virtio_gpu_resource_flush *f = g->cmd;
    hdr_init(f, VIRTIO_GPU_CMD_RESOURCE_FLUSH);
    f->r.x = x; f->r.y = y; f->r.width = w; f->r.height = h;
    f->resource_id = 1;
    cmd(g, sizeof(*f), sizeof(struct virtio_gpu_ctrl_hdr));
  }
}

static void
vgpu_present(struct device *d, int x, int y, int w, int h)
{
  present_g(vgpu_of(d), x, y, w, h);
}

// No wait_vblank. The device presents whole rectangles atomically, so there is
// nothing to synchronise against -- and a null op is more honest than one that
// spins and achieves nothing. display_wait_vblank() handles the absence.
// The guest-side framebuffer, which is this device's own memory and not the
// console's -- on every machine this driver has run on. It is written as a
// comparison rather than as `return 0` because a machine whose *only* display
// is a virtio-gpu would have the console here, and then the honest answer
// changes without this line needing to.
static uint64
vgpu_scanout_phys(struct device *d)
{
  struct vgpu *g = vgpu_of(d);
  return (g && g->fb) ? dma_phys(g->fb) : 0;
}

const struct display_ops virtio_gpu_display_ops = {
  .mode        = vgpu_mode,
  .acquire     = vgpu_acquire,
  .present     = vgpu_present,
  .wait_vblank = 0,
  .scanout_phys = vgpu_scanout_phys,
};

// --- probe -----------------------------------------------------------------

// Acknowledge the device's interrupt.
//
// This driver polls for command completion and wants no notifications at all,
// which is why nothing here reacts to the cause -- but "wants none" and "gets
// none" are different things, and a PCI interrupt line is level-triggered.
// The device raises INTx, the CPU takes the interrupt, the kernel signals EOI,
// and if nobody has cleared the device's interrupt status the line is still
// asserted, so it fires again immediately. Forever.
//
// That is not hypothetical. With no handler registered here at all, this
// device shared IRQ 11 with the e1000 under QEMU and storme at roughly 72,000
// interrupts a second, eating 83% of CPU0. The visible symptom was nowhere
// near the graphics driver: the timer interrupt was starved, so `ticks`
// advanced at about 14 Hz instead of 100, and every duration in the system --
// uptime, `time`, sleep, ping's one-second interval -- came out ~7x short.
// See docs/history/kernel.md.
//
// Reading the ISR byte clears it and deasserts the line. That single read is
// the entire handler, and it is not optional.
static void
vgpu_intr(void *arg)
{
  struct vgpu *g = arg;
  if (g && g->isr)
    (void)*g->isr;
}

static int
vgpu_probe(struct pci_dev *d)
{
  struct vgpu *g = 0;

  for (int i = 0; i < VGPU_MAX; i++)
    if (!gpus[i].ready) {
      g = &gpus[i];
      break;
    }
  if (!g)
    return -1;                     // every slot already drives an adapter
  g->pdev = d;

  pci_enable(d, 0x0006);           // memory space + bus master

  g->common = find_cfg(d, VIRTIO_PCI_CAP_COMMON_CFG, 0);
  g->notify = find_cfg(d, VIRTIO_PCI_CAP_NOTIFY_CFG, &g->notify_mult);
  g->isr    = find_cfg(d, VIRTIO_PCI_CAP_ISR_CFG, 0);
  if (!g->common || !g->notify) {
    cprintf("virtio-gpu: no modern virtio capabilities; not a 1.0 device\n");
    return -1;
  }

  // Reset, then the status handshake in the specified order.
  w8(g->common, VCOMMON_STATUS, 0);
  while (r8(g->common, VCOMMON_STATUS) != 0)
    ;
  w8(g->common, VCOMMON_STATUS, VSTATUS_ACK);
  w8(g->common, VCOMMON_STATUS, VSTATUS_ACK | VSTATUS_DRIVER);

  // Accept VIRTIO_F_VERSION_1 and nothing else. Feature bit 32 lives in the
  // second 32-bit window, which is what the select registers are for.
  w32(g->common, VCOMMON_DFEAT_SEL, 1);
  uint32 hi = r32(g->common, VCOMMON_DFEAT);
  if (!(hi & (1u << (VIRTIO_F_VERSION_1 - 32)))) {
    cprintf("virtio-gpu: device is not virtio 1.0\n");
    w8(g->common, VCOMMON_STATUS, VSTATUS_FAILED);
    return -1;
  }
  w32(g->common, VCOMMON_GFEAT_SEL, 0);
  w32(g->common, VCOMMON_GFEAT, 0);
  w32(g->common, VCOMMON_GFEAT_SEL, 1);
  w32(g->common, VCOMMON_GFEAT, 1u << (VIRTIO_F_VERSION_1 - 32));

  w8(g->common, VCOMMON_STATUS, VSTATUS_ACK | VSTATUS_DRIVER | VSTATUS_FEATURES_OK);
  if (!(r8(g->common, VCOMMON_STATUS) & VSTATUS_FEATURES_OK)) {
    // The device is allowed to clear this bit to reject the feature set, and
    // this readback is the only way it ever tells you.
    cprintf("virtio-gpu: device refused our features\n");
    w8(g->common, VCOMMON_STATUS, VSTATUS_FAILED);
    return -1;
  }

  g->cmd  = dma_alloc(4096, 16);
  g->resp = dma_alloc(4096, 16);
  if (!g->cmd || !g->resp || queue_setup(g) != 0) {
    cprintf("virtio-gpu: queue setup failed\n");
    w8(g->common, VCOMMON_STATUS, VSTATUS_FAILED);
    return -1;
  }

  w8(g->common, VCOMMON_STATUS,
     VSTATUS_ACK | VSTATUS_DRIVER | VSTATUS_FEATURES_OK | VSTATUS_DRIVER_OK);

  if (setup_scanout(g) != 0) {
    w8(g->common, VCOMMON_STATUS, VSTATUS_FAILED);
    return -1;
  }

  // Level-triggered and active low, like every other PCI INTx line -- see
  // vgpu_intr for what happens when this registration is missing.
  if (g->isr)
    irq_register(d->irq, vgpu_intr, g, "virtio-gpu", 1);
  else
    cprintf("virtio-gpu: no ISR capability; leaving irq %d unclaimed\n", d->irq);

  g->ready = 1;
  kinfo("virtio-gpu: %dx%d, %d KB framebuffer, queue %d, irq %d\n",
          g->width, g->height, g->fb_bytes / 1024, g->qsize, d->irq);
  return 0;
}

// Registers kept from before a handover, per device. Filled in by
// vgpu_release() and read only by vgpu_panic_present(); see both.
static struct {
  struct pci_dev *pdev;
  volatile uint8 *common, *notify;
  uint32 notify_mult;
} handed_over[VGPU_MAX];

// Hand the device to a userspace driver.
//
// docs/plan/hybrid.md's first eviction, and the reason it is this device: the
// boundary already exists (struct display_ops), it has exactly one consumer,
// and there is a working fallback underneath. So letting go is safe in a way
// it would not be for the disk the root filesystem is on.
//
// What "letting go" has to mean, in order:
//
//   1. stop drawing -- put the VBE framebuffer back on screen *first*, so
//      there is never a moment where gfx points at hardware nobody drives
//   2. reset the device, so the next driver starts from a known state rather
//      than inheriting a configured queue it did not set up
//   3. forget the instance, so a later probe or a second claim cannot find
//      half-initialised state
static int
vgpu_release(struct pci_dev *d)
{
  struct vgpu *g = 0;

  for (int i = 0; i < VGPU_MAX; i++)
    if (gpus[i].ready && gpus[i].pdev == d) { g = &gpus[i]; break; }
  if (!g)
    return -1;

  // The rule from docs/plan/hybrid.md, enforced rather than described: **the
  // kernel keeps a minimal driver for everything on the path to a shell.** If
  // there is no framebuffer to fall back to, this device is the only screen the
  // machine has, and handing it to a process that might crash would make a
  // failed driver indistinguishable from a dead machine.
  //
  // Refusing here is what lets the userspace driver be started unconditionally
  // at boot: on hardware where it is safe it takes the GPU, and on hardware
  // where it is not it is told no and says so, rather than each machine needing
  // to be configured for its own display topology.
  if (!vga_fb_restore()) {
    cprintf("virtio-gpu: refusing to release -- it is the only display, and "
            "there is no framebuffer to fall back to\n");
    return -1;
  }

  // Status 0 is the virtio reset. The spec requires a driver to wait for the
  // device to acknowledge by reading the register back as zero.
  if (g->common) {
    w8(g->common, VCOMMON_STATUS, 0);
    for (int spin = 0; spin < 1000000 && r8(g->common, VCOMMON_STATUS); spin++)
      ;
  }

  if (g->isr)
    irq_unregister(d->irq, vgpu_intr, g);

  // Give the arena its memory back.
  //
  // Releasing the *device* without releasing its memory is half a handover:
  // the DMA arena is 16 MB and contiguous, a framebuffer at this resolution is
  // 4 MB of it, and the process taking over needs one of its own. Keeping ours
  // means the new driver asks for a framebuffer and is told there is no room,
  // which is what happened the first time this was tried.
  if (g->fb)   { dma_free(g->fb, g->fb_bytes); g->fb = 0; }
  if (g->ring) { dma_free(g->ring, 4096 * 3);  g->ring = 0; }
  if (g->cmd)  { dma_free(g->cmd, 4096);       g->cmd = 0; }
  if (g->resp) { dma_free(g->resp, 4096);      g->resp = 0; }

  // Keep the addresses, and only the addresses.
  //
  // This is what makes the crash screen possible on a device the kernel no
  // longer drives: panic() cannot ioremap -- that allocates, and the allocator
  // is exactly the sort of thing that may have been what died -- so the
  // mappings made at probe are kept, and the panic path finds the device
  // through them rather than through anything it has to build. Registers, not
  // state: there is no queue here, no framebuffer, nothing the userspace
  // driver could be confused by.
  for (int i = 0; i < VGPU_MAX; i++)
    if (handed_over[i].pdev == d || handed_over[i].pdev == 0) {
      handed_over[i].pdev        = d;
      handed_over[i].common      = g->common;
      handed_over[i].notify      = g->notify;
      handed_over[i].notify_mult = g->notify_mult;
      break;
    }

  g->ready = 0;
  g->pdev  = 0;
  kinfo("virtio-gpu: released to userspace; screen is the VBE framebuffer\n");
  return 0;
}

// --- the crash screen, on a device this driver was told to let go ----------
//
// docs/plan/hybrid.md's bargain has a clause nobody wrote down: a device the
// kernel hands away is a device the kernel cannot use to tell you the machine
// died. The second monitor sat there showing a shell prompt from before the
// panic, which is the most misleading thing a screen can do.
//
// It does not have to be that way, and the reason is the whole argument for
// virtio: presenting is an explicit operation over a queue, and a queue can be
// rebuilt from nothing in a few hundred instructions. So on a panic the kernel
// takes the device back -- resets it, builds one queue out of static memory,
// points a scanout at the framebuffer the userspace driver registered, and
// flushes it once.
//
// The 20 KB below is the price, and it is paid whether or not the machine ever
// panics. Allocating it on demand would be cheaper and would not work: panic
// context has no allocator it can trust, because a corrupted heap is one of
// the things that gets a machine here.
static uint8 panic_pool[4096 * 5] __attribute__((aligned(4096)));

// Every spin in here is bounded. The unbounded ones in vgpu_probe() are
// correct where they are -- a device that never answers during boot is a
// machine that cannot work -- but a crash screen that hangs instead of
// appearing has taken away the serial log's last chance to be read.
static int
panic_wait_status(volatile uint8 *common, uint8 want)
{
  for (int i = 0; i < 1000000; i++)
    if (r8(common, VCOMMON_STATUS) == want)
      return 0;
  return -1;
}

static int
vgpu_panic_present(struct pci_dev *d, uint64 fb_paddr, int w, int h)
{
  static struct vgpu pg;          // static: panic context allocates nothing
  struct vgpu *g = &pg;
  int slot = -1;

  for (int i = 0; i < VGPU_MAX; i++)
    if (handed_over[i].pdev == d) { slot = i; break; }
  if (slot < 0 || !handed_over[slot].common || !handed_over[slot].notify)
    return -1;
  if (w <= 0 || h <= 0)
    return -1;

  memset(g, 0, sizeof(*g));
  g->common      = handed_over[slot].common;
  g->notify      = handed_over[slot].notify;
  g->notify_mult = handed_over[slot].notify_mult;
  g->width       = w;
  g->height      = h;

  // Memory space and bus mastering. The userspace driver has them on, but
  // asserting them costs one config write and covers the case where it died
  // and pciuser_stop() turned them off -- which is a likely way to arrive
  // here.
  pci_enable(d, 0x0006);

  // Reset. Everything the userspace driver set up -- its queue, its resources,
  // its scanout -- goes away here, which is the point: what follows must not
  // depend on any of it, because a driver that crashed may have left it in any
  // state at all.
  w8(g->common, VCOMMON_STATUS, 0);
  if (panic_wait_status(g->common, 0) != 0)
    return -1;
  w8(g->common, VCOMMON_STATUS, VSTATUS_ACK);
  w8(g->common, VCOMMON_STATUS, VSTATUS_ACK | VSTATUS_DRIVER);

  w32(g->common, VCOMMON_DFEAT_SEL, 1);
  if (!(r32(g->common, VCOMMON_DFEAT) & (1u << (VIRTIO_F_VERSION_1 - 32))))
    return -1;
  w32(g->common, VCOMMON_GFEAT_SEL, 0);
  w32(g->common, VCOMMON_GFEAT, 0);
  w32(g->common, VCOMMON_GFEAT_SEL, 1);
  w32(g->common, VCOMMON_GFEAT, 1u << (VIRTIO_F_VERSION_1 - 32));
  w8(g->common, VCOMMON_STATUS,
     VSTATUS_ACK | VSTATUS_DRIVER | VSTATUS_FEATURES_OK);
  if (!(r8(g->common, VCOMMON_STATUS) & VSTATUS_FEATURES_OK))
    return -1;

  g->cmd  = panic_pool + 4096 * 3;
  g->resp = panic_pool + 4096 * 4;
  if (queue_setup_mem(g, panic_pool) != 0)
    return -1;

  w8(g->common, VCOMMON_STATUS,
     VSTATUS_ACK | VSTATUS_DRIVER | VSTATUS_FEATURES_OK | VSTATUS_DRIVER_OK);

  // A resource the size of the report, backed by the memory the userspace
  // driver registered and the panic path has already painted into.
  {
    struct virtio_gpu_resource_create_2d *c = g->cmd;
    hdr_init(c, VIRTIO_GPU_CMD_RESOURCE_CREATE_2D);
    c->resource_id = 1;
    c->format      = VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM;
    c->width       = (uint32)w;
    c->height      = (uint32)h;
    if (cmd(g, sizeof(*c), sizeof(struct virtio_gpu_ctrl_hdr)) != 0)
      return -1;
  }
  {
    struct virtio_gpu_resource_attach_backing *a = g->cmd;
    hdr_init(a, VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING);
    a->resource_id = 1;
    a->nr_entries  = 1;
    struct virtio_gpu_mem_entry *e = (struct virtio_gpu_mem_entry *)(a + 1);
    e->addr    = fb_paddr;
    e->length  = (uint32)w * (uint32)h * 4;
    e->padding = 0;
    if (cmd(g, sizeof(*a) + sizeof(*e),
            sizeof(struct virtio_gpu_ctrl_hdr)) != 0)
      return -1;
  }
  {
    struct virtio_gpu_set_scanout *so = g->cmd;
    hdr_init(so, VIRTIO_GPU_CMD_SET_SCANOUT);
    so->r.x = so->r.y = 0;
    so->r.width  = (uint32)w;
    so->r.height = (uint32)h;
    so->scanout_id  = 0;
    so->resource_id = 1;
    if (cmd(g, sizeof(*so), sizeof(struct virtio_gpu_ctrl_hdr)) != 0)
      return -1;
  }

  g->ready = 1;                   // so vgpu_present will talk to the device
  present_g(g, 0, 0, w, h);
  return 0;
}

static const struct pci_match virtio_gpu_match[] = {
  { VIRTIO_VENDOR, VIRTIO_DEV_GPU, 0, 0 },
  { 0, 0, 0, 0 },
};

// Hang the instance on the device the bus has just published. probe() runs
// before device_add(), so there is nothing to attach it to in there -- see the
// comment on `attached` in include/dev/pci.h.
static void
vgpu_attached(struct pci_dev *d, struct device *dev)
{
  for (int i = 0; i < VGPU_MAX; i++)
    if (gpus[i].ready && gpus[i].pdev == d) {
      dev->priv = &gpus[i];
      return;
    }
}

const struct pci_driver virtio_gpu_driver = {
  .name  = "virtio-gpu",
  .match = virtio_gpu_match,
  .ops   = &virtio_gpu_display_ops,
  .probe = vgpu_probe,
  .attached = vgpu_attached,
    .release  = vgpu_release,
  .panic_present = vgpu_panic_present,
};