SchOS/cmd/vgpud/main.c

// SPDX-License-Identifier: Apache-2.0
//
// vgpud -- the virtio-gpu driver, outside the kernel.
//
// docs/plan/hybrid.md's first eviction. The same device the kernel drives in
// kernel/drivers/gpu/virtio_gpu/, driven instead by a process that asks the
// kernel for four things and nothing else:
//
//   PCIIOC_CLAIM   take the device (the kernel driver is asked to let go)
//   PCIIOC_CFG     read config space, to find the vendor capabilities
//   mmap(BAR)      reach the registers
//   PCIIOC_DMA     memory the device can be pointed at
//
// The point is not that this is better than the in-kernel driver -- it is the
// same code in a worse language for the job. The point is that it is a real
// driver rather than a test program, so the interface has something to be
// judged by, and what it asks for is what a foreign driver would ask for too.
//
// Note what is *not* here: no syscall knows this is a GPU. The kernel handed
// over a PCI device and this decided it was a GPU, which is the property that
// makes the interface general rather than a hole shaped like one thing.

#include "types.h"
#include "user.h"
#include "fcntl.h"
#include "posix.h"
#include "pciuser.h"
#include "virtio.h"
#include "schinit.h"
#include "socket.h"
#include "theme.h"   // SCH_BG, SCH_ACCENT -- one palette, see docs/branding
#include "logo.h"    // the mark, the same bytes the loader and console draw

// --- small output, since lib/ has no printf ---------------------------------

// How the machine was booted. A driver coming up is progress on a boot you are
// watching and noise on every other one; a driver *failing* is neither, and
// stays on puts(). See kinfo() in kernel/core/console.c for the same rule one
// level down.
static int verbose_boot;

static void
read_bootmode(void)
{
    char b[16];
    int fd = open("/proc/bootmode", O_RDONLY);
    int n;

    if (fd < 0)
        return;
    n = read(fd, b, sizeof(b) - 1);
    close(fd);
    if (n > 0) { b[n] = 0; verbose_boot = (b[0] == 'v'); }
}

static void
puts(const char *s)
{
    int n = 0;
    while (s[n]) n++;
    write(1, (void *)s, n);
}

// Progress, as opposed to news.
static void
vputs(const char *s)
{
    if (verbose_boot)
        puts(s);
}

static void putn(long long v);

static void
vputn(long long v)
{
    if (verbose_boot)
        putn(v);
}

static void
putn(long long v)
{
    char b[24];
    int i = 0;
    if (v < 0) { write(1, "-", 1); v = -v; }
    if (v == 0) { write(1, "0", 1); return; }
    while (v > 0) { b[i++] = '0' + (int)(v % 10); v /= 10; }
    while (i-- > 0) write(1, &b[i], 1);
}

static void
puthex(unsigned long long v, int digits)
{
    static const char h[] = "0123456789abcdef";
    char b[17];
    for (int i = digits - 1; i >= 0; i--) { b[i] = h[v & 0xF]; v >>= 4; }
    write(1, b, digits);
}

static void
die(const char *why)
{
    puts("vgpud: "); puts(why); puts("\n");
    exit();
}

// Not a failure: a report. There is no virtio-gpu here, or the kernel will not
// give it up, and neither of those changes by trying again -- so this tells the
// supervisor to stop, rather than being restarted every three seconds for the
// rest of the uptime. See SCHINIT_EXIT_NORESTART in include/schinit.h.
static void
nothing_to_do(const char *why)
{
    // Quiet on a quiet boot: "this machine has no virtio-gpu" is a fact about
    // the hardware and not a fault, and it is the same fact every time. It is
    // still an exit the supervisor is told about, which is what stops it being
    // restarted fifteen times in forty-five seconds.
    vputs("vgpud: "); vputs(why); vputs("\n");
    exit_group(SCHINIT_EXIT_NORESTART);
}

// --- register access --------------------------------------------------------

static inline unsigned char  r8 (volatile unsigned char *p, int o) { return p[o]; }
static inline void           w8 (volatile unsigned char *p, int o, unsigned char v) { p[o] = v; }
static inline unsigned short r16(volatile unsigned char *p, int o) { return *(volatile unsigned short *)(p + o); }
static inline void           w16(volatile unsigned char *p, int o, unsigned short v) { *(volatile unsigned short *)(p + o) = v; }
static inline unsigned int   r32(volatile unsigned char *p, int o) { return *(volatile unsigned int *)(p + o); }
static inline void           w32(volatile unsigned char *p, int o, unsigned int v) { *(volatile unsigned int *)(p + o) = v; }
static inline void           w64(volatile unsigned char *p, int o, unsigned long long v)
{
    // Two 32-bit writes, low half first. The virtio spec requires it and the
    // device latches on the high write; a single 64-bit store is a different
    // bus transaction and some implementations simply drop it.
    w32(p, o, (unsigned int)v);
    w32(p, o + 4, (unsigned int)(v >> 32));
}

// --- the device -------------------------------------------------------------

static int pcifd;
static int index_;

static volatile unsigned char *common;
static volatile unsigned char *notify;
static volatile unsigned char *isr;
static unsigned int notify_mult;

static struct pciuser_info info;

static unsigned int
cfg32(unsigned int off)
{
    struct pciuser_cfg c;
    c.index = (unsigned int)index_;
    c.offset = off & ~3u;
    c.value = 0;
    if (ioctl(pcifd, PCIIOC_CFG, &c) < 0)
        die("config read failed");
    return c.value;
}

static unsigned char
cfg8(unsigned int off)
{
    return (unsigned char)(cfg32(off) >> ((off & 3) * 8));
}

// Find a virtio configuration structure.
//
// A virtio 1.0 device does not put its registers at a fixed offset. It walks
// you there: a vendor capability in config space names which BAR, and how far
// into it, each structure lives. So a driver that cannot read config space
// cannot find its own registers -- which is why PCIIOC_CFG exists, and why the
// test program that came before this never needed it.
static volatile unsigned char *
find_cfg(int want, unsigned int *out_mult)
{
    unsigned char off = cfg8(0x34) & 0xFC;

    while (off) {
        unsigned char id   = cfg8(off);
        unsigned char next = cfg8(off + 1);

        if (id == PCI_CAP_ID_VNDR) {
            unsigned char type = cfg8(off + 3);
            unsigned char bar  = cfg8(off + 4);
            unsigned int  o    = cfg32(off + 8);
            unsigned int  len  = cfg32(off + 12);

            if (type == want && bar < PCIUSER_MAXBAR && info.bar[bar].base) {
                if (out_mult)
                    *out_mult = cfg32(off + 16);

                // Map from the start of the BAR, not from the capability's
                // offset: two capabilities usually live in one BAR, and
                // mapping each separately would hand out two mappings of
                // overlapping pages.
                unsigned long long want_len = (unsigned long long)o + len;
                if (want_len < 4096) want_len = 4096;
                if (want_len > info.bar[bar].size) want_len = info.bar[bar].size;

                volatile unsigned char *p =
                    (volatile unsigned char *)mmap(0, want_len,
                                                   PROT_READ | PROT_WRITE,
                                                   MAP_SHARED, pcifd,
                                                   info.bar[bar].base);
                if (p == 0 || (long)p == -1)
                    return 0;
                return p + o;
            }
        }
        off = next & 0xFC;
    }
    return 0;
}

// --- the queue --------------------------------------------------------------

#define CTRL_QUEUE 0

static struct vring_desc  *desc;
static struct vring_avail *avail;
static struct vring_used  *used;
static unsigned long long  ring_phys;
static unsigned short      qsize, notify_off, last_used, next_desc;

static void      *cmdbuf,  *respbuf;
static unsigned long long cmdphys, respphys;

static void
queue_setup(void)
{
    w16(common, VCOMMON_Q_SELECT, CTRL_QUEUE);
    qsize = r16(common, VCOMMON_Q_SIZE);
    if (qsize == 0)
        die("control queue has no slots");
    if (qsize > 64)
        qsize = 64;                 // more than one command in flight is not
                                    // something this driver has any use for
    w16(common, VCOMMON_Q_SIZE, qsize);
    notify_off = r16(common, VCOMMON_Q_NOTIFY_OFF);

    // One contiguous block for the three rings, laid out the way the spec
    // requires: descriptors, then available, then used, each aligned. The
    // device reads all three by physical address, so they have to come from
    // the kernel's DMA arena rather than from anything this process could
    // allocate for itself.
    unsigned int desc_sz  = 16 * qsize;
    unsigned int avail_sz = 6 + 2 * qsize;

    struct pciuser_dma d;
    d.size = 16384;
    if (ioctl(pcifd, PCIIOC_DMA, &d) < 0)
        die("no DMA memory for the queue");

    unsigned long long base  = d.uaddr;
    unsigned long long basep = d.paddr;

    unsigned int off_avail = (desc_sz + 1) & ~1u;
    unsigned int off_used  = (off_avail + avail_sz + 3) & ~3u;

    desc  = (struct vring_desc  *)(base);
    avail = (struct vring_avail *)(base + off_avail);
    used  = (struct vring_used  *)(base + off_used);
    ring_phys = basep;

    for (unsigned int i = 0; i < 16384; i++)
        ((volatile unsigned char *)base)[i] = 0;

    // Do not interrupt us. docmd() polls the used ring, so an interrupt is
    // not merely unwanted, it is dangerous: this process holds the device but
    // has no handler registered, and the line is shared with the e1000. A
    // level-triggered line nobody acknowledges re-fires for ever -- 77,000
    // times a second, which is what the first working run of this driver did
    // before this line existed. The storm detector in kernel/dev/irq.c named
    // the line and the neighbour it was drowning.
    avail->flags = VRING_AVAIL_F_NO_INTERRUPT;
    __sync_synchronize();

    w64(common, VCOMMON_Q_DESC,   basep);
    w64(common, VCOMMON_Q_DRIVER, basep + off_avail);
    w64(common, VCOMMON_Q_DEVICE, basep + off_used);
    w16(common, VCOMMON_Q_ENABLE, 1);

    // Command and response buffers, also from the arena for the same reason.
    struct pciuser_dma b;
    b.size = 8192;
    if (ioctl(pcifd, PCIIOC_DMA, &b) < 0)
        die("no DMA memory for command buffers");
    cmdbuf   = (void *)b.uaddr;
    cmdphys  = b.paddr;
    respbuf  = (void *)(b.uaddr + 4096);
    respphys = b.paddr + 4096;
}

// Submit one command and wait for the answer.
//
// Polled, not interrupt-driven. This driver issues a handful of commands at
// startup and one flush per frame, and a poll on a used-ring index that the
// device updates by DMA is both simpler and lower-latency than waking a
// process. The interrupt path exists (PCIIOC_IRQ) and belongs in whatever
// wants to sleep between frames.
static int
docmd(unsigned int cmdlen, unsigned int resplen)
{
    unsigned short head = next_desc;

    desc[head].addr  = cmdphys;
    desc[head].len   = cmdlen;
    desc[head].flags = VRING_DESC_F_NEXT;
    desc[head].next  = (unsigned short)(head + 1);

    desc[head + 1].addr  = respphys;
    desc[head + 1].len   = resplen;
    desc[head + 1].flags = VRING_DESC_F_WRITE;
    desc[head + 1].next  = 0;

    avail->ring[avail->idx % qsize] = head;
    __sync_synchronize();
    avail->idx++;
    __sync_synchronize();

    *(volatile unsigned short *)(notify + notify_off * notify_mult) = CTRL_QUEUE;

    // Spin with a bound. A device that never answers is a bug somewhere, and
    // hanging for ever is a worse way to report it than saying so.
    for (int spin = 0; spin < 100000000; spin++) {
        __sync_synchronize();
        if (used->idx != last_used) {
            last_used = used->idx;
            next_desc = (unsigned short)((next_desc + 2) % qsize);
            return 0;
        }
    }
    return -1;
}

static void
hdr_init(void *p, unsigned int type)
{
    struct virtio_gpu_ctrl_hdr *h = (struct virtio_gpu_ctrl_hdr *)p;
    for (unsigned int i = 0; i < sizeof(*h); i++) ((unsigned char *)h)[i] = 0;
    h->type = type;
}

static int
resp_ok(void)
{
    struct virtio_gpu_ctrl_hdr *h = (struct virtio_gpu_ctrl_hdr *)respbuf;
    return h->type == VIRTIO_GPU_RESP_OK_NODATA ||
           h->type == VIRTIO_GPU_RESP_OK_DISPLAY_INFO;
}

// --- bringing the device up -------------------------------------------------

static int width, height;
static unsigned long long fb_uaddr, fb_paddr;
static unsigned int fb_bytes;          // what was allocated, not what is in use
static int max_w, max_h;               // the mode the allocation was sized for

// The resource is recreated on every modeset; the framebuffer behind it is
// not. PCIIOC_DMA has no free -- a driver's blocks come back when it exits and
// not before -- so a modeset that allocated would exhaust sixteen slots and
// then the arena, and the fault would look like a GPU bug rather than like
// counting. Allocating once at the largest mode and reusing it for every
// smaller one costs nothing: a 1024x768 frame is a prefix of a 1280x800 one.
static unsigned int resource_id = 1;

static void
bring_up(void)
{
    // Reset, then the status handshake in the order the spec sets out. A
    // device that finds the sequence wrong fails the feature step rather than
    // telling you which line was at fault, so it is written out plainly.
    w8(common, VCOMMON_STATUS, 0);
    while (r8(common, VCOMMON_STATUS) != 0)
        ;
    w8(common, VCOMMON_STATUS, VSTATUS_ACK);
    w8(common, VCOMMON_STATUS, VSTATUS_ACK | VSTATUS_DRIVER);

    w32(common, VCOMMON_DFEAT_SEL, 1);
    unsigned int hi = r32(common, VCOMMON_DFEAT);
    if (!(hi & (1u << (VIRTIO_F_VERSION_1 - 32)))) {
        w8(common, VCOMMON_STATUS, VSTATUS_FAILED);
        die("device is not virtio 1.0");
    }

    w32(common, VCOMMON_GFEAT_SEL, 0);
    w32(common, VCOMMON_GFEAT, 0);
    w32(common, VCOMMON_GFEAT_SEL, 1);
    w32(common, VCOMMON_GFEAT, 1u << (VIRTIO_F_VERSION_1 - 32));

    w8(common, VCOMMON_STATUS, VSTATUS_ACK | VSTATUS_DRIVER | VSTATUS_FEATURES_OK);
    if (!(r8(common, VCOMMON_STATUS) & VSTATUS_FEATURES_OK)) {
        w8(common, VCOMMON_STATUS, VSTATUS_FAILED);
        die("device refused the feature set");
    }

    queue_setup();

    w8(common, VCOMMON_STATUS,
       VSTATUS_ACK | VSTATUS_DRIVER | VSTATUS_FEATURES_OK | VSTATUS_DRIVER_OK);

    // What size is the screen?
    hdr_init(cmdbuf, VIRTIO_GPU_CMD_GET_DISPLAY_INFO);
    if (docmd(sizeof(struct virtio_gpu_ctrl_hdr),
              sizeof(struct virtio_gpu_resp_display_info)) < 0)
        die("the device never answered GET_DISPLAY_INFO");
    if (!resp_ok())
        die("GET_DISPLAY_INFO refused");

    struct virtio_gpu_resp_display_info *di =
        (struct virtio_gpu_resp_display_info *)respbuf;
    width  = (int)di->pmodes[0].r.width;
    height = (int)di->pmodes[0].r.height;
    if (width <= 0 || height <= 0)
        die("the device reports no display");

    vputs("vgpud: display is "); vputn(width); vputs("x"); vputn(height); vputs("\n");

    // The framebuffer is claimed **before** the resource is created, because
    // the resource has to be created at a size we can actually back.
    //
    // virtio-gpu's display size is the size of the host's window, which the
    // user drags to whatever they like -- and every pixel of it is four bytes
    // of contiguous DMA that has to be found twice during the handover, once
    // for the kernel driver and once for this one. A big enough window does
    // not fit, and what that used to look like was not a message, it was a
    // loop:
    //
    //   vgpud: wanted 7365 KB of DMA memory for a 1920x982 framebuffer
    //   schinitd: restarting os.schos.vgpud
    //   vgpud: wanted 7365 KB ...                (and again, and again, until
    //   schinit: supervisor stopped answering     schinit replaced schinitd)
    //
    // So: ask for what the display says, and if that cannot be had, ask for
    // less. A smaller scanout on a larger display is a worse picture; a driver
    // that will not start is no picture and a supervisor fighting itself.
    {
        struct pciuser_dma d;
        int w = width, h = height, tries;

        for (tries = 0; ; tries++) {
            d.size = (unsigned long long)w * h * 4;
            if (ioctl(pcifd, PCIIOC_DMA, &d) >= 0)
                break;

            // Two thirds each time, on an eight-pixel grid. Not halving:
            // halving overshoots badly, and the arena is usually short by a
            // little rather than by a lot.
            w = ((w * 2 / 3) + 7) & ~7;
            h = ((h * 2 / 3) + 7) & ~7;
            if (w < 640 || h < 480) {
                puts("vgpud: wanted "); putn(((long long)width * height * 4) / 1024);
                puts(" KB of DMA memory for a "); putn(width); puts("x");
                putn(height); puts(" framebuffer, and nothing down to 640x480\n");
                puts("vgpud: `schdbg dma` shows the arena -- this is "
                     "DMA_ARENA_BYTES, not fragmentation\n");
                // Not die(): a died driver is restarted, and restarting will
                // fail identically for ever. The kernel keeps the VBE console,
                // which is a working screen.
                nothing_to_do("no room for a framebuffer; the console keeps the "
                              "VBE framebuffer");
            }
        }

        if (tries) {
            puts("vgpud: the display is "); putn(width); puts("x"); putn(height);
            puts(" but only "); putn(w); puts("x"); putn(h);
            puts(" of DMA could be had; using that\n");
            width = w;
            height = h;
        }
        fb_uaddr = d.uaddr;
        fb_paddr = d.paddr;
    }
    fb_bytes = (unsigned int)(width * height * 4);
    max_w = width;
    max_h = height;

    // A 2D resource the size of the screen.
    struct virtio_gpu_resource_create_2d *c =
        (struct virtio_gpu_resource_create_2d *)cmdbuf;
    hdr_init(c, VIRTIO_GPU_CMD_RESOURCE_CREATE_2D);
    c->resource_id = resource_id;
    c->format      = VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM;
    c->width       = (unsigned int)width;
    c->height      = (unsigned int)height;
    if (docmd(sizeof(*c), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok())
        die("RESOURCE_CREATE_2D refused");

    // The framebuffer was claimed above, before the resource was created.

    struct virtio_gpu_resource_attach_backing *a =
        (struct virtio_gpu_resource_attach_backing *)cmdbuf;
    hdr_init(a, VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING);
    a->resource_id = resource_id;
    a->nr_entries  = 1;
    // The entries follow the struct rather than being in it: the count is
    // variable, so the command is a header with an array glued on the end.
    struct virtio_gpu_mem_entry *e = (struct virtio_gpu_mem_entry *)(a + 1);
    e->addr    = fb_paddr;
    e->length  = fb_bytes;
    e->padding = 0;
    if (docmd(sizeof(*a) + sizeof(*e), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 ||
        !resp_ok())
        die("RESOURCE_ATTACH_BACKING refused");

    // Put it on the screen.
    struct virtio_gpu_set_scanout *so = (struct virtio_gpu_set_scanout *)cmdbuf;
    hdr_init(so, VIRTIO_GPU_CMD_SET_SCANOUT);
    so->r.x = so->r.y = 0;
    so->r.width  = (unsigned int)width;
    so->r.height = (unsigned int)height;
    so->scanout_id  = 0;
    so->resource_id = resource_id;
    if (docmd(sizeof(*so), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok())
        die("SET_SCANOUT refused");

    if (verbose_boot) {
        puts("vgpud: scanout is ours, framebuffer at physical 0x");
        puthex(fb_paddr, 16); puts("\n");
    }
}

// Push what has been drawn to the host. Two commands, and both are needed:
// the transfer copies from the backing into the device's copy of the resource,
// the flush tells it to put that on screen.
static void
present(void)
{
    struct virtio_gpu_transfer_to_host_2d *t =
        (struct virtio_gpu_transfer_to_host_2d *)cmdbuf;
    hdr_init(t, VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D);
    t->r.x = t->r.y = 0;
    t->r.width  = (unsigned int)width;
    t->r.height = (unsigned int)height;
    t->offset      = 0;
    t->resource_id = resource_id;
    t->padding     = 0;
    if (docmd(sizeof(*t), sizeof(struct virtio_gpu_ctrl_hdr)) < 0)
        die("TRANSFER_TO_HOST_2D never answered");

    struct virtio_gpu_resource_flush *f =
        (struct virtio_gpu_resource_flush *)cmdbuf;
    hdr_init(f, VIRTIO_GPU_CMD_RESOURCE_FLUSH);
    f->r.x = f->r.y = 0;
    f->r.width  = (unsigned int)width;
    f->r.height = (unsigned int)height;
    f->resource_id = resource_id;
    f->padding     = 0;
    if (docmd(sizeof(*f), sizeof(struct virtio_gpu_ctrl_hdr)) < 0)
        die("RESOURCE_FLUSH never answered");
}

// --- something to look at ---------------------------------------------------

// The mark, on the theme ground.
//
// This used to be a test card: three colour bands, four gradients, a
// checkerboard, and this instance's pid in binary along the bottom edge. That
// was scaffolding for hybrid.md's H2 and H3 -- a screenshot had to answer "did
// a *userspace process* put this here", and "did the driver come back" needed
// a second run to look different from a first that never stopped. Both
// milestones are closed, and what was left was a garish test pattern on a
// second display with no explanation of itself.
//
// The mark answers the same question a black screen cannot -- something is
// driving this device -- and answers it as the system rather than as a test
// rig. Same include/logo.h the loader splash and the kernel console draw, so
// there is one mark and one set of bytes.
static void
draw_mark(void)
{
    volatile unsigned int *fb = (volatile unsigned int *)fb_uaddr;
    int x0 = (width  - LOGO_W) / 2;
    int y0 = (height - LOGO_H) / 2;

    for (int y = 0; y < height; y++)
        for (int x = 0; x < width; x++)
            fb[y * width + x] = SCH_BG;

    if (x0 < 0 || y0 < 0)
        return;                   // a display smaller than the mark: leave it

    for (int dy = 0; dy < LOGO_H; dy++) {
        for (int dx = 0; dx < LOGO_W; dx++) {
            unsigned ink = LOGO_INK(dy * LOGO_W + dx);
            if (!ink)
                continue;         // the mask is a square; the mark is not
            // Coverage, not a stencil: the mark has soft edges, and stamping
            // them as solid would show every one of them as a staircase.
            unsigned r = (((SCH_ACCENT >> 16) & 0xFF) * ink +
                          ((SCH_BG     >> 16) & 0xFF) * (255 - ink)) / 255;
            unsigned g = (((SCH_ACCENT >>  8) & 0xFF) * ink +
                          ((SCH_BG     >>  8) & 0xFF) * (255 - ink)) / 255;
            unsigned b = (((SCH_ACCENT      ) & 0xFF) * ink +
                          ((SCH_BG          ) & 0xFF) * (255 - ink)) / 255;
            fb[(y0 + dy) * width + (x0 + dx)] = (r << 16) | (g << 8) | b;
        }
    }
}

// --- the modeset --------------------------------------------------------------
//
// docs/plan/hybrid.md milestone H3: *"a modeset API, which is the thing an
// out-of-kernel GPU driver makes affordable and an in-kernel one never got."*
//
// The claim is worth testing rather than repeating, so here is the whole of
// it: five commands to the device, no kernel change, and a driver that can be
// killed and restarted if it goes wrong. In the kernel the same sequence would
// be five commands *and* a reason to be nervous, because getting it wrong
// takes the machine rather than one process -- which is precisely why the
// in-kernel driver never grew one.
static int
set_mode(int w, int h)
{
    if (w <= 0 || h <= 0)
        return -1;
    // Bounded by the allocation, not by the device. See fb_bytes.
    if ((unsigned int)(w * h * 4) > fb_bytes)
        return -2;

    // Let go of the old resource first. The backing pages are ours and survive
    // this; only the device's view of them goes away.
    struct virtio_gpu_resource_unref *u =
        (struct virtio_gpu_resource_unref *)cmdbuf;
    hdr_init(u, VIRTIO_GPU_CMD_RESOURCE_UNREF);
    u->resource_id = resource_id;
    u->padding     = 0;
    if (docmd(sizeof(*u), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok())
        return -3;

    // A fresh id rather than the same one again: the spec does not promise the
    // old id is reusable the instant the unref completes, and an id is free.
    resource_id++;

    struct virtio_gpu_resource_create_2d *c =
        (struct virtio_gpu_resource_create_2d *)cmdbuf;
    hdr_init(c, VIRTIO_GPU_CMD_RESOURCE_CREATE_2D);
    c->resource_id = resource_id;
    c->format      = VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM;
    c->width       = (unsigned int)w;
    c->height      = (unsigned int)h;
    if (docmd(sizeof(*c), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok())
        return -4;

    struct virtio_gpu_resource_attach_backing *a =
        (struct virtio_gpu_resource_attach_backing *)cmdbuf;
    hdr_init(a, VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING);
    a->resource_id = resource_id;
    a->nr_entries  = 1;
    struct virtio_gpu_mem_entry *e = (struct virtio_gpu_mem_entry *)(a + 1);
    e->addr    = fb_paddr;
    e->length  = (unsigned int)(w * h * 4);
    e->padding = 0;
    if (docmd(sizeof(*a) + sizeof(*e), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 ||
        !resp_ok())
        return -5;

    struct virtio_gpu_set_scanout *so = (struct virtio_gpu_set_scanout *)cmdbuf;
    hdr_init(so, VIRTIO_GPU_CMD_SET_SCANOUT);
    so->r.x = so->r.y = 0;
    so->r.width  = (unsigned int)w;
    so->r.height = (unsigned int)h;
    so->scanout_id  = 0;
    so->resource_id = resource_id;
    if (docmd(sizeof(*so), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok())
        return -6;

    width  = w;
    height = h;
    return 0;
}

// --- the control socket -------------------------------------------------------
//
// Lines of text, the same choice and the same reasons as
// docs/architecture/debugging.md's: it can be driven by hand, it needs no
// client to be useful, and a garbled answer is still mostly readable.
//
// The name is a *name*, not a path. AF_UNIX sockets in this kernel live in a
// table keyed by string and never touch the filesystem -- that is item 3 of
// docs/plan/foundations.md and it is not built. Nothing here depends on which
// it is, and the day it becomes a real path this keeps working.
#define VGPUD_SOCKET "/run/vgpud"

static void
reply(int fd, const char *s)
{
    int n = 0;
    while (s[n]) n++;
    write(fd, (void *)s, n);
}

static void
reply_dim(int fd, int w, int h, const char *tail)
{
    char b[48];
    int n = 0, i, v;

    v = w; i = 0; { char t[12]; if (!v) t[i++]='0'; while (v) { t[i++]='0'+v%10; v/=10; }
                    while (i-- > 0) b[n++] = t[i]; }
    b[n++] = 'x';
    v = h; i = 0; { char t[12]; if (!v) t[i++]='0'; while (v) { t[i++]='0'+v%10; v/=10; }
                    while (i-- > 0) b[n++] = t[i]; }
    b[n] = 0;
    reply(fd, b);
    reply(fd, tail);
}

static int
num(const char **sp)
{
    const char *s = *sp;
    int v = 0, any = 0;
    while (*s == ' ' || *s == 'x' || *s == '\t') s++;
    while (*s >= '0' && *s <= '9') { v = v * 10 + (*s++ - '0'); any = 1; }
    *sp = s;
    return any ? v : -1;
}

// The modes offered. Not a list the device publishes -- virtio-gpu has no mode
// table, it will scan out whatever size it is given -- so this is the set that
// fits the framebuffer, which is the only real constraint.
static const struct { int w, h; } offered[] = {
    { 1280, 800 }, { 1280, 720 }, { 1024, 768 }, { 800, 600 }, { 640, 480 },
};
#define NOFFERED ((int)(sizeof(offered) / sizeof(offered[0])))

static void
serve(int fd)
{
    char line[128];
    int  n;

    while ((n = read(fd, line, sizeof(line) - 1)) > 0) {
        const char *p = line;
        line[n] = 0;

        while (*p == ' ') p++;

        if (p[0] == 'i') {                       /* info */
            reply_dim(fd, width, height, " current\n");
            reply_dim(fd, max_w, max_h, " maximum, framebuffer ");
            {
                char b[16]; int i = 0, v = (int)(fb_bytes / 1024);
                char t[12]; int j = 0;
                if (!v) t[j++] = '0';
                while (v) { t[j++] = '0' + v % 10; v /= 10; }
                while (j-- > 0) b[i++] = t[j];
                b[i] = 0;
                reply(fd, b);
            }
            reply(fd, " KB\n");
        } else if (p[0] == 'm' && p[1] == 'o' && p[2] == 'd' && p[3] == 'e' &&
                   p[4] == 's') {                /* modes */
            for (int i = 0; i < NOFFERED; i++) {
                if ((unsigned int)(offered[i].w * offered[i].h * 4) > fb_bytes)
                    continue;
                reply_dim(fd, offered[i].w, offered[i].h,
                          (offered[i].w == width && offered[i].h == height)
                              ? "  *\n" : "\n");
            }
        } else if (p[0] == 'm') {                /* mode W H  or  mode WxH */
            int w, h;
            while (*p && *p != ' ') p++;
            w = num(&p);
            h = num(&p);
            if (w < 0 || h < 0) {
                reply(fd, "usage: mode <width> <height>\n");
            } else {
                int r = set_mode(w, h);
                if (r == -2) {
                    reply(fd, "refused: larger than the framebuffer, which is "
                              "fixed at startup\n");
                } else if (r < 0) {
                    reply(fd, "the device refused the modeset\n");
                } else {
                    draw_mark();
                    present();
                    reply_dim(fd, width, height, " ok\n");
                    puts("vgpud: mode is now "); putn(width); puts("x");
                    putn(height); puts("\n");
                }
            }
        } else if (p[0] == 'q') {                /* quit */
            return;
        } else {
            reply(fd, "info | modes | mode <w> <h> | quit\n");
        }
    }
}

static int
control_socket(void)
{
    struct sockaddr_un sa;
    int fd, i;

    fd = socket(AF_UNIX, SOCK_STREAM, 0);
    if (fd < 0)
        return -1;

    for (i = 0; i < (int)sizeof(sa.sun_path); i++)
        sa.sun_path[i] = 0;
    sa.sun_family = AF_UNIX;
    for (i = 0; VGPUD_SOCKET[i] && i < (int)sizeof(sa.sun_path) - 1; i++)
        sa.sun_path[i] = VGPUD_SOCKET[i];

    if (bind(fd, (struct sockaddr *)&sa, sizeof(sa)) < 0) {
        close(fd);
        return -1;
    }
    if (listen(fd, 4) < 0) {
        close(fd);
        return -1;
    }
    return fd;
}

// --- main -------------------------------------------------------------------

int
main(int argc, char *argv[])
{
    read_bootmode();

    pcifd = open("/dev/pci", O_RDWR);
    if (pcifd < 0)
        die("cannot open /dev/pci");

    int count = 0;
    if (ioctl(pcifd, PCIIOC_COUNT, &count) < 0)
        die("PCIIOC_COUNT failed");

    // Find it by what it is, not by where it is. A driver that takes a bus
    // address on the command line is one nobody can start automatically.
    index_ = -1;
    for (int i = 0; i < count; i++) {
        info.index = (unsigned int)i;
        if (ioctl(pcifd, PCIIOC_INFO, &info) < 0)
            continue;
        if (info.vendor == VIRTIO_VENDOR && info.device == VIRTIO_DEV_GPU) {
            index_ = i;
            break;
        }
    }
    if (index_ < 0)
        nothing_to_do("no virtio-gpu on this machine");

    if (verbose_boot) {
        puts("vgpud: virtio-gpu at "); puthex(info.bus, 2); puts(":");
        puthex(info.slot, 2); puts("."); puthex(info.func, 1);
        if (info.driver[0]) { puts(", currently held by "); puts(info.driver); }
        puts("\n");
    }

    // The kernel driver is asked to let go here. It puts the VBE framebuffer
    // back on screen before returning, so there is never a moment with no
    // display -- see vgpu_release() and vga_fb_restore().
    if (ioctl(pcifd, PCIIOC_CLAIM, &index_) < 0)
        nothing_to_do("claim refused -- the kernel driver would not release it");
    vputs("vgpud: claimed\n");

    // The info is re-read because claiming is what enables memory space and
    // bus mastering, and a BAR read before that can legitimately be zero.
    info.index = (unsigned int)index_;
    if (ioctl(pcifd, PCIIOC_INFO, &info) < 0)
        die("PCIIOC_INFO failed after claim");

    common = find_cfg(VIRTIO_PCI_CAP_COMMON_CFG, 0);
    notify = find_cfg(VIRTIO_PCI_CAP_NOTIFY_CFG, &notify_mult);
    isr    = find_cfg(VIRTIO_PCI_CAP_ISR_CFG, 0);
    if (!common || !notify)
        die("no modern virtio capabilities -- not a 1.0 device");

    bring_up();
    draw_mark();
    present();

    vputs("vgpud: pid ");
    vputn(getpid());
    vputs(" drew a test pattern from userspace\n");

    if (argc > 1 && argv[1][0] == '-' && argv[1][1] == 'x') {
        // Exit without releasing, to show what a crashed driver leaves behind:
        // the kernel takes the device back and the screen is whatever the
        // device was last told to scan out.
        puts("vgpud: exiting without cleaning up (simulated crash)\n");
        exit();
    }

    // Hold the device, and answer.
    //
    // This used to sleep for ever, with a comment arguing that building a
    // control socket before there is a compositor to judge it is the mistake
    // the plan warns about. That argument was right about a *compositor*
    // protocol and wrong as a reason to have no socket at all: H3 is a modeset
    // API, and a modeset has a consumer that exists -- somebody changing the
    // mode and looking at the screen. `schmode` is that consumer, and it is
    // enough to judge an interface this small.
    //
    // What is deliberately *not* here is anything a compositor would need:
    // no buffer submission, no damage, no vblank. Those wait for a compositor,
    // for the original reason.
    //
    // Blocking accept, one client at a time. A driver whose whole job is to
    // hold a device has nothing else to be doing, and a poll loop would be
    // machinery in place of a call.
    //
    // Being killed here is not an error path, it is the H2 path: schinit
    // restarts this process, and the device has to come back with it.
    int srv = control_socket();
    if (srv < 0) {
        vputs("vgpud: no control socket; the mode is fixed at "
             "whatever the device reported\n");
        for (;;)
            sleep(100);
    }

    vputs("vgpud: listening on " VGPUD_SOCKET "\n");
    for (;;) {
        int c = accept(srv, 0, 0);
        if (c < 0) {
            sleep(10);
            continue;
        }
        serve(c);
        close(c);
    }
}