// SPDX-License-Identifier: GPL-2.0-or-later // // vgpud -- the virtio-gpu driver, outside the kernel. // // docs/plan/hybrid.md's first eviction. The same device the kernel drives in // kernel/drivers/gpu/virtio_gpu/, driven instead by a process that asks the // kernel for four things and nothing else: // // PCIIOC_CLAIM take the device (the kernel driver is asked to let go) // PCIIOC_CFG read config space, to find the vendor capabilities // mmap(BAR) reach the registers // PCIIOC_DMA memory the device can be pointed at // // The point is not that this is better than the in-kernel driver -- it is the // same code in a worse language for the job. The point is that it is a real // driver rather than a test program, so the interface has something to be // judged by, and what it asks for is what a foreign driver would ask for too. // // Note what is *not* here: no syscall knows this is a GPU. The kernel handed // over a PCI device and this decided it was a GPU, which is the property that // makes the interface general rather than a hole shaped like one thing. #include "core/types.h" #include "uapi/user.h" #include "uapi/fcntl.h" #include "uapi/posix.h" #include "dev/pciuser.h" #include "dev/virtio.h" #include "uapi/schinit.h" #include "net/socket.h" #include "gfx/theme.h" // SCH_BG, SCH_ACCENT -- one palette, see docs/branding #include "gfx/logo.h" // the mark, the same bytes the loader and console draw #include "gfx/fb.h" // the console framebuffer, which this mirrors // lib/glue.c has these -- they exist because the compiler emits calls to them // whether or not anything asks -- but include/uapi/user.h never declared them. void *memcpy(void *, const void *, unsigned long); int memcmp(const void *, const void *, unsigned long); // --- small output, since lib/ has no printf --------------------------------- // How the machine was booted. A driver coming up is progress on a boot you are // watching and noise on every other one; a driver *failing* is neither, and // stays on puts(). See kinfo() in kernel/core/console.c for the same rule one // level down. static int verbose_boot; static void read_bootmode(void) { char b[16]; int fd = open("/proc/bootmode", O_RDONLY); int n; if (fd < 0) return; n = read(fd, b, sizeof(b) - 1); close(fd); if (n > 0) { b[n] = 0; verbose_boot = (b[0] == 'v'); } } // Whole lines, in one write() each. // // Every message here is assembled from pieces -- a prefix, a job's label, a // number -- and each piece used to be its own write(). The console keeps one // write() together, but nothing keeps two of them together, so a job starting // on another CPU landed in the middle of the line: "schinitd: starting vgpud: // os.schos.resolvconfno virtio-gpu on this machine" was this process and // vgpud taking turns a word at a time. The pieces now collect here and go out // at the newline. static char lbuf[512]; static int lbn; static void lflush(void) { if (lbn) write(1, lbuf, lbn); lbn = 0; } static void emit(const char *s, int n) { for (int i = 0; i < n; i++) { lbuf[lbn++] = s[i]; if (s[i] == '\n' || lbn == (int)sizeof lbuf) lflush(); } } static void puts(const char *s) { int n = 0; while (s[n]) n++; emit(s, n); } // Progress, as opposed to news. static void vputs(const char *s) { if (verbose_boot) puts(s); } static void putn(long long v); static void vputn(long long v) { if (verbose_boot) putn(v); } static void putn(long long v) { char b[24]; int i = 0; if (v < 0) { emit("-", 1); v = -v; } if (v == 0) { emit("0", 1); return; } while (v > 0) { b[i++] = '0' + (int)(v % 10); v /= 10; } while (i-- > 0) emit(&b[i], 1); } static void puthex(unsigned long long v, int digits) { static const char h[] = "0123456789abcdef"; char b[17]; for (int i = digits - 1; i >= 0; i--) { b[i] = h[v & 0xF]; v >>= 4; } emit(b, digits); } static void die(const char *why) { puts("vgpud: "); puts(why); puts("\n"); exit(); } // Not a failure: a report. There is no virtio-gpu here, or the kernel will not // give it up, and neither of those changes by trying again -- so this tells the // supervisor to stop, rather than being restarted every three seconds for the // rest of the uptime. See SCHINIT_EXIT_NORESTART in include/uapi/schinit.h. static void nothing_to_do(const char *why) { // Quiet on a quiet boot: "this machine has no virtio-gpu" is a fact about // the hardware and not a fault, and it is the same fact every time. It is // still an exit the supervisor is told about, which is what stops it being // restarted fifteen times in forty-five seconds. vputs("vgpud: "); vputs(why); vputs("\n"); lflush(); exit_group(SCHINIT_EXIT_NORESTART); } // --- register access -------------------------------------------------------- static inline unsigned char r8 (volatile unsigned char *p, int o) { return p[o]; } static inline void w8 (volatile unsigned char *p, int o, unsigned char v) { p[o] = v; } static inline unsigned short r16(volatile unsigned char *p, int o) { return *(volatile unsigned short *)(p + o); } static inline void w16(volatile unsigned char *p, int o, unsigned short v) { *(volatile unsigned short *)(p + o) = v; } static inline unsigned int r32(volatile unsigned char *p, int o) { return *(volatile unsigned int *)(p + o); } static inline void w32(volatile unsigned char *p, int o, unsigned int v) { *(volatile unsigned int *)(p + o) = v; } static inline void w64(volatile unsigned char *p, int o, unsigned long long v) { // Two 32-bit writes, low half first. The virtio spec requires it and the // device latches on the high write; a single 64-bit store is a different // bus transaction and some implementations simply drop it. w32(p, o, (unsigned int)v); w32(p, o + 4, (unsigned int)(v >> 32)); } // --- the device ------------------------------------------------------------- static int pcifd; static int index_; static volatile unsigned char *common; static volatile unsigned char *notify; static volatile unsigned char *isr; static unsigned int notify_mult; static struct pciuser_info info; static unsigned int cfg32(unsigned int off) { struct pciuser_cfg c; c.index = (unsigned int)index_; c.offset = off & ~3u; c.value = 0; if (ioctl(pcifd, PCIIOC_CFG, &c) < 0) die("config read failed"); return c.value; } static unsigned char cfg8(unsigned int off) { return (unsigned char)(cfg32(off) >> ((off & 3) * 8)); } // Find a virtio configuration structure. // // A virtio 1.0 device does not put its registers at a fixed offset. It walks // you there: a vendor capability in config space names which BAR, and how far // into it, each structure lives. So a driver that cannot read config space // cannot find its own registers -- which is why PCIIOC_CFG exists, and why the // test program that came before this never needed it. static volatile unsigned char * find_cfg(int want, unsigned int *out_mult) { unsigned char off = cfg8(0x34) & 0xFC; while (off) { unsigned char id = cfg8(off); unsigned char next = cfg8(off + 1); if (id == PCI_CAP_ID_VNDR) { unsigned char type = cfg8(off + 3); unsigned char bar = cfg8(off + 4); unsigned int o = cfg32(off + 8); unsigned int len = cfg32(off + 12); if (type == want && bar < PCIUSER_MAXBAR && info.bar[bar].base) { if (out_mult) *out_mult = cfg32(off + 16); // Map from the start of the BAR, not from the capability's // offset: two capabilities usually live in one BAR, and // mapping each separately would hand out two mappings of // overlapping pages. unsigned long long want_len = (unsigned long long)o + len; if (want_len < 4096) want_len = 4096; if (want_len > info.bar[bar].size) want_len = info.bar[bar].size; volatile unsigned char *p = (volatile unsigned char *)mmap(0, want_len, PROT_READ | PROT_WRITE, MAP_SHARED, pcifd, info.bar[bar].base); if (p == 0 || (long)p == -1) return 0; return p + o; } } off = next & 0xFC; } return 0; } // --- the queue -------------------------------------------------------------- #define CTRL_QUEUE 0 static struct vring_desc *desc; static struct vring_avail *avail; static struct vring_used *used; static unsigned long long ring_phys; static unsigned short qsize, notify_off, last_used, next_desc; static void *cmdbuf, *respbuf; static unsigned long long cmdphys, respphys; static void queue_setup(void) { w16(common, VCOMMON_Q_SELECT, CTRL_QUEUE); qsize = r16(common, VCOMMON_Q_SIZE); if (qsize == 0) die("control queue has no slots"); if (qsize > 64) qsize = 64; // more than one command in flight is not // something this driver has any use for w16(common, VCOMMON_Q_SIZE, qsize); notify_off = r16(common, VCOMMON_Q_NOTIFY_OFF); // One contiguous block for the three rings, laid out the way the spec // requires: descriptors, then available, then used, each aligned. The // device reads all three by physical address, so they have to come from // the kernel's DMA arena rather than from anything this process could // allocate for itself. unsigned int desc_sz = 16 * qsize; unsigned int avail_sz = 6 + 2 * qsize; struct pciuser_dma d; d.size = 16384; if (ioctl(pcifd, PCIIOC_DMA, &d) < 0) die("no DMA memory for the queue"); unsigned long long base = d.uaddr; unsigned long long basep = d.paddr; unsigned int off_avail = (desc_sz + 1) & ~1u; unsigned int off_used = (off_avail + avail_sz + 3) & ~3u; desc = (struct vring_desc *)(base); avail = (struct vring_avail *)(base + off_avail); used = (struct vring_used *)(base + off_used); ring_phys = basep; for (unsigned int i = 0; i < 16384; i++) ((volatile unsigned char *)base)[i] = 0; // Do not interrupt us. docmd() polls the used ring, so an interrupt is // not merely unwanted, it is dangerous: this process holds the device but // has no handler registered, and the line is shared with the e1000. A // level-triggered line nobody acknowledges re-fires for ever -- 77,000 // times a second, which is what the first working run of this driver did // before this line existed. The storm detector in kernel/dev/irq.c named // the line and the neighbour it was drowning. avail->flags = VRING_AVAIL_F_NO_INTERRUPT; __sync_synchronize(); w64(common, VCOMMON_Q_DESC, basep); w64(common, VCOMMON_Q_DRIVER, basep + off_avail); w64(common, VCOMMON_Q_DEVICE, basep + off_used); w16(common, VCOMMON_Q_ENABLE, 1); // Command and response buffers, also from the arena for the same reason. struct pciuser_dma b; b.size = 8192; if (ioctl(pcifd, PCIIOC_DMA, &b) < 0) die("no DMA memory for command buffers"); cmdbuf = (void *)b.uaddr; cmdphys = b.paddr; respbuf = (void *)(b.uaddr + 4096); respphys = b.paddr + 4096; } // Submit one command and wait for the answer. // // Polled, not interrupt-driven. This driver issues a handful of commands at // startup and one flush per frame, and a poll on a used-ring index that the // device updates by DMA is both simpler and lower-latency than waking a // process. The interrupt path exists (PCIIOC_IRQ) and belongs in whatever // wants to sleep between frames. static int docmd(unsigned int cmdlen, unsigned int resplen) { unsigned short head = next_desc; desc[head].addr = cmdphys; desc[head].len = cmdlen; desc[head].flags = VRING_DESC_F_NEXT; desc[head].next = (unsigned short)(head + 1); desc[head + 1].addr = respphys; desc[head + 1].len = resplen; desc[head + 1].flags = VRING_DESC_F_WRITE; desc[head + 1].next = 0; avail->ring[avail->idx % qsize] = head; __sync_synchronize(); avail->idx++; __sync_synchronize(); *(volatile unsigned short *)(notify + notify_off * notify_mult) = CTRL_QUEUE; // Spin with a bound. A device that never answers is a bug somewhere, and // hanging for ever is a worse way to report it than saying so. for (int spin = 0; spin < 100000000; spin++) { __sync_synchronize(); if (used->idx != last_used) { last_used = used->idx; next_desc = (unsigned short)((next_desc + 2) % qsize); return 0; } } return -1; } static void hdr_init(void *p, unsigned int type) { struct virtio_gpu_ctrl_hdr *h = (struct virtio_gpu_ctrl_hdr *)p; for (unsigned int i = 0; i < sizeof(*h); i++) ((unsigned char *)h)[i] = 0; h->type = type; } static int resp_ok(void) { struct virtio_gpu_ctrl_hdr *h = (struct virtio_gpu_ctrl_hdr *)respbuf; return h->type == VIRTIO_GPU_RESP_OK_NODATA || h->type == VIRTIO_GPU_RESP_OK_DISPLAY_INFO; } // --- bringing the device up ------------------------------------------------- static int width, height; static unsigned long long fb_uaddr, fb_paddr; static unsigned int fb_bytes; // what was allocated, not what is in use static int max_w, max_h; // the mode the allocation was sized for // The console, learned by console_open() and read by bring_up() one line // before it picks a mode. Declared up here rather than beside the mirror // because that ordering -- the console decides the mode, not the other way // round -- is the whole of why the head comes up the size the F8 menu asked // for. static volatile unsigned char *con_fb; // the console's framebuffer static unsigned con_w, con_h, con_stride; // The resource is recreated on every modeset; the framebuffer behind it is // not. PCIIOC_DMA has no free -- a driver's blocks come back when it exits and // not before -- so a modeset that allocated would exhaust sixteen slots and // then the arena, and the fault would look like a GPU bug rather than like // counting. Allocating once at the largest mode and reusing it for every // smaller one costs nothing: a 1024x768 frame is a prefix of a 1280x800 one. static unsigned int resource_id = 1; static void bring_up(void) { // Reset, then the status handshake in the order the spec sets out. A // device that finds the sequence wrong fails the feature step rather than // telling you which line was at fault, so it is written out plainly. w8(common, VCOMMON_STATUS, 0); while (r8(common, VCOMMON_STATUS) != 0) ; w8(common, VCOMMON_STATUS, VSTATUS_ACK); w8(common, VCOMMON_STATUS, VSTATUS_ACK | VSTATUS_DRIVER); w32(common, VCOMMON_DFEAT_SEL, 1); unsigned int hi = r32(common, VCOMMON_DFEAT); if (!(hi & (1u << (VIRTIO_F_VERSION_1 - 32)))) { w8(common, VCOMMON_STATUS, VSTATUS_FAILED); die("device is not virtio 1.0"); } w32(common, VCOMMON_GFEAT_SEL, 0); w32(common, VCOMMON_GFEAT, 0); w32(common, VCOMMON_GFEAT_SEL, 1); w32(common, VCOMMON_GFEAT, 1u << (VIRTIO_F_VERSION_1 - 32)); w8(common, VCOMMON_STATUS, VSTATUS_ACK | VSTATUS_DRIVER | VSTATUS_FEATURES_OK); if (!(r8(common, VCOMMON_STATUS) & VSTATUS_FEATURES_OK)) { w8(common, VCOMMON_STATUS, VSTATUS_FAILED); die("device refused the feature set"); } queue_setup(); w8(common, VCOMMON_STATUS, VSTATUS_ACK | VSTATUS_DRIVER | VSTATUS_FEATURES_OK | VSTATUS_DRIVER_OK); // What size is the screen? hdr_init(cmdbuf, VIRTIO_GPU_CMD_GET_DISPLAY_INFO); if (docmd(sizeof(struct virtio_gpu_ctrl_hdr), sizeof(struct virtio_gpu_resp_display_info)) < 0) die("the device never answered GET_DISPLAY_INFO"); if (!resp_ok()) die("GET_DISPLAY_INFO refused"); struct virtio_gpu_resp_display_info *di = (struct virtio_gpu_resp_display_info *)respbuf; width = (int)di->pmodes[0].r.width; height = (int)di->pmodes[0].r.height; if (width <= 0 || height <= 0) die("the device reports no display"); vputs("vgpud: display is "); vputn(width); vputs("x"); vputn(height); vputs("\n"); // But the console's mode wins, when there is a console. // // GET_DISPLAY_INFO answers with the size of the *host's window*, which the // person running QEMU drags to whatever they like and which has nothing to // do with the machine. The resolution the F8 menu offered at boot is the // one that was actually chosen, and it is the console's -- so a head that // mirrors the console and ignores that choice shows the right pixels in // the wrong mode, letterboxed or cropped, on a display that could simply // have been the right size. // // This is also the only chance to get it right without a second modeset: // the DMA allocation below is sized from these numbers and fixed for the // life of the process, so asking for the console's size here is what makes // `schmode` able to reach it later. if (con_w && con_h) { if ((int)con_w != width || (int)con_h != height) { vputs("vgpud: following the console at "); vputn((int)con_w); vputs("x"); vputn((int)con_h); vputs(" rather than the window\n"); } width = (int)con_w; height = (int)con_h; } // The framebuffer is claimed **before** the resource is created, because // the resource has to be created at a size we can actually back. // // virtio-gpu's display size is the size of the host's window, which the // user drags to whatever they like -- and every pixel of it is four bytes // of contiguous DMA that has to be found twice during the handover, once // for the kernel driver and once for this one. A big enough window does // not fit, and what that used to look like was not a message, it was a // loop: // // vgpud: wanted 7365 KB of DMA memory for a 1920x982 framebuffer // schinitd: restarting os.schos.vgpud // vgpud: wanted 7365 KB ... (and again, and again, until // schinit: supervisor stopped answering schinit replaced schinitd) // // So: ask for what the display says, and if that cannot be had, ask for // less. A smaller scanout on a larger display is a worse picture; a driver // that will not start is no picture and a supervisor fighting itself. { struct pciuser_dma d; int w = width, h = height, tries; for (tries = 0; ; tries++) { d.size = (unsigned long long)w * h * 4; if (ioctl(pcifd, PCIIOC_DMA, &d) >= 0) break; // Two thirds each time, on an eight-pixel grid. Not halving: // halving overshoots badly, and the arena is usually short by a // little rather than by a lot. w = ((w * 2 / 3) + 7) & ~7; h = ((h * 2 / 3) + 7) & ~7; if (w < 640 || h < 480) { puts("vgpud: wanted "); putn(((long long)width * height * 4) / 1024); puts(" KB of DMA memory for a "); putn(width); puts("x"); putn(height); puts(" framebuffer, and nothing down to 640x480\n"); puts("vgpud: `schdbg dma` shows the arena -- this is " "DMA_ARENA_BYTES, not fragmentation\n"); // Not die(): a died driver is restarted, and restarting will // fail identically for ever. The kernel keeps the VBE console, // which is a working screen. nothing_to_do("no room for a framebuffer; the console keeps the " "VBE framebuffer"); } } if (tries) { puts("vgpud: the display is "); putn(width); puts("x"); putn(height); puts(" but only "); putn(w); puts("x"); putn(h); puts(" of DMA could be had; using that\n"); width = w; height = h; } fb_uaddr = d.uaddr; fb_paddr = d.paddr; } fb_bytes = (unsigned int)(width * height * 4); max_w = width; max_h = height; // A 2D resource the size of the screen. struct virtio_gpu_resource_create_2d *c = (struct virtio_gpu_resource_create_2d *)cmdbuf; hdr_init(c, VIRTIO_GPU_CMD_RESOURCE_CREATE_2D); c->resource_id = resource_id; c->format = VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM; c->width = (unsigned int)width; c->height = (unsigned int)height; if (docmd(sizeof(*c), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok()) die("RESOURCE_CREATE_2D refused"); // The framebuffer was claimed above, before the resource was created. struct virtio_gpu_resource_attach_backing *a = (struct virtio_gpu_resource_attach_backing *)cmdbuf; hdr_init(a, VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING); a->resource_id = resource_id; a->nr_entries = 1; // The entries follow the struct rather than being in it: the count is // variable, so the command is a header with an array glued on the end. struct virtio_gpu_mem_entry *e = (struct virtio_gpu_mem_entry *)(a + 1); e->addr = fb_paddr; e->length = fb_bytes; e->padding = 0; if (docmd(sizeof(*a) + sizeof(*e), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok()) die("RESOURCE_ATTACH_BACKING refused"); // Put it on the screen. struct virtio_gpu_set_scanout *so = (struct virtio_gpu_set_scanout *)cmdbuf; hdr_init(so, VIRTIO_GPU_CMD_SET_SCANOUT); so->r.x = so->r.y = 0; so->r.width = (unsigned int)width; so->r.height = (unsigned int)height; so->scanout_id = 0; so->resource_id = resource_id; if (docmd(sizeof(*so), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok()) die("SET_SCANOUT refused"); if (verbose_boot) { puts("vgpud: scanout is ours, framebuffer at physical 0x"); puthex(fb_paddr, 16); puts("\n"); } } // Push what has been drawn to the host. Two commands, and both are needed: // the transfer copies from the backing into the device's copy of the resource, // the flush tells it to put that on screen. // // A rectangle rather than the screen, because the mirror below repaints a few // changed rows at a time and a full-screen transfer of a 1280x800 resource to // move one line of text is most of a megabyte for nothing. `offset` is where // the rectangle's top-left pixel sits in the *backing*, which the device // cannot work out for itself -- getting it wrong shifts the update diagonally // rather than failing, so it is written once here and nowhere else. static void present_rect(int x, int y, int w, int h) { if (w <= 0 || h <= 0) return; struct virtio_gpu_transfer_to_host_2d *t = (struct virtio_gpu_transfer_to_host_2d *)cmdbuf; hdr_init(t, VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D); t->r.x = (unsigned int)x; t->r.y = (unsigned int)y; t->r.width = (unsigned int)w; t->r.height = (unsigned int)h; t->offset = ((unsigned long long)y * (unsigned)width + x) * 4; t->resource_id = resource_id; t->padding = 0; if (docmd(sizeof(*t), sizeof(struct virtio_gpu_ctrl_hdr)) < 0) die("TRANSFER_TO_HOST_2D never answered"); struct virtio_gpu_resource_flush *f = (struct virtio_gpu_resource_flush *)cmdbuf; hdr_init(f, VIRTIO_GPU_CMD_RESOURCE_FLUSH); f->r.x = (unsigned int)x; f->r.y = (unsigned int)y; f->r.width = (unsigned int)w; f->r.height = (unsigned int)h; f->resource_id = resource_id; f->padding = 0; if (docmd(sizeof(*f), sizeof(struct virtio_gpu_ctrl_hdr)) < 0) die("RESOURCE_FLUSH never answered"); } static void present(void) { present_rect(0, 0, width, height); } // --- something to look at --------------------------------------------------- // The mark, on the theme ground. // // This used to be a test card: three colour bands, four gradients, a // checkerboard, and this instance's pid in binary along the bottom edge. That // was scaffolding for hybrid.md's H2 and H3 -- a screenshot had to answer "did // a *userspace process* put this here", and "did the driver come back" needed // a second run to look different from a first that never stopped. Both // milestones are closed, and what was left was a garish test pattern on a // second display with no explanation of itself. // // The mark answers the same question a black screen cannot -- something is // driving this device -- and answers it as the system rather than as a test // rig. Same include/gfx/logo.h the loader splash and the kernel console draw, so // there is one mark and one set of bytes. static void draw_mark(void) { volatile unsigned int *fb = (volatile unsigned int *)fb_uaddr; int x0 = (width - LOGO_W) / 2; int y0 = (height - LOGO_H) / 2; for (int y = 0; y < height; y++) for (int x = 0; x < width; x++) fb[y * width + x] = SCH_BG; if (x0 < 0 || y0 < 0) return; // a display smaller than the mark: leave it for (int dy = 0; dy < LOGO_H; dy++) { for (int dx = 0; dx < LOGO_W; dx++) { unsigned ink = LOGO_INK(dy * LOGO_W + dx); if (!ink) continue; // the mask is a square; the mark is not // Coverage, not a stencil: the mark has soft edges, and stamping // them as solid would show every one of them as a staircase. unsigned r = (((SCH_ACCENT >> 16) & 0xFF) * ink + ((SCH_BG >> 16) & 0xFF) * (255 - ink)) / 255; unsigned g = (((SCH_ACCENT >> 8) & 0xFF) * ink + ((SCH_BG >> 8) & 0xFF) * (255 - ink)) / 255; unsigned b = (((SCH_ACCENT ) & 0xFF) * ink + ((SCH_BG ) & 0xFF) * (255 - ink)) / 255; fb[(y0 + dy) * width + (x0 + dx)] = (r << 16) | (g << 8) | b; } } } // --- the mirror -------------------------------------------------------------- // // What this display is *for*, which the mark alone never answered. // // The mark said "a userspace process is driving this device", which was the // question H2 and H3 asked and is not a question any more. What was left was // a second monitor showing a logo for ever, which is indistinguishable from a // second monitor that does not work -- and that is exactly what it got called. // // So: the virtio-gpu head shows the console. Not because mirroring is the // endpoint -- a compositor (Gate 3) will want this head for itself, and when // there is one this goes away -- but because until then the honest thing for // a working display to show is the system that is running on it. // // The mechanism is a *copy with a diff*, and the diff is what makes it cheap // enough to run in a loop. The console framebuffer is mapped read-only-ish // through /dev/fb0; the virtio backing store is already mapped. The backing // therefore holds, at all times, exactly what was last sent to the device -- // which means it doubles as the previous frame, and no third buffer is needed // to answer "what changed". Rows that compare equal are skipped; the run of // rows that did not is copied and transferred as one rectangle. // // Mapping /dev/fb0 does not take the screen away from the console: the // mapping is of the same physical pages the console draws into, and the // kernel's fb_mapped() only arranges for a redraw when the fd is *closed*. // This one is never closed. // Ten passes a second. Fast enough that typing does not feel posted from // elsewhere, slow enough that the comparison -- a few megabytes of memcmp // against rows that are usually identical -- is not what the machine spends // its time on. #define MIRROR_MS 100 #define MIRROR_TICKS 10 // the same interval, in sleep()'s units static int mirror_x, mirror_y, mirror_w, mirror_h; static int mirror_full; // next pass copies every row // Where the console lands on this display. The two are not the same size -- // the console is whatever mode the firmware left, this head is whatever mode // it was last told -- so the smaller of each dimension is what gets copied. // // A head larger than the console centres it, on the theme ground draw_mark() // leaves behind. A head smaller than the console shows its top-left corner, // which is not a compromise: a terminal fills from the top-left, so that is // where everything worth seeing is. Scaling is the alternative and it is the // wrong one -- text resampled to a non-integer ratio is text you cannot read. // // Called again after every modeset, because both halves of that arithmetic // move. static void mirror_geometry(void) { mirror_w = (int)con_w < width ? (int)con_w : width; mirror_h = (int)con_h < height ? (int)con_h : height; mirror_x = (width - mirror_w) / 2; mirror_y = (height - mirror_h) / 2; mirror_full = 1; } // Learn the console: its size, its format, and where its pixels are. // // Called *before* the device is brought up, because the answer decides what // mode to bring it up in -- see bring_up(). Every failure here is a reason to // keep showing the mark rather than a reason to die: a machine with no // framebuffer console is a machine this daemon still has to hold the device // on. static int console_open(void) { struct fb_var_screeninfo var; struct fb_fix_screeninfo fix; unsigned char *p; int fd = open("/dev/fb0", O_RDWR); if (fd < 0) return -1; if (ioctl(fd, FBIOGET_VSCREENINFO, &var) < 0 || ioctl(fd, FBIOGET_FSCREENINFO, &fix) < 0) { close(fd); return -1; } // The copy below is a run of 32-bit words, so the two formats have to // agree exactly rather than approximately. They do today -- both sides are // XRGB8888 -- and if a console ever comes up 16-bit or BGR, converting per // pixel is a different function and this one should decline instead of // painting the screen in the wrong colours. if (var.bits_per_pixel != 32 || var.red.offset != 16 || var.green.offset != 8 || var.blue.offset != 0) { vputs("vgpud: console is not XRGB8888; not mirroring it\n"); close(fd); return -1; } p = mmap(0, fix.smem_len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if (p == 0 || (long)p < 0) { close(fd); return -1; } con_fb = p; con_w = var.xres; con_h = var.yres; con_stride = fix.line_length; // NOT xres*4; the card may pad each row return 0; // fd deliberately left open; see above } // One pass. Cheap when nothing changed, which is almost always. static void mirror_pass(void) { volatile unsigned int *dst; int y, first = -1, last = -1; unsigned rowbytes; if (!con_fb) return; dst = (volatile unsigned int *)fb_uaddr; rowbytes = (unsigned)mirror_w * 4; for (y = 0; y < mirror_h; y++) { const void *src = (const void *)(con_fb + (unsigned long)y * con_stride); void *row = (void *)&dst[(unsigned long)(mirror_y + y) * width + mirror_x]; if (!mirror_full && memcmp(row, src, rowbytes) == 0) continue; memcpy(row, src, rowbytes); if (first < 0) first = y; last = y; } mirror_full = 0; // One rectangle covering every changed row, rather than one per run. Text // scrolling dirties everything anyway, and a cursor blinking at the bottom // of a full screen is the worst case -- two rows apart, one transfer of // the span between them. Chasing that with a list of runs is more code // than the copy it saves. if (first >= 0) present_rect(mirror_x, mirror_y + first, mirror_w, last - first + 1); } // Tell the kernel where the scanout is, so it can put a crash screen there. // // The one thing a userspace display driver cannot do for itself. When the // machine panics this process does not get another turn -- there is no // scheduling after a panic -- so the head it owns would sit there showing a // shell prompt from before the crash, which is the most misleading thing a // screen can do. // // So the kernel is told, in advance, where to paint: it writes the report into // this framebuffer and asks the kernel driver it took this device from to // reset it and flush one frame. See PCIIOC_PANICFB in include/dev/pciuser.h. // // Called again after every modeset, because the size is half of what was // registered. Failing is not fatal to anything: it costs the crash screen on // this head and nothing else, which is exactly what the situation was before // the kernel could do this at all. static void register_panic_fb(void) { struct pciuser_panicfb pfb; pfb.index = (unsigned int)index_; pfb.paddr = fb_paddr; pfb.width = (unsigned int)width; pfb.height = (unsigned int)height; pfb.stride = (unsigned int)width * 4; if (ioctl(pcifd, PCIIOC_PANICFB, &pfb) < 0) vputs("vgpud: the kernel would not take a panic framebuffer; " "a crash will leave this head as it is\n"); } // --- the modeset -------------------------------------------------------------- // // docs/plan/hybrid.md milestone H3: *"a modeset API, which is the thing an // out-of-kernel GPU driver makes affordable and an in-kernel one never got."* // // The claim is worth testing rather than repeating, so here is the whole of // it: five commands to the device, no kernel change, and a driver that can be // killed and restarted if it goes wrong. In the kernel the same sequence would // be five commands *and* a reason to be nervous, because getting it wrong // takes the machine rather than one process -- which is precisely why the // in-kernel driver never grew one. static int set_mode(int w, int h) { if (w <= 0 || h <= 0) return -1; // Bounded by the allocation, not by the device. See fb_bytes. if ((unsigned int)(w * h * 4) > fb_bytes) return -2; // Let go of the old resource first. The backing pages are ours and survive // this; only the device's view of them goes away. struct virtio_gpu_resource_unref *u = (struct virtio_gpu_resource_unref *)cmdbuf; hdr_init(u, VIRTIO_GPU_CMD_RESOURCE_UNREF); u->resource_id = resource_id; u->padding = 0; if (docmd(sizeof(*u), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok()) return -3; // A fresh id rather than the same one again: the spec does not promise the // old id is reusable the instant the unref completes, and an id is free. resource_id++; struct virtio_gpu_resource_create_2d *c = (struct virtio_gpu_resource_create_2d *)cmdbuf; hdr_init(c, VIRTIO_GPU_CMD_RESOURCE_CREATE_2D); c->resource_id = resource_id; c->format = VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM; c->width = (unsigned int)w; c->height = (unsigned int)h; if (docmd(sizeof(*c), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok()) return -4; struct virtio_gpu_resource_attach_backing *a = (struct virtio_gpu_resource_attach_backing *)cmdbuf; hdr_init(a, VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING); a->resource_id = resource_id; a->nr_entries = 1; struct virtio_gpu_mem_entry *e = (struct virtio_gpu_mem_entry *)(a + 1); e->addr = fb_paddr; e->length = (unsigned int)(w * h * 4); e->padding = 0; if (docmd(sizeof(*a) + sizeof(*e), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok()) return -5; struct virtio_gpu_set_scanout *so = (struct virtio_gpu_set_scanout *)cmdbuf; hdr_init(so, VIRTIO_GPU_CMD_SET_SCANOUT); so->r.x = so->r.y = 0; so->r.width = (unsigned int)w; so->r.height = (unsigned int)h; so->scanout_id = 0; so->resource_id = resource_id; if (docmd(sizeof(*so), sizeof(struct virtio_gpu_ctrl_hdr)) < 0 || !resp_ok()) return -6; width = w; height = h; return 0; } // --- the control socket ------------------------------------------------------- // // Lines of text, the same choice and the same reasons as // docs/architecture/debugging.md's: it can be driven by hand, it needs no // client to be useful, and a garbled answer is still mostly readable. // // The name is a *name*, not a path. AF_UNIX sockets in this kernel live in a // table keyed by string and never touch the filesystem -- that is item 3 of // docs/plan/foundations.md and it is not built. Nothing here depends on which // it is, and the day it becomes a real path this keeps working. #define VGPUD_SOCKET "/run/vgpud" static void reply(int fd, const char *s) { int n = 0; while (s[n]) n++; write(fd, (void *)s, n); } static void reply_dim(int fd, int w, int h, const char *tail) { char b[48]; int n = 0, i, v; v = w; i = 0; { char t[12]; if (!v) t[i++]='0'; while (v) { t[i++]='0'+v%10; v/=10; } while (i-- > 0) b[n++] = t[i]; } b[n++] = 'x'; v = h; i = 0; { char t[12]; if (!v) t[i++]='0'; while (v) { t[i++]='0'+v%10; v/=10; } while (i-- > 0) b[n++] = t[i]; } b[n] = 0; reply(fd, b); reply(fd, tail); } static int num(const char **sp) { const char *s = *sp; int v = 0, any = 0; while (*s == ' ' || *s == 'x' || *s == '\t') s++; while (*s >= '0' && *s <= '9') { v = v * 10 + (*s++ - '0'); any = 1; } *sp = s; return any ? v : -1; } // The modes offered. Not a list the device publishes -- virtio-gpu has no mode // table, it will scan out whatever size it is given -- so this is the set that // fits the framebuffer, which is the only real constraint. static const struct { int w, h; } offered[] = { { 1280, 800 }, { 1280, 720 }, { 1024, 768 }, { 800, 600 }, { 640, 480 }, }; #define NOFFERED ((int)(sizeof(offered) / sizeof(offered[0]))) static void serve(int fd) { char line[128]; int n; while ((n = read(fd, line, sizeof(line) - 1)) > 0) { const char *p = line; line[n] = 0; while (*p == ' ') p++; if (p[0] == 'i') { /* info */ reply_dim(fd, width, height, " current\n"); reply_dim(fd, max_w, max_h, " maximum, framebuffer "); { char b[16]; int i = 0, v = (int)(fb_bytes / 1024); char t[12]; int j = 0; if (!v) t[j++] = '0'; while (v) { t[j++] = '0' + v % 10; v /= 10; } while (j-- > 0) b[i++] = t[j]; b[i] = 0; reply(fd, b); } reply(fd, " KB\n"); } else if (p[0] == 'm' && p[1] == 'o' && p[2] == 'd' && p[3] == 'e' && p[4] == 's') { /* modes */ for (int i = 0; i < NOFFERED; i++) { if ((unsigned int)(offered[i].w * offered[i].h * 4) > fb_bytes) continue; reply_dim(fd, offered[i].w, offered[i].h, (offered[i].w == width && offered[i].h == height) ? " *\n" : "\n"); } } else if (p[0] == 'm') { /* mode W H or mode WxH */ int w, h; while (*p && *p != ' ') p++; w = num(&p); h = num(&p); if (w < 0 || h < 0) { reply(fd, "usage: mode \n"); } else { int r = set_mode(w, h); if (r == -2) { reply(fd, "refused: larger than the framebuffer, which is " "fixed at startup\n"); } else if (r < 0) { reply(fd, "the device refused the modeset\n"); } else { // The ground first, then the console re-laid onto it: the // mirror's placement is derived from both sizes, and one // of them just changed. register_panic_fb(); draw_mark(); mirror_geometry(); mirror_pass(); present(); reply_dim(fd, width, height, " ok\n"); puts("vgpud: mode is now "); putn(width); puts("x"); putn(height); puts("\n"); } } } else if (p[0] == 'q') { /* quit */ return; } else { reply(fd, "info | modes | mode | quit\n"); } } } static int control_socket(void) { struct sockaddr_un sa; int fd, i; fd = socket(AF_UNIX, SOCK_STREAM, 0); if (fd < 0) return -1; for (i = 0; i < (int)sizeof(sa.sun_path); i++) sa.sun_path[i] = 0; sa.sun_family = AF_UNIX; for (i = 0; VGPUD_SOCKET[i] && i < (int)sizeof(sa.sun_path) - 1; i++) sa.sun_path[i] = VGPUD_SOCKET[i]; // The node from the last time this ran, if there is one. // // /run is a directory on the root filesystem rather than a tmpfs, so a // socket node outlives the process that bound it *and* the boot it was // bound in. bind() onto an existing name fails, and what that produced was // a vgpud holding the device with no way to talk to it -- on every boot // after the first, on any image whose root is writable. `schmode` said // "it is not running" about a process that was running perfectly well. // // Unlinking first is what a Unix daemon does, and the usual objection -- // that it lets a second instance steal a live socket -- does not apply // here: two vgpuds cannot coexist anyway, because the first holds the PCI // claim and the second is refused the device long before it reaches this // function. The claim is the mutual exclusion; this node never was. unlink(VGPUD_SOCKET); if (bind(fd, (struct sockaddr *)&sa, sizeof(sa)) < 0) { close(fd); return -1; } if (listen(fd, 4) < 0) { close(fd); return -1; } return fd; } // --- main ------------------------------------------------------------------- int main(int argc, char *argv[]) { read_bootmode(); pcifd = open("/dev/pci", O_RDWR); if (pcifd < 0) die("cannot open /dev/pci"); int count = 0; if (ioctl(pcifd, PCIIOC_COUNT, &count) < 0) die("PCIIOC_COUNT failed"); // Find it by what it is, not by where it is. A driver that takes a bus // address on the command line is one nobody can start automatically. index_ = -1; for (int i = 0; i < count; i++) { info.index = (unsigned int)i; if (ioctl(pcifd, PCIIOC_INFO, &info) < 0) continue; if (info.vendor == VIRTIO_VENDOR && info.device == VIRTIO_DEV_GPU) { index_ = i; break; } } if (index_ < 0) nothing_to_do("no virtio-gpu on this machine"); if (verbose_boot) { puts("vgpud: virtio-gpu at "); puthex(info.bus, 2); puts(":"); puthex(info.slot, 2); puts("."); puthex(info.func, 1); if (info.driver[0]) { puts(", currently held by "); puts(info.driver); } puts("\n"); } // The kernel driver is asked to let go here. It puts the VBE framebuffer // back on screen before returning, so there is never a moment with no // display -- see vgpu_release() and vga_fb_restore(). if (ioctl(pcifd, PCIIOC_CLAIM, &index_) < 0) nothing_to_do("claim refused -- the kernel driver would not release it"); vputs("vgpud: claimed\n"); // The info is re-read because claiming is what enables memory space and // bus mastering, and a BAR read before that can legitimately be zero. info.index = (unsigned int)index_; if (ioctl(pcifd, PCIIOC_INFO, &info) < 0) die("PCIIOC_INFO failed after claim"); common = find_cfg(VIRTIO_PCI_CAP_COMMON_CFG, 0); notify = find_cfg(VIRTIO_PCI_CAP_NOTIFY_CFG, ¬ify_mult); isr = find_cfg(VIRTIO_PCI_CAP_ISR_CFG, 0); if (!common || !notify) die("no modern virtio capabilities -- not a 1.0 device"); // Before bring_up(), which needs to know the console's size to pick a // mode. A machine with no console to mirror brings up whatever the device // reports, as it always did. int mirroring = console_open() == 0; bring_up(); register_panic_fb(); draw_mark(); if (mirroring) mirror_geometry(); present(); vputs("vgpud: pid "); vputn(getpid()); vputs(" drew the mark from userspace\n"); // From here the head shows the console rather than the mark. The mark // stays on screen if there is nothing to mirror, and stays as the ground // around the console when the two sizes do not match. if (mirroring) { mirror_pass(); vputs("vgpud: mirroring the console\n"); } if (argc > 1 && argv[1][0] == '-' && argv[1][1] == 'x') { // Exit without releasing, to show what a crashed driver leaves behind: // the kernel takes the device back and the screen is whatever the // device was last told to scan out. puts("vgpud: exiting without cleaning up (simulated crash)\n"); exit(); } // Hold the device, and answer. // // This used to sleep for ever, with a comment arguing that building a // control socket before there is a compositor to judge it is the mistake // the plan warns about. That argument was right about a *compositor* // protocol and wrong as a reason to have no socket at all: H3 is a modeset // API, and a modeset has a consumer that exists -- somebody changing the // mode and looking at the screen. `schmode` is that consumer, and it is // enough to judge an interface this small. // // What is deliberately *not* here is anything a compositor would need: // no buffer submission, no damage, no vblank. Those wait for a compositor, // for the original reason. // // Blocking accept, one client at a time. A driver whose whole job is to // hold a device has nothing else to be doing, and a poll loop would be // machinery in place of a call. // // Being killed here is not an error path, it is the H2 path: schinit // restarts this process, and the device has to come back with it. int srv = control_socket(); if (srv < 0) { vputs("vgpud: no control socket; the mode is fixed at " "whatever the device reported\n"); for (;;) { sleep(MIRROR_TICKS); mirror_pass(); } } vputs("vgpud: listening on " VGPUD_SOCKET "\n"); // Not a blocking accept any more. That was right when holding the device // was the whole job; now there is a second thing to do between calls, and // poll() is how a process waits for one of two reasons to wake up. The // timeout is the frame interval: with nothing happening, this wakes ten // times a second, compares rows, finds none changed, and goes back to // sleep. struct pollfd pf; pf.fd = srv; pf.events = POLLIN; for (;;) { pf.revents = 0; if (poll(&pf, 1, MIRROR_MS) > 0 && (pf.revents & POLLIN)) { int c = accept(srv, 0, 0); if (c >= 0) { serve(c); close(c); } } mirror_pass(); } }