gfx: draw the console in RAM, and write it to the card write-combined

On the first real machine with a discrete card (an RTX 3060, 800x600 from
the UEFI GOP) the console was slow enough to photograph mid-scroll, and the
boot died on CPU 1's "cpu1: starting": CPU 0 spun 200M times on the console
lock while CPU 1 held it, scrolling, and called it a deadlock.

The framebuffer is device memory in the uncached 3-4 GB page of the direct
map. QEMU's stdvga is RAM underneath, so nothing showed there; on a card,
gfx_scroll_up's memmove read the whole screen back across the bus one byte
at a time, and every anti-aliased glyph read back each pixel it blended.

So the console now draws into a copy of the screen in RAM (vmap_alloc: kalloc
pages behind a page table of their own, since nothing needs it contiguous),
and gfx_flush() copies the changed rectangle onto the card through an
ioremap_wc() mapping -- PAT entry 4 set to write-combining on every CPU.
Flushes happen where drawing ends: a message, a terminal write, an echo, a
terminal switch. Scrolling moves eight bytes at a time instead of one.

/dev/fb0 reads and writes the card itself, as before. The early console's
fb_info now carries the physical address it always should have.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4

docs/architecture/graphics.md

@@ -41,6 +41,27 @@ is now a second caller of one function rather than an edit to the crash screen.
 left running says so before any bus is enumerated, because the console draws
 long before PCI is walked.
 
+### Drawing in RAM
+
+The console does not draw on the card. Once there is a page allocator,
+`gfx_accelerate()` gives it a copy of the screen in RAM (`vmap_alloc()`, so it
+needs no contiguous memory) and maps the real framebuffer write-combined
+(`ioremap_wc()`, PAT entry 4). Every drawing routine reads and writes the copy;
+`gfx_flush()` copies the rectangle that changed onto the card, and is called
+where drawing ends — after a message, a `write()` to a terminal, an echo, a
+terminal switch, the crash screen — not where it happens.
+
+On QEMU's stdvga the framebuffer is RAM, so none of this was visible. On a real
+graphics card it is device memory behind the PCI hole, uncached: a scroll is a
+`memmove` that reads the whole screen back across the bus a byte at a time,
+and an anti-aliased glyph reads every pixel it blends. Measured on an RTX 3060
+at 800×600, one scroll took long enough to photograph half done, and a second
+CPU waiting to print declared the console lock deadlocked.
+
+`/dev/fb0` still reads and writes the card itself (`gfx_hw_ptr()`), and its
+`mmap` still maps the physical framebuffer, so a program drawing on the screen
+and `vgpud` mirroring it see what is on the screen.
+
 ### `display` — the class
 
 `struct display_ops`, found with `device_find_class(DEVCLASS_DISPLAY)`.

include/gfx.h

@@ -74,6 +74,13 @@ void gfx_putc(int c, uint32 color);
 void gfx_puts(int x, int y, char *s, uint32 color);
 void gfx_set_bg(uint32 color);
 void gfx_screen_reset(void);
+
+// See kernel/dev/gfx.c: drawing goes to a copy of the screen in RAM once
+// gfx_accelerate() has run, and reaches the display at gfx_flush().
+void gfx_accelerate(void);
+void gfx_flush(void);
+void gfx_flush_all(void);         // the crash screen: unconditionally
+volatile uchar *gfx_hw_ptr(void);
 int  gfx_cols(void);
 int  gfx_rows(void);
 

include/vmap.h

@@ -0,0 +1,21 @@
+#pragma once
+
+// Kernel mappings that are neither the direct map nor plain device registers.
+// Both take one 1 GB slot of the device window ioremap() hands out, so each is
+// limited to a gigabyte and meant to be called a handful of times at boot.
+
+// Program this CPU's PAT so that a mapping can ask for write-combining. Every
+// CPU runs it before it touches a write-combined mapping -- the PAT is per
+// processor, and one left at its reset value reads the same bits as
+// write-back, which on device memory the MTRRs then turn back into uncached.
+void  pat_init(void);
+
+// Device memory, write-combined: stores are gathered into bursts rather than
+// sent one at a time, and reads are as slow as uncached. For a framebuffer,
+// never for registers. Returns 0 if there is no window left.
+void *ioremap_wc(uint64 pa, uint64 size);
+
+// `size` bytes of ordinary RAM, contiguous in the kernel's address space and
+// not physically: kalloc() pages behind a page table of their own. Zeroed.
+// For buffers too big for the DMA arena that no device reads. Never freed.
+void *vmap_alloc(uint64 size);

kernel/arch/x86_64/vm.c

@@ -7,6 +7,7 @@
 #include "mmu.h"
 #include "proc.h"
 #include "ioperm.h"
+#include "vmap.h"
 
 int mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm);
 static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc);
@@ -336,6 +337,100 @@ ioremap(uint64 pa, uint64 size)
   return (void *)(KERNBASE + ((uint64)first << 30) + (pa - base));
 }
 
+// --- write-combining, and big buffers -----------------------------------------
+//
+// See include/vmap.h. The console used to draw straight into the framebuffer
+// through the direct map, which is uncached across the whole PCI hole: every
+// store was its own bus transaction and every read -- a scroll is a memmove,
+// so a read for every byte on screen -- waited on the card. In QEMU that is
+// free. On a real graphics card a single scroll took seconds, long enough
+// that a second CPU waiting to print gave up and called it a deadlock.
+
+#define MSR_PAT    0x277
+#define PAT_WC     0x01
+#define PDE_PAT    (1ULL << 12)    // in a 2 MB entry; PAT index 4 with PCD=PWT=0
+#define PGSIZE_2M  (2ULL << 20)
+
+void
+pat_init(void)
+{
+  // Entry 4 is write-back at reset and nothing here sets the PAT bit, so
+  // repurposing it changes no existing mapping. Entries 0-3, which every
+  // other mapping selects through PCD and PWT, keep their reset values.
+  uint64 pat = rdmsr(MSR_PAT);
+  pat = (pat & ~(0xFFULL << 32)) | ((uint64)PAT_WC << 32);
+  wrmsr(MSR_PAT, pat);
+}
+
+// A slot of the device window pointing at a fresh page directory.
+static pde_t *
+window_pd(int *slot)
+{
+  pde_t *pd;
+
+  if (ioremap_next >= IOREMAP_LAST_PDPT)
+    return 0;
+  if ((pd = (pde_t *)kalloc()) == 0)
+    return 0;
+  memset(pd, 0, PGSIZE);
+  *slot = ioremap_next++;
+  kpdpt[*slot] = v2p(pd) | PTE_P | PTE_W;
+  return pd;
+}
+
+void *
+ioremap_wc(uint64 pa, uint64 size)
+{
+  uint64 base = pa & ~(uint64)(PGSIZE_2M - 1);
+  uint64 n = (pa + size - base + PGSIZE_2M - 1) / PGSIZE_2M;
+  pde_t *pd;
+  int slot;
+
+  if (size == 0 || n > 512 || (pd = window_pd(&slot)) == 0)
+    return 0;
+  // An uncached alias of the same frames is the direct map's 3-4 GB page.
+  // Two uncached types for one device page is allowed; what the manuals
+  // warn against is a cached alias, and there is none.
+  for (uint64 i = 0; i < n; i++)
+    pd[i] = (base + i * PGSIZE_2M) | PTE_P | PTE_W | PTE_PS | PDE_PAT;
+  switchkvm();
+  return (void *)(KERNBASE + ((uint64)slot << 30) + (pa - base));
+}
+
+void *
+vmap_alloc(uint64 size)
+{
+  uint64 pages = (size + PGSIZE - 1) / PGSIZE;
+  pde_t *pd;
+  int slot;
+
+  if (size == 0 || pages > 512 * 512 || (pd = window_pd(&slot)) == 0)
+    return 0;
+  for (uint64 i = 0; i < pages; i++) {
+    pte_t *pt;
+    char *page;
+    if (pd[i / 512] == 0) {
+      if ((pt = (pte_t *)kalloc()) == 0)
+        goto oom;
+      memset(pt, 0, PGSIZE);
+      pd[i / 512] = v2p(pt) | PTE_P | PTE_W;
+    }
+    pt = (pte_t *)P2V(PTE_ADDR(pd[i / 512]));
+    if ((page = kalloc()) == 0)
+      goto oom;
+    memset(page, 0, PGSIZE);
+    pt[i % 512] = v2p(page) | PTE_P | PTE_W;
+  }
+  switchkvm();
+  return (void *)(KERNBASE + ((uint64)slot << 30));
+
+oom:
+  // Whatever was mapped stays mapped and is lost. This runs once at boot,
+  // and a machine out of memory by then has bigger problems.
+  cprintf("vmap: out of memory for %d KB\n", (int)(size / 1024));
+  return 0;
+}
+
 void
 switchuvm(struct proc *p)
 {

kernel/core/console.c

@@ -167,6 +167,7 @@ cprintf(char *fmt, ...)
   if (locking)
     acquire(&cons.lock);
   vcprintf(fmt, ap);
+  gfx_flush();
   if (locking)
     release(&cons.lock);
   va_end(ap);
@@ -200,6 +201,7 @@ kinfo(char *fmt, ...)
   vcprintf(fmt, ap);
   if (!boot_verbose)
     cons_logonly--;
+  gfx_flush();
   if (locking)
     release(&cons.lock);
   va_end(ap);
@@ -645,6 +647,7 @@ consoleintr(int (*getc)(void))
     }
   }
   release(&input_lock);
+  gfx_flush();                    // the echo
 }
 
 int
@@ -759,6 +762,7 @@ consolewrite(struct inode *ip, uint64 off, char *buf, int n)
   }
   for(i = 0; i < n; i++)
     vt_out(v, buf[i] & 0xff);
+  gfx_flush();
   release(&cons.lock);
 
   // And onto a second display, if the kernel drives one.

kernel/core/main.c

@@ -12,6 +12,7 @@
 #include "klog.h"
 #include "usb.h"
 #include "acpi.h"
+#include "vmap.h"
 
 static void *klog_base;   // the persistent log, just above the kernel image
 
@@ -138,6 +139,8 @@ kmain(uint32 fb_addr)
       bi.bpp    = *(uchar *)P2V(SCHBOOT_VBE_BPP);
     }
     bi.ptr  = (volatile uchar *)P2V(fb_phys);
+    bi.phys = fb_phys;
+    bi.size = (uint64)bi.stride * bi.height;
 
     // Only if the loader actually described something, and only if the direct
     // map reaches it. A framebuffer above 4 GB needs ioremap(), and ioremap()
@@ -201,6 +204,7 @@ kmain(uint32 fb_addr)
   dma_init(P2V(lowtop - DMA_ARENA_BYTES), P2V(lowtop));
   if (boot_verbose) cprintf("verbose boot: dma_init done\n");
   kvmalloc();
+  pat_init();
   if (boot_verbose) cprintf("verbose boot: kvmalloc done\n");
 
   // Now that the direct map reaches it, the rest of RAM can be allocated from.
@@ -231,6 +235,10 @@ kmain(uint32 fb_addr)
               (void *)fb_early);
     }
   }
+  // The console has been drawing straight onto the card. From here it draws
+  // into RAM and copies out. See gfx_flush().
+  gfx_accelerate();
+
   // ACPI first, the legacy MP table only if it finds nothing. Modern firmware
   // publishes a complete MADT and an MP table with one processor in it, so
   // asking the older interface first means never seeing more than one CPU.
@@ -409,6 +417,7 @@ mpenter(void)
   // AP keeps whatever mode the firmware left its APIC in across INIT, so the
   // boot processor having switched itself back says nothing about this one.
   lapic_xapic_mode();
+  pat_init();
   switchkvm();
   seginit();
   lapicinit();

kernel/dev/fb.c

@@ -67,7 +67,7 @@ pixel_format(struct fb_var_screeninfo *v, unsigned int bpp)
 int
 fbread(struct inode *ip, uint64 off, char *dst, int n)
 {
-  volatile uchar *fb = gfx_fb_ptr();
+  volatile uchar *fb = gfx_hw_ptr();
   uint64 size = gfx_fb_size();
 
   if (!fb)
@@ -101,7 +101,7 @@ fb_mapped(void)
 int
 fbwrite(struct inode *ip, uint64 off, char *src, int n)
 {
-  volatile uchar *fb = gfx_fb_ptr();
+  volatile uchar *fb = gfx_hw_ptr();
   uint64 size = gfx_fb_size();
 
   if (!fb)

kernel/dev/gfx.c

@@ -23,9 +23,8 @@
 #include "vt.h"
 #include "text.h"
 #include "schboot.h"
-#include "schboot.h"
-#include "memlayout.h"
 #include "ttf.h"
+#include "vmap.h"
 
 #define BACKSPACE 0x100
 #define COLOR_BG SCH_BG
@@ -86,6 +85,45 @@ static void   draw_glyph_px(int x, int y, uint32 cp, uint32 fg, uint32 bg);
 static struct fb_info scanout;
 static volatile uchar *fb;
 
+// Where `fb` points once gfx_accelerate() has run: a copy of the screen in
+// RAM, which every drawing routine reads and writes, and which gfx_flush()
+// copies onto the scanout -- `hw` -- a rectangle at a time.
+//
+// The framebuffer itself is device memory, uncached. Drawing into it directly
+// cost nothing in QEMU, where it is RAM, and on a real graphics card it was
+// the slowest thing the kernel did: a scroll reads every byte on the screen
+// back across the bus, one uncached read per byte, and an anti-aliased glyph
+// reads every pixel it blends. A screenful of boot log took long enough that
+// photographs of the monitor caught the scroll half done, and a second CPU
+// waiting for the console lock declared a deadlock. The copy is read instead,
+// and the card is only ever written -- through a write-combining mapping, so
+// a row goes out as bursts rather than one transaction per pixel.
+static volatile uchar *hw;
+static uchar  *shadow;
+static uint64  shadow_bytes;
+static uint64  wc_phys;            // the scanout ioremap_wc() mapped, and where
+static volatile uchar *wc_ptr;
+
+// What gfx_flush() still owes the scanout, in pixels, inclusive. Separate from
+// the row span below: that one is for a second screen and is taken by
+// somebody else.
+static int fl_x0 = -1, fl_y0, fl_x1, fl_y1;
+
+static void
+dirty(int x0, int y0, int x1, int y1)
+{
+  if (fb != (volatile uchar *)shadow)
+    return;
+  if (fl_x0 < 0) {
+    fl_x0 = x0; fl_y0 = y0; fl_x1 = x1; fl_y1 = y1;
+    return;
+  }
+  if (x0 < fl_x0) fl_x0 = x0;
+  if (y0 < fl_y0) fl_y0 = y0;
+  if (x1 > fl_x1) fl_x1 = x1;
+  if (y1 > fl_y1) fl_y1 = y1;
+}
+
 // What has been drawn since anyone last asked.
 //
 // The console framebuffer is one screen on a machine that may have two, and
@@ -133,7 +171,17 @@ void
 gfx_attach_scanout(const struct fb_info *info)
 {
   scanout = *info;
-  fb      = info->ptr;
+  hw      = info->ptr;
+  // The framebuffer gfx_accelerate() mapped write-combined, coming back --
+  // vga_fb_restore() hands over its own uncached pointer to the same frames.
+  if (wc_ptr && info->phys == wc_phys)
+    hw = scanout.ptr = wc_ptr;
+  fb = hw;
+  if (shadow && (uint64)scanout.stride * scanout.height <= shadow_bytes) {
+    fb = (volatile uchar *)shadow;
+    // A new screen has none of what the copy holds.
+    dirty(0, 0, (int)scanout.width - 1, (int)scanout.height - 1);
+  }
 
   // Tell the terminals how big the screen actually is. They were sized before
   // this driver existed -- the PCI bus is walked long after the terminals are
@@ -178,6 +226,7 @@ gfx_draw_pixel(int x, int y, uint32 color)
   uint32 offset = (y * scanout.stride) + (x * (scanout.bpp / 8));
 
   damage(y, y);
+  dirty(x, y, x, y);
 
   if (scanout.bpp == 32) {
     *(volatile uint32*)(fb + offset) = color;
@@ -219,6 +268,25 @@ gfx_puts(int x, int y, char *s, uint32 color)
   }
 }
 
+// The screen, `by` bytes towards the top. memmove() goes a byte at a time,
+// which against RAM is merely slow; eight at a time is what a scroll of a
+// screenful wants. Forward is safe because the destination is below the
+// source.
+static void
+move_up(uint64 by, uint64 n)
+{
+  volatile uchar *d = fb;
+  volatile uchar *s = fb + by;
+
+  if (((by | n | (uint64)fb) & 7) == 0) {
+    for (uint64 i = 0; i < n; i += 8)
+      *(volatile uint64 *)(d + i) = *(volatile uint64 *)(s + i);
+    return;
+  }
+  for (uint64 i = 0; i < n; i++)
+    d[i] = s[i];
+}
+
 void
 gfx_putc(int c, uint32 color)
 {
@@ -276,7 +344,7 @@ gfx_putc(int c, uint32 color)
     uint32 total_size = scanout.stride * scanout.height;
     uint32 copy_size = total_size - line_size;
 
-    memmove((void*)fb, (void*)(fb + line_size), copy_size);
+    move_up(line_size, copy_size);
 
     for (int y = scanout.height - CHAR_HEIGHT; y < scanout.height; y++) {
       for (int x = 0; x < scanout.width; x++) {
@@ -352,6 +420,7 @@ draw_glyph_px(int x, int y, uint32 cp, uint32 fg, uint32 bg)
 
   cell = surface_sub(&screen, x, y, cell_w, cell_h);
   text_draw_glyph(&cell, cfont, 0, cfont_baseline, cp, fg);
+  dirty(x, y, x + cell_w - 1, y + cell_h - 1);
 }
 
 void
@@ -481,8 +550,9 @@ gfx_scroll_up(uint32 bg)
   uint32 line_size  = scanout.stride * CHAR_HEIGHT;
   uint32 total_size = scanout.stride * scanout.height;
 
-  memmove((void*)fb, (void*)(fb + line_size), total_size - line_size);
+  move_up(line_size, total_size - line_size);
   damage(0, (int)scanout.height - 1);   // a scroll moves every row
+  dirty(0, 0, (int)scanout.width - 1, (int)scanout.height - 1);
 
   for (int y = (int)scanout.height - CHAR_HEIGHT; y < (int)scanout.height; y++)
     for (int x = 0; x < (int)scanout.width; x++)
@@ -496,6 +566,116 @@ gfx_screen_reset(void)
   gfx_clear(COLOR_BG);
   cursor_x = 0;
   cursor_y = 0;
+  gfx_flush();
+}
+
+// --- the copy in RAM ---------------------------------------------------------
+
+// Copy what has been drawn since the last flush onto the scanout.
+//
+// Called where drawing ends rather than where it happens: after a message, a
+// write(2) to a terminal, a terminal switch, the crash screen. A line of
+// output is then one rectangle however many glyphs it took.
+//
+// Not a lock. Two CPUs flushing at once would only copy the same pixels
+// twice, and a flush that finds another in progress leaves its damage for
+// the next one -- so it can be called from the panic path and from inside an
+// interrupt without either of them waiting on anything.
+static uint64 flushing;
+
+void
+gfx_flush(void)
+{
+  int x0, y0, x1, y1, bpp;
+
+  if (!hw || fb != (volatile uchar *)shadow || fl_x0 < 0)
+    return;
+  if (xchg(&flushing, 1) != 0)
+    return;
+
+  x0 = fl_x0; y0 = fl_y0; x1 = fl_x1; y1 = fl_y1;
+  fl_x0 = -1;
+
+  if (x0 < 0) x0 = 0;
+  if (y0 < 0) y0 = 0;
+  if (x1 >= (int)scanout.width)  x1 = (int)scanout.width - 1;
+  if (y1 >= (int)scanout.height) y1 = (int)scanout.height - 1;
+
+  bpp = (int)scanout.bpp / 8;
+  for (int y = y0; y <= y1 && x0 <= x1; y++) {
+    uint64 off = (uint64)y * scanout.stride + (uint64)x0 * bpp;
+    uint64 end = (uint64)y * scanout.stride + (uint64)(x1 + 1) * bpp;
+    // Bytes up to an eight-byte boundary, words, and bytes after. The
+    // source and the destination share an offset, so one alignment serves
+    // both.
+    while (off < end && (off & 7))
+      { hw[off] = shadow[off]; off++; }
+    for (; off + 8 <= end; off += 8)
+      *(volatile uint64 *)(hw + off) = *(uint64 *)(shadow + off);
+    for (; off < end; off++)
+      hw[off] = shadow[off];
+  }
+  // Write-combined stores sit in the CPU's buffers until something drains
+  // them. They would get there; this says when.
+  asm volatile("sfence" ::: "memory");
+  flushing = 0;
+}
+
+// The crash screen, whatever else was happening. The CPU that was flushing
+// may have been stopped halfway through by the panic, holding the flag for
+// ever, and a crash report that never reaches the screen is the one outcome
+// worse than a slow one.
+void
+gfx_flush_all(void)
+{
+  flushing = 0;
+  dirty(0, 0, (int)scanout.width - 1, (int)scanout.height - 1);
+  gfx_flush();
+}
+
+// Move drawing into RAM, and map the scanout write-combined.
+//
+// Once, from kmain, as soon as there is a page allocator. Everything drawn
+// before that went straight to the framebuffer, and is drawn again here from
+// the terminal's grid rather than read back from it -- reading the screen is
+// the slow thing this exists to stop doing.
+void
+gfx_accelerate(void)
+{
+  uint64 bytes;
+  void *wc;
+
+  if (!fb || shadow || scanout.phys == 0 || scanout.bpp < 8)
+    return;
+  bytes = (uint64)scanout.stride * scanout.height;
+
+  if ((wc = ioremap_wc(scanout.phys, bytes)) != 0) {
+    wc_phys = scanout.phys;
+    wc_ptr  = (volatile uchar *)wc;
+    hw = scanout.ptr = wc_ptr;
+    fb = hw;
+  }
+
+  if ((shadow = vmap_alloc(bytes)) == 0) {
+    kinfo("gfx: no memory for a copy of the screen; drawing to it directly\n");
+    return;
+  }
+  shadow_bytes = bytes;
+  fb = (volatile uchar *)shadow;
+
+  gfx_clear(gfx_bg);
+  vt_repaint(vt_current());
+  gfx_flush();
+  kinfo("gfx: %d KB copy of the screen in RAM, %s\n", (int)(bytes / 1024),
+        wc ? "write-combined to the card" : "copied out uncached");
+}
+
+// The scanout itself, for /dev/fb0: a program drawing on the screen draws on
+// the screen, not on the console's copy of it.
+volatile uchar *
+gfx_hw_ptr(void)
+{
+  return hw;
 }
 
 /* ------------------------------------------------------------------ */
@@ -646,6 +826,7 @@ gfx_fill_rect(int x0, int y0, int w, int h, uint32 color)
     return;
 
   damage(y0, y0 + h - 1);
+  dirty(x0, y0, x0 + w - 1, y0 + h - 1);
 
   // Only the 32bpp case gets the fast path; anything else is rare enough that
   // correctness through gfx_draw_pixel is the better trade.

kernel/dev/vt.c

@@ -842,6 +842,7 @@ vt_switch(int i)
   // needed to be: the outgoing terminal's grid has been current all along,
   // which is the whole reason the grid exists.
   vt_repaint(&vts[i]);
+  gfx_flush();
   return 0;
 }