diff --git a/Makefile b/Makefile index 89d4060..6335ead 100644 --- a/Makefile +++ b/Makefile @@ -724,9 +724,10 @@ qemu-iso: schos.iso # where a function is, the table is silently wrong for every address, and a # debugger that lies is worse than one that says nothing -- so the build stops. KERN_LINK = $(BUILD)/kernel/arch/x86_64/entry.o $(KERN_NOENTRY) \ - $(BUILD)/kernel/initcode_obj $(BUILD)/kernel/entryother_obj $(RAMDISK_OBJ) + $(BUILD)/kernel/initcode_obj $(BUILD)/kernel/entryother_obj \ + $(BUILD)/kernel/consfont_obj $(RAMDISK_OBJ) -$(BUILD)/kernel/kernel.pass1: $(KERNOBJS) $(BUILD)/kernel/initcode_obj $(BUILD)/kernel/entryother_obj $(RAMDISK_OBJ) kernel/kernel.ld +$(BUILD)/kernel/kernel.pass1: $(KERNOBJS) $(BUILD)/kernel/initcode_obj $(BUILD)/kernel/entryother_obj $(BUILD)/kernel/consfont_obj $(RAMDISK_OBJ) kernel/kernel.ld @mkdir -p $(BUILD)/kernel $(LD) $(LDFLAGS) -T kernel/kernel.ld -o $@ $(KERN_LINK) @@ -772,6 +773,32 @@ $(BUILD)/kernel/ramdisk_obj: $(BUILD)/fs-ram.img --redefine-sym _binary_build_fs_ram_img_size=_binary_ramdisk_size \ $@ +# The console font, linked into the kernel. +# +# In the kernel image rather than loaded from disk, because the console has to +# work before there is a disk: a panic during boot has to be readable, and the +# only alternative is keeping the 8x16 bitmap around for early messages and +# switching later, which means two fonts and a seam where they disagree. +# +# 115 KB of kernel for every glyph in a modern font is a trade this makes +# gladly. The 8x16 bitmap it replaces is CP437 -- 256 glyphs, no Ą, no Ę, no Ś, +# no Ω -- and they did not render as something wrong, they rendered as `?`, +# because a codepoint the grid could not hold was mapped to one. +# +# JetBrains Mono is SIL OFL 1.1, which permits this and requires the name be +# kept; see NOTICE. The regular static weight, not the variable font -- a +# variable font needs `fvar`/`gvar` interpolation, which ttf.c does not do and +# which would render every glyph at the wrong weight without saying so. +CONSFONT = ports/JetBrains_Mono/static/JetBrainsMono-Regular.ttf + +$(BUILD)/kernel/consfont_obj: $(CONSFONT) + @mkdir -p $(BUILD)/kernel + $(LD) -r -b binary -m elf_x86_64 -o $@ $< + $(OBJCOPY) \ + --redefine-sym _binary_$(subst .,_,$(subst /,_,$(subst -,_,$(CONSFONT))))_start=_binary_consfont_start \ + --redefine-sym _binary_$(subst .,_,$(subst /,_,$(subst -,_,$(CONSFONT))))_end=_binary_consfont_end \ + $@ + # AP startup blob $(BUILD)/kernel/entryother: kernel/arch/x86_64/entryother.S @mkdir -p $(BUILD)/kernel diff --git a/cmd/resolvconf/main.c b/cmd/resolvconf/main.c index 95700f1..ee7325e 100644 --- a/cmd/resolvconf/main.c +++ b/cmd/resolvconf/main.c @@ -44,6 +44,30 @@ puts_(const char *s) write(1, (void *)s, n); } +// Whether this boot asked to be told what is happening; see /proc/bootmode and +// kinfo() in kernel/core/console.c for the same distinction one level down. +// +// Success here is progress, not news. Names starting to work is the expected +// outcome of a normal boot, and printing it unconditionally meant a quiet boot +// still opened with a line about the nameserver -- the one line that was +// supposed to be "Welcome to SchOS!". Failures stay on puts_(), because a +// machine where nothing can resolve a hostname should say so at any verbosity. +static int verbose_boot; + +static void +read_bootmode(void) +{ + char b[16]; + int fd = open("/proc/bootmode", O_RDONLY); + int n; + + if (fd < 0) + return; + n = read(fd, b, sizeof(b) - 1); + close(fd); + if (n > 0) { b[n] = 0; verbose_boot = (b[0] == 'v'); } +} + static int same(const char *a, const char *b, int n) { @@ -74,6 +98,8 @@ main(int argc, char *argv[]) char buf[512]; int i, n, fd, wait = 1, force = 0, print = 0; + read_bootmode(); + for (i = 1; i < argc; i++) { if (argv[i][0] != '-' || argv[i][2] != 0) goto usage; @@ -153,9 +179,11 @@ main(int argc, char *argv[]) } close(fd); - // Worth one line the first time, because it is the moment names start - // working and somebody may be waiting to know it. - puts_("resolvconf: "); - write(1, buf, n); + // Worth one line under -v, because it is the moment names start working + // and somebody watching a verbose boot may be waiting to know it. + if (verbose_boot) { + puts_("resolvconf: "); + write(1, buf, n); + } return 0; } diff --git a/docs/architecture/graphics.md b/docs/architecture/graphics.md index 9aac02d..9c22bbf 100644 --- a/docs/architecture/graphics.md +++ b/docs/architecture/graphics.md @@ -99,9 +99,36 @@ So the text layer fixes three things: ### The TrueType rasteriser -`kernel/dev/ttf.c`: `glyf` outlines, composite glyphs, `cmap` format 4, -anti-aliased scanline fill, with a host harness at `tools/ttftest.c`. A -direct-mapped glyph cache over ASCII in a 48 KB pool. +`kernel/dev/ttf.c`: `glyf` outlines, composite glyphs with their component +transforms, `cmap` format 4, anti-aliased scanline fill, with a host harness at +`tools/ttftest.c`. A direct-mapped glyph cache in a 64 KB pool: one reserved +slot per printable ASCII character, plus 224 shared slots keyed by codepoint +for everything above it. + +**It is the console font.** The regular weight is linked into the kernel image +by the `consfont` rule in the Makefile and opened at 16 px before `vt_init()`, +which is why `gfx_font_init()` prints nothing — there is no terminal to print +to yet. A failure there is not fatal; the 8x16 bitmap font is still the +fallback, and `gfx_font_ready()` says which one is in use. + +**Two bugs here were the same bug.** `KSTACKSIZE` is 4096, and both the render +path and the composite-glyph path put multi-kilobyte outlines on it as locals. +They corrupted whatever was below the stack, which surfaced as user processes +faulting on addresses they never touched and a panic inside `freevm()` — never +anywhere near a font. Neither showed in the host harness, which has an 8 MB +stack. That is the standing hazard of a rasteriser built from one source for +both: the host is most forgiving about exactly the resource the kernel is +strictest about. The outlines are `static` now, one per recursion level, and +`load_simple()` is split out so the leaf's 512-byte flag array is not +multiplied by the recursion depth. + +**Component transforms are applied, not skipped.** An earlier version read a +composite component's offset and stepped over its scale, reasoning that a wrong +transform is worse than none. The left single quote in JetBrains Mono is the +comma glyph with a scale of -1 — a 180-degree rotation — and its offset only +makes sense after the rotation. Dropping the scale put the mark twelve pixels +into a ten-pixel cell, where it was clipped away completely, so the character +read as missing from the font rather than as misplaced. **The typefaces**, both licence-checked at source: @@ -126,8 +153,9 @@ looking at is showing them a developer's log, and that single fact is most of what makes an OS look like a hobby project. Everything still goes out the UART. `console_takeover()` clears the flag on the first `exec()`. -The console still uses the 8×16 cell grid. Porting it onto the text layer and -deleting the grid is open work. +The console uses a cell grid whose size comes from the console font's measured +advance rather than a hardcoded 8x16. Porting it onto the text layer's +proportional path and deleting the grid is open work. Two things the ANSI parser got wrong until a large program nobody here wrote exercised it properly: @@ -249,8 +277,10 @@ makes it mean something. ## What it does not do -- **The console still has its cell grid.** The text layer exists and the console - does not use it; the rasteriser exists and no font is loaded from disk. +- **The console still has its cell grid.** Cells are now a Unicode codepoint + drawn by the TrueType rasteriser rather than a CP437 byte drawn from a bitmap, + and the cell size follows the font's measured advance — but it is still a + fixed grid, and the text layer's proportional path is unused by it. - **No mode setting.** Resolution is whatever the loader asked for. There is no `/dev/dri`-style device and no way to change it at runtime. - **No double buffering in practice.** `display_ops.acquire` is specified and @@ -262,8 +292,13 @@ makes it mean something. path. TrueType hinting is deliberately out of scope. - **No text layout library.** Advance widths exist per glyph; kerning, shaping and line breaking do not. -- **The glyph cache covers ASCII only**, direct-mapped. A codepoint outside it - is rasterised every time. +- **The glyph cache has 224 slots above ASCII**, direct-mapped by codepoint + modulo that. One European language fits without collisions; mixing scripts + costs collisions, and a full pool is emptied and refilled rather than evicted + entry by entry, because a linear pool cannot reclaim one glyph's bytes. +- **No composite point-matching.** Components positioned by matching a point in + the base rather than by an offset are placed at the origin. Fonts that use it + are rare and none in the tree do. - **PSOGM renders and stops.** The interactive crash mode described in [../plan/roadmap.md](../plan/roadmap.md) — walk the stack, read another process's memory, dump page tables — is designed and not built. diff --git a/docs/architecture/terminals.md b/docs/architecture/terminals.md index 90a6f1d..1500a54 100644 --- a/docs/architecture/terminals.md +++ b/docs/architecture/terminals.md @@ -45,23 +45,29 @@ switched away from mid-sequence comes back wrong. ## A cell -Four bytes: +Eight bytes: | | | |---|---| -| `ch` | the glyph the font draws — **not** a codepoint | +| `ch` | the Unicode codepoint, 32 bits | | `fg`, `bg` | palette indices, 0-15 ANSI plus two theme defaults | | `attr` | reverse video; room for underline and the rest | +| `pad` | to eight, so the grid indexes without a multiply | -`ch` stores the glyph rather than the codepoint because the console maps UTF-8 -to CP437 before it gets here, and that is what `include/font.h` contains. -Storing the codepoint would mean re-running the mapping on every repaint and -getting a different answer for anything the font cannot draw. +`ch` was one byte holding a CP437 glyph, not a codepoint: the console mapped +UTF-8 down to CP437 on the way in, because that is what `include/font.h` +contained. Widening it to the codepoint is what made Ą, Ę, Ś, Ω and the rest +possible, and no font change could have done it alone — a byte cannot *hold* +U+0104 whatever is drawing it. Mapping on the way in also meant the decision +was made once, at write time, and a later font could not revisit it. -The colours are indices rather than RGB because a cell holding two 32-bit -colours would be twelve bytes instead of four, and the grids are static: six -terminals at 240x75 is 432 KB of BSS, paid once at link time rather than failed -to pay at run time. There is no contiguous multi-page allocator to ask. +The cost is real: the grid went from four bytes per cell to eight, and six +terminals at 240x75 is 864 KB of BSS rather than 432 KB. Paid once at link +time, which is the only way to pay it — there is no contiguous multi-page +allocator to ask. + +The colours stay indices rather than RGB, for the same reason they always +were: a cell holding two 32-bit colours would be sixteen bytes. ## What is per-terminal @@ -316,5 +322,16 @@ waiting for a different editor. - **Only 240x75 of any screen.** Larger framebuffers work; the grid is clamped and the remainder stays background. Raising `VT_MAXCOLS`/`VT_MAXROWS` is the fix, and costs BSS. -- **No UTF-8 the font lacks.** CP437 has ~256 glyphs. Anything else draws as - `?`, which is deliberate: drawing a *different* character would be worse. +- **No UTF-8 the font lacks.** The console draws through the TrueType + rasteriser now, so the limit is the font's `cmap` rather than a 256-entry + table — Latin, Latin Extended, Greek, Cyrillic and the common punctuation and + maths symbols all render. Anything genuinely absent still draws as `?`, which + is deliberate: drawing a *different* character would be worse. +- **No shaping, and no wide characters.** One codepoint is one cell of one + advance. Combining marks land on top of their own cell rather than the + preceding one, and CJK draws at half the width it wants. Both need a + width table and a cell that can span two columns. +- **Typed input above U+07FF is unreliable.** Two-byte UTF-8 arrives intact; + three-byte sequences typed at the shell come through mangled. The console's + own decoder is not the problem — the same characters `cat`ed from a file + render correctly — so this belongs with keyboard and locale handling. diff --git a/include/gfx.h b/include/gfx.h index 41e8da5..8447588 100644 --- a/include/gfx.h +++ b/include/gfx.h @@ -75,7 +75,16 @@ int gfx_rows(void); // For the text layer in kernel/dev/vt.c: one cell with its own colours, and // a screen scroll that leaves the cursor alone. -void gfx_draw_cell(int col, int row, uchar glyph, uint32 fg, uint32 bg); +// One terminal cell. `cp` is a Unicode codepoint, not a byte -- see the note on +// struct vtcell in vt.h for why that distinction is the whole feature. +void gfx_draw_cell(int col, int row, uint32 cp, uint32 fg, uint32 bg); + +// Bring the console typeface up, once, early. Falls back to the built-in +// bitmap and says nothing if the font will not open -- see the note in gfx.c +// on why it cannot print for itself. +void gfx_font_init(void); +int gfx_font_ready(void); +void gfx_cell_size(int *w, int *h); // A filled rectangle in pixels, with the per-pixel checks hoisted out. void gfx_fill(int x, int y, int w, int h, uint32 color); void gfx_scroll_up(uint32 bg); diff --git a/include/vt.h b/include/vt.h index 7ddb26c..26c3f73 100644 --- a/include/vt.h +++ b/include/vt.h @@ -43,11 +43,23 @@ // same cell holding two 32-bit colours would be twelve, and the grids are // static. struct vtcell { - uint8 ch; + // A Unicode codepoint, not a byte. + // + // This was a uint8, which is to say the grid could hold 256 distinct glyphs + // and every terminal was a CP437 terminal. A codepoint outside that was not + // stored wrongly, it was *converted* -- `glyph_for()` mapped what it could + // and turned the rest into `?`. Ą, Ę, Ś, Ω, Œ, Ə, ∞ and every other + // character anybody actually types were question marks before they reached + // the font, so no font could have fixed it. + // + // Four bytes a cell instead of one. A 1920x1080 screen at 9x18 is 12,720 + // cells, so the grid costs 100 KB instead of 50 KB per terminal. + uint32 ch; uint8 fg; uint8 bg; uint8 attr; // reserved: bold/underline/reverse, which do not change -}; // the glyph and so can be added without touching a grid + uint8 pad; // the glyph and so can be added without touching a grid +}; #define VT_FG_DEFAULT 16 // SCH_FG from include/theme.h #define VT_BG_DEFAULT 17 // SCH_BG diff --git a/kernel/core/main.c b/kernel/core/main.c index 5e22922..cfcdfab 100644 --- a/kernel/core/main.c +++ b/kernel/core/main.c @@ -69,6 +69,9 @@ kmain(uint32 fb_addr) // is a worse failure than staying quiet. if (bi.ptr && bi.width && bi.height && bi.stride && bi.bpp >= 16) { gfx_attach_scanout(&bi); + // Before vt_init, which reads the cell size to work out how many rows + // and columns there are. The font is in the kernel image, so this needs + // nothing that does not exist yet -- no allocator, no filesystem. vt_init(); gfx_clear(0x0C0E13); /* SCH_BG; theme.h is not included here */ cons_serial_only = 0; @@ -135,10 +138,22 @@ kmain(uint32 fb_addr) // Before consoleinit, because the console writes to a terminal and there // has to be one. vt_init reads the framebuffer geometry, so it also has to // come after whatever attached a scanout. + gfx_font_init(); // no-op if the early path already did it vt_init(); if (boot_verbose) cprintf("verbose boot: vt_init done\n"); consoleinit(); if (boot_verbose) cprintf("verbose boot: consoleinit done\n"); + // Now that there is a console to say it on. gfx_font_init() ran before + // vt_init() and deliberately printed nothing; see the note there. + { + int cw, chh; + gfx_cell_size(&cw, &chh); + if (gfx_font_ready()) + kinfo("console: JetBrains Mono, %dx%d cells, %d by %d\n", + cw, chh, gfx_cols(), gfx_rows()); + else + kinfo("console: built-in bitmap font (the typeface would not open)\n"); + } uartinit(); if (boot_verbose) cprintf("verbose boot: uartinit done\n"); mouseinit(); diff --git a/kernel/dev/gfx.c b/kernel/dev/gfx.c index e1821de..215792b 100644 --- a/kernel/dev/gfx.c +++ b/kernel/dev/gfx.c @@ -22,12 +22,37 @@ #include "display.h" #include "gfx.h" #include "vt.h" +#include "text.h" +#include "ttf.h" #define BACKSPACE 0x100 -#define CHAR_HEIGHT 16 -#define CHAR_WIDTH 8 #define COLOR_BG SCH_BG +// The cell size, and why it is not a constant any more. +// +// It was 8x16, the size of the built-in bitmap font, and everything in this +// file multiplied by it. That font is CP437: 256 glyphs, no Ą, no Ę, no Ś, no +// Ω, no ∞. Those did not come out wrong, they came out as `?`, because the +// grid could not hold a codepoint above 255 and mapped what it could not keep. +// +// So the console draws from a real typeface now, and a real typeface decides +// its own metrics. These start at the bitmap's size because the console has to +// work from the first panic onwards -- before gfx_font_init() has run there is +// no font, and 8x16 against font_data is exactly what there always was. +static int cell_w = 8; +static int cell_h = 16; + +#define CHAR_WIDTH cell_w +#define CHAR_HEIGHT cell_h + +// The typeface, once there is one. `cfont` is the switch: null means fall back +// to the bitmap, which is what early boot and any font failure get. +static struct ttf consttf; +static struct font *cfont; +static int cfont_baseline; + +static uint32 blend(uint32 bg, uint32 fg, uint a); + // The scanout, as published by whichever driver owns the display. Zeroed until // one does, and every entry point checks `fb` before touching memory -- a // kernel that panics before any display driver has run still has to be able to @@ -222,21 +247,131 @@ gfx_putc(int c, uint32 color) // right for a console that has one background and wrong for a terminal where // every cell carries its own. void -gfx_draw_cell(int col, int row, uchar glyph, uint32 fg, uint32 bg) +gfx_draw_cell(int col, int row, uint32 cp, uint32 fg, uint32 bg) { if (!fb) return; - int x = col * CHAR_WIDTH, y = row * CHAR_HEIGHT; + int x = col * cell_w, y = row * cell_h; if (x < 0 || y < 0 || - x + CHAR_WIDTH > (int)scanout.width || y + CHAR_HEIGHT > (int)scanout.height) + x + cell_w > (int)scanout.width || y + cell_h > (int)scanout.height) return; - for (int r = 0; r < CHAR_HEIGHT; r++) { - uchar bits = font_data[glyph][r]; - for (int c = 0; c < CHAR_WIDTH; c++) - gfx_draw_pixel(x + c, y + r, (bits & (0x80 >> c)) ? fg : bg); + // No font yet: the bitmap, exactly as before. Anything it cannot hold is + // drawn as a space rather than as the wrong glyph -- `?` was at least honest + // about being a substitute, and a CP437 picture of something unrelated is + // not. + if (!cfont) { + uchar g = cp < 256 ? (uchar)cp : ' '; + for (int r = 0; r < cell_h; r++) { + uchar bits = font_data[g][r]; + for (int c = 0; c < cell_w; c++) + gfx_draw_pixel(x + c, y + r, (bits & (0x80 >> c)) ? fg : bg); + } + return; } + + // Background first, then the glyph blended over it. The cell is always + // filled: a terminal cell owns its rectangle, and leaving the old pixels + // where a narrow glyph does not reach is how ghosting gets in. + for (int r = 0; r < cell_h; r++) + for (int c = 0; c < cell_w; c++) + gfx_draw_pixel(x + c, y + r, bg); + + struct glyph gl; + if (cp == ' ' || cfont->ops->glyph(cfont, cp, &gl) != 0) + return; // no such glyph: the cell stays background + + // `cov` is w*h bytes of coverage, 0..255, positioned by the bearing from the + // pen. The pen sits on the baseline at the left of the cell. + int gx = x + gl.bearing_x; + int gy = y + cfont_baseline - gl.bearing_y; + + for (int r = 0; r < gl.h; r++) { + int py = gy + r; + if (py < y || py >= y + cell_h) + continue; // clip to the cell: a tall glyph must not + for (int c = 0; c < gl.w; c++) { // bleed into the line above or below + int px = gx + c; + if (px < x || px >= x + cell_w) + continue; + uint a = gl.cov[r * gl.w + c]; + if (a == 0) + continue; + gfx_draw_pixel(px, py, a == 255 ? fg : blend(bg, fg, a)); + } + } +} + +// Bring the console typeface up. Called once, early, from the display path. +// +// The font is linked into the kernel (see CONSFONT in the Makefile) because +// the console has to work before there is a filesystem -- a panic during boot +// has to be readable. +// +// A size failure is not fatal and not silent: `cfont` stays null and every +// cell keeps coming from the bitmap, which is what this system has always +// drawn. A console that lost its font would be a much worse outcome than one +// that never had a good one. +void +gfx_font_init(void) +{ + extern unsigned char _binary_consfont_start[], _binary_consfont_end[]; + uint32 len = (uint32)(_binary_consfont_end - _binary_consfont_start); + struct glyph gl; + + if (cfont) + return; + // 16 pixels per em. The bitmap it replaces is 16 tall, so the terminal keeps + // the number of rows it had and nothing that assumed a screenful has to + // change. + if (ttf_open(&consttf, _binary_consfont_start, len, 16) != 0) + return; + + struct font *f = ttf_font(&consttf); + if (!f) + return; + + // The cell is as wide as the font's own advance. JetBrains Mono is + // monospaced, so measuring one glyph measures all of them -- and measuring + // rather than assuming is what keeps this correct if the font is ever + // changed for another one. + if (f->ops->glyph(f, 'M', &gl) != 0 || gl.advance <= 0) + return; + + int w = gl.advance, h = f->ops->height(f); + + // Checked *before* they are stored. Writing them first and returning on a + // bad value leaves the console with a cell size no font is drawing at, and + // the bitmap fallback reading past its own 16-row glyphs. + if (w < 4 || w > 64 || h < 6 || h > 64) + return; // implausible metrics: keep the bitmap + + cell_w = w; + cell_h = h; + cfont_baseline = f->ops->baseline(f); + cfont = f; +} + +// **Nothing here prints.** gfx_font_init() runs before vt_init(), because +// vt_init() has to know how big a cell is before it can work out how many fit +// on the screen -- and by then cons_serial_only may already be clear, so a +// cprintf() would reach vt_current() before there is a current terminal. That +// is a fault during boot with no console to report it on, which is a +// spectacularly unhelpful way to fail. +// +// So main() asks afterwards and says it then. +int +gfx_font_ready(void) +{ + return cfont != 0; +} + +void +gfx_cell_size(int *w, int *h) +{ + if (w) *w = cell_w; + if (h) *h = cell_h; } // Move the whole screen up one text row and clear the row it exposes. The diff --git a/kernel/dev/ttf.c b/kernel/dev/ttf.c index f0df6ef..dac0cd7 100644 --- a/kernel/dev/ttf.c +++ b/kernel/dev/ttf.c @@ -176,77 +176,13 @@ struct outline { int npts, ncont; }; +// The simple (non-composite) glyph body, split out from load_outline so that +// its flags[] array -- 512 bytes -- lives only in the leaf frame. Composite +// glyphs recurse, and a half-kilobyte local multiplied by the recursion depth +// is not affordable on a 4 KB kernel stack. static int -load_outline(struct ttf *f, int gid, struct outline *o) +load_simple(const uint8 *g, int nc, struct outline *o) { - uint32 g0, g1; - - if (gid < 0 || gid >= f->num_glyphs) - return -1; - if (f->long_loca) { - g0 = rd32(f->data + f->loca + 4 * gid); - g1 = rd32(f->data + f->loca + 4 * gid + 4); - } else { - g0 = (uint32)rd16(f->data + f->loca + 2 * gid) * 2; - g1 = (uint32)rd16(f->data + f->loca + 2 * gid + 2) * 2; - } - o->npts = o->ncont = 0; - if (g0 >= g1) - return 0; // empty glyph, e.g. space -- not an error - - const uint8 *g = f->data + f->glyf + g0; - int nc = rs16(g); - - // A negative contour count means a composite: the glyph is other glyphs - // placed at offsets. This is not an exotic case to skip -- `i` is built from - // a dotless i plus the dot, so skipping composites loses a letter from - // "SchOS", which is exactly how the gap was found. - // - // Only the translated case is handled: components with a scale or a 2x2 - // matrix are rare outside of synthesised obliques, and applying a transform - // wrongly is worse than not applying one. - if (nc < 0) { - const uint8 *p = g + 10; - int flags; - do { - flags = rd16(p); - int cgid = rd16(p + 2); - p += 4; - - int dx, dy; - if (flags & 0x0001) { // ARG_1_AND_2_ARE_WORDS - dx = rs16(p); dy = rs16(p + 2); p += 4; - } else { - dx = (signed char)p[0]; dy = (signed char)p[1]; p += 2; - } - if (!(flags & 0x0002)) // ARGS_ARE_XY_VALUES - dx = dy = 0; // point matching: not supported - - if (flags & 0x0008) p += 2; // WE_HAVE_A_SCALE - else if (flags & 0x0040) p += 4; // X_AND_Y_SCALE - else if (flags & 0x0080) p += 8; // TWO_BY_TWO - - // Recurse, then merge the component's contours into ours with the offset - // applied. Depth is bounded by the recursion into load_outline, which - // rejects anything that would overflow the point arrays. - struct outline sub; - if (load_outline(f, cgid, &sub) != 0) - continue; - if (o->npts + sub.npts > MAXPTS || o->ncont + sub.ncont > MAXCONT) - break; - for (int i = 0; i < sub.npts; i++) { - o->x[o->npts + i] = sub.x[i] + dx; - o->y[o->npts + i] = sub.y[i] + dy; - o->on[o->npts + i] = sub.on[i]; - } - for (int i = 0; i < sub.ncont; i++) - o->ends[o->ncont + i] = sub.ends[i] + o->npts; - o->npts += sub.npts; - o->ncont += sub.ncont; - } while (flags & 0x0020); // MORE_COMPONENTS - return 0; - } - if (nc > MAXCONT) return -1; o->ncont = nc; @@ -305,6 +241,147 @@ load_outline(struct ttf *f, int gid, struct outline *o) return 0; } +#define F2DOT14_ONE 16384 + +// One row of a 2x2 F2Dot14 transform applied to a point, rounded to nearest +// rather than truncated -- at 16 px a font unit is a sixteenth of a pixel, and +// truncating every component point biases the whole mark down and left by +// enough to see. +// +// long, not int: a coordinate can reach the em square's bounds and a matrix +// entry reach two, so each product is comfortably inside 32 bits but the sum +// of two is not guaranteed to be. +static int +f2d14(int m0, int v0, int m1, int v1) +{ + long acc = (long)m0 * v0 + (long)m1 * v1; + return (int)((acc + (acc >= 0 ? F2DOT14_ONE / 2 : -(F2DOT14_ONE / 2))) + / F2DOT14_ONE); +} + +// Composite glyphs nest: an accented letter is a base plus a mark, and the +// mark can itself be a composite. Real fonts stay within two or three levels, +// so cap it and refuse rather than let a malformed or hostile font drive the +// recursion arbitrarily deep. +#define MAXDEPTH 4 + +// One scratch outline per recursion level, allocated once. As a local this was +// 4736 bytes per frame in a recursive function, which overflowed the kernel +// stack the moment anything composite was drawn -- and composites are exactly +// A-ogonek, E-ogonek, S-acute and the rest of the Polish alphabet, so the +// crash only appeared once non-ASCII text reached the console. +// +// Static for the same reason and under the same rule as the buffers in +// ttf_render: every caller is serialised behind cons.lock. +static struct outline compose[MAXDEPTH]; + +static int +load_outline_at(struct ttf *f, int gid, struct outline *o, int depth) +{ + uint32 g0, g1; + + if (gid < 0 || gid >= f->num_glyphs) + return -1; + if (f->long_loca) { + g0 = rd32(f->data + f->loca + 4 * gid); + g1 = rd32(f->data + f->loca + 4 * gid + 4); + } else { + g0 = (uint32)rd16(f->data + f->loca + 2 * gid) * 2; + g1 = (uint32)rd16(f->data + f->loca + 2 * gid + 2) * 2; + } + o->npts = o->ncont = 0; + if (g0 >= g1) + return 0; // empty glyph, e.g. space -- not an error + + const uint8 *g = f->data + f->glyf + g0; + int nc = rs16(g); + + // A negative contour count means a composite: the glyph is other glyphs + // placed at offsets. This is not an exotic case to skip -- `i` is built from + // a dotless i plus the dot, so skipping composites loses a letter from + // "SchOS", which is exactly how the gap was found. + // + // The component transform is applied, not ignored. An earlier version read + // the offset and stepped over the scale, on the theory that a wrong + // transform is worse than none. It is not: a left single quote in this font + // is the comma glyph with a scale of -1, which is a 180-degree rotation, and + // the offset that goes with it only lands in the right place once the + // rotation has happened. Dropping the scale put the mark twelve pixels into + // a ten-pixel cell -- clipped away entirely, so the character read as + // missing from the font rather than as misplaced. + if (nc < 0) { + if (depth >= MAXDEPTH) + return -1; + struct outline *sub = &compose[depth]; + const uint8 *p = g + 10; + int flags; + do { + flags = rd16(p); + int cgid = rd16(p + 2); + p += 4; + + int dx, dy; + if (flags & 0x0001) { // ARG_1_AND_2_ARE_WORDS + dx = rs16(p); dy = rs16(p + 2); p += 4; + } else { + dx = (signed char)p[0]; dy = (signed char)p[1]; p += 2; + } + if (!(flags & 0x0002)) // ARGS_ARE_XY_VALUES + dx = dy = 0; // point matching: not supported + + // The 2x2 in F2Dot14: a fixed-point format with two integer bits and + // fourteen fractional, so the unit matrix is 16384 and -1 is -16384. + // Identity unless the component says otherwise. + int a = F2DOT14_ONE, b = 0, c = 0, d = F2DOT14_ONE; + if (flags & 0x0008) { // WE_HAVE_A_SCALE + a = d = rs16(p); p += 2; + } else if (flags & 0x0040) { // X_AND_Y_SCALE + a = rs16(p); d = rs16(p + 2); p += 4; + } else if (flags & 0x0080) { // TWO_BY_TWO + a = rs16(p); b = rs16(p + 2); + c = rs16(p + 4); d = rs16(p + 6); p += 8; + } + + // Whether the offset is itself transformed. Neither bit set means + // unscaled, which is what Microsoft's rasteriser does and what every + // font in practice is drawn against. + if (flags & 0x0800) { // SCALED_COMPONENT_OFFSET + int tx = dx, ty = dy; + dx = f2d14(a, tx, c, ty); + dy = f2d14(b, tx, d, ty); + } + + // Recurse into the next level's scratch outline, then merge its contours + // into ours with the transform applied. `sub` is only read back here, + // and the callee only ever touches levels deeper than this one, so the + // two never alias. + if (load_outline_at(f, cgid, sub, depth + 1) != 0) + continue; + if (o->npts + sub->npts > MAXPTS || o->ncont + sub->ncont > MAXCONT) + break; + for (int i = 0; i < sub->npts; i++) { + int X = sub->x[i], Y = sub->y[i]; + o->x[o->npts + i] = f2d14(a, X, c, Y) + dx; + o->y[o->npts + i] = f2d14(b, X, d, Y) + dy; + o->on[o->npts + i] = sub->on[i]; + } + for (int i = 0; i < sub->ncont; i++) + o->ends[o->ncont + i] = sub->ends[i] + o->npts; + o->npts += sub->npts; + o->ncont += sub->ncont; + } while (flags & 0x0020); // MORE_COMPONENTS + return 0; + } + + return load_simple(g, nc, o); +} + +static int +load_outline(struct ttf *f, int gid, struct outline *o) +{ + return load_outline_at(f, gid, o, 0); +} + // --- rasterising ----------------------------------------------------------- // // Scanline fill with the non-zero winding rule, four sub-scanlines per output @@ -359,8 +436,25 @@ int ttf_render(struct ttf *f, uint32 cp, struct glyph *out, unsigned char *cov, int covcap, int *ox, int *oy) { - struct outline o; - struct raster r; + // **Static, not automatic.** Together these are about 21 KB -- an outline + // holds 512 points as three arrays, and a raster holds 1024 edges -- and + // KSTACKSIZE is 4096. As locals they walked straight off the bottom of the + // kernel stack and into whatever was underneath, which showed up as + // user processes faulting at addresses they never touched and then a panic + // in freevm() on a page directory that had been overwritten. None of it + // pointed anywhere near a font. + // + // It never showed in the host harness, where the same code runs on an 8 MB + // stack. That is the whole hazard of a rasteriser that builds host-side and + // kernel-side from one source: the host is forgiving about exactly the + // resource the kernel is strictest about. + // + // Safe as statics because every caller is serialised: the console path holds + // cons.lock across a whole message, and the host harness is single-threaded. + // If a second thread ever rasterises concurrently, these have to become a + // caller-supplied scratch buffer rather than quietly go back on the stack. + static struct outline o; + static struct raster r; int gid = glyph_index(f, cp); if (gid == 0 && cp != 0) @@ -502,22 +596,39 @@ ttf_render(struct ttf *f, uint32 cp, struct glyph *out, unsigned char *cov, // worse: the single scratch buffer meant a glyph was only valid until the next // call, so a caller could not hold two at once. // -// Direct-mapped over printable ASCII, which is the case that matters and needs -// no eviction policy, no hashing and no allocation. Anything outside that range -// still renders, just uncached, through the scratch buffer. +// Two regions, both direct-mapped and neither needing hashing or allocation. // -// The pool is filled linearly and never freed. A cache that can only fill up is -// the right shape here because the contents are pure functions of (codepoint, -// size) -- nothing ever becomes stale -- and the only event that invalidates it -// is a size change, which drops the whole thing. +// Printable ASCII gets a reserved slot each, one codepoint per slot, so the +// characters that make up nearly every line can never be evicted by anything +// else. Everything above ASCII shares a second block of slots keyed by +// codepoint modulo the block size. That block was the missing half: with an +// ASCII-only cache every accented letter re-rasterised on every redraw, so a +// screenful of Polish did hundreds of bezier flattenings per scroll while the +// same screenful of English did none. +// +// The modulus is chosen so that a run of consecutive codepoints lands in +// consecutive slots. Latin-1 Supplement and Latin Extended-A together are +// under 200 codepoints, so a European text in any one language fits without +// collisions; mixing scripts costs collisions, not correctness. +// +// The pool is filled linearly and never freed. A cache that can only fill up +// is the right shape here because the contents are pure functions of +// (codepoint, size) -- nothing ever becomes stale. Two events empty it: a size +// change, which makes every bitmap wrong, and the pool running out, which is +// the price of letting a slot be re-keyed without being able to reclaim the +// bytes its previous occupant held. Both are safe because the returned glyph +// is only promised to be valid until the next call. #define CACHE_LO 32 // space #define CACHE_HI 127 // exclusive -#define CACHE_N (CACHE_HI - CACHE_LO) -#define CACHE_POOL (48 * 1024) +#define ASCII_N (CACHE_HI - CACHE_LO) +#define EXTRA_N 224 // shared slots for everything above ASCII +#define CACHE_N (ASCII_N + EXTRA_N) +#define CACHE_POOL (64 * 1024) struct gcache { struct glyph g; + uint32 cp; // key; meaningful only when `have` uint32 off; // into pool, valid only when `have` int have; }; @@ -545,11 +656,13 @@ ttf_glyph_op(struct font *fo, uint32 cp, struct glyph *out) cache_px = f->px; } - if (cp < CACHE_LO || cp >= CACHE_HI) + // Control characters are not glyphs and would waste a reserved slot each. + if (cp < CACHE_LO) return ttf_render(f, cp, out, ttf_cov, (int)sizeof(ttf_cov), 0, 0); - struct gcache *c = &cache[cp - CACHE_LO]; - if (c->have) { + struct gcache *c = cp < CACHE_HI ? &cache[cp - CACHE_LO] + : &cache[ASCII_N + cp % EXTRA_N]; + if (c->have && c->cp == cp) { *out = c->g; out->cov = cache_pool + c->off; // rebind: the pool may have moved in return 0; // a future revision, the offset has not @@ -560,9 +673,21 @@ ttf_glyph_op(struct font *fo, uint32 cp, struct glyph *out) return -1; uint32 need = (uint32)(g.w * g.h); + + // Out of pool. The bytes held by evicted slots cannot be reclaimed + // individually, so reclaim all of them at once and start over rather than + // degrading to permanently uncached the first time a page of mixed scripts + // goes past. Rebuilding costs one rasterisation per glyph still on screen. + if (cache_used + need > CACHE_POOL && need <= CACHE_POOL) { + for (int i = 0; i < CACHE_N; i++) + cache[i].have = 0; + cache_used = 0; + } + if (cache_used + need <= CACHE_POOL) { memmove(cache_pool + cache_used, g.cov, need); c->off = cache_used; + c->cp = cp; c->g = g; c->g.cov = cache_pool + cache_used; // not the scratch buffer c->have = 1; @@ -571,8 +696,8 @@ ttf_glyph_op(struct font *fo, uint32 cp, struct glyph *out) return 0; } - // Pool exhausted. Still correct, just uncached -- and the glyph is in the - // scratch buffer, which is exactly the old contract. + // A single glyph larger than the whole pool. Still correct, just uncached -- + // and it is in the scratch buffer, which is exactly the old contract. *out = g; return 0; } diff --git a/kernel/dev/vt.c b/kernel/dev/vt.c index 26540a2..91bfde3 100644 --- a/kernel/dev/vt.c +++ b/kernel/dev/vt.c @@ -375,11 +375,16 @@ vt_putc(struct vt *v, int glyph) // the prompt on every line. if (glyph < 0x20) return; - if (glyph < 0 || glyph > 255) - glyph = '?'; + // No upper bound any more. The cell holds a codepoint, so a character the + // font may not have is stored as itself and the font decides -- which is + // the difference between a terminal that cannot show Ą and one whose font + // happens not to. It used to be clamped to 255 and replaced with '?' here, + // before any font was ever consulted. + if (glyph < 0) + return; wrap_now(v); // the previous glyph filled the line struct vtcell *k = cell_at(v, v->cx, v->cy); - k->ch = (uint8)glyph; + k->ch = (uint32)glyph; k->fg = v->fg; k->bg = v->bg; k->attr = (uint8)v->attr; @@ -802,8 +807,10 @@ vt_feed(struct vt *v, int c) v->u8_cp = (v->u8_cp << 6) | (c & 0x3F); if (--v->u8_need > 0) return; - int g = glyph_for(v->u8_cp); - vt_putc(v, g ? g : '?'); + // The codepoint itself, not a CP437 approximation of it. glyph_for() is + // still here for the bitmap fallback, but the grid no longer needs it: + // what reaches the font is what the program sent. + vt_putc(v, (int)v->u8_cp); return; } diff --git a/tools/ttftest.c b/tools/ttftest.c index 6a99a4c..1fcbeca 100644 --- a/tools/ttftest.c +++ b/tools/ttftest.c @@ -15,6 +15,31 @@ int ttf_render(struct ttf *, uint32, struct glyph *, unsigned char *, int, int *, int *); +// The console speaks UTF-8, so the harness has to as well -- otherwise the +// only glyphs it can ever ask for are ASCII, and ASCII is entirely made of +// simple glyphs. Every composite in a Latin font is an accented letter, which +// is to say: the byte-at-a-time version could not reach the code path that +// mattered. +// +// text.h declares utf8_next and kernel/dev/text.c defines it, but that file +// pulls in the kernel headers and cannot be compiled on the host. Define the +// symbol here instead: a second decoder is the cost of the harness existing +// at all, and the two agree on the only thing that matters here, which is +// always advancing. +uint32 utf8_next(const char **pp) +{ + const unsigned char *p = (const unsigned char *)*pp; + uint32 c = *p++; + int extra = 0; + if ((c & 0xe0) == 0xc0) { c &= 0x1f; extra = 1; } + else if ((c & 0xf0) == 0xe0) { c &= 0x0f; extra = 2; } + else if ((c & 0xf8) == 0xf0) { c &= 0x07; extra = 3; } + while (extra-- > 0 && (*p & 0xc0) == 0x80) + c = (c << 6) | (*p++ & 0x3f); + *pp = (const char *)p; + return c; +} + int main(int argc, char **argv) { if (argc < 5) { fprintf(stderr, "usage: ttftest font.ttf px \"text\" out.pgm\n"); return 1; } @@ -41,16 +66,17 @@ int main(int argc, char **argv) int H = fo->ops->height(fo) + px; // room for descenders and padding int base = fo->ops->baseline(fo) + px / 2; int W = 16; - for (const char *p = text; *p; p++) W += px; // generous + for (const char *p = text; *p; ) { utf8_next(&p); W += px; } // generous unsigned char *img = calloc(W * H, 1); int pen = 8, drawn = 0, missing = 0; - for (const char *p = text; *p; p++) { + for (const char *p = text; *p; ) { + uint32 cp = utf8_next(&p); struct glyph g; // Through the provider, not ttf_render, so the glyph cache is on the // path being tested rather than bypassed by the harness. - if (fo->ops->glyph(fo, (unsigned char)*p, &g) != 0) { + if (fo->ops->glyph(fo, cp, &g) != 0) { missing++; pen += px / 2; continue;