debugger: the kernel can be asked a question

docs/plan/debugger.md, milestones D0, D1 and D2. What they replace is the
ninety-second loop -- add a cprintf, rebuild, boot, read the serial log --
which answers exactly the one question you thought of before the rebuild.

The rule the shape follows: **it observes, it does not control.** No
stepping, no software breakpoints, no stopping the world. Half the bugs
this kernel produces are timing -- an interrupt storm, a level-triggered
line nobody acknowledges, a driver that faults only on its second start --
and every one of them disappears when you halt the machine to look. And
the layering rule, which is the circularity rule the display already has:
a debugger that needs lwIP cannot debug lwIP.

D0 -- symbols. tools/ksyms.pl turns the first link pass into a table the
second pass carries. Two passes is enough because .ksym is placed after
.text and .rodata, so carrying the table cannot move anything recorded in
it; the build lists both passes' text symbols and refuses to link if they
ever disagree, because a symbol table that is subtly wrong gives confident
false answers. ksym.c reads it through weak symbols, so pass one -- which
has no table -- still links. A crash chain now reads panic+0x5f <-
proc_sysrq_cmd+0x99 <- ... <- syscall+0xf6.

D1 -- a log that outlives the kernel. Every character through consputc()
lands in a byte ring with a timestamp per line, readable as /proc/kmsg.
It sits in a region carved before the page allocator gets anything, with a
magic header, so a kernel finds the previous kernel's log still there --
including the whole PSOGM report. panic.c's own comment said "there is no
core dump and no log to go back to"; that is no longer true.

  The obvious place for it is the top of RAM beside the DMA arena. That
  was the first attempt and the magic came back as ASCII on the reboot,
  because the top of usable memory is exactly where the firmware makes its
  own allocations. It lives above the kernel image instead, which is a
  stable address across boots of the same build.

D2 -- the introspection registry. A subsystem publishes its state with one
line at the bottom of the file that owns the data:

  dbg_register("dma", dma_dump, "the DMA arena's free extents");

Seven views -- dma, mem, proc, vma, claims, pci, sock -- reachable three
ways: sysrq 'g' (always, no process and no filesystem), /dev/dbg (echo a
line, cat the answer), and schdbg. The protocol is lines of text because
the same bytes have to work where nothing can allocate or frame a record,
and because a protocol you cannot drive by hand is one you cannot debug.
struct sbuf moved out of kernel/dev/procfs.c into include/sbuf.h rather
than being written a second time.

Several views exist because of a specific bug. `vma` is the table that
decides whether exit() frees a page, and the DMA ownership bug that made
this worth building was a block that was never in it.

Also fixed, found by building with clang for the first time in a while:
struct iovec was defined in both posix.h and socket.h, and every file that
included both was a redefinition. GCC 15 defaults to C23, which permits an
identical struct redeclaration, so the gcc path never noticed -- while
clang defaults to gnu17 and rejected it, breaking the one toolchain
shell.nix exists to provide and the UEFI loader requires. Both are now
guarded.

D3 (`whose <addr>`, the cross-subsystem ownership query) and D4 (hardware
watchpoints on DR0-DR3) are not built and are described honestly as
missing in the architecture doc's "What it does not do".

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VQ7Qx8E1q1M1bbCq8QLHS2
This commit is contained in:
Nexoniarz 2026-09-14 00:58:45 +02:00
commit 1dd4ba0186
35 changed files with 2296 additions and 83 deletions

View file

@ -1,6 +1,7 @@
#include <stdarg.h>
#include "types.h"
#include "defs.h"
#include "klog.h"
#include "vt.h"
#include "kbd.h" /* KEY_VT_SWITCH -- the terminal-switch chords */
#include "gfx.h"
@ -205,6 +206,11 @@ consputc(int c)
for(;;) hlt();
}
// Every kernel message passes through here, which is what makes this the
// one hook the log needs: cprintf, panic's report, a driver's complaint at
// three in the morning. See include/klog.h.
klog_putc(c);
if (c == BACKSPACE) {
uartputc('\b'); uartputc(' '); uartputc('\b');
if (!cons_serial_only)

View file

@ -7,6 +7,8 @@
#include "mmu.h"
#include "spinlock.h"
#include "dma.h"
#include "sbuf.h"
#include "dbg.h"
#define DMA_GRAIN 64 // one cache line; also xHCI's ring alignment
#define NEXTENT 64 // free extents we can track
@ -30,6 +32,59 @@ roundup(uint64 v, uint64 to)
return (v + to - 1) & ~(to - 1);
}
// What the arena is holding, extent by extent.
//
// This is the view that would have ended the H2 investigation in one line. The
// bug was that a 4 MB block was on this free list *and* on kalloc's at the same
// time, and the only way to see it at the time was to read two allocators and
// reason about PHYSTOP. Printing the list is not clever; not being able to
// print it was the problem.
static void
dma_dump(struct sbuf *out, const char *arg)
{
(void)arg;
acquire(&arena.lock);
sb_puts(out, "arena 0x");
sb_puthex(out, arena.lo, 16);
sb_puts(out, "-0x");
sb_puthex(out, arena.hi, 16);
sb_puts(out, " (virtual; physical is this minus KERNBASE)\n");
sb_puts(out, " ");
sb_putu(out, arena.total / 1024);
sb_puts(out, " KB total, ");
{
uint64 free = 0;
for (int i = 0; i < arena.n; i++)
free += arena.free[i].len;
sb_putu(out, free / 1024);
sb_puts(out, " KB free in ");
sb_putu(out, arena.n);
sb_puts(out, " extent(s), ");
sb_putu(out, (arena.total - free) / 1024);
sb_puts(out, " KB out\n");
}
sb_puts(out, "\n # base phys bytes\n");
for (int i = 0; i < arena.n; i++) {
sb_puts(out, " ");
sb_putu_pad(out, i, 2);
sb_puts(out, " 0x");
sb_puthex(out, arena.free[i].base, 16);
sb_puts(out, " 0x");
sb_puthex(out, arena.free[i].base - (addr_t)KERNBASE, 16);
sb_puts(out, " ");
sb_putu_pad(out, arena.free[i].len, 10);
sb_putc(out, '\n');
}
if (arena.n == NEXTENT)
sb_puts(out, "\n the extent table is FULL -- a free here leaks rather "
"than corrupting the list\n");
release(&arena.lock);
}
void
dma_init(void *vstart, void *vend)
{
@ -42,6 +97,8 @@ dma_init(void *vstart, void *vend)
arena.free[0].len = arena.hi - arena.lo;
arena.n = 1;
arena.total = arena.free[0].len;
dbg_register("dma", dma_dump, "the DMA arena's free extents");
}
// Remove free[i], shifting the tail down.

View file

@ -8,6 +8,8 @@
#include "memlayout.h"
#include "mmu.h"
#include "spinlock.h"
#include "sbuf.h"
#include "dbg.h"
void freerange(void *vstart, void *vend);
extern char end[]; // first address after kernel loaded from ELF file
@ -80,10 +82,45 @@ kinit1(void *vstart, void *vend)
}
}
// The page allocator, from the outside.
//
// Counting the free list is O(the free list), which on a gigabyte is a quarter
// of a million links. That is fine for a command someone typed and would not
// be fine anywhere else, so it lives here and nowhere near a hot path -- and
// the walk is bounded, because the one time you most want to run this is when
// you suspect the list is a loop.
static void
mem_dump(struct sbuf *out, const char *arg)
{
(void)arg;
struct run *r;
uint64 n = 0;
int looped = 0;
if (kmem.use_lock)
acquire(&kmem.lock);
for (r = kmem.freelist; r; r = r->next) {
if (++n > (PHYSTOP / PGSIZE) + 16) { looped = 1; break; }
}
if (kmem.use_lock)
release(&kmem.lock);
sb_puts(out, "window 0x");
sb_puthex(out, (addr_t)end, 16);
sb_puts(out, " upward; PHYSTOP is 0x");
sb_puthex(out, (addr_t)PHYSTOP, 16);
sb_putc(out, '\n');
sb_kv(out, "free ", n, "pages");
sb_kv(out, " ", n * PGSIZE / 1024, "KB");
if (looped)
sb_puts(out, "\n the free list did not end -- it has a CYCLE in it\n");
}
void
kinit2()
{
kmem.use_lock = 1;
dbg_register("mem", mem_dump, "the page allocator's free list");
}
void

267
kernel/core/klog.c Normal file
View file

@ -0,0 +1,267 @@
// SPDX-License-Identifier: Apache-2.0
//
// The kernel log that outlives the kernel. See include/klog.h.
//
// Two properties are worth reading the code for, because both are easy to
// write and easy to get subtly wrong:
//
// **It takes no lock.** klog_putc() runs inside consputc(), which runs with
// cons.lock held on the ordinary path and with interrupts off and every other
// CPU spinning on the panic path. A lock here would add a way for the logger
// to deadlock the thing it exists to report on, which is the one failure it
// must not have. The cost is that two CPUs printing at once can interleave
// characters -- which is already true of the console itself, so the log is no
// worse than the screen it mirrors.
//
// **It buffers before it exists.** The region is carved after e820_init(), so
// everything printed before that -- the loader's handover, the memory map
// itself -- has nowhere to go yet. Dropping it would mean the log is blank for
// exactly the part of the boot that is hardest to debug any other way, so
// early output goes into a small static array and is flushed into the ring the
// moment there is one.
#include "types.h"
#include "defs.h"
#include "memlayout.h"
#include "klog.h"
// consputc()'s cursor-movement escape, from kernel/core/console.c.
#define BACKSPACE 0x100
static struct klog_hdr *klog;
static int recovered; // the buffer held a previous boot when we found it
static uint64 prev_bytes; // how much of it that was
// Early output, before there is a ring to put it in. One page is enough for
// the handful of lines between the first cprintf and klog_init(); anything
// past it is dropped rather than overwritten, because the *first* lines of a
// boot that died early are the ones worth having.
static char early[4096];
static int nearly;
static int early_lost;
// Line state, kept here rather than derived, so a timestamp is emitted once
// per line and not once per write.
static int at_line_start = 1;
// Over the fields that do not move while a kernel is running -- deliberately
// not over head and mark.
//
// Those two change on every character written, and a sum that covered them
// would have to be recomputed on every character too: eight multiplies per
// byte of log, for a check whose entire job is to notice that some *other*
// program used this memory between two boots. head and mark are validated by
// range instead, which catches the same thing without the cost.
//
// Not a CRC either, for the same reason of proportion: this does not need to
// detect a clever corruption, and dragging kernel/fs/schfs's crc32c into core
// to do it would be a dependency in the wrong direction.
static uint32
checksum(const struct klog_hdr *h)
{
uint32 s = 0x9e3779b9u;
s = s * 31 + h->magic;
s = s * 31 + h->version;
s = s * 31 + h->size;
return s;
}
static int
looks_valid(const struct klog_hdr *h, uint64 bytes)
{
if (h->magic != KLOG_MAGIC || h->version != KLOG_VERSION)
return 0;
if (h->size != bytes - sizeof(struct klog_hdr))
return 0;
if (h->sum != checksum(h))
return 0;
// head and mark are outside the sum, so they are checked for sense instead:
// a boot cannot have ended before it started, and nothing this kernel does
// writes a quarter of a petabyte to a log.
if (h->mark > h->head)
return 0;
if (h->head > (1ULL << 48))
return 0;
return 1;
}
static void
raw_putc(char c)
{
klog->data[klog->head % klog->size] = c;
klog->head++;
}
static void
raw_puts(const char *s)
{
while (*s)
raw_putc(*s++);
}
// "[ 2.84] ", from the timer tick. Written by hand because the only
// formatter in the kernel is cprintf, and cprintf is what calls this.
static void
stamp(void)
{
uint t = ticks;
uint sec = t / 100;
uint cs = t % 100;
char b[16];
int i = 0, j;
raw_putc('[');
if (sec == 0) {
b[i++] = '0';
} else {
while (sec && i < 12) { b[i++] = '0' + (sec % 10); sec /= 10; }
}
for (j = i; j < 6; j++)
raw_putc(' ');
while (i-- > 0)
raw_putc(b[i]);
raw_putc('.');
raw_putc('0' + (cs / 10));
raw_putc('0' + (cs % 10));
raw_putc(']');
raw_putc(' ');
}
void
klog_putc(int c)
{
char ch = (char)c;
// Carriage returns and the console's backspace are cursor movement, not
// content. A log full of them reads as corruption.
if (ch == '\r' || c == BACKSPACE || ch == 0)
return;
if (!klog) {
if (nearly < (int)sizeof(early))
early[nearly++] = ch;
else
early_lost = 1;
return;
}
if (at_line_start && ch != '\n')
stamp();
raw_putc(ch);
at_line_start = (ch == '\n');
}
void
klog_init(void *base, uint64 bytes)
{
struct klog_hdr *h = (struct klog_hdr *)base;
if (bytes <= sizeof(struct klog_hdr))
return;
if (looks_valid(h, bytes)) {
recovered = 1;
prev_bytes = h->head - h->mark; // what the *last* boot wrote
} else {
// Either the first boot on this machine or a region something else has
// been using. Either way the contents cannot be trusted, and printing
// somebody else's bytes as if they were a log would be worse than
// printing nothing.
h->magic = KLOG_MAGIC;
h->version = KLOG_VERSION;
h->size = (uint32)(bytes - sizeof(struct klog_hdr));
h->boot = 0;
h->head = 0;
recovered = 0;
prev_bytes = 0;
}
h->mark = h->head;
h->boot++;
h->sum = checksum(h);
klog = h;
// A divider in the log itself rather than a flag a reader has to know about:
// whoever ends up reading this will be reading text, possibly by eye, and a
// line that says where one boot ends is worth more than a field that has to
// be interpreted.
at_line_start = 1;
raw_puts("\n--- boot ");
{
char b[12];
int i = 0;
uint32 n = h->boot;
if (n == 0) b[i++] = '0';
while (n) { b[i++] = '0' + (n % 10); n /= 10; }
while (i-- > 0) raw_putc(b[i]);
}
raw_puts(" ---\n");
for (int i = 0; i < nearly; i++) {
char ch = early[i];
if (ch == '\r' || ch == 0)
continue;
if (at_line_start && ch != '\n')
stamp();
raw_putc(ch);
at_line_start = (ch == '\n');
}
if (early_lost)
raw_puts("klog: early output overflowed and some was dropped\n");
nearly = 0;
h->sum = checksum(h);
// Said out loud on every boot, because "the log survived" and "the log was
// silently started over" look identical in a file that begins with a boot
// divider either way.
if (recovered)
cprintf("klog: %d KB at %p, %d bytes carried over from the previous boot\n",
(int)(bytes / 1024), (uint64)base, (int)prev_bytes);
else
cprintf("klog: %d KB at %p, nothing recoverable was there\n",
(int)(bytes / 1024), (uint64)base);
}
// How many bytes the ring currently holds, oldest first.
uint64
klog_len(void)
{
if (!klog)
return 0;
return klog->head < klog->size ? klog->head : klog->size;
}
uint64 klog_prev_bytes(void) { return prev_bytes; }
int klog_recovered(void) { return recovered; }
// Read at a byte offset into the log, where 0 is the oldest byte still held.
//
// Serving straight out of the ring rather than rendering into a scratch buffer
// the way every other /proc file does, because this file is 128 KB and that
// pattern regenerates the whole thing on every read into a stack array. See
// procread() in kernel/dev/procfs.c.
int
klog_read(uint64 off, char *dst, int n)
{
uint64 have, first;
int i;
if (!klog || n <= 0)
return 0;
have = klog_len();
if (off >= have)
return 0; // EOF
if ((uint64)n > have - off)
n = (int)(have - off);
first = klog->head - have; // logical position of the oldest byte
for (i = 0; i < n; i++)
dst[i] = klog->data[(first + off + i) % klog->size];
return n;
}

128
kernel/core/ksym.c Normal file
View file

@ -0,0 +1,128 @@
// SPDX-License-Identifier: Apache-2.0
//
// Naming an address.
//
// docs/plan/debugger.md milestone D0. It is the least clever thing on that
// plan and the one everything else is worth more with: a call chain of six
// hex numbers has to be resolved by hand against a disassembly that may not
// be the build that was running, and the resolving is done at exactly the
// moment when you least want a second task.
//
// The table is built by tools/ksyms.pl from the *first* of two link passes and
// linked into the second. kernel/kernel.ld puts .ksym after .text and .rodata,
// so carrying the table cannot move the addresses recorded in it; the Makefile
// checks that rather than trusting it, by listing the text symbols of both
// passes and refusing to build if they differ.
#include "types.h"
#include "defs.h"
// Weak, and the reason is worth stating: the first link pass has no table --
// it is the input to the program that writes one. An undefined weak symbol
// links as zero, so that pass resolves these to nothing and still links. The
// same property means a kernel built with the generator removed boots fine and
// simply cannot name anything, which is a much better failure than a build
// that stops.
extern const uint32 ksym_count __attribute__((weak));
extern const uint64 ksym_base __attribute__((weak));
extern const uint64 ksym_textend __attribute__((weak));
extern const uint32 ksym_addr[] __attribute__((weak));
extern const uint32 ksym_nameoff[] __attribute__((weak));
extern const char ksym_names[] __attribute__((weak));
int
ksym_ready(void)
{
return &ksym_count != 0 && ksym_count > 0;
}
// The greatest symbol at or below addr.
//
// Returns 0 for anything outside the kernel's text, which includes every
// userspace address -- naming a user rip with a kernel symbol would be worse
// than printing the number, because it would look like an answer.
const char *
ksym_lookup(uint64 addr, uint64 *off)
{
if (!ksym_ready())
return 0;
if (addr < ksym_base || addr >= ksym_textend)
return 0;
uint32 want = (uint32)(addr - ksym_base);
// Binary search for the last entry <= want, in the half-open form so the
// indices stay unsigned and nothing has to go below zero. ksym_addr[0] is 0
// by construction -- the base *is* the first symbol -- so the invariant
// "ksym_addr[lo] <= want" holds on entry and there is no empty case.
uint32 lo = 0, hi = ksym_count;
while (hi - lo > 1) {
uint32 mid = lo + (hi - lo) / 2;
if (ksym_addr[mid] <= want)
lo = mid;
else
hi = mid;
}
if (off)
*off = want - ksym_addr[lo];
return &ksym_names[ksym_nameoff[lo]];
}
static int
put(char *buf, int cap, int n, const char *s)
{
while (*s && n < cap - 1)
buf[n++] = *s++;
return n;
}
static int
puthex(char *buf, int cap, int n, uint64 v)
{
static const char h[] = "0123456789abcdef";
char tmp[17];
int i = 0;
if (v == 0)
return put(buf, cap, n, "0");
while (v && i < 16) { tmp[i++] = h[v & 0xF]; v >>= 4; }
while (i-- > 0 && n < cap - 1)
buf[n++] = tmp[i];
return n;
}
// "sched+0x2a", or the bare address when there is nothing to say.
//
// Writes into a caller's buffer rather than printing, because both callers
// that matter are in panic context: one is building a line for the crash
// screen and the other is building the QR payload. Neither can call cprintf.
int
ksym_fmt(char *buf, int cap, uint64 addr)
{
uint64 off = 0;
const char *name;
int n = 0;
if (cap <= 1) {
if (cap == 1)
buf[0] = 0;
return 0;
}
name = ksym_lookup(addr, &off);
if (!name) {
n = put(buf, cap, n, "0x");
n = puthex(buf, cap, n, addr);
buf[n] = 0;
return n;
}
n = put(buf, cap, n, name);
if (off) {
n = put(buf, cap, n, "+0x");
n = puthex(buf, cap, n, off);
}
buf[n] = 0;
return n;
}

View file

@ -9,6 +9,9 @@
#include "x86.h"
#include "irq.h"
#include "dma.h"
#include "klog.h"
static void *klog_base; // the persistent log, just above the kernel image
static void startothers(void);
static void mpmain(void) __attribute__((noreturn));
@ -83,7 +86,23 @@ kmain(uint32 fb_addr)
// Before anything touches PHYSTOP, which is now a variable read from the
// firmware's memory map rather than a constant asserting 224 MB exists.
e820_init();
kinit1(end, P2V(PHYSTOP - DMA_ARENA_BYTES));
// The log region: the page after the kernel image, and the page allocator is
// started above it rather than at `end`.
//
// Low memory, deliberately, and it took a measurement to learn why. The
// obvious place is the top of RAM beside the DMA arena -- and on a reboot
// the magic came back as ASCII, because that is exactly where the firmware
// puts its own allocations before it hands the machine over. The bytes below
// the kernel image are the ones the loader has already committed to leaving
// alone, and `end` is the same address on every boot of the same build,
// which is the property a region has to have to be found again.
//
// No hole has to be punched in the allocator for this: kinit1's lower bound
// is a parameter, so moving it up by KLOG_BYTES is the whole of it.
klog_base = (void *)PGROUNDUP((addr_t)end);
klog_init(klog_base, KLOG_BYTES);
kinit1((char *)klog_base + KLOG_BYTES, P2V(PHYSTOP - DMA_ARENA_BYTES));
if (boot_verbose) cprintf("verbose boot: kinit1 done\n");
dma_init(P2V(PHYSTOP - DMA_ARENA_BYTES), P2V(PHYSTOP));
if (boot_verbose) cprintf("verbose boot: dma_init done\n");

View file

@ -414,7 +414,19 @@ trace_line(char *buf, int cap, int i, addr_t pc)
char *p = put_str(buf, e, " #");
p = put_dec(p, e, (uint64)i);
p = put_str(p, e, i < 10 ? " " : " ");
put_hex(p, e, (uint64)pc, 16);
p = put_hex(p, e, (uint64)pc, 16);
// And the name, when the kernel is carrying a symbol table. This is the
// difference between a crash screen you photograph and then decode against
// a disassembly, and one you can read. See kernel/core/ksym.c.
if (ksym_ready()) {
char name[64];
if (ksym_lookup((uint64)pc, 0)) {
ksym_fmt(name, sizeof(name), (uint64)pc);
p = put_str(p, e, " ");
put_str(p, e, name);
}
}
}
/* ------------------------------------------------------------------ */
@ -520,10 +532,16 @@ report_serial(char *reason, addr_t *pcs, int npcs)
}
cprintf("\n Call chain\n");
for (int i = 0; i < npcs && pcs[i]; i++)
cprintf(" #%d %p\n", i, pcs[i]);
for (int i = 0; i < npcs && pcs[i]; i++) {
char name[64];
ksym_fmt(name, sizeof(name), (uint64)pcs[i]);
cprintf(" #%d %p %s\n", i, pcs[i], name);
}
cprintf("\n Resolve with: addr2line -e build/kernel/kernel <address>\n");
if (!ksym_ready())
cprintf("\n Resolve with: addr2line -e build/kernel/kernel <address>\n");
else
cprintf("\n For a line number: addr2line -e build/kernel/kernel <address>\n");
cprintf(" Reason: %s\n", reason);
cprintf(" System halted.\n");
}

82
kernel/core/sbuf.c Normal file
View file

@ -0,0 +1,82 @@
// SPDX-License-Identifier: Apache-2.0
//
// The bounded string builder. See include/sbuf.h for what it is and why it
// moved out of kernel/dev/procfs.c.
#include "types.h"
#include "sbuf.h"
void
sb_putc(struct sbuf *sb, char c)
{
if (sb->len < sb->cap - 1)
sb->p[sb->len++] = c;
}
void
sb_puts(struct sbuf *sb, const char *s)
{
while (*s)
sb_putc(sb, *s++);
}
void
sb_pad(struct sbuf *sb, const char *s, int width)
{
int n = 0;
while (s[n]) {
sb_putc(sb, s[n]);
n++;
}
while (n++ < width)
sb_putc(sb, ' ');
}
void
sb_puthex4(struct sbuf *sb, uint32 v)
{
static const char hex[] = "0123456789abcdef";
for (int i = 3; i >= 0; i--)
sb_putc(sb, hex[(v >> (i * 4)) & 0xF]);
}
void
sb_puthex(struct sbuf *sb, uint64 v, int digits)
{
static const char hex[] = "0123456789abcdef";
for (int i = digits - 1; i >= 0; i--)
sb_putc(sb, hex[(v >> (i * 4)) & 0xF]);
}
void
sb_putu(struct sbuf *sb, uint64 v)
{
char tmp[24];
int i = 0;
if (v == 0) { sb_putc(sb, '0'); return; }
while (v && i < (int)sizeof(tmp)) { tmp[i++] = '0' + (v % 10); v /= 10; }
while (i--) sb_putc(sb, tmp[i]);
}
void
sb_putu_pad(struct sbuf *sb, uint64 v, int width)
{
char tmp[24];
int i = 0, n;
if (v == 0) { tmp[i++] = '0'; }
while (v && i < (int)sizeof(tmp)) { tmp[i++] = '0' + (v % 10); v /= 10; }
for (n = i; n < width; n++)
sb_putc(sb, ' ');
while (i--)
sb_putc(sb, tmp[i]);
}
void
sb_kv(struct sbuf *sb, const char *label, uint64 v, const char *unit)
{
sb_puts(sb, label);
sb_putu(sb, v);
if (unit && *unit) { sb_putc(sb, ' '); sb_puts(sb, unit); }
sb_putc(sb, '\n');
}