diff --git a/include/memlayout.h b/include/memlayout.h index 41d492e..d3054bc 100644 --- a/include/memlayout.h +++ b/include/memlayout.h @@ -27,7 +27,7 @@ extern uint64 phystop; // PHYSTOP still means what it always did -- the top of what kalloc may hand // out, and the bound kfree() rejects pages above -- so every existing call // site is unaffected. -#define MEMREGION_MAX 32 +#define MEMREGION_MAX 64 #ifndef __ASSEMBLER__ struct memregion { @@ -57,10 +57,14 @@ extern uint64 lowtop; // // kvmalloc() now maps as many 1 GB pages as the machine has RAM, so the limit // is the page table rather than a constant: the kernel PDPT has 512 entries -// and ioremap takes what the direct map does not, so 64 GB of RAM still leaves -// 448 GB of device window. A machine with more than this gets the rest +// and ioremap takes what the direct map does not, so 256 GB of RAM still +// leaves 248 GB of device window. A machine with more than this gets the rest // ignored, with a line saying so. -#define PHYSTOP_MAX (64ULL << 30) +// +// It was 64 GB while handing memory to the allocator meant writing every page +// of it at boot. kalloc.c keeps never-used memory as ranges now, so a bigger +// machine costs nothing until its memory is used. +#define PHYSTOP_MAX (256ULL << 30) // What the *loader's* page tables cover, and therefore the ceiling on anything // that runs before kvmalloc() builds the kernel's own. diff --git a/kernel/core/kalloc.c b/kernel/core/kalloc.c index f9557e1..2257ad1 100644 --- a/kernel/core/kalloc.c +++ b/kernel/core/kalloc.c @@ -19,12 +19,53 @@ struct run { struct run *next; }; +// Free memory is two things: pages that have been allocated and given back, +// on a list through their first word, and ranges that have never been handed +// out at all, as a cursor and an end. kalloc() takes from the list first. +// +// Everything used to go on the list at boot, which means writing every page +// of RAM -- a pointer and, from kfree(), 4 KB of junk -- before the first +// process ran. On a 64 GB machine that is 64 GB of memset to say "this is +// free". A range costs sixteen bytes however large it is, and a page is first +// touched by whoever asked for it. +// +// `nfree` counts both, so the free total is a read rather than a walk of +// millions of links under the lock every time top refreshes. +#define NFRESH (2 * MEMREGION_MAX) + +struct fresh { + addr_t next, end; // physical, page aligned +}; + struct { struct spinlock lock; int use_lock; struct run *freelist; + struct fresh fresh[NFRESH]; + int nfresh, cur; // ranges, and the first that may have pages + uint64 nfree; // pages, list and ranges together } kmem; +// Before the allocator has a lock, or under it. +static void +add_fresh(addr_t lo, addr_t hi) +{ + lo = PGROUNDUP(lo); + hi &= ~(addr_t)(PGSIZE - 1); + if (lo >= hi) + return; + if (kmem.nfresh == NFRESH) { + // Not reachable with MEMREGION_MAX regions per pass; but if it were, the + // old way still works. + freerange(P2V(lo), P2V(hi)); + return; + } + kmem.fresh[kmem.nfresh].next = lo; + kmem.fresh[kmem.nfresh].end = hi; + kmem.nfresh++; + kmem.nfree += (hi - lo) / PGSIZE; +} + // Catch kmem.freelist corruption at the point it's touched, rather than // letting a bad pointer be silently returned by kalloc() and crash whatever // code dereferences it minutes later on an unrelated call stack. @@ -60,7 +101,7 @@ kinit1(void *vstart, void *vend) // synthetic region behind for exactly this reason, so this is belt and // braces rather than a path anything takes. if(nmem_usable == 0){ - freerange(vstart, vend); + add_fresh(V2P((addr_t)vstart), V2P((addr_t)vend)); return; } @@ -78,7 +119,7 @@ kinit1(void *vstart, void *vend) if(lo >= hi) continue; - freerange(P2V(lo), P2V(hi)); + add_fresh(lo, hi); } } @@ -110,8 +151,16 @@ mem_dump(struct sbuf *out, const char *arg) sb_puts(out, " upward; PHYSTOP is 0x"); sb_puthex(out, (addr_t)PHYSTOP, 16); sb_putc(out, '\n'); - sb_kv(out, "free ", n, "pages"); - sb_kv(out, " ", n * PGSIZE / 1024, "KB"); + sb_kv(out, "listed ", n, "pages"); + sb_kv(out, "free ", kmem.nfree, "pages"); + sb_kv(out, " ", kmem.nfree * PGSIZE / 1024, "KB"); + for (int i = 0; i < kmem.nfresh; i++) { + sb_puts(out, " never used 0x"); + sb_puthex(out, kmem.fresh[i].next, 12); + sb_puts(out, "-0x"); + sb_puthex(out, kmem.fresh[i].end, 12); + sb_putc(out, '\n'); + } if (looped) sb_puts(out, "\n the free list did not end -- it has a CYCLE in it\n"); } @@ -154,6 +203,14 @@ mem_whose(struct sbuf *out, struct whose *w) if (kmem.use_lock) release(&kmem.lock); + for (int i = 0; !found && i < kmem.nfresh; i++) + if (w->phys >= kmem.fresh[i].next && w->phys < kmem.fresh[i].end) { + w->owners++; + w->free_owners++; + sb_puts(out, " kalloc FREE -- never handed out yet\n"); + return; + } + if (found) { w->owners++; w->free_owners++; @@ -189,7 +246,11 @@ kinit_high(void) if (lo >= hi) continue; - freerange(P2V(lo), P2V(hi)); + if (kmem.use_lock) + acquire(&kmem.lock); + add_fresh(lo, hi); + if (kmem.use_lock) + release(&kmem.lock); added += hi - lo; } @@ -236,6 +297,7 @@ kfree(char *v) r = (struct run*)v; r->next = kmem.freelist; kmem.freelist = r; + kmem.nfree++; if(kmem.use_lock) release(&kmem.lock); } @@ -252,27 +314,32 @@ kalloc(void) acquire(&kmem.lock); check_run(kmem.freelist, "kalloc entry"); r = kmem.freelist; - if(r) + if(r){ kmem.freelist = r->next; + } else { + while(kmem.cur < kmem.nfresh && + kmem.fresh[kmem.cur].next >= kmem.fresh[kmem.cur].end) + kmem.cur++; + if(kmem.cur < kmem.nfresh){ + r = (struct run*)P2V(kmem.fresh[kmem.cur].next); + kmem.fresh[kmem.cur].next += PGSIZE; + } + } + if(r) + kmem.nfree--; if(kmem.use_lock) release(&kmem.lock); return (char*)r; } -// Total free physical memory, in bytes. Walks the free list; only /proc -// readers call it, so the O(n) scan is not on any hot path. +// Total free physical memory, in bytes. +// +// This walked the free list under the allocator's lock: two million links on +// 8 GB, cache misses all, every second that top or a status bar was running -- +// and every CPU that wanted a page in that time waited for it. One word now. uint64 kalloc_free_bytes(void) { - struct run *r; - uint64 n = 0; - - if(kmem.use_lock) - acquire(&kmem.lock); - for(r = kmem.freelist; r; r = r->next) - n += PGSIZE; - if(kmem.use_lock) - release(&kmem.lock); - return n; + return kmem.nfree * PGSIZE; } diff --git a/kernel/dev/procfs.c b/kernel/dev/procfs.c index ce7ca82..22a2ab3 100644 --- a/kernel/dev/procfs.c +++ b/kernel/dev/procfs.c @@ -88,7 +88,11 @@ gen_meminfo(struct sbuf *sb) sb_kv(sb, "MemTotal: ", total, "kB"); sb_kv(sb, "MemFree: ", freek, "kB"); sb_kv(sb, "MemAvailable: ", freek, "kB"); - sb_kv(sb, "Buffers: ", 0, "kB"); + // The buffer cache, which grows to 1/32 of RAM and keeps it: without this + // line that showed up in free(1) as memory "used" by nothing. + uint nbuf, maxbuf; + bcache_size(&nbuf, &maxbuf); + sb_kv(sb, "Buffers: ", (uint64)nbuf * (BSIZE / 1024), "kB"); sb_kv(sb, "Cached: ", 0, "kB"); sb_kv(sb, "SwapTotal: ", 0, "kB"); sb_kv(sb, "SwapFree: ", 0, "kB"); diff --git a/kernel/sys/sys_proc.c b/kernel/sys/sys_proc.c index 0e633b5..2cdcfb7 100644 --- a/kernel/sys/sys_proc.c +++ b/kernel/sys/sys_proc.c @@ -12,6 +12,10 @@ #include "errno.h" #include "acpi.h" #include "fat.h" +#include "spinlock.h" +#include "sleeplock.h" +#include "fs.h" +#include "buf.h" addr_t sys_fork(void) { return fork(); } addr_t sys_wait(void) { return wait(); } @@ -522,6 +526,10 @@ addr_t sys_sysinfo(void) // then PHYSTOP, an address, which counted the PCI hole as memory. si->totalram = ram_total_bytes(); si->freeram = kalloc_free_bytes(); + // The buffer cache, so free(1) does not count it as used by nothing. + uint nbuf, maxbuf; + bcache_size(&nbuf, &maxbuf); + si->bufferram = (uint64)nbuf * BSIZE; si->mem_unit = 1; si->procs = (uint16)proc_num(); return 0;