nexoniarz/SchOS
nexoniarz/SchOS
Watch
1
0
Fork
You've already forked SchOS
 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions
SchOS/kernel/arch/x86_64/vm.c
Nexoniarz 3185685997 syscall: three globals on the entry path, shared by every CPU 
One to four processes died per boot with a fault they could not have caused:
`rip 0x0` -- an instruction fetch at zero, which a process does not branch to
-- or a dereference of `0xffffffffffffffea`, which is `-EINVAL`. It hit shells
and schinitd alike and had been carried for a long time as "zsh sometimes jumps
into .data".

`syscall_entry` has to swap FS_BASE from the user's TLS to the kernel's before
it can touch any `%fs:` data, and `wrmsr` destroys rax, rcx and rdx. Those have
to be parked somewhere first -- not on the user stack, which is untrusted and
has not been switched away from yet. They were parked in three globals:

    // Single-CPU scratch for syscall_entry FS swap
    // (safe while IF cleared by SYSCALL/SFMASK)
    uint64 _sc_rax, _sc_rcx, _sc_rdx;

IF being clear answers for an interrupt arriving on *this* CPU. It says nothing
whatever about a second CPU executing the same three stores at the same
instant, and with more than one processor entering syscalls that is not a rare
interleaving -- it is the ordinary one.

**%rcx is the user return address.** SYSCALL puts the user's rip there and
SYSRET takes it back from there. So the CPU that lost the race returned its
process into *another process's* instruction stream, with %rax -- the syscall
number on the way in, the return value on the way out -- swapped alongside it.
Both symptoms fall straight out of that: a zero handed over and followed, and
an errno used as a pointer.

Per-CPU words at %gs:32..48 now. %gs is usable there, before the FS swap, for
exactly the reason RESTORE_KERNEL_FS can already read %gs:0: GS base is per-CPU
in hardware, seginit() sets it once, and userspace cannot change it --
sys_arch_prctl() implements ARCH_SET_FS and ARCH_GET_FS and rejects the rest.

Measured rather than argued. Both variants built once, `build/esp` saved for
each, the same pristine schfs.img restored before every boot, -smp 4:

    three globals     10 boots    10 faults, in five of the ten
    three per-CPU     18 boots     0 faults

A _Static_assert pins all seven per-CPU offsets, because trapasm.S spells them
as literals and cannot include a header -- a slot that moves would leave a
register quietly holding somebody else's value, which is the hardest possible
thing to find.

**This is the second bug of this exact shape in this file.** The first was
`kernel_fs_base`, a single global every CPU wrote during its own seginit(), so
the last writer won and every CPU restored one CPU's per-CPU block;
RESTORE_KERNEL_FS reads %gs:0 for that reason. The lesson generalises and is
now written down in both places: **on the entry path, a global is a bug.**

Closes the row added to Gate 0 four commits ago. It was the right row to put
first -- a compositor built on this would have failed in ways indistinguishable
from its own bugs.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-21 22:56:45 +02:00

772 lines
23 KiB
C
Raw Blame History

#include "param.h"
#include "types.h"
#include "defs.h"
#include "gfx.h"
#include "x86.h"
#include "memlayout.h"
#include "mmu.h"
#include "proc.h"
#include "ioperm.h"
int mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm);
static pte_t * walkpgdir(pml4e_t *pml4, const void *va, int alloc);
// `cpu` and `proc` are no longer __thread variables; they live at fixed offsets
// in the per-CPU block and are read through %gs at each use. See the long
// comment in include/proc.h for why -- the short version is that __thread means
// per *thread*, the compiler caches the thread pointer across calls on that
// basis, and a kernel thread changes CPU at every sched().
// Kernel FS_BASE set by seginit(); read by trapasm.S to restore after user TLS
uint64 kernel_fs_base;
// The per-CPU block's layout is shared with trapasm.S, which spells the offsets
// as literals because there is no way to include a C header from there. These
// pin the C side: if a slot moves, the assembly keeps reading the old address
// and the failure is a register quietly holding somebody else's value, which is
// the hardest possible thing to find. Ask the question at compile time instead.
_Static_assert(PERCPU_SELF == 0 && PERCPU_SCRATCH == 8 && PERCPU_CPU == 16 &&
PERCPU_PROC == 24 && PERCPU_SC_RAX == 32 &&
PERCPU_SC_RCX == 40 && PERCPU_SC_RDX == 48,
"per-CPU block layout changed -- update kernel/arch/x86_64/trapasm.S");
static pml4e_t *kpml4;
static pdpe_t *kpdpt;
struct tassdesc {
uint lim_15_0 : 16;
uint base_15_0 : 16;
uint base_23_16 : 8;
uint type : 4;
uint s : 1;
uint dpl : 2;
uint p : 1;
uint lim_19_16 : 4;
uint avl : 1;
uint l : 1;
uint d : 1;
uint g : 1;
uint base_31_24 : 8;
uint32 base_63_32;
uint32 res;
};
void
syscallinit(void)
{
// SYSRETQ does NOT use USER_CS/USER_DS directly. Per the AMD64/Intel
// convention, it computes CS = STAR[63:48]+16 and SS = STAR[63:48]+8
// (RPL forced to 3), so STAR[63:48] must hold the *compat* 32-bit user
// code selector (SEG_UCODE32), relying on the GDT's consecutive
// [UCODE32, UDATA, UCODE] layout — NOT USER_CS itself. Using USER_CS
// here made every sysretq load CS=USER_CS+16 (an invalid selector,
// pointing past UCODE into the TSS descriptor) and SS=USER_CS+8. QEMU's
// TCG apparently doesn't validate segment descriptors on SYSRET, so this
// went unnoticed until an interrupt forced a strictly-validated iretq
// with that same bogus CS still loaded, which correctly #GP'd.
wrmsr(MSR_STAR,
((((uint64)((SEG_UCODE32 << 3) | DPL_USER) << 16) | (uint64)KERNEL_CS) << 32));
wrmsr(MSR_LSTAR, (addr_t)syscall_entry);
wrmsr(MSR_CSTAR, (addr_t)ignore_sysret);
wrmsr(MSR_SFMASK, FL_TF|FL_DF|FL_IF|FL_IOPL_MASK|FL_AC|FL_NT);
}
void
seginit(void)
{
struct segdesc *gdt;
uint *tss;
uint64 addr, tsslim;
void *local;
struct cpu *c;
// The per-CPU page, preallocated by the boot processor for every AP.
//
// It must not be kalloc()'d here on an application processor. kalloc takes a
// spinlock; acquire() calls holding(); holding() reads `cpu`, which is a
// __thread variable reached through %fs -- and %fs is set up four lines
// below. So an AP allocating its own page reads the per-CPU pointer before
// the per-CPU pointer exists, and the kernel dies in panic("acquire") with
// every lock appearing to be held.
//
// startothers() fills c->local in before starting each AP, on the boot
// processor, where locking works.
c = &cpus[cpunum()];
local = c->local;
if (local == 0) {
local = kalloc(); // the boot processor's own, taken normally
memset(local, 0, PGSIZE);
}
gdt = (struct segdesc*) local;
// The TSS moved out of this page. It now carries an I/O permission bitmap
// (include/ioperm.h) which does not fit here, and it has to be one object
// with it. ioperm_tss() returns it and the descriptor limit that reaches the
// end of the bitmap.
tss = (uint*) ioperm_tss(cpunum(), &tsslim);
kernel_fs_base = ((uint64) local) + 2048;
wrmsr(0xC0000100, kernel_fs_base);
// The TCB self-pointer: the word at %fs:0 must hold the thread pointer.
//
// This is part of the x86-64 TLS ABI and it was missing, which worked only
// by accident. GCC with -mtls-direct-seg-refs compiles `cpu = c` into a
// single `movq %rcx, %fs:-0x10` and never needs the thread pointer as a
// value, so nothing ever read this word. clang materialises it instead --
// `movq %fs:0x0, %rax; addq $-0x10, %rax; movq %rcx, (%rax)` -- which read
// the zero left by the memset above and stored through address
// 0xfffffffffffffff0, three instructions later.
//
// Writing it costs one store per CPU and removes the dependency on a
// particular compiler's TLS optimisation.
*(uint64 *) kernel_fs_base = kernel_fs_base;
// Clear the per-CPU slots before anything can read them. Their offsets are
// the PERCPU_* constants in include/proc.h and are shared with trapasm.S,
// which reads %gs:24 for proc on the way out to user mode and parks three
// registers at %gs:32..48 across the FS_BASE swap on the way in.
((uint64 *) kernel_fs_base)[2] = 0; // struct cpu * (PERCPU_CPU)
((uint64 *) kernel_fs_base)[3] = 0; // struct proc * (PERCPU_PROC)
((uint64 *) kernel_fs_base)[4] = 0; // syscall rax (PERCPU_SC_RAX)
((uint64 *) kernel_fs_base)[5] = 0; // syscall rcx (PERCPU_SC_RCX)
((uint64 *) kernel_fs_base)[6] = 0; // syscall rdx (PERCPU_SC_RDX)
// The same address in GS base, per CPU.
//
// trapasm.S needs to find *this* CPU's kernel FS base on every entry from
// user mode, and it has no register to work from at that instant -- which is
// why it used to read the kernel_fs_base global. That global is written by
// every CPU during its own seginit(), so with more than one processor the
// last writer wins and every CPU restores one CPU's per-CPU block. GS base is
// per-CPU in hardware, so %gs:0 answers the question with no global at all.
//
// Safe because GS belongs entirely to the kernel here: sys_arch_prctl()
// implements ARCH_SET_FS and ARCH_GET_FS only, and musl uses FS for TLS on
// x86-64. See the comment on RESTORE_KERNEL_FS.
wrmsr(0xC0000101, kernel_fs_base); // IA32_GS_BASE
c->local = local;
cpu_set(c);
proc_set(0);
// Two of the entries below are deliberately null descriptors, and neither is
// dead code even though both look it:
//
// SEG_UCODE32 is never loaded, but SYSRETQ computes CS from STAR[63:48]+16
// and SS from +8, so the *slot* must exist to keep the
// [UCODE32, UDATA, UCODE] triple consecutive. See syscallinit.
// SEG_KCPU is a leftover slot number from 32-bit xv6, where %gs held the
// per-CPU pointer. SchOS uses FS_BASE/TLS instead, but the
// index has to stay occupied or SEG_TSS stops landing at 7.
//
// Deleting either moves every descriptor after it and breaks sysret.
addr = (uint64) tss;
gdt[0] = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
gdt[SEG_KCODE] = SEG(STA_X|STA_R, 0, 0, 0);
gdt[SEG_KDATA] = SEG(STA_W, 0, 0, 0);
gdt[SEG_UCODE32] = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
gdt[SEG_UDATA] = SEG(STA_W, 0, 0, DPL_USER);
gdt[SEG_UCODE] = SEG(STA_X|STA_R, 0, 0, DPL_USER);
gdt[SEG_KCPU] = (struct segdesc) {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
struct tassdesc *td = (struct tassdesc*)(gdt + SEG_TSS);
td->lim_15_0 = tsslim & 0xffff;
td->base_15_0 = addr & 0xffff;
td->base_23_16 = (addr >> 16) & 0xff;
td->type = STS_T64A;
td->s = 0;
td->dpl = 0;
td->p = 1;
td->lim_19_16 = (tsslim >> 16) & 0xf;
td->avl = 0;
td->l = 0;
td->d = 0;
td->g = 0;
td->base_31_24 = (addr >> 24) & 0xff;
td->base_63_32 = (addr >> 32) & 0xffffffff;
td->res = 0;
lgdt((void*) gdt, (NSEGS+1) * sizeof(struct segdesc));
ltr(SEG_TSS << 3);
}
pml4e_t*
setupkvm(void)
{
pml4e_t *pml4 = (pml4e_t*) kalloc();
if(pml4 == 0) return 0;
memset(pml4, 0, PGSIZE);
pml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W;
// The framebuffer is deliberately NOT mapped here.
//
// This used to identity-map 0xfd000000 for 800*600*4 bytes, user-writable,
// into every process that ever existed. Three things were wrong with that.
// The address and the size were both guesses that only hold on QEMU's
// stdvga at one resolution; setupkvm runs before vga_init, so on the first
// call nothing had even read the VBE block yet; and every process could
// scribble on the screen without asking anyone.
//
// Nothing used it -- userspace reaches the framebuffer through
// mmap("/dev/fb0"), which maps the real base at the real size. So it is
// gone, and a process gets the framebuffer exactly when it asks for it.
return pml4;
}
void
kvmalloc(void)
{
kpml4 = (pml4e_t*) kalloc();
memset(kpml4, 0, PGSIZE);
kpdpt = (pdpe_t*) kalloc();
memset(kpdpt, 0, PGSIZE);
kpml4[0] = v2p(kpdpt) | PTE_P | PTE_W;
kpml4[PMX(KERNBASE)] = v2p(kpdpt) | PTE_P | PTE_W;
for(uint64 i = 0; i < 4; i++) {
uint64 flags = PTE_PS | PTE_P | PTE_W;
if(i >= 3) {
flags |= PTE_PCD | PTE_PWT;
}
kpdpt[i] = (i << 30) | flags;
}
switchkvm();
}
// Map a device's registers so the kernel can reach them.
//
// Every driver used to say P2V(bar.base), which quietly assumes the BAR lands
// inside the 0-4 GB the loop above maps. Under SeaBIOS it always does. Under
// OVMF it does not: firmware is free to put a 64-bit BAR anywhere, and it puts
// virtio-gpu's at about 768 GB. P2V then produces an address with no
// translation behind it, and the driver's first register write takes a page
// fault -- which, coming from a driver bind, looks like broken hardware rather
// than a missing mapping.
//
// The direct map uses PDPT entries 0-3 of a page directory pointer table that
// kpml4[0], kpml4[KERNBASE] and every process's PML4 (see setupkvm) all share.
// Entries 4-511 are free, so each one can carry a 1 GB page of device memory:
// no page table to allocate, nothing to free, and the mapping appears in every
// address space at once because they all point at this one table.
//
// Cache-disabled and write-through, for the same reason the top GB of the
// direct map is: these are registers, and a write that sits in a cache line
// waiting for company is a write the device never sees.
#define IOREMAP_FIRST_PDPT 4
#define IOREMAP_LAST_PDPT 511
#define GIGABYTE (1ULL << 30)
static int ioremap_next = IOREMAP_FIRST_PDPT;
void *
ioremap(uint64 pa, uint64 size)
{
if (size == 0)
return 0;
// Below 4 GB the direct map already covers it, so hand back the same address
// the old code would have produced. This is what keeps the BIOS path -- where
// every BAR is low -- byte for byte what it was.
if (pa + size <= 4 * GIGABYTE)
return P2V(pa);
uint64 base = pa & ~(GIGABYTE - 1); // 1 GB pages, so align to one
uint64 last = (pa + size - 1) & ~(GIGABYTE - 1);
int span = (int)((last - base) / GIGABYTE) + 1;
// Already mapped? A second BAR in the same gigabyte should not consume a
// second window, and drivers do re-map the same device.
for (int i = IOREMAP_FIRST_PDPT; i < ioremap_next; i++) {
// Clearing the low 30 bits drops every flag as well, leaving the frame.
if ((kpdpt[i] & PTE_P) && (kpdpt[i] & ~(GIGABYTE - 1)) == base)
return (void *)(KERNBASE + ((uint64)i << 30) + (pa - base));
}
if (ioremap_next + span > IOREMAP_LAST_PDPT) {
cprintf("ioremap: out of device window for %p\n", (void *)pa);
return 0;
}
int first = ioremap_next;
for (int i = 0; i < span; i++) {
kpdpt[ioremap_next] = (base + (uint64)i * GIGABYTE)
| PTE_PS | PTE_P | PTE_W | PTE_PCD | PTE_PWT;
ioremap_next++;
}
// The entries were not present a moment ago. Reloading CR3 is heavier than
// invlpg but this runs a handful of times at boot, and it cannot be the thing
// that is subtly wrong.
switchkvm();
return (void *)(KERNBASE + ((uint64)first << 30) + (pa - base));
}
void
switchuvm(struct proc *p)
{
pushcli();
if(p->pgdir == 0)
panic("switchuvm: no pgdir");
void *tss = ioperm_tss_of(cpunum());
const addr_t stktop = (addr_t)p->kstack + KSTACKSIZE;
ioperm_set_rsp0(tss, stktop);
// Which ports this process may reach. Cheap when nobody uses any, which is
// every process that is not hosting a driver.
ioperm_switch(p);
lcr3(v2p(p->pgdir));
popcli();
}
static pte_t *
walkpgdir(pml4e_t *pml4, const void *va, int alloc)
{
pml4e_t *pml4e;
pdpe_t *pdp, *pdpe;
pde_t *pd, *pde;
pte_t *pgtab;
pml4e = &pml4[PMX(va)];
if(*pml4e & PTE_P)
pdp = (pdpe_t*)P2V(PTE_ADDR(*pml4e));
else {
if(!alloc || (pdp = (pdpe_t*)kalloc()) == 0)
return 0;
memset(pdp, 0, PGSIZE);
*pml4e = V2P(pdp) | PTE_P | PTE_W | PTE_U;
}
pdpe = &pdp[PDPX(va)];
if(*pdpe & PTE_P)
pd = (pde_t*)P2V(PTE_ADDR(*pdpe));
else {
if(!alloc || (pd = (pde_t*)kalloc()) == 0)
return 0;
memset(pd, 0, PGSIZE);
*pdpe = V2P(pd) | PTE_P | PTE_W | PTE_U;
}
pde = &pd[PDX(va)];
if(*pde & PTE_P)
pgtab = (pte_t*)P2V(PTE_ADDR(*pde));
else {
if(!alloc || (pgtab = (pte_t*)kalloc()) == 0)
return 0;
memset(pgtab, 0, PGSIZE);
*pde = V2P(pgtab) | PTE_P | PTE_W | PTE_U;
}
return (pte_t*)&pgtab[PTX(va)];
}
void
switchkvm(void)
{
lcr3(v2p(kpml4));
}
int
mappages(pml4e_t *pgdir, void *va, addr_t size, addr_t pa, int perm)
{
char *a, *last;
pte_t *pte;
a = (char*)PGROUNDDOWN((addr_t)va);
last = (char*)PGROUNDDOWN(((addr_t)va) + size - 1);
for(;;){
if((pte = walkpgdir(pgdir, a, 1)) == 0)
return -1;
if(*pte & PTE_P)
panic("remap");
*pte = pa | perm | PTE_P | PTE_U;
invlpg(a);
if(a == last)
break;
a += PGSIZE;
pa += PGSIZE;
}
return 0;
}
void
vm_unmap_range(pml4e_t *pgdir, uint64 va, uint64 length)
{
uint64 a = PGROUNDDOWN(va);
uint64 end = PGROUNDUP(va + length);
for (; a < end; a += PGSIZE) {
pte_t *pte = walkpgdir(pgdir, (void*)a, 0);
if (pte && (*pte & PTE_P)) {
uint64 pa = PTE_ADDR(*pte);
kfree((char*)P2V(pa));
*pte = 0;
invlpg((void*)a);
}
}
}
// Tear down a mapping without freeing what is underneath it.
//
// vm_unmap_range() kfree()s every page it unmaps, which is right for an
// anonymous mapping and catastrophic for anything shared: the framebuffer is
// device memory that was never allocated, and a shared memory segment's pages
// belong to the segment, not to whichever process happens to be detaching.
// Handing either of those to kfree puts them on the free list while another
// process is still reading and writing them.
void
vm_unmap_norelease(pml4e_t *pgdir, uint64 va, uint64 length)
{
uint64 a = PGROUNDDOWN(va);
uint64 end = PGROUNDUP(va + length);
for (; a < end; a += PGSIZE) {
pte_t *pte = walkpgdir(pgdir, (void*)a, 0);
if (pte && (*pte & PTE_P)) {
*pte = 0;
invlpg((void*)a);
}
}
}
// Duplicate an mmap()'d VMA region from src into dst for fork().
// Framebuffer mappings are re-pointed at the same physical MMIO range
// (shared, never copied/freed); anonymous mappings get fresh pages with
// the parent's contents copied in — mirrors what copyuvm() does for the
// [0, sz) heap range, which never covers mmap()'d addresses.
int
copy_vma_range(pml4e_t *dstpgdir, pml4e_t *srcpgdir, uint64 va, uint64 length, int share)
{
uint64 a = PGROUNDDOWN(va);
uint64 end = PGROUNDUP(va + length);
for (; a < end; a += PGSIZE) {
if (share) {
// Point the child at the very same physical pages, whatever they are.
//
// This used to special-case the framebuffer and map gfx_fb_phys()
// explicitly. Reading the parent's own PTE instead covers device memory
// and shared memory segments with one rule, and cannot disagree with
// what the parent actually has mapped -- which the explicit version
// could, and did, when the address was hardcoded.
pte_t *spte = walkpgdir(srcpgdir, (void*)a, 0);
if (!spte || !(*spte & PTE_P))
continue;
if (mappages(dstpgdir, (void*)a, PGSIZE, PTE_ADDR(*spte), PTE_FLAGS(*spte)) < 0)
return -1;
continue;
}
pte_t *spte = walkpgdir(srcpgdir, (void*)a, 0);
if (!spte || !(*spte & PTE_P))
continue;
char *mem = kalloc();
if (!mem)
return -1;
memmove(mem, (char*)P2V(PTE_ADDR(*spte)), PGSIZE);
if (mappages(dstpgdir, (void*)a, PGSIZE, V2P(mem), PTE_FLAGS(*spte)) < 0) {
kfree(mem);
return -1;
}
}
return 0;
}
void
inituvm(pde_t *pgdir, char *sinit, uint64 sz)
{
char *mem;
if(sz >= PGSIZE)
panic("inituvm: more than a page");
mem = kalloc();
memset(mem, 0, PGSIZE);
mappages(pgdir, 0, PGSIZE, v2p(mem), PTE_W|PTE_U);
memmove(mem, sinit, sz);
}
int
loaduvm(pml4e_t *pgdir, char *addr, struct inode *ip, uint64 offset, uint64 sz)
{
uint64 done, n, off_in_page;
addr_t pa;
pte_t *pte;
char *cur_page;
for(done = 0; done < sz; done += n){
cur_page = (char*)PGROUNDDOWN((uint64)(addr + done));
off_in_page = (uint64)(addr + done) - (uint64)cur_page;
if((pte = walkpgdir(pgdir, cur_page, 0)) == 0)
panic("loaduvm: address should exist");
pa = PTE_ADDR(*pte);
n = PGSIZE - off_in_page;
if(n > sz - done)
n = sz - done;
if(readi(ip, 0, (uint64)(P2V(pa) + off_in_page), offset + done, n) != (int)n)
return -1;
}
return 0;
}
uint64
allocuvm(pml4e_t *pgdir, uint64 oldsz, uint64 newsz)
{
char *mem;
addr_t a;
if(newsz >= KERNBASE)
return 0;
if(newsz < oldsz)
return oldsz;
a = PGROUNDUP(oldsz);
for(; a < newsz; a += PGSIZE){
mem = kalloc();
if(mem == 0){
deallocuvm(pgdir, newsz, oldsz);
return 0;
}
memset(mem, 0, PGSIZE);
if(mappages(pgdir, (char*)a, PGSIZE, V2P(mem), PTE_W|PTE_U) < 0){
deallocuvm(pgdir, newsz, oldsz);
kfree(mem);
return 0;
}
}
return newsz;
}
uint64
deallocuvm(pml4e_t *pgdir, uint64 oldsz, uint64 newsz)
{
pte_t *pte;
addr_t a, pa;
if(newsz >= oldsz)
return oldsz;
a = PGROUNDUP(newsz);
for(; a < oldsz; a += PGSIZE){
pte = walkpgdir(pgdir, (char*)a, 0);
if(pte && (*pte & PTE_P) != 0){
pa = PTE_ADDR(*pte);
if(pa == 0)
panic("kfree");
if(pa < PHYSTOP) {
char *v = P2V(pa);
kfree(v);
}
*pte = 0;
}
}
return newsz;
}
// Walk every page a process has mapped in the low half, calling back for each.
//
// Written for `whose` (docs/architecture/debugging.md): answering "which
// process has this physical page mapped" means a reverse lookup, and there is
// no reverse map in this kernel -- so the only way is forward, over every
// entry. That is O(pages mapped), a few thousand iterations for a real
// process, which is nothing for a command somebody typed and would be
// unacceptable anywhere else.
//
// The low half only. The kernel half is the direct map and is identical in
// every address space, so reporting it per process would be noise: every
// physical page is mapped there, always, and saying so once is the useful
// form of that fact.
void
vm_walk_user(pml4e_t *pml4, void (*fn)(void *ctx, uint64 va, uint64 pa, uint64 pte),
void *ctx)
{
uint i, j, k, l;
pdpe_t *pdp;
pde_t *pd;
pte_t *pt;
if (pml4 == 0)
return;
for (i = 0; i < (512/2); i++) {
if (!(pml4[i] & PTE_P))
continue;
pdp = (pdpe_t*)P2V(PTE_ADDR(pml4[i]));
for (j = 0; j < 512; j++) {
if (!(pdp[j] & PTE_P))
continue;
pd = (pde_t*)P2V(PTE_ADDR(pdp[j]));
for (k = 0; k < 512; k++) {
if (!(pd[k] & PTE_P))
continue;
pt = (pte_t*)P2V(PTE_ADDR(pd[k]));
for (l = 0; l < 512; l++) {
if (!(pt[l] & PTE_P))
continue;
fn(ctx,
((uint64)i << 39) | ((uint64)j << 30) | ((uint64)k << 21) |
((uint64)l << 12),
PTE_ADDR(pt[l]), pt[l]);
}
}
}
}
}
void
freevm(pml4e_t *pml4)
{
uint i, j, k, l;
pdpe_t *pdp;
pde_t *pd;
pte_t *pt;
if(pml4 == 0)
panic("freevm: no pgdir");
for(i = 0; i < (512/2); i++){
if(pml4[i] & PTE_P){
pdp = (pdpe_t*)P2V(PTE_ADDR(pml4[i]));
for(j = 0; j < 512; j++){
if(pdp[j] & PTE_P){
pd = (pde_t*)P2V(PTE_ADDR(pdp[j]));
for(k = 0; k < 512; k++){
if(pd[k] & PTE_P) {
pt = (pte_t*)P2V(PTE_ADDR(pd[k]));
for(l = 0; l < 512; l++){
if(pt[l] & PTE_P) {
addr_t pa = PTE_ADDR(pt[l]);
if(pa < PHYSTOP && pa != 0) {
kfree(P2V(pa));
}
}
}
kfree((char*)pt);
}
}
kfree((char*)pd);
}
}
kfree((char*)pdp);
}
}
kfree((char*)pml4);
}
void
clearpteu(pml4e_t *pgdir, char *uva)
{
pte_t *pte;
pte = walkpgdir(pgdir, uva, 0);
if(pte == 0)
panic("clearpteu");
*pte &= ~PTE_U;
}
pml4e_t*
copyuvm(pml4e_t *pgdir, uint64 sz)
{
pml4e_t *d;
pte_t *pte;
addr_t pa, i, flags;
char *mem;
if((d = setupkvm()) == 0)
return 0;
for(i = 0; i < sz; i += PGSIZE){
if((pte = walkpgdir(pgdir, (void*)i, 0)) == 0)
continue;
if(!(*pte & PTE_P))
continue;
pa = PTE_ADDR(*pte);
flags = PTE_FLAGS(*pte);
if(pa < PHYSTOP) {
if((mem = kalloc()) == 0)
goto bad;
memmove(mem, (char*)P2V(pa), PGSIZE);
if(mappages(d, (void*)i, PGSIZE, V2P(mem), flags) < 0) {
kfree(mem);
goto bad;
}
}
}
return d;
bad:
freevm(d);
return 0;
}
char*
uva2ka(pml4e_t *pgdir, char *uva)
{
if((addr_t)uva >= KERNBASE)
return 0;
pte_t *pte;
pte = walkpgdir(pgdir, uva, 0);
if(pte == 0 || (*pte & PTE_P) == 0)
return 0;
if((*pte & PTE_U) == 0)
return 0;
addr_t pa = (*pte) & 0x000FFFFFFFFFF000ull;
// Include uva's in-page offset — callers dereference the returned pointer
// directly at uva's address, not just somewhere within its page.
return (char*)P2V(pa) + ((addr_t)uva & 0xFFF);
}
int
copyout(pml4e_t *pgdir, addr_t va, void *p, uint64 len)
{
char *buf, *pa0;
addr_t n, va0;
buf = (char*)p;
while(len > 0){
va0 = PGROUNDDOWN(va);
pa0 = uva2ka(pgdir, (char*)va0);
if(pa0 == 0) {
return -1;
}
n = PGSIZE - (va - va0);
if(n > len)
n = len;
memmove(pa0 + (va - va0), buf, n);
len -= n;
buf += n;
va += n;
}
return 0;
}
int
copyin(pml4e_t *pgdir, addr_t srcva, void *dst, uint64 len)
{
uint64 n, va0;
char *pa0;
char *dst_buf = (char*)dst;
while(len > 0){
va0 = PGROUNDDOWN(srcva);
pa0 = uva2ka(pgdir, (char*)va0);
if(pa0 == 0)
return -1;
n = PGSIZE - (srcva - va0);
if(n > len)
n = len;
memmove(dst_buf, pa0 + (srcva - va0), n);
len -= n;
dst_buf += n;
srcva += n;
}
return 0;
}
Reference in a new issue View git blame Copy permalink