diff --git a/docs/history/kernel.md b/docs/history/kernel.md index 6b52cdb..13c7e0d 100644 --- a/docs/history/kernel.md +++ b/docs/history/kernel.md @@ -1079,6 +1079,23 @@ memory. Three translations from the truth. `NVMA` is 256 and the failure returns `-ENOMEM`. +256 ran out too, later, and silently: `x=($(user) + return s->u ? fetchaddr(s->u + i * sizeof(addr_t), p) : (*p = 0, 0); + *p = s->k ? (addr_t)s->k[i] : 0; + return 0; +} + +static long +arg_len(struct argsrc *s, addr_t p) +{ + return s->user ? strlen_user(p) : (long)strlen((char *)p); +} + +// Count an array and its bytes, against Linux's limits: ARG_MAX in all, +// pointers included, and ARG_STRLEN_MAX for any one string. +static int +arg_measure(struct argsrc *s, uint64 *count, uint64 *bytes, uint64 already) +{ + for (uint64 i = 0; ; i++) { + addr_t p; + if (arg_ptr(s, i, &p) < 0) + return -EFAULT; + if (p == 0) { + *count = i; + return 0; + } + long n = arg_len(s, p); + if (n < 0) + return n == -EFAULT ? -EFAULT : -E2BIG; + if (n + 1 > ARG_STRLEN_MAX) + return -E2BIG; + *bytes += (uint64)n + 1; + if (*bytes + (already + i + 1) * sizeof(addr_t) > ARG_MAX) + return -E2BIG; + } +} + +// The new stack's pointer table, written in order a page at a time. +struct tbl { + pml4e_t *pgdir; + uint64 *buf; + addr_t at; // where buf[0] goes in the new address space + int n; +}; + +static int +tbl_flush(struct tbl *t) +{ + if (t->n && copyout(t->pgdir, t->at, (char *)t->buf, t->n * 8) < 0) + return -1; + t->at += (addr_t)t->n * 8; + t->n = 0; + return 0; +} + +static int +tbl_put(struct tbl *t, uint64 v) +{ + t->buf[t->n++] = v; + return t->n == PGSIZE / 8 ? tbl_flush(t) : 0; +} + +// Copy `count` strings to `*at` onward in the new address space, each NUL +// terminated whatever happened to it since it was measured, and put each +// one's address in the table. The strings must not have grown: the room for +// them was measured once, and running past it would write over the table. +static int +arg_copy(struct argsrc *s, uint64 count, pml4e_t *pgdir, addr_t *at, + addr_t end, char *bounce, struct tbl *t) +{ + for (uint64 i = 0; i < count; i++) { + addr_t p; + if (arg_ptr(s, i, &p) < 0 || p == 0) + return -1; + long n = arg_len(s, p); + if (n < 0 || *at + (addr_t)n + 1 > end) + return -1; + if (tbl_put(t, *at) < 0) + return -1; + for (long done = 0; done < n; ) { + long c = n - done > PGSIZE ? PGSIZE : n - done; + if (s->user) { + if (copy_from_user(bounce, p + done, c) < 0) + return -1; + } else { + memmove(bounce, (char *)p + done, c); + } + if (copyout(pgdir, *at + done, bounce, c) < 0) + return -1; + done += c; + } + char nul = 0; + if (copyout(pgdir, *at + n, &nul, 1) < 0) + return -1; + *at += (addr_t)n + 1; + } + return 0; +} + +static int do_exec(char *path, struct argsrc *av, struct argsrc *ev); + // envp may be null, which means "no environment" -- the same thing this // function used to do unconditionally. int exec(char *path, char **argv, char **envp) +{ + struct argsrc a = { 0, argv, 0 }, e = { 0, envp, 0 }; + return do_exec(path, &a, &e); +} + +// execve(2): the arrays are in the caller's memory. See struct argsrc. +int +exec_user(char *path, addr_t uargv, addr_t uenvp) +{ + struct argsrc a = { 1, 0, uargv }, e = { 1, 0, uenvp }; + return do_exec(path, &a, &e); +} + +static int +do_exec(char *path, struct argsrc *av, struct argsrc *ev) { uint64 _t0 = rdtsc(); fsstat.exec_calls++; char *s, *last; int i, off; - uint64 argc, envc, sz, sp; + uint64 sz, sp; struct elfhdr elf; struct inode *ip; struct proghdr ph; pml4e_t *pgdir, *oldpgdir; - int slot; - // The three pointer arrays live on the heap, not on the kernel stack. + // The arguments are measured before anything is loaded, so that too many + // of them is E2BIG with nothing to undo -- and so the stack can be sized to + // hold them. // - // Together they are about 1.8 KB, and KSTACKSIZE is 4096 -- with the call - // chain under this function running exec -> loaduvm -> readi -> SchFS -> - // btree, there is not 1.8 KB to spare. Putting them here overflowed the - // stack on the first exec after the environment was added, and the canary - // in param.h caught it on pid 1 before it could corrupt anything. - // - // One page holds all three with room over. + // There used to be a fixed 32 arguments and 64 variables, in arrays on the + // kernel stack and then in one page. `rm *.o` in a build directory, or any + // glob over more than 31 files, failed with E2BIG. Now the limit is Linux's: + // 2 MB of strings and pointers together. + uint64 argc = 0, envc = 0, strbytes = 0; + int r; + if ((r = arg_measure(av, &argc, &strbytes, 0)) < 0 || + (r = arg_measure(ev, &envc, &strbytes, argc + 1)) < 0) + return r; + + // Two pages, off the kernel stack, which has no room for either: a bounce + // buffer for the strings and a batch of pointer-table entries. + char *bounce = kalloc(); uint64 *scratch = (uint64*)kalloc(); - if(!scratch) - return -1; - uint64 *ustack = scratch; /* MAXARG+1 */ - uint64 *estack = ustack + MAXARG + 1; /* MAXENV+1 */ - uint64 *stack_slots = estack + MAXENV + 1; /* the rest */ - _Static_assert((MAXARG + 1 + MAXENV + 1 + MAXARG + MAXENV + 32) * 8 <= 4096, - "exec scratch does not fit in one page"); + if(!bounce || !scratch){ + if (bounce) kfree(bounce); + if (scratch) kfree((char*)scratch); + return -ENOMEM; + } uint64 max_sz = 0; uint64 rand_va = 0; @@ -105,91 +245,65 @@ exec(char *path, char **argv, char **envp) end_op(); ip = 0; + // The stack: 63 pages plus a guard, as always, and on top of that room for + // the arguments, the pointer table and AT_RANDOM. + uint64 nslots = 1 + argc + 1 + envc + 1 + 20; + uint64 argpages = (strbytes + nslots * 8 + 64 + PGSIZE - 1) / PGSIZE; + uint64 stackpages = 64 + argpages; sz = PGROUNDUP(max_sz); - if((sz = allocuvm(pgdir, sz, sz + 64*PGSIZE)) == 0) + if((sz = allocuvm(pgdir, sz, sz + stackpages*PGSIZE)) == 0) goto bad; - clearpteu(pgdir, (char*)(sz - 64*PGSIZE)); - sp = sz; + clearpteu(pgdir, (char*)(sz - stackpages*PGSIZE)); - for(argc = 0; argv[argc]; argc++) { - if(argc >= MAXARG) - goto bad; - sp -= strlen(argv[argc]) + 1; - sp &= ~7; - if(copyout(pgdir, sp, argv[argc], strlen(argv[argc]) + 1) < 0) - goto bad; - ustack[argc] = sp; - } + // From the top: the strings, then 16 random bytes, then the table that + // points at them -- argc, argv, 0, envp, 0, the auxiliary vector -- with + // the stack pointer at argc, 16-byte aligned, as the ELF ABI requires. + addr_t strbase = (sz - strbytes) & ~0xFULL; + rand_va = (strbase - 16) & ~0xFULL; + sp = (rand_va - nslots * 8) & ~0xFULL; - // The environment, copied the same way and for the same reason: these - // strings live in the *old* address space, which is still mapped, and will - // not be once the new pgdir is installed. - for(envc = 0; envp && envp[envc]; envc++) { - if(envc >= MAXENV) - goto bad; - sp -= strlen(envp[envc]) + 1; - sp &= ~7; - if(copyout(pgdir, sp, envp[envc], strlen(envp[envc]) + 1) < 0) - goto bad; - estack[envc] = sp; - } - - // Push 16 pseudo-random bytes for AT_RANDOM. { uint8 randbuf[16]; - uint rseed = (uint)(uintptr_t)myproc() ^ (uint)myproc()->pid; + uint rseed = (uint)(uintptr_t)myproc() ^ (uint)myproc()->pid ^ (uint)rdtsc(); for(int ri = 0; ri < 16; ri++){ rseed = rseed * 1664525u + 1013904223u; randbuf[ri] = (uint8)(rseed >> 16); } - sp -= 16; sp &= ~0xFUL; - rand_va = sp; if(copyout(pgdir, rand_va, (char*)randbuf, 16) < 0) goto bad; } - slot = 0; - stack_slots[slot++] = argc; - for(i = 0; i < argc; i++) { - stack_slots[slot++] = ustack[i]; - } - stack_slots[slot++] = 0; // end of argv - + struct tbl t = { pgdir, scratch, sp, 0 }; + addr_t at = strbase; + if (tbl_put(&t, argc) < 0) + goto bad; + if (arg_copy(av, argc, pgdir, &at, sz, bounce, &t) < 0 || tbl_put(&t, 0) < 0) + goto bad; // The environment used to be a single zero here, with the comment "(no // environment)". Nothing on SchOS had one: not a shell, not a musl program // that had just called execve with a perfectly good envp, not anything. So // $TERM was unset, $PATH was whatever each program invented, and a variable // exported in a shell did not survive the exec that was supposed to carry it. - for(i = 0; i < envc; i++) - stack_slots[slot++] = estack[i]; - stack_slots[slot++] = 0; // end of envp + if (arg_copy(ev, envc, pgdir, &at, sz, bounce, &t) < 0 || tbl_put(&t, 0) < 0) + goto bad; // Auxiliary vector — musl reads these for page size, clock, etc. - stack_slots[slot++] = 6; // AT_PAGESZ - stack_slots[slot++] = 4096; - stack_slots[slot++] = 17; // AT_CLKTCK - stack_slots[slot++] = USER_HZ; // what times() is scaled to, not a literal - stack_slots[slot++] = 23; // AT_SECURE = 0 (not setuid) - stack_slots[slot++] = 0; - stack_slots[slot++] = 11; // AT_UID - stack_slots[slot++] = 0; - stack_slots[slot++] = 12; // AT_EUID - stack_slots[slot++] = 0; - stack_slots[slot++] = 13; // AT_GID - stack_slots[slot++] = 0; - stack_slots[slot++] = 14; // AT_EGID - stack_slots[slot++] = 0; - stack_slots[slot++] = 16; // AT_HWCAP - stack_slots[slot++] = 0; - stack_slots[slot++] = 25; // AT_RANDOM - stack_slots[slot++] = rand_va; - stack_slots[slot++] = 0; // AT_NULL - stack_slots[slot++] = 0; - - sp -= slot * sizeof(uint64); - sp &= ~0xF; - - if(copyout(pgdir, sp, (char*)stack_slots, slot * sizeof(uint64)) < 0) - goto bad; + { + static const uint64 fixed[] = { + 6, 4096, // AT_PAGESZ + 17, USER_HZ, // AT_CLKTCK: what times() is scaled to, not a literal + 23, 0, // AT_SECURE: not setuid + 11, 0, 12, 0, // AT_UID, AT_EUID + 13, 0, 14, 0, // AT_GID, AT_EGID + 16, 0, // AT_HWCAP + }; + for (int k = 0; k < (int)NELEM(fixed); k++) + if (tbl_put(&t, fixed[k]) < 0) + goto bad; + if (tbl_put(&t, 25) < 0 || tbl_put(&t, rand_va) < 0 || // AT_RANDOM + tbl_put(&t, 0) < 0 || tbl_put(&t, 0) < 0 || // AT_NULL + tbl_flush(&t) < 0) + goto bad; + } for(last=s=path; *s; s++) if(*s == '/') @@ -277,11 +391,13 @@ exec(char *path, char **argv, char **envp) switchuvm(myproc()); freevm(oldpgdir); kfree((char*)scratch); + kfree(bounce); fsstat.exec_cycles += rdtsc() - _t0; return 0; bad: kfree((char*)scratch); + kfree(bounce); if(pgdir) freevm(pgdir); if(ip){ diff --git a/kernel/sys/sys_fs.c b/kernel/sys/sys_fs.c index ed4d20f..269f0f9 100644 --- a/kernel/sys/sys_fs.c +++ b/kernel/sys/sys_fs.c @@ -987,34 +987,20 @@ addr_t sys_unlink(void) addr_t sys_exec(void) { - char *path, *argv[MAXARG], *envp[MAXENV]; - addr_t uargv, uenvp, uarg; + char *path; + addr_t uargv, uenvp; if (argstr(0, &path) < 0 || argaddr(1, &uargv) < 0) return -EFAULT; - memset(argv, 0, sizeof(argv)); - for (int i = 0; ; i++) { - if (i >= NELEM(argv)) return -E2BIG; - if (fetchaddr(uargv + sizeof(addr_t) * i, &uarg) < 0) return -EFAULT; - if (!uarg) { argv[i] = 0; break; } - if (fetchstr(uarg, &argv[i]) < 0) return -EFAULT; - } - // The third argument. This is Linux's execve(2), so musl has been passing - // an environment here all along -- it was simply never read, and every - // program on the system started with an empty one. + // The arrays are read where they are, by exec_user(), rather than copied + // into fixed arrays here: there used to be room for 32 arguments and 64 + // variables, and `rm *` in a directory of 40 files failed with E2BIG. // - // A null envp is legal and means exactly that. It is also what the - // first-party exec() wrapper passes, so old callers keep their old - // behaviour without knowing about this. - memset(envp, 0, sizeof(envp)); - if (argaddr(2, &uenvp) == 0 && uenvp) { - for (int i = 0; ; i++) { - if (i >= NELEM(envp)) return -E2BIG; - if (fetchaddr(uenvp + sizeof(addr_t) * i, &uarg) < 0) return -EFAULT; - if (!uarg) { envp[i] = 0; break; } - if (fetchstr(uarg, &envp[i]) < 0) return -EFAULT; - } - } - return exec(path, argv, envp); + // envp is Linux's third argument, which musl had been passing all along + // while nothing read it. A null envp is legal and means "none"; it is also + // what the first-party exec() wrapper passes. + if (argaddr(2, &uenvp) < 0) + uenvp = 0; + return exec_user(path, uargv, uenvp); } // ftruncate/truncate both grow and shrink. Growing used to be quietly dropped diff --git a/kernel/sys/sys_mem.c b/kernel/sys/sys_mem.c index 503c439..998ec95 100644 --- a/kernel/sys/sys_mem.c +++ b/kernel/sys/sys_mem.c @@ -31,6 +31,51 @@ bad_user_range(uint64 va, uint64 length) return length == 0 || va + length < va || va + length > KERNBASE; } +// Anonymous memory -- pages this process owns outright, with no segment or +// device behind them. These are the only mappings that may be merged, trimmed +// or split, because for them the page table is the whole truth: fork copies +// and munmap frees whatever is present in a range and skips the holes. +static int +vma_anon(const struct vma *v) +{ + return v->used && !v->no_free && v->shmid < 0; +} + +static struct vma * +vma_free_slot(struct proc *p) +{ + for (int i = 0; i < NVMA; i++) + if (!p->vma[i].used) + return &p->vma[i]; + return 0; +} + +// Record [va, va+len) as anonymous memory. +// +// Every mapping used to take a slot of its own, so a process could hold NVMA +// of them and no more -- and zsh takes one per 16 KB heap block, so splitting +// 30000 words into an array failed after 4 MB of heap with a silent exit 1. +// Addresses come from a bump allocator, so a new mapping almost always starts +// where the last one ended, and it extends that one instead. Linux merges +// neighbouring anonymous mappings for the same reason. +static int +vma_note_anon(struct proc *p, uint64 va, uint64 len) +{ + for (int i = 0; i < NVMA; i++) { + struct vma *v = &p->vma[i]; + if (vma_anon(v) && v->addr + v->length == va) { + v->length += len; + return 0; + } + } + struct vma *v = vma_free_slot(p); + if (!v) + return -ENOMEM; + v->addr = va; v->length = len; v->used = 1; + v->no_free = 0; v->shmid = -1; + return 0; +} + addr_t sys_sbrk(void) { @@ -90,15 +135,15 @@ sys_mmap(void) return va; } - struct vma *v = 0; - for (int i = 0; i < NVMA; i++) { - if (!myproc()->vma[i].used) { v = &myproc()->vma[i]; break; } - } + // A slot of its own, for the mappings that cannot be merged: a segment, + // device memory, the framebuffer. Anonymous memory goes through + // vma_note_anon() instead and usually needs none. + // // -ENOMEM, not -1. A bare -1 reaches musl as -EPERM, so a process that // had simply used all NVMA slots was told "operation not permitted" -- // which sends you looking for a permission model this system does not // have. malloc() turns either into NULL, but only one of them is a clue. - if (!v) return -ENOMEM; + struct vma *v = vma_free_slot(myproc()); // Ignore non-zero hint addresses (not MAP_FIXED); always pick a fresh VA. // Bump-allocated: each mapping gets its own disjoint, exactly-length-sized @@ -139,6 +184,8 @@ sys_mmap(void) // it is refused with the errno that says the range is wrong. if (offset + length > have) return -EINVAL; + if (!v) + return -ENOMEM; shm_ref(id); for (uint64 o = 0; o < length; o += PGSIZE) { @@ -176,10 +223,14 @@ sys_mmap(void) for (uint64 a = va; a < va + length; a += PGSIZE) { char *mem = kalloc(); - if (!mem) return -ENOMEM; + if (!mem) { + vm_unmap_range(myproc()->pgdir, va, a - va); + return -ENOMEM; + } memset(mem, 0, PGSIZE); if (mappages(myproc()->pgdir, (void*)a, PGSIZE, V2P((uint64)mem), PTE_W | PTE_U) < 0) { kfree(mem); + vm_unmap_range(myproc()->pgdir, va, a - va); return -ENOMEM; } } @@ -190,8 +241,11 @@ sys_mmap(void) readi(f->ip, 0, va, offset, length); iunlock(f->ip); - v->addr = va; v->length = length; v->used = 1; - v->no_free = 0; v->shmid = -1; + // A private copy, so from here it is anonymous memory like any other. + if (vma_note_anon(myproc(), va, length) < 0) { + vm_unmap_range(myproc()->pgdir, va, length); + return -ENOMEM; + } return va; } @@ -206,6 +260,8 @@ sys_mmap(void) uint64 len = PGROUNDUP(length); if (!pciuser_mmio_ok(phys, len)) return -EACCES; + if (!v) + return -ENOMEM; // PTE_PWT: device registers are not memory, and a cached write to a // doorbell arrives whenever the cache feels like it, which for a // driver is indistinguishable from hardware that ignores you. @@ -230,6 +286,8 @@ sys_mmap(void) return -ENODEV; if (length > fbsz) length = fbsz; + if (!v) + return -ENOMEM; if (mappages(myproc()->pgdir, (void*)va, length, fb, PTE_W | PTE_U) < 0) return -1; is_fb = 1; @@ -240,11 +298,22 @@ sys_mmap(void) } else { for (uint64 a = va; a < va + length; a += PGSIZE) { char *mem = kalloc(); - if (!mem) return -1; + if (!mem) { + vm_unmap_range(myproc()->pgdir, va, a - va); + return -ENOMEM; + } memset(mem, 0, PGSIZE); - if (mappages(myproc()->pgdir, (void*)a, PGSIZE, V2P((uint64)mem), PTE_W | PTE_U) < 0) - return -1; + if (mappages(myproc()->pgdir, (void*)a, PGSIZE, V2P((uint64)mem), PTE_W | PTE_U) < 0) { + kfree(mem); + vm_unmap_range(myproc()->pgdir, va, a - va); + return -ENOMEM; + } } + if (vma_note_anon(myproc(), va, length) < 0) { + vm_unmap_range(myproc()->pgdir, va, length); + return -ENOMEM; + } + return va; } v->addr = va; v->length = length; v->used = 1; @@ -256,27 +325,66 @@ addr_t sys_munmap(void) { uint64 addr, length; + struct proc *p = myproc(); if (argaddr(0, &addr) < 0 || argaddr(1, &length) < 0) return -1; length = PGROUNDUP(length); - if (bad_user_range(addr, length)) + if (bad_user_range(addr, length) || (addr % PGSIZE)) return -EINVAL; + uint64 end = addr + length; + + // A segment or device mapping goes only as a whole, by its own address. + // Part of one is refused: unmapping it the anonymous way below would kfree + // pages that are the framebuffer's or another process's. for (int i = 0; i < NVMA; i++) { - if (myproc()->vma[i].used && myproc()->vma[i].addr == addr) { - if (myproc()->vma[i].shmid >= 0) - shm_detach(myproc()->vma[i].shmid); - if (myproc()->vma[i].no_free) - vm_unmap_norelease(myproc()->pgdir, addr, myproc()->vma[i].length); + struct vma *v = &p->vma[i]; + if (!v->used || vma_anon(v)) + continue; + if (v->addr == addr) { + if (v->shmid >= 0) + shm_detach(v->shmid); + if (v->no_free) + vm_unmap_norelease(p->pgdir, addr, v->length); else - vm_unmap_range(myproc()->pgdir, addr, myproc()->vma[i].length); - myproc()->vma[i].used = myproc()->vma[i].addr = myproc()->vma[i].length = 0; - myproc()->vma[i].no_free = 0; - myproc()->vma[i].shmid = -1; + vm_unmap_range(p->pgdir, addr, v->length); + v->used = v->addr = v->length = 0; + v->no_free = 0; + v->shmid = -1; return 0; } + if (addr < v->addr + v->length && end > v->addr) + return -EINVAL; } - // Partial / untracked unmap — free pages anyway - vm_unmap_range(myproc()->pgdir, addr, length); + + // Anonymous memory: trim every mapping the range touches, splitting one + // it falls in the middle of. With no slot for the split, the record keeps + // covering the hole, which is harmless -- fork and munmap skip pages that + // are not there, and the bump allocator never hands the addresses out + // again. + for (int i = 0; i < NVMA; i++) { + struct vma *v = &p->vma[i]; + if (!vma_anon(v)) + continue; + uint64 vs = v->addr, ve = v->addr + v->length; + if (end <= vs || addr >= ve) + continue; + if (addr <= vs && end >= ve) { + v->used = v->addr = v->length = 0; + } else if (addr <= vs) { + v->addr = end; + v->length = ve - end; + } else if (end >= ve) { + v->length = addr - vs; + } else { + struct vma *w = vma_free_slot(p); + if (w) { + w->addr = end; w->length = ve - end; w->used = 1; + w->no_free = 0; w->shmid = -1; + v->length = addr - vs; + } + } + } + vm_unmap_range(p->pgdir, addr, length); return 0; }