nexoniarz/SchOS
nexoniarz/SchOS
Watch
1
0
Fork
You've already forked SchOS
 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions
SchOS/kernel/sys/sys_mem.c
Nexoniarz 54b25708c3 Demand paging, the RTL8168H, links that come up late, and Ctrl+Alt+F1..F12 
- Anonymous memory can be reserved without being backed: large mappings,
  MAP_NORESERVE and PROT_NONE are recorded and each page appears, zeroed,
  on first touch -- from trap() for the program, from uva2ka() for the
  kernel on its behalf. WebKit reserves 4 GB at start and touches a sliver;
  backing it all failed with ENOMEM and Epiphany aborted.
- r8169: the 8168G generation and later (the laptop's 8168H, XID 541) are
  taken from the firmware's out-of-band mode, get their PLL powered, the
  receive gate opened, and their PHY reached through the OCP window. The
  PHY's power-down bit is cleared; the old code kept it while restarting
  autonegotiation. Untested on the hardware so far.
- The network thread follows each device's link into lwIP. The link was
  read once at attach, 0.02 s into boot; real autonegotiation takes
  seconds, so on hardware the interface stayed down for good.
- Twelve terminals, switched with Ctrl+Alt+F1..F12 by default: the keys X
  uses, so the console and the desktop agree. AltGr+F-key is gone from
  the defaults (chvt -c altgr+F brings it back).
- SO_PASSCRED on Unix sockets, with SCM_CREDENTIALS delivered by recvmsg.
  WebKit sets it on its IPC sockets and aborted when refused.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4
2026-10-25 11:27:48 +01:00

1149 lines
39 KiB
C
Raw Blame History

#include "core/types.h"
#include "core/defs.h"
#include "gfx/gfx.h"
#include "core/param.h"
#include "uapi/stat.h"
#include "uapi/fcntl.h"
#include "arch/memlayout.h"
#include "arch/mmu.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "core/proc.h"
#include "fs/fs.h"
#include "fs/file.h"
#include "core/syscall.h"
#include "uapi/errno.h"
#include "core/uaccess.h"
#include "uapi/posix.h"
#include "dev/pciuser.h"
extern int mappages(pml4e_t *pgdir, void *va, uint64 size, uint64 pa, int perm);
/* MAP_* and PROT_* now live in include/uapi/posix.h, so userspace and the kernel
compare the same numbers. */
// Reject user mapping requests that fall outside user space. Without this,
// MAP_FIXED at an arbitrary address could create user-accessible PTEs over
// (or vm_unmap_range/kfree pages belonging to) the kernel half of the
// address space.
static int
bad_user_range(uint64 va, uint64 length)
{
return length == 0 || va + length < va || va + length > KERNBASE;
}
// Anonymous memory -- pages this process owns outright, with no segment or
// device behind them. These are the only mappings that may be merged, trimmed
// or split, because for them the page table is the whole truth: fork copies
// and munmap frees whatever is present in a range and skips the holes.
static int
vma_anon(const struct vma *v)
{
return v->used && !v->no_free && v->shmid < 0;
}
static struct vma *
vma_free_slot(struct proc *p)
{
for (int i = 0; i < NVMA; i++)
if (!p->mm->vma[i].used)
return &p->mm->vma[i];
return 0;
}
// Record [va, va+len) as anonymous memory.
//
// Every mapping used to take a slot of its own, so a process could hold NVMA
// of them and no more -- and zsh takes one per 16 KB heap block, so splitting
// 30000 words into an array failed after 4 MB of heap with a silent exit 1.
// Addresses come from a bump allocator, so a new mapping almost always starts
// where the last one ended, and it extends that one instead. Linux merges
// neighbouring anonymous mappings for the same reason.
static int
vma_note_anon(struct proc *p, uint64 va, uint64 len)
{
for (int i = 0; i < NVMA; i++) {
struct vma *v = &p->mm->vma[i];
if (vma_anon(v) && v->addr + v->length == va) {
v->length += len;
return 0;
}
}
struct vma *v = vma_free_slot(p);
if (!v)
return -ENOMEM;
v->addr = va; v->length = len; v->used = 1;
v->no_free = 0; v->shmid = -1;
return 0;
}
// Nothing recorded and nothing mapped anywhere in [va, va+len).
static int
range_free(struct proc *p, uint64 va, uint64 len)
{
for (int i = 0; i < NVMA; i++) {
struct vma *v = &p->mm->vma[i];
if (v->used && va < v->addr + v->length && v->addr < va + len)
return 0;
}
for (uint64 a = va; a < va + len; a += PGSIZE)
if (uva2ka(p->pgdir, (char *)a))
return 0;
return 1;
}
// Where a new mapping of `len` goes.
//
// Fresh addresses from the bump pointer, except for memory that can merge:
// that goes straight after an anonymous mapping whose end has room, so it
// extends the record rather than taking a slot. Without this a slot was
// spent each time the record that ended at the bump pointer had been trimmed,
// and every time munmap split one -- and musl's malloc frees 32 KB pieces out
// of the middle of larger ones all day. apk ran out of NVMA slots within a
// second of starting to install, with 780 MB free, and said "Out of memory".
// It also gives freed address space back, which the bump pointer never did.
static uint64
mmap_va(struct proc *p, uint64 len, int mergeable)
{
if (p->mm->mmap_next == 0)
p->mm->mmap_next = 0x40000000;
if (mergeable) {
for (int i = 0; i < NVMA; i++) {
struct vma *v = &p->mm->vma[i];
uint64 e = v->addr + v->length;
if (vma_anon(v) && e >= 0x40000000 && e + len <= p->mm->mmap_next &&
range_free(p, e, len))
return e;
}
}
uint64 va = p->mm->mmap_next;
if (bad_user_range(va, len))
return 0;
p->mm->mmap_next += len;
return va;
}
// --- demand paging ---------------------------------------------------------
//
// Anonymous memory may be recorded without being backed: the page appears,
// zeroed, the first time it is touched -- by the program, through trap(), or
// by the kernel on its behalf, through uva2ka(). Large mappings are made this
// way, and so are the ones that say outright they will not all be used.
// WebKit reserves 4 GB of address space at start for its JIT and heaps and
// touches a sliver of it; backing all of it at mmap time failed with ENOMEM
// and Epiphany aborted. Go, Java and Firefox reserve the same way.
//
// fork and munmap already skip the holes in anonymous memory, so a page that
// was never touched is simply one they do not see, and a child that touches
// it gets zeroes -- which is what it would have read in the parent.
#define LAZY_MIN (1024 * 1024)
static int
mmap_lazy(int prot, int flags, uint64 length)
{
return (flags & MAP_NORESERVE) || prot == PROT_NONE || length >= LAZY_MIN;
}
static struct spinlock faultlock;
static int faultlock_ready;
// Back the page at va if it lies in this process's anonymous memory and is
// not there yet. 0 if it is there now, -1 if this is a real fault. Takes a
// spinlock and nothing that sleeps, so uva2ka() may call it from anywhere;
// the lock is what keeps two threads of one process from both mapping it.
int
vm_fault(addr_t va)
{
struct proc *p = myproc();
pte_t *pte;
char *mem;
int found = 0;
if (!p || !p->mm || va >= KERNBASE)
return -1;
va = PGROUNDDOWN(va);
for (int i = 0; i < NVMA; i++) {
struct vma *v = &p->mm->vma[i];
if (vma_anon(v) && v->addr <= va && va < v->addr + v->length) {
found = 1;
break;
}
}
if (!found)
return -1;
if ((mem = kalloc()) == 0)
return -1;
memset(mem, 0, PGSIZE);
if (!faultlock_ready) {
initlock(&faultlock, "vmfault");
faultlock_ready = 1;
}
acquire(&faultlock);
pte = walkpte(p->pgdir, (void *)va);
if (pte && (*pte & PTE_P)) {
// Another thread got here first, or the page was never missing and
// this is a protection fault: either way, not ours to fix.
int ours = (*pte & PTE_U) != 0;
release(&faultlock);
kfree(mem);
return ours ? 0 : -1;
}
if (mappages(p->pgdir, (void *)va, PGSIZE, V2P((uint64)mem), PTE_W | PTE_U) < 0) {
release(&faultlock);
kfree(mem);
return -1;
}
release(&faultlock);
return 0;
}
// Record [va, va+len) as anonymous memory wherever no mapping already covers
// it, for a MAP_FIXED that may land anywhere: inside an earlier mapping, across
// the end of one, or in a gap. Below sz is the program image and its heap,
// which fork copies as [0, sz) and needs no record -- a second one there would
// have fork map those pages twice.
static int
vma_cover_anon(struct proc *p, uint64 va, uint64 len)
{
uint64 cur = va, end = va + len;
if (cur < PGROUNDUP(p->sz))
cur = PGROUNDUP(p->sz);
while (cur < end) {
uint64 next = end;
int inside = -1;
for (int i = 0; i < NVMA; i++) {
struct vma *v = &p->mm->vma[i];
if (!v->used)
continue;
if (v->addr <= cur && cur < v->addr + v->length)
inside = i;
else if (v->addr > cur && v->addr < next)
next = v->addr;
}
if (inside >= 0) {
cur = p->mm->vma[inside].addr + p->mm->vma[inside].length;
continue;
}
if (vma_note_anon(p, cur, next - cur) < 0)
return -ENOMEM;
cur = next;
}
return 0;
}
addr_t
sys_sbrk(void)
{
int n;
if (argint(0, &n) < 0)
return -1;
addr_t addr = myproc()->sz;
if (growproc(n) < 0)
return -1;
return addr;
}
addr_t
sys_brk(void)
{
addr_t addr;
if (argaddr(0, &addr) < 0 || addr == 0)
return myproc()->sz;
if (addr > myproc()->sz && growproc(addr - myproc()->sz) < 0)
return myproc()->sz;
return myproc()->sz;
}
static addr_t
mmap_locked(void)
{
uint64 addr, length, offset;
int prot, flags, fd;
struct file *f = 0;
if (argaddr(0, &addr) < 0 || argaddr(1, &length) < 0 ||
argint(2, &prot) < 0 || argint(3, &flags) < 0 ||
argint(4, &fd) < 0 || argaddr(5, &offset) < 0)
return -1;
if (fd >= 0 && fd < NOFILE)
f = myproc()->files->ofile[fd];
length = PGROUNDUP(length);
uint64 va = addr;
if (length == 0 || length > KERNBASE)
return -EINVAL;
if (flags & MAP_FIXED) {
if (bad_user_range(va, length) || (va % PGSIZE))
return -EINVAL;
vm_unmap_range(myproc()->pgdir, va, length);
int fixed_file = f && f->type == FD_INODE && f->ip->type != T_DEV &&
!(flags & MAP_ANONYMOUS);
// Reserved, not backed. PROT_NONE is recorded too: mprotect() does not
// change pages here, so a range reserved PROT_NONE and opened up later
// has to be memory that can appear when it is touched.
if (!fixed_file && mmap_lazy(prot, flags, length)) {
if (vma_cover_anon(myproc(), va, length) < 0)
return -ENOMEM;
return va;
}
for (uint64 a = va; a < va + length; a += PGSIZE) {
char *mem = kalloc();
if (!mem) return -ENOMEM;
memset(mem, 0, PGSIZE);
if (mappages(myproc()->pgdir, (void*)a, PGSIZE, V2P((uint64)mem), PTE_W | PTE_U) < 0) {
kfree(mem);
return -ENOMEM;
}
}
// The file, when there is one. This path used to ignore it and hand
// back zeroed pages whatever fd said -- and a dynamic linker maps
// every segment of every library after the first exactly this way,
// MAP_FIXED at an offset into the file, so libc came up as a library
// of zeros. A private copy, as in the non-fixed path below.
if (f && f->type == FD_INODE && f->ip->type != T_DEV &&
!(flags & MAP_ANONYMOUS)) {
if ((flags & MAP_SHARED) && (prot & PROT_WRITE))
return -ENODEV;
if (!f->readable)
return -EACCES;
ilock(f->ip);
readi(f->ip, 0, va, offset, length);
iunlock(f->ip);
}
// And remembered. Without a record fork() did not copy these pages,
// so a child of a dynamically linked program had no libc.
if (vma_cover_anon(myproc(), va, length) < 0)
return -ENOMEM;
return va;
}
// A slot of its own, for the mappings that cannot be merged: a segment,
// device memory, the framebuffer. Anonymous memory goes through
// vma_note_anon() instead and usually needs none.
//
// -ENOMEM, not -1. A bare -1 reaches musl as -EPERM, so a process that
// had simply used all NVMA slots was told "operation not permitted" --
// which sends you looking for a permission model this system does not
// have. malloc() turns either into NULL, but only one of them is a clue.
struct vma *v = vma_free_slot(myproc());
// Ignore non-zero hint addresses (not MAP_FIXED); always pick a fresh VA.
// Bump-allocated: each mapping gets its own disjoint, exactly-length-sized
// range, so a single mmap() can never run into a neighboring live mapping
// regardless of size (the old scheme assigned a fixed 8MB window per vma[]
// slot index, which a single allocation bigger than 8MB — e.g. a large
// realloc() growth — could map straight past, panicking mappages("remap")
// when it hit an already-present PTE in the next slot's territory).
// Anonymous memory and private file copies may also reuse a freed range
// next to an existing one; see mmap_va().
int mergeable = !(f && (f->type == FD_MEMFD ||
(f->type == FD_INODE && f->ip->type == T_DEV)));
va = mmap_va(myproc(), length, mergeable);
if (va == 0)
return -ENOMEM;
int is_fb = 0;
int shmid = -1;
// A memfd. The pages already exist and belong to the segment, so this maps
// them rather than allocating -- which is the entire point: the other
// process that maps the same segment gets the same physical pages, and a
// store by one is a load by the other because there is only one copy.
//
// A private mapping is allowed read-only, and is then the same pages
// mapped without write access: until somebody writes, a private mapping
// sees the object's stores on Linux too. GLib's GMappedFile maps a memfd
// this way, and that is how gdk-pixbuf reads every image glycin decodes.
// Private and writable would need a copy that is not made; it is refused.
if (f && f->type == FD_MEMFD) {
int id = f->memfd_seg;
uint64 have = PGROUNDUP(memfd_size(id)); // the tail page maps whole
if (!(flags & MAP_SHARED) && (prot & PROT_WRITE))
return -EINVAL;
int pte = (flags & MAP_SHARED) ? PTE_W | PTE_U : PTE_U;
if ((prot & PROT_WRITE) &&
(memfd_seals(id, -1) & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)))
return -EPERM;
if (offset % PGSIZE)
return -EINVAL;
// Mapping past the end is not an error on Linux -- the pages simply
// fault -- but there is no fault handler here to make that true, so
// it is refused with the errno that says the range is wrong.
if (offset + length > have)
return -EINVAL;
if (!v)
return -ENOMEM;
shm_ref(id);
for (uint64 o = 0; o < length; o += PGSIZE) {
char *pg = shm_page(id, (int)((offset + o) / PGSIZE));
if (!pg || mappages(myproc()->pgdir, (void*)(va + o), PGSIZE,
V2P((uint64)pg), pte) < 0) {
vm_unmap_norelease(myproc()->pgdir, va, o);
shm_detach(id);
return -ENOMEM;
}
}
v->addr = va; v->length = length; v->used = 1;
v->no_free = 1; // the segment owns these pages
v->shmid = id;
return va;
}
// File-backed mapping of a regular file.
//
// This used to be ignored entirely: any file mapping silently returned
// zeroed anonymous pages, so a program that mmap'd a font or a config
// file got a buffer full of nothing and no error to explain it. Reading
// the contents in is not demand paging and never will be without a page
// cache -- but it is the difference between a wrong answer and a right
// one.
//
// MAP_SHARED that is also writable is refused rather than faked. Its
// whole contract is that stores land in the file and other mappers see
// them, and there is nothing here that could make that true; quietly
// giving back a private copy would be the same lie in a new place.
if (f && f->type == FD_INODE && f->ip->type != T_DEV) {
if ((flags & MAP_SHARED) && (prot & PROT_WRITE))
return -ENODEV;
if (!f->readable)
return -EACCES;
for (uint64 a = va; a < va + length; a += PGSIZE) {
char *mem = kalloc();
if (!mem) {
vm_unmap_range(myproc()->pgdir, va, a - va);
return -ENOMEM;
}
memset(mem, 0, PGSIZE);
if (mappages(myproc()->pgdir, (void*)a, PGSIZE, V2P((uint64)mem), PTE_W | PTE_U) < 0) {
kfree(mem);
vm_unmap_range(myproc()->pgdir, va, a - va);
return -ENOMEM;
}
}
// Read through the file's own offset-independent path, so this does
// not disturb a descriptor the caller is also reading normally.
ilock(f->ip);
readi(f->ip, 0, va, offset, length);
iunlock(f->ip);
// A private copy, so from here it is anonymous memory like any other.
if (vma_note_anon(myproc(), va, length) < 0) {
vm_unmap_range(myproc()->pgdir, va, length);
return -ENOMEM;
}
return va;
}
// /dev/pci: the offset is the physical address of a BAR.
//
// Checked against the BARs of the devices *this process has claimed*,
// which is what stops it being /dev/mem with extra steps -- without that
// test a process could map the kernel's own memory and call it a driver.
if (f && f->type == FD_INODE && f->ip->type == T_DEV &&
f->ip->major == PCIUSER_MAJOR) {
uint64 phys = PGROUNDDOWN(offset);
uint64 len = PGROUNDUP(length);
if (!pciuser_mmio_ok(phys, len))
return -EACCES;
if (!v)
return -ENOMEM;
// PTE_PWT: device registers are not memory, and a cached write to a
// doorbell arrives whenever the cache feels like it, which for a
// driver is indistinguishable from hardware that ignores you.
if (mappages(myproc()->pgdir, (void*)va, len, phys,
PTE_W | PTE_U | PTE_PWT) < 0)
return -ENOMEM;
v->addr = va; v->length = len; v->used = 1;
v->no_free = 1; // device memory is not ours to free
v->shmid = -1;
return va;
}
if (f && f->type == FD_INODE && f->ip->type == T_DEV && f->ip->major == 2) {
// Map the framebuffer where it actually is. This used to map a
// hardcoded 0xfd000000 for a hardcoded length, so a request for more
// than the screen holds handed out user-writable PTEs to whatever
// MMIO sits above the framebuffer, and any machine that put the LFB
// somewhere else got a window onto the wrong device entirely.
uint64 fb = gfx_fb_phys();
uint64 fbsz = PGROUNDUP(gfx_fb_size());
if (fb == 0 || fbsz == 0)
return -ENODEV;
if (length > fbsz)
length = fbsz;
if (!v)
return -ENOMEM;
if (mappages(myproc()->pgdir, (void*)va, length, fb, PTE_W | PTE_U) < 0)
return -1;
is_fb = 1;
// From here the drawing happens with no further syscalls, so this is
// the kernel's only chance to notice that the screen no longer belongs
// to the text console. fbclose() puts it back.
fb_mapped();
} else {
if (mmap_lazy(prot, flags, length)) {
if (vma_note_anon(myproc(), va, length) < 0)
return -ENOMEM;
return va;
}
for (uint64 a = va; a < va + length; a += PGSIZE) {
char *mem = kalloc();
if (!mem) {
vm_unmap_range(myproc()->pgdir, va, a - va);
return -ENOMEM;
}
memset(mem, 0, PGSIZE);
if (mappages(myproc()->pgdir, (void*)a, PGSIZE, V2P((uint64)mem), PTE_W | PTE_U) < 0) {
kfree(mem);
vm_unmap_range(myproc()->pgdir, va, a - va);
return -ENOMEM;
}
}
if (vma_note_anon(myproc(), va, length) < 0) {
vm_unmap_range(myproc()->pgdir, va, length);
return -ENOMEM;
}
return va;
}
v->addr = va; v->length = length; v->used = 1;
v->no_free = is_fb; v->shmid = shmid;
return va;
}
static addr_t
munmap_locked(void)
{
uint64 addr, length;
struct proc *p = myproc();
if (argaddr(0, &addr) < 0 || argaddr(1, &length) < 0)
return -1;
length = PGROUNDUP(length);
if (bad_user_range(addr, length) || (addr % PGSIZE))
return -EINVAL;
uint64 end = addr + length;
// A segment or device mapping goes only as a whole, by its own address.
// Part of one is refused: unmapping it the anonymous way below would kfree
// pages that are the framebuffer's or another process's.
for (int i = 0; i < NVMA; i++) {
struct vma *v = &p->mm->vma[i];
if (!v->used || vma_anon(v))
continue;
if (v->addr == addr) {
if (v->shmid >= 0)
shm_detach(v->shmid);
if (v->no_free)
vm_unmap_norelease(p->pgdir, addr, v->length);
else
vm_unmap_range(p->pgdir, addr, v->length);
v->used = v->addr = v->length = 0;
v->no_free = 0;
v->shmid = -1;
return 0;
}
if (addr < v->addr + v->length && end > v->addr)
return -EINVAL;
}
// Anonymous memory: trim every mapping the range touches, splitting one
// it falls in the middle of. With no slot for the split, the record keeps
// covering the hole, which is harmless -- fork and munmap skip pages that
// are not there, and the bump allocator never hands the addresses out
// again.
for (int i = 0; i < NVMA; i++) {
struct vma *v = &p->mm->vma[i];
if (!vma_anon(v))
continue;
uint64 vs = v->addr, ve = v->addr + v->length;
if (end <= vs || addr >= ve)
continue;
if (addr <= vs && end >= ve) {
v->used = v->addr = v->length = 0;
} else if (addr <= vs) {
v->addr = end;
v->length = ve - end;
} else if (end >= ve) {
v->length = addr - vs;
} else {
struct vma *w = vma_free_slot(p);
if (w) {
w->addr = end; w->length = ve - end; w->used = 1;
w->no_free = 0; w->shmid = -1;
v->length = addr - vs;
}
}
}
vm_unmap_range(p->pgdir, addr, length);
return 0;
}
addr_t sys_mprotect(void) { return 0; }
// ---------------------------------------------------------------------------
// Shared memory
//
// Two processes mapping the same physical pages. This is what a display
// server and its clients need: without it every frame is a copy through a
// pipe, and the copy is the entire cost of drawing anything.
//
// Deliberately simple. Pages are allocated up front rather than on demand,
// there is no swapping, and a segment lives until the last attachment goes
// away. What it does have is the property that matters: a store by one
// process is visible to the other, because it is the same page.
// ---------------------------------------------------------------------------
#define NSHM 16 // segments
#define SHM_MAXPAGES 512 // 2 MB each -- a 800x600x32 surface is 1.9 MB
#define IPC_PRIVATE 0
#define IPC_CREAT 01000
#define IPC_EXCL 02000
struct shmseg {
int used;
int key;
uint64 size; // rounded up to a page
uint64 bytes; // a memfd's length as ftruncate gave it
int seals; // a memfd's F_SEAL_* bits
int npages;
int nattach; // live attachments across all processes
char *pages[SHM_MAXPAGES]; // kernel virtual addresses
};
static struct shmseg shm[NSHM];
static struct spinlock shmlock;
static int shm_ready;
static void
shm_init_once(void)
{
if (!shm_ready) {
initlock(&shmlock, "shm");
shm_ready = 1;
}
}
static void
shm_free(struct shmseg *s)
{
for (int i = 0; i < s->npages; i++)
if (s->pages[i])
kfree(s->pages[i]);
memset(s, 0, sizeof(*s));
}
// fork() gives the child its own attachment to the same segment. Without
// this the count would say one attachment while two processes had it mapped,
// and the first to exit would free the pages under the other.
void
shm_ref(int id)
{
if (id < 0 || id >= NSHM)
return;
shm_init_once();
acquire(&shmlock);
if (shm[id].used)
shm[id].nattach++;
release(&shmlock);
}
// Called from munmap and from exit, once per detached mapping.
void
shm_detach(int id)
{
if (id < 0 || id >= NSHM)
return;
shm_init_once();
acquire(&shmlock);
if (shm[id].used && shm[id].nattach > 0) {
if (--shm[id].nattach == 0)
shm_free(&shm[id]);
}
release(&shmlock);
}
// --- memfd ------------------------------------------------------------------
//
// memfd_create(2): a shared memory object reached by *descriptor* rather than
// by name or by System V key. That is the whole difference and the whole
// reason it exists -- two processes share memory by passing the fd over a
// socket (SCM_RIGHTS, which already works), so neither has to agree on a path
// or race to invent a key.
//
// It is the same segment machinery underneath, which is exactly what
// docs/plan/foundations.md proposed: shmget()'s pages, refcounts, no_free
// VMAs and the fork/exit accounting are all already right. What memfd adds is
// a segment that starts empty and is sized by ftruncate(), and a file that
// owns one reference to it.
//
// This is narrower than a page cache and deliberately so. Writable MAP_SHARED
// of a *real file* still returns ENODEV, because nothing here could make its
// contract true. An anonymous object has no such contract to break.
// Grow or shrink a segment. Growing allocates zeroed pages; shrinking frees
// from the end.
//
// Only legal while nothing has it mapped. Linux allows shrinking a mapped
// memfd and faults the truncated pages out from under the mapper, which needs
// a page fault handler that can say "this page is gone"; here there is no such
// handler and a freed page would simply stay mapped to memory somebody else
// now owns. Refusing is the honest version.
static int
shm_resize(int id, uint64 newsize)
{
struct shmseg *sg;
int want, i;
uint64 len = newsize;
if (id < 0 || id >= NSHM)
return -EINVAL;
newsize = PGROUNDUP(newsize);
want = (int)(newsize / PGSIZE);
if (want > SHM_MAXPAGES)
return -ENOMEM;
shm_init_once();
acquire(&shmlock);
sg = &shm[id];
if (!sg->used) { release(&shmlock); return -EINVAL; }
// nattach is 1 for the file itself; anything above that is a live mapping.
if (sg->nattach > 1 && want < sg->npages) {
release(&shmlock);
return -EBUSY;
}
release(&shmlock);
// Allocate outside the lock: kalloc takes its own, and a page allocator
// called with a spinlock held is how the two orders meet.
for (i = sg->npages; i < want; i++) {
char *pg = kalloc();
if (!pg) {
// Undo this call's own allocations. Anything from an earlier
// successful resize stays: the object is still that size.
for (int j = sg->npages; j < i; j++) { kfree(sg->pages[j]); sg->pages[j] = 0; }
return -ENOMEM;
}
memset(pg, 0, PGSIZE);
sg->pages[i] = pg;
}
for (i = want; i < sg->npages; i++) {
if (sg->pages[i]) { kfree(sg->pages[i]); sg->pages[i] = 0; }
}
acquire(&shmlock);
sg->npages = want;
sg->size = newsize;
sg->bytes = len;
release(&shmlock);
return 0;
}
// One page of a segment, by index. Used by the mmap path; the caller has
// already taken a reference, so the page cannot go away underneath it.
char *
shm_page(int id, int pg)
{
char *p;
if (id < 0 || id >= NSHM || pg < 0)
return 0;
shm_init_once();
acquire(&shmlock);
p = (shm[id].used && pg < shm[id].npages) ? shm[id].pages[pg] : 0;
release(&shmlock);
return p;
}
uint64
memfd_size(int id)
{
uint64 n;
if (id < 0 || id >= NSHM)
return 0;
shm_init_once();
acquire(&shmlock);
// The length asked for, not the pages behind it. fstat reports this, and
// memmap2 -- which glycin uses to share every decoded image -- sizes its
// mapping by it: 24x24 icons came back as 4096 bytes instead of 2304, and
// the PNG decoder asserted.
n = shm[id].used ? shm[id].bytes : 0;
release(&shmlock);
return n;
}
int
memfd_truncate(int id, uint64 len)
{
if (id < 0 || id >= NSHM)
return -EINVAL;
uint64 now = memfd_size(id);
if ((len < now && (shm[id].seals & F_SEAL_SHRINK)) ||
(len > now && (shm[id].seals & F_SEAL_GROW)))
return -EPERM;
return shm_resize(id, len);
}
// File seals: promises about a memfd that whoever receives it can rely on.
// glycin seals each decoded image before handing it over, and gdk-pixbuf
// first asks for the seals just to learn the descriptor is a memfd at all.
//
// Linux refuses F_SEAL_WRITE while a writable mapping exists (EBUSY). That
// needs every mapping of the segment to be made read-only on the spot, which
// takes the page fault handling this kernel does not have, so the seal is
// recorded and later writes, truncates and writable mappings are refused --
// the mapping that already exists keeps writing.
int
memfd_seals(int id, int add)
{
int r;
if (id < 0 || id >= NSHM)
return -EINVAL;
acquire(&shmlock);
if (!shm[id].used)
r = -EINVAL;
else if (add < 0)
r = shm[id].seals;
else if (shm[id].seals & F_SEAL_SEAL)
r = -EPERM;
else if (add & ~(F_SEAL_SEAL | F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
r = -EINVAL;
else {
shm[id].seals |= add;
r = 0;
}
release(&shmlock);
return r;
}
// read()/write() on a memfd, so it is a file rather than a token that only
// mmap understands. Wayland clients use ftruncate and mmap, but a descriptor
// that refuses read() is a surprise nobody needs.
int
memfd_rw(int id, int user, uint64 buf, uint64 off, int n, int writing)
{
struct shmseg *sg;
int done = 0;
if (id < 0 || id >= NSHM || n <= 0)
return 0;
shm_init_once();
sg = &shm[id];
if (!sg->used)
return -EINVAL;
if (writing && (sg->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)))
return -EPERM;
if (off >= sg->bytes)
return 0; // EOF, or a write past the end
if (off + (uint64)n > sg->bytes)
n = (int)(sg->bytes - off);
while (done < n) {
uint64 pos = off + (uint64)done;
int pg = (int)(pos / PGSIZE);
int inpg = (int)(pos % PGSIZE);
int want = PGSIZE - inpg;
if (want > n - done)
want = n - done;
char *p = sg->pages[pg] + inpg;
if (writing) {
if (user) { if (copy_from_user(p, buf + done, want) < 0) return -EFAULT; }
else memmove(p, (char*)(buf + done), want);
} else {
if (user) { if (copy_to_user(buf + done, p, want) < 0) return -EFAULT; }
else memmove((char*)(buf + done), p, want);
}
done += want;
}
return done;
}
addr_t
sys_memfd_create(void)
{
addr_t namep;
int flags, id = -1, fd;
struct file *f;
argaddr(0, &namep);
argint(1, &flags);
// The name is for /proc and for debuggers. There is no /proc/<pid>/fd
// here, so it is read for validation and dropped -- which is honest, and
// better than storing a string nothing will ever show.
if (namep) {
char *nm;
if (fetchstr(namep, &nm) < 0)
return -EFAULT;
}
shm_init_once();
acquire(&shmlock);
for (int i = 0; i < NSHM; i++)
if (!shm[i].used) {
memset(&shm[i], 0, sizeof(shm[i]));
shm[i].used = 1;
shm[i].key = IPC_PRIVATE;
shm[i].size = 0;
shm[i].npages = 0;
shm[i].nattach = 1; // the file holds this one
// Without MFD_ALLOW_SEALING the object is sealed against sealing.
shm[i].seals = (flags & MFD_ALLOW_SEALING) ? 0 : F_SEAL_SEAL;
id = i;
break;
}
release(&shmlock);
if (id < 0)
return -ENFILE;
if ((f = filealloc()) == 0) { shm_detach(id); return -ENFILE; }
if ((fd = fdalloc(f)) < 0) { fileclose(f); return -EMFILE; }
if (flags & MFD_CLOEXEC)
myproc()->files->flags[fd] = FD_CLOEXEC;
f->type = FD_MEMFD;
f->memfd_seg = id;
f->readable = 1;
f->writable = 1;
f->off = 0;
return fd;
}
addr_t
sys_shmget(void)
{
int key, flags;
addr_t size;
if (argint(0, &key) < 0 || argaddr(1, &size) < 0 || argint(2, &flags) < 0)
return -EINVAL;
size = PGROUNDUP(size);
if (size == 0 || size > (uint64)SHM_MAXPAGES * PGSIZE)
return -EINVAL;
shm_init_once();
acquire(&shmlock);
if (key != IPC_PRIVATE) {
for (int i = 0; i < NSHM; i++) {
if (shm[i].used && shm[i].key == key) {
if ((flags & IPC_CREAT) && (flags & IPC_EXCL)) {
release(&shmlock);
return -EEXIST;
}
if (shm[i].size < size) {
release(&shmlock);
return -EINVAL;
}
release(&shmlock);
return i;
}
}
if (!(flags & IPC_CREAT)) {
release(&shmlock);
return -ENOENT;
}
}
for (int i = 0; i < NSHM; i++) {
if (shm[i].used)
continue;
struct shmseg *s = &shm[i];
s->npages = (int)(size / PGSIZE);
for (int pg = 0; pg < s->npages; pg++) {
s->pages[pg] = kalloc();
if (!s->pages[pg]) {
for (int j = 0; j < pg; j++) kfree(s->pages[j]);
memset(s, 0, sizeof(*s));
release(&shmlock);
return -ENOMEM;
}
// Zeroed, because a fresh segment handing out the last tenant's
// bytes would leak them between processes.
memset(s->pages[pg], 0, PGSIZE);
}
s->used = 1;
s->key = key;
s->size = size;
s->nattach = 0;
release(&shmlock);
return i;
}
release(&shmlock);
return -ENOSPC;
}
static addr_t
shmat_locked(void)
{
int id, flags;
addr_t addr;
if (argint(0, &id) < 0 || argaddr(1, &addr) < 0 || argint(2, &flags) < 0)
return -EINVAL;
if (id < 0 || id >= NSHM)
return -EINVAL;
shm_init_once();
acquire(&shmlock);
if (!shm[id].used) {
release(&shmlock);
return -EINVAL;
}
uint64 len = shm[id].size;
int npages = shm[id].npages;
shm[id].nattach++; // claimed before the lock is dropped
release(&shmlock);
struct vma *v = 0;
for (int i = 0; i < NVMA; i++)
if (!myproc()->mm->vma[i].used) { v = &myproc()->mm->vma[i]; break; }
if (!v) { shm_detach(id); return -ENOMEM; }
// Ignore the hint and take a fresh range, the same way mmap does.
if (myproc()->mm->mmap_next == 0)
myproc()->mm->mmap_next = 0x40000000;
uint64 va = myproc()->mm->mmap_next;
if (bad_user_range(va, len)) { shm_detach(id); return -ENOMEM; }
myproc()->mm->mmap_next += len;
for (int pg = 0; pg < npages; pg++) {
if (mappages(myproc()->pgdir, (void*)(va + (uint64)pg * PGSIZE), PGSIZE,
V2P((uint64)shm[id].pages[pg]), PTE_W | PTE_U) < 0) {
vm_unmap_norelease(myproc()->pgdir, va, (uint64)pg * PGSIZE);
shm_detach(id);
return -ENOMEM;
}
}
v->addr = va; v->length = len; v->used = 1;
v->no_free = 1; // the segment owns these pages
v->shmid = id;
return va;
}
// Drop every mapping this process holds that it does not own the pages of,
// before its page tables are torn down.
//
// freevm() frees any mapped page below PHYSTOP, which is right for the heap
// and the stack and wrong for anything shared. The framebuffer survived only
// because it sits above PHYSTOP and was skipped by accident; shared memory is
// kalloc'd, sits below it, and would be handed to kfree while another process
// was still using it -- the worst kind of bug, silent and remote.
//
// Called from exit() and from exec() before the old address space goes away.
void
mm_release_shared(struct mm *m)
{
for (int i = 0; i < NVMA; i++) {
if (!m->vma[i].used || !m->vma[i].no_free)
continue;
if (m->pgdir)
vm_unmap_norelease(m->pgdir, m->vma[i].addr, m->vma[i].length);
if (m->vma[i].shmid >= 0)
shm_detach(m->vma[i].shmid);
m->vma[i].used = 0;
m->vma[i].no_free = 0;
m->vma[i].shmid = -1;
}
}
static addr_t
shmdt_locked(void)
{
addr_t addr;
if (argaddr(0, &addr) < 0)
return -EINVAL;
for (int i = 0; i < NVMA; i++) {
if (myproc()->mm->vma[i].used && myproc()->mm->vma[i].addr == addr &&
myproc()->mm->vma[i].shmid >= 0) {
vm_unmap_norelease(myproc()->pgdir, addr, myproc()->mm->vma[i].length);
shm_detach(myproc()->mm->vma[i].shmid);
myproc()->mm->vma[i].used = myproc()->mm->vma[i].addr = myproc()->mm->vma[i].length = 0;
myproc()->mm->vma[i].no_free = 0;
myproc()->mm->vma[i].shmid = -1;
return 0;
}
}
return -EINVAL;
}
// Memory advice/hints — all no-ops; kernel decides on its own.
addr_t sys_madvise(void) { return 0; }
addr_t sys_msync(void) { return 0; }
addr_t sys_mincore(void) { return 0; }
addr_t sys_mlock(void) { return 0; }
addr_t sys_munlock(void) { return 0; }
addr_t sys_mlockall(void) { return 0; }
addr_t sys_munlockall(void){ return 0; }
addr_t sys_fadvise64(void) { return 0; }
// mremap: we don't support region relocation; return ENOMEM.
addr_t sys_mremap(void) { return -ENOMEM; }
// Resource limits: always report unlimited.
addr_t sys_setrlimit(void) { return 0; }
// Process priority. These returned a flat 0 for as long as the scheduler had
// no notion of priority to report; it has one now (kernel/proc/proc.c), and a
// stub that says "everything is nice 0" is a lie the moment `renice` works.
//
// PRIO_PROCESS is 0, and `who` of 0 means the caller -- both from POSIX.
#define PRIO_PROCESS 0
addr_t
sys_getpriority(void)
{
int which, who, nice = 0, r;
argint(0, &which); argint(1, &who);
if (which != PRIO_PROCESS)
return -EINVAL;
if ((r = proc_getnice(who, &nice)) < 0)
return r;
// getpriority(2) returns the nice value *offset by 20*, because -1 is a
// legal nice level and also how a syscall says "error". libc subtracts the
// 20 back off. Returning the raw nice value here makes `renice -n -1` look
// like a failure to every caller that checks.
return (addr_t)(20 - nice);
}
addr_t
sys_setpriority(void)
{
int which, who, nice;
argint(0, &which); argint(1, &who); argint(2, &nice);
if (which != PRIO_PROCESS)
return -EINVAL;
return (addr_t)proc_setnice(who, nice);
}
// One mapping change at a time per address space: threads share the VMA table
// and the page tables, and two of them allocating the same missing page-table
// level at once would each install their own and lose the other's mappings.
#define MM_LOCKED(name) \
addr_t \
sys_##name(void) \
{ \
struct mm *m = myproc()->mm; \
mm_lock(m); \
addr_t r = name##_locked(); \
mm_unlock(m); \
return r; \
}
MM_LOCKED(mmap)
MM_LOCKED(munmap)
MM_LOCKED(shmat)
MM_LOCKED(shmdt)
Reference in a new issue View git blame Copy permalink