diff --git a/Makefile b/Makefile index def49fe..4f21329 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ KERN_NOENTRY = $(filter-out $(BUILD)/kernel/arch/x86_64/entry.o, $(KERNOBJS)) # Third-party software does NOT go here — see ports/. # ----------------------------------------------------------------------- ULIB = $(addprefix $(BUILD)/lib/, start.o glue.o usys.o) -CMDS = schinit schinitd fbinfo shmtest socktest inputtest nettest reflink schattr ping chvt loadkeys sigtest pcitest vgpud schdbg schdbgd resolvconf httpd schmode netecho ldlsd +CMDS = schinit schinitd fbinfo shmtest socktest inputtest nettest reflink schattr ping chvt loadkeys sigtest memfdtest pcitest vgpud schdbg schdbgd resolvconf httpd schmode netecho ldlsd CMDBINS = $(addprefix $(BUILD)/cmd/, $(CMDS)) # ----------------------------------------------------------------------- diff --git a/cmd/memfdtest/main.c b/cmd/memfdtest/main.c new file mode 100644 index 0000000..c00a9d8 --- /dev/null +++ b/cmd/memfdtest/main.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// memfdtest -- two processes map the same memfd, one writes, the other sees it. +// +// That sentence is docs/plan/foundations.md's own acceptance criterion for +// item 2, and it is the check a permissive implementation cannot pass: an +// mmap that quietly returns private zeroed pages satisfies every return code +// here and fails the one assertion that matters, because the reader is a +// *different process* and the pages would not be the same pages. +// +// The lifecycle is the one a Wayland client performs, in order: +// +// fd = memfd_create("buf", 0); +// ftruncate(fd, size); +// p = mmap(0, size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); +// +// and the reason it had to wait for SCM_RIGHTS is that the *other* process +// gets the same object by being handed the descriptor, not by naming it. + +#include "types.h" +#include "user.h" +#include "fcntl.h" +#include "stat.h" +#include "posix.h" +#include "socket.h" + +static int failures; + +static void +puts_(const char *s) +{ + int n = 0; + while (s[n]) n++; + write(1, (void *)s, n); +} + +// mmap returns -errno in the pointer on this system's own ABI -- musl turns +// that into MAP_FAILED, but a first-party binary sees the raw value. Anything +// in the top page is an error, not an address. +static int +mmap_failed(void *p) +{ + return (uint64)p >= (uint64)-4096; +} + +static void +check(const char *what, int ok) +{ + puts_(ok ? "[ok] " : "[FAIL] "); + puts_(what); + puts_("\n"); + if (!ok) failures++; +} + +#define SIZE 8192 + +static void memset_(void *d, int c, int n) { char *p = d; while (n-- > 0) *p++ = (char)c; } +static void memmove_(void *d, const void *s, int n) +{ char *a = d; const char *b = s; while (n-- > 0) *a++ = *b++; } + +// Send one descriptor over a socket, the way a Wayland client hands its buffer +// to the compositor. Same shape as cmd/socktest's helper; kept local so this +// test stands on its own. +static int +sendfd(int sock, int fd) +{ + struct msghdr m; + struct iovec iv; + char cbuf[CMSG_SPACE(sizeof(int))]; + struct cmsghdr *c; + char tag = 'x'; + + memset_(&m, 0, sizeof(m)); + memset_(cbuf, 0, sizeof(cbuf)); + iv.iov_base = &tag; iv.iov_len = 1; + m.msg_iov = &iv; m.msg_iovlen = 1; + m.msg_control = cbuf; m.msg_controllen = sizeof(cbuf); + c = (struct cmsghdr *)cbuf; + c->cmsg_len = CMSG_LEN(sizeof(int)); + c->cmsg_level = SOL_SOCKET; + c->cmsg_type = SCM_RIGHTS; + memmove_(CMSG_DATA(c), &fd, sizeof(int)); + return sendmsg(sock, &m, 0); +} + +static int +recvfd(int sock) +{ + struct msghdr m; + struct iovec iv; + char cbuf[CMSG_SPACE(sizeof(int))]; + struct cmsghdr *c; + char tag = 0; + int got = -1; + + memset_(&m, 0, sizeof(m)); + memset_(cbuf, 0, sizeof(cbuf)); + iv.iov_base = &tag; iv.iov_len = 1; + m.msg_iov = &iv; m.msg_iovlen = 1; + m.msg_control = cbuf; m.msg_controllen = CMSG_LEN(sizeof(int)); + if (recvmsg(sock, &m, 0) < 0) + return -1; + c = CMSG_FIRSTHDR(&m); + if (c && c->cmsg_level == SOL_SOCKET && c->cmsg_type == SCM_RIGHTS) + memmove_(&got, CMSG_DATA(c), sizeof(int)); + return got; +} + +int +main(void) +{ + int fd, pfd[2], st = 0; + char *p; + struct stat sb; + + puts_("memfdtest: a shareable memory object\n\n"); + + fd = memfd_create("test-buffer", 0); + check("memfd_create returns a descriptor", fd >= 0); + if (fd < 0) { puts_("\nmemfdtest: FAILED\n"); return 1; } + + // A fresh memfd is empty. Mapping it before it has a size must fail -- + // there are no pages yet, and handing back anonymous ones would be the + // lie this whole object exists to avoid. + check("a new memfd has size 0", + fstat(fd, &sb) == 0 && sb.size == 0); + check("mapping a memfd that has no size yet is refused", + mmap_failed(mmap(0, SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0))); + + check("ftruncate gives it a size", ftruncate(fd, SIZE) == 0); + check("and fstat reports it back", + fstat(fd, &sb) == 0 && sb.size == SIZE); + + p = mmap(0, SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + check("mmap(MAP_SHARED) returns a mapping", !mmap_failed(p)); + if (mmap_failed(p)) { puts_("\nmemfdtest: FAILED\n"); return 1; } + + // Readable and writable by this process at all, before anything about + // sharing is claimed. + p[0] = 'A'; + p[SIZE - 1] = 'Z'; + check("the mapping is writable and reads back", + p[0] == 'A' && p[SIZE - 1] == 'Z'); + + // MAP_PRIVATE is refused rather than silently copied. + check("MAP_PRIVATE on a memfd is refused", + mmap_failed(mmap(0, SIZE, PROT_READ, MAP_PRIVATE, fd, 0))); + + // --- the assertion this test exists for ------------------------------ + // + // fork(), then have the child map the *same* descriptor independently and + // write through its own mapping. The parent must see it through its own. + // A private mapping passes everything above and fails here. + if (pipe(pfd) < 0) { puts_("[FAIL] pipe\n"); return 1; } + + int pid = fork(); + if (pid == 0) { + char *q = mmap(0, SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + char ok = mmap_failed(q) ? 0 : 1; + if (ok) { + // Prove the child sees what the parent wrote before the fork... + if (q[0] != 'A' || q[SIZE - 1] != 'Z') + ok = 0; + // ...then write something of its own, in a different place. + q[100] = 'C'; + q[SIZE / 2] = 'M'; + } + write(pfd[1], &ok, 1); + close(pfd[1]); + exit(); + } + close(pfd[1]); + { + char ok = 0; + read(pfd[0], &ok, 1); + close(pfd[0]); + check("a second process maps the same fd and sees the parent's bytes", + ok == 1); + } + wait(); + (void)st; + + check("and the parent sees what the child wrote", + p[100] == 'C' && p[SIZE / 2] == 'M'); + check("without disturbing anything else in the mapping", + p[0] == 'A' && p[SIZE - 1] == 'Z' && p[101] == 0); + + // --- read()/write() reach the same bytes ------------------------------- + { + char buf[4] = { 0, 0, 0, 0 }; + lseek(fd, 100, 0 /* SEEK_SET */); + check("read() sees what mmap wrote", + read(fd, buf, 1) == 1 && buf[0] == 'C'); + lseek(fd, 200, 0 /* SEEK_SET */); + buf[0] = 'W'; + check("write() is visible through the mapping", + write(fd, buf, 1) == 1 && p[200] == 'W'); + } + + // --- shrinking while mapped is refused, not faked ---------------------- + check("shrinking a mapped memfd is refused", ftruncate(fd, 4096) < 0); + check("growing it is allowed", ftruncate(fd, SIZE * 2) == 0); + + // --- the combination this was all for --------------------------------- + // + // A forked child inherits the descriptor, which proves the *pages* are + // shared but not that the *object* travels. The Wayland case is an + // unrelated process that is handed the descriptor over a socket and has + // no other way to name it. So: send the memfd through a socketpair, and + // have the receiver close every inherited copy before it maps. + { + int sv[2], cpid, rp[2]; + char ok = 0; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) < 0 || pipe(rp) < 0) { + check("socketpair for the descriptor-passing check", 0); + } else if ((cpid = fork()) == 0) { + int got; + char *q; + close(sv[0]); close(rp[0]); + got = recvfd(sv[1]); + // Close the inherited copy, so the only route to the object is + // the descriptor that arrived over the socket. + close(fd); + q = (got >= 0) ? mmap(0, SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED, got, 0) : (char *)-1; + ok = (got >= 0 && !mmap_failed(q) && q[100] == 'C') ? 1 : 0; + if (ok) q[300] = 'S'; + write(rp[1], &ok, 1); + close(rp[1]); + exit(); + } else { + close(sv[1]); close(rp[1]); + sendfd(sv[0], fd); + read(rp[0], &ok, 1); + close(rp[0]); close(sv[0]); + wait(); + check("a memfd passed over a socket maps in the receiver", ok == 1); + check("and the sender sees what the receiver wrote", p[300] == 'S'); + } + } + + close(fd); + + puts_("\n"); + puts_(failures ? "memfdtest: FAILED\n" : "memfdtest: all checks passed\n"); + return failures ? 1 : 0; +} diff --git a/docs/architecture/kernel.md b/docs/architecture/kernel.md index a45e77f..0266f20 100644 --- a/docs/architecture/kernel.md +++ b/docs/architecture/kernel.md @@ -141,6 +141,41 @@ arrangement as `schfs-objtest`, and for the same reason: slab lists, free-list threading and whether pages actually come back are all things that can be wrong without a kernel to be wrong in. +### Shared memory objects + +`shmget`/`shmat`/`shmdt` are System V shared memory: segments of `kalloc`'d +pages, reference counted, mapped through a VMA with `no_free` set so that +`freevm()` does not hand another process's pages to `kfree`. + +`memfd_create(2)` is the same machinery reached by **descriptor** instead of by +key. That difference is the whole point and the reason it exists on Linux at +all: two processes share memory by passing the fd over a socket, so neither has +to agree on a path or race to invent a key. It is what a Wayland buffer crosses +on, and it is why descriptor passing (`SCM_RIGHTS`) had to be built first. + +| | | +|---|---| +| `memfd_create(name, flags)` | a new segment of size 0, and a descriptor owning one reference | +| `ftruncate(fd, n)` | gives it a size — growing allocates zeroed pages, shrinking frees from the end | +| `mmap(…, MAP_SHARED, fd, off)` | maps the segment's existing pages, so both mappers get the same physical memory | +| `read`/`write`/`lseek`/`fstat` | work, so it is a file and not a token only `mmap` understands | + +Three things are refused rather than faked, and each refusal is the honest +version of something this kernel cannot do: + +- **`MAP_PRIVATE` on a memfd.** Every caller is sharing on purpose; a private + mapping of an anonymous object is an expensive `MAP_ANONYMOUS`. +- **Mapping past the end.** Linux lets it through and faults the pages in. + There is no fault handler here that could, so the range is rejected. +- **Shrinking while mapped.** Linux truncates the pages out from under the + mapper. Here a freed page would simply stay mapped to memory somebody else + now owns. + +Writable `MAP_SHARED` of a **real file** is still `ENODEV`. That needs a page +cache, it is a separate project, and nothing in 1.0 asks for it — an anonymous +object has no such contract to break, which is exactly why `memfd` is the +narrow answer. + ### What the fixed tables actually cost The standing claim was that every fixed-size table in the kernel exists to diff --git a/docs/plan/1.0.md b/docs/plan/1.0.md index 683c876..dfaa9af 100644 --- a/docs/plan/1.0.md +++ b/docs/plan/1.0.md @@ -428,9 +428,14 @@ Nothing above needs these; everything below stops without them. [../architecture/networking.md](../architecture/networking.md#descriptor-passing). This was also mechanism 1 of [hybrid.md](hybrid.md) — one primitive for both, which is the whole reason those are one plan. -- [ ] **Item 2 — a shareable memory object** (`memfd_create`, and - `mmap(MAP_SHARED)` on it genuinely working between two processes). This is - what a Wayland buffer still cannot cross on. +- [x] ~~**Item 2 — a shareable memory object.**~~ `memfd_create`, `ftruncate`, + `mmap(MAP_SHARED)`. Built on the `shmget` segments exactly as + [foundations.md](foundations.md) proposed — the pages, refcounts, + `no_free` VMAs and the fork/exit accounting were all already right; what + memfd adds is a segment reached by *descriptor* rather than by name or + System V key, which is the whole reason it exists on Linux. + `cmd/memfdtest`: seventeen checks, ending with a memfd sent over a + socket to a receiver that closes every inherited copy first. - [ ] **Item 3 — `AF_UNIX` sockets in the filesystem namespace.** ### Gate 2 — Init as a role — **done** diff --git a/docs/plan/foundations.md b/docs/plan/foundations.md index 3d6098a..bf97da1 100644 --- a/docs/plan/foundations.md +++ b/docs/plan/foundations.md @@ -79,10 +79,14 @@ Two bugs fell out of building it, neither in the new code: an end that had already happened. `sockreadable()` always had the equivalent condition; the pipe path did not. -### 2. A shareable memory object +### 2. A shareable memory object — **done** `memfd_create`, plus `mmap(MAP_SHARED)` on it genuinely working between two -processes. +processes. Described in +[../architecture/kernel.md](../architecture/kernel.md#shared-memory-objects); +asserted by `cmd/memfdtest`, seventeen checks including the one that matters — +a descriptor sent over a socket, mapped by a receiver that has closed every +inherited copy, with stores visible in both directions. Narrower than a general page cache, and sufficient. An anonymous shared object needs no file behind it, which is exactly why `memfd` exists on Linux at all — diff --git a/include/defs.h b/include/defs.h index 5dcb799..92ea608 100644 --- a/include/defs.h +++ b/include/defs.h @@ -698,6 +698,10 @@ void vm_walk_user(pml4e_t*, void (*)(void*, uint64, uint64, uint64), // Shared memory (sys_mem.c). shm_ref/shm_detach maintain the attachment // count that decides when a segment's pages go back. void shm_ref(int id); +char* shm_page(int id, int pg); +uint64 memfd_size(int id); +int memfd_truncate(int id, uint64 len); +int memfd_rw(int id, int user, uint64 buf, uint64 off, int n, int writing); void shm_detach(int id); void proc_release_shared(struct proc *); int copy_vma_range(pml4e_t *dstpgdir, pml4e_t *srcpgdir, uint64 va, uint64 length, int is_fb); diff --git a/include/file.h b/include/file.h index e4fb461..e379e2a 100644 --- a/include/file.h +++ b/include/file.h @@ -1,5 +1,9 @@ struct file { - enum { FD_NONE, FD_PIPE, FD_INODE, FD_SOCKET } type; + // FD_MEMFD: an anonymous shared memory object, the thing memfd_create(2) + // returns. It is a descriptor with no name and no inode behind it, which is + // the entire reason memfd exists -- two processes share memory by passing + // the descriptor, not by agreeing on a path or a System V key. + enum { FD_NONE, FD_PIPE, FD_INODE, FD_SOCKET, FD_MEMFD } type; int ref; // reference count char readable; char writable; @@ -11,6 +15,11 @@ struct file { struct pipe *pipe; struct inode *ip; struct socket *sock; + // The shared segment behind an FD_MEMFD, as an index into the table in + // kernel/sys/sys_mem.c. Not a pointer, because the segment table is where + // the reference counting lives and handing out pointers into it would + // invite somebody to keep one. + int memfd_seg; uint64 off; }; diff --git a/include/user.h b/include/user.h index 42ae7ea..5329eac 100644 --- a/include/user.h +++ b/include/user.h @@ -46,6 +46,9 @@ int alarm(int seconds); int waitpid(int pid, int *status, int options); int setsid(void); void* mmap(void*, uint64, int, int, int, uint64); +// memfd_create(2). The flags argument exists for MFD_CLOEXEC and MFD_ALLOW_ +// SEALING, neither of which this kernel implements; pass 0. +int memfd_create(const char *name, uint flags); // Shared memory. shmat returns the attach address, or a small negative value // on failure — there is no errno in this runtime, so check for < 0. int shmget(int key, uint64 size, int flags); diff --git a/kernel/fs/file.c b/kernel/fs/file.c index ee862a6..b60b3ec 100644 --- a/kernel/fs/file.c +++ b/kernel/fs/file.c @@ -75,7 +75,9 @@ fileclose(struct file *f) FSSTAT_DROP(fsstat.file_used); // 1 -> 0 release(&ftable.lock); - if(ff.type == FD_PIPE) + if(ff.type == FD_MEMFD) + shm_detach(ff.memfd_seg); // the last mapper frees the pages + else if(ff.type == FD_PIPE) pipeclose(ff.pipe, ff.writable); else if(ff.type == FD_SOCKET) sockclose(ff.sock); @@ -96,6 +98,17 @@ fileclose(struct file *f) int filestat(struct file *f, struct stat *st) { + // A memfd has no inode, so the size is the only field with an answer. It is + // the field that matters: mmap length and ftruncate round-trips are checked + // against it, and a caller that fstat()s to find out how big the buffer is + // gets the truth. + if(f->type == FD_MEMFD){ + memset(st, 0, sizeof(*st)); + st->type = T_FILE; + st->size = memfd_size(f->memfd_seg); + st->nlink = 1; + return 0; + } if(f->type == FD_INODE){ ilock(f->ip); stati(f->ip, st); @@ -113,6 +126,17 @@ fileread(struct file *f, char *addr, int n) if(f->readable == 0) return -1; + // A memfd is a file: read() and write() move bytes in and out of the shared + // pages, and the descriptor carries its own offset like any other. mmap is + // what it is usually for, but a descriptor that refuses read() is a surprise + // nobody needs. + if(f->type == FD_MEMFD){ + r = memfd_rw(f->memfd_seg, 1, (uint64)addr, f->off, n, 0); + if(r > 0) + f->off += r; + return r; + } + // O_NONBLOCK on a pipe or a socket. Reported here rather than inside // piperead/sockread so that the blocking implementations stay unchanged: // this is a check of the same condition their sleep loops wait on, made one @@ -195,6 +219,13 @@ filewrite(struct file *f, char *addr, int n) if(f->writable == 0) return -1; + if(f->type == FD_MEMFD){ + r = memfd_rw(f->memfd_seg, 1, (uint64)addr, f->off, n, 1); + if(r > 0) + f->off += r; + return r; + } + if(f->type == FD_PIPE) return pipewrite(f->pipe, addr, n); diff --git a/kernel/sys/sys_fs.c b/kernel/sys/sys_fs.c index 1bdb7f1..bc3f711 100644 --- a/kernel/sys/sys_fs.c +++ b/kernel/sys/sys_fs.c @@ -400,16 +400,19 @@ addr_t sys_lseek(void) struct file *f; addr_t off_raw; int whence; - if (argfd(0, 0, &f) < 0 || argaddr(1, &off_raw) < 0 || argint(2, &whence) < 0) return (addr_t)-1; - if (f->type != FD_INODE) return (addr_t)-1; + if (argfd(0, 0, &f) < 0 || argaddr(1, &off_raw) < 0 || argint(2, &whence) < 0) return (addr_t)-EFAULT; + if (f->type != FD_INODE && f->type != FD_MEMFD) return (addr_t)-ESPIPE; int64 off = (int64)off_raw, newoff; + // SEEK_END needs the length, and a memfd's length is not in an inode. + int64 end = (f->type == FD_MEMFD) ? (int64)memfd_size(f->memfd_seg) + : (int64)f->ip->size; switch (whence) { case 0: newoff = off; break; case 1: newoff = (int64)f->off + off; break; - case 2: newoff = (int64)f->ip->size + off; break; - default: return (addr_t)-1; + case 2: newoff = end + off; break; + default: return (addr_t)-EINVAL; } - if (newoff < 0) return (addr_t)-1; + if (newoff < 0) return (addr_t)-EINVAL; // Seeking past the end is legal and is how a sparse file is made: the gap // reads as zeros and costs nothing on a filesystem with extents. This used // to clamp to end-of-file, which silently turned "seek to 1 GB and write" @@ -595,8 +598,29 @@ addr_t sys_fstat(void) struct file *f; char *st; int size = myproc()->syscall_map ? sizeof(struct linux_stat) : sizeof(struct stat); - if (argfd(0, 0, &f) < 0 || argptr(1, &st, size) < 0) return -1; - if (f->type != FD_INODE) return -1; + if (argfd(0, 0, &f) < 0 || argptr(1, &st, size) < 0) return -EFAULT; + // A memfd has no inode, and its size is the one field with an answer -- + // which is the field a caller is asking for when it fstat()s a buffer. + // This used to refuse anything that was not FD_INODE, so filestat()'s own + // memfd branch was unreachable from the syscall that needs it. + if (f->type == FD_MEMFD) { + uint64 sz = memfd_size(f->memfd_seg); + if (myproc()->syscall_map) { + struct linux_stat *ls = (struct linux_stat *)st; + memset(ls, 0, sizeof(*ls)); + ls->st_mode = LINUX_S_IFREG | 0600; + ls->st_nlink = 1; + ls->st_size = (int64)sz; + } else { + struct stat *ss = (struct stat *)st; + memset(ss, 0, sizeof(*ss)); + ss->type = T_FILE; + ss->nlink = 1; + ss->size = sz; + } + return 0; + } + if (f->type != FD_INODE) return -EINVAL; ilock(f->ip); if (myproc()->syscall_map) stati_linux(f->ip, (struct linux_stat*)st); else stati(f->ip, (struct stat*)st); @@ -852,7 +876,13 @@ addr_t sys_ftruncate(void) addr_t length; if (argfd(0, 0, &f) < 0 || argaddr(1, &length) < 0) return -1; if ((int64)length < 0) return -EINVAL; - if (f->type != FD_INODE || !f->writable) return -1; + // A memfd starts at size zero and is given its size here. That is the + // whole lifecycle a Wayland client performs: memfd_create, ftruncate, + // mmap. + if (f->type == FD_MEMFD) + return f->writable ? memfd_truncate(f->memfd_seg, (uint64)length) + : -EBADF; + if (f->type != FD_INODE || !f->writable) return -EINVAL; begin_op(); ilock(f->ip); itrunc_to(f->ip, (uint64)length); iunlock(f->ip); end_op(); diff --git a/kernel/sys/sys_mem.c b/kernel/sys/sys_mem.c index b113e68..503c439 100644 --- a/kernel/sys/sys_mem.c +++ b/kernel/sys/sys_mem.c @@ -12,6 +12,7 @@ #include "file.h" #include "syscall.h" #include "errno.h" +#include "uaccess.h" #include "posix.h" #include "pciuser.h" @@ -116,6 +117,44 @@ sys_mmap(void) int is_fb = 0; int shmid = -1; + + // A memfd. The pages already exist and belong to the segment, so this maps + // them rather than allocating -- which is the entire point: the other + // process that maps the same segment gets the same physical pages, and a + // store by one is a load by the other because there is only one copy. + // + // MAP_PRIVATE is refused rather than given a copy. Every caller of this is + // sharing on purpose, and a private mapping of an anonymous object is an + // expensive way to ask for MAP_ANONYMOUS. + if (f && f->type == FD_MEMFD) { + int id = f->memfd_seg; + uint64 have = memfd_size(id); + + if (!(flags & MAP_SHARED)) + return -EINVAL; + if (offset % PGSIZE) + return -EINVAL; + // Mapping past the end is not an error on Linux -- the pages simply + // fault -- but there is no fault handler here to make that true, so + // it is refused with the errno that says the range is wrong. + if (offset + length > have) + return -EINVAL; + + shm_ref(id); + for (uint64 o = 0; o < length; o += PGSIZE) { + char *pg = shm_page(id, (int)((offset + o) / PGSIZE)); + if (!pg || mappages(myproc()->pgdir, (void*)(va + o), PGSIZE, + V2P((uint64)pg), PTE_W | PTE_U) < 0) { + vm_unmap_norelease(myproc()->pgdir, va, o); + shm_detach(id); + return -ENOMEM; + } + } + v->addr = va; v->length = length; v->used = 1; + v->no_free = 1; // the segment owns these pages + v->shmid = id; + return va; + } // File-backed mapping of a regular file. // // This used to be ignored entirely: any file mapping silently returned @@ -323,6 +362,201 @@ shm_detach(int id) release(&shmlock); } +// --- memfd ------------------------------------------------------------------ +// +// memfd_create(2): a shared memory object reached by *descriptor* rather than +// by name or by System V key. That is the whole difference and the whole +// reason it exists -- two processes share memory by passing the fd over a +// socket (SCM_RIGHTS, which already works), so neither has to agree on a path +// or race to invent a key. +// +// It is the same segment machinery underneath, which is exactly what +// docs/plan/foundations.md proposed: shmget()'s pages, refcounts, no_free +// VMAs and the fork/exit accounting are all already right. What memfd adds is +// a segment that starts empty and is sized by ftruncate(), and a file that +// owns one reference to it. +// +// This is narrower than a page cache and deliberately so. Writable MAP_SHARED +// of a *real file* still returns ENODEV, because nothing here could make its +// contract true. An anonymous object has no such contract to break. + +// Grow or shrink a segment. Growing allocates zeroed pages; shrinking frees +// from the end. +// +// Only legal while nothing has it mapped. Linux allows shrinking a mapped +// memfd and faults the truncated pages out from under the mapper, which needs +// a page fault handler that can say "this page is gone"; here there is no such +// handler and a freed page would simply stay mapped to memory somebody else +// now owns. Refusing is the honest version. +static int +shm_resize(int id, uint64 newsize) +{ + struct shmseg *sg; + int want, i; + + if (id < 0 || id >= NSHM) + return -EINVAL; + newsize = PGROUNDUP(newsize); + want = (int)(newsize / PGSIZE); + if (want > SHM_MAXPAGES) + return -ENOMEM; + + shm_init_once(); + acquire(&shmlock); + sg = &shm[id]; + if (!sg->used) { release(&shmlock); return -EINVAL; } + // nattach is 1 for the file itself; anything above that is a live mapping. + if (sg->nattach > 1 && want < sg->npages) { + release(&shmlock); + return -EBUSY; + } + release(&shmlock); + + // Allocate outside the lock: kalloc takes its own, and a page allocator + // called with a spinlock held is how the two orders meet. + for (i = sg->npages; i < want; i++) { + char *pg = kalloc(); + if (!pg) { + // Undo this call's own allocations. Anything from an earlier + // successful resize stays: the object is still that size. + for (int j = sg->npages; j < i; j++) { kfree(sg->pages[j]); sg->pages[j] = 0; } + return -ENOMEM; + } + memset(pg, 0, PGSIZE); + sg->pages[i] = pg; + } + for (i = want; i < sg->npages; i++) { + if (sg->pages[i]) { kfree(sg->pages[i]); sg->pages[i] = 0; } + } + + acquire(&shmlock); + sg->npages = want; + sg->size = newsize; + release(&shmlock); + return 0; +} + +// One page of a segment, by index. Used by the mmap path; the caller has +// already taken a reference, so the page cannot go away underneath it. +char * +shm_page(int id, int pg) +{ + char *p; + if (id < 0 || id >= NSHM || pg < 0) + return 0; + shm_init_once(); + acquire(&shmlock); + p = (shm[id].used && pg < shm[id].npages) ? shm[id].pages[pg] : 0; + release(&shmlock); + return p; +} + +uint64 +memfd_size(int id) +{ + uint64 n; + if (id < 0 || id >= NSHM) + return 0; + shm_init_once(); + acquire(&shmlock); + n = shm[id].used ? shm[id].size : 0; + release(&shmlock); + return n; +} + +int +memfd_truncate(int id, uint64 len) +{ + return shm_resize(id, len); +} + +// read()/write() on a memfd, so it is a file rather than a token that only +// mmap understands. Wayland clients use ftruncate and mmap, but a descriptor +// that refuses read() is a surprise nobody needs. +int +memfd_rw(int id, int user, uint64 buf, uint64 off, int n, int writing) +{ + struct shmseg *sg; + int done = 0; + + if (id < 0 || id >= NSHM || n <= 0) + return 0; + shm_init_once(); + sg = &shm[id]; + if (!sg->used) + return -EINVAL; + if (off >= sg->size) + return 0; // EOF, or a write past the end + if (off + (uint64)n > sg->size) + n = (int)(sg->size - off); + + while (done < n) { + uint64 pos = off + (uint64)done; + int pg = (int)(pos / PGSIZE); + int inpg = (int)(pos % PGSIZE); + int want = PGSIZE - inpg; + if (want > n - done) + want = n - done; + char *p = sg->pages[pg] + inpg; + + if (writing) { + if (user) { if (copy_from_user(p, buf + done, want) < 0) return -EFAULT; } + else memmove(p, (char*)(buf + done), want); + } else { + if (user) { if (copy_to_user(buf + done, p, want) < 0) return -EFAULT; } + else memmove((char*)(buf + done), p, want); + } + done += want; + } + return done; +} + +addr_t +sys_memfd_create(void) +{ + addr_t namep; + int flags, id = -1, fd; + struct file *f; + + argaddr(0, &namep); + argint(1, &flags); + // The name is for /proc and for debuggers. There is no /proc//fd + // here, so it is read for validation and dropped -- which is honest, and + // better than storing a string nothing will ever show. + if (namep) { + char *nm; + if (fetchstr(namep, &nm) < 0) + return -EFAULT; + } + + shm_init_once(); + acquire(&shmlock); + for (int i = 0; i < NSHM; i++) + if (!shm[i].used) { + memset(&shm[i], 0, sizeof(shm[i])); + shm[i].used = 1; + shm[i].key = IPC_PRIVATE; + shm[i].size = 0; + shm[i].npages = 0; + shm[i].nattach = 1; // the file holds this one + id = i; + break; + } + release(&shmlock); + if (id < 0) + return -ENFILE; + + if ((f = filealloc()) == 0) { shm_detach(id); return -ENFILE; } + if ((fd = fdalloc(f)) < 0) { fileclose(f); return -EMFILE; } + + f->type = FD_MEMFD; + f->memfd_seg = id; + f->readable = 1; + f->writable = 1; + f->off = 0; + return fd; +} + addr_t sys_shmget(void) { diff --git a/kernel/sys/table.inc b/kernel/sys/table.inc index 6210d02..6ecb48b 100644 --- a/kernel/sys/table.inc +++ b/kernel/sys/table.inc @@ -144,6 +144,10 @@ SYSCALL(198, 288, accept4) // from one process to another (SCM_RIGHTS). docs/plan/foundations.md item 1. SYSCALL(199, 46, sendmsg) SYSCALL(200, 47, recvmsg) +// memfd_create: an anonymous shared memory object reached by descriptor. +// docs/plan/foundations.md item 2 -- the thing a Wayland buffer crosses on, +// and the reason the descriptor passing above had to exist first. +SYSCALL(201, 319, memfd_create) // --- time --- SYSCALL( 28, 37, alarm) diff --git a/lib/usys.S b/lib/usys.S index e970b85..b96b616 100644 --- a/lib/usys.S +++ b/lib/usys.S @@ -103,6 +103,7 @@ SYSCALL(clock_gettime) SYSCALL(getppid) SYSCALL(rt_sigaction) SYSCALL(rt_sigsuspend) +SYSCALL(memfd_create) SYSCALL(shmget) SYSCALL(shmat) SYSCALL(shmdt)