diff --git a/Makefile b/Makefile index bf2f490..fff6338 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ KERN_PROC = $(addprefix $(BUILD)/kernel/proc/, \ KERN_SYS = $(addprefix $(BUILD)/kernel/sys/, \ syscall.o sys_proc.o sys_fs.o sys_mem.o sys_net.o \ - sys_time.o sys_sig.o sys_compat.o sys_poll.o linux_abi.o) + sys_time.o sys_sig.o sys_compat.o sys_poll.o sys_event.o linux_abi.o) # vectors.o lives alongside trapasm in the arch dir KERN_ARCH += $(BUILD)/kernel/arch/x86_64/vectors.o @@ -128,7 +128,7 @@ KERN_NOENTRY = $(filter-out $(BUILD)/kernel/arch/x86_64/entry.o, $(KERNOBJS)) # Third-party software does NOT go here — see ports/. # ----------------------------------------------------------------------- ULIB = $(addprefix $(BUILD)/lib/, start.o glue.o usys.o) -CMDS = schinit schinitd fbinfo shmtest socktest inputtest nettest reflink schattr ping chvt loadkeys sigtest memfdtest pcitest vgpud schdbg schdbgd resolvconf httpd schmode netecho ldlsd tone hwreport dynhello +CMDS = schinit schinitd fbinfo shmtest socktest inputtest nettest reflink schattr ping chvt loadkeys sigtest memfdtest pcitest vgpud schdbg schdbgd resolvconf httpd schmode netecho ldlsd tone hwreport dynhello evtest CMDBINS = $(addprefix $(BUILD)/cmd/, $(CMDS)) # ----------------------------------------------------------------------- @@ -1070,6 +1070,16 @@ $(BUILD)/cmd/dynhello: cmd/dynhello/main.c $(BUILD)/cmd/libdynhello.so $(MUSLPREFIX)/lib/crt1.o $(MUSLPREFIX)/lib/crti.o $(BUILD)/cmd/dynhello.o \ -L$(BUILD)/cmd -ldynhello -L$(MUSLPREFIX)/lib -lc $(PORTEND) +# evtest: eventfd, timerfd, signalfd and epoll through musl's wrappers. A +# musl program for the same reason as dynhello, and dynamic because it may as +# well be. +$(BUILD)/cmd/evtest: cmd/evtest/main.c $(MUSLPREFIX)/lib/libc.so + @mkdir -p $(BUILD)/cmd + $(DYNCC) -fno-pic -c $< -o $(BUILD)/cmd/evtest.o + $(DYNCC) -no-pie -nostdlib $(DYNLD) -o $@ \ + $(MUSLPREFIX)/lib/crt1.o $(MUSLPREFIX)/lib/crti.o $(BUILD)/cmd/evtest.o \ + -L$(MUSLPREFIX)/lib -lc $(PORTEND) + $(SYSROOT)/lib/libdynhello.so: $(BUILD)/cmd/libdynhello.so @mkdir -p $(SYSROOT)/lib cp $< $@ diff --git a/cmd/evtest/main.c b/cmd/evtest/main.c new file mode 100644 index 0000000..687ceaa --- /dev/null +++ b/cmd/evtest/main.c @@ -0,0 +1,128 @@ +// evtest: eventfd, timerfd, signalfd and epoll, each checked for the one +// property an event loop depends on, and then all four in one epoll set -- +// which is how libwayland's event loop uses them. +// +// A musl program, linked dynamically, so it goes through libc's wrappers +// exactly as a ported program would. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int bad; + +#define CHECK(cond, ...) do { \ + if (!(cond)) { printf("evtest: FAIL " __VA_ARGS__); printf("\n"); bad++; } \ + } while (0) + +static long +ms_since(struct timespec *t0) +{ + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return (t.tv_sec - t0->tv_sec) * 1000 + (t.tv_nsec - t0->tv_nsec) / 1000000; +} + +int +main(void) +{ + uint64_t v; + struct timespec t0; + + // eventfd: writes add, a read takes the sum and empties it, and an empty + // nonblocking one says EAGAIN rather than blocking. + int efd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + CHECK(efd >= 0, "eventfd: %d", errno); + v = 3; write(efd, &v, 8); + v = 4; write(efd, &v, 8); + v = 0; + CHECK(read(efd, &v, 8) == 8 && v == 7, "eventfd sum %llu", (unsigned long long)v); + CHECK(read(efd, &v, 8) < 0 && errno == EAGAIN, "eventfd empty: errno %d", errno); + + int sem = eventfd(2, EFD_SEMAPHORE); + CHECK(read(sem, &v, 8) == 8 && v == 1, "semaphore eventfd gave %llu", (unsigned long long)v); + + // timerfd: a one-shot 50 ms timer is not ready at once, is after it, and a + // blocking read waits for it rather than returning. + int tfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC); + CHECK(tfd >= 0, "timerfd_create: %d", errno); + struct itimerspec its = { { 0, 0 }, { 0, 50 * 1000000 } }; + clock_gettime(CLOCK_MONOTONIC, &t0); + CHECK(timerfd_settime(tfd, 0, &its, 0) == 0, "timerfd_settime: %d", errno); + v = 0; + CHECK(read(tfd, &v, 8) == 8 && v == 1, "timerfd expirations %llu", (unsigned long long)v); + long waited = ms_since(&t0); + CHECK(waited >= 40 && waited < 1000, "timerfd fired after %ld ms", waited); + + // signalfd: a blocked SIGUSR1 sent to ourselves is read back as data. + sigset_t m; + sigemptyset(&m); + sigaddset(&m, SIGUSR1); + sigprocmask(SIG_BLOCK, &m, 0); + int sfd = signalfd(-1, &m, SFD_NONBLOCK); + CHECK(sfd >= 0, "signalfd: %d", errno); + kill(getpid(), SIGUSR1); + struct signalfd_siginfo si; + memset(&si, 0, sizeof(si)); + CHECK(read(sfd, &si, sizeof(si)) == sizeof(si) && si.ssi_signo == SIGUSR1, + "signalfd read signo %u", si.ssi_signo); + + // epoll: all of it in one set. Nothing ready -> times out; then the + // timer, the eventfd and the signal each arrive as their own event, + // carrying the data they were registered with. + int ep = epoll_create1(EPOLL_CLOEXEC); + CHECK(ep >= 0, "epoll_create1: %d", errno); + struct epoll_event ev; + ev.events = EPOLLIN; ev.data.u64 = 1; epoll_ctl(ep, EPOLL_CTL_ADD, efd, &ev); + ev.events = EPOLLIN; ev.data.u64 = 2; epoll_ctl(ep, EPOLL_CTL_ADD, tfd, &ev); + ev.events = EPOLLIN; ev.data.u64 = 3; epoll_ctl(ep, EPOLL_CTL_ADD, sfd, &ev); + CHECK(epoll_ctl(ep, EPOLL_CTL_ADD, efd, &ev) < 0 && errno == EEXIST, "double add"); + + struct epoll_event out[8]; + clock_gettime(CLOCK_MONOTONIC, &t0); + CHECK(epoll_wait(ep, out, 8, 30) == 0, "an empty set was ready"); + CHECK(ms_since(&t0) >= 20, "epoll_wait timed out after %ld ms", ms_since(&t0)); + + its.it_value.tv_nsec = 20 * 1000000; + timerfd_settime(tfd, 0, &its, 0); + int n = epoll_wait(ep, out, 8, 1000); + CHECK(n == 1 && out[0].data.u64 == 2, "timer through epoll: n=%d data=%llu", + n, n > 0 ? (unsigned long long)out[0].data.u64 : 0ULL); + read(tfd, &v, 8); + + v = 1; write(efd, &v, 8); + kill(getpid(), SIGUSR1); + n = epoll_wait(ep, out, 8, 1000); + int seen = 0; + for (int i = 0; i < n; i++) + seen |= 1 << out[i].data.u64; + CHECK(n == 2 && seen == ((1 << 1) | (1 << 3)), "eventfd+signal: n=%d mask=%x", n, seen); + + // Level-triggered: still ready until drained. Then edge-triggered: once. + n = epoll_wait(ep, out, 8, 0); + CHECK(n == 2, "level-triggered repeat gave %d", n); + read(efd, &v, 8); + read(sfd, &si, sizeof(si)); + + ev.events = EPOLLIN | EPOLLET; ev.data.u64 = 1; + epoll_ctl(ep, EPOLL_CTL_MOD, efd, &ev); + v = 1; write(efd, &v, 8); + CHECK(epoll_wait(ep, out, 8, 0) == 1, "edge-triggered first report"); + CHECK(epoll_wait(ep, out, 8, 0) == 0, "edge-triggered reported twice"); + + // A closed descriptor leaves the set. + close(efd); + CHECK(epoll_wait(ep, out, 8, 0) == 0, "closed fd still in the set"); + CHECK(epoll_ctl(ep, EPOLL_CTL_DEL, tfd, 0) == 0, "EPOLL_CTL_DEL"); + + printf("evtest: %s\n", bad ? "FAILED" : "ok"); + return bad != 0; +} diff --git a/docs/architecture/syscalls.md b/docs/architecture/syscalls.md index e6aa044..13fa504 100644 --- a/docs/architecture/syscalls.md +++ b/docs/architecture/syscalls.md @@ -82,11 +82,11 @@ Roughly 190 native numbers covering: | Identity | the full real/effective/saved set for both uid and gid, groups, umask | | Memory | mmap, munmap, mprotect, brk/sbrk, madvise, System V shared memory | | Filesystem | the ordinary set, the whole `*at()` family, statfs, xattr, mount/umount2 | -| I/O | read/write, readv/writev, pread/pwrite, sendfile, poll/select/ppoll/pselect6, epoll, ioctl, fcntl | +| I/O | read/write, readv/writev, pread/pwrite, sendfile, poll/select/ppoll/pselect6, epoll (create, create1, ctl, wait, pwait), ioctl, fcntl | | Sockets | socket, bind, listen, accept, connect, sendto, recvfrom, socketpair, setsockopt, getsockopt | | Signals | rt_sigaction, rt_sigreturn, rt_sigsuspend, rt_sigpending, rt_sigtimedwait, sigaltstack | | Time | gettimeofday, clock_gettime, clock_getres, nanosleep, clock_nanosleep, alarm, timers | -| Notification | eventfd2, timerfd_create/settime/gettime, signalfd4 | +| Notification | eventfd, eventfd2, timerfd_create/settime/gettime, signalfd, signalfd4 -- see `kernel/sys/sys_event.c` | ### Argument fetching diff --git a/docs/plan/1.0.md b/docs/plan/1.0.md index e7a3fce..0936c68 100644 --- a/docs/plan/1.0.md +++ b/docs/plan/1.0.md @@ -512,7 +512,7 @@ attention. Nothing here is cancelled. | | Why it waits | |---|---| | **The rest of the hybrid kernel** ([hybrid.md](hybrid.md)) | Still not a completeness requirement — but the mechanisms and the first eviction happened early, because the alternative to a driver compatibility layer is writing a driver for every device that exists, and that is not a plan. What is deferred is the foreign driver host, not the interface it needs. | -| **Dynamic linking, Mesa, SDL3** | Everything at 1.0 is statically linked. This is the single biggest post-1.0 unlock and it is a whole project. | +| **Mesa, SDL3** | The kernel and musl do dynamic linking now (`cmd/dynhello`), but every shipped program is still static. Mesa is the single biggest post-1.0 unlock and it is a whole project. | | **A page cache** | `memfd` covers the desktop's needs. General writable `MAP_SHARED` of a file is a real project with no 1.0 consumer. | | **`.sip` and the installer** | 1.0 installs by writing a USB stick. That is how most systems start and it is honest. | | **SchFS snapshots, FUSE, Linux module, sparse files, online fsck** | The format version (Gate 0) is the prerequisite for all of them, and it is the only part on a clock. | diff --git a/docs/plan/foundations.md b/docs/plan/foundations.md index 02d3157..1e0252d 100644 --- a/docs/plan/foundations.md +++ b/docs/plan/foundations.md @@ -37,10 +37,10 @@ same lie in a new place. --- -## The five items +## The six items In dependency order. The first three are what Wayland cannot start without. -Item 1 is built; items 2 and 3 are what a buffer still cannot cross on. +All but item 4 are built. ### 1. `sendmsg` / `recvmsg` with `SCM_RIGHTS` — **done** @@ -128,13 +128,41 @@ strength of the lie. Small, and grouped here because it is in the same file as item 3 and will be found by the same testing. -### 5. Dynamic linking +### 5. Dynamic linking — **done** -`PT_INTERP` and `ET_DYN` in `kernel/fs/exec.c`. +`PT_INTERP` and `ET_DYN` in `kernel/fs/exec.c`. A program that names an +interpreter gets it loaded at the bottom of the mmap area and started instead, +with `AT_PHDR`, `AT_PHENT`, `AT_PHNUM`, `AT_BASE` and `AT_ENTRY` to find the +program by; a PIE program is placed where a fixed one would be. The interpreter +is musl's own: `ports/musl` now builds `libc.so` as well, shipped as +`/lib/ld-musl-x86_64.so.1`, which is libc and loader in one file. -**Deferrable, and deferred.** Static binaries are enough to reach a working -desktop with software rendering. This becomes necessary when Mesa does — EGL -means `dlopen` means a loader — and that is a separate decision made later. +The kernel half that was missing besides `exec` was `mmap(MAP_FIXED)` of a +file. It handed back zeroed pages whatever the descriptor said, and kept no +record of them for `fork` — and a dynamic linker maps every library segment +after the first exactly that way. + +Every port still links `-static`, on purpose: this makes the other kind +possible, it does not convert anything. `cmd/dynhello` is the test — a +program against `libc.so` and a library of its own, checking TLS, the heap and +a forked child that still has its libraries. + +`mprotect` is accepted and not enforced, so RELRO is not read-only yet. + +### 6. The event descriptors — **done** + +`epoll`, `eventfd`, `timerfd` and `signalfd`, in `kernel/sys/sys_event.c`. +libwayland's event loop is all four in one epoll set, which is why they arrive +together; not in the original list because they were stubs that looked like +syscalls, returning `ENOSYS` from `table.inc` entries that read as done. + +They wait the way `poll()` does, a tick at a time, so the latency is 10 ms. +`cmd/evtest` checks each one, then all four in one set, level- and +edge-triggered. + +Building it found a signal bug: a default-fatal signal killed on arrival even +when it was blocked. A blocked signal now stays pending, which is the contract +`signalfd` is built on. --- diff --git a/docs/plan/wayland.md b/docs/plan/wayland.md index fb10d62..1008c2b 100644 --- a/docs/plan/wayland.md +++ b/docs/plan/wayland.md @@ -3,9 +3,10 @@ Schcom survives and changes its wire format. SchWM and SchDE are deleted. What follows is why that is not a retreat, and what it costs. -Blocked on [foundations.md](foundations.md), items 1–4. **Item 1 is built** — -descriptor passing, without which no Wayland client can hand over a buffer. Item -2, a shareable memory object, is the next thing it stops at. +Blocked on [foundations.md](foundations.md). Everything there is built except +item 4, `setsockopt` that does not lie: descriptor passing, shareable memory, +sockets in the filesystem, dynamic linking, and the epoll/eventfd/timerfd/ +signalfd set libwayland's event loop is made of. --- @@ -106,9 +107,10 @@ through `display_ops`, which already exists. ### Software rendering first, and probably for a long time `pixman` for composition, `wl_shm` for buffers. Mesa means EGL means `dlopen` -means a dynamic loader, and that is a large kernel project that does not have to -be on this path. Static binaries are enough to reach a working desktop; keep it -that way until something concrete demands otherwise. +means a dynamic loader -- which now exists (see [foundations.md](foundations.md) +item 5), so it is no longer what stands in the way of Mesa; a GPU driver is. +Static binaries are still enough to reach a working desktop, and the desktop's +own programs can stay static until something concrete demands otherwise. ### One sharp edge @@ -160,8 +162,8 @@ something proves nothing. ## What it will still not do -- **No hardware acceleration.** Software rendering only, until dynamic linking - and Mesa, neither of which is on this plan. +- **No hardware acceleration.** Software rendering only, until Mesa, which is + not on this plan. Dynamic linking, which it also needs, is in. - **No XWayland**, so no X11 clients, ever, unless someone ports it. - **No mode setting.** Resolution stays whatever the loader asked for — see [hybrid.md](hybrid.md), where the GPU driver's eviction is also where a modeset diff --git a/include/core/defs.h b/include/core/defs.h index 58aa1bc..237bba2 100644 --- a/include/core/defs.h +++ b/include/core/defs.h @@ -108,6 +108,13 @@ int exec(char*, char**, char**); // file.c struct file* filealloc(void); int fdalloc(struct file*); + +// sys_event.c: eventfd, timerfd, signalfd and epoll, as files +struct evobj; +void ev_close(struct evobj *); +int ev_poll(struct file *, int events); +int ev_read(struct file *, char *addr, int n); +int ev_write(struct file *, char *addr, int n); void fileclose(struct file*); struct file* filedup(struct file*); void fileinit(void); diff --git a/include/fs/file.h b/include/fs/file.h index edb03e6..6da4970 100644 --- a/include/fs/file.h +++ b/include/fs/file.h @@ -3,7 +3,10 @@ struct file { // returns. It is a descriptor with no name and no inode behind it, which is // the entire reason memfd exists -- two processes share memory by passing // the descriptor, not by agreeing on a path or a System V key. - enum { FD_NONE, FD_PIPE, FD_INODE, FD_SOCKET, FD_MEMFD } type; + // + // FD_EVENT: an eventfd, timerfd, signalfd or epoll set -- see + // kernel/sys/sys_event.c, which is the only file that looks inside `ev`. + enum { FD_NONE, FD_PIPE, FD_INODE, FD_SOCKET, FD_MEMFD, FD_EVENT } type; int ref; // reference count char readable; char writable; @@ -20,6 +23,7 @@ struct file { // the reference counting lives and handing out pointers into it would // invite somebody to keep one. int memfd_seg; + struct evobj *ev; uint64 off; struct file *fnext; // file.c's free list, while ref is 0 }; diff --git a/kernel/fs/file.c b/kernel/fs/file.c index 8b05164..50ab253 100644 --- a/kernel/fs/file.c +++ b/kernel/fs/file.c @@ -106,6 +106,8 @@ fileclose(struct file *f) if(ff.type == FD_MEMFD) shm_detach(ff.memfd_seg); // the last mapper frees the pages + else if(ff.type == FD_EVENT) + ev_close(ff.ev); else if(ff.type == FD_PIPE) pipeclose(ff.pipe, ff.writable); else if(ff.type == FD_SOCKET) @@ -144,6 +146,14 @@ filestat(struct file *f, struct stat *st) iunlock(f->ip); return 0; } + // An eventfd, timerfd, signalfd or epoll set: Linux calls these anonymous + // inodes, and fstat succeeds on them with nothing much to say. A program + // that fstat()s every descriptor it is handed should not fail on one. + if(f->type == FD_EVENT){ + memset(st, 0, sizeof(*st)); + st->nlink = 1; + return 0; + } return -1; } @@ -155,6 +165,9 @@ fileread(struct file *f, char *addr, int n) if(f->readable == 0) return -1; + if(f->type == FD_EVENT) + return ev_read(f, addr, n); + // A memfd is a file: read() and write() move bytes in and out of the shared // pages, and the descriptor carries its own offset like any other. mmap is // what it is usually for, but a descriptor that refuses read() is a surprise @@ -248,6 +261,9 @@ filewrite(struct file *f, char *addr, int n) if(f->writable == 0) return -1; + if(f->type == FD_EVENT) + return ev_write(f, addr, n); + if(f->type == FD_MEMFD){ r = memfd_rw(f->memfd_seg, 1, (uint64)addr, f->off, n, 1); if(r > 0) @@ -326,6 +342,9 @@ filewrite(struct file *f, char *addr, int n) int file_poll(struct file *f, int events) { + if(f->type == FD_EVENT) + return ev_poll(f, events); + if(f->type == FD_PIPE) return pipe_poll(f->pipe, f->readable, f->writable) & (events | POLLERR | POLLHUP); diff --git a/kernel/sys/sys_compat.c b/kernel/sys/sys_compat.c index 555c101..6e67b34 100644 --- a/kernel/sys/sys_compat.c +++ b/kernel/sys/sys_compat.c @@ -226,14 +226,4 @@ addr_t sys_getrandom(void) // pselect6 / ppoll live in sys_poll.c alongside select/poll. -// epoll: not implemented. -addr_t sys_epoll_create1(void) { return -ENOSYS; } -addr_t sys_epoll_ctl(void) { return -ENOSYS; } -addr_t sys_epoll_wait(void) { return -ENOSYS; } - -// signalfd / eventfd / timerfd: not implemented. -addr_t sys_signalfd4(void) { return -ENOSYS; } -addr_t sys_eventfd2(void) { return -ENOSYS; } -addr_t sys_timerfd_create(void) { return -ENOSYS; } -addr_t sys_timerfd_settime(void){ return -ENOSYS; } -addr_t sys_timerfd_gettime(void){ return -ENOSYS; } +// epoll, eventfd, timerfd and signalfd live in sys_event.c. diff --git a/kernel/sys/sys_event.c b/kernel/sys/sys_event.c new file mode 100644 index 0000000..6846177 --- /dev/null +++ b/kernel/sys/sys_event.c @@ -0,0 +1,745 @@ +// eventfd, timerfd, signalfd and epoll: the descriptors an event loop is +// built from. +// +// Four kinds of file with no inode behind them, all FD_EVENT, told apart by +// struct evobj's kind. Each one is a counter or a set that read() drains and +// poll() reports on -- which is the whole design, and the reason they arrive +// together: an event loop like libwayland's puts all of them in one epoll set +// and sleeps on that, so a timer firing, a signal arriving and another thread +// poking an eventfd all become "this descriptor is readable". +// +// Waiting is the same as poll()'s: check, and if nothing is ready sleep one +// timer tick and check again. Not a wakeup per event -- nothing here has a +// wait queue -- so the latency is a tick, 10 ms. That matches every other +// descriptor poll() already sees, and the day poll() gets wait queues these +// get them in the same change. + +#include "core/types.h" +#include "core/defs.h" +#include "core/param.h" +#include "core/spinlock.h" +#include "core/sleeplock.h" +#include "core/kmem.h" +#include "arch/mmu.h" +#include "fs/fs.h" +#include "fs/file.h" +#include "core/proc.h" +#include "core/syscall.h" +#include "core/uaccess.h" +#include "uapi/posix.h" +#include "uapi/fcntl.h" +#include "uapi/stat.h" +#include "uapi/errno.h" + +enum { EV_EVENTFD = 1, EV_TIMERFD, EV_SIGNALFD, EV_EPOLL }; + +// Linux's flag values, which are O_NONBLOCK and O_CLOEXEC for every one of +// these: one spelling per meaning, so they are tested as such. +#define EFD_SEMAPHORE 1 +#define TFD_TIMER_ABSTIME 1 + +#define EPOLL_CTL_ADD 1 +#define EPOLL_CTL_DEL 2 +#define EPOLL_CTL_MOD 3 +#define EPOLLIN 0x001 +#define EPOLLOUT 0x004 +#define EPOLLERR 0x008 +#define EPOLLHUP 0x010 +#define EPOLLRDHUP 0x2000 +#define EPOLLEXCLUSIVE (1u << 28) +#define EPOLLWAKEUP (1u << 29) +#define EPOLLONESHOT (1u << 30) +#define EPOLLET (1u << 31) + +// What a set can hold: one page of entries, about a hundred. libwayland's +// loop has a handful; a compositor with every client is a few dozen. A page +// from kalloc() rather than kmalloc(), whose objects stop short of a page. +#define EP_MAX ((int)(PGSIZE / sizeof(struct epitem))) + +// x86-64 packs it: 12 bytes, the data unaligned. Getting this wrong puts every +// event's data four bytes off, which reads as a valid but wrong pointer. +struct epoll_event { + uint32 events; + uint64 data; +} __attribute__((packed)); + +// struct signalfd_siginfo is 128 bytes and the program reads it whole; only +// the signal number has a meaning here yet. +struct sfd_info { + uint32 ssi_signo; + int32 ssi_errno; + int32 ssi_code; + uint32 ssi_pid; + uint32 ssi_uid; + uint8 rest[108]; +}; + +struct epitem { + int fd; + struct file *f; // checked against the table, never dereferenced alone + uint32 events; + uint64 data; + int was_ready; // for EPOLLET: report on becoming ready, not on being + int disabled; // EPOLLONESHOT fired, until EPOLL_CTL_MOD +}; + +struct evobj { + int kind; + // A sleep lock, because an epoll set is held across file_poll() of + // everything in it, and some of those -- a socket -- may sleep. + struct sleeplock lock; + union { + struct { uint64 count; int sema; } efd; + struct { + int clock; + uint64 next_ns; // 0 when disarmed + uint64 interval_ns; + uint64 expired; // expirations not yet read + } tfd; + struct { uint64 mask; } sfd; // kernel bit layout, as signal_mask + struct { int n; struct epitem *items; } ep; + }; +}; + +static uint64 +clock_ns(int clock) +{ + uint64 sec, usec; + // The same clock clock_gettime() reads, so an absolute time a program + // computed from it means the same instant here. + wallclock(&sec, &usec); + if (clock != CLOCK_REALTIME) + sec -= boot_epoch; + return sec * 1000000000ULL + usec * 1000ULL; +} + +// One tick, or until a signal or kill makes waiting pointless. +static int +ev_block(void) +{ + struct proc *p = myproc(); + if (p->killed || (p->pending_signals & ~p->signal_mask)) + return -EINTR; + acquire(&tickslock); + uint t0 = ticks; + while (ticks == t0 && !p->killed && !(p->pending_signals & ~p->signal_mask)) + sleep(&ticks, &tickslock); + release(&tickslock); + return 0; +} + +static int +ev_open(struct evobj *o, int flags, int readable, int writable) +{ + struct file *f; + int fd; + + initsleeplock(&o->lock, "evobj"); + if ((f = filealloc()) == 0) { + kmfree(o); + return -ENFILE; + } + f->type = FD_EVENT; + f->ev = o; + f->readable = readable; + f->writable = writable; + f->nonblock = (flags & O_NONBLOCK) != 0; + if ((fd = fdalloc(f)) < 0) { + fileclose(f); + return -EMFILE; + } + if (flags & O_CLOEXEC) + myproc()->ofile_flags[fd] |= FD_CLOEXEC; + return fd; +} + +static struct evobj * +ev_new(int kind) +{ + struct evobj *o = kmalloc(sizeof(*o)); + if (o) { + memset(o, 0, sizeof(*o)); + o->kind = kind; + } + return o; +} + +static struct evobj * +ev_arg(int n, int kind, struct file **pf) +{ + int fd; + struct file *f; + if (argint(n, &fd) < 0 || fd < 0 || fd >= NOFILE || !(f = myproc()->ofile[fd])) + return 0; + if (f->type != FD_EVENT || f->ev->kind != kind) + return 0; + if (pf) + *pf = f; + return f->ev; +} + +// Called by fileclose() for the last reference. +void +ev_close(struct evobj *o) +{ + if (o->kind == EV_EPOLL && o->ep.items) + kfree((char *)o->ep.items); + kmfree(o); +} + +// --- timerfd's arithmetic ---------------------------------------------------- + +// Fold expirations up to now into the count. Lazily, on every look: there is +// no timer interrupt behind a timerfd, only the clock and this. +static void +tfd_catch_up(struct evobj *o) +{ + if (!o->tfd.next_ns) + return; + uint64 now = clock_ns(o->tfd.clock); + if (now < o->tfd.next_ns) + return; + if (o->tfd.interval_ns) { + uint64 n = 1 + (now - o->tfd.next_ns) / o->tfd.interval_ns; + o->tfd.expired += n; + o->tfd.next_ns += n * o->tfd.interval_ns; + } else { + o->tfd.expired++; + o->tfd.next_ns = 0; + } +} + +// --- readiness ----------------------------------------------------------------- + +static int ep_ready(struct evobj *o, struct epoll_event *out, int max); + +int +ev_poll(struct file *f, int events) +{ + struct evobj *o = f->ev; + int rev = 0; + + acquiresleep(&o->lock); + switch (o->kind) { + case EV_EVENTFD: + if (o->efd.count > 0) + rev |= POLLIN; + if (o->efd.count < 0xfffffffffffffffeULL) + rev |= POLLOUT; + break; + case EV_TIMERFD: + tfd_catch_up(o); + if (o->tfd.expired) + rev |= POLLIN; + break; + case EV_SIGNALFD: + if (myproc()->pending_signals & o->sfd.mask) + rev |= POLLIN; + break; + } + releasesleep(&o->lock); + + // An epoll set is readable when something in it is: which is how sets + // nest, and how a program polls an epoll descriptor along with others. + if (o->kind == EV_EPOLL && ep_ready(o, 0, 1) > 0) + rev |= POLLIN; + return rev & (events | POLLERR | POLLHUP); +} + +// --- read and write ------------------------------------------------------------ + +int +ev_read(struct file *f, char *addr, int n) +{ + struct evobj *o = f->ev; + + for (;;) { + acquiresleep(&o->lock); + switch (o->kind) { + case EV_EVENTFD: + if (n < 8) { releasesleep(&o->lock); return -EINVAL; } + if (o->efd.count) { + uint64 v = o->efd.sema ? 1 : o->efd.count; + o->efd.count -= v; + releasesleep(&o->lock); + return copy_to_user((addr_t)addr, &v, 8) < 0 ? -EFAULT : 8; + } + break; + case EV_TIMERFD: + if (n < 8) { releasesleep(&o->lock); return -EINVAL; } + tfd_catch_up(o); + if (o->tfd.expired) { + uint64 v = o->tfd.expired; + o->tfd.expired = 0; + releasesleep(&o->lock); + return copy_to_user((addr_t)addr, &v, 8) < 0 ? -EFAULT : 8; + } + break; + case EV_SIGNALFD: { + if (n < (int)sizeof(struct sfd_info)) { releasesleep(&o->lock); return -EINVAL; } + // The reader's own pending signals, as Linux does it: a signalfd shared + // across fork reads whichever process is reading. + struct proc *p = myproc(); + uint64 have = p->pending_signals & o->sfd.mask; + if (have) { + int sig = 0; + while (!(have & (1ULL << sig))) + sig++; + p->pending_signals &= ~(1ULL << sig); + releasesleep(&o->lock); + struct sfd_info si; + memset(&si, 0, sizeof(si)); + si.ssi_signo = sig; + return copy_to_user((addr_t)addr, &si, sizeof(si)) < 0 ? -EFAULT + : (int)sizeof(si); + } + break; + } + default: + releasesleep(&o->lock); + return -EINVAL; + } + releasesleep(&o->lock); + if (f->nonblock) + return -EAGAIN; + if (ev_block() < 0) + return -EINTR; + } +} + +int +ev_write(struct file *f, char *addr, int n) +{ + struct evobj *o = f->ev; + uint64 v; + + if (o->kind != EV_EVENTFD) + return -EINVAL; + if (n < 8) + return -EINVAL; + if (copy_from_user(&v, (addr_t)addr, 8) < 0) + return -EFAULT; + if (v == 0xffffffffffffffffULL) + return -EINVAL; + for (;;) { + acquiresleep(&o->lock); + // The counter stops one short of all ones, and a write that would pass + // that waits for a read to make room. + if (o->efd.count + v >= o->efd.count && + o->efd.count + v < 0xffffffffffffffffULL) { + o->efd.count += v; + releasesleep(&o->lock); + return 8; + } + releasesleep(&o->lock); + if (f->nonblock) + return -EAGAIN; + if (ev_block() < 0) + return -EINTR; + } +} + +// --- eventfd ------------------------------------------------------------------- + +static addr_t +eventfd_common(uint64 init, int flags) +{ + if (flags & ~(EFD_SEMAPHORE | O_NONBLOCK | O_CLOEXEC)) + return -EINVAL; + struct evobj *o = ev_new(EV_EVENTFD); + if (!o) + return -ENOMEM; + o->efd.count = init; + o->efd.sema = (flags & EFD_SEMAPHORE) != 0; + return ev_open(o, flags, 1, 1); +} + +addr_t +sys_eventfd2(void) +{ + int init, flags; + if (argint(0, &init) < 0 || argint(1, &flags) < 0) + return -EINVAL; + return eventfd_common((uint32)init, flags); +} + +// The original, with no flags argument -- the register that would hold one +// holds whatever the caller left there. +addr_t +sys_eventfd(void) +{ + int init; + if (argint(0, &init) < 0) + return -EINVAL; + return eventfd_common((uint32)init, 0); +} + +// --- timerfd ------------------------------------------------------------------- + +struct itimerspec_u { + int64 it_interval_sec, it_interval_nsec; + int64 it_value_sec, it_value_nsec; +}; + +addr_t +sys_timerfd_create(void) +{ + int clock, flags; + if (argint(0, &clock) < 0 || argint(1, &flags) < 0) + return -EINVAL; + // REALTIME, MONOTONIC and BOOTTIME (7); the last is MONOTONIC here, since + // nothing on SchOS suspends. + if (clock != CLOCK_REALTIME && clock != CLOCK_MONOTONIC && clock != 7) + return -EINVAL; + if (flags & ~(O_NONBLOCK | O_CLOEXEC)) + return -EINVAL; + struct evobj *o = ev_new(EV_TIMERFD); + if (!o) + return -ENOMEM; + o->tfd.clock = clock; + return ev_open(o, flags, 1, 0); +} + +static void +tfd_current(struct evobj *o, struct itimerspec_u *cur) +{ + memset(cur, 0, sizeof(*cur)); + cur->it_interval_sec = (int64)(o->tfd.interval_ns / 1000000000ULL); + cur->it_interval_nsec = (int64)(o->tfd.interval_ns % 1000000000ULL); + if (o->tfd.next_ns) { + uint64 now = clock_ns(o->tfd.clock); + uint64 left = o->tfd.next_ns > now ? o->tfd.next_ns - now : 1; + cur->it_value_sec = (int64)(left / 1000000000ULL); + cur->it_value_nsec = (int64)(left % 1000000000ULL); + } +} + +addr_t +sys_timerfd_settime(void) +{ + int flags; + addr_t unew, uold; + struct itimerspec_u nv, ov; + struct evobj *o = ev_arg(0, EV_TIMERFD, 0); + + if (!o) + return -EBADF; + if (argint(1, &flags) < 0 || argaddr(2, &unew) < 0 || argaddr(3, &uold) < 0) + return -EINVAL; + if (copy_from_user(&nv, unew, sizeof(nv)) < 0) + return -EFAULT; + if (nv.it_value_nsec < 0 || nv.it_value_nsec >= 1000000000 || + nv.it_interval_nsec < 0 || nv.it_interval_nsec >= 1000000000 || + nv.it_value_sec < 0 || nv.it_interval_sec < 0) + return -EINVAL; + + acquiresleep(&o->lock); + tfd_current(o, &ov); + uint64 value = (uint64)nv.it_value_sec * 1000000000ULL + (uint64)nv.it_value_nsec; + o->tfd.interval_ns = (uint64)nv.it_interval_sec * 1000000000ULL + + (uint64)nv.it_interval_nsec; + o->tfd.expired = 0; + if (value == 0) { + o->tfd.next_ns = 0; // disarm + } else if (flags & TFD_TIMER_ABSTIME) { + o->tfd.next_ns = value; + } else { + o->tfd.next_ns = clock_ns(o->tfd.clock) + value; + } + releasesleep(&o->lock); + + if (uold && copy_to_user(uold, &ov, sizeof(ov)) < 0) + return -EFAULT; + return 0; +} + +addr_t +sys_timerfd_gettime(void) +{ + addr_t ucur; + struct itimerspec_u cur; + struct evobj *o = ev_arg(0, EV_TIMERFD, 0); + + if (!o) + return -EBADF; + if (argaddr(1, &ucur) < 0) + return -EINVAL; + acquiresleep(&o->lock); + tfd_catch_up(o); + tfd_current(o, &cur); + releasesleep(&o->lock); + return copy_to_user(ucur, &cur, sizeof(cur)) < 0 ? -EFAULT : 0; +} + +// --- signalfd ------------------------------------------------------------------ + +static addr_t +signalfd_common(int fd, addr_t umask, uint64 size, int flags) +{ + uint64 umaskv; + + if (size != 8) + return -EINVAL; + if (flags & ~(O_NONBLOCK | O_CLOEXEC)) + return -EINVAL; + if (copy_from_user(&umaskv, umask, 8) < 0) + return -EFAULT; + // User sigsets put signal n at bit n-1; the kernel keeps it at bit n. + uint64 mask = umaskv << 1; + + // An existing signalfd: change its mask, keep the descriptor. + if (fd != -1) { + struct file *f; + if (fd < 0 || fd >= NOFILE || !(f = myproc()->ofile[fd]) || + f->type != FD_EVENT || f->ev->kind != EV_SIGNALFD) + return -EINVAL; + acquiresleep(&f->ev->lock); + f->ev->sfd.mask = mask; + releasesleep(&f->ev->lock); + return fd; + } + struct evobj *o = ev_new(EV_SIGNALFD); + if (!o) + return -ENOMEM; + o->sfd.mask = mask; + return ev_open(o, flags, 1, 0); +} + +addr_t +sys_signalfd4(void) +{ + int fd, flags; + addr_t umask, size; + if (argint(0, &fd) < 0 || argaddr(1, &umask) < 0 || argaddr(2, &size) < 0 || + argint(3, &flags) < 0) + return -EINVAL; + return signalfd_common(fd, umask, size, flags); +} + +addr_t +sys_signalfd(void) +{ + int fd; + addr_t umask, size; + if (argint(0, &fd) < 0 || argaddr(1, &umask) < 0 || argaddr(2, &size) < 0) + return -EINVAL; + return signalfd_common(fd, umask, size, 0); +} + +// --- epoll --------------------------------------------------------------------- +// +// A set of (descriptor, file) pairs. The pair, not the descriptor alone: Linux +// registers the *file*, and an entry quietly leaves the set when that file is +// closed. Here an entry whose descriptor no longer holds the same file is +// treated as gone and dropped the next time the set is looked at -- which is +// that rule, checked lazily. No reference is held, so a registered descriptor +// closes exactly as it would otherwise. + +static addr_t +epoll_common(int flags) +{ + if (flags & ~O_CLOEXEC) + return -EINVAL; + struct evobj *o = ev_new(EV_EPOLL); + if (!o) + return -ENOMEM; + o->ep.items = (struct epitem *)kalloc(); + if (!o->ep.items) { + kmfree(o); + return -ENOMEM; + } + return ev_open(o, flags, 1, 0); +} + +addr_t +sys_epoll_create1(void) +{ + int flags; + if (argint(0, &flags) < 0) + return -EINVAL; + return epoll_common(flags); +} + +// The size argument has been ignored since Linux 2.6.8, apart from having to +// be positive. +addr_t +sys_epoll_create(void) +{ + int size; + if (argint(0, &size) < 0 || size <= 0) + return -EINVAL; + return epoll_common(0); +} + +static int +ep_find(struct evobj *o, int fd) +{ + for (int i = 0; i < o->ep.n; i++) + if (o->ep.items[i].fd == fd) + return i; + return -1; +} + +static void +ep_drop(struct evobj *o, int i) +{ + o->ep.items[i] = o->ep.items[--o->ep.n]; +} + +addr_t +sys_epoll_ctl(void) +{ + int op, fd; + addr_t uev; + struct epoll_event ev; + struct file *ef; + struct evobj *o = ev_arg(0, EV_EPOLL, &ef); + + if (!o) + return -EBADF; + if (argint(1, &op) < 0 || argint(2, &fd) < 0 || argaddr(3, &uev) < 0) + return -EINVAL; + if (fd < 0 || fd >= NOFILE || !myproc()->ofile[fd]) + return -EBADF; + struct file *tf = myproc()->ofile[fd]; + // A set cannot watch itself. (Nor contain a set that contains it; that + // check is the deep one Linux does, and nothing here builds such loops.) + if (tf == ef) + return -EINVAL; + // Regular files are refused, as on Linux: they are always ready, and a + // program that adds one has a bug that EPERM tells it about. + if (tf->type == FD_INODE && tf->ip && tf->ip->type == T_FILE) + return -EPERM; + if (op != EPOLL_CTL_DEL && copy_from_user(&ev, uev, sizeof(ev)) < 0) + return -EFAULT; + + acquiresleep(&o->lock); + int i = ep_find(o, fd); + if (i >= 0 && o->ep.items[i].f != tf) { + ep_drop(o, i); // a stale entry for a closed-and-reused fd + i = -1; + } + int r = 0; + switch (op) { + case EPOLL_CTL_ADD: + if (i >= 0) { r = -EEXIST; break; } + if (o->ep.n == EP_MAX) { r = -ENOSPC; break; } + o->ep.items[o->ep.n++] = (struct epitem){ fd, tf, ev.events, ev.data, 0, 0 }; + break; + case EPOLL_CTL_MOD: + if (i < 0) { r = -ENOENT; break; } + o->ep.items[i].events = ev.events; + o->ep.items[i].data = ev.data; + o->ep.items[i].was_ready = 0; + o->ep.items[i].disabled = 0; + break; + case EPOLL_CTL_DEL: + if (i < 0) { r = -ENOENT; break; } + ep_drop(o, i); + break; + default: + r = -EINVAL; + } + releasesleep(&o->lock); + return r; +} + +// Fill up to `max` events that are ready now. With `out` null, only count, +// and change nothing -- that is the question poll() on the set itself asks, +// and it must not use up an edge or a one-shot. +static int +ep_ready(struct evobj *o, struct epoll_event *out, int max) +{ + int got = 0; + + acquiresleep(&o->lock); + for (int i = 0; i < o->ep.n && got < max; ) { + struct epitem *it = &o->ep.items[i]; + struct file *now = (it->fd >= 0 && it->fd < NOFILE) ? myproc()->ofile[it->fd] : 0; + if (now != it->f) { + ep_drop(o, i); // closed: it leaves the set + continue; + } + i++; + if (it->disabled) + continue; + // file_poll() is called with the set's lock held; for an event + // descriptor in the set it takes that object's own lock, never this one + // -- a set cannot hold itself. + int want = (int)(it->events & (EPOLLIN | EPOLLOUT | EPOLLRDHUP)) | POLLERR | POLLHUP; + int rev = file_poll(it->f, want); + if (rev == POLLNVAL) + rev = POLLERR; + if (!rev) { + it->was_ready = 0; + continue; + } + if ((it->events & EPOLLET) && it->was_ready) + continue; + if (out) { + out[got].events = (uint32)rev; + out[got].data = it->data; + it->was_ready = 1; + if (it->events & EPOLLONESHOT) + it->disabled = 1; + } + got++; + } + releasesleep(&o->lock); + return got; +} + +static addr_t +epoll_wait_common(int timeout_ms) +{ + addr_t uevents; + int max; + struct file *ef; + struct evobj *o = ev_arg(0, EV_EPOLL, &ef); + + if (!o) + return -EBADF; + if (argaddr(1, &uevents) < 0 || argint(2, &max) < 0 || max <= 0) + return -EINVAL; + if (max > EP_MAX) + max = EP_MAX; + + struct epoll_event *buf = kmalloc(max * sizeof(*buf)); + if (!buf) + return -ENOMEM; + + int ticks_left = timeout_ms < 0 ? -1 : (timeout_ms + MSEC_PER_TICK - 1) / MSEC_PER_TICK; + int got; + for (;;) { + got = ep_ready(o, buf, max); + if (got > 0 || ticks_left == 0) + break; + if (ev_block() < 0) { + got = -EINTR; + break; + } + if (ticks_left > 0) + ticks_left--; + } + if (got > 0 && copy_to_user(uevents, buf, got * sizeof(*buf)) < 0) + got = -EFAULT; + kmfree(buf); + return got; +} + +addr_t +sys_epoll_wait(void) +{ + int timeout; + if (argint(3, &timeout) < 0) + return -EINVAL; + return epoll_wait_common(timeout); +} + +// epoll_pwait: the signal mask is not swapped in for the wait. A signal that +// arrives ends the wait with EINTR whatever the mask, as poll() already does. +addr_t +sys_epoll_pwait(void) +{ + return sys_epoll_wait(); +} diff --git a/kernel/sys/table.inc b/kernel/sys/table.inc index c7a1218..83b1bf0 100644 --- a/kernel/sys/table.inc +++ b/kernel/sys/table.inc @@ -280,14 +280,21 @@ SYSCALL(175, 270, pselect6) SYSCALL(176, 271, ppoll) // --- epoll / eventfd / signalfd / timerfd --- +// signalfd4 and eventfd2 are 289 and 290. 282 and 284 are the originals, +// which have no flags argument -- they were once aliased to the new ones, +// which then read their flags from whatever the caller left in a register. SYSCALL(177, 291, epoll_create1) SYSCALL(178, 233, epoll_ctl) SYSCALL(179, 232, epoll_wait) -SYSCALL(180, 282, signalfd4) -SYSCALL(181, 284, eventfd2) +SYSCALL(180, 289, signalfd4) +SYSCALL(181, 290, eventfd2) SYSCALL(182, 283, timerfd_create) SYSCALL(183, 286, timerfd_settime) SYSCALL(184, 287, timerfd_gettime) +SYSCALL(202, 213, epoll_create) +SYSCALL(203, 281, epoll_pwait) +SYSCALL(204, 282, signalfd) +SYSCALL(205, 284, eventfd) // --- mounts --- SYSCALL(185, 165, mount) @@ -297,8 +304,5 @@ SYSCALL(186, 166, umount2) // --- additional Linux numbers routed to an existing handler --- SYSCALL_ALIAS( 58, fork) SYSCALL_ALIAS( 280, utimensat) -SYSCALL_ALIAS( 281, epoll_wait) -SYSCALL_ALIAS( 289, signalfd4) -SYSCALL_ALIAS( 290, eventfd2) SYSCALL_ALIAS( 295, readv) SYSCALL_ALIAS( 296, writev)