SchOS/kernel/sys/sys_event.c

// eventfd, timerfd, signalfd, epoll and inotify: the descriptors an event
// loop is built from.
//
// Four kinds of file with no inode behind them, all FD_EVENT, told apart by
// struct evobj's kind. Each one is a counter or a set that read() drains and
// poll() reports on -- which is the whole design, and the reason they arrive
// together: an event loop like libwayland's puts all of them in one epoll set
// and sleeps on that, so a timer firing, a signal arriving and another thread
// poking an eventfd all become "this descriptor is readable".
//
// Waiting is the same as poll()'s: check, and if nothing is ready sleep one
// timer tick and check again. Not a wakeup per event -- nothing here has a
// wait queue -- so the latency is a tick, 10 ms. That matches every other
// descriptor poll() already sees, and the day poll() gets wait queues these
// get them in the same change.

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "core/kmem.h"
#include "arch/mmu.h"
#include "fs/fs.h"
#include "fs/file.h"
#include "core/proc.h"
#include "core/syscall.h"
#include "core/uaccess.h"
#include "uapi/posix.h"
#include "uapi/fcntl.h"
#include "uapi/stat.h"
#include "uapi/errno.h"
#include "uapi/inotify.h"

enum { EV_EVENTFD = 1, EV_TIMERFD, EV_SIGNALFD, EV_EPOLL, EV_INOTIFY };

struct inotify;

// Linux's flag values, which are O_NONBLOCK and O_CLOEXEC for every one of
// these: one spelling per meaning, so they are tested as such.
#define EFD_SEMAPHORE   1
#define TFD_TIMER_ABSTIME 1

#define EPOLL_CTL_ADD   1
#define EPOLL_CTL_DEL   2
#define EPOLL_CTL_MOD   3
#define EPOLLIN         0x001
#define EPOLLOUT        0x004
#define EPOLLERR        0x008
#define EPOLLHUP        0x010
#define EPOLLRDHUP      0x2000
#define EPOLLEXCLUSIVE  (1u << 28)
#define EPOLLWAKEUP     (1u << 29)
#define EPOLLONESHOT    (1u << 30)
#define EPOLLET         (1u << 31)

// What a set can hold: one page of entries, about a hundred. libwayland's
// loop has a handful; a compositor with every client is a few dozen. A page
// from kalloc() rather than kmalloc(), whose objects stop short of a page.
#define EP_MAX ((int)(PGSIZE / sizeof(struct epitem)))

// x86-64 packs it: 12 bytes, the data unaligned. Getting this wrong puts every
// event's data four bytes off, which reads as a valid but wrong pointer.
struct epoll_event {
  uint32 events;
  uint64 data;
} __attribute__((packed));

// struct signalfd_siginfo is 128 bytes and the program reads it whole; only
// the signal number has a meaning here yet.
struct sfd_info {
  uint32 ssi_signo;
  int32  ssi_errno;
  int32  ssi_code;
  uint32 ssi_pid;
  uint32 ssi_uid;
  uint8  rest[108];
};

struct epitem {
  int          fd;
  struct file *f;      // checked against the table, never dereferenced alone
  uint32       events;
  uint64       data;
  uint         seq;         // EPOLLET: poll_generation() when last reported
  int          was_ready;   // for EPOLLET: report on becoming ready, not on being
  int          disabled;    // EPOLLONESHOT fired, until EPOLL_CTL_MOD
};

struct evobj {
  int kind;
  // A sleep lock, because an epoll set is held across file_poll() of
  // everything in it, and some of those -- a socket -- may sleep.
  struct sleeplock lock;
  union {
    struct { uint64 count; int sema; } efd;
    struct {
      int    clock;
      uint64 next_ns;     // 0 when disarmed
      uint64 interval_ns;
      uint64 expired;     // expirations not yet read
    } tfd;
    struct { uint64 mask; } sfd;   // kernel bit layout, as signal_mask
    struct { int n; struct epitem *items; } ep;
    struct inotify *in;
  };
};

static uint64
clock_ns(int clock)
{
  uint64 sec, usec;
  // The same clock clock_gettime() reads, so an absolute time a program
  // computed from it means the same instant here.
  wallclock(&sec, &usec);
  if (clock != CLOCK_REALTIME)
    sec -= boot_epoch;
  return sec * 1000000000ULL + usec * 1000ULL;
}

// One tick, a poll_kick, or a signal or kill that makes waiting pointless:
// poll's wait, so data arriving wakes an epoll_wait as it wakes a poll.
// Returns 1 for a tick, 0 for a kick, -EINTR.
static int
ev_block(void)
{
  return poll_wait_tick();
}

static int
ev_open(struct evobj *o, int flags, int readable, int writable)
{
  struct file *f;
  int fd;

  initsleeplock(&o->lock, "evobj");
  if ((f = filealloc()) == 0) {
    kmfree(o);
    return -ENFILE;
  }
  f->type     = FD_EVENT;
  f->ev       = o;
  f->readable = readable;
  f->writable = writable;
  f->nonblock = (flags & O_NONBLOCK) != 0;
  if ((fd = fdalloc(f)) < 0) {
    fileclose(f);
    return -EMFILE;
  }
  if (flags & O_CLOEXEC)
    myproc()->files->flags[fd] |= FD_CLOEXEC;
  return fd;
}

static struct evobj *
ev_new(int kind)
{
  struct evobj *o = kmalloc(sizeof(*o));
  if (o) {
    memset(o, 0, sizeof(*o));
    o->kind = kind;
  }
  return o;
}

static struct evobj *
ev_arg(int n, int kind, struct file **pf)
{
  int fd;
  struct file *f;
  if (argint(n, &fd) < 0 || fd < 0 || fd >= NOFILE || !(f = myproc()->files->ofile[fd]))
    return 0;
  if (f->type != FD_EVENT || f->ev->kind != kind)
    return 0;
  if (pf)
    *pf = f;
  return f->ev;
}

// The name Linux gives each kind in /proc/<pid>/fd.
const char *
ev_kind_name(struct evobj *o)
{
  switch (o ? o->kind : 0) {
  case EV_EVENTFD:  return "anon_inode:[eventfd]";
  case EV_TIMERFD:  return "anon_inode:[timerfd]";
  case EV_SIGNALFD: return "anon_inode:[signalfd]";
  case EV_EPOLL:    return "anon_inode:[eventpoll]";
  case EV_INOTIFY:  return "anon_inode:inotify";
  }
  return "anon_inode:[event]";
}

static void in_free(struct inotify *in);
static int  in_pending(struct inotify *in);
static int  in_read(struct file *f, struct inotify *in, char *addr, int n);

// Called by fileclose() for the last reference.
void
ev_close(struct evobj *o)
{
  if (o->kind == EV_INOTIFY)
    in_free(o->in);
  if (o->kind == EV_EPOLL && o->ep.items)
    kfree((char *)o->ep.items);
  kmfree(o);
}

// --- timerfd's arithmetic ----------------------------------------------------

// Fold expirations up to now into the count. Lazily, on every look: there is
// no timer interrupt behind a timerfd, only the clock and this.
static void
tfd_catch_up(struct evobj *o)
{
  if (!o->tfd.next_ns)
    return;
  uint64 now = clock_ns(o->tfd.clock);
  if (now < o->tfd.next_ns)
    return;
  if (o->tfd.interval_ns) {
    uint64 n = 1 + (now - o->tfd.next_ns) / o->tfd.interval_ns;
    o->tfd.expired += n;
    o->tfd.next_ns += n * o->tfd.interval_ns;
  } else {
    o->tfd.expired++;
    o->tfd.next_ns = 0;
  }
}

// --- readiness -----------------------------------------------------------------

static int ep_ready(struct evobj *o, struct epoll_event *out, int max);

int
ev_poll(struct file *f, int events)
{
  struct evobj *o = f->ev;
  int rev = 0;

  acquiresleep(&o->lock);
  switch (o->kind) {
  case EV_EVENTFD:
    if (o->efd.count > 0)
      rev |= POLLIN;
    if (o->efd.count < 0xfffffffffffffffeULL)
      rev |= POLLOUT;
    break;
  case EV_TIMERFD:
    tfd_catch_up(o);
    if (o->tfd.expired)
      rev |= POLLIN;
    break;
  case EV_SIGNALFD:
    if (myproc()->pending_signals & o->sfd.mask)
      rev |= POLLIN;
    break;
  case EV_INOTIFY:
    if (in_pending(o->in))
      rev |= POLLIN;
    break;
  }
  releasesleep(&o->lock);

  // An epoll set is readable when something in it is: which is how sets
  // nest, and how a program polls an epoll descriptor along with others.
  if (o->kind == EV_EPOLL && ep_ready(o, 0, 1) > 0)
    rev |= POLLIN;
  return rev & (events | POLLERR | POLLHUP);
}

// --- read and write ------------------------------------------------------------

int
ev_read(struct file *f, char *addr, int n)
{
  struct evobj *o = f->ev;

  if (o->kind == EV_INOTIFY)
    return in_read(f, o->in, addr, n);

  for (;;) {
    acquiresleep(&o->lock);
    switch (o->kind) {
    case EV_EVENTFD:
      if (n < 8) { releasesleep(&o->lock); return -EINVAL; }
      if (o->efd.count) {
        uint64 v = o->efd.sema ? 1 : o->efd.count;
        o->efd.count -= v;
        releasesleep(&o->lock);
        return copy_to_user((addr_t)addr, &v, 8) < 0 ? -EFAULT : 8;
      }
      break;
    case EV_TIMERFD:
      if (n < 8) { releasesleep(&o->lock); return -EINVAL; }
      tfd_catch_up(o);
      if (o->tfd.expired) {
        uint64 v = o->tfd.expired;
        o->tfd.expired = 0;
        releasesleep(&o->lock);
        return copy_to_user((addr_t)addr, &v, 8) < 0 ? -EFAULT : 8;
      }
      break;
    case EV_SIGNALFD: {
      if (n < (int)sizeof(struct sfd_info)) { releasesleep(&o->lock); return -EINVAL; }
      // The reader's own pending signals, as Linux does it: a signalfd shared
      // across fork reads whichever process is reading.
      struct proc *p = myproc();
      uint64 have = p->pending_signals & o->sfd.mask;
      if (have) {
        int sig = 0;
        while (!(have & (1ULL << sig)))
          sig++;
        p->pending_signals &= ~(1ULL << sig);
        releasesleep(&o->lock);
        struct sfd_info si;
        memset(&si, 0, sizeof(si));
        si.ssi_signo = sig;
        return copy_to_user((addr_t)addr, &si, sizeof(si)) < 0 ? -EFAULT
                                                              : (int)sizeof(si);
      }
      break;
    }
    default:
      releasesleep(&o->lock);
      return -EINVAL;
    }
    releasesleep(&o->lock);
    if (f->nonblock)
      return -EAGAIN;
    if (ev_block() < 0)
      return -EINTR;
  }
}

int
ev_write(struct file *f, char *addr, int n)
{
  struct evobj *o = f->ev;
  uint64 v;

  if (o->kind != EV_EVENTFD)
    return -EINVAL;
  if (n < 8)
    return -EINVAL;
  if (copy_from_user(&v, (addr_t)addr, 8) < 0)
    return -EFAULT;
  if (v == 0xffffffffffffffffULL)
    return -EINVAL;
  for (;;) {
    acquiresleep(&o->lock);
    // The counter stops one short of all ones, and a write that would pass
    // that waits for a read to make room.
    if (o->efd.count + v >= o->efd.count &&
        o->efd.count + v < 0xffffffffffffffffULL) {
      o->efd.count += v;
      releasesleep(&o->lock);
      return 8;
    }
    releasesleep(&o->lock);
    if (f->nonblock)
      return -EAGAIN;
    if (ev_block() < 0)
      return -EINTR;
  }
}

// --- eventfd -------------------------------------------------------------------

static addr_t
eventfd_common(uint64 init, int flags)
{
  if (flags & ~(EFD_SEMAPHORE | O_NONBLOCK | O_CLOEXEC))
    return -EINVAL;
  struct evobj *o = ev_new(EV_EVENTFD);
  if (!o)
    return -ENOMEM;
  o->efd.count = init;
  o->efd.sema  = (flags & EFD_SEMAPHORE) != 0;
  return ev_open(o, flags, 1, 1);
}

addr_t
sys_eventfd2(void)
{
  int init, flags;
  if (argint(0, &init) < 0 || argint(1, &flags) < 0)
    return -EINVAL;
  return eventfd_common((uint32)init, flags);
}

// The original, with no flags argument -- the register that would hold one
// holds whatever the caller left there.
addr_t
sys_eventfd(void)
{
  int init;
  if (argint(0, &init) < 0)
    return -EINVAL;
  return eventfd_common((uint32)init, 0);
}

// --- timerfd -------------------------------------------------------------------

struct itimerspec_u {
  int64 it_interval_sec, it_interval_nsec;
  int64 it_value_sec, it_value_nsec;
};

addr_t
sys_timerfd_create(void)
{
  int clock, flags;
  if (argint(0, &clock) < 0 || argint(1, &flags) < 0)
    return -EINVAL;
  // REALTIME, MONOTONIC and BOOTTIME (7); the last is MONOTONIC here, since
  // nothing on SchOS suspends.
  if (clock != CLOCK_REALTIME && clock != CLOCK_MONOTONIC && clock != 7)
    return -EINVAL;
  if (flags & ~(O_NONBLOCK | O_CLOEXEC))
    return -EINVAL;
  struct evobj *o = ev_new(EV_TIMERFD);
  if (!o)
    return -ENOMEM;
  o->tfd.clock = clock;
  return ev_open(o, flags, 1, 0);
}

static void
tfd_current(struct evobj *o, struct itimerspec_u *cur)
{
  memset(cur, 0, sizeof(*cur));
  cur->it_interval_sec  = (int64)(o->tfd.interval_ns / 1000000000ULL);
  cur->it_interval_nsec = (int64)(o->tfd.interval_ns % 1000000000ULL);
  if (o->tfd.next_ns) {
    uint64 now = clock_ns(o->tfd.clock);
    uint64 left = o->tfd.next_ns > now ? o->tfd.next_ns - now : 1;
    cur->it_value_sec  = (int64)(left / 1000000000ULL);
    cur->it_value_nsec = (int64)(left % 1000000000ULL);
  }
}

addr_t
sys_timerfd_settime(void)
{
  int flags;
  addr_t unew, uold;
  struct itimerspec_u nv, ov;
  struct evobj *o = ev_arg(0, EV_TIMERFD, 0);

  if (!o)
    return -EBADF;
  if (argint(1, &flags) < 0 || argaddr(2, &unew) < 0 || argaddr(3, &uold) < 0)
    return -EINVAL;
  if (copy_from_user(&nv, unew, sizeof(nv)) < 0)
    return -EFAULT;
  if (nv.it_value_nsec < 0 || nv.it_value_nsec >= 1000000000 ||
      nv.it_interval_nsec < 0 || nv.it_interval_nsec >= 1000000000 ||
      nv.it_value_sec < 0 || nv.it_interval_sec < 0)
    return -EINVAL;

  acquiresleep(&o->lock);
  tfd_current(o, &ov);
  uint64 value = (uint64)nv.it_value_sec * 1000000000ULL + (uint64)nv.it_value_nsec;
  o->tfd.interval_ns = (uint64)nv.it_interval_sec * 1000000000ULL +
                       (uint64)nv.it_interval_nsec;
  o->tfd.expired = 0;
  if (value == 0) {
    o->tfd.next_ns = 0;             // disarm
  } else if (flags & TFD_TIMER_ABSTIME) {
    o->tfd.next_ns = value;
  } else {
    o->tfd.next_ns = clock_ns(o->tfd.clock) + value;
  }
  releasesleep(&o->lock);

  if (uold && copy_to_user(uold, &ov, sizeof(ov)) < 0)
    return -EFAULT;
  return 0;
}

addr_t
sys_timerfd_gettime(void)
{
  addr_t ucur;
  struct itimerspec_u cur;
  struct evobj *o = ev_arg(0, EV_TIMERFD, 0);

  if (!o)
    return -EBADF;
  if (argaddr(1, &ucur) < 0)
    return -EINVAL;
  acquiresleep(&o->lock);
  tfd_catch_up(o);
  tfd_current(o, &cur);
  releasesleep(&o->lock);
  return copy_to_user(ucur, &cur, sizeof(cur)) < 0 ? -EFAULT : 0;
}

// --- signalfd ------------------------------------------------------------------

static addr_t
signalfd_common(int fd, addr_t umask, uint64 size, int flags)
{
  uint64 umaskv;

  if (size != 8)
    return -EINVAL;
  if (flags & ~(O_NONBLOCK | O_CLOEXEC))
    return -EINVAL;
  if (copy_from_user(&umaskv, umask, 8) < 0)
    return -EFAULT;
  // User sigsets put signal n at bit n-1; the kernel keeps it at bit n.
  uint64 mask = umaskv << 1;

  // An existing signalfd: change its mask, keep the descriptor.
  if (fd != -1) {
    struct file *f;
    if (fd < 0 || fd >= NOFILE || !(f = myproc()->files->ofile[fd]) ||
        f->type != FD_EVENT || f->ev->kind != EV_SIGNALFD)
      return -EINVAL;
    acquiresleep(&f->ev->lock);
    f->ev->sfd.mask = mask;
    releasesleep(&f->ev->lock);
    return fd;
  }
  struct evobj *o = ev_new(EV_SIGNALFD);
  if (!o)
    return -ENOMEM;
  o->sfd.mask = mask;
  return ev_open(o, flags, 1, 0);
}

addr_t
sys_signalfd4(void)
{
  int fd, flags;
  addr_t umask, size;
  if (argint(0, &fd) < 0 || argaddr(1, &umask) < 0 || argaddr(2, &size) < 0 ||
      argint(3, &flags) < 0)
    return -EINVAL;
  return signalfd_common(fd, umask, size, flags);
}

addr_t
sys_signalfd(void)
{
  int fd;
  addr_t umask, size;
  if (argint(0, &fd) < 0 || argaddr(1, &umask) < 0 || argaddr(2, &size) < 0)
    return -EINVAL;
  return signalfd_common(fd, umask, size, 0);
}

// --- epoll ---------------------------------------------------------------------
//
// A set of (descriptor, file) pairs. The pair, not the descriptor alone: Linux
// registers the *file*, and an entry quietly leaves the set when that file is
// closed. Here an entry whose descriptor no longer holds the same file is
// treated as gone and dropped the next time the set is looked at -- which is
// that rule, checked lazily. No reference is held, so a registered descriptor
// closes exactly as it would otherwise.

static addr_t
epoll_common(int flags)
{
  if (flags & ~O_CLOEXEC)
    return -EINVAL;
  struct evobj *o = ev_new(EV_EPOLL);
  if (!o)
    return -ENOMEM;
  o->ep.items = (struct epitem *)kalloc();
  if (!o->ep.items) {
    kmfree(o);
    return -ENOMEM;
  }
  return ev_open(o, flags, 1, 0);
}

addr_t
sys_epoll_create1(void)
{
  int flags;
  if (argint(0, &flags) < 0)
    return -EINVAL;
  return epoll_common(flags);
}

// The size argument has been ignored since Linux 2.6.8, apart from having to
// be positive.
addr_t
sys_epoll_create(void)
{
  int size;
  if (argint(0, &size) < 0 || size <= 0)
    return -EINVAL;
  return epoll_common(0);
}

static int
ep_find(struct evobj *o, int fd)
{
  for (int i = 0; i < o->ep.n; i++)
    if (o->ep.items[i].fd == fd)
      return i;
  return -1;
}

static void
ep_drop(struct evobj *o, int i)
{
  o->ep.items[i] = o->ep.items[--o->ep.n];
}

addr_t
sys_epoll_ctl(void)
{
  int op, fd;
  addr_t uev;
  struct epoll_event ev;
  struct file *ef;
  struct evobj *o = ev_arg(0, EV_EPOLL, &ef);

  if (!o)
    return -EBADF;
  if (argint(1, &op) < 0 || argint(2, &fd) < 0 || argaddr(3, &uev) < 0)
    return -EINVAL;
  if (fd < 0 || fd >= NOFILE || !myproc()->files->ofile[fd])
    return -EBADF;
  struct file *tf = myproc()->files->ofile[fd];
  // A set cannot watch itself. (Nor contain a set that contains it; that
  // check is the deep one Linux does, and nothing here builds such loops.)
  if (tf == ef)
    return -EINVAL;
  // Regular files are refused, as on Linux: they are always ready, and a
  // program that adds one has a bug that EPERM tells it about.
  if (tf->type == FD_INODE && tf->ip && tf->ip->type == T_FILE)
    return -EPERM;
  if (op != EPOLL_CTL_DEL && copy_from_user(&ev, uev, sizeof(ev)) < 0)
    return -EFAULT;

  acquiresleep(&o->lock);
  int i = ep_find(o, fd);
  if (i >= 0 && o->ep.items[i].f != tf) {
    ep_drop(o, i);                 // a stale entry for a closed-and-reused fd
    i = -1;
  }
  int r = 0;
  switch (op) {
  case EPOLL_CTL_ADD:
    if (i >= 0)                { r = -EEXIST; break; }
    if (o->ep.n == EP_MAX)     { r = -ENOSPC; break; }
    o->ep.items[o->ep.n++] = (struct epitem){ .fd = fd, .f = tf, .events = ev.events,
                                             .data = ev.data };
    break;
  case EPOLL_CTL_MOD:
    if (i < 0)                 { r = -ENOENT; break; }
    o->ep.items[i].events    = ev.events;
    o->ep.items[i].data      = ev.data;
    o->ep.items[i].was_ready = 0;
    o->ep.items[i].disabled  = 0;
    break;
  case EPOLL_CTL_DEL:
    if (i < 0)                 { r = -ENOENT; break; }
    ep_drop(o, i);
    break;
  default:
    r = -EINVAL;
  }
  releasesleep(&o->lock);
  return r;
}

// Fill up to `max` events that are ready now. With `out` null, only count,
// and change nothing -- that is the question poll() on the set itself asks,
// and it must not use up an edge or a one-shot.
static int
ep_ready(struct evobj *o, struct epoll_event *out, int max)
{
  int got = 0;
  // Taken before any descriptor is looked at, so a kick that lands during the
  // scan is newer than what is recorded and is reported next time.
  uint gen = poll_generation();

  acquiresleep(&o->lock);
  for (int i = 0; i < o->ep.n && got < max; ) {
    struct epitem *it = &o->ep.items[i];
    struct file *now = (it->fd >= 0 && it->fd < NOFILE) ? myproc()->files->ofile[it->fd] : 0;
    if (now != it->f) {
      ep_drop(o, i);                // closed: it leaves the set
      continue;
    }
    i++;
    if (it->disabled)
      continue;
    // file_poll() is called with the set's lock held; for an event
    // descriptor in the set it takes that object's own lock, never this one
    // -- a set cannot hold itself.
    int want = (int)(it->events & (EPOLLIN | EPOLLOUT | EPOLLRDHUP)) | POLLERR | POLLHUP;
    int rev = file_poll(it->f, want);
    if (rev == POLLNVAL)
      rev = POLLERR;
    if (!rev) {
      it->was_ready = 0;
      continue;
    }
    // Edge-triggered: an edge is new data, not only a not-ready-to-ready
    // change. A descriptor still ready since it was last reported is
    // reported again if anything has signalled since -- possibly spuriously,
    // which EPOLLET allows; never missed, which it does not. Xorg watches its
    // clients this way. Waiting for a not-ready scan in between lost the
    // edge whenever X read part of a client's requests and more arrived
    // before it looked again: on one CPU that almost never happened, on
    // several it hung the desktop -- X waiting on the client, the client on X.
    if ((it->events & EPOLLET) && it->was_ready && it->seq == gen)
      continue;
    if (out) {
      out[got].events = (uint32)rev;
      out[got].data   = it->data;
      it->was_ready = 1;
      it->seq = gen;
      if (it->events & EPOLLONESHOT)
        it->disabled = 1;
    }
    got++;
  }
  releasesleep(&o->lock);
  return got;
}

static addr_t
epoll_wait_common(int timeout_ms)
{
  addr_t uevents;
  int max;
  struct file *ef;
  struct evobj *o = ev_arg(0, EV_EPOLL, &ef);

  if (!o)
    return -EBADF;
  if (argaddr(1, &uevents) < 0 || argint(2, &max) < 0 || max <= 0)
    return -EINVAL;
  if (max > EP_MAX)
    max = EP_MAX;

  struct epoll_event *buf = kmalloc(max * sizeof(*buf));
  if (!buf)
    return -ENOMEM;

  int ticks_left = timeout_ms < 0 ? -1 : (timeout_ms + MSEC_PER_TICK - 1) / MSEC_PER_TICK;
  int got;
  for (;;) {
    got = ep_ready(o, buf, max);
    if (got > 0 || ticks_left == 0)
      break;
    int w = ev_block();
    if (w < 0) {
      got = -EINTR;
      break;
    }
    if (w > 0 && ticks_left > 0)
      ticks_left--;
  }
  if (got > 0 && copy_to_user(uevents, buf, got * sizeof(*buf)) < 0)
    got = -EFAULT;
  kmfree(buf);
  return got;
}

addr_t
sys_epoll_wait(void)
{
  int timeout;
  if (argint(3, &timeout) < 0)
    return -EINVAL;
  return epoll_wait_common(timeout);
}

// epoll_pwait: the signal mask is not swapped in for the wait. A signal that
// arrives ends the wait with EINTR whatever the mask, as poll() already does.
addr_t
sys_epoll_pwait(void)
{
  return sys_epoll_wait();
}

// --- inotify -------------------------------------------------------------------
//
// Watches name an inode by device and number. The filesystem calls
// inotify_event() as it creates, removes, renames and changes things -- for
// the inode itself, and for the directory it is in, with the name -- and every
// watch on that inode gets the event queued on its descriptor.
//
// The hooks cost one load when nothing is watched anywhere, which is the
// usual case. GLib watches a directory for every GSettings and file monitor
// it keeps, and without inotify at all it found no file monitor and GNOME
// programs aborted at start.

// The masks are in uapi/inotify.h.

struct inwatch {
  int    wd;
  uint   dev;
  uint64 inum;
  uint32 mask;
};

// One page for the instance and its watches, one for the queue.
#define IN_QBYTES  PGSIZE
#define IN_MAXW    ((int)((PGSIZE - 64) / sizeof(struct inwatch)))

struct inotify {
  struct inotify *next;     // in_list
  int    nw, next_wd;
  int    qlen, overflowed;
  char  *q;
  struct inwatch w[];
};

struct in_event {
  int    wd;
  uint32 mask, cookie, len;
};

static struct spinlock in_lock;
static int             in_lock_ready;
static struct inotify *in_list;
static int             in_nwatch;     // across every instance: the fast path
static uint32          in_cookie;

static void
in_lock_once(void)
{
  if (!in_lock_ready) {
    initlock(&in_lock, "inotify");
    in_lock_ready = 1;
  }
}

// Queue one event. in_lock held. Linux's padding: the name, its NUL, and
// zeros up to a multiple of the header's size.
static void
in_queue(struct inotify *in, int wd, uint32 mask, uint32 cookie, const char *name)
{
  int nlen = name ? strlen(name) + 1 : 0;
  int pad = nlen ? (nlen + 15) & ~15 : 0;
  int need = (int)sizeof(struct in_event) + pad;
  struct in_event e;

  // Room is kept for one overflow event, which says "you missed some".
  if (in->qlen + need > IN_QBYTES - (int)sizeof(struct in_event)) {
    if (!in->overflowed) {
      e.wd = -1; e.mask = IN_Q_OVERFLOW; e.cookie = 0; e.len = 0;
      memmove(in->q + in->qlen, &e, sizeof(e));
      in->qlen += sizeof(e);
      in->overflowed = 1;
    }
    return;
  }
  e.wd = wd; e.mask = mask; e.cookie = cookie; e.len = (uint32)pad;
  // The same event as the last one queued is not queued twice, as on Linux:
  // a file written in a hundred pieces is one IN_MODIFY until someone reads.
  if (in->qlen >= need && !name && pad == 0) {
    struct in_event last;
    memmove(&last, in->q + in->qlen - sizeof(last), sizeof(last));
    if (last.wd == wd && last.mask == mask && last.cookie == cookie && last.len == 0)
      return;
  }
  memmove(in->q + in->qlen, &e, sizeof(e));
  if (pad) {
    memset(in->q + in->qlen + sizeof(e), 0, pad);
    memmove(in->q + in->qlen + sizeof(e), name, nlen - 1);
  }
  in->qlen += need;
}

static void
in_drop_watch(struct inotify *in, int i)
{
  in_queue(in, in->w[i].wd, IN_IGNORED, 0, 0);
  in->w[i] = in->w[--in->nw];
  in_nwatch--;
}

uint32
inotify_cookie(void)
{
  in_lock_once();
  acquire(&in_lock);
  uint32 c = ++in_cookie;
  release(&in_lock);
  return c;
}

// Something happened to inode (dev, inum): to the inode itself when name is
// null, or to the entry `name` in it, a directory. Safe with inode locks
// held; takes a spinlock and nothing else.
void
inotify_event(uint dev, uint64 inum, uint32 mask, const char *name, uint32 cookie)
{
  struct inotify *in;
  int queued = 0;

  if (in_nwatch == 0)
    return;
  in_lock_once();
  acquire(&in_lock);
  for (in = in_list; in; in = in->next) {
    for (int i = 0; i < in->nw; i++) {
      struct inwatch *w = &in->w[i];
      if (w->dev != dev || w->inum != inum)
        continue;
      if (w->mask & mask & IN_ALL_EVENTS) {
        in_queue(in, w->wd, mask, cookie, name);
        queued = 1;
        if (w->mask & IN_ONESHOT) {
          in_drop_watch(in, i--);
          continue;
        }
      }
      // The watched thing is gone; so is the watch, and the reader is told.
      if (!name && (mask & IN_DELETE_SELF)) {
        in_drop_watch(in, i--);
        queued = 1;
      }
    }
  }
  release(&in_lock);
  if (queued)
    poll_kick();
}

// An event on an open file, for the file and for its directory. The
// directory's needs the name, which only the path the file was opened by
// gives -- looked up again here, and only when something is being watched.
void
inotify_file(struct file *f, uint32 mask)
{
  char name[VFS_NAME_MAX + 1];
  struct inode *dp;

  if (in_nwatch == 0 || f->type != FD_INODE || !f->ip)
    return;
  inotify_event(f->ip->dev, f->ip->inum, mask, 0, 0);
  if (!f->path)
    return;
  begin_op();
  if ((dp = nameiparent(f->path, name)) != 0) {
    inotify_event(dp->dev, dp->inum, mask, name, 0);
    iput(dp);
  }
  end_op();
}

static int
in_pending(struct inotify *in)
{
  in_lock_once();
  acquire(&in_lock);
  int r = in->qlen > 0;
  release(&in_lock);
  return r;
}

static void
in_free(struct inotify *in)
{
  struct inotify **pp;

  in_lock_once();
  acquire(&in_lock);
  for (pp = &in_list; *pp; pp = &(*pp)->next)
    if (*pp == in) {
      *pp = in->next;
      break;
    }
  in_nwatch -= in->nw;
  release(&in_lock);
  kfree(in->q);
  kfree((char *)in);
}

// As many whole events as fit; EINVAL if not even the first does.
static int
in_read(struct file *f, struct inotify *in, char *addr, int n)
{
  char *tmp;
  int take;

  for (;;) {
    in_lock_once();
    acquire(&in_lock);
    if (in->qlen > 0)
      break;
    release(&in_lock);
    if (f->nonblock)
      return -EAGAIN;
    if (ev_block() < 0)
      return -EINTR;
  }
  take = 0;
  while (take < in->qlen) {
    struct in_event e;
    memmove(&e, in->q + take, sizeof(e));
    int sz = (int)sizeof(e) + (int)e.len;
    if (take + sz > n)
      break;
    take += sz;
  }
  if (take == 0) {
    release(&in_lock);
    return -EINVAL;
  }
  release(&in_lock);
  // Copied out of the queue under the lock into a page of our own, and from
  // there to the reader without it: copy_to_user may fault.
  if ((tmp = kalloc()) == 0)
    return -ENOMEM;
  acquire(&in_lock);
  memmove(tmp, in->q, take);
  memmove(in->q, in->q + take, in->qlen - take);
  in->qlen -= take;
  if (in->qlen == 0)
    in->overflowed = 0;
  release(&in_lock);
  int r = copy_to_user((addr_t)addr, tmp, take) < 0 ? -EFAULT : take;
  kfree(tmp);
  return r;
}

static int
in_create(int flags)
{
  struct evobj *o;
  struct inotify *in;
  int fd;

  if (flags & ~(O_NONBLOCK | O_CLOEXEC))
    return -EINVAL;
  if ((o = ev_new(EV_INOTIFY)) == 0)
    return -ENOMEM;
  if ((in = (struct inotify *)kalloc()) == 0) {
    kmfree(o);
    return -ENOMEM;
  }
  memset(in, 0, PGSIZE);
  if ((in->q = kalloc()) == 0) {
    kfree((char *)in);
    kmfree(o);
    return -ENOMEM;
  }
  in->next_wd = 1;
  o->in = in;
  // ev_open frees o on failure; past filealloc it does so through fileclose,
  // and ev_close frees the instance too.
  if ((fd = ev_open(o, flags, 1, 0)) < 0) {
    if (fd == -ENFILE) {
      kfree(in->q);
      kfree((char *)in);
    }
    return fd;
  }
  in_lock_once();
  acquire(&in_lock);
  in->next = in_list;
  in_list = in;
  release(&in_lock);
  return fd;
}

addr_t sys_inotify_init(void) { return in_create(0); }

addr_t
sys_inotify_init1(void)
{
  int flags;
  argint(0, &flags);
  return in_create(flags);
}

addr_t
sys_inotify_add_watch(void)
{
  struct evobj *o;
  struct inotify *in;
  char *path;
  uint32 mask;
  int err = 0, wd = -1;
  uint dev;
  uint64 inum;

  if ((o = ev_arg(0, EV_INOTIFY, 0)) == 0)
    return -EBADF;
  in = o->in;
  if (argstr(1, &path) < 0)
    return -EFAULT;
  argint(2, (int *)&mask);
  if (!(mask & IN_ALL_EVENTS))
    return -EINVAL;
  if ((mask & IN_MASK_ADD) && (mask & IN_MASK_CREATE))
    return -EINVAL;
  if ((err = path_identity(path, !(mask & IN_DONT_FOLLOW),
                           (mask & IN_ONLYDIR) != 0, &dev, &inum)) < 0)
    return err;

  mask &= IN_ALL_EVENTS | IN_ONESHOT | IN_EXCL_UNLINK | IN_MASK_ADD;
  in_lock_once();
  acquire(&in_lock);
  for (int i = 0; i < in->nw; i++) {
    if (in->w[i].dev == dev && in->w[i].inum == inum) {
      if (mask & IN_MASK_CREATE) {
        release(&in_lock);
        return -EEXIST;
      }
      in->w[i].mask = (mask & IN_MASK_ADD) ? in->w[i].mask | (mask & ~IN_MASK_ADD)
                                           : mask;
      wd = in->w[i].wd;
      release(&in_lock);
      return wd;
    }
  }
  if (in->nw >= IN_MAXW) {
    release(&in_lock);
    return -ENOSPC;
  }
  wd = in->next_wd++;
  in->w[in->nw].wd = wd;
  in->w[in->nw].dev = dev;
  in->w[in->nw].inum = inum;
  in->w[in->nw].mask = mask & ~IN_MASK_ADD;
  in->nw++;
  in_nwatch++;
  release(&in_lock);
  return wd;
}

addr_t
sys_inotify_rm_watch(void)
{
  struct evobj *o;
  struct inotify *in;
  int wd;

  if ((o = ev_arg(0, EV_INOTIFY, 0)) == 0)
    return -EBADF;
  in = o->in;
  argint(1, &wd);
  in_lock_once();
  acquire(&in_lock);
  for (int i = 0; i < in->nw; i++) {
    if (in->w[i].wd == wd) {
      in_drop_watch(in, i);
      release(&in_lock);
      poll_kick();
      return 0;
    }
  }
  release(&in_lock);
  return -EINVAL;
}

// FIONREAD: bytes a read would return, for the kinds where that is a number
// worth asking. inotifywait sizes its read buffer by it, and stopped at
// "Not a tty" without.
int
ev_fionread(struct evobj *o)
{
  if (o->kind == EV_INOTIFY) {
    in_lock_once();
    acquire(&in_lock);
    int n = o->in->qlen;
    release(&in_lock);
    return n;
  }
  return -ENOTTY;
}