SchOS/kernel/sys/sys_event.c

// eventfd, timerfd, signalfd and epoll: the descriptors an event loop is
// built from.
//
// Four kinds of file with no inode behind them, all FD_EVENT, told apart by
// struct evobj's kind. Each one is a counter or a set that read() drains and
// poll() reports on -- which is the whole design, and the reason they arrive
// together: an event loop like libwayland's puts all of them in one epoll set
// and sleeps on that, so a timer firing, a signal arriving and another thread
// poking an eventfd all become "this descriptor is readable".
//
// Waiting is the same as poll()'s: check, and if nothing is ready sleep one
// timer tick and check again. Not a wakeup per event -- nothing here has a
// wait queue -- so the latency is a tick, 10 ms. That matches every other
// descriptor poll() already sees, and the day poll() gets wait queues these
// get them in the same change.

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "core/kmem.h"
#include "arch/mmu.h"
#include "fs/fs.h"
#include "fs/file.h"
#include "core/proc.h"
#include "core/syscall.h"
#include "core/uaccess.h"
#include "uapi/posix.h"
#include "uapi/fcntl.h"
#include "uapi/stat.h"
#include "uapi/errno.h"

enum { EV_EVENTFD = 1, EV_TIMERFD, EV_SIGNALFD, EV_EPOLL };

// Linux's flag values, which are O_NONBLOCK and O_CLOEXEC for every one of
// these: one spelling per meaning, so they are tested as such.
#define EFD_SEMAPHORE   1
#define TFD_TIMER_ABSTIME 1

#define EPOLL_CTL_ADD   1
#define EPOLL_CTL_DEL   2
#define EPOLL_CTL_MOD   3
#define EPOLLIN         0x001
#define EPOLLOUT        0x004
#define EPOLLERR        0x008
#define EPOLLHUP        0x010
#define EPOLLRDHUP      0x2000
#define EPOLLEXCLUSIVE  (1u << 28)
#define EPOLLWAKEUP     (1u << 29)
#define EPOLLONESHOT    (1u << 30)
#define EPOLLET         (1u << 31)

// What a set can hold: one page of entries, about a hundred. libwayland's
// loop has a handful; a compositor with every client is a few dozen. A page
// from kalloc() rather than kmalloc(), whose objects stop short of a page.
#define EP_MAX ((int)(PGSIZE / sizeof(struct epitem)))

// x86-64 packs it: 12 bytes, the data unaligned. Getting this wrong puts every
// event's data four bytes off, which reads as a valid but wrong pointer.
struct epoll_event {
  uint32 events;
  uint64 data;
} __attribute__((packed));

// struct signalfd_siginfo is 128 bytes and the program reads it whole; only
// the signal number has a meaning here yet.
struct sfd_info {
  uint32 ssi_signo;
  int32  ssi_errno;
  int32  ssi_code;
  uint32 ssi_pid;
  uint32 ssi_uid;
  uint8  rest[108];
};

struct epitem {
  int          fd;
  struct file *f;      // checked against the table, never dereferenced alone
  uint32       events;
  uint64       data;
  int          was_ready;   // for EPOLLET: report on becoming ready, not on being
  int          disabled;    // EPOLLONESHOT fired, until EPOLL_CTL_MOD
};

struct evobj {
  int kind;
  // A sleep lock, because an epoll set is held across file_poll() of
  // everything in it, and some of those -- a socket -- may sleep.
  struct sleeplock lock;
  union {
    struct { uint64 count; int sema; } efd;
    struct {
      int    clock;
      uint64 next_ns;     // 0 when disarmed
      uint64 interval_ns;
      uint64 expired;     // expirations not yet read
    } tfd;
    struct { uint64 mask; } sfd;   // kernel bit layout, as signal_mask
    struct { int n; struct epitem *items; } ep;
  };
};

static uint64
clock_ns(int clock)
{
  uint64 sec, usec;
  // The same clock clock_gettime() reads, so an absolute time a program
  // computed from it means the same instant here.
  wallclock(&sec, &usec);
  if (clock != CLOCK_REALTIME)
    sec -= boot_epoch;
  return sec * 1000000000ULL + usec * 1000ULL;
}

// One tick, a poll_kick, or a signal or kill that makes waiting pointless:
// poll's wait, so data arriving wakes an epoll_wait as it wakes a poll.
// Returns 1 for a tick, 0 for a kick, -EINTR.
static int
ev_block(void)
{
  return poll_wait_tick();
}

static int
ev_open(struct evobj *o, int flags, int readable, int writable)
{
  struct file *f;
  int fd;

  initsleeplock(&o->lock, "evobj");
  if ((f = filealloc()) == 0) {
    kmfree(o);
    return -ENFILE;
  }
  f->type     = FD_EVENT;
  f->ev       = o;
  f->readable = readable;
  f->writable = writable;
  f->nonblock = (flags & O_NONBLOCK) != 0;
  if ((fd = fdalloc(f)) < 0) {
    fileclose(f);
    return -EMFILE;
  }
  if (flags & O_CLOEXEC)
    myproc()->files->flags[fd] |= FD_CLOEXEC;
  return fd;
}

static struct evobj *
ev_new(int kind)
{
  struct evobj *o = kmalloc(sizeof(*o));
  if (o) {
    memset(o, 0, sizeof(*o));
    o->kind = kind;
  }
  return o;
}

static struct evobj *
ev_arg(int n, int kind, struct file **pf)
{
  int fd;
  struct file *f;
  if (argint(n, &fd) < 0 || fd < 0 || fd >= NOFILE || !(f = myproc()->files->ofile[fd]))
    return 0;
  if (f->type != FD_EVENT || f->ev->kind != kind)
    return 0;
  if (pf)
    *pf = f;
  return f->ev;
}

// The name Linux gives each kind in /proc/<pid>/fd.
const char *
ev_kind_name(struct evobj *o)
{
  switch (o ? o->kind : 0) {
  case EV_EVENTFD:  return "anon_inode:[eventfd]";
  case EV_TIMERFD:  return "anon_inode:[timerfd]";
  case EV_SIGNALFD: return "anon_inode:[signalfd]";
  case EV_EPOLL:    return "anon_inode:[eventpoll]";
  }
  return "anon_inode:[event]";
}

// Called by fileclose() for the last reference.
void
ev_close(struct evobj *o)
{
  if (o->kind == EV_EPOLL && o->ep.items)
    kfree((char *)o->ep.items);
  kmfree(o);
}

// --- timerfd's arithmetic ----------------------------------------------------

// Fold expirations up to now into the count. Lazily, on every look: there is
// no timer interrupt behind a timerfd, only the clock and this.
static void
tfd_catch_up(struct evobj *o)
{
  if (!o->tfd.next_ns)
    return;
  uint64 now = clock_ns(o->tfd.clock);
  if (now < o->tfd.next_ns)
    return;
  if (o->tfd.interval_ns) {
    uint64 n = 1 + (now - o->tfd.next_ns) / o->tfd.interval_ns;
    o->tfd.expired += n;
    o->tfd.next_ns += n * o->tfd.interval_ns;
  } else {
    o->tfd.expired++;
    o->tfd.next_ns = 0;
  }
}

// --- readiness -----------------------------------------------------------------

static int ep_ready(struct evobj *o, struct epoll_event *out, int max);

int
ev_poll(struct file *f, int events)
{
  struct evobj *o = f->ev;
  int rev = 0;

  acquiresleep(&o->lock);
  switch (o->kind) {
  case EV_EVENTFD:
    if (o->efd.count > 0)
      rev |= POLLIN;
    if (o->efd.count < 0xfffffffffffffffeULL)
      rev |= POLLOUT;
    break;
  case EV_TIMERFD:
    tfd_catch_up(o);
    if (o->tfd.expired)
      rev |= POLLIN;
    break;
  case EV_SIGNALFD:
    if (myproc()->pending_signals & o->sfd.mask)
      rev |= POLLIN;
    break;
  }
  releasesleep(&o->lock);

  // An epoll set is readable when something in it is: which is how sets
  // nest, and how a program polls an epoll descriptor along with others.
  if (o->kind == EV_EPOLL && ep_ready(o, 0, 1) > 0)
    rev |= POLLIN;
  return rev & (events | POLLERR | POLLHUP);
}

// --- read and write ------------------------------------------------------------

int
ev_read(struct file *f, char *addr, int n)
{
  struct evobj *o = f->ev;

  for (;;) {
    acquiresleep(&o->lock);
    switch (o->kind) {
    case EV_EVENTFD:
      if (n < 8) { releasesleep(&o->lock); return -EINVAL; }
      if (o->efd.count) {
        uint64 v = o->efd.sema ? 1 : o->efd.count;
        o->efd.count -= v;
        releasesleep(&o->lock);
        return copy_to_user((addr_t)addr, &v, 8) < 0 ? -EFAULT : 8;
      }
      break;
    case EV_TIMERFD:
      if (n < 8) { releasesleep(&o->lock); return -EINVAL; }
      tfd_catch_up(o);
      if (o->tfd.expired) {
        uint64 v = o->tfd.expired;
        o->tfd.expired = 0;
        releasesleep(&o->lock);
        return copy_to_user((addr_t)addr, &v, 8) < 0 ? -EFAULT : 8;
      }
      break;
    case EV_SIGNALFD: {
      if (n < (int)sizeof(struct sfd_info)) { releasesleep(&o->lock); return -EINVAL; }
      // The reader's own pending signals, as Linux does it: a signalfd shared
      // across fork reads whichever process is reading.
      struct proc *p = myproc();
      uint64 have = p->pending_signals & o->sfd.mask;
      if (have) {
        int sig = 0;
        while (!(have & (1ULL << sig)))
          sig++;
        p->pending_signals &= ~(1ULL << sig);
        releasesleep(&o->lock);
        struct sfd_info si;
        memset(&si, 0, sizeof(si));
        si.ssi_signo = sig;
        return copy_to_user((addr_t)addr, &si, sizeof(si)) < 0 ? -EFAULT
                                                              : (int)sizeof(si);
      }
      break;
    }
    default:
      releasesleep(&o->lock);
      return -EINVAL;
    }
    releasesleep(&o->lock);
    if (f->nonblock)
      return -EAGAIN;
    if (ev_block() < 0)
      return -EINTR;
  }
}

int
ev_write(struct file *f, char *addr, int n)
{
  struct evobj *o = f->ev;
  uint64 v;

  if (o->kind != EV_EVENTFD)
    return -EINVAL;
  if (n < 8)
    return -EINVAL;
  if (copy_from_user(&v, (addr_t)addr, 8) < 0)
    return -EFAULT;
  if (v == 0xffffffffffffffffULL)
    return -EINVAL;
  for (;;) {
    acquiresleep(&o->lock);
    // The counter stops one short of all ones, and a write that would pass
    // that waits for a read to make room.
    if (o->efd.count + v >= o->efd.count &&
        o->efd.count + v < 0xffffffffffffffffULL) {
      o->efd.count += v;
      releasesleep(&o->lock);
      return 8;
    }
    releasesleep(&o->lock);
    if (f->nonblock)
      return -EAGAIN;
    if (ev_block() < 0)
      return -EINTR;
  }
}

// --- eventfd -------------------------------------------------------------------

static addr_t
eventfd_common(uint64 init, int flags)
{
  if (flags & ~(EFD_SEMAPHORE | O_NONBLOCK | O_CLOEXEC))
    return -EINVAL;
  struct evobj *o = ev_new(EV_EVENTFD);
  if (!o)
    return -ENOMEM;
  o->efd.count = init;
  o->efd.sema  = (flags & EFD_SEMAPHORE) != 0;
  return ev_open(o, flags, 1, 1);
}

addr_t
sys_eventfd2(void)
{
  int init, flags;
  if (argint(0, &init) < 0 || argint(1, &flags) < 0)
    return -EINVAL;
  return eventfd_common((uint32)init, flags);
}

// The original, with no flags argument -- the register that would hold one
// holds whatever the caller left there.
addr_t
sys_eventfd(void)
{
  int init;
  if (argint(0, &init) < 0)
    return -EINVAL;
  return eventfd_common((uint32)init, 0);
}

// --- timerfd -------------------------------------------------------------------

struct itimerspec_u {
  int64 it_interval_sec, it_interval_nsec;
  int64 it_value_sec, it_value_nsec;
};

addr_t
sys_timerfd_create(void)
{
  int clock, flags;
  if (argint(0, &clock) < 0 || argint(1, &flags) < 0)
    return -EINVAL;
  // REALTIME, MONOTONIC and BOOTTIME (7); the last is MONOTONIC here, since
  // nothing on SchOS suspends.
  if (clock != CLOCK_REALTIME && clock != CLOCK_MONOTONIC && clock != 7)
    return -EINVAL;
  if (flags & ~(O_NONBLOCK | O_CLOEXEC))
    return -EINVAL;
  struct evobj *o = ev_new(EV_TIMERFD);
  if (!o)
    return -ENOMEM;
  o->tfd.clock = clock;
  return ev_open(o, flags, 1, 0);
}

static void
tfd_current(struct evobj *o, struct itimerspec_u *cur)
{
  memset(cur, 0, sizeof(*cur));
  cur->it_interval_sec  = (int64)(o->tfd.interval_ns / 1000000000ULL);
  cur->it_interval_nsec = (int64)(o->tfd.interval_ns % 1000000000ULL);
  if (o->tfd.next_ns) {
    uint64 now = clock_ns(o->tfd.clock);
    uint64 left = o->tfd.next_ns > now ? o->tfd.next_ns - now : 1;
    cur->it_value_sec  = (int64)(left / 1000000000ULL);
    cur->it_value_nsec = (int64)(left % 1000000000ULL);
  }
}

addr_t
sys_timerfd_settime(void)
{
  int flags;
  addr_t unew, uold;
  struct itimerspec_u nv, ov;
  struct evobj *o = ev_arg(0, EV_TIMERFD, 0);

  if (!o)
    return -EBADF;
  if (argint(1, &flags) < 0 || argaddr(2, &unew) < 0 || argaddr(3, &uold) < 0)
    return -EINVAL;
  if (copy_from_user(&nv, unew, sizeof(nv)) < 0)
    return -EFAULT;
  if (nv.it_value_nsec < 0 || nv.it_value_nsec >= 1000000000 ||
      nv.it_interval_nsec < 0 || nv.it_interval_nsec >= 1000000000 ||
      nv.it_value_sec < 0 || nv.it_interval_sec < 0)
    return -EINVAL;

  acquiresleep(&o->lock);
  tfd_current(o, &ov);
  uint64 value = (uint64)nv.it_value_sec * 1000000000ULL + (uint64)nv.it_value_nsec;
  o->tfd.interval_ns = (uint64)nv.it_interval_sec * 1000000000ULL +
                       (uint64)nv.it_interval_nsec;
  o->tfd.expired = 0;
  if (value == 0) {
    o->tfd.next_ns = 0;             // disarm
  } else if (flags & TFD_TIMER_ABSTIME) {
    o->tfd.next_ns = value;
  } else {
    o->tfd.next_ns = clock_ns(o->tfd.clock) + value;
  }
  releasesleep(&o->lock);

  if (uold && copy_to_user(uold, &ov, sizeof(ov)) < 0)
    return -EFAULT;
  return 0;
}

addr_t
sys_timerfd_gettime(void)
{
  addr_t ucur;
  struct itimerspec_u cur;
  struct evobj *o = ev_arg(0, EV_TIMERFD, 0);

  if (!o)
    return -EBADF;
  if (argaddr(1, &ucur) < 0)
    return -EINVAL;
  acquiresleep(&o->lock);
  tfd_catch_up(o);
  tfd_current(o, &cur);
  releasesleep(&o->lock);
  return copy_to_user(ucur, &cur, sizeof(cur)) < 0 ? -EFAULT : 0;
}

// --- signalfd ------------------------------------------------------------------

static addr_t
signalfd_common(int fd, addr_t umask, uint64 size, int flags)
{
  uint64 umaskv;

  if (size != 8)
    return -EINVAL;
  if (flags & ~(O_NONBLOCK | O_CLOEXEC))
    return -EINVAL;
  if (copy_from_user(&umaskv, umask, 8) < 0)
    return -EFAULT;
  // User sigsets put signal n at bit n-1; the kernel keeps it at bit n.
  uint64 mask = umaskv << 1;

  // An existing signalfd: change its mask, keep the descriptor.
  if (fd != -1) {
    struct file *f;
    if (fd < 0 || fd >= NOFILE || !(f = myproc()->files->ofile[fd]) ||
        f->type != FD_EVENT || f->ev->kind != EV_SIGNALFD)
      return -EINVAL;
    acquiresleep(&f->ev->lock);
    f->ev->sfd.mask = mask;
    releasesleep(&f->ev->lock);
    return fd;
  }
  struct evobj *o = ev_new(EV_SIGNALFD);
  if (!o)
    return -ENOMEM;
  o->sfd.mask = mask;
  return ev_open(o, flags, 1, 0);
}

addr_t
sys_signalfd4(void)
{
  int fd, flags;
  addr_t umask, size;
  if (argint(0, &fd) < 0 || argaddr(1, &umask) < 0 || argaddr(2, &size) < 0 ||
      argint(3, &flags) < 0)
    return -EINVAL;
  return signalfd_common(fd, umask, size, flags);
}

addr_t
sys_signalfd(void)
{
  int fd;
  addr_t umask, size;
  if (argint(0, &fd) < 0 || argaddr(1, &umask) < 0 || argaddr(2, &size) < 0)
    return -EINVAL;
  return signalfd_common(fd, umask, size, 0);
}

// --- epoll ---------------------------------------------------------------------
//
// A set of (descriptor, file) pairs. The pair, not the descriptor alone: Linux
// registers the *file*, and an entry quietly leaves the set when that file is
// closed. Here an entry whose descriptor no longer holds the same file is
// treated as gone and dropped the next time the set is looked at -- which is
// that rule, checked lazily. No reference is held, so a registered descriptor
// closes exactly as it would otherwise.

static addr_t
epoll_common(int flags)
{
  if (flags & ~O_CLOEXEC)
    return -EINVAL;
  struct evobj *o = ev_new(EV_EPOLL);
  if (!o)
    return -ENOMEM;
  o->ep.items = (struct epitem *)kalloc();
  if (!o->ep.items) {
    kmfree(o);
    return -ENOMEM;
  }
  return ev_open(o, flags, 1, 0);
}

addr_t
sys_epoll_create1(void)
{
  int flags;
  if (argint(0, &flags) < 0)
    return -EINVAL;
  return epoll_common(flags);
}

// The size argument has been ignored since Linux 2.6.8, apart from having to
// be positive.
addr_t
sys_epoll_create(void)
{
  int size;
  if (argint(0, &size) < 0 || size <= 0)
    return -EINVAL;
  return epoll_common(0);
}

static int
ep_find(struct evobj *o, int fd)
{
  for (int i = 0; i < o->ep.n; i++)
    if (o->ep.items[i].fd == fd)
      return i;
  return -1;
}

static void
ep_drop(struct evobj *o, int i)
{
  o->ep.items[i] = o->ep.items[--o->ep.n];
}

addr_t
sys_epoll_ctl(void)
{
  int op, fd;
  addr_t uev;
  struct epoll_event ev;
  struct file *ef;
  struct evobj *o = ev_arg(0, EV_EPOLL, &ef);

  if (!o)
    return -EBADF;
  if (argint(1, &op) < 0 || argint(2, &fd) < 0 || argaddr(3, &uev) < 0)
    return -EINVAL;
  if (fd < 0 || fd >= NOFILE || !myproc()->files->ofile[fd])
    return -EBADF;
  struct file *tf = myproc()->files->ofile[fd];
  // A set cannot watch itself. (Nor contain a set that contains it; that
  // check is the deep one Linux does, and nothing here builds such loops.)
  if (tf == ef)
    return -EINVAL;
  // Regular files are refused, as on Linux: they are always ready, and a
  // program that adds one has a bug that EPERM tells it about.
  if (tf->type == FD_INODE && tf->ip && tf->ip->type == T_FILE)
    return -EPERM;
  if (op != EPOLL_CTL_DEL && copy_from_user(&ev, uev, sizeof(ev)) < 0)
    return -EFAULT;

  acquiresleep(&o->lock);
  int i = ep_find(o, fd);
  if (i >= 0 && o->ep.items[i].f != tf) {
    ep_drop(o, i);                 // a stale entry for a closed-and-reused fd
    i = -1;
  }
  int r = 0;
  switch (op) {
  case EPOLL_CTL_ADD:
    if (i >= 0)                { r = -EEXIST; break; }
    if (o->ep.n == EP_MAX)     { r = -ENOSPC; break; }
    o->ep.items[o->ep.n++] = (struct epitem){ fd, tf, ev.events, ev.data, 0, 0 };
    break;
  case EPOLL_CTL_MOD:
    if (i < 0)                 { r = -ENOENT; break; }
    o->ep.items[i].events    = ev.events;
    o->ep.items[i].data      = ev.data;
    o->ep.items[i].was_ready = 0;
    o->ep.items[i].disabled  = 0;
    break;
  case EPOLL_CTL_DEL:
    if (i < 0)                 { r = -ENOENT; break; }
    ep_drop(o, i);
    break;
  default:
    r = -EINVAL;
  }
  releasesleep(&o->lock);
  return r;
}

// Fill up to `max` events that are ready now. With `out` null, only count,
// and change nothing -- that is the question poll() on the set itself asks,
// and it must not use up an edge or a one-shot.
static int
ep_ready(struct evobj *o, struct epoll_event *out, int max)
{
  int got = 0;

  acquiresleep(&o->lock);
  for (int i = 0; i < o->ep.n && got < max; ) {
    struct epitem *it = &o->ep.items[i];
    struct file *now = (it->fd >= 0 && it->fd < NOFILE) ? myproc()->files->ofile[it->fd] : 0;
    if (now != it->f) {
      ep_drop(o, i);                // closed: it leaves the set
      continue;
    }
    i++;
    if (it->disabled)
      continue;
    // file_poll() is called with the set's lock held; for an event
    // descriptor in the set it takes that object's own lock, never this one
    // -- a set cannot hold itself.
    int want = (int)(it->events & (EPOLLIN | EPOLLOUT | EPOLLRDHUP)) | POLLERR | POLLHUP;
    int rev = file_poll(it->f, want);
    if (rev == POLLNVAL)
      rev = POLLERR;
    if (!rev) {
      it->was_ready = 0;
      continue;
    }
    if ((it->events & EPOLLET) && it->was_ready)
      continue;
    if (out) {
      out[got].events = (uint32)rev;
      out[got].data   = it->data;
      it->was_ready = 1;
      if (it->events & EPOLLONESHOT)
        it->disabled = 1;
    }
    got++;
  }
  releasesleep(&o->lock);
  return got;
}

static addr_t
epoll_wait_common(int timeout_ms)
{
  addr_t uevents;
  int max;
  struct file *ef;
  struct evobj *o = ev_arg(0, EV_EPOLL, &ef);

  if (!o)
    return -EBADF;
  if (argaddr(1, &uevents) < 0 || argint(2, &max) < 0 || max <= 0)
    return -EINVAL;
  if (max > EP_MAX)
    max = EP_MAX;

  struct epoll_event *buf = kmalloc(max * sizeof(*buf));
  if (!buf)
    return -ENOMEM;

  int ticks_left = timeout_ms < 0 ? -1 : (timeout_ms + MSEC_PER_TICK - 1) / MSEC_PER_TICK;
  int got;
  for (;;) {
    got = ep_ready(o, buf, max);
    if (got > 0 || ticks_left == 0)
      break;
    int w = ev_block();
    if (w < 0) {
      got = -EINTR;
      break;
    }
    if (w > 0 && ticks_left > 0)
      ticks_left--;
  }
  if (got > 0 && copy_to_user(uevents, buf, got * sizeof(*buf)) < 0)
    got = -EFAULT;
  kmfree(buf);
  return got;
}

addr_t
sys_epoll_wait(void)
{
  int timeout;
  if (argint(3, &timeout) < 0)
    return -EINVAL;
  return epoll_wait_common(timeout);
}

// epoll_pwait: the signal mask is not swapped in for the wait. A signal that
// arrives ends the wait with EINTR whatever the mask, as poll() already does.
addr_t
sys_epoll_pwait(void)
{
  return sys_epoll_wait();
}