SchOS/kernel/net/inet.c

// SPDX-License-Identifier: GPL-2.0-or-later
//
// AF_INET sockets, on top of lwIP's raw API.
//
// The raw API rather than lwIP's own sockets layer, because lwIP's needs
// NO_SYS=0 -- threads, mailboxes and semaphores of its own -- and this kernel
// already has a socket layer with a shape of its own that AF_INET has to fit
// into rather than replace. See kernel/net/lwip/lwipopts.h.
//
// ---------------------------------------------------------------------------
// Locking, and why there is only one lock here
//
// Everything in an inet socket is protected by the lwIP lock and nothing else.
// That is not a shortcut: lwIP's callbacks (data arriving, a connection
// completing, a peer closing) run in the network thread with that lock held,
// and they are the only things that write to a socket's receive buffer. A
// syscall reading that buffer takes the same lock. There is no second lock
// because there is no second writer.
//
// ---------------------------------------------------------------------------
// Blocking, and how a waiter is woken
//
// A blocking call sleeps until an lwIP callback says something changed. It used
// to wake on every timer tick and look, which cost a wakeup per socket per
// 10 ms and answered up to 10 ms late even when the data was already there.
//
// The obstacle was a lock mismatch, and it is worth writing down because it is
// the reason this is not simply sleep(&socket, &lwip_lock): sleep() wants a
// spinlock, and the lwIP lock is a sleeplock -- it must be, because a call into
// lwIP can reach the driver and the driver can sleep. A waiter cannot hold the
// lwIP lock while it sleeps either, or the network thread could never deliver
// the very thing it is waiting for.
//
// So the wait is on a counter with a spinlock of its own:
//
//   reader:  lwip_lock; check condition; s = net_seq(); lwip_unlock;
//            net_wait(s)
//   waker:   (in a callback, lwIP lock held) net_wake()
//
// No wakeup can be lost. `s` is read while the reader still holds the lwIP
// lock, so anything that changes the condition must do so after that -- and
// therefore bumps the counter after the read. net_wait re-checks the counter
// under its spinlock before sleeping, sees it has moved, and does not sleep.
//
// One counter and one channel for every inet socket, rather than one each. At
// sixteen sockets the cost of waking all of them is a handful of instructions
// in the ones with nothing to do, and a per-socket channel would have to be
// torn down carefully in close() against callbacks that can still arrive.

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "core/proc.h"
#include "fs/fs.h"      /* NDIRECT, which file.h's struct inode needs */
#include "fs/file.h"
#define _KERNEL_SOCKET_INTERNALS
#include "net/socket.h"
#include "uapi/errno.h"

#include "lwip/opt.h"
#include "lwip/tcp.h"
#include "lwip/udp.h"
#include "lwip/raw.h"
#include "lwip/icmp.h"
#include "lwip/prot/ip4.h"
#include "lwip/ip_addr.h"
#include "lwip/pbuf.h"

#define NINET      16        // AF_INET sockets in the system
// Per socket. More than TCP_WND, which it must be (see inet_recv_cb), and a
// power of two, which it must also be: rhead and rtail run freely and are
// reduced modulo this, which stays continuous across their wrap at 2^32 only
// when it divides 2^32.
#define INET_RXBUF (512 * 1024)
_Static_assert(INET_RXBUF > TCP_WND, "the ring must hold a whole window");
_Static_assert((INET_RXBUF & (INET_RXBUF - 1)) == 0, "the ring must be a power of two");
#define NDGRAM     16        // datagrams queued on a UDP socket

// Where one datagram ends and the next begins, and who sent it.
//
// The bytes themselves stay in the same ring TCP uses -- a datagram queue of
// whole buffers would be 16 sockets x 16 datagrams x an MTU of static memory
// for something that is nearly always empty. Only the boundaries and the
// source need to be kept alongside, and those are eight bytes each.
struct dgram {
  uint   addr;               // source, network byte order
  ushort port;               // source, host byte order
  ushort len;
};

struct inet_sock {
  int    inuse;
  int    type;               // SOCK_STREAM or SOCK_DGRAM
  int    proto;              // 0, or IPPROTO_ICMP for a ping socket
  int    iphdr;              // SOCK_RAW: reads start at the IP header

  struct tcp_pcb *tpcb;
  struct udp_pcb *upcb;
  struct raw_pcb *rpcb;      // SOCK_DGRAM + IPPROTO_ICMP

  // Receive ring. lwIP hands over pbufs from the network thread; a reader
  // drains bytes. Full means the sender is told to back off, which for TCP is
  // the window closing rather than data being dropped.
  char   rx[INET_RXBUF];
  uint   rhead, rtail;       // rhead == rtail is empty

  // UDP only: the boundaries within that ring. A stream socket has none --
  // that is the whole difference between the two -- so these stay at zero.
  struct dgram dq[NDGRAM];
  uint   dhead, dtail;       // dhead == dtail is empty

  int    connecting;         // connect() issued, no answer yet
  int    connected;
  int    conn_failed;
  int    peer_closed;        // FIN seen, or the connection died
  int    wr_shut;            // shutdown(SHUT_WR): our FIN is sent
  int    listening;

  // Connections lwIP has completed but nobody has accept()ed yet.
  struct inet_sock *acceptq[NBACKLOG];
  int    naccept;
};

static struct inet_sock itab[NINET];

/* ------------------------------------------------------------------ */
/* Waiting. See the note on blocking at the top of this file.          */
/* ------------------------------------------------------------------ */

static struct spinlock netwait_lock;
static uint            netwait_seq;

void
inet_init(void)
{
  initlock(&netwait_lock, "netwait");
}

// The current wakeup count. Read by a caller that has just checked its
// condition and is still holding the lwIP lock.
static uint
net_seq(void)
{
  uint s;

  acquire(&netwait_lock);
  s = netwait_seq;
  release(&netwait_lock);
  return s;
}

// A blocked socket call gives up for a kill and for any signal it can take.
// The second used to be missing, so a signal woke the sleeper and the loop
// below put it straight back: busybox's ping sends each request from a
// SIGALRM handler while it sits in recvfrom, and got one reply and then
// silence. There is no SA_RESTART, so the call returns EINTR and the caller
// asks again, which is what every one of them is written to do.
//
// Only signals with a handler count, as on Linux. A SIGCHLD left at its
// default is posted to a parent all the same, and ending a server's accept()
// for it would be an EINTR that program had never been handed before.
static int
net_interrupted(void)
{
  struct proc *p = myproc();
  if (p->killed)
    return 1;
  uint64 live = p->pending_signals & ~p->signal_mask;
  for (int sig = 1; sig < 32; sig++)
    if ((live & (1ULL << sig)) && p->sighand->handler[sig] > 1)   // not DFL/IGN
      return 1;
  return 0;
}

// Sleep until the count moves past `seen`, or a signal arrives. kill() makes
// a sleeping process runnable, so the loop re-checks and leaves.
static void
net_wait(uint seen)
{
  acquire(&netwait_lock);
  while (netwait_seq == seen && !net_interrupted())
    sleep(&netwait_seq, &netwait_lock);
  release(&netwait_lock);
}

// Something a waiter might care about has happened. Called from the lwIP
// callbacks, which run in the network thread with the lwIP lock held -- taking
// a spinlock there is fine, since nothing between acquire and release sleeps.
static void
net_wake(void)
{
  acquire(&netwait_lock);
  netwait_seq++;
  release(&netwait_lock);
  wakeup(&netwait_seq);
  poll_kick();                 // and poll/select/epoll waiters, not at the tick
}

/* ------------------------------------------------------------------ */
/* The receive ring. Callers hold the lwIP lock.                       */
/* ------------------------------------------------------------------ */

static uint
rx_used(struct inet_sock *is)
{
  return is->rhead - is->rtail;
}

static uint
rx_space(struct inet_sock *is)
{
  return INET_RXBUF - rx_used(is);
}

// Two copies at most, one each side of the ring's end. A byte at a time with a
// modulo each was a measurable share of a fast download.
static void
rx_put(struct inet_sock *is, const char *p, uint n)
{
  uint at = is->rhead % INET_RXBUF, first = INET_RXBUF - at;
  if (first > n)
    first = n;
  memmove(is->rx + at, p, first);
  memmove(is->rx, p + first, n - first);
  is->rhead += n;
}

static uint
rx_take(struct inet_sock *is, char *p, uint n)
{
  uint have = rx_used(is);
  if (n > have)
    n = have;
  uint at = is->rtail % INET_RXBUF, first = INET_RXBUF - at;
  if (first > n)
    first = n;
  memmove(p, is->rx + at, first);
  memmove(p + first, is->rx, n - first);
  is->rtail += n;
  return n;
}

// Throw bytes away without copying them. A datagram larger than the buffer it
// was read into is truncated, and the remainder is discarded rather than left
// to be delivered as if it were the next datagram.
static void
rx_drop(struct inet_sock *is, uint n)
{
  uint have = rx_used(is);
  is->rtail += n > have ? have : n;
}

/* ------------------------------------------------------------------ */
/* Datagram boundaries. UDP only; callers hold the lwIP lock.          */
/* ------------------------------------------------------------------ */

static uint
dq_used(struct inet_sock *is)
{
  return is->dhead - is->dtail;
}

// Take one datagram, whole. Returns the bytes copied, or -1 if none is queued.
//
// Whole is the point: read() on a UDP socket returns one datagram or nothing,
// never the tail of one followed by the head of the next. Before this queue
// existed the bytes went into the ring unmarked, so two datagrams that arrived
// together came back as one -- which is a bug that only appears when the
// timing is right, and is then blamed on the sender.
static int
dq_take(struct inet_sock *is, char *p, uint n, uint *addr, ushort *port)
{
  if (dq_used(is) == 0)
    return -1;

  struct dgram *d = &is->dq[is->dtail % NDGRAM];
  uint want = d->len;
  uint got  = want < n ? want : n;

  rx_take(is, p, got);
  rx_drop(is, want - got);

  if (addr) *addr = d->addr;
  if (port) *port = d->port;

  is->dtail++;
  return (int)got;
}

/* ------------------------------------------------------------------ */
/* lwIP callbacks. All of these run in the network thread, holding the */
/* lwIP lock.                                                          */
/* ------------------------------------------------------------------ */

static void
tcp_fail(struct inet_sock *is)
{
  is->peer_closed = 1;
  is->conn_failed = 1;
  is->connecting  = 0;
  // The pcb is already freed by lwIP when err_cb fires; keeping the pointer
  // would leave a dangling handle that close() would then free a second time.
  is->tpcb = 0;
  // A failure is as much of a wakeup as data: whoever is blocked in connect,
  // read or write is now waiting for something that will never arrive.
  net_wake();
}

static void
inet_err_cb(void *arg, err_t err)
{
  (void)err;
  if (arg)
    tcp_fail((struct inet_sock *)arg);
}

static err_t
inet_recv_cb(void *arg, struct tcp_pcb *pcb, struct pbuf *p, err_t err)
{
  struct inet_sock *is = arg;

  if (is == 0) {
    if (p) pbuf_free(p);
    return ERR_OK;
  }
  if (err != ERR_OK) {
    if (p) pbuf_free(p);
    return err;
  }
  // A null pbuf is the peer's FIN, not an error: the connection stays readable
  // until the buffer drains, and only then does read() report end of file.
  if (p == NULL) {
    is->peer_closed = 1;
    net_wake();
    return ERR_OK;
  }

  // Refusing what will not fit leaves it queued in lwIP. That cannot happen
  // while the window is reopened only as the reader drains the ring (see
  // inet_recvfrom) and the ring is larger than the window -- and it must not:
  // lwIP drops every new data segment while refused data is pending, so the
  // sender stalls until its retransmission timer, a second or more. That was
  // downloads at five kilobytes a second, when the ring was 8 KB, the window
  // 11 KB, and the window reopened the moment data entered the ring.
  if (p->tot_len > rx_space(is))
    return ERR_MEM;

  for (struct pbuf *q = p; q != NULL; q = q->next)
    rx_put(is, (const char *)q->payload, q->len);
  pbuf_free(p);
  net_wake();
  return ERR_OK;
}

// An acknowledgement freed space in the send buffer. Nothing reads `len`: the
// only waiter is a write that could not fit, and it re-reads tcp_sndbuf itself.
static err_t
inet_sent_cb(void *arg, struct tcp_pcb *pcb, u16_t len)
{
  (void)arg; (void)pcb; (void)len;
  net_wake();
  return ERR_OK;
}

static err_t
inet_connected_cb(void *arg, struct tcp_pcb *pcb, err_t err)
{
  struct inet_sock *is = arg;
  (void)pcb;

  if (is == 0)
    return ERR_OK;
  if (err != ERR_OK) {
    tcp_fail(is);
    return ERR_OK;
  }
  is->connecting = 0;
  is->connected  = 1;
  net_wake();
  return ERR_OK;
}

static struct inet_sock *inet_alloc(int type);

static err_t
inet_accept_cb(void *arg, struct tcp_pcb *newpcb, err_t err)
{
  struct inet_sock *ls = arg;

  if (ls == 0 || err != ERR_OK)
    return ERR_VAL;
  if (ls->naccept >= NBACKLOG)
    return ERR_MEM;            // backlog full: lwIP refuses the connection

  struct inet_sock *is = inet_alloc(SOCK_STREAM);
  if (is == 0)
    return ERR_MEM;

  is->tpcb      = newpcb;
  is->connected = 1;
  tcp_arg(newpcb, is);
  tcp_recv(newpcb, inet_recv_cb);
  tcp_err(newpcb, inet_err_cb);
  tcp_sent(newpcb, inet_sent_cb);

  ls->acceptq[ls->naccept++] = is;
  net_wake();
  return ERR_OK;
}

// UDP delivers datagrams with no connection, so each one carries its own
// sender. Both the sender and the length are recorded next to the bytes, which
// is what lets recvfrom answer "who sent this" and read() return exactly one
// datagram.
static void
inet_udp_recv_cb(void *arg, struct udp_pcb *pcb, struct pbuf *p,
                 const ip_addr_t *addr, u16_t port)
{
  struct inet_sock *is = arg;
  (void)pcb;

  if (is == 0 || p == NULL) {
    if (p) pbuf_free(p);
    return;
  }

  // Two ways to be full, and both mean the same thing for UDP: the datagram is
  // dropped. That is the protocol's own contract -- there is no window to
  // close and nothing to retransmit -- but it must be dropped *whole*, because
  // half a datagram in the ring would corrupt every one after it.
  if (p->tot_len <= rx_space(is) && dq_used(is) < NDGRAM) {
    struct dgram *d = &is->dq[is->dhead % NDGRAM];
    d->addr = addr ? ip4_addr_get_u32(ip_2_ip4(addr)) : 0;
    d->port = port;
    d->len  = p->tot_len;
    is->dhead++;

    for (struct pbuf *q = p; q != NULL; q = q->next)
      rx_put(is, (const char *)q->payload, q->len);
  }

  pbuf_free(p);
  net_wake();
}

// ICMP, for ping sockets.
//
// Linux calls these "ping sockets": SOCK_DGRAM with IPPROTO_ICMP, giving an
// unprivileged process the one thing raw sockets were used for. The contract is
// that the caller reads and writes the ICMP message alone, and the kernel deals
// with the IP header -- which is why this strips it on the way in, where
// lwIP's raw layer hands over a pbuf that starts at the IP header.
//
// **Only echo replies are eaten.** raw_input runs before lwIP's own
// icmp_input, so a callback that returned 1 for everything would swallow the
// echo *requests* too and this machine would silently stop answering pings
// from outside. Returning 0 leaves the packet for lwIP, which is exactly what
// should happen to every ICMP message we did not ask for.
//
// No matching against the echo id: a reply goes to every open ICMP socket.
// Linux rewrites the id per socket and filters on it, which matters when two
// processes ping at once; with one ping at a time the difference is invisible,
// and a socket that sees a reply to somebody else's request will read a
// sequence number it never sent and ignore it.
//
// A SOCK_RAW ICMP socket is the same socket, except that a read starts at the
// IP header: that is the raw-socket contract, and busybox's ping reads the TTL
// out of it and finds the ICMP message by IHL. Writing is the same for both --
// without IP_HDRINCL a raw socket's caller sends the ICMP message alone too.
static u8_t
inet_icmp_recv_cb(void *arg, struct raw_pcb *pcb, struct pbuf *p,
                  const ip_addr_t *addr)
{
  struct inet_sock *is = arg;
  (void)pcb;

  if (is == 0 || p == NULL)
    return 0;

  // The IP header is variable length: IHL counts 32-bit words.
  if (p->tot_len < sizeof(struct ip_hdr))
    return 0;
  struct ip_hdr *ih = (struct ip_hdr *)p->payload;
  u16_t hlen = (u16_t)(IPH_HL(ih) * 4);
  if (hlen < sizeof(struct ip_hdr) || p->tot_len < hlen + 8)
    return 0;

  // Type 0 is ICMP Echo Reply. Everything else -- including the requests this
  // host must keep answering -- is left for lwIP.
  u8_t type;
  if (pbuf_copy_partial(p, &type, 1, hlen) != 1 || type != 0)
    return 0;

  u16_t skip    = is->iphdr ? 0 : hlen;
  u16_t icmplen = (u16_t)(p->tot_len - skip);

  if (icmplen <= rx_space(is) && dq_used(is) < NDGRAM) {
    struct dgram *d = &is->dq[is->dhead % NDGRAM];
    d->addr = addr ? ip4_addr_get_u32(ip_2_ip4(addr)) : 0;
    d->port = 0;                       // ICMP has no ports
    d->len  = icmplen;
    is->dhead++;

    // Copy the message out, skipping the IP header unless this is a raw
    // socket, across whatever pbuf chain it arrived in.
    u16_t off = skip;
    while (off < p->tot_len) {
      char tmp[64];
      u16_t want = (u16_t)(p->tot_len - off);
      if (want > sizeof(tmp)) want = sizeof(tmp);
      if (pbuf_copy_partial(p, tmp, want, off) != want)
        break;
      rx_put(is, tmp, want);
      off = (u16_t)(off + want);
    }
  }

  pbuf_free(p);
  net_wake();
  return 1;                            // eaten
}

/* ------------------------------------------------------------------ */
/* Allocation                                                          */
/* ------------------------------------------------------------------ */

static struct inet_sock *
inet_alloc(int type)
{
  for (int i = 0; i < NINET; i++)
    if (!itab[i].inuse) {
      struct inet_sock *is = &itab[i];
      memset(is, 0, sizeof(*is));
      is->inuse = 1;
      is->type  = type;
      return is;
    }
  return 0;
}

/* ------------------------------------------------------------------ */
/* The interface kernel/net/unix.c dispatches to                       */
/* ------------------------------------------------------------------ */

void *
inet_new(int type, int proto)
{
  if (type != SOCK_STREAM && type != SOCK_DGRAM && type != SOCK_RAW)
    return 0;
  if (type == SOCK_RAW && proto != IPPROTO_ICMP)
    return 0;
  // Past this point a raw ICMP socket is a ping socket with one flag: every
  // datagram path below already does the right thing for it.
  int iphdr = (type == SOCK_RAW);
  if (iphdr)
    type = SOCK_DGRAM;
  // Ready, not *up*: see net_is_ready() in kernel/net/net.c. Requiring an
  // address here made every socket() fail until DHCP answered.
  if (!net_is_ready())
    return 0;

  lwip_sys_lock();
  struct inet_sock *is = inet_alloc(type);
  if (is == 0) {
    lwip_sys_unlock();
    return 0;
  }
  is->proto = proto;
  is->iphdr = iphdr;

  if (proto == IPPROTO_ICMP) {
    // A ping socket. raw_new registers for one IP protocol number; the
    // callback below filters to echo replies so lwIP keeps answering the
    // requests this host receives. See inet_icmp_recv_cb.
    is->rpcb = raw_new(IP_PROTO_ICMP);
    if (is->rpcb == 0) {
      is->inuse = 0;
      lwip_sys_unlock();
      return 0;
    }
    raw_recv(is->rpcb, inet_icmp_recv_cb, is);
    raw_bind(is->rpcb, IP_ANY_TYPE);
  } else if (type == SOCK_STREAM) {
    is->tpcb = tcp_new();
    if (is->tpcb == 0) {
      is->inuse = 0;
      lwip_sys_unlock();
      return 0;
    }
    tcp_arg(is->tpcb, is);
    tcp_err(is->tpcb, inet_err_cb);
    tcp_recv(is->tpcb, inet_recv_cb);
    tcp_sent(is->tpcb, inet_sent_cb);
  } else {
    is->upcb = udp_new();
    if (is->upcb == 0) {
      is->inuse = 0;
      lwip_sys_unlock();
      return 0;
    }
    udp_recv(is->upcb, inet_udp_recv_cb, is);
  }

  lwip_sys_unlock();
  return is;
}

int
inet_bind(void *h, uint32 addr, uint16 port)
{
  struct inet_sock *is = h;
  ip_addr_t ip;
  err_t e;

  ip_addr_set_ip4_u32(&ip, addr);      // already network byte order

  lwip_sys_lock();
  e = (is->type == SOCK_STREAM) ? tcp_bind(is->tpcb, &ip, port)
                                : udp_bind(is->upcb, &ip, port);
  lwip_sys_unlock();

  if (e == ERR_USE)
    return -EADDRINUSE;
  return e == ERR_OK ? 0 : -EINVAL;
}

// Apply a socket option to the lwIP pcb behind this handle.
//
// Only the four that lwIP can genuinely honour. Everything else is refused by
// the caller (kernel/sys/sys_net.c) before it reaches here, so this never has
// to pretend.
//
// Called with the option already recorded on the struct socket, so a flag set
// before bind() is not lost: lwIP's pcb exists from inet_new() onwards, which
// is early enough for all four, but recording it there as well means
// getsockopt has one answer and not two.
int
inet_setopt(void *h, int level, int name, int val)
{
  struct inet_sock *is = h;

  lwip_sys_lock();
  if (level == SOL_SOCKET) {
    struct ip_pcb *p = (is->type == SOCK_STREAM) ? (struct ip_pcb *)is->tpcb
                                                 : (struct ip_pcb *)is->upcb;
    if (p == 0) { lwip_sys_unlock(); return -EINVAL; }
    switch (name) {
    case SO_REUSEADDR:
      if (val) ip_set_option(p, SOF_REUSEADDR);
      else     ip_reset_option(p, SOF_REUSEADDR);
      break;
    case SO_KEEPALIVE:
      if (val) ip_set_option(p, SOF_KEEPALIVE);
      else     ip_reset_option(p, SOF_KEEPALIVE);
      break;
    case SO_BROADCAST:
      if (val) ip_set_option(p, SOF_BROADCAST);
      else     ip_reset_option(p, SOF_BROADCAST);
      break;
    default:
      lwip_sys_unlock();
      return -ENOPROTOOPT;
    }
  } else if (level == IPPROTO_TCP && name == TCP_NODELAY) {
    if (is->type != SOCK_STREAM || is->tpcb == 0) {
      lwip_sys_unlock();
      return -EINVAL;
    }
    if (val) tcp_nagle_disable(is->tpcb);
    else     tcp_nagle_enable(is->tpcb);
  } else {
    lwip_sys_unlock();
    return -ENOPROTOOPT;
  }
  lwip_sys_unlock();
  return 0;
}

int
inet_listen(void *h, int backlog)
{
  struct inet_sock *is = h;

  if (is->type != SOCK_STREAM)
    return -EOPNOTSUPP;

  lwip_sys_lock();
  // tcp_listen frees the old pcb and returns a smaller listening one, so the
  // handle has to be replaced rather than kept. Losing this is a use-after-free
  // that only shows up on the second connection.
  struct tcp_pcb *lp = tcp_listen_with_backlog(is->tpcb,
                          backlog > 0 && backlog < NBACKLOG ? (u8_t)backlog
                                                            : NBACKLOG);
  if (lp == 0) {
    lwip_sys_unlock();
    return -ENOMEM;
  }
  is->tpcb      = lp;
  is->listening = 1;
  tcp_arg(lp, is);
  tcp_accept(lp, inet_accept_cb);
  lwip_sys_unlock();
  return 0;
}

void *
inet_accept(void *h)
{
  struct inet_sock *ls = h;

  if (!ls->listening)
    return 0;

  for (;;) {
    lwip_sys_lock();
    if (ls->naccept > 0) {
      struct inet_sock *is = ls->acceptq[0];
      for (int i = 1; i < ls->naccept; i++)
        ls->acceptq[i - 1] = ls->acceptq[i];
      ls->naccept--;
      lwip_sys_unlock();
      return is;
    }
    // Taken while the lwIP lock is still held, so no connection can arrive
    // between the check above and the sleep below without moving it.
    uint seen = net_seq();
    lwip_sys_unlock();

    if (net_interrupted())
      return 0;
    net_wait(seen);
  }
}

int
inet_connect(void *h, uint32 addr, uint16 port)
{
  struct inet_sock *is = h;
  ip_addr_t ip;

  ip_addr_set_ip4_u32(&ip, addr);

  if (is->type == SOCK_DGRAM) {
    // A connected UDP socket just remembers where to send; there is no
    // handshake to wait for.
    lwip_sys_lock();
    err_t e = udp_connect(is->upcb, &ip, port);
    lwip_sys_unlock();
    return e == ERR_OK ? 0 : -EINVAL;
  }

  lwip_sys_lock();
  is->connecting = 1;
  err_t e = tcp_connect(is->tpcb, &ip, port, inet_connected_cb);
  lwip_sys_unlock();
  if (e != ERR_OK) {
    is->connecting = 0;
    return -ECONNREFUSED;
  }

  for (;;) {
    lwip_sys_lock();
    int  done = is->connected, failed = is->conn_failed;
    uint seen = net_seq();
    lwip_sys_unlock();
    if (done)
      return 0;
    if (failed)
      return -ECONNREFUSED;
    if (net_interrupted())
      return -EINTR;
    net_wait(seen);
  }
}

// Read, and say where it came from.
//
// `addr` and `port` may be null, which is what read() and recv() pass: the
// source is then simply not reported. For a stream socket the answer is the
// peer it is connected to and is the same for every byte; for a datagram
// socket it belongs to the individual datagram, which is why UDP needs the
// queue above and TCP does not.
// getsockname / getpeername: this end's address or the other's, the address
// in network order and the port in host order, as inet_recvfrom gives them. A
// socket not yet bound reads as 0.0.0.0:0 locally, which is what Linux says;
// one with no peer is ENOTCONN. A listening pcb has no remote fields at all
// -- lwIP's listen pcb is the smaller struct -- so it is never asked for them.
int
inet_name(void *h, int peer, uint *addr, ushort *port)
{
  struct inet_sock *is = h;
  int r = 0;

  *addr = 0;
  *port = 0;
  lwip_sys_lock();
  if (is->tpcb) {
    if (peer && (is->listening || is->tpcb->state < SYN_SENT)) {
      r = -ENOTCONN;
    } else if (peer) {
      *addr = ip4_addr_get_u32(ip_2_ip4(&is->tpcb->remote_ip));
      *port = is->tpcb->remote_port;
    } else {
      *addr = ip4_addr_get_u32(ip_2_ip4(&is->tpcb->local_ip));
      *port = is->tpcb->local_port;
    }
  } else if (is->upcb) {
    if (peer && !(is->upcb->flags & UDP_FLAGS_CONNECTED)) {
      r = -ENOTCONN;
    } else if (peer) {
      *addr = ip4_addr_get_u32(ip_2_ip4(&is->upcb->remote_ip));
      *port = is->upcb->remote_port;
    } else {
      *addr = ip4_addr_get_u32(ip_2_ip4(&is->upcb->local_ip));
      *port = is->upcb->local_port;
    }
  } else if (peer) {
    r = -ENOTCONN;                     // a ping socket has no peer
  }
  lwip_sys_unlock();
  return r;
}

int
inet_recvfrom(void *h, char *buf, int n, uint *addr, ushort *port)
{
  struct inet_sock *is = h;

  if (n <= 0)
    return 0;

  for (;;) {
    int  got  = 0;
    int  have = 0;      // something was delivered, even if it was zero bytes
    int  eof;
    uint seen;

    lwip_sys_lock();
    if (is->type == SOCK_DGRAM) {
      // A zero-length datagram is a real thing a sender can send, and the
      // right answer to it is 0 rather than "nothing arrived yet" -- which is
      // why dq_take reports an empty queue as -1 and not as zero bytes.
      int r = dq_take(is, buf, (uint)n, addr, port);
      have  = r >= 0;
      got   = r > 0 ? r : 0;
    } else {
      got  = (int)rx_take(is, buf, (uint)n);
      have = got > 0;
      // The bytes have left the ring, so the window may reopen by as much.
      // In pieces: the length is sixteen bits, and one read can take 64 KB.
      for (int left = got; left > 0 && is->tpcb; left -= 0x8000)
        tcp_recved(is->tpcb, (u16_t)(left > 0x8000 ? 0x8000 : left));
      // A stream's source is the peer it is connected to: the same answer for
      // every byte, which is why it is read from the pcb rather than stored.
      if (have && addr && is->tpcb) {
        *addr = ip4_addr_get_u32(ip_2_ip4(&is->tpcb->remote_ip));
        if (port)
          *port = is->tpcb->remote_port;
      }
    }
    eof  = is->peer_closed;
    seen = net_seq();
    lwip_sys_unlock();

    if (have)
      return got;
    if (eof)
      return 0;                       // end of file, not an error
    if (net_interrupted())
      return -EINTR;
    net_wait(seen);
  }
}

int
inet_read(void *h, char *buf, int n)
{
  return inet_recvfrom(h, buf, n, 0, 0);
}

// Write, optionally to a named destination.
//
// `addr` of zero means "wherever this socket is already pointed", which is what
// write() and send() pass. For TCP that is the only possibility and a
// destination is ignored rather than refused: the connection decides where the
// bytes go, and a program that passes one anyway -- many do, out of habit -- is
// better served by having it work.
int
inet_sendto(void *h, char *buf, int n, uint addr, ushort port)
{
  struct inet_sock *is = h;
  int done = 0;

  if (n <= 0)
    return 0;

  // ICMP. The caller hands over the ICMP message alone -- header, checksum and
  // payload -- and lwIP puts the IP header on the front, which is what makes
  // this a ping socket rather than a raw one.
  //
  // PBUF_IP rather than PBUF_TRANSPORT for the headroom: the IP header is what
  // goes in front here, and asking for transport headroom would have lwIP
  // reallocate to make room for it.
  if (is->proto == IPPROTO_ICMP) {
    if (is->rpcb == 0)
      return -ENOTCONN;
    if (addr == 0)
      return -EDESTADDRREQ;

    lwip_sys_lock();
    struct pbuf *p = pbuf_alloc(PBUF_IP, (u16_t)n, PBUF_RAM);
    if (p == 0) {
      lwip_sys_unlock();
      return -ENOBUFS;
    }
    pbuf_take(p, buf, (u16_t)n);

    ip_addr_t ip;
    ip_addr_set_ip4_u32(&ip, addr);
    err_t e = raw_sendto(is->rpcb, p, &ip);

    pbuf_free(p);
    lwip_sys_unlock();
    if (e == ERR_OK)
      return n;
    return e == ERR_RTE ? -ENETUNREACH : -EIO;
  }

  if (is->type == SOCK_DGRAM) {
    lwip_sys_lock();
    struct pbuf *p = pbuf_alloc(PBUF_TRANSPORT, (u16_t)n, PBUF_RAM);
    if (p == 0) {
      lwip_sys_unlock();
      return -ENOBUFS;
    }
    pbuf_take(p, buf, (u16_t)n);

    err_t e;
    if (addr) {
      ip_addr_t ip;
      ip_addr_set_ip4_u32(&ip, addr);
      e = udp_sendto(is->upcb, p, &ip, port);
    } else {
      // No destination and none remembered is a program error, not a network
      // one: lwIP would send to 0.0.0.0 and the datagram would vanish.
      e = udp_send(is->upcb, p);
    }

    pbuf_free(p);
    lwip_sys_unlock();
    if (e == ERR_OK)
      return n;
    return e == ERR_RTE ? -EDESTADDRREQ : -EIO;
  }

  // TCP: the send buffer is finite, so a large write becomes several passes.
  while (done < n) {
    uint seen;

    lwip_sys_lock();
    if (is->peer_closed || is->wr_shut || is->tpcb == 0) {
      lwip_sys_unlock();
      // Same contract as a pipe with no reader, which is what the AF_UNIX side
      // does: the signal first, then the error.
      return done > 0 ? done : -EPIPE;
    }

    int room = tcp_sndbuf(is->tpcb);
    int want = n - done;
    if (want > room)
      want = room;

    if (want > 0) {
      // TCP_WRITE_FLAG_COPY because `buf` is a kernel bounce buffer the caller
      // reuses the moment this returns; lwIP would otherwise hold a pointer
      // into memory that has changed by the time the segment goes out.
      err_t e = tcp_write(is->tpcb, buf + done, (u16_t)want,
                          TCP_WRITE_FLAG_COPY);
      if (e == ERR_OK) {
        tcp_output(is->tpcb);
        done += want;
      } else if (e != ERR_MEM) {
        lwip_sys_unlock();
        return done > 0 ? done : -EIO;
      }
    }
    seen = net_seq();
    lwip_sys_unlock();

    if (done < n) {
      if (net_interrupted())
        return done > 0 ? done : -EINTR;
      // Woken by inet_sent_cb when an acknowledgement frees send buffer, or by
      // a failure. Without tcp_sent registered this would be a sleep with
      // nobody left to end it, which is why that callback exists.
      net_wait(seen);
    }
  }
  return done;
}

int
inet_write(void *h, char *buf, int n)
{
  return inet_sendto(h, buf, n, 0, 0);
}

// shutdown(2) on a TCP socket. SHUT_WR sends our FIN and the peer can keep
// sending; SHUT_RD is local, reads see EOF. The pcb is never freed here --
// tcp_shutdown(pcb, 1, 1) would free it -- so close() still owns it.
int
inet_shutdown(void *h, int how)
{
  struct inet_sock *is = h;

  if (is->type != SOCK_STREAM)
    return 0;
  lwip_sys_lock();
  if (is->tpcb == 0 || is->listening || !is->connected) {
    lwip_sys_unlock();
    return -ENOTCONN;
  }
  if ((how == SHUT_WR || how == SHUT_RDWR) && !is->wr_shut) {
    is->wr_shut = 1;
    tcp_shutdown(is->tpcb, 0, 1);
  }
  if (how == SHUT_RD || how == SHUT_RDWR)
    is->peer_closed = 1;
  lwip_sys_unlock();
  net_wake();
  return 0;
}

int
inet_readable(void *h)
{
  struct inet_sock *is = h;
  int r;

  lwip_sys_lock();
  r = rx_used(is) > 0 || is->peer_closed || is->naccept > 0;
  lwip_sys_unlock();
  return r;
}

int
inet_hup(void *h)
{
  struct inet_sock *is = h;
  int r;

  lwip_sys_lock();
  r = is->peer_closed && rx_used(is) == 0;
  lwip_sys_unlock();
  return r;
}

void
inet_close(void *h)
{
  struct inet_sock *is = h;

  if (is == 0)
    return;

  lwip_sys_lock();
  // Anything queued but never accepted is closed too, or those connections
  // would sit half-open until the peer gave up on them.
  for (int i = 0; i < is->naccept; i++) {
    struct inet_sock *q = is->acceptq[i];
    if (q->tpcb) {
      tcp_arg(q->tpcb, 0);
      tcp_close(q->tpcb);
    }
    q->inuse = 0;
  }
  is->naccept = 0;

  if (is->tpcb) {
    // Clearing the argument first: lwIP may still deliver a callback for this
    // pcb after tcp_close, and it would otherwise arrive with a pointer to a
    // socket that has just been freed. `callback_arg` is in the part of the
    // structure both kinds of pcb share, so this one is safe either way.
    tcp_arg(is->tpcb, 0);

    // **A listening pcb is a different, smaller structure.** tcp_listen()
    // frees the connection pcb and returns a `struct tcp_pcb_listen`, which
    // has no recv, sent, err or poll field at all -- so tcp_recv() on one does
    // not merely set something useless, it writes past the end of the
    // allocation. lwIP asserts rather than allowing that, and the assert is a
    // kernel panic here:
    //
    //   lwip assert: invalid socket state for recv callback  tcp.c:2024
    //   #1 tcp_recv  #2 inet_close  #3 sockclose  #4 fileclose  #5 exit
    //
    // which is a user process taking the machine down by pressing Ctrl-C.
    // Nothing had ever closed a listening TCP socket before -- the first
    // program to listen and then exit was cmd/httpd, and it found this on its
    // first run.
    //
    // The callback a listening pcb *does* have is `accept`, and clearing that
    // is the equivalent gesture.
    if (is->listening) {
      tcp_accept(is->tpcb, 0);
      // Always ERR_OK for a LISTEN pcb, and tcp_abort() would be wrong on one:
      // it reaches for connection state that a listening pcb does not have.
      tcp_close(is->tpcb);
    } else {
      tcp_recv(is->tpcb, 0);
      tcp_err(is->tpcb, 0);
      tcp_sent(is->tpcb, 0);
      if (tcp_close(is->tpcb) != ERR_OK)
        tcp_abort(is->tpcb);
    }
    is->tpcb = 0;
    is->listening = 0;
  }
  if (is->upcb) {
    udp_recv(is->upcb, 0, 0);
    udp_remove(is->upcb);
    is->upcb = 0;
  }
  if (is->rpcb) {
    raw_recv(is->rpcb, 0, 0);
    raw_remove(is->rpcb);
    is->rpcb = 0;
  }
  is->inuse = 0;
  lwip_sys_unlock();
}