net: downloads at tens of megabits, not one

Measured against the host fetching the same 67 MB from Alpine's CDN at
11.5 MB/s, SchOS managed about 120 KB/s. Three causes:

- The receive window reopened when data entered the socket buffer,
  before the reader drained it; lwIP then refused segments and the
  sender waited on retransmission timers. It reopens on read now.
- The e1000 receive ring held 16 frames, so a full window arriving back
  to back lost everything past the sixteenth. 256 now.
- Without window scaling the window stops at 64 KB, one window per
  round trip. Scaling on, 256 KB window, 512 KB socket ring (a power of
  two, which the free-running indices need), copied in blocks.

Now 5.8 MB/s for the same file, and apk update in about two seconds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4

kernel/net/inet.c

@@ -69,7 +69,13 @@
 #include "lwip/pbuf.h"
 
 #define NINET      16        // AF_INET sockets in the system
-#define INET_RXBUF 8192      // per socket; one lwIP window plus slack
+// Per socket. More than TCP_WND, which it must be (see inet_recv_cb), and a
+// power of two, which it must also be: rhead and rtail run freely and are
+// reduced modulo this, which stays continuous across their wrap at 2^32 only
+// when it divides 2^32.
+#define INET_RXBUF (512 * 1024)
+_Static_assert(INET_RXBUF > TCP_WND, "the ring must hold a whole window");
+_Static_assert((INET_RXBUF & (INET_RXBUF - 1)) == 0, "the ring must be a power of two");
 #define NDGRAM     16        // datagrams queued on a UDP socket
 
 // Where one datagram ends and the next begins, and who sent it.
@@ -182,11 +188,16 @@ rx_space(struct inet_sock *is)
   return INET_RXBUF - rx_used(is);
 }
 
+// Two copies at most, one each side of the ring's end. A byte at a time with a
+// modulo each was a measurable share of a fast download.
 static void
 rx_put(struct inet_sock *is, const char *p, uint n)
 {
-  for (uint i = 0; i < n; i++)
-    is->rx[(is->rhead + i) % INET_RXBUF] = p[i];
+  uint at = is->rhead % INET_RXBUF, first = INET_RXBUF - at;
+  if (first > n)
+    first = n;
+  memmove(is->rx + at, p, first);
+  memmove(is->rx, p + first, n - first);
   is->rhead += n;
 }
 
@@ -196,8 +207,11 @@ rx_take(struct inet_sock *is, char *p, uint n)
   uint have = rx_used(is);
   if (n > have)
     n = have;
-  for (uint i = 0; i < n; i++)
-    p[i] = is->rx[(is->rtail + i) % INET_RXBUF];
+  uint at = is->rtail % INET_RXBUF, first = INET_RXBUF - at;
+  if (first > n)
+    first = n;
+  memmove(p, is->rx + at, first);
+  memmove(p + first, is->rx, n - first);
   is->rtail += n;
   return n;
 }
@@ -297,17 +311,18 @@ inet_recv_cb(void *arg, struct tcp_pcb *pcb, struct pbuf *p, err_t err)
     return ERR_OK;
   }
 
-  // Refusing what will not fit leaves it queued in lwIP, which closes the
-  // window and makes the sender wait -- which is the whole point of a window.
-  // Accepting and dropping would look like a working receive that loses data.
+  // Refusing what will not fit leaves it queued in lwIP. That cannot happen
+  // while the window is reopened only as the reader drains the ring (see
+  // inet_recvfrom) and the ring is larger than the window -- and it must not:
+  // lwIP drops every new data segment while refused data is pending, so the
+  // sender stalls until its retransmission timer, a second or more. That was
+  // downloads at five kilobytes a second, when the ring was 8 KB, the window
+  // 11 KB, and the window reopened the moment data entered the ring.
   if (p->tot_len > rx_space(is))
     return ERR_MEM;
 
   for (struct pbuf *q = p; q != NULL; q = q->next)
     rx_put(is, (const char *)q->payload, q->len);
-
-  // Only now is the data ours, so only now may the window reopen.
-  tcp_recved(pcb, p->tot_len);
   pbuf_free(p);
   net_wake();
   return ERR_OK;
@@ -752,6 +767,10 @@ inet_recvfrom(void *h, char *buf, int n, uint *addr, ushort *port)
     } else {
       got  = (int)rx_take(is, buf, (uint)n);
       have = got > 0;
+      // The bytes have left the ring, so the window may reopen by as much.
+      // In pieces: the length is sixteen bits, and one read can take 64 KB.
+      for (int left = got; left > 0 && is->tpcb; left -= 0x8000)
+        tcp_recved(is->tpcb, (u16_t)(left > 0x8000 ? 0x8000 : left));
       // A stream's source is the peer it is connected to: the same answer for
       // every byte, which is why it is read from the pcb rather than stored.
       if (have && addr && is->tpcb) {

kernel/net/lwip/lwipopts.h

@@ -35,10 +35,12 @@
 #define MEMP_NUM_UDP_PCB            8
 #define MEMP_NUM_TCP_PCB            16
 #define MEMP_NUM_TCP_PCB_LISTEN     8
-#define MEMP_NUM_TCP_SEG            64
+#define MEMP_NUM_TCP_SEG            256   // a window held out of order
 #define MEMP_NUM_ARP_QUEUE          16
 #define MEMP_NUM_SYS_TIMEOUT        16
-#define PBUF_POOL_SIZE              48
+// Enough for a full receive window held out of order, plus the frames in
+// flight from the driver: a window of TCP_WND below, in pool-sized pieces.
+#define PBUF_POOL_SIZE              256
 #define PBUF_POOL_BUFSIZE           1536   // one Ethernet frame plus room
 
 // Protocols. IPv4 only: the driver is proven for it, the QEMU test network is
@@ -77,7 +79,17 @@
 
 #define TCP_MSS                     1460
 #define TCP_SND_BUF                 (8 * TCP_MSS)
-#define TCP_WND                     (8 * TCP_MSS)
+// The receive window, and with it the download speed: one window per round
+// trip. Eight segments held a transfer from a CDN to well under a megabyte a
+// second; thirty-two, the most that fits in TCP's 16-bit window field, held
+// it to 1.5 MB/s at 25 ms -- an eighth of a 100 Mbit line, measured against
+// the host fetching the same file. Window scaling (RFC 7323) lifts the 64 KB
+// ceiling: the field is shifted by TCP_RCV_SCALE, so 256 KB is advertised as
+// 16384 << 4. Every socket's buffer in kernel/net/inet.c is larger than this,
+// which is what makes the window an honest promise -- see inet_recv_cb.
+#define LWIP_WND_SCALE              1
+#define TCP_RCV_SCALE               4
+#define TCP_WND                     (256 * 1024)
 #define TCP_SND_QUEUELEN            ((4 * TCP_SND_BUF) / TCP_MSS)
 #define TCP_LISTEN_BACKLOG          1