diff --git a/kernel/drivers/net/e1000/e1000.h b/kernel/drivers/net/e1000/e1000.h index 281826f..16c5fe8 100644 --- a/kernel/drivers/net/e1000/e1000.h +++ b/kernel/drivers/net/e1000/e1000.h @@ -192,6 +192,12 @@ struct e1000_rx_desc { // Ring sizes. The manual requires the descriptor ring length to be a multiple // of 128 bytes, i.e. a multiple of 8 descriptors. -#define E1000_NTX 16 -#define E1000_NRX 16 +// +// The receive ring has to hold a whole TCP window arriving at once, and more: +// a sender that has the window open sends it back to back, and every frame +// past the ring's end is dropped and waits for a retransmission. Sixteen was +// fine for a window of eight segments; with thirty-two it held downloads to a +// megabit a second. 256 buffers is half a megabyte of the 64 MB DMA arena. +#define E1000_NTX 64 +#define E1000_NRX 256 #define E1000_BUFSZ 2048 diff --git a/kernel/net/inet.c b/kernel/net/inet.c index d4450fc..47426ab 100644 --- a/kernel/net/inet.c +++ b/kernel/net/inet.c @@ -69,7 +69,13 @@ #include "lwip/pbuf.h" #define NINET 16 // AF_INET sockets in the system -#define INET_RXBUF 8192 // per socket; one lwIP window plus slack +// Per socket. More than TCP_WND, which it must be (see inet_recv_cb), and a +// power of two, which it must also be: rhead and rtail run freely and are +// reduced modulo this, which stays continuous across their wrap at 2^32 only +// when it divides 2^32. +#define INET_RXBUF (512 * 1024) +_Static_assert(INET_RXBUF > TCP_WND, "the ring must hold a whole window"); +_Static_assert((INET_RXBUF & (INET_RXBUF - 1)) == 0, "the ring must be a power of two"); #define NDGRAM 16 // datagrams queued on a UDP socket // Where one datagram ends and the next begins, and who sent it. @@ -182,11 +188,16 @@ rx_space(struct inet_sock *is) return INET_RXBUF - rx_used(is); } +// Two copies at most, one each side of the ring's end. A byte at a time with a +// modulo each was a measurable share of a fast download. static void rx_put(struct inet_sock *is, const char *p, uint n) { - for (uint i = 0; i < n; i++) - is->rx[(is->rhead + i) % INET_RXBUF] = p[i]; + uint at = is->rhead % INET_RXBUF, first = INET_RXBUF - at; + if (first > n) + first = n; + memmove(is->rx + at, p, first); + memmove(is->rx, p + first, n - first); is->rhead += n; } @@ -196,8 +207,11 @@ rx_take(struct inet_sock *is, char *p, uint n) uint have = rx_used(is); if (n > have) n = have; - for (uint i = 0; i < n; i++) - p[i] = is->rx[(is->rtail + i) % INET_RXBUF]; + uint at = is->rtail % INET_RXBUF, first = INET_RXBUF - at; + if (first > n) + first = n; + memmove(p, is->rx + at, first); + memmove(p + first, is->rx, n - first); is->rtail += n; return n; } @@ -297,17 +311,18 @@ inet_recv_cb(void *arg, struct tcp_pcb *pcb, struct pbuf *p, err_t err) return ERR_OK; } - // Refusing what will not fit leaves it queued in lwIP, which closes the - // window and makes the sender wait -- which is the whole point of a window. - // Accepting and dropping would look like a working receive that loses data. + // Refusing what will not fit leaves it queued in lwIP. That cannot happen + // while the window is reopened only as the reader drains the ring (see + // inet_recvfrom) and the ring is larger than the window -- and it must not: + // lwIP drops every new data segment while refused data is pending, so the + // sender stalls until its retransmission timer, a second or more. That was + // downloads at five kilobytes a second, when the ring was 8 KB, the window + // 11 KB, and the window reopened the moment data entered the ring. if (p->tot_len > rx_space(is)) return ERR_MEM; for (struct pbuf *q = p; q != NULL; q = q->next) rx_put(is, (const char *)q->payload, q->len); - - // Only now is the data ours, so only now may the window reopen. - tcp_recved(pcb, p->tot_len); pbuf_free(p); net_wake(); return ERR_OK; @@ -752,6 +767,10 @@ inet_recvfrom(void *h, char *buf, int n, uint *addr, ushort *port) } else { got = (int)rx_take(is, buf, (uint)n); have = got > 0; + // The bytes have left the ring, so the window may reopen by as much. + // In pieces: the length is sixteen bits, and one read can take 64 KB. + for (int left = got; left > 0 && is->tpcb; left -= 0x8000) + tcp_recved(is->tpcb, (u16_t)(left > 0x8000 ? 0x8000 : left)); // A stream's source is the peer it is connected to: the same answer for // every byte, which is why it is read from the pcb rather than stored. if (have && addr && is->tpcb) { diff --git a/kernel/net/lwip/lwipopts.h b/kernel/net/lwip/lwipopts.h index 3e98f7f..4e4fab3 100644 --- a/kernel/net/lwip/lwipopts.h +++ b/kernel/net/lwip/lwipopts.h @@ -35,10 +35,12 @@ #define MEMP_NUM_UDP_PCB 8 #define MEMP_NUM_TCP_PCB 16 #define MEMP_NUM_TCP_PCB_LISTEN 8 -#define MEMP_NUM_TCP_SEG 64 +#define MEMP_NUM_TCP_SEG 256 // a window held out of order #define MEMP_NUM_ARP_QUEUE 16 #define MEMP_NUM_SYS_TIMEOUT 16 -#define PBUF_POOL_SIZE 48 +// Enough for a full receive window held out of order, plus the frames in +// flight from the driver: a window of TCP_WND below, in pool-sized pieces. +#define PBUF_POOL_SIZE 256 #define PBUF_POOL_BUFSIZE 1536 // one Ethernet frame plus room // Protocols. IPv4 only: the driver is proven for it, the QEMU test network is @@ -77,7 +79,17 @@ #define TCP_MSS 1460 #define TCP_SND_BUF (8 * TCP_MSS) -#define TCP_WND (8 * TCP_MSS) +// The receive window, and with it the download speed: one window per round +// trip. Eight segments held a transfer from a CDN to well under a megabyte a +// second; thirty-two, the most that fits in TCP's 16-bit window field, held +// it to 1.5 MB/s at 25 ms -- an eighth of a 100 Mbit line, measured against +// the host fetching the same file. Window scaling (RFC 7323) lifts the 64 KB +// ceiling: the field is shifted by TCP_RCV_SCALE, so 256 KB is advertised as +// 16384 << 4. Every socket's buffer in kernel/net/inet.c is larger than this, +// which is what makes the window an honest promise -- see inet_recv_cb. +#define LWIP_WND_SCALE 1 +#define TCP_RCV_SCALE 4 +#define TCP_WND (256 * 1024) #define TCP_SND_QUEUELEN ((4 * TCP_SND_BUF) / TCP_MSS) #define TCP_LISTEN_BACKLOG 1