SchOS/kernel/drivers/net/e1000/e1000.c

// Intel 8254x gigabit Ethernet, written from Intel's public datasheet (317453).
//
// The first network device. There is no IP stack yet, so what this provides is
// the layer under one: a MAC address, a transmit path, and a receive ring that
// fills with frames. That is deliberately the right order -- an IP stack
// written before a driver has nothing to be tested against, and every bug in it
// looks like a driver bug.
//
// Bring-up follows the manual's initialisation sequence (section 14.3-14.5):
// reset, read the MAC out of EEPROM, clear the multicast table, set up the
// receive and transmit rings, then enable each. The order matters -- enabling a
// receiver whose ring base is still zero makes the device DMA into physical
// address 0, which on this kernel is the boot page table.

#include "types.h"
#include "defs.h"
#include "param.h"
#include "x86.h"
#include "spinlock.h"
#include "sleeplock.h"
#include "fs.h"
#include "file.h"
#include "mmu.h"
#include "memlayout.h"
#include "pci.h"
#include "device.h"
#include "dma.h"
#include "irq.h"
#include "e1000.h"

struct e1000 {
  volatile uint8 *regs;

  struct e1000_tx_desc *tx;
  struct e1000_rx_desc *rx;
  uint8 *txbuf[E1000_NTX];
  uint8 *rxbuf[E1000_NRX];
  uint16 tx_next;
  uint16 rx_next;

  uint8 mac[6];
  int   ready;
  uint64 rx_packets, tx_packets, rx_dropped;

  struct spinlock lock;
};

static struct e1000 nic;

static inline uint32 rd(struct e1000 *e, int off) { return *(volatile uint32 *)(e->regs + off); }
static inline void   wr(struct e1000 *e, int off, uint32 v) { *(volatile uint32 *)(e->regs + off) = v; }

// The EEPROM is read one 16-bit word at a time through a single register: write
// the address with the start bit, poll for done, take the data out of the top
// half.
//
// Bounded, because a device whose EEPROM never reports done would otherwise
// hang the boot -- and QEMU's e1000 does answer, but a real card with a blank
// EEPROM is a thing that exists.
static int
eeprom_read(struct e1000 *e, int addr, uint16 *out)
{
  wr(e, E1000_EERD, (addr << E1000_EERD_ADDR_SHIFT) | E1000_EERD_START);
  for (int i = 0; i < 100000; i++) {
    uint32 v = rd(e, E1000_EERD);
    if (v & E1000_EERD_DONE) {
      *out = (uint16)(v >> E1000_EERD_DATA_SHIFT);
      return 0;
    }
  }
  return -1;
}

// The MAC address, from EEPROM words 0..2, little-endian within each word.
//
// Falls back to the receive-address registers, which the firmware may already
// have programmed. QEMU fills both; some cards leave the EEPROM path
// unimplemented, and a driver that only knows one of the two ends up with a MAC
// of all zeroes and no idea why nothing answers it.
static void
read_mac(struct e1000 *e)
{
  uint16 w;
  int ok = 1;

  for (int i = 0; i < 3; i++) {
    if (eeprom_read(e, i, &w) != 0) { ok = 0; break; }
    e->mac[i * 2]     = (uint8)(w & 0xFF);
    e->mac[i * 2 + 1] = (uint8)(w >> 8);
  }
  if (ok)
    return;

  uint32 lo = rd(e, E1000_RAL), hi = rd(e, E1000_RAH);
  e->mac[0] = (uint8)lo; e->mac[1] = (uint8)(lo >> 8);
  e->mac[2] = (uint8)(lo >> 16); e->mac[3] = (uint8)(lo >> 24);
  e->mac[4] = (uint8)hi; e->mac[5] = (uint8)(hi >> 8);
}

static int
rings_setup(struct e1000 *e)
{
  // One block for both rings and every buffer, from the DMA arena: the device
  // reads all of it by physical address, and kalloc's pages are contiguous only
  // by accident.
  uint32 txring = E1000_NTX * sizeof(struct e1000_tx_desc);
  uint32 rxring = E1000_NRX * sizeof(struct e1000_rx_desc);

  e->tx = dma_alloc(txring, 16);
  e->rx = dma_alloc(rxring, 16);
  if (!e->tx || !e->rx)
    return -1;
  memset(e->tx, 0, txring);
  memset(e->rx, 0, rxring);

  for (int i = 0; i < E1000_NTX; i++) {
    e->txbuf[i] = dma_alloc(E1000_BUFSZ, 16);
    if (!e->txbuf[i])
      return -1;
    e->tx[i].addr   = dma_phys(e->txbuf[i]);
    // DD set on every transmit descriptor at start-up, so the first pass round
    // the ring sees them as free. The hardware only sets it after it has used
    // one, and a driver that waits for a bit the device has never had reason to
    // write waits forever.
    e->tx[i].status = E1000_TXD_STAT_DD;
  }

  for (int i = 0; i < E1000_NRX; i++) {
    e->rxbuf[i] = dma_alloc(E1000_BUFSZ, 16);
    if (!e->rxbuf[i])
      return -1;
    e->rx[i].addr   = dma_phys(e->rxbuf[i]);
    e->rx[i].status = 0;
  }

  wr(e, E1000_TDBAL, (uint32)dma_phys(e->tx));
  wr(e, E1000_TDBAH, (uint32)(dma_phys(e->tx) >> 32));
  wr(e, E1000_TDLEN, txring);
  wr(e, E1000_TDH, 0);
  wr(e, E1000_TDT, 0);

  wr(e, E1000_RDBAL, (uint32)dma_phys(e->rx));
  wr(e, E1000_RDBAH, (uint32)(dma_phys(e->rx) >> 32));
  wr(e, E1000_RDLEN, rxring);
  wr(e, E1000_RDH, 0);
  // Tail points at the last descriptor the hardware may fill. Setting it to
  // NRX-1 hands it every descriptor but one; the ring is full when head meets
  // tail, so one must always be held back or "full" and "empty" look identical.
  wr(e, E1000_RDT, E1000_NRX - 1);

  return 0;
}

// Queue one frame. Returns 0, or -1 if the ring is full.
//
// Copies into the descriptor's own buffer rather than mapping the caller's.
// A zero-copy path wants the caller to guarantee the memory stays put and is
// physically contiguous, which nothing above this can promise yet.
int
e1000_transmit(const void *frame, int len)
{
  struct e1000 *e = &nic;

  if (!e->ready || len <= 0 || len > E1000_BUFSZ)
    return -1;

  acquire(&e->lock);
  struct e1000_tx_desc *d = &e->tx[e->tx_next];
  if (!(d->status & E1000_TXD_STAT_DD)) {
    release(&e->lock);
    return -1;                       // the device has not finished with it
  }

  memmove(e->txbuf[e->tx_next], frame, len);
  d->length  = (uint16)len;
  d->cmd     = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS | E1000_TXD_CMD_RS;
  d->status  = 0;

  e->tx_next = (uint16)((e->tx_next + 1) % E1000_NTX);
  __sync_synchronize();              // descriptor visible before the tail moves
  wr(e, E1000_TDT, e->tx_next);
  e->tx_packets++;
  release(&e->lock);
  return 0;
}

// Take one received frame, if there is one. Returns its length, or 0.
//
// Polled. There is nothing above this to wake yet, and a receive path that
// wakes a nonexistent protocol stack is harder to reason about than one that is
// asked. The interrupt handler below exists to keep the cause register clear
// and to count; when an IP stack arrives, this is where the hand-off changes.
int
e1000_receive(void *out, int cap)
{
  struct e1000 *e = &nic;

  if (!e->ready)
    return 0;

  acquire(&e->lock);
  struct e1000_rx_desc *d = &e->rx[e->rx_next];
  if (!(d->status & E1000_RXD_STAT_DD)) {
    release(&e->lock);
    return 0;
  }

  int len = d->length;
  if (len > cap)
    len = cap;
  memmove(out, e->rxbuf[e->rx_next], len);

  d->status = 0;
  __sync_synchronize();
  // Hand the descriptor back by moving the tail to it. The tail always trails
  // the descriptor just released, which is why it is the *previous* index.
  wr(e, E1000_RDT, e->rx_next);
  e->rx_next = (uint16)((e->rx_next + 1) % E1000_NRX);
  e->rx_packets++;
  release(&e->lock);
  return len;
}

static void
e1000_intr(void *arg)
{
  struct e1000 *e = arg;
  // Reading ICR clears it. Not reading it leaves the cause asserted, and with a
  // level-triggered PCI interrupt that means the handler is re-entered forever.
  uint32 cause = rd(e, E1000_ICR);
  (void)cause;
}

int
e1000_info(char *buf, int cap)
{
  struct e1000 *e = &nic;
  int n = 0;

  if (!e->ready)
    return 0;

  static const char hex[] = "0123456789abcdef";
  n += 0;
  for (int i = 0; i < 6 && n + 3 < cap; i++) {
    buf[n++] = hex[e->mac[i] >> 4];
    buf[n++] = hex[e->mac[i] & 0xF];
    if (i < 5)
      buf[n++] = ':';
  }
  if (n < cap) buf[n++] = '\n';
  return n;
}

static int
e1000_probe(struct pci_dev *d)
{
  struct e1000 *e = &nic;

  if (e->ready)
    return -1;
  if (!d->bar[0].base)
    return -1;

  initlock(&e->lock, "e1000");
  pci_enable(d, 0x0006);              // memory space + bus master
  // ioremap rather than P2V: firmware may place a 64-bit BAR above the 4 GB the
  // direct map covers. It returns the direct-map address unchanged for a low
  // BAR, which is every BAR under SeaBIOS.
  e->regs = (volatile uint8 *)ioremap(d->bar[0].base, d->bar[0].size);
  if (!e->regs)
    return -1;

  // Reset, then wait for it to clear. The manual asks for a delay after reset
  // before any other register is touched; polling CTRL is that delay.
  wr(e, E1000_CTRL, rd(e, E1000_CTRL) | E1000_CTRL_RST);
  for (int i = 0; i < 1000000; i++)
    if (!(rd(e, E1000_CTRL) & E1000_CTRL_RST))
      break;

  wr(e, E1000_IMC, 0xFFFFFFFF);       // mask everything while we set up
  rd(e, E1000_ICR);                   // and clear whatever was pending

  wr(e, E1000_CTRL, rd(e, E1000_CTRL) | E1000_CTRL_SLU | E1000_CTRL_ASDE);

  read_mac(e);

  // The multicast table is undefined after reset. Left as it comes up, the
  // device accepts multicast traffic for addresses nobody asked for.
  for (int i = 0; i < 128; i++)
    wr(e, E1000_MTA + i * 4, 0);

  // Program our own MAC into the receive-address filter, with the valid bit.
  {
    uint32 lo = (uint32)e->mac[0] | ((uint32)e->mac[1] << 8) |
                ((uint32)e->mac[2] << 16) | ((uint32)e->mac[3] << 24);
    uint32 hi = (uint32)e->mac[4] | ((uint32)e->mac[5] << 8) | (1u << 31);
    wr(e, E1000_RAL, lo);
    wr(e, E1000_RAH, hi);
  }

  if (rings_setup(e) != 0) {
    cprintf("e1000: no DMA memory for the rings\n");
    return -1;
  }

  wr(e, E1000_TIPG, 10 | (8 << 10) | (6 << 20));   // the manual's IEEE defaults
  wr(e, E1000_TCTL, E1000_TCTL_EN | E1000_TCTL_PSP |
                    (0x10 << E1000_TCTL_CT_SHIFT) |
                    (0x40 << E1000_TCTL_COLD_SHIFT));

  wr(e, E1000_RCTL, E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_SECRC |
                    E1000_RCTL_SZ_2048 | E1000_RCTL_LBM_NONE);

  if (d->irq != 0xFF)
    // level_low = 1: this is a PCI interrupt, which is level-triggered and
    // active low. Registering it as edge-triggered makes it fire once and then
    // wedge, because the device holds the line asserted and no further edge
    // ever arrives. See the comment on irq_register in include/irq.h.
    irq_register(d->irq, e1000_intr, e, "e1000", 1);
  wr(e, E1000_IMS, E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_LSC);

  e->ready = 1;

  cprintf("e1000: %x:%x:%x:%x:%x:%x, link %s, irq %d\n",
          e->mac[0], e->mac[1], e->mac[2], e->mac[3], e->mac[4], e->mac[5],
          (rd(e, E1000_STATUS) & E1000_STATUS_LU) ? "up" : "down", d->irq);
  return 0;
}

// Every 8254x that shares the register map above. See include/e1000.h for
// where the IDs come from and why the PCIe parts are excluded.
#define M(id) { E1000_VENDOR, (id), 0, 0 }
static const struct pci_match e1000_match[] = {
  M(E1000_DEV_82540EM),        M(E1000_DEV_82540EM_LOM),
  M(E1000_DEV_82540EP_LOM),    M(E1000_DEV_82540EP),
  M(E1000_DEV_82540EP_LP),
  M(E1000_DEV_82541EI),        M(E1000_DEV_82541EI_MOBILE),
  M(E1000_DEV_82541ER_LOM),    M(E1000_DEV_82541ER),
  M(E1000_DEV_82541GI),        M(E1000_DEV_82541GI_MOBILE),
  M(E1000_DEV_82541GI_LF),
  M(E1000_DEV_82544EI_FIBER),  M(E1000_DEV_82544GC_COPPER),
  M(E1000_DEV_82544GC_LOM),
  M(E1000_DEV_82545EM_COPPER), M(E1000_DEV_82545EM_FIBER),
  M(E1000_DEV_82545GM_COPPER), M(E1000_DEV_82545GM_FIBER),
  M(E1000_DEV_82545GM_SERDES),
  M(E1000_DEV_82546EB_COPPER), M(E1000_DEV_82546EB_FIBER),
  M(E1000_DEV_82546EB_QUAD),
  M(E1000_DEV_82546GB_COPPER), M(E1000_DEV_82546GB_FIBER),
  M(E1000_DEV_82546GB_SERDES),
  M(E1000_DEV_82547EI),
  { 0, 0, 0, 0 },
};
#undef M

const struct pci_driver e1000_driver = {
  .name    = "e1000",
  .devname = "eth",
  .match = e1000_match,
  .ops   = 0,                 // no class vtable yet; there is no net class
  .probe = e1000_probe,
};

// --- self-test: does this thing actually move packets? ---------------------
//
// The transmit and receive paths were written together and neither had ever
// carried a frame. That is exactly the state in which a driver looks finished
// and is not: the rings are set up, the registers read back plausibly, and the
// first thing to discover otherwise would have been a protocol stack failing
// for reasons indistinguishable from its own bugs.
//
// So: build an ARP request by hand, send it, and wait for the reply. It proves
// the whole path at once -- descriptor format, DD handshake, tail bookkeeping,
// DMA addresses, the receive ring handback, and the MAC filter -- because an
// ARP reply is addressed to *us* and will not arrive if the filter is wrong.
//
// Under QEMU's user-mode networking the guest is 10.0.2.15 and the gateway is
// 10.0.2.2. SLIRP is silent until spoken to, which is why this has to transmit
// before it can expect to receive.
//
// `echo n > /proc/sysrq`.

static void
put_be16(uint8 *p, uint16 v)
{
  p[0] = (uint8)(v >> 8);
  p[1] = (uint8)v;
}

const unsigned char *
e1000_mac(void)
{
  return nic.mac;
}

void
e1000_selftest(void)
{
  struct e1000 *e = &nic;
  uint8 frame[42];
  uint8 reply[E1000_BUFSZ];

  if (!e->ready) {
    cprintf("e1000: no device\n");
    return;
  }

  static const uint8 me[4]  = { 10, 0, 2, 15 };   // QEMU SLIRP guest address
  static const uint8 them[4] = { 10, 0, 2, 2 };   // and its gateway

  memset(frame, 0, sizeof(frame));
  for (int i = 0; i < 6; i++)
    frame[i] = 0xFF;                              // broadcast
  for (int i = 0; i < 6; i++)
    frame[6 + i] = e->mac[i];
  put_be16(&frame[12], 0x0806);                   // ARP

  put_be16(&frame[14], 1);                        // Ethernet
  put_be16(&frame[16], 0x0800);                   // IPv4
  frame[18] = 6;                                  // hardware address length
  frame[19] = 4;                                  // protocol address length
  put_be16(&frame[20], 1);                        // request
  for (int i = 0; i < 6; i++)
    frame[22 + i] = e->mac[i];                    // sender hardware address
  for (int i = 0; i < 4; i++)
    frame[28 + i] = me[i];                        // sender protocol address
  // target hardware address stays zero -- that is what is being asked
  for (int i = 0; i < 4; i++)
    frame[38 + i] = them[i];

  cprintf("e1000: asking who has %d.%d.%d.%d\n", them[0], them[1], them[2], them[3]);
  if (e1000_transmit(frame, sizeof(frame)) != 0) {
    cprintf("e1000: transmit ring full\n");
    return;
  }

  // Bounded poll. There is no interrupt path to sleep on yet and this runs from
  // a sysrq write, so spinning briefly is the honest way to wait.
  for (int spin = 0; spin < 20000000; spin++) {
    int n = e1000_receive(reply, sizeof(reply));
    if (n <= 0)
      continue;

    cprintf("e1000: %d bytes from %x:%x:%x:%x:%x:%x type %x\n",
            n, reply[6], reply[7], reply[8], reply[9], reply[10], reply[11],
            ((uint32)reply[12] << 8) | reply[13]);

    if (n >= 42 && reply[12] == 0x08 && reply[13] == 0x06 &&
        reply[20] == 0 && reply[21] == 2) {
      cprintf("e1000: ARP reply -- %d.%d.%d.%d is at %x:%x:%x:%x:%x:%x\n",
              reply[28], reply[29], reply[30], reply[31],
              reply[22], reply[23], reply[24], reply[25],
              reply[26], reply[27]);
      cprintf("e1000: self-test passed, %d tx %d rx\n",
              (int)e->tx_packets, (int)e->rx_packets);
      return;
    }
  }
  cprintf("e1000: no reply -- %d tx, %d rx\n",
          (int)e->tx_packets, (int)e->rx_packets);
}