SchOS/kernel/drivers/net/e1000/e1000.c

// Intel 8254x gigabit Ethernet, written from Intel's public datasheet (317453).
//
// What it provides the rest of the system is a struct netdev_ops and nothing
// else: a station address, a transmit path and a receive ring. It does not
// know that lwIP exists, and lwIP does not know that it does -- see
// include/dev/netdev.h for why that boundary is where it is. Adding a second NIC
// means writing another driver that fills in the same four function pointers,
// and touching no file outside its own directory.
//
// Bring-up follows the manual's initialisation sequence (section 14.3-14.5):
// reset, read the MAC out of EEPROM, clear the multicast table, set up the
// receive and transmit rings, then enable each. The order matters -- enabling a
// receiver whose ring base is still zero makes the device DMA into physical
// address 0, which on this kernel is the boot page table.

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "arch/x86.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "fs/fs.h"
#include "fs/file.h"
#include "arch/mmu.h"
#include "arch/memlayout.h"
#include "dev/pci.h"
#include "dev/device.h"
#include "dev/netdev.h"
#include "core/dma.h"
#include "core/irq.h"
#include "e1000.h"

struct e1000 {
  volatile uint8 *regs;

  struct e1000_tx_desc *tx;
  struct e1000_rx_desc *rx;
  uint8 *txbuf[E1000_NTX];
  uint8 *rxbuf[E1000_NRX];
  uint16 tx_next;
  uint16 rx_next;

  uint8 mac[6];
  int   ready;
  // Which generation's EERD layout this part speaks. See E1000_EERD_DONE_8257X
  // in e1000.h -- it is the only register whose shape differs between the
  // 8254x and the PCIe 8257x, and getting it wrong yields a MAC of zeroes.
  int   nvm_8257x;
  uint64 rx_packets, tx_packets, rx_dropped;

  // Which PCI device this instance drives. Kept so that attached() can match
  // the device the bus just published against the instance probe() set up --
  // the two calls are separate, and nothing else connects them.
  struct pci_dev *pdev;

  // And the network device the stack knows it by, so that letting go of the
  // hardware can also take the interface out of the stack. Set by attached(),
  // which is where the two are first introduced.
  struct device *ndev;

  struct spinlock lock;
};

// More than one, because a machine really can have two of the same card and
// this driver used to refuse the second outright. Four is the same limit the
// stack puts on interfaces; each unused entry costs a few dozen bytes, and the
// rings behind a used one come from the DMA arena at probe time.
#define E1000_MAX_NICS 4

static struct e1000 nics[E1000_MAX_NICS];

// The instance behind a device, or 0. Every op goes through this rather than a
// file-static pointer, which is what makes two cards possible at all.
static struct e1000 *
nic_of(struct device *dev)
{
  return dev ? (struct e1000 *)dev->priv : 0;
}

static inline uint32 rd(struct e1000 *e, int off) { return *(volatile uint32 *)(e->regs + off); }
static inline void   wr(struct e1000 *e, int off, uint32 v) { *(volatile uint32 *)(e->regs + off) = v; }

// The EEPROM is read one 16-bit word at a time through a single register: write
// the address with the start bit, poll for done, take the data out of the top
// half.
//
// Bounded, because a device whose EEPROM never reports done would otherwise
// hang the boot -- and QEMU's e1000 does answer, but a real card with a blank
// EEPROM is a thing that exists.
static int
eeprom_read(struct e1000 *e, int addr, uint16 *out)
{
  int  shift = e->nvm_8257x ? E1000_EERD_ADDR_SHIFT_8257X
                            : E1000_EERD_ADDR_SHIFT;
  uint32 done = e->nvm_8257x ? E1000_EERD_DONE_8257X : E1000_EERD_DONE;

  wr(e, E1000_EERD, (addr << shift) | E1000_EERD_START);
  for (int i = 0; i < 100000; i++) {
    uint32 v = rd(e, E1000_EERD);
    if (v & done) {
      *out = (uint16)(v >> E1000_EERD_DATA_SHIFT);
      return 0;
    }
  }
  return -1;
}

// The MAC address, from EEPROM words 0..2, little-endian within each word.
//
// Falls back to the receive-address registers, which the firmware may already
// have programmed. QEMU fills both; some cards leave the EEPROM path
// unimplemented, and a driver that only knows one of the two ends up with a MAC
// of all zeroes and no idea why nothing answers it.
static void
read_mac(struct e1000 *e)
{
  uint16 w;
  int ok = 1;

  for (int i = 0; i < 3; i++) {
    if (eeprom_read(e, i, &w) != 0) { ok = 0; break; }
    e->mac[i * 2]     = (uint8)(w & 0xFF);
    e->mac[i * 2 + 1] = (uint8)(w >> 8);
  }
  if (ok)
    return;

  uint32 lo = rd(e, E1000_RAL), hi = rd(e, E1000_RAH);
  e->mac[0] = (uint8)lo; e->mac[1] = (uint8)(lo >> 8);
  e->mac[2] = (uint8)(lo >> 16); e->mac[3] = (uint8)(lo >> 24);
  e->mac[4] = (uint8)hi; e->mac[5] = (uint8)(hi >> 8);
}

static int
rings_setup(struct e1000 *e)
{
  // One block for both rings and every buffer, from the DMA arena: the device
  // reads all of it by physical address, and kalloc's pages are contiguous only
  // by accident.
  uint32 txring = E1000_NTX * sizeof(struct e1000_tx_desc);
  uint32 rxring = E1000_NRX * sizeof(struct e1000_rx_desc);

  e->tx = dma_alloc(txring, 16);
  e->rx = dma_alloc(rxring, 16);
  if (!e->tx || !e->rx)
    return -1;
  memset(e->tx, 0, txring);
  memset(e->rx, 0, rxring);

  for (int i = 0; i < E1000_NTX; i++) {
    e->txbuf[i] = dma_alloc(E1000_BUFSZ, 16);
    if (!e->txbuf[i])
      return -1;
    e->tx[i].addr   = dma_phys(e->txbuf[i]);
    // DD set on every transmit descriptor at start-up, so the first pass round
    // the ring sees them as free. The hardware only sets it after it has used
    // one, and a driver that waits for a bit the device has never had reason to
    // write waits forever.
    e->tx[i].status = E1000_TXD_STAT_DD;
  }

  for (int i = 0; i < E1000_NRX; i++) {
    e->rxbuf[i] = dma_alloc(E1000_BUFSZ, 16);
    if (!e->rxbuf[i])
      return -1;
    e->rx[i].addr   = dma_phys(e->rxbuf[i]);
    e->rx[i].status = 0;
  }

  wr(e, E1000_TDBAL, (uint32)dma_phys(e->tx));
  wr(e, E1000_TDBAH, (uint32)(dma_phys(e->tx) >> 32));
  wr(e, E1000_TDLEN, txring);
  wr(e, E1000_TDH, 0);
  wr(e, E1000_TDT, 0);

  wr(e, E1000_RDBAL, (uint32)dma_phys(e->rx));
  wr(e, E1000_RDBAH, (uint32)(dma_phys(e->rx) >> 32));
  wr(e, E1000_RDLEN, rxring);
  wr(e, E1000_RDH, 0);
  // Tail points at the last descriptor the hardware may fill. Setting it to
  // NRX-1 hands it every descriptor but one; the ring is full when head meets
  // tail, so one must always be held back or "full" and "empty" look identical.
  wr(e, E1000_RDT, E1000_NRX - 1);

  return 0;
}

// Queue one frame. Returns 0, or -1 if the ring is full.
//
// Copies into the descriptor's own buffer rather than mapping the caller's.
// A zero-copy path wants the caller to guarantee the memory stays put and is
// physically contiguous, which nothing above this can promise yet.
//
static int
e1000_transmit(struct device *dev, const void *frame, int len)
{
  struct e1000 *e = nic_of(dev);

  if (!e || !e->ready || len <= 0 || len > E1000_BUFSZ)
    return -1;

  acquire(&e->lock);
  struct e1000_tx_desc *d = &e->tx[e->tx_next];
  if (!(d->status & E1000_TXD_STAT_DD)) {
    release(&e->lock);
    return -1;                       // the device has not finished with it
  }

  memmove(e->txbuf[e->tx_next], frame, len);
  d->length  = (uint16)len;
  d->cmd     = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS | E1000_TXD_CMD_RS;
  d->status  = 0;

  e->tx_next = (uint16)((e->tx_next + 1) % E1000_NTX);
  __sync_synchronize();              // descriptor visible before the tail moves
  wr(e, E1000_TDT, e->tx_next);
  e->tx_packets++;
  release(&e->lock);
  return 0;
}

// Take one received frame, if there is one. Returns its length, or 0.
//
// Polled. There is nothing above this to wake yet, and a receive path that
// wakes a nonexistent protocol stack is harder to reason about than one that is
// asked. The interrupt handler below exists to keep the cause register clear
// and to count; when an IP stack arrives, this is where the hand-off changes.
static int
e1000_receive(struct device *dev, void *out, int cap)
{
  struct e1000 *e = nic_of(dev);

  if (!e || !e->ready)
    return 0;

  acquire(&e->lock);
  struct e1000_rx_desc *d = &e->rx[e->rx_next];
  if (!(d->status & E1000_RXD_STAT_DD)) {
    release(&e->lock);
    return 0;
  }

  int len = d->length;
  if (len > cap)
    len = cap;
  memmove(out, e->rxbuf[e->rx_next], len);

  d->status = 0;
  __sync_synchronize();
  // Hand the descriptor back by moving the tail to it. The tail always trails
  // the descriptor just released, which is why it is the *previous* index.
  wr(e, E1000_RDT, e->rx_next);
  e->rx_next = (uint16)((e->rx_next + 1) % E1000_NRX);
  e->rx_packets++;
  release(&e->lock);
  return len;
}

static void
e1000_intr(void *arg)
{
  struct e1000 *e = arg;
  // Reading ICR clears it. Not reading it leaves the cause asserted, and with a
  // level-triggered PCI interrupt that means the handler is re-entered forever.
  uint32 cause = rd(e, E1000_ICR);

  // Nothing is read from the ring here. The frame is pulled by the network
  // thread through .receive, which may sleep on the buffer cache and on the
  // lwIP lock; an interrupt handler may do neither. All this does is end that
  // thread's wait, so the frame is taken now instead of up to 10 ms later.
  //
  // RXT0 is the receive timer expiring -- a frame, or a burst, has landed.
  // RXDMT0 is the ring falling below its minimum threshold, which means frames
  // are arriving faster than they are being taken and is exactly when waiting
  // for the clock hurts most.
  if (cause & (E1000_ICR_RXT0 | E1000_ICR_RXDMT0))
    netdev_rx_signal();
}

static int
e1000_probe(struct pci_dev *d)
{
  struct e1000 *e = 0;

  // A free slot, rather than "the one instance, if it is still free". The old
  // spelling is why a machine with two of these cards drove only the first.
  for (int i = 0; i < E1000_MAX_NICS; i++)
    if (!nics[i].ready) {
      e = &nics[i];
      break;
    }
  if (!e)
    return -1;
  if (!d->bar[0].base)
    return -1;

  e->pdev = d;
  // Everything from 0x105E upwards in the table is PCIe; the 8254x ids are all
  // below it. Spelled as a list rather than a comparison because an id is not
  // a magnitude -- 0x150C is a PCIe part and 0x1078 is not.
  switch (d->device) {
  case E1000_DEV_82571EB_COPPER: case E1000_DEV_82572EI_COPPER:
  case E1000_DEV_82572EI:        case E1000_DEV_82573E:
  case E1000_DEV_82573L:         case E1000_DEV_82574L:
  case E1000_DEV_82583V:
    e->nvm_8257x = 1;
    break;
  default:
    e->nvm_8257x = 0;
    break;
  }
  initlock(&e->lock, "e1000");
  pci_enable(d, 0x0006);              // memory space + bus master
  // ioremap rather than P2V: firmware may place a 64-bit BAR above the 4 GB the
  // direct map covers. It returns the direct-map address unchanged for a low
  // BAR, which is every BAR under SeaBIOS.
  e->regs = (volatile uint8 *)ioremap(d->bar[0].base, d->bar[0].size);
  if (!e->regs)
    return -1;

  // Reset, then wait for it to clear. The manual asks for a delay after reset
  // before any other register is touched; polling CTRL is that delay.
  wr(e, E1000_CTRL, rd(e, E1000_CTRL) | E1000_CTRL_RST);
  for (int i = 0; i < 1000000; i++)
    if (!(rd(e, E1000_CTRL) & E1000_CTRL_RST))
      break;

  wr(e, E1000_IMC, 0xFFFFFFFF);       // mask everything while we set up
  rd(e, E1000_ICR);                   // and clear whatever was pending

  wr(e, E1000_CTRL, rd(e, E1000_CTRL) | E1000_CTRL_SLU | E1000_CTRL_ASDE);

  read_mac(e);

  // The multicast table is undefined after reset. Left as it comes up, the
  // device accepts multicast traffic for addresses nobody asked for.
  for (int i = 0; i < 128; i++)
    wr(e, E1000_MTA + i * 4, 0);

  // Program our own MAC into the receive-address filter, with the valid bit.
  {
    uint32 lo = (uint32)e->mac[0] | ((uint32)e->mac[1] << 8) |
                ((uint32)e->mac[2] << 16) | ((uint32)e->mac[3] << 24);
    uint32 hi = (uint32)e->mac[4] | ((uint32)e->mac[5] << 8) | (1u << 31);
    wr(e, E1000_RAL, lo);
    wr(e, E1000_RAH, hi);
  }

  if (rings_setup(e) != 0) {
    cprintf("e1000: no DMA memory for the rings\n");
    return -1;
  }

  wr(e, E1000_TIPG, 10 | (8 << 10) | (6 << 20));   // the manual's IEEE defaults
  wr(e, E1000_TCTL, E1000_TCTL_EN | E1000_TCTL_PSP |
                    (0x10 << E1000_TCTL_CT_SHIFT) |
                    (0x40 << E1000_TCTL_COLD_SHIFT));

  wr(e, E1000_RCTL, E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_SECRC |
                    E1000_RCTL_SZ_2048 | E1000_RCTL_LBM_NONE);

  // MSI where the part has it -- the 82574 and the rest of the PCIe
  // generation do -- and INTx otherwise. MSI-X is not offered: the 82574 in
  // MSI-X mode routes its causes through IVAR, which this driver does not
  // program, and its interrupts would go nowhere. If neither works the
  // network thread still collects frames on its tick.
  pci_irq_attach(d, e1000_intr, e, "e1000", 0);
  wr(e, E1000_IMS, E1000_ICR_RXT0 | E1000_ICR_RXDMT0 | E1000_ICR_LSC);

  e->ready = 1;

  kinfo("e1000: %x:%x:%x:%x:%x:%x, link %s, irq %d (%s)\n",
          e->mac[0], e->mac[1], e->mac[2], e->mac[3], e->mac[4], e->mac[5],
          (rd(e, E1000_STATUS) & E1000_STATUS_LU) ? "up" : "down",
          d->irq_line, pci_irq_kind(d));
  return 0;
}

// Every 8254x that shares the register map above. See include/e1000.h for
// where the IDs come from and why the PCIe parts are excluded.
#define M(id) { E1000_VENDOR, (id), 0, 0 }
static const struct pci_match e1000_match[] = {
  M(E1000_DEV_82540EM),        M(E1000_DEV_82540EM_LOM),
  M(E1000_DEV_82540EP_LOM),    M(E1000_DEV_82540EP),
  M(E1000_DEV_82540EP_LP),
  M(E1000_DEV_82541EI),        M(E1000_DEV_82541EI_MOBILE),
  M(E1000_DEV_82541ER_LOM),    M(E1000_DEV_82541ER),
  M(E1000_DEV_82541GI),        M(E1000_DEV_82541GI_MOBILE),
  M(E1000_DEV_82541GI_LF),
  M(E1000_DEV_82544EI_FIBER),  M(E1000_DEV_82544GC_COPPER),
  M(E1000_DEV_82544GC_LOM),
  M(E1000_DEV_82545EM_COPPER), M(E1000_DEV_82545EM_FIBER),
  M(E1000_DEV_82545GM_COPPER), M(E1000_DEV_82545GM_FIBER),
  M(E1000_DEV_82545GM_SERDES),
  M(E1000_DEV_82546EB_COPPER), M(E1000_DEV_82546EB_FIBER),
  M(E1000_DEV_82546EB_QUAD),
  M(E1000_DEV_82546GB_COPPER), M(E1000_DEV_82546GB_FIBER),
  M(E1000_DEV_82546GB_SERDES),
  M(E1000_DEV_82547EI),
  // The PCIe generation. 82574L is emulated by QEMU and is what this was
  // tested against; the others share its register map and NVM layout.
  M(E1000_DEV_82571EB_COPPER), M(E1000_DEV_82572EI_COPPER),
  M(E1000_DEV_82572EI),        M(E1000_DEV_82573E),
  M(E1000_DEV_82573L),         M(E1000_DEV_82574L),
  M(E1000_DEV_82583V),
  { 0, 0, 0, 0 },
};
#undef M

static const unsigned char *
e1000_mac_op(struct device *dev)
{
  struct e1000 *e = nic_of(dev);
  return (e && e->ready) ? e->mac : 0;
}

static int
e1000_link_up(struct device *dev)
{
  struct e1000 *e = nic_of(dev);
  return e && e->ready && (rd(e, E1000_STATUS) & E1000_STATUS_LU) != 0;
}

// Let go of the device, so a userspace driver can have it.
//
// The same hook virtio-gpu grew for H1, on a NIC. docs/architecture/devices.md's rule
// is that the kernel keeps a minimal driver for everything on the path to a
// shell -- and a network interface is not on that path, so this one may leave.
//
// "Letting go" is more than saying yes. The device is still bus-mastering into
// descriptor rings this driver allocated, and the process about to claim it
// will be handed those pages back by the arena. So the receiver and the
// transmitter are switched off and the interrupt is masked *at the device*
// before the handler is unregistered: masking after unregistering leaves a
// window where a level-triggered line is asserted with nobody to clear it,
// which is the failure docs/history/kernel.md section 13 is about.
//
// `ready` going false is what makes the class vtable safe afterwards --
// transmit, receive and link_up all check it, so the stack sees an interface
// whose link is down rather than one that swallows packets.
static int
e1000_release(struct pci_dev *d)
{
  struct e1000 *e = 0;

  // Found by the device it is bound to. struct pci_dev carries no private
  // pointer -- this driver keeps its instances in a table and matches on
  // `pdev`, the same way e1000_attached() does.
  for (int i = 0; i < E1000_MAX_NICS; i++)
    if (nics[i].pdev == d) { e = &nics[i]; break; }

  if (!e || !e->ready)
    return 0;                     // never came up, or already let go

  e->ready = 0;

  wr(e, E1000_IMC, 0xFFFFFFFF);            // no more interrupts from it
  wr(e, E1000_RCTL, 0);                    // stop receiving
  wr(e, E1000_TCTL, 0);                    // stop transmitting
  rd(e, E1000_STATUS);                     // post the writes

  pci_irq_detach(d, e1000_intr, e);

  // And take the interface out of the stack.
  //
  // Leaving it there is worse than it sounds. The netif keeps its address and
  // the route to its subnet, so the stack goes on choosing it -- and every
  // frame it sends reaches e1000_transmit, which now correctly refuses. The
  // packets stop and nothing reports an error, which is indistinguishable
  // from a driver in a process that does not work. It was mistaken for
  // exactly that for an afternoon.
  if (e->ndev)
    net_detach(e->ndev);

  cprintf("e1000: released to userspace; the interface is down\n");
  return 0;
}

// Hang the instance on the device the bus has just published. See the comment
// on `attached` in include/dev/pci.h for why this cannot happen inside probe().
static void
e1000_attached(struct pci_dev *d, struct device *dev)
{
  for (int i = 0; i < E1000_MAX_NICS; i++)
    if (nics[i].ready && nics[i].pdev == d) {
      dev->priv = &nics[i];
      nics[i].ndev = dev;
      return;
    }
}

// The whole of what the network stack sees of this driver.
static const struct netdev_ops e1000_netdev_ops = {
  .transmit = e1000_transmit,
  .receive  = e1000_receive,
  .mac      = e1000_mac_op,
  .link_up  = e1000_link_up,
};

const struct pci_driver e1000_driver = {
  .name    = "e1000",
  .devname = "eth",
  .match = e1000_match,
  .ops   = &e1000_netdev_ops,
  .probe = e1000_probe,
  .attached = e1000_attached,
  .release = e1000_release,
};