SchOS/kernel/drivers/net/r8169/r8169.c

// Realtek RTL8169/8168/8111 gigabit Ethernet.
//
// The NIC in a very large number of laptops and desktop boards: every
// RTL8111/8168 variant answers to 10EC:8168, and the RTL8111H in particular is
// what prompted this driver.
//
// **Native rather than through LDLS**, and the reason is worth recording
// because the first plan was the other way round. Running Linux's r8169.ko
// under kernel/../ldls would mean implementing 130 kernel functions it calls,
// and 35 of those are *phylib* -- `phy_connect_direct`, `phy_start`,
// `mdiobus_read`, `genphy_soft_reset` and the rest. phylib is a subsystem with
// a link state machine, not a set of leaf calls, and LDLS's usual answer for a
// symbol it does not implement -- a stub that reports itself and returns zero
// -- is exactly wrong for it: `phy_start` returning zero means the link never
// comes up and there is nothing to see. A native driver reaches the PHY
// through the chip's own MII window instead and skips the whole subsystem.
//
// The register layout below is a *fact about the hardware*, taken the way
// docs/architecture/ldls.md §3 says facts may be taken. None of the expression
// is Linux's: this is a descriptor-ring driver in the same shape as the e1000
// beside it, because that is the shape this kernel's netdev class wants.

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "arch/x86.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "fs/fs.h"
#include "fs/file.h"
#include "arch/memlayout.h"
#include "arch/mmu.h"
#include "dev/pci.h"
#include "dev/device.h"
#include "dev/netdev.h"
#include "core/dma.h"
#include "core/irq.h"

// --- registers --------------------------------------------------------------

#define R_IDR0      0x00    // station address, six bytes
#define R_MAR0      0x08    // multicast filter
#define R_TNPDS     0x20    // Tx normal-priority descriptors, 64-bit
#define R_CR        0x37    // command
#define R_TPPOLL    0x38    // tell the chip to look at the Tx ring
#define R_IMR       0x3C
#define R_ISR       0x3E
#define R_TCR       0x40    // Tx config
#define R_RCR       0x44    // Rx config
#define R_9346CR    0x50    // unlocks the config registers
#define R_CONFIG1   0x52
#define R_PHYAR     0x60    // the MII window
#define R_PHYSTATUS 0x6C
#define R_RMS       0xDA    // largest frame the chip will accept
#define R_CPCR      0xE0    // "C+" command
#define R_RDSAR     0xE4    // Rx descriptors, 64-bit
#define R_MTPS      0xEC

#define CR_RST      0x10
#define CR_RE       0x08
#define CR_TE       0x04

#define TPPOLL_NPQ  0x40

#define C9346_UNLOCK 0xC0
#define C9346_LOCK   0x00

// Interrupt causes. Write-1-to-clear, which is the detail that matters: a
// cause left set on a level-triggered line re-enters the handler for ever.
#define ISR_ROK     0x0001
#define ISR_RER     0x0002
#define ISR_TOK     0x0004
#define ISR_TER     0x0008
#define ISR_RDU     0x0010    // ran out of Rx descriptors
#define ISR_LINKCHG 0x0020
#define ISR_TDU     0x0080

#define PHYSTATUS_LINK   0x02

// Rx config: accept frames addressed to us, broadcast and multicast, with the
// largest DMA burst and Rx FIFO threshold the chip offers. Promiscuous is
// deliberately not set -- a NIC that hands the stack every frame on the wire
// is a NIC that makes its own bugs look like somebody else's traffic.
#define RCR_AAP     0x00000001   // (not set)
#define RCR_APM     0x00000002   // physical match
#define RCR_AM      0x00000004   // multicast
#define RCR_AB      0x00000008   // broadcast
// MXDMA is bits 8-10 and RXFTH is bits 13-15, and "unlimited" is all ones in
// each: 0x700 | 0xE000. The first version of this had 0x7700, which sets bits
// 12-14 -- an Rx FIFO threshold of *half* rather than none, and one bit of
// MXDMA short. It would have worked and been slower, which is the worst kind
// of wrong to leave in a register constant.
#define RCR_UNLIMITED 0x0000E700

#define TCR_IFG_STD 0x03000000
#define TCR_MXDMA   0x00000700

// --- descriptors ------------------------------------------------------------
//
// Sixteen bytes, and the ring is a ring only because the last descriptor
// carries EOR. The chip walks forward and wraps when it sees it.

struct rtl_desc {
  uint32 opts1;
  uint32 opts2;
  uint64 addr;
};

#define DESC_OWN    0x80000000   // the chip owns it
#define DESC_EOR    0x40000000   // last descriptor in the ring
#define DESC_FS     0x20000000   // first fragment of a frame
#define DESC_LS     0x10000000   // last fragment
#define DESC_LEN    0x00003FFF

#define NRX         32
#define NTX         32
#define BUFSZ       2048         // one frame, with room for a VLAN tag

#define R8169_MAX_NICS 2

struct r8169 {
  volatile uint8 *mmio;
  struct rtl_desc *rx, *tx;
  uint8  *rxbuf, *txbuf;
  uint64 rx_phys, tx_phys, rxbuf_phys, txbuf_phys;

  int    rx_next;               // the descriptor we will look at next
  int    tx_next;
  uint8  mac[6];
  int    ready;
  uint64 rx_packets, tx_packets;

  struct spinlock lock;
  struct pci_dev *pdev;
  struct device  *ndev;
};

static struct r8169 nics[R8169_MAX_NICS];

static struct r8169 *
of(struct device *d)
{
  return d ? (struct r8169 *)d->priv : 0;
}

static inline uint8  rd8 (struct r8169 *r, int o) { return *(volatile uint8  *)(r->mmio + o); }
static inline uint16 rd16(struct r8169 *r, int o) { return *(volatile uint16 *)(r->mmio + o); }
static inline uint32 rd32(struct r8169 *r, int o) { return *(volatile uint32 *)(r->mmio + o); }
static inline void   wr8 (struct r8169 *r, int o, uint8 v)  { *(volatile uint8  *)(r->mmio + o) = v; }
static inline void   wr16(struct r8169 *r, int o, uint16 v) { *(volatile uint16 *)(r->mmio + o) = v; }
static inline void   wr32(struct r8169 *r, int o, uint32 v) { *(volatile uint32 *)(r->mmio + o) = v; }

static void
wr64(struct r8169 *r, int o, uint64 v)
{
  // Low half first. The chip latches the pair on the high write, so the other
  // order hands it half of one address and half of the previous one.
  wr32(r, o, (uint32)v);
  wr32(r, o + 4, (uint32)(v >> 32));
}

// --- the PHY, through the chip's own window ---------------------------------
//
// This is the part that makes the driver native rather than a phylib port:
// PHYAR is a one-register MII bus. Write it with the register number and a
// value to write; read it back until the chip says the access finished.

#define PHYAR_FLAG  0x80000000

static void
mii_write(struct r8169 *r, int reg, uint16 val)
{
  wr32(r, R_PHYAR, PHYAR_FLAG | ((reg & 0x1F) << 16) | val);
  for (int i = 0; i < 1000; i++) {
    microdelay(20);
    if (!(rd32(r, R_PHYAR) & PHYAR_FLAG))
      return;
  }
}

static int
mii_read(struct r8169 *r, int reg)
{
  wr32(r, R_PHYAR, (reg & 0x1F) << 16);
  for (int i = 0; i < 1000; i++) {
    microdelay(20);
    uint32 v = rd32(r, R_PHYAR);
    if (v & PHYAR_FLAG)
      return (int)(v & 0xFFFF);
  }
  return -1;
}

// --- the class ops ----------------------------------------------------------

static const unsigned char *
r8169_mac_op(struct device *d)
{
  struct r8169 *r = of(d);
  static const unsigned char zero[6];
  return r ? r->mac : zero;
}

static int
r8169_link_up(struct device *d)
{
  struct r8169 *r = of(d);
  return r ? (rd8(r, R_PHYSTATUS) & PHYSTATUS_LINK) != 0 : 0;
}

static int
r8169_transmit(struct device *d, const void *frame, int len)
{
  struct r8169 *r = of(d);
  struct rtl_desc *t;
  int idx;

  if (!r || !r->ready)
    return -1;
  if (len <= 0 || len > BUFSZ)
    return -1;

  acquire(&r->lock);
  idx = r->tx_next;
  t = &r->tx[idx];

  // Still the chip's? Then the ring is full. Congestion rather than failure:
  // the caller retries, which is what the netdev contract says -1 means.
  if (t->opts1 & DESC_OWN) {
    release(&r->lock);
    return -1;
  }

  // The runt rule. Ethernet's minimum frame is 60 bytes before the FCS, and
  // this chip does not pad for you -- a shorter frame goes out short and the
  // other end drops it. ARP requests are 42 bytes, so getting this wrong is
  // not a corner case, it is "the machine cannot resolve anything".
  int n = len < 60 ? 60 : len;
  memmove(r->txbuf + (uint64)idx * BUFSZ, frame, len);
  if (n > len)
    memset(r->txbuf + (uint64)idx * BUFSZ + len, 0, n - len);

  t->addr  = r->txbuf_phys + (uint64)idx * BUFSZ;
  t->opts2 = 0;
  __sync_synchronize();
  t->opts1 = DESC_OWN | DESC_FS | DESC_LS | (uint32)n |
             (idx == NTX - 1 ? DESC_EOR : 0);
  __sync_synchronize();

  // Poke it. Without this the chip may sit on a ring it already owns until
  // something else wakes it.
  wr8(r, R_TPPOLL, TPPOLL_NPQ);

  r->tx_next = (idx + 1) % NTX;
  r->tx_packets++;
  release(&r->lock);
  return 0;
}

static int
r8169_receive(struct device *d, void *out, int cap)
{
  struct r8169 *r = of(d);
  struct rtl_desc *rx;
  int idx, len;

  if (!r || !r->ready)
    return 0;

  acquire(&r->lock);
  idx = r->rx_next;
  rx  = &r->rx[idx];

  if (rx->opts1 & DESC_OWN) {          // still the chip's: nothing arrived
    release(&r->lock);
    return 0;
  }

  // The length includes the four-byte FCS, which the stack does not want and
  // lwIP would otherwise carry to the end of the world as trailing garbage.
  len = (int)(rx->opts1 & DESC_LEN) - 4;
  if (len < 0)
    len = 0;
  if (len > cap)
    len = cap;
  if (len > 0)
    memmove(out, r->rxbuf + (uint64)idx * BUFSZ, len);

  // Hand the descriptor back. Address and size first, OWN last: the chip may
  // start filling it the instant OWN is set.
  rx->addr  = r->rxbuf_phys + (uint64)idx * BUFSZ;
  rx->opts2 = 0;
  __sync_synchronize();
  rx->opts1 = DESC_OWN | (uint32)BUFSZ | (idx == NRX - 1 ? DESC_EOR : 0);
  __sync_synchronize();

  r->rx_next = (idx + 1) % NRX;
  r->rx_packets++;
  release(&r->lock);
  return len;
}

// --- interrupt --------------------------------------------------------------

static void
r8169_intr(void *arg)
{
  struct r8169 *r = arg;
  uint16 isr;

  if (!r || !r->mmio)
    return;

  isr = rd16(r, R_ISR);
  if (!isr)
    return;

  // Write-1-to-clear, and it has to happen whatever else does: a cause left
  // asserted on a level-triggered PCI line re-enters this handler until the
  // machine does nothing else. See docs/history/kernel.md section 13 for the
  // last time that was learned here.
  wr16(r, R_ISR, isr);

  // Nothing is read from the ring here, for the same reason as the e1000: the
  // frame is pulled by the network thread, which may sleep on the buffer cache
  // and take the lwIP lock, and an interrupt handler may do neither. All this
  // does is end that thread's wait early.
  if (isr & (ISR_ROK | ISR_RDU))
    netdev_rx_signal();
}

// --- bring-up ---------------------------------------------------------------

static int
rings_init(struct r8169 *r)
{
  // 256-byte alignment is the chip's requirement for both descriptor bases,
  // and it is not advisory: a misaligned base is a ring the chip walks
  // somewhere else entirely.
  r->rx = dma_alloc(NRX * sizeof(struct rtl_desc), 256);
  r->tx = dma_alloc(NTX * sizeof(struct rtl_desc), 256);
  r->rxbuf = dma_alloc(NRX * BUFSZ, 8);
  r->txbuf = dma_alloc(NTX * BUFSZ, 8);
  if (!r->rx || !r->tx || !r->rxbuf || !r->txbuf)
    return -1;

  r->rx_phys    = dma_phys(r->rx);
  r->tx_phys    = dma_phys(r->tx);
  r->rxbuf_phys = dma_phys(r->rxbuf);
  r->txbuf_phys = dma_phys(r->txbuf);

  memset(r->rx, 0, NRX * sizeof(struct rtl_desc));
  memset(r->tx, 0, NTX * sizeof(struct rtl_desc));

  for (int i = 0; i < NRX; i++) {
    r->rx[i].addr  = r->rxbuf_phys + (uint64)i * BUFSZ;
    r->rx[i].opts1 = DESC_OWN | (uint32)BUFSZ |
                     (i == NRX - 1 ? DESC_EOR : 0);
  }
  for (int i = 0; i < NTX; i++)
    r->tx[i].opts1 = (i == NTX - 1 ? DESC_EOR : 0);

  r->rx_next = 0;
  r->tx_next = 0;
  return 0;
}

static int
r8169_probe(struct pci_dev *d)
{
  struct r8169 *r = 0;
  uint64 bar = 0;
  uint64 barlen = 0;

  for (int i = 0; i < R8169_MAX_NICS; i++)
    if (!nics[i].ready) { r = &nics[i]; break; }
  if (!r)
    return -1;

  // BAR 2 on a PCIe part, BAR 1 on the older PCI ones, and BAR 0 is I/O on
  // both. Picking the first memory BAR rather than a number is what makes one
  // driver cover the family.
  for (int i = 0; i < 6; i++)
    if (d->bar[i].base && !d->bar[i].is_io) {
      bar = d->bar[i].base;
      barlen = d->bar[i].size;
      break;
    }
  if (!bar) {
    cprintf("r8169: no memory BAR; not driving this device\n");
    return -1;
  }

  r->pdev = d;
  initlock(&r->lock, "r8169");
  pci_enable(d, PCI_CMD_MEMORY | PCI_CMD_BUS_MASTER);

  // ioremap, not P2V: a 64-bit BAR on a laptop routinely sits above the
  // 4 GB the direct map covers, and this is the same lesson the virtio-gpu
  // driver's comment records.
  r->mmio = ioremap(bar, barlen ? barlen : 0x1000);
  if (!r->mmio) {
    cprintf("r8169: cannot map registers at %p\n", (void *)bar);
    return -1;
  }

  // Soft reset, and wait for the chip to say it finished.
  wr8(r, R_CR, CR_RST);
  for (int i = 0; i < 1000; i++) {
    microdelay(100);
    if (!(rd8(r, R_CR) & CR_RST))
      break;
  }
  if (rd8(r, R_CR) & CR_RST) {
    cprintf("r8169: the chip did not come out of reset\n");
    return -1;
  }

  for (int i = 0; i < 6; i++)
    r->mac[i] = rd8(r, R_IDR0 + i);

  if (rings_init(r) != 0) {
    cprintf("r8169: no DMA memory for the rings\n");
    return -1;
  }

  wr8(r, R_9346CR, C9346_UNLOCK);       // config registers are writable now

  // CPCR is deliberately left alone.
  //
  // An earlier version set bit 3 here with a comment calling it "PCI multiple
  // read/write". Bit 3 is PCIDAC -- dual address cycle, which is 64-bit DMA
  // addressing -- and the comment was describing bit 4. Neither is wanted:
  // this driver's rings come from the DMA arena, which is below 4 GB, so DAC
  // would be asking the chip to do something it does not need to do on the
  // strength of a constant nobody had checked.
  //
  // The descriptor registers written below *are* the C+ ones on this part;
  // unlike the 8139 it has no other programming model to be in.

  wr16(r, R_RMS, BUFSZ);
  wr8(r, R_MTPS, 0x3B);

  wr64(r, R_RDSAR, r->rx_phys);
  wr64(r, R_TNPDS, r->tx_phys);

  wr32(r, R_TCR, TCR_IFG_STD | TCR_MXDMA);
  wr32(r, R_RCR, RCR_APM | RCR_AM | RCR_AB | RCR_UNLIMITED);

  // Multicast: everything. The stack filters, and a hash filter that is wrong
  // drops IPv6 neighbour discovery in a way that looks like a routing problem.
  wr32(r, R_MAR0, 0xFFFFFFFF);
  wr32(r, R_MAR0 + 4, 0xFFFFFFFF);

  wr8(r, R_CR, CR_RE | CR_TE);
  wr8(r, R_9346CR, C9346_LOCK);

  // Restart autonegotiation. The PHY normally does this out of reset, and
  // asking again costs nothing and covers the case where firmware left it
  // parked -- BMCR bit 12 is autoneg enable, bit 9 restarts it.
  {
    int bmcr = mii_read(r, 0);
    if (bmcr >= 0)
      mii_write(r, 0, (uint16)(bmcr | 0x1200));
  }

  wr16(r, R_ISR, 0xFFFF);               // clear anything left over
  wr16(r, R_IMR, ISR_ROK | ISR_RER | ISR_TOK | ISR_TER |
                 ISR_RDU | ISR_LINKCHG);

  // MSI first: every 8111 has it, and on the boards these are soldered to it
  // is the only interrupt that is sure to arrive. See pci_irq_attach().
  if (pci_irq_attach(d, r8169_intr, r, "r8169", 0) < 0)
    cprintf("r8169: no usable interrupt line; the link will be polled\n");

  r->ready = 1;

  // Said with cprintf rather than kinfo, and said in full.
  //
  // This driver's first run is on a machine nobody here can attach a debugger
  // to, so the one line it prints has to carry everything a bug report would
  // need: which variant answered, what address it claims, whether the PHY sees
  // anything, and which line it will interrupt on. XID is the hardware version
  // -- the top bits of TxConfig -- and is how Realtek parts are told apart,
  // because the PCI device id is 8168 for two decades of silicon.
  {
    uint32 xid = (rd32(r, R_TCR) >> 20) & 0xFCF;
    int    bmsr = mii_read(r, 1);

    cprintf("r8169: xid %x, %x:%x:%x:%x:%x:%x, irq %d (%s), %s (bmsr %x)\n",
            xid,
            r->mac[0], r->mac[1], r->mac[2], r->mac[3], r->mac[4], r->mac[5],
            d->irq_line, pci_irq_kind(d),
            (rd8(r, R_PHYSTATUS) & PHYSTATUS_LINK) ? "link up" : "link down",
            bmsr);
  }
  return 0;
}

static void
r8169_attached(struct pci_dev *d, struct device *dev)
{
  for (int i = 0; i < R8169_MAX_NICS; i++)
    if (nics[i].ready && nics[i].pdev == d) {
      dev->priv = &nics[i];
      nics[i].ndev = dev;
      return;
    }
}

static const struct netdev_ops r8169_netdev_ops = {
  .transmit = r8169_transmit,
  .receive  = r8169_receive,
  .mac      = r8169_mac_op,
  .link_up  = r8169_link_up,
};

// The whole RTL8111/8168 family answers to 8168, which is why one entry covers
// most of the laptops this will ever run on. The others are the older PCI
// 8169, the Fast Ethernet 8101/8106, and the 2.5G 8125 -- listed because they
// share this programming model, and untested because none of them is here.
static const struct pci_match r8169_match[] = {
  { 0x10EC, 0x8168, 0, 0 },     // RTL8111/8168/8411 -- includes the 8111H
  { 0x10EC, 0x8169, 0, 0 },     // the original gigabit part
  { 0x10EC, 0x8136, 0, 0 },     // RTL8101/8102/8106, Fast Ethernet
  { 0x10EC, 0x8161, 0, 0 },
  { 0x10EC, 0x2502, 0, 0 },
  { 0x10EC, 0x2600, 0, 0 },
  { 0, 0, 0, 0 },
};

const struct pci_driver r8169_driver = {
  .name    = "r8169",
  .devname = "eth",
  .match   = r8169_match,
  .ops     = &r8169_netdev_ops,
  .probe   = r8169_probe,
  .attached = r8169_attached,
};