SchOS/kernel/drivers/net/r8169/r8169.c

// Realtek RTL8169/8168/8111 gigabit Ethernet.
//
// The NIC in a very large number of laptops and desktop boards: every
// RTL8111/8168 variant answers to 10EC:8168, and the RTL8111H in particular is
// what prompted this driver.
//
// **Native rather than through LDLS**, and the reason is worth recording
// because the first plan was the other way round. Running Linux's r8169.ko
// under kernel/../ldls would mean implementing 130 kernel functions it calls,
// and 35 of those are *phylib* -- `phy_connect_direct`, `phy_start`,
// `mdiobus_read`, `genphy_soft_reset` and the rest. phylib is a subsystem with
// a link state machine, not a set of leaf calls, and LDLS's usual answer for a
// symbol it does not implement -- a stub that reports itself and returns zero
// -- is exactly wrong for it: `phy_start` returning zero means the link never
// comes up and there is nothing to see. A native driver reaches the PHY
// through the chip's own MII window instead and skips the whole subsystem.
//
// The register layout below is a *fact about the hardware*, taken the way
// docs/architecture/ldls.md §3 says facts may be taken. None of the expression
// is Linux's: this is a descriptor-ring driver in the same shape as the e1000
// beside it, because that is the shape this kernel's netdev class wants.

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "arch/x86.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "fs/fs.h"
#include "fs/file.h"
#include "arch/memlayout.h"
#include "arch/mmu.h"
#include "dev/pci.h"
#include "dev/device.h"
#include "dev/netdev.h"
#include "core/dma.h"
#include "core/irq.h"

// --- registers --------------------------------------------------------------

#define R_IDR0      0x00    // station address, six bytes
#define R_MAR0      0x08    // multicast filter
#define R_TNPDS     0x20    // Tx normal-priority descriptors, 64-bit
#define R_CR        0x37    // command
#define R_TPPOLL    0x38    // tell the chip to look at the Tx ring
#define R_IMR       0x3C
#define R_ISR       0x3E
#define R_TCR       0x40    // Tx config
#define R_RCR       0x44    // Rx config
#define R_9346CR    0x50    // unlocks the config registers
#define R_CONFIG1   0x52
#define R_CONFIG2   0x53
#define R_CONFIG3   0x54
#define R_CONFIG5   0x56
#define R_EEE_LED   0x1B
#define R_EPHYAR    0x80    // the PCIe PHY's registers
#define R_DLLPR     0xD0
#define R_MISC1     0xF2
#define R_INTRMIT   0xE2    // interrupt mitigation
#define R_PHYAR     0x60    // the MII window
#define R_PHYSTATUS 0x6C
#define R_PMCH      0x6F    // PLL power
#define R_ERIDR     0x70    // extended register window: data
#define R_ERIAR     0x74    //   and address
#define R_GPHY_OCP  0xB8    // the PHY's registers, 8168G and later
#define R_MAC_OCP   0xB0    // the MAC's own OCP registers
#define R_MCU       0xD3
#define R_MISC      0xF0
#define R_RMS       0xDA    // largest frame the chip will accept
#define R_CPCR      0xE0    // "C+" command
#define R_RDSAR     0xE4    // Rx descriptors, 64-bit
#define R_MTPS      0xEC

#define CR_RST      0x10
#define CR_RE       0x08
#define CR_TE       0x04

#define TPPOLL_NPQ  0x40

#define C9346_UNLOCK 0xC0
#define C9346_LOCK   0x00

// Interrupt causes. Write-1-to-clear, which is the detail that matters: a
// cause left set on a level-triggered line re-enters the handler for ever.
#define ISR_ROK     0x0001
#define ISR_RER     0x0002
#define ISR_TOK     0x0004
#define ISR_TER     0x0008
#define ISR_RDU     0x0010    // ran out of Rx descriptors
#define ISR_LINKCHG 0x0020
#define ISR_TDU     0x0080

#define PHYSTATUS_LINK   0x02

#define MCU_NOW_IS_OOB   0x80     // the firmware, not a driver, owns the MAC
#define MCU_LL_READY     0x02     // link-list FIFO ready
#define MISC_RXDV_GATE   0x00080000
#define OCP_FLAG         0x80000000
#define OCP_STD_PHY_BASE 0xA400
#define BMCR_PDOWN       0x0800
#define BMCR_ANENABLE    0x1000
#define BMCR_ANRESTART   0x0200

// Rx config: accept frames addressed to us, broadcast and multicast, with the
// largest DMA burst and Rx FIFO threshold the chip offers. Promiscuous is
// deliberately not set -- a NIC that hands the stack every frame on the wire
// is a NIC that makes its own bugs look like somebody else's traffic.
#define RCR_AAP     0x00000001   // (not set)
#define RCR_APM     0x00000002   // physical match
#define RCR_AM      0x00000004   // multicast
#define RCR_AB      0x00000008   // broadcast
// MXDMA is bits 8-10 and RXFTH is bits 13-15, and "unlimited" is all ones in
// each: 0x700 | 0xE000. The first version of this had 0x7700, which sets bits
// 12-14 -- an Rx FIFO threshold of *half* rather than none, and one bit of
// MXDMA short. It would have worked and been slower, which is the worst kind
// of wrong to leave in a register constant.
#define RCR_UNLIMITED 0x0000E700
// What Linux's rtl_init_rxcfg() writes on the 8168G and later instead:
// 128-byte early interrupt, multiple descriptors per DMA, unlimited burst,
// early receive off.
#define RCR_GEN_G   (0x8000 | 0x4000 | 0x0700 | 0x0800)
#define TCR_AUTO_FIFO 0x00000080 // 8168E-VL and later: the chip sizes its FIFOs

#define TCR_IFG_STD 0x03000000
#define TCR_MXDMA   0x00000700

// --- descriptors ------------------------------------------------------------
//
// Sixteen bytes, and the ring is a ring only because the last descriptor
// carries EOR. The chip walks forward and wraps when it sees it.

struct rtl_desc {
  uint32 opts1;
  uint32 opts2;
  uint64 addr;
};

#define DESC_OWN    0x80000000   // the chip owns it
#define DESC_EOR    0x40000000   // last descriptor in the ring
#define DESC_FS     0x20000000   // first fragment of a frame
#define DESC_LS     0x10000000   // last fragment
#define DESC_LEN    0x00003FFF

// 256, as the e1000 has. It was 32, and TCP advertises a 256 KB window --
// some 180 full frames in flight -- which a gigabit link fills 32 slots with
// in under half a millisecond. Everything past them was dropped, TCP read the
// drops as congestion, and a download on the laptop crawled: apk sat on its
// first index for minutes. The chip takes up to 1024.
#define NRX         256
#define NTX         32
#define BUFSZ       2048         // one frame, with room for a VLAN tag

#define R8169_MAX_NICS 2

struct r8169 {
  volatile uint8 *mmio;
  struct rtl_desc *rx, *tx;
  uint8  *rxbuf, *txbuf;
  uint64 rx_phys, tx_phys, rxbuf_phys, txbuf_phys;

  int    rx_next;               // the descriptor we will look at next
  int    tx_next;
  uint8  mac[6];
  int    ready;
  int    gen_g;                 // 8168G or later: see r8169_gen_g()
  uint16 ocp_base;              // the PHY page selected through register 0x1F
  uint64 rx_packets, tx_packets;

  struct spinlock lock;
  struct pci_dev *pdev;
  struct device  *ndev;
};

static struct r8169 nics[R8169_MAX_NICS];

static struct r8169 *
of(struct device *d)
{
  return d ? (struct r8169 *)d->priv : 0;
}

static inline uint8  rd8 (struct r8169 *r, int o) { return *(volatile uint8  *)(r->mmio + o); }
static inline uint16 rd16(struct r8169 *r, int o) { return *(volatile uint16 *)(r->mmio + o); }
static inline uint32 rd32(struct r8169 *r, int o) { return *(volatile uint32 *)(r->mmio + o); }
static inline void   wr8 (struct r8169 *r, int o, uint8 v)  { *(volatile uint8  *)(r->mmio + o) = v; }
static inline void   wr16(struct r8169 *r, int o, uint16 v) { *(volatile uint16 *)(r->mmio + o) = v; }
static inline void   wr32(struct r8169 *r, int o, uint32 v) { *(volatile uint32 *)(r->mmio + o) = v; }

static void
wr64(struct r8169 *r, int o, uint64 v)
{
  // Low half first. The chip latches the pair on the high write, so the other
  // order hands it half of one address and half of the previous one.
  wr32(r, o, (uint32)v);
  wr32(r, o + 4, (uint32)(v >> 32));
}

// --- the PHY, through the chip's own window ---------------------------------
//
// This is the part that makes the driver native rather than a phylib port:
// PHYAR is a one-register MII bus. Write it with the register number and a
// value to write; read it back until the chip says the access finished.

#define PHYAR_FLAG  0x80000000

static void
phyar_write(struct r8169 *r, int reg, uint16 val)
{
  wr32(r, R_PHYAR, PHYAR_FLAG | ((reg & 0x1F) << 16) | val);
  for (int i = 0; i < 1000; i++) {
    microdelay(20);
    if (!(rd32(r, R_PHYAR) & PHYAR_FLAG))
      return;
  }
}

static int
phyar_read(struct r8169 *r, int reg)
{
  wr32(r, R_PHYAR, (reg & 0x1F) << 16);
  for (int i = 0; i < 1000; i++) {
    microdelay(20);
    uint32 v = rd32(r, R_PHYAR);
    if (v & PHYAR_FLAG)
      return (int)(v & 0xFFFF);
  }
  return -1;
}

// From the 8168G on, the PHY sits behind an OCP window instead: its standard
// registers are at 0xA400 + 2*reg, and "select page N" through register 0x1F
// becomes a base address. PHYAR still answers reads on some of these chips,
// which is how this driver got away without it -- until a write mattered.
static void
ocp_phy_write(struct r8169 *r, uint16 addr, uint16 val)
{
  wr32(r, R_GPHY_OCP, OCP_FLAG | ((uint32)(addr >> 1) << 16) | val);
  for (int i = 0; i < 1000; i++) {
    microdelay(20);
    if (!(rd32(r, R_GPHY_OCP) & OCP_FLAG))
      return;
  }
}

static int
ocp_phy_read(struct r8169 *r, uint16 addr)
{
  wr32(r, R_GPHY_OCP, (uint32)(addr >> 1) << 16);
  for (int i = 0; i < 1000; i++) {
    microdelay(20);
    uint32 v = rd32(r, R_GPHY_OCP);
    if (v & OCP_FLAG)
      return (int)(v & 0xFFFF);
  }
  return -1;
}

static void
mii_write(struct r8169 *r, int reg, uint16 val)
{
  if (!r->gen_g) {
    phyar_write(r, reg, val);
    return;
  }
  if (reg == 0x1F) {
    r->ocp_base = val ? (uint16)(val << 4) : OCP_STD_PHY_BASE;
    return;
  }
  if (r->ocp_base != OCP_STD_PHY_BASE)
    reg -= 0x10;
  ocp_phy_write(r, (uint16)(r->ocp_base + reg * 2), val);
}

static int
mii_read(struct r8169 *r, int reg)
{
  if (!r->gen_g)
    return phyar_read(r, reg);
  if (r->ocp_base != OCP_STD_PHY_BASE)
    reg -= 0x10;
  return ocp_phy_read(r, (uint16)(r->ocp_base + reg * 2));
}

// The MAC's OCP registers. No completion flag to wait on.
static uint16
mac_ocp_read(struct r8169 *r, uint16 reg)
{
  wr32(r, R_MAC_OCP, (uint32)(reg >> 1) << 16);
  return (uint16)rd32(r, R_MAC_OCP);
}

static void
mac_ocp_write(struct r8169 *r, uint16 reg, uint16 val)
{
  wr32(r, R_MAC_OCP, OCP_FLAG | ((uint32)(reg >> 1) << 16) | val);
}

// The extended (ERI) registers. `mask` picks the bytes, bit 0 the lowest:
// several of these registers share a word with ones that must not change.
static void
eri_write_mask(struct r8169 *r, int addr, int mask, uint32 val)
{
  wr32(r, R_ERIDR, val);
  wr32(r, R_ERIAR, OCP_FLAG | ((uint32)mask << 12) | addr);
  for (int i = 0; i < 100; i++) {
    microdelay(10);
    if (!(rd32(r, R_ERIAR) & OCP_FLAG))
      return;
  }
}

// All four bytes at once.
static uint32
eri_read(struct r8169 *r, int addr)
{
  wr32(r, R_ERIAR, (0xFu << 12) | addr);
  for (int i = 0; i < 100; i++) {
    microdelay(10);
    if (rd32(r, R_ERIAR) & OCP_FLAG)
      return rd32(r, R_ERIDR);
  }
  return 0;
}

static void
eri_write(struct r8169 *r, int addr, uint32 val)
{
  wr32(r, R_ERIDR, val);
  wr32(r, R_ERIAR, OCP_FLAG | (0xFu << 12) | addr);
  for (int i = 0; i < 100; i++) {
    microdelay(10);
    if (!(rd32(r, R_ERIAR) & OCP_FLAG))
      return;
  }
}

// Which XIDs are the 8168G generation and later: 8168G, GU, 8411B, EP, H.
// They share the OCP windows and the firmware handover below.
static int
r8169_gen_g(uint32 xid)
{
  switch (xid) {
  case 0x4C0: case 0x4C1: case 0x5C8: case 0x509:
  case 0x500: case 0x501: case 0x502:
  case 0x540: case 0x541: case 0x6C0:
    return 1;
  }
  return 0;
}

static void
wait_ll_ready(struct r8169 *r)
{
  for (int i = 0; i < 42; i++) {
    if (rd8(r, R_MCU) & MCU_LL_READY)
      return;
    microdelay(100);
  }
}

// Taking the chip from the firmware. A laptop that was last running Windows,
// or that keeps Wake-on-LAN armed, hands it over in out-of-band mode with the
// receive path gated off: the MAC answers its registers and moves no frames.
static void
r8169_takeover_g(struct r8169 *r)
{
  wr32(r, R_MISC, rd32(r, R_MISC) | MISC_RXDV_GATE);
  wr8(r, R_CR, rd8(r, R_CR) & ~(CR_TE | CR_RE));
  microdelay(1000);
  wr8(r, R_MCU, rd8(r, R_MCU) & ~MCU_NOW_IS_OOB);
  mac_ocp_write(r, 0xE8DE, mac_ocp_read(r, 0xE8DE) & ~(1u << 14));
  wait_ll_ready(r);
  mac_ocp_write(r, 0xE8DE, mac_ocp_read(r, 0xE8DE) | (1u << 15));
  wait_ll_ready(r);
}

// --- the 8168H as Linux sets it up ------------------------------------------
//
// From Linux 6.18 drivers/net/ethernet/realtek/r8169_main.c
// (rtl_hw_start_8168h_1, rtl_hw_aspm_clkreq_enable) and r8169_phy_config.c
// (rtl8168h_2_hw_phy_config), GPL-2.0, carried over register for register.
// Without them the chip runs on its power-on FIFO sizes and PHY tuning and
// with ASPM as the firmware left it, and moves frames, slowly and with drops.
// The one part left out is the PHY firmware patch (rtl_nic/rtl8168h-2.fw),
// which needs the root filesystem this runs before.

static void
ephy_write(struct r8169 *r, int reg, uint16 val)
{
  wr32(r, R_EPHYAR, 0x80000000u | ((uint32)(reg & 0x1F) << 16) | val);
  for (int i = 0; i < 100; i++) {
    microdelay(10);
    if (!(rd32(r, R_EPHYAR) & 0x80000000u))
      break;
  }
  microdelay(10);
}

static uint16
ephy_read(struct r8169 *r, int reg)
{
  wr32(r, R_EPHYAR, (uint32)(reg & 0x1F) << 16);
  for (int i = 0; i < 100; i++) {
    microdelay(10);
    uint32 v = rd32(r, R_EPHYAR);
    if (v & 0x80000000u)
      return (uint16)v;
  }
  return 0xFFFF;
}

static void
mac_ocp_modify(struct r8169 *r, uint16 reg, uint16 clear, uint16 set)
{
  mac_ocp_write(r, reg, (mac_ocp_read(r, reg) & ~clear) | set);
}

static int
phy_read_paged(struct r8169 *r, int page, int reg)
{
  mii_write(r, 0x1F, (uint16)page);
  int v = mii_read(r, reg);
  mii_write(r, 0x1F, 0);
  return v;
}

static void
phy_write_paged(struct r8169 *r, int page, int reg, uint16 val)
{
  mii_write(r, 0x1F, (uint16)page);
  mii_write(r, reg, val);
  mii_write(r, 0x1F, 0);
}

static void
phy_modify_paged(struct r8169 *r, int page, int reg, uint16 clear, uint16 set)
{
  mii_write(r, 0x1F, (uint16)page);
  int v = mii_read(r, reg);
  if (v >= 0)
    mii_write(r, reg, (uint16)((v & ~clear) | set));
  mii_write(r, 0x1F, 0);
}

static void
phy_param(struct r8169 *r, uint16 parm, uint16 clear, uint16 set)
{
  mii_write(r, 0x1F, 0x0A43);
  mii_write(r, 0x13, parm);
  int v = mii_read(r, 0x14);
  if (v >= 0)
    mii_write(r, 0x14, (uint16)((v & ~clear) | set));
  mii_write(r, 0x1F, 0);
}

// PCIe Link Control, low two bits: L0s and L1. Linux clears L1 on this chip
// and on the bridge above it unless the vendor has marked it safe, for "random
// device stop working problems as well as full system hangs". Firmware
// leaves it on. A link that drops into L1 between frames costs a wake-up per
// burst and on some boards stalls outright, which is a download that crawls.
static void
aspm_off(struct pci_dev *d)
{
  uint8 cap = pci_find_cap(d, 0x10);
  if (cap) {
    uint32 v = pci_read32(d->bus, d->slot, d->func, cap + 0x10);
    pci_write32(d->bus, d->slot, d->func, cap + 0x10, v & ~3u);
  }
}

static void
r8169_aspm_off(struct r8169 *r)
{
  struct pci_dev *d = r->pdev;

  // The vendor's flag that ASPM is safe on this board (rtl_aspm_is_safe).
  if (mac_ocp_read(r, 0xC0B2) & 0xF)
    return;
  aspm_off(d);
  for (int i = 0; i < pci_count(); i++) {
    struct pci_dev *b = pci_get(i);
    if (b && (b->header_type & 0x7F) == 1 &&
        pci_read8(b->bus, b->slot, b->func, 0x19) == d->bus)
      aspm_off(b);
  }
  // And in the chip: no L1.2 trigger, no clock request, ASPM off.
  mac_ocp_modify(r, 0xE092, 0x00FF, 0);
  wr8(r, R_CONFIG2, rd8(r, R_CONFIG2) & ~0x80);
  wr8(r, R_CONFIG5, rd8(r, R_CONFIG5) & ~0x01);
}

// rtl_hw_start_8168h_1. Config registers must be unlocked.
static void
r8169_start_8168h(struct r8169 *r)
{
  static const struct { uint8 reg; uint16 clear, set; } ephy[] = {
    { 0x1E, 0x0800, 0x0001 },
    { 0x1D, 0x0000, 0x0800 },
    { 0x05, 0xFFFF, 0x2089 },
    { 0x06, 0xFFFF, 0x5881 },
    { 0x04, 0xFFFF, 0x854A },
    { 0x01, 0xFFFF, 0x068B },
  };
  for (unsigned i = 0; i < sizeof ephy / sizeof ephy[0]; i++)
    ephy_write(r, ephy[i].reg,
               (ephy_read(r, ephy[i].reg) & ~ephy[i].clear) | ephy[i].set);

  eri_write(r, 0xC8, (0x08u << 16) | 0x02);     // FIFO sizes
  eri_write(r, 0xE8, (0x10u << 16) | 0x06);
  eri_write_mask(r, 0xCC, 0x1, 0x38);           // pause thresholds
  eri_write_mask(r, 0xD0, 0x1, 0x48);

  eri_write(r, 0xDC, eri_read(r, 0xDC) & ~1u);  // reset the packet filter
  eri_write(r, 0xDC, eri_read(r, 0xDC) | 1u);
  eri_write(r, 0xDC, eri_read(r, 0xDC) | 0x1C);
  eri_write_mask(r, 0x5F0, 0x3, 0x4F87);
  wr32(r, R_MISC, rd32(r, R_MISC) & ~MISC_RXDV_GATE);
  eri_write_mask(r, 0xC0, 0x3, 0);
  eri_write_mask(r, 0xB8, 0x3, 0);

  wr8(r, R_EEE_LED, rd8(r, R_EEE_LED) & ~0x07); // rtl8168_config_eee_mac
  eri_write(r, 0x1B0, eri_read(r, 0x1B0) | 0x3);

  wr8(r, R_DLLPR, rd8(r, R_DLLPR) & ~0x40);     // PFM off
  wr8(r, R_MISC1, rd8(r, R_MISC1) & ~0x40);     // PFM in D3cold off
  wr8(r, R_DLLPR, rd8(r, R_DLLPR) & ~0x80);     // 10M power saving off
  eri_write(r, 0x1B0, eri_read(r, 0x1B0) & ~(1u << 12));
  wr8(r, R_CONFIG3, rd8(r, R_CONFIG3) & ~0x02); // Rdy_to_L23 off

  int saw = phy_read_paged(r, 0x0C42, 0x13);
  if (saw > 0 && (saw & 0x3FFF) > 0)
    mac_ocp_modify(r, 0xD412, 0x0FFF, (16000000 / (saw & 0x3FFF)) & 0x0FFF);

  mac_ocp_modify(r, 0xE056, 0x00F0, 0x0000);
  mac_ocp_modify(r, 0xE052, 0x6000, 0x8008);
  mac_ocp_modify(r, 0xE0D6, 0x01FF, 0x017F);
  mac_ocp_modify(r, 0xD420, 0x0FFF, 0x047F);
  mac_ocp_write(r, 0xE63E, 0x0001);
  mac_ocp_write(r, 0xE63E, 0x0000);
  mac_ocp_write(r, 0xC094, 0x0000);
  mac_ocp_write(r, 0xC09E, 0x0000);
}

// rtl8168h_2_hw_phy_config, less the firmware patch.
static void
r8169_phy_8168h(struct r8169 *r)
{
  phy_param(r, 0x808A, 0x003F, 0x000A);         // CHIN EST parameter
  phy_param(r, 0x0811, 0x0000, 0x0800);         // R-tune and PGA-retune on
  phy_modify_paged(r, 0x0A42, 0x16, 0x0000, 0x0002);
  phy_modify_paged(r, 0x0A44, 0x11, 0, 1u << 11); // gphy 10M

  // ADC bias offset, from the MAC's OCP space.
  mac_ocp_write(r, 0xDD02, 0x807D);
  uint16 d1 = mac_ocp_read(r, 0xDD02), d2 = mac_ocp_read(r, 0xDD00);
  uint16 ioffset = ((d2 >> 1) & 0x7FF8) | (d2 & 0x0007) | ((d1 & 0x80) ? 0x8000 : 0);
  if (ioffset != 0xFFFF)
    phy_write_paged(r, 0x0BCF, 0x16, ioffset);

  // TX low-pass corner.
  int data = phy_read_paged(r, 0x0BCD, 0x16);
  if (data >= 0) {
    uint16 rlen = (data & 0xF) > 3 ? (data & 0xF) - 3 : 0;
    phy_write_paged(r, 0x0BCD, 0x17, rlen | rlen << 4 | rlen << 8 | rlen << 12);
  }

  phy_modify_paged(r, 0x0A44, 0x11, 1u << 7, 0); // PHY PFM off
  phy_modify_paged(r, 0x0A43, 0x10, 1u << 0, 0); // 10M PLL off: off
  phy_modify_paged(r, 0x0A43, 0x10, 1u << 2, 0); // ALDPS off
  phy_modify_paged(r, 0x0A43, 0x11, 0, 1u << 4); // EEE
}

// The PHY up and negotiating: page 0, out of power-down, autoneg on and
// restarted. Power-down is what firmware leaves set after a shutdown, and
// the old code ORed autoneg into BMCR and kept it.
static void
r8169_phy_up(struct r8169 *r)
{
  mii_write(r, 0x1F, 0);
  int bmcr = mii_read(r, 0);
  if (bmcr < 0)
    return;
  bmcr &= ~BMCR_PDOWN;
  mii_write(r, 0, (uint16)bmcr);
  if (r->gen_g) {
    // Wait for the PHY to report it is on: its state, low three bits of
    // OCP 0xA420, reads 3 once it is.
    for (int i = 0; i < 100; i++) {
      int st = ocp_phy_read(r, 0xA420);
      if (st >= 0 && (st & 7) == 3)
        break;
      microdelay(1000);
    }
  }
  mii_write(r, 0, (uint16)(bmcr | BMCR_ANENABLE | BMCR_ANRESTART));
}

// --- the class ops ----------------------------------------------------------

static const unsigned char *
r8169_mac_op(struct device *d)
{
  struct r8169 *r = of(d);
  static const unsigned char zero[6];
  return r ? r->mac : zero;
}

static int
r8169_link_up(struct device *d)
{
  struct r8169 *r = of(d);
  return r ? (rd8(r, R_PHYSTATUS) & PHYSTATUS_LINK) != 0 : 0;
}

static int
r8169_transmit(struct device *d, const void *frame, int len)
{
  struct r8169 *r = of(d);
  struct rtl_desc *t;
  int idx;

  if (!r || !r->ready)
    return -1;
  if (len <= 0 || len > BUFSZ)
    return -1;

  acquire(&r->lock);
  idx = r->tx_next;
  t = &r->tx[idx];

  // Still the chip's? Then the ring is full. Congestion rather than failure:
  // the caller retries, which is what the netdev contract says -1 means.
  if (t->opts1 & DESC_OWN) {
    release(&r->lock);
    return -1;
  }

  // The runt rule. Ethernet's minimum frame is 60 bytes before the FCS, and
  // this chip does not pad for you -- a shorter frame goes out short and the
  // other end drops it. ARP requests are 42 bytes, so getting this wrong is
  // not a corner case, it is "the machine cannot resolve anything".
  int n = len < 60 ? 60 : len;
  memmove(r->txbuf + (uint64)idx * BUFSZ, frame, len);
  if (n > len)
    memset(r->txbuf + (uint64)idx * BUFSZ + len, 0, n - len);

  t->addr  = r->txbuf_phys + (uint64)idx * BUFSZ;
  t->opts2 = 0;
  __sync_synchronize();
  t->opts1 = DESC_OWN | DESC_FS | DESC_LS | (uint32)n |
             (idx == NTX - 1 ? DESC_EOR : 0);
  __sync_synchronize();

  // Poke it. Without this the chip may sit on a ring it already owns until
  // something else wakes it.
  wr8(r, R_TPPOLL, TPPOLL_NPQ);

  r->tx_next = (idx + 1) % NTX;
  r->tx_packets++;
  release(&r->lock);
  return 0;
}

static int
r8169_receive(struct device *d, void *out, int cap)
{
  struct r8169 *r = of(d);
  struct rtl_desc *rx;
  int idx, len;

  if (!r || !r->ready)
    return 0;

  acquire(&r->lock);
  idx = r->rx_next;
  rx  = &r->rx[idx];

  if (rx->opts1 & DESC_OWN) {          // still the chip's: nothing arrived
    release(&r->lock);
    return 0;
  }

  // The length includes the four-byte FCS, which the stack does not want and
  // lwIP would otherwise carry to the end of the world as trailing garbage.
  len = (int)(rx->opts1 & DESC_LEN) - 4;
  if (len < 0)
    len = 0;
  if (len > cap)
    len = cap;
  if (len > 0)
    memmove(out, r->rxbuf + (uint64)idx * BUFSZ, len);

  // Hand the descriptor back. Address and size first, OWN last: the chip may
  // start filling it the instant OWN is set.
  rx->addr  = r->rxbuf_phys + (uint64)idx * BUFSZ;
  rx->opts2 = 0;
  __sync_synchronize();
  rx->opts1 = DESC_OWN | (uint32)BUFSZ | (idx == NRX - 1 ? DESC_EOR : 0);
  __sync_synchronize();

  r->rx_next = (idx + 1) % NRX;
  r->rx_packets++;
  release(&r->lock);
  return len;
}

// --- interrupt --------------------------------------------------------------

static void
r8169_intr(void *arg)
{
  struct r8169 *r = arg;
  uint16 isr;

  if (!r || !r->mmio)
    return;

  isr = rd16(r, R_ISR);
  if (!isr)
    return;

  // Write-1-to-clear, and it has to happen whatever else does: a cause left
  // asserted on a level-triggered PCI line re-enters this handler until the
  // machine does nothing else. See docs/history/kernel.md section 13 for the
  // last time that was learned here.
  wr16(r, R_ISR, isr);

  // Nothing is read from the ring here, for the same reason as the e1000: the
  // frame is pulled by the network thread, which may sleep on the buffer cache
  // and take the lwIP lock, and an interrupt handler may do neither. All this
  // does is end that thread's wait early.
  if (isr & (ISR_ROK | ISR_RDU))
    netdev_rx_signal();
}

// --- bring-up ---------------------------------------------------------------

static int
rings_init(struct r8169 *r)
{
  // 256-byte alignment is the chip's requirement for both descriptor bases,
  // and it is not advisory: a misaligned base is a ring the chip walks
  // somewhere else entirely.
  r->rx = dma_alloc(NRX * sizeof(struct rtl_desc), 256);
  r->tx = dma_alloc(NTX * sizeof(struct rtl_desc), 256);
  r->rxbuf = dma_alloc(NRX * BUFSZ, 8);
  r->txbuf = dma_alloc(NTX * BUFSZ, 8);
  if (!r->rx || !r->tx || !r->rxbuf || !r->txbuf)
    return -1;

  r->rx_phys    = dma_phys(r->rx);
  r->tx_phys    = dma_phys(r->tx);
  r->rxbuf_phys = dma_phys(r->rxbuf);
  r->txbuf_phys = dma_phys(r->txbuf);

  memset(r->rx, 0, NRX * sizeof(struct rtl_desc));
  memset(r->tx, 0, NTX * sizeof(struct rtl_desc));

  for (int i = 0; i < NRX; i++) {
    r->rx[i].addr  = r->rxbuf_phys + (uint64)i * BUFSZ;
    r->rx[i].opts1 = DESC_OWN | (uint32)BUFSZ |
                     (i == NRX - 1 ? DESC_EOR : 0);
  }
  for (int i = 0; i < NTX; i++)
    r->tx[i].opts1 = (i == NTX - 1 ? DESC_EOR : 0);

  r->rx_next = 0;
  r->tx_next = 0;
  return 0;
}

static int
r8169_probe(struct pci_dev *d)
{
  struct r8169 *r = 0;
  uint64 bar = 0;
  uint64 barlen = 0;

  for (int i = 0; i < R8169_MAX_NICS; i++)
    if (!nics[i].ready) { r = &nics[i]; break; }
  if (!r)
    return -1;

  // BAR 2 on a PCIe part, BAR 1 on the older PCI ones, and BAR 0 is I/O on
  // both. Picking the first memory BAR rather than a number is what makes one
  // driver cover the family.
  for (int i = 0; i < 6; i++)
    if (d->bar[i].base && !d->bar[i].is_io) {
      bar = d->bar[i].base;
      barlen = d->bar[i].size;
      break;
    }
  if (!bar) {
    cprintf("r8169: no memory BAR; not driving this device\n");
    return -1;
  }

  r->pdev = d;
  r->ocp_base = OCP_STD_PHY_BASE;
  initlock(&r->lock, "r8169");
  pci_enable(d, PCI_CMD_MEMORY | PCI_CMD_BUS_MASTER);

  // ioremap, not P2V: a 64-bit BAR on a laptop routinely sits above the
  // 4 GB the direct map covers, and this is the same lesson the virtio-gpu
  // driver's comment records.
  r->mmio = ioremap(bar, barlen ? barlen : 0x1000);
  if (!r->mmio) {
    cprintf("r8169: cannot map registers at %p\n", (void *)bar);
    return -1;
  }

  r->gen_g = r8169_gen_g((rd32(r, R_TCR) >> 20) & 0xFCF);
  if (r->gen_g)
    r8169_takeover_g(r);

  // Soft reset, and wait for the chip to say it finished.
  wr8(r, R_CR, CR_RST);
  for (int i = 0; i < 1000; i++) {
    microdelay(100);
    if (!(rd8(r, R_CR) & CR_RST))
      break;
  }
  if (rd8(r, R_CR) & CR_RST) {
    cprintf("r8169: the chip did not come out of reset\n");
    return -1;
  }

  for (int i = 0; i < 6; i++)
    r->mac[i] = rd8(r, R_IDR0 + i);

  if (rings_init(r) != 0) {
    cprintf("r8169: no DMA memory for the rings\n");
    return -1;
  }

  wr8(r, R_9346CR, C9346_UNLOCK);       // config registers are writable now

  if (r->gen_g) {
    // The PLL powered, the extra ERI bits the 8168H wants with it, and the
    // receive gate opened again.
    wr8(r, R_PMCH, rd8(r, R_PMCH) | 0xC0);
    uint32 xid = (rd32(r, R_TCR) >> 20) & 0xFCF;
    if (xid == 0x540 || xid == 0x541)
      eri_write(r, 0x1A8, eri_read(r, 0x1A8) | 0xFC000000);
    wr32(r, R_MISC, rd32(r, R_MISC) & ~MISC_RXDV_GATE);
    if (xid == 0x541 || xid == 0x6C0) {
      r8169_aspm_off(r);
      r8169_start_8168h(r);
    }
  }

  // CPCR is deliberately left alone.
  //
  // An earlier version set bit 3 here with a comment calling it "PCI multiple
  // read/write". Bit 3 is PCIDAC -- dual address cycle, which is 64-bit DMA
  // addressing -- and the comment was describing bit 4. Neither is wanted:
  // this driver's rings come from the DMA arena, which is below 4 GB, so DAC
  // would be asking the chip to do something it does not need to do on the
  // strength of a constant nobody had checked.
  //
  // The descriptor registers written below *are* the C+ ones on this part;
  // unlike the 8139 it has no other programming model to be in.

  wr16(r, R_RMS, BUFSZ);
  // EarlySize on the 8168E-VL and later, TxPacketMax before, as in Linux.
  wr8(r, R_MTPS, r->gen_g ? 0x27 : 0x3B);
  wr16(r, R_INTRMIT, 0);                // no interrupt coalescing

  wr64(r, R_RDSAR, r->rx_phys);
  wr64(r, R_TNPDS, r->tx_phys);

  wr32(r, R_TCR, TCR_IFG_STD | TCR_MXDMA | (r->gen_g ? TCR_AUTO_FIFO : 0));
  wr32(r, R_RCR, RCR_APM | RCR_AM | RCR_AB |
                 (r->gen_g ? RCR_GEN_G : RCR_UNLIMITED));

  // Multicast: everything. The stack filters, and a hash filter that is wrong
  // drops IPv6 neighbour discovery in a way that looks like a routing problem.
  wr32(r, R_MAR0, 0xFFFFFFFF);
  wr32(r, R_MAR0 + 4, 0xFFFFFFFF);

  wr8(r, R_CR, CR_RE | CR_TE);
  wr8(r, R_9346CR, C9346_LOCK);

  {
    uint32 xid = (rd32(r, R_TCR) >> 20) & 0xFCF;
    if (xid == 0x541 || xid == 0x6C0)
      r8169_phy_8168h(r);
  }
  r8169_phy_up(r);

  wr16(r, R_ISR, 0xFFFF);               // clear anything left over
  wr16(r, R_IMR, ISR_ROK | ISR_RER | ISR_TOK | ISR_TER |
                 ISR_RDU | ISR_LINKCHG);

  // MSI first: every 8111 has it, and on the boards these are soldered to it
  // is the only interrupt that is sure to arrive. See pci_irq_attach().
  if (pci_irq_attach(d, r8169_intr, r, "r8169", 0) < 0)
    kinfo("r8169: no usable interrupt line; the link will be polled\n");

  r->ready = 1;

  // Said with cprintf rather than kinfo, and said in full.
  //
  // This driver's first run is on a machine nobody here can attach a debugger
  // to, so the one line it prints has to carry everything a bug report would
  // need: which variant answered, what address it claims, whether the PHY sees
  // anything, and which line it will interrupt on. XID is the hardware version
  // -- the top bits of TxConfig -- and is how Realtek parts are told apart,
  // because the PCI device id is 8168 for two decades of silicon.
  {
    uint32 xid = (rd32(r, R_TCR) >> 20) & 0xFCF;
    int    bmsr = mii_read(r, 1);

    kinfo("r8169: xid %x, %x:%x:%x:%x:%x:%x, irq %d (%s), %s (bmsr %x)\n",
            xid,
            r->mac[0], r->mac[1], r->mac[2], r->mac[3], r->mac[4], r->mac[5],
            d->irq_line, pci_irq_kind(d),
            (rd8(r, R_PHYSTATUS) & PHYSTATUS_LINK) ? "link up" : "link down",
            bmsr);
  }
  return 0;
}

static void
r8169_attached(struct pci_dev *d, struct device *dev)
{
  for (int i = 0; i < R8169_MAX_NICS; i++)
    if (nics[i].ready && nics[i].pdev == d) {
      dev->priv = &nics[i];
      nics[i].ndev = dev;
      return;
    }
}

static const struct netdev_ops r8169_netdev_ops = {
  .transmit = r8169_transmit,
  .receive  = r8169_receive,
  .mac      = r8169_mac_op,
  .link_up  = r8169_link_up,
};

// The whole RTL8111/8168 family answers to 8168, which is why one entry covers
// most of the laptops this will ever run on. The others are the older PCI
// 8169, the Fast Ethernet 8101/8106, and the 2.5G 8125 -- listed because they
// share this programming model, and untested because none of them is here.
static const struct pci_match r8169_match[] = {
  { 0x10EC, 0x8168, 0, 0 },     // RTL8111/8168/8411 -- includes the 8111H
  { 0x10EC, 0x8169, 0, 0 },     // the original gigabit part
  { 0x10EC, 0x8136, 0, 0 },     // RTL8101/8102/8106, Fast Ethernet
  { 0x10EC, 0x8161, 0, 0 },
  { 0x10EC, 0x2502, 0, 0 },
  { 0x10EC, 0x2600, 0, 0 },
  { 0, 0, 0, 0 },
};

const struct pci_driver r8169_driver = {
  .name    = "r8169",
  .devname = "eth",
  .match   = r8169_match,
  .ops     = &r8169_netdev_ops,
  .probe   = r8169_probe,
  .attached = r8169_attached,
};