SchOS/kernel/drivers/net/rtl8139/rtl8139.c

// Realtek RTL8139 Fast Ethernet.
//
// The other Realtek, and a genuinely different chip from the RTL8111 next
// door despite the neighbouring name. The 8139 has no descriptor rings: the
// receiver is **one circular buffer** the chip writes packets into back to
// back, each behind a four-byte header, and the transmitter is **four fixed
// slots** used round-robin. Sharing a driver with the 8169 would mean two
// programming models behind one set of function pointers, which is how a
// driver ends up with a mode flag threaded through every path.
//
// It is here for two reasons. It is what a great many older machines have --
// and it is the one Realtek part QEMU emulates, so unlike the 8169 beside it
// every line of this has been run.

#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "arch/x86.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "fs/fs.h"
#include "fs/file.h"
#include "arch/memlayout.h"
#include "arch/mmu.h"
#include "dev/pci.h"
#include "dev/device.h"
#include "dev/netdev.h"
#include "core/dma.h"
#include "core/irq.h"

// --- registers --------------------------------------------------------------

#define R_IDR0     0x00    // station address
#define R_MAR0     0x08    // multicast filter
#define R_TSD0     0x10    // four transmit status registers
#define R_TSAD0    0x20    // four transmit buffer addresses
#define R_RBSTART  0x30    // receive buffer, 32-bit physical
#define R_CR       0x37    // command
#define R_CAPR     0x38    // how far the driver has read
#define R_CBR      0x3A    // how far the chip has written
#define R_IMR      0x3C
#define R_ISR      0x3E
#define R_TCR      0x40
#define R_RCR      0x44
#define R_9346CR   0x50
#define R_CONFIG1  0x52
#define R_MSR      0x58    // media status

#define CR_RST     0x10
#define CR_RE      0x08
#define CR_TE      0x04
#define CR_BUFE    0x01    // receive buffer empty

#define TSD_OWN    0x00002000   // the chip has taken the data
#define TSD_TOK    0x00008000   // and put it on the wire
#define TSD_SIZE   0x00001FFF

#define ISR_ROK    0x0001
#define ISR_RER    0x0002
#define ISR_TOK    0x0004
#define ISR_TER    0x0008
#define ISR_RXOVW  0x0010
#define ISR_LINKCHG 0x0020
#define ISR_FOVW   0x0040

#define MSR_LINKB  0x04    // link *bad* -- the sense is inverted

// Accept our own address, broadcast and multicast; no promiscuous. WRAP lets
// the chip run a packet past the end of the buffer rather than splitting it,
// which is why the allocation below is larger than the buffer length it
// declares. MXDMA and RXFTH are both "unlimited".
#define RCR_APM    0x00000002
#define RCR_AM     0x00000004
#define RCR_AB     0x00000008
#define RCR_WRAP   0x00000080
#define RCR_RBLEN_8K 0x00000000
#define RCR_UNLIMITED 0x0000E700

// 8 KB of buffer, 16 bytes the chip requires past it, and one maximum frame of
// slack because WRAP is set. Getting this last part wrong is a chip writing
// over whatever follows the allocation, at line rate.
#define RXBUF_LEN   8192
#define RXBUF_PAD   16
#define RXBUF_SLACK 1536
#define RXBUF_TOTAL (RXBUF_LEN + RXBUF_PAD + RXBUF_SLACK)

#define NTX         4
#define TXBUF       2048

#define RTL8139_MAX_NICS 2

struct rtl8139 {
  volatile uint8 *mmio;
  uint8  *rxbuf, *txbuf;
  uint64 rxbuf_phys, txbuf_phys;

  int    rx_ptr;                // our offset into the circular buffer
  int    tx_slot;               // the next of the four to use
  uint8  mac[6];
  int    ready;
  uint64 rx_packets, tx_packets;

  struct spinlock lock;
  struct pci_dev *pdev;
  struct device  *ndev;
};

static struct rtl8139 nics[RTL8139_MAX_NICS];

static struct rtl8139 *
of(struct device *d)
{
  return d ? (struct rtl8139 *)d->priv : 0;
}

static inline uint8  rd8 (struct rtl8139 *r, int o) { return *(volatile uint8  *)(r->mmio + o); }
static inline uint16 rd16(struct rtl8139 *r, int o) { return *(volatile uint16 *)(r->mmio + o); }
static inline uint32 rd32(struct rtl8139 *r, int o) { return *(volatile uint32 *)(r->mmio + o); }
static inline void   wr8 (struct rtl8139 *r, int o, uint8 v)  { *(volatile uint8  *)(r->mmio + o) = v; }
static inline void   wr16(struct rtl8139 *r, int o, uint16 v) { *(volatile uint16 *)(r->mmio + o) = v; }
static inline void   wr32(struct rtl8139 *r, int o, uint32 v) { *(volatile uint32 *)(r->mmio + o) = v; }

// --- the class ops ----------------------------------------------------------

static const unsigned char *
rtl8139_mac_op(struct device *d)
{
  struct rtl8139 *r = of(d);
  static const unsigned char zero[6];
  return r ? r->mac : zero;
}

static int
rtl8139_link_up(struct device *d)
{
  struct rtl8139 *r = of(d);
  // LINKB is set when the link is *down*. Reading it the obvious way round is
  // a driver that reports a cable as unplugged whenever it is plugged in.
  return r ? (rd8(r, R_MSR) & MSR_LINKB) == 0 : 0;
}

static int
rtl8139_transmit(struct device *d, const void *frame, int len)
{
  struct rtl8139 *r = of(d);
  int slot;
  uint32 tsd;

  if (!r || !r->ready)
    return -1;
  if (len <= 0 || len > TXBUF)
    return -1;

  acquire(&r->lock);
  slot = r->tx_slot;
  tsd  = rd32(r, R_TSD0 + slot * 4);

  // A slot is ours again once the chip has taken the data (OWN) -- not once it
  // has reached the wire. Waiting for TOK instead would idle three quarters of
  // the transmitter for no benefit.
  if (r->tx_packets >= NTX && !(tsd & TSD_OWN)) {
    release(&r->lock);
    return -1;                  // all four busy: congestion, so retry
  }

  // Ethernet's minimum is 60 bytes before the FCS and this chip does not pad.
  // An ARP request is 42, so a driver that skips this cannot resolve anything.
  int n = len < 60 ? 60 : len;
  memmove(r->txbuf + (uint64)slot * TXBUF, frame, len);
  if (n > len)
    memset(r->txbuf + (uint64)slot * TXBUF + len, 0, n - len);

  wr32(r, R_TSAD0 + slot * 4, (uint32)(r->txbuf_phys + (uint64)slot * TXBUF));
  __sync_synchronize();
  // Writing the size is what starts it: this clears OWN, and the chip sets OWN
  // again when it has the data. The early-transmit threshold in bits 16-21 is
  // left at zero, which means "start when the whole frame is in the FIFO" --
  // the safe end of a trade whose other end is underruns.
  wr32(r, R_TSD0 + slot * 4, (uint32)n);

  r->tx_slot = (slot + 1) % NTX;
  r->tx_packets++;
  release(&r->lock);
  return 0;
}

static int
rtl8139_receive(struct device *d, void *out, int cap)
{
  struct rtl8139 *r = of(d);
  uint32 hdr;
  int status, len, avail;

  if (!r || !r->ready)
    return 0;

  acquire(&r->lock);

  if (rd8(r, R_CR) & CR_BUFE) {        // nothing waiting
    release(&r->lock);
    return 0;
  }

  // The four-byte header the chip writes in front of every packet: status in
  // the low half, length in the high half. The length counts the 4-byte CRC.
  hdr    = *(volatile uint32 *)(r->rxbuf + r->rx_ptr);
  status = (int)(hdr & 0xFFFF);
  len    = (int)((hdr >> 16) & 0xFFFF);

  // 0xFFF0 is the chip saying "still writing this one". Treating it as a
  // length is a driver that copies a partial packet and then loses the buffer
  // alignment for everything after it.
  if (len == 0xFFF0 || len == 0) {
    release(&r->lock);
    return 0;
  }

  if (!(status & 0x0001) || len < 8 || len > RXBUF_LEN) {
    // A bad frame. Resynchronise rather than try to skip it: the length is
    // the only thing that says where the next one starts, and it is the field
    // that is not to be trusted here.
    wr8(r, R_CR, CR_TE);               // stop the receiver
    r->rx_ptr = 0;
    wr32(r, R_RBSTART, (uint32)r->rxbuf_phys);
    wr16(r, R_CAPR, (uint16)(0 - RXBUF_PAD));
    wr8(r, R_CR, CR_RE | CR_TE);
    release(&r->lock);
    return 0;
  }

  avail = len - 4;                     // drop the CRC the stack does not want
  if (avail > cap)
    avail = cap;

  // The copy may run past the end of the 8 KB window, which is exactly what
  // WRAP is for and why the allocation has a frame of slack: the bytes are
  // there, contiguously, so one memmove is correct.
  if (avail > 0)
    memmove(out, r->rxbuf + r->rx_ptr + 4, avail);

  // Four-byte align the next offset, wrap it, and tell the chip how far we
  // have read. CAPR is *sixteen behind* the true position -- a quirk of the
  // part, not an off-by-one.
  r->rx_ptr = (r->rx_ptr + len + 4 + 3) & ~3;
  r->rx_ptr %= RXBUF_LEN;
  wr16(r, R_CAPR, (uint16)(r->rx_ptr - RXBUF_PAD));

  r->rx_packets++;
  release(&r->lock);
  return avail;
}

// --- interrupt --------------------------------------------------------------

static void
rtl8139_intr(void *arg)
{
  struct rtl8139 *r = arg;
  uint16 isr;

  if (!r || !r->mmio)
    return;

  isr = rd16(r, R_ISR);
  if (!isr)
    return;
  wr16(r, R_ISR, isr);                 // write-1-to-clear, or it re-enters

  // As with every other NIC here: nothing is read from the buffer in interrupt
  // context. This only ends the network thread's wait early.
  if (isr & (ISR_ROK | ISR_RXOVW | ISR_FOVW))
    netdev_rx_signal();
}

// --- probe ------------------------------------------------------------------

static int
rtl8139_probe(struct pci_dev *d)
{
  struct rtl8139 *r = 0;
  uint64 bar = 0, barlen = 0;

  for (int i = 0; i < RTL8139_MAX_NICS; i++)
    if (!nics[i].ready) { r = &nics[i]; break; }
  if (!r)
    return -1;

  // The first memory BAR. BAR 0 is I/O on this part and BAR 1 is the same
  // registers in memory space; using memory keeps this the same shape as
  // every other driver here.
  for (int i = 0; i < 6; i++)
    if (d->bar[i].base && !d->bar[i].is_io) {
      bar = d->bar[i].base;
      barlen = d->bar[i].size;
      break;
    }
  if (!bar) {
    cprintf("rtl8139: no memory BAR; not driving this device\n");
    return -1;
  }

  r->pdev = d;
  initlock(&r->lock, "rtl8139");
  pci_enable(d, PCI_CMD_MEMORY | PCI_CMD_BUS_MASTER);

  r->mmio = ioremap(bar, barlen ? barlen : 0x100);
  if (!r->mmio) {
    cprintf("rtl8139: cannot map registers at %p\n", (void *)bar);
    return -1;
  }

  // Wake the chip if the firmware left it asleep, then reset.
  wr8(r, R_CONFIG1, 0x00);
  wr8(r, R_CR, CR_RST);
  for (int i = 0; i < 10000; i++) {
    microdelay(10);
    if (!(rd8(r, R_CR) & CR_RST))
      break;
  }
  if (rd8(r, R_CR) & CR_RST) {
    cprintf("rtl8139: the chip did not come out of reset\n");
    return -1;
  }

  for (int i = 0; i < 6; i++)
    r->mac[i] = rd8(r, R_IDR0 + i);

  r->rxbuf = dma_alloc(RXBUF_TOTAL, 8);
  r->txbuf = dma_alloc(NTX * TXBUF, 8);
  if (!r->rxbuf || !r->txbuf) {
    cprintf("rtl8139: no DMA memory for the buffers\n");
    return -1;
  }
  r->rxbuf_phys = dma_phys(r->rxbuf);
  r->txbuf_phys = dma_phys(r->txbuf);
  memset(r->rxbuf, 0, RXBUF_TOTAL);

  // A 32-bit part: the buffer address register is 32 bits and there is no
  // dual-address cycle to fall back on. The DMA arena is below 4 GB, so this
  // holds -- but it holds for a reason rather than by luck, and if the arena
  // ever moves this is where it breaks.
  if (r->rxbuf_phys + RXBUF_TOTAL > 0xFFFFFFFFULL ||
      r->txbuf_phys + NTX * TXBUF > 0xFFFFFFFFULL) {
    cprintf("rtl8139: buffers above 4 GB; this chip cannot reach them\n");
    return -1;
  }

  wr32(r, R_RBSTART, (uint32)r->rxbuf_phys);
  r->rx_ptr = 0;
  wr16(r, R_CAPR, (uint16)(0 - RXBUF_PAD));

  wr32(r, R_RCR, RCR_APM | RCR_AM | RCR_AB | RCR_WRAP |
                 RCR_RBLEN_8K | RCR_UNLIMITED);
  wr32(r, R_TCR, 0x00000700);          // unlimited DMA burst
  wr32(r, R_MAR0, 0xFFFFFFFF);         // multicast: everything; the stack filters
  wr32(r, R_MAR0 + 4, 0xFFFFFFFF);

  wr8(r, R_CR, CR_RE | CR_TE);
  wr16(r, R_ISR, 0xFFFF);
  wr16(r, R_IMR, ISR_ROK | ISR_RER | ISR_TOK | ISR_TER |
                 ISR_RXOVW | ISR_FOVW | ISR_LINKCHG);

  pci_irq_attach(d, rtl8139_intr, r, "rtl8139", 0);   // no MSI on this part

  r->ready = 1;
  kinfo("rtl8139: %x:%x:%x:%x:%x:%x, %s, irq %d\n",
        r->mac[0], r->mac[1], r->mac[2], r->mac[3], r->mac[4], r->mac[5],
        (rd8(r, R_MSR) & MSR_LINKB) ? "link down" : "link up", d->irq);
  return 0;
}

// Let go of the device, so another driver can have it.
//
// Without this the kernel's answer to "may I have the 8139" was always no, and
// a hosted driver for it could never run on a machine where this one had
// probed first -- which is every machine with one:
//
//   found   10ec:8139 at 00:04.0, held by rtl8139
//   the kernel would not release it
//
// The order is the one e1000_release() follows and for the same reasons. The
// chip is silenced *before* the handler goes, because a level-triggered line
// left asserted with nobody to clear it is an interrupt storm. Then it is
// reset, which stops the receiver writing into a buffer that is about to be
// freed, and bus mastering is cleared so that nothing it still has queued can
// reach memory. Only then are the buffers given back: the arena will hand them
// to the next driver, and a chip still pointed at them would be writing frames
// into somebody else's descriptors.
static int
rtl8139_release(struct pci_dev *d)
{
  struct rtl8139 *r = 0;

  for (int i = 0; i < RTL8139_MAX_NICS; i++)
    if (nics[i].pdev == d) { r = &nics[i]; break; }

  if (!r || !r->ready)
    return 0;                     // never came up, or already let go

  acquire(&r->lock);
  r->ready = 0;                   // transmit and receive refuse from here on
  wr16(r, R_IMR, 0);              // no more interrupts from it
  wr16(r, R_ISR, 0xFFFF);         // and none left pending
  wr8(r, R_CR, 0);                // receiver and transmitter off
  release(&r->lock);

  pci_irq_detach(d, rtl8139_intr, r);

  // Reset, and wait for it, so the chip has truly stopped before its buffers
  // are reused. The reset bit clears itself when the chip is done.
  wr8(r, R_CR, CR_RST);
  for (int i = 0; i < 10000 && (rd8(r, R_CR) & CR_RST); i++)
    microdelay(10);
  pci_disable(d, PCI_CMD_BUS_MASTER);

  // Out of the stack, for the reason e1000_release() spells out: an interface
  // left behind keeps its address and its route, and the stack goes on sending
  // into a driver that now refuses every frame.
  if (r->ndev)
    net_detach(r->ndev);

  dma_free(r->rxbuf, RXBUF_TOTAL);
  dma_free(r->txbuf, NTX * TXBUF);
  r->rxbuf = r->txbuf = 0;
  r->tx_slot = 0;
  r->tx_packets = 0;
  r->rx_packets = 0;

  cprintf("rtl8139: released; the interface is down\n");
  return 0;
}

static void
rtl8139_attached(struct pci_dev *d, struct device *dev)
{
  for (int i = 0; i < RTL8139_MAX_NICS; i++)
    if (nics[i].ready && nics[i].pdev == d) {
      dev->priv = &nics[i];
      nics[i].ndev = dev;
      return;
    }
}

static const struct netdev_ops rtl8139_netdev_ops = {
  .transmit = rtl8139_transmit,
  .receive  = rtl8139_receive,
  .mac      = rtl8139_mac_op,
  .link_up  = rtl8139_link_up,
};

static const struct pci_match rtl8139_match[] = {
  { 0x10EC, 0x8139, 0, 0 },
  { 0, 0, 0, 0 },
};

const struct pci_driver rtl8139_driver = {
  .name    = "rtl8139",
  .devname = "eth",
  .match   = rtl8139_match,
  .ops     = &rtl8139_netdev_ops,
  .probe   = rtl8139_probe,
  .attached = rtl8139_attached,
  .release = rtl8139_release,
};