SchOS/ldls/kpi/net.c

// SPDX-License-Identifier: Apache-2.0
//
// The network half of the KPI. See ldls/kpi/kpi.h for the provenance rule.
//
// `alloc_etherdev_mqs` is the interesting one and it is where BTF earns its
// keep a second time. A driver reaches its own private area with
// `netdev_priv(dev)`, which is an *inline function* compiled into the driver:
// the net_device, rounded up to an alignment, and the private area after it.
// So the size this allocates is not a free choice -- get it wrong and the
// driver writes its state over the tail of its own control block, and nothing
// says so until something much later reads a field that has been quietly
// overwritten.
//
// The size comes from the same BTF the offsets do.

#include "types.h"
#include "user.h"
#include "fcntl.h"
#include "posix.h"
#include "netuser.h"
#include "kpi.h"

// Filled in by the host from BTF before the driver runs. Zero means the host
// could not find `struct net_device`, in which case allocating is a guess and
// is refused rather than made.
unsigned int kpi_netdev_size;

// NETDEV_ALIGN. Not a layout and not in BTF -- it is a constant in a header,
// and it has been 32 for as long as x86-64 has existed. A wrong value here is
// visible immediately as a driver that reads zeroes out of its own private
// area, which is why it is worth stating rather than hiding in an expression.
#define KPI_NETDEV_ALIGN 32

static void *g_netdev;          // the one interface, for register_netdev

// What alloc_etherdev_mqs handed back.
//
// The host needs it to reach ndo_open and ndo_start_xmit, and it is not the
// pci_dev -- passing that instead reads netdev_ops out of the middle of a
// different struct and finds nothing, which is exactly what happened the first
// time this was wired up.
void *kpi_netdev(void) { return g_netdev; }

void *
kpi_alloc_etherdev_mqs(int priv, unsigned int txqs, unsigned int rxqs)
{
  unsigned long base, total;
  void *p;

  (void)txqs; (void)rxqs;

  if (kpi_netdev_size == 0) {
    kpi_log("kpi: no sizeof(struct net_device) -- cannot allocate one\n");
    return 0;
  }

  base  = (kpi_netdev_size + (KPI_NETDEV_ALIGN - 1)) & ~(unsigned long)(KPI_NETDEV_ALIGN - 1);
  total = base + (unsigned long)priv + KPI_NETDEV_ALIGN;

  p = mmap(0, total, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  if (p == 0 || (long)p == -1)
    return 0;

  // Linux's alloc_etherdev does more than allocate, and one of the things it
  // does is not optional: `dev->dev_addr` is a *pointer*, aimed at the
  // device's own `perm_addr` array. A driver programs its MAC into the chip
  // with `*(u32 *)dev->dev_addr`, so leaving it null is a null dereference
  // inside the driver's hardware setup -- which is exactly where this stopped
  // the first time.
  if (kpi_l.nd_dev_addr && kpi_l.nd_perm_addr) {
    unsigned char *d = (unsigned char *)p;
    unsigned char *perm = d + kpi_l.nd_perm_addr;
    int i;
    for (i = 0; i < 8; i++)
      d[kpi_l.nd_dev_addr + i] = (unsigned char)((unsigned long)perm >> (i * 8));
    if (kpi_l.nd_addr_len)
      d[kpi_l.nd_addr_len] = 6;
  } else {
    kpi_log("kpi: no net_device.dev_addr offset -- the driver will fault\n");
  }

  // The address lists have to be *empty*, not zeroed.
  //
  // A Linux list_head is empty when it points at itself; zeroed means a null
  // `next`, and the first thing that walks it dereferences that. 8139too walks
  // the multicast list in its receive-mode setup, so this is reached on the
  // way to bringing the interface up -- and it is the second thing Linux's
  // alloc_etherdev does that allocating alone does not.
  if (kpi_l.nd_mc && kpi_l.nd_uc) {
    unsigned int heads[2];
    int h;
    heads[0] = kpi_l.nd_mc;
    heads[1] = kpi_l.nd_uc;
    for (h = 0; h < 2; h++) {
      unsigned char *lh = (unsigned char *)p + heads[h];
      unsigned long self = (unsigned long)lh;
      int i;
      for (i = 0; i < 8; i++) {
        lh[i]     = (unsigned char)(self >> (i * 8));   /* next */
        lh[8 + i] = (unsigned char)(self >> (i * 8));   /* prev */
      }
    }
  }

  // And the transmit queues, which are an array hanging off the device rather
  // than part of it.
  //
  // `netif_start_queue` is an inline that clears a bit in `dev->_tx[0].state`,
  // so a driver bringing its interface up writes through this pointer before
  // it sends anything. Allocating it is the third thing Linux's alloc_etherdev
  // does that allocating the device alone does not -- and like the other two,
  // it was found by the driver faulting on it rather than by reading a list.
  if (kpi_l.nd_tx && kpi_l.nq_size) {
    unsigned int n = txqs ? txqs : 1;
    unsigned char *q = mmap(0, (unsigned long)n * kpi_l.nq_size,
                            PROT_READ | PROT_WRITE,
                            MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
    if (q && (long)q != -1) {
      unsigned char *d = (unsigned char *)p;
      unsigned int k;
      int i;
      for (i = 0; i < 8; i++)
        d[kpi_l.nd_tx + i] = (unsigned char)((unsigned long)q >> (i * 8));
      for (k = 0; k < n; k++) {
        unsigned char *e = q + (unsigned long)k * kpi_l.nq_size;
        for (i = 0; i < 8; i++)
          e[kpi_l.nq_dev + i] = (unsigned char)((unsigned long)p >> (i * 8));
      }
      for (i = 0; i < 4; i++) {
        d[kpi_l.nd_num_tx + i]      = (unsigned char)(n >> (i * 8));
        d[kpi_l.nd_real_num_tx + i] = (unsigned char)(n >> (i * 8));
      }
    }
  }

  kpi_log("  kpi: net_device "); kpi_lognum(kpi_netdev_size);
  kpi_log(" bytes + "); kpi_lognum(priv); kpi_log(" private\n");

  g_netdev = p;
  return p;
}

void
kpi_free_netdev(void *dev)
{
  (void)dev;
}

// The MAC, as the driver read it off the chip.
//
// This is the payoff of the whole exercise and it is worth being precise about
// why: nothing here invented it. The driver bit-banged the card's EEPROM,
// serially, hundreds of register accesses, and handed the result to this. If
// it matches what the machine was actually configured with, then every layer
// underneath -- the loader, the relocations, the BTF offsets, the BAR
// mapping, the register accessors -- is right.
void
kpi_dev_addr_mod(void *dev, unsigned int off, const unsigned char *addr,
                 unsigned int len)
{
  int i;

  (void)dev;
  if (off != 0 || len != 6 || !addr)
    return;

  for (i = 0; i < 6; i++)
    kpi.mac[i] = addr[i];
  kpi.have_mac = 1;

  // And through dev->dev_addr, which is where the driver will read it back
  // from when it programs the chip.
  if (g_netdev && kpi_l.nd_dev_addr) {
    unsigned char *d = (unsigned char *)g_netdev;
    unsigned long q = 0;
    for (i = 0; i < 8; i++)
      q |= (unsigned long)d[kpi_l.nd_dev_addr + i] << (i * 8);
    if (q)
      for (i = 0; i < 6; i++)
        ((unsigned char *)q)[i] = addr[i];
  }

  kpi_log("  kpi: the driver read a MAC off the hardware: ");
  for (i = 0; i < 6; i++) {
    kpi_loghex(kpi.mac[i], 2);
    if (i < 5) kpi_log(":");
  }
  kpi_log("\n");
}

// Become an interface.
//
// /dev/netdev is the bridge docs/architecture/ldls.md calls the class bridge, built
// and proved at L3a by `netecho` with no hardware behind it at all. Here there
// is: the same four calls, answered by a Linux driver.
int
kpi_register_netdev(void *dev)
{
  struct netuser_create c;
  int i;

  (void)dev;

  if (!kpi.have_mac) {
    kpi_log("kpi: register_netdev before the driver found a MAC\n");
    return -1;
  }

  // Non-blocking, because the host's loop has to give the receive side a turn
  // whether or not the stack has anything to send. netuser_ready() is what
  // makes O_NONBLOCK work here; see struct devsw.
  kpi.netfd = open("/dev/netdev", O_RDWR | O_NONBLOCK);
  if (kpi.netfd < 0) {
    kpi_log("kpi: cannot open /dev/netdev\n");
    return -1;
  }

  for (i = 0; i < (int)sizeof(c.name); i++) c.name[i] = 0;
  for (i = 0; i < 6; i++) c.mac[i] = kpi.mac[i];
  c.pad[0] = c.pad[1] = 0;
  c.name[0] = 'l'; c.name[1] = 'd'; c.name[2] = 'l'; c.name[3] = 's';

  if (ioctl(kpi.netfd, NETUSERIOC_CREATE, &c) < 0) {
    kpi_log("kpi: /dev/netdev is already somebody else's\n");
    close(kpi.netfd);
    kpi.netfd = -1;
    return -1;
  }

  kpi_log("  kpi: registered as an interface, with the hardware's own MAC\n");
  return 0;
}

void
kpi_unregister_netdev(void *dev)
{
  (void)dev;
  if (kpi.netfd >= 0) {
    ioctl(kpi.netfd, NETUSERIOC_RELEASE, 0);
    close(kpi.netfd);
    kpi.netfd = -1;
  }
}

// netif_carrier_off / netif_carrier_on.
//
// These were no-ops, and it cost a day. `netif_carrier_ok()` is an inline that
// tests __LINK_STATE_NOCARRIER in `dev->state`, so with the bit never set a
// driver reads "the carrier is already up" and skips its entire link-up path.
// e1000's link-up path is where it enables the transmitter, so the visible
// symptom was frames accepted and mapped and never sent -- and TCTL read back
// from the chip with every field set but the enable.
//
// The bit number comes from the kernel's own BTF; see btf_enum_value().
void
kpi_netif_carrier_off(void *dev)
{
  unsigned long *st = (unsigned long *)((char *)dev + kpi_l.nd_state);
  *st |= 1UL << kpi_l.link_nocarrier;
}

void
kpi_netif_carrier_on(void *dev)
{
  unsigned long *st = (unsigned long *)((char *)dev + kpi_l.nd_state);
  *st &= ~(1UL << kpi_l.link_nocarrier);
  kpi_log("kpi: the driver says the link is up\n");
}