SchOS/ldls/kpi/base.c

// SPDX-License-Identifier: GPL-2.0-or-later
//
// The rest of the kernel a modern driver stands on: the driver core, device-
// managed resources, printf, time, bitmaps, firmware, and the data symbols.
// See ldls/kpi/kpi.h.
//
// r8169 is the reason for most of it. Unlike 8139too and e1000 it does not
// talk to its PHY itself: it registers an MDIO bus with libphy, and libphy
// finds the PHY on it and binds a PHY driver (realtek.ko) through Linux's
// driver core -- bus_type.match, then device_driver.probe. r8169 then checks
// that a dedicated PHY driver was bound and refuses the device if not. So the
// driver core here is not decoration; it is what makes the device usable.
//
// Several functions below exist because a stub's zero is a lie in their
// case, and each says which lie it was:
//   __list_add_valid_or_report  0 = "corrupt list", and with CONFIG_DEBUG_LIST
//                               every list_add asks, so nothing is ever added;
//   try_module_get              0 = "module going away", phylib refuses the PHY;
//   dev_err_probe               returns its error, or failure looks like success;
//   eth_platform_get_mac_address 0 = "here is your MAC", all zeroes;
//   firmware_request_nowarn     0 = "loaded", with a NULL firmware;
//   __bitmap_*                  phylib computes link modes with them;
//   __const_udelay              a delay of nothing turns every poll-with-
//                               timeout into an immediate timeout.

#include "core/types.h"
#include "uapi/user.h"
#include "uapi/fcntl.h"
#include "uapi/posix.h"
#include "kpi.h"

typedef __builtin_va_list va_list;
#define va_start(a, l) __builtin_va_start(a, l)
#define va_end(a)      __builtin_va_end(a)
#define va_arg(a, t)   __builtin_va_arg(a, t)

extern void *kpi_kmalloc(unsigned long n, unsigned int flags);
extern int   kpi_pci_enable_device(void *pdev);
extern void *kpi_pci_iomap(void *pdev, int bar, unsigned long max);
extern int   kpi_pci_read_config_word(void *pdev, int where, unsigned short *v);
extern void *kpi_alloc_etherdev_mqs(int priv, unsigned int tx, unsigned int rx);
extern void *kpi_dma_alloc_attrs(void *dev, unsigned long size,
                                 unsigned long long *handle, unsigned int gfp,
                                 unsigned long attrs);
extern void *kpi_alloc_pages(unsigned int gfp, unsigned int order);

static void *zalloc(unsigned long n) { return kpi_kmalloc(n, 0); }

static unsigned long rdp(const void *base, unsigned int off)
{ unsigned long v; __builtin_memcpy(&v, (const char *)base + off, 8); return v; }
static void wrp(void *base, unsigned int off, unsigned long v)
{ __builtin_memcpy((char *)base + off, &v, 8); }

static int slen(const char *s) { int n = 0; while (s && s[n]) n++; return n; }

// --- printf ---------------------------------------------------------------------
//
// Linux's format language: C's, plus %p followed by letters that say what the
// pointer is. Of those, only the MAC address (%pM) is worth expanding here;
// the rest print as a pointer and consume their letters, as Linux does, so
// that the text after them is not mistaken for part of the format.

struct out { char *buf; unsigned long cap, n; };

static void put(struct out *o, char c)
{
  if (o->n + 1 < o->cap)
    o->buf[o->n] = c;
  o->n++;
}

static void
putnum(struct out *o, unsigned long long v, int base, int neg, int width,
       int zero, int left, int upper, int prec)
{
  char tmp[24];
  const char *dig = upper ? "0123456789ABCDEF" : "0123456789abcdef";
  int n = 0, len, pad;

  if (v == 0 && prec != 0)
    tmp[n++] = '0';
  while (v) { tmp[n++] = dig[v % (unsigned)base]; v /= (unsigned)base; }
  while (n < prec && n < (int)sizeof(tmp)) tmp[n++] = '0';
  len = n + (neg ? 1 : 0);
  pad = width > len ? width - len : 0;
  if (!left && !zero) while (pad-- > 0) put(o, ' ');
  if (neg) put(o, '-');
  if (!left && zero) while (pad-- > 0) put(o, '0');
  while (n > 0) put(o, tmp[--n]);
  if (left) while (pad-- > 0) put(o, ' ');
}

int
kpi_vsnprintf(char *buf, unsigned long cap, const char *f, va_list ap)
{
  struct out o = { buf, cap, 0 };

  if (!f) f = "";
  while (*f) {
    if (*f != '%') { put(&o, *f++); continue; }
    f++;
    int left = 0, zero = 0, width = 0, prec = -1, lng = 0;
    for (;; f++) {
      if (*f == '-') left = 1;
      else if (*f == '0') zero = 1;
      else if (*f == '+' || *f == ' ' || *f == '#') ;
      else break;
    }
    if (*f == '*') { width = va_arg(ap, int); f++; }
    else while (*f >= '0' && *f <= '9') width = width * 10 + (*f++ - '0');
    if (*f == '.') {
      f++; prec = 0;
      if (*f == '*') { prec = va_arg(ap, int); f++; }
      else while (*f >= '0' && *f <= '9') prec = prec * 10 + (*f++ - '0');
    }
    for (;; f++) {
      if (*f == 'l') lng++;
      else if (*f == 'z' || *f == 't' || *f == 'j') lng = 2;
      else if (*f == 'h') ;
      else break;
    }
    char c = *f ? *f++ : 0;
    switch (c) {
    case 'd': case 'i': {
      long long v = lng ? va_arg(ap, long long) : va_arg(ap, int);
      int neg = v < 0;
      putnum(&o, neg ? (unsigned long long)-v : (unsigned long long)v, 10,
             neg, width, zero, left, 0, prec);
      break;
    }
    case 'u': case 'x': case 'X': case 'o': {
      unsigned long long v = lng ? va_arg(ap, unsigned long long)
                                 : va_arg(ap, unsigned int);
      putnum(&o, v, c == 'u' ? 10 : c == 'o' ? 8 : 16, 0, width, zero, left,
             c == 'X', prec);
      break;
    }
    case 'c':
      put(&o, (char)va_arg(ap, int));
      break;
    case 's': {
      const char *s = va_arg(ap, const char *);
      int n, pad;
      if (!s) s = "(null)";
      n = slen(s);
      if (prec >= 0 && n > prec) n = prec;
      pad = width > n ? width - n : 0;
      if (!left) while (pad-- > 0) put(&o, ' ');
      for (int i = 0; i < n; i++) put(&o, s[i]);
      if (left) while (pad-- > 0) put(&o, ' ');
      break;
    }
    case 'p': {
      const unsigned char *p = va_arg(ap, const unsigned char *);
      if ((*f == 'M' || *f == 'm') && p) {
        for (int i = 0; i < 6; i++) {
          if (i && *f == 'M') put(&o, ':');
          putnum(&o, p[i], 16, 0, 2, 1, 0, 0, -1);
        }
      } else if (*f == 'e') {
        put(&o, 'E');
        long e = (long)p;
        putnum(&o, (unsigned long long)(e < 0 ? -e : e), 10, e < 0, 0, 0, 0, 0, -1);
      } else {
        put(&o, '0'); put(&o, 'x');
        putnum(&o, (unsigned long)p, 16, 0, 0, 0, 0, 0, -1);
      }
      while ((*f >= 'a' && *f <= 'z') || (*f >= 'A' && *f <= 'Z') ||
             (*f >= '0' && *f <= '9'))
        f++;
      break;
    }
    case '%':
      put(&o, '%');
      break;
    default:
      break;
    }
  }
  if (cap)
    buf[o.n < cap ? o.n : cap - 1] = 0;
  return (int)o.n;
}

int
kpi_snprintf(char *buf, unsigned long cap, const char *f, ...)
{
  va_list ap;
  va_start(ap, f);
  int n = kpi_vsnprintf(buf, cap, f, ap);
  va_end(ap);
  return n;
}

int
kpi_sprintf(char *buf, const char *f, ...)
{
  va_list ap;
  va_start(ap, f);
  int n = kpi_vsnprintf(buf, 1UL << 20, f, ap);
  va_end(ap);
  return n;
}

char *
kpi_kvasprintf(const char *f, va_list ap)
{
  char tmp[512];
  va_list ap2;
  __builtin_va_copy(ap2, ap);
  int n = kpi_vsnprintf(tmp, sizeof(tmp), f, ap2);
  va_end(ap2);
  char *s = zalloc((unsigned long)n + 1);
  if (s)
    kpi_vsnprintf(s, (unsigned long)n + 1, f, ap);
  return s;
}

char *
kpi_kasprintf(unsigned int gfp, const char *f, ...)
{
  (void)gfp;
  va_list ap;
  va_start(ap, f);
  char *s = kpi_kvasprintf(f, ap);
  va_end(ap);
  return s;
}

// A kernel message: the level byte off the front, the driver's text after a
// prefix that says whose it is.
static void
emit(const char *who, const char *f, va_list ap)
{
  char line[512];

  if (f && (unsigned char)f[0] == 0x01 && f[1])
    f += 2;
  kpi_vsnprintf(line, sizeof(line), f, ap);
  kpi_log("  ");
  if (who && *who) { kpi_log(who); kpi_log(": "); }
  kpi_log(line);
  int n = slen(line);
  if (n == 0 || line[n - 1] != '\n')
    kpi_log("\n");
}

static const char *
dev_name_of(void *dev)
{
  if (!dev || !kpi_l.dev_kobj_ok)
    return "";
  const char *n = (const char *)rdp(dev, kpi_l.dev_kobj + kpi_l.kobj_name);
  return n ? n : "";
}

int kpi_vprintk(const char *f, va_list ap) { emit(0, f, ap); return 0; }

int
kpi_printk_fmt(const char *f, ...)
{
  va_list ap; va_start(ap, f); emit(0, f, ap); va_end(ap); return 0;
}

void
kpi_dev_printk(void *dev, const char *f, ...)
{
  va_list ap; va_start(ap, f); emit(dev_name_of(dev), f, ap); va_end(ap);
}

// netdev_* take the net_device, whose name is not kept anywhere this can
// read; "ldls0" is what the interface is called on this side anyway.
void
kpi_netdev_printk_fmt(void *nd, const char *f, ...)
{
  (void)nd;
  va_list ap; va_start(ap, f); emit("ldls0", f, ap); va_end(ap);
}

void
kpi_netdev_printk_lvl(const char *lvl, void *nd, const char *f, ...)
{
  (void)lvl; (void)nd;
  va_list ap; va_start(ap, f); emit("ldls0", f, ap); va_end(ap);
}

int
kpi_dev_err_probe(void *dev, int err, const char *f, ...)
{
  va_list ap; va_start(ap, f); emit(dev_name_of(dev), f, ap); va_end(ap);
  return err;
}

void
kpi_warn_printk(const char *f, ...)
{
  kpi_log("  WARNING ");
  va_list ap; va_start(ap, f); emit(0, f, ap); va_end(ap);
}

// --- strings ----------------------------------------------------------------------

int
kpi_strcmp(const char *a, const char *b)
{
  while (*a && *a == *b) { a++; b++; }
  return (unsigned char)*a - (unsigned char)*b;
}

char *
kpi_strcpy(char *d, const char *s)
{
  char *r = d;
  while ((*d++ = *s++)) ;
  return r;
}

const char *
kpi_kstrdup_const(const char *s, unsigned int gfp)
{
  (void)gfp;
  if (!s) return 0;
  char *d = zalloc((unsigned long)slen(s) + 1);
  if (d) kpi_strcpy(d, s);
  return d;
}

int
kpi_kstrtoull(const char *s, unsigned int base, unsigned long long *res)
{
  unsigned long long v = 0;
  int any = 0;
  if (base == 0) {
    base = 10;
    if (s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) { base = 16; s += 2; }
    else if (s[0] == '0') base = 8;
  }
  for (; *s && *s != '\n'; s++) {
    int d = (*s >= '0' && *s <= '9') ? *s - '0' :
            (*s >= 'a' && *s <= 'f') ? *s - 'a' + 10 :
            (*s >= 'A' && *s <= 'F') ? *s - 'A' + 10 : 99;
    if (d >= (int)base) return -22;
    v = v * base + (unsigned)d;
    any = 1;
  }
  if (!any) return -22;
  *res = v;
  return 0;
}

// --- time --------------------------------------------------------------------------
//
// HZ is a compile-time constant in the module. It is 1000 on the kernels this
// has been built against, and jiffies has to count at the module's HZ or
// every delay it computes in jiffies is wrong by the ratio -- phylib's
// one-second link poll became ten seconds at the old 100.

static unsigned long long
now_ns(void)
{
  struct timespec ts;
  if (clock_gettime(CLOCK_MONOTONIC, &ts) < 0)
    return (unsigned long long)uptime() * 10000000ULL;
  return (unsigned long long)ts.tv_sec * 1000000000ULL + (unsigned long long)ts.tv_nsec;
}

unsigned long long kpi_ktime_get(void) { return now_ns(); }

unsigned long
kpi_jiffies_ms(void)
{
  return (unsigned long)(now_ns() / 1000000ULL);
}

static void
spin_us(unsigned long us)
{
  unsigned long long end = now_ns() + (unsigned long long)us * 1000ULL;
  while (now_ns() < end)
    __builtin_ia32_pause();
}

static void
sleep_us(unsigned long us)
{
  // Below a scheduler tick, a sleep would round up to the tick; the waits
  // that short are MDIO and register polls, where that would be a hundredfold.
  if (us < 10000) {
    spin_us(us);
    return;
  }
  struct timespec ts = { (long)(us / 1000000), (long)(us % 1000000) * 1000 };
  nanosleep(&ts);
}

// udelay(n) arrives as __const_udelay(n * 0x10c7): microseconds scaled by
// 2^32 / 10^6.
void kpi_const_udelay(unsigned long xloops) { spin_us((unsigned long)(((unsigned long long)xloops * 1000000ULL) >> 32) + 1); }
void kpi_udelay(unsigned long us)           { spin_us(us); }
void kpi_ndelay(unsigned long ns)           { spin_us(ns / 1000 + 1); }
void kpi_msleep(unsigned int ms)            { sleep_us((unsigned long)ms * 1000); }
void kpi_usleep_range(unsigned long min, unsigned long max, unsigned int st)
{ (void)max; (void)st; sleep_us(min); }

// --- bitmaps -------------------------------------------------------------------------
//
// phylib keeps link modes in 121-bit maps, past the size Linux inlines, so
// these are called for every "what can both ends do" question it asks.

#define WORDS(n) (((n) + 63) / 64)

static unsigned long
lastmask(unsigned int nbits)
{
  return (nbits % 64) ? (1UL << (nbits % 64)) - 1 : ~0UL;
}

int
kpi_bitmap_and(unsigned long *d, const unsigned long *a, const unsigned long *b,
               unsigned int nbits)
{
  unsigned long any = 0;
  unsigned int w = WORDS(nbits);
  for (unsigned int i = 0; i < w; i++) {
    d[i] = a[i] & b[i];
    any |= (i == w - 1) ? (d[i] & lastmask(nbits)) : d[i];
  }
  return any != 0;
}

int
kpi_bitmap_andnot(unsigned long *d, const unsigned long *a,
                  const unsigned long *b, unsigned int nbits)
{
  unsigned long any = 0;
  unsigned int w = WORDS(nbits);
  for (unsigned int i = 0; i < w; i++) {
    d[i] = a[i] & ~b[i];
    any |= (i == w - 1) ? (d[i] & lastmask(nbits)) : d[i];
  }
  return any != 0;
}

void
kpi_bitmap_or(unsigned long *d, const unsigned long *a, const unsigned long *b,
              unsigned int nbits)
{
  for (unsigned int i = 0; i < WORDS(nbits); i++)
    d[i] = a[i] | b[i];
}

int
kpi_bitmap_equal(const unsigned long *a, const unsigned long *b,
                 unsigned int nbits)
{
  unsigned int w = WORDS(nbits);
  for (unsigned int i = 0; i < w; i++) {
    unsigned long m = (i == w - 1) ? lastmask(nbits) : ~0UL;
    if ((a[i] ^ b[i]) & m) return 0;
  }
  return 1;
}

int
kpi_bitmap_intersects(const unsigned long *a, const unsigned long *b,
                      unsigned int nbits)
{
  unsigned int w = WORDS(nbits);
  for (unsigned int i = 0; i < w; i++) {
    unsigned long m = (i == w - 1) ? lastmask(nbits) : ~0UL;
    if (a[i] & b[i] & m) return 1;
  }
  return 0;
}

unsigned long
kpi_find_first_bit(const unsigned long *addr, unsigned long size)
{
  for (unsigned long i = 0; i < size; i++)
    if (addr[i / 64] & (1UL << (i % 64)))
      return i;
  return size;
}

// --- the driver core -----------------------------------------------------------------
//
// Enough of it to bind a driver to a device on a bus, which is all libphy
// asks of it: drivers and devices are remembered, and whenever one arrives it
// is offered to the other side -- bus_type.match decides, and the bus's probe
// or else the driver's runs with device.driver already set, as Linux does.
// Nothing is reference-counted and nothing is ever torn down; see the note on
// memory in kpi.c.

#define NCORE 64
static void *core_drv[NCORE];
static int   core_ndrv;
static void *core_dev[NCORE];
static int   core_ndev;

static int
core_bind(void *dev, void *drv)
{
  void *bus = (void *)rdp(dev, kpi_l.dev_bus);
  int (*match)(void *, void *) = bus ? (int (*)(void *, void *))rdp(bus, kpi_l.bus_match) : 0;
  int (*bprobe)(void *) = bus ? (int (*)(void *))rdp(bus, kpi_l.bus_probe) : 0;
  int (*dprobe)(void *) = (int (*)(void *))rdp(drv, kpi_l.drv_probe);

  if (match && match(dev, drv) <= 0)
    return 0;
  wrp(dev, kpi_l.dev_driver, (unsigned long)drv);
  int r = bprobe ? bprobe(dev) : dprobe ? dprobe(dev) : 0;
  const char *dn = (const char *)rdp(drv, kpi_l.drv_name);
  kpi_log("  kpi: bound "); kpi_log(dn ? dn : "a driver");
  kpi_log(" to "); kpi_log(dev_name_of(dev));
  if (r != 0) {
    kpi_log(" -- probe said "); kpi_lognum(r); kpi_log(", unbound\n");
    wrp(dev, kpi_l.dev_driver, 0);
    return 0;
  }
  kpi_log("\n");
  return 1;
}

static void
core_offer_dev(void *dev)
{
  void *bus;
  if (!kpi_l.core_ok || rdp(dev, kpi_l.dev_driver))
    return;
  bus = (void *)rdp(dev, kpi_l.dev_bus);
  if (!bus)
    return;
  for (int i = 0; i < core_ndrv; i++)
    if ((void *)rdp(core_drv[i], kpi_l.drv_bus) == bus && core_bind(dev, core_drv[i]))
      return;
}

int
kpi_driver_register(void *drv)
{
  if (core_ndrv < NCORE)
    core_drv[core_ndrv++] = drv;
  if (!kpi_l.core_ok)
    return 0;
  void *bus = (void *)rdp(drv, kpi_l.drv_bus);
  for (int i = 0; i < core_ndev; i++)
    if (!rdp(core_dev[i], kpi_l.dev_driver) &&
        (void *)rdp(core_dev[i], kpi_l.dev_bus) == bus)
      core_bind(core_dev[i], drv);
  return 0;
}

void
kpi_driver_unregister(void *drv)
{
  for (int i = 0; i < core_ndrv; i++)
    if (core_drv[i] == drv) core_drv[i] = core_drv[--core_ndrv];
}

int
kpi_device_add(void *dev)
{
  int i;
  for (i = 0; i < core_ndev; i++)
    if (core_dev[i] == dev) break;
  if (i == core_ndev && core_ndev < NCORE)
    core_dev[core_ndev++] = dev;
  core_offer_dev(dev);
  return 0;
}

void
kpi_device_del(void *dev)
{
  for (int i = 0; i < core_ndev; i++)
    if (core_dev[i] == dev) core_dev[i] = core_dev[--core_ndev];
}

void kpi_device_release_driver(void *dev)
{ if (kpi_l.core_ok) wrp(dev, kpi_l.dev_driver, 0); }

void *kpi_get_device(void *dev) { return dev; }

int
kpi_dev_set_name(void *dev, const char *f, ...)
{
  if (!kpi_l.dev_kobj_ok)
    return 0;
  va_list ap;
  va_start(ap, f);
  char *s = kpi_kvasprintf(f, ap);
  va_end(ap);
  if (!s)
    return -12;
  wrp(dev, kpi_l.dev_kobj + kpi_l.kobj_name, (unsigned long)s);
  return 0;
}

const char *
kpi_dev_driver_string(void *dev)
{
  if (kpi_l.core_ok && dev) {
    void *drv = (void *)rdp(dev, kpi_l.dev_driver);
    if (drv) {
      const char *n = (const char *)rdp(drv, kpi_l.drv_name);
      if (n) return n;
    }
  }
  return "";
}

// --- device-managed resources ----------------------------------------------------------
//
// Freed when the device goes, which here is never. What has to be real is
// devres_find: mdio_devres registers a bus only after finding the record
// devm_mdiobus_alloc made, and says -EINVAL otherwise.

#define NRES 128
static struct { void *dev, *release, *data; } res[NRES];
static int nres;

void *
kpi_devres_alloc(void *release, unsigned long size, unsigned int gfp, int nid,
                 const char *name)
{
  (void)gfp; (void)nid; (void)name;
  void *d = zalloc(size ? size : 8);
  if (d && nres < NRES) {
    res[nres].dev = 0; res[nres].release = release; res[nres].data = d;
    nres++;
  }
  return d;
}

void
kpi_devres_add(void *dev, void *data)
{
  for (int i = 0; i < nres; i++)
    if (res[i].data == data) { res[i].dev = dev; return; }
}

void *
kpi_devres_find(void *dev, void *release, int (*match)(void *, void *, void *),
                void *mdata)
{
  for (int i = 0; i < nres; i++)
    if (res[i].dev == dev && res[i].release == release &&
        (!match || match(dev, res[i].data, mdata)))
      return res[i].data;
  return 0;
}

void
kpi_devres_free(void *data)
{
  for (int i = 0; i < nres; i++)
    if (res[i].data == data) { res[i] = res[--nres]; return; }
}

void *kpi_devm_kmalloc(void *dev, unsigned long size, unsigned int gfp)
{ (void)dev; (void)gfp; return zalloc(size); }

void *kpi_devm_alloc_etherdev(void *dev, int priv, unsigned int tx, unsigned int rx)
{ (void)dev; return kpi_alloc_etherdev_mqs(priv, tx, rx); }

void *
kpi_dmam_alloc_attrs(void *dev, unsigned long size, unsigned long long *h,
                     unsigned int gfp, unsigned long attrs)
{ return kpi_dma_alloc_attrs(dev, size, h, gfp, attrs); }

// --- PCI, the managed and newer calls ----------------------------------------------------

int kpi_pcim_enable_device(void *pdev) { return kpi_pci_enable_device(pdev); }

void *
kpi_pcim_iomap_region(void *pdev, int bar, const char *name)
{
  (void)name;
  void *p = kpi_pci_iomap(pdev, bar, 0);
  return p ? p : (void *)-12L;          // ERR_PTR(-ENOMEM); NULL is not an error
}

// One vector, and its number is whatever request_irq is then given back --
// the handler is called from the poll loop either way. See kpi_request_irq.
int kpi_pci_alloc_irq_vectors(void *pdev, unsigned int min, unsigned int max,
                              unsigned int flags)
{ (void)pdev; (void)min; (void)max; (void)flags; return 1; }

int kpi_pci_irq_vector(void *pdev, unsigned int nr) { (void)pdev; (void)nr; return 16; }

int
kpi_pci_read_config_byte(void *pdev, int where, unsigned char *v)
{
  unsigned short w = 0;
  int r = kpi_pci_read_config_word(pdev, where & ~1, &w);
  if (v) *v = (unsigned char)(w >> ((where & 1) * 8));
  return r;
}

// /dev/pci has no config write (see pci.c). The ones r8169 makes are tuning,
// and the ASPM change it wants is made by the kernel at boot instead.
int
kpi_pci_write_config_byte(void *pdev, int where, unsigned char v)
{
  (void)pdev; (void)v;
  kpi_log("  kpi: config write to 0x"); kpi_loghex((unsigned)where, 2);
  kpi_log(" not made\n");
  return 0;
}

void *kpi_alloc_pages_node(unsigned int gfp, unsigned int order, int nid, void *mask)
{ (void)nid; (void)mask; return kpi_alloc_pages(gfp, order); }

// --- work queues, the rest of the API ----------------------------------------------------

static char kpi_wq_dummy[64];
void *kpi_system_wq = kpi_wq_dummy;     // the *variable* is the symbol

void *kpi_alloc_workqueue(const char *f, unsigned int flags, int max, ...)
{ (void)f; (void)flags; (void)max; return kpi_wq_dummy; }

int
kpi_mod_delayed_work_on(int cpu, void *q, void *dwork, unsigned long delay)
{
  kpi_queue_delayed_work_on(cpu, q, dwork, delay);
  return 0;
}

int kpi_disable_work_sync(void *w) { kpi_cancel_work_sync(w); return 0; }

// --- firmware ------------------------------------------------------------------------------
//
// From /lib/firmware, which is where Alpine's linux-firmware-* packages put
// it. struct firmware's layout comes from BTF like everything else.

int
kpi_request_firmware(void **out, const char *name, void *dev)
{
  char path[256];
  (void)dev;
  if (out) *out = 0;
  if (!kpi_l.fw_ok || !name)
    return -2;
  kpi_snprintf(path, sizeof(path), "/lib/firmware/%s", name);
  int fd = open(path, O_RDONLY);
  if (fd < 0) {
    kpi_log("  kpi: no firmware "); kpi_log(path); kpi_log("\n");
    return -2;
  }
  unsigned long cap = 4UL << 20, n = 0;
  unsigned char *buf = zalloc(cap);
  if (!buf) { close(fd); return -12; }
  for (;;) {
    int r = read(fd, buf + n, (int)(cap - n > 65536 ? 65536 : cap - n));
    if (r <= 0) break;
    n += (unsigned long)r;
    if (n == cap) break;
  }
  close(fd);
  void *fw = zalloc(kpi_l.fw_size_of ? kpi_l.fw_size_of : 64);
  if (!fw) return -12;
  wrp(fw, kpi_l.fw_size, n);
  wrp(fw, kpi_l.fw_data, (unsigned long)buf);
  if (out) *out = fw;
  kpi_log("  kpi: firmware "); kpi_log(path); kpi_log(", ");
  kpi_lognum((long long)n); kpi_log(" bytes\n");
  return 0;
}

// --- small things that must not answer zero ------------------------------------------------

int  kpi_true(void)  { return 1; }
int  kpi_enodev(void) { return -19; }
long kpi_ida_alloc(void *ida, unsigned int min, unsigned int max, unsigned int gfp)
{
  static long next;
  (void)ida; (void)max; (void)gfp;
  return (long)min + next++;
}

unsigned int
kpi_crc32_le(unsigned int crc, const unsigned char *p, unsigned long len)
{
  while (len--) {
    crc ^= *p++;
    for (int k = 0; k < 8; k++)
      crc = (crc >> 1) ^ (0xEDB88320u & (0u - (crc & 1)));
  }
  return crc;
}

void
kpi_get_random_bytes(void *buf, unsigned long n)
{
  static unsigned long long x = 0x9E3779B97F4A7C15ULL;
  unsigned char *b = buf;
  x ^= now_ns();
  for (unsigned long i = 0; i < n; i++) {
    x ^= x << 13; x ^= x >> 7; x ^= x << 17;
    b[i] = (unsigned char)x;
  }
}

// __skb_pad(skb, pad, free_on_error): zero the bytes after the data, so a
// runt frame goes out padded with zeroes rather than with whatever was there.
int
kpi_skb_pad(void *skb, int pad, int free_on_error)
{
  (void)free_on_error;
  unsigned char *head = (unsigned char *)rdp(skb, kpi_l.skb_head);
  unsigned int tail, end;
  __builtin_memcpy(&tail, (char *)skb + kpi_l.skb_tail, 4);
  __builtin_memcpy(&end, (char *)skb + kpi_l.skb_end, 4);
  if (pad < 0 || tail + (unsigned)pad > end)
    return -12;
  for (int i = 0; i < pad; i++) head[tail + i] = 0;
  return 0;
}

// --- data -------------------------------------------------------------------------------------
//
// Undefined data symbols. A stub would resolve them to machine code, which
// reads as nonsense and, written to, overwrites a trampoline.

// net/ethtool/common.c's link_mode_params: speed, lanes, duplex per link mode
// bit, which phylib reads at init to build its capability table. A fact table,
// generated from Linux 6.18; struct link_mode_info is int, u8, u8 (8 bytes).
static const struct { int speed; unsigned char lanes, duplex; } kpi_link_mode_params[] = {
#include "link_modes.inc"
};

static unsigned char kpi_zero_data[16][4096] __attribute__((aligned(64)));
static unsigned long kpi_cpu_online_mask[16] = { 1 };   // CPU 0
static int kpi_system_state = 3;                         // SYSTEM_RUNNING
// A list_head: empty means pointing at itself, not zero.
static void *kpi_leds_list[2] = { kpi_leds_list, kpi_leds_list };

// --- calls that must not touch a register ---------------------------------------------
//
// preempt_schedule is reached through a thunk that saves every register, so
// its callers keep live values in caller-saved ones across the call. A
// logging stub makes a syscall, and `syscall` writes RFLAGS into r11: mdio-
// bus kept the PHY register it had just read in r11, and every MDIO read on
// the RTL8168H came back 0x0246 -- the PHY ID read as 0x02460246 and no
// driver matched it. Nothing to schedule here anyway; a bare ret.
__asm__(".text\n.globl kpi_ret\nkpi_ret:\n\tret\n");
extern void kpi_ret(void);

// --- the table ------------------------------------------------------------------------------------

extern int  kpi_queue_delayed_work_on(int cpu, void *q, void *dwork, unsigned long delay);
extern int  kpi_cancel_work_sync(void *work);
extern void kpi_kfree_skb(void *skb);

const struct kpi_sym kpi_base_table[] = {
  // printing, formatted
  { "_printk",                 (void *)kpi_printk_fmt },
  { "vprintk",                 (void *)kpi_vprintk },
  { "_dev_info",               (void *)kpi_dev_printk },
  { "_dev_err",                (void *)kpi_dev_printk },
  { "_dev_warn",               (void *)kpi_dev_printk },
  { "_dev_notice",             (void *)kpi_dev_printk },
  { "netdev_info",             (void *)kpi_netdev_printk_fmt },
  { "netdev_err",              (void *)kpi_netdev_printk_fmt },
  { "netdev_warn",             (void *)kpi_netdev_printk_fmt },
  { "netdev_alert",            (void *)kpi_netdev_printk_fmt },
  { "netdev_printk",           (void *)kpi_netdev_printk_lvl },
  { "dev_err_probe",           (void *)kpi_dev_err_probe },
  { "__warn_printk",           (void *)kpi_warn_printk },
  { "snprintf",                (void *)kpi_snprintf },
  { "sprintf",                 (void *)kpi_sprintf },
  { "kasprintf",               (void *)kpi_kasprintf },

  // strings
  { "strcmp",                  (void *)kpi_strcmp },
  { "strcpy",                  (void *)kpi_strcpy },
  { "kstrdup_const",           (void *)kpi_kstrdup_const },
  { "kfree_const",             (void *)kpi_nop },
  { "kstrtoull",               (void *)kpi_kstrtoull },

  // time
  { "ktime_get",               (void *)kpi_ktime_get },
  { "__const_udelay",          (void *)kpi_const_udelay },
  { "__udelay",                (void *)kpi_udelay },
  { "__ndelay",                (void *)kpi_ndelay },
  { "msleep",                  (void *)kpi_msleep },
  { "usleep_range_state",      (void *)kpi_usleep_range },

  // bitmaps
  { "__bitmap_and",            (void *)kpi_bitmap_and },
  { "__bitmap_andnot",         (void *)kpi_bitmap_andnot },
  { "__bitmap_or",             (void *)kpi_bitmap_or },
  { "__bitmap_equal",          (void *)kpi_bitmap_equal },
  { "__bitmap_intersects",     (void *)kpi_bitmap_intersects },
  { "_find_first_bit",         (void *)kpi_find_first_bit },

  // the driver core
  { "driver_register",         (void *)kpi_driver_register },
  { "driver_unregister",       (void *)kpi_driver_unregister },
  { "device_add",              (void *)kpi_device_add },
  { "device_register",         (void *)kpi_device_add },
  { "device_del",              (void *)kpi_device_del },
  { "device_unregister",       (void *)kpi_device_del },
  { "device_initialize",       (void *)kpi_nop },
  { "device_bind_driver",      (void *)kpi_zero },
  { "device_release_driver",   (void *)kpi_device_release_driver },
  { "get_device",              (void *)kpi_get_device },
  { "put_device",              (void *)kpi_nop },
  { "dev_set_name",            (void *)kpi_dev_set_name },
  { "dev_driver_string",       (void *)kpi_dev_driver_string },
  { "bus_register",            (void *)kpi_zero },
  { "class_register",          (void *)kpi_zero },
  { "try_module_get",          (void *)kpi_true },
  { "module_put",              (void *)kpi_nop },

  // device-managed resources
  { "__devres_alloc_node",     (void *)kpi_devres_alloc },
  { "devres_add",              (void *)kpi_devres_add },
  { "devres_find",             (void *)kpi_devres_find },
  { "devres_free",             (void *)kpi_devres_free },
  { "devm_kmalloc",            (void *)kpi_devm_kmalloc },
  { "devm_alloc_etherdev_mqs", (void *)kpi_devm_alloc_etherdev },
  { "dmam_alloc_attrs",        (void *)kpi_dmam_alloc_attrs },

  // PCI
  { "pcim_enable_device",      (void *)kpi_pcim_enable_device },
  { "pcim_iomap_region",       (void *)kpi_pcim_iomap_region },
  { "pci_alloc_irq_vectors",   (void *)kpi_pci_alloc_irq_vectors },
  { "pci_irq_vector",          (void *)kpi_pci_irq_vector },
  { "pci_read_config_byte",    (void *)kpi_pci_read_config_byte },
  { "pci_write_config_byte",   (void *)kpi_pci_write_config_byte },
  { "__alloc_pages_noprof",    (void *)kpi_alloc_pages_node },
  { "__free_pages",            (void *)kpi_nop },

  // work
  { "mod_delayed_work_on",     (void *)kpi_mod_delayed_work_on },
  { "alloc_workqueue_noprof",  (void *)kpi_alloc_workqueue },
  { "destroy_workqueue",       (void *)kpi_nop },
  { "disable_work_sync",       (void *)kpi_disable_work_sync },
  { "disable_work",            (void *)kpi_disable_work_sync },
  { "enable_work",             (void *)kpi_true },
  { "system_power_efficient_wq", (void *)&kpi_system_wq },
  { "system_percpu_wq",        (void *)&kpi_system_wq },
  { "system_wq",               (void *)&kpi_system_wq },

  // firmware
  { "firmware_request_nowarn", (void *)kpi_request_firmware },
  { "request_firmware",        (void *)kpi_request_firmware },
  { "release_firmware",        (void *)kpi_nop },

  // must preserve every register
  { "__SCT__preempt_schedule",         (void *)kpi_ret },
  { "__SCT__preempt_schedule_notrace", (void *)kpi_ret },
  { "preempt_schedule_thunk",          (void *)kpi_ret },
  { "preempt_schedule_notrace_thunk",  (void *)kpi_ret },

  // must not be zero
  { "__list_add_valid_or_report",       (void *)kpi_true },
  { "__list_del_entry_valid_or_report", (void *)kpi_true },
  { "mutex_is_locked",         (void *)kpi_true },
  { "eth_platform_get_mac_address", (void *)kpi_enodev },
  { "ida_alloc_range",         (void *)kpi_ida_alloc },
  { "crc32_le",                (void *)kpi_crc32_le },
  { "get_random_bytes",        (void *)kpi_get_random_bytes },
  { "__skb_pad",               (void *)kpi_skb_pad },
  // r8169 frees every transmitted skb this way; a stub leaked one per frame
  // until the DMA heap ran out. The budget argument changes only which cache
  // Linux returns it to.
  { "napi_consume_skb",        (void *)kpi_kfree_skb },

  // data
  { "link_mode_params",        (void *)kpi_link_mode_params },
  { "__cpu_online_mask",       (void *)kpi_cpu_online_mask },
  { "system_state",            (void *)&kpi_system_state },
  { "phylib_stubs",            (void *)kpi_zero_data[0] },
  { "phys_base",               (void *)kpi_zero_data[7] },
  { "kmalloc_caches",          (void *)kpi_zero_data[1] },
  { "random_kmalloc_seed",     (void *)kpi_zero_data[2] },
  { "mem_alloc_profiling_key", (void *)kpi_zero_data[3] },
  { "of_fwnode_ops",           (void *)kpi_zero_data[4] },
  { "leds_list",               (void *)kpi_leds_list },
  { "leds_list_lock",          (void *)kpi_zero_data[6] },
};

const int kpi_base_count = (int)(sizeof(kpi_base_table) / sizeof(kpi_base_table[0]));