SchOS/ldls/kpi/kpi.c

// SPDX-License-Identifier: GPL-2.0-or-later
//
// The KPI, and the table the loader resolves against. See ldls/kpi/kpi.h.
//
// Provenance, once, for every function below: the *signature* of each is a
// fact -- a compiled driver calls it with these arguments in these registers
// and there is no other way to express that. What each one *does* is written
// here from the published behaviour and from what the driver requires of it.
// No comment, macro body or implementation has been carried over from
// anywhere. docs/architecture/ldls.md §3.

#include "core/types.h"
#include "uapi/user.h"
#include "uapi/fcntl.h"
#include "uapi/posix.h"
#include "dev/pciuser.h"
#include "net/netuser.h"
#include "kpi.h"

struct kpi_ctx    kpi;
struct kpi_layout kpi_l;
int               kpi_verbose;

// The driver's NAPI poll, captured at registration. See kpi.h.
int  (*kpi_napi_poll)(void *napi, int budget);
void *kpi_napi;

// --- the driver's interrupt handler ------------------------------------------
//
// LDLS has no interrupt to run this from, and for a while that was taken to
// mean the handler could simply never run: request_irq returned 0, the NAPI
// poll was called from a loop, and 8139too was perfectly happy.
//
// e1000 is not, and the reason is worth writing down because it is not about
// receiving packets. Its handler does two things -- schedule the NAPI poll,
// which LDLS was already doing by hand, and, on a link-status change, queue
// the watchdog. The watchdog is what enables the transmitter. So a driver
// whose handler never runs was a driver whose transmitter was never switched
// on, and the symptom was frames accepted, mapped, and silently not sent.
//
// Calling the handler from the loop is safe in a way that it would not be in a
// kernel: a handler's first act is to read its device's cause register, and a
// device with no cause pending returns IRQ_NONE and does nothing.
static int (*kpi_irq_fn)(int irq, void *dev);
static void *kpi_irq_dev;
static int   kpi_irq_no;

// request_threaded_irq(irq, handler, thread_fn, flags, name, dev)
int
kpi_request_irq(unsigned int irq, void *handler, void *thread_fn,
                unsigned long flags, const char *name, void *dev)
{
  (void)thread_fn; (void)flags; (void)name;
  kpi_irq_no = (int)irq;
  kpi_irq_fn = (int (*)(int, void *))handler;
  kpi_irq_dev = dev;
  kpi_log("kpi: the driver asked for irq "); kpi_lognum((int)irq);
  kpi_log("; it will be called from the poll loop\n");
  return 0;
}

int
kpi_free_irq(unsigned int irq, void *dev)
{
  (void)irq; (void)dev;
  kpi_irq_fn = 0;
  return 0;
}

// Returns 1 if the handler said it was ours, which the loop counts as work.
int
kpi_irq_poll(void)
{
  if (!kpi_irq_fn)
    return 0;
  return kpi_irq_fn(kpi_irq_no, kpi_irq_dev) == 1;
}

// netif_napi_add_weight_locked(dev, napi, poll, weight)
//
// Recorded rather than ignored, because the poll function is the receive path:
// a NAPI driver's interrupt handler only schedules, and everything that
// actually takes packets off the chip happens in here.
void
kpi_netif_napi_add(void *dev, void *napi, void *poll, int weight)
{
  (void)dev; (void)weight;
  kpi_napi      = napi;
  kpi_napi_poll = (int (*)(void *, int))poll;
  kpi_log("  kpi: the driver registered a NAPI poll\n");
}

void
kpi_log(const char *s)
{
  int n = 0;
  while (s[n]) n++;
  write(1, (void *)s, n);
}

void
kpi_lognum(long long v)
{
  char b[24];
  int i = 0;
  if (v < 0) { write(1, "-", 1); v = -v; }
  if (v == 0) { write(1, "0", 1); return; }
  while (v > 0) { b[i++] = (char)('0' + (v % 10)); v /= 10; }
  while (i-- > 0) write(1, &b[i], 1);
}

void
kpi_loghex(unsigned long long v, int digits)
{
  static const char h[] = "0123456789abcdef";
  char b[16];
  int  i;
  if (digits > 16) digits = 16;
  for (i = digits - 1; i >= 0; i--) { b[i] = h[v & 0xF]; v >>= 4; }
  write(1, b, digits);
}

static int
eq(const char *a, const char *b)
{
  while (*a && *a == *b) { a++; b++; }
  return *a == 0 && *b == 0;
}

void *
kpi_find(const char *name)
{
  int i;
  for (i = 0; i < kpi_base_count; i++)
    if (eq(kpi_base_table[i].name, name))
      return kpi_base_table[i].fn;
  for (i = 0; i < kpi_count; i++)
    if (eq(kpi_table[i].name, name))
      return kpi_table[i].fn;
  return 0;
}

// --- memory -------------------------------------------------------------------
//
// There is no allocator in lib/, and a driver's allocations are neither many
// nor small, so every one of these is an mmap. Nothing is ever freed: a hosted
// driver's lifetime is the process's, and the kernel reclaims it all at exit.
// That is a leak in the strict sense and the right trade here -- writing an
// allocator to serve a program that will be killed to restart it is work spent
// on the wrong problem.

static void *
kpi_alloc(unsigned long n)
{
  void *p;
  if (n == 0)
    n = 8;
  p = mmap(0, n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  if (p == 0 || (long)p == -1)
    return 0;
  return p;                      // mmap gives zeroed pages, which kzalloc wants
}

void *kpi_kmalloc(unsigned long n, unsigned int flags)
{ (void)flags; return kpi_alloc(n); }

// vmalloc's promise is that the result is virtually contiguous and need not be
// physically so, which is what mmap already gives. The distinction only matters
// to a kernel that has both kinds of address, and a process has one kind.
//
// This is not a detail: e1000 allocates its per-descriptor bookkeeping with
// vzalloc, so without it ndo_open fails at the first Tx ring with "Allocation
// for Tx Queue 0 failed" and -ENOMEM -- an honest error from the driver about
// a function that was quietly returning zero.
void *kpi_vmalloc(unsigned long n)          { return kpi_alloc(n); }
void  kpi_vfree(void *p)                    { (void)p; }
int   kpi_is_vmalloc_addr(const void *p)    { (void)p; return 0; }

// __kmalloc_cache_noprof(cache, flags, size) -- the slab is somebody else's
// bookkeeping, and the size is the third argument.
void *
kpi_kmalloc_cache(void *cache, unsigned int flags, unsigned long n)
{ (void)cache; (void)flags; return kpi_alloc(n); }

void kpi_kfree(void *p) { (void)p; }

void *
kpi_krealloc(void *old, unsigned long n, unsigned int flags)
{
  (void)flags;
  // Never shrinks and never reuses. A driver that reallocs in its fast path
  // would notice; none of the ones this hosts do it at all.
  {
    unsigned char *p = kpi_alloc(n);
    unsigned char *o = (unsigned char *)old;
    unsigned long i;
    if (p && o)
      for (i = 0; i < n; i++)
        p[i] = o[i];
    return p;
  }
}

// dma_alloc_attrs(dev, size, dma_handle, gfp, attrs)
//
// The one allocation that cannot be an ordinary mmap: the device is going to
// read it by physical address, so it has to come from the kernel's DMA arena.
// PCIIOC_DMA is exactly that call, and it reports both addresses because the
// physical one is precisely what a process cannot work out for itself.
void *
kpi_dma_alloc_attrs(void *dev, unsigned long size, unsigned long long *handle,
                    unsigned int gfp, unsigned long attrs)
{
  struct pciuser_dma d;

  (void)dev; (void)gfp; (void)attrs;
  d.size = size;
  if (ioctl(kpi.pcifd, PCIIOC_DMA, &d) < 0) {
    kpi_log("kpi: no DMA memory for "); kpi_lognum(size); kpi_log(" bytes\n");
    return 0;
  }
  if (handle)
    *handle = d.paddr;
  if (kpi_verbose) {
    kpi_log("kpi: dma "); kpi_lognum(size); kpi_log(" bytes at phys 0x");
    kpi_loghex(d.paddr, 16); kpi_log("\n");
  }
  return (void *)d.uaddr;
}

// --- memory a device can be pointed at, in small pieces -----------------------
//
// dma_alloc_coherent is one big block per call and PCIIOC_DMA matches it
// exactly. dma_map_single is the other shape: the driver allocates an ordinary
// buffer, hands it to the device and expects a bus address back.
//
// 8139too never needed this -- it copies each frame into a ring it allocated
// coherently up front -- so it went unimplemented and the stub returned 0. On
// e1000 that is a NIC pointed at physical address zero, which is not a fault,
// not an error, and not something any log would have shown. The first driver
// to need it is the reason this exists.
//
// A process cannot know the physical address of an ordinary page, so the
// buffers have to come from the arena too: a chunk at a time, handed out by a
// bump pointer, with the pairing remembered so a virtual address can be turned
// back into a bus address. Nothing is freed, like everything else here.

#define KPI_DMA_CHUNK   (1024 * 1024)
#define KPI_DMA_CHUNKS  8

static struct {
  unsigned long long uaddr, paddr, size, used;
} kpi_chunk[KPI_DMA_CHUNKS];
static int kpi_nchunk;

static int
kpi_chunk_more(void)
{
  struct pciuser_dma d;

  if (kpi_nchunk >= KPI_DMA_CHUNKS)
    return 0;
  d.size = KPI_DMA_CHUNK;
  if (ioctl(kpi.pcifd, PCIIOC_DMA, &d) < 0)
    return 0;
  kpi_chunk[kpi_nchunk].uaddr = d.uaddr;
  kpi_chunk[kpi_nchunk].paddr = d.paddr;
  kpi_chunk[kpi_nchunk].size  = KPI_DMA_CHUNK;
  kpi_chunk[kpi_nchunk].used  = 0;
  kpi_nchunk++;
  return 1;
}

void *
kpi_dma_heap(unsigned long n, unsigned long align)
{
  int i;

  if (align == 0)
    align = 64;
  for (i = 0; i < 2; i++) {
    int c;
    for (c = 0; c < kpi_nchunk; c++) {
      unsigned long long at = kpi_chunk[c].uaddr + kpi_chunk[c].used;
      unsigned long long pad = (align - (at % align)) % align;
      if (kpi_chunk[c].used + pad + n > kpi_chunk[c].size)
        continue;
      kpi_chunk[c].used += pad + n;
      return (void *)(unsigned long)(at + pad);
    }
    if (!kpi_chunk_more())
      break;
  }
  kpi_log("kpi: out of DMA heap for "); kpi_lognum(n); kpi_log(" bytes\n");
  return 0;
}

// The bus address of something the heap handed out, or 0.
unsigned long long
kpi_phys_of(const void *va)
{
  unsigned long long a = (unsigned long long)(unsigned long)va;
  int c;

  for (c = 0; c < kpi_nchunk; c++)
    if (a >= kpi_chunk[c].uaddr && a < kpi_chunk[c].uaddr + kpi_chunk[c].size)
      return kpi_chunk[c].paddr + (a - kpi_chunk[c].uaddr);
  return 0;
}

// The two globals that make the module's own address arithmetic ours.
//
// `dma_map_single` is a static inline, so what reaches the shim is already
// expanded: the module computes `pa = va - page_offset_base`, then
// `page = vmemmap_base + (pa >> 12) * 64`, and calls dma_map_page_attrs with
// that. Both bases are undefined *data* symbols, which means the shim supplies
// them -- and setting both to zero makes the whole computation reversible with
// no lookup table at all:
//
//     page  = (va >> 12) * 64          on the way in
//     va    = (page / 64) << 12 | off  on the way back
//
// The 64 is sizeof(struct page) and is compiled into the module as `shl $0x6`,
// so it is not a choice either of us gets to make.
unsigned long kpi_page_offset_base;
unsigned long kpi_vmemmap_base;

#define KPI_PAGE_TO_VA(p) ((unsigned long)(((unsigned long)(p) >> 6) << 12))

// dma_map_page_attrs(dev, page, offset, size, dir, attrs)
unsigned long long
kpi_dma_map_page(void *dev, void *page, unsigned long offset,
                 unsigned long size, int dir, unsigned long attrs)
{
  unsigned long va = KPI_PAGE_TO_VA(page) + offset;
  unsigned long long pa = kpi_phys_of((const void *)va);

  static int said;
  (void)dev; (void)size; (void)dir; (void)attrs;
  if (!said) {
    said = 1;
    kpi_log("kpi: the driver mapped its first buffer for the device\n");
  }
  if (!pa) {
    // DMA_MAPPING_ERROR. Saying so is the whole point: a driver checks this
    // with dma_mapping_error() and drops the buffer, where a zero would have
    // aimed the device at the bottom of memory.
    kpi_log("kpi: dma_map of memory that is not in the arena\n");
    return ~0ULL;
  }
  return pa;
}

void
kpi_dma_unmap_page(void *dev, unsigned long long h, unsigned long size,
                   int dir, unsigned long attrs)
{ (void)dev; (void)h; (void)size; (void)dir; (void)attrs; }

// The arena is uncached-coherent as far as this host is concerned, and x86 is
// cache-coherent for DMA anyway, so a sync is genuinely nothing.
void kpi_dma_sync(void *dev, unsigned long long h, unsigned long size, int dir)
{ (void)dev; (void)h; (void)size; (void)dir; }

// alloc_pages(gfp, order) -> struct page *, in the same fictional page space.
void *
kpi_alloc_pages(unsigned int gfp, unsigned int order)
{
  unsigned long n = 4096UL << order;
  void *p;

  static int said;
  (void)gfp;
  p = kpi_dma_heap(n, 4096);
  if (!p)
    return 0;
  if (!said) {
    said = 1;
    kpi_log("kpi: the driver is allocating pages, order ");
    kpi_lognum(order); kpi_log("\n");
  }
  return (void *)((((unsigned long)p) >> 12) << 6);
}

void kpi_page_frag_free(void *p) { (void)p; }

// __netdev_alloc_frag_align(fragsz, align_mask)
void *
kpi_netdev_alloc_frag(unsigned long fragsz, unsigned long align_mask)
{
  static int said;
  unsigned long align = (~align_mask) + 1;
  void *p;

  if (align == 0 || align > 4096)
    align = 64;
  p = kpi_dma_heap(fragsz, align);
  if (!said) {
    said = 1;
    kpi_log("kpi: the driver is allocating receive buffers, ");
    kpi_lognum((long long)fragsz); kpi_log(" bytes each\n");
  }
  return p;
}

void
kpi_dma_free_attrs(void *dev, unsigned long size, void *va,
                   unsigned long long handle, unsigned long attrs)
{
  // PCIIOC_DMA has no free; a driver's blocks come back when it exits. See
  // the note in kernel/dev/pciuser.c.
  (void)dev; (void)size; (void)va; (void)handle; (void)attrs;
}

// --- bitmaps ------------------------------------------------------------------

// _find_next_bit(addr, nbits, start) -- the first set bit at or after `start`,
// or `nbits` if there is none.
//
// A pure function on the caller's own memory, so there is nothing to emulate:
// this is simply it, written from the signature. It is here because the stub
// version cost an hour. `for_each_cpu` is a loop whose body is
// `i = _find_next_bit(mask, nbits, i + 1)` and whose exit test is `i < nbits`,
// so a stub returning 0 does not mean "nothing found" -- it means "bit zero is
// set", for ever. ndo_open hung with no error and no fault.
//
// The general lesson is the one memcpy taught in docs/architecture/ldls.md,
// pointed a different way: for a function that *searches*, zero is not a
// neutral answer, it is the most specific answer there is.
unsigned long
kpi_find_next_bit(const unsigned long *addr, unsigned long nbits,
                  unsigned long start)
{
  unsigned long i;

  for (i = start; i < nbits; i++)
    if (addr[i / 64] & (1UL << (i % 64)))
      return i;
  return nbits;
}

// --- things that only have to not crash ---------------------------------------
//
// A hosted driver takes locks, starts timers and waits for things. In one
// process with no preemption inside the driver, a lock is a no-op that is
// *correct* rather than merely convenient: there is nobody to race with.
//
// This stops being true the moment a driver has an interrupt thread, and the
// note in docs/architecture/ldls.md about sleeping-context fidelity is about exactly
// this. It is the deepest unfixed assumption in the whole layer.

void kpi_nop(void) { }
int  kpi_zero(void) { return 0; }
int  kpi_one(void)  { return 1; }

// --- printing ------------------------------------------------------------------

// _printk(fmt, ...) and the netdev/dev variants, which put the format second.
//
// Linux format strings are printf's with a leading level byte and %p
// extensions that expand pointers by kind. Nothing here expands them, so the
// string is printed as it stands -- which is honest and is enough for the
// lines that matter, the ones a driver prints when it has found its hardware.
static void
print_fmt(const char *f)
{
  int n = 0;
  if (!f)
    return;
  if ((unsigned char)f[0] == 0x01 && f[1] >= '0' && f[1] <= '7')
    f += 2;
  kpi_log("  driver: ");
  kpi_log(f);
  while (f[n]) n++;
  if (n == 0 || f[n - 1] != '\n')
    kpi_log("\n");
}

int kpi_printk(const char *fmt) { print_fmt(fmt); return 0; }
int kpi_netdev_print(void *dev, const char *fmt) { (void)dev; print_fmt(fmt); return 0; }

// --- data, not functions -------------------------------------------------------
//
// Not every undefined symbol in a module is a call. `byte_rev_table` is a
// 256-byte lookup of bit-reversed bytes, and 8139too indexes it directly while
// computing its multicast hash. A stub trampoline resolves the name and gives
// the driver machine code to read as data -- which does not fault, and
// produces a quietly wrong answer, which is worse.
//
// The ELF symbol table cannot tell these apart: every undefined symbol in a
// module is STT_NOTYPE. BTF can (VAR against FUNC) and that is the general
// fix; this is the specific one, and the values are a fact about bit order.
static const unsigned char kpi_byte_rev_table[256] = {
  0x00,0x80,0x40,0xc0,0x20,0xa0,0x60,0xe0,0x10,0x90,0x50,0xd0,0x30,0xb0,0x70,0xf0,
  0x08,0x88,0x48,0xc8,0x28,0xa8,0x68,0xe8,0x18,0x98,0x58,0xd8,0x38,0xb8,0x78,0xf8,
  0x04,0x84,0x44,0xc4,0x24,0xa4,0x64,0xe4,0x14,0x94,0x54,0xd4,0x34,0xb4,0x74,0xf4,
  0x0c,0x8c,0x4c,0xcc,0x2c,0xac,0x6c,0xec,0x1c,0x9c,0x5c,0xdc,0x3c,0xbc,0x7c,0xfc,
  0x02,0x82,0x42,0xc2,0x22,0xa2,0x62,0xe2,0x12,0x92,0x52,0xd2,0x32,0xb2,0x72,0xf2,
  0x0a,0x8a,0x4a,0xca,0x2a,0xaa,0x6a,0xea,0x1a,0x9a,0x5a,0xda,0x3a,0xba,0x7a,0xfa,
  0x06,0x86,0x46,0xc6,0x26,0xa6,0x66,0xe6,0x16,0x96,0x56,0xd6,0x36,0xb6,0x76,0xf6,
  0x0e,0x8e,0x4e,0xce,0x2e,0xae,0x6e,0xee,0x1e,0x9e,0x5e,0xde,0x3e,0xbe,0x7e,0xfe,
  0x01,0x81,0x41,0xc1,0x21,0xa1,0x61,0xe1,0x11,0x91,0x51,0xd1,0x31,0xb1,0x71,0xf1,
  0x09,0x89,0x49,0xc9,0x29,0xa9,0x69,0xe9,0x19,0x99,0x59,0xd9,0x39,0xb9,0x79,0xf9,
  0x05,0x85,0x45,0xc5,0x25,0xa5,0x65,0xe5,0x15,0x95,0x55,0xd5,0x35,0xb5,0x75,0xf5,
  0x0d,0x8d,0x4d,0xcd,0x2d,0xad,0x6d,0xed,0x1d,0x9d,0x5d,0xdd,0x3d,0xbd,0x7d,0xfd,
  0x03,0x83,0x43,0xc3,0x23,0xa3,0x63,0xe3,0x13,0x93,0x53,0xd3,0x33,0xb3,0x73,0xf3,
  0x0b,0x8b,0x4b,0xcb,0x2b,0xab,0x6b,0xeb,0x1b,0x9b,0x5b,0xdb,0x3b,0xbb,0x7b,0xfb,
  0x07,0x87,0x47,0xc7,0x27,0xa7,0x67,0xe7,0x17,0x97,0x57,0xd7,0x37,0xb7,0x77,0xf7,
  0x0f,0x8f,0x4f,0xcf,0x2f,0xaf,0x6f,0xef,0x1f,0x9f,0x5f,0xdf,0x3f,0xbf,0x7f,0xff,
};

// jiffies, also data -- and now it has to move.
//
// It was a static zero for as long as nothing ran a driver's deferred work,
// and that was honest: a clock nobody reads need not tick. e1000's watchdog
// reads it, re-arms itself against it, and enables the transmitter from
// inside it, so a stopped clock is a NIC that never transmits.
//
// It counts at the module's HZ, which is compiled into the module and is 1000
// on the kernels LDLS loads from -- phylib re-arms its link poll HZ jiffies
// ahead, and at the 100 Hz this used to count that poll ran every ten seconds.
// Milliseconds from CLOCK_MONOTONIC; see kpi_jiffies_ms() in base.c.
unsigned long kpi_jiffies;

unsigned long
kpi_jiffies_now(void)
{
  kpi_jiffies = kpi_jiffies_ms();
  return kpi_jiffies;
}

// --- the table -----------------------------------------------------------------
//
// Names on the left are Linux's and are facts. Everything on the right is ours.

extern void kpi_thunk_rax(void), kpi_thunk_rbx(void), kpi_thunk_rcx(void);
extern void kpi_thunk_rdx(void), kpi_thunk_rsi(void), kpi_thunk_rdi(void);
extern void kpi_thunk_rbp(void);
extern void kpi_thunk_r8(void),  kpi_thunk_r9(void),  kpi_thunk_r10(void);
extern void kpi_thunk_r11(void), kpi_thunk_r12(void), kpi_thunk_r13(void);
extern void kpi_thunk_r14(void), kpi_thunk_r15(void);
extern void kpi_return_thunk(void), kpi_fentry(void);
extern void kpi_netif_carrier_off(void *dev);
extern void kpi_netif_carrier_on(void *dev);
extern void *kpi_pci_iomap(void *pdev, int bar, unsigned long max);
extern void  kpi_pci_iounmap(void *pdev, void *addr);
extern int   kpi_pci_enable_device(void *pdev);
extern int   kpi_pci_request_regions(void *pdev, const char *name);
extern void  kpi_pci_release_regions(void *pdev);
extern void  kpi_pci_set_master(void *pdev);
extern int   kpi_pci_read_config_word(void *pdev, int where, unsigned short *v);
extern int   kpi_pci_write_config_word(void *pdev, int where, unsigned short v);
extern void *kpi_pci_ioremap_bar(void *pdev, int bar);
extern int   kpi_pci_select_bars(void *pdev, unsigned long flags);
extern int   kpi_pci_request_selected_regions(void *pdev, int bars, const char *n);
extern void  kpi_pci_release_selected_regions(void *pdev, int bars);
extern int   kpi_pci_save_state(void *pdev);
extern void  kpi_pci_restore_state(void *pdev);

extern unsigned char  kpi_ioread8(void *addr);
extern unsigned short kpi_ioread16(void *addr);
extern unsigned int   kpi_ioread32(void *addr);
extern void kpi_iowrite8(unsigned char v, void *addr);
extern void kpi_iowrite16(unsigned short v, void *addr);
extern void kpi_iowrite32(unsigned int v, void *addr);

extern void *kpi_alloc_etherdev_mqs(int priv, unsigned int tx, unsigned int rx);
extern void  kpi_free_netdev(void *dev);
extern int   kpi_register_netdev(void *dev);
extern void  kpi_unregister_netdev(void *dev);
extern void  kpi_dev_addr_mod(void *dev, unsigned int off,
                              const unsigned char *addr, unsigned int len);

extern void *kpi_napi_alloc_skb(void *napi, unsigned int len);
extern void *kpi_build_skb(void *data, unsigned int frag_len);
extern int kpi_gro_receive_skb(void *gro, void *skb);
extern void *kpi_netdev_alloc_skb(void *dev, unsigned int len);
extern void *kpi_skb_put(void *skb, unsigned int n);
extern void *kpi_skb_reserve(void *skb, int n);
extern void  kpi_skb_copy_and_csum_dev(void *skb, unsigned char *to);
extern unsigned short kpi_eth_type_trans(void *skb, void *dev);
extern int   kpi_netif_receive_skb(void *skb);
extern void  kpi_kfree_skb(void *skb);
extern void  kpi_kfree_skb_reason(void *skb, int r);

extern void *kpi_memcpy(void *d, const void *s, unsigned long n);
extern void *kpi_memmove(void *d, const void *s, unsigned long n);
extern void *kpi_memset(void *d, int c, unsigned long n);
extern int   kpi_memcmp(const void *a, const void *b, unsigned long n);
extern unsigned long kpi_strlen(const char *s);
extern unsigned long kpi_strnlen(const char *s, unsigned long max);
extern long  kpi_sized_strscpy(char *d, const char *s, unsigned long n);
extern void  kpi_memcpy_fromio(void *d, const void *s, unsigned long n);
extern void  kpi_memcpy_toio(void *d, const void *s, unsigned long n);
extern void  kpi_fortify_panic(unsigned int, unsigned long, unsigned long);

const struct kpi_sym kpi_table[] = {
  // PCI
  { "pci_enable_device",       (void *)kpi_pci_enable_device },
  { "pci_disable_device",      (void *)kpi_nop },
  { "pci_request_regions",     (void *)kpi_pci_request_regions },
  { "pci_release_regions",     (void *)kpi_pci_release_regions },
  { "pci_set_master",          (void *)kpi_pci_set_master },
  { "pci_iomap",               (void *)kpi_pci_iomap },
  { "pci_iounmap",             (void *)kpi_pci_iounmap },
  { "pci_ioremap_bar",         (void *)kpi_pci_ioremap_bar },
  { "pci_select_bars",         (void *)kpi_pci_select_bars },
  { "pci_request_selected_regions", (void *)kpi_pci_request_selected_regions },
  { "pci_release_selected_regions", (void *)kpi_pci_release_selected_regions },
  { "pci_save_state",          (void *)kpi_pci_save_state },
  { "pci_restore_state",       (void *)kpi_pci_restore_state },
  { "pci_read_config_word",    (void *)kpi_pci_read_config_word },
  { "pci_write_config_word",   (void *)kpi_pci_write_config_word },

  // Registers
  { "ioread8",                 (void *)kpi_ioread8 },
  { "ioread16",                (void *)kpi_ioread16 },
  { "ioread32",                (void *)kpi_ioread32 },
  { "iowrite8",                (void *)kpi_iowrite8 },
  { "iowrite16",               (void *)kpi_iowrite16 },
  { "iowrite32",               (void *)kpi_iowrite32 },

  // Memory
  { "netif_carrier_off",       (void *)kpi_netif_carrier_off },
  { "netif_carrier_on",        (void *)kpi_netif_carrier_on },
  // The compiler's, not Linux's -- see thunk.c. A stub here is not a missing
  // feature, it is a discarded jump.
  { "__x86_indirect_thunk_rax", (void *)kpi_thunk_rax },
  { "__x86_indirect_thunk_rbx", (void *)kpi_thunk_rbx },
  { "__x86_indirect_thunk_rcx", (void *)kpi_thunk_rcx },
  { "__x86_indirect_thunk_rdx", (void *)kpi_thunk_rdx },
  { "__x86_indirect_thunk_rsi", (void *)kpi_thunk_rsi },
  { "__x86_indirect_thunk_rdi", (void *)kpi_thunk_rdi },
  { "__x86_indirect_thunk_rbp", (void *)kpi_thunk_rbp },
  { "__x86_indirect_thunk_r8",  (void *)kpi_thunk_r8 },
  { "__x86_indirect_thunk_r9",  (void *)kpi_thunk_r9 },
  { "__x86_indirect_thunk_r10", (void *)kpi_thunk_r10 },
  { "__x86_indirect_thunk_r11", (void *)kpi_thunk_r11 },
  { "__x86_indirect_thunk_r12", (void *)kpi_thunk_r12 },
  { "__x86_indirect_thunk_r13", (void *)kpi_thunk_r13 },
  { "__x86_indirect_thunk_r14", (void *)kpi_thunk_r14 },
  { "__x86_indirect_thunk_r15", (void *)kpi_thunk_r15 },
  { "__x86_return_thunk",       (void *)kpi_return_thunk },
  { "__fentry__",               (void *)kpi_fentry },

  { "_find_next_bit",          (void *)kpi_find_next_bit },
  { "queue_work_on",           (void *)kpi_queue_work_on },
  { "queue_delayed_work_on",   (void *)kpi_queue_delayed_work_on },
  { "cancel_work_sync",        (void *)kpi_cancel_work_sync },
  { "cancel_delayed_work_sync",(void *)kpi_cancel_delayed_work_sync },
  { "cancel_delayed_work",     (void *)kpi_cancel_delayed_work },
  { "flush_work",              (void *)kpi_flush_work },
  { "dma_map_page_attrs",      (void *)kpi_dma_map_page },
  { "dma_unmap_page_attrs",    (void *)kpi_dma_unmap_page },
  { "__dma_sync_single_for_cpu",    (void *)kpi_dma_sync },
  { "__dma_sync_single_for_device", (void *)kpi_dma_sync },
  { "alloc_pages_noprof",      (void *)kpi_alloc_pages },
  { "page_frag_free",          (void *)kpi_page_frag_free },
  { "__netdev_alloc_frag_align", (void *)kpi_netdev_alloc_frag },
  { "page_offset_base",        (void *)&kpi_page_offset_base },
  { "vmemmap_base",            (void *)&kpi_vmemmap_base },
  { "__kmalloc_noprof",        (void *)kpi_kmalloc },
  { "vzalloc_noprof",          (void *)kpi_vmalloc },
  { "vmalloc_noprof",          (void *)kpi_vmalloc },
  { "vfree",                   (void *)kpi_vfree },
  { "is_vmalloc_addr",         (void *)kpi_is_vmalloc_addr },
  { "__kmalloc_cache_noprof",  (void *)kpi_kmalloc_cache },
  { "kmalloc_noprof",          (void *)kpi_kmalloc },
  { "kfree",                   (void *)kpi_kfree },
  { "krealloc_noprof",         (void *)kpi_krealloc },
  { "dma_alloc_attrs",         (void *)kpi_dma_alloc_attrs },
  { "dma_free_attrs",          (void *)kpi_dma_free_attrs },

  // Network
  { "alloc_etherdev_mqs",      (void *)kpi_alloc_etherdev_mqs },
  { "free_netdev",             (void *)kpi_free_netdev },
  { "register_netdev",         (void *)kpi_register_netdev },
  { "unregister_netdev",       (void *)kpi_unregister_netdev },
  { "dev_addr_mod",            (void *)kpi_dev_addr_mod },

  // Data, not functions. See the note above kpi_byte_rev_table.
  { "byte_rev_table",          (void *)kpi_byte_rev_table },
  { "jiffies",                 (void *)&kpi_jiffies },

  // Bytes. Not optional, and not safe to stub -- see ldls/kpi/string.c.
  { "memcpy",                  (void *)kpi_memcpy },
  { "__memcpy",                (void *)kpi_memcpy },
  { "memmove",                 (void *)kpi_memmove },
  { "memset",                  (void *)kpi_memset },
  { "__memset",                (void *)kpi_memset },
  { "memcmp",                  (void *)kpi_memcmp },
  { "strlen",                  (void *)kpi_strlen },
  { "strnlen",                 (void *)kpi_strnlen },
  { "sized_strscpy",           (void *)kpi_sized_strscpy },
  { "memcpy_fromio",           (void *)kpi_memcpy_fromio },
  { "memcpy_toio",             (void *)kpi_memcpy_toio },
  { "__fortify_panic",         (void *)kpi_fortify_panic },

  // Printing
  { "_printk",                 (void *)kpi_printk },
  { "netdev_info",             (void *)kpi_netdev_print },
  { "netdev_err",              (void *)kpi_netdev_print },
  { "_dev_info",               (void *)kpi_netdev_print },
  { "_dev_err",                (void *)kpi_netdev_print },

  // Packets
  { "napi_build_skb",          (void *)kpi_build_skb },
  { "build_skb",               (void *)kpi_build_skb },
  { "napi_alloc_skb",          (void *)kpi_napi_alloc_skb },
  { "__napi_alloc_skb",        (void *)kpi_napi_alloc_skb },
  { "__netdev_alloc_skb",      (void *)kpi_netdev_alloc_skb },
  { "skb_put",                 (void *)kpi_skb_put },
  { "skb_reserve",             (void *)kpi_skb_reserve },
  { "skb_copy_and_csum_dev",   (void *)kpi_skb_copy_and_csum_dev },
  { "eth_type_trans",          (void *)kpi_eth_type_trans },
  { "netif_receive_skb",       (void *)kpi_netif_receive_skb },
  { "napi_gro_receive",        (void *)kpi_netif_receive_skb },
  { "gro_receive_skb",         (void *)kpi_gro_receive_skb },
  { "dev_kfree_skb_any_reason",(void *)kpi_kfree_skb_reason },
  { "consume_skb",             (void *)kpi_kfree_skb },
  { "kfree_skb_reason",        (void *)kpi_kfree_skb_reason },

  // Bringing the interface up. LDLS polls rather than taking interrupts, but
  // the handler is kept and called from the loop -- see kpi_request_irq.
  { "request_threaded_irq",    (void *)kpi_request_irq },
  { "free_irq",                (void *)kpi_free_irq },
  { "disable_hardirq",         (void *)kpi_nop },
  { "enable_irq",              (void *)kpi_nop },
  { "disable_irq_nosync",      (void *)kpi_nop },
  { "napi_enable",             (void *)kpi_nop },
  { "napi_enable_locked",      (void *)kpi_nop },
  { "napi_disable",            (void *)kpi_nop },
  { "napi_disable_locked",     (void *)kpi_nop },
  { "__napi_schedule",         (void *)kpi_nop },
  { "napi_schedule_prep",      (void *)kpi_one },
  { "napi_complete_done",      (void *)kpi_one },
  { "netif_tx_wake_queue",     (void *)kpi_nop },
  { "netif_device_attach",     (void *)kpi_nop },
  { "netif_device_detach",     (void *)kpi_nop },
  { "netif_napi_add_weight_locked", (void *)kpi_netif_napi_add },

  // The PHY. 8139too reaches it through the mii helpers, which call back into
  // the driver's own mdio routines -- so "the link is up" is the only thing
  // this has to say, and saying anything else stops the driver bringing the
  // interface up at all.
  { "mii_link_ok",             (void *)kpi_one },
  { "mii_nway_restart",        (void *)kpi_zero },
  { "mii_check_media",         (void *)kpi_nop },
  { "generic_mii_ioctl",       (void *)kpi_zero },
  { "mii_ethtool_get_link_ksettings", (void *)kpi_zero },
  { "mii_ethtool_set_link_ksettings", (void *)kpi_zero },

  // Deferred work. Nothing here runs it: the driver's periodic thread is a
  // watchdog, and a watchdog that never fires is better than one firing in a
  // context this host cannot provide.
  { "queue_delayed_work_on",   (void *)kpi_one },
  { "cancel_delayed_work_sync",(void *)kpi_zero },
  { "delayed_work_timer_fn",   (void *)kpi_nop },
  { "wake_up_process",         (void *)kpi_one },

  // Things a driver calls at probe that need only to return sensibly. Each is
  // here because 8139too calls it and because doing nothing is the right
  // answer in one process -- see the note above kpi_nop().
  { "mutex_lock",              (void *)kpi_nop },
  { "mutex_unlock",            (void *)kpi_nop },
  { "__mutex_init",            (void *)kpi_nop },
  { "_raw_spin_lock",          (void *)kpi_nop },
  { "_raw_spin_unlock",        (void *)kpi_nop },
  { "_raw_spin_lock_irq",      (void *)kpi_nop },
  { "_raw_spin_unlock_irq",    (void *)kpi_nop },
  { "_raw_spin_lock_bh",       (void *)kpi_nop },
  { "_raw_spin_unlock_bh",     (void *)kpi_nop },
  { "_raw_spin_lock_irqsave",  (void *)kpi_zero },
  { "_raw_spin_unlock_irqrestore", (void *)kpi_nop },
  { "timer_init_key",          (void *)kpi_nop },
  { "netif_napi_add_weight_locked", (void *)kpi_nop },
  { "__const_udelay",          (void *)kpi_nop },
  { "synchronize_rcu",         (void *)kpi_nop },
};

const int kpi_count = (int)(sizeof(kpi_table) / sizeof(kpi_table[0]));