// SPDX-License-Identifier: Apache-2.0
//
// The KPI, and the table the loader resolves against. See ldls/kpi/kpi.h.
//
// Provenance, once, for every function below: the *signature* of each is a
// fact -- a compiled driver calls it with these arguments in these registers
// and there is no other way to express that. What each one *does* is written
// here from the published behaviour and from what the driver requires of it.
// No comment, macro body or implementation has been carried over from
// anywhere. docs/architecture/ldls.md §3.

#include "types.h"
#include "user.h"
#include "fcntl.h"
#include "posix.h"
#include "pciuser.h"
#include "netuser.h"
#include "kpi.h"

struct kpi_ctx    kpi;
struct kpi_layout kpi_l;
int               kpi_verbose;

// The driver's NAPI poll, captured at registration. See kpi.h.
int  (*kpi_napi_poll)(void *napi, int budget);
void *kpi_napi;

// netif_napi_add_weight_locked(dev, napi, poll, weight)
//
// Recorded rather than ignored, because the poll function is the receive path:
// a NAPI driver's interrupt handler only schedules, and everything that
// actually takes packets off the chip happens in here.
void
kpi_netif_napi_add(void *dev, void *napi, void *poll, int weight)
{
  (void)dev; (void)weight;
  kpi_napi      = napi;
  kpi_napi_poll = (int (*)(void *, int))poll;
  kpi_log("  kpi: the driver registered a NAPI poll\n");
}

void
kpi_log(const char *s)
{
  int n = 0;
  while (s[n]) n++;
  write(1, (void *)s, n);
}

void
kpi_lognum(long long v)
{
  char b[24];
  int i = 0;
  if (v < 0) { write(1, "-", 1); v = -v; }
  if (v == 0) { write(1, "0", 1); return; }
  while (v > 0) { b[i++] = (char)('0' + (v % 10)); v /= 10; }
  while (i-- > 0) write(1, &b[i], 1);
}

void
kpi_loghex(unsigned long long v, int digits)
{
  static const char h[] = "0123456789abcdef";
  char b[16];
  int  i;
  if (digits > 16) digits = 16;
  for (i = digits - 1; i >= 0; i--) { b[i] = h[v & 0xF]; v >>= 4; }
  write(1, b, digits);
}

static int
eq(const char *a, const char *b)
{
  while (*a && *a == *b) { a++; b++; }
  return *a == 0 && *b == 0;
}

void *
kpi_find(const char *name)
{
  int i;
  for (i = 0; i < kpi_count; i++)
    if (eq(kpi_table[i].name, name))
      return kpi_table[i].fn;
  return 0;
}

// --- memory -------------------------------------------------------------------
//
// There is no allocator in lib/, and a driver's allocations are neither many
// nor small, so every one of these is an mmap. Nothing is ever freed: a hosted
// driver's lifetime is the process's, and the kernel reclaims it all at exit.
// That is a leak in the strict sense and the right trade here -- writing an
// allocator to serve a program that will be killed to restart it is work spent
// on the wrong problem.

static void *
kpi_alloc(unsigned long n)
{
  void *p;
  if (n == 0)
    n = 8;
  p = mmap(0, n, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  if (p == 0 || (long)p == -1)
    return 0;
  return p;                      // mmap gives zeroed pages, which kzalloc wants
}

void *kpi_kmalloc(unsigned long n, unsigned int flags)
{ (void)flags; return kpi_alloc(n); }

void kpi_kfree(void *p) { (void)p; }

void *
kpi_krealloc(void *old, unsigned long n, unsigned int flags)
{
  (void)flags;
  // Never shrinks and never reuses. A driver that reallocs in its fast path
  // would notice; none of the ones this hosts do it at all.
  {
    unsigned char *p = kpi_alloc(n);
    unsigned char *o = (unsigned char *)old;
    unsigned long i;
    if (p && o)
      for (i = 0; i < n; i++)
        p[i] = o[i];
    return p;
  }
}

// dma_alloc_attrs(dev, size, dma_handle, gfp, attrs)
//
// The one allocation that cannot be an ordinary mmap: the device is going to
// read it by physical address, so it has to come from the kernel's DMA arena.
// PCIIOC_DMA is exactly that call, and it reports both addresses because the
// physical one is precisely what a process cannot work out for itself.
void *
kpi_dma_alloc_attrs(void *dev, unsigned long size, unsigned long long *handle,
                    unsigned int gfp, unsigned long attrs)
{
  struct pciuser_dma d;

  (void)dev; (void)gfp; (void)attrs;
  d.size = size;
  if (ioctl(kpi.pcifd, PCIIOC_DMA, &d) < 0) {
    kpi_log("kpi: no DMA memory for "); kpi_lognum(size); kpi_log(" bytes\n");
    return 0;
  }
  if (handle)
    *handle = d.paddr;
  if (kpi_verbose) {
    kpi_log("kpi: dma "); kpi_lognum(size); kpi_log(" bytes at phys 0x");
    kpi_loghex(d.paddr, 16); kpi_log("\n");
  }
  return (void *)d.uaddr;
}

void
kpi_dma_free_attrs(void *dev, unsigned long size, void *va,
                   unsigned long long handle, unsigned long attrs)
{
  // PCIIOC_DMA has no free; a driver's blocks come back when it exits. See
  // the note in kernel/dev/pciuser.c.
  (void)dev; (void)size; (void)va; (void)handle; (void)attrs;
}

// --- things that only have to not crash ---------------------------------------
//
// A hosted driver takes locks, starts timers and waits for things. In one
// process with no preemption inside the driver, a lock is a no-op that is
// *correct* rather than merely convenient: there is nobody to race with.
//
// This stops being true the moment a driver has an interrupt thread, and the
// note in docs/architecture/ldls.md about sleeping-context fidelity is about exactly
// this. It is the deepest unfixed assumption in the whole layer.

void kpi_nop(void) { }
int  kpi_zero(void) { return 0; }
int  kpi_one(void)  { return 1; }

// --- printing ------------------------------------------------------------------

// _printk(fmt, ...) and the netdev/dev variants, which put the format second.
//
// Linux format strings are printf's with a leading level byte and %p
// extensions that expand pointers by kind. Nothing here expands them, so the
// string is printed as it stands -- which is honest and is enough for the
// lines that matter, the ones a driver prints when it has found its hardware.
static void
print_fmt(const char *f)
{
  int n = 0;
  if (!f)
    return;
  if ((unsigned char)f[0] == 0x01 && f[1] >= '0' && f[1] <= '7')
    f += 2;
  kpi_log("  driver: ");
  kpi_log(f);
  while (f[n]) n++;
  if (n == 0 || f[n - 1] != '\n')
    kpi_log("\n");
}

int kpi_printk(const char *fmt) { print_fmt(fmt); return 0; }
int kpi_netdev_print(void *dev, const char *fmt) { (void)dev; print_fmt(fmt); return 0; }

// --- data, not functions -------------------------------------------------------
//
// Not every undefined symbol in a module is a call. `byte_rev_table` is a
// 256-byte lookup of bit-reversed bytes, and 8139too indexes it directly while
// computing its multicast hash. A stub trampoline resolves the name and gives
// the driver machine code to read as data -- which does not fault, and
// produces a quietly wrong answer, which is worse.
//
// The ELF symbol table cannot tell these apart: every undefined symbol in a
// module is STT_NOTYPE. BTF can (VAR against FUNC) and that is the general
// fix; this is the specific one, and the values are a fact about bit order.
static const unsigned char kpi_byte_rev_table[256] = {
  0x00,0x80,0x40,0xc0,0x20,0xa0,0x60,0xe0,0x10,0x90,0x50,0xd0,0x30,0xb0,0x70,0xf0,
  0x08,0x88,0x48,0xc8,0x28,0xa8,0x68,0xe8,0x18,0x98,0x58,0xd8,0x38,0xb8,0x78,0xf8,
  0x04,0x84,0x44,0xc4,0x24,0xa4,0x64,0xe4,0x14,0x94,0x54,0xd4,0x34,0xb4,0x74,0xf4,
  0x0c,0x8c,0x4c,0xcc,0x2c,0xac,0x6c,0xec,0x1c,0x9c,0x5c,0xdc,0x3c,0xbc,0x7c,0xfc,
  0x02,0x82,0x42,0xc2,0x22,0xa2,0x62,0xe2,0x12,0x92,0x52,0xd2,0x32,0xb2,0x72,0xf2,
  0x0a,0x8a,0x4a,0xca,0x2a,0xaa,0x6a,0xea,0x1a,0x9a,0x5a,0xda,0x3a,0xba,0x7a,0xfa,
  0x06,0x86,0x46,0xc6,0x26,0xa6,0x66,0xe6,0x16,0x96,0x56,0xd6,0x36,0xb6,0x76,0xf6,
  0x0e,0x8e,0x4e,0xce,0x2e,0xae,0x6e,0xee,0x1e,0x9e,0x5e,0xde,0x3e,0xbe,0x7e,0xfe,
  0x01,0x81,0x41,0xc1,0x21,0xa1,0x61,0xe1,0x11,0x91,0x51,0xd1,0x31,0xb1,0x71,0xf1,
  0x09,0x89,0x49,0xc9,0x29,0xa9,0x69,0xe9,0x19,0x99,0x59,0xd9,0x39,0xb9,0x79,0xf9,
  0x05,0x85,0x45,0xc5,0x25,0xa5,0x65,0xe5,0x15,0x95,0x55,0xd5,0x35,0xb5,0x75,0xf5,
  0x0d,0x8d,0x4d,0xcd,0x2d,0xad,0x6d,0xed,0x1d,0x9d,0x5d,0xdd,0x3d,0xbd,0x7d,0xfd,
  0x03,0x83,0x43,0xc3,0x23,0xa3,0x63,0xe3,0x13,0x93,0x53,0xd3,0x33,0xb3,0x73,0xf3,
  0x0b,0x8b,0x4b,0xcb,0x2b,0xab,0x6b,0xeb,0x1b,0x9b,0x5b,0xdb,0x3b,0xbb,0x7b,0xfb,
  0x07,0x87,0x47,0xc7,0x27,0xa7,0x67,0xe7,0x17,0x97,0x57,0xd7,0x37,0xb7,0x77,0xf7,
  0x0f,0x8f,0x4f,0xcf,0x2f,0xaf,0x6f,0xef,0x1f,0x9f,0x5f,0xdf,0x3f,0xbf,0x7f,0xff,
};

// jiffies, also data. Nothing advances it: the drivers hosted so far read it
// only to compare against a timeout they will never reach, because nothing
// here runs their watchdog either.
static unsigned long kpi_jiffies;

// --- the table -----------------------------------------------------------------
//
// Names on the left are Linux's and are facts. Everything on the right is ours.

extern void *kpi_pci_iomap(void *pdev, int bar, unsigned long max);
extern void  kpi_pci_iounmap(void *pdev, void *addr);
extern int   kpi_pci_enable_device(void *pdev);
extern int   kpi_pci_request_regions(void *pdev, const char *name);
extern void  kpi_pci_release_regions(void *pdev);
extern void  kpi_pci_set_master(void *pdev);
extern int   kpi_pci_read_config_word(void *pdev, int where, unsigned short *v);
extern int   kpi_pci_write_config_word(void *pdev, int where, unsigned short v);

extern unsigned char  kpi_ioread8(void *addr);
extern unsigned short kpi_ioread16(void *addr);
extern unsigned int   kpi_ioread32(void *addr);
extern void kpi_iowrite8(unsigned char v, void *addr);
extern void kpi_iowrite16(unsigned short v, void *addr);
extern void kpi_iowrite32(unsigned int v, void *addr);

extern void *kpi_alloc_etherdev_mqs(int priv, unsigned int tx, unsigned int rx);
extern void  kpi_free_netdev(void *dev);
extern int   kpi_register_netdev(void *dev);
extern void  kpi_unregister_netdev(void *dev);
extern void  kpi_dev_addr_mod(void *dev, unsigned int off,
                              const unsigned char *addr, unsigned int len);

extern void *kpi_napi_alloc_skb(void *napi, unsigned int len);
extern void *kpi_netdev_alloc_skb(void *dev, unsigned int len);
extern void *kpi_skb_put(void *skb, unsigned int n);
extern void *kpi_skb_reserve(void *skb, int n);
extern void  kpi_skb_copy_and_csum_dev(void *skb, unsigned char *to);
extern unsigned short kpi_eth_type_trans(void *skb, void *dev);
extern int   kpi_netif_receive_skb(void *skb);
extern void  kpi_kfree_skb(void *skb);
extern void  kpi_kfree_skb_reason(void *skb, int r);

extern void *kpi_memcpy(void *d, const void *s, unsigned long n);
extern void *kpi_memmove(void *d, const void *s, unsigned long n);
extern void *kpi_memset(void *d, int c, unsigned long n);
extern int   kpi_memcmp(const void *a, const void *b, unsigned long n);
extern unsigned long kpi_strlen(const char *s);
extern unsigned long kpi_strnlen(const char *s, unsigned long max);
extern long  kpi_sized_strscpy(char *d, const char *s, unsigned long n);
extern void  kpi_memcpy_fromio(void *d, const void *s, unsigned long n);
extern void  kpi_memcpy_toio(void *d, const void *s, unsigned long n);
extern void  kpi_fortify_panic(void);

const struct kpi_sym kpi_table[] = {
  // PCI
  { "pci_enable_device",       (void *)kpi_pci_enable_device },
  { "pci_disable_device",      (void *)kpi_nop },
  { "pci_request_regions",     (void *)kpi_pci_request_regions },
  { "pci_release_regions",     (void *)kpi_pci_release_regions },
  { "pci_set_master",          (void *)kpi_pci_set_master },
  { "pci_iomap",               (void *)kpi_pci_iomap },
  { "pci_iounmap",             (void *)kpi_pci_iounmap },
  { "pci_read_config_word",    (void *)kpi_pci_read_config_word },
  { "pci_write_config_word",   (void *)kpi_pci_write_config_word },

  // Registers
  { "ioread8",                 (void *)kpi_ioread8 },
  { "ioread16",                (void *)kpi_ioread16 },
  { "ioread32",                (void *)kpi_ioread32 },
  { "iowrite8",                (void *)kpi_iowrite8 },
  { "iowrite16",               (void *)kpi_iowrite16 },
  { "iowrite32",               (void *)kpi_iowrite32 },

  // Memory
  { "__kmalloc_noprof",        (void *)kpi_kmalloc },
  { "kmalloc_noprof",          (void *)kpi_kmalloc },
  { "kfree",                   (void *)kpi_kfree },
  { "krealloc_noprof",         (void *)kpi_krealloc },
  { "dma_alloc_attrs",         (void *)kpi_dma_alloc_attrs },
  { "dma_free_attrs",          (void *)kpi_dma_free_attrs },

  // Network
  { "alloc_etherdev_mqs",      (void *)kpi_alloc_etherdev_mqs },
  { "free_netdev",             (void *)kpi_free_netdev },
  { "register_netdev",         (void *)kpi_register_netdev },
  { "unregister_netdev",       (void *)kpi_unregister_netdev },
  { "dev_addr_mod",            (void *)kpi_dev_addr_mod },

  // Data, not functions. See the note above kpi_byte_rev_table.
  { "byte_rev_table",          (void *)kpi_byte_rev_table },
  { "jiffies",                 (void *)&kpi_jiffies },

  // Bytes. Not optional, and not safe to stub -- see ldls/kpi/string.c.
  { "memcpy",                  (void *)kpi_memcpy },
  { "__memcpy",                (void *)kpi_memcpy },
  { "memmove",                 (void *)kpi_memmove },
  { "memset",                  (void *)kpi_memset },
  { "__memset",                (void *)kpi_memset },
  { "memcmp",                  (void *)kpi_memcmp },
  { "strlen",                  (void *)kpi_strlen },
  { "strnlen",                 (void *)kpi_strnlen },
  { "sized_strscpy",           (void *)kpi_sized_strscpy },
  { "memcpy_fromio",           (void *)kpi_memcpy_fromio },
  { "memcpy_toio",             (void *)kpi_memcpy_toio },
  { "__fortify_panic",         (void *)kpi_fortify_panic },

  // Printing
  { "_printk",                 (void *)kpi_printk },
  { "netdev_info",             (void *)kpi_netdev_print },
  { "netdev_err",              (void *)kpi_netdev_print },
  { "_dev_info",               (void *)kpi_netdev_print },
  { "_dev_err",                (void *)kpi_netdev_print },

  // Packets
  { "napi_alloc_skb",          (void *)kpi_napi_alloc_skb },
  { "__napi_alloc_skb",        (void *)kpi_napi_alloc_skb },
  { "__netdev_alloc_skb",      (void *)kpi_netdev_alloc_skb },
  { "skb_put",                 (void *)kpi_skb_put },
  { "skb_reserve",             (void *)kpi_skb_reserve },
  { "skb_copy_and_csum_dev",   (void *)kpi_skb_copy_and_csum_dev },
  { "eth_type_trans",          (void *)kpi_eth_type_trans },
  { "netif_receive_skb",       (void *)kpi_netif_receive_skb },
  { "napi_gro_receive",        (void *)kpi_netif_receive_skb },
  { "dev_kfree_skb_any_reason",(void *)kpi_kfree_skb_reason },
  { "consume_skb",             (void *)kpi_kfree_skb },
  { "kfree_skb_reason",        (void *)kpi_kfree_skb_reason },

  // Bringing the interface up. LDLS polls rather than taking interrupts, so
  // request_irq succeeds and nothing is ever delivered through it -- see the
  // note in cmd/ldlsd on why a poll loop is the honest shape here.
  { "request_threaded_irq",    (void *)kpi_zero },
  { "free_irq",                (void *)kpi_nop },
  { "enable_irq",              (void *)kpi_nop },
  { "disable_irq_nosync",      (void *)kpi_nop },
  { "napi_enable",             (void *)kpi_nop },
  { "napi_enable_locked",      (void *)kpi_nop },
  { "napi_disable",            (void *)kpi_nop },
  { "napi_disable_locked",     (void *)kpi_nop },
  { "__napi_schedule",         (void *)kpi_nop },
  { "napi_schedule_prep",      (void *)kpi_one },
  { "napi_complete_done",      (void *)kpi_one },
  { "netif_tx_wake_queue",     (void *)kpi_nop },
  { "netif_device_attach",     (void *)kpi_nop },
  { "netif_device_detach",     (void *)kpi_nop },
  { "netif_napi_add_weight_locked", (void *)kpi_netif_napi_add },

  // The PHY. 8139too reaches it through the mii helpers, which call back into
  // the driver's own mdio routines -- so "the link is up" is the only thing
  // this has to say, and saying anything else stops the driver bringing the
  // interface up at all.
  { "mii_link_ok",             (void *)kpi_one },
  { "mii_nway_restart",        (void *)kpi_zero },
  { "mii_check_media",         (void *)kpi_nop },
  { "generic_mii_ioctl",       (void *)kpi_zero },
  { "mii_ethtool_get_link_ksettings", (void *)kpi_zero },
  { "mii_ethtool_set_link_ksettings", (void *)kpi_zero },

  // Deferred work. Nothing here runs it: the driver's periodic thread is a
  // watchdog, and a watchdog that never fires is better than one firing in a
  // context this host cannot provide.
  { "queue_delayed_work_on",   (void *)kpi_one },
  { "cancel_delayed_work_sync",(void *)kpi_zero },
  { "delayed_work_timer_fn",   (void *)kpi_nop },
  { "wake_up_process",         (void *)kpi_one },

  // Things a driver calls at probe that need only to return sensibly. Each is
  // here because 8139too calls it and because doing nothing is the right
  // answer in one process -- see the note above kpi_nop().
  { "mutex_lock",              (void *)kpi_nop },
  { "mutex_unlock",            (void *)kpi_nop },
  { "__mutex_init",            (void *)kpi_nop },
  { "_raw_spin_lock",          (void *)kpi_nop },
  { "_raw_spin_unlock",        (void *)kpi_nop },
  { "_raw_spin_lock_irq",      (void *)kpi_nop },
  { "_raw_spin_unlock_irq",    (void *)kpi_nop },
  { "_raw_spin_lock_bh",       (void *)kpi_nop },
  { "_raw_spin_unlock_bh",     (void *)kpi_nop },
  { "_raw_spin_lock_irqsave",  (void *)kpi_zero },
  { "_raw_spin_unlock_irqrestore", (void *)kpi_nop },
  { "timer_init_key",          (void *)kpi_nop },
  { "netif_napi_add_weight_locked", (void *)kpi_nop },
  { "__const_udelay",          (void *)kpi_nop },
  { "synchronize_rcu",         (void *)kpi_nop },
};

const int kpi_count = (int)(sizeof(kpi_table) / sizeof(kpi_table[0]));