ldls: a second driver, and everything it found
e1000 -- Linux's, unmodified -- probes, opens, gets an address by DHCP and
carries ping traffic in a SchOS process. The kernel was driving that NIC and
gave it up, which makes it the first real handover as well as the second
driver.
None of what it needed was a missing feature. Every one was a call that
succeeded, returned zero and did nothing:
_find_next_bit "bit 0 is set" -- for_each_cpu never ended
netif_carrier_off carrier already up -- TCTL.EN never set
__x86_indirect_thunk_r8 the jump discarded -- 255 rx buffers, none made
dma_map_page_attrs bus address 0 -- NIC aimed at physical 0
so the rule from memcpy is now general: a stub returning zero is unsafe
wherever zero is a meaningful answer.
The loader grew ko_fixup_percpu(). The stack guard turned out to be one
per-CPU variable among several; `current` cannot be defused because the driver
dereferences it. Taking the %gs off instead leaves the same opcode reading the
same width from storage the host supplies -- a per-CPU variable in a
single-threaded host is just a variable. Eight bytes become seven and a nop,
and the displacement is not always last, so there is now just enough of an x86
decoder in ko.c to know how long the instruction is.
ldls/kpi/thunk.c is fifteen lines of assembly and the highest-value file here:
retpolines are control flow, and a stub cannot stand in for a jump.
The kernel side:
- ioperm.c, the TSS I/O permission bitmap. A driver's outl is an instruction,
not a call, and e1000's reset path picks it deliberately. IOPL=3 would hand
over the PIC and the disk with it, so it is per-port. Writing it found that
the I/O map base had been written at offset 64 rather than 102, leaving the
CPU checking ports 0-831 against the TSS's own RSP0.
- claim masks the device's IRQ line: 450,000 a second, otherwise.
- net_detach(), the missing half of net_attach_late(). An interface left in
the stack keeps its route and sends every frame to a driver that refuses
it, which looks exactly like a userspace driver that does not work.
- e1000_release() gives up the interface, not just the hardware.
8139too still passes 4/4; ldls-check still passes 27; four CPUs, no panic.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VQ7Qx8E1q1M1bbCq8QLHS2
This commit is contained in:
Nexoniarz
parent
f38eb8a7fd
commit
071499a378
26 changed files with 2009 additions and 46 deletions
374
ldls/kpi/kpi.c
374
ldls/kpi/kpi.c
|
|
@ -25,6 +25,57 @@ int kpi_verbose;
|
|||
int (*kpi_napi_poll)(void *napi, int budget);
|
||||
void *kpi_napi;
|
||||
|
||||
// --- the driver's interrupt handler ------------------------------------------
|
||||
//
|
||||
// LDLS has no interrupt to run this from, and for a while that was taken to
|
||||
// mean the handler could simply never run: request_irq returned 0, the NAPI
|
||||
// poll was called from a loop, and 8139too was perfectly happy.
|
||||
//
|
||||
// e1000 is not, and the reason is worth writing down because it is not about
|
||||
// receiving packets. Its handler does two things -- schedule the NAPI poll,
|
||||
// which LDLS was already doing by hand, and, on a link-status change, queue
|
||||
// the watchdog. The watchdog is what enables the transmitter. So a driver
|
||||
// whose handler never runs was a driver whose transmitter was never switched
|
||||
// on, and the symptom was frames accepted, mapped, and silently not sent.
|
||||
//
|
||||
// Calling the handler from the loop is safe in a way that it would not be in a
|
||||
// kernel: a handler's first act is to read its device's cause register, and a
|
||||
// device with no cause pending returns IRQ_NONE and does nothing.
|
||||
static int (*kpi_irq_fn)(int irq, void *dev);
|
||||
static void *kpi_irq_dev;
|
||||
static int kpi_irq_no;
|
||||
|
||||
// request_threaded_irq(irq, handler, thread_fn, flags, name, dev)
|
||||
int
|
||||
kpi_request_irq(unsigned int irq, void *handler, void *thread_fn,
|
||||
unsigned long flags, const char *name, void *dev)
|
||||
{
|
||||
(void)thread_fn; (void)flags; (void)name;
|
||||
kpi_irq_no = (int)irq;
|
||||
kpi_irq_fn = (int (*)(int, void *))handler;
|
||||
kpi_irq_dev = dev;
|
||||
kpi_log("kpi: the driver asked for irq "); kpi_lognum((int)irq);
|
||||
kpi_log("; it will be called from the poll loop\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
kpi_free_irq(unsigned int irq, void *dev)
|
||||
{
|
||||
(void)irq; (void)dev;
|
||||
kpi_irq_fn = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Returns 1 if the handler said it was ours, which the loop counts as work.
|
||||
int
|
||||
kpi_irq_poll(void)
|
||||
{
|
||||
if (!kpi_irq_fn)
|
||||
return 0;
|
||||
return kpi_irq_fn(kpi_irq_no, kpi_irq_dev) == 1;
|
||||
}
|
||||
|
||||
// netif_napi_add_weight_locked(dev, napi, poll, weight)
|
||||
//
|
||||
// Recorded rather than ignored, because the poll function is the receive path:
|
||||
|
|
@ -110,6 +161,24 @@ kpi_alloc(unsigned long n)
|
|||
void *kpi_kmalloc(unsigned long n, unsigned int flags)
|
||||
{ (void)flags; return kpi_alloc(n); }
|
||||
|
||||
// vmalloc's promise is that the result is virtually contiguous and need not be
|
||||
// physically so, which is what mmap already gives. The distinction only matters
|
||||
// to a kernel that has both kinds of address, and a process has one kind.
|
||||
//
|
||||
// This is not a detail: e1000 allocates its per-descriptor bookkeeping with
|
||||
// vzalloc, so without it ndo_open fails at the first Tx ring with "Allocation
|
||||
// for Tx Queue 0 failed" and -ENOMEM -- an honest error from the driver about
|
||||
// a function that was quietly returning zero.
|
||||
void *kpi_vmalloc(unsigned long n) { return kpi_alloc(n); }
|
||||
void kpi_vfree(void *p) { (void)p; }
|
||||
int kpi_is_vmalloc_addr(const void *p) { (void)p; return 0; }
|
||||
|
||||
// __kmalloc_cache_noprof(cache, flags, size) -- the slab is somebody else's
|
||||
// bookkeeping, and the size is the third argument.
|
||||
void *
|
||||
kpi_kmalloc_cache(void *cache, unsigned int flags, unsigned long n)
|
||||
{ (void)cache; (void)flags; return kpi_alloc(n); }
|
||||
|
||||
void kpi_kfree(void *p) { (void)p; }
|
||||
|
||||
void *
|
||||
|
|
@ -156,6 +225,180 @@ kpi_dma_alloc_attrs(void *dev, unsigned long size, unsigned long long *handle,
|
|||
return (void *)d.uaddr;
|
||||
}
|
||||
|
||||
// --- memory a device can be pointed at, in small pieces -----------------------
|
||||
//
|
||||
// dma_alloc_coherent is one big block per call and PCIIOC_DMA matches it
|
||||
// exactly. dma_map_single is the other shape: the driver allocates an ordinary
|
||||
// buffer, hands it to the device and expects a bus address back.
|
||||
//
|
||||
// 8139too never needed this -- it copies each frame into a ring it allocated
|
||||
// coherently up front -- so it went unimplemented and the stub returned 0. On
|
||||
// e1000 that is a NIC pointed at physical address zero, which is not a fault,
|
||||
// not an error, and not something any log would have shown. The first driver
|
||||
// to need it is the reason this exists.
|
||||
//
|
||||
// A process cannot know the physical address of an ordinary page, so the
|
||||
// buffers have to come from the arena too: a chunk at a time, handed out by a
|
||||
// bump pointer, with the pairing remembered so a virtual address can be turned
|
||||
// back into a bus address. Nothing is freed, like everything else here.
|
||||
|
||||
#define KPI_DMA_CHUNK (1024 * 1024)
|
||||
#define KPI_DMA_CHUNKS 8
|
||||
|
||||
static struct {
|
||||
unsigned long long uaddr, paddr, size, used;
|
||||
} kpi_chunk[KPI_DMA_CHUNKS];
|
||||
static int kpi_nchunk;
|
||||
|
||||
static int
|
||||
kpi_chunk_more(void)
|
||||
{
|
||||
struct pciuser_dma d;
|
||||
|
||||
if (kpi_nchunk >= KPI_DMA_CHUNKS)
|
||||
return 0;
|
||||
d.size = KPI_DMA_CHUNK;
|
||||
if (ioctl(kpi.pcifd, PCIIOC_DMA, &d) < 0)
|
||||
return 0;
|
||||
kpi_chunk[kpi_nchunk].uaddr = d.uaddr;
|
||||
kpi_chunk[kpi_nchunk].paddr = d.paddr;
|
||||
kpi_chunk[kpi_nchunk].size = KPI_DMA_CHUNK;
|
||||
kpi_chunk[kpi_nchunk].used = 0;
|
||||
kpi_nchunk++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
void *
|
||||
kpi_dma_heap(unsigned long n, unsigned long align)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (align == 0)
|
||||
align = 64;
|
||||
for (i = 0; i < 2; i++) {
|
||||
int c;
|
||||
for (c = 0; c < kpi_nchunk; c++) {
|
||||
unsigned long long at = kpi_chunk[c].uaddr + kpi_chunk[c].used;
|
||||
unsigned long long pad = (align - (at % align)) % align;
|
||||
if (kpi_chunk[c].used + pad + n > kpi_chunk[c].size)
|
||||
continue;
|
||||
kpi_chunk[c].used += pad + n;
|
||||
return (void *)(unsigned long)(at + pad);
|
||||
}
|
||||
if (!kpi_chunk_more())
|
||||
break;
|
||||
}
|
||||
kpi_log("kpi: out of DMA heap for "); kpi_lognum(n); kpi_log(" bytes\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// The bus address of something the heap handed out, or 0.
|
||||
unsigned long long
|
||||
kpi_phys_of(const void *va)
|
||||
{
|
||||
unsigned long long a = (unsigned long long)(unsigned long)va;
|
||||
int c;
|
||||
|
||||
for (c = 0; c < kpi_nchunk; c++)
|
||||
if (a >= kpi_chunk[c].uaddr && a < kpi_chunk[c].uaddr + kpi_chunk[c].size)
|
||||
return kpi_chunk[c].paddr + (a - kpi_chunk[c].uaddr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// The two globals that make the module's own address arithmetic ours.
|
||||
//
|
||||
// `dma_map_single` is a static inline, so what reaches the shim is already
|
||||
// expanded: the module computes `pa = va - page_offset_base`, then
|
||||
// `page = vmemmap_base + (pa >> 12) * 64`, and calls dma_map_page_attrs with
|
||||
// that. Both bases are undefined *data* symbols, which means the shim supplies
|
||||
// them -- and setting both to zero makes the whole computation reversible with
|
||||
// no lookup table at all:
|
||||
//
|
||||
// page = (va >> 12) * 64 on the way in
|
||||
// va = (page / 64) << 12 | off on the way back
|
||||
//
|
||||
// The 64 is sizeof(struct page) and is compiled into the module as `shl $0x6`,
|
||||
// so it is not a choice either of us gets to make.
|
||||
unsigned long kpi_page_offset_base;
|
||||
unsigned long kpi_vmemmap_base;
|
||||
|
||||
#define KPI_PAGE_TO_VA(p) ((unsigned long)(((unsigned long)(p) >> 6) << 12))
|
||||
|
||||
// dma_map_page_attrs(dev, page, offset, size, dir, attrs)
|
||||
unsigned long long
|
||||
kpi_dma_map_page(void *dev, void *page, unsigned long offset,
|
||||
unsigned long size, int dir, unsigned long attrs)
|
||||
{
|
||||
unsigned long va = KPI_PAGE_TO_VA(page) + offset;
|
||||
unsigned long long pa = kpi_phys_of((const void *)va);
|
||||
|
||||
static int said;
|
||||
(void)dev; (void)size; (void)dir; (void)attrs;
|
||||
if (!said) {
|
||||
said = 1;
|
||||
kpi_log("kpi: the driver mapped its first buffer for the device\n");
|
||||
}
|
||||
if (!pa) {
|
||||
// DMA_MAPPING_ERROR. Saying so is the whole point: a driver checks this
|
||||
// with dma_mapping_error() and drops the buffer, where a zero would have
|
||||
// aimed the device at the bottom of memory.
|
||||
kpi_log("kpi: dma_map of memory that is not in the arena\n");
|
||||
return ~0ULL;
|
||||
}
|
||||
return pa;
|
||||
}
|
||||
|
||||
void
|
||||
kpi_dma_unmap_page(void *dev, unsigned long long h, unsigned long size,
|
||||
int dir, unsigned long attrs)
|
||||
{ (void)dev; (void)h; (void)size; (void)dir; (void)attrs; }
|
||||
|
||||
// The arena is uncached-coherent as far as this host is concerned, and x86 is
|
||||
// cache-coherent for DMA anyway, so a sync is genuinely nothing.
|
||||
void kpi_dma_sync(void *dev, unsigned long long h, unsigned long size, int dir)
|
||||
{ (void)dev; (void)h; (void)size; (void)dir; }
|
||||
|
||||
// alloc_pages(gfp, order) -> struct page *, in the same fictional page space.
|
||||
void *
|
||||
kpi_alloc_pages(unsigned int gfp, unsigned int order)
|
||||
{
|
||||
unsigned long n = 4096UL << order;
|
||||
void *p;
|
||||
|
||||
static int said;
|
||||
(void)gfp;
|
||||
p = kpi_dma_heap(n, 4096);
|
||||
if (!p)
|
||||
return 0;
|
||||
if (!said) {
|
||||
said = 1;
|
||||
kpi_log("kpi: the driver is allocating pages, order ");
|
||||
kpi_lognum(order); kpi_log("\n");
|
||||
}
|
||||
return (void *)((((unsigned long)p) >> 12) << 6);
|
||||
}
|
||||
|
||||
void kpi_page_frag_free(void *p) { (void)p; }
|
||||
|
||||
// __netdev_alloc_frag_align(fragsz, align_mask)
|
||||
void *
|
||||
kpi_netdev_alloc_frag(unsigned long fragsz, unsigned long align_mask)
|
||||
{
|
||||
static int said;
|
||||
unsigned long align = (~align_mask) + 1;
|
||||
void *p;
|
||||
|
||||
if (align == 0 || align > 4096)
|
||||
align = 64;
|
||||
p = kpi_dma_heap(fragsz, align);
|
||||
if (!said) {
|
||||
said = 1;
|
||||
kpi_log("kpi: the driver is allocating receive buffers, ");
|
||||
kpi_lognum((long long)fragsz); kpi_log(" bytes each\n");
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
void
|
||||
kpi_dma_free_attrs(void *dev, unsigned long size, void *va,
|
||||
unsigned long long handle, unsigned long attrs)
|
||||
|
|
@ -165,6 +408,33 @@ kpi_dma_free_attrs(void *dev, unsigned long size, void *va,
|
|||
(void)dev; (void)size; (void)va; (void)handle; (void)attrs;
|
||||
}
|
||||
|
||||
// --- bitmaps ------------------------------------------------------------------
|
||||
|
||||
// _find_next_bit(addr, nbits, start) -- the first set bit at or after `start`,
|
||||
// or `nbits` if there is none.
|
||||
//
|
||||
// A pure function on the caller's own memory, so there is nothing to emulate:
|
||||
// this is simply it, written from the signature. It is here because the stub
|
||||
// version cost an hour. `for_each_cpu` is a loop whose body is
|
||||
// `i = _find_next_bit(mask, nbits, i + 1)` and whose exit test is `i < nbits`,
|
||||
// so a stub returning 0 does not mean "nothing found" -- it means "bit zero is
|
||||
// set", for ever. ndo_open hung with no error and no fault.
|
||||
//
|
||||
// The general lesson is the one memcpy taught in docs/architecture/ldls.md,
|
||||
// pointed a different way: for a function that *searches*, zero is not a
|
||||
// neutral answer, it is the most specific answer there is.
|
||||
unsigned long
|
||||
kpi_find_next_bit(const unsigned long *addr, unsigned long nbits,
|
||||
unsigned long start)
|
||||
{
|
||||
unsigned long i;
|
||||
|
||||
for (i = start; i < nbits; i++)
|
||||
if (addr[i / 64] & (1UL << (i % 64)))
|
||||
return i;
|
||||
return nbits;
|
||||
}
|
||||
|
||||
// --- things that only have to not crash ---------------------------------------
|
||||
//
|
||||
// A hosted driver takes locks, starts timers and waits for things. In one
|
||||
|
|
@ -235,15 +505,41 @@ static const unsigned char kpi_byte_rev_table[256] = {
|
|||
0x0f,0x8f,0x4f,0xcf,0x2f,0xaf,0x6f,0xef,0x1f,0x9f,0x5f,0xdf,0x3f,0xbf,0x7f,0xff,
|
||||
};
|
||||
|
||||
// jiffies, also data. Nothing advances it: the drivers hosted so far read it
|
||||
// only to compare against a timeout they will never reach, because nothing
|
||||
// here runs their watchdog either.
|
||||
static unsigned long kpi_jiffies;
|
||||
// jiffies, also data -- and now it has to move.
|
||||
//
|
||||
// It was a static zero for as long as nothing ran a driver's deferred work,
|
||||
// and that was honest: a clock nobody reads need not tick. e1000's watchdog
|
||||
// reads it, re-arms itself against it, and enables the transmitter from
|
||||
// inside it, so a stopped clock is a NIC that never transmits.
|
||||
//
|
||||
// SchOS's uptime() counts 100 Hz ticks, which is a plausible HZ and is the
|
||||
// only monotonic source a process has. What matters to a driver is that it
|
||||
// advances and never goes backwards; the exact rate matters only to timeouts,
|
||||
// and a timeout measured against a slower clock is a longer timeout.
|
||||
unsigned long kpi_jiffies;
|
||||
|
||||
unsigned long
|
||||
kpi_jiffies_now(void)
|
||||
{
|
||||
int t = uptime();
|
||||
if (t > 0)
|
||||
kpi_jiffies = (unsigned long)t;
|
||||
return kpi_jiffies;
|
||||
}
|
||||
|
||||
// --- the table -----------------------------------------------------------------
|
||||
//
|
||||
// Names on the left are Linux's and are facts. Everything on the right is ours.
|
||||
|
||||
extern void kpi_thunk_rax(void), kpi_thunk_rbx(void), kpi_thunk_rcx(void);
|
||||
extern void kpi_thunk_rdx(void), kpi_thunk_rsi(void), kpi_thunk_rdi(void);
|
||||
extern void kpi_thunk_rbp(void);
|
||||
extern void kpi_thunk_r8(void), kpi_thunk_r9(void), kpi_thunk_r10(void);
|
||||
extern void kpi_thunk_r11(void), kpi_thunk_r12(void), kpi_thunk_r13(void);
|
||||
extern void kpi_thunk_r14(void), kpi_thunk_r15(void);
|
||||
extern void kpi_return_thunk(void), kpi_fentry(void);
|
||||
extern void kpi_netif_carrier_off(void *dev);
|
||||
extern void kpi_netif_carrier_on(void *dev);
|
||||
extern void *kpi_pci_iomap(void *pdev, int bar, unsigned long max);
|
||||
extern void kpi_pci_iounmap(void *pdev, void *addr);
|
||||
extern int kpi_pci_enable_device(void *pdev);
|
||||
|
|
@ -252,6 +548,12 @@ extern void kpi_pci_release_regions(void *pdev);
|
|||
extern void kpi_pci_set_master(void *pdev);
|
||||
extern int kpi_pci_read_config_word(void *pdev, int where, unsigned short *v);
|
||||
extern int kpi_pci_write_config_word(void *pdev, int where, unsigned short v);
|
||||
extern void *kpi_pci_ioremap_bar(void *pdev, int bar);
|
||||
extern int kpi_pci_select_bars(void *pdev, unsigned long flags);
|
||||
extern int kpi_pci_request_selected_regions(void *pdev, int bars, const char *n);
|
||||
extern void kpi_pci_release_selected_regions(void *pdev, int bars);
|
||||
extern int kpi_pci_save_state(void *pdev);
|
||||
extern void kpi_pci_restore_state(void *pdev);
|
||||
|
||||
extern unsigned char kpi_ioread8(void *addr);
|
||||
extern unsigned short kpi_ioread16(void *addr);
|
||||
|
|
@ -268,6 +570,8 @@ extern void kpi_dev_addr_mod(void *dev, unsigned int off,
|
|||
const unsigned char *addr, unsigned int len);
|
||||
|
||||
extern void *kpi_napi_alloc_skb(void *napi, unsigned int len);
|
||||
extern void *kpi_build_skb(void *data, unsigned int frag_len);
|
||||
extern int kpi_gro_receive_skb(void *gro, void *skb);
|
||||
extern void *kpi_netdev_alloc_skb(void *dev, unsigned int len);
|
||||
extern void *kpi_skb_put(void *skb, unsigned int n);
|
||||
extern void *kpi_skb_reserve(void *skb, int n);
|
||||
|
|
@ -297,6 +601,12 @@ const struct kpi_sym kpi_table[] = {
|
|||
{ "pci_set_master", (void *)kpi_pci_set_master },
|
||||
{ "pci_iomap", (void *)kpi_pci_iomap },
|
||||
{ "pci_iounmap", (void *)kpi_pci_iounmap },
|
||||
{ "pci_ioremap_bar", (void *)kpi_pci_ioremap_bar },
|
||||
{ "pci_select_bars", (void *)kpi_pci_select_bars },
|
||||
{ "pci_request_selected_regions", (void *)kpi_pci_request_selected_regions },
|
||||
{ "pci_release_selected_regions", (void *)kpi_pci_release_selected_regions },
|
||||
{ "pci_save_state", (void *)kpi_pci_save_state },
|
||||
{ "pci_restore_state", (void *)kpi_pci_restore_state },
|
||||
{ "pci_read_config_word", (void *)kpi_pci_read_config_word },
|
||||
{ "pci_write_config_word", (void *)kpi_pci_write_config_word },
|
||||
|
||||
|
|
@ -309,7 +619,50 @@ const struct kpi_sym kpi_table[] = {
|
|||
{ "iowrite32", (void *)kpi_iowrite32 },
|
||||
|
||||
// Memory
|
||||
{ "netif_carrier_off", (void *)kpi_netif_carrier_off },
|
||||
{ "netif_carrier_on", (void *)kpi_netif_carrier_on },
|
||||
// The compiler's, not Linux's -- see thunk.c. A stub here is not a missing
|
||||
// feature, it is a discarded jump.
|
||||
{ "__x86_indirect_thunk_rax", (void *)kpi_thunk_rax },
|
||||
{ "__x86_indirect_thunk_rbx", (void *)kpi_thunk_rbx },
|
||||
{ "__x86_indirect_thunk_rcx", (void *)kpi_thunk_rcx },
|
||||
{ "__x86_indirect_thunk_rdx", (void *)kpi_thunk_rdx },
|
||||
{ "__x86_indirect_thunk_rsi", (void *)kpi_thunk_rsi },
|
||||
{ "__x86_indirect_thunk_rdi", (void *)kpi_thunk_rdi },
|
||||
{ "__x86_indirect_thunk_rbp", (void *)kpi_thunk_rbp },
|
||||
{ "__x86_indirect_thunk_r8", (void *)kpi_thunk_r8 },
|
||||
{ "__x86_indirect_thunk_r9", (void *)kpi_thunk_r9 },
|
||||
{ "__x86_indirect_thunk_r10", (void *)kpi_thunk_r10 },
|
||||
{ "__x86_indirect_thunk_r11", (void *)kpi_thunk_r11 },
|
||||
{ "__x86_indirect_thunk_r12", (void *)kpi_thunk_r12 },
|
||||
{ "__x86_indirect_thunk_r13", (void *)kpi_thunk_r13 },
|
||||
{ "__x86_indirect_thunk_r14", (void *)kpi_thunk_r14 },
|
||||
{ "__x86_indirect_thunk_r15", (void *)kpi_thunk_r15 },
|
||||
{ "__x86_return_thunk", (void *)kpi_return_thunk },
|
||||
{ "__fentry__", (void *)kpi_fentry },
|
||||
|
||||
{ "_find_next_bit", (void *)kpi_find_next_bit },
|
||||
{ "queue_work_on", (void *)kpi_queue_work_on },
|
||||
{ "queue_delayed_work_on", (void *)kpi_queue_delayed_work_on },
|
||||
{ "cancel_work_sync", (void *)kpi_cancel_work_sync },
|
||||
{ "cancel_delayed_work_sync",(void *)kpi_cancel_delayed_work_sync },
|
||||
{ "cancel_delayed_work", (void *)kpi_cancel_delayed_work },
|
||||
{ "flush_work", (void *)kpi_flush_work },
|
||||
{ "dma_map_page_attrs", (void *)kpi_dma_map_page },
|
||||
{ "dma_unmap_page_attrs", (void *)kpi_dma_unmap_page },
|
||||
{ "__dma_sync_single_for_cpu", (void *)kpi_dma_sync },
|
||||
{ "__dma_sync_single_for_device", (void *)kpi_dma_sync },
|
||||
{ "alloc_pages_noprof", (void *)kpi_alloc_pages },
|
||||
{ "page_frag_free", (void *)kpi_page_frag_free },
|
||||
{ "__netdev_alloc_frag_align", (void *)kpi_netdev_alloc_frag },
|
||||
{ "page_offset_base", (void *)&kpi_page_offset_base },
|
||||
{ "vmemmap_base", (void *)&kpi_vmemmap_base },
|
||||
{ "__kmalloc_noprof", (void *)kpi_kmalloc },
|
||||
{ "vzalloc_noprof", (void *)kpi_vmalloc },
|
||||
{ "vmalloc_noprof", (void *)kpi_vmalloc },
|
||||
{ "vfree", (void *)kpi_vfree },
|
||||
{ "is_vmalloc_addr", (void *)kpi_is_vmalloc_addr },
|
||||
{ "__kmalloc_cache_noprof", (void *)kpi_kmalloc_cache },
|
||||
{ "kmalloc_noprof", (void *)kpi_kmalloc },
|
||||
{ "kfree", (void *)kpi_kfree },
|
||||
{ "krealloc_noprof", (void *)kpi_krealloc },
|
||||
|
|
@ -349,6 +702,8 @@ const struct kpi_sym kpi_table[] = {
|
|||
{ "_dev_err", (void *)kpi_netdev_print },
|
||||
|
||||
// Packets
|
||||
{ "napi_build_skb", (void *)kpi_build_skb },
|
||||
{ "build_skb", (void *)kpi_build_skb },
|
||||
{ "napi_alloc_skb", (void *)kpi_napi_alloc_skb },
|
||||
{ "__napi_alloc_skb", (void *)kpi_napi_alloc_skb },
|
||||
{ "__netdev_alloc_skb", (void *)kpi_netdev_alloc_skb },
|
||||
|
|
@ -358,15 +713,16 @@ const struct kpi_sym kpi_table[] = {
|
|||
{ "eth_type_trans", (void *)kpi_eth_type_trans },
|
||||
{ "netif_receive_skb", (void *)kpi_netif_receive_skb },
|
||||
{ "napi_gro_receive", (void *)kpi_netif_receive_skb },
|
||||
{ "gro_receive_skb", (void *)kpi_gro_receive_skb },
|
||||
{ "dev_kfree_skb_any_reason",(void *)kpi_kfree_skb_reason },
|
||||
{ "consume_skb", (void *)kpi_kfree_skb },
|
||||
{ "kfree_skb_reason", (void *)kpi_kfree_skb_reason },
|
||||
|
||||
// Bringing the interface up. LDLS polls rather than taking interrupts, so
|
||||
// request_irq succeeds and nothing is ever delivered through it -- see the
|
||||
// note in cmd/ldlsd on why a poll loop is the honest shape here.
|
||||
{ "request_threaded_irq", (void *)kpi_zero },
|
||||
{ "free_irq", (void *)kpi_nop },
|
||||
// Bringing the interface up. LDLS polls rather than taking interrupts, but
|
||||
// the handler is kept and called from the loop -- see kpi_request_irq.
|
||||
{ "request_threaded_irq", (void *)kpi_request_irq },
|
||||
{ "free_irq", (void *)kpi_free_irq },
|
||||
{ "disable_hardirq", (void *)kpi_nop },
|
||||
{ "enable_irq", (void *)kpi_nop },
|
||||
{ "disable_irq_nosync", (void *)kpi_nop },
|
||||
{ "napi_enable", (void *)kpi_nop },
|
||||
|
|
|
|||
|
|
@ -78,8 +78,18 @@ struct kpi_layout {
|
|||
unsigned int nd_tx, nd_num_tx, nd_real_num_tx;
|
||||
unsigned int nq_size, nq_dev;
|
||||
unsigned int nd_state, nd_flags;
|
||||
unsigned int pd_dev, dev_kobj, kobj_name;
|
||||
unsigned int ndo_open, ndo_stop, ndo_start_xmit;
|
||||
|
||||
// work_struct.func -- where a deferred work item keeps the function to call.
|
||||
// `struct delayed_work` starts with a work_struct, so one offset serves both.
|
||||
unsigned int work_func;
|
||||
|
||||
// __LINK_STATE_NOCARRIER, from BTF. `netif_carrier_ok` is an inline that
|
||||
// tests this bit of net_device.state, so it is the one number that decides
|
||||
// whether a driver believes its link came up.
|
||||
unsigned int link_nocarrier;
|
||||
|
||||
unsigned int skb_size;
|
||||
unsigned int skb_len, skb_data, skb_head, skb_tail, skb_end, skb_dev;
|
||||
};
|
||||
|
|
@ -95,6 +105,29 @@ extern struct kpi_layout kpi_l;
|
|||
extern int (*kpi_napi_poll)(void *napi, int budget);
|
||||
extern void *kpi_napi;
|
||||
|
||||
// The driver's own interrupt handler, kept and called from the loop. See the
|
||||
// note in kpi.c: this is not about receiving packets, it is about everything a
|
||||
// driver only ever does from inside its handler.
|
||||
int kpi_request_irq(unsigned int irq, void *handler, void *thread_fn,
|
||||
unsigned long flags, const char *name, void *dev);
|
||||
int kpi_free_irq(unsigned int irq, void *dev);
|
||||
int kpi_irq_poll(void);
|
||||
|
||||
// A driver's deferred work, run from the same loop that polls. See work.c --
|
||||
// e1000 enables its transmitter from a watchdog, so this is not optional.
|
||||
int kpi_queue_work_on(int cpu, void *q, void *work);
|
||||
int kpi_queue_delayed_work_on(int cpu, void *q, void *dwork, unsigned long delay);
|
||||
int kpi_cancel_work_sync(void *work);
|
||||
int kpi_cancel_delayed_work_sync(void *dwork);
|
||||
int kpi_cancel_delayed_work(void *dwork);
|
||||
int kpi_flush_work(void *work);
|
||||
int kpi_work_run(void);
|
||||
|
||||
// The clock those deadlines are measured against, and the `jiffies` a driver
|
||||
// reads directly.
|
||||
extern unsigned long kpi_jiffies;
|
||||
unsigned long kpi_jiffies_now(void);
|
||||
|
||||
// Output. The KPI is inside a process with no libc, and every implementation
|
||||
// wants to be able to say what it did.
|
||||
void kpi_log(const char *s);
|
||||
|
|
@ -105,6 +138,12 @@ void kpi_loghex(unsigned long long v, int digits);
|
|||
// default -- 538 of 8139too's 552 probe calls are ioread8/iowrite8.
|
||||
extern int kpi_verbose;
|
||||
|
||||
// Memory a device can be pointed at, in pieces smaller than a PCIIOC_DMA call.
|
||||
// The skb path uses it, because a driver that maps an skb for its device needs
|
||||
// the bus address of the data. See the long note in kpi.c.
|
||||
void *kpi_dma_heap(unsigned long n, unsigned long align);
|
||||
unsigned long long kpi_phys_of(const void *va);
|
||||
|
||||
// Is this a register address we handed out? Reports the first few that are
|
||||
// not, then goes quiet -- a driver that is off the rails produces thousands.
|
||||
int kpi_io_ok(void *addr, int width, int write);
|
||||
|
|
|
|||
|
|
@ -241,3 +241,28 @@ kpi_unregister_netdev(void *dev)
|
|||
kpi.netfd = -1;
|
||||
}
|
||||
}
|
||||
|
||||
// netif_carrier_off / netif_carrier_on.
|
||||
//
|
||||
// These were no-ops, and it cost a day. `netif_carrier_ok()` is an inline that
|
||||
// tests __LINK_STATE_NOCARRIER in `dev->state`, so with the bit never set a
|
||||
// driver reads "the carrier is already up" and skips its entire link-up path.
|
||||
// e1000's link-up path is where it enables the transmitter, so the visible
|
||||
// symptom was frames accepted and mapped and never sent -- and TCTL read back
|
||||
// from the chip with every field set but the enable.
|
||||
//
|
||||
// The bit number comes from the kernel's own BTF; see btf_enum_value().
|
||||
void
|
||||
kpi_netif_carrier_off(void *dev)
|
||||
{
|
||||
unsigned long *st = (unsigned long *)((char *)dev + kpi_l.nd_state);
|
||||
*st |= 1UL << kpi_l.link_nocarrier;
|
||||
}
|
||||
|
||||
void
|
||||
kpi_netif_carrier_on(void *dev)
|
||||
{
|
||||
unsigned long *st = (unsigned long *)((char *)dev + kpi_l.nd_state);
|
||||
*st &= ~(1UL << kpi_l.link_nocarrier);
|
||||
kpi_log("kpi: the driver says the link is up\n");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,11 @@
|
|||
#include "pciuser.h"
|
||||
#include "kpi.h"
|
||||
|
||||
// The PCI resource flag bits, which are ABI between the kernel and every
|
||||
// driver written against it. Not layout, so BTF does not carry them.
|
||||
#define LDLS_IORESOURCE_IO 0x00000100
|
||||
#define LDLS_IORESOURCE_MEM 0x00000200
|
||||
|
||||
// The device is claimed by the host before the driver runs, because claiming
|
||||
// is what enables memory space and bus mastering -- and a BAR read before that
|
||||
// can legitimately be zero. So by the time a driver calls this, it has already
|
||||
|
|
@ -145,6 +150,57 @@ kpi_pci_iomap(void *pdev, int bar, unsigned long max)
|
|||
return p;
|
||||
}
|
||||
|
||||
// pci_ioremap_bar(pdev, bar) -- the same thing as pci_iomap with no cap. A
|
||||
// second spelling, which is the sort of thing a second driver turns up.
|
||||
void *
|
||||
kpi_pci_ioremap_bar(void *pdev, int bar)
|
||||
{
|
||||
return kpi_pci_iomap(pdev, bar, 0);
|
||||
}
|
||||
|
||||
// pci_select_bars(pdev, flags) -- which BARs match a resource kind.
|
||||
//
|
||||
// A driver asks this before requesting regions, so the answer decides which
|
||||
// BARs it thinks exist. Reporting one that is not there gets it refused later;
|
||||
// omitting one it needs gets it to give up here.
|
||||
int
|
||||
kpi_pci_select_bars(void *pdev, unsigned long flags)
|
||||
{
|
||||
int bars = 0, b;
|
||||
|
||||
(void)pdev;
|
||||
for (b = 0; b < 6; b++) {
|
||||
if (!kpi.bar_base[b])
|
||||
continue;
|
||||
if ((kpi.bar_flags[b] & PCIUSER_BAR_IO) && (flags & LDLS_IORESOURCE_IO))
|
||||
bars |= 1 << b;
|
||||
if ((kpi.bar_flags[b] & PCIUSER_BAR_MMIO) && (flags & LDLS_IORESOURCE_MEM))
|
||||
bars |= 1 << b;
|
||||
}
|
||||
return bars;
|
||||
}
|
||||
|
||||
int
|
||||
kpi_pci_request_selected_regions(void *pdev, int bars, const char *name)
|
||||
{
|
||||
(void)pdev; (void)bars; (void)name;
|
||||
return 0; // nobody to arbitrate with; see above
|
||||
}
|
||||
|
||||
void
|
||||
kpi_pci_release_selected_regions(void *pdev, int bars)
|
||||
{
|
||||
(void)pdev; (void)bars;
|
||||
}
|
||||
|
||||
// Saving and restoring config space is what a driver does around a reset it
|
||||
// expects to clear the BARs. LDLS never resets the device at that level and
|
||||
// has no config write to restore with, so there is nothing to save -- and
|
||||
// saying so by succeeding is right, because a driver that cannot save state
|
||||
// refuses to probe at all.
|
||||
int kpi_pci_save_state(void *pdev) { (void)pdev; return 0; }
|
||||
void kpi_pci_restore_state(void *pdev) { (void)pdev; }
|
||||
|
||||
void
|
||||
kpi_pci_iounmap(void *pdev, void *addr)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -23,6 +23,19 @@
|
|||
// somewhere to go.
|
||||
#define KPI_HEADROOM 64
|
||||
|
||||
// Room past `end` for `struct skb_shared_info`.
|
||||
//
|
||||
// `skb_shinfo(skb)` is `(struct skb_shared_info *)(skb->head + skb->end)` --
|
||||
// it is not a field, it lives immediately after the data, and every allocator
|
||||
// in Linux reserves space for it. 8139too never looked, so an skb that ended
|
||||
// exactly at `end` worked; e1000 reads `nr_frags` on every single transmit,
|
||||
// and read it from whatever followed the allocation.
|
||||
//
|
||||
// The struct is 320 bytes on x86-64 and this is deliberately more. It is not
|
||||
// read from BTF only because getting it wrong in the small direction is
|
||||
// silent, and a fixed size that is comfortably too large cannot be.
|
||||
#define KPI_SHINFO 512
|
||||
|
||||
static unsigned int
|
||||
rd32(void *base, unsigned int off)
|
||||
{
|
||||
|
|
@ -67,15 +80,26 @@ void *
|
|||
kpi_skb_alloc(unsigned int len)
|
||||
{
|
||||
unsigned char *p;
|
||||
unsigned int total = kpi_l.skb_size + KPI_HEADROOM + len + 64;
|
||||
unsigned int total = kpi_l.skb_size + KPI_HEADROOM + len + 64 + KPI_SHINFO;
|
||||
unsigned char *buf;
|
||||
|
||||
if (kpi_l.skb_size == 0)
|
||||
return 0;
|
||||
|
||||
p = mmap(0, total, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
if (p == 0 || (long)p == -1)
|
||||
// From the DMA arena, not from mmap.
|
||||
//
|
||||
// A driver that maps an skb for the device -- e1000 does, 8139too did not --
|
||||
// needs the bus address of its data, and a process can only know that for
|
||||
// memory the kernel handed it through PCIIOC_DMA. Putting the control block
|
||||
// in there too wastes a little arena and keeps one buffer one allocation.
|
||||
p = kpi_dma_heap(total, 64);
|
||||
if (p == 0)
|
||||
return 0;
|
||||
{
|
||||
unsigned int i;
|
||||
for (i = 0; i < total; i++)
|
||||
p[i] = 0; // the arena is recycled; mmap's zeroes are not
|
||||
}
|
||||
|
||||
buf = p + kpi_l.skb_size;
|
||||
wrptr(p, kpi_l.skb_head, buf);
|
||||
|
|
@ -86,6 +110,48 @@ kpi_skb_alloc(unsigned int len)
|
|||
return p;
|
||||
}
|
||||
|
||||
// napi_build_skb(data, frag_len) -- an skb wrapped around a buffer that
|
||||
// already exists.
|
||||
//
|
||||
// The other half of the receive path, and the one that carries anything
|
||||
// larger than `copybreak` (256 bytes by default). e1000 hands the device a
|
||||
// raw fragment, and when a frame arrives it wraps that same fragment rather
|
||||
// than copying it -- so the control block is allocated here and the data is
|
||||
// not. A DHCP offer is 590 bytes, which is why nothing larger than a ping
|
||||
// reply worked until this existed.
|
||||
//
|
||||
// `skb_shared_info` lives at the *end of the buffer* in this arrangement,
|
||||
// which is what makes `end` smaller than the fragment: the driver picked a
|
||||
// fragment size that already includes room for it.
|
||||
void *
|
||||
kpi_build_skb(void *data, unsigned int frag_len)
|
||||
{
|
||||
unsigned char *p;
|
||||
unsigned int i, room;
|
||||
|
||||
if (kpi_l.skb_size == 0 || data == 0 || frag_len <= KPI_SHINFO)
|
||||
return 0;
|
||||
|
||||
p = kpi_dma_heap(kpi_l.skb_size, 64);
|
||||
if (!p)
|
||||
return 0;
|
||||
for (i = 0; i < kpi_l.skb_size; i++)
|
||||
p[i] = 0;
|
||||
|
||||
room = frag_len - KPI_SHINFO;
|
||||
// And zero the shared info, because this buffer is recycled arena and the
|
||||
// driver reads nr_frags out of it without writing it first.
|
||||
for (i = 0; i < KPI_SHINFO; i++)
|
||||
((unsigned char *)data)[room + i] = 0;
|
||||
|
||||
wrptr(p, kpi_l.skb_head, data);
|
||||
wrptr(p, kpi_l.skb_data, data);
|
||||
wr32(p, kpi_l.skb_tail, 0);
|
||||
wr32(p, kpi_l.skb_end, room);
|
||||
wr32(p, kpi_l.skb_len, 0);
|
||||
return p;
|
||||
}
|
||||
|
||||
void *kpi_napi_alloc_skb(void *napi, unsigned int len)
|
||||
{ (void)napi; return kpi_skb_alloc(len); }
|
||||
|
||||
|
|
@ -146,6 +212,18 @@ kpi_netif_receive_skb(void *skb)
|
|||
return 0;
|
||||
}
|
||||
|
||||
// gro_receive_skb(gro, skb) -- the same thing, with the GRO node in front.
|
||||
//
|
||||
// `napi_gro_receive` is an inline wrapper in this kernel, so what a module
|
||||
// actually calls is this, and the skb is the *second* argument. Taking the
|
||||
// first would hand the stack a `struct gro_node` and call it a frame.
|
||||
int
|
||||
kpi_gro_receive_skb(void *gro, void *skb)
|
||||
{
|
||||
(void)gro;
|
||||
return kpi_netif_receive_skb(skb);
|
||||
}
|
||||
|
||||
// eth_type_trans(skb, dev) -- consume the Ethernet header and report the
|
||||
// protocol. The stack below wants the whole frame including that header, so
|
||||
// unlike Linux's this does not pull it off; what it returns is the type field,
|
||||
|
|
|
|||
|
|
@ -58,10 +58,33 @@ kpi_memcmp(const void *x, const void *y, unsigned long n)
|
|||
return 0;
|
||||
}
|
||||
|
||||
// A null string is survivable, and reported.
|
||||
//
|
||||
// A hosted driver reads names out of structures this host only partly fills
|
||||
// in, and the first one it found was `pci_name(pdev)` -- a pointer into a
|
||||
// `struct device` that had never been given one. Faulting inside strnlen tells
|
||||
// you nothing about which field was missing; answering zero and saying so
|
||||
// leaves the driver running and names the problem.
|
||||
static long kpi_nullstr;
|
||||
|
||||
static int
|
||||
str_ok(const char *s, const char *who)
|
||||
{
|
||||
if (s)
|
||||
return 1;
|
||||
if (++kpi_nullstr <= 4) {
|
||||
kpi_log(" kpi: "); kpi_log(who);
|
||||
kpi_log(" on a null string -- some field of a struct this host fills in "
|
||||
"only partly\n");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
unsigned long
|
||||
kpi_strlen(const char *s)
|
||||
{
|
||||
unsigned long n = 0;
|
||||
if (!str_ok(s, "strlen")) return 0;
|
||||
while (s[n]) n++;
|
||||
return n;
|
||||
}
|
||||
|
|
@ -70,6 +93,7 @@ unsigned long
|
|||
kpi_strnlen(const char *s, unsigned long max)
|
||||
{
|
||||
unsigned long n = 0;
|
||||
if (!str_ok(s, "strnlen")) return 0;
|
||||
while (n < max && s[n]) n++;
|
||||
return n;
|
||||
}
|
||||
|
|
@ -83,6 +107,7 @@ kpi_sized_strscpy(char *d, const char *s, unsigned long n)
|
|||
|
||||
if (n == 0)
|
||||
return -7; /* -E2BIG */
|
||||
if (!str_ok(s, "strscpy")) { d[0] = 0; return 0; }
|
||||
while (i + 1 < n && s[i]) { d[i] = s[i]; i++; }
|
||||
d[i] = 0;
|
||||
return s[i] ? -7 : (long)i;
|
||||
|
|
|
|||
71
ldls/kpi/thunk.c
Normal file
71
ldls/kpi/thunk.c
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
// Retpoline thunks. See ldls/kpi/kpi.h for the provenance rule.
|
||||
//
|
||||
// These are not API. They are the compiler's, emitted by `-mretpoline` /
|
||||
// `-mfunction-return`, and a module built for a mitigated kernel is full of
|
||||
// them: every indirect call becomes `jmp __x86_indirect_thunk_rN` and every
|
||||
// return becomes `jmp __x86_return_thunk`.
|
||||
//
|
||||
// They have to be assembly, because what they are *is* control flow. A C
|
||||
// function cannot express "jump to whatever is in %r8 without touching any
|
||||
// other register", and the stub the loader supplies for an unimplemented
|
||||
// symbol -- which logs its name and returns zero -- is exactly the wrong
|
||||
// answer: it discards the jump.
|
||||
//
|
||||
// That cost a day, and the shape of the failure is the reason this file has a
|
||||
// long comment. e1000 refills its receive ring through a function pointer, and
|
||||
// the compiler made the call a *tail* call:
|
||||
//
|
||||
// mov 0x2a0(%rbx),%r8 ; adapter->alloc_rx_buf
|
||||
// ... ; compute E1000_DESC_UNUSED(ring)
|
||||
// jmp __x86_indirect_thunk_r8
|
||||
//
|
||||
// so the one thing in the log was `__x86_indirect_thunk_r8(...)` among four
|
||||
// thousand other stub calls, and the visible symptom was a NIC that
|
||||
// transmitted perfectly and received nothing, with RDT sitting at zero. The
|
||||
// driver had asked for 255 receive buffers and the shim had answered "yes".
|
||||
//
|
||||
// **These are the plain versions, with no mitigation in them.** A real
|
||||
// retpoline traps speculation in a loop; this is `jmp *%reg`. Spectre-v2
|
||||
// hardening for foreign code in a process is not something LDLS can offer
|
||||
// honestly -- the kernel it runs under is not mitigated either -- and a thunk
|
||||
// that pretends otherwise would be worse than one that says so here.
|
||||
|
||||
// One per register, because the register is in the name -- that is the whole
|
||||
// point of the encoding. `endbr64` heads each one because a module built with
|
||||
// IBT reaches them by indirect jump.
|
||||
#define THUNK(reg) \
|
||||
".globl kpi_thunk_" #reg "\n" \
|
||||
".type kpi_thunk_" #reg ", @function\n" \
|
||||
"kpi_thunk_" #reg ":\n" \
|
||||
" endbr64\n" \
|
||||
" jmp *%" #reg "\n" \
|
||||
".size kpi_thunk_" #reg ", .-kpi_thunk_" #reg "\n"
|
||||
|
||||
__asm__(
|
||||
".text\n"
|
||||
THUNK(rax) THUNK(rbx) THUNK(rcx) THUNK(rdx)
|
||||
THUNK(rsi) THUNK(rdi) THUNK(rbp)
|
||||
THUNK(r8) THUNK(r9) THUNK(r10) THUNK(r11)
|
||||
THUNK(r12) THUNK(r13) THUNK(r14) THUNK(r15)
|
||||
|
||||
// __x86_return_thunk is just a return. It had been resolving to a stub,
|
||||
// which happened to work -- the stub is reached by `jmp`, so its own `ret`
|
||||
// returned to the right place -- while clobbering %rax with zero on the way
|
||||
// out. Every function whose return value came back as 0 for no reason was
|
||||
// this.
|
||||
".globl kpi_return_thunk\n"
|
||||
".type kpi_return_thunk, @function\n"
|
||||
"kpi_return_thunk:\n"
|
||||
" ret\n"
|
||||
".size kpi_return_thunk, .-kpi_return_thunk\n"
|
||||
|
||||
// __fentry__ is the ftrace hook: on an untraced kernel it is a bare return,
|
||||
// and it is called at the top of essentially every function in the module.
|
||||
".globl kpi_fentry\n"
|
||||
".type kpi_fentry, @function\n"
|
||||
"kpi_fentry:\n"
|
||||
" ret\n"
|
||||
".size kpi_fentry, .-kpi_fentry\n"
|
||||
);
|
||||
142
ldls/kpi/work.c
Normal file
142
ldls/kpi/work.c
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
// A driver's periodic work. See ldls/kpi/kpi.h for the provenance rule.
|
||||
//
|
||||
// e1000 is the reason this exists, and the way it failed is worth keeping.
|
||||
// The driver probed, opened, accepted frames from the stack, mapped them and
|
||||
// wrote the transmit tail register -- and the wire stayed silent. Reading the
|
||||
// chip's own registers said why:
|
||||
//
|
||||
// TCTL=0x010000f8 PSP, collision threshold, retry-on-late-collision
|
||||
// -- and bit 1, the enable, clear
|
||||
// RDT=0 no receive descriptors offered to the device
|
||||
//
|
||||
// Neither is set by `ndo_open`. e1000 enables its transmitter in
|
||||
// `e1000_watchdog`, once it has seen the link come up, and refills the receive
|
||||
// ring from the same place. The watchdog is a `delayed_work`, and LDLS was
|
||||
// dropping every one on the floor: `queue_delayed_work_on` resolved to a stub
|
||||
// that returned zero, which is the correct return value and does nothing.
|
||||
//
|
||||
// So a hosted driver needs somewhere for deferred work to actually run. There
|
||||
// is no thread to run it on, which is the same constraint that made the NAPI
|
||||
// poll a loop -- and the same answer: keep the function, and call it from the
|
||||
// loop that was going to spin anyway.
|
||||
//
|
||||
// **A work item here runs on the main loop, not concurrently.** That is why
|
||||
// there are no locks, and it is also the limitation: a driver whose work item
|
||||
// blocks stops everything, including the transmit path it is probably waiting
|
||||
// for.
|
||||
|
||||
#include "types.h"
|
||||
#include "user.h"
|
||||
#include "posix.h"
|
||||
#include "kpi.h"
|
||||
|
||||
#define KPI_NWORK 16
|
||||
|
||||
static struct {
|
||||
void *work; // struct work_struct *, our identity
|
||||
unsigned long due; // in jiffies
|
||||
int armed;
|
||||
} wq[KPI_NWORK];
|
||||
|
||||
static void
|
||||
rdfn(void *work, void (**fn)(void *))
|
||||
{
|
||||
unsigned char *p = (unsigned char *)work + kpi_l.work_func;
|
||||
int i;
|
||||
for (i = 0; i < 8; i++) ((unsigned char *)fn)[i] = p[i];
|
||||
}
|
||||
|
||||
static int
|
||||
slot_for(void *work)
|
||||
{
|
||||
int i, free = -1;
|
||||
for (i = 0; i < KPI_NWORK; i++) {
|
||||
if (wq[i].armed && wq[i].work == work)
|
||||
return i;
|
||||
if (!wq[i].armed && free < 0)
|
||||
free = i;
|
||||
}
|
||||
return free;
|
||||
}
|
||||
|
||||
static int
|
||||
arm(void *work, unsigned long delay)
|
||||
{
|
||||
int i;
|
||||
|
||||
if (kpi_l.work_func == 0) {
|
||||
kpi_log("kpi: deferred work with no work_struct.func offset\n");
|
||||
return 0;
|
||||
}
|
||||
i = slot_for(work);
|
||||
if (i < 0) {
|
||||
kpi_log("kpi: too many deferred work items\n");
|
||||
return 0;
|
||||
}
|
||||
// Re-arming an item that is already queued moves its deadline, which is what
|
||||
// queue_delayed_work does *not* do -- it leaves the first deadline standing.
|
||||
// The difference only shows up as work running sooner than a driver asked,
|
||||
// and running a watchdog early is harmless in a way that running it late is
|
||||
// not.
|
||||
wq[i].work = work;
|
||||
wq[i].due = kpi_jiffies_now() + delay;
|
||||
wq[i].armed = 1;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// queue_work_on(cpu, wq, work)
|
||||
int kpi_queue_work_on(int cpu, void *q, void *work)
|
||||
{ (void)cpu; (void)q; return arm(work, 0); }
|
||||
|
||||
// queue_delayed_work_on(cpu, wq, dwork, delay)
|
||||
//
|
||||
// `struct delayed_work` begins with its `work_struct`, so the pointer is
|
||||
// already the right one -- which is the only reason this does not need a
|
||||
// second BTF offset.
|
||||
int
|
||||
kpi_queue_delayed_work_on(int cpu, void *q, void *dwork, unsigned long delay)
|
||||
{ (void)cpu; (void)q; return arm(dwork, delay); }
|
||||
|
||||
int
|
||||
kpi_cancel_work_sync(void *work)
|
||||
{
|
||||
int i = slot_for(work);
|
||||
if (i >= 0 && wq[i].armed && wq[i].work == work) { wq[i].armed = 0; return 1; }
|
||||
return 0;
|
||||
}
|
||||
|
||||
int kpi_cancel_delayed_work_sync(void *dwork) { return kpi_cancel_work_sync(dwork); }
|
||||
int kpi_cancel_delayed_work(void *dwork) { return kpi_cancel_work_sync(dwork); }
|
||||
int kpi_flush_work(void *work) { (void)work; return 0; }
|
||||
void kpi_flush_workqueue(void *q) { (void)q; }
|
||||
|
||||
// Run whatever is due. Returns how many ran, so the caller's loop can tell
|
||||
// whether anything happened.
|
||||
//
|
||||
// The slot is disarmed *before* the call, because the overwhelming majority of
|
||||
// periodic work in a driver ends by queueing itself again -- and disarming
|
||||
// after would throw that away and stop the clock.
|
||||
int
|
||||
kpi_work_run(void)
|
||||
{
|
||||
unsigned long now = kpi_jiffies_now();
|
||||
int i, n = 0;
|
||||
|
||||
for (i = 0; i < KPI_NWORK; i++) {
|
||||
void (*fn)(void *) = 0;
|
||||
void *w;
|
||||
|
||||
if (!wq[i].armed || (long)(now - wq[i].due) < 0)
|
||||
continue;
|
||||
w = wq[i].work;
|
||||
wq[i].armed = 0;
|
||||
rdfn(w, &fn);
|
||||
if (!fn)
|
||||
continue;
|
||||
fn(w);
|
||||
n++;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
Loading…
Reference in a new issue