diff --git a/Makefile b/Makefile index 91ee2ac..0acbd29 100644 --- a/Makefile +++ b/Makefile @@ -1098,7 +1098,7 @@ $(BUILD)/lib/%.o: lib/%.S # stub.c are freestanding and build unchanged for either side, which is the # whole point of having written them that way. LDLS_GUEST = $(addprefix $(BUILD)/ldls/guest/, ko.o btf.o stub.o) \ - $(addprefix $(BUILD)/ldls/guest/kpi/, kpi.o pci.o io.o net.o skb.o string.o work.o thunk.o) + $(addprefix $(BUILD)/ldls/guest/kpi/, kpi.o pci.o io.o net.o skb.o string.o work.o thunk.o base.o) $(BUILD)/ldls/guest/%.o: ldls/%.c ldls/ko.h ldls/btf.h ldls/stub.h @mkdir -p $(BUILD)/ldls/guest @@ -1106,7 +1106,7 @@ $(BUILD)/ldls/guest/%.o: ldls/%.c ldls/ko.h ldls/btf.h ldls/stub.h # The KPI -- the Linux symbols, implemented. Guest-only: it reaches /dev/pci # and /dev/netdev, which exist on SchOS and nowhere else. -$(BUILD)/ldls/guest/kpi/%.o: ldls/kpi/%.c ldls/kpi/kpi.h +$(BUILD)/ldls/guest/kpi/%.o: ldls/kpi/%.c ldls/kpi/kpi.h ldls/kpi/link_modes.inc @mkdir -p $(BUILD)/ldls/guest/kpi $(CC) $(CFLAGS) -I./ldls -c $< -o $@ @@ -1378,6 +1378,33 @@ ldls-stage: @test -z "$(BTF)" || cp "$(BTF)" $(SYSROOT)/lib/modules/btf @echo "staged into $(SYSROOT)/lib/modules -- this image is now yours alone" +# Linux's r8169 and what it needs, from the kernel this machine runs, for +# etc/schinit/ldls-r8169.job. The module set is r8169's dependency closure less +# what LDLS answers itself (led-class, of_mdio, fwnode_mdio, fixed_phy), in load +# order; the firmware is the RTL8168/8111 PHY patches. SchOS is GPL-2.0-or-later +# and ships these knowingly -- see docs/plan/gpu.md. +LDLS_KDIR ?= /run/current-system/kernel-modules/lib/modules/$(shell uname -r)/kernel +LDLS_FWDIR ?= /run/current-system/firmware +LDLS_R8169 = drivers/net/phy/mdio-bus drivers/net/phy/libphy \ + drivers/net/phy/mdio_devres drivers/net/phy/realtek/realtek \ + drivers/net/ethernet/realtek/r8169 +ldls-r8169: + @mkdir -p $(SYSROOT)/lib/modules $(SYSROOT)/lib/firmware/rtl_nic + @set -e; list=; for m in $(LDLS_R8169); do \ + f=$$(ls $(LDLS_KDIR)/$$m.ko* | head -1); n=$$(basename $$m).ko; \ + case $$f in *.xz) xz -dc $$f;; *.zst) zstd -qdc $$f;; *) cat $$f;; esac \ + > $(SYSROOT)/lib/modules/$$n; \ + list=$$list$${list:+,}/lib/modules/$$n; done; \ + cp /sys/kernel/btf/vmlinux $(SYSROOT)/lib/modules/btf; \ + printf '#!/bin/sh\nmkdir -p /var/log\nexec /bin/ldlsd %s /lib/modules/btf > /var/log/ldls-r8169.log 2>&1\n' \ + "$$list" > $(SYSROOT)/lib/modules/ldls-r8169.sh + @for f in $(LDLS_FWDIR)/rtl_nic/rtl8168*.fw*; do \ + n=$$(basename $$f); case $$n in \ + *.zst) zstd -qdc $$f > $(SYSROOT)/lib/firmware/rtl_nic/$${n%.zst};; \ + *.xz) xz -dc $$f > $(SYSROOT)/lib/firmware/rtl_nic/$${n%.xz};; \ + *) cp $$f $(SYSROOT)/lib/firmware/rtl_nic/;; esac; done + @echo "staged Linux r8169 ($(shell uname -r)) into $(SYSROOT)" + ldls-unstage: rm -rf $(SYSROOT)/lib/modules @echo "removed" diff --git a/cmd/ldlsd/main.c b/cmd/ldlsd/main.c index 379ace4..5105b84 100644 --- a/cmd/ldlsd/main.c +++ b/cmd/ldlsd/main.c @@ -185,6 +185,32 @@ ldls_stub(long idx, unsigned long *args) struct rctx { struct ko *k; struct stubs *s; }; +// Modules loaded before the one being relocated. A module set is loaded in +// dependency order -- libphy before r8169, ttm before amdgpu -- and what an +// earlier one defines is what a later one imports from it. +#define MAXMOD 32 +static struct ko *g_mods[MAXMOD]; +static int g_nmods; + +static unsigned long long +find_global(struct ko *k, const char *want) +{ + int i; + for (i = 0; i < k->nsym; i++) { + if ((k->sym[i].st_info >> 4) == 0) // STB_LOCAL: not for others + continue; + if (k->sym[i].st_shndx == KO_SHN_UNDEF || + k->sym[i].st_shndx >= (unsigned short)k->nsh) + continue; + if (!k->sec[k->sym[i].st_shndx].alloc) + continue; + if (!streq(k->str + k->sym[i].st_name, want)) + continue; + return k->sec[k->sym[i].st_shndx].addr + k->sym[i].st_value; + } + return 0; +} + // The KPI first, a logging stub second. // // An unimplemented symbol still resolves -- to something that reports itself @@ -208,6 +234,14 @@ resolve(void *ctx, const char *name, unsigned long long *out) return 0; } + for (i = 0; i < g_nmods; i++) { + unsigned long long a = find_global(g_mods[i], name); + if (a) { + *out = a; + return 0; + } + } + for (i = 0; i < r->k->nundef; i++) if (streq(r->k->undef[i], name)) { *out = stub_addr(r->s, i); @@ -445,8 +479,42 @@ kpi_layout_from_btf(struct btf *b) GET(pd, "dev", kpi_l.pd_dev); GET(dv, "kobj", kpi_l.dev_kobj); GET(ko, "name", kpi_l.kobj_name); + kpi_l.dev_kobj_ok = 1; } } + // The driver core and struct firmware. Optional: a driver that never + // registers a bus or loads firmware does not need them, so a BTF without + // one of these turns the feature off rather than refusing to run. + { + int dv = btf_find(b, "device", BTF_KIND_STRUCT); + int dd = btf_find(b, "device_driver", BTF_KIND_STRUCT); + int bt = btf_find(b, "bus_type", BTF_KIND_STRUCT); + int fw = btf_find(b, "firmware", BTF_KIND_STRUCT); + int ok = dv && dd && bt; +#define OPT(id, name, field) \ + do { if (btf_member_off(b, id, name, &off, &size) < 0) ok = 0; \ + else field = off; } while (0) + if (ok) { + OPT(dv, "bus", kpi_l.dev_bus); + OPT(dv, "driver", kpi_l.dev_driver); + OPT(dd, "bus", kpi_l.drv_bus); + OPT(dd, "probe", kpi_l.drv_probe); + OPT(dd, "name", kpi_l.drv_name); + OPT(bt, "match", kpi_l.bus_match); + OPT(bt, "probe", kpi_l.bus_probe); + } + kpi_l.core_ok = ok; + if (fw) { + ok = 1; + OPT(fw, "size", kpi_l.fw_size); + OPT(fw, "data", kpi_l.fw_data); + kpi_l.fw_size_of = btf_type_size(b, fw); + kpi_l.fw_ok = ok; + } +#undef OPT + puts_(" kpi driver core "); puts_(kpi_l.core_ok ? "on" : "off"); + puts_(", firmware "); puts_(kpi_l.fw_ok ? "on" : "off"); puts_("\n"); + } { int nq = btf_find(b, "netdev_queue", BTF_KIND_STRUCT); if (!nq) { puts_("ldlsd: no struct netdev_queue\n"); return -1; } @@ -682,8 +750,11 @@ drive_probe(const char *btfpath) { struct pciuser_info in; + // No device: not a failure worth restarting for. schinit's + // KeepAlive would otherwise start this again every few seconds on + // every machine without the NIC. See SCHINIT_EXIT_NORESTART. if (claim_for(idtab, id_size, id_vendor, id_device, &in) < 0) - return; + exit_group(78); pdev = (unsigned long)getmem(dev_size); if (!pdev) { puts_("ldlsd: no memory for a pci_dev\n"); return; } @@ -714,6 +785,26 @@ drive_probe(const char *btfpath) (unsigned char)(q >> (i * 8)); } + // pdev->bus. pci_domain_nr() and pci_dev_id() are inlines that + // dereference it -- r8169 names its MDIO bus with both -- and a null + // one is a fault at the first line of MDIO setup. + { + unsigned int pd_bus = lay(&b, "pci_dev", "bus"); + unsigned int pb_num = lay(&b, "pci_bus", "number"); + unsigned int pb_sys = lay(&b, "pci_bus", "sysdata"); + unsigned int pb_size = lay(&b, "pci_bus", 0); + if (pd_bus != 0xffffffffu && pb_num != 0xffffffffu && + pb_sys != 0xffffffffu && pb_size != 0xffffffffu) { + unsigned long bus = (unsigned long)getmem(pb_size); + unsigned long sys = (unsigned long)getmem(4096); + if (bus && sys) { + ((unsigned char *)bus)[pb_num] = (unsigned char)in.bus; + bcopy_((void *)(bus + pb_sys), &sys, 8); + bcopy_((void *)(pdev + pd_bus), &bus, 8); + } + } + } + wr16(pdev, dev_vendor, in.vendor); wr16(pdev, dev_device, in.device); wr32(pdev, dev_devfn, (unsigned int)((in.slot << 3) | in.func)); @@ -795,21 +886,139 @@ drive_probe(const char *btfpath) } } -// --- main ---------------------------------------------------------------------- - -int -main(int argc, char *argv[]) +// Read the BTF and fill in the KPI's layouts, before any module runs. +static void +early_layout(const char *btfpath) { - struct ko k; + static struct btf b; + unsigned long long blen; + unsigned char *bimg = slurp(btfpath, &blen); + long nt; + unsigned int *boff; + + if (!bimg) return; + nt = btf_count(bimg, blen); + if (nt < 0) return; + boff = getmem(((unsigned long long)nt + 2) * 4); + if (!boff || btf_open(&b, bimg, blen, boff, (unsigned int)nt + 2) < 0) + return; + { + extern unsigned int kpi_netdev_size; + int nd = btf_find(&b, "net_device", BTF_KIND_STRUCT); + kpi_netdev_size = nd ? btf_type_size(&b, nd) : 0; + } + kpi_layout_from_btf(&b); +} + +// One module: read, laid out, relocated, its %gs references fixed, and its +// init_module called. Kept loaded for the ones after it to link against. +static void +load_module(const char *path) +{ + struct ko *k; struct ko_section *sec; - struct stubs stubs; + struct stubs *stubs; struct rctx rc; unsigned char *img, *mem, *code; unsigned long long len, span, entry; int nsec, i; + k = getmem(sizeof(*k)); + stubs = getmem(sizeof(*stubs)); + if (!k || !stubs) { puts_("ldlsd: out of memory\n"); exit(); } + + img = slurp(path, &len); + if (!img) { puts_("ldlsd: cannot read "); puts_(path); puts_("\n"); exit(); } + + nsec = ko_sections_needed(img, len); + if (nsec < 0) { + puts_("ldlsd: that is not an ELF64 relocatable object\n"); + exit(); + } + sec = getmem((unsigned long long)(nsec + 1) * sizeof(struct ko_section)); + if (!sec) { puts_("ldlsd: out of memory\n"); exit(); } + + if (ko_open(k, img, len, sec, nsec) < 0) { + puts_("ldlsd: "); puts_(ko_err(k)); puts_("\n"); + exit(); + } + + puts_(path); puts_("\n"); + puts_(" module "); + puts_(ko_name(k) ? ko_name(k) : "(not a kernel module)"); puts_("\n"); + puts_(" license "); + puts_(ko_modinfo(k, "license") ? ko_modinfo(k, "license") + : "(none declared)"); puts_("\n"); + puts_(" wants "); putn(k->nundef); puts_(" symbols\n"); + + span = ko_span(k); + mem = getmem(span ? span : 4096); + code = getmem((unsigned long long)(k->nundef + 1) * STUB_SIZE); + if (!mem || !code) { puts_("ldlsd: out of memory\n"); exit(); } + + stub_build_from(stubs, code, k->nundef, g_count); + for (i = 0; i < k->nundef; i++) { + g_names[g_count + i] = k->undef[i]; + if (stub_is_bare(k->undef[i])) + stub_make_bare(code + (unsigned long long)i * STUB_SIZE); + } + g_count += k->nundef; + k_nundef_global = g_count; + + if (ko_layout(k, mem, span, (unsigned long long)mem) < 0) { + puts_("ldlsd: layout: "); puts_(ko_err(k)); puts_("\n"); + exit(); + } + + rc.k = k; rc.s = stubs; + if (ko_relocate(k, resolve, &rc) < 0) { + puts_("ldlsd: relocate: "); puts_(ko_err(k)); puts_("\n"); + exit(); + } + // The stack canary is fetched through %gs, which belongs to the kernel + // here. See ko_defuse_gs(). + { + int g = ko_defuse_gs(k, "__ref_stack_chk_guard"); + if (g < 0) g = 0; + if (g == 0) g = ko_defuse_gs(k, "__stack_chk_guard"); + if (g > 0) { puts_(" defused "); putn(g); puts_(" %gs stack-guard load(s)\n"); } + } + // Whatever %gs references are left name real per-CPU variables, and those + // cannot be defused -- the driver dereferences what it reads. + { + int p = ko_fixup_percpu(k, percpu_for, 0); + if (p < 0) { + puts_("ldlsd: per-CPU: "); puts_(ko_err(k)); puts_("\n"); + exit(); + } + if (p > 0) { puts_(" rehomed "); putn(p); puts_(" %gs per-CPU reference(s)\n"); } + } + + puts_(" loaded "); putn(span); puts_(" bytes at "); puthex((unsigned long long)mem); + puts_("\n"); + + entry = find_symbol(k, "init_module"); + if (g_nmods < MAXMOD) + g_mods[g_nmods++] = k; + if (!entry) { puts_(" no init_module -- nothing to call\n"); return; } + + puts_("\n calling init_module\n\n"); + { + int (*init)(void) = (int (*)(void))(void *)entry; + int r = init(); + puts_("\n init_module returned "); putn(r); + puts_(" after "); putn(g_calls); puts_(" stub call(s)\n"); + } + +} + +// --- main ---------------------------------------------------------------------- + +int +main(int argc, char *argv[]) +{ if (argc < 2) { - puts_("usage: ldlsd [vmlinux-btf] [bus:slot.func]\n"); + puts_("usage: ldlsd [,...] [vmlinux-btf] [bus:slot.func]\n"); puts_(" neither file ships with SchOS; both are yours.\n"); exit(); } @@ -837,85 +1046,32 @@ main(int argc, char *argv[]) puthexn(sel_slot, 2); puts_("."); puthexn(sel_func, 1); puts_("\n"); } - img = slurp(argv[1], &len); - if (!img) { puts_("ldlsd: cannot read "); puts_(argv[1]); puts_("\n"); exit(); } - - nsec = ko_sections_needed(img, len); - if (nsec < 0) { - puts_("ldlsd: that is not an ELF64 relocatable object\n"); - exit(); - } - sec = getmem((unsigned long long)(nsec + 1) * sizeof(struct ko_section)); - if (!sec) { puts_("ldlsd: out of memory\n"); exit(); } - - if (ko_open(&k, img, len, sec, nsec) < 0) { - puts_("ldlsd: "); puts_(ko_err(&k)); puts_("\n"); - exit(); - } - - puts_(argv[1]); puts_("\n"); - puts_(" module "); - puts_(ko_name(&k) ? ko_name(&k) : "(not a kernel module)"); puts_("\n"); - puts_(" license "); - puts_(ko_modinfo(&k, "license") ? ko_modinfo(&k, "license") - : "(none declared)"); puts_("\n"); - puts_(" wants "); putn(k.nundef); puts_(" symbols\n"); - - span = ko_span(&k); - mem = getmem(span ? span : 4096); - code = getmem((unsigned long long)(k.nundef + 1) * STUB_SIZE); - if (!mem || !code) { puts_("ldlsd: out of memory\n"); exit(); } - - stub_build(&stubs, code, k.nundef); - g_names = k.undef; - g_count = k.nundef; - k_nundef_global = k.nundef; - kpi.netfd = -1; - for (i = 0; i < k.nundef; i++) - if (stub_is_bare(k.undef[i])) - stub_make_bare(code + (unsigned long long)i * STUB_SIZE); - - if (ko_layout(&k, mem, span, (unsigned long long)mem) < 0) { - puts_("ldlsd: layout: "); puts_(ko_err(&k)); puts_("\n"); - exit(); - } - - rc.k = &k; rc.s = &stubs; - if (ko_relocate(&k, resolve, &rc) < 0) { - puts_("ldlsd: relocate: "); puts_(ko_err(&k)); puts_("\n"); - exit(); - } - // The stack canary is fetched through %gs, which belongs to the kernel - // here. See ko_defuse_gs(). + // argv[1] is one module, or a set in dependency order separated by + // commas: libphy.ko,realtek.ko,r8169.ko. Each is loaded, relocated + // against the KPI and the ones before it, and initialised before the next. { - int g = ko_defuse_gs(&k, "__ref_stack_chk_guard"); - if (g < 0) g = 0; - if (g == 0) g = ko_defuse_gs(&k, "__stack_chk_guard"); - if (g > 0) { puts_(" defused "); putn(g); puts_(" %gs stack-guard load(s)\n"); } - } - // Whatever %gs references are left name real per-CPU variables, and those - // cannot be defused -- the driver dereferences what it reads. - { - int p = ko_fixup_percpu(&k, percpu_for, 0); - if (p < 0) { - puts_("ldlsd: per-CPU: "); puts_(ko_err(&k)); puts_("\n"); - exit(); + static char list[4096]; + char *p = list, *name; + int n = 0; + while (argv[1][n] && n < (int)sizeof(list) - 1) { list[n] = argv[1][n]; n++; } + list[n] = 0; + // The layouts first: init_module is where a module registers its + // drivers, and the driver core needs struct offsets to do that. + if (argc > 2) + early_layout(argv[2]); + g_names = getmem(sizeof(char *) * (unsigned long long)MAXMOD * KO_MAXUNDEF); + if (!g_names) { puts_("ldlsd: out of memory\n"); exit(); } + kpi.netfd = -1; + for (;;) { + name = p; + while (*p && *p != ',') p++; + int last = *p == 0; + *p = 0; + if (*name) + load_module(name); + if (last) break; + p++; } - if (p > 0) { puts_(" rehomed "); putn(p); puts_(" %gs per-CPU reference(s)\n"); } - } - - puts_(" loaded "); putn(span); puts_(" bytes at "); puthex((unsigned long long)mem); - puts_("\n"); - - entry = find_symbol(&k, "init_module"); - if (!entry) { puts_(" no init_module -- nothing to call\n"); exit(); } - - puts_("\n calling init_module\n\n"); - { - int (*init)(void) = (int (*)(void))(void *)entry; - int r = init(); - puts_("\n init_module returned "); putn(r); - puts_(" after "); putn(g_calls); puts_(" stub call(s)\n"); } if (argc > 2 && g_pci_driver) diff --git a/docs/plan/gpu.md b/docs/plan/gpu.md new file mode 100644 index 0000000..481ce29 --- /dev/null +++ b/docs/plan/gpu.md @@ -0,0 +1,125 @@ +# The GPU, through Linux's own driver + +The laptop this is for (ASUS TUF A15 FA507NU) has a Radeon 680M in the CPU and +an RTX 4050 beside it. SchOS draws everything with the CPU on an EFI +framebuffer. Rendering a browser page that way keeps every core busy, and the +machine runs hot doing it. + +Writing a Radeon driver is not on the table: amdgpu is around four million lines +before its register headers. The plan is to **run Linux's amdgpu unmodified**, +with its firmware, in a process under LDLS ([../architecture/ldls.md](../architecture/ldls.md)), +and to give Mesa the `/dev/dri` it expects. Licensing is not an obstacle: +SchOS is GPL-2.0-or-later, and the user has said that Linux code and firmware +blobs may be shipped. That supersedes LDLS's rule 1. + +## Why LDLS and not an in-kernel linuxkpi + +FreeBSD's drm-kmod compiles Linux's DRM source into kernel modules against a +shim (linuxkpi). SchOS has already chosen the other route, and for good reasons +([hybrid.md](hybrid.md)): GPU drivers live outside the kernel, a crashed driver +falls back to the framebuffer instead of taking the machine down, and LDLS +loads unmodified `.ko` files against the running kernel's BTF instead of +needing a source port per Linux release. vgpud already drives a GPU from a +process, and LDLS already runs 8139too and e1000 that way. + +## What it costs: the numbers + +Measured against Linux 6.18.54 (the host's kernel and modules), with +`modprobe --show-depends`, `nm -u` and the export tables: + +| Driver | Modules in the closure | Symbols needed from the core kernel | LDLS provides today | +|---|---|---|---| +| r8169 | 12 (libphy, mdio-bus, realtek PHY, ...) | 330 | 70 | +| amdgpu | 15 (ttm, gpu-sched, drm_buddy, drm_exec, drm_display_helper, ...) | 1,166 | 76 | + +Of amdgpu's 1,166 symbols, **314 are the DRM core** (`drm_*`). NixOS builds +`drm.ko` and `drm_kms_helper.ko` into the kernel, so a module set that has them +as modules is needed; see step 0. A further 45 are dma-fence, dma-buf and +dma-resv. The rest is ordinary kernel: memory, locks, workqueues, timers, PCI, +ACPI, firmware loading, i2c, backlight. + +## What LDLS lacks, and must grow + +LDLS today assumes one thread, no-op locks and a polled interrupt handler +(ldls.md, "What it costs"). amdgpu cannot work under that: + +1. **Threads.** `kthread_run`, workqueues with real worker threads, + `gpu-sched`'s scheduler threads. Map them to pthreads in the driver process. +2. **Locks that lock.** mutex, spinlock, rwsem, `ww_mutex` (TTM and dma-resv + depend on wound/wait semantics), completions and wait queues, on futexes. +3. **Interrupts as interrupts.** A thread blocked in `read()` on `/dev/pci` + calls the handler and any threaded handler, instead of the poll loop. +4. **Timers.** `timer_list`, `hrtimer`, `delayed_work` and `jiffies`, from a + timer thread. +5. **Memory with physical addresses.** TTM allocates system pages for GTT and + maps them for the GPU. Without an IOMMU, DMA address equals physical + address, so the `PCIIOC_DMA` arena has to grow to hundreds of megabytes, + with `struct page` (vmemmap) coverage for all of it. +6. **Firmware.** `request_firmware` reads `/lib/firmware/amdgpu/*.bin`, from + Alpine's `linux-firmware-amdgpu` package. +7. **ACPI.** On an APU the VBIOS comes from the VFCT table, and amdgpu calls + ATIF/ATCS. A small `/dev/acpi` (read a table, evaluate a method through + uACPI) covers both. + +Each of these is general LDLS work, not amdgpu work. Every later driver needs +the same pieces: e1000e, iwlwifi, nouveau. + +## The new kernel piece: `/dev/dri` served by a process + +Mesa and Xorg open `/dev/dri/card0` and `/dev/dri/renderD128`, then use +`ioctl`, `mmap` and `poll`, and pass buffers to each other as dma-buf file +descriptors. The kernel has to forward each of those to the driver process: + +- **open, ioctl, poll, close** go to the server process with the client's + identity attached, the same pattern `/dev/netdev` uses for NICs, extended to + ioctls with their argument buffers. +- **mmap** is the part the kernel must do itself. The server replies with + physical ranges (VRAM through the BAR, or GTT pages from the arena), and the + kernel maps those into the client's address space. +- **dma-buf** becomes a kernel file object naming a buffer the server owns. + `SCM_RIGHTS` passes it between processes (already built, foundations.md + item 1), and X's DRI3 needs exactly that. +- **sysfs**: `/sys/class/drm/card0`, `/sys/dev/char/226:*`, and the + vendor/device files that libdrm and Mesa read to pick a driver. + +## Order + +Each step ends with something you can run. + +0. **One module set.** Build Linux 6.18 from source, once, with `DRM=m`, + `DRM_AMDGPU=m` and BTF on. That gives `drm.ko`, `drm_kms_helper.ko` and the + rest, plus a matching `vmlinux` BTF. `make ldls-modules` does the build, and + packaging it as an apk makes it installable. +1. **LDLS concurrency and memory** (lacks 1–5 above), proven on **r8169**. + The real Linux r8169 brings PHY tuning, the firmware patch, ASPM handling + and every RTL8168 variant. It is the laptop NIC, so this step also fixes the + network properly. Done when the laptop's ethernet runs at gigabit speed + through `ldls0`. +2. **amdgpu probes.** The driver process loads firmware, brings up the IP + blocks and takes the display from the EFI framebuffer. The kernel console + must hand over the screen and get it back if the process dies. Done when + `dmesg` shows amdgpu's "SMU is initialized" and the panel stays lit. +3. **KMS.** `/dev/dri/card0` with ioctl forwarding and the mmap path. Done + when `modetest` (Alpine `libdrm-tests`) sets a mode and draws a frame. +4. **Rendering.** `renderD128`, GEM mmap, dma-buf, syncobj. Done when + `eglinfo` names radeonsi and `glmark2-es2-drm` and `vkcube` (RADV) run. +5. **The desktop.** Xorg's modesetting driver with glamor, then Epiphany with + GL compositing. Done when the fans stay quiet while a page scrolls. + +## The RTX 4050 + +Later, and in two parts: + +- **Powered off, now.** Without a driver the dGPU sits in D0 and draws power. + Putting it in D3cold through its ACPI power resources (`_PR3` / `_OFF`) is + a few dozen lines in the kernel and helps the heat and the battery today. +- **Driven, after amdgpu.** nouveau with GSP firmware (Ada needs it), through + the same LDLS and `/dev/dri` machinery, for PRIME render offload. + +## Not GPU, but also why the laptop runs hot + +Until now, idle CPUs halted at C1 and never reached the deep states listed in +ACPI `_CST`. That kept every core powered at an idle shell prompt. It is fixed +alongside this plan: the scheduler's idle path now enters `_CST`'s C2 (CC6 on +Zen). P-state control (AMD CPPC) is still missing, so the cores run at the +firmware's default clock under load. diff --git a/etc/schinit/ldls-r8169.job b/etc/schinit/ldls-r8169.job new file mode 100644 index 0000000..1746390 --- /dev/null +++ b/etc/schinit/ldls-r8169.job @@ -0,0 +1,17 @@ +# The Realtek RTL8111/8168 ethernet, driven by Linux's own r8169. +# +# The modules (mdio-bus, libphy, mdio_devres, the Realtek PHY driver and r8169 +# itself), the kernel's BTF and the RTL PHY firmware are put in the image by +# `make ldls-r8169`. ldlsd loads them in that order, binds the PHY driver +# through its driver core, and probes the NIC; the interface is ldls0. +# +# On a machine with no RTL8168 ldlsd exits SCHINIT_EXIT_NORESTART and that is +# the end of it. KeepAlive is off while this is new: a driver that crashes on +# real hardware would otherwise be restarted into the same crash every few +# seconds. Its output, every line of it, is in /var/log/ldls-r8169.log. + +Label = os.schos.ldls-r8169 +Program = /bin/sh +Args = sh /lib/modules/ldls-r8169.sh +RunAtLoad = yes +KeepAlive = no diff --git a/include/dev/acpi.h b/include/dev/acpi.h index de49970..5ff868a 100644 --- a/include/dev/acpi.h +++ b/include/dev/acpi.h @@ -50,3 +50,15 @@ struct acpi_psu { }; int acpi_psu_count(void); int acpi_psu_get(int i, struct acpi_psu *out); + +// How an idle CPU sleeps: kernel/acpi/aml.c reads it from _CST, and the +// scheduler's idle path in kernel/proc/proc.c acts on it. +#define ACPI_IDLE_HLT 0 +#define ACPI_IDLE_IO 1 // read a port +#define ACPI_IDLE_MWAIT 2 // MWAIT with a hint +struct acpi_idle { + int kind; + uint64 addr; // the port, or the MWAIT hint + uint latency; // microseconds +}; +extern struct acpi_idle acpi_idle; diff --git a/kernel/acpi/aml.c b/kernel/acpi/aml.c index 99a478b..dfc4cc0 100644 --- a/kernel/acpi/aml.c +++ b/kernel/acpi/aml.c @@ -41,6 +41,7 @@ static int ready; static int s5_typ_a = -1, s5_typ_b = -1; static uacpi_interrupt_ret power_button(uacpi_handle ctx); +static void cstate_find(void); void acpi_aml_init(void) @@ -98,6 +99,7 @@ acpi_aml_init(void) } ready = 1; kinfo("acpi: namespace loaded\n"); + cstate_find(); } // --- power ------------------------------------------------------------------- @@ -526,3 +528,111 @@ acpi_psu_get(int i, struct acpi_psu *b) } return 0; } + +// --- idle states -------------------------------------------------------------- +// +// An idle CPU used to run hlt and nothing else. hlt is C1: the core stops +// executing and stays powered. The states below it -- CC6 on Zen, C3 and +// deeper on Intel -- switch the core off, and the firmware lists how to reach +// them in each processor's _CST. Without them a laptop sitting at a shell +// keeps every core warm and the fans going. +// +// _CST is a count followed by one package per state: a register (a Generic +// Register descriptor in a buffer), a type (1 = C1, 2 = C2, 3 = C3), the +// worst-case wake-up latency in microseconds, and the power. A register in +// system I/O space is entered by reading it; one in functional fixed hardware +// is MWAIT, with the hint in the address. +// +// Only types 1 and 2 are taken. C3 asks the OS to stop bus-master activity +// or flush caches first (ACPI 8.1.4), which this does not do. + +struct acpi_idle acpi_idle; + +static int cst_done; + +static void +cst_parse(uacpi_namespace_node *node) +{ + uacpi_object *o = 0; + uacpi_object_array p, e; + uacpi_data_view d; + + if (uacpi_eval_package(node, "_CST", 0, &o) != UACPI_STATUS_OK) + return; + if (uacpi_object_get_package(o, &p) == UACPI_STATUS_OK) { + for (uacpi_size i = 1; i < p.count; i++) { + if (uacpi_object_get_package(p.objects[i], &e) != UACPI_STATUS_OK || + e.count < 4) + continue; + uint32 type = pkg_int(&e, 1), lat = pkg_int(&e, 2); + if (type != 2 || lat > 1000) + continue; + if (uacpi_object_get_buffer(e.objects[0], &d) != UACPI_STATUS_OK || + d.length < 15 || d.const_bytes[0] != 0x82) + continue; + uint8 space = d.const_bytes[3]; + uint64 addr = 0; + for (int k = 0; k < 8; k++) + addr |= (uint64)d.const_bytes[7 + k] << (8 * k); + if (space == UACPI_ADDRESS_SPACE_SYSTEM_IO && addr && addr < 0x10000) { + acpi_idle.kind = ACPI_IDLE_IO; + } else if (space == UACPI_ADDRESS_SPACE_FFIXEDHW && (d.const_bytes[4] == 1)) { + // Intel's FFH encoding: vendor 1, class 2 (native C-state), and + // the MWAIT hint in the low byte of the address. + acpi_idle.kind = ACPI_IDLE_MWAIT; + } else { + continue; + } + acpi_idle.addr = addr; + acpi_idle.latency = lat; + break; + } + } + uacpi_object_unref(o); +} + +static uacpi_iteration_decision +found_cpu(void *user, uacpi_namespace_node *node, uacpi_u32 depth) +{ + (void)user; (void)depth; + if (cst_done) + return UACPI_ITERATION_DECISION_BREAK; + cst_parse(node); + cst_done = 1; // every processor's _CST is the same in practice + return UACPI_ITERATION_DECISION_BREAK; +} + +static void +cstate_find(void) +{ + uint32 a, b, c, dd; + + // The local APIC timer is what wakes an idle CPU for the scheduler tick. + // Below C1 it may stop, unless the CPU says it keeps running (ARAT, + // CPUID 6 EAX bit 2); without that an idle CPU would sleep for good. + cpuid_leaf(6, &a, &b, &c, &dd); + if (!(a & 4)) { + kinfo("acpi: no always-running APIC timer; idle stays at C1\n"); + return; + } + + uacpi_namespace_for_each_child(uacpi_namespace_root(), found_cpu, 0, + UACPI_OBJECT_PROCESSOR_BIT, + UACPI_MAX_DEPTH_ANY, 0); + if (!cst_done) + uacpi_find_devices("ACPI0007", found_cpu, 0); + + if (acpi_idle.kind == ACPI_IDLE_MWAIT) { + cpuid_leaf(1, &a, &b, &c, &dd); + if (!(c & (1u << 3))) // no MONITOR/MWAIT after all + acpi_idle.kind = ACPI_IDLE_HLT; + } + if (acpi_idle.kind == ACPI_IDLE_IO) + kinfo("acpi: idle CPUs enter C2 through port %x (wake-up %d us)\n", + (uint)acpi_idle.addr, acpi_idle.latency); + else if (acpi_idle.kind == ACPI_IDLE_MWAIT) + kinfo("acpi: idle CPUs enter C2 by MWAIT hint %x (wake-up %d us)\n", + (uint)acpi_idle.addr, acpi_idle.latency); + else + kinfo("acpi: no usable C2 in _CST; idle stays at C1\n"); +} diff --git a/kernel/dev/pci.c b/kernel/dev/pci.c index e938763..3b8f18a 100644 --- a/kernel/dev/pci.c +++ b/kernel/dev/pci.c @@ -678,6 +678,41 @@ pci_dump(struct sbuf *out, const char *arg) } } +// ASPM off on Realtek NICs, as Linux's r8169 does at probe through +// pci_disable_link_state(): "Disable ASPM L1 as that cause random device stop +// working problems as well as full system hangs". Here because the driver now +// runs in a process, and /dev/pci gives a process no config writes. Both ends +// of the link: the NIC's PCIe Link Control and the bridge above it, low two +// bits, L0s and L1. +static void +link_aspm_off(struct pci_dev *d) +{ + uint8 cap = pci_find_cap(d, 0x10); + if (cap) { + uint32 v = pci_read32(d->bus, d->slot, d->func, cap + 0x10); + if (v & 3) + pci_write32(d->bus, d->slot, d->func, cap + 0x10, v & ~3u); + } +} + +static void +realtek_aspm_off(void) +{ + for (int i = 0; i < ndevs; i++) { + struct pci_dev *d = &devs[i]; + if (d->vendor != 0x10EC || (d->device != 0x8168 && d->device != 0x8161 && + d->device != 0x8125 && d->device != 0x8136)) + continue; + link_aspm_off(d); + for (int j = 0; j < ndevs; j++) + if ((devs[j].header_type & 0x7F) == 1 && + pci_read8(devs[j].bus, devs[j].slot, devs[j].func, PCI_SECONDARY_BUS) == d->bus) + link_aspm_off(&devs[j]); + kinfo("pci: %02x:%02x.%d Realtek NIC, ASPM off on its link\n", + d->bus, d->slot, d->func); + } +} + void pci_init(void) { @@ -693,6 +728,7 @@ pci_init(void) scan_bus(0); kinfo("pci: %d device%s\n", ndevs, ndevs == 1 ? "" : "s"); + realtek_aspm_off(); // Before any driver binds: the line each device's config space names is // the 8259's number, which is not the IOAPIC pin it is wired to. ACPI's diff --git a/kernel/drivers/net/r8169/r8169.c b/kernel/drivers/net/r8169/r8169.c index 727d4f6..7fae24e 100644 --- a/kernel/drivers/net/r8169/r8169.c +++ b/kernel/drivers/net/r8169/r8169.c @@ -49,6 +49,14 @@ #define R_RCR 0x44 // Rx config #define R_9346CR 0x50 // unlocks the config registers #define R_CONFIG1 0x52 +#define R_CONFIG2 0x53 +#define R_CONFIG3 0x54 +#define R_CONFIG5 0x56 +#define R_EEE_LED 0x1B +#define R_EPHYAR 0x80 // the PCIe PHY's registers +#define R_DLLPR 0xD0 +#define R_MISC1 0xF2 +#define R_INTRMIT 0xE2 // interrupt mitigation #define R_PHYAR 0x60 // the MII window #define R_PHYSTATUS 0x6C #define R_PMCH 0x6F // PLL power @@ -107,6 +115,11 @@ // MXDMA short. It would have worked and been slower, which is the worst kind // of wrong to leave in a register constant. #define RCR_UNLIMITED 0x0000E700 +// What Linux's rtl_init_rxcfg() writes on the 8168G and later instead: +// 128-byte early interrupt, multiple descriptors per DMA, unlimited burst, +// early receive off. +#define RCR_GEN_G (0x8000 | 0x4000 | 0x0700 | 0x0800) +#define TCR_AUTO_FIFO 0x00000080 // 8168E-VL and later: the chip sizes its FIFOs #define TCR_IFG_STD 0x03000000 #define TCR_MXDMA 0x00000700 @@ -282,7 +295,21 @@ mac_ocp_write(struct r8169 *r, uint16 reg, uint16 val) wr32(r, R_MAC_OCP, OCP_FLAG | ((uint32)(reg >> 1) << 16) | val); } -// The extended (ERI) registers, all four bytes at once. +// The extended (ERI) registers. `mask` picks the bytes, bit 0 the lowest: +// several of these registers share a word with ones that must not change. +static void +eri_write_mask(struct r8169 *r, int addr, int mask, uint32 val) +{ + wr32(r, R_ERIDR, val); + wr32(r, R_ERIAR, OCP_FLAG | ((uint32)mask << 12) | addr); + for (int i = 0; i < 100; i++) { + microdelay(10); + if (!(rd32(r, R_ERIAR) & OCP_FLAG)) + return; + } +} + +// All four bytes at once. static uint32 eri_read(struct r8169 *r, int addr) { @@ -315,7 +342,7 @@ r8169_gen_g(uint32 xid) switch (xid) { case 0x4C0: case 0x4C1: case 0x5C8: case 0x509: case 0x500: case 0x501: case 0x502: - case 0x540: case 0x541: + case 0x540: case 0x541: case 0x6C0: return 1; } return 0; @@ -347,6 +374,202 @@ r8169_takeover_g(struct r8169 *r) wait_ll_ready(r); } +// --- the 8168H as Linux sets it up ------------------------------------------ +// +// From Linux 6.18 drivers/net/ethernet/realtek/r8169_main.c +// (rtl_hw_start_8168h_1, rtl_hw_aspm_clkreq_enable) and r8169_phy_config.c +// (rtl8168h_2_hw_phy_config), GPL-2.0, carried over register for register. +// Without them the chip runs on its power-on FIFO sizes and PHY tuning and +// with ASPM as the firmware left it, and moves frames, slowly and with drops. +// The one part left out is the PHY firmware patch (rtl_nic/rtl8168h-2.fw), +// which needs the root filesystem this runs before. + +static void +ephy_write(struct r8169 *r, int reg, uint16 val) +{ + wr32(r, R_EPHYAR, 0x80000000u | ((uint32)(reg & 0x1F) << 16) | val); + for (int i = 0; i < 100; i++) { + microdelay(10); + if (!(rd32(r, R_EPHYAR) & 0x80000000u)) + break; + } + microdelay(10); +} + +static uint16 +ephy_read(struct r8169 *r, int reg) +{ + wr32(r, R_EPHYAR, (uint32)(reg & 0x1F) << 16); + for (int i = 0; i < 100; i++) { + microdelay(10); + uint32 v = rd32(r, R_EPHYAR); + if (v & 0x80000000u) + return (uint16)v; + } + return 0xFFFF; +} + +static void +mac_ocp_modify(struct r8169 *r, uint16 reg, uint16 clear, uint16 set) +{ + mac_ocp_write(r, reg, (mac_ocp_read(r, reg) & ~clear) | set); +} + +static int +phy_read_paged(struct r8169 *r, int page, int reg) +{ + mii_write(r, 0x1F, (uint16)page); + int v = mii_read(r, reg); + mii_write(r, 0x1F, 0); + return v; +} + +static void +phy_write_paged(struct r8169 *r, int page, int reg, uint16 val) +{ + mii_write(r, 0x1F, (uint16)page); + mii_write(r, reg, val); + mii_write(r, 0x1F, 0); +} + +static void +phy_modify_paged(struct r8169 *r, int page, int reg, uint16 clear, uint16 set) +{ + mii_write(r, 0x1F, (uint16)page); + int v = mii_read(r, reg); + if (v >= 0) + mii_write(r, reg, (uint16)((v & ~clear) | set)); + mii_write(r, 0x1F, 0); +} + +static void +phy_param(struct r8169 *r, uint16 parm, uint16 clear, uint16 set) +{ + mii_write(r, 0x1F, 0x0A43); + mii_write(r, 0x13, parm); + int v = mii_read(r, 0x14); + if (v >= 0) + mii_write(r, 0x14, (uint16)((v & ~clear) | set)); + mii_write(r, 0x1F, 0); +} + +// PCIe Link Control, low two bits: L0s and L1. Linux clears L1 on this chip +// and on the bridge above it unless the vendor has marked it safe, for "random +// device stop working problems as well as full system hangs". Firmware +// leaves it on. A link that drops into L1 between frames costs a wake-up per +// burst and on some boards stalls outright, which is a download that crawls. +static void +aspm_off(struct pci_dev *d) +{ + uint8 cap = pci_find_cap(d, 0x10); + if (cap) { + uint32 v = pci_read32(d->bus, d->slot, d->func, cap + 0x10); + pci_write32(d->bus, d->slot, d->func, cap + 0x10, v & ~3u); + } +} + +static void +r8169_aspm_off(struct r8169 *r) +{ + struct pci_dev *d = r->pdev; + + // The vendor's flag that ASPM is safe on this board (rtl_aspm_is_safe). + if (mac_ocp_read(r, 0xC0B2) & 0xF) + return; + aspm_off(d); + for (int i = 0; i < pci_count(); i++) { + struct pci_dev *b = pci_get(i); + if (b && (b->header_type & 0x7F) == 1 && + pci_read8(b->bus, b->slot, b->func, 0x19) == d->bus) + aspm_off(b); + } + // And in the chip: no L1.2 trigger, no clock request, ASPM off. + mac_ocp_modify(r, 0xE092, 0x00FF, 0); + wr8(r, R_CONFIG2, rd8(r, R_CONFIG2) & ~0x80); + wr8(r, R_CONFIG5, rd8(r, R_CONFIG5) & ~0x01); +} + +// rtl_hw_start_8168h_1. Config registers must be unlocked. +static void +r8169_start_8168h(struct r8169 *r) +{ + static const struct { uint8 reg; uint16 clear, set; } ephy[] = { + { 0x1E, 0x0800, 0x0001 }, + { 0x1D, 0x0000, 0x0800 }, + { 0x05, 0xFFFF, 0x2089 }, + { 0x06, 0xFFFF, 0x5881 }, + { 0x04, 0xFFFF, 0x854A }, + { 0x01, 0xFFFF, 0x068B }, + }; + for (unsigned i = 0; i < sizeof ephy / sizeof ephy[0]; i++) + ephy_write(r, ephy[i].reg, + (ephy_read(r, ephy[i].reg) & ~ephy[i].clear) | ephy[i].set); + + eri_write(r, 0xC8, (0x08u << 16) | 0x02); // FIFO sizes + eri_write(r, 0xE8, (0x10u << 16) | 0x06); + eri_write_mask(r, 0xCC, 0x1, 0x38); // pause thresholds + eri_write_mask(r, 0xD0, 0x1, 0x48); + + eri_write(r, 0xDC, eri_read(r, 0xDC) & ~1u); // reset the packet filter + eri_write(r, 0xDC, eri_read(r, 0xDC) | 1u); + eri_write(r, 0xDC, eri_read(r, 0xDC) | 0x1C); + eri_write_mask(r, 0x5F0, 0x3, 0x4F87); + wr32(r, R_MISC, rd32(r, R_MISC) & ~MISC_RXDV_GATE); + eri_write_mask(r, 0xC0, 0x3, 0); + eri_write_mask(r, 0xB8, 0x3, 0); + + wr8(r, R_EEE_LED, rd8(r, R_EEE_LED) & ~0x07); // rtl8168_config_eee_mac + eri_write(r, 0x1B0, eri_read(r, 0x1B0) | 0x3); + + wr8(r, R_DLLPR, rd8(r, R_DLLPR) & ~0x40); // PFM off + wr8(r, R_MISC1, rd8(r, R_MISC1) & ~0x40); // PFM in D3cold off + wr8(r, R_DLLPR, rd8(r, R_DLLPR) & ~0x80); // 10M power saving off + eri_write(r, 0x1B0, eri_read(r, 0x1B0) & ~(1u << 12)); + wr8(r, R_CONFIG3, rd8(r, R_CONFIG3) & ~0x02); // Rdy_to_L23 off + + int saw = phy_read_paged(r, 0x0C42, 0x13); + if (saw > 0 && (saw & 0x3FFF) > 0) + mac_ocp_modify(r, 0xD412, 0x0FFF, (16000000 / (saw & 0x3FFF)) & 0x0FFF); + + mac_ocp_modify(r, 0xE056, 0x00F0, 0x0000); + mac_ocp_modify(r, 0xE052, 0x6000, 0x8008); + mac_ocp_modify(r, 0xE0D6, 0x01FF, 0x017F); + mac_ocp_modify(r, 0xD420, 0x0FFF, 0x047F); + mac_ocp_write(r, 0xE63E, 0x0001); + mac_ocp_write(r, 0xE63E, 0x0000); + mac_ocp_write(r, 0xC094, 0x0000); + mac_ocp_write(r, 0xC09E, 0x0000); +} + +// rtl8168h_2_hw_phy_config, less the firmware patch. +static void +r8169_phy_8168h(struct r8169 *r) +{ + phy_param(r, 0x808A, 0x003F, 0x000A); // CHIN EST parameter + phy_param(r, 0x0811, 0x0000, 0x0800); // R-tune and PGA-retune on + phy_modify_paged(r, 0x0A42, 0x16, 0x0000, 0x0002); + phy_modify_paged(r, 0x0A44, 0x11, 0, 1u << 11); // gphy 10M + + // ADC bias offset, from the MAC's OCP space. + mac_ocp_write(r, 0xDD02, 0x807D); + uint16 d1 = mac_ocp_read(r, 0xDD02), d2 = mac_ocp_read(r, 0xDD00); + uint16 ioffset = ((d2 >> 1) & 0x7FF8) | (d2 & 0x0007) | ((d1 & 0x80) ? 0x8000 : 0); + if (ioffset != 0xFFFF) + phy_write_paged(r, 0x0BCF, 0x16, ioffset); + + // TX low-pass corner. + int data = phy_read_paged(r, 0x0BCD, 0x16); + if (data >= 0) { + uint16 rlen = (data & 0xF) > 3 ? (data & 0xF) - 3 : 0; + phy_write_paged(r, 0x0BCD, 0x17, rlen | rlen << 4 | rlen << 8 | rlen << 12); + } + + phy_modify_paged(r, 0x0A44, 0x11, 1u << 7, 0); // PHY PFM off + phy_modify_paged(r, 0x0A43, 0x10, 1u << 0, 0); // 10M PLL off: off + phy_modify_paged(r, 0x0A43, 0x10, 1u << 2, 0); // ALDPS off + phy_modify_paged(r, 0x0A43, 0x11, 0, 1u << 4); // EEE +} + // The PHY up and negotiating: page 0, out of power-down, autoneg on and // restarted. Power-down is what firmware leaves set after a shutdown, and // the old code ORed autoneg into BMCR and kept it. @@ -620,6 +843,10 @@ r8169_probe(struct pci_dev *d) if (xid == 0x540 || xid == 0x541) eri_write(r, 0x1A8, eri_read(r, 0x1A8) | 0xFC000000); wr32(r, R_MISC, rd32(r, R_MISC) & ~MISC_RXDV_GATE); + if (xid == 0x541 || xid == 0x6C0) { + r8169_aspm_off(r); + r8169_start_8168h(r); + } } // CPCR is deliberately left alone. @@ -635,13 +862,16 @@ r8169_probe(struct pci_dev *d) // unlike the 8139 it has no other programming model to be in. wr16(r, R_RMS, BUFSZ); - wr8(r, R_MTPS, 0x3B); + // EarlySize on the 8168E-VL and later, TxPacketMax before, as in Linux. + wr8(r, R_MTPS, r->gen_g ? 0x27 : 0x3B); + wr16(r, R_INTRMIT, 0); // no interrupt coalescing wr64(r, R_RDSAR, r->rx_phys); wr64(r, R_TNPDS, r->tx_phys); - wr32(r, R_TCR, TCR_IFG_STD | TCR_MXDMA); - wr32(r, R_RCR, RCR_APM | RCR_AM | RCR_AB | RCR_UNLIMITED); + wr32(r, R_TCR, TCR_IFG_STD | TCR_MXDMA | (r->gen_g ? TCR_AUTO_FIFO : 0)); + wr32(r, R_RCR, RCR_APM | RCR_AM | RCR_AB | + (r->gen_g ? RCR_GEN_G : RCR_UNLIMITED)); // Multicast: everything. The stack filters, and a hash filter that is wrong // drops IPv6 neighbour discovery in a way that looks like a routing problem. @@ -651,6 +881,11 @@ r8169_probe(struct pci_dev *d) wr8(r, R_CR, CR_RE | CR_TE); wr8(r, R_9346CR, C9346_LOCK); + { + uint32 xid = (rd32(r, R_TCR) >> 20) & 0xFCF; + if (xid == 0x541 || xid == 0x6C0) + r8169_phy_8168h(r); + } r8169_phy_up(r); wr16(r, R_ISR, 0xFFFF); // clear anything left over diff --git a/kernel/drivers/registry.c b/kernel/drivers/registry.c index 191899c..15d9be5 100644 --- a/kernel/drivers/registry.c +++ b/kernel/drivers/registry.c @@ -33,7 +33,6 @@ extern const struct pci_driver xhci_driver; extern const struct pci_driver virtio_gpu_driver; extern const struct pci_driver e1000_driver; extern const struct pci_driver ac97_driver; -extern const struct pci_driver r8169_driver; extern const struct pci_driver rtl8139_driver; extern const struct pci_driver ahci_driver; extern const struct pci_driver nvme_driver; @@ -47,7 +46,9 @@ static const struct pci_driver *drivers[] = { &virtio_gpu_driver, &vga_pci_driver, &e1000_driver, - &r8169_driver, + // &r8169_driver -- the RTL8168 is driven by Linux's own r8169 through + // LDLS (etc/schinit/ldls-r8169.job), which a kernel driver here would + // claim first. The native driver stays in the tree as the fallback. &rtl8139_driver, &xhci_driver, &ahci_driver, diff --git a/kernel/proc/proc.c b/kernel/proc/proc.c index 9c15972..83fa5af 100644 --- a/kernel/proc/proc.c +++ b/kernel/proc/proc.c @@ -16,6 +16,7 @@ #include "uapi/errno.h" #include "fs/blkdev.h" #include "core/uaccess.h" +#include "dev/acpi.h" // Verify myproc()->fs_base offset matches the hardcoded 0x688 in trapasm.S. _Static_assert(__builtin_offsetof(struct proc, fs_base) == 0x18, @@ -1983,6 +1984,33 @@ sched_slice_expired(void) return (p->slice_used + ns) >= p->slice_ns; } +// Wait for an interrupt as deeply as the firmware allows. See acpi_idle in +// kernel/acpi/aml.c. Interrupts are off on the way down so that one arriving +// between the decision and the sleep is not lost: a pending interrupt still +// ends the sleep (MWAIT's ECX bit 0 says so explicitly; an I/O-entered C2 +// wakes on it by design), and is taken at the sti after. +static void +cpu_idle(void) +{ + static volatile int monitor_line[16]; + + switch (acpi_idle.kind) { + case ACPI_IDLE_IO: + cli(); + (void)inb((ushort)acpi_idle.addr); + sti(); + break; + case ACPI_IDLE_MWAIT: + cli(); + asm volatile("monitor" : : "a"(monitor_line), "c"(0), "d"(0)); + asm volatile("mwait" : : "a"((uint32)acpi_idle.addr), "c"(1)); + sti(); + break; + default: + hlt(); + } +} + void scheduler(void) { @@ -1999,7 +2027,7 @@ scheduler(void) // hlt() would stop every other CPU from scheduling until an interrupt // happened to arrive here. release(&ptable.lock); - hlt(); + cpu_idle(); continue; } diff --git a/ldls/ko.c b/ldls/ko.c index 46783bb..8874496 100644 --- a/ldls/ko.c +++ b/ldls/ko.c @@ -708,6 +708,8 @@ gs_form(const unsigned char *p, int want_pre) default: if (op >= 0x0f90 && op <= 0x0f9f) /* setcc */ { imm = 0; break; } + if (op >= 0x0f40 && op <= 0x0f4f) /* cmovcc -- r8169 reads numa_node so */ + { imm = 0; break; } return 0; } return pre + 4 + imm; diff --git a/ldls/ko.h b/ldls/ko.h index 73abd79..d39dd5e 100644 --- a/ldls/ko.h +++ b/ldls/ko.h @@ -37,7 +37,7 @@ // Undefined symbols the report will hold. Measured over the same corpus the // median is 25 and the worst seen is 245, so this is room to spare -- and // `undef_lost` says so rather than the list quietly ending. -#define KO_MAXUNDEF 512 +#define KO_MAXUNDEF 4096 // amdgpu alone imports well over a thousand // --- ELF64, the relocatable subset ------------------------------------------ // diff --git a/ldls/kpi/base.c b/ldls/kpi/base.c new file mode 100644 index 0000000..2813745 --- /dev/null +++ b/ldls/kpi/base.c @@ -0,0 +1,963 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// +// The rest of the kernel a modern driver stands on: the driver core, device- +// managed resources, printf, time, bitmaps, firmware, and the data symbols. +// See ldls/kpi/kpi.h. +// +// r8169 is the reason for most of it. Unlike 8139too and e1000 it does not +// talk to its PHY itself: it registers an MDIO bus with libphy, and libphy +// finds the PHY on it and binds a PHY driver (realtek.ko) through Linux's +// driver core -- bus_type.match, then device_driver.probe. r8169 then checks +// that a dedicated PHY driver was bound and refuses the device if not. So the +// driver core here is not decoration; it is what makes the device usable. +// +// Several functions below exist because a stub's zero is a lie in their +// case, and each says which lie it was: +// __list_add_valid_or_report 0 = "corrupt list", and with CONFIG_DEBUG_LIST +// every list_add asks, so nothing is ever added; +// try_module_get 0 = "module going away", phylib refuses the PHY; +// dev_err_probe returns its error, or failure looks like success; +// eth_platform_get_mac_address 0 = "here is your MAC", all zeroes; +// firmware_request_nowarn 0 = "loaded", with a NULL firmware; +// __bitmap_* phylib computes link modes with them; +// __const_udelay a delay of nothing turns every poll-with- +// timeout into an immediate timeout. + +#include "core/types.h" +#include "uapi/user.h" +#include "uapi/fcntl.h" +#include "uapi/posix.h" +#include "kpi.h" + +typedef __builtin_va_list va_list; +#define va_start(a, l) __builtin_va_start(a, l) +#define va_end(a) __builtin_va_end(a) +#define va_arg(a, t) __builtin_va_arg(a, t) + +extern void *kpi_kmalloc(unsigned long n, unsigned int flags); +extern int kpi_pci_enable_device(void *pdev); +extern void *kpi_pci_iomap(void *pdev, int bar, unsigned long max); +extern int kpi_pci_read_config_word(void *pdev, int where, unsigned short *v); +extern void *kpi_alloc_etherdev_mqs(int priv, unsigned int tx, unsigned int rx); +extern void *kpi_dma_alloc_attrs(void *dev, unsigned long size, + unsigned long long *handle, unsigned int gfp, + unsigned long attrs); +extern void *kpi_alloc_pages(unsigned int gfp, unsigned int order); + +static void *zalloc(unsigned long n) { return kpi_kmalloc(n, 0); } + +static unsigned long rdp(const void *base, unsigned int off) +{ unsigned long v; __builtin_memcpy(&v, (const char *)base + off, 8); return v; } +static void wrp(void *base, unsigned int off, unsigned long v) +{ __builtin_memcpy((char *)base + off, &v, 8); } + +static int slen(const char *s) { int n = 0; while (s && s[n]) n++; return n; } + +// --- printf --------------------------------------------------------------------- +// +// Linux's format language: C's, plus %p followed by letters that say what the +// pointer is. Of those, only the MAC address (%pM) is worth expanding here; +// the rest print as a pointer and consume their letters, as Linux does, so +// that the text after them is not mistaken for part of the format. + +struct out { char *buf; unsigned long cap, n; }; + +static void put(struct out *o, char c) +{ + if (o->n + 1 < o->cap) + o->buf[o->n] = c; + o->n++; +} + +static void +putnum(struct out *o, unsigned long long v, int base, int neg, int width, + int zero, int left, int upper, int prec) +{ + char tmp[24]; + const char *dig = upper ? "0123456789ABCDEF" : "0123456789abcdef"; + int n = 0, len, pad; + + if (v == 0 && prec != 0) + tmp[n++] = '0'; + while (v) { tmp[n++] = dig[v % (unsigned)base]; v /= (unsigned)base; } + while (n < prec && n < (int)sizeof(tmp)) tmp[n++] = '0'; + len = n + (neg ? 1 : 0); + pad = width > len ? width - len : 0; + if (!left && !zero) while (pad-- > 0) put(o, ' '); + if (neg) put(o, '-'); + if (!left && zero) while (pad-- > 0) put(o, '0'); + while (n > 0) put(o, tmp[--n]); + if (left) while (pad-- > 0) put(o, ' '); +} + +int +kpi_vsnprintf(char *buf, unsigned long cap, const char *f, va_list ap) +{ + struct out o = { buf, cap, 0 }; + + if (!f) f = ""; + while (*f) { + if (*f != '%') { put(&o, *f++); continue; } + f++; + int left = 0, zero = 0, width = 0, prec = -1, lng = 0; + for (;; f++) { + if (*f == '-') left = 1; + else if (*f == '0') zero = 1; + else if (*f == '+' || *f == ' ' || *f == '#') ; + else break; + } + if (*f == '*') { width = va_arg(ap, int); f++; } + else while (*f >= '0' && *f <= '9') width = width * 10 + (*f++ - '0'); + if (*f == '.') { + f++; prec = 0; + if (*f == '*') { prec = va_arg(ap, int); f++; } + else while (*f >= '0' && *f <= '9') prec = prec * 10 + (*f++ - '0'); + } + for (;; f++) { + if (*f == 'l') lng++; + else if (*f == 'z' || *f == 't' || *f == 'j') lng = 2; + else if (*f == 'h') ; + else break; + } + char c = *f ? *f++ : 0; + switch (c) { + case 'd': case 'i': { + long long v = lng ? va_arg(ap, long long) : va_arg(ap, int); + int neg = v < 0; + putnum(&o, neg ? (unsigned long long)-v : (unsigned long long)v, 10, + neg, width, zero, left, 0, prec); + break; + } + case 'u': case 'x': case 'X': case 'o': { + unsigned long long v = lng ? va_arg(ap, unsigned long long) + : va_arg(ap, unsigned int); + putnum(&o, v, c == 'u' ? 10 : c == 'o' ? 8 : 16, 0, width, zero, left, + c == 'X', prec); + break; + } + case 'c': + put(&o, (char)va_arg(ap, int)); + break; + case 's': { + const char *s = va_arg(ap, const char *); + int n, pad; + if (!s) s = "(null)"; + n = slen(s); + if (prec >= 0 && n > prec) n = prec; + pad = width > n ? width - n : 0; + if (!left) while (pad-- > 0) put(&o, ' '); + for (int i = 0; i < n; i++) put(&o, s[i]); + if (left) while (pad-- > 0) put(&o, ' '); + break; + } + case 'p': { + const unsigned char *p = va_arg(ap, const unsigned char *); + if ((*f == 'M' || *f == 'm') && p) { + for (int i = 0; i < 6; i++) { + if (i && *f == 'M') put(&o, ':'); + putnum(&o, p[i], 16, 0, 2, 1, 0, 0, -1); + } + } else if (*f == 'e') { + put(&o, 'E'); + long e = (long)p; + putnum(&o, (unsigned long long)(e < 0 ? -e : e), 10, e < 0, 0, 0, 0, 0, -1); + } else { + put(&o, '0'); put(&o, 'x'); + putnum(&o, (unsigned long)p, 16, 0, 0, 0, 0, 0, -1); + } + while ((*f >= 'a' && *f <= 'z') || (*f >= 'A' && *f <= 'Z') || + (*f >= '0' && *f <= '9')) + f++; + break; + } + case '%': + put(&o, '%'); + break; + default: + break; + } + } + if (cap) + buf[o.n < cap ? o.n : cap - 1] = 0; + return (int)o.n; +} + +int +kpi_snprintf(char *buf, unsigned long cap, const char *f, ...) +{ + va_list ap; + va_start(ap, f); + int n = kpi_vsnprintf(buf, cap, f, ap); + va_end(ap); + return n; +} + +int +kpi_sprintf(char *buf, const char *f, ...) +{ + va_list ap; + va_start(ap, f); + int n = kpi_vsnprintf(buf, 1UL << 20, f, ap); + va_end(ap); + return n; +} + +char * +kpi_kvasprintf(const char *f, va_list ap) +{ + char tmp[512]; + va_list ap2; + __builtin_va_copy(ap2, ap); + int n = kpi_vsnprintf(tmp, sizeof(tmp), f, ap2); + va_end(ap2); + char *s = zalloc((unsigned long)n + 1); + if (s) + kpi_vsnprintf(s, (unsigned long)n + 1, f, ap); + return s; +} + +char * +kpi_kasprintf(unsigned int gfp, const char *f, ...) +{ + (void)gfp; + va_list ap; + va_start(ap, f); + char *s = kpi_kvasprintf(f, ap); + va_end(ap); + return s; +} + +// A kernel message: the level byte off the front, the driver's text after a +// prefix that says whose it is. +static void +emit(const char *who, const char *f, va_list ap) +{ + char line[512]; + + if (f && (unsigned char)f[0] == 0x01 && f[1]) + f += 2; + kpi_vsnprintf(line, sizeof(line), f, ap); + kpi_log(" "); + if (who && *who) { kpi_log(who); kpi_log(": "); } + kpi_log(line); + int n = slen(line); + if (n == 0 || line[n - 1] != '\n') + kpi_log("\n"); +} + +static const char * +dev_name_of(void *dev) +{ + if (!dev || !kpi_l.dev_kobj_ok) + return ""; + const char *n = (const char *)rdp(dev, kpi_l.dev_kobj + kpi_l.kobj_name); + return n ? n : ""; +} + +int kpi_vprintk(const char *f, va_list ap) { emit(0, f, ap); return 0; } + +int +kpi_printk_fmt(const char *f, ...) +{ + va_list ap; va_start(ap, f); emit(0, f, ap); va_end(ap); return 0; +} + +void +kpi_dev_printk(void *dev, const char *f, ...) +{ + va_list ap; va_start(ap, f); emit(dev_name_of(dev), f, ap); va_end(ap); +} + +// netdev_* take the net_device, whose name is not kept anywhere this can +// read; "ldls0" is what the interface is called on this side anyway. +void +kpi_netdev_printk_fmt(void *nd, const char *f, ...) +{ + (void)nd; + va_list ap; va_start(ap, f); emit("ldls0", f, ap); va_end(ap); +} + +void +kpi_netdev_printk_lvl(const char *lvl, void *nd, const char *f, ...) +{ + (void)lvl; (void)nd; + va_list ap; va_start(ap, f); emit("ldls0", f, ap); va_end(ap); +} + +int +kpi_dev_err_probe(void *dev, int err, const char *f, ...) +{ + va_list ap; va_start(ap, f); emit(dev_name_of(dev), f, ap); va_end(ap); + return err; +} + +void +kpi_warn_printk(const char *f, ...) +{ + kpi_log(" WARNING "); + va_list ap; va_start(ap, f); emit(0, f, ap); va_end(ap); +} + +// --- strings ---------------------------------------------------------------------- + +int +kpi_strcmp(const char *a, const char *b) +{ + while (*a && *a == *b) { a++; b++; } + return (unsigned char)*a - (unsigned char)*b; +} + +char * +kpi_strcpy(char *d, const char *s) +{ + char *r = d; + while ((*d++ = *s++)) ; + return r; +} + +const char * +kpi_kstrdup_const(const char *s, unsigned int gfp) +{ + (void)gfp; + if (!s) return 0; + char *d = zalloc((unsigned long)slen(s) + 1); + if (d) kpi_strcpy(d, s); + return d; +} + +int +kpi_kstrtoull(const char *s, unsigned int base, unsigned long long *res) +{ + unsigned long long v = 0; + int any = 0; + if (base == 0) { + base = 10; + if (s[0] == '0' && (s[1] == 'x' || s[1] == 'X')) { base = 16; s += 2; } + else if (s[0] == '0') base = 8; + } + for (; *s && *s != '\n'; s++) { + int d = (*s >= '0' && *s <= '9') ? *s - '0' : + (*s >= 'a' && *s <= 'f') ? *s - 'a' + 10 : + (*s >= 'A' && *s <= 'F') ? *s - 'A' + 10 : 99; + if (d >= (int)base) return -22; + v = v * base + (unsigned)d; + any = 1; + } + if (!any) return -22; + *res = v; + return 0; +} + +// --- time -------------------------------------------------------------------------- +// +// HZ is a compile-time constant in the module. It is 1000 on the kernels this +// has been built against, and jiffies has to count at the module's HZ or +// every delay it computes in jiffies is wrong by the ratio -- phylib's +// one-second link poll became ten seconds at the old 100. + +static unsigned long long +now_ns(void) +{ + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) < 0) + return (unsigned long long)uptime() * 10000000ULL; + return (unsigned long long)ts.tv_sec * 1000000000ULL + (unsigned long long)ts.tv_nsec; +} + +unsigned long long kpi_ktime_get(void) { return now_ns(); } + +unsigned long +kpi_jiffies_ms(void) +{ + return (unsigned long)(now_ns() / 1000000ULL); +} + +static void +spin_us(unsigned long us) +{ + unsigned long long end = now_ns() + (unsigned long long)us * 1000ULL; + while (now_ns() < end) + __builtin_ia32_pause(); +} + +static void +sleep_us(unsigned long us) +{ + // Below a scheduler tick, a sleep would round up to the tick; the waits + // that short are MDIO and register polls, where that would be a hundredfold. + if (us < 10000) { + spin_us(us); + return; + } + struct timespec ts = { (long)(us / 1000000), (long)(us % 1000000) * 1000 }; + nanosleep(&ts); +} + +// udelay(n) arrives as __const_udelay(n * 0x10c7): microseconds scaled by +// 2^32 / 10^6. +void kpi_const_udelay(unsigned long xloops) { spin_us((unsigned long)(((unsigned long long)xloops * 1000000ULL) >> 32) + 1); } +void kpi_udelay(unsigned long us) { spin_us(us); } +void kpi_ndelay(unsigned long ns) { spin_us(ns / 1000 + 1); } +void kpi_msleep(unsigned int ms) { sleep_us((unsigned long)ms * 1000); } +void kpi_usleep_range(unsigned long min, unsigned long max, unsigned int st) +{ (void)max; (void)st; sleep_us(min); } + +// --- bitmaps ------------------------------------------------------------------------- +// +// phylib keeps link modes in 121-bit maps, past the size Linux inlines, so +// these are called for every "what can both ends do" question it asks. + +#define WORDS(n) (((n) + 63) / 64) + +static unsigned long +lastmask(unsigned int nbits) +{ + return (nbits % 64) ? (1UL << (nbits % 64)) - 1 : ~0UL; +} + +int +kpi_bitmap_and(unsigned long *d, const unsigned long *a, const unsigned long *b, + unsigned int nbits) +{ + unsigned long any = 0; + unsigned int w = WORDS(nbits); + for (unsigned int i = 0; i < w; i++) { + d[i] = a[i] & b[i]; + any |= (i == w - 1) ? (d[i] & lastmask(nbits)) : d[i]; + } + return any != 0; +} + +int +kpi_bitmap_andnot(unsigned long *d, const unsigned long *a, + const unsigned long *b, unsigned int nbits) +{ + unsigned long any = 0; + unsigned int w = WORDS(nbits); + for (unsigned int i = 0; i < w; i++) { + d[i] = a[i] & ~b[i]; + any |= (i == w - 1) ? (d[i] & lastmask(nbits)) : d[i]; + } + return any != 0; +} + +void +kpi_bitmap_or(unsigned long *d, const unsigned long *a, const unsigned long *b, + unsigned int nbits) +{ + for (unsigned int i = 0; i < WORDS(nbits); i++) + d[i] = a[i] | b[i]; +} + +int +kpi_bitmap_equal(const unsigned long *a, const unsigned long *b, + unsigned int nbits) +{ + unsigned int w = WORDS(nbits); + for (unsigned int i = 0; i < w; i++) { + unsigned long m = (i == w - 1) ? lastmask(nbits) : ~0UL; + if ((a[i] ^ b[i]) & m) return 0; + } + return 1; +} + +int +kpi_bitmap_intersects(const unsigned long *a, const unsigned long *b, + unsigned int nbits) +{ + unsigned int w = WORDS(nbits); + for (unsigned int i = 0; i < w; i++) { + unsigned long m = (i == w - 1) ? lastmask(nbits) : ~0UL; + if (a[i] & b[i] & m) return 1; + } + return 0; +} + +unsigned long +kpi_find_first_bit(const unsigned long *addr, unsigned long size) +{ + for (unsigned long i = 0; i < size; i++) + if (addr[i / 64] & (1UL << (i % 64))) + return i; + return size; +} + +// --- the driver core ----------------------------------------------------------------- +// +// Enough of it to bind a driver to a device on a bus, which is all libphy +// asks of it: drivers and devices are remembered, and whenever one arrives it +// is offered to the other side -- bus_type.match decides, and the bus's probe +// or else the driver's runs with device.driver already set, as Linux does. +// Nothing is reference-counted and nothing is ever torn down; see the note on +// memory in kpi.c. + +#define NCORE 64 +static void *core_drv[NCORE]; +static int core_ndrv; +static void *core_dev[NCORE]; +static int core_ndev; + +static int +core_bind(void *dev, void *drv) +{ + void *bus = (void *)rdp(dev, kpi_l.dev_bus); + int (*match)(void *, void *) = bus ? (int (*)(void *, void *))rdp(bus, kpi_l.bus_match) : 0; + int (*bprobe)(void *) = bus ? (int (*)(void *))rdp(bus, kpi_l.bus_probe) : 0; + int (*dprobe)(void *) = (int (*)(void *))rdp(drv, kpi_l.drv_probe); + + if (match && match(dev, drv) <= 0) + return 0; + wrp(dev, kpi_l.dev_driver, (unsigned long)drv); + int r = bprobe ? bprobe(dev) : dprobe ? dprobe(dev) : 0; + const char *dn = (const char *)rdp(drv, kpi_l.drv_name); + kpi_log(" kpi: bound "); kpi_log(dn ? dn : "a driver"); + kpi_log(" to "); kpi_log(dev_name_of(dev)); + if (r != 0) { + kpi_log(" -- probe said "); kpi_lognum(r); kpi_log(", unbound\n"); + wrp(dev, kpi_l.dev_driver, 0); + return 0; + } + kpi_log("\n"); + return 1; +} + +static void +core_offer_dev(void *dev) +{ + void *bus; + if (!kpi_l.core_ok || rdp(dev, kpi_l.dev_driver)) + return; + bus = (void *)rdp(dev, kpi_l.dev_bus); + if (!bus) + return; + for (int i = 0; i < core_ndrv; i++) + if ((void *)rdp(core_drv[i], kpi_l.drv_bus) == bus && core_bind(dev, core_drv[i])) + return; +} + +int +kpi_driver_register(void *drv) +{ + if (core_ndrv < NCORE) + core_drv[core_ndrv++] = drv; + if (!kpi_l.core_ok) + return 0; + void *bus = (void *)rdp(drv, kpi_l.drv_bus); + for (int i = 0; i < core_ndev; i++) + if (!rdp(core_dev[i], kpi_l.dev_driver) && + (void *)rdp(core_dev[i], kpi_l.dev_bus) == bus) + core_bind(core_dev[i], drv); + return 0; +} + +void +kpi_driver_unregister(void *drv) +{ + for (int i = 0; i < core_ndrv; i++) + if (core_drv[i] == drv) core_drv[i] = core_drv[--core_ndrv]; +} + +int +kpi_device_add(void *dev) +{ + int i; + for (i = 0; i < core_ndev; i++) + if (core_dev[i] == dev) break; + if (i == core_ndev && core_ndev < NCORE) + core_dev[core_ndev++] = dev; + core_offer_dev(dev); + return 0; +} + +void +kpi_device_del(void *dev) +{ + for (int i = 0; i < core_ndev; i++) + if (core_dev[i] == dev) core_dev[i] = core_dev[--core_ndev]; +} + +void kpi_device_release_driver(void *dev) +{ if (kpi_l.core_ok) wrp(dev, kpi_l.dev_driver, 0); } + +void *kpi_get_device(void *dev) { return dev; } + +int +kpi_dev_set_name(void *dev, const char *f, ...) +{ + if (!kpi_l.dev_kobj_ok) + return 0; + va_list ap; + va_start(ap, f); + char *s = kpi_kvasprintf(f, ap); + va_end(ap); + if (!s) + return -12; + wrp(dev, kpi_l.dev_kobj + kpi_l.kobj_name, (unsigned long)s); + return 0; +} + +const char * +kpi_dev_driver_string(void *dev) +{ + if (kpi_l.core_ok && dev) { + void *drv = (void *)rdp(dev, kpi_l.dev_driver); + if (drv) { + const char *n = (const char *)rdp(drv, kpi_l.drv_name); + if (n) return n; + } + } + return ""; +} + +// --- device-managed resources ---------------------------------------------------------- +// +// Freed when the device goes, which here is never. What has to be real is +// devres_find: mdio_devres registers a bus only after finding the record +// devm_mdiobus_alloc made, and says -EINVAL otherwise. + +#define NRES 128 +static struct { void *dev, *release, *data; } res[NRES]; +static int nres; + +void * +kpi_devres_alloc(void *release, unsigned long size, unsigned int gfp, int nid, + const char *name) +{ + (void)gfp; (void)nid; (void)name; + void *d = zalloc(size ? size : 8); + if (d && nres < NRES) { + res[nres].dev = 0; res[nres].release = release; res[nres].data = d; + nres++; + } + return d; +} + +void +kpi_devres_add(void *dev, void *data) +{ + for (int i = 0; i < nres; i++) + if (res[i].data == data) { res[i].dev = dev; return; } +} + +void * +kpi_devres_find(void *dev, void *release, int (*match)(void *, void *, void *), + void *mdata) +{ + for (int i = 0; i < nres; i++) + if (res[i].dev == dev && res[i].release == release && + (!match || match(dev, res[i].data, mdata))) + return res[i].data; + return 0; +} + +void +kpi_devres_free(void *data) +{ + for (int i = 0; i < nres; i++) + if (res[i].data == data) { res[i] = res[--nres]; return; } +} + +void *kpi_devm_kmalloc(void *dev, unsigned long size, unsigned int gfp) +{ (void)dev; (void)gfp; return zalloc(size); } + +void *kpi_devm_alloc_etherdev(void *dev, int priv, unsigned int tx, unsigned int rx) +{ (void)dev; return kpi_alloc_etherdev_mqs(priv, tx, rx); } + +void * +kpi_dmam_alloc_attrs(void *dev, unsigned long size, unsigned long long *h, + unsigned int gfp, unsigned long attrs) +{ return kpi_dma_alloc_attrs(dev, size, h, gfp, attrs); } + +// --- PCI, the managed and newer calls ---------------------------------------------------- + +int kpi_pcim_enable_device(void *pdev) { return kpi_pci_enable_device(pdev); } + +void * +kpi_pcim_iomap_region(void *pdev, int bar, const char *name) +{ + (void)name; + void *p = kpi_pci_iomap(pdev, bar, 0); + return p ? p : (void *)-12L; // ERR_PTR(-ENOMEM); NULL is not an error +} + +// One vector, and its number is whatever request_irq is then given back -- +// the handler is called from the poll loop either way. See kpi_request_irq. +int kpi_pci_alloc_irq_vectors(void *pdev, unsigned int min, unsigned int max, + unsigned int flags) +{ (void)pdev; (void)min; (void)max; (void)flags; return 1; } + +int kpi_pci_irq_vector(void *pdev, unsigned int nr) { (void)pdev; (void)nr; return 16; } + +int +kpi_pci_read_config_byte(void *pdev, int where, unsigned char *v) +{ + unsigned short w = 0; + int r = kpi_pci_read_config_word(pdev, where & ~1, &w); + if (v) *v = (unsigned char)(w >> ((where & 1) * 8)); + return r; +} + +// /dev/pci has no config write (see pci.c). The ones r8169 makes are tuning, +// and the ASPM change it wants is made by the kernel at boot instead. +int +kpi_pci_write_config_byte(void *pdev, int where, unsigned char v) +{ + (void)pdev; (void)v; + kpi_log(" kpi: config write to 0x"); kpi_loghex((unsigned)where, 2); + kpi_log(" not made\n"); + return 0; +} + +void *kpi_alloc_pages_node(unsigned int gfp, unsigned int order, int nid, void *mask) +{ (void)nid; (void)mask; return kpi_alloc_pages(gfp, order); } + +// --- work queues, the rest of the API ---------------------------------------------------- + +static char kpi_wq_dummy[64]; +void *kpi_system_wq = kpi_wq_dummy; // the *variable* is the symbol + +void *kpi_alloc_workqueue(const char *f, unsigned int flags, int max, ...) +{ (void)f; (void)flags; (void)max; return kpi_wq_dummy; } + +int +kpi_mod_delayed_work_on(int cpu, void *q, void *dwork, unsigned long delay) +{ + kpi_queue_delayed_work_on(cpu, q, dwork, delay); + return 0; +} + +int kpi_disable_work_sync(void *w) { kpi_cancel_work_sync(w); return 0; } + +// --- firmware ------------------------------------------------------------------------------ +// +// From /lib/firmware, which is where Alpine's linux-firmware-* packages put +// it. struct firmware's layout comes from BTF like everything else. + +int +kpi_request_firmware(void **out, const char *name, void *dev) +{ + char path[256]; + (void)dev; + if (out) *out = 0; + if (!kpi_l.fw_ok || !name) + return -2; + kpi_snprintf(path, sizeof(path), "/lib/firmware/%s", name); + int fd = open(path, O_RDONLY); + if (fd < 0) { + kpi_log(" kpi: no firmware "); kpi_log(path); kpi_log("\n"); + return -2; + } + unsigned long cap = 4UL << 20, n = 0; + unsigned char *buf = zalloc(cap); + if (!buf) { close(fd); return -12; } + for (;;) { + int r = read(fd, buf + n, (int)(cap - n > 65536 ? 65536 : cap - n)); + if (r <= 0) break; + n += (unsigned long)r; + if (n == cap) break; + } + close(fd); + void *fw = zalloc(kpi_l.fw_size_of ? kpi_l.fw_size_of : 64); + if (!fw) return -12; + wrp(fw, kpi_l.fw_size, n); + wrp(fw, kpi_l.fw_data, (unsigned long)buf); + if (out) *out = fw; + kpi_log(" kpi: firmware "); kpi_log(path); kpi_log(", "); + kpi_lognum((long long)n); kpi_log(" bytes\n"); + return 0; +} + +// --- small things that must not answer zero ------------------------------------------------ + +int kpi_true(void) { return 1; } +int kpi_enodev(void) { return -19; } +long kpi_ida_alloc(void *ida, unsigned int min, unsigned int max, unsigned int gfp) +{ + static long next; + (void)ida; (void)max; (void)gfp; + return (long)min + next++; +} + +unsigned int +kpi_crc32_le(unsigned int crc, const unsigned char *p, unsigned long len) +{ + while (len--) { + crc ^= *p++; + for (int k = 0; k < 8; k++) + crc = (crc >> 1) ^ (0xEDB88320u & (0u - (crc & 1))); + } + return crc; +} + +void +kpi_get_random_bytes(void *buf, unsigned long n) +{ + static unsigned long long x = 0x9E3779B97F4A7C15ULL; + unsigned char *b = buf; + x ^= now_ns(); + for (unsigned long i = 0; i < n; i++) { + x ^= x << 13; x ^= x >> 7; x ^= x << 17; + b[i] = (unsigned char)x; + } +} + +// __skb_pad(skb, pad, free_on_error): zero the bytes after the data, so a +// runt frame goes out padded with zeroes rather than with whatever was there. +int +kpi_skb_pad(void *skb, int pad, int free_on_error) +{ + (void)free_on_error; + unsigned char *head = (unsigned char *)rdp(skb, kpi_l.skb_head); + unsigned int tail, end; + __builtin_memcpy(&tail, (char *)skb + kpi_l.skb_tail, 4); + __builtin_memcpy(&end, (char *)skb + kpi_l.skb_end, 4); + if (pad < 0 || tail + (unsigned)pad > end) + return -12; + for (int i = 0; i < pad; i++) head[tail + i] = 0; + return 0; +} + +// --- data ------------------------------------------------------------------------------------- +// +// Undefined data symbols. A stub would resolve them to machine code, which +// reads as nonsense and, written to, overwrites a trampoline. + +// net/ethtool/common.c's link_mode_params: speed, lanes, duplex per link mode +// bit, which phylib reads at init to build its capability table. A fact table, +// generated from Linux 6.18; struct link_mode_info is int, u8, u8 (8 bytes). +static const struct { int speed; unsigned char lanes, duplex; } kpi_link_mode_params[] = { +#include "link_modes.inc" +}; + +static unsigned char kpi_zero_data[16][4096] __attribute__((aligned(64))); +static unsigned long kpi_cpu_online_mask[16] = { 1 }; // CPU 0 +static int kpi_system_state = 3; // SYSTEM_RUNNING +// A list_head: empty means pointing at itself, not zero. +static void *kpi_leds_list[2] = { kpi_leds_list, kpi_leds_list }; + +// --- the table ------------------------------------------------------------------------------------ + +extern int kpi_queue_delayed_work_on(int cpu, void *q, void *dwork, unsigned long delay); +extern int kpi_cancel_work_sync(void *work); + +const struct kpi_sym kpi_base_table[] = { + // printing, formatted + { "_printk", (void *)kpi_printk_fmt }, + { "vprintk", (void *)kpi_vprintk }, + { "_dev_info", (void *)kpi_dev_printk }, + { "_dev_err", (void *)kpi_dev_printk }, + { "_dev_warn", (void *)kpi_dev_printk }, + { "_dev_notice", (void *)kpi_dev_printk }, + { "netdev_info", (void *)kpi_netdev_printk_fmt }, + { "netdev_err", (void *)kpi_netdev_printk_fmt }, + { "netdev_warn", (void *)kpi_netdev_printk_fmt }, + { "netdev_alert", (void *)kpi_netdev_printk_fmt }, + { "netdev_printk", (void *)kpi_netdev_printk_lvl }, + { "dev_err_probe", (void *)kpi_dev_err_probe }, + { "__warn_printk", (void *)kpi_warn_printk }, + { "snprintf", (void *)kpi_snprintf }, + { "sprintf", (void *)kpi_sprintf }, + { "kasprintf", (void *)kpi_kasprintf }, + + // strings + { "strcmp", (void *)kpi_strcmp }, + { "strcpy", (void *)kpi_strcpy }, + { "kstrdup_const", (void *)kpi_kstrdup_const }, + { "kfree_const", (void *)kpi_nop }, + { "kstrtoull", (void *)kpi_kstrtoull }, + + // time + { "ktime_get", (void *)kpi_ktime_get }, + { "__const_udelay", (void *)kpi_const_udelay }, + { "__udelay", (void *)kpi_udelay }, + { "__ndelay", (void *)kpi_ndelay }, + { "msleep", (void *)kpi_msleep }, + { "usleep_range_state", (void *)kpi_usleep_range }, + + // bitmaps + { "__bitmap_and", (void *)kpi_bitmap_and }, + { "__bitmap_andnot", (void *)kpi_bitmap_andnot }, + { "__bitmap_or", (void *)kpi_bitmap_or }, + { "__bitmap_equal", (void *)kpi_bitmap_equal }, + { "__bitmap_intersects", (void *)kpi_bitmap_intersects }, + { "_find_first_bit", (void *)kpi_find_first_bit }, + + // the driver core + { "driver_register", (void *)kpi_driver_register }, + { "driver_unregister", (void *)kpi_driver_unregister }, + { "device_add", (void *)kpi_device_add }, + { "device_register", (void *)kpi_device_add }, + { "device_del", (void *)kpi_device_del }, + { "device_unregister", (void *)kpi_device_del }, + { "device_initialize", (void *)kpi_nop }, + { "device_bind_driver", (void *)kpi_zero }, + { "device_release_driver", (void *)kpi_device_release_driver }, + { "get_device", (void *)kpi_get_device }, + { "put_device", (void *)kpi_nop }, + { "dev_set_name", (void *)kpi_dev_set_name }, + { "dev_driver_string", (void *)kpi_dev_driver_string }, + { "bus_register", (void *)kpi_zero }, + { "class_register", (void *)kpi_zero }, + { "try_module_get", (void *)kpi_true }, + { "module_put", (void *)kpi_nop }, + + // device-managed resources + { "__devres_alloc_node", (void *)kpi_devres_alloc }, + { "devres_add", (void *)kpi_devres_add }, + { "devres_find", (void *)kpi_devres_find }, + { "devres_free", (void *)kpi_devres_free }, + { "devm_kmalloc", (void *)kpi_devm_kmalloc }, + { "devm_alloc_etherdev_mqs", (void *)kpi_devm_alloc_etherdev }, + { "dmam_alloc_attrs", (void *)kpi_dmam_alloc_attrs }, + + // PCI + { "pcim_enable_device", (void *)kpi_pcim_enable_device }, + { "pcim_iomap_region", (void *)kpi_pcim_iomap_region }, + { "pci_alloc_irq_vectors", (void *)kpi_pci_alloc_irq_vectors }, + { "pci_irq_vector", (void *)kpi_pci_irq_vector }, + { "pci_read_config_byte", (void *)kpi_pci_read_config_byte }, + { "pci_write_config_byte", (void *)kpi_pci_write_config_byte }, + { "__alloc_pages_noprof", (void *)kpi_alloc_pages_node }, + { "__free_pages", (void *)kpi_nop }, + + // work + { "mod_delayed_work_on", (void *)kpi_mod_delayed_work_on }, + { "alloc_workqueue_noprof", (void *)kpi_alloc_workqueue }, + { "destroy_workqueue", (void *)kpi_nop }, + { "disable_work_sync", (void *)kpi_disable_work_sync }, + { "disable_work", (void *)kpi_disable_work_sync }, + { "enable_work", (void *)kpi_true }, + { "system_power_efficient_wq", (void *)&kpi_system_wq }, + { "system_percpu_wq", (void *)&kpi_system_wq }, + { "system_wq", (void *)&kpi_system_wq }, + + // firmware + { "firmware_request_nowarn", (void *)kpi_request_firmware }, + { "request_firmware", (void *)kpi_request_firmware }, + { "release_firmware", (void *)kpi_nop }, + + // must not be zero + { "__list_add_valid_or_report", (void *)kpi_true }, + { "__list_del_entry_valid_or_report", (void *)kpi_true }, + { "mutex_is_locked", (void *)kpi_true }, + { "eth_platform_get_mac_address", (void *)kpi_enodev }, + { "ida_alloc_range", (void *)kpi_ida_alloc }, + { "crc32_le", (void *)kpi_crc32_le }, + { "get_random_bytes", (void *)kpi_get_random_bytes }, + { "__skb_pad", (void *)kpi_skb_pad }, + + // data + { "link_mode_params", (void *)kpi_link_mode_params }, + { "__cpu_online_mask", (void *)kpi_cpu_online_mask }, + { "system_state", (void *)&kpi_system_state }, + { "phylib_stubs", (void *)kpi_zero_data[0] }, + { "phys_base", (void *)kpi_zero_data[7] }, + { "kmalloc_caches", (void *)kpi_zero_data[1] }, + { "random_kmalloc_seed", (void *)kpi_zero_data[2] }, + { "mem_alloc_profiling_key", (void *)kpi_zero_data[3] }, + { "of_fwnode_ops", (void *)kpi_zero_data[4] }, + { "leds_list", (void *)kpi_leds_list }, + { "leds_list_lock", (void *)kpi_zero_data[6] }, +}; + +const int kpi_base_count = (int)(sizeof(kpi_base_table) / sizeof(kpi_base_table[0])); diff --git a/ldls/kpi/kpi.c b/ldls/kpi/kpi.c index a0fd940..f1d9d0c 100644 --- a/ldls/kpi/kpi.c +++ b/ldls/kpi/kpi.c @@ -131,6 +131,9 @@ void * kpi_find(const char *name) { int i; + for (i = 0; i < kpi_base_count; i++) + if (eq(kpi_base_table[i].name, name)) + return kpi_base_table[i].fn; for (i = 0; i < kpi_count; i++) if (eq(kpi_table[i].name, name)) return kpi_table[i].fn; @@ -512,18 +515,16 @@ static const unsigned char kpi_byte_rev_table[256] = { // reads it, re-arms itself against it, and enables the transmitter from // inside it, so a stopped clock is a NIC that never transmits. // -// SchOS's uptime() counts 100 Hz ticks, which is a plausible HZ and is the -// only monotonic source a process has. What matters to a driver is that it -// advances and never goes backwards; the exact rate matters only to timeouts, -// and a timeout measured against a slower clock is a longer timeout. +// It counts at the module's HZ, which is compiled into the module and is 1000 +// on the kernels LDLS loads from -- phylib re-arms its link poll HZ jiffies +// ahead, and at the 100 Hz this used to count that poll ran every ten seconds. +// Milliseconds from CLOCK_MONOTONIC; see kpi_jiffies_ms() in base.c. unsigned long kpi_jiffies; unsigned long kpi_jiffies_now(void) { - int t = uptime(); - if (t > 0) - kpi_jiffies = (unsigned long)t; + kpi_jiffies = kpi_jiffies_ms(); return kpi_jiffies; } diff --git a/ldls/kpi/kpi.h b/ldls/kpi/kpi.h index 31df6cf..0aaafe9 100644 --- a/ldls/kpi/kpi.h +++ b/ldls/kpi/kpi.h @@ -35,6 +35,15 @@ extern const int kpi_count; // driver ever calls it. void *kpi_find(const char *name); +// The second table, in base.c. Searched first, so that a formatted printk +// there replaces the plain one in kpi.c. +extern const struct kpi_sym kpi_base_table[]; +extern const int kpi_base_count; + +void kpi_nop(void); +int kpi_zero(void); +unsigned long kpi_jiffies_ms(void); + // --- what the host gives the KPI --------------------------------------------- // // The KPI cannot open these for itself: `/dev/pci` is one device per process @@ -92,6 +101,17 @@ struct kpi_layout { unsigned int skb_size; unsigned int skb_len, skb_data, skb_head, skb_tail, skb_end, skb_dev; + + // The driver core (ldls/kpi/base.c): device.bus and .driver, + // device_driver.bus, .probe and .name, bus_type.match and .probe. core_ok + // says all of them were found; without them nothing is ever bound. + unsigned int dev_bus, dev_driver, drv_bus, drv_probe, drv_name; + unsigned int bus_match, bus_probe; + int core_ok, dev_kobj_ok; + + // struct firmware: size and data, and how big it is. + unsigned int fw_size, fw_data, fw_size_of; + int fw_ok; }; extern struct kpi_layout kpi_l; diff --git a/ldls/kpi/link_modes.inc b/ldls/kpi/link_modes.inc new file mode 100644 index 0000000..57151e5 --- /dev/null +++ b/ldls/kpi/link_modes.inc @@ -0,0 +1,122 @@ +// 121 entries, generated from Linux 6.18 net/ethtool/common.c + { 10, 1, 0x0 }, + { 10, 1, 0x1 }, + { 100, 1, 0x0 }, + { 100, 1, 0x1 }, + { 1000, 1, 0x0 }, + { 1000, 1, 0x1 }, + { -1, 0, 0xff }, + { -1, 0, 0xff }, + { -1, 0, 0xff }, + { -1, 0, 0xff }, + { -1, 0, 0xff }, + { -1, 0, 0xff }, + { 10000, 1, 0x1 }, + { -1, 0, 0xff }, + { -1, 0, 0xff }, + { 2500, 1, 0x1 }, + { -1, 0, 0xff }, + { 1000, 1, 0x1 }, + { 10000, 4, 0x1 }, + { 10000, 1, 0x1 }, + { 10000, 1, 0x1 }, + { 20000, 2, 0x1 }, + { 20000, 2, 0x1 }, + { 40000, 4, 0x1 }, + { 40000, 4, 0x1 }, + { 40000, 4, 0x1 }, + { 40000, 4, 0x1 }, + { 56000, 4, 0x1 }, + { 56000, 4, 0x1 }, + { 56000, 4, 0x1 }, + { 56000, 4, 0x1 }, + { 25000, 1, 0x1 }, + { 25000, 1, 0x1 }, + { 25000, 1, 0x1 }, + { 50000, 2, 0x1 }, + { 50000, 2, 0x1 }, + { 100000, 4, 0x1 }, + { 100000, 4, 0x1 }, + { 100000, 4, 0x1 }, + { 100000, 4, 0x1 }, + { 50000, 2, 0x1 }, + { 1000, 1, 0x1 }, + { 10000, 1, 0x1 }, + { 10000, 1, 0x1 }, + { 10000, 1, 0x1 }, + { 10000, 1, 0x1 }, + { 10000, 1, 0x1 }, + { 2500, 1, 0x1 }, + { 5000, 1, 0x1 }, + { -1, 0, 0xff }, + { -1, 0, 0xff }, + { -1, 0, 0xff }, + { 50000, 1, 0x1 }, + { 50000, 1, 0x1 }, + { 50000, 1, 0x1 }, + { 50000, 1, 0x1 }, + { 50000, 1, 0x1 }, + { 100000, 2, 0x1 }, + { 100000, 2, 0x1 }, + { 100000, 2, 0x1 }, + { 100000, 2, 0x1 }, + { 100000, 2, 0x1 }, + { 200000, 4, 0x1 }, + { 200000, 4, 0x1 }, + { 200000, 4, 0x1 }, + { 200000, 4, 0x1 }, + { 200000, 4, 0x1 }, + { 100, 1, 0x1 }, + { 1000, 1, 0x1 }, + { 400000, 8, 0x1 }, + { 400000, 8, 0x1 }, + { 400000, 8, 0x1 }, + { 400000, 8, 0x1 }, + { 400000, 8, 0x1 }, + { -1, 0, 0xff }, + { 100000, 1, 0x1 }, + { 100000, 1, 0x1 }, + { 100000, 1, 0x1 }, + { 100000, 1, 0x1 }, + { 100000, 1, 0x1 }, + { 200000, 2, 0x1 }, + { 200000, 2, 0x1 }, + { 200000, 2, 0x1 }, + { 200000, 2, 0x1 }, + { 200000, 2, 0x1 }, + { 400000, 4, 0x1 }, + { 400000, 4, 0x1 }, + { 400000, 4, 0x1 }, + { 400000, 4, 0x1 }, + { 400000, 4, 0x1 }, + { 100, 1, 0x0 }, + { 100, 1, 0x1 }, + { 10, 1, 0x1 }, + { 800000, 8, 0x1 }, + { 800000, 8, 0x1 }, + { 800000, 8, 0x1 }, + { 800000, 8, 0x1 }, + { 800000, 8, 0x1 }, + { 800000, 8, 0x1 }, + { 10, 1, 0x1 }, + { 10, 1, 0x0 }, + { 10, 1, 0x0 }, + { 10, 1, 0x1 }, + { 200000, 1, 0x1 }, + { 200000, 1, 0x1 }, + { 200000, 1, 0x1 }, + { 200000, 1, 0x1 }, + { 200000, 1, 0x1 }, + { 200000, 1, 0x1 }, + { 400000, 2, 0x1 }, + { 400000, 2, 0x1 }, + { 400000, 2, 0x1 }, + { 400000, 2, 0x1 }, + { 400000, 2, 0x1 }, + { 400000, 2, 0x1 }, + { 800000, 4, 0x1 }, + { 800000, 4, 0x1 }, + { 800000, 4, 0x1 }, + { 800000, 4, 0x1 }, + { 800000, 4, 0x1 }, + { 800000, 4, 0x1 }, diff --git a/ldls/stub.c b/ldls/stub.c index 8d22310..35fb518 100644 --- a/ldls/stub.c +++ b/ldls/stub.c @@ -85,6 +85,12 @@ stub_is_bare(const char *name) void stub_build(struct stubs *s, unsigned char *code, int n) +{ + stub_build_from(s, code, n, 0); +} + +void +stub_build_from(struct stubs *s, unsigned char *code, int n, int first) { int i; @@ -107,7 +113,7 @@ stub_build(struct stubs *s, unsigned char *code, int n) // movl $i, %r11d -- 41 bb imm32. The upper half of %r11 is zeroed by the // 32-bit form, which is what we want for a small index. p[4] = 0x41; p[5] = 0xBB; - put32(p + 6, (unsigned int)i); + put32(p + 6, (unsigned int)(first + i)); // movabsq $ldls_thunk, %r10 -- 49 ba imm64 p[10] = 0x49; p[11] = 0xBA; diff --git a/ldls/stub.h b/ldls/stub.h index 0b662a4..af0f0ee 100644 --- a/ldls/stub.h +++ b/ldls/stub.h @@ -35,6 +35,9 @@ struct stubs { // // `code` has to be executable memory; this only writes the bytes. void stub_build(struct stubs *s, unsigned char *code, int n); +// The same, numbering the stubs from `first`: with several modules loaded, +// each module's stubs follow the previous module's in one shared name table. +void stub_build_from(struct stubs *s, unsigned char *code, int n, int first); // The address of stub `i`, to hand to the loader as a symbol value. unsigned long long stub_addr(const struct stubs *s, int i);