ldls: a second driver, and everything it found
e1000 -- Linux's, unmodified -- probes, opens, gets an address by DHCP and
carries ping traffic in a SchOS process. The kernel was driving that NIC and
gave it up, which makes it the first real handover as well as the second
driver.
None of what it needed was a missing feature. Every one was a call that
succeeded, returned zero and did nothing:
_find_next_bit "bit 0 is set" -- for_each_cpu never ended
netif_carrier_off carrier already up -- TCTL.EN never set
__x86_indirect_thunk_r8 the jump discarded -- 255 rx buffers, none made
dma_map_page_attrs bus address 0 -- NIC aimed at physical 0
so the rule from memcpy is now general: a stub returning zero is unsafe
wherever zero is a meaningful answer.
The loader grew ko_fixup_percpu(). The stack guard turned out to be one
per-CPU variable among several; `current` cannot be defused because the driver
dereferences it. Taking the %gs off instead leaves the same opcode reading the
same width from storage the host supplies -- a per-CPU variable in a
single-threaded host is just a variable. Eight bytes become seven and a nop,
and the displacement is not always last, so there is now just enough of an x86
decoder in ko.c to know how long the instruction is.
ldls/kpi/thunk.c is fifteen lines of assembly and the highest-value file here:
retpolines are control flow, and a stub cannot stand in for a jump.
The kernel side:
- ioperm.c, the TSS I/O permission bitmap. A driver's outl is an instruction,
not a call, and e1000's reset path picks it deliberately. IOPL=3 would hand
over the PIC and the disk with it, so it is per-port. Writing it found that
the I/O map base had been written at offset 64 rather than 102, leaving the
CPU checking ports 0-831 against the TSS's own RSP0.
- claim masks the device's IRQ line: 450,000 a second, otherwise.
- net_detach(), the missing half of net_attach_late(). An interface left in
the stack keeps its route and sends every frame to a driver that refuses
it, which looks exactly like a userspace driver that does not work.
- e1000_release() gives up the interface, not just the hardware.
8139too still passes 4/4; ldls-check still passes 27; four CPUs, no panic.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01VQ7Qx8E1q1M1bbCq8QLHS2
This commit is contained in:
Nexoniarz
parent
f38eb8a7fd
commit
071499a378
26 changed files with 2009 additions and 46 deletions
244
ldls/ko.c
244
ldls/ko.c
|
|
@ -597,6 +597,250 @@ ko_defuse_gs(struct ko *k, const char *sym)
|
|||
return n;
|
||||
}
|
||||
|
||||
// Could this symbol name a per-CPU variable at all?
|
||||
//
|
||||
// Imports can: `const_current_task` and `softnet_data` live in vmlinux. A
|
||||
// module's own per-CPU variables live in a section called .data..percpu.
|
||||
// Anything else -- a section symbol for .text, an ordinary global -- cannot,
|
||||
// and saying so is what keeps a coincidental 0x65 from being read as a prefix.
|
||||
static int
|
||||
percpu_symbol(struct ko *k, unsigned int si)
|
||||
{
|
||||
unsigned short shndx = k->sym[si].st_shndx;
|
||||
const struct ko_shdr *sh;
|
||||
const char *nm;
|
||||
int i;
|
||||
|
||||
if (shndx == 0) /* undefined: an import */
|
||||
return 1;
|
||||
if (shndx >= (unsigned short)k->nsh || !k->shstr)
|
||||
return 0;
|
||||
sh = ko_sh(k, shndx);
|
||||
if (!sh)
|
||||
return 0;
|
||||
nm = k->shstr + sh->sh_name;
|
||||
for (i = 0; ".data..percpu"[i]; i++)
|
||||
if (nm[i] != ".data..percpu"[i])
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Take the `%gs` off.
|
||||
//
|
||||
// The first idea here was to rewrite the load into a `lea` of memory we
|
||||
// supply, which works for `current` -- a pointer -- and is wrong for
|
||||
// `cpu_number`, which is a value and needs to read as 0. Dropping the segment
|
||||
// prefix instead serves both: the instruction keeps its opcode, its width and
|
||||
// its direction, and simply addresses memory we own rather than memory only a
|
||||
// kernel has. A per-CPU variable becomes a variable, which on a host with one
|
||||
// thread pretending to be one CPU is what it was all along.
|
||||
//
|
||||
// It is one byte shorter, so the last byte becomes a nop and nothing after it
|
||||
// moves. The displacement is measured from the end of the *shortened*
|
||||
// instruction, which is the subtlety worth stating: taking it from the end of
|
||||
// the original puts every access one byte off, and reads plausible data from
|
||||
// the wrong place.
|
||||
|
||||
// How long the instruction is, and how much of it comes before the
|
||||
// displacement.
|
||||
//
|
||||
// Enough of an x86 decoder to know that and no more. The displacement is not
|
||||
// always last -- `cmpb $0x0, %gs:softnet_data+0x75(%rip)` has an immediate
|
||||
// after it, and assuming otherwise moved the wrong four bytes. So the opcode
|
||||
// has to be read far enough to know how big the immediate is.
|
||||
//
|
||||
// Returns the total length, or 0 for a shape we have not established the
|
||||
// meaning of. Refusing is the useful answer there: rule 5 says the driver tells
|
||||
// us what to implement next, and a silently mis-decoded instruction tells us
|
||||
// nothing at all.
|
||||
static int
|
||||
gs_form(const unsigned char *p, int want_pre)
|
||||
{
|
||||
int j = 1, op, modrm, imm = 0, pre;
|
||||
|
||||
if (p[0] != 0x65)
|
||||
return 0;
|
||||
if ((p[1] & 0xF0) == 0x40) /* REX */
|
||||
j = 2;
|
||||
if (p[j] == 0x0f) { /* two-byte opcode */
|
||||
op = 0x0f00 | p[j + 1];
|
||||
modrm = p[j + 2];
|
||||
pre = j + 3;
|
||||
} else {
|
||||
op = p[j];
|
||||
modrm = p[j + 1];
|
||||
pre = j + 2;
|
||||
}
|
||||
if (pre != want_pre)
|
||||
return 0;
|
||||
if ((modrm & 0xC7) != 0x05) /* rip-relative, no SIB */
|
||||
return 0;
|
||||
|
||||
switch (op) {
|
||||
/* mov, arithmetic, test, exchange -- register or implicit operand */
|
||||
case 0x88: case 0x89: case 0x8a: case 0x8b: case 0x8d:
|
||||
case 0x00: case 0x01: case 0x02: case 0x03:
|
||||
case 0x08: case 0x09: case 0x0a: case 0x0b:
|
||||
case 0x20: case 0x21: case 0x22: case 0x23:
|
||||
case 0x28: case 0x29: case 0x2a: case 0x2b:
|
||||
case 0x30: case 0x31: case 0x32: case 0x33:
|
||||
case 0x38: case 0x39: case 0x3a: case 0x3b:
|
||||
case 0x84: case 0x85: case 0x86: case 0x87:
|
||||
case 0xff: case 0xfe:
|
||||
imm = 0; break;
|
||||
/* the same, with an immediate */
|
||||
case 0x80: case 0x83: case 0xc0: case 0xc1: case 0xc6:
|
||||
imm = 1; break;
|
||||
case 0x81: case 0xc7:
|
||||
imm = 4; break;
|
||||
/* group 3: only /0 (test) carries an immediate */
|
||||
case 0xf6:
|
||||
imm = ((modrm >> 3) & 7) == 0 ? 1 : 0; break;
|
||||
case 0xf7:
|
||||
imm = ((modrm >> 3) & 7) == 0 ? 4 : 0; break;
|
||||
/* movzx, movsx, setcc, bit tests, cmpxchg, xadd */
|
||||
case 0x0fb6: case 0x0fb7: case 0x0fbe: case 0x0fbf:
|
||||
case 0x0fa3: case 0x0fab: case 0x0fb3: case 0x0fbb:
|
||||
case 0x0fb0: case 0x0fb1: case 0x0fc0: case 0x0fc1:
|
||||
imm = 0; break;
|
||||
case 0x0fba:
|
||||
imm = 1; break;
|
||||
default:
|
||||
if (op >= 0x0f90 && op <= 0x0f9f) /* setcc */
|
||||
{ imm = 0; break; }
|
||||
return 0;
|
||||
}
|
||||
return pre + 4 + imm;
|
||||
}
|
||||
|
||||
// A null `insn` measures without writing, which is how the first pass asks
|
||||
// whether every site is one this can do before the second pass does any of it.
|
||||
static int
|
||||
percpu_one(unsigned char *insn, int pre, int len, unsigned long long at,
|
||||
unsigned long long tgt)
|
||||
{
|
||||
long long rel;
|
||||
int i;
|
||||
|
||||
rel = (long long)tgt - (long long)(at + len - 1);
|
||||
if (rel < -0x80000000LL || rel > 0x7fffffffLL)
|
||||
return -2; /* too far for a rip-relative access */
|
||||
if (!insn)
|
||||
return 0;
|
||||
|
||||
for (i = 0; i < len - 1; i++)
|
||||
insn[i] = insn[i + 1];
|
||||
insn[len - 1] = 0x90;
|
||||
|
||||
insn[pre - 1] = (unsigned char)rel;
|
||||
insn[pre + 0] = (unsigned char)(rel >> 8);
|
||||
insn[pre + 1] = (unsigned char)(rel >> 16);
|
||||
insn[pre + 2] = (unsigned char)(rel >> 24);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Two passes, for the reason ko_defuse_gs() has two: a half-patched object is
|
||||
// worse than an unpatched one, because it runs.
|
||||
int
|
||||
ko_fixup_percpu(struct ko *k,
|
||||
unsigned long long (*percpu)(void *ctx, const char *name),
|
||||
void *ctx)
|
||||
{
|
||||
int pass, i, n = 0;
|
||||
|
||||
if (!k->laid_out)
|
||||
{ ko_fail(k, "percpu fixup before layout", 0); return -1; }
|
||||
|
||||
for (pass = 0; pass < 2; pass++) {
|
||||
n = 0;
|
||||
for (i = 0; i < k->nsh; i++) {
|
||||
const struct ko_shdr *sh = ko_sh(k, i);
|
||||
struct ko_section *tgt;
|
||||
unsigned long long cnt, j;
|
||||
|
||||
if (!sh || sh->sh_type != KO_SHT_RELA || !k->sec[i].data)
|
||||
continue;
|
||||
if (sh->sh_info >= (unsigned int)k->nsh)
|
||||
continue;
|
||||
tgt = &k->sec[sh->sh_info];
|
||||
if (!tgt->alloc || tgt->size == 0 || !(tgt->flags & KO_SHF_EXECINSTR))
|
||||
continue;
|
||||
|
||||
cnt = sh->sh_size / sizeof(struct ko_rela);
|
||||
for (j = 0; j < cnt; j++) {
|
||||
const struct ko_rela *rl =
|
||||
(const struct ko_rela *)(k->sec[i].data) + j;
|
||||
unsigned int si = KO_R_SYM(rl->r_info);
|
||||
unsigned char *insn = 0;
|
||||
unsigned long long addr, t;
|
||||
const char *nm;
|
||||
int pre, len = 0, r;
|
||||
|
||||
if ((int)si >= k->nsym || rl->r_offset < 5 ||
|
||||
rl->r_offset + 4 > tgt->size)
|
||||
continue;
|
||||
|
||||
// Only PC32. PLT32 is the displacement of a call or a jump, and the
|
||||
// bytes in front of one are whatever the previous instruction ended
|
||||
// with -- which can be 0x65, and was: a call to `eth_type_trans` read
|
||||
// as a %gs access until this line existed.
|
||||
if (KO_R_TYPE(rl->r_info) != KO_R_X86_64_PC32)
|
||||
continue;
|
||||
|
||||
// The prefix sits three to five bytes before the displacement,
|
||||
// depending on REX and on whether the opcode is two bytes. A REX byte
|
||||
// is never 0x65 and neither is 0x0f, so the longest match that decodes
|
||||
// is the right one.
|
||||
for (pre = 5; pre >= 3; pre--) {
|
||||
unsigned char *c = tgt->where + rl->r_offset - pre;
|
||||
len = gs_form(c, pre);
|
||||
if (len) { insn = c; break; }
|
||||
}
|
||||
|
||||
if (!insn) {
|
||||
// Nothing decoded. Most of the time that is because this is an
|
||||
// ordinary rip-relative reference ko_relocate already resolved and
|
||||
// there is no %gs anywhere near it.
|
||||
const unsigned char *q = tgt->where + rl->r_offset;
|
||||
if (q[-3] != 0x65 && q[-4] != 0x65 && q[-5] != 0x65)
|
||||
continue;
|
||||
// A stray 0x65 in the tail of the previous instruction looks exactly
|
||||
// like a prefix from here, so the byte alone is not evidence. Only a
|
||||
// symbol that could actually *be* per-CPU is: an import, or
|
||||
// something the module defines in .data..percpu.
|
||||
if (!percpu_symbol(k, si))
|
||||
continue;
|
||||
// Then it is one, in a shape we have not established the meaning of.
|
||||
// Naming it is the point: it is the next thing to implement.
|
||||
ko_fail(k, "unrecognised %gs instruction for ",
|
||||
k->str + k->sym[si].st_name);
|
||||
return -1;
|
||||
}
|
||||
if (rl->r_offset - pre + len > tgt->size)
|
||||
{ ko_fail(k, "%gs instruction runs off the section for ",
|
||||
k->str + k->sym[si].st_name); return -1; }
|
||||
|
||||
nm = k->str + k->sym[si].st_name;
|
||||
addr = tgt->addr + (unsigned long long)(insn - tgt->where);
|
||||
|
||||
t = percpu ? percpu(ctx, nm) : 0;
|
||||
if (!t)
|
||||
{ ko_fail(k, "no per-CPU storage for ", nm); return -1; }
|
||||
|
||||
r = percpu_one(pass ? insn : 0, pre, len, addr, t);
|
||||
if (r == -2)
|
||||
{ ko_fail(k, "per-CPU storage is too far to reach for ", nm);
|
||||
return -1; }
|
||||
if (r < 0)
|
||||
{ ko_fail(k, "cannot rewrite the per-CPU access to ", nm);
|
||||
return -1; }
|
||||
n++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
// --- what the object says about itself ---------------------------------------
|
||||
|
||||
static const struct ko_section *
|
||||
|
|
|
|||
Loading…
Reference in a new issue