SchOS/kernel/dev/pci.c

// PCI bus enumeration.
//
// Everything here is done through the two-port configuration mechanism at
// 0xCF8/0xCFC: write a (bus, slot, function, register) address to one port,
// read or write the value at the other. It is the oldest and least clever way
// to reach config space, and it is the one that works everywhere without
// first having to find an ACPI table to tell you where the memory-mapped
// version lives.
//
// The scan is recursive rather than a flat sweep of all 256 buses. A flat
// sweep does find everything on a PC, but it also probes 8192 slots that are
// not there, and it cannot tell you which bridge a device sits behind -- and
// that matters as soon as interrupt routing does.

#include "types.h"
#include "defs.h"
#include "param.h"
#include "x86.h"
#include "spinlock.h"
#include "sleeplock.h"
#include "fs.h"
#include "file.h"
#include "pci.h"
#include "sbuf.h"
#include "dbg.h"
#include "device.h"

static struct pci_dev devs[NPCI];
static int ndevs;

static void scan_bus(uint8 bus);

// ---------------------------------------------------------------------------
// Configuration space access
// ---------------------------------------------------------------------------

static uint32
cfg_addr(uint8 bus, uint8 slot, uint8 func, uint8 off)
{
  return 0x80000000u
       | ((uint32)bus  << 16)
       | ((uint32)slot << 11)
       | ((uint32)func << 8)
       | (off & 0xFC);          // the low two bits are not part of the address
}

uint32
pci_read32(uint8 bus, uint8 slot, uint8 func, uint8 off)
{
  outl(PCI_CONFIG_ADDR, cfg_addr(bus, slot, func, off));
  return inl(PCI_CONFIG_DATA);
}

void
pci_write32(uint8 bus, uint8 slot, uint8 func, uint8 off, uint32 val)
{
  outl(PCI_CONFIG_ADDR, cfg_addr(bus, slot, func, off));
  outl(PCI_CONFIG_DATA, val);
}

// Config space is only addressable in dwords, so narrower reads are a shift
// and a mask of the containing dword rather than a narrower bus cycle.
uint16
pci_read16(uint8 bus, uint8 slot, uint8 func, uint8 off)
{
  return (uint16)(pci_read32(bus, slot, func, off) >> ((off & 2) * 8));
}

uint8
pci_read8(uint8 bus, uint8 slot, uint8 func, uint8 off)
{
  return (uint8)(pci_read32(bus, slot, func, off) >> ((off & 3) * 8));
}

void
pci_enable(struct pci_dev *d, uint16 bits)
{
  uint32 cmd = pci_read32(d->bus, d->slot, d->func, PCI_COMMAND);
  pci_write32(d->bus, d->slot, d->func, PCI_COMMAND, cmd | bits);
}

// The other direction, and PCI_CMD_BUS_MASTER is the reason it exists.
//
// Clearing that bit is the only way this kernel can stop a device it does not
// understand from touching memory. Everything else about halting a device is
// device-specific -- a different register on every chip -- but bus mastering is
// in the standard config header of every PCI device ever made, and clearing it
// means the next DMA cycle does not happen.
//
// That makes it the correct thing to do when a userspace driver dies: the
// kernel is about to take its DMA buffers back, and a device still scanning out
// of them would read whatever is put there next. See kernel/dev/pciuser.c.
void
pci_disable(struct pci_dev *d, uint16 bits)
{
  uint32 cmd = pci_read32(d->bus, d->slot, d->func, PCI_COMMAND);
  pci_write32(d->bus, d->slot, d->func, PCI_COMMAND, cmd & ~(uint32)bits);
}

// ---------------------------------------------------------------------------
// Capabilities
//
// Everything a device can do beyond the 64 bytes of standard config header --
// MSI, MSI-X, power management, and the whole of virtio's configuration -- is
// described by a singly linked list of capability structures living in the
// device's config space. A driver that needs any of it has to walk that list.
// ---------------------------------------------------------------------------

// Returns the config-space offset of the first capability with this ID, or 0.
uint8
pci_find_cap(struct pci_dev *d, uint8 id)
{
  uint8 off;
  int guard;

  if(!(pci_read16(d->bus, d->slot, d->func, PCI_STATUS) & PCI_STATUS_CAPLIST))
    return 0;

  off = pci_read8(d->bus, d->slot, d->func, PCI_CAP_PTR) & 0xFC;

  // The list is built by the device, so it can be circular if the device is
  // broken or if we misread it. 48 is more capabilities than the 192 usable
  // bytes can hold; past that we are certainly looping.
  for(guard = 0; off >= 0x40 && guard < 48; guard++){
    if(pci_read8(d->bus, d->slot, d->func, off) == id)
      return off;
    off = pci_read8(d->bus, d->slot, d->func, off + 1) & 0xFC;
  }
  return 0;
}

// Point a device's MSI at a CPU and a vector.
//
// MSI is not a wire: it is a memory write the device performs to a magic
// address that the local APIC decodes as an interrupt. That makes it strictly
// better than INTx for anything modern -- no sharing, no level/edge question,
// and no chance of the interrupt arriving before the DMA it announces, since
// both are posted writes travelling the same path in order.
//
// Returns 0 on success, -1 if the device has no MSI capability.
int
pci_msi_enable(struct pci_dev *d, int vector, int cpu)
{
  uint8 cap = pci_find_cap(d, PCI_CAP_MSI);
  uint16 ctl;

  if(cap == 0)
    return -1;

  ctl = pci_read16(d->bus, d->slot, d->func, cap + 2);

  // 0xFEE00000 | (apic id << 12) is the LAPIC's message address; the data
  // word is the vector, with delivery mode 000 (fixed) and edge trigger.
  pci_write32(d->bus, d->slot, d->func, cap + 4,
              0xFEE00000u | ((uint32)cpu << 12));

  if(ctl & (1 << 7)){          // 64-bit address capable: data moved to +12
    pci_write32(d->bus, d->slot, d->func, cap + 8, 0);
    pci_write32(d->bus, d->slot, d->func, cap + 12, (uint32)vector);
  } else {
    pci_write32(d->bus, d->slot, d->func, cap + 8, (uint32)vector);
  }

  // Enable, and request exactly one vector (multiple-message-enable = 0).
  // Message control shares its dword with the capability ID and next pointer,
  // and config space is only writable a dword at a time, so read-modify-write
  // the whole thing rather than clobbering the list this capability is on.
  {
    uint32 dw = pci_read32(d->bus, d->slot, d->func, cap);
    ctl = (ctl & ~0x0070) | 1;
    dw = (dw & 0x0000FFFF) | ((uint32)ctl << 16);
    pci_write32(d->bus, d->slot, d->func, cap, dw);
  }

  // INTx must be masked or the device will signal both.
  pci_write32(d->bus, d->slot, d->func, PCI_COMMAND,
              pci_read32(d->bus, d->slot, d->func, PCI_COMMAND) | (1 << 10));
  return 0;
}

// ---------------------------------------------------------------------------
// Base address registers
// ---------------------------------------------------------------------------

// A BAR does not report its own size. You find it by writing all ones and
// reading back: the device leaves the bits it does not decode as zero, so the
// size is one more than the inverse of what comes back. The original value
// has to be put back afterwards -- this is a live device whose decoder is
// briefly pointed at the top of the address space while we look.
static void
read_bars(struct pci_dev *d)
{
  for (int i = 0; i < 6; i++) {
    uint8  off = PCI_BAR0 + i * 4;
    uint32 lo  = pci_read32(d->bus, d->slot, d->func, off);

    if (lo == 0)
      continue;

    pci_write32(d->bus, d->slot, d->func, off, 0xFFFFFFFF);
    uint32 mask = pci_read32(d->bus, d->slot, d->func, off);
    pci_write32(d->bus, d->slot, d->func, off, lo);

    if (mask == 0)
      continue;

    if (lo & 1) {                       // I/O space
      d->bar[i].is_io = 1;
      d->bar[i].base  = lo & ~0x3u;
      d->bar[i].size  = (~(mask & ~0x3u) + 1) & 0xFFFF;
      continue;
    }

    d->bar[i].prefetch = (lo >> 3) & 1;

    if (((lo >> 1) & 3) == 2) {         // 64-bit BAR: eats the next slot
      uint32 hi = pci_read32(d->bus, d->slot, d->func, off + 4);
      pci_write32(d->bus, d->slot, d->func, off + 4, 0xFFFFFFFF);
      uint32 hmask = pci_read32(d->bus, d->slot, d->func, off + 4);
      pci_write32(d->bus, d->slot, d->func, off + 4, hi);

      d->bar[i].is_64 = 1;
      d->bar[i].base  = ((uint64)hi << 32) | (lo & ~0xFu);
      d->bar[i].size  = ~(((uint64)hmask << 32) | (mask & ~0xFu)) + 1;
      i++;                              // the upper half is not its own BAR
      continue;
    }

    d->bar[i].base = lo & ~0xFu;
    d->bar[i].size = (~(mask & ~0xFu) + 1) & 0xFFFFFFFF;
  }
}

// ---------------------------------------------------------------------------
// Enumeration
// ---------------------------------------------------------------------------

static void
add_func(uint8 bus, uint8 slot, uint8 func)
{
  uint16 vendor = pci_read16(bus, slot, func, PCI_VENDOR_ID);

  if (vendor == 0xFFFF)                 // nothing plugged in
    return;

  if (ndevs >= NPCI) {
    cprintf("pci: more than %d devices, ignoring the rest\n", NPCI);
    return;
  }

  struct pci_dev *d = &devs[ndevs++];
  d->bus  = bus;
  d->slot = slot;
  d->func = func;

  d->vendor      = vendor;
  d->device      = pci_read16(bus, slot, func, PCI_DEVICE_ID);
  d->revision    = pci_read8(bus, slot, func, PCI_REVISION);
  d->prog_if     = pci_read8(bus, slot, func, PCI_PROG_IF);
  d->subclass    = pci_read8(bus, slot, func, PCI_SUBCLASS);
  d->class       = pci_read8(bus, slot, func, PCI_CLASS);
  d->header_type = pci_read8(bus, slot, func, PCI_HEADER_TYPE);
  d->irq         = pci_read8(bus, slot, func, PCI_INTERRUPT_LINE);

  // Only header type 0 has six BARs. A bridge (type 1) has two and a very
  // different layout, and reading it as though it had six would hand out
  // window registers as if they were device apertures.
  if ((d->header_type & 0x7F) == 0)
    read_bars(d);

  // Walk down through a PCI-to-PCI bridge to whatever is behind it.
  if ((d->header_type & 0x7F) == 1) {
    uint8 secondary = pci_read8(bus, slot, func, PCI_SECONDARY_BUS);
    if (secondary != 0 && secondary != bus)
      scan_bus(secondary);
  }
}

static void
scan_bus(uint8 bus)
{
  for (uint8 slot = 0; slot < 32; slot++) {
    if (pci_read16(bus, slot, 0, PCI_VENDOR_ID) == 0xFFFF)
      continue;

    add_func(bus, slot, 0);

    // Bit 7 of the header type says the device has more than one function.
    // Probing all eight unconditionally is harmless on real hardware but
    // produces phantom entries on some emulated devices, which answer every
    // function with a copy of function 0.
    if (pci_read8(bus, slot, 0, PCI_HEADER_TYPE) & 0x80)
      for (uint8 func = 1; func < 8; func++)
        add_func(bus, slot, func);
  }
}

// ---------------------------------------------------------------------------
// Lookup
// ---------------------------------------------------------------------------

int
pci_count(void)
{
  return ndevs;
}

struct pci_dev *
pci_get(int i)
{
  return (i >= 0 && i < ndevs) ? &devs[i] : 0;
}

struct pci_dev *
pci_find_class(uint8 class, uint8 subclass)
{
  for (int i = 0; i < ndevs; i++)
    if (devs[i].class == class && devs[i].subclass == subclass)
      return &devs[i];
  return 0;
}

struct pci_dev *
pci_find_id(uint16 vendor, uint16 device)
{
  for (int i = 0; i < ndevs; i++)
    if (devs[i].vendor == vendor && devs[i].device == device)
      return &devs[i];
  return 0;
}

// ---------------------------------------------------------------------------
// Driver binding
// ---------------------------------------------------------------------------

static int
matches(const struct pci_match *m, struct pci_dev *d)
{
  if (m->vendor != PCI_ANY)
    return m->vendor == d->vendor &&
           (m->device == PCI_ANY || m->device == d->device);

  // A class match, for drivers that handle a kind of device rather than a
  // particular model.
  return m->class == d->class &&
         (m->subclass == 0xFF || m->subclass == d->subclass);
}

// Offer every unclaimed device to a driver. Called by pci_init() for each
// entry in the registry; see kernel/drivers/registry.c.
// Defined further down with the /proc/pci formatting it was written for.
static const char *class_name(uint8 class, uint8 subclass);

// Translate a PCI class code into the bus-agnostic one the device model uses.
//
// The two vocabularies are deliberately separate. PCI's class byte describes
// how the card presents itself on that bus; DEVCLASS describes what the rest of
// the kernel can *do* with it, and a USB disk has to land in the same bucket as
// a SATA one despite sharing no numbering with it at all.
enum dev_class
pci_class_of(const struct pci_dev *d)
{
  switch (d->class) {
  case PCI_CLASS_DISPLAY:    return DEVCLASS_DISPLAY;
  case PCI_CLASS_STORAGE:    return DEVCLASS_STORAGE;
  case PCI_CLASS_NETWORK:    return DEVCLASS_NET;
  case PCI_CLASS_SERIAL_BUS: return DEVCLASS_BUS;
  case PCI_CLASS_MULTIMEDIA: return DEVCLASS_AUDIO;
  default:                   return DEVCLASS_OTHER;
  }
}

// Human-readable description, the same text /proc/pci prints for the device.
void
pci_describe(const struct pci_dev *d, char *out, int cap)
{
  const char *s = class_name(d->class, d->subclass);
  int n = 0;
  while (s[n] && n < cap - 1) {
    out[n] = s[n];
    n++;
  }
  out[n] = '\0';
}

void
pci_bind(const struct pci_driver *drv)
{
  for (int i = 0; i < ndevs; i++) {
    struct pci_dev *d = &devs[i];
    if (d->driver)
      continue;
    for (const struct pci_match *m = drv->match; m->vendor || m->class; m++) {
      if (!matches(m, d))
        continue;
      if (drv->probe(d) == 0) {
        d->driver = drv->name;
        d->drv    = drv;
        // Publish it in the bus-agnostic device list. Done here rather than in
        // each driver's probe() so that a driver cannot forget to, and so the
        // list is populated in bind order -- which is the order a device
        // manager should show.
        struct device *dev = device_add(drv->devname ? drv->devname : drv->name,
                                        DEVBUS_PCI, pci_class_of(d), drv->name);
        if (dev) {
          dev->ops     = (void *)drv->ops;
          dev->vendor  = d->vendor;
          dev->product = d->device;
          dev->b = d->bus; dev->s = d->slot; dev->f = d->func;
          pci_describe(d, dev->desc, sizeof(dev->desc));
          // Now, and only now, can a multi-instance driver say which of its
          // own this device is: the object it would hang that on did not
          // exist until the line above.
          if (drv->attached)
            drv->attached(d, dev);
        }
      }
      break;
    }
  }
}

// ---------------------------------------------------------------------------

// Fixed-width lowercase hex, the way lspci prints addresses.
static int
put_hex(char *out, uint64 v, int digits)
{
  static const char d[] = "0123456789abcdef";
  for (int i = digits - 1; i >= 0; i--)
    *out++ = d[(v >> (i * 4)) & 0xF];
  return digits;
}

static int
put_str(char *out, const char *s)
{
  int n = 0;
  while (s[n]) { out[n] = s[n]; n++; }
  return n;
}

static const char *
class_name(uint8 class, uint8 subclass)
{
  switch (class) {
  case PCI_CLASS_STORAGE:
    return subclass == 0x01 ? "IDE controller"
         : subclass == 0x06 ? "SATA controller"
         : subclass == 0x08 ? "NVMe controller"
                            : "storage controller";
  case PCI_CLASS_NETWORK:    return "network controller";
  case PCI_CLASS_DISPLAY:    return "display controller";
  case PCI_CLASS_MULTIMEDIA: return "multimedia device";
  case PCI_CLASS_BRIDGE:
    return subclass == 0x00 ? "host bridge"
         : subclass == 0x01 ? "ISA bridge"
         : subclass == 0x04 ? "PCI bridge"
                            : "bridge";
  case PCI_CLASS_SERIAL_BUS:
    return subclass == 0x03 ? "USB controller" : "serial bus controller";
  }
  return "device";
}

// Fills a buffer for /proc/pci. Returns bytes written.
//
// There is no snprintf in the kernel, so this appends by hand in the same
// style as kernel/dev/procfs.c. The output is deliberately lspci-shaped --
// "bus:slot.func vendor:device description" -- because that is the format
// anyone reading it will already know how to scan.
int
pci_info(char *buf, int cap)
{
  int len = 0;

  for (int i = 0; i < ndevs; i++) {
    struct pci_dev *d = &devs[i];

    if (len > cap - 96)                 // room for one more line
      break;

    len += put_hex(buf + len, d->bus, 2);
    buf[len++] = ':';
    len += put_hex(buf + len, d->slot, 2);
    buf[len++] = '.';
    len += put_hex(buf + len, d->func, 1);
    buf[len++] = ' ';
    len += put_hex(buf + len, d->vendor, 4);
    buf[len++] = ':';
    len += put_hex(buf + len, d->device, 4);
    buf[len++] = ' ';
    len += put_str(buf + len, class_name(d->class, d->subclass));
    len += put_str(buf + len, "  [");
    len += put_str(buf + len, d->driver ? d->driver : "-");
    buf[len++] = ']';
    buf[len++] = '\n';
  }

  return len;
}

// The same lines /proc/pci carries, plus the command register, which is the
// bit that says whether a device can still reach memory. pciuser.c clears bus
// mastering when a userspace driver dies -- see "Taking it back" in
// docs/architecture/devices.md -- and "did that actually happen" is a question
// worth being able to ask without a userspace program.
static void
pci_dump(struct sbuf *out, const char *arg)
{
  (void)arg;
  sb_puts(out, "  bdf       vendor:device  class                    "
               "cmd   bm  io  mem  driver\n");

  for (int i = 0; i < ndevs; i++) {
    struct pci_dev *d = &devs[i];
    uint32 cmd = pci_read32(d->bus, d->slot, d->func, PCI_COMMAND) & 0xFFFF;

    sb_puts(out, "  ");
    sb_puthex(out, d->bus, 2);  sb_putc(out, ':');
    sb_puthex(out, d->slot, 2); sb_putc(out, '.');
    sb_puthex(out, d->func, 1);
    sb_puts(out, "   ");
    sb_puthex4(out, d->vendor); sb_putc(out, ':');
    sb_puthex4(out, d->device);
    sb_puts(out, "    ");
    sb_pad(out, class_name(d->class, d->subclass), 25);
    sb_puthex4(out, cmd);
    sb_puts(out, (cmd & PCI_CMD_BUS_MASTER) ? "  on " : "  -- ");
    sb_puts(out, (cmd & PCI_CMD_IO)         ? "  on " : "  -- ");
    sb_puts(out, (cmd & PCI_CMD_MEMORY)     ? "  on " : "  -- ");
    sb_puts(out, " ");
    {
      int pid = pciuser_owner_pid(i);
      if (pid) {
        sb_puts(out, "userspace pid ");
        sb_putu(out, pid);
      } else {
        sb_puts(out, d->driver ? d->driver : "-");
      }
    }
    sb_putc(out, '\n');
  }
}

void
pci_init(void)
{
  ndevs = 0;
  dbg_register("pci", pci_dump, "the bus, with each device's command register");

  // If there is no host bridge at 0:0.0 there is no PCI, and every config
  // read will come back 0xFFFFFFFF. Saying so once beats 8192 silent probes.
  if (pci_read16(0, 0, 0, PCI_VENDOR_ID) == 0xFFFF) {
    cprintf("pci: no host bridge; bus not present\n");
    return;
  }

  scan_bus(0);
  kinfo("pci: %d device%s\n", ndevs, ndevs == 1 ? "" : "s");

  pci_bind_all();
}