SchOS/kernel/drivers/bus/xhci/xhci.c

// xHCI host controller driver.
//
// This is written from the xHCI 1.2 specification rather than ported from
// FreeBSD or NetBSD. That was a deliberate choice and worth recording,
// because "take the USB stack from BSD" is the obvious plan and it does not
// survive contact with the code: sys/dev/usb is welded to newbus, devclass,
// bus_dma, callouts, taskqueues and FreeBSD's locking primitives, none of
// which exist here. Porting it means porting FreeBSD's device framework and
// then discovering the USB stack underneath. The specification is public and
// the controller is not complicated; BSD is much more valuable as the
// reference you check yourself against than as the code you paste.
//
// (Licensing points the same way. The BSD stacks are BSD-2/3-clause, which is
// compatible with Apache-2.0 -- Linux's drivers are GPL-2.0-only, which is
// not. See docs/guides/adding-a-driver.md.)
//
// Enumeration polls the event ring instead of using interrupts. That is not
// laziness: pci_bind_all() runs from kmain() with interrupts disabled and no
// scheduler, so a driver that waited for an interrupt during probe would wait
// forever. Interrupts are for later, when a device is delivering data.

#include "types.h"
#include "defs.h"
#include "irq.h"
#include "dma.h"
#include "param.h"
#include "memlayout.h"
#include "mmu.h"
#include "spinlock.h"
#include "sleeplock.h"
#include "fs.h"
#include "file.h"
#include "pci.h"
#include "usb.h"
#include "xhci.h"

#define RING_TRBS   64                  // TRBs per ring, including the link
#define RING_BYTES  (RING_TRBS * 16)
#define MAX_SLOTS   16                  // device slots we are willing to use

// A producer ring: we write TRBs, the controller reads them. The last entry
// is a Link TRB pointing back at the start, which is how a fixed-size buffer
// becomes an endless queue.
struct ring {
  volatile struct trb *trb;
  addr_t phys;
  int    enq;                           // next index to write
  int    cycle;                         // cycle bit we are currently producing
};

// The event ring is the other direction: the controller writes, we read. It
// has no link TRB -- it wraps at the end of the segment and inverts the cycle
// bit, and the segment is described to the controller by a separate table.
struct event_ring {
  volatile struct trb *trb;
  addr_t phys;
  void  *erst;
  addr_t erst_phys;
  int    deq;
  int    cycle;
};

#define MAX_EPS     4                   // endpoints beyond EP0, per device

// One non-control endpoint. `buf` stays allocated for the life of the
// endpoint and is re-queued after every completion, because an interrupt
// endpoint is a standing request rather than a one-off: the device has
// nothing to say until it does, and there must already be a TRB waiting when
// that happens.
struct xhci_ep {
  struct ring ring;
  int    dci;
  int    addr;                          // bEndpointAddress
  int    type;                          // USB_EP_INTERRUPT or USB_EP_BULK
  void  *buf;
  int    len;
  int    open;

  // Completion state, used only by bulk transfers. An interrupt endpoint
  // never waits -- its whole point is that the report arrives when it
  // arrives -- but a bulk transfer is a caller sitting in a read() who has to
  // be put to sleep and woken when the data lands.
  volatile int done;
  int    cc;
  int    residue;
};

struct xhci {
  struct pci_dev *pci;
  volatile uchar *cap;                  // BAR0
  volatile uchar *op;                   // cap + CAPLENGTH
  volatile uchar *rt;                   // cap + RTSOFF
  volatile uint32 *db;                  // cap + DBOFF

  int maxslots, maxports, ctxsize;

  uint64 *dcbaa;
  addr_t  dcbaa_phys;

  struct ring cmd;
  struct event_ring ev;

  // Guards the bulk completion handshake between a sleeping caller and the
  // interrupt handler. Deliberately not held across the interrupt-endpoint
  // callback path, which reaches into the console.
  struct spinlock lock;

  // Set once probing is over. Until then there is no scheduler and no
  // process table -- sleep() would have nothing to switch to and wakeup()
  // would take a lock pinit() has not initialised yet -- so a bulk transfer
  // has to poll instead. See xhci_bulk().
  int can_sleep;

  // Per-slot state. Slot 0 is not a device slot; index by slot ID directly.
  struct {
    void  *devctx;
    addr_t devctx_phys;
    struct ring ep0;
    struct xhci_ep ep[MAX_EPS];
    int    nep;
    int    max_dci;                     // highest DCI configured so far
    int    port;
    int    speed;
    struct usb_dev dev;
  } slot[MAX_SLOTS + 1];

  int ndevices;
};

static void xhci_attach(struct usb_dev *hub, int hub_port, int speed);

static struct xhci xhcis[2];
static int nxhci;

static void transfer_done(struct xhci *x, struct trb *ev);

// ---------------------------------------------------------------------------
// Register access
//
// BAR0 is device memory, so every access has to actually happen -- hence
// volatile. It reaches the CPU uncached because the kernel's page tables mark
// the whole 3-4 GB range PCD|PWT (kvmalloc in vm.c), which is where a PC's
// PCI apertures live. A BAR above 4 GB would need a mapping of its own; we
// check for that rather than silently reading nonsense.
// ---------------------------------------------------------------------------

static inline uint32 rd32(volatile uchar *p, int off)
{ return *(volatile uint32*)(p + off); }

static inline void wr32(volatile uchar *p, int off, uint32 v)
{ *(volatile uint32*)(p + off) = v; }

static inline uint64 rd64(volatile uchar *p, int off)
{ return *(volatile uint64*)(p + off); }

static inline void wr64(volatile uchar *p, int off, uint64 v)
{ *(volatile uint64*)(p + off) = v; }

// ---------------------------------------------------------------------------
// Rings
// ---------------------------------------------------------------------------

static int
ring_init(struct ring *r)
{
  volatile struct trb *link;

  r->trb = dma_alloc(RING_BYTES, 4096);
  if(r->trb == 0)
    return -1;
  r->phys  = dma_phys((void*)r->trb);
  r->enq   = 0;
  r->cycle = 1;

  // The link TRB's Toggle Cycle bit is what makes the ring work: when the
  // controller follows it back to the start it flips the cycle bit it expects,
  // so entries we have not written yet still look stale to it.
  link = &r->trb[RING_TRBS - 1];
  link->param   = r->phys;
  link->status  = 0;
  link->control = TRB_TYPE(TRB_LINK) | TRB_TC;
  return 0;
}

// Write one TRB and advance. The cycle bit is set *last*, because it is what
// tells the controller the entry is ready -- a controller reading the ring
// concurrently must never see a valid cycle bit over a half-written TRB.
static void
ring_push(struct ring *r, uint64 param, uint32 status, uint32 control)
{
  volatile struct trb *t = &r->trb[r->enq];

  t->param  = param;
  t->status = status;
  __sync_synchronize();
  t->control = (control & ~TRB_C) | (r->cycle ? TRB_C : 0);
  __sync_synchronize();

  r->enq++;
  if(r->enq == RING_TRBS - 1){
    // Hand the link TRB the current cycle too, then wrap and invert.
    volatile struct trb *link = &r->trb[RING_TRBS - 1];
    link->control = TRB_TYPE(TRB_LINK) | TRB_TC | (r->cycle ? TRB_C : 0);
    __sync_synchronize();
    r->enq = 0;
    r->cycle ^= 1;
  }
}

static int
event_ring_init(struct xhci *x)
{
  uint64 *erst;

  x->ev.trb = dma_alloc(RING_BYTES, 4096);
  if(x->ev.trb == 0)
    return -1;
  x->ev.phys  = dma_phys((void*)x->ev.trb);
  x->ev.deq   = 0;
  x->ev.cycle = 1;

  // The Event Ring Segment Table: one 16-byte entry per segment. One segment
  // is plenty here.
  erst = dma_alloc(64, 64);
  if(erst == 0)
    return -1;
  erst[0] = x->ev.phys;
  erst[1] = RING_TRBS;                  // size in TRBs, high half reserved
  x->ev.erst      = erst;
  x->ev.erst_phys = dma_phys(erst);
  return 0;
}

// Pull the next event, if the controller has produced one. Returns 1 and
// copies it out, or 0 if the ring is empty.
//
// "Empty" is decided by the cycle bit, not by a pointer comparison: the
// controller does not tell us where it stopped writing, it just leaves the
// cycle bit of every entry it has produced matching the value we expect.
static int
event_pop(struct xhci *x, struct trb *out)
{
  volatile struct trb *e = &x->ev.trb[x->ev.deq];
  uint32 ctrl = e->control;

  if(((ctrl & TRB_C) ? 1 : 0) != x->ev.cycle)
    return 0;

  __sync_synchronize();
  out->param   = e->param;
  out->status  = e->status;
  out->control = ctrl;

  x->ev.deq++;
  if(x->ev.deq == RING_TRBS){
    x->ev.deq = 0;
    x->ev.cycle ^= 1;
  }

  // Tell the controller how far we have read, and clear Event Handler Busy.
  wr64(x->rt, XHCI_IR0 + IR_ERDP,
       (x->ev.phys + x->ev.deq * 16) | ERDP_EHB);
  return 1;
}

// Wait for an event of a given type. Returns 0 on success.
static int
event_wait(struct xhci *x, int type, struct trb *out, int ms)
{
  int i;

  for(i = 0; i < ms * 1000; i++){
    while(event_pop(x, out)){
      int t = TRB_GET_TYPE(out->control);
      if(t == type)
        return 0;
      // Port status changes arrive unbidden throughout enumeration. They are
      // not errors and must not be mistaken for the event being waited on.
      if(t != TRB_PORT_STATUS)
        cprintf("xhci: unexpected event type %d while waiting for %d\n",
                t, type);
    }
    microdelay(1);
  }
  return -1;
}

// Submit a command and wait for its completion event. Returns the completion
// code, or -1 on timeout.
static int
command(struct xhci *x, uint64 param, uint32 status, uint32 control,
        struct trb *result)
{
  struct trb ev;

  ring_push(&x->cmd, param, status, control);
  x->db[0] = 0;                         // doorbell 0 = command ring
  __sync_synchronize();

  if(event_wait(x, TRB_CMD_COMPLETE, &ev, 1000) < 0)
    return -1;
  if(result)
    *result = ev;
  return TRB_CC(ev.status);
}

// ---------------------------------------------------------------------------
// Device and input contexts
//
// A context is the controller's own record of a device or an endpoint, living
// in memory it reads directly. Their size is 32 or 64 bytes depending on the
// HCCPARAMS1.CSZ bit -- an easy thing to hardcode and a miserable thing to
// debug, because a 32-byte assumption on a 64-byte controller puts every
// field at half the right offset and the controller simply rejects commands
// with no explanation.
// ---------------------------------------------------------------------------

static inline uint32 *
ctx_at(struct xhci *x, void *base, int index)
{
  return (uint32*)((uchar*)base + index * x->ctxsize);
}

// Default max packet size for endpoint 0, by speed. Full-speed devices are
// the awkward case: theirs may be 8, 16, 32 or 64 and the only way to find
// out is to ask the device -- which requires a working control endpoint. The
// standard answer is to start at 8, read the first 8 bytes of the device
// descriptor, and correct it.
static int
default_mps0(int speed)
{
  switch(speed){
  case XSPEED_LOW:   return 8;
  case XSPEED_FULL:  return 8;
  case XSPEED_HIGH:  return 64;
  case XSPEED_SUPER: return 512;
  default:           return 8;
  }
}

static const char *
speed_name(int speed)
{
  switch(speed){
  case XSPEED_LOW:   return "low (1.5 Mb/s)";
  case XSPEED_FULL:  return "full (12 Mb/s)";
  case XSPEED_HIGH:  return "high (480 Mb/s)";
  case XSPEED_SUPER: return "super (5 Gb/s)";
  default:           return "unknown";
  }
}

// ---------------------------------------------------------------------------
// Control transfers
// ---------------------------------------------------------------------------

// A control transfer is three stages on the endpoint ring: Setup, an optional
// Data stage, and Status. Only the last carries IOC, so one transfer produces
// one event.
static int
control_xfer(struct xhci *x, int slot, struct usb_setup *s, void *buf, int len)
{
  struct ring *r = &x->slot[slot].ep0;
  struct trb ev;
  addr_t bufphys = 0;
  int in = (s->bmRequestType & USB_DIR_IN) != 0;
  uint64 setup;

  if(len > 0){
    if(buf == 0)
      return -1;
    bufphys = dma_phys(buf);
  }

  // The setup packet travels inside the TRB itself (Immediate Data) rather
  // than by pointer -- it is exactly eight bytes, which is what the parameter
  // field holds.
  setup = *(uint64*)s;
  ring_push(r, setup, 8,
            TRB_TYPE(TRB_SETUP) | TRB_IDT |
            ((uint32)(len ? (in ? TRT_IN : TRT_OUT) : TRT_NONE) << 16));

  if(len > 0)
    ring_push(r, bufphys, len,
              TRB_TYPE(TRB_DATA) | (in ? DIR_IN : 0));

  // The status stage runs in the opposite direction to the data stage, which
  // is how the device acknowledges. With no data stage it is always IN.
  ring_push(r, 0, 0,
            TRB_TYPE(TRB_STATUS) | TRB_IOC |
            ((len == 0 || !in) ? DIR_IN : 0));

  x->db[slot] = DCI_EP0;
  __sync_synchronize();

  if(event_wait(x, TRB_XFER_EVENT, &ev, 1000) < 0){
    cprintf("xhci: control transfer timed out (slot %d, request 0x%x)\n",
            slot, s->bRequest);
    return -1;
  }

  // A short packet is not a failure: it is the device saying "that is all I
  // have", which is the normal reply to asking for more than it holds.
  int cc = TRB_CC(ev.status);
  if(cc != CC_SUCCESS && cc != CC_SHORT_PACKET){
    cprintf("xhci: control transfer failed, completion code %d\n", cc);
    return -1;
  }
  return 0;
}

static int
get_descriptor(struct xhci *x, int slot, int type, int index, void *buf,
               int len)
{
  struct usb_setup s;

  s.bmRequestType = USB_DIR_IN | USB_TYPE_STANDARD | USB_RECIP_DEVICE;
  s.bRequest      = USB_REQ_GET_DESCRIPTOR;
  s.wValue        = (type << 8) | index;
  s.wIndex        = 0;
  s.wLength       = len;
  return control_xfer(x, slot, &s, buf, len);
}

// ---------------------------------------------------------------------------
// Enumeration
// ---------------------------------------------------------------------------

// Build the input context for Address Device and issue it. The input context
// is a request, not a record: the Input Control Context's add-flags say which
// of the following contexts the controller should actually take.
static int
address_device(struct xhci *x, int slot, struct usb_dev *dev)
{
  void *input;
  uint32 *ctrl, *sc, *ep0;
  int cc;

  // Input context = control context + slot context + endpoint contexts, so it
  // is one context larger than the device context it describes.
  input = dma_alloc(x->ctxsize * 3, 64);
  if(input == 0)
    return -1;

  ctrl = ctx_at(x, input, 0);
  sc   = ctx_at(x, input, 1);
  ep0  = ctx_at(x, input, 2);

  ctrl[0] = 0;                          // drop nothing
  ctrl[1] = 0x3;                        // add slot context and EP0

  // Route string plus speed, and one valid endpoint context. The route is
  // what lets the controller find a device several hubs deep: the root port
  // number says which tree, the route says which branch of it.
  sc[0] = (dev->route & 0xFFFFF) | ((uint32)dev->speed << 20) | (1u << 27);
  sc[1] = (uint32)dev->port << 16;

  // A full or low speed device behind a high-speed hub is reached through
  // that hub's transaction translator, and the controller has to be told
  // which one -- it schedules against the hub, not the device.
  if(dev->tt_slot)
    sc[2] = (uint32)dev->tt_slot | ((uint32)dev->tt_port << 8);
  else
    sc[2] = 0;
  sc[3] = 0;

  // Control endpoint, three retries before giving up, dequeue pointer at the
  // start of the ring with the cycle state we are producing.
  ep0[0] = 0;
  ep0[1] = (EPTYPE_CONTROL << 3) | (3 << 1) |
           ((uint32)default_mps0(dev->speed) << 16);
  *(uint64*)&ep0[2] = x->slot[slot].ep0.phys | 1;
  ep0[4] = 8;                           // average TRB length

  cc = command(x, dma_phys(input), 0,
               TRB_TYPE(TRB_ADDRESS_DEV) | TRB_SLOT(slot), 0);

  dma_free(input, x->ctxsize * 3);

  if(cc != CC_SUCCESS){
    cprintf("xhci: Address Device failed on port %d route %x, code %d\n",
            dev->port, dev->route, cc);
    return -1;
  }
  return 0;
}

// Mark a slot as a hub. Must happen before anything behind it is addressed.
static int
xhci_set_hub(struct usb_dev *dev, int nports, int mtt)
{
  struct xhci *x = dev->hc;
  void *input;
  uint32 *ctrl, *sc;
  int cc;

  input = dma_alloc(x->ctxsize * 3, 64);
  if(input == 0)
    return -1;

  ctrl = ctx_at(x, input, 0);
  sc   = ctx_at(x, input, 1);

  ctrl[0] = 0;
  ctrl[1] = 1;                          // slot context only

  sc[0] = (dev->route & 0xFFFFF) | ((uint32)dev->speed << 20) |
          ((uint32)x->slot[dev->slot].max_dci << 27) |
          (1u << 26) | (mtt ? (1u << 25) : 0);      // Hub, Multi-TT
  sc[1] = ((uint32)dev->port << 16) | ((uint32)nports << 24);
  sc[2] = dev->tt_slot ? ((uint32)dev->tt_slot | ((uint32)dev->tt_port << 8))
                       : 0;

  cc = command(x, dma_phys(input), 0,
               TRB_TYPE(TRB_EVAL_CTX) | TRB_SLOT(dev->slot), 0);
  dma_free(input, x->ctxsize * 3);

  if(cc != CC_SUCCESS){
    cprintf("xhci: could not mark slot %d as a hub, code %d\n",
            dev->slot, cc);
    return -1;
  }
  return 0;
}

// Correct EP0's max packet size once the device has told us what it really is.
static int
set_mps0(struct xhci *x, int slot, int mps)
{
  void *input;
  uint32 *ctrl, *ep0;
  int cc;

  input = dma_alloc(x->ctxsize * 3, 64);
  if(input == 0)
    return -1;

  ctrl = ctx_at(x, input, 0);
  ep0  = ctx_at(x, input, 2);
  ctrl[0] = 0;
  ctrl[1] = 0x2;                        // EP0 only
  ep0[1] = (EPTYPE_CONTROL << 3) | (3 << 1) | ((uint32)mps << 16);
  *(uint64*)&ep0[2] = x->slot[slot].ep0.phys | 1;
  ep0[4] = 8;

  cc = command(x, dma_phys(input), 0,
               TRB_TYPE(TRB_EVAL_CTX) | TRB_SLOT(slot), 0);
  dma_free(input, x->ctxsize * 3);
  return cc == CC_SUCCESS ? 0 : -1;
}

// ---------------------------------------------------------------------------
// Non-control endpoints
// ---------------------------------------------------------------------------

// Device Context Index. Endpoint 0 is DCI 1; after that each endpoint number
// has two, OUT then IN. This packing is why an endpoint context array has 31
// entries for 15 endpoints.
static int
endpoint_dci(int addr)
{
  return USB_EP_NUM(addr) * 2 + (USB_EP_IS_IN(addr) ? 1 : 0);
}

// xHCI wants the polling interval as a power-of-two number of 125 us
// microframes; the descriptor gives it in two different units depending on
// speed. High and super speed already report an exponent. Full and low speed
// report whole 1 ms frames, and a frame is eight microframes, hence the +3.
static int
endpoint_interval(int speed, const struct usb_endpoint_desc *ep)
{
  int i;

  if(speed == XSPEED_HIGH || speed == XSPEED_SUPER){
    i = ep->bInterval - 1;
  } else {
    int frames = ep->bInterval ? ep->bInterval : 1;
    for(i = 0; (1 << (i + 1)) <= frames; i++)
      ;
    i += 3;
  }
  if(i < 0) i = 0;
  if(i > 15) i = 15;
  return i;
}

static int
xhci_open_ep(struct usb_dev *dev, const struct usb_endpoint_desc *epd)
{
  struct xhci *x = dev->hc;
  int slot = dev->slot;
  struct xhci_ep *ep;
  void *input;
  uint32 *ctrl, *sc, *ec;
  int dci, cc, type, mps;

  int xfer = USB_EP_XFERTYPE(epd->bmAttributes);

  if(x->slot[slot].nep >= MAX_EPS)
    return -1;
  if(xfer != USB_EP_INTERRUPT && xfer != USB_EP_BULK){
    cprintf("xhci: endpoint 0x%x is neither interrupt nor bulk\n",
            epd->bEndpointAddress);
    return -1;
  }

  ep  = &x->slot[slot].ep[x->slot[slot].nep];
  dci = endpoint_dci(epd->bEndpointAddress);
  mps = epd->wMaxPacketSize & 0x7FF;

  if(ring_init(&ep->ring) < 0)
    return -1;
  ep->dci  = dci;
  ep->addr = epd->bEndpointAddress;
  ep->type = xfer;
  ep->len  = mps;

  // An interrupt endpoint owns a standing buffer, because it is re-armed
  // immediately after every report and there is nobody to supply one. A bulk
  // endpoint is driven by a caller who brings their own.
  if(xfer == USB_EP_INTERRUPT){
    ep->buf = dma_alloc(mps, 64);
    if(ep->buf == 0)
      return -1;
  }

  // An input context has to be big enough to hold every context up to the one
  // being added: control, slot, then one per DCI.
  input = dma_alloc(x->ctxsize * 33, 64);
  if(input == 0)
    return -1;

  ctrl = ctx_at(x, input, 0);
  sc   = ctx_at(x, input, 1);
  ec   = ctx_at(x, input, dci + 1);

  // The slot context has to be in the add list as well as the endpoint: its
  // Context Entries field says how far the device context extends, and the
  // controller will not look past it. Adding an endpoint without raising it
  // configures an endpoint the controller then refuses to see.
  ctrl[0] = 0;
  ctrl[1] = 1 | (1u << dci);

  // Context Entries must be the *highest* DCI in use, not the one being
  // added. Writing the new endpoint's own DCI works only while endpoints
  // happen to be opened in increasing order -- open a bulk IN (DCI 3) and
  // then something at DCI 2 and the field shrinks, silently invalidating the
  // endpoint that was already working. Mass storage got away with it because
  // its IN endpoint happens to come first.
  if(dci > x->slot[slot].max_dci)
    x->slot[slot].max_dci = dci;

  sc[0] = ((uint32)dev->speed << 20) | ((uint32)x->slot[slot].max_dci << 27);
  sc[1] = (uint32)dev->port << 16;

  if(xfer == USB_EP_INTERRUPT)
    type = EPTYPE_INT_IN;
  else
    type = USB_EP_IS_IN(epd->bEndpointAddress) ? EPTYPE_BULK_IN
                                               : EPTYPE_BULK_OUT;

  // Bulk has no service interval -- it is "whenever there is bus time left",
  // which is exactly why it is the right transfer type for a disk and the
  // wrong one for a keyboard.
  ec[0] = (xfer == USB_EP_INTERRUPT)
        ? ((uint32)endpoint_interval(dev->speed, epd) << 16) : 0;
  ec[1] = ((uint32)type << 3) | (3 << 1) | ((uint32)mps << 16);
  *(uint64*)&ec[2] = ep->ring.phys | 1;

  // Max ESIT Payload is how much the endpoint may move per service interval,
  // which only means anything for a periodic endpoint. Bulk has no service
  // interval, so it must be left zero rather than filled in with the packet
  // size for symmetry.
  ec[4] = (uint32)mps;
  if(xfer == USB_EP_INTERRUPT)
    ec[4] |= (uint32)mps << 16;

  cc = command(x, dma_phys(input), 0,
               TRB_TYPE(TRB_CONFIG_EP) | TRB_SLOT(slot), 0);
  dma_free(input, x->ctxsize * 33);

  if(cc != CC_SUCCESS){
    cprintf("xhci: Configure Endpoint failed for ep 0x%x, code %d\n",
            epd->bEndpointAddress, cc);
    return -1;
  }

  ep->open = 1;
  x->slot[slot].nep++;
  return 0;
}

static struct xhci_ep *
find_ep(struct xhci *x, int slot, int dci)
{
  int i;

  for(i = 0; i < x->slot[slot].nep; i++)
    if(x->slot[slot].ep[i].dci == dci)
      return &x->slot[slot].ep[i];
  return 0;
}

// Caller holds x->lock, or is still probing and single-threaded.
static int
queue_in_locked(struct usb_dev *dev, int ep_addr, void *buf, int len)
{
  struct xhci *x = dev->hc;
  struct xhci_ep *ep = find_ep(x, dev->slot, endpoint_dci(ep_addr));

  if(ep == 0 || !ep->open)
    return -1;
  if(buf == 0){
    buf = ep->buf;
    len = ep->len;
  }

  // ISP asks for an event even when the device sends less than a full packet,
  // which for an 8-byte HID report on an 8-byte endpoint is every time it has
  // nothing new to add.
  ring_push(&ep->ring, dma_phys(buf), len, TRB_TYPE(TRB_NORMAL) | TRB_IOC | TRB_ISP);
  x->db[dev->slot] = ep->dci;
  __sync_synchronize();
  return 0;
}

static int
xhci_queue_in(struct usb_dev *dev, int ep_addr, void *buf, int len)
{
  struct xhci *x = dev->hc;
  int rc;

  acquire(&x->lock);
  rc = queue_in_locked(dev, ep_addr, buf, len);
  release(&x->lock);
  return rc;
}

static int
xhci_control(struct usb_dev *dev, struct usb_setup *s, void *buf, int len)
{
  return control_xfer(dev->hc, dev->slot, s, buf, len);
}

// One bulk transfer, synchronously.
//
// Unlike everything else here this runs in process context, from somewhere
// under bread(), so it can and must sleep rather than spin -- a disk read
// that busy-waited would burn a CPU for the duration and, on a single-core
// machine, prevent the very interrupt it is waiting for from being handled.
//
// Returns the number of bytes transferred, or -1.
static int
xhci_bulk(struct usb_dev *dev, int ep_addr, void *buf, int len)
{
  struct xhci *x = dev->hc;
  struct xhci_ep *ep = find_ep(x, dev->slot, endpoint_dci(ep_addr));
  int in = USB_EP_IS_IN(ep_addr);
  int rc, cc;

  if(ep == 0 || !ep->open)
    return -1;

  acquire(&x->lock);
  ep->done = 0;
  ep->cc = 0;
  ep->residue = 0;

  ring_push(&ep->ring, dma_phys(buf), len,
            TRB_TYPE(TRB_NORMAL) | TRB_IOC | (in ? TRB_ISP : 0));
  x->db[dev->slot] = ep->dci;
  __sync_synchronize();

  if(!x->can_sleep){
    // Still probing. This is the same constraint that makes enumeration poll
    // the event ring: pci_bind_all() runs from kmain() with interrupts off,
    // before pinit(), so there is no scheduler to sleep into and no process
    // table to be woken from. Sleeping here does not fail loudly -- it wedges
    // the machine mid-enumeration with no message at all.
    int i;
    for(i = 0; i < 5000000 && !ep->done; i++){
      struct trb ev;
      while(event_pop(x, &ev))
        if(TRB_GET_TYPE(ev.control) == TRB_XFER_EVENT)
          transfer_done(x, &ev);
      release(&x->lock);
      microdelay(1);
      acquire(&x->lock);
    }
    if(!ep->done){
      release(&x->lock);
      cprintf("xhci: bulk transfer on ep 0x%x timed out\n", ep_addr);
      return -1;
    }
  } else {
    while(!ep->done)
      sleep(ep, &x->lock);
  }

  cc = ep->cc;
  rc = (cc == CC_SUCCESS || cc == CC_SHORT_PACKET) ? len - ep->residue : -1;
  release(&x->lock);

  // Read out of ep before releasing: once the lock is gone another transfer
  // on the same endpoint may already have overwritten the result.
  if(rc < 0)
    cprintf("xhci: bulk transfer on ep 0x%x failed, code %d\n", ep_addr, cc);
  return rc;
}

static const struct usb_hc_ops xhci_ops = {
  .control  = xhci_control,
  .open_ep  = xhci_open_ep,
  .queue_in = xhci_queue_in,
  .bulk     = xhci_bulk,
  .attach   = xhci_attach,
  .set_hub  = xhci_set_hub,
};

static const char *
class_name(int cls)
{
  switch(cls){
  case USB_CLASS_HID:          return "HID";
  case USB_CLASS_MASS_STORAGE: return "mass storage";
  case USB_CLASS_HUB:          return "hub";
  case 0:                      return "per-interface";
  default:                     return "other";
  }
}

// Bring up whatever is plugged into one root port.
// Bring up one device, wherever it is in the tree.
//
// `proto` carries everything the caller already knows about where the device
// sits: root port, route string, tier, speed, and which transaction
// translator serves it. Everything past that is the same whether it is
// plugged into the machine or four hubs away, which is the point of splitting
// it out of the root-port scan.
static void
attach_device(struct xhci *x, const struct usb_dev *proto)
{
  struct trb ev;
  struct usb_device_desc *desc;
  struct usb_dev *dev;
  int slot, cc;
  int speed = proto->speed;

  cc = command(x, 0, 0, TRB_TYPE(TRB_ENABLE_SLOT), &ev);
  if(cc != CC_SUCCESS){
    cprintf("xhci: Enable Slot failed on port %d, completion code %d\n",
            proto->port, cc);
    return;
  }
  slot = TRB_GET_SLOT(ev.control);
  if(slot <= 0 || slot > MAX_SLOTS){
    cprintf("xhci: controller returned slot %d, out of range\n", slot);
    return;
  }

  // The device context is the controller's; we allocate it and hand over the
  // address, and from then on it writes there and we only read.
  x->slot[slot].devctx = dma_alloc(x->ctxsize * 32, 64);
  if(x->slot[slot].devctx == 0)
    return;
  x->slot[slot].devctx_phys = dma_phys(x->slot[slot].devctx);
  x->dcbaa[slot] = x->slot[slot].devctx_phys;

  if(ring_init(&x->slot[slot].ep0) < 0)
    return;

  x->slot[slot].port    = proto->port;
  x->slot[slot].speed   = speed;
  x->slot[slot].max_dci = DCI_EP0;      // EP0 exists from Address Device on
  x->slot[slot].nep     = 0;

  dev = &x->slot[slot].dev;
  *dev = *proto;
  dev->ops    = &xhci_ops;
  dev->hc     = x;
  dev->slot   = slot;
  dev->in_use = 1;

  if(address_device(x, slot, dev) < 0)
    return;

  desc = dma_alloc(sizeof(*desc), 64);
  if(desc == 0)
    return;

  // Ask for eight bytes first. That is all a full-speed device is guaranteed
  // to manage in one packet, and byte 7 is the max packet size that tells us
  // how much it can manage from here on.
  if(get_descriptor(x, slot, USB_DT_DEVICE, 0, desc, 8) < 0){
    dma_free(desc, sizeof(*desc));
    return;
  }

  if(desc->bMaxPacketSize0 != default_mps0(speed) &&
     desc->bMaxPacketSize0 >= 8)
    set_mps0(x, slot, desc->bMaxPacketSize0);

  if(get_descriptor(x, slot, USB_DT_DEVICE, 0, desc, sizeof(*desc)) < 0){
    dma_free(desc, sizeof(*desc));
    return;
  }

  // cprintf understands %d %u %l %x %p %b %s and nothing else -- no field
  // widths. "%04x" is not a directive to it: it prints '%' and '0' literally,
  // consumes no argument, and every argument after it lands in the wrong
  // place. That is what turned this line into rubbish, not the enumeration.
  cprintf("usb: port %d%s slot %d %x:%x %s, %s speed, %d configuration%s\n",
          proto->port, proto->route ? " (via hub)" : "", slot,
          desc->idVendor, desc->idProduct,
          class_name(desc->bDeviceClass), speed_name(speed),
          desc->bNumConfigurations,
          desc->bNumConfigurations == 1 ? "" : "s");

  x->ndevices++;
  dev->desc = *desc;
  dma_free(desc, sizeof(*desc));

  // Hand it to the core, which knows about configurations and class drivers
  // and nothing about TRBs.
  usb_attach(dev);
}

// A device found on a downstream port of a hub.
static void
xhci_attach(struct usb_dev *hub, int hub_port, int speed)
{
  struct xhci *x = hub->hc;
  struct usb_dev proto;

  memset(&proto, 0, sizeof(proto));
  proto.port  = hub->port;              // same root port, deeper in the tree
  proto.speed = speed;
  proto.tier  = hub->tier + 1;

  // Each tier contributes one nibble. Five tiers is the limit the route
  // string can express, and also the limit USB itself allows.
  if(proto.tier > 5){
    cprintf("usb: device too deep in the hub tree to address\n");
    return;
  }
  proto.route = hub->route |
                (((uint32)hub_port & 0xF) << (4 * (proto.tier - 1)));

  // If this device is slower than high speed, it is reached through a
  // transaction translator. That is the hub's, unless the hub is itself
  // behind one -- in which case the one further up is still doing the work.
  if(speed == XSPEED_LOW || speed == XSPEED_FULL){
    if(hub->speed == XSPEED_HIGH){
      proto.tt_slot = hub->slot;
      proto.tt_port = hub_port;
    } else {
      proto.tt_slot = hub->tt_slot;
      proto.tt_port = hub->tt_port;
    }
  }

  attach_device(x, &proto);
}

static void
enumerate_port(struct xhci *x, int port)
{
  uint32 sc = rd32(x->op, XHCI_PORTSC(port));
  struct usb_dev proto;
  int i;

  if(!(sc & PORT_CCS))
    return;

  // USB2 ports need an explicit reset before they will enable. USB3 ports
  // train themselves and come up already enabled, so resetting one is at best
  // wasted time. PED tells the two apart without having to know which is
  // which from the extended capabilities.
  if(!(sc & PORT_PED)){
    wr32(x->op, XHCI_PORTSC(port), (sc & ~PORT_RW1C) | PORT_PR);
    for(i = 0; i < 1000; i++){
      microdelay(1000);
      sc = rd32(x->op, XHCI_PORTSC(port));
      if(sc & PORT_PRC)
        break;
    }
    // Acknowledge the reset-complete and connect-status change bits.
    wr32(x->op, XHCI_PORTSC(port),
         (sc & ~PORT_RW1C) | PORT_PRC | PORT_CSC);
    sc = rd32(x->op, XHCI_PORTSC(port));
  }

  if(!(sc & PORT_PED)){
    cprintf("xhci: port %d did not enable after reset\n", port);
    return;
  }

  // A root port device is tier 0: no route string, no transaction translator.
  memset(&proto, 0, sizeof(proto));
  proto.port  = port;
  proto.speed = PORT_SPEED(sc);
  proto.tier  = 0;

  attach_device(x, &proto);
}

// ---------------------------------------------------------------------------
// Controller bring-up
// ---------------------------------------------------------------------------

static int
reset_controller(struct xhci *x)
{
  int i;

  // Stop it before resetting it. A controller that is still running when
  // HCRST is written is not required to do anything sensible.
  wr32(x->op, XHCI_USBCMD, rd32(x->op, XHCI_USBCMD) & ~CMD_RS);
  for(i = 0; i < 1000; i++){
    if(rd32(x->op, XHCI_USBSTS) & STS_HCH)
      break;
    microdelay(1000);
  }
  if(!(rd32(x->op, XHCI_USBSTS) & STS_HCH)){
    cprintf("xhci: controller would not halt\n");
    return -1;
  }

  wr32(x->op, XHCI_USBCMD, CMD_HCRST);
  for(i = 0; i < 1000; i++){
    microdelay(1000);
    if(!(rd32(x->op, XHCI_USBCMD) & CMD_HCRST) &&
       !(rd32(x->op, XHCI_USBSTS) & STS_CNR))
      return 0;
  }
  cprintf("xhci: reset did not complete\n");
  return -1;
}

// Scratchpad buffers are memory the controller asks to borrow for its own
// internal use. If it asks and does not get them, it will not run.
static int
alloc_scratchpad(struct xhci *x, int count)
{
  uint64 *array;
  int i;

  if(count == 0)
    return 0;

  array = dma_alloc(count * 8, 4096);
  if(array == 0)
    return -1;
  for(i = 0; i < count; i++){
    void *p = dma_alloc(4096, 4096);
    if(p == 0)
      return -1;
    array[i] = dma_phys(p);
  }
  // The scratchpad array hangs off DCBAA entry 0, which is otherwise unused
  // because slot numbering starts at 1.
  x->dcbaa[0] = dma_phys(array);
  return 0;
}

// One completed transfer on a non-control endpoint. Caller holds x->lock,
// which is also what makes it safe for the debug dump and the interrupt
// handler to drain the same event ring.
static void
transfer_done(struct xhci *x, struct trb *ev)
{
  int slot = TRB_GET_SLOT(ev->control);
  int dci  = TRB_GET_EPID(ev->control);
  struct xhci_ep *ep;
  struct usb_dev *dev;
  int cc, got;

  if(slot <= 0 || slot > MAX_SLOTS)
    return;
  ep = find_ep(x, slot, dci);
  if(ep == 0)
    return;
  dev = &x->slot[slot].dev;

  cc = TRB_CC(ev->status);

  // A bulk transfer has somebody asleep waiting for it; an interrupt endpoint
  // has nobody, and must instead be re-armed at once or the device is never
  // heard from again.
  if(ep->type == USB_EP_BULK){
    ep->cc      = cc;
    ep->residue = (int)(ev->status & 0xFFFFFF);
    __sync_synchronize();
    ep->done    = 1;
    // Before the scheduler exists the caller is spinning on ep->done a few
    // frames away, and there is no process table to wake anything in.
    if(x->can_sleep)
      wakeup(ep);
    return;
  }

  if(cc == CC_SUCCESS || cc == CC_SHORT_PACKET){
    // The status field holds the residue -- what was *not* transferred --
    // so the length actually received is the request minus that.
    got = ep->len - (int)(ev->status & 0xFFFFFF);
    if(got > 0 && dev->complete)
      dev->complete(dev, ep->buf, got);
  }

  queue_in_locked(dev, ep->addr, 0, 0);  // re-arm, even after an error
}

static void
xhci_irq(void *arg)
{
  struct xhci *x = arg;
  struct trb ev;
  uint32 sts = rd32(x->op, XHCI_USBSTS);

  // A shared line means this may not be us at all. Say nothing and return.
  if(!(sts & STS_EINT))
    return;

  wr32(x->op, XHCI_USBSTS, STS_EINT);
  wr32(x->rt, XHCI_IR0 + IR_IMAN,
       rd32(x->rt, XHCI_IR0 + IR_IMAN) | IMAN_IP);

  // The event ring has exactly one consumer at a time. Without this the debug
  // dump, which drains it from process context, could take an event out from
  // under this handler mid-update and leave the dequeue index wrong for good.
  acquire(&x->lock);
  while(event_pop(x, &ev)){
    switch(TRB_GET_TYPE(ev.control)){
    case TRB_XFER_EVENT:
      transfer_done(x, &ev);
      break;
    case TRB_PORT_STATUS:
      // Hotplug. Acknowledging is enough for now; enumerating from interrupt
      // context would need the control transfers to become asynchronous.
      break;
    default:
      break;
    }
  }
  release(&x->lock);
}

// Print everything the controller thinks is true, and drain the event ring by
// hand.
//
// This exists because "nothing happened" has at least two causes that look
// identical from outside: the device never produced an event, or it did and
// the interrupt carrying the news was lost. Printing the ring separates them
// in one look. Reached with `echo u > /proc/sysrq`.
void
xhci_dump(void)
{
  struct trb ev;
  int i, p, n;

  for(i = 0; i < nxhci; i++){
    struct xhci *x = &xhcis[i];

    cprintf("xhci%d: usbcmd %x usbsts %x crcr %p\n", i,
            rd32(x->op, XHCI_USBCMD), rd32(x->op, XHCI_USBSTS),
            rd64(x->op, XHCI_CRCR));
    cprintf("  iman %x imod %x erstsz %d erdp %p (deq %d cycle %d)\n",
            rd32(x->rt, XHCI_IR0 + IR_IMAN), rd32(x->rt, XHCI_IR0 + IR_IMOD),
            rd32(x->rt, XHCI_IR0 + IR_ERSTSZ),
            rd64(x->rt, XHCI_IR0 + IR_ERDP), x->ev.deq, x->ev.cycle);
    cprintf("  pci irq %d\n", x->pci->irq);

    for(p = 1; p <= x->maxports; p++){
      uint32 sc = rd32(x->op, XHCI_PORTSC(p));
      if(sc & PORT_CCS)
        cprintf("  port %d: portsc %x (ccs=%d ped=%d speed=%d)\n",
                p, sc, sc & PORT_CCS ? 1 : 0, sc & PORT_PED ? 1 : 0,
                PORT_SPEED(sc));
    }

    for(p = 1; p <= MAX_SLOTS; p++){
      int e;
      if(x->slot[p].nep == 0)
        continue;
      for(e = 0; e < x->slot[p].nep; e++)
        cprintf("  slot %d ep dci %d addr %x enq %d cycle %d\n",
                p, x->slot[p].ep[e].dci, x->slot[p].ep[e].addr,
                x->slot[p].ep[e].ring.enq, x->slot[p].ep[e].ring.cycle);
    }

    n = 0;
    acquire(&x->lock);          // the interrupt handler drains the same ring
    while(event_pop(x, &ev)){
      cprintf("  event: type %d cc %d slot %d epid %d len %d\n",
              TRB_GET_TYPE(ev.control), TRB_CC(ev.status),
              TRB_GET_SLOT(ev.control), TRB_GET_EPID(ev.control),
              ev.status & 0xFFFFFF);
      if(TRB_GET_TYPE(ev.control) == TRB_XFER_EVENT)
        transfer_done(x, &ev);
      if(++n > 16)
        break;
    }
    release(&x->lock);
    if(n == 0)
      cprintf("  event ring empty\n");
  }
}

static int
xhci_probe(struct pci_dev *d)
{
  struct xhci *x;
  uint32 hcs1, hcs2, hcc1;
  int caplen, port, spb;

  if(nxhci >= (int)(sizeof(xhcis)/sizeof(xhcis[0])))
    return -1;
  if(d->bar[0].base == 0)
    return -1;

  x = &xhcis[nxhci];
  x->pci = d;
  // A BAR above 4 GB used to be refused here: the direct map covers only the
  // first 4 GB, so there was nowhere to put it. ioremap() gives it somewhere,
  // and returns the direct-map address unchanged when the BAR is low -- which
  // every BAR is under SeaBIOS.
  x->cap = (volatile uchar*)ioremap(d->bar[0].base, d->bar[0].size);
  if(x->cap == 0){
    cprintf("xhci: cannot map BAR0 at 0x%p\n", d->bar[0].base);
    return -1;
  }
  initlock(&x->lock, "xhci");

  // Bus mastering is not optional: every structure below is read by the
  // controller itself, and without this bit set those reads never leave it.
  pci_enable(d, PCI_CMD_MEMORY | PCI_CMD_BUS_MASTER);

  caplen = rd32(x->cap, XHCI_CAPLENGTH) & 0xFF;
  x->op  = x->cap + caplen;
  x->rt  = x->cap + (rd32(x->cap, XHCI_RTSOFF) & ~0x1Fu);
  x->db  = (volatile uint32*)(x->cap + (rd32(x->cap, XHCI_DBOFF) & ~0x3u));

  hcs1 = rd32(x->cap, XHCI_HCSPARAMS1);
  hcs2 = rd32(x->cap, XHCI_HCSPARAMS2);
  hcc1 = rd32(x->cap, XHCI_HCCPARAMS1);

  x->maxslots = HCS1_MAXSLOTS(hcs1);
  x->maxports = HCS1_MAXPORTS(hcs1);
  x->ctxsize  = (hcc1 & HCC1_CSZ) ? 64 : 32;

  // HCIVERSION is a *word* at offset 2, and MMIO does not promise anything
  // about an unaligned access -- reading a uint32 from offset 2 returned zero
  // rather than the version. It shares its dword with CAPLENGTH, so read that
  // dword and take the top half.
  {
    uint32 v = rd32(x->cap, 0) >> 16;
    cprintf("xhci: version %d.%d%d, %d ports, %d slots, %d-byte contexts\n",
            (v >> 8) & 0xFF, (v >> 4) & 0xF, v & 0xF,
            x->maxports, x->maxslots, x->ctxsize);
  }

  if(reset_controller(x) < 0)
    return -1;

  if(x->maxslots > MAX_SLOTS)
    x->maxslots = MAX_SLOTS;

  // Device Context Base Address Array: one 64-bit pointer per slot, plus
  // entry 0 for the scratchpad array.
  x->dcbaa = dma_alloc((x->maxslots + 1) * 8, 4096);
  if(x->dcbaa == 0)
    return -1;
  x->dcbaa_phys = dma_phys(x->dcbaa);

  spb = (HCS2_SPB_HI(hcs2) << 5) | HCS2_SPB_LO(hcs2);
  if(alloc_scratchpad(x, spb) < 0){
    cprintf("xhci: could not allocate %d scratchpad buffers\n", spb);
    return -1;
  }

  if(ring_init(&x->cmd) < 0 || event_ring_init(x) < 0)
    return -1;

  wr32(x->op, XHCI_CONFIG, x->maxslots);
  wr64(x->op, XHCI_DCBAAP, x->dcbaa_phys);
  wr64(x->op, XHCI_CRCR, x->cmd.phys | CRCR_RCS);

  wr32(x->rt, XHCI_IR0 + IR_ERSTSZ, 1);
  wr64(x->rt, XHCI_IR0 + IR_ERDP, x->ev.phys);
  // ERSTBA last: writing it is what makes the controller start using the
  // event ring, so everything it will consult must already be valid.
  wr64(x->rt, XHCI_IR0 + IR_ERSTBA, x->ev.erst_phys);

  // Interrupts stay OFF through enumeration -- IMAN.IE and USBCMD.INTE are
  // both deliberately clear here. Enumeration polls, so it does not need
  // them, and enabling them early is actively harmful: the controller would
  // assert its INTx line during enumeration, while the IOAPIC entry for that
  // line is still masked. An interrupt that arrives at a masked pin is
  // *dropped*, not deferred -- and because INTx is a level, the line then
  // simply stays high with no further transition for the IOAPIC to notice.
  // Unmasking afterwards finds nothing pending and the device is never heard
  // from again, which is exactly the symptom this produced: successful
  // transfers, a completion event sitting in the ring, and an interrupt count
  // stuck at zero.
  wr32(x->op, XHCI_USBCMD, CMD_RS);
  for(port = 0; port < 100; port++){
    if(!(rd32(x->op, XHCI_USBSTS) & STS_HCH))
      break;
    microdelay(1000);
  }
  if(rd32(x->op, XHCI_USBSTS) & STS_HCH){
    cprintf("xhci: controller would not start\n");
    return -1;
  }

  // Prove the ring and the event path work before trusting them with a real
  // command. A No-Op does nothing except complete, which is exactly the point
  // -- if this fails, the fault is in our plumbing and not in any device.
  if(command(x, 0, 0, TRB_TYPE(TRB_NOOP_CMD), 0) != CC_SUCCESS){
    cprintf("xhci: command ring no-op failed; not enumerating\n");
    return -1;
  }

  for(port = 1; port <= x->maxports; port++)
    enumerate_port(x, port);

  if(x->ndevices == 0)
    cprintf("xhci: no devices attached\n");

  // Probing is over: from here a bulk transfer has a process behind it and
  // can sleep rather than spin.
  x->can_sleep = 1;

  // Now, and only now, turn interrupts on. The order is: claim the line
  // (which unmasks it and sets level/active-low), clear anything the
  // controller already considers pending so that it drives INTx low, and then
  // let it interrupt. The next event is a genuine low-to-high transition,
  // which is the only thing an interrupt controller can act on.
  if(d->irq != 0xFF){
    irq_register(d->irq, xhci_irq, x, "xhci", 1);   // PCI: level, active low

    wr32(x->op, XHCI_USBSTS, STS_EINT);
    wr32(x->rt, XHCI_IR0 + IR_IMAN, IMAN_IP | IMAN_IE);
    wr32(x->op, XHCI_USBCMD, rd32(x->op, XHCI_USBCMD) | CMD_INTE);
  }

  nxhci++;
  return 0;
}

// Class 0x0C subclass 0x03 prog-if 0x30 is xHCI. Matching on class rather
// than on vendor/device is the whole point of a standardised controller: one
// entry covers every xHCI ever made.
static const struct pci_match xhci_match[] = {
  { PCI_ANY, PCI_ANY, PCI_CLASS_SERIAL_BUS, 0x03 },
  { 0, 0, 0, 0 },
};

const struct pci_driver xhci_driver = {
  .name  = "xhci",
  .match = xhci_match,
  .probe = xhci_probe,
};