SchOS/kernel/drivers/storage/ahci/ahci.c

// SPDX-License-Identifier: GPL-2.0-or-later
//
// AHCI: SATA disks, written from the Serial ATA AHCI 1.3.1 specification.
//
// This is how every PC since about 2008 attaches its internal disk, and the
// IDE driver beside it is how almost none of them do. Under QEMU's q35 the
// boot disk is here, which is why that machine used to stop with "no root
// filesystem" while the very same image booted on i440fx.
//
// The model is small. The HBA has up to 32 ports; each has a command list of
// up to 32 slots, each slot points at a command table, and the table holds
// the ATA command as a register FIS plus a scatter list of where the data
// goes. Issue a command by setting its bit in PxCI; the HBA clears the bit
// when the command is done.
//
// One slot per port and **completion by polling**, deliberately. A disk is
// on the path to a shell, and polling PxCI is the one way of finishing a
// command that does not depend on interrupt routing -- which on real hardware
// without an AML interpreter is the part that cannot be trusted (see
// kernel/dev/irq.c). A SATA command completes in tens of microseconds to a
// few milliseconds, so the cost is that, spent spinning, and it is yielded
// once a wait goes long.

#include "types.h"
#include "defs.h"
#include "param.h"
#include "x86.h"
#include "spinlock.h"
#include "sleeplock.h"
#include "fs.h"
#include "file.h"
#include "proc.h"
#include "pci.h"
#include "dma.h"
#include "blkdev.h"

// --- HBA registers (section 3.1) ---------------------------------------------

#define HBA_CAP     0x00
#define   CAP_S64A    (1u << 31)       // 64-bit DMA addresses
#define   CAP_SSS     (1u << 27)       // staggered spin-up: ports start spun down
#define HBA_GHC     0x04
#define   GHC_AE      (1u << 31)       // AHCI enable
#define   GHC_HR      (1u << 0)        // HBA reset
#define HBA_PI      0x0C               // ports implemented
#define HBA_VS      0x10
#define HBA_CAP2    0x24
#define   CAP2_BOH    (1u << 0)        // BIOS/OS handoff supported
#define HBA_BOHC    0x28
#define   BOHC_BOS    (1u << 0)        // the BIOS owns the HBA
#define   BOHC_OOS    (1u << 1)        // the OS asks for it
#define   BOHC_BB     (1u << 4)        // BIOS busy

// --- port registers (section 3.3), at 0x100 + port * 0x80 --------------------

#define PX_CLB      0x00
#define PX_CLBU     0x04
#define PX_FB       0x08
#define PX_FBU      0x0C
#define PX_IS       0x10
#define PX_IE       0x14
#define PX_CMD      0x18
#define   CMD_ST      (1u << 0)        // start processing the command list
#define   CMD_SUD     (1u << 1)        // spin up
#define   CMD_POD     (1u << 2)        // power on
#define   CMD_FRE     (1u << 4)        // FIS receive enable
#define   CMD_FR      (1u << 14)       // FIS receive running
#define   CMD_CR      (1u << 15)       // command list running
#define PX_TFD      0x20
#define   TFD_ERR     0x01
#define   TFD_DRQ     0x08
#define   TFD_BSY     0x80
#define PX_SIG      0x24
#define   SIG_ATA     0x00000101
#define   SIG_ATAPI   0xEB140101
#define PX_SSTS     0x28
#define PX_SCTL     0x2C
#define PX_SERR     0x30
#define PX_CI       0x38

#define IS_TFES     (1u << 30)         // task file error

// --- ATA ---------------------------------------------------------------------

#define ATA_IDENTIFY        0xEC
#define ATA_READ_DMA_EXT    0x25
#define ATA_WRITE_DMA_EXT   0x35
#define ATA_FLUSH_EXT       0xEA

#define FIS_H2D             0x27

// One command header (section 4.2.2).
struct cmdhdr {
  uint16 flags;          // CFL in 4:0, A 5, W 6, P 7, R 8, B 9, C 10
  uint16 prdtl;          // scatter entries in the table
  uint32 prdbc;          // bytes transferred, written by the HBA
  uint32 ctba, ctbau;    // the command table, 128-byte aligned
  uint32 rsv[4];
};

struct prd {
  uint32 dba, dbau;
  uint32 rsv;
  uint32 dbc;            // byte count - 1 in 21:0, interrupt-on-completion 31
};

// The command table (section 4.2.3): the FIS, an ATAPI command we never send,
// and one scatter entry -- every transfer here is one bounce buffer.
struct cmdtbl {
  uint8  cfis[64];
  uint8  acmd[16];
  uint8  rsv[48];
  struct prd prdt[1];
};

#define AHCI_MAX_PORTS 32
#define AHCI_MAX_DISKS 8

struct ahci_port {
  volatile uint8 *r;              // this port's registers
  struct cmdhdr  *cl;             // command list, 1 KB
  uint8          *fis;            // received FIS area, 256 bytes
  struct cmdtbl  *ct;             // the one command table
  uint8          *xfer;           // bounce buffer, BSIZE
  uint64          nsectors;
  uint32          secsize;
  struct sleeplock lock;          // one command at a time on this port
  char            name[8];
  int             dev;
};

static struct ahci_port disks[AHCI_MAX_DISKS];
static int ndisks;
static int nhba;

static inline uint32 rd(volatile uint8 *b, int o)        { return *(volatile uint32 *)(b + o); }
static inline void   wr(volatile uint8 *b, int o, uint32 v) { *(volatile uint32 *)(b + o) = v; }

// Spin until (reg & mask) == want, for at most `ms`. Returns 0 or -1.
static int
waitreg(volatile uint8 *b, int o, uint32 mask, uint32 want, int ms)
{
  for (int i = 0; i < ms * 100; i++) {
    if ((rd(b, o) & mask) == want)
      return 0;
    microdelay(10);
  }
  return (rd(b, o) & mask) == want ? 0 : -1;
}

// Stop the port's engines, which is the precondition for touching CLB and FB.
// Section 10.1.2: clear ST and wait for CR, clear FRE and wait for FR.
static int
port_stop(volatile uint8 *r)
{
  wr(r, PX_CMD, rd(r, PX_CMD) & ~CMD_ST);
  if (waitreg(r, PX_CMD, CMD_CR, 0, 500) < 0)
    return -1;
  wr(r, PX_CMD, rd(r, PX_CMD) & ~CMD_FRE);
  return waitreg(r, PX_CMD, CMD_FR, 0, 500);
}

// The link, if any. DET = 3 is "device present and communication established"
// and IPM = 1 is "active". Firmware normally leaves a link up; when it has not,
// a COMRESET (section 10.4.2) brings it up, and a port with nothing on it
// simply never answers.
static int
port_link(volatile uint8 *r)
{
  // A drive told to spin up a moment ago takes a few milliseconds to be seen
  // at all; give it twenty before calling the port empty.
  uint32 ssts = rd(r, PX_SSTS);
  for (int i = 0; i < 20 && (ssts & 0xF) == 0; i++) {
    microdelay(1000);
    ssts = rd(r, PX_SSTS);
  }
  if ((ssts & 0xF) == 3)
    return 1;
  // DET = 0 is "no device detected": an empty port, not a slow one. This
  // used to COMRESET every empty port and wait a second for an answer that
  // could not come -- four seconds of boot on a q35 with one disk, all of it
  // with interrupts off. Only a device that is detected without a link
  // (DET = 1) is worth the reset.
  if ((ssts & 0xF) != 1)
    return 0;

  wr(r, PX_SCTL, (rd(r, PX_SCTL) & ~0xFu) | 1);    // DET = 1: COMRESET
  microdelay(2000);                                  // at least 1 ms
  wr(r, PX_SCTL, rd(r, PX_SCTL) & ~0xFu);
  for (int i = 0; i < 100; i++) {                    // up to 1 s for the link
    if ((rd(r, PX_SSTS) & 0xF) == 3)
      return 1;
    microdelay(10000);
  }
  return 0;
}

// Issue the command in slot 0 and wait for it. `buf` is BSIZE bytes of the
// port's bounce buffer, which is where the data goes or comes from.
static int
port_exec(struct ahci_port *p, uint8 cmd, uint64 lba, uint32 count,
          uint32 bytes, int write)
{
  volatile uint8 *r = p->r;
  struct cmdtbl *t = p->ct;

  // Wait out anything still busy from before. A port that stays busy is one
  // the drive has wedged; failing is better than issuing into it.
  if (waitreg(r, PX_TFD, TFD_BSY | TFD_DRQ, 0, 1000) < 0)
    return -1;

  memset(t, 0, sizeof(*t));
  uint8 *f = t->cfis;
  f[0] = FIS_H2D;
  f[1] = 0x80;                        // C: this is a command, not a control
  f[2] = cmd;
  f[4] = lba;  f[5] = lba >> 8;  f[6] = lba >> 16;
  f[7] = 0x40;                        // LBA mode
  f[8] = lba >> 24;  f[9] = lba >> 32;  f[10] = lba >> 40;
  f[12] = count;  f[13] = count >> 8;

  struct cmdhdr *h = &p->cl[0];
  h->flags = 5 | (write ? (1 << 6) : 0);   // a register FIS is five dwords
  h->prdbc = 0;
  if (bytes) {
    uint64 pa = dma_phys(p->xfer);
    t->prdt[0].dba  = (uint32)pa;
    t->prdt[0].dbau = (uint32)(pa >> 32);
    t->prdt[0].dbc  = bytes - 1;
    h->prdtl = 1;
  } else {
    h->prdtl = 0;
  }

  wr(r, PX_IS, 0xFFFFFFFF);
  __sync_synchronize();
  wr(r, PX_CI, 1);

  // Spin for the first few milliseconds, which is when nearly every command
  // finishes; after that give the processor away between looks. yield() only
  // from a process -- the root probe and IDENTIFY run before there is one.
  uint64 start = rdtsc(), limit = tsc_hz * 10;     // ten seconds
  while (rd(r, PX_CI) & 1) {
    if (rd(r, PX_IS) & IS_TFES)
      break;
    uint64 now = rdtsc() - start;
    if (now > limit) {
      cprintf("ahci: %s: command %x timed out\n", p->name, cmd);
      return -1;
    }
    if (now > tsc_hz / 200 && myproc())
      yield();
    else
      asm volatile("pause");
  }

  if ((rd(r, PX_IS) & IS_TFES) || (rd(r, PX_TFD) & TFD_ERR)) {
    cprintf("ahci: %s: command %x failed, tfd %x\n", p->name, cmd,
            rd(r, PX_TFD));
    return -1;
  }
  return 0;
}

static int
ahci_rw(void *priv, uint64 lba, uint32 count, void *buf, int write)
{
  struct ahci_port *p = priv;
  uint32 bytes = count * p->secsize;
  int rc;

  if (bytes > BSIZE)
    return -1;
  acquiresleep(&p->lock);
  if (write)
    memmove(p->xfer, buf, bytes);
  rc = port_exec(p, write ? ATA_WRITE_DMA_EXT : ATA_READ_DMA_EXT,
                 lba, count, bytes, write);
  if (rc == 0 && !write)
    memmove(buf, p->xfer, bytes);
  releasesleep(&p->lock);
  return rc;
}

static int
ahci_flush(void *priv)
{
  struct ahci_port *p = priv;
  acquiresleep(&p->lock);
  int rc = port_exec(p, ATA_FLUSH_EXT, 0, 0, 0, 0);
  releasesleep(&p->lock);
  return rc;
}

static const struct blkdev_ops ahci_ops = {
  .rw    = ahci_rw,
  .flush = ahci_flush,
};

// IDENTIFY DEVICE's model string: forty characters, byte-swapped in pairs,
// padded with spaces.
static void
model_of(const uint16 *id, char *out, int cap)
{
  int n = 0;
  for (int w = 27; w < 47 && n + 2 < cap; w++) {
    out[n++] = id[w] >> 8;
    out[n++] = id[w] & 0xFF;
  }
  while (n > 0 && out[n - 1] == ' ')
    n--;
  out[n] = 0;
}

// Bring one port up and, if a SATA disk answers on it, register it.
static void
port_init(volatile uint8 *hba, int port, uint32 cap)
{
  volatile uint8 *r = hba + 0x100 + port * 0x80;

  if (ndisks >= AHCI_MAX_DISKS)
    return;
  if (port_stop(r) < 0) {
    cprintf("ahci: port %d will not stop; skipping it\n", port);
    return;
  }

  // Power and spin up first: with staggered spin-up the drive is not even
  // turning until SUD is set, and its link never comes up until it is.
  if (cap & CAP_SSS)
    wr(r, PX_CMD, rd(r, PX_CMD) | CMD_POD | CMD_SUD);

  if (!port_link(r))
    return;                                   // nothing plugged in

  struct ahci_port *p = &disks[ndisks];
  memset(p, 0, sizeof(*p));
  p->r    = r;
  p->cl   = dma_alloc(1024, 1024);
  p->fis  = dma_alloc(256, 256);
  p->ct   = dma_alloc(sizeof(struct cmdtbl), 128);
  p->xfer = dma_alloc(BSIZE, 4096);
  if (!p->cl || !p->fis || !p->ct || !p->xfer) {
    cprintf("ahci: no DMA memory for port %d\n", port);
    return;
  }
  // Without S64A the HBA cannot see above 4 GB. The DMA arena is below it
  // (see include/dma.h), so this holds, and says so if that ever changes.
  if (!(cap & CAP_S64A) && dma_phys(p->xfer) + BSIZE > 0xFFFFFFFFULL) {
    cprintf("ahci: DMA memory above 4 GB and the HBA is 32-bit\n");
    return;
  }
  memset(p->cl, 0, 1024);
  memset(p->fis, 0, 256);

  uint64 cl = dma_phys(p->cl), fb = dma_phys(p->fis), ct = dma_phys(p->ct);
  p->cl[0].ctba  = (uint32)ct;
  p->cl[0].ctbau = (uint32)(ct >> 32);
  wr(r, PX_CLB,  (uint32)cl);
  wr(r, PX_CLBU, (uint32)(cl >> 32));
  wr(r, PX_FB,   (uint32)fb);
  wr(r, PX_FBU,  (uint32)(fb >> 32));

  wr(r, PX_SERR, 0xFFFFFFFF);                 // write-1-to-clear
  wr(r, PX_IS,   0xFFFFFFFF);
  wr(r, PX_IE,   0);                          // polled; see the top of the file
  wr(r, PX_CMD,  rd(r, PX_CMD) | CMD_FRE);

  // The drive's first D2H FIS sets the signature; until then it reads all
  // ones. ATAPI (an optical drive) and port multipliers are not disks here.
  if (waitreg(r, PX_TFD, TFD_BSY | TFD_DRQ, 0, 3000) < 0) {
    cprintf("ahci: port %d: the drive stays busy\n", port);
    return;
  }
  uint32 sig = rd(r, PX_SIG);
  if (sig != SIG_ATA) {
    if (sig == SIG_ATAPI)
      kinfo("ahci: port %d: ATAPI device, not used\n", port);
    return;
  }

  wr(r, PX_CMD, rd(r, PX_CMD) | CMD_ST);

  p->name[0] = 's'; p->name[1] = 'a'; p->name[2] = 't'; p->name[3] = 'a';
  p->name[4] = '0' + ndisks; p->name[5] = 0;
  initsleeplock(&p->lock, "ahci");
  p->secsize = 512;

  if (port_exec(p, ATA_IDENTIFY, 0, 0, 512, 0) < 0) {
    cprintf("ahci: port %d: IDENTIFY failed\n", port);
    return;
  }
  uint16 *id = (uint16 *)p->xfer;
  if (!(id[83] & (1 << 10))) {
    cprintf("ahci: port %d: no 48-bit LBA; not supported\n", port);
    return;
  }
  p->nsectors = (uint64)id[100] | (uint64)id[101] << 16 |
                (uint64)id[102] << 32 | (uint64)id[103] << 48;
  // A logical sector larger than 512 (word 106 bit 12, size in 117-118 as
  // words). Rare, and handled: the block layer works in BSIZE and asks for
  // BSIZE / secsize of them.
  if ((id[106] & 0xC000) == 0x4000 && (id[106] & (1 << 12)))
    p->secsize = 2 * ((uint32)id[117] | (uint32)id[118] << 16);
  if (p->secsize < 512 || p->secsize > BSIZE || BSIZE % p->secsize) {
    cprintf("ahci: port %d: sector size %d not supported\n", port, p->secsize);
    return;
  }

  char model[44];
  model_of(id, model, sizeof(model));
  ndisks++;
  kinfo("ahci: %s: %s, %d MB, port %d\n", p->name, model,
        (int)((p->nsectors * p->secsize) >> 20), port);

  p->dev = blkdev_register(p->name, &ahci_ops, p, p->nsectors, p->secsize);
}

// Take the HBA from the firmware if it is holding it (section 10.6). Most
// firmware lets go at ExitBootServices; the specification says to ask, and
// some does not let go until asked.
static void
bios_handoff(volatile uint8 *hba)
{
  if (!(rd(hba, HBA_CAP2) & CAP2_BOH))
    return;
  wr(hba, HBA_BOHC, rd(hba, HBA_BOHC) | BOHC_OOS);
  if (waitreg(hba, HBA_BOHC, BOHC_BOS, 0, 25) < 0 ||
      (rd(hba, HBA_BOHC) & BOHC_BB)) {
    // BIOS busy: it is allowed two seconds to finish what it is doing.
    waitreg(hba, HBA_BOHC, BOHC_BB | BOHC_BOS, 0, 2000);
  }
}

static int
ahci_probe(struct pci_dev *d)
{
  if (d->prog_if != 0x01)
    return -1;                         // a RAID or IDE-mode controller
  if (!d->bar[5].base || d->bar[5].is_io) {
    cprintf("ahci: no ABAR; not driving this controller\n");
    return -1;
  }

  pci_enable(d, PCI_CMD_MEMORY | PCI_CMD_BUS_MASTER);
  volatile uint8 *hba = ioremap(d->bar[5].base,
                                d->bar[5].size ? d->bar[5].size : 0x1100);
  if (!hba)
    return -1;

  bios_handoff(hba);

  // AHCI mode, with interrupts off at the HBA. No HBA reset: firmware has
  // already brought the links up, and resetting throws that away for a
  // second or more of COMRESETs per port to get it back.
  wr(hba, HBA_GHC, (rd(hba, HBA_GHC) | GHC_AE) & ~(1u << 1));   // IE off

  uint32 cap = rd(hba, HBA_CAP), pi = rd(hba, HBA_PI);
  uint32 vs = rd(hba, HBA_VS);
  kinfo("ahci: controller %x:%x, AHCI %d.%d, %d ports\n",
        d->vendor, d->device, vs >> 16, (vs >> 8) & 0xFF, (cap & 0x1F) + 1);

  for (int port = 0; port < AHCI_MAX_PORTS; port++)
    if (pi & (1u << port))
      port_init(hba, port, cap);

  nhba++;
  return 0;
}

static const struct pci_match ahci_match[] = {
  { PCI_ANY, PCI_ANY, 0x01, 0x06 },     // mass storage, SATA
  { 0, 0, 0, 0 },
};

const struct pci_driver ahci_driver = {
  .name    = "ahci",
  .devname = "sata",
  .match   = ahci_match,
  .probe   = ahci_probe,
};