// SPDX-License-Identifier: GPL-2.0-or-later // // AHCI: SATA disks, written from the Serial ATA AHCI 1.3.1 specification. // // This is how every PC since about 2008 attaches its internal disk, and the // IDE driver beside it is how almost none of them do. Under QEMU's q35 the // boot disk is here, which is why that machine used to stop with "no root // filesystem" while the very same image booted on i440fx. // // The model is small. The HBA has up to 32 ports; each has a command list of // up to 32 slots, each slot points at a command table, and the table holds // the ATA command as a register FIS plus a scatter list of where the data // goes. Issue a command by setting its bit in PxCI; the HBA clears the bit // when the command is done. // // One slot per port and **completion by polling**, deliberately. A disk is // on the path to a shell, and polling PxCI is the one way of finishing a // command that does not depend on interrupt routing -- which on real hardware // without an AML interpreter is the part that cannot be trusted (see // kernel/dev/irq.c). A SATA command completes in tens of microseconds to a // few milliseconds, so the cost is that, spent spinning, and it is yielded // once a wait goes long. #include "core/types.h" #include "core/defs.h" #include "core/param.h" #include "arch/x86.h" #include "core/spinlock.h" #include "core/sleeplock.h" #include "fs/fs.h" #include "fs/file.h" #include "core/proc.h" #include "dev/pci.h" #include "core/dma.h" #include "fs/blkdev.h" // --- HBA registers (section 3.1) --------------------------------------------- #define HBA_CAP 0x00 #define CAP_S64A (1u << 31) // 64-bit DMA addresses #define CAP_SSS (1u << 27) // staggered spin-up: ports start spun down #define HBA_GHC 0x04 #define GHC_AE (1u << 31) // AHCI enable #define GHC_HR (1u << 0) // HBA reset #define HBA_PI 0x0C // ports implemented #define HBA_VS 0x10 #define HBA_CAP2 0x24 #define CAP2_BOH (1u << 0) // BIOS/OS handoff supported #define HBA_BOHC 0x28 #define BOHC_BOS (1u << 0) // the BIOS owns the HBA #define BOHC_OOS (1u << 1) // the OS asks for it #define BOHC_BB (1u << 4) // BIOS busy // --- port registers (section 3.3), at 0x100 + port * 0x80 -------------------- #define PX_CLB 0x00 #define PX_CLBU 0x04 #define PX_FB 0x08 #define PX_FBU 0x0C #define PX_IS 0x10 #define PX_IE 0x14 #define PX_CMD 0x18 #define CMD_ST (1u << 0) // start processing the command list #define CMD_SUD (1u << 1) // spin up #define CMD_POD (1u << 2) // power on #define CMD_FRE (1u << 4) // FIS receive enable #define CMD_FR (1u << 14) // FIS receive running #define CMD_CR (1u << 15) // command list running #define PX_TFD 0x20 #define TFD_ERR 0x01 #define TFD_DRQ 0x08 #define TFD_BSY 0x80 #define PX_SIG 0x24 #define SIG_ATA 0x00000101 #define SIG_ATAPI 0xEB140101 #define PX_SSTS 0x28 #define PX_SCTL 0x2C #define PX_SERR 0x30 #define PX_CI 0x38 #define IS_TFES (1u << 30) // task file error // --- ATA --------------------------------------------------------------------- #define ATA_IDENTIFY 0xEC #define ATA_READ_DMA_EXT 0x25 #define ATA_WRITE_DMA_EXT 0x35 #define ATA_FLUSH_EXT 0xEA #define FIS_H2D 0x27 // One command header (section 4.2.2). struct cmdhdr { uint16 flags; // CFL in 4:0, A 5, W 6, P 7, R 8, B 9, C 10 uint16 prdtl; // scatter entries in the table uint32 prdbc; // bytes transferred, written by the HBA uint32 ctba, ctbau; // the command table, 128-byte aligned uint32 rsv[4]; }; struct prd { uint32 dba, dbau; uint32 rsv; uint32 dbc; // byte count - 1 in 21:0, interrupt-on-completion 31 }; // The command table (section 4.2.3): the FIS, an ATAPI command we never send, // and one scatter entry -- every transfer here is one bounce buffer. struct cmdtbl { uint8 cfis[64]; uint8 acmd[16]; uint8 rsv[48]; struct prd prdt[1]; }; #define AHCI_MAX_PORTS 32 #define AHCI_MAX_DISKS 8 struct ahci_port { volatile uint8 *r; // this port's registers struct cmdhdr *cl; // command list, 1 KB uint8 *fis; // received FIS area, 256 bytes struct cmdtbl *ct; // the one command table uint8 *xfer; // bounce buffer, BSIZE uint64 nsectors; uint32 secsize; struct sleeplock lock; // one command at a time on this port char name[8]; int dev; }; static struct ahci_port disks[AHCI_MAX_DISKS]; static int ndisks; static int nhba; static inline uint32 rd(volatile uint8 *b, int o) { return *(volatile uint32 *)(b + o); } static inline void wr(volatile uint8 *b, int o, uint32 v) { *(volatile uint32 *)(b + o) = v; } // Spin until (reg & mask) == want, for at most `ms`. Returns 0 or -1. static int waitreg(volatile uint8 *b, int o, uint32 mask, uint32 want, int ms) { for (int i = 0; i < ms * 100; i++) { if ((rd(b, o) & mask) == want) return 0; microdelay(10); } return (rd(b, o) & mask) == want ? 0 : -1; } // Stop the port's engines, which is the precondition for touching CLB and FB. // Section 10.1.2: clear ST and wait for CR, clear FRE and wait for FR. static int port_stop(volatile uint8 *r) { wr(r, PX_CMD, rd(r, PX_CMD) & ~CMD_ST); if (waitreg(r, PX_CMD, CMD_CR, 0, 500) < 0) return -1; wr(r, PX_CMD, rd(r, PX_CMD) & ~CMD_FRE); return waitreg(r, PX_CMD, CMD_FR, 0, 500); } // The link, if any. DET = 3 is "device present and communication established" // and IPM = 1 is "active". Firmware normally leaves a link up; when it has not, // a COMRESET (section 10.4.2) brings it up, and a port with nothing on it // simply never answers. static int port_link(volatile uint8 *r) { // A drive told to spin up a moment ago takes a few milliseconds to be seen // at all; give it twenty before calling the port empty. uint32 ssts = rd(r, PX_SSTS); for (int i = 0; i < 20 && (ssts & 0xF) == 0; i++) { microdelay(1000); ssts = rd(r, PX_SSTS); } if ((ssts & 0xF) == 3) return 1; // DET = 0 is "no device detected": an empty port, not a slow one. This // used to COMRESET every empty port and wait a second for an answer that // could not come -- four seconds of boot on a q35 with one disk, all of it // with interrupts off. Only a device that is detected without a link // (DET = 1) is worth the reset. if ((ssts & 0xF) != 1) return 0; wr(r, PX_SCTL, (rd(r, PX_SCTL) & ~0xFu) | 1); // DET = 1: COMRESET microdelay(2000); // at least 1 ms wr(r, PX_SCTL, rd(r, PX_SCTL) & ~0xFu); for (int i = 0; i < 100; i++) { // up to 1 s for the link if ((rd(r, PX_SSTS) & 0xF) == 3) return 1; microdelay(10000); } return 0; } // Issue the command in slot 0 and wait for it. `buf` is BSIZE bytes of the // port's bounce buffer, which is where the data goes or comes from. static int port_exec(struct ahci_port *p, uint8 cmd, uint64 lba, uint32 count, uint32 bytes, int write) { volatile uint8 *r = p->r; struct cmdtbl *t = p->ct; // Wait out anything still busy from before. A port that stays busy is one // the drive has wedged; failing is better than issuing into it. if (waitreg(r, PX_TFD, TFD_BSY | TFD_DRQ, 0, 1000) < 0) return -1; memset(t, 0, sizeof(*t)); uint8 *f = t->cfis; f[0] = FIS_H2D; f[1] = 0x80; // C: this is a command, not a control f[2] = cmd; f[4] = lba; f[5] = lba >> 8; f[6] = lba >> 16; f[7] = 0x40; // LBA mode f[8] = lba >> 24; f[9] = lba >> 32; f[10] = lba >> 40; f[12] = count; f[13] = count >> 8; struct cmdhdr *h = &p->cl[0]; h->flags = 5 | (write ? (1 << 6) : 0); // a register FIS is five dwords h->prdbc = 0; if (bytes) { uint64 pa = dma_phys(p->xfer); t->prdt[0].dba = (uint32)pa; t->prdt[0].dbau = (uint32)(pa >> 32); t->prdt[0].dbc = bytes - 1; h->prdtl = 1; } else { h->prdtl = 0; } wr(r, PX_IS, 0xFFFFFFFF); __sync_synchronize(); wr(r, PX_CI, 1); // Spin for the first few milliseconds, which is when nearly every command // finishes; after that give the processor away between looks. yield() only // from a process -- the root probe and IDENTIFY run before there is one. uint64 start = rdtsc(), limit = tsc_hz * 10; // ten seconds while (rd(r, PX_CI) & 1) { if (rd(r, PX_IS) & IS_TFES) break; uint64 now = rdtsc() - start; if (now > limit) { cprintf("ahci: %s: command %x timed out\n", p->name, cmd); return -1; } if (now > tsc_hz / 200 && myproc()) yield(); else asm volatile("pause"); } if ((rd(r, PX_IS) & IS_TFES) || (rd(r, PX_TFD) & TFD_ERR)) { cprintf("ahci: %s: command %x failed, tfd %x\n", p->name, cmd, rd(r, PX_TFD)); return -1; } return 0; } static int ahci_rw(void *priv, uint64 lba, uint32 count, void *buf, int write) { struct ahci_port *p = priv; uint32 bytes = count * p->secsize; int rc; if (bytes > BSIZE) return -1; acquiresleep(&p->lock); if (write) memmove(p->xfer, buf, bytes); rc = port_exec(p, write ? ATA_WRITE_DMA_EXT : ATA_READ_DMA_EXT, lba, count, bytes, write); if (rc == 0 && !write) memmove(buf, p->xfer, bytes); releasesleep(&p->lock); return rc; } static int ahci_flush(void *priv) { struct ahci_port *p = priv; acquiresleep(&p->lock); int rc = port_exec(p, ATA_FLUSH_EXT, 0, 0, 0, 0); releasesleep(&p->lock); return rc; } static const struct blkdev_ops ahci_ops = { .rw = ahci_rw, .flush = ahci_flush, }; // IDENTIFY DEVICE's model string: forty characters, byte-swapped in pairs, // padded with spaces. static void model_of(const uint16 *id, char *out, int cap) { int n = 0; for (int w = 27; w < 47 && n + 2 < cap; w++) { out[n++] = id[w] >> 8; out[n++] = id[w] & 0xFF; } while (n > 0 && out[n - 1] == ' ') n--; out[n] = 0; } // Bring one port up and, if a SATA disk answers on it, register it. static void port_init(volatile uint8 *hba, int port, uint32 cap) { volatile uint8 *r = hba + 0x100 + port * 0x80; if (ndisks >= AHCI_MAX_DISKS) return; if (port_stop(r) < 0) { cprintf("ahci: port %d will not stop; skipping it\n", port); return; } // Power and spin up first: with staggered spin-up the drive is not even // turning until SUD is set, and its link never comes up until it is. if (cap & CAP_SSS) wr(r, PX_CMD, rd(r, PX_CMD) | CMD_POD | CMD_SUD); if (!port_link(r)) return; // nothing plugged in struct ahci_port *p = &disks[ndisks]; memset(p, 0, sizeof(*p)); p->r = r; p->cl = dma_alloc(1024, 1024); p->fis = dma_alloc(256, 256); p->ct = dma_alloc(sizeof(struct cmdtbl), 128); p->xfer = dma_alloc(BSIZE, 4096); if (!p->cl || !p->fis || !p->ct || !p->xfer) { cprintf("ahci: no DMA memory for port %d\n", port); return; } // Without S64A the HBA cannot see above 4 GB. The DMA arena is below it // (see include/core/dma.h), so this holds, and says so if that ever changes. if (!(cap & CAP_S64A) && dma_phys(p->xfer) + BSIZE > 0xFFFFFFFFULL) { cprintf("ahci: DMA memory above 4 GB and the HBA is 32-bit\n"); return; } memset(p->cl, 0, 1024); memset(p->fis, 0, 256); uint64 cl = dma_phys(p->cl), fb = dma_phys(p->fis), ct = dma_phys(p->ct); p->cl[0].ctba = (uint32)ct; p->cl[0].ctbau = (uint32)(ct >> 32); wr(r, PX_CLB, (uint32)cl); wr(r, PX_CLBU, (uint32)(cl >> 32)); wr(r, PX_FB, (uint32)fb); wr(r, PX_FBU, (uint32)(fb >> 32)); wr(r, PX_SERR, 0xFFFFFFFF); // write-1-to-clear wr(r, PX_IS, 0xFFFFFFFF); wr(r, PX_IE, 0); // polled; see the top of the file wr(r, PX_CMD, rd(r, PX_CMD) | CMD_FRE); // The drive's first D2H FIS sets the signature; until then it reads all // ones. ATAPI (an optical drive) and port multipliers are not disks here. if (waitreg(r, PX_TFD, TFD_BSY | TFD_DRQ, 0, 3000) < 0) { cprintf("ahci: port %d: the drive stays busy\n", port); return; } uint32 sig = rd(r, PX_SIG); if (sig != SIG_ATA) { if (sig == SIG_ATAPI) kinfo("ahci: port %d: ATAPI device, not used\n", port); return; } wr(r, PX_CMD, rd(r, PX_CMD) | CMD_ST); p->name[0] = 's'; p->name[1] = 'a'; p->name[2] = 't'; p->name[3] = 'a'; p->name[4] = '0' + ndisks; p->name[5] = 0; initsleeplock(&p->lock, "ahci"); p->secsize = 512; if (port_exec(p, ATA_IDENTIFY, 0, 0, 512, 0) < 0) { cprintf("ahci: port %d: IDENTIFY failed\n", port); return; } uint16 *id = (uint16 *)p->xfer; if (!(id[83] & (1 << 10))) { cprintf("ahci: port %d: no 48-bit LBA; not supported\n", port); return; } p->nsectors = (uint64)id[100] | (uint64)id[101] << 16 | (uint64)id[102] << 32 | (uint64)id[103] << 48; // A logical sector larger than 512 (word 106 bit 12, size in 117-118 as // words). Rare, and handled: the block layer works in BSIZE and asks for // BSIZE / secsize of them. if ((id[106] & 0xC000) == 0x4000 && (id[106] & (1 << 12))) p->secsize = 2 * ((uint32)id[117] | (uint32)id[118] << 16); if (p->secsize < 512 || p->secsize > BSIZE || BSIZE % p->secsize) { cprintf("ahci: port %d: sector size %d not supported\n", port, p->secsize); return; } char model[44]; model_of(id, model, sizeof(model)); ndisks++; kinfo("ahci: %s: %s, %d MB, port %d\n", p->name, model, (int)((p->nsectors * p->secsize) >> 20), port); p->dev = blkdev_register(p->name, &ahci_ops, p, p->nsectors, p->secsize); } // Take the HBA from the firmware if it is holding it (section 10.6). Most // firmware lets go at ExitBootServices; the specification says to ask, and // some does not let go until asked. static void bios_handoff(volatile uint8 *hba) { if (!(rd(hba, HBA_CAP2) & CAP2_BOH)) return; wr(hba, HBA_BOHC, rd(hba, HBA_BOHC) | BOHC_OOS); if (waitreg(hba, HBA_BOHC, BOHC_BOS, 0, 25) < 0 || (rd(hba, HBA_BOHC) & BOHC_BB)) { // BIOS busy: it is allowed two seconds to finish what it is doing. waitreg(hba, HBA_BOHC, BOHC_BB | BOHC_BOS, 0, 2000); } } static int ahci_probe(struct pci_dev *d) { if (d->prog_if != 0x01) return -1; // a RAID or IDE-mode controller if (!d->bar[5].base || d->bar[5].is_io) { cprintf("ahci: no ABAR; not driving this controller\n"); return -1; } pci_enable(d, PCI_CMD_MEMORY | PCI_CMD_BUS_MASTER); volatile uint8 *hba = ioremap(d->bar[5].base, d->bar[5].size ? d->bar[5].size : 0x1100); if (!hba) return -1; bios_handoff(hba); // AHCI mode, with interrupts off at the HBA. No HBA reset: firmware has // already brought the links up, and resetting throws that away for a // second or more of COMRESETs per port to get it back. wr(hba, HBA_GHC, (rd(hba, HBA_GHC) | GHC_AE) & ~(1u << 1)); // IE off uint32 cap = rd(hba, HBA_CAP), pi = rd(hba, HBA_PI); uint32 vs = rd(hba, HBA_VS); kinfo("ahci: controller %x:%x, AHCI %d.%d, %d ports\n", d->vendor, d->device, vs >> 16, (vs >> 8) & 0xFF, (cap & 0x1F) + 1); for (int port = 0; port < AHCI_MAX_PORTS; port++) if (pi & (1u << port)) port_init(hba, port, cap); nhba++; return 0; } static const struct pci_match ahci_match[] = { { PCI_ANY, PCI_ANY, 0x01, 0x06 }, // mass storage, SATA { 0, 0, 0, 0 }, }; const struct pci_driver ahci_driver = { .name = "ahci", .devname = "sata", .match = ahci_match, .probe = ahci_probe, };