nexoniarz/SchOS
nexoniarz/SchOS
Watch
1
0
Fork
You've already forked SchOS
 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions
SchOS/kernel/drivers/storage/ahci/ahci.c
Nexoniarz a1c965da1e include: one folder per subsystem, and includes that name it 
Sixty-five headers sat side by side in include/, so `#include "irq.h"`
said nothing about where irq.h was or what it belonged to, and errno.h,
stat.h and fcntl.h at the top shadowed the system's for any host tool
built with -I./include. They are in folders now -- arch/ core/ fs/
dev/ gfx/ net/ uapi/ (and ldls/ for the Linux driver layer) -- and
every include names the folder, as Linux's do: "core/proc.h".

Done mechanically and checked the hard way: the kernel built from this
tree disassembles identically to the one built from the tree before
it. The full image boots, and userland is the same set of programs.
Paths in docs and comments follow; docs/guides/layout.md has a table
of what is where, and tools/fix-includes.py rewrites old-style
includes from the folders in the tree, for code written before this.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S1giNvrHTrPMPw7V6x1kQ4
2026-10-23 15:05:12 +02:00

481 lines
16 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0-or-later
//
// AHCI: SATA disks, written from the Serial ATA AHCI 1.3.1 specification.
//
// This is how every PC since about 2008 attaches its internal disk, and the
// IDE driver beside it is how almost none of them do. Under QEMU's q35 the
// boot disk is here, which is why that machine used to stop with "no root
// filesystem" while the very same image booted on i440fx.
//
// The model is small. The HBA has up to 32 ports; each has a command list of
// up to 32 slots, each slot points at a command table, and the table holds
// the ATA command as a register FIS plus a scatter list of where the data
// goes. Issue a command by setting its bit in PxCI; the HBA clears the bit
// when the command is done.
//
// One slot per port and **completion by polling**, deliberately. A disk is
// on the path to a shell, and polling PxCI is the one way of finishing a
// command that does not depend on interrupt routing -- which on real hardware
// without an AML interpreter is the part that cannot be trusted (see
// kernel/dev/irq.c). A SATA command completes in tens of microseconds to a
// few milliseconds, so the cost is that, spent spinning, and it is yielded
// once a wait goes long.
#include "core/types.h"
#include "core/defs.h"
#include "core/param.h"
#include "arch/x86.h"
#include "core/spinlock.h"
#include "core/sleeplock.h"
#include "fs/fs.h"
#include "fs/file.h"
#include "core/proc.h"
#include "dev/pci.h"
#include "core/dma.h"
#include "fs/blkdev.h"
// --- HBA registers (section 3.1) ---------------------------------------------
#define HBA_CAP 0x00
#define CAP_S64A (1u << 31) // 64-bit DMA addresses
#define CAP_SSS (1u << 27) // staggered spin-up: ports start spun down
#define HBA_GHC 0x04
#define GHC_AE (1u << 31) // AHCI enable
#define GHC_HR (1u << 0) // HBA reset
#define HBA_PI 0x0C // ports implemented
#define HBA_VS 0x10
#define HBA_CAP2 0x24
#define CAP2_BOH (1u << 0) // BIOS/OS handoff supported
#define HBA_BOHC 0x28
#define BOHC_BOS (1u << 0) // the BIOS owns the HBA
#define BOHC_OOS (1u << 1) // the OS asks for it
#define BOHC_BB (1u << 4) // BIOS busy
// --- port registers (section 3.3), at 0x100 + port * 0x80 --------------------
#define PX_CLB 0x00
#define PX_CLBU 0x04
#define PX_FB 0x08
#define PX_FBU 0x0C
#define PX_IS 0x10
#define PX_IE 0x14
#define PX_CMD 0x18
#define CMD_ST (1u << 0) // start processing the command list
#define CMD_SUD (1u << 1) // spin up
#define CMD_POD (1u << 2) // power on
#define CMD_FRE (1u << 4) // FIS receive enable
#define CMD_FR (1u << 14) // FIS receive running
#define CMD_CR (1u << 15) // command list running
#define PX_TFD 0x20
#define TFD_ERR 0x01
#define TFD_DRQ 0x08
#define TFD_BSY 0x80
#define PX_SIG 0x24
#define SIG_ATA 0x00000101
#define SIG_ATAPI 0xEB140101
#define PX_SSTS 0x28
#define PX_SCTL 0x2C
#define PX_SERR 0x30
#define PX_CI 0x38
#define IS_TFES (1u << 30) // task file error
// --- ATA ---------------------------------------------------------------------
#define ATA_IDENTIFY 0xEC
#define ATA_READ_DMA_EXT 0x25
#define ATA_WRITE_DMA_EXT 0x35
#define ATA_FLUSH_EXT 0xEA
#define FIS_H2D 0x27
// One command header (section 4.2.2).
struct cmdhdr {
uint16 flags; // CFL in 4:0, A 5, W 6, P 7, R 8, B 9, C 10
uint16 prdtl; // scatter entries in the table
uint32 prdbc; // bytes transferred, written by the HBA
uint32 ctba, ctbau; // the command table, 128-byte aligned
uint32 rsv[4];
};
struct prd {
uint32 dba, dbau;
uint32 rsv;
uint32 dbc; // byte count - 1 in 21:0, interrupt-on-completion 31
};
// The command table (section 4.2.3): the FIS, an ATAPI command we never send,
// and one scatter entry -- every transfer here is one bounce buffer.
struct cmdtbl {
uint8 cfis[64];
uint8 acmd[16];
uint8 rsv[48];
struct prd prdt[1];
};
#define AHCI_MAX_PORTS 32
#define AHCI_MAX_DISKS 8
struct ahci_port {
volatile uint8 *r; // this port's registers
struct cmdhdr *cl; // command list, 1 KB
uint8 *fis; // received FIS area, 256 bytes
struct cmdtbl *ct; // the one command table
uint8 *xfer; // bounce buffer, BSIZE
uint64 nsectors;
uint32 secsize;
struct sleeplock lock; // one command at a time on this port
char name[8];
int dev;
};
static struct ahci_port disks[AHCI_MAX_DISKS];
static int ndisks;
static int nhba;
static inline uint32 rd(volatile uint8 *b, int o) { return *(volatile uint32 *)(b + o); }
static inline void wr(volatile uint8 *b, int o, uint32 v) { *(volatile uint32 *)(b + o) = v; }
// Spin until (reg & mask) == want, for at most `ms`. Returns 0 or -1.
static int
waitreg(volatile uint8 *b, int o, uint32 mask, uint32 want, int ms)
{
for (int i = 0; i < ms * 100; i++) {
if ((rd(b, o) & mask) == want)
return 0;
microdelay(10);
}
return (rd(b, o) & mask) == want ? 0 : -1;
}
// Stop the port's engines, which is the precondition for touching CLB and FB.
// Section 10.1.2: clear ST and wait for CR, clear FRE and wait for FR.
static int
port_stop(volatile uint8 *r)
{
wr(r, PX_CMD, rd(r, PX_CMD) & ~CMD_ST);
if (waitreg(r, PX_CMD, CMD_CR, 0, 500) < 0)
return -1;
wr(r, PX_CMD, rd(r, PX_CMD) & ~CMD_FRE);
return waitreg(r, PX_CMD, CMD_FR, 0, 500);
}
// The link, if any. DET = 3 is "device present and communication established"
// and IPM = 1 is "active". Firmware normally leaves a link up; when it has not,
// a COMRESET (section 10.4.2) brings it up, and a port with nothing on it
// simply never answers.
static int
port_link(volatile uint8 *r)
{
// A drive told to spin up a moment ago takes a few milliseconds to be seen
// at all; give it twenty before calling the port empty.
uint32 ssts = rd(r, PX_SSTS);
for (int i = 0; i < 20 && (ssts & 0xF) == 0; i++) {
microdelay(1000);
ssts = rd(r, PX_SSTS);
}
if ((ssts & 0xF) == 3)
return 1;
// DET = 0 is "no device detected": an empty port, not a slow one. This
// used to COMRESET every empty port and wait a second for an answer that
// could not come -- four seconds of boot on a q35 with one disk, all of it
// with interrupts off. Only a device that is detected without a link
// (DET = 1) is worth the reset.
if ((ssts & 0xF) != 1)
return 0;
wr(r, PX_SCTL, (rd(r, PX_SCTL) & ~0xFu) | 1); // DET = 1: COMRESET
microdelay(2000); // at least 1 ms
wr(r, PX_SCTL, rd(r, PX_SCTL) & ~0xFu);
for (int i = 0; i < 100; i++) { // up to 1 s for the link
if ((rd(r, PX_SSTS) & 0xF) == 3)
return 1;
microdelay(10000);
}
return 0;
}
// Issue the command in slot 0 and wait for it. `buf` is BSIZE bytes of the
// port's bounce buffer, which is where the data goes or comes from.
static int
port_exec(struct ahci_port *p, uint8 cmd, uint64 lba, uint32 count,
uint32 bytes, int write)
{
volatile uint8 *r = p->r;
struct cmdtbl *t = p->ct;
// Wait out anything still busy from before. A port that stays busy is one
// the drive has wedged; failing is better than issuing into it.
if (waitreg(r, PX_TFD, TFD_BSY | TFD_DRQ, 0, 1000) < 0)
return -1;
memset(t, 0, sizeof(*t));
uint8 *f = t->cfis;
f[0] = FIS_H2D;
f[1] = 0x80; // C: this is a command, not a control
f[2] = cmd;
f[4] = lba; f[5] = lba >> 8; f[6] = lba >> 16;
f[7] = 0x40; // LBA mode
f[8] = lba >> 24; f[9] = lba >> 32; f[10] = lba >> 40;
f[12] = count; f[13] = count >> 8;
struct cmdhdr *h = &p->cl[0];
h->flags = 5 | (write ? (1 << 6) : 0); // a register FIS is five dwords
h->prdbc = 0;
if (bytes) {
uint64 pa = dma_phys(p->xfer);
t->prdt[0].dba = (uint32)pa;
t->prdt[0].dbau = (uint32)(pa >> 32);
t->prdt[0].dbc = bytes - 1;
h->prdtl = 1;
} else {
h->prdtl = 0;
}
wr(r, PX_IS, 0xFFFFFFFF);
__sync_synchronize();
wr(r, PX_CI, 1);
// Spin for the first few milliseconds, which is when nearly every command
// finishes; after that give the processor away between looks. yield() only
// from a process -- the root probe and IDENTIFY run before there is one.
uint64 start = rdtsc(), limit = tsc_hz * 10; // ten seconds
while (rd(r, PX_CI) & 1) {
if (rd(r, PX_IS) & IS_TFES)
break;
uint64 now = rdtsc() - start;
if (now > limit) {
cprintf("ahci: %s: command %x timed out\n", p->name, cmd);
return -1;
}
if (now > tsc_hz / 200 && myproc())
yield();
else
asm volatile("pause");
}
if ((rd(r, PX_IS) & IS_TFES) || (rd(r, PX_TFD) & TFD_ERR)) {
cprintf("ahci: %s: command %x failed, tfd %x\n", p->name, cmd,
rd(r, PX_TFD));
return -1;
}
return 0;
}
static int
ahci_rw(void *priv, uint64 lba, uint32 count, void *buf, int write)
{
struct ahci_port *p = priv;
uint32 bytes = count * p->secsize;
int rc;
if (bytes > BSIZE)
return -1;
acquiresleep(&p->lock);
if (write)
memmove(p->xfer, buf, bytes);
rc = port_exec(p, write ? ATA_WRITE_DMA_EXT : ATA_READ_DMA_EXT,
lba, count, bytes, write);
if (rc == 0 && !write)
memmove(buf, p->xfer, bytes);
releasesleep(&p->lock);
return rc;
}
static int
ahci_flush(void *priv)
{
struct ahci_port *p = priv;
acquiresleep(&p->lock);
int rc = port_exec(p, ATA_FLUSH_EXT, 0, 0, 0, 0);
releasesleep(&p->lock);
return rc;
}
static const struct blkdev_ops ahci_ops = {
.rw = ahci_rw,
.flush = ahci_flush,
};
// IDENTIFY DEVICE's model string: forty characters, byte-swapped in pairs,
// padded with spaces.
static void
model_of(const uint16 *id, char *out, int cap)
{
int n = 0;
for (int w = 27; w < 47 && n + 2 < cap; w++) {
out[n++] = id[w] >> 8;
out[n++] = id[w] & 0xFF;
}
while (n > 0 && out[n - 1] == ' ')
n--;
out[n] = 0;
}
// Bring one port up and, if a SATA disk answers on it, register it.
static void
port_init(volatile uint8 *hba, int port, uint32 cap)
{
volatile uint8 *r = hba + 0x100 + port * 0x80;
if (ndisks >= AHCI_MAX_DISKS)
return;
if (port_stop(r) < 0) {
cprintf("ahci: port %d will not stop; skipping it\n", port);
return;
}
// Power and spin up first: with staggered spin-up the drive is not even
// turning until SUD is set, and its link never comes up until it is.
if (cap & CAP_SSS)
wr(r, PX_CMD, rd(r, PX_CMD) | CMD_POD | CMD_SUD);
if (!port_link(r))
return; // nothing plugged in
struct ahci_port *p = &disks[ndisks];
memset(p, 0, sizeof(*p));
p->r = r;
p->cl = dma_alloc(1024, 1024);
p->fis = dma_alloc(256, 256);
p->ct = dma_alloc(sizeof(struct cmdtbl), 128);
p->xfer = dma_alloc(BSIZE, 4096);
if (!p->cl || !p->fis || !p->ct || !p->xfer) {
cprintf("ahci: no DMA memory for port %d\n", port);
return;
}
// Without S64A the HBA cannot see above 4 GB. The DMA arena is below it
// (see include/core/dma.h), so this holds, and says so if that ever changes.
if (!(cap & CAP_S64A) && dma_phys(p->xfer) + BSIZE > 0xFFFFFFFFULL) {
cprintf("ahci: DMA memory above 4 GB and the HBA is 32-bit\n");
return;
}
memset(p->cl, 0, 1024);
memset(p->fis, 0, 256);
uint64 cl = dma_phys(p->cl), fb = dma_phys(p->fis), ct = dma_phys(p->ct);
p->cl[0].ctba = (uint32)ct;
p->cl[0].ctbau = (uint32)(ct >> 32);
wr(r, PX_CLB, (uint32)cl);
wr(r, PX_CLBU, (uint32)(cl >> 32));
wr(r, PX_FB, (uint32)fb);
wr(r, PX_FBU, (uint32)(fb >> 32));
wr(r, PX_SERR, 0xFFFFFFFF); // write-1-to-clear
wr(r, PX_IS, 0xFFFFFFFF);
wr(r, PX_IE, 0); // polled; see the top of the file
wr(r, PX_CMD, rd(r, PX_CMD) | CMD_FRE);
// The drive's first D2H FIS sets the signature; until then it reads all
// ones. ATAPI (an optical drive) and port multipliers are not disks here.
if (waitreg(r, PX_TFD, TFD_BSY | TFD_DRQ, 0, 3000) < 0) {
cprintf("ahci: port %d: the drive stays busy\n", port);
return;
}
uint32 sig = rd(r, PX_SIG);
if (sig != SIG_ATA) {
if (sig == SIG_ATAPI)
kinfo("ahci: port %d: ATAPI device, not used\n", port);
return;
}
wr(r, PX_CMD, rd(r, PX_CMD) | CMD_ST);
p->name[0] = 's'; p->name[1] = 'a'; p->name[2] = 't'; p->name[3] = 'a';
p->name[4] = '0' + ndisks; p->name[5] = 0;
initsleeplock(&p->lock, "ahci");
p->secsize = 512;
if (port_exec(p, ATA_IDENTIFY, 0, 0, 512, 0) < 0) {
cprintf("ahci: port %d: IDENTIFY failed\n", port);
return;
}
uint16 *id = (uint16 *)p->xfer;
if (!(id[83] & (1 << 10))) {
cprintf("ahci: port %d: no 48-bit LBA; not supported\n", port);
return;
}
p->nsectors = (uint64)id[100] | (uint64)id[101] << 16 |
(uint64)id[102] << 32 | (uint64)id[103] << 48;
// A logical sector larger than 512 (word 106 bit 12, size in 117-118 as
// words). Rare, and handled: the block layer works in BSIZE and asks for
// BSIZE / secsize of them.
if ((id[106] & 0xC000) == 0x4000 && (id[106] & (1 << 12)))
p->secsize = 2 * ((uint32)id[117] | (uint32)id[118] << 16);
if (p->secsize < 512 || p->secsize > BSIZE || BSIZE % p->secsize) {
cprintf("ahci: port %d: sector size %d not supported\n", port, p->secsize);
return;
}
char model[44];
model_of(id, model, sizeof(model));
ndisks++;
kinfo("ahci: %s: %s, %d MB, port %d\n", p->name, model,
(int)((p->nsectors * p->secsize) >> 20), port);
p->dev = blkdev_register(p->name, &ahci_ops, p, p->nsectors, p->secsize);
}
// Take the HBA from the firmware if it is holding it (section 10.6). Most
// firmware lets go at ExitBootServices; the specification says to ask, and
// some does not let go until asked.
static void
bios_handoff(volatile uint8 *hba)
{
if (!(rd(hba, HBA_CAP2) & CAP2_BOH))
return;
wr(hba, HBA_BOHC, rd(hba, HBA_BOHC) | BOHC_OOS);
if (waitreg(hba, HBA_BOHC, BOHC_BOS, 0, 25) < 0 ||
(rd(hba, HBA_BOHC) & BOHC_BB)) {
// BIOS busy: it is allowed two seconds to finish what it is doing.
waitreg(hba, HBA_BOHC, BOHC_BB | BOHC_BOS, 0, 2000);
}
}
static int
ahci_probe(struct pci_dev *d)
{
if (d->prog_if != 0x01)
return -1; // a RAID or IDE-mode controller
if (!d->bar[5].base || d->bar[5].is_io) {
cprintf("ahci: no ABAR; not driving this controller\n");
return -1;
}
pci_enable(d, PCI_CMD_MEMORY | PCI_CMD_BUS_MASTER);
volatile uint8 *hba = ioremap(d->bar[5].base,
d->bar[5].size ? d->bar[5].size : 0x1100);
if (!hba)
return -1;
bios_handoff(hba);
// AHCI mode, with interrupts off at the HBA. No HBA reset: firmware has
// already brought the links up, and resetting throws that away for a
// second or more of COMRESETs per port to get it back.
wr(hba, HBA_GHC, (rd(hba, HBA_GHC) | GHC_AE) & ~(1u << 1)); // IE off
uint32 cap = rd(hba, HBA_CAP), pi = rd(hba, HBA_PI);
uint32 vs = rd(hba, HBA_VS);
kinfo("ahci: controller %x:%x, AHCI %d.%d, %d ports\n",
d->vendor, d->device, vs >> 16, (vs >> 8) & 0xFF, (cap & 0x1F) + 1);
for (int port = 0; port < AHCI_MAX_PORTS; port++)
if (pi & (1u << port))
port_init(hba, port, cap);
nhba++;
return 0;
}
static const struct pci_match ahci_match[] = {
{ PCI_ANY, PCI_ANY, 0x01, 0x06 }, // mass storage, SATA
{ 0, 0, 0, 0 },
};
const struct pci_driver ahci_driver = {
.name = "ahci",
.devname = "sata",
.match = ahci_match,
.probe = ahci_probe,
};
Reference in a new issue View git blame Copy permalink