1#include <asm.h>
2#include <block/bio.h>
3#include <block/block.h>
4#include <drivers/ahci.h>
5#include <drivers/ata.h>
6#include <drivers/mmio.h>
7#include <irq/idt.h>
8#include <math/align.h>
9#include <math/bit.h>
10#include <math/min_max.h>
11#include <math/units.h>
12#include <mem/alloc.h>
13#include <mem/vmm.h>
14#include <sch/sched.h>
15#include <stdbool.h>
16#include <stdint.h>
17#include <string.h>
18#include <thread/thread.h>
19#include <time/spin_sleep.h>
20
21#define MAX_PRDT_ENTRY_SIZE MIB(4)
22
23/* TODO: horrible code - bit-op spam */
24static void ahci_process_completions(struct ahci_device *dev, uint32_t port) {
25 struct ahci_full_port *fp = &dev->regs[port];
26 struct ahci_port cc_mem_io *p = fp->port;
27
28 uint32_t ci = mmio_read_32(address: &p->ci);
29 uint32_t sact = mmio_read_32(address: &p->sact);
30 uint32_t completed = ~(ci | sact);
31
32 for (uint32_t slot = 0; slot < AHCI_MAX_SLOTS; slot++) {
33 uint32_t mask = BIT(slot);
34 if (!(completed & mask))
35 continue;
36
37 struct ahci_request *req = dev->io_requests[port][slot];
38
39 dev->io_requests[port][slot] = NULL;
40 if (req && req->trigger_completion) {
41 bool has_waiter = req->has_waiter;
42 req->done = true;
43 req->status = 0;
44 atomic_fetch_and(&fp->slot_bitmap, ~mask);
45 if (req->on_complete)
46 req->on_complete(req);
47 if (has_waiter)
48 event_set_full(e: &req->evt, reason: THREAD_WAKE_REASON_BLOCKING_IO, NULL);
49 }
50 }
51
52 mmio_write_32(address: &p->is, value: p->is);
53}
54
55enum irq_result ahci_isr_handler(struct irq_context *ictx, void *ctx) {
56 cc_ignore(ictx);
57
58 struct ahci_device *dev = ctx;
59 for (uint32_t port = 0; port < AHCI_MAX_PORTS; port++) {
60 if (!dev->regs[port].port)
61 continue;
62
63 ahci_process_completions(dev, port);
64 }
65
66 return IRQ_HANDLED;
67}
68
69void ahci_send_command(struct ahci_disk *disk, struct ahci_full_port *port,
70 struct ahci_request *req) {
71 uint32_t slot = req->slot;
72
73 mmio_write_32(address: &port->port->is, value: 0xFFFFFFFF);
74 disk->device->io_requests[disk->port][slot] = req;
75
76 uint32_t command_issue = mmio_read_32(address: &port->port->ci);
77 command_issue |= BIT(slot);
78 mmio_write_32(address: &port->port->ci, value: command_issue);
79}
80
81static uint32_t try_find_slot(struct ahci_full_port *p) {
82 while (true) {
83 uint32_t old = atomic_load_rlx(&p->slot_bitmap);
84 for (int slot = 0; slot < AHCI_MAX_SLOTS; slot++) {
85 uint32_t mask = BIT(slot);
86 if (!(old & mask)) {
87 uint32_t new_bitmap = old | mask;
88 if (atomic_cas_weak(&p->slot_bitmap, &old, new_bitmap,
89 mo_acquire, mo_rlx)) {
90 return slot;
91 }
92 break;
93 }
94 }
95 return (uint32_t) -1; // no free slot found
96 }
97}
98
99uint32_t ahci_find_slot(struct ahci_full_port *p) {
100
101 uint32_t slot = try_find_slot(p);
102
103 while (slot == (uint32_t) -1)
104 slot = try_find_slot(p);
105
106 return slot;
107}
108
109/* TODO: BUG here when there are too many concurrent requests
110 * eating up all the available slots */
111void ahci_prepare_command(struct ahci_full_port *port, uint32_t slot,
112 bool write, uint8_t *buf, uint64_t size) {
113
114 struct ahci_cmd_header *hdr = port->cmd_hdrs[slot];
115 struct ahci_cmd_table *cmd_tbl = port->cmd_tables[slot];
116
117 if (!hdr || !cmd_tbl || size == 0)
118 return;
119
120 uint64_t prdt_count = DIV_ROUND_UP(size, MAX_PRDT_ENTRY_SIZE);
121
122 if (prdt_count > 65535)
123 return;
124
125 hdr->cfl = sizeof(struct ahci_fis_reg_h2d) / sizeof(uint32_t);
126 hdr->w = write ? 1 : 0;
127 hdr->p = 0;
128 hdr->a = 0;
129 hdr->c = 1;
130 hdr->prdtl = prdt_count;
131 hdr->prdbc = 0;
132
133 uint64_t remaining = size;
134 uint64_t offset = 0;
135 uint64_t phys_base = PAGE_ALIGN_DOWN(vmm_get_phys((uint64_t) buf));
136
137 for (uint32_t i = 0; i < prdt_count; i++) {
138 uint64_t chunk = MIN(remaining, MAX_PRDT_ENTRY_SIZE);
139
140 uint64_t phys_addr = phys_base + offset;
141
142 cmd_tbl->prdt_entry[i].dba = (uint32_t) (phys_addr & 0xFFFFFFFF);
143 cmd_tbl->prdt_entry[i].dbau = (uint32_t) (phys_addr >> 32);
144 cmd_tbl->prdt_entry[i].dbc = (uint32_t) (chunk - 1); // size - 1
145 cmd_tbl->prdt_entry[i].i = (i == prdt_count - 1) ? 1 : 0;
146
147 offset += chunk;
148 remaining -= chunk;
149 }
150}
151
152void ahci_setup_fis(struct ahci_cmd_table *cmd_tbl, uint8_t command,
153 bool is_atapi) {
154 struct ahci_fis_reg_h2d *fis = (struct ahci_fis_reg_h2d *) cmd_tbl->cfis;
155 memset(fis, 0, sizeof(struct ahci_fis_reg_h2d));
156
157 fis->fis_type = FIS_TYPE_REG_H2D;
158 fis->c = 1;
159 fis->command = command;
160
161 if (is_atapi) {
162 fis->device = BIT(6); // LBA bit
163 }
164}
165
166void ahci_identify(struct ahci_disk *disk) {
167 struct ahci_full_port *port = &disk->device->regs[disk->port];
168 uint32_t slot = ahci_find_slot(p: port);
169
170 uint8_t *buffer = valloc_aligned(PAGE_SIZE, PAGE_SIZE);
171 if (!buffer)
172 return;
173
174 ahci_prepare_command(port, slot, false, buf: buffer, PAGE_SIZE);
175
176 if (!buffer)
177 return;
178
179 ahci_setup_fis(cmd_tbl: port->cmd_tables[slot], AHCI_CMD_IDENTIFY, false);
180
181 struct ahci_request req = {
182 .slot = slot, .port = disk->port, .buffer = buffer};
183 ahci_send_command(disk, port, req: &req);
184
185 uint32_t logical_sector_size = 512;
186
187 /*if (buffer[106] & (1 << 14)) {
188 uint32_t low = buffer[117];
189 uint32_t high = buffer[118];
190 logical_sector_size = ((uint32_t) high << 16) | low;
191 }*/
192
193 disk->sector_size = logical_sector_size;
194
195 ahci_log(LOG_INFO, "Sector size is %u bytes", disk->sector_size);
196
197 vfree_aligned(buffer);
198}
199
200static void ahci_on_bio_complete(struct ahci_request *req) {
201 struct bio_request *bio = (struct bio_request *) req->user_data;
202
203 bio->done = true;
204 bio->status = req->status;
205
206 if (bio->on_complete)
207 bio->on_complete(bio);
208
209 vfree(req);
210}
211
212bool ahci_submit_bio_request(struct block_device *disk,
213 struct bio_request *bio) {
214 struct ahci_disk *ahci_disk = (struct ahci_disk *) disk->driver_data;
215 struct ahci_request *ahci_req = vnew(ahci_req, ALLOC_ZERO);
216 if (!ahci_req)
217 return false;
218
219 struct ahci_full_port *p = &ahci_disk->device->regs[ahci_disk->port];
220
221 ahci_req->port = ahci_disk->port;
222 ahci_req->slot = ahci_find_slot(p);
223 ahci_req->lba = bio->lba;
224 ahci_req->buffer = bio->buffer;
225 ahci_req->sector_count = bio->sector_count;
226 ahci_req->size = bio->size;
227 ahci_req->write = bio->write;
228 ahci_req->done = false;
229
230 ahci_req->on_complete = ahci_on_bio_complete;
231 ahci_req->user_data = bio;
232
233 if (bio->write) {
234 return ahci_write_sector_async_wrapper(disk, lba: bio->lba, buf: bio->buffer,
235 cnt: bio->sector_count, req: ahci_req);
236 } else {
237 return ahci_read_sector_async_wrapper(disk, lba: bio->lba, buf: bio->buffer,
238 cnt: bio->sector_count, req: ahci_req);
239 }
240}
241