1#include <asm.h>
2#include <block/bio.h>
3#include <block/block.h>
4#include <block/sched.h>
5#include <console/printf.h>
6#include <drivers/mmio.h>
7#include <drivers/nvme.h>
8#include <enforce.h>
9#include <irq/idt.h>
10#include <math/bit.h>
11#include <mem/alloc.h>
12#include <mem/pmm.h>
13#include <mem/vmm.h>
14#include <stdbool.h>
15#include <stdint.h>
16#include <string.h>
17#include <test/export.h>
18#include <thread/workqueue.h>
19#include <time/spin_sleep.h>
20
21#include "internal.h"
22#include <thread/thread.h>
23
24static enum bio_request_status nvme_to_bio_status(uint16_t status_word) {
25 uint16_t status = (status_word >> 1) & 0x7FFF;
26 if (status == 0)
27 return BIO_STATUS_OK;
28 if (status == NVME_STATUS_CONFLICTING_ATTRIBUTES)
29 return BIO_STATUS_INVAL_ARG;
30 if (status == NVME_STATUS_INVALID_PROT_INFO)
31 return BIO_STATUS_INVAL_INTERNAL;
32
33 return BIO_STATUS_UNKNOWN_ERR;
34}
35TEST_EXPORT(nvme_to_bio_status);
36
37void nvme_send_waiters(struct nvme_device *dev) {
38 struct nvme_waiting_requests *waiters = &dev->waiting_requests;
39 struct nvme_request *next = NULL;
40
41 enum irql irql = spin_lock_high(&waiters->lock);
42
43 struct list_node *pop = list_head_pop_front_init(h: &waiters->list);
44 if (!pop)
45 goto done;
46
47 next = container_of(pop, struct nvme_request, list_node);
48
49done:
50 spin_unlock(&waiters->lock, irql);
51
52 if (next)
53 nvme_send_nvme_req(d: dev->generic_disk, r: next);
54}
55
56static void nvme_process_one(struct nvme_device *dev,
57 struct nvme_request *req) {
58 cc_ignore(dev);
59 bool has_waiter = req->has_waiter;
60
61 if (--req->remaining_parts == 0) {
62 if (req->bio_data->prp_list_phys)
63 pmm_free_page(addr: req->bio_data->prp_list_phys);
64
65 vfree(req->bio_data->prps);
66 vfree(req->bio_data);
67 req->done = true;
68 req->status = nvme_to_bio_status(status_word: req->status);
69 if (req->on_complete)
70 req->on_complete(req);
71 if (has_waiter)
72 event_set_full(e: &req->evt, reason: THREAD_WAKE_REASON_BLOCKING_IO, NULL);
73 }
74}
75
76static struct nvme_request *nvme_finished_pop_front(struct nvme_device *dev) {
77 enum irql irql = spin_lock_high(&dev->finished_requests.lock);
78
79 struct list_node *lh =
80 list_head_pop_front_init(h: &dev->finished_requests.list);
81
82 spin_unlock(&dev->finished_requests.lock, irql);
83
84 if (!lh)
85 return NULL;
86
87 return container_of(lh, struct nvme_request, list_node);
88}
89
90void nvme_work(void *dvoid, void *nothing) {
91 cc_ignore(nothing);
92
93 struct nvme_device *dev = dvoid;
94 struct nvme_request *req;
95 while (true) {
96 while ((req = nvme_finished_pop_front(dev)) != NULL) {
97 nvme_process_one(dev, req);
98 nvme_send_waiters(dev);
99 }
100
101 atomic_store(&dev->on_sem, true);
102 semaphore_wait(s: &dev->sem);
103 }
104}
105
106static void nvme_process_completions(struct nvme_device *dev, uint32_t qid) {
107 struct nvme_queue *queue = dev->io_queues[qid];
108
109 enum irql irql = spin_lock_high(&queue->lock);
110
111 while (true) {
112 struct nvme_completion *entry = &queue->cq[queue->cq_head];
113
114 if ((mmio_read_32(address: (void cc_mem_io *) &entry->status) & 1) !=
115 queue->cq_phase)
116 break;
117
118 uint16_t status =
119 mmio_read_32(address: (void cc_mem_io *) &entry->status) & 0xFFFE;
120 uint16_t cid = mmio_read_32(address: (void cc_mem_io *) &entry->cid);
121
122 /* CIDs are owned by one in-flight rq, and we release the slot here.
123 *
124 * CIDs out of range, or second completions for retired CIDs
125 * refer to requests that could already be freed, so we need
126 * to drop those and keep draining, else other
127 * completions later on get lost */
128 struct nvme_request *req =
129 cid < queue->sq_depth ? queue->sq_requests[cid] : NULL;
130
131 if (cc_unlikely(!req)) {
132 nvme_log(LOG_ERROR,
133 "spurious completion on qid %u: cid %u, status 0x%x", qid,
134 cid, status);
135 } else {
136 queue->sq_requests[cid] = NULL;
137
138 req->status = status;
139
140 enum irql irql2 = spin_lock_high(&dev->finished_requests.lock);
141
142 list_head_push_back(h: &dev->finished_requests.list, n: &req->list_node);
143
144 spin_unlock(&dev->finished_requests.lock, irql2);
145
146 atomic_dec(&queue->outstanding);
147 atomic_dec(&dev->total_outstanding);
148 }
149
150 queue->cq_head = (queue->cq_head + 1) % queue->cq_depth;
151 if (queue->cq_head == 0)
152 queue->cq_phase ^= 1;
153
154 mmio_write_32(address: queue->cq_db, value: queue->cq_head);
155 }
156
157 spin_unlock(&queue->lock, irql);
158
159 semaphore_post(s: &dev->sem);
160}
161
162enum irq_result nvme_isr_handler(struct irq_context *rsp, void *ctx) {
163 cc_ignore(rsp);
164 struct nvme_device *dev = ctx;
165 nvme_process_completions(dev, THIS_QID(dev));
166 return IRQ_HANDLED;
167}
168
169/* Place a command on the queue and use the tail slot's index as the CID.
170 *
171 * Return false if the thread is owned by an in-flight request, with
172 * outstanding counters moving under a lock */
173bool nvme_submit_io_cmd(struct nvme_device *nvme, struct nvme_command *cmd,
174 uint32_t qid, struct nvme_request *req) {
175 struct nvme_queue *this_queue = nvme->io_queues[qid];
176
177 enum irql irql = spin_lock_high(&this_queue->lock);
178
179 uint16_t tail = this_queue->sq_tail;
180
181 /* A ring of sq_depth entries holds at MOST sq_depth -1 */
182 if (atomic_load(&this_queue->outstanding) >= this_queue->sq_depth - 1) {
183 spin_unlock(&this_queue->lock, irql);
184 return false;
185 }
186
187 if (this_queue->sq_requests[tail]) {
188 spin_unlock(&this_queue->lock, irql);
189 return false;
190 }
191
192 uint16_t next_tail = (tail + 1) % this_queue->sq_depth;
193
194 cmd->cid = tail;
195
196 this_queue->sq[tail] = *cmd;
197 this_queue->sq_requests[tail] = req;
198
199 req->status = BIO_STATUS_INFLIGHT; /* In flight */
200
201 this_queue->sq_tail = next_tail;
202
203 atomic_inc(&this_queue->outstanding);
204 atomic_inc(&nvme->total_outstanding);
205
206 mmio_write_32(address: this_queue->sq_db, value: next_tail);
207
208 spin_unlock(&this_queue->lock, irql);
209
210 return true;
211}
212
213uint16_t nvme_submit_admin_cmd(struct nvme_device *nvme,
214 struct nvme_command *cmd, uint32_t *dw0_out) {
215 uint16_t tail = nvme->admin_sq_tail;
216 uint16_t next_tail = (tail + 1) % nvme->admin_q_depth;
217
218 cmd->cid = tail;
219 nvme->admin_sq[tail] = *cmd;
220
221 nvme->admin_sq_tail = next_tail;
222
223 mmio_write_32(address: nvme->admin_sq_db, value: nvme->admin_sq_tail);
224
225 uint64_t timeout = NVME_ADMIN_TIMEOUT_MS * 1000;
226 while (true) {
227 struct nvme_completion *entry = &nvme->admin_cq[nvme->admin_cq_head];
228
229 if ((mmio_read_16(address: (void cc_mem_io *) &entry->status) & 1) ==
230 nvme->admin_cq_phase) {
231 if (mmio_read_16(address: (void cc_mem_io *) &entry->cid) == cmd->cid) {
232 uint16_t status = entry->status & 0xFFFE;
233
234 nvme->admin_cq_head =
235 (nvme->admin_cq_head + 1) % nvme->admin_q_depth;
236
237 if (nvme->admin_cq_head == 0)
238 nvme->admin_cq_phase ^= 1;
239
240 mmio_write_32(address: nvme->admin_cq_db, value: nvme->admin_cq_head);
241 if (dw0_out)
242 *dw0_out = entry->result;
243 return status;
244 }
245 }
246 sleep_spin_us(us: 10);
247 timeout--;
248 if (timeout == 0)
249 return 0xFFFF;
250 }
251}
252
253uint8_t *nvme_identify_controller(struct nvme_device *nvme) {
254 uint64_t buffer_phys = pmm_alloc_page();
255 nvme_check_dma_addr(phys: buffer_phys, what: "identify buffer");
256
257 void *buffer = mmio_map_dma(phys: buffer_phys, PAGE_SIZE);
258
259 memset(buffer, 0, PAGE_SIZE);
260
261 struct nvme_command cmd = {0};
262 cmd.opc = NVME_OP_ADMIN_IDENT; // IDENTIFY opcode
263 cmd.fuse = 0; // normal
264 cmd.nsid = 1; // not used for controller ID
265 cmd.prp1 = buffer_phys;
266 cmd.cdw10 = 1; // identify controller
267
268 uint16_t status = nvme_submit_admin_cmd(nvme, cmd: &cmd, NULL);
269
270 if (status) {
271 nvme_log(LOG_ERROR, "IDENTIFY failed! Status: 0x%04X\n", status);
272 return NULL;
273 }
274
275 uint8_t *data = (uint8_t *) buffer;
276 return data;
277}
278
279uint32_t nvme_set_num_queues(struct nvme_device *nvme, uint16_t desired_sq,
280 uint16_t desired_cq) {
281 struct nvme_command cmd = {0};
282 cmd.opc = NVME_OP_ADMIN_SET_FEATS;
283 cmd.cdw10 = 0x07;
284 cmd.cdw11 =
285 ((uint32_t) (desired_cq - 1) << 16) | ((desired_sq - 1) & 0xFFFF);
286
287 uint32_t cdw0;
288 uint16_t status = nvme_submit_admin_cmd(nvme, cmd: &cmd, dw0_out: &cdw0);
289
290 if (status) {
291 nvme_log(LOG_ERROR,
292 "SET FEATURES (Number of Queues) failed! Status: 0x%04X",
293 status);
294 return 0;
295 }
296
297 uint16_t actual_sq = (cdw0 & 0xFFFF) + 1;
298 uint16_t actual_cq = ((cdw0 >> 16) & 0xFFFF) + 1;
299
300 return (actual_cq << 16) | actual_sq;
301}
302
303uint8_t *nvme_identify_namespace(struct nvme_device *nvme, uint32_t nsid) {
304 uint64_t buffer_phys = pmm_alloc_page();
305 nvme_check_dma_addr(phys: buffer_phys, what: "identify buffer");
306
307 void *buffer = mmio_map_dma(phys: buffer_phys, PAGE_SIZE);
308
309 memset(buffer, 0, PAGE_SIZE);
310
311 struct nvme_command cmd = {0};
312 cmd.opc = NVME_OP_ADMIN_IDENT; // IDENTIFY opcode
313 cmd.fuse = 0; // normal
314 cmd.nsid = nsid; // namespace ID to identify
315 cmd.prp1 = buffer_phys;
316 cmd.cdw10 = 0; // Identify Namespace (CNS=0)
317
318 uint16_t status = nvme_submit_admin_cmd(nvme, cmd: &cmd, NULL);
319
320 if (status) {
321 nvme_log(LOG_ERROR, "IDENTIFY namespace failed! Status: 0x%04X",
322 status);
323 return NULL;
324 }
325
326 struct nvme_identify_namespace *ns = (void *) buffer;
327 uint8_t flbas_index = ns->flbas & 0xF; // lower 4 bits = selected format
328 uint8_t lbads = ns->lbaf[flbas_index].lbads;
329 uint32_t sector_size = BIT(lbads);
330 nvme_log(LOG_INFO, "Device sector size is %u bytes", sector_size);
331
332 nvme->sector_size = sector_size;
333 return (uint8_t *) buffer;
334}
335