1#include <acpi/lapic.h>
2#include <asm.h>
3#include <block/bio.h>
4#include <block/block.h>
5#include <block/sched.h>
6#include <console/printf.h>
7#include <drivers/mmio.h>
8#include <drivers/nvme.h>
9#include <irq/idt.h>
10#include <kassert.h>
11#include <mem/alloc.h>
12#include <mem/pmm.h>
13#include <mem/vmm.h>
14#include <stdbool.h>
15#include <stdint.h>
16#include <string.h>
17#include <test/export.h>
18#include <thread/workqueue.h>
19#include <time/spin_sleep.h>
20
21#include "internal.h"
22#include <thread/thread.h>
23
24static enum bio_request_status nvme_to_bio_status(uint16_t status_word) {
25 uint16_t status = (status_word >> 1) & 0x7FFF;
26 if (status == 0)
27 return BIO_STATUS_OK;
28 if (status == NVME_STATUS_CONFLICTING_ATTRIBUTES)
29 return BIO_STATUS_INVAL_ARG;
30 if (status == NVME_STATUS_INVALID_PROT_INFO)
31 return BIO_STATUS_INVAL_INTERNAL;
32
33 return BIO_STATUS_UNKNOWN_ERR;
34}
35TEST_EXPORT(nvme_to_bio_status);
36
37static void nvme_send_waiters(struct nvme_device *dev) {
38 struct nvme_waiting_requests *waiters = &dev->waiting_requests;
39 struct nvme_request *next = NULL;
40
41 enum irql irql = spin_lock_irq_disable(&waiters->lock);
42
43 struct list_head *pop = list_pop_front_init(head: &waiters->list);
44 if (!pop)
45 goto done;
46
47 next = container_of(pop, struct nvme_request, list_node);
48
49done:
50 spin_unlock(&waiters->lock, irql);
51
52 if (next)
53 nvme_send_nvme_req(d: dev->generic_disk, r: next);
54}
55
56static void nvme_process_one(struct nvme_device *dev,
57 struct nvme_request *req) {
58 struct thread *t = req->waiter;
59
60 if (--req->remaining_parts == 0) {
61 if (req->bio_data->prp_list_phys)
62 pmm_free_page(addr: req->bio_data->prp_list_phys);
63
64 kfree(req->bio_data->prps);
65 kfree(req->bio_data);
66 req->done = true;
67 req->status = nvme_to_bio_status(status_word: req->status);
68 if (req->on_complete)
69 req->on_complete(req);
70 }
71
72 if (t)
73 thread_wake_from_io_block(t, wake_src: dev);
74}
75
76static struct nvme_request *nvme_finished_pop_front(struct nvme_device *dev) {
77 enum irql irql = spin_lock_irq_disable(&dev->finished_requests.lock);
78
79 struct list_head *lh = list_pop_front_init(head: &dev->finished_requests.list);
80
81 spin_unlock(&dev->finished_requests.lock, irql);
82
83 if (!lh)
84 return NULL;
85
86 return container_of(lh, struct nvme_request, list_node);
87}
88
89void nvme_work(void *dvoid, void *nothing) {
90 (void) nothing;
91
92 struct nvme_device *dev = dvoid;
93 struct nvme_request *req;
94 while (true) {
95 while ((req = nvme_finished_pop_front(dev)) != NULL) {
96 nvme_process_one(dev, req);
97 nvme_send_waiters(dev);
98 }
99
100 atomic_store(&dev->on_sem, true);
101 semaphore_wait(s: &dev->sem);
102 }
103}
104
105void nvme_process_completions(struct nvme_device *dev, uint32_t qid) {
106 struct nvme_queue *queue = dev->io_queues[qid];
107
108 enum irql irql = spin_lock_irq_disable(&queue->lock);
109
110 while (true) {
111 struct nvme_completion *entry = &queue->cq[queue->cq_head];
112
113 if ((mmio_read_32(address: &entry->status) & 1) != queue->cq_phase)
114 break;
115
116 uint16_t status = mmio_read_32(address: &entry->status) & 0xFFFE;
117 uint16_t cid = mmio_read_32(address: &entry->cid);
118
119 struct nvme_request *req = queue->sq_requests[cid];
120
121 req->status = status;
122
123 enum irql irql2 = spin_lock_irq_disable(&dev->finished_requests.lock);
124
125 list_add_tail(new: &req->list_node, head: &dev->finished_requests.list);
126
127 spin_unlock(&dev->finished_requests.lock, irql2);
128
129 atomic_fetch_sub(&queue->outstanding, 1);
130 atomic_fetch_sub(&dev->total_outstanding, 1);
131
132 queue->cq_head = (queue->cq_head + 1) % queue->cq_depth;
133 if (queue->cq_head == 0)
134 queue->cq_phase ^= 1;
135
136 mmio_write_32(address: queue->cq_db, value: queue->cq_head);
137 }
138
139 spin_unlock(&queue->lock, irql);
140
141 semaphore_post(s: &dev->sem);
142}
143
144enum irq_result nvme_isr_handler(void *ctx, uint8_t vector,
145 struct irq_context *rsp) {
146 (void) vector, (void) rsp;
147 struct nvme_device *dev = ctx;
148 nvme_process_completions(dev, THIS_QID(dev));
149 return IRQ_HANDLED;
150}
151
152void nvme_submit_io_cmd(struct nvme_device *nvme, struct nvme_command *cmd,
153 uint32_t qid, struct nvme_request *req) {
154 struct nvme_queue *this_queue = nvme->io_queues[qid];
155
156 atomic_fetch_add(&this_queue->outstanding, 1);
157 atomic_fetch_add(&nvme->total_outstanding, 1);
158
159 enum irql irql = spin_lock_irq_disable(&this_queue->lock);
160
161 uint16_t tail = this_queue->sq_tail;
162 uint16_t next_tail = (tail + 1) % this_queue->sq_depth;
163
164 cmd->cid = tail;
165
166 this_queue->sq[tail] = *cmd;
167 this_queue->sq_requests[tail] = req;
168
169 req->status = BIO_STATUS_INFLIGHT; /* In flight */
170
171 this_queue->sq_tail = next_tail;
172
173 mmio_write_32(address: this_queue->sq_db, value: next_tail);
174
175 spin_unlock(&this_queue->lock, irql);
176}
177
178uint16_t nvme_submit_admin_cmd(struct nvme_device *nvme,
179 struct nvme_command *cmd, uint32_t *dw0_out) {
180 uint16_t tail = nvme->admin_sq_tail;
181 uint16_t next_tail = (tail + 1) % nvme->admin_q_depth;
182
183 cmd->cid = tail;
184 nvme->admin_sq[tail] = *cmd;
185
186 nvme->admin_sq_tail = next_tail;
187
188 mmio_write_32(address: nvme->admin_sq_db, value: nvme->admin_sq_tail);
189
190 uint64_t timeout = NVME_ADMIN_TIMEOUT_MS * 1000;
191 while (true) {
192 struct nvme_completion *entry = &nvme->admin_cq[nvme->admin_cq_head];
193
194 if ((mmio_read_16(address: &entry->status) & 1) == nvme->admin_cq_phase) {
195 if (mmio_read_16(address: &entry->cid) == cmd->cid) {
196 uint16_t status = entry->status & 0xFFFE;
197
198 nvme->admin_cq_head =
199 (nvme->admin_cq_head + 1) % nvme->admin_q_depth;
200
201 if (nvme->admin_cq_head == 0)
202 nvme->admin_cq_phase ^= 1;
203
204 mmio_write_32(address: nvme->admin_cq_db, value: nvme->admin_cq_head);
205 if (dw0_out)
206 *dw0_out = entry->result;
207 return status;
208 }
209 }
210 sleep_spin_us(us: 10);
211 timeout--;
212 if (timeout == 0)
213 return 0xFFFF;
214 }
215}
216
217uint8_t *nvme_identify_controller(struct nvme_device *nvme) {
218 uint64_t buffer_phys = pmm_alloc_page();
219 nvme_check_dma_addr(phys: buffer_phys, what: "identify buffer");
220
221 void *buffer = mmio_map(phys: buffer_phys, PAGE_SIZE);
222
223 memset(buffer, 0, PAGE_SIZE);
224
225 struct nvme_command cmd = {0};
226 cmd.opc = NVME_OP_ADMIN_IDENT; // IDENTIFY opcode
227 cmd.fuse = 0; // normal
228 cmd.nsid = 1; // not used for controller ID
229 cmd.prp1 = buffer_phys;
230 cmd.cdw10 = 1; // identify controller
231
232 uint16_t status = nvme_submit_admin_cmd(nvme, cmd: &cmd, NULL);
233
234 if (status) {
235 nvme_log(LOG_ERROR, "IDENTIFY failed! Status: 0x%04X\n", status);
236 return NULL;
237 }
238
239 uint8_t *data = (uint8_t *) buffer;
240 return data;
241}
242
243uint32_t nvme_set_num_queues(struct nvme_device *nvme, uint16_t desired_sq,
244 uint16_t desired_cq) {
245 struct nvme_command cmd = {0};
246 cmd.opc = NVME_OP_ADMIN_SET_FEATS;
247 cmd.cdw10 = 0x07;
248 cmd.cdw11 =
249 ((uint32_t) (desired_cq - 1) << 16) | ((desired_sq - 1) & 0xFFFF);
250
251 uint32_t cdw0;
252 uint16_t status = nvme_submit_admin_cmd(nvme, cmd: &cmd, dw0_out: &cdw0);
253
254 if (status) {
255 nvme_log(LOG_ERROR,
256 "SET FEATURES (Number of Queues) failed! Status: 0x%04X",
257 status);
258 return 0;
259 }
260
261 uint16_t actual_sq = (cdw0 & 0xFFFF) + 1;
262 uint16_t actual_cq = ((cdw0 >> 16) & 0xFFFF) + 1;
263
264 return (actual_cq << 16) | actual_sq;
265}
266
267uint8_t *nvme_identify_namespace(struct nvme_device *nvme, uint32_t nsid) {
268 uint64_t buffer_phys = pmm_alloc_page();
269 nvme_check_dma_addr(phys: buffer_phys, what: "identify buffer");
270
271 void *buffer = mmio_map(phys: buffer_phys, PAGE_SIZE);
272
273 memset(buffer, 0, PAGE_SIZE);
274
275 struct nvme_command cmd = {0};
276 cmd.opc = NVME_OP_ADMIN_IDENT; // IDENTIFY opcode
277 cmd.fuse = 0; // normal
278 cmd.nsid = nsid; // namespace ID to identify
279 cmd.prp1 = buffer_phys;
280 cmd.cdw10 = 0; // Identify Namespace (CNS=0)
281
282 uint16_t status = nvme_submit_admin_cmd(nvme, cmd: &cmd, NULL);
283
284 if (status) {
285 nvme_log(LOG_ERROR, "IDENTIFY namespace failed! Status: 0x%04X",
286 status);
287 return NULL;
288 }
289
290 struct nvme_identify_namespace *ns = (void *) buffer;
291 uint8_t flbas_index = ns->flbas & 0xF; // lower 4 bits = selected format
292 uint8_t lbads = ns->lbaf[flbas_index].lbads;
293 uint32_t sector_size = 1U << lbads;
294 nvme_log(LOG_INFO, "Device sector size is %u bytes", sector_size);
295
296 nvme->sector_size = sector_size;
297 return (uint8_t *) buffer;
298}
299