1#include <acpi/lapic.h>
2#include <asm.h>
3#include <block/bio.h>
4#include <block/block.h>
5#include <block/sched.h>
6#include <console/printf.h>
7#include <drivers/mmio.h>
8#include <drivers/nvme.h>
9#include <irq/idt.h>
10#include <kassert.h>
11#include <mem/alloc.h>
12#include <mem/pmm.h>
13#include <mem/vmm.h>
14#include <stdbool.h>
15#include <stdint.h>
16#include <string.h>
17#include <thread/workqueue.h>
18#include <time/spin_sleep.h>
19
20#include "internal.h"
21
22static enum bio_request_status nvme_to_bio_status(uint16_t status) {
23 if (status == NVME_STATUS_CONFLICTING_ATTRIBUTES) {
24 return BIO_STATUS_INVAL_ARG;
25 } else if (status == NVME_STATUS_INVALID_PROT_INFO) {
26 return BIO_STATUS_INVAL_INTERNAL;
27 }
28
29 return BIO_STATUS_OK;
30}
31
32static void nvme_send_waiters(struct nvme_device *dev) {
33 struct nvme_waiting_requests *waiters = &dev->waiting_requests;
34 struct nvme_request *next = NULL;
35
36 enum irql irql = spin_lock_irq_disable(lock: &waiters->lock);
37
38 struct list_head *pop = list_pop_front_init(head: &waiters->list);
39 if (!pop)
40 goto done;
41
42 next = container_of(pop, struct nvme_request, list_node);
43
44done:
45 spin_unlock(lock: &waiters->lock, old: irql);
46
47 if (next)
48 nvme_send_nvme_req(d: dev->generic_disk, r: next);
49}
50
51static void nvme_process_one(struct nvme_device *dev,
52 struct nvme_request *req) {
53 struct thread *t = req->waiter;
54
55 if (--req->remaining_parts == 0) {
56 kfree(req->bio_data->prps);
57 kfree(req->bio_data);
58 req->done = true;
59 req->status = nvme_to_bio_status(status: req->status);
60 if (req->on_complete)
61 req->on_complete(req);
62 }
63
64 if (t)
65 thread_wake_from_io_block(t, wake_src: dev);
66}
67
68static struct nvme_request *nvme_finished_pop_front(struct nvme_device *dev) {
69 enum irql irql = spin_lock_irq_disable(lock: &dev->finished_requests.lock);
70
71 struct list_head *lh = list_pop_front_init(head: &dev->finished_requests.list);
72
73 spin_unlock(lock: &dev->finished_requests.lock, old: irql);
74
75 if (!lh)
76 return NULL;
77
78 return container_of(lh, struct nvme_request, list_node);
79}
80
81void nvme_work(void *dvoid, void *nothing) {
82 (void) nothing;
83
84 struct nvme_device *dev = dvoid;
85 struct nvme_request *req;
86 while (true) {
87 while ((req = nvme_finished_pop_front(dev)) != NULL) {
88 nvme_process_one(dev, req);
89 nvme_send_waiters(dev);
90 }
91
92 atomic_store(&dev->on_sem, true);
93 semaphore_wait(s: &dev->sem);
94 }
95}
96
97void nvme_process_completions(struct nvme_device *dev, uint32_t qid) {
98 struct nvme_queue *queue = dev->io_queues[qid];
99
100 enum irql irql = spin_lock_irq_disable(lock: &queue->lock);
101
102 while (true) {
103 struct nvme_completion *entry = &queue->cq[queue->cq_head];
104
105 if ((mmio_read_32(address: &entry->status) & 1) != queue->cq_phase)
106 break;
107
108 uint16_t status = mmio_read_32(address: &entry->status) & 0xFFFE;
109 uint16_t cid = mmio_read_32(address: &entry->cid);
110
111 struct nvme_request *req = queue->sq_requests[cid];
112
113 req->status = status;
114
115 enum irql irql2 = spin_lock_irq_disable(lock: &dev->finished_requests.lock);
116
117 list_add_tail(new: &req->list_node, head: &dev->finished_requests.list);
118
119 spin_unlock(lock: &dev->finished_requests.lock, old: irql2);
120
121 atomic_fetch_sub(&queue->outstanding, 1);
122 atomic_fetch_sub(&dev->total_outstanding, 1);
123
124 queue->cq_head = (queue->cq_head + 1) % queue->cq_depth;
125 if (queue->cq_head == 0)
126 queue->cq_phase ^= 1;
127
128 mmio_write_32(address: queue->cq_db, value: queue->cq_head);
129 }
130
131 spin_unlock(lock: &queue->lock, old: irql);
132
133 semaphore_post(s: &dev->sem);
134}
135
136enum irq_result nvme_isr_handler(void *ctx, uint8_t vector,
137 struct irq_context *rsp) {
138 (void) vector, (void) rsp;
139 struct nvme_device *dev = ctx;
140 nvme_process_completions(dev, THIS_QID(dev));
141 return IRQ_HANDLED;
142}
143
144void nvme_submit_io_cmd(struct nvme_device *nvme, struct nvme_command *cmd,
145 uint32_t qid, struct nvme_request *req) {
146 struct nvme_queue *this_queue = nvme->io_queues[qid];
147
148 atomic_fetch_add(&this_queue->outstanding, 1);
149 atomic_fetch_add(&nvme->total_outstanding, 1);
150
151 enum irql irql = spin_lock_irq_disable(lock: &this_queue->lock);
152
153 uint16_t tail = this_queue->sq_tail;
154 uint16_t next_tail = (tail + 1) % this_queue->sq_depth;
155
156 cmd->cid = tail;
157
158 this_queue->sq[tail] = *cmd;
159 this_queue->sq_requests[tail] = req;
160
161 req->status = BIO_STATUS_INFLIGHT; /* In flight */
162
163 this_queue->sq_tail = next_tail;
164
165 mmio_write_32(address: this_queue->sq_db, value: next_tail);
166
167 spin_unlock(lock: &this_queue->lock, old: irql);
168}
169
170uint16_t nvme_submit_admin_cmd(struct nvme_device *nvme,
171 struct nvme_command *cmd, uint32_t *dw0_out) {
172 uint16_t tail = nvme->admin_sq_tail;
173 uint16_t next_tail = (tail + 1) % nvme->admin_q_depth;
174
175 cmd->cid = tail;
176 nvme->admin_sq[tail] = *cmd;
177
178 nvme->admin_sq_tail = next_tail;
179
180 mmio_write_32(address: nvme->admin_sq_db, value: nvme->admin_sq_tail);
181
182 uint64_t timeout = NVME_ADMIN_TIMEOUT_MS * 1000;
183 while (true) {
184 struct nvme_completion *entry = &nvme->admin_cq[nvme->admin_cq_head];
185
186 if ((mmio_read_16(address: &entry->status) & 1) == nvme->admin_cq_phase) {
187 if (mmio_read_16(address: &entry->cid) == cmd->cid) {
188 uint16_t status = entry->status & 0xFFFE;
189
190 nvme->admin_cq_head =
191 (nvme->admin_cq_head + 1) % nvme->admin_q_depth;
192
193 if (nvme->admin_cq_head == 0)
194 nvme->admin_cq_phase ^= 1;
195
196 mmio_write_32(address: nvme->admin_cq_db, value: nvme->admin_cq_head);
197 if (dw0_out)
198 *dw0_out = entry->result;
199 return status;
200 }
201 }
202 sleep_spin_us(us: 10);
203 timeout--;
204 if (timeout == 0)
205 return 0xFFFF;
206 }
207}
208
209uint8_t *nvme_identify_controller(struct nvme_device *nvme) {
210 uint64_t buffer_phys = pmm_alloc_page();
211
212 void *buffer = mmio_map(phys: buffer_phys, PAGE_SIZE);
213
214 memset(buffer, 0, PAGE_SIZE);
215
216 struct nvme_command cmd = {0};
217 cmd.opc = NVME_OP_ADMIN_IDENT; // IDENTIFY opcode
218 cmd.fuse = 0; // normal
219 cmd.nsid = 1; // not used for controller ID
220 cmd.prp1 = buffer_phys;
221 cmd.cdw10 = 1; // identify controller
222
223 uint16_t status = nvme_submit_admin_cmd(nvme, cmd: &cmd, NULL);
224
225 if (status) {
226 nvme_log(LOG_ERROR, "IDENTIFY failed! Status: 0x%04X\n", status);
227 return NULL;
228 }
229
230 uint8_t *data = (uint8_t *) buffer;
231 return data;
232}
233
234uint32_t nvme_set_num_queues(struct nvme_device *nvme, uint16_t desired_sq,
235 uint16_t desired_cq) {
236 struct nvme_command cmd = {0};
237 cmd.opc = NVME_OP_ADMIN_SET_FEATS;
238 cmd.cdw10 = 0x07;
239 cmd.cdw11 =
240 ((uint32_t) (desired_cq - 1) << 16) | ((desired_sq - 1) & 0xFFFF);
241
242 uint32_t cdw0;
243 uint16_t status = nvme_submit_admin_cmd(nvme, cmd: &cmd, dw0_out: &cdw0);
244
245 if (status) {
246 nvme_log(LOG_ERROR,
247 "SET FEATURES (Number of Queues) failed! Status: 0x%04X",
248 status);
249 return 0;
250 }
251
252 uint16_t actual_sq = (cdw0 & 0xFFFF) + 1;
253 uint16_t actual_cq = ((cdw0 >> 16) & 0xFFFF) + 1;
254
255 return (actual_cq << 16) | actual_sq;
256}
257
258uint8_t *nvme_identify_namespace(struct nvme_device *nvme, uint32_t nsid) {
259 uint64_t buffer_phys = pmm_alloc_page();
260
261 void *buffer = mmio_map(phys: buffer_phys, PAGE_SIZE);
262
263 memset(buffer, 0, PAGE_SIZE);
264
265 struct nvme_command cmd = {0};
266 cmd.opc = NVME_OP_ADMIN_IDENT; // IDENTIFY opcode
267 cmd.fuse = 0; // normal
268 cmd.nsid = nsid; // namespace ID to identify
269 cmd.prp1 = buffer_phys;
270 cmd.cdw10 = 0; // Identify Namespace (CNS=0)
271
272 uint16_t status = nvme_submit_admin_cmd(nvme, cmd: &cmd, NULL);
273
274 if (status) {
275 nvme_log(LOG_ERROR, "IDENTIFY namespace failed! Status: 0x%04X",
276 status);
277 return NULL;
278 }
279
280 struct nvme_identify_namespace *ns = (void *) buffer;
281 uint8_t flbas_index = ns->flbas & 0xF; // lower 4 bits = selected format
282 uint8_t lbads = ns->lbaf[flbas_index].lbads;
283 uint32_t sector_size = 1U << lbads;
284 nvme_log(LOG_INFO, "Device sector size is %u bytes", sector_size);
285
286 nvme->sector_size = sector_size;
287 return (uint8_t *) buffer;
288}
289