1#include <acpi/lapic.h>
2#include <asm.h>
3#include <compiler/core.h>
4#include <console/printf.h>
5#include <drivers/mmio.h>
6#include <drivers/nvme.h>
7#include <irq/idt.h>
8#include <mem/alloc.h>
9#include <mem/nofail.h>
10#include <mem/pmm.h>
11#include <mem/vmm.h>
12#include <stdbool.h>
13#include <stdint.h>
14#include <string.h>
15#include <time/spin_sleep.h>
16
17#include "internal.h"
18
19/* we poll in setup */
20void nvme_enable_controller(struct nvme_device *nvme) {
21
22 struct nvme_cc cc = {0};
23
24 cc.mps = 0;
25
26 cc.iocqes = 4; // 2 ^ 4 = 16
27
28 cc.iosqes = 6; // 2 ^ 6 = 64
29
30 cc.ams = 0;
31
32 cc.css = 0b110;
33
34 cc.en = 0;
35
36 mmio_write_32(address: &nvme->regs->cc, value: *(uint32_t *) &cc);
37
38 mmio_spin_wait(reg: &nvme->regs->csts, mask: 1, NVME_CMD_TIMEOUT_MS);
39
40 cc.en = 1;
41
42 mmio_write_32(address: &nvme->regs->cc, value: *(uint32_t *) &cc);
43
44 uint64_t timeout = NVME_CMD_TIMEOUT_MS * 1000;
45 while ((mmio_read_32(address: &nvme->regs->csts) & 1) == 0) {
46 sleep_spin_us(us: 10);
47 timeout--;
48 if (timeout == 0)
49 return;
50 }
51}
52
53void nvme_setup_admin_queues(struct nvme_device *nvme) {
54 uint32_t q_depth_minus_1 = nvme->admin_q_depth - 1;
55
56 uint32_t aqa = (q_depth_minus_1 << 16) | q_depth_minus_1;
57 mmio_write_32(address: &nvme->regs->aqa, value: aqa);
58
59 mmio_write_32(address: &nvme->regs->asq_lo, value: (nvme->admin_sq_phys & 0xFFFFFFFF));
60 mmio_write_32(address: &nvme->regs->asq_hi, value: (nvme->admin_sq_phys >> 32));
61
62 mmio_write_32(address: &nvme->regs->acq_lo, value: (nvme->admin_cq_phys & 0xFFFFFFFF));
63 mmio_write_32(address: &nvme->regs->acq_hi, value: (nvme->admin_cq_phys >> 32));
64
65 nvme->admin_sq_tail = 0;
66 nvme->admin_cq_head = 0;
67 nvme->admin_cq_phase = 1;
68}
69
70void nvme_alloc_admin_queues(struct nvme_device *nvme) {
71 uint64_t asq_size = nvme->admin_q_depth * sizeof(struct nvme_command);
72 uint64_t acq_size = nvme->admin_q_depth * sizeof(struct nvme_completion);
73
74 uint64_t asq_pages = DIV_ROUND_UP(asq_size, nvme->page_size);
75 uint64_t acq_pages = DIV_ROUND_UP(acq_size, nvme->page_size);
76
77 uint64_t asq_phys = pmm_alloc_pages(asq_pages);
78 nvme_check_dma_addr(phys: asq_phys, what: "admin submission queue");
79
80 struct nvme_command *asq_virt =
81 mmio_map_dma(phys: asq_phys, size: asq_pages * nvme->page_size);
82
83 memset(asq_virt, 0, asq_pages * nvme->page_size);
84
85 uint64_t acq_phys = pmm_alloc_pages(acq_pages);
86 nvme_check_dma_addr(phys: acq_phys, what: "admin completion queue");
87
88 struct nvme_completion *acq_virt =
89 mmio_map_dma(phys: acq_phys, size: acq_pages * nvme->page_size);
90
91 memset(acq_virt, 0, acq_pages * nvme->page_size);
92
93 nvme->admin_sq = asq_virt;
94 nvme->admin_sq_phys = asq_phys;
95 nvme->admin_cq = acq_virt;
96 nvme->admin_cq_phys = acq_phys;
97}
98
99void nvme_alloc_io_queues(struct nvme_device *nvme, uint32_t qid) {
100 if (!qid)
101 panic(OUT_OF_MEMORY, "IO queue zero");
102
103 nvme->io_queues[qid] = nofail_vnew(nvme->io_queues[qid], ALLOC_ZERO);
104
105 struct nvme_queue *this_queue = nvme->io_queues[qid];
106
107 uint64_t sq_pages = 2;
108 uint64_t cq_pages = 2;
109
110 uint64_t sq_phys = pmm_alloc_pages(sq_pages);
111 nvme_check_dma_addr(phys: sq_phys, what: "IO submission queue");
112
113 this_queue->sq = vmm_map_bump(sq_phys, sq_pages * nvme->page_size);
114 memset(this_queue->sq, 0, sq_pages * nvme->page_size);
115
116 uint64_t cq_phys = pmm_alloc_pages(cq_pages);
117 nvme_check_dma_addr(phys: cq_phys, what: "IO completion queue");
118
119 this_queue->cq = vmm_map_bump(cq_phys, cq_pages * nvme->page_size);
120 memset(this_queue->cq, 0, cq_pages * nvme->page_size);
121
122 this_queue->sq_phys = sq_phys;
123 this_queue->cq_phys = cq_phys;
124 this_queue->sq_tail = 0;
125 this_queue->cq_head = 0;
126 this_queue->cq_phase = 1;
127 /* match sizes */
128 this_queue->sq_depth = 64; // TODO: #define these or something
129 this_queue->cq_depth = 64;
130 this_queue->sq_db =
131 (uint32_t cc_mem_io *) ((uint8_t cc_mem_io *) nvme->regs +
132 NVME_DOORBELL_BASE +
133 (2 * qid * nvme->doorbell_stride));
134 this_queue->cq_db =
135 (uint32_t cc_mem_io *) ((uint8_t cc_mem_io *) nvme->regs +
136 NVME_DOORBELL_BASE +
137 ((2 * qid + 1) * nvme->doorbell_stride));
138
139 uint8_t this_isr = nvme->isr_index[qid];
140
141 this_queue->sq_requests =
142 nofail_vnews(this_queue->sq_requests, this_queue->sq_depth, ALLOC_ZERO);
143
144 // complete queue
145 struct nvme_command cq_cmd = {0};
146 cq_cmd.opc = NVME_OP_ADMIN_CREATE_IOCQ;
147 cq_cmd.prp1 = cq_phys;
148
149 cq_cmd.cdw10 = (this_queue->cq_depth - 1) << 16 | qid;
150
151 /* isr enabled, physicall contiguous */
152 cq_cmd.cdw11 = this_isr << 16 | 0b11;
153
154 irq_register(name: "nvme", vector: this_isr, handler: nvme_isr_handler, ctx: nvme, flags: IRQ_FLAG_NONE);
155 irq_set_chip(vector: this_isr, chip: lapic_get_chip(), NULL);
156
157 if (nvme_submit_admin_cmd(nvme, cmd: &cq_cmd, NULL) != 0) {
158 nvme_log(LOG_ERROR, "failed to create IOCQ %u, code 0x%x, ISR %u", qid,
159 cq_cmd.opc, this_isr);
160 return;
161 }
162
163 // submit queue
164 struct nvme_command sq_cmd = {0};
165 sq_cmd.opc = NVME_OP_ADMIN_CREATE_IOSQ;
166 sq_cmd.prp1 = sq_phys;
167
168 sq_cmd.cdw10 = (this_queue->sq_depth - 1) << 16 | qid;
169 sq_cmd.cdw11 = qid << 16 | 1;
170
171 if (nvme_submit_admin_cmd(nvme, cmd: &sq_cmd, NULL) != 0) {
172 nvme_log(LOG_ERROR, "failed to create IOSQ %u, code 0x%x, ISR %u", qid,
173 sq_cmd.opc, this_isr);
174 return;
175 }
176 nvme_log(LOG_INFO, "NVMe QID %u created - ISR %u", qid, this_isr);
177 spinlock_init(&this_queue->lock);
178}
179