1#include <asm.h>
2#include <compiler.h>
3#include <console/printf.h>
4#include <drivers/mmio.h>
5#include <drivers/nvme.h>
6#include <irq/idt.h>
7#include <mem/alloc.h>
8#include <mem/alloc_or_die.h>
9#include <mem/pmm.h>
10#include <mem/vmm.h>
11#include <stdbool.h>
12#include <stdint.h>
13#include <string.h>
14#include <time/spin_sleep.h>
15
16#include "internal.h"
17
18/* we poll in setup */
19void nvme_enable_controller(struct nvme_device *nvme) {
20
21 struct nvme_cc cc = {0};
22
23 cc.mps = 0;
24
25 cc.iocqes = 4; // 2 ^ 4 = 16
26
27 cc.iosqes = 6; // 2 ^ 6 = 64
28
29 cc.ams = 0;
30
31 cc.css = 0b110;
32
33 cc.en = 0;
34
35 mmio_write_32(address: &nvme->regs->cc, value: *(uint32_t *) &cc);
36
37 mmio_spin_wait(reg: &nvme->regs->csts, mask: 1, NVME_CMD_TIMEOUT_MS);
38
39 cc.en = 1;
40
41 mmio_write_32(address: &nvme->regs->cc, value: *(uint32_t *) &cc);
42
43 uint64_t timeout = NVME_CMD_TIMEOUT_MS * 1000;
44 while ((mmio_read_32(address: &nvme->regs->csts) & 1) == 0) {
45 sleep_spin_us(us: 10);
46 timeout--;
47 if (timeout == 0)
48 return;
49 }
50}
51
52void nvme_setup_admin_queues(struct nvme_device *nvme) {
53 uint32_t q_depth_minus_1 = nvme->admin_q_depth - 1;
54
55 uint32_t aqa = (q_depth_minus_1 << 16) | q_depth_minus_1;
56 mmio_write_32(address: &nvme->regs->aqa, value: aqa);
57
58 mmio_write_32(address: &nvme->regs->asq_lo, value: (nvme->admin_sq_phys & 0xFFFFFFFF));
59 mmio_write_32(address: &nvme->regs->asq_hi, value: (nvme->admin_sq_phys >> 32));
60
61 mmio_write_32(address: &nvme->regs->acq_lo, value: (nvme->admin_cq_phys & 0xFFFFFFFF));
62 mmio_write_32(address: &nvme->regs->acq_hi, value: (nvme->admin_cq_phys >> 32));
63
64 nvme->admin_sq_tail = 0;
65 nvme->admin_cq_head = 0;
66 nvme->admin_cq_phase = 1;
67}
68
69void nvme_alloc_admin_queues(struct nvme_device *nvme) {
70 uint64_t asq_size = nvme->admin_q_depth * sizeof(struct nvme_command);
71 uint64_t acq_size = nvme->admin_q_depth * sizeof(struct nvme_completion);
72
73 uint64_t asq_pages = DIV_ROUND_UP(asq_size, nvme->page_size);
74 uint64_t acq_pages = DIV_ROUND_UP(acq_size, nvme->page_size);
75
76 uint64_t asq_phys = pmm_alloc_pages(asq_pages);
77 nvme_check_dma_addr(phys: asq_phys, what: "admin submission queue");
78
79 struct nvme_command *asq_virt =
80 mmio_map(phys: asq_phys, size: asq_pages * nvme->page_size);
81
82 memset(asq_virt, 0, asq_pages * nvme->page_size);
83
84 uint64_t acq_phys = pmm_alloc_pages(acq_pages);
85 nvme_check_dma_addr(phys: acq_phys, what: "admin completion queue");
86
87 struct nvme_completion *acq_virt =
88 mmio_map(phys: acq_phys, size: acq_pages * nvme->page_size);
89
90 memset(acq_virt, 0, acq_pages * nvme->page_size);
91
92 nvme->admin_sq = asq_virt;
93 nvme->admin_sq_phys = asq_phys;
94 nvme->admin_cq = acq_virt;
95 nvme->admin_cq_phys = acq_phys;
96}
97
98void nvme_alloc_io_queues(struct nvme_device *nvme, uint32_t qid) {
99 if (!qid)
100 panic("Can't allocate IO queue zero!");
101
102 nvme->io_queues[qid] =
103 kmalloc_or_die(sizeof(struct nvme_queue), ALLOC_FLAGS_ZERO);
104
105 struct nvme_queue *this_queue = nvme->io_queues[qid];
106
107 uint64_t sq_pages = 2;
108 uint64_t cq_pages = 2;
109
110 uint64_t sq_phys = pmm_alloc_pages(sq_pages);
111 nvme_check_dma_addr(phys: sq_phys, what: "IO submission queue");
112
113 this_queue->sq =
114 vmm_map_bump(sq_phys, sq_pages * nvme->page_size, PAGE_NO_FLAGS);
115 memset(this_queue->sq, 0, sq_pages * nvme->page_size);
116
117 uint64_t cq_phys = pmm_alloc_pages(cq_pages);
118 nvme_check_dma_addr(phys: cq_phys, what: "IO completion queue");
119
120 this_queue->cq =
121 vmm_map_bump(cq_phys, cq_pages * nvme->page_size, PAGE_NO_FLAGS);
122 memset(this_queue->cq, 0, cq_pages * nvme->page_size);
123
124 this_queue->sq_phys = sq_phys;
125 this_queue->cq_phys = cq_phys;
126 this_queue->sq_tail = 0;
127 this_queue->cq_head = 0;
128 this_queue->cq_phase = 1;
129 this_queue->sq_depth = 64; // TODO: #define these or something
130 this_queue->cq_depth = 16;
131 this_queue->sq_db =
132 (uint32_t *) ((uint8_t *) nvme->regs + NVME_DOORBELL_BASE +
133 (2 * qid * nvme->doorbell_stride));
134 this_queue->cq_db =
135 (uint32_t *) ((uint8_t *) nvme->regs + NVME_DOORBELL_BASE +
136 ((2 * qid + 1) * nvme->doorbell_stride));
137
138 uint8_t this_isr = nvme->isr_index[qid];
139
140 this_queue->sq_requests = alloc_or_die(
141 kmalloc(sizeof(struct nvme_request *) * this_queue->sq_depth,
142 ALLOC_FLAGS_ZERO));
143
144 // complete queue
145 struct nvme_command cq_cmd = {0};
146 cq_cmd.opc = NVME_OP_ADMIN_CREATE_IOCQ;
147 cq_cmd.prp1 = cq_phys;
148
149 cq_cmd.cdw10 = (15) << 16 | qid;
150
151 /* isr enabled, physicall contiguous */
152 cq_cmd.cdw11 = this_isr << 16 | 0b11;
153
154 irq_register(name: "nvme", vector: this_isr, handler: nvme_isr_handler, ctx: nvme, flags: IRQ_FLAG_NONE);
155 irq_set_chip(vector: this_isr, chip: lapic_get_chip(), NULL);
156
157 if (nvme_submit_admin_cmd(nvme, cmd: &cq_cmd, NULL) != 0) {
158 nvme_log(LOG_ERROR, "failed to create IOCQ %u, code 0x%x, ISR %u", qid,
159 cq_cmd.opc, this_isr);
160 return;
161 }
162
163 // submit queue
164 struct nvme_command sq_cmd = {0};
165 sq_cmd.opc = NVME_OP_ADMIN_CREATE_IOSQ;
166 sq_cmd.prp1 = sq_phys;
167
168 sq_cmd.cdw10 = (63) << 16 | qid;
169 sq_cmd.cdw11 = qid << 16 | 1;
170
171 if (nvme_submit_admin_cmd(nvme, cmd: &sq_cmd, NULL) != 0) {
172 nvme_log(LOG_ERROR, "failed to create IOSQ %u, code 0x%x, ISR %u", qid,
173 sq_cmd.opc, this_isr);
174 return;
175 }
176 nvme_log(LOG_INFO, "NVMe QID %u created - ISR %u", qid, this_isr);
177 spinlock_init(&this_queue->lock);
178}
179