1#include <asm.h>
2#include <compiler.h>
3#include <console/printf.h>
4#include <drivers/mmio.h>
5#include <drivers/nvme.h>
6#include <irq/idt.h>
7#include <mem/alloc.h>
8#include <mem/alloc_or_die.h>
9#include <mem/pmm.h>
10#include <mem/vmm.h>
11#include <stdbool.h>
12#include <stdint.h>
13#include <string.h>
14#include <time/spin_sleep.h>
15
16#include "internal.h"
17
18/* we poll in setup */
19void nvme_enable_controller(struct nvme_device *nvme) {
20
21 struct nvme_cc cc = {0};
22
23 cc.mps = 0;
24
25 cc.iocqes = 4; // 2 ^ 4 = 16
26
27 cc.iosqes = 6; // 2 ^ 6 = 64
28
29 cc.ams = 0;
30
31 cc.css = 0b110;
32
33 cc.en = 0;
34
35 mmio_write_32(address: &nvme->regs->cc, value: *(uint32_t *) &cc);
36
37 mmio_spin_wait(reg: &nvme->regs->csts, mask: 1, NVME_CMD_TIMEOUT_MS);
38
39 cc.en = 1;
40
41 mmio_write_32(address: &nvme->regs->cc, value: *(uint32_t *) &cc);
42
43 uint64_t timeout = NVME_CMD_TIMEOUT_MS * 1000;
44 while ((mmio_read_32(address: &nvme->regs->csts) & 1) == 0) {
45 sleep_spin_us(us: 10);
46 timeout--;
47 if (timeout == 0)
48 return;
49 }
50}
51
52void nvme_setup_admin_queues(struct nvme_device *nvme) {
53 uint32_t q_depth_minus_1 = nvme->admin_q_depth - 1;
54
55 uint32_t aqa = (q_depth_minus_1 << 16) | q_depth_minus_1;
56 mmio_write_32(address: &nvme->regs->aqa, value: aqa);
57
58 mmio_write_32(address: &nvme->regs->asq_lo, value: (nvme->admin_sq_phys & 0xFFFFFFFF));
59 mmio_write_32(address: &nvme->regs->asq_hi, value: (nvme->admin_sq_phys >> 32));
60
61 mmio_write_32(address: &nvme->regs->acq_lo, value: (nvme->admin_cq_phys & 0xFFFFFFFF));
62 mmio_write_32(address: &nvme->regs->acq_hi, value: (nvme->admin_cq_phys >> 32));
63
64 nvme->admin_sq_tail = 0;
65 nvme->admin_cq_head = 0;
66 nvme->admin_cq_phase = 1;
67}
68
69void nvme_alloc_admin_queues(struct nvme_device *nvme) {
70 uint64_t asq_size = nvme->admin_q_depth * sizeof(struct nvme_command);
71 uint64_t acq_size = nvme->admin_q_depth * sizeof(struct nvme_completion);
72
73 uint64_t asq_pages = DIV_ROUND_UP(asq_size, nvme->page_size);
74 uint64_t acq_pages = DIV_ROUND_UP(acq_size, nvme->page_size);
75
76 uint64_t asq_phys = pmm_alloc_pages(asq_pages);
77
78 struct nvme_command *asq_virt =
79 mmio_map(phys: asq_phys, size: asq_pages * nvme->page_size);
80
81 memset(asq_virt, 0, asq_pages * nvme->page_size);
82
83 uint64_t acq_phys = pmm_alloc_pages(acq_pages);
84
85 struct nvme_completion *acq_virt =
86 mmio_map(phys: acq_phys, size: acq_pages * nvme->page_size);
87
88 memset(acq_virt, 0, acq_pages * nvme->page_size);
89
90 nvme->admin_sq = asq_virt;
91 nvme->admin_sq_phys = asq_phys;
92 nvme->admin_cq = acq_virt;
93 nvme->admin_cq_phys = acq_phys;
94}
95
96void nvme_alloc_io_queues(struct nvme_device *nvme, uint32_t qid) {
97 if (!qid)
98 panic("Can't allocate IO queue zero!");
99
100 nvme->io_queues[qid] =
101 kmalloc_or_die(sizeof(struct nvme_queue), ALLOC_FLAGS_ZERO);
102
103 struct nvme_queue *this_queue = nvme->io_queues[qid];
104
105 uint64_t sq_pages = 2;
106 uint64_t cq_pages = 2;
107
108 uint64_t sq_phys = pmm_alloc_pages(sq_pages);
109
110 this_queue->sq =
111 vmm_map_bump(sq_phys, sq_pages * nvme->page_size, PAGE_NO_FLAGS);
112 memset(this_queue->sq, 0, sq_pages * nvme->page_size);
113
114 uint64_t cq_phys = pmm_alloc_pages(cq_pages);
115
116 this_queue->cq =
117 vmm_map_bump(cq_phys, cq_pages * nvme->page_size, PAGE_NO_FLAGS);
118 memset(this_queue->cq, 0, cq_pages * nvme->page_size);
119
120 this_queue->sq_phys = sq_phys;
121 this_queue->cq_phys = cq_phys;
122 this_queue->sq_tail = 0;
123 this_queue->cq_head = 0;
124 this_queue->cq_phase = 1;
125 this_queue->sq_depth = 64; // TODO: #define these or something
126 this_queue->cq_depth = 16;
127 this_queue->sq_db =
128 (uint32_t *) ((uint8_t *) nvme->regs + NVME_DOORBELL_BASE +
129 (2 * qid * nvme->doorbell_stride));
130 this_queue->cq_db =
131 (uint32_t *) ((uint8_t *) nvme->regs + NVME_DOORBELL_BASE +
132 ((2 * qid + 1) * nvme->doorbell_stride));
133
134 uint8_t this_isr = nvme->isr_index[qid];
135
136 this_queue->sq_requests = alloc_or_die(
137 kmalloc(sizeof(struct nvme_request *) * this_queue->sq_depth,
138 ALLOC_FLAGS_ZERO));
139
140 // complete queue
141 struct nvme_command cq_cmd = {0};
142 cq_cmd.opc = NVME_OP_ADMIN_CREATE_IOCQ;
143 cq_cmd.prp1 = cq_phys;
144
145 cq_cmd.cdw10 = (15) << 16 | qid;
146
147 /* isr enabled, physicall contiguous */
148 cq_cmd.cdw11 = this_isr << 16 | 0b11;
149
150 irq_register(name: "nvme", vector: this_isr, handler: nvme_isr_handler, ctx: nvme, flags: IRQ_FLAG_NONE);
151 irq_set_chip(vector: this_isr, chip: lapic_get_chip(), NULL);
152
153 if (nvme_submit_admin_cmd(nvme, cmd: &cq_cmd, NULL) != 0) {
154 nvme_log(LOG_ERROR, "failed to create IOCQ %u, code 0x%x, ISR %u", qid,
155 cq_cmd.opc, this_isr);
156 return;
157 }
158
159 // submit queue
160 struct nvme_command sq_cmd = {0};
161 sq_cmd.opc = NVME_OP_ADMIN_CREATE_IOSQ;
162 sq_cmd.prp1 = sq_phys;
163
164 sq_cmd.cdw10 = (63) << 16 | qid;
165 sq_cmd.cdw11 = qid << 16 | 1;
166
167 if (nvme_submit_admin_cmd(nvme, cmd: &sq_cmd, NULL) != 0) {
168 nvme_log(LOG_ERROR, "failed to create IOSQ %u, code 0x%x, ISR %u", qid,
169 sq_cmd.opc, this_isr);
170 return;
171 }
172 nvme_log(LOG_INFO, "NVMe QID %u created - ISR %u", qid, this_isr);
173 spinlock_init(lock: &this_queue->lock);
174}
175