1#include <asm.h>
2#include <block/bcache.h>
3#include <block/block.h>
4#include <block/sched.h>
5#include <compiler/core.h>
6#include <console/printf.h>
7#include <drivers/mmio.h>
8#include <drivers/nvme.h>
9#include <drivers/pci.h>
10#include <irq/idt.h>
11#include <math/bit.h>
12#include <math/min_max.h>
13#include <mem/alloc.h>
14#include <mem/vmm.h>
15#include <registry.h>
16#include <sch/sched.h>
17#include <smp/core.h>
18#include <stdbool.h>
19#include <stdint.h>
20
21#include "internal.h"
22LOG_HANDLE_DEFINE_PRINT(nvme);
23LOG_SITE_DEFINE_PRINT(nvme);
24
25struct nvme_device *nvme_discover_device(uint8_t bus, uint8_t slot,
26 uint8_t func) {
27
28 uint32_t original_bar0 = pci_read(bus, slot, func, offset: 0x10);
29 uint32_t original_bar1 = pci_read(bus, slot, func, offset: 0x14);
30
31 bool is_io = original_bar0 & 1;
32
33 if (is_io) {
34 panic(UNSUPPORTED, "doesnt look like mmio to me");
35 }
36
37 pci_write(bus, slot, func, offset: 0x10, value: 0xFFFFFFFF);
38 uint32_t size_mask = pci_read(bus, slot, func, offset: 0x10);
39 pci_write(bus, slot, func, offset: 0x10, value: original_bar0);
40 uint32_t size = ~(size_mask & ~0xFU) + 1;
41
42 if (size == 0) {
43 panic(UNSUPPORTED, "bar0 reports zero size ?");
44 }
45
46 uint64_t phys_addr =
47 ((uint64_t) original_bar1 << 32) | (original_bar0 & ~0xFU);
48
49 void cc_mem_io *mmio = mmio_map(phys: phys_addr, size);
50
51 struct nvme_regs cc_mem_io *regs = (struct nvme_regs cc_mem_io *) mmio;
52 uint64_t cap = ((uint64_t) regs->cap_hi << 32) | regs->cap_lo;
53 uint32_t version = regs->version;
54
55 uint32_t dstrd = (cap >> 32) & 0xF;
56
57 struct nvme_device *nvme = vnew(nvme, ALLOC_ZERO);
58 if (!nvme)
59 panic(OUT_OF_MEMORY, "NVMe drive");
60
61 nvme->doorbell_stride = 4U << dstrd;
62 nvme->page_size = PAGE_SIZE;
63 nvme->cap = cap;
64
65 nvme->version = version;
66 nvme->regs = regs;
67 nvme->admin_q_depth = ((nvme->cap) & 0xFFFF) + 1;
68 nvme->io_queues = vnew(nvme->io_queues, ALLOC_ZERO);
69 if (!nvme->io_queues)
70 panic(OUT_OF_MEMORY, "NVMe IO queues");
71
72 nvme->admin_sq_db =
73 (uint32_t cc_mem_io *) ((uint8_t cc_mem_io *) nvme->regs +
74 NVME_DOORBELL_BASE);
75 nvme->admin_cq_db =
76 (uint32_t cc_mem_io *) ((uint8_t cc_mem_io *) nvme->regs +
77 NVME_DOORBELL_BASE + nvme->doorbell_stride);
78
79 if (nvme->admin_q_depth > 32)
80 nvme->admin_q_depth = 32;
81
82 struct nvme_cc cc = {0};
83 cc.raw = (uint32_t) mmio_read_32(address: &nvme->regs->cc);
84 cc.en = 0;
85
86 mmio_write_32(address: &nvme->regs->cc, value: cc.raw);
87
88 uint64_t core_count = global.core_count;
89
90 nvme_alloc_admin_queues(nvme);
91 nvme_setup_admin_queues(nvme);
92 nvme_enable_controller(nvme);
93 nvme_identify_namespace(nvme, nsid: 1);
94 struct nvme_identify_controller *c =
95 (void *) nvme_identify_controller(nvme);
96
97 uint32_t actual = nvme_set_num_queues(nvme, desired_sq: core_count, desired_cq: core_count);
98 uint32_t total_sq = actual & 0xffff;
99 uint32_t total_cq = actual >> 16;
100 nvme_log(LOG_INFO, "Controller supports %u SQs and %u CQs", total_sq,
101 total_cq);
102
103 nvme_set_num_queues(nvme, desired_sq: total_sq, desired_cq: total_cq);
104
105 uint64_t supported_sqs = total_sq;
106 uint64_t sqs_to_make_wide = MIN(core_count, supported_sqs);
107 enforce(sqs_to_make_wide <= UINT32_MAX, CAPACITY_EXCEEDED);
108 uint32_t sqs_to_make = (uint32_t) sqs_to_make_wide;
109
110 nvme->max_transfer_size = BIT(c->mdts) * PAGE_SIZE;
111
112 /* Commands describe one page via PRP1 and the rest through one
113 * PRP List page, so we just cap the transfer at that */
114 uint64_t prp_limit = NVME_PRPS_PER_PAGE * PAGE_SIZE;
115 nvme->max_transfer_size = MIN(nvme->max_transfer_size, prp_limit);
116 nvme_log(LOG_INFO, "Controller max transfer size is %lu bytes",
117 nvme->max_transfer_size);
118
119 nvme->isr_index = vnews(nvme->isr_index, (sqs_to_make + 1), ALLOC_ZERO);
120 nvme->io_queues = vnews(nvme->io_queues, (sqs_to_make + 1), ALLOC_ZERO);
121 if (cc_unlikely(!nvme->isr_index || !nvme->io_queues))
122 panic(OUT_OF_MEMORY, "NVMe structures");
123
124 nvme->queue_count = sqs_to_make;
125
126 pci_enable_msix(bus, slot, func);
127 for (uint32_t i = 1; i <= sqs_to_make; i++) {
128 uint8_t nvme_isr = irq_alloc_entry();
129
130 pci_enable_msix_on_core(bus, slot, func, vector: nvme_isr, apic_id: smp_apic_id(cpu: i - 1));
131 nvme->isr_index[i] = nvme_isr;
132
133 nvme_alloc_io_queues(nvme, qid: i);
134 }
135
136 list_head_init(h: &nvme->finished_requests.list);
137 list_head_init(h: &nvme->waiting_requests.list);
138 list_node_init(n: &nvme->work.list_node);
139 nvme->work.args = WORK_ARGS(nvme, NULL);
140 nvme->work.func = nvme_work;
141 spinlock_init(&nvme->waiting_requests.lock);
142 spinlock_init(&nvme->finished_requests.lock);
143
144 struct cpu_mask mask;
145 cpu_mask_init(m: &mask, nbits: global.core_count);
146
147 cpu_mask_set_all(&mask);
148
149 struct workqueue_attributes attrs = {
150 .idle_check = WORKQUEUE_DEFAULT_IDLE_CHECK,
151 .max_workers = 1,
152 .spawn_delay = WORKQUEUE_DEFAULT_SPAWN_DELAY,
153 .flags = WORKQUEUE_FLAG_DEFAULTS | WORKQUEUE_FLAG_NO_WORKER_GC |
154 WORKQUEUE_FLAG_ISR_SAFE,
155 .worker_cpu_mask = mask,
156 };
157
158 nvme->workqueue = workqueue_create("nvme_wq", &attrs);
159 if (!nvme->workqueue)
160 panic(OUT_OF_MEMORY, "workqueue");
161
162 semaphore_init(s: &nvme->sem, value: 0, SEMAPHORE_INIT_IRQ_DISABLE);
163 nvme_work_enqueue(dev: nvme, work: &nvme->work);
164 workqueue_kick(queue: nvme->workqueue);
165
166 return nvme;
167}
168
169void nvme_print_wrapper(struct block_device *d) {
170 struct nvme_device *dev = (struct nvme_device *) d->driver_data;
171 uint8_t *n = nvme_identify_namespace(nvme: dev, nsid: 1);
172 nvme_print_namespace(ns: (struct nvme_identify_namespace *) n);
173 uint8_t *i = nvme_identify_controller(nvme: dev);
174 nvme_print_identify(ctrl: (struct nvme_identify_controller *) i);
175}
176
177static struct bio_scheduler_ops nvme_bio_sched_ops = {
178 .should_coalesce = noop_should_coalesce,
179 .reorder = noop_reorder,
180 .do_coalesce = noop_do_coalesce,
181 .max_wait_time =
182 {
183 [BIO_RQ_BACKGROUND] = 20,
184 [BIO_RQ_LOW] = 15,
185 [BIO_RQ_MEDIUM] = 10,
186 [BIO_RQ_HIGH] = 4,
187 [BIO_RQ_URGENT] = 0,
188 },
189 .dispatch_threshold = 128,
190 .boost_occupance_limit =
191 {
192 [BIO_RQ_BACKGROUND] = 64,
193 [BIO_RQ_LOW] = 56,
194 [BIO_RQ_MEDIUM] = 48,
195 [BIO_RQ_HIGH] = 40,
196 [BIO_RQ_URGENT] = 32,
197 },
198 .min_wait_ms = 1,
199 .tick_ms = 20,
200};
201
202struct block_device *nvme_create_generic(struct nvme_device *nvme) {
203 struct block_device *d = vnew(d, ALLOC_ZERO);
204 if (!d)
205 panic(OUT_OF_MEMORY, "NVMe device");
206
207 d->driver_data = nvme;
208 d->sector_size = nvme->sector_size;
209 d->read_sector = nvme_read_sector_wrapper;
210 d->write_sector = nvme_write_sector_wrapper;
211 d->submit_bio_async = nvme_submit_bio_request;
212 d->flags = BDEV_FLAG_NO_REORDER | BDEV_FLAG_NO_COALESCE;
213 d->cache = vnew(d->cache, ALLOC_ZERO);
214 if (cc_unlikely(!d->cache))
215 panic(OUT_OF_MEMORY, "NVMe block cache");
216
217 d->scheduler = bio_sched_create(disk: d, ops: &nvme_bio_sched_ops);
218
219 bcache_init(cache: d->cache, DEFAULT_BLOCK_CACHE_SIZE);
220 d->type = BDEV_NVME_DRIVE;
221 nvme->generic_disk = d;
222 return d;
223}
224
225static uint64_t nvme_cnt = 1;
226
227static enum err nvme_pci_init(struct device *d) {
228 struct pci_device *dev = d->driver_data;
229 uint8_t bus = dev->bus, device = dev->dev, function = dev->function;
230
231 struct nvme_device *nd = nvme_discover_device(bus, slot: device, func: function);
232 struct block_device *disk = nvme_create_generic(nvme: nd);
233 registry_mkname(disk, prefix: "nvme", counter: nvme_cnt++);
234 registry_register(disk);
235 k_print_register(disk->name);
236 return ERR_OK;
237}
238
239PCI_DEV_REGISTER(nvme, PCI_CLASS_MASS_STORAGE, PCI_SUBCLASS_NVM,
240 PCI_PROGIF_NVME, 0xFFFF, nvme_pci_init);
241