1#include <asm.h>
2#include <block/bcache.h>
3#include <block/block.h>
4#include <block/sched.h>
5#include <compiler.h>
6#include <console/printf.h>
7#include <drivers/mmio.h>
8#include <drivers/nvme.h>
9#include <drivers/pci.h>
10#include <irq/idt.h>
11#include <mem/alloc.h>
12#include <mem/vmm.h>
13#include <registry.h>
14#include <sch/sched.h>
15#include <stdbool.h>
16#include <stdint.h>
17
18#include "internal.h"
19LOG_HANDLE_DECLARE_DEFAULT(nvme);
20LOG_SITE_DECLARE_DEFAULT(nvme);
21
22struct nvme_device *nvme_discover_device(uint8_t bus, uint8_t slot,
23 uint8_t func) {
24
25 uint32_t original_bar0 = pci_read(bus, slot, func, offset: 0x10);
26 uint32_t original_bar1 = pci_read(bus, slot, func, offset: 0x14);
27
28 bool is_io = original_bar0 & 1;
29
30 if (is_io) {
31 panic("doesnt look like mmio to me");
32 }
33
34 pci_write(bus, slot, func, offset: 0x10, value: 0xFFFFFFFF);
35 uint32_t size_mask = pci_read(bus, slot, func, offset: 0x10);
36 pci_write(bus, slot, func, offset: 0x10, value: original_bar0);
37 uint32_t size = ~(size_mask & ~0xFU) + 1;
38
39 if (size == 0) {
40 panic("bar0 reports zero size ?");
41 }
42
43 uint64_t phys_addr =
44 ((uint64_t) original_bar1 << 32) | (original_bar0 & ~0xFU);
45
46 void *mmio = mmio_map(phys: phys_addr, size);
47
48 struct nvme_regs *regs = (struct nvme_regs *) mmio;
49 uint64_t cap = ((uint64_t) regs->cap_hi << 32) | regs->cap_lo;
50 uint32_t version = regs->version;
51
52 uint32_t dstrd = (cap >> 32) & 0xF;
53
54 struct nvme_device *nvme =
55 kmalloc(sizeof(struct nvme_device), ALLOC_FLAGS_ZERO);
56 if (!nvme)
57 panic("Could not allocate space for NVMe drive");
58
59 nvme->doorbell_stride = 4U << dstrd;
60 nvme->page_size = PAGE_SIZE;
61 nvme->cap = cap;
62
63 nvme->version = version;
64 nvme->regs = regs;
65 nvme->admin_q_depth = ((nvme->cap) & 0xFFFF) + 1;
66 nvme->io_queues = kmalloc(sizeof(struct nvme_queue *), ALLOC_FLAGS_ZERO);
67 if (!nvme->io_queues)
68 panic("Could not allocate space for NVMe IO queues");
69
70 nvme->admin_sq_db =
71 (uint32_t *) ((uint8_t *) nvme->regs + NVME_DOORBELL_BASE);
72 nvme->admin_cq_db =
73 (uint32_t *) ((uint8_t *) nvme->regs + NVME_DOORBELL_BASE +
74 nvme->doorbell_stride);
75
76 if (nvme->admin_q_depth > 32)
77 nvme->admin_q_depth = 32;
78
79 struct nvme_cc cc = {0};
80 cc.raw = (uint32_t) mmio_read_32(address: &nvme->regs->cc);
81 cc.en = 0;
82
83 mmio_write_32(address: &nvme->regs->cc, value: cc.raw);
84
85 uint64_t core_count = global.core_count;
86
87 nvme_alloc_admin_queues(nvme);
88 nvme_setup_admin_queues(nvme);
89 nvme_enable_controller(nvme);
90 nvme_identify_namespace(nvme, nsid: 1);
91 struct nvme_identify_controller *c =
92 (void *) nvme_identify_controller(nvme);
93
94 uint32_t actual = nvme_set_num_queues(nvme, desired_sq: core_count, desired_cq: core_count);
95 uint32_t total_sq = actual & 0xffff;
96 uint32_t total_cq = actual >> 16;
97 nvme_log(LOG_INFO, "Controller supports %u SQs and %u CQs", total_sq,
98 total_cq);
99
100 nvme_set_num_queues(nvme, desired_sq: total_sq, desired_cq: total_cq);
101
102 uint32_t sqs_to_make = core_count > total_sq ? total_sq : core_count;
103
104 nvme->max_transfer_size = (1 << c->mdts) * PAGE_SIZE;
105 nvme_log(LOG_INFO, "Controller max transfer size is %u bytes",
106 nvme->max_transfer_size);
107
108 nvme->isr_index =
109 kmalloc(sizeof(uint8_t) * (sqs_to_make + 1), ALLOC_FLAGS_ZERO);
110 nvme->io_queues = kmalloc(sizeof(struct nvme_queue *) * (sqs_to_make + 1),
111 ALLOC_FLAGS_ZERO);
112 if (unlikely(!nvme->isr_index || !nvme->io_queues))
113 panic("Could not allocate space for NVMe structures");
114
115 nvme->queue_count = sqs_to_make;
116
117 pci_enable_msix(bus, slot, func);
118 for (uint32_t i = 1; i <= sqs_to_make; i++) {
119 uint8_t nvme_isr = irq_alloc_entry();
120
121 pci_enable_msix_on_core(bus, slot, func, vector: nvme_isr, core: i - 1);
122 nvme->isr_index[i] = nvme_isr;
123
124 nvme_alloc_io_queues(nvme, qid: i);
125 }
126
127 INIT_LIST_HEAD(list: &nvme->finished_requests.list);
128 INIT_LIST_HEAD(list: &nvme->waiting_requests.list);
129 INIT_LIST_HEAD(list: &nvme->work.list_node);
130 nvme->work.args = WORK_ARGS(nvme, NULL);
131 nvme->work.func = nvme_work;
132 spinlock_init(lock: &nvme->waiting_requests.lock);
133 spinlock_init(lock: &nvme->finished_requests.lock);
134
135 struct cpu_mask mask;
136 if (!cpu_mask_init(m: &mask, nbits: global.core_count))
137 panic("Could not initialize CPU mask");
138
139 cpu_mask_set_all(m: &mask);
140
141 struct workqueue_attributes attrs = {
142 .capacity = 64, /* small, oneshots are rare */
143 .idle_check = WORKQUEUE_DEFAULT_IDLE_CHECK,
144 .max_workers = 1,
145 .spawn_delay = WORKQUEUE_DEFAULT_SPAWN_DELAY,
146 .flags = WORKQUEUE_FLAG_DEFAULTS | WORKQUEUE_FLAG_NO_WORKER_GC |
147 WORKQUEUE_FLAG_ISR_SAFE,
148 .worker_cpu_mask = mask,
149 };
150
151 nvme->workqueue = workqueue_create(fmt: "nvme_wq", attrs: &attrs);
152 if (!nvme->workqueue)
153 panic("Could not allocate workqueue");
154
155 semaphore_init(s: &nvme->sem, value: 0, SEMAPHORE_INIT_IRQ_DISABLE);
156 nvme_work_enqueue(dev: nvme, work: &nvme->work);
157 workqueue_kick(queue: nvme->workqueue);
158
159 return nvme;
160}
161
162void nvme_print_wrapper(struct block_device *d) {
163 struct nvme_device *dev = (struct nvme_device *) d->driver_data;
164 uint8_t *n = nvme_identify_namespace(nvme: dev, nsid: 1);
165 nvme_print_namespace(ns: (struct nvme_identify_namespace *) n);
166 uint8_t *i = nvme_identify_controller(nvme: dev);
167 nvme_print_identify(ctrl: (struct nvme_identify_controller *) i);
168}
169
170static struct bio_scheduler_ops nvme_bio_sched_ops = {
171 .should_coalesce = noop_should_coalesce,
172 .reorder = noop_reorder,
173 .do_coalesce = noop_do_coalesce,
174 .max_wait_time =
175 {
176 [BIO_RQ_BACKGROUND] = 20,
177 [BIO_RQ_LOW] = 15,
178 [BIO_RQ_MEDIUM] = 10,
179 [BIO_RQ_HIGH] = 4,
180 [BIO_RQ_URGENT] = 0,
181 },
182 .dispatch_threshold = 128,
183 .boost_occupance_limit =
184 {
185 [BIO_RQ_BACKGROUND] = 64,
186 [BIO_RQ_LOW] = 56,
187 [BIO_RQ_MEDIUM] = 48,
188 [BIO_RQ_HIGH] = 40,
189 [BIO_RQ_URGENT] = 32,
190 },
191 .min_wait_ms = 1,
192 .tick_ms = 20,
193};
194
195struct block_device *nvme_create_generic(struct nvme_device *nvme) {
196 struct block_device *d =
197 kmalloc(sizeof(struct block_device), ALLOC_FLAGS_ZERO);
198 if (!d)
199 panic("Could not allocate space for NVMe device");
200
201 d->driver_data = nvme;
202 d->sector_size = nvme->sector_size;
203 d->read_sector = nvme_read_sector_wrapper;
204 d->write_sector = nvme_write_sector_wrapper;
205 d->submit_bio_async = nvme_submit_bio_request;
206 d->flags = BDEV_FLAG_NO_REORDER | BDEV_FLAG_NO_COALESCE;
207 d->cache = kmalloc(sizeof(struct bcache), ALLOC_FLAGS_ZERO);
208 if (unlikely(!d->cache))
209 panic("Could not allocate space for NVMe block cache");
210
211 d->scheduler = bio_sched_create(disk: d, ops: &nvme_bio_sched_ops);
212
213 bcache_init(cache: d->cache, DEFAULT_BLOCK_CACHE_SIZE);
214 d->type = BDEV_NVME_DRIVE;
215 nvme->generic_disk = d;
216 return d;
217}
218
219static uint64_t nvme_cnt = 1;
220
221static enum errno nvme_pci_init(struct device *d) {
222 struct pci_device *dev = d->driver_data;
223 uint8_t bus = dev->bus, device = dev->dev, function = dev->function;
224
225 struct nvme_device *nd = nvme_discover_device(bus, slot: device, func: function);
226 struct block_device *disk = nvme_create_generic(nvme: nd);
227 registry_mkname(disk, prefix: "nvme", counter: nvme_cnt++);
228 registry_register(disk);
229 k_print_register(disk->name);
230 return ERR_OK;
231}
232
233PCI_DEV_REGISTER(nvme, PCI_CLASS_MASS_STORAGE, PCI_SUBCLASS_NVM,
234 PCI_PROGIF_NVME, 0xFFFF, nvme_pci_init);
235