1#include <block/bio.h>
2#include <block/block.h>
3#include <block/sched.h>
4#include <drivers/nvme.h>
5#include <math/min_max.h>
6#include <mem/alloc.h>
7#include <mem/hhdm.h>
8#include <mem/pmm.h>
9#include <mem/vmm.h>
10#include <stdbool.h>
11#include <stddef.h>
12#include <stdint.h>
13#include <structures/sll.h>
14#include <thread/workqueue.h>
15
16#include "internal.h"
17#include <thread/thread.h>
18
19typedef bool (*sync_fn)(struct block_device *, uint64_t, uint8_t *, uint16_t);
20typedef bool (*async_fn)(struct block_device *, struct nvme_request *);
21
22static void enqueue_request(struct nvme_device *dev, struct nvme_request *req) {
23 struct nvme_waiting_requests *q = &dev->waiting_requests;
24
25 enum irql irql = spin_lock_high(&q->lock);
26
27 list_head_push_back(h: &q->list, n: &req->list_node);
28
29 spin_unlock(&q->lock, irql);
30
31 /* Retry with nvme_send_waiters(), completion decrements
32 * total_outstanding before looking at the list */
33 if (atomic_load(&dev->total_outstanding) == 0)
34 nvme_send_waiters(dev);
35}
36
37static bool nvme_bio_fill_prps(struct nvme_bio_data *data, const void *buffer,
38 uint64_t size) {
39 uint64_t offset = (uintptr_t) buffer & (PAGE_SIZE - 1);
40 uint64_t num_pages = PAGES_NEEDED_FOR(offset + size);
41
42 data->prps = vnews(data->prps, num_pages);
43 if (!data->prps)
44 return false;
45
46 /* PRP1 is the only one that can have a non-zero offset, and
47 * because the transfer begins some way into the first page, it needs
48 * to use this. Later entries describe full pages so must be page aligned */
49 vaddr_t vaddr = (vaddr_t) buffer;
50
51 for (size_t i = 0; i < num_pages; i++) {
52 data->prps[i] = vmm_get_phys(vaddr);
53 nvme_check_dma_addr(phys: data->prps[i], what: "PRP entry");
54 vaddr = PAGE_ALIGN_DOWN(vaddr) + PAGE_SIZE;
55 }
56
57 data->prp_count = num_pages;
58
59 return true;
60}
61
62static bool nvme_setup_prps(struct nvme_command *cmd,
63 struct nvme_bio_data *data) {
64 enforce(data->prp_count > 0);
65
66 cmd->prp1 = data->prps[0];
67 nvme_check_dma_addr(phys: cmd->prp1, what: "prp1");
68
69 enforce(IS_ALIGNED(cmd->prp1, 4)); /* dword aligned */
70 if (data->prp_count == 1) {
71 cmd->prp2 = 0;
72 goto free_prps;
73 } else if (data->prp_count == 2) {
74 cmd->prp2 = data->prps[1];
75 nvme_check_dma_addr(phys: cmd->prp2, what: "prp2");
76 goto free_prps;
77 }
78
79 /* After 2 pages PRP2 is a PRP List pointer. This means
80 * we need to provide another page for the list,
81 * with max_transfer_size always preventing too many */
82 uint64_t entries = data->prp_count - 1;
83 enforce(entries <= NVME_PRPS_PER_PAGE, CAPACITY_EXCEEDED);
84
85 paddr_t list_phys = pmm_alloc_page();
86 if (!list_phys)
87 goto free_prps_fail;
88
89 uint64_t *list = hhdm_paddr_to_ptr(p: list_phys);
90 for (uint64_t i = 0; i < entries; i++)
91 list[i] = data->prps[i + 1];
92
93 data->prp_list_phys = list_phys;
94 cmd->prp2 = list_phys;
95 nvme_check_dma_addr(phys: cmd->prp2, what: "PRP list pointer");
96
97 /* List page has everything needed by the controller */
98free_prps:
99 vfree(data->prps);
100 data->prps = NULL;
101 return true;
102
103free_prps_fail:
104 vfree(data->prps);
105 data->prps = NULL;
106 return false;
107}
108
109static bool rw_send_command(struct block_device *disk, struct nvme_request *req,
110 uint8_t opc) {
111 struct nvme_device *nvme = disk->driver_data;
112 uint16_t qid = THIS_QID(nvme);
113 uint64_t lba = req->lba;
114 uint64_t count = req->sector_count;
115 void *buffer = req->buffer;
116
117 struct nvme_queue *q = nvme->io_queues[qid];
118
119 /* sq_depth - 1, avoid the off by one */
120 if (atomic_load(&q->outstanding) >= q->sq_depth - 1) {
121 enqueue_request(dev: nvme, req);
122
123 /* No room */
124 return true;
125 }
126
127 struct nvme_bio_data *data = vnew(data, ALLOC_ZERO);
128 if (!data)
129 return false;
130
131 if (!nvme_bio_fill_prps(data, buffer, size: count * disk->sector_size)) {
132 vfree(data);
133 return false;
134 }
135
136 req->bio_data = data;
137
138 struct nvme_command cmd = {0};
139 cmd.opc = opc;
140 cmd.nsid = 1;
141 cmd.cdw10 = lba & 0xFFFFFFFFULL;
142 cmd.cdw11 = lba >> 32ULL;
143 cmd.cdw12 = count - 1;
144
145 if (!nvme_setup_prps(cmd: &cmd, data)) {
146 req->bio_data = NULL;
147 vfree(data);
148 return false;
149 }
150
151 req->lba = lba;
152 req->buffer = buffer;
153 req->sector_count = count;
154 req->done = false;
155 req->status = -1;
156
157 if (!nvme_submit_io_cmd(nvme, cmd: &cmd, qid, req)) {
158 /* Queue filled up. Enqueue a request and a completion
159 * will hand it back */
160 if (data->prp_list_phys)
161 pmm_free_page(addr: data->prp_list_phys);
162
163 vfree(data->prps);
164 vfree(data);
165 req->bio_data = NULL;
166
167 enqueue_request(dev: nvme, req);
168 }
169
170 return true;
171}
172
173static bool rw_sync(struct block_device *disk, uint64_t lba, uint8_t *buffer,
174 uint16_t count, async_fn function) {
175 struct nvme_request req = {0};
176 req.lba = lba;
177 req.buffer = buffer;
178 req.sector_count = count;
179 req.remaining_parts = 1;
180 list_node_init(n: &req.list_node);
181
182 event_init(e: &req.evt, type: EVENT_NOTIFICATION, false);
183 req.has_waiter = true;
184
185 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
186 bool sent = function(disk, &req);
187 irql_lower(old_level: irql);
188
189 if (!sent)
190 return false;
191
192 /* Go run something else now */
193 event_wait(e: &req.evt, type: THREAD_WAIT_UNINTERRUPTIBLE, reason: THREAD_BLOCK_REASON_IO);
194
195 return !req.status;
196}
197
198static bool rw_wrapper(struct block_device *disk, uint64_t lba, uint8_t *buf,
199 uint64_t cnt, sync_fn function) {
200 struct nvme_device *nvme = (struct nvme_device *) disk->driver_data;
201 uint64_t max_sectors = nvme->max_transfer_size / disk->sector_size;
202 enforce(max_sectors > 0 && max_sectors <= UINT16_MAX, CAPACITY_EXCEEDED);
203
204 while (cnt > 0) {
205 uint16_t chunk = (uint16_t) MIN(cnt, max_sectors);
206 if (!function(disk, lba, buf, chunk))
207 return false;
208
209 lba += chunk;
210 buf += chunk * disk->sector_size;
211 cnt -= chunk;
212 }
213
214 return true;
215}
216
217static bool rw_async_wrapper(struct block_device *disk,
218 struct nvme_request *req, async_fn function) {
219 struct nvme_device *nvme = (struct nvme_device *) disk->driver_data;
220 uint64_t max_sectors = nvme->max_transfer_size / disk->sector_size;
221 uint64_t chunk;
222
223 uint64_t cnt = req->sector_count;
224
225 int part_count = 0;
226 uint64_t tmp_cnt = cnt;
227 while (tmp_cnt > 0) {
228 chunk = MIN(tmp_cnt, max_sectors);
229 tmp_cnt -= chunk;
230 part_count++;
231 }
232
233 req->remaining_parts = part_count;
234
235 while (cnt > 0) {
236 chunk = MIN(cnt, max_sectors);
237 cnt -= chunk;
238
239 if (!function(disk, req))
240 return false;
241 }
242
243 return true;
244}
245
246static bool nvme_read_sector(struct block_device *disk, uint64_t lba,
247 uint8_t *buffer, uint16_t count) {
248 return rw_sync(disk, lba, buffer, count, function: nvme_read_sector_async);
249}
250
251static bool nvme_write_sector(struct block_device *disk, uint64_t lba,
252 uint8_t *buffer, uint16_t count) {
253 return rw_sync(disk, lba, buffer, count, function: nvme_write_sector_async);
254}
255
256bool nvme_read_sector_wrapper(struct block_device *disk, uint64_t lba,
257 uint8_t *buf, uint64_t cnt) {
258 return rw_wrapper(disk, lba, buf, cnt, function: nvme_read_sector);
259}
260
261bool nvme_write_sector_wrapper(struct block_device *disk, uint64_t lba,
262 const uint8_t *buf, uint64_t cnt) {
263 return rw_wrapper(disk, lba, buf: (uint8_t *) buf, cnt, function: nvme_write_sector);
264}
265
266bool nvme_read_sector_async(struct block_device *disk,
267 struct nvme_request *req) {
268 return rw_send_command(disk, req, NVME_OP_IO_READ);
269}
270
271bool nvme_write_sector_async(struct block_device *disk,
272 struct nvme_request *req) {
273 return rw_send_command(disk, req, NVME_OP_IO_WRITE);
274}
275
276bool nvme_write_sector_async_wrapper(struct block_device *disk,
277 struct nvme_request *req) {
278 return rw_async_wrapper(disk, req, function: nvme_write_sector_async);
279}
280
281bool nvme_read_sector_async_wrapper(struct block_device *disk,
282 struct nvme_request *req) {
283 return rw_async_wrapper(disk, req, function: nvme_read_sector_async);
284}
285
286bool nvme_send_nvme_req(struct block_device *d, struct nvme_request *r) {
287 return rw_send_command(disk: d, req: r, opc: r->write ? NVME_OP_IO_WRITE : NVME_OP_IO_READ);
288}
289