1#include <block/bio.h>
2#include <block/block.h>
3#include <block/sched.h>
4#include <drivers/nvme.h>
5#include <mem/alloc.h>
6#include <mem/hhdm.h>
7#include <mem/pmm.h>
8#include <mem/vmm.h>
9#include <stdbool.h>
10#include <stddef.h>
11#include <stdint.h>
12#include <structures/sll.h>
13#include <thread/io_wait.h>
14#include <thread/workqueue.h>
15
16#include "internal.h"
17#include <thread/thread.h>
18
19typedef bool (*sync_fn)(struct block_device *, uint64_t, uint8_t *, uint16_t,
20 struct io_wait_token *iowt);
21typedef bool (*async_fn)(struct block_device *, struct nvme_request *);
22
23static void enqueue_request(struct nvme_device *dev, struct nvme_request *req) {
24 struct nvme_waiting_requests *q = &dev->waiting_requests;
25
26 enum irql irql = spin_lock_irq_disable(&q->lock);
27
28 list_add_tail(new: &req->list_node, head: &q->list);
29
30 spin_unlock(&q->lock, irql);
31}
32
33static bool nvme_bio_fill_prps(struct nvme_bio_data *data, const void *buffer,
34 uint64_t size) {
35 uint64_t offset = (uintptr_t) buffer & (PAGE_SIZE - 1);
36 uint64_t num_pages = PAGES_NEEDED_FOR(offset + size);
37
38 data->prps = kmalloc(sizeof(uint64_t) * num_pages);
39 if (!data->prps)
40 return false;
41
42 /* PRP1 is the only one that can have a non-zero offset, and
43 * because the transfer begins some way into the first page, it needs
44 * to use this. Later entries describe full pages so must be page aligned */
45 vaddr_t vaddr = (vaddr_t) buffer;
46
47 for (size_t i = 0; i < num_pages; i++) {
48 data->prps[i] = vmm_get_phys(vaddr, VMM_FLAG_NONE);
49 nvme_check_dma_addr(phys: data->prps[i], what: "PRP entry");
50 vaddr = PAGE_ALIGN_DOWN(vaddr) + PAGE_SIZE;
51 }
52
53 data->prp_count = num_pages;
54
55 return true;
56}
57
58static bool nvme_setup_prps(struct nvme_command *cmd,
59 struct nvme_bio_data *data) {
60 kassert(data->prp_count > 0);
61
62 cmd->prp1 = data->prps[0];
63 nvme_check_dma_addr(phys: cmd->prp1, what: "prp1");
64
65 kassert(IS_ALIGNED(cmd->prp1, 4)); /* dword aligned */
66 if (data->prp_count == 1) {
67 cmd->prp2 = 0;
68 goto free_prps;
69 } else if (data->prp_count == 2) {
70 cmd->prp2 = data->prps[1];
71 nvme_check_dma_addr(phys: cmd->prp2, what: "prp2");
72 goto free_prps;
73 }
74
75 /* After 2 pages PRP2 is a PRP List pointer. This means
76 * we need to provide another page for the list,
77 * with max_transfer_size always preventing too many */
78 uint64_t entries = data->prp_count - 1;
79 kassert(entries <= NVME_PRPS_PER_PAGE);
80
81 paddr_t list_phys = pmm_alloc_page();
82 if (!list_phys)
83 goto free_prps_fail;
84
85 uint64_t *list = hhdm_paddr_to_ptr(p: list_phys);
86 for (uint64_t i = 0; i < entries; i++)
87 list[i] = data->prps[i + 1];
88
89 data->prp_list_phys = list_phys;
90 cmd->prp2 = list_phys;
91 nvme_check_dma_addr(phys: cmd->prp2, what: "PRP list pointer");
92
93 /* List page has everything needed by the controller */
94free_prps:
95 kfree(data->prps);
96 data->prps = NULL;
97 return true;
98
99free_prps_fail:
100 kfree(data->prps);
101 data->prps = NULL;
102 return false;
103}
104
105static bool rw_send_command(struct block_device *disk, struct nvme_request *req,
106 uint8_t opc) {
107 struct nvme_device *nvme = disk->driver_data;
108 uint16_t qid = THIS_QID(nvme);
109 uint64_t lba = req->lba;
110 uint64_t count = req->sector_count;
111 void *buffer = req->buffer;
112
113 struct nvme_queue *q = nvme->io_queues[qid];
114
115 if (atomic_load(&q->outstanding) >= q->sq_depth) {
116 enqueue_request(dev: nvme, req);
117
118 /* No room */
119 return true;
120 }
121
122 struct nvme_bio_data *data =
123 kmalloc(sizeof(struct nvme_bio_data), ALLOC_FLAGS_ZERO);
124 if (!data)
125 return false;
126
127 if (!nvme_bio_fill_prps(data, buffer, size: count * disk->sector_size)) {
128 kfree(data);
129 return false;
130 }
131
132 req->bio_data = data;
133
134 struct nvme_command cmd = {0};
135 cmd.opc = opc;
136 cmd.nsid = 1;
137 cmd.cdw10 = lba & 0xFFFFFFFFULL;
138 cmd.cdw11 = lba >> 32ULL;
139 cmd.cdw12 = count - 1;
140
141 if (!nvme_setup_prps(cmd: &cmd, data)) {
142 req->bio_data = NULL;
143 kfree(data);
144 return false;
145 }
146
147 req->lba = lba;
148 req->buffer = buffer;
149 req->sector_count = count;
150 req->done = false;
151 req->status = -1;
152
153 nvme_submit_io_cmd(nvme, cmd: &cmd, qid, req);
154
155 return true;
156}
157
158static bool rw_sync(struct block_device *disk, uint64_t lba, uint8_t *buffer,
159 uint16_t count, async_fn function,
160 struct io_wait_token *iowt) {
161 struct nvme_request req = {0};
162 req.lba = lba;
163 req.buffer = buffer;
164 req.sector_count = count;
165 req.remaining_parts = 1;
166 INIT_LIST_HEAD(list: &req.list_node);
167
168 struct thread *curr = thread_get_current();
169
170 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
171 req.waiter = curr;
172
173 if (io_wait_token_active(t: iowt))
174 io_wait_end(t: iowt, act: IO_WAIT_END_NO_OP);
175
176 io_wait_begin(out: iowt, io_object: disk->driver_data);
177
178 function(disk, &req);
179 irql_lower(old_level: irql);
180
181 /* Go run something else now */
182 thread_yield_until_wake_match();
183
184 return !req.status;
185}
186
187static bool rw_wrapper(struct block_device *disk, uint64_t lba, uint8_t *buf,
188 uint64_t cnt, sync_fn function) {
189 struct nvme_device *nvme = (struct nvme_device *) disk->driver_data;
190 uint16_t max_sectors = nvme->max_transfer_size / disk->sector_size;
191 struct io_wait_token iowt = IO_WAIT_TOKEN_EMPTY;
192
193 while (cnt > 0) {
194 uint16_t chunk = (cnt > max_sectors) ? max_sectors : (uint16_t) cnt;
195 if (!function(disk, lba, buf, chunk, &iowt)) {
196 io_wait_end(t: &iowt, act: IO_WAIT_END_YIELD);
197 return false;
198 }
199
200 lba += chunk;
201 buf += chunk * disk->sector_size;
202 cnt -= chunk;
203 }
204
205 /* Reset priority after boost */
206 io_wait_end(t: &iowt, act: IO_WAIT_END_YIELD);
207 return true;
208}
209
210static bool rw_async_wrapper(struct block_device *disk,
211 struct nvme_request *req, async_fn function) {
212 struct nvme_device *nvme = (struct nvme_device *) disk->driver_data;
213 uint16_t max_sectors = nvme->max_transfer_size / disk->sector_size;
214 uint16_t chunk;
215
216 uint64_t cnt = req->sector_count;
217
218 int part_count = 0;
219 uint64_t tmp_cnt = cnt;
220 while (tmp_cnt > 0) {
221 chunk = (tmp_cnt > max_sectors) ? max_sectors : (uint16_t) tmp_cnt;
222 tmp_cnt -= chunk;
223 part_count++;
224 }
225
226 req->remaining_parts = part_count;
227
228 while (cnt > 0) {
229 chunk = (cnt > max_sectors) ? max_sectors : (uint16_t) cnt;
230 cnt -= chunk;
231
232 if (!function(disk, req))
233 return false;
234 }
235
236 return true;
237}
238
239bool nvme_read_sector(struct block_device *disk, uint64_t lba, uint8_t *buffer,
240 uint16_t count, struct io_wait_token *i) {
241 return rw_sync(disk, lba, buffer, count, function: nvme_read_sector_async, iowt: i);
242}
243
244bool nvme_write_sector(struct block_device *disk, uint64_t lba, uint8_t *buffer,
245 uint16_t count, struct io_wait_token *i) {
246 return rw_sync(disk, lba, buffer, count, function: nvme_write_sector_async, iowt: i);
247}
248
249bool nvme_read_sector_wrapper(struct block_device *disk, uint64_t lba,
250 uint8_t *buf, uint64_t cnt) {
251 return rw_wrapper(disk, lba, buf, cnt, function: nvme_read_sector);
252}
253
254bool nvme_write_sector_wrapper(struct block_device *disk, uint64_t lba,
255 const uint8_t *buf, uint64_t cnt) {
256 return rw_wrapper(disk, lba, buf: (uint8_t *) buf, cnt, function: nvme_write_sector);
257}
258
259bool nvme_read_sector_async(struct block_device *disk,
260 struct nvme_request *req) {
261 return rw_send_command(disk, req, NVME_OP_IO_READ);
262}
263
264bool nvme_write_sector_async(struct block_device *disk,
265 struct nvme_request *req) {
266 return rw_send_command(disk, req, NVME_OP_IO_WRITE);
267}
268
269bool nvme_write_sector_async_wrapper(struct block_device *disk,
270 struct nvme_request *req) {
271 return rw_async_wrapper(disk, req, function: nvme_write_sector_async);
272}
273
274bool nvme_read_sector_async_wrapper(struct block_device *disk,
275 struct nvme_request *req) {
276 return rw_async_wrapper(disk, req, function: nvme_read_sector_async);
277}
278
279bool nvme_send_nvme_req(struct block_device *d, struct nvme_request *r) {
280 return rw_send_command(disk: d, req: r, opc: r->write ? NVME_OP_IO_WRITE : NVME_OP_IO_READ);
281}
282