1#include <block/bio.h>
2#include <block/block.h>
3#include <block/sched.h>
4#include <drivers/nvme.h>
5#include <mem/alloc.h>
6#include <mem/vmm.h>
7#include <stdbool.h>
8#include <stddef.h>
9#include <stdint.h>
10#include <structures/sll.h>
11#include <thread/io_wait.h>
12#include <thread/workqueue.h>
13
14#include "internal.h"
15
16typedef bool (*sync_fn)(struct block_device *, uint64_t, uint8_t *, uint16_t,
17 struct io_wait_token *iowt);
18typedef bool (*async_fn)(struct block_device *, struct nvme_request *);
19
20static void enqueue_request(struct nvme_device *dev, struct nvme_request *req) {
21 struct nvme_waiting_requests *q = &dev->waiting_requests;
22
23 enum irql irql = spin_lock_irq_disable(lock: &q->lock);
24
25 list_add_tail(new: &req->list_node, head: &q->list);
26
27 spin_unlock(lock: &q->lock, old: irql);
28}
29
30static bool nvme_bio_fill_prps(struct nvme_bio_data *data, const void *buffer,
31 uint64_t size) {
32 uint64_t offset = (uintptr_t) buffer & (PAGE_SIZE - 1);
33 uint64_t num_pages = PAGES_NEEDED_FOR(offset + size);
34
35 data->prps = kmalloc(sizeof(uint64_t) * num_pages);
36 if (!data->prps)
37 return false;
38
39 vaddr_t vaddr = PAGE_ALIGN_DOWN(buffer);
40
41 for (size_t i = 0; i < num_pages; i++) {
42 data->prps[i] = vmm_get_phys(vaddr, VMM_FLAG_NONE);
43 vaddr += PAGE_SIZE;
44 }
45
46 data->prp_count = num_pages;
47
48 return true;
49}
50
51static void nvme_setup_prps(struct nvme_command *cmd,
52 struct nvme_bio_data *data) {
53 kassert(data->prp_count > 0);
54
55 cmd->prp1 = data->prps[0];
56
57 if (data->prp_count == 1) {
58 cmd->prp2 = 0;
59 goto free_prps;
60 } else if (data->prp_count == 2) {
61 cmd->prp2 = data->prps[1];
62 goto free_prps;
63 } else {
64 cmd->prp2 = vmm_get_phys((uint64_t) (&data->prps[1]), VMM_FLAG_NONE);
65 }
66
67 return;
68
69 /* For when there is no need for multiple PRPs */
70free_prps:
71 kfree(data->prps);
72 data->prps = NULL;
73 return;
74}
75
76static bool rw_send_command(struct block_device *disk, struct nvme_request *req,
77 uint8_t opc) {
78 struct nvme_device *nvme = disk->driver_data;
79 uint16_t qid = THIS_QID(nvme);
80 uint64_t lba = req->lba;
81 uint64_t count = req->sector_count;
82 void *buffer = req->buffer;
83
84 struct nvme_queue *q = nvme->io_queues[qid];
85
86 if (atomic_load(&q->outstanding) >= q->sq_depth) {
87 enqueue_request(dev: nvme, req);
88
89 /* No room */
90 return true;
91 }
92
93 struct nvme_bio_data *data =
94 kmalloc(sizeof(struct nvme_bio_data), ALLOC_FLAGS_ZERO);
95 if (!data)
96 return false;
97
98 if (!nvme_bio_fill_prps(data, buffer, size: count * disk->sector_size)) {
99 kfree(data);
100 return false;
101 }
102
103 req->bio_data = data;
104
105 struct nvme_command cmd = {0};
106 cmd.opc = opc;
107 cmd.nsid = 1;
108 cmd.cdw10 = lba & 0xFFFFFFFFULL;
109 cmd.cdw11 = lba >> 32ULL;
110 cmd.cdw12 = count - 1;
111
112 nvme_setup_prps(cmd: &cmd, data);
113
114 req->lba = lba;
115 req->buffer = buffer;
116 req->sector_count = count;
117 req->done = false;
118 req->status = -1;
119
120 nvme_submit_io_cmd(nvme, cmd: &cmd, qid, req);
121
122 return true;
123}
124
125static bool rw_sync(struct block_device *disk, uint64_t lba, uint8_t *buffer,
126 uint16_t count, async_fn function,
127 struct io_wait_token *iowt) {
128 struct nvme_request req = {0};
129 req.lba = lba;
130 req.buffer = buffer;
131 req.sector_count = count;
132 req.remaining_parts = 1;
133 INIT_LIST_HEAD(list: &req.list_node);
134
135 struct thread *curr = thread_get_current();
136
137 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
138 req.waiter = curr;
139
140 if (io_wait_token_active(t: iowt))
141 io_wait_end(t: iowt, act: IO_WAIT_END_NO_OP);
142
143 io_wait_begin(out: iowt, io_object: disk->driver_data);
144
145 function(disk, &req);
146 irql_lower(old_level: irql);
147
148 /* Go run something else now */
149 thread_yield_until_wake_match();
150
151 return !req.status;
152}
153
154static bool rw_wrapper(struct block_device *disk, uint64_t lba, uint8_t *buf,
155 uint64_t cnt, sync_fn function) {
156 struct nvme_device *nvme = (struct nvme_device *) disk->driver_data;
157 uint16_t max_sectors = nvme->max_transfer_size / disk->sector_size;
158 struct io_wait_token iowt = IO_WAIT_TOKEN_EMPTY;
159
160 while (cnt > 0) {
161 uint16_t chunk = (cnt > max_sectors) ? max_sectors : (uint16_t) cnt;
162 if (!function(disk, lba, buf, chunk, &iowt)) {
163 io_wait_end(t: &iowt, act: IO_WAIT_END_YIELD);
164 return false;
165 }
166
167 lba += chunk;
168 buf += chunk * disk->sector_size;
169 cnt -= chunk;
170 }
171
172 /* Reset priority after boost */
173 io_wait_end(t: &iowt, act: IO_WAIT_END_YIELD);
174 return true;
175}
176
177static bool rw_async_wrapper(struct block_device *disk,
178 struct nvme_request *req, async_fn function) {
179 struct nvme_device *nvme = (struct nvme_device *) disk->driver_data;
180 uint16_t max_sectors = nvme->max_transfer_size / disk->sector_size;
181 uint16_t chunk;
182
183 uint64_t cnt = req->sector_count;
184
185 int part_count = 0;
186 uint64_t tmp_cnt = cnt;
187 while (tmp_cnt > 0) {
188 chunk = (tmp_cnt > max_sectors) ? max_sectors : (uint16_t) tmp_cnt;
189 tmp_cnt -= chunk;
190 part_count++;
191 }
192
193 req->remaining_parts = part_count;
194
195 while (cnt > 0) {
196 chunk = (cnt > max_sectors) ? max_sectors : (uint16_t) cnt;
197 cnt -= chunk;
198
199 if (!function(disk, req))
200 return false;
201 }
202
203 return true;
204}
205
206bool nvme_read_sector(struct block_device *disk, uint64_t lba, uint8_t *buffer,
207 uint16_t count, struct io_wait_token *i) {
208 return rw_sync(disk, lba, buffer, count, function: nvme_read_sector_async, iowt: i);
209}
210
211bool nvme_write_sector(struct block_device *disk, uint64_t lba, uint8_t *buffer,
212 uint16_t count, struct io_wait_token *i) {
213 return rw_sync(disk, lba, buffer, count, function: nvme_write_sector_async, iowt: i);
214}
215
216bool nvme_read_sector_wrapper(struct block_device *disk, uint64_t lba,
217 uint8_t *buf, uint64_t cnt) {
218 return rw_wrapper(disk, lba, buf, cnt, function: nvme_read_sector);
219}
220
221bool nvme_write_sector_wrapper(struct block_device *disk, uint64_t lba,
222 const uint8_t *buf, uint64_t cnt) {
223 return rw_wrapper(disk, lba, buf: (uint8_t *) buf, cnt, function: nvme_write_sector);
224}
225
226bool nvme_read_sector_async(struct block_device *disk,
227 struct nvme_request *req) {
228 return rw_send_command(disk, req, NVME_OP_IO_READ);
229}
230
231bool nvme_write_sector_async(struct block_device *disk,
232 struct nvme_request *req) {
233 return rw_send_command(disk, req, NVME_OP_IO_WRITE);
234}
235
236bool nvme_write_sector_async_wrapper(struct block_device *disk,
237 struct nvme_request *req) {
238 return rw_async_wrapper(disk, req, function: nvme_write_sector_async);
239}
240
241bool nvme_read_sector_async_wrapper(struct block_device *disk,
242 struct nvme_request *req) {
243 return rw_async_wrapper(disk, req, function: nvme_read_sector_async);
244}
245
246bool nvme_send_nvme_req(struct block_device *d, struct nvme_request *r) {
247 return rw_send_command(disk: d, req: r, opc: r->write ? NVME_OP_IO_WRITE : NVME_OP_IO_READ);
248}
249