1/* @title: nVME */
2#pragma once
3#include <block/block.h>
4#include <block/sched.h>
5#include <compiler.h>
6#include <stdatomic.h>
7#include <stdint.h>
8#include <sync/semaphore.h>
9#include <sync/spinlock.h>
10#include <thread/workqueue.h>
11
12struct nvme_command {
13 uint8_t opc;
14 uint8_t fuse;
15 uint16_t cid;
16 uint32_t nsid;
17 uint64_t rsvd2;
18 uint64_t mptr;
19 uint64_t prp1;
20 uint64_t prp2;
21 uint32_t cdw10;
22 uint32_t cdw11;
23 uint32_t cdw12;
24 uint32_t cdw13;
25 uint32_t cdw14;
26 uint32_t cdw15;
27} __packed;
28
29struct nvme_completion {
30 uint32_t result;
31 uint32_t rsvd;
32 uint16_t sq_head;
33 uint16_t sq_id;
34 uint16_t cid;
35 uint16_t status;
36} __packed;
37
38struct nvme_cc {
39 union {
40 uint32_t raw;
41 struct {
42 uint32_t en : 1;
43 uint32_t __reserved0 : 3;
44 uint32_t css : 3;
45 uint32_t mps : 4;
46 uint32_t ams : 3;
47 uint32_t shn : 2;
48 uint32_t iosqes : 4;
49 uint32_t iocqes : 4;
50 uint32_t __reserved1 : 8;
51 };
52 };
53} __packed;
54static_assert_struct_size_eq(nvme_cc, sizeof(uint32_t));
55
56struct nvme_regs {
57 uint32_t cap_lo;
58 uint32_t cap_hi;
59 uint32_t version;
60 uint32_t intms;
61 uint32_t intmc;
62 struct nvme_cc cc;
63 uint32_t nssr;
64 uint32_t csts;
65 uint32_t reserved1;
66 uint32_t aqa;
67 uint32_t asq_lo;
68 uint32_t asq_hi;
69 uint32_t acq_lo;
70 uint32_t acq_hi;
71 uint32_t reserved4[1018];
72} __attribute__((aligned));
73
74struct nvme_bio_data {
75 uint64_t *prps;
76 uint64_t prp_count;
77};
78
79struct nvme_request {
80 uint32_t qid;
81 uint64_t lba;
82 void *buffer;
83 uint64_t size;
84 uint64_t sector_count;
85 bool write;
86
87 volatile bool done;
88 volatile uint16_t status;
89 int32_t remaining_parts;
90
91 void (*on_complete)(struct nvme_request *);
92 struct nvme_bio_data *bio_data;
93 struct thread *waiter;
94
95 void *user_data;
96
97 struct list_head list_node;
98};
99
100struct nvme_waiting_requests {
101 struct spinlock lock;
102 struct list_head list;
103};
104
105struct nvme_queue {
106 struct nvme_command *sq; // Submission queue (virtual)
107 struct nvme_completion *cq; // Completion queue (virtual)
108 uint64_t sq_phys; // Submission queue physical address
109 uint64_t cq_phys; // Completion queue physical address
110
111 uint16_t sq_tail; // Tail index for submission
112 uint16_t cq_head; // Head index for completion
113 uint16_t sq_depth; // Queue depth (entries)
114 uint16_t cq_depth; // Queue depth (entries)
115 uint8_t cq_phase; // Phase bit for completion
116 uint32_t *sq_db;
117 uint32_t *cq_db;
118
119 struct nvme_request **sq_requests;
120 _Atomic uint16_t outstanding;
121
122 struct spinlock lock;
123};
124
125struct nvme_device {
126 struct nvme_regs *regs;
127 uint64_t cap;
128 uint32_t version;
129 uint32_t doorbell_stride;
130 uint32_t page_size;
131 uint32_t *admin_sq_db;
132 uint32_t *admin_cq_db;
133
134 struct nvme_command *admin_sq;
135 struct nvme_completion *admin_cq;
136 uint64_t admin_sq_phys;
137 uint64_t admin_cq_phys;
138
139 uint16_t admin_sq_tail;
140 uint16_t admin_cq_head;
141 uint16_t admin_q_depth;
142 uint8_t admin_cq_phase;
143
144 /* Array of pointers to queues */
145 struct nvme_queue **io_queues;
146
147 struct nvme_waiting_requests waiting_requests;
148 struct nvme_waiting_requests finished_requests;
149 struct work work;
150
151 atomic_bool on_sem;
152 struct semaphore sem;
153
154 uint8_t *isr_index;
155 uint32_t queue_count;
156
157 uint32_t sector_size;
158 uint64_t max_transfer_size;
159 struct block_device *generic_disk;
160
161 _Atomic uint64_t total_outstanding;
162
163 struct workqueue *workqueue;
164};
165
166struct nvme_identify {
167 uint8_t data[PAGE_SIZE];
168};
169
170struct nvme_lbaf {
171 uint16_t ms; // Metadata size
172 uint8_t lbads; // LBA data size (log2 of sector size)
173 uint8_t rp : 2; // Relative performance
174 uint8_t reserved : 6;
175} __packed;
176
177struct nvme_identify_namespace {
178 uint64_t nsze; // Namespace Size
179 uint64_t ncap; // Namespace Capacity
180 uint64_t nuse; // Namespace Utilization
181 uint8_t nsfeat; // Namespace Features
182 uint8_t nlbaf; // Number of LBA formats
183 uint8_t flbas; // Formatted LBA Size
184 uint8_t mc; // Metadata Capabilities
185 uint8_t dpc; // End-to-end Data Protection Capabilities
186 uint8_t dps; // End-to-end Data Protection Type Settings
187 uint8_t nmic; // Namespace Multipath I/O and Namespace Sharing Capabilities
188 uint8_t rescap; // Reservation Capabilities
189 uint8_t fpi; // Format Progress Indicator
190 uint8_t dlfeat; // Deallocate Logical Block Features
191 uint16_t nawun; // Namespace Atomic Write Unit Normal
192 uint16_t nawupf; // Namespace Atomic Write Unit Power Fail
193 uint16_t nacwu; // Namespace Atomic Compare & Write Unit
194 uint16_t nabsn; // Namespace Atomic Boundary Size Normal
195 uint16_t nabo; // Namespace Atomic Boundary Offset
196 uint16_t nabspf; // Namespace Atomic Boundary Size Power Fail
197 uint16_t noiob; // Namespace Optimal IO Boundary
198 uint64_t nvmcap[2]; // Namespace NVM Capacity
199 uint16_t npwg;
200 uint16_t npwa;
201 uint16_t npdg;
202 uint16_t npda;
203 uint16_t nows;
204 uint16_t mssrl;
205 uint32_t mcl;
206 uint8_t msrc;
207 uint8_t reserved0[11];
208 uint32_t adagrpid;
209 uint8_t reserved1[3];
210 uint8_t nsattr;
211 uint16_t nvmsetid;
212 uint16_t endgid;
213 uint64_t nguid[2];
214 uint64_t eui64;
215 struct nvme_lbaf lbaf[64]; // LBA format descriptions
216 uint8_t vendor_specific[3712];
217} __packed;
218static_assert_struct_size_eq(nvme_identify_namespace, 0x1000);
219
220struct nvme_identify_controller {
221 uint16_t vid; // PCI Vendor ID
222 uint16_t ssvid; // Subsystem Vendor ID
223 char sn[20]; // Serial Number (ASCII)
224 char mn[40]; // Model Number (ASCII)
225 char fr[8]; // Firmware Revision (ASCII)
226 uint8_t rab; // Recommended Arbitration Burst
227 uint8_t ieee[3]; // IEEE OUI Identifier
228 uint8_t mic; // Multi-interface Capabilities
229 uint8_t mdts; // Maximum Data Transfer Size
230 uint16_t cntlid; // Controller ID
231 uint32_t ver; // Version
232 uint32_t rtd3r; // RTD3 Resume Latency
233 uint32_t rtd3e; // RTD3 Entry Latency
234 uint32_t oaes; // Optional Admin Command Support
235 uint32_t ctratt; // Controller Attributes
236 uint8_t rsvd96[156];
237 uint16_t oacs; // Optional Admin Command Support (bit flags)
238 uint8_t acl; // Abort Command Limit
239 uint8_t aerl; // Asynchronous Event Request Limit
240 uint8_t frmw; // Firmware Updates
241 uint8_t lpa; // Log Page Attributes
242 uint8_t elpe; // Error Log Page Entries
243 uint8_t npss; // Number of Power States Support
244 uint8_t avscc; // Admin Vendor Specific Command Configuration
245 uint8_t apsta; // Autonomous Power State Transition Attributes
246 uint16_t wctemp; // Warning Composite Temperature Threshold
247 uint16_t cctemp; // Critical Composite Temperature Threshold
248 uint16_t mtfa; // Maximum Time for Firmware Activation
249 uint32_t hmpre; // Host Memory Buffer Preferred Size
250 uint32_t hmmin; // Host Memory Buffer Minimum Size
251 uint64_t tnvmcap[2]; // Total NVM Capacity
252 uint64_t unvmcap[2]; // Unallocated NVM Capacity
253 uint32_t rpmbs; // Replay Protected Memory Block Support
254 uint16_t edstt; // Extended Device Self-test Time
255 uint8_t dsto; // Device Self-test Options
256 uint8_t fwug; // Firmware Update Granularity
257 uint16_t kas; // Keep Alive Support
258 uint16_t hctma; // Host Controlled Thermal Management Attributes
259 uint16_t mntmt; // Minimum Thermal Management Temperature
260 uint16_t mxtmt; // Maximum Thermal Management Temperature
261 uint32_t sanicap; // Sanitize Capabilities
262 uint8_t rsvd228[180];
263 uint8_t sqes; // Submission Queue Entry Size
264 uint8_t cqes; // Completion Queue Entry Size
265 // TODO: there is more but me lazy and dont need it
266} __packed;
267
268uint16_t nvme_submit_admin_cmd(struct nvme_device *nvme,
269 struct nvme_command *cmd, uint32_t *);
270uint32_t nvme_set_num_queues(struct nvme_device *nvme, uint16_t desired_sq,
271 uint16_t desired_cq);
272
273void nvme_submit_io_cmd(struct nvme_device *nvme, struct nvme_command *cmd,
274 uint32_t qid, struct nvme_request *req);
275
276uint8_t *nvme_identify_controller(struct nvme_device *nvme);
277uint8_t *nvme_identify_namespace(struct nvme_device *nvme, uint32_t nsid);
278void nvme_enable_controller(struct nvme_device *nvme);
279void nvme_setup_admin_queues(struct nvme_device *nvme);
280void nvme_alloc_admin_queues(struct nvme_device *nvme);
281void nvme_alloc_io_queues(struct nvme_device *nvme, uint32_t qid);
282struct nvme_device *nvme_discover_device(uint8_t bus, uint8_t slot,
283 uint8_t func);
284struct block_device *nvme_create_generic(struct nvme_device *nvme);
285void nvme_print_identify(const struct nvme_identify_controller *ctrl);
286void nvme_print_namespace(const struct nvme_identify_namespace *ns);
287
288bool nvme_read_sector_wrapper(struct block_device *disk, uint64_t lba,
289 uint8_t *buf, uint64_t cnt);
290
291bool nvme_read_sector_async_wrapper(struct block_device *disk,
292 struct nvme_request *req);
293
294bool nvme_write_sector_async_wrapper(struct block_device *disk,
295 struct nvme_request *req);
296
297bool nvme_send_nvme_req(struct block_device *d, struct nvme_request *r);
298
299bool nvme_write_sector_wrapper(struct block_device *disk, uint64_t lba,
300 const uint8_t *buf, uint64_t cnt);
301
302enum irq_result nvme_isr_handler(void *ctx, uint8_t vector,
303 struct irq_context *rsp);
304
305bool nvme_submit_bio_request(struct block_device *disk,
306 struct bio_request *bio);
307
308bool nvme_should_coalesce(struct block_device *disk,
309 const struct bio_request *a,
310 const struct bio_request *b);
311
312void nvme_do_coalesce(struct block_device *disk, struct bio_request *into,
313 struct bio_request *from);
314
315void nvme_reorder(struct block_device *disk);
316void nvme_work(void *rvoid, void *dvoid);
317