1/* @title: nVME */
2#pragma once
3#include <block/block.h>
4#include <block/sched.h>
5#include <compiler.h>
6#include <stdatomic.h>
7#include <stdint.h>
8#include <sync/semaphore.h>
9#include <sync/spinlock.h>
10#include <thread/workqueue.h>
11
12struct nvme_command {
13 uint8_t opc;
14 uint8_t fuse;
15 uint16_t cid;
16 uint32_t nsid;
17 uint64_t rsvd2;
18 uint64_t mptr;
19 uint64_t prp1;
20 uint64_t prp2;
21 uint32_t cdw10;
22 uint32_t cdw11;
23 uint32_t cdw12;
24 uint32_t cdw13;
25 uint32_t cdw14;
26 uint32_t cdw15;
27} __packed;
28
29struct nvme_completion {
30 uint32_t result;
31 uint32_t rsvd;
32 uint16_t sq_head;
33 uint16_t sq_id;
34 uint16_t cid;
35 uint16_t status;
36} __packed;
37
38struct nvme_cc {
39 union {
40 uint32_t raw;
41 struct {
42 uint32_t en : 1;
43 uint32_t __reserved0 : 3;
44 uint32_t css : 3;
45 uint32_t mps : 4;
46 uint32_t ams : 3;
47 uint32_t shn : 2;
48 uint32_t iosqes : 4;
49 uint32_t iocqes : 4;
50 uint32_t __reserved1 : 8;
51 };
52 };
53} __packed;
54static_assert_struct_size_eq(nvme_cc, sizeof(uint32_t));
55
56struct nvme_regs {
57 uint32_t cap_lo;
58 uint32_t cap_hi;
59 uint32_t version;
60 uint32_t intms;
61 uint32_t intmc;
62 struct nvme_cc cc;
63 uint32_t nssr;
64 uint32_t csts;
65 uint32_t reserved1;
66 uint32_t aqa;
67 uint32_t asq_lo;
68 uint32_t asq_hi;
69 uint32_t acq_lo;
70 uint32_t acq_hi;
71 uint32_t reserved4[1018];
72} __attribute__((aligned));
73
74struct nvme_bio_data {
75 uint64_t *prps;
76 uint64_t prp_count;
77
78 /* PRP List page handed to the controller, or just
79 * 0 when the transfer fits in PRP1/2 */
80 paddr_t prp_list_phys;
81};
82
83struct nvme_request {
84 uint32_t qid;
85 uint64_t lba;
86 void *buffer;
87 uint64_t size;
88 uint64_t sector_count;
89 bool write;
90
91 volatile bool done;
92 volatile uint16_t status;
93 int32_t remaining_parts;
94
95 void (*on_complete)(struct nvme_request *);
96 struct nvme_bio_data *bio_data;
97 struct thread *waiter;
98
99 void *user_data;
100
101 struct list_head list_node;
102};
103
104struct nvme_waiting_requests {
105 struct spinlock lock;
106 struct list_head list;
107};
108
109struct nvme_queue {
110 struct nvme_command *sq; // Submission queue (virtual)
111 struct nvme_completion *cq; // Completion queue (virtual)
112 uint64_t sq_phys; // Submission queue physical address
113 uint64_t cq_phys; // Completion queue physical address
114
115 uint16_t sq_tail; // Tail index for submission
116 uint16_t cq_head; // Head index for completion
117 uint16_t sq_depth; // Queue depth (entries)
118 uint16_t cq_depth; // Queue depth (entries)
119 uint8_t cq_phase; // Phase bit for completion
120 uint32_t *sq_db;
121 uint32_t *cq_db;
122
123 struct nvme_request **sq_requests;
124 _Atomic uint16_t outstanding;
125
126 struct spinlock lock;
127};
128
129struct nvme_device {
130 struct nvme_regs *regs;
131 uint64_t cap;
132 uint32_t version;
133 uint32_t doorbell_stride;
134 uint32_t page_size;
135 uint32_t *admin_sq_db;
136 uint32_t *admin_cq_db;
137
138 struct nvme_command *admin_sq;
139 struct nvme_completion *admin_cq;
140 uint64_t admin_sq_phys;
141 uint64_t admin_cq_phys;
142
143 uint16_t admin_sq_tail;
144 uint16_t admin_cq_head;
145 uint16_t admin_q_depth;
146 uint8_t admin_cq_phase;
147
148 /* Array of pointers to queues */
149 struct nvme_queue **io_queues;
150
151 struct nvme_waiting_requests waiting_requests;
152 struct nvme_waiting_requests finished_requests;
153 struct work work;
154
155 atomic_bool on_sem;
156 struct semaphore sem;
157
158 uint8_t *isr_index;
159 uint32_t queue_count;
160
161 uint32_t sector_size;
162 uint64_t max_transfer_size;
163 struct block_device *generic_disk;
164
165 _Atomic uint64_t total_outstanding;
166
167 struct workqueue *workqueue;
168};
169
170struct nvme_identify {
171 uint8_t data[PAGE_SIZE];
172};
173
174struct nvme_lbaf {
175 uint16_t ms; // Metadata size
176 uint8_t lbads; // LBA data size (log2 of sector size)
177 uint8_t rp : 2; // Relative performance
178 uint8_t reserved : 6;
179} __packed;
180
181struct nvme_identify_namespace {
182 uint64_t nsze; // Namespace Size
183 uint64_t ncap; // Namespace Capacity
184 uint64_t nuse; // Namespace Utilization
185 uint8_t nsfeat; // Namespace Features
186 uint8_t nlbaf; // Number of LBA formats
187 uint8_t flbas; // Formatted LBA Size
188 uint8_t mc; // Metadata Capabilities
189 uint8_t dpc; // End-to-end Data Protection Capabilities
190 uint8_t dps; // End-to-end Data Protection Type Settings
191 uint8_t nmic; // Namespace Multipath I/O and Namespace Sharing Capabilities
192 uint8_t rescap; // Reservation Capabilities
193 uint8_t fpi; // Format Progress Indicator
194 uint8_t dlfeat; // Deallocate Logical Block Features
195 uint16_t nawun; // Namespace Atomic Write Unit Normal
196 uint16_t nawupf; // Namespace Atomic Write Unit Power Fail
197 uint16_t nacwu; // Namespace Atomic Compare & Write Unit
198 uint16_t nabsn; // Namespace Atomic Boundary Size Normal
199 uint16_t nabo; // Namespace Atomic Boundary Offset
200 uint16_t nabspf; // Namespace Atomic Boundary Size Power Fail
201 uint16_t noiob; // Namespace Optimal IO Boundary
202 uint64_t nvmcap[2]; // Namespace NVM Capacity
203 uint16_t npwg;
204 uint16_t npwa;
205 uint16_t npdg;
206 uint16_t npda;
207 uint16_t nows;
208 uint16_t mssrl;
209 uint32_t mcl;
210 uint8_t msrc;
211 uint8_t reserved0[11];
212 uint32_t adagrpid;
213 uint8_t reserved1[3];
214 uint8_t nsattr;
215 uint16_t nvmsetid;
216 uint16_t endgid;
217 uint64_t nguid[2];
218 uint64_t eui64;
219 struct nvme_lbaf lbaf[64]; // LBA format descriptions
220 uint8_t vendor_specific[3712];
221} __packed;
222static_assert_struct_size_eq(nvme_identify_namespace, 0x1000);
223
224struct nvme_identify_controller {
225 uint16_t vid; // PCI Vendor ID
226 uint16_t ssvid; // Subsystem Vendor ID
227 char sn[20]; // Serial Number (ASCII)
228 char mn[40]; // Model Number (ASCII)
229 char fr[8]; // Firmware Revision (ASCII)
230 uint8_t rab; // Recommended Arbitration Burst
231 uint8_t ieee[3]; // IEEE OUI Identifier
232 uint8_t mic; // Multi-interface Capabilities
233 uint8_t mdts; // Maximum Data Transfer Size
234 uint16_t cntlid; // Controller ID
235 uint32_t ver; // Version
236 uint32_t rtd3r; // RTD3 Resume Latency
237 uint32_t rtd3e; // RTD3 Entry Latency
238 uint32_t oaes; // Optional Admin Command Support
239 uint32_t ctratt; // Controller Attributes
240 uint8_t rsvd96[156];
241 uint16_t oacs; // Optional Admin Command Support (bit flags)
242 uint8_t acl; // Abort Command Limit
243 uint8_t aerl; // Asynchronous Event Request Limit
244 uint8_t frmw; // Firmware Updates
245 uint8_t lpa; // Log Page Attributes
246 uint8_t elpe; // Error Log Page Entries
247 uint8_t npss; // Number of Power States Support
248 uint8_t avscc; // Admin Vendor Specific Command Configuration
249 uint8_t apsta; // Autonomous Power State Transition Attributes
250 uint16_t wctemp; // Warning Composite Temperature Threshold
251 uint16_t cctemp; // Critical Composite Temperature Threshold
252 uint16_t mtfa; // Maximum Time for Firmware Activation
253 uint32_t hmpre; // Host Memory Buffer Preferred Size
254 uint32_t hmmin; // Host Memory Buffer Minimum Size
255 uint64_t tnvmcap[2]; // Total NVM Capacity
256 uint64_t unvmcap[2]; // Unallocated NVM Capacity
257 uint32_t rpmbs; // Replay Protected Memory Block Support
258 uint16_t edstt; // Extended Device Self-test Time
259 uint8_t dsto; // Device Self-test Options
260 uint8_t fwug; // Firmware Update Granularity
261 uint16_t kas; // Keep Alive Support
262 uint16_t hctma; // Host Controlled Thermal Management Attributes
263 uint16_t mntmt; // Minimum Thermal Management Temperature
264 uint16_t mxtmt; // Maximum Thermal Management Temperature
265 uint32_t sanicap; // Sanitize Capabilities
266 uint8_t rsvd228[180];
267 uint8_t sqes; // Submission Queue Entry Size
268 uint8_t cqes; // Completion Queue Entry Size
269 // TODO: there is more but me lazy and dont need it
270} __packed;
271
272uint16_t nvme_submit_admin_cmd(struct nvme_device *nvme,
273 struct nvme_command *cmd, uint32_t *);
274uint32_t nvme_set_num_queues(struct nvme_device *nvme, uint16_t desired_sq,
275 uint16_t desired_cq);
276
277void nvme_submit_io_cmd(struct nvme_device *nvme, struct nvme_command *cmd,
278 uint32_t qid, struct nvme_request *req);
279
280uint8_t *nvme_identify_controller(struct nvme_device *nvme);
281uint8_t *nvme_identify_namespace(struct nvme_device *nvme, uint32_t nsid);
282void nvme_enable_controller(struct nvme_device *nvme);
283void nvme_setup_admin_queues(struct nvme_device *nvme);
284void nvme_alloc_admin_queues(struct nvme_device *nvme);
285void nvme_alloc_io_queues(struct nvme_device *nvme, uint32_t qid);
286struct nvme_device *nvme_discover_device(uint8_t bus, uint8_t slot,
287 uint8_t func);
288struct block_device *nvme_create_generic(struct nvme_device *nvme);
289void nvme_print_identify(const struct nvme_identify_controller *ctrl);
290void nvme_print_namespace(const struct nvme_identify_namespace *ns);
291
292bool nvme_read_sector_wrapper(struct block_device *disk, uint64_t lba,
293 uint8_t *buf, uint64_t cnt);
294
295bool nvme_read_sector_async_wrapper(struct block_device *disk,
296 struct nvme_request *req);
297
298bool nvme_write_sector_async_wrapper(struct block_device *disk,
299 struct nvme_request *req);
300
301bool nvme_send_nvme_req(struct block_device *d, struct nvme_request *r);
302
303bool nvme_write_sector_wrapper(struct block_device *disk, uint64_t lba,
304 const uint8_t *buf, uint64_t cnt);
305
306enum irq_result nvme_isr_handler(void *ctx, uint8_t vector,
307 struct irq_context *rsp);
308
309bool nvme_submit_bio_request(struct block_device *disk,
310 struct bio_request *bio);
311
312bool nvme_should_coalesce(struct block_device *disk,
313 const struct bio_request *a,
314 const struct bio_request *b);
315
316void nvme_do_coalesce(struct block_device *disk, struct bio_request *into,
317 struct bio_request *from);
318
319void nvme_reorder(struct block_device *disk);
320void nvme_work(void *rvoid, void *dvoid);
321