1/* @title: nVME */
2#pragma once
3#include <atomic.h>
4#include <block/block.h>
5#include <block/sched.h>
6#include <compiler/core.h>
7#include <stdint.h>
8#include <sync/event.h>
9#include <sync/semaphore.h>
10#include <sync/spinlock.h>
11#include <thread/workqueue.h>
12
13/* NOTE: offsets and sizes asserted in kernel/drivers/nvme/layout.c */
14struct nvme_command {
15 uint8_t opc;
16 uint8_t fuse;
17 uint16_t cid;
18 uint32_t nsid;
19 uint64_t rsvd2;
20 uint64_t mptr;
21 uint64_t prp1;
22 uint64_t prp2;
23 uint32_t cdw10;
24 uint32_t cdw11;
25 uint32_t cdw12;
26 uint32_t cdw13;
27 uint32_t cdw14;
28 uint32_t cdw15;
29} cc_packed;
30
31struct nvme_completion {
32 uint32_t result;
33 uint32_t rsvd;
34 uint16_t sq_head;
35 uint16_t sq_id;
36 uint16_t cid;
37 uint16_t status;
38} cc_packed;
39
40struct nvme_cc {
41 union {
42 uint32_t raw;
43 struct {
44 uint32_t en : 1;
45 uint32_t __reserved0 : 3;
46 uint32_t css : 3;
47 uint32_t mps : 4;
48 uint32_t ams : 3;
49 uint32_t shn : 2;
50 uint32_t iosqes : 4;
51 uint32_t iocqes : 4;
52 uint32_t __reserved1 : 8;
53 };
54 };
55} cc_packed;
56struct nvme_regs {
57 uint32_t cap_lo;
58 uint32_t cap_hi;
59 uint32_t version;
60 uint32_t intms;
61 uint32_t intmc;
62 struct nvme_cc cc;
63 uint32_t nssr;
64 uint32_t csts;
65 uint32_t reserved1;
66 uint32_t aqa;
67 uint32_t asq_lo;
68 uint32_t asq_hi;
69 uint32_t acq_lo;
70 uint32_t acq_hi;
71 uint32_t reserved4[1018];
72} __attribute__((aligned));
73
74struct nvme_bio_data {
75 uint64_t *prps;
76 uint64_t prp_count;
77
78 /* PRP List page handed to the controller, or just
79 * 0 when the transfer fits in PRP1/2 */
80 paddr_t prp_list_phys;
81};
82
83struct nvme_request {
84 uint32_t qid;
85 uint64_t lba;
86 void *buffer;
87 uint64_t size;
88 uint64_t sector_count;
89 bool write;
90
91 volatile bool done;
92 volatile uint16_t status;
93 int32_t remaining_parts;
94
95 void (*on_complete)(struct nvme_request *);
96 struct nvme_bio_data *bio_data;
97 struct event evt;
98 bool has_waiter;
99
100 void *user_data;
101
102 struct list_node list_node;
103};
104
105struct nvme_waiting_requests {
106 struct spinlock lock;
107 struct list_head list;
108};
109
110struct nvme_queue {
111 struct nvme_command *sq; // Submission queue (virtual)
112 struct nvme_completion *cq; // Completion queue (virtual)
113 uint64_t sq_phys; // Submission queue physical address
114 uint64_t cq_phys; // Completion queue physical address
115
116 uint16_t sq_tail; // Tail index for submission
117 uint16_t cq_head; // Head index for completion
118 uint16_t sq_depth; // Queue depth (entries)
119 uint16_t cq_depth; // Queue depth (entries)
120 uint8_t cq_phase; // Phase bit for completion
121 uint32_t cc_mem_io *sq_db;
122 uint32_t cc_mem_io *cq_db;
123
124 struct nvme_request **sq_requests;
125 atomic_uint16_t outstanding;
126
127 struct spinlock lock;
128};
129
130struct nvme_device {
131 struct nvme_regs cc_mem_io *regs;
132 uint64_t cap;
133 uint32_t version;
134 uint32_t doorbell_stride;
135 uint32_t page_size;
136 uint32_t cc_mem_io *admin_sq_db;
137 uint32_t cc_mem_io *admin_cq_db;
138
139 struct nvme_command *admin_sq;
140 struct nvme_completion *admin_cq;
141 uint64_t admin_sq_phys;
142 uint64_t admin_cq_phys;
143
144 uint16_t admin_sq_tail;
145 uint16_t admin_cq_head;
146 uint16_t admin_q_depth;
147 uint8_t admin_cq_phase;
148
149 /* Array of pointers to queues */
150 struct nvme_queue **io_queues;
151
152 struct nvme_waiting_requests waiting_requests;
153 struct nvme_waiting_requests finished_requests;
154 struct work work;
155
156 atomic_bool on_sem;
157 struct semaphore sem;
158
159 uint8_t *isr_index;
160 uint32_t queue_count;
161
162 uint32_t sector_size;
163 uint64_t max_transfer_size;
164 struct block_device *generic_disk;
165
166 atomic_uint64_t total_outstanding;
167
168 struct workqueue *workqueue;
169};
170
171struct nvme_identify {
172 uint8_t data[PAGE_SIZE];
173};
174
175struct nvme_lbaf {
176 uint16_t ms; // Metadata size
177 uint8_t lbads; // LBA data size (log2 of sector size)
178 uint8_t rp : 2; // Relative performance
179 uint8_t reserved : 6;
180} cc_packed;
181
182struct nvme_identify_namespace {
183 uint64_t nsze; // Namespace Size
184 uint64_t ncap; // Namespace Capacity
185 uint64_t nuse; // Namespace Utilization
186 uint8_t nsfeat; // Namespace Features
187 uint8_t nlbaf; // Number of LBA formats
188 uint8_t flbas; // Formatted LBA Size
189 uint8_t mc; // Metadata Capabilities
190 uint8_t dpc; // End-to-end Data Protection Capabilities
191 uint8_t dps; // End-to-end Data Protection Type Settings
192 uint8_t nmic; // Namespace Multipath I/O and Namespace Sharing Capabilities
193 uint8_t rescap; // Reservation Capabilities
194 uint8_t fpi; // Format Progress Indicator
195 uint8_t dlfeat; // Deallocate Logical Block Features
196 uint16_t nawun; // Namespace Atomic Write Unit Normal
197 uint16_t nawupf; // Namespace Atomic Write Unit Power Fail
198 uint16_t nacwu; // Namespace Atomic Compare & Write Unit
199 uint16_t nabsn; // Namespace Atomic Boundary Size Normal
200 uint16_t nabo; // Namespace Atomic Boundary Offset
201 uint16_t nabspf; // Namespace Atomic Boundary Size Power Fail
202 uint16_t noiob; // Namespace Optimal IO Boundary
203 uint64_t nvmcap[2]; // Namespace NVM Capacity
204 uint16_t npwg;
205 uint16_t npwa;
206 uint16_t npdg;
207 uint16_t npda;
208 uint16_t nows;
209 uint16_t mssrl;
210 uint32_t mcl;
211 uint8_t msrc;
212 uint8_t reserved0[11];
213 uint32_t adagrpid;
214 uint8_t reserved1[3];
215 uint8_t nsattr;
216 uint16_t nvmsetid;
217 uint16_t endgid;
218 uint64_t nguid[2];
219 uint64_t eui64;
220 struct nvme_lbaf lbaf[64]; // LBA format descriptions
221 uint8_t vendor_specific[3712];
222} cc_packed;
223struct nvme_identify_controller {
224 uint16_t vid; // PCI Vendor ID
225 uint16_t ssvid; // Subsystem Vendor ID
226 char sn[20]; // Serial Number (ASCII)
227 char mn[40]; // Model Number (ASCII)
228 char fr[8]; // Firmware Revision (ASCII)
229 uint8_t rab; // Recommended Arbitration Burst
230 uint8_t ieee[3]; // IEEE OUI Identifier
231 uint8_t mic; // Multi-interface Capabilities
232 uint8_t mdts; // Maximum Data Transfer Size
233 uint16_t cntlid; // Controller ID
234 uint32_t ver; // Version
235 uint32_t rtd3r; // RTD3 Resume Latency
236 uint32_t rtd3e; // RTD3 Entry Latency
237 uint32_t oaes; // Optional Admin Command Support
238 uint32_t ctratt; // Controller Attributes
239 uint8_t rsvd96[156];
240 uint16_t oacs; // Optional Admin Command Support (bit flags)
241 uint8_t acl; // Abort Command Limit
242 uint8_t aerl; // Asynchronous Event Request Limit
243 uint8_t frmw; // Firmware Updates
244 uint8_t lpa; // Log Page Attributes
245 uint8_t elpe; // Error Log Page Entries
246 uint8_t npss; // Number of Power States Support
247 uint8_t avscc; // Admin Vendor Specific Command Configuration
248 uint8_t apsta; // Autonomous Power State Transition Attributes
249 uint16_t wctemp; // Warning Composite Temperature Threshold
250 uint16_t cctemp; // Critical Composite Temperature Threshold
251 uint16_t mtfa; // Maximum Time for Firmware Activation
252 uint32_t hmpre; // Host Memory Buffer Preferred Size
253 uint32_t hmmin; // Host Memory Buffer Minimum Size
254 uint64_t tnvmcap[2]; // Total NVM Capacity
255 uint64_t unvmcap[2]; // Unallocated NVM Capacity
256 uint32_t rpmbs; // Replay Protected Memory Block Support
257 uint16_t edstt; // Extended Device Self-test Time
258 uint8_t dsto; // Device Self-test Options
259 uint8_t fwug; // Firmware Update Granularity
260 uint16_t kas; // Keep Alive Support
261 uint16_t hctma; // Host Controlled Thermal Management Attributes
262 uint16_t mntmt; // Minimum Thermal Management Temperature
263 uint16_t mxtmt; // Maximum Thermal Management Temperature
264 uint32_t sanicap; // Sanitize Capabilities
265 uint8_t rsvd228[180];
266 uint8_t sqes; // Submission Queue Entry Size
267 uint8_t cqes; // Completion Queue Entry Size
268 // TODO: there is more but me lazy and dont need it
269} cc_packed;
270
271uint16_t nvme_submit_admin_cmd(struct nvme_device *nvme,
272 struct nvme_command *cmd, uint32_t *);
273uint32_t nvme_set_num_queues(struct nvme_device *nvme, uint16_t desired_sq,
274 uint16_t desired_cq);
275
276bool nvme_submit_io_cmd(struct nvme_device *nvme, struct nvme_command *cmd,
277 uint32_t qid, struct nvme_request *req);
278
279uint8_t *nvme_identify_controller(struct nvme_device *nvme);
280uint8_t *nvme_identify_namespace(struct nvme_device *nvme, uint32_t nsid);
281void nvme_enable_controller(struct nvme_device *nvme);
282void nvme_setup_admin_queues(struct nvme_device *nvme);
283void nvme_alloc_admin_queues(struct nvme_device *nvme);
284void nvme_alloc_io_queues(struct nvme_device *nvme, uint32_t qid);
285struct nvme_device *nvme_discover_device(uint8_t bus, uint8_t slot,
286 uint8_t func);
287struct block_device *nvme_create_generic(struct nvme_device *nvme);
288void nvme_print_identify(const struct nvme_identify_controller *ctrl);
289void nvme_print_namespace(const struct nvme_identify_namespace *ns);
290
291bool nvme_read_sector_wrapper(struct block_device *disk, uint64_t lba,
292 uint8_t *buf, uint64_t cnt);
293
294bool nvme_read_sector_async_wrapper(struct block_device *disk,
295 struct nvme_request *req);
296
297bool nvme_write_sector_async_wrapper(struct block_device *disk,
298 struct nvme_request *req);
299
300bool nvme_send_nvme_req(struct block_device *d, struct nvme_request *r);
301
302bool nvme_write_sector_wrapper(struct block_device *disk, uint64_t lba,
303 const uint8_t *buf, uint64_t cnt);
304
305enum irq_result nvme_isr_handler(struct irq_context *rsp, void *ctx);
306
307bool nvme_submit_bio_request(struct block_device *disk,
308 struct bio_request *bio);
309
310bool nvme_should_coalesce(struct block_device *disk,
311 const struct bio_request *a,
312 const struct bio_request *b);
313
314void nvme_do_coalesce(struct block_device *disk, struct bio_request *into,
315 struct bio_request *from);
316
317void nvme_reorder(struct block_device *disk);
318void nvme_work(void *rvoid, void *dvoid);
319
320void nvme_print_wrapper(struct block_device *d);
321