1#include <console/printf.h>
2#include <drivers/nvme.h>
3#include <log.h>
4#include <math/div.h>
5#include <mem/vmm.h>
6
7#define NVME_CMD_TIMEOUT_MS 2000 // Normal command timeout
8#define NVME_ADMIN_TIMEOUT_MS 5000 // Admin commands
9#define NVME_RESET_TIMEOUT_MS 30000 // Controller reset or format NVM
10
11/* _raw is fine here: the driver only uses per-cpu queues
12 * as an optimization, and we'll rewrite a lot of this anyways */
13#define THIS_QID(nvme) (1 + (smp_id_raw() % (nvme->queue_count)))
14
15LOG_SITE_EXTERN(nvme);
16LOG_HANDLE_EXTERN(nvme);
17#define nvme_log(log_level, fmt, ...) \
18 log(LOG_SITE(nvme), LOG_HANDLE(nvme), log_level, fmt, ##__VA_ARGS__)
19
20#define NVME_COMPLETION_PHASE(cpl) ((cpl)->status & 0x1)
21#define NVME_COMPLETION_STATUS(cpl) (((cpl)->status >> 1) & 0x7FFF)
22
23#define NVME_DOORBELL_BASE 0x1000
24
25#define NVME_PRPS_PER_PAGE (PAGE_SIZE / sizeof(uint64_t))
26
27#define NVME_OP_ADMIN_DELETE_IOSQ 0x0
28#define NVME_OP_ADMIN_CREATE_IOSQ 0x1
29
30#define NVME_OP_ADMIN_GET_LOG_PG 0x2
31
32#define NVME_OP_ADMIN_DELETE_IOCQ 0x4
33#define NVME_OP_ADMIN_CREATE_IOCQ 0x5
34
35#define NVME_OP_ADMIN_IDENT 0x6
36#define NVME_OP_ADMIN_SET_FEATS 0x9
37#define NVME_OP_ADMIN_GET_FEATS 0x10
38
39#define NVME_OP_IO_READ 0x02
40#define NVME_OP_IO_WRITE 0x01
41
42#define NVME_STATUS_CONFLICTING_ATTRIBUTES 0x80
43#define NVME_STATUS_INVALID_PROT_INFO 0x81
44
45/* We funnel addresses through this so we can catch bad ones */
46static inline void nvme_check_dma_addr(uint64_t phys, const char *what) {
47 if (phys == (uint64_t) -1 || phys == 0)
48 panic("NVMe: %s has no physical mapping (0x%lx)", what, phys);
49
50 if (vmm_phys_is_kernel_text(phys))
51 panic("NVMe: %s aims at kernel text (phys 0x%lx)", what, phys);
52}
53
54bool nvme_read_sector_async(struct block_device *disk,
55 struct nvme_request *req);
56
57bool nvme_write_sector_async(struct block_device *disk,
58 struct nvme_request *req);
59
60static inline enum workqueue_error nvme_work_enqueue(struct nvme_device *dev,
61 struct work *work) {
62 return workqueue_enqueue(queue: dev->workqueue, work);
63}
64