struct nvme_command {
uint8_t opc;
uint8_t fuse;
uint16_t cid;
uint32_t nsid;
uint64_t rsvd2;
uint64_t mptr;
uint64_t prp1;
uint64_t prp2;
uint32_t cdw10;
uint32_t cdw11;
uint32_t cdw12;
uint32_t cdw13;
uint32_t cdw14;
uint32_t cdw15;
};
struct nvme_completion {
uint32_t result;
uint32_t rsvd;
uint16_t sq_head;
uint16_t sq_id;
uint16_t cid;
uint16_t status;
};
struct nvme_cc {
union {
uint32_t raw;
struct {
uint32_t en : 1;
uint32_t __reserved0 : 3;
uint32_t css : 3;
uint32_t mps : 4;
uint32_t ams : 3;
uint32_t shn : 2;
uint32_t iosqes : 4;
uint32_t iocqes : 4;
uint32_t __reserved1 : 8;
};
};
};
struct nvme_regs {
uint32_t cap_lo;
uint32_t cap_hi;
uint32_t version;
uint32_t intms;
uint32_t intmc;
struct nvme_cc cc;
uint32_t nssr;
uint32_t csts;
uint32_t reserved1;
uint32_t aqa;
uint32_t asq_lo;
uint32_t asq_hi;
uint32_t acq_lo;
uint32_t acq_hi;
uint32_t reserved4[1018];
};
struct nvme_bio_data {
uint64_t *prps;
uint64_t prp_count;
};
struct nvme_request {
uint32_t qid;
uint64_t lba;
void *buffer;
uint64_t size;
uint64_t sector_count;
bool write;
volatile bool done;
volatile uint16_t status;
int32_t remaining_parts;
void (*on_complete)(struct nvme_request *);
struct nvme_bio_data *bio_data;
struct thread *waiter;
void *user_data;
struct list_head list_node;
};
struct nvme_waiting_requests {
struct spinlock lock;
struct list_head list;
};
struct nvme_queue {
struct nvme_command *sq;
struct nvme_completion *cq;
uint64_t sq_phys;
uint64_t cq_phys;
uint16_t sq_tail;
uint16_t cq_head;
uint16_t sq_depth;
uint16_t cq_depth;
uint8_t cq_phase;
uint32_t *sq_db;
uint32_t *cq_db;
struct nvme_request **sq_requests;
_Atomic uint16_t outstanding;
struct spinlock lock;
};
struct nvme_device {
struct nvme_regs *regs;
uint64_t cap;
uint32_t version;
uint32_t doorbell_stride;
uint32_t page_size;
uint32_t *admin_sq_db;
uint32_t *admin_cq_db;
struct nvme_command *admin_sq;
struct nvme_completion *admin_cq;
uint64_t admin_sq_phys;
uint64_t admin_cq_phys;
uint16_t admin_sq_tail;
uint16_t admin_cq_head;
uint16_t admin_q_depth;
uint8_t admin_cq_phase;
struct nvme_queue **io_queues;
struct nvme_waiting_requests waiting_requests;
struct nvme_waiting_requests finished_requests;
struct work work;
atomic_bool on_sem;
struct semaphore sem;
uint8_t *isr_index;
uint32_t queue_count;
uint32_t sector_size;
uint64_t max_transfer_size;
struct block_device *generic_disk;
_Atomic uint64_t total_outstanding;
struct workqueue *workqueue;
};
struct nvme_identify {
uint8_t data[PAGE_SIZE];
};
struct nvme_lbaf {
uint16_t ms;
uint8_t lbads;
uint8_t rp : 2;
uint8_t reserved : 6;
};
struct nvme_identify_namespace {
uint64_t nsze;
uint64_t ncap;
uint64_t nuse;
uint8_t nsfeat;
uint8_t nlbaf;
uint8_t flbas;
uint8_t mc;
uint8_t dpc;
uint8_t dps;
uint8_t nmic;
uint8_t rescap;
uint8_t fpi;
uint8_t dlfeat;
uint16_t nawun;
uint16_t nawupf;
uint16_t nacwu;
uint16_t nabsn;
uint16_t nabo;
uint16_t nabspf;
uint16_t noiob;
uint64_t nvmcap[2];
uint16_t npwg;
uint16_t npwa;
uint16_t npdg;
uint16_t npda;
uint16_t nows;
uint16_t mssrl;
uint32_t mcl;
uint8_t msrc;
uint8_t reserved0[11];
uint32_t adagrpid;
uint8_t reserved1[3];
uint8_t nsattr;
uint16_t nvmsetid;
uint16_t endgid;
uint64_t nguid[2];
uint64_t eui64;
struct nvme_lbaf lbaf[64];
uint8_t vendor_specific[3712];
};
struct nvme_identify_controller {
uint16_t vid;
uint16_t ssvid;
char sn[20];
char mn[40];
char fr[8];
uint8_t rab;
uint8_t ieee[3];
uint8_t mic;
uint8_t mdts;
uint16_t cntlid;
uint32_t ver;
uint32_t rtd3r;
uint32_t rtd3e;
uint32_t oaes;
uint32_t ctratt;
uint8_t rsvd96[156];
uint16_t oacs;
uint8_t acl;
uint8_t aerl;
uint8_t frmw;
uint8_t lpa;
uint8_t elpe;
uint8_t npss;
uint8_t avscc;
uint8_t apsta;
uint16_t wctemp;
uint16_t cctemp;
uint16_t mtfa;
uint32_t hmpre;
uint32_t hmmin;
uint64_t tnvmcap[2];
uint64_t unvmcap[2];
uint32_t rpmbs;
uint16_t edstt;
uint8_t dsto;
uint8_t fwug;
uint16_t kas;
uint16_t hctma;
uint16_t mntmt;
uint16_t mxtmt;
uint32_t sanicap;
uint8_t rsvd228[180];
uint8_t sqes;
uint8_t cqes;
};
uint16_t nvme_submit_admin_cmd(struct nvme_device *nvme, struct nvme_command *cmd, uint32_t * *);
uint32_t nvme_set_num_queues(struct nvme_device *nvme, uint16_t desired_sq, uint16_t desired_cq);
void nvme_submit_io_cmd(struct nvme_device *nvme, struct nvme_command *cmd, uint32_t qid, struct nvme_request *req);
uint8_t * nvme_identify_controller(struct nvme_device *nvme);
uint8_t * nvme_identify_namespace(struct nvme_device *nvme, uint32_t nsid);
void nvme_enable_controller(struct nvme_device *nvme);
void nvme_setup_admin_queues(struct nvme_device *nvme);
void nvme_alloc_admin_queues(struct nvme_device *nvme);
void nvme_alloc_io_queues(struct nvme_device *nvme, uint32_t qid);
struct nvme_device * nvme_discover_device(uint8_t bus, uint8_t slot, uint8_t func);
struct block_device * nvme_create_generic(struct nvme_device *nvme);
void nvme_print_identify(const struct nvme_identify_controller *ctrl);
void nvme_print_namespace(const struct nvme_identify_namespace *ns);
bool nvme_read_sector_wrapper(struct block_device *disk, uint64_t lba, uint8_t *buf, uint64_t cnt);
bool nvme_read_sector_async_wrapper(struct block_device *disk, struct nvme_request *req);
bool nvme_write_sector_async_wrapper(struct block_device *disk, struct nvme_request *req);
bool nvme_send_nvme_req(struct block_device *d, struct nvme_request *r);
bool nvme_write_sector_wrapper(struct block_device *disk, uint64_t lba, const uint8_t *buf, uint64_t cnt);
enum irq_result nvme_isr_handler(void *ctx, uint8_t vector, struct irq_context *rsp);
bool nvme_submit_bio_request(struct block_device *disk, struct bio_request *bio);
bool nvme_should_coalesce(struct block_device *disk, const struct bio_request *a, const struct bio_request *b);
void nvme_do_coalesce(struct block_device *disk, struct bio_request *into, struct bio_request *from);
void nvme_reorder(struct block_device *disk);
void nvme_work(void *rvoid, void *dvoid);