1#include <asm.h>
2#include <console/printf.h>
3#include <drivers/mmio.h>
4#include <drivers/usb/usb.h>
5#include <drivers/usb/xhci.h>
6#include <sch/sched.h>
7#include <string.h>
8#include <thread/thread.h>
9
10#define XHCI_PORTSC_CHANGE_MASK \
11 (PORTSC_CSC | PORTSC_PEC | PORTSC_WRC | PORTSC_OCC | PORTSC_PRC | \
12 PORTSC_PLC | PORTSC_CEC)
13
14/* Bits in PORTSC that trigger an action when written as 1 */
15#define XHCI_PORTSC_ACTION_MASK \
16 (PORTSC_PED | PORTSC_PR | PORTSC_WPR | PORTSC_LWS)
17
18void xhci_nop(struct xhci_device *dev);
19void xhci_request_move(struct xhci_device *dev, struct xhci_request *req,
20 enum xhci_request_list new_list);
21void xhci_reset_slot(struct usb_device *dev);
22enum usb_error xhci_port_init(struct xhci_port *p);
23enum irq_result xhci_isr(struct irq_context *ic, void *ctx);
24struct xhci_return xhci_wait_for_port_status_change(struct xhci_device *dev,
25 uint32_t port_id);
26void xhci_device_start_interrupts(uint8_t bus, uint8_t slot, uint8_t func,
27 struct xhci_device *dev);
28void xhci_emit_singular(struct xhci_command *cmd, struct xhci_ring *ring);
29enum usb_error xhci_address_device(struct xhci_port *p, uint8_t slot_id,
30 struct xhci_slot *publish_to);
31void xhci_teardown_slot(struct xhci_slot *me);
32void xhci_wake_waiter(struct xhci_device *dev, struct xhci_request *request);
33void xhci_cleanup(struct xhci_device *dev, struct xhci_request *req);
34struct xhci_ring *cc_nullable xhci_allocate_ring(void);
35struct xhci_ring *cc_nullable xhci_allocate_event_ring(void);
36void xhci_free_ring(struct xhci_ring *ring);
37void cc_mem_io *xhci_map_mmio(uint8_t bus, uint8_t slot, uint8_t func);
38struct xhci_device *xhci_device_create(void cc_mem_io *mmio);
39bool xhci_controller_stop(struct xhci_device *dev);
40bool xhci_controller_reset(struct xhci_device *dev);
41bool xhci_controller_start(struct xhci_device *dev);
42void xhci_controller_enable_ints(struct xhci_device *dev);
43void xhci_setup_event_ring(struct xhci_device *dev);
44void xhci_setup_command_ring(struct xhci_device *dev);
45
46enum usb_error xhci_submit_interrupt_transfer(struct usb_request *r);
47enum usb_error xhci_control_transfer(struct usb_request *request);
48
49bool xhci_send_command(struct xhci_device *dev, struct xhci_command *cmd);
50
51/* returns CONTROL */
52struct xhci_return xhci_wait_for_response(struct xhci_device *dev);
53
54/* returns STATUS */
55struct xhci_return xhci_wait_for_transfer_event(struct xhci_device *dev,
56 uint8_t slot_id);
57uint8_t xhci_enable_slot(struct xhci_device *dev);
58void xhci_disable_slot(struct xhci_device *dev, uint8_t slot_id);
59void xhci_parse_ext_caps(struct xhci_device *dev);
60enum usb_error xhci_reset_port(struct xhci_device *dev, uint32_t port_index);
61void xhci_detect_usb3_ports(struct xhci_device *dev);
62
63static inline void xhci_clear_interrupt_pending(struct xhci_device *dev) {
64 uint32_t iman = mmio_read_32(address: &dev->intr_regs->iman);
65 iman |= XHCI_IMAN_INT_PENDING;
66 mmio_write_32(address: &dev->intr_regs->iman, value: iman);
67}
68
69static inline void xhci_interrupt_enable_ints(struct xhci_device *dev) {
70 uint32_t iman = mmio_read_32(address: &dev->intr_regs->iman);
71 mmio_write_32(address: &dev->intr_regs->iman, value: iman | (XHCI_IMAN_INT_ENABLE));
72}
73
74static inline void xhci_interrupt_disable_ints(struct xhci_device *dev) {
75 uint32_t iman = mmio_read_32(address: &dev->intr_regs->iman);
76 mmio_write_32(address: &dev->intr_regs->iman, value: iman & (~XHCI_IMAN_INT_ENABLE));
77}
78
79static inline void xhci_erdp_ack(struct xhci_device *dev, uint64_t erdp) {
80 mmio_write_64(address: &dev->intr_regs->erdp,
81 value: erdp | XHCI_ERDP_EHB_BIT | dev->event_ring->cycle);
82}
83
84static inline uint8_t usb_to_xhci_ep_type(bool in, uint8_t type) {
85 if (in) {
86 switch (type) {
87 case USB_ENDPOINT_ATTR_TRANS_TYPE_BULK:
88 return XHCI_ENDPOINT_TYPE_BULK_IN;
89
90 case USB_ENDPOINT_ATTR_TRANS_TYPE_CONTROL:
91 return XHCI_ENDPOINT_TYPE_CONTROL_BI;
92
93 case USB_ENDPOINT_ATTR_TRANS_TYPE_INTERRUPT:
94 return XHCI_ENDPOINT_TYPE_INTERRUPT_IN;
95
96 case USB_ENDPOINT_ATTR_TRANS_TYPE_ISOCHRONOUS:
97 return XHCI_ENDPOINT_TYPE_ISOCH_IN;
98
99 default:
100 xhci_log(LOG_ERROR, "Invalid type detected: %u", type);
101 return 0;
102 }
103 }
104 switch (type) {
105 case USB_ENDPOINT_ATTR_TRANS_TYPE_BULK: return XHCI_ENDPOINT_TYPE_BULK_OUT;
106
107 case USB_ENDPOINT_ATTR_TRANS_TYPE_CONTROL:
108 return XHCI_ENDPOINT_TYPE_CONTROL_BI;
109
110 case USB_ENDPOINT_ATTR_TRANS_TYPE_INTERRUPT:
111 return XHCI_ENDPOINT_TYPE_INTERRUPT_OUT;
112
113 case USB_ENDPOINT_ATTR_TRANS_TYPE_ISOCHRONOUS:
114 return XHCI_ENDPOINT_TYPE_ISOCH_OUT;
115
116 default: xhci_log(LOG_ERROR, "Invalid type detected: %u", type); return 0;
117 }
118}
119
120static inline void xhci_ring_doorbell(struct xhci_device *dev, uint32_t slot_id,
121 uint32_t ep_id) {
122 uint32_t cc_mem_io *doorbell =
123 (void cc_mem_io *) dev->cap_regs + dev->cap_regs->dboff;
124 mmio_write_32(address: &doorbell[slot_id], value: ep_id);
125}
126
127static inline void xhci_controller_restart(struct xhci_device *dev) {
128 xhci_controller_stop(dev);
129 xhci_controller_start(dev);
130}
131
132static inline enum usb_error xhci_rq_to_usb_status(struct xhci_request *req) {
133 /* These two meta-statuses are returned first if they are detected */
134 if (req->status == XHCI_REQUEST_DISCONNECT)
135 return USB_ERR_DISCONNECT;
136
137 if (req->status == XHCI_REQUEST_CANCELLED)
138 return USB_ERR_CANCELLED;
139
140 switch (req->completion_code) {
141 case CC_SUCCESS: return USB_OK;
142 case CC_STALL_ERROR: return USB_ERR_STALL;
143 case CC_SHORT_PACKET: return USB_OK;
144 case CC_USB_TRANSACTION_ERROR: return USB_ERR_CRC;
145 case CC_BABBLE_DETECTED: return USB_ERR_OVERFLOW;
146 case CC_RING_OVERRUN:
147 case CC_RING_UNDERRUN: return USB_ERR_IO;
148 case CC_CONTEXT_STATE_ERROR: return USB_ERR_PROTO;
149 case CC_STOPPED: return USB_ERR_CANCELLED;
150 case CC_NO_SLOTS_AVAILABLE: return USB_ERR_NO_DEVICE;
151 default: return USB_ERR_IO;
152 }
153}
154
155static inline void
156xhci_request_init_blocking(struct xhci_request *req, struct xhci_command *cmd,
157 uint8_t port, enum xhci_request_command_type t) {
158 req->status = XHCI_REQUEST_SENDING;
159 req->type = t;
160 req->completion_code = 0;
161 req->command = cmd;
162 req->private = NULL;
163 req->callback = xhci_wake_waiter;
164 req->list_owner = XHCI_REQ_LIST_NONE;
165 req->port = port;
166 list_node_init(n: &req->list);
167}
168
169static inline void xhci_request_init(struct xhci_request *req,
170 struct xhci_command *cmd,
171 struct usb_request *rq,
172 enum xhci_request_command_type t) {
173 req->list_owner = XHCI_REQ_LIST_NONE;
174 req->status = XHCI_REQUEST_SENDING;
175 req->completion_code = 0;
176 req->type = t;
177 req->command = cmd;
178 list_node_init(n: &req->list);
179 req->urb = rq;
180 req->private = NULL;
181 req->callback = xhci_cleanup;
182 req->port = rq ? rq->dev->port : 0;
183}
184
185static inline void xhci_clear_usbsts_ei(struct xhci_device *dev) {
186 mmio_write_32(address: &dev->op_regs->usbsts,
187 value: mmio_read_32(address: &dev->op_regs->usbsts) | XHCI_USBSTS_EI);
188}
189
190static inline bool xhci_send_command_and_block(struct xhci_device *dev,
191 struct xhci_command *cmd) {
192 event_init(e: &cmd->request->event, type: EVENT_NOTIFICATION, false);
193
194 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
195 bool sent = xhci_send_command(dev, cmd);
196 irql_lower(old_level: irql);
197
198 if (!sent)
199 return false;
200
201 event_wait(e: &cmd->request->event, type: THREAD_WAIT_UNINTERRUPTIBLE,
202 reason: THREAD_BLOCK_REASON_IO);
203 return true;
204}
205
206static inline uint64_t xhci_get_trb_phys(struct xhci_ring *ring,
207 struct xhci_trb *trb) {
208 uint64_t offset = (uint8_t *) trb - (uint8_t *) ring->trbs;
209 return ring->phys + offset;
210}
211
212static inline uint32_t cc_mem_io *xhci_portsc_ptr(struct xhci_device *dev,
213 uint8_t port) {
214 return &dev->port_regs[port - 1].portsc;
215}
216
217static inline uint32_t xhci_read_portsc(struct xhci_device *dev, uint8_t port) {
218 return mmio_read_32(address: &dev->port_regs[port - 1]);
219}
220
221static inline struct xhci_slot *xhci_get_slot(struct xhci_device *dev,
222 uint8_t id) {
223 return &dev->slots[id - 1];
224}
225
226static inline enum xhci_slot_state xhci_slot_get_state(struct xhci_slot *slot) {
227 return atomic_load_acq(&slot->state);
228}
229
230static inline void xhci_slot_set_state(struct xhci_slot *slot,
231 enum xhci_slot_state new) {
232 atomic_store_release(&slot->state, new);
233}
234
235/* A request is OK if it is CC_SUCCESS and PROCESSED */
236static inline bool xhci_request_ok(struct xhci_request *rq) {
237 return rq->completion_code == CC_SUCCESS && rq->status == XHCI_REQUEST_OK;
238}
239
240REFCOUNT_GENERATE_GETREF_FOR_STRUCT_WITH_FAILURE_COND(
241 xhci_slot, refcount, state, >= XHCI_SLOT_STATE_DISCONNECTING);
242
243static inline void xhci_slot_putref(struct xhci_slot *slot) {
244 if (refcount_dec_and_test(rc: &slot->refcount)) {
245 xhci_teardown_slot(me: slot);
246 }
247}
248
249static inline bool xhci_ring_can_reserve(struct xhci_ring *ring, size_t ntrbs) {
250 return ring->outgoing + ntrbs < ring->size;
251}
252
253static inline void xhci_ring_reserve(struct xhci_ring *ring, size_t ntrbs) {
254 ring->outgoing += ntrbs;
255}
256
257static inline void xhci_ring_unreserve(struct xhci_ring *ring, size_t ntrbs) {
258 ring->outgoing -= ntrbs;
259}
260
261static inline void xhci_advance_dequeue(struct xhci_ring *ring) {
262 ring->dequeue_index++;
263 if (ring->dequeue_index == ring->size) {
264 ring->dequeue_index = 0;
265 ring->cycle ^= 1;
266 }
267}
268
269static inline void xhci_advance_enqueue(struct xhci_ring *ring) {
270 ring->enqueue_index++;
271
272 if (ring->enqueue_index == ring->size - 1) {
273 ring->trbs[ring->size - 1].control &= ~TRB_CYCLE_BIT;
274 ring->trbs[ring->size - 1].control |= ring->cycle;
275 ring->enqueue_index = 0;
276 ring->cycle ^= 1;
277 }
278}
279
280static inline struct xhci_trb *xhci_ring_next_trb(struct xhci_ring *ring) {
281 struct xhci_trb *trb = &ring->trbs[ring->enqueue_index];
282
283 memset(trb, 0, sizeof(*trb));
284 trb->control |= ring->cycle;
285
286 xhci_advance_enqueue(ring);
287 return trb;
288}
289
290static inline struct xhci_slot *xhci_usb_slot(struct usb_device *dev) {
291 return dev->slot;
292}
293
294static inline void xhci_port_set_state(struct xhci_port *port,
295 enum xhci_port_state state) {
296 port->state = state;
297 port->generation++;
298}
299