1#include <asm.h>
2#include <console/printf.h>
3#include <drivers/usb/usb.h>
4#include <drivers/usb/xhci.h>
5#include <sch/sched.h>
6#include <string.h>
7#include <thread/io_wait.h>
8#include <thread/thread.h>
9
10#define XHCI_PORTSC_CHANGE_MASK \
11 (PORTSC_CSC | PORTSC_PEC | PORTSC_WRC | PORTSC_OCC | PORTSC_PRC | \
12 PORTSC_PLC | PORTSC_CEC)
13
14/* Bits in PORTSC that trigger an action when written as 1 */
15#define XHCI_PORTSC_ACTION_MASK \
16 (PORTSC_PED | PORTSC_PR | PORTSC_WPR | PORTSC_LWS)
17
18void xhci_nop(struct xhci_device *dev);
19void xhci_request_move(struct xhci_device *dev, struct xhci_request *req,
20 enum xhci_request_list new_list);
21void xhci_reset_slot(struct usb_device *dev);
22enum usb_error xhci_port_init(struct xhci_port *p);
23enum irq_result xhci_isr(void *ctx, uint8_t vector, struct irq_context *rsp);
24struct xhci_return xhci_wait_for_port_status_change(struct xhci_device *dev,
25 uint32_t port_id);
26void xhci_device_start_interrupts(uint8_t bus, uint8_t slot, uint8_t func,
27 struct xhci_device *dev);
28void xhci_emit_singular(struct xhci_command *cmd, struct xhci_ring *ring);
29enum usb_error xhci_address_device(struct xhci_port *p, uint8_t slot_id,
30 struct xhci_slot *publish_to);
31void xhci_teardown_slot(struct xhci_slot *me);
32void xhci_wake_waiter(struct xhci_device *dev, struct xhci_request *request);
33void xhci_cleanup(struct xhci_device *dev, struct xhci_request *req);
34struct xhci_ring *xhci_allocate_ring();
35struct xhci_ring *xhci_allocate_event_ring(void);
36void xhci_free_ring(struct xhci_ring *ring);
37void *xhci_map_mmio(uint8_t bus, uint8_t slot, uint8_t func);
38struct xhci_device *xhci_device_create(void *mmio);
39bool xhci_controller_stop(struct xhci_device *dev);
40bool xhci_controller_reset(struct xhci_device *dev);
41bool xhci_controller_start(struct xhci_device *dev);
42void xhci_controller_enable_ints(struct xhci_device *dev);
43void xhci_setup_event_ring(struct xhci_device *dev);
44void xhci_setup_command_ring(struct xhci_device *dev);
45
46enum usb_error xhci_submit_interrupt_transfer(struct usb_request *r);
47enum usb_error xhci_control_transfer(struct usb_request *request);
48
49bool xhci_send_command(struct xhci_device *dev, struct xhci_command *cmd);
50
51/* returns CONTROL */
52struct xhci_return xhci_wait_for_response(struct xhci_device *dev);
53
54/* returns STATUS */
55struct xhci_return xhci_wait_for_transfer_event(struct xhci_device *dev,
56 uint8_t slot_id);
57uint8_t xhci_enable_slot(struct xhci_device *dev);
58void xhci_disable_slot(struct xhci_device *dev, uint8_t slot_id);
59void xhci_parse_ext_caps(struct xhci_device *dev);
60enum usb_error xhci_reset_port(struct xhci_device *dev, uint32_t port_index);
61void xhci_detect_usb3_ports(struct xhci_device *dev);
62
63static inline void xhci_clear_interrupt_pending(struct xhci_device *dev) {
64 uint32_t iman = mmio_read_32(address: &dev->intr_regs->iman);
65 iman |= XHCI_IMAN_INT_PENDING;
66 mmio_write_32(address: &dev->intr_regs->iman, value: iman);
67}
68
69static inline void xhci_interrupt_enable_ints(struct xhci_device *dev) {
70 uint32_t iman = mmio_read_32(address: &dev->intr_regs->iman);
71 mmio_write_32(address: &dev->intr_regs->iman, value: iman | (XHCI_IMAN_INT_ENABLE));
72}
73
74static inline void xhci_interrupt_disable_ints(struct xhci_device *dev) {
75 uint32_t iman = mmio_read_32(address: &dev->intr_regs->iman);
76 mmio_write_32(address: &dev->intr_regs->iman, value: iman & (~XHCI_IMAN_INT_ENABLE));
77}
78
79static inline void xhci_erdp_ack(struct xhci_device *dev, uint64_t erdp) {
80 mmio_write_64(address: &dev->intr_regs->erdp,
81 value: erdp | XHCI_ERDP_EHB_BIT | dev->event_ring->cycle);
82}
83
84static inline uint8_t usb_to_xhci_ep_type(bool in, uint8_t type) {
85 if (in) {
86 switch (type) {
87 case USB_ENDPOINT_ATTR_TRANS_TYPE_BULK:
88 return XHCI_ENDPOINT_TYPE_BULK_IN;
89
90 case USB_ENDPOINT_ATTR_TRANS_TYPE_CONTROL:
91 return XHCI_ENDPOINT_TYPE_CONTROL_BI;
92
93 case USB_ENDPOINT_ATTR_TRANS_TYPE_INTERRUPT:
94 return XHCI_ENDPOINT_TYPE_INTERRUPT_IN;
95
96 case USB_ENDPOINT_ATTR_TRANS_TYPE_ISOCHRONOUS:
97 return XHCI_ENDPOINT_TYPE_ISOCH_IN;
98
99 default:
100 xhci_log(LOG_ERROR, "Invalid type detected: %u", type);
101 return 0;
102 }
103 }
104 switch (type) {
105 case USB_ENDPOINT_ATTR_TRANS_TYPE_BULK: return XHCI_ENDPOINT_TYPE_BULK_OUT;
106
107 case USB_ENDPOINT_ATTR_TRANS_TYPE_CONTROL:
108 return XHCI_ENDPOINT_TYPE_CONTROL_BI;
109
110 case USB_ENDPOINT_ATTR_TRANS_TYPE_INTERRUPT:
111 return XHCI_ENDPOINT_TYPE_INTERRUPT_OUT;
112
113 case USB_ENDPOINT_ATTR_TRANS_TYPE_ISOCHRONOUS:
114 return XHCI_ENDPOINT_TYPE_ISOCH_OUT;
115
116 default: xhci_log(LOG_ERROR, "Invalid type detected: %u", type); return 0;
117 }
118}
119
120static inline void xhci_ring_doorbell(struct xhci_device *dev, uint32_t slot_id,
121 uint32_t ep_id) {
122 uint32_t *doorbell = (void *) dev->cap_regs + dev->cap_regs->dboff;
123 mmio_write_32(address: &doorbell[slot_id], value: ep_id);
124}
125
126static inline void xhci_controller_restart(struct xhci_device *dev) {
127 xhci_controller_stop(dev);
128 xhci_controller_start(dev);
129}
130
131static inline enum usb_error xhci_rq_to_usb_status(struct xhci_request *req) {
132 /* These two meta-statuses are returned first if they are detected */
133 if (req->status == XHCI_REQUEST_DISCONNECT)
134 return USB_ERR_DISCONNECT;
135
136 if (req->status == XHCI_REQUEST_CANCELLED)
137 return USB_ERR_CANCELLED;
138
139 switch (req->completion_code) {
140 case CC_SUCCESS: return USB_OK;
141 case CC_STALL_ERROR: return USB_ERR_STALL;
142 case CC_SHORT_PACKET: return USB_OK;
143 case CC_USB_TRANSACTION_ERROR: return USB_ERR_CRC;
144 case CC_BABBLE_DETECTED: return USB_ERR_OVERFLOW;
145 case CC_RING_OVERRUN:
146 case CC_RING_UNDERRUN: return USB_ERR_IO;
147 case CC_CONTEXT_STATE_ERROR: return USB_ERR_PROTO;
148 case CC_STOPPED: return USB_ERR_CANCELLED;
149 case CC_NO_SLOTS_AVAILABLE: return USB_ERR_NO_DEVICE;
150 default: return USB_ERR_IO;
151 }
152}
153
154static inline void
155xhci_request_init_blocking(struct xhci_request *req, struct xhci_command *cmd,
156 uint8_t port, enum xhci_request_command_type t) {
157 req->status = XHCI_REQUEST_SENDING;
158 req->type = t;
159 req->completion_code = 0;
160 req->command = cmd;
161 req->private = thread_get_current();
162 req->callback = xhci_wake_waiter;
163 req->list_owner = XHCI_REQ_LIST_NONE;
164 req->port = port;
165 INIT_LIST_HEAD(list: &req->list);
166}
167
168static inline void xhci_request_init(struct xhci_request *req,
169 struct xhci_command *cmd,
170 struct usb_request *rq,
171 enum xhci_request_command_type t) {
172 req->list_owner = XHCI_REQ_LIST_NONE;
173 req->status = XHCI_REQUEST_SENDING;
174 req->completion_code = 0;
175 req->type = t;
176 req->command = cmd;
177 INIT_LIST_HEAD(list: &req->list);
178 req->urb = rq;
179 req->private = NULL;
180 req->callback = xhci_cleanup;
181 req->port = rq->dev->port;
182}
183
184static inline void xhci_clear_usbsts_ei(struct xhci_device *dev) {
185 mmio_write_32(address: &dev->op_regs->usbsts,
186 value: mmio_read_32(address: &dev->op_regs->usbsts) | XHCI_USBSTS_EI);
187}
188
189static inline bool xhci_send_command_and_block(struct xhci_device *dev,
190 struct xhci_command *cmd,
191 struct io_wait_token *iot) {
192 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
193
194 struct io_wait_token tok;
195
196 if (iot) {
197 io_wait_begin(out: iot, io_object: dev);
198 } else {
199 io_wait_begin(out: &tok, io_object: dev);
200 }
201
202 if (!xhci_send_command(dev, cmd)) {
203 thread_wake_unlocked(t: thread_get_current(),
204 r: THREAD_WAKE_REASON_BLOCKING_MANUAL, wake_src: dev);
205
206 if (iot) {
207 io_wait_end(t: iot, act: IO_WAIT_END_NO_OP);
208 } else {
209 io_wait_end(t: &tok, act: IO_WAIT_END_NO_OP);
210 }
211
212 irql_lower(old_level: irql);
213 return false;
214 }
215
216 irql_lower(old_level: irql);
217
218 thread_yield_until_wake_match();
219
220 if (!iot)
221 io_wait_end(t: &tok, act: IO_WAIT_END_YIELD);
222
223 return true;
224}
225
226static inline uint64_t xhci_get_trb_phys(struct xhci_ring *ring,
227 struct xhci_trb *trb) {
228 uint64_t offset = (uint8_t *) trb - (uint8_t *) ring->trbs;
229 return ring->phys + offset;
230}
231
232static inline uint32_t *xhci_portsc_ptr(struct xhci_device *dev, uint8_t port) {
233 return &dev->port_regs[port - 1].portsc;
234}
235
236static inline uint32_t xhci_read_portsc(struct xhci_device *dev, uint8_t port) {
237 return mmio_read_32(address: &dev->port_regs[port - 1]);
238}
239
240static inline struct xhci_slot *xhci_get_slot(struct xhci_device *dev,
241 uint8_t id) {
242 return &dev->slots[id - 1];
243}
244
245static inline enum xhci_slot_state xhci_slot_get_state(struct xhci_slot *slot) {
246 return atomic_load_explicit(&slot->state, memory_order_acquire);
247}
248
249static inline void xhci_slot_set_state(struct xhci_slot *slot,
250 enum xhci_slot_state new) {
251 atomic_store_explicit(&slot->state, new, memory_order_release);
252}
253
254/* A request is OK if it is CC_SUCCESS and PROCESSED */
255static inline bool xhci_request_ok(struct xhci_request *rq) {
256 return rq->completion_code == CC_SUCCESS && rq->status == XHCI_REQUEST_OK;
257}
258
259REFCOUNT_GENERATE_GET_FOR_STRUCT_WITH_FAILURE_COND(
260 xhci_slot, refcount, state, >= XHCI_SLOT_STATE_DISCONNECTING);
261
262static inline void xhci_slot_put(struct xhci_slot *slot) {
263 if (refcount_dec_and_test(rc: &slot->refcount)) {
264 xhci_teardown_slot(me: slot);
265 }
266}
267
268static inline bool xhci_ring_can_reserve(struct xhci_ring *ring, size_t ntrbs) {
269 return ring->outgoing + ntrbs < ring->size;
270}
271
272static inline void xhci_ring_reserve(struct xhci_ring *ring, size_t ntrbs) {
273 ring->outgoing += ntrbs;
274}
275
276static inline void xhci_ring_unreserve(struct xhci_ring *ring, size_t ntrbs) {
277 ring->outgoing -= ntrbs;
278}
279
280static inline void xhci_advance_dequeue(struct xhci_ring *ring) {
281 ring->dequeue_index++;
282 if (ring->dequeue_index == ring->size) {
283 ring->dequeue_index = 0;
284 ring->cycle ^= 1;
285 }
286}
287
288static inline void xhci_advance_enqueue(struct xhci_ring *ring) {
289 ring->enqueue_index++;
290
291 if (ring->enqueue_index == ring->size - 1) {
292 ring->trbs[ring->size - 1].control &= ~TRB_CYCLE_BIT;
293 ring->trbs[ring->size - 1].control |= ring->cycle;
294 ring->enqueue_index = 0;
295 ring->cycle ^= 1;
296 }
297}
298
299static inline struct xhci_trb *xhci_ring_next_trb(struct xhci_ring *ring) {
300 struct xhci_trb *trb = &ring->trbs[ring->enqueue_index];
301
302 memset(trb, 0, sizeof(*trb));
303 trb->control |= ring->cycle;
304
305 xhci_advance_enqueue(ring);
306 return trb;
307}
308
309static inline struct xhci_slot *xhci_usb_slot(struct usb_device *dev) {
310 return dev->slot;
311}
312
313static inline void xhci_port_set_state(struct xhci_port *port,
314 enum xhci_port_state state) {
315 port->state = state;
316 port->generation++;
317}
318