1#include <asm.h>
2#include <compiler/core.h>
3#include <console/printf.h>
4#include <drivers/pci.h>
5#include <drivers/usb/xhci.h>
6#include <irq/idt.h>
7#include <math/bit.h>
8#include <math/min_max.h>
9#include <mem/alloc.h>
10#include <mem/nofail.h>
11#include <mem/page.h>
12#include <mem/vmm.h>
13#include <stdbool.h>
14#include <stdint.h>
15#include <string.h>
16#include <thread/workqueue.h>
17
18#include "internal.h"
19
20struct workqueue *xhci_wq;
21LOG_HANDLE_DEFINE_PRINT(xhci);
22
23#ifdef DEBUG_USB_XHCI
24LOG_SITE_DEFINE_PRINT(xhci);
25#else
26LOG_SITE_DEFINE_PRINT(xhci, .enabled_mask = LOG_SITE_LEVEL(LOG_ERROR));
27#endif
28
29enum usb_error xhci_address_device(struct xhci_port *p, uint8_t slot_id,
30 struct xhci_slot *publish_to) {
31 struct xhci_device *xhci = p->dev;
32 uint8_t speed = p->speed;
33 uint8_t port = p->port_id;
34
35 struct xhci_input_ctx *input_ctx =
36 valloc_aligned(PAGE_SIZE, PAGE_SIZE, ALLOC_ZERO);
37 if (!input_ctx)
38 return USB_ERR_OOM;
39
40 uintptr_t input_ctx_phys = vmm_get_phys((uintptr_t) input_ctx);
41
42 struct xhci_ring *ring = xhci_allocate_ring();
43 if (!ring) {
44 vfree_aligned(input_ctx);
45 return USB_ERR_OOM;
46 }
47
48 struct xhci_device_ctx *dev_ctx =
49 valloc_aligned(PAGE_SIZE, PAGE_SIZE, ALLOC_ZERO);
50 if (!dev_ctx) {
51 xhci_free_ring(ring);
52 vfree_aligned(input_ctx);
53 return USB_ERR_OOM;
54 }
55
56 uintptr_t dev_ctx_phys = vmm_get_phys((uintptr_t) dev_ctx);
57
58 enum irql irql = spin_lock_high(&xhci->lock);
59 input_ctx->ctrl_ctx.add_flags = XHCI_INPUT_CTX_ADD_FLAGS;
60 input_ctx->ctrl_ctx.drop_flags = 0;
61
62 struct xhci_slot_ctx *slot = &input_ctx->slot_ctx;
63 slot->route_string = 0;
64 slot->speed = speed;
65 slot->context_entries = 1;
66 slot->root_hub_port = port;
67 slot->mtt = 0;
68 slot->hub = 0;
69 slot->num_ports = 0;
70
71 publish_to->ep_rings[0] = ring;
72
73 struct xhci_ep_ctx *ep0 = &input_ctx->ep_ctx[0];
74 ep0->ep_type = XHCI_ENDPOINT_TYPE_CONTROL_BI;
75 ep0->max_packet_size =
76 (speed == PORT_SPEED_LOW || speed == PORT_SPEED_FULL) ? 8 : 64;
77 ep0->max_burst_size = 0;
78 ep0->interval = 0;
79 ep0->dequeue_ptr_raw = ring->phys | TRB_CYCLE_BIT;
80
81 xhci->dcbaa->ptrs[slot_id] = dev_ctx_phys;
82
83 uint32_t control = 0;
84 control |= TRB_SET_TYPE(TRB_TYPE_ADDRESS_DEVICE);
85 control |= TRB_SET_SLOT_ID(slot_id);
86
87 struct xhci_request request = {0};
88 struct xhci_command cmd = {0};
89 xhci_request_init_blocking(req: &request, cmd: &cmd, port,
90 t: XHCI_CMD_TYPE_ADDRESS_DEVICE);
91
92 struct xhci_trb outgoing = {
93 .control = control,
94 .parameter = input_ctx_phys,
95 .status = 0,
96 };
97
98 cmd = (struct xhci_command){
99 .ring = xhci->cmd_ring,
100 .private = &outgoing,
101 .emit = xhci_emit_singular,
102 .num_trbs = 1,
103 .ep_id = 0,
104 .slot = NULL,
105 .request = &request,
106 };
107
108 spin_unlock(&xhci->lock, irql);
109 bool ret = xhci_send_command_and_block(dev: xhci, cmd: &cmd);
110
111 vfree_aligned(input_ctx);
112
113 if (!ret || !xhci_request_ok(rq: &request)) {
114 enum usb_error err =
115 !ret ? USB_ERR_NO_DEVICE : xhci_rq_to_usb_status(req: &request);
116 vfree_aligned(dev_ctx);
117 xhci_free_ring(ring);
118 publish_to->ep_rings[0] = NULL;
119 return err;
120 }
121
122 return USB_OK;
123}
124
125static uint8_t xhci_ep_to_input_ctx_idx(struct usb_endpoint *ep) {
126 return ep->number * 2 - (ep->in ? 0 : 1);
127}
128
129/* free the rings `copy` holds that `orig` doesn't */
130static void xhci_free_new_rings(struct xhci_slot *orig,
131 struct xhci_slot *copy) {
132 for (size_t i = 0; i < ct_array_size(copy->ep_rings); i++)
133 if (copy->ep_rings[i] && copy->ep_rings[i] != orig->ep_rings[i])
134 xhci_free_ring(ring: copy->ep_rings[i]);
135}
136
137static enum usb_error xhci_configure_device_endpoints(struct usb_device *usb) {
138 struct xhci_slot *xslot = usb->slot;
139 if (!xhci_slot_getref(obj: xslot))
140 return USB_ERR_NO_DEVICE;
141
142 enum usb_error err = USB_OK;
143 struct xhci_device *xhci = usb->driver_private;
144 struct xhci_input_ctx *input_ctx =
145 valloc_aligned(PAGE_SIZE, PAGE_SIZE, ALLOC_ZERO);
146 if (!input_ctx) {
147 xhci_slot_putref(slot: xslot);
148 return USB_ERR_OOM;
149 }
150
151 uintptr_t input_ctx_phys = vmm_get_phys((uintptr_t) input_ctx);
152
153 input_ctx->ctrl_ctx.add_flags = 1;
154 uint8_t max_ep_index = 0;
155
156 struct xhci_slot local_copy = *xslot;
157
158 for (size_t i = 0; i < usb->num_endpoints; i++) {
159 struct usb_endpoint *ep = usb->endpoints[i];
160 /* endpoint 0 is the default control pipe */
161 if (ep->number == 0) {
162 err = USB_ERR_INVALID_ARGUMENT;
163 goto err_free;
164 }
165
166 uint8_t ep_index = get_ep_index(ep);
167
168 uint8_t input_ctx_idx = xhci_ep_to_input_ctx_idx(ep);
169
170 max_ep_index = MAX(input_ctx_idx, max_ep_index);
171
172 /* Add one, there is a slot that the add flags account for */
173 input_ctx->ctrl_ctx.add_flags |= BIT(input_ctx_idx + 1);
174
175 struct xhci_ep_ctx *ep_ctx =
176 &input_ctx->ep_ctx[input_ctx_idx]; // NOLINT(*ArrayBound)
177
178 ep_ctx->ep_type = usb_to_xhci_ep_type(in: ep->in, type: ep->type);
179
180 ep_ctx->max_packet_size = ep->max_packet_size;
181 ep_ctx->interval = ep->interval;
182
183 ep_ctx->max_burst_size = 0;
184
185 struct xhci_ring *ring = xhci_allocate_ring();
186 if (!ring) {
187 err = USB_ERR_OOM;
188 goto err_free;
189 }
190
191 ep_ctx->dequeue_ptr_raw = ring->phys | TRB_CYCLE_BIT;
192 ep_ctx->ep_state = 1;
193
194 local_copy.ep_rings[ep_index] = ring;
195 }
196
197 input_ctx->slot_ctx.context_entries = max_ep_index;
198
199 uint32_t control = TRB_SET_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT);
200 control |= TRB_SET_SLOT_ID(xslot->slot_id);
201
202 struct xhci_request request = {0};
203 struct xhci_command cmd = {0};
204 xhci_request_init_blocking(req: &request, cmd: &cmd, /* port = */ 0,
205 t: XHCI_CMD_TYPE_CONFIGURE_ENDPOINT);
206
207 struct xhci_trb outgoing = {
208 .parameter = input_ctx_phys,
209 .control = control,
210 .status = 0,
211 };
212
213 cmd = (struct xhci_command){
214 .slot = NULL,
215 .ep_id = 0,
216 .private = &outgoing,
217 .emit = xhci_emit_singular,
218 .ring = xhci->cmd_ring,
219 .request = &request,
220 .num_trbs = 1,
221 };
222
223 bool ret = xhci_send_command_and_block(dev: xhci, cmd: &cmd);
224
225 vfree_aligned(input_ctx);
226 if (!ret)
227 return USB_ERR_NO_DEVICE;
228
229 enum irql irql = spin_lock_high(&xhci->lock);
230
231 *xslot = local_copy;
232
233 spin_unlock(&xhci->lock, irql);
234 xhci_slot_putref(slot: xslot);
235
236 if (!xhci_request_ok(rq: &request)) {
237 xhci_warn("Failed to configure endpoints for slot %u",
238 local_copy.slot_id);
239 return xhci_rq_to_usb_status(req: &request);
240 }
241
242 return USB_OK;
243
244err_free:
245 xhci_free_new_rings(orig: xslot, copy: &local_copy);
246 vfree_aligned(input_ctx);
247 xhci_slot_putref(slot: xslot);
248 return err;
249}
250
251static void xhci_work_port_disconnect(void *arg1) {
252 struct xhci_device *d = arg1;
253 while (true) {
254 xhci_trace("port_disconnect work down");
255 semaphore_wait(s: &d->port_disconnect);
256 xhci_trace("port_disconnect work up");
257
258 /* Do repeated scans until we don't find a port */
259 bool keep_going = true;
260 while (keep_going) {
261 struct xhci_port *port = NULL;
262 struct xhci_slot *slot = NULL;
263 struct usb_device *dev = NULL;
264
265 enum irql irql = spin_lock_high(&d->lock);
266 for (size_t i = 0; i < XHCI_PORT_COUNT; i++) {
267 port = &d->port_info[i];
268 if (port->state == XHCI_PORT_STATE_DISCONNECTING) {
269 slot = port->slot;
270
271 if (slot)
272 dev = slot->udev;
273
274 port->slot = NULL;
275 xhci_port_set_state(port, state: XHCI_PORT_STATE_DISCONNECTED);
276 break;
277 }
278 }
279
280 if (slot) {
281 list_node_delete_init(n: &dev->hc_list);
282 d->num_devices--;
283
284 spin_unlock(&d->lock, irql);
285
286 spin_lock_raw(&port->update_lock);
287
288 usb_teardown_device(dev);
289 xhci_slot_putref(slot);
290 spin_unlock_raw(&port->update_lock);
291 } else {
292 spin_unlock(&d->lock, irql);
293 keep_going = false;
294 }
295 }
296 }
297}
298
299static void xhci_work_port_connect(void *arg1) {
300 struct xhci_device *d = arg1;
301
302 while (true) {
303 xhci_trace("port_connect work down");
304 semaphore_wait(s: &d->port_connect);
305 xhci_trace("port_connect work up");
306
307 bool keep_going = true;
308 while (keep_going) {
309 struct xhci_port *port = NULL;
310
311 enum irql irql = spin_lock_high(&d->lock);
312 for (size_t i = 0; i < XHCI_PORT_COUNT; i++) {
313 if (d->port_info[i].state == XHCI_PORT_STATE_CONNECTING) {
314 port = &d->port_info[i];
315 break;
316 }
317 }
318 spin_unlock(&d->lock, irql);
319
320 /* Done */
321 if (!port) {
322 keep_going = false;
323 break;
324 }
325
326 spin_lock_raw(&port->update_lock);
327
328 enum usb_error err;
329
330 /* Device can vanish in between connect and now, if it is gone,
331 * port_init never ran, no slot to tear down... */
332 if (!(xhci_read_portsc(dev: d, port: port->port_id) & PORTSC_CCS))
333 err = USB_ERR_NO_DEVICE;
334 else
335 err = xhci_port_init(p: port);
336
337 if (err == USB_OK) {
338 struct usb_device *dev = port->slot->udev;
339 spin_unlock_raw(&port->update_lock);
340 usb_init_device(dev);
341 continue;
342 }
343
344 /* Init failed, or device disappeared. retire the port if
345 * connecting, a disconnect could've moved it to DISCONNECTING,
346 * which would need the disconnect worker to handle it */
347 spin_unlock_raw(&port->update_lock);
348
349 irql = spin_lock_high(&d->lock);
350
351 if (port->state == XHCI_PORT_STATE_CONNECTING)
352 xhci_port_set_state(port, state: XHCI_PORT_STATE_DISCONNECTED);
353
354 spin_unlock(&d->lock, irql);
355 }
356 }
357}
358
359static struct xhci_request *
360xhci_finished_requests_pop_front(struct xhci_device *dev) {
361 enum irql irql = spin_lock_high(&dev->lock);
362
363 struct xhci_request *ret = NULL;
364 struct list_node *lh =
365 list_head_pop_front_init(h: &dev->requests[XHCI_REQ_LIST_PROCESSED]);
366 if (!lh)
367 goto out;
368
369 ret = container_of(lh, struct xhci_request, list);
370
371out:
372 spin_unlock(&dev->lock, irql);
373 return ret;
374}
375
376static struct xhci_request *
377xhci_waiting_requests_pop_front(struct xhci_device *dev) {
378 enum irql irql = spin_lock_high(&dev->lock);
379 struct xhci_request *ret = NULL;
380 struct list_node *lh =
381 list_head_pop_front_init(h: &dev->requests[XHCI_REQ_LIST_WAITING]);
382
383 if (!lh)
384 goto out;
385
386 ret = container_of(lh, struct xhci_request, list);
387
388out:
389 spin_unlock(&dev->lock, irql);
390 return ret;
391}
392
393static void xhci_process_single(struct xhci_device *dev,
394 struct xhci_request *request) {
395 struct xhci_slot *rslot = request->command->slot;
396
397 enforce(request->callback);
398 request->callback(dev, request);
399
400 if (rslot)
401 xhci_slot_putref(slot: rslot);
402}
403
404static void xhci_worker_submit_waiting(struct xhci_device *dev) {
405 struct xhci_request *req;
406 while ((req = xhci_waiting_requests_pop_front(dev))) {
407 /* Command failed to send, we are responsible
408 * for now processing it */
409 if (!xhci_send_command(dev, cmd: req->command))
410 xhci_process_single(dev, request: req);
411 }
412}
413
414static void xhci_worker(void *arg) {
415 struct xhci_device *dev = arg;
416 struct xhci_request *req;
417 while (true) {
418 while ((req = xhci_finished_requests_pop_front(dev)))
419 xhci_process_single(dev, request: req);
420
421 xhci_worker_submit_waiting(dev);
422 atomic_store(&dev->worker_waiting, true);
423 semaphore_wait(s: &dev->sem);
424 }
425}
426
427/*
428 *
429 * whenever a port becomes disconnected, the following steps happen:
430 *
431 * 1. in the ISR, move all requests for that port's slot into the
432 * finished requests list, and go through and mark all of the requests
433 * as having the status of DISCONNECTED
434 *
435 * 2. mark the slot as being DISCONNECTING (no one can grab refs, but the port
436 * is still there and nothing has been freed yet)
437 *
438 * 3. wake worker
439 * then the worker goes through and does the normal biz of handling all the
440 * requests. the requests will all get the DISCONNECTED status forwarded.
441 *
442 * -------------------------------------------------------------------------
443 * at this point, there is still the possibility of an outgoing request being
444 * sent. because this is the case, the ISR will ALWAYS do a final pass of the
445 * request list (after processing event TRBs) to find any of these "oddball
446 * requests" that are going to a slot that no longer exists
447 * -------------------------------------------------------------------------
448 *
449 * 4. in the worker thread, it will then check all slots (under the lock of the
450 * xhci device) to see if any slots have become DISCONNECTING. for each slot
451 * that is DISCONNECTING, it will first call into the usb_device to tell it
452 * to clean up (no more requests being sent out. usb_device itself will
453 * handle how this works internally. probably just dropping the initial ref
454 * is fine), and then drop the initial ref of the slot.
455 *
456 * 5. later on, as the xhci device generates interrupts, the list will be
457 * checked in the ISR for any requests that might have a slot that doesnt
458 * exist.
459 *
460 */
461
462static struct xhci_request *xhci_lookup_trb(struct xhci_device *dev,
463 struct xhci_trb *trb) {
464 enforce(TRB_TYPE(trb->control) == TRB_TYPE_TRANSFER_EVENT ||
465 TRB_TYPE(trb->control) == TRB_TYPE_COMMAND_COMPLETION);
466
467 paddr_t phys = trb->parameter;
468
469 struct xhci_request *req, *tmp, *found = NULL;
470 list_for_each_entry_safe(req, tmp, &dev->requests[XHCI_REQ_LIST_OUTGOING],
471 list) {
472 if (req->trb_phys == phys) {
473 found = req;
474 break;
475 }
476 }
477
478 return found;
479}
480
481void xhci_request_move(struct xhci_device *dev, struct xhci_request *req,
482 enum xhci_request_list new_list) {
483 enforce(req->list_owner != new_list);
484 /* remove from old list */
485 if (req->list_owner != XHCI_REQ_LIST_NONE) {
486 list_node_delete_init(n: &req->list);
487
488 if (req->list_owner == XHCI_REQ_LIST_OUTGOING) {
489 req->command->ring->outgoing -= req->command->num_trbs;
490 }
491 }
492
493 req->list_owner = new_list;
494
495 if (new_list == XHCI_REQ_LIST_NONE)
496 return;
497
498 list_head_push_back(h: &dev->requests[new_list], n: &req->list);
499}
500
501static void xhci_disconnect_requests_on_port(struct xhci_device *dev,
502 uint8_t port) {
503 for (int i = XHCI_REQ_LIST_OUTGOING; i <= XHCI_REQ_LIST_WAITING; i++) {
504 struct xhci_request *req, *tmp;
505 list_for_each_entry_safe(req, tmp, &dev->requests[i], list) {
506 if (req->port != port)
507 continue;
508
509 /* An outgoing request has its TRB queued on the ring
510 * which always posts a completion event
511 *
512 * Completing it here would orphan that
513 * completion (xhci_lookup_trb would later miss it) */
514 if (i == XHCI_REQ_LIST_OUTGOING &&
515 req->command->ring == dev->cmd_ring)
516 continue;
517
518 req->status = XHCI_REQUEST_DISCONNECT;
519 xhci_request_move(dev, req, new_list: XHCI_REQ_LIST_PROCESSED);
520 }
521 }
522}
523
524static void catch_stragglers_on_list(struct xhci_device *dev,
525 enum xhci_request_list l) {
526 struct xhci_request *req, *tmp;
527 list_for_each_entry_safe(req, tmp, &dev->requests[l], list) {
528 struct xhci_slot *slot = req->command->slot;
529 if (!slot)
530 continue;
531
532 enum xhci_slot_state state = xhci_slot_get_state(slot);
533 bool slot_here = state == XHCI_SLOT_STATE_ENABLED &&
534 slot->port->generation == req->generation;
535
536 if (!slot_here) {
537 req->status = XHCI_REQUEST_DISCONNECT;
538 xhci_request_move(dev, req, new_list: XHCI_REQ_LIST_PROCESSED);
539 }
540 }
541}
542
543/* sends DISCONNECTED to any requests that are not on a valid port */
544static void xhci_catch_stragglers(struct xhci_device *dev) {
545 catch_stragglers_on_list(dev, l: XHCI_REQ_LIST_OUTGOING);
546 catch_stragglers_on_list(dev, l: XHCI_REQ_LIST_WAITING);
547}
548
549static enum xhci_request_status
550xhci_make_request_status(struct xhci_device *dev,
551 struct xhci_request *request) {
552 if (request->port) {
553 struct xhci_port *port = &dev->port_info[request->port - 1];
554
555 if (request->generation && request->generation != port->generation) {
556 xhci_warn("gen mismatch");
557 return XHCI_REQUEST_DISCONNECT;
558 }
559 }
560
561 switch (request->completion_code) {
562 case CC_STOPPED:
563 case CC_STOPPED_SHORT_PACKET: return XHCI_REQUEST_CANCELLED;
564
565 case CC_SUCCESS:
566 case CC_STOPPED_LEN_INVALID:
567 case CC_SHORT_PACKET: return XHCI_REQUEST_OK;
568
569 case CC_STALL_ERROR:
570 case CC_ENDPOINT_NOT_ENABLED:
571 case CC_TRB_ERROR: return XHCI_REQUEST_ERR;
572
573 case CC_RING_OVERRUN:
574 case CC_RING_UNDERRUN:
575 xhci_warn("Ring overrun/underrun detected");
576 return XHCI_REQUEST_OK;
577
578 case CC_CONTEXT_STATE_ERROR:
579 case CC_NO_SLOTS_AVAILABLE:
580 xhci_warn("Context state error or no slots available");
581 return XHCI_REQUEST_ERR;
582
583 case CC_BABBLE_DETECTED:
584 case CC_USB_TRANSACTION_ERROR:
585 xhci_warn("Babble detected/transaction error");
586 return XHCI_REQUEST_ERR;
587 }
588
589 xhci_warn("Other error");
590 return XHCI_REQUEST_ERR;
591}
592
593static void xhci_process_trb_into_request(struct xhci_device *dev,
594 struct xhci_request *request,
595 struct xhci_trb *trb) {
596 request->completion_code = TRB_CC(trb->status);
597 request->return_status = trb->status;
598 request->return_control = trb->control;
599 request->return_parameter = trb->parameter;
600 request->status = xhci_make_request_status(dev, request);
601}
602
603static bool xhci_trb_slot_exists(struct xhci_device *dev, struct xhci_trb *trb,
604 struct xhci_request *request) {
605 struct xhci_trb *source = request->last_trb;
606 uint8_t type = TRB_TYPE(source->control);
607 if (type == TRB_TYPE_ENABLE_SLOT || type == TRB_TYPE_NO_OP ||
608 type == TRB_TYPE_ADDRESS_DEVICE || type == TRB_TYPE_DISABLE_SLOT ||
609 type == TRB_TYPE_RESET_DEVICE)
610 return true;
611
612 return xhci_slot_get_state(slot: xhci_get_slot(dev, TRB_SLOT(trb->control))) ==
613 XHCI_SLOT_STATE_ENABLED;
614}
615
616static void xhci_process_request(struct xhci_device *dev,
617 struct xhci_trb *trb) {
618 struct xhci_request *found = xhci_lookup_trb(dev, trb);
619
620 if (!found) {
621 xhci_warn("Completion TRB with no matching request, dropping");
622 return;
623 }
624
625 xhci_process_trb_into_request(dev, request: found, trb);
626
627 if (!xhci_trb_slot_exists(dev, trb, request: found))
628 found->status = XHCI_REQUEST_DISCONNECT;
629
630 xhci_request_move(dev, req: found, new_list: XHCI_REQ_LIST_PROCESSED);
631}
632
633static void xhci_ack_port_changes(uint32_t cc_mem_io *portsc_ptr,
634 uint32_t observed) {
635 uint32_t change = observed & XHCI_PORTSC_CHANGE_MASK;
636 if (!change)
637 return;
638
639 uint32_t ack =
640 (observed & ~(XHCI_PORTSC_ACTION_MASK | XHCI_PORTSC_CHANGE_MASK)) |
641 change;
642 mmio_write_32(address: portsc_ptr, value: ack);
643}
644
645static void xhci_start_port_connect(struct xhci_device *dev,
646 struct xhci_port *port) {
647 xhci_port_set_state(port, state: XHCI_PORT_STATE_CONNECTING);
648 semaphore_post(s: &dev->port_connect);
649}
650
651static void xhci_start_port_disconnect(struct xhci_device *dev,
652 struct xhci_port *port) {
653 xhci_warn("dsc");
654 xhci_port_set_state(port, state: XHCI_PORT_STATE_DISCONNECTING);
655
656 if (port->slot) {
657 struct xhci_slot *s = xhci_get_slot(dev, id: port->slot->slot_id);
658 xhci_slot_set_state(slot: s, new: XHCI_SLOT_STATE_DISCONNECTING);
659 xhci_disconnect_requests_on_port(dev, port: port->port_id);
660 }
661
662 semaphore_post(s: &dev->port_disconnect);
663}
664
665static void xhci_reconcile_port(struct xhci_device *dev,
666 struct xhci_port *port) {
667 bool present = xhci_read_portsc(dev, port: port->port_id) & PORTSC_CCS;
668 bool tracked = port->slot != NULL;
669 enum xhci_port_state st = port->state;
670
671 if (present && !tracked && st != XHCI_PORT_STATE_CONNECTING)
672 xhci_start_port_connect(dev, port);
673 else if (!present && tracked && st != XHCI_PORT_STATE_DISCONNECTING)
674 xhci_start_port_disconnect(dev, port);
675}
676
677static void xhci_scan_ports_locked(struct xhci_device *dev) {
678 for (size_t i = 0; i < dev->ports; i++) {
679 struct xhci_port *port = &dev->port_info[i];
680 uint32_t cc_mem_io *portsc_ptr = xhci_portsc_ptr(dev, port: port->port_id);
681 xhci_ack_port_changes(portsc_ptr, observed: mmio_read_32(address: portsc_ptr));
682 xhci_reconcile_port(dev, port);
683 }
684}
685
686static void xhci_process_port_status_change(struct xhci_device *dev,
687 struct xhci_trb *evt) {
688 cc_ignore(evt);
689 /* Scan all ports */
690 xhci_scan_ports_locked(dev);
691}
692
693static cc_unused void xhci_scan_ports(struct xhci_device *dev) {
694 enum irql irql = spin_lock_high(&dev->lock);
695 xhci_scan_ports_locked(dev);
696 spin_unlock(&dev->lock, irql);
697}
698
699static void xhci_process_event(struct xhci_device *dev, struct xhci_trb *trb) {
700 uint32_t trb_type = TRB_TYPE(trb->control);
701 switch (trb_type) {
702 case TRB_TYPE_PORT_STATUS_CHANGE:
703 xhci_process_port_status_change(dev, evt: trb);
704 break;
705 case TRB_TYPE_TRANSFER_EVENT:
706 case TRB_TYPE_COMMAND_COMPLETION: xhci_process_request(dev, trb); break;
707 default: xhci_warn("Unknown TRB type %u", trb_type);
708 }
709}
710
711static void xhci_process_event_ring(struct xhci_device *xhci) {
712 struct xhci_ring *ring = xhci->event_ring;
713
714 enum irql irql = spin_lock_high(&xhci->lock);
715
716 while (true) {
717 struct xhci_trb *evt = &ring->trbs[ring->dequeue_index];
718 uint32_t control = mmio_read_32(address: (void cc_mem_io *) &evt->control);
719
720 if ((control & TRB_CYCLE_BIT) != ring->cycle)
721 break;
722
723 xhci_process_event(dev: xhci, trb: evt);
724
725 xhci_advance_dequeue(ring);
726
727 uint64_t erdp =
728 ring->phys + ring->dequeue_index * sizeof(struct xhci_trb);
729 xhci_erdp_ack(dev: xhci, erdp);
730 }
731
732 xhci_catch_stragglers(dev: xhci);
733
734 spin_unlock(&xhci->lock, irql);
735}
736
737enum irq_result xhci_isr(struct irq_context *rsp, void *ctx) {
738 cc_ignore(rsp);
739 xhci_trace("Interrupt caught");
740 struct xhci_device *dev = ctx;
741
742 xhci_process_event_ring(xhci: dev);
743
744 xhci_clear_interrupt_pending(dev);
745 xhci_clear_usbsts_ei(dev);
746 semaphore_post(s: &dev->sem);
747
748 return IRQ_HANDLED;
749}
750
751static struct usb_controller_ops xhci_ctrl_ops = {
752 .submit_control_transfer = xhci_control_transfer,
753 .submit_bulk_transfer = NULL,
754 .submit_interrupt_transfer = xhci_submit_interrupt_transfer,
755 .reset_slot = xhci_reset_slot,
756 .configure_endpoint = xhci_configure_device_endpoints,
757};
758
759void xhci_init(uint8_t bus, uint8_t slot, uint8_t func,
760 struct pci_device *pci) {
761 struct cpu_mask cmask;
762 cpu_mask_init(m: &cmask, nbits: global.core_count);
763
764 cpu_mask_set_all(&cmask);
765 struct workqueue_attributes attrs = {
766 .idle_check = WORKQUEUE_DEFAULT_IDLE_CHECK,
767 .min_workers = 1,
768 .max_workers = 1,
769 .spawn_delay = WORKQUEUE_DEFAULT_SPAWN_DELAY,
770 .worker_cpu_mask = cmask,
771 .worker_niceness = 0,
772 .flags = WORKQUEUE_FLAG_DEFAULTS | WORKQUEUE_FLAG_ISR_SAFE,
773 };
774
775 xhci_wq = workqueue_create("xhci_wq", &attrs);
776
777 xhci_info("Found device at %02x:%02x.%02x", bus, slot, func);
778 void cc_mem_io *mmio = xhci_map_mmio(bus, slot, func);
779
780 struct xhci_device *dev = xhci_device_create(mmio);
781 xhci_info("Device at %p, offset is %zu", dev,
782 offsetof(struct xhci_request, list));
783
784 semaphore_init(s: &dev->sem, value: 0, SEMAPHORE_INIT_IRQ_DISABLE);
785 spinlock_init(&dev->lock);
786 thread_spawn("xhci_worker", xhci_worker, .arg = dev);
787
788 /* Wait till we know our worker is on the sem */
789 while (!atomic_load(&dev->worker_waiting))
790 scheduler_yield();
791
792 dev->pci = pci;
793
794 pci_enable_msix(bus, slot, func);
795 xhci_device_start_interrupts(bus, slot, func, dev);
796
797 if (!xhci_controller_stop(dev))
798 return;
799
800 if (!xhci_controller_reset(dev))
801 return;
802
803 xhci_parse_ext_caps(dev);
804 xhci_detect_usb3_ports(dev);
805 xhci_setup_event_ring(dev);
806
807 xhci_setup_command_ring(dev);
808
809 xhci_controller_start(dev);
810 xhci_controller_enable_ints(dev);
811 xhci_interrupt_enable_ints(dev);
812
813 struct usb_controller *ctrl = nofail_vnew(ctrl, ALLOC_ZERO);
814 ctrl->driver_data = dev;
815 ctrl->type = USB_CONTROLLER_XHCI;
816 ctrl->ops = &xhci_ctrl_ops;
817 dev->controller = ctrl;
818
819 enum irql irql = spin_lock_high(&dev->lock);
820 for (uint32_t port = 1; port <= dev->ports; port++) {
821 struct xhci_port *p = &dev->port_info[port - 1];
822 uint32_t cc_mem_io *portsc_ptr = xhci_portsc_ptr(dev, port);
823 uint32_t v = mmio_read_32(address: portsc_ptr);
824 xhci_ack_port_changes(portsc_ptr, observed: v);
825 if ((v & PORTSC_CCS) && p->state == XHCI_PORT_STATE_DISCONNECTED)
826 xhci_port_set_state(port: p, state: XHCI_PORT_STATE_CONNECTING);
827 }
828 spin_unlock(&dev->lock, irql);
829
830 for (uint32_t port = 1; port <= dev->ports; port++) {
831 struct xhci_port *p = &dev->port_info[port - 1];
832 if (p->state != XHCI_PORT_STATE_CONNECTING)
833 continue;
834
835 spin_lock_raw(&p->update_lock);
836 enum usb_error err = xhci_port_init(p);
837 spin_unlock_raw(&p->update_lock);
838
839 if (err == USB_OK) {
840 usb_init_device(dev: p->slot->udev);
841 continue;
842 }
843
844 irql = spin_lock_high(&dev->lock);
845 if (p->state == XHCI_PORT_STATE_CONNECTING)
846 xhci_port_set_state(port: p, state: XHCI_PORT_STATE_DISCONNECTED);
847 spin_unlock(&dev->lock, irql);
848 }
849
850 thread_spawn("xhci_disconnect_worker", xhci_work_port_disconnect,
851 .arg = dev);
852 thread_spawn("xhci_connect_worker", xhci_work_port_connect, .arg = dev);
853
854#ifdef DEBUG_USB_XHCI
855
856 /*
857 while (true)
858 cpu_halt(); */
859
860#endif
861
862 xhci_info("Device initialized successfully");
863}
864
865static enum err xhci_pci_init(struct device *d) {
866 struct pci_device *dev = d->driver_data;
867 switch (dev->prog_if) {
868 case 0x30: xhci_init(bus: dev->bus, slot: dev->dev, func: dev->function, pci: dev); break;
869 default: break;
870 }
871
872 return ERR_OK;
873}
874
875PCI_DEV_REGISTER(xhci, 0x0C, 0x03, 0x030, 0xFFFF, xhci_pci_init)
876