1#include <acpi/lapic.h>
2#include <asm.h>
3#include <compiler.h>
4#include <console/printf.h>
5#include <drivers/pci.h>
6#include <drivers/usb/xhci.h>
7#include <irq/idt.h>
8#include <mem/alloc.h>
9#include <mem/alloc_or_die.h>
10#include <mem/page.h>
11#include <mem/vmm.h>
12#include <stdbool.h>
13#include <stdint.h>
14#include <string.h>
15#include <thread/workqueue.h>
16
17#include "internal.h"
18
19struct workqueue *xhci_wq;
20LOG_HANDLE_DECLARE_DEFAULT(xhci);
21
22#ifdef DEBUG_USB_XHCI
23LOG_SITE_DECLARE_DEFAULT(xhci);
24#else
25LOG_SITE_DECLARE_DEFAULT(xhci, .flags = LOG_SITE_LEVEL(LOG_ERROR));
26#endif
27
28enum usb_error xhci_address_device(struct xhci_port *p, uint8_t slot_id,
29 struct xhci_slot *publish_to) {
30 struct xhci_device *xhci = p->dev;
31 uint8_t speed = p->speed;
32 uint8_t port = p->port_id;
33
34 struct xhci_input_ctx *input_ctx =
35 kmalloc_aligned(PAGE_SIZE, PAGE_SIZE, ALLOC_FLAGS_ZERO);
36 if (!input_ctx)
37 return USB_ERR_OOM;
38
39 uintptr_t input_ctx_phys =
40 vmm_get_phys((uintptr_t) input_ctx, VMM_FLAG_NONE);
41
42 struct xhci_ring *ring = xhci_allocate_ring();
43 if (!ring) {
44 kfree_aligned(input_ctx);
45 return USB_ERR_OOM;
46 }
47
48 struct xhci_device_ctx *dev_ctx =
49 kmalloc_aligned(PAGE_SIZE, PAGE_SIZE, ALLOC_FLAGS_ZERO);
50 if (!dev_ctx) {
51 xhci_free_ring(ring);
52 kfree_aligned(input_ctx);
53 return USB_ERR_OOM;
54 }
55
56 uintptr_t dev_ctx_phys = vmm_get_phys((uintptr_t) dev_ctx, VMM_FLAG_NONE);
57
58 enum irql irql = spin_lock_irq_disable(lock: &xhci->lock);
59 input_ctx->ctrl_ctx.add_flags = XHCI_INPUT_CTX_ADD_FLAGS;
60 input_ctx->ctrl_ctx.drop_flags = 0;
61
62 struct xhci_slot_ctx *slot = &input_ctx->slot_ctx;
63 slot->route_string = 0;
64 slot->speed = speed;
65 slot->context_entries = 1;
66 slot->root_hub_port = port;
67 slot->mtt = 0;
68 slot->hub = 0;
69 slot->num_ports = 0;
70
71 publish_to->ep_rings[0] = ring;
72
73 struct xhci_ep_ctx *ep0 = &input_ctx->ep_ctx[0];
74 ep0->ep_type = XHCI_ENDPOINT_TYPE_CONTROL_BI;
75 ep0->max_packet_size =
76 (speed == PORT_SPEED_LOW || speed == PORT_SPEED_FULL) ? 8 : 64;
77 ep0->max_burst_size = 0;
78 ep0->interval = 0;
79 ep0->dequeue_ptr_raw = ring->phys | TRB_CYCLE_BIT;
80
81 xhci->dcbaa->ptrs[slot_id] = dev_ctx_phys;
82
83 uint32_t control = 0;
84 control |= TRB_SET_TYPE(TRB_TYPE_ADDRESS_DEVICE);
85 control |= TRB_SET_SLOT_ID(slot_id);
86
87 struct xhci_request request = {0};
88 struct xhci_command cmd = {0};
89 xhci_request_init_blocking(req: &request, cmd: &cmd, port,
90 t: XHCI_CMD_TYPE_ADDRESS_DEVICE);
91
92 struct xhci_trb outgoing = {
93 .control = control,
94 .parameter = input_ctx_phys,
95 .status = 0,
96 };
97
98 cmd = (struct xhci_command){
99 .ring = xhci->cmd_ring,
100 .private = &outgoing,
101 .emit = xhci_emit_singular,
102 .num_trbs = 1,
103 .ep_id = 0,
104 .slot = NULL,
105 .request = &request,
106 };
107
108 spin_unlock(lock: &xhci->lock, old: irql);
109 bool ret = xhci_send_command_and_block(dev: xhci, cmd: &cmd, NULL);
110
111 kfree_aligned(input_ctx);
112
113 if (!ret || !xhci_request_ok(rq: &request)) {
114 enum usb_error err =
115 !ret ? USB_ERR_NO_DEVICE : xhci_rq_to_usb_status(req: &request);
116 kfree_aligned(dev_ctx);
117 xhci_free_ring(ring);
118 publish_to->ep_rings[0] = NULL;
119 return err;
120 }
121
122 return USB_OK;
123}
124
125static uint8_t xhci_ep_to_input_ctx_idx(struct usb_endpoint *ep) {
126 return ep->number * 2 - (ep->in ? 0 : 1);
127}
128
129enum usb_error xhci_configure_device_endpoints(struct usb_device *usb) {
130 struct xhci_slot *xslot = usb->slot;
131 if (!xhci_slot_get(obj: xslot))
132 return USB_ERR_NO_DEVICE;
133
134 struct xhci_device *xhci = usb->driver_private;
135 struct xhci_input_ctx *input_ctx =
136 kmalloc_aligned(PAGE_SIZE, PAGE_SIZE, ALLOC_FLAGS_ZERO);
137
138 uintptr_t input_ctx_phys =
139 vmm_get_phys((uintptr_t) input_ctx, VMM_FLAG_NONE);
140
141 input_ctx->ctrl_ctx.add_flags = 1;
142 uint8_t max_ep_index = 0;
143
144 struct xhci_slot local_copy = *xslot;
145
146 for (size_t i = 0; i < usb->num_endpoints; i++) {
147 struct usb_endpoint *ep = usb->endpoints[i];
148 uint8_t ep_index = get_ep_index(ep);
149
150 uint8_t input_ctx_idx = xhci_ep_to_input_ctx_idx(ep);
151
152 max_ep_index =
153 (input_ctx_idx > max_ep_index) ? input_ctx_idx : max_ep_index;
154
155 /* Add one, there is a slot that the add flags account for */
156 input_ctx->ctrl_ctx.add_flags |= (1 << (input_ctx_idx + 1));
157
158 struct xhci_ep_ctx *ep_ctx = &input_ctx->ep_ctx[input_ctx_idx];
159
160 ep_ctx->ep_type = usb_to_xhci_ep_type(in: ep->in, type: ep->type);
161
162 ep_ctx->max_packet_size = ep->max_packet_size;
163 ep_ctx->interval = ep->interval;
164
165 ep_ctx->max_burst_size = 0;
166
167 struct xhci_ring *ring = xhci_allocate_ring();
168
169 ep_ctx->dequeue_ptr_raw = ring->phys | TRB_CYCLE_BIT;
170 ep_ctx->ep_state = 1;
171
172 local_copy.ep_rings[ep_index] = ring;
173 }
174
175 input_ctx->slot_ctx.context_entries = max_ep_index;
176
177 uint32_t control = TRB_SET_TYPE(TRB_TYPE_CONFIGURE_ENDPOINT);
178 control |= TRB_SET_SLOT_ID(xslot->slot_id);
179
180 struct xhci_request request = {0};
181 struct xhci_command cmd = {0};
182 xhci_request_init_blocking(req: &request, cmd: &cmd, /* port = */ 0,
183 t: XHCI_CMD_TYPE_CONFIGURE_ENDPOINT);
184
185 struct xhci_trb outgoing = {
186 .parameter = input_ctx_phys,
187 .control = control,
188 .status = 0,
189 };
190
191 cmd = (struct xhci_command){
192 .slot = NULL,
193 .ep_id = 0,
194 .private = &outgoing,
195 .emit = xhci_emit_singular,
196 .ring = xhci->cmd_ring,
197 .request = &request,
198 .num_trbs = 1,
199 };
200
201 bool ret = xhci_send_command_and_block(dev: xhci, cmd: &cmd, NULL);
202
203 kfree_aligned(input_ctx);
204 if (!ret)
205 return USB_ERR_NO_DEVICE;
206
207 enum irql irql = spin_lock_irq_disable(lock: &xhci->lock);
208
209 *xslot = local_copy;
210
211 spin_unlock(lock: &xhci->lock, old: irql);
212 xhci_slot_put(slot: xslot);
213
214 if (!xhci_request_ok(rq: &request)) {
215 xhci_warn("Failed to configure endpoints for slot %u",
216 local_copy.slot_id);
217 return xhci_rq_to_usb_status(req: &request);
218 }
219
220 return USB_OK;
221}
222
223static void xhci_work_port_disconnect(void *arg1) {
224 struct xhci_device *d = arg1;
225 while (true) {
226 xhci_trace("port_disconnect work down");
227 semaphore_wait(s: &d->port_disconnect);
228 xhci_trace("port_disconnect work up");
229
230 /* Do repeated scans until we don't find a port */
231 bool keep_going = true;
232 while (keep_going) {
233 struct xhci_port *port = NULL;
234 struct xhci_slot *slot = NULL;
235 struct usb_device *dev = NULL;
236
237 enum irql irql = spin_lock_irq_disable(lock: &d->lock);
238 for (size_t i = 0; i < XHCI_PORT_COUNT; i++) {
239 port = &d->port_info[i];
240 if (port->state == XHCI_PORT_STATE_DISCONNECTING) {
241 slot = port->slot;
242
243 if (slot)
244 dev = slot->udev;
245
246 port->slot = NULL;
247 xhci_port_set_state(port, state: XHCI_PORT_STATE_DISCONNECTED);
248 break;
249 }
250 }
251
252 if (slot) {
253 list_del_init(entry: &dev->hc_list);
254 d->num_devices--;
255
256 spin_unlock(lock: &d->lock, old: irql);
257
258 spin_lock_raw(lock: &port->update_lock);
259
260 usb_teardown_device(dev);
261 xhci_slot_put(slot);
262 spin_unlock_raw(lock: &port->update_lock);
263 } else {
264 spin_unlock(lock: &d->lock, old: irql);
265 keep_going = false;
266 }
267 }
268 }
269}
270
271static void xhci_work_port_connect(void *arg1) {
272 struct xhci_device *d = arg1;
273
274 while (true) {
275 xhci_trace("port_connect work down");
276 semaphore_wait(s: &d->port_connect);
277 xhci_trace("port_connect work up");
278
279 /* Level triggered drain: handle each port the ISR marked CONNECTING...
280 *
281 * Cannot depend on future interrupt to rescue a port, device present
282 * now may not produce a change bit transition, so CONNECTING port
283 * always leaves CONNECTED or DISCONNECTED */
284
285 bool keep_going = true;
286 while (keep_going) {
287 struct xhci_port *port = NULL;
288
289 enum irql irql = spin_lock_irq_disable(lock: &d->lock);
290 for (size_t i = 0; i < XHCI_PORT_COUNT; i++) {
291 if (d->port_info[i].state == XHCI_PORT_STATE_CONNECTING) {
292 port = &d->port_info[i];
293 break;
294 }
295 }
296 spin_unlock(lock: &d->lock, old: irql);
297
298 /* Done */
299 if (!port) {
300 keep_going = false;
301 break;
302 }
303
304 spin_lock_raw(lock: &port->update_lock);
305
306 enum usb_error err;
307
308 /* Device can vanish in between connect and now, if it is gone,
309 * port_init never ran, no slot to tear down... */
310 if (!(xhci_read_portsc(dev: d, port: port->port_id) & PORTSC_CCS))
311 err = USB_ERR_NO_DEVICE;
312 else
313 err = xhci_port_init(p: port);
314
315 if (err == USB_OK) {
316 struct usb_device *dev = port->slot->udev;
317 spin_unlock_raw(lock: &port->update_lock);
318 usb_init_device(dev);
319 continue;
320 }
321
322 /* Init failed, or device disappeared. retire the port if
323 * connecting, a disconnect could've moved it to DISCONNECTING,
324 * which would need the disconnect worker to handle it */
325 spin_unlock_raw(lock: &port->update_lock);
326
327 irql = spin_lock_irq_disable(lock: &d->lock);
328
329 if (port->state == XHCI_PORT_STATE_CONNECTING)
330 xhci_port_set_state(port, state: XHCI_PORT_STATE_DISCONNECTED);
331
332 spin_unlock(lock: &d->lock, old: irql);
333 }
334 }
335}
336
337static struct xhci_request *
338xhci_finished_requests_pop_front(struct xhci_device *dev) {
339 enum irql irql = spin_lock_irq_disable(lock: &dev->lock);
340
341 struct xhci_request *ret = NULL;
342 struct list_head *lh =
343 list_pop_front_init(head: &dev->requests[XHCI_REQ_LIST_PROCESSED]);
344 if (!lh)
345 goto out;
346
347 ret = container_of(lh, struct xhci_request, list);
348
349out:
350 spin_unlock(lock: &dev->lock, old: irql);
351 return ret;
352}
353
354static struct xhci_request *
355xhci_waiting_requests_pop_front(struct xhci_device *dev) {
356 enum irql irql = spin_lock_irq_disable(lock: &dev->lock);
357 struct xhci_request *ret = NULL;
358 struct list_head *lh =
359 list_pop_front_init(head: &dev->requests[XHCI_REQ_LIST_WAITING]);
360
361 if (!lh)
362 goto out;
363
364 ret = container_of(lh, struct xhci_request, list);
365
366out:
367 spin_unlock(lock: &dev->lock, old: irql);
368 return ret;
369}
370
371static void xhci_process_single(struct xhci_device *dev,
372 struct xhci_request *request) {
373 struct xhci_slot *rslot = request->command->slot;
374
375 kassert(request->callback);
376 request->callback(dev, request);
377
378 if (rslot)
379 xhci_slot_put(slot: rslot);
380}
381
382static void xhci_worker_submit_waiting(struct xhci_device *dev) {
383 struct xhci_request *req;
384 while ((req = xhci_waiting_requests_pop_front(dev))) {
385 /* Command failed to send, we are responsible
386 * for now processing it */
387 if (!xhci_send_command(dev, cmd: req->command))
388 xhci_process_single(dev, request: req);
389 }
390}
391
392static void xhci_worker(void *arg) {
393 struct xhci_device *dev = arg;
394 struct xhci_request *req;
395 while (true) {
396 while ((req = xhci_finished_requests_pop_front(dev)))
397 xhci_process_single(dev, request: req);
398
399 xhci_worker_submit_waiting(dev);
400 atomic_store(&dev->worker_waiting, true);
401 semaphore_wait(s: &dev->sem);
402 }
403}
404
405/*
406 *
407 * whenever a port becomes disconnected, the following steps happen:
408 *
409 * 1. in the ISR, move all requests for that port's slot into the
410 * finished requests list, and go through and mark all of the requests
411 * as having the status of DISCONNECTED
412 *
413 * 2. mark the slot as being DISCONNECTING (no one can grab refs, but the port
414 * is still there and nothing has been freed yet)
415 *
416 * 3. wake worker
417 * then the worker goes through and does the normal biz of handling all the
418 * requests. the requests will all get the DISCONNECTED status forwarded.
419 *
420 * -------------------------------------------------------------------------
421 * at this point, there is still the possibility of an outgoing request being
422 * sent. because this is the case, the ISR will ALWAYS do a final pass of the
423 * request list (after processing event TRBs) to find any of these "oddball
424 * requests" that are going to a slot that no longer exists
425 * -------------------------------------------------------------------------
426 *
427 * 4. in the worker thread, it will then check all slots (under the lock of the
428 * xhci device) to see if any slots have become DISCONNECTING. for each slot
429 * that is DISCONNECTING, it will first call into the usb_device to tell it
430 * to clean up (no more requests being sent out. usb_device itself will
431 * handle how this works internally. probably just dropping the initial ref
432 * is fine), and then drop the initial ref of the slot.
433 *
434 * 5. later on, as the xhci device generates interrupts, the list will be
435 * checked in the ISR for any requests that might have a slot that doesnt
436 * exist.
437 *
438 */
439
440static struct xhci_request *xhci_lookup_trb(struct xhci_device *dev,
441 struct xhci_trb *trb) {
442 kassert(TRB_TYPE(trb->control) == TRB_TYPE_TRANSFER_EVENT ||
443 TRB_TYPE(trb->control) == TRB_TYPE_COMMAND_COMPLETION);
444
445 paddr_t phys = trb->parameter;
446
447 struct xhci_request *req, *tmp, *found = NULL;
448 list_for_each_entry_safe(req, tmp, &dev->requests[XHCI_REQ_LIST_OUTGOING],
449 list) {
450 if (req->trb_phys == phys) {
451 found = req;
452 break;
453 }
454 }
455
456 return found;
457}
458
459void xhci_request_move(struct xhci_device *dev, struct xhci_request *req,
460 enum xhci_request_list new_list) {
461 kassert(req->list_owner != new_list);
462 /* remove from old list */
463 if (req->list_owner != XHCI_REQ_LIST_NONE) {
464 list_del_init(entry: &req->list);
465
466 if (req->list_owner == XHCI_REQ_LIST_OUTGOING) {
467 req->command->ring->outgoing -= req->command->num_trbs;
468 }
469 }
470
471 req->list_owner = new_list;
472
473 if (new_list == XHCI_REQ_LIST_NONE)
474 return;
475
476 list_add_tail(new: &req->list, head: &dev->requests[new_list]);
477}
478
479static void xhci_disconnect_requests_on_port(struct xhci_device *dev,
480 uint8_t port) {
481 for (int i = XHCI_REQ_LIST_OUTGOING; i <= XHCI_REQ_LIST_WAITING; i++) {
482 struct xhci_request *req, *tmp;
483 list_for_each_entry_safe(req, tmp, &dev->requests[i], list) {
484 if (req->port != port)
485 continue;
486
487 /* An outgoing request has its TRB queued on the ring
488 * which always posts a completion event
489 *
490 * Completing it here would orphan that
491 * completion (xhci_lookup_trb would later miss it) */
492 if (i == XHCI_REQ_LIST_OUTGOING &&
493 req->command->ring == dev->cmd_ring)
494 continue;
495
496 req->status = XHCI_REQUEST_DISCONNECT;
497 xhci_request_move(dev, req, new_list: XHCI_REQ_LIST_PROCESSED);
498 }
499 }
500}
501
502static void catch_stragglers_on_list(struct xhci_device *dev,
503 enum xhci_request_list l) {
504 struct xhci_request *req, *tmp;
505 list_for_each_entry_safe(req, tmp, &dev->requests[l], list) {
506 struct xhci_slot *slot = req->command->slot;
507 if (!slot)
508 continue;
509
510 enum xhci_slot_state state = xhci_slot_get_state(slot);
511 bool slot_here = state == XHCI_SLOT_STATE_ENABLED &&
512 slot->port->generation == req->generation;
513
514 if (!slot_here) {
515 req->status = XHCI_REQUEST_DISCONNECT;
516 xhci_request_move(dev, req, new_list: XHCI_REQ_LIST_PROCESSED);
517 }
518 }
519}
520
521/* sends DISCONNECTED to any requests that are not on a valid port */
522static void xhci_catch_stragglers(struct xhci_device *dev) {
523 catch_stragglers_on_list(dev, l: XHCI_REQ_LIST_OUTGOING);
524 catch_stragglers_on_list(dev, l: XHCI_REQ_LIST_WAITING);
525}
526
527enum xhci_request_status
528xhci_make_request_status(struct xhci_device *dev,
529 struct xhci_request *request) {
530 if (request->port) {
531 struct xhci_port *port = &dev->port_info[request->port - 1];
532
533 if (request->generation && request->generation != port->generation) {
534 xhci_warn("gen mismatch");
535 return XHCI_REQUEST_DISCONNECT;
536 }
537 }
538
539 switch (request->completion_code) {
540 case CC_STOPPED:
541 case CC_STOPPED_SHORT_PACKET: return XHCI_REQUEST_CANCELLED;
542
543 case CC_SUCCESS:
544 case CC_STOPPED_LEN_INVALID:
545 case CC_SHORT_PACKET: return XHCI_REQUEST_OK;
546
547 case CC_STALL_ERROR:
548 case CC_ENDPOINT_NOT_ENABLED:
549 case CC_TRB_ERROR: return XHCI_REQUEST_ERR;
550
551 case CC_RING_OVERRUN:
552 case CC_RING_UNDERRUN:
553 xhci_warn("Ring overrun/underrun detected");
554 return XHCI_REQUEST_OK;
555
556 case CC_CONTEXT_STATE_ERROR:
557 case CC_NO_SLOTS_AVAILABLE:
558 xhci_warn("Context state error or no slots available");
559 return XHCI_REQUEST_ERR;
560
561 case CC_BABBLE_DETECTED:
562 case CC_USB_TRANSACTION_ERROR:
563 xhci_warn("Babble detected/transaction error");
564 return XHCI_REQUEST_ERR;
565 }
566
567 xhci_warn("Other error");
568 return XHCI_REQUEST_ERR;
569}
570
571static void xhci_process_trb_into_request(struct xhci_device *dev,
572 struct xhci_request *request,
573 struct xhci_trb *trb) {
574 request->completion_code = TRB_CC(trb->status);
575 request->return_status = trb->status;
576 request->return_control = trb->control;
577 request->return_parameter = trb->parameter;
578 request->status = xhci_make_request_status(dev, request);
579}
580
581static bool xhci_trb_slot_exists(struct xhci_device *dev, struct xhci_trb *trb,
582 struct xhci_request *request) {
583 struct xhci_trb *source = request->last_trb;
584 uint8_t type = TRB_TYPE(source->control);
585 if (type == TRB_TYPE_ENABLE_SLOT || type == TRB_TYPE_NO_OP ||
586 type == TRB_TYPE_ADDRESS_DEVICE || type == TRB_TYPE_DISABLE_SLOT ||
587 type == TRB_TYPE_RESET_DEVICE)
588 return true;
589
590 return xhci_slot_get_state(slot: xhci_get_slot(dev, TRB_SLOT(trb->control))) ==
591 XHCI_SLOT_STATE_ENABLED;
592}
593
594static void xhci_process_request(struct xhci_device *dev,
595 struct xhci_trb *trb) {
596 struct xhci_request *found = xhci_lookup_trb(dev, trb);
597
598 /* No owner left for the completion: hotplug can create this scenario:
599 * a transfer request force completed by disconnect may post stale
600 * completion for already queued TRB */
601
602 if (!found) {
603 xhci_warn("Completion TRB with no matching request, dropping");
604 return;
605 }
606
607 xhci_process_trb_into_request(dev, request: found, trb);
608
609 if (!xhci_trb_slot_exists(dev, trb, request: found))
610 found->status = XHCI_REQUEST_DISCONNECT;
611
612 xhci_request_move(dev, req: found, new_list: XHCI_REQ_LIST_PROCESSED);
613}
614
615/* ack the change bits. multiple changes can coalesce into one event. only
616 * write back a 1 for the bits we saw */
617static void xhci_ack_port_changes(uint32_t *portsc_ptr, uint32_t observed) {
618 uint32_t change = observed & XHCI_PORTSC_CHANGE_MASK;
619 if (!change)
620 return;
621
622 uint32_t ack =
623 (observed & ~(XHCI_PORTSC_ACTION_MASK | XHCI_PORTSC_CHANGE_MASK)) |
624 change;
625 mmio_write_32(address: portsc_ptr, value: ack);
626}
627
628static void xhci_start_port_connect(struct xhci_device *dev,
629 struct xhci_port *port) {
630 xhci_port_set_state(port, state: XHCI_PORT_STATE_CONNECTING);
631 semaphore_post(s: &dev->port_connect);
632}
633
634static void xhci_start_port_disconnect(struct xhci_device *dev,
635 struct xhci_port *port) {
636 xhci_warn("dsc");
637 xhci_port_set_state(port, state: XHCI_PORT_STATE_DISCONNECTING);
638
639 if (port->slot) {
640 struct xhci_slot *s = xhci_get_slot(dev, id: port->slot->slot_id);
641 xhci_slot_set_state(slot: s, new: XHCI_SLOT_STATE_DISCONNECTING);
642 xhci_disconnect_requests_on_port(dev, port: port->port_id);
643 }
644
645 semaphore_post(s: &dev->port_disconnect);
646}
647
648/* xHC only raises PSC on 0->1 bit change, edges get coalesced. driving
649 * from the CCS level makes missed edges fix themselves, as lost transitions
650 * are fixed on the next reconcile which converges the port to match HW */
651static void xhci_reconcile_port_locked(struct xhci_device *dev,
652 struct xhci_port *port) {
653 bool present = xhci_read_portsc(dev, port: port->port_id) & PORTSC_CCS;
654 bool tracked = port->slot != NULL;
655 enum xhci_port_state st = port->state;
656
657 if (present && !tracked && st != XHCI_PORT_STATE_CONNECTING)
658 xhci_start_port_connect(dev, port);
659 else if (!present && tracked && st != XHCI_PORT_STATE_DISCONNECTING)
660 xhci_start_port_disconnect(dev, port);
661}
662
663/* Sweep each port... ONE PSC is generated even when many ports change at once,
664 * never re-posting for a set change bit. A bit slower, but the safe route */
665
666static void xhci_scan_ports_locked(struct xhci_device *dev) {
667 for (size_t i = 0; i < dev->ports; i++) {
668 struct xhci_port *port = &dev->port_info[i];
669 uint32_t *portsc_ptr = xhci_portsc_ptr(dev, port: port->port_id);
670 xhci_ack_port_changes(portsc_ptr, observed: mmio_read_32(address: portsc_ptr));
671 xhci_reconcile_port_locked(dev, port);
672 }
673}
674
675static void xhci_process_port_status_change(struct xhci_device *dev,
676 struct xhci_trb *evt) {
677 (void) evt;
678 /* Scan all ports */
679 xhci_scan_ports_locked(dev);
680}
681
682static void xhci_scan_ports(struct xhci_device *dev) {
683 enum irql irql = spin_lock_irq_disable(lock: &dev->lock);
684 xhci_scan_ports_locked(dev);
685 spin_unlock(lock: &dev->lock, old: irql);
686}
687
688static void xhci_process_event(struct xhci_device *dev, struct xhci_trb *trb) {
689 uint32_t trb_type = TRB_TYPE(trb->control);
690 switch (trb_type) {
691 case TRB_TYPE_PORT_STATUS_CHANGE:
692 xhci_process_port_status_change(dev, evt: trb);
693 break;
694 case TRB_TYPE_TRANSFER_EVENT:
695 case TRB_TYPE_COMMAND_COMPLETION: xhci_process_request(dev, trb); break;
696 default: xhci_warn("Unknown TRB type %u", trb_type);
697 }
698}
699
700void xhci_process_event_ring(struct xhci_device *xhci) {
701 struct xhci_ring *ring = xhci->event_ring;
702
703 enum irql irql = spin_lock_irq_disable(lock: &xhci->lock);
704
705 while (true) {
706 struct xhci_trb *evt = &ring->trbs[ring->dequeue_index];
707 uint32_t control = mmio_read_32(address: &evt->control);
708
709 if ((control & TRB_CYCLE_BIT) != ring->cycle)
710 break;
711
712 xhci_process_event(dev: xhci, trb: evt);
713
714 xhci_advance_dequeue(ring);
715
716 uint64_t erdp =
717 ring->phys + ring->dequeue_index * sizeof(struct xhci_trb);
718 xhci_erdp_ack(dev: xhci, erdp);
719 }
720
721 xhci_catch_stragglers(dev: xhci);
722
723 spin_unlock(lock: &xhci->lock, old: irql);
724}
725
726enum irq_result xhci_isr(void *ctx, uint8_t vector, struct irq_context *rsp) {
727 (void) vector, (void) rsp;
728 xhci_trace("Interrupt caught");
729 struct xhci_device *dev = ctx;
730
731 xhci_process_event_ring(xhci: dev);
732
733 xhci_clear_interrupt_pending(dev);
734 xhci_clear_usbsts_ei(dev);
735 semaphore_post(s: &dev->sem);
736
737 return IRQ_HANDLED;
738}
739
740static struct usb_controller_ops xhci_ctrl_ops = {
741 .submit_control_transfer = xhci_control_transfer,
742 .submit_bulk_transfer = NULL,
743 .submit_interrupt_transfer = xhci_submit_interrupt_transfer,
744 .reset_slot = xhci_reset_slot,
745 .configure_endpoint = xhci_configure_device_endpoints,
746};
747
748void xhci_init(uint8_t bus, uint8_t slot, uint8_t func,
749 struct pci_device *pci) {
750 struct cpu_mask cmask;
751 alloc_or_die(cpu_mask_init(&cmask, global.core_count));
752
753 cpu_mask_set_all(m: &cmask);
754 struct workqueue_attributes attrs = {
755 .capacity = WORKQUEUE_DEFAULT_CAPACITY,
756 .idle_check = WORKQUEUE_DEFAULT_IDLE_CHECK,
757 .min_workers = 1,
758 .max_workers = 1,
759 .spawn_delay = WORKQUEUE_DEFAULT_SPAWN_DELAY,
760 .worker_cpu_mask = cmask,
761 .worker_niceness = 0,
762 .flags = WORKQUEUE_FLAG_DEFAULTS | WORKQUEUE_FLAG_ISR_SAFE,
763 };
764
765 xhci_wq = workqueue_create(fmt: "xhci_wq", attrs: &attrs);
766
767 xhci_info("Found device at %02x:%02x.%02x", bus, slot, func);
768 void *mmio = xhci_map_mmio(bus, slot, func);
769
770 struct xhci_device *dev = xhci_device_create(mmio);
771 xhci_info("Device at %p, offset is %u", dev,
772 offsetof(struct xhci_request, list));
773
774 semaphore_init(s: &dev->sem, value: 0, SEMAPHORE_INIT_IRQ_DISABLE);
775 spinlock_init(lock: &dev->lock);
776 thread_spawn(name: "xhci_worker", entry: xhci_worker, arg: dev);
777
778 /* Wait till we know our worker is on the sem */
779 while (!atomic_load(&dev->worker_waiting))
780 scheduler_yield();
781
782 dev->pci = pci;
783
784 pci_enable_msix(bus, slot, func);
785 xhci_device_start_interrupts(bus, slot, func, dev);
786
787 if (!xhci_controller_stop(dev))
788 return;
789
790 if (!xhci_controller_reset(dev))
791 return;
792
793 xhci_parse_ext_caps(dev);
794 xhci_detect_usb3_ports(dev);
795 xhci_setup_event_ring(dev);
796
797 xhci_setup_command_ring(dev);
798
799 xhci_controller_start(dev);
800 xhci_controller_enable_ints(dev);
801 xhci_interrupt_enable_ints(dev);
802
803 struct usb_controller *ctrl =
804 kmalloc(sizeof(struct usb_controller), ALLOC_FLAGS_ZERO);
805 ctrl->driver_data = dev;
806 ctrl->type = USB_CONTROLLER_XHCI;
807 ctrl->ops = &xhci_ctrl_ops;
808 dev->controller = ctrl;
809
810 enum irql irql = spin_lock_irq_disable(lock: &dev->lock);
811 for (uint32_t port = 1; port <= dev->ports; port++) {
812 struct xhci_port *p = &dev->port_info[port - 1];
813 uint32_t *portsc_ptr = xhci_portsc_ptr(dev, port);
814 uint32_t v = mmio_read_32(address: portsc_ptr);
815 xhci_ack_port_changes(portsc_ptr, observed: v);
816 if ((v & PORTSC_CCS) && p->state == XHCI_PORT_STATE_DISCONNECTED)
817 xhci_port_set_state(port: p, state: XHCI_PORT_STATE_CONNECTING);
818 }
819 spin_unlock(lock: &dev->lock, old: irql);
820
821 for (uint32_t port = 1; port <= dev->ports; port++) {
822 struct xhci_port *p = &dev->port_info[port - 1];
823 if (p->state != XHCI_PORT_STATE_CONNECTING)
824 continue;
825
826 spin_lock_raw(lock: &p->update_lock);
827 enum usb_error err = xhci_port_init(p);
828 spin_unlock_raw(lock: &p->update_lock);
829
830 if (err == USB_OK) {
831 usb_init_device(dev: p->slot->udev);
832 continue;
833 }
834
835 irql = spin_lock_irq_disable(lock: &dev->lock);
836 if (p->state == XHCI_PORT_STATE_CONNECTING)
837 xhci_port_set_state(port: p, state: XHCI_PORT_STATE_DISCONNECTED);
838 spin_unlock(lock: &dev->lock, old: irql);
839 }
840
841 thread_spawn(name: "xhci_disconnect_worker", entry: xhci_work_port_disconnect, arg: dev);
842 thread_spawn(name: "xhci_connect_worker", entry: xhci_work_port_connect, arg: dev);
843
844#ifdef DEBUG_USB_XHCI
845
846 while (1)
847 wait_for_interrupt();
848
849#endif
850
851 xhci_info("Device initialized successfully");
852}
853
854static enum errno xhci_pci_init(struct device *d) {
855 struct pci_device *dev = d->driver_data;
856 switch (dev->prog_if) {
857 case 0x30: xhci_init(bus: dev->bus, slot: dev->dev, func: dev->function, pci: dev);
858 default: break;
859 }
860
861 return ERR_OK;
862}
863
864PCI_DEV_REGISTER(xhci, 0x0C, 0x03, 0x030, 0xFFFF, xhci_pci_init)
865