1#include <acpi/lapic.h>
2#include <asm.h>
3#include <compiler/core.h>
4#include <drivers/mmio.h>
5#include <drivers/pci.h>
6#include <drivers/usb/xhci.h>
7#include <mem/alloc.h>
8#include <mem/nofail.h>
9#include <mem/page.h>
10#include <mem/vmm.h>
11#include <smp/core.h>
12#include <stdbool.h>
13#include <stdint.h>
14#include <time/spin_sleep.h>
15
16#include "internal.h"
17
18void xhci_setup_event_ring(struct xhci_device *dev) {
19 struct xhci_erst_entry *erst =
20 nofail(valloc_aligned(PAGE_SIZE, PAGE_SIZE, ALLOC_ZERO));
21
22 paddr_t erst_phys = vmm_get_phys((vaddr_t) erst);
23
24 dev->event_ring = nofail(xhci_allocate_event_ring());
25 erst[0].ring_segment_base = dev->event_ring->phys;
26 erst[0].ring_segment_size = dev->event_ring->size;
27 erst[0].reserved = 0;
28
29 struct xhci_interrupter_regs cc_mem_io *ir = dev->intr_regs;
30
31 mmio_write_32(address: &ir->imod, value: 0);
32 mmio_write_32(address: &ir->erstsz, value: 1);
33 mmio_write_64(address: &ir->erdp, value: dev->event_ring->phys);
34 mmio_write_64(address: &ir->erstba, value: erst_phys);
35}
36
37void xhci_setup_command_ring(struct xhci_device *dev) {
38 struct xhci_op_regs cc_mem_io *op = dev->op_regs;
39 dev->cmd_ring = nofail(xhci_allocate_ring());
40 uintptr_t trb_phys = dev->cmd_ring->phys;
41
42 struct xhci_dcbaa *dcbaa_virt =
43 nofail(valloc_aligned(PAGE_SIZE, PAGE_SIZE, ALLOC_ZERO));
44 uintptr_t dcbaa_phys = vmm_get_phys((uintptr_t) dcbaa_virt);
45
46 dev->dcbaa = dcbaa_virt;
47 mmio_write_64(address: &op->crcr, value: trb_phys | 1);
48 mmio_write_64(address: &op->dcbaap, value: dcbaa_phys | 1);
49}
50
51void xhci_nop(struct xhci_device *dev) {
52 struct xhci_request request = {0};
53 struct xhci_command cmd = {0};
54 xhci_request_init_blocking(req: &request, cmd: &cmd, /* port = */ 0,
55 t: XHCI_CMD_TYPE_NO_OP);
56
57 struct xhci_trb outgoing = {
58 .parameter = 0,
59 .control = TRB_SET_TYPE(TRB_TYPE_NO_OP),
60 .status = 0,
61 };
62
63 cmd = (struct xhci_command){
64 .private = &outgoing,
65 .emit = xhci_emit_singular,
66 .ep_id = 0,
67 .slot = NULL,
68 .ring = dev->cmd_ring,
69 .request = &request,
70 .num_trbs = 1,
71 };
72
73 xhci_send_command_and_block(dev, cmd: &cmd);
74}
75
76uint8_t xhci_enable_slot(struct xhci_device *dev) {
77 struct xhci_request request = {0};
78 struct xhci_command cmd = {0};
79 xhci_request_init_blocking(req: &request, cmd: &cmd, /* port = */ 0,
80 t: XHCI_CMD_TYPE_ENABLE_SLOT);
81
82 struct xhci_trb outgoing = {
83 .parameter = 0,
84 .control = TRB_SET_TYPE(TRB_TYPE_ENABLE_SLOT),
85 .status = 0,
86 };
87
88 cmd = (struct xhci_command){
89 .private = &outgoing,
90 .emit = xhci_emit_singular,
91 .ep_id = 0,
92 .slot = NULL,
93 .ring = dev->cmd_ring,
94 .request = &request,
95 .num_trbs = 1,
96 };
97
98 xhci_send_command_and_block(dev, cmd: &cmd);
99
100 return TRB_SLOT(request.return_control);
101}
102
103/* valloc state for DISABLE SLOT command because lifetime requires this
104 * outlives the caller */
105struct xhci_disable_slot_async {
106 struct xhci_request req;
107 struct xhci_command cmd;
108 struct xhci_trb trb;
109};
110
111static void xhci_disable_slot_done(struct xhci_device *dev,
112 struct xhci_request *req) {
113 cc_ignore(dev);
114 vfree(container_of(req, struct xhci_disable_slot_async, req));
115}
116
117/* DISABLE SLOT is issued asynchronously: its completion result is never
118 * consumed, and it can be triggered by xhci_slot_putref() dropping
119 * the last slot reference from inside xhci_process_single(), which runs on the
120 * single command-completion worker
121 *
122 * If that worker blocked here waiting for the DISABLE SLOT completion,
123 * it would deadlock against itself, stranding every other pending request
124 *
125 * Fire-and-forget semantics keep the completion path non-blocking
126 *
127 * Command-ring FIFO ordering guarantees this disable is processed
128 * before any later ENABLE SLOT, so slot reuse is
129 * still safe. */
130void xhci_disable_slot(struct xhci_device *dev, uint8_t slot_id) {
131 struct xhci_disable_slot_async *a = vnew(a, ALLOC_ZERO);
132 if (!a)
133 return;
134
135 xhci_request_init(req: &a->req, cmd: &a->cmd, NULL, t: XHCI_CMD_TYPE_DISABLE_SLOT);
136
137 a->req.callback = xhci_disable_slot_done;
138 a->req.port = 0;
139
140 a->trb = (struct xhci_trb){
141 .parameter = 0,
142 .status = 0,
143 .control =
144 TRB_SET_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_SET_SLOT_ID(slot_id),
145 };
146
147 a->cmd = (struct xhci_command){
148 .private = &a->trb,
149 .emit = xhci_emit_singular,
150 .ep_id = 0,
151 .slot = NULL,
152 .ring = dev->cmd_ring,
153 .request = &a->req,
154 .num_trbs = 1,
155 };
156
157 if (!xhci_send_command(dev, cmd: &a->cmd))
158 vfree(a);
159}
160
161static enum usb_error xhci_spin_wait_port_reset(uint32_t cc_mem_io *portsc,
162 bool is_usb3) {
163 const uint64_t timeout_us = 100 * 1000;
164 uint64_t start = time_get_us();
165
166 while (time_get_us() - start < timeout_us) {
167 uint32_t v = mmio_read_32(address: portsc);
168
169 if (is_usb3) {
170 if (!(v & PORTSC_WPR)) {
171 uint32_t pls = (v >> PORTSC_PLS_SHIFT) & PORTSC_PLS_MASK;
172 if (pls == PORTSC_PLS_RXDETECT)
173 return USB_ERR_IO;
174 if (pls == PORTSC_PLS_U0)
175 return USB_OK;
176 }
177 } else {
178 if (!(v & PORTSC_PR)) {
179 if (v & PORTSC_PED)
180 return USB_OK;
181 return USB_ERR_IO;
182 }
183 }
184
185 cpu_pause();
186 sleep_spin_us(us: 10);
187 }
188
189 return USB_ERR_TIMEOUT;
190}
191
192enum usb_error xhci_reset_port(struct xhci_device *dev, uint32_t portnum) {
193 uint32_t cc_mem_io *portsc = xhci_portsc_ptr(dev, port: portnum);
194 bool is_usb3 = dev->port_info[portnum - 1].usb3;
195
196 uint32_t v = mmio_read_32(address: portsc);
197
198 /* Power on */
199 if (!(v & PORTSC_PP)) {
200 mmio_write_32(address: portsc, value: v | PORTSC_PP);
201 sleep_spin_us(us: 2000);
202 v = mmio_read_32(address: portsc);
203 if (!(v & PORTSC_PP))
204 return USB_ERR_IO;
205 }
206
207 /* Clear change bits */
208 mmio_write_32(address: portsc,
209 value: v | PORTSC_CSC | PORTSC_PEC | PORTSC_PRC | PORTSC_WRC);
210
211 sleep_spin_us(us: 100);
212
213 /* Initiate reset */
214 if (is_usb3) {
215 mmio_write_32(address: portsc, PORTSC_WPR);
216 } else {
217 mmio_write_32(address: portsc, PORTSC_PR);
218 }
219
220 /* Spin-poll completion */
221 enum usb_error st = xhci_spin_wait_port_reset(portsc, is_usb3);
222
223 if (st != USB_OK)
224 return st;
225
226 /* Final sanity check */
227 v = mmio_read_32(address: portsc);
228 if (!(v & PORTSC_PED))
229 return USB_ERR_IO;
230
231 return USB_OK;
232}
233
234void xhci_parse_ext_caps(struct xhci_device *dev) {
235 uint32_t hcc_params1 = mmio_read_32(address: &dev->cap_regs->hcc_params1);
236 uint32_t offset = (hcc_params1 >> 16) & 0xFFFF;
237
238 while (offset) {
239 void cc_mem_io *ext_cap_addr =
240 (uint8_t cc_mem_io *) dev->cap_regs + offset * 4;
241 uint32_t cap_header = mmio_read_32(address: ext_cap_addr);
242
243 uint8_t cap_id = cap_header & 0xFF;
244 uint8_t next = (cap_header >> 8) & 0xFF;
245
246 if (cap_id != XHCI_EXT_CAP_ID_LEGACY_SUPPORT) {
247 offset = next;
248 continue;
249 }
250
251 void cc_mem_io *bios_owns_addr = (uint8_t cc_mem_io *) ext_cap_addr + 4;
252 void cc_mem_io *os_owns_addr = (uint8_t cc_mem_io *) ext_cap_addr + 8;
253
254 mmio_write_32(address: os_owns_addr, value: 1);
255
256 uint64_t timeout = 1000 * 1000;
257 while ((mmio_read_32(address: bios_owns_addr) & 1) && timeout--) {
258 sleep_spin_us(us: 1);
259 }
260
261 uint32_t own_data = mmio_read_32(address: bios_owns_addr);
262 if (own_data & 1) {
263 xhci_warn("BIOS ownership handoff failed");
264 } else {
265 xhci_info("BIOS ownership handoff completed");
266 }
267
268 break;
269 }
270}
271
272void xhci_detect_usb3_ports(struct xhci_device *dev) {
273 uint32_t hcc_params1 = mmio_read_32(address: &dev->cap_regs->hcc_params1);
274 uint32_t offset = (hcc_params1 >> 16) & 0xFFFF;
275
276 while (offset) {
277 void cc_mem_io *ext_cap_addr =
278 (uint8_t cc_mem_io *) dev->cap_regs + offset * 4;
279 uint32_t cap_header = mmio_read_32(address: ext_cap_addr);
280
281 uint8_t cap_id = cap_header & 0xFF;
282 uint8_t next = (cap_header >> 8) & 0xFF;
283
284 if (cap_id == XHCI_EXT_CAP_ID_USB) {
285 uint32_t cap[4];
286 for (int i = 0; i < 4; i++)
287 cap[i] =
288 mmio_read_32(address: (uint8_t cc_mem_io *) ext_cap_addr + i * 4);
289
290 uint8_t portcount = (cap[2] >> 8) & 0xFF;
291 uint8_t portoffset = (cap[2]) & 0xFF;
292 uint8_t major = (cap[0] >> 24) & 0xFF;
293
294 if (portoffset == 0 || portcount == 0) {
295 xhci_warn("USB capability with invalid port offset/count");
296 } else {
297 uint8_t start = portoffset - 1;
298
299 for (uint8_t i = start; i < start + portcount; i++) {
300 dev->port_info[i].usb3 = major >= 3;
301 xhci_info("Port %u detected as USB%u", i + 1, major);
302 }
303 }
304 }
305
306 offset = next;
307 }
308}
309
310void cc_mem_io *xhci_map_mmio(uint8_t bus, uint8_t slot, uint8_t func) {
311 uint32_t original_bar0 = pci_read(bus, slot, func, offset: 0x10);
312
313 pci_write(bus, slot, func, offset: 0x10, value: 0xFFFFFFFF);
314 uint32_t size_mask = pci_read(bus, slot, func, offset: 0x10);
315 pci_write(bus, slot, func, offset: 0x10, value: original_bar0);
316
317 uint32_t size = ~(size_mask & ~0xF) + 1;
318
319 uint32_t phys_addr = original_bar0 & ~0xF;
320 return mmio_map(phys: phys_addr, size);
321}
322
323struct xhci_device *xhci_device_create(void cc_mem_io *mmio) {
324 struct xhci_device *dev = vnew(dev, ALLOC_ZERO);
325 if (cc_unlikely(!dev))
326 panic(OUT_OF_MEMORY, "XHCI device");
327
328 struct xhci_cap_regs cc_mem_io *cap = mmio;
329 struct xhci_op_regs cc_mem_io *op = mmio + cap->cap_length;
330 void cc_mem_io *runtime_regs = (void cc_mem_io *) mmio + cap->rtsoff;
331 struct xhci_interrupter_regs cc_mem_io *ir_base =
332 (void cc_mem_io *) ((uint8_t cc_mem_io *) runtime_regs + 0x20);
333
334 for (size_t i = 0; i < XHCI_REQ_LIST_MAX; i++) {
335 list_head_init(h: &dev->requests[i]);
336 }
337
338 dev->num_devices = 0;
339 dev->port_regs = op->regs;
340 dev->intr_regs = ir_base;
341 dev->cap_regs = cap;
342 dev->op_regs = op;
343 dev->ports = cap->hcs_params1 & 0xff;
344 list_head_init(h: &dev->devices);
345 semaphore_init(s: &dev->port_disconnect, value: 0, SEMAPHORE_INIT_IRQ_DISABLE);
346 semaphore_init(s: &dev->port_connect, value: 0, SEMAPHORE_INIT_IRQ_DISABLE);
347 semaphore_init(s: &dev->sem, value: 0, SEMAPHORE_INIT_IRQ_DISABLE);
348
349 for (uint32_t i = 0; i < XHCI_PORT_COUNT; i++) {
350 struct xhci_port *p = &dev->port_info[i];
351 p->generation = 0;
352 p->usb3 = false;
353 p->state = XHCI_PORT_STATE_DISCONNECTED;
354 uint32_t portsc = xhci_read_portsc(dev, port: i + 1);
355 uint8_t speed = portsc & 0xF;
356 p->speed = speed;
357 p->dev = dev;
358 p->port_id = (i + 1);
359 spinlock_init(&p->update_lock, LOCK_UNCHKD);
360 }
361
362 for (size_t i = 0; i < XHCI_SLOT_COUNT; i++) {
363 struct xhci_slot *xs = &dev->slots[i];
364 xs->dev = dev;
365 atomic_init(&xs->state, XHCI_SLOT_STATE_DISCONNECTED);
366 xs->slot_id = (i + 1);
367 }
368
369 return dev;
370}
371
372void xhci_device_start_interrupts(uint8_t bus, uint8_t slot, uint8_t func,
373 struct xhci_device *dev) {
374 dev->irq = irq_alloc_entry();
375 irq_register(name: "xhci", vector: dev->irq, handler: xhci_isr, ctx: dev, flags: IRQ_FLAG_NONE);
376 irq_set_chip(vector: dev->irq, chip: lapic_get_chip(), NULL);
377 pci_program_msix_entry(bus, slot, func, table_index: 0, vector: dev->irq, apic_id: smp_apic_id(cpu: 0));
378}
379
380enum usb_error xhci_port_init(struct xhci_port *p) {
381 struct xhci_device *dev = p->dev;
382 uint8_t port = p->port_id;
383 enum usb_error err = USB_OK;
384 uint8_t slot_id;
385 struct usb_device *usb;
386 if (!(usb = vnew(usb, ALLOC_ZERO))) {
387 return USB_ERR_OOM;
388 }
389
390 xhci_trace("reset_port sent");
391 if ((err = xhci_reset_port(dev, portnum: port)) != USB_OK) {
392 xhci_trace("reset_port fail");
393 vfree(usb);
394 return err;
395 }
396
397 xhci_trace("reset_port returned");
398
399 xhci_trace("enable_slot sent");
400 if ((slot_id = xhci_enable_slot(dev)) == 0) {
401 xhci_trace("enable_slot fail");
402 vfree(usb);
403 return USB_ERR_NO_DEVICE;
404 }
405 xhci_trace("enable_slot returned");
406
407 struct xhci_slot temp_slot = {0};
408 atomic_init(&temp_slot.state, XHCI_SLOT_STATE_ENABLED);
409 temp_slot.slot_id = slot_id;
410 temp_slot.dev = dev;
411
412 /* The device can be yanked from the port in the window between Enable Slot
413 * completing and Address Device being processed... Once the device is gone
414 * the controller releases the port, so Address Device is rejected with a
415 * TRB Error */
416 if (!(xhci_read_portsc(dev, port) & PORTSC_CCS)) {
417 xhci_debug("port %u departed before address_device; aborting", port);
418 xhci_disable_slot(dev, slot_id);
419 vfree(usb);
420 return USB_ERR_NO_DEVICE;
421 }
422
423 xhci_trace("address_device sent");
424 if ((err = xhci_address_device(p, slot_id, publish_to: &temp_slot)) != USB_OK) {
425 xhci_trace("address_device fail");
426 if (!(xhci_read_portsc(dev, port) & PORTSC_CCS))
427 xhci_trace("address_device aborted: port %u departed", port);
428 else
429 xhci_warn("address_device failed on port %u: %d", port, err);
430 xhci_trace("disable_slot sent");
431 xhci_disable_slot(dev, slot_id);
432 xhci_trace("disable_slot returned");
433 vfree(usb);
434 return err;
435 }
436 xhci_trace("address_device returned");
437
438 usb->speed = p->speed;
439 usb->port = port;
440 usb->configured = false;
441 usb->host = dev->controller;
442 usb->driver_private = dev;
443
444 refcount_init(rc: &usb->refcount, val: 1);
445 list_node_init(n: &usb->hc_list);
446
447 struct xhci_slot *this_slot = xhci_get_slot(dev, id: slot_id);
448 memcpy(this_slot, &temp_slot, sizeof(struct xhci_slot));
449
450 xhci_port_set_state(port: p, state: XHCI_PORT_STATE_CONNECTED);
451 refcount_init(rc: &this_slot->refcount, val: 1);
452 p->slot = this_slot;
453 this_slot->port = p;
454 this_slot->udev = usb;
455
456 list_head_push_back(h: &dev->devices, n: &usb->hc_list);
457 dev->num_devices++;
458 usb->slot = this_slot;
459
460 return err;
461}
462