1#include <asm.h>
2#include <compiler.h>
3#include <drivers/mmio.h>
4#include <drivers/pci.h>
5#include <drivers/usb/xhci.h>
6#include <mem/alloc.h>
7#include <mem/page.h>
8#include <mem/vmm.h>
9#include <stdbool.h>
10#include <stdint.h>
11#include <time/spin_sleep.h>
12
13#include "internal.h"
14
15void xhci_setup_event_ring(struct xhci_device *dev) {
16 struct xhci_erst_entry *erst =
17 kmalloc_aligned(PAGE_SIZE, PAGE_SIZE, ALLOC_FLAGS_ZERO);
18
19 paddr_t erst_phys = vmm_get_phys((vaddr_t) erst, VMM_FLAG_NONE);
20
21 dev->event_ring = xhci_allocate_event_ring();
22 erst[0].ring_segment_base = dev->event_ring->phys;
23 erst[0].ring_segment_size = dev->event_ring->size;
24 erst[0].reserved = 0;
25
26 struct xhci_interrupter_regs *ir = dev->intr_regs;
27
28 mmio_write_32(address: &ir->imod, value: 0);
29 mmio_write_32(address: &ir->erstsz, value: 1);
30 mmio_write_64(address: &ir->erdp, value: dev->event_ring->phys);
31 mmio_write_64(address: &ir->erstba, value: erst_phys);
32}
33
34void xhci_setup_command_ring(struct xhci_device *dev) {
35 struct xhci_op_regs *op = dev->op_regs;
36 dev->cmd_ring = xhci_allocate_ring();
37 uintptr_t trb_phys = dev->cmd_ring->phys;
38
39 struct xhci_dcbaa *dcbaa_virt =
40 kmalloc_aligned(PAGE_SIZE, PAGE_SIZE, ALLOC_FLAGS_ZERO);
41 uintptr_t dcbaa_phys = vmm_get_phys((uintptr_t) dcbaa_virt, VMM_FLAG_NONE);
42
43 dev->dcbaa = dcbaa_virt;
44 mmio_write_64(address: &op->crcr, value: trb_phys | 1);
45 mmio_write_64(address: &op->dcbaap, value: dcbaa_phys | 1);
46}
47
48void xhci_nop(struct xhci_device *dev) {
49 struct xhci_request request = {0};
50 struct xhci_command cmd = {0};
51 xhci_request_init_blocking(req: &request, cmd: &cmd, /* port = */ 0,
52 t: XHCI_CMD_TYPE_NO_OP);
53
54 struct xhci_trb outgoing = {
55 .parameter = 0,
56 .control = TRB_SET_TYPE(TRB_TYPE_NO_OP),
57 .status = 0,
58 };
59
60 cmd = (struct xhci_command){
61 .private = &outgoing,
62 .emit = xhci_emit_singular,
63 .ep_id = 0,
64 .slot = NULL,
65 .ring = dev->cmd_ring,
66 .request = &request,
67 .num_trbs = 1,
68 };
69
70 xhci_send_command_and_block(dev, cmd: &cmd, NULL);
71}
72
73uint8_t xhci_enable_slot(struct xhci_device *dev) {
74 struct xhci_request request = {0};
75 struct xhci_command cmd = {0};
76 xhci_request_init_blocking(req: &request, cmd: &cmd, /* port = */ 0,
77 t: XHCI_CMD_TYPE_ENABLE_SLOT);
78
79 struct xhci_trb outgoing = {
80 .parameter = 0,
81 .control = TRB_SET_TYPE(TRB_TYPE_ENABLE_SLOT),
82 .status = 0,
83 };
84
85 cmd = (struct xhci_command){
86 .private = &outgoing,
87 .emit = xhci_emit_singular,
88 .ep_id = 0,
89 .slot = NULL,
90 .ring = dev->cmd_ring,
91 .request = &request,
92 .num_trbs = 1,
93 };
94
95 xhci_send_command_and_block(dev, cmd: &cmd, NULL);
96
97 return TRB_SLOT(request.return_control);
98}
99
100/* kmalloc state for DISABLE SLOT command because lifetime requires this
101 * outlives the caller */
102struct xhci_disable_slot_async {
103 struct xhci_request req;
104 struct xhci_command cmd;
105 struct xhci_trb trb;
106};
107
108static void xhci_disable_slot_done(struct xhci_device *dev,
109 struct xhci_request *req) {
110 (void) dev;
111 kfree(container_of(req, struct xhci_disable_slot_async, req));
112}
113
114/* DISABLE SLOT is issued asynchronously: its completion result is never
115 * consumed, and it can be triggered by xhci_slot_put() dropping
116 * the last slot reference from inside xhci_process_single(), which runs on the
117 * single command-completion worker
118 *
119 * If that worker blocked here waiting for the DISABLE SLOT completion,
120 * it would deadlock against itself, stranding every other pending request
121 *
122 * Fire-and-forget semantics keep the completion path non-blocking
123 *
124 * Command-ring FIFO ordering guarantees this disable is processed
125 * before any later ENABLE SLOT, so slot reuse is
126 * still safe. */
127void xhci_disable_slot(struct xhci_device *dev, uint8_t slot_id) {
128 struct xhci_disable_slot_async *a = kmalloc(sizeof(*a), ALLOC_FLAGS_ZERO);
129 if (!a)
130 return;
131
132 xhci_request_init(req: &a->req, cmd: &a->cmd, NULL, t: XHCI_CMD_TYPE_DISABLE_SLOT);
133
134 a->req.callback = xhci_disable_slot_done;
135 a->req.port = 0;
136
137 a->trb = (struct xhci_trb){
138 .parameter = 0,
139 .status = 0,
140 .control =
141 TRB_SET_TYPE(TRB_TYPE_DISABLE_SLOT) | TRB_SET_SLOT_ID(slot_id),
142 };
143
144 a->cmd = (struct xhci_command){
145 .private = &a->trb,
146 .emit = xhci_emit_singular,
147 .ep_id = 0,
148 .slot = NULL,
149 .ring = dev->cmd_ring,
150 .request = &a->req,
151 .num_trbs = 1,
152 };
153
154 if (!xhci_send_command(dev, cmd: &a->cmd))
155 kfree(a);
156}
157
158static enum usb_error xhci_spin_wait_port_reset(uint32_t *portsc,
159 bool is_usb3) {
160 const uint64_t timeout_us = 100 * 1000;
161 uint64_t start = time_get_us();
162
163 while (time_get_us() - start < timeout_us) {
164 uint32_t v = mmio_read_32(address: portsc);
165
166 if (is_usb3) {
167 if (!(v & PORTSC_WPR)) {
168 uint32_t pls = (v >> PORTSC_PLS_SHIFT) & PORTSC_PLS_MASK;
169 if (pls == PORTSC_PLS_RXDETECT)
170 return USB_ERR_IO;
171 if (pls == PORTSC_PLS_U0)
172 return USB_OK;
173 }
174 } else {
175 if (!(v & PORTSC_PR)) {
176 if (v & PORTSC_PED)
177 return USB_OK;
178 return USB_ERR_IO;
179 }
180 }
181
182 cpu_relax();
183 sleep_spin_us(us: 10);
184 }
185
186 return USB_ERR_TIMEOUT;
187}
188
189enum usb_error xhci_reset_port(struct xhci_device *dev, uint32_t portnum) {
190 uint32_t *portsc = xhci_portsc_ptr(dev, port: portnum);
191 bool is_usb3 = dev->port_info[portnum - 1].usb3;
192
193 uint32_t v = mmio_read_32(address: portsc);
194
195 /* Power on */
196 if (!(v & PORTSC_PP)) {
197 mmio_write_32(address: portsc, value: v | PORTSC_PP);
198 sleep_spin_us(us: 2000);
199 v = mmio_read_32(address: portsc);
200 if (!(v & PORTSC_PP))
201 return USB_ERR_IO;
202 }
203
204 /* Clear change bits */
205 mmio_write_32(address: portsc,
206 value: v | PORTSC_CSC | PORTSC_PEC | PORTSC_PRC | PORTSC_WRC);
207
208 sleep_spin_us(us: 100);
209
210 /* Initiate reset */
211 if (is_usb3) {
212 mmio_write_32(address: portsc, PORTSC_WPR);
213 } else {
214 mmio_write_32(address: portsc, PORTSC_PR);
215 }
216
217 /* Spin-poll completion */
218 enum usb_error st = xhci_spin_wait_port_reset(portsc, is_usb3);
219
220 if (st != USB_OK)
221 return st;
222
223 /* Final sanity check */
224 v = mmio_read_32(address: portsc);
225 if (!(v & PORTSC_PED))
226 return USB_ERR_IO;
227
228 return USB_OK;
229}
230
231void xhci_parse_ext_caps(struct xhci_device *dev) {
232 uint32_t hcc_params1 = mmio_read_32(address: &dev->cap_regs->hcc_params1);
233 uint32_t offset = (hcc_params1 >> 16) & 0xFFFF;
234
235 while (offset) {
236 void *ext_cap_addr = (uint8_t *) dev->cap_regs + offset * 4;
237 uint32_t cap_header = mmio_read_32(address: ext_cap_addr);
238
239 uint8_t cap_id = cap_header & 0xFF;
240 uint8_t next = (cap_header >> 8) & 0xFF;
241
242 if (cap_id != XHCI_EXT_CAP_ID_LEGACY_SUPPORT) {
243 offset = next;
244 continue;
245 }
246
247 void *bios_owns_addr = (uint8_t *) ext_cap_addr + 4;
248 void *os_owns_addr = (uint8_t *) ext_cap_addr + 8;
249
250 mmio_write_32(address: os_owns_addr, value: 1);
251
252 uint64_t timeout = 1000 * 1000;
253 while ((mmio_read_32(address: bios_owns_addr) & 1) && timeout--) {
254 sleep_spin_us(us: 1);
255 }
256
257 uint32_t own_data = mmio_read_32(address: bios_owns_addr);
258 if (own_data & 1) {
259 xhci_warn("BIOS ownership handoff failed");
260 } else {
261 xhci_info("BIOS ownership handoff completed");
262 }
263
264 break;
265 }
266}
267
268void xhci_detect_usb3_ports(struct xhci_device *dev) {
269 uint32_t hcc_params1 = mmio_read_32(address: &dev->cap_regs->hcc_params1);
270 uint32_t offset = (hcc_params1 >> 16) & 0xFFFF;
271
272 while (offset) {
273 void *ext_cap_addr = (uint8_t *) dev->cap_regs + offset * 4;
274 uint32_t cap_header = mmio_read_32(address: ext_cap_addr);
275
276 uint8_t cap_id = cap_header & 0xFF;
277 uint8_t next = (cap_header >> 8) & 0xFF;
278
279 if (cap_id == XHCI_EXT_CAP_ID_USB) {
280 uint32_t cap[4];
281 for (int i = 0; i < 4; i++)
282 cap[i] = mmio_read_32(address: (uint8_t *) ext_cap_addr + i * 4);
283
284 uint8_t portcount = (cap[2] >> 8) & 0xFF;
285 uint8_t portoffset = (cap[2]) & 0xFF;
286 uint8_t major = (cap[0] >> 24) & 0xFF;
287
288 if (portoffset == 0 || portcount == 0) {
289 xhci_warn("USB capability with invalid port offset/count");
290 } else {
291 uint8_t start = portoffset - 1;
292
293 for (uint8_t i = start; i < start + portcount; i++) {
294 dev->port_info[i].usb3 = major >= 3;
295 xhci_info("Port %u detected as USB%u", i + 1, major);
296 }
297 }
298 }
299
300 offset = next;
301 }
302}
303
304void *xhci_map_mmio(uint8_t bus, uint8_t slot, uint8_t func) {
305 uint32_t original_bar0 = pci_read(bus, slot, func, offset: 0x10);
306
307 pci_write(bus, slot, func, offset: 0x10, value: 0xFFFFFFFF);
308 uint32_t size_mask = pci_read(bus, slot, func, offset: 0x10);
309 pci_write(bus, slot, func, offset: 0x10, value: original_bar0);
310
311 uint32_t size = ~(size_mask & ~0xF) + 1;
312
313 uint32_t phys_addr = original_bar0 & ~0xF;
314 return mmio_map(phys: phys_addr, size);
315}
316
317struct xhci_device *xhci_device_create(void *mmio) {
318 struct xhci_device *dev =
319 kmalloc(sizeof(struct xhci_device), ALLOC_FLAGS_ZERO);
320 if (unlikely(!dev))
321 panic("Could not allocate space for XHCI device");
322
323 struct xhci_cap_regs *cap = mmio;
324 struct xhci_op_regs *op = mmio + cap->cap_length;
325 void *runtime_regs = (void *) mmio + cap->rtsoff;
326 struct xhci_interrupter_regs *ir_base =
327 (void *) ((uint8_t *) runtime_regs + 0x20);
328
329 for (size_t i = 0; i < XHCI_REQ_LIST_MAX; i++) {
330 INIT_LIST_HEAD(list: &dev->requests[i]);
331 }
332
333 dev->num_devices = 0;
334 dev->port_regs = op->regs;
335 dev->intr_regs = ir_base;
336 dev->cap_regs = cap;
337 dev->op_regs = op;
338 dev->ports = cap->hcs_params1 & 0xff;
339 INIT_LIST_HEAD(list: &dev->devices);
340 semaphore_init(s: &dev->port_disconnect, value: 0, SEMAPHORE_INIT_IRQ_DISABLE);
341 semaphore_init(s: &dev->port_connect, value: 0, SEMAPHORE_INIT_IRQ_DISABLE);
342 semaphore_init(s: &dev->sem, value: 0, SEMAPHORE_INIT_IRQ_DISABLE);
343
344 for (uint32_t i = 0; i < XHCI_PORT_COUNT; i++) {
345 struct xhci_port *p = &dev->port_info[i];
346 p->generation = 0;
347 p->usb3 = false;
348 p->state = XHCI_PORT_STATE_DISCONNECTED;
349 uint32_t portsc = xhci_read_portsc(dev, port: i + 1);
350 uint8_t speed = portsc & 0xF;
351 p->speed = speed;
352 p->dev = dev;
353 p->port_id = (i + 1);
354 spinlock_init(lock: &p->update_lock);
355 }
356
357 for (size_t i = 0; i < XHCI_SLOT_COUNT; i++) {
358 struct xhci_slot *xs = &dev->slots[i];
359 xs->dev = dev;
360 xs->state = XHCI_SLOT_STATE_DISCONNECTED;
361 xs->slot_id = (i + 1);
362 }
363
364 return dev;
365}
366
367void xhci_device_start_interrupts(uint8_t bus, uint8_t slot, uint8_t func,
368 struct xhci_device *dev) {
369 dev->irq = irq_alloc_entry();
370 irq_register(name: "xhci", vector: dev->irq, handler: xhci_isr, ctx: dev, flags: IRQ_FLAG_NONE);
371 irq_set_chip(vector: dev->irq, chip: lapic_get_chip(), NULL);
372 pci_program_msix_entry(bus, slot, func, table_index: 0, vector: dev->irq, /*core=*/apic_id: 0);
373}
374
375enum usb_error xhci_port_init(struct xhci_port *p) {
376 struct xhci_device *dev = p->dev;
377 uint8_t port = p->port_id;
378 enum usb_error err = USB_OK;
379 uint8_t slot_id;
380 struct usb_device *usb;
381 if (!(usb = kmalloc(sizeof(struct usb_device), ALLOC_FLAGS_ZERO))) {
382 return USB_ERR_OOM;
383 }
384
385 xhci_trace("reset_port sent");
386 if ((err = xhci_reset_port(dev, portnum: port)) != USB_OK) {
387 xhci_trace("reset_port fail");
388 kfree(usb);
389 return err;
390 }
391
392 xhci_trace("reset_port returned");
393
394 xhci_trace("enable_slot sent");
395 if ((slot_id = xhci_enable_slot(dev)) == 0) {
396 xhci_trace("enable_slot fail");
397 kfree(usb);
398 return USB_ERR_NO_DEVICE;
399 }
400 xhci_trace("enable_slot returned");
401
402 struct xhci_slot temp_slot = {0};
403 temp_slot.state = XHCI_SLOT_STATE_ENABLED;
404 temp_slot.slot_id = slot_id;
405 temp_slot.dev = dev;
406
407 /* The device can be yanked from the port in the window between Enable Slot
408 * completing and Address Device being processed... Once the device is gone
409 * the controller releases the port, so Address Device is rejected with a
410 * TRB Error */
411 if (!(xhci_read_portsc(dev, port) & PORTSC_CCS)) {
412 xhci_debug("port %u departed before address_device; aborting", port);
413 xhci_disable_slot(dev, slot_id);
414 kfree(usb);
415 return USB_ERR_NO_DEVICE;
416 }
417
418 xhci_trace("address_device sent");
419 if ((err = xhci_address_device(p, slot_id, publish_to: &temp_slot)) != USB_OK) {
420 xhci_trace("address_device fail");
421 if (!(xhci_read_portsc(dev, port) & PORTSC_CCS))
422 xhci_trace("address_device aborted: port %u departed", port);
423 else
424 xhci_warn("address_device failed on port %u: %d", port, err);
425 xhci_trace("disable_slot sent");
426 xhci_disable_slot(dev, slot_id);
427 xhci_trace("disable_slot returned");
428 kfree(usb);
429 return err;
430 }
431 xhci_trace("address_device returned");
432
433 usb->speed = p->speed;
434 usb->port = port;
435 usb->configured = false;
436 usb->host = dev->controller;
437 usb->driver_private = dev;
438
439 refcount_init(rc: &usb->refcount, val: 1);
440 INIT_LIST_HEAD(list: &usb->hc_list);
441
442 struct xhci_slot *this_slot = xhci_get_slot(dev, id: slot_id);
443 memcpy(this_slot, &temp_slot, sizeof(struct xhci_slot));
444
445 xhci_port_set_state(port: p, state: XHCI_PORT_STATE_CONNECTED);
446 refcount_init(rc: &this_slot->refcount, val: 1);
447 p->slot = this_slot;
448 this_slot->port = p;
449 this_slot->udev = usb;
450
451 list_add_tail(new: &usb->hc_list, head: &dev->devices);
452 dev->num_devices++;
453 usb->slot = this_slot;
454
455 return err;
456}
457