1#include <acpi/lapic.h>
2#include <console/printf.h>
3#include <drivers/mmio.h>
4#include <drivers/pci.h>
5#include <drivers/usb/xhci.h>
6#include <log.h>
7#include <math/align.h>
8#include <math/bit.h>
9#include <math/min_max.h>
10#include <mem/alloc.h>
11#include <mem/page.h>
12#include <mem/vmm.h>
13#include <stddef.h>
14#include <stdint.h>
15
16static struct pci_device *pci_devices = NULL;
17static uint64_t pci_device_count;
18LOG_HANDLE_DEFINE_PRINT(pci);
19LOG_SITE_DEFINE_PRINT(pci);
20
21const char *pci_class_name(uint8_t class_code, uint8_t subclass) {
22 switch (class_code) {
23 case 0x01:
24 switch (subclass) {
25 case 0x00: return "SCSI Controller";
26 case 0x01: return "IDE Controller";
27 case 0x02: return "Floppy Controller";
28 case 0x03: return "IPI Bus Controller";
29 case 0x04: return "RAID Controller";
30 case 0x05: return "ATA Controller";
31 case 0x06: return "SATA Controller (AHCI)";
32 case 0x08: return "NVMe Controller";
33 default: return "Other Mass Storage Controller";
34 }
35 case 0x02: return "Network Controller";
36 case 0x03: return "Display Controller";
37 case 0x04: return "Multimedia Controller";
38 case 0x06: return "Bridge Device";
39 case 0x0C:
40 if (subclass == 0x03)
41 return "USB Controller";
42 break;
43 }
44 return "Unknown Device";
45}
46
47static void init_device(struct pci_device *dev) {
48 struct pci_driver *start = __skernel_pci_devices;
49 struct pci_driver *end = __ekernel_pci_devices;
50
51 for (struct pci_driver *d = start; d < end; d++) {
52 bool class, subclass, prog_if, vendor;
53
54 class = dev->class_code == d->class_code;
55 subclass = dev->subclass == d->subclass;
56 prog_if = d->prog_if == 0xFF ? true : dev->prog_if == d->prog_if;
57 vendor = d->vendor_id == 0xFFFF ? true : dev->vendor_id == d->vendor_id;
58 dev->device.driver_data = dev;
59
60 if (class && subclass && prog_if && vendor) {
61 d->driver.probe(&dev->device);
62 }
63 }
64}
65
66void pci_init_devices(struct pci_device *devices, uint64_t count) {
67 struct pci_driver *start = __skernel_pci_devices;
68 struct pci_driver *end = __ekernel_pci_devices;
69
70 pci_log(LOG_INFO, "There are %zu PCI drivers", (size_t) (end - start));
71
72 for (uint64_t i = 0; i < count; i++) {
73 struct pci_device *dev = &devices[i];
74 init_device(dev);
75 }
76}
77
78void pci_scan_devices(struct pci_device **devices_out, uint64_t *count_out) {
79 pci_device_count = 0;
80
81 uint64_t space_to_alloc = 0;
82
83 for (uint16_t bus = 0; bus < 256; ++bus) {
84 for (uint8_t device = 0; device < 32; ++device) {
85 for (uint8_t function = 0; function < 8; ++function) {
86 uint16_t vendor = pci_read_word(bus, slot: device, func: function, offset: 0x00);
87 if (vendor == 0xFFFF)
88 continue;
89
90 space_to_alloc++;
91
92 /* TODO: Do not blindly enable all PCI devices */
93 union pci_command_reg cmd;
94 cmd.value = pci_read_config16(bus, device, function, offset: 0x04);
95
96 cmd.bus_master = 1;
97 cmd.memory_space = 1;
98
99 pci_write_config16(bus, device, function, offset: 0x04, value: cmd.value);
100
101 if (function == 0) {
102 uint8_t header_type =
103 pci_read_byte(bus, slot: device, func: function, offset: 0x0E);
104 if ((header_type & 0x80) == 0)
105 break;
106 }
107 }
108 }
109 }
110
111 pci_devices = vnews(pci_devices, space_to_alloc);
112 if (!pci_devices)
113 panic(OUT_OF_MEMORY, "PCI devices");
114
115 for (uint16_t bus = 0; bus < 256; ++bus) {
116 for (uint8_t device = 0; device < 32; ++device) {
117 for (uint8_t function = 0; function < 8; ++function) {
118 uint16_t vendor_id = pci_read_word(bus, slot: device, func: function, offset: 0x00);
119 if (vendor_id == 0xFFFF)
120 continue;
121
122 uint16_t device_id = pci_read_word(bus, slot: device, func: function, offset: 0x02);
123 uint32_t class_info = pci_read(bus, slot: device, func: function, offset: 0x08);
124 uint8_t class_code = (class_info >> 24) & 0xFF;
125 uint8_t subclass = (class_info >> 16) & 0xFF;
126 uint8_t prog_if = (class_info >> 8) & 0xFF;
127 uint8_t revision = class_info & 0xFF;
128
129 pci_devices[pci_device_count++] =
130 (struct pci_device){.bus = bus,
131 .dev = device,
132 .function = function,
133 .vendor_id = vendor_id,
134 .device_id = device_id,
135 .class_code = class_code,
136 .subclass = subclass,
137 .prog_if = prog_if,
138 .revision = revision};
139
140 pci_log(LOG_INFO, "Found device '%s' at %02x:%02x.%x",
141 pci_class_name(class_code, subclass), bus, device,
142 function);
143
144 if (function == 0) {
145 uint8_t header_type =
146 pci_read_byte(bus, slot: device, func: function, offset: 0x0E);
147 if ((header_type & 0x80) == 0)
148 break;
149 }
150 }
151 }
152 }
153
154 *devices_out = pci_devices;
155 *count_out = pci_device_count;
156}
157
158uint8_t pci_find_capability(uint8_t bus, uint8_t slot, uint8_t func,
159 uint8_t cap_id) {
160 uint8_t cap_ptr = pci_read_byte(bus, slot, func, PCI_CAP_PTR);
161
162 while (cap_ptr != 0 && cap_ptr != 0xFF) {
163 uint8_t current_id = pci_read_byte(bus, slot, func, offset: cap_ptr);
164 if (current_id == cap_id) {
165 return cap_ptr;
166 }
167 cap_ptr = pci_read_byte(bus, slot, func, offset: cap_ptr + 1);
168 }
169
170 return 0;
171}
172
173uint32_t pci_read_bar(uint8_t bus, uint8_t device, uint8_t function,
174 uint8_t bar_index) {
175 uint8_t offset = 0x10 + (bar_index * 4);
176 return pci_read(bus, slot: device, func: function, offset);
177}
178
179static uint64_t pci_read_bar64(uint8_t bus, uint8_t slot, uint8_t func,
180 uint8_t bar_index) {
181 uint32_t low = pci_read(bus, slot, func, offset: 0x10 + 4 * bar_index);
182 uint8_t type = (low >> 1) & 0x3;
183 if (type == 0x2) {
184 uint32_t high = pci_read(bus, slot, func, offset: 0x10 + 4 * (bar_index + 1));
185 return (((uint64_t) high) << 32) | (low & ~0xFULL);
186 } else {
187 return (uint64_t) (low & ~0xFULL);
188 }
189}
190
191void pci_program_msix_entry(uint8_t bus, uint8_t slot, uint8_t func,
192 uint32_t table_index, uint8_t vector,
193 uint32_t apic_id) {
194 uint8_t cap = pci_find_capability(bus, slot, func, PCI_CAP_ID_MSIX);
195 if (!cap) {
196 pci_log(LOG_ERROR, "MSI-X capability not found");
197 return;
198 }
199
200 uint32_t table_offset_bir = pci_read(bus, slot, func, offset: cap + 4);
201 uint8_t bir = table_offset_bir & 0x7;
202 uint32_t table_offset = table_offset_bir & ~0x7;
203
204 if (bir > 5) {
205 pci_log(LOG_ERROR, "MSIX BIR out of range");
206 return;
207 }
208
209 uint64_t bar_addr = pci_read_bar64(bus, slot, func, bar_index: bir);
210 if (bar_addr == 0) {
211 pci_log(LOG_ERROR, "PCI BAR%u is zero/unassigned", bir);
212 return;
213 }
214
215 size_t entry_size = sizeof(struct pci_msix_table_entry);
216 uint64_t table_base = bar_addr + table_offset; // physical
217 uint64_t entry_phys = table_base + (uint64_t) table_index * entry_size;
218
219 uint64_t map_base = PAGE_ALIGN_DOWN(entry_phys);
220 size_t map_size =
221 PAGE_ALIGN_UP((entry_phys & (PAGE_SIZE - 1)) + entry_size);
222
223 void cc_mem_io *map = mmio_map(phys: map_base, size: map_size);
224 if (!map) {
225 pci_log(LOG_ERROR, "vmm_map_phys failed for MSI-X table");
226 return;
227 }
228
229 struct pci_msix_table_entry cc_mem_io *entry =
230 (struct pci_msix_table_entry cc_mem_io *) ((uintptr_t) map +
231 (entry_phys &
232 (PAGE_SIZE - 1)));
233
234 uint64_t msg_addr =
235 0xFEE00000ull | ((uint64_t) lapic_dest_8bit(apic_id) << 12);
236 mmio_write_32(address: &entry->msg_addr_low, value: (uint32_t) (msg_addr & 0xFFFFFFFF));
237 mmio_write_32(address: &entry->msg_addr_high, value: (uint32_t) (msg_addr >> 32));
238 mmio_write_32(address: &entry->msg_data, value: (uint32_t) vector);
239
240 uint32_t ctrl = mmio_read_32(address: &entry->vector_ctrl);
241 ctrl &= ~1u;
242 mmio_write_32(address: &entry->vector_ctrl, value: ctrl);
243}
244
245void pci_enable_msix_on_core(uint8_t bus, uint8_t slot, uint8_t func,
246 uint8_t vector_index, uint32_t apic_id) {
247 uint8_t cap = pci_find_capability(bus, slot, func, PCI_CAP_ID_MSIX);
248 if (cap == 0) {
249 pci_log(LOG_ERROR, "MSI-X capability not found");
250 return;
251 }
252 uint32_t table_offset_bir = pci_read(bus, slot, func, offset: cap + 4);
253
254 uint8_t bir = table_offset_bir & 0x7;
255 uint32_t table_offset = table_offset_bir & ~0x7;
256
257 uint32_t bar_low = pci_read(bus, slot, func, offset: 0x10 + 4 * bir);
258 uint32_t bar_high = pci_read(bus, slot, func, offset: 0x10 + 4 * bir + 4);
259
260 uint64_t bar_addr = 0;
261
262 if (bir == 0) {
263 bar_addr = ((uint64_t) bar_high << 32) | (bar_low & ~0xFU);
264 } else if (bir == 1) {
265 pci_log(LOG_ERROR, "unsupported BIR");
266 }
267
268 uint64_t map_size =
269 (vector_index + 1) * sizeof(struct pci_msix_table_entry);
270 map_size = MAX(map_size, PAGE_SIZE);
271 void cc_mem_io *msix_table = mmio_map(phys: bar_addr + table_offset, size: map_size);
272
273 struct pci_msix_table_entry cc_mem_io *entry_addr =
274 (void cc_mem_io *) msix_table +
275 vector_index * sizeof(struct pci_msix_table_entry);
276
277 uint64_t msg_addr =
278 0xFEE00000ull | ((uint64_t) lapic_dest_8bit(apic_id) << 12);
279
280 mmio_write_32(address: &entry_addr->msg_addr_low, value: msg_addr);
281 mmio_write_32(address: &entry_addr->msg_addr_high, value: 0);
282 mmio_write_32(address: &entry_addr->msg_data, value: vector_index);
283
284 uint32_t vector_ctrl = mmio_read_32(address: &entry_addr->vector_ctrl);
285 vector_ctrl &= ~0x1;
286 mmio_write_32(address: &entry_addr->vector_ctrl, value: vector_ctrl);
287}
288
289void pci_enable_msix(uint8_t bus, uint8_t slot, uint8_t func) {
290 uint8_t cap_ptr = pci_read_byte(bus, slot, func, PCI_CAP_PTR);
291
292 while (cap_ptr != 0) {
293 uint8_t cap_id = pci_read_byte(bus, slot, func, offset: cap_ptr);
294 if (cap_id == PCI_CAP_ID_MSIX) {
295 uint16_t msg_ctl = pci_read_word(bus, slot, func, offset: cap_ptr + 2);
296
297 msg_ctl |= BIT(15);
298 msg_ctl &= ~BIT(14);
299
300 pci_write_word(bus, slot, func, offset: cap_ptr + 2, value: msg_ctl);
301
302 uint16_t verify = pci_read_word(bus, slot, func, offset: cap_ptr + 2);
303
304 if (BIT_TEST(verify, 15) && !BIT_TEST(verify, 14)) {
305 pci_log(LOG_INFO, "MSI-X enabled");
306 } else {
307 pci_log(LOG_ERROR, "Failed to enable MSI-X");
308 }
309 return;
310 }
311 cap_ptr = pci_read_byte(bus, slot, func, offset: cap_ptr + 1);
312 }
313 pci_log(LOG_ERROR, "MSI-X capability not found");
314}
315