1#include <console/printf.h>
2#include <drivers/mmio.h>
3#include <drivers/pci.h>
4#include <drivers/usb/xhci.h>
5#include <log.h>
6#include <mem/alloc.h>
7#include <mem/page.h>
8#include <mem/vmm.h>
9#include <stddef.h>
10#include <stdint.h>
11
12static struct pci_device *pci_devices = NULL;
13static uint64_t pci_device_count;
14LOG_HANDLE_DECLARE_DEFAULT(pci);
15LOG_SITE_DECLARE_DEFAULT(pci);
16
17const char *pci_class_name(uint8_t class_code, uint8_t subclass) {
18 switch (class_code) {
19 case 0x01:
20 switch (subclass) {
21 case 0x00: return "SCSI Controller";
22 case 0x01: return "IDE Controller";
23 case 0x02: return "Floppy Controller";
24 case 0x03: return "IPI Bus Controller";
25 case 0x04: return "RAID Controller";
26 case 0x05: return "ATA Controller";
27 case 0x06: return "SATA Controller (AHCI)";
28 case 0x08: return "NVMe Controller";
29 default: return "Other Mass Storage Controller";
30 }
31 case 0x02: return "Network Controller";
32 case 0x03: return "Display Controller";
33 case 0x04: return "Multimedia Controller";
34 case 0x06: return "Bridge Device";
35 case 0x0C:
36 if (subclass == 0x03)
37 return "USB Controller";
38 break;
39 }
40 return "Unknown Device";
41}
42
43static void init_device(struct pci_device *dev) {
44 struct pci_driver *start = __skernel_pci_devices;
45 struct pci_driver *end = __ekernel_pci_devices;
46
47 for (struct pci_driver *d = start; d < end; d++) {
48 bool class, subclass, prog_if, vendor;
49
50 class = dev->class_code == d->class_code;
51 subclass = dev->subclass == d->subclass;
52 prog_if = d->prog_if == 0xFF ? true : dev->prog_if == d->prog_if;
53 vendor = d->vendor_id == 0xFFFF ? true : dev->vendor_id == d->vendor_id;
54 dev->device.driver_data = dev;
55
56 if (class && subclass && prog_if && vendor) {
57 d->driver.probe(&dev->device);
58 }
59 }
60}
61
62void pci_init_devices(struct pci_device *devices, uint64_t count) {
63 struct pci_driver *start = __skernel_pci_devices;
64 struct pci_driver *end = __ekernel_pci_devices;
65
66 pci_log(LOG_INFO, "There are %u PCI drivers", end - start);
67
68 for (uint64_t i = 0; i < count; i++) {
69 struct pci_device *dev = &devices[i];
70 init_device(dev);
71 }
72}
73
74void pci_scan_devices(struct pci_device **devices_out, uint64_t *count_out) {
75 pci_device_count = 0;
76
77 uint64_t space_to_alloc = 0;
78
79 for (uint16_t bus = 0; bus < 256; ++bus) {
80 for (uint8_t device = 0; device < 32; ++device) {
81 for (uint8_t function = 0; function < 8; ++function) {
82 uint16_t vendor = pci_read_word(bus, slot: device, func: function, offset: 0x00);
83 if (vendor == 0xFFFF)
84 continue;
85
86 space_to_alloc++;
87
88 /* TODO: Do not blindly enable all PCI devices */
89 union pci_command_reg cmd;
90 cmd.value = pci_read_config16(bus, device, function, offset: 0x04);
91
92 cmd.bus_master = 1;
93 cmd.memory_space = 1;
94
95 pci_write_config16(bus, device, function, offset: 0x04, value: cmd.value);
96
97 if (function == 0) {
98 uint8_t header_type =
99 pci_read_byte(bus, slot: device, func: function, offset: 0x0E);
100 if ((header_type & 0x80) == 0)
101 break;
102 }
103 }
104 }
105 }
106
107 pci_devices = kmalloc(space_to_alloc * sizeof(struct pci_device));
108 if (!pci_devices)
109 panic("Could not allocate space for PCI devices");
110
111 for (uint16_t bus = 0; bus < 256; ++bus) {
112 for (uint8_t device = 0; device < 32; ++device) {
113 for (uint8_t function = 0; function < 8; ++function) {
114 uint16_t vendor_id = pci_read_word(bus, slot: device, func: function, offset: 0x00);
115 if (vendor_id == 0xFFFF)
116 continue;
117
118 uint16_t device_id = pci_read_word(bus, slot: device, func: function, offset: 0x02);
119 uint32_t class_info = pci_read(bus, slot: device, func: function, offset: 0x08);
120 uint8_t class_code = (class_info >> 24) & 0xFF;
121 uint8_t subclass = (class_info >> 16) & 0xFF;
122 uint8_t prog_if = (class_info >> 8) & 0xFF;
123 uint8_t revision = class_info & 0xFF;
124
125 pci_devices[pci_device_count++] =
126 (struct pci_device){.bus = bus,
127 .dev = device,
128 .function = function,
129 .vendor_id = vendor_id,
130 .device_id = device_id,
131 .class_code = class_code,
132 .subclass = subclass,
133 .prog_if = prog_if,
134 .revision = revision};
135
136 pci_log(LOG_INFO, "Found device '%s' at %02x:%02x.%x",
137 pci_class_name(class_code, subclass), bus, device,
138 function);
139
140 if (function == 0) {
141 uint8_t header_type =
142 pci_read_byte(bus, slot: device, func: function, offset: 0x0E);
143 if ((header_type & 0x80) == 0)
144 break;
145 }
146 }
147 }
148 }
149
150 *devices_out = pci_devices;
151 *count_out = pci_device_count;
152}
153
154uint8_t pci_find_capability(uint8_t bus, uint8_t slot, uint8_t func,
155 uint8_t cap_id) {
156 uint8_t cap_ptr = pci_read_byte(bus, slot, func, PCI_CAP_PTR);
157
158 while (cap_ptr != 0 && cap_ptr != 0xFF) {
159 uint8_t current_id = pci_read_byte(bus, slot, func, offset: cap_ptr);
160 if (current_id == cap_id) {
161 return cap_ptr;
162 }
163 cap_ptr = pci_read_byte(bus, slot, func, offset: cap_ptr + 1);
164 }
165
166 return 0;
167}
168
169uint32_t pci_read_bar(uint8_t bus, uint8_t device, uint8_t function,
170 uint8_t bar_index) {
171 uint8_t offset = 0x10 + (bar_index * 4);
172 return pci_read(bus, slot: device, func: function, offset);
173}
174
175static uint64_t pci_read_bar64(uint8_t bus, uint8_t slot, uint8_t func,
176 uint8_t bar_index) {
177 uint32_t low = pci_read(bus, slot, func, offset: 0x10 + 4 * bar_index);
178 uint8_t type = (low >> 1) & 0x3;
179 if (type == 0x2) {
180 uint32_t high = pci_read(bus, slot, func, offset: 0x10 + 4 * (bar_index + 1));
181 return (((uint64_t) high) << 32) | (low & ~0xFULL);
182 } else {
183 return (uint64_t) (low & ~0xFULL);
184 }
185}
186
187void pci_program_msix_entry(uint8_t bus, uint8_t slot, uint8_t func,
188 uint32_t table_index, uint8_t vector,
189 uint8_t apic_id) {
190 uint8_t cap = pci_find_capability(bus, slot, func, PCI_CAP_ID_MSIX);
191 if (!cap) {
192 pci_log(LOG_ERROR, "MSI-X capability not found");
193 return;
194 }
195
196 uint32_t table_offset_bir = pci_read(bus, slot, func, offset: cap + 4);
197 uint8_t bir = table_offset_bir & 0x7;
198 uint32_t table_offset = table_offset_bir & ~0x7;
199
200 if (bir > 5) {
201 pci_log(LOG_ERROR, "MSIX BIR out of range");
202 return;
203 }
204
205 uint64_t bar_addr = pci_read_bar64(bus, slot, func, bar_index: bir);
206 if (bar_addr == 0) {
207 pci_log(LOG_ERROR, "PCI BAR%u is zero/unassigned", bir);
208 return;
209 }
210
211 size_t entry_size = sizeof(struct pci_msix_table_entry);
212 uint64_t table_base = bar_addr + table_offset; // physical
213 uint64_t entry_phys = table_base + (uint64_t) table_index * entry_size;
214
215 uint64_t map_base = entry_phys & ~(PAGE_SIZE - 1);
216 size_t map_size = (entry_phys & (PAGE_SIZE - 1)) + entry_size;
217 map_size = (map_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
218
219 void *map = mmio_map(phys: map_base, size: map_size);
220 if (!map) {
221 pci_log(LOG_ERROR, "vmm_map_phys failed for MSI-X table");
222 return;
223 }
224
225 struct pci_msix_table_entry *entry =
226 (struct pci_msix_table_entry *) ((uintptr_t) map +
227 (entry_phys & (PAGE_SIZE - 1)));
228
229 uint64_t msg_addr = 0xFEE00000ull | ((uint64_t) apic_id << 12);
230 mmio_write_32(address: &entry->msg_addr_low, value: (uint32_t) (msg_addr & 0xFFFFFFFF));
231 mmio_write_32(address: &entry->msg_addr_high, value: (uint32_t) (msg_addr >> 32));
232 mmio_write_32(address: &entry->msg_data, value: (uint32_t) vector);
233
234 uint32_t ctrl = mmio_read_32(address: &entry->vector_ctrl);
235 ctrl &= ~1u;
236 mmio_write_32(address: &entry->vector_ctrl, value: ctrl);
237}
238
239void pci_enable_msix_on_core(uint8_t bus, uint8_t slot, uint8_t func,
240 uint8_t vector_index, uint8_t apic_id) {
241 uint8_t cap = pci_find_capability(bus, slot, func, PCI_CAP_ID_MSIX);
242 if (cap == 0) {
243 pci_log(LOG_ERROR, "MSI-X capability not found");
244 return;
245 }
246 uint32_t table_offset_bir = pci_read(bus, slot, func, offset: cap + 4);
247
248 uint8_t bir = table_offset_bir & 0x7;
249 uint32_t table_offset = table_offset_bir & ~0x7;
250
251 uint32_t bar_low = pci_read(bus, slot, func, offset: 0x10 + 4 * bir);
252 uint32_t bar_high = pci_read(bus, slot, func, offset: 0x10 + 4 * bir + 4);
253
254 uint64_t bar_addr = 0;
255
256 if (bir == 0) {
257 bar_addr = ((uint64_t) bar_high << 32) | (bar_low & ~0xFU);
258 } else if (bir == 1) {
259 pci_log(LOG_ERROR, "unsupported BIR");
260 }
261
262 uint64_t map_size =
263 (vector_index + 1) * sizeof(struct pci_msix_table_entry);
264 if (map_size < PAGE_SIZE) {
265 map_size = PAGE_SIZE;
266 }
267 void *msix_table = mmio_map(phys: bar_addr + table_offset, size: map_size);
268
269 struct pci_msix_table_entry *entry_addr =
270 (void *) msix_table +
271 vector_index * sizeof(struct pci_msix_table_entry);
272
273 uint64_t msg_addr = 0xFEE00000 | (apic_id << 12);
274
275 mmio_write_32(address: &entry_addr->msg_addr_low, value: msg_addr);
276 mmio_write_32(address: &entry_addr->msg_addr_high, value: 0);
277 mmio_write_32(address: &entry_addr->msg_data, value: vector_index);
278
279 uint32_t vector_ctrl = mmio_read_32(address: &entry_addr->vector_ctrl);
280 vector_ctrl &= ~0x1;
281 mmio_write_32(address: &entry_addr->vector_ctrl, value: vector_ctrl);
282}
283
284void pci_enable_msix(uint8_t bus, uint8_t slot, uint8_t func) {
285 uint8_t cap_ptr = pci_read_byte(bus, slot, func, PCI_CAP_PTR);
286
287 while (cap_ptr != 0) {
288 uint8_t cap_id = pci_read_byte(bus, slot, func, offset: cap_ptr);
289 if (cap_id == PCI_CAP_ID_MSIX) {
290 uint16_t msg_ctl = pci_read_word(bus, slot, func, offset: cap_ptr + 2);
291
292 msg_ctl |= (1 << 15);
293 msg_ctl &= ~(1 << 14);
294
295 pci_write_word(bus, slot, func, offset: cap_ptr + 2, value: msg_ctl);
296
297 uint16_t verify = pci_read_word(bus, slot, func, offset: cap_ptr + 2);
298
299 if ((verify & (1 << 15)) && !(verify & (1 << 14))) {
300 pci_log(LOG_INFO, "MSI-X enabled");
301 } else {
302 pci_log(LOG_ERROR, "Failed to enable MSI-X");
303 }
304 return;
305 }
306 cap_ptr = pci_read_byte(bus, slot, func, offset: cap_ptr + 1);
307 }
308 pci_log(LOG_ERROR, "MSI-X capability not found");
309}
310