1#include <asm.h>
2#include <console/printf.h>
3#include <drivers/iommu/vt_d.h>
4#include <drivers/mmio.h>
5#include <log.h>
6#include <mem/page.h>
7#include <mem/pmm.h>
8#include <mem/vmm.h>
9#include <string.h>
10
11LOG_HANDLE_DECLARE_DEFAULT(vtd);
12LOG_SITE_DECLARE_DEFAULT(vtd);
13
14const struct iommu_ops vtd_iommu_ops = {};
15
16void vtd_write_gcmd(struct vtd_unit *u, uint32_t cmd) {
17 uint32_t status = mmio_read_32(address: &u->regs->global_status);
18
19 uint32_t preserved =
20 status & (GSTS_TRANSLATION_ENABLED | GSTS_ROOT_TABLE_PTR_SET |
21 GSTS_QUEUED_INVAL_ENABLED | GSTS_INTERRUPT_REMAP_ENABLED |
22 GSTS_COMPAT_FORMAT_INTERRUPT_STATUS);
23
24 mmio_write_32(address: &u->regs->global_command, value: preserved | cmd);
25}
26
27void vtd_wait_gsts(struct vtd_unit *u, uint32_t bit, bool set) {
28 while (1) {
29 uint32_t s = mmio_read_32(address: &u->regs->global_status);
30 if (set && (s & bit))
31 break;
32
33 if (!set && !(s & bit))
34 break;
35
36 cpu_relax(); /* pause */
37 }
38}
39
40enum iommu_error vtd_iq_init(struct vtd_unit *u) {
41 if (!ECAP_QUEUED_INVALIDATION(u->ecap)) {
42 vtd_warn("QI not supported by this unit, cannot continue");
43 return IOMMU_ERR_UNSUPPORTED;
44 }
45
46 if (mmio_read_32(address: &u->regs->global_status) & GSTS_QUEUED_INVAL_ENABLED) {
47 vtd_info("QI already enabled, disabling before reset");
48 vtd_write_gcmd(u, cmd: mmio_read_32(address: &u->regs->global_status) &
49 ~GCMD_QUEUED_INVAL_ENABLE);
50 vtd_wait_gsts(u, GSTS_QUEUED_INVAL_ENABLED, false);
51 }
52
53 mmio_write_64(address: &u->regs->invalidation_queue_tail, value: 0);
54 mmio_write_64(address: &u->regs->invalidation_queue_head, value: 0);
55
56 u->iq_size = 256;
57 paddr_t phys = pmm_alloc_page();
58 if (!phys)
59 return IOMMU_ERR_NO_MEM;
60
61 u->iq_phys = phys;
62 u->iq_base = mmio_map(phys, PAGE_SIZE);
63 u->iq_head = 0;
64 u->iq_tail = 0;
65 memset(u->iq_base, 0, PAGE_SIZE);
66
67 mmio_write_64(address: &u->regs->invalidation_queue_addr,
68 IQA_REG_BUILD(phys, IQA_SIZE_256_ENTRIES));
69
70 vtd_write_gcmd(u, GCMD_QUEUED_INVAL_ENABLE);
71 vtd_wait_gsts(u, GSTS_QUEUED_INVAL_ENABLED, true);
72
73 vtd_info("IQ initialized: phys=0x%llx entries=%u", phys, u->iq_size);
74 return IOMMU_ERR_OK;
75}
76
77void vtd_iq_submit(struct vtd_unit *u, struct vtd_inv_desc desc) {
78 struct vtd_inv_desc *ring = u->iq_base;
79 mmio_write_64(address: &ring[u->iq_tail].hi, value: desc.hi);
80 mmio_write_64(address: &ring[u->iq_tail].lo, value: desc.lo);
81
82 u->iq_tail = (u->iq_tail + 1) % u->iq_size;
83 mmio_write_64(address: &u->regs->invalidation_queue_tail, value: u->iq_tail << 4);
84}
85
86void vtd_iq_flush(struct vtd_unit *u) {
87 static volatile uint32_t wait_token __attribute__((aligned(4)));
88 paddr_t token_phys = vmm_get_phys((vaddr_t) &wait_token, VMM_FLAG_NONE);
89
90 wait_token = 0;
91
92 vtd_iq_submit(u, WAIT_DESC(token_phys, 1));
93
94 while (wait_token != 1)
95 cpu_relax();
96}
97
98enum iommu_error vtd_root_table_init(struct vtd_unit *u) {
99 paddr_t phys = pmm_alloc_page();
100 if (!phys)
101 return IOMMU_ERR_NO_MEM;
102
103 u->root_table_phys = phys;
104 u->root_table = mmio_map(phys, PAGE_SIZE);
105 memset(u->root_table, 0, PAGE_SIZE);
106
107 mmio_write_64(address: &u->regs->root_table_addr,
108 value: phys | RTADDR_TRANSLATION_MODE_LEGACY);
109
110 vtd_write_gcmd(u, GCMD_SET_ROOT_TABLE_PTR);
111 vtd_wait_gsts(u, GSTS_ROOT_TABLE_PTR_SET, true);
112
113 vtd_iq_submit(u, CTX_INVAL_DESC_GLOBAL);
114 vtd_iq_submit(u, IOTLB_INVAL_DESC_GLOBAL);
115 vtd_iq_flush(u);
116
117 vtd_info("root table initialized: phys=0x%llx", phys);
118 return IOMMU_ERR_OK;
119}
120
121/*
122 * vtd_build_identity_sl
123 *
124 * Build minimal second-level page table that identity-maps the full
125 * 512 gb low address space using 1 gb hugepages
126 *
127 * One PML4 entry covers all 512 PDPT slots = 512 GiB
128 */
129static paddr_t vtd_build_identity_sl(struct vtd_unit *u) {
130 (void) u;
131
132 paddr_t pml4_phys = pmm_alloc_page();
133 if (!pml4_phys)
134 return 0;
135 uint64_t *pml4 = mmio_map(phys: pml4_phys, PAGE_SIZE);
136 memset(pml4, 0, PAGE_SIZE);
137
138 paddr_t pdpt_phys = pmm_alloc_page();
139 if (!pdpt_phys)
140 return 0;
141 uint64_t *pdpt = mmio_map(phys: pdpt_phys, PAGE_SIZE);
142
143 for (int i = 0; i < 512; i++) {
144 pdpt[i] = ((uint64_t) i << 30) | SL_PTE_LARGE_PAGE |
145 SL_PTE_MEMORY_TYPE_WRITE_BACK | SL_PTE_IGNORE_PAT |
146 SL_PTE_READ | SL_PTE_WRITE;
147 }
148
149 /* SL_TABLE_ENTRY sets the physical address + PRESENT (R|W) */
150 pml4[0] = SL_TABLE_ENTRY(pdpt_phys);
151
152 return pml4_phys;
153}
154
155/*
156 * Populate every root+context entry so no BDF produces a translation
157 * fault after GCMD_TRANSLATION_ENABLE is set
158 *
159 * All devices are placed in domain 1 and
160 * pointed at the shared identity SL table built above
161 */
162static enum iommu_error vtd_passthrough_all_devices(struct vtd_unit *u) {
163 paddr_t sl_phys = vtd_build_identity_sl(u);
164 if (!sl_phys)
165 return IOMMU_ERR_NO_MEM;
166
167 const uint16_t domain_id = 1;
168
169 uint64_t ctx_lo =
170 CTX_ENTRY_SET_LO(sl_phys, CTX_ENTRY_TRANSLATION_TYPE_UNTRANSLATED);
171 uint64_t ctx_hi = CTX_ENTRY_SET_HI(domain_id, CTX_ENTRY_ADDR_WIDTH_48BIT);
172
173 struct vtd_root_entry *root = (struct vtd_root_entry *) u->root_table;
174
175 for (int bus = 0; bus < 256; bus++) {
176 if (!(root[bus].lo & ROOT_ENTRY_PRESENT)) {
177 paddr_t ctx_phys = pmm_alloc_page();
178 if (!ctx_phys)
179 return IOMMU_ERR_NO_MEM;
180
181 void *ctx_virt = mmio_map(phys: ctx_phys, PAGE_SIZE);
182 memset(ctx_virt, 0, PAGE_SIZE);
183
184 root[bus].lo = ROOT_ENTRY_SET_CONTEXT_TABLE_PTR(ctx_phys);
185 root[bus].hi = 0;
186 }
187
188 paddr_t ctx_phys = root[bus].lo & ROOT_ENTRY_CONTEXT_TABLE_PTR_MASK;
189 struct vtd_context_entry *ctx = mmio_map(phys: ctx_phys, PAGE_SIZE);
190
191 for (int df = 0; df < 256; df++) {
192 ctx[df].lo = ctx_lo;
193 ctx[df].hi = ctx_hi;
194 }
195 }
196
197 vtd_iq_submit(u, CTX_INVAL_DESC_GLOBAL);
198 vtd_iq_submit(u, IOTLB_INVAL_DESC_GLOBAL);
199 vtd_iq_flush(u);
200
201 vtd_info("passthrough identity domain installed for all devices (dom=%u "
202 "sl=0x%llx)",
203 domain_id, sl_phys);
204 return IOMMU_ERR_OK;
205}
206
207enum iommu_error vtd_unit_init(struct iommu *unit) {
208 struct vtd_unit *u = unit->private;
209
210 vtd_info("initializing VT-d unit seg=%u regs=%p", u->segment, u->regs);
211 vtd_info(" CAP: SAGAW=%02x MGAW=%u CM=%u RWBF=%u domains=%u",
212 CAP_SUPPORTED_ADDR_WIDTHS(u->cap), CAP_MAX_ADDR_WIDTH(u->cap),
213 CAP_CACHING_MODE(u->cap), CAP_REQUIRES_WRITE_BUF_FLUSH(u->cap),
214 vtd_cap_domain_count(u->cap));
215 vtd_info(" ECAP: QI=%u IR=%u PT=%u SC=%u C=%u",
216 ECAP_QUEUED_INVALIDATION(u->ecap),
217 ECAP_INTERRUPT_REMAPPING(u->ecap), ECAP_PASS_THROUGH(u->ecap),
218 ECAP_SNOOP_CONTROL(u->ecap), ECAP_COHERENCY(u->ecap));
219
220 if (!(CAP_SUPPORTED_ADDR_WIDTHS(u->cap) & ADDR_WIDTH_48BIT)) {
221 vtd_warn("4-level page tables not supported, aborting");
222 return IOMMU_ERR_UNSUPPORTED;
223 }
224
225 if (CAP_REQUIRES_WRITE_BUF_FLUSH(u->cap)) {
226 vtd_write_gcmd(u, GCMD_WRITE_BUFFER_FLUSH);
227 vtd_wait_gsts(u, GSTS_WRITE_BUFFER_FLUSH_STATUS, false);
228 }
229
230 enum iommu_error err;
231
232 err = vtd_iq_init(u);
233 if (err != IOMMU_ERR_OK)
234 return err;
235
236 err = vtd_root_table_init(u);
237 if (err != IOMMU_ERR_OK)
238 return err;
239
240 err = vtd_passthrough_all_devices(u);
241 if (err != IOMMU_ERR_OK)
242 return err;
243
244 vtd_write_gcmd(u, GCMD_TRANSLATION_ENABLE);
245 vtd_wait_gsts(u, GSTS_TRANSLATION_ENABLED, true);
246
247 vtd_info("translation enabled");
248 unit->status = IOMMU_STATUS_ACTIVE;
249 return IOMMU_ERR_OK;
250}
251
252struct iommu *vtd_unit_create(uint64_t base_phys, uint16_t segment,
253 uint8_t size_field) {
254 struct iommu *unit = kmalloc(sizeof(*unit));
255 if (!unit)
256 return NULL;
257
258 struct vtd_unit *u = kmalloc(sizeof(*u));
259 if (!u) {
260 kfree(unit);
261 return NULL;
262 }
263
264 memset(unit, 0, sizeof(*unit));
265 memset(u, 0, sizeof(*u));
266
267 u->segment = segment;
268 u->regs = mmio_map(phys: base_phys, size: 1u << (size_field + 12));
269 u->cap = u->regs->capabilities;
270 u->ecap = u->regs->extended_capabilities;
271 u->domain_count = vtd_cap_domain_count(cap: u->cap);
272
273 unit->ops = &vtd_iommu_ops;
274 unit->private = u;
275 unit->status = IOMMU_STATUS_INACTIVE;
276
277 vtd_unit_init(unit);
278 return unit;
279}
280