1#include <asm.h>
2#include <console/printf.h>
3#include <drivers/iommu/vt_d.h>
4#include <drivers/mmio.h>
5#include <log.h>
6#include <mem/page.h>
7#include <mem/pmm.h>
8#include <mem/vmm.h>
9#include <string.h>
10
11#include "internal.h"
12
13LOG_HANDLE_DEFINE_PRINT(vtd);
14LOG_SITE_DEFINE_PRINT(vtd);
15
16const struct iommu_ops vtd_iommu_ops = {};
17
18void vtd_write_gcmd(struct vtd_unit *u, uint32_t cmd) {
19 uint32_t status = mmio_read_32(address: &u->regs->global_status);
20
21 uint32_t preserved =
22 status & (GSTS_TRANSLATION_ENABLED | GSTS_ROOT_TABLE_PTR_SET |
23 GSTS_QUEUED_INVAL_ENABLED | GSTS_INTERRUPT_REMAP_ENABLED |
24 GSTS_COMPAT_FORMAT_INTERRUPT_STATUS);
25
26 mmio_write_32(address: &u->regs->global_command, value: preserved | cmd);
27}
28
29void vtd_wait_gsts(struct vtd_unit *u, uint32_t bit, bool set) {
30 while (true) {
31 uint32_t s = mmio_read_32(address: &u->regs->global_status);
32 if (set && (s & bit))
33 break;
34
35 if (!set && !(s & bit))
36 break;
37
38 cpu_pause(); /* pause */
39 }
40}
41
42enum iommu_error vtd_iq_init(struct vtd_unit *u) {
43 if (!ECAP_QUEUED_INVALIDATION(u->ecap)) {
44 vtd_warn("QI not supported by this unit, cannot continue");
45 return IOMMU_ERR_UNSUPPORTED;
46 }
47
48 if (mmio_read_32(address: &u->regs->global_status) & GSTS_QUEUED_INVAL_ENABLED) {
49 vtd_info("QI already enabled, disabling before reset");
50 vtd_write_gcmd(u, cmd: mmio_read_32(address: &u->regs->global_status) &
51 ~GCMD_QUEUED_INVAL_ENABLE);
52 vtd_wait_gsts(u, GSTS_QUEUED_INVAL_ENABLED, false);
53 }
54
55 mmio_write_64(address: &u->regs->invalidation_queue_tail, value: 0);
56 mmio_write_64(address: &u->regs->invalidation_queue_head, value: 0);
57
58 u->iq_size = 256;
59 paddr_t phys = pmm_alloc_page();
60 if (!phys)
61 return IOMMU_ERR_NO_MEM;
62
63 u->iq_phys = phys;
64 u->iq_base = mmio_map_dma(phys, PAGE_SIZE);
65 u->iq_head = 0;
66 u->iq_tail = 0;
67 memset(u->iq_base, 0, PAGE_SIZE);
68
69 mmio_write_64(address: &u->regs->invalidation_queue_addr,
70 IQA_REG_BUILD(phys, IQA_SIZE_256_ENTRIES));
71
72 vtd_write_gcmd(u, GCMD_QUEUED_INVAL_ENABLE);
73 vtd_wait_gsts(u, GSTS_QUEUED_INVAL_ENABLED, true);
74
75 vtd_info("IQ initialized: phys=0x%lx entries=%u", phys, u->iq_size);
76 return IOMMU_ERR_OK;
77}
78
79void vtd_iq_submit(struct vtd_unit *u, struct vtd_inv_desc desc) {
80 struct vtd_inv_desc *ring = u->iq_base;
81 mmio_write_64(address: (uint64_t cc_mem_io *) &ring[u->iq_tail].hi, value: desc.hi);
82 mmio_write_64(address: (uint64_t cc_mem_io *) &ring[u->iq_tail].lo, value: desc.lo);
83
84 u->iq_tail = (u->iq_tail + 1) % u->iq_size;
85 mmio_write_64(address: &u->regs->invalidation_queue_tail, value: u->iq_tail << 4);
86}
87
88void vtd_iq_flush(struct vtd_unit *u) {
89 static volatile uint32_t wait_token cc_aligned(4);
90 paddr_t token_phys = vmm_get_phys((vaddr_t) &wait_token);
91
92 wait_token = 0;
93
94 vtd_iq_submit(u, WAIT_DESC(token_phys, 1));
95
96 while (wait_token != 1)
97 cpu_pause();
98}
99
100enum iommu_error vtd_root_table_init(struct vtd_unit *u) {
101 paddr_t phys = pmm_alloc_page();
102 if (!phys)
103 return IOMMU_ERR_NO_MEM;
104
105 u->root_table_phys = phys;
106 u->root_table = mmio_map_dma(phys, PAGE_SIZE);
107 memset(u->root_table, 0, PAGE_SIZE);
108
109 mmio_write_64(address: &u->regs->root_table_addr,
110 value: phys | RTADDR_TRANSLATION_MODE_LEGACY);
111
112 vtd_write_gcmd(u, GCMD_SET_ROOT_TABLE_PTR);
113 vtd_wait_gsts(u, GSTS_ROOT_TABLE_PTR_SET, true);
114
115 vtd_iq_submit(u, CTX_INVAL_DESC_GLOBAL);
116 vtd_iq_submit(u, IOTLB_INVAL_DESC_GLOBAL);
117 vtd_iq_flush(u);
118
119 vtd_info("root table initialized: phys=0x%lx", phys);
120 return IOMMU_ERR_OK;
121}
122
123/*
124 * vtd_build_identity_sl
125 *
126 * Build minimal second-level page table that identity-maps the full
127 * 512 gb low address space using 1 gb hugepages
128 *
129 * One PML4 entry covers all 512 PDPT slots = 512 GiB
130 */
131static paddr_t vtd_build_identity_sl(struct vtd_unit *u) {
132 cc_ignore(u);
133
134 paddr_t pml4_phys = pmm_alloc_page();
135 if (!pml4_phys)
136 return 0;
137 uint64_t *pml4 = mmio_map_dma(phys: pml4_phys, PAGE_SIZE);
138 memset(pml4, 0, PAGE_SIZE);
139
140 paddr_t pdpt_phys = pmm_alloc_page();
141 if (!pdpt_phys)
142 return 0;
143 uint64_t *pdpt = mmio_map_dma(phys: pdpt_phys, PAGE_SIZE);
144
145 for (int i = 0; i < 512; i++) {
146 pdpt[i] = ((uint64_t) i << 30) | SL_PTE_LARGE_PAGE |
147 SL_PTE_MEMORY_TYPE_WRITE_BACK | SL_PTE_IGNORE_PAT |
148 SL_PTE_READ | SL_PTE_WRITE;
149 }
150
151 /* SL_TABLE_ENTRY sets the physical address + PRESENT (R|W) */
152 pml4[0] = SL_TABLE_ENTRY(pdpt_phys);
153
154 return pml4_phys;
155}
156
157/*
158 * Populate every root+context entry so no BDF produces a translation
159 * fault after GCMD_TRANSLATION_ENABLE is set
160 *
161 * All devices are placed in domain 1 and
162 * pointed at the shared identity SL table built above
163 */
164static enum iommu_error vtd_passthrough_all_devices(struct vtd_unit *u) {
165 paddr_t sl_phys = vtd_build_identity_sl(u);
166 if (!sl_phys)
167 return IOMMU_ERR_NO_MEM;
168
169 const uint16_t domain_id = 1;
170
171 uint64_t ctx_lo =
172 CTX_ENTRY_SET_LO(sl_phys, CTX_ENTRY_TRANSLATION_TYPE_UNTRANSLATED);
173 uint64_t ctx_hi = CTX_ENTRY_SET_HI(domain_id, CTX_ENTRY_ADDR_WIDTH_48BIT);
174
175 struct vtd_root_entry *root = (struct vtd_root_entry *) u->root_table;
176
177 for (int bus = 0; bus < 256; bus++) {
178 if (!(root[bus].lo & ROOT_ENTRY_PRESENT)) {
179 paddr_t ctx_phys = pmm_alloc_page();
180 if (!ctx_phys)
181 return IOMMU_ERR_NO_MEM;
182
183 void *ctx_virt = mmio_map_dma(phys: ctx_phys, PAGE_SIZE);
184 memset(ctx_virt, 0, PAGE_SIZE);
185
186 root[bus].lo = ROOT_ENTRY_SET_CONTEXT_TABLE_PTR(ctx_phys);
187 root[bus].hi = 0;
188 }
189
190 paddr_t ctx_phys = root[bus].lo & ROOT_ENTRY_CONTEXT_TABLE_PTR_MASK;
191 struct vtd_context_entry *ctx = mmio_map_dma(phys: ctx_phys, PAGE_SIZE);
192
193 for (int df = 0; df < 256; df++) {
194 ctx[df].lo = ctx_lo;
195 ctx[df].hi = ctx_hi;
196 }
197 }
198
199 vtd_iq_submit(u, CTX_INVAL_DESC_GLOBAL);
200 vtd_iq_submit(u, IOTLB_INVAL_DESC_GLOBAL);
201 vtd_iq_flush(u);
202
203 vtd_info("passthrough identity domain installed for all devices (dom=%u "
204 "sl=0x%lx)",
205 domain_id, sl_phys);
206 return IOMMU_ERR_OK;
207}
208
209static enum iommu_error vtd_unit_init(struct iommu *unit) {
210 struct vtd_unit *u = unit->private;
211
212 vtd_info("initializing VT-d unit seg=%u regs=%p", u->segment, u->regs);
213 vtd_info(" CAP: SAGAW=%02lx MGAW=%lu CM=%lu RWBF=%lu domains=%u",
214 CAP_SUPPORTED_ADDR_WIDTHS(u->cap), CAP_MAX_ADDR_WIDTH(u->cap),
215 CAP_CACHING_MODE(u->cap), CAP_REQUIRES_WRITE_BUF_FLUSH(u->cap),
216 vtd_cap_domain_count(u->cap));
217 vtd_info(" ECAP: QI=%lu IR=%lu PT=%lu SC=%lu C=%lu",
218 ECAP_QUEUED_INVALIDATION(u->ecap),
219 ECAP_INTERRUPT_REMAPPING(u->ecap), ECAP_PASS_THROUGH(u->ecap),
220 ECAP_SNOOP_CONTROL(u->ecap), ECAP_COHERENCY(u->ecap));
221
222 if (!(CAP_SUPPORTED_ADDR_WIDTHS(u->cap) & ADDR_WIDTH_48BIT)) {
223 vtd_warn("4-level page tables not supported, aborting");
224 return IOMMU_ERR_UNSUPPORTED;
225 }
226
227 if (CAP_REQUIRES_WRITE_BUF_FLUSH(u->cap)) {
228 vtd_write_gcmd(u, GCMD_WRITE_BUFFER_FLUSH);
229 vtd_wait_gsts(u, GSTS_WRITE_BUFFER_FLUSH_STATUS, false);
230 }
231
232 enum iommu_error err;
233
234 err = vtd_iq_init(u);
235 if (err != IOMMU_ERR_OK)
236 return err;
237
238 err = vtd_root_table_init(u);
239 if (err != IOMMU_ERR_OK)
240 return err;
241
242 err = vtd_passthrough_all_devices(u);
243 if (err != IOMMU_ERR_OK)
244 return err;
245
246 vtd_write_gcmd(u, GCMD_TRANSLATION_ENABLE);
247 vtd_wait_gsts(u, GSTS_TRANSLATION_ENABLED, true);
248
249 vtd_info("translation enabled");
250 unit->status = IOMMU_STATUS_ACTIVE;
251 return IOMMU_ERR_OK;
252}
253
254struct iommu *vtd_unit_create(uint64_t base_phys, uint16_t segment,
255 uint8_t size_field) {
256 struct iommu *unit = vnew(unit);
257 if (!unit)
258 return NULL;
259
260 struct vtd_unit *u = vnew(u);
261 if (!u) {
262 vfree(unit);
263 return NULL;
264 }
265
266 memset(unit, 0, sizeof(*unit));
267 memset(u, 0, sizeof(*u));
268
269 u->segment = segment;
270 u->regs = mmio_map(phys: base_phys, size: 1u << (size_field + 12));
271 u->cap = u->regs->capabilities;
272 u->ecap = u->regs->extended_capabilities;
273 u->domain_count = vtd_cap_domain_count(cap: u->cap);
274
275 unit->ops = &vtd_iommu_ops;
276 unit->private = u;
277 unit->status = IOMMU_STATUS_INACTIVE;
278
279 vtd_unit_init(unit);
280 return unit;
281}
282