1#include <asm.h>
2#include <console/printf.h>
3#include <drivers/iommu/vt_d.h>
4#include <drivers/mmio.h>
5#include <log.h>
6#include <mem/hhdm.h>
7#include <mem/page.h>
8#include <mem/pmm.h>
9#include <mem/vmm.h>
10#include <string.h>
11
12#include "internal.h"
13
14static bool vtd_pt_empty(uint64_t *pt) {
15 for (size_t i = 0; i < SL_ENTRY_COUNT; i++) {
16 if (pt[i] & SL_PTE_READ)
17 return false;
18 }
19 return true;
20}
21
22static uint64_t *vtd_sl_ensure_child(sl_pte_atomic_t *entry) {
23 uint64_t val = atomic_load_rlx(entry);
24 val &= ~SL_PTE_LOCK_BIT;
25
26 if (val & SL_PTE_READ)
27 return hhdm_paddr_to_ptr(SL_PTE_ADDR(val));
28
29 paddr_t phys = pmm_alloc_page();
30 if (!phys)
31 return NULL;
32
33 void *virt = hhdm_paddr_to_ptr(p: phys);
34 memset(virt, 0, PAGE_SIZE);
35
36 atomic_store_release(entry, SL_TABLE_ENTRY(phys) | SL_PTE_LOCK_BIT);
37 return virt;
38}
39
40static enum iommu_error vtd_sl_map_page(uint64_t *sl_pgd, iova_t iova,
41 paddr_t pa, uint32_t perm) {
42 sl_pte_atomic_t *locked[PT_LEVELS] = {NULL, NULL, NULL, NULL};
43 enum irql saved[PT_LEVELS];
44
45 sl_pte_atomic_t *e4 = (sl_pte_atomic_t *) &sl_pgd[SL_PML4_INDEX(iova)];
46 saved[0] = vtd_pt_lock(pte: e4);
47 locked[0] = e4;
48
49 uint64_t *pdp = vtd_sl_ensure_child(entry: e4);
50 if (!pdp)
51 goto fail;
52
53 sl_pte_atomic_t *e3 = (sl_pte_atomic_t *) &pdp[SL_PDPT_INDEX(iova)];
54 saved[1] = vtd_pt_lock(pte: e3);
55 locked[1] = e3;
56
57 uint64_t *pd = vtd_sl_ensure_child(entry: e3);
58 if (!pd)
59 goto fail;
60
61 sl_pte_atomic_t *e2 = (sl_pte_atomic_t *) &pd[SL_PD_INDEX(iova)];
62 saved[2] = vtd_pt_lock(pte: e2);
63 locked[2] = e2;
64
65 uint64_t *pt = vtd_sl_ensure_child(entry: e2);
66 if (!pt)
67 goto fail;
68
69 sl_pte_atomic_t *e1 = (sl_pte_atomic_t *) &pt[SL_PT_INDEX(iova)];
70 saved[3] = vtd_pt_lock(pte: e1);
71 locked[3] = e1;
72
73 atomic_store_release(e1, SL_PAGE_ENTRY(pa, perm));
74
75 /* Unlock in reverse: leaf → root */
76 for (int i = PT_LEVELS - 1; i >= 0; i--)
77 vtd_pt_unlock(pte: locked[i], old_irql: saved[i]);
78
79 return IOMMU_ERR_OK;
80
81fail:
82 for (int i = PT_LEVELS - 1; i >= 0; i--) {
83 if (locked[i])
84 vtd_pt_unlock(pte: locked[i], old_irql: saved[i]);
85 }
86 return IOMMU_ERR_NO_MEM;
87}
88
89static cc_unused bool vtd_sl_unmap_page(uint64_t *sl_pgd, iova_t iova) {
90 pte_t pte;
91
92 pte = sl_pgd[SL_PML4_INDEX(iova)];
93 if (!(pte & SL_PTE_READ))
94 return false;
95
96 uint64_t *pdp = hhdm_paddr_to_ptr(SL_PTE_ADDR(pte));
97 pte = pdp[SL_PDPT_INDEX(iova)];
98 if (!(pte & SL_PTE_READ))
99 return false;
100
101 uint64_t *pd = hhdm_paddr_to_ptr(SL_PTE_ADDR(pte));
102 pte = pd[SL_PD_INDEX(iova)];
103 if (!(pte & SL_PTE_READ))
104 return false;
105
106 uint64_t *pt = hhdm_paddr_to_ptr(SL_PTE_ADDR(pte));
107 if (!(pt[SL_PT_INDEX(iova)] & SL_PTE_READ))
108 return false;
109
110 pt[SL_PT_INDEX(iova)] = 0;
111 return true;
112}
113
114static cc_unused paddr_t vtd_sl_translate(uint64_t *sl_pgd, iova_t iova) {
115 pte_t pte;
116
117 pte = sl_pgd[SL_PML4_INDEX(iova)];
118 if (!(pte & SL_PTE_READ))
119 return 0;
120
121 uint64_t *pdp = hhdm_paddr_to_ptr(SL_PTE_ADDR(pte));
122 pte = pdp[SL_PDPT_INDEX(iova)];
123 if (!(pte & SL_PTE_READ))
124 return 0;
125
126 uint64_t *pd = hhdm_paddr_to_ptr(SL_PTE_ADDR(pte));
127 pte = pd[SL_PD_INDEX(iova)];
128 if (!(pte & SL_PTE_READ))
129 return 0;
130
131 uint64_t *pt = hhdm_paddr_to_ptr(SL_PTE_ADDR(pte));
132 pte = pt[SL_PT_INDEX(iova)];
133 if (!(pte & SL_PTE_READ))
134 return 0;
135
136 return SL_PTE_ADDR(pte) | SL_PAGE_OFFSET(iova);
137}
138
139static cc_unused enum iommu_error vtd_map(struct iommu_domain *domain,
140 iova_t iova, paddr_t pa, size_t size,
141 uint32_t perm) {
142 struct vtd_unit *u = domain->unit->private;
143 struct vtd_domain *vd = domain->priv;
144
145 if (!IS_PAGE_ALIGNED(iova) || !IS_PAGE_ALIGNED(pa) ||
146 !IS_PAGE_ALIGNED(size))
147 return IOMMU_ERR_INVALID;
148
149 for (size_t offset = 0; offset < size; offset += PAGE_SIZE) {
150 enum iommu_error err =
151 vtd_sl_map_page(sl_pgd: vd->sl_pgd, iova: iova + offset, pa: pa + offset, perm);
152 if (err != IOMMU_ERR_OK)
153 return err;
154 }
155
156 vtd_iq_submit(u, IOTLB_INVAL_DESC_DOMAIN(vd->domain_id));
157 vtd_iq_flush(u);
158
159 return IOMMU_ERR_OK;
160}
161
162static cc_unused void vtd_flush_iotlb_domain(struct iommu_domain *domain) {
163 struct vtd_unit *u = domain->unit->private;
164 struct vtd_domain *vd = domain->priv;
165
166 vtd_iq_submit(u, IOTLB_INVAL_DESC_DOMAIN(vd->domain_id));
167 vtd_iq_flush(u);
168}
169
170static cc_unused void vtd_flush_iotlb_range(struct iommu_domain *domain,
171 iova_t iova, size_t size) {
172 struct vtd_unit *u = domain->unit->private;
173 struct vtd_domain *vd = domain->priv;
174
175 if (CAP_PAGE_SELECTIVE_INVALIDATION(u->cap) && size <= 32 * PAGE_SIZE) {
176 for (size_t off = 0; off < size; off += PAGE_SIZE)
177 vtd_iq_submit(u,
178 IOTLB_INVAL_DESC_PAGE(vd->domain_id, iova + off, 0));
179 } else {
180 vtd_iq_submit(u, IOTLB_INVAL_DESC_DOMAIN(vd->domain_id));
181 }
182
183 vtd_iq_flush(u);
184}
185
186struct vtd_walk_state {
187 uint64_t *tables[PT_LEVELS];
188 size_t indices[PT_LEVELS];
189};
190
191static bool vtd_sl_unmap_locked(uint64_t *sl_pgd, iova_t iova,
192 struct vtd_walk_state *ws) {
193 sl_pte_atomic_t *locked[PT_LEVELS];
194 enum irql saved[PT_LEVELS];
195
196 ws->tables[0] = sl_pgd;
197 ws->indices[0] = SL_PML4_INDEX(iova);
198 ws->indices[1] = SL_PDPT_INDEX(iova);
199 ws->indices[2] = SL_PD_INDEX(iova);
200 ws->indices[3] = SL_PT_INDEX(iova);
201
202 uint64_t *cur = sl_pgd;
203
204 for (int lvl = 0; lvl < PT_LEVELS; lvl++) {
205 sl_pte_atomic_t *entry = (sl_pte_atomic_t *) &cur[ws->indices[lvl]];
206 saved[lvl] = vtd_pt_lock(pte: entry);
207 locked[lvl] = entry;
208
209 uint64_t val = atomic_load_rlx(entry);
210 val &= ~SL_PTE_LOCK_BIT;
211
212 if (!(val & SL_PTE_READ)) {
213 /* not mapped, unlock everything and bail */
214 for (int i = lvl; i >= 0; i--)
215 vtd_pt_unlock(pte: locked[i], old_irql: saved[i]);
216 return false;
217 }
218
219 if (lvl == PT_LEVELS - 1) {
220 /* leaf, clear it, keep lock bit until unlock */
221 atomic_store_release(entry, SL_PTE_LOCK_BIT);
222 } else {
223 ws->tables[lvl + 1] = hhdm_paddr_to_ptr(SL_PTE_ADDR(val));
224 cur = ws->tables[lvl + 1];
225 }
226 }
227
228 for (int i = PT_LEVELS - 1; i >= 0; i--)
229 vtd_pt_unlock(pte: locked[i], old_irql: saved[i]);
230
231 return true;
232}
233
234/* bottom-up reclaim of empty intermediate tables */
235static void vtd_sl_reclaim_walk(struct vtd_walk_state *ws) {
236 /* lvl 3 is the leaf (PT entry) which is already cleared */
237 for (int lvl = PT_LEVELS - 2; lvl >= 0; lvl--) {
238 sl_pte_atomic_t *parent =
239 (sl_pte_atomic_t *) &ws->tables[lvl][ws->indices[lvl]];
240
241 enum irql old_irql = vtd_pt_lock(pte: parent);
242
243 uint64_t val = atomic_load_rlx(parent);
244 val &= ~SL_PTE_LOCK_BIT;
245
246 if (!(val & SL_PTE_READ)) {
247 /* someone else already freed it */
248 vtd_pt_unlock(pte: parent, old_irql);
249 break;
250 }
251
252 uint64_t *child = ws->tables[lvl + 1];
253
254 if (!vtd_pt_empty(pt: child)) {
255 vtd_pt_unlock(pte: parent, old_irql);
256 break;
257 }
258
259 /* child is empty, clear parent entry and free child */
260 atomic_store_release(parent, SL_PTE_LOCK_BIT);
261 vtd_pt_mark_dead(pte: parent);
262 vtd_pt_unlock(pte: parent, old_irql);
263
264 pmm_free_page(addr: hhdm_ptr_to_paddr(ptr: child));
265 }
266}
267
268static cc_unused void vtd_unmap(struct iommu_domain *domain, iova_t iova,
269 size_t size) {
270 struct vtd_unit *u = domain->unit->private;
271 struct vtd_domain *vd = domain->priv;
272
273 if (!IS_PAGE_ALIGNED(iova) || !IS_PAGE_ALIGNED(size))
274 return;
275
276 size_t page_count = size / PAGE_SIZE;
277
278 struct vtd_walk_state *walks = vnews(walks, page_count);
279 bool *unmapped = vnews(unmapped, page_count);
280 if (!walks || !unmapped) {
281 for (size_t off = 0; off < size; off += PAGE_SIZE) {
282 struct vtd_walk_state dummy;
283 vtd_sl_unmap_locked(sl_pgd: vd->sl_pgd, iova: iova + off, ws: &dummy);
284 }
285 vtd_iotlb_flush_range_batched(u, domain_id: vd->domain_id, iova, size);
286 vfree(walks);
287 vfree(unmapped);
288 return;
289 }
290
291 /* clear all leaves */
292 for (size_t i = 0; i < page_count; i++)
293 unmapped[i] =
294 vtd_sl_unmap_locked(sl_pgd: vd->sl_pgd, iova: iova + i * PAGE_SIZE, ws: &walks[i]);
295
296 /* send invalidation */
297 vtd_iotlb_flush_range_batched(u, domain_id: vd->domain_id, iova, size);
298
299 /* go and reclaim */
300 for (size_t i = 0; i < page_count; i++) {
301 if (unmapped[i])
302 vtd_sl_reclaim_walk(ws: &walks[i]);
303 }
304
305 vfree(walks);
306 vfree(unmapped);
307}
308