1#include <asm.h>
2#include <console/printf.h>
3#include <drivers/iommu/vt_d.h>
4#include <log.h>
5#include <mem/page.h>
6#include <mem/pmm.h>
7#include <mem/vmm.h>
8
9#define PT_LEVELS 4
10
11/*
12 * VT-d Second-Level PTE locking
13 */
14#define SL_PTE_LOCK_BIT BIT(61)
15#define SL_PTE_DEAD_BIT BIT(60)
16
17typedef atomic_uint64_t sl_pte_atomic_t;
18
19static inline bool vtd_pt_trylock(sl_pte_atomic_t *pte,
20 enum irql *irql_out) TSA_NO_ANALYSIS {
21 *irql_out = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
22
23 uint64_t old = atomic_load_rlx(pte);
24 do {
25 if (old & (SL_PTE_LOCK_BIT | SL_PTE_DEAD_BIT)) {
26 irql_lower(old_level: *irql_out);
27 return false;
28 }
29
30 } while (
31 !atomic_cas_weak(pte, &old, old | SL_PTE_LOCK_BIT, mo_acquire, mo_rlx));
32
33 return true;
34}
35
36static inline void vtd_pt_unlock_internal(sl_pte_atomic_t *pte) {
37 atomic_fetch_and_release(pte, ~SL_PTE_LOCK_BIT);
38}
39
40static inline void vtd_pt_mark_dead(sl_pte_atomic_t *pte) {
41 atomic_fetch_or_release(pte, SL_PTE_DEAD_BIT);
42}
43
44enum vtd_pt_lock_result {
45 VTD_PT_LOCK_OK = 0,
46 VTD_PT_LOCK_NOT_PRESENT,
47 VTD_PT_LOCK_DEAD,
48};
49
50static inline enum vtd_pt_lock_result
51vtd_pt_lock_internal(sl_pte_atomic_t *pte) {
52 while (true) {
53 uint64_t old = atomic_load_rlx(pte);
54
55 if (!(old & SL_PTE_PRESENT))
56 return VTD_PT_LOCK_NOT_PRESENT;
57 if (old & SL_PTE_DEAD_BIT)
58 return VTD_PT_LOCK_DEAD;
59 if (old & SL_PTE_LOCK_BIT) {
60 cpu_pause();
61 continue;
62 }
63
64 if (atomic_cas_weak(pte, &old, old | SL_PTE_LOCK_BIT, mo_acquire,
65 mo_rlx))
66 return VTD_PT_LOCK_OK;
67
68 cpu_pause();
69 }
70}
71
72static inline enum irql vtd_pt_lock(sl_pte_atomic_t *pte) TSA_NO_ANALYSIS {
73 enum irql old = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
74 while (true) {
75 uint64_t val = atomic_load_rlx(pte);
76 if (val & SL_PTE_LOCK_BIT) {
77 cpu_pause();
78 continue;
79 }
80 if (atomic_cas_weak(pte, &val, val | SL_PTE_LOCK_BIT, mo_acquire,
81 mo_rlx))
82 return old;
83 }
84}
85
86static inline void vtd_pt_unlock(sl_pte_atomic_t *pte,
87 enum irql old_irql) TSA_NO_ANALYSIS {
88 vtd_pt_unlock_internal(pte);
89 irql_lower(old_level: old_irql);
90}
91
92void vtd_write_gcmd(struct vtd_unit *u, uint32_t cmd);
93void vtd_wait_gsts(struct vtd_unit *u, uint32_t bit, bool set);
94enum iommu_error vtd_iq_init(struct vtd_unit *u);
95void vtd_iq_submit(struct vtd_unit *u, struct vtd_inv_desc desc);
96void vtd_iq_flush(struct vtd_unit *u);
97enum iommu_error vtd_root_table_init(struct vtd_unit *u);
98enum iommu_error vtd_root_table_init(struct vtd_unit *u);
99void vtd_iotlb_flush_global(struct vtd_unit *u);
100void vtd_iotlb_flush_domain(struct vtd_unit *u, uint16_t domain_id);
101void vtd_iotlb_flush_page(struct vtd_unit *u, uint16_t domain_id, iova_t iova,
102 uint8_t am);
103void vtd_iotlb_flush_range(struct vtd_unit *u, uint16_t domain_id, iova_t iova,
104 size_t size);
105void vtd_iotlb_flush_range_batched(struct vtd_unit *u, uint16_t domain_id,
106 iova_t iova, size_t size);
107
108ssize_t vtd_domain_alloc(struct vtd_unit *unit);
109ssize_t vtd_domain_free(struct vtd_unit *unit, size_t domain);
110bool vtd_domain_init(struct vtd_unit *unit);
111extern const struct iommu_ops vtd_iommu_ops;
112