1#include <asm.h>
2#include <console/printf.h>
3#include <drivers/iommu/vt_d.h>
4#include <log.h>
5#include <mem/page.h>
6#include <mem/pmm.h>
7#include <mem/vmm.h>
8
9#define PT_LEVELS 4
10
11/*
12 * VT-d Second-Level PTE locking
13 */
14#define SL_PTE_LOCK_BIT BIT(61)
15#define SL_PTE_DEAD_BIT BIT(60)
16
17typedef _Atomic uint64_t sl_pte_atomic_t;
18
19static inline bool vtd_pt_trylock(sl_pte_atomic_t *pte, enum irql *irql_out) {
20 *irql_out = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
21
22 uint64_t old = atomic_load_explicit(pte, memory_order_relaxed);
23 do {
24 if (old & (SL_PTE_LOCK_BIT | SL_PTE_DEAD_BIT)) {
25 irql_lower(old_level: *irql_out);
26 return false;
27 }
28
29 } while (!atomic_compare_exchange_weak_explicit(
30 pte, &old, old | SL_PTE_LOCK_BIT, memory_order_acquire,
31 memory_order_relaxed));
32
33 return true;
34}
35
36static inline void vtd_pt_unlock_internal(sl_pte_atomic_t *pte) {
37 atomic_fetch_and_explicit(pte, ~SL_PTE_LOCK_BIT, memory_order_release);
38}
39
40static inline void vtd_pt_mark_dead(sl_pte_atomic_t *pte) {
41 atomic_fetch_or_explicit(pte, SL_PTE_DEAD_BIT, memory_order_release);
42}
43
44enum vtd_pt_lock_result {
45 VTD_PT_LOCK_OK = 0,
46 VTD_PT_LOCK_NOT_PRESENT,
47 VTD_PT_LOCK_DEAD,
48};
49
50static inline enum vtd_pt_lock_result
51vtd_pt_lock_internal(sl_pte_atomic_t *pte) {
52 for (;;) {
53 uint64_t old = atomic_load_explicit(pte, memory_order_relaxed);
54
55 if (!(old & SL_PTE_PRESENT))
56 return VTD_PT_LOCK_NOT_PRESENT;
57 if (old & SL_PTE_DEAD_BIT)
58 return VTD_PT_LOCK_DEAD;
59 if (old & SL_PTE_LOCK_BIT) {
60 cpu_relax();
61 continue;
62 }
63
64 if (atomic_compare_exchange_weak_explicit(
65 pte, &old, old | SL_PTE_LOCK_BIT, memory_order_acquire,
66 memory_order_relaxed))
67 return VTD_PT_LOCK_OK;
68
69 cpu_relax();
70 }
71}
72
73static inline enum irql vtd_pt_lock(sl_pte_atomic_t *pte) {
74 enum irql old = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
75 for (;;) {
76 uint64_t val = atomic_load_explicit(pte, memory_order_relaxed);
77 if (val & SL_PTE_LOCK_BIT) {
78 cpu_relax();
79 continue;
80 }
81 if (atomic_compare_exchange_weak_explicit(
82 pte, &val, val | SL_PTE_LOCK_BIT, memory_order_acquire,
83 memory_order_relaxed))
84 return old;
85 }
86}
87
88static inline void vtd_pt_unlock(sl_pte_atomic_t *pte, enum irql old_irql) {
89 vtd_pt_unlock_internal(pte);
90 irql_lower(old_level: old_irql);
91}
92
93void vtd_write_gcmd(struct vtd_unit *u, uint32_t cmd);
94void vtd_wait_gsts(struct vtd_unit *u, uint32_t bit, bool set);
95enum iommu_error vtd_iq_init(struct vtd_unit *u);
96void vtd_iq_submit(struct vtd_unit *u, struct vtd_inv_desc desc);
97void vtd_iq_flush(struct vtd_unit *u);
98enum iommu_error vtd_root_table_init(struct vtd_unit *u);
99enum iommu_error vtd_root_table_init(struct vtd_unit *u);
100void vtd_iotlb_flush_global(struct vtd_unit *u);
101void vtd_iotlb_flush_domain(struct vtd_unit *u, uint16_t domain_id);
102void vtd_iotlb_flush_page(struct vtd_unit *u, uint16_t domain_id, iova_t iova,
103 uint8_t am);
104void vtd_iotlb_flush_range(struct vtd_unit *u, uint16_t domain_id, iova_t iova,
105 size_t size);
106void vtd_iotlb_flush_range_batched(struct vtd_unit *u, uint16_t domain_id,
107 iova_t iova, size_t size);
108
109size_t vtd_domain_alloc(struct vtd_unit *unit);
110size_t vtd_domain_free(struct vtd_unit *unit, size_t domain);
111bool vtd_domain_init(struct vtd_unit *unit);
112extern const struct iommu_ops vtd_iommu_ops;
113