1/* @title: VT-d */
2#pragma once
3#include <console/panic.h>
4#include <console/printf.h>
5#include <drivers/iommu/iommu.h>
6#include <log.h>
7#include <math/bit.h>
8#include <sync/spinlock.h>
9LOG_SITE_EXTERN(vtd);
10LOG_HANDLE_EXTERN(vtd);
11
12#define vtd_log(lvl, fmt, ...) \
13 log(LOG_SITE(vtd), LOG_HANDLE(vtd), lvl, fmt, ##__VA_ARGS__)
14
15#define vtd_err(fmt, ...) vtd_log(LOG_ERROR, fmt, ##__VA_ARGS__)
16#define vtd_warn(fmt, ...) vtd_log(LOG_WARN, fmt, ##__VA_ARGS__)
17#define vtd_info(fmt, ...) vtd_log(LOG_INFO, fmt, ##__VA_ARGS__)
18#define vtd_debug(fmt, ...) vtd_log(LOG_DEBUG, fmt, ##__VA_ARGS__)
19#define vtd_trace(fmt, ...) vtd_log(LOG_TRACE, fmt, ##__VA_ARGS__)
20
21/*
22 * Register-name prefixes used below mirror the names in the Intel VT-d spec
23 * so that bit-field macros can be cross-referenced with the spec text.
24 * CAP - Capability Register
25 * ECAP - Extended Capability Register
26 * GCMD - Global Command Register
27 * GSTS - Global Status Register
28 * RTADDR - Root Table Address Register
29 * FSTS - Fault Status Register
30 * IQA - Invalidation Queue Address Register
31 */
32
33struct vtd_regs {
34 uint32_t version; /* 0x000 - Version */
35 uint32_t _res0;
36 uint64_t capabilities; /* 0x008 - Capability */
37 uint64_t extended_capabilities; /* 0x010 - Extended Capability */
38 uint32_t global_command; /* 0x018 - Global Command */
39 uint32_t global_status; /* 0x01C - Global Status */
40 uint64_t root_table_addr; /* 0x020 - Root Table Address */
41 uint64_t context_command; /* 0x028 - Context Command */
42 uint32_t _res1;
43 uint32_t fault_status; /* 0x034 - Fault Status */
44 uint32_t fault_event_control; /* 0x038 - Fault Event Control */
45 uint32_t fault_event_data; /* 0x03C - Fault Event Data */
46 uint32_t fault_event_addr; /* 0x040 - Fault Event Address */
47 uint32_t fault_event_addr_upper; /* 0x044 - Fault Event Upper Address*/
48 uint64_t _res2[2];
49 uint64_t advanced_fault_log; /* 0x058 - Advanced Fault Log */
50 uint32_t _res3;
51 uint32_t protected_memory_enable; /* 0x064 - Protected Memory Enable */
52 uint32_t protected_low_mem_base; /* 0x068 - Protected Low Mem Base */
53 uint32_t protected_low_mem_limit; /* 0x06C - Protected Low Mem Limit */
54 uint64_t protected_high_mem_base; /* 0x070 - Protected High Mem Base */
55 uint64_t protected_high_mem_limit; /* 0x078 - Protected High Mem Limit */
56 uint64_t invalidation_queue_head; /* 0x080 - Invalidation Queue Head */
57 uint64_t invalidation_queue_tail; /* 0x088 - Invalidation Queue Tail */
58 uint64_t invalidation_queue_addr; /* 0x090 - Invalidation Queue Addr */
59 uint32_t _res4;
60 uint32_t
61 invalidation_comp_status; /* 0x09C - Invalidation Completion Status */
62 uint32_t invalidation_event_ctrl; /* 0x0A0 - Invalidation Event Control */
63 uint32_t invalidation_event_data; /* 0x0A4 - Invalidation Event Data */
64 uint32_t invalidation_event_addr; /* 0x0A8 - Invalidation Event Address */
65 uint32_t invalidation_event_addr_upper; /* 0x0AC - Invalidation Event Upper
66 Addr */
67 uint64_t invalidation_queue_error_record; /* 0x0B0 - IQ Error Record */
68 uint64_t interrupt_remapping_table_addr; /* 0x0B8 - IR Table Address */
69} __packed;
70
71_Static_assert(offsetof(struct vtd_regs, invalidation_queue_head) == 0x080,
72 "vtd_regs layout mismatch");
73_Static_assert(offsetof(struct vtd_regs, interrupt_remapping_table_addr) ==
74 0x0B8,
75 "vtd_regs layout mismatch");
76_Static_assert(offsetof(struct vtd_regs, fault_status) == 0x034,
77 "vtd_regs layout mismatch");
78
79struct vtd_invl_queue {};
80
81struct vtd_unit {
82 struct vtd_regs *regs;
83 uint64_t cap;
84 uint64_t ecap;
85 uint16_t segment;
86 uint32_t domain_count;
87 uint8_t *domain_bitmap;
88 struct spinlock domain_bitmap_lock;
89
90 uint64_t *root_table;
91 paddr_t root_table_phys;
92
93 void *iq_base;
94 paddr_t iq_phys;
95 uint32_t iq_head;
96 uint32_t iq_tail;
97 uint32_t iq_size;
98};
99
100struct vtd_domain {
101 uint16_t domain_id;
102 uint64_t *sl_pgd;
103 uint64_t sl_pgd_phys;
104};
105
106/*
107 * CAP - Capability Register (64-bit, offset 0x008)
108 * Intel VT-d spec 11.4.2
109 */
110#define CAP_NUM_DOMAINS(cap) \
111 BIT_RANGE((cap), 0, 2) /* ND - Number of Domains supported (encoded) \
112 */
113#define CAP_REQUIRES_WRITE_BUF_FLUSH(cap) \
114 BIT_RANGE((cap), 4, 4) /* RWBF - Required Write-Buffer Flushing */
115#define CAP_PROTECTED_LOW_MEMORY(cap) \
116 BIT_RANGE((cap), 5, 5) /* PLMR - Protected Low-Memory Region */
117#define CAP_PROTECTED_HIGH_MEMORY(cap) \
118 BIT_RANGE((cap), 6, 6) /* PHMR - Protected High-Memory Region */
119#define CAP_CACHING_MODE(cap) BIT_RANGE((cap), 7, 7) /* CM */
120#define CAP_SUPPORTED_ADDR_WIDTHS(cap) \
121 BIT_RANGE((cap), 8, \
122 12) /* SAGAW - Supported Adjusted Guest Address Widths */
123#define CAP_MAX_ADDR_WIDTH(cap) \
124 BIT_RANGE((cap), 16, 21) /* MGAW - Maximum Guest Address Width */
125#define CAP_ZERO_LENGTH_READ(cap) BIT_RANGE((cap), 22, 22) /* ZLR */
126#define CAP_FAULT_RECORD_OFFSET(cap) \
127 BIT_RANGE((cap), 24, 33) /* FRO - Fault-Recording Register Offset */
128#define CAP_LARGE_PAGE_SUPPORT(cap) \
129 BIT_RANGE((cap), 34, 37) /* SLLPS - Second-Level Large Page Support */
130#define CAP_PAGE_SELECTIVE_INVALIDATION(cap) \
131 BIT_RANGE((cap), 39, 39) /* PSI \
132 */
133#define CAP_NUM_FAULT_RECORDS(cap) \
134 BIT_RANGE((cap), 40, 47) /* NFR - Number of Fault-Recording Registers \
135 */
136#define CAP_MAX_ADDR_MASK_VALUE(cap) BIT_RANGE((cap), 48, 53) /* MAMV */
137#define CAP_WRITE_DRAINING(cap) BIT_RANGE((cap), 54, 54) /* DWD */
138#define CAP_READ_DRAINING(cap) BIT_RANGE((cap), 55, 55) /* DRD */
139#define CAP_FIRST_LEVEL_1GB_PAGE(cap) BIT_RANGE((cap), 56, 56) /* FL1GP */
140#define CAP_POSTED_INTERRUPTS(cap) BIT_RANGE((cap), 59, 59) /* PI */
141#define CAP_FIRST_STAGE_5LEVEL_PAGING(cap) \
142 BIT_RANGE((cap), 60, 60) /* FL5LP / FS5LP */
143#define CAP_ENHANCED_COMMAND_SUPPORT(cap) BIT_RANGE((cap), 61, 61) /* ECMDS */
144#define CAP_ENHANCED_SET_INTR_REMAP_TABLE_PTR_SUPPORT(cap) \
145 BIT_RANGE( \
146 (cap), 62, \
147 62) /* ESIRTPS - Enhanced Set Interrupt Remap Table Pointer Support */
148#define CAP_ENHANCED_SET_ROOT_TABLE_PTR_SUPPORT(cap) \
149 BIT_RANGE((cap), 63, \
150 63) /* ESRTPS - Enhanced Set Root Table Pointer Support */
151
152/*
153 * ECAP - Extended Capability Register (64-bit, offset 0x010)
154 * Intel VT-d spec 11.4.3
155 */
156#define ECAP_COHERENCY(ecap) BIT_RANGE((ecap), 0, 0) /* C */
157#define ECAP_QUEUED_INVALIDATION(ecap) BIT_RANGE((ecap), 1, 1) /* QI */
158#define ECAP_DEVICE_TLB(ecap) BIT_RANGE((ecap), 2, 2) /* DT */
159#define ECAP_INTERRUPT_REMAPPING(ecap) BIT_RANGE((ecap), 3, 3) /* IR */
160#define ECAP_EXTENDED_INTERRUPT_MODE(ecap) BIT_RANGE((ecap), 4, 4) /* EIM */
161#define ECAP_PASS_THROUGH(ecap) BIT_RANGE((ecap), 6, 6) /* PT */
162#define ECAP_SNOOP_CONTROL(ecap) BIT_RANGE((ecap), 7, 7) /* SC */
163#define ECAP_IOTLB_REGISTER_OFFSET(ecap) BIT_RANGE((ecap), 8, 17) /* IRO */
164#define ECAP_MAX_HANDLE_MASK_VALUE(ecap) BIT_RANGE((ecap), 20, 23) /* MHMV */
165#define ECAP_MEMORY_TYPE_SUPPORT(ecap) BIT_RANGE((ecap), 25, 25) /* MTS */
166#define ECAP_NESTED_TRANSLATION(ecap) BIT_RANGE((ecap), 26, 26) /* NEST */
167#define ECAP_PAGE_REQUEST_SUPPORT(arg) BIT_TEST((arg), 29) /* PRS */
168#define ECAP_EXECUTE_REQUEST_SUPPORT(arg) BIT_TEST((arg), 30) /* ERS */
169#define ECAP_SUPERVISOR_REQUEST_SUPPORT(arg) BIT_TEST((arg), 31) /* SRS */
170#define ECAP_NO_WRITE_FLAG_SUPPORT(arg) BIT_TEST((arg), 33) /* NWFS */
171#define ECAP_EXTENDED_ACCESSED_FLAG_SUPPORT(arg) \
172 BIT_TEST((arg), 34) /* EAFS \
173 */
174#define ECAP_PASID_SIZE_SUPPORTED(arg) \
175 BIT_RANGE((arg), 35, 39) /* PSS - Process Address Space ID size */
176#define ECAP_PASID_SUPPORT(arg) BIT_TEST((arg), 40) /* PASID */
177#define ECAP_DEVICE_TLB_INVAL_THROTTLE(arg) \
178 BIT_TEST((arg), 41) /* DIT \
179 */
180#define ECAP_PAGE_REQUEST_DRAIN_SUPPORT(arg) BIT_TEST((arg), 42) /* PDS */
181#define ECAP_SCALABLE_MODE_TRANSLATION_SUPPORT(arg) \
182 BIT_TEST((arg), 43) /* SMTS */
183#define ECAP_VIRTUAL_COMMAND_SUPPORT(arg) BIT_TEST((arg), 44) /* VCS */
184#define ECAP_SECOND_STAGE_ACCESSED_DIRTY_SUPPORT(arg) \
185 BIT_TEST((arg), 45) /* SSADS */
186#define ECAP_SECOND_STAGE_TRANSLATION_SUPPORT(arg) \
187 BIT_TEST((arg), 46) /* SSTS */
188#define ECAP_FIRST_STAGE_TRANSLATION_SUPPORT(arg) \
189 BIT_TEST((arg), 47) /* FSTS (do not confuse with Fault Status register) */
190#define ECAP_SCALABLE_MODE_PAGE_WALK_COHERENCY_SUPPORT(arg) \
191 BIT_TEST((arg), 48) /* SMPWCS */
192#define ECAP_RID_PASID_SUPPORT(arg) \
193 BIT_TEST((arg), 49) /* RPS - Requester-ID/PASID */
194#define ECAP_PERFORMANCE_MONITORING_SUPPORT(arg) BIT_TEST((arg), 51) /* PMS */
195#define ECAP_ABORT_DMA_MODE_SUPPORT(arg) BIT_TEST((arg), 52) /* ADMS */
196#define ECAP_RID_PRIV_SUPPORT(arg) \
197 BIT_TEST((arg), 53) /* RPRIVS - Requester-ID Privilege */
198#define ECAP_STOP_MARKER_SUPPORT(arg) BIT_TEST((arg), 58) /* SMS */
199
200/*
201 * Bit positions in CAP_SUPPORTED_ADDR_WIDTHS (CAP.SAGAW)
202 * Each bit indicates a supported guest address width / page-table depth.
203 */
204#define ADDR_WIDTH_39BIT BIT(1) /* 3-level page table */
205#define ADDR_WIDTH_48BIT BIT(2) /* 4-level page table */
206#define ADDR_WIDTH_57BIT BIT(3) /* 5-level page table */
207
208/*
209 * GCMD - Global Command Register (32-bit, offset 0x018)
210 * GSTS - Global Status Register (32-bit, offset 0x01C)
211 * Intel VT-d spec 11.4.4, 11.4.5
212 */
213#define GCMD_TRANSLATION_ENABLE BIT(31) /* TE */
214#define GCMD_SET_ROOT_TABLE_PTR BIT(30) /* SRTP - Set Root Table Pointer */
215#define GCMD_WRITE_BUFFER_FLUSH BIT(27) /* WBF */
216#define GCMD_QUEUED_INVAL_ENABLE BIT(26) /* QIE */
217#define GCMD_INTERRUPT_REMAP_ENABLE BIT(25) /* IRE */
218#define GCMD_SET_INTR_REMAP_TABLE_PTR \
219 BIT(24) /* SIRTP - Set Interrupt Remap Table Pointer */
220#define GCMD_COMPAT_FORMAT_INTERRUPT \
221 BIT(23) /* CFI - Compatibility Format Interrupt */
222
223#define GSTS_TRANSLATION_ENABLED BIT(31) /* TES */
224#define GSTS_ROOT_TABLE_PTR_SET BIT(30) /* RTPS */
225#define GSTS_WRITE_BUFFER_FLUSH_STATUS BIT(27) /* WBFS */
226#define GSTS_QUEUED_INVAL_ENABLED BIT(26) /* QIES */
227#define GSTS_INTERRUPT_REMAP_ENABLED BIT(25) /* IRES */
228#define GSTS_INTR_REMAP_TABLE_PTR_SET BIT(24) /* IRTPS */
229#define GSTS_COMPAT_FORMAT_INTERRUPT_STATUS BIT(23) /* CFIS */
230
231/*
232 * RTADDR - Root Table Address Register (64-bit, offset 0x020)
233 * TTM = Translation Table Mode (bits 10-11).
234 */
235#define RTADDR_TRANSLATION_MODE_LEGACY (0ULL << 10)
236#define RTADDR_TRANSLATION_MODE_SCALABLE (1ULL << 10)
237
238/*
239 * FSTS - Fault Status Register (32-bit, offset 0x034)
240 */
241#define FSTS_PRIMARY_FAULT_OVERFLOW BIT(0) /* PFO */
242#define FSTS_PRIMARY_PENDING_FAULT BIT(1) /* PPF */
243#define FSTS_ADVANCED_FAULT_OVERFLOW BIT(2) /* AFO */
244#define FSTS_ADVANCED_PENDING_FAULT BIT(3) /* APF */
245#define FSTS_INVAL_QUEUE_ERROR BIT(4) /* IQE */
246#define FSTS_INVAL_COMPLETION_ERROR BIT(5) /* ICE */
247#define FSTS_INVAL_TIMEOUT_ERROR BIT(6) /* ITE */
248#define FSTS_PAGE_REQUEST_OVERFLOW BIT(7) /* PRO */
249#define FSTS_FAULT_RECORD_INDEX(s) BIT_RANGE((s), 8, 15) /* FRI */
250
251/*
252 * Root Entry (128-bit / two 64-bit words) - Intel VT-d spec 9.1
253 * CTP = Context-Table Pointer (bits 12-63 of the lo word).
254 */
255#define ROOT_ENTRY_PRESENT BIT(0)
256#define ROOT_ENTRY_CONTEXT_TABLE_PTR_SHIFT 12
257#define ROOT_ENTRY_CONTEXT_TABLE_PTR_MASK (~0xFFFULL)
258
259#define ROOT_ENTRY_SET_CONTEXT_TABLE_PTR(phys) \
260 (((phys) & ROOT_ENTRY_CONTEXT_TABLE_PTR_MASK) | ROOT_ENTRY_PRESENT)
261
262struct vtd_root_entry {
263 uint64_t lo;
264 uint64_t hi; /* must be zero */
265} __packed;
266
267_Static_assert(sizeof(struct vtd_root_entry) == 16, "root entry must be 16B");
268
269/*
270 * Context Entry (128-bit / two 64-bit words) - Intel VT-d spec 9.3
271 * Field acronyms used as suffixes:
272 * FPD - Fault Processing Disable
273 * TT - Translation Type
274 * SLPTPTR - Second-Level Page-Table Pointer
275 * AW - Address Width
276 * DID - Domain Identifier
277 */
278#define CTX_ENTRY_PRESENT BIT(0)
279#define CTX_ENTRY_FAULT_PROCESSING_DISABLE BIT(1)
280#define CTX_ENTRY_TRANSLATION_TYPE_SHIFT 2
281#define CTX_ENTRY_TRANSLATION_TYPE_MASK BIT_MASK(2, 3)
282#define CTX_ENTRY_TRANSLATION_TYPE_UNTRANSLATED (0ULL << 2)
283#define CTX_ENTRY_TRANSLATION_TYPE_PASSTHROUGH (2ULL << 2)
284#define CTX_ENTRY_SL_PTR_SHIFT 12
285#define CTX_ENTRY_SL_PTR_MASK (~0xFFFULL)
286
287#define CTX_ENTRY_ADDR_WIDTH_SHIFT 0
288#define CTX_ENTRY_ADDR_WIDTH_39BIT 1
289#define CTX_ENTRY_ADDR_WIDTH_48BIT 2
290#define CTX_ENTRY_ADDR_WIDTH_57BIT 3
291
292#define CTX_ENTRY_DOMAIN_ID_SHIFT 8
293#define CTX_ENTRY_DOMAIN_ID_MASK BIT_MASK(8, 23)
294
295#define CTX_ENTRY_SET_LO(slptptr, tt) \
296 (((slptptr) & CTX_ENTRY_SL_PTR_MASK) | (tt) | CTX_ENTRY_PRESENT)
297#define CTX_ENTRY_SET_HI(domain_id, aw) \
298 (((uint64_t) (domain_id) << CTX_ENTRY_DOMAIN_ID_SHIFT) | (aw))
299
300/* Extract fields from a context entry */
301#define CTX_ENTRY_TRANSLATION_TYPE(lo) BIT_RANGE((lo), 2, 3)
302#define CTX_ENTRY_DOMAIN_ID(hi) BIT_RANGE((hi), 8, 23)
303#define CTX_ENTRY_ADDR_WIDTH(hi) BIT_RANGE((hi), 0, 2)
304
305struct vtd_context_entry {
306 uint64_t lo;
307 uint64_t hi;
308} __packed;
309
310_Static_assert(sizeof(struct vtd_context_entry) == 16, "ctx entry must be 16B");
311
312/*
313 * Second-Level Page-Table Entry (PTE).
314 */
315#define SL_PTE_READ BIT(0)
316#define SL_PTE_WRITE BIT(1)
317#define SL_PTE_PRESENT (SL_PTE_READ | SL_PTE_WRITE)
318#define SL_PTE_EXECUTE BIT(2)
319#define SL_PTE_MEMORY_TYPE_SHIFT 3 /* EMT - Extended Memory Type */
320#define SL_PTE_MEMORY_TYPE_WRITE_BACK (6ULL << 3) /* WB */
321#define SL_PTE_MEMORY_TYPE_UNCACHEABLE (0ULL << 3) /* UC */
322#define SL_PTE_IGNORE_PAT BIT(6) /* IPAT - Ignore Page Attribute Table */
323#define SL_PTE_LARGE_PAGE BIT(7) /* PS - Page Size */
324#define SL_PTE_SNOOP BIT(11) /* SNP */
325#define SL_PTE_ADDR_MASK (~0xFFFULL & ((1ULL << 52) - 1))
326
327#define SL_PTE_ADDR(pte) ((pte) & SL_PTE_ADDR_MASK)
328#define SL_PTE_MEMORY_TYPE(pte) BIT_RANGE((pte), 3, 5)
329#define SL_PTE_IS_LARGE(pte) BIT_RANGE((pte), 7, 7)
330
331#define SL_TABLE_ENTRY(phys) (((phys) & SL_PTE_ADDR_MASK) | SL_PTE_PRESENT)
332
333#define SL_PAGE_ENTRY(phys, perm) \
334 (((phys) & SL_PTE_ADDR_MASK) | SL_PTE_MEMORY_TYPE_WRITE_BACK | \
335 SL_PTE_IGNORE_PAT | ((perm) & 0x3))
336
337/*
338 * IOVA index helpers, identical breakdown to x86-64 VA.
339 * PML4 / PDPT / PD / PT are the standard x86-64 page-table levels.
340 */
341#define SL_PML4_INDEX(iova) (((iova) >> 39) & 0x1FF)
342#define SL_PDPT_INDEX(iova) (((iova) >> 30) & 0x1FF)
343#define SL_PD_INDEX(iova) (((iova) >> 21) & 0x1FF)
344#define SL_PT_INDEX(iova) (((iova) >> 12) & 0x1FF)
345#define SL_PAGE_OFFSET(iova) ((iova) & 0xFFF)
346#define SL_ENTRY_COUNT 512
347
348/*
349 * IQA - Invalidation Queue Address Register, and queue layout
350 * Intel VT-d spec 6.5
351 */
352#define IQA_SIZE_SHIFT 0
353#define IQA_SIZE_256_ENTRIES 0
354#define IQA_SIZE_512_ENTRIES 1
355#define IQA_SIZE_1K_ENTRIES 2
356#define IQA_SIZE_2K_ENTRIES 3
357#define IQA_SIZE_4K_ENTRIES 4
358#define IQA_SIZE_8K_ENTRIES 5
359#define IQA_SIZE_16K_ENTRIES 6
360#define IQA_SIZE_32K_ENTRIES 7
361#define IQA_ADDR_MASK (~0xFFFULL)
362
363#define IQA_REG_BUILD(phys, size_enc) (((phys) & IQA_ADDR_MASK) | (size_enc))
364#define IQA_SIZE(iqa) BIT_RANGE((iqa), 0, 2)
365
366#define INVAL_QUEUE_ENTRY_SIZE 16
367#define INVAL_QUEUE_TAIL_SHIFT 4
368
369/* Invalidation descriptor type codes (low 4 bits of descriptor.lo) */
370#define INVAL_DESC_TYPE_CONTEXT_CACHE 0x1
371#define INVAL_DESC_TYPE_IOTLB \
372 0x2 /* I/O Translation Lookaside Buffer \
373 */
374#define INVAL_DESC_TYPE_DEVICE_IOTLB 0x3
375#define INVAL_DESC_TYPE_INTERRUPT_ENTRY_CACHE 0x4 /* IEC */
376#define INVAL_DESC_TYPE_WAIT 0x5
377
378#define INVAL_DESC_TYPE(lo) BIT_RANGE((lo), 0, 3)
379#define INVAL_DESC_GRANULARITY(lo) BIT_RANGE((lo), 4, 5)
380
381/* Context-cache invalidation descriptor */
382#define CTX_INVAL_GRANULARITY_GLOBAL (1ULL << 4)
383#define CTX_INVAL_GRANULARITY_DOMAIN (2ULL << 4)
384#define CTX_INVAL_GRANULARITY_DEVICE (3ULL << 4)
385#define CTX_INVAL_DOMAIN_ID_SHIFT 48
386
387#define CTX_INVAL_DESC_GLOBAL \
388 ((struct vtd_inv_desc) {.lo = INVAL_DESC_TYPE_CONTEXT_CACHE | \
389 CTX_INVAL_GRANULARITY_GLOBAL, \
390 .hi = 0})
391#define CTX_INVAL_DESC_DOMAIN(did) \
392 ((struct vtd_inv_desc) { \
393 .lo = INVAL_DESC_TYPE_CONTEXT_CACHE | CTX_INVAL_GRANULARITY_DOMAIN | \
394 ((uint64_t) (did) << CTX_INVAL_DOMAIN_ID_SHIFT), \
395 .hi = 0})
396
397/* IOTLB invalidation descriptor */
398#define IOTLB_INVAL_GRANULARITY_GLOBAL (1ULL << 4)
399#define IOTLB_INVAL_GRANULARITY_DOMAIN (2ULL << 4)
400#define IOTLB_INVAL_GRANULARITY_PAGE (3ULL << 4)
401#define IOTLB_INVAL_DRAIN_READS BIT(7) /* DR */
402#define IOTLB_INVAL_DRAIN_WRITES BIT(6) /* DW */
403#define IOTLB_INVAL_DOMAIN_ID_SHIFT 32
404#define IOTLB_INVAL_IOVA_SHIFT 12
405#define IOTLB_INVAL_ADDR_MASK_FIELD_MASK \
406 0x3FULL /* AM (Address Mask) field width */
407#define IOTLB_INVAL_HINT \
408 BIT(6) /* IH (in hi dword): non-leaf entries unchanged */
409
410#define IOTLB_INVAL_DOMAIN_ID(lo) BIT_RANGE((lo), 32, 47)
411#define IOTLB_INVAL_GRANULARITY(lo) BIT_RANGE((lo), 4, 5)
412#define IOTLB_REG_OFFSET(ecap) ((ECAP_IOTLB_REGISTER_OFFSET(ecap) << 4) + 8)
413#define IVA_REG_OFFSET(ecap) ((ECAP_IOTLB_REGISTER_OFFSET(ecap) << 4) + 0)
414#define IOTLB_INVAL_DESC_GLOBAL \
415 ((struct vtd_inv_desc) { \
416 .lo = INVAL_DESC_TYPE_IOTLB | IOTLB_INVAL_GRANULARITY_GLOBAL | \
417 IOTLB_INVAL_DRAIN_READS | IOTLB_INVAL_DRAIN_WRITES, \
418 .hi = 0})
419#define IOTLB_INVAL_DESC_DOMAIN(did) \
420 ((struct vtd_inv_desc) { \
421 .lo = INVAL_DESC_TYPE_IOTLB | IOTLB_INVAL_GRANULARITY_DOMAIN | \
422 IOTLB_INVAL_DRAIN_READS | IOTLB_INVAL_DRAIN_WRITES | \
423 ((uint64_t) (did) << IOTLB_INVAL_DOMAIN_ID_SHIFT), \
424 .hi = 0})
425#define IOTLB_INVAL_DESC_PAGE(did, iova, am) \
426 ((struct vtd_inv_desc) { \
427 .lo = INVAL_DESC_TYPE_IOTLB | IOTLB_INVAL_GRANULARITY_PAGE | \
428 IOTLB_INVAL_DRAIN_READS | IOTLB_INVAL_DRAIN_WRITES | \
429 ((uint64_t) (did) << IOTLB_INVAL_DOMAIN_ID_SHIFT), \
430 .hi = \
431 ((iova) & ~0xFFFULL) | ((am) & IOTLB_INVAL_ADDR_MASK_FIELD_MASK)})
432
433#define IOTLB_REG_INVALIDATE BIT(63) /* IVT - triggers invalidation */
434#define IOTLB_REG_DRAIN_READS BIT(49) /* DR */
435#define IOTLB_REG_DRAIN_WRITES BIT(48) /* DW */
436#define IOTLB_REG_GLOBAL (1ULL << 60) /* IIRG = 01 */
437#define IOTLB_REG_DOMAIN (2ULL << 60) /* IIRG = 10 */
438#define IOTLB_REG_PAGE (3ULL << 60) /* IIRG = 11 */
439#define IOTLB_REG_DOMAIN_ID_SHIFT 32
440
441#define IVA_REG_ADDR_MASK (~0xFFFULL)
442#define IVA_REG_HINT BIT(6) /* IH */
443#define IVA_REG_AM_SHIFT 0 /* address mask */
444
445/* Wait descriptor */
446#define WAIT_DESC_INTERRUPT_FLAG BIT(4) /* IF */
447#define WAIT_DESC_STATUS_WRITE BIT(5) /* SW */
448#define WAIT_DESC_FENCE BIT(6) /* FN */
449#define WAIT_DESC_DATA_SHIFT 32
450
451#define WAIT_DESC(status_phys, status_data) \
452 ((struct vtd_inv_desc) { \
453 .lo = INVAL_DESC_TYPE_WAIT | WAIT_DESC_STATUS_WRITE | \
454 WAIT_DESC_FENCE | \
455 ((uint64_t) (status_data) << WAIT_DESC_DATA_SHIFT), \
456 .hi = (status_phys)})
457
458struct vtd_inv_desc {
459 uint64_t lo;
460 uint64_t hi;
461} __packed;
462
463_Static_assert(sizeof(struct vtd_inv_desc) == 16, "inv desc must be 16B");
464
465struct iommu *vtd_unit_create(uint64_t base_phys, uint16_t segment,
466 uint8_t size_field);
467
468/* ND encoding to actual domain count */
469static inline uint32_t vtd_cap_domain_count(uint64_t cap) {
470 static const uint32_t nd_table[] = {16, 64, 256, 1024,
471 4096, 16384, 65536, 0};
472 return nd_table[CAP_NUM_DOMAINS(cap)];
473}
474