1/* @title: VT-d */
2#pragma once
3#include <compiler/core.h>
4#include <console/panic.h>
5#include <console/printf.h>
6#include <drivers/iommu/iommu.h>
7#include <log.h>
8#include <math/bit.h>
9#include <sync/spinlock.h>
10LOG_SITE_EXTERN(vtd);
11LOG_HANDLE_EXTERN(vtd);
12
13#define vtd_log(lvl, fmt, ...) \
14 log(LOG_SITE(vtd), LOG_HANDLE(vtd), lvl, fmt, ##__VA_ARGS__)
15
16#define vtd_err(fmt, ...) vtd_log(LOG_ERROR, fmt, ##__VA_ARGS__)
17#define vtd_warn(fmt, ...) vtd_log(LOG_WARN, fmt, ##__VA_ARGS__)
18#define vtd_info(fmt, ...) vtd_log(LOG_INFO, fmt, ##__VA_ARGS__)
19#define vtd_debug(fmt, ...) vtd_log(LOG_DEBUG, fmt, ##__VA_ARGS__)
20#define vtd_trace(fmt, ...) vtd_log(LOG_TRACE, fmt, ##__VA_ARGS__)
21
22/*
23 * Register-name prefixes used below mirror the names in the Intel VT-d spec
24 * so that bit-field macros can be cross-referenced with the spec text.
25 * CAP - Capability Register
26 * ECAP - Extended Capability Register
27 * GCMD - Global Command Register
28 * GSTS - Global Status Register
29 * RTADDR - Root Table Address Register
30 * FSTS - Fault Status Register
31 * IQA - Invalidation Queue Address Register
32 */
33
34struct vtd_regs {
35 /* NOTE: offsets and sizes in kernel/drivers/iommu/vt_d/layout.c */
36 uint32_t version; /* Version */
37 uint32_t _res0;
38 uint64_t capabilities; /* Capability */
39 uint64_t extended_capabilities; /* Extended Capability */
40 uint32_t global_command; /* Global Command */
41 uint32_t global_status; /* Global Status */
42 uint64_t root_table_addr; /* Root Table Address */
43 uint64_t context_command; /* Context Command */
44 uint32_t _res1;
45 uint32_t fault_status; /* Fault Status */
46 uint32_t fault_event_control; /* Fault Event Control */
47 uint32_t fault_event_data; /* Fault Event Data */
48 uint32_t fault_event_addr; /* Fault Event Address */
49 uint32_t fault_event_addr_upper; /* Fault Event Upper Address*/
50 uint64_t _res2[2];
51 uint64_t advanced_fault_log; /* Advanced Fault Log */
52 uint32_t _res3;
53 uint32_t protected_memory_enable; /* Protected Memory Enable */
54 uint32_t protected_low_mem_base; /* Protected Low Mem Base */
55 uint32_t protected_low_mem_limit; /* Protected Low Mem Limit */
56 uint64_t protected_high_mem_base; /* Protected High Mem Base */
57 uint64_t protected_high_mem_limit; /* Protected High Mem Limit */
58 uint64_t invalidation_queue_head; /* Invalidation Queue Head */
59 uint64_t invalidation_queue_tail; /* Invalidation Queue Tail */
60 uint64_t invalidation_queue_addr; /* Invalidation Queue Addr */
61 uint32_t _res4;
62 uint32_t invalidation_comp_status; /* Invalidation Completion Status */
63 uint32_t invalidation_event_ctrl; /* Invalidation Event Control */
64 uint32_t invalidation_event_data; /* Invalidation Event Data */
65 uint32_t invalidation_event_addr; /* Invalidation Event Address */
66 uint32_t invalidation_event_addr_upper; /* Invalidation Event Upper
67 Addr */
68 uint64_t invalidation_queue_error_record; /* IQ Error Record */
69 uint64_t interrupt_remapping_table_addr; /* IR Table Address */
70} cc_packed;
71
72struct vtd_invl_queue {};
73
74struct vtd_unit {
75 struct vtd_regs cc_mem_io *regs;
76 uint64_t cap;
77 uint64_t ecap;
78 uint16_t segment;
79 uint32_t domain_count;
80 uint8_t *domain_bitmap;
81 struct spinlock domain_bitmap_lock;
82
83 uint64_t *root_table;
84 paddr_t root_table_phys;
85
86 void *iq_base;
87 paddr_t iq_phys;
88 uint32_t iq_head;
89 uint32_t iq_tail;
90 uint32_t iq_size;
91};
92
93struct vtd_domain {
94 uint16_t domain_id;
95 uint64_t *sl_pgd;
96 uint64_t sl_pgd_phys;
97};
98
99/*
100 * CAP - Capability Register (64-bit, offset 0x008)
101 * Intel VT-d spec 11.4.2
102 */
103#define CAP_NUM_DOMAINS(cap) \
104 BIT_RANGE((cap), 0, 2) /* ND - Number of Domains supported (encoded) \
105 */
106#define CAP_REQUIRES_WRITE_BUF_FLUSH(cap) \
107 BIT_RANGE((cap), 4, 4) /* RWBF - Required Write-Buffer Flushing */
108#define CAP_PROTECTED_LOW_MEMORY(cap) \
109 BIT_RANGE((cap), 5, 5) /* PLMR - Protected Low-Memory Region */
110#define CAP_PROTECTED_HIGH_MEMORY(cap) \
111 BIT_RANGE((cap), 6, 6) /* PHMR - Protected High-Memory Region */
112#define CAP_CACHING_MODE(cap) BIT_RANGE((cap), 7, 7) /* CM */
113#define CAP_SUPPORTED_ADDR_WIDTHS(cap) \
114 BIT_RANGE((cap), 8, \
115 12) /* SAGAW - Supported Adjusted Guest Address Widths */
116#define CAP_MAX_ADDR_WIDTH(cap) \
117 BIT_RANGE((cap), 16, 21) /* MGAW - Maximum Guest Address Width */
118#define CAP_ZERO_LENGTH_READ(cap) BIT_RANGE((cap), 22, 22) /* ZLR */
119#define CAP_FAULT_RECORD_OFFSET(cap) \
120 BIT_RANGE((cap), 24, 33) /* FRO - Fault-Recording Register Offset */
121#define CAP_LARGE_PAGE_SUPPORT(cap) \
122 BIT_RANGE((cap), 34, 37) /* SLLPS - Second-Level Large Page Support */
123#define CAP_PAGE_SELECTIVE_INVALIDATION(cap) \
124 BIT_RANGE((cap), 39, 39) /* PSI \
125 */
126#define CAP_NUM_FAULT_RECORDS(cap) \
127 BIT_RANGE((cap), 40, 47) /* NFR - Number of Fault-Recording Registers \
128 */
129#define CAP_MAX_ADDR_MASK_VALUE(cap) BIT_RANGE((cap), 48, 53) /* MAMV */
130#define CAP_WRITE_DRAINING(cap) BIT_RANGE((cap), 54, 54) /* DWD */
131#define CAP_READ_DRAINING(cap) BIT_RANGE((cap), 55, 55) /* DRD */
132#define CAP_FIRST_LEVEL_1GB_PAGE(cap) BIT_RANGE((cap), 56, 56) /* FL1GP */
133#define CAP_POSTED_INTERRUPTS(cap) BIT_RANGE((cap), 59, 59) /* PI */
134#define CAP_FIRST_STAGE_5LEVEL_PAGING(cap) \
135 BIT_RANGE((cap), 60, 60) /* FL5LP / FS5LP */
136#define CAP_ENHANCED_COMMAND_SUPPORT(cap) BIT_RANGE((cap), 61, 61) /* ECMDS */
137#define CAP_ENHANCED_SET_INTR_REMAP_TABLE_PTR_SUPPORT(cap) \
138 BIT_RANGE( \
139 (cap), 62, \
140 62) /* ESIRTPS - Enhanced Set Interrupt Remap Table Pointer Support */
141#define CAP_ENHANCED_SET_ROOT_TABLE_PTR_SUPPORT(cap) \
142 BIT_RANGE((cap), 63, \
143 63) /* ESRTPS - Enhanced Set Root Table Pointer Support */
144
145/*
146 * ECAP - Extended Capability Register (64-bit, offset 0x010)
147 * Intel VT-d spec 11.4.3
148 */
149#define ECAP_COHERENCY(ecap) BIT_RANGE((ecap), 0, 0) /* C */
150#define ECAP_QUEUED_INVALIDATION(ecap) BIT_RANGE((ecap), 1, 1) /* QI */
151#define ECAP_DEVICE_TLB(ecap) BIT_RANGE((ecap), 2, 2) /* DT */
152#define ECAP_INTERRUPT_REMAPPING(ecap) BIT_RANGE((ecap), 3, 3) /* IR */
153#define ECAP_EXTENDED_INTERRUPT_MODE(ecap) BIT_RANGE((ecap), 4, 4) /* EIM */
154#define ECAP_PASS_THROUGH(ecap) BIT_RANGE((ecap), 6, 6) /* PT */
155#define ECAP_SNOOP_CONTROL(ecap) BIT_RANGE((ecap), 7, 7) /* SC */
156#define ECAP_IOTLB_REGISTER_OFFSET(ecap) BIT_RANGE((ecap), 8, 17) /* IRO */
157#define ECAP_MAX_HANDLE_MASK_VALUE(ecap) BIT_RANGE((ecap), 20, 23) /* MHMV */
158#define ECAP_MEMORY_TYPE_SUPPORT(ecap) BIT_RANGE((ecap), 25, 25) /* MTS */
159#define ECAP_NESTED_TRANSLATION(ecap) BIT_RANGE((ecap), 26, 26) /* NEST */
160#define ECAP_PAGE_REQUEST_SUPPORT(arg) BIT_TEST((arg), 29) /* PRS */
161#define ECAP_EXECUTE_REQUEST_SUPPORT(arg) BIT_TEST((arg), 30) /* ERS */
162#define ECAP_SUPERVISOR_REQUEST_SUPPORT(arg) BIT_TEST((arg), 31) /* SRS */
163#define ECAP_NO_WRITE_FLAG_SUPPORT(arg) BIT_TEST((arg), 33) /* NWFS */
164#define ECAP_EXTENDED_ACCESSED_FLAG_SUPPORT(arg) \
165 BIT_TEST((arg), 34) /* EAFS \
166 */
167#define ECAP_PASID_SIZE_SUPPORTED(arg) \
168 BIT_RANGE((arg), 35, 39) /* PSS - Process Address Space ID size */
169#define ECAP_PASID_SUPPORT(arg) BIT_TEST((arg), 40) /* PASID */
170#define ECAP_DEVICE_TLB_INVAL_THROTTLE(arg) \
171 BIT_TEST((arg), 41) /* DIT \
172 */
173#define ECAP_PAGE_REQUEST_DRAIN_SUPPORT(arg) BIT_TEST((arg), 42) /* PDS */
174#define ECAP_SCALABLE_MODE_TRANSLATION_SUPPORT(arg) \
175 BIT_TEST((arg), 43) /* SMTS */
176#define ECAP_VIRTUAL_COMMAND_SUPPORT(arg) BIT_TEST((arg), 44) /* VCS */
177#define ECAP_SECOND_STAGE_ACCESSED_DIRTY_SUPPORT(arg) \
178 BIT_TEST((arg), 45) /* SSADS */
179#define ECAP_SECOND_STAGE_TRANSLATION_SUPPORT(arg) \
180 BIT_TEST((arg), 46) /* SSTS */
181#define ECAP_FIRST_STAGE_TRANSLATION_SUPPORT(arg) \
182 BIT_TEST((arg), 47) /* FSTS (do not confuse with Fault Status register) */
183#define ECAP_SCALABLE_MODE_PAGE_WALK_COHERENCY_SUPPORT(arg) \
184 BIT_TEST((arg), 48) /* SMPWCS */
185#define ECAP_RID_PASID_SUPPORT(arg) \
186 BIT_TEST((arg), 49) /* RPS - Requester-ID/PASID */
187#define ECAP_PERFORMANCE_MONITORING_SUPPORT(arg) BIT_TEST((arg), 51) /* PMS */
188#define ECAP_ABORT_DMA_MODE_SUPPORT(arg) BIT_TEST((arg), 52) /* ADMS */
189#define ECAP_RID_PRIV_SUPPORT(arg) \
190 BIT_TEST((arg), 53) /* RPRIVS - Requester-ID Privilege */
191#define ECAP_STOP_MARKER_SUPPORT(arg) BIT_TEST((arg), 58) /* SMS */
192
193/*
194 * Bit positions in CAP_SUPPORTED_ADDR_WIDTHS (CAP.SAGAW)
195 * Each bit indicates a supported guest address width / page-table depth.
196 */
197#define ADDR_WIDTH_39BIT BIT(1) /* 3-level page table */
198#define ADDR_WIDTH_48BIT BIT(2) /* 4-level page table */
199#define ADDR_WIDTH_57BIT BIT(3) /* 5-level page table */
200
201/*
202 * GCMD - Global Command Register (32-bit, offset 0x018)
203 * GSTS - Global Status Register (32-bit, offset 0x01C)
204 * Intel VT-d spec 11.4.4, 11.4.5
205 */
206#define GCMD_TRANSLATION_ENABLE BIT(31) /* TE */
207#define GCMD_SET_ROOT_TABLE_PTR BIT(30) /* SRTP - Set Root Table Pointer */
208#define GCMD_WRITE_BUFFER_FLUSH BIT(27) /* WBF */
209#define GCMD_QUEUED_INVAL_ENABLE BIT(26) /* QIE */
210#define GCMD_INTERRUPT_REMAP_ENABLE BIT(25) /* IRE */
211#define GCMD_SET_INTR_REMAP_TABLE_PTR \
212 BIT(24) /* SIRTP - Set Interrupt Remap Table Pointer */
213#define GCMD_COMPAT_FORMAT_INTERRUPT \
214 BIT(23) /* CFI - Compatibility Format Interrupt */
215
216#define GSTS_TRANSLATION_ENABLED BIT(31) /* TES */
217#define GSTS_ROOT_TABLE_PTR_SET BIT(30) /* RTPS */
218#define GSTS_WRITE_BUFFER_FLUSH_STATUS BIT(27) /* WBFS */
219#define GSTS_QUEUED_INVAL_ENABLED BIT(26) /* QIES */
220#define GSTS_INTERRUPT_REMAP_ENABLED BIT(25) /* IRES */
221#define GSTS_INTR_REMAP_TABLE_PTR_SET BIT(24) /* IRTPS */
222#define GSTS_COMPAT_FORMAT_INTERRUPT_STATUS BIT(23) /* CFIS */
223
224/*
225 * RTADDR - Root Table Address Register (64-bit, offset 0x020)
226 * TTM = Translation Table Mode (bits 10-11).
227 */
228#define RTADDR_TRANSLATION_MODE_LEGACY (0ULL << 10)
229#define RTADDR_TRANSLATION_MODE_SCALABLE BIT(10)
230
231/*
232 * FSTS - Fault Status Register (32-bit, offset 0x034)
233 */
234#define FSTS_PRIMARY_FAULT_OVERFLOW BIT(0) /* PFO */
235#define FSTS_PRIMARY_PENDING_FAULT BIT(1) /* PPF */
236#define FSTS_ADVANCED_FAULT_OVERFLOW BIT(2) /* AFO */
237#define FSTS_ADVANCED_PENDING_FAULT BIT(3) /* APF */
238#define FSTS_INVAL_QUEUE_ERROR BIT(4) /* IQE */
239#define FSTS_INVAL_COMPLETION_ERROR BIT(5) /* ICE */
240#define FSTS_INVAL_TIMEOUT_ERROR BIT(6) /* ITE */
241#define FSTS_PAGE_REQUEST_OVERFLOW BIT(7) /* PRO */
242#define FSTS_FAULT_RECORD_INDEX(s) BIT_RANGE((s), 8, 15) /* FRI */
243
244/*
245 * Root Entry (128-bit / two 64-bit words) - Intel VT-d spec 9.1
246 * CTP = Context-Table Pointer (bits 12-63 of the lo word).
247 */
248#define ROOT_ENTRY_PRESENT BIT(0)
249#define ROOT_ENTRY_CONTEXT_TABLE_PTR_SHIFT 12
250#define ROOT_ENTRY_CONTEXT_TABLE_PTR_MASK (~0xFFFULL)
251
252#define ROOT_ENTRY_SET_CONTEXT_TABLE_PTR(phys) \
253 (((phys) & ROOT_ENTRY_CONTEXT_TABLE_PTR_MASK) | ROOT_ENTRY_PRESENT)
254
255struct vtd_root_entry {
256 uint64_t lo;
257 uint64_t hi; /* must be zero */
258} cc_packed;
259
260/*
261 * Context Entry (128-bit / two 64-bit words) - Intel VT-d spec 9.3
262 * Field acronyms used as suffixes:
263 * FPD - Fault Processing Disable
264 * TT - Translation Type
265 * SLPTPTR - Second-Level Page-Table Pointer
266 * AW - Address Width
267 * DID - Domain Identifier
268 */
269#define CTX_ENTRY_PRESENT BIT(0)
270#define CTX_ENTRY_FAULT_PROCESSING_DISABLE BIT(1)
271#define CTX_ENTRY_TRANSLATION_TYPE_SHIFT 2
272#define CTX_ENTRY_TRANSLATION_TYPE_UNTRANSLATED (0ULL << 2)
273#define CTX_ENTRY_TRANSLATION_TYPE_PASSTHROUGH (2ULL << 2)
274#define CTX_ENTRY_SL_PTR_SHIFT 12
275#define CTX_ENTRY_SL_PTR_MASK (~0xFFFULL)
276
277#define CTX_ENTRY_ADDR_WIDTH_SHIFT 0
278#define CTX_ENTRY_ADDR_WIDTH_39BIT 1
279#define CTX_ENTRY_ADDR_WIDTH_48BIT 2
280#define CTX_ENTRY_ADDR_WIDTH_57BIT 3
281
282#define CTX_ENTRY_DOMAIN_ID_SHIFT 8
283
284#define CTX_ENTRY_SET_LO(slptptr, tt) \
285 (((slptptr) & CTX_ENTRY_SL_PTR_MASK) | (tt) | CTX_ENTRY_PRESENT)
286#define CTX_ENTRY_SET_HI(domain_id, aw) \
287 (((uint64_t) (domain_id) << CTX_ENTRY_DOMAIN_ID_SHIFT) | (aw))
288
289/* Extract fields from a context entry */
290#define CTX_ENTRY_TRANSLATION_TYPE(lo) BIT_RANGE((lo), 2, 3)
291#define CTX_ENTRY_DOMAIN_ID(hi) BIT_RANGE((hi), 8, 23)
292#define CTX_ENTRY_ADDR_WIDTH(hi) BIT_RANGE((hi), 0, 2)
293
294struct vtd_context_entry {
295 uint64_t lo;
296 uint64_t hi;
297} cc_packed;
298
299/*
300 * Second-Level Page-Table Entry (PTE).
301 */
302#define SL_PTE_READ BIT(0)
303#define SL_PTE_WRITE BIT(1)
304#define SL_PTE_PRESENT (SL_PTE_READ | SL_PTE_WRITE)
305#define SL_PTE_EXECUTE BIT(2)
306#define SL_PTE_MEMORY_TYPE_SHIFT 3 /* EMT - Extended Memory Type */
307#define SL_PTE_MEMORY_TYPE_WRITE_BACK (6ULL << 3) /* WB */
308#define SL_PTE_MEMORY_TYPE_UNCACHEABLE (0ULL << 3) /* UC */
309#define SL_PTE_IGNORE_PAT BIT(6) /* IPAT - Ignore Page Attribute Table */
310#define SL_PTE_LARGE_PAGE BIT(7) /* PS - Page Size */
311#define SL_PTE_SNOOP BIT(11) /* SNP */
312#define SL_PTE_ADDR_MASK (~0xFFFULL & (BIT(52) - 1))
313
314#define SL_PTE_ADDR(pte) ((pte) & SL_PTE_ADDR_MASK)
315#define SL_PTE_MEMORY_TYPE(pte) BIT_RANGE((pte), 3, 5)
316#define SL_PTE_IS_LARGE(pte) BIT_RANGE((pte), 7, 7)
317
318#define SL_TABLE_ENTRY(phys) (((phys) & SL_PTE_ADDR_MASK) | SL_PTE_PRESENT)
319
320#define SL_PAGE_ENTRY(phys, perm) \
321 (((phys) & SL_PTE_ADDR_MASK) | SL_PTE_MEMORY_TYPE_WRITE_BACK | \
322 SL_PTE_IGNORE_PAT | ((perm) & 0x3))
323
324/*
325 * IOVA index helpers, identical breakdown to x86-64 VA.
326 * PML4 / PDPT / PD / PT are the standard x86-64 page-table levels.
327 */
328#define SL_PML4_INDEX(iova) (((iova) >> 39) & 0x1FF)
329#define SL_PDPT_INDEX(iova) (((iova) >> 30) & 0x1FF)
330#define SL_PD_INDEX(iova) (((iova) >> 21) & 0x1FF)
331#define SL_PT_INDEX(iova) (((iova) >> 12) & 0x1FF)
332#define SL_PAGE_OFFSET(iova) ((iova) & 0xFFF)
333#define SL_ENTRY_COUNT 512
334
335/*
336 * IQA - Invalidation Queue Address Register, and queue layout
337 * Intel VT-d spec 6.5
338 */
339#define IQA_SIZE_SHIFT 0
340#define IQA_SIZE_256_ENTRIES 0
341#define IQA_SIZE_512_ENTRIES 1
342#define IQA_SIZE_1K_ENTRIES 2
343#define IQA_SIZE_2K_ENTRIES 3
344#define IQA_SIZE_4K_ENTRIES 4
345#define IQA_SIZE_8K_ENTRIES 5
346#define IQA_SIZE_16K_ENTRIES 6
347#define IQA_SIZE_32K_ENTRIES 7
348#define IQA_ADDR_MASK (~0xFFFULL)
349
350#define IQA_REG_BUILD(phys, size_enc) (((phys) & IQA_ADDR_MASK) | (size_enc))
351#define IQA_SIZE(iqa) BIT_RANGE((iqa), 0, 2)
352
353#define INVAL_QUEUE_ENTRY_SIZE 16
354#define INVAL_QUEUE_TAIL_SHIFT 4
355
356/* Invalidation descriptor type codes (low 4 bits of descriptor.lo) */
357#define INVAL_DESC_TYPE_CONTEXT_CACHE 0x1
358#define INVAL_DESC_TYPE_IOTLB \
359 0x2 /* I/O Translation Lookaside Buffer \
360 */
361#define INVAL_DESC_TYPE_DEVICE_IOTLB 0x3
362#define INVAL_DESC_TYPE_INTERRUPT_ENTRY_CACHE 0x4 /* IEC */
363#define INVAL_DESC_TYPE_WAIT 0x5
364
365#define INVAL_DESC_TYPE(lo) BIT_RANGE((lo), 0, 3)
366#define INVAL_DESC_GRANULARITY(lo) BIT_RANGE((lo), 4, 5)
367
368/* Context-cache invalidation descriptor */
369#define CTX_INVAL_GRANULARITY_GLOBAL BIT(4)
370#define CTX_INVAL_GRANULARITY_DOMAIN (2ULL << 4)
371#define CTX_INVAL_GRANULARITY_DEVICE (3ULL << 4)
372#define CTX_INVAL_DOMAIN_ID_SHIFT 48
373
374#define CTX_INVAL_DESC_GLOBAL \
375 ((struct vtd_inv_desc) {.lo = INVAL_DESC_TYPE_CONTEXT_CACHE | \
376 CTX_INVAL_GRANULARITY_GLOBAL, \
377 .hi = 0})
378#define CTX_INVAL_DESC_DOMAIN(did) \
379 ((struct vtd_inv_desc) { \
380 .lo = INVAL_DESC_TYPE_CONTEXT_CACHE | CTX_INVAL_GRANULARITY_DOMAIN | \
381 ((uint64_t) (did) << CTX_INVAL_DOMAIN_ID_SHIFT), \
382 .hi = 0})
383
384/* IOTLB invalidation descriptor */
385#define IOTLB_INVAL_GRANULARITY_GLOBAL BIT(4)
386#define IOTLB_INVAL_GRANULARITY_DOMAIN (2ULL << 4)
387#define IOTLB_INVAL_GRANULARITY_PAGE (3ULL << 4)
388#define IOTLB_INVAL_DRAIN_READS BIT(7) /* DR */
389#define IOTLB_INVAL_DRAIN_WRITES BIT(6) /* DW */
390#define IOTLB_INVAL_DOMAIN_ID_SHIFT 32
391#define IOTLB_INVAL_IOVA_SHIFT 12
392#define IOTLB_INVAL_ADDR_MASK_FIELD_MASK \
393 0x3FULL /* AM (Address Mask) field width */
394#define IOTLB_INVAL_HINT \
395 BIT(6) /* IH (in hi dword): non-leaf entries unchanged */
396
397#define IOTLB_INVAL_DOMAIN_ID(lo) BIT_RANGE((lo), 32, 47)
398#define IOTLB_INVAL_GRANULARITY(lo) BIT_RANGE((lo), 4, 5)
399#define IOTLB_REG_OFFSET(ecap) ((ECAP_IOTLB_REGISTER_OFFSET(ecap) << 4) + 8)
400#define IVA_REG_OFFSET(ecap) ((ECAP_IOTLB_REGISTER_OFFSET(ecap) << 4) + 0)
401#define IOTLB_INVAL_DESC_GLOBAL \
402 ((struct vtd_inv_desc) { \
403 .lo = INVAL_DESC_TYPE_IOTLB | IOTLB_INVAL_GRANULARITY_GLOBAL | \
404 IOTLB_INVAL_DRAIN_READS | IOTLB_INVAL_DRAIN_WRITES, \
405 .hi = 0})
406#define IOTLB_INVAL_DESC_DOMAIN(did) \
407 ((struct vtd_inv_desc) { \
408 .lo = INVAL_DESC_TYPE_IOTLB | IOTLB_INVAL_GRANULARITY_DOMAIN | \
409 IOTLB_INVAL_DRAIN_READS | IOTLB_INVAL_DRAIN_WRITES | \
410 ((uint64_t) (did) << IOTLB_INVAL_DOMAIN_ID_SHIFT), \
411 .hi = 0})
412#define IOTLB_INVAL_DESC_PAGE(did, iova, am) \
413 ((struct vtd_inv_desc) { \
414 .lo = INVAL_DESC_TYPE_IOTLB | IOTLB_INVAL_GRANULARITY_PAGE | \
415 IOTLB_INVAL_DRAIN_READS | IOTLB_INVAL_DRAIN_WRITES | \
416 ((uint64_t) (did) << IOTLB_INVAL_DOMAIN_ID_SHIFT), \
417 .hi = \
418 ((iova) & ~0xFFFULL) | ((am) & IOTLB_INVAL_ADDR_MASK_FIELD_MASK)})
419
420#define IOTLB_REG_INVALIDATE BIT(63) /* IVT - triggers invalidation */
421#define IOTLB_REG_DRAIN_READS BIT(49) /* DR */
422#define IOTLB_REG_DRAIN_WRITES BIT(48) /* DW */
423#define IOTLB_REG_GLOBAL BIT(60) /* IIRG = 01 */
424#define IOTLB_REG_DOMAIN (2ULL << 60) /* IIRG = 10 */
425#define IOTLB_REG_PAGE (3ULL << 60) /* IIRG = 11 */
426#define IOTLB_REG_DOMAIN_ID_SHIFT 32
427
428#define IVA_REG_ADDR_MASK (~0xFFFULL)
429#define IVA_REG_HINT BIT(6) /* IH */
430#define IVA_REG_AM_SHIFT 0 /* address mask */
431
432/* Wait descriptor */
433#define WAIT_DESC_INTERRUPT_FLAG BIT(4) /* IF */
434#define WAIT_DESC_STATUS_WRITE BIT(5) /* SW */
435#define WAIT_DESC_FENCE BIT(6) /* FN */
436#define WAIT_DESC_DATA_SHIFT 32
437
438#define WAIT_DESC(status_phys, status_data) \
439 ((struct vtd_inv_desc) { \
440 .lo = INVAL_DESC_TYPE_WAIT | WAIT_DESC_STATUS_WRITE | \
441 WAIT_DESC_FENCE | \
442 ((uint64_t) (status_data) << WAIT_DESC_DATA_SHIFT), \
443 .hi = (status_phys)})
444
445struct vtd_inv_desc {
446 uint64_t lo;
447 uint64_t hi;
448} cc_packed;
449
450struct iommu *vtd_unit_create(uint64_t base_phys, uint16_t segment,
451 uint8_t size_field);
452
453/* ND encoding to actual domain count */
454static inline uint32_t vtd_cap_domain_count(uint64_t cap) {
455 static const uint32_t nd_table[] = {16, 64, 256, 1024,
456 4096, 16384, 65536, 0};
457 return nd_table[CAP_NUM_DOMAINS(cap)];
458}
459