1#include <drivers/mmio.h>
2
3#include "internal.h"
4
5static uint64_t cc_mem_io *vtd_iotlb_reg(struct vtd_unit *u) {
6 return (uint64_t cc_mem_io *) ((uintptr_t) u->regs +
7 IOTLB_REG_OFFSET(u->ecap));
8}
9
10static uint64_t cc_mem_io *vtd_iva_reg(struct vtd_unit *u) {
11 return (uint64_t cc_mem_io *) ((uintptr_t) u->regs +
12 IVA_REG_OFFSET(u->ecap));
13}
14
15static void vtd_iotlb_reg_flush(struct vtd_unit *u, uint64_t val) {
16 mmio_write_64(address: vtd_iotlb_reg(u), value: val | IOTLB_REG_INVALIDATE);
17
18 while (mmio_read_64(address: vtd_iotlb_reg(u)) & IOTLB_REG_INVALIDATE)
19 cpu_pause();
20}
21
22void vtd_iotlb_flush_global(struct vtd_unit *u) {
23 if (ECAP_QUEUED_INVALIDATION(u->ecap)) {
24 vtd_iq_submit(u, IOTLB_INVAL_DESC_GLOBAL);
25 vtd_iq_flush(u);
26 return;
27 }
28
29 vtd_iotlb_reg_flush(u, IOTLB_REG_GLOBAL | IOTLB_REG_DRAIN_READS |
30 IOTLB_REG_DRAIN_WRITES);
31}
32
33void vtd_iotlb_flush_domain(struct vtd_unit *u, uint16_t domain_id) {
34 if (ECAP_QUEUED_INVALIDATION(u->ecap)) {
35 vtd_iq_submit(u, IOTLB_INVAL_DESC_DOMAIN(domain_id));
36 vtd_iq_flush(u);
37 return;
38 }
39
40 vtd_iotlb_reg_flush(
41 u, IOTLB_REG_DOMAIN | IOTLB_REG_DRAIN_READS | IOTLB_REG_DRAIN_WRITES |
42 ((uint64_t) domain_id << IOTLB_REG_DOMAIN_ID_SHIFT));
43}
44
45void vtd_iotlb_flush_page(struct vtd_unit *u, uint16_t domain_id, iova_t iova,
46 uint8_t am) {
47 if (ECAP_QUEUED_INVALIDATION(u->ecap)) {
48 vtd_iq_submit(u, IOTLB_INVAL_DESC_PAGE(domain_id, iova, am));
49 vtd_iq_flush(u);
50 return;
51 }
52
53 mmio_write_64(address: vtd_iva_reg(u),
54 value: (iova & IVA_REG_ADDR_MASK) | IVA_REG_HINT | am);
55
56 vtd_iotlb_reg_flush(
57 u, IOTLB_REG_PAGE | IOTLB_REG_DRAIN_READS | IOTLB_REG_DRAIN_WRITES |
58 ((uint64_t) domain_id << IOTLB_REG_DOMAIN_ID_SHIFT));
59}
60
61void vtd_iotlb_flush_range(struct vtd_unit *u, uint16_t domain_id, iova_t iova,
62 size_t size) {
63 if (CAP_PAGE_SELECTIVE_INVALIDATION(u->cap) && size <= 32 * PAGE_SIZE) {
64 for (size_t off = 0; off < size; off += PAGE_SIZE)
65 vtd_iotlb_flush_page(u, domain_id, iova: iova + off, am: 0);
66 return;
67 }
68
69 vtd_iotlb_flush_domain(u, domain_id);
70}
71
72void vtd_iotlb_flush_range_batched(struct vtd_unit *u, uint16_t domain_id,
73 iova_t iova, size_t size) {
74 if (!ECAP_QUEUED_INVALIDATION(u->ecap)) {
75 vtd_iotlb_flush_range(u, domain_id, iova, size);
76 return;
77 }
78
79 if (CAP_PAGE_SELECTIVE_INVALIDATION(u->cap) && size <= 32 * PAGE_SIZE) {
80 for (size_t off = 0; off < size; off += PAGE_SIZE)
81 vtd_iq_submit(u, IOTLB_INVAL_DESC_PAGE(domain_id, iova + off, 0));
82 } else {
83 vtd_iq_submit(u, IOTLB_INVAL_DESC_DOMAIN(domain_id));
84 }
85
86 vtd_iq_flush(u);
87}
88