1#include <drivers/mmio.h>
2
3#include "internal.h"
4
5static uint64_t *vtd_iotlb_reg(struct vtd_unit *u) {
6 return (uint64_t *) ((uintptr_t) u->regs + IOTLB_REG_OFFSET(u->ecap));
7}
8
9static uint64_t *vtd_iva_reg(struct vtd_unit *u) {
10 return (uint64_t *) ((uintptr_t) u->regs + IVA_REG_OFFSET(u->ecap));
11}
12
13static void vtd_iotlb_reg_flush(struct vtd_unit *u, uint64_t val) {
14 mmio_write_64(address: vtd_iotlb_reg(u), value: val | IOTLB_REG_INVALIDATE);
15
16 while (mmio_read_64(address: vtd_iotlb_reg(u)) & IOTLB_REG_INVALIDATE)
17 cpu_relax();
18}
19
20void vtd_iotlb_flush_global(struct vtd_unit *u) {
21 if (ECAP_QUEUED_INVALIDATION(u->ecap)) {
22 vtd_iq_submit(u, IOTLB_INVAL_DESC_GLOBAL);
23 vtd_iq_flush(u);
24 return;
25 }
26
27 vtd_iotlb_reg_flush(u, IOTLB_REG_GLOBAL | IOTLB_REG_DRAIN_READS |
28 IOTLB_REG_DRAIN_WRITES);
29}
30
31void vtd_iotlb_flush_domain(struct vtd_unit *u, uint16_t domain_id) {
32 if (ECAP_QUEUED_INVALIDATION(u->ecap)) {
33 vtd_iq_submit(u, IOTLB_INVAL_DESC_DOMAIN(domain_id));
34 vtd_iq_flush(u);
35 return;
36 }
37
38 vtd_iotlb_reg_flush(
39 u, IOTLB_REG_DOMAIN | IOTLB_REG_DRAIN_READS | IOTLB_REG_DRAIN_WRITES |
40 ((uint64_t) domain_id << IOTLB_REG_DOMAIN_ID_SHIFT));
41}
42
43void vtd_iotlb_flush_page(struct vtd_unit *u, uint16_t domain_id, iova_t iova,
44 uint8_t am) {
45 if (ECAP_QUEUED_INVALIDATION(u->ecap)) {
46 vtd_iq_submit(u, IOTLB_INVAL_DESC_PAGE(domain_id, iova, am));
47 vtd_iq_flush(u);
48 return;
49 }
50
51 mmio_write_64(address: vtd_iva_reg(u),
52 value: (iova & IVA_REG_ADDR_MASK) | IVA_REG_HINT | am);
53
54 vtd_iotlb_reg_flush(
55 u, IOTLB_REG_PAGE | IOTLB_REG_DRAIN_READS | IOTLB_REG_DRAIN_WRITES |
56 ((uint64_t) domain_id << IOTLB_REG_DOMAIN_ID_SHIFT));
57}
58
59void vtd_iotlb_flush_range(struct vtd_unit *u, uint16_t domain_id, iova_t iova,
60 size_t size) {
61 if (CAP_PAGE_SELECTIVE_INVALIDATION(u->cap) && size <= 32 * PAGE_SIZE) {
62 for (size_t off = 0; off < size; off += PAGE_SIZE)
63 vtd_iotlb_flush_page(u, domain_id, iova: iova + off, am: 0);
64 return;
65 }
66
67 vtd_iotlb_flush_domain(u, domain_id);
68}
69
70void vtd_iotlb_flush_range_batched(struct vtd_unit *u, uint16_t domain_id,
71 iova_t iova, size_t size) {
72 if (!ECAP_QUEUED_INVALIDATION(u->ecap)) {
73 vtd_iotlb_flush_range(u, domain_id, iova, size);
74 return;
75 }
76
77 if (CAP_PAGE_SELECTIVE_INVALIDATION(u->cap) && size <= 32 * PAGE_SIZE) {
78 for (size_t off = 0; off < size; off += PAGE_SIZE)
79 vtd_iq_submit(u, IOTLB_INVAL_DESC_PAGE(domain_id, iova + off, 0));
80 } else {
81 vtd_iq_submit(u, IOTLB_INVAL_DESC_DOMAIN(domain_id));
82 }
83
84 vtd_iq_flush(u);
85}
86