1#include <acpi/lapic.h>
2#include <mem/alloc.h>
3#include <mem/page.h>
4#include <mem/tlb.h>
5#include <sch/sched.h>
6#include <stdatomic.h>
7#include <stdint.h>
8#include <thread/dpc.h>
9
10struct spinlock tlb_shootdown_lock = SPINLOCK_INIT;
11
12static void tlb_shootdown_internal(void) {
13 size_t cpu = smp_id(cond: TOPC_IRQ);
14 struct tlb_shootdown_cpu *c = &global.shootdown_data[cpu];
15
16 uint64_t done = atomic_load_explicit(&c->done_gen, memory_order_relaxed);
17
18 while (true) {
19 uint64_t req = atomic_load_explicit(&c->req_gen, memory_order_acquire);
20
21 if (done >= req)
22 break;
23
24 for (;;) {
25 uint32_t tail =
26 atomic_load_explicit(&c->tail, memory_order_relaxed);
27 uint32_t head =
28 atomic_load_explicit(&c->head, memory_order_acquire);
29
30 if (tail == head)
31 break;
32
33 while (tail != head) {
34 uintptr_t addr =
35 atomic_load_explicit(&c->queue[tail & (TLB_QUEUE_SIZE - 1)],
36 memory_order_acquire);
37
38 if (addr)
39 invlpg(virt: addr);
40
41 tail++;
42 }
43
44 atomic_store_explicit(&c->tail, tail, memory_order_release);
45 }
46
47 if (atomic_exchange_explicit(&c->flush_all, false,
48 memory_order_acq_rel)) {
49 tlb_flush();
50 uint32_t h = atomic_load_explicit(&c->head, memory_order_acquire);
51 atomic_store_explicit(&c->tail, h, memory_order_release);
52 }
53
54 /* A drain satisfies each gen up to the `req` we read before
55 * draining. Stepping one at a time makes cost of ack ~ prop to
56 * how many shootdowns the rest of the machine had done, which
57 * causes some larger slowdowns */
58 done = req;
59 atomic_store_explicit(&c->done_gen, done, memory_order_release);
60 }
61}
62
63enum irq_result tlb_shootdown_isr(void *ctx, irq_t irq,
64 struct irq_context *rsp) {
65 (void) ctx;
66 (void) irq;
67 (void) rsp;
68
69 tlb_shootdown_internal();
70 return IRQ_HANDLED;
71}
72
73void tlb_shootdown(uintptr_t addr, bool synchronous) {
74 if (global.current_bootstage < BOOTSTAGE_MID_MP)
75 return;
76
77 /* TODO: scale up */
78 enum irql lirql = spin_lock(&tlb_shootdown_lock);
79
80 uint64_t gen = atomic_fetch_add_explicit(&global.next_tlb_gen, 1,
81 memory_order_relaxed) +
82 1;
83
84 size_t this_cpu = smp_id(cond: TOPC_IRQL);
85
86 size_t i;
87 for_each_cpu_id(i) {
88 if (i == this_cpu) {
89 invlpg(virt: addr);
90 continue;
91 }
92
93 struct tlb_shootdown_cpu *t = &global.shootdown_data[i];
94
95 uint32_t head = atomic_load_explicit(&t->head, memory_order_relaxed);
96 uint32_t tail = atomic_load_explicit(&t->tail, memory_order_acquire);
97
98 if ((head - tail) >= TLB_QUEUE_SIZE) {
99 atomic_store_explicit(&t->flush_all, true, memory_order_release);
100 } else {
101 atomic_store_explicit(&t->queue[head & (TLB_QUEUE_SIZE - 1)], addr,
102 memory_order_release);
103 atomic_store_explicit(&t->head, head + 1, memory_order_release);
104 }
105
106 atomic_store_explicit(&t->req_gen, gen, memory_order_release);
107 ipi_send(apic_id: i, IRQ_TLB_SHOOTDOWN);
108 }
109
110 if (synchronous) {
111 for_each_cpu_id(i) {
112 if (i == this_cpu)
113 continue;
114
115 struct tlb_shootdown_cpu *o = &global.shootdown_data[i];
116
117 int spins = 0;
118
119 while (atomic_load_explicit(&o->done_gen, memory_order_acquire) <
120 gen) {
121 if (spins < 100) {
122 cpu_relax();
123 spins++;
124 continue;
125 }
126
127 spins = 0;
128 ipi_send(apic_id: i, IRQ_TLB_SHOOTDOWN);
129 }
130 }
131
132 atomic_fetch_add_explicit(&global.pt_epoch, 1, memory_order_release);
133 }
134
135 spin_unlock(&tlb_shootdown_lock, lirql);
136}
137