1#include <acpi/lapic.h>
2#include <mem/alloc.h>
3#include <mem/page.h>
4#include <mem/tlb.h>
5#include <sch/sched.h>
6#include <stdatomic.h>
7#include <stdint.h>
8#include <thread/dpc.h>
9
10struct spinlock tlb_shootdown_lock = SPINLOCK_INIT;
11
12static void tlb_shootdown_internal(void) {
13 size_t cpu = smp_core_id();
14 struct tlb_shootdown_cpu *c = &global.shootdown_data[cpu];
15
16 uint64_t done = atomic_load_explicit(&c->done_gen, memory_order_relaxed);
17
18 while (true) {
19 uint64_t req = atomic_load_explicit(&c->req_gen, memory_order_acquire);
20
21 if (done >= req)
22 break;
23
24 for (;;) {
25 uint32_t tail =
26 atomic_load_explicit(&c->tail, memory_order_relaxed);
27 uint32_t head =
28 atomic_load_explicit(&c->head, memory_order_acquire);
29
30 if (tail == head)
31 break;
32
33 while (tail != head) {
34 uintptr_t addr =
35 atomic_load_explicit(&c->queue[tail & (TLB_QUEUE_SIZE - 1)],
36 memory_order_acquire);
37
38 if (addr)
39 invlpg(virt: addr);
40
41 tail++;
42 }
43
44 atomic_store_explicit(&c->tail, tail, memory_order_release);
45 }
46
47 if (atomic_exchange_explicit(&c->flush_all, false,
48 memory_order_acq_rel)) {
49 tlb_flush();
50 uint32_t h = atomic_load_explicit(&c->head, memory_order_acquire);
51 atomic_store_explicit(&c->tail, h, memory_order_release);
52 }
53
54 done++;
55 atomic_store_explicit(&c->done_gen, done, memory_order_release);
56 }
57}
58
59enum irq_result tlb_shootdown_isr(void *ctx, irq_t irq,
60 struct irq_context *rsp) {
61 (void) ctx;
62 (void) irq;
63 (void) rsp;
64
65 tlb_shootdown_internal();
66 return IRQ_HANDLED;
67}
68
69void tlb_shootdown(uintptr_t addr, bool synchronous) {
70 if (global.current_bootstage < BOOTSTAGE_MID_MP)
71 return;
72
73 enum irql lirql = spin_lock(lock: &tlb_shootdown_lock);
74
75 uint64_t gen = atomic_fetch_add_explicit(&global.next_tlb_gen, 1,
76 memory_order_relaxed) +
77 1;
78
79 size_t this_cpu = smp_core_id();
80
81 size_t i;
82 for_each_cpu_id(i) {
83 if (i == this_cpu) {
84 invlpg(virt: addr);
85 continue;
86 }
87
88 struct tlb_shootdown_cpu *t = &global.shootdown_data[i];
89
90 uint32_t head = atomic_load_explicit(&t->head, memory_order_relaxed);
91 uint32_t tail = atomic_load_explicit(&t->tail, memory_order_acquire);
92
93 if ((head - tail) >= TLB_QUEUE_SIZE) {
94 atomic_store_explicit(&t->flush_all, true, memory_order_release);
95 } else {
96 atomic_store_explicit(&t->queue[head & (TLB_QUEUE_SIZE - 1)], addr,
97 memory_order_release);
98 atomic_store_explicit(&t->head, head + 1, memory_order_release);
99 }
100
101 atomic_store_explicit(&t->req_gen, gen, memory_order_release);
102 ipi_send(apic_id: i, IRQ_TLB_SHOOTDOWN);
103 }
104
105 if (synchronous) {
106 for_each_cpu_id(i) {
107 if (i == this_cpu)
108 continue;
109
110 struct tlb_shootdown_cpu *o = &global.shootdown_data[i];
111
112 int spins = 0;
113
114 while (atomic_load_explicit(&o->done_gen, memory_order_acquire) <
115 gen) {
116 if (spins < 100) {
117 cpu_relax();
118 spins++;
119 continue;
120 }
121
122 spins = 0;
123 ipi_send(apic_id: i, IRQ_TLB_SHOOTDOWN);
124 }
125 }
126
127 atomic_fetch_add_explicit(&global.pt_epoch, 1, memory_order_release);
128 }
129
130 spin_unlock(lock: &tlb_shootdown_lock, old: lirql);
131}
132