1#include <atomic.h>
2#include <irq/ipi.h>
3#include <mem/alloc.h>
4#include <mem/page.h>
5#include <mem/tlb.h>
6#include <sch/sched.h>
7#include <smp/percpu.h>
8#include <smp/perdomain.h>
9#include <stdint.h>
10#include <thread/dpc.h>
11#include <time/tsc.h>
12
13#include "tlb_internal.h"
14
15static void tlb_cpu_init(struct tlb_cpu *cpu, cpu_id_t id);
16static void tlb_domain_init(struct tlb_domain *cpu, domain_id_t id);
17
18struct tlb_globals tlb_global = {0};
19PERDOMAIN_DEFINE(struct tlb_domain, tlb_domains, tlb_domain_init);
20PERCPU_DEFINE(struct tlb_cpu, tlb_cpus, tlb_cpu_init);
21
22static void tlb_cpu_init(struct tlb_cpu *cpu, cpu_id_t id) {
23 tlb_queue_init(q: &cpu->eager_ring.queue, TLB_QUEUE_SIZE);
24 tlb_queue_init(q: &cpu->lazy_ring.queue, TLB_QUEUE_SIZE);
25 cpu->id = id;
26}
27
28static void tlb_domain_init(struct tlb_domain *d, domain_id_t id) {
29 d->domain = global.domains[id];
30 struct cpu_mask mask = CPU_MASK_INIT;
31 domain_set_cpu_mask(mask: &mask, domain: d->domain);
32
33 d->ipi_cost = ipi_send_cost(targets: &mask);
34}
35
36static void payload_invalidate(struct tlb_payload *pl) {
37 switch (pl->type) {
38 case TLB_PAYLOAD_FLUSH: tlb_flush(); break;
39 case TLB_PAYLOAD_PAGE: tlb_invlpg(virt: pl->addr); break;
40 case TLB_PAYLOAD_RANGE:
41 for (vaddr_t start = pl->range.low; start < pl->range.hi;
42 start += pl->stride) {
43 tlb_invlpg(virt: start);
44 }
45 }
46}
47
48/* Checking flush_req once is ok. If anyone else wants another
49 * shootdown, they'd bump and send another IPI anyways */
50static void ring_invalidate(struct tlb_ring *ring) {
51 uint64_t fr = atomic_load_acq(&ring->flush_req);
52
53 if (fr != atomic_load_rlx(&ring->flush_done)) {
54 tlb_queue_drain(q: &ring->queue);
55 tlb_flush();
56 atomic_store_release(&ring->flush_done, fr);
57 } else {
58 struct tlb_queue_entry got;
59 while (tlb_queue_dequeue(q: &ring->queue, out: &got))
60 payload_invalidate(pl: &got.payload);
61 }
62
63 atomic_store_release(&ring->done_pos, tlb_queue_consumed(&ring->queue));
64}
65
66static void ring_enqueue(struct tlb_ring *ring, struct tlb_payload *pl) {
67 struct tlb_queue_entry qent = {.payload = *pl};
68 if (!tlb_queue_enqueue(q: &ring->queue, v: qent))
69 atomic_fetch_add_release(&ring->flush_req, 1);
70}
71
72/* Re-load every loop since other shooters can bump the counter */
73static void ring_wait(struct tlb_ring *ring) {
74 uint64_t want_pos = tlb_queue_produced(q: &ring->queue);
75 uint64_t want_flush = atomic_load_acq(&ring->flush_req);
76
77 while (atomic_load_acq(&ring->done_pos) < want_pos ||
78 atomic_load_acq(&ring->flush_done) < want_flush)
79 cpu_pause();
80}
81
82static void tlb_shootdown_internal(void) {
83 ring_invalidate(ring: &PERCPU_PTR(TOPC_IRQ, tlb_cpus)->eager_ring);
84}
85
86enum irq_result tlb_shootdown_isr(struct irq_context *rsp, void *ctx) {
87 cc_ignore(ctx, rsp);
88
89 tlb_shootdown_internal();
90 return IRQ_HANDLED;
91}
92
93void tlb_shootdown_full(struct tlb_request rq) {
94 if (rq.mode == TLB_MODE_NONE)
95 return;
96
97 if (global.current_bootstage < BOOTSTAGE_MID_MP)
98 return;
99
100 enum irql lirql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
101
102 size_t this_cpu = smp_id(cond: TOPC_IRQL);
103
104 /* TODO: lazy */
105 size_t i;
106 cpu_mask_for_each(i, rq.targets) {
107 if (i == this_cpu) {
108 payload_invalidate(pl: &rq.payload);
109 continue;
110 }
111
112 ring_enqueue(ring: &PERCPU_PTR_FOR(tlb_cpus, i)->eager_ring, pl: &rq.payload);
113
114 ipi_send(cpu: i, IRQ_TLB_SHOOTDOWN);
115 }
116
117 if (rq.mode == TLB_MODE_SYNC) {
118 cpu_mask_for_each(i, rq.targets) {
119 if (i == this_cpu)
120 continue;
121
122 ring_wait(ring: &PERCPU_PTR_FOR(tlb_cpus, i)->eager_ring);
123 }
124
125 atomic_inc_release(&global.pt_epoch);
126 }
127
128 irql_lower(old_level: lirql);
129}
130
131void tlb_init(void) {
132 if (tsc_has_invariant()) {
133 tlb_global.stamp_source = TLB_STAMP_TSC;
134 tlb_global.stamp_guard =
135 tsc_global.max_warp + tsc_global.min_rtt + TLB_STAMP_GUARD_SLACK;
136 } else {
137 tlb_global.stamp_source = TLB_STAMP_GEN;
138 tlb_global.stamp_guard = 0;
139 }
140}
141