1#include <acpi/lapic.h>
2#include <asm.h>
3#include <atomic.h>
4#include <enforce.h>
5#include <global.h>
6#include <irq/ipi.h>
7#include <log.h>
8#include <math/bit.h>
9#include <sch/irql.h>
10#include <smp/core.h>
11#include <structures/cpu_mask.h>
12#include <virt/kvm.h>
13
14#define IPI_ICR_FIXED (LAPIC_DELIVERY_FIXED | LAPIC_LEVEL_ASSERT)
15
16enum ipi_mask_backend {
17 IPI_BACKEND_UNKNOWN = 0,
18 IPI_BACKEND_XAPIC_PHYSICAL,
19 IPI_BACKEND_X2APIC_CLUSTER,
20 IPI_BACKEND_KVM_PV,
21};
22
23/* All CPUs would compute the same thing so it's fine to race */
24static atomic_uint32_t ipi_backend = IPI_BACKEND_UNKNOWN;
25
26void ipi_send(cpu_id_t cpu, uint8_t vector) {
27 lapic_icr_write(dest: smp_apic_id(cpu),
28 lo: vector | IPI_ICR_FIXED | LAPIC_DEST_PHYSICAL);
29}
30
31bool ipi_send_try(cpu_id_t cpu, uint8_t vector) {
32 return lapic_icr_try_write(dest: smp_apic_id(cpu),
33 lo: vector | IPI_ICR_FIXED | LAPIC_DEST_PHYSICAL);
34}
35
36void nmi_send(cpu_id_t cpu) {
37 enforce(cpu != smp_id(TOPC_NONE)); /* NMI'ing ourselves is a MASSIVE
38 * risk, likely buggy code, panic */
39
40 lapic_icr_write(dest: smp_apic_id(cpu), LAPIC_DELIVERY_NMI | LAPIC_LEVEL_ASSERT |
41 LAPIC_DEST_PHYSICAL);
42}
43
44void nmi_send_others(cpu_id_t self) {
45 size_t i;
46 for_each_cpu_id(i) {
47 if (i == self)
48 continue;
49
50 nmi_send(cpu: i);
51 }
52}
53
54static void ipi_send_shorthand(uint32_t shorthand, uint8_t vector) {
55 lapic_icr_write(dest: 0,
56 lo: vector | IPI_ICR_FIXED | LAPIC_DEST_PHYSICAL | shorthand);
57}
58
59struct x2apic_cluster_acc {
60 uint32_t cluster;
61 uint16_t bits;
62};
63
64static void x2apic_cluster_flush(struct x2apic_cluster_acc *acc,
65 uint8_t vector) {
66 if (!acc->bits)
67 return;
68
69 uint32_t ldr = (acc->cluster << X2APIC_LDR_CLUSTER_SHIFT) | acc->bits;
70 lapic_icr_write(dest: ldr, lo: vector | IPI_ICR_FIXED | LAPIC_DEST_LOGICAL);
71 acc->bits = 0;
72}
73
74/* Batches consecutive same-cluster IDs into one ICR write */
75static void x2apic_cluster_add(struct x2apic_cluster_acc *acc, uint32_t apic_id,
76 uint8_t vector) {
77 uint32_t cluster =
78 (apic_id >> X2APIC_CLUSTER_SHIFT) & X2APIC_CLUSTER_ID_MASK;
79
80 if (acc->bits && cluster != acc->cluster)
81 x2apic_cluster_flush(acc, vector);
82
83 acc->cluster = cluster;
84 acc->bits |= (uint16_t) BIT(apic_id & (X2APIC_CLUSTER_SIZE - 1));
85}
86
87static void ipi_send_mask_cluster(const struct cpu_mask *targets,
88 uint8_t vector) {
89 struct x2apic_cluster_acc acc = {0};
90 size_t cpu;
91
92 for_each_cpu(cpu, targets) {
93 x2apic_cluster_add(acc: &acc, apic_id: smp_apic_id(cpu), vector);
94 }
95
96 x2apic_cluster_flush(acc: &acc, vector);
97}
98
99static void ipi_send_mask_physical(const struct cpu_mask *targets,
100 uint8_t vector) {
101 size_t cpu;
102 for_each_cpu(cpu, targets) {
103 ipi_send(cpu, vector);
104 }
105}
106
107static enum ipi_mask_backend ipi_backend_get(void) {
108 enum ipi_mask_backend b = atomic_load_acq(&ipi_backend);
109 if (b != IPI_BACKEND_UNKNOWN)
110 return b;
111
112 if (!x2apic_enabled)
113 b = IPI_BACKEND_XAPIC_PHYSICAL;
114 else if (kvm_has_feature(KVM_FEATURE_PV_SEND_IPI))
115 b = IPI_BACKEND_KVM_PV;
116 else
117 b = IPI_BACKEND_X2APIC_CLUSTER;
118
119 atomic_store_release(&ipi_backend, b);
120 return b;
121}
122
123static void kvm_pv_window_send(const uint64_t win[2], uint32_t min,
124 uint8_t vector) {
125 enum err e = kvm_hypercall4(KVM_HC_SEND_IPI, a0: win[0], a1: win[1], a2: min,
126 a3: vector | LAPIC_DELIVERY_FIXED, NULL);
127 if (e == ERR_OK)
128 return;
129
130 log_msg_once(LOG_WARN, "KVM PV IPI failed (%s), using x2APIC clusters",
131 errno_to_str(e));
132 atomic_store_release(&ipi_backend, IPI_BACKEND_X2APIC_CLUSTER);
133
134 struct x2apic_cluster_acc acc = {0};
135 for (uint32_t i = 0; i < KVM_PV_IPI_WINDOW; i++) {
136 if (win[i / 64] & BIT(i % 64))
137 x2apic_cluster_add(acc: &acc, apic_id: min + i, vector);
138 }
139
140 x2apic_cluster_flush(acc: &acc, vector);
141}
142
143/* One hypercall per 128-ID window starting at the lowest ID in it */
144static void ipi_send_mask_kvm_pv(const struct cpu_mask *targets,
145 uint8_t vector) {
146 uint64_t win[2] = {0, 0};
147 uint32_t min = 0;
148 bool open = false;
149 size_t cpu;
150
151 cpu_full_fence();
152 for_each_cpu(cpu, targets) {
153 uint32_t apic_id = smp_apic_id(cpu);
154
155 if (!open || apic_id < min || apic_id - min >= KVM_PV_IPI_WINDOW) {
156 if (open)
157 kvm_pv_window_send(win, min, vector);
158
159 win[0] = win[1] = 0;
160 min = apic_id;
161 open = true;
162 }
163
164 uint32_t off = apic_id - min;
165 win[off / 64] |= BIT(off % 64);
166 }
167
168 if (open)
169 kvm_pv_window_send(win, min, vector);
170}
171
172struct ipi_mask_plan {
173 size_t others; /* targets excluding self */
174 size_t sends; /* writes/hypercalls the exact path would issue */
175 bool self;
176};
177
178static struct ipi_mask_plan ipi_mask_plan(const struct cpu_mask *targets,
179 enum ipi_mask_backend backend,
180 size_t this_cpu) {
181 struct ipi_mask_plan p = {0};
182 uint32_t group = UINT32_MAX;
183 size_t cpu;
184
185 for_each_cpu(cpu, targets) {
186 if (cpu == this_cpu)
187 p.self = true;
188 else
189 p.others++;
190
191 uint32_t apic_id = smp_apic_id(cpu);
192 switch (backend) {
193 case IPI_BACKEND_X2APIC_CLUSTER: {
194 uint32_t c = apic_id >> X2APIC_CLUSTER_SHIFT;
195 if (c != group) {
196 group = c;
197 p.sends++;
198 }
199 break;
200 }
201
202 case IPI_BACKEND_KVM_PV:
203 if (group == UINT32_MAX || apic_id < group ||
204 apic_id - group >= KVM_PV_IPI_WINDOW) {
205 group = apic_id;
206 p.sends++;
207 }
208 break;
209
210 default: p.sends++; break;
211 }
212 }
213
214 return p;
215}
216
217static bool ipi_mask_should_broadcast(const struct ipi_mask_plan *p,
218 size_t online,
219 enum ipi_mask_flags flags) {
220 if (global.current_bootstage < BOOTSTAGE_MID_MP)
221 return false;
222
223 if (online > CPU_MASK_BITS)
224 return false;
225
226 size_t bystanders = online - 1 - p->others;
227 if (!bystanders)
228 return true;
229
230 if (!(flags & IPI_MASK_SUPERSET_OK) || p->sends <= 1)
231 return false;
232
233 return (p->sends - 1) * 256 >= bystanders * IPI_BCAST_RATIO_Q8;
234}
235
236size_t ipi_send_cost(const struct cpu_mask *targets) {
237 enum irql irql = irql_raise(new_level: IRQL_HIGH_LEVEL);
238
239 cpu_id_t this_cpu = smp_id(cond: TOPC_IRQL);
240
241 enum ipi_mask_backend backend = ipi_backend_get();
242 struct ipi_mask_plan plan = ipi_mask_plan(targets, backend, this_cpu);
243
244 irql_lower(old_level: irql);
245
246 return plan.sends;
247}
248
249void ipi_send_mask(const struct cpu_mask *targets, uint8_t vector,
250 enum ipi_mask_flags flags) {
251 enum irql irql = irql_raise(new_level: IRQL_HIGH_LEVEL);
252
253 cpu_id_t this_cpu = smp_id(cond: TOPC_IRQL);
254 size_t online = global.core_count;
255 enum ipi_mask_backend backend = ipi_backend_get();
256 struct ipi_mask_plan plan = ipi_mask_plan(targets, backend, this_cpu);
257
258 if (!plan.others && !plan.self)
259 goto out;
260
261 if (plan.others && ipi_mask_should_broadcast(p: &plan, online, flags)) {
262 ipi_send_shorthand(shorthand: plan.self ? LAPIC_DEST_SHORTHAND_ALL
263 : LAPIC_DEST_SHORTHAND_OTHERS,
264 vector);
265 goto out;
266 }
267
268 switch (backend) {
269 case IPI_BACKEND_KVM_PV: ipi_send_mask_kvm_pv(targets, vector); break;
270 case IPI_BACKEND_X2APIC_CLUSTER:
271 ipi_send_mask_cluster(targets, vector);
272 break;
273
274 default: ipi_send_mask_physical(targets, vector); break;
275 }
276
277out:
278 irql_lower(old_level: irql);
279}
280