1#include <acpi/lapic.h>
2#include <boot/gdt.h>
3#include <crypto/prng.h>
4#include <irq/idt.h>
5#include <irq/ipi.h>
6#include <limine.h>
7#include <math/bit.h>
8#include <mem/alloc.h>
9#include <mem/domain.h>
10#include <mem/nofail.h>
11#include <mem/tlb.h>
12#include <sch/sched.h>
13#include <smp/domain.h>
14#include <smp/percpu.h>
15#include <smp/smp.h>
16#include <string.h>
17#include <sync/once_latch.h>
18#include <sync/spinlock.h>
19#include <thread/dpc.h>
20#include <thread/thread.h>
21#include <time/lapic_timer.h>
22#include <time/time.h>
23#include <time/tsc.h>
24
25#include "mem/tlb_internal.h" /* for tlb_shootdown_cpu */
26
27/* TODO: This file is one that I had thought would stay small back when
28 * I first got SMP working, but day by day it grows and grows. Might
29 * want to refactor and clean this up into state machines */
30
31static atomic_uint64_t cr3 = 0;
32static atomic_uint32_t cores_awake = 0;
33
34/* NOTE: this is used because it is contiguous in memory,
35 * whereas percpu is NOT.
36 *
37 * TODO: implement PERCPU flags so that we can
38 * initialize in a contiguous manner, and avoid this hassle
39 * down the line. */
40static struct dpc timer_init_dpcs[CPU_MASK_BITS];
41
42static void timer_init_fanout_fn(void) {
43 timers_init_ap(cpu: smp_id_raw());
44}
45#define CPUID_LEAF_HYBRID 0x1A
46
47static void detect_cpu_features(struct cpu_capability *cap) {
48 uint32_t eax, ebx, ecx, edx;
49
50 cap->feature_bits = 0;
51
52 /* CPUID.1 */
53 cpuid_count(leaf: 1, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
54
55 if (BIT_TEST(edx, 26))
56 cap->feature_bits |= CPU_FEAT_SSE2;
57 if (BIT_TEST(ecx, 28))
58 cap->feature_bits |= CPU_FEAT_AVX;
59
60 /* CPUID.7.0 */
61 cpuid_count(leaf: 7, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
62
63 if (BIT_TEST(ebx, 5))
64 cap->feature_bits |= CPU_FEAT_AVX2;
65 if (BIT_TEST(ebx, 16))
66 cap->feature_bits |= CPU_FEAT_AVX512F;
67}
68
69static void detect_cpu_class(struct cpu_capability *cap) {
70 uint32_t eax, ebx, ecx, edx;
71
72 cpuid_count(CPUID_LEAF_HYBRID, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
73
74 uint32_t core_type = (eax >> 24) & 0xFF;
75
76 switch (core_type) {
77 case 0x40: cap->class = CPU_CLASS_PERFORMANCE; break;
78 case 0x20: cap->class = CPU_CLASS_EFFICIENCY; break;
79 default: cap->class = CPU_CLASS_UNKNOWN; break;
80 }
81
82 cap->uarch_id = core_type;
83}
84
85static uint32_t detect_uarch_id(void) {
86 uint32_t eax, ebx, ecx, edx;
87
88 cpuid_count(leaf: 1, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
89
90 uint32_t family = (eax >> 8) & 0xF;
91 uint32_t model = (eax >> 4) & 0xF;
92
93 if (family == 6)
94 model |= ((eax >> 16) & 0xF) << 4;
95
96 switch (model) {
97 case 0x97: /* Alder Lake P */
98 case 0x9A: /* Raptor Lake P */ return UARCH_GOLDEN_COVE;
99 case 0x9C: /* Alder Lake E */ return UARCH_GRACEMONT;
100 default: return UARCH_UNKNOWN;
101 }
102}
103
104static void detect_pipeline_width(struct cpu_capability *cap) {
105 switch (cap->uarch_id) {
106 case UARCH_GOLDEN_COVE:
107 cap->issue_width = 6;
108 cap->retire_width = 6;
109 break;
110
111 case UARCH_GRACEMONT:
112 cap->issue_width = 3;
113 cap->retire_width = 3;
114 break;
115
116 default:
117 cap->issue_width = 2;
118 cap->retire_width = 2;
119 break;
120 }
121}
122
123static const char *cpu_class_str(enum cpu_class c) {
124 switch (c) {
125 case CPU_CLASS_PERFORMANCE: return "P-core";
126 case CPU_CLASS_EFFICIENCY: return "E-core";
127 default: return "unknown";
128 }
129}
130
131static const char *uarch_str(uint32_t uarch) {
132 switch (uarch) {
133 case UARCH_GOLDEN_COVE: return "Golden Cove";
134 case UARCH_GRACEMONT: return "Gracemont";
135 default: return "unknown";
136 }
137}
138
139static void dump_cpu_features(uint64_t f) {
140 char buf[128];
141 buf[0] = '\0';
142
143 if (f & CPU_FEAT_SSE2)
144 strcat(buf, " SSE2");
145 if (f & CPU_FEAT_AVX)
146 strcat(buf, " AVX");
147 if (f & CPU_FEAT_AVX2)
148 strcat(buf, " AVX2");
149 if (f & CPU_FEAT_AVX512F)
150 strcat(buf, " AVX-512F");
151
152 if (buf[0] == '\0')
153 strcpy(buf, " (none)");
154
155 log_msg(LOG_INFO, " Features:%s", buf);
156}
157
158void smp_dump_core(struct core *c) {
159 log_msg(LOG_INFO, "CPU%zu: pkg=%u core=%u smt=%u numa=%zu domain_cpu=%zu",
160 c->id, c->package_id, c->core_id, c->smt_id, c->numa_node,
161 c->domain_cpu_id);
162
163 /* SMT / topology */
164 log_msg(LOG_INFO, " Topology: smt_mask=0x%x llc_shared=%u", c->smt_mask,
165 c->llc.cores_sharing);
166
167 /* Cache */
168 log_msg(LOG_INFO, " LLC: L%u size=%uKB line=%uB type=%u", c->llc.level,
169 c->llc.size_kb, c->llc.line_size, c->llc.type);
170
171 /* CPU capability block */
172 struct cpu_capability *cap = &c->cap;
173
174 log_msg(LOG_INFO, " Class: %s (%s)", cpu_class_str(cap->class),
175 uarch_str(cap->uarch_id));
176
177 log_msg(LOG_INFO, " Widths: issue=%u retire=%u", cap->issue_width,
178 cap->retire_width);
179
180 log_msg(LOG_INFO, " Scores: perf=%u energy=%u", cap->perf_score,
181 cap->energy_score);
182
183 dump_cpu_features(f: cap->feature_bits);
184}
185
186static void detect_llc(struct topology_cache_info *llc) {
187 uint32_t eax, ebx, ecx, edx;
188 for (uint32_t idx = 0;; idx++) {
189
190 cpuid_count(leaf: 4, subleaf: idx, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
191
192 uint32_t cache_type = eax & 0x1F;
193 if (cache_type == 0)
194 break;
195
196 uint32_t cache_level = (eax >> 5) & 0x7;
197 if (cache_level != 3)
198 continue;
199
200 llc->level = cache_level;
201 llc->type = cache_type;
202 llc->line_size = (ebx & 0xFFF) + 1;
203 llc->cores_sharing = ((eax >> 14) & 0xFFF) + 1;
204
205 uint32_t sets = ecx + 1;
206 uint32_t ways = ((ebx >> 22) & 0x3FF) + 1;
207 llc->size_kb = (ways * sets * llc->line_size) / 1024;
208
209 break;
210 }
211}
212
213static void init_smt_info(struct core *c) {
214 uint32_t eax, ebx, ecx, edx;
215
216 uint32_t smt_width = 0;
217 uint32_t core_width = 0;
218 uint32_t apic_id;
219
220 for (uint32_t level = 0;; level++) {
221 cpuid_count(leaf: 0xB, subleaf: level, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
222 uint32_t level_type = (ecx >> 8) & 0xFF;
223 if (level_type == 0)
224 break;
225
226 if (level_type == 1) {
227 smt_width = eax & 0x1F;
228 } else if (level_type == 2) {
229 core_width = eax & 0x1F;
230 }
231 }
232
233 apic_id = smp_apic_id(cpu: c->id);
234 c->package_id = apic_id >> core_width;
235 c->smt_mask = BIT(smt_width) - 1;
236 c->smt_id = apic_id & c->smt_mask;
237 c->core_id = (apic_id >> smt_width) & (BIT(core_width - smt_width) - 1);
238}
239
240static void detect_cpu_capability(struct core *c) {
241 struct cpu_capability *cap = &c->cap;
242
243 detect_cpu_features(cap);
244
245 cap->uarch_id = detect_uarch_id();
246
247 detect_cpu_class(cap);
248 detect_pipeline_width(cap);
249}
250
251static struct core *setup_cpu(uint64_t cpu) {
252 struct core *c = global.cores[cpu];
253 enforce(c->id == cpu);
254 c->self = c;
255 c->current_irql = IRQL_PASSIVE_LEVEL;
256 c->tsc_hz = tsc_calibrate();
257 init_smt_info(c);
258 detect_llc(llc: &c->llc);
259 detect_cpu_capability(c);
260
261 wrmsr(MSR_GS_BASE, value: (uint64_t) c);
262 return c;
263}
264
265static inline void set_core_awake(void) {
266 if (atomic_inc_return_new_acq_rel(&cores_awake) ==
267 (global.core_count - 1)) {
268 bootstage_advance(new: BOOTSTAGE_MID_MP);
269 }
270}
271
272/* returns with interrupts disabled; the scheduler takes it from here */
273static void smp_ap_wakeup(struct limine_mp_info *info) TSA_ACQUIRES_IRQS {
274 irq_disable();
275
276 cr3_write(atomic_load_acq(&cr3));
277
278 x2apic_init();
279 cpu_id_t cpu = (cpu_id_t) info->extra_argument; /* smp_map_apic_ids */
280 enforce(lapic_this_id() == smp_apic_id(cpu), UNSUPPORTED,
281 "AP APIC ID mismatch");
282 setup_cpu(cpu);
283
284 gdt_load();
285 wrmsr(MSR_GS_BASE, value: (uint64_t) global.cores[cpu]);
286 idt_load();
287
288 lapic_timer_init(core_id: cpu);
289 tsc_sync_check_ap(self: cpu);
290 set_core_awake();
291
292 scheduler_yield();
293}
294
295void smp_init(void) {
296 for (size_t i = 0; i < global.core_count; i++) {
297 size_t d = domain_for_cpu(cpu: i);
298
299 if (i != 0) {
300 global.cores[i] =
301 nofail(valloc_from_domain(d, sizeof(struct core)));
302
303 memset(global.cores[i], 0, sizeof(struct core));
304 }
305
306 global.cores[i]->id = i;
307 global.cores[i]->numa_node = numa_node_for_cpu(cpu: i);
308 global.cores[i]->domain = global.domains[d];
309 }
310}
311
312void smp_timer_init(void) {
313 struct cpu_mask all = CPU_MASK_INIT;
314 cpu_mask_set_all(&all);
315
316 dpc_fanout(storage: timer_init_dpcs, cpus: all, fn: timer_init_fanout_fn);
317}
318
319void smp_wait_for_others_to_idle(void) {
320 /* wait for them to enter idle threads */
321 size_t expected_idle = global.core_count - 1;
322 while (atomic_load(&global.idle_core_count) < expected_idle) {
323 cpu_pause();
324 }
325}
326
327void smp_wake(struct limine_mp_response *mpr) {
328 atomic_store_release(&cr3, cr3_read());
329 for (uint64_t i = 0; i < mpr->cpu_count; i++)
330 if (mpr->cpus[i]->lapic_id != mpr->bsp_lapic_id)
331 mpr->cpus[i]->goto_address = smp_ap_wakeup;
332
333 smp_core(cond: TOPC_NONE)->tsc_hz = tsc_calibrate();
334 if (global.core_count == 1)
335 return;
336
337 tsc_sync_check_all_aps();
338 /* wait for bootstage to progress, TODO: Find a better
339 * way to do this, reading a volatile and NOT using
340 * atomics here is problematic */
341 while (global.current_bootstage != BOOTSTAGE_MID_MP)
342 cpu_pause();
343}
344
345uint32_t *smp_apic_ids;
346
347static void smp_map_apic_ids(struct limine_mp_response *mpr) {
348 smp_apic_ids = nofail_vnews(smp_apic_ids, global.core_count);
349 smp_apic_ids[0] = mpr->bsp_lapic_id;
350
351 cpu_id_t next = 1;
352 for (uint64_t i = 0; i < mpr->cpu_count; i++) {
353 struct limine_mp_info *info = mpr->cpus[i];
354 if (info->lapic_id == mpr->bsp_lapic_id) {
355 info->extra_argument = 0;
356 continue;
357 }
358
359 enforce(next < global.core_count, UNSUPPORTED, "limine lists no BSP");
360 info->extra_argument = next;
361 smp_apic_ids[next++] = info->lapic_id;
362 }
363}
364
365cpu_id_t smp_cpu_from_apic_id(uint32_t apic_id) {
366 cpu_id_t cpu;
367 for_each_cpu_id(cpu) {
368 if (smp_apic_ids[cpu] == apic_id)
369 return cpu;
370 }
371
372 return CPU_ID_NONE;
373}
374
375void smp_setup_bsp(struct limine_mp_response *mpr) {
376 smp_map_apic_ids(mpr);
377
378 struct core *c = nofail_vnew(c, ALLOC_ZERO);
379
380 c->id = 0;
381 c->self = c;
382 c->current_irql = IRQL_PASSIVE_LEVEL;
383 wrmsr(MSR_GS_BASE, value: (uint64_t) c);
384 global.cores = nofail_vnews(global.cores, global.core_count, ALLOC_ZERO);
385
386 global.cores[0] = c;
387 init_smt_info(c);
388 detect_llc(llc: &c->llc);
389 detect_cpu_capability(c);
390}
391
392static atomic_uint32_t tick_change_state = 0;
393static atomic_bool enable = false;
394static uint8_t entry = 0;
395
396static enum irq_result tick_op_isr(struct irq_context *rsp, void *ctx) {
397 cc_ignore(ctx, rsp);
398 if (atomic_load_acq(&enable)) {
399 scheduler_tick_enable();
400 } else {
401 scheduler_tick_disable();
402 }
403
404 atomic_inc(&tick_change_state);
405 return IRQ_HANDLED;
406}
407
408static void send_em_all_out(bool e) {
409 atomic_store_release(&enable, e);
410
411 atomic_store(&tick_change_state, 0);
412 size_t i;
413 for_each_cpu_id(i) {
414 if (i == 0)
415 continue;
416
417 ipi_send(cpu: i, vector: entry);
418 }
419
420 /* wait for everyone to change their tick state */
421 while (atomic_load(&tick_change_state) < (global.core_count - 1))
422 cpu_pause();
423}
424
425void smp_disable_all_ticks(void) {
426 entry = irq_alloc_entry();
427 irq_register(name: "tick_op", vector: entry, handler: tick_op_isr, NULL, flags: IRQ_FLAG_NONE);
428 irq_set_chip(vector: entry, chip: lapic_get_chip(), NULL);
429 send_em_all_out(false);
430}
431
432void smp_enable_all_ticks(void) {
433 send_em_all_out(true);
434 irq_free_entry(entry);
435 irq_set_chip(vector: entry, NULL, NULL);
436}
437
438/* Force every AP into the top spin part of its idel loop while the BSP
439 * advances to BOOTSTAGE_LATE. This is because IRQL operations
440 * treat < BOOTSTAGE_LATE as a no-op and return IRQL_NONE.
441 *
442 * We have to do this instead of just disabling ticks because the tick
443 * disabling IPI can wake idle APs and go into scheduler_yield() */
444static atomic_bool boot_fence_up = false;
445static atomic_uint32_t boot_fence_parked = 0;
446
447void smp_boot_fence_raise(void) {
448 atomic_store(&boot_fence_parked, 0);
449 atomic_store_release(&boot_fence_up, true);
450
451 size_t i;
452 for_each_cpu_id(i) {
453 if (i == 0)
454 continue;
455
456 ipi_send(cpu: i, IRQ_NOP);
457 }
458
459 while (atomic_load_acq(&boot_fence_parked) < (global.core_count - 1))
460 cpu_pause();
461}
462
463void smp_boot_fence_lower(void) {
464 atomic_store_release(&boot_fence_up, false);
465}
466
467void smp_boot_fence_park(void) {
468 if (cc_likely(!atomic_load_acq(&boot_fence_up)))
469 return;
470
471 enforce(!irqs_enabled(), INVALID_CONTEXT);
472 atomic_inc(&boot_fence_parked);
473
474 while (atomic_load_acq(&boot_fence_up))
475 cpu_pause();
476
477 /* serializing instruction for bootstage_advance(), since that
478 * would patch kernel text and do stuff */
479 uint32_t a, b, c, d;
480 cpuid_count(leaf: 0, subleaf: 0, eax: &a, ebx: &b, ecx: &c, edx: &d);
481}
482
483struct core *smp_bsp(void) {
484 return global.cores[0];
485}
486
487void smp_caller_verify(enum topology_caller caller) {
488 struct topology_contract tct = {
489 .caller = caller,
490 .scope = TOPOLOGY_LEVEL_SMT,
491 };
492
493 bool valid = topology_contract_verify(c: tct);
494 enforce(valid, "Caller 0x%x was not satisfied", caller);
495}
496