1#include <acpi/lapic.h>
2#include <boot/gdt.h>
3#include <crypto/prng.h>
4#include <irq/idt.h>
5#include <limine.h>
6#include <mem/alloc.h>
7#include <mem/alloc_or_die.h>
8#include <mem/domain.h>
9#include <mem/tlb.h>
10#include <sch/sched.h>
11#include <smp/domain.h>
12#include <smp/percpu.h>
13#include <smp/smp.h>
14#include <string.h>
15#include <sync/spinlock.h>
16#include <thread/dpc.h>
17#include <thread/thread.h>
18#include <time/time.h>
19#include <time/tsc.h>
20
21/* TODO: This file is one that I had thought would stay small back when
22 * I first got SMP working, but day by day it grows and grows. Might
23 * want to refactor and clean this up into state machines */
24
25static volatile uint64_t cr3 = 0;
26static _Atomic uint32_t cores_awake = 0;
27#define CPUID_LEAF_HYBRID 0x1A
28
29static void detect_cpu_features(struct cpu_capability *cap) {
30 uint32_t eax, ebx, ecx, edx;
31
32 cap->feature_bits = 0;
33
34 /* CPUID.1 */
35 cpuid_count(leaf: 1, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
36
37 if (edx & (1 << 26))
38 cap->feature_bits |= CPU_FEAT_SSE2;
39 if (ecx & (1 << 28))
40 cap->feature_bits |= CPU_FEAT_AVX;
41
42 /* CPUID.7.0 */
43 cpuid_count(leaf: 7, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
44
45 if (ebx & (1 << 5))
46 cap->feature_bits |= CPU_FEAT_AVX2;
47 if (ebx & (1 << 16))
48 cap->feature_bits |= CPU_FEAT_AVX512F;
49}
50
51static void detect_cpu_class(struct cpu_capability *cap) {
52 uint32_t eax, ebx, ecx, edx;
53
54 cpuid_count(CPUID_LEAF_HYBRID, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
55
56 uint32_t core_type = (eax >> 24) & 0xFF;
57
58 switch (core_type) {
59 case 0x40: cap->class = CPU_CLASS_PERFORMANCE; break;
60 case 0x20: cap->class = CPU_CLASS_EFFICIENCY; break;
61 default: cap->class = CPU_CLASS_UNKNOWN; break;
62 }
63
64 cap->uarch_id = core_type;
65}
66
67static uint32_t detect_uarch_id(void) {
68 uint32_t eax, ebx, ecx, edx;
69
70 cpuid_count(leaf: 1, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
71
72 uint32_t family = (eax >> 8) & 0xF;
73 uint32_t model = (eax >> 4) & 0xF;
74
75 if (family == 6)
76 model |= ((eax >> 16) & 0xF) << 4;
77
78 switch (model) {
79 case 0x97: /* Alder Lake P */
80 case 0x9A: /* Raptor Lake P */ return UARCH_GOLDEN_COVE;
81 case 0x9C: /* Alder Lake E */ return UARCH_GRACEMONT;
82 default: return UARCH_UNKNOWN;
83 }
84}
85
86static void detect_pipeline_width(struct cpu_capability *cap) {
87 switch (cap->uarch_id) {
88 case UARCH_GOLDEN_COVE:
89 cap->issue_width = 6;
90 cap->retire_width = 6;
91 break;
92
93 case UARCH_GRACEMONT:
94 cap->issue_width = 3;
95 cap->retire_width = 3;
96 break;
97
98 default:
99 cap->issue_width = 2;
100 cap->retire_width = 2;
101 break;
102 }
103}
104
105static const char *cpu_class_str(enum cpu_class c) {
106 switch (c) {
107 case CPU_CLASS_PERFORMANCE: return "P-core";
108 case CPU_CLASS_EFFICIENCY: return "E-core";
109 default: return "unknown";
110 }
111}
112
113static const char *uarch_str(uint32_t uarch) {
114 switch (uarch) {
115 case UARCH_GOLDEN_COVE: return "Golden Cove";
116 case UARCH_GRACEMONT: return "Gracemont";
117 default: return "unknown";
118 }
119}
120
121static void dump_cpu_features(uint64_t f) {
122 char buf[128];
123 buf[0] = '\0';
124
125 if (f & CPU_FEAT_SSE2)
126 strcat(dest: buf, src: " SSE2");
127 if (f & CPU_FEAT_AVX)
128 strcat(dest: buf, src: " AVX");
129 if (f & CPU_FEAT_AVX2)
130 strcat(dest: buf, src: " AVX2");
131 if (f & CPU_FEAT_AVX512F)
132 strcat(dest: buf, src: " AVX-512F");
133
134 if (buf[0] == '\0')
135 strcpy(dest: buf, src: " (none)");
136
137 log_msg(LOG_INFO, " Features:%s", buf);
138}
139
140void smp_dump_core(struct core *c) {
141 log_msg(LOG_INFO, "CPU%zu: pkg=%u core=%u smt=%u numa=%zu domain_cpu=%zu",
142 c->id, c->package_id, c->core_id, c->smt_id, c->numa_node,
143 c->domain_cpu_id);
144
145 /* SMT / topology */
146 log_msg(LOG_INFO, " Topology: smt_mask=0x%x llc_shared=%u", c->smt_mask,
147 c->llc.cores_sharing);
148
149 /* Cache */
150 log_msg(LOG_INFO, " LLC: L%u size=%uKB line=%uB type=%u", c->llc.level,
151 c->llc.size_kb, c->llc.line_size, c->llc.type);
152
153 /* CPU capability block */
154 struct cpu_capability *cap = &c->cap;
155
156 log_msg(LOG_INFO, " Class: %s (%s)", cpu_class_str(cap->class),
157 uarch_str(cap->uarch_id));
158
159 log_msg(LOG_INFO, " Widths: issue=%u retire=%u", cap->issue_width,
160 cap->retire_width);
161
162 log_msg(LOG_INFO, " Scores: perf=%u energy=%u", cap->perf_score,
163 cap->energy_score);
164
165 dump_cpu_features(f: cap->feature_bits);
166}
167
168static void detect_llc(struct topology_cache_info *llc) {
169 uint32_t eax, ebx, ecx, edx;
170 for (uint32_t idx = 0;; idx++) {
171
172 cpuid_count(leaf: 4, subleaf: idx, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
173
174 uint32_t cache_type = eax & 0x1F;
175 if (cache_type == 0)
176 break;
177
178 uint32_t cache_level = (eax >> 5) & 0x7;
179 if (cache_level != 3)
180 continue;
181
182 llc->level = cache_level;
183 llc->type = cache_type;
184 llc->line_size = (ebx & 0xFFF) + 1;
185 llc->cores_sharing = ((eax >> 14) & 0xFFF) + 1;
186
187 uint32_t sets = ecx + 1;
188 uint32_t ways = ((ebx >> 22) & 0x3FF) + 1;
189 llc->size_kb = (ways * sets * llc->line_size) / 1024;
190
191 break;
192 }
193}
194
195static void init_smt_info(struct core *c) {
196 uint32_t eax, ebx, ecx, edx;
197
198 uint32_t smt_width = 0;
199 uint32_t core_width = 0;
200 uint32_t apic_id;
201
202 for (uint32_t level = 0;; level++) {
203 cpuid_count(leaf: 0xB, subleaf: level, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
204 uint32_t level_type = (ecx >> 8) & 0xFF;
205 if (level_type == 0)
206 break;
207
208 if (level_type == 1) {
209 smt_width = eax & 0x1F;
210 } else if (level_type == 2) {
211 core_width = eax & 0x1F;
212 }
213 }
214
215 apic_id = c->id;
216 c->package_id = c->id >> core_width;
217 c->smt_mask = (1 << smt_width) - 1;
218 c->smt_id = apic_id & c->smt_mask;
219 c->core_id = (apic_id >> smt_width) & ((1 << (core_width - smt_width)) - 1);
220}
221
222static void detect_cpu_capability(struct core *c) {
223 struct cpu_capability *cap = &c->cap;
224
225 detect_cpu_features(cap);
226
227 cap->uarch_id = detect_uarch_id();
228
229 detect_cpu_class(cap);
230 detect_pipeline_width(cap);
231}
232
233static struct core *setup_cpu(uint64_t cpu) {
234 struct core *c = global.cores[cpu];
235 kassert(c->id == cpu);
236 c->self = c;
237 c->current_irql = IRQL_PASSIVE_LEVEL;
238 c->tsc_hz = tsc_calibrate();
239 init_smt_info(c);
240 detect_llc(llc: &c->llc);
241 detect_cpu_capability(c);
242
243 wrmsr(MSR_GS_BASE, value: (uint64_t) c);
244 return c;
245}
246
247static inline void set_core_awake(void) {
248 atomic_fetch_add_explicit(&cores_awake, 1, memory_order_release);
249 if (atomic_load_explicit(&cores_awake, memory_order_acquire) ==
250 (global.core_count - 1)) {
251 bootstage_advance(new: BOOTSTAGE_MID_MP);
252 }
253}
254
255void smp_wakeup() {
256 disable_interrupts();
257
258 asm volatile("mov %0, %%cr3" ::"r"(cr3));
259
260 x2apic_init();
261 uint64_t cpu = cpu_get_this_id();
262 setup_cpu(cpu);
263
264 gdt_install();
265 wrmsr(MSR_GS_BASE, value: (uint64_t) global.cores[cpu]);
266 irq_load();
267
268 lapic_timer_init(core_id: cpu);
269 tsc_sync_check_ap(self: cpu);
270 set_core_awake();
271
272 scheduler_yield();
273}
274
275void smp_init() {
276 for (size_t i = 0; i < global.core_count; i++) {
277 size_t d = domain_for_cpu(cpu: i);
278
279 if (i != 0) {
280 global.cores[i] =
281 alloc_or_die(kmalloc_from_domain(d, sizeof(struct core)));
282
283 memset(global.cores[i], 0, sizeof(struct core));
284 }
285
286 global.cores[i]->irq_entered_irql = IRQL_NONE;
287 global.cores[i]->id = i;
288 global.cores[i]->numa_node = numa_node_for_cpu(cpu: i);
289 global.cores[i]->domain = global.domains[d];
290 }
291}
292
293void smp_wait_for_others_to_idle() {
294 /* wait for them to enter idle threads */
295 size_t expected_idle = global.core_count - 1;
296 while (atomic_load(&global.idle_core_count) < expected_idle) {
297 cpu_relax();
298 }
299}
300
301void smp_wake(struct limine_mp_response *mpr) {
302 asm volatile("mov %%cr3, %0" : "=r"(cr3));
303 for (uint64_t i = 1; i < mpr->cpu_count; i++)
304 mpr->cpus[i]->goto_address = smp_wakeup;
305
306 smp_core(cond: TOPC_NONE)->tsc_hz = tsc_calibrate();
307 if (global.core_count == 1)
308 return;
309
310 tsc_sync_check_all_aps();
311 /* wait for bootstage to progress, TODO: Find a better
312 * way to do this, reading a volatile and NOT using
313 * atomics here is problematic */
314 while (global.current_bootstage != BOOTSTAGE_MID_MP)
315 cpu_relax();
316
317 smp_wait_for_others_to_idle();
318}
319
320void smp_setup_bsp(void) {
321 struct core *c = kmalloc(sizeof(struct core), ALLOC_FLAGS_ZERO);
322 if (!c)
323 panic("Could not allocate space for core structure on BSP");
324
325 c->irq_entered_irql = IRQL_NONE;
326 c->id = 0;
327 c->self = c;
328 c->current_irql = IRQL_PASSIVE_LEVEL;
329 wrmsr(MSR_GS_BASE, value: (uint64_t) c);
330 global.cores =
331 kmalloc(sizeof(struct core *) * global.core_count, ALLOC_FLAGS_ZERO);
332
333 if (unlikely(!global.cores))
334 panic("Could not allocate space for global core structures");
335
336 global.shootdown_data = kmalloc(
337 sizeof(struct tlb_shootdown_cpu) * global.core_count, ALLOC_FLAGS_ZERO);
338 if (!global.shootdown_data)
339 panic("Could not allocate global shootdown data");
340
341 global.cores[0] = c;
342 init_smt_info(c);
343 detect_llc(llc: &c->llc);
344 detect_cpu_capability(c);
345 tsc_mailboxes_init();
346}
347
348static atomic_uint tick_change_state = 0;
349static bool enable = false;
350static uint8_t entry = 0;
351
352static enum irq_result tick_op_isr(void *ctx, uint8_t vector,
353 struct irq_context *rsp) {
354 if (enable) {
355 scheduler_tick_enable();
356 } else {
357 scheduler_tick_disable();
358 }
359
360 atomic_fetch_add(&tick_change_state, 1);
361 return IRQ_HANDLED;
362}
363
364static void send_em_all_out(bool e) {
365 enable = e;
366
367 atomic_store(&tick_change_state, 0);
368 size_t i;
369 for_each_cpu_id(i) {
370 if (i == 0)
371 continue;
372
373 ipi_send(apic_id: i, vector: entry);
374 }
375
376 /* wait for everyone to change their tick state */
377 while (atomic_load(&tick_change_state) < (global.core_count - 1))
378 cpu_relax();
379}
380
381void smp_disable_all_ticks() {
382 entry = irq_alloc_entry();
383 irq_register(name: "tick_op", vector: entry, handler: tick_op_isr, NULL, flags: IRQ_FLAG_NONE);
384 irq_set_chip(vector: entry, chip: lapic_get_chip(), NULL);
385 send_em_all_out(false);
386}
387
388extern void nop_handler(void *, uint8_t, void *);
389void smp_enable_all_ticks() {
390 send_em_all_out(true);
391 irq_free_entry(entry);
392 irq_set_chip(vector: entry, NULL, NULL);
393}
394
395struct core *smp_bsp(void) {
396 return global.cores[0];
397}
398
399void smp_caller_verify(enum topology_caller caller) {
400 struct topology_contract tct = {
401 .caller = caller,
402 .scope = TOPOLOGY_LEVEL_SMT,
403 };
404
405 bool valid = topology_contract_verify(c: tct);
406 kassert(valid, "Caller 0x%lx was not satisfied", caller);
407}
408