1#include <acpi/lapic.h>
2#include <boot/gdt.h>
3#include <crypto/prng.h>
4#include <irq/idt.h>
5#include <limine.h>
6#include <mem/alloc.h>
7#include <mem/alloc_or_die.h>
8#include <mem/domain.h>
9#include <mem/tlb.h>
10#include <sch/sched.h>
11#include <smp/domain.h>
12#include <smp/percpu.h>
13#include <smp/smp.h>
14#include <string.h>
15#include <sync/spinlock.h>
16#include <thread/dpc.h>
17#include <time/time.h>
18
19static volatile uint64_t cr3 = 0;
20static _Atomic uint32_t cores_awake = 0;
21#define CPUID_LEAF_HYBRID 0x1A
22
23static void detect_cpu_features(struct cpu_capability *cap) {
24 uint32_t eax, ebx, ecx, edx;
25
26 cap->feature_bits = 0;
27
28 /* CPUID.1 */
29 cpuid_count(leaf: 1, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
30
31 if (edx & (1 << 26))
32 cap->feature_bits |= CPU_FEAT_SSE2;
33 if (ecx & (1 << 28))
34 cap->feature_bits |= CPU_FEAT_AVX;
35
36 /* CPUID.7.0 */
37 cpuid_count(leaf: 7, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
38
39 if (ebx & (1 << 5))
40 cap->feature_bits |= CPU_FEAT_AVX2;
41 if (ebx & (1 << 16))
42 cap->feature_bits |= CPU_FEAT_AVX512F;
43}
44
45static void detect_cpu_class(struct cpu_capability *cap) {
46 uint32_t eax, ebx, ecx, edx;
47
48 cpuid_count(CPUID_LEAF_HYBRID, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
49
50 uint32_t core_type = (eax >> 24) & 0xFF;
51
52 switch (core_type) {
53 case 0x40: cap->class = CPU_CLASS_PERFORMANCE; break;
54 case 0x20: cap->class = CPU_CLASS_EFFICIENCY; break;
55 default: cap->class = CPU_CLASS_UNKNOWN; break;
56 }
57
58 cap->uarch_id = core_type;
59}
60
61static uint32_t detect_uarch_id(void) {
62 uint32_t eax, ebx, ecx, edx;
63
64 cpuid_count(leaf: 1, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
65
66 uint32_t family = (eax >> 8) & 0xF;
67 uint32_t model = (eax >> 4) & 0xF;
68
69 if (family == 6)
70 model |= ((eax >> 16) & 0xF) << 4;
71
72 switch (model) {
73 case 0x97: /* Alder Lake P */
74 case 0x9A: /* Raptor Lake P */ return UARCH_GOLDEN_COVE;
75 case 0x9C: /* Alder Lake E */ return UARCH_GRACEMONT;
76 default: return UARCH_UNKNOWN;
77 }
78}
79
80static void detect_pipeline_width(struct cpu_capability *cap) {
81 switch (cap->uarch_id) {
82 case UARCH_GOLDEN_COVE:
83 cap->issue_width = 6;
84 cap->retire_width = 6;
85 break;
86
87 case UARCH_GRACEMONT:
88 cap->issue_width = 3;
89 cap->retire_width = 3;
90 break;
91
92 default:
93 cap->issue_width = 2;
94 cap->retire_width = 2;
95 break;
96 }
97}
98
99static const char *cpu_class_str(enum cpu_class c) {
100 switch (c) {
101 case CPU_CLASS_PERFORMANCE: return "P-core";
102 case CPU_CLASS_EFFICIENCY: return "E-core";
103 default: return "unknown";
104 }
105}
106
107static const char *uarch_str(uint32_t uarch) {
108 switch (uarch) {
109 case UARCH_GOLDEN_COVE: return "Golden Cove";
110 case UARCH_GRACEMONT: return "Gracemont";
111 default: return "unknown";
112 }
113}
114
115static void dump_cpu_features(uint64_t f) {
116 char buf[128];
117 buf[0] = '\0';
118
119 if (f & CPU_FEAT_SSE2)
120 strcat(dest: buf, src: " SSE2");
121 if (f & CPU_FEAT_AVX)
122 strcat(dest: buf, src: " AVX");
123 if (f & CPU_FEAT_AVX2)
124 strcat(dest: buf, src: " AVX2");
125 if (f & CPU_FEAT_AVX512F)
126 strcat(dest: buf, src: " AVX-512F");
127
128 if (buf[0] == '\0')
129 strcpy(dest: buf, src: " (none)");
130
131 log_msg(LOG_INFO, " Features:%s", buf);
132}
133
134void smp_dump_core(struct core *c) {
135 log_msg(LOG_INFO, "CPU%zu: pkg=%u core=%u smt=%u numa=%zu domain_cpu=%zu",
136 c->id, c->package_id, c->core_id, c->smt_id, c->numa_node,
137 c->domain_cpu_id);
138
139 /* SMT / topology */
140 log_msg(LOG_INFO, " Topology: smt_mask=0x%x llc_shared=%u", c->smt_mask,
141 c->llc.cores_sharing);
142
143 /* Cache */
144 log_msg(LOG_INFO, " LLC: L%u size=%uKB line=%uB type=%u", c->llc.level,
145 c->llc.size_kb, c->llc.line_size, c->llc.type);
146
147 /* CPU capability block */
148 struct cpu_capability *cap = &c->cap;
149
150 log_msg(LOG_INFO, " Class: %s (%s)", cpu_class_str(cap->class),
151 uarch_str(cap->uarch_id));
152
153 log_msg(LOG_INFO, " Widths: issue=%u retire=%u", cap->issue_width,
154 cap->retire_width);
155
156 log_msg(LOG_INFO, " Scores: perf=%u energy=%u", cap->perf_score,
157 cap->energy_score);
158
159 dump_cpu_features(f: cap->feature_bits);
160}
161
162static void detect_llc(struct topology_cache_info *llc) {
163 uint32_t eax, ebx, ecx, edx;
164 for (uint32_t idx = 0;; idx++) {
165
166 cpuid_count(leaf: 4, subleaf: idx, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
167
168 uint32_t cache_type = eax & 0x1F;
169 if (cache_type == 0)
170 break;
171
172 uint32_t cache_level = (eax >> 5) & 0x7;
173 if (cache_level != 3)
174 continue;
175
176 llc->level = cache_level;
177 llc->type = cache_type;
178 llc->line_size = (ebx & 0xFFF) + 1;
179 llc->cores_sharing = ((eax >> 14) & 0xFFF) + 1;
180
181 uint32_t sets = ecx + 1;
182 uint32_t ways = ((ebx >> 22) & 0x3FF) + 1;
183 llc->size_kb = (ways * sets * llc->line_size) / 1024;
184
185 break;
186 }
187}
188
189static void init_smt_info(struct core *c) {
190 uint32_t eax, ebx, ecx, edx;
191
192 uint32_t smt_width = 0;
193 uint32_t core_width = 0;
194 uint32_t apic_id;
195
196 for (uint32_t level = 0;; level++) {
197 cpuid_count(leaf: 0xB, subleaf: level, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
198 uint32_t level_type = (ecx >> 8) & 0xFF;
199 if (level_type == 0)
200 break;
201
202 if (level_type == 1) {
203 smt_width = eax & 0x1F;
204 } else if (level_type == 2) {
205 core_width = eax & 0x1F;
206 }
207 }
208
209 apic_id = c->id;
210 c->package_id = c->id >> core_width;
211 c->smt_mask = (1 << smt_width) - 1;
212 c->smt_id = apic_id & c->smt_mask;
213 c->core_id = (apic_id >> smt_width) & ((1 << (core_width - smt_width)) - 1);
214}
215
216static void detect_cpu_capability(struct core *c) {
217 struct cpu_capability *cap = &c->cap;
218
219 detect_cpu_features(cap);
220
221 cap->uarch_id = detect_uarch_id();
222
223 detect_cpu_class(cap);
224 detect_pipeline_width(cap);
225}
226
227static struct core *setup_cpu(uint64_t cpu) {
228 struct core *c = global.cores[cpu];
229 kassert(c->id == cpu);
230 c->self = c;
231 c->current_irql = IRQL_PASSIVE_LEVEL;
232 c->tsc_hz = tsc_calibrate();
233 init_smt_info(c);
234 detect_llc(llc: &c->llc);
235 detect_cpu_capability(c);
236
237 wrmsr(MSR_GS_BASE, value: (uint64_t) c);
238 return c;
239}
240
241static inline void set_core_awake(void) {
242 atomic_fetch_add_explicit(&cores_awake, 1, memory_order_release);
243 if (atomic_load_explicit(&cores_awake, memory_order_acquire) ==
244 (global.core_count - 1)) {
245 bootstage_advance(new: BOOTSTAGE_MID_MP);
246 }
247}
248
249void smp_wakeup() {
250 disable_interrupts();
251
252 asm volatile("mov %0, %%cr3" ::"r"(cr3));
253
254 x2apic_init();
255 uint64_t cpu = cpu_get_this_id();
256 setup_cpu(cpu);
257
258 gdt_install();
259 wrmsr(MSR_GS_BASE, value: (uint64_t) global.cores[cpu]);
260 irq_load();
261
262 lapic_timer_init(core_id: cpu);
263 set_core_awake();
264
265 scheduler_yield();
266}
267
268void smp_init() {
269 for (size_t i = 0; i < global.core_count; i++) {
270 size_t d = domain_for_core(cpu: i);
271
272 if (i != 0) {
273 global.cores[i] =
274 alloc_or_die(kmalloc_from_domain(d, sizeof(struct core)));
275
276 memset(global.cores[i], 0, sizeof(struct core));
277 }
278
279 global.cores[i]->irq_entered_irql = IRQL_NONE;
280 global.cores[i]->id = i;
281 global.cores[i]->numa_node = domain_for_core(cpu: i);
282 global.cores[i]->domain = global.domains[d];
283 }
284}
285
286void smp_wait_for_others_to_idle() {
287 /* wait for them to enter idle threads */
288 size_t expected_idle = global.core_count - 1;
289 while (atomic_load(&global.idle_core_count) < expected_idle) {
290 cpu_relax();
291 }
292}
293
294void smp_wake(struct limine_mp_response *mpr) {
295 asm volatile("mov %%cr3, %0" : "=r"(cr3));
296 for (uint64_t i = 1; i < mpr->cpu_count; i++)
297 mpr->cpus[i]->goto_address = smp_wakeup;
298
299 smp_core()->tsc_hz = tsc_calibrate();
300 if (global.core_count == 1)
301 return;
302
303 /* wait for bootstage to progress */
304 while (global.current_bootstage != BOOTSTAGE_MID_MP)
305 cpu_relax();
306
307 smp_wait_for_others_to_idle();
308}
309
310void smp_setup_bsp() {
311 struct core *c = kmalloc(sizeof(struct core), ALLOC_FLAGS_ZERO);
312 if (!c)
313 panic("Could not allocate space for core structure on BSP");
314
315 c->irq_entered_irql = IRQL_NONE;
316 c->id = 0;
317 c->self = c;
318 c->current_irql = IRQL_PASSIVE_LEVEL;
319 wrmsr(MSR_GS_BASE, value: (uint64_t) c);
320 global.cores =
321 kmalloc(sizeof(struct core *) * global.core_count, ALLOC_FLAGS_ZERO);
322
323 if (unlikely(!global.cores))
324 panic("Could not allocate space for global core structures");
325
326 global.shootdown_data = kmalloc(
327 sizeof(struct tlb_shootdown_cpu) * global.core_count, ALLOC_FLAGS_ZERO);
328 if (!global.shootdown_data)
329 panic("Could not allocate global shootdown data");
330
331 global.cores[0] = c;
332 init_smt_info(c);
333 detect_llc(llc: &c->llc);
334 detect_cpu_capability(c);
335}
336
337static atomic_uint tick_change_state = 0;
338static bool enable = false;
339static uint8_t entry = 0;
340
341static enum irq_result tick_op_isr(void *ctx, uint8_t vector,
342 struct irq_context *rsp) {
343 if (enable) {
344 scheduler_tick_enable();
345 } else {
346 scheduler_tick_disable();
347 }
348
349 atomic_fetch_add(&tick_change_state, 1);
350 return IRQ_HANDLED;
351}
352
353static void send_em_all_out(bool e) {
354 enable = e;
355
356 atomic_store(&tick_change_state, 0);
357 size_t i;
358 for_each_cpu_id(i) {
359 if (i == 0)
360 continue;
361
362 ipi_send(apic_id: i, vector: entry);
363 }
364
365 /* wait for everyone to change their tick state */
366 while (atomic_load(&tick_change_state) < (global.core_count - 1))
367 cpu_relax();
368}
369
370void smp_disable_all_ticks() {
371 entry = irq_alloc_entry();
372 irq_register(name: "tick_op", vector: entry, handler: tick_op_isr, NULL, flags: IRQ_FLAG_NONE);
373 irq_set_chip(vector: entry, chip: lapic_get_chip(), NULL);
374 send_em_all_out(false);
375}
376
377extern void nop_handler(void *, uint8_t, void *);
378void smp_enable_all_ticks() {
379 send_em_all_out(true);
380 irq_free_entry(entry);
381 irq_set_chip(vector: entry, NULL, NULL);
382}
383