1/* @title: Per-CPU structure */
2#pragma once
3#include <compiler.h>
4#include <console/panic.h>
5#include <sch/irql.h>
6#include <smp/topology.h>
7#include <stdatomic.h>
8#include <stdbool.h>
9#include <stddef.h>
10#include <stdint.h>
11#include <thread/dpc.h>
12#include <types/types.h>
13
14#define CPU_FEAT_SSE2 (1ULL << 0)
15#define CPU_FEAT_AVX (1ULL << 1)
16#define CPU_FEAT_AVX2 (1ULL << 2)
17#define CPU_FEAT_AVX512F (1ULL << 3)
18
19enum cpu_class {
20 CPU_CLASS_UNKNOWN,
21 CPU_CLASS_PERFORMANCE,
22 CPU_CLASS_EFFICIENCY,
23};
24
25enum {
26 UARCH_UNKNOWN,
27 UARCH_GOLDEN_COVE,
28 UARCH_GRACEMONT,
29 UARCH_SKYLAKE,
30};
31
32struct cpu_capability {
33 enum cpu_class class;
34
35 uint32_t uarch_id; /* e.g. golden cove, gracemont */
36
37 uint32_t issue_width;
38 uint32_t retire_width;
39
40 cpu_perf_t perf_score; /* Relative to everyone else on a 0-255
41 * scale, how performant are we? The
42 * higher this number is, the more "performant"
43 * this CPU currently is, and the more likely
44 * the scheduler will decide to migrate a
45 * thread that needs such perf scores onto here. */
46
47 uint32_t energy_score; /* lower is better */
48
49 uint64_t feature_bits; /* ISA features, vector width, etc */
50};
51
52/* Let's put commonly accessed fields up here
53 * to make the cache a bit happier */
54struct core {
55 struct core *self;
56 cpu_id_t id;
57 struct thread *current_thread;
58 struct cpu_capability cap;
59
60 size_t domain_cpu_id; /* what CPU in the domain? */
61
62 /* array [domain_levels_enabled] -> domain reference */
63 struct scheduler_domain *domains[TOPOLOGY_LEVEL_MAX];
64
65 /* index within each domain's groups */
66 int32_t group_index[TOPOLOGY_LEVEL_MAX];
67
68 atomic_bool executing_dpcs;
69 atomic_bool idle;
70
71 /* This scratch buffer is stack allocated, and is set upon
72 * IRQ entry, allowing the top half of the IRQ to modify it
73 *
74 * For exception_sync_cb, it is passed into the callback as a parameter */
75 uint8_t *irq_stack_scratch_buf;
76
77 /* Execution context in one word, using SMP_CTX_* below */
78 uint32_t ctx;
79
80 enum irql current_irql;
81
82 /* Remains valid in the top half, once the bottom half is
83 * reached, this becomes IRQL_NONE */
84 enum irql irq_entered_irql; /* What IRQL were we at before
85 * entering an ISR (if !in_interrupt,
86 * this should be IRQL_NONE */
87
88 atomic_bool needs_run_dpcs; /* Set before sending IRQ_NOP, which is then
89 * checked in the isr_common_entry */
90
91 atomic_bool needs_resched;
92 atomic_bool in_resched; /* in scheduler_yield() */
93
94 struct domain *domain;
95 struct domain_arena *domain_arena;
96 size_t rr_current_domain;
97
98 struct tss *tss;
99
100 freq_khz_t lapic_khz;
101
102 struct topology_node *topo_node;
103 struct topology_cache_info llc;
104
105 numa_node_t numa_node;
106 uint32_t package_id;
107 uint32_t smt_mask;
108 uint32_t smt_id;
109 uint32_t core_id;
110
111 freq_hz_t tsc_hz;
112 time_us_t last_us;
113 uint64_t last_tsc; /* For time.c */
114
115 _Atomic uint64_t pt_seen_epoch;
116 bool reclaiming_page_tables;
117};
118
119void smp_caller_verify(enum topology_caller caller);
120#define smp_read8(off) \
121 ({ \
122 uint8_t __v; \
123 asm volatile("movb %%gs:%c1, %b0" : "=q"(__v) : "i"(off) : "memory"); \
124 __v; \
125 })
126
127#define smp_read16(off) \
128 ({ \
129 uint16_t __v; \
130 asm volatile("movw %%gs:%c1, %w0" : "=r"(__v) : "i"(off) : "memory"); \
131 __v; \
132 })
133
134#define smp_read32(off) \
135 ({ \
136 uint32_t __v; \
137 asm volatile("movl %%gs:%c1, %k0" : "=r"(__v) : "i"(off) : "memory"); \
138 __v; \
139 })
140
141#define smp_read64(off) \
142 ({ \
143 uint64_t __v; \
144 asm volatile("movq %%gs:%c1, %0" : "=r"(__v) : "i"(off) : "memory"); \
145 __v; \
146 })
147
148#define smp_member_size(member) sizeof(typeof(((struct core *) 0)->member))
149
150#define smp_read(cond, member) \
151 ({ \
152 smp_caller_verify(cond); \
153 static_assert( \
154 smp_member_size(member) == 1 || smp_member_size(member) == 2 || \
155 smp_member_size(member) == 4 || smp_member_size(member) == 8, \
156 "smp_core_read: unsupported member size"); \
157 uint64_t _raw; \
158 switch (smp_member_size(member)) { \
159 case 1: _raw = smp_read8(offsetof(struct core, member)); break; \
160 case 2: _raw = smp_read16(offsetof(struct core, member)); break; \
161 case 4: _raw = smp_read32(offsetof(struct core, member)); break; \
162 case 8: _raw = smp_read64(offsetof(struct core, member)); break; \
163 default: __builtin_unreachable(); \
164 } \
165 (typeof(__comptime_decay( \
166 ((struct core *) 0)->member))) (uintptr_t) _raw; \
167 })
168
169#define smp_write8(off, v) \
170 ({ \
171 uint8_t __v = (v); \
172 asm volatile("movb %b0, %%gs:%c1" : : "q"(__v), "i"(off) : "memory"); \
173 })
174
175#define smp_write16(off, v) \
176 ({ \
177 uint16_t __v = (v); \
178 asm volatile("movw %w0, %%gs:%c1" : : "r"(__v), "i"(off) : "memory"); \
179 })
180
181#define smp_write32(off, v) \
182 ({ \
183 uint32_t __v = (v); \
184 asm volatile("movl %0, %%gs:%c1" : : "r"(__v), "i"(off) : "memory"); \
185 })
186
187#define smp_write64(off, v) \
188 ({ \
189 uint64_t __v = (v); \
190 asm volatile("movq %0, %%gs:%c1" : : "r"(__v), "i"(off) : "memory"); \
191 })
192
193#define smp_write(cond, member, val) \
194 do { \
195 smp_caller_verify(cond); \
196 static_assert( \
197 smp_member_size(member) == 1 || smp_member_size(member) == 2 || \
198 smp_member_size(member) == 4 || smp_member_size(member) == 8, \
199 "smp_core_write: unsupported member size"); \
200 typeof(((struct core *) 0)->member) _val = (val); \
201 uint64_t _raw = (uint64_t) (uintptr_t) _val; \
202 switch (smp_member_size(member)) { \
203 case 1: \
204 smp_write8(offsetof(struct core, member), (uint8_t) _raw); \
205 break; \
206 case 2: \
207 smp_write16(offsetof(struct core, member), (uint16_t) _raw); \
208 break; \
209 case 4: \
210 smp_write32(offsetof(struct core, member), (uint32_t) _raw); \
211 break; \
212 case 8: smp_write64(offsetof(struct core, member), _raw); break; \
213 default: __builtin_unreachable(); \
214 } \
215 } while (0)
216
217static inline cpu_id_t smp_id(enum topology_caller cond) {
218 return smp_read(cond, id);
219}
220
221static inline cpu_id_t smp_id_raw(void) {
222 return smp_read(TOPC_NONE, id);
223}
224
225static inline struct core *smp_core(enum topology_caller cond) {
226 return smp_read(cond, self);
227}
228
229static inline struct core *smp_core_raw(void) {
230 return smp_read(TOPC_NONE, self);
231}
232
233/* The idea with this (TODO: Consider an enum) is that if a migration happens
234 * across reads of two different words, the whole result becomes invalid,
235 * and squishing it all into one word guarantees that such behavior
236 * is not possible. We also use counters here, as opposed to flags,
237 * so reentrancy bugs can be identified from the get-go.
238 *
239 * NOTE: needs_resched is not tracked here because that's for other
240 * CPUs to write, and this is purely local
241 */
242#define SMP_CTX_PREEMPT_SHIFT 0
243#define SMP_CTX_PREEMPT_BITS 8
244#define SMP_CTX_IRQ_SHIFT 8
245#define SMP_CTX_IRQ_BITS 8
246#define SMP_CTX_NMI_SHIFT 16
247#define SMP_CTX_NMI_BITS 4
248
249#define SMP_CTX_FIELD_MAX(bits) ((1u << (bits)) - 1u)
250
251#define SMP_CTX_PREEMPT_MASK \
252 (SMP_CTX_FIELD_MAX(SMP_CTX_PREEMPT_BITS) << SMP_CTX_PREEMPT_SHIFT)
253#define SMP_CTX_IRQ_MASK \
254 (SMP_CTX_FIELD_MAX(SMP_CTX_IRQ_BITS) << SMP_CTX_IRQ_SHIFT)
255#define SMP_CTX_NMI_MASK \
256 (SMP_CTX_FIELD_MAX(SMP_CTX_NMI_BITS) << SMP_CTX_NMI_SHIFT)
257
258#define SMP_CTX_PREEMPT_ONE (1u << SMP_CTX_PREEMPT_SHIFT)
259#define SMP_CTX_IRQ_ONE (1u << SMP_CTX_IRQ_SHIFT)
260#define SMP_CTX_NMI_ONE (1u << SMP_CTX_NMI_SHIFT)
261
262/* "not plain thread context" */
263#define SMP_CTX_IN_INTERRUPT_MASK (SMP_CTX_IRQ_MASK | SMP_CTX_NMI_MASK)
264
265struct core *smp_bsp(void);
266static inline uint32_t smp_ctx(enum topology_caller c) {
267 return smp_read(c, ctx);
268}
269
270static inline uint32_t smp_ctx_preempt_count(uint32_t smp_ctx) {
271 return (smp_ctx & SMP_CTX_PREEMPT_MASK) >> SMP_CTX_PREEMPT_SHIFT;
272}
273
274static inline uint32_t smp_ctx_irq_count(uint32_t smp_ctx) {
275 return (smp_ctx & SMP_CTX_IRQ_MASK) >> SMP_CTX_IRQ_SHIFT;
276}
277
278static inline uint32_t smp_ctx_nmi_count(uint32_t smp_ctx) {
279 return (smp_ctx & SMP_CTX_NMI_MASK) >> SMP_CTX_NMI_SHIFT;
280}
281
282/* Add a SMP_CTX_*_ONE to its field */
283static inline uint32_t smp_ctx_add(enum topology_caller c, uint32_t one,
284 uint32_t mask) {
285 struct core *cpu = smp_core(cond: c);
286 if (unlikely((cpu->ctx & mask) == mask))
287 panic("smp_ctx field overflow, mask %#x, smp_ctx %#x", mask, cpu->ctx);
288
289 cpu->ctx += one;
290 return cpu->ctx;
291}
292
293static inline uint32_t smp_ctx_sub(enum topology_caller c, uint32_t one,
294 uint32_t mask) {
295 struct core *cpu = smp_core(cond: c);
296 if (unlikely((cpu->ctx & mask) == 0))
297 panic("smp_ctx field underflow, mask %#x, smp_ctx %#x", mask, cpu->ctx);
298
299 cpu->ctx -= one;
300 return cpu->ctx;
301}
302
303#define for_each_cpu_struct(__iter) \
304 for (size_t __id = 0; \
305 ((__iter = global.cores[__id]), __id < global.core_count); __id++)
306
307#define for_each_cpu_id(__id) for (__id = 0; __id < global.core_count; __id++)
308