1/* @title: Per-CPU structure */
2#pragma once
3#include <atomic.h>
4#include <compiler/core.h>
5#include <compiler/intrinsic.h>
6#include <console/panic.h>
7#include <math/bit.h>
8#include <sch/irql.h>
9#include <smp/topology.h>
10#include <stdbool.h>
11#include <stddef.h>
12#include <stdint.h>
13#include <thread/dpc.h>
14#include <types/types.h>
15
16#define CPU_FEAT_SSE2 BIT(0)
17#define CPU_FEAT_AVX BIT(1)
18#define CPU_FEAT_AVX2 BIT(2)
19#define CPU_FEAT_AVX512F BIT(3)
20
21enum cpu_class {
22 CPU_CLASS_UNKNOWN,
23 CPU_CLASS_PERFORMANCE,
24 CPU_CLASS_EFFICIENCY,
25};
26
27enum {
28 UARCH_UNKNOWN,
29 UARCH_GOLDEN_COVE,
30 UARCH_GRACEMONT,
31 UARCH_SKYLAKE,
32};
33
34struct cpu_capability {
35 enum cpu_class class;
36
37 uint32_t uarch_id; /* e.g. golden cove, gracemont */
38
39 uint32_t issue_width;
40 uint32_t retire_width;
41
42 cpu_perf_t perf_score; /* Relative to everyone else on a 0-255
43 * scale, how performant are we? The
44 * higher this number is, the more "performant"
45 * this CPU currently is, and the more likely
46 * the scheduler will decide to migrate a
47 * thread that needs such perf scores onto here. */
48
49 uint32_t energy_score; /* lower is better */
50
51 uint64_t feature_bits; /* ISA features, vector width, etc */
52};
53
54/* Let's put commonly accessed fields up here
55 * to make the cache a bit happier */
56struct core {
57 struct core *self;
58 cpu_id_t id;
59 struct thread *current_thread;
60 struct cpu_capability cap;
61
62 size_t domain_cpu_id; /* what CPU in the domain? */
63
64 /* array [domain_levels_enabled] -> domain reference */
65 struct scheduler_domain *domains[TOPOLOGY_LEVEL_MAX];
66
67 /* index within each domain's groups */
68 int32_t group_index[TOPOLOGY_LEVEL_MAX];
69
70 atomic_bool executing_dpcs;
71 atomic_bool idle;
72
73 /* Execution context in one word, using SMP_CTX_* below */
74 uint32_t ctx;
75
76 enum irql current_irql;
77
78 atomic_bool needs_run_dpcs; /* Set before sending IRQ_NOP, which is then
79 * checked in the isr_common_entry */
80
81 atomic_bool needs_resched;
82 atomic_bool in_resched; /* in scheduler_yield() */
83
84 struct domain *domain;
85 struct domain_arena *domain_arena;
86 size_t rr_current_domain;
87
88 struct tss *tss;
89
90 freq_khz_t lapic_khz;
91
92 struct topology_node *topo_node;
93 struct topology_cache_info llc;
94
95 numa_node_t numa_node;
96 uint32_t package_id;
97 uint32_t smt_mask;
98 uint32_t smt_id;
99 uint32_t core_id;
100
101 freq_hz_t tsc_hz;
102 time_us_t last_us;
103 uint64_t last_tsc; /* For time.c */
104
105 atomic_uint64_t pt_seen_epoch;
106 bool reclaiming_page_tables;
107};
108
109void smp_caller_verify(enum topology_caller caller);
110#define smp_read8(off) \
111 ({ \
112 uint8_t __smp_v; \
113 asm volatile("movb %%gs:%c1, %b0" \
114 : "=q"(__smp_v) \
115 : "i"(off) \
116 : "memory"); \
117 __smp_v; \
118 })
119
120#define smp_read16(off) \
121 ({ \
122 uint16_t __smp_v; \
123 asm volatile("movw %%gs:%c1, %w0" \
124 : "=r"(__smp_v) \
125 : "i"(off) \
126 : "memory"); \
127 __smp_v; \
128 })
129
130#define smp_read32(off) \
131 ({ \
132 uint32_t __smp_v; \
133 asm volatile("movl %%gs:%c1, %k0" \
134 : "=r"(__smp_v) \
135 : "i"(off) \
136 : "memory"); \
137 __smp_v; \
138 })
139
140#define smp_read64(off) \
141 ({ \
142 uint64_t __smp_v; \
143 asm volatile("movq %%gs:%c1, %0" \
144 : "=r"(__smp_v) \
145 : "i"(off) \
146 : "memory"); \
147 __smp_v; \
148 })
149
150#define smp_member_size(member) sizeof(typeof(((struct core *) 0)->member))
151
152#if defined(cn_id_clang)
153#define _smp_read_type(member) (typeof(ct_decay(((struct core *) 0)->member)))
154#else
155#define _smp_read_type(member) (typeof(((struct core *) 0)->member))
156#endif
157
158#define smp_read(cond, member) \
159 ({ \
160 if ((cond) != TOPC_NONE) \
161 smp_caller_verify(cond); \
162 static_assert( \
163 smp_member_size(member) == 1 || smp_member_size(member) == 2 || \
164 smp_member_size(member) == 4 || smp_member_size(member) == 8, \
165 "smp_core_read: unsupported member size"); \
166 uint64_t __smp_raw; \
167 switch (smp_member_size(member)) { \
168 case 1: __smp_raw = smp_read8(offsetof(struct core, member)); break; \
169 case 2: __smp_raw = smp_read16(offsetof(struct core, member)); break; \
170 case 4: __smp_raw = smp_read32(offsetof(struct core, member)); break; \
171 case 8: __smp_raw = smp_read64(offsetof(struct core, member)); break; \
172 default: ci_unreachable(); \
173 } \
174 _smp_read_type(member)(uintptr_t) __smp_raw; \
175 })
176
177#define smp_write8(off, v) \
178 ({ \
179 uint8_t __smp_v = (v); \
180 asm volatile("movb %b0, %%gs:%c1" \
181 : \
182 : "q"(__smp_v), "i"(off) \
183 : "memory"); \
184 })
185
186#define smp_write16(off, v) \
187 ({ \
188 uint16_t __smp_v = (v); \
189 asm volatile("movw %w0, %%gs:%c1" \
190 : \
191 : "r"(__smp_v), "i"(off) \
192 : "memory"); \
193 })
194
195#define smp_write32(off, v) \
196 ({ \
197 uint32_t __smp_v = (v); \
198 asm volatile("movl %0, %%gs:%c1" \
199 : \
200 : "r"(__smp_v), "i"(off) \
201 : "memory"); \
202 })
203
204#define smp_write64(off, v) \
205 ({ \
206 uint64_t __smp_v = (v); \
207 asm volatile("movq %0, %%gs:%c1" \
208 : \
209 : "r"(__smp_v), "i"(off) \
210 : "memory"); \
211 })
212
213#define smp_write(cond, member, val) \
214 do { \
215 if ((cond) != TOPC_NONE) \
216 smp_caller_verify(cond); \
217 static_assert( \
218 smp_member_size(member) == 1 || smp_member_size(member) == 2 || \
219 smp_member_size(member) == 4 || smp_member_size(member) == 8, \
220 "smp_core_write: unsupported member size"); \
221 typeof(((struct core *) 0)->member) __smp_val = (val); \
222 uint64_t __smp_raw = (uint64_t) (uintptr_t) __smp_val; \
223 switch (smp_member_size(member)) { \
224 case 1: \
225 smp_write8(offsetof(struct core, member), (uint8_t) __smp_raw); \
226 break; \
227 case 2: \
228 smp_write16(offsetof(struct core, member), (uint16_t) __smp_raw); \
229 break; \
230 case 4: \
231 smp_write32(offsetof(struct core, member), (uint32_t) __smp_raw); \
232 break; \
233 case 8: smp_write64(offsetof(struct core, member), __smp_raw); break; \
234 default: ci_unreachable(); \
235 } \
236 } while (0)
237
238static inline cpu_id_t smp_id(enum topology_caller cond) {
239 return smp_read(cond, id);
240}
241
242static inline cpu_id_t smp_id_raw(void) {
243 return smp_read(TOPC_NONE, id);
244}
245
246extern uint32_t *smp_apic_ids;
247
248static inline uint32_t smp_apic_id(cpu_id_t cpu) {
249 return smp_apic_ids[cpu];
250}
251
252/* CPU_ID_NONE if no CPU booted has apic_id */
253cpu_id_t smp_cpu_from_apic_id(uint32_t apic_id);
254
255static inline struct core *smp_core(enum topology_caller cond) {
256 return smp_read(cond, self);
257}
258
259static inline struct core *smp_core_raw(void) {
260 return smp_read(TOPC_NONE, self);
261}
262
263/* The idea with this (TODO: Consider an enum) is that if a migration happens
264 * across reads of two different words, the whole result becomes invalid,
265 * and squishing it all into one word guarantees that such behavior
266 * is not possible. We also use counters here, as opposed to flags,
267 * so reentrancy bugs can be identified from the get-go.
268 *
269 * NOTE: needs_resched is not tracked here because that's for other
270 * CPUs to write, and this is purely local
271 */
272#define SMP_CTX_PREEMPT_SHIFT 0
273#define SMP_CTX_PREEMPT_BITS 8
274#define SMP_CTX_IRQ_SHIFT 8
275#define SMP_CTX_IRQ_BITS 8
276#define SMP_CTX_NMI_SHIFT 16
277#define SMP_CTX_NMI_BITS 4
278
279#define SMP_CTX_FIELD_MAX(bits) ((1u << (bits)) - 1u)
280
281#define SMP_CTX_PREEMPT_MASK \
282 (SMP_CTX_FIELD_MAX(SMP_CTX_PREEMPT_BITS) << SMP_CTX_PREEMPT_SHIFT)
283#define SMP_CTX_IRQ_MASK \
284 (SMP_CTX_FIELD_MAX(SMP_CTX_IRQ_BITS) << SMP_CTX_IRQ_SHIFT)
285#define SMP_CTX_NMI_MASK \
286 (SMP_CTX_FIELD_MAX(SMP_CTX_NMI_BITS) << SMP_CTX_NMI_SHIFT)
287
288#define SMP_CTX_PREEMPT_ONE (1u << SMP_CTX_PREEMPT_SHIFT)
289#define SMP_CTX_IRQ_ONE (1u << SMP_CTX_IRQ_SHIFT)
290#define SMP_CTX_NMI_ONE (1u << SMP_CTX_NMI_SHIFT)
291
292/* "not plain thread context" */
293#define SMP_CTX_IN_INTERRUPT_MASK (SMP_CTX_IRQ_MASK | SMP_CTX_NMI_MASK)
294
295struct core *smp_bsp(void);
296static inline uint32_t smp_ctx(enum topology_caller c) {
297 return smp_read(c, ctx);
298}
299
300static inline uint32_t smp_ctx_preempt_count(uint32_t smp_ctx) {
301 return (smp_ctx & SMP_CTX_PREEMPT_MASK) >> SMP_CTX_PREEMPT_SHIFT;
302}
303
304static inline uint32_t smp_ctx_irq_count(uint32_t smp_ctx) {
305 return (smp_ctx & SMP_CTX_IRQ_MASK) >> SMP_CTX_IRQ_SHIFT;
306}
307
308static inline uint32_t smp_ctx_nmi_count(uint32_t smp_ctx) {
309 return (smp_ctx & SMP_CTX_NMI_MASK) >> SMP_CTX_NMI_SHIFT;
310}
311
312/* Add a SMP_CTX_*_ONE to its field */
313static inline uint32_t smp_ctx_add(enum topology_caller c, uint32_t one,
314 uint32_t mask) {
315 struct core *cpu = smp_core(cond: c);
316 if (cc_unlikely((cpu->ctx & mask) == mask))
317 panic(OVERFLOW, "smp_ctx mask %#x, ctx %#x", mask, cpu->ctx);
318
319 cpu->ctx += one;
320 return cpu->ctx;
321}
322
323static inline uint32_t smp_ctx_sub(enum topology_caller c, uint32_t one,
324 uint32_t mask) {
325 struct core *cpu = smp_core(cond: c);
326 if (cc_unlikely((cpu->ctx & mask) == 0))
327 panic(UNDERFLOW, "smp_ctx mask %#x, ctx %#x", mask, cpu->ctx);
328
329 cpu->ctx -= one;
330 return cpu->ctx;
331}
332
333#define for_each_cpu_struct(iter_) \
334 for (size_t __cpu_idx = 0; __cpu_idx < global.core_count && \
335 (((iter_) = global.cores[__cpu_idx]), true); \
336 __cpu_idx++)
337
338#define for_each_cpu_id(id_) for ((id_) = 0; (id_) < global.core_count; (id_)++)
339