1#include <global.h>
2#include <inject.h>
3#include <irq/ipi.h>
4#include <irq/irq.h>
5#include <mem/alloc.h>
6#include <sch/sched.h>
7#include <string.h>
8#include <structures/cpu_mask.h>
9#include <test/perturb.h>
10#include <thread/apc.h>
11#include <thread/dpc.h>
12#include <thread/thread.h>
13#include <time/time.h>
14
15#if defined(TEST_ENABLED) || defined(TEST_NIGHTMARE_ENABLED)
16
17static struct test_perturb_config perturb_configs[] = {
18 {.name = "migrator"}, {.name = "waker"}, {.name = "apc_spammer"},
19 {.name = "stutter"}, {.name = "alloc_pressure"}, {.name = "inject_armer"},
20 {.name = "ipi_storm"}, {.name = "dpc_storm"},
21};
22
23struct test_perturb_config *test_perturb_config_lookup_n(const char *name,
24 size_t len) {
25 for (size_t i = 0; i < sizeof(perturb_configs) / sizeof(perturb_configs[0]);
26 i++) {
27 if (strlen(str: perturb_configs[i].name) == len &&
28 strncmp(s1: perturb_configs[i].name, s2: name, n: len) == 0)
29 return &perturb_configs[i];
30 }
31 return NULL;
32}
33
34struct test_perturb_config *test_perturb_config_lookup(const char *name) {
35 return test_perturb_config_lookup_n(name, len: strlen(str: name));
36}
37
38static time_us_t perturb_interval_us(const char *name, time_us_t fallback) {
39 const struct test_perturb_config *cfg = test_perturb_config_lookup(name);
40 return (cfg && cfg->interval_us) ? NS_TO_US(cfg->interval_us) : fallback;
41}
42
43static bool perturb_continue(const struct test_perturb_env *env,
44 struct test_conc_worker *self) {
45 while (!test_conc_must_stop(c: env->conc)) {
46 if (!test_conc_must_park(c: env->conc))
47 return true;
48 test_conc_park(c: env->conc, w: self);
49 }
50 return false;
51}
52
53static struct thread *perturb_pick_subject(const struct test_perturb_env *env,
54 struct test_conc_worker *self) {
55 if (!env->subject_count)
56 return NULL;
57
58 size_t idx = test_rng_next(rng: &self->rng) % env->subject_count;
59 return atomic_load_acq(&env->conc->workers[idx].th);
60}
61
62static void perturb_report(const struct test_perturb_env *env, const char *kind,
63 const char *msg) {
64 if (env->finding)
65 env->finding(kind, msg);
66}
67
68void test_perturb_migrator(const struct test_perturb_env *env,
69 struct test_conc_worker *self) {
70 time_us_t interval_us = perturb_interval_us(name: "migrator", fallback: 200);
71
72 while (perturb_continue(env, self)) {
73 struct thread *target = perturb_pick_subject(env, self);
74 if (target && global.core_count > 1) {
75 uint64_t cpu = test_rng_next(rng: &self->rng) % global.core_count;
76 thread_set_migration_target(t: target, new: (int64_t) cpu);
77 }
78
79 thread_sleep_for_us(us: interval_us);
80 }
81}
82
83void test_perturb_waker(const struct test_perturb_env *env,
84 struct test_conc_worker *self) {
85 time_us_t interval_us = perturb_interval_us(name: "waker", fallback: 250);
86
87 while (perturb_continue(env, self)) {
88 struct thread *target = perturb_pick_subject(env, self);
89 if (target)
90 thread_alert(thread: target);
91
92 thread_sleep_for_us(us: interval_us);
93 }
94}
95
96static void perturb_apc_probe(struct apc_context *ac, void *arg) {
97 cc_ignore(ac);
98 cc_ignore(arg);
99}
100
101void test_perturb_apc_spammer(const struct test_perturb_env *env,
102 struct test_conc_worker *self) {
103 time_us_t interval_us = perturb_interval_us(name: "apc_spammer", fallback: 300);
104
105 while (perturb_continue(env, self)) {
106 struct thread *target = perturb_pick_subject(env, self);
107 if (target && thread_getref(obj: target)) {
108 struct apc *apc = apc_create();
109 if (apc) {
110 apc_init(a: apc, fn: perturb_apc_probe, NULL, destroy: apc_destroy_free);
111 apc_enqueue(t: target, a: apc, type: APC_TYPE_KERNEL);
112 apc_putref(a: apc);
113 }
114 thread_putref(t: target);
115 }
116
117 thread_sleep_for_us(us: interval_us);
118 }
119}
120
121#define ALLOC_PRESSURE_SLOTS 64
122
123void test_perturb_alloc_pressure(const struct test_perturb_env *env,
124 struct test_conc_worker *self) {
125 time_us_t interval_us = perturb_interval_us(name: "alloc_pressure", fallback: 200);
126
127 void *slots[ALLOC_PRESSURE_SLOTS] = {0};
128 size_t slot_sizes[ALLOC_PRESSURE_SLOTS] = {0};
129 uint32_t slot_patterns[ALLOC_PRESSURE_SLOTS] = {0};
130
131 while (perturb_continue(env, self)) {
132 size_t slot = test_rng_next(rng: &self->rng) % ALLOC_PRESSURE_SLOTS;
133 if (slots[slot]) {
134 uint32_t *p = slots[slot];
135 size_t words = slot_sizes[slot] / sizeof(uint32_t);
136 uint32_t expected = slot_patterns[slot];
137 for (size_t w = 0; w < words; w++) {
138 if (p[w] != (expected ^ (uint32_t) w)) {
139 perturb_report(
140 env, kind: "alloc_corruption",
141 msg: "heap memory corruption in alloc_pressure slot");
142 break;
143 }
144 }
145 vfree(slots[slot]);
146 slots[slot] = NULL;
147 slot_sizes[slot] = 0; // NOLINT(*Malloc)
148 slot_patterns[slot] = 0;
149 } else {
150 size_t size = 16 + (test_rng_next(rng: &self->rng) % 8176);
151 uint32_t pat = (uint32_t) test_rng_next(rng: &self->rng);
152 void *ptr = valloc(size, ALLOC_ZERO);
153 if (ptr) {
154 uint32_t *p = ptr;
155 size_t words = size / sizeof(uint32_t);
156 for (size_t w = 0; w < words; w++)
157 p[w] = pat ^ (uint32_t) w;
158 slots[slot] = ptr;
159 slot_sizes[slot] = size;
160 slot_patterns[slot] = pat;
161 }
162 }
163
164 thread_sleep_for_us(us: interval_us);
165 }
166
167 for (size_t i = 0; i < ALLOC_PRESSURE_SLOTS; i++) {
168 if (slots[i]) {
169 vfree(slots[i]);
170 slots[i] = NULL;
171 }
172 }
173}
174
175void test_perturb_inject_armer(const struct test_perturb_env *env,
176 struct test_conc_worker *self) {
177 time_us_t interval_us = perturb_interval_us(name: "inject_armer", fallback: 500);
178
179 size_t site_count = __ekernel_inject_sites - __skernel_inject_sites;
180 if (site_count == 0) {
181 while (perturb_continue(env, self))
182 thread_sleep_for_ms(ms: 50);
183 return;
184 }
185
186 while (perturb_continue(env, self)) {
187 size_t idx = test_rng_next(rng: &self->rng) % site_count;
188 struct inject_site *site = &__skernel_inject_sites[idx];
189 uint32_t seed = (uint32_t) test_rng_next(rng: &self->rng);
190 uint32_t nth = 1 + (test_rng_next(rng: &self->rng) % 10);
191
192 inject_arm(s: site, seed, nth);
193 thread_sleep_for_us(us: interval_us);
194 inject_disarm(s: site);
195 }
196}
197
198enum storm_target : uint8_t {
199 STORM_TARGET_VICTIM,
200 STORM_TARGET_RANDOM,
201 STORM_TARGET_BROADCAST,
202};
203
204static cpu_id_t storm_pick_victim(const struct test_perturb_env *env,
205 struct test_conc_worker *self) {
206 if (env->victim_cpu >= 0 && (size_t) env->victim_cpu < global.core_count)
207 return (cpu_id_t) env->victim_cpu;
208
209 return (cpu_id_t) (test_rng_next(rng: &self->rng) % global.core_count);
210}
211
212static enum storm_target storm_roll(struct test_conc_worker *self,
213 cpu_id_t victim, cpu_id_t *out_cpu) {
214 uint64_t roll = test_rng_next(rng: &self->rng) % (TEST_PERTURB_VICTIM_BIAS + 2);
215
216 if (roll < TEST_PERTURB_VICTIM_BIAS) {
217 *out_cpu = victim;
218 return STORM_TARGET_VICTIM;
219 }
220
221 if (roll == TEST_PERTURB_VICTIM_BIAS) {
222 *out_cpu = (cpu_id_t) (test_rng_next(rng: &self->rng) % global.core_count);
223 return STORM_TARGET_RANDOM;
224 }
225
226 return STORM_TARGET_BROADCAST;
227}
228
229/* Bother with reschedule IPIs */
230void test_perturb_ipi_storm(const struct test_perturb_env *env,
231 struct test_conc_worker *self) {
232 time_us_t interval_us = perturb_interval_us(name: "ipi_storm", fallback: 100);
233 cpu_id_t victim = storm_pick_victim(env, self);
234
235 struct cpu_mask all = CPU_MASK_INIT;
236 cpu_mask_set_all(&all);
237
238 while (perturb_continue(env, self)) {
239 cpu_id_t cpu;
240 if (storm_roll(self, victim, out_cpu: &cpu) == STORM_TARGET_BROADCAST) {
241 size_t i;
242 for_each_cpu_id(i) {
243 scheduler_mark_core_needs_resched(c: global.cores[i], true);
244 }
245 ipi_send_mask(targets: &all, IRQ_SCHEDULER, flags: IPI_MASK_SUPERSET_OK);
246 } else {
247 scheduler_force_resched(sched: global.schedulers[cpu]);
248 }
249
250 thread_sleep_for_us(us: interval_us);
251 }
252}
253
254/* How long storm DPC stays DISPATCH_LEVEL */
255#define DPC_STORM_SPIN 256
256
257#define DPC_STORM_POOL 64
258
259static void dpc_storm_spin(void) {
260 for (volatile size_t i = 0; i < DPC_STORM_SPIN; i++)
261 cpu_pause();
262}
263
264static void dpc_storm_fn(struct dpc_context *dc, void *a, void *b) {
265 cc_ignore(dc);
266 cc_ignore(a, b);
267 dpc_storm_spin();
268}
269
270static struct dpc dpc_storm_pool[DPC_STORM_POOL] = {
271 [0 ... DPC_STORM_POOL - 1] = {.func = dpc_storm_fn},
272};
273
274void test_perturb_dpc_storm(const struct test_perturb_env *env,
275 struct test_conc_worker *self) {
276 time_us_t interval_us = perturb_interval_us(name: "dpc_storm", fallback: 200);
277 cpu_id_t victim = storm_pick_victim(env, self);
278
279 /* dpc_fanout() waits for every CPU, so this can be per-thread */
280 struct dpc *fanout = vnews(fanout, global.core_count, ALLOC_ZERO);
281
282 struct cpu_mask all = CPU_MASK_INIT;
283 cpu_mask_set_all(&all);
284
285 while (perturb_continue(env, self)) {
286 cpu_id_t cpu;
287 if (storm_roll(self, victim, out_cpu: &cpu) == STORM_TARGET_BROADCAST) {
288 if (fanout)
289 dpc_fanout(storage: fanout, cpus: all, fn: dpc_storm_spin);
290 } else {
291 size_t slot = test_rng_next(rng: &self->rng) % DPC_STORM_POOL;
292 dpc_enqueue_on_cpu(cpu, d: &dpc_storm_pool[slot]);
293 }
294
295 thread_sleep_for_us(us: interval_us);
296 }
297
298 vfree(fanout);
299}
300
301static const struct test_perturb_desc perturb_descs[] = {
302 {.name = "migrator", .fn = test_perturb_migrator},
303 {.name = "waker", .fn = test_perturb_waker},
304 {.name = "apc_spammer", .fn = test_perturb_apc_spammer},
305 {.name = "alloc_pressure", .fn = test_perturb_alloc_pressure},
306 {.name = "inject_armer", .fn = test_perturb_inject_armer},
307 {.name = "ipi_storm", .fn = test_perturb_ipi_storm},
308 {.name = "dpc_storm", .fn = test_perturb_dpc_storm},
309};
310
311const struct test_perturb_desc *test_perturb_lookup(const char *name) {
312 if (!name)
313 return NULL;
314
315 for (size_t i = 0; i < sizeof(perturb_descs) / sizeof(perturb_descs[0]);
316 i++) {
317 if (strcmp(str1: perturb_descs[i].name, str2: name) == 0)
318 return &perturb_descs[i];
319 }
320 return NULL;
321}
322
323#endif /* TEST_ENABLED || TEST_NIGHTMARE_ENABLED */
324