1#ifdef TEST_NIGHTMARE_ENABLED
2#include "internal.h"
3
4#include <asm.h>
5#include <console/panic.h>
6#include <console/printf.h>
7#include <global.h>
8#include <math/units.h>
9#include <mem/alloc.h>
10#include <mem/pmm.h>
11#include <ndjson.h>
12#include <nightmare/perturb.h>
13#include <nightmare/record.h>
14#include <smp/smp.h>
15#include <string.h>
16#include <thread/thread.h>
17#include <time/time.h>
18
19#ifndef CHARMOS_COMMIT
20#define CHARMOS_COMMIT "unknown"
21#endif
22
23#define NIGHTMARE_MIB MIB(1)
24#define NIGHTMARE_DEFAULT_DURATION_MS 1000
25#define NIGHTMARE_DEFAULT_DRAIN_MS 20000
26#define NIGHTMARE_DEFAULT_STAT_MS 5000
27
28struct nightmare_runtime nightmare_runtime;
29
30static void nightmare_exit(uint8_t code, const char *reason) {
31 ndjson_bye(code, reason);
32 qemu_exit(code);
33 while (true)
34 cpu_pause();
35}
36
37static uint8_t nightmare_exit_for_result(enum nightmare_result result) {
38 switch (result) {
39 case NIGHTMARE_RESULT_OK: return NIGHTMARE_EXIT_OK;
40 case NIGHTMARE_RESULT_FINDING: return NIGHTMARE_EXIT_FINDING;
41 case NIGHTMARE_RESULT_FAIL: return NIGHTMARE_EXIT_FAIL;
42 case NIGHTMARE_RESULT_STALL: return NIGHTMARE_EXIT_STALL;
43 case NIGHTMARE_RESULT_SKIP: return NIGHTMARE_EXIT_SKIP;
44 default: return NIGHTMARE_EXIT_FAIL;
45 }
46}
47
48static uint64_t nightmare_progress_snapshot(void) {
49#ifdef TEST_NIGHTMARE_ENABLED
50 return test_conc_progress_sum();
51#else
52 return 0;
53#endif
54}
55
56static void append_cap(char *caps, size_t cap, const char *value) {
57 size_t used = strlen(str: caps);
58 if (used >= cap - 1)
59 return;
60 snprintf(caps + used, (int) (cap - used), "%s%s", used ? "," : "", value);
61}
62
63static void nightmare_build_caps(void) {
64 nightmare_runtime.caps[0] = '\0';
65 append_cap(caps: nightmare_runtime.caps, cap: sizeof(nightmare_runtime.caps),
66 value: "preempt");
67
68#ifdef DEBUG_ASAN
69 append_cap(nightmare_runtime.caps, sizeof(nightmare_runtime.caps), "asan");
70#endif
71
72#ifdef DEBUG_LOCK_CHK
73 append_cap(nightmare_runtime.caps, sizeof(nightmare_runtime.caps),
74 "lock_chk");
75#endif
76
77#ifdef INJECT_ENABLED
78 append_cap(nightmare_runtime.caps, sizeof(nightmare_runtime.caps),
79 "inject");
80#endif
81}
82
83#ifdef TEST_NIGHTMARE_ENABLED
84static const struct nightmare *nightmare_resolve(const char *name) {
85 const struct nightmare *match = NULL;
86 for (struct nightmare *nm = __skernel_nightmares; nm < __ekernel_nightmares;
87 nm++) {
88 if (!nm->name)
89 nightmare_panic("nightmare registry contains an unnamed entry");
90 if (strcmp(str1: nm->name, str2: name) != 0)
91 continue;
92 if (match)
93 nightmare_panic("duplicate nightmare name '%s' in %s and %s", name,
94 match->fname, nm->fname);
95 match = nm;
96 }
97 return match;
98}
99#endif
100
101static void
102nightmare_record_resolved_boot(const struct nightmare_cmdline_config *config,
103 const char *requested) {
104 const struct nightmare *nm = nightmare_runtime.ctx.nm;
105 nightmare_record_boot(record: &(struct nightmare_boot_record){
106 .test = nm ? nm->name : requested,
107 .seed = config->seed,
108 .seed_policy =
109 nm ? nightmare_seed_policy_to_str(stringify_in: nm->seed_policy) : "unknown",
110 .seed_mode = nightmare_seed_mode_to_str(stringify_in: config->seed_mode),
111 .intensity = nightmare_runtime.ctx.intensity,
112 .intensity_val = nightmare_runtime.ctx.intensity_val,
113 .workers = nightmare_runtime.ctx.worker_count,
114 .smp = global.core_count,
115 .mem_mib = pmm_get_usable_ram() / NIGHTMARE_MIB,
116 .caps = nightmare_runtime.caps,
117 .campaign_id = config->campaign_id ? config->campaign_id : "",
118 .boot_index = config->boot_index,
119 .commit = CHARMOS_COMMIT,
120 });
121}
122
123static void nightmare_emit_verdict(struct nightmare_verdict verdict,
124 const char *fallback_reason) {
125 size_t findings = atomic_load_acq(&nightmare_runtime.finding_count);
126 enum nightmare_result result =
127 nightmare_result_with_findings(result: verdict.result, findings);
128
129 const char *reason = verdict.reason;
130 if (result == NIGHTMARE_RESULT_SKIP)
131 reason = nightmare_skip_to_str(stringify_in: verdict.skip_reason);
132 if (!reason)
133 reason = fallback_reason;
134
135 time_ms_t now = time_get_ms();
136 uint64_t elapsed =
137 nightmare_runtime.started_ms ? now - nightmare_runtime.started_ms : 0;
138
139 nightmare_record_verdict(record: &(struct nightmare_verdict_record){
140 .result = nightmare_result_to_str(stringify_in: result),
141 .reason = reason ? reason : "unknown",
142 .duration_ms = elapsed,
143 .progress = nightmare_progress_snapshot(),
144 .findings = findings,
145 .msg = verdict.msg ? verdict.msg : "",
146 });
147
148 nightmare_exit(code: nightmare_exit_for_result(result),
149 reason: reason ? reason : nightmare_result_to_str(stringify_in: result));
150}
151
152enum nightmare_result
153nightmare_result_with_findings(enum nightmare_result result, size_t findings) {
154 if (result == NIGHTMARE_RESULT_OK && findings)
155 return NIGHTMARE_RESULT_FINDING;
156 return result;
157}
158
159#ifdef TEST_NIGHTMARE_ENABLED
160static enum nightmare_skip_reason
161nightmare_seed_refusal(const struct nightmare *nm,
162 const struct nightmare_cmdline_config *config) {
163 if (config->seed_mode != NIGHTMARE_SEED_SEEDLESS && !config->seed_present)
164 return NIGHTMARE_SKIP_SEED_MISSING;
165
166 if (nm->seed_policy == NIGHTMARE_SEED_IGNORED &&
167 config->seed_mode == NIGHTMARE_SEED_SEEDFUL)
168 return NIGHTMARE_SKIP_SEED_UNUSED;
169
170 if (nm->seed_policy == NIGHTMARE_SEED_REQUIRED &&
171 config->seed_mode != NIGHTMARE_SEED_SEEDFUL)
172 return NIGHTMARE_SKIP_SEED_MISSING;
173
174 return NIGHTMARE_SKIP_NONE;
175}
176
177static bool
178nightmare_has_missing_service(const struct nightmare *nm,
179 const struct nightmare_cmdline_config *config) {
180 if (nm->perturb) {
181 for (const char *const *name = nm->perturb; *name; name++) {
182 if (!nightmare_perturb_lookup(name: *name))
183 return true;
184 }
185 }
186
187 if (!config->perturb_present)
188 return false;
189 for (size_t i = 0; i < config->perturb.count; i++) {
190 const char *name = NULL;
191 if (cmdline_extract_const_string(val: &config->perturb.items[i], out: &name) !=
192 ERR_OK ||
193 !nightmare_perturb_lookup(name))
194 return true;
195 }
196 return false;
197}
198
199static enum nightmare_skip_reason
200nightmare_preflight(const struct nightmare *nm,
201 const struct nightmare_cmdline_config *config) {
202 if ((nm->requires & NIGHTMARE_REQ_SMP) && global.core_count < 2)
203 return NIGHTMARE_SKIP_NEEDS_SMP;
204 /* Today, we always have preemption enabled, however TODO: someday
205 * we can implement support for a non-preemptive kernel*/
206 if ((nm->requires & NIGHTMARE_REQ_PREEMPT) == 0) {
207 /* No-op */
208 }
209
210#ifndef DEBUG_ASAN
211 if (nm->requires & NIGHTMARE_REQ_ASAN)
212 return NIGHTMARE_SKIP_NEEDS_ASAN;
213#endif
214
215#ifndef INJECT_ENABLED
216 if (nm->requires & NIGHTMARE_REQ_INJECT)
217 return NIGHTMARE_SKIP_NEEDS_INJECT;
218#endif
219
220 if (pmm_get_usable_ram() / NIGHTMARE_MIB < nm->min_mem_mib)
221 return NIGHTMARE_SKIP_RAM_LOW;
222
223 enum nightmare_skip_reason seed = nightmare_seed_refusal(nm, config);
224 if (seed != NIGHTMARE_SKIP_NONE)
225 return seed;
226 if (nightmare_has_missing_service(nm, config))
227 return NIGHTMARE_SKIP_SERVICE_MISSING;
228 return NIGHTMARE_SKIP_NONE;
229}
230
231static void nightmare_soft_deadline(struct timer *timer) {
232 cc_ignore(timer);
233 nightmare_publish_stop(reason: TEST_STOP_BUDGET);
234}
235
236static void nightmare_hard_deadline(struct timer *timer) {
237 cc_ignore(timer);
238 bool expected = false;
239 if (!atomic_cas_strong(&nightmare_runtime.terminal, &expected, true,
240 mo_acq_rel, mo_acquire))
241 return;
242
243 nightmare_publish_stop(reason: TEST_STOP_STALL);
244 ndjson_enter_panic();
245 nightmare_record_verdict(record: &(struct nightmare_verdict_record){
246 .result = "stall",
247 .reason = "drain_timeout",
248 .duration_ms = time_get_ms() - nightmare_runtime.started_ms,
249 .progress = test_conc_progress_sum(),
250 .findings = atomic_load_rlx(&nightmare_runtime.finding_count),
251 .msg = "hard deadline expired before teardown completed",
252 });
253 nightmare_exit(code: NIGHTMARE_EXIT_STALL, reason: "hard deadline");
254}
255
256static void nightmare_arm_deadlines(time_ms_t duration_ms,
257 time_ms_t drain_grace_ms) {
258 time_us_t now = time_get_us();
259 timer_init(timer: &nightmare_runtime.soft_timer, func: nightmare_soft_deadline, NULL);
260 nightmare_runtime.soft_timer.flags = TIMER_FLAG_IRQ;
261 nightmare_runtime.soft_timer.expiration_us = now + MS_TO_US(duration_ms);
262 timer_add_global(timer: &nightmare_runtime.soft_timer);
263
264 timer_init(timer: &nightmare_runtime.hard_timer, func: nightmare_hard_deadline, NULL);
265 nightmare_runtime.hard_timer.flags = TIMER_FLAG_IRQ;
266 nightmare_runtime.hard_timer.expiration_us =
267 now + MS_TO_US(duration_ms + drain_grace_ms);
268 timer_add_global(timer: &nightmare_runtime.hard_timer);
269}
270
271static uint64_t nightmare_worker_seed(const struct nightmare_ctx *ctx,
272 size_t index) {
273 bool seeded =
274 ctx->seed_mode == NIGHTMARE_SEED_SEEDFUL ||
275 (ctx->seed_mode == NIGHTMARE_SEED_SPLIT && index >= ctx->worker_count);
276 uint64_t base = seeded ? ctx->seed : time_get_ns();
277 return test_rng_seed_for(base_seed: base, index);
278}
279
280static bool nightmare_spawn_threads(void) {
281 size_t created = 0;
282 for (size_t i = 0; i < nightmare_runtime.conc.worker_count; i++) {
283 struct test_conc_worker *worker = &nightmare_runtime.conc.workers[i];
284 worker->index = i;
285 if (i < nightmare_runtime.ctx.worker_count) {
286 worker->role = "worker";
287 } else {
288 size_t pidx = i - nightmare_runtime.ctx.worker_count;
289 worker->role = nightmare_runtime.perturbers[pidx]->name;
290 }
291 worker->rng.state =
292 nightmare_worker_seed(ctx: &nightmare_runtime.ctx, index: worker->index);
293 struct thread *thread = thread_spawn(
294 ("nightmare_%s_%zu", worker->role, i), nightmare_thread_main,
295 .arg = worker, .joinable = true);
296 if (!thread)
297 goto fail;
298 enforce(thread_getref(thread), STALE_REFERENCE);
299 atomic_store_release(&worker->th, thread);
300 created++;
301 }
302
303 /* Not actually a worker, but store in conc.aux */
304 struct thread *heartbeat = thread_spawn(
305 "nightmare_heartbeat", nightmare_heartbeat_main, .joinable = true);
306 if (!heartbeat)
307 goto fail;
308 enforce(thread_getref(heartbeat), STALE_REFERENCE);
309 atomic_store_release(&nightmare_runtime.conc.aux, heartbeat);
310 return true;
311
312fail:
313 nightmare_publish_stop(reason: TEST_STOP_FAIL);
314 event_set(e: &nightmare_runtime.conc.start);
315 for (size_t i = created; i > 0; i--) {
316 struct thread *th =
317 atomic_load_acq(&nightmare_runtime.conc.workers[i - 1].th);
318 if (th)
319 thread_join(t: th);
320 }
321 return false;
322}
323
324static void nightmare_join_threads(void) {
325 for (size_t i = nightmare_runtime.ctx.worker_count;
326 i < nightmare_runtime.conc.worker_count; i++) {
327 struct thread *thread =
328 atomic_load_acq(&nightmare_runtime.conc.workers[i].th);
329 if (thread)
330 thread_join(t: thread);
331 }
332 for (size_t i = 0; i < nightmare_runtime.ctx.worker_count; i++) {
333 struct thread *thread =
334 atomic_load_acq(&nightmare_runtime.conc.workers[i].th);
335 if (thread)
336 thread_join(t: thread);
337 }
338 struct thread *heartbeat = atomic_load_acq(&nightmare_runtime.conc.aux);
339 if (heartbeat)
340 thread_join(t: heartbeat);
341}
342
343/* All workers are joined and timer/probe callbacks are quiescent */
344static void nightmare_release_threads(void) {
345 for (size_t i = 0; i < nightmare_runtime.conc.worker_count; i++) {
346 struct thread *thread =
347 atomic_xchg_acq_rel(&nightmare_runtime.conc.workers[i].th, NULL);
348 if (thread)
349 thread_putref(t: thread);
350 }
351 struct thread *heartbeat =
352 atomic_xchg_acq_rel(&nightmare_runtime.conc.aux, NULL);
353 if (heartbeat)
354 thread_putref(t: heartbeat);
355}
356
357void nightmare_publish_perturb_verdict(struct nightmare_verdict verdict) {
358 if (verdict.result == NIGHTMARE_RESULT_OK ||
359 atomic_load_acq(&nightmare_runtime.perturb_verdict_ready))
360 return;
361
362 if (verdict.reason) {
363 snprintf(nightmare_runtime.perturb_reason,
364 sizeof(nightmare_runtime.perturb_reason), "%s",
365 verdict.reason);
366 verdict.reason = nightmare_runtime.perturb_reason;
367 }
368 if (verdict.msg) {
369 snprintf(nightmare_runtime.perturb_msg,
370 sizeof(nightmare_runtime.perturb_msg), "%s", verdict.msg);
371 verdict.msg = nightmare_runtime.perturb_msg;
372 }
373 nightmare_runtime.perturb_verdict = verdict;
374 atomic_store_release(&nightmare_runtime.perturb_verdict_ready, true);
375}
376
377bool nightmare_load_perturb_verdict(struct nightmare_verdict *out) {
378 if (!atomic_load_acq(&nightmare_runtime.perturb_verdict_ready))
379 return false;
380 *out = nightmare_runtime.perturb_verdict;
381 return true;
382}
383static void nightmare_add_perturber(const struct test_perturb_desc *desc) {
384 if (!desc || nightmare_runtime.perturber_count >= NIGHTMARE_MAX_PERTURBERS)
385 return;
386 for (size_t i = 0; i < nightmare_runtime.perturber_count; i++) {
387 if (nightmare_runtime.perturbers[i] == desc)
388 return;
389 }
390 nightmare_runtime.perturbers[nightmare_runtime.perturber_count++] = desc;
391}
392
393static void
394nightmare_collect_perturbers(const struct nightmare *nm,
395 const struct nightmare_cmdline_config *config) {
396 nightmare_runtime.perturber_count = 0;
397 if (nm->perturb) {
398 for (const char *const *name = nm->perturb; *name; name++)
399 nightmare_add_perturber(desc: nightmare_perturb_lookup(name: *name));
400 }
401 if (config->perturb_present) {
402 for (size_t i = 0; i < config->perturb.count; i++) {
403 const char *name = NULL;
404 if (cmdline_extract_const_string(val: &config->perturb.items[i],
405 out: &name) == ERR_OK &&
406 name)
407 nightmare_add_perturber(desc: nightmare_perturb_lookup(name));
408 }
409 }
410}
411
412static struct nightmare_verdict
413nightmare_finalize_verdict(const struct nightmare *nm) {
414 struct nightmare_verdict final = NIGHTMARE_OK;
415 struct nightmare_verdict perturb_verdict;
416 bool has_perturb_verdict = nightmare_load_perturb_verdict(out: &perturb_verdict);
417 if (nm->ops && nm->ops->quiesce_check) {
418 final = nm->ops->quiesce_check(&nightmare_runtime.ctx);
419 nightmare_record_quiesce(record: &(struct nightmare_quiesce_record){
420 .result = nightmare_result_to_str(stringify_in: final.result),
421 .checks = 1,
422 });
423 }
424
425 if (has_perturb_verdict) {
426 final = perturb_verdict;
427 } else if (final.result == NIGHTMARE_RESULT_OK && nm->ops &&
428 nm->ops->finish) {
429 final = nm->ops->finish(&nightmare_runtime.ctx);
430 }
431
432 enum test_stop stop = atomic_load_acq(&nightmare_runtime.conc.stop);
433 return nightmare_verdict_for_stop(verdict: final, stop);
434}
435
436struct nightmare_verdict
437nightmare_verdict_for_stop(struct nightmare_verdict final,
438 enum test_stop stop) {
439 if (stop == TEST_STOP_STALL)
440 final = (struct nightmare_verdict){
441 .result = NIGHTMARE_RESULT_STALL,
442 .reason = "liveness",
443 };
444 else if (stop == TEST_STOP_FAIL && final.result == NIGHTMARE_RESULT_OK)
445 final = NIGHTMARE_FAIL("harness", "harness requested termination");
446
447 return final;
448}
449#endif
450
451void nightmare_run(void) {
452 struct nightmare_cmdline_config config;
453 nightmare_cmdline_get(config: &config);
454 if (!config.selector)
455 return;
456
457 nightmare_runtime = (struct nightmare_runtime){
458 .ctx = {.seed = config.seed,
459 .seed_present = config.seed_present,
460 .seed_mode = config.seed_mode},
461 .finding_count = ATOMIC_VAR_INIT(0),
462 .terminal = ATOMIC_VAR_INIT(false),
463 .perturb_verdict_ready = ATOMIC_VAR_INIT(false),
464 .stat_interval_ms = config.stat_interval_ms ? config.stat_interval_ms
465 : NIGHTMARE_DEFAULT_STAT_MS,
466 .campaign_id = config.campaign_id,
467 .boot_index = config.boot_index,
468 };
469 /* Worker slots don't get sized until perturbers get collected,
470 * so we init empty and re-point once perturbers arrive */
471 test_conc_init(c: &nightmare_runtime.conc, NULL, worker_count: 0);
472 nightmare_runtime.started_ms = time_get_ms();
473 nightmare_build_caps();
474
475#ifndef TEST_NIGHTMARE_ENABLED
476 nightmare_record_resolved_boot(&config, config.selector);
477 nightmare_emit_verdict(NIGHTMARE_SKIP(NIGHTMARE_SKIP_NOT_COMPILED),
478 "not_compiled");
479#else
480 const struct nightmare *nm = nightmare_resolve(name: config.selector);
481 nightmare_runtime.ctx.nm = nm;
482 if (!nm) {
483 nightmare_record_resolved_boot(config: &config, requested: config.selector);
484 nightmare_emit_verdict(NIGHTMARE_SKIP(NIGHTMARE_SKIP_NO_SUCH_NIGHTMARE),
485 fallback_reason: "no_such_nightmare");
486 }
487
488 nightmare_runtime.ctx.intensity = NIGHTMARE_INTENSITY_DEFAULT;
489
490 if (config.intensity != NIGHTMARE_INTENSITY_SENTINEL) {
491 nightmare_runtime.ctx.intensity = config.intensity;
492 } else if (nm->intensity != NIGHTMARE_INTENSITY_SENTINEL) {
493 nightmare_runtime.ctx.intensity = nm->intensity;
494 }
495
496 nightmare_runtime.ctx.intensity_val =
497 scaled_param_eval(desc: &nm->intensity_desc, value: nightmare_runtime.ctx.intensity);
498
499 nightmare_runtime.ctx.worker_count =
500 nightmare_runtime.ctx.intensity_val
501 ? nightmare_runtime.ctx.intensity_val
502 : 1;
503
504 time_ms_t duration_ms = NIGHTMARE_DEFAULT_DURATION_MS;
505
506 if (config.duration_ms) {
507 duration_ms = config.duration_ms;
508 } else if (nm->default_duration_ms) {
509 duration_ms = nm->default_duration_ms;
510 }
511
512 time_ms_t drain_ms = config.drain_grace_ms ? config.drain_grace_ms
513 : NIGHTMARE_DEFAULT_DRAIN_MS;
514 nightmare_runtime.ctx.soft_deadline_ms =
515 nightmare_runtime.started_ms + duration_ms;
516
517 nightmare_runtime.ctx.hard_deadline_ms =
518 nightmare_runtime.ctx.soft_deadline_ms + drain_ms;
519
520 nightmare_record_resolved_boot(config: &config, requested: config.selector);
521
522 enum nightmare_skip_reason refusal = nightmare_preflight(nm, config: &config);
523 if (refusal != NIGHTMARE_SKIP_NONE)
524 nightmare_emit_verdict(NIGHTMARE_SKIP(refusal),
525 fallback_reason: nightmare_skip_to_str(stringify_in: refusal));
526
527 atomic_store_release(&nightmare_runtime.conc.active, true);
528 nightmare_arm_deadlines(duration_ms, drain_grace_ms: drain_ms);
529
530 struct nightmare_verdict prepared = NIGHTMARE_OK;
531 if (nm->ops && nm->ops->prepare)
532 prepared = nm->ops->prepare(&nightmare_runtime.ctx);
533 if (prepared.result != NIGHTMARE_RESULT_OK) {
534 timer_shutdown_sync(timer: &nightmare_runtime.soft_timer);
535 timer_shutdown_sync(timer: &nightmare_runtime.hard_timer);
536 if (prepared.result == NIGHTMARE_RESULT_SKIP)
537 prepared = NIGHTMARE_SKIP(NIGHTMARE_SKIP_PREPARE_REFUSED);
538 nightmare_emit_verdict(verdict: prepared, fallback_reason: "prepare");
539 }
540
541 nightmare_collect_perturbers(nm, config: &config);
542
543 size_t worker_count =
544 nightmare_runtime.ctx.worker_count + nightmare_runtime.perturber_count;
545 struct test_conc_worker *workers = vnews(workers, worker_count, ALLOC_ZERO);
546
547 nightmare_runtime.conc.workers = workers;
548 nightmare_runtime.conc.worker_count = worker_count;
549
550 if (!nightmare_runtime.conc.workers) {
551 timer_shutdown_sync(timer: &nightmare_runtime.soft_timer);
552 timer_shutdown_sync(timer: &nightmare_runtime.hard_timer);
553 nightmare_emit_verdict(
554 NIGHTMARE_FAIL("worker_alloc", "could not allocate worker state"),
555 fallback_reason: "worker_alloc");
556 }
557
558 if (!nightmare_spawn_threads()) {
559 timer_shutdown_sync(timer: &nightmare_runtime.soft_timer);
560 timer_shutdown_sync(timer: &nightmare_runtime.hard_timer);
561 nightmare_release_threads();
562 nightmare_emit_verdict(
563 NIGHTMARE_FAIL("thread_create", "could not create all threads"),
564 fallback_reason: "thread_create");
565 }
566
567 if (!nightmare_liveness_start(threshold_ms: config.stall_threshold_ms, policy: config.on_stall))
568 nightmare_panic("could not start liveness detector");
569 event_set(e: &nightmare_runtime.conc.start);
570 nightmare_join_threads();
571 nightmare_liveness_stop();
572 nightmare_publish_stop(reason: TEST_STOP_BUDGET);
573
574 timer_shutdown_sync(timer: &nightmare_runtime.soft_timer);
575 timer_shutdown_sync(timer: &nightmare_runtime.hard_timer);
576
577 struct nightmare_verdict final = nightmare_finalize_verdict(nm);
578 atomic_store_release(&nightmare_runtime.conc.active, false);
579
580 nightmare_release_threads();
581
582 vfree(nightmare_runtime.conc.workers);
583 nightmare_runtime.conc.workers = NULL;
584 if (nightmare_runtime.ctx.private) {
585 vfree(nightmare_runtime.ctx.private);
586 nightmare_runtime.ctx.private = NULL;
587 }
588
589 bool expected = false;
590 if (atomic_cas_strong(&nightmare_runtime.terminal, &expected, true,
591 mo_acq_rel, mo_acquire))
592 nightmare_emit_verdict(verdict: final, fallback_reason: "completed");
593#endif
594}
595#endif
596