1#include "internal.h"
2
3#include <nightmare/record.h>
4#include <sch/sched.h>
5#include <thread/thread.h>
6
7#ifdef TEST_NIGHTMARE_ENABLED
8void nightmare_publish_stop(enum nightmare_stop reason) {
9 enum nightmare_stop observed =
10 atomic_load_explicit(&nightmare_runtime.stop, memory_order_acquire);
11 bool advanced = false;
12 while (observed < reason) {
13 if (atomic_compare_exchange_weak_explicit(
14 &nightmare_runtime.stop, &observed, reason,
15 memory_order_release, memory_order_acquire)) {
16 advanced = true;
17 break;
18 }
19 }
20
21 /* Workers and heartbeat may be between polls in timed sleeps, so
22 * we need to wake them here so teardown doesn't depend on their timers */
23
24 /* Threads blocked on other stuff don't care because the expected
25 * wake source won't match what we give them here */
26 if (advanced && observed == NM_RUN) {
27 for (size_t i = 0; i < nightmare_runtime.total_worker_count; i++) {
28 struct thread *thread = atomic_load_explicit(
29 &nightmare_runtime.workers[i].th, memory_order_acquire);
30 if (thread)
31 scheduler_wake_manual(t: thread, wake_src: thread);
32 }
33 if (nightmare_runtime.heartbeat)
34 scheduler_wake_manual(t: nightmare_runtime.heartbeat,
35 wake_src: nightmare_runtime.heartbeat);
36 }
37}
38
39bool nightmare_must_stop_irq(void) {
40 return atomic_load_explicit(&nightmare_runtime.stop,
41 memory_order_acquire) != NM_RUN;
42}
43
44bool nightmare_must_stop(void) {
45 return nightmare_must_stop_irq();
46}
47
48void nightmare_stop_after_finding(void) {
49 nightmare_publish_stop(reason: NM_STOP_FINDING);
50}
51
52bool nightmare_must_park(void) {
53 return atomic_load_explicit(&nightmare_runtime.quiesce_requested,
54 memory_order_acquire);
55}
56
57void nightmare_park(struct nightmare_worker *worker) {
58 bool was_parked =
59 atomic_exchange_explicit(&worker->parked, true, memory_order_acq_rel);
60 if (!was_parked)
61 atomic_fetch_add_explicit(&nightmare_runtime.parked_count, 1,
62 memory_order_release);
63
64 while (nightmare_must_park() && !nightmare_must_stop())
65 scheduler_yield();
66
67 if (!was_parked) {
68 atomic_fetch_sub_explicit(&nightmare_runtime.parked_count, 1,
69 memory_order_release);
70 atomic_store_explicit(&worker->parked, false, memory_order_release);
71 }
72}
73
74void nightmare_thread_main(void *arg) {
75 struct nightmare_worker *worker = arg;
76 completion_wait(c: &nightmare_runtime.start);
77
78 if (!nightmare_must_stop()) {
79 if (worker->index < nightmare_runtime.ctx.worker_count) {
80 if (nightmare_runtime.ctx.nm && nightmare_runtime.ctx.nm->ops &&
81 nightmare_runtime.ctx.nm->ops->worker)
82 nightmare_runtime.ctx.nm->ops->worker(&nightmare_runtime.ctx,
83 worker);
84 } else {
85 size_t pidx = worker->index - nightmare_runtime.ctx.worker_count;
86 if (pidx < nightmare_runtime.perturber_count &&
87 nightmare_runtime.perturbers[pidx] &&
88 nightmare_runtime.perturbers[pidx]->thread) {
89 nightmare_runtime.perturbers[pidx]->thread(
90 &nightmare_runtime.ctx, worker);
91 }
92 }
93 }
94
95 if (atomic_load_explicit(&worker->parked, memory_order_acquire)) {
96 atomic_fetch_sub_explicit(&nightmare_runtime.parked_count, 1,
97 memory_order_release);
98 atomic_store_explicit(&worker->parked, false, memory_order_release);
99 }
100}
101
102void nightmare_heartbeat_main(void *arg) {
103 (void) arg;
104 completion_wait(c: &nightmare_runtime.start);
105
106 time_ms_t next = time_get_ms() + nightmare_runtime.stat_interval_ms;
107 do {
108 nightmare_liveness_poll();
109
110 time_ms_t now = time_get_ms();
111 if (now >= next) {
112 nightmare_record_stat(progress: nightmare_progress_sum_irq(),
113 workers: nightmare_runtime.ctx.worker_count);
114 next = now + nightmare_runtime.stat_interval_ms;
115 }
116 thread_sleep_for_ms(ms: 10);
117 } while (!nightmare_must_stop());
118
119 nightmare_liveness_poll();
120
121 nightmare_record_stat(progress: nightmare_progress_sum_irq(),
122 workers: nightmare_runtime.ctx.worker_count);
123}
124#endif
125