| 1 | #include "internal.h" |
| 2 | |
| 3 | #include <nightmare/record.h> |
| 4 | #include <sch/sched.h> |
| 5 | |
| 6 | #ifdef TEST_NIGHTMARE_ENABLED |
| 7 | bool nightmare_liveness_eval(struct nightmare_liveness_state *state, |
| 8 | uint64_t current_progress, time_ms_t now_ms, |
| 9 | bool quiesce_requested, bool run_active, |
| 10 | enum nightmare_stop stop, cpu_id_t observer_cpu) { |
| 11 | if (!state || !run_active || stop != NM_RUN) |
| 12 | return false; |
| 13 | |
| 14 | if (atomic_load_explicit(&state->phase, memory_order_acquire) != |
| 15 | NM_LIVE_ARMED) |
| 16 | return false; |
| 17 | |
| 18 | if (quiesce_requested) { |
| 19 | state->last_progress = current_progress; |
| 20 | state->last_change_ms = now_ms; |
| 21 | state->was_quiesced = true; |
| 22 | return false; |
| 23 | } |
| 24 | |
| 25 | if (state->was_quiesced) { |
| 26 | state->last_progress = current_progress; |
| 27 | state->last_change_ms = now_ms; |
| 28 | state->was_quiesced = false; |
| 29 | return false; |
| 30 | } |
| 31 | |
| 32 | if (current_progress != state->last_progress) { |
| 33 | state->last_progress = current_progress; |
| 34 | state->last_change_ms = now_ms; |
| 35 | return false; |
| 36 | } |
| 37 | |
| 38 | if (now_ms < state->last_change_ms || |
| 39 | now_ms - state->last_change_ms < state->threshold_ms) |
| 40 | return false; |
| 41 | |
| 42 | state->pending = (struct nightmare_stall_evidence){ |
| 43 | .silent_ms = now_ms - state->last_change_ms, |
| 44 | .progress = current_progress, |
| 45 | .observer_cpu = observer_cpu, |
| 46 | }; |
| 47 | |
| 48 | enum nightmare_liveness_phase expected = NM_LIVE_ARMED; |
| 49 | if (!atomic_compare_exchange_strong_explicit( |
| 50 | &state->phase, &expected, NM_LIVE_PENDING, memory_order_release, |
| 51 | memory_order_acquire)) |
| 52 | return false; |
| 53 | |
| 54 | return true; |
| 55 | } |
| 56 | |
| 57 | static void nightmare_liveness_watchdog_cb(struct watchdog_callback *cb) { |
| 58 | struct nightmare_liveness_state *state = |
| 59 | container_of(cb, struct nightmare_liveness_state, callback); |
| 60 | |
| 61 | if (atomic_load_explicit(&state->phase, memory_order_acquire) != |
| 62 | NM_LIVE_ARMED) |
| 63 | return; |
| 64 | |
| 65 | bool active = |
| 66 | atomic_load_explicit(&nightmare_runtime.active, memory_order_acquire); |
| 67 | if (!active) |
| 68 | return; |
| 69 | |
| 70 | enum nightmare_stop stop = |
| 71 | atomic_load_explicit(&nightmare_runtime.stop, memory_order_acquire); |
| 72 | if (stop != NM_RUN) |
| 73 | return; |
| 74 | |
| 75 | bool quiesce = atomic_load_explicit(&nightmare_runtime.quiesce_requested, |
| 76 | memory_order_acquire); |
| 77 | uint64_t progress = nightmare_progress_sum_irq(); |
| 78 | time_ms_t now = time_get_ms(); |
| 79 | cpu_id_t cpu = smp_id(cond: TOPC_NONE); /* Heuristic */ |
| 80 | |
| 81 | if (nightmare_liveness_eval(state, current_progress: progress, now_ms: now, quiesce_requested: quiesce, run_active: active, stop, |
| 82 | observer_cpu: cpu)) { |
| 83 | nightmare_publish_stop(reason: NM_STOP_STALL); |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | struct nightmare_liveness_state nightmare_liveness; |
| 88 | |
| 89 | bool nightmare_liveness_start(time_ms_t threshold_ms, |
| 90 | enum nightmare_on_stall policy) { |
| 91 | if (threshold_ms == 0 || nightmare_liveness.registered) |
| 92 | return false; |
| 93 | |
| 94 | nightmare_liveness = (struct nightmare_liveness_state){ |
| 95 | .callback = {.fn = nightmare_liveness_watchdog_cb}, |
| 96 | .phase = ATOMIC_VAR_INIT(NM_LIVE_ARMED), |
| 97 | .last_progress = nightmare_progress_sum_irq(), |
| 98 | .last_change_ms = time_get_ms(), |
| 99 | .threshold_ms = threshold_ms, |
| 100 | .was_quiesced = false, |
| 101 | .policy = policy, |
| 102 | .coordinator_cpu = 0, |
| 103 | .registered = false, |
| 104 | }; |
| 105 | |
| 106 | watchdog_callback_add(cpu: nightmare_liveness.coordinator_cpu, |
| 107 | cb: &nightmare_liveness.callback); |
| 108 | nightmare_liveness.registered = true; |
| 109 | return true; |
| 110 | } |
| 111 | |
| 112 | void nightmare_liveness_stop(void) { |
| 113 | if (nightmare_liveness.registered) { |
| 114 | watchdog_callback_remove(cpu: nightmare_liveness.coordinator_cpu, |
| 115 | cb: &nightmare_liveness.callback); |
| 116 | nightmare_liveness.registered = false; |
| 117 | } |
| 118 | atomic_store_explicit(&nightmare_liveness.phase, NM_LIVE_OFF, |
| 119 | memory_order_release); |
| 120 | } |
| 121 | |
| 122 | void nightmare_report_stall(const struct nightmare_stall_evidence *evidence) { |
| 123 | if (!evidence) |
| 124 | return; |
| 125 | NIGHTMARE_FINDING_TIER("stall" , NIGHTMARE_TIER_AMBIGUOUS, 0, |
| 126 | "silent_ms=%lu progress=%lu observer_cpu=%u" , |
| 127 | (unsigned long) evidence->silent_ms, |
| 128 | (unsigned long) evidence->progress, |
| 129 | (unsigned int) evidence->observer_cpu); |
| 130 | nightmare_publish_stop(reason: NM_STOP_STALL); |
| 131 | } |
| 132 | |
| 133 | void nightmare_liveness_poll(void) { |
| 134 | enum nightmare_liveness_phase phase = |
| 135 | atomic_load_explicit(&nightmare_liveness.phase, memory_order_acquire); |
| 136 | if (phase == NM_LIVE_PENDING) { |
| 137 | enum nightmare_liveness_phase expected = NM_LIVE_PENDING; |
| 138 | if (atomic_compare_exchange_strong_explicit( |
| 139 | &nightmare_liveness.phase, &expected, NM_LIVE_REPORTED, |
| 140 | memory_order_acq_rel, memory_order_acquire)) { |
| 141 | nightmare_report_stall(evidence: &nightmare_liveness.pending); |
| 142 | if (nightmare_liveness.policy != NIGHTMARE_ON_STALL_REPORT) |
| 143 | nightmare_panic("aggregate nightmare progress stalled" ); |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | if (nightmare_runtime.ctx.nm && nightmare_runtime.ctx.nm->ops && |
| 148 | nightmare_runtime.ctx.nm->ops->probe) { |
| 149 | nightmare_runtime.ctx.nm->ops->probe(&nightmare_runtime.ctx); |
| 150 | } |
| 151 | } |
| 152 | #endif |
| 153 | |