1#include <acpi/lapic.h>
2#include <cmdline.h>
3#include <irq/ipi.h>
4#include <irq/irq.h>
5#include <math/min_max.h>
6#include <mem/nofail.h>
7#include <pit.h>
8#include <smp/core.h>
9#include <smp/percpu.h>
10#include <string.h>
11#include <sync/seqlock.h>
12#include <thread/dpc.h>
13#include <time/time.h>
14#include <watchdog.h>
15
16#include "watchdog/internal.h"
17#include <mem/alloc.h>
18
19#define watchdog_master_log(lvl, fmt, ...) \
20 log(LOG_SITE(watchdog_master), LOG_HANDLE(watchdog_master), lvl, fmt, \
21 ##__VA_ARGS__)
22
23#define watchdog_master_err(fmt, ...) \
24 watchdog_master_log(LOG_ERROR, fmt, ##__VA_ARGS__)
25#define watchdog_master_warn(fmt, ...) \
26 watchdog_master_log(LOG_WARN, fmt, ##__VA_ARGS__)
27#define watchdog_master_info(fmt, ...) \
28 watchdog_master_log(LOG_INFO, fmt, ##__VA_ARGS__)
29#define watchdog_master_debug(fmt, ...) \
30 watchdog_master_log(LOG_DEBUG, fmt, ##__VA_ARGS__)
31#define watchdog_master_trace(fmt, ...) \
32 watchdog_master_log(LOG_TRACE, fmt, ##__VA_ARGS__)
33
34static void watchdog_worker_timer_func(struct timer *t);
35static void watchdog_buckets_init(struct watchdog_buckets *b);
36static void watchdog_percpu_ctor(struct watchdog_percpu *pcpu, cpu_id_t cpu);
37
38static struct watchdog_config config = {
39 .bucket_interval = SECONDS_TO_NS(1),
40 .master_tick_interval = MS_TO_NS(100),
41 .worker_heartbeat_interval = MS_TO_NS(50),
42 .master_print_interval = MS_TO_NS(250),
43 .master_stall_timeout = MS_TO_NS(WATCHDOG_STALL_TIMEOUT_DEFAULT_MS),
44
45 .master_panic_score = FX(0.9), /* We missed 60% of heartbeats for a while,
46 * AND when tested, we failed 75% */
47 .master_critical_score = FX(0.6), /* We're missing 60% of our heartbeats */
48 .master_suspect_score = FX(0.25), /* We're missing 25% of our heartbeats */
49
50};
51static struct watchdog_master watchdog_master = {0};
52static struct watchdog_globals watchdog_global = {0};
53
54/* TODO: a LOG_SITE + HANDLE EXTERN, with printing explicitly turned off
55 * because we cannot take the implicit printf lock(s) from the NMI */
56LOG_SITE_DEFINE(watchdog_master);
57LOG_HANDLE_DEFINE(watchdog_master);
58
59PERCPU_DEFINE(struct watchdog_percpu, watchdog_percpu, watchdog_percpu_ctor);
60CMDLINE_DEFINE_STATIC(watchdog, .flags = CMDLINE_ENTRY_SYMBOLIC,
61 .desc = "Watchdog command line namespace");
62
63CMDLINE_CHILD_DEFINE(watchdog, master, .flags = CMDLINE_ENTRY_SYMBOLIC);
64
65CMDLINE_CHILDREN_DEFINE(
66 CMDLINE_NODE(watchdog, master),
67 CMDLINE_INNER_DURATION(heartbeat_interval, config.master_tick_interval,
68 .range = RANGE(MS_TO_NS(1), SECONDS_TO_NS(60))),
69 CMDLINE_INNER_DURATION(bucket_interval, config.bucket_interval,
70 .range = RANGE(MS_TO_NS(1), SECONDS_TO_NS(60))),
71 CMDLINE_INNER_FX(panic_score, config.master_panic_score,
72 .range = RANGE(0, FX_ONE)),
73 CMDLINE_INNER_FX(warn_score, config.master_critical_score,
74 .range = RANGE(0, FX_ONE)),
75 CMDLINE_INNER_FX(suspect_threshold, config.master_suspect_score,
76 .range = RANGE(0, FX_ONE)),
77 CMDLINE_INNER_DURATION(stall_timeout, config.master_stall_timeout,
78 .range = RANGE(0, SECONDS_TO_NS(600))));
79
80/* PERCPU_DEFINE zero-initializes, but we explicitly init buckets */
81static void watchdog_percpu_ctor(struct watchdog_percpu *pcpu, cpu_id_t cpu) {
82 pcpu->id = cpu;
83 pcpu->pets_enabled = false;
84 atomic_init(&pcpu->pets, 0);
85 atomic_init(&pcpu->anti_pets, 0);
86 atomic_init(&pcpu->heartbeat_seq, 0);
87 watchdog_buckets_init(b: &pcpu->buckets);
88 pcpu->timer.func = watchdog_worker_timer_func;
89 pcpu->timer.flags =
90 TIMER_FLAG_PINNED | TIMER_FLAG_CPU(cpu) | TIMER_FLAG_IRQ;
91}
92
93static inline size_t time_to_bucket(time_ms_t time) {
94 /* TODO: Decide if this condition is not permitted? */
95 if (cc_unlikely(watchdog_global.bucket_interval_ms == 0))
96 return 0;
97
98 return time / watchdog_global.bucket_interval_ms;
99}
100
101static void watchdog_buckets_init(struct watchdog_buckets *b) {
102 seqcount_init(s: &b->seq);
103 b->idx = 0;
104 b->curr_epoch = 0;
105 b->last_heartbeat_ms = 0;
106 for (size_t i = 0; i < WATCHDOG_NUM_BUCKETS; i++) {
107 b->buckets_internal[i].epoch = 0;
108 b->buckets_internal[i].heartbeats = 0;
109 }
110}
111
112/* Must be called with write lock held */
113static void watchdog_buckets_advance_internal(struct watchdog_buckets *buckets,
114 time_ms_t new_time) {
115 if (buckets->last_heartbeat_ms == 0) {
116 buckets->last_heartbeat_ms = new_time;
117 buckets->buckets_internal[buckets->idx].epoch = buckets->curr_epoch;
118 buckets->buckets_internal[buckets->idx].heartbeats = 0;
119 return;
120 }
121
122 if (cc_unlikely(new_time < buckets->last_heartbeat_ms))
123 return;
124
125 size_t old_bucket = time_to_bucket(time: buckets->last_heartbeat_ms);
126 size_t new_bucket = time_to_bucket(time: new_time);
127 if (new_bucket <= old_bucket)
128 return;
129
130 size_t elapsed = new_bucket - old_bucket;
131 if (elapsed >= WATCHDOG_NUM_BUCKETS) {
132 size_t idx_absolute = buckets->idx + elapsed;
133 size_t wraps = idx_absolute / WATCHDOG_NUM_BUCKETS;
134 buckets->curr_epoch += wraps;
135 buckets->idx = idx_absolute % WATCHDOG_NUM_BUCKETS;
136
137 for (size_t i = 0; i < WATCHDOG_NUM_BUCKETS; i++) {
138 buckets->buckets_internal[i].heartbeats = 0;
139 buckets->buckets_internal[i].epoch = 0;
140 }
141
142 buckets->buckets_internal[buckets->idx].epoch = buckets->curr_epoch;
143 buckets->buckets_internal[buckets->idx].heartbeats = 0;
144 } else {
145 for (size_t i = 0; i < elapsed; i++) {
146 buckets->idx++;
147 if (buckets->idx == WATCHDOG_NUM_BUCKETS) {
148 buckets->idx = 0;
149 buckets->curr_epoch++;
150 }
151
152 size_t idx = buckets->idx;
153 buckets->buckets_internal[idx].heartbeats = 0;
154 buckets->buckets_internal[idx].epoch = buckets->curr_epoch;
155 }
156 }
157
158 buckets->last_heartbeat_ms = new_time;
159}
160
161/* Only called by percpu owner */
162static void watchdog_buckets_inc_heartbeat(struct watchdog_buckets *buckets,
163 time_ms_t now) {
164 seqcount_begin_write(s: &buckets->seq);
165 watchdog_buckets_advance_internal(buckets, new_time: now);
166
167 struct watchdog_bucket *bucket = &buckets->buckets_internal[buckets->idx];
168 if (bucket->epoch != buckets->curr_epoch) {
169 bucket->epoch = buckets->curr_epoch;
170 bucket->heartbeats = 0;
171 }
172
173 bucket->heartbeats++;
174 buckets->last_heartbeat_ms = now;
175 seqcount_end_write(s: &buckets->seq);
176}
177
178static void watchdog_do_percpu_heartbeat(time_ms_t now) {
179 enforce(PERCPU_READY(watchdog_percpu), UNINITIALIZED);
180
181 atomic_inc_rlx(&PERCPU_PTR(TOPC_IRQL, watchdog_percpu)->heartbeat_seq);
182
183 watchdog_buckets_inc_heartbeat(
184 buckets: &PERCPU_READ(TOPC_IRQL, watchdog_percpu).buckets, now);
185}
186
187/* Non blocking snapshot */
188static bool watchdog_buckets_snapshot(const struct watchdog_buckets *b,
189 struct watchdog_bucket_snapshot *out) {
190 uint32_t seq;
191 int retries = WATCHDOG_MASTER_SEQCOUNT_SPINS;
192
193 /* NOTE: It is highly unlikely, but not impossible,
194 * for us to end up in a scenario where a worker somehow gets stuck
195 * within the code section under the seqcount write begin/end,
196 * which could lead to the master never reading data from its buckets,
197 * and never promoting past NORMAL in its criticality
198 *
199 * TODO: I can only imagine such a scenario happening with a
200 * kernel live patch gone wrong, but needless to say, this is
201 * a highly unlikely, but theoretically possible state that
202 * we may want to track and handle via bail and panic() */
203 do {
204 seq = seqcount_begin_read_raw(s: &b->seq);
205 if (cc_unlikely((seq & 1) != 0))
206 return false;
207
208 out->idx = b->idx;
209 out->curr_epoch = b->curr_epoch;
210 out->last_heartbeat_ms = b->last_heartbeat_ms;
211 for (size_t i = 0; i < WATCHDOG_NUM_BUCKETS; i++)
212 out->buckets[i] = b->buckets_internal[i];
213
214 } while (seqcount_read_retry(s: &b->seq, start: seq) && --retries > 0);
215
216 return retries > 0;
217}
218
219bool watchdog_count_heartbeats_at(const struct watchdog_buckets *buckets,
220 size_t window_buckets, time_ms_t now,
221 time_ms_t bucket_interval_ms,
222 size_t expected_per_bucket,
223 size_t *out_heartbeats, size_t *out_expected,
224 fx32_32_t *out_score) {
225 struct watchdog_bucket_snapshot snap;
226 if (!watchdog_buckets_snapshot(b: buckets, out: &snap))
227 return false;
228
229 enforce_precond(window_buckets && window_buckets <= WATCHDOG_NUM_BUCKETS,
230 CAPACITY_EXCEEDED);
231 enforce_precond(bucket_interval_ms);
232 enforce_precond(expected_per_bucket);
233
234 if (snap.last_heartbeat_ms == 0)
235 goto not_ready;
236
237 now = MAX(now, snap.last_heartbeat_ms);
238
239 size_t cursor = snap.curr_epoch * WATCHDOG_NUM_BUCKETS + snap.idx;
240 size_t last_bucket = snap.last_heartbeat_ms / bucket_interval_ms;
241 size_t now_bucket = now / bucket_interval_ms;
242 enforce(last_bucket >= cursor);
243
244 size_t first_bucket = last_bucket - cursor;
245 size_t completed = MIN(now_bucket - first_bucket, window_buckets);
246 if (!completed)
247 goto not_ready;
248
249 size_t virtual_cursor = cursor + now_bucket - last_bucket;
250 size_t total = 0, total_expect = 0;
251 for (size_t i = 1; i <= completed; i++) {
252 size_t absolute = virtual_cursor - i;
253 size_t idx = absolute % WATCHDOG_NUM_BUCKETS;
254 size_t expected_epoch = absolute / WATCHDOG_NUM_BUCKETS;
255
256 if (snap.buckets[idx].epoch == expected_epoch)
257 total += snap.buckets[idx].heartbeats;
258 total_expect += expected_per_bucket;
259 }
260
261 if (!total_expect)
262 goto not_ready;
263
264 /* If we're here, out_expect should never be zero, since one tick
265 * must fire for something to enter SUSPECT */
266 enforce(total_expect);
267
268 if (out_score)
269 *out_score = fx_div(a: fx_from_int(x: total_expect - total),
270 b: fx_from_int(x: total_expect));
271
272 if (out_expected)
273 *out_expected = total_expect;
274
275 if (out_heartbeats)
276 *out_heartbeats = total;
277
278 return true;
279
280not_ready:
281 if (out_heartbeats)
282 *out_heartbeats = 0;
283
284 if (out_expected)
285 *out_expected = 0;
286
287 if (out_score)
288 *out_score = 0;
289
290 return false;
291}
292
293static void watchdog_worker_timer_func(struct timer *t) {
294 enforce(irq_in_interrupt(), INVALID_CONTEXT);
295
296 struct watchdog_percpu *pcpu = PERCPU_PTR(TOPC_IRQ, watchdog_percpu);
297 enum irql irql = spin_lock_high(&pcpu->callback_list.lock);
298
299 struct watchdog_callback *cb;
300 list_for_each_entry(cb, &pcpu->callback_list.list, list) {
301 cb->fn(cb);
302 }
303
304 spin_unlock(&pcpu->callback_list.lock, irql);
305
306 time_ms_t now = time_get_ms();
307 watchdog_do_percpu_heartbeat(now);
308
309 timer_modify(timer: t, new: timer_delta_us(NS_TO_US(config.worker_heartbeat_interval)));
310}
311
312static void watchdog_start_petting(cpu_id_t cpu) {
313 struct watchdog_percpu *pcpu = PERCPU_PTR_FOR(watchdog_percpu, cpu);
314 if (cpu == 0) {
315 pcpu->pets_enabled = true;
316 atomic_store_rlx(&pcpu->anti_pets, 0);
317 atomic_store_rlx(&pcpu->pets, 0);
318 return; /* We don't touch the seqcount for BSP */
319 }
320
321 seqcount_begin_write(s: &pcpu->pets_seq);
322 nmi_send(cpu);
323}
324
325static void watchdog_read_pets_for(cpu_id_t cpu, size_t *out_pets,
326 size_t *out_anti_pets) {
327 struct watchdog_percpu *pcpu = PERCPU_PTR_FOR(watchdog_percpu, cpu);
328 *out_pets = atomic_load_rlx(&pcpu->pets);
329 *out_anti_pets = atomic_load_rlx(&pcpu->anti_pets);
330}
331
332/* This is the real meat and potatoes of this whole subsystem, and it
333 * needs a bit of explanation that is specifically tied to code
334 * so the implementation is clear
335 *
336 * At the high level, the steps are
337 *
338 * (1) Check on any CRITICAL CPUs -> (2) Check on any SUSPECT CPUs ->
339 * (3) Check on NORMAL domains and CPUs -> (4) Panic last, so that
340 * we can capture all possible CPUs to panic on in (1) and do
341 * any necessary promotions for this tick
342 *
343 * NOTE: Anything that bothers/inspects another CPU must account for
344 * the case where cpu == 0
345 */
346static void watchdog_cpu_promote(cpu_id_t cpu,
347 enum watchdog_master_state new_state) {
348 enforce(cpu_mask_test(&watchdog_master.cpu_masks[new_state - 1], cpu));
349 cpu_mask_clear(m: &watchdog_master.cpu_masks[new_state - 1], cpu);
350 cpu_mask_set(m: &watchdog_master.cpu_masks[new_state], cpu);
351 watchdog_master.cpus[cpu].state = new_state;
352}
353
354static void watchdog_cpu_demote(cpu_id_t cpu,
355 enum watchdog_master_state new_state) {
356 enforce(cpu_mask_test(&watchdog_master.cpu_masks[new_state + 1], cpu));
357 cpu_mask_clear(m: &watchdog_master.cpu_masks[new_state + 1], cpu);
358 cpu_mask_set(m: &watchdog_master.cpu_masks[new_state], cpu);
359 watchdog_master.cpus[cpu].state = new_state;
360}
361
362static bool watchdog_test_outstanding(struct watchdog_master_cpu *cpu) {
363 return seqcount_read_raw(s: &cpu->pcpu->response.seqcount) & 1;
364}
365
366static time_ms_t watchdog_finished_time(struct watchdog_master_cpu *cpu) {
367 return cpu->pcpu->response.finished_ms;
368}
369
370static void watchdog_start_test_on(struct watchdog_master_cpu *cpu,
371 time_ms_t now) {
372 /* Cannot be called with seqcount write active */
373 struct watchdog_percpu *pcpu = cpu->pcpu;
374 struct watchdog_percpu_response *resp = &pcpu->response;
375
376 /* If odd, it's mid-test, cannot start another */
377 enforce(!(seqcount_read_raw(&resp->seqcount) & 1));
378 seqcount_begin_write(s: &resp->seqcount);
379 cpu->critical_test_start = now;
380
381 /* Fine to not guard against cpu == 0, since the IRQ
382 * here is safe as it's not an NMI */
383 ipi_send(cpu: cpu->id, vector: watchdog_global.critical_test_irq);
384}
385
386static time_ms_t watchdog_spin_for_response(struct watchdog_master_cpu *cpu) {
387 uint32_t seq;
388 int retries = WATCHDOG_MASTER_SEQCOUNT_SPINS;
389 struct watchdog_percpu_response *resp = &cpu->pcpu->response;
390
391 do {
392 seq = seqcount_begin_read_raw(s: &resp->seqcount);
393 if (seq & 1) {
394 cpu_pause();
395 continue;
396 }
397
398 enforce(resp->finished_ms >= cpu->critical_test_start);
399 return resp->finished_ms - cpu->critical_test_start;
400
401 } while ((seqcount_read_retry(s: &resp->seqcount, start: seq) || (seq & 1)) &&
402 --retries > 0);
403
404 return TIME_MS_MAX;
405}
406
407static void watchdog_enter_suspect(cpu_id_t cpu) {
408 struct watchdog_master_cpu *mcpu = &watchdog_master.cpus[cpu];
409
410 enforce(mcpu->state == WATCHDOG_STATE_CRITICAL ||
411 mcpu->state == WATCHDOG_STATE_NORMAL);
412
413 if (mcpu->state == WATCHDOG_STATE_NORMAL) {
414 watchdog_cpu_promote(cpu, new_state: WATCHDOG_STATE_SUSPECT);
415 } else {
416 watchdog_cpu_demote(cpu, new_state: WATCHDOG_STATE_SUSPECT);
417 }
418
419 /* Demote and reset score, tests have been fine */
420 mcpu->lockup_score = config.master_suspect_score;
421 mcpu->lockup_ewma.ewma = mcpu->lockup_score;
422 mcpu->suspect_start_tick = watchdog_master.tick;
423}
424
425static inline void watchdog_record_delta(struct watchdog_master_cpu *mcpu,
426 time_ms_t delta) {
427 mcpu->critical_tests[mcpu->critical_tests_done++] = delta;
428}
429
430/* Score pet count against the expected amount over
431 * the window */
432static inline fx32_32_t watchdog_pets_score(size_t pets) {
433 size_t expected = watchdog_global.expected_heartbeats_per_bucket;
434 if (!expected)
435 return pets > 0 ? FX(0.0) : FX_ONE;
436
437 if (pets >= expected)
438 return FX(0.0);
439
440 return fx_div(a: fx_from_int(x: expected - pets), b: fx_from_int(x: expected));
441}
442
443static void watchdog_master_process_critical(time_ms_t now) {
444 /* All CPUs in here have been CRITICAL for at *least* one tick,
445 * so we never start the petting here, only end + demote
446 *
447 * Here, we look at each CPU, and for each one, we investigate
448 * (1) the pet state, (2) pet status, (3) IPI status
449 */
450
451 cpu_id_t i;
452 watchdog_cpu_for_each(i, WATCHDOG_STATE_CRITICAL) {
453 size_t failures = 0;
454 size_t target_tests, pets, anti_pets;
455 watchdog_read_pets_for(cpu: i, out_pets: &pets, out_anti_pets: &anti_pets);
456
457 if (pets > 0) {
458 target_tests = WATCHDOG_CRITICAL_IPI_TESTS;
459 } else if (!anti_pets) {
460 target_tests = (WATCHDOG_CRITICAL_IPI_TESTS * 3) / 4;
461 } else {
462 target_tests = WATCHDOG_CRITICAL_IPI_TESTS / 2;
463 }
464
465 enforce(target_tests);
466
467 struct watchdog_master_cpu *mcpu = &watchdog_master.cpus[i];
468
469 /* TODO: Logging should be safe in NMI context as
470 * the log site does not acquire locks when passthrough
471 * printing is off */
472
473 /* We have to first check this condition before anything else */
474 bool outstanding = false;
475 if (watchdog_test_outstanding(cpu: mcpu)) {
476 outstanding = true;
477 if (now - mcpu->critical_test_start >=
478 WATCHDOG_CRITICAL_PANIC_THRESHOLD_MS)
479 goto promote_to_panic;
480 }
481
482 if (mcpu->critical_tests_done >= target_tests)
483 goto score;
484
485 if (outstanding)
486 continue;
487
488 /* No outstanding tests, record the last one, start another
489 *
490 * NOTE: the promotion process to CRITICAL will have started the first
491 * test in the sequence, we don't handle that here */
492 time_ms_t finished = watchdog_finished_time(cpu: mcpu);
493 enforce(finished >= mcpu->critical_test_start);
494
495 time_ms_t delta = finished - mcpu->critical_test_start;
496
497 /* NOTE: If a test finishes at all, don't panic */
498 watchdog_record_delta(mcpu, delta);
499
500 /* Start a new one */
501 watchdog_start_test_on(cpu: mcpu, now);
502
503 /* Spin a few times to see if we get a response,
504 * and record it if we do */
505 delta = watchdog_spin_for_response(cpu: mcpu);
506 if (delta != TIME_MS_MAX)
507 watchdog_record_delta(mcpu, delta);
508
509 continue;
510
511 score:
512 /* Use the scoring formula:
513 *
514 * failed/total * TEST_WEIGHT + pet_factor * PET_WEIGHT
515 *
516 * mapped to [warn_score, 1]
517 *
518 */
519
520 for (int j = 0; j < WATCHDOG_CRITICAL_IPI_TESTS; j++) {
521 if (mcpu->critical_tests[j] > WATCHDOG_CRITICAL_PASS_THRESHOLD_MS)
522 failures++;
523 }
524
525 fx32_32_t test_part =
526 fx_mul(a: fx_div(a: fx_from_int(x: failures),
527 b: fx_from_int(WATCHDOG_CRITICAL_IPI_TESTS)),
528 WATCHDOG_CRITICAL_TESTS_FACTOR);
529 fx32_32_t pet_part =
530 fx_mul(a: watchdog_pets_score(pets), WATCHDOG_CRITICAL_PETS_FACTOR);
531 fx32_32_t total = test_part + pet_part;
532 enforce(total < FX_ONE);
533
534 if (total == 0) {
535 watchdog_enter_suspect(cpu: i);
536 } else {
537 /* map it from [0, 1] -> [warn_score, 1] */
538 fx32_32_t mapped =
539 fx_map(value: total, from_low: 0, FX_ONE, to_low: config.master_critical_score, FX_ONE);
540
541 enforce(IN_RANGE(mapped, config.master_critical_score, FX_ONE));
542 mcpu->lockup_score = mapped;
543
544 /* Panic promotion only bothers with a bitmap,
545 * everything else stays, since it never demotes from there */
546 if (mcpu->lockup_score >= config.master_panic_score) {
547 promote_to_panic:
548 watchdog_cpu_promote(cpu: i, new_state: WATCHDOG_STATE_PANIC);
549 }
550 }
551 }
552}
553
554static void watchdog_enter_critical(struct watchdog_master_cpu *mcpu,
555 time_ms_t now) {
556 /* The idea here: we'll need to start the tests so
557 * the critical processing's invariants hold */
558 watchdog_cpu_promote(cpu: mcpu->id, new_state: WATCHDOG_STATE_CRITICAL);
559 memset(&mcpu->critical_tests, 0, sizeof(mcpu->critical_tests));
560 mcpu->critical_tests_done = 0;
561 mcpu->critical_start_tick = watchdog_master.tick;
562
563 watchdog_start_petting(cpu: mcpu->id);
564
565 mcpu->critical_test_start = now;
566 watchdog_start_test_on(cpu: mcpu, now);
567}
568
569static void watchdog_enter_normal(struct watchdog_master_cpu *mcpu) {
570 ewma_init(e: &mcpu->lockup_ewma, WATCHDOG_EWMA_ALPHA);
571 watchdog_cpu_demote(cpu: mcpu->id, new_state: WATCHDOG_STATE_NORMAL);
572 mcpu->lockup_score = 0;
573}
574
575static void watchdog_master_process_suspect(time_ms_t now) {
576 cpu_id_t i;
577 watchdog_cpu_for_each(i, WATCHDOG_STATE_SUSPECT) {
578 /*
579 * SUSPECT CPUs are subject to EWMA monitoring,
580 * which can either result in it being good enough,
581 * or above warn_score, and if it lingers for
582 * long enough in SUSPECT limbo, we emit warnings
583 */
584 struct watchdog_percpu *pcpu = PERCPU_PTR_FOR(watchdog_percpu, i);
585 struct watchdog_master_cpu *mcpu = &watchdog_master.cpus[i];
586
587 fx32_32_t new;
588 if (!watchdog_count_heartbeats_at(
589 buckets: &pcpu->buckets, WATCHDOG_WINDOW_BUCKETS, now,
590 bucket_interval_ms: watchdog_global.bucket_interval_ms,
591 expected_per_bucket: watchdog_global.expected_heartbeats_per_bucket, NULL, NULL,
592 out_score: &new))
593 continue;
594
595 ewma_update(e: &mcpu->lockup_ewma, new);
596
597 mcpu->lockup_score = mcpu->lockup_ewma.ewma;
598 if (mcpu->lockup_score >= config.master_critical_score) {
599 watchdog_enter_critical(mcpu, now);
600 } else if (mcpu->lockup_score < config.master_suspect_score) {
601 watchdog_enter_normal(mcpu);
602 }
603 }
604}
605
606static void watchdog_master_process_normal(void) {
607 cpu_id_t i;
608 watchdog_cpu_for_each(i, WATCHDOG_STATE_NORMAL) {
609 struct watchdog_percpu *pcpu = PERCPU_PTR_FOR(watchdog_percpu, i);
610 fx32_32_t score;
611 if (!watchdog_count_heartbeats_at(
612 buckets: &pcpu->buckets, WATCHDOG_WINDOW_BUCKETS, now: time_get_ms(),
613 bucket_interval_ms: watchdog_global.bucket_interval_ms,
614 expected_per_bucket: watchdog_global.expected_heartbeats_per_bucket, NULL, NULL,
615 out_score: &score))
616 continue;
617
618 if (score >= config.master_suspect_score)
619 watchdog_enter_suspect(cpu: i);
620 }
621}
622
623static void watchdog_master_process_panic(void) {
624 cpu_id_t i;
625
626 /* TODO: Aggregate, print other data, for this first version we
627 * just print the first CPU that appears hung */
628 cpu_mask_for_each(i, watchdog_master.cpu_masks[WATCHDOG_STATE_PANIC]) {
629 watchdog_panic("CPU %zu hard lockup (watchdog missed heartbeats)", i);
630 }
631}
632
633/* We check here if the heartbeat counter changed, measuring elapsed
634 * time in only master ticks. This is because places like time_get_ms()
635 * have a seqcount that can get stuck, and this is effectively
636 * the "quick check" for the edge case stall */
637static void watchdog_master_process_stall(void) {
638 if (!watchdog_global.stall_timeout_ticks)
639 return;
640
641 /* Give the workers time to arm their timers before judging them. */
642 if (watchdog_master.tick < WATCHDOG_STALL_GRACE_TICKS)
643 return;
644
645 for (cpu_id_t i = 0; i < global.core_count; i++) {
646 struct watchdog_master_cpu *mcpu = &watchdog_master.cpus[i];
647 struct watchdog_percpu *pcpu = PERCPU_PTR_FOR(watchdog_percpu, i);
648
649 uint64_t seen = atomic_load_rlx(&pcpu->heartbeat_seq);
650
651 if (seen != mcpu->last_seen_heartbeat) {
652 mcpu->last_seen_heartbeat = seen;
653 mcpu->last_progress_tick = watchdog_master.tick;
654 continue;
655 }
656
657 if (watchdog_master.tick - mcpu->last_progress_tick <
658 watchdog_global.stall_timeout_ticks)
659 continue;
660
661 watchdog_panic("CPU %zu stalled: no heartbeat for %zu master ticks "
662 "(seq stuck at %llu)",
663 (size_t) i,
664 watchdog_master.tick - mcpu->last_progress_tick,
665 (unsigned long long) seen);
666 }
667}
668
669static enum irq_result watchdog_master_nmi_handler(struct irq_context *regs,
670 void *ctx) {
671 cc_ignore(ctx, regs);
672
673 /* Since the other NMI ISRs run well before this one does, we can
674 * guarantee that this IS us, since we register last, but we do
675 * a CPU ID check to really make sure
676 *
677 * HACK: We simply use the call site ordering to guarantee list_add_tail
678 * happens in the right order, but we will want to probably need to
679 * make this more robust, reusable, and less reliant on call ordering */
680 if (smp_id(cond: TOPC_IRQ) != 0)
681 return IRQ_NONE; /* Not us */
682
683 watchdog_master.tick++;
684
685 /* Process what could've stalled before anything else */
686 watchdog_master_process_stall();
687
688 /* If anything is wedged on the timekeeper seqlock, this
689 * prevents the NMI from stalling too, so we only try_ here */
690 time_ms_t now;
691 if (!time_try_get_ms(out: &now))
692 return IRQ_HANDLED;
693
694 watchdog_master_process_critical(now);
695 watchdog_master_process_suspect(now);
696 watchdog_master_process_normal();
697 watchdog_master_process_panic();
698
699 return IRQ_HANDLED;
700}
701
702static enum irq_result watchdog_pet_nmi_handler(struct irq_context *regs,
703 void *ctx) {
704 cc_ignore(ctx, regs);
705 struct watchdog_percpu *pcpu = PERCPU_PTR(TOPC_IRQ, watchdog_percpu);
706
707 /* Must be set, if it doesn't, something happened */
708 if (seqcount_read_raw(s: &pcpu->pets_seq) & 1) {
709 pcpu->pets_enabled = true;
710 seqcount_end_write(s: &pcpu->pets_seq);
711 return IRQ_HANDLED;
712 }
713
714 return IRQ_NONE;
715}
716
717static enum irq_result watchdog_test_handler(struct irq_context *regs,
718 void *ctx) {
719 cc_ignore(ctx, regs);
720 struct watchdog_percpu *pcpu = PERCPU_PTR(TOPC_IRQ, watchdog_percpu);
721
722 if (seqcount_read_raw(s: &pcpu->response.seqcount) & 1) {
723 pcpu->response.finished_ms = time_get_ms();
724 seqcount_end_write(s: &pcpu->response.seqcount);
725 return IRQ_HANDLED;
726 }
727
728 return IRQ_NONE;
729}
730
731/* We'll want to set up the master and all the workers */
732void watchdog_init(void) {
733 enforce(PERCPU_READY(watchdog_percpu), UNINITIALIZED);
734
735 for (int i = 0; i < WATCHDOG_STATE_MAX; i++) {
736 cpu_mask_init(m: &watchdog_master.cpu_masks[i], nbits: global.core_count);
737 }
738
739 cpu_mask_init(m: &watchdog_master.scratch_mask, nbits: global.core_count);
740
741 cpu_mask_set_all(&watchdog_master.cpu_masks[WATCHDOG_STATE_NORMAL]);
742 watchdog_master.cpus =
743 nofail_vnews(watchdog_master.cpus, global.core_count, ALLOC_ZERO);
744
745 watchdog_global.critical_test_irq = irq_alloc_entry();
746 irq_register(name: "watchdog_test", vector: watchdog_global.critical_test_irq,
747 handler: watchdog_test_handler, NULL, flags: IRQ_FLAG_NONE);
748 irq_set_chip(vector: watchdog_global.critical_test_irq, chip: lapic_get_chip(), NULL);
749 irq_register(name: "watchdog_master", IRQ_NMI, handler: watchdog_master_nmi_handler, NULL,
750 flags: IRQ_FLAG_SHARED);
751
752 irq_register(name: "watchdog_pet", IRQ_NMI, handler: watchdog_pet_nmi_handler, NULL,
753 flags: IRQ_FLAG_SHARED);
754
755 for (cpu_id_t i = 0; i < global.core_count; i++) {
756 watchdog_master.cpus[i].id = i;
757 watchdog_master.cpus[i].state = WATCHDOG_STATE_NORMAL;
758 ewma_init(e: &watchdog_master.cpus[i].lockup_ewma, WATCHDOG_EWMA_ALPHA);
759 watchdog_master.cpus[i].lockup_score = FX(0.0);
760 watchdog_master.cpus[i].pcpu = PERCPU_PTR_FOR(watchdog_percpu, i);
761 locked_list_init(ll: &PERCPU_PTR_FOR(watchdog_percpu, i)->callback_list,
762 LOCKED_LIST_INIT_IRQ_DISABLE);
763 }
764}
765
766/* TODO: give PERCPU contiguous functionality */
767static struct dpc watchdog_arm_dpcs[CPU_MASK_BITS];
768
769static void watchdog_arm_local(void) {
770 struct watchdog_percpu *pcpu = PERCPU_PTR(TOPC_NONE, watchdog_percpu);
771 timer_modify(timer: &pcpu->timer,
772 new: timer_delta_us(NS_TO_US(config.master_tick_interval)));
773}
774
775void watchdog_start(void) {
776 watchdog_global.bucket_interval_ms = NS_TO_MS(config.bucket_interval);
777 watchdog_global.expected_heartbeats_per_bucket =
778 watchdog_global.bucket_interval_ms /
779 NS_TO_MS(config.worker_heartbeat_interval);
780
781 time_ms_t tick_ms = NS_TO_MS(config.master_tick_interval);
782 watchdog_global.stall_timeout_ticks =
783 (config.master_stall_timeout && tick_ms)
784 ? NS_TO_MS(config.master_stall_timeout) / tick_ms
785 : 0;
786
787 for (cpu_id_t i = 0; i < global.core_count; i++) {
788 watchdog_master.cpus[i].last_seen_heartbeat = 0;
789 watchdog_master.cpus[i].last_progress_tick = 0;
790 }
791
792 struct cpu_mask all = CPU_MASK_INIT;
793 cpu_mask_set_all(&all);
794 dpc_fanout(storage: watchdog_arm_dpcs, cpus: all, fn: watchdog_arm_local);
795
796 pit_init();
797 pit_wire_periodic_nmi(interval_ns: config.master_tick_interval);
798 ioapic_route_isa_nmi(isa_irq: 0, dest_apic_id: smp_apic_id(cpu: 0));
799}
800
801void watchdog_pet(void) {
802 if (PERCPU_READY(watchdog_percpu)) {
803 struct watchdog_percpu *this = PERCPU_PTR(TOPC_NONE, watchdog_percpu);
804 /* The caller must make sure we cannot be preempted */
805 if (this->pets_enabled) {
806 enforce(irql_get() >= IRQL_DISPATCH_LEVEL, INVALID_CONTEXT);
807 atomic_inc_rlx(&this->pets);
808 }
809 }
810}
811
812void watchdog_anti_pet(void) {
813 if (PERCPU_READY(watchdog_percpu)) {
814 struct watchdog_percpu *this = PERCPU_PTR(TOPC_NONE, watchdog_percpu);
815 /* The caller must make sure we cannot be preempted */
816 if (this->pets_enabled) {
817 enforce(irql_get() >= IRQL_DISPATCH_LEVEL, INVALID_CONTEXT);
818 atomic_inc_rlx(&this->anti_pets);
819 }
820 }
821}
822
823void watchdog_callback_add(cpu_id_t id, struct watchdog_callback *cb) {
824 struct watchdog_percpu *percpu = PERCPU_PTR_FOR(watchdog_percpu, id);
825 locked_list_add(ll: &percpu->callback_list, lh: &cb->list);
826}
827
828void watchdog_callback_remove(cpu_id_t id, struct watchdog_callback *cb) {
829 struct watchdog_percpu *percpu = PERCPU_PTR_FOR(watchdog_percpu, id);
830 locked_list_del(ll: &percpu->callback_list, lh: &cb->list);
831}
832