1#include <acpi/lapic.h>
2#include <cmdline.h>
3#include <irq/irq.h>
4#include <mem/alloc_or_die.h>
5#include <pit.h>
6#include <smp/percpu.h>
7#include <string.h>
8#include <sync/seqlock.h>
9#include <time/time.h>
10#include <watchdog.h>
11
12#include "watchdog/internal.h"
13#include <mem/alloc.h>
14
15#define watchdog_master_log(lvl, fmt, ...) \
16 log(LOG_SITE(watchdog_master), LOG_HANDLE(watchdog_master), lvl, fmt, \
17 ##__VA_ARGS__)
18
19#define watchdog_master_err(fmt, ...) \
20 watchdog_master_log(LOG_ERROR, fmt, ##__VA_ARGS__)
21#define watchdog_master_warn(fmt, ...) \
22 watchdog_master_log(LOG_WARN, fmt, ##__VA_ARGS__)
23#define watchdog_master_info(fmt, ...) \
24 watchdog_master_log(LOG_INFO, fmt, ##__VA_ARGS__)
25#define watchdog_master_debug(fmt, ...) \
26 watchdog_master_log(LOG_DEBUG, fmt, ##__VA_ARGS__)
27#define watchdog_master_trace(fmt, ...) \
28 watchdog_master_log(LOG_TRACE, fmt, ##__VA_ARGS__)
29
30static void watchdog_worker_timer_func(struct timer *t);
31static void watchdog_buckets_init(struct watchdog_buckets *b);
32static void watchdog_percpu_ctor(struct watchdog_percpu *pcpu, cpu_id_t cpu);
33
34static struct watchdog_config config = {
35 .bucket_interval = SECONDS_TO_NS(1),
36 .master_tick_interval = MS_TO_NS(100),
37 .worker_heartbeat_interval = MS_TO_NS(50),
38 .master_print_interval = MS_TO_NS(250),
39 .master_stall_timeout = MS_TO_NS(WATCHDOG_STALL_TIMEOUT_DEFAULT_MS),
40
41 .master_panic_score = FX(0.9), /* We missed 60% of heartbeats for a while,
42 * AND when tested, we failed 75% */
43 .master_critical_score = FX(0.6), /* We're missing 60% of our heartbeats */
44 .master_suspect_score = FX(0.25), /* We're missing 25% of our heartbeats */
45
46};
47static struct watchdog_master watchdog_master = {0};
48static struct watchdog_globals watchdog_global = {0};
49
50/* TODO: a LOG_SITE + HANDLE DECLARE, with printing explicitly turned off
51 * because we cannot take the implicit printf lock(s) from the NMI */
52LOG_SITE_DECLARE(watchdog_master);
53LOG_HANDLE_DECLARE(watchdog_master);
54
55PERCPU_DECLARE(watchdog_percpu, struct watchdog_percpu, watchdog_percpu_ctor);
56static CMDLINE_DECLARE(watchdog, .flags = CMDLINE_ENTRY_SYMBOLIC,
57 .desc = "Watchdog command line namespace");
58
59CMDLINE_CHILD_DECLARE(watchdog, master, .flags = CMDLINE_ENTRY_SYMBOLIC);
60
61CMDLINE_CHILDREN_DECLARE(
62 CMDLINE_NODE(watchdog, master),
63 CMDLINE_INNER_DURATION(heartbeat_interval, config.master_tick_interval,
64 .range = RANGE(MS_TO_NS(1), SECONDS_TO_NS(60))),
65 CMDLINE_INNER_DURATION(bucket_interval, config.bucket_interval,
66 .range = RANGE(MS_TO_NS(1), SECONDS_TO_NS(60))),
67 CMDLINE_INNER_FX(panic_score, config.master_panic_score,
68 .range = RANGE(0, FX_ONE)),
69 CMDLINE_INNER_FX(warn_score, config.master_critical_score,
70 .range = RANGE(0, FX_ONE)),
71 CMDLINE_INNER_FX(suspect_threshold, config.master_suspect_score,
72 .range = RANGE(0, FX_ONE)),
73 CMDLINE_INNER_DURATION(stall_timeout, config.master_stall_timeout,
74 .range = RANGE(0, SECONDS_TO_NS(600))));
75
76/* PERCPU_DECLARE zero-initializes, but we explicitly init buckets */
77static void watchdog_percpu_ctor(struct watchdog_percpu *pcpu, cpu_id_t cpu) {
78 pcpu->id = cpu;
79 pcpu->pets_enabled = false;
80 pcpu->pets = 0;
81 pcpu->anti_pets = 0;
82 watchdog_buckets_init(b: &pcpu->buckets);
83 pcpu->timer.func = watchdog_worker_timer_func;
84 pcpu->timer.flags =
85 TIMER_FLAG_PINNED | TIMER_FLAG_CPU(cpu) | TIMER_FLAG_IRQ;
86}
87
88static inline size_t time_to_bucket(time_ms_t time) {
89 /* TODO: Decide if this condition is not permitted? */
90 if (unlikely(watchdog_global.bucket_interval_ms == 0))
91 return 0;
92
93 return time / watchdog_global.bucket_interval_ms;
94}
95
96static void watchdog_buckets_init(struct watchdog_buckets *b) {
97 seqcount_init(s: &b->seq);
98 b->idx = 0;
99 b->curr_epoch = 0;
100 b->last_heartbeat_ms = 0;
101 for (size_t i = 0; i < WATCHDOG_NUM_BUCKETS; i++) {
102 b->buckets_internal[i].epoch = 0;
103 b->buckets_internal[i].heartbeats = 0;
104 }
105}
106
107/* Must be called with write lock held */
108static void watchdog_buckets_advance_internal(struct watchdog_buckets *buckets,
109 time_ms_t new_time) {
110 if (buckets->last_heartbeat_ms == 0) {
111 buckets->last_heartbeat_ms = new_time;
112 buckets->buckets_internal[buckets->idx].epoch = buckets->curr_epoch;
113 buckets->buckets_internal[buckets->idx].heartbeats = 0;
114 return;
115 }
116
117 if (unlikely(new_time < buckets->last_heartbeat_ms))
118 return;
119
120 size_t old_bucket = time_to_bucket(time: buckets->last_heartbeat_ms);
121 size_t new_bucket = time_to_bucket(time: new_time);
122 if (new_bucket <= old_bucket)
123 return;
124
125 size_t elapsed = new_bucket - old_bucket;
126 if (elapsed >= WATCHDOG_NUM_BUCKETS) {
127 size_t idx_absolute = buckets->idx + elapsed;
128 size_t wraps = idx_absolute / WATCHDOG_NUM_BUCKETS;
129 buckets->curr_epoch += wraps;
130 buckets->idx = idx_absolute % WATCHDOG_NUM_BUCKETS;
131
132 for (size_t i = 0; i < WATCHDOG_NUM_BUCKETS; i++) {
133 buckets->buckets_internal[i].heartbeats = 0;
134 buckets->buckets_internal[i].epoch = 0;
135 }
136
137 buckets->buckets_internal[buckets->idx].epoch = buckets->curr_epoch;
138 buckets->buckets_internal[buckets->idx].heartbeats = 0;
139 } else {
140 for (size_t i = 0; i < elapsed; i++) {
141 buckets->idx++;
142 if (buckets->idx == WATCHDOG_NUM_BUCKETS) {
143 buckets->idx = 0;
144 buckets->curr_epoch++;
145 }
146
147 size_t idx = buckets->idx;
148 buckets->buckets_internal[idx].heartbeats = 0;
149 buckets->buckets_internal[idx].epoch = buckets->curr_epoch;
150 }
151 }
152
153 buckets->last_heartbeat_ms = new_time;
154}
155
156/* Only called by percpu owner */
157static void watchdog_buckets_inc_heartbeat(struct watchdog_buckets *buckets,
158 time_ms_t now) {
159 seqcount_begin_write(s: &buckets->seq);
160 watchdog_buckets_advance_internal(buckets, new_time: now);
161
162 struct watchdog_bucket *bucket = &buckets->buckets_internal[buckets->idx];
163 if (bucket->epoch != buckets->curr_epoch) {
164 bucket->epoch = buckets->curr_epoch;
165 bucket->heartbeats = 0;
166 }
167
168 bucket->heartbeats++;
169 buckets->last_heartbeat_ms = now;
170 seqcount_end_write(s: &buckets->seq);
171}
172
173static void watchdog_do_percpu_heartbeat(time_ms_t now) {
174 kassert(PERCPU_READY(watchdog_percpu));
175
176 atomic_fetch_add_explicit(
177 &PERCPU_PTR(TOPC_IRQL, watchdog_percpu)->heartbeat_seq, 1,
178 memory_order_relaxed);
179
180 watchdog_buckets_inc_heartbeat(
181 buckets: &PERCPU_READ(TOPC_IRQL, watchdog_percpu).buckets, now);
182}
183
184/* Non blocking snapshot */
185static bool watchdog_buckets_snapshot(const struct watchdog_buckets *b,
186 struct watchdog_bucket_snapshot *out) {
187 uint32_t seq;
188 int retries = WATCHDOG_MASTER_SEQCOUNT_SPINS;
189
190 /* NOTE: It is highly unlikely, but not impossible,
191 * for us to end up in a scenario where a worker somehow gets stuck
192 * within the code section under the seqcount write begin/end,
193 * which could lead to the master never reading data from its buckets,
194 * and never promoting past NORMAL in its criticality
195 *
196 * TODO: I can only imagine such a scenario happening with a
197 * kernel live patch gone wrong, but needless to say, this is
198 * a highly unlikely, but theoretically possible state that
199 * we may want to track and handle via bail and panic() */
200 do {
201 seq = seqcount_begin_read_raw(s: &b->seq);
202 if (unlikely((seq & 1) != 0))
203 return false;
204
205 out->idx = b->idx;
206 out->curr_epoch = b->curr_epoch;
207 out->last_heartbeat_ms = b->last_heartbeat_ms;
208 for (size_t i = 0; i < WATCHDOG_NUM_BUCKETS; i++)
209 out->buckets[i] = b->buckets_internal[i];
210
211 } while (seqcount_read_retry(s: &b->seq, start: seq) && --retries > 0);
212
213 return retries > 0;
214}
215
216bool watchdog_count_heartbeats_at(const struct watchdog_buckets *buckets,
217 size_t window_buckets, time_ms_t now,
218 time_ms_t bucket_interval_ms,
219 size_t expected_per_bucket,
220 size_t *out_heartbeats, size_t *out_expected,
221 fx32_32_t *out_score) {
222 struct watchdog_bucket_snapshot snap;
223 if (!watchdog_buckets_snapshot(b: buckets, out: &snap))
224 return false;
225
226 kassert(window_buckets && window_buckets <= WATCHDOG_NUM_BUCKETS);
227 kassert(bucket_interval_ms);
228 kassert(expected_per_bucket);
229
230 if (snap.last_heartbeat_ms == 0)
231 goto not_ready;
232
233 if (unlikely(now < snap.last_heartbeat_ms))
234 now = snap.last_heartbeat_ms;
235
236 size_t cursor = snap.curr_epoch * WATCHDOG_NUM_BUCKETS + snap.idx;
237 size_t last_bucket = snap.last_heartbeat_ms / bucket_interval_ms;
238 size_t now_bucket = now / bucket_interval_ms;
239 kassert(last_bucket >= cursor);
240
241 size_t first_bucket = last_bucket - cursor;
242 size_t completed = now_bucket - first_bucket;
243 if (completed > window_buckets)
244 completed = window_buckets;
245 if (!completed)
246 goto not_ready;
247
248 size_t virtual_cursor = cursor + now_bucket - last_bucket;
249 size_t total = 0, total_expect = 0;
250 for (size_t i = 1; i <= completed; i++) {
251 size_t absolute = virtual_cursor - i;
252 size_t idx = absolute % WATCHDOG_NUM_BUCKETS;
253 size_t expected_epoch = absolute / WATCHDOG_NUM_BUCKETS;
254
255 if (snap.buckets[idx].epoch == expected_epoch)
256 total += snap.buckets[idx].heartbeats;
257 total_expect += expected_per_bucket;
258 }
259
260 if (!total_expect)
261 goto not_ready;
262
263 /* If we're here, out_expect should never be zero, since one tick
264 * must fire for something to enter SUSPECT */
265 kassert(total_expect);
266
267 if (out_score)
268 *out_score = fx_div(a: fx_from_int(x: total_expect - total),
269 b: fx_from_int(x: total_expect));
270
271 if (out_expected)
272 *out_expected = total_expect;
273
274 if (out_heartbeats)
275 *out_heartbeats = total;
276
277 return true;
278
279not_ready:
280 if (out_heartbeats)
281 *out_heartbeats = 0;
282
283 if (out_expected)
284 *out_expected = 0;
285
286 if (out_score)
287 *out_score = 0;
288
289 return false;
290}
291
292static void watchdog_worker_timer_func(struct timer *t) {
293 kassert(irq_in_interrupt());
294
295 struct watchdog_percpu *pcpu = PERCPU_PTR(TOPC_IRQ, watchdog_percpu);
296 enum irql irql = spin_lock_irq_disable(&pcpu->callback_list.lock);
297
298 struct watchdog_callback *cb;
299 list_for_each_entry(cb, &pcpu->callback_list.list, list) {
300 cb->fn(cb);
301 }
302
303 spin_unlock(&pcpu->callback_list.lock, irql);
304
305 time_ms_t now = time_get_ms();
306 watchdog_do_percpu_heartbeat(now);
307
308 timer_modify(timer: t, new: timer_delta_us(NS_TO_US(config.worker_heartbeat_interval)));
309}
310
311static void watchdog_start_petting(cpu_id_t cpu) {
312 struct watchdog_percpu *pcpu = PERCPU_PTR_FOR_CPU(watchdog_percpu, cpu);
313 if (cpu == 0) {
314 pcpu->pets_enabled = true;
315 pcpu->anti_pets = 0;
316 pcpu->pets = 0;
317 return; /* We don't touch the seqcount for BSP */
318 }
319
320 seqcount_begin_write(s: &pcpu->pets_seq);
321 nmi_send(apic_id: cpu);
322}
323
324static void watchdog_read_pets_for(cpu_id_t cpu, size_t *out_pets,
325 size_t *out_anti_pets) {
326 struct watchdog_percpu *pcpu = PERCPU_PTR_FOR_CPU(watchdog_percpu, cpu);
327 *out_pets = atomic_load_explicit(&pcpu->pets, memory_order_relaxed);
328 *out_anti_pets =
329 atomic_load_explicit(&pcpu->anti_pets, memory_order_relaxed);
330}
331
332/* This is the real meat and potatoes of this whole subsystem, and it
333 * needs a bit of explanation that is specifically tied to code
334 * so the implementation is clear
335 *
336 * At the high level, the steps are
337 *
338 * (1) Check on any CRITICAL CPUs -> (2) Check on any SUSPECT CPUs ->
339 * (3) Check on NORMAL domains and CPUs -> (4) Panic last, so that
340 * we can capture all possible CPUs to panic on in (1) and do
341 * any necessary promotions for this tick
342 *
343 * NOTE: Anything that bothers/inspects another CPU must account for
344 * the case where cpu == 0
345 */
346static void watchdog_cpu_promote(cpu_id_t cpu,
347 enum watchdog_master_state new_state) {
348 kassert(cpu_mask_test(&watchdog_master.cpu_masks[new_state - 1], cpu));
349 cpu_mask_clear(m: &watchdog_master.cpu_masks[new_state - 1], cpu);
350 cpu_mask_set(m: &watchdog_master.cpu_masks[new_state], cpu);
351 watchdog_master.cpus[cpu].state = new_state;
352}
353
354static void watchdog_cpu_demote(cpu_id_t cpu,
355 enum watchdog_master_state new_state) {
356 kassert(cpu_mask_test(&watchdog_master.cpu_masks[new_state + 1], cpu));
357 cpu_mask_clear(m: &watchdog_master.cpu_masks[new_state + 1], cpu);
358 cpu_mask_set(m: &watchdog_master.cpu_masks[new_state], cpu);
359 watchdog_master.cpus[cpu].state = new_state;
360}
361
362static bool watchdog_test_outstanding(struct watchdog_master_cpu *cpu) {
363 return seqcount_read_raw(s: &cpu->pcpu->response.seqcount) & 1;
364}
365
366static time_ms_t watchdog_finished_time(struct watchdog_master_cpu *cpu) {
367 return cpu->pcpu->response.finished_ms;
368}
369
370static void watchdog_start_test_on(struct watchdog_master_cpu *cpu,
371 time_ms_t now) {
372 /* Cannot be called with seqcount write active */
373 struct watchdog_percpu *pcpu = cpu->pcpu;
374 struct watchdog_percpu_response *resp = &pcpu->response;
375
376 /* If odd, it's mid-test, cannot start another */
377 kassert(!(seqcount_read_raw(&resp->seqcount) & 1));
378 seqcount_begin_write(s: &resp->seqcount);
379 cpu->critical_test_start = now;
380
381 /* Fine to not guard against cpu == 0, since the IRQ
382 * here is safe as it's not an NMI */
383 ipi_send(apic_id: cpu->id, vector: watchdog_global.critical_test_irq);
384}
385
386static time_ms_t watchdog_spin_for_response(struct watchdog_master_cpu *cpu) {
387 uint32_t seq;
388 int retries = WATCHDOG_MASTER_SEQCOUNT_SPINS;
389 struct watchdog_percpu_response *resp = &cpu->pcpu->response;
390
391 do {
392 seq = seqcount_begin_read_raw(s: &resp->seqcount);
393 if (seq & 1) {
394 cpu_relax();
395 continue;
396 }
397
398 kassert(resp->finished_ms >= cpu->critical_test_start);
399 return resp->finished_ms - cpu->critical_test_start;
400
401 } while ((seqcount_read_retry(s: &resp->seqcount, start: seq) || (seq & 1)) &&
402 --retries > 0);
403
404 return TIME_MS_MAX;
405}
406
407static void watchdog_enter_suspect(cpu_id_t cpu) {
408 struct watchdog_master_cpu *mcpu = &watchdog_master.cpus[cpu];
409
410 kassert(mcpu->state == WATCHDOG_STATE_CRITICAL ||
411 mcpu->state == WATCHDOG_STATE_NORMAL);
412
413 if (mcpu->state == WATCHDOG_STATE_NORMAL) {
414 watchdog_cpu_promote(cpu, new_state: WATCHDOG_STATE_SUSPECT);
415 } else {
416 watchdog_cpu_demote(cpu, new_state: WATCHDOG_STATE_SUSPECT);
417 }
418
419 /* Demote and reset score, tests have been fine */
420 mcpu->lockup_score = config.master_suspect_score;
421 mcpu->lockup_ewma.ewma = mcpu->lockup_score;
422 mcpu->suspect_start_tick = watchdog_master.tick;
423}
424
425static inline void watchdog_record_delta(struct watchdog_master_cpu *mcpu,
426 time_ms_t delta) {
427 mcpu->critical_tests[mcpu->critical_tests_done++] = delta;
428}
429
430/* Score pet count against the expected amount over
431 * the window */
432static inline fx32_32_t watchdog_pets_score(size_t pets) {
433 size_t expected = watchdog_global.expected_heartbeats_per_bucket;
434 if (!expected)
435 return pets > 0 ? FX(0.0) : FX_ONE;
436
437 if (pets >= expected)
438 return FX(0.0);
439
440 return fx_div(a: fx_from_int(x: expected - pets), b: fx_from_int(x: expected));
441}
442
443static void watchdog_master_process_critical(time_ms_t now) {
444 /* All CPUs in here have been CRITICAL for at *least* one tick,
445 * so we never start the petting here, only end + demote
446 *
447 * Here, we look at each CPU, and for each one, we investigate
448 * (1) the pet state, (2) pet status, (3) IPI status
449 */
450
451 cpu_id_t i;
452 watchdog_cpu_for_each(i, WATCHDOG_STATE_CRITICAL) {
453 size_t failures = 0;
454 size_t target_tests, pets, anti_pets;
455 watchdog_read_pets_for(cpu: i, out_pets: &pets, out_anti_pets: &anti_pets);
456
457 if (pets > 0) {
458 target_tests = WATCHDOG_CRITICAL_IPI_TESTS;
459 } else if (!anti_pets) {
460 target_tests = (WATCHDOG_CRITICAL_IPI_TESTS * 3) / 4;
461 } else {
462 target_tests = WATCHDOG_CRITICAL_IPI_TESTS / 2;
463 }
464
465 kassert(target_tests);
466
467 struct watchdog_master_cpu *mcpu = &watchdog_master.cpus[i];
468
469 /* TODO: Logging should be safe in NMI context as
470 * the log site does not acquire locks when passthrough
471 * printing is off */
472
473 /* We have to first check this condition before anything else */
474 bool outstanding = false;
475 if (watchdog_test_outstanding(cpu: mcpu)) {
476 outstanding = true;
477 if (now - mcpu->critical_test_start >=
478 WATCHDOG_CRITICAL_PANIC_THRESHOLD_MS)
479 goto promote_to_panic;
480 }
481
482 if (mcpu->critical_tests_done >= target_tests)
483 goto score;
484
485 if (outstanding)
486 continue;
487
488 /* No outstanding tests, record the last one, start another
489 *
490 * NOTE: the promotion process to CRITICAL will have started the first
491 * test in the sequence, we don't handle that here */
492 time_ms_t finished = watchdog_finished_time(cpu: mcpu);
493 kassert(finished >= mcpu->critical_test_start);
494
495 time_ms_t delta = finished - mcpu->critical_test_start;
496
497 /* NOTE: If a test finishes at all, don't panic */
498 watchdog_record_delta(mcpu, delta);
499
500 /* Start a new one */
501 watchdog_start_test_on(cpu: mcpu, now);
502
503 /* Spin a few times to see if we get a response,
504 * and record it if we do */
505 delta = watchdog_spin_for_response(cpu: mcpu);
506 if (delta != TIME_MS_MAX)
507 watchdog_record_delta(mcpu, delta);
508
509 continue;
510
511 score:
512 /* Use the scoring formula:
513 *
514 * failed/total * TEST_WEIGHT + pet_factor * PET_WEIGHT
515 *
516 * mapped to [warn_score, 1]
517 *
518 */
519
520 for (int j = 0; j < WATCHDOG_CRITICAL_IPI_TESTS; j++) {
521 if (mcpu->critical_tests[j] > WATCHDOG_CRITICAL_PASS_THRESHOLD_MS)
522 failures++;
523 }
524
525 fx32_32_t test_part =
526 fx_mul(a: fx_div(a: fx_from_int(x: failures),
527 b: fx_from_int(WATCHDOG_CRITICAL_IPI_TESTS)),
528 WATCHDOG_CRITICAL_TESTS_FACTOR);
529 fx32_32_t pet_part =
530 fx_mul(a: watchdog_pets_score(pets), WATCHDOG_CRITICAL_PETS_FACTOR);
531 fx32_32_t total = test_part + pet_part;
532 kassert(total < FX_ONE);
533
534 if (total == 0) {
535 watchdog_enter_suspect(cpu: i);
536 } else {
537 /* map it from [0, 1] -> [warn_score, 1] */
538 fx32_32_t mapped =
539 fx_map(value: total, from_low: 0, FX_ONE, to_low: config.master_critical_score, FX_ONE);
540
541 kassert(IN_RANGE(mapped, config.master_critical_score, FX_ONE));
542 mcpu->lockup_score = mapped;
543
544 /* Panic promotion only bothers with a bitmap,
545 * everything else stays, since it never demotes from there */
546 if (mcpu->lockup_score >= config.master_panic_score) {
547 promote_to_panic:
548 watchdog_cpu_promote(cpu: i, new_state: WATCHDOG_STATE_PANIC);
549 }
550 }
551 }
552}
553
554static void watchdog_enter_critical(struct watchdog_master_cpu *mcpu,
555 time_ms_t now) {
556 /* The idea here: we'll need to start the tests so
557 * the critical processing's invariants hold */
558 watchdog_cpu_promote(cpu: mcpu->id, new_state: WATCHDOG_STATE_CRITICAL);
559 memset(&mcpu->critical_tests, 0, sizeof(mcpu->critical_tests));
560 mcpu->critical_tests_done = 0;
561 mcpu->critical_start_tick = watchdog_master.tick;
562
563 watchdog_start_petting(cpu: mcpu->id);
564
565 mcpu->critical_test_start = now;
566 watchdog_start_test_on(cpu: mcpu, now);
567}
568
569static void watchdog_enter_normal(struct watchdog_master_cpu *mcpu) {
570 ewma_init(e: &mcpu->lockup_ewma, WATCHDOG_EWMA_ALPHA);
571 watchdog_cpu_demote(cpu: mcpu->id, new_state: WATCHDOG_STATE_NORMAL);
572 mcpu->lockup_score = 0;
573}
574
575static void watchdog_master_process_suspect(time_ms_t now) {
576 cpu_id_t i;
577 watchdog_cpu_for_each(i, WATCHDOG_STATE_SUSPECT) {
578 /*
579 * SUSPECT CPUs are subject to EWMA monitoring,
580 * which can either result in it being good enough,
581 * or above warn_score, and if it lingers for
582 * long enough in SUSPECT limbo, we emit warnings
583 */
584 struct watchdog_percpu *pcpu = PERCPU_PTR_FOR_CPU(watchdog_percpu, i);
585 struct watchdog_master_cpu *mcpu = &watchdog_master.cpus[i];
586
587 fx32_32_t new;
588 if (!watchdog_count_heartbeats_at(
589 buckets: &pcpu->buckets, WATCHDOG_WINDOW_BUCKETS, now,
590 bucket_interval_ms: watchdog_global.bucket_interval_ms,
591 expected_per_bucket: watchdog_global.expected_heartbeats_per_bucket, NULL, NULL,
592 out_score: &new))
593 continue;
594
595 ewma_update(e: &mcpu->lockup_ewma, new);
596
597 mcpu->lockup_score = mcpu->lockup_ewma.ewma;
598 if (mcpu->lockup_score >= config.master_critical_score) {
599 watchdog_enter_critical(mcpu, now);
600 } else if (mcpu->lockup_score < config.master_suspect_score) {
601 watchdog_enter_normal(mcpu);
602 }
603 }
604}
605
606static void watchdog_master_process_normal(void) {
607 cpu_id_t i;
608 watchdog_cpu_for_each(i, WATCHDOG_STATE_NORMAL) {
609 struct watchdog_percpu *pcpu = PERCPU_PTR_FOR_CPU(watchdog_percpu, i);
610 fx32_32_t score;
611 if (!watchdog_count_heartbeats_at(
612 buckets: &pcpu->buckets, WATCHDOG_WINDOW_BUCKETS, now: time_get_ms(),
613 bucket_interval_ms: watchdog_global.bucket_interval_ms,
614 expected_per_bucket: watchdog_global.expected_heartbeats_per_bucket, NULL, NULL,
615 out_score: &score))
616 continue;
617
618 if (score >= config.master_suspect_score)
619 watchdog_enter_suspect(cpu: i);
620 }
621}
622
623static void watchdog_master_process_panic(void) {
624 cpu_id_t i;
625
626 /* TODO: Aggregate, print other data, for this first version we
627 * just print the first CPU that appears hung */
628 cpu_mask_for_each(i, watchdog_master.cpu_masks[WATCHDOG_STATE_PANIC]) {
629 watchdog_panic("CPU %zu hard lockup (watchdog missed heartbeats)", i);
630 }
631}
632
633/* We check here if the heartbeat counter changed, measuring elapsed
634 * time in only master ticks. This is because places like time_get_ms()
635 * have a seqcount that can get stuck, and this is effectively
636 * the "quick check" for the edge case stall */
637static void watchdog_master_process_stall(void) {
638 if (!watchdog_global.stall_timeout_ticks)
639 return;
640
641 /* Give the workers time to arm their timers before judging them. */
642 if (watchdog_master.tick < WATCHDOG_STALL_GRACE_TICKS)
643 return;
644
645 for (cpu_id_t i = 0; i < global.core_count; i++) {
646 struct watchdog_master_cpu *mcpu = &watchdog_master.cpus[i];
647 struct watchdog_percpu *pcpu = PERCPU_PTR_FOR_CPU(watchdog_percpu, i);
648
649 uint64_t seen =
650 atomic_load_explicit(&pcpu->heartbeat_seq, memory_order_relaxed);
651
652 if (seen != mcpu->last_seen_heartbeat) {
653 mcpu->last_seen_heartbeat = seen;
654 mcpu->last_progress_tick = watchdog_master.tick;
655 continue;
656 }
657
658 if (watchdog_master.tick - mcpu->last_progress_tick <
659 watchdog_global.stall_timeout_ticks)
660 continue;
661
662 watchdog_panic("CPU %zu stalled: no heartbeat for %zu master ticks "
663 "(seq stuck at %llu)",
664 (size_t) i,
665 watchdog_master.tick - mcpu->last_progress_tick,
666 (unsigned long long) seen);
667 }
668}
669
670static enum irq_result watchdog_master_nmi_handler(void *ctx, irq_t irq,
671 struct irq_context *regs) {
672 (void) ctx, (void) irq, (void) regs;
673
674 /* Since the other NMI ISRs run well before this one does, we can
675 * guarantee that this IS us, since we register last, but we do
676 * a CPU ID check to really make sure
677 *
678 * HACK: We simply use the call site ordering to guarantee list_add_tail
679 * happens in the right order, but we will want to probably need to
680 * make this more robust, reusable, and less reliant on call ordering */
681 if (smp_id(cond: TOPC_IRQ) != 0)
682 return IRQ_NONE; /* Not us */
683
684 watchdog_master.tick++;
685
686 /* Process what could've stalled before anything else */
687 watchdog_master_process_stall();
688
689 /* If anything is wedged on the timekeeper seqlock, this
690 * prevents the NMI from stalling too, so we only try_ here */
691 time_ms_t now;
692 if (!time_try_get_ms(out: &now))
693 return IRQ_HANDLED;
694
695 watchdog_master_process_critical(now);
696 watchdog_master_process_suspect(now);
697 watchdog_master_process_normal();
698 watchdog_master_process_panic();
699
700 return IRQ_HANDLED;
701}
702
703static enum irq_result watchdog_pet_nmi_handler(void *ctx, irq_t irq,
704 struct irq_context *regs) {
705 (void) ctx, (void) irq, (void) regs;
706 struct watchdog_percpu *pcpu = PERCPU_PTR(TOPC_IRQ, watchdog_percpu);
707
708 /* Must be set, if it doesn't, something happened */
709 if (seqcount_read_raw(s: &pcpu->pets_seq) & 1) {
710 pcpu->pets_enabled = true;
711 seqcount_end_write(s: &pcpu->pets_seq);
712 return IRQ_HANDLED;
713 }
714
715 return IRQ_NONE;
716}
717
718static enum irq_result watchdog_test_handler(void *ctx, irq_t irq,
719 struct irq_context *regs) {
720 (void) ctx, (void) irq, (void) regs;
721 struct watchdog_percpu *pcpu = PERCPU_PTR(TOPC_IRQ, watchdog_percpu);
722
723 if (seqcount_read_raw(s: &pcpu->response.seqcount) & 1) {
724 pcpu->response.finished_ms = time_get_ms();
725 seqcount_end_write(s: &pcpu->response.seqcount);
726 return IRQ_HANDLED;
727 }
728
729 return IRQ_NONE;
730}
731
732/* We'll want to set up the master and all the workers */
733void watchdog_init(void) {
734 kassert(PERCPU_READY(watchdog_percpu));
735
736 for (int i = 0; i < WATCHDOG_STATE_MAX; i++) {
737 alloc_or_die(
738 cpu_mask_init(&watchdog_master.cpu_masks[i], global.core_count));
739 }
740
741 alloc_or_die(
742 cpu_mask_init(&watchdog_master.scratch_mask, global.core_count));
743
744 cpu_mask_set_all(&watchdog_master.cpu_masks[WATCHDOG_STATE_NORMAL]);
745 watchdog_master.cpus =
746 kmalloc_or_die(sizeof(struct watchdog_master_cpu) * global.core_count,
747 ALLOC_FLAGS_ZERO);
748
749 watchdog_global.critical_test_irq = irq_alloc_entry();
750 irq_register(name: "watchdog_test", vector: watchdog_global.critical_test_irq,
751 handler: watchdog_test_handler, NULL, flags: IRQ_FLAG_NONE);
752 irq_set_chip(vector: watchdog_global.critical_test_irq, chip: lapic_get_chip(), NULL);
753 irq_register(name: "watchdog_master", IRQ_NMI, handler: watchdog_master_nmi_handler, NULL,
754 flags: IRQ_FLAG_SHARED);
755
756 irq_register(name: "watchdog_pet", IRQ_NMI, handler: watchdog_pet_nmi_handler, NULL,
757 flags: IRQ_FLAG_SHARED);
758
759 for (cpu_id_t i = 0; i < global.core_count; i++) {
760 watchdog_master.cpus[i].id = i;
761 watchdog_master.cpus[i].state = WATCHDOG_STATE_NORMAL;
762 ewma_init(e: &watchdog_master.cpus[i].lockup_ewma, WATCHDOG_EWMA_ALPHA);
763 watchdog_master.cpus[i].lockup_score = FX(0.0);
764 watchdog_master.cpus[i].pcpu = PERCPU_PTR_FOR_CPU(watchdog_percpu, i);
765 locked_list_init(ll: &PERCPU_PTR_FOR_CPU(watchdog_percpu, i)->callback_list,
766 LOCKED_LIST_INIT_IRQ_DISABLE);
767 }
768}
769
770void watchdog_start(void) {
771 watchdog_global.bucket_interval_ms = NS_TO_MS(config.bucket_interval);
772 watchdog_global.expected_heartbeats_per_bucket =
773 watchdog_global.bucket_interval_ms /
774 NS_TO_MS(config.worker_heartbeat_interval);
775
776 time_ms_t tick_ms = NS_TO_MS(config.master_tick_interval);
777 watchdog_global.stall_timeout_ticks =
778 (config.master_stall_timeout && tick_ms)
779 ? NS_TO_MS(config.master_stall_timeout) / tick_ms
780 : 0;
781
782 for (cpu_id_t i = 0; i < global.core_count; i++) {
783 watchdog_master.cpus[i].last_seen_heartbeat = 0;
784 watchdog_master.cpus[i].last_progress_tick = 0;
785 }
786
787 struct watchdog_percpu *pcpu;
788 percpu_for_each(watchdog_percpu, pcpu, cpu) {
789 timer_modify(timer: &pcpu->timer,
790 new: timer_delta_us(NS_TO_US(config.master_tick_interval)));
791 }
792
793 pit_init();
794 pit_wire_periodic_nmi(interval_ns: config.master_tick_interval);
795 ioapic_route_isa_nmi(isa_irq: 0, /* cpu id */ dest_apic_id: 0);
796}
797
798void watchdog_pet(void) {
799 if (PERCPU_READY(watchdog_percpu)) {
800 struct watchdog_percpu *this = PERCPU_PTR(TOPC_NONE, watchdog_percpu);
801 /* The caller must make sure we cannot be preempted */
802 if (this->pets_enabled) {
803 kassert(irql_get() >= IRQL_DISPATCH_LEVEL);
804 this->pets++;
805 }
806 }
807}
808
809void watchdog_anti_pet(void) {
810 if (PERCPU_READY(watchdog_percpu)) {
811 struct watchdog_percpu *this = PERCPU_PTR(TOPC_NONE, watchdog_percpu);
812 /* The caller must make sure we cannot be preempted */
813 if (this->pets_enabled) {
814 kassert(irql_get() >= IRQL_DISPATCH_LEVEL);
815 this->anti_pets++;
816 }
817 }
818}
819
820void watchdog_callback_add(cpu_id_t id, struct watchdog_callback *cb) {
821 struct watchdog_percpu *percpu = PERCPU_PTR_FOR_CPU(watchdog_percpu, id);
822 locked_list_add(ll: &percpu->callback_list, lh: &cb->list);
823}
824
825void watchdog_callback_remove(cpu_id_t id, struct watchdog_callback *cb) {
826 struct watchdog_percpu *percpu = PERCPU_PTR_FOR_CPU(watchdog_percpu, id);
827 locked_list_del(ll: &percpu->callback_list, lh: &cb->list);
828}
829