1#include <console/panic.h>
2#include <global.h>
3#include <log.h>
4#include <sch/sched.h>
5#include <watchdog.h>
6
7#include "internal.h"
8
9#ifdef DEBUG_SCHED
10
11#define SCHED_DEBUG_STRAND_MS 5000
12
13/* Actually count */
14static size_t sched_debug_count_tree(struct scheduler *sched, struct rbt *tree,
15 const char *which) {
16 size_t n = 0;
17 struct rbt_node *prev = NULL;
18 struct rbt_node *node;
19
20 rbt_for_each(node, tree) {
21 enforce(!prev || tree->compare(prev, node) <= 0, CORRUPTION,
22 "cpu %zu %s out of key order", (size_t) sched->core_id, which);
23 prev = node;
24 n++;
25 }
26
27 return n;
28}
29
30static size_t sched_debug_count_list(struct list_head *q) {
31 size_t n = 0;
32 struct list_node *pos;
33
34 list_for_each(pos, q) {
35 n++;
36 }
37
38 return n;
39}
40
41void scheduler_debug_validate(struct scheduler *sched) {
42 size_t linked[THREAD_PRIO_CLASS_COUNT] = {0};
43
44 linked[THREAD_PRIO_CLASS_TIMESHARE] =
45 sched_debug_count_tree(sched, &sched->thread_rbt, "thread_rbt") +
46 sched_debug_count_tree(sched, &sched->completed_rbt, "completed_rbt");
47
48 linked[THREAD_PRIO_CLASS_URGENT] =
49 sched_debug_count_list(&sched->urgent_threads);
50 linked[THREAD_PRIO_CLASS_REALTIME] =
51 sched_debug_count_list(&sched->rt_threads);
52 linked[THREAD_PRIO_CLASS_BACKGROUND] =
53 sched_debug_count_list(&sched->bg_threads);
54
55 uint8_t bitmap = atomic_load_rlx(&sched->queue_bitmap);
56 size_t total = 0;
57
58 for (size_t prio = 0; prio < THREAD_PRIO_CLASS_COUNT; prio++) {
59 enforce(linked[prio] == sched->thread_count[prio], CORRUPTION,
60 "cpu %zu class %zu: %zu linked, %zu counted",
61 (size_t) sched->core_id, prio, linked[prio],
62 sched->thread_count[prio]);
63
64 bool bit = bitmap & (1u << prio);
65 enforce(bit == (linked[prio] != 0), CORRUPTION,
66 "cpu %zu class %zu: bitmap says %d with %zu linked",
67 (size_t) sched->core_id, prio, (int) bit, linked[prio]);
68
69 total += linked[prio];
70 }
71
72 enforce(total == sched->total_thread_count, CORRUPTION,
73 "cpu %zu: %zu linked, total_thread_count %zu",
74 (size_t) sched->core_id, total, sched->total_thread_count);
75}
76
77static void sched_debug_dump(cpu_id_t cpu, uint64_t now_ms) {
78 struct scheduler *s = global.schedulers[cpu];
79 struct core *c = global.cores[cpu];
80 struct thread *curr = s->current;
81
82 log_msg(LOG_ERROR,
83 "rq %zu: queued=%zu [bg %zu ts %zu rt %zu urg %zu] bitmap=0x%x",
84 (size_t) cpu, s->total_thread_count, s->thread_count[0],
85 s->thread_count[1], s->thread_count[2], s->thread_count[3],
86 (unsigned) atomic_load_rlx(&s->queue_bitmap));
87
88 log_msg(LOG_ERROR,
89 "rq %zu: idle=%d needs_resched=%d tick=%d progress %lu ms ago",
90 (size_t) cpu, (int) atomic_load_rlx(&c->idle),
91 (int) atomic_load_rlx(&c->needs_resched),
92 (int) atomic_load_rlx(&s->tick_enabled),
93 (unsigned long) (now_ms - atomic_load_rlx(&s->debug_progress_ms)));
94
95 log_msg(LOG_ERROR, "rq %zu: current '%s' state=%d", (size_t) cpu,
96 curr ? curr->name : "?", curr ? (int) thread_get_state(curr) : -1);
97}
98
99static atomic_bool sched_debug_tripped = false;
100
101static void sched_debug_strand_check(struct watchdog_callback *cb) {
102 cc_ignore(cb);
103
104 if (atomic_load_rlx(&sched_debug_tripped))
105 return;
106
107 uint64_t now_ms = (uint64_t) time_get_ms();
108 size_t i;
109
110 for_each_cpu_id(i) {
111 struct scheduler *s = global.schedulers[i];
112
113 if (s->total_thread_count == 0)
114 continue;
115
116 uint64_t progress = atomic_load_rlx(&s->debug_progress_ms);
117 if (!progress || progress >= now_ms ||
118 now_ms - progress < SCHED_DEBUG_STRAND_MS)
119 continue;
120
121 /* RT threads run with tick off and can hold the core */
122 struct thread *curr = s->current;
123 if (curr && thread_is_rt(curr))
124 continue;
125
126 if (atomic_exchange(&sched_debug_tripped, true))
127 return;
128
129 size_t j;
130 for_each_cpu_id(j) {
131 sched_debug_dump(j, now_ms);
132 }
133
134 panic("rq %zu stranded: %zu queued, nothing picked for %lu ms", i,
135 s->total_thread_count, (unsigned long) (now_ms - progress));
136 }
137}
138
139static struct watchdog_callback sched_debug_callbacks[CPU_MASK_BITS];
140
141void scheduler_debug_start(void) {
142 uint64_t now_ms = (uint64_t) time_get_ms();
143 size_t i;
144
145 for_each_cpu_id(i) {
146 atomic_store_rlx(&global.schedulers[i]->debug_progress_ms, now_ms);
147 sched_debug_callbacks[i].fn = sched_debug_strand_check;
148 watchdog_callback_add((cpu_id_t) i, &sched_debug_callbacks[i]);
149 }
150}
151
152#endif /* DEBUG_SCHED */
153