1#include <math/min_max.h>
2#include <sch/sched.h>
3#include <string.h>
4#include <thread/apc.h>
5#include <thread/thread.h>
6
7#include "sch/internal.h"
8
9static inline bool resume_is_from_sleep(uint8_t wake_reason) {
10 return wake_reason == THREAD_WAKE_REASON_SLEEP_TIMEOUT ||
11 wake_reason == THREAD_WAKE_REASON_SLEEP_MANUAL;
12}
13
14static inline bool wait_reason_is_sleep(thread_act_reason_t reason) {
15 return reason == THREAD_SLEEP_REASON_MANUAL;
16}
17
18static const char *thread_prio_class_str(enum thread_prio_class c) {
19 switch (c) {
20 case THREAD_PRIO_CLASS_URGENT: return "URGENT";
21 case THREAD_PRIO_CLASS_REALTIME: return "RT";
22 case THREAD_PRIO_CLASS_TIMESHARE: return "TS";
23 case THREAD_PRIO_CLASS_BACKGROUND: return "BG";
24 default: return "?";
25 }
26}
27
28static const char *reason_str(thread_act_reason_t reason) {
29 switch (reason) {
30 case THREAD_WAKE_REASON_BLOCKING_IO: return "WAKE_IO";
31 case THREAD_WAKE_REASON_BLOCKING_MANUAL: return "WAKE_MANUAL";
32 case THREAD_WAKE_REASON_SLEEP_TIMEOUT: return "WAKE_TIMEOUT";
33 case THREAD_WAKE_REASON_SLEEP_MANUAL: return "WAKE_SLEEP_MANUAL";
34 case THREAD_BLOCK_REASON_IO: return "BLOCK_IO";
35 case THREAD_BLOCK_REASON_MANUAL: return "BLOCK_MANUAL";
36 case THREAD_SLEEP_REASON_MANUAL: return "SLEEP_MANUAL";
37 case THREAD_EVENT_REASON_NONE: return "NONE";
38 default: return "?";
39 }
40}
41
42static void print_events(const struct thread *t) {
43 const struct thread_activity_data *d = &t->activity_data;
44 size_t newest = (d->head - 1) & THREAD_EVENT_RINGBUFFER_MASK;
45 bool has_newest = d->events[newest].wake_reason != THREAD_EVENT_REASON_NONE;
46
47 if (d->active_wait_reason != THREAD_EVENT_REASON_NONE) {
48 printf(format: " active_wait: { reason: %s, start_ms: %u },\n",
49 reason_str(reason: d->active_wait_reason), d->wait_start_ms);
50 }
51
52 printf(format: " events: [\n");
53 for (size_t i = 0; i < THREAD_EVENT_RINGBUFFER_CAPACITY; i++) {
54 const struct thread_event_reason *e = &d->events[i];
55 if (e->wake_reason == THREAD_EVENT_REASON_NONE &&
56 e->wait_reason == THREAD_EVENT_REASON_NONE)
57 continue;
58
59 printf(format: " { wait: %s, wake: %s, dur: %u ms",
60 reason_str(reason: e->wait_reason), reason_str(reason: e->wake_reason),
61 e->duration_ms);
62
63 if (has_newest && i == newest)
64 printf(format: " <-- newest");
65
66 printf(format: " },\n");
67 }
68 printf(format: " ],\n");
69}
70
71void thread_print(const struct thread *t) {
72 printf(format: "Thread %s {\n", t->name);
73 printf(format: " id: %llu,\n", (unsigned long long) t->id);
74 printf(format: " state: %s,\n", thread_state_str(atomic_load(&t->state)));
75 printf(format: " core: %lld,\n", (long long) t->curr_core);
76 printf(format: " stack_size: %zu,\n", t->stack_size);
77
78 /* priorities */
79 printf(format: " base_prio: %s,\n", thread_prio_class_str(c: t->base_prio_class));
80 printf(format: " perceived_prio: %s,\n",
81 thread_prio_class_str(c: t->perceived_prio_class));
82 printf(format: " activity_score: %u, dynamic_delta: %d, weight: %llu,\n",
83 t->activity_score, t->dynamic_delta, (unsigned long long) t->weight);
84
85#ifdef DEBUG_THREAD
86 printf(" boost_count: %u\n", t->boost_count);
87#endif
88
89 printf(format: " activity_class: %s,\n",
90 thread_activity_class_str(c: t->activity_class));
91
92 /* time / slice info */
93 printf(format: " run: %llu ms / budget: %llu ms,\n",
94 (unsigned long long) t->period_runtime_raw_ms,
95 (unsigned long long) t->budget_time_raw_ms);
96 printf(format: " timeslice_length: %llu ms,\n",
97 (unsigned long long) t->timeslice_length_raw_ms);
98 printf(format: " completed_period: %llu,\n",
99 (unsigned long long) t->completed_period);
100 printf(format: " virtual_runtime: %llu / virtual_budget: %llu,\n",
101 (unsigned long long) t->virtual_period_runtime,
102 (unsigned long long) t->virtual_budget);
103
104 /* metrics overview */
105 printf(format: " activity_metrics: { run_ratio: %llu, block_ratio: %llu, "
106 "sleep_ratio: %llu, wake_freq: %llu },\n",
107 (unsigned long long) t->activity_metrics.run_ratio,
108 (unsigned long long) t->activity_metrics.block_ratio,
109 (unsigned long long) t->activity_metrics.sleep_ratio,
110 (unsigned long long) t->activity_metrics.wake_freq);
111
112 /* profiling */
113#ifdef DEBUG_THREAD
114 printf(" context_switches: %zu,\n", t->context_switches);
115 printf(" preemptions: %zu,\n", t->preemptions);
116 printf(" wakes: %u, blocks: %u, sleeps: %u,\n", t->total_wake_count,
117 t->total_block_count, t->total_sleep_count);
118 printf(" apcs: %u,\n", t->total_apcs_ran);
119 printf(" creation_time: %lld ms,\n", (long long) t->creation_time_ms);
120#endif
121
122 /* APC state */
123 printf(format: " executing_apc: %s,\n",
124 (atomic_load_rlx(&t->flags) & THREAD_FLAG_EXECUTING_APC) ? "true"
125 : "false");
126 printf(format: " special_apc_disable: %u, kernel_apc_disable: %u,\n",
127 t->special_apc_disable, t->kernel_apc_disable);
128
129 /* activity events ringbuffer */
130 print_events(t);
131
132 printf(format: "}\n");
133}
134
135enum thread_bucket_span_kind {
136 SPAN_RUN,
137 SPAN_BLOCK,
138 SPAN_SLEEP,
139};
140
141static inline void ensure_bucket_epoch(struct thread_activity_bucket *b,
142 uint32_t epoch) {
143 if (b->epoch != epoch) {
144 b->epoch = epoch;
145 b->run_ms = 0;
146 b->block_ms = 0;
147 b->sleep_ms = 0;
148 b->wake_count = 0;
149 }
150}
151
152static void bucket_add_span(struct thread_activity_stats *stats,
153 time_ms_t start_ms, time_ms_t end_ms,
154 enum thread_bucket_span_kind kind) {
155 if (end_ms <= start_ms)
156 return;
157
158 uint64_t start = start_ms;
159 uint64_t end = end_ms;
160
161 uint64_t start_epoch = start >> THREAD_ACTIVITY_BUCKET_SHIFT;
162 uint64_t end_epoch = end >> THREAD_ACTIVITY_BUCKET_SHIFT;
163 uint64_t orig_start_epoch = start_epoch;
164
165 if (end_epoch - start_epoch >= THREAD_ACTIVITY_BUCKET_COUNT)
166 start_epoch = end_epoch - (THREAD_ACTIVITY_BUCKET_COUNT - 1);
167
168 for (uint64_t ep = start_epoch; ep <= end_epoch; ++ep) {
169 size_t idx = ep & THREAD_ACTIVITY_INDEX_MASK;
170 struct thread_activity_bucket *b = &stats->buckets[idx];
171
172 ensure_bucket_epoch(b, epoch: (uint32_t) ep);
173
174 uint64_t bucket_start = ep << THREAD_ACTIVITY_BUCKET_SHIFT;
175 uint64_t bucket_end = (ep + 1) << THREAD_ACTIVITY_BUCKET_SHIFT;
176
177 uint64_t slice_start = ep == orig_start_epoch ? start : bucket_start;
178 uint64_t slice_end = ep == end_epoch ? end : bucket_end;
179 uint32_t overlap =
180 slice_end > slice_start ? (uint32_t) (slice_end - slice_start) : 0;
181
182 uint16_t *field;
183 if (kind == SPAN_RUN) {
184 field = &b->run_ms;
185 } else if (kind == SPAN_BLOCK) {
186 field = &b->block_ms;
187 } else {
188 field = &b->sleep_ms;
189 }
190
191 uint32_t sum = *field + overlap;
192 *field = sum > THREAD_ACTIVITY_BUCKET_DURATION
193 ? THREAD_ACTIVITY_BUCKET_DURATION
194 : sum;
195 }
196}
197
198void thread_add_wake_reason(struct thread *t, thread_act_reason_t reason) {
199 struct thread_activity_data *d = &t->activity_data;
200 time_ms_t now = time_get_ms();
201 uint16_t clamped = 0;
202 thread_act_reason_t wait_reason = d->active_wait_reason;
203
204 if (wait_reason != THREAD_EVENT_REASON_NONE) {
205 /* Explicitly truncate and cut off the high bits to save some space */
206 uint32_t elapsed_32 = (uint32_t) now - d->wait_start_ms;
207 clamped = CLAMP(elapsed_32, 0, UINT16_MAX);
208
209 time_ms_t start = now > elapsed_32 ? now - elapsed_32 : 0;
210 bool is_sleep =
211 wait_reason_is_sleep(reason: wait_reason) || resume_is_from_sleep(wake_reason: reason);
212
213 bucket_add_span(stats: &t->activity_stats, start_ms: start, end_ms: now,
214 kind: is_sleep ? SPAN_SLEEP : SPAN_BLOCK);
215
216 d->active_wait_reason = THREAD_EVENT_REASON_NONE;
217 }
218
219 uint32_t now_epoch = (uint32_t) (now >> THREAD_ACTIVITY_BUCKET_SHIFT);
220 size_t wbi = now_epoch & THREAD_ACTIVITY_INDEX_MASK;
221 struct thread_activity_bucket *wb = &t->activity_stats.buckets[wbi];
222 ensure_bucket_epoch(b: wb, epoch: now_epoch);
223 if (wb->wake_count < UINT16_MAX)
224 wb->wake_count++;
225
226 struct thread_event_reason *slot =
227 &d->events[d->head & THREAD_EVENT_RINGBUFFER_MASK];
228
229 slot->duration_ms = clamped;
230 slot->wait_reason = wait_reason;
231 slot->wake_reason = reason;
232 d->head++;
233
234#ifdef DEBUG_THREAD
235 t->total_wake_count++;
236#endif
237}
238
239void thread_update_runtime_buckets(struct thread *thread, time_ms_t time) {
240 if (time <= thread->run_start_time)
241 return;
242
243 bucket_add_span(stats: &thread->activity_stats, start_ms: thread->run_start_time, end_ms: time,
244 kind: SPAN_RUN);
245
246 uint64_t new_ms = (uint64_t) (time - thread->run_start_time);
247
248 thread->period_runtime_raw_ms += new_ms;
249
250 uint64_t mult = thread->activity_score == 0 ? 1 : thread->activity_score;
251
252 thread->virtual_period_runtime += new_ms * mult;
253}
254
255void thread_add_block_reason(struct thread *t, thread_act_reason_t reason) {
256#ifdef DEBUG_THREAD
257 t->total_block_count++;
258#endif
259 t->activity_data.wait_start_ms = (uint32_t) time_get_ms();
260 t->activity_data.active_wait_reason = reason;
261}
262
263void thread_add_sleep_reason(struct thread *t, thread_act_reason_t reason) {
264#ifdef DEBUG_THREAD
265 t->total_sleep_count++;
266#endif
267 t->activity_data.wait_start_ms = (uint32_t) time_get_ms();
268 t->activity_data.active_wait_reason = reason;
269}
270
271/* NOTE: these two are UNSAFE to call outside of initializing threads.
272 *
273 * TODO: implement lock nest tracking and whatnot to perform base prio
274 * changes, since much of locking depends on base prios */
275void thread_set_timesharing(struct thread *t) {
276 enum irql pirql = spin_lock_high(&t->pi_lock);
277
278 t->base_prio_class = THREAD_PRIO_CLASS_TIMESHARE;
279 cc_discard(thread_recompute_prio_class_locked(t));
280
281 spin_unlock(&t->pi_lock, pirql);
282}
283
284void thread_set_background(struct thread *t) {
285 enum irql pirql = spin_lock_high(&t->pi_lock);
286
287 t->base_prio_class = THREAD_PRIO_CLASS_BACKGROUND;
288 cc_discard(thread_recompute_prio_class_locked(t));
289
290 spin_unlock(&t->pi_lock, pirql);
291}
292