1#include <sch/sched.h>
2#include <string.h>
3#include <thread/thread.h>
4
5#include "sch/internal.h"
6
7static inline bool thread_wake_is_from_block(uint8_t wake_reason) {
8 return wake_reason == THREAD_WAKE_REASON_BLOCKING_IO ||
9 wake_reason == THREAD_WAKE_REASON_BLOCKING_MANUAL;
10}
11
12static inline bool thread_wake_is_from_sleep(uint8_t wake_reason) {
13 return wake_reason == THREAD_WAKE_REASON_SLEEP_TIMEOUT ||
14 wake_reason == THREAD_WAKE_REASON_SLEEP_MANUAL;
15}
16
17static const char *thread_prio_class_str(enum thread_prio_class c) {
18 switch (c) {
19 case THREAD_PRIO_CLASS_URGENT: return "URGENT";
20 case THREAD_PRIO_CLASS_RT: return "RT";
21 case THREAD_PRIO_CLASS_TIMESHARE: return "TS";
22 case THREAD_PRIO_CLASS_BACKGROUND: return "BG";
23 default: return "?";
24 }
25}
26
27static const char *reason_str(uint8_t reason) {
28 switch (reason) {
29 case THREAD_WAKE_REASON_BLOCKING_IO: return "WAKE_IO";
30 case THREAD_WAKE_REASON_BLOCKING_MANUAL: return "WAKE_MANUAL";
31 case THREAD_WAKE_REASON_SLEEP_TIMEOUT: return "WAKE_TIMEOUT";
32 case THREAD_WAKE_REASON_SLEEP_MANUAL: return "WAKE_SLEEP_MANUAL";
33 case THREAD_BLOCK_REASON_IO: return "BLOCK_IO";
34 case THREAD_BLOCK_REASON_MANUAL: return "BLOCK_MANUAL";
35 case THREAD_SLEEP_REASON_MANUAL: return "SLEEP_MANUAL";
36 case THREAD_EVENT_REASON_NONE: return "NONE";
37 default: return "?";
38 }
39}
40
41static void print_ringbuffer(const struct thread *t, bool wake_reasons,
42 const char *label, struct thread_event_reason *buf,
43 size_t head) {
44 printf(format: " %s: [\n", label);
45 for (size_t i = 0; i < THREAD_EVENT_RINGBUFFER_CAPACITY; i++) {
46 struct thread_event_reason *e = &buf[i];
47 if (e->reason == THREAD_ASSOCIATED_REASON_NONE)
48 continue;
49
50 printf(format: " { reason: %s, ts: %lld, cycle: %llu",
51 reason_str(reason: e->reason), (long long) e->timestamp,
52 (unsigned long long) e->cycle);
53
54 if (e->associated_reason.reason != THREAD_ASSOCIATED_REASON_NONE) {
55 if (!wake_reasons)
56 printf(
57 format: ", assoc: { reason: %s, ts: %lld, cycle: %llu }",
58 reason_str(reason: t->activity_data
59 ->wake_reasons[e->associated_reason.reason]
60 .reason),
61 t->activity_data->wake_reasons[e->associated_reason.reason]
62 .timestamp,
63 (unsigned long long) e->associated_reason.cycle);
64 }
65
66 if (i == head % THREAD_EVENT_RINGBUFFER_CAPACITY)
67 printf(format: " <-- head");
68
69 printf(format: " },\n");
70 }
71 printf(format: " ],\n");
72}
73
74void thread_print(const struct thread *t) {
75 printf(format: "Thread %s {\n", t->name);
76 printf(format: " id: %llu,\n", (unsigned long long) t->id);
77 printf(format: " state: %s,\n", thread_state_str(atomic_load(&t->state)));
78 printf(format: " core: %lld,\n", (long long) t->curr_core);
79 printf(format: " stack_size: %zu,\n", t->stack_size);
80
81 /* priorities */
82 printf(format: " base_prio: %s,\n", thread_prio_class_str(c: t->base_prio_class));
83 printf(format: " perceived_prio: %s,\n",
84 thread_prio_class_str(c: t->perceived_prio_class));
85 printf(format: " effective_priority: %llu,\n",
86 (unsigned long long) t->effective_priority);
87 printf(format: " activity_score: %u, dynamic_delta: %d, weight: %llu,\n",
88 t->activity_score, t->dynamic_delta, (unsigned long long) t->weight);
89 printf(format: " boost_count: %u\n", t->boost_count);
90
91 printf(format: " activity_class: %s,\n",
92 thread_activity_class_str(c: t->activity_class));
93
94 /* time / slice info */
95 printf(format: " run: %llu ms / budget: %llu ms,\n",
96 (unsigned long long) t->period_runtime_raw_ms,
97 (unsigned long long) t->budget_time_raw_ms);
98 printf(format: " timeslice_length: %llu ms,\n",
99 (unsigned long long) t->timeslice_length_raw_ms);
100 printf(format: " completed_period: %llu,\n",
101 (unsigned long long) t->completed_period);
102 printf(format: " virtual_runtime: %llu / virtual_budget: %llu,\n",
103 (unsigned long long) t->virtual_period_runtime,
104 (unsigned long long) t->virtual_budget);
105
106 /* metrics overview */
107 printf(format: " activity_metrics: { run_ratio: %llu, block_ratio: %llu, "
108 "sleep_ratio: %llu, wake_freq: %llu },\n",
109 (unsigned long long) t->activity_metrics.run_ratio,
110 (unsigned long long) t->activity_metrics.block_ratio,
111 (unsigned long long) t->activity_metrics.sleep_ratio,
112 (unsigned long long) t->activity_metrics.wake_freq);
113
114 /* profiling */
115 printf(format: " context_switches: %zu,\n", t->context_switches);
116 printf(format: " preemptions: %zu,\n", t->preemptions);
117 printf(format: " wakes: %zu, blocks: %zu, sleeps: %zu,\n", t->total_wake_count,
118 t->total_block_count, t->total_sleep_count);
119 printf(format: " apcs: %zu,\n", t->total_apcs_ran);
120 printf(format: " creation_time: %lld ms,\n", (long long) t->creation_time_ms);
121
122 /* APC state */
123 printf(format: " executing_apc: %s,\n",
124 (t->flags & THREAD_FLAG_EXECUTING_APC) ? "true" : "false");
125 printf(format: " special_apc_disable: %u, kernel_apc_disable: %u,\n",
126 t->special_apc_disable, t->kernel_apc_disable);
127
128 /* activity ringbuffers */
129 if (t->activity_data) {
130 print_ringbuffer(t, /* wake_reasons = */ true, label: "wake_reasons",
131 buf: t->activity_data->wake_reasons,
132 head: t->activity_data->wake_reasons_head);
133 print_ringbuffer(t, /* wake_reasons = */ false, label: "block_reasons",
134 buf: t->activity_data->block_reasons,
135 head: t->activity_data->block_reasons_head);
136 print_ringbuffer(t, /* wake_reasons = */ false, label: "sleep_reasons",
137 buf: t->activity_data->sleep_reasons,
138 head: t->activity_data->sleep_reasons_head);
139 }
140
141 printf(format: "}\n");
142}
143
144static struct thread_event_reason *
145most_recent(struct thread_event_reason *reasons, size_t head) {
146 size_t past_head = head - 1;
147 return &reasons[past_head % THREAD_EVENT_RINGBUFFER_CAPACITY];
148}
149
150static struct thread_event_reason *
151wake_reason_associated_reason(struct thread_activity_data *data,
152 struct thread_event_reason *wake) {
153 if (thread_wake_is_from_block(wake_reason: wake->reason)) {
154 return &data->block_reasons[wake->associated_reason.reason %
155 THREAD_EVENT_RINGBUFFER_CAPACITY];
156 } else if (thread_wake_is_from_sleep(wake_reason: wake->reason)) {
157 return &data->sleep_reasons[wake->associated_reason.reason %
158 THREAD_EVENT_RINGBUFFER_CAPACITY];
159 }
160 return NULL;
161}
162
163static bool thread_event_reason_is_valid(struct thread_activity_data *data,
164 struct thread_event_reason *reason) {
165 struct thread_event_reason *assoc =
166 wake_reason_associated_reason(data, wake: reason);
167 return assoc->cycle == reason->associated_reason.cycle;
168}
169
170static bool is_block(uint8_t reason) {
171 return reason == THREAD_BLOCK_REASON_IO ||
172 reason == THREAD_BLOCK_REASON_MANUAL;
173}
174
175static bool is_sleep(uint8_t reason) {
176 return reason == THREAD_SLEEP_REASON_MANUAL;
177}
178
179static size_t get_bucket_index(time_t timestamp_ms) {
180 return (timestamp_ms / THREAD_ACTIVITY_BUCKET_DURATION) %
181 THREAD_ACTIVITY_BUCKET_COUNT;
182}
183
184static void clear_bucket(struct thread_activity_bucket *b) {
185 b->block_count = 0;
186 b->sleep_count = 0;
187 b->wake_count = 0;
188 b->block_duration = 0;
189 b->sleep_duration = 0;
190}
191
192static void clear_bucket_set_cycle(struct thread_activity_bucket *b,
193 uint64_t cycle) {
194 clear_bucket(b);
195 b->cycle = cycle;
196}
197
198static void advance_to_next_bucket(struct thread_activity_stats *stats,
199 size_t steps, time_t now) {
200 for (size_t i = 1; i <= steps && i <= THREAD_ACTIVITY_BUCKET_COUNT; i++) {
201 size_t next_bucket = stats->current_bucket + i;
202 size_t index = next_bucket % THREAD_ACTIVITY_BUCKET_COUNT;
203
204 struct thread_activity_bucket *b = &stats->buckets[index];
205
206 if (b->cycle != stats->current_cycle)
207 clear_bucket_set_cycle(b, cycle: stats->current_cycle);
208 }
209
210 size_t new_bucket = stats->current_bucket + steps;
211 stats->current_bucket = new_bucket % THREAD_ACTIVITY_BUCKET_COUNT;
212
213 if (new_bucket > stats->current_bucket)
214 stats->current_cycle++;
215
216 stats->last_update_ms = now - (now % THREAD_ACTIVITY_BUCKET_DURATION);
217}
218
219static void advance_buckets_to_time(struct thread_activity_stats *stats,
220 time_t ts) {
221 if (ts <= stats->last_update_ms)
222 return;
223
224 time_t elapsed = ts - stats->last_update_ms;
225
226 if (elapsed >= TOTAL_BUCKET_DURATION) {
227 /* Jumped past everything, reset it */
228 stats->current_cycle++;
229 stats->current_bucket = get_bucket_index(timestamp_ms: ts);
230
231 for (size_t i = 0; i < THREAD_ACTIVITY_BUCKET_COUNT; i++)
232 clear_bucket_set_cycle(b: &stats->buckets[i], cycle: stats->current_cycle);
233
234 stats->last_update_ms = ts - (ts % THREAD_ACTIVITY_BUCKET_DURATION);
235 return;
236 }
237
238 size_t steps = elapsed / THREAD_ACTIVITY_BUCKET_DURATION;
239 if (steps)
240 advance_to_next_bucket(stats, steps, now: ts);
241}
242
243static void clear_event_slot(struct thread_event_reason *slot) {
244 slot->associated_reason.reason = THREAD_ASSOCIATED_REASON_NONE;
245 slot->associated_reason.cycle = 0;
246 slot->reason = THREAD_EVENT_REASON_NONE;
247 slot->timestamp = 0;
248}
249
250static struct thread_event_reason *
251thread_add_event_reason(struct thread_event_reason *ring, uint8_t *head,
252 uint8_t reason, time_t time,
253 struct thread_activity_stats *stats) {
254
255 struct thread_event_reason *slot =
256 &ring[*head % THREAD_EVENT_RINGBUFFER_CAPACITY];
257
258 if (slot->timestamp != 0)
259 slot->cycle++;
260
261 clear_event_slot(slot);
262
263 slot->reason = reason;
264 slot->timestamp = time;
265 *head = *head + 1;
266
267 if (is_block(reason) || is_sleep(reason)) {
268 advance_buckets_to_time(stats, ts: time);
269 size_t i = get_bucket_index(timestamp_ms: time);
270 struct thread_activity_bucket *b = &stats->buckets[i];
271 if (is_block(reason))
272 b->block_count++;
273 else
274 b->sleep_count++;
275 }
276
277 return slot;
278}
279
280static inline void link_wake_reason(struct thread_event_reason *target_reason,
281 struct thread_event_reason *this_reason,
282 size_t target_link, size_t this_link) {
283 struct thread_event_association *target = &target_reason->associated_reason;
284 struct thread_event_association *asso = &this_reason->associated_reason;
285
286 target->reason = this_link % THREAD_EVENT_RINGBUFFER_CAPACITY;
287 asso->reason = target_link % THREAD_EVENT_RINGBUFFER_CAPACITY;
288
289 target->cycle = this_reason->cycle;
290 asso->cycle = target_reason->cycle;
291}
292
293static void update_bucket_data(struct thread_event_reason *wake,
294 struct thread_activity_bucket *bucket,
295 uint64_t overlap, uint32_t current_cycle) {
296 if (bucket->cycle != current_cycle)
297 clear_bucket_set_cycle(b: bucket, cycle: current_cycle);
298
299 if (thread_wake_is_from_block(wake_reason: wake->reason)) {
300 bucket->block_duration += overlap;
301 } else if (thread_wake_is_from_sleep(wake_reason: wake->reason)) {
302 bucket->sleep_duration += overlap;
303 }
304}
305
306static inline uint64_t find_overlap(time_t effective_start,
307 time_t effective_end) {
308 uint64_t diff = effective_end - effective_start;
309 return effective_end > effective_start ? diff : 0;
310}
311
312static void update_bucket(struct thread_activity_stats *stats,
313 struct thread_event_reason *wake, time_t start,
314 time_t end) {
315 time_t bucket_start = start - (start % THREAD_ACTIVITY_BUCKET_DURATION);
316 uint32_t current_cycle = stats->current_cycle;
317
318 size_t max_buckets = THREAD_ACTIVITY_BUCKET_COUNT;
319 size_t buckets_updated = 0;
320
321 while (bucket_start < end && buckets_updated < max_buckets) {
322 time_t bucket_end = bucket_start + THREAD_ACTIVITY_BUCKET_DURATION;
323 size_t bucket_index = get_bucket_index(timestamp_ms: bucket_start);
324
325 time_t effective_start = start > bucket_start ? start : bucket_start;
326 time_t effective_end = end < bucket_end ? end : bucket_end;
327 uint64_t overlap = find_overlap(effective_start, effective_end);
328
329 struct thread_activity_bucket *bucket = &stats->buckets[bucket_index];
330
331 update_bucket_data(wake, bucket, overlap, current_cycle);
332
333 bucket_start += THREAD_ACTIVITY_BUCKET_DURATION;
334 buckets_updated++;
335 }
336}
337
338void thread_update_activity_stats(struct thread *t, time_t time) {
339 struct thread_activity_stats *stats = t->activity_stats;
340 struct thread_activity_data *data = t->activity_data;
341
342 time_t now = time;
343
344 /* Advance to next bucket if a new time window has happened */
345 time_t elapsed = now - stats->last_update_ms;
346 size_t steps = elapsed / THREAD_ACTIVITY_BUCKET_DURATION;
347
348 if (steps > 0)
349 advance_to_next_bucket(stats, steps, now);
350
351 /* Gather wake event associated data */
352 size_t wake_head = data->wake_reasons_head;
353 size_t last = stats->last_wake_index;
354
355 for (size_t i = last; i < wake_head; i++) {
356 size_t idx = i % THREAD_EVENT_RINGBUFFER_CAPACITY;
357 struct thread_event_reason *wake = &data->wake_reasons[idx];
358
359 if (wake->associated_reason.reason == THREAD_ASSOCIATED_REASON_NONE)
360 continue;
361
362 struct thread_event_reason *start_evt =
363 wake_reason_associated_reason(data, wake);
364
365 bool start_evt_is_valid = thread_event_reason_is_valid(data, reason: wake);
366 if (!start_evt || !start_evt_is_valid)
367 continue;
368
369 time_t start = start_evt->timestamp;
370 time_t end = wake->timestamp;
371
372 if (start > end)
373 start = end;
374
375 update_bucket(stats, wake, start, end);
376 }
377
378 stats->last_wake_index = wake_head;
379}
380
381void thread_add_wake_reason(struct thread *t, uint8_t reason) {
382 struct thread_activity_data *d = t->activity_data;
383 time_t now = time_get_ms();
384
385 struct thread_event_reason *curr = thread_add_event_reason(
386 ring: d->wake_reasons, head: &d->wake_reasons_head, reason, time: now, stats: t->activity_stats);
387
388 advance_buckets_to_time(stats: t->activity_stats, ts: now);
389 size_t wbi = get_bucket_index(timestamp_ms: now);
390 t->activity_stats->buckets[wbi].wake_count++;
391
392 size_t this_past_head = d->wake_reasons_head - 1;
393 struct thread_event_reason *past = NULL;
394 size_t past_head = 0;
395
396 if (thread_wake_is_from_block(wake_reason: reason)) {
397 past_head = d->block_reasons_head - 1;
398 past = most_recent(reasons: d->block_reasons, head: d->block_reasons_head);
399 } else if (thread_wake_is_from_sleep(wake_reason: reason)) {
400 past_head = d->sleep_reasons_head - 1;
401 past = most_recent(reasons: d->sleep_reasons, head: d->sleep_reasons_head);
402 } else {
403 curr->associated_reason.reason = THREAD_ASSOCIATED_REASON_NONE;
404 }
405
406 if (past)
407 link_wake_reason(target_reason: past, this_reason: curr, target_link: past_head, this_link: this_past_head);
408
409 thread_update_activity_stats(t, time: now);
410
411 t->total_wake_count++;
412}
413
414void thread_update_runtime_buckets(struct thread *thread, time_t time) {
415 uint64_t now = time;
416
417 /* Which seconds does this delta span? */
418 uint64_t start_sec = thread->run_start_time / 1000;
419 uint64_t end_sec = now / 1000;
420
421 for (uint64_t sec = start_sec; sec <= end_sec; ++sec) {
422 uint64_t bucket_index = sec % THREAD_EVENT_RINGBUFFER_CAPACITY;
423
424 struct thread_runtime_bucket *bucket =
425 &thread->activity_stats->rt_buckets[bucket_index];
426
427 /* Reset it if it's for a different second */
428 if (bucket->wall_clock_sec != sec) {
429 bucket->wall_clock_sec = sec;
430 bucket->run_time_ms = 0;
431 }
432
433 /* How much of delta belongs to this second? */
434 uint64_t slice_start =
435 (sec == start_sec) ? thread->run_start_time : sec * 1000;
436 uint64_t slice_end = (sec == end_sec) ? now : (sec + 1) * 1000;
437
438 bucket->run_time_ms += slice_end - slice_start;
439 }
440
441 uint64_t new_ms = time - thread->run_start_time;
442
443 thread->period_runtime_raw_ms += new_ms;
444
445 uint64_t mult = thread->activity_score == 0 ? 1 : thread->activity_score;
446
447 thread->virtual_period_runtime += new_ms * mult;
448}
449
450void thread_add_block_reason(struct thread *t, uint8_t reason) {
451 struct thread_activity_data *d = t->activity_data;
452 t->total_block_count++;
453 thread_add_event_reason(ring: d->block_reasons, head: &d->block_reasons_head, reason,
454 time: time_get_ms(), stats: t->activity_stats);
455}
456
457void thread_add_sleep_reason(struct thread *t, uint8_t reason) {
458 struct thread_activity_data *d = t->activity_data;
459 t->total_sleep_count++;
460 thread_add_event_reason(ring: d->sleep_reasons, head: &d->sleep_reasons_head, reason,
461 time: time_get_ms(), stats: t->activity_stats);
462}
463
464static bool set_state_and_update_reason(
465 struct thread *t, uint8_t reason, enum thread_state state,
466 void (*callback)(struct thread *, uint8_t), void *wake_src,
467 bool already_locked, enum thread_wait_type type, bool exit_if_match) {
468 /* do not preempt us. this is raised to HIGH because it is sometimes
469 * called from HIGH and we can't "raise from HIGH to DISPATCH" */
470 enum irql irql = IRQL_NONE;
471 bool aok = true;
472
473 if (!already_locked)
474 irql = thread_acquire(t, success: &aok);
475 else
476 SPINLOCK_ASSERT_HELD(&t->lock);
477
478 kassert(aok);
479
480 bool ok = false;
481 if (exit_if_match) {
482 if ((thread_get_flags(t) & THREAD_FLAG_WAKE_MATCHED) &&
483 t->wake_token == t->wait_token) {
484 ok = true;
485 goto out;
486 }
487 }
488
489 if (state == THREAD_STATE_READY) {
490 atomic_store_explicit(&t->wake_src, wake_src, memory_order_release);
491 if (wake_src == t->expected_wake_src) {
492 t->wake_token = t->wait_token;
493 thread_or_flags(t, flags: THREAD_FLAG_WAKE_MATCHED);
494 }
495 } else {
496 thread_and_flags(t, flags: ~(THREAD_FLAG_YIELDED | THREAD_FLAG_WAKE_MATCHED));
497 atomic_store_explicit(&t->wait_type, type, memory_order_release);
498 t->last_action_reason = reason;
499 t->last_action = state;
500 t->wait_token = ++t->token_ctr;
501 t->wake_token = 0;
502 t->expected_wake_src = wake_src;
503 }
504
505 /* only change the state if it is NOT both RUNNING and being set to READY */
506 if (!(state == THREAD_STATE_READY &&
507 thread_get_state(t) == THREAD_STATE_RUNNING))
508 atomic_store(&t->state, state);
509
510 /* NOTE: special case: the thread has not re-entered runqueues yet */
511 if ((!(thread_get_flags(t) & THREAD_FLAG_YIELDED)) &&
512 state == THREAD_STATE_READY)
513 atomic_store(&t->state, THREAD_STATE_RUNNING);
514
515 callback(t, reason);
516
517 time_t time = time_get_ms();
518
519 if (state != THREAD_STATE_READY)
520 thread_update_runtime_buckets(thread: t, time);
521
522out:
523 if (!already_locked)
524 thread_release(t, irql);
525
526 return ok;
527}
528
529void thread_wake_unlocked(struct thread *t, enum thread_wake_reason r,
530 void *wake_src) {
531 set_state_and_update_reason(
532 t, reason: r, state: THREAD_STATE_READY, callback: thread_add_wake_reason, wake_src,
533 /*already_locked=*/false,
534 /* set this to 0 because it is no-op on wake */ type: 0,
535 /* exit_if_match = */ false);
536}
537
538void thread_prepare_to_wake_locked(struct thread *t, enum thread_wake_reason r,
539 void *wake_src) {
540 set_state_and_update_reason(
541 t, reason: r, state: THREAD_STATE_READY, callback: thread_add_wake_reason, wake_src,
542 /*already_locked=*/true, /* type = */ 0, /* exit_if_match = */ false);
543}
544
545void thread_set_timesharing(struct thread *t) {
546 t->base_prio_class = THREAD_PRIO_CLASS_TIMESHARE;
547 t->perceived_prio_class = THREAD_PRIO_CLASS_TIMESHARE;
548}
549
550void thread_set_background(struct thread *t) {
551 t->base_prio_class = THREAD_PRIO_CLASS_BACKGROUND;
552 t->perceived_prio_class = THREAD_PRIO_CLASS_BACKGROUND;
553}
554
555void thread_prepare_to_block(struct thread *t, enum thread_block_reason r,
556 enum thread_wait_type type,
557 void *expect_wake_src) {
558 set_state_and_update_reason(
559 t, reason: r, state: THREAD_STATE_BLOCKED, callback: thread_add_block_reason, wake_src: expect_wake_src,
560 /*already_locked=*/false, type, /* exit_if_match = */ false);
561}
562
563void thread_prepare_to_block_locked(struct thread *t,
564 enum thread_block_reason r,
565 enum thread_wait_type type,
566 void *expect_wake_src) {
567 set_state_and_update_reason(
568 t, reason: r, state: THREAD_STATE_BLOCKED, callback: thread_add_block_reason, wake_src: expect_wake_src,
569 /*already_locked=*/true, type, /* exit_if_match = */ false);
570}
571
572void thread_prepare_to_sleep(struct thread *t, enum thread_sleep_reason r,
573 enum thread_wait_type type,
574 void *expect_wake_src) {
575 set_state_and_update_reason(
576 t, reason: r, state: THREAD_STATE_SLEEPING, callback: thread_add_sleep_reason, wake_src: expect_wake_src,
577 /*already_locked=*/false, type, /* exit_if_match = */ false);
578}
579
580static bool block_interruptible(struct thread *t, enum thread_block_reason r,
581 enum thread_wait_type type,
582 void *expect_wake_src) {
583 return set_state_and_update_reason(
584 t, reason: r, state: THREAD_STATE_BLOCKED, callback: thread_add_block_reason, wake_src: expect_wake_src,
585 /*already_locked=*/false, type, /* exit_if_match = */ true);
586}
587
588static bool sleep_interruptible(struct thread *t, enum thread_block_reason r,
589 enum thread_wait_type type,
590 void *expect_wake_src) {
591 return set_state_and_update_reason(
592 t, reason: r, state: THREAD_STATE_SLEEPING, callback: thread_add_sleep_reason, wake_src: expect_wake_src,
593 /*already_locked=*/false, type, /* exit_if_match = */ true);
594}
595
596void thread_yield_until_wake_match() {
597 struct thread *curr = thread_get_current();
598
599 /* If our wake has matched we take the fast path down
600 * and out of the function */
601 if (!(thread_get_flags(t: curr) & THREAD_FLAG_WAKE_MATCHED))
602 scheduler_yield();
603
604 if (curr->last_action != THREAD_STATE_BLOCKED &&
605 curr->last_action != THREAD_STATE_SLEEPING)
606 return;
607
608 /* we can safely avoid checking the token in the loop because that will
609 * become set if wake_matched is set... */
610 while (!(thread_get_flags(t: curr) & THREAD_FLAG_WAKE_MATCHED)) {
611 if (curr->last_action != THREAD_STATE_BLOCKED &&
612 curr->last_action != THREAD_STATE_SLEEPING)
613 panic("uh oh");
614
615 if (curr->last_action == THREAD_STATE_BLOCKED) {
616 if (block_interruptible(t: curr, r: curr->last_action_reason,
617 type: curr->wait_type, expect_wake_src: curr->expected_wake_src))
618 break;
619 } else {
620 if (sleep_interruptible(t: curr, r: curr->last_action_reason,
621 type: curr->wait_type, expect_wake_src: curr->expected_wake_src))
622 break;
623 }
624
625 scheduler_yield();
626 }
627
628 thread_clear_wake_data(t: curr);
629}
630