1/* @title: Threads */
2/* File defines thread structures and public APIs
3 * for boost and event recording + scoring */
4
5#pragma once
6#include <asm.h>
7#include <compiler.h>
8#include <log.h>
9#include <mem/page.h>
10#include <sch/climb.h>
11#include <sch/rt_sched_types.h>
12#include <stdarg.h>
13#include <stdatomic.h>
14#include <stddef.h>
15#include <stdint.h>
16#include <structures/list.h>
17#include <structures/pairing_heap.h>
18#include <structures/rbt.h>
19#include <sync/condvar.h>
20#include <sync/lock_chk_types.h>
21#include <sync/rcu.h>
22#include <sync/spinlock.h>
23#include <thread/apc_types.h>
24#include <thread/thread_types.h>
25#include <time/time.h>
26#include <types/refcount.h>
27#include <types/types.h>
28
29#define THREAD_DEFAULT_TIMESLICE 15 /* 15 ms */
30
31#define THREAD_CLASS_WIDTH 1024
32#define THREAD_CLASS_HALF (THREAD_CLASS_WIDTH / 2)
33
34#define THREAD_BAND_MIN(avg) ((avg) - THREAD_CLASS_HALF)
35#define THREAD_BAND_MAX(avg) ((avg) + THREAD_CLASS_HALF)
36
37#define THREAD_ACT_INTERACTIVE_AVG 4000u
38#define THREAD_ACT_IO_BOUND_AVG 2500u
39#define THREAD_ACT_CPU_BOUND_AVG 1200u
40#define THREAD_ACT_SLEEPY_AVG 4500u
41
42#define THREAD_ACT_INTERACTIVE_MIN THREAD_BAND_MIN(THREAD_ACT_INTERACTIVE_AVG)
43#define THREAD_ACT_INTERACTIVE_MAX THREAD_BAND_MAX(THREAD_ACT_INTERACTIVE_AVG)
44
45#define THREAD_ACT_IO_BOUND_MIN THREAD_BAND_MIN(THREAD_ACT_IO_BOUND_AVG)
46#define THREAD_ACT_IO_BOUND_MAX THREAD_BAND_MAX(THREAD_ACT_IO_BOUND_AVG)
47
48#define THREAD_ACT_CPU_BOUND_MIN THREAD_BAND_MIN(THREAD_ACT_CPU_BOUND_AVG)
49#define THREAD_ACT_CPU_BOUND_MAX THREAD_BAND_MAX(THREAD_ACT_CPU_BOUND_AVG)
50
51#define THREAD_ACT_SLEEPY_MIN THREAD_BAND_MIN(THREAD_ACT_SLEEPY_AVG)
52#define THREAD_ACT_SLEEPY_MAX THREAD_BAND_MAX(THREAD_ACT_SLEEPY_AVG)
53
54#define THREAD_NICENESS_VALID(n) \
55 ((((nice_t) (n)) >= -19) && (((nice_t) (n)) <= 20))
56
57/* pluh */
58struct cpu_context {
59 uint64_t rbx;
60 uint64_t rbp;
61 uint64_t r12;
62 uint64_t r13;
63 uint64_t r14;
64 uint64_t r15;
65 uint64_t rsp;
66 uint64_t rip;
67};
68
69#define THREAD_EVENT_REASON_NONE 0xFF
70
71struct thread_event_association {
72 uint8_t reason;
73 uint64_t cycle; /* Cycle for the associated reason */
74};
75
76#define THREAD_ASSOCIATED_REASON_NONE 0xFF
77struct thread_event_reason {
78 uint8_t reason;
79 struct thread_event_association associated_reason;
80 time_ms_t timestamp;
81 uint64_t cycle;
82};
83
84#define THREAD_PRIO_IS_TIMESHARING(prio) (prio == THREAD_PRIO_CLASS_TIMESHARE)
85
86/* Background threads share timeslices */
87#define THREAD_PRIO_HAS_TIMESLICE(prio) \
88 (THREAD_PRIO_IS_TIMESHARING(prio) || prio == THREAD_PRIO_CLASS_BACKGROUND)
89
90#define THREAD_ACTIVITY_BUCKET_COUNT 4
91#define THREAD_ACTIVITY_BUCKET_DURATION 1000 /* 1 second per bucket */
92
93static_assert(THREAD_ACTIVITY_BUCKET_COUNT < UINT16_MAX,
94 "Thread activity bucket granularity too large for a u16");
95
96#define THREAD_EVENT_RINGBUFFER_CAPACITY THREAD_ACTIVITY_BUCKET_COUNT
97#define TOTAL_BUCKET_DURATION \
98 (THREAD_ACTIVITY_BUCKET_COUNT * THREAD_ACTIVITY_BUCKET_DURATION)
99
100/* Buckets */
101struct thread_runtime_bucket {
102 uint16_t run_time_ms; /* can safely do a u16 since 2^16 > 1000 */
103 uint64_t wall_clock_sec;
104};
105
106struct thread_activity_bucket {
107 uint64_t cycle;
108
109 /* please do not block/sleep/wake more than 2^32 times a second */
110 uint32_t block_count;
111 uint32_t sleep_count;
112 uint32_t wake_count;
113
114 uint16_t block_duration;
115 uint16_t sleep_duration;
116};
117
118/* Fine grained, exact activity stats */
119struct thread_activity_stats {
120 struct thread_runtime_bucket rt_buckets[THREAD_ACTIVITY_BUCKET_COUNT];
121 struct thread_activity_bucket buckets[THREAD_ACTIVITY_BUCKET_COUNT];
122 time_ms_t last_update_ms;
123 uint64_t current_cycle;
124 uint8_t current_bucket; /* idx of bucket representing 'now' */
125 uint8_t last_wake_index;
126};
127
128#define MAKE_THREAD_RINGBUFFER(name) \
129 struct thread_event_reason name[THREAD_EVENT_RINGBUFFER_CAPACITY]; \
130 uint8_t name##_head;
131
132struct thread_activity_data {
133 MAKE_THREAD_RINGBUFFER(wake_reasons);
134 MAKE_THREAD_RINGBUFFER(block_reasons);
135 MAKE_THREAD_RINGBUFFER(sleep_reasons);
136};
137
138/* Activity aggregations */
139enum thread_activity_class {
140 THREAD_ACTIVITY_CLASS_CPU_BOUND,
141 THREAD_ACTIVITY_CLASS_IO_BOUND,
142 THREAD_ACTIVITY_CLASS_INTERACTIVE,
143 THREAD_ACTIVITY_CLASS_SLEEPY,
144 THREAD_ACTIVITY_CLASS_UNKNOWN
145};
146
147struct thread_activity_metrics {
148 uint8_t run_ratio;
149 uint8_t block_ratio;
150 uint8_t sleep_ratio;
151 uint8_t wake_freq;
152};
153
154struct thread {
155 /* ========== Metadata ========== */
156 /* Unique ID allocated from global thread ID tree */
157 thread_id_t id;
158 char *name;
159 void (*entry)(void *); /* For debug */
160
161 /* ========== Processor context data ========== */
162
163 /* Stack */
164 void *stack;
165 size_t stack_size;
166
167 /* TODO: we should isolate this into the debug nesting
168 * mode, as this helps us identify scheduler_yield() nesting */
169 uint32_t yield_nesting;
170
171 /* Preserves the highest point yield_nesting ever reached */
172 uint32_t yield_nesting_max;
173
174 /* Registers */
175 struct cpu_context regs;
176
177 /* ========== Transparent structure nodes ========== */
178
179 /* TODO: Use unions and combine these. Do make sure to keep
180 * in mind that all nodes get need to get
181 * reset if unions are to be used */
182
183 struct crash_perthread crash_data;
184 struct list_head reaper_list; /* reaper list */
185 struct list_head thread_list; /* global list of threads */
186
187 /* Runqueue nodes */
188
189 union {
190 struct rbt_node rq_tree_node; /* runqueue tree node */
191 struct rbt_node rt_tree_node; /* alias for rt scheduling */
192 };
193
194 union {
195 struct list_head rq_list_node; /* runqueue list node */
196 struct list_head rt_list_node; /* alias for rt scheduling */
197 };
198
199 /* Waitqueue nodes */
200 struct rbt_node wq_tree_node; /* waitqueue tree node */
201 struct list_head wq_list_node; /* waitqueue list node */
202 struct pairing_node wq_pairing_node; /* waitqueue pairing node */
203
204 struct list_head rcu_list_node; /* blocked reader node on RCU leaf */
205
206 /* ========== State ========== */
207
208 /* State */
209 _Atomic enum thread_state state;
210
211 /* Who is running us? */
212 cpu_id_t curr_core; /* -1 if not being ran */
213
214 cpu_id_t core_to_wake_on; /* When I run again, where should I be placed?
215 * -1 if the scheduler should select the most
216 * optimal core */
217
218 _Atomic(struct scheduler *) scheduler;
219
220 time_ms_t run_start_time; /* When did we start running */
221
222 /* Who is allowed to run us? */
223 struct cpu_mask allowed_cpus;
224 _Atomic int64_t migrate_to; /* -1 if no migration target */
225
226 /* Flags */
227 enum rt_scheduler_capability accepted_rt_caps;
228 _Atomic enum thread_flags flags;
229 _Atomic size_t migration_generation;
230
231 /* ======== Raw priority + timeslice data ======== */
232
233 /* Priorities */
234 thread_prio_t activity_score;
235 int32_t dynamic_delta; /* Signed delta applied to base */
236 size_t weight;
237 nice_t niceness; /* -20 .. + 19 */
238
239 cpu_perf_t wanted_perf; /* This is the cpu_perf_t the thread's current
240 * CPU should try to match or do better than.
241 *
242 * If the current CPU's current wanted_perf
243 * does not satisfy the thread, we will try
244 * to either increase it for this CPU, or
245 * migrate the thread to another CPU. */
246
247 /* Class changes */
248 time_ms_t last_class_change_ms;
249
250 size_t effective_priority;
251
252 /* Timeslice info and periods */
253 uint64_t completed_period;
254 time_ms_t period_runtime_raw_ms; /* Raw MS time of runtime this period */
255 time_ms_t budget_time_raw_ms; /* Raw MS time of budget */
256 time_ms_t timeslice_length_raw_ms;
257
258 uint32_t virtual_period_runtime;
259 uint32_t virtual_budget;
260 uint32_t virtual_runtime_left;
261
262 /* ========== Thread activity stats ========== */
263
264 enum thread_activity_class activity_class;
265
266 enum thread_prio_class base_prio_class; /* for class boosts */
267 enum thread_prio_class perceived_prio_class;
268
269 /* Activity data */
270 struct thread_activity_data *activity_data;
271 struct thread_activity_stats *activity_stats;
272
273 /* "Overview" derived from data and stats */
274 struct thread_activity_metrics activity_metrics;
275
276 /* ========== Synchronization data ========== */
277
278 /* Lock + rc */
279 struct spinlock lock;
280 refcount_t refcount;
281
282 /* Join */
283 struct spinlock join_lock; /* guards exit_status + the ZOMBIE publish */
284 struct condvar join_cv; /* joiners wait here */
285 int exit_status;
286
287 /* For condvar */
288 volatile enum wake_reason wake_reason;
289 size_t wait_cookie;
290
291 /* RCU stuff:
292 *
293 * rcu_nesting and rcu_read_seq are written ONLY by this thread
294 *
295 * rcu_leaf, rcu_blocked_seq, and rcu_list_node are how readers
296 * that get preempted mid-section get registered in RCU,
297 *
298 * those are written under the leaf's lock,
299 * either by us or the CPU switching us out
300 */
301 _Atomic uint32_t rcu_nesting; /* read section depth */
302 _Atomic uint64_t rcu_read_seq; /* GP sequence at the outermost lock */
303 struct rcu_node *rcu_leaf; /* leaf we are registered on or NULL */
304 uint64_t rcu_blocked_seq; /* GP we are counted against or 0 */
305 struct rcu_cb free_rcu;
306
307 /* Block/sleep and wake sync. */
308 _Atomic enum thread_wait_type wait_type;
309 void *expected_wake_src;
310 uint64_t wait_token;
311
312 uint8_t last_action_reason;
313
314 /* used in wait_for_wake */
315 enum thread_state last_action;
316
317 _Atomic(void *) wake_src;
318 uint64_t wake_token;
319
320 uint64_t token_ctr;
321
322 struct condvar_with_cb cv_cb_object; /* wait object */
323 struct list_head io_wait_tokens; /* list of tokens */
324
325 struct turnstile *turnstile; /* my turnstile */
326 _Atomic(struct turnstile *) blocked_ts; /* what am I blocked on */
327
328 struct climb_thread_state climb_state;
329
330#ifdef DEBUG_LOCK_CHK
331
332 struct lock_chk_thread_data lock_chk;
333
334#endif
335
336 /* ========== APC data ========== */
337 /* Standard APC queues */
338 struct apc_queue apc_head[APC_TYPE_COUNT];
339
340 /* Any APC pending */
341 _Atomic uint8_t apc_pending_mask; /* bitmask of APC_TYPE_* pending */
342
343 /* APC disable counts */
344 uint32_t special_apc_disable;
345 uint32_t kernel_apc_disable;
346
347 struct apc_queue event_apcs; /* yet to execute */
348 struct apc_queue to_exec_event_apcs; /* to be executed */
349
350 /* ========== Profiling data ========== */
351 struct log_site *log_site;
352 struct log_handle log_handle;
353 size_t context_switches; /* Total context switches */
354
355 size_t preemptions;
356
357 time_ms_t creation_time_ms; /* When were we created? */
358
359 uint32_t boost_count;
360 uint32_t total_wake_count; /* Aggregate count of all wake events */
361 uint32_t total_block_count; /* Aggregate count of all block events */
362 uint32_t total_sleep_count; /* Aggregate count of all sleep events */
363 uint32_t total_apcs_ran; /* Total APCs executed on a given thread */
364
365 /* Misc. private field for whatever needs it */
366 void *private;
367};
368
369#define thread_from_rq_rbt_node(node) \
370 rbt_entry(node, struct thread, rq_tree_node)
371#define thread_from_rq_list_node(ln) \
372 (container_of(ln, struct thread, rq_list_node))
373
374#define thread_from_rcu_list_node(ln) \
375 (container_of(ln, struct thread, rcu_list_node))
376
377#define thread_from_wq_pairing_node(pn) \
378 (container_of(pn, struct thread, wq_pairing_node))
379#define thread_from_wq_list_node(ln) \
380 (container_of(ln, struct thread, wq_list_node))
381#define thread_from_wq_rbt_node(ln) \
382 (container_of(ln, struct thread, wq_tree_node))
383
384#define thread_log(lvl, fmt, ...) \
385 ({ \
386 if (global.current_bootstage >= BOOTSTAGE_LATE) \
387 log(thread_get_current()->log_site, \
388 &thread_get_current()->log_handle, lvl, fmt, ##__VA_ARGS__); \
389 })
390
391#define thread_err(fmt, ...) thread_log(LOG_ERROR, fmt, ##__VA_ARGS__)
392#define thread_warn(fmt, ...) thread_log(LOG_WARN, fmt, ##__VA_ARGS__)
393#define thread_info(fmt, ...) thread_log(LOG_INFO, fmt, ##__VA_ARGS__)
394#define thread_debug(fmt, ...) thread_log(LOG_DEBUG, fmt, ##__VA_ARGS__)
395#define thread_trace(fmt, ...) thread_log(LOG_TRACE, fmt, ##__VA_ARGS__)
396
397struct thread *thread_create_internal(char *name, void (*entry_point)(void *),
398 void *arg, size_t stack_size,
399 va_list args);
400
401struct thread *thread_create(char *name, void (*entry_point)(void *), void *arg,
402 ...);
403
404struct thread *thread_create_custom_stack(char *name,
405 void (*entry_point)(void *),
406 void *arg, size_t stack_size, ...);
407void thread_free(struct thread *t);
408
409void thread_init_thread_ids(void);
410void thread_sleep_for_ms(uint64_t ms);
411void thread_sleep_for_us(uint64_t us);
412void thread_exit(void);
413void thread_print(const struct thread *t);
414
415void thread_update_activity_stats(struct thread *t, time_ms_t time);
416void thread_classify_activity(struct thread *t, uint64_t now_ms);
417void thread_update_runtime_buckets(struct thread *thread, time_ms_t time);
418void thread_apply_wake_boost(struct thread *t);
419void thread_update_effective_priority(struct thread *t);
420void thread_apply_cpu_penalty(struct thread *t);
421
422void thread_add_wake_reason(struct thread *t, uint8_t reason);
423void scheduler_wake_manual(struct thread *t, void *wake_src);
424void thread_calculate_activity_data(struct thread *t);
425
426void thread_add_block_reason(struct thread *t, uint8_t reason);
427void thread_add_sleep_reason(struct thread *t, uint8_t reason);
428
429void thread_prepare_to_wait(struct thread *t, enum thread_state state,
430 uint8_t reason, enum thread_wait_type type,
431 void *expect_wake_src);
432void thread_prepare_to_wait_locked(struct thread *t, enum thread_state state,
433 uint8_t reason, enum thread_wait_type type,
434 void *expect_wake_src);
435bool thread_rearm_wait(struct thread *t);
436enum thread_wait_status thread_wait_yield(void);
437void thread_yield_until_wake_match(void);
438enum thread_wait_status thread_yield_interruptible(void);
439enum thread_wait_status thread_yield_arbitrary(enum thread_wait_type type);
440
441void thread_prepare_to_block(struct thread *t, enum thread_block_reason r,
442 enum thread_wait_type wait_type,
443 void *expect_wake_src);
444void thread_prepare_to_sleep(struct thread *t, enum thread_sleep_reason r,
445 enum thread_wait_type wait_type,
446 void *expect_wake_src);
447
448/* Turnstile wants this */
449void thread_prepare_to_block_locked(struct thread *t,
450 enum thread_block_reason r,
451 enum thread_wait_type type,
452 void *expect_wake_src);
453
454void thread_set_timesharing(struct thread *t);
455void thread_set_background(struct thread *t);
456void thread_wake_unlocked(struct thread *t, enum thread_wake_reason r,
457 void *wake_src);
458void thread_migrate(struct thread *t, size_t dest_core);
459enum thread_prio_class thread_unboost_self();
460enum thread_prio_class thread_boost_self(enum thread_prio_class new);
461struct scheduler *thread_get_scheduler(struct thread *t, enum irql *sirql_out);
462
463struct thread_queue;
464void thread_block_on(struct thread_queue *q, enum thread_wait_type type,
465 void *wake_src);
466
467void thread_enqueue(struct thread *t);
468void thread_enqueue_on_core(struct thread *t, uint64_t core_id);
469
470bool thread_wake(struct thread *t, enum thread_wake_reason reason,
471 enum thread_prio_class prio, void *wake_src);
472void thread_wake_from_io_block(struct thread *t, void *wake_src);
473bool thread_inherit_priority(struct thread *boosted, struct thread *from,
474 enum thread_prio_class *old_class_out);
475
476void thread_uninherit_priority(enum thread_prio_class class);
477void thread_remove_boost();
478
479void thread_exit_with_status(int status);
480
481int thread_join(struct thread *t);
482bool thread_join_timeout(struct thread *t, time_ms_t timeout_ms,
483 int *status_out);
484void thread_detach(struct thread *t);
485
486void thread_lock_two_runqueues(struct thread *a, struct thread *b,
487 struct scheduler **out_rq_a,
488 struct scheduler **out_rq_b, enum irql *irq_a,
489 enum irql *irq_b);
490
491void thread_lock_thread_and_rq(struct thread *t, struct scheduler *other_rq,
492 struct scheduler **out_thread_rq,
493 enum irql *irq_first, enum irql *irq_second);
494void thread_unlock_thread_and_rq(struct scheduler *thread_rq,
495 struct scheduler *other_rq,
496 enum irql irq_first, enum irql irq_second);
497
498bool thread_in_context(void);
499REFCOUNT_GENERATE_GET_FOR_STRUCT_WITH_FAILURE_COND(thread, refcount, flags,
500 &THREAD_FLAG_DYING);
501void reaper_enqueue(struct thread *t);
502void thread_put(struct thread *t);
503
504/* Yield nesting work:
505 *
506 * scheduler_yield() can re-enter, and we're using this to debug,
507 * with a counter bumped per thread. The max nesting depth today
508 * is 1, and it will likely stay this way, but I'm keeping
509 * it a tunable counter in case this changes. */
510#ifndef SCHED_MAX_YIELD_NESTING
511#define SCHED_MAX_YIELD_NESTING 1
512#endif
513
514static inline uint32_t scheduler_yield_nesting(struct thread *t) {
515 return t ? t->yield_nesting : 0;
516}
517
518static inline void scheduler_yield_nesting_enter(struct thread *t) {
519 if (!t)
520 return;
521
522 t->yield_nesting++;
523 if (t->yield_nesting > t->yield_nesting_max)
524 t->yield_nesting_max = t->yield_nesting;
525#ifdef DEBUG_SCHED_NESTING
526 kassert(t->yield_nesting <= SCHED_MAX_YIELD_NESTING,
527 "scheduler_yield nested %u deep on thread '%s'", t->yield_nesting,
528 t->name);
529#endif
530}
531
532static inline uint32_t scheduler_yield_nesting_max(struct thread *t) {
533 return t ? t->yield_nesting_max : 0;
534}
535
536static inline void scheduler_yield_nesting_exit(struct thread *t) {
537 if (!t)
538 return;
539
540#ifdef DEBUG_SCHED_NESTING
541 kassert(t->yield_nesting > 0,
542 "scheduler_yield nesting underflow on thread '%s'", t->name);
543#endif
544
545 if (t->yield_nesting)
546 t->yield_nesting--;
547}
548
549static inline void scheduler_yield_nesting_reset(struct thread *t) {
550 if (t)
551 t->yield_nesting = 0;
552}
553
554static inline struct thread *thread_get_current(void) {
555 uintptr_t thread;
556 asm volatile("movq %%gs:%c1, %0"
557 : "=r"(thread)
558 : "i"(offsetof(struct core, current_thread)));
559 return (struct thread *) thread;
560}
561
562static inline int64_t thread_set_migration_target(struct thread *t,
563 int64_t new) {
564 return atomic_exchange(&t->migrate_to, new);
565}
566
567static inline enum thread_state thread_get_state(struct thread *t) {
568 return atomic_load(&t->state);
569}
570
571static inline void thread_set_state(struct thread *t, enum thread_state state) {
572 atomic_store(&t->state, state);
573}
574
575static inline enum thread_flags thread_get_flags(struct thread *t) {
576 return atomic_load(&t->flags);
577}
578
579static inline void thread_set_flags(struct thread *t, enum thread_flags new) {
580 atomic_store(&t->flags, new);
581}
582
583static inline bool thread_pin(struct thread *t) {
584 return atomic_fetch_or(&t->flags, THREAD_FLAG_PINNED) & THREAD_FLAG_PINNED;
585}
586
587static inline void thread_unpin(struct thread *t) {
588 atomic_fetch_and(&t->flags, ~THREAD_FLAG_PINNED);
589}
590
591static inline enum thread_flags thread_or_flags(struct thread *t,
592 enum thread_flags flags) {
593 return atomic_fetch_or(&t->flags, flags);
594}
595
596static inline enum thread_flags thread_and_flags(struct thread *t,
597 enum thread_flags flags) {
598 return atomic_fetch_and(&t->flags, flags);
599}
600
601static inline size_t thread_get_migration_generation(struct thread *t) {
602 return atomic_load_explicit(&t->migration_generation, memory_order_acquire);
603}
604
605static inline struct scheduler *thread_get_scheduler_unsafe(struct thread *t) {
606 return atomic_load_explicit(&t->scheduler, memory_order_acquire);
607}
608
609static inline void thread_set_runqueue(struct thread *t, struct scheduler *s) {
610 atomic_fetch_add_explicit(&t->migration_generation, 1,
611 memory_order_release);
612 atomic_store_explicit(&t->scheduler, s, memory_order_release);
613 atomic_fetch_add_explicit(&t->migration_generation, 1,
614 memory_order_release);
615}
616
617/* RCU keeps the thread memory allocated so we inc_not_zero here
618 * and everything is fine and dandy, everything must be in a
619 * read side critical section, though */
620static inline bool thread_get_rcu(struct thread *t) {
621 if (!refcount_inc_not_zero(rc: &t->refcount))
622 return false;
623
624 if (thread_get_flags(t) & THREAD_FLAG_DYING) {
625 thread_put(t);
626 return false;
627 }
628
629 return true;
630}
631
632static inline enum irql thread_acquire(struct thread *t, bool *success) {
633 if (!thread_get(obj: t)) {
634 if (success)
635 *success = false;
636 return IRQL_NONE;
637 }
638
639 if (success)
640 *success = true;
641 return spin_lock_irq_disable(&t->lock);
642}
643
644static inline void thread_release(struct thread *t, enum irql irql) {
645 spin_unlock(&t->lock, irql);
646 thread_put(t);
647}
648
649static inline bool thread_is_rt(struct thread *t) {
650 return t->perceived_prio_class == THREAD_PRIO_CLASS_URGENT ||
651 t->perceived_prio_class == THREAD_PRIO_CLASS_RT;
652}
653
654static inline void thread_clear_wake_data_raw(struct thread *t) {
655 atomic_store_explicit(&t->wake_src, NULL, memory_order_release);
656 thread_and_flags(t, flags: ~THREAD_FLAG_WAKE_MATCHED);
657 t->expected_wake_src = NULL;
658 t->wait_type = THREAD_WAIT_NONE;
659 t->last_action_reason = 0;
660 t->last_action = THREAD_STATE_READY;
661 t->wake_token = 0;
662}
663
664static inline void thread_finish_wait_raw(struct thread *t) {
665 enum thread_state s = thread_get_state(t);
666 if (s == THREAD_STATE_BLOCKED || s == THREAD_STATE_SLEEPING)
667 thread_set_state(t, state: THREAD_STATE_RUNNING);
668
669 thread_clear_wake_data_raw(t);
670}
671
672static inline void thread_finish_wait_locked(struct thread *t) {
673 thread_finish_wait_raw(t);
674}
675
676static inline void thread_finish_wait(struct thread *t) {
677 bool aok;
678 enum irql irql = thread_acquire(t, success: &aok);
679 kassert(aok);
680
681 thread_finish_wait_locked(t);
682
683 thread_release(t, irql);
684}
685
686static inline void thread_clear_wake_data(struct thread *t) {
687 thread_finish_wait(t);
688}
689
690static inline enum thread_wait_type thread_get_wait_type(struct thread *t) {
691 return atomic_load_explicit(&t->wait_type, memory_order_acquire);
692}
693
694static inline struct thread *thread_spawn(char *name, void (*entry)(void *),
695 void *arg, ...) {
696 va_list args;
697 va_start(args, arg);
698 struct thread *t =
699 thread_create_internal(name, entry_point: entry, arg, THREAD_STACK_SIZE, args);
700 va_end(args);
701 thread_enqueue(t);
702 return t;
703}
704
705static inline struct thread *thread_spawn_custom_stack(char *name,
706 void (*entry)(void *),
707 void *arg,
708 size_t stack_size, ...) {
709 va_list args;
710 va_start(args, stack_size);
711 struct thread *t =
712 thread_create_internal(name, entry_point: entry, arg, stack_size, args);
713 va_end(args);
714
715 thread_enqueue(t);
716 return t;
717}
718
719static inline struct thread *thread_spawn_on_core(char *name,
720 void (*entry)(void *),
721 void *arg, uint64_t core_id,
722 ...) {
723 va_list args;
724 va_start(args, core_id);
725 struct thread *t =
726 thread_create_internal(name, entry_point: entry, arg, THREAD_STACK_SIZE, args);
727 va_end(args);
728
729 thread_enqueue_on_core(t, core_id);
730 return t;
731}
732
733/* Must be called before the thread can run: taking the join reference
734 * relies on THREAD_FLAG_DYING not being observable yet. */
735static inline void thread_set_joinable(struct thread *t) {
736 kassert(!(thread_get_flags(t) & THREAD_FLAG_JOINABLE));
737 kassert(refcount_inc(&t->refcount));
738 thread_or_flags(t, flags: THREAD_FLAG_JOINABLE);
739}
740
741static inline struct thread *
742thread_spawn_joinable(char *name, void (*entry)(void *), void *arg, ...) {
743 va_list args;
744 va_start(args, arg);
745 struct thread *t =
746 thread_create_internal(name, entry_point: entry, arg, THREAD_STACK_SIZE, args);
747 va_end(args);
748
749 if (unlikely(!t))
750 return NULL;
751
752 thread_set_joinable(t);
753 thread_enqueue(t);
754 return t;
755}
756
757static inline struct thread *
758thread_spawn_joinable_custom_stack(char *name, void (*entry)(void *), void *arg,
759 size_t stack_size, ...) {
760 va_list args;
761 va_start(args, stack_size);
762 struct thread *t =
763 thread_create_internal(name, entry_point: entry, arg, stack_size, args);
764 va_end(args);
765
766 if (unlikely(!t))
767 return NULL;
768
769 thread_set_joinable(t);
770 thread_enqueue(t);
771 return t;
772}
773
774static inline struct thread *
775thread_spawn_joinable_on_core(char *name, void (*entry)(void *), void *arg,
776 uint64_t core_id, ...) {
777 va_list args;
778 va_start(args, core_id);
779 struct thread *t =
780 thread_create_internal(name, entry_point: entry, arg, THREAD_STACK_SIZE, args);
781 va_end(args);
782
783 if (unlikely(!t))
784 return NULL;
785
786 thread_set_joinable(t);
787 thread_enqueue_on_core(t, core_id);
788 return t;
789}
790