1/* @title: Threads */
2/* File defines thread structures and public APIs
3 * for boost and event recording + scoring */
4
5#pragma once
6#include <asm.h>
7#include <atomic.h>
8#include <compiler/core.h>
9#include <log.h>
10#include <math/range.h>
11#include <mem/page.h>
12#include <mem/tlb.h> /* TODO: This header has little business being in here,
13 * we should consider moving it out of this */
14#include <sch/climb.h>
15#include <sch/rt_sched_types.h>
16#include <sch/wroost.h>
17#include <stdarg.h>
18#include <stddef.h>
19#include <stdint.h>
20#include <structures/list.h>
21#include <structures/pairing_heap.h>
22#include <structures/rbt.h>
23#include <sync/condvar.h>
24#include <sync/lock_chk_types.h>
25#include <sync/rcu.h>
26#include <sync/spinlock.h>
27#include <thread/apc_types.h>
28#include <thread/thread_diag.h>
29#include <thread/thread_types.h>
30#include <thread/wait.h>
31#include <time/time.h>
32#include <time/timer.h>
33#include <types/refcount.h>
34#include <types/types.h>
35
36#define THREAD_DEFAULT_TIMESLICE 15 /* 15 ms */
37
38#define THREAD_CLASS_WIDTH 1024
39#define THREAD_CLASS_HALF (THREAD_CLASS_WIDTH / 2)
40
41#define THREAD_BAND_MIN(avg) ((avg) - THREAD_CLASS_HALF)
42#define THREAD_BAND_MAX(avg) ((avg) + THREAD_CLASS_HALF)
43
44#define THREAD_ACT_INTERACTIVE_AVG 4000u
45#define THREAD_ACT_IO_BOUND_AVG 2500u
46#define THREAD_ACT_CPU_BOUND_AVG 1200u
47#define THREAD_ACT_SLEEPY_AVG 4500u
48
49#define THREAD_ACT_INTERACTIVE_MIN THREAD_BAND_MIN(THREAD_ACT_INTERACTIVE_AVG)
50#define THREAD_ACT_INTERACTIVE_MAX THREAD_BAND_MAX(THREAD_ACT_INTERACTIVE_AVG)
51
52#define THREAD_ACT_IO_BOUND_MIN THREAD_BAND_MIN(THREAD_ACT_IO_BOUND_AVG)
53#define THREAD_ACT_IO_BOUND_MAX THREAD_BAND_MAX(THREAD_ACT_IO_BOUND_AVG)
54
55#define THREAD_ACT_CPU_BOUND_MIN THREAD_BAND_MIN(THREAD_ACT_CPU_BOUND_AVG)
56#define THREAD_ACT_CPU_BOUND_MAX THREAD_BAND_MAX(THREAD_ACT_CPU_BOUND_AVG)
57
58#define THREAD_ACT_SLEEPY_MIN THREAD_BAND_MIN(THREAD_ACT_SLEEPY_AVG)
59#define THREAD_ACT_SLEEPY_MAX THREAD_BAND_MAX(THREAD_ACT_SLEEPY_AVG)
60
61#define THREAD_NICENESS_VALID(n) IN_RANGE((nice_t) (n), NICE_MIN, NICE_MAX)
62
63/* pluh */
64struct cpu_context {
65 uint64_t rbx;
66 uint64_t rbp;
67 uint64_t r12;
68 uint64_t r13;
69 uint64_t r14;
70 uint64_t r15;
71 uint64_t rsp;
72 uint64_t rip;
73};
74
75#define THREAD_EVENT_REASON_NONE 0xFF
76
77struct thread_event_reason {
78 uint16_t duration_ms; /* wait duration (clamped to UINT16_MAX ms) */
79 thread_act_reason_t
80 wait_reason; /* BLOCK_IO, BLOCK_MANUAL, SLEEP_MANUAL... */
81 thread_act_reason_t wake_reason; /* WAKE_IO, WAKE_MANUAL, WAKE_TIMEOUT... */
82};
83
84/* Background threads share timeslices */
85#define THREAD_PRIO_HAS_TIMESLICE(prio) \
86 (prio == THREAD_PRIO_CLASS_TIMESHARE || \
87 prio == THREAD_PRIO_CLASS_BACKGROUND)
88
89#define THREAD_ACTIVITY_BUCKET_COUNT 4
90#define THREAD_ACTIVITY_BUCKET_SHIFT 10
91#define THREAD_ACTIVITY_BUCKET_DURATION \
92 (1U << THREAD_ACTIVITY_BUCKET_SHIFT) /* 1024 ms (~1s) */
93#define THREAD_ACTIVITY_BUCKET_MASK (THREAD_ACTIVITY_BUCKET_DURATION - 1U)
94#define THREAD_ACTIVITY_INDEX_MASK (THREAD_ACTIVITY_BUCKET_COUNT - 1U)
95
96#define THREAD_EVENT_RINGBUFFER_CAPACITY THREAD_ACTIVITY_BUCKET_COUNT
97#define THREAD_EVENT_RINGBUFFER_MASK (THREAD_EVENT_RINGBUFFER_CAPACITY - 1U)
98#define TOTAL_BUCKET_DURATION \
99 (THREAD_ACTIVITY_BUCKET_COUNT * THREAD_ACTIVITY_BUCKET_DURATION)
100
101/* We're using 1024 for power of 2 division strength reduction optimizations */
102struct thread_activity_bucket {
103 uint32_t epoch; /* time_ms >> THREAD_ACTIVITY_BUCKET_SHIFT */
104 uint16_t run_ms; /* run duration in window (0..1024) */
105 uint16_t block_ms; /* blocked duration in window (0..1024) */
106 uint16_t sleep_ms; /* sleeping duration in window (0..1024) */
107 uint16_t wake_count; /* saturating wake count in window */
108};
109
110struct thread_activity_stats {
111 struct thread_activity_bucket buckets[THREAD_ACTIVITY_BUCKET_COUNT];
112};
113
114struct thread_activity_data {
115 uint32_t wait_start_ms; /*low 32 bits of wait start time */
116 struct thread_event_reason events[THREAD_EVENT_RINGBUFFER_CAPACITY];
117 uint8_t head; /* next slot index */
118
119 thread_act_reason_t active_wait_reason; /* NONE if not waiting */
120};
121
122/* Activity aggregations */
123enum thread_activity_class {
124 THREAD_ACTIVITY_CLASS_CPU_BOUND,
125 THREAD_ACTIVITY_CLASS_IO_BOUND,
126 THREAD_ACTIVITY_CLASS_INTERACTIVE,
127 THREAD_ACTIVITY_CLASS_SLEEPY,
128 THREAD_ACTIVITY_CLASS_UNKNOWN
129};
130
131struct thread_create_params {
132 enum thread_prio_class base_prio_class;
133 enum thread_flags flags;
134
135 size_t stack_pages;
136
137 void *arg;
138 void *stack_internal;
139
140 struct cpu_mask allowed_cpus; /* Default: All CPUs (functionally
141 * identical, so technically a mask) */
142
143 cpu_id_t on_cpu; /* Default: CPU_ID_MAX */
144
145 bool pinned;
146 bool joinable;
147
148 void *private;
149};
150
151struct thread_activity_metrics {
152 uint8_t run_ratio;
153 uint8_t block_ratio;
154 uint8_t sleep_ratio;
155 uint8_t wake_freq;
156};
157
158struct thread {
159 /* ========== Metadata ========== */
160 /* Unique ID allocated from global thread ID tree */
161 thread_id_t id;
162 char *name;
163 thread_entry_fn_t entry;
164
165 /* ========== Processor context data ========== */
166
167 /* Stack */
168 void *stack cc_sized_by(stack_size);
169 size_t stack_size;
170
171 /* TODO: we should isolate this into the debug nesting
172 * mode, as this helps us identify scheduler_yield() nesting */
173 uint32_t yield_nesting;
174
175 /* Preserves the highest point yield_nesting ever reached */
176 uint32_t yield_nesting_max;
177
178 /* Registers */
179 struct cpu_context regs;
180
181 /* ========== Transparent structure nodes ========== */
182
183 /* TODO: Use unions and combine these. Do make sure to keep
184 * in mind that all nodes get need to get
185 * reset if unions are to be used */
186
187#ifdef DEBUG_CRASH
188 struct crash_thread_data crash_data;
189#endif
190
191 union {
192 struct list_node reaper_list; /* reaper list */
193 struct list_node thread_list; /* global list of threads */
194 };
195
196 /* Runqueue nodes */
197
198 union {
199 struct rbt_node rq_tree_node; /* runqueue tree node */
200 struct rbt_node rt_tree_node; /* alias for rt scheduling */
201 };
202
203 union {
204 struct list_node rq_list_node; /* runqueue list node */
205 struct list_node rt_list_node; /* alias for rt scheduling */
206 };
207
208 /* Waitqueue nodes. TODO: union */
209 struct rbt_node wq_tree_node; /* waitqueue tree node */
210 int32_t wq_prio; /* turnstile sort key
211 * TODO: there is probably a smarter way
212 */
213
214 uint64_t wq_seq; /* Turnstile arrival order sequence count */
215 struct list_node wq_list_node; /* waitqueue list node */
216 struct list_node rcu_list_node; /* blocked reader node on RCU leaf */
217
218 /* ========== State ========== */
219
220 /* State */
221 atomic(enum thread_state) state;
222
223 /* Who is running us? */
224 cpu_id_t curr_core; /* -1 if not being ran */
225
226 /* TODO: implement a core_to_wake_on */
227
228 atomic(struct scheduler *) scheduler;
229
230 time_ms_t run_start_time; /* When did we start running */
231
232 /* Who is allowed to run us? */
233 atomic_uint32_t pinned_depth;
234 struct cpu_mask allowed_cpus;
235 atomic_int64_t migrate_to; /* -1 if no migration target */
236
237 /* Flags */
238 enum rt_scheduler_capability accepted_rt_caps;
239 atomic(enum thread_flags) flags;
240 atomic_size_t migration_generation;
241
242 /* ======== Raw priority + timeslice data ======== */
243
244 /* Priorities */
245 thread_prio_t activity_score;
246 int32_t dynamic_delta; /* Signed delta applied to base */
247 size_t weight;
248 nice_t niceness; /* -20 .. + 19 */
249
250 cpu_perf_t wanted_perf; /* This is the cpu_perf_t the thread's current
251 * CPU should try to match or do better than.
252 *
253 * If the current CPU's current wanted_perf
254 * does not satisfy the thread, we will try
255 * to either increase it for this CPU, or
256 * migrate the thread to another CPU. */
257
258 /* Class changes */
259 time_ms_t last_class_change_ms;
260
261 /* Timeslice info and periods */
262 uint64_t completed_period;
263 time_ms_t period_runtime_raw_ms; /* Raw MS time of runtime this period */
264 time_ms_t budget_time_raw_ms; /* Raw MS time of budget */
265 time_ms_t timeslice_length_raw_ms;
266
267 uint32_t virtual_period_runtime;
268 uint32_t virtual_budget;
269 uint32_t virtual_runtime_left;
270
271 /* ========== Thread activity stats ========== */
272
273 enum thread_activity_class activity_class : 4;
274
275 enum thread_prio_class perceived_prio_class : THREAD_PRIO_CLASS_BIT_COUNT;
276 enum thread_prio_class base_prio_class; /* for class boosts */
277
278 /*
279 * Rules with this lock:
280 *
281 * This lock is meant to synchronize updates to perceived_prio_class
282 * and base_prio_class. It is only meant to be used when
283 *
284 * (1) you are updating a thread on another CPU OR a non-running thread on THIS CPU
285 * and ALSO have the runqueue lock held prior to taking this, and will update
286 * the thread positions after doing this, OR
287 *
288 * (2) you are updating the currently running thread on the calling
289 * CPU, it is OK to NOT have the runqueue lock held.
290 *
291 * In any case, the scheduler will not see an invalid
292 * perceived_prio_class, and should be able to make decisions
293 * based on it, as it would be properly locked.
294 */
295 struct spinlock pi_lock;
296
297 /* Also protected by pi_lock, perceived_prio_class = max(base_prio_class,
298 * highest class with a boost). remember to
299 * thread_recompute_prio_class_locked after changing things */
300 uint8_t class_floors[THREAD_PRIO_CLASS_COUNT];
301
302 /* Activity data */
303 struct thread_activity_data activity_data;
304 struct thread_activity_stats activity_stats;
305
306 /* "Overview" derived from data and stats */
307 struct thread_activity_metrics activity_metrics;
308
309 /* ========== Synchronization data ========== */
310
311 /* Lock + rc */
312 struct spinlock lock;
313 refcount_t refcount;
314
315 /* Join */
316 struct spinlock join_lock; /* guards exit_status + the ZOMBIE publish */
317 struct condvar join_cv; /* joiners wait here */
318 int exit_status;
319
320 /* RCU stuff:
321 *
322 * rcu_nesting and rcu_read_seq are written ONLY by this thread
323 *
324 * rcu_leaf, rcu_blocked_seq, and rcu_list_node are how readers
325 * that get preempted mid-section get registered in RCU,
326 *
327 * those are written under the leaf's lock,
328 * either by us or the CPU switching us out
329 */
330 uint32_t rcu_nesting; /* read section depth */
331 uint64_t rcu_read_seq; /* GP sequence at the outermost lock */
332 struct rcu_node *rcu_leaf; /* leaf we are registered on or NULL */
333 uint64_t rcu_blocked_seq; /* GP we are counted against or 0 */
334 struct rcu_cb free_rcu;
335
336 enum thread_state wait_state;
337 enum thread_resume_reason wait_completion_reason;
338
339 /* Storage */
340 struct spinlock wait_lock;
341 struct thread_wait_block wait_blocks[THREAD_NUM_WAIT_BLOCKS];
342 struct thread_wait_block *active_wait_blocks;
343
344 size_t wait_block_count;
345
346 /* Every new wait bumps this, and it gets updated under wait_lock.
347 *
348 * While any block of the wait remains linked, the wait header's lock
349 * prevents this from changing, since waits can't end, and new ones
350 * can't start, without unlinking the respective block.
351 *
352 */
353 atomic_uint64_t wait_epoch;
354 uint16_t wait_key;
355
356 enum thread_wait_status wait_status;
357
358 bool wait_done : 1;
359 bool alert_pending : 1;
360 bool object_wait : 1;
361
362 atomic(enum thread_wait_type) wait_type;
363
364 struct timer wait_timer; /* for thread_wait_complete_timeout */
365 uint64_t wait_timer_epoch; /* epoch the armed timer could end */
366
367 struct list_head io_wait_tokens; /* list of tokens. NOTE: deprecated */
368
369 struct turnstile *turnstile; /* my turnstile */
370 atomic(struct turnstile *) blocked_ts; /* what am I blocked on */
371
372 struct climb_thread_data climb_data;
373 struct wroost_thread_data wroost_data;
374
375#ifdef DEBUG_LOCK_CHK
376 struct lock_chk_thread_data lock_chk;
377#endif
378
379 /* ========== Memory management data ========== */
380 tlb_stamp_t lazy_tlb_gen;
381
382 /* ========== APC data ========== */
383 /* Standard APC queues */
384 struct apc_queue apc_head[APC_TYPE_COUNT];
385
386 /* Any APC pending */
387 atomic_uint8_t apc_pending_mask; /* bitmask of APC_TYPE_* pending */
388
389 /* APC disable counts */
390 uint32_t special_apc_disable;
391 uint32_t kernel_apc_disable;
392
393 /* ========== Debug data ========== */
394#ifdef DEBUG_THREAD
395 struct log_site *log_site;
396 struct log_handle log_handle;
397
398 size_t context_switches; /* Total context switches */
399
400 size_t preemptions;
401
402 time_ms_t creation_time_ms; /* When were we created? */
403
404 uint32_t boost_count;
405 uint32_t total_wake_count; /* Aggregate count of all wake events */
406 uint32_t total_block_count; /* Aggregate count of all block events */
407 uint32_t total_sleep_count; /* Aggregate count of all sleep events */
408 uint32_t total_apcs_ran; /* Total APCs executed on a given thread */
409#endif
410
411 /* Misc. private field for whatever needs it */
412 void *private;
413};
414
415#define thread_from_rq_rbt_node(node) \
416 rbt_entry(node, struct thread, rq_tree_node)
417#define thread_from_rq_list_node(ln) \
418 (container_of(ln, struct thread, rq_list_node))
419
420#define thread_from_rcu_list_node(ln) \
421 (container_of(ln, struct thread, rcu_list_node))
422
423#define thread_from_wq_pairing_node(pn) \
424 (container_of(pn, struct thread, wq_pairing_node))
425#define thread_from_wq_list_node(ln) \
426 (container_of(ln, struct thread, wq_list_node))
427#define thread_from_wq_rbt_node(ln) \
428 (container_of(ln, struct thread, wq_tree_node))
429
430#define thread_log(lvl, fmt, ...) \
431 ({ \
432 if (global.current_bootstage >= BOOTSTAGE_LATE) \
433 log(thread_get_current()->log_site, \
434 &thread_get_current()->log_handle, lvl, fmt, ##__VA_ARGS__); \
435 })
436
437#define thread_err(fmt, ...) thread_log(LOG_ERROR, fmt, ##__VA_ARGS__)
438#define thread_warn(fmt, ...) thread_log(LOG_WARN, fmt, ##__VA_ARGS__)
439#define thread_info(fmt, ...) thread_log(LOG_INFO, fmt, ##__VA_ARGS__)
440#define thread_debug(fmt, ...) thread_log(LOG_DEBUG, fmt, ##__VA_ARGS__)
441#define thread_trace(fmt, ...) thread_log(LOG_TRACE, fmt, ##__VA_ARGS__)
442
443struct thread *thread_create_internal(thread_entry_fn_t entry,
444 struct thread_create_params params,
445 char *fmt, ...) cc_printf_like(3, 4);
446
447struct thread *thread_spawn_internal(thread_entry_fn_t entry,
448 struct thread_create_params params,
449 char *fmt, ...) cc_printf_like(3, 4);
450
451void thread_set_joinable(struct thread *t);
452
453void thread_free(struct thread *t);
454
455void thread_init_thread_ids(void);
456void thread_sleep_for_ms(uint64_t ms) TSA_MAY_SLEEP;
457void thread_sleep_for_us(uint64_t us) TSA_MAY_SLEEP;
458cc_noreturn void thread_exit(void);
459void thread_print(const struct thread *t);
460
461void thread_classify_activity(struct thread *t, uint64_t now_ms);
462void thread_update_runtime_buckets(struct thread *thread, time_ms_t time);
463void thread_apply_wake_boost(struct thread *t);
464void thread_update_effective_priority(struct thread *t);
465void thread_apply_cpu_penalty(struct thread *t);
466
467void thread_add_wake_reason(struct thread *t, thread_act_reason_t reason);
468void thread_calculate_activity_data(struct thread *t);
469
470void thread_add_block_reason(struct thread *t, thread_act_reason_t reason);
471void thread_add_sleep_reason(struct thread *t, thread_act_reason_t reason);
472
473void thread_set_timesharing(struct thread *t);
474void thread_set_background(struct thread *t);
475void thread_migrate(struct thread *t, size_t dest_core);
476
477void thread_boost_self(enum thread_prio_class class);
478void thread_unboost_self(enum thread_prio_class class);
479
480void thread_enqueue(struct thread *t);
481void thread_enqueue_on_core(struct thread *t, uint64_t core_id);
482
483struct wroost_kicks cc_warn_unused_result
484thread_recompute_prio_class_locked(struct thread *t);
485
486/* Dry run: see what the new prio would be, don't set anything */
487enum thread_prio_class
488thread_recompute_prio_class_dry_locked(struct thread *t,
489 enum thread_prio_class *old_out,
490 enum thread_prio_class *new_out);
491
492void thread_recompute_prio_class_raw_locked(struct thread *t,
493 enum thread_prio_class *old_out,
494 enum thread_prio_class *new_out);
495
496bool thread_inherit_priority(struct thread *boosted, struct thread *from,
497 struct thread_class_donation *d,
498 struct wroost_kicks *kicks_out);
499
500void thread_donate_class(struct thread *boosted, enum thread_prio_class class,
501 struct thread_class_donation *d);
502
503void thread_uninherit_priority(struct thread_class_donation *d);
504void thread_remove_boost(void);
505
506cc_noreturn void thread_exit_with_status(int status);
507
508int thread_join(struct thread *t);
509bool thread_join_timeout(struct thread *t, time_ms_t timeout_ms,
510 int *status_out);
511void thread_detach(struct thread *t);
512
513void thread_lock_two_runqueues(struct thread *a, struct thread *b,
514 struct scheduler **out_rq_a,
515 struct scheduler **out_rq_b, enum irql *irq_a,
516 enum irql *irq_b);
517
518void thread_lock_thread_and_rq(struct thread *t, struct scheduler *other_rq,
519 struct scheduler **out_thread_rq,
520 enum irql *irq_first, enum irql *irq_second);
521void thread_unlock_thread_and_rq(struct scheduler *thread_rq,
522 struct scheduler *other_rq,
523 enum irql irq_first, enum irql irq_second);
524
525void thread_lock_two_pi_locks(struct thread *cc_nonnull_ptr a,
526 struct thread *cc_nonnull_ptr b, enum irql *irq_a,
527 enum irql *irq_b) TSA_NO_ANALYSIS;
528
529void thread_unlock_two_pi_locks(struct thread *cc_nonnull_ptr a,
530 struct thread *cc_nonnull_ptr b,
531 enum irql *irq_a,
532 enum irql *irq_b) TSA_NO_ANALYSIS;
533
534bool thread_in_context(void);
535REFCOUNT_GENERATE_GETREF_FOR_STRUCT_WITH_FAILURE_COND(thread, refcount, flags,
536 &THREAD_FLAG_DYING);
537void reaper_enqueue(struct thread *t);
538void thread_putref(struct thread *t);
539
540/* Yield nesting work:
541 *
542 * scheduler_yield() can re-enter, and we're using this to debug,
543 * with a counter bumped per thread. The max nesting depth today
544 * is 1, and it will likely stay this way, but I'm keeping
545 * it a tunable counter in case this changes. */
546#ifndef SCHED_MAX_YIELD_NESTING
547#define SCHED_MAX_YIELD_NESTING 1
548#endif
549
550static inline uint32_t scheduler_yield_nesting(struct thread *t) {
551 return t ? t->yield_nesting : 0;
552}
553
554static inline void scheduler_yield_nesting_enter(struct thread *t) {
555 if (!t)
556 return;
557
558 t->yield_nesting++;
559 if (t->yield_nesting > t->yield_nesting_max)
560 t->yield_nesting_max = t->yield_nesting;
561
562#ifdef DEBUG_SCHED_NESTING
563 enforce(t->yield_nesting <= SCHED_MAX_YIELD_NESTING, OVERFLOW,
564 "scheduler_yield nested %u deep on thread '%s'", t->yield_nesting,
565 t->name);
566#endif
567}
568
569static inline uint32_t scheduler_yield_nesting_max(struct thread *t) {
570 return t ? t->yield_nesting_max : 0;
571}
572
573static inline void scheduler_yield_nesting_exit(struct thread *t) {
574 if (!t)
575 return;
576
577#ifdef DEBUG_SCHED_NESTING
578 enforce(t->yield_nesting > 0, UNDERFLOW,
579 "scheduler_yield nesting on thread '%s'", t->name);
580#endif
581
582 if (t->yield_nesting)
583 t->yield_nesting--;
584}
585
586static inline void scheduler_yield_nesting_reset(struct thread *t) {
587 if (t)
588 t->yield_nesting = 0;
589}
590
591static inline struct thread *thread_get_current(void) {
592 uintptr_t thread;
593 asm volatile("movq %%gs:%c1, %0"
594 : "=r"(thread)
595 : "i"(offsetof(struct core, current_thread)));
596 return (struct thread *) thread;
597}
598
599static inline int64_t thread_set_migration_target(struct thread *t,
600 int64_t new) {
601 return atomic_exchange(&t->migrate_to, new);
602}
603
604static inline enum thread_state thread_get_state(struct thread *t) {
605 return atomic_load(&t->state);
606}
607
608static inline void thread_set_state(struct thread *t, enum thread_state state) {
609 atomic_store(&t->state, state);
610}
611
612static inline enum thread_flags thread_get_flags(struct thread *t) {
613 return atomic_load(&t->flags);
614}
615
616static inline void thread_set_flags(struct thread *t, enum thread_flags new) {
617 atomic_store(&t->flags, new);
618}
619
620static inline uint32_t thread_pin(struct thread *t) {
621 uint32_t old = atomic_inc(&t->pinned_depth);
622 enforce(old != UINT32_MAX, OVERFLOW);
623 return old;
624}
625
626static inline uint32_t thread_unpin(struct thread *t) {
627 uint32_t old = atomic_dec(&t->pinned_depth);
628 enforce(old != 0, UNDERFLOW);
629 return old;
630}
631
632static inline bool thread_is_pinned(struct thread *t) {
633 return atomic_load(&t->pinned_depth) > 0;
634}
635
636static inline enum thread_flags thread_or_flags(struct thread *t,
637 enum thread_flags flags) {
638 return atomic_fetch_or(&t->flags, flags);
639}
640
641static inline enum thread_flags thread_and_flags(struct thread *t,
642 enum thread_flags flags) {
643 return atomic_fetch_and(&t->flags, flags);
644}
645
646static inline enum thread_flags thread_set_flag(struct thread *t,
647 enum thread_flags flag) {
648 return thread_or_flags(t, flags: flag);
649}
650
651static inline enum thread_flags thread_clear_flag(struct thread *t,
652 enum thread_flags flag) {
653 return thread_and_flags(t, flags: ~flag);
654}
655
656static inline bool thread_test_flag(struct thread *t, enum thread_flags flag) {
657 return (thread_get_flags(t) & flag) == flag;
658}
659
660static inline size_t thread_get_migration_generation(struct thread *t) {
661 return atomic_load_acq(&t->migration_generation);
662}
663
664static inline struct scheduler *thread_get_scheduler_unsafe(struct thread *t) {
665 return atomic_load_acq(&t->scheduler);
666}
667
668static inline void thread_set_runqueue(struct thread *t, struct scheduler *s) {
669 atomic_inc_release(&t->migration_generation);
670 atomic_store_release(&t->scheduler, s);
671 atomic_inc_release(&t->migration_generation);
672}
673
674/* RCU keeps the thread memory allocated so we inc_not_zero here
675 * and everything is fine and dandy, everything must be in a
676 * read side critical section, though */
677static inline bool thread_getref_rcu(struct thread *t) {
678 if (!refcount_inc_not_zero(rc: &t->refcount))
679 return false;
680
681 if (thread_get_flags(t) & THREAD_FLAG_DYING) {
682 thread_putref(t);
683 return false;
684 }
685
686 return true;
687}
688
689static inline enum irql thread_acquire_internal(struct thread *t,
690 bool (*fn)(struct thread *),
691 bool *success) TSA_NO_ANALYSIS {
692 if (!fn(t)) {
693 if (success)
694 *success = false;
695 return IRQL_NONE;
696 }
697
698 if (success)
699 *success = true;
700 return spin_lock_high(&t->lock);
701}
702
703static inline enum irql thread_acquire(struct thread *t,
704 bool *success) TSA_NO_ANALYSIS {
705 return thread_acquire_internal(t, fn: thread_getref, success);
706}
707
708static inline enum irql thread_acquire_rcu(struct thread *t,
709 bool *success) TSA_NO_ANALYSIS {
710 return thread_acquire_internal(t, fn: thread_getref_rcu, success);
711}
712
713static inline void thread_release(struct thread *t,
714 enum irql irql) TSA_NO_ANALYSIS {
715 spin_unlock(&t->lock, irql);
716 thread_putref(t);
717}
718
719static inline bool thread_is_rt(struct thread *t) {
720 return t->perceived_prio_class == THREAD_PRIO_CLASS_URGENT ||
721 t->perceived_prio_class == THREAD_PRIO_CLASS_REALTIME;
722}
723
724static inline enum thread_wait_type thread_get_wait_type(struct thread *t) {
725 return atomic_load_acq(&t->wait_type);
726}
727
728#define THREAD_CREATE_PARAMS_DEFAULTS \
729 .stack_pages = (THREAD_STACK_SIZE / PAGE_SIZE), .on_cpu = CPU_ID_MAX, \
730 .joinable = false, .pinned = false, .allowed_cpus = CPU_MASK_INIT_ALL, \
731 .arg = NULL, .private = NULL, .stack_internal = NULL, .flags = 0, \
732 .base_prio_class = THREAD_PRIO_CLASS_TIMESHARE
733
734#define thread_params_with_defaults(...) \
735 cc_wno_override_init_expr( \
736 struct thread_create_params, \
737 ((struct thread_create_params) {THREAD_CREATE_PARAMS_DEFAULTS, \
738 ##__VA_ARGS__}))
739
740#define thread_create(name, entry, ...) \
741 thread_create_internal((entry), \
742 (struct thread_create_params) \
743 thread_params_with_defaults(__VA_ARGS__), \
744 PP_UNPAREN(name))
745
746#define thread_spawn(name, entry, ...) \
747 thread_spawn_internal((entry), \
748 (struct thread_create_params) \
749 thread_params_with_defaults(__VA_ARGS__), \
750 PP_UNPAREN(name))
751