1#include "internal.h"
2
3struct scheduler *scheduler_select_best_for_thread(struct thread *t) {
4 /* Pinned threads are already placed, don't bother here */
5 struct scheduler *pinned_sched = atomic_load_acq(&t->scheduler);
6 if (thread_is_pinned(t) && pinned_sched)
7 return pinned_sched;
8
9 /* IMPORTANT NOTE: If a thread that was newly spawned as PINNED,
10 * we do NOT fail the selection. We select as if it's simply
11 * a non-pinned thread. We've got real consumers of this behavior,
12 * notably, in the testing infrastructure, so if we
13 * make changes, we'll need to preserve this */
14 struct scheduler *sched = NULL;
15 size_t i, min_load = SIZE_MAX;
16 cpu_mask_for_each(i, t->allowed_cpus) {
17 size_t this_load = global.schedulers[i]->total_thread_count;
18
19 if (global.cores && global.cores[i] &&
20 !scheduler_core_idle(c: global.cores[i]))
21 this_load++;
22
23 if (this_load < min_load) {
24 min_load = this_load;
25 sched = global.schedulers[i];
26 }
27 }
28
29 return enforce(
30 sched, "No scheduler was found, likely due to allowed_cpus being 0");
31}
32
33/* Only scheduler entries are object completion and APC execution */
34static void resume_thread(struct thread *t, enum thread_resume_reason reason,
35 bool completion) TSA_NO_ANALYSIS {
36 enum irql birql, lirql;
37
38 /* Force reschedule OUTSIDE of any runqueue locks to avoid deadlocking */
39 struct scheduler *to_force_resched = NULL;
40
41 struct scheduler *best = scheduler_select_best_for_thread(t);
42 struct scheduler *last;
43 thread_lock_thread_and_rq(t, other_rq: best, out_thread_rq: &last, irq_first: &lirql, irq_second: &birql);
44
45 bool ok;
46 enum irql tirql = thread_acquire(t, success: &ok);
47 if (!ok)
48 goto end;
49
50 enum thread_state state = thread_get_state(t);
51 if (completion) {
52 enforce(t->object_wait);
53 } else if (!t->object_wait ||
54 thread_get_wait_type(t) != THREAD_WAIT_INTERRUPTIBLE) {
55
56 if (state == THREAD_STATE_RUNNING || state == THREAD_STATE_READY)
57 to_force_resched = last;
58
59 goto out;
60 }
61
62 enum irql pirql = spin_lock_high(&t->pi_lock);
63 bool yielded = thread_test_flag(t, flag: THREAD_FLAG_YIELDED);
64
65 /* A prepared waiter can preempt before complete(), so we need
66 * to remove the old queue entry and then change class for completion */
67 bool requeue = completion && state == THREAD_STATE_READY;
68 if (requeue)
69 scheduler_remove_thread_locked(sched: last, t);
70
71 enum thread_state to_state =
72 yielded ? THREAD_STATE_READY : THREAD_STATE_RUNNING;
73
74 if (state != THREAD_STATE_RUNNING && state != THREAD_STATE_READY)
75 thread_set_state(t, state: to_state);
76
77 /* NOTE: completion is automatically given the boost *here* */
78 if (completion) {
79 thread_add_wake_reason(t, reason);
80 thread_apply_wake_boost(t);
81 }
82
83 bool queuable =
84 state != THREAD_STATE_RUNNING && state != THREAD_STATE_READY;
85
86 if (requeue || (yielded && queuable)) {
87 scheduler_add_thread_locked(sched: best, thread: t);
88
89 if (last != best)
90 thread_post_migrate(t, old_cpu: last->core_id, new_cpu: best->core_id);
91
92 to_force_resched = best;
93 } else if (!completion) {
94 to_force_resched = last;
95 }
96
97 spin_unlock(&t->pi_lock, pirql);
98
99out:
100 thread_release(t, irql: tirql);
101
102end:
103 thread_unlock_thread_and_rq(thread_rq: last, other_rq: best, irq_first: lirql, irq_second: birql);
104
105 if (to_force_resched)
106 scheduler_force_resched(sched: to_force_resched);
107}
108
109void scheduler_complete_object_wait(struct thread *t,
110 enum thread_resume_reason reason) {
111 SPINLOCK_ASSERT_HELD(&t->wait_lock);
112 resume_thread(t, reason, true);
113}
114
115void scheduler_resume_for_apc(struct thread *t) {
116 resume_thread(t, reason: THREAD_WAKE_REASON_BLOCKING_MANUAL, false);
117}
118