1#include <thread/io_wait.h>
2
3#include "internal.h"
4
5struct scheduler *scheduler_select_best_for_thread(struct thread *t) {
6 /* Pinned threads are already placed, don't bother here */
7 if ((thread_get_flags(t) & THREAD_FLAG_PINNED) && t->scheduler)
8 return t->scheduler;
9
10 struct scheduler *sched = NULL;
11 size_t i, min_load = SIZE_MAX;
12 cpu_mask_for_each(i, t->allowed_cpus) {
13 size_t this_load = global.schedulers[i]->total_thread_count;
14
15 if (global.cores && global.cores[i] &&
16 !scheduler_core_idle(c: global.cores[i]))
17 this_load++;
18
19 if (this_load < min_load) {
20 min_load = this_load;
21 sched = global.schedulers[i];
22 }
23 }
24
25 return kassert(sched);
26}
27
28bool thread_wake(struct thread *t, enum thread_wake_reason reason,
29 enum thread_prio_class prio, void *wake_src) {
30 kassert(t);
31
32 enum irql birql, lirql;
33
34 struct scheduler *best = scheduler_select_best_for_thread(t);
35 struct scheduler *last_sch;
36
37 /* Lock the thread's runqueue and the best one. If the thread
38 * can be placed on the best one, we put it over there */
39 thread_lock_thread_and_rq(t, other_rq: best, out_thread_rq: &last_sch, irq_first: &lirql, irq_second: &birql);
40
41 /* this is a fun one. because threads can sleep/block in modes
42 * that aren't just wakeable in one way, we must take care here.
43 *
44 * first, we acquire the scheduler lock so the thread doesn't enter/exit
45 * the runqueues. then we acquire the thread lock
46 * so it doesn't decide to block/sleep (this is because of
47 * wait_for_wake_match -- the yield() loop will abort if it sees
48 * that someone else has set wake_matched).
49 *
50 * this puts us in a position where by the time the thread sees us publish
51 * the `wake` changes we make to it, it will absolutely wake up.
52 */
53
54 bool woke = false;
55 bool ok;
56 enum irql tirql = thread_acquire(t, success: &ok);
57 if (!ok)
58 goto end;
59
60 /* now that we have acquired the locks, we will take a
61 * peek at the wait type.
62 *
63 * if it is UNINTERRUPTIBLE and we are NOT the expected waker, then we leave
64 */
65 enum thread_wait_type wt = thread_get_wait_type(t);
66 if ((wt == THREAD_WAIT_UNINTERRUPTIBLE &&
67 t->expected_wake_src != THREAD_WAIT_ANY_SRC &&
68 t->expected_wake_src != wake_src) ||
69 wt == THREAD_WAIT_NONE) {
70 goto out;
71 }
72
73 woke = true;
74
75 /* we get the earlier state here */
76 enum thread_state state = thread_get_state(t);
77 bool yielded = thread_get_flags(t) & THREAD_FLAG_YIELDED;
78
79 thread_prepare_to_wake_locked(t, r: reason, wake_src);
80 thread_apply_wake_boost(t);
81 t->perceived_prio_class = prio;
82
83 /* if the thread has NOT yielded after it set itself blocked it is
84 * completely unsafe to put it back on the runqueues as it is currently
85 * running, but is marked as BLOCKED or SLEEPING. This can happen when an
86 * ISR enters this code, when the thread we are looking at is on the same
87 * CPU and marked as BLOCKED/SLEEPING when in reality it is actually running
88 * but wanting to block/sleep but has not yielded */
89 if (yielded && state != THREAD_STATE_RUNNING &&
90 state != THREAD_STATE_READY) {
91 scheduler_add_thread(sched: best, thread: t, /* lock_held = */ true);
92 if (last_sch != best) {
93 thread_post_migrate(t, old_cpu: last_sch->core_id, new_cpu: best->core_id);
94 }
95
96 scheduler_force_resched(sched: best);
97 }
98
99out:
100 thread_release(t, irql: tirql);
101end:
102
103 thread_unlock_thread_and_rq(thread_rq: last_sch, other_rq: best, irq_first: lirql, irq_second: birql);
104 return woke;
105}
106
107void thread_wake_from_io_block(struct thread *t, void *wake_src) {
108 /* we are just inspecting the thread to see if this structure exists, no
109 * synchronization needed */
110 struct io_wait_token *iter;
111 bool found = false;
112 list_for_each_entry(iter, &t->io_wait_tokens, list) {
113 if (iter->wait_object == wake_src)
114 found = true;
115 }
116 if (!found) {
117 printf(format: "Problem with %s %p\n", t->name, t);
118 list_for_each_entry(iter, &t->io_wait_tokens, list) {
119 printf(format: "wait obj %p src %p\n", iter->wait_object, wake_src);
120 }
121 }
122
123 kassert(found, "On thread %p", t);
124
125 thread_wake(t, reason: THREAD_WAKE_REASON_BLOCKING_IO, prio: THREAD_PRIO_CLASS_URGENT,
126 wake_src);
127}
128