1#include <thread/io_wait.h>
2
3#include "internal.h"
4
5struct scheduler *scheduler_select_best_for_thread(struct thread *t) {
6 struct scheduler *sched = NULL;
7 size_t i, min_load = SIZE_MAX;
8 cpu_mask_for_each(i, t->allowed_cpus) {
9 size_t this_load = global.schedulers[i]->total_thread_count;
10
11 if (global.cores && global.cores[i] &&
12 !scheduler_core_idle(c: global.cores[i]))
13 this_load++;
14
15 if (this_load < min_load) {
16 min_load = this_load;
17 sched = global.schedulers[i];
18 }
19 }
20
21 kassert(sched);
22 return sched;
23}
24
25bool thread_wake(struct thread *t, enum thread_wake_reason reason,
26 enum thread_prio_class prio, void *wake_src) {
27 kassert(t);
28
29 enum irql birql, lirql;
30
31 struct scheduler *best = scheduler_select_best_for_thread(t);
32 struct scheduler *last_sch;
33
34 /* Lock the thread's runqueue and the best one. If the thread
35 * can be placed on the best one, we put it over there */
36 thread_lock_thread_and_rq(t, other_rq: best, out_thread_rq: &last_sch, irq_first: &lirql, irq_second: &birql);
37
38 /* this is a fun one. because threads can sleep/block in modes
39 * that aren't just wakeable in one way, we must take care here.
40 *
41 * first, we acquire the scheduler lock so the thread doesn't enter/exit
42 * the runqueues. then we acquire the thread lock
43 * so it doesn't decide to block/sleep (this is because of
44 * wait_for_wake_match -- the yield() loop will abort if it sees
45 * that someone else has set wake_matched).
46 *
47 * this puts us in a position where by the time the thread sees us publish
48 * the `wake` changes we make to it, it will absolutely wake up.
49 */
50
51 bool woke = false;
52 bool ok;
53 enum irql tirql = thread_acquire(t, success: &ok);
54 if (!ok)
55 goto end;
56
57 /* now that we have acquired the locks, we will take a
58 * peek at the wait type.
59 *
60 * if it is UNINTERRUPTIBLE and we are NOT the expected waker, then we leave
61 */
62 enum thread_wait_type wt = thread_get_wait_type(t);
63 if ((wt == THREAD_WAIT_UNINTERRUPTIBLE &&
64 t->expected_wake_src != wake_src) ||
65 wt == THREAD_WAIT_NONE) {
66 goto out;
67 }
68
69 woke = true;
70
71 /* we get the earlier state here */
72 enum thread_state state = thread_get_state(t);
73 bool yielded = thread_get_flags(t) & THREAD_FLAG_YIELDED;
74
75 thread_prepare_to_wake_locked(t, r: reason, wake_src);
76 thread_apply_wake_boost(t);
77 t->perceived_prio_class = prio;
78
79 /* if the thread has NOT yielded after it set itself blocked it is
80 * completely unsafe to put it back on the runqueues as it is currently
81 * running, but is marked as BLOCKED or SLEEPING. This can happen when an
82 * ISR enters this code, when the thread we are looking at is on the same
83 * CPU and marked as BLOCKED/SLEEPING when in reality it is actually running
84 * but wanting to block/sleep but has not yielded */
85 if (yielded && state != THREAD_STATE_RUNNING &&
86 state != THREAD_STATE_READY) {
87 scheduler_add_thread(sched: best, thread: t, /* lock_held = */ true);
88 if (last_sch != best) {
89 thread_post_migrate(t, old_cpu: last_sch->core_id, new_cpu: best->core_id);
90 }
91
92 scheduler_force_resched(sched: best);
93 }
94
95out:
96 thread_release(t, irql: tirql);
97end:
98
99 thread_unlock_thread_and_rq(thread_rq: last_sch, other_rq: best, irq_first: lirql, irq_second: birql);
100 return woke;
101}
102
103void thread_wake_from_io_block(struct thread *t, void *wake_src) {
104 /* we are just inspecting the thread to see if this structure exists, no
105 * synchronization needed */
106 struct io_wait_token *iter;
107 bool found = false;
108 list_for_each_entry(iter, &t->io_wait_tokens, list) {
109 if (iter->wait_object == wake_src)
110 found = true;
111 }
112 if (!found) {
113 printf(format: "Problem with %s %p\n", t->name, t);
114 list_for_each_entry(iter, &t->io_wait_tokens, list) {
115 printf(format: "wait obj %p src %p\n", iter->wait_object, wake_src);
116 }
117 }
118
119 kassert(found, "On thread %p", t);
120
121 thread_wake(t, reason: THREAD_WAKE_REASON_BLOCKING_IO, prio: THREAD_PRIO_CLASS_URGENT,
122 wake_src);
123}
124