1#include <sch/sched.h>
2#include <thread/thread.h>
3
4#include "internal.h"
5
6enum thread_prio_class
7thread_recompute_prio_class_dry_locked(struct thread *t,
8 enum thread_prio_class *old_out,
9 enum thread_prio_class *new_out) {
10 SPINLOCK_ASSERT_HELD(&t->pi_lock);
11
12 enum thread_prio_class c = t->base_prio_class;
13 for (int i = THREAD_PRIO_CLASS_COUNT - 1; i > c; i--) {
14 if (t->class_floors[i]) {
15 c = (enum thread_prio_class) i;
16 break;
17 }
18 }
19
20 if (old_out)
21 *old_out = t->perceived_prio_class;
22
23 if (new_out)
24 *new_out = c;
25
26 return c;
27}
28
29void thread_recompute_prio_class_raw_locked(struct thread *t,
30 enum thread_prio_class *old_out,
31 enum thread_prio_class *new_out) {
32 t->perceived_prio_class =
33 thread_recompute_prio_class_dry_locked(t, old_out, new_out);
34}
35
36struct wroost_kicks thread_recompute_prio_class_locked(struct thread *t) {
37 enum thread_prio_class old, new;
38
39 thread_recompute_prio_class_raw_locked(t, old_out: &old, new_out: &new);
40 return wroost_post_prio_change_locked(t, old, new);
41}
42
43void thread_class_floor_inc_locked(struct thread *t,
44 enum thread_prio_class class) {
45 SPINLOCK_ASSERT_HELD(&t->pi_lock);
46
47 enforce_precond(class < THREAD_PRIO_CLASS_COUNT);
48 enforce(t->class_floors[class] != UINT8_MAX, OVERFLOW);
49 t->class_floors[class]++;
50}
51
52void thread_class_floor_dec_locked(struct thread *t,
53 enum thread_prio_class class) {
54 SPINLOCK_ASSERT_HELD(&t->pi_lock);
55
56 enforce_precond(class < THREAD_PRIO_CLASS_COUNT);
57 enforce(t->class_floors[class] > 0, UNDERFLOW);
58 t->class_floors[class]--;
59}
60
61/* Raise `d` on `t` to at least class */
62static bool thread_donation_raise_locked(struct thread *t,
63 struct thread_class_donation *d,
64 enum thread_prio_class class,
65 struct wroost_kicks *kicks) {
66 enum thread_prio_class old = t->perceived_prio_class;
67
68 if (d->active) {
69 enforce(d->target == t, "donation applied to a different thread");
70 if (class <= d->class)
71 return false;
72
73 thread_class_floor_dec_locked(t, class: d->class);
74 }
75
76 d->target = t;
77 d->class = class;
78 d->active = true;
79 thread_class_floor_inc_locked(t, class);
80 *kicks = thread_recompute_prio_class_locked(t);
81
82 return t->perceived_prio_class > old;
83}
84
85static void thread_donation_drop_locked(struct thread_class_donation *d) {
86 struct thread *t = d->target;
87 thread_class_floor_dec_locked(t, class: d->class);
88
89 /* NOTE: we can ignore re-sending DPCs when boosting down */
90 cc_discard(thread_recompute_prio_class_locked(t));
91
92 d->target = NULL;
93 d->class = THREAD_PRIO_CLASS_BACKGROUND;
94 d->active = false;
95}
96
97#define THREAD_CLASS_UPDATE(t, ...) \
98 do { \
99 struct thread *__t = (t); \
100 if (__t == thread_get_current()) { \
101 enum irql __pirql = spin_lock_high(&__t->pi_lock); \
102 __VA_ARGS__; \
103 spin_unlock(&__t->pi_lock, __pirql); \
104 } else { \
105 struct scheduler *__sched; \
106 enum irql __sirql = thread_lock_scheduler(__t, &__sched); \
107 enum irql __pirql = spin_lock_high(&__t->pi_lock); \
108 bool __ready = thread_get_state(__t) == THREAD_STATE_READY; \
109 if (__ready) \
110 scheduler_remove_thread_locked(__sched, __t); \
111 __VA_ARGS__; \
112 if (__ready) \
113 scheduler_add_thread_locked(__sched, __t); \
114 spin_unlock(&__t->pi_lock, __pirql); \
115 spin_unlock(&__sched->lock, __sirql); \
116 } \
117 } while (0)
118
119static bool scheduler_boost_thread_internal(struct thread *boosted,
120 struct thread *from,
121 struct thread_class_donation *d,
122 struct wroost_kicks *kicks) {
123 if (thread_donation_raise_locked(t: boosted, d, class: from->perceived_prio_class,
124 kicks))
125 goto ok;
126
127 /* Here comes the fun part. We always assume that `from` is a thread that
128 * can be safely modified, and we use the runqueue lock to protected
129 * `boosted`'s CLIMB structures. We first check that `from`'s CLIMB
130 * handle has either not been used or was used on `boosted`, and then
131 * we perform our boost. We initialize `from`'s CLIMB handle, and then
132 * attach it to `boosted` and apply relevant boosts */
133 struct climb_handle *h = &from->climb_data.handle;
134 if (h->given_to == boosted)
135 goto ok;
136
137 if (h->given_to)
138 return false;
139
140 h->pressure = climb_thread_compute_pressure_to_apply(t: from);
141 climb_handle_apply_locked(t: boosted, h);
142
143 return false;
144
145ok:
146
147#ifdef DEBUG_THREAD
148 boosted->boost_count++;
149#endif
150
151 return true;
152}
153
154bool thread_inherit_priority(struct thread *boosted, struct thread *from,
155 struct thread_class_donation *d,
156 struct wroost_kicks *kicks_out) TSA_NO_ANALYSIS {
157 struct scheduler *sched, *sched2;
158 enum irql irql, irql2;
159 enum irql birql, firql;
160 struct wroost_kicks kicks = {0};
161 struct wroost_kicks *use = kicks_out ? kicks_out : &kicks;
162
163 thread_lock_two_runqueues(a: boosted, b: from, out_rq_a: &sched, out_rq_b: &sched2, irq_a: &irql, irq_b: &irql2);
164
165 thread_lock_two_pi_locks(a: boosted, b: from, irq_a: &birql, irq_b: &firql);
166
167 bool did_boost = false;
168 if (thread_get_state(t: boosted) == THREAD_STATE_READY) {
169 /* thread is READY - we remove it from the runqueue and then we
170 * re-insert it */
171 scheduler_remove_thread_locked(sched, t: boosted);
172 did_boost = scheduler_boost_thread_internal(boosted, from, d, kicks: use);
173 scheduler_add_thread_locked(sched, thread: boosted);
174 } else {
175 /* if the thread is off doing anything else (maybe it's blocking, maybe
176 * it's running), we go ahead and just boost it. when it is saved those
177 * new values will be read and everything will be all splendid */
178 did_boost = scheduler_boost_thread_internal(boosted, from, d, kicks: use);
179 }
180
181 thread_unlock_two_pi_locks(a: boosted, b: from, irq_a: &birql, irq_b: &firql);
182
183 scheduler_release_two_locks(a: sched, b: sched2, a_irql: irql, b_irql: irql2);
184
185 if (!kicks_out)
186 wroost_kick(kicks: &kicks);
187
188 return did_boost;
189}
190
191void thread_donate_class(struct thread *boosted, enum thread_prio_class class,
192 struct thread_class_donation *d) TSA_NO_ANALYSIS {
193 struct wroost_kicks kicks = {0};
194 THREAD_CLASS_UPDATE(boosted, {
195 if (thread_donation_raise_locked(boosted, d, class, &kicks)) {
196#ifdef DEBUG_THREAD
197 boosted->boost_count++;
198#endif
199 }
200 });
201
202 wroost_kick(kicks: &kicks);
203}
204
205void thread_uninherit_priority(struct thread_class_donation *d)
206 TSA_NO_ANALYSIS {
207 if (!d->active)
208 return;
209
210 THREAD_CLASS_UPDATE(d->target, thread_donation_drop_locked(d));
211}
212
213void thread_boost_self(enum thread_prio_class class) {
214 struct thread *curr = thread_get_current();
215 struct wroost_kicks kicks;
216
217 {
218 spin_guard_high(&curr->pi_lock);
219
220 thread_class_floor_inc_locked(t: curr, class);
221 kicks = thread_recompute_prio_class_locked(t: curr);
222 }
223
224 wroost_kick(kicks: &kicks);
225}
226
227void thread_unboost_self(enum thread_prio_class class) {
228 struct thread *curr = thread_get_current();
229 struct wroost_kicks kicks;
230
231 {
232 spin_guard_high(&curr->pi_lock);
233
234 thread_class_floor_dec_locked(t: curr, class);
235 kicks = thread_recompute_prio_class_locked(t: curr);
236 }
237
238 wroost_kick(kicks: &kicks);
239}
240
241void thread_remove_boost(void) {
242 climb_handle_remove(h: &thread_get_current()->climb_data.handle);
243}
244