1#include <compiler/core.h>
2#include <compiler/intrinsic.h>
3#include <enforce.h>
4#include <math/bit.h>
5#include <mem/alloc.h>
6#include <sch/sched.h>
7#include <smp/core.h>
8#include <thread/apc.h>
9#include <thread/thread.h>
10#include <thread/thread_diag.h>
11
12#include "sch/internal.h"
13
14static inline bool safe_to_exec_apcs(void) {
15 if (irql_get() != IRQL_PASSIVE_LEVEL || !irq_not_in_interrupt())
16 return false;
17
18 struct thread *curr = thread_get_current();
19 if (!curr)
20 return false;
21
22 enum thread_state s = thread_get_state(t: curr);
23 if (s == THREAD_STATE_RUNNING)
24 return true;
25
26 if ((s == THREAD_STATE_SLEEPING || s == THREAD_STATE_BLOCKED) &&
27 !(thread_get_flags(t: curr) & THREAD_FLAG_YIELDED))
28 return true;
29
30 return false;
31}
32
33static inline enum apc_state apc_state_load(struct apc *a) {
34 return atomic_load_acq(&a->state);
35}
36
37static inline bool apc_queue_empty(struct apc_queue *q) {
38 return q->head == NULL;
39}
40
41static inline void apc_enqueue_tail(struct apc_queue *q, struct apc *a) {
42 a->next = NULL;
43
44 if (!q->head) {
45 q->head = q->tail = a;
46 } else {
47 q->tail->next = a;
48 q->tail = a;
49 }
50}
51
52static inline struct apc *apc_dequeue_head(struct apc_queue *q) {
53 struct apc *a = q->head;
54 if (!a)
55 return NULL;
56
57 q->head = a->next;
58 if (!q->head)
59 q->tail = NULL;
60
61 a->next = NULL;
62 return a;
63}
64
65static inline void apc_add_tail(struct thread *t, struct apc *a,
66 enum apc_type type) {
67 apc_enqueue_tail(q: &t->apc_head[type], a);
68}
69
70static inline bool apc_list_empty(struct thread *t, enum apc_type type) {
71 return apc_queue_empty(q: &t->apc_head[type]);
72}
73
74static inline void apc_unset_bitmask(struct thread *t, enum apc_type type) {
75 atomic_clear_bit(&t->apc_pending_mask, type);
76}
77
78static inline void apc_set_bitmask(struct thread *t, enum apc_type type) {
79 atomic_set_bit(&t->apc_pending_mask, type);
80}
81
82static inline bool thread_can_exec_special_apcs(struct thread *t) {
83 return t->special_apc_disable == 0 &&
84 atomic_test_bit(&t->apc_pending_mask, APC_TYPE_SPECIAL_KERNEL);
85}
86
87static inline bool thread_can_exec_kernel_apcs(struct thread *t) {
88 return t->kernel_apc_disable == 0 &&
89 atomic_test_bit(&t->apc_pending_mask, APC_TYPE_KERNEL);
90}
91
92static inline bool thread_is_dying(struct thread *t) {
93 if (thread_get_flags(t) & THREAD_FLAG_DYING)
94 return true;
95 enum thread_state s = thread_get_state(t);
96 return s == THREAD_STATE_TERMINATED || s == THREAD_STATE_ZOMBIE;
97}
98
99static bool thread_apc_sanity_check(struct thread *t) {
100 if (cc_unlikely(thread_get_state(t) == THREAD_STATE_IDLE_THREAD))
101 panic("Attempted to put an APC on the idle thread");
102
103 if (cc_unlikely(thread_is_dying(t)))
104 return false;
105
106 return true;
107}
108
109static void apc_execute(struct apc *a) {
110 enforce(irql_get() == IRQL_APC_LEVEL, INVALID_CONTEXT);
111
112 struct thread *curr = thread_get_current();
113
114 thread_or_flags(t: curr, flags: THREAD_FLAG_EXECUTING_APC);
115
116 struct apc_context ac = {.apc = a};
117 a->func(&ac, a->ctx);
118
119 thread_and_flags(t: curr, flags: ~THREAD_FLAG_EXECUTING_APC);
120
121#ifdef DEBUG_THREAD
122 curr->total_apcs_ran++;
123#endif
124}
125
126/* TODO: Work on this budget thing */
127#define APC_DELIVER_BUDGET 32
128
129static void deliver_apc_type(struct thread *t, enum apc_type type) {
130 uint32_t budget = APC_DELIVER_BUDGET;
131 uint64_t drained = 0;
132
133 while (budget--) {
134 thread_diag_apc_deliver_batch(t, drained);
135
136 bool ok;
137 enum irql irql = thread_acquire(t, success: &ok);
138 if (!ok)
139 return;
140
141 struct apc *apc = apc_dequeue_head(q: &t->apc_head[type]);
142
143 if (!apc) {
144 apc_unset_bitmask(t, type);
145 thread_release(t, irql);
146 return;
147 }
148
149 enforce(apc->owner == t);
150 enforce(apc_state_load(apc) == APC_STATE_QUEUED);
151 apc->owner = NULL;
152 atomic_store_release(&apc->state, APC_STATE_EXECUTING);
153
154 thread_release(t, irql);
155
156 apc_execute(a: apc);
157 atomic_store_release(&apc->state, APC_STATE_IDLE);
158 apc_putref(a: apc);
159 drained++;
160 }
161}
162
163static void add_apc_to_thread(struct thread *t, struct apc *a,
164 enum apc_type type) {
165 a->owner = t;
166 apc_add_tail(t, a, type);
167 apc_set_bitmask(t, type);
168}
169
170static void wake_if_waiting(struct thread *t) {
171 if (thread_apc_sanity_check(t))
172 scheduler_resume_for_apc(t);
173}
174
175bool apc_enqueue(struct thread *t, struct apc *a, enum apc_type type) {
176 if (!t || !a || type >= APC_TYPE_COUNT || !thread_apc_sanity_check(t))
177 return false;
178
179 bool ok;
180 enum irql irql = thread_acquire(t, success: &ok);
181 if (!ok)
182 return false;
183
184 if (thread_is_dying(t)) {
185 thread_release(t, irql);
186 return false;
187 }
188
189 if (!apc_getref(a)) {
190 thread_release(t, irql);
191 return false;
192 }
193
194 enum apc_state expected = APC_STATE_IDLE;
195 if (!atomic_cas_strong(&a->state, &expected, APC_STATE_QUEUED, mo_acq_rel,
196 mo_acquire)) {
197 thread_release(t, irql);
198 apc_putref(a);
199 return false;
200 }
201
202 enforce(a->owner == NULL);
203 add_apc_to_thread(t, a, type);
204 thread_release(t, irql);
205
206 /* Let's go and execute em */
207 if (t == thread_get_current()) {
208 apc_check_and_deliver(t);
209 } else {
210 /* Not us, go wake up the other guy */
211 wake_if_waiting(t);
212 }
213 return true;
214}
215
216static bool try_cancel_from_queue(struct thread *t, struct apc *a,
217 enum apc_type type) {
218 struct apc_queue *q = &t->apc_head[type];
219
220 struct apc *prev = NULL;
221 struct apc *curr = q->head;
222
223 while (curr) {
224 struct apc *next = curr->next;
225
226 if (curr == a) {
227 enforce(curr->owner == t);
228 enforce(apc_state_load(curr) == APC_STATE_QUEUED);
229 if (prev)
230 prev->next = next;
231 else
232 q->head = next;
233
234 if (q->tail == curr)
235 q->tail = prev;
236
237 curr->next = NULL;
238 curr->owner = NULL;
239
240 return true;
241 }
242
243 prev = curr;
244 curr = next;
245 }
246
247 return false;
248}
249
250/* update pending mask if queue now empty */
251static inline void update_pending_mask(struct thread *t, enum apc_type type) {
252 if (apc_list_empty(t, type))
253 atomic_clear_bit(&t->apc_pending_mask, type);
254}
255
256bool apc_cancel(struct thread *t, struct apc *a) {
257 if (!t || !a)
258 return false;
259
260 bool removed = false;
261 bool ok;
262 enum irql irql = thread_acquire(t, success: &ok);
263 if (!ok)
264 return false;
265
266 for (int type = 0; type < APC_TYPE_COUNT; type++) {
267 removed = try_cancel_from_queue(t, a, type);
268
269 if (removed) {
270 update_pending_mask(t, type);
271 break;
272 }
273 }
274
275 thread_release(t, irql);
276 if (removed) {
277 atomic_store_release(&a->state, APC_STATE_IDLE);
278 apc_putref(a);
279 }
280 return removed;
281}
282
283struct apc *apc_create(void) {
284 struct apc *a = vnew(a);
285 return a;
286}
287
288void apc_init(struct apc *cc_nonnull_ptr a, apc_func_t cc_nonnull_ptr fn,
289 void *arg1, apc_destroy_t destroy) {
290 enforce_precond(a);
291 enforce_precond(fn);
292 a->func = fn;
293 a->ctx = arg1;
294 a->next = NULL;
295 a->owner = NULL;
296 refcount_init(rc: &a->refcount, val: 1);
297 atomic_store_rlx(&a->state, APC_STATE_IDLE);
298 a->destroy = destroy;
299}
300
301bool apc_getref(struct apc *a) {
302 return a && refcount_inc_not_zero(rc: &a->refcount);
303}
304
305void apc_putref(struct apc *cc_nonnull_ptr a) {
306 enforce_precond(a);
307 if (!refcount_dec_and_test(rc: &a->refcount))
308 return;
309
310 enforce(apc_state_load(a) == APC_STATE_IDLE);
311 enforce(a->owner == NULL);
312 enforce(a->next == NULL);
313 if (a->destroy)
314 a->destroy(a);
315}
316
317void apc_destroy_free(struct apc *a) {
318 vfree(a);
319}
320
321static void apc_queue_splice(struct apc_queue *from, struct apc_queue *to) {
322 struct apc *a;
323
324 while ((a = apc_dequeue_head(q: from)))
325 apc_enqueue_tail(q: to, a);
326}
327
328void apc_rundown_thread(struct thread *t) {
329 struct apc_queue drained = {0};
330
331 /* Every other queue mutator will hold t->lock,
332 * and rundown has to do that too */
333 enum irql irql = spin_lock_high(&t->lock);
334
335 for (size_t type = 0; type < APC_TYPE_COUNT; type++)
336 apc_queue_splice(from: &t->apc_head[type], to: &drained);
337
338 atomic_store_release(&t->apc_pending_mask, 0);
339
340 spin_unlock(&t->lock, irql);
341
342 /* We can't run the teardown from the critical section */
343 struct apc *a;
344 while ((a = apc_dequeue_head(q: &drained))) {
345 a->owner = NULL;
346 atomic_store_release(&a->state, APC_STATE_IDLE);
347 apc_putref(a);
348 }
349}
350
351void apc_disable_special(void) {
352 thread_get_current()->special_apc_disable++;
353}
354
355void apc_enable_special(void) {
356 struct thread *t = thread_get_current();
357 enforce(t->special_apc_disable > 0, UNDERFLOW);
358
359 if (--t->special_apc_disable == 0)
360 apc_check_and_deliver(t);
361}
362
363void apc_disable_kernel(void) {
364 thread_get_current()->kernel_apc_disable++;
365}
366
367void apc_enable_kernel(void) {
368 struct thread *t = thread_get_current();
369 enforce(t->kernel_apc_disable > 0, UNDERFLOW);
370
371 if (--t->kernel_apc_disable == 0)
372 apc_check_and_deliver(t);
373}
374
375static inline bool thread_can_exec_any_apcs(struct thread *t) {
376 return thread_can_exec_special_apcs(t) || thread_can_exec_kernel_apcs(t);
377}
378
379static void thread_exec_apcs(struct thread *t) {
380 if (thread_can_exec_special_apcs(t))
381 deliver_apc_type(t, type: APC_TYPE_SPECIAL_KERNEL);
382
383 if (thread_can_exec_kernel_apcs(t))
384 deliver_apc_type(t, type: APC_TYPE_KERNEL);
385}
386
387void apc_check_and_deliver(struct thread *t) {
388 if (!t || !safe_to_exec_apcs() || !thread_can_exec_any_apcs(t))
389 return;
390
391 if (thread_get_flags(t) & (THREAD_FLAG_EXECUTING_APC |
392 THREAD_FLAG_DELIVERING_APCS | THREAD_FLAG_DYING))
393 return;
394
395 /* Which IRQL-lowering site keeps handing this thread back to delivery */
396 thread_diag_apc_deliver_enter(t, ci_return_address(0));
397
398 thread_or_flags(t, flags: THREAD_FLAG_DELIVERING_APCS);
399
400 enum irql irql = irql_raise(new_level: IRQL_APC_LEVEL);
401
402 thread_exec_apcs(t);
403
404 irql_lower(old_level: irql);
405
406 thread_and_flags(t, flags: ~THREAD_FLAG_DELIVERING_APCS);
407}
408