1#include <kassert.h>
2#include <sch/sched.h>
3#include <smp/core.h>
4#include <thread/apc.h>
5#include <thread/thread.h>
6
7#include "sch/internal.h"
8#include <mem/alloc.h>
9
10static inline bool safe_to_exec_apcs(void) {
11 if (irql_get() != IRQL_PASSIVE_LEVEL || !irq_not_in_interrupt())
12 return false;
13
14 struct thread *curr = thread_get_current();
15 if (!curr || thread_get_state(t: curr) != THREAD_STATE_RUNNING)
16 return false;
17
18 return true;
19}
20
21static inline enum apc_state apc_state_load(struct apc *a) {
22 return atomic_load_explicit(&a->state, memory_order_acquire);
23}
24
25static inline size_t apc_type_bit(enum apc_type t) {
26 return (size_t) 1ULL << (size_t) t;
27}
28
29static inline bool apc_queue_empty(struct apc_queue *q) {
30 return q->head == NULL;
31}
32
33static inline void apc_enqueue_tail(struct apc_queue *q, struct apc *a) {
34 a->next = NULL;
35
36 if (!q->head) {
37 q->head = q->tail = a;
38 } else {
39 q->tail->next = a;
40 q->tail = a;
41 }
42}
43
44static inline struct apc *apc_dequeue_head(struct apc_queue *q) {
45 struct apc *a = q->head;
46 if (!a)
47 return NULL;
48
49 q->head = a->next;
50 if (!q->head)
51 q->tail = NULL;
52
53 a->next = NULL;
54 return a;
55}
56
57static inline void apc_add_tail(struct thread *t, struct apc *a,
58 enum apc_type type) {
59 apc_enqueue_tail(q: &t->apc_head[type], a);
60}
61
62static inline bool apc_list_empty(struct thread *t, enum apc_type type) {
63 return apc_queue_empty(q: &t->apc_head[type]);
64}
65
66static inline void apc_unset_bitmask(struct thread *t, enum apc_type type) {
67 atomic_fetch_and(&t->apc_pending_mask, ~apc_type_bit(type));
68}
69
70static inline void apc_set_bitmask(struct thread *t, enum apc_type type) {
71 atomic_fetch_or(&t->apc_pending_mask, apc_type_bit(type));
72}
73
74static inline bool thread_can_exec_special_apcs(struct thread *t) {
75 return t->special_apc_disable == 0 &&
76 (atomic_load(&t->apc_pending_mask) &
77 apc_type_bit(t: APC_TYPE_SPECIAL_KERNEL));
78}
79
80static inline bool thread_can_exec_kernel_apcs(struct thread *t) {
81 return t->kernel_apc_disable == 0 &&
82 (atomic_load(&t->apc_pending_mask) & apc_type_bit(t: APC_TYPE_KERNEL));
83}
84
85static inline bool thread_is_dying(struct thread *t) {
86 if (thread_get_flags(t) & THREAD_FLAG_DYING)
87 return true;
88 enum thread_state s = thread_get_state(t);
89 return s == THREAD_STATE_TERMINATED || s == THREAD_STATE_ZOMBIE;
90}
91
92static bool thread_apc_sanity_check(struct thread *t) {
93 if (unlikely(thread_get_state(t) == THREAD_STATE_IDLE_THREAD))
94 panic("Attempted to put an APC on the idle thread");
95
96 if (unlikely(thread_is_dying(t)))
97 return false;
98
99 return true;
100}
101
102static void apc_execute(struct apc *a) {
103 kassert(irql_get() == IRQL_APC_LEVEL);
104
105 struct thread *curr = thread_get_current();
106
107 thread_or_flags(t: curr, flags: THREAD_FLAG_EXECUTING_APC);
108
109 a->func(a->ctx);
110
111 thread_and_flags(t: curr, flags: ~THREAD_FLAG_EXECUTING_APC);
112 curr->total_apcs_ran++;
113}
114
115static void deliver_apc_type(struct thread *t, enum apc_type type) {
116 while (true) {
117 bool ok;
118 enum irql irql = thread_acquire(t, success: &ok);
119 if (!ok)
120 return;
121
122 struct apc *apc = apc_dequeue_head(q: &t->apc_head[type]);
123
124 if (!apc) {
125 apc_unset_bitmask(t, type);
126 thread_release(t, irql);
127 return;
128 }
129
130 kassert(apc->owner == t);
131 kassert(apc_state_load(apc) == APC_STATE_QUEUED);
132 apc->owner = NULL;
133 atomic_store_explicit(&apc->state, APC_STATE_EXECUTING,
134 memory_order_release);
135
136 thread_release(t, irql);
137
138 apc_execute(a: apc);
139 atomic_store_explicit(&apc->state, APC_STATE_IDLE,
140 memory_order_release);
141 apc_put(a: apc);
142 }
143}
144
145static void add_apc_to_thread(struct thread *t, struct apc *a,
146 enum apc_type type) {
147 a->owner = t;
148 apc_add_tail(t, a, type);
149 apc_set_bitmask(t, type);
150}
151
152static inline bool thread_is_active(struct thread *t) {
153 enum thread_state s = thread_get_state(t);
154 return s == THREAD_STATE_READY || s == THREAD_STATE_RUNNING;
155}
156
157static void maybe_force_resched(struct thread *t) {
158 enum irql irql;
159 struct scheduler *sched = thread_get_scheduler(t, sirql_out: &irql);
160
161 scheduler_force_resched(sched);
162
163 spin_unlock(&sched->lock, irql);
164}
165
166static void wake_if_waiting(struct thread *t) {
167 if (thread_is_active(t))
168 maybe_force_resched(t);
169
170 /* Get it running again */
171 if (!thread_apc_sanity_check(t))
172 return;
173
174 /* set the wake_src as the thread that enqueued the APC */
175 scheduler_wake_manual(t, /* wake_src = */ t);
176}
177
178bool apc_enqueue(struct thread *t, struct apc *a, enum apc_type type) {
179 if (!t || !a || type >= APC_TYPE_COUNT || !thread_apc_sanity_check(t))
180 return false;
181
182 bool ok;
183 enum irql irql = thread_acquire(t, success: &ok);
184 if (!ok)
185 return false;
186
187 if (!apc_get(a)) {
188 thread_release(t, irql);
189 return false;
190 }
191
192 enum apc_state expected = APC_STATE_IDLE;
193 if (!atomic_compare_exchange_strong_explicit(
194 &a->state, &expected, APC_STATE_QUEUED, memory_order_acq_rel,
195 memory_order_acquire)) {
196 thread_release(t, irql);
197 apc_put(a);
198 return false;
199 }
200
201 kassert(a->owner == NULL);
202 add_apc_to_thread(t, a, type);
203 thread_release(t, irql);
204
205 /* Let's go and execute em */
206 if (t == thread_get_current()) {
207 apc_check_and_deliver(t);
208 } else {
209 /* Not us, go wake up the other guy */
210 wake_if_waiting(t);
211 }
212 return true;
213}
214
215/* We can only enqueue and run from ourselves, no sync needed */
216bool apc_enqueue_event_apc(struct event_apc *a, struct apc_event_desc *desc) {
217 kassert(desc);
218 if (!a || !apc_get(a: &a->apc))
219 return false;
220
221 enum apc_state expected = APC_STATE_IDLE;
222 if (!atomic_compare_exchange_strong_explicit(
223 &a->apc.state, &expected, APC_STATE_QUEUED, memory_order_acq_rel,
224 memory_order_acquire)) {
225 apc_put(a: &a->apc);
226 return false;
227 }
228
229 kassert(!a->apc.owner);
230
231 a->desc = desc;
232
233 struct thread *t = thread_get_current();
234 if (!thread_apc_sanity_check(t)) {
235 atomic_store_explicit(&a->apc.state, APC_STATE_IDLE,
236 memory_order_release);
237 apc_put(a: &a->apc);
238 return false;
239 }
240
241 apc_enqueue_tail(q: &t->event_apcs, a: &a->apc);
242
243 a->apc.owner = t;
244 apc_set_bitmask(t, type: APC_TYPE_KERNEL);
245 return true;
246}
247
248static bool try_cancel_from_queue(struct thread *t, struct apc *a,
249 enum apc_type type) {
250 struct apc_queue *q = &t->apc_head[type];
251
252 struct apc *prev = NULL;
253 struct apc *curr = q->head;
254
255 while (curr) {
256 struct apc *next = curr->next;
257
258 if (curr == a) {
259 kassert(curr->owner == t);
260 kassert(apc_state_load(curr) == APC_STATE_QUEUED);
261 if (prev)
262 prev->next = next;
263 else
264 q->head = next;
265
266 if (q->tail == curr)
267 q->tail = prev;
268
269 curr->next = NULL;
270 curr->owner = NULL;
271
272 return true;
273 }
274
275 prev = curr;
276 curr = next;
277 }
278
279 return false;
280}
281
282/* update pending mask if queue now empty */
283static inline void update_pending_mask(struct thread *t, enum apc_type type) {
284 if (apc_list_empty(t, type))
285 atomic_fetch_and(&t->apc_pending_mask, ~apc_type_bit(type));
286}
287
288bool apc_cancel(struct thread *t, struct apc *a) {
289 if (!t || !a)
290 return false;
291
292 bool removed = false;
293 bool ok;
294 enum irql irql = thread_acquire(t, success: &ok);
295 if (!ok)
296 return false;
297
298 for (int type = 0; type < APC_TYPE_COUNT; type++) {
299 removed = try_cancel_from_queue(t, a, type);
300
301 if (removed) {
302 update_pending_mask(t, type);
303 break;
304 }
305 }
306
307 thread_release(t, irql);
308 if (removed) {
309 atomic_store_explicit(&a->state, APC_STATE_IDLE, memory_order_release);
310 apc_put(a);
311 }
312 return removed;
313}
314
315struct apc *apc_create(void) {
316 return kmalloc(sizeof(struct apc));
317}
318
319struct event_apc *apc_event_apc_create(void) {
320 return kmalloc(sizeof(struct event_apc));
321}
322
323void apc_init(struct apc *a, apc_func_t fn, void *arg1, apc_destroy_t destroy) {
324 kassert(a);
325 kassert(fn);
326 a->func = fn;
327 a->ctx = arg1;
328 a->next = NULL;
329 a->owner = NULL;
330 refcount_init(rc: &a->refcount, val: 1);
331 atomic_store_explicit(&a->state, APC_STATE_IDLE, memory_order_relaxed);
332 a->destroy = destroy;
333}
334
335void apc_event_apc_init(struct event_apc *a, apc_func_t fn, void *arg1,
336 apc_destroy_t destroy) {
337 apc_init(a: &a->apc, fn, arg1, destroy);
338 a->execute_times = 0;
339}
340
341bool apc_get(struct apc *a) {
342 return a && refcount_inc_not_zero(rc: &a->refcount);
343}
344
345void apc_put(struct apc *a) {
346 kassert(a);
347 if (!refcount_dec_and_test(rc: &a->refcount))
348 return;
349
350 kassert(apc_state_load(a) == APC_STATE_IDLE);
351 kassert(a->owner == NULL);
352 kassert(a->next == NULL);
353 if (a->destroy)
354 a->destroy(a);
355}
356
357void apc_destroy_free(struct apc *a) {
358 kfree(a);
359}
360
361static void apc_queue_splice(struct apc_queue *from, struct apc_queue *to) {
362 struct apc *a;
363
364 while ((a = apc_dequeue_head(q: from)))
365 apc_enqueue_tail(q: to, a);
366}
367
368void apc_rundown_thread(struct thread *t) {
369 struct apc_queue drained = {0};
370
371 /* Every other queue mutator will hold t->lock,
372 * and rundown has to do that too */
373 enum irql irql = spin_lock_irq_disable(&t->lock);
374
375 for (size_t type = 0; type < APC_TYPE_COUNT; type++)
376 apc_queue_splice(from: &t->apc_head[type], to: &drained);
377
378 apc_queue_splice(from: &t->event_apcs, to: &drained);
379 apc_queue_splice(from: &t->to_exec_event_apcs, to: &drained);
380 atomic_store_explicit(&t->apc_pending_mask, 0, memory_order_release);
381
382 spin_unlock(&t->lock, irql);
383
384 /* We can't run the teardown from the critical section */
385 struct apc *a;
386 while ((a = apc_dequeue_head(q: &drained))) {
387 a->owner = NULL;
388 atomic_store_explicit(&a->state, APC_STATE_IDLE, memory_order_release);
389 apc_put(a);
390 }
391}
392
393static void bump_counters_on_queue(struct apc_queue *from,
394 struct apc_event_desc *desc,
395 struct apc_queue *to) {
396 struct apc *prev = NULL;
397 struct apc *curr = from->head;
398
399 while (curr) {
400 struct apc *next = curr->next;
401 struct event_apc *eapc = container_of(curr, struct event_apc, apc);
402
403 if (eapc->desc == desc) {
404 eapc->execute_times++;
405
406 if (to) {
407 /* unlink */
408 if (prev)
409 prev->next = next;
410 else
411 from->head = next;
412
413 if (from->tail == curr)
414 from->tail = prev;
415
416 /* enqueue into target */
417 curr->next = NULL;
418 apc_enqueue_tail(q: to, a: curr);
419
420 curr = next;
421 continue;
422 }
423 }
424
425 prev = curr;
426 curr = next;
427 }
428}
429
430static void apc_execute_event(struct apc *a) {
431 kassert(apc_state_load(a) == APC_STATE_QUEUED);
432 atomic_store_explicit(&a->state, APC_STATE_EXECUTING, memory_order_release);
433 apc_execute(a);
434 atomic_store_explicit(&a->state, APC_STATE_QUEUED, memory_order_release);
435}
436
437void apc_event_signal(struct apc_event_desc *desc) {
438 /* here we want to do two things: first, we identify if it is safe to
439 * execute APCs. if it is, it must be guaranteed that the to_execute
440 * tree of event APCs is empty, because the irql_lower that should've
441 * happened would have executed anything on that tree. in this case,
442 * we check our event_apcs tree, and execute anything of relevance
443 * in there. if it is not safe to execute APCs, we will check
444 * the to_execute tree, increment counters for all relevant APCs, and then
445 * check the event_apcs tree, and move anything necessary over */
446 struct thread *curr = thread_get_current();
447
448 if (safe_to_exec_apcs() && curr->kernel_apc_disable == 0) {
449 kassert(apc_queue_empty(&curr->to_exec_event_apcs));
450
451 enum irql irql = irql_raise(new_level: IRQL_APC_LEVEL);
452
453 struct apc *a = curr->event_apcs.head;
454
455 /* This will give us the "first node in a list" that matches our `desc`
456 * value. We can keep going this->right->right to find everyone else to
457 * execute */
458 while (a) {
459 if (container_of(a, struct event_apc, apc)->desc == desc)
460 apc_execute_event(a);
461
462 a = a->next;
463 }
464
465 irql_lower(old_level: irql);
466 } else {
467 /* Cannot execute APCs right now. Search both trees, bump counters. */
468 bump_counters_on_queue(from: &curr->to_exec_event_apcs, desc, NULL);
469 bump_counters_on_queue(from: &curr->event_apcs, desc,
470 to: &curr->to_exec_event_apcs);
471
472 apc_set_bitmask(t: curr, type: APC_TYPE_KERNEL);
473 }
474}
475
476void thread_exec_event_apcs(struct thread *t) {
477 struct apc *a;
478
479 while ((a = apc_dequeue_head(q: &t->to_exec_event_apcs))) {
480 struct event_apc *eapc = container_of(a, struct event_apc, apc);
481 kassert(eapc->execute_times);
482
483 for (size_t i = 0; i < eapc->execute_times; i++)
484 apc_execute_event(a);
485
486 eapc->execute_times = 0;
487
488 apc_enqueue_tail(q: &t->event_apcs, a);
489 }
490
491 /* Just in case */
492 apc_unset_bitmask(t, type: APC_TYPE_KERNEL);
493}
494
495void apc_disable_special() {
496 thread_get_current()->special_apc_disable++;
497}
498
499void apc_enable_special() {
500 struct thread *t = thread_get_current();
501 kassert(t->special_apc_disable > 0);
502
503 if (--t->special_apc_disable == 0)
504 apc_check_and_deliver(t);
505}
506
507void apc_disable_kernel() {
508 thread_get_current()->kernel_apc_disable++;
509}
510
511void apc_enable_kernel() {
512 struct thread *t = thread_get_current();
513 kassert(t->kernel_apc_disable > 0);
514
515 if (--t->kernel_apc_disable == 0)
516 apc_check_and_deliver(t);
517}
518
519static inline bool thread_can_exec_any_apcs(struct thread *t) {
520 return thread_can_exec_special_apcs(t) || thread_can_exec_kernel_apcs(t);
521}
522
523void thread_exec_apcs(struct thread *t) {
524 if (thread_can_exec_special_apcs(t))
525 deliver_apc_type(t, type: APC_TYPE_SPECIAL_KERNEL);
526
527 if (thread_can_exec_kernel_apcs(t)) {
528 deliver_apc_type(t, type: APC_TYPE_KERNEL);
529 thread_exec_event_apcs(t);
530 }
531}
532
533void apc_check_and_deliver(struct thread *t) {
534 if (!t || !safe_to_exec_apcs() || !thread_can_exec_any_apcs(t))
535 return;
536
537 if (thread_get_flags(t) & (THREAD_FLAG_EXECUTING_APC | THREAD_FLAG_DYING))
538 return;
539
540 enum irql irql = irql_raise(new_level: IRQL_APC_LEVEL);
541
542 thread_exec_apcs(t);
543
544 irql_lower(old_level: irql);
545}
546