1#include <kassert.h>
2#include <sch/sched.h>
3#include <smp/core.h>
4#include <thread/apc.h>
5#include <thread/thread.h>
6
7#include "sch/internal.h"
8
9#define apc_from_list_node(n) container_of(n, struct apc, list)
10
11static inline bool safe_to_exec_apcs(void) {
12 return irql_get() == IRQL_PASSIVE_LEVEL && irq_in_thread_context();
13}
14
15static inline bool thread_has_apcs(struct thread *t) {
16 return t->apc_pending_mask != 0;
17}
18
19static inline size_t apc_type_bit(enum apc_type t) {
20 return (size_t) 1ULL << (size_t) t;
21}
22
23static inline bool apc_queue_empty(struct apc_queue *q) {
24 return q->head == NULL;
25}
26
27static inline void apc_enqueue_tail(struct apc_queue *q, struct apc *a) {
28 a->next = NULL;
29
30 if (!q->head) {
31 q->head = q->tail = a;
32 } else {
33 q->tail->next = a;
34 q->tail = a;
35 }
36}
37
38static inline struct apc *apc_dequeue_head(struct apc_queue *q) {
39 struct apc *a = q->head;
40 if (!a)
41 return NULL;
42
43 q->head = a->next;
44 if (!q->head)
45 q->tail = NULL;
46
47 a->next = NULL;
48 return a;
49}
50
51static inline void apc_add_tail(struct thread *t, struct apc *a,
52 enum apc_type type) {
53 apc_enqueue_tail(q: &t->apc_head[type], a);
54}
55
56static inline bool apc_list_empty(struct thread *t, enum apc_type type) {
57 return apc_queue_empty(q: &t->apc_head[type]);
58}
59
60static inline void apc_unset_bitmask(struct thread *t, enum apc_type type) {
61 atomic_fetch_and(&t->apc_pending_mask, ~apc_type_bit(type));
62}
63
64static inline void apc_set_bitmask(struct thread *t, enum apc_type type) {
65 atomic_fetch_or(&t->apc_pending_mask, apc_type_bit(type));
66}
67
68static inline bool thread_can_exec_special_apcs(struct thread *t) {
69 return t->special_apc_disable == 0 &&
70 (atomic_load(&t->apc_pending_mask) &
71 apc_type_bit(t: APC_TYPE_SPECIAL_KERNEL));
72}
73
74static inline bool thread_can_exec_kernel_apcs(struct thread *t) {
75 return t->kernel_apc_disable == 0 &&
76 (atomic_load(&t->apc_pending_mask) & apc_type_bit(t: APC_TYPE_KERNEL));
77}
78
79static inline bool thread_is_dying(struct thread *t) {
80 enum thread_state s = thread_get_state(t);
81 return s == THREAD_STATE_TERMINATED || s == THREAD_STATE_ZOMBIE;
82}
83
84static bool thread_apc_sanity_check(struct thread *t) {
85 if (unlikely(thread_get_state(t) == THREAD_STATE_IDLE_THREAD))
86 panic("Attempted to put an APC on the idle thread");
87
88 if (unlikely(thread_is_dying(t)))
89 return false;
90
91 return true;
92}
93
94static void apc_execute(struct apc *a) {
95 kassert(irql_get() == IRQL_APC_LEVEL);
96
97 struct thread *curr = thread_get_current();
98
99 thread_or_flags(t: curr, flags: THREAD_FLAG_EXECUTING_APC);
100
101 a->func(a->ctx);
102
103 thread_and_flags(t: curr, flags: ~THREAD_FLAG_EXECUTING_APC);
104 curr->total_apcs_ran++;
105}
106
107static void deliver_apc_type(struct thread *t, enum apc_type type) {
108 while (true) {
109 bool ok;
110 enum irql irql = thread_acquire(t, success: &ok);
111 kassert(ok);
112
113 struct apc *apc = apc_dequeue_head(q: &t->apc_head[type]);
114
115 if (!apc) {
116 apc_unset_bitmask(t, type);
117 thread_release(t, irql);
118 return;
119 }
120
121 apc->owner = NULL;
122
123 thread_release(t, irql);
124
125 apc_execute(a: apc);
126 }
127}
128
129static void add_apc_to_thread(struct thread *t, struct apc *a,
130 enum apc_type type) {
131 a->owner = t;
132 apc_add_tail(t, a, type);
133 apc_set_bitmask(t, type);
134}
135
136static inline bool thread_is_active(struct thread *t) {
137 enum thread_state s = thread_get_state(t);
138 return s == THREAD_STATE_READY || s == THREAD_STATE_RUNNING;
139}
140
141static void maybe_force_resched(struct thread *t) {
142 /* it's ok if the read of tick_enabled races here. if we read it as
143 * `enabled`, it means that it is either truly enabled or is in
144 * the schedule() routine about to disable it, meaning that
145 * if it does get disabled, it'll still have a chance to check
146 * and run the APCs of the only thread active */
147 enum irql irql;
148 struct scheduler *sched = thread_get_scheduler(t, sirql_out: &irql);
149
150 bool needs_resched = !sched->tick_enabled;
151 if (needs_resched)
152 scheduler_force_resched(sched);
153
154 spin_unlock(lock: &sched->lock, old: irql);
155}
156
157static void wake_if_waiting(struct thread *t) {
158 if (thread_is_active(t))
159 maybe_force_resched(t);
160
161 /* Get it running again */
162 if (!thread_apc_sanity_check(t))
163 return;
164
165 /* set the wake_src as the thread that enqueued the APC */
166 scheduler_wake_manual(t, /* wake_src = */ t);
167}
168
169void apc_enqueue(struct thread *t, struct apc *a, enum apc_type type) {
170 if (!thread_apc_sanity_check(t))
171 return;
172
173 bool ok;
174 enum irql irql = thread_acquire(t, success: &ok);
175 if (!ok)
176 return;
177
178 if (a->owner) {
179 thread_release(t, irql);
180 return;
181 }
182
183 add_apc_to_thread(t, a, type);
184 thread_release(t, irql);
185
186 /* Let's go and execute em */
187 if (t == thread_get_current()) {
188 apc_check_and_deliver(t);
189 } else {
190 /* Not us, go wake up the other guy */
191 wake_if_waiting(t);
192 }
193}
194
195/* We can only enqueue and run from ourselves, no sync needed */
196void apc_enqueue_event_apc(struct event_apc *a, struct apc_event_desc *desc) {
197 kassert(desc);
198 kassert(!a->apc.owner);
199
200 a->desc = desc;
201
202 struct thread *t = thread_get_current();
203 if (!thread_apc_sanity_check(t))
204 return;
205
206 apc_enqueue_tail(q: &t->event_apcs, a: &a->apc);
207
208 a->apc.owner = t;
209 apc_set_bitmask(t, type: APC_TYPE_KERNEL);
210}
211
212static bool try_cancel_from_queue(struct thread *t, struct apc *a,
213 enum apc_type type) {
214 struct apc_queue *q = &t->apc_head[type];
215
216 struct apc *prev = NULL;
217 struct apc *curr = q->head;
218
219 while (curr) {
220 struct apc *next = curr->next;
221
222 if (curr == a) {
223 if (prev)
224 prev->next = next;
225 else
226 q->head = next;
227
228 if (q->tail == curr)
229 q->tail = prev;
230
231 curr->next = NULL;
232 curr->owner = NULL;
233
234 return true;
235 }
236
237 prev = curr;
238 curr = next;
239 }
240
241 return false;
242}
243
244/* update pending mask if queue now empty */
245static inline void update_pending_mask(struct thread *t, enum apc_type type) {
246 if (apc_list_empty(t, type))
247 atomic_fetch_and(&t->apc_pending_mask, ~apc_type_bit(type));
248}
249
250bool apc_cancel(struct apc *a) {
251 if (!a || !a->owner)
252 return false;
253
254 struct thread *t = a->owner;
255 bool removed = false;
256 bool ok;
257 enum irql irql = thread_acquire(t, success: &ok);
258 if (!ok)
259 return false;
260
261 for (int type = 0; type < APC_TYPE_COUNT; type++) {
262 removed = try_cancel_from_queue(t, a, type);
263
264 if (removed) {
265 update_pending_mask(t, type);
266 break;
267 }
268 }
269
270 thread_release(t, irql);
271 return removed;
272}
273
274struct apc *apc_create(void) {
275 return kmalloc(sizeof(struct apc));
276}
277
278struct event_apc *apc_event_apc_create(void) {
279 return kmalloc(sizeof(struct event_apc));
280}
281
282void apc_init(struct apc *a, apc_func_t fn, void *arg1) {
283 a->func = fn;
284 a->ctx = arg1;
285 a->next = NULL;
286 a->owner = NULL;
287}
288
289void apc_event_apc_init(struct event_apc *a, apc_func_t fn, void *arg1) {
290 apc_init(a: &a->apc, fn, arg1);
291 a->execute_times = 0;
292}
293
294void apc_free_on_thread(struct thread *t) {
295 struct apc *a;
296
297 while ((a = apc_dequeue_head(q: &t->event_apcs))) {
298 kfree(a);
299 }
300}
301
302static void bump_counters_on_queue(struct apc_queue *from,
303 struct apc_event_desc *desc,
304 struct apc_queue *to) {
305 struct apc *prev = NULL;
306 struct apc *curr = from->head;
307
308 while (curr) {
309 struct apc *next = curr->next;
310 struct event_apc *eapc = container_of(curr, struct event_apc, apc);
311
312 if (eapc->desc == desc) {
313 eapc->execute_times++;
314
315 if (to) {
316 /* unlink */
317 if (prev)
318 prev->next = next;
319 else
320 from->head = next;
321
322 if (from->tail == curr)
323 from->tail = prev;
324
325 /* enqueue into target */
326 curr->next = NULL;
327 apc_enqueue_tail(q: to, a: curr);
328
329 curr = next;
330 continue;
331 }
332 }
333
334 prev = curr;
335 curr = next;
336 }
337}
338
339void apc_event_signal(struct apc_event_desc *desc) {
340 /* here we want to do two things: first, we identify if it is safe to
341 * execute APCs. if it is, it must be guaranteed that the to_execute
342 * tree of event APCs is empty, because the irql_lower that should've
343 * happened would have executed anything on that tree. in this case,
344 * we check our event_apcs tree, and execute anything of relevance
345 * in there. if it is not safe to execute APCs, we will check
346 * the to_execute tree, increment counters for all relevant APCs, and then
347 * check the event_apcs tree, and move anything necessary over */
348 struct thread *curr = thread_get_current();
349
350 if (safe_to_exec_apcs() && curr->kernel_apc_disable == 0) {
351 kassert(apc_queue_empty(&curr->to_exec_event_apcs));
352
353 enum irql irql = irql_raise(new_level: IRQL_APC_LEVEL);
354
355 struct apc *a = curr->event_apcs.head;
356
357 /* This will give us the "first node in a list" that matches our `desc`
358 * value. We can keep going this->right->right to find everyone else to
359 * execute */
360 while (a) {
361 if (container_of(a, struct event_apc, apc)->desc == desc)
362 apc_execute(a);
363
364 a = a->next;
365 }
366
367 irql_lower(old_level: irql);
368 } else {
369 /* Cannot execute APCs right now. Search both trees, bump counters. */
370 bump_counters_on_queue(from: &curr->to_exec_event_apcs, desc, NULL);
371 bump_counters_on_queue(from: &curr->event_apcs, desc,
372 to: &curr->to_exec_event_apcs);
373
374 apc_set_bitmask(t: curr, type: APC_TYPE_KERNEL);
375 }
376}
377
378void thread_exec_event_apcs(struct thread *t) {
379 struct apc *a;
380
381 while ((a = apc_dequeue_head(q: &t->to_exec_event_apcs))) {
382 struct event_apc *eapc = container_of(a, struct event_apc, apc);
383 kassert(eapc->execute_times);
384
385 for (size_t i = 0; i < eapc->execute_times; i++)
386 apc_execute(a);
387
388 eapc->execute_times = 0;
389
390 apc_enqueue_tail(q: &t->event_apcs, a);
391 }
392
393 /* Just in case */
394 apc_unset_bitmask(t, type: APC_TYPE_KERNEL);
395}
396
397void apc_disable_special() {
398 thread_get_current()->special_apc_disable++;
399}
400
401void apc_enable_special() {
402 struct thread *t = thread_get_current();
403 kassert(t->special_apc_disable > 0);
404
405 if (--t->special_apc_disable == 0)
406 apc_check_and_deliver(t);
407}
408
409void apc_disable_kernel() {
410 thread_get_current()->kernel_apc_disable++;
411}
412
413void apc_enable_kernel() {
414 struct thread *t = thread_get_current();
415 kassert(t->kernel_apc_disable > 0);
416
417 if (--t->kernel_apc_disable == 0)
418 apc_check_and_deliver(t);
419}
420
421void thread_exec_apcs(struct thread *t) {
422 if (thread_can_exec_special_apcs(t))
423 deliver_apc_type(t, type: APC_TYPE_SPECIAL_KERNEL);
424
425 if (thread_can_exec_kernel_apcs(t)) {
426 deliver_apc_type(t, type: APC_TYPE_KERNEL);
427 thread_exec_event_apcs(t);
428 }
429}
430
431void apc_check_and_deliver(struct thread *t) {
432 if (!t || !thread_has_apcs(t) || !safe_to_exec_apcs())
433 return;
434
435 enum irql irql = irql_raise(new_level: IRQL_APC_LEVEL);
436
437 thread_exec_apcs(t);
438
439 irql_lower(old_level: irql);
440}
441