1#include <bootstage_condition.h>
2#include <sch/periodic_work.h>
3#include <sch/sched.h>
4#include <smp/core.h>
5#include <thread/apc.h>
6#include <thread/dpc.h>
7#include <watchdog.h>
8
9enum irql irql_get(void) {
10 /* We cannot change this away from NONE because
11 * the verification routine relies on this function */
12 return smp_read(TOPC_NONE, current_irql);
13}
14
15static void irql_set(enum irql irql) {
16 /* Internal function */
17 smp_write(TOPC_NONE, current_irql, irql);
18}
19
20static inline uint32_t scheduler_preemption_disable(void) {
21 kassert(!are_interrupts_enabled());
22
23 /* We enforce that interrupts are disabled upon raise, no migration */
24 return smp_ctx_preempt_count(
25 smp_ctx: smp_ctx_add(c: TOPC_NONE, SMP_CTX_PREEMPT_ONE, SMP_CTX_PREEMPT_MASK));
26}
27
28static inline uint32_t scheduler_preemption_enable(void) {
29 /* Similar enforcement simply because this *is* the point where
30 * migrations can happen */
31 return smp_ctx_preempt_count(
32 smp_ctx: smp_ctx_sub(c: TOPC_NONE, SMP_CTX_PREEMPT_ONE, SMP_CTX_PREEMPT_MASK));
33}
34
35enum irql irql_raise(enum irql new_level) {
36 BOOTSTAGE_IF_LT(BOOTSTAGE_LATE) {
37 return IRQL_NONE;
38 }
39
40 bool iflag = are_interrupts_enabled();
41 disable_interrupts();
42
43 enum irql old = irql_get();
44
45 irql_set(irql: new_level);
46 if (new_level > old) {
47 if (old < IRQL_DISPATCH_LEVEL && new_level >= IRQL_DISPATCH_LEVEL)
48 scheduler_preemption_disable();
49
50 if (new_level >= IRQL_HIGH_LEVEL)
51 disable_interrupts();
52
53 } else if (new_level < old) {
54 panic("Raising to lower IRQL, from %s to %s", irql_to_str(old),
55 irql_to_str(new_level));
56 }
57
58 /* ok now we re-enable interrupts if we had disabled them prior */
59 if (iflag && new_level < IRQL_HIGH_LEVEL)
60 enable_interrupts();
61
62 return old;
63}
64
65static void irql_lower_internal(enum irql new_level, bool allow_resched) {
66 BOOTSTAGE_IF_LT(BOOTSTAGE_LATE) {
67 return;
68 }
69
70 if (new_level == IRQL_NONE)
71 return;
72
73 enum irql old = irql_get();
74
75 if (new_level > old)
76 panic("Lowering to higher IRQL, from %s to %s", irql_to_str(old),
77 irql_to_str(new_level));
78
79 if (new_level == old)
80 return;
81
82 bool in_thread = irq_not_in_interrupt();
83 struct thread *curr = thread_get_current();
84
85 if (old >= IRQL_HIGH_LEVEL && new_level < IRQL_HIGH_LEVEL) {
86 enum irql intermediate =
87 (new_level < IRQL_DISPATCH_LEVEL) ? IRQL_DISPATCH_LEVEL : new_level;
88 irql_set(irql: intermediate);
89 if (in_thread)
90 enable_interrupts();
91 }
92
93 if (old >= IRQL_DISPATCH_LEVEL && new_level < IRQL_DISPATCH_LEVEL) {
94 irql_set(irql: IRQL_DISPATCH_LEVEL);
95 if (in_thread)
96 dpc_drain_local();
97
98 watchdog_pet();
99 }
100
101 /* Step down first so current_irql matches before preemption re enables */
102 irql_set(irql: new_level);
103
104 if (old >= IRQL_DISPATCH_LEVEL && new_level < IRQL_DISPATCH_LEVEL)
105 scheduler_preemption_enable();
106
107 if (in_thread && new_level == IRQL_PASSIVE_LEVEL) {
108 if (old >= IRQL_APC_LEVEL)
109 apc_check_and_deliver(t: curr);
110
111 if (allow_resched && !scheduler_preemption_disabled(c: TOPC_NONE))
112 scheduler_resched_if_needed();
113 }
114}
115
116void irql_lower(enum irql new_level) {
117 irql_lower_internal(new_level, /* allow_resched = */ true);
118}
119
120void irql_lower_no_resched(enum irql new_level) {
121 irql_lower_internal(new_level, /* allow_resched = */ false);
122}
123