1#include <bootstage_condition.h>
2#include <math/min_max.h>
3#include <sch/periodic_work.h>
4#include <sch/sched.h>
5#include <smp/core.h>
6#include <thread/apc.h>
7#include <thread/dpc.h>
8#include <watchdog.h>
9
10enum irql irql_get(void) {
11 /* We cannot change this away from NONE because
12 * the verification routine relies on this function */
13 return smp_read(TOPC_NONE, current_irql);
14}
15
16static void irql_set(enum irql irql) {
17 /* Internal function */
18 smp_write(TOPC_NONE, current_irql, irql);
19}
20
21static inline uint32_t scheduler_preemption_disable(void) {
22 enforce(!irqs_enabled(), INVALID_CONTEXT);
23
24 /* We enforce that interrupts are disabled upon raise, no migration */
25 return smp_ctx_preempt_count(
26 smp_ctx: smp_ctx_add(c: TOPC_NONE, SMP_CTX_PREEMPT_ONE, SMP_CTX_PREEMPT_MASK));
27}
28
29static inline uint32_t scheduler_preemption_enable(void) {
30 /* Similar enforcement simply because this *is* the point where
31 * migrations can happen */
32 return smp_ctx_preempt_count(
33 smp_ctx: smp_ctx_sub(c: TOPC_NONE, SMP_CTX_PREEMPT_ONE, SMP_CTX_PREEMPT_MASK));
34}
35
36enum irql irql_raise(enum irql new_level) TSA_NO_ANALYSIS {
37 BOOTSTAGE_IF_LT(BOOTSTAGE_LATE) {
38 return IRQL_NONE;
39 }
40
41 bool iflag = irq_disable_save();
42
43 enum irql old = irql_get();
44
45 irql_set(irql: new_level);
46 if (new_level > old) {
47 if (old < IRQL_DISPATCH_LEVEL && new_level >= IRQL_DISPATCH_LEVEL)
48 scheduler_preemption_disable();
49
50 if (new_level >= IRQL_HIGH_LEVEL)
51 irq_disable();
52
53 } else if (new_level < old) {
54 panic(INVALID_CONTEXT, "Raising to lower IRQL, from %s to %s",
55 irql_to_str(old), irql_to_str(new_level));
56 }
57
58 /* ok now we re-enable interrupts if we had disabled them prior */
59 if (iflag && new_level < IRQL_HIGH_LEVEL)
60 irq_enable();
61
62 return old;
63}
64
65static void irql_lower_internal(enum irql new_level,
66 bool allow_resched) TSA_NO_ANALYSIS {
67 BOOTSTAGE_IF_LT(BOOTSTAGE_LATE) {
68 return;
69 }
70
71 if (new_level == IRQL_NONE)
72 return;
73
74 enum irql old = irql_get();
75
76 if (new_level > old)
77 panic(INVALID_CONTEXT, "Lowering to higher IRQL, from %s to %s",
78 irql_to_str(old), irql_to_str(new_level));
79
80 if (new_level == old)
81 return;
82
83 bool in_thread = irq_not_in_interrupt();
84 struct thread *curr = thread_get_current();
85
86 if (old >= IRQL_HIGH_LEVEL && new_level < IRQL_HIGH_LEVEL) {
87 enum irql intermediate = MAX(new_level, IRQL_DISPATCH_LEVEL);
88 irql_set(irql: intermediate);
89 if (in_thread)
90 irq_enable();
91 }
92
93 if (old >= IRQL_DISPATCH_LEVEL && new_level < IRQL_DISPATCH_LEVEL) {
94 irql_set(irql: IRQL_DISPATCH_LEVEL);
95 if (in_thread)
96 dpc_drain_local();
97
98 watchdog_pet();
99 }
100
101 /* Step down first so current_irql matches before preemption re enables */
102 irql_set(irql: new_level);
103
104 if (old >= IRQL_DISPATCH_LEVEL && new_level < IRQL_DISPATCH_LEVEL)
105 scheduler_preemption_enable();
106
107 if (in_thread && new_level == IRQL_PASSIVE_LEVEL) {
108 if (old >= IRQL_APC_LEVEL)
109 apc_check_and_deliver(t: curr);
110
111 if (allow_resched && !scheduler_preemption_disabled(c: TOPC_NONE))
112 scheduler_resched_if_needed();
113 }
114}
115
116void irql_lower(enum irql new_level) TSA_NO_ANALYSIS {
117 irql_lower_internal(new_level, /* allow_resched = */ true);
118}
119
120void irql_lower_no_resched(enum irql new_level) TSA_NO_ANALYSIS {
121 irql_lower_internal(new_level, /* allow_resched = */ false);
122}
123