1#include <bootstage_condition.h>
2#include <sch/periodic_work.h>
3#include <sch/sched.h>
4#include <smp/core.h>
5#include <thread/apc.h>
6#include <thread/dpc.h>
7
8enum irql irql_get(void) {
9 return smp_core()->current_irql;
10}
11
12static enum irql irql_set(enum irql irql) {
13 return smp_core()->current_irql = irql;
14}
15
16static inline uint32_t scheduler_preemption_disable(void) {
17 kassert(!are_interrupts_enabled());
18 struct core *cpu = smp_core();
19
20 uint32_t old = cpu->preempt_disable_depth;
21
22 if (old == UINT32_MAX)
23 panic("overflow");
24
25 cpu->preempt_disable_depth++;
26 return old + 1;
27}
28
29static inline uint32_t scheduler_preemption_enable(void) {
30 struct core *cpu = smp_core();
31
32 uint32_t old = cpu->preempt_disable_depth;
33
34 if (old == 0) {
35 panic("underflow");
36 }
37
38 cpu->preempt_disable_depth--;
39 return old - 1;
40}
41
42enum irql irql_raise(enum irql new_level) {
43 BOOTSTAGE_IF_LT(BOOTSTAGE_LATE) {
44 return IRQL_NONE;
45 }
46
47 bool iflag = are_interrupts_enabled();
48 disable_interrupts();
49
50 enum irql old = irql_get();
51
52 irql_set(irql: new_level);
53 if (new_level > old) {
54 if (old < IRQL_APC_LEVEL && new_level >= IRQL_APC_LEVEL)
55 scheduler_preemption_disable();
56
57 if (new_level >= IRQL_HIGH_LEVEL)
58 disable_interrupts();
59
60 } else if (new_level < old) {
61 panic("Raising to lower IRQL, from %s to %s", irql_to_str(old),
62 irql_to_str(new_level));
63 }
64
65 /* ok now we re-enable interrupts if we had disabled them prior */
66 if (iflag && new_level < IRQL_HIGH_LEVEL)
67 enable_interrupts();
68
69 return old;
70}
71
72void irql_lower(enum irql new_level) {
73 BOOTSTAGE_IF_LT(BOOTSTAGE_LATE) {
74 return;
75 }
76
77 kassert(new_level != IRQL_NONE);
78
79 enum irql old = irql_get();
80
81 /* hook into here */
82 if (old == IRQL_DISPATCH_LEVEL && new_level == IRQL_PASSIVE_LEVEL &&
83 !scheduler_in_periodic_work())
84 scheduler_periodic_work_execute(type: PERIODIC_WORK_TIME_BASED);
85
86 struct thread *curr = thread_get_current();
87 bool in_thread = irq_in_thread_context();
88
89 irql_set(irql: new_level);
90 if (new_level < old) {
91 if (in_thread && old >= IRQL_HIGH_LEVEL && new_level < IRQL_HIGH_LEVEL)
92 enable_interrupts();
93
94 if (in_thread && old >= IRQL_DISPATCH_LEVEL &&
95 new_level < IRQL_DISPATCH_LEVEL)
96 dpc_run_local();
97
98 bool preempt_re_enabled = false;
99 if (old >= IRQL_APC_LEVEL && new_level < IRQL_APC_LEVEL)
100 preempt_re_enabled = (scheduler_preemption_enable() == 0);
101
102 if (in_thread && old > IRQL_APC_LEVEL && new_level <= IRQL_APC_LEVEL)
103 apc_check_and_deliver(t: curr);
104
105 if (in_thread && preempt_re_enabled)
106 scheduler_resched_if_needed();
107
108 } else if (new_level > old) {
109 panic("Lowering to higher IRQL, from %s to %s", irql_to_str(old),
110 irql_to_str(new_level));
111 }
112}
113