1#include <acpi/lapic.h>
2#include <asm.h>
3#include <compiler.h>
4#include <console/printf.h>
5#include <dbg.h>
6#include <global.h>
7#include <irq/exception_sync_cb.h>
8#include <irq/idt.h>
9#include <mem/alloc.h>
10#include <mem/alloc_or_die.h>
11#include <mem/page_fault.h>
12#include <mem/tlb.h>
13#include <mem/vmm.h>
14#include <sch/sched.h>
15#include <smp/core.h>
16#include <smp/smp.h>
17#include <stdbool.h>
18#include <stddef.h>
19#include <stdint.h>
20#include <string.h>
21#include <sync/rcu.h>
22#include <thread/apc.h>
23#include <thread/thread.h>
24
25/* Lock is only used for allocation/free and registering */
26static struct spinlock irq_table_lock = SPINLOCK_INIT;
27static struct irq_desc irq_table[IDT_ENTRIES] = {0};
28static struct exception_sync_cb exception_cbs[IRQ_EXCEPTION_COUNT] = {0};
29static struct idt_table idts = {0};
30static struct idt_ptr idtps = {0};
31
32#include "fault_isrs.h"
33#include "isr_stubs.h"
34#include "isr_vectors_array.h"
35
36void isr_common_entry(uint8_t vector, struct irq_context *irq_ctx) {
37 irq_mark_self_in_interrupt(true);
38
39 enum irql old = irql_raise(new_level: IRQL_HIGH_LEVEL);
40
41 bool is_exception = irq_vector_is_exception(vector);
42 uint8_t scratch_buf[EXCEPTION_SYNC_CB_SCRATCH_BUFFER_SIZE] = {0};
43
44 if (vector != IRQ_NMI)
45 kassert(smp_core()->irq_entered_irql == IRQL_NONE, "Potential race");
46
47 smp_core()->irq_entered_irql = old;
48 smp_core()->irq_stack_scratch_buf = scratch_buf;
49
50 struct irq_desc *desc = &irq_table[vector];
51 if (!desc->present || list_empty(head: &irq_table[vector].actions))
52 panic("Unhandled ISR vector: %u", vector);
53
54 bool handled = false;
55 struct list_head *lh;
56 list_for_each(lh, &desc->actions) {
57 struct irq_action *act = container_of(lh, struct irq_action, list);
58 if (act->handler(act->data, vector, irq_ctx) == IRQ_HANDLED) {
59 handled = true;
60 break;
61 }
62 }
63
64 if (handled && desc->chip && desc->chip->eoi)
65 desc->chip->eoi(desc);
66
67 smp_core()->irq_entered_irql = IRQL_NONE;
68 smp_core()->irq_stack_scratch_buf = NULL;
69
70 /* Here's an odd bit: we'll have very different
71 * behavior if we came from an exception. Namely,
72 * irql_lower will NOT `sti` if irq_in_interrupt()
73 *
74 * Thus, if we are in an exception, we FIRST
75 * irq_mark_self_in_interrupt(false), and the we
76 * irql_lower(old)
77 *
78 * This results in interrupts being enabled
79 * when we come out of irql_lower(), and the
80 * reason why this only applies to exceptions
81 * is because they are synchronous, and more
82 * importantly, have an exception_sync_cb that
83 * can run, but only from this context in which we
84 * are at our old pre-exception IRQL and have
85 * interrupts enabled */
86 if (is_exception) {
87 irq_mark_self_in_interrupt(false);
88 irql_lower(old_level: old);
89
90 struct exception_sync_cb *escb = &exception_cbs[vector];
91 if (escb->fn)
92 escb->fn(escb, irq_ctx, scratch_buf);
93
94 } else {
95 irql_lower(old_level: old);
96 irq_mark_self_in_interrupt(false);
97 }
98
99 /* in reschedule, don't check if we need to preempt */
100 if (scheduler_self_in_resched())
101 return;
102
103 if (!scheduler_preemption_disabled() &&
104 scheduler_mark_self_needs_resched(false)) {
105 struct thread *curr = thread_get_current();
106 if (curr)
107 curr->preemptions++;
108
109 kassert(old != IRQL_DISPATCH_LEVEL);
110 scheduler_yield();
111 }
112}
113
114void irq_register(char *name, uint8_t vector, irq_handler_t handler, void *ctx,
115 enum irq_flags flags) {
116 enum irql irql = spin_lock(lock: &irq_table_lock);
117 struct irq_desc *me = &irq_table[vector];
118
119 bool was = me->present;
120 me->present = true;
121 me->allocated = true;
122 me->enabled = true;
123
124 if (was && !(flags & IRQ_FLAG_SHARED))
125 panic("need to be shared to have many, registered by %s", me->name);
126
127 struct irq_action *act =
128 kmalloc_or_die(sizeof(struct irq_action), ALLOC_FLAGS_ZERO);
129
130 act->handler = handler;
131 INIT_LIST_HEAD(list: &act->list);
132 act->data = ctx;
133
134 list_add_tail(new: &act->list, head: &me->actions);
135 if (!me->name)
136 me->name = name;
137
138 me->flags = flags;
139
140 spin_unlock(lock: &irq_table_lock, old: irql);
141}
142
143void irq_set_chip(uint8_t vec, struct irq_chip *chip, void *data) {
144 enum irql irql = spin_lock(lock: &irq_table_lock);
145
146 if (irq_table[vec].chip && chip)
147 panic("IRQ chip %u exists", vec);
148
149 irq_table[vec].chip = chip;
150 irq_table[vec].chip_data = data;
151
152 spin_unlock(lock: &irq_table_lock, old: irql);
153}
154
155void idt_set_gate(uint8_t num, uint16_t sel, uint8_t flags) {
156 struct idt_entry *idt = idts.entries;
157
158 uint64_t base = (uint64_t) isr_vectors[num];
159
160 idt[num].base_low = (base & 0xFFFF);
161 idt[num].base_mid = (base >> 16) & 0xFFFF;
162 idt[num].base_high = (base >> 32) & 0xFFFFFFFF;
163 idt[num].selector = sel;
164
165 /* TODO: maybe don't hardcode this */
166 if (num == IRQ_NMI || num == IRQ_DBF || num == IRQ_PAGE_FAULT) {
167 idt[num].ist = 1;
168 } else {
169 idt[num].ist = 0;
170 }
171
172 /* debug */
173 idt[num].ist = 0;
174
175 idt[num].flags = flags;
176 idt[num].reserved = 0;
177}
178
179void irq_load(void) {
180 idtps.limit = sizeof(struct idt_entry) * IDT_ENTRIES - 1;
181 idtps.base = (uint64_t) &idts;
182 asm volatile("lidt %0" : : "m"(idtps));
183}
184
185int32_t irq_alloc_entry() {
186 enum irql irql = spin_lock(lock: &irq_table_lock);
187 for (int32_t i = 32; i < IDT_ENTRIES; i++) {
188 if (!irq_table[i].allocated) {
189 irq_table[i].allocated = true;
190 spin_unlock(lock: &irq_table_lock, old: irql);
191 return i;
192 }
193 }
194 spin_unlock(lock: &irq_table_lock, old: irql);
195 return -1;
196}
197
198static void irq_desc_clear(struct irq_desc *desc) {
199 cpu_mask_set_all(m: &desc->masked_cpus);
200 cpu_mask_set_all(m: &desc->affinity);
201 INIT_LIST_HEAD(list: &desc->actions);
202 desc->present = false;
203 desc->allocated = false;
204 desc->enabled = false;
205 desc->flags = 0;
206 desc->name = "none";
207}
208
209void irq_free_entry(int32_t entry) {
210 if (entry < 32 || entry >= IDT_ENTRIES)
211 return;
212
213 enum irql irql = spin_lock(lock: &irq_table_lock);
214
215 struct irq_desc *desc = &irq_table[entry];
216 desc->allocated = false;
217 desc->present = false;
218
219 struct irq_action *iter, *tmp;
220 list_for_each_entry_safe(iter, tmp, &desc->actions, list) {
221 list_del_init(entry: &iter->list);
222 kfree(iter);
223 }
224
225 irq_desc_clear(desc);
226
227 spin_unlock(lock: &irq_table_lock, old: irql);
228}
229
230void irq_disable(irq_t irq) {
231 struct irq_desc *desc = &irq_table[irq];
232 desc->enabled = false;
233 if (desc->chip && desc->chip->mask)
234 desc->chip->mask(desc);
235}
236
237void irq_enable(irq_t irq) {
238 struct irq_desc *desc = &irq_table[irq];
239 desc->enabled = true;
240 if (desc->chip && desc->chip->unmask)
241 desc->chip->unmask(desc);
242}
243
244static void exception_sync_cbs_init() {
245 struct exception_sync_cb *esc;
246 linker_section_for_each_object(esc, exception_sync_cbs) {
247 kassert(exception_cbs[esc->vector].vector == 0);
248 exception_cbs[esc->vector] = *esc;
249 }
250}
251
252void irq_init() {
253 for (size_t i = 0; i < IDT_ENTRIES; i++) {
254 struct irq_desc *desc = &irq_table[i];
255 alloc_or_die(cpu_mask_init(&desc->masked_cpus, global.core_count));
256 alloc_or_die(cpu_mask_init(&desc->affinity, global.core_count));
257
258 desc->vector = i;
259 irq_desc_clear(desc);
260
261 idt_set_gate(num: i, sel: 0x08, flags: 0x8e);
262 }
263
264 exception_sync_cbs_init();
265
266 irq_register(name: "division_by_zero", IRQ_DIV_BY_Z, handler: divbyz_handler, NULL,
267 flags: IRQ_FLAG_NONE);
268 irq_register(name: "debug", IRQ_DEBUG, handler: debug_handler, NULL, flags: IRQ_FLAG_NONE);
269 irq_register(name: "breakpoint", IRQ_BREAKPOINT, handler: breakpoint_handler, NULL,
270 flags: IRQ_FLAG_NONE);
271
272 irq_register(name: "ssf", IRQ_SSF, handler: ss_handler, NULL, flags: IRQ_FLAG_NONE);
273
274 irq_register(name: "gpf", IRQ_GPF, handler: gpf_handler, NULL, flags: IRQ_FLAG_NONE);
275 irq_register(name: "double_fault", IRQ_DBF, handler: double_fault_handler, NULL,
276 flags: IRQ_FLAG_NONE);
277 irq_register(name: "page_fault", IRQ_PAGE_FAULT, handler: page_fault_isr, NULL,
278 flags: IRQ_FLAG_NONE);
279
280 irq_register(name: "timer", IRQ_TIMER, handler: scheduler_timer_isr, NULL, flags: IRQ_FLAG_NONE);
281 irq_set_chip(IRQ_TIMER, chip: lapic_get_chip(), NULL);
282
283 irq_register(name: "nmi", IRQ_NMI, handler: nmi_isr, NULL, flags: IRQ_FLAG_NONE);
284 irq_register(name: "tlb_shootdown", IRQ_TLB_SHOOTDOWN, handler: tlb_shootdown_isr, NULL,
285 flags: IRQ_FLAG_NONE);
286 irq_set_chip(IRQ_TLB_SHOOTDOWN, chip: lapic_get_chip(), NULL);
287
288 irq_register(name: "nop", IRQ_NOP, handler: nop_handler, NULL, flags: IRQ_FLAG_NONE);
289 idt_set_gate(num: 0x80, sel: 0x2b, flags: 0xee);
290 irq_load();
291}
292