1#include <acpi/lapic.h>
2#include <asm.h>
3#include <compiler.h>
4#include <console/printf.h>
5#include <dbg.h>
6#include <global.h>
7#include <irq/exception_sync_cb.h>
8#include <irq/idt.h>
9#include <mem/alloc.h>
10#include <mem/alloc_or_die.h>
11#include <mem/page_fault.h>
12#include <mem/tlb.h>
13#include <mem/vmm.h>
14#include <sch/sched.h>
15#include <smp/core.h>
16#include <smp/smp.h>
17#include <stdbool.h>
18#include <stddef.h>
19#include <stdint.h>
20#include <string.h>
21#include <sync/rcu.h>
22#include <thread/apc.h>
23#include <thread/thread.h>
24#include <time/timer.h>
25#include <watchdog.h>
26
27/* Lock is only used for allocation/free and registering */
28static struct spinlock irq_table_lock = SPINLOCK_INIT;
29static struct irq_desc irq_table[IDT_ENTRIES] = {0};
30static struct exception_sync_cb exception_cbs[IRQ_EXCEPTION_COUNT] = {0};
31static struct idt_table idts = {0};
32static struct idt_ptr idtps = {0};
33
34#include "fault_isrs.h"
35#include "isr_stubs.h"
36#include "isr_vectors_array.h"
37
38static void irq_execute_vector_handlers(irq_t vector,
39 struct irq_context *irq_ctx) {
40 struct irq_desc *desc = &irq_table[vector];
41 if (!desc->present || list_empty(head: &irq_table[vector].actions))
42 panic("Unhandled ISR vector: %u", vector);
43
44 bool handled = false;
45 struct list_head *lh;
46 list_for_each(lh, &desc->actions) {
47 struct irq_action *act = container_of(lh, struct irq_action, list);
48 if (act->handler(act->data, vector, irq_ctx) == IRQ_HANDLED) {
49 if (vector != IRQ_NMI)
50 watchdog_pet();
51 handled = true;
52 break;
53 }
54 }
55
56 if (handled && desc->chip && desc->chip->eoi)
57 desc->chip->eoi(desc);
58}
59
60/* TODO: Someday we will need to handle nmi_depth > 1 and do a fancy asm
61 * trampoline to handle cases where IRQs fire in NMIs due to the IRET issue */
62void isr_nmi_entry(struct irq_context *irq_ctx) {
63 /* vector == IRQ_NMI, of course */
64 kassert(!irq_mark_self_in_nmi(true));
65
66 irq_execute_vector_handlers(IRQ_NMI, irq_ctx);
67
68 kassert(irq_mark_self_in_nmi(false) == 1);
69}
70
71void isr_standard_entry(irq_t vector, struct irq_context *irq_ctx) {
72 irq_mark_self_in_interrupt(true);
73
74 enum irql old = irql_raise(new_level: IRQL_HIGH_LEVEL);
75
76 bool is_exception = irq_vector_is_exception(vector);
77 uint8_t scratch_buf[EXCEPTION_SYNC_CB_SCRATCH_BUFFER_SIZE] = {0};
78
79 if (!is_exception)
80 kassert(old != IRQL_HIGH_LEVEL);
81
82 /* All IRQ scoped, _raw is fine */
83 kassert(smp_core_raw()->irq_entered_irql == IRQL_NONE, "Potential race");
84
85 smp_core_raw()->irq_entered_irql = old;
86 smp_core_raw()->irq_stack_scratch_buf = scratch_buf;
87
88 irq_execute_vector_handlers(vector, irq_ctx);
89
90 smp_core_raw()->irq_entered_irql = IRQL_NONE;
91 smp_core_raw()->irq_stack_scratch_buf = NULL;
92
93 /* We explicitly exclude exceptions, since the context
94 * might be anywhere, including within RCU itself */
95 if (!is_exception)
96 rcu_note_irq_exit();
97
98 /* Here's an odd bit: we'll have very different
99 * behavior if we came from an exception. Namely,
100 * irql_lower will NOT `sti` if irq_in_interrupt()
101 *
102 * Thus, if we are in an exception, we FIRST
103 * irq_mark_self_in_interrupt(false), and the we
104 * irql_lower(old)
105 *
106 * This results in interrupts being enabled
107 * when we come out of irql_lower(), and the
108 * reason why this only applies to exceptions
109 * is because they are synchronous, and more
110 * importantly, have an exception_sync_cb that
111 * can run, but only from this context in which we
112 * are at our old pre-exception IRQL and have
113 * interrupts enabled */
114 if (is_exception) {
115 irq_mark_self_in_interrupt(false);
116 irql_lower(old_level: old);
117
118 struct exception_sync_cb *escb = &exception_cbs[vector];
119 if (escb->fn)
120 escb->fn(escb, irq_ctx, scratch_buf);
121
122 } else {
123 irql_lower(old_level: old);
124 irq_mark_self_in_interrupt(false);
125 }
126
127 /* This function just raises and lowers the IRQL, because
128 * now we're irq_mark_self_in_interrupt == false */
129 if (scheduler_mark_self_needs_run_dpcs(false))
130 dpc_run_dpcs_from_irq();
131
132 /* in reschedule, don't check if we need to preempt */
133 if (scheduler_self_in_resched())
134 return;
135
136 if (scheduler_yield_nesting(t: thread_get_current()) != 0)
137 return;
138
139 if (!scheduler_preemption_disabled(c: TOPC_NONE) &&
140 scheduler_mark_self_needs_resched(false)) {
141 struct thread *curr = thread_get_current();
142 if (curr)
143 curr->preemptions++;
144
145 kassert(old != IRQL_DISPATCH_LEVEL);
146 scheduler_yield();
147 }
148}
149
150void isr_common_entry(irq_t vector, struct irq_context *irq_ctx) {
151 if (vector != IRQ_NMI) {
152 isr_standard_entry(vector, irq_ctx);
153 } else {
154 isr_nmi_entry(irq_ctx);
155 }
156}
157
158void irq_register(char *name, uint8_t vector, irq_handler_t handler, void *ctx,
159 enum irq_flags flags) {
160 enum irql irql = spin_lock(&irq_table_lock);
161 struct irq_desc *me = &irq_table[vector];
162
163 bool was = me->present;
164 me->present = true;
165 me->allocated = true;
166 me->enabled = true;
167
168 if (was && !(flags & IRQ_FLAG_SHARED))
169 panic("need to be shared to have many, registered by %s", me->name);
170
171 struct irq_action *act =
172 kmalloc_or_die(sizeof(struct irq_action), ALLOC_FLAGS_ZERO);
173
174 act->handler = handler;
175 INIT_LIST_HEAD(list: &act->list);
176 act->data = ctx;
177
178 list_add_tail(new: &act->list, head: &me->actions);
179 if (!me->name)
180 me->name = name;
181
182 me->flags = flags;
183
184 spin_unlock(&irq_table_lock, irql);
185}
186
187void irq_set_chip(uint8_t vec, struct irq_chip *chip, void *data) {
188 enum irql irql = spin_lock(&irq_table_lock);
189
190 if (irq_table[vec].chip && chip)
191 panic("IRQ chip %u exists", vec);
192
193 irq_table[vec].chip = chip;
194 irq_table[vec].chip_data = data;
195
196 spin_unlock(&irq_table_lock, irql);
197}
198
199void idt_set_gate(uint8_t num, uint16_t sel, uint8_t flags) {
200 struct idt_entry *idt = idts.entries;
201
202 uint64_t base = (uint64_t) isr_vectors[num];
203
204 idt[num].base_low = (base & 0xFFFF);
205 idt[num].base_mid = (base >> 16) & 0xFFFF;
206 idt[num].base_high = (base >> 32) & 0xFFFFFFFF;
207 idt[num].selector = sel;
208
209 /* TODO: maybe don't hardcode this (?) */
210 if (num == IRQ_NMI || num == IRQ_DBF) {
211 idt[num].ist = 1;
212 } else {
213 idt[num].ist = 0;
214 }
215
216 idt[num].flags = flags;
217 idt[num].reserved = 0;
218}
219
220void irq_load(void) {
221 idtps.limit = sizeof(struct idt_entry) * IDT_ENTRIES - 1;
222 idtps.base = (uint64_t) &idts;
223 asm volatile("lidt %0" : : "m"(idtps));
224}
225
226int32_t irq_alloc_entry() {
227 enum irql irql = spin_lock(&irq_table_lock);
228 for (int32_t i = 32; i < IDT_ENTRIES; i++) {
229 if (!irq_table[i].allocated) {
230 irq_table[i].allocated = true;
231 spin_unlock(&irq_table_lock, irql);
232 return i;
233 }
234 }
235 spin_unlock(&irq_table_lock, irql);
236 return -1;
237}
238
239static void irq_desc_clear(struct irq_desc *desc) {
240 cpu_mask_set_all(&desc->masked_cpus);
241 cpu_mask_set_all(&desc->affinity);
242 INIT_LIST_HEAD(list: &desc->actions);
243 desc->present = false;
244 desc->allocated = false;
245 desc->enabled = false;
246 desc->flags = 0;
247 desc->name = NULL;
248}
249
250void irq_free_entry(int32_t entry) {
251 if (entry < 32 || entry >= IDT_ENTRIES)
252 return;
253
254 enum irql irql = spin_lock(&irq_table_lock);
255
256 struct irq_desc *desc = &irq_table[entry];
257 desc->allocated = false;
258 desc->present = false;
259
260 struct irq_action *iter, *tmp;
261 list_for_each_entry_safe(iter, tmp, &desc->actions, list) {
262 list_del_init(entry: &iter->list);
263 kfree(iter);
264 }
265
266 irq_desc_clear(desc);
267
268 spin_unlock(&irq_table_lock, irql);
269}
270
271void irq_disable(irq_t irq) {
272 struct irq_desc *desc = &irq_table[irq];
273 desc->enabled = false;
274 if (desc->chip && desc->chip->mask)
275 desc->chip->mask(desc);
276}
277
278void irq_enable(irq_t irq) {
279 struct irq_desc *desc = &irq_table[irq];
280 desc->enabled = true;
281 if (desc->chip && desc->chip->unmask)
282 desc->chip->unmask(desc);
283}
284
285static void exception_sync_cbs_init() {
286 struct exception_sync_cb *esc;
287 linker_section_for_each_object(esc, exception_sync_cbs) {
288 kassert(exception_cbs[esc->vector].vector == 0);
289 exception_cbs[esc->vector] = *esc;
290 }
291}
292
293void irq_init() {
294 for (size_t i = 0; i < IDT_ENTRIES; i++) {
295 struct irq_desc *desc = &irq_table[i];
296 alloc_or_die(cpu_mask_init(&desc->masked_cpus, global.core_count));
297 alloc_or_die(cpu_mask_init(&desc->affinity, global.core_count));
298
299 desc->vector = i;
300 irq_desc_clear(desc);
301
302 idt_set_gate(num: i, sel: 0x08, flags: 0x8e);
303 }
304
305 exception_sync_cbs_init();
306
307 irq_register(name: "division_by_zero", IRQ_DIV_BY_Z, handler: divbyz_handler, NULL,
308 flags: IRQ_FLAG_NONE);
309 irq_register(name: "debug", IRQ_DEBUG, handler: debug_handler, NULL, flags: IRQ_FLAG_NONE);
310 irq_register(name: "breakpoint", IRQ_BREAKPOINT, handler: breakpoint_handler, NULL,
311 flags: IRQ_FLAG_NONE);
312
313 irq_register(name: "ssf", IRQ_SSF, handler: ss_handler, NULL, flags: IRQ_FLAG_NONE);
314
315 irq_register(name: "gpf", IRQ_GPF, handler: gpf_handler, NULL, flags: IRQ_FLAG_NONE);
316 irq_register(name: "double_fault", IRQ_DBF, handler: double_fault_handler, NULL,
317 flags: IRQ_FLAG_NONE);
318 irq_register(name: "page_fault", IRQ_PAGE_FAULT, handler: page_fault_isr, NULL,
319 flags: IRQ_FLAG_NONE);
320
321 irq_register(name: "timer", IRQ_TIMER, handler: timer_isr, NULL, flags: IRQ_FLAG_NONE);
322 irq_set_chip(IRQ_TIMER, chip: lapic_get_chip(), NULL);
323
324 /* NOTE: Ordering MATTERS here. We have to register this first, because
325 * this means that the panic check fires *before* any other NMI check,
326 * which is probably one of the biggest debug correctness parts
327 * of this interrupt subsystem. This MUST stay here, or its registration
328 * must guarantee that it is at the head of the NMI ISR list */
329 irq_register(name: "nmi", IRQ_NMI, handler: panic_nmi_isr, NULL, flags: IRQ_FLAG_SHARED);
330
331 /* HACK: secondary names are not added onto additional registrations,
332 * they just disappear. It's not at all important for correctness,
333 * and there are no consumers of name anyways, but it's something we'll
334 * need to implement some time down the line */
335 irq_register(name: "hardware_nmi", IRQ_NMI, handler: hw_error_nmi_isr, NULL,
336 flags: IRQ_FLAG_SHARED);
337
338 irq_register(name: "tlb_shootdown", IRQ_TLB_SHOOTDOWN, handler: tlb_shootdown_isr, NULL,
339 flags: IRQ_FLAG_NONE);
340 irq_set_chip(IRQ_TLB_SHOOTDOWN, chip: lapic_get_chip(), NULL);
341
342 irq_register(name: "nop", IRQ_NOP, handler: nop_handler, NULL, flags: IRQ_FLAG_NONE);
343 irq_set_chip(IRQ_NOP, chip: lapic_get_chip(), NULL);
344 irq_register(name: "dpc", IRQ_DPC, handler: dpc_handler, NULL, flags: IRQ_FLAG_NONE);
345 irq_set_chip(IRQ_DPC, chip: lapic_get_chip(), NULL);
346
347 idt_set_gate(num: 0x80, sel: 0x2b, flags: 0xee);
348 irq_load();
349}
350