1#include <acpi/lapic.h>
2#include <asm.h>
3#include <compiler/core.h>
4#include <console/printf.h>
5#include <dbg.h>
6#include <global.h>
7#include <irq/exception_sync_cb.h>
8#include <irq/idt.h>
9#include <mem/alloc.h>
10#include <mem/nofail.h>
11#include <mem/page_fault.h>
12#include <mem/tlb.h>
13#include <mem/vmm.h>
14#include <sch/sched.h>
15#include <smp/core.h>
16#include <smp/smp.h>
17#include <stdbool.h>
18#include <stddef.h>
19#include <stdint.h>
20#include <string.h>
21#include <sync/rcu.h>
22#include <thread/apc.h>
23#include <thread/thread.h>
24#include <time/timer.h>
25#include <watchdog.h>
26
27/* Lock is only used for allocation/free and registering */
28static struct spinlock irq_table_lock = SPINLOCK_INIT;
29static struct irq_desc irq_table[IDT_ENTRIES] = {0};
30static struct exception_sync_cb exception_cbs[IRQ_EXCEPTION_COUNT] = {0};
31static struct idt_table idts = {0};
32static struct idt_ptr idtps = {0};
33
34#include "fault_isrs.h"
35#include "isr_stubs.h"
36#include "isr_vectors_array.h"
37
38static void irq_execute_vector_handlers(irq_t vector,
39 struct irq_context *irq_ctx) {
40 irq_ctx->vector = vector;
41 struct irq_desc *desc = &irq_table[vector];
42 if (!desc->present || list_head_empty(h: &irq_table[vector].actions))
43 panic("Unhandled ISR vector: %u", vector);
44
45 /* HACK: There is this specific use of smp_enable_all_ticks() in bringup
46 * where a TOCTOU can happen. Basically, CPU 0 will send all the IPIs out,
47 * then everyone will return from the handler, but before they send
48 * the EOI, the chip can get swapped out, meaning they never end up
49 * actually sending the LAPIC EOI. In the future, I will likely
50 * want to use DPCs to handle that initial tick stop operation
51 * or something, since this is not a very good design.
52 *
53 * TODO: Also likely will want to rework IRQ registration
54 * and unregistration atomicity and synchronization too. */
55 struct irq_chip *chip = desc->chip;
56
57 bool handled = false;
58 struct list_node *lh;
59 list_for_each(lh, &desc->actions) {
60 struct irq_action *act = container_of(lh, struct irq_action, list);
61 if (act->handler(irq_ctx, act->data) == IRQ_HANDLED) {
62 if (vector != IRQ_NMI)
63 watchdog_pet();
64 handled = true;
65 break;
66 }
67 }
68
69 if (handled && chip && chip->eoi)
70 chip->eoi(desc);
71}
72
73/* TODO: Someday we will need to handle nmi_depth > 1 and do a fancy asm
74 * trampoline to handle cases where IRQs fire in NMIs due to the IRET issue */
75static void isr_nmi_entry(struct irq_registers *irq_regs) {
76 /* vector == IRQ_NMI, of course */
77 enforce(!irq_mark_self_in_nmi(true));
78 uint32_t icr_high = lapic_icr_nmi_save();
79
80 struct irq_context irq_ctx = {.regs = irq_regs};
81 irq_execute_vector_handlers(IRQ_NMI, irq_ctx: &irq_ctx);
82
83 lapic_icr_nmi_restore(high: icr_high);
84 enforce(irq_mark_self_in_nmi(false) == 1);
85}
86
87static void isr_standard_entry(irq_t vector, struct irq_registers *irq_regs) {
88 irq_mark_self_in_interrupt(true);
89
90 /* before anyone can enter a read section */
91 rcu_idle_exit();
92
93 enum irql old = irql_raise(new_level: IRQL_HIGH_LEVEL);
94
95 bool is_exception = irq_vector_is_exception(vector);
96 uint8_t scratch_buf[EXCEPTION_SYNC_CB_SCRATCH_BUFFER_SIZE] = {0};
97
98 if (!is_exception)
99 enforce(old != IRQL_HIGH_LEVEL, INVALID_CONTEXT);
100
101 struct irq_context irq_ctx = {.irq_entered_irql = old,
102 .irq_stack_scratch_buf = scratch_buf,
103 .regs = irq_regs};
104 irq_execute_vector_handlers(vector, irq_ctx: &irq_ctx);
105
106 /* We explicitly exclude exceptions, since the context
107 * might be anywhere, including within RCU itself */
108 if (!is_exception)
109 rcu_note_irq_exit();
110
111 /* Here's an odd bit: we'll have very different
112 * behavior if we came from an exception. Namely,
113 * irql_lower will NOT `sti` if irq_in_interrupt()
114 *
115 * Thus, if we are in an exception, we FIRST
116 * irq_mark_self_in_interrupt(false), and the we
117 * irql_lower(old)
118 *
119 * This results in interrupts being enabled
120 * when we come out of irql_lower(), and the
121 * reason why this only applies to exceptions
122 * is because they are synchronous, and more
123 * importantly, have an exception_sync_cb that
124 * can run, but only from this context in which we
125 * are at our old pre-exception IRQL and have
126 * interrupts enabled */
127 if (is_exception) {
128 irq_mark_self_in_interrupt(false);
129 irql_lower(old_level: old);
130
131 struct exception_sync_cb *escb = &exception_cbs[vector];
132 if (escb->fn)
133 escb->fn(escb, &irq_ctx, scratch_buf);
134
135 } else {
136 irql_lower(old_level: old);
137 irq_mark_self_in_interrupt(false);
138 }
139
140 /* This function just raises and lowers the IRQL, because
141 * now we're irq_mark_self_in_interrupt == false.
142 *
143 * No running DPCs at or above IRQL_DISPATCH_LEVEL. */
144 if (old < IRQL_DISPATCH_LEVEL && scheduler_mark_self_needs_run_dpcs(false))
145 dpc_run_dpcs_from_irq();
146
147 /* in reschedule, don't check if we need to preempt */
148 if (scheduler_self_in_resched())
149 return;
150
151 if (scheduler_yield_nesting(t: thread_get_current()) != 0)
152 return;
153
154 if (!scheduler_preemption_disabled(c: TOPC_NONE) &&
155 scheduler_mark_self_needs_resched(false)) {
156
157#ifdef DEBUG_THREAD
158 struct thread *curr = thread_get_current();
159 if (curr)
160 curr->preemptions++;
161#endif
162
163 enforce(old != IRQL_DISPATCH_LEVEL, INVALID_CONTEXT);
164 scheduler_yield();
165 }
166}
167
168/* Only called from isrs.asm */
169void isr_common_entry(irq_t vector, struct irq_registers *irq_regs);
170void isr_common_entry(irq_t vector, struct irq_registers *irq_regs) {
171 if (vector != IRQ_NMI) {
172 isr_standard_entry(vector, irq_regs);
173 } else {
174 isr_nmi_entry(irq_regs);
175 }
176}
177
178void irq_register(char *name, uint8_t vector, irq_handler_t handler, void *ctx,
179 enum irq_flags flags) {
180 enum irql irql = spin_lock(&irq_table_lock);
181 struct irq_desc *me = &irq_table[vector];
182
183 bool was = me->present;
184 me->present = true;
185 me->allocated = true;
186 me->enabled = true;
187
188 if (was && !(flags & IRQ_FLAG_SHARED))
189 panic(DUPLICATE, "need to be shared to have many, registered by %s",
190 me->name);
191
192 struct irq_action *act = nofail_vnew(act, ALLOC_ZERO);
193
194 act->handler = handler;
195 list_node_init(n: &act->list);
196 act->data = ctx;
197
198 list_head_push_back(h: &me->actions, n: &act->list);
199 if (!me->name)
200 me->name = name;
201
202 me->flags = flags;
203
204 spin_unlock(&irq_table_lock, irql);
205}
206
207void irq_set_chip(uint8_t vec, struct irq_chip *chip, void *data) {
208 enum irql irql = spin_lock(&irq_table_lock);
209
210 if (irq_table[vec].chip && chip)
211 panic(DUPLICATE, "IRQ chip %u exists", vec);
212
213 irq_table[vec].chip = chip;
214 irq_table[vec].chip_data = data;
215
216 spin_unlock(&irq_table_lock, irql);
217}
218
219void idt_set_gate(uint8_t num, uint16_t sel, uint8_t flags) {
220 struct idt_entry *idt = idts.entries;
221
222 uint64_t base = (uint64_t) isr_vectors[num];
223
224 idt[num].base_low = (base & 0xFFFF);
225 idt[num].base_mid = (base >> 16) & 0xFFFF;
226 idt[num].base_high = (base >> 32) & 0xFFFFFFFF;
227 idt[num].selector = sel;
228
229 /* TODO: maybe don't hardcode this (?) */
230 if (num == IRQ_NMI || num == IRQ_DBF) {
231 idt[num].ist = 1;
232 } else {
233 idt[num].ist = 0;
234 }
235
236 idt[num].flags = flags;
237 idt[num].reserved = 0;
238}
239
240void idt_load(void) {
241 idtps.limit = sizeof(struct idt_entry) * IDT_ENTRIES - 1;
242 idtps.base = (uint64_t) &idts;
243 asm volatile("lidt %0" : : "m"(idtps));
244}
245
246int32_t irq_alloc_entry(void) {
247 enum irql irql = spin_lock(&irq_table_lock);
248 for (int32_t i = 32; i < IDT_ENTRIES; i++) {
249 if (!irq_table[i].allocated) {
250 irq_table[i].allocated = true;
251 spin_unlock(&irq_table_lock, irql);
252 return i;
253 }
254 }
255 spin_unlock(&irq_table_lock, irql);
256 return -1;
257}
258
259static void irq_desc_clear(struct irq_desc *desc) {
260 cpu_mask_set_all(&desc->masked_cpus);
261 cpu_mask_set_all(&desc->affinity);
262 list_head_init(h: &desc->actions);
263 desc->present = false;
264 desc->allocated = false;
265 desc->enabled = false;
266 desc->flags = 0;
267 desc->name = NULL;
268}
269
270void irq_free_entry(int32_t entry) {
271 if (entry < 32 || entry >= IDT_ENTRIES)
272 return;
273
274 enum irql irql = spin_lock(&irq_table_lock);
275
276 struct irq_desc *desc = &irq_table[entry];
277 desc->allocated = false;
278 desc->present = false;
279
280 struct irq_action *iter, *tmp;
281 list_for_each_entry_safe(iter, tmp, &desc->actions, list) {
282 list_node_delete_init(n: &iter->list);
283 vfree(iter);
284 }
285
286 irq_desc_clear(desc);
287
288 spin_unlock(&irq_table_lock, irql);
289}
290
291void irq_vector_disable(irq_t irq) {
292 struct irq_desc *desc = &irq_table[irq];
293 desc->enabled = false;
294 if (desc->chip && desc->chip->mask)
295 desc->chip->mask(desc);
296}
297
298void irq_vector_enable(irq_t irq) {
299 struct irq_desc *desc = &irq_table[irq];
300 desc->enabled = true;
301 if (desc->chip && desc->chip->unmask)
302 desc->chip->unmask(desc);
303}
304
305static void exception_sync_cbs_init(void) {
306 struct exception_sync_cb *esc;
307 linker_section_for_each_object(esc, exception_sync_cbs) {
308 enforce(exception_cbs[esc->vector].vector == 0, DUPLICATE);
309 exception_cbs[esc->vector] = *esc;
310 }
311}
312
313void irq_init(void) {
314 for (size_t i = 0; i < IDT_ENTRIES; i++) {
315 struct irq_desc *desc = &irq_table[i];
316 cpu_mask_init(m: &desc->masked_cpus, nbits: global.core_count);
317 cpu_mask_init(m: &desc->affinity, nbits: global.core_count);
318
319 desc->vector = i;
320 irq_desc_clear(desc);
321
322 idt_set_gate(num: i, sel: 0x08, flags: 0x8e);
323 }
324
325 exception_sync_cbs_init();
326
327 irq_register(name: "division_by_zero", IRQ_DIV_BY_Z, handler: divbyz_handler, NULL,
328 flags: IRQ_FLAG_NONE);
329 irq_register(name: "debug", IRQ_DEBUG, handler: debug_handler, NULL, flags: IRQ_FLAG_NONE);
330 irq_register(name: "breakpoint", IRQ_BREAKPOINT, handler: breakpoint_handler, NULL,
331 flags: IRQ_FLAG_NONE);
332
333 irq_register(name: "ssf", IRQ_SSF, handler: ss_handler, NULL, flags: IRQ_FLAG_NONE);
334
335 irq_register(name: "gpf", IRQ_GPF, handler: gpf_handler, NULL, flags: IRQ_FLAG_NONE);
336 irq_register(name: "double_fault", IRQ_DBF, handler: double_fault_handler, NULL,
337 flags: IRQ_FLAG_NONE);
338 irq_register(name: "page_fault", IRQ_PAGE_FAULT, handler: page_fault_isr, NULL,
339 flags: IRQ_FLAG_NONE);
340
341 irq_register(name: "scheduler", IRQ_SCHEDULER, handler: scheduler_reschedule_isr, NULL,
342 flags: IRQ_FLAG_NONE);
343 irq_set_chip(IRQ_SCHEDULER, chip: lapic_get_chip(), NULL);
344
345 irq_register(name: "timer", IRQ_TIMER, handler: timer_isr, NULL, flags: IRQ_FLAG_NONE);
346 irq_set_chip(IRQ_TIMER, chip: lapic_get_chip(), NULL);
347
348 /* NOTE: Ordering MATTERS here. We have to register this first, because
349 * this means that the panic check fires *before* any other NMI check,
350 * which is probably one of the biggest debug correctness parts
351 * of this interrupt subsystem. This MUST stay here, or its registration
352 * must guarantee that it is at the head of the NMI ISR list */
353 irq_register(name: "nmi", IRQ_NMI, handler: panic_nmi_isr, NULL, flags: IRQ_FLAG_SHARED);
354
355 /* HACK: secondary names are not added onto additional registrations,
356 * they just disappear. It's not at all important for correctness,
357 * and there are no consumers of name anyways, but it's something we'll
358 * need to implement some time down the line */
359 irq_register(name: "hardware_nmi", IRQ_NMI, handler: hw_error_nmi_isr, NULL,
360 flags: IRQ_FLAG_SHARED);
361
362 irq_register(name: "tlb_shootdown", IRQ_TLB_SHOOTDOWN, handler: tlb_shootdown_isr, NULL,
363 flags: IRQ_FLAG_NONE);
364 irq_set_chip(IRQ_TLB_SHOOTDOWN, chip: lapic_get_chip(), NULL);
365
366 irq_register(name: "nop", IRQ_NOP, handler: nop_handler, NULL, flags: IRQ_FLAG_NONE);
367 irq_set_chip(IRQ_NOP, chip: lapic_get_chip(), NULL);
368 irq_register(name: "dpc", IRQ_DPC, handler: dpc_handler, NULL, flags: IRQ_FLAG_NONE);
369 irq_set_chip(IRQ_DPC, chip: lapic_get_chip(), NULL);
370
371 idt_set_gate(num: 0x80, sel: 0x2b, flags: 0xee);
372 idt_load();
373}
374