1#include <acpi/lapic.h>
2#include <asm.h>
3#include <drivers/mmio.h>
4#include <global.h>
5#include <irq/idt.h>
6#include <log.h>
7#include <mem/alloc.h>
8#include <mem/page.h>
9#include <mem/vmm.h>
10#include <smp/core.h>
11#include <smp/percpu.h>
12#include <string.h>
13#include <time/clock_evdev.h>
14#include <time/names.h>
15#include <time/spin_sleep.h>
16
17uint32_t *lapic;
18bool x2apic_enabled = false;
19static LOG_HANDLE_DECLARE_PRINT(lapic);
20
21void lapic_init(void) {
22 uintptr_t lapic_phys = rdmsr(IA32_APIC_BASE_MSR) & IA32_APIC_BASE_MASK;
23 lapic = mmio_map(phys: lapic_phys, PAGE_SIZE);
24}
25
26void lapic_timer_init(cpu_id_t core_id) {
27 uint32_t calibration_sleep_ms = 2;
28
29 lapic_write(LAPIC_REG_SVR, LAPIC_ENABLE | 0xFF);
30 lapic_write(LAPIC_REG_TIMER_DIV, val: 0b0011);
31 lapic_write(LAPIC_REG_LVT_TIMER, IRQ_TIMER | LAPIC_LVT_MASK);
32 lapic_write(LAPIC_REG_TIMER_INIT, val: 0xFFFFFFFF);
33
34 sleep_spin_ms(msec: calibration_sleep_ms);
35
36 uint32_t curr = lapic_read(reg: (LAPIC_REG_TIMER_CUR));
37 uint32_t elapsed = 0xFFFFFFFF - curr;
38
39 freq_khz_t lapic_calibrated_khz = elapsed / calibration_sleep_ms;
40
41 global.cores[core_id]->lapic_khz = lapic_calibrated_khz;
42 lapic_timer_disable();
43}
44
45void lapic_timer_disable() {
46 uint32_t lvt = lapic_read(LAPIC_REG_LVT_TIMER);
47 lvt |= LAPIC_LVT_MASK;
48 lapic_write(LAPIC_REG_LVT_TIMER, val: lvt);
49}
50
51void lapic_timer_enable() {
52 uint32_t lvt = lapic_read(LAPIC_REG_LVT_TIMER);
53 lvt &= ~LAPIC_LVT_MASK;
54 lapic_write(LAPIC_REG_LVT_TIMER, val: lvt);
55}
56
57void lapic_eoi(struct irq_desc *unused) {
58 (void) unused;
59 lapic_write(LAPIC_REG_EOI, val: 0);
60}
61
62bool lapic_timer_is_enabled() {
63 uint32_t lvt = lapic_read(LAPIC_REG_LVT_TIMER);
64 return !(lvt & (1 << 16));
65}
66
67static void lapic_send_ipi(uint8_t apic_id, uint8_t vector) {
68 enum irql irql = irql_raise(new_level: IRQL_HIGH_LEVEL);
69 while (lapic_read(LAPIC_ICR_LOW) & LAPIC_IPI_IN_FLIGHT)
70 cpu_relax();
71
72 lapic_write(LAPIC_ICR_HIGH, val: apic_id << LAPIC_DEST_SHIFT);
73 lapic_write(LAPIC_ICR_LOW, val: vector | LAPIC_DELIVERY_FIXED |
74 LAPIC_LEVEL_ASSERT | LAPIC_DEST_PHYSICAL);
75 irql_lower(old_level: irql);
76}
77
78void x2apic_send_ipi(uint32_t apic_id, uint8_t vector) {
79 uint64_t icr = 0;
80 icr |= vector;
81 icr |= LAPIC_DELIVERY_FIXED;
82 icr |= LAPIC_LEVEL_ASSERT;
83 icr |= LAPIC_DEST_PHYSICAL;
84 icr |= ((uint64_t) apic_id << 32);
85
86 wrmsr(IA32_X2APIC_ICR, value: icr);
87}
88
89void panic_broadcast(uint64_t exclude_core) {
90 size_t i;
91 for_each_cpu_id(i) {
92 if (i == exclude_core)
93 continue;
94
95 nmi_send(apic_id: i);
96 }
97}
98
99void nmi_send(uint32_t apic_id) {
100 kassert(apic_id != smp_id(TOPC_NONE)); /* NMI'ing ourselves is a MASSIVE
101 * risk, likely buggy code, panic */
102
103 if (x2apic_enabled) {
104 uint64_t icr = 0;
105
106 /* Delivery mode = NMI. Vector ignored. */
107 icr |= LAPIC_DELIVERY_NMI; /* bits 10:8 = 100b for NMI */
108 icr |= LAPIC_DEST_PHYSICAL; /* physical dest mode */
109 icr |= ((uint64_t) apic_id << 32);
110
111 wrmsr(IA32_X2APIC_ICR, value: icr);
112 return;
113 }
114
115 uint32_t hi = apic_id << LAPIC_DEST_SHIFT;
116 uint32_t lo = 0;
117
118 lo |= LAPIC_DELIVERY_NMI;
119 lo |= LAPIC_DEST_PHYSICAL;
120 lo |= LAPIC_LEVEL_ASSERT;
121
122 /* High must be written before low */
123 lapic_write(LAPIC_ICR_HIGH, val: hi);
124 lapic_write(LAPIC_ICR_LOW, val: lo);
125
126 while (lapic_read(LAPIC_ICR_LOW) & LAPIC_IPI_IN_FLIGHT)
127 cpu_relax();
128}
129
130static int cpu_has_x2apic(void) {
131 uint32_t eax, ebx, ecx, edx;
132
133 asm volatile("cpuid"
134 : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx)
135 : "a"(1)
136 :);
137
138 return (ecx >> 21) & 1;
139}
140
141void x2apic_init(void) {
142 if (!cpu_has_x2apic())
143 return;
144
145 x2apic_enabled = true;
146 uint64_t apic_base;
147 apic_base = rdmsr(IA32_APIC_BASE);
148 apic_base |= APIC_X2APIC_ENABLE;
149 wrmsr(IA32_APIC_BASE, value: apic_base);
150 log_info_global(LOG_HANDLE(lapic), "X2APIC enabled");
151}
152
153uint32_t x2apic_get_id(void) {
154 return rdmsr(IA32_X2APIC_ID) & 0xFFFFFFFF;
155}
156
157uint64_t lapic_get_id(void) {
158 uint32_t lapic_id_raw = lapic_read(LAPIC_REG_ID);
159 uint64_t cpu = (lapic_id_raw >> 24) & 0xFF;
160 return cpu;
161}
162
163uint32_t cpu_get_this_id(void) {
164 return cpu_has_x2apic() ? x2apic_get_id() : lapic_get_id();
165}
166
167void ipi_send(uint32_t apic_id, uint8_t vector) {
168 if (x2apic_enabled)
169 return x2apic_send_ipi(apic_id, vector);
170
171 lapic_send_ipi(apic_id, vector);
172}
173
174static struct irq_chip lapic_irq_chip = {
175 .eoi = lapic_eoi,
176 .mask = NULL,
177 .unmask = NULL,
178 .set_affinity = NULL,
179 .set_rate_limit = NULL,
180 .name = "lapic",
181};
182
183struct irq_chip *lapic_get_chip() {
184 return &lapic_irq_chip;
185}
186
187void lapic_timer_init_bsp(void) {
188 lapic_timer_init(core_id: 0);
189}
190
191static enum errno lapic_evdev_set_next_event(struct clock_evdev *ced,
192 time_ns_t delta_ns) {
193 (void) ced;
194
195 /* The timer context guarantees that set_next_event happens under HIGH */
196 freq_khz_t freq_khz = smp_core(cond: TOPC_IRQL)->lapic_khz;
197 uint64_t ticks = (freq_khz * (uint64_t) delta_ns) / 1000000ULL;
198
199 if (ticks == 0)
200 ticks = 1;
201
202 if (ticks > 0xFFFFFFFFULL)
203 ticks = 0xFFFFFFFFULL;
204
205 uint32_t lvt = lapic_read(LAPIC_REG_LVT_TIMER);
206 lvt &= ~(TIMER_MODE_PERIODIC | LAPIC_LVT_MASK);
207 lvt |= (IRQ_TIMER | TIMER_MODE_ONESHOT);
208 lapic_write(LAPIC_REG_LVT_TIMER, val: lvt);
209
210 /* Oneshot countdown */
211 lapic_write(LAPIC_REG_TIMER_INIT, val: (uint32_t) ticks);
212
213 return 0;
214}
215
216static enum errno lapic_evdev_change_state(struct clock_evdev *ced,
217 enum clock_evdev_state state) {
218 (void) ced;
219 uint32_t lvt = lapic_read(LAPIC_REG_LVT_TIMER);
220
221 switch (state) {
222 case CLOCK_EVDEV_STATE_ONESHOT:
223 lvt &= ~(TIMER_MODE_PERIODIC | LAPIC_LVT_MASK);
224 lvt |= (IRQ_TIMER | TIMER_MODE_ONESHOT);
225 lapic_write(LAPIC_REG_LVT_TIMER, val: lvt);
226 break;
227
228 case CLOCK_EVDEV_STATE_PERIODIC:
229 lvt &= ~LAPIC_LVT_MASK;
230 lvt |= (IRQ_TIMER | TIMER_MODE_PERIODIC);
231 lapic_write(LAPIC_REG_LVT_TIMER, val: lvt);
232 break;
233
234 case CLOCK_EVDEV_STATE_OFF:
235 case CLOCK_EVDEV_STATE_ONESHOT_STOPPED:
236 lapic_write(LAPIC_REG_TIMER_INIT, val: 0);
237 lvt |= LAPIC_LVT_MASK;
238 lapic_write(LAPIC_REG_LVT_TIMER, val: lvt);
239 break;
240 }
241
242 ced->state = state;
243 return 0;
244}
245
246static struct clock_evdev *lapic_clock_evdev_create(cpu_id_t core_id) {
247 struct clock_evdev *ced = clock_evdev_create(fmt: "lapic_timer_%zu", core_id);
248
249 ced->set_next_event = lapic_evdev_set_next_event;
250 ced->change_state = lapic_evdev_change_state;
251
252 /* bounds based on LAPIC frequency */
253 freq_khz_t freq_khz = global.cores[core_id]->lapic_khz;
254
255 ced->min_delta_ns = (1000000ULL + freq_khz - 1) / freq_khz;
256 ced->max_delta_ns = (0xFFFFFFFFULL * 1000000ULL) / freq_khz;
257
258 ced->min_delta_ticks = 1;
259 ced->max_delta_ticks = 0xFFFFFFFF;
260
261 ced->flags =
262 CLOCK_EVDEV_ONESHOT | CLOCK_EVDEV_PERCPU | CLOCK_EVDEV_TICK_SUITABLE;
263 ced->rating = CLOCK_RATING_BEST;
264 ced->bound_to_cpu = core_id;
265 ced->irq = IRQ_TIMER;
266
267 /* ready LAPIC divider and vector */
268 lapic_write(LAPIC_REG_TIMER_DIV, val: 0b0011);
269
270 lapic_evdev_change_state(ced, state: CLOCK_EVDEV_STATE_ONESHOT_STOPPED);
271 return ced;
272}
273
274void lapic_clock_evdev_group_init(void) {
275 struct clock_evdev_group *cedg =
276 alloc_or_die(clock_evdev_group_create(CLOCK_NAME_LAPIC));
277
278 /* No need to set evdev_for_cpu if CLOCK_EVDEV_GROUP_PERCPU set */
279 cedg->flags = CLOCK_EVDEV_GROUP_PERCPU;
280
281 size_t i;
282 for_each_cpu_id(i) {
283 clock_evdev_group_add(cedg, ced: lapic_clock_evdev_create(core_id: i));
284 }
285
286 clock_evdev_group_register(cedg);
287}
288