1#include <acpi/acpi.h>
2#include <acpi/cst.h>
3#include <acpi/hpet.h>
4#include <acpi/ioapic.h>
5#include <acpi/lapic.h>
6#include <acpi/uacpi_interface.h>
7#include <asm.h>
8#include <boot/gdt.h>
9#include <bootstage.h>
10#include <cmdline.h>
11#include <compiler/core.h>
12#include <console/printf.h>
13#include <console/term.h>
14#include <crypto/prng.h>
15#include <drivers/iommu/iommu.h>
16#include <drivers/mmio.h>
17#include <elf.h>
18#include <fs/vfs.h>
19#include <global.h>
20#include <irq/idt.h>
21#include <limine.h>
22#include <log.h>
23#include <logo.h>
24#include <mem/address_range.h>
25#include <mem/alloc.h>
26#include <mem/arena.h>
27#include <mem/asan.h>
28#include <mem/buddy.h>
29#include <mem/domain.h>
30#include <mem/movealloc.h>
31#include <mem/page_alloc.h>
32#include <mem/pmm.h>
33#include <mem/slab.h>
34#include <mem/tlb.h>
35#include <mem/vmm.h>
36#include <ndjson.h>
37#include <nightmare/nightmare.h>
38#include <registry.h>
39#include <requests.h>
40#include <sch/domain.h>
41#include <sch/periodic_work.h>
42#include <sch/sched.h>
43#include <smp/core.h>
44#include <smp/domain.h>
45#include <smp/percpu.h>
46#include <smp/perdomain.h>
47#include <smp/pertopo.h>
48#include <smp/smp.h>
49#include <stack_depot.h>
50#include <stdint.h>
51#include <sync/lock_chk.h>
52#include <sync/rcu.h>
53#include <sync/spinlock.h>
54#include <sync/turnstile.h>
55#include <syscall.h>
56#include <test/test.h>
57#include <thread/dpc.h>
58#include <thread/reaper.h>
59#include <thread/thread.h>
60#include <thread/workqueue.h>
61#include <time/clock.h>
62#include <time/lapic_timer.h>
63#include <time/timekeeper.h>
64#include <time/timer.h>
65#include <watchdog.h>
66
67struct globals global = {0};
68
69#define BEHAVIOR /* avoids undefined behavior */
70
71/* Only called from the linker script ENTRY */
72cc_no_asan void k_main(void) TSA_ACQUIRES_IRQS {
73 irq_disable();
74 global.core_count = mp_request.response->cpu_count;
75 global.hhdm_offset = hhdm_request.response->offset;
76 atomic_init(&global.pt_epoch, 1);
77
78 printf_init(fb: framebuffer_request.response->framebuffers[0]);
79 err_facilities_init();
80 enforce(global.core_count <= CPU_MASK_BITS, CAPACITY_EXCEEDED,
81 "%zu CPUs exceed CPU_MASK_BITS (%d)", global.core_count,
82 CPU_MASK_BITS);
83 ndjson_early_init();
84 bootstage_advance(new: BOOTSTAGE_EARLY_FB);
85
86 const char *cmdline =
87 cmdline_request.response ? cmdline_request.response->cmdline : "";
88
89 if (cmdline_wants_help(input: cmdline))
90 cmdline_dump_help();
91
92 pmm_early_init(m: memmap_request);
93 vmm_init(memmap: memmap_request.response, xa: xa_request.response);
94 pmm_mid_init();
95
96#ifdef DEBUG_ASAN
97 asan_init();
98#endif
99
100 address_ranges_init();
101 slab_allocator_init();
102 page_alloc_init();
103
104 stack_depot_init();
105 log_sites_init();
106 bootstage_advance(new: BOOTSTAGE_EARLY_ALLOCATORS);
107 gdt_load();
108 syscall_setup(syscall_entry);
109 smp_setup_bsp(mpr: mp_request.response);
110
111 clocks_init();
112
113 mmio_init();
114 irq_init();
115 uacpi_init(rsdp: rsdp_request.response->address);
116 x2apic_init();
117 lapic_init();
118 hpet_init();
119 ioapic_init();
120 acpi_find_cst();
121 bootstage_advance(new: BOOTSTAGE_EARLY_DEVICES);
122
123 srat_init();
124 slit_init();
125 iommu_init();
126
127 domain_init();
128 pmm_late_init();
129 slab_domain_init();
130
131 percpu_obj_init();
132 watchdog_init();
133 smp_init();
134 tlb_init();
135
136 domain_init_after_smp();
137 domain_buddies_init_after_smp();
138 thread_init_thread_ids();
139
140 scheduler_init();
141 turnstiles_init();
142
143 cmdline_parse(input: cmdline);
144 ndjson_init();
145 cmdline_debug_hook();
146
147 lapic_timer_init_bsp();
148 dpc_init_percpu();
149 smp_wake(mpr: mp_request.response);
150 timekeeper_init();
151 term_probe();
152
153 topology_init();
154 pertopo_obj_init();
155 perdomain_obj_init();
156 scheduler_domains_init();
157 tlb_init();
158 bootstage_advance(new: BOOTSTAGE_MID_TOPOLOGY);
159
160 lapic_clock_evdev_group_init();
161 timers_init_bsp();
162
163 scheduler_periodic_work_init();
164 movealloc_exec_all();
165 bootstage_advance(new: BOOTSTAGE_MID_ALLOCATORS);
166
167 scheduler_yield();
168}
169
170void k_sch_main(void *nop) {
171 cc_ignore(nop);
172 /* make sure everyone else is idle before we
173 * advance the bootstage here... */
174 smp_wait_for_others_to_idle();
175
176 /* we have to force everyone to disable their
177 * ticks because this prevents anyone from
178 * possibly entering an ISR since IRQL
179 * operations check bootstages and they can
180 * see the MID_ALLOCATORS bootstage (no-op) in
181 * an early `irql_raise` and the LATE_DEVICES
182 * bootstage later on (causing mis-raised IRQLs) */
183 smp_disable_all_ticks();
184
185 smp_boot_fence_raise();
186 bootstage_advance(new: BOOTSTAGE_LATE);
187 smp_boot_fence_lower();
188 lock_chk_init();
189 smp_timer_init();
190
191 smp_enable_all_ticks();
192 watchdog_start();
193 scheduler_debug_start();
194
195 rcu_init();
196 workqueues_permanent_init();
197 slab_domain_init_late();
198 domain_buddies_init_late();
199 reaper_init();
200 arena_global_init();
201
202 registry_setup();
203
204#ifdef TEST_NIGHTMARE_ENABLED
205 nightmare_run();
206#endif
207
208 tests_run();
209 bootstage_advance(new: BOOTSTAGE_COMPLETE);
210
211#ifdef TEST_ENABLED
212 return;
213#endif
214
215 thread_print(t: thread_get_current());
216
217 domain_buddy_dump();
218}
219