1#include <console/crash.h>
2#include <console/printf.h>
3#include <dbg.h>
4#include <irq/exception_sync_cb.h>
5#include <irq/irq.h>
6#include <mem/address_range.h>
7#include <mem/demand_page.h>
8#include <mem/hhdm.h>
9#include <mem/page_fault.h>
10#include <mem/page_table.h>
11#include <mem/pmm.h>
12#include <mem/vmm.h>
13#include <sch/sched.h>
14#include <string.h>
15#include <sync/spinlock.h>
16#include <thread/thread.h>
17
18static enum exception_sync_cb_result
19page_fault_sync_cb(struct exception_sync_cb *this, struct irq_context *irqc,
20 uint8_t buf[EXCEPTION_SYNC_CB_SCRATCH_BUFFER_SIZE]);
21
22static void __noreturn page_fault_report_crash(vaddr_t fault_addr,
23 uint64_t error_code,
24 struct irq_context *irqc);
25
26EXCEPTION_SYNC_CB_REGISTER(page_fault, IRQ_PAGE_FAULT, page_fault_sync_cb,
27 NULL);
28
29static struct spinlock pf_lock = SPINLOCK_INIT;
30
31enum irq_result page_fault_isr(void *context, uint8_t vector,
32 struct irq_context *rsp) {
33 (void) context, (void) vector;
34 /* Synchronization here is actually OK: we can only receive one
35 * IRQ per CPU at any moment, and the caller passes the thread-local buffer
36 * to the exception_sync_cb below, this is just how the ISR
37 * gets access to said buffer without extra parameters */
38 struct page_fault_scratch_buffer *pfsb =
39 (struct page_fault_scratch_buffer *) smp_core(cond: TOPC_IRQ)
40 ->irq_stack_scratch_buf;
41
42 uint64_t error_code = rsp->error_code;
43 uint64_t fault_addr;
44 asm volatile("mov %%cr2, %0" : "=r"(fault_addr));
45 pfsb->error_code = error_code;
46 pfsb->virt = fault_addr;
47
48 return IRQ_HANDLED;
49}
50
51static enum exception_sync_cb_result
52page_fault_sync_cb(struct exception_sync_cb *this, struct irq_context *irqc,
53 uint8_t buf[EXCEPTION_SYNC_CB_SCRATCH_BUFFER_SIZE]) {
54 struct page_fault_scratch_buffer *pfsb =
55 (struct page_fault_scratch_buffer *) buf;
56
57 vaddr_t vaddr = pfsb->virt;
58 uint64_t error = pfsb->error_code;
59
60 enum page_fault_access access_error;
61 if (error & PAGE_FAULT_EC_WRITE) {
62 access_error = PAGE_FAULT_WRITE;
63 } else if (error & PAGE_FAULT_EC_INSTRUCTION) {
64 access_error = PAGE_FAULT_EXEC;
65 } else {
66 access_error = PAGE_FAULT_READ;
67 }
68
69 struct page_fault_info pfi = {
70 .addr = vaddr,
71 .user = error & PAGE_FAULT_EC_USER,
72 .was_present = error & PAGE_FAULT_EC_PRESENT,
73 .access = access_error,
74 };
75
76 /*
77 * I'll need to rework this model... just leave it absent for now,
78 * no harm done, yet... TODO:
79 if (smp_core()->irq_entered_irql != IRQL_PASSIVE_LEVEL)
80 goto crash; */
81
82 /* Do the full crash here, it's not a valid address_range,
83 * we need the other information here */
84 struct address_range *adr = address_range_for_addr(vaddr);
85 if (!adr)
86 goto crash;
87
88 /* it just needs to exist */
89 struct page_fault_handler *pfh = adr->page_fault_handler;
90 kassert(pfh);
91 kassert(pfh->ops && pfh->ops->is_valid_fault);
92
93 if (!pfh->ops->is_valid_fault(&pfi))
94 goto crash;
95
96 /* Great, we have a valid vaddr, let's bring it in */
97
98 /* TODO: order > 0, i.e. hugepages */
99 pte_t pte = vmm_get_leaf_pte(vaddr);
100
101 /* Nice, another CPU mapped this in
102 * for us while we were dillying */
103 if (pte & PAGE_PRESENT)
104 goto done;
105
106 struct pte_tagged ptag = pte_tagged_unpack(pte);
107 kassert(ptag.type == PTE_TAG_TYPE_DEMAND_PAGED);
108 kassert((ptag.payload & DEMAND_PAGE_FLAG_ZERO_MEMORY) ||
109 (ptag.payload & DEMAND_PAGE_FLAG_NONE));
110 bool zeroed_out = ptag.payload & DEMAND_PAGE_FLAG_ZERO_MEMORY;
111 paddr_t paddr;
112
113 /* TODO: memory locality in allocations, we can use the alloc_pages
114 * function to supply that, but we do need to take note of this */
115 if (!pfh->ops->alloc_pages) {
116 paddr = pmm_alloc_page(); /* TODO: order > 0 */
117 } else {
118 paddr = pfh->ops->alloc_pages(vaddr, 0);
119 }
120
121 kassert(paddr); /* TODO: might be recoverable? many say no */
122
123 if (zeroed_out)
124 memset(hhdm_paddr_to_ptr(paddr), 0, PAGE_SIZE);
125
126 enum errno e = vmm_map_demand_page(vaddr, paddr, ptag.payload);
127 if (e == ERR_EXIST) {
128 pmm_free_page(addr: paddr);
129 return EXCEPTION_SYNC_CB_OK;
130 }
131
132 if (pfh->ops->update_after_map)
133 if (!pfh->ops->update_after_map(vaddr, page_for_paddr(paddr)))
134 pmm_free_page(addr: paddr);
135
136done:
137 return EXCEPTION_SYNC_CB_OK;
138
139crash:
140 page_fault_report_crash(fault_addr: vaddr, error_code: error, irqc);
141}
142
143static bool addr_is_mapped(uint64_t addr) {
144 return vmm_get_phys((vaddr_t) PAGE_ALIGN_DOWN(addr), VMM_FLAG_NONE) !=
145 (uintptr_t) -1;
146}
147
148static void dump_slab_exec_fault(struct thread *curr, struct irq_context *rsp) {
149 printf(format: "\n=== SLAB EXEC FAULT DEBUG ===\n");
150
151 printf(format: "Faulting RIP (from CPU): %p\n", rsp->rip);
152 printf(format: "Faulting RSP (from CPU): %p\n", rsp->rsp);
153
154 printf(format: "\n--- Current thread struct ---\n");
155 printf(format: "Thread struct addr: %p\n", (uint64_t) curr);
156
157 if (!addr_is_mapped(addr: (uint64_t) curr)) {
158 printf(format: " Thread pointer is NOT MAPPED - cannot dump\n");
159 return;
160 }
161
162 printf(format: " tid: %lu\n", curr->id);
163 printf(format: " name: %p", (uint64_t) curr->name);
164 if (curr->name && addr_is_mapped(addr: (uint64_t) curr->name))
165 printf(format: " -> \"%s\"", curr->name);
166 printf(format: "\n");
167 printf(format: " entry: %p\n", (uint64_t) curr->entry);
168 printf(format: " stack: %p (size %lu)\n", (uint64_t) curr->stack,
169 curr->stack_size);
170 printf(format: " state: %u\n", (uint32_t) curr->state);
171 printf(format: " core: %u\n", (uint32_t) curr->curr_core);
172 printf(format: " flags: 0x%lx\n", (uint64_t) thread_get_flags(t: curr));
173 printf(format: " ref: %lu\n", (uint64_t) refcount_read(rc: &curr->refcount));
174
175 printf(format: "\n--- Saved regs (post-switch residual) ---\n");
176 printf(format: " rbx = %p\n", curr->regs.rbx);
177 printf(format: " rbp = %p\n", curr->regs.rbp);
178 printf(format: " r12 = %p\n", curr->regs.r12);
179 printf(format: " r13 = %p\n", curr->regs.r13);
180 printf(format: " r14 = %p\n", curr->regs.r14);
181 printf(format: " r15 = %p\n", curr->regs.r15);
182 printf(format: " rsp = %p\n", curr->regs.rsp);
183 printf(format: " rip = %p\n", curr->regs.rip);
184
185 printf(format: "\n--- Thread struct raw dump (992 bytes) ---\n");
186 debug_print_memory(addr: (void *) curr, size: 992);
187
188 uint8_t *prev_obj = (uint8_t *) curr - 992;
189 printf(format: "\n--- Preceding slab object tail (last 256 bytes) ---\n");
190 if (addr_is_mapped(addr: (uint64_t) prev_obj)) {
191 debug_print_memory(addr: prev_obj + 992 - 256, size: 256);
192 } else {
193 printf(format: " Preceding object at %p is not mapped\n", (uint64_t) prev_obj);
194 }
195
196 printf(format: "\n--- Stack at fault RSP (%p) ---\n", rsp->rsp);
197 if (addr_is_mapped(addr: rsp->rsp)) {
198 debug_print_memory(addr: (void *) rsp->rsp, size: 256);
199 } else {
200 printf(format: " RSP is not mapped!\n");
201 }
202
203 printf(format: "\n--- ISR context (regs at fault time) ---\n");
204 printf(format: " rax=%p rbx=%p\n", rsp->rax, rsp->rbx);
205 printf(format: " rcx=%p rdx=%p\n", rsp->rcx, rsp->rdx);
206 printf(format: " rdi=%p rsi=%p\n", rsp->rdi, rsp->rsi);
207 printf(format: " rbp=%p rsp=%p\n", rsp->rbp, rsp->rsp);
208 printf(format: " r8 =%p r9 =%p\n", rsp->r8, rsp->r9);
209 printf(format: " r10=%p r11=%p\n", rsp->r10, rsp->r11);
210 printf(format: " r12=%p r13=%p\n", rsp->r12, rsp->r13);
211 printf(format: " r14=%p r15=%p\n", rsp->r14, rsp->r15);
212 printf(format: " rip=%p rfl=%p\n", rsp->rip, rsp->rflags);
213 printf(format: " cs=%p ss=%p\n", rsp->cs, rsp->ss);
214
215 /* Crash: raw smp_id is fine */
216 struct scheduler *sched = global.schedulers[smp_id_raw()];
217 printf(format: "\n--- Scheduler state (core %u) ---\n", smp_id_raw());
218 printf(format: " sched->current = %p\n", (uint64_t) sched->current);
219 printf(format: " sched->drop_last_ref = %p\n", (uint64_t) sched->drop_last_ref);
220 printf(format: " sched->other_locked = %p\n", (uint64_t) sched->other_locked);
221 printf(format: " sched->stealing_work = %u\n", (uint32_t) sched->stealing_work);
222
223 printf(format: "\n--- Thread migration info ---\n");
224 printf(format: " migrate_to = %ld\n", (int64_t) atomic_load(&curr->migrate_to));
225 printf(format: " migration_gen = 0x%lx\n", curr->migration_generation);
226 printf(format: " scheduler = %p\n", (uint64_t) atomic_load(&curr->scheduler));
227}
228
229static void __noreturn page_fault_report_crash(vaddr_t fault_addr,
230 uint64_t error_code,
231 struct irq_context *irqc) {
232
233 struct thread *curr = thread_get_current();
234
235 struct address_range *ar = address_range_for_addr(vaddr: fault_addr);
236 const char *name = ar ? ar->name : "UNKNOWN";
237
238 spin_lock_raw(&pf_lock);
239
240 printf(format: "\n=== PAGE FAULT === @ %p\n", irqc->rip);
241 printf(format: "Faulting Address (CR2): %p (ar: %s)\n", fault_addr, name);
242 printf(format: "Error Code: %p\n", error_code);
243 printf(format: " - Page not Present (P): %s\n",
244 (error_code & PAGE_FAULT_EC_PRESENT) ? "Yes" : "No");
245 printf(format: " - Write Access (W/R): %s\n",
246 (error_code & PAGE_FAULT_EC_WRITE) ? "Write" : "Read");
247 printf(format: " - User Mode (U/S): %s\n",
248 (error_code & PAGE_FAULT_EC_USER) ? "User" : "Supervisor");
249 printf(format: " - Reserved Bit Set (RSVD): %s\n",
250 (error_code & PAGE_FAULT_EC_RESERVED) ? "Yes" : "No");
251 printf(format: " - Instruction Fetch (I/D): %s\n",
252 (error_code & PAGE_FAULT_EC_INSTRUCTION) ? "Yes" : "No");
253 printf(format: " - Protection Key Violation (PK): %s\n",
254 (error_code & PAGE_FAULT_EC_PROTECTION_KEY) ? "Yes" : "No");
255 printf(format: " - Kernel stack %p -> %p\n", curr->stack,
256 (uintptr_t) curr->stack + curr->stack_size);
257
258 vaddr_t protector_base = (uintptr_t) curr->stack - PAGE_SIZE;
259 vaddr_t protector_top = (uintptr_t) curr->stack;
260 if (fault_addr >= protector_base && fault_addr <= protector_top)
261 printf(format: "Likely stack overflow!! Fault in protector page!!!\n");
262
263 vaddr_t code = PAGE_ALIGN_DOWN(irqc->rip);
264 if (vmm_get_phys(code, VMM_FLAG_NONE) != (paddr_t) -1) {
265 printf(format: "\n--- Bytes at RIP %p ---\n", (void *) irqc->rip);
266
267 for (int64_t i = -16; i < 16; i++) {
268 vaddr_t at = irqc->rip + i;
269
270 if (PAGE_ALIGN_DOWN(at) != code &&
271 vmm_get_phys(at, VMM_FLAG_NONE) == (paddr_t) -1)
272 continue;
273
274 printf(format: "%02x ", *(const uint8_t *) at);
275 }
276
277 printf(format: "\n");
278 }
279
280 printf(format: "\n--- Stack at fault RSP ---\n");
281 debug_print_stack_from(start: (uint64_t *) irqc->rsp, max_scan: 0);
282
283 bool is_slab_exec = (error_code & PAGE_FAULT_EC_INSTRUCTION) && ar &&
284 strcmp(str1: ar->name, str2: "slab") == 0;
285
286 if (!is_slab_exec && (error_code & PAGE_FAULT_EC_INSTRUCTION)) {
287 struct address_range *rip_ar = address_range_for_addr(vaddr: irqc->rip);
288 if (rip_ar && strcmp(str1: rip_ar->name, str2: "slab") == 0)
289 is_slab_exec = true;
290 }
291
292 if (is_slab_exec)
293 dump_slab_exec_fault(curr, rsp: irqc);
294
295 struct crash_regs pregs;
296 irq_context_to_crash_regs(ictx: irqc, out: &pregs);
297 char msg[CRASH_MSG_MAX];
298 snprintf(buffer: msg, buffer_len: sizeof(msg),
299 format: "Kernel Page Fault at %p (CR2: %p, ar: %s, ec: 0x%lx, thread: %s)",
300 (void *) irqc->rip, (void *) fault_addr, name, error_code,
301 curr ? curr->name : "none");
302
303 spin_unlock_raw(&pf_lock);
304
305 crash_full(ctx: &(struct crash_context){
306 .source = CRASH_SOURCE_CPU_EXCEPTION,
307 .formats = CRASH_FMT_DEFAULT,
308 .file = __FILE__,
309 .line = __LINE__,
310 .func = __func__,
311 .msg = msg,
312 .regs = &pregs,
313 });
314}
315