1#include <console/crash.h>
2#include <console/printf.h>
3#include <dbg.h>
4#include <irq/exception_sync_cb.h>
5#include <irq/irq.h>
6#include <math/range.h>
7#include <mem/address_range.h>
8#include <mem/demand_page.h>
9#include <mem/hhdm.h>
10#include <mem/page_fault.h>
11#include <mem/page_table.h>
12#include <mem/pmm.h>
13#include <mem/vmm.h>
14#include <sch/sched.h>
15#include <string.h>
16#include <sync/spinlock.h>
17#include <thread/thread.h>
18
19static enum exception_sync_cb_result
20page_fault_sync_cb(struct exception_sync_cb *this, struct irq_context *irqc,
21 uint8_t buf[EXCEPTION_SYNC_CB_SCRATCH_BUFFER_SIZE]);
22
23static void cc_noreturn page_fault_report_crash(vaddr_t fault_addr,
24 uint64_t error_code,
25 struct irq_context *irqc);
26
27EXCEPTION_SYNC_CB_REGISTER(page_fault, IRQ_PAGE_FAULT, page_fault_sync_cb,
28 NULL);
29
30static struct spinlock pf_lock = SPINLOCK_INIT;
31
32enum irq_result page_fault_isr(struct irq_context *rsp, void *ctx) {
33 cc_ignore(ctx);
34 /* Synchronization here is actually OK: we can only receive one
35 * IRQ per CPU at any moment, and the caller passes the thread-local buffer
36 * to the exception_sync_cb below, this is just how the ISR
37 * gets access to said buffer without extra parameters */
38 struct page_fault_scratch_buffer *pfsb =
39 (struct page_fault_scratch_buffer *) rsp->irq_stack_scratch_buf;
40
41 uint64_t error_code = rsp->regs->error_code;
42 uint64_t fault_addr;
43 asm volatile("mov %%cr2, %0" : "=r"(fault_addr));
44 pfsb->error_code = error_code;
45 pfsb->virt = fault_addr;
46
47 return IRQ_HANDLED;
48}
49
50static enum exception_sync_cb_result
51page_fault_sync_cb(struct exception_sync_cb *this, struct irq_context *irqc,
52 uint8_t buf[EXCEPTION_SYNC_CB_SCRATCH_BUFFER_SIZE]) {
53 cc_ignore(this);
54 struct page_fault_scratch_buffer *pfsb =
55 (struct page_fault_scratch_buffer *) buf;
56
57 vaddr_t vaddr = pfsb->virt;
58 uint64_t error = pfsb->error_code;
59
60 enum page_fault_access access_error;
61 if (error & PAGE_FAULT_EC_WRITE) {
62 access_error = PAGE_FAULT_WRITE;
63 } else if (error & PAGE_FAULT_EC_INSTRUCTION) {
64 access_error = PAGE_FAULT_EXEC;
65 } else {
66 access_error = PAGE_FAULT_READ;
67 }
68
69 struct page_fault_info pfi = {
70 .addr = vaddr,
71 .user = error & PAGE_FAULT_EC_USER,
72 .was_present = error & PAGE_FAULT_EC_PRESENT,
73 .access = access_error,
74 };
75
76 /*
77 * I'll need to rework this model... just leave it absent for now,
78 * no harm done, yet... TODO:
79 if (smp_core()->irq_entered_irql != IRQL_PASSIVE_LEVEL)
80 goto crash; */
81
82 /* Do the full crash here, it's not a valid address_range,
83 * we need the other information here */
84 struct address_range *adr = address_range_for_addr(vaddr);
85 if (!adr)
86 goto crash;
87
88 /* it just needs to exist */
89 struct page_fault_handler *pfh = adr->page_fault_handler;
90 enforce(pfh);
91 enforce(pfh->ops && pfh->ops->is_valid_fault);
92
93 if (!pfh->ops->is_valid_fault(&pfi))
94 goto crash;
95
96 /* Great, we have a valid vaddr, let's bring it in */
97
98 /* TODO: order > 0, i.e. hugepages */
99 pte_t pte = vmm_get_leaf_pte(vaddr);
100
101 /* Nice, another CPU mapped this in
102 * for us while we were dillying */
103 if (pte & PAGE_PRESENT)
104 goto done;
105
106 struct pte_tagged ptag = pte_tagged_unpack(pte);
107 enforce(ptag.type == PTE_TAG_TYPE_DEMAND_PAGED);
108 enforce((ptag.payload & DEMAND_PAGE_FLAG_ZERO_MEMORY) ||
109 (ptag.payload & DEMAND_PAGE_FLAG_NONE));
110 bool zeroed_out = ptag.payload & DEMAND_PAGE_FLAG_ZERO_MEMORY;
111 paddr_t paddr;
112
113 /* TODO: memory locality in allocations, we can use the alloc_pages
114 * function to supply that, but we do need to take note of this */
115 if (!pfh->ops->alloc_pages) {
116 paddr = pmm_alloc_page(); /* TODO: order > 0 */
117 } else {
118 paddr = pfh->ops->alloc_pages(vaddr, 0);
119 }
120
121 enforce(paddr); /* TODO: might be recoverable? many say no */
122
123 if (zeroed_out)
124 memset(hhdm_paddr_to_ptr(paddr), 0, PAGE_SIZE);
125
126 enum err e = vmm_map_demand_page(vaddr, paddr, ptag.payload);
127 if (e == ERR_EXIST) {
128 pmm_free_page(addr: paddr);
129 return EXCEPTION_SYNC_CB_OK;
130 }
131
132 if (pfh->ops->update_after_map)
133 if (!pfh->ops->update_after_map(vaddr, page_for_paddr(paddr)))
134 pmm_free_page(addr: paddr);
135
136done:
137 return EXCEPTION_SYNC_CB_OK;
138
139crash:
140 page_fault_report_crash(fault_addr: vaddr, error_code: error, irqc);
141}
142
143static bool addr_is_mapped(uint64_t addr) {
144 return vmm_get_phys((vaddr_t) PAGE_ALIGN_DOWN(addr)) != (uintptr_t) -1;
145}
146
147static void dump_slab_exec_fault(struct thread *curr, struct irq_context *ctx) {
148 printf(format: "\n=== SLAB EXEC FAULT DEBUG ===\n");
149
150 struct irq_registers *rsp = ctx->regs;
151 printf(format: "Faulting RIP (from CPU): %p\n", (void *) rsp->rip);
152 printf(format: "Faulting RSP (from CPU): %p\n", (void *) rsp->rsp);
153
154 printf(format: "\n--- Current thread struct ---\n");
155 printf(format: "Thread struct addr: %p\n", (void *) curr);
156
157 if (!addr_is_mapped(addr: (uint64_t) curr)) {
158 printf(format: " Thread pointer is NOT MAPPED - cannot dump\n");
159 return;
160 }
161
162 printf(format: " tid: %lu\n", curr->id);
163 printf(format: " name: %p", (void *) curr->name);
164 if (curr->name && addr_is_mapped(addr: (uint64_t) curr->name))
165 printf(format: " -> \"%s\"", curr->name);
166 printf(format: "\n");
167 printf(format: " entry: %p\n", (void *) curr->entry);
168 printf(format: " stack: %p (size %lu)\n", (void *) curr->stack, curr->stack_size);
169 printf(format: " state: %u\n", (uint32_t) atomic_load_rlx(&curr->state));
170 printf(format: " core: %u\n", (uint32_t) curr->curr_core);
171 printf(format: " flags: 0x%lx\n", (uint64_t) thread_get_flags(t: curr));
172 printf(format: " ref: %lu\n", (uint64_t) refcount_read(rc: &curr->refcount));
173
174 printf(format: "\n--- Saved regs (post-switch residual) ---\n");
175 printf(format: " rbx = %p\n", (void *) curr->regs.rbx);
176 printf(format: " rbp = %p\n", (void *) curr->regs.rbp);
177 printf(format: " r12 = %p\n", (void *) curr->regs.r12);
178 printf(format: " r13 = %p\n", (void *) curr->regs.r13);
179 printf(format: " r14 = %p\n", (void *) curr->regs.r14);
180 printf(format: " r15 = %p\n", (void *) curr->regs.r15);
181 printf(format: " rsp = %p\n", (void *) curr->regs.rsp);
182 printf(format: " rip = %p\n", (void *) curr->regs.rip);
183
184 printf(format: "\n--- Thread struct raw dump (992 bytes) ---\n");
185 debug_print_memory(addr: (void *) curr, size: 992);
186
187 uint8_t *prev_obj = (uint8_t *) curr - 992;
188 printf(format: "\n--- Preceding slab object tail (last 256 bytes) ---\n");
189 if (addr_is_mapped(addr: (uint64_t) prev_obj)) {
190 debug_print_memory(addr: prev_obj + 992 - 256, size: 256);
191 } else {
192 printf(format: " Preceding object at %p is not mapped\n", (void *) prev_obj);
193 }
194
195 printf(format: "\n--- Stack at fault RSP (%p) ---\n", (void *) rsp->rsp);
196 if (addr_is_mapped(addr: rsp->rsp)) {
197 debug_print_memory(addr: (void *) rsp->rsp, size: 256);
198 } else {
199 printf(format: " RSP is not mapped!\n");
200 }
201
202 printf(format: "\n--- ISR context (regs at fault time) ---\n");
203 printf(format: " rax=%p rbx=%p\n", (void *) rsp->rax, (void *) rsp->rbx);
204 printf(format: " rcx=%p rdx=%p\n", (void *) rsp->rcx, (void *) rsp->rdx);
205 printf(format: " rdi=%p rsi=%p\n", (void *) rsp->rdi, (void *) rsp->rsi);
206 printf(format: " rbp=%p rsp=%p\n", (void *) rsp->rbp, (void *) rsp->rsp);
207 printf(format: " r8 =%p r9 =%p\n", (void *) rsp->r8, (void *) rsp->r9);
208 printf(format: " r10=%p r11=%p\n", (void *) rsp->r10, (void *) rsp->r11);
209 printf(format: " r12=%p r13=%p\n", (void *) rsp->r12, (void *) rsp->r13);
210 printf(format: " r14=%p r15=%p\n", (void *) rsp->r14, (void *) rsp->r15);
211 printf(format: " rip=%p rfl=%p\n", (void *) rsp->rip, (void *) rsp->rflags);
212 printf(format: " cs=%p ss=%p\n", (void *) rsp->cs, (void *) rsp->ss);
213
214 /* Crash: raw smp_id is fine */
215 struct scheduler *sched = global.schedulers[smp_id_raw()];
216 printf(format: "\n--- Scheduler state (core %zu) ---\n", smp_id_raw());
217 printf(format: " sched->current = %p\n", (void *) sched->current);
218 printf(format: " sched->drop_last_ref = %p\n", (void *) sched->drop_last_ref);
219 printf(format: " sched->other_locked = %p\n", (void *) sched->other_locked);
220 printf(format: " sched->stealing_work = %u\n",
221 (uint32_t) atomic_load_rlx(&sched->stealing_work));
222
223 printf(format: "\n--- Thread migration info ---\n");
224 printf(format: " migrate_to = %ld\n", (int64_t) atomic_load(&curr->migrate_to));
225 printf(format: " migration_gen = 0x%lx\n",
226 atomic_load_rlx(&curr->migration_generation));
227 printf(format: " scheduler = %p\n", (void *) atomic_load(&curr->scheduler));
228}
229
230static cc_noreturn void page_fault_report_crash(vaddr_t fault_addr,
231 uint64_t error_code,
232 struct irq_context *ctx) {
233
234 struct thread *curr = thread_get_current();
235
236 struct address_range *ar = address_range_for_addr(vaddr: fault_addr);
237 const char *name = ar ? ar->name : "UNKNOWN";
238
239 struct irq_registers *irqc = ctx->regs;
240
241 spin_lock_raw(&pf_lock);
242
243 printf(format: "\n=== PAGE FAULT === @ %p\n", (void *) irqc->rip);
244 printf(format: "Faulting Address (CR2): %p (ar: %s)\n", (void *) fault_addr, name);
245 printf(format: "Error Code: 0x%lx\n", error_code);
246 printf(format: " - Page not Present (P): %s\n",
247 (error_code & PAGE_FAULT_EC_PRESENT) ? "Yes" : "No");
248 printf(format: " - Write Access (W/R): %s\n",
249 (error_code & PAGE_FAULT_EC_WRITE) ? "Write" : "Read");
250 printf(format: " - User Mode (U/S): %s\n",
251 (error_code & PAGE_FAULT_EC_USER) ? "User" : "Supervisor");
252 printf(format: " - Reserved Bit Set (RSVD): %s\n",
253 (error_code & PAGE_FAULT_EC_RESERVED) ? "Yes" : "No");
254 printf(format: " - Instruction Fetch (I/D): %s\n",
255 (error_code & PAGE_FAULT_EC_INSTRUCTION) ? "Yes" : "No");
256 printf(format: " - Protection Key Violation (PK): %s\n",
257 (error_code & PAGE_FAULT_EC_PROTECTION_KEY) ? "Yes" : "No");
258 printf(format: " - Kernel stack %p -> %p\n", curr->stack,
259 (void *) ((uintptr_t) curr->stack + curr->stack_size));
260
261 vaddr_t protector_base = (uintptr_t) curr->stack - PAGE_SIZE;
262 vaddr_t protector_top = (uintptr_t) curr->stack;
263 if (IN_RANGE(fault_addr, protector_base, protector_top))
264 printf(format: "Likely stack overflow!! Fault in protector page!!!\n");
265
266 vaddr_t code = PAGE_ALIGN_DOWN(irqc->rip);
267 if (vmm_get_phys(code) != (paddr_t) -1) {
268 printf(format: "\n--- Bytes at RIP %p ---\n", (void *) irqc->rip);
269
270 for (int64_t i = -16; i < 16; i++) {
271 vaddr_t at = irqc->rip + i;
272
273 if (PAGE_ALIGN_DOWN(at) != code && vmm_get_phys(at) == (paddr_t) -1)
274 continue;
275
276 printf(format: "%02x ", *(const uint8_t *) at);
277 }
278
279 printf(format: "\n");
280 }
281
282 printf(format: "\n--- Stack at fault RSP ---\n");
283 debug_print_stack_from(start: (uint64_t *) irqc->rsp, max_scan: 0);
284
285 bool is_slab_exec = (error_code & PAGE_FAULT_EC_INSTRUCTION) && ar &&
286 strcmp(str1: ar->name, str2: "slab") == 0;
287
288 if (!is_slab_exec && (error_code & PAGE_FAULT_EC_INSTRUCTION)) {
289 struct address_range *rip_ar = address_range_for_addr(vaddr: irqc->rip);
290 if (rip_ar && strcmp(str1: rip_ar->name, str2: "slab") == 0)
291 is_slab_exec = true;
292 }
293
294 if (is_slab_exec)
295 dump_slab_exec_fault(curr, ctx);
296
297 struct crash_regs pregs;
298 irq_context_to_crash_regs(ictx: irqc, out: &pregs);
299 char msg[CRASH_MSG_MAX];
300 snprintf(msg, sizeof(msg),
301 "Kernel Page Fault at %p (CR2: %p, ar: %s, ec: 0x%lx, thread: %s)",
302 (void *) irqc->rip, (void *) fault_addr, name, error_code,
303 curr ? curr->name : "none");
304
305 spin_unlock_raw(&pf_lock);
306
307 crash_full(ctx: &(struct crash_context){
308 .source = CRASH_SOURCE_CPU_EXCEPTION,
309 .formats = CRASH_FMT_DEFAULT,
310 .file = __FILE__,
311 .line = __LINE__,
312 .func = __func__,
313 .msg = msg,
314 .regs = &pregs,
315 });
316}
317