1#include <console/printf.h>
2#include <dbg.h>
3#include <irq/exception_sync_cb.h>
4#include <irq/irq.h>
5#include <mem/address_range.h>
6#include <mem/demand_page.h>
7#include <mem/hhdm.h>
8#include <mem/page_fault.h>
9#include <mem/page_table.h>
10#include <mem/pmm.h>
11#include <mem/vmm.h>
12#include <sch/sched.h>
13#include <string.h>
14#include <sync/spinlock.h>
15#include <thread/thread.h>
16
17static enum exception_sync_cb_result
18page_fault_sync_cb(struct exception_sync_cb *this, struct irq_context *irqc,
19 uint8_t buf[EXCEPTION_SYNC_CB_SCRATCH_BUFFER_SIZE]);
20
21static void __noreturn page_fault_report_crash(vaddr_t fault_addr,
22 uint64_t error_code,
23 struct irq_context *irqc);
24
25EXCEPTION_SYNC_CB_REGISTER(page_fault, IRQ_PAGE_FAULT, page_fault_sync_cb,
26 NULL);
27
28static struct spinlock pf_lock = SPINLOCK_INIT;
29
30enum irq_result page_fault_isr(void *context, uint8_t vector,
31 struct irq_context *rsp) {
32 (void) context, (void) vector;
33 struct page_fault_scratch_buffer *pfsb =
34 (struct page_fault_scratch_buffer *) smp_core()->irq_stack_scratch_buf;
35
36 uint64_t error_code = rsp->error_code;
37 uint64_t fault_addr;
38 asm volatile("mov %%cr2, %0" : "=r"(fault_addr));
39 pfsb->error_code = error_code;
40 pfsb->virt = fault_addr;
41
42 return IRQ_HANDLED;
43}
44
45static enum exception_sync_cb_result
46page_fault_sync_cb(struct exception_sync_cb *this, struct irq_context *irqc,
47 uint8_t buf[EXCEPTION_SYNC_CB_SCRATCH_BUFFER_SIZE]) {
48 struct page_fault_scratch_buffer *pfsb =
49 (struct page_fault_scratch_buffer *) buf;
50
51 vaddr_t vaddr = pfsb->virt;
52 uint64_t error = pfsb->error_code;
53
54 enum page_fault_access access_error;
55 if (error & PAGE_FAULT_EC_WRITE) {
56 access_error = PAGE_FAULT_WRITE;
57 } else if (error & PAGE_FAULT_EC_INSTRUCTION) {
58 access_error = PAGE_FAULT_EXEC;
59 } else {
60 access_error = PAGE_FAULT_READ;
61 }
62
63 struct page_fault_info pfi = {
64 .addr = vaddr,
65 .user = error & PAGE_FAULT_EC_USER,
66 .was_present = error & PAGE_FAULT_EC_PRESENT,
67 .access = access_error,
68 };
69
70 /*
71 * I'll need to rework this model... just leave it absent for now,
72 * no harm done, yet... TODO:
73 if (smp_core()->irq_entered_irql != IRQL_PASSIVE_LEVEL)
74 goto crash; */
75
76 /* Do the full crash here, it's not a valid address_range,
77 * we need the other information here */
78 struct address_range *adr = address_range_for_addr(vaddr);
79 if (!adr)
80 goto crash;
81
82 /* it just needs to exist */
83 struct page_fault_handler *pfh = adr->page_fault_handler;
84 kassert(pfh);
85 kassert(pfh->ops && pfh->ops->is_valid_fault);
86
87 if (!pfh->ops->is_valid_fault(&pfi))
88 goto crash;
89
90 /* Great, we have a valid vaddr, let's bring it in */
91
92 /* TODO: order > 0, i.e. hugepages */
93 pte_t pte = vmm_get_leaf_pte(vaddr);
94
95 /* Nice, another CPU mapped this in
96 * for us while we were dillying */
97 if (pte & PAGE_PRESENT)
98 goto done;
99
100 struct pte_tagged ptag = pte_tagged_unpack(pte);
101 kassert(ptag.type == PTE_TAG_TYPE_DEMAND_PAGED);
102 kassert((ptag.payload & DEMAND_PAGE_FLAG_ZERO_MEMORY) ||
103 (ptag.payload & DEMAND_PAGE_FLAG_NONE));
104 bool zeroed_out = ptag.payload & DEMAND_PAGE_FLAG_ZERO_MEMORY;
105 paddr_t paddr;
106
107 /* TODO: memory locality in allocations, we can use the alloc_pages
108 * function to supply that, but we do need to take note of this */
109 if (!pfh->ops->alloc_pages) {
110 paddr = pmm_alloc_page(); /* TODO: order > 0 */
111 } else {
112 paddr = pfh->ops->alloc_pages(vaddr, 0);
113 }
114
115 kassert(paddr); /* TODO: might be recoverable? many say no */
116
117 if (zeroed_out)
118 memset(hhdm_paddr_to_ptr(paddr), 0, PAGE_SIZE);
119
120 enum errno e = vmm_map_demand_page(vaddr, paddr, ptag.payload);
121 if (e == ERR_EXIST) {
122 pmm_free_page(addr: paddr);
123 return EXCEPTION_SYNC_CB_OK;
124 }
125
126 if (pfh->ops->update_after_map)
127 if (!pfh->ops->update_after_map(vaddr, page_for_paddr(paddr)))
128 pmm_free_page(addr: paddr);
129
130done:
131 return EXCEPTION_SYNC_CB_OK;
132
133crash:
134 page_fault_report_crash(fault_addr: vaddr, error_code: error, irqc);
135}
136
137static bool addr_is_mapped(uint64_t addr) {
138 return vmm_get_phys((vaddr_t) PAGE_ALIGN_DOWN(addr), VMM_FLAG_NONE) !=
139 (uintptr_t) -1;
140}
141
142static void dump_slab_exec_fault(struct thread *curr, struct irq_context *rsp) {
143 printf(format: "\n=== SLAB EXEC FAULT DEBUG ===\n");
144
145 printf(format: "Faulting RIP (from CPU): %p\n", rsp->rip);
146 printf(format: "Faulting RSP (from CPU): %p\n", rsp->rsp);
147
148 printf(format: "\n--- Current thread struct ---\n");
149 printf(format: "Thread struct addr: %p\n", (uint64_t) curr);
150
151 if (!addr_is_mapped(addr: (uint64_t) curr)) {
152 printf(format: " Thread pointer is NOT MAPPED - cannot dump\n");
153 return;
154 }
155
156 printf(format: " tid: %lu\n", curr->id);
157 printf(format: " name: %p", (uint64_t) curr->name);
158 if (curr->name && addr_is_mapped(addr: (uint64_t) curr->name))
159 printf(format: " -> \"%s\"", curr->name);
160 printf(format: "\n");
161 printf(format: " entry: %p\n", (uint64_t) curr->entry);
162 printf(format: " stack: %p (size %lu)\n", (uint64_t) curr->stack,
163 curr->stack_size);
164 printf(format: " state: %u\n", (uint32_t) curr->state);
165 printf(format: " core: %u\n", (uint32_t) curr->curr_core);
166 printf(format: " flags: 0x%lx\n", (uint64_t) thread_get_flags(t: curr));
167 printf(format: " ref: %lu\n", (uint64_t) refcount_read(rc: &curr->refcount));
168
169 printf(format: "\n--- Saved regs (post-switch residual) ---\n");
170 printf(format: " rbx = %p\n", curr->regs.rbx);
171 printf(format: " rbp = %p\n", curr->regs.rbp);
172 printf(format: " r12 = %p\n", curr->regs.r12);
173 printf(format: " r13 = %p\n", curr->regs.r13);
174 printf(format: " r14 = %p\n", curr->regs.r14);
175 printf(format: " r15 = %p\n", curr->regs.r15);
176 printf(format: " rsp = %p\n", curr->regs.rsp);
177 printf(format: " rip = %p\n", curr->regs.rip);
178
179 printf(format: "\n--- Thread struct raw dump (992 bytes) ---\n");
180 debug_print_memory(addr: (void *) curr, size: 992);
181
182 uint8_t *prev_obj = (uint8_t *) curr - 992;
183 printf(format: "\n--- Preceding slab object tail (last 256 bytes) ---\n");
184 if (addr_is_mapped(addr: (uint64_t) prev_obj)) {
185 debug_print_memory(addr: prev_obj + 992 - 256, size: 256);
186 } else {
187 printf(format: " Preceding object at %p is not mapped\n", (uint64_t) prev_obj);
188 }
189
190 printf(format: "\n--- Stack at fault RSP (%p) ---\n", rsp->rsp);
191 if (addr_is_mapped(addr: rsp->rsp)) {
192 debug_print_memory(addr: (void *) rsp->rsp, size: 256);
193 } else {
194 printf(format: " RSP is not mapped!\n");
195 }
196
197 printf(format: "\n--- ISR context (regs at fault time) ---\n");
198 printf(format: " rax=%p rbx=%p\n", rsp->rax, rsp->rbx);
199 printf(format: " rcx=%p rdx=%p\n", rsp->rcx, rsp->rdx);
200 printf(format: " rdi=%p rsi=%p\n", rsp->rdi, rsp->rsi);
201 printf(format: " rbp=%p rsp=%p\n", rsp->rbp, rsp->rsp);
202 printf(format: " r8 =%p r9 =%p\n", rsp->r8, rsp->r9);
203 printf(format: " r10=%p r11=%p\n", rsp->r10, rsp->r11);
204 printf(format: " r12=%p r13=%p\n", rsp->r12, rsp->r13);
205 printf(format: " r14=%p r15=%p\n", rsp->r14, rsp->r15);
206 printf(format: " rip=%p rfl=%p\n", rsp->rip, rsp->rflags);
207 printf(format: " cs=%p ss=%p\n", rsp->cs, rsp->ss);
208
209 struct scheduler *sched = global.schedulers[smp_core_id()];
210 printf(format: "\n--- Scheduler state (core %u) ---\n", smp_core_id());
211 printf(format: " sched->current = %p\n", (uint64_t) sched->current);
212 printf(format: " sched->drop_last_ref = %p\n", (uint64_t) sched->drop_last_ref);
213 printf(format: " sched->other_locked = %p\n", (uint64_t) sched->other_locked);
214 printf(format: " sched->stealing_work = %u\n", (uint32_t) sched->stealing_work);
215
216 printf(format: "\n--- Thread migration info ---\n");
217 printf(format: " migrate_to = %ld\n", (int64_t) atomic_load(&curr->migrate_to));
218 printf(format: " migration_gen = 0x%lx\n", curr->migration_generation);
219 printf(format: " scheduler = %p\n", (uint64_t) atomic_load(&curr->scheduler));
220}
221
222static void __noreturn page_fault_report_crash(vaddr_t fault_addr,
223 uint64_t error_code,
224 struct irq_context *irqc) {
225
226 struct thread *curr = thread_get_current();
227
228 struct address_range *ar = address_range_for_addr(vaddr: fault_addr);
229 const char *name = ar ? ar->name : "UNKNOWN";
230
231 spin_lock_raw(lock: &pf_lock);
232
233 printf(format: "\n=== PAGE FAULT === @ %p\n", irqc->rip);
234 printf(format: "Faulting Address (CR2): %p (ar: %s)\n", fault_addr, name);
235 printf(format: "Error Code: %p\n", error_code);
236 printf(format: " - Page not Present (P): %s\n",
237 (error_code & PAGE_FAULT_EC_PRESENT) ? "Yes" : "No");
238 printf(format: " - Write Access (W/R): %s\n",
239 (error_code & PAGE_FAULT_EC_WRITE) ? "Write" : "Read");
240 printf(format: " - User Mode (U/S): %s\n",
241 (error_code & PAGE_FAULT_EC_USER) ? "User" : "Supervisor");
242 printf(format: " - Reserved Bit Set (RSVD): %s\n",
243 (error_code & PAGE_FAULT_EC_RESERVED) ? "Yes" : "No");
244 printf(format: " - Instruction Fetch (I/D): %s\n",
245 (error_code & PAGE_FAULT_EC_INSTRUCTION) ? "Yes" : "No");
246 printf(format: " - Protection Key Violation (PK): %s\n",
247 (error_code & PAGE_FAULT_EC_PROTECTION_KEY) ? "Yes" : "No");
248 printf(format: " - Kernel stack %p -> %p\n", curr->stack,
249 (uintptr_t) curr->stack + curr->stack_size);
250
251 vaddr_t protector_base = (uintptr_t) curr->stack - PAGE_SIZE;
252 vaddr_t protector_top = (uintptr_t) curr->stack;
253 if (fault_addr >= protector_base && fault_addr <= protector_top)
254 printf(format: "Likely stack overflow!! Fault in protector page!!!\n");
255
256 printf(format: "\n--- Stack at fault RSP ---\n");
257 debug_print_stack_from(start: (uint64_t *) irqc->rsp, max_scan: 0);
258
259 bool is_slab_exec = (error_code & PAGE_FAULT_EC_INSTRUCTION) && ar &&
260 strcmp(str1: ar->name, str2: "slab") == 0;
261
262 if (!is_slab_exec && (error_code & PAGE_FAULT_EC_INSTRUCTION)) {
263 struct address_range *rip_ar = address_range_for_addr(vaddr: irqc->rip);
264 if (rip_ar && strcmp(str1: rip_ar->name, str2: "slab") == 0)
265 is_slab_exec = true;
266 }
267
268 if (is_slab_exec)
269 dump_slab_exec_fault(curr, rsp: irqc);
270
271 if (!(error_code & PAGE_FAULT_EC_USER)) {
272 spin_unlock_raw(lock: &pf_lock);
273 panic("KERNEL PAGE FAULT ON CORE %llu under thread %s", smp_core_id(),
274 thread_get_current()->name);
275 while (true) {
276 disable_interrupts();
277 wait_for_interrupt();
278 }
279 }
280
281 spin_unlock_raw(lock: &pf_lock);
282 panic("?");
283}
284