1/* @title: Kernel Crash Engine */
2#include <acpi/lapic.h>
3#include <asm.h>
4#include <bootstage.h>
5#include <console/crash.h>
6#include <console/panic_scene.h>
7#include <console/printf.h>
8#include <console/report.h>
9#include <console/statusbar.h>
10#include <console/term.h>
11#include <dbg.h>
12#include <global.h>
13#include <irq/irq.h>
14#include <linker/symbols.h>
15#include <log.h>
16#include <logo.h>
17#include <math/sort.h>
18#include <ndjson.h>
19#include <smp/core.h>
20#include <smp/percpu.h>
21#include <stdarg.h>
22#include <string.h>
23#include <sync/mutex.h>
24#include <sync/qspinlock.h>
25#include <sync/raw_spinlock.h>
26#include <sync/rwlock.h>
27#include <sync/spinlock.h>
28#include <thread/thread.h>
29#include <time/spin_sleep.h>
30#include <time/time.h>
31
32static _Atomic int64_t crash_owner = -1;
33static _Atomic uint32_t crash_depth = 0;
34static struct raw_spinlock crash_lock = RAW_SPINLOCK_INIT;
35
36PERCPU_DECLARE(crash_quiesced, _Atomic uint32_t, NULL);
37PERCPU_DECLARE(crash_regs, struct crash_regs, NULL);
38static struct crash_regs boot_crash_regs = {0};
39
40NDJSON_DECLARE(panic_at, NDJSON_SECTION_PANIC, NDJSON_KIND_AT, 1,
41 NDJSON_STR(file), NDJSON_U64(line), NDJSON_STR(func),
42 NDJSON_STR(msg), NDJSON_STR(bootstage), NDJSON_STR(thread),
43 NDJSON_U64(depth));
44
45NDJSON_DECLARE(panic_frame, NDJSON_SECTION_PANIC, NDJSON_KIND_FRAME, 1,
46 NDJSON_U64(idx), NDJSON_HEX(addr), NDJSON_STR(sym),
47 NDJSON_U64(off), NDJSON_STR(file), NDJSON_U64(line));
48
49extern void crash_capture_regs(struct crash_regs *out);
50static struct crash_facility *crash_facility_for(enum crash_code code);
51
52bool crash_cpu_is_owner(uint64_t id) {
53 return atomic_load_explicit(&crash_owner, memory_order_relaxed) ==
54 (int64_t) id;
55}
56
57void crash_nmi_handoff(void *p, struct irq_context *irqc) {
58 (void) p;
59 /* _NONE here for safety reasons (the validator could
60 * crash again depending on why we crashed),
61 * it's the crash context anyways */
62 struct crash_regs *this_regs_buf = PERCPU_PTR(TOPC_NONE, crash_regs);
63 this_regs_buf->r8 = irqc->r8;
64 this_regs_buf->r9 = irqc->r9;
65 this_regs_buf->r10 = irqc->r10;
66 this_regs_buf->r11 = irqc->r11;
67 this_regs_buf->r12 = irqc->r12;
68 this_regs_buf->r13 = irqc->r13;
69 this_regs_buf->r14 = irqc->r14;
70 this_regs_buf->r15 = irqc->r15;
71 this_regs_buf->rdi = irqc->rdi;
72 this_regs_buf->rsi = irqc->rsi;
73 this_regs_buf->rbp = irqc->rbp;
74 this_regs_buf->rsp = irqc->rsp;
75 this_regs_buf->rax = irqc->rax;
76 this_regs_buf->rbx = irqc->rbx;
77 this_regs_buf->rcx = irqc->rcx;
78 this_regs_buf->rdx = irqc->rdx;
79 this_regs_buf->rip = irqc->rip;
80 this_regs_buf->rflags = irqc->rflags;
81 this_regs_buf->cr2 = read_cr2();
82 this_regs_buf->cr3 = read_cr3();
83 atomic_store(PERCPU_PTR(TOPC_NONE, crash_quiesced), 1);
84 while (1)
85 hcf();
86}
87
88void crash_broadcast_nmi(void) {
89 panic_broadcast(exclude_core: smp_id(cond: TOPC_NONE));
90}
91
92void panic_handler(struct crash_regs *regs) {
93 disable_interrupts();
94
95 if (PERCPU_READY(crash_regs)) {
96 PERCPU_READ(TOPC_NONE, crash_regs) = *regs;
97 } else {
98 boot_crash_regs = *regs;
99 }
100
101 if (global.current_bootstage >= BOOTSTAGE_MID_MP) {
102 crash_broadcast_nmi();
103 sleep_spin_ms(msec: 50);
104 }
105}
106
107static bool crash_regs_captured(const struct crash_regs *r) {
108 for (int i = 0; i < CRASH_REG_COUNT; i++) {
109 if (r->regs[i])
110 return true;
111 }
112 return false;
113}
114
115struct crash_reg_entry {
116 const char *name;
117 uint64_t val;
118 bool ret_addr;
119};
120
121static size_t crash_regs_collect(const struct crash_regs *r,
122 struct crash_reg_entry *out) {
123 size_t n = 0;
124 out[n++] = (struct crash_reg_entry){"rax", r->rax, false};
125 out[n++] = (struct crash_reg_entry){"rbx", r->rbx, false};
126 out[n++] = (struct crash_reg_entry){"rcx", r->rcx, false};
127 out[n++] = (struct crash_reg_entry){"rdx", r->rdx, false};
128 out[n++] = (struct crash_reg_entry){"rsi", r->rsi, false};
129 out[n++] = (struct crash_reg_entry){"rdi", r->rdi, false};
130 out[n++] = (struct crash_reg_entry){"rbp", r->rbp, false};
131 out[n++] = (struct crash_reg_entry){"rsp", r->rsp, false};
132 out[n++] = (struct crash_reg_entry){" r8", r->r8, false};
133 out[n++] = (struct crash_reg_entry){" r9", r->r9, false};
134 out[n++] = (struct crash_reg_entry){"r10", r->r10, false};
135 out[n++] = (struct crash_reg_entry){"r11", r->r11, false};
136 out[n++] = (struct crash_reg_entry){"r12", r->r12, false};
137 out[n++] = (struct crash_reg_entry){"r13", r->r13, false};
138 out[n++] = (struct crash_reg_entry){"r14", r->r14, false};
139 out[n++] = (struct crash_reg_entry){"r15", r->r15, false};
140 return n;
141}
142
143struct crash_addr_info {
144 const char *sym;
145 uint64_t offset;
146 const char *file;
147 uint32_t line;
148};
149
150static struct crash_addr_info crash_resolve_addr(uint64_t addr,
151 bool is_ret_addr) {
152 struct crash_addr_info info = {0};
153 uint64_t lookup_addr = (is_ret_addr && addr) ? addr - 1 : addr;
154
155 if (addr >= (uint64_t) &__stext && addr < (uint64_t) &__etext) {
156 info.sym = debug_symbolize(addr: lookup_addr, out_off: &info.offset);
157 info.file = debug_line_for(addr: lookup_addr, out_line: &info.line);
158 }
159 return info;
160}
161
162static void crash_describe_addr(uint64_t v, char *out, size_t cap) {
163 if (v >= (uint64_t) &__stext && v < (uint64_t) &__etext) {
164 struct crash_addr_info info = crash_resolve_addr(addr: v, false);
165 if (info.sym && info.file)
166 snprintf(buffer: out, buffer_len: (int) cap, format: "%s+0x%lx at %s:%u", info.sym, info.offset,
167 info.file, info.line);
168 else if (info.sym)
169 snprintf(buffer: out, buffer_len: (int) cap, format: "%s+0x%lx", info.sym, info.offset);
170 else if (info.file)
171 snprintf(buffer: out, buffer_len: (int) cap, format: "<text> at %s:%u", info.file, info.line);
172 else
173 snprintf(buffer: out, buffer_len: (int) cap, format: "<text>");
174 return;
175 }
176
177 if (v >= (uint64_t) &__srodata && v < (uint64_t) &__erodata)
178 snprintf(buffer: out, buffer_len: (int) cap, format: "<rodata>");
179 else if (v >= (uint64_t) &__sdata && v < (uint64_t) &__edata)
180 snprintf(buffer: out, buffer_len: (int) cap, format: "<data>");
181 else if (v >= (uint64_t) &__sbss && v < (uint64_t) &__ebss)
182 snprintf(buffer: out, buffer_len: (int) cap, format: "<bss>");
183 else
184 snprintf(buffer: out, buffer_len: (int) cap, format: "<none>");
185}
186
187static void crash_reg_str(char *out, size_t cap, const char *name, uint64_t v,
188 const char *note) {
189 snprintf(buffer: out, buffer_len: (int) cap, format: "%s%s%s %016lx %s%s%s",
190 term_style(sev: TERM_SEV_LABEL), name, term_style_reset(), v,
191 term_style(sev: TERM_SEV_DIM), note, term_style_reset());
192}
193
194static void crash_reg_fmt(char *out, size_t cap, const char *name, uint64_t v,
195 bool ret_addr) {
196 char note[REPORT_PANE_LINE_MAX];
197 crash_describe_addr(v: ret_addr && v ? v - 1 : v, out: note, cap: sizeof(note));
198 crash_reg_str(out, cap, name, v, note);
199}
200
201static void crash_reg_note(struct report_target *tgt, const char *name,
202 uint64_t v, const char *note) {
203 char line[REPORT_PANE_LINE_MAX];
204 crash_reg_str(out: line, cap: sizeof(line), name, v, note);
205 report_puts(tgt, s: line);
206}
207
208static void crash_reg_line(struct report_target *tgt, const char *name,
209 uint64_t v, bool ret_addr) {
210 char line[REPORT_PANE_LINE_MAX];
211 crash_reg_fmt(out: line, cap: sizeof(line), name, v, ret_addr);
212 report_puts(tgt, s: line);
213}
214
215#define CRASH_REG_FIELD_MIN 28
216#define CRASH_REG_FIELD_GAP 2
217
218static void crash_reg_field(struct report_line *l,
219 const struct crash_reg_entry *r, size_t width) {
220 char cell[REPORT_PANE_LINE_MAX];
221 crash_reg_fmt(out: cell, cap: sizeof(cell), name: r->name, v: r->val, ret_addr: r->ret_addr);
222 report_line_field(l, s: cell, width);
223}
224
225static void crash_regs_grid(struct report_target *tgt,
226 const struct crash_reg_entry *r, size_t n) {
227 size_t width = report_target_width(tgt);
228 size_t per_row = 1;
229 size_t cell = width;
230
231 if (width >= 2 * CRASH_REG_FIELD_MIN + CRASH_REG_FIELD_GAP) {
232 per_row = 2;
233 cell = (width - CRASH_REG_FIELD_GAP) / 2;
234 }
235
236 for (size_t i = 0; i < n; i += per_row) {
237 REPORT_LINE(l, width);
238 crash_reg_field(l: &l, r: &r[i], width: cell);
239 if (per_row == 2 && i + 1 < n) {
240 report_line_repeat(l: &l, glyph: " ", CRASH_REG_FIELD_GAP);
241 crash_reg_field(l: &l, r: &r[i + 1], width: cell);
242 }
243 report_line_emit(tgt, l: &l);
244 }
245}
246
247static void crash_rflags_decode(uint64_t f, char *out, size_t cap) {
248 static const struct {
249 uint8_t bit;
250 const char *name;
251 } bits[] = {{0, "CF"}, {2, "PF"}, {4, "AF"}, {6, "ZF"},
252 {7, "SF"}, {8, "TF"}, {9, "IF"}, {10, "DF"},
253 {11, "OF"}, {16, "RF"}, {17, "VM"}, {18, "AC"}};
254
255 size_t n = 0;
256 if (cap < 2)
257 return;
258
259 out[n++] = '[';
260 out[n] = '\0';
261
262 for (size_t i = 0; i < sizeof(bits) / sizeof(*bits); i++) {
263 if (!(f & (1ull << bits[i].bit)))
264 continue;
265 if (n + 5 >= cap)
266 break;
267 n += (size_t) snprintf(buffer: out + n, buffer_len: (int) (cap - n), format: "%s%s",
268 n > 1 ? " " : "", bits[i].name);
269 }
270
271 if (n + 1 < cap)
272 snprintf(buffer: out + n, buffer_len: (int) (cap - n), format: "] iopl=%lu", (f >> 12) & 3);
273}
274
275static void crash_regs_panel(struct report_target *tgt,
276 const struct crash_regs *regs) {
277 struct crash_reg_entry r[CRASH_REG_COUNT];
278 size_t n;
279
280 if (!crash_regs_captured(r: regs)) {
281 report_printf(tgt, fmt: "%s<no registers>%s", term_style(sev: TERM_SEV_WARN),
282 term_style_reset());
283 return;
284 }
285
286 char flags[REPORT_PANE_LINE_MAX];
287 crash_reg_line(tgt, name: "rip", v: regs->rip, true);
288 crash_rflags_decode(f: regs->rflags, out: flags, cap: sizeof(flags));
289 crash_reg_note(tgt, name: "flg", v: regs->rflags, note: flags);
290 crash_reg_line(tgt, name: "cr2", v: regs->cr2, false);
291 crash_reg_note(tgt, name: "cr3", v: regs->cr3, note: "<phys>");
292
293 report_puts(tgt, s: "");
294 n = crash_regs_collect(r: regs, out: r);
295 crash_regs_grid(tgt, r, n);
296}
297
298static void crash_backtrace_panel(struct report_target *tgt,
299 const struct crash_regs *regs) {
300 uint64_t entries[STACK_TRACE_MAX_DEPTH];
301 size_t nr = 0;
302
303 if (crash_regs_captured(r: regs) && regs->rip)
304 entries[nr++] = regs->rip;
305
306 if (global.current_bootstage >= BOOTSTAGE_EARLY_ALLOCATORS) {
307 if (crash_regs_captured(r: regs) && regs->rbp)
308 nr += stack_unwind(frame: regs->rbp, entries: entries + nr,
309 STACK_TRACE_MAX_DEPTH - nr);
310 else
311 nr += stack_unwind(frame: (uint64_t) __builtin_frame_address(0),
312 entries: entries + nr, STACK_TRACE_MAX_DEPTH - nr);
313 }
314
315 if (!debug_syms_present())
316 report_printf(tgt, fmt: "%s<no symbol table: rebuild to symbolize>%s",
317 term_style(sev: TERM_SEV_WARN), term_style_reset());
318
319 if (!nr) {
320 report_printf(tgt, fmt: "%s<no kernel frames found>%s",
321 term_style(sev: TERM_SEV_WARN), term_style_reset());
322 return;
323 }
324
325 for (size_t i = 0; i < nr; i++) {
326 uint64_t off = 0;
327 const char *sym = debug_symbolize(addr: entries[i], out_off: &off);
328 uint32_t line = 0;
329 const char *file;
330 char frame[REPORT_PANE_LINE_MAX];
331 char at[REPORT_PANE_LINE_MAX];
332
333 if (sym)
334 snprintf(buffer: frame, buffer_len: (int) sizeof(frame),
335 format: "%s#%-2zu%s %012lx %s%s+0x%lx%s", term_style(sev: TERM_SEV_DIM),
336 i, term_style_reset(), entries[i],
337 term_style(sev: TERM_SEV_HEAD), sym, off, term_style_reset());
338 else
339 snprintf(buffer: frame, buffer_len: (int) sizeof(frame), format: "%s#%-2zu%s %012lx <unknown>",
340 term_style(sev: TERM_SEV_DIM), i, term_style_reset(),
341 entries[i]);
342
343 file = debug_line_for(addr: entries[i] - 1, out_line: &line);
344 if (!file) {
345 report_puts(tgt, s: frame);
346 continue;
347 }
348
349 snprintf(buffer: at, buffer_len: (int) sizeof(at), format: "%sat %s:%u%s", term_style(sev: TERM_SEV_DIM),
350 file, line, term_style_reset());
351
352 if (report_strwidth(s: frame) + 2 + report_strwidth(s: at) <=
353 report_target_width(tgt)) {
354 report_printf(tgt, fmt: "%s %s", frame, at);
355 } else {
356 report_puts(tgt, s: frame);
357 report_printf(tgt, fmt: " %s", at);
358 }
359 }
360}
361
362#define CRASH_CPU_MIN_WIDTH 52
363
364static uint32_t crash_cpu_panes(void) {
365 uint16_t total = term_size().cols;
366 for (uint32_t n = REPORT_PANES_MAX; n > 1; n--) {
367 if (total >= n * CRASH_CPU_MIN_WIDTH + (n - 1) * REPORT_PANE_GAP)
368 return n;
369 }
370 return 1;
371}
372
373static void crash_cpu_box(struct report_target *tgt, uint64_t id,
374 uint16_t inner) {
375 const struct crash_regs *r = PERCPU_PTR_FOR_CPU(crash_regs, id);
376 _Atomic uint32_t *quiesced = PERCPU_PTR_FOR_CPU(crash_quiesced, id);
377 time_us_t end = time_get_us() + CRASH_WAIT_US;
378 uint64_t entries[6];
379 char line[REPORT_PANE_LINE_MAX];
380 char title[24];
381 struct report_box box;
382 size_t nr;
383
384 snprintf(buffer: title, buffer_len: (int) sizeof(title), format: "cpu %lu", id);
385
386 while (!atomic_load(quiesced) && time_get_us() <= end)
387 sleep_spin_us(CRASH_SPIN_ONE_US);
388
389 report_box_open(b: &box, target: *tgt, title, inner);
390
391 if (!atomic_load(quiesced)) {
392 report_box_printf(b: &box, fmt: "%sno response to the crash NMI%s",
393 term_style(sev: TERM_SEV_WARN), term_style_reset());
394 report_box_close(b: &box);
395 return;
396 }
397
398 crash_reg_fmt(out: line, cap: sizeof(line), name: "rip", v: r->rip, true);
399 report_box_printf(b: &box, fmt: "%s", line);
400 crash_reg_fmt(out: line, cap: sizeof(line), name: "rsp", v: r->rsp, false);
401 report_box_printf(b: &box, fmt: "%s", line);
402 crash_reg_fmt(out: line, cap: sizeof(line), name: "rbp", v: r->rbp, false);
403 report_box_printf(b: &box, fmt: "%s", line);
404
405 nr = r->rbp ? stack_unwind(frame: r->rbp, entries, max: 6) : 0;
406 if (!nr) {
407 report_box_printf(b: &box, fmt: "%s<no frames>%s", term_style(sev: TERM_SEV_DIM),
408 term_style_reset());
409 report_box_close(b: &box);
410 return;
411 }
412
413 for (size_t i = 0; i < nr; i++) {
414 uint64_t off = 0;
415 uint32_t line = 0;
416 const char *sym = debug_symbolize(addr: entries[i], out_off: &off);
417 const char *file = debug_line_for(addr: entries[i] - 1, out_line: &line);
418
419 if (sym && file)
420 report_box_printf(b: &box, fmt: "%s#%-2zu%s %s+0x%lx %sat %s:%u%s",
421 term_style(sev: TERM_SEV_DIM), i, term_style_reset(),
422 sym, off, term_style(sev: TERM_SEV_DIM), file, line,
423 term_style_reset());
424 else if (sym)
425 report_box_printf(b: &box, fmt: "%s#%-2zu%s %s+0x%lx",
426 term_style(sev: TERM_SEV_DIM), i, term_style_reset(),
427 sym, off);
428 else
429 report_box_printf(b: &box, fmt: "%s#%-2zu%s %016lx",
430 term_style(sev: TERM_SEV_DIM), i, term_style_reset(),
431 entries[i]);
432 }
433
434 report_box_close(b: &box);
435}
436
437static void crash_other_cpus(struct report_panes *panes) {
438 struct report_target col0 = report_pane(panes, pane: 0);
439 uint64_t self = smp_id(cond: TOPC_NONE);
440 uint16_t inner = 0;
441 uint32_t count = 0;
442 uint32_t per_col;
443 uint32_t slot = 0;
444 uint64_t id;
445
446 if (global.current_bootstage < BOOTSTAGE_MID_MP) {
447 report_printf(tgt: &col0, fmt: " %s<single core at this bootstage>%s",
448 term_style(sev: TERM_SEV_DIM), term_style_reset());
449 return;
450 }
451
452 if (!PERCPU_READY(crash_regs)) {
453 report_printf(tgt: &col0, fmt: " %s<percpu regs not initialised yet>%s",
454 term_style(sev: TERM_SEV_DIM), term_style_reset());
455 return;
456 }
457
458 for_each_cpu_id(id) {
459 if (id != self)
460 count++;
461 }
462
463 if (!count) {
464 report_printf(tgt: &col0, fmt: " %s<no other cores>%s", term_style(sev: TERM_SEV_DIM),
465 term_style_reset());
466 return;
467 }
468
469 for (uint32_t i = 0; i < panes->n; i++) {
470 struct report_target s = report_pane(panes, pane: i);
471 uint16_t room = s.width > 4 ? (uint16_t) (s.width - 4) : 1;
472 if (!inner || room < inner)
473 inner = room;
474 }
475
476 per_col = (count + panes->n - 1) / panes->n;
477 if (!per_col)
478 per_col = 1;
479
480 for_each_cpu_id(id) {
481 if (id == self)
482 continue;
483
484 uint32_t which = slot++ / per_col;
485 struct report_target target = report_pane(
486 panes, pane: which < panes->n ? which : (uint32_t) panes->n - 1);
487 crash_cpu_box(tgt: &target, id, inner);
488 }
489}
490
491#define CRASH_LOGO_INDENT 2
492
493static size_t crash_logo_indent(const char *) {
494 size_t least = (size_t) -1;
495 for (const char *p = logo; *p;) {
496 size_t indent = 0;
497
498 while (*p == ' ') {
499 indent++;
500 p++;
501 }
502
503 if (*p && *p != '\n' && indent < least)
504 least = indent;
505
506 p = strchrnul(s: p, c: '\n');
507
508 if (*p)
509 p++;
510 }
511 return least == (size_t) -1 ? 0 : least;
512}
513
514static size_t crash_logo_width(const char *, size_t skip) {
515 size_t widest = 0;
516 for (const char *p = logo; *p;) {
517 const char *start;
518
519 for (size_t i = 0; i < skip && *p == ' '; i++)
520 p++;
521
522 start = p;
523 p = strchrnul(s: p, c: '\n');
524
525 if ((size_t) (p - start) > widest)
526 widest = (size_t) (p - start);
527
528 if (*p)
529 p++;
530 }
531 return widest;
532}
533
534static void crash_logo_panel(struct report_target *tgt, const char *) {
535 size_t skip = crash_logo_indent(logo);
536 size_t art = crash_logo_width(logo, skip);
537 size_t width = report_target_width(tgt);
538 size_t pad = art < width ? (width - art) / 2 : 0;
539
540 for (const char *p = logo; *p;) {
541 char row[REPORT_PANE_LINE_MAX];
542 const char *eol;
543 size_t n;
544
545 for (size_t i = 0; i < skip && *p == ' '; i++)
546 p++;
547
548 eol = strchrnul(s: p, c: '\n');
549 n = (size_t) (eol - p);
550 if (n + 1 > sizeof(row))
551 n = sizeof(row) - 1;
552
553 memcpy(row, p, n);
554 p = eol;
555
556 while (n && row[n - 1] == ' ')
557 n--;
558
559 row[n] = '\0';
560 if (*p)
561 p++;
562
563 report_printf(tgt, fmt: "%*s%s%s%s", (int) pad, "", ANSI_RED, row,
564 ANSI_RESET);
565 }
566}
567
568static void crash_where_box(struct report_target *tgt, const char *file,
569 int line, const char *func) {
570 const char *sep = term_unicode() ? " · " : " | ";
571 char loc[REPORT_LINE_MAX];
572 char ctx[REPORT_LINE_MAX];
573 struct report_box box;
574 size_t w;
575
576 snprintf(buffer: loc, buffer_len: (int) sizeof(loc), format: "%s%s:%d%s %s%s()%s", ANSI_GREEN,
577 file ? file : "<unknown>", line, ANSI_RESET, ANSI_CYAN,
578 func ? func : "<unknown>", ANSI_RESET);
579
580 char *thread_name = global.current_bootstage >= BOOTSTAGE_LATE
581 ? thread_get_current()->name
582 : "(null)";
583 if (global.current_bootstage < BOOTSTAGE_EARLY_DEVICES)
584 snprintf(buffer: ctx, buffer_len: (int) sizeof(ctx),
585 format: "cpu %lu%stime unknown%sbootstage '%s'", smp_id(cond: TOPC_NONE),
586 sep, sep, bootstage_str[global.current_bootstage]);
587 else
588 snprintf(buffer: ctx, buffer_len: (int) sizeof(ctx),
589 format: "cpu %lu%s%lu ms%sbootstage '%s'\nthread '%s'",
590 smp_id(cond: TOPC_NONE), sep, time_get_ms(), sep,
591 bootstage_str[global.current_bootstage], thread_name);
592
593 w = report_strwidth(s: loc);
594 if (report_strwidth(s: ctx) > w)
595 w = report_strwidth(s: ctx);
596
597 report_box_open(b: &box, target: *tgt, title: "where", inner: (uint16_t) w);
598 report_box_printf(b: &box, fmt: "%s", loc);
599 report_box_printf(b: &box, fmt: "%s%s%s", term_style(sev: TERM_SEV_DIM), ctx,
600 term_style_reset());
601 report_box_close(b: &box);
602}
603
604static void crash_empty_box_panel(struct report_target *tgt) {
605 for (size_t i = 0; i < panic_scene_count; i++)
606 report_puts(tgt, s: panic_scene[i]);
607}
608
609static struct report_target *active_facility_target = NULL;
610
611void crash_facility_printf(const char *fmt, ...) {
612 va_list ap;
613 va_start(ap, fmt);
614
615 if (active_facility_target) {
616 char buf[REPORT_PANE_LINE_MAX];
617 vsnprintf(buffer: buf, buffer_len: sizeof(buf), format: fmt, args: ap);
618 report_wrap(tgt: active_facility_target, text: buf);
619 } else {
620 vprintf(NULL, format: fmt, args: ap);
621 }
622
623 va_end(ap);
624}
625
626static void crash_emit_ndjson_records(const struct crash_context *ctx,
627 const struct crash_regs *regs,
628 uint32_t depth) {
629 const char *thread = global.current_bootstage >= BOOTSTAGE_LATE
630 ? thread_get_current()->name
631 : NULL;
632
633 ndjson_emit(panic_at, .file = ctx->file ? ctx->file : "",
634 .line = (uint64_t) ctx->line,
635 .func = ctx->func ? ctx->func : "",
636 .msg = ctx->msg ? ctx->msg : "",
637 .bootstage = bootstage_str[global.current_bootstage],
638 .thread = thread, .depth = depth);
639
640 uint64_t entries[STACK_TRACE_MAX_DEPTH];
641 size_t nr = 0;
642
643 if (regs && regs->rip)
644 entries[nr++] = regs->rip;
645
646 if (global.current_bootstage >= BOOTSTAGE_EARLY_ALLOCATORS) {
647 if (regs && regs->rbp)
648 nr += stack_unwind(frame: regs->rbp, entries: entries + nr,
649 STACK_TRACE_MAX_DEPTH - nr);
650 else
651 nr += stack_unwind(frame: (uint64_t) __builtin_frame_address(0),
652 entries: entries + nr, STACK_TRACE_MAX_DEPTH - nr);
653 }
654
655 for (size_t i = 0; i < nr; i++) {
656 uint64_t off = 0;
657 uint32_t srcline = 0;
658 const char *sym = debug_symbolize(addr: entries[i], out_off: &off);
659 const char *srcfile = debug_line_for(addr: entries[i] - 1, out_line: &srcline);
660
661 ndjson_emit(panic_frame, .idx = i, .addr = entries[i], .sym = sym,
662 .off = off, .file = srcfile, .line = srcline);
663 }
664}
665
666static const char *crash_source_title(enum crash_source s) {
667 switch (s) {
668 case CRASH_SOURCE_PANIC: return "kernel panic";
669 case CRASH_SOURCE_ASSERT: return "assertion failure";
670 case CRASH_SOURCE_KASAN: return "kasan fault";
671 case CRASH_SOURCE_UBSAN: return "ubsan violation";
672 case CRASH_SOURCE_NMI_WATCHDOG: return "watchdog lockup";
673 case CRASH_SOURCE_CPU_EXCEPTION: return "cpu exception";
674 case CRASH_SOURCE_NIGHTMARE: return "nightmare finding";
675 case CRASH_SOURCE_LOCK_CHK: return "lock validator violation";
676 default: return "kernel crash";
677 }
678}
679
680static const char *crash_source_name(enum crash_source s) {
681 switch (s) {
682 case CRASH_SOURCE_PANIC: return "PANIC";
683 case CRASH_SOURCE_ASSERT: return "ASSERTION FAILED";
684 case CRASH_SOURCE_KASAN: return "KASAN";
685 case CRASH_SOURCE_UBSAN: return "UBSAN";
686 case CRASH_SOURCE_NMI_WATCHDOG: return "WATCHDOG";
687 case CRASH_SOURCE_CPU_EXCEPTION: return "CPU EXCEPTION";
688 case CRASH_SOURCE_NIGHTMARE: return "NIGHTMARE";
689 case CRASH_SOURCE_LOCK_CHK: return "LOCK_CHK";
690 default: return "CRASH";
691 }
692}
693
694static void crash_report_visual(const struct crash_context *ctx,
695 const struct crash_regs *regs) {
696 struct report_target con = report_console();
697 struct report_panes *panes = report_panes_panic();
698
699 printf(format: "\033[H\033[2J");
700
701 report_panes_begin(panes, n: 2, weights: (const uint8_t[]){42, 58});
702 const char *title = crash_source_title(s: ctx->source);
703 report_panes_title(panes, pane: 0, sev: TERM_SEV_CRIT, title);
704 report_panes_title(panes, pane: 1, sev: TERM_SEV_LABEL, title: "registers");
705 report_panes_top(panes);
706
707 struct report_target left = report_pane(panes, pane: 0);
708 struct report_target right = report_pane(panes, pane: 1);
709
710 enum crash_code code = ctx->payload.code;
711 struct crash_facility *facility = crash_facility_for(code);
712 crash_logo_panel(tgt: &left, logo: OS_LOGO_PANIC_CENTERED);
713 report_blank(tgt: &left);
714
715 crash_where_box(tgt: &left, file: ctx->file, line: ctx->line, func: ctx->func);
716
717 if (report_section_begin_at(tgt: &left, name: "message")) {
718 report_wrap_printf(tgt: &left, fmt: "%s%s%s", term_style(sev: TERM_SEV_HEAD),
719 ctx->msg ? ctx->msg : "<no message>",
720 term_style_reset());
721 }
722 report_section_end();
723
724 if (report_section_claim_at(tgt: &right, name: "registers"))
725 crash_regs_panel(tgt: &right, regs);
726 report_section_end();
727
728 if (report_section_begin_at(tgt: &right, name: "backtrace"))
729 crash_backtrace_panel(tgt: &right, regs);
730 report_section_end();
731
732 char name_top[128];
733 if (facility) {
734 uint16_t delta = CRASH_CODE_GET_DELTA(code);
735 snprintf(buffer: name_top, buffer_len: sizeof(name_top),
736 format: "\"%s\" " ANSI_RED "crashed" ANSI_BRIGHT_BLUE
737 " with code " ANSI_BOLD ANSI_WHITE "0x%x" ANSI_RESET
738 " - " ANSI_CYAN "\"%s\"" ANSI_RESET,
739 facility->name ? facility->name : "unknown", code,
740 facility->to_str ? facility->to_str(delta) : "unknown");
741 } else if (code != 0) {
742 char *to_str = (char *) crash_code_to_str(code);
743 if (!to_str)
744 to_str = "unknown";
745
746 snprintf(buffer: name_top, buffer_len: sizeof(name_top),
747 ANSI_RED "crashed" ANSI_BRIGHT_BLUE
748 " with code " ANSI_BOLD ANSI_WHITE "0x%x" ANSI_RESET
749 " - " ANSI_CYAN "\"%s\"" ANSI_RESET,
750 code, to_str);
751 } else {
752 strcpy(dest: name_top, src: "empty box");
753 }
754
755 if (report_section_begin_at(tgt: &right, name: name_top)) {
756 if (facility && facility->dump) {
757 active_facility_target = &right;
758 facility->dump(CRASH_CODE_GET_DELTA(code), ctx->payload);
759 active_facility_target = NULL;
760 } else {
761 crash_empty_box_panel(tgt: &right);
762 }
763 }
764
765 report_section_end();
766
767 report_panes_flush(panes);
768
769 if (ctx->formats & CRASH_FMT_PEER_CPUS) {
770 report_panes_carry(panes);
771 report_panes_begin(panes, n: crash_cpu_panes(), NULL);
772 report_panes_undivided(panes);
773 report_panes_title(panes, pane: 0, sev: TERM_SEV_LABEL, title: "other CPUs");
774 report_panes_top(panes);
775
776 struct report_target cpu0 = report_pane(panes, pane: 0);
777 if (report_section_claim_at(tgt: &cpu0, name: "other CPUs"))
778 crash_other_cpus(panes);
779 report_section_end();
780 report_panes_flush(panes);
781 }
782
783 if (ctx->formats & CRASH_FMT_DUMP_LOGS) {
784 report_panes_bottom(panes, sev: TERM_SEV_LABEL, title: "logs");
785 if (report_section_claim(name: "logs"))
786 log_dump_panic();
787 report_section_end();
788 }
789
790 report_blank(tgt: &con);
791}
792
793static void crash_report_raw_serial(const struct crash_context *ctx,
794 const struct crash_regs *regs) {
795 printf_unlocked(format: "\n*** KERNEL CRASH: %s ***\n",
796 crash_source_name(s: ctx->source));
797 if (ctx->file || ctx->func)
798 printf_unlocked(format: "Location: %s:%d in %s()\n",
799 ctx->file ? ctx->file : "<unknown>", ctx->line,
800 ctx->func ? ctx->func : "<unknown>");
801 if (ctx->msg)
802 printf_unlocked(format: "Message : %s\n", ctx->msg);
803 if (regs) {
804 printf_unlocked(
805 format: "RIP: %016lx RSP: %016lx RFLAGS: %016lx CR2: %016lx\n",
806 regs->rip, regs->rsp, regs->rflags, regs->cr2);
807 printf_unlocked(format: "RAX: %016lx RBX: %016lx RCX: %016lx RDX: %016lx\n",
808 regs->rax, regs->rbx, regs->rcx, regs->rdx);
809 }
810}
811
812__noreturn void crash_full(const struct crash_context *ctx) {
813 disable_interrupts();
814
815 uint32_t depth =
816 atomic_fetch_add_explicit(&crash_depth, 1, memory_order_relaxed);
817
818 if (depth >= CRASH_MAX_DEPTH) {
819 printf_unlocked(format: "\n[crash depth %u, aborting report]\n", depth);
820#if defined(TEST_ENABLED) || defined(TEST_NIGHTMARE_ENABLED)
821 ndjson_bye(code: QEMU_EXIT_PANIC, reason: "nested_crash");
822 qemu_exit(code: QEMU_EXIT_PANIC);
823#endif
824 while (true)
825 wait_for_interrupt();
826 }
827
828 if (depth == 0)
829 raw_spin_lock(lock: &crash_lock);
830
831 int64_t unowned = -1;
832 atomic_compare_exchange_strong(&crash_owner, &unowned,
833 (int64_t) smp_id(TOPC_NONE));
834 atomic_store(&global.panicked, true);
835
836 struct crash_regs captured_regs;
837 if (ctx->regs) {
838 captured_regs = *ctx->regs;
839 } else {
840 crash_capture_regs(out: &captured_regs);
841 }
842
843 if (PERCPU_READY(crash_regs)) {
844 PERCPU_READ(TOPC_NONE, crash_regs) = captured_regs;
845 } else {
846 boot_crash_regs = captured_regs;
847 }
848
849 if (global.current_bootstage >= BOOTSTAGE_MID_MP) {
850 crash_broadcast_nmi();
851 sleep_spin_ms(msec: 500);
852 }
853
854 report_enter_panic();
855 ndjson_enter_panic();
856
857 uint32_t formats = ctx->formats;
858 if (global.current_bootstage < BOOTSTAGE_EARLY_DEVICES) {
859 formats &= ~CRASH_FMT_VISUAL_PANES;
860 formats |= CRASH_FMT_RAW_SERIAL;
861 }
862
863 if (formats & CRASH_FMT_NDJSON)
864 crash_emit_ndjson_records(ctx, regs: &captured_regs, depth);
865
866 if (formats & CRASH_FMT_VISUAL_PANES)
867 crash_report_visual(ctx, regs: &captured_regs);
868 else if (formats & CRASH_FMT_RAW_SERIAL)
869 crash_report_raw_serial(ctx, regs: &captured_regs);
870
871 if ((formats & CRASH_FMT_DUMP_LOGS) && !(formats & CRASH_FMT_VISUAL_PANES))
872 log_dump_panic();
873
874 if (depth == 0)
875 raw_spin_unlock(lock: &crash_lock);
876
877#if defined(TEST_ENABLED) || defined(TEST_NIGHTMARE_ENABLED)
878 ndjson_bye(code: QEMU_EXIT_PANIC, reason: "crash");
879 qemu_exit(code: QEMU_EXIT_PANIC);
880#endif
881
882 while (true)
883 wait_for_interrupt();
884}
885
886static int cmp_facility_prefix(const void *key, const void *elem) {
887 uint16_t pref = *(const uint16_t *) key;
888 const struct crash_facility *f = elem;
889 return (pref > f->prefix) - (pref < f->prefix);
890}
891
892/* bsearch __skernel_crash_facilities to __ekernel_crash_facilities */
893static struct crash_facility *facility_for(uint16_t pref) {
894 if (!pref)
895 return NULL;
896
897 size_t count = __ekernel_crash_facilities - __skernel_crash_facilities;
898 return bsearch(key: &pref, base: __skernel_crash_facilities, nmemb: count,
899 size: sizeof(struct crash_facility), compar: cmp_facility_prefix);
900}
901
902static struct crash_facility *crash_facility_for(enum crash_code code) {
903 return facility_for(CRASH_CODE_GET_FACILITY(code));
904}
905
906const char *crash_code_from_facility_to_str(enum crash_code code) {
907 uint16_t pref = CRASH_CODE_GET_FACILITY(code);
908 uint16_t del = CRASH_CODE_GET_DELTA(code);
909 kassert(pref && del);
910 struct crash_facility *this = kassert(facility_for(pref));
911 if (this->to_str)
912 return this->to_str(del);
913
914 return NULL;
915}
916
917void crash_facilities_init(void) {
918 kassert(__ekernel_crash_facilities - __skernel_crash_facilities <=
919 UINT16_MAX,
920 "too many?");
921
922 /* Simple: 1 + index in array, keeps it sorted, avoids 0 */
923 for (struct crash_facility *f = __skernel_crash_facilities;
924 f < __ekernel_crash_facilities; f++)
925 f->prefix = (f - __skernel_crash_facilities) + 1;
926}
927
928void crash_perthread_init(struct thread *t) {
929 struct crash_perthread *pt = &t->crash_data;
930 INIT_LIST_HEAD(list: &pt->crash_hooks);
931 pt->in_hook = false;
932 INIT_LIST_HEAD(list: &pt->unwind.free_list);
933 INIT_LIST_HEAD(list: &pt->unwind.in_use);
934 for (int i = 0; i < CRASH_UNWIND_NODES; i++)
935 list_add_tail(new: &pt->unwind.nodes[i].list, head: &pt->unwind.free_list);
936}
937
938static struct crash_unwind_node *
939unwind_node_alloc(struct thread *t, enum crash_unwind_type type) {
940 struct list_head *n = list_pop_tail(head: &t->crash_data.unwind.free_list);
941 if (!n)
942 panic("ran out of nodes");
943
944 struct crash_unwind_node *node =
945 container_of(n, struct crash_unwind_node, list);
946 kassert(node->type == CRASH_UNWIND_NONE);
947 node->type = type;
948 return node;
949}
950
951static void unwind_node_free(struct thread *t, struct crash_unwind_node *n) {
952 n->type = CRASH_UNWIND_NONE;
953 n->data.raw = 0;
954 list_add_tail(new: &n->list, head: &t->crash_data.unwind.free_list);
955}
956
957/* Small optimization here: it doesn't matter if we go backwards or
958 * forwards, simply that we get to the node, but it's more likely
959 * to be found faster if we iterate in reverse */
960static void crash_unwind_node_add(struct crash_unwind_node_data *data,
961 enum crash_unwind_type type) {
962 struct thread *t = thread_get_current();
963 kassert(!t->crash_data.unwinding);
964 if (type == CRASH_UNWIND_RCU) {
965 struct crash_unwind_node *n;
966 list_for_each_entry_rev(n, &t->crash_data.unwind.in_use, list) {
967 if (n->type == type) {
968 n->data.rcu_lock_times++;
969 return;
970 }
971 }
972
973 n = unwind_node_alloc(t, type);
974 n->data.rcu_lock_times = 1;
975 list_add_tail(new: &n->list, head: &t->crash_data.unwind.in_use);
976 return;
977 }
978
979 struct crash_unwind_node *node = unwind_node_alloc(t, type);
980 node->data = *data;
981 list_add_tail(new: &node->list, head: &t->crash_data.unwind.in_use);
982}
983
984static void crash_unwind_node_remove(struct crash_unwind_node_data *data,
985 enum crash_unwind_type type) {
986 struct thread *t = thread_get_current();
987 kassert(!t->crash_data.unwinding);
988 bool found = false;
989
990 if (type == CRASH_UNWIND_RCU) {
991 struct crash_unwind_node *n;
992 list_for_each_entry_rev(n, &t->crash_data.unwind.in_use, list) {
993 if (n->type == type) {
994 if (n->data.rcu_lock_times == 1) {
995 list_del(entry: &n->list);
996 unwind_node_free(t, n);
997 } else {
998 n->data.rcu_lock_times--;
999 }
1000 return;
1001 }
1002 }
1003
1004 goto out;
1005 }
1006
1007 struct crash_unwind_node *iter = NULL;
1008
1009 list_for_each_entry_rev(iter, &t->crash_data.unwind.in_use, list) {
1010 if (iter->data.raw == data->raw && iter->type == type) {
1011 list_del(entry: &iter->list);
1012 unwind_node_free(t, n: iter);
1013 found = true;
1014 break;
1015 }
1016 }
1017
1018out:
1019 kassert(found, "Likely double remove");
1020}
1021
1022static void unwind_rcu(struct crash_unwind_node_data *d) {
1023 for (uintptr_t i = 0; i < d->rcu_lock_times; i++)
1024 rcu_read_unlock();
1025}
1026
1027static void unwind_mutex(struct crash_unwind_node_data *d) {
1028 mutex_unlock(d->ptr);
1029}
1030
1031static void unwind_rwlock(struct crash_unwind_node_data *d) {
1032 rw_unlock(d->ptr);
1033}
1034
1035static void unwind_spinlock(struct crash_unwind_node_data *d) {
1036 spin_unlock(d->ptr, d->arg);
1037}
1038
1039static void unwind_qspinlock(struct crash_unwind_node_data *d) {
1040 qspin_unlock(d->ptr, d->arg);
1041}
1042
1043static void (*unwind_cbs[CRASH_UNWIND_MAX])(struct crash_unwind_node_data *) = {
1044 [CRASH_UNWIND_RCU] = unwind_rcu,
1045 [CRASH_UNWIND_MUTEX] = unwind_mutex,
1046 [CRASH_UNWIND_RWLOCK] = unwind_rwlock,
1047 [CRASH_UNWIND_SPINLOCK] = unwind_spinlock,
1048 [CRASH_UNWIND_QSPINLOCK] = unwind_qspinlock,
1049};
1050
1051static inline const char *
1052crash_unwind_type_to_str(enum crash_unwind_type type) {
1053 switch (type) {
1054 case CRASH_UNWIND_RCU: return "RCU";
1055 case CRASH_UNWIND_MUTEX: return "MUTEX";
1056 case CRASH_UNWIND_RWLOCK: return "RWLOCK";
1057 case CRASH_UNWIND_SPINLOCK: return "SPINLOCK";
1058 case CRASH_UNWIND_QSPINLOCK: return "QSPINLOCK";
1059 default: unreachable("Invalid %u", type);
1060 }
1061}
1062
1063/* The idea here: we first traverse backwards and unwind
1064 * one by one, detaching as we go */
1065void crash_unwind(void) {
1066 struct thread *t = thread_get_current();
1067 struct crash_perthread *pt = &t->crash_data;
1068 pt->unwinding = true;
1069
1070 struct crash_unwind_perthread *upt = &pt->unwind;
1071 struct crash_unwind_node *cun, *tmp;
1072 list_for_each_entry_safe_rev(cun, tmp, &upt->in_use, list) {
1073 kassert(cun->type != CRASH_UNWIND_NONE);
1074 thread_warn("unwinding %s (%p)", crash_unwind_type_to_str(cun->type),
1075 cun->data.ptr);
1076 unwind_cbs[cun->type](&cun->data);
1077 list_del(entry: &cun->list);
1078 unwind_node_free(t, n: cun);
1079 }
1080
1081 irql_lower(old_level: IRQL_PASSIVE_LEVEL);
1082
1083 pt->unwinding = false;
1084}
1085
1086static void crash_unwind_enter(uintptr_t data, enum crash_unwind_type type,
1087 uintptr_t arg) {
1088 if (global.current_bootstage >= BOOTSTAGE_LATE &&
1089 !thread_get_current()->crash_data.unwinding) {
1090 struct crash_unwind_node_data nd = {
1091 .raw = data,
1092 .arg = arg,
1093 };
1094
1095 crash_unwind_node_add(data: &nd, type);
1096 }
1097}
1098
1099static void crash_unwind_exit(uintptr_t data, enum crash_unwind_type type) {
1100 if (global.current_bootstage >= BOOTSTAGE_LATE &&
1101 !thread_get_current()->crash_data.unwinding) {
1102 struct crash_unwind_node_data nd = {
1103 .raw = data,
1104 };
1105
1106 crash_unwind_node_remove(data: &nd, type);
1107 }
1108}
1109
1110void crash_unwind_enter_rcu(void) {
1111 crash_unwind_enter(data: 0, type: CRASH_UNWIND_RCU, arg: 0);
1112}
1113
1114void crash_unwind_exit_rcu(void) {
1115 crash_unwind_exit(data: 0, type: CRASH_UNWIND_RCU);
1116}
1117
1118void crash_unwind_enter_mutex(struct mutex *m) {
1119 crash_unwind_enter(data: (uintptr_t) m, type: CRASH_UNWIND_MUTEX, arg: 0);
1120}
1121
1122void crash_unwind_exit_mutex(struct mutex *m) {
1123 crash_unwind_exit(data: (uintptr_t) m, type: CRASH_UNWIND_MUTEX);
1124}
1125
1126void crash_unwind_enter_rwlock(struct rwlock *r) {
1127 crash_unwind_enter(data: (uintptr_t) r, type: CRASH_UNWIND_RWLOCK, arg: 0);
1128}
1129
1130void crash_unwind_exit_rwlock(struct rwlock *r) {
1131 crash_unwind_exit(data: (uintptr_t) r, type: CRASH_UNWIND_RWLOCK);
1132}
1133
1134void crash_unwind_enter_spinlock(struct spinlock *s, enum irql old) {
1135 crash_unwind_enter(data: (uintptr_t) s, type: CRASH_UNWIND_SPINLOCK, arg: (uintptr_t) old);
1136}
1137
1138void crash_unwind_exit_spinlock(struct spinlock *s) {
1139 crash_unwind_exit(data: (uintptr_t) s, type: CRASH_UNWIND_SPINLOCK);
1140}
1141
1142void crash_unwind_enter_qspinlock(struct qspinlock *q, enum irql old) {
1143 crash_unwind_enter(data: (uintptr_t) q, type: CRASH_UNWIND_QSPINLOCK, arg: (uintptr_t) old);
1144}
1145
1146void crash_unwind_exit_qspinlock(struct qspinlock *q) {
1147 crash_unwind_exit(data: (uintptr_t) q, type: CRASH_UNWIND_QSPINLOCK);
1148}
1149