1/* @title: Kernel Crash Engine */
2#include <asm.h>
3#include <bootstage.h>
4#include <compiler/core.h>
5#include <compiler/intrinsic.h>
6#include <console/crash.h>
7#include <console/panic_scene.h>
8#include <console/printf.h>
9#include <console/report.h>
10#include <console/statusbar.h>
11#include <console/term.h>
12#include <dbg.h>
13#include <global.h>
14#include <irq/ipi.h>
15#include <irq/irq.h>
16#include <linker/symbols.h>
17#include <log.h>
18#include <logo.h>
19#include <math/align.h>
20#include <math/bit.h>
21#include <math/sort.h>
22#include <ndjson.h>
23#include <smp/core.h>
24#include <smp/percpu.h>
25#include <stdarg.h>
26#include <string.h>
27#include <stringify.h>
28#include <sync/mutex.h>
29#include <sync/qspinlock.h>
30#include <sync/raw_spinlock.h>
31#include <sync/rwlock.h>
32#include <sync/spinlock.h>
33#include <thread/thread.h>
34#include <time/spin_sleep.h>
35#include <time/time.h>
36
37static atomic_int64_t crash_owner = -1;
38static atomic_uint32_t crash_depth = 0;
39static struct raw_spinlock crash_lock = RAW_SPINLOCK_INIT;
40
41PERCPU_DEFINE(atomic_uint32_t, crash_quiesced);
42PERCPU_DEFINE(struct crash_regs, crash_regs);
43static cc_unused struct crash_regs boot_crash_regs = {0};
44
45NDJSON_DEFINE(panic_at, NDJSON_SECTION_PANIC, NDJSON_KIND_AT, 1,
46 NDJSON_STR(file), NDJSON_U64(line), NDJSON_STR(func),
47 NDJSON_STR(msg), NDJSON_STR(bootstage), NDJSON_STR(thread),
48 NDJSON_U64(depth));
49
50NDJSON_DEFINE(panic_frame, NDJSON_SECTION_PANIC, NDJSON_KIND_FRAME, 1,
51 NDJSON_U64(idx), NDJSON_HEX(addr), NDJSON_STR(sym),
52 NDJSON_U64(off), NDJSON_STR(file), NDJSON_U64(line));
53
54extern void crash_capture_regs(struct crash_regs *out);
55
56bool crash_cpu_is_owner(uint64_t id) {
57 return atomic_load_rlx(&crash_owner) == (int64_t) id;
58}
59
60void crash_nmi_handoff(void *p, struct irq_registers *irqc) {
61 cc_ignore(p);
62 /* _NONE here for safety reasons (the validator could
63 * crash again depending on why we crashed),
64 * it's the crash context anyways */
65 struct crash_regs *this_regs_buf = PERCPU_PTR(TOPC_NONE, crash_regs);
66 this_regs_buf->r8 = irqc->r8;
67 this_regs_buf->r9 = irqc->r9;
68 this_regs_buf->r10 = irqc->r10;
69 this_regs_buf->r11 = irqc->r11;
70 this_regs_buf->r12 = irqc->r12;
71 this_regs_buf->r13 = irqc->r13;
72 this_regs_buf->r14 = irqc->r14;
73 this_regs_buf->r15 = irqc->r15;
74 this_regs_buf->rdi = irqc->rdi;
75 this_regs_buf->rsi = irqc->rsi;
76 this_regs_buf->rbp = irqc->rbp;
77 this_regs_buf->rsp = irqc->rsp;
78 this_regs_buf->rax = irqc->rax;
79 this_regs_buf->rbx = irqc->rbx;
80 this_regs_buf->rcx = irqc->rcx;
81 this_regs_buf->rdx = irqc->rdx;
82 this_regs_buf->rip = irqc->rip;
83 this_regs_buf->rflags = irqc->rflags;
84 this_regs_buf->cr2 = cr2_read();
85 this_regs_buf->cr3 = cr3_read();
86 atomic_store(PERCPU_PTR(TOPC_NONE, crash_quiesced), 1);
87 while (true)
88 cpu_freeze();
89}
90
91void crash_broadcast_nmi(void) {
92 nmi_send_others(self: smp_id(cond: TOPC_NONE));
93}
94
95/* called from panic.asm, returns with interrupts disabled */
96void panic_handler(struct crash_regs *regs) TSA_ACQUIRES_IRQS;
97void panic_handler(struct crash_regs *regs) TSA_ACQUIRES_IRQS {
98 irq_disable();
99
100 if (PERCPU_READY(crash_regs)) {
101 PERCPU_READ(TOPC_NONE, crash_regs) = *regs;
102 } else {
103 boot_crash_regs = *regs;
104 }
105
106 if (global.current_bootstage >= BOOTSTAGE_MID_MP) {
107 crash_broadcast_nmi();
108 sleep_spin_ms(msec: 50);
109 }
110}
111
112static bool crash_regs_captured(const struct crash_regs *r) {
113 for (int i = 0; i < CRASH_REG_COUNT; i++) {
114 if (r->regs[i])
115 return true;
116 }
117 return false;
118}
119
120struct crash_reg_entry {
121 const char *name;
122 uint64_t val;
123 bool ret_addr;
124};
125
126static size_t crash_regs_collect(const struct crash_regs *r,
127 struct crash_reg_entry *out) {
128 size_t n = 0;
129 out[n++] = (struct crash_reg_entry){"rax", r->rax, false};
130 out[n++] = (struct crash_reg_entry){"rbx", r->rbx, false};
131 out[n++] = (struct crash_reg_entry){"rcx", r->rcx, false};
132 out[n++] = (struct crash_reg_entry){"rdx", r->rdx, false};
133 out[n++] = (struct crash_reg_entry){"rsi", r->rsi, false};
134 out[n++] = (struct crash_reg_entry){"rdi", r->rdi, false};
135 out[n++] = (struct crash_reg_entry){"rbp", r->rbp, false};
136 out[n++] = (struct crash_reg_entry){"rsp", r->rsp, false};
137 out[n++] = (struct crash_reg_entry){" r8", r->r8, false};
138 out[n++] = (struct crash_reg_entry){" r9", r->r9, false};
139 out[n++] = (struct crash_reg_entry){"r10", r->r10, false};
140 out[n++] = (struct crash_reg_entry){"r11", r->r11, false};
141 out[n++] = (struct crash_reg_entry){"r12", r->r12, false};
142 out[n++] = (struct crash_reg_entry){"r13", r->r13, false};
143 out[n++] = (struct crash_reg_entry){"r14", r->r14, false};
144 out[n++] = (struct crash_reg_entry){"r15", r->r15, false};
145 return n;
146}
147
148struct crash_addr_info {
149 const char *sym;
150 uint64_t offset;
151 const char *file;
152 uint32_t line;
153};
154
155static struct crash_addr_info crash_resolve_addr(uint64_t addr,
156 bool is_ret_addr) {
157 struct crash_addr_info info = {0};
158 uint64_t lookup_addr = (is_ret_addr && addr) ? addr - 1 : addr;
159
160 if (addr >= (uint64_t) &__stext && addr < (uint64_t) &__etext) {
161 info.sym = debug_symbolize(addr: lookup_addr, out_off: &info.offset);
162 info.file = debug_line_for(addr: lookup_addr, out_line: &info.line);
163 }
164 return info;
165}
166
167static void crash_describe_addr(uint64_t v, char *out, size_t cap) {
168 if (v >= (uint64_t) &__stext && v < (uint64_t) &__etext) {
169 struct crash_addr_info info = crash_resolve_addr(addr: v, false);
170 if (info.sym && info.file)
171 snprintf(out, (int) cap, "%s+0x%lx at %s:%u", info.sym, info.offset,
172 info.file, info.line);
173 else if (info.sym)
174 snprintf(out, (int) cap, "%s+0x%lx", info.sym, info.offset);
175 else if (info.file)
176 snprintf(out, (int) cap, "<text> at %s:%u", info.file, info.line);
177 else
178 snprintf(out, (int) cap, "<text>");
179 return;
180 }
181
182 if (v >= (uint64_t) &__srodata && v < (uint64_t) &__erodata)
183 snprintf(out, (int) cap, "<rodata>");
184 else if (v >= (uint64_t) &__sdata && v < (uint64_t) &__edata)
185 snprintf(out, (int) cap, "<data>");
186 else if (v >= (uint64_t) &__sbss && v < (uint64_t) &__ebss)
187 snprintf(out, (int) cap, "<bss>");
188 else
189 snprintf(out, (int) cap, "<none>");
190}
191
192static void crash_reg_str(char *out, size_t cap, const char *name, uint64_t v,
193 const char *note) {
194 snprintf(out, (int) cap, "%s%s%s %016lx %s%s%s",
195 term_style(TERM_SEV_LABEL), name, term_style_reset(), v,
196 term_style(TERM_SEV_DIM), note, term_style_reset());
197}
198
199static void crash_reg_fmt(char *out, size_t cap, const char *name, uint64_t v,
200 bool ret_addr) {
201 char note[REPORT_PANE_LINE_MAX];
202 crash_describe_addr(v: ret_addr && v ? v - 1 : v, out: note, cap: sizeof(note));
203 crash_reg_str(out, cap, name, v, note);
204}
205
206static void crash_reg_note(struct report_target *tgt, const char *name,
207 uint64_t v, const char *note) {
208 char line[REPORT_PANE_LINE_MAX];
209 crash_reg_str(out: line, cap: sizeof(line), name, v, note);
210 report_puts(tgt, s: line);
211}
212
213static void crash_reg_line(struct report_target *tgt, const char *name,
214 uint64_t v, bool ret_addr) {
215 char line[REPORT_PANE_LINE_MAX];
216 crash_reg_fmt(out: line, cap: sizeof(line), name, v, ret_addr);
217 report_puts(tgt, s: line);
218}
219
220#define CRASH_REG_FIELD_MIN 28
221#define CRASH_REG_FIELD_GAP 2
222
223static void crash_reg_field(struct report_line *l,
224 const struct crash_reg_entry *r, size_t width) {
225 char cell[REPORT_PANE_LINE_MAX];
226 crash_reg_fmt(out: cell, cap: sizeof(cell), name: r->name, v: r->val, ret_addr: r->ret_addr);
227 report_line_field(l, s: cell, width);
228}
229
230static void crash_regs_grid(struct report_target *tgt,
231 const struct crash_reg_entry *r, size_t n) {
232 size_t width = report_target_width(tgt);
233 size_t per_row = 1;
234 size_t cell = width;
235
236 if (width >= 2 * CRASH_REG_FIELD_MIN + CRASH_REG_FIELD_GAP) {
237 per_row = 2;
238 cell = (width - CRASH_REG_FIELD_GAP) / 2;
239 }
240
241 for (size_t i = 0; i < n; i += per_row) {
242 REPORT_LINE(l, width);
243 crash_reg_field(l: &l, r: &r[i], width: cell);
244 if (per_row == 2 && i + 1 < n) {
245 report_line_repeat(l: &l, glyph: " ", CRASH_REG_FIELD_GAP);
246 crash_reg_field(l: &l, r: &r[i + 1], width: cell);
247 }
248 report_line_emit(tgt, l: &l);
249 }
250}
251
252static void crash_rflags_decode(uint64_t f, char *out, size_t cap) {
253 static const struct {
254 uint8_t bit;
255 const char *name;
256 } bits[] = {{0, "CF"}, {2, "PF"}, {4, "AF"}, {6, "ZF"},
257 {7, "SF"}, {8, "TF"}, {9, "IF"}, {10, "DF"},
258 {11, "OF"}, {16, "RF"}, {17, "VM"}, {18, "AC"}};
259
260 size_t n = 0;
261 if (cap < 2)
262 return;
263
264 out[n++] = '[';
265 out[n] = '\0';
266
267 for (size_t i = 0; i < sizeof(bits) / sizeof(*bits); i++) {
268 if (!BIT_TEST(f, bits[i].bit))
269 continue;
270 if (n + 5 >= cap)
271 break;
272 n += (size_t) snprintf(out + n, (int) (cap - n), "%s%s",
273 n > 1 ? " " : "", bits[i].name);
274 }
275
276 if (n + 1 < cap)
277 snprintf(out + n, (int) (cap - n), "] iopl=%lu", (f >> 12) & 3);
278}
279
280static void crash_regs_panel(struct report_target *tgt,
281 const struct crash_regs *regs) {
282 struct crash_reg_entry r[CRASH_REG_COUNT];
283 size_t n;
284
285 if (!crash_regs_captured(r: regs)) {
286 report_printf(tgt, fmt: "%s<no registers>%s", term_style(sev: TERM_SEV_WARN),
287 term_style_reset());
288 return;
289 }
290
291 char flags[REPORT_PANE_LINE_MAX];
292 crash_reg_line(tgt, name: "rip", v: regs->rip, true);
293 crash_rflags_decode(f: regs->rflags, out: flags, cap: sizeof(flags));
294 crash_reg_note(tgt, name: "flg", v: regs->rflags, note: flags);
295 crash_reg_line(tgt, name: "cr2", v: regs->cr2, false);
296 crash_reg_note(tgt, name: "cr3", v: regs->cr3, note: "<phys>");
297
298 report_puts(tgt, s: "");
299 n = crash_regs_collect(r: regs, out: r);
300 crash_regs_grid(tgt, r, n);
301}
302
303static void crash_backtrace_panel(struct report_target *tgt,
304 const struct crash_regs *regs) {
305 uint64_t entries[STACK_TRACE_MAX_DEPTH];
306 size_t nr = 0;
307
308 if (crash_regs_captured(r: regs) && regs->rip)
309 entries[nr++] = regs->rip;
310
311 if (global.current_bootstage >= BOOTSTAGE_EARLY_ALLOCATORS) {
312 if (crash_regs_captured(r: regs) && regs->rbp)
313 nr += stack_unwind(frame: regs->rbp, entries: entries + nr,
314 STACK_TRACE_MAX_DEPTH - nr);
315 else
316 nr += stack_unwind(frame: (uint64_t) ci_frame_address(0), entries: entries + nr,
317 STACK_TRACE_MAX_DEPTH - nr);
318 }
319
320 if (!debug_syms_present())
321 report_printf(tgt, fmt: "%s<no symbol table: rebuild to symbolize>%s",
322 term_style(sev: TERM_SEV_WARN), term_style_reset());
323
324 if (!nr) {
325 report_printf(tgt, fmt: "%s<no kernel frames found>%s",
326 term_style(sev: TERM_SEV_WARN), term_style_reset());
327 return;
328 }
329
330 for (size_t i = 0; i < nr; i++) {
331 uint64_t off = 0;
332 const char *sym = debug_symbolize(addr: entries[i], out_off: &off);
333 uint32_t line = 0;
334 const char *file;
335 char frame[REPORT_PANE_LINE_MAX];
336 char at[REPORT_PANE_LINE_MAX];
337
338 if (sym)
339 snprintf(frame, (int) sizeof(frame),
340 "%s#%-2zu%s %012lx %s%s+0x%lx%s", term_style(TERM_SEV_DIM),
341 i, term_style_reset(), entries[i],
342 term_style(TERM_SEV_HEAD), sym, off, term_style_reset());
343 else
344 snprintf(frame, (int) sizeof(frame), "%s#%-2zu%s %012lx <unknown>",
345 term_style(TERM_SEV_DIM), i, term_style_reset(),
346 entries[i]);
347
348 file = debug_line_for(addr: entries[i] - 1, out_line: &line);
349 if (!file) {
350 report_puts(tgt, s: frame);
351 continue;
352 }
353
354 snprintf(at, (int) sizeof(at), "%sat %s:%u%s", term_style(TERM_SEV_DIM),
355 file, line, term_style_reset());
356
357 if (report_strwidth(s: frame) + 2 + report_strwidth(s: at) <=
358 report_target_width(tgt)) {
359 report_printf(tgt, fmt: "%s %s", frame, at);
360 } else {
361 report_puts(tgt, s: frame);
362 report_printf(tgt, fmt: " %s", at);
363 }
364 }
365}
366
367#define CRASH_CPU_MIN_WIDTH 52
368
369static uint32_t crash_cpu_panes(void) {
370 uint16_t total = term_size().cols;
371 for (uint32_t n = REPORT_PANES_MAX; n > 1; n--) {
372 if (total >= n * CRASH_CPU_MIN_WIDTH + (n - 1) * REPORT_PANE_GAP)
373 return n;
374 }
375 return 1;
376}
377
378static void crash_cpu_box(struct report_target *tgt, uint64_t id,
379 uint16_t inner) {
380 const struct crash_regs *r = PERCPU_PTR_FOR(crash_regs, id);
381 atomic_uint32_t *quiesced = PERCPU_PTR_FOR(crash_quiesced, id);
382 time_us_t end = time_get_us() + CRASH_WAIT_US;
383 uint64_t entries[6];
384 char line[REPORT_PANE_LINE_MAX];
385 char title[24];
386 struct report_box box;
387 size_t nr;
388
389 snprintf(title, (int) sizeof(title), "cpu %lu", id);
390
391 while (!atomic_load(quiesced) && time_get_us() <= end)
392 sleep_spin_us(CRASH_SPIN_ONE_US);
393
394 report_box_open(b: &box, target: *tgt, title, inner);
395
396 if (!atomic_load(quiesced)) {
397 report_box_printf(b: &box, fmt: "%sno response to the crash NMI%s",
398 term_style(sev: TERM_SEV_WARN), term_style_reset());
399 report_box_close(b: &box);
400 return;
401 }
402
403 crash_reg_fmt(out: line, cap: sizeof(line), name: "rip", v: r->rip, true);
404 report_box_printf(b: &box, fmt: "%s", line);
405 crash_reg_fmt(out: line, cap: sizeof(line), name: "rsp", v: r->rsp, false);
406 report_box_printf(b: &box, fmt: "%s", line);
407 crash_reg_fmt(out: line, cap: sizeof(line), name: "rbp", v: r->rbp, false);
408 report_box_printf(b: &box, fmt: "%s", line);
409
410 nr = r->rbp ? stack_unwind(frame: r->rbp, entries, max: 6) : 0;
411 if (!nr) {
412 report_box_printf(b: &box, fmt: "%s<no frames>%s", term_style(sev: TERM_SEV_DIM),
413 term_style_reset());
414 report_box_close(b: &box);
415 return;
416 }
417
418 for (size_t i = 0; i < nr; i++) {
419 uint64_t off = 0;
420 uint32_t lineno = 0;
421 const char *sym = debug_symbolize(addr: entries[i], out_off: &off);
422 const char *file = debug_line_for(addr: entries[i] - 1, out_line: &lineno);
423
424 if (sym && file)
425 report_box_printf(b: &box, fmt: "%s#%-2zu%s %s+0x%lx %sat %s:%u%s",
426 term_style(sev: TERM_SEV_DIM), i, term_style_reset(),
427 sym, off, term_style(sev: TERM_SEV_DIM), file, lineno,
428 term_style_reset());
429 else if (sym)
430 report_box_printf(b: &box, fmt: "%s#%-2zu%s %s+0x%lx",
431 term_style(sev: TERM_SEV_DIM), i, term_style_reset(),
432 sym, off);
433 else
434 report_box_printf(b: &box, fmt: "%s#%-2zu%s %016lx",
435 term_style(sev: TERM_SEV_DIM), i, term_style_reset(),
436 entries[i]);
437 }
438
439 report_box_close(b: &box);
440}
441
442static void crash_other_cpus(struct report_panes *panes) {
443 struct report_target col0 = report_pane(panes, pane: 0);
444 uint64_t self = smp_id(cond: TOPC_NONE);
445 uint16_t inner = 0;
446 uint32_t count = 0;
447 uint32_t per_col;
448 uint32_t slot = 0;
449 uint64_t id;
450
451 if (global.current_bootstage < BOOTSTAGE_MID_MP) {
452 report_printf(tgt: &col0, fmt: " %s<single core at this bootstage>%s",
453 term_style(sev: TERM_SEV_DIM), term_style_reset());
454 return;
455 }
456
457 if (!PERCPU_READY(crash_regs)) {
458 report_printf(tgt: &col0, fmt: " %s<percpu regs not initialised yet>%s",
459 term_style(sev: TERM_SEV_DIM), term_style_reset());
460 return;
461 }
462
463 for_each_cpu_id(id) {
464 if (id != self)
465 count++;
466 }
467
468 if (!count) {
469 report_printf(tgt: &col0, fmt: " %s<no other cores>%s", term_style(sev: TERM_SEV_DIM),
470 term_style_reset());
471 return;
472 }
473
474 for (uint32_t i = 0; i < panes->n; i++) {
475 struct report_target s = report_pane(panes, pane: i);
476 uint16_t room = s.width > 4 ? (uint16_t) (s.width - 4) : 1;
477 if (!inner || room < inner)
478 inner = room;
479 }
480
481 per_col = DIV_ROUND_UP(count, panes->n);
482 if (!per_col)
483 per_col = 1;
484
485 for_each_cpu_id(id) {
486 if (id == self)
487 continue;
488
489 uint32_t which = slot++ / per_col;
490 struct report_target target = report_pane(
491 panes, pane: which < panes->n ? which : (uint32_t) panes->n - 1);
492 crash_cpu_box(tgt: &target, id, inner);
493 }
494}
495
496#define CRASH_LOGO_INDENT 2
497
498static size_t crash_logo_indent(const char *) {
499 size_t least = (size_t) -1;
500 for (const char *p = logo; *p;) {
501 size_t indent = 0;
502
503 while (*p == ' ') {
504 indent++;
505 p++;
506 }
507
508 if (*p && *p != '\n' && indent < least)
509 least = indent;
510
511 p = strchrnul(s: p, c: '\n');
512
513 if (*p)
514 p++;
515 }
516 return least == (size_t) -1 ? 0 : least;
517}
518
519static size_t crash_logo_width(const char *, size_t skip) {
520 size_t widest = 0;
521 for (const char *p = logo; *p;) {
522 const char *start;
523
524 for (size_t i = 0; i < skip && *p == ' '; i++)
525 p++;
526
527 start = p;
528 p = strchrnul(s: p, c: '\n');
529
530 if ((size_t) (p - start) > widest)
531 widest = (size_t) (p - start);
532
533 if (*p)
534 p++;
535 }
536 return widest;
537}
538
539static void crash_logo_panel(struct report_target *tgt, const char *) {
540 size_t skip = crash_logo_indent(logo);
541 size_t art = crash_logo_width(logo, skip);
542 size_t width = report_target_width(tgt);
543 size_t pad = art < width ? (width - art) / 2 : 0;
544
545 for (const char *p = logo; *p;) {
546 char row[REPORT_PANE_LINE_MAX];
547 const char *eol;
548 size_t n;
549
550 for (size_t i = 0; i < skip && *p == ' '; i++)
551 p++;
552
553 eol = strchrnul(s: p, c: '\n');
554 n = (size_t) (eol - p);
555 if (n + 1 > sizeof(row))
556 n = sizeof(row) - 1;
557
558 memcpy(row, p, n);
559 p = eol;
560
561 while (n && row[n - 1] == ' ')
562 n--;
563
564 row[n] = '\0';
565 if (*p)
566 p++;
567
568 report_printf(tgt, fmt: "%*s%s%s%s", (int) pad, "", ANSI_RED, row,
569 ANSI_RESET);
570 }
571}
572
573static void crash_where_box(struct report_target *tgt, const char *file,
574 int line, const char *func) {
575 const char *sep = term_unicode() ? " · " : " | ";
576 char loc[REPORT_LINE_MAX];
577 char ctx[REPORT_LINE_MAX];
578 struct report_box box;
579 size_t w;
580
581 snprintf(loc, (int) sizeof(loc), "%s%s:%d%s %s%s()%s", ANSI_GREEN,
582 file ? file : "<unknown>", line, ANSI_RESET, ANSI_CYAN,
583 func ? func : "<unknown>", ANSI_RESET);
584
585 char *thread_name = global.current_bootstage >= BOOTSTAGE_LATE
586 ? thread_get_current()->name
587 : "(null)";
588 if (global.current_bootstage < BOOTSTAGE_EARLY_DEVICES)
589 snprintf(ctx, (int) sizeof(ctx),
590 "cpu %lu%stime unknown%sbootstage '%s'", smp_id(TOPC_NONE),
591 sep, sep, bootstage_str[global.current_bootstage]);
592 else
593 snprintf(ctx, (int) sizeof(ctx),
594 "cpu %lu%s%lu ms%sbootstage '%s'\nthread '%s'",
595 smp_id(TOPC_NONE), sep, time_get_ms(), sep,
596 bootstage_str[global.current_bootstage], thread_name);
597
598 w = report_strwidth(s: loc);
599 if (report_strwidth(s: ctx) > w)
600 w = report_strwidth(s: ctx);
601
602 report_box_open(b: &box, target: *tgt, title: "where", inner: (uint16_t) w);
603 report_box_printf(b: &box, fmt: "%s", loc);
604 report_box_printf(b: &box, fmt: "%s%s%s", term_style(sev: TERM_SEV_DIM), ctx,
605 term_style_reset());
606 report_box_close(b: &box);
607}
608
609static void crash_code_name_box(struct report_target *tgt, const char *name) {
610 struct report_box box;
611
612 report_box_open(b: &box, target: *tgt, NULL, inner: (uint16_t) report_strwidth(s: name));
613 report_box_printf(b: &box, fmt: "%s%s%s", term_style(sev: TERM_SEV_CRIT), name,
614 term_style_reset());
615 report_box_close(b: &box);
616}
617
618static void crash_empty_box_panel(struct report_target *tgt) {
619 for (size_t i = 0; i < panic_scene_count; i++)
620 report_puts(tgt, s: panic_scene[i]);
621}
622
623static struct report_target *active_facility_target = NULL;
624
625void crash_facility_printf(const char *fmt, ...) {
626 va_list ap;
627 va_start(ap, fmt);
628
629 if (active_facility_target) {
630 char buf[REPORT_PANE_LINE_MAX];
631 vsnprintf(buf, sizeof(buf), fmt, ap);
632 report_wrap(tgt: active_facility_target, text: buf);
633 } else {
634 vprintf(NULL, format: fmt, args: ap);
635 printf_unlocked(format: "\n");
636 }
637
638 va_end(ap);
639}
640
641void crash_facility_line(const char *fmt, ...) {
642 va_list ap;
643 va_start(ap, fmt);
644
645 char buf[REPORT_PANE_LINE_MAX];
646 vsnprintf(buf, sizeof(buf), fmt, ap);
647
648 if (!active_facility_target) {
649 printf_unlocked(format: "%s\n", buf);
650 } else if (report_strwidth(s: buf) <=
651 report_target_width(tgt: active_facility_target)) {
652 report_puts(tgt: active_facility_target, s: buf);
653 } else {
654 size_t indent = strspn(s: buf, accept: " ");
655 struct report_target hang =
656 report_target_indent(tgt: *active_facility_target, indent: (uint16_t) indent);
657 report_wrap(tgt: &hang, text: buf + indent);
658 }
659
660 va_end(ap);
661}
662
663static void crash_facility_dump(const struct crash_facility *facility,
664 struct report_target *tgt,
665 struct crash_payload pl) {
666 active_facility_target = tgt;
667 facility->dump(pl);
668 active_facility_target = NULL;
669}
670
671static void crash_emit_ndjson_records(const struct crash_context *ctx,
672 const struct crash_regs *regs,
673 uint32_t depth) {
674 const char *thread = global.current_bootstage >= BOOTSTAGE_LATE
675 ? thread_get_current()->name
676 : NULL;
677
678 ndjson_emit(panic_at, .file = ctx->file ? ctx->file : "",
679 .line = (uint64_t) ctx->line,
680 .func = ctx->func ? ctx->func : "",
681 .msg = ctx->msg ? ctx->msg : "",
682 .bootstage = bootstage_str[global.current_bootstage],
683 .thread = thread, .depth = depth);
684
685 uint64_t entries[STACK_TRACE_MAX_DEPTH];
686 size_t nr = 0;
687
688 if (regs && regs->rip)
689 entries[nr++] = regs->rip;
690
691 if (global.current_bootstage >= BOOTSTAGE_EARLY_ALLOCATORS) {
692 if (regs && regs->rbp)
693 nr += stack_unwind(frame: regs->rbp, entries: entries + nr,
694 STACK_TRACE_MAX_DEPTH - nr);
695 else
696 nr += stack_unwind(frame: (uint64_t) ci_frame_address(0), entries: entries + nr,
697 STACK_TRACE_MAX_DEPTH - nr);
698 }
699
700 for (size_t i = 0; i < nr; i++) {
701 uint64_t off = 0;
702 uint32_t srcline = 0;
703 const char *sym = debug_symbolize(addr: entries[i], out_off: &off);
704 const char *srcfile = debug_line_for(addr: entries[i] - 1, out_line: &srcline);
705
706 ndjson_emit(panic_frame, .idx = i, .addr = entries[i], .sym = sym,
707 .off = off, .file = srcfile, .line = srcline);
708 }
709}
710
711static const char *crash_source_title(enum crash_source s) {
712 switch (s) {
713 case CRASH_SOURCE_PANIC: return "kernel panic";
714 case CRASH_SOURCE_ENFORCE: return "enforcement failure";
715 case CRASH_SOURCE_KASAN: return "kasan fault";
716 case CRASH_SOURCE_UBSAN: return "ubsan violation";
717 case CRASH_SOURCE_NMI_WATCHDOG: return "watchdog lockup";
718 case CRASH_SOURCE_CPU_EXCEPTION: return "cpu exception";
719 case CRASH_SOURCE_NIGHTMARE: return "nightmare finding";
720 case CRASH_SOURCE_LOCK_CHK: return "lock validator violation";
721 default: return "kernel crash";
722 }
723}
724
725static const char *crash_source_name(enum crash_source s) {
726 switch (s) {
727 case CRASH_SOURCE_PANIC: return "PANIC";
728 case CRASH_SOURCE_ENFORCE: return "ENFORCEMENT FAILED";
729 case CRASH_SOURCE_KASAN: return "KASAN";
730 case CRASH_SOURCE_UBSAN: return "UBSAN";
731 case CRASH_SOURCE_NMI_WATCHDOG: return "WATCHDOG";
732 case CRASH_SOURCE_CPU_EXCEPTION: return "CPU EXCEPTION";
733 case CRASH_SOURCE_NIGHTMARE: return "NIGHTMARE";
734 case CRASH_SOURCE_LOCK_CHK: return "LOCK_CHK";
735 default: return "CRASH";
736 }
737}
738
739static void crash_report_visual(const struct crash_context *ctx,
740 const struct crash_regs *regs) {
741 struct report_target con = report_console();
742 struct report_panes *panes = report_panes_panic();
743
744 printf(format: "\033[H\033[2J");
745
746 report_panes_begin(panes, n: 2, weights: (const uint8_t[]){42, 58});
747 const char *title = crash_source_title(s: ctx->source);
748 report_panes_title(panes, pane: 0, sev: TERM_SEV_CRIT, title);
749 report_panes_title(panes, pane: 1, sev: TERM_SEV_LABEL, title: "registers");
750 report_panes_top(panes);
751
752 struct report_target left = report_pane(panes, pane: 0);
753 struct report_target right = report_pane(panes, pane: 1);
754
755 const struct crash_code *code = ctx->payload.code;
756 const struct crash_facility *facility = code ? code->facility : NULL;
757 crash_logo_panel(tgt: &left, logo: OS_LOGO_PANIC_CENTERED);
758 report_blank(tgt: &left);
759
760 crash_where_box(tgt: &left, file: ctx->file, line: ctx->line, func: ctx->func);
761
762 if (report_section_begin_at(tgt: &left, name: "message")) {
763 report_wrap_printf(tgt: &left, fmt: "%s%s%s", term_style(sev: TERM_SEV_HEAD),
764 ctx->msg ? ctx->msg : "<no message>",
765 term_style_reset());
766 }
767 report_section_end();
768
769 if (report_section_claim_at(tgt: &right, name: "registers"))
770 crash_regs_panel(tgt: &right, regs);
771 report_section_end();
772
773 if (report_section_begin_at(tgt: &right, name: "backtrace"))
774 crash_backtrace_panel(tgt: &right, regs);
775 report_section_end();
776
777 char name_top[128];
778 char code_id[64];
779 const char *code_name = NULL;
780 if (code) {
781 crash_code_name(code, buf: code_id, len: sizeof(code_id));
782 snprintf(name_top, sizeof(name_top),
783 ANSI_BOLD ANSI_WHITE "%s" ANSI_RESET, code_id);
784 code_name = code->desc;
785 } else {
786 strcpy(name_top, "empty box");
787 }
788
789 if (report_section_begin_at(tgt: &right, name: name_top)) {
790 if (code_name)
791 crash_code_name_box(tgt: &right, name: code_name);
792
793 if (facility && facility->dump)
794 crash_facility_dump(facility, tgt: &right, pl: ctx->payload);
795 else if (!code_name)
796 crash_empty_box_panel(tgt: &right);
797 }
798
799 report_section_end();
800
801 report_panes_flush(panes);
802
803 if (ctx->formats & CRASH_FMT_PEER_CPUS) {
804 report_panes_carry(panes);
805 report_panes_begin(panes, n: crash_cpu_panes(), NULL);
806 report_panes_undivided(panes);
807 report_panes_title(panes, pane: 0, sev: TERM_SEV_LABEL, title: "other CPUs");
808 report_panes_top(panes);
809
810 struct report_target cpu0 = report_pane(panes, pane: 0);
811 if (report_section_claim_at(tgt: &cpu0, name: "other CPUs"))
812 crash_other_cpus(panes);
813 report_section_end();
814 report_panes_flush(panes);
815 }
816
817 if (ctx->formats & CRASH_FMT_DUMP_LOGS) {
818 report_panes_bottom(panes, sev: TERM_SEV_LABEL, title: "logs");
819 if (report_section_claim(name: "logs"))
820 log_dump_panic();
821 report_section_end();
822 }
823
824 report_blank(tgt: &con);
825}
826
827static void crash_report_raw_serial(const struct crash_context *ctx,
828 const struct crash_regs *regs) {
829 printf_unlocked(format: "\n*** KERNEL CRASH: %s ***\n",
830 crash_source_name(s: ctx->source));
831 if (ctx->file || ctx->func)
832 printf_unlocked(format: "Location: %s:%d in %s()\n",
833 ctx->file ? ctx->file : "<unknown>", ctx->line,
834 ctx->func ? ctx->func : "<unknown>");
835 if (ctx->msg)
836 printf_unlocked(format: "Message : %s\n", ctx->msg);
837
838 const struct crash_code *code = ctx->payload.code;
839 const struct crash_facility *facility = code ? code->facility : NULL;
840 if (code) {
841 char code_id[64];
842 crash_code_name(code, buf: code_id, len: sizeof(code_id));
843 printf_unlocked(format: "Code : %s%s%s%s\n", code_id, code->desc ? " (" : "",
844 code->desc ? code->desc : "", code->desc ? ")" : "");
845 }
846
847 if (facility && facility->dump)
848 crash_facility_dump(facility, NULL, pl: ctx->payload);
849
850 if (regs) {
851 printf_unlocked(
852 format: "RIP: %016lx RSP: %016lx RFLAGS: %016lx CR2: %016lx\n",
853 regs->rip, regs->rsp, regs->rflags, regs->cr2);
854 printf_unlocked(format: "RAX: %016lx RBX: %016lx RCX: %016lx RDX: %016lx\n",
855 regs->rax, regs->rbx, regs->rcx, regs->rdx);
856 }
857}
858
859cc_noreturn void crash_full(const struct crash_context *ctx) TSA_NO_ANALYSIS {
860 irq_disable();
861
862 irql_raise(new_level: IRQL_HIGH_LEVEL);
863
864 uint32_t depth = atomic_fetch_add_rlx(&crash_depth, 1);
865
866 if (depth >= CRASH_MAX_DEPTH) {
867 printf_unlocked(format: "\n[crash depth %u, aborting report]\n", depth);
868
869#if defined(TEST_ENABLED) || defined(TEST_NIGHTMARE_ENABLED)
870 ndjson_bye(code: QEMU_EXIT_PANIC, reason: "nested_crash");
871 qemu_exit(code: QEMU_EXIT_PANIC);
872#endif
873
874 while (true)
875 cpu_halt();
876 }
877
878 if (depth == 0)
879 raw_spin_lock(lock: &crash_lock);
880
881 int64_t unowned = -1;
882 atomic_cas_strong(&crash_owner, &unowned, (int64_t) smp_id(TOPC_NONE));
883 atomic_store(&global.panicked, true);
884
885 struct crash_regs captured_regs;
886 if (ctx->regs) {
887 captured_regs = *ctx->regs;
888 } else {
889 crash_capture_regs(out: &captured_regs);
890 }
891
892 if (PERCPU_READY(crash_regs)) {
893 PERCPU_READ(TOPC_NONE, crash_regs) = captured_regs;
894 } else {
895 boot_crash_regs = captured_regs;
896 }
897
898 if (global.current_bootstage >= BOOTSTAGE_MID_MP) {
899 crash_broadcast_nmi();
900 sleep_spin_ms(msec: 500);
901 }
902
903 report_enter_panic();
904 ndjson_enter_panic();
905
906 uint32_t formats = ctx->formats;
907 if (global.current_bootstage < BOOTSTAGE_EARLY_DEVICES) {
908 formats &= ~CRASH_FMT_VISUAL_PANES;
909 formats |= CRASH_FMT_RAW_SERIAL;
910 }
911
912 if (formats & CRASH_FMT_NDJSON)
913 crash_emit_ndjson_records(ctx, regs: &captured_regs, depth);
914
915 if (formats & CRASH_FMT_VISUAL_PANES)
916 crash_report_visual(ctx, regs: &captured_regs);
917 else if (formats & CRASH_FMT_RAW_SERIAL)
918 crash_report_raw_serial(ctx, regs: &captured_regs);
919
920 if ((formats & CRASH_FMT_DUMP_LOGS) && !(formats & CRASH_FMT_VISUAL_PANES))
921 log_dump_panic();
922
923 if (depth == 0)
924 raw_spin_unlock(lock: &crash_lock);
925
926#if defined(TEST_ENABLED) || defined(TEST_NIGHTMARE_ENABLED)
927 ndjson_bye(code: QEMU_EXIT_PANIC, reason: "crash");
928 qemu_exit(code: QEMU_EXIT_PANIC);
929#endif
930
931 while (true)
932 cpu_halt();
933}
934
935CRASH_FACILITY_DEFINE(global, NULL);
936
937size_t crash_code_name(const struct crash_code *code, char *buf, size_t len) {
938 if (!code)
939 return (size_t) snprintf(buf, len, "GENERIC");
940
941 if (code->facility == &CRASH_FACILITY(global))
942 return (size_t) snprintf(buf, len, "%s", code->name);
943
944 return (size_t) snprintf(buf, len, "%s.%s", code->facility->name,
945 code->name);
946}
947
948#ifdef DEBUG_CRASH
949
950void crash_thread_init(struct thread *t) {
951 cc_ignore(t);
952
953 struct crash_thread_data *pt = &t->crash_data;
954 list_head_init(&pt->crash_hooks);
955 pt->in_hook = false;
956 list_head_init(&pt->unwind.free_list);
957 list_head_init(&pt->unwind.in_use);
958 for (int i = 0; i < CRASH_UNWIND_NODES; i++)
959 list_head_push_back(&pt->unwind.free_list, &pt->unwind.nodes[i].list);
960}
961
962static struct crash_unwind_node *
963unwind_node_alloc(struct thread *t, enum crash_unwind_type type) {
964 struct list_node *n = list_head_pop_back(&t->crash_data.unwind.free_list);
965 if (!n)
966 panic(CAPACITY_EXCEEDED, "unwind node pool");
967
968 struct crash_unwind_node *node =
969 container_of(n, struct crash_unwind_node, list);
970 enforce(node->type == CRASH_UNWIND_NONE);
971 node->type = type;
972 return node;
973}
974
975static void unwind_node_free(struct thread *t, struct crash_unwind_node *n) {
976 n->type = CRASH_UNWIND_NONE;
977 n->data.raw = 0;
978 list_head_push_back(&t->crash_data.unwind.free_list, &n->list);
979}
980
981/* Small optimization here: it doesn't matter if we go backwards or
982 * forwards, simply that we get to the node, but it's more likely
983 * to be found faster if we iterate in reverse */
984static void crash_unwind_node_add(struct crash_unwind_node_data *data,
985 enum crash_unwind_type type) {
986 struct thread *t = thread_get_current();
987 enforce(!t->crash_data.unwinding);
988 if (type == CRASH_UNWIND_RCU) {
989 struct crash_unwind_node *n;
990 list_for_each_entry_rev(n, &t->crash_data.unwind.in_use, list) {
991 if (n->type == type) {
992 n->data.rcu_lock_times++;
993 return;
994 }
995 }
996
997 n = unwind_node_alloc(t, type);
998 n->data.rcu_lock_times = 1;
999 list_head_push_back(&t->crash_data.unwind.in_use, &n->list);
1000 return;
1001 }
1002
1003 struct crash_unwind_node *node = unwind_node_alloc(t, type);
1004 node->data = *data;
1005 list_head_push_back(&t->crash_data.unwind.in_use, &node->list);
1006}
1007
1008static void crash_unwind_node_remove(struct crash_unwind_node_data *data,
1009 enum crash_unwind_type type) {
1010 struct thread *t = thread_get_current();
1011 enforce(!t->crash_data.unwinding);
1012 bool found = false;
1013
1014 if (type == CRASH_UNWIND_RCU) {
1015 struct crash_unwind_node *n;
1016 list_for_each_entry_rev(n, &t->crash_data.unwind.in_use, list) {
1017 if (n->type == type) {
1018 if (n->data.rcu_lock_times == 1) {
1019 list_node_delete(&n->list);
1020 unwind_node_free(t, n);
1021 } else {
1022 n->data.rcu_lock_times--;
1023 }
1024 return;
1025 }
1026 }
1027
1028 goto out;
1029 }
1030
1031 struct crash_unwind_node *iter = NULL;
1032
1033 list_for_each_entry_rev(iter, &t->crash_data.unwind.in_use, list) {
1034 if (iter->data.raw == data->raw && iter->type == type) {
1035 list_node_delete(&iter->list);
1036 unwind_node_free(t, iter);
1037 found = true;
1038 break;
1039 }
1040 }
1041
1042out:
1043 enforce(found, BAD_FREE, "unwind node not in use");
1044}
1045
1046static void unwind_rcu(struct crash_unwind_node_data *d) TSA_NO_ANALYSIS {
1047 for (uintptr_t i = 0; i < d->rcu_lock_times; i++)
1048 rcu_read_unlock();
1049}
1050
1051static void unwind_mutex(struct crash_unwind_node_data *d) TSA_NO_ANALYSIS {
1052 mutex_unlock(d->ptr);
1053}
1054
1055static void unwind_rwlock(struct crash_unwind_node_data *d) TSA_NO_ANALYSIS {
1056 rw_unlock(d->ptr);
1057}
1058
1059static void unwind_spinlock(struct crash_unwind_node_data *d) TSA_NO_ANALYSIS {
1060 spin_unlock(d->ptr, d->arg);
1061}
1062
1063static void unwind_qspinlock(struct crash_unwind_node_data *d) TSA_NO_ANALYSIS {
1064 qspin_unlock(d->ptr, d->arg);
1065}
1066
1067static void (*unwind_cbs[CRASH_UNWIND_MAX])(struct crash_unwind_node_data *) = {
1068 [CRASH_UNWIND_RCU] = unwind_rcu,
1069 [CRASH_UNWIND_MUTEX] = unwind_mutex,
1070 [CRASH_UNWIND_RWLOCK] = unwind_rwlock,
1071 [CRASH_UNWIND_SPINLOCK] = unwind_spinlock,
1072 [CRASH_UNWIND_QSPINLOCK] = unwind_qspinlock,
1073};
1074
1075STRINGIFY_DECLARE_ENUM(crash_unwind_type) {
1076 switch (stringify_in) {
1077 case CRASH_UNWIND_RCU: return "RCU";
1078 case CRASH_UNWIND_MUTEX: return "MUTEX";
1079 case CRASH_UNWIND_RWLOCK: return "RWLOCK";
1080 case CRASH_UNWIND_SPINLOCK: return "SPINLOCK";
1081 case CRASH_UNWIND_QSPINLOCK: return "QSPINLOCK";
1082 default: cc_unreachable();
1083 }
1084}
1085
1086static void crash_unwind_enter(uintptr_t data, enum crash_unwind_type type,
1087 uintptr_t arg) {
1088 if (global.current_bootstage >= BOOTSTAGE_LATE &&
1089 !thread_get_current()->crash_data.unwinding) {
1090 struct crash_unwind_node_data nd = {
1091 .raw = data,
1092 .arg = arg,
1093 };
1094
1095 crash_unwind_node_add(&nd, type);
1096 }
1097}
1098
1099static void crash_unwind_exit(uintptr_t data, enum crash_unwind_type type) {
1100 if (global.current_bootstage >= BOOTSTAGE_LATE &&
1101 !thread_get_current()->crash_data.unwinding) {
1102 struct crash_unwind_node_data nd = {
1103 .raw = data,
1104 };
1105
1106 crash_unwind_node_remove(&nd, type);
1107 }
1108}
1109
1110/* The idea here: we first traverse backwards and unwind
1111 * one by one, detaching as we go */
1112void crash_unwind(void) TSA_NO_ANALYSIS {
1113 struct thread *t = thread_get_current();
1114 struct crash_thread_data *pt = &t->crash_data;
1115 pt->unwinding = true;
1116
1117 struct crash_unwind_thread_data *upt = &pt->unwind;
1118 struct crash_unwind_node *cun, *tmp;
1119 list_for_each_entry_safe_rev(cun, tmp, &upt->in_use, list) {
1120 enforce(cun->type != CRASH_UNWIND_NONE);
1121 thread_warn("unwinding %s (%p)", crash_unwind_type_to_str(cun->type),
1122 cun->data.ptr);
1123 unwind_cbs[cun->type](&cun->data);
1124 list_node_delete(&cun->list);
1125 unwind_node_free(t, cun);
1126 }
1127
1128 irql_lower(IRQL_PASSIVE_LEVEL);
1129
1130 pt->unwinding = false;
1131}
1132
1133void crash_unwind_enter_rcu(void) {
1134 crash_unwind_enter(0, CRASH_UNWIND_RCU, 0);
1135}
1136
1137void crash_unwind_exit_rcu(void) {
1138 crash_unwind_exit(0, CRASH_UNWIND_RCU);
1139}
1140
1141void crash_unwind_enter_mutex(struct mutex *m) {
1142 crash_unwind_enter((uintptr_t) m, CRASH_UNWIND_MUTEX, 0);
1143}
1144
1145void crash_unwind_exit_mutex(struct mutex *m) {
1146 crash_unwind_exit((uintptr_t) m, CRASH_UNWIND_MUTEX);
1147}
1148
1149void crash_unwind_enter_rwlock(struct rwlock *r) {
1150 crash_unwind_enter((uintptr_t) r, CRASH_UNWIND_RWLOCK, 0);
1151}
1152
1153void crash_unwind_exit_rwlock(struct rwlock *r) {
1154 crash_unwind_exit((uintptr_t) r, CRASH_UNWIND_RWLOCK);
1155}
1156
1157void crash_unwind_enter_spinlock(struct spinlock *s, enum irql old) {
1158 crash_unwind_enter((uintptr_t) s, CRASH_UNWIND_SPINLOCK, (uintptr_t) old);
1159}
1160
1161void crash_unwind_exit_spinlock(struct spinlock *s) {
1162 crash_unwind_exit((uintptr_t) s, CRASH_UNWIND_SPINLOCK);
1163}
1164
1165void crash_unwind_enter_qspinlock(struct qspinlock *q, enum irql old) {
1166 crash_unwind_enter((uintptr_t) q, CRASH_UNWIND_QSPINLOCK, (uintptr_t) old);
1167}
1168
1169void crash_unwind_exit_qspinlock(struct qspinlock *q) {
1170 crash_unwind_exit((uintptr_t) q, CRASH_UNWIND_QSPINLOCK);
1171}
1172
1173#endif
1174