1/* @title: Crash Engine */
2#pragma once
3#include <asm.h>
4#include <atomic.h>
5#include <compiler/core.h>
6#include <linker/symbols.h>
7#include <sch/irql.h>
8#include <setjmp.h>
9#include <stdbool.h>
10#include <stddef.h>
11#include <stdint.h>
12#include <structures/list.h>
13#include <time/time.h>
14#include <types/types.h>
15
16struct crash_facility;
17struct spinlock;
18struct qspinlock;
19struct rwlock;
20struct mutex;
21struct thread;
22struct irq_registers;
23struct irq_context;
24
25#define CRASH_REG_COUNT 20
26
27enum qemu_exit_codes {
28 QEMU_EXIT_OK = 0,
29 QEMU_EXIT_FAIL = 1,
30 QEMU_EXIT_PANIC = 2,
31};
32
33struct crash_regs {
34 union {
35 struct {
36 uint64_t rip, rflags, cr2, cr3;
37 uint64_t rax, rbx, rcx, rdx, rbp, rdi, rsi;
38 uint64_t r8, r9, r10, r11, r12, r13, r14, r15;
39 uint64_t rsp;
40 };
41
42 uint64_t regs[CRASH_REG_COUNT];
43 };
44};
45
46ct_assert_struct_size_eq(crash_regs, CRASH_REG_COUNT * 8);
47
48/* NULL crashes are the generic, code-less crash */
49struct crash_code {
50 const char *name;
51 const char *desc;
52 const struct crash_facility *facility;
53};
54
55/* Codes that don't have a facility, like enforce(p, INVALID_ARGUMENT) */
56#define CRASH_CODES_global \
57 (INVALID_ARGUMENT, "Argument was NULL or out of range"), \
58 (INVALID_STATE, "Object was in a state this operation forbids"), \
59 (INVALID_CONTEXT, \
60 "Operation ran at a forbidden IRQL, IRQ or boot stage"), \
61 (OUT_OF_MEMORY, "Allocation that must not fail did fail"), \
62 (UNSUPPORTED, "Hardware or firmware lacks a required feature"), \
63 (CORRUPTION, "Data structure or allocator invariant was broken"), \
64 (CAPACITY_EXCEEDED, "A fixed-size limit was exceeded"), \
65 (UNREACHABLE, "Control reached a path that must not execute"), \
66 (DUPLICATE, "Object was registered, inserted, or defined twice"), \
67 (OVERFLOW, "Counter or arithmetic result exceeded its range"), \
68 (UNDERFLOW, "Counter was decremented below zero"), \
69 (STALE_REFERENCE, "Reference to an object that is dying or dead"), \
70 (BAD_FREE, "Freed memory was already free or never allocated"), \
71 (UNINITIALIZED, "Object was used before initialization"), \
72 (INIT_FAILED, "A required initialization step failed"), \
73 (NOT_FOUND, "Lookup that must succeed found nothing")
74
75enum crash_source {
76 CRASH_SOURCE_PANIC = 0, /* panic() call */
77 CRASH_SOURCE_ENFORCE, /* enforce() failure */
78 CRASH_SOURCE_KASAN, /* ASAN check failure */
79 CRASH_SOURCE_UBSAN, /* UBSAN failure */
80 CRASH_SOURCE_NMI_WATCHDOG, /* Watchdog / Liveness monitor hard stall */
81 CRASH_SOURCE_CPU_EXCEPTION, /* Hardware CPU fault */
82 CRASH_SOURCE_NIGHTMARE, /* Nightmare test harness failure */
83 CRASH_SOURCE_LOCK_CHK, /* Lock validator order / dependency violation */
84};
85
86enum crash_format_flags {
87 CRASH_FMT_RAW_SERIAL = 1 << 0, /* Minimal serial printf */
88 CRASH_FMT_VISUAL_PANES = 1 << 1, /* Dual pane ANSI console report */
89 CRASH_FMT_NDJSON = 1 << 2, /* NDJSON stream */
90 CRASH_FMT_DUMP_LOGS = 1 << 3, /* circular log buffer dump */
91 CRASH_FMT_PEER_CPUS = 1 << 4, /* Quiesce and render peer CPU frames */
92
93 /* Default formatting */
94 CRASH_FMT_DEFAULT = CRASH_FMT_RAW_SERIAL | CRASH_FMT_VISUAL_PANES |
95 CRASH_FMT_NDJSON | CRASH_FMT_DUMP_LOGS |
96 CRASH_FMT_PEER_CPUS,
97
98 /* Minimal formatting for early boot or nested crashes */
99 CRASH_FMT_MINIMAL = CRASH_FMT_RAW_SERIAL | CRASH_FMT_NDJSON,
100};
101
102enum crash_hook_flags {
103 CRASH_HOOK_DEFAULT = 0,
104 CRASH_HOOK_FACILITY = 1 << 0, /* Facility granularity: match
105 * any code whose ->facility is the
106 * code's facility
107 */
108
109 CRASH_HOOK_NO_UNWIND = 1 << 1, /* By default, the crash handler will make
110 * a best-effort attempt to unwind: it'll
111 * try to safely drop locks, exit RCU
112 * read-side critical sections, although
113 * memory may still leak (this is dependent
114 * on the facility implementation).
115 *
116 * NO_UNWIND allows this behavior to be
117 * skipped. This is because in certain cases,
118 * the unwinding cannot happen (e.g. lock
119 * checking is disabled), and also because
120 * the stale state not unwinding
121 * leaves can be used as a postmortem
122 * for state verification.
123 */
124
125};
126
127enum crash_unwind_type {
128 CRASH_UNWIND_NONE, /* should not be reachable */
129 CRASH_UNWIND_RCU,
130 CRASH_UNWIND_MUTEX,
131 CRASH_UNWIND_RWLOCK,
132 CRASH_UNWIND_SPINLOCK,
133 CRASH_UNWIND_QSPINLOCK,
134
135 CRASH_UNWIND_MAX,
136};
137
138struct crash_payload {
139 const struct crash_code *code;
140 void *data;
141 uintptr_t params[4];
142};
143
144struct report_target;
145struct crash_facility {
146 const char *name;
147 const char *desc;
148 const struct crash_code *const *codes; /* list order */
149 size_t nr_codes;
150 void (*const dump)(struct crash_payload pl);
151};
152
153/* The idea here:
154 *
155 * Each thread has a fixed pool of unwind nodes, with a list that stores
156 * the currently active nodes, which are unwound in LIFO order.
157 *
158 * The reason why it's a pool and not a stack is because you can have code
159 * such as
160 *
161 * mutex_lock(&lock);
162 * rcu_read_lock();
163 *
164 * mutex_unlock(&lock);
165 * rcu_read_unlock();
166 *
167 * and with a naive stack, the semantics break here/become tricky because
168 * the thread would have to reorganize the stack, whereas a list_head
169 * can just be yanked out of the middle.
170 *
171 * Thus, the unwind node enqueue path is:
172 *
173 * (1) try to reserve one
174 * (2) list_add_tail it
175 *
176 * And dequeue becomes:
177 *
178 * (1) find the pointer to the current thing in question
179 * (2) dequeue it
180 *
181 * RCU is even simpler: the first rcu_read_lock gets a node,
182 * and every subsequent lock bumps a counter, dec'ing the counter
183 * upon unlock, and the crash path simply rcu_read_unlock's it
184 * that many times.
185 */
186
187struct crash_unwind_node_data {
188 uintptr_t arg;
189 union {
190 void *ptr;
191 size_t rcu_lock_times;
192 enum irql irql;
193 uintptr_t raw;
194 };
195};
196
197struct crash_unwind_node {
198 enum crash_unwind_type type;
199 struct list_node list;
200 struct crash_unwind_node_data data;
201};
202
203#define CRASH_UNWIND_NODES 128
204struct crash_unwind_thread_data {
205 struct list_head free_list;
206 struct list_head in_use; /* LIFO */
207 struct crash_unwind_node nodes[CRASH_UNWIND_NODES];
208};
209
210struct crash_thread_data {
211 struct crash_unwind_thread_data unwind;
212 struct list_head crash_hooks; /* struct crash_hook */
213 bool unwinding;
214 bool in_hook;
215 jmp_buf env;
216};
217
218/*
219 * The idea of crash hooks:
220 *
221 * A hook can hook into a specific code, which is checked
222 * against a facility to verify if it is hookable.
223 *
224 * They are hooked PER-THREAD, so as to not introduce strange
225 * non-determinism bugs and the chance that the crash hook
226 * registration path itself can crash (would be problematic)
227 * due to synchronization violations.
228 *
229 * The generic (NULL) code is not hookable
230 */
231struct crash_hook {
232 char *name;
233 struct list_node list;
234 const struct crash_code *code;
235 enum crash_source source_mask;
236};
237
238struct crash_context {
239 struct crash_payload payload;
240 enum crash_source source;
241 enum crash_format_flags formats;
242 const char *stmt;
243 const char *file;
244 int line;
245 const char *func;
246 const char *msg;
247 const struct crash_regs *regs; /* NULL = capture caller's frame via asm */
248 void *source_data;
249};
250
251#define CRASH_WAIT_US MS_TO_US(500) /* Quiesce timeout per peer CPU */
252#define CRASH_SPIN_ONE_US 5
253#define CRASH_MAX_DEPTH 2 /* Max recursive fault depth */
254#define CRASH_MSG_MAX 256
255
256#define CRASH_PAYLOAD(c, d) ((struct crash_payload) {.code = c, .data = d})
257#define CRASH_PARAMS(c, p0, p1, p2, p3) \
258 ((struct crash_payload) {.code = (c), \
259 .params = {(uintptr_t) (p0), (uintptr_t) (p1), \
260 (uintptr_t) (p2), (uintptr_t) (p3)}})
261
262#define CRASH_CODE_TO_PAYLOAD(c) ((struct crash_payload) {.code = c})
263
264/*
265 * Use like this:
266 * #define CRASH_CODES_asan \
267 * (USE_AFTER_FREE, "Use after free"), \
268 * (PARTIAL_GRANULE)
269 *
270 * CRASH_CODE(asan, USE_AFTER_FREE)
271 */
272#define CRASH_FACILITY(f) __crash_facility_##f
273
274#define CRASH_CODE(f, n) _crash_code_sym(f, n)
275#define CRASH_CODE_NTH(f, i) __crash_codes_##f[(i)]
276
277#define CRASH_CODES_ENUM(f, prefix) \
278 PP_FOREACH_CTX(_crash_code_enumerator, prefix, CRASH_CODES_##f)
279#define crash_code_in(c, f) crash_code_in_facility((c), &CRASH_FACILITY(f))
280
281#define CRASH_CODES_DECLARE(f) \
282 extern const struct crash_facility CRASH_FACILITY(f); \
283 extern const struct crash_code *const __crash_codes_##f[]; \
284 PP_FOREACH_CTX(_crash_code_extern, f, CRASH_CODES_##f)
285
286#define CRASH_FACILITY_DEFINE(f, d, ...) \
287 CRASH_CODES_DECLARE(f); \
288 PP_FOREACH_CTX(_crash_code_def, f, CRASH_CODES_##f) \
289 const struct crash_code *const __crash_codes_##f[] = { \
290 PP_FOREACH_CTX(_crash_code_ptr, f, CRASH_CODES_##f)}; \
291 const struct crash_facility CRASH_FACILITY(f) = { \
292 .name = #f, \
293 .desc = d, \
294 .codes = __crash_codes_##f, \
295 .nr_codes = ct_array_size(__crash_codes_##f), \
296 __VA_ARGS__}
297
298#define _crash_code_sym(f, n) __crash_code_##f##_##n
299#define _crash_code_name(t) PP_GET_1 t
300#define _crash_code_desc(t) _crash_code_desc_(PP_UNPAREN(t), NULL, ~)
301#define _crash_code_desc_(...) _crash_code_desc_pick(__VA_ARGS__)
302#define _crash_code_desc_pick(n, d, ...) d
303
304#define _crash_code_extern(f, t) _crash_code_extern_(f, _crash_code_name(t))
305#define _crash_code_extern_(f, n) \
306 extern const struct crash_code _crash_code_sym(f, n)[1];
307#define _crash_code_def(f, t) \
308 _crash_code_def_(f, _crash_code_name(t), _crash_code_desc(t))
309#define _crash_code_def_(f, n, d) \
310 const struct crash_code _crash_code_sym(f, n)[1] = { \
311 {PP_STRINGIZE(n), d, &CRASH_FACILITY(f)}};
312#define _crash_code_ptr(f, t) _crash_code_ptr_(f, _crash_code_name(t))
313#define _crash_code_ptr_(f, n) _crash_code_sym(f, n),
314#define _crash_code_enumerator(p, t) PP_CONCAT(p, _crash_code_name(t)),
315
316CRASH_CODES_DECLARE(global);
317
318#define CRASH_CODE_ARG(a) PP_IIF(PP_IS_PAREN(a))(_crash_tuple_code(a), a)
319
320#define _crash_tuple_code(t) \
321 PP_CONCAT(_crash_tuple_code_, _crash_narg(PP_UNPAREN(t))) t
322#define _crash_narg(...) PP_NARG(__VA_ARGS__)
323#define _crash_tuple_code_1(n) CRASH_CODE(global, n)
324#define _crash_tuple_code_2(f, n) CRASH_CODE(f, n)
325
326/* Wraps stmt in the bare-name scope unless code was a tuple */
327#define CRASH_CODE_SCOPED(a, ...) \
328 PP_CONCAT(_crash_code_scoped_, PP_IS_PAREN(a))(__VA_ARGS__)
329#define _crash_code_scoped_1(...) __VA_ARGS__
330#define _crash_code_scoped_0(...) \
331 ({ \
332 _CRASH_CODE_SCOPE \
333 __VA_ARGS__; \
334 })
335
336#define _crash_code_alias(f, t) _crash_code_alias_(f, _crash_code_name(t))
337#define _crash_code_alias_(f, n) \
338 cc_unused extern const struct crash_code n[1] asm(PP_STRINGIZE(__USER_LABEL_PREFIX__) PP_STRINGIZE(_crash_code_sym(f, n)));
339
340#define _CRASH_CODE_SCOPE \
341 PP_FOREACH_CTX(_crash_code_alias, global, CRASH_CODES_global)
342
343cc_noreturn void enforce_impl_msg(const char *file, int line, const char *func,
344 const char *prefix, const char *expr,
345 const char *fmt, ...) cc_printf_like(6, 7);
346/* enforce_impl_msg() with INVALID_ARGUMENT for enforce_precond() */
347cc_noreturn void enforce_impl_precond_msg(const char *file, int line,
348 const char *func, const char *prefix,
349 const char *expr, const char *fmt,
350 ...) cc_printf_like(6, 7);
351cc_noreturn void enforce_impl_code(const char *file, int line, const char *func,
352 const char *prefix, const char *expr,
353 const struct crash_code *code,
354 const char *fmt, ...) cc_printf_like(7, 8);
355cc_noreturn void enforce_impl_code_only(const char *file, int line,
356 const char *func, const char *prefix,
357 const char *expr,
358 const struct crash_code *code);
359
360cc_noreturn void crash_full(const struct crash_context *ctx);
361
362bool crash_cpu_is_owner(uint64_t id);
363void crash_broadcast_nmi(void);
364size_t crash_code_name(const struct crash_code *code, char *buf, size_t len);
365void crash_nmi_handoff(void *p, struct irq_registers *irqc);
366void debug_print_stack(void);
367void crash_facility_printf(const char *fmt, ...) cc_printf_like(1, 2);
368void crash_facility_line(const char *fmt, ...) cc_printf_like(1, 2);
369
370static inline bool crash_code_in_facility(const struct crash_code *code,
371 const struct crash_facility *f) {
372 return code && code->facility == f;
373}
374
375#ifdef DEBUG_CRASH
376
377void crash_thread_init(struct thread *t);
378void crash_unwind(void);
379
380/* Enter/exit pairs */
381void crash_unwind_enter_rcu(void);
382void crash_unwind_exit_rcu(void);
383void crash_unwind_enter_mutex(struct mutex *m);
384void crash_unwind_exit_mutex(struct mutex *m);
385void crash_unwind_enter_rwlock(struct rwlock *r);
386void crash_unwind_exit_rwlock(struct rwlock *r);
387void crash_unwind_enter_spinlock(struct spinlock *s, enum irql old);
388void crash_unwind_exit_spinlock(struct spinlock *s);
389void crash_unwind_enter_qspinlock(struct qspinlock *q, enum irql old);
390void crash_unwind_exit_qspinlock(struct qspinlock *q);
391
392#else
393/* Empty static inline versions that cc_ignore the parameters */
394
395static inline void crash_thread_init(struct thread *t) {
396 cc_ignore(t);
397}
398
399static inline void crash_unwind(void) {}
400
401static inline void crash_unwind_enter_rcu(void) {}
402static inline void crash_unwind_exit_rcu(void) {}
403
404static inline void crash_unwind_enter_mutex(struct mutex *m) {
405 cc_ignore(m);
406}
407
408static inline void crash_unwind_exit_mutex(struct mutex *m) {
409 cc_ignore(m);
410}
411
412static inline void crash_unwind_enter_rwlock(struct rwlock *r) {
413 cc_ignore(r);
414}
415
416static inline void crash_unwind_exit_rwlock(struct rwlock *r) {
417 cc_ignore(r);
418}
419
420static inline void crash_unwind_enter_spinlock(struct spinlock *s,
421 enum irql old) {
422 cc_ignore(s, old);
423}
424
425static inline void crash_unwind_exit_spinlock(struct spinlock *s) {
426 cc_ignore(s);
427}
428
429static inline void crash_unwind_enter_qspinlock(struct qspinlock *q,
430 enum irql old) {
431 cc_ignore(q, old);
432}
433
434static inline void crash_unwind_exit_qspinlock(struct qspinlock *q) {
435 cc_ignore(q);
436}
437
438#endif
439
440static inline void qemu_exit(int code) {
441 outb(port: 0xf4, value: (uint8_t) code);
442}
443