1#include <compiler/intrinsic.h>
2#include <console/crash.h>
3#include <console/term.h>
4#include <dbg.h>
5#include <enforce.h>
6#include <global.h>
7#include <math/align.h>
8#include <math/min_max.h>
9#include <mem/asan.h>
10#include <mem/hhdm.h>
11#include <mem/page.h>
12#include <mem/pmm.h>
13#include <mem/vmm.h>
14#include <ndjson.h>
15#include <stack_depot.h>
16#include <stdint.h>
17#include <string.h>
18#include <sync/spinlock.h>
19
20#include "mem/slab/internal.h"
21
22#ifdef DEBUG_ASAN
23LOG_SITE_DEFINE_PRINT(asan);
24LOG_HANDLE_DEFINE_PRINT(asan);
25
26/* If we widen this, ASAN_SHADOW_OFFSET must be recomputed, because right now
27 * it maps exactly this window into the shadow region */
28#define ASAN_COVERED_START SLAB_HEAP_START
29#define ASAN_COVERED_END SLAB_HEAP_END
30
31static bool asan_ready = false;
32
33static cc_noreturn void asan_runtime_panic(const char *what, const void *addr,
34 size_t size, const char *file,
35 int line, const char *func);
36
37#define kasan_panic(what, addr, size) \
38 asan_runtime_panic((what), (addr), (size), __FILE__, __LINE__, __func__)
39
40/* Two pages that everything aliases to for pages that are fully OK/not OK to
41 * read/modify */
42static paddr_t asan_zero_shadow_phys; /* never had a chunk: reads accessible */
43static paddr_t asan_freed_shadow_phys; /* had one, gave it back: reads freed */
44
45static inline bool asan_addr_covered(uintptr_t a, size_t size) {
46 return a >= ASAN_COVERED_START && (a + size) <= ASAN_COVERED_END;
47}
48
49static inline uint8_t *asan_shadow_for_internal(const void *addr) {
50 return (uint8_t *) ASAN_SHADOW_ADDR(addr);
51}
52
53/* Number of shadow bytes covering [addr, addr + size) */
54static inline size_t asan_shadow_span(const void *addr, size_t size) {
55 const uint8_t *end = (const uint8_t *) addr + size - 1;
56 return (size_t) (asan_shadow_for_internal(end) -
57 asan_shadow_for_internal(addr)) +
58 1;
59}
60
61static void asan_mark_valid(void *addr, size_t size) {
62 enforce(IS_ALIGNED((uintptr_t) addr, ASAN_GRANULE));
63
64 uint8_t *shadow = asan_shadow_for_internal(addr);
65 size_t full = size >> ASAN_SHADOW_SCALE;
66 size_t tail = size & (ASAN_GRANULE - 1);
67
68 memset(shadow, 0, full);
69 if (tail)
70 shadow[full] = (uint8_t) tail;
71}
72
73static void asan_mark_poisoned(void *addr, size_t size, uint8_t value) {
74 enforce(IS_ALIGNED((uintptr_t) addr, ASAN_GRANULE));
75
76 memset(asan_shadow_for_internal(addr), value, asan_shadow_span(addr, size));
77}
78
79void asan_alloc(void *addr, size_t requested, size_t slot) {
80 ASAN_ABORT_IF_NOT_READY();
81
82 if (!slot || !asan_addr_covered((uintptr_t) addr, slot))
83 return;
84
85 enforce(requested && requested <= slot);
86
87 asan_mark_valid(addr, requested);
88
89 size_t redzone = ALIGN_UP(requested, ASAN_GRANULE);
90 if (redzone < slot)
91 asan_mark_poisoned((uint8_t *) addr + redzone, slot - redzone,
92 ASAN_POISON_HEAP_REDZONE);
93}
94
95void asan_free(void *addr, size_t slot) {
96 ASAN_ABORT_IF_NOT_READY();
97
98 if (!slot || !asan_addr_covered((uintptr_t) addr, slot))
99 return;
100
101 asan_mark_poisoned(addr, slot, ASAN_POISON_HEAP_FREED);
102}
103
104void asan_poison(void *addr, size_t size) {
105 ASAN_ABORT_IF_NOT_READY();
106
107 if (!size || !asan_addr_covered((uintptr_t) addr, size))
108 return;
109
110 asan_mark_poisoned(addr, size, ASAN_POISON_VALUE);
111}
112
113void asan_unpoison(void *addr, size_t size) {
114 ASAN_ABORT_IF_NOT_READY();
115
116 if (!size || !asan_addr_covered((uintptr_t) addr, size))
117 return;
118
119 asan_mark_valid(addr, size);
120}
121
122static inline bool asan_shadow_is_shared(vaddr_t shadow_page) {
123 paddr_t p = vmm_get_phys(shadow_page);
124 return p == asan_zero_shadow_phys || p == asan_freed_shadow_phys;
125}
126
127static bool asan_page_is_uniform(const void *addr, uint8_t value) {
128 const uint64_t *p = addr;
129 uint64_t want = value * 0x0101010101010101ULL;
130
131 for (size_t i = 0; i < PAGE_SIZE / sizeof(uint64_t); i++) {
132 if (p[i] != want)
133 return false;
134 }
135
136 return true;
137}
138
139static inline vaddr_t asan_shadow_page_first(vaddr_t base) {
140 return ALIGN_DOWN(ASAN_SHADOW_ADDR(base), PAGE_SIZE);
141}
142
143static inline vaddr_t asan_shadow_page_limit(vaddr_t base, size_t len) {
144 return ALIGN_UP(ASAN_SHADOW_ADDR(base + len - 1) + 1, PAGE_SIZE);
145}
146
147enum err asan_shadow_install(vaddr_t base, size_t len) {
148 if (!asan_ready)
149 return ERR_OK;
150
151 if (!len || !asan_addr_covered(base, len))
152 return ERR_OK;
153
154 vaddr_t limit = asan_shadow_page_limit(base, len);
155
156 for (vaddr_t v = asan_shadow_page_first(base); v < limit; v += PAGE_SIZE) {
157 if (!asan_shadow_is_shared(v))
158 continue;
159
160 /* Per page rather than once up front, so a range of any length works:
161 * the walk is cheap and only happens for a page we are about to back */
162 enum err err = vmm_unshare_path(v);
163 if (err < 0)
164 return err;
165
166 paddr_t phys = pmm_alloc_page();
167 if (!phys)
168 return ERR_NO_MEM;
169
170 err = vmm_map_page(v, phys,
171 .page_flags = PAGE_PRESENT | PAGE_WRITE | PAGE_XD,
172 .vmm_flags = VMM_FLAG_MODIFY_LEAF);
173 if (err < 0) {
174 pmm_free_page(phys);
175 return err;
176 }
177
178 memset((void *) v, ASAN_POISON_VALUE, PAGE_SIZE);
179 }
180
181 return ERR_OK;
182}
183
184void asan_shadow_release(vaddr_t base, size_t len) {
185 if (!asan_ready)
186 return;
187
188 if (!len || !asan_addr_covered(base, len))
189 return;
190
191 vaddr_t limit = asan_shadow_page_limit(base, len);
192
193 for (vaddr_t v = asan_shadow_page_first(base); v < limit; v += PAGE_SIZE) {
194 if (asan_shadow_is_shared(v))
195 continue;
196
197 if (!asan_page_is_uniform((void *) v, ASAN_POISON_HEAP_FREED))
198 continue;
199
200 paddr_t phys = vmm_get_phys(v);
201
202 if (vmm_map_page(v, asan_freed_shadow_phys,
203 .page_flags = PAGE_PRESENT | PAGE_XD,
204 .vmm_flags = VMM_FLAG_MODIFY_LEAF) < 0)
205 continue;
206
207 pmm_free_page(phys);
208 }
209}
210
211static void asan_map_early_shadow(void) {
212 vaddr_t start = ASAN_SHADOW_ADDR(ASAN_COVERED_START);
213 vaddr_t end = ASAN_SHADOW_ADDR(ASAN_COVERED_END);
214
215 asan_zero_shadow_phys = pmm_alloc_page(ALLOC_FLAGS_ZERO);
216 if (!asan_zero_shadow_phys)
217 kasan_panic("could not allocate the shared shadow page", NULL, 0);
218
219 asan_freed_shadow_phys = pmm_alloc_page();
220 if (!asan_freed_shadow_phys)
221 kasan_panic("could not allocate the shared freed shadow page", NULL, 0);
222
223 memset(hhdm_paddr_to_ptr(asan_freed_shadow_phys), ASAN_POISON_HEAP_FREED,
224 PAGE_SIZE);
225
226 /* Deliberately RO and aliased */
227 enum err err = vmm_map_aliased(start, end - start, asan_zero_shadow_phys,
228 .page_flags = PAGE_PRESENT | PAGE_XD);
229 if (err < 0)
230 kasan_panic("could not map the shared shadow window",
231 (const void *) start, end - start);
232
233 asan_info("shared shadow: [%lx, %lx), %zu GiB of window on one zero page "
234 "at %lx\n",
235 start, end, (size_t) ((end - start) / GIB(1)),
236 asan_zero_shadow_phys);
237}
238
239void asan_init(void) {
240 asan_info("bringing ASAN up...");
241
242 asan_map_early_shadow();
243
244 asan_ready = true;
245}
246
247NDJSON_DEFINE(asan_fault, NDJSON_SECTION_ASAN, NDJSON_KIND_FAULT, 1,
248 NDJSON_STR(what), NDJSON_HEX(addr), NDJSON_U64(size),
249 NDJSON_STR(access));
250
251NDJSON_DEFINE(asan_frame, NDJSON_SECTION_ASAN, NDJSON_KIND_FRAME, 1,
252 NDJSON_U64(idx), NDJSON_HEX(addr), NDJSON_STR(sym),
253 NDJSON_U64(off));
254
255#define CRASH_CODES_asan \
256 (USE_AFTER_FREE, "Use after free"), \
257 (HEAP_REDZONE, "Heap out of bounds (redzone)"), \
258 (UNALLOCATED, "Access to unallocated memory"), \
259 (PARTIAL_GRANULE, "Heap overflow (partial granule)"), \
260 (POISONED, "Access to poisoned memory"), \
261 (UNCLASSIFIED, "Bad memory access"), (RUNTIME, "ASAN runtime failure")
262CRASH_CODES_DECLARE(asan);
263
264#define ASAN_REPORT_SHADOW_ROWS 3
265#define ASAN_REPORT_SHADOW_COLS 8 /* fits the facility box at 80 columns */
266#define ASAN_REPORT_NOTE_MAX 128
267
268struct asan_report {
269 const void *addr;
270 size_t size;
271 bool is_write;
272 bool has_shadow;
273 uint8_t shadow[ASAN_REPORT_SHADOW_ROWS][ASAN_REPORT_SHADOW_COLS];
274 char owner[ASAN_REPORT_NOTE_MAX]; /* empty = not a slab object */
275 stack_handle_t owner_trace;
276 struct slab *owner_slab; /* NULL until the object is pinned down */
277 size_t owner_idx;
278};
279
280static const struct crash_code *asan_poison_code(uint8_t shadow) {
281 switch (shadow) {
282 case ASAN_POISON_HEAP_FREED: return CRASH_CODE(asan, USE_AFTER_FREE);
283 case ASAN_POISON_HEAP_REDZONE: return CRASH_CODE(asan, HEAP_REDZONE);
284 case ASAN_POISON_VALUE: return CRASH_CODE(asan, UNALLOCATED);
285 default:
286 return asan_shadow_is_poison(shadow)
287 ? CRASH_CODE(asan, POISONED)
288 : CRASH_CODE(asan, PARTIAL_GRANULE);
289 }
290}
291
292static void asan_crash_dump(struct crash_payload pl);
293
294CRASH_FACILITY_DEFINE(asan, "Kernel address sanitizer",
295 .dump = asan_crash_dump);
296
297static void asan_dump_shadow(const struct asan_report *r) {
298 char row[256]; /* room for a color escape around every byte */
299
300 crash_facility_line("%sshadow bytes%s", term_style(TERM_SEV_LABEL),
301 term_style_reset());
302
303 for (int i = 0; i < ASAN_REPORT_SHADOW_ROWS; i++) {
304 int rel = i - ASAN_REPORT_SHADOW_ROWS / 2;
305 size_t n = 0;
306
307 for (int j = 0; j < ASAN_REPORT_SHADOW_COLS; j++) {
308 bool fault = rel == 0 && j == 0;
309 uint8_t b = r->shadow[i][j];
310 const char *style = fault ? term_style(TERM_SEV_CRIT)
311 : b ? term_style(TERM_SEV_WARN)
312 : term_style(TERM_SEV_DIM);
313
314 n += (size_t) snprintf(row + n, (int) (sizeof(row) - n),
315 "%s%02x%s ", style, b, term_style_reset());
316 if (n >= sizeof(row))
317 break;
318 }
319
320 char off[16];
321 snprintf(off, (int) sizeof(off), "%s%d", rel > 0 ? "+" : "",
322 rel * ASAN_REPORT_SHADOW_COLS * (int) ASAN_GRANULE);
323 crash_facility_line("%s%6s%s %s", term_style(TERM_SEV_DIM), off,
324 term_style_reset(), row);
325 }
326}
327
328static void asan_dump_trace(stack_handle_t h) {
329 struct stack_depot_record *rec = stack_depot_get_record(h);
330
331 if (!rec) {
332 crash_facility_line(" %s<trace no longer in the depot>%s",
333 term_style(TERM_SEV_DIM), term_style_reset());
334 return;
335 }
336
337 for (size_t i = 0; i < rec->num_entries; i++) {
338 uint64_t off = 0;
339 const char *sym = debug_symbolize(rec->entries[i], &off);
340
341 if (sym)
342 crash_facility_line(" %s#%-2zu%s %s+0x%lx",
343 term_style(TERM_SEV_DIM), i, term_style_reset(),
344 sym, off);
345 else
346 crash_facility_line(" %s#%-2zu%s %016lx", term_style(TERM_SEV_DIM),
347 i, term_style_reset(), rec->entries[i]);
348 }
349}
350
351static void asan_crash_dump(struct crash_payload pl) {
352 const struct asan_report *r = pl.data;
353
354 if (!r)
355 return;
356
357 crash_facility_line("%saddress%s %p", term_style(TERM_SEV_LABEL),
358 term_style_reset(), r->addr);
359
360 if (pl.code != CRASH_CODE(asan, RUNTIME))
361 crash_facility_line("%saccess%s %zu-byte %s",
362 term_style(TERM_SEV_LABEL), term_style_reset(),
363 r->size, r->is_write ? "store" : "load");
364 else if (r->size)
365 crash_facility_line("%ssize%s 0x%zx", term_style(TERM_SEV_LABEL),
366 term_style_reset(), r->size);
367
368 if (r->has_shadow) {
369 crash_facility_line("");
370 asan_dump_shadow(r);
371 }
372
373 if (r->owner[0]) {
374 crash_facility_line("");
375 crash_facility_printf("%sowner%s %s", term_style(TERM_SEV_LABEL),
376 term_style_reset(), r->owner);
377 if (r->owner_trace)
378 asan_dump_trace(r->owner_trace);
379 }
380}
381
382static void asan_collect_shadow(struct asan_report *r) {
383 if (!asan_ready || !asan_addr_covered((uintptr_t) r->addr, 1))
384 return;
385
386 const uint8_t *sh = asan_shadow_for_internal(r->addr);
387 int mid = ASAN_REPORT_SHADOW_ROWS / 2;
388
389 for (int i = 0; i < ASAN_REPORT_SHADOW_ROWS; i++)
390 for (int j = 0; j < ASAN_REPORT_SHADOW_COLS; j++)
391 r->shadow[i][j] = sh[(i - mid) * ASAN_REPORT_SHADOW_COLS + j];
392
393 r->has_shadow = true;
394}
395
396static void asan_emit_trace(stack_handle_t h) {
397 struct stack_depot_record *rec = stack_depot_get_record(h);
398
399 if (!rec)
400 return;
401
402 for (size_t i = 0; i < rec->num_entries; i++) {
403 uint64_t off = 0;
404 const char *sym = debug_symbolize(rec->entries[i], &off);
405
406 ndjson_emit(asan_frame, .idx = i, .addr = rec->entries[i], .sym = sym,
407 .off = off);
408 }
409}
410
411#define asan_owner_note(r, ...) \
412 snprintf((r)->owner, sizeof((r)->owner), __VA_ARGS__)
413
414/* Slabs keep a stack depot handle per object when DEBUG_SLAB_DEEP, so
415 * we must tread carefully when retrieving the backtrace, lest we
416 * dereference an unmapped page or cause other problems */
417static void asan_collect_owner(struct asan_report *r) {
418#ifdef DEBUG_SLAB_DEEP
419 static bool reporting;
420 void *addr = (void *) r->addr;
421
422 if (reporting || !slab_ptr_in_slab(addr))
423 return;
424 reporting = true;
425
426 if (slab_order_map_get((vaddr_t) addr) == SLAB_POW2_ORDER_EMPTY) {
427 asan_owner_note(r, "in the slab heap but no chunk is mapped there");
428 goto out;
429 }
430
431 struct slab *s = slab_for_ptr(addr);
432 struct slab_cache *cache = s->parent_cache;
433
434 /* Metadata may be what went wrong, so sanity check before trusting it */
435 if ((uintptr_t) cache < SLAB_HEAP_START || !cache->obj_stride ||
436 !cache->objs_per_slab || s->mem < (vaddr_t) s ||
437 (vaddr_t) addr < s->mem) {
438 asan_owner_note(r, "slab metadata at %p is not usable (cache=%p)",
439 (void *) s, (void *) cache);
440 goto out;
441 }
442
443 size_t idx = slab_allocation_index(s, addr);
444 if (idx >= cache->objs_per_slab) {
445 asan_owner_note(r, "object index %zu out of range for slab %p", idx,
446 (void *) s);
447 goto out;
448 }
449
450 stack_handle_t h = s->traces[idx];
451 if (!h) {
452 asan_owner_note(r, "object %zu of slab %p has no owner", idx,
453 (void *) s);
454 goto out;
455 }
456
457 asan_owner_note(r, "object %zu of slab %p, last allocated at:", idx,
458 (void *) s);
459 r->owner_slab = s;
460 r->owner_idx = idx;
461 r->owner_trace = h;
462 asan_emit_trace(h);
463
464out:
465 reporting = false;
466#else
467 cc_ignore(r);
468#endif
469}
470
471static cc_noreturn void asan_crash(const struct crash_code *code,
472 const char *msg, struct asan_report *r,
473 const char *file, int line,
474 const char *func) {
475 crash_full(&(struct crash_context){
476 .payload = CRASH_PAYLOAD(code, r),
477 .source = CRASH_SOURCE_KASAN,
478 .formats = CRASH_FMT_DEFAULT,
479 .file = file,
480 .line = line,
481 .func = func,
482 .msg = msg,
483 });
484}
485
486static cc_noreturn void asan_runtime_panic(const char *what, const void *addr,
487 size_t size, const char *file,
488 int line, const char *func) {
489 struct asan_report r = {.addr = addr, .size = size};
490
491 ndjson_emit(asan_fault, .what = what, .addr = (uint64_t) (uintptr_t) addr,
492 .size = size, .access = "");
493 asan_crash(CRASH_CODE(asan, RUNTIME), what, &r, file, line, func);
494}
495
496/* First shadow byte that makes [addr, addr + size) bad, 0 if all of it is OK */
497static inline uint8_t asan_bad_shadow(const void *addr, size_t size) {
498 if (size == 0 || !asan_addr_covered((uintptr_t) addr, size))
499 return 0;
500
501 uintptr_t start = (uintptr_t) addr;
502 uintptr_t last = start + (size - 1);
503
504 for (uintptr_t g = ALIGN_DOWN(start, ASAN_GRANULE); g <= last;
505 g += ASAN_GRANULE) {
506 uint8_t s = *asan_shadow_for_internal((const void *) g);
507 if (s == 0)
508 continue;
509
510 /* Bytes of this granule the access actually touches: [lo, hi) */
511 uintptr_t hi = MIN(last - g + 1, ASAN_GRANULE);
512
513 /* A partial granule keeps its first `s` bytes accessible */
514 if (!asan_shadow_is_poison(s) && hi <= s)
515 continue;
516
517 return s;
518 }
519
520 return 0;
521}
522
523static cc_noreturn void asan_report_access(const void *addr, size_t size,
524 bool is_write, void *pc) {
525 uint8_t s = asan_ready ? asan_bad_shadow(addr, size) : 0;
526 const struct crash_code *code =
527 s ? asan_poison_code(s) : CRASH_CODE(asan, UNCLASSIFIED);
528 const char *access = is_write ? "store" : "load";
529 struct asan_report r = {.addr = addr, .size = size, .is_write = is_write};
530 char msg[CRASH_MSG_MAX];
531
532 ndjson_emit(asan_fault, .what = code->desc,
533 .addr = (uint64_t) (uintptr_t) addr, .size = size,
534 .access = access);
535
536 asan_collect_shadow(&r);
537 asan_collect_owner(&r);
538
539 int n = snprintf(msg, sizeof(msg), "%zu-byte %s at %p", size, access, addr);
540 if (r.owner_slab && n > 0 && (size_t) n < sizeof(msg))
541 snprintf(msg + n, sizeof(msg) - (size_t) n, " (object %zu of slab %p)",
542 r.owner_idx, (void *) r.owner_slab);
543
544 uint64_t site = (uint64_t) (uintptr_t) pc - 1;
545 uint64_t off = 0;
546 uint32_t line = 0;
547 const char *func = debug_symbolize(site, &off);
548 const char *file = debug_line_for(site, &line);
549
550 asan_crash(code, msg, &r, file, (int) line, func);
551}
552
553static inline void asan_check_access_core(const void *addr, size_t size,
554 bool is_write, void *pc) {
555 ASAN_ABORT_IF_NOT_READY();
556
557 if (asan_bad_shadow(addr, size))
558 asan_report_access(addr, size, is_write, pc);
559}
560
561void __asan_load1(const void *addr) {
562 asan_check_access_core(addr, 1, false, ci_return_address(0));
563}
564void __asan_load2(const void *addr) {
565 asan_check_access_core(addr, 2, false, ci_return_address(0));
566}
567void __asan_load4(const void *addr) {
568 asan_check_access_core(addr, 4, false, ci_return_address(0));
569}
570void __asan_load8(const void *addr) {
571 asan_check_access_core(addr, 8, false, ci_return_address(0));
572}
573void __asan_load16(const void *addr) {
574 asan_check_access_core(addr, 16, false, ci_return_address(0));
575} /* some compilers */
576
577void __asan_store1(const void *addr) {
578 asan_check_access_core(addr, 1, true, ci_return_address(0));
579}
580void __asan_store2(const void *addr) {
581 asan_check_access_core(addr, 2, true, ci_return_address(0));
582}
583void __asan_store4(const void *addr) {
584 asan_check_access_core(addr, 4, true, ci_return_address(0));
585}
586void __asan_store8(const void *addr) {
587 asan_check_access_core(addr, 8, true, ci_return_address(0));
588}
589void __asan_store16(const void *addr) {
590 asan_check_access_core(addr, 16, true, ci_return_address(0));
591}
592
593/* Generic wrappers the compiler sometimes uses */
594void __asan_loadN(const void *addr, size_t size) {
595 asan_check_access_core(addr, size, false, ci_return_address(0));
596}
597void __asan_storeN(const void *addr, size_t size) {
598 asan_check_access_core(addr, size, true, ci_return_address(0));
599}
600
601/* The compiler's spelling of the same two operations */
602void __asan_poison_memory_region(void *addr, size_t size) {
603 asan_poison(addr, size);
604}
605
606void __asan_unpoison_memory_region(void *addr, size_t size) {
607 asan_unpoison(addr, size);
608}
609
610struct __asan_global {
611 void *addr;
612 size_t size;
613 const char *name;
614 /* some targets include more fields; we ignore them */
615};
616
617void __asan_register_globals(struct __asan_global *globals, size_t n) {
618 ASAN_ABORT_IF_NOT_READY();
619 const size_t redzone = 16;
620 for (size_t i = 0; i < n; i++) {
621 void *addr = globals[i].addr;
622 size_t size = globals[i].size;
623 if (!addr || size == 0)
624 continue;
625
626 /* poison left redzone (if address is valid) */
627 if ((uintptr_t) addr >= redzone) /* simple check */
628 __asan_poison_memory_region((uint8_t *) addr - redzone, redzone);
629 __asan_unpoison_memory_region(addr, size);
630 __asan_poison_memory_region((uint8_t *) addr + size, redzone);
631 }
632}
633
634void __asan_unregister_globals(void *globals, size_t n) {
635 cc_ignore(globals, n);
636}
637
638#define ASAN_MAX_STACK_RECORDS 1024
639struct stack_record {
640 void *addr;
641 size_t size;
642};
643
644static struct stack_record stack_records[ASAN_MAX_STACK_RECORDS];
645static size_t stack_records_count = 0;
646
647void __asan_stack_malloc(void *addr, size_t size) {
648 ASAN_ABORT_IF_NOT_READY();
649 if (!addr || size == 0)
650 return;
651 const size_t rz = 16;
652 __asan_poison_memory_region((uint8_t *) addr - rz, size + rz * 2);
653 __asan_unpoison_memory_region(addr, size);
654
655 if (stack_records_count < ASAN_MAX_STACK_RECORDS) {
656 stack_records[stack_records_count].addr = addr;
657 stack_records[stack_records_count].size = size;
658 stack_records_count++;
659 }
660}
661
662void __asan_stack_free(void *addr) {
663 ASAN_ABORT_IF_NOT_READY();
664 for (size_t i = 0; i < stack_records_count; i++) {
665 if (stack_records[i].addr == addr) {
666 size_t size = stack_records[i].size;
667 const size_t rz = 16;
668 __asan_poison_memory_region((uint8_t *) addr - rz, size + rz * 2);
669 /* shift tail down */
670 stack_records[i] = stack_records[stack_records_count - 1];
671 stack_records_count--;
672 return;
673 }
674 }
675}
676
677void __asan_report_load1(void *addr) {
678 asan_report_access(addr, 1, false, ci_return_address(0));
679}
680void __asan_report_load2(void *addr) {
681 asan_report_access(addr, 2, false, ci_return_address(0));
682}
683void __asan_report_load4(void *addr) {
684 asan_report_access(addr, 4, false, ci_return_address(0));
685}
686void __asan_report_load8(void *addr) {
687 asan_report_access(addr, 8, false, ci_return_address(0));
688}
689void __asan_report_load16(void *addr) {
690 asan_report_access(addr, 16, false, ci_return_address(0));
691}
692
693void __asan_report_load32(void *addr) {
694 asan_report_access(addr, 32, false, ci_return_address(0));
695}
696
697void __asan_report_load64(void *addr) {
698 asan_report_access(addr, 64, false, ci_return_address(0));
699}
700
701void __asan_report_load_n(void *addr, size_t size) {
702 asan_report_access(addr, size, false, ci_return_address(0));
703}
704
705void __asan_report_store1(void *addr) {
706 asan_report_access(addr, 1, true, ci_return_address(0));
707}
708void __asan_report_store2(void *addr) {
709 asan_report_access(addr, 2, true, ci_return_address(0));
710}
711void __asan_report_store4(void *addr) {
712 asan_report_access(addr, 4, true, ci_return_address(0));
713}
714void __asan_report_store8(void *addr) {
715 asan_report_access(addr, 8, true, ci_return_address(0));
716}
717void __asan_report_store16(void *addr) {
718 asan_report_access(addr, 16, true, ci_return_address(0));
719}
720
721void __asan_report_store32(void *addr) {
722 asan_report_access(addr, 32, true, ci_return_address(0));
723}
724
725void __asan_report_store64(void *addr) {
726 asan_report_access(addr, 64, true, ci_return_address(0));
727}
728
729void __asan_report_store_n(void *addr, size_t size) {
730 asan_report_access(addr, size, true, ci_return_address(0));
731}
732
733int __asan_option_detect_stack_use_after_return = 0;
734
735void *__asan_stack_malloc_0(size_t size) {
736 cc_ignore(size);
737 return NULL;
738}
739void *__asan_stack_malloc_1(size_t size) {
740 cc_ignore(size);
741 return NULL;
742}
743void *__asan_stack_malloc_2(size_t size) {
744 cc_ignore(size);
745 return NULL;
746}
747void *__asan_stack_malloc_3(size_t size) {
748 cc_ignore(size);
749 return NULL;
750}
751void *__asan_stack_malloc_4(size_t size) {
752 cc_ignore(size);
753 return NULL;
754}
755void *__asan_stack_malloc_5(size_t size) {
756 cc_ignore(size);
757 return NULL;
758}
759void *__asan_stack_malloc_6(size_t size) {
760 cc_ignore(size);
761 return NULL;
762}
763void *__asan_stack_malloc_7(size_t size) {
764 cc_ignore(size);
765 return NULL;
766}
767void *__asan_stack_malloc_8(size_t size) {
768 cc_ignore(size);
769 return NULL;
770}
771void *__asan_stack_malloc_9(size_t size) {
772 cc_ignore(size);
773 return NULL;
774}
775
776void __asan_stack_free_0(void *p, size_t size) {
777 cc_ignore(p, size);
778}
779void __asan_stack_free_1(void *p, size_t size) {
780 cc_ignore(p, size);
781}
782void __asan_stack_free_2(void *p, size_t size) {
783 cc_ignore(p, size);
784}
785void __asan_stack_free_3(void *p, size_t size) {
786 cc_ignore(p, size);
787}
788void __asan_stack_free_4(void *p, size_t size) {
789 cc_ignore(p, size);
790}
791void __asan_stack_free_5(void *p, size_t size) {
792 cc_ignore(p, size);
793}
794void __asan_stack_free_6(void *p, size_t size) {
795 cc_ignore(p, size);
796}
797void __asan_stack_free_7(void *p, size_t size) {
798 cc_ignore(p, size);
799}
800void __asan_stack_free_8(void *p, size_t size) {
801 cc_ignore(p, size);
802}
803void __asan_stack_free_9(void *p, size_t size) {
804 cc_ignore(p, size);
805}
806
807void *__asan_malloc(size_t size) {
808 /* Not relevant for kernel: just panic if somehow called */
809 kasan_panic("__asan_malloc called", NULL, size);
810}
811
812void __asan_free(void *p) {
813 kasan_panic("__asan_free called", p, 0);
814}
815
816void __asan_malloc_hook(void *ptr, size_t size) {
817 cc_ignore(ptr, size);
818}
819void __asan_free_hook(void *ptr) {
820 cc_ignore(ptr);
821}
822
823void __asan_init(void) {
824 asan_info("__asan_init runtime stub called");
825}
826
827void __asan_before_dynamic_init(const char *module_name) {
828 cc_ignore(module_name);
829}
830void __asan_after_dynamic_init(void) {}
831
832/* Compiler sometimes emits these for non-instrumented copies. */
833void *__asan_memcpy(void *dst, const void *src, size_t n) {
834 return memcpy(dst, src, n);
835}
836void *__asan_memmove(void *dst, const void *src, size_t n) {
837 return memmove(dst, src, n);
838}
839void *__asan_memset(void *s, int c, size_t n) {
840 return memset(s, c, n);
841}
842
843/* Some compilers also expect these “weak” entrypoints */
844void __asan_handle_no_return(void) {}
845void __asan_after_load(void) {}
846void __asan_after_store(void) {}
847void __asan_before_memory_access(void) {}
848void __asan_after_memory_access(void) {}
849
850/* Optional runtime interface for global poisoning/unpoisoning */
851void __asan_set_shadow_00_to_0x00(void) {}
852void __asan_set_shadow_f8_to_0x00(void) {}
853
854void __asan_alloca_poison(void *addr, size_t size) {
855 ASAN_ABORT_IF_NOT_READY();
856 if (!addr || size == 0)
857 return;
858 __asan_poison_memory_region(addr, size);
859}
860
861void __asan_allocas_unpoison(void *addr, size_t size) {
862 ASAN_ABORT_IF_NOT_READY();
863 if (!addr || size == 0)
864 return;
865 __asan_unpoison_memory_region(addr, size);
866}
867
868cc_weak void __asan_alloca_poison_0(void *addr, size_t size) {
869 __asan_alloca_poison(addr, size);
870}
871
872cc_weak void __asan_allocas_unpoison_0(void *addr, size_t size) {
873 __asan_allocas_unpoison(addr, size);
874}
875
876#define ASAN_ALIAS(name, target) cc_alias(target) void name
877
878/* Outline callbacks (forced via -asan-instrumentation-with-call-threshold=0).
879 */
880ASAN_ALIAS(__asan_load1_noabort, __asan_load1)(const void *addr);
881ASAN_ALIAS(__asan_load2_noabort, __asan_load2)(const void *addr);
882ASAN_ALIAS(__asan_load4_noabort, __asan_load4)(const void *addr);
883ASAN_ALIAS(__asan_load8_noabort, __asan_load8)(const void *addr);
884ASAN_ALIAS(__asan_load16_noabort, __asan_load16)(const void *addr);
885ASAN_ALIAS(__asan_store1_noabort, __asan_store1)(const void *addr);
886ASAN_ALIAS(__asan_store2_noabort, __asan_store2)(const void *addr);
887ASAN_ALIAS(__asan_store4_noabort, __asan_store4)(const void *addr);
888ASAN_ALIAS(__asan_store8_noabort, __asan_store8)(const void *addr);
889ASAN_ALIAS(__asan_store16_noabort, __asan_store16)(const void *addr);
890ASAN_ALIAS(__asan_loadN_noabort, __asan_loadN)(const void *addr, size_t size);
891ASAN_ALIAS(__asan_storeN_noabort, __asan_storeN)(const void *addr, size_t size);
892
893ASAN_ALIAS(__asan_report_load1_noabort, __asan_report_load1)(void *addr);
894ASAN_ALIAS(__asan_report_load2_noabort, __asan_report_load2)(void *addr);
895ASAN_ALIAS(__asan_report_load4_noabort, __asan_report_load4)(void *addr);
896ASAN_ALIAS(__asan_report_load8_noabort, __asan_report_load8)(void *addr);
897ASAN_ALIAS(__asan_report_load16_noabort, __asan_report_load16)(void *addr);
898ASAN_ALIAS(__asan_report_store1_noabort, __asan_report_store1)(void *addr);
899ASAN_ALIAS(__asan_report_store2_noabort, __asan_report_store2)(void *addr);
900ASAN_ALIAS(__asan_report_store4_noabort, __asan_report_store4)(void *addr);
901ASAN_ALIAS(__asan_report_store8_noabort, __asan_report_store8)(void *addr);
902ASAN_ALIAS(__asan_report_store16_noabort, __asan_report_store16)(void *addr);
903ASAN_ALIAS(__asan_report_load_n_noabort, __asan_report_load_n)(void *addr,
904 size_t size);
905ASAN_ALIAS(__asan_report_store_n_noabort, __asan_report_store_n)(void *addr,
906 size_t size);
907#endif
908