1#include "internal.h"
2#include <mem/address_range.h>
3
4#ifdef DEBUG_SLAB_DEEP
5static const char *slab_addr_region(vaddr_t a) {
6 if (address_range_for_addr(a))
7 return address_range_for_addr(a)->name;
8
9 return "none";
10}
11
12#define SLAB_TRACK_SLOTS 16384
13struct slab_track {
14 vaddr_t addr;
15 uint64_t alloc_ra[2];
16 uint64_t free_ra[2];
17 uint32_t seq;
18 uint8_t live;
19};
20static struct slab_track slab_track_table[SLAB_TRACK_SLOTS];
21static atomic_uint32_t slab_track_seq;
22
23void slab_track_event(vaddr_t addr, uint64_t ra0, uint64_t ra1, bool is_alloc) {
24 struct slab_track *e = &slab_track_table[(addr >> 6) % SLAB_TRACK_SLOTS];
25 e->addr = addr;
26 e->seq = atomic_fetch_add_rlx(&slab_track_seq, 1);
27 if (is_alloc) {
28 e->alloc_ra[0] = ra0;
29 e->alloc_ra[1] = ra1;
30 e->live = 1;
31 } else {
32 e->free_ra[0] = ra0;
33 e->free_ra[1] = ra1;
34 e->live = 0;
35 }
36}
37
38void slab_track_dump(const char *label, vaddr_t addr) {
39 struct slab_track *e = &slab_track_table[(addr >> 6) % SLAB_TRACK_SLOTS];
40 if (e->addr != addr) {
41 slab_err(" track[%s %p]: no record (last-seen addr %p)", label,
42 (void *) addr, (void *) e->addr);
43 return;
44 }
45 slab_err(" track[%s %p]: state=%s seq=%u alloc_ra=%#lx/%#lx "
46 "free_ra=%#lx/%#lx",
47 label, (void *) addr, e->live ? "LIVE(alloc)" : "freed", e->seq,
48 e->alloc_ra[0], e->alloc_ra[1], e->free_ra[0], e->free_ra[1]);
49}
50
51void slab_debug_assert_not_already_free(vaddr_t v, int32_t class) {
52 if (class < 0)
53 return;
54 for (size_t d = 0; d < global.domain_count; d++) {
55 struct slab_domain *sd = global.domains[d]->slab_domain;
56 if (!sd)
57 continue;
58
59 if (sd->percpu_caches) {
60 size_t cpus = sd->domain->num_cores;
61 for (size_t c = 0; c < cpus; c++) {
62 struct slab_percpu_cache *pc = sd->percpu_caches[c];
63 if (!pc)
64 continue;
65 for (int t = 0; t < SLAB_MAGAZINE_TYPE_COUNT; t++) {
66 if (!pc->mags[t])
67 continue;
68 struct slab_magazine *m = &pc->mags[t][class];
69 for (size_t i = 0; i < m->count; i++)
70 if (m->objs[i] == v)
71 panic(BAD_FREE,
72 "%p already parked in d=%zu cpu=%zu mag "
73 "type=%d class=%d idx=%zu",
74 (void *) v, d, c, t, class, i);
75 }
76 }
77 }
78
79 struct slab_free_queue *fq = &sd->free_queue;
80 if (fq->mpmc.slots) {
81 uint64_t tail = atomic_load_acq(&fq->mpmc.hdr.tail);
82 uint64_t head = atomic_load_acq(&fq->mpmc.hdr.head);
83 for (uint64_t pos = tail; pos != head; pos++)
84 if (fq->mpmc.slots[pos & fq->mpmc.hdr.mask].val == v)
85 panic(BAD_FREE, "%p already in d=%zu free_queue slot=%zu",
86 (void *) v, d, (size_t) (pos & fq->mpmc.hdr.mask));
87 }
88 }
89}
90
91cc_no_asan void slab_dump_corruption(void *obj,
92 struct slab_magazine *popped_mag,
93 size_t obj_size) {
94 crash_broadcast_nmi(); /* Get everyone to stop yapping
95 * so we have a clean view */
96 vaddr_t v = (vaddr_t) obj;
97 struct slab *s = slab_for_ptr(obj);
98 uint64_t byte_idx;
99 uint8_t bit_mask;
100 slab_index_and_mask(s, obj, &byte_idx, &bit_mask);
101 size_t obj_idx = (v - s->mem) / s->parent_cache->obj_stride;
102
103 slab_err("obj=%p region=%s", (void *) v, slab_addr_region(v));
104 slab_err("slab=%p base_off=%#zx obj_idx=%zu type=%d obj_size=%zu "
105 "page_count=%zu used=%zu bit_set=%d",
106 s, (size_t) (v - (vaddr_t) s), obj_idx, s->type,
107 s->parent_cache->obj_size, s->page_count, s->used,
108 SLAB_BITMAP_TEST(s->bitmap[byte_idx], bit_mask) != 0);
109
110 slab_track_dump("victim", v);
111
112 size_t words;
113
114 if (popped_mag) {
115 words = popped_mag->obj_size / sizeof(uint64_t);
116 } else {
117 words = obj_size / sizeof(uint64_t);
118 }
119
120 for (size_t w = 0; w < words; w++) {
121 uint64_t val = ((uint64_t *) obj)[w];
122 if (!val)
123 continue;
124 slab_err(" word[%zu] (off %zu) = %#lx region=%s", w, w * 8, val,
125 slab_addr_region((vaddr_t) val));
126 if ((vaddr_t) val >= SLAB_HEAP_START && (vaddr_t) val < SLAB_HEAP_END) {
127 struct slab *ps = slab_for_ptr((void *) val);
128 slab_err(" -> points into slab %p type=%d obj_size=%zu", ps,
129 ps->type, ps->parent_cache->obj_size);
130 slab_track_dump("target", (vaddr_t) val);
131 }
132 /* Dump 8 words around the target if it looks like a kernel pointer. */
133 if ((vaddr_t) val >= SLAB_HEAP_START &&
134 vmm_get_phys(PAGE_ALIGN_DOWN((vaddr_t) val)) != (paddr_t) -1) {
135 uint64_t *t = (uint64_t *) (vaddr_t) val;
136 for (int j = 0; j < 6; j++)
137 slab_err(" [%p +%d] = %#lx", t, j * 8, t[j]);
138 }
139 }
140
141 /* Search every magazine, shadow buffer, and free_queue for this address. */
142 size_t found = 0;
143 for (size_t d = 0; d < global.domain_count; d++) {
144 struct slab_domain *sd = global.domains[d]->slab_domain;
145 if (!sd)
146 continue;
147
148 if (sd->percpu_caches) {
149 size_t cpus = sd->domain->num_cores;
150 for (size_t c = 0; c < cpus; c++) {
151 struct slab_percpu_cache *pc = sd->percpu_caches[c];
152 if (!pc)
153 continue;
154
155 vaddr_t pcv = (vaddr_t) pc;
156 if (v >= pcv && v < pcv + sizeof(struct slab_percpu_cache))
157 slab_err(
158 " !! obj OVERLAPS percpu_cache %p (d=%zu cpu=%zu) "
159 "off_into_pc=%#zx",
160 pc, d, c, (size_t) (v - pcv));
161 for (int t = 0; t < SLAB_MAGAZINE_TYPE_COUNT; t++) {
162 if ((vaddr_t) pc->mags[t] ==
163 (vaddr_t) ((uint64_t *) obj)[6])
164 slab_err(" !! obj word[6] == percpu_cache %p mags[%d] "
165 "(d=%zu cpu=%zu)",
166 pc, t, d, c);
167 }
168 for (int t = 0; t < SLAB_MAGAZINE_TYPE_COUNT; t++) {
169 if (!pc->mags[t])
170 continue;
171 for (size_t k = 0; k < slab_global.num_sizes; k++) {
172 struct slab_magazine *m = &pc->mags[t][k];
173 for (size_t i = 0; i < SLAB_MAG_ENTRIES; i++) {
174 if (m->objs[i] == v) {
175 slab_err(" ALSO IN mag d=%zu cpu=%zu type=%d "
176 "class=%zu idx=%zu",
177 d, c, t, k, i);
178 found++;
179 }
180 }
181 }
182 for (size_t i = 0; i < SLAB_MAG_ENTRIES + 1; i++) {
183 if (pc->shadow_objs[i] == v) {
184 slab_err(" ALSO IN shadow d=%zu cpu=%zu idx=%zu",
185 d, c, i);
186 found++;
187 }
188 }
189 }
190 }
191 }
192
193 struct slab_free_queue *fq = &sd->free_queue;
194 if (fq->mpmc.slots) {
195 uint64_t tail = atomic_load_acq(&fq->mpmc.hdr.tail);
196 uint64_t head = atomic_load_acq(&fq->mpmc.hdr.head);
197 for (size_t i = 0; i < slab_free_ring_capacity(&fq->mpmc); i++) {
198 if (fq->mpmc.slots[i].val != v)
199 continue;
200 bool occupied = false;
201 for (uint64_t pos = tail; pos != head; pos++) {
202 if ((pos & fq->mpmc.hdr.mask) == i) {
203 occupied = true;
204 break;
205 }
206 }
207 slab_err(" ALSO IN free_queue d=%zu slot=%zu %s", d, i,
208 occupied ? "(OCCUPIED -- real dup)" : "(stale addr)");
209 if (occupied)
210 found++;
211 }
212 }
213 }
214 slab_err("total duplicate locations found (excl. nothing): %zu", found);
215}
216
217stack_handle_t slab_stack_handle_for(void *ptr) {
218 enforce(slab_ptr_in_slab(ptr), BAD_FREE);
219 struct slab *s = slab_for_ptr(ptr);
220 size_t idx = slab_allocation_index(s, ptr);
221 return s->traces[idx];
222}
223#endif
224