1#include "internal.h"
2#include <mem/address_range.h>
3
4#ifdef DEBUG_SLAB_DEEP
5const char *slab_addr_region(vaddr_t a) {
6 if (address_range_for_addr(a))
7 return address_range_for_addr(a)->name;
8
9 return "none";
10}
11
12#define SLAB_TRACK_SLOTS 16384
13struct slab_track {
14 vaddr_t addr;
15 uint64_t alloc_ra[2];
16 uint64_t free_ra[2];
17 uint32_t seq;
18 uint8_t live;
19};
20static struct slab_track slab_track_table[SLAB_TRACK_SLOTS];
21static _Atomic uint32_t slab_track_seq;
22
23void slab_track_event(vaddr_t addr, uint64_t ra0, uint64_t ra1, bool is_alloc) {
24 struct slab_track *e = &slab_track_table[(addr >> 6) % SLAB_TRACK_SLOTS];
25 e->addr = addr;
26 e->seq =
27 atomic_fetch_add_explicit(&slab_track_seq, 1, memory_order_relaxed);
28 if (is_alloc) {
29 e->alloc_ra[0] = ra0;
30 e->alloc_ra[1] = ra1;
31 e->live = 1;
32 } else {
33 e->free_ra[0] = ra0;
34 e->free_ra[1] = ra1;
35 e->live = 0;
36 }
37}
38
39void slab_track_dump(const char *label, vaddr_t addr) {
40 struct slab_track *e = &slab_track_table[(addr >> 6) % SLAB_TRACK_SLOTS];
41 if (e->addr != addr) {
42 slab_err(" track[%s %p]: no record (last-seen addr %p)", label,
43 (void *) addr, (void *) e->addr);
44 return;
45 }
46 slab_err(" track[%s %p]: state=%s seq=%u alloc_ra=%#lx/%#lx "
47 "free_ra=%#lx/%#lx",
48 label, (void *) addr, e->live ? "LIVE(alloc)" : "freed", e->seq,
49 e->alloc_ra[0], e->alloc_ra[1], e->free_ra[0], e->free_ra[1]);
50}
51
52void slab_debug_assert_not_already_free(vaddr_t v, int32_t class) {
53 if (class < 0)
54 return;
55 for (size_t d = 0; d < global.domain_count; d++) {
56 struct slab_domain *sd = global.domains[d]->slab_domain;
57 if (!sd)
58 continue;
59
60 if (sd->percpu_caches) {
61 size_t cpus = sd->domain->num_cores;
62 for (size_t c = 0; c < cpus; c++) {
63 struct slab_percpu_cache *pc = sd->percpu_caches[c];
64 if (!pc)
65 continue;
66 for (int t = 0; t < SLAB_MAGAZINE_TYPE_COUNT; t++) {
67 if (!pc->mags[t])
68 continue;
69 struct slab_magazine *m = &pc->mags[t][class];
70 for (size_t i = 0; i < m->count; i++)
71 if (m->objs[i] == v)
72 panic("DOUBLE FREE: %p already parked in d=%zu "
73 "cpu=%zu mag type=%d class=%d idx=%zu",
74 (void *) v, d, c, t, class, i);
75 }
76 }
77 }
78
79 struct slab_free_queue *fq = &sd->free_queue;
80 if (fq->mpmc.slots) {
81 uint64_t tail =
82 atomic_load_explicit(&fq->mpmc.tail, memory_order_acquire);
83 uint64_t head =
84 atomic_load_explicit(&fq->mpmc.head, memory_order_acquire);
85 for (uint64_t pos = tail; pos != head; pos++)
86 if (fq->mpmc.slots[pos % fq->mpmc.capacity].data == v)
87 panic(
88 "DOUBLE FREE: %p already in d=%zu free_queue slot=%zu",
89 (void *) v, d, (size_t) (pos % fq->mpmc.capacity));
90 }
91 }
92}
93
94__no_sanitize_address void
95slab_dump_corruption(void *obj, struct slab_magazine *popped_mag,
96 size_t obj_size) {
97 crash_broadcast_nmi(); /* Get everyone to stop yapping
98 * so we have a clean view */
99 vaddr_t v = (vaddr_t) obj;
100 struct slab *s = slab_for_ptr(obj);
101 uint64_t byte_idx;
102 uint8_t bit_mask;
103 slab_index_and_mask(s, obj, &byte_idx, &bit_mask);
104 size_t obj_idx = (v - s->mem) / s->parent_cache->obj_stride;
105
106 slab_err("obj=%p region=%s", (void *) v, slab_addr_region(v));
107 slab_err("slab=%p base_off=%#zx obj_idx=%zu type=%d obj_size=%zu "
108 "page_count=%zu used=%zu bit_set=%d",
109 s, (size_t) (v - (vaddr_t) s), obj_idx, s->type,
110 s->parent_cache->obj_size, s->page_count, s->used,
111 SLAB_BITMAP_TEST(s->bitmap[byte_idx], bit_mask) != 0);
112
113 slab_track_dump("victim", v);
114
115 size_t words;
116
117 if (popped_mag) {
118 words = popped_mag->obj_size / sizeof(uint64_t);
119 } else {
120 words = obj_size / sizeof(uint64_t);
121 }
122
123 for (size_t w = 0; w < words; w++) {
124 uint64_t val = ((uint64_t *) obj)[w];
125 if (!val)
126 continue;
127 slab_err(" word[%zu] (off %zu) = %#lx region=%s", w, w * 8, val,
128 slab_addr_region((vaddr_t) val));
129 if ((vaddr_t) val >= SLAB_HEAP_START && (vaddr_t) val < SLAB_HEAP_END) {
130 struct slab *ps = slab_for_ptr((void *) val);
131 slab_err(" -> points into slab %p type=%d obj_size=%zu", ps,
132 ps->type, ps->parent_cache->obj_size);
133 slab_track_dump("target", (vaddr_t) val);
134 }
135 /* Dump 8 words around the target if it looks like a kernel pointer. */
136 if ((vaddr_t) val >= SLAB_HEAP_START &&
137 vmm_get_phys(PAGE_ALIGN_DOWN((vaddr_t) val), VMM_FLAG_NONE) !=
138 (paddr_t) -1) {
139 uint64_t *t = (uint64_t *) (vaddr_t) val;
140 for (int j = 0; j < 6; j++)
141 slab_err(" [%p +%d] = %#lx", t, j * 8, t[j]);
142 }
143 }
144
145 /* Search every magazine, shadow buffer, and free_queue for this address. */
146 size_t found = 0;
147 for (size_t d = 0; d < global.domain_count; d++) {
148 struct slab_domain *sd = global.domains[d]->slab_domain;
149 if (!sd)
150 continue;
151
152 if (sd->percpu_caches) {
153 size_t cpus = sd->domain->num_cores;
154 for (size_t c = 0; c < cpus; c++) {
155 struct slab_percpu_cache *pc = sd->percpu_caches[c];
156 if (!pc)
157 continue;
158
159 vaddr_t pcv = (vaddr_t) pc;
160 if (v >= pcv && v < pcv + sizeof(struct slab_percpu_cache))
161 slab_err(
162 " !! obj OVERLAPS percpu_cache %p (d=%zu cpu=%zu) "
163 "off_into_pc=%#zx",
164 pc, d, c, (size_t) (v - pcv));
165 for (int t = 0; t < SLAB_MAGAZINE_TYPE_COUNT; t++) {
166 if ((vaddr_t) pc->mags[t] ==
167 (vaddr_t) ((uint64_t *) obj)[6])
168 slab_err(" !! obj word[6] == percpu_cache %p mags[%d] "
169 "(d=%zu cpu=%zu)",
170 pc, t, d, c);
171 }
172 for (int t = 0; t < SLAB_MAGAZINE_TYPE_COUNT; t++) {
173 if (!pc->mags[t])
174 continue;
175 for (size_t k = 0; k < slab_global.num_sizes; k++) {
176 struct slab_magazine *m = &pc->mags[t][k];
177 for (size_t i = 0; i < SLAB_MAG_ENTRIES; i++) {
178 if (m->objs[i] == v) {
179 slab_err(" ALSO IN mag d=%zu cpu=%zu type=%d "
180 "class=%zu idx=%zu",
181 d, c, t, k, i);
182 found++;
183 }
184 }
185 }
186 for (size_t i = 0; i < SLAB_MAG_ENTRIES + 1; i++) {
187 if (pc->shadow_objs[i] == v) {
188 slab_err(" ALSO IN shadow d=%zu cpu=%zu idx=%zu",
189 d, c, i);
190 found++;
191 }
192 }
193 }
194 }
195 }
196
197 struct slab_free_queue *fq = &sd->free_queue;
198 if (fq->mpmc.slots) {
199 uint64_t tail =
200 atomic_load_explicit(&fq->mpmc.tail, memory_order_acquire);
201 uint64_t head =
202 atomic_load_explicit(&fq->mpmc.head, memory_order_acquire);
203 for (size_t i = 0; i < fq->mpmc.capacity; i++) {
204 if (fq->mpmc.slots[i].data != v)
205 continue;
206 bool occupied = false;
207 for (uint64_t pos = tail; pos != head; pos++) {
208 if (pos % fq->mpmc.capacity == i) {
209 occupied = true;
210 break;
211 }
212 }
213 slab_err(" ALSO IN free_queue d=%zu slot=%zu %s", d, i,
214 occupied ? "(OCCUPIED -- real dup)" : "(stale addr)");
215 if (occupied)
216 found++;
217 }
218 }
219 }
220 slab_err("total duplicate locations found (excl. nothing): %zu", found);
221}
222
223stack_handle_t slab_stack_handle_for(void *ptr) {
224 kassert(slab_ptr_in_slab(ptr));
225 struct slab *s = slab_for_ptr(ptr);
226 size_t idx = slab_allocation_index(s, ptr);
227 return s->traces[idx];
228}
229#endif
230