1#include "internal.h"
2#include <mem/address_range.h>
3
4#ifdef DEBUG_SLAB_DEEP
5const char *slab_addr_region(vaddr_t a) {
6 if (address_range_for_addr(a))
7 return address_range_for_addr(a)->name;
8
9 return "none";
10}
11
12#define SLAB_TRACK_SLOTS 16384
13struct slab_track {
14 vaddr_t addr;
15 uint64_t alloc_ra[2];
16 uint64_t free_ra[2];
17 uint32_t seq;
18 uint8_t live;
19};
20static struct slab_track slab_track_table[SLAB_TRACK_SLOTS];
21static _Atomic uint32_t slab_track_seq;
22
23void slab_track_event(vaddr_t addr, uint64_t ra0, uint64_t ra1, bool is_alloc) {
24 struct slab_track *e = &slab_track_table[(addr >> 6) % SLAB_TRACK_SLOTS];
25 e->addr = addr;
26 e->seq =
27 atomic_fetch_add_explicit(&slab_track_seq, 1, memory_order_relaxed);
28 if (is_alloc) {
29 e->alloc_ra[0] = ra0;
30 e->alloc_ra[1] = ra1;
31 e->live = 1;
32 } else {
33 e->free_ra[0] = ra0;
34 e->free_ra[1] = ra1;
35 e->live = 0;
36 }
37}
38
39void slab_track_dump(const char *label, vaddr_t addr) {
40 struct slab_track *e = &slab_track_table[(addr >> 6) % SLAB_TRACK_SLOTS];
41 if (e->addr != addr) {
42 slab_err(" track[%s %p]: no record (last-seen addr %p)", label,
43 (void *) addr, (void *) e->addr);
44 return;
45 }
46 slab_err(" track[%s %p]: state=%s seq=%u alloc_ra=%#lx/%#lx "
47 "free_ra=%#lx/%#lx",
48 label, (void *) addr, e->live ? "LIVE(alloc)" : "freed", e->seq,
49 e->alloc_ra[0], e->alloc_ra[1], e->free_ra[0], e->free_ra[1]);
50}
51
52void slab_debug_assert_not_already_free(vaddr_t v, int32_t class) {
53 if (class < 0)
54 return;
55 for (size_t d = 0; d < global.domain_count; d++) {
56 struct slab_domain *sd = global.domains[d]->slab_domain;
57 if (!sd)
58 continue;
59
60 if (sd->percpu_caches) {
61 size_t cpus = sd->domain->num_cores;
62 for (size_t c = 0; c < cpus; c++) {
63 struct slab_percpu_cache *pc = sd->percpu_caches[c];
64 if (!pc)
65 continue;
66 for (int t = 0; t < SLAB_MAGAZINE_TYPE_COUNT; t++) {
67 if (!pc->mags[t])
68 continue;
69 struct slab_magazine *m = &pc->mags[t][class];
70 for (size_t i = 0; i < m->count; i++)
71 if (m->objs[i] == v)
72 panic("DOUBLE FREE: %p already parked in d=%zu "
73 "cpu=%zu mag type=%d class=%d idx=%zu",
74 (void *) v, d, c, t, class, i);
75 }
76 }
77 }
78
79 struct slab_free_queue *fq = &sd->free_queue;
80 if (fq->slots) {
81 uint64_t tail =
82 atomic_load_explicit(&fq->tail, memory_order_acquire);
83 uint64_t head =
84 atomic_load_explicit(&fq->head, memory_order_acquire);
85 for (uint64_t pos = tail; pos != head; pos++)
86 if (fq->slots[pos % fq->capacity].addr == v)
87 panic(
88 "DOUBLE FREE: %p already in d=%zu free_queue slot=%zu",
89 (void *) v, d, (size_t) (pos % fq->capacity));
90 }
91 }
92}
93
94void slab_dump_corruption(void *obj, struct slab_magazine *popped_mag,
95 size_t obj_size) {
96 panic_broadcast_nmi(); /* Get everyone to stop yapping
97 * so we have a clean view */
98 vaddr_t v = (vaddr_t) obj;
99 struct slab *s = slab_for_ptr(obj);
100 uint64_t byte_idx;
101 uint8_t bit_mask;
102 slab_index_and_mask(s, obj, &byte_idx, &bit_mask);
103 size_t obj_idx = (v - s->mem) / s->parent_cache->obj_stride;
104
105 slab_err("obj=%p region=%s", (void *) v, slab_addr_region(v));
106 slab_err("slab=%p base_off=%#zx obj_idx=%zu type=%d obj_size=%zu "
107 "page_count=%zu used=%zu bit_set=%d",
108 s, (size_t) (v - (vaddr_t) s), obj_idx, s->type,
109 s->parent_cache->obj_size, s->page_count, s->used,
110 SLAB_BITMAP_TEST(s->bitmap[byte_idx], bit_mask) != 0);
111
112 slab_track_dump("victim", v);
113
114 size_t words;
115
116 if (popped_mag) {
117 words = popped_mag->obj_size / sizeof(uint64_t);
118 } else {
119 words = obj_size / sizeof(uint64_t);
120 }
121
122 for (size_t w = 0; w < words; w++) {
123 uint64_t val = ((uint64_t *) obj)[w];
124 if (!val)
125 continue;
126 slab_err(" word[%zu] (off %zu) = %#lx region=%s", w, w * 8, val,
127 slab_addr_region((vaddr_t) val));
128 if ((vaddr_t) val >= SLAB_HEAP_START && (vaddr_t) val < SLAB_HEAP_END) {
129 struct slab *ps = slab_for_ptr((void *) val);
130 slab_err(" -> points into slab %p type=%d obj_size=%zu", ps,
131 ps->type, ps->parent_cache->obj_size);
132 slab_track_dump("target", (vaddr_t) val);
133 }
134 /* Dump 8 words around the target if it looks like a kernel pointer. */
135 if ((vaddr_t) val >= SLAB_HEAP_START &&
136 vmm_get_phys(PAGE_ALIGN_DOWN((vaddr_t) val), VMM_FLAG_NONE) !=
137 (paddr_t) -1) {
138 uint64_t *t = (uint64_t *) (vaddr_t) val;
139 for (int j = 0; j < 6; j++)
140 slab_err(" [%p +%d] = %#lx", t, j * 8, t[j]);
141 }
142 }
143
144 /* Search every magazine, shadow buffer, and free_queue for this address. */
145 size_t found = 0;
146 for (size_t d = 0; d < global.domain_count; d++) {
147 struct slab_domain *sd = global.domains[d]->slab_domain;
148 if (!sd)
149 continue;
150
151 if (sd->percpu_caches) {
152 size_t cpus = sd->domain->num_cores;
153 for (size_t c = 0; c < cpus; c++) {
154 struct slab_percpu_cache *pc = sd->percpu_caches[c];
155 if (!pc)
156 continue;
157
158 vaddr_t pcv = (vaddr_t) pc;
159 if (v >= pcv && v < pcv + sizeof(struct slab_percpu_cache))
160 slab_err(
161 " !! obj OVERLAPS percpu_cache %p (d=%zu cpu=%zu) "
162 "off_into_pc=%#zx",
163 pc, d, c, (size_t) (v - pcv));
164 for (int t = 0; t < SLAB_MAGAZINE_TYPE_COUNT; t++) {
165 if ((vaddr_t) pc->mags[t] ==
166 (vaddr_t) ((uint64_t *) obj)[6])
167 slab_err(" !! obj word[6] == percpu_cache %p mags[%d] "
168 "(d=%zu cpu=%zu)",
169 pc, t, d, c);
170 }
171 for (int t = 0; t < SLAB_MAGAZINE_TYPE_COUNT; t++) {
172 if (!pc->mags[t])
173 continue;
174 for (size_t k = 0; k < slab_global.num_sizes; k++) {
175 struct slab_magazine *m = &pc->mags[t][k];
176 for (size_t i = 0; i < SLAB_MAG_ENTRIES; i++) {
177 if (m->objs[i] == v) {
178 slab_err(" ALSO IN mag d=%zu cpu=%zu type=%d "
179 "class=%zu idx=%zu",
180 d, c, t, k, i);
181 found++;
182 }
183 }
184 }
185 for (size_t i = 0; i < SLAB_MAG_ENTRIES + 1; i++) {
186 if (pc->shadow_objs[i] == v) {
187 slab_err(" ALSO IN shadow d=%zu cpu=%zu idx=%zu",
188 d, c, i);
189 found++;
190 }
191 }
192 }
193 }
194 }
195
196 struct slab_free_queue *fq = &sd->free_queue;
197 if (fq->slots) {
198 uint64_t tail =
199 atomic_load_explicit(&fq->tail, memory_order_acquire);
200 uint64_t head =
201 atomic_load_explicit(&fq->head, memory_order_acquire);
202 for (size_t i = 0; i < fq->capacity; i++) {
203 if (fq->slots[i].addr != v)
204 continue;
205 bool occupied = false;
206 for (uint64_t pos = tail; pos != head; pos++) {
207 if (pos % fq->capacity == i) {
208 occupied = true;
209 break;
210 }
211 }
212 slab_err(" ALSO IN free_queue d=%zu slot=%zu %s", d, i,
213 occupied ? "(OCCUPIED -- real dup)" : "(stale addr)");
214 if (occupied)
215 found++;
216 }
217 }
218 }
219 slab_err("total duplicate locations found (excl. nothing): %zu", found);
220}
221#endif
222