1#include <mem/alloc_or_die.h>
2#include <mem/slab.h>
3#include <sch/sched.h>
4#include <smp/domain.h>
5
6#include "internal.h"
7
8__no_sanitize_address static void
9slab_magazine_zero_check(struct slab_magazine *mag) {
10#ifdef DEBUG_SLAB_DEEP
11 if (mag->type != SLAB_MAGAZINE_ZERO)
12 return;
13
14 for (size_t i = 0; i < SLAB_MAG_ENTRIES; i++) {
15 if (mag->objs[i]) {
16 if (!is_buffer_uniform((void *) mag->objs[i], mag->obj_size, 0)) {
17 vaddr_t obj = mag->objs[i];
18 struct slab *s = slab_for_ptr((void *) obj);
19 uint64_t byte_idx;
20 uint8_t bit_mask;
21 slab_index_and_mask(s, (void *) obj, &byte_idx, &bit_mask);
22 slab_err("CORRUPT parked obj %p mag_objsize=%zu", obj,
23 mag->obj_size);
24 slab_err(" slab=%p type=%d obj_size=%zu page_count=%zu "
25 "used=%zu bit_set=%d",
26 s, s->type, s->parent_cache->obj_size, s->page_count,
27 s->used,
28 SLAB_BITMAP_TEST(s->bitmap[byte_idx], bit_mask) != 0);
29 for (size_t b = 0; b < mag->obj_size; b++) {
30 uint8_t v = ((uint8_t *) obj)[b];
31 if (v)
32 slab_err(" byte %zu = 0x%02x", b, v);
33 }
34 panic("not uniform at idx %zu on mag %p with count %zu\n", i,
35 mag, mag->count);
36 }
37 }
38 }
39#endif
40}
41
42static bool slab_magazine_has(struct slab_magazine *mag, vaddr_t obj) {
43 for (size_t i = 0; i < SLAB_MAG_ENTRIES; i++) {
44 if (mag->objs[i] == obj)
45 return true;
46 }
47 return false;
48}
49
50bool slab_magazine_push(struct slab_magazine *mag, vaddr_t obj) {
51#ifdef DEBUG_SLAB
52 if (mag->type == SLAB_MAGAZINE_ZERO &&
53 !is_buffer_uniform((void *) obj, mag->obj_size, 0))
54 panic("buffer of size %zu is not uniform\n", mag->obj_size);
55#endif
56
57 kassert(mag->parent == slab_percpu_cache_local());
58 kassert(!slab_magazine_has(mag, obj));
59 slab_magazine_zero_check(mag);
60
61 if (mag->count < SLAB_MAG_ENTRIES) {
62 mag->objs[mag->count++] = obj;
63 return true;
64 }
65
66 return false;
67}
68
69vaddr_t slab_magazine_pop(struct slab_magazine *mag) {
70 kassert(mag->parent == slab_percpu_cache_local());
71
72 vaddr_t ret = 0x0;
73
74 if (mag->count > 0) {
75 ret = mag->objs[--mag->count];
76 mag->objs[mag->count] = 0x0; /* Reset it */
77 }
78
79 if (ret)
80 kassert(!slab_magazine_has(mag, ret));
81
82 slab_magazine_zero_check(mag);
83 return ret;
84}
85
86static size_t slab_cache_bulk_alloc(struct slab_cache *cache,
87 vaddr_t *addr_array, size_t num_objects,
88 enum alloc_behavior behavior) {
89 size_t total_allocated = 0;
90
91 for (size_t i = 0; i < num_objects; i++) {
92
93 /* We don't allow allocations of new slabs - that
94 * is not the point of our percpu caches */
95 kassert(!(behavior & SLAB_ALLOC_BEHAVIOR_FROM_ALLOC));
96 void *obj = slab_alloc(cache, /*handle=*/NULL, behavior);
97 if (!obj)
98 break;
99
100 total_allocated++;
101 addr_array[i] = (vaddr_t) obj;
102 }
103
104 return total_allocated;
105}
106
107static void slab_cache_bulk_free(struct slab_domain *domain,
108 vaddr_t *addr_array, size_t num_objects) {
109 for (size_t i = 0; i < num_objects; i++) {
110 vaddr_t addr = addr_array[i];
111
112 if (!slab_free_queue_ringbuffer_enqueue(q: &domain->free_queue, addr))
113 slab_free(domain, obj: (void *) addr);
114 }
115
116 return;
117}
118
119/* NOTE: unused? */
120void slab_percpu_flush(struct slab_domain *dom, struct slab_percpu_cache *pc,
121 size_t class_idx, vaddr_t overflow_obj) {
122 for (int i = 0; i < SLAB_MAGAZINE_TYPE_COUNT; i++) {
123 struct slab_magazine *mag = &pc->mags[i][class_idx];
124
125 /* capture how many valid items we have */
126 size_t valid = mag->count;
127
128 /* copy only valid objects */
129 for (size_t i = 0; i < valid; i++)
130 pc->shadow_objs[i] = mag->objs[i];
131
132 /* reset magazine */
133 mag->count = 0;
134 for (size_t i = 0; i < valid; i++)
135 mag->objs[i] = 0x0;
136
137 /* add overflow object to the end and free (valid + 1) objects */
138 pc->shadow_objs[valid] = overflow_obj;
139 slab_cache_bulk_free(domain: dom, addr_array: pc->shadow_objs, num_objects: valid + 1);
140 }
141}
142
143static vaddr_t slab_percpu_refill_for_mag_and_cache(
144 struct slab_percpu_cache *pc, struct slab_magazine *mag,
145 struct slab_cache *cache, enum alloc_behavior behavior) {
146 size_t space = SLAB_MAG_ENTRIES - mag->count;
147
148 if (space == 0)
149 space = 1; /* we still want 1 so we can return an object */
150
151 kassert(space <= SLAB_MAG_ENTRIES);
152
153 if (mag->type == SLAB_MAGAZINE_ZERO)
154 kassert(cache->type == SLAB_TYPE_PAGEABLE_ZERO ||
155 cache->type == SLAB_TYPE_NONPAGEABLE_ZERO);
156
157 size_t want = space;
158 size_t got = slab_cache_bulk_alloc(cache, addr_array: pc->shadow_objs, num_objects: want, behavior);
159 if (got == 0)
160 return slab_magazine_pop(mag);
161
162 size_t can_insert = SLAB_MAG_ENTRIES - mag->count;
163 size_t to_insert = (got > 0) ? (got - 1) : 0;
164 if (to_insert > can_insert)
165 to_insert = can_insert;
166
167 kassert(mag->obj_size == cache->obj_size);
168 vaddr_t first = pc->shadow_objs[0];
169
170 for (size_t i = 1; i <= to_insert; i++) {
171
172#ifdef DEBUG_SLAB
173 if (mag->type == SLAB_MAGAZINE_ZERO &&
174 !is_buffer_uniform((void *) pc->shadow_objs[i], mag->obj_size, 0))
175 panic("buffer of size %zu is not uniform\n", mag->obj_size);
176#endif
177
178 mag->objs[mag->count++] = pc->shadow_objs[i];
179 pc->shadow_objs[i] = 0;
180 }
181
182 slab_magazine_zero_check(mag);
183
184 return first;
185}
186
187static vaddr_t slab_percpu_refill_class(struct slab_domain *dom,
188 struct slab_percpu_cache *pc,
189 size_t class_idx,
190 enum alloc_flags flags,
191 enum alloc_behavior behavior) {
192 struct slab_magazine *mag;
193 struct slab_cache *cache;
194
195 if (flags & ALLOC_FLAG_ZERO_ON_ALLOC) {
196 mag = &pc->mags[SLAB_MAGAZINE_ZERO][class_idx];
197 cache = &dom->caches[SLAB_TYPE_NONPAGEABLE_ZERO]->caches[class_idx];
198 } else {
199 mag = &pc->mags[SLAB_MAGAZINE_NORMAL][class_idx];
200 cache = &dom->caches[SLAB_TYPE_NONPAGEABLE]->caches[class_idx];
201 }
202
203 return slab_percpu_refill_for_mag_and_cache(pc, mag, cache, behavior);
204}
205
206void slab_percpu_refill(struct slab_domain *dom,
207 struct slab_percpu_cache *cache, enum alloc_flags flags,
208 enum alloc_behavior behavior) {
209 /* This flushes a portion of the freequeue into the percpu cache */
210 slab_free_queue_drain_limited(pc: cache, dom, /* pct = */ 100, bh: behavior);
211 for (size_t class = 0; class < slab_global.num_sizes; class++)
212 slab_percpu_refill_class(dom, pc: cache, class_idx: class, flags, behavior);
213}
214
215/* TODO: memory locality */
216void slab_domain_percpu_init(struct slab_domain *domain) {
217 size_t cpus = domain->domain->num_cores;
218 domain->percpu_caches = alloc_or_die(
219 kmalloc(sizeof(struct slab_percpu_cache *) * cpus, ALLOC_FLAGS_ZERO));
220
221 for (size_t i = 0; i < cpus; i++) {
222 domain->percpu_caches[i] = alloc_or_die(
223 kmalloc(sizeof(struct slab_percpu_cache), ALLOC_FLAGS_ZERO));
224 for (int j = 0; j < SLAB_MAGAZINE_TYPE_COUNT; j++) {
225 domain->percpu_caches[i]->mags[j] = alloc_or_die(
226 kmalloc(sizeof(struct slab_magazine) * slab_global.num_sizes,
227 ALLOC_FLAGS_ZERO));
228 for (size_t k = 0; k < slab_global.num_sizes; k++) {
229 domain->percpu_caches[i]->mags[j][k].type = j;
230 domain->percpu_caches[i]->mags[j][k].obj_size =
231 slab_global.class_sizes[k].size;
232 domain->percpu_caches[i]->mags[j][k].parent =
233 domain->percpu_caches[i];
234 }
235 }
236
237 domain->percpu_caches[i]->domain = domain;
238 mpsc_slist_init(q: &domain->percpu_caches[i]->defer_frees);
239 }
240}
241