1#include <math/min_max.h>
2#include <mem/nofail.h>
3#include <mem/slab.h>
4#include <sch/sched.h>
5#include <smp/domain.h>
6
7#include "internal.h"
8
9static cc_no_asan void slab_magazine_zero_check(struct slab_magazine *mag) {
10#ifndef DEBUG_SLAB_DEEP
11 cc_ignore(mag);
12#endif
13
14#ifdef DEBUG_SLAB_DEEP
15 if (mag->type != SLAB_MAGAZINE_ZERO)
16 return;
17
18 for (size_t i = 0; i < SLAB_MAG_ENTRIES; i++) {
19 if (mag->objs[i]) {
20 if (!is_buffer_uniform((void *) mag->objs[i], mag->obj_size, 0)) {
21 vaddr_t obj = mag->objs[i];
22 struct slab *s = slab_for_ptr((void *) obj);
23 uint64_t byte_idx;
24 uint8_t bit_mask;
25 slab_index_and_mask(s, (void *) obj, &byte_idx, &bit_mask);
26 slab_err("CORRUPT parked obj %p mag_objsize=%zu", (void *) obj,
27 mag->obj_size);
28 slab_err(" slab=%p type=%d obj_size=%zu page_count=%zu "
29 "used=%zu bit_set=%d",
30 s, s->type, s->parent_cache->obj_size, s->page_count,
31 s->used,
32 SLAB_BITMAP_TEST(s->bitmap[byte_idx], bit_mask) != 0);
33 for (size_t b = 0; b < mag->obj_size; b++) {
34 uint8_t v = ((uint8_t *) obj)[b];
35 if (v)
36 slab_err(" byte %zu = 0x%02x", b, v);
37 }
38 panic(CORRUPTION,
39 "not uniform at idx %zu on mag %p with count %zu", i, mag,
40 mag->count);
41 }
42 }
43 }
44#endif
45}
46
47static bool slab_magazine_has(struct slab_magazine *mag, vaddr_t obj) {
48 for (size_t i = 0; i < SLAB_MAG_ENTRIES; i++) {
49 if (mag->objs[i] == obj)
50 return true;
51 }
52 return false;
53}
54
55bool slab_magazine_push(struct slab_magazine *mag, vaddr_t obj) {
56#ifdef DEBUG_SLAB
57 if (mag->type == SLAB_MAGAZINE_ZERO &&
58 !is_buffer_uniform((void *) obj, mag->obj_size, 0))
59 panic(CORRUPTION, "buffer of size %zu is not uniform", mag->obj_size);
60#endif
61
62 enforce(mag->parent == slab_percpu_cache_local());
63 enforce(!slab_magazine_has(mag, obj));
64 slab_magazine_zero_check(mag);
65
66 if (mag->count < SLAB_MAG_ENTRIES) {
67 mag->objs[mag->count++] = obj;
68 return true;
69 }
70
71 return false;
72}
73
74vaddr_t slab_magazine_pop(struct slab_magazine *mag) {
75 enforce(mag->parent == slab_percpu_cache_local());
76
77 vaddr_t ret = 0x0;
78
79 if (mag->count > 0) {
80 ret = mag->objs[--mag->count];
81 mag->objs[mag->count] = 0x0; /* Reset it */
82 }
83
84 if (ret)
85 enforce(!slab_magazine_has(mag, ret));
86
87 slab_magazine_zero_check(mag);
88 return ret;
89}
90
91static size_t slab_cache_bulk_alloc(struct slab_cache *cache,
92 vaddr_t *addr_array, size_t num_objects,
93 enum alloc_behavior behavior) {
94 size_t total_allocated = 0;
95
96 for (size_t i = 0; i < num_objects; i++) {
97
98 /* We don't allow allocations of new slabs - that
99 * is not the point of our percpu caches */
100 enforce_precond(!(behavior & SLAB_ALLOC_BEHAVIOR_FROM_ALLOC));
101 void *obj = slab_alloc(cache, /*handle=*/NULL, behavior);
102 if (!obj)
103 break;
104
105 total_allocated++;
106 addr_array[i] = (vaddr_t) obj;
107 }
108
109 return total_allocated;
110}
111
112static void slab_cache_bulk_free(struct slab_domain *domain,
113 vaddr_t *addr_array, size_t num_objects) {
114 for (size_t i = 0; i < num_objects; i++) {
115 vaddr_t addr = addr_array[i];
116
117 if (!slab_free_queue_ringbuffer_enqueue(q: &domain->free_queue, addr))
118 slab_free(domain, obj: (void *) addr);
119 }
120
121 return;
122}
123
124/* NOTE: unused? */
125void slab_percpu_flush(struct slab_domain *dom, struct slab_percpu_cache *pc,
126 size_t class_idx, vaddr_t overflow_obj) {
127 for (int type = 0; type < SLAB_MAGAZINE_TYPE_COUNT; type++) {
128 struct slab_magazine *mag = &pc->mags[type][class_idx];
129
130 /* capture how many valid items we have */
131 size_t valid = mag->count;
132
133 /* copy only valid objects */
134 for (size_t i = 0; i < valid; i++)
135 pc->shadow_objs[i] = mag->objs[i];
136
137 /* reset magazine */
138 mag->count = 0;
139 for (size_t i = 0; i < valid; i++)
140 mag->objs[i] = 0x0;
141
142 /* add overflow object to the end and free (valid + 1) objects */
143 pc->shadow_objs[valid] = overflow_obj;
144 slab_cache_bulk_free(domain: dom, addr_array: pc->shadow_objs, num_objects: valid + 1);
145 }
146}
147
148static vaddr_t slab_percpu_refill_for_mag_and_cache(
149 struct slab_percpu_cache *pc, struct slab_magazine *mag,
150 struct slab_cache *cache, enum alloc_behavior behavior) {
151 size_t space = SLAB_MAG_ENTRIES - mag->count;
152
153 if (space == 0)
154 space = 1; /* we still want 1 so we can return an object */
155
156 enforce(space <= SLAB_MAG_ENTRIES, CAPACITY_EXCEEDED);
157
158 if (mag->type == SLAB_MAGAZINE_ZERO)
159 enforce(cache->type == SLAB_TYPE_PAGEABLE_ZERO ||
160 cache->type == SLAB_TYPE_NONPAGEABLE_ZERO);
161
162 size_t want = space;
163 size_t got = slab_cache_bulk_alloc(cache, addr_array: pc->shadow_objs, num_objects: want, behavior);
164 if (got == 0)
165 return slab_magazine_pop(mag);
166
167 size_t can_insert = SLAB_MAG_ENTRIES - mag->count;
168 size_t to_insert = got > 0 ? MIN(got - 1, can_insert) : 0;
169
170 enforce(mag->obj_size == cache->obj_size);
171 vaddr_t first = pc->shadow_objs[0];
172
173 for (size_t i = 1; i <= to_insert; i++) {
174
175#ifdef DEBUG_SLAB
176 if (mag->type == SLAB_MAGAZINE_ZERO &&
177 !is_buffer_uniform((void *) pc->shadow_objs[i], mag->obj_size, 0))
178 panic(CORRUPTION, "buffer of size %zu is not uniform",
179 mag->obj_size);
180#endif
181
182 mag->objs[mag->count++] = pc->shadow_objs[i];
183 pc->shadow_objs[i] = 0;
184 }
185
186 slab_magazine_zero_check(mag);
187
188 return first;
189}
190
191static vaddr_t slab_percpu_refill_class(struct slab_domain *dom,
192 struct slab_percpu_cache *pc,
193 size_t class_idx,
194 struct alloc_params params) {
195 struct slab_magazine *mag;
196 struct slab_cache *cache;
197
198 if (params.flags & ALLOC_FLAG_ZERO_ON_ALLOC) {
199 mag = &pc->mags[SLAB_MAGAZINE_ZERO][class_idx];
200 cache = &dom->caches[SLAB_TYPE_NONPAGEABLE_ZERO]->caches[class_idx];
201 } else {
202 mag = &pc->mags[SLAB_MAGAZINE_NORMAL][class_idx];
203 cache = &dom->caches[SLAB_TYPE_NONPAGEABLE]->caches[class_idx];
204 }
205
206 return slab_percpu_refill_for_mag_and_cache(pc, mag, cache,
207 behavior: params.behavior);
208}
209
210void slab_percpu_refill(struct slab_domain *dom,
211 struct slab_percpu_cache *cache,
212 struct alloc_params params) {
213 /* This flushes a portion of the freequeue into the percpu cache */
214 slab_free_queue_drain_limited(pc: cache, dom, /* pct = */ 100, bh: params.behavior);
215 for (size_t class = 0; class < slab_global.num_sizes; class++)
216 slab_percpu_refill_class(dom, pc: cache, class_idx: class, params);
217}
218
219/* TODO: memory locality */
220void slab_domain_percpu_init(struct slab_domain *domain) {
221 size_t cpus = domain->domain->num_cores;
222 domain->percpu_caches =
223 nofail_vnews(domain->percpu_caches, cpus, ALLOC_ZERO);
224
225 for (size_t i = 0; i < cpus; i++) {
226 domain->percpu_caches[i] =
227 nofail_vnew(domain->percpu_caches[i], ALLOC_ZERO);
228 for (int j = 0; j < SLAB_MAGAZINE_TYPE_COUNT; j++) {
229 domain->percpu_caches[i]->mags[j] =
230 nofail_vnews(domain->percpu_caches[i]->mags[j],
231 slab_global.num_sizes, ALLOC_ZERO);
232 for (size_t k = 0; k < slab_global.num_sizes; k++) {
233 domain->percpu_caches[i]->mags[j][k].type = j;
234 domain->percpu_caches[i]->mags[j][k].obj_size =
235 slab_global.class_sizes[k].size;
236 domain->percpu_caches[i]->mags[j][k].parent =
237 domain->percpu_caches[i];
238 }
239 }
240
241 domain->percpu_caches[i]->domain = domain;
242 mpsc_lifo_list_init(q: &domain->percpu_caches[i]->defer_frees);
243 }
244}
245