1#include <mem/alloc_or_die.h>
2#include <mem/slab.h>
3#include <sch/sched.h>
4#include <smp/domain.h>
5
6#include "internal.h"
7
8static void slab_magazine_zero_check(struct slab_magazine *mag) {
9#ifdef DEBUG_SLAB_DEEP
10 if (mag->type != SLAB_MAGAZINE_ZERO)
11 return;
12
13 for (size_t i = 0; i < SLAB_MAG_ENTRIES; i++) {
14 if (mag->objs[i]) {
15 if (!is_buffer_uniform((void *) mag->objs[i], mag->obj_size, 0)) {
16 vaddr_t obj = mag->objs[i];
17 struct slab *s = slab_for_ptr((void *) obj);
18 uint64_t byte_idx;
19 uint8_t bit_mask;
20 slab_index_and_mask(s, (void *) obj, &byte_idx, &bit_mask);
21 slab_err("CORRUPT parked obj %p mag_objsize=%zu", obj,
22 mag->obj_size);
23 slab_err(" slab=%p type=%d obj_size=%zu page_count=%zu "
24 "used=%zu bit_set=%d",
25 s, s->type, s->parent_cache->obj_size, s->page_count,
26 s->used,
27 SLAB_BITMAP_TEST(s->bitmap[byte_idx], bit_mask) != 0);
28 for (size_t b = 0; b < mag->obj_size; b++) {
29 uint8_t v = ((uint8_t *) obj)[b];
30 if (v)
31 slab_err(" byte %zu = 0x%02x", b, v);
32 }
33 panic("not uniform at idx %zu on mag %p with count %zu\n", i,
34 mag, mag->count);
35 }
36 }
37 }
38#endif
39}
40
41static bool slab_magazine_has(struct slab_magazine *mag, vaddr_t obj) {
42 for (size_t i = 0; i < SLAB_MAG_ENTRIES; i++) {
43 if (mag->objs[i] == obj)
44 return true;
45 }
46 return false;
47}
48
49bool slab_magazine_push(struct slab_magazine *mag, vaddr_t obj) {
50#ifdef DEBUG_SLAB
51 if (mag->type == SLAB_MAGAZINE_ZERO &&
52 !is_buffer_uniform((void *) obj, mag->obj_size, 0))
53 panic("buffer of size %zu is not uniform\n", mag->obj_size);
54#endif
55
56 kassert(mag->parent == slab_percpu_cache_local());
57 kassert(!slab_magazine_has(mag, obj));
58 slab_magazine_zero_check(mag);
59
60 if (mag->count < SLAB_MAG_ENTRIES) {
61 mag->objs[mag->count++] = obj;
62 return true;
63 }
64
65 return false;
66}
67
68vaddr_t slab_magazine_pop(struct slab_magazine *mag) {
69 kassert(mag->parent == slab_percpu_cache_local());
70
71 vaddr_t ret = 0x0;
72
73 if (mag->count > 0) {
74 ret = mag->objs[--mag->count];
75 mag->objs[mag->count] = 0x0; /* Reset it */
76 }
77
78 if (ret)
79 kassert(!slab_magazine_has(mag, ret));
80
81 slab_magazine_zero_check(mag);
82 return ret;
83}
84
85static size_t slab_cache_bulk_alloc(struct slab_cache *cache,
86 vaddr_t *addr_array, size_t num_objects,
87 enum alloc_behavior behavior) {
88 size_t total_allocated = 0;
89
90 for (size_t i = 0; i < num_objects; i++) {
91
92 /* We don't allow allocations of new slabs - that
93 * is not the point of our percpu caches */
94 kassert(!(behavior & SLAB_ALLOC_BEHAVIOR_FROM_ALLOC));
95 void *obj = slab_alloc(cache, behavior);
96 if (!obj)
97 break;
98
99 total_allocated++;
100 addr_array[i] = (vaddr_t) obj;
101 }
102
103 return total_allocated;
104}
105
106static void slab_cache_bulk_free(struct slab_domain *domain,
107 vaddr_t *addr_array, size_t num_objects) {
108 for (size_t i = 0; i < num_objects; i++) {
109 vaddr_t addr = addr_array[i];
110
111 if (!slab_free_queue_ringbuffer_enqueue(q: &domain->free_queue, addr))
112 slab_free(domain, obj: (void *) addr);
113 }
114
115 return;
116}
117
118/* NOTE: unused? */
119void slab_percpu_flush(struct slab_domain *dom, struct slab_percpu_cache *pc,
120 size_t class_idx, vaddr_t overflow_obj) {
121 for (int i = 0; i < SLAB_MAGAZINE_TYPE_COUNT; i++) {
122 struct slab_magazine *mag = &pc->mags[i][class_idx];
123
124 /* capture how many valid items we have */
125 size_t valid = mag->count;
126
127 /* copy only valid objects */
128 for (size_t i = 0; i < valid; i++)
129 pc->shadow_objs[i] = mag->objs[i];
130
131 /* reset magazine */
132 mag->count = 0;
133 for (size_t i = 0; i < valid; i++)
134 mag->objs[i] = 0x0;
135
136 /* add overflow object to the end and free (valid + 1) objects */
137 pc->shadow_objs[valid] = overflow_obj;
138 slab_cache_bulk_free(domain: dom, addr_array: pc->shadow_objs, num_objects: valid + 1);
139 }
140}
141
142static vaddr_t slab_percpu_refill_for_mag_and_cache(
143 struct slab_percpu_cache *pc, struct slab_magazine *mag,
144 struct slab_cache *cache, enum alloc_behavior behavior) {
145 size_t space = SLAB_MAG_ENTRIES - mag->count;
146
147 if (space == 0)
148 space = 1; /* we still want 1 so we can return an object */
149
150 kassert(space <= SLAB_MAG_ENTRIES);
151
152 if (mag->type == SLAB_MAGAZINE_ZERO)
153 kassert(cache->type == SLAB_TYPE_PAGEABLE_ZERO ||
154 cache->type == SLAB_TYPE_NONPAGEABLE_ZERO);
155
156 size_t want = space;
157 size_t got = slab_cache_bulk_alloc(cache, addr_array: pc->shadow_objs, num_objects: want, behavior);
158 if (got == 0)
159 return slab_magazine_pop(mag);
160
161 size_t can_insert = SLAB_MAG_ENTRIES - mag->count;
162 size_t to_insert = (got > 0) ? (got - 1) : 0;
163 if (to_insert > can_insert)
164 to_insert = can_insert;
165
166 kassert(mag->obj_size == cache->obj_size);
167 vaddr_t first = pc->shadow_objs[0];
168
169 for (size_t i = 1; i <= to_insert; i++) {
170
171#ifdef DEBUG_SLAB
172 if (mag->type == SLAB_MAGAZINE_ZERO &&
173 !is_buffer_uniform((void *) pc->shadow_objs[i], mag->obj_size, 0))
174 panic("buffer of size %zu is not uniform\n", mag->obj_size);
175#endif
176
177 mag->objs[mag->count++] = pc->shadow_objs[i];
178 pc->shadow_objs[i] = 0;
179 }
180
181 slab_magazine_zero_check(mag);
182
183 return first;
184}
185
186static vaddr_t slab_percpu_refill_class(struct slab_domain *dom,
187 struct slab_percpu_cache *pc,
188 size_t class_idx,
189 enum alloc_flags flags,
190 enum alloc_behavior behavior) {
191 struct slab_magazine *mag;
192 struct slab_cache *cache;
193
194 if (flags & ALLOC_FLAG_ZERO_ON_ALLOC) {
195 mag = &pc->mags[SLAB_MAGAZINE_ZERO][class_idx];
196 cache = &dom->caches[SLAB_TYPE_NONPAGEABLE_ZERO]->caches[class_idx];
197 } else {
198 mag = &pc->mags[SLAB_MAGAZINE_NORMAL][class_idx];
199 cache = &dom->caches[SLAB_TYPE_NONPAGEABLE]->caches[class_idx];
200 }
201
202 return slab_percpu_refill_for_mag_and_cache(pc, mag, cache, behavior);
203}
204
205void slab_percpu_refill(struct slab_domain *dom,
206 struct slab_percpu_cache *cache, enum alloc_flags flags,
207 enum alloc_behavior behavior) {
208 /* This flushes a portion of the freequeue into the percpu cache */
209 slab_free_queue_drain_limited(pc: cache, dom, /* pct = */ 100, bh: behavior);
210 for (size_t class = 0; class < slab_global.num_sizes; class++)
211 slab_percpu_refill_class(dom, pc: cache, class_idx: class, flags, behavior);
212}
213
214/* TODO: memory locality */
215void slab_domain_percpu_init(struct slab_domain *domain) {
216 size_t cpus = domain->domain->num_cores;
217 domain->percpu_caches = alloc_or_die(
218 kmalloc(sizeof(struct slab_percpu_cache *) * cpus, ALLOC_FLAGS_ZERO));
219
220 for (size_t i = 0; i < cpus; i++) {
221 domain->percpu_caches[i] = alloc_or_die(
222 kmalloc(sizeof(struct slab_percpu_cache), ALLOC_FLAGS_ZERO));
223 for (int j = 0; j < SLAB_MAGAZINE_TYPE_COUNT; j++) {
224 domain->percpu_caches[i]->mags[j] = alloc_or_die(
225 kmalloc(sizeof(struct slab_magazine) * slab_global.num_sizes,
226 ALLOC_FLAGS_ZERO));
227 for (size_t k = 0; k < slab_global.num_sizes; k++) {
228 domain->percpu_caches[i]->mags[j][k].type = j;
229 domain->percpu_caches[i]->mags[j][k].obj_size =
230 slab_global.class_sizes[k].size;
231 domain->percpu_caches[i]->mags[j][k].parent =
232 domain->percpu_caches[i];
233 }
234 }
235
236 domain->percpu_caches[i]->domain = domain;
237 mpsc_slist_init(q: &domain->percpu_caches[i]->defer_frees);
238 }
239}
240