1#include "mem/tests/test_internal.h"
2#include "mem/vas_internal.h"
3
4#define SMP_VAS_BASE 0x710000000000ULL
5
6static struct thread *vas_worker_on(size_t cpu, void (*entry)(void *),
7 void *arg) {
8 struct thread *thread = thread_create(name: "vas_worker", entry_point: entry, arg);
9 if (!thread)
10 return NULL;
11 cpu_mask_clear_all(m: &thread->allowed_cpus);
12 cpu_mask_set(m: &thread->allowed_cpus, cpu);
13 thread_or_flags(t: thread, flags: THREAD_FLAG_PINNED);
14 thread_set_joinable(t: thread);
15 thread_enqueue_on_core(t: thread, core_id: cpu);
16 return thread;
17}
18
19struct handoff {
20 struct vas *vas;
21 vaddr_t addr;
22 size_t cpu;
23 size_t size;
24 bool valid;
25};
26
27static void handoff_alloc(void *arg) {
28 struct handoff *h = arg;
29 h->cpu = smp_id(cond: TOPC_NONE);
30 h->addr = vas_alloc(vas: h->vas, size: h->size, PAGE_SIZE);
31}
32
33static void handoff_free(void *arg) {
34 struct handoff *h = arg;
35 h->cpu = smp_id(cond: TOPC_NONE);
36 h->valid = vas_vaddr_is_allocated(vas: h->vas, addr: h->addr + h->size - 1);
37 vas_free(vas: h->vas, addr: h->addr, size: h->size);
38 h->valid &= !vas_vaddr_is_allocated(vas: h->vas, addr: h->addr);
39}
40
41TEST_DECLARE_INTEGRATION(vas, cross_cpu_free_without_any_free_gap) {
42 if (global.core_count < 2)
43 return TEST_SKIP(TEST_SKIP_NONE);
44
45 struct handoff h = {
46 .vas = vas_create(SMP_VAS_BASE, SMP_VAS_BASE + VAS_CHUNK_SIZE),
47 .size = VAS_CHUNK_SIZE,
48 };
49
50 TEST_ASSERT_NONNULL(h.vas);
51 struct thread *producer = vas_worker_on(cpu: 0, entry: handoff_alloc, arg: &h);
52 TEST_ASSERT_NONNULL(producer);
53
54 thread_join(t: producer);
55 TEST_ASSERT_EQ(h.cpu, 0);
56 TEST_ASSERT_EQ(h.addr, SMP_VAS_BASE);
57 TEST_ASSERT_EQ(atomic_load(&h.vas->chunk_owner[0]), 0);
58 TEST_ASSERT_EQ(h.vas->local[0].total_free, 0);
59
60 struct thread *consumer = vas_worker_on(cpu: 1, entry: handoff_free, arg: &h);
61 TEST_ASSERT_NONNULL(consumer);
62
63 thread_join(t: consumer);
64 TEST_ASSERT_EQ(h.cpu, 1);
65 TEST_ASSERT(h.valid);
66 TEST_ASSERT_EQ(h.vas->local[0].total_free, VAS_CHUNK_SIZE);
67 TEST_ASSERT_EQ(h.vas->local[1].total_free, 0);
68 TEST_ASSERT(vas_destroy(h.vas));
69
70 return TEST_SUCCESS;
71}
72
73struct churn_worker {
74 struct vas *vas;
75 size_t expected_cpu;
76 bool passed;
77};
78
79static void concurrent_churn(void *arg) {
80 struct churn_worker *w = arg;
81 w->passed = smp_id(cond: TOPC_NONE) == w->expected_cpu;
82 uint64_t random = 0x123456789ULL + w->expected_cpu;
83 for (size_t i = 0; i < 600; i++) {
84 const size_t classes[] = {PAGE_SIZE, 5 * PAGE_SIZE, 17 * PAGE_SIZE,
85 PAGE_2MB};
86 size_t size =
87 i % 2 ? classes[(i / 2) % TEST_ARRAY_LEN(classes)]
88 : (1 + prng_splitmix64_next(state: &random) % 512) * PAGE_SIZE;
89 size_t align = i % 3 ? PAGE_SIZE : PAGE_2MB;
90 vaddr_t addr = vas_alloc(vas: w->vas, size, align);
91 if (!addr) {
92 w->passed = false;
93 break;
94 }
95 w->passed &= vas_vaddr_is_allocated(vas: w->vas, addr: addr + size - 1);
96 if (i % 7 == 0)
97 vas_reclaim(vas: w->vas);
98 vas_free(vas: w->vas, addr, size);
99 }
100}
101
102TEST_DECLARE_INTEGRATION(vas, concurrent_import_query_and_reclaim) {
103 if (global.core_count < 2)
104 return TEST_SKIP(TEST_SKIP_NONE);
105
106 struct vas *vas =
107 vas_create(SMP_VAS_BASE, SMP_VAS_BASE + 8 * VAS_CHUNK_SIZE);
108
109 TEST_ASSERT_NONNULL(vas);
110 struct thread *threads[4] = {0};
111 struct churn_worker workers[4] = {0};
112 size_t count = global.core_count < 4 ? global.core_count : 4;
113 size_t started = 0;
114
115 for (; started < count; started++) {
116 workers[started].vas = vas;
117 workers[started].expected_cpu = started;
118 threads[started] =
119 vas_worker_on(cpu: started, entry: concurrent_churn, arg: &workers[started]);
120
121 if (!threads[started])
122 break;
123 }
124
125 bool passed = started == count;
126 for (size_t i = 0; i < started; i++) {
127 thread_join(t: threads[i]);
128 passed &= workers[i].passed;
129 }
130
131 vas_reclaim(vas);
132 TEST_ASSERT_EQ(vas->global.total_free, vas->limit - vas->base);
133 TEST_ASSERT(vas_destroy(vas));
134 TEST_ASSERT(passed);
135 return TEST_SUCCESS;
136}
137
138static bool vas_run_on(size_t cpu, void (*entry)(void *), void *arg) {
139 struct thread *thread = vas_worker_on(cpu, entry, arg);
140 if (!thread)
141 return false;
142
143 thread_join(t: thread);
144 return true;
145}
146
147TEST_DECLARE_INTEGRATION(vas, magazine_remote_free_does_not_cache) {
148 if (global.core_count < 2)
149 return TEST_SKIP(TEST_SKIP_NONE);
150
151 struct handoff h = {
152 .vas = vas_create(SMP_VAS_BASE, SMP_VAS_BASE + VAS_CHUNK_SIZE),
153 .size = PAGE_SIZE,
154 };
155
156 TEST_ASSERT_NONNULL(h.vas);
157 TEST_ASSERT(vas_run_on(0, handoff_alloc, &h));
158 TEST_ASSERT_NE(h.addr, 0);
159 size_t enrolled = atomic_load(&h.vas->mag_reserved_bytes);
160 TEST_ASSERT_GE(enrolled, PAGE_SIZE);
161 TEST_ASSERT(vas_run_on(1, handoff_free, &h));
162 TEST_ASSERT(h.valid);
163 TEST_ASSERT_EQ(atomic_load(&h.vas->mag_reserved_bytes),
164 enrolled - PAGE_SIZE);
165
166 TEST_ASSERT_EQ(h.vas->local[1].mag_free_hits, 0);
167 TEST_ASSERT_EQ(h.vas->local[0].mag_free_hits, 0);
168 TEST_ASSERT(vas_destroy(h.vas));
169 return TEST_SUCCESS;
170}
171
172/* explicit phase boundaries to verify remote visibility */
173struct magazine_visibility {
174 struct vas *vas;
175 vaddr_t addr;
176 _Atomic uint32_t phase;
177 bool valid;
178};
179
180static bool await_phase(struct magazine_visibility *v, uint32_t phase) {
181 while (true) {
182 uint32_t current =
183 atomic_load_explicit(&v->phase, memory_order_acquire);
184 if (current == UINT32_MAX)
185 return false;
186 if (current == phase)
187 return true;
188 scheduler_yield();
189 }
190}
191
192static void visibility_owner(void *arg) {
193 struct magazine_visibility *v = arg;
194 for (uint32_t i = 0; i < 200; i++) {
195 v->addr = vas_alloc(vas: v->vas, PAGE_SIZE, PAGE_SIZE);
196 if (!v->addr) {
197 atomic_store(&v->phase, UINT32_MAX);
198 return;
199 }
200 atomic_store_explicit(&v->phase, 1, memory_order_release);
201 if (!await_phase(v, phase: 2)) {
202 vas_free(vas: v->vas, addr: v->addr, PAGE_SIZE);
203 return;
204 }
205 vas_free(vas: v->vas, addr: v->addr, PAGE_SIZE);
206 atomic_store_explicit(&v->phase, 3, memory_order_release);
207 if (!await_phase(v, phase: 4))
208 return;
209 }
210}
211
212static void visibility_observer(void *arg) {
213 struct magazine_visibility *v = arg;
214 v->valid = true;
215 for (uint32_t i = 0; i < 200; i++) {
216 if (!await_phase(v, phase: 1)) {
217 v->valid = false;
218 return;
219 }
220 vas_reclaim(vas: v->vas); /* Must keep the owner's live slot intact */
221 v->valid &= vas_vaddr_is_allocated(vas: v->vas, addr: v->addr + PAGE_SIZE - 1);
222 atomic_store_explicit(&v->phase, 2, memory_order_release);
223 if (!await_phase(v, phase: 3)) {
224 v->valid = false;
225 return;
226 }
227 v->valid &= !vas_vaddr_is_allocated(vas: v->vas, addr: v->addr);
228 vas_reclaim(vas: v->vas);
229 v->valid &= atomic_load(&v->vas->mag_reserved_bytes) == 0;
230 atomic_store_explicit(&v->phase, 4, memory_order_release);
231 }
232}
233
234TEST_DECLARE_INTEGRATION(vas, magazine_remote_visibility_and_drain) {
235 if (global.core_count < 2)
236 return TEST_SKIP(TEST_SKIP_NONE);
237 struct magazine_visibility v = {
238 .vas = vas_create(SMP_VAS_BASE, SMP_VAS_BASE + VAS_CHUNK_SIZE),
239 };
240 TEST_ASSERT_NONNULL(v.vas);
241 atomic_init(&v.phase, 0);
242 struct thread *owner = vas_worker_on(cpu: 0, entry: visibility_owner, arg: &v);
243 TEST_ASSERT_NONNULL(owner);
244 struct thread *observer = vas_worker_on(cpu: 1, entry: visibility_observer, arg: &v);
245 if (!observer)
246 atomic_store(&v.phase, UINT32_MAX);
247 thread_join(t: owner);
248 if (observer)
249 thread_join(t: observer);
250 TEST_ASSERT(vas_destroy(v.vas));
251 TEST_ASSERT(v.valid);
252 return TEST_SUCCESS;
253}
254
255struct budget_worker {
256 struct vas *vas;
257 vaddr_t addresses[2];
258};
259
260static void budget_alloc(void *arg) {
261 struct budget_worker *w = arg;
262 for (size_t i = 0; i < TEST_ARRAY_LEN(w->addresses); i++)
263 w->addresses[i] = vas_alloc(vas: w->vas, PAGE_2MB, PAGE_2MB);
264}
265
266static void budget_free(void *arg) {
267 struct budget_worker *w = arg;
268 for (size_t i = 0; i < TEST_ARRAY_LEN(w->addresses); i++)
269 if (w->addresses[i])
270 vas_free(vas: w->vas, addr: w->addresses[i], PAGE_2MB);
271}
272
273TEST_DECLARE_INTEGRATION(vas, magazine_per_vas_byte_budget) {
274 if (global.core_count < 2)
275 return TEST_SKIP(TEST_SKIP_NONE);
276 struct vas *vas =
277 vas_create(SMP_VAS_BASE, SMP_VAS_BASE + 4 * VAS_CHUNK_SIZE);
278 TEST_ASSERT_NONNULL(vas);
279 struct budget_worker workers[2] = {{.vas = vas}, {.vas = vas}};
280 for (uint32_t cpu = 0; cpu < 2; cpu++) {
281 TEST_ASSERT(vas_run_on(cpu, budget_alloc, &workers[cpu]));
282 for (size_t i = 0; i < TEST_ARRAY_LEN(workers[cpu].addresses); i++)
283 TEST_ASSERT_NE(workers[cpu].addresses[i], 0);
284 }
285 TEST_ASSERT_EQ(atomic_load(&vas->mag_reserved_bytes), VAS_MAG_BYTE_LIMIT);
286 struct handoff extra = {.vas = vas, .size = PAGE_SIZE};
287 TEST_ASSERT(vas_run_on(0, handoff_alloc, &extra));
288 TEST_ASSERT_NE(extra.addr, 0); /* Budget denies caching */
289 TEST_ASSERT_EQ(atomic_load(&vas->mag_reserved_bytes), VAS_MAG_BYTE_LIMIT);
290 TEST_ASSERT(vas_run_on(0, handoff_free, &extra));
291 TEST_ASSERT_EQ(vas->local[0].mag_free_hits, 0);
292 for (uint32_t cpu = 0; cpu < 2; cpu++)
293 TEST_ASSERT(vas_run_on(cpu, budget_free, &workers[cpu]));
294 vas_reclaim(vas);
295 TEST_ASSERT_EQ(atomic_load(&vas->mag_reserved_bytes), 0);
296 TEST_ASSERT_EQ(vas->global.total_free, vas->limit - vas->base);
297 TEST_ASSERT(vas_destroy(vas));
298 return TEST_SUCCESS;
299}
300