1#include "mem/tests/test_internal.h"
2#include "mem/vas_internal.h"
3
4#define SMP_VAS_BASE 0x710000000000ULL
5
6static struct thread *vas_worker_on(size_t cpu, void (*entry)(void *),
7 void *arg) {
8 return thread_spawn("vas_worker", entry, .arg = arg, .joinable = true,
9 .allowed_cpus = cpu_mask_of(cpu), .pinned = true,
10 .on_cpu = cpu);
11}
12
13struct handoff {
14 struct vas *vas;
15 vaddr_t addr;
16 size_t cpu;
17 size_t size;
18 bool valid;
19};
20
21static void handoff_alloc(void *arg) {
22 struct handoff *h = arg;
23 h->cpu = smp_id(cond: TOPC_NONE);
24 h->addr = vas_alloc(vas: h->vas, size: h->size, PAGE_SIZE);
25}
26
27static void handoff_free(void *arg) {
28 struct handoff *h = arg;
29 h->cpu = smp_id(cond: TOPC_NONE);
30 h->valid = vas_vaddr_is_allocated(vas: h->vas, addr: h->addr + h->size - 1);
31 vas_free(vas: h->vas, addr: h->addr, size: h->size);
32 h->valid &= !vas_vaddr_is_allocated(vas: h->vas, addr: h->addr);
33}
34
35TEST_DEFINE_INTEGRATION(vas, cross_cpu_free_without_any_free_gap,
36 .min_cores = 2) {
37 struct handoff h = {
38 .vas = vas_create(SMP_VAS_BASE, SMP_VAS_BASE + VAS_CHUNK_SIZE),
39 .size = VAS_CHUNK_SIZE,
40 };
41
42 TEST_ASSERT_NONNULL(h.vas);
43 struct thread *producer = vas_worker_on(cpu: 0, entry: handoff_alloc, arg: &h);
44 TEST_ASSERT_NONNULL(producer);
45
46 thread_join(t: producer);
47 TEST_ASSERT_EQ(h.cpu, 0);
48 TEST_ASSERT_EQ(h.addr, SMP_VAS_BASE);
49 TEST_ASSERT_EQ(atomic_load(&h.vas->chunk_owner[0]), 0);
50 TEST_ASSERT_EQ(h.vas->local[0].total_free, 0);
51
52 struct thread *consumer = vas_worker_on(cpu: 1, entry: handoff_free, arg: &h);
53 TEST_ASSERT_NONNULL(consumer);
54
55 thread_join(t: consumer);
56 TEST_ASSERT_EQ(h.cpu, 1);
57 TEST_ASSERT(h.valid);
58 TEST_ASSERT_EQ(h.vas->local[0].total_free, VAS_CHUNK_SIZE);
59 TEST_ASSERT_EQ(h.vas->local[1].total_free, 0);
60 TEST_ASSERT(vas_destroy(h.vas));
61
62 return TEST_SUCCESS;
63}
64
65struct churn_worker {
66 struct vas *vas;
67 size_t expected_cpu;
68 bool passed;
69};
70
71static void concurrent_churn(void *arg) {
72 struct churn_worker *w = arg;
73 w->passed = smp_id(cond: TOPC_NONE) == w->expected_cpu;
74 uint64_t random = 0x123456789ULL + w->expected_cpu;
75 for (size_t i = 0; i < 600; i++) {
76 const size_t classes[] = {PAGE_SIZE, 5 * PAGE_SIZE, 17 * PAGE_SIZE,
77 PAGE_2MB};
78 size_t size =
79 i % 2 ? classes[(i / 2) % TEST_ARRAY_LEN(classes)]
80 : (1 + prng_splitmix64_next(state: &random) % 512) * PAGE_SIZE;
81 size_t align = i % 3 ? PAGE_SIZE : PAGE_2MB;
82 vaddr_t addr = vas_alloc(vas: w->vas, size, align);
83 if (!addr) {
84 w->passed = false;
85 break;
86 }
87 w->passed &= vas_vaddr_is_allocated(vas: w->vas, addr: addr + size - 1);
88 if (i % 7 == 0)
89 vas_reclaim(vas: w->vas);
90 vas_free(vas: w->vas, addr, size);
91 }
92}
93
94TEST_DEFINE_INTEGRATION(vas, mt_import_query_and_reclaim, .min_cores = 2) {
95 struct vas *vas =
96 vas_create(SMP_VAS_BASE, SMP_VAS_BASE + 8 * VAS_CHUNK_SIZE);
97
98 struct thread *threads[4] = {0};
99 struct churn_worker workers[4] = {0};
100 size_t thread_capacity = sizeof(threads) / sizeof(*threads);
101 size_t count = MIN(global.core_count, thread_capacity);
102
103 TEST_ASSERT_NONNULL(vas);
104 size_t started = 0;
105
106 for (; started < count; started++) {
107 workers[started].vas = vas;
108 workers[started].expected_cpu = started;
109 threads[started] =
110 vas_worker_on(cpu: started, entry: concurrent_churn, arg: &workers[started]);
111
112 if (!threads[started])
113 break;
114 }
115
116 bool passed = started == count;
117 for (size_t i = 0; i < started; i++) {
118 thread_join(t: threads[i]);
119 passed &= workers[i].passed;
120 }
121
122 vas_reclaim(vas);
123 TEST_ASSERT_EQ(vas->global.total_free, vas->limit - vas->base);
124 TEST_ASSERT(vas_destroy(vas));
125 TEST_ASSERT(passed);
126 return TEST_SUCCESS;
127}
128
129static bool vas_run_on(size_t cpu, void (*entry)(void *), void *arg) {
130 struct thread *thread = vas_worker_on(cpu, entry, arg);
131 if (!thread)
132 return false;
133
134 thread_join(t: thread);
135 return true;
136}
137
138TEST_DEFINE_INTEGRATION(vas, magazine_remote_free_does_not_cache,
139 .min_cores = 2) {
140 struct handoff h = {
141 .vas = vas_create(SMP_VAS_BASE, SMP_VAS_BASE + VAS_CHUNK_SIZE),
142 .size = PAGE_SIZE,
143 };
144
145 TEST_ASSERT_NONNULL(h.vas);
146 TEST_ASSERT(vas_run_on(0, handoff_alloc, &h));
147 TEST_ASSERT_NE(h.addr, 0);
148 size_t enrolled = atomic_load(&h.vas->mag_reserved_bytes);
149 TEST_ASSERT_GE(enrolled, PAGE_SIZE);
150 TEST_ASSERT(vas_run_on(1, handoff_free, &h));
151 TEST_ASSERT(h.valid);
152 TEST_ASSERT_EQ(atomic_load(&h.vas->mag_reserved_bytes),
153 enrolled - PAGE_SIZE);
154
155 TEST_ASSERT_EQ(h.vas->local[1].mag_free_hits, 0);
156 TEST_ASSERT_EQ(h.vas->local[0].mag_free_hits, 0);
157 TEST_ASSERT(vas_destroy(h.vas));
158 return TEST_SUCCESS;
159}
160
161/* explicit phase boundaries to verify remote visibility */
162struct magazine_visibility {
163 struct vas *vas;
164 vaddr_t addr;
165 atomic_uint32_t phase;
166 bool valid;
167};
168
169static bool await_phase(struct magazine_visibility *v, uint32_t phase) {
170 while (true) {
171 uint32_t current = atomic_load_acq(&v->phase);
172 if (current == UINT32_MAX)
173 return false;
174 if (current == phase)
175 return true;
176 scheduler_yield();
177 }
178}
179
180static void visibility_owner(void *arg) {
181 struct magazine_visibility *v = arg;
182 for (uint32_t i = 0; i < 200; i++) {
183 v->addr = vas_alloc(vas: v->vas, PAGE_SIZE, PAGE_SIZE);
184 if (!v->addr) {
185 atomic_store(&v->phase, UINT32_MAX);
186 return;
187 }
188 atomic_store_release(&v->phase, 1);
189 if (!await_phase(v, phase: 2)) {
190 vas_free(vas: v->vas, addr: v->addr, PAGE_SIZE);
191 return;
192 }
193 vas_free(vas: v->vas, addr: v->addr, PAGE_SIZE);
194 atomic_store_release(&v->phase, 3);
195 if (!await_phase(v, phase: 4))
196 return;
197 }
198}
199
200static void visibility_observer(void *arg) {
201 struct magazine_visibility *v = arg;
202 v->valid = true;
203 for (uint32_t i = 0; i < 200; i++) {
204 if (!await_phase(v, phase: 1)) {
205 v->valid = false;
206 return;
207 }
208 vas_reclaim(vas: v->vas); /* Must keep the owner's live slot intact */
209 v->valid &= vas_vaddr_is_allocated(vas: v->vas, addr: v->addr + PAGE_SIZE - 1);
210 atomic_store_release(&v->phase, 2);
211 if (!await_phase(v, phase: 3)) {
212 v->valid = false;
213 return;
214 }
215 v->valid &= !vas_vaddr_is_allocated(vas: v->vas, addr: v->addr);
216 vas_reclaim(vas: v->vas);
217 v->valid &= atomic_load(&v->vas->mag_reserved_bytes) == 0;
218 atomic_store_release(&v->phase, 4);
219 }
220}
221
222TEST_DEFINE_INTEGRATION(vas, magazine_remote_visibility_and_drain,
223 .min_cores = 2) {
224 struct magazine_visibility v = {
225 .vas = vas_create(SMP_VAS_BASE, SMP_VAS_BASE + VAS_CHUNK_SIZE),
226 };
227 TEST_ASSERT_NONNULL(v.vas);
228 atomic_init(&v.phase, 0);
229 struct thread *owner = vas_worker_on(cpu: 0, entry: visibility_owner, arg: &v);
230 TEST_ASSERT_NONNULL(owner);
231 struct thread *observer = vas_worker_on(cpu: 1, entry: visibility_observer, arg: &v);
232 if (!observer)
233 atomic_store(&v.phase, UINT32_MAX);
234 thread_join(t: owner);
235 if (observer)
236 thread_join(t: observer);
237 TEST_ASSERT(vas_destroy(v.vas));
238 TEST_ASSERT(v.valid);
239 return TEST_SUCCESS;
240}
241
242struct budget_worker {
243 struct vas *vas;
244 vaddr_t addresses[2];
245};
246
247static void budget_alloc(void *arg) {
248 struct budget_worker *w = arg;
249 for (size_t i = 0; i < TEST_ARRAY_LEN(w->addresses); i++)
250 w->addresses[i] = vas_alloc(vas: w->vas, PAGE_2MB, PAGE_2MB);
251}
252
253static void budget_free(void *arg) {
254 struct budget_worker *w = arg;
255 for (size_t i = 0; i < TEST_ARRAY_LEN(w->addresses); i++)
256 if (w->addresses[i])
257 vas_free(vas: w->vas, addr: w->addresses[i], PAGE_2MB);
258}
259
260TEST_DEFINE_INTEGRATION(vas, magazine_per_vas_byte_budget, .min_cores = 2) {
261 struct vas *vas =
262 vas_create(SMP_VAS_BASE, SMP_VAS_BASE + 4 * VAS_CHUNK_SIZE);
263 TEST_ASSERT_NONNULL(vas);
264 struct budget_worker workers[2] = {{.vas = vas}, {.vas = vas}};
265 for (uint32_t cpu = 0; cpu < 2; cpu++) {
266 TEST_ASSERT(vas_run_on(cpu, budget_alloc, &workers[cpu]));
267 for (size_t i = 0; i < TEST_ARRAY_LEN(workers[cpu].addresses); i++)
268 TEST_ASSERT_NE(workers[cpu].addresses[i], 0);
269 }
270 TEST_ASSERT_EQ(atomic_load(&vas->mag_reserved_bytes), VAS_MAG_BYTE_LIMIT);
271 struct handoff extra = {.vas = vas, .size = PAGE_SIZE};
272 TEST_ASSERT(vas_run_on(0, handoff_alloc, &extra));
273 TEST_ASSERT_NE(extra.addr, 0); /* Budget denies caching */
274 TEST_ASSERT_EQ(atomic_load(&vas->mag_reserved_bytes), VAS_MAG_BYTE_LIMIT);
275 TEST_ASSERT(vas_run_on(0, handoff_free, &extra));
276 TEST_ASSERT_EQ(vas->local[0].mag_free_hits, 0);
277 for (uint32_t cpu = 0; cpu < 2; cpu++)
278 TEST_ASSERT(vas_run_on(cpu, budget_free, &workers[cpu]));
279 vas_reclaim(vas);
280 TEST_ASSERT_EQ(atomic_load(&vas->mag_reserved_bytes), 0);
281 TEST_ASSERT_EQ(vas->global.total_free, vas->limit - vas->base);
282 TEST_ASSERT(vas_destroy(vas));
283 return TEST_SUCCESS;
284}
285