1#include "mem/tests/test_internal.h"
2
3#define DP_PAGES 16
4#define DP_STRIDE (PAGE_SIZE / sizeof(uint64_t))
5#define DP_MAX_BUFS ((size_t) 8)
6#define DP_MAX_THREADS ((size_t) 64)
7
8struct dp_worker {
9 atomic_uint64_t **bufs; /* nbuf demand buffers, counter at page head */
10 size_t nbuf;
11 size_t pages;
12 atomic_uint *done;
13};
14
15static void dp_hammer(void *arg) {
16 struct dp_worker *w = arg;
17
18 /* touch every page of every buffer */
19 for (size_t b = 0; b < w->nbuf; b++)
20 for (size_t p = 0; p < w->pages; p++)
21 atomic_inc_rlx(&w->bufs[b][p * DP_STRIDE]);
22
23 atomic_inc(w->done);
24}
25
26static bool dp_alloc_bufs(atomic_uint64_t **bufs, size_t nbuf, size_t pages) {
27 for (size_t b = 0; b < nbuf; b++) {
28 bufs[b] = page_alloc_demand(pages, ALLOC_ZERO);
29 if (!bufs[b]) {
30 for (size_t j = 0; j < b; j++)
31 page_free((void *) bufs[j], pages);
32 return false;
33 }
34 }
35 return true;
36}
37
38/* every page was faulted in by workers, so all frames are present */
39static void dp_free_bufs(atomic_uint64_t **bufs, size_t nbuf, size_t pages) {
40 for (size_t b = 0; b < nbuf; b++)
41 page_free((void *) bufs[b], pages);
42}
43
44static bool dp_verify(atomic_uint64_t **bufs, size_t nbuf, size_t pages,
45 uint64_t expect) {
46 for (size_t b = 0; b < nbuf; b++)
47 for (size_t p = 0; p < pages; p++)
48 if (atomic_load(&bufs[b][p * DP_STRIDE]) != expect)
49 return false;
50
51 return true;
52}
53
54/* Spawn nthreads workers over shared buffer set. single_core pins them,
55 * and we can test all on one CPU vs spread out */
56static void dp_spawn(struct thread **t, size_t nthreads, struct dp_worker *w,
57 bool single_core) {
58 for (size_t i = 0; i < nthreads; i++) {
59 uint64_t core = single_core ? 0 : (i % global.core_count);
60 t[i] = enforce(thread_spawn("dp_hammer", dp_hammer, .arg = w,
61 .on_cpu = core, .joinable = true,
62 .pinned = single_core));
63 }
64}
65
66static void dp_join(struct thread **t, size_t nthreads) {
67 for (size_t i = 0; i < nthreads; i++)
68 thread_join(t: t[i]);
69}
70
71/* 1 buffer, N threads, N CPUs = many CPUs racing for same PTEs */
72TEST_DEFINE_INTEGRATION(mem, mt_demand_single_buf,
73 TEST_INTENSITY_CORES(1, 1, 4, "threads/core"),
74 .min_cores = 2, .min_ram_mib = 8) {
75 const size_t pages = DP_PAGES, nbuf = 1;
76 size_t nthreads =
77 MIN(ctx->intensity_val ? ctx->intensity_val : global.core_count,
78 DP_MAX_THREADS);
79
80 atomic_uint64_t *bufs[1];
81 TEST_ASSERT(dp_alloc_bufs(bufs, nbuf, pages));
82
83 atomic_uint done = 0;
84 struct dp_worker w = {bufs, nbuf, pages, &done};
85 struct thread *t[DP_MAX_THREADS];
86 dp_spawn(t, nthreads, w: &w, /*single_core=*/false);
87
88 dp_join(t, nthreads);
89
90 TEST_ASSERT_EQ(atomic_load(&done), nthreads);
91 TEST_ASSERT(dp_verify(bufs, nbuf, pages, nthreads));
92 dp_free_bufs(bufs, nbuf, pages);
93 return TEST_SUCCESS;
94}
95
96/* N buffers, M threads (M > N), N CPUs = contention spread over multiple
97 * regions */
98TEST_DEFINE_INTEGRATION(mem, mt_demand_multi_buf,
99 TEST_INTENSITY_CORES(1, 2, 4, "threads/core"),
100 .min_cores = 2, .min_ram_mib = 8) {
101 const size_t pages = DP_PAGES;
102 size_t nbuf = MIN(global.core_count, DP_MAX_BUFS);
103 size_t nthreads = MIN(ctx->intensity_val ? ctx->intensity_val : (2 * nbuf),
104 DP_MAX_THREADS); /* M > N */
105
106 atomic_uint64_t *bufs[DP_MAX_BUFS];
107 TEST_ASSERT(dp_alloc_bufs(bufs, nbuf, pages));
108
109 atomic_uint done = 0;
110 struct dp_worker w = {bufs, nbuf, pages, &done};
111 struct thread *t[DP_MAX_THREADS];
112 dp_spawn(t, nthreads, w: &w, /*single_core=*/false);
113
114 dp_join(t, nthreads);
115
116 TEST_ASSERT_EQ(atomic_load(&done), nthreads);
117 TEST_ASSERT(dp_verify(bufs, nbuf, pages, nthreads));
118 dp_free_bufs(bufs, nbuf, pages);
119 return TEST_SUCCESS;
120}
121