1#include "mem/tests/test_internal.h"
2
3#define DP_PAGES 16
4#define DP_STRIDE (PAGE_SIZE / sizeof(uint64_t))
5#define DP_MAX_BUFS 8
6#define DP_MAX_THREADS 64
7
8struct dp_worker {
9 _Atomic uint64_t **bufs; /* nbuf demand buffers, counter at page head */
10 size_t nbuf;
11 size_t pages;
12 atomic_uint *done;
13};
14
15static void dp_hammer(void *arg) {
16 struct dp_worker *w = arg;
17
18 /* touch every page of every buffer */
19 for (size_t b = 0; b < w->nbuf; b++)
20 for (size_t p = 0; p < w->pages; p++)
21 atomic_fetch_add_explicit(&w->bufs[b][p * DP_STRIDE], 1,
22 memory_order_relaxed);
23
24 atomic_fetch_add(w->done, 1);
25}
26
27static bool dp_alloc_bufs(_Atomic uint64_t **bufs, size_t nbuf, size_t pages) {
28 for (size_t b = 0; b < nbuf; b++) {
29 bufs[b] = page_alloc_demand(pages, ALLOC_FLAGS_ZERO);
30 if (!bufs[b]) {
31 for (size_t j = 0; j < b; j++)
32 page_free((void *) bufs[j], pages);
33 return false;
34 }
35 }
36 return true;
37}
38
39/* every page was faulted in by workers, so all frames are present */
40static void dp_free_bufs(_Atomic uint64_t **bufs, size_t nbuf, size_t pages) {
41 for (size_t b = 0; b < nbuf; b++)
42 page_free((void *) bufs[b], pages);
43}
44
45static bool dp_verify(_Atomic uint64_t **bufs, size_t nbuf, size_t pages,
46 uint64_t expect) {
47 for (size_t b = 0; b < nbuf; b++)
48 for (size_t p = 0; p < pages; p++)
49 if (atomic_load(&bufs[b][p * DP_STRIDE]) != expect)
50 return false;
51
52 return true;
53}
54
55/* Spawn nthreads workers over shared buffer set. single_core pins them,
56 * and we can test all on one CPU vs spread out */
57static void dp_spawn(struct thread **t, size_t nthreads, struct dp_worker *w,
58 bool single_core) {
59 for (size_t i = 0; i < nthreads; i++) {
60 uint64_t core = single_core ? 0 : (i % global.core_count);
61 /* Join reference is what makes thread_pin safe, worker may
62 * have already exited by then */
63 t[i] = kassert(
64 thread_spawn_joinable_on_core("dp_hammer", dp_hammer, w, core));
65 if (single_core)
66 thread_pin(t: t[i]);
67 }
68}
69
70static void dp_join(struct thread **t, size_t nthreads) {
71 for (size_t i = 0; i < nthreads; i++)
72 thread_join(t: t[i]);
73}
74
75/* 1 buffer, N threads, N CPUs = many CPUs racing for same PTEs */
76TEST_DECLARE_INTEGRATION(mem, demand_single_buf_smp,
77 TEST_INTENSITY_CORES(1, 1, 4, "threads/core")) {
78 ABORT_IF_RAM_LOW();
79
80 if (global.core_count < 2) {
81 return TEST_SKIP(TEST_SKIP_NONE);
82 }
83
84 const size_t pages = DP_PAGES, nbuf = 1;
85 size_t nthreads =
86 ctx->intensity_val ? ctx->intensity_val : global.core_count;
87 if (nthreads > DP_MAX_THREADS)
88 nthreads = DP_MAX_THREADS;
89
90 _Atomic uint64_t *bufs[1];
91 TEST_ASSERT(dp_alloc_bufs(bufs, nbuf, pages));
92
93 atomic_uint done = 0;
94 struct dp_worker w = {bufs, nbuf, pages, &done};
95 struct thread *t[DP_MAX_THREADS];
96 dp_spawn(t, nthreads, w: &w, /*single_core=*/false);
97
98 dp_join(t, nthreads);
99
100 TEST_ASSERT_EQ(atomic_load(&done), nthreads);
101 TEST_ASSERT(dp_verify(bufs, nbuf, pages, nthreads));
102 dp_free_bufs(bufs, nbuf, pages);
103 return TEST_SUCCESS;
104}
105
106/* N buffers, M threads (M > N), N CPUs = contention spread over multiple
107 * regions */
108TEST_DECLARE_INTEGRATION(mem, demand_multi_buf_smp,
109 TEST_INTENSITY_CORES(1, 2, 4, "threads/core")) {
110 ABORT_IF_RAM_LOW();
111
112 if (global.core_count < 2) {
113 return TEST_SKIP(TEST_SKIP_NONE);
114 }
115
116 const size_t pages = DP_PAGES;
117 size_t nbuf = global.core_count;
118 if (nbuf > DP_MAX_BUFS)
119 nbuf = DP_MAX_BUFS;
120 size_t nthreads =
121 ctx->intensity_val ? ctx->intensity_val : (2 * nbuf); /* M > N */
122 if (nthreads > DP_MAX_THREADS)
123 nthreads = DP_MAX_THREADS;
124
125 _Atomic uint64_t *bufs[DP_MAX_BUFS];
126 TEST_ASSERT(dp_alloc_bufs(bufs, nbuf, pages));
127
128 atomic_uint done = 0;
129 struct dp_worker w = {bufs, nbuf, pages, &done};
130 struct thread *t[DP_MAX_THREADS];
131 dp_spawn(t, nthreads, w: &w, /*single_core=*/false);
132
133 dp_join(t, nthreads);
134
135 TEST_ASSERT_EQ(atomic_load(&done), nthreads);
136 TEST_ASSERT(dp_verify(bufs, nbuf, pages, nthreads));
137 dp_free_bufs(bufs, nbuf, pages);
138 return TEST_SUCCESS;
139}
140