1#include "mem/tests/test_internal.h"
2
3#define TLB_MAX_TEST_THREADS 64
4
5static volatile uint64_t tlb_seen[TLB_MAX_TEST_THREADS];
6static atomic_bool tlb_go = false;
7static atomic_uint tlb_threads_done = 0;
8
9static void tlb_reader(void *arg) {
10 size_t id = (size_t) arg;
11
12 while (!atomic_load(&tlb_go))
13 cpu_relax();
14
15 volatile uint64_t *va = thread_get_current()->private;
16 tlb_seen[id] = *va;
17
18 atomic_fetch_add(&tlb_threads_done, 1);
19}
20
21TEST_DECLARE_INTEGRATION(mem, tlb_shootdown_sync,
22 TEST_INTENSITY_CORES(1, 1, 4, "threads/core")) {
23 ABORT_IF_RAM_LOW();
24
25 if (global.core_count < 2) {
26 return TEST_SKIP(TEST_SKIP_NONE);
27 }
28
29 size_t nthreads =
30 ctx->intensity_val ? ctx->intensity_val : global.core_count;
31 if (nthreads > TLB_MAX_TEST_THREADS)
32 nthreads = TLB_MAX_TEST_THREADS;
33
34 atomic_store(&tlb_go, false);
35 atomic_store(&tlb_threads_done, 0);
36
37 paddr_t p1 = pmm_alloc_page();
38 paddr_t p2 = pmm_alloc_page();
39 TEST_ASSERT(p1 && p2);
40
41 void *va = vmm_map_bump(p1, PAGE_SIZE, 0);
42 *(volatile uint64_t *) va = 0xAAAAAAAAAAAAAAAA;
43
44 struct thread *threads[TLB_MAX_TEST_THREADS];
45 for (size_t i = 0; i < nthreads; i++) {
46 threads[i] =
47 thread_spawn_joinable(name: "tlb_reader", entry: tlb_reader, arg: (void *) i);
48 threads[i]->private = va;
49 }
50
51 /* Wait a tick so they spin on tlb_go */
52 time_ms_t start = time_get_ms();
53 while (time_get_ms() - start < 10)
54 scheduler_yield();
55
56 /* Remap under them */
57 vmm_unmap_virt(addr: va, PAGE_SIZE, vflags: VMM_FLAG_NONE);
58 vmm_map_page((vaddr_t) va, p2, PAGE_PRESENT | PAGE_WRITE);
59 *(volatile uint64_t *) va = 0xBBBBBBBBBBBBBBBB;
60
61 /* Sync shootdown -- must complete on all cores before returning */
62 tlb_shootdown(addr: (uintptr_t) va, true);
63
64 /* Release workers to read */
65 atomic_store(&tlb_go, true);
66
67 for (size_t i = 0; i < nthreads; i++)
68 thread_join(t: threads[i]);
69
70 /* Every worker must have seen the new value, not old cached translation */
71 for (size_t i = 0; i < nthreads; i++) {
72 TEST_ASSERT_EQ(tlb_seen[i], 0xBBBBBBBBBBBBBBBB);
73 }
74
75 return TEST_SUCCESS;
76}
77
78TEST_DECLARE_INTEGRATION(mem, tlb_shootdown_async) {
79 ABORT_IF_RAM_LOW();
80
81 paddr_t p1 = pmm_alloc_page();
82 paddr_t p2 = pmm_alloc_page();
83 TEST_ASSERT(p1 && p2);
84
85 void *va = vmm_map_bump(p1, PAGE_SIZE, 0);
86 *(volatile uint64_t *) va = 0x1234;
87
88 vmm_unmap_virt(addr: va, PAGE_SIZE, vflags: VMM_FLAG_NONE);
89 va = vmm_map_bump(p2, PAGE_SIZE, 0);
90 *(volatile uint64_t *) va = 0x5678;
91
92 tlb_shootdown(addr: (uintptr_t) va, false);
93
94 /* Wait for IPIs to land */
95 time_ms_t start = time_get_ms();
96 while (time_get_ms() - start < 100) {
97 if (*(volatile uint64_t *) va == 0x5678)
98 return TEST_SUCCESS;
99 scheduler_yield();
100 }
101
102 test_info("async TLB shootdown did not converge");
103 return TEST_FAIL("async TLB shootdown did not converge within timeout");
104}
105
106TEST_DECLARE_INTEGRATION(mem, tlb_shootdown_flush_all,
107 TEST_INTENSITY(64, 256, 4096)) {
108 ABORT_IF_RAM_LOW();
109
110 size_t iters =
111 ctx->intensity_val ? ctx->intensity_val : (TLB_QUEUE_SIZE * 4);
112
113 paddr_t p = pmm_alloc_page();
114 TEST_ASSERT(p);
115
116 void *va = vmm_map_bump(p, PAGE_SIZE, 0);
117
118 /* Flood shootdown queue */
119 for (size_t i = 0; i < iters; i++) {
120 tlb_shootdown(addr: (uintptr_t) va, false);
121 }
122
123 /* Now do remap */
124 paddr_t p2 = pmm_alloc_page();
125 vmm_unmap_virt(addr: va, PAGE_SIZE, vflags: VMM_FLAG_NONE);
126 va = vmm_map_bump(p2, PAGE_SIZE, 0);
127 *(volatile uint64_t *) va = 0xDEADBEEF;
128
129 tlb_shootdown(addr: (uintptr_t) va, true);
130
131 TEST_ASSERT_EQ(*(volatile uint64_t *) va, 0xDEADBEEF);
132 return TEST_SUCCESS;
133}
134
135#define TLB_CONTENTION_MAX_THREADS 64
136
137static void tlb_spammer(void *) {
138 paddr_t p = pmm_alloc_page();
139 void *va = vmm_map_bump(p, PAGE_SIZE, 0);
140
141 for (int i = 0; i < 1000; i++) {
142 tlb_shootdown(addr: (uintptr_t) va, false);
143 }
144}
145
146TEST_DECLARE_INTEGRATION(mem, tlb_shootdown_contention,
147 TEST_INTENSITY_CORES(1, 1, 2, "threads/core")) {
148 size_t nthreads = ctx->intensity_val ? ctx->intensity_val : 4;
149 if (nthreads > TLB_CONTENTION_MAX_THREADS)
150 nthreads = TLB_CONTENTION_MAX_THREADS;
151
152 struct thread *t[TLB_CONTENTION_MAX_THREADS];
153 for (size_t i = 0; i < nthreads; i++) {
154 t[i] = thread_spawn_joinable(name: "tlb_spammer", entry: tlb_spammer, NULL);
155 TEST_ASSERT_NONNULL(t[i]);
156 }
157
158 for (size_t i = 0; i < nthreads; i++)
159 thread_join(t: t[i]);
160
161 test_info("concurrent shootdown stress completed");
162 return TEST_SUCCESS;
163}
164