1#include "mem/tests/test_internal.h"
2
3#define TLB_MAX_TEST_THREADS ((size_t) 64)
4
5static volatile uint64_t tlb_seen[TLB_MAX_TEST_THREADS];
6static atomic_bool tlb_go = false;
7static atomic_uint tlb_threads_done = 0;
8
9static void tlb_reader(void *arg) {
10 size_t id = (size_t) arg;
11
12 while (!atomic_load(&tlb_go))
13 cpu_pause();
14
15 volatile uint64_t *va = thread_get_current()->private;
16 tlb_seen[id] = *va;
17
18 atomic_inc(&tlb_threads_done);
19}
20
21TEST_DEFINE_INTEGRATION(mem, tlb_shootdown_sync,
22 TEST_INTENSITY_CORES(1, 1, 4, "threads/core"),
23 .min_cores = 2, .min_ram_mib = 8) {
24 size_t nthreads =
25 MIN(ctx->intensity_val ? ctx->intensity_val : global.core_count,
26 TLB_MAX_TEST_THREADS);
27
28 atomic_store(&tlb_go, false);
29 atomic_store(&tlb_threads_done, 0);
30
31 paddr_t p1 = pmm_alloc_page();
32 paddr_t p2 = pmm_alloc_page();
33 TEST_ASSERT(p1 && p2);
34
35 void *va = vmm_map_bump(p1, PAGE_SIZE);
36 *(volatile uint64_t *) va = 0xAAAAAAAAAAAAAAAA;
37
38 struct thread *threads[TLB_MAX_TEST_THREADS];
39 for (size_t i = 0; i < nthreads; i++) {
40 threads[i] = thread_spawn("tlb_reader", tlb_reader, .arg = (void *) i,
41 .private = va, .joinable = true);
42 }
43
44 /* Wait a tick so they spin on tlb_go */
45 time_ms_t start = time_get_ms();
46 while (time_get_ms() - start < 10)
47 scheduler_yield();
48
49 /* Remap under them */
50 vmm_unmap(va, PAGE_SIZE);
51 vmm_map_page((vaddr_t) va, p2, .page_flags = PAGE_PRESENT | PAGE_WRITE);
52 *(volatile uint64_t *) va = 0xBBBBBBBBBBBBBBBB;
53
54 /* Sync shootdown -- must complete on all cores before returning */
55 tlb_shootdown(.payload.addr = (uintptr_t) va);
56
57 /* Release workers to read */
58 atomic_store(&tlb_go, true);
59
60 for (size_t i = 0; i < nthreads; i++)
61 thread_join(t: threads[i]);
62
63 /* Every worker must have seen the new value, not old cached translation */
64 for (size_t i = 0; i < nthreads; i++) {
65 TEST_ASSERT_EQ(tlb_seen[i], 0xBBBBBBBBBBBBBBBB);
66 }
67
68 return TEST_SUCCESS;
69}
70
71TEST_DEFINE_INTEGRATION(mem, tlb_shootdown_async, .min_ram_mib = 8) {
72 paddr_t p1 = pmm_alloc_page();
73 paddr_t p2 = pmm_alloc_page();
74 TEST_ASSERT(p1 && p2);
75
76 void *va = vmm_map_bump(p1, PAGE_SIZE);
77 *(volatile uint64_t *) va = 0x1234;
78
79 vmm_unmap(va, PAGE_SIZE);
80 va = vmm_map_bump(p2, PAGE_SIZE);
81 *(volatile uint64_t *) va = 0x5678;
82
83 tlb_shootdown(.mode = TLB_MODE_ASYNC, .payload.addr = (uintptr_t) va);
84
85 /* Wait for IPIs to land */
86 time_ms_t start = time_get_ms();
87 while (time_get_ms() - start < 100) {
88 if (*(volatile uint64_t *) va == 0x5678)
89 return TEST_SUCCESS;
90 scheduler_yield();
91 }
92
93 test_info("async TLB shootdown did not converge");
94 return TEST_FAIL("async TLB shootdown did not converge within timeout");
95}
96
97TEST_DEFINE_INTEGRATION(mem, tlb_shootdown_flush_all,
98 TEST_INTENSITY(64, 256, 4096), .min_ram_mib = 8) {
99 size_t iters =
100 ctx->intensity_val ? ctx->intensity_val : (TLB_QUEUE_SIZE * 4);
101
102 paddr_t p = pmm_alloc_page();
103 TEST_ASSERT(p);
104
105 void *va = vmm_map_bump(p, PAGE_SIZE);
106
107 /* Flood shootdown queue */
108 for (size_t i = 0; i < iters; i++) {
109 tlb_shootdown(.mode = TLB_MODE_ASYNC, .payload.addr = (uintptr_t) va);
110 }
111
112 /* Now do remap */
113 paddr_t p2 = pmm_alloc_page();
114 vmm_unmap(va, PAGE_SIZE);
115 va = vmm_map_bump(p2, PAGE_SIZE);
116 *(volatile uint64_t *) va = 0xDEADBEEF;
117
118 tlb_shootdown(.payload.addr = (uintptr_t) va, );
119
120 TEST_ASSERT_EQ(*(volatile uint64_t *) va, 0xDEADBEEF);
121 return TEST_SUCCESS;
122}
123
124#define TLB_CONTENTION_MAX_THREADS ((size_t) 64)
125
126static void tlb_spammer(void *arg) {
127 cc_ignore(arg);
128 paddr_t p = pmm_alloc_page();
129 void *va = vmm_map_bump(p, PAGE_SIZE);
130
131 for (int i = 0; i < 1000; i++) {
132 tlb_shootdown(.mode = TLB_MODE_ASYNC, .payload.addr = (uintptr_t) va);
133 }
134}
135
136TEST_DEFINE_INTEGRATION(mem, mt_tlb_shootdown_contention,
137 TEST_INTENSITY_CORES(1, 1, 2, "threads/core")) {
138 size_t nthreads = MIN(ctx->intensity_val ? ctx->intensity_val : 4,
139 TLB_CONTENTION_MAX_THREADS);
140
141 struct thread *t[TLB_CONTENTION_MAX_THREADS];
142 for (size_t i = 0; i < nthreads; i++) {
143 t[i] = thread_spawn("tlb_spammer", tlb_spammer, .joinable = true);
144 TEST_ASSERT_NONNULL(t[i]);
145 }
146
147 for (size_t i = 0; i < nthreads; i++)
148 thread_join(t: t[i]);
149
150 test_info("concurrent shootdown stress completed");
151 return TEST_SUCCESS;
152}
153