1#ifdef DEBUG_LOCK_CHK
2
3#include <asm.h>
4#include <console/panic.h>
5#include <console/printf.h>
6#include <irq/irq.h>
7#include <kassert.h>
8#include <smp/core.h>
9#include <smp/percpu.h>
10#include <stdatomic.h>
11#include <string.h>
12#include <thread/thread.h>
13
14#include "lock_chk_internal.h"
15
16static struct lock_chk_graph lock_chk_global_graph;
17static _Atomic enum lock_chk_engine_state lock_chk_deep_state =
18 LOCK_CHK_INACTIVE;
19PERCPU_DECLARE(lock_chk_recursion_depth, uint8_t, NULL);
20
21struct lock_chk_guard {
22 uint8_t *depth;
23 bool irqs_enabled;
24};
25
26static struct lock_chk_guard lock_chk_enter(void) {
27 struct lock_chk_guard guard = {
28 .irqs_enabled = are_interrupts_enabled(),
29 };
30 disable_interrupts();
31 guard.depth = PERCPU_PTR(TOPC_IFLAG, lock_chk_recursion_depth);
32 if (*guard.depth != 0)
33 panic("Recursive lock validator entry");
34 *guard.depth = 1;
35 return guard;
36}
37
38static void lock_chk_leave(const struct lock_chk_guard *guard) {
39 kassert(*guard->depth == 1);
40 *guard->depth = 0;
41 if (guard->irqs_enabled)
42 enable_interrupts();
43}
44
45static bool lock_chk_deep_is_active(void) {
46 return atomic_load_explicit(&lock_chk_deep_state, memory_order_acquire) ==
47 LOCK_CHK_ACTIVE;
48}
49
50static enum lock_chk_result
51lock_chk_validate_acquire(const struct lock_chk_acquire_request *request,
52 struct lock_chk_thread_data *thread_data) {
53 if (request->type == LOCK_CHK_TYPE_MUTEX ||
54 request->type == LOCK_CHK_TYPE_MUTEX_SIMPLE ||
55 request->type == LOCK_CHK_TYPE_RWLOCK) {
56 if (request->in_nmi || request->in_irq || irq_in_interrupt() ||
57 request->prev_irql > IRQL_APC_LEVEL)
58 return LOCK_CHK_RESULT_BAD_CONTEXT;
59 } else {
60 /* Spinlock / Qspinlock */
61 if (request->in_nmi)
62 return LOCK_CHK_RESULT_BAD_CONTEXT;
63
64 if (!request->irq_safe && !request->raw_operation) {
65 if (request->in_irq || irq_in_interrupt())
66 return LOCK_CHK_RESULT_BAD_CONTEXT;
67 }
68 }
69
70 if (request->subclass >= LOCK_CHK_MAX_SUBCLASSES)
71 return LOCK_CHK_RESULT_INTERNAL;
72
73 if (thread_data->depth == LOCK_CHK_MAX_HELD_LOCKS)
74 return LOCK_CHK_RESULT_HELD_CAPACITY;
75
76 for (uint8_t i = 0; i < thread_data->depth; i++)
77 if (thread_data->held[i].instance == request->instance)
78 return LOCK_CHK_RESULT_RECURSION;
79
80 return LOCK_CHK_RESULT_OK;
81}
82
83void lock_chk_deep_activate(void) {
84 kassert(PERCPU_READY(lock_chk_recursion_depth));
85 lock_chk_graph_init(&lock_chk_global_graph);
86 atomic_store_explicit(&lock_chk_deep_state, LOCK_CHK_ACTIVE,
87 memory_order_release);
88}
89
90static void lock_chk_degrade(void) {
91 atomic_store_explicit(&lock_chk_deep_state, LOCK_CHK_DEGRADED,
92 memory_order_release);
93}
94
95static void
96lock_chk_handle_graph_failure(enum lock_chk_result result,
97 const struct lock_chk_acquire_request *request,
98 struct lock_chk_failure *failure) {
99 if (result == LOCK_CHK_RESULT_BAD_CONTEXT) {
100 *failure = (struct lock_chk_failure){
101 .kind = LOCK_CHK_FAIL_CONTEXT,
102 .site = request->site,
103 .class = request->map ? request->map->class : NULL,
104 .instance = request->instance,
105 .type = request->type,
106 .mode = request->mode,
107 .subclass = request->subclass,
108 };
109 lock_chk_fail(failure, "Lock class changed interrupt-safety context");
110 return;
111 }
112
113 if (result == LOCK_CHK_RESULT_NODE_CAPACITY) {
114 *failure = (struct lock_chk_failure){
115 .kind = LOCK_CHK_FAIL_CAPACITY,
116 .site = request->site,
117 .class = request->map ? request->map->class : NULL,
118 .instance = request->instance,
119 .type = request->type,
120 .mode = request->mode,
121 .subclass = request->subclass,
122 .capacity_pool = "nodes",
123 .capacity_used = LOCK_CHK_MAX_NODES,
124 .capacity_limit = LOCK_CHK_MAX_NODES,
125 };
126 lock_chk_fail(failure, "Node capacity exhausted (%u/%u)",
127 LOCK_CHK_MAX_NODES, LOCK_CHK_MAX_NODES);
128 } else if (result == LOCK_CHK_RESULT_CYCLE ||
129 result == LOCK_CHK_RESULT_EDGE_CAPACITY) {
130 lock_chk_report_failure(failure);
131 } else {
132 *failure = (struct lock_chk_failure){
133 .kind = LOCK_CHK_FAIL_UNINITIALIZED,
134 .site = request->site,
135 .instance = request->instance,
136 .type = request->type,
137 .mode = request->mode,
138 .subclass = request->subclass,
139 };
140 lock_chk_fail(failure, "Internal graph preparation failure (%u)",
141 (unsigned) result);
142 return;
143 }
144
145 if ((result == LOCK_CHK_RESULT_NODE_CAPACITY ||
146 result == LOCK_CHK_RESULT_EDGE_CAPACITY) &&
147 !lock_chk_capacity_should_panic())
148 lock_chk_degrade();
149}
150
151static void lock_chk_handle_validation_failure(
152 enum lock_chk_result result,
153 const struct lock_chk_acquire_request *request) {
154 struct lock_chk_failure fail = {
155 .site = request->site,
156 .class = request->map ? request->map->class : NULL,
157 .instance = request->instance,
158 .type = request->type,
159 .mode = request->mode,
160 .subclass = request->subclass,
161 };
162
163 if (result == LOCK_CHK_RESULT_BAD_CONTEXT) {
164 fail.kind = LOCK_CHK_FAIL_CONTEXT;
165 lock_chk_fail(
166 &fail, "Invalid acquisition context (nmi=%d, irq=%d, irql=%u)",
167 request->in_nmi, request->in_irq, (unsigned) request->prev_irql);
168 return;
169 }
170 if (result == LOCK_CHK_RESULT_RECURSION) {
171 fail.kind = LOCK_CHK_FAIL_RECURSION;
172 lock_chk_fail(&fail, "Recursive lock acquisition (instance %p)",
173 request->instance);
174 return;
175 }
176 if (result == LOCK_CHK_RESULT_HELD_CAPACITY) {
177 fail.kind = LOCK_CHK_FAIL_CAPACITY;
178 fail.capacity_pool = "per-thread held";
179 fail.capacity_used = LOCK_CHK_MAX_HELD_LOCKS;
180 fail.capacity_limit = LOCK_CHK_MAX_HELD_LOCKS;
181 lock_chk_fail(&fail, "Per-thread held capacity exhausted (%u/%u)",
182 LOCK_CHK_MAX_HELD_LOCKS, LOCK_CHK_MAX_HELD_LOCKS);
183 if (!lock_chk_capacity_should_panic())
184 lock_chk_degrade();
185 return;
186 }
187
188 fail.kind = LOCK_CHK_FAIL_UNINITIALIZED;
189 lock_chk_fail(&fail, "Subclass out of range (%u >= %u)", request->subclass,
190 LOCK_CHK_MAX_SUBCLASSES);
191}
192
193void lock_chk_before_acquire(struct lock_chk_acquire_token *token,
194 const struct lock_chk_acquire_request *request) {
195 *token = (struct lock_chk_acquire_token){0};
196 if (!lock_chk_deep_is_active() || request->flags == LOCK_UNCHKD)
197 return;
198
199 struct lock_chk_guard guard = lock_chk_enter();
200
201 struct thread *thread = thread_get_current();
202 if (thread == NULL) {
203 struct lock_chk_failure fail = {
204 .kind = LOCK_CHK_FAIL_CONTEXT,
205 .site = request->site,
206 .type = request->type,
207 .mode = request->mode,
208 .subclass = request->subclass,
209 };
210 lock_chk_fail(&fail, "Checked acquisition without a current thread");
211 goto out;
212 }
213
214 struct lock_chk_thread_data *thread_data = &thread->lock_chk;
215 enum lock_chk_result result =
216 lock_chk_validate_acquire(request, thread_data);
217 if (result != LOCK_CHK_RESULT_OK) {
218 lock_chk_handle_validation_failure(result, request);
219 goto out;
220 }
221
222 struct lock_chk_node *node = NULL;
223 if ((request->flags & LOCK_CHKD_ORDER) != 0) {
224 struct lock_chk_failure fail = {0};
225 result = lock_chk_graph_prepare_acquire(
226 &lock_chk_global_graph, request->map, request->subclass, request,
227 thread_data, &node, &fail);
228 if (result != LOCK_CHK_RESULT_OK) {
229 lock_chk_handle_graph_failure(result, request, &fail);
230 goto out;
231 }
232 }
233
234 token->node = node;
235 token->context_node = node;
236 token->request = request;
237 token->thread_data = thread_data;
238 token->active = true;
239
240out:
241 lock_chk_leave(&guard);
242}
243
244void lock_chk_acquired(struct lock_chk_acquire_token *token) {
245 if (!token->active)
246 return;
247
248 struct lock_chk_guard guard = lock_chk_enter();
249
250 const struct lock_chk_acquire_request *request = token->request;
251 struct lock_chk_thread_data *thread_data = token->thread_data;
252 kassert(thread_data->depth < LOCK_CHK_MAX_HELD_LOCKS);
253
254 thread_data->held[thread_data->depth++] = (struct lock_chk_held){
255 .node = token->node,
256 .instance = request->instance,
257 .acquire_site = request->site,
258 .acquire_tsc = rdtsc_ordered(),
259 .prev_irql = request->prev_irql,
260 .cpu = smp_id(TOPC_IFLAG),
261 .flags = request->flags,
262 .type = request->type,
263 .mode = request->mode,
264 .subclass = request->subclass,
265 .trylock = request->wait_kind == LOCK_CHK_WAIT_TRY,
266 .raw_operation = request->raw_operation,
267 };
268
269 if ((request->flags & LOCK_CHKD_THREAD) != 0) {
270 thread_data->thread_checked_depth++;
271 if (!request->raw_operation && (request->type == LOCK_CHK_TYPE_SPIN ||
272 request->type == LOCK_CHK_TYPE_QSPIN))
273 thread_data->thread_checked_spin_depth++;
274 }
275
276 token->active = false;
277
278 lock_chk_leave(&guard);
279}
280
281void lock_chk_cancel(struct lock_chk_acquire_token *token) {
282 token->active = false;
283}
284
285void lock_chk_before_release(struct lock_chk_release_token *token,
286 const struct lock_chk_release_request *request) {
287 *token = (struct lock_chk_release_token){0};
288 if (!lock_chk_deep_is_active() || request->flags == LOCK_UNCHKD)
289 return;
290
291 struct lock_chk_guard guard = lock_chk_enter();
292
293 struct thread *thread = thread_get_current();
294 if (thread == NULL) {
295 struct lock_chk_failure fail = {
296 .kind = LOCK_CHK_FAIL_CONTEXT,
297 .site = request->site,
298 .type = request->type,
299 .mode = request->mode,
300 };
301 lock_chk_fail(&fail, "Checked release without a current thread");
302 goto out;
303 }
304
305 struct lock_chk_thread_data *thread_data = &thread->lock_chk;
306 for (uint8_t i = 0; i < thread_data->depth; i++) {
307 struct lock_chk_held *held = &thread_data->held[i];
308 if (held->instance != request->instance ||
309 held->type != request->type || held->mode != request->mode)
310 continue;
311
312 token->thread_data = thread_data;
313 token->instance = request->instance;
314 token->held_index = i;
315 token->active = true;
316 goto out;
317 }
318
319 struct lock_chk_failure fail = {
320 .kind = LOCK_CHK_FAIL_RELEASE,
321 .site = request->site,
322 .class = request->map ? request->map->class : NULL,
323 .instance = request->instance,
324 .type = request->type,
325 .mode = request->mode,
326 };
327 lock_chk_fail(&fail, "Foreign or unbalanced lock release (instance %p)",
328 request->instance);
329
330out:
331 lock_chk_leave(&guard);
332}
333
334void lock_chk_released(struct lock_chk_release_token *token) {
335 if (!token->active)
336 return;
337
338 struct lock_chk_guard guard = lock_chk_enter();
339
340 struct lock_chk_thread_data *thread_data = token->thread_data;
341 kassert(token->held_index < thread_data->depth);
342 struct lock_chk_held released = thread_data->held[token->held_index];
343 kassert(released.instance == token->instance);
344
345 if ((released.flags & LOCK_CHKD_THREAD) != 0) {
346 kassert(thread_data->thread_checked_depth != 0);
347 thread_data->thread_checked_depth--;
348 if (!released.raw_operation && (released.type == LOCK_CHK_TYPE_SPIN ||
349 released.type == LOCK_CHK_TYPE_QSPIN)) {
350 kassert(thread_data->thread_checked_spin_depth != 0);
351 thread_data->thread_checked_spin_depth--;
352 }
353 }
354
355 for (uint8_t i = token->held_index; i + 1 < thread_data->depth; i++)
356 thread_data->held[i] = thread_data->held[i + 1];
357
358 thread_data->depth--;
359 token->active = false;
360
361 lock_chk_leave(&guard);
362}
363
364bool lock_chk_assert_held_deep(struct lock_chk_lock *lock,
365 enum lock_chk_mode mode, bool want_held,
366 const struct lock_chk_site *site) {
367 bool mode_specific = mode != LOCK_CHK_MODE_IGNORED;
368 void *instance = lock->instance;
369 enum lock_chk_type type = lock->type;
370 if (!lock_chk_deep_is_active())
371 return false;
372
373 struct lock_chk_guard guard = lock_chk_enter();
374 bool handled = true;
375
376 struct thread *thread = thread_get_current();
377 if (thread == NULL) {
378 handled = false;
379 goto out;
380 }
381
382 struct lock_chk_thread_data *thread_data = &thread->lock_chk;
383 bool found = false;
384 enum lock_chk_mode found_mode = mode;
385 for (uint8_t i = 0; i < thread_data->depth; i++) {
386 struct lock_chk_held *held = &thread_data->held[i];
387 if (held->instance != instance || held->type != type)
388 continue;
389 if (mode_specific && held->mode != mode)
390 continue;
391 found = true;
392 found_mode = held->mode;
393 break;
394 }
395
396 if (found != want_held) {
397 struct lock_chk_failure fail = {
398 .kind = want_held ? LOCK_CHK_FAIL_NOT_HELD
399 : LOCK_CHK_FAIL_UNEXPECTED_HELD,
400 .site = site,
401 .class = lock ? lock->map.class : NULL,
402 .instance = instance,
403 .type = type,
404 .mode = found_mode,
405 };
406 lock_chk_fail(&fail,
407 want_held ? "Lock assumed held but not held by current "
408 "thread (instance %p)"
409 : "Lock assumed not held but held by current "
410 "thread (instance %p)",
411 instance);
412 }
413
414out:
415 lock_chk_leave(&guard);
416 return handled;
417}
418
419void lock_chk_assert_schedulable(const struct lock_chk_site *site) {
420 if (!lock_chk_deep_is_active())
421 return;
422
423 struct lock_chk_guard guard = lock_chk_enter();
424 struct thread *thread = thread_get_current();
425 if (thread != NULL && thread->lock_chk.thread_checked_spin_depth != 0) {
426 struct lock_chk_failure fail = {
427 .kind = LOCK_CHK_FAIL_CONTEXT,
428 .site = site,
429 };
430 lock_chk_fail(&fail,
431 "Scheduling while holding a thread-checked spinlock");
432 }
433 lock_chk_leave(&guard);
434}
435
436void lock_chk_thread_init(struct thread *thread) {
437 memset(&thread->lock_chk, 0, sizeof(thread->lock_chk));
438}
439
440void lock_chk_thread_exit(struct thread *thread) {
441 if (!lock_chk_deep_is_active())
442 return;
443
444 struct lock_chk_guard guard = lock_chk_enter();
445 if (thread->lock_chk.thread_checked_depth != 0) {
446 struct lock_chk_failure fail = {
447 .kind = LOCK_CHK_FAIL_THREAD_EXIT,
448 };
449 lock_chk_fail(&fail, "Thread %p exited with %u checked locks held",
450 thread, thread->lock_chk.thread_checked_depth);
451 }
452 lock_chk_leave(&guard);
453}
454
455#endif /* DEBUG_LOCK_CHK */
456