1#ifdef DEBUG_LOCK_CHK
2
3#include <asm.h>
4#include <atomic.h>
5#include <console/panic.h>
6#include <console/printf.h>
7#include <enforce.h>
8#include <irq/irq.h>
9#include <smp/core.h>
10#include <smp/percpu.h>
11#include <string.h>
12#include <thread/thread.h>
13
14#include "internal.h"
15
16PERCPU_DEFINE(uint8_t, lock_chk_recursion_depth);
17
18struct lock_chk_guard lock_chk_enter(void) {
19 struct lock_chk_guard guard = {
20 .irqs_enabled = irq_disable_save(),
21 };
22 guard.depth = PERCPU_PTR(TOPC_IFLAG, lock_chk_recursion_depth);
23 if (*guard.depth != 0)
24 panic("Recursive lock validator entry");
25 *guard.depth = 1;
26 return guard;
27}
28
29void lock_chk_leave(const struct lock_chk_guard *guard) {
30 enforce(*guard->depth == 1);
31 *guard->depth = 0;
32 irq_restore(guard->irqs_enabled);
33}
34
35static enum lock_chk_result
36validate_acq(const struct lock_chk_acq_req *request,
37 struct lock_chk_thread_data *thread_data) {
38 if (request->in_nmi)
39 return LOCK_CHK_RESULT_BAD_CONTEXT;
40
41 if (lock_chk_type_is_blocking(request->lock.type)) {
42 if (request->in_irq || irq_in_interrupt() ||
43 request->prev_irql > IRQL_APC_LEVEL)
44 return LOCK_CHK_RESULT_BAD_CONTEXT;
45 } else {
46 /* Spinlock / Qspinlock */
47 bool irq_safe = lock_op_irq_safe(request->op_flags);
48 bool raw_op = request->op_flags & LOCK_OP_RAW;
49 if (!irq_safe && !raw_op) {
50 if (request->in_irq || irq_in_interrupt())
51 return LOCK_CHK_RESULT_BAD_CONTEXT;
52 }
53 }
54
55 if (request->subclass >= LOCK_CHK_MAX_SUBCLASSES)
56 return LOCK_CHK_RESULT_INTERNAL;
57
58 if (thread_data->depth == LOCK_CHK_MAX_HELD_LOCKS)
59 return LOCK_CHK_RESULT_HELD_CAPACITY;
60
61 if (lock_chk_find_held(thread_data, request->lock.instance, 0,
62 /*match_type=*/false, LOCK_CHK_MODE_IGNORED,
63 /*match_mode=*/false) != NULL)
64 return LOCK_CHK_RESULT_RECURSION;
65
66 return LOCK_CHK_RESULT_OK;
67}
68
69void lock_chk_deep_activate(void) {
70 enforce(PERCPU_READY(lock_chk_recursion_depth), UNINITIALIZED);
71 lock_chk_graph_init(&lock_chk_global.graph);
72 atomic_store_release(&lock_chk_global.deep, LOCK_CHK_ACTIVE);
73}
74
75static void lock_chk_degrade(void) {
76 atomic_store_release(&lock_chk_global.deep, LOCK_CHK_DEGRADED);
77}
78
79static void lock_chk_note_capacity_exhausted(enum lock_chk_result result) {
80 if ((result == LOCK_CHK_RESULT_NODE_CAPACITY ||
81 result == LOCK_CHK_RESULT_EDGE_CAPACITY) &&
82 !lock_chk_global.panic_on_exhaustion)
83 lock_chk_degrade();
84}
85
86static void lock_chk_do_graph_fail(enum lock_chk_result result,
87 const struct lock_chk_acq_req *request,
88 struct lock_chk_fault *fault) {
89 enforce(result != LOCK_CHK_RESULT_CYCLE &&
90 result != LOCK_CHK_RESULT_EDGE_CAPACITY);
91
92 if (result == LOCK_CHK_RESULT_BAD_CONTEXT) {
93 *fault = lock_chk_fault_from_req(LOCK_CHK_FAIL_CONTEXT, request);
94 lock_chk_fail(fault, "%s %p (class interrupt safety changed)",
95 lock_chk_type_to_str(request->lock.type),
96 request->lock.instance);
97 return;
98 }
99
100 if (result == LOCK_CHK_RESULT_NODE_CAPACITY) {
101 *fault = lock_chk_fault_from_req(LOCK_CHK_FAIL_CAPACITY, request);
102 fault->capacity_pool = "nodes";
103 fault->capacity_used = LOCK_CHK_MAX_NODES;
104 fault->capacity_limit = LOCK_CHK_MAX_NODES;
105 lock_chk_fail(fault, "%s %p (nodes %u/%u)",
106 lock_chk_type_to_str(request->lock.type),
107 request->lock.instance, LOCK_CHK_MAX_NODES,
108 LOCK_CHK_MAX_NODES);
109 lock_chk_note_capacity_exhausted(result);
110 return;
111 }
112
113 *fault = lock_chk_fault_from_req(LOCK_CHK_FAIL_UNINITIALIZED, request);
114 lock_chk_fail(fault, "%s %p (graph preparation result %u)",
115 lock_chk_type_to_str(request->lock.type),
116 request->lock.instance, result);
117}
118
119static void
120lock_chk_do_validation_fail(enum lock_chk_result result,
121 const struct lock_chk_acq_req *request) {
122 struct lock_chk_fault fail =
123 lock_chk_fault_from_req(LOCK_CHK_FAIL_UNINITIALIZED, request);
124
125 if (result == LOCK_CHK_RESULT_BAD_CONTEXT) {
126 fail.kind = LOCK_CHK_FAIL_CONTEXT;
127 lock_chk_fail(&fail, "%s %p (irq=%d, irql=%u)",
128 lock_chk_type_to_str(request->lock.type),
129 request->lock.instance, request->in_irq,
130 (unsigned) request->prev_irql);
131 return;
132 }
133 if (result == LOCK_CHK_RESULT_RECURSION) {
134 fail.kind = LOCK_CHK_FAIL_RECURSION;
135 lock_chk_fail(&fail, "%s %p", lock_chk_type_to_str(request->lock.type),
136 request->lock.instance);
137 return;
138 }
139 if (result == LOCK_CHK_RESULT_HELD_CAPACITY) {
140 fail.kind = LOCK_CHK_FAIL_CAPACITY;
141 fail.capacity_pool = "per-thread held";
142 fail.capacity_used = LOCK_CHK_MAX_HELD_LOCKS;
143 fail.capacity_limit = LOCK_CHK_MAX_HELD_LOCKS;
144 lock_chk_fail(&fail, "%s %p (per-thread held %u/%u)",
145 lock_chk_type_to_str(request->lock.type),
146 request->lock.instance, LOCK_CHK_MAX_HELD_LOCKS,
147 LOCK_CHK_MAX_HELD_LOCKS);
148 if (!lock_chk_global.panic_on_exhaustion)
149 lock_chk_degrade();
150 return;
151 }
152
153 fail.kind = LOCK_CHK_FAIL_UNINITIALIZED;
154 lock_chk_fail(&fail, "%s %p (subclass %u >= %u)",
155 lock_chk_type_to_str(request->lock.type),
156 request->lock.instance, request->subclass,
157 LOCK_CHK_MAX_SUBCLASSES);
158}
159
160void lock_chk_before_acq(struct lock_chk_acq_token *token,
161 const struct lock_chk_acq_req *request) {
162 *token = (struct lock_chk_acq_token){0};
163 if (!lock_chk_state_active(&lock_chk_global.deep) ||
164 request->lock.flags == LOCK_UNCHKD)
165 return;
166
167 struct lock_chk_guard guard = lock_chk_enter();
168
169 struct thread *thread = enforce(thread_get_current());
170 struct lock_chk_thread_data *thread_data = &thread->lock_chk;
171 enum lock_chk_result result = validate_acq(request, thread_data);
172 if (result != LOCK_CHK_RESULT_OK) {
173 lock_chk_do_validation_fail(result, request);
174 goto out;
175 }
176
177 struct lock_chk_node *node = NULL;
178 if ((request->lock.flags & LOCK_CHKD_ORDER) != 0) {
179 struct lock_chk_fault fail = {0};
180 struct lock_chk_ctx ctx = {
181 .graph = &lock_chk_global.graph,
182 .req = request,
183 .thread_data = thread_data,
184 .site = request->site,
185 .fault = &fail,
186 };
187 result = lock_chk_graph_prepare_acq(&ctx, request->map,
188 request->subclass, &node);
189 if (result != LOCK_CHK_RESULT_OK) {
190 if (result == LOCK_CHK_RESULT_CYCLE ||
191 result == LOCK_CHK_RESULT_EDGE_CAPACITY) {
192 lock_chk_report_fail(&fail);
193 lock_chk_note_capacity_exhausted(result);
194 } else {
195 lock_chk_do_graph_fail(result, request, &fail);
196 }
197 goto out;
198 }
199 }
200
201 token->node = node;
202 token->context_node = node;
203 token->request = request;
204 token->thread_data = thread_data;
205 token->active = true;
206
207out:
208 lock_chk_leave(&guard);
209}
210
211void lock_chk_acqd(struct lock_chk_acq_token *token) {
212 if (!token->active)
213 return;
214
215 struct lock_chk_guard guard = lock_chk_enter();
216
217 const struct lock_chk_acq_req *request = token->request;
218 struct lock_chk_thread_data *thread_data = token->thread_data;
219 enforce(thread_data->depth < LOCK_CHK_MAX_HELD_LOCKS, CAPACITY_EXCEEDED);
220
221 thread_data->held[thread_data->depth++] = (struct lock_chk_held){
222 .node = token->node,
223 .lock = request->lock,
224 .acquire_site = request->site,
225 .acquire_tsc = rdtsc_ordered(),
226 .prev_irql = request->prev_irql,
227 .cpu = (uint16_t) smp_id(TOPC_IFLAG),
228 .mode = request->mode,
229 .subclass = request->subclass,
230 .trylock = (request->op_flags & LOCK_OP_KIND_MASK) == LOCK_OP_KIND_TRY,
231 .raw_operation = request->op_flags & LOCK_OP_RAW,
232 };
233
234 if ((request->lock.flags & LOCK_CHKD_THREAD) != 0) {
235 thread_data->thread_checked_depth++;
236 if (!(request->op_flags & LOCK_OP_RAW) &&
237 lock_chk_type_is_spin(request->lock.type))
238 thread_data->thread_checked_spin_depth++;
239 }
240
241 token->active = false;
242
243 lock_chk_leave(&guard);
244}
245
246void lock_chk_cancel(struct lock_chk_acq_token *token) {
247 token->active = false;
248}
249
250void lock_chk_before_rel(struct lock_chk_rel_token *token,
251 const struct lock_chk_rel_req *request) {
252 *token = (struct lock_chk_rel_token){0};
253 if (!lock_chk_state_active(&lock_chk_global.deep) ||
254 request->lock.flags == LOCK_UNCHKD)
255 return;
256
257 struct lock_chk_guard guard = lock_chk_enter();
258
259 struct thread *thread = enforce(thread_get_current());
260
261 struct lock_chk_thread_data *thread_data = &thread->lock_chk;
262 struct lock_chk_held *held = lock_chk_find_held(
263 thread_data, request->lock.instance, request->lock.type,
264 /*match_type=*/true, request->mode, /*match_mode=*/true);
265
266 if (held != NULL) {
267 token->thread_data = thread_data;
268 token->instance = request->lock.instance;
269 token->held_index = (uint8_t) (held - thread_data->held);
270 token->active = true;
271 goto out;
272 }
273
274 struct lock_chk_fault fail =
275 lock_chk_fault_from_rel(LOCK_CHK_FAIL_RELEASE, request);
276 lock_chk_fail(&fail, "%s %p (not held by this thread)",
277 lock_chk_type_to_str(request->lock.type),
278 request->lock.instance);
279
280out:
281 lock_chk_leave(&guard);
282}
283
284void lock_chk_reld(struct lock_chk_rel_token *token) {
285 if (!token->active)
286 return;
287
288 struct lock_chk_guard guard = lock_chk_enter();
289
290 struct lock_chk_thread_data *thread_data = token->thread_data;
291 enforce(token->held_index < thread_data->depth);
292 struct lock_chk_held released = thread_data->held[token->held_index];
293 enforce(released.lock.instance == token->instance);
294
295 if ((released.lock.flags & LOCK_CHKD_THREAD) != 0) {
296 enforce(thread_data->thread_checked_depth != 0);
297 thread_data->thread_checked_depth--;
298 if (!released.raw_operation &&
299 lock_chk_type_is_spin(released.lock.type)) {
300 enforce(thread_data->thread_checked_spin_depth != 0);
301 thread_data->thread_checked_spin_depth--;
302 }
303 }
304
305 for (uint8_t i = token->held_index; i + 1 < thread_data->depth; i++)
306 thread_data->held[i] = thread_data->held[i + 1];
307
308 thread_data->depth--;
309 token->active = false;
310
311 lock_chk_leave(&guard);
312}
313
314bool lock_chk_assert_held_deep(struct lock_chk_lock *lock,
315 enum lock_chk_mode mode, bool want_held,
316 const struct lock_chk_site *site) {
317 bool mode_specific = mode != LOCK_CHK_MODE_IGNORED;
318 void *instance = lock->instance;
319 enum lock_chk_type type = lock->type;
320 if (!lock_chk_state_active(&lock_chk_global.deep))
321 return false;
322
323 struct lock_chk_guard guard = lock_chk_enter();
324 bool handled = true;
325
326 struct thread *thread = thread_get_current();
327 if (thread == NULL) {
328 handled = false;
329 goto out;
330 }
331
332 struct lock_chk_thread_data *thread_data = &thread->lock_chk;
333 const struct lock_chk_held *held = lock_chk_find_held(
334 thread_data, instance, type, /*match_type=*/true, mode, mode_specific);
335 bool found = held != NULL;
336 enum lock_chk_mode found_mode = found ? held->mode : mode;
337
338 if (found != want_held) {
339 struct lock_chk_fault fail = lock_chk_fault_from_lock(
340 want_held ? LOCK_CHK_FAIL_NOT_HELD : LOCK_CHK_FAIL_UNEXPECTED_HELD,
341 lock, site, found_mode);
342 lock_chk_fail(&fail, "%s %p", lock_chk_type_to_str(type), instance);
343 }
344
345out:
346 lock_chk_leave(&guard);
347 return handled;
348}
349
350void lock_chk_assert_schedulable(const struct lock_chk_site *site) {
351 if (!lock_chk_state_active(&lock_chk_global.deep))
352 return;
353
354 struct lock_chk_guard guard = lock_chk_enter();
355 struct thread *thread = thread_get_current();
356 if (thread != NULL && thread->lock_chk.thread_checked_spin_depth != 0) {
357 struct lock_chk_fault fail = {
358 .kind = LOCK_CHK_FAIL_CONTEXT,
359 .site = site,
360 };
361 lock_chk_fail(&fail,
362 "thread '%s' (scheduling with %u thread-checked "
363 "spinlocks held)",
364 thread->name, thread->lock_chk.thread_checked_spin_depth);
365 }
366 lock_chk_leave(&guard);
367}
368
369void lock_chk_thread_init(struct thread *thread) {
370 memset(&thread->lock_chk, 0, sizeof(thread->lock_chk));
371}
372
373void lock_chk_thread_exit(struct thread *thread) {
374 if (!lock_chk_state_active(&lock_chk_global.deep))
375 return;
376
377 struct lock_chk_guard guard = lock_chk_enter();
378 if (thread->lock_chk.thread_checked_depth != 0) {
379 struct lock_chk_fault fail = {
380 .kind = LOCK_CHK_FAIL_THREAD_EXIT,
381 };
382 lock_chk_fail(&fail, "thread '%s' %p (%u checked locks held)",
383 thread->name, thread,
384 thread->lock_chk.thread_checked_depth);
385 }
386 lock_chk_leave(&guard);
387}
388
389#endif /* DEBUG_LOCK_CHK */
390