1#ifdef DEBUG_LOCK_CHK
2
3#include <asm.h>
4#include <atomic.h>
5#include <enforce.h>
6#include <smp/percpu.h>
7#include <sync/lock_chk.h>
8
9#include "internal.h"
10
11PERCPU_DEFINE(struct lock_debug_cpu, lock_debug_cpu);
12
13static bool debug_is_active(void) {
14 return lock_chk_state_active(&lock_chk_global.debug);
15}
16
17static struct lock_debug_cpu *debug_cpu_here(void) {
18 enforce(!irqs_enabled(), INVALID_CONTEXT);
19 return PERCPU_PTR(TOPC_IFLAG, lock_debug_cpu);
20}
21
22void lock_debug_activate(void) {
23 enforce(PERCPU_READY(lock_debug_cpu), UNINITIALIZED);
24 atomic_store_release(&lock_chk_global.debug, LOCK_CHK_ACTIVE);
25}
26
27void lock_debug_spin_classify(atomic(enum lock_op_flags) * usage,
28 enum lock_op_flags requested,
29 struct lock_chk_lock *lock,
30 const struct lock_chk_site *site) {
31 if (!debug_is_active())
32 return;
33
34 uint8_t expected = LOCK_OP_IRQ_NONE;
35 if (atomic_cas_strong(usage, &expected, requested, mo_rlx, mo_rlx))
36 return;
37
38 if ((expected & LOCK_OP_IRQ_MASK) != (requested & LOCK_OP_IRQ_MASK)) {
39 struct lock_chk_fault fail = lock_chk_fault_from_lock(
40 LOCK_CHK_FAIL_CONTEXT, lock, site, LOCK_CHK_MODE_IGNORED);
41 lock_chk_fail(&fail, "%s %p (IRQL usage changed)",
42 lock_chk_type_to_str(lock->type), lock->instance);
43 }
44}
45
46bool lock_debug_spin_push(struct lock_chk_lock *lock, enum irql prev_irql,
47 const struct lock_chk_site *site) {
48 if (!debug_is_active())
49 return false;
50
51 struct lock_debug_cpu *cpu = debug_cpu_here();
52 if (cpu->depth == LOCK_CHK_MAX_SPIN_DEPTH) {
53 struct lock_chk_fault fail = lock_chk_fault_from_lock(
54 LOCK_CHK_FAIL_CAPACITY, lock, site, LOCK_CHK_MODE_IGNORED);
55 fail.capacity_pool = "shallow spin stack";
56 fail.capacity_used = LOCK_CHK_MAX_SPIN_DEPTH;
57 fail.capacity_limit = LOCK_CHK_MAX_SPIN_DEPTH;
58 lock_chk_fail(&fail, "%s %p (shallow spin stack %u/%u)",
59 lock_chk_type_to_str(lock->type), lock->instance,
60 LOCK_CHK_MAX_SPIN_DEPTH, LOCK_CHK_MAX_SPIN_DEPTH);
61 atomic_store_release(&lock_chk_global.debug, LOCK_CHK_DEGRADED);
62 return false;
63 }
64
65 cpu->stack[cpu->depth++] = (struct lock_debug_spin_entry){
66 .instance = lock->instance,
67 .acquire_site = site,
68 .prev_irql = prev_irql,
69 .type = lock->type,
70 };
71 return true;
72}
73
74void lock_debug_spin_validate_top(struct lock_chk_lock *lock,
75 enum irql prev_irql,
76 const struct lock_chk_site *site) {
77 void *instance = lock->instance;
78 enum lock_chk_type type = lock->type;
79 if (!debug_is_active())
80 return;
81
82 struct lock_debug_cpu *cpu = debug_cpu_here();
83 if (cpu->depth == 0) {
84 struct lock_chk_fault fail = lock_chk_fault_from_lock(
85 LOCK_CHK_FAIL_SPIN_ORDER, lock, site, LOCK_CHK_MODE_IGNORED);
86 lock_chk_fail(&fail, "%s %p (untracked)", lock_chk_type_to_str(type),
87 instance);
88 return;
89 }
90
91 struct lock_debug_spin_entry *top = &cpu->stack[cpu->depth - 1];
92 if (top->instance != instance || top->type != type ||
93 top->prev_irql != prev_irql) {
94 struct lock_chk_fault fail = lock_chk_fault_from_lock(
95 LOCK_CHK_FAIL_SPIN_ORDER, lock, site, LOCK_CHK_MODE_IGNORED);
96 lock_chk_fail(&fail, "%s %p (top of stack is %s %p)",
97 lock_chk_type_to_str(type), instance,
98 lock_chk_type_to_str(top->type), top->instance);
99 }
100}
101
102void lock_debug_spin_pop(struct lock_chk_lock *lock) {
103 if (!debug_is_active())
104 return;
105
106 struct lock_debug_cpu *cpu = debug_cpu_here();
107 enforce(cpu->depth != 0);
108 struct lock_debug_spin_entry *top = &cpu->stack[cpu->depth - 1];
109 enforce(top->instance == lock->instance);
110 enforce(top->type == lock->type);
111 cpu->depth--;
112}
113
114#endif /* DEBUG_LOCK_CHK */
115