1#ifdef DEBUG_LOCK_CHK
2
3#include <asm.h>
4#include <kassert.h>
5#include <smp/percpu.h>
6#include <stdatomic.h>
7#include <sync/lock_chk.h>
8
9#include "lock_chk_internal.h"
10
11struct lock_debug_spin_entry {
12 void *instance;
13 const struct lock_chk_site *acquire_site;
14 enum irql prev_irql;
15 enum lock_chk_type type;
16};
17
18struct lock_debug_cpu {
19 struct lock_debug_spin_entry stack[LOCK_CHK_MAX_SPIN_DEPTH];
20 uint8_t depth;
21};
22
23static _Atomic enum lock_chk_engine_state lock_debug_state = LOCK_CHK_INACTIVE;
24PERCPU_DECLARE(lock_debug_cpu, struct lock_debug_cpu, NULL);
25
26static const char *lock_chk_type_name(enum lock_chk_type type) {
27 switch (type) {
28 case LOCK_CHK_TYPE_SPIN: return "spinlock";
29 case LOCK_CHK_TYPE_QSPIN: return "qspinlock";
30 case LOCK_CHK_TYPE_MUTEX: return "mutex";
31 case LOCK_CHK_TYPE_MUTEX_SIMPLE: return "simple mutex";
32 case LOCK_CHK_TYPE_RWLOCK: return "rwlock";
33 }
34
35 return "unknown lock";
36}
37
38void lock_debug_activate(void) {
39 kassert(PERCPU_READY(lock_debug_cpu));
40 atomic_store_explicit(&lock_debug_state, LOCK_CHK_ACTIVE,
41 memory_order_release);
42}
43
44void lock_debug_spin_classify(_Atomic uint8_t *usage,
45 enum lock_debug_irq_usage requested,
46 struct lock_chk_lock *lock,
47 const struct lock_chk_site *site) {
48 if (atomic_load_explicit(&lock_debug_state, memory_order_acquire) !=
49 LOCK_CHK_ACTIVE)
50 return;
51
52 uint8_t expected = LOCK_DEBUG_IRQ_NONE;
53 if (atomic_compare_exchange_strong_explicit(usage, &expected, requested,
54 memory_order_relaxed,
55 memory_order_relaxed))
56 return;
57
58 if (expected != requested) {
59 struct lock_chk_failure fail = {
60 .kind = LOCK_CHK_FAIL_CONTEXT,
61 .site = site,
62 .instance = lock->instance,
63 .type = lock->type,
64 };
65 lock_chk_fail(&fail, "%s %p changed IRQL usage",
66 lock_chk_type_name(lock->type), lock->instance);
67 }
68}
69
70bool lock_debug_spin_push(struct lock_chk_lock *lock, enum irql prev_irql,
71 const struct lock_chk_site *site) {
72 if (atomic_load_explicit(&lock_debug_state, memory_order_acquire) !=
73 LOCK_CHK_ACTIVE)
74 return false;
75
76 kassert(!are_interrupts_enabled());
77 struct lock_debug_cpu *cpu = PERCPU_PTR(TOPC_IFLAG, lock_debug_cpu);
78 if (cpu->depth == LOCK_CHK_MAX_SPIN_DEPTH) {
79 struct lock_chk_failure fail = {
80 .kind = LOCK_CHK_FAIL_CAPACITY,
81 .site = site,
82 .instance = lock->instance,
83 .type = lock->type,
84 .capacity_pool = "shallow spin stack",
85 .capacity_used = LOCK_CHK_MAX_SPIN_DEPTH,
86 .capacity_limit = LOCK_CHK_MAX_SPIN_DEPTH,
87 };
88 lock_chk_fail(&fail, "Shallow spin stack capacity exhausted (%u/%u)",
89 LOCK_CHK_MAX_SPIN_DEPTH, LOCK_CHK_MAX_SPIN_DEPTH);
90 atomic_store_explicit(&lock_debug_state, LOCK_CHK_DEGRADED,
91 memory_order_release);
92 return false;
93 }
94
95 cpu->stack[cpu->depth++] = (struct lock_debug_spin_entry){
96 .instance = lock->instance,
97 .acquire_site = site,
98 .prev_irql = prev_irql,
99 .type = lock->type,
100 };
101 return true;
102}
103
104void lock_debug_spin_validate_top(struct lock_chk_lock *lock,
105 enum irql prev_irql,
106 const struct lock_chk_site *site) {
107 void *instance = lock->instance;
108 enum lock_chk_type type = lock->type;
109 if (atomic_load_explicit(&lock_debug_state, memory_order_acquire) !=
110 LOCK_CHK_ACTIVE)
111 return;
112
113 kassert(!are_interrupts_enabled());
114 struct lock_debug_cpu *cpu = PERCPU_PTR(TOPC_IFLAG, lock_debug_cpu);
115 if (cpu->depth == 0) {
116 struct lock_chk_failure fail = {
117 .kind = LOCK_CHK_FAIL_SPIN_ORDER,
118 .site = site,
119 .instance = instance,
120 .type = type,
121 };
122 lock_chk_fail(&fail, "Releasing untracked %s %p",
123 lock_chk_type_name(type), instance);
124 return;
125 }
126
127 struct lock_debug_spin_entry *top = &cpu->stack[cpu->depth - 1];
128 if (top->instance != instance || top->type != type ||
129 top->prev_irql != prev_irql) {
130 struct lock_chk_failure fail = {
131 .kind = LOCK_CHK_FAIL_SPIN_ORDER,
132 .site = site,
133 .instance = instance,
134 .type = type,
135 };
136 lock_chk_fail(&fail, "Non-LIFO %s release %p", lock_chk_type_name(type),
137 instance);
138 }
139}
140
141void lock_debug_spin_pop(struct lock_chk_lock *lock) {
142 if (atomic_load_explicit(&lock_debug_state, memory_order_acquire) !=
143 LOCK_CHK_ACTIVE)
144 return;
145
146 kassert(!are_interrupts_enabled());
147 struct lock_debug_cpu *cpu = PERCPU_PTR(TOPC_IFLAG, lock_debug_cpu);
148 kassert(cpu->depth != 0);
149 struct lock_debug_spin_entry *top = &cpu->stack[cpu->depth - 1];
150 kassert(top->instance == lock->instance);
151 kassert(top->type == lock->type);
152 cpu->depth--;
153}
154
155#endif /* DEBUG_LOCK_CHK */
156