1#include <sch/sched.h>
2#include <sync/mutex_simple.h>
3#include <sync/spinlock.h>
4#include <thread/thread.h>
5
6#ifdef DEBUG_LOCK_CHK
7
8#include "lock_chk/internal.h"
9
10static inline void mutex_simple_chk_stamp(struct mutex_simple *lock) {
11 lock->chk.instance = lock;
12 lock->chk.type = LOCK_CHK_TYPE_MUTEX_SIMPLE;
13}
14
15struct mutex_simple_chk_acquire_state {
16 struct lock_chk_acq_req request;
17 struct lock_chk_acq_token token;
18};
19
20struct mutex_simple_chk_release_state {
21 struct lock_chk_rel_req request;
22 struct lock_chk_rel_token token;
23};
24
25static void mutex_simple_chk_before_lock(
26 struct mutex_simple_chk_acquire_state *state, struct mutex_simple *m,
27 enum lock_subclass subclass, const struct lock_chk_site *site) {
28 enum lock_op_flags flags = LOCK_OP_IRQ_NONE | LOCK_OP_KIND_BLOCKING;
29 lock_chk_note_use(&m->chk, flags);
30 mutex_simple_chk_stamp(m);
31 state->request = lock_chk_acq_req_make(
32 &m->chk, site, LOCK_CHK_MODE_EXCLUSIVE, subclass, flags);
33 lock_chk_before_acq(&state->token, &state->request);
34}
35
36static void
37mutex_simple_chk_locked(struct mutex_simple_chk_acquire_state *state) {
38 lock_chk_acqd(&state->token);
39}
40
41static void
42mutex_simple_chk_before_unlock(struct mutex_simple_chk_release_state *state,
43 struct mutex_simple *m,
44 const struct lock_chk_site *site) {
45 mutex_simple_chk_stamp(m);
46 state->request =
47 lock_chk_rel_req_make(&m->chk, site, LOCK_CHK_MODE_EXCLUSIVE);
48 lock_chk_before_rel(&state->token, &state->request);
49}
50
51static void
52mutex_simple_chk_unlocked(struct mutex_simple_chk_release_state *state) {
53 lock_chk_reld(&state->token);
54}
55
56static void mutex_simple_chk_state_init(struct mutex_simple *m,
57 const struct lock_chk_class *class,
58 enum lock_chk_flags flags) {
59 enforce_precond((flags & ~LOCK_CHKD_FULL) == 0);
60 enforce_precond(flags == LOCK_UNCHKD || class != NULL);
61 m->chk.flags = flags;
62 m->chk.initialized = true;
63 atomic_store_rlx(&m->chk.used, false);
64 lock_chk_map_runtime_init(&m->chk.map, class);
65}
66
67void mutex_simple_set_chk_flags(struct mutex_simple *m,
68 enum lock_chk_flags flags) {
69 enforce(m->chk.initialized, UNINITIALIZED);
70 enforce(m->owner == NULL);
71 enforce(list_head_empty(&m->waiters.waiters));
72 enforce(!spinlock_locked(&m->waiters.lock));
73 enforce(!spinlock_locked(&m->lock));
74 enforce(!atomic_load_rlx(&m->chk.used));
75 enforce_precond((flags & ~LOCK_CHKD_FULL) == 0);
76 m->chk.flags = flags;
77}
78
79void mutex_simple_reinit_chk(struct mutex_simple *m,
80 const struct lock_chk_class *class,
81 enum lock_chk_flags flags) {
82 enforce(m->chk.initialized, UNINITIALIZED);
83 enforce(m->owner == NULL);
84 enforce(list_head_empty(&m->waiters.waiters));
85 enforce(!spinlock_locked(&m->waiters.lock));
86 enforce(!spinlock_locked(&m->lock));
87 mutex_simple_init_chk_full(m, class, flags);
88}
89
90#else /* !defined(DEBUG_LOCK_CHK) */
91
92struct mutex_simple_chk_acquire_state {
93 bool unused;
94};
95
96struct mutex_simple_chk_release_state {
97 bool unused;
98};
99
100static inline void mutex_simple_chk_before_lock(
101 struct mutex_simple_chk_acquire_state *state, struct mutex_simple *m,
102 enum lock_subclass subclass, const struct lock_chk_site *site) {
103 cc_ignore(state, m, subclass, site);
104}
105
106static inline void
107mutex_simple_chk_locked(struct mutex_simple_chk_acquire_state *state) {
108 cc_ignore(state);
109}
110
111static inline void
112mutex_simple_chk_before_unlock(struct mutex_simple_chk_release_state *state,
113 struct mutex_simple *m,
114 const struct lock_chk_site *site) {
115 cc_ignore(state, m, site);
116}
117
118static inline void
119mutex_simple_chk_unlocked(struct mutex_simple_chk_release_state *state) {
120 cc_ignore(state);
121}
122
123static void mutex_simple_chk_state_init(struct mutex_simple *m,
124 const struct lock_chk_class *class,
125 enum lock_chk_flags flags) {
126 cc_ignore(m, class, flags);
127}
128
129void mutex_simple_set_chk_flags(struct mutex_simple *m,
130 enum lock_chk_flags flags) {
131 cc_ignore(m, flags);
132}
133
134void mutex_simple_reinit_chk(struct mutex_simple *m,
135 const struct lock_chk_class *class,
136 enum lock_chk_flags flags) {
137 mutex_simple_init_chk_full(m, class, flags);
138}
139
140#endif /* DEBUG_LOCK_CHK */
141
142static void mutex_simple_sanity_check(void) {
143 enforce(irq_not_in_interrupt(), INVALID_CONTEXT);
144 enforce(irql_get() <= IRQL_APC_LEVEL, INVALID_CONTEXT);
145}
146
147static bool try_acquire_simple_mutex(struct mutex_simple *m,
148 struct thread *curr) {
149 enum irql irql = spin_lock(&m->lock);
150 if (m->owner == NULL) {
151 m->owner = curr;
152 spin_unlock(&m->lock, irql);
153 return true;
154 }
155 spin_unlock(&m->lock, irql);
156 return false;
157}
158
159static bool should_spin_on_mutex(struct mutex_simple *m) {
160 enum irql irql = spin_lock(&m->lock);
161 struct thread *owner = m->owner;
162 bool active =
163 owner && atomic_load_rlx(&owner->state) == THREAD_STATE_RUNNING;
164 spin_unlock(&m->lock, irql);
165 return active;
166}
167
168static bool spin_wait_simple_mutex(struct mutex_simple *m,
169 struct thread *curr) {
170 for (int i = 0; i < 500; i++)
171 if (try_acquire_simple_mutex(m, curr))
172 return true;
173
174 return false;
175}
176
177static void block_on_simple_mutex(struct mutex_simple *m) {
178 enum irql irql = spin_lock(&m->lock);
179 /* owner can release between failed fast path and this lock */
180 if (!m->owner) {
181 spin_unlock(&m->lock, irql);
182 return;
183 }
184 thread_wait_prepare_one(header: &m->waiters, object: m, type: THREAD_WAIT_UNINTERRUPTIBLE,
185 reason: THREAD_BLOCK_REASON_MANUAL);
186 spin_unlock(&m->lock, irql);
187 thread_wait_complete();
188}
189
190void mutex_simple_init_chk_full(struct mutex_simple *m,
191 const struct lock_chk_class *class,
192 enum lock_chk_flags flags) {
193 m->owner = NULL;
194 list_head_init(h: &m->waiters.waiters);
195 spinlock_init(&m->waiters.lock, LOCK_UNCHKD);
196 spinlock_init(&m->lock, LOCK_UNCHKD);
197 mutex_simple_chk_state_init(m, class, flags);
198}
199
200void mutex_simple_lock_subclass_full(
201 struct mutex_simple *m, enum lock_subclass subclass,
202 const struct lock_chk_site *site) TSA_NO_ANALYSIS {
203 mutex_simple_sanity_check();
204
205 struct mutex_simple_chk_acquire_state chk_state;
206 mutex_simple_chk_before_lock(state: &chk_state, m, subclass, site);
207
208 struct thread *curr = thread_get_current();
209
210 while (true) {
211 if (try_acquire_simple_mutex(m, curr))
212 break;
213
214 if (should_spin_on_mutex(m))
215 if (spin_wait_simple_mutex(m, curr))
216 break;
217
218 block_on_simple_mutex(m);
219 }
220
221 enforce(m->owner == curr);
222 mutex_simple_chk_locked(state: &chk_state);
223}
224
225void mutex_simple_lock_full(struct mutex_simple *m,
226 const struct lock_chk_site *site) TSA_NO_ANALYSIS {
227 mutex_simple_lock_subclass_full(m, subclass: 0, site);
228}
229
230void mutex_simple_unlock_full(
231 struct mutex_simple *m, const struct lock_chk_site *site) TSA_NO_ANALYSIS {
232 mutex_simple_sanity_check();
233
234 struct thread *curr = thread_get_current();
235
236 if (m->owner != curr) {
237 panic("mutex_simple unlock by non-owner thread. owner is %p, current "
238 "is %p",
239 m->owner, curr);
240 }
241
242 struct mutex_simple_chk_release_state chk_state;
243 mutex_simple_chk_before_unlock(state: &chk_state, m, site);
244
245 enum irql irql = spin_lock(&m->lock);
246
247 m->owner = NULL;
248
249 thread_wait_header_satisfy(hdr: &m->waiters, reason: THREAD_WAKE_REASON_BLOCKING_MANUAL,
250 NULL);
251
252 spin_unlock(&m->lock, irql);
253
254 mutex_simple_chk_unlocked(state: &chk_state);
255}
256
257bool mutex_simple_locked(struct mutex_simple *m) {
258 return mutex_simple_get_owner(m) != NULL;
259}
260
261struct thread *mutex_simple_get_owner(struct mutex_simple *m) {
262 enum irql irql = spin_lock(&m->lock);
263 struct thread *owner = m->owner;
264 spin_unlock(&m->lock, irql);
265 return owner;
266}
267
268void mutex_simple_assert_held_full(struct mutex_simple *m,
269 const struct lock_chk_site *site) {
270#ifdef DEBUG_LOCK_CHK
271 mutex_simple_chk_stamp(m);
272 if (m->chk.flags != LOCK_UNCHKD && lock_chk_tracking_active() &&
273 lock_chk_assert_held_deep(&m->chk, LOCK_CHK_MODE_IGNORED,
274 /*want_held=*/true, site))
275 return;
276#else
277 cc_ignore(site);
278#endif
279
280 enforce(mutex_simple_get_owner(m) == thread_get_current(),
281 "mutex_simple not held by current thread");
282}
283
284void mutex_simple_assert_not_held_full(struct mutex_simple *m,
285 const struct lock_chk_site *site) {
286#ifdef DEBUG_LOCK_CHK
287 mutex_simple_chk_stamp(m);
288 if (m->chk.flags != LOCK_UNCHKD && lock_chk_tracking_active() &&
289 lock_chk_assert_held_deep(&m->chk, LOCK_CHK_MODE_IGNORED,
290 /*want_held=*/false, site))
291 return;
292#else
293 cc_ignore(site);
294#endif
295
296 enforce(mutex_simple_get_owner(m) != thread_get_current(),
297 "mutex_simple unexpectedly held by current thread");
298}
299