1#include <sch/sched.h>
2#include <sync/mutex_simple.h>
3#include <sync/spinlock.h>
4#include <thread/thread.h>
5
6#ifdef DEBUG_LOCK_CHK
7
8#include "lock_chk_internal.h"
9
10struct mutex_simple_chk_acquire_state {
11 struct lock_chk_acquire_request request;
12 struct lock_chk_acquire_token token;
13};
14
15struct mutex_simple_chk_release_state {
16 struct lock_chk_release_request request;
17 struct lock_chk_release_token token;
18};
19
20static void
21mutex_simple_chk_before_lock(struct mutex_simple_chk_acquire_state *state,
22 struct mutex_simple *m, unsigned int subclass,
23 const struct lock_chk_site *site) {
24 lock_chk_note_lock_use(&m->chk, false, false);
25 m->chk.instance = m;
26 m->chk.type = LOCK_CHK_TYPE_MUTEX_SIMPLE;
27 state->request = lock_chk_acquire_request_make(
28 &m->chk, site, LOCK_CHK_MODE_EXCLUSIVE, LOCK_CHK_WAIT_BLOCKING,
29 subclass, false, false);
30 lock_chk_before_acquire(&state->token, &state->request);
31}
32
33static void
34mutex_simple_chk_locked(struct mutex_simple_chk_acquire_state *state) {
35 lock_chk_acquired(&state->token);
36}
37
38static void
39mutex_simple_chk_before_unlock(struct mutex_simple_chk_release_state *state,
40 struct mutex_simple *m,
41 const struct lock_chk_site *site) {
42 m->chk.instance = m;
43 m->chk.type = LOCK_CHK_TYPE_MUTEX_SIMPLE;
44 state->request =
45 lock_chk_release_request_make(&m->chk, site, LOCK_CHK_MODE_EXCLUSIVE);
46 lock_chk_before_release(&state->token, &state->request);
47}
48
49static void
50mutex_simple_chk_unlocked(struct mutex_simple_chk_release_state *state) {
51 lock_chk_released(&state->token);
52}
53
54static void mutex_simple_chk_state_init(struct mutex_simple *m,
55 const struct lock_chk_class *class,
56 enum lock_chk_flags flags) {
57 kassert((flags & ~LOCK_CHKD_FULL) == 0);
58 kassert(flags == LOCK_UNCHKD || class != NULL);
59 m->chk.flags = flags;
60 m->chk.initialized = true;
61 atomic_store_explicit(&m->chk.used, false, memory_order_relaxed);
62 lock_chk_map_runtime_init(&m->chk.map, class);
63}
64
65void mutex_simple_set_chk_flags(struct mutex_simple *m,
66 enum lock_chk_flags flags) {
67 kassert(m->chk.initialized);
68 kassert(m->owner == NULL);
69 kassert(list_empty(&m->waiters.list));
70 kassert(!spinlock_locked(&m->waiters.lock));
71 kassert(!spinlock_locked(&m->lock));
72 kassert(!atomic_load_explicit(&m->chk.used, memory_order_relaxed));
73 kassert((flags & ~LOCK_CHKD_FULL) == 0);
74 m->chk.flags = flags;
75}
76
77void mutex_simple_reinit_chk(struct mutex_simple *m,
78 const struct lock_chk_class *class,
79 enum lock_chk_flags flags) {
80 kassert(m->chk.initialized);
81 kassert(m->owner == NULL);
82 kassert(list_empty(&m->waiters.list));
83 kassert(!spinlock_locked(&m->waiters.lock));
84 kassert(!spinlock_locked(&m->lock));
85 mutex_simple_init_chk_internal(m, class, flags);
86}
87
88#else /* !defined(DEBUG_LOCK_CHK) */
89
90struct mutex_simple_chk_acquire_state {
91 bool unused;
92};
93
94struct mutex_simple_chk_release_state {
95 bool unused;
96};
97
98static inline void
99mutex_simple_chk_before_lock(struct mutex_simple_chk_acquire_state *state,
100 struct mutex_simple *m, unsigned int subclass,
101 const struct lock_chk_site *site) {
102 unused(state, m, subclass, site);
103}
104
105static inline void
106mutex_simple_chk_locked(struct mutex_simple_chk_acquire_state *state) {
107 unused(state);
108}
109
110static inline void
111mutex_simple_chk_before_unlock(struct mutex_simple_chk_release_state *state,
112 struct mutex_simple *m,
113 const struct lock_chk_site *site) {
114 unused(state, m, site);
115}
116
117static inline void
118mutex_simple_chk_unlocked(struct mutex_simple_chk_release_state *state) {
119 unused(state);
120}
121
122static void mutex_simple_chk_state_init(struct mutex_simple *m,
123 const struct lock_chk_class *class,
124 enum lock_chk_flags flags) {
125 unused(m, class, flags);
126}
127
128void mutex_simple_set_chk_flags(struct mutex_simple *m,
129 enum lock_chk_flags flags) {
130 unused(m, flags);
131}
132
133void mutex_simple_reinit_chk(struct mutex_simple *m,
134 const struct lock_chk_class *class,
135 enum lock_chk_flags flags) {
136 mutex_simple_init_chk_internal(m, class, flags);
137}
138
139#endif /* DEBUG_LOCK_CHK */
140
141static void mutex_simple_sanity_check(void) {
142 kassert(irq_not_in_interrupt());
143 kassert(irql_get() <= IRQL_APC_LEVEL);
144}
145
146static bool try_acquire_simple_mutex(struct mutex_simple *m,
147 struct thread *curr) {
148 enum irql irql = spin_lock(&m->lock);
149 if (m->owner == NULL) {
150 m->owner = curr;
151 spin_unlock(&m->lock, irql);
152 return true;
153 }
154 spin_unlock(&m->lock, irql);
155 return false;
156}
157
158static bool should_spin_on_mutex(struct mutex_simple *m) {
159 enum irql irql = spin_lock(&m->lock);
160 struct thread *owner = m->owner;
161 bool active = owner && atomic_load(&owner->state) == THREAD_STATE_RUNNING;
162 spin_unlock(&m->lock, irql);
163 return active;
164}
165
166static bool spin_wait_simple_mutex(struct mutex_simple *m,
167 struct thread *curr) {
168 for (int i = 0; i < 500; i++)
169 if (try_acquire_simple_mutex(m, curr))
170 return true;
171
172 return false;
173}
174
175static void block_on_simple_mutex(struct mutex_simple *m) {
176 enum irql irql = spin_lock(&m->lock);
177 thread_block_on(q: &m->waiters, type: THREAD_WAIT_UNINTERRUPTIBLE, wake_src: m);
178 spin_unlock(&m->lock, irql);
179 scheduler_yield();
180}
181
182void mutex_simple_init_chk_internal(struct mutex_simple *m,
183 const struct lock_chk_class *class,
184 enum lock_chk_flags flags) {
185 m->owner = NULL;
186 INIT_LIST_HEAD(list: &m->waiters.list);
187 spinlock_init(&m->waiters.lock, LOCK_UNCHKD);
188 spinlock_init(&m->lock, LOCK_UNCHKD);
189 mutex_simple_chk_state_init(m, class, flags);
190}
191
192void mutex_simple_lock_subclass_internal(struct mutex_simple *m,
193 uint8_t subclass,
194 const struct lock_chk_site *site) {
195 mutex_simple_sanity_check();
196
197 struct mutex_simple_chk_acquire_state chk_state;
198 mutex_simple_chk_before_lock(state: &chk_state, m, subclass, site);
199
200 struct thread *curr = thread_get_current();
201
202 while (true) {
203 if (try_acquire_simple_mutex(m, curr))
204 break;
205
206 if (should_spin_on_mutex(m))
207 if (spin_wait_simple_mutex(m, curr))
208 break;
209
210 block_on_simple_mutex(m);
211 }
212
213 kassert(m->owner == curr);
214 mutex_simple_chk_locked(state: &chk_state);
215}
216
217void mutex_simple_lock_internal(struct mutex_simple *m,
218 const struct lock_chk_site *site) {
219 mutex_simple_lock_subclass_internal(m, subclass: 0, site);
220}
221
222void mutex_simple_unlock_internal(struct mutex_simple *m,
223 const struct lock_chk_site *site) {
224 mutex_simple_sanity_check();
225
226 struct thread *curr = thread_get_current();
227
228 if (m->owner != curr) {
229 panic("mutex_simple unlock by non-owner thread. owner is %p, current "
230 "is %p",
231 m->owner, curr);
232 }
233
234 struct mutex_simple_chk_release_state chk_state;
235 mutex_simple_chk_before_unlock(state: &chk_state, m, site);
236
237 enum irql irql = spin_lock(&m->lock);
238
239 m->owner = NULL;
240
241 struct thread *next = thread_queue_pop_front(q: &m->waiters);
242 if (next != NULL)
243 thread_wake(t: next, reason: THREAD_WAKE_REASON_BLOCKING_MANUAL,
244 prio: next->perceived_prio_class, wake_src: m);
245
246 spin_unlock(&m->lock, irql);
247
248 mutex_simple_chk_unlocked(state: &chk_state);
249}
250
251bool mutex_simple_locked(struct mutex_simple *m) {
252 return mutex_simple_get_owner(m) != NULL;
253}
254
255struct thread *mutex_simple_get_owner(struct mutex_simple *m) {
256 enum irql irql = spin_lock(&m->lock);
257 struct thread *owner = m->owner;
258 spin_unlock(&m->lock, irql);
259 return owner;
260}
261
262void mutex_simple_assert_held_internal(struct mutex_simple *m,
263 const struct lock_chk_site *site) {
264#ifdef DEBUG_LOCK_CHK
265 m->chk.instance = m;
266 m->chk.type = LOCK_CHK_TYPE_MUTEX_SIMPLE;
267 if (m->chk.flags != LOCK_UNCHKD && lock_chk_tracking_active() &&
268 lock_chk_assert_held_deep(&m->chk, LOCK_CHK_MODE_IGNORED,
269 /*want_held=*/true, site))
270 return;
271#else
272 unused(site);
273#endif
274 kassert(mutex_simple_get_owner(m) == thread_get_current(),
275 "mutex_simple not held by current thread");
276}
277
278void mutex_simple_assert_not_held_internal(struct mutex_simple *m,
279 const struct lock_chk_site *site) {
280#ifdef DEBUG_LOCK_CHK
281 m->chk.instance = m;
282 m->chk.type = LOCK_CHK_TYPE_MUTEX_SIMPLE;
283 if (m->chk.flags != LOCK_UNCHKD && lock_chk_tracking_active() &&
284 lock_chk_assert_held_deep(&m->chk, LOCK_CHK_MODE_IGNORED,
285 /*want_held=*/false, site))
286 return;
287#else
288 unused(site);
289#endif
290 kassert(mutex_simple_get_owner(m) != thread_get_current(),
291 "mutex_simple unexpectedly held by current thread");
292}
293