1#pragma once
2#include <asm.h>
3#include <bootstage.h>
4#include <compiler.h>
5#include <console/panic.h>
6#include <irq/irq.h>
7#include <kassert.h>
8#include <sch/irql.h>
9#include <smp/core.h>
10#include <stdatomic.h>
11#include <stdbool.h>
12#include <sync/lock_chk_types.h>
13#include <sync/raw_spinlock.h>
14
15struct spinlock {
16 struct raw_spinlock raw;
17
18#ifdef DEBUG_LOCK_CHK
19 struct lock_chk_lock chk;
20 _Atomic uint8_t irq_usage;
21#endif /* DEBUG_LOCK_CHK */
22};
23
24#define SPINLOCK_INIT SPINLOCK_INIT_CHK(NULL, LOCK_CHKD_FULL)
25#define SPINLOCK_DEFINE(id) struct spinlock id = SPINLOCK_INIT
26#define SPINLOCK_DEFINE_CHK(id, class_, flags_) \
27 struct spinlock id = SPINLOCK_INIT_CHK((class_), (flags_))
28
29#define spinlock_init_1(lock_) \
30 spinlock_init_auto_internal((lock_), LOCK_CHKD_FULL)
31#define spinlock_init_2(lock_, flags_) \
32 spinlock_init_auto_internal((lock_), (flags_))
33#define spinlock_init(...) \
34 _DISPATCH(spinlock_init, PP_NARG(__VA_ARGS__))(__VA_ARGS__)
35
36static inline void
37spinlock_init_chk_internal(struct spinlock *lock,
38 const struct lock_chk_class *class,
39 enum lock_chk_flags flags);
40
41#ifdef DEBUG_LOCK_CHK
42
43#define __SPINLOCK_SHALLOW_VALUE_INIT \
44 , .irq_usage = ATOMIC_VAR_INIT(LOCK_DEBUG_IRQ_NONE)
45#define __SPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_) \
46 , .chk = LOCK_CHK_LOCK_VALUE_INIT((class_), (flags_))
47#define SPINLOCK_INIT_CHK(class_, flags_) \
48 ((struct spinlock) { \
49 .raw = RAW_SPINLOCK_INIT __SPINLOCK_LOCK_CHK_VALUE_INIT( \
50 (class_), (flags_)) __SPINLOCK_SHALLOW_VALUE_INIT})
51
52#define spinlock_init_chk(lock_, class_, flags_) \
53 spinlock_init_chk_internal((lock_), (class_), (flags_))
54#define spinlock_init_auto_internal(lock_, flags_) \
55 do { \
56 static const struct lock_chk_class __auto_class = { \
57 .name = #lock_, \
58 .file = __RELFILE__, \
59 .line = __LINE__, \
60 }; \
61 spinlock_init_chk_internal((lock_), &__auto_class, (flags_)); \
62 } while (0)
63
64static inline void spinlock_policy_init_internal(struct spinlock *lock,
65 enum lock_chk_flags flags) {
66 kassert((flags & ~LOCK_CHKD_FULL) == 0);
67 lock->chk.flags = flags;
68 lock->chk.initialized = true;
69 atomic_store_explicit(&lock->chk.used, false, memory_order_relaxed);
70}
71static inline void spinlock_shallow_init_internal(struct spinlock *lock) {
72 atomic_store_explicit(&lock->irq_usage, LOCK_DEBUG_IRQ_NONE,
73 memory_order_relaxed);
74}
75static inline void
76spinlock_map_init_internal(struct spinlock *lock,
77 const struct lock_chk_class *class,
78 enum lock_chk_flags flags) {
79 kassert(flags == LOCK_UNCHKD || class != NULL);
80 lock_chk_map_runtime_init(&lock->chk.map, class);
81}
82
83static inline void spinlock_note_use(struct spinlock *lock,
84 bool raw_operation) {
85 lock_chk_note_lock_use(&lock->chk, true, raw_operation);
86}
87
88static inline bool spinlock_order_checked(struct spinlock *lock) {
89 return lock_chk_tracking_active() &&
90 (lock->chk.flags & LOCK_CHKD_ORDER) != 0;
91}
92
93static inline void spinlock_classify(struct spinlock *lock,
94 enum lock_debug_irq_usage usage,
95 const struct lock_chk_site *site) {
96 lock->chk.instance = lock;
97 lock->chk.type = LOCK_CHK_TYPE_SPIN;
98 lock_debug_spin_classify(&lock->irq_usage, usage, &lock->chk, site);
99}
100
101static inline bool spinlock_deep_checked(struct spinlock *lock) {
102 return lock_chk_tracking_active() && lock->chk.flags != LOCK_UNCHKD;
103}
104
105static inline struct lock_chk_acquire_request spinlock_chk_acquire_request(
106 struct spinlock *lock, const struct lock_chk_site *site,
107 enum lock_chk_wait_kind wait_kind, uint8_t subclass, bool raw_operation,
108 bool irq_safe) {
109 lock->chk.instance = lock;
110 lock->chk.type = LOCK_CHK_TYPE_SPIN;
111 return lock_chk_acquire_request_make(&lock->chk, site,
112 LOCK_CHK_MODE_EXCLUSIVE, wait_kind,
113 subclass, raw_operation, irq_safe);
114}
115
116static inline struct lock_chk_release_request
117spinlock_chk_release_request(struct spinlock *lock,
118 const struct lock_chk_site *site) {
119 lock->chk.instance = lock;
120 lock->chk.type = LOCK_CHK_TYPE_SPIN;
121 return lock_chk_release_request_make(&lock->chk, site,
122 LOCK_CHK_MODE_EXCLUSIVE);
123}
124
125#else /* !defined(DEBUG_LOCK_CHK) */
126
127#define __SPINLOCK_SHALLOW_VALUE_INIT
128#define __SPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_)
129#define SPINLOCK_INIT_CHK(class_, flags_) \
130 ((struct spinlock) {.raw = RAW_SPINLOCK_INIT})
131
132#define spinlock_init_chk(lock_, class_, flags_) \
133 spinlock_init_chk_internal((lock_), NULL, LOCK_UNCHKD)
134#define spinlock_init_auto_internal(lock_, flags_) \
135 spinlock_init_chk_internal((lock_), NULL, LOCK_UNCHKD)
136
137static inline void spinlock_policy_init_internal(struct spinlock *lock,
138 enum lock_chk_flags flags) {
139 unused(lock, flags);
140}
141
142static inline void spinlock_shallow_init_internal(struct spinlock *lock) {
143 unused(lock);
144}
145
146static inline void
147spinlock_map_init_internal(struct spinlock *lock,
148 const struct lock_chk_class *class,
149 enum lock_chk_flags flags) {
150 unused(lock, class, flags);
151}
152
153static inline void spinlock_note_use(struct spinlock *lock,
154 bool raw_operation) {
155 unused(lock, raw_operation);
156}
157
158static inline bool spinlock_order_checked(struct spinlock *lock) {
159 unused(lock);
160 return false;
161}
162
163static inline void spinlock_classify(struct spinlock *lock,
164 enum lock_debug_irq_usage usage,
165 const struct lock_chk_site *site) {
166 unused(lock, usage, site);
167}
168
169static inline bool spinlock_deep_checked(struct spinlock *lock) {
170 unused(lock);
171 return false;
172}
173
174#endif /* DEBUG_LOCK_CHK */
175
176static inline bool
177 __warn_unused_result spin_trylock_physical(struct spinlock *lock) {
178 return raw_spin_trylock(lock: &lock->raw);
179}
180
181static inline void spin_lock_physical(struct spinlock *lock) {
182 raw_spin_lock(lock: &lock->raw);
183}
184
185static inline void spin_unlock_physical(struct spinlock *lock) {
186 raw_spin_unlock(lock: &lock->raw);
187}
188
189static inline void
190spinlock_init_chk_internal(struct spinlock *lock,
191 const struct lock_chk_class *class,
192 enum lock_chk_flags flags) {
193 raw_spinlock_init(lock: &lock->raw);
194 spinlock_policy_init_internal(lock, flags);
195 spinlock_shallow_init_internal(lock);
196 spinlock_map_init_internal(lock, class, flags);
197}
198
199static inline void spinlock_restore_interrupts(bool enabled) {
200 if (enabled)
201 enable_interrupts();
202}
203
204static inline bool __warn_unused_result spin_trylock_raw_internal(
205 struct spinlock *lock, const struct lock_chk_site *site) {
206 spinlock_note_use(lock, true);
207
208#ifdef DEBUG_LOCK_CHK
209 struct lock_chk_acquire_request req;
210 struct lock_chk_acquire_token token;
211 bool checked_deep = spinlock_deep_checked(lock);
212 if (checked_deep) {
213 req = spinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_TRY, 0,
214 true, false);
215 lock_chk_before_acquire(&token, &req);
216 }
217#endif
218
219 if (spin_trylock_physical(lock)) {
220
221#ifdef DEBUG_LOCK_CHK
222 if (checked_deep)
223 lock_chk_acquired(&token);
224#endif
225
226 return true;
227 }
228
229#ifdef DEBUG_LOCK_CHK
230 if (checked_deep)
231 lock_chk_cancel(&token);
232#endif
233
234 return false;
235}
236
237static inline void spin_lock_raw_internal(struct spinlock *lock,
238 const struct lock_chk_site *site) {
239 spinlock_note_use(lock, true);
240
241#ifdef DEBUG_LOCK_CHK
242 struct lock_chk_acquire_request req;
243 struct lock_chk_acquire_token token;
244 bool checked_deep = spinlock_deep_checked(lock);
245 if (checked_deep) {
246 req = spinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_BLOCKING,
247 0, true, false);
248 lock_chk_before_acquire(&token, &req);
249 }
250#endif
251
252 spin_lock_physical(lock);
253
254#ifdef DEBUG_LOCK_CHK
255 if (checked_deep)
256 lock_chk_acquired(&token);
257#endif
258}
259
260static inline void spin_unlock_raw_internal(struct spinlock *lock,
261 const struct lock_chk_site *site) {
262
263#ifdef DEBUG_LOCK_CHK
264 struct lock_chk_release_request req;
265 struct lock_chk_release_token token;
266 bool checked_deep = spinlock_deep_checked(lock);
267 if (checked_deep) {
268 req = spinlock_chk_release_request(lock, site);
269 lock_chk_before_release(&token, &req);
270 }
271#endif
272
273 spin_unlock_physical(lock);
274
275#ifdef DEBUG_LOCK_CHK
276 if (checked_deep)
277 lock_chk_released(&token);
278#endif
279}
280
281static inline void spin_unlock_internal(struct spinlock *lock, enum irql old,
282 const struct lock_chk_site *site) {
283
284#ifdef DEBUG_LOCK_CHK
285 bool checked_shallow = spinlock_order_checked(lock);
286 struct lock_chk_release_request req;
287 struct lock_chk_release_token token;
288 bool checked_deep = spinlock_deep_checked(lock);
289#endif
290
291 bool irqs_enabled = are_interrupts_enabled();
292 disable_interrupts();
293
294#ifdef DEBUG_LOCK_CHK
295 /* TODO: the ad-hoc chk.x = y should be moved to init EVERYWHERE */
296 lock->chk.type = LOCK_CHK_TYPE_SPIN;
297 lock->chk.instance = lock;
298 if (checked_shallow)
299 lock_debug_spin_validate_top(&lock->chk, old, site);
300 if (checked_deep) {
301 req = spinlock_chk_release_request(lock, site);
302 lock_chk_before_release(&token, &req);
303 }
304#endif
305
306 spin_unlock_physical(lock);
307
308#ifdef DEBUG_LOCK_CHK
309 if (checked_deep)
310 lock_chk_released(&token);
311 if (checked_shallow)
312 lock_debug_spin_pop(&lock->chk);
313#endif
314
315 spinlock_restore_interrupts(enabled: irqs_enabled);
316
317 irql_lower(old_level: old);
318}
319
320static inline enum irql __warn_unused_result spin_lock_subclass_internal(
321 struct spinlock *lock, uint8_t subclass, const struct lock_chk_site *site) {
322 kassert(subclass < LOCK_CHK_MAX_SUBCLASSES);
323 if (bootstage_get() >= BOOTSTAGE_MID_MP &&
324 (irq_in_interrupt() || irq_in_nmi()))
325 panic("Attempted to take non-ISR safe spinlock outside thread context");
326
327 spinlock_note_use(lock, false);
328 bool checked_shallow = spinlock_order_checked(lock);
329 if (checked_shallow)
330 spinlock_classify(lock, usage: LOCK_DEBUG_IRQ_DISPATCH, site);
331
332#ifdef DEBUG_LOCK_CHK
333 struct lock_chk_acquire_request req;
334 struct lock_chk_acquire_token token;
335 bool checked_deep = spinlock_deep_checked(lock);
336 if (checked_deep) {
337 req = spinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_BLOCKING,
338 subclass, false, false);
339 lock_chk_before_acquire(&token, &req);
340 }
341#endif
342
343 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
344 spin_lock_physical(lock);
345
346 bool irqs_enabled = are_interrupts_enabled();
347 disable_interrupts();
348
349#ifdef DEBUG_LOCK_CHK
350 lock->chk.instance = lock;
351 lock->chk.type = LOCK_CHK_TYPE_SPIN;
352 if (checked_shallow)
353 lock_debug_spin_push(&lock->chk, irql, site);
354
355 if (checked_deep)
356 lock_chk_acquired(&token);
357#endif
358
359 spinlock_restore_interrupts(enabled: irqs_enabled);
360
361 return irql;
362}
363
364static inline enum irql __warn_unused_result
365spin_lock_internal(struct spinlock *lock, const struct lock_chk_site *site) {
366 return spin_lock_subclass_internal(lock, subclass: 0, site);
367}
368
369static inline enum irql __warn_unused_result spin_lock_irq_disable_internal(
370 struct spinlock *lock, const struct lock_chk_site *site) {
371 if (bootstage_get() >= BOOTSTAGE_MID_MP && irq_in_nmi())
372 panic("Attempted to take non-raw spinlock from an NMI");
373
374 spinlock_note_use(lock, false);
375 bool checked_shallow = spinlock_order_checked(lock);
376 if (checked_shallow)
377 spinlock_classify(lock, usage: LOCK_DEBUG_IRQ_HIGH, site);
378
379#ifdef DEBUG_LOCK_CHK
380 struct lock_chk_acquire_request req;
381 struct lock_chk_acquire_token token;
382 bool checked_deep = spinlock_deep_checked(lock);
383 if (checked_deep) {
384 req = spinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_BLOCKING,
385 0, false, true);
386 lock_chk_before_acquire(&token, &req);
387 }
388#endif
389
390 enum irql irql = irql_raise(new_level: IRQL_HIGH_LEVEL);
391 spin_lock_physical(lock);
392
393#ifdef DEBUG_LOCK_CHK
394 lock->chk.instance = lock;
395 lock->chk.type = LOCK_CHK_TYPE_SPIN;
396 if (checked_shallow)
397 lock_debug_spin_push(&lock->chk, irql, site);
398
399 if (checked_deep)
400 lock_chk_acquired(&token);
401#endif
402
403 return irql;
404}
405
406static inline bool __warn_unused_result spin_trylock_internal(
407 struct spinlock *lock, enum irql *out, const struct lock_chk_site *site) {
408 if (bootstage_get() >= BOOTSTAGE_MID_MP &&
409 (irq_in_interrupt() || irq_in_nmi()))
410 panic("Attempted to take non-ISR safe spinlock outside thread context");
411
412 spinlock_note_use(lock, false);
413 bool checked_shallow = spinlock_order_checked(lock);
414 if (checked_shallow)
415 spinlock_classify(lock, usage: LOCK_DEBUG_IRQ_DISPATCH, site);
416
417#ifdef DEBUG_LOCK_CHK
418 struct lock_chk_acquire_request req;
419 struct lock_chk_acquire_token token;
420 bool checked_deep = spinlock_deep_checked(lock);
421 if (checked_deep) {
422 req = spinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_TRY, 0,
423 false, false);
424 lock_chk_before_acquire(&token, &req);
425 }
426#endif
427
428 *out = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
429 if (spin_trylock_physical(lock)) {
430 bool irqs_enabled = are_interrupts_enabled();
431 disable_interrupts();
432
433#ifdef DEBUG_LOCK_CHK
434 lock->chk.instance = lock;
435 lock->chk.type = LOCK_CHK_TYPE_SPIN;
436 if (checked_shallow)
437 lock_debug_spin_push(&lock->chk, *out, site);
438 if (checked_deep)
439 lock_chk_acquired(&token);
440#endif
441
442 spinlock_restore_interrupts(enabled: irqs_enabled);
443 return true;
444 }
445
446#ifdef DEBUG_LOCK_CHK
447 if (checked_deep)
448 lock_chk_cancel(&token);
449#endif
450
451 irql_lower(old_level: *out);
452 return false;
453}
454
455static inline bool __warn_unused_result spin_trylock_irq_disable_internal(
456 struct spinlock *lock, enum irql *out, const struct lock_chk_site *site) {
457 if (bootstage_get() >= BOOTSTAGE_MID_MP && irq_in_nmi())
458 panic("Attempted to take non-raw spinlock from an NMI");
459
460 spinlock_note_use(lock, false);
461 bool checked_shallow = spinlock_order_checked(lock);
462 if (checked_shallow)
463 spinlock_classify(lock, usage: LOCK_DEBUG_IRQ_HIGH, site);
464
465#ifdef DEBUG_LOCK_CHK
466 struct lock_chk_acquire_request req;
467 struct lock_chk_acquire_token token;
468 bool checked_deep = spinlock_deep_checked(lock);
469 if (checked_deep) {
470 req = spinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_TRY, 0,
471 false, true);
472 lock_chk_before_acquire(&token, &req);
473 }
474#endif
475
476 *out = irql_raise(new_level: IRQL_HIGH_LEVEL);
477 if (spin_trylock_physical(lock)) {
478
479#ifdef DEBUG_LOCK_CHK
480 lock->chk.instance = lock;
481 lock->chk.type = LOCK_CHK_TYPE_SPIN;
482 if (checked_shallow)
483 lock_debug_spin_push(&lock->chk, *out, site);
484 if (checked_deep)
485 lock_chk_acquired(&token);
486#endif
487
488 return true;
489 }
490
491#ifdef DEBUG_LOCK_CHK
492 if (checked_deep)
493 lock_chk_cancel(&token);
494#endif
495 irql_lower(old_level: *out);
496 return false;
497}
498
499#define spin_lock(lock_) spin_lock_internal((lock_), LOCK_CHK_SITE_HERE())
500#define spin_lock_subclass(lock_, subclass_) \
501 spin_lock_subclass_internal((lock_), (subclass_), LOCK_CHK_SITE_HERE())
502#define spin_lock_irq_disable(lock_) \
503 spin_lock_irq_disable_internal((lock_), LOCK_CHK_SITE_HERE())
504#define spin_trylock(lock_, out_) \
505 spin_trylock_internal((lock_), (out_), LOCK_CHK_SITE_HERE())
506#define spin_trylock_irq_disable(lock_, out_) \
507 spin_trylock_irq_disable_internal((lock_), (out_), LOCK_CHK_SITE_HERE())
508#define spin_unlock(lock_, old_) \
509 spin_unlock_internal((lock_), (old_), LOCK_CHK_SITE_HERE())
510#define spin_lock_raw(lock_) \
511 spin_lock_raw_internal((lock_), LOCK_CHK_SITE_HERE())
512#define spin_trylock_raw(lock_) \
513 spin_trylock_raw_internal((lock_), LOCK_CHK_SITE_HERE())
514#define spin_unlock_raw(lock_) \
515 spin_unlock_raw_internal((lock_), LOCK_CHK_SITE_HERE())
516
517static inline bool spinlock_locked(struct spinlock *lock) {
518 return atomic_load(&lock->raw.state);
519}
520
521/* A raw check to make sure it is locked, but could be by anyone */
522#define SPINLOCK_ASSERT_LOCKED(l) \
523 kassert(spinlock_locked(l), "spinlock not locked")
524
525static inline void spinlock_set_chk_flags(struct spinlock *lock,
526 enum lock_chk_flags flags) {
527
528#ifdef DEBUG_LOCK_CHK
529 kassert(lock->chk.initialized);
530 kassert(!spinlock_locked(lock));
531 kassert(!atomic_load_explicit(&lock->chk.used, memory_order_relaxed));
532 kassert((flags & ~LOCK_CHKD_FULL) == 0);
533 lock->chk.flags = flags;
534#endif
535}
536
537static inline void spinlock_reinit_chk(struct spinlock *lock,
538 const struct lock_chk_class *class,
539 enum lock_chk_flags flags) {
540
541#ifdef DEBUG_LOCK_CHK
542 kassert(lock->chk.initialized);
543 kassert(!spinlock_locked(lock));
544#endif
545
546 spinlock_init_chk_internal(lock, class, flags);
547}
548
549static inline void
550spinlock_assert_held_internal(struct spinlock *lock,
551 const struct lock_chk_site *site) {
552#ifdef DEBUG_LOCK_CHK
553 lock->chk.instance = lock;
554 lock->chk.type = LOCK_CHK_TYPE_SPIN;
555 if (spinlock_deep_checked(lock) &&
556 lock_chk_assert_held_deep(&lock->chk, LOCK_CHK_MODE_IGNORED,
557 /*want_held=*/true, site))
558 return;
559#else
560 unused(site);
561#endif
562 SPINLOCK_ASSERT_LOCKED(lock);
563}
564
565static inline void
566spinlock_assert_not_held_internal(struct spinlock *lock,
567 const struct lock_chk_site *site) {
568#ifdef DEBUG_LOCK_CHK
569 lock->chk.instance = lock;
570 lock->chk.type = LOCK_CHK_TYPE_SPIN;
571 if (spinlock_deep_checked(lock) &&
572 lock_chk_assert_held_deep(&lock->chk, LOCK_CHK_MODE_IGNORED,
573 /*want_held=*/false, site))
574 return;
575#else
576 unused(site);
577#endif
578 kassert(!spinlock_locked(lock), "spinlock unexpectedly locked");
579}
580
581#define SPINLOCK_ASSERT_HELD(l) \
582 spinlock_assert_held_internal((l), LOCK_CHK_SITE_HERE())
583#define SPINLOCK_ASSERT_NOT_HELD(l) \
584 spinlock_assert_not_held_internal((l), LOCK_CHK_SITE_HERE())
585