1/* @title: Queued Spinlock (MCS-based 4-byte qspinlock) */
2#pragma once
3#include "console/crash.h"
4#include <asm.h>
5#include <bootstage.h>
6#include <compiler.h>
7#include <console/panic.h>
8#include <irq/irq.h>
9#include <kassert.h>
10#include <sch/irql.h>
11#include <stdatomic.h>
12#include <stdbool.h>
13#include <stdint.h>
14#include <sync/lock_chk_types.h>
15
16/* TODO: I would really like to use a CNA lock/cohorting as an extension to
17 * qspinlock, possibly also with a boot-time flag to enable/disable it
18 * for testing. Could be super interesting! */
19
20/*
21 * Bit layout of the lock word:
22 *
23 * 0 - 7: locked byte (Q_SPIN_LOCKED_VAL = 0x01)
24 * 8 : pending bit (Q_SPIN_PENDING_VAL = 0x100)
25 * 9 - 10: tail context index
26 * 16 - 31: tail CPU ID + 1 (16 bits)
27 */
28#define Q_SPIN_LOCKED_OFFSET 0
29#define Q_SPIN_LOCKED_BITS 8
30#define Q_SPIN_LOCKED_MASK 0x000000FFU
31#define Q_SPIN_LOCKED_VAL 0x00000001U
32
33#define Q_SPIN_PENDING_OFFSET 8
34#define Q_SPIN_PENDING_BITS 1
35#define Q_SPIN_PENDING_MASK 0x00000100U
36#define Q_SPIN_PENDING_VAL 0x00000100U
37
38#define Q_SPIN_LOCKED_PENDING_MASK (Q_SPIN_LOCKED_MASK | Q_SPIN_PENDING_MASK)
39
40#define Q_SPIN_TAIL_LVL_OFFSET 9
41#define Q_SPIN_TAIL_LVL_BITS 2
42#define Q_SPIN_TAIL_LVL_MASK 0x00000600U
43
44#define Q_SPIN_TAIL_CPU_OFFSET 16
45#define Q_SPIN_TAIL_CPU_BITS 16
46#define Q_SPIN_TAIL_CPU_MASK 0xFFFF0000U
47
48#define Q_SPIN_TAIL_MASK (Q_SPIN_TAIL_LVL_MASK | Q_SPIN_TAIL_CPU_MASK)
49
50enum qspinlock_level {
51 QSPINLOCK_LEVEL_NORMAL, /* prev irql = DISPATCH */
52 QSPINLOCK_LEVEL_IRQ, /* prev irql = HIGH */
53 QSPINLOCK_LEVEL_NMI, /* For later usage */
54 QSPINLOCK_LEVEL_MAX,
55};
56
57struct qspinlock {
58 _Atomic uint32_t val;
59
60#ifdef DEBUG_LOCK_CHK
61 struct lock_chk_lock chk;
62 _Atomic uint8_t irq_usage;
63#endif /* DEBUG_LOCK_CHK */
64};
65
66#define QSPINLOCK_INIT QSPINLOCK_INIT_CHK(NULL, LOCK_CHKD_FULL)
67#define QSPINLOCK_DEFINE(id) struct qspinlock id = QSPINLOCK_INIT
68#define QSPINLOCK_DEFINE_CHK(id, class_, flags_) \
69 struct qspinlock id = QSPINLOCK_INIT_CHK((class_), (flags_))
70
71static inline void
72qspinlock_init_chk_internal(struct qspinlock *lock,
73 const struct lock_chk_class *class,
74 enum lock_chk_flags flags);
75static inline bool qspin_is_locked(const struct qspinlock *lock);
76
77#ifdef DEBUG_LOCK_CHK
78
79#define __QSPINLOCK_SHALLOW_VALUE_INIT \
80 , .irq_usage = ATOMIC_VAR_INIT(LOCK_DEBUG_IRQ_NONE)
81#define __QSPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_) \
82 , .chk = LOCK_CHK_LOCK_VALUE_INIT((class_), (flags_))
83#define QSPINLOCK_INIT_CHK(class_, flags_) \
84 ((struct qspinlock) { \
85 .val = ATOMIC_VAR_INIT(0) __QSPINLOCK_LOCK_CHK_VALUE_INIT( \
86 (class_), (flags_)) __QSPINLOCK_SHALLOW_VALUE_INIT})
87
88#define qspinlock_init_chk(lock_, class_, flags_) \
89 qspinlock_init_chk_internal((lock_), (class_), (flags_))
90#define qspinlock_init_auto_internal(lock_, flags_) \
91 do { \
92 static const struct lock_chk_class __auto_class = { \
93 .name = #lock_, \
94 .file = __RELFILE__, \
95 .line = __LINE__, \
96 }; \
97 qspinlock_init_chk_internal((lock_), &__auto_class, (flags_)); \
98 } while (0)
99
100static inline void qspinlock_policy_init_internal(struct qspinlock *lock,
101 enum lock_chk_flags flags) {
102 kassert((flags & ~LOCK_CHKD_FULL) == 0);
103 lock->chk.flags = flags;
104 lock->chk.initialized = true;
105 atomic_store_explicit(&lock->chk.used, false, memory_order_relaxed);
106}
107
108static inline void
109qspinlock_map_init_internal(struct qspinlock *lock,
110 const struct lock_chk_class *class,
111 enum lock_chk_flags flags) {
112 kassert(flags == LOCK_UNCHKD || class != NULL);
113 lock_chk_map_runtime_init(&lock->chk.map, class);
114}
115
116static inline void qspinlock_shallow_init_internal(struct qspinlock *lock) {
117 atomic_store_explicit(&lock->irq_usage, LOCK_DEBUG_IRQ_NONE,
118 memory_order_relaxed);
119}
120
121static inline void qspinlock_set_chk_flags(struct qspinlock *lock,
122 enum lock_chk_flags flags) {
123 kassert(lock->chk.initialized);
124 kassert(!qspin_is_locked(lock));
125 kassert(!atomic_load_explicit(&lock->chk.used, memory_order_relaxed));
126 kassert((flags & ~LOCK_CHKD_FULL) == 0);
127 lock->chk.flags = flags;
128}
129
130static inline void qspinlock_reinit_chk(struct qspinlock *lock,
131 const struct lock_chk_class *class,
132 enum lock_chk_flags flags) {
133 kassert(lock->chk.initialized);
134 kassert(!qspin_is_locked(lock));
135 qspinlock_init_chk_internal(lock, class, flags);
136}
137
138static inline void qspinlock_note_use(struct qspinlock *lock,
139 bool raw_operation) {
140 lock_chk_note_lock_use(&lock->chk, /*manages_irql=*/true, raw_operation);
141}
142
143static inline bool qspinlock_order_checked(struct qspinlock *lock) {
144 return lock_chk_tracking_active() &&
145 (lock->chk.flags & LOCK_CHKD_ORDER) != 0;
146}
147
148static inline void qspinlock_classify(struct qspinlock *lock,
149 enum lock_debug_irq_usage usage,
150 const struct lock_chk_site *site) {
151 lock->chk.type = LOCK_CHK_TYPE_QSPIN;
152 lock->chk.instance = lock;
153 lock_debug_spin_classify(&lock->irq_usage, usage, &lock->chk, site);
154}
155
156static inline bool qspinlock_deep_checked(struct qspinlock *lock) {
157 return lock_chk_tracking_active() && lock->chk.flags != LOCK_UNCHKD;
158}
159
160static inline struct lock_chk_acquire_request qspinlock_chk_acquire_request(
161 struct qspinlock *lock, const struct lock_chk_site *site,
162 enum lock_chk_wait_kind wait_kind, uint8_t subclass, bool raw_operation,
163 bool irq_safe) {
164 lock->chk.instance = lock;
165 lock->chk.type = LOCK_CHK_TYPE_QSPIN;
166 return lock_chk_acquire_request_make(&lock->chk, site,
167 LOCK_CHK_MODE_EXCLUSIVE, wait_kind,
168 subclass, raw_operation, irq_safe);
169}
170
171static inline struct lock_chk_release_request
172qspinlock_chk_release_request(struct qspinlock *lock,
173 const struct lock_chk_site *site) {
174 lock->chk.instance = lock;
175 lock->chk.type = LOCK_CHK_TYPE_QSPIN;
176 return lock_chk_release_request_make(&lock->chk, site,
177 LOCK_CHK_MODE_EXCLUSIVE);
178}
179
180#else /* !defined(DEBUG_LOCK_CHK) */
181
182#define __QSPINLOCK_SHALLOW_VALUE_INIT
183#define __QSPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_)
184#define QSPINLOCK_INIT_CHK(class_, flags_) \
185 ((struct qspinlock) {.val = ATOMIC_VAR_INIT(0)})
186
187#define qspinlock_init_chk(lock_, class_, flags_) \
188 qspinlock_init_chk_internal((lock_), NULL, LOCK_UNCHKD)
189#define qspinlock_init_auto_internal(lock_, flags_) \
190 qspinlock_init_chk_internal((lock_), NULL, LOCK_UNCHKD)
191
192static inline void qspinlock_policy_init_internal(struct qspinlock *lock,
193 enum lock_chk_flags flags) {
194 unused(lock, flags);
195}
196
197static inline void
198qspinlock_map_init_internal(struct qspinlock *lock,
199 const struct lock_chk_class *class,
200 enum lock_chk_flags flags) {
201 unused(lock, class, flags);
202}
203
204static inline void qspinlock_shallow_init_internal(struct qspinlock *lock) {
205 unused(lock);
206}
207
208static inline void qspinlock_set_chk_flags(struct qspinlock *lock,
209 enum lock_chk_flags flags) {
210 unused(lock, flags);
211}
212
213static inline void qspinlock_reinit_chk(struct qspinlock *lock,
214 const struct lock_chk_class *class,
215 enum lock_chk_flags flags) {
216 qspinlock_init_chk_internal(lock, class, flags);
217}
218
219static inline void qspinlock_note_use(struct qspinlock *lock,
220 bool raw_operation) {
221 unused(lock, raw_operation);
222}
223
224static inline bool qspinlock_order_checked(struct qspinlock *lock) {
225 unused(lock);
226 return false;
227}
228
229static inline void qspinlock_classify(struct qspinlock *lock,
230 enum lock_debug_irq_usage usage,
231 const struct lock_chk_site *site) {
232 unused(lock, usage, site);
233}
234
235static inline bool qspinlock_deep_checked(struct qspinlock *lock) {
236 unused(lock);
237 return false;
238}
239
240#endif /* DEBUG_LOCK_CHK */
241
242static inline void
243qspinlock_init_chk_internal(struct qspinlock *lock,
244 const struct lock_chk_class *class,
245 enum lock_chk_flags flags) {
246 atomic_store_explicit(&lock->val, 0, memory_order_relaxed);
247 qspinlock_policy_init_internal(lock, flags);
248 qspinlock_shallow_init_internal(lock);
249 qspinlock_map_init_internal(lock, class, flags);
250}
251
252#define qspinlock_init_1(lock_) \
253 qspinlock_init_auto_internal((lock_), LOCK_CHKD_FULL)
254#define qspinlock_init_2(lock_, flags_) \
255 qspinlock_init_auto_internal((lock_), (flags_))
256#define qspinlock_init(...) \
257 _DISPATCH(qspinlock_init, PP_NARG(__VA_ARGS__))(__VA_ARGS__)
258#define qspin_init(...) qspinlock_init(__VA_ARGS__)
259#define qspin_init_chk(...) qspinlock_init_chk(__VA_ARGS__)
260
261static inline bool
262 __warn_unused_result qspin_trylock_physical(struct qspinlock *lock) {
263 uint32_t expected = 0;
264 return atomic_compare_exchange_strong_explicit(
265 &lock->val, &expected, Q_SPIN_LOCKED_VAL, memory_order_acquire,
266 memory_order_relaxed);
267}
268
269void qspin_lock_slowpath(struct qspinlock *lock, uint32_t val);
270
271static inline void qspin_lock_physical(struct qspinlock *lock) {
272 uint32_t val = 0;
273 if (likely(atomic_compare_exchange_strong_explicit(
274 &lock->val, &val, Q_SPIN_LOCKED_VAL, memory_order_acquire,
275 memory_order_relaxed)))
276 return;
277
278 qspin_lock_slowpath(lock, val);
279}
280
281static inline void qspin_unlock_physical(struct qspinlock *lock) {
282 /* Clear only the locked byte so pending and tail bits remain valid */
283 atomic_store_explicit((_Atomic uint8_t *) &lock->val, 0,
284 memory_order_release);
285}
286
287static inline bool qspin_is_locked(const struct qspinlock *lock) {
288 return (atomic_load_explicit(&lock->val, memory_order_relaxed) &
289 Q_SPIN_LOCKED_MASK) != 0;
290}
291
292/* Same deal as SPINLOCK_ASSERT_LOCKED, just checks the bit,
293 * but that doesn't say anything about thread ownership */
294#define QSPINLOCK_ASSERT_LOCKED(l) \
295 kassert(qspin_is_locked(l), "qspinlock not locked")
296
297static inline bool __warn_unused_result qspin_trylock_raw_internal(
298 struct qspinlock *lock, const struct lock_chk_site *site) {
299 qspinlock_note_use(lock, true);
300
301#ifdef DEBUG_LOCK_CHK
302 struct lock_chk_acquire_request req;
303 struct lock_chk_acquire_token token;
304 bool checked_deep = qspinlock_deep_checked(lock);
305 if (checked_deep) {
306 req = qspinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_TRY, 0,
307 true, false);
308 lock_chk_before_acquire(&token, &req);
309 }
310#endif
311
312 if (qspin_trylock_physical(lock)) {
313
314#ifdef DEBUG_LOCK_CHK
315 if (checked_deep)
316 lock_chk_acquired(&token);
317#endif
318
319 return true;
320 }
321
322#ifdef DEBUG_LOCK_CHK
323 if (checked_deep)
324 lock_chk_cancel(&token);
325#endif
326
327 return false;
328}
329
330static inline void qspin_lock_raw_internal(struct qspinlock *lock,
331 const struct lock_chk_site *site) {
332 qspinlock_note_use(lock, true);
333
334#ifdef DEBUG_LOCK_CHK
335 struct lock_chk_acquire_request req;
336 struct lock_chk_acquire_token token;
337 bool checked_deep = qspinlock_deep_checked(lock);
338 if (checked_deep) {
339 req = qspinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_BLOCKING,
340 0, true, false);
341 lock_chk_before_acquire(&token, &req);
342 }
343#endif
344
345 qspin_lock_physical(lock);
346
347#ifdef DEBUG_LOCK_CHK
348 if (checked_deep)
349 lock_chk_acquired(&token);
350#endif
351}
352
353static inline void qspin_unlock_raw_internal(struct qspinlock *lock,
354 const struct lock_chk_site *site) {
355
356#ifdef DEBUG_LOCK_CHK
357 struct lock_chk_release_request req;
358 struct lock_chk_release_token token;
359 bool checked_deep = qspinlock_deep_checked(lock);
360 if (checked_deep) {
361 req = qspinlock_chk_release_request(lock, site);
362 lock_chk_before_release(&token, &req);
363 }
364#endif
365
366 qspin_unlock_physical(lock);
367
368#ifdef DEBUG_LOCK_CHK
369 if (checked_deep)
370 lock_chk_released(&token);
371#endif
372}
373
374static inline void qspin_unlock_internal(struct qspinlock *lock,
375 enum irql old_irql,
376 const struct lock_chk_site *site) {
377#ifdef DEBUG_LOCK_CHK
378 bool checked_shallow = qspinlock_order_checked(lock);
379 struct lock_chk_release_request req;
380 struct lock_chk_release_token token;
381 bool checked_deep = qspinlock_deep_checked(lock);
382#endif
383
384 bool irqs_enabled = are_interrupts_enabled();
385 disable_interrupts();
386
387#ifdef DEBUG_LOCK_CHK
388 lock->chk.instance = lock;
389 lock->chk.type = LOCK_CHK_TYPE_QSPIN;
390 if (checked_shallow)
391 lock_debug_spin_validate_top(&lock->chk, old_irql, site);
392 if (checked_deep) {
393 req = qspinlock_chk_release_request(lock, site);
394 lock_chk_before_release(&token, &req);
395 }
396#endif
397
398 qspin_unlock_physical(lock);
399
400#ifdef DEBUG_LOCK_CHK
401 if (checked_deep)
402 lock_chk_released(&token);
403 if (checked_shallow)
404 lock_debug_spin_pop(&lock->chk);
405#endif
406
407 crash_unwind_exit_qspinlock(q: lock);
408 if (irqs_enabled)
409 enable_interrupts();
410
411 irql_lower(old_level: old_irql);
412}
413
414static inline enum irql __warn_unused_result
415qspin_lock_subclass_internal(struct qspinlock *lock, uint8_t subclass,
416 const struct lock_chk_site *site) {
417 kassert(subclass < LOCK_CHK_MAX_SUBCLASSES);
418 if (bootstage_get() >= BOOTSTAGE_MID_MP &&
419 (irq_in_interrupt() || irq_in_nmi()))
420 panic(
421 "Attempted to take non-ISR safe qspinlock outside thread context");
422
423 qspinlock_note_use(lock, false);
424 bool checked_shallow = qspinlock_order_checked(lock);
425 if (checked_shallow)
426 qspinlock_classify(lock, usage: LOCK_DEBUG_IRQ_DISPATCH, site);
427
428#ifdef DEBUG_LOCK_CHK
429 struct lock_chk_acquire_request req;
430 struct lock_chk_acquire_token token;
431 bool checked_deep = qspinlock_deep_checked(lock);
432 if (checked_deep) {
433 req = qspinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_BLOCKING,
434 subclass, false, false);
435 lock_chk_before_acquire(&token, &req);
436 }
437#endif
438
439 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
440 qspin_lock_physical(lock);
441
442 kassert(are_interrupts_enabled());
443 disable_interrupts();
444
445#ifdef DEBUG_LOCK_CHK
446 lock->chk.instance = lock;
447 lock->chk.type = LOCK_CHK_TYPE_QSPIN;
448
449 if (checked_shallow)
450 lock_debug_spin_push(&lock->chk, irql, site);
451
452 if (checked_deep)
453 lock_chk_acquired(&token);
454#endif
455
456 crash_unwind_enter_qspinlock(q: lock, old: irql);
457 enable_interrupts();
458
459 return irql;
460}
461
462static inline enum irql __warn_unused_result
463qspin_lock_internal(struct qspinlock *lock, const struct lock_chk_site *site) {
464 return qspin_lock_subclass_internal(lock, subclass: 0, site);
465}
466
467static inline enum irql __warn_unused_result qspin_lock_irq_disable_internal(
468 struct qspinlock *lock, const struct lock_chk_site *site) {
469 if (bootstage_get() >= BOOTSTAGE_MID_MP && irq_in_nmi())
470 panic("Attempted to take non-raw qspinlock from an NMI");
471
472 qspinlock_note_use(lock, false);
473 bool checked_shallow = qspinlock_order_checked(lock);
474 if (checked_shallow)
475 qspinlock_classify(lock, usage: LOCK_DEBUG_IRQ_HIGH, site);
476
477#ifdef DEBUG_LOCK_CHK
478 struct lock_chk_acquire_request req;
479 struct lock_chk_acquire_token token;
480 bool checked_deep = qspinlock_deep_checked(lock);
481 if (checked_deep) {
482 req = qspinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_BLOCKING,
483 0, false, true);
484 lock_chk_before_acquire(&token, &req);
485 }
486#endif
487
488 enum irql irql = irql_raise(new_level: IRQL_HIGH_LEVEL);
489 qspin_lock_physical(lock);
490
491#ifdef DEBUG_LOCK_CHK
492 lock->chk.instance = lock;
493 lock->chk.type = LOCK_CHK_TYPE_QSPIN;
494 if (checked_shallow)
495 lock_debug_spin_push(&lock->chk, irql, site);
496
497 if (checked_deep)
498 lock_chk_acquired(&token);
499#endif
500
501 crash_unwind_enter_qspinlock(q: lock, old: irql);
502
503 return irql;
504}
505
506static inline bool __warn_unused_result qspin_trylock_internal(
507 struct qspinlock *lock, enum irql *out, const struct lock_chk_site *site) {
508 if (bootstage_get() >= BOOTSTAGE_MID_MP &&
509 (irq_in_interrupt() || irq_in_nmi()))
510 panic(
511 "Attempted to take non-ISR safe qspinlock outside thread context");
512
513 qspinlock_note_use(lock, false);
514 bool checked_shallow = qspinlock_order_checked(lock);
515 if (checked_shallow)
516 qspinlock_classify(lock, usage: LOCK_DEBUG_IRQ_DISPATCH, site);
517
518#ifdef DEBUG_LOCK_CHK
519 struct lock_chk_acquire_request req;
520 struct lock_chk_acquire_token token;
521 bool checked_deep = qspinlock_deep_checked(lock);
522 if (checked_deep) {
523 req = qspinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_TRY, 0,
524 false, false);
525 lock_chk_before_acquire(&token, &req);
526 }
527#endif
528
529 *out = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
530 if (qspin_trylock_physical(lock)) {
531 kassert(are_interrupts_enabled()); /* Should not go false */
532 disable_interrupts();
533#ifdef DEBUG_LOCK_CHK
534 lock->chk.instance = lock;
535 lock->chk.type = LOCK_CHK_TYPE_QSPIN;
536 if (checked_shallow)
537 lock_debug_spin_push(&lock->chk, *out, site);
538
539 if (checked_deep)
540 lock_chk_acquired(&token);
541#endif
542 enable_interrupts();
543 crash_unwind_enter_qspinlock(q: lock, old: *out);
544 return true;
545 }
546
547#ifdef DEBUG_LOCK_CHK
548 if (checked_deep)
549 lock_chk_cancel(&token);
550#endif
551
552 irql_lower(old_level: *out);
553 return false;
554}
555
556static inline bool __warn_unused_result qspin_trylock_irq_disable_internal(
557 struct qspinlock *lock, enum irql *out, const struct lock_chk_site *site) {
558 if (bootstage_get() >= BOOTSTAGE_MID_MP && irq_in_nmi())
559 panic("Attempted to take non-raw qspinlock from an NMI");
560
561 qspinlock_note_use(lock, false);
562 bool checked_shallow = qspinlock_order_checked(lock);
563 if (checked_shallow)
564 qspinlock_classify(lock, usage: LOCK_DEBUG_IRQ_HIGH, site);
565
566#ifdef DEBUG_LOCK_CHK
567 struct lock_chk_acquire_request req;
568 struct lock_chk_acquire_token token;
569 bool checked_deep = qspinlock_deep_checked(lock);
570 if (checked_deep) {
571 req = qspinlock_chk_acquire_request(lock, site, LOCK_CHK_WAIT_TRY, 0,
572 false, true);
573 lock_chk_before_acquire(&token, &req);
574 }
575#endif
576
577 *out = irql_raise(new_level: IRQL_HIGH_LEVEL);
578 if (qspin_trylock_physical(lock)) {
579
580#ifdef DEBUG_LOCK_CHK
581 lock->chk.instance = lock;
582 lock->chk.type = LOCK_CHK_TYPE_QSPIN;
583 if (checked_shallow)
584 lock_debug_spin_push(&lock->chk, *out, site);
585
586 if (checked_deep)
587 lock_chk_acquired(&token);
588#endif
589
590 crash_unwind_enter_qspinlock(q: lock, old: *out);
591 return true;
592 }
593
594#ifdef DEBUG_LOCK_CHK
595 if (checked_deep)
596 lock_chk_cancel(&token);
597#endif
598
599 irql_lower(old_level: *out);
600 return false;
601}
602
603#define qspin_lock(lock_) qspin_lock_internal((lock_), LOCK_CHK_SITE_HERE())
604#define qspin_lock_subclass(lock_, subclass_) \
605 qspin_lock_subclass_internal((lock_), (subclass_), LOCK_CHK_SITE_HERE())
606#define qspin_lock_irq_disable(lock_) \
607 qspin_lock_irq_disable_internal((lock_), LOCK_CHK_SITE_HERE())
608#define qspin_unlock(lock_, old_) \
609 qspin_unlock_internal((lock_), (old_), LOCK_CHK_SITE_HERE())
610#define qspin_unlock_irq_restore(lock_, old_) \
611 qspin_unlock_internal((lock_), (old_), LOCK_CHK_SITE_HERE())
612#define qspin_trylock(lock_, out_) \
613 qspin_trylock_internal((lock_), (out_), LOCK_CHK_SITE_HERE())
614#define qspin_trylock_irq_disable(lock_, out_) \
615 qspin_trylock_irq_disable_internal((lock_), (out_), LOCK_CHK_SITE_HERE())
616#define qspin_lock_raw(lock_) \
617 qspin_lock_raw_internal((lock_), LOCK_CHK_SITE_HERE())
618#define qspin_trylock_raw(lock_) \
619 qspin_trylock_raw_internal((lock_), LOCK_CHK_SITE_HERE())
620#define qspin_unlock_raw(lock_) \
621 qspin_unlock_raw_internal((lock_), LOCK_CHK_SITE_HERE())
622
623static inline void
624qspin_assert_held_internal(struct qspinlock *lock,
625 const struct lock_chk_site *site) {
626#ifdef DEBUG_LOCK_CHK
627 lock->chk.instance = lock;
628 lock->chk.type = LOCK_CHK_TYPE_QSPIN;
629 if (qspinlock_deep_checked(lock) &&
630 lock_chk_assert_held_deep(&lock->chk, LOCK_CHK_MODE_IGNORED,
631 /*want_held=*/true, site))
632 return;
633#else
634 unused(site);
635#endif
636 QSPINLOCK_ASSERT_LOCKED(lock);
637}
638
639static inline void
640qspin_assert_not_held_internal(struct qspinlock *lock,
641 const struct lock_chk_site *site) {
642#ifdef DEBUG_LOCK_CHK
643 lock->chk.instance = lock;
644 lock->chk.type = LOCK_CHK_TYPE_QSPIN;
645 if (qspinlock_deep_checked(lock) &&
646 lock_chk_assert_held_deep(&lock->chk, LOCK_CHK_MODE_IGNORED,
647 /*want_held=*/false, site))
648 return;
649#else
650 unused(site);
651#endif
652 kassert(!qspin_is_locked(lock), "qspinlock unexpectedly locked");
653}
654
655#define QSPINLOCK_ASSERT_HELD(l) \
656 qspin_assert_held_internal((l), LOCK_CHK_SITE_HERE())
657#define QSPINLOCK_ASSERT_NOT_HELD(l) \
658 qspin_assert_not_held_internal((l), LOCK_CHK_SITE_HERE())
659