1/* @title: Queued Spinlock (MCS-based 4-byte qspinlock) */
2#pragma once
3#include <asm.h>
4#include <atomic.h>
5#include <bootstage.h>
6#include <compiler/core.h>
7#include <console/crash.h>
8#include <console/panic.h>
9#include <enforce.h>
10#include <irq/irq.h>
11#include <sch/irql.h>
12#include <stdbool.h>
13#include <stdint.h>
14#include <sync/lock_chk_types.h>
15#include <sync/lock_general.h>
16#include <tsa.h>
17
18/* TODO: I would really like to use a CNA lock/cohorting as an extension to
19 * qspinlock, possibly also with a boot-time flag to enable/disable it
20 * for testing. Could be super interesting! */
21
22/*
23 * Bit layout of the lock word:
24 *
25 * 0 - 7: locked byte (Q_SPIN_LOCKED_VAL = 0x01)
26 * 8 : pending bit (Q_SPIN_PENDING_VAL = 0x100)
27 * 9 - 10: tail context index
28 * 16 - 31: tail CPU ID + 1 (16 bits)
29 */
30#define Q_SPIN_LOCKED_OFFSET 0
31#define Q_SPIN_LOCKED_BITS 8
32#define Q_SPIN_LOCKED_MASK 0x000000FFU
33#define Q_SPIN_LOCKED_VAL 0x00000001U
34
35#define Q_SPIN_PENDING_OFFSET 8
36#define Q_SPIN_PENDING_BITS 1
37#define Q_SPIN_PENDING_MASK 0x00000100U
38#define Q_SPIN_PENDING_VAL 0x00000100U
39
40#define Q_SPIN_LOCKED_PENDING_MASK (Q_SPIN_LOCKED_MASK | Q_SPIN_PENDING_MASK)
41
42#define Q_SPIN_TAIL_LVL_OFFSET 9
43#define Q_SPIN_TAIL_LVL_BITS 2
44#define Q_SPIN_TAIL_LVL_MASK 0x00000600U
45
46#define Q_SPIN_TAIL_CPU_OFFSET 16
47#define Q_SPIN_TAIL_CPU_BITS 16
48#define Q_SPIN_TAIL_CPU_MASK 0xFFFF0000U
49
50#define Q_SPIN_TAIL_MASK (Q_SPIN_TAIL_LVL_MASK | Q_SPIN_TAIL_CPU_MASK)
51
52enum qspinlock_level {
53 QSPINLOCK_LEVEL_NORMAL, /* prev irql = DISPATCH */
54 QSPINLOCK_LEVEL_IRQ, /* prev irql = HIGH */
55 QSPINLOCK_LEVEL_NMI, /* For later usage */
56 QSPINLOCK_LEVEL_MAX,
57};
58
59struct TSA_CAPABILITY("spinlock") qspinlock {
60 atomic_uint32_t val;
61
62#ifdef DEBUG_LOCK_CHK
63 struct lock_chk_lock chk;
64 atomic(enum lock_op_flags) irq_usage;
65#endif /* DEBUG_LOCK_CHK */
66};
67
68#define QSPINLOCK_INIT QSPINLOCK_INIT_CHK(NULL, LOCK_CHKD_FULL)
69#define QSPINLOCK_DEFINE(id) struct qspinlock id = QSPINLOCK_INIT
70#define QSPINLOCK_DEFINE_CHK(id, class_, flags_) \
71 struct qspinlock id = QSPINLOCK_INIT_CHK((class_), (flags_))
72
73static inline void qspinlock_init_chk_full(struct qspinlock *lock,
74 const struct lock_chk_class *class,
75 enum lock_chk_flags flags);
76static inline bool qspin_is_locked(const struct qspinlock *lock);
77
78#ifdef DEBUG_LOCK_CHK
79
80#define __QSPINLOCK_SHALLOW_VALUE_INIT \
81 , .irq_usage = ATOMIC_VAR_INIT(LOCK_OP_IRQ_NONE)
82#define __QSPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_) \
83 , .chk = LOCK_CHK_LOCK_VALUE_INIT((class_), (flags_))
84#define QSPINLOCK_INIT_CHK(class_, flags_) \
85 ((struct qspinlock) { \
86 .val = ATOMIC_VAR_INIT(0) __QSPINLOCK_LOCK_CHK_VALUE_INIT( \
87 (class_), (flags_)) __QSPINLOCK_SHALLOW_VALUE_INIT})
88
89#define qspinlock_init_chk(lock_, class_, flags_) \
90 qspinlock_init_chk_full((lock_), (class_), (flags_))
91#define qspinlock_init_auto_internal(lock_, flags_) \
92 do { \
93 static const struct lock_chk_class __auto_class = { \
94 .name = #lock_, \
95 .file = __RELFILE__, \
96 .line = __LINE__, \
97 }; \
98 qspinlock_init_chk_full((lock_), &__auto_class, (flags_)); \
99 } while (0)
100
101static inline void qspinlock_policy_init_internal(struct qspinlock *lock,
102 enum lock_chk_flags flags) {
103 enforce((flags & ~LOCK_CHKD_FULL) == 0);
104 lock->chk.flags = flags;
105 lock->chk.initialized = true;
106 atomic_store_rlx(&lock->chk.used, false);
107}
108
109static inline void
110qspinlock_map_init_internal(struct qspinlock *lock,
111 const struct lock_chk_class *class,
112 enum lock_chk_flags flags) {
113 enforce(flags == LOCK_UNCHKD || class != NULL);
114 lock_chk_map_runtime_init(&lock->chk.map, class);
115}
116
117static inline void qspinlock_shallow_init_internal(struct qspinlock *lock) {
118 atomic_store_rlx(&lock->irq_usage, LOCK_OP_IRQ_NONE);
119}
120
121static inline void qspinlock_set_chk_flags(struct qspinlock *lock,
122 enum lock_chk_flags flags) {
123 enforce(lock->chk.initialized, UNINITIALIZED);
124 enforce(!qspin_is_locked(lock));
125 enforce(!atomic_load_rlx(&lock->chk.used));
126 enforce((flags & ~LOCK_CHKD_FULL) == 0);
127 lock->chk.flags = flags;
128}
129
130static inline void qspinlock_reinit_chk(struct qspinlock *lock,
131 const struct lock_chk_class *class,
132 enum lock_chk_flags flags) {
133 enforce(lock->chk.initialized, UNINITIALIZED);
134 enforce(!qspin_is_locked(lock));
135 qspinlock_init_chk_full(lock, class, flags);
136}
137
138static inline void qspinlock_note_use(struct qspinlock *lock,
139 enum lock_op_flags flags) {
140 lock_chk_note_use(&lock->chk, flags);
141}
142
143static inline bool qspinlock_order_checked(struct qspinlock *lock) {
144 return lock_chk_tracking_active() &&
145 (lock->chk.flags & LOCK_CHKD_ORDER) != 0;
146}
147
148/* TODO: move stamping to initialization, but
149 * we are using this HACK:
150 * because it's fine anyways */
151static inline void qspinlock_stamp(struct qspinlock *lock) {
152 lock->chk.instance = lock;
153 lock->chk.type = LOCK_CHK_TYPE_QSPIN;
154}
155
156static inline void qspinlock_classify(struct qspinlock *lock,
157 enum lock_op_flags usage,
158 const struct lock_chk_site *site) {
159 qspinlock_stamp(lock);
160 lock_debug_spin_classify(&lock->irq_usage, usage, &lock->chk, site);
161}
162
163static inline bool qspinlock_deep_checked(struct qspinlock *lock) {
164 return lock_chk_tracking_active() && lock->chk.flags != LOCK_UNCHKD;
165}
166
167static inline struct lock_chk_acq_req
168qspinlock_chk_acq_req(struct qspinlock *lock, const struct lock_chk_site *site,
169 enum lock_subclass subclass, enum lock_op_flags flags) {
170 qspinlock_stamp(lock);
171 return lock_chk_acq_req_make(&lock->chk, site, LOCK_CHK_MODE_EXCLUSIVE,
172 subclass, flags);
173}
174
175static inline struct lock_chk_rel_req
176qspinlock_chk_rel_req(struct qspinlock *lock,
177 const struct lock_chk_site *site) {
178 qspinlock_stamp(lock);
179 return lock_chk_rel_req_make(&lock->chk, site, LOCK_CHK_MODE_EXCLUSIVE);
180}
181
182struct qspinlock_acq_scope {
183 struct lock_chk_acq_req req;
184 struct lock_chk_acq_token token;
185 bool checked;
186};
187
188struct qspinlock_rel_scope {
189 struct lock_chk_rel_req req;
190 struct lock_chk_rel_token token;
191 bool checked;
192};
193
194static inline void qspinlock_acq_begin(struct qspinlock_acq_scope *scope,
195 struct qspinlock *lock,
196 const struct lock_chk_site *site,
197 enum lock_subclass subclass,
198 enum lock_op_flags flags) {
199 scope->checked = qspinlock_deep_checked(lock);
200 if (!scope->checked)
201 return;
202
203 scope->req = qspinlock_chk_acq_req(lock, site, subclass, flags);
204 lock_chk_before_acq(&scope->token, &scope->req);
205}
206
207static inline void qspinlock_acq_commit(struct qspinlock_acq_scope *scope) {
208 if (scope->checked)
209 lock_chk_acqd(&scope->token);
210}
211
212static inline void qspinlock_acq_abort(struct qspinlock_acq_scope *scope) {
213 if (scope->checked)
214 lock_chk_cancel(&scope->token);
215}
216
217static inline void qspinlock_rel_begin(struct qspinlock_rel_scope *scope,
218 struct qspinlock *lock,
219 const struct lock_chk_site *site) {
220 scope->checked = qspinlock_deep_checked(lock);
221 if (!scope->checked)
222 return;
223
224 scope->req = qspinlock_chk_rel_req(lock, site);
225 lock_chk_before_rel(&scope->token, &scope->req);
226}
227
228static inline void qspinlock_rel_commit(struct qspinlock_rel_scope *scope) {
229 if (scope->checked)
230 lock_chk_reld(&scope->token);
231}
232
233static inline void qspinlock_shallow_push(struct qspinlock *lock,
234 enum irql irql,
235 const struct lock_chk_site *site) {
236 qspinlock_stamp(lock);
237 lock_debug_spin_push(&lock->chk, irql, site);
238}
239
240static inline void
241qspinlock_shallow_validate_top(struct qspinlock *lock, enum irql old,
242 const struct lock_chk_site *site) {
243 qspinlock_stamp(lock);
244 lock_debug_spin_validate_top(&lock->chk, old, site);
245}
246
247static inline void qspinlock_shallow_pop(struct qspinlock *lock) {
248 lock_debug_spin_pop(&lock->chk);
249}
250
251static inline bool
252qspinlock_assert_held_deep(struct qspinlock *lock, bool want_held,
253 const struct lock_chk_site *site) {
254 qspinlock_stamp(lock);
255 return qspinlock_deep_checked(lock) &&
256 lock_chk_assert_held_deep(&lock->chk, LOCK_CHK_MODE_IGNORED,
257 want_held, site);
258}
259
260#else /* !defined(DEBUG_LOCK_CHK) */
261
262#define __QSPINLOCK_SHALLOW_VALUE_INIT
263#define __QSPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_)
264#define QSPINLOCK_INIT_CHK(class_, flags_) \
265 ((struct qspinlock) {.val = ATOMIC_VAR_INIT(0)})
266
267#define qspinlock_init_chk(lock_, class_, flags_) \
268 qspinlock_init_chk_full((lock_), NULL, LOCK_UNCHKD)
269#define qspinlock_init_auto_internal(lock_, flags_) \
270 qspinlock_init_chk_full((lock_), NULL, LOCK_UNCHKD)
271
272static inline void qspinlock_policy_init_internal(struct qspinlock *lock,
273 enum lock_chk_flags flags) {
274 cc_ignore(lock, flags);
275}
276
277static inline void
278qspinlock_map_init_internal(struct qspinlock *lock,
279 const struct lock_chk_class *class,
280 enum lock_chk_flags flags) {
281 cc_ignore(lock, class, flags);
282}
283
284static inline void qspinlock_shallow_init_internal(struct qspinlock *lock) {
285 cc_ignore(lock);
286}
287
288static inline void qspinlock_set_chk_flags(struct qspinlock *lock,
289 enum lock_chk_flags flags) {
290 cc_ignore(lock, flags);
291}
292
293static inline void qspinlock_reinit_chk(struct qspinlock *lock,
294 const struct lock_chk_class *class,
295 enum lock_chk_flags flags) {
296 cc_ignore(lock, class, flags);
297}
298
299static inline void qspinlock_note_use(struct qspinlock *lock,
300 enum lock_op_flags flags) {
301 cc_ignore(lock, flags);
302}
303
304static inline bool qspinlock_order_checked(struct qspinlock *lock) {
305 cc_ignore(lock);
306 return false;
307}
308
309static inline void qspinlock_classify(struct qspinlock *lock,
310 enum lock_op_flags usage,
311 const struct lock_chk_site *site) {
312 cc_ignore(lock, usage, site);
313}
314
315static inline bool qspinlock_deep_checked(struct qspinlock *lock) {
316 cc_ignore(lock);
317 return false;
318}
319
320struct qspinlock_acq_scope {
321 char unused_;
322};
323
324struct qspinlock_rel_scope {
325 char unused_;
326};
327
328static inline void qspinlock_acq_begin(struct qspinlock_acq_scope *scope,
329 struct qspinlock *lock,
330 const struct lock_chk_site *site,
331 enum lock_subclass subclass,
332 enum lock_op_flags flags) {
333 cc_ignore(scope, lock, site, subclass, flags);
334}
335
336static inline void qspinlock_acq_commit(struct qspinlock_acq_scope *scope) {
337 cc_ignore(scope);
338}
339
340static inline void qspinlock_acq_abort(struct qspinlock_acq_scope *scope) {
341 cc_ignore(scope);
342}
343
344static inline void qspinlock_rel_begin(struct qspinlock_rel_scope *scope,
345 struct qspinlock *lock,
346 const struct lock_chk_site *site) {
347 cc_ignore(scope, lock, site);
348}
349
350static inline void qspinlock_rel_commit(struct qspinlock_rel_scope *scope) {
351 cc_ignore(scope);
352}
353
354static inline void qspinlock_shallow_push(struct qspinlock *lock,
355 enum irql irql,
356 const struct lock_chk_site *site) {
357 cc_ignore(lock, irql, site);
358}
359
360static inline void
361qspinlock_shallow_validate_top(struct qspinlock *lock, enum irql old,
362 const struct lock_chk_site *site) {
363 cc_ignore(lock, old, site);
364}
365
366static inline void qspinlock_shallow_pop(struct qspinlock *lock) {
367 cc_ignore(lock);
368}
369
370static inline bool
371qspinlock_assert_held_deep(struct qspinlock *lock, bool want_held,
372 const struct lock_chk_site *site) {
373 cc_ignore(lock, want_held, site);
374 return false;
375}
376
377#endif /* DEBUG_LOCK_CHK */
378
379static inline void qspinlock_init_chk_full(struct qspinlock *lock,
380 const struct lock_chk_class *class,
381 enum lock_chk_flags flags) {
382 atomic_store_rlx(&lock->val, 0);
383 qspinlock_policy_init_internal(lock, flags);
384 qspinlock_shallow_init_internal(lock);
385 qspinlock_map_init_internal(lock, class, flags);
386}
387
388#define qspinlock_init_1(lock_) qspinlock_init_2(lock_, LOCK_CHKD_FULL)
389#define qspinlock_init_2(lock_, flags_) \
390 qspinlock_init_auto_internal((lock_), (flags_))
391#define qspinlock_init(...) PP_CALL(qspinlock_init, __VA_ARGS__)
392#define qspin_init(...) qspinlock_init(__VA_ARGS__)
393#define qspin_init_chk(...) qspinlock_init_chk(__VA_ARGS__)
394
395static inline cc_warn_unused_result bool
396qspin_trylock_physical(struct qspinlock *lock) TSA_NO_ANALYSIS {
397 uint32_t expected = 0;
398 return atomic_cas_strong(&lock->val, &expected, Q_SPIN_LOCKED_VAL,
399 mo_acquire, mo_rlx);
400}
401
402void qspin_lock_slowpath(struct qspinlock *lock, uint32_t val);
403
404static inline void qspin_lock_physical(struct qspinlock *lock) TSA_NO_ANALYSIS {
405 uint32_t val = 0;
406 if (cc_likely(atomic_cas_strong(&lock->val, &val, Q_SPIN_LOCKED_VAL,
407 mo_acquire, mo_rlx)))
408 return;
409
410 qspin_lock_slowpath(lock, val);
411}
412
413static inline void
414qspin_unlock_physical(struct qspinlock *lock) TSA_NO_ANALYSIS {
415 /* Clear only the locked byte so pending and tail bits remain valid */
416 atomic_store_release((atomic_uint8_t *) &lock->val, 0);
417}
418
419static inline bool qspin_is_locked(const struct qspinlock *lock) {
420 return (atomic_load_rlx(&lock->val) & Q_SPIN_LOCKED_MASK) != 0;
421}
422
423/* Same deal as SPINLOCK_ASSERT_HELD_RAW, just checks the bit,
424 * but that doesn't say anything about thread ownership */
425#define QSPINLOCK_ASSERT_HELD_RAW(l) \
426 enforce(qspin_is_locked(l), "qspinlock not locked")
427
428static inline cc_warn_unused_result bool
429qspin_trylock_raw_full(struct qspinlock *lock, enum lock_subclass sc,
430 const struct lock_chk_site *site)
431 TSA_TRY_ACQUIRES(true, lock) TSA_NO_ANALYSIS {
432 enum lock_op_flags flags =
433 LOCK_OP_RAW | LOCK_OP_IRQ_NONE | LOCK_OP_KIND_TRY;
434 qspinlock_note_use(lock, flags);
435
436 struct qspinlock_acq_scope chk;
437 qspinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
438
439 if (qspin_trylock_physical(lock)) {
440
441 qspinlock_acq_commit(scope: &chk);
442
443 return true;
444 }
445
446 qspinlock_acq_abort(scope: &chk);
447
448 return false;
449}
450
451static inline void qspin_lock_raw_full(struct qspinlock *lock,
452 enum lock_subclass sc,
453 const struct lock_chk_site *site)
454 TSA_ACQUIRES(lock) TSA_NO_ANALYSIS {
455 enum lock_op_flags flags =
456 LOCK_OP_RAW | LOCK_OP_IRQ_NONE | LOCK_OP_KIND_BLOCKING;
457 qspinlock_note_use(lock, flags);
458
459 struct qspinlock_acq_scope chk;
460 qspinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
461
462 qspin_lock_physical(lock);
463
464 qspinlock_acq_commit(scope: &chk);
465}
466
467static inline void qspin_unlock_raw_full(struct qspinlock *lock,
468 const struct lock_chk_site *site)
469 TSA_RELEASES(lock) TSA_NO_ANALYSIS {
470
471 struct qspinlock_rel_scope chk;
472 qspinlock_rel_begin(scope: &chk, lock, site);
473
474 qspin_unlock_physical(lock);
475
476 qspinlock_rel_commit(scope: &chk);
477}
478
479static inline void qspin_unlock_full(struct qspinlock *lock, enum irql old_irql,
480 const struct lock_chk_site *site)
481 TSA_RELEASES_SPIN(lock) TSA_NO_ANALYSIS {
482 bool checked_shallow = qspinlock_order_checked(lock);
483 struct qspinlock_rel_scope chk;
484
485 bool irqs_enabled = irq_disable_save();
486
487 if (checked_shallow)
488 qspinlock_shallow_validate_top(lock, old: old_irql, site);
489 qspinlock_rel_begin(scope: &chk, lock, site);
490
491 qspin_unlock_physical(lock);
492
493 qspinlock_rel_commit(scope: &chk);
494 if (checked_shallow)
495 qspinlock_shallow_pop(lock);
496
497 crash_unwind_exit_qspinlock(q: lock);
498 if (irqs_enabled)
499 irq_enable();
500
501 irql_lower(old_level: old_irql);
502}
503
504static inline cc_warn_unused_result enum irql
505qspin_lock_full(struct qspinlock *lock, enum lock_subclass sc,
506 const struct lock_chk_site *site)
507 TSA_ACQUIRES_SPIN(lock) TSA_NO_ANALYSIS {
508 enforce(sc < LOCK_CHK_MAX_SUBCLASSES, CAPACITY_EXCEEDED);
509 if (bootstage_get() >= BOOTSTAGE_MID_MP &&
510 (irq_in_interrupt() || irq_in_nmi()))
511 panic(INVALID_CONTEXT,
512 "non-ISR-safe qspinlock taken outside thread context");
513
514 enum lock_op_flags flags = LOCK_OP_IRQ_DISPATCH | LOCK_OP_KIND_BLOCKING;
515 qspinlock_note_use(lock, flags);
516 bool checked_shallow = qspinlock_order_checked(lock);
517 if (checked_shallow)
518 qspinlock_classify(lock, usage: flags, site);
519
520 struct qspinlock_acq_scope chk;
521 qspinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
522
523 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
524 qspin_lock_physical(lock);
525
526 enforce(irqs_enabled(), INVALID_CONTEXT);
527 irq_disable();
528
529 if (checked_shallow)
530 qspinlock_shallow_push(lock, irql, site);
531 qspinlock_acq_commit(scope: &chk);
532
533 crash_unwind_enter_qspinlock(q: lock, old: irql);
534 irq_enable();
535
536 return irql;
537}
538
539static inline cc_warn_unused_result enum irql
540qspin_lock_high_full(struct qspinlock *lock, enum lock_subclass sc,
541 const struct lock_chk_site *site)
542 TSA_ACQUIRES_SPIN(lock) TSA_NO_ANALYSIS {
543 enforce(sc < LOCK_CHK_MAX_SUBCLASSES, CAPACITY_EXCEEDED);
544 if (bootstage_get() >= BOOTSTAGE_MID_MP && irq_in_nmi())
545 panic(INVALID_CONTEXT, "non-raw qspinlock taken in NMI");
546
547 enum lock_op_flags flags = LOCK_OP_KIND_BLOCKING | LOCK_OP_IRQ_HIGH;
548 qspinlock_note_use(lock, flags);
549 bool checked_shallow = qspinlock_order_checked(lock);
550 if (checked_shallow)
551 qspinlock_classify(lock, usage: flags, site);
552
553 struct qspinlock_acq_scope chk;
554 qspinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
555
556 enum irql irql = irql_raise(new_level: IRQL_HIGH_LEVEL);
557 qspin_lock_physical(lock);
558
559 if (checked_shallow)
560 qspinlock_shallow_push(lock, irql, site);
561 qspinlock_acq_commit(scope: &chk);
562
563 crash_unwind_enter_qspinlock(q: lock, old: irql);
564
565 return irql;
566}
567
568static inline cc_warn_unused_result bool
569qspin_trylock_full(struct qspinlock *lock, enum irql *out,
570 enum lock_subclass sc, const struct lock_chk_site *site)
571 TSA_TRY_ACQUIRES_SPIN(true, lock) TSA_NO_ANALYSIS {
572 if (bootstage_get() >= BOOTSTAGE_MID_MP &&
573 (irq_in_interrupt() || irq_in_nmi()))
574 panic(INVALID_CONTEXT,
575 "non-ISR-safe qspinlock taken outside thread context");
576
577 enum lock_op_flags flags = LOCK_OP_KIND_TRY | LOCK_OP_IRQ_DISPATCH;
578 qspinlock_note_use(lock, flags);
579 bool checked_shallow = qspinlock_order_checked(lock);
580 if (checked_shallow)
581 qspinlock_classify(lock, usage: flags, site);
582
583 struct qspinlock_acq_scope chk;
584 qspinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
585
586 *out = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
587 if (qspin_trylock_physical(lock)) {
588 enforce(irqs_enabled(), INVALID_CONTEXT); /* Should not go false */
589 irq_disable();
590 if (checked_shallow)
591 qspinlock_shallow_push(lock, irql: *out, site);
592 qspinlock_acq_commit(scope: &chk);
593 irq_enable();
594 crash_unwind_enter_qspinlock(q: lock, old: *out);
595 return true;
596 }
597
598 qspinlock_acq_abort(scope: &chk);
599
600 irql_lower(old_level: *out);
601 return false;
602}
603
604static inline cc_warn_unused_result bool
605qspin_trylock_high_full(struct qspinlock *lock, enum irql *out,
606 enum lock_subclass sc, const struct lock_chk_site *site)
607 TSA_TRY_ACQUIRES_SPIN(true, lock) TSA_NO_ANALYSIS {
608 if (bootstage_get() >= BOOTSTAGE_MID_MP && irq_in_nmi())
609 panic(INVALID_CONTEXT, "non-raw qspinlock taken in NMI");
610
611 enum lock_op_flags flags = LOCK_OP_KIND_BLOCKING | LOCK_OP_IRQ_HIGH;
612 qspinlock_note_use(lock, flags);
613 bool checked_shallow = qspinlock_order_checked(lock);
614 if (checked_shallow)
615 qspinlock_classify(lock, usage: flags, site);
616
617 struct qspinlock_acq_scope chk;
618 qspinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
619
620 *out = irql_raise(new_level: IRQL_HIGH_LEVEL);
621 if (qspin_trylock_physical(lock)) {
622
623 if (checked_shallow)
624 qspinlock_shallow_push(lock, irql: *out, site);
625 qspinlock_acq_commit(scope: &chk);
626
627 crash_unwind_enter_qspinlock(q: lock, old: *out);
628 return true;
629 }
630
631 qspinlock_acq_abort(scope: &chk);
632
633 irql_lower(old_level: *out);
634 return false;
635}
636
637#define qspin_lock(lock_) \
638 qspin_lock_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())
639#define qspin_lock_subclass(lock_, subclass_) \
640 qspin_lock_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())
641
642#define qspin_lock_high(lock_) \
643 qspin_lock_high_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())
644#define qspin_lock_high_subclass(lock_, subclass_) \
645 qspin_lock_high_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())
646
647#define qspin_trylock(lock_, out_) \
648 qspin_trylock_full((lock_), (out_), LOCK_SUBCLASS_PRIMARY, \
649 LOCK_CHK_SITE_HERE())
650#define qspin_trylock_subclass(lock_, out_, subclass_) \
651 qspin_trylock_full((lock_), (out_), (subclass_), LOCK_CHK_SITE_HERE())
652
653#define qspin_trylock_high(lock_, out_) \
654 qspin_trylock_high_full((lock_), (out_), LOCK_SUBCLASS_PRIMARY, \
655 LOCK_CHK_SITE_HERE())
656#define qspin_trylock_high_subclass(lock_, out_, subclass_) \
657 qspin_trylock_high_full((lock_), (out_), (subclass_), LOCK_CHK_SITE_HERE())
658
659#define qspin_unlock(lock_, old_) \
660 qspin_unlock_full((lock_), (old_), LOCK_CHK_SITE_HERE())
661#define qspin_unlock_irq_restore(lock_, old_) \
662 qspin_unlock_full((lock_), (old_), LOCK_CHK_SITE_HERE())
663
664#define qspin_lock_raw(lock_) \
665 qspin_lock_raw_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())
666#define qspin_lock_raw_subclass(lock_, subclass_) \
667 qspin_lock_raw_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())
668
669#define qspin_trylock_raw(lock_) \
670 qspin_trylock_raw_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())
671#define qspin_trylock_raw_subclass(lock_, subclass_) \
672 qspin_trylock_raw_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())
673
674#define qspin_unlock_raw(lock_) \
675 qspin_unlock_raw_full((lock_), LOCK_CHK_SITE_HERE())
676
677static inline void qspin_assert_held_full(struct qspinlock *lock,
678 const struct lock_chk_site *site)
679 TSA_ASSERT_CAPABILITY(lock) {
680 if (qspinlock_assert_held_deep(lock, /*want_held=*/true, site))
681 return;
682
683 QSPINLOCK_ASSERT_HELD_RAW(lock);
684}
685
686static inline void
687qspin_assert_not_held_full(struct qspinlock *lock,
688 const struct lock_chk_site *site) {
689 if (qspinlock_assert_held_deep(lock, /*want_held=*/false, site))
690 return;
691
692 if (bootstage_get() < BOOTSTAGE_EARLY_DEVICES)
693 enforce(!qspin_is_locked(lock), "qspinlock unexpectedly locked");
694}
695
696#define QSPINLOCK_ASSERT_HELD(l) \
697 qspin_assert_held_full((l), LOCK_CHK_SITE_HERE())
698#define QSPINLOCK_ASSERT_NOT_HELD(l) \
699 qspin_assert_not_held_full((l), LOCK_CHK_SITE_HERE())
700
701struct qspinlock_guard {
702 struct qspinlock *lock;
703 enum irql irql;
704};
705
706static inline cc_always_inline cc_unused void
707qspinlock_guard_assume(struct qspinlock *lock) TSA_ASSERT_CAPABILITY(lock) {
708 cc_ignore(lock);
709}
710
711static inline cc_always_inline cc_unused void qspinlock_guard_exit(
712 struct qspinlock_guard *g) TSA_RELEASES_SPIN_IRQL TSA_NO_ANALYSIS {
713 if (g->lock)
714 qspin_unlock(g->lock, g->irql);
715}
716
717static inline cc_always_inline cc_unused struct qspinlock_guard
718qspin_guard_enter(struct qspinlock *lock)
719 TSA_ACQUIRES_SPIN_IRQL TSA_NO_ANALYSIS {
720 return (struct qspinlock_guard) {
721 .lock = lock,
722 .irql = qspin_lock(lock),
723 };
724}
725
726static inline cc_always_inline cc_unused struct qspinlock_guard
727qspin_guard_high_enter(struct qspinlock *lock)
728 TSA_ACQUIRES_SPIN_IRQL TSA_NO_ANALYSIS {
729 return (struct qspinlock_guard) {
730 .lock = lock,
731 .irql = qspin_lock_high(lock),
732 };
733}
734
735static inline cc_always_inline cc_unused struct qspinlock_guard
736qspin_guard_high_subclass_enter(struct qspinlock *lock, enum lock_subclass sc)
737 TSA_ACQUIRES_SPIN_IRQL TSA_NO_ANALYSIS {
738 return (struct qspinlock_guard) {
739 .lock = lock,
740 .irql = qspin_lock_high_subclass(lock, sc),
741 };
742}
743
744static inline cc_always_inline cc_unused struct qspinlock_guard
745qspin_guard_subclass_enter(struct qspinlock *lock, enum lock_subclass sc)
746 TSA_ACQUIRES_SPIN_IRQL TSA_NO_ANALYSIS {
747 return (struct qspinlock_guard) {
748 .lock = lock,
749 .irql = qspin_lock_subclass(lock, sc),
750 };
751}
752
753#define qspin_guard(lock_) \
754 cc_cleanup(qspinlock_guard_exit) struct qspinlock_guard PP_CONCAT( \
755 qspin_guard_, __COUNTER__) = \
756 (qspinlock_guard_assume(lock_), qspin_guard_enter(lock_))
757
758#define qspin_guard_high(lock_) \
759 cc_cleanup(qspinlock_guard_exit) struct qspinlock_guard PP_CONCAT( \
760 qspin_guard_high_, __COUNTER__) = \
761 (qspinlock_guard_assume(lock_), qspin_guard_high_enter(lock_))
762
763#define qspin_guard_subclass(lock_, subclass_) \
764 cc_cleanup(qspinlock_guard_exit) struct qspinlock_guard PP_CONCAT( \
765 qspin_guard_, __COUNTER__) = \
766 (qspinlock_guard_assume(lock_), \
767 qspin_guard_subclass_enter((lock_), (subclass_)))
768
769#define qspin_guard_high_subclass(lock_, subclass_) \
770 cc_cleanup(qspinlock_guard_exit) struct qspinlock_guard PP_CONCAT( \
771 qspin_guard_high_, __COUNTER__) = \
772 (qspinlock_guard_assume(lock_), \
773 qspin_guard_high_subclass_enter((lock_), (subclass_)))
774