1#pragma once
2#include <asm.h>
3#include <atomic.h>
4#include <bootstage.h>
5#include <compiler/core.h>
6#include <console/panic.h>
7#include <enforce.h>
8#include <irq/irq.h>
9#include <sch/irql.h>
10#include <smp/core.h>
11#include <stdbool.h>
12#include <sync/lock_chk_types.h>
13#include <sync/lock_general.h>
14#include <sync/raw_spinlock.h>
15#include <tsa.h>
16
17struct TSA_CAPABILITY("spinlock") spinlock {
18 struct raw_spinlock raw;
19
20#ifdef DEBUG_LOCK_CHK
21 struct lock_chk_lock chk;
22 atomic(enum lock_op_flags) irq_usage;
23#endif /* DEBUG_LOCK_CHK */
24};
25
26TSA_DEFINE_ASSUME(struct spinlock)
27
28#define SPINLOCK_INIT SPINLOCK_INIT_CHK(NULL, LOCK_CHKD_FULL)
29#define SPINLOCK_DEFINE(id) struct spinlock id = SPINLOCK_INIT
30#define SPINLOCK_DEFINE_CHK(id, class_, flags_) \
31 struct spinlock id = SPINLOCK_INIT_CHK((class_), (flags_))
32
33#define spinlock_init_1(lock_) spinlock_init_2(lock_, LOCK_CHKD_FULL)
34#define spinlock_init_2(lock_, flags_) \
35 spinlock_init_auto_internal((lock_), (flags_))
36#define spinlock_init(...) PP_CALL(spinlock_init, __VA_ARGS__)
37
38static inline void spinlock_init_chk_full(struct spinlock *lock,
39 const struct lock_chk_class *class,
40 enum lock_chk_flags flags);
41
42#ifdef DEBUG_LOCK_CHK
43
44#define __SPINLOCK_SHALLOW_VALUE_INIT \
45 , .irq_usage = ATOMIC_VAR_INIT(LOCK_OP_IRQ_NONE)
46#define __SPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_) \
47 , .chk = LOCK_CHK_LOCK_VALUE_INIT((class_), (flags_))
48#define SPINLOCK_INIT_CHK(class_, flags_) \
49 ((struct spinlock) { \
50 .raw = RAW_SPINLOCK_INIT __SPINLOCK_LOCK_CHK_VALUE_INIT( \
51 (class_), (flags_)) __SPINLOCK_SHALLOW_VALUE_INIT})
52
53#define spinlock_init_chk(lock_, class_, flags_) \
54 spinlock_init_chk_full((lock_), (class_), (flags_))
55#define spinlock_init_auto_internal(lock_, flags_) \
56 do { \
57 static const struct lock_chk_class __auto_class = { \
58 .name = #lock_, \
59 .file = __RELFILE__, \
60 .line = __LINE__, \
61 }; \
62 spinlock_init_chk_full((lock_), &__auto_class, (flags_)); \
63 } while (0)
64
65static inline void spinlock_policy_init_internal(struct spinlock *lock,
66 enum lock_chk_flags flags) {
67 enforce((flags & ~LOCK_CHKD_FULL) == 0);
68 lock->chk.flags = flags;
69 lock->chk.initialized = true;
70 atomic_store_rlx(&lock->chk.used, false);
71}
72static inline void spinlock_shallow_init_internal(struct spinlock *lock) {
73 atomic_store_rlx(&lock->irq_usage, LOCK_OP_IRQ_NONE);
74}
75static inline void
76spinlock_map_init_internal(struct spinlock *lock,
77 const struct lock_chk_class *class,
78 enum lock_chk_flags flags) {
79 enforce(flags == LOCK_UNCHKD || class != NULL);
80 lock_chk_map_runtime_init(&lock->chk.map, class);
81}
82
83static inline void spinlock_note_use(struct spinlock *lock,
84 enum lock_op_flags flags) {
85 lock_chk_note_use(&lock->chk, flags);
86}
87
88static inline bool spinlock_order_checked(struct spinlock *lock) {
89 return lock_chk_tracking_active() &&
90 (lock->chk.flags & LOCK_CHKD_ORDER) != 0;
91}
92
93/* TODO: unify stamping at initialization too */
94static inline void spinlock_stamp(struct spinlock *lock) {
95 lock->chk.instance = lock;
96 lock->chk.type = LOCK_CHK_TYPE_SPIN;
97}
98
99static inline void spinlock_classify(struct spinlock *lock,
100 enum lock_op_flags usage,
101 const struct lock_chk_site *site) {
102 spinlock_stamp(lock);
103 lock_debug_spin_classify(&lock->irq_usage, usage, &lock->chk, site);
104}
105
106static inline bool spinlock_deep_checked(struct spinlock *lock) {
107 return lock_chk_tracking_active() && lock->chk.flags != LOCK_UNCHKD;
108}
109
110static inline struct lock_chk_acq_req
111spinlock_chk_acq_req(struct spinlock *lock, const struct lock_chk_site *site,
112 enum lock_subclass subclass, enum lock_op_flags flags) {
113 spinlock_stamp(lock);
114 return lock_chk_acq_req_make(&lock->chk, site, LOCK_CHK_MODE_EXCLUSIVE,
115 subclass, flags);
116}
117
118static inline struct lock_chk_rel_req
119spinlock_chk_rel_req(struct spinlock *lock, const struct lock_chk_site *site) {
120 spinlock_stamp(lock);
121 return lock_chk_rel_req_make(&lock->chk, site, LOCK_CHK_MODE_EXCLUSIVE);
122}
123
124struct spinlock_acq_scope {
125 struct lock_chk_acq_req req;
126 struct lock_chk_acq_token token;
127 bool checked;
128};
129
130struct spinlock_rel_scope {
131 struct lock_chk_rel_req req;
132 struct lock_chk_rel_token token;
133 bool checked;
134};
135
136static inline void spinlock_acq_begin(struct spinlock_acq_scope *scope,
137 struct spinlock *lock,
138 const struct lock_chk_site *site,
139 enum lock_subclass subclass,
140 enum lock_op_flags flags) {
141 scope->checked = spinlock_deep_checked(lock);
142 if (!scope->checked)
143 return;
144
145 scope->req = spinlock_chk_acq_req(lock, site, subclass, flags);
146 lock_chk_before_acq(&scope->token, &scope->req);
147}
148
149static inline void spinlock_acq_commit(struct spinlock_acq_scope *scope) {
150 if (scope->checked)
151 lock_chk_acqd(&scope->token);
152}
153
154static inline void spinlock_acq_abort(struct spinlock_acq_scope *scope) {
155 if (scope->checked)
156 lock_chk_cancel(&scope->token);
157}
158
159static inline void spinlock_rel_begin(struct spinlock_rel_scope *scope,
160 struct spinlock *lock,
161 const struct lock_chk_site *site) {
162 scope->checked = spinlock_deep_checked(lock);
163 if (!scope->checked)
164 return;
165
166 scope->req = spinlock_chk_rel_req(lock, site);
167 lock_chk_before_rel(&scope->token, &scope->req);
168}
169
170static inline void spinlock_rel_commit(struct spinlock_rel_scope *scope) {
171 if (scope->checked)
172 lock_chk_reld(&scope->token);
173}
174
175static inline void spinlock_shallow_push(struct spinlock *lock, enum irql irql,
176 const struct lock_chk_site *site) {
177 spinlock_stamp(lock);
178 lock_debug_spin_push(&lock->chk, irql, site);
179}
180
181static inline void
182spinlock_shallow_validate_top(struct spinlock *lock, enum irql old,
183 const struct lock_chk_site *site) {
184 spinlock_stamp(lock);
185 lock_debug_spin_validate_top(&lock->chk, old, site);
186}
187
188static inline void spinlock_shallow_pop(struct spinlock *lock) {
189 lock_debug_spin_pop(&lock->chk);
190}
191
192static inline bool spinlock_assert_held_deep(struct spinlock *lock,
193 bool want_held,
194 const struct lock_chk_site *site) {
195 spinlock_stamp(lock);
196 return spinlock_deep_checked(lock) &&
197 lock_chk_assert_held_deep(&lock->chk, LOCK_CHK_MODE_IGNORED,
198 want_held, site);
199}
200
201static inline void spinlock_assert_initialized(struct spinlock *lock) {
202 enforce(lock->chk.initialized, UNINITIALIZED);
203}
204
205static inline void spinlock_policy_set_internal(struct spinlock *lock,
206 enum lock_chk_flags flags) {
207 enforce(lock->chk.initialized, UNINITIALIZED);
208 enforce(!atomic_load_rlx(&lock->chk.used));
209 enforce((flags & ~LOCK_CHKD_FULL) == 0);
210 lock->chk.flags = flags;
211}
212
213#else /* !defined(DEBUG_LOCK_CHK) */
214
215#define __SPINLOCK_SHALLOW_VALUE_INIT
216#define __SPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_)
217#define SPINLOCK_INIT_CHK(class_, flags_) \
218 ((struct spinlock) {.raw = RAW_SPINLOCK_INIT})
219
220#define spinlock_init_chk(lock_, class_, flags_) \
221 spinlock_init_chk_full((lock_), NULL, LOCK_UNCHKD)
222#define spinlock_init_auto_internal(lock_, flags_) \
223 spinlock_init_chk_full((lock_), NULL, LOCK_UNCHKD)
224
225static inline void spinlock_policy_init_internal(struct spinlock *lock,
226 enum lock_chk_flags flags) {
227 cc_ignore(lock, flags);
228}
229
230static inline void spinlock_shallow_init_internal(struct spinlock *lock) {
231 cc_ignore(lock);
232}
233
234static inline void
235spinlock_map_init_internal(struct spinlock *lock,
236 const struct lock_chk_class *class,
237 enum lock_chk_flags flags) {
238 cc_ignore(lock, class, flags);
239}
240
241static inline void spinlock_note_use(struct spinlock *lock,
242 enum lock_op_flags flags) {
243 cc_ignore(lock, flags);
244}
245
246static inline bool spinlock_order_checked(struct spinlock *lock) {
247 cc_ignore(lock);
248 return false;
249}
250
251static inline void spinlock_classify(struct spinlock *lock,
252 enum lock_op_flags usage,
253 const struct lock_chk_site *site) {
254 cc_ignore(lock, usage, site);
255}
256
257static inline bool spinlock_deep_checked(struct spinlock *lock) {
258 cc_ignore(lock);
259 return false;
260}
261
262struct spinlock_acq_scope {
263 char unused_;
264};
265
266struct spinlock_rel_scope {
267 char unused_;
268};
269
270static inline void spinlock_acq_begin(struct spinlock_acq_scope *scope,
271 struct spinlock *lock,
272 const struct lock_chk_site *site,
273 enum lock_subclass subclass,
274 enum lock_op_flags flags) {
275 cc_ignore(scope, lock, site, subclass, flags);
276}
277
278static inline void spinlock_acq_commit(struct spinlock_acq_scope *scope) {
279 cc_ignore(scope);
280}
281
282static inline void spinlock_acq_abort(struct spinlock_acq_scope *scope) {
283 cc_ignore(scope);
284}
285
286static inline void spinlock_rel_begin(struct spinlock_rel_scope *scope,
287 struct spinlock *lock,
288 const struct lock_chk_site *site) {
289 cc_ignore(scope, lock, site);
290}
291
292static inline void spinlock_rel_commit(struct spinlock_rel_scope *scope) {
293 cc_ignore(scope);
294}
295
296static inline void spinlock_shallow_push(struct spinlock *lock, enum irql irql,
297 const struct lock_chk_site *site) {
298 cc_ignore(lock, irql, site);
299}
300
301static inline void
302spinlock_shallow_validate_top(struct spinlock *lock, enum irql old,
303 const struct lock_chk_site *site) {
304 cc_ignore(lock, old, site);
305}
306
307static inline void spinlock_shallow_pop(struct spinlock *lock) {
308 cc_ignore(lock);
309}
310
311static inline bool spinlock_assert_held_deep(struct spinlock *lock,
312 bool want_held,
313 const struct lock_chk_site *site) {
314 cc_ignore(lock, want_held, site);
315 return false;
316}
317
318static inline void spinlock_assert_initialized(struct spinlock *lock) {
319 cc_ignore(lock);
320}
321
322static inline void spinlock_policy_set_internal(struct spinlock *lock,
323 enum lock_chk_flags flags) {
324 cc_ignore(lock, flags);
325}
326
327#endif /* DEBUG_LOCK_CHK */
328
329static inline cc_warn_unused_result bool
330spin_trylock_physical(struct spinlock *lock) TSA_NO_ANALYSIS {
331 return raw_spin_trylock(lock: &lock->raw);
332}
333
334static inline void spin_lock_physical(struct spinlock *lock) TSA_NO_ANALYSIS {
335 raw_spin_lock(lock: &lock->raw);
336}
337
338static inline void spin_unlock_physical(struct spinlock *lock) TSA_NO_ANALYSIS {
339 raw_spin_unlock(lock: &lock->raw);
340}
341
342static inline void spinlock_init_chk_full(struct spinlock *lock,
343 const struct lock_chk_class *class,
344 enum lock_chk_flags flags) {
345 raw_spinlock_init(lock: &lock->raw);
346 spinlock_policy_init_internal(lock, flags);
347 spinlock_shallow_init_internal(lock);
348 spinlock_map_init_internal(lock, class, flags);
349}
350
351static inline void spinlock_restore_interrupts(bool enabled) TSA_RELEASES_IRQS {
352 irq_restore(was_enabled: enabled);
353}
354
355static inline cc_warn_unused_result bool
356spin_trylock_raw_full(struct spinlock *lock, enum lock_subclass sc,
357 const struct lock_chk_site *site)
358 TSA_TRY_ACQUIRES(true, lock) TSA_NO_ANALYSIS {
359 enum lock_op_flags flags =
360 LOCK_OP_RAW | LOCK_OP_KIND_TRY | LOCK_OP_IRQ_NONE;
361 spinlock_note_use(lock, flags);
362
363 struct spinlock_acq_scope chk;
364 spinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
365
366 if (spin_trylock_physical(lock)) {
367 spinlock_acq_commit(scope: &chk);
368 return true;
369 }
370
371 spinlock_acq_abort(scope: &chk);
372 return false;
373}
374
375static inline void spin_lock_raw_full(struct spinlock *lock,
376 enum lock_subclass sc,
377 const struct lock_chk_site *site)
378 TSA_ACQUIRES(lock) TSA_NO_ANALYSIS {
379 enum lock_op_flags flags =
380 LOCK_OP_RAW | LOCK_OP_KIND_BLOCKING | LOCK_OP_IRQ_NONE;
381 spinlock_note_use(lock, flags);
382
383 struct spinlock_acq_scope chk;
384 spinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
385
386 spin_lock_physical(lock);
387 spinlock_acq_commit(scope: &chk);
388}
389
390static inline void spin_unlock_raw_full(struct spinlock *lock,
391 const struct lock_chk_site *site)
392 TSA_RELEASES(lock) TSA_NO_ANALYSIS {
393
394 struct spinlock_rel_scope chk;
395 spinlock_rel_begin(scope: &chk, lock, site);
396
397 spin_unlock_physical(lock);
398 spinlock_rel_commit(scope: &chk);
399}
400
401static inline void spin_unlock_full(struct spinlock *lock, enum irql old,
402 const struct lock_chk_site *site)
403 TSA_RELEASES_SPIN(lock) TSA_NO_ANALYSIS {
404
405 bool checked_shallow = spinlock_order_checked(lock);
406 struct spinlock_rel_scope chk;
407
408 bool irqs_enabled = irq_disable_save();
409
410 if (checked_shallow)
411 spinlock_shallow_validate_top(lock, old, site);
412 spinlock_rel_begin(scope: &chk, lock, site);
413
414 spin_unlock_physical(lock);
415
416 spinlock_rel_commit(scope: &chk);
417 if (checked_shallow)
418 spinlock_shallow_pop(lock);
419
420 spinlock_restore_interrupts(enabled: irqs_enabled);
421
422 irql_lower(old_level: old);
423}
424
425static inline cc_warn_unused_result enum irql
426spin_lock_full(struct spinlock *lock, enum lock_subclass sc,
427 const struct lock_chk_site *site)
428 TSA_ACQUIRES_SPIN(lock) TSA_NO_ANALYSIS {
429 enforce(sc < LOCK_CHK_MAX_SUBCLASSES, CAPACITY_EXCEEDED);
430 if (bootstage_get() >= BOOTSTAGE_MID_MP &&
431 (irq_in_interrupt() || irq_in_nmi()))
432 panic(INVALID_CONTEXT,
433 "non-ISR-safe spinlock taken outside thread context");
434
435 enum lock_op_flags flags = LOCK_OP_KIND_BLOCKING | LOCK_OP_IRQ_DISPATCH;
436
437 spinlock_note_use(lock, flags);
438 bool checked_shallow = spinlock_order_checked(lock);
439 if (checked_shallow)
440 spinlock_classify(lock, usage: flags, site);
441
442 struct spinlock_acq_scope chk;
443 spinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
444
445 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
446 spin_lock_physical(lock);
447
448 bool irqs_enabled = irq_disable_save();
449
450 if (checked_shallow)
451 spinlock_shallow_push(lock, irql, site);
452 spinlock_acq_commit(scope: &chk);
453
454 spinlock_restore_interrupts(enabled: irqs_enabled);
455
456 return irql;
457}
458
459static inline cc_warn_unused_result enum irql
460spin_lock_high_full(struct spinlock *lock, enum lock_subclass sc,
461 const struct lock_chk_site *site)
462 TSA_ACQUIRES_SPIN(lock) TSA_NO_ANALYSIS {
463 enforce(sc < LOCK_CHK_MAX_SUBCLASSES, CAPACITY_EXCEEDED);
464 if (bootstage_get() >= BOOTSTAGE_MID_MP && irq_in_nmi())
465 panic(INVALID_CONTEXT, "non-raw spinlock taken in NMI");
466
467 enum lock_op_flags flags = LOCK_OP_KIND_BLOCKING | LOCK_OP_IRQ_HIGH;
468
469 spinlock_note_use(lock, flags);
470 bool checked_shallow = spinlock_order_checked(lock);
471 if (checked_shallow)
472 spinlock_classify(lock, usage: flags, site);
473
474 struct spinlock_acq_scope chk;
475 spinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
476
477 enum irql irql = irql_raise(new_level: IRQL_HIGH_LEVEL);
478 spin_lock_physical(lock);
479
480 if (checked_shallow)
481 spinlock_shallow_push(lock, irql, site);
482 spinlock_acq_commit(scope: &chk);
483
484 return irql;
485}
486
487static inline cc_warn_unused_result bool
488spin_trylock_full(struct spinlock *lock, enum irql *out, enum lock_subclass sc,
489 const struct lock_chk_site *site)
490 TSA_TRY_ACQUIRES_SPIN(true, lock) TSA_NO_ANALYSIS {
491 if (bootstage_get() >= BOOTSTAGE_MID_MP &&
492 (irq_in_interrupt() || irq_in_nmi()))
493 panic(INVALID_CONTEXT,
494 "non-ISR-safe spinlock taken outside thread context");
495
496 enum lock_op_flags flags = LOCK_OP_KIND_TRY | LOCK_OP_IRQ_DISPATCH;
497
498 spinlock_note_use(lock, flags);
499
500 bool checked_shallow = spinlock_order_checked(lock);
501 if (checked_shallow)
502 spinlock_classify(lock, usage: flags, site);
503
504 struct spinlock_acq_scope chk;
505 spinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
506
507 *out = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
508 if (spin_trylock_physical(lock)) {
509 bool irqs_enabled = irq_disable_save();
510
511 if (checked_shallow)
512 spinlock_shallow_push(lock, irql: *out, site);
513 spinlock_acq_commit(scope: &chk);
514
515 spinlock_restore_interrupts(enabled: irqs_enabled);
516 return true;
517 }
518
519 spinlock_acq_abort(scope: &chk);
520 irql_lower(old_level: *out);
521 return false;
522}
523
524static inline cc_warn_unused_result bool
525spin_trylock_high_full(struct spinlock *lock, enum irql *out,
526 enum lock_subclass sc, const struct lock_chk_site *site)
527 TSA_TRY_ACQUIRES_SPIN(true, lock) TSA_NO_ANALYSIS {
528 if (bootstage_get() >= BOOTSTAGE_MID_MP && irq_in_nmi())
529 panic(INVALID_CONTEXT, "non-raw spinlock taken in NMI");
530
531 enum lock_op_flags flags = LOCK_OP_KIND_TRY | LOCK_OP_IRQ_HIGH;
532
533 spinlock_note_use(lock, flags);
534 bool checked_shallow = spinlock_order_checked(lock);
535 if (checked_shallow)
536 spinlock_classify(lock, usage: flags, site);
537
538 struct spinlock_acq_scope chk;
539 spinlock_acq_begin(scope: &chk, lock, site, subclass: sc, flags);
540
541 *out = irql_raise(new_level: IRQL_HIGH_LEVEL);
542 if (spin_trylock_physical(lock)) {
543 if (checked_shallow)
544 spinlock_shallow_push(lock, irql: *out, site);
545 spinlock_acq_commit(scope: &chk);
546
547 return true;
548 }
549
550 spinlock_acq_abort(scope: &chk);
551 irql_lower(old_level: *out);
552 return false;
553}
554
555#define spin_lock(lock_) \
556 spin_lock_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())
557#define spin_lock_subclass(lock_, subclass_) \
558 spin_lock_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())
559
560#define spin_lock_high(lock_) \
561 spin_lock_high_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())
562#define spin_lock_high_subclass(lock_, subclass_) \
563 spin_lock_high_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())
564
565#define spin_trylock(lock_, out_) \
566 spin_trylock_full((lock_), (out_), LOCK_SUBCLASS_PRIMARY, \
567 LOCK_CHK_SITE_HERE())
568#define spin_trylock_subclass(lock_, out_, subclass_) \
569 spin_trylock_full((lock_), (out_), (subclass_), LOCK_CHK_SITE_HERE())
570
571#define spin_trylock_high(lock_, out_) \
572 spin_trylock_high_full((lock_), (out_), LOCK_SUBCLASS_PRIMARY, \
573 LOCK_CHK_SITE_HERE())
574#define spin_trylock_high_subclass(lock_, out_, subclass_) \
575 spin_trylock_high_full((lock_), (out_) (subclass_), LOCK_CHK_SITE_HERE())
576
577#define spin_unlock(lock_, old_) \
578 spin_unlock_full((lock_), (old_), LOCK_CHK_SITE_HERE())
579
580#define spin_lock_raw(lock_) \
581 spin_lock_raw_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())
582#define spin_lock_raw_subclass(lock_) \
583 spin_lock_raw_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())
584
585#define spin_trylock_raw(lock_) \
586 spin_trylock_raw_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())
587#define spin_trylock_raw_subclass(lock_) \
588 spin_trylock_raw_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())
589
590#define spin_unlock_raw(lock_) \
591 spin_unlock_raw_full((lock_), LOCK_CHK_SITE_HERE())
592
593static inline bool spinlock_locked(struct spinlock *lock) {
594 return atomic_load_rlx(&lock->raw.state);
595}
596
597/* A raw check to make sure it is locked, but could be by anyone */
598#define SPINLOCK_ASSERT_HELD_RAW(l) \
599 enforce(spinlock_locked(l), "spinlock not locked")
600
601static inline void spinlock_set_chk_flags(struct spinlock *lock,
602 enum lock_chk_flags flags) {
603 enforce(!spinlock_locked(lock));
604 spinlock_policy_set_internal(lock, flags);
605}
606
607static inline void spinlock_reinit_chk(struct spinlock *lock,
608 const struct lock_chk_class *class,
609 enum lock_chk_flags flags) {
610 enforce(!spinlock_locked(lock));
611 spinlock_assert_initialized(lock);
612 spinlock_init_chk_full(lock, class, flags);
613}
614
615static inline void spinlock_assert_held_full(struct spinlock *lock,
616 const struct lock_chk_site *site)
617 TSA_ASSERT_CAPABILITY(lock) {
618 if (spinlock_assert_held_deep(lock, /*want_held=*/true, site))
619 return;
620
621 SPINLOCK_ASSERT_HELD_RAW(lock);
622}
623
624static inline void
625spinlock_assert_not_held_full(struct spinlock *lock,
626 const struct lock_chk_site *site) {
627 if (spinlock_assert_held_deep(lock, /*want_held=*/false, site))
628 return;
629
630 /* Cannot check here */
631 if (bootstage_get() < BOOTSTAGE_EARLY_DEVICES)
632 enforce(!spinlock_locked(lock), "spinlock unexpectedly locked");
633}
634
635#define SPINLOCK_ASSERT_HELD(l) \
636 spinlock_assert_held_full((l), LOCK_CHK_SITE_HERE())
637#define SPINLOCK_ASSERT_NOT_HELD(l) \
638 spinlock_assert_not_held_full((l), LOCK_CHK_SITE_HERE())
639
640/* Scoped guards. TSA can't connect guard variables to lock expressions.
641 * IRQL_RAISED is acquired on entry and released on cleanup, and
642 * this prevents sleeping inside the guard. Lock expression is evaluated
643 * twice, remember to pass a lock address */
644struct spinlock_guard {
645 struct spinlock *lock;
646 enum irql irql;
647};
648
649static inline cc_always_inline cc_unused void
650spinlock_guard_assume(struct spinlock *lock) TSA_ASSERT_CAPABILITY(lock) {
651 cc_ignore(lock);
652}
653
654static inline cc_always_inline cc_unused void spinlock_guard_exit(
655 struct spinlock_guard *g) TSA_RELEASES_SPIN_IRQL TSA_NO_ANALYSIS {
656 if (g->lock)
657 spin_unlock(g->lock, g->irql);
658}
659
660static inline cc_always_inline cc_unused struct spinlock_guard
661spin_guard_enter(struct spinlock *lock) TSA_ACQUIRES_SPIN_IRQL TSA_NO_ANALYSIS {
662 return (struct spinlock_guard) {
663 .lock = lock,
664 .irql = spin_lock(lock),
665 };
666}
667
668static inline cc_always_inline cc_unused struct spinlock_guard
669spin_guard_high_enter(struct spinlock *lock)
670 TSA_ACQUIRES_SPIN_IRQL TSA_NO_ANALYSIS {
671 return (struct spinlock_guard) {
672 .lock = lock,
673 .irql = spin_lock_high(lock),
674 };
675}
676
677static inline cc_always_inline cc_unused struct spinlock_guard
678spin_guard_high_subclass_enter(struct spinlock *lock, enum lock_subclass sc)
679 TSA_ACQUIRES_SPIN_IRQL TSA_NO_ANALYSIS {
680 return (struct spinlock_guard) {
681 .lock = lock,
682 .irql = spin_lock_high_subclass(lock, sc),
683 };
684}
685
686static inline cc_always_inline cc_unused struct spinlock_guard
687spin_guard_subclass_enter(struct spinlock *lock, enum lock_subclass sc)
688 TSA_ACQUIRES_SPIN_IRQL TSA_NO_ANALYSIS {
689 return (struct spinlock_guard) {
690 .lock = lock,
691 .irql = spin_lock_subclass(lock, sc),
692 };
693}
694
695#define spin_guard(lock_) \
696 cc_cleanup(spinlock_guard_exit) struct spinlock_guard PP_CONCAT( \
697 spin_guard_, __COUNTER__) = \
698 (spinlock_guard_assume(lock_), spin_guard_enter(lock_))
699
700#define spin_guard_subclass(lock_, subclass_) \
701 cc_cleanup(spinlock_guard_exit) struct spinlock_guard PP_CONCAT( \
702 spin_guard_, __COUNTER__) = \
703 (spinlock_guard_assume(lock_), \
704 spin_guard_subclass_enter((lock_), (subclass_)))
705
706#define spin_guard_high(lock_) \
707 cc_cleanup(spinlock_guard_exit) struct spinlock_guard PP_CONCAT( \
708 spin_guard_high_, __COUNTER__) = \
709 (spinlock_guard_assume(lock_), spin_guard_high_enter(lock_))
710
711#define spin_guard_high_subclass(lock_, subclass_) \
712 cc_cleanup(spinlock_guard_exit) struct spinlock_guard PP_CONCAT( \
713 spin_guard_high_, __COUNTER__) = \
714 (spinlock_guard_assume(lock_), \
715 spin_guard_high_enter((lock_), (subclass_)))
716