1/* @title: Raw Spinlock */
2#pragma once
3#include <asm.h>
4#include <atomic.h>
5#include <compiler/core.h>
6#include <stdbool.h>
7#include <stdint.h>
8#include <tsa.h>
9
10struct TSA_CAPABILITY("spinlock") raw_spinlock {
11 atomic_uint8_t state;
12};
13
14#define RAW_SPINLOCK_INIT ((struct raw_spinlock) {.state = ATOMIC_VAR_INIT(0)})
15#define RAW_SPINLOCK_DEFINE(id) struct raw_spinlock id = RAW_SPINLOCK_INIT
16
17static inline void raw_spinlock_init(struct raw_spinlock *lock) {
18 atomic_store_rlx(&lock->state, 0);
19}
20
21static inline cc_warn_unused_result bool
22raw_spin_trylock(struct raw_spinlock *lock)
23 TSA_TRY_ACQUIRES(true, lock) TSA_NO_ANALYSIS {
24 uint8_t expected = 0;
25 return atomic_cas_strong(&lock->state, &expected, 1, mo_acquire, mo_rlx);
26}
27
28static inline void raw_spin_lock(struct raw_spinlock *lock)
29 TSA_ACQUIRES(lock) TSA_NO_ANALYSIS {
30 while (true) {
31 if (raw_spin_trylock(lock))
32 return;
33
34 while (atomic_load_rlx(&lock->state) != 0)
35 cpu_pause();
36 }
37}
38
39static inline void raw_spin_unlock(struct raw_spinlock *lock)
40 TSA_RELEASES(lock) TSA_NO_ANALYSIS {
41 atomic_store_release(&lock->state, 0);
42}
43
44/* whether interrupts were enabled on entry */
45static inline cc_warn_unused_result bool
46raw_spin_lock_high(struct raw_spinlock *lock)
47 TSA_ACQUIRES_IRQS_LOCK(lock) TSA_NO_ANALYSIS {
48 bool irqs_were_enabled = irq_disable_save();
49 raw_spin_lock(lock);
50 return irqs_were_enabled;
51}
52
53static inline void raw_spin_unlock_irq_restore(struct raw_spinlock *lock,
54 bool irqs_were_enabled)
55 TSA_RELEASES_IRQS_LOCK(lock) TSA_NO_ANALYSIS {
56 raw_spin_unlock(lock);
57 irq_restore(was_enabled: irqs_were_enabled);
58}
59
60struct raw_spinlock_high_guard {
61 struct raw_spinlock *lock;
62 bool irqs_were_enabled;
63};
64
65static inline cc_always_inline cc_unused void
66raw_spin_guard_assume(struct raw_spinlock *lock) TSA_ASSERT_CAPABILITY(lock) {
67 cc_ignore(lock);
68}
69
70static inline cc_always_inline cc_unused void
71raw_spin_guard_exit(struct raw_spinlock **lock) TSA_NO_ANALYSIS {
72 if (*lock)
73 raw_spin_unlock(lock: *lock);
74}
75
76static inline cc_always_inline cc_unused struct raw_spinlock *
77raw_spin_guard_enter(struct raw_spinlock *lock) TSA_NO_ANALYSIS {
78 raw_spin_lock(lock);
79 return lock;
80}
81
82static inline cc_always_inline cc_unused void raw_spin_high_guard_exit(
83 struct raw_spinlock_high_guard *g) TSA_RELEASES_IRQS TSA_NO_ANALYSIS {
84 if (g->lock)
85 raw_spin_unlock_irq_restore(lock: g->lock, irqs_were_enabled: g->irqs_were_enabled);
86}
87
88static inline cc_always_inline cc_unused struct raw_spinlock_high_guard
89raw_spin_high_guard_enter(struct raw_spinlock *lock)
90 TSA_ACQUIRES_IRQS TSA_NO_ANALYSIS {
91 return (struct raw_spinlock_high_guard) {
92 .lock = lock,
93 .irqs_were_enabled = raw_spin_lock_high(lock),
94 };
95}
96
97#define raw_spin_guard(lock_) \
98 cc_cleanup(raw_spin_guard_exit) struct raw_spinlock *PP_CONCAT( \
99 raw_spin_guard_, __COUNTER__) = \
100 (raw_spin_guard_assume(lock_), raw_spin_guard_enter(lock_))
101
102#define raw_spin_guard_high(lock_) \
103 cc_cleanup(raw_spin_high_guard_exit) struct raw_spinlock_high_guard \
104 PP_CONCAT(raw_spin_guard_high_, __COUNTER__) = \
105 (raw_spin_guard_assume(lock_), raw_spin_high_guard_enter(lock_))
106