1#pragma once
2#include <asm.h>
3#include <bootstage.h>
4#include <compiler.h>
5#include <console/panic.h>
6#include <irq/irq.h>
7#include <kassert.h>
8#include <sch/irql.h>
9#include <smp/core.h>
10#include <stdatomic.h>
11#include <stdbool.h>
12
13#ifdef DEBUG_LOCK
14#define SPINLOCK_COOKIE_MAGIC 0xBEEB00
15#endif
16
17struct spinlock {
18 _Atomic uint8_t state;
19
20#ifdef DEBUG_LOCK
21 bool acquired_high;
22 uint32_t initialized_magic;
23#endif
24};
25
26#ifdef DEBUG_LOCK
27#define SPINLOCK_INIT \
28 {.state = ATOMIC_VAR_INIT(0), \
29 .acquired_high = false, \
30 .initialized_magic = SPINLOCK_COOKIE_MAGIC}
31#else
32#define SPINLOCK_INIT {ATOMIC_VAR_INIT(0)}
33#endif
34
35static inline void spinlock_init(struct spinlock *lock) {
36
37#ifdef DEBUG_LOCK
38 lock->initialized_magic = SPINLOCK_COOKIE_MAGIC;
39 lock->acquired_high = false;
40#endif
41
42 atomic_store(&lock->state, 0);
43}
44
45static inline bool __warn_unused_result
46spin_trylock_raw(struct spinlock *lock) {
47
48#ifdef DEBUG_LOCK
49 kassert(lock->initialized_magic == SPINLOCK_COOKIE_MAGIC);
50#endif
51
52 uint8_t expected = 0;
53 return atomic_compare_exchange_strong_explicit(
54 &lock->state, &expected, 1, memory_order_acquire, memory_order_relaxed);
55}
56
57static inline void spin_raw(struct spinlock *lock) {
58 while (atomic_load_explicit(&lock->state, memory_order_relaxed) != 0)
59 cpu_relax();
60}
61
62static inline void spin_lock_raw(struct spinlock *lock) {
63 while (true) {
64 if (spin_trylock_raw(lock))
65 return;
66
67 spin_raw(lock);
68 }
69}
70
71static inline void spin_unlock_raw(struct spinlock *lock) {
72 atomic_store_explicit(&lock->state, 0, memory_order_release);
73}
74
75static inline void spin_unlock(struct spinlock *lock, enum irql old) {
76 atomic_exchange_explicit(&lock->state, 0, memory_order_release);
77 irql_lower(old_level: old);
78}
79
80static inline enum irql __warn_unused_result spin_lock(struct spinlock *lock) {
81 if (bootstage_get() >= BOOTSTAGE_MID_MP && irq_in_interrupt())
82 panic("Attempted to take non-ISR safe spinlock from an ISR!");
83
84#ifdef DEBUG_LOCK
85 kassert(!lock->acquired_high);
86#endif
87
88 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
89 spin_lock_raw(lock);
90 return irql;
91}
92
93static inline enum irql __warn_unused_result
94spin_lock_irq_disable(struct spinlock *lock) {
95
96#ifdef DEBUG_LOCK
97 lock->acquired_high = true;
98#endif
99
100 enum irql irql = irql_raise(new_level: IRQL_HIGH_LEVEL);
101 spin_lock_raw(lock);
102 return irql;
103}
104
105static inline bool __warn_unused_result spin_trylock(struct spinlock *lock,
106 enum irql *out) {
107
108#ifdef DEBUG_LOCK
109 kassert(!lock->acquired_high);
110#endif
111
112 *out = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
113 if (spin_trylock_raw(lock))
114 return true;
115
116 irql_lower(old_level: *out);
117 return false;
118}
119
120static inline bool __warn_unused_result
121spin_trylock_irq_disable(struct spinlock *lock, enum irql *out) {
122
123#ifdef DEBUG_LOCK
124 lock->acquired_high = true;
125#endif
126
127 *out = irql_raise(new_level: IRQL_HIGH_LEVEL);
128 if (spin_trylock_raw(lock))
129 return true;
130
131 irql_lower(old_level: *out);
132 return false;
133}
134
135static inline bool spinlock_held(struct spinlock *lock) {
136 return atomic_load(&lock->state);
137}
138#define SPINLOCK_ASSERT_HELD(l) kassert(spinlock_held(l))
139