Warning: That file was not part of the compilation database. It may have many parsing errors.

1/* @title: Bit Spinlock */
2#pragma once
3#include <asm.h>
4#include <atomic.h>
5#include <compiler/core.h>
6#include <compiler/intrinsic.h>
7#include <enforce.h>
8#include <math/bit.h>
9#include <sch/irql.h>
10#include <stdbool.h>
11#include <stdint.h>
12
13/* bit_spinlock: not an actual struct, but a series of macros for locking any
14 * single bit within a byte, word, dword, and qword with comptime bounds check
15 */
16
17#define BIT_SPINLOCK_MASK(bit, ptr) ((typeof(*(ptr))) BIT(bit))
18
19#define BIT_SPINLOCK_CHECK(bit, ptr) \
20 do { \
21 static_assert( \
22 ci_constant_p(bit) ? ((size_t) (bit) < sizeof(*(ptr)) * 8) : 1, \
23 "bit index exceeds type width"); \
24 enforce((size_t) (bit) < sizeof(*(ptr)) * 8, \
25 "bit index out of bounds for pointer type"); \
26 } while (0)
27
28#define bit_spin_is_locked_raw(bit, ptr) \
29 ({ \
30 BIT_SPINLOCK_CHECK(bit, ptr); \
31 atomic_test_bit_rlx((atomic(typeof(*(ptr))) *) (ptr), bit); \
32 })
33
34#define bit_spin_trylock_raw(bit, ptr) \
35 ({ \
36 BIT_SPINLOCK_CHECK(bit, ptr); \
37 atomic(typeof(*(ptr))) *__bs_ptr = (atomic(typeof(*(ptr))) *) (ptr); \
38 typeof(*(ptr)) __bs_mask = BIT_SPINLOCK_MASK(bit, ptr); \
39 typeof(*(ptr)) __bs_old = atomic_load_rlx(__bs_ptr); \
40 bool __bs_acquired = false; \
41 if (!(__bs_old & __bs_mask)) { \
42 __bs_acquired = atomic_cas_weak( \
43 __bs_ptr, &__bs_old, (typeof(*(ptr))) (__bs_old | __bs_mask), \
44 mo_acquire, mo_rlx); \
45 } \
46 __bs_acquired; \
47 })
48
49#define bit_spin_lock_raw(bit, ptr) \
50 do { \
51 BIT_SPINLOCK_CHECK(bit, ptr); \
52 atomic(typeof(*(ptr))) *__bs_ptr = (atomic(typeof(*(ptr))) *) (ptr); \
53 typeof(*(ptr)) __bs_mask = BIT_SPINLOCK_MASK(bit, ptr); \
54 while (true) { \
55 typeof(*(ptr)) __bs_old = atomic_load_rlx(__bs_ptr); \
56 if (__bs_old & __bs_mask) { \
57 cpu_pause(); \
58 continue; \
59 } \
60 if (atomic_cas_weak(__bs_ptr, &__bs_old, \
61 (typeof(*(ptr))) (__bs_old | __bs_mask), \
62 mo_acquire, mo_rlx)) \
63 break; \
64 cpu_pause(); \
65 } \
66 } while (0)
67
68#define bit_spin_unlock_raw(bit, ptr) \
69 do { \
70 BIT_SPINLOCK_CHECK(bit, ptr); \
71 enforce(atomic_test_and_clear_bit_release( \
72 (atomic(typeof(*(ptr))) *) (ptr), bit), \
73 "bit spinlock unlock on unheld lock"); \
74 } while (0)
75
76#define bit_spin_lock(bit, ptr) \
77 ({ \
78 enum irql __bs_old_irql = irql_raise(IRQL_DISPATCH_LEVEL); \
79 bit_spin_lock_raw(bit, ptr); \
80 __bs_old_irql; \
81 })
82
83#define bit_spin_unlock(bit, ptr, old_irql) \
84 do { \
85 bit_spin_unlock_raw(bit, ptr); \
86 irql_lower(old_irql); \
87 } while (0)
88
89#define bit_spin_lock_high(bit, ptr) \
90 ({ \
91 enum irql __bs_old_irql = irql_raise(IRQL_HIGH_LEVEL); \
92 bit_spin_lock_raw(bit, ptr); \
93 __bs_old_irql; \
94 })
95
96#define bit_spin_unlock_irq_restore(bit, ptr, old_irql) \
97 do { \
98 bit_spin_unlock_raw(bit, ptr); \
99 irql_lower(old_irql); \
100 } while (0)
101
102#define bit_spin_trylock(bit, ptr, out_irql) \
103 ({ \
104 enum irql *__bs_out_irql = (out_irql); \
105 *__bs_out_irql = irql_raise(IRQL_DISPATCH_LEVEL); \
106 bool __bs_ok = bit_spin_trylock_raw(bit, ptr); \
107 if (!__bs_ok) \
108 irql_lower(*__bs_out_irql); \
109 __bs_ok; \
110 })
111
112#define bit_spin_trylock_high(bit, ptr, out_irql) \
113 ({ \
114 enum irql *__bs_out_irql = (out_irql); \
115 *__bs_out_irql = irql_raise(IRQL_HIGH_LEVEL); \
116 bool __bs_ok = bit_spin_trylock_raw(bit, ptr); \
117 if (!__bs_ok) \
118 irql_lower(*__bs_out_irql); \
119 __bs_ok; \
120 })
121
122#define BIT_SPIN_LOCK_ASSERT_HELD(bit, ptr) \
123 enforce(bit_spin_is_locked_raw(bit, ptr), "bitlock not held")
124
125struct bit_spinlock_guard {
126 void *ptr;
127 uint8_t bit;
128 uint8_t size;
129 enum irql irql;
130};
131
132struct bit_spinlock_raw_guard {
133 void *ptr;
134 uint8_t bit;
135 uint8_t size;
136};
137
138static inline cc_always_inline void
139bit_spin_unlock_sized_raw(uint8_t bit, void *ptr, uint8_t size) {
140 switch (size) {
141 case 1: bit_spin_unlock_raw(bit, (uint8_t *) ptr); break;
142 case 2: bit_spin_unlock_raw(bit, (uint16_t *) ptr); break;
143 case 4: bit_spin_unlock_raw(bit, (uint32_t *) ptr); break;
144 case 8:
145 default: bit_spin_unlock_raw(bit, (uint64_t *) ptr); break;
146 }
147}
148
149static inline cc_always_inline cc_unused void
150bit_spin_raw_guard_exit(struct bit_spinlock_raw_guard *g) {
151 if (g->ptr)
152 bit_spin_unlock_sized_raw(bit: g->bit, ptr: g->ptr, size: g->size);
153}
154
155static inline cc_always_inline cc_unused void
156bit_spin_guard_exit(struct bit_spinlock_guard *g)
157 TSA_RELEASES(IRQL_RAISED) TSA_NO_ANALYSIS {
158 if (g->ptr) {
159 bit_spin_unlock_sized_raw(bit: g->bit, ptr: g->ptr, size: g->size);
160 irql_lower(old_level: g->irql);
161 }
162}
163
164#define bit_spin_guard(bit_, ptr_) \
165 cc_cleanup(bit_spin_guard_exit) struct bit_spinlock_guard PP_CONCAT( \
166 bit_spin_guard_, __COUNTER__) = { \
167 .ptr = (void *) (ptr_), \
168 .bit = (uint8_t) (bit_), \
169 .size = (uint8_t) sizeof(*(ptr_)), \
170 .irql = bit_spin_lock((bit_), (ptr_)), \
171 }
172
173#define bit_spin_guard_high(bit_, ptr_) \
174 cc_cleanup(bit_spin_guard_exit) struct bit_spinlock_guard PP_CONCAT( \
175 bit_spin_guard_high_, __COUNTER__) = { \
176 .ptr = (void *) (ptr_), \
177 .bit = (uint8_t) (bit_), \
178 .size = (uint8_t) sizeof(*(ptr_)), \
179 .irql = bit_spin_lock_high((bit_), (ptr_)), \
180 }
181
182#define bit_spin_guard_raw(bit_, ptr_) \
183 bit_spin_lock_raw((bit_), (ptr_)); \
184 cc_cleanup(bit_spin_raw_guard_exit) struct bit_spinlock_raw_guard \
185 PP_CONCAT(bit_spin_guard_raw_, __COUNTER__) = { \
186 .ptr = (void *) (ptr_), \
187 .bit = (uint8_t) (bit_), \
188 .size = (uint8_t) sizeof(*(ptr_)), \
189 }
190

Warning: That file was not part of the compilation database. It may have many parsing errors.