1/* @title: Atomic Operations, Types & Primitives */
2#pragma once
3
4#include <compiler/atomic.h>
5#include <compiler/core.h>
6#include <stdatomic.h>
7#include <stdbool.h>
8#include <stddef.h>
9#include <stdint.h>
10
11typedef _Atomic uint8_t atomic_uint8_t;
12typedef _Atomic uint16_t atomic_uint16_t;
13typedef _Atomic uint32_t atomic_uint32_t;
14typedef _Atomic uint64_t atomic_uint64_t;
15typedef _Atomic __uint128_t atomic_uint128_t;
16
17typedef _Atomic int8_t atomic_int8_t;
18typedef _Atomic int16_t atomic_int16_t;
19typedef _Atomic int32_t atomic_int32_t;
20typedef _Atomic int64_t atomic_int64_t;
21typedef _Atomic __int128_t atomic_iint128_t;
22
23typedef _Atomic bool atomic_bool;
24typedef _Atomic size_t atomic_size_t;
25typedef _Atomic uintptr_t atomic_uintptr_t;
26typedef _Atomic intptr_t atomic_intptr_t;
27typedef _Atomic ptrdiff_t atomic_ptrdiff_t;
28
29/* Nice alias */
30#define atomic(type) _Atomic(type)
31
32/* ==== Memory Order ==== */
33#define mo_rlx memory_order_relaxed
34#define mo_consume memory_order_consume
35#define mo_acquire memory_order_acquire
36#define mo_release memory_order_release
37#define mo_acq_rel memory_order_acq_rel
38#define mo_seq_cst memory_order_seq_cst
39
40/* ==== Memory Order Validations ==== */
41#define atomic_check_not_consume_internal(mo) \
42 static_assert(!__builtin_constant_p(mo) || (mo) != memory_order_consume, \
43 "mo_consume is prohibited, use mo_acquire or ca_read_once")
44
45#define atomic_check_load_mo_internal(mo) \
46 static_assert( \
47 !__builtin_constant_p(mo) || \
48 ((mo) == memory_order_relaxed || (mo) == memory_order_acquire || \
49 (mo) == memory_order_seq_cst), \
50 "Invalid memory order for atomic_load: cannot use release or acq_rel")
51
52#define atomic_check_store_mo_internal(mo) \
53 static_assert(!__builtin_constant_p(mo) || \
54 ((mo) == memory_order_relaxed || \
55 (mo) == memory_order_release || \
56 (mo) == memory_order_seq_cst), \
57 "Invalid memory order for atomic_store: cannot use " \
58 "acquire or acq_rel")
59
60#define atomic_check_cas_fail_mo_internal(fail_mo) \
61 static_assert(!__builtin_constant_p(fail_mo) || \
62 ((fail_mo) != memory_order_release && \
63 (fail_mo) != memory_order_acq_rel), \
64 "CAS failure memory order cannot be release or acq_rel")
65
66#define atomic_overload_internal(name, ...) \
67 PP_DISPATCH(name, PP_NARG(__VA_ARGS__))
68#define atomic_call_internal(name, ...) \
69 atomic_overload_internal(name, __VA_ARGS__)(__VA_ARGS__)
70
71#undef atomic_load
72#define atomic_load_checked_internal(ptr, mo) \
73 ({ \
74 atomic_check_not_consume_internal(mo); \
75 atomic_check_load_mo_internal(mo); \
76 atomic_load_explicit((ptr), (mo)); \
77 })
78#define atomic_load_1(ptr) atomic_load_2((ptr), mo_seq_cst)
79#define atomic_load_2(ptr, mo) atomic_load_checked_internal((ptr), (mo))
80#define atomic_load(...) atomic_call_internal(atomic_load, __VA_ARGS__)
81
82#undef atomic_store
83#define atomic_store_checked_internal(ptr, val, mo) \
84 ({ \
85 atomic_check_not_consume_internal(mo); \
86 atomic_check_store_mo_internal(mo); \
87 atomic_store_explicit((ptr), (val), (mo)); \
88 })
89#define atomic_store_2(ptr, val) atomic_store_3((ptr), (val), mo_seq_cst)
90#define atomic_store_3(ptr, val, mo) \
91 atomic_store_checked_internal((ptr), (val), (mo))
92#define atomic_store(...) atomic_call_internal(atomic_store, __VA_ARGS__)
93
94#undef atomic_exchange
95#define atomic_xchg_checked_internal(ptr, val, mo) \
96 ({ \
97 atomic_check_not_consume_internal(mo); \
98 atomic_exchange_explicit((ptr), (val), (mo)); \
99 })
100#define atomic_exchange_2(ptr, val) atomic_exchange_3((ptr), (val), mo_seq_cst)
101#define atomic_exchange_3(ptr, val, mo) \
102 atomic_xchg_checked_internal((ptr), (val), (mo))
103#define atomic_exchange(...) atomic_call_internal(atomic_exchange, __VA_ARGS__)
104
105#undef atomic_compare_exchange_weak
106#define atomic_cas_weak_checked_internal(ptr, exp, des, succ, fail) \
107 ({ \
108 atomic_check_not_consume_internal(succ); \
109 atomic_check_not_consume_internal(fail); \
110 atomic_check_cas_fail_mo_internal(fail); \
111 atomic_compare_exchange_weak_explicit((ptr), (exp), (des), (succ), \
112 (fail)); \
113 })
114#define atomic_compare_exchange_weak_3(ptr, exp, des) \
115 atomic_compare_exchange_weak_5((ptr), (exp), (des), mo_seq_cst, mo_seq_cst)
116#define atomic_compare_exchange_weak_4(ptr, exp, des, mo) \
117 ({ \
118 static_assert(0, "4-argument CAS is prohibited"); \
119 false; \
120 })
121#define atomic_compare_exchange_weak_5(ptr, exp, des, succ, fail) \
122 atomic_cas_weak_checked_internal((ptr), (exp), (des), (succ), (fail))
123#define atomic_compare_exchange_weak(...) \
124 atomic_call_internal(atomic_compare_exchange_weak, __VA_ARGS__)
125#define atomic_cas_weak(...) atomic_compare_exchange_weak(__VA_ARGS__)
126
127#undef atomic_compare_exchange_strong
128#define atomic_cas_strong_checked_internal(ptr, exp, des, succ, fail) \
129 ({ \
130 atomic_check_not_consume_internal(succ); \
131 atomic_check_not_consume_internal(fail); \
132 atomic_check_cas_fail_mo_internal(fail); \
133 atomic_compare_exchange_strong_explicit((ptr), (exp), (des), (succ), \
134 (fail)); \
135 })
136#define atomic_compare_exchange_strong_3(ptr, exp, des) \
137 atomic_compare_exchange_strong_5((ptr), (exp), (des), mo_seq_cst, \
138 mo_seq_cst)
139#define atomic_compare_exchange_strong_4(ptr, exp, des, mo) \
140 ({ \
141 static_assert(0, "4-argument CAS is prohibited"); \
142 false; \
143 })
144#define atomic_compare_exchange_strong_5(ptr, exp, des, succ, fail) \
145 atomic_cas_strong_checked_internal((ptr), (exp), (des), (succ), (fail))
146#define atomic_compare_exchange_strong(...) \
147 atomic_call_internal(atomic_compare_exchange_strong, __VA_ARGS__)
148#define atomic_cas_strong(...) atomic_compare_exchange_strong(__VA_ARGS__)
149
150/* Linux-style value-returning atomic_cmpxchg */
151#define atomic_cmpxchg_checked_internal(ptr, old_val, new_val, succ, fail) \
152 ({ \
153 __auto_type cpx_ptr_internal = (ptr); \
154 typeof((void) 0, *cpx_ptr_internal) cpx_exp_internal = (old_val); \
155 (void) atomic_compare_exchange_strong_explicit( \
156 cpx_ptr_internal, &cpx_exp_internal, (new_val), (succ), (fail)); \
157 cpx_exp_internal; \
158 })
159#define atomic_cmpxchg_3(ptr, old_val, new_val) \
160 atomic_cmpxchg_5((ptr), (old_val), (new_val), mo_seq_cst, mo_seq_cst)
161#define atomic_cmpxchg_4(ptr, old_val, new_val, mo) \
162 ({ \
163 static_assert(0, "4-argument cmpxchg is prohibited"); \
164 *(ptr); \
165 })
166#define atomic_cmpxchg_5(ptr, old_val, new_val, succ, fail) \
167 atomic_cmpxchg_checked_internal((ptr), (old_val), (new_val), (succ), (fail))
168#define atomic_cmpxchg(...) atomic_call_internal(atomic_cmpxchg, __VA_ARGS__)
169
170/* Arithmetic and Bitwise */
171#undef atomic_fetch_add
172#define atomic_fetch_add_checked_internal(ptr, val, mo) \
173 ({ \
174 atomic_check_not_consume_internal(mo); \
175 atomic_fetch_add_explicit((ptr), (val), (mo)); \
176 })
177#define atomic_fetch_add_2(ptr, val) \
178 atomic_fetch_add_3((ptr), (val), mo_seq_cst)
179#define atomic_fetch_add_3(ptr, val, mo) \
180 atomic_fetch_add_checked_internal((ptr), (val), (mo))
181#define atomic_fetch_add(...) \
182 atomic_call_internal(atomic_fetch_add, __VA_ARGS__)
183
184#undef atomic_fetch_sub
185#define atomic_fetch_sub_checked_internal(ptr, val, mo) \
186 ({ \
187 atomic_check_not_consume_internal(mo); \
188 atomic_fetch_sub_explicit((ptr), (val), (mo)); \
189 })
190#define atomic_fetch_sub_2(ptr, val) \
191 atomic_fetch_sub_3((ptr), (val), mo_seq_cst)
192#define atomic_fetch_sub_3(ptr, val, mo) \
193 atomic_fetch_sub_checked_internal((ptr), (val), (mo))
194#define atomic_fetch_sub(...) \
195 atomic_call_internal(atomic_fetch_sub, __VA_ARGS__)
196
197#undef atomic_fetch_and
198#define atomic_fetch_and_checked_internal(ptr, val, mo) \
199 ({ \
200 atomic_check_not_consume_internal(mo); \
201 atomic_fetch_and_explicit((ptr), (val), (mo)); \
202 })
203#define atomic_fetch_and_2(ptr, val) \
204 atomic_fetch_and_3((ptr), (val), mo_seq_cst)
205#define atomic_fetch_and_3(ptr, val, mo) \
206 atomic_fetch_and_checked_internal((ptr), (val), (mo))
207#define atomic_fetch_and(...) \
208 atomic_call_internal(atomic_fetch_and, __VA_ARGS__)
209
210#undef atomic_fetch_or
211#define atomic_fetch_or_checked_internal(ptr, val, mo) \
212 ({ \
213 atomic_check_not_consume_internal(mo); \
214 atomic_fetch_or_explicit((ptr), (val), (mo)); \
215 })
216#define atomic_fetch_or_2(ptr, val) atomic_fetch_or_3((ptr), (val), mo_seq_cst)
217#define atomic_fetch_or_3(ptr, val, mo) \
218 atomic_fetch_or_checked_internal((ptr), (val), (mo))
219#define atomic_fetch_or(...) atomic_call_internal(atomic_fetch_or, __VA_ARGS__)
220
221#undef atomic_fetch_xor
222#define atomic_fetch_xor_checked_internal(ptr, val, mo) \
223 ({ \
224 atomic_check_not_consume_internal(mo); \
225 atomic_fetch_xor_explicit((ptr), (val), (mo)); \
226 })
227#define atomic_fetch_xor_2(ptr, val) \
228 atomic_fetch_xor_3((ptr), (val), mo_seq_cst)
229#define atomic_fetch_xor_3(ptr, val, mo) \
230 atomic_fetch_xor_checked_internal((ptr), (val), (mo))
231#define atomic_fetch_xor(...) \
232 atomic_call_internal(atomic_fetch_xor, __VA_ARGS__)
233
234/* Atomic Flags */
235#undef atomic_flag_test_and_set
236#define atomic_flag_test_and_set_checked_internal(ptr, mo) \
237 ({ \
238 atomic_check_not_consume_internal(mo); \
239 atomic_flag_test_and_set_explicit((ptr), (mo)); \
240 })
241#define atomic_flag_test_and_set_1(ptr) \
242 atomic_flag_test_and_set_2((ptr), mo_seq_cst)
243#define atomic_flag_test_and_set_2(ptr, mo) \
244 atomic_flag_test_and_set_checked_internal((ptr), (mo))
245#define atomic_flag_test_and_set(...) \
246 atomic_call_internal(atomic_flag_test_and_set, __VA_ARGS__)
247
248#undef atomic_flag_clear
249#define atomic_flag_clear_checked_internal(ptr, mo) \
250 ({ \
251 atomic_check_not_consume_internal(mo); \
252 atomic_check_store_mo_internal(mo); \
253 atomic_flag_clear_explicit((ptr), (mo)); \
254 })
255#define atomic_flag_clear_1(ptr) atomic_flag_clear_2((ptr), mo_seq_cst)
256#define atomic_flag_clear_2(ptr, mo) \
257 atomic_flag_clear_checked_internal((ptr), (mo))
258#define atomic_flag_clear(...) \
259 atomic_call_internal(atomic_flag_clear, __VA_ARGS__)
260
261/* ==== Mnemonic Suffixes ==== */
262#define atomic_load_rlx(ptr) atomic_load((ptr), mo_rlx)
263#define atomic_load_acq(ptr) atomic_load((ptr), mo_acquire)
264
265#define atomic_store_rlx(ptr, val) atomic_store((ptr), (val), mo_rlx)
266#define atomic_store_release(ptr, val) atomic_store((ptr), (val), mo_release)
267
268#define atomic_xchg_rlx(ptr, val) atomic_exchange((ptr), (val), mo_rlx)
269#define atomic_xchg_acq(ptr, val) atomic_exchange((ptr), (val), mo_acquire)
270#define atomic_xchg_release(ptr, val) atomic_exchange((ptr), (val), mo_release)
271#define atomic_xchg_acq_rel(ptr, val) atomic_exchange((ptr), (val), mo_acq_rel)
272
273/* Fetch arithmetic and bitwise shorthands */
274#define atomic_fetch_add_rlx(ptr, val) atomic_fetch_add((ptr), (val), mo_rlx)
275#define atomic_fetch_add_acq(ptr, val) \
276 atomic_fetch_add((ptr), (val), mo_acquire)
277#define atomic_fetch_add_release(ptr, val) \
278 atomic_fetch_add((ptr), (val), mo_release)
279#define atomic_fetch_add_acq_rel(ptr, val) \
280 atomic_fetch_add((ptr), (val), mo_acq_rel)
281
282#define atomic_fetch_sub_rlx(ptr, val) atomic_fetch_sub((ptr), (val), mo_rlx)
283#define atomic_fetch_sub_acq(ptr, val) \
284 atomic_fetch_sub((ptr), (val), mo_acquire)
285#define atomic_fetch_sub_release(ptr, val) \
286 atomic_fetch_sub((ptr), (val), mo_release)
287#define atomic_fetch_sub_acq_rel(ptr, val) \
288 atomic_fetch_sub((ptr), (val), mo_acq_rel)
289
290#define atomic_fetch_or_rlx(ptr, val) atomic_fetch_or((ptr), (val), mo_rlx)
291#define atomic_fetch_or_acq(ptr, val) atomic_fetch_or((ptr), (val), mo_acquire)
292#define atomic_fetch_or_release(ptr, val) \
293 atomic_fetch_or((ptr), (val), mo_release)
294#define atomic_fetch_or_acq_rel(ptr, val) \
295 atomic_fetch_or((ptr), (val), mo_acq_rel)
296
297#define atomic_fetch_and_rlx(ptr, val) atomic_fetch_and((ptr), (val), mo_rlx)
298#define atomic_fetch_and_acq(ptr, val) \
299 atomic_fetch_and((ptr), (val), mo_acquire)
300#define atomic_fetch_and_release(ptr, val) \
301 atomic_fetch_and((ptr), (val), mo_release)
302#define atomic_fetch_and_acq_rel(ptr, val) \
303 atomic_fetch_and((ptr), (val), mo_acq_rel)
304
305#define atomic_fetch_xor_rlx(ptr, val) atomic_fetch_xor((ptr), (val), mo_rlx)
306#define atomic_fetch_xor_acq(ptr, val) \
307 atomic_fetch_xor((ptr), (val), mo_acquire)
308#define atomic_fetch_xor_release(ptr, val) \
309 atomic_fetch_xor((ptr), (val), mo_release)
310#define atomic_fetch_xor_acq_rel(ptr, val) \
311 atomic_fetch_xor((ptr), (val), mo_acq_rel)
312
313/* Flag shorthands */
314#define atomic_flag_test_and_set_rlx(ptr) \
315 atomic_flag_test_and_set((ptr), mo_rlx)
316#define atomic_flag_test_and_set_acq(ptr) \
317 atomic_flag_test_and_set((ptr), mo_acquire)
318#define atomic_flag_test_and_set_acq_rel(ptr) \
319 atomic_flag_test_and_set((ptr), mo_acq_rel)
320#define atomic_flag_clear_rlx(ptr) atomic_flag_clear((ptr), mo_rlx)
321#define atomic_flag_clear_release(ptr) atomic_flag_clear((ptr), mo_release)
322
323/* ==== Arithmetic Shorthands ==== */
324#define atomic_inc_rlx(ptr) atomic_inc((ptr), mo_rlx)
325#define atomic_inc_acq(ptr) atomic_inc((ptr), mo_acquire)
326#define atomic_inc_release(ptr) atomic_inc((ptr), mo_release)
327#define atomic_inc_acq_rel(ptr) atomic_inc((ptr), mo_acq_rel)
328
329#define atomic_dec_rlx(ptr) atomic_dec((ptr), mo_rlx)
330#define atomic_dec_acq(ptr) atomic_dec((ptr), mo_acquire)
331#define atomic_dec_release(ptr) atomic_dec((ptr), mo_release)
332#define atomic_dec_acq_rel(ptr) atomic_dec((ptr), mo_acq_rel)
333#define atomic_inc_1(ptr) atomic_inc_2((ptr), mo_seq_cst)
334#define atomic_inc_2(ptr, mo) atomic_fetch_add_checked_internal((ptr), 1, (mo))
335#define atomic_inc(...) atomic_call_internal(atomic_inc, __VA_ARGS__)
336
337#define atomic_dec_1(ptr) atomic_dec_2((ptr), mo_seq_cst)
338#define atomic_dec_2(ptr, mo) atomic_fetch_sub_checked_internal((ptr), 1, (mo))
339#define atomic_dec(...) atomic_call_internal(atomic_dec, __VA_ARGS__)
340
341#define atomic_inc_return_new_1(ptr) atomic_inc_return_new_2((ptr), mo_seq_cst)
342#define atomic_inc_return_new_2(ptr, mo) \
343 (atomic_fetch_add_checked_internal((ptr), 1, (mo)) + 1)
344#define atomic_inc_return_new(...) \
345 atomic_call_internal(atomic_inc_return_new, __VA_ARGS__)
346
347#define atomic_dec_return_new_1(ptr) atomic_dec_return_new_2((ptr), mo_seq_cst)
348#define atomic_dec_return_new_2(ptr, mo) \
349 (atomic_fetch_sub_checked_internal((ptr), 1, (mo)) - 1)
350#define atomic_dec_return_new(...) \
351 atomic_call_internal(atomic_dec_return_new, __VA_ARGS__)
352
353#define atomic_inc_return_new_rlx(ptr) atomic_inc_return_new((ptr), mo_rlx)
354#define atomic_inc_return_new_acq(ptr) atomic_inc_return_new((ptr), mo_acquire)
355#define atomic_inc_return_new_release(ptr) \
356 atomic_inc_return_new((ptr), mo_release)
357#define atomic_inc_return_new_acq_rel(ptr) \
358 atomic_inc_return_new((ptr), mo_acq_rel)
359
360#define atomic_dec_return_new_rlx(ptr) atomic_dec_return_new((ptr), mo_rlx)
361#define atomic_dec_return_new_acq(ptr) atomic_dec_return_new((ptr), mo_acquire)
362#define atomic_dec_return_new_release(ptr) \
363 atomic_dec_return_new((ptr), mo_release)
364#define atomic_dec_return_new_acq_rel(ptr) \
365 atomic_dec_return_new((ptr), mo_acq_rel)
366
367#define atomic_dec_and_test(ptr) \
368 (atomic_fetch_sub_checked_internal((ptr), 1, mo_acq_rel) == 1)
369
370/* ==== Extremum & Watermark Operations ==== */
371#define atomic_fetch_max_explicit(ptr, val, succ_mo, fail_mo) \
372 ({ \
373 __auto_type fm_ptr_internal = (ptr); \
374 typeof((void) 0, *fm_ptr_internal) fm_val_internal = (val); \
375 typeof((void) 0, *fm_ptr_internal) fm_old_internal = \
376 atomic_load_checked_internal(fm_ptr_internal, (fail_mo)); \
377 while (fm_val_internal > fm_old_internal && \
378 !atomic_cas_weak_checked_internal( \
379 fm_ptr_internal, &fm_old_internal, fm_val_internal, \
380 (succ_mo), (fail_mo))) \
381 ; \
382 fm_old_internal; \
383 })
384
385#define atomic_fetch_max_2(ptr, val) \
386 atomic_fetch_max_4((ptr), (val), mo_seq_cst, mo_seq_cst)
387#define atomic_fetch_max_3(ptr, val, mo) \
388 ({ \
389 static_assert(0, "3-argument atomic_fetch_max is prohibited"); \
390 *(ptr); \
391 })
392#define atomic_fetch_max_4(ptr, val, succ_mo, fail_mo) \
393 atomic_fetch_max_explicit((ptr), (val), (succ_mo), (fail_mo))
394#define atomic_fetch_max(...) \
395 atomic_call_internal(atomic_fetch_max, __VA_ARGS__)
396
397#define atomic_fetch_min_explicit(ptr, val, succ_mo, fail_mo) \
398 ({ \
399 __auto_type fn_ptr_internal = (ptr); \
400 typeof((void) 0, *fn_ptr_internal) fn_val_internal = (val); \
401 typeof((void) 0, *fn_ptr_internal) fn_old_internal = \
402 atomic_load_checked_internal(fn_ptr_internal, (fail_mo)); \
403 while (fn_val_internal < fn_old_internal && \
404 !atomic_cas_weak_checked_internal( \
405 fn_ptr_internal, &fn_old_internal, fn_val_internal, \
406 (succ_mo), (fail_mo))) \
407 ; \
408 fn_old_internal; \
409 })
410
411#define atomic_fetch_min_2(ptr, val) \
412 atomic_fetch_min_4((ptr), (val), mo_seq_cst, mo_seq_cst)
413#define atomic_fetch_min_3(ptr, val, mo) \
414 ({ \
415 static_assert(0, "3-argument atomic_fetch_min is prohibited"); \
416 *(ptr); \
417 })
418#define atomic_fetch_min_4(ptr, val, succ_mo, fail_mo) \
419 atomic_fetch_min_explicit((ptr), (val), (succ_mo), (fail_mo))
420#define atomic_fetch_min(...) \
421 atomic_call_internal(atomic_fetch_min, __VA_ARGS__)
422
423/* ==== RMW CAS Loop Helper ==== */
424#define atomic_update_explicit(ptr, old_var, new_expr, succ_mo, fail_mo) \
425 ({ \
426 __auto_type u_ptr_internal = (ptr); \
427 typeof((void) 0, *u_ptr_internal) old_var = \
428 atomic_load_checked_internal(u_ptr_internal, (fail_mo)); \
429 while (!atomic_cas_weak_checked_internal( \
430 u_ptr_internal, &(old_var), (new_expr), (succ_mo), (fail_mo))) \
431 ; \
432 old_var; \
433 })
434
435#define atomic_update(ptr, old_var, new_expr) \
436 atomic_update_explicit((ptr), old_var, (new_expr), mo_seq_cst, mo_seq_cst)
437
438/* ==== Bitwise Operations ==== */
439#define atomic_set_bit_explicit(ptr, bit, mo) \
440 atomic_fetch_or_checked_internal( \
441 (ptr), ((typeof((void) 0, *(ptr))) 1) << (bit), (mo))
442
443#define atomic_clear_bit_explicit(ptr, bit, mo) \
444 atomic_fetch_and_checked_internal( \
445 (ptr), ~(((typeof((void) 0, *(ptr))) 1) << (bit)), (mo))
446
447#define atomic_toggle_bit_explicit(ptr, bit, mo) \
448 atomic_fetch_xor_checked_internal( \
449 (ptr), ((typeof((void) 0, *(ptr))) 1) << (bit), (mo))
450
451#define atomic_test_bit_explicit(ptr, bit, mo) \
452 ((atomic_load_checked_internal((ptr), (mo)) & \
453 (((typeof((void) 0, *(ptr))) 1) << (bit))) != 0)
454
455#define atomic_test_and_set_bit_explicit(ptr, bit, mo) \
456 ((atomic_fetch_or_checked_internal( \
457 (ptr), ((typeof((void) 0, *(ptr))) 1) << (bit), (mo)) & \
458 (((typeof((void) 0, *(ptr))) 1) << (bit))) != 0)
459
460#define atomic_test_and_clear_bit_explicit(ptr, bit, mo) \
461 ((atomic_fetch_and_checked_internal( \
462 (ptr), ~(((typeof((void) 0, *(ptr))) 1) << (bit)), (mo)) & \
463 (((typeof((void) 0, *(ptr))) 1) << (bit))) != 0)
464
465#define atomic_set_bit_2(ptr, bit) atomic_set_bit_3((ptr), (bit), mo_seq_cst)
466#define atomic_set_bit_3(ptr, bit, mo) \
467 atomic_set_bit_explicit((ptr), (bit), (mo))
468#define atomic_set_bit(...) atomic_call_internal(atomic_set_bit, __VA_ARGS__)
469
470#define atomic_clear_bit_2(ptr, bit) \
471 atomic_clear_bit_3((ptr), (bit), mo_seq_cst)
472#define atomic_clear_bit_3(ptr, bit, mo) \
473 atomic_clear_bit_explicit((ptr), (bit), (mo))
474#define atomic_clear_bit(...) \
475 atomic_call_internal(atomic_clear_bit, __VA_ARGS__)
476
477#define atomic_toggle_bit_2(ptr, bit) \
478 atomic_toggle_bit_3((ptr), (bit), mo_seq_cst)
479#define atomic_toggle_bit_3(ptr, bit, mo) \
480 atomic_toggle_bit_explicit((ptr), (bit), (mo))
481#define atomic_toggle_bit(...) \
482 atomic_call_internal(atomic_toggle_bit, __VA_ARGS__)
483
484#define atomic_test_bit_2(ptr, bit) atomic_test_bit_3((ptr), (bit), mo_seq_cst)
485#define atomic_test_bit_3(ptr, bit, mo) \
486 atomic_test_bit_explicit((ptr), (bit), (mo))
487#define atomic_test_bit(...) atomic_call_internal(atomic_test_bit, __VA_ARGS__)
488
489#define atomic_test_and_set_bit_2(ptr, bit) \
490 atomic_test_and_set_bit_3((ptr), (bit), mo_seq_cst)
491#define atomic_test_and_set_bit_3(ptr, bit, mo) \
492 atomic_test_and_set_bit_explicit((ptr), (bit), (mo))
493#define atomic_test_and_set_bit(...) \
494 atomic_call_internal(atomic_test_and_set_bit, __VA_ARGS__)
495
496#define atomic_test_and_clear_bit_2(ptr, bit) \
497 atomic_test_and_clear_bit_3((ptr), (bit), mo_seq_cst)
498#define atomic_test_and_clear_bit_3(ptr, bit, mo) \
499 atomic_test_and_clear_bit_explicit((ptr), (bit), (mo))
500#define atomic_test_and_clear_bit(...) \
501 atomic_call_internal(atomic_test_and_clear_bit, __VA_ARGS__)
502
503#define atomic_set_bit_rlx(ptr, bit) atomic_set_bit((ptr), (bit), mo_rlx)
504#define atomic_set_bit_release(ptr, bit) \
505 atomic_set_bit((ptr), (bit), mo_release)
506
507#define atomic_clear_bit_rlx(ptr, bit) atomic_clear_bit((ptr), (bit), mo_rlx)
508#define atomic_clear_bit_release(ptr, bit) \
509 atomic_clear_bit((ptr), (bit), mo_release)
510
511#define atomic_toggle_bit_rlx(ptr, bit) atomic_toggle_bit((ptr), (bit), mo_rlx)
512#define atomic_test_bit_rlx(ptr, bit) atomic_test_bit((ptr), (bit), mo_rlx)
513
514#define atomic_test_bit_acq(ptr, bit) atomic_test_bit((ptr), (bit), mo_acquire)
515
516#define atomic_test_and_set_bit_acq(ptr, bit) \
517 atomic_test_and_set_bit((ptr), (bit), mo_acquire)
518#define atomic_test_and_set_bit_acq_rel(ptr, bit) \
519 atomic_test_and_set_bit((ptr), (bit), mo_acq_rel)
520#define atomic_test_and_clear_bit_release(ptr, bit) \
521 atomic_test_and_clear_bit((ptr), (bit), mo_release)
522#define atomic_test_and_clear_bit_acq_rel(ptr, bit) \
523 atomic_test_and_clear_bit((ptr), (bit), mo_acq_rel)
524
525/* ==== Polling Primitives ==== */
526
527#define atomic_poll_until(condition_expr) \
528 do { \
529 while (!(condition_expr)) { \
530 ca_pause(); \
531 } \
532 } while (0)
533
534#define atomic_poll_until_eq_3(ptr, val, mo) \
535 do { \
536 __auto_type p_ptr_internal = (ptr); \
537 typeof((void) 0, *p_ptr_internal) p_val_internal = (val); \
538 while (atomic_load_checked_internal(p_ptr_internal, (mo)) != \
539 p_val_internal) { \
540 ca_pause(); \
541 } \
542 } while (0)
543
544#define atomic_poll_until_eq_2(ptr, val) \
545 atomic_poll_until_eq_3((ptr), (val), mo_seq_cst)
546
547#define atomic_poll_until_eq(...) \
548 atomic_call_internal(atomic_poll_until_eq, __VA_ARGS__)
549
550/* ==== Cacheline Alignment ==== */
551
552#ifndef ca_cache_aligned
553#define ca_cache_aligned cc_cache_aligned
554#endif
555
556#define CA_CACHELINE_SIZE 64
557
558#define ca_cacheline_member(T, name) \
559 struct { \
560 uint8_t ca_pre_##name[CA_CACHELINE_SIZE - cc_alignof(T)]; \
561 union { \
562 T name; \
563 uint8_t ca_post_##name[CA_CACHELINE_SIZE]; \
564 }; \
565 }
566
567#ifndef TSA_LOCKLESS
568#define TSA_LOCKLESS
569#endif
570
571/* ==== Lock-Free Assertions ==== */
572
573#define ct_assert_atomic_lock_free(type) \
574 static_assert(__atomic_always_lock_free(sizeof(type), 0), \
575 #type " must be natively lock-free on this architecture")
576
577ct_assert_atomic_lock_free(uint8_t);
578ct_assert_atomic_lock_free(uint16_t);
579ct_assert_atomic_lock_free(uint32_t);
580ct_assert_atomic_lock_free(uint64_t);
581ct_assert_atomic_lock_free(void *);
582