1#include <atomic.h>
2#include <types/types.h>
3
4#pragma GCC diagnostic ignored "-Wmissing-prototypes"
5
6/* GCC references these when expecting libatomic, and we don't
7 * bother with the memory ordering constraint since cmpxchg16b is
8 * a full barrier anyways */
9static inline bool cmpxchg16b(volatile void *ptr, uint128_t *expected,
10 uint128_t desired) {
11 uint64_t lo = (uint64_t) *expected;
12 uint64_t hi = (uint64_t) (*expected >> 64);
13 bool ok;
14
15 asm volatile("lock cmpxchg16b %[mem]"
16 : [mem] "+m"(*(volatile uint128_t *) ptr), "+a"(lo), "+d"(hi),
17 "=@ccz"(ok)
18 : "b"((uint64_t) desired), "c"((uint64_t) (desired >> 64))
19 : "memory");
20
21 *expected = ((uint128_t) hi << 64) | lo;
22 return ok;
23}
24
25uint128_t __atomic_load_16(const volatile void *ptr, int mo) {
26 cc_ignore(mo);
27 uint128_t cur = 0;
28 cmpxchg16b(ptr: (volatile void *) ptr, expected: &cur, desired: 0);
29 return cur;
30}
31
32void __atomic_store_16(volatile void *ptr, uint128_t val, int mo) {
33 cc_ignore(mo);
34 uint128_t cur = 0;
35 while (!cmpxchg16b(ptr, expected: &cur, desired: val))
36 ;
37}
38
39uint128_t __atomic_exchange_16(volatile void *ptr, uint128_t val, int mo) {
40 cc_ignore(mo);
41 uint128_t cur = 0;
42 while (!cmpxchg16b(ptr, expected: &cur, desired: val))
43 ;
44 return cur;
45}
46
47bool atomic_compare_exchange_16(
48 volatile void *ptr, void *expected, uint128_t desired, int success,
49 int failure) asm("__atomic_compare_exchange_16");
50
51bool atomic_compare_exchange_16(volatile void *ptr, void *expected,
52 uint128_t desired, int success, int failure) {
53 cc_ignore(success, failure);
54 return cmpxchg16b(ptr, expected, desired);
55}
56