Bit Spinlock

include/sync/bit_spinlock.h View source View on GitHub

bit_spinlock: not an actual struct, but a series of macros for locking any single bit within a byte, word, dword, and qword with comptime bounds check

Structs

bit_spinlock_guard

struct bit_spinlock_guard (4 fields) source

§ void *ptr

§ uint8_t bit

§ uint8_t size

§ enum irql irql

bit_spinlock_raw_guard

struct bit_spinlock_raw_guard (3 fields) source

§ void *ptr

§ uint8_t bit

§ uint8_t size

Functions

bit_spin_unlock_sized_raw

cc_always_inline void bit_spin_unlock_sized_raw(
    uint8_t bit,
    void *ptr,
    uint8_t size
)

source

bit_spin_raw_guard_exit

void bit_spin_raw_guard_exit(struct bit_spinlock_raw_guard *g)

source

bit_spin_guard_exit

void bit_spin_guard_exit(struct bit_spinlock_guard *g)

source

Macros

BIT_SPINLOCK_MASK

#define BIT_SPINLOCK_MASK(bit, ptr)

source

Expansion

#define BIT_SPINLOCK_MASK(bit, ptr) ((typeof(*(ptr))) BIT(bit))

BIT_SPINLOCK_CHECK

#define BIT_SPINLOCK_CHECK(bit, ptr)

source

Expansion

#define BIT_SPINLOCK_CHECK(bit, ptr) \
    do { \
        static_assert( \
            ci_constant_p(bit) ? ((size_t) (bit) < sizeof(*(ptr)) * 8) : 1, \
            "bit index exceeds type width"); \
        enforce((size_t) (bit) < sizeof(*(ptr)) * 8, \
                "bit index out of bounds for pointer type"); \
    } while (0)

bit_spin_is_locked_raw

#define bit_spin_is_locked_raw(bit, ptr)

source

Expansion

#define bit_spin_is_locked_raw(bit, ptr) \
    ({ \
        BIT_SPINLOCK_CHECK(bit, ptr); \
        atomic_test_bit_rlx((atomic(typeof(*(ptr))) *) (ptr), bit); \
    })

bit_spin_trylock_raw

#define bit_spin_trylock_raw(bit, ptr)

source

Expansion

#define bit_spin_trylock_raw(bit, ptr) \
    ({ \
        BIT_SPINLOCK_CHECK(bit, ptr); \
        atomic(typeof(*(ptr))) *__bs_ptr = (atomic(typeof(*(ptr))) *) (ptr); \
        typeof(*(ptr)) __bs_mask = BIT_SPINLOCK_MASK(bit, ptr); \
        typeof(*(ptr)) __bs_old = atomic_load_rlx(__bs_ptr); \
        bool __bs_acquired = false; \
        if (!(__bs_old & __bs_mask)) { \
            __bs_acquired = atomic_cas_weak( \
                __bs_ptr, &__bs_old, (typeof(*(ptr))) (__bs_old | __bs_mask), \
                mo_acquire, mo_rlx); \
        } \
        __bs_acquired; \
    })

bit_spin_lock_raw

#define bit_spin_lock_raw(bit, ptr)

source

Expansion

#define bit_spin_lock_raw(bit, ptr) \
    do { \
        BIT_SPINLOCK_CHECK(bit, ptr); \
        atomic(typeof(*(ptr))) *__bs_ptr = (atomic(typeof(*(ptr))) *) (ptr); \
        typeof(*(ptr)) __bs_mask = BIT_SPINLOCK_MASK(bit, ptr); \
        while (true) { \
            typeof(*(ptr)) __bs_old = atomic_load_rlx(__bs_ptr); \
            if (__bs_old & __bs_mask) { \
                cpu_pause(); \
                continue; \
            } \
            if (atomic_cas_weak(__bs_ptr, &__bs_old, \
                                (typeof(*(ptr))) (__bs_old | __bs_mask), \
                                mo_acquire, mo_rlx)) \
                break; \
            cpu_pause(); \
        } \
    } while (0)

bit_spin_unlock_raw

#define bit_spin_unlock_raw(bit, ptr)

source

Expansion

#define bit_spin_unlock_raw(bit, ptr) \
    do { \
        BIT_SPINLOCK_CHECK(bit, ptr); \
        enforce(atomic_test_and_clear_bit_release( \
                    (atomic(typeof(*(ptr))) *) (ptr), bit), \
                "bit spinlock unlock on unheld lock"); \
    } while (0)

bit_spin_lock

#define bit_spin_lock(bit, ptr)

source

Expansion

#define bit_spin_lock(bit, ptr) \
    ({ \
        enum irql __bs_old_irql = irql_raise(IRQL_DISPATCH_LEVEL); \
        bit_spin_lock_raw(bit, ptr); \
        __bs_old_irql; \
    })

bit_spin_unlock

#define bit_spin_unlock(bit, ptr, old_irql)

source

Expansion

#define bit_spin_unlock(bit, ptr, old_irql) \
    do { \
        bit_spin_unlock_raw(bit, ptr); \
        irql_lower(old_irql); \
    } while (0)

bit_spin_lock_high

#define bit_spin_lock_high(bit, ptr)

source

Expansion

#define bit_spin_lock_high(bit, ptr) \
    ({ \
        enum irql __bs_old_irql = irql_raise(IRQL_HIGH_LEVEL); \
        bit_spin_lock_raw(bit, ptr); \
        __bs_old_irql; \
    })

bit_spin_unlock_irq_restore

#define bit_spin_unlock_irq_restore(bit, ptr, old_irql)

source

Expansion

#define bit_spin_unlock_irq_restore(bit, ptr, old_irql) \
    do { \
        bit_spin_unlock_raw(bit, ptr); \
        irql_lower(old_irql); \
    } while (0)

bit_spin_trylock

#define bit_spin_trylock(bit, ptr, out_irql)

source

Expansion

#define bit_spin_trylock(bit, ptr, out_irql) \
    ({ \
        enum irql *__bs_out_irql = (out_irql); \
        *__bs_out_irql = irql_raise(IRQL_DISPATCH_LEVEL); \
        bool __bs_ok = bit_spin_trylock_raw(bit, ptr); \
        if (!__bs_ok) \
            irql_lower(*__bs_out_irql); \
        __bs_ok; \
    })

bit_spin_trylock_high

#define bit_spin_trylock_high(bit, ptr, out_irql)

source

Expansion

#define bit_spin_trylock_high(bit, ptr, out_irql) \
    ({ \
        enum irql *__bs_out_irql = (out_irql); \
        *__bs_out_irql = irql_raise(IRQL_HIGH_LEVEL); \
        bool __bs_ok = bit_spin_trylock_raw(bit, ptr); \
        if (!__bs_ok) \
            irql_lower(*__bs_out_irql); \
        __bs_ok; \
    })

BIT_SPIN_LOCK_ASSERT_HELD

#define BIT_SPIN_LOCK_ASSERT_HELD(bit, ptr)

source

Expansion

#define BIT_SPIN_LOCK_ASSERT_HELD(bit, ptr) \
    enforce(bit_spin_is_locked_raw(bit, ptr), "bitlock not held")

bit_spin_guard

#define bit_spin_guard(bit_, ptr_)

source

Expansion

#define bit_spin_guard(bit_, ptr_) \
    cc_cleanup(bit_spin_guard_exit) struct bit_spinlock_guard PP_CONCAT( \
        bit_spin_guard_, __COUNTER__) = { \
        .ptr = (void *) (ptr_), \
        .bit = (uint8_t) (bit_), \
        .size = (uint8_t) sizeof(*(ptr_)), \
        .irql = bit_spin_lock((bit_), (ptr_)), \
    }

bit_spin_guard_high

#define bit_spin_guard_high(bit_, ptr_)

source

Expansion

#define bit_spin_guard_high(bit_, ptr_) \
    cc_cleanup(bit_spin_guard_exit) struct bit_spinlock_guard PP_CONCAT( \
        bit_spin_guard_high_, __COUNTER__) = { \
        .ptr = (void *) (ptr_), \
        .bit = (uint8_t) (bit_), \
        .size = (uint8_t) sizeof(*(ptr_)), \
        .irql = bit_spin_lock_high((bit_), (ptr_)), \
    }

bit_spin_guard_raw

#define bit_spin_guard_raw(bit_, ptr_)

source

Expansion

#define bit_spin_guard_raw(bit_, ptr_) \
    bit_spin_lock_raw((bit_), (ptr_)); \
    cc_cleanup(bit_spin_raw_guard_exit) struct bit_spinlock_raw_guard \
    PP_CONCAT(bit_spin_guard_raw_, __COUNTER__) = { \
        .ptr = (void *) (ptr_), \
        .bit = (uint8_t) (bit_), \
        .size = (uint8_t) sizeof(*(ptr_)), \
    }