Bit Spinlock

include/sync/bit_spinlock.h View source View on GitHub

Macros

BIT_SPINLOCK_MASK

#define BIT_SPINLOCK_MASK(bit, ptr) ((typeof(*(ptr))) (1ULL << (bit)))

BIT_SPINLOCK_CHECK

#define BIT_SPINLOCK_CHECK(bit, ptr) \
    do { \
        static_assert(__builtin_constant_p(bit) \
                          ? ((size_t) (bit) < sizeof(*(ptr)) * 8) \
                          : 1, \
                      "bit index exceeds type width"); \
        kassert((size_t) (bit) < sizeof(*(ptr)) * 8, \
                "bit index out of bounds for pointer type"); \
    } while (0)

bit_spin_is_locked_raw

#define bit_spin_is_locked_raw(bit, ptr) \
    ({ \
        BIT_SPINLOCK_CHECK(bit, ptr); \
        typeof(*(ptr)) __m = BIT_SPINLOCK_MASK(bit, ptr); \
        (atomic_load_explicit((_Atomic typeof(*(ptr)) *) (ptr), \
                              memory_order_relaxed) & \
         __m) != 0; \
    })

bit_spin_trylock_raw

#define bit_spin_trylock_raw(bit, ptr) \
    ({ \
        BIT_SPINLOCK_CHECK(bit, ptr); \
        typeof(*(ptr)) __m = BIT_SPINLOCK_MASK(bit, ptr); \
        typeof(*(ptr)) __old = atomic_load_explicit( \
            (_Atomic typeof(*(ptr)) *) (ptr), memory_order_relaxed); \
        bool __acquired = false; \
        if (!(__old & __m)) { \
            __acquired = atomic_compare_exchange_weak_explicit( \
                (_Atomic typeof(*(ptr)) *) (ptr), &__old, \
                (typeof(*(ptr))) (__old | __m), memory_order_acquire, \
                memory_order_relaxed); \
        } \
        __acquired; \
    })

bit_spin_lock_raw

#define bit_spin_lock_raw(bit, ptr) \
    do { \
        BIT_SPINLOCK_CHECK(bit, ptr); \
        typeof(*(ptr)) __m = BIT_SPINLOCK_MASK(bit, ptr); \
        while (true) { \
            typeof(*(ptr)) __old = atomic_load_explicit( \
                (_Atomic typeof(*(ptr)) *) (ptr), memory_order_relaxed); \
            if (__old & __m) { \
                cpu_relax(); \
                continue; \
            } \
            if (atomic_compare_exchange_weak_explicit( \
                    (_Atomic typeof(*(ptr)) *) (ptr), &__old, \
                    (typeof(*(ptr))) (__old | __m), memory_order_acquire, \
                    memory_order_relaxed)) \
                break; \
            cpu_relax(); \
        } \
    } while (0)

bit_spin_unlock_raw

#define bit_spin_unlock_raw(bit, ptr) \
    do { \
        BIT_SPINLOCK_CHECK(bit, ptr); \
        typeof(*(ptr)) __m = BIT_SPINLOCK_MASK(bit, ptr); \
        typeof(*(ptr)) __not_m = (typeof(*(ptr))) (~__m); \
        kassert(atomic_fetch_and_explicit((_Atomic typeof(*(ptr)) *) (ptr), \
                                          __not_m, memory_order_release) & \
                    __m, \
                "bit spinlock unlock on unheld lock"); \
    } while (0)

bit_spin_lock

#define bit_spin_lock(bit, ptr) \
    ({ \
        enum irql __old_irql = irql_raise(IRQL_DISPATCH_LEVEL); \
        bit_spin_lock_raw(bit, ptr); \
        __old_irql; \
    })

bit_spin_unlock

#define bit_spin_unlock(bit, ptr, old_irql) \
    do { \
        bit_spin_unlock_raw(bit, ptr); \
        irql_lower(old_irql); \
    } while (0)

bit_spin_lock_irq_disable

#define bit_spin_lock_irq_disable(bit, ptr) \
    ({ \
        enum irql __old_irql = irql_raise(IRQL_HIGH_LEVEL); \
        bit_spin_lock_raw(bit, ptr); \
        __old_irql; \
    })

bit_spin_unlock_irq_restore

#define bit_spin_unlock_irq_restore(bit, ptr, old_irql) \
    do { \
        bit_spin_unlock_raw(bit, ptr); \
        irql_lower(old_irql); \
    } while (0)

bit_spin_trylock

#define bit_spin_trylock(bit, ptr, out_irql) \
    ({ \
        *(out_irql) = irql_raise(IRQL_DISPATCH_LEVEL); \
        bool __ok = bit_spin_trylock_raw(bit, ptr); \
        if (!__ok) \
            irql_lower(*(out_irql)); \
        __ok; \
    })

bit_spin_trylock_irq_disable

#define bit_spin_trylock_irq_disable(bit, ptr, out_irql) \
    ({ \
        *(out_irql) = irql_raise(IRQL_HIGH_LEVEL); \
        bool __ok = bit_spin_trylock_raw(bit, ptr); \
        if (!__ok) \
            irql_lower(*(out_irql)); \
        __ok; \
    })

BIT_SPIN_LOCK_ASSERT_HELD

#define BIT_SPIN_LOCK_ASSERT_HELD(bit, ptr) \
    kassert(bit_spin_is_locked_raw(bit, ptr), "bitlock not held")

---