spinlock

include/sync/spinlock.h View source View on GitHub

Structs

spinlock_acq_scope

struct spinlock_acq_scope (3 fields)

§ struct lock_chk_acq_req req

§ struct lock_chk_acq_token token

§ bool checked

spinlock_rel_scope

struct spinlock_rel_scope (3 fields)

§ struct lock_chk_rel_req req

§ struct lock_chk_rel_token token

§ bool checked

spinlock_acq_scope

struct spinlock_acq_scope (1 field)

§ char unused_

spinlock_rel_scope

struct spinlock_rel_scope (1 field)

§ char unused_

spinlock_guard

struct spinlock_guard (2 fields)

Scoped guards. TSA can’t connect guard variables to lock expressions. IRQL_RAISED is acquired on entry and released on cleanup, and this prevents sleeping inside the guard. Lock expression is evaluated twice, remember to pass a lock address

§ struct spinlock *lock

§ enum irql irql

Functions

spinlock_init_chk_full

void spinlock_init_chk_full(
    struct spinlock *lock,
    const struct lock_chk_class *class,
    enum lock_chk_flags flags
)

spinlock_policy_init_internal

void spinlock_policy_init_internal(
    struct spinlock *lock,
    enum lock_chk_flags flags
)

spinlock_shallow_init_internal

void spinlock_shallow_init_internal(struct spinlock *lock)

spinlock_map_init_internal

void spinlock_map_init_internal(
    struct spinlock *lock,
    const struct lock_chk_class *class,
    enum lock_chk_flags flags
)

spinlock_note_use

void spinlock_note_use(
    struct spinlock *lock,
    enum lock_op_flags flags
)

spinlock_order_checked

bool spinlock_order_checked(struct spinlock *lock)

spinlock_stamp

void spinlock_stamp(struct spinlock *lock)

TODO

unify stamping at initialization too

spinlock_classify

void spinlock_classify(
    struct spinlock *lock,
    enum lock_op_flags usage,
    const struct lock_chk_site *site
)

spinlock_deep_checked

bool spinlock_deep_checked(struct spinlock *lock)

spinlock_chk_acq_req

struct lock_chk_acq_req spinlock_chk_acq_req(
    struct spinlock *lock,
    const struct lock_chk_site *site,
    enum lock_subclass subclass,
    enum lock_op_flags flags
)

spinlock_chk_rel_req

struct lock_chk_rel_req spinlock_chk_rel_req(
    struct spinlock *lock,
    const struct lock_chk_site *site
)

spinlock_acq_begin

void spinlock_acq_begin(
    struct spinlock_acq_scope *scope,
    struct spinlock *lock,
    const struct lock_chk_site *site,
    enum lock_subclass subclass,
    enum lock_op_flags flags
)

spinlock_acq_commit

void spinlock_acq_commit(struct spinlock_acq_scope *scope)

spinlock_acq_abort

void spinlock_acq_abort(struct spinlock_acq_scope *scope)

spinlock_rel_begin

void spinlock_rel_begin(
    struct spinlock_rel_scope *scope,
    struct spinlock *lock,
    const struct lock_chk_site *site
)

spinlock_rel_commit

void spinlock_rel_commit(struct spinlock_rel_scope *scope)

spinlock_shallow_push

void spinlock_shallow_push(
    struct spinlock *lock,
    enum irql irql,
    const struct lock_chk_site *site
)

spinlock_shallow_validate_top

void spinlock_shallow_validate_top(
    struct spinlock *lock,
    enum irql old,
    const struct lock_chk_site *site
)

spinlock_shallow_pop

void spinlock_shallow_pop(struct spinlock *lock)

spinlock_assert_held_deep

bool spinlock_assert_held_deep(
    struct spinlock *lock,
    bool want_held,
    const struct lock_chk_site *site
)

spinlock_assert_initialized

void spinlock_assert_initialized(struct spinlock *lock)

spinlock_policy_set_internal

void spinlock_policy_set_internal(
    struct spinlock *lock,
    enum lock_chk_flags flags
)

spinlock_policy_init_internal

void spinlock_policy_init_internal(
    struct spinlock *lock,
    enum lock_chk_flags flags
)

spinlock_shallow_init_internal

void spinlock_shallow_init_internal(struct spinlock *lock)

spinlock_map_init_internal

void spinlock_map_init_internal(
    struct spinlock *lock,
    const struct lock_chk_class *class,
    enum lock_chk_flags flags
)

spinlock_note_use

void spinlock_note_use(
    struct spinlock *lock,
    enum lock_op_flags flags
)

spinlock_order_checked

bool spinlock_order_checked(struct spinlock *lock)

spinlock_classify

void spinlock_classify(
    struct spinlock *lock,
    enum lock_op_flags usage,
    const struct lock_chk_site *site
)

spinlock_deep_checked

bool spinlock_deep_checked(struct spinlock *lock)

spinlock_acq_begin

void spinlock_acq_begin(
    struct spinlock_acq_scope *scope,
    struct spinlock *lock,
    const struct lock_chk_site *site,
    enum lock_subclass subclass,
    enum lock_op_flags flags
)

spinlock_acq_commit

void spinlock_acq_commit(struct spinlock_acq_scope *scope)

spinlock_acq_abort

void spinlock_acq_abort(struct spinlock_acq_scope *scope)

spinlock_rel_begin

void spinlock_rel_begin(
    struct spinlock_rel_scope *scope,
    struct spinlock *lock,
    const struct lock_chk_site *site
)

spinlock_rel_commit

void spinlock_rel_commit(struct spinlock_rel_scope *scope)

spinlock_shallow_push

void spinlock_shallow_push(
    struct spinlock *lock,
    enum irql irql,
    const struct lock_chk_site *site
)

spinlock_shallow_validate_top

void spinlock_shallow_validate_top(
    struct spinlock *lock,
    enum irql old,
    const struct lock_chk_site *site
)

spinlock_shallow_pop

void spinlock_shallow_pop(struct spinlock *lock)

spinlock_assert_held_deep

bool spinlock_assert_held_deep(
    struct spinlock *lock,
    bool want_held,
    const struct lock_chk_site *site
)

spinlock_assert_initialized

void spinlock_assert_initialized(struct spinlock *lock)

spinlock_policy_set_internal

void spinlock_policy_set_internal(
    struct spinlock *lock,
    enum lock_chk_flags flags
)

spin_trylock_physical

cc_warn_unused_result bool spin_trylock_physical(struct spinlock *lock)

spin_lock_physical

void spin_lock_physical(struct spinlock *lock)

spin_unlock_physical

void spin_unlock_physical(struct spinlock *lock)

spinlock_init_chk_full

void spinlock_init_chk_full(
    struct spinlock *lock,
    const struct lock_chk_class *class,
    enum lock_chk_flags flags
)

spinlock_restore_interrupts

void spinlock_restore_interrupts(bool enabled)

spin_trylock_raw_full

cc_warn_unused_result bool spin_trylock_raw_full(
    struct spinlock *lock,
    enum lock_subclass sc,
    const struct lock_chk_site *site
)

spin_lock_raw_full

void spin_lock_raw_full(
    struct spinlock *lock,
    enum lock_subclass sc,
    const struct lock_chk_site *site
)

spin_unlock_raw_full

void spin_unlock_raw_full(
    struct spinlock *lock,
    const struct lock_chk_site *site
)

spin_unlock_full

void spin_unlock_full(
    struct spinlock *lock,
    enum irql old,
    const struct lock_chk_site *site
)

spin_lock_full

irql spin_lock_full(
    struct spinlock *lock,
    enum lock_subclass sc,
    const struct lock_chk_site *site
)

spin_lock_high_full

irql spin_lock_high_full(
    struct spinlock *lock,
    enum lock_subclass sc,
    const struct lock_chk_site *site
)

spin_trylock_full

cc_warn_unused_result bool spin_trylock_full(
    struct spinlock *lock,
    enum irql *out,
    enum lock_subclass sc,
    const struct lock_chk_site *site
)

spin_trylock_high_full

cc_warn_unused_result bool spin_trylock_high_full(
    struct spinlock *lock,
    enum irql *out,
    enum lock_subclass sc,
    const struct lock_chk_site *site
)

spinlock_locked

bool spinlock_locked(struct spinlock *lock)

spinlock_set_chk_flags

void spinlock_set_chk_flags(
    struct spinlock *lock,
    enum lock_chk_flags flags
)

spinlock_reinit_chk

void spinlock_reinit_chk(
    struct spinlock *lock,
    const struct lock_chk_class *class,
    enum lock_chk_flags flags
)

spinlock_assert_held_full

void spinlock_assert_held_full(
    struct spinlock *lock,
    const struct lock_chk_site *site
)

spinlock_assert_not_held_full

void spinlock_assert_not_held_full(
    struct spinlock *lock,
    const struct lock_chk_site *site
)

spinlock_guard_assume

void spinlock_guard_assume(struct spinlock *lock)

spinlock_guard_exit

void spinlock_guard_exit(struct spinlock_guard *g)

spin_guard_enter

struct spinlock_guard spin_guard_enter(struct spinlock *lock)

spin_guard_high_enter

struct spinlock_guard spin_guard_high_enter(struct spinlock *lock)

spin_guard_high_subclass_enter

struct spinlock_guard spin_guard_high_subclass_enter(
    struct spinlock *lock,
    enum lock_subclass sc
)

spin_guard_subclass_enter

struct spinlock_guard spin_guard_subclass_enter(
    struct spinlock *lock,
    enum lock_subclass sc
)

Variables

raw

struct raw_spinlock raw

chk

struct lock_chk_lock chk

irq_usage

atomic(enum lock_op_flags) irq_usage

Macros

SPINLOCK_INIT

#define SPINLOCK_INIT SPINLOCK_INIT_CHK(NULL, LOCK_CHKD_FULL)

SPINLOCK_DEFINE

#define SPINLOCK_DEFINE(id)

Expansion

#define SPINLOCK_DEFINE(id) struct spinlock id = SPINLOCK_INIT

SPINLOCK_DEFINE_CHK

#define SPINLOCK_DEFINE_CHK(id, class_, flags_)

Expansion

#define SPINLOCK_DEFINE_CHK(id, class_, flags_) \
    struct spinlock id = SPINLOCK_INIT_CHK((class_), (flags_))

spinlock_init_1

#define spinlock_init_1(lock_)

Expansion

#define spinlock_init_1(lock_) spinlock_init_2(lock_, LOCK_CHKD_FULL)

spinlock_init_2

#define spinlock_init_2(lock_, flags_)

Expansion

#define spinlock_init_2(lock_, flags_) \
    spinlock_init_auto_internal((lock_), (flags_))

spinlock_init

#define spinlock_init(...)

Expansion

#define spinlock_init(...) PP_CALL(spinlock_init, __VA_ARGS__)

__SPINLOCK_SHALLOW_VALUE_INIT

#define __SPINLOCK_SHALLOW_VALUE_INIT

Expansion

#define __SPINLOCK_SHALLOW_VALUE_INIT \
    , .irq_usage = ATOMIC_VAR_INIT(LOCK_OP_IRQ_NONE)

__SPINLOCK_LOCK_CHK_VALUE_INIT

#define __SPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_)

Expansion

#define __SPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_) \
    , .chk = LOCK_CHK_LOCK_VALUE_INIT((class_), (flags_))

SPINLOCK_INIT_CHK

#define SPINLOCK_INIT_CHK(class_, flags_)

Expansion

#define SPINLOCK_INIT_CHK(class_, flags_) \
    ((struct spinlock) { \
        .raw = RAW_SPINLOCK_INIT __SPINLOCK_LOCK_CHK_VALUE_INIT( \
            (class_), (flags_)) __SPINLOCK_SHALLOW_VALUE_INIT})

spinlock_init_chk

#define spinlock_init_chk(lock_, class_, flags_)

Expansion

#define spinlock_init_chk(lock_, class_, flags_) \
    spinlock_init_chk_full((lock_), (class_), (flags_))

spinlock_init_auto_internal

#define spinlock_init_auto_internal(lock_, flags_)

Expansion

#define spinlock_init_auto_internal(lock_, flags_) \
    do { \
        static const struct lock_chk_class __auto_class = { \
            .name = #lock_, \
            .file = __RELFILE__, \
            .line = __LINE__, \
        }; \
        spinlock_init_chk_full((lock_), &__auto_class, (flags_)); \
    } while (0)

__SPINLOCK_SHALLOW_VALUE_INIT

#define __SPINLOCK_SHALLOW_VALUE_INIT

__SPINLOCK_LOCK_CHK_VALUE_INIT

#define __SPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_)

Expansion

#define __SPINLOCK_LOCK_CHK_VALUE_INIT(class_, flags_)

SPINLOCK_INIT_CHK

#define SPINLOCK_INIT_CHK(class_, flags_)

Expansion

#define SPINLOCK_INIT_CHK(class_, flags_) \
    ((struct spinlock) {.raw = RAW_SPINLOCK_INIT})

spinlock_init_chk

#define spinlock_init_chk(lock_, class_, flags_)

Expansion

#define spinlock_init_chk(lock_, class_, flags_) \
    spinlock_init_chk_full((lock_), NULL, LOCK_UNCHKD)

spinlock_init_auto_internal

#define spinlock_init_auto_internal(lock_, flags_)

Expansion

#define spinlock_init_auto_internal(lock_, flags_) \
    spinlock_init_chk_full((lock_), NULL, LOCK_UNCHKD)

spin_lock

#define spin_lock(lock_)

Expansion

#define spin_lock(lock_) \
    spin_lock_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())

spin_lock_subclass

#define spin_lock_subclass(lock_, subclass_)

Expansion

#define spin_lock_subclass(lock_, subclass_) \
    spin_lock_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())

spin_lock_high

#define spin_lock_high(lock_)

Expansion

#define spin_lock_high(lock_) \
    spin_lock_high_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())

spin_lock_high_subclass

#define spin_lock_high_subclass(lock_, subclass_)

Expansion

#define spin_lock_high_subclass(lock_, subclass_) \
    spin_lock_high_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())

spin_trylock

#define spin_trylock(lock_, out_)

Expansion

#define spin_trylock(lock_, out_) \
    spin_trylock_full((lock_), (out_), LOCK_SUBCLASS_PRIMARY, \
                      LOCK_CHK_SITE_HERE())

spin_trylock_subclass

#define spin_trylock_subclass(lock_, out_, subclass_)

Expansion

#define spin_trylock_subclass(lock_, out_, subclass_) \
    spin_trylock_full((lock_), (out_), (subclass_), LOCK_CHK_SITE_HERE())

spin_trylock_high

#define spin_trylock_high(lock_, out_)

Expansion

#define spin_trylock_high(lock_, out_) \
    spin_trylock_high_full((lock_), (out_), LOCK_SUBCLASS_PRIMARY, \
                           LOCK_CHK_SITE_HERE())

spin_trylock_high_subclass

#define spin_trylock_high_subclass(lock_, out_, subclass_)

Expansion

#define spin_trylock_high_subclass(lock_, out_, subclass_) \
    spin_trylock_high_full((lock_), (out_) (subclass_), LOCK_CHK_SITE_HERE())

spin_unlock

#define spin_unlock(lock_, old_)

Expansion

#define spin_unlock(lock_, old_) \
    spin_unlock_full((lock_), (old_), LOCK_CHK_SITE_HERE())

spin_lock_raw

#define spin_lock_raw(lock_)

Expansion

#define spin_lock_raw(lock_) \
    spin_lock_raw_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())

spin_lock_raw_subclass

#define spin_lock_raw_subclass(lock_)

Expansion

#define spin_lock_raw_subclass(lock_) \
    spin_lock_raw_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())

spin_trylock_raw

#define spin_trylock_raw(lock_)

Expansion

#define spin_trylock_raw(lock_) \
    spin_trylock_raw_full((lock_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())

spin_trylock_raw_subclass

#define spin_trylock_raw_subclass(lock_)

Expansion

#define spin_trylock_raw_subclass(lock_) \
    spin_trylock_raw_full((lock_), (subclass_), LOCK_CHK_SITE_HERE())

spin_unlock_raw

#define spin_unlock_raw(lock_)

Expansion

#define spin_unlock_raw(lock_) \
    spin_unlock_raw_full((lock_), LOCK_CHK_SITE_HERE())

SPINLOCK_ASSERT_HELD_RAW

#define SPINLOCK_ASSERT_HELD_RAW(l)

A raw check to make sure it is locked, but could be by anyone

Expansion

#define SPINLOCK_ASSERT_HELD_RAW(l) \
    enforce(spinlock_locked(l), "spinlock not locked")

SPINLOCK_ASSERT_HELD

#define SPINLOCK_ASSERT_HELD(l)

Expansion

#define SPINLOCK_ASSERT_HELD(l) \
    spinlock_assert_held_full((l), LOCK_CHK_SITE_HERE())

SPINLOCK_ASSERT_NOT_HELD

#define SPINLOCK_ASSERT_NOT_HELD(l)

Expansion

#define SPINLOCK_ASSERT_NOT_HELD(l) \
    spinlock_assert_not_held_full((l), LOCK_CHK_SITE_HERE())

spin_guard

#define spin_guard(lock_)

Expansion

#define spin_guard(lock_) \
    cc_cleanup(spinlock_guard_exit) struct spinlock_guard PP_CONCAT( \
        spin_guard_, __COUNTER__) = \
        (spinlock_guard_assume(lock_), spin_guard_enter(lock_))

spin_guard_subclass

#define spin_guard_subclass(lock_, subclass_)

Expansion

#define spin_guard_subclass(lock_, subclass_) \
    cc_cleanup(spinlock_guard_exit) struct spinlock_guard PP_CONCAT( \
        spin_guard_, __COUNTER__) = \
        (spinlock_guard_assume(lock_), \
         spin_guard_subclass_enter((lock_), (subclass_)))

spin_guard_high

#define spin_guard_high(lock_)

Expansion

#define spin_guard_high(lock_) \
    cc_cleanup(spinlock_guard_exit) struct spinlock_guard PP_CONCAT( \
        spin_guard_high_, __COUNTER__) = \
        (spinlock_guard_assume(lock_), spin_guard_high_enter(lock_))

spin_guard_high_subclass

#define spin_guard_high_subclass(lock_, subclass_)

Expansion

#define spin_guard_high_subclass(lock_, subclass_) \
    cc_cleanup(spinlock_guard_exit) struct spinlock_guard PP_CONCAT( \
        spin_guard_high_, __COUNTER__) = \
        (spinlock_guard_assume(lock_), \
         spin_guard_high_enter((lock_), (subclass_)))