Sequence Lock

include/sync/seqlock.h View source View on GitHub

The naming here is rather… unpleasant, so it’s worth specifying upfront:

seqcount_ is for all the struct seqcount functions, seq_ and seqlock_ are for the struct seqlock functions, we keep it this way so that function signatures don’t explode in length, and to mirror the general naming conventions of the lock primitives with lock in their name (see spinlock.h, rwlock.h)

(begin_|end_)(read|write)(_raw)(_high)(_retry)

is the syntax ordering, broadly

Structs

seqcount

struct seqcount (1 field)

Sequence counters for lock free reader synchronization with serialized writers

Odd sequence indicates an in progress write, even count indicates quiescent data.

TODO

the thread API and thread.c uses what is effectively a sequence counter, just not with this API. someday we can change it over

§ atomic_uint32_t sequence

seqlock

struct seqlock (2 fields)

Sequence Lock (seqlock)

Combines a sequence counter with a spinlock to serialize writers

§ struct seqcount seqcount

§ struct spinlock lock

seqlock_write_guard

struct seqlock_write_guard (2 fields)

§ struct seqlock *sl

§ enum irql irql

seqlock_write_raw_guard

struct seqlock_write_raw_guard (1 field)

§ struct seqlock *sl

Type Aliases

seqcount_t

typedef struct seqcount seqcount_t

seqlock_t

typedef struct seqlock seqlock_t

Functions

seqcount_init

void seqcount_init(struct seqcount *s)

seqcount_read_raw

uint32_t seqcount_read_raw(const struct seqcount *s)

seqcount_begin_read_raw

uint32_t seqcount_begin_read_raw(const struct seqcount *s)

seqcount_begin_read

uint32_t seqcount_begin_read(const struct seqcount *s)

Wait for any active writer to complete and return the sequence with acquire barrier

seqcount_read_retry

bool seqcount_read_retry(const struct seqcount *s, uint32_t start)

Check for change since start

seqcount_begin_write

void seqcount_begin_write(struct seqcount *s)

even to odd with wmb

NOTE

the separate rlx load and store here is intentional: atomic_fetch_add is RMW, and unneeded when there is only one writer

seqcount_end_write

void seqcount_end_write(struct seqcount *s)

odd to even with wmb

seqlock_init_chk_full

void seqlock_init_chk_full(
    struct seqlock *sl,
    const struct lock_chk_class *class,
    enum lock_chk_flags flags
)

seq_begin_read

uint32_t seq_begin_read(const struct seqlock *sl)

seq_read_retry

bool seq_read_retry(const struct seqlock *sl, uint32_t start)

seq_begin_read_raw

uint32_t seq_begin_read_raw(const struct seqlock *sl)

seq_read_raw

uint32_t seq_read_raw(const struct seqlock *sl)

seq_write_lock

irql seq_write_lock(struct seqlock *sl)

seq_write_lock_high

irql seq_write_lock_high(struct seqlock *sl)

Writer APIs

seq_write_unlock

void seq_write_unlock(struct seqlock *sl, enum irql old)

seq_write_lock_raw

void seq_write_lock_raw(struct seqlock *sl)

Raw, no IRQL

seq_write_unlock_raw

void seq_write_unlock_raw(struct seqlock *sl)

seq_try_write_lock

cc_warn_unused_result bool seq_try_write_lock(
    struct seqlock *sl,
    enum irql *out
)

Trylock

seq_try_write_lock_high

cc_warn_unused_result bool seq_try_write_lock_high(
    struct seqlock *sl,
    enum irql *out
)

seq_try_write_lock_raw

cc_warn_unused_result bool seq_try_write_lock_raw(struct seqlock *sl)

seqlock_is_writing

bool seqlock_is_writing(const struct seqlock *sl)

Query

seqlock_held

bool seqlock_held(const struct seqlock *sl)

seqcount_write_guard_exit

void seqcount_write_guard_exit(struct seqcount **s)

seqcount_write_guard_enter

struct seqcount * seqcount_write_guard_enter(struct seqcount *s)

seqlock_write_guard_assume

void seqlock_write_guard_assume(struct seqlock *sl)

seqlock_write_guard_exit

void seqlock_write_guard_exit(struct seqlock_write_guard *g)

seqlock_write_guard_enter

struct seqlock_write_guard seqlock_write_guard_enter(struct seqlock *sl)

seqlock_write_high_guard_enter

struct seqlock_write_guard seqlock_write_high_guard_enter(struct seqlock *sl)

seqlock_write_raw_guard_exit

void seqlock_write_raw_guard_exit(struct seqlock_write_raw_guard *g)

seqlock_write_raw_guard_enter

struct seqlock_write_raw_guard seqlock_write_raw_guard_enter(struct seqlock *sl)

Macros

SEQCOUNT_INIT

#define SEQCOUNT_INIT

Expansion

#define SEQCOUNT_INIT \
    (struct seqcount) { \
        .sequence = ATOMIC_VAR_INIT(0) \
    }

SEQLOCK_INIT_CHK

#define SEQLOCK_INIT_CHK(class_, flags_)

Expansion

#define SEQLOCK_INIT_CHK(class_, flags_) \
    (struct seqlock) { \
        .seqcount = SEQCOUNT_INIT, \
        .lock = SPINLOCK_INIT_CHK((class_), (flags_)) \
    }

SEQLOCK_INIT

#define SEQLOCK_INIT SEQLOCK_INIT_CHK(NULL, LOCK_CHKD_FULL)

SEQLOCK_DEFINE

#define SEQLOCK_DEFINE(id)

Expansion

#define SEQLOCK_DEFINE(id) struct seqlock id = SEQLOCK_INIT

SEQLOCK_DEFINE_CHK

#define SEQLOCK_DEFINE_CHK(id, class_, flags_)

Expansion

#define SEQLOCK_DEFINE_CHK(id, class_, flags_) \
    struct seqlock id = SEQLOCK_INIT_CHK((class_), (flags_))

seqlock_init_chk

#define seqlock_init_chk(sl_, class_, flags_)

Expansion

#define seqlock_init_chk(sl_, class_, flags_) \
    seqlock_init_chk_full((sl_), (class_), (flags_))

seqlock_init_auto_internal

#define seqlock_init_auto_internal(sl_, flags_)

Expansion

#define seqlock_init_auto_internal(sl_, flags_) \
    do { \
        static const struct lock_chk_class __auto_class = { \
            .name = #sl_, \
            .file = __RELFILE__, \
            .line = __LINE__, \
        }; \
        seqlock_init_chk_full((sl_), &__auto_class, (flags_)); \
    } while (0)

seqlock_init_chk

#define seqlock_init_chk(sl_, class_, flags_)

Expansion

#define seqlock_init_chk(sl_, class_, flags_) \
    seqlock_init_chk_full((sl_), NULL, LOCK_UNCHKD)

seqlock_init_auto_internal

#define seqlock_init_auto_internal(sl_, flags_)

Expansion

#define seqlock_init_auto_internal(sl_, flags_) \
    seqlock_init_chk_full((sl_), NULL, LOCK_UNCHKD)

seqlock_init_1

#define seqlock_init_1(sl_)

Expansion

#define seqlock_init_1(sl_) seqlock_init_2(sl_, LOCK_CHKD_FULL)

seqlock_init_2

#define seqlock_init_2(sl_, flags_)

Expansion

#define seqlock_init_2(sl_, flags_) seqlock_init_auto_internal((sl_), (flags_))

seqlock_init

#define seqlock_init(...)

Expansion

#define seqlock_init(...) PP_CALL(seqlock_init, __VA_ARGS__)

SEQLOCK_ASSERT_HELD

#define SEQLOCK_ASSERT_HELD(sl)

Expansion

#define SEQLOCK_ASSERT_HELD(sl) enforce(seqlock_held(sl))

seqcount_write_guard

#define seqcount_write_guard(s_)

Expansion

#define seqcount_write_guard(s_) \
    cc_cleanup(seqcount_write_guard_exit) struct seqcount *PP_CONCAT( \
        seqc_guard_, __COUNTER__) = seqcount_write_guard_enter(s_)

seq_write_guard

#define seq_write_guard(sl_)

Expansion

#define seq_write_guard(sl_) \
    cc_cleanup(seqlock_write_guard_exit) struct seqlock_write_guard PP_CONCAT( \
        seq_wguard_, __COUNTER__) = \
        (seqlock_write_guard_assume(sl_), seqlock_write_guard_enter(sl_))

seq_write_guard_high

#define seq_write_guard_high(sl_)

Expansion

#define seq_write_guard_high(sl_) \
    cc_cleanup(seqlock_write_guard_exit) struct seqlock_write_guard PP_CONCAT( \
        seq_wguard_high_, __COUNTER__) = \
        (seqlock_write_guard_assume(sl_), seqlock_write_high_guard_enter(sl_))

seq_write_guard_raw

#define seq_write_guard_raw(sl_)

Expansion

#define seq_write_guard_raw(sl_) \
    cc_cleanup(seqlock_write_raw_guard_exit) struct seqlock_write_raw_guard \
    PP_CONCAT(seq_wguard_raw_, __COUNTER__) = \
        (seqlock_write_guard_assume(sl_), seqlock_write_raw_guard_enter(sl_))