1/* @title: Reader writer lock */
2#pragma once
3#include <atomic.h>
4#include <compiler/core.h>
5#include <math/bit.h>
6#include <sch/irql.h>
7#include <stddef.h>
8#include <stdint.h>
9#include <sync/lock_chk_types.h>
10#include <sync/lock_general.h>
11#include <sync/rwlock_types.h>
12#include <thread/thread_types.h>
13#include <tsa.h>
14
15/* rwlock: pointer sized shared reader writer lock
16 *
17 * note: the writer bit leads to two separate
18 * possible encodings for the rest of the bits in the lock word.
19 *
20 * ┌─────────────────────────┐
21 * Bits │ .... .... .... 3..0 │
22 * Use when w = 1 │ %%%% %%%% %%%% %ppw │
23 * Use when w = 0 │ RRRR RRRR RRRW appw │
24 * └─────────────────────────┘
25 *
26 *
27 * w - writer bit -> a writer holds the lock
28 * a - waiter bit -> threads are waiting on the lock
29 * W - writer want -> a writer wants the lock
30 * R - reader count -> used to store the number of readers
31 * |
32 * | The way the reader count actually works,
33 * | though, is that it uses the base priority
34 * | of the reader to create multiple separate
35 * | counters that are all the same few bits,
36 * | so writers have an exact snapshot of
37 * | who is reading the lock.
38 * |
39 *
40 * m - max prio class -> panic if reader > this
41 *
42 * %%%% - pointer to owner thread
43 *
44 */
45struct TSA_CAPABILITY("rwlock") rwlock {
46 atomic_uintptr_t lock_word;
47
48#ifdef DEBUG_LOCK_CHK
49 struct lock_chk_lock chk;
50#endif /* DEBUG_LOCK_CHK */
51};
52
53void rw_lock_full(struct rwlock *lock, enum rwlock_acquire_type type,
54 enum lock_subclass subclass, const struct lock_chk_site *site)
55 TSA_ACQUIRES(lock) TSA_MAY_SLEEP;
56void rw_unlock_full(struct rwlock *lock, const struct lock_chk_site *site)
57 TSA_RELEASES(lock);
58void rwlock_init_chk_full(struct rwlock *lock, enum thread_prio_class ceiling,
59 const struct lock_chk_class *class,
60 enum lock_chk_flags flags);
61void rwlock_set_chk_flags(struct rwlock *lock, enum lock_chk_flags flags);
62void rwlock_reinit_chk(struct rwlock *lock, enum thread_prio_class ceiling,
63 const struct lock_chk_class *class,
64 enum lock_chk_flags flags);
65bool rwlock_locked(struct rwlock *lock, enum rwlock_acquire_type type);
66void rwlock_assert_held_full(struct rwlock *lock, enum rwlock_acquire_type type,
67 const struct lock_chk_site *site)
68 TSA_ASSERT_CAPABILITY(lock);
69void rwlock_assert_not_held_full(struct rwlock *lock,
70 const struct lock_chk_site *site);
71
72#define RWLOCK_ASSERT_HELD(lock, type) \
73 rwlock_assert_held_full((lock), (type), LOCK_CHK_SITE_HERE())
74#define RWLOCK_ASSERT_READ(lock) RWLOCK_ASSERT_HELD((lock), RWLOCK_READ)
75#define RWLOCK_ASSERT_WRITE(lock) RWLOCK_ASSERT_HELD((lock), RWLOCK_WRITE)
76#define RWLOCK_ASSERT_NOT_HELD(lock) \
77 rwlock_assert_not_held_full((lock), LOCK_CHK_SITE_HERE())
78
79#define RWLOCK_PRIO_CEIL_SHIFT 1
80
81#ifdef DEBUG_LOCK_CHK
82
83#define RWLOCK_INIT_CHK(ceil_, class_, flags_) \
84 ((struct rwlock) { \
85 .lock_word = ATOMIC_VAR_INIT((ceil_) << RWLOCK_PRIO_CEIL_SHIFT), \
86 .chk = LOCK_CHK_LOCK_VALUE_INIT((class_), (flags_)), \
87 })
88
89#define rwlock_init_chk(lock_, ceil_, class_, flags_) \
90 rwlock_init_chk_full((lock_), (ceil_), (class_), (flags_))
91#define rwlock_init_auto_internal(lock_, ceil_, flags_) \
92 do { \
93 static const struct lock_chk_class __auto_class = { \
94 .name = #lock_, \
95 .file = __RELFILE__, \
96 .line = __LINE__, \
97 }; \
98 rwlock_init_chk_full((lock_), (ceil_), &__auto_class, (flags_)); \
99 } while (0)
100
101#else /* !defined(DEBUG_LOCK_CHK) */
102
103#define RWLOCK_INIT_CHK(ceil_, class_, flags_) \
104 ((struct rwlock) { \
105 .lock_word = ATOMIC_VAR_INIT((ceil_) << RWLOCK_PRIO_CEIL_SHIFT), \
106 })
107
108#define rwlock_init_chk(lock_, ceil_, class_, flags_) \
109 rwlock_init_chk_full((lock_), (ceil_), NULL, LOCK_UNCHKD)
110#define rwlock_init_auto_internal(lock_, ceil_, flags_) \
111 rwlock_init_chk_full((lock_), (ceil_), NULL, LOCK_UNCHKD)
112
113#endif /* DEBUG_LOCK_CHK */
114
115#define RWLOCK_INIT(ceil_) RWLOCK_INIT_CHK((ceil_), NULL, LOCK_CHKD_FULL)
116#define RWLOCK_DEFINE(id, ceil_) struct rwlock id = RWLOCK_INIT(ceil_)
117#define RWLOCK_DEFINE_CHK(id, ceil_, class_, flags_) \
118 struct rwlock id = RWLOCK_INIT_CHK((ceil_), (class_), (flags_))
119
120#define rwlock_init_2(lock_, ceil_) rwlock_init_3(lock_, ceil_, LOCK_CHKD_FULL)
121#define rwlock_init_3(lock_, ceil_, flags_) \
122 rwlock_init_auto_internal((lock_), (ceil_), (flags_))
123#define rwlock_init(...) PP_CALL(rwlock_init, __VA_ARGS__)
124
125#define rw_lock(lock_, type_) \
126 rw_lock_full((lock_), (type_), LOCK_SUBCLASS_PRIMARY, LOCK_CHK_SITE_HERE())
127#define rw_read_lock(lock_) \
128 rw_lock_full((lock_), RWLOCK_READ, LOCK_SUBCLASS_PRIMARY, \
129 LOCK_CHK_SITE_HERE())
130#define rw_write_lock(lock_) \
131 rw_lock_full((lock_), RWLOCK_WRITE, LOCK_SUBCLASS_PRIMARY, \
132 LOCK_CHK_SITE_HERE())
133#define rw_read_lock_subclass(lock_, subclass_) \
134 rw_lock_full((lock_), RWLOCK_READ, (subclass_), LOCK_CHK_SITE_HERE())
135#define rw_write_lock_subclass(lock_, subclass_) \
136 rw_lock_full((lock_), RWLOCK_WRITE, (subclass_), LOCK_CHK_SITE_HERE())
137#define rw_unlock(lock_) rw_unlock_full((lock_), LOCK_CHK_SITE_HERE())
138
139static inline cc_always_inline
140cc_unused void rwlock_guard_exit(struct rwlock **lock) TSA_NO_ANALYSIS {
141 if (*lock)
142 rw_unlock(*lock);
143}
144
145static inline cc_always_inline cc_unused void
146rwlock_guard_assume(struct rwlock *lock) TSA_ASSERT_CAPABILITY(lock) {
147 cc_ignore(lock);
148}
149
150static inline cc_always_inline cc_unused struct rwlock *
151rw_read_guard_enter(struct rwlock *lock) TSA_MAY_SLEEP TSA_NO_ANALYSIS {
152 rw_read_lock(lock);
153 return lock;
154}
155
156static inline cc_always_inline cc_unused struct rwlock *
157rw_write_guard_enter(struct rwlock *lock) TSA_MAY_SLEEP TSA_NO_ANALYSIS {
158 rw_write_lock(lock);
159 return lock;
160}
161
162#define rw_read_guard(lock_) \
163 cc_cleanup(rwlock_guard_exit) struct rwlock *PP_CONCAT(rw_read_guard_, \
164 __COUNTER__) = \
165 (rwlock_guard_assume(lock_), rw_read_guard_enter(lock_))
166
167#define rw_write_guard(lock_) \
168 cc_cleanup(rwlock_guard_exit) struct rwlock *PP_CONCAT(rw_write_guard_, \
169 __COUNTER__) = \
170 (rwlock_guard_assume(lock_), rw_write_guard_enter(lock_))
171