1#include <errno.h>
2#include <global.h>
3#include <kassert.h>
4#include <mem/alloc.h>
5#include <sync/percpu_rc.h>
6#include <sync/rcu.h>
7
8static void percpu_rc_switch_to_atomic_rcu(struct rcu_cb *cb, void *arg) {
9 (void) cb;
10 struct percpu_rc *rc = arg;
11 uintptr_t pcpu =
12 atomic_load_explicit(&rc->percpu_count_ptr, memory_order_relaxed);
13
14 int64_t *counters = PERCPU_RC_PTR(pcpu);
15 int64_t sum = 0;
16
17 if (counters) {
18 for (size_t i = 0; i < global.core_count; i++) {
19 sum += counters[i];
20 counters[i] = 0;
21 }
22
23 if (!rc->allow_reinit) {
24 kfree(counters);
25 atomic_store_explicit(&rc->percpu_count_ptr,
26 PERCPU_RC_DEAD | PERCPU_RC_ATOMIC,
27 memory_order_release);
28 }
29 }
30
31 /* Add pcpu sum and remove bias + initial kill ref, == 0, release */
32 int64_t adjustment = sum - (PERCPU_COUNT_BIAS + 1);
33 int64_t prev =
34 atomic_fetch_add_explicit(&rc->count, adjustment, memory_order_acq_rel);
35
36 if (prev + adjustment == 0) {
37 if (rc->release)
38 rc->release(rc);
39 }
40}
41
42int percpu_rc_init(struct percpu_rc *rc, percpu_rc_release_fn release,
43 enum percpu_rc_flags flags) {
44 rc->release = release;
45 rc->allow_reinit = (flags & PERCPU_RC_ALLOW_REINIT) != 0;
46
47 if (flags & PERCPU_RC_INIT_ATOMIC) {
48 atomic_store_explicit(&rc->count, 1, memory_order_relaxed);
49 atomic_store_explicit(&rc->percpu_count_ptr, PERCPU_RC_ATOMIC,
50 memory_order_relaxed);
51 return 0;
52 }
53
54 size_t size = sizeof(int64_t) * global.core_count;
55 int64_t *counters = kmalloc(size, ALLOC_FLAGS_ZERO);
56 if (!counters)
57 return ERR_NO_MEM;
58
59 /* Init with PERCPU_COUNT_BIAS + 1 so early put()s during kill don't
60 * prematurely trigger release */
61 atomic_store_explicit(&rc->count, 1 + PERCPU_COUNT_BIAS,
62 memory_order_relaxed);
63 atomic_store_explicit(&rc->percpu_count_ptr, (uintptr_t) counters,
64 memory_order_release);
65
66 return 0;
67}
68
69void percpu_rc_destroy(struct percpu_rc *ref) {
70 uintptr_t pcpu =
71 atomic_load_explicit(&ref->percpu_count_ptr, memory_order_relaxed);
72 int64_t *counters = PERCPU_RC_PTR(pcpu);
73
74 if (counters) {
75 kfree(counters);
76 atomic_store_explicit(&ref->percpu_count_ptr,
77 PERCPU_RC_DEAD | PERCPU_RC_ATOMIC,
78 memory_order_relaxed);
79 }
80}
81
82void percpu_rc_kill(struct percpu_rc *ref) {
83 uintptr_t pcpu =
84 atomic_load_explicit(&ref->percpu_count_ptr, memory_order_relaxed);
85
86 if (pcpu & PERCPU_RC_DEAD)
87 return;
88
89 atomic_fetch_or_explicit(&ref->percpu_count_ptr,
90 PERCPU_RC_DEAD | PERCPU_RC_ATOMIC,
91 memory_order_release);
92
93 rcu_defer(cb: &ref->rcu, fn: percpu_rc_switch_to_atomic_rcu, arg: ref);
94}
95
96void percpu_rc_reinit(struct percpu_rc *rc) {
97 kassert(rc->allow_reinit,
98 "percpu_rc was not initialized with allow_reinit");
99
100 uintptr_t pcpu =
101 atomic_load_explicit(&rc->percpu_count_ptr, memory_order_relaxed);
102 int64_t *counters = PERCPU_RC_PTR(pcpu);
103 kassert(counters, "percpu_rc counters missing during reinit");
104
105 atomic_store_explicit(&rc->count, 1 + PERCPU_COUNT_BIAS,
106 memory_order_relaxed);
107 atomic_store_explicit(&rc->percpu_count_ptr, (uintptr_t) counters,
108 memory_order_release);
109}
110
111void percpu_rc_resurrect(struct percpu_rc *rc) {
112 atomic_store_explicit(&rc->count, 1, memory_order_relaxed);
113}
114
115int64_t percpu_rc_read(struct percpu_rc *rc) {
116 uintptr_t pcpu =
117 atomic_load_explicit(&rc->percpu_count_ptr, memory_order_relaxed);
118
119 if (percpu_rc_is_percpu(pcpu)) {
120 int64_t sum = 0;
121 int64_t *counters = PERCPU_RC_PTR(pcpu);
122 if (counters) {
123 for (size_t i = 0; i < global.core_count; i++)
124 sum += counters[i];
125 }
126 return (atomic_load_explicit(&rc->count, memory_order_relaxed) -
127 PERCPU_COUNT_BIAS) +
128 sum;
129 }
130
131 return atomic_load_explicit(&rc->count, memory_order_relaxed);
132}
133