1#include <enforce.h>
2#include <err.h>
3#include <global.h>
4#include <mem/alloc.h>
5#include <sync/percpu_rc.h>
6#include <sync/rcu.h>
7
8static void percpu_rc_switch_to_atomic_rcu(struct rcu_cb *cb) {
9 cc_ignore(cb);
10 struct percpu_rc *rc = container_of(cb, struct percpu_rc, rcu);
11 uintptr_t pcpu = atomic_load_rlx(&rc->percpu_count_ptr);
12
13 atomic_int64_t *counters = PERCPU_RC_PTR(pcpu);
14 int64_t sum = 0;
15
16 if (counters) {
17 for (size_t i = 0; i < global.core_count; i++) {
18 sum += atomic_load_rlx(&counters[i]);
19 atomic_store_rlx(&counters[i], 0);
20 }
21
22 if (!rc->allow_reinit) {
23 vfree(counters);
24 atomic_store_release(&rc->percpu_count_ptr,
25 PERCPU_RC_DEAD | PERCPU_RC_ATOMIC);
26 }
27 }
28
29 /* Add pcpu sum and remove bias + initial kill ref, == 0, release */
30 int64_t adjustment = sum - (PERCPU_COUNT_BIAS + 1);
31 int64_t prev = atomic_fetch_add_acq_rel(&rc->count, adjustment);
32
33 if (prev + adjustment == 0) {
34 if (rc->release)
35 rc->release(rc);
36 }
37}
38
39int percpu_rc_init(struct percpu_rc *rc, percpu_rc_release_fn release,
40 enum percpu_rc_flags flags) {
41 rc->release = release;
42 rc->allow_reinit = (flags & PERCPU_RC_ALLOW_REINIT) != 0;
43
44 if (flags & PERCPU_RC_INIT_ATOMIC) {
45 atomic_store_rlx(&rc->count, 1);
46 atomic_store_rlx(&rc->percpu_count_ptr, PERCPU_RC_ATOMIC);
47 return 0;
48 }
49
50 size_t size = sizeof(int64_t) * global.core_count;
51 int64_t *counters = valloc(size, ALLOC_ZERO);
52 if (!counters)
53 return ERR_NO_MEM;
54
55 /* Init with PERCPU_COUNT_BIAS + 1 so early putref()s during kill don't
56 * prematurely trigger release */
57 atomic_store_rlx(&rc->count, 1 + PERCPU_COUNT_BIAS);
58 atomic_store_release(&rc->percpu_count_ptr, (uintptr_t) counters);
59
60 return 0;
61}
62
63void percpu_rc_destroy(struct percpu_rc *ref) {
64 uintptr_t pcpu = atomic_load_rlx(&ref->percpu_count_ptr);
65 atomic_int64_t *counters = PERCPU_RC_PTR(pcpu);
66
67 if (counters) {
68 vfree(counters);
69 atomic_store_rlx(&ref->percpu_count_ptr,
70 PERCPU_RC_DEAD | PERCPU_RC_ATOMIC);
71 }
72}
73
74void percpu_rc_kill(struct percpu_rc *ref) {
75 uintptr_t pcpu = atomic_load_rlx(&ref->percpu_count_ptr);
76
77 if (pcpu & PERCPU_RC_DEAD)
78 return;
79
80 atomic_fetch_or_release(&ref->percpu_count_ptr,
81 PERCPU_RC_DEAD | PERCPU_RC_ATOMIC);
82
83 rcu_defer(cb: &ref->rcu, func: percpu_rc_switch_to_atomic_rcu);
84}
85
86void percpu_rc_reinit(struct percpu_rc *rc) {
87 enforce(rc->allow_reinit,
88 "percpu_rc was not initialized with allow_reinit");
89
90 uintptr_t pcpu = atomic_load_rlx(&rc->percpu_count_ptr);
91 atomic_int64_t *counters = PERCPU_RC_PTR(pcpu);
92 enforce(counters, UNINITIALIZED,
93 "percpu_rc counters missing during reinit");
94
95 atomic_store_rlx(&rc->count, 1 + PERCPU_COUNT_BIAS);
96 atomic_store_release(&rc->percpu_count_ptr, (uintptr_t) counters);
97}
98
99void percpu_rc_resurrect(struct percpu_rc *rc) {
100 atomic_store_rlx(&rc->count, 1);
101}
102
103int64_t percpu_rc_read(struct percpu_rc *rc) {
104 uintptr_t pcpu = atomic_load_rlx(&rc->percpu_count_ptr);
105
106 if (percpu_rc_is_percpu(pcpu)) {
107 int64_t sum = 0;
108 atomic_int64_t *counters = PERCPU_RC_PTR(pcpu);
109 if (counters) {
110 for (size_t i = 0; i < global.core_count; i++)
111 sum += atomic_load_rlx(&counters[i]);
112 }
113 return (atomic_load_rlx(&rc->count) - PERCPU_COUNT_BIAS) + sum;
114 }
115
116 return atomic_load_rlx(&rc->count);
117}
118