1/* @title: Per-CPU dynamic objects */
2#pragma once
3#include <compiler/core.h>
4#include <global.h>
5#include <linker/symbols.h>
6#include <smp/core.h>
7#include <stddef.h>
8#include <stdint.h>
9
10struct percpu_descriptor;
11typedef void (*percpu_descriptor_constructor)(void *, size_t);
12
13struct percpu_descriptor {
14 const char *name;
15 size_t size;
16 size_t align;
17 void **percpu_ptrs;
18 percpu_descriptor_constructor constructor;
19 atomic_bool ready;
20};
21
22LINKER_SECTION_EXTERN(struct percpu_descriptor, percpu_desc);
23
24#define PERCPU_DEFINE_3(type_, name_, ctor_) \
25 static typeof(type_) __percpu_##name_ cc_unused; \
26 static struct percpu_descriptor __percpu_desc_##name_; \
27 static void __percpu_ctor_##name_(void *inst, size_t cpu) { \
28 void (*const __typed_ctor)(typeof(type_) *, size_t) = (ctor_); \
29 if (__typed_ctor != NULL) \
30 __typed_ctor((typeof(type_) *) inst, cpu); \
31 if (cpu == global.core_count - 1) \
32 atomic_store(&__percpu_desc_##name_.ready, true); \
33 } \
34 static LINKER_SECTION_OBJECT(struct percpu_descriptor, percpu_desc) \
35 __percpu_desc_##name_ = { \
36 .name = #name_, \
37 .size = sizeof(typeof(type_)), \
38 .align = cc_alignof(typeof(type_)), \
39 .percpu_ptrs = NULL, \
40 .constructor = __percpu_ctor_##name_, \
41 .ready = false, \
42 }; \
43 static struct percpu_descriptor *const __percpu_desc_ref_##name_ \
44 cc_unused = &__percpu_desc_##name_
45
46#define PERCPU_DEFINE_2(type_, name_) PERCPU_DEFINE_3(type_, name_, NULL)
47#define PERCPU_DEFINE(...) PP_CALL(PERCPU_DEFINE, __VA_ARGS__)
48
49#define PERCPU_EXPORT_AS(sym_name, name) \
50 extern struct percpu_descriptor __percpu_desc_sym_##sym_name \
51 cc_alias(__percpu_desc_##name) cc_used
52
53#define PERCPU_EXPORT(name) PERCPU_EXPORT_AS(name, name)
54
55#define PERCPU_EXTERN_AS(type, name, sym_name) \
56 extern struct percpu_descriptor __percpu_desc_sym_##sym_name; \
57 static typeof(type) __percpu_##name cc_unused; \
58 static struct percpu_descriptor *const __percpu_desc_ref_##name \
59 cc_unused = &__percpu_desc_sym_##sym_name
60
61#define PERCPU_EXTERN(type, name) PERCPU_EXTERN_AS(type, name, name)
62
63static inline
64 cc_pure bool percpu_desc_ready(const struct percpu_descriptor *desc) {
65 return atomic_load(&desc->ready);
66}
67
68#define PERCPU(name) &(__percpu_##name)
69#define PERCPU_READY(name) (atomic_load(&(__percpu_desc_ref_##name)->ready))
70
71#define PERCPU_PTR_FOR(name, cpu) \
72 ({ \
73 assume(percpu_desc_ready(__percpu_desc_ref_##name)); \
74 ((typeof(__percpu_##name) *) (__percpu_desc_ref_##name) \
75 ->percpu_ptrs[cpu]); \
76 })
77
78#define PERCPU_READ_FOR(name, cpu) \
79 (*((typeof(__percpu_##name) *) PERCPU_PTR_FOR(name, cpu)))
80
81#define PERCPU_PTR(clr, name) PERCPU_PTR_FOR(name, smp_id(clr))
82#define PERCPU_READ(clr, name) \
83 (*((typeof(__percpu_##name) *) PERCPU_PTR(clr, name)))
84
85#define PERCPU_WRITE(clr, name, val) (PERCPU_READ(clr, name) = (val))
86
87#define percpu_for_each_internal_3(var, cpu, name) \
88 for (cpu = 0; cpu < global.core_count; cpu++) \
89 for (var = PERCPU_PTR_FOR(name, cpu); var != NULL; var = NULL)
90
91#define percpu_for_each_internal_2(var, name) \
92 for (cpu_id_t __percpu_idx = 0; __percpu_idx < global.core_count; \
93 __percpu_idx++) \
94 for (var = PERCPU_PTR_FOR(name, __percpu_idx); var != NULL; var = NULL)
95
96#define percpu_for_each(...) PP_CALL(percpu_for_each_internal, __VA_ARGS__)
97
98void percpu_obj_init(void);
99
100void **percpu_alloc_array(size_t size);
101