1/* @title: Per-CPU dynamic objects */
2#pragma once
3#include <compiler.h>
4#include <global.h>
5#include <linker/symbols.h>
6#include <smp/core.h>
7#include <stdatomic.h>
8#include <stddef.h>
9#include <stdint.h>
10
11struct percpu_descriptor;
12typedef void (*percpu_descriptor_constructor)(void *, size_t);
13
14struct percpu_descriptor {
15 const char *name;
16 size_t size;
17 size_t align;
18 void **percpu_ptrs;
19 percpu_descriptor_constructor constructor;
20 atomic_bool ready;
21};
22
23LINKER_SECTION_DEFINE(struct percpu_descriptor, percpu_desc);
24
25#define PERCPU_DECLARE(__n, __type, __ctor) \
26 static typeof(__type) __percpu_##__n __unused; \
27 static struct percpu_descriptor __percpu_desc_##__n; \
28 static void __percpu_ctor_##__n(void *inst, size_t cpu) { \
29 void (*const __typed_ctor)(typeof(__type) *, size_t) = (__ctor); \
30 if (__typed_ctor != NULL) \
31 __typed_ctor((typeof(__type) *) inst, cpu); \
32 if (cpu == global.core_count - 1) \
33 atomic_store(&__percpu_desc_##__n.ready, true); \
34 } \
35 static LINKER_SECTION_OBJECT(struct percpu_descriptor, percpu_desc) \
36 __percpu_desc_##__n = { \
37 .name = #__n, \
38 .size = sizeof(typeof(__type)), \
39 .align = _Alignof(typeof(__type)), \
40 .percpu_ptrs = NULL, \
41 .constructor = __percpu_ctor_##__n, \
42 .ready = false, \
43 }; \
44 static struct percpu_descriptor *const __percpu_desc_ref_##__n __unused = \
45 &__percpu_desc_##__n
46
47#define PERCPU_EXPORT_AS(sym_name, name) \
48 extern struct percpu_descriptor __percpu_desc_sym_##sym_name \
49 __attribute__((alias("__percpu_desc_" #name), used))
50
51#define PERCPU_EXPORT(name) PERCPU_EXPORT_AS(name, name)
52
53#define PERCPU_DEFINE_AS(name, sym_name, type) \
54 extern struct percpu_descriptor __percpu_desc_sym_##sym_name; \
55 static typeof(type) __percpu_##name __unused; \
56 static struct percpu_descriptor *const __percpu_desc_ref_##name __unused = \
57 &__percpu_desc_sym_##sym_name
58
59#define PERCPU_DEFINE(name, type) PERCPU_DEFINE_AS(name, name, type)
60
61#define PERCPU(name) &(__percpu_##name)
62#define PERCPU_READY(name) (atomic_load(&(__percpu_desc_ref_##name)->ready))
63
64#define PERCPU_PTR_FOR_CPU(name, cpu) \
65 ({ \
66 (void) kassert(PERCPU_READY(name)); \
67 ((typeof(__percpu_##name) *) (__percpu_desc_ref_##name) \
68 ->percpu_ptrs[cpu]); \
69 })
70
71#define PERCPU_READ_FOR_CPU(name, cpu) \
72 (*((typeof(__percpu_##name) *) PERCPU_PTR_FOR_CPU(name, cpu)))
73
74#define PERCPU_PTR(clr, name) PERCPU_PTR_FOR_CPU(name, smp_id(clr))
75#define PERCPU_READ(clr, name) \
76 (*((typeof(__percpu_##name) *) PERCPU_PTR(clr, name)))
77
78#define PERCPU_WRITE(clr, name, val) (PERCPU_READ(clr, name) = (val))
79
80#define percpu_for_each_internal(name, var, cpu) \
81 for (cpu_id_t cpu = 0; cpu < global.core_count; cpu++) \
82 for (var = PERCPU_PTR_FOR_CPU(name, cpu); var != NULL; var = NULL)
83
84#define percpu_for_each_internal_3(name, var, cpu) \
85 percpu_for_each_internal(name, var, cpu)
86#define percpu_for_each_internal_2(name, var) \
87 percpu_for_each_internal(name, var, __cpu)
88
89#define percpu_for_each(...) \
90 _DISPATCH(percpu_for_each_internal, PP_NARG(__VA_ARGS__))(__VA_ARGS__)
91
92void percpu_obj_init(void);
93