1/* @title: Per-Node dynamic objects */
2#pragma once
3#include <compiler.h>
4#include <global.h>
5#include <linker/symbols.h>
6#include <mem/numa.h>
7#include <stddef.h>
8#include <stdint.h>
9
10struct pernode_descriptor;
11typedef void (*pernode_descriptor_constructor)(void *, size_t);
12
13struct pernode_descriptor {
14 const char *name;
15 size_t size;
16 size_t align;
17 void **pernode_ptrs;
18 pernode_descriptor_constructor constructor;
19 atomic_bool ready;
20};
21
22LINKER_SECTION_DEFINE(struct pernode_descriptor, pernode_desc);
23
24#define PERNODE_DECLARE(__n, __type, __ctor) \
25 static typeof(__type) __pernode_##__n __unused; \
26 static struct pernode_descriptor __pernode_desc_##__n; \
27 static void __pernode_ctor_##__n(void *inst, size_t node) { \
28 void (*const __typed_ctor)(typeof(__type) *, size_t) = (__ctor); \
29 if (__typed_ctor != NULL) \
30 __typed_ctor((typeof(__type) *) inst, node); \
31 if (node == global.node_count - 1) \
32 atomic_store(&__pernode_desc_##__n.ready, true); \
33 } \
34 static LINKER_SECTION_OBJECT(struct pernode_descriptor, pernode_desc) \
35 __pernode_desc_##__n = { \
36 .name = #__n, \
37 .size = sizeof(typeof(__type)), \
38 .align = _Alignof(typeof(__type)), \
39 .pernode_ptrs = NULL, \
40 .constructor = __pernode_ctor_##__n, \
41 .ready = false, \
42 }; \
43 static struct pernode_descriptor *const __pernode_desc_ref_##__n \
44 __unused = &__pernode_desc_##__n
45
46#define PERNODE_EXPORT_AS(sym_name, name) \
47 extern struct pernode_descriptor __pernode_desc_sym_##sym_name \
48 __attribute__((alias("__pernode_desc_" #name), used))
49
50#define PERNODE_EXPORT(name) PERNODE_EXPORT_AS(name, name)
51
52#define PERNODE_DEFINE_AS(name, sym_name, type) \
53 extern struct pernode_descriptor __pernode_desc_sym_##sym_name; \
54 static typeof(type) __pernode_##name __unused; \
55 static struct pernode_descriptor *const __pernode_desc_ref_##name \
56 __unused = &__pernode_desc_sym_##sym_name
57
58#define PERNODE_DEFINE(name, type) PERNODE_DEFINE_AS(name, name, type)
59
60void pernode_obj_init(void);
61
62#define PERNODE(name) &(__pernode_##name)
63#define PERNODE_READY(name) (atomic_load(&(__pernode_desc_ref_##name)->ready))
64
65#define PERNODE_PTR_FOR_NODE(name, d) \
66 ({ \
67 (void) kassert(PERNODE_READY(name)); \
68 ((typeof(__pernode_##name) *) (__pernode_desc_ref_##name) \
69 ->pernode_ptrs[d]); \
70 })
71
72#define PERNODE_READ_FOR_NODE(name, d) \
73 (*((typeof(__pernode_##name) *) PERNODE_PTR_FOR_NODE(name, d)))
74
75#define PERNODE_PTR(clr, name) \
76 PERNODE_PTR_FOR_NODE(name, smp_core(clr)->numa_node)
77#define PERNODE_READ(clr, name) \
78 (*((typeof(__pernode_##name) *) PERNODE_PTR(clr, name)))
79
80#define PERNODE_WRITE(clr, name, val) (PERNODE_READ(clr, name) = (val))
81
82#define pernode_for_each_internal(name, var, node) \
83 for (node_id_t node = 0; node < global.node_count; node++) \
84 for (var = PERNODE_PTR_FOR_NODE(name, node); var != NULL; var = NULL)
85
86#define pernode_for_each_internal_3(name, var, node) \
87 pernode_for_each_internal(name, var, node)
88#define pernode_for_each_internal_2(name, var) \
89 pernode_for_each_internal(name, var, __node)
90
91#define pernode_for_each(...) \
92 _DISPATCH(pernode_for_each_internal, PP_NARG(__VA_ARGS__))(__VA_ARGS__)
93