Per-CPU dynamic objects

include/smp/percpu.h View source View on GitHub

Structs

percpu_descriptor

struct percpu_descriptor {
    const char                     *name;
    size_t                         size;
    size_t                         align;
    void                           **percpu_ptrs;
    percpu_descriptor_constructor  constructor;
    atomic_bool                    ready;
};

Type Aliases

percpu_descriptor_constructor

typedef void (*percpu_descriptor_constructor)(void * *, size_t);

Functions

percpu_obj_init

void percpu_obj_init(void);

Macros

PERCPU_DECLARE

#define PERCPU_DECLARE(__n, __type, __ctor) \
    static typeof(__type) __percpu_##__n __unused; \
    static struct percpu_descriptor __percpu_desc_##__n; \
    static void __percpu_ctor_##__n(void *inst, size_t cpu) { \
        void (*const __typed_ctor)(typeof(__type) *, size_t) = (__ctor); \
        if (__typed_ctor != NULL) \
            __typed_ctor((typeof(__type) *) inst, cpu); \
        if (cpu == global.core_count - 1) \
            atomic_store(&__percpu_desc_##__n.ready, true); \
    } \
    static LINKER_SECTION_OBJECT(struct percpu_descriptor, percpu_desc) \
        __percpu_desc_##__n = { \
            .name = #__n, \
            .size = sizeof(typeof(__type)), \
            .align = _Alignof(typeof(__type)), \
            .percpu_ptrs = NULL, \
            .constructor = __percpu_ctor_##__n, \
            .ready = false, \
    }; \
    static struct percpu_descriptor *const __percpu_desc_ref_##__n __unused = \
        &__percpu_desc_##__n

PERCPU_EXPORT_AS

#define PERCPU_EXPORT_AS(sym_name, name) \
    extern struct percpu_descriptor __percpu_desc_sym_##sym_name \
        __attribute__((alias("__percpu_desc_" #name), used))

PERCPU_EXPORT

#define PERCPU_EXPORT(name) PERCPU_EXPORT_AS(name, name)

PERCPU_DEFINE_AS

#define PERCPU_DEFINE_AS(name, sym_name, type) \
    extern struct percpu_descriptor __percpu_desc_sym_##sym_name; \
    static typeof(type) __percpu_##name __unused; \
    static struct percpu_descriptor *const __percpu_desc_ref_##name __unused = \
        &__percpu_desc_sym_##sym_name

PERCPU_DEFINE

#define PERCPU_DEFINE(name, type) PERCPU_DEFINE_AS(name, name, type)

PERCPU

#define PERCPU(name) &(__percpu_##name)

PERCPU_READY

#define PERCPU_READY(name) (atomic_load(&(__percpu_desc_ref_##name)->ready))

PERCPU_PTR_FOR_CPU

#define PERCPU_PTR_FOR_CPU(name, cpu) \
    ({ \
        (void) kassert(PERCPU_READY(name)); \
        ((typeof(__percpu_##name) *) (__percpu_desc_ref_##name) \
             ->percpu_ptrs[cpu]); \
    })

PERCPU_READ_FOR_CPU

#define PERCPU_READ_FOR_CPU(name, cpu) \
    (*((typeof(__percpu_##name) *) PERCPU_PTR_FOR_CPU(name, cpu)))

PERCPU_PTR

#define PERCPU_PTR(clr, name) PERCPU_PTR_FOR_CPU(name, smp_id(clr))

PERCPU_READ

#define PERCPU_READ(clr, name) \
    (*((typeof(__percpu_##name) *) PERCPU_PTR(clr, name)))

PERCPU_WRITE

#define PERCPU_WRITE(clr, name, val) (PERCPU_READ(clr, name) = (val))

percpu_for_each_internal

#define percpu_for_each_internal(name, var, cpu) \
    for (cpu_id_t cpu = 0; cpu < global.core_count; cpu++) \
        for (var = PERCPU_PTR_FOR_CPU(name, cpu); var != NULL; var = NULL)

percpu_for_each_internal_3

#define percpu_for_each_internal_3(name, var, cpu) \
    percpu_for_each_internal(name, var, cpu)

percpu_for_each_internal_2

#define percpu_for_each_internal_2(name, var) \
    percpu_for_each_internal(name, var, __cpu)

percpu_for_each

#define percpu_for_each(...) \
    _DISPATCH(percpu_for_each_internal, PP_NARG(__VA_ARGS__))(__VA_ARGS__)

---