Per-CPU dynamic objects

include/smp/percpu.h View source View on GitHub

Structs

percpu_descriptor

struct percpu_descriptor (6 fields) source

§ const char *name

§ size_t size

§ size_t align

§ void **percpu_ptrs

§ percpu_descriptor_constructor constructor

§ atomic_bool ready

Type Aliases

percpu_descriptor_constructor

typedef void (*percpu_descriptor_constructor)(void * *, size_t)

source

Functions

percpu_desc_ready

cc_pure bool percpu_desc_ready(const struct percpu_descriptor *desc)

source

percpu_obj_init

void percpu_obj_init(void)

source

percpu_alloc_array

void * * percpu_alloc_array(size_t size)

source

Macros

PERCPU_DEFINE_3

#define PERCPU_DEFINE_3(type_, name_, ctor_)

source

Expansion

#define PERCPU_DEFINE_3(type_, name_, ctor_) \
    static typeof(type_) __percpu_##name_ cc_unused; \
    static struct percpu_descriptor __percpu_desc_##name_; \
    static void __percpu_ctor_##name_(void *inst, size_t cpu) { \
        void (*const __typed_ctor)(typeof(type_) *, size_t) = (ctor_); \
        if (__typed_ctor != NULL) \
            __typed_ctor((typeof(type_) *) inst, cpu); \
        if (cpu == global.core_count - 1) \
            atomic_store(&__percpu_desc_##name_.ready, true); \
    } \
    static LINKER_SECTION_OBJECT(struct percpu_descriptor, percpu_desc) \
        __percpu_desc_##name_ = { \
            .name = #name_, \
            .size = sizeof(typeof(type_)), \
            .align = cc_alignof(typeof(type_)), \
            .percpu_ptrs = NULL, \
            .constructor = __percpu_ctor_##name_, \
            .ready = false, \
    }; \
    static struct percpu_descriptor *const __percpu_desc_ref_##name_ \
        cc_unused = &__percpu_desc_##name_

PERCPU_DEFINE_2

#define PERCPU_DEFINE_2(type_, name_)

source

Expansion

#define PERCPU_DEFINE_2(type_, name_) PERCPU_DEFINE_3(type_, name_, NULL)

PERCPU_DEFINE

#define PERCPU_DEFINE(...)

source

Expansion

#define PERCPU_DEFINE(...) PP_CALL(PERCPU_DEFINE, __VA_ARGS__)

PERCPU_EXPORT_AS

#define PERCPU_EXPORT_AS(sym_name, name)

source

Expansion

#define PERCPU_EXPORT_AS(sym_name, name) \
    extern struct percpu_descriptor __percpu_desc_sym_##sym_name \
        cc_alias(__percpu_desc_##name) cc_used

PERCPU_EXPORT

#define PERCPU_EXPORT(name)

source

Expansion

#define PERCPU_EXPORT(name) PERCPU_EXPORT_AS(name, name)

PERCPU_EXTERN_AS

#define PERCPU_EXTERN_AS(type, name, sym_name)

source

Expansion

#define PERCPU_EXTERN_AS(type, name, sym_name) \
    extern struct percpu_descriptor __percpu_desc_sym_##sym_name; \
    static typeof(type) __percpu_##name cc_unused; \
    static struct percpu_descriptor *const __percpu_desc_ref_##name \
        cc_unused = &__percpu_desc_sym_##sym_name

PERCPU_EXTERN

#define PERCPU_EXTERN(type, name)

source

Expansion

#define PERCPU_EXTERN(type, name) PERCPU_EXTERN_AS(type, name, name)

PERCPU

#define PERCPU(name)

source

Expansion

#define PERCPU(name) &(__percpu_##name)

PERCPU_READY

#define PERCPU_READY(name)

source

Expansion

#define PERCPU_READY(name) (atomic_load(&(__percpu_desc_ref_##name)->ready))

PERCPU_PTR_FOR

#define PERCPU_PTR_FOR(name, cpu)

source

Expansion

#define PERCPU_PTR_FOR(name, cpu) \
    ({ \
        assume(percpu_desc_ready(__percpu_desc_ref_##name)); \
        ((typeof(__percpu_##name) *) (__percpu_desc_ref_##name) \
             ->percpu_ptrs[cpu]); \
    })

PERCPU_READ_FOR

#define PERCPU_READ_FOR(name, cpu)

source

Expansion

#define PERCPU_READ_FOR(name, cpu) \
    (*((typeof(__percpu_##name) *) PERCPU_PTR_FOR(name, cpu)))

PERCPU_PTR

#define PERCPU_PTR(clr, name)

source

Expansion

#define PERCPU_PTR(clr, name) PERCPU_PTR_FOR(name, smp_id(clr))

PERCPU_READ

#define PERCPU_READ(clr, name)

source

Expansion

#define PERCPU_READ(clr, name) \
    (*((typeof(__percpu_##name) *) PERCPU_PTR(clr, name)))

PERCPU_WRITE

#define PERCPU_WRITE(clr, name, val)

source

Expansion

#define PERCPU_WRITE(clr, name, val) (PERCPU_READ(clr, name) = (val))

percpu_for_each_internal_3

#define percpu_for_each_internal_3(var, cpu, name)

source

Expansion

#define percpu_for_each_internal_3(var, cpu, name) \
    for (cpu = 0; cpu < global.core_count; cpu++) \
        for (var = PERCPU_PTR_FOR(name, cpu); var != NULL; var = NULL)

percpu_for_each_internal_2

#define percpu_for_each_internal_2(var, name)

source

Expansion

#define percpu_for_each_internal_2(var, name) \
    for (cpu_id_t __percpu_idx = 0; __percpu_idx < global.core_count; \
         __percpu_idx++) \
        for (var = PERCPU_PTR_FOR(name, __percpu_idx); var != NULL; var = NULL)

percpu_for_each

#define percpu_for_each(...)

source

Expansion

#define percpu_for_each(...) PP_CALL(percpu_for_each_internal, __VA_ARGS__)