Per-CPU structure

include/smp/core.h View source View on GitHub

Structs

cpu_capability

struct cpu_capability (7 fields) source

§ enum cpu_class class

§ uint32_t uarch_id

e.g. golden cove, gracemont

§ uint32_t issue_width

§ uint32_t retire_width

§ cpu_perf_t perf_score

Relative to everyone else on a 0-255 scale, how performant are we? The higher this number is, the more “performant” this CPU currently is, and the more likely the scheduler will decide to migrate a thread that needs such perf scores onto here.

§ uint32_t energy_score

lower is better

§ uint64_t feature_bits

ISA features, vector width, etc

core

struct core (31 fields) source

Let’s put commonly accessed fields up here to make the cache a bit happier

§ struct core *self

§ cpu_id_t id

§ struct thread *current_thread

§ struct cpu_capability cap

§ size_t domain_cpu_id

what CPU in the domain?

§ struct scheduler_domain *domains[TOPOLOGY_LEVEL_MAX]

array domain_levels_enabled -> domain reference

§ int32_t group_index[TOPOLOGY_LEVEL_MAX]

index within each domain’s groups

§ atomic_bool executing_dpcs

§ atomic_bool idle

§ uint32_t ctx

Execution context in one word, using SMP_CTX_* below

§ enum irql current_irql

§ atomic_bool needs_run_dpcs

Set before sending IRQ_NOP, which is then checked in the isr_common_entry

§ atomic_bool needs_resched

§ atomic_bool in_resched

in scheduler_yield()

§ struct domain *domain

§ struct domain_arena *domain_arena

§ size_t rr_current_domain

§ struct tss *tss

§ freq_khz_t lapic_khz

§ struct topology_node *topo_node

§ struct topology_cache_info llc

§ numa_node_t numa_node

§ uint32_t package_id

§ uint32_t smt_mask

§ uint32_t smt_id

§ uint32_t core_id

§ freq_hz_t tsc_hz

§ time_us_t last_us

§ uint64_t last_tsc

For time.c

§ atomic_uint64_t pt_seen_epoch

§ bool reclaiming_page_tables

Enums

cpu_class

enum cpu_class (3 variants) source

§ CPU_CLASS_UNKNOWN

§ CPU_CLASS_PERFORMANCE

§ CPU_CLASS_EFFICIENCY

Functions

smp_caller_verify

void smp_caller_verify(enum topology_caller caller)

source

smp_id

cpu_id_t smp_id(enum topology_caller cond)

source

smp_id_raw

cpu_id_t smp_id_raw(void)

source

smp_apic_id

uint32_t smp_apic_id(cpu_id_t cpu)

source

smp_cpu_from_apic_id

cpu_id_t smp_cpu_from_apic_id(uint32_t apic_id)

source

CPU_ID_NONE if no CPU booted has apic_id

smp_core

struct core * smp_core(enum topology_caller cond)

source

smp_core_raw

struct core * smp_core_raw(void)

source

smp_bsp

struct core * smp_bsp(void)

source

smp_ctx

uint32_t smp_ctx(enum topology_caller c)

source

smp_ctx_preempt_count

uint32_t smp_ctx_preempt_count(uint32_t smp_ctx)

source

smp_ctx_irq_count

uint32_t smp_ctx_irq_count(uint32_t smp_ctx)

source

smp_ctx_nmi_count

uint32_t smp_ctx_nmi_count(uint32_t smp_ctx)

source

smp_ctx_add

uint32_t smp_ctx_add(
    enum topology_caller c,
    uint32_t one,
    uint32_t mask
)

source

Add a SMP_CTX_*_ONE to its field

smp_ctx_sub

uint32_t smp_ctx_sub(
    enum topology_caller c,
    uint32_t one,
    uint32_t mask
)

source

Variables

smp_apic_ids

extern uint32_t *smp_apic_ids

source

Macros

CPU_FEAT_SSE2

#define CPU_FEAT_SSE2 BIT(0)

source

CPU_FEAT_AVX

#define CPU_FEAT_AVX BIT(1)

source

CPU_FEAT_AVX2

#define CPU_FEAT_AVX2 BIT(2)

source

CPU_FEAT_AVX512F

#define CPU_FEAT_AVX512F BIT(3)

source

smp_read8

#define smp_read8(off)

source

Expansion

#define smp_read8(off) \
    ({ \
        uint8_t __smp_v; \
        asm volatile("movb %%gs:%c1, %b0" \
                     : "=q"(__smp_v) \
                     : "i"(off) \
                     : "memory"); \
        __smp_v; \
    })

smp_read16

#define smp_read16(off)

source

Expansion

#define smp_read16(off) \
    ({ \
        uint16_t __smp_v; \
        asm volatile("movw %%gs:%c1, %w0" \
                     : "=r"(__smp_v) \
                     : "i"(off) \
                     : "memory"); \
        __smp_v; \
    })

smp_read32

#define smp_read32(off)

source

Expansion

#define smp_read32(off) \
    ({ \
        uint32_t __smp_v; \
        asm volatile("movl %%gs:%c1, %k0" \
                     : "=r"(__smp_v) \
                     : "i"(off) \
                     : "memory"); \
        __smp_v; \
    })

smp_read64

#define smp_read64(off)

source

Expansion

#define smp_read64(off) \
    ({ \
        uint64_t __smp_v; \
        asm volatile("movq %%gs:%c1, %0" \
                     : "=r"(__smp_v) \
                     : "i"(off) \
                     : "memory"); \
        __smp_v; \
    })

smp_member_size

#define smp_member_size(member)

source

Expansion

#define smp_member_size(member) sizeof(typeof(((struct core *) 0)->member))

_smp_read_type

#define _smp_read_type(member)

source

Expansion

#define _smp_read_type(member) (typeof(ct_decay(((struct core *) 0)->member)))

_smp_read_type

#define _smp_read_type(member)

source

Expansion

#define _smp_read_type(member) (typeof(((struct core *) 0)->member))

smp_read

#define smp_read(cond, member)

source

Expansion

#define smp_read(cond, member) \
    ({ \
        if ((cond) != TOPC_NONE) \
            smp_caller_verify(cond); \
        static_assert( \
            smp_member_size(member) == 1 || smp_member_size(member) == 2 || \
                smp_member_size(member) == 4 || smp_member_size(member) == 8, \
            "smp_core_read: unsupported member size"); \
        uint64_t __smp_raw; \
        switch (smp_member_size(member)) { \
        case 1: __smp_raw = smp_read8(offsetof(struct core, member)); break; \
        case 2: __smp_raw = smp_read16(offsetof(struct core, member)); break; \
        case 4: __smp_raw = smp_read32(offsetof(struct core, member)); break; \
        case 8: __smp_raw = smp_read64(offsetof(struct core, member)); break; \
        default: ci_unreachable(); \
        } \
        _smp_read_type(member)(uintptr_t) __smp_raw; \
    })

smp_write8

#define smp_write8(off, v)

source

Expansion

#define smp_write8(off, v) \
    ({ \
        uint8_t __smp_v = (v); \
        asm volatile("movb %b0, %%gs:%c1" \
                     : \
                     : "q"(__smp_v), "i"(off) \
                     : "memory"); \
    })

smp_write16

#define smp_write16(off, v)

source

Expansion

#define smp_write16(off, v) \
    ({ \
        uint16_t __smp_v = (v); \
        asm volatile("movw %w0, %%gs:%c1" \
                     : \
                     : "r"(__smp_v), "i"(off) \
                     : "memory"); \
    })

smp_write32

#define smp_write32(off, v)

source

Expansion

#define smp_write32(off, v) \
    ({ \
        uint32_t __smp_v = (v); \
        asm volatile("movl %0, %%gs:%c1" \
                     : \
                     : "r"(__smp_v), "i"(off) \
                     : "memory"); \
    })

smp_write64

#define smp_write64(off, v)

source

Expansion

#define smp_write64(off, v) \
    ({ \
        uint64_t __smp_v = (v); \
        asm volatile("movq %0, %%gs:%c1" \
                     : \
                     : "r"(__smp_v), "i"(off) \
                     : "memory"); \
    })

smp_write

#define smp_write(cond, member, val)

source

Expansion

#define smp_write(cond, member, val) \
    do { \
        if ((cond) != TOPC_NONE) \
            smp_caller_verify(cond); \
        static_assert( \
            smp_member_size(member) == 1 || smp_member_size(member) == 2 || \
                smp_member_size(member) == 4 || smp_member_size(member) == 8, \
            "smp_core_write: unsupported member size"); \
        typeof(((struct core *) 0)->member) __smp_val = (val); \
        uint64_t __smp_raw = (uint64_t) (uintptr_t) __smp_val; \
        switch (smp_member_size(member)) { \
        case 1: \
            smp_write8(offsetof(struct core, member), (uint8_t) __smp_raw); \
            break; \
        case 2: \
            smp_write16(offsetof(struct core, member), (uint16_t) __smp_raw); \
            break; \
        case 4: \
            smp_write32(offsetof(struct core, member), (uint32_t) __smp_raw); \
            break; \
        case 8: smp_write64(offsetof(struct core, member), __smp_raw); break; \
        default: ci_unreachable(); \
        } \
    } while (0)

SMP_CTX_PREEMPT_SHIFT

#define SMP_CTX_PREEMPT_SHIFT 0

source

The idea with this (TODO: Consider an enum) is that if a migration happens across reads of two different words, the whole result becomes invalid, and squishing it all into one word guarantees that such behavior is not possible. We also use counters here, as opposed to flags, so reentrancy bugs can be identified from the get-go.

NOTE

needs_resched is not tracked here because that’s for other CPUs to write, and this is purely local

SMP_CTX_PREEMPT_BITS

#define SMP_CTX_PREEMPT_BITS 8

source

SMP_CTX_IRQ_SHIFT

#define SMP_CTX_IRQ_SHIFT 8

source

SMP_CTX_IRQ_BITS

#define SMP_CTX_IRQ_BITS 8

source

SMP_CTX_NMI_SHIFT

#define SMP_CTX_NMI_SHIFT 16

source

SMP_CTX_NMI_BITS

#define SMP_CTX_NMI_BITS 4

source

SMP_CTX_FIELD_MAX

#define SMP_CTX_FIELD_MAX(bits)

source

Expansion

#define SMP_CTX_FIELD_MAX(bits) ((1u << (bits)) - 1u)

SMP_CTX_PREEMPT_MASK

#define SMP_CTX_PREEMPT_MASK

source

Expansion

#define SMP_CTX_PREEMPT_MASK \
    (SMP_CTX_FIELD_MAX(SMP_CTX_PREEMPT_BITS) << SMP_CTX_PREEMPT_SHIFT)

SMP_CTX_IRQ_MASK

#define SMP_CTX_IRQ_MASK

source

Expansion

#define SMP_CTX_IRQ_MASK \
    (SMP_CTX_FIELD_MAX(SMP_CTX_IRQ_BITS) << SMP_CTX_IRQ_SHIFT)

SMP_CTX_NMI_MASK

#define SMP_CTX_NMI_MASK

source

Expansion

#define SMP_CTX_NMI_MASK \
    (SMP_CTX_FIELD_MAX(SMP_CTX_NMI_BITS) << SMP_CTX_NMI_SHIFT)

SMP_CTX_PREEMPT_ONE

#define SMP_CTX_PREEMPT_ONE (1u << SMP_CTX_PREEMPT_SHIFT)

source

SMP_CTX_IRQ_ONE

#define SMP_CTX_IRQ_ONE (1u << SMP_CTX_IRQ_SHIFT)

source

SMP_CTX_NMI_ONE

#define SMP_CTX_NMI_ONE (1u << SMP_CTX_NMI_SHIFT)

source

SMP_CTX_IN_INTERRUPT_MASK

#define SMP_CTX_IN_INTERRUPT_MASK (SMP_CTX_IRQ_MASK | SMP_CTX_NMI_MASK)

source

“not plain thread context”

for_each_cpu_struct

#define for_each_cpu_struct(iter_)

source

Expansion

#define for_each_cpu_struct(iter_) \
    for (size_t __cpu_idx = 0; __cpu_idx < global.core_count && \
                               (((iter_) = global.cores[__cpu_idx]), true); \
         __cpu_idx++)

for_each_cpu_id

#define for_each_cpu_id(id_)

source

Expansion

#define for_each_cpu_id(id_) for ((id_) = 0; (id_) < global.core_count; (id_)++)