include/thread/thread.h View source View on GitHubstruct cpu_context {
uint64_t rbx;
uint64_t rbp;
uint64_t r12;
uint64_t r13;
uint64_t r14;
uint64_t r15;
uint64_t rsp;
uint64_t rip;
};
struct thread_event_association {
uint8_t reason;
uint64_t cycle;
};
struct thread_event_reason {
uint8_t reason;
struct thread_event_association associated_reason;
time_t timestamp;
uint64_t cycle;
};
struct thread_runtime_bucket {
uint16_t run_time_ms;
uint64_t wall_clock_sec;
};
struct thread_activity_bucket {
uint64_t cycle;
uint32_t block_count;
uint32_t sleep_count;
uint32_t wake_count;
uint16_t block_duration;
uint16_t sleep_duration;
};
struct thread_activity_stats {
struct thread_runtime_bucket rt_buckets[THREAD_ACTIVITY_BUCKET_COUNT];
struct thread_activity_bucket buckets[THREAD_ACTIVITY_BUCKET_COUNT];
time_t last_update_ms;
uint64_t current_cycle;
uint8_t current_bucket;
uint8_t last_wake_index;
};
struct thread_activity_data {
MAKE_THREAD_RINGBUFFER(wake_reasons) ;
MAKE_THREAD_RINGBUFFER(block_reasons) ;
MAKE_THREAD_RINGBUFFER(sleep_reasons) ;
};
struct thread_activity_metrics {
uint8_t run_ratio;
uint8_t block_ratio;
uint8_t sleep_ratio;
uint8_t wake_freq;
};
struct thread {
uint64_t id;
char *name;
void (*entry)(void *);
void *stack;
size_t stack_size;
struct cpu_context regs;
struct list_head reaper_list;
struct list_head thread_list;
union {
struct rbt_node rq_tree_node;
struct rbt_node rt_tree_node;
};
union {
struct list_head rq_list_node;
struct list_head rt_list_node;
};
struct rbt_node wq_tree_node;
struct list_head wq_list_node;
struct pairing_node wq_pairing_node;
struct list_head rcu_list_node;
_Atomic enum thread_state state;
cpu_id_t curr_core;
cpu_id_t core_to_wake_on;
_Atomic (struct scheduler *) scheduler;
time_t run_start_time;
struct cpu_mask allowed_cpus;
_Atomic int64_t migrate_to;
enum rt_scheduler_capability accepted_rt_caps;
_Atomic (enum thread_flags) flags;
_Atomic size_t migration_generation;
thread_prio_t activity_score;
int32_t dynamic_delta;
size_t weight;
nice_t niceness;
cpu_perf_t wanted_perf;
time_t last_class_change_ms;
size_t effective_priority;
uint64_t completed_period;
time_t period_runtime_raw_ms;
time_t budget_time_raw_ms;
time_t timeslice_length_raw_ms;
uint32_t virtual_period_runtime;
uint32_t virtual_budget;
uint32_t virtual_runtime_left;
enum thread_activity_class activity_class;
enum thread_prio_class base_prio_class;
enum thread_prio_class perceived_prio_class;
struct thread_activity_data *activity_data;
struct thread_activity_stats *activity_stats;
struct thread_activity_metrics activity_metrics;
struct spinlock lock;
refcount_t refcount;
volatile enum wake_reason wake_reason;
size_t wait_cookie;
_Atomic uint32_t rcu_nesting;
_Atomic uint64_t rcu_start_gen;
_Atomic uint64_t rcu_quiescent_gen;
_Atomic enum thread_wait_type wait_type;
void *expected_wake_src;
uint64_t wait_token;
uint8_t last_action_reason;
enum thread_state last_action;
_Atomic (void *) wake_src;
uint64_t wake_token;
uint64_t token_ctr;
struct condvar_with_cb cv_cb_object;
struct list_head io_wait_tokens;
struct turnstile *turnstile;
_Atomic (struct turnstile *) blocked_ts;
struct climb_thread_state climb_state;
struct apc_queue apc_head[APC_TYPE_COUNT];
_Atomic uint8_t apc_pending_mask;
uint32_t special_apc_disable;
uint32_t kernel_apc_disable;
struct apc_queue event_apcs;
struct apc_queue to_exec_event_apcs;
struct log_site *log_site;
size_t context_switches;
size_t preemptions;
time_t creation_time_ms;
uint32_t boost_count;
uint32_t total_wake_count;
uint32_t total_block_count;
uint32_t total_sleep_count;
uint32_t total_apcs_ran;
void *private;
};
enum thread_activity_class {
THREAD_ACTIVITY_CLASS_CPU_BOUND,
THREAD_ACTIVITY_CLASS_IO_BOUND,
THREAD_ACTIVITY_CLASS_INTERACTIVE,
THREAD_ACTIVITY_CLASS_SLEEPY,
THREAD_ACTIVITY_CLASS_UNKNOWN,
};
struct thread * thread_create_internal(char *name, void (*entry_point)(void *), void *arg, size_t stack_size, va_list args);
struct thread * thread_create(char *name, void (*entry_point)(void *), void *arg);
struct thread * thread_create_custom_stack(char *name, void (*entry_point)(void *), void *arg, size_t stack_size);
void thread_free(struct thread *t);
void thread_init_thread_ids(void);
void thread_sleep_for_ms(uint64_t ms);
void thread_exit(void);
void thread_print(const struct thread *t);
void thread_update_activity_stats(struct thread *t, time_t time);
void thread_classify_activity(struct thread *t, uint64_t now_ms);
void thread_update_runtime_buckets(struct thread *thread, time_t time);
void thread_apply_wake_boost(struct thread *t);
void thread_update_effective_priority(struct thread *t);
void thread_apply_cpu_penalty(struct thread *t);
void thread_add_wake_reason(struct thread *t, uint8_t reason);
void scheduler_wake_manual(struct thread *t, void *wake_src);
void thread_calculate_activity_data(struct thread *t);
void thread_add_block_reason(struct thread *t, uint8_t reason);
void thread_add_sleep_reason(struct thread *t, uint8_t reason);
void thread_prepare_to_block(struct thread *t, enum thread_block_reason r, enum thread_wait_type wait_type, void *expect_wake_src);
void thread_prepare_to_sleep(struct thread *t, enum thread_sleep_reason r, enum thread_wait_type wait_type, void *expect_wake_src);
void thread_prepare_to_block_locked(struct thread *t, enum thread_block_reason r, enum thread_wait_type type, void *expect_wake_src);
void thread_set_timesharing(struct thread *t);
void thread_set_background(struct thread *t);
void thread_wake_unlocked(struct thread *t, enum thread_wake_reason r, void *wake_src);
void thread_migrate(struct thread *t, size_t dest_core);
void thread_yield_until_wake_match();
enum thread_prio_class thread_unboost_self();
enum thread_prio_class thread_boost_self(enum thread_prio_class new);
struct scheduler * thread_get_scheduler(struct thread *t, enum irql *sirql_out);
void thread_block_on(struct thread_queue *q, enum thread_wait_type type, void *wake_src);
void thread_enqueue(struct thread *t);
void thread_enqueue_on_core(struct thread *t, uint64_t core_id);
bool thread_wake(struct thread *t, enum thread_wake_reason reason, enum thread_prio_class prio, void *wake_src);
void thread_wake_from_io_block(struct thread *t, void *wake_src);
bool thread_inherit_priority(struct thread *boosted, struct thread *from, enum thread_prio_class *old_class_out);
void thread_uninherit_priority(enum thread_prio_class class);
void thread_remove_boost();
void thread_lock_two_runqueues(struct thread *a, struct thread *b, struct scheduler **out_rq_a, struct scheduler **out_rq_b, enum irql *irq_a, enum irql *irq_b);
void thread_lock_thread_and_rq(struct thread *t, struct scheduler *other_rq, struct scheduler **out_thread_rq, enum irql *irq_first, enum irql *irq_second);
void thread_unlock_thread_and_rq(struct scheduler *thread_rq, struct scheduler *other_rq, enum irql irq_first, enum irql irq_second);
struct thread thread_get_current();
int64_t thread_set_migration_target(struct thread *t, int64_t new);
enum thread_state thread_get_state(struct thread *t);
void thread_set_state(struct thread *t, enum thread_state state);
enum thread_flags thread_get_flags(struct thread *t);
void thread_set_flags(struct thread *t, enum thread_flags new);
bool thread_pin(struct thread *t);
void thread_unpin(struct thread *t);
enum thread_flags thread_or_flags(struct thread *t, enum thread_flags flags);
enum thread_flags thread_and_flags(struct thread *t, enum thread_flags flags);
size_t thread_get_migration_generation(struct thread *t);
struct scheduler thread_get_scheduler_unsafe(struct thread *t);
void thread_set_runqueue(struct thread *t, struct scheduler *s);
void reaper_enqueue(struct thread *t);
void thread_put(struct thread *t);
enum irql thread_acquire(struct thread *t, bool *success);
void thread_release(struct thread *t, enum irql irql);
bool thread_is_rt(struct thread *t);
void thread_clear_wake_data_raw(struct thread *t);
void thread_clear_wake_data(struct thread *t);
enum thread_wait_type thread_get_wait_type(struct thread *t);
struct thread thread_spawn(char *name, void (*entry)(void *), void *arg);
struct thread thread_spawn_custom_stack(char *name, void (*entry)(void *), void *arg, size_t stack_size);
struct thread thread_spawn_on_core(char *name, void (*entry)(void *), void *arg, uint64_t core_id);
#define THREAD_DEFAULT_TIMESLICE 15
#define THREAD_CLASS_WIDTH 1024
#define THREAD_CLASS_HALF (THREAD_CLASS_WIDTH / 2)
#define THREAD_BAND_MIN(avg) ((avg) - THREAD_CLASS_HALF)
#define THREAD_BAND_MAX(avg) ((avg) + THREAD_CLASS_HALF)
#define THREAD_ACT_INTERACTIVE_AVG 4000u
#define THREAD_ACT_IO_BOUND_AVG 2500u
#define THREAD_ACT_CPU_BOUND_AVG 1200u
#define THREAD_ACT_SLEEPY_AVG 4500u
#define THREAD_ACT_INTERACTIVE_MIN THREAD_BAND_MIN(THREAD_ACT_INTERACTIVE_AVG)
#define THREAD_ACT_INTERACTIVE_MAX THREAD_BAND_MAX(THREAD_ACT_INTERACTIVE_AVG)
#define THREAD_ACT_IO_BOUND_MIN THREAD_BAND_MIN(THREAD_ACT_IO_BOUND_AVG)
#define THREAD_ACT_IO_BOUND_MAX THREAD_BAND_MAX(THREAD_ACT_IO_BOUND_AVG)
#define THREAD_ACT_CPU_BOUND_MIN THREAD_BAND_MIN(THREAD_ACT_CPU_BOUND_AVG)
#define THREAD_ACT_CPU_BOUND_MAX THREAD_BAND_MAX(THREAD_ACT_CPU_BOUND_AVG)
#define THREAD_ACT_SLEEPY_MIN THREAD_BAND_MIN(THREAD_ACT_SLEEPY_AVG)
#define THREAD_ACT_SLEEPY_MAX THREAD_BAND_MAX(THREAD_ACT_SLEEPY_AVG)
#define THREAD_NICENESS_VALID(n) \
((((nice_t) (n)) >= -19) && (((nice_t) (n)) <= 20))
#define THREAD_EVENT_REASON_NONE 0xFF
#define THREAD_ASSOCIATED_REASON_NONE 0xFF
#define THREAD_PRIO_IS_TIMESHARING(prio) (prio == THREAD_PRIO_CLASS_TIMESHARE)
#define THREAD_PRIO_HAS_TIMESLICE(prio) \
(THREAD_PRIO_IS_TIMESHARING(prio) || prio == THREAD_PRIO_CLASS_BACKGROUND)
#define THREAD_ACTIVITY_BUCKET_COUNT 4
#define THREAD_ACTIVITY_BUCKET_DURATION 1000
#define THREAD_EVENT_RINGBUFFER_CAPACITY THREAD_ACTIVITY_BUCKET_COUNT
#define TOTAL_BUCKET_DURATION \
(THREAD_ACTIVITY_BUCKET_COUNT * THREAD_ACTIVITY_BUCKET_DURATION)
#define MAKE_THREAD_RINGBUFFER(name) \
struct thread_event_reason name[THREAD_EVENT_RINGBUFFER_CAPACITY]; \
uint8_t name##_head;
#define thread_from_rq_rbt_node(node) \
rbt_entry(node, struct thread, rq_tree_node)
#define thread_from_rq_list_node(ln) \
(container_of(ln, struct thread, rq_list_node))
#define thread_from_rcu_list_node(ln) \
(container_of(ln, struct thread, rcu_list_node))
#define thread_from_wq_pairing_node(pn) \
(container_of(pn, struct thread, wq_pairing_node))
#define thread_from_wq_list_node(ln) \
(container_of(ln, struct thread, wq_list_node))
#define thread_from_wq_rbt_node(ln) \
(container_of(ln, struct thread, wq_tree_node))