1#pragma once
2#include <mem/buddy.h>
3#include <mem/page.h>
4#include <mem/vmm.h>
5#include <smp/domain.h>
6#include <stdbool.h>
7#include <stdint.h>
8#include <sync/semaphore.h>
9#include <sync/spinlock.h>
10
11#define DISTANCE_WEIGHT 100000 /* distance is heavily weighted */
12#define FREE_PAGES_WEIGHT 1 /* free pages count less */
13
14#define DOMAIN_ARENA_SIZE 64
15#define DOMAIN_FREE_QUEUE_SIZE 64
16
17#define ARENA_SCALE_PERMILLE 10 /* 1% of domain pages per-core arena */
18#define FREEQUEUE_SCALE_PERMILLE 5 /* 0.5% of total pages for freequeues */
19
20#define MAX_ARENA_PAGES ((size_t) 4096) /* absolute cap per-core arena */
21#define MAX_FREEQUEUE_PAGES \
22 ((size_t) 16384) /* absolute cap per-domain freequeue */
23
24struct page;
25struct buddy_page;
26struct domain_arena {
27 struct buddy_page **pages;
28 size_t head;
29 size_t tail;
30 size_t capacity;
31 atomic_size_t num_pages;
32 struct spinlock lock;
33};
34
35struct domain_free_queue {
36 struct {
37 paddr_t addr;
38 size_t pages;
39 } *queue;
40
41 size_t head;
42 size_t tail;
43 size_t capacity;
44 atomic_size_t num_elements;
45 atomic_bool free_in_progress; /* Simple flag to indicate that
46 * a free is in progress to prevent
47 * overly aggressive freeing. Enqueue/dequeue
48 * is still allowed since that uses
49 * the spinlock, but this just prevents
50 * aggressive concurrent access to the free
51 * queue so I don't free aggressively (not a
52 * 'race condition', just suboptimal) */
53
54 struct spinlock lock;
55};
56
57struct domain_zonelist_entry {
58 struct domain_buddy *domain;
59 uint8_t distance;
60 size_t free_pages;
61};
62
63struct domain_zonelist {
64 struct domain_zonelist_entry *entries;
65 size_t count;
66};
67
68struct domain_buddy_stats {
69 atomic_size_t alloc_count;
70 atomic_size_t free_count;
71 atomic_size_t remote_alloc_count;
72 atomic_size_t failed_alloc_count;
73 atomic_size_t interleaved_alloc_count;
74};
75
76struct domain_flush_worker {
77 struct domain_buddy *domain;
78 struct thread *thread;
79 struct semaphore sema;
80 atomic_bool enqueued;
81 bool stop;
82};
83
84struct domain_buddy {
85 struct domain_buddy_stats stats;
86 struct domain_zonelist zonelist;
87
88 struct buddy_page *buddy;
89 struct buddy_free_area *free_area;
90
91 struct domain_arena **arenas; /* One per core */
92 struct core **cores;
93 size_t core_count; /* # cores on this domain */
94
95 struct domain_free_queue *free_queue;
96
97 paddr_t start; /* physical start address */
98 paddr_t end; /* physical end address */
99 size_t length; /* total bytes */
100
101 atomic_size_t pages_used;
102 atomic_size_t total_pages;
103
104 struct spinlock lock;
105 struct domain_flush_worker worker;
106 struct domain *domain;
107};
108
109void domain_buddies_init(void);
110void domain_free(paddr_t address, size_t page_count);
111paddr_t domain_alloc(size_t pages, enum alloc_flags flags);
112bool domain_free_queue_enqueue(struct domain_free_queue *fq, paddr_t addr,
113 size_t pages);
114bool domain_free_queue_dequeue(struct domain_free_queue *fq, paddr_t *addr_out,
115 size_t *pages_out);
116bool domain_arena_push(struct domain_arena *arena, struct buddy_page *page);
117struct buddy_page *domain_arena_pop(struct domain_arena *arena);
118
119void domain_flush_free_queue(struct domain_buddy *domain,
120 struct domain_free_queue *queue);
121void domain_flush_thread(void *arg);
122void domain_enqueue_flush_worker(struct domain_flush_worker *worker);
123
124#define domain_for_each_arena(arena_ptr_, domain_) \
125 for (uint32_t __arena_idx = 0; \
126 __arena_idx < (domain_)->core_count && \
127 (((arena_ptr_) = ((domain_)->arenas[__arena_idx])), true); \
128 __arena_idx++)
129
130static inline struct domain_buddy *domain_buddy_for_addr(paddr_t addr) {
131 for (size_t i = 0; i < global.domain_count; i++) {
132 struct domain_buddy *d = &global.domain_buddies[i];
133 if (addr >= d->start && addr < d->end)
134 return d;
135 }
136
137 /* None? */
138 return NULL;
139}
140
141/* These are all fine, they're accessed in either
142 * pinned or non-preemptible ctx */
143static inline struct domain_buddy *domain_buddy_on_this_core(void) {
144 return smp_core(cond: TOPC_IRQL | TOPC_PINNED)->domain->domain_buddy;
145}
146
147static inline struct domain_arena *domain_arena_on_this_core(void) {
148 return smp_core(cond: TOPC_IRQL | TOPC_PINNED)->domain_arena;
149}
150
151static inline struct domain_free_queue *domain_free_queue_on_this_core(void) {
152 return domain_buddy_on_this_core()->free_queue;
153}
154
155static inline size_t *domain_rr_on_this_core(void) {
156 return &smp_core(cond: TOPC_IRQL | TOPC_PINNED)->rr_current_domain;
157}
158
159static inline void domain_stat_alloc(struct domain_buddy *d, bool remote,
160 bool interleaved) {
161
162 atomic_inc_rlx(&d->stats.alloc_count);
163
164 if (remote)
165 atomic_inc_rlx(&d->stats.remote_alloc_count);
166 if (interleaved)
167 atomic_inc_rlx(&d->stats.interleaved_alloc_count);
168}
169
170static inline void domain_stat_free(struct domain_buddy *d) {
171 atomic_inc_rlx(&d->stats.free_count);
172}
173
174static inline void domain_stat_mark_interleaved(struct domain_buddy *d) {
175 atomic_inc_rlx(&d->stats.interleaved_alloc_count);
176}
177
178static inline void domain_stat_failed_alloc(struct domain_buddy *d) {
179 atomic_inc_rlx(&d->stats.failed_alloc_count);
180}
181
182static inline bool is_free_in_progress(struct domain_free_queue *fq) {
183 return atomic_load_rlx(&fq->free_in_progress);
184}
185
186static inline void mark_free_in_progress(struct domain_free_queue *fq, bool s) {
187 atomic_store_rlx(&fq->free_in_progress, s);
188}
189
190static inline struct buddy_page *buddy_page_for_addr(paddr_t address) {
191 return (struct buddy_page *) page_for_paddr(paddr: address);
192}
193
194static inline void free_from_buddy_internal(struct domain_buddy *target,
195 paddr_t address,
196 size_t page_count) {
197 enum irql irql = spin_lock(&target->lock);
198 buddy_free_pages(addr: address, count: page_count, free_area: target->free_area,
199 atomic_load_rlx(&target->total_pages));
200 domain_stat_free(d: target);
201 atomic_fetch_sub(&target->pages_used, page_count);
202
203 spin_unlock(&target->lock, irql);
204}
205
206struct domain *domain_alloc_pick_best_domain(struct domain *local, size_t pages,
207 size_t max_scan,
208 bool flexible_locality);
209