1#pragma once
2#include <mem/buddy.h>
3#include <mem/page.h>
4#include <mem/vmm.h>
5#include <smp/domain.h>
6#include <stdbool.h>
7#include <stdint.h>
8#include <sync/semaphore.h>
9#include <sync/spinlock.h>
10
11#define DISTANCE_WEIGHT 100000 /* distance is heavily weighted */
12#define FREE_PAGES_WEIGHT 1 /* free pages count less */
13
14#define DOMAIN_ARENA_SIZE 64
15#define DOMAIN_FREE_QUEUE_SIZE 64
16
17#define ARENA_SCALE_PERMILLE 10 /* 1% of domain pages per-core arena */
18#define FREEQUEUE_SCALE_PERMILLE 5 /* 0.5% of total pages for freequeues */
19
20#define MAX_ARENA_PAGES 4096 /* absolute cap per-core arena */
21#define MAX_FREEQUEUE_PAGES 16384 /* absolute cap per-domain freequeue */
22
23struct page;
24struct buddy_page;
25struct domain_arena {
26 struct buddy_page **pages;
27 size_t head;
28 size_t tail;
29 size_t capacity;
30 atomic_size_t num_pages;
31 struct spinlock lock;
32};
33
34struct domain_free_queue {
35 struct {
36 paddr_t addr;
37 size_t pages;
38 } *queue;
39
40 size_t head;
41 size_t tail;
42 size_t capacity;
43 atomic_size_t num_elements;
44 atomic_bool free_in_progress; /* Simple flag to indicate that
45 * a free is in progress to prevent
46 * overly aggressive freeing. Enqueue/dequeue
47 * is still allowed since that uses
48 * the spinlock, but this just prevents
49 * aggressive concurrent access to the free
50 * queue so I don't free aggressively (not a
51 * 'race condition', just suboptimal) */
52
53 struct spinlock lock;
54};
55
56struct domain_zonelist_entry {
57 struct domain_buddy *domain;
58 uint8_t distance;
59 size_t free_pages;
60};
61
62struct domain_zonelist {
63 struct domain_zonelist_entry *entries;
64 size_t count;
65};
66
67struct domain_buddy_stats {
68 atomic_size_t alloc_count;
69 atomic_size_t free_count;
70 atomic_size_t remote_alloc_count;
71 atomic_size_t failed_alloc_count;
72 atomic_size_t interleaved_alloc_count;
73};
74
75struct domain_flush_worker {
76 struct domain_buddy *domain;
77 struct thread *thread;
78 struct semaphore sema;
79 atomic_bool enqueued;
80 bool stop;
81};
82
83struct domain_buddy {
84 struct domain_buddy_stats stats;
85 struct domain_zonelist zonelist;
86
87 struct buddy_page *buddy;
88 struct buddy_free_area *free_area;
89
90 struct domain_arena **arenas; /* One per core */
91 struct core **cores;
92 size_t core_count; /* # cores on this domain */
93
94 struct domain_free_queue *free_queue;
95
96 paddr_t start; /* physical start address */
97 paddr_t end; /* physical end address */
98 size_t length; /* total bytes */
99
100 atomic_size_t pages_used;
101 atomic_size_t total_pages;
102
103 struct spinlock lock;
104 struct domain_flush_worker worker;
105 struct domain *domain;
106};
107
108void domain_buddies_init(void);
109void domain_free(paddr_t address, size_t page_count);
110paddr_t domain_alloc(size_t pages, enum alloc_flags flags);
111bool domain_free_queue_enqueue(struct domain_free_queue *fq, paddr_t addr,
112 size_t pages);
113bool domain_free_queue_dequeue(struct domain_free_queue *fq, paddr_t *addr_out,
114 size_t *pages_out);
115bool domain_arena_push(struct domain_arena *arena, struct buddy_page *page);
116struct buddy_page *domain_arena_pop(struct domain_arena *arena);
117
118void domain_flush_free_queue(struct domain_buddy *domain,
119 struct domain_free_queue *queue);
120void domain_flush_thread(void *arg);
121void domain_enqueue_flush_worker(struct domain_flush_worker *worker);
122
123#define domain_for_each_arena(domain, arena_ptr) \
124 for (uint32_t __i = 0; \
125 (arena_ptr = ((domain)->arenas[__i]), __i < (domain)->core_count); \
126 __i++)
127
128static inline struct domain_buddy *domain_buddy_for_addr(paddr_t addr) {
129 for (size_t i = 0; i < global.domain_count; i++) {
130 struct domain_buddy *d = &global.domain_buddies[i];
131 if (addr >= d->start && addr < d->end)
132 return d;
133 }
134
135 /* None? */
136 return NULL;
137}
138
139static inline struct domain_buddy *domain_buddy_on_this_core(void) {
140 return smp_core()->domain->domain_buddy;
141}
142
143static inline struct domain_arena *domain_arena_on_this_core(void) {
144 return smp_core()->domain_arena;
145}
146
147static inline struct domain_free_queue *domain_free_queue_on_this_core(void) {
148 return domain_buddy_on_this_core()->free_queue;
149}
150
151static inline size_t *domain_rr_on_this_core(void) {
152 return &smp_core()->rr_current_domain;
153}
154
155static inline void domain_stat_alloc(struct domain_buddy *d, bool remote,
156 bool interleaved) {
157
158 atomic_fetch_add_explicit(&d->stats.alloc_count, 1, memory_order_relaxed);
159
160 if (remote)
161 atomic_fetch_add_explicit(&d->stats.remote_alloc_count, 1,
162 memory_order_relaxed);
163 if (interleaved)
164 atomic_fetch_add_explicit(&d->stats.interleaved_alloc_count, 1,
165 memory_order_relaxed);
166}
167
168static inline void domain_stat_free(struct domain_buddy *d) {
169 atomic_fetch_add_explicit(&d->stats.free_count, 1, memory_order_relaxed);
170}
171
172static inline void domain_stat_mark_interleaved(struct domain_buddy *d) {
173
174 atomic_fetch_add_explicit(&d->stats.interleaved_alloc_count, 1,
175 memory_order_relaxed);
176}
177
178static inline void domain_stat_failed_alloc(struct domain_buddy *d) {
179
180 atomic_fetch_add_explicit(&d->stats.failed_alloc_count, 1,
181 memory_order_relaxed);
182}
183
184static inline bool is_free_in_progress(struct domain_free_queue *fq) {
185 return atomic_load_explicit(&fq->free_in_progress, memory_order_relaxed);
186}
187
188static inline void mark_free_in_progress(struct domain_free_queue *fq, bool s) {
189 atomic_store_explicit(&fq->free_in_progress, s, memory_order_relaxed);
190}
191
192static inline struct buddy_page *buddy_page_for_addr(paddr_t address) {
193 return (struct buddy_page *) page_for_paddr(paddr: address);
194}
195
196static inline void free_from_buddy_internal(struct domain_buddy *target,
197 paddr_t address,
198 size_t page_count) {
199 enum irql irql = spin_lock(lock: &target->lock);
200 buddy_free_pages(addr: address, count: page_count, free_area: target->free_area,
201 total_pages: target->total_pages);
202 domain_stat_free(d: target);
203 atomic_fetch_sub(&target->pages_used, page_count);
204
205 spin_unlock(lock: &target->lock, old: irql);
206}
207
208struct domain *domain_alloc_pick_best_domain(struct domain *local, size_t pages,
209 size_t max_scan,
210 bool flexible_locality);
211