1/* @title: Allocator API */
2#pragma once
3#include <console/printf.h>
4#include <log.h>
5#include <mem/alloc_api_internal.h>
6#include <stdbool.h>
7#include <stddef.h>
8#include <stdint.h>
9
10LOG_SITE_EXTERN(slab);
11LOG_HANDLE_EXTERN(slab_flags);
12
13/*
14 * TL;DR: ALLOCATION FLAGS TELL THE ALLOCATOR WHAT KIND OF MEMORY
15 * YOU WANT, ALLOCATION BEHAVIORS TELL THE ALLOCATOR
16 * WHAT IT IS ALLOWED TO DO TO GET THAT MEMORY.
17 */
18
19/* ─────────────────────────── ALLOC FLAGS ─────────────────────────── */
20
21#define ALLOC_LOCALITY_SHIFT 25
22#define ALLOC_CLASS_SHIFT 28
23#define ALLOC_CLASS_MASK 0xF
24
25/* The larger the locality, the closer it must be */
26#define ALLOC_LOCALITY_MAX 7
27#define ALLOC_LOCALITY_MIN 0
28#define ALLOC_LOCALITY_MASK 0x7
29
30#define ALLOC_LOCALITY_FROM_FLAGS(flags) \
31 (((flags) >> ALLOC_LOCALITY_SHIFT) & ALLOC_LOCALITY_MASK)
32
33#define ALLOC_LOCALITY_TO_FLAGS(locality) \
34 (((locality) & ALLOC_LOCALITY_MASK) << ALLOC_LOCALITY_SHIFT)
35
36#define ALLOC_FLAG_TEST(flags, mask) (flags & mask)
37#define ALLOC_FLAG_CLASS(flags) \
38 ((flags >> ALLOC_CLASS_SHIFT) & ALLOC_CLASS_MASK)
39
40/* Bits 16..23 are available, this gives the 0-indexed Nth available bit */
41#define ALLOC_FLAG_AVAIL_BIT(n) (1 << (ALLOC_CLASS_SHIFT - 4 + (n)))
42
43/* alloc_flags: 32 bit bitflags
44 *
45 * ┌──────────────────────────────────────────────────────┐
46 * Bits │ 31..28 27..24 23..20 19..16 15..12 11..8 7..4 3..0 │
47 * Use │ %%%% ###* AAAA AAAA **** **** **Zc MPFC │
48 * └──────────────────────────────────────────────────────┘
49 *
50 * C - "Prefer cache alignment"
51 *
52 * F - "Allow flexible NUMA locality"
53 *
54 * P - "Allow memory to be pageable"
55 *
56 * M - "Allow memory to be movable"
57 *
58 * c - Physically contiguous - applies only to page_alloc
59 *
60 * Z - Zero on alloc, memory is zeroed
61 *
62 * ### - Locality bits
63 *
64 * %%%% - Allocation class bits
65 *
66 * A - Unused (available)
67 * * - Unused (unavailable)
68 *
69 */
70
71/* Flags define properties regarding
72 * the memory the allocator will return */
73enum alloc_flags : uint32_t {
74 /* Cache alignment */
75 ALLOC_FLAG_PREFER_CACHE_ALIGNED = (1 << 0),
76 ALLOC_FLAG_NO_CACHE_ALIGN = 0,
77
78 /* Flexible locality */
79 ALLOC_FLAG_FLEXIBLE_LOCALITY = (1 << 1),
80 ALLOC_FLAG_STRICT_LOCALITY = 0,
81
82 /* Pageable */
83 ALLOC_FLAG_PAGEABLE = (1 << 2),
84 ALLOC_FLAG_NONPAGEABLE = 0,
85
86 /* Movable */
87 ALLOC_FLAG_MOVABLE = (1 << 3),
88 ALLOC_FLAG_NONMOVABLE = 0,
89
90 /* Contiguous - applies only to page_alloc() */
91 ALLOC_FLAG_CONTIGUOUS = (1 << 4),
92 ALLOC_FLAG_NONCONTIGUOUS = 0,
93
94 /* Zero on alloc */
95 ALLOC_FLAG_ZERO_ON_ALLOC = (1 << 5),
96 ALLOC_FLAG_NON_ZERO = 0,
97
98 /* Allocation classes */
99 ALLOC_FLAG_CLASS_DEFAULT = (1 << ALLOC_CLASS_SHIFT),
100 ALLOC_FLAG_CLASS_INTERLEAVED = (2 << ALLOC_CLASS_SHIFT),
101 ALLOC_FLAG_CLASS_HIGH_BANDWIDTH = (3 << ALLOC_CLASS_SHIFT),
102};
103
104#define ALLOC_FLAGS_NONE 0
105
106/* Bits 6..15 and 24 */
107#define ALLOC_FLAGS_UNAVAILABLE_BITS ((1 << 24) | (0x3FF << 16))
108
109#define ALLOC_FLAGS_DEFAULT \
110 (ALLOC_FLAG_CLASS_DEFAULT | ALLOC_FLAG_FLEXIBLE_LOCALITY | \
111 ALLOC_FLAG_NONMOVABLE | ALLOC_FLAG_NONPAGEABLE | \
112 ALLOC_FLAG_NO_CACHE_ALIGN | ALLOC_LOCALITY_TO_FLAGS(ALLOC_LOCALITY_MIN))
113#define ALLOC_FLAGS_ZERO (ALLOC_FLAGS_DEFAULT | ALLOC_FLAG_ZERO_ON_ALLOC)
114
115#define ALLOC_FLAGS_PAGEABLE \
116 ALLOC_FLAG_PAGEABLE | ALLOC_FLAG_CLASS_DEFAULT | \
117 ALLOC_FLAG_FLEXIBLE_LOCALITY
118
119static inline bool alloc_flags_valid(enum alloc_flags flags) {
120 /* If an unavailable bit is set, it is not valid */
121 return !(flags & ALLOC_FLAGS_UNAVAILABLE_BITS);
122}
123
124/* ─────────────────────────── ALLOC BEHAVIORS ─────────────────────────── */
125
126#define ALLOC_BEHAVIOR_FLAG_SHIFT 4
127#define ALLOC_BEHAVIOR_MASK (0xF)
128#define ALLOC_BEHAVIOR_AVAILABLE_SHIFT 12
129#define ALLOC_BEHAVIOR_AVAIL_BIT(n) (1 << (ALLOC_BEHAVIOR_AVAILABLE_SHIFT + n))
130
131/* alloc_behavior: 16 bits for a behavior and flags
132 *
133 * ┌───────────────────────────┐
134 * Bits │ 15..12 11..8 7..4 3..0 │
135 * Use │ AAAA **** **MF %%%% │
136 * └───────────────────────────┘
137 *
138 * %%%% - Allocation behavior bits
139 *
140 * F - "Prefer fast allocation" -- may fail fast/early
141 * M - Perform "minimal" allocation -- e.g. do not bother with
142 * new slab cache construction
143 * A - Unused (Available)
144 * * - Unused (Unavailable)
145 *
146 */
147
148/* Behaviors define what the allocator is
149 * allowed to do in a given invocation. */
150
151/* Allocation behavior restricts flags. If non-faulting behaviors
152 * are selected, then the allocator cannot allocate pageable memory */
153enum alloc_behavior : uint16_t {
154 ALLOC_BEHAVIOR_NORMAL,
155 ALLOC_BEHAVIOR_ATOMIC,
156 ALLOC_BEHAVIOR_NO_WAIT,
157 ALLOC_BEHAVIOR_NO_RECLAIM,
158 ALLOC_BEHAVIOR_FAULT_SAFE,
159 ALLOC_BEHAVIOR_FLAG_FAST = 1 << ALLOC_BEHAVIOR_FLAG_SHIFT,
160
161 /* Used by various allocators to prevent
162 * recursion in things they interact with */
163 ALLOC_BEHAVIOR_FLAG_MINIMAL = 1 << (ALLOC_BEHAVIOR_FLAG_SHIFT + 1),
164
165};
166#define ALLOC_BEHAVIOR_DEFAULT ALLOC_BEHAVIOR_NORMAL
167
168/* ────────────────────────────────────────────────────────────────────────── */
169
170/* Allocation Behavior Semantics
171 *
172 * Behaviors define what the allocator is ALLOWED to do
173 * Behaviors allow/forbid blocking, faulting, and use in ISRs
174 *
175 * ┌───────────────────────┐
176 * │ Allowed Behaviors │
177 * ┌────────────┼───────┬───────┬───────┼──────────────────────────────────────┐
178 * │ Behavior │ Fault │ Block │ ISR │ Comments │
179 * ├────────────┼───────┼───────┼───────┼──────────────────────────────────────┤
180 * │ NORMAL │ ✅ │ ✅ │ ❌ │ General-purpose + unrestricted │
181 * ├────────────┼───────┼───────┼───────┼──────────────────────────────────────┤
182 * │ ATOMIC │ ❌ │ ❌ │ ✅ │ For ISRs or hard contexts. Only uses │
183 * │ │ │ │ │ preresident, nonpageable memory │
184 * ├────────────┼───────┼───────┼───────┼──────────────────────────────────────┤
185 * │ NO_WAIT │ ✅ │ ❌ │ ❌ │ Nonblocking but can fault. For soft │
186 * │ │ │ │ │ realtime / fastpath code │
187 * ├────────────┼───────┼───────┼───────┼──────────────────────────────────────┤
188 * │ NO_RECLAIM │ ✅ │ ✅ │ ❌ │ May block but cannot trigger GC or │
189 * │ │ │ │ │ reclaim. For paging or lowmem code │
190 * ├────────────┼───────┼───────┼───────┼──────────────────────────────────────┤
191 * │ FAULT_SAFE │ ❌ │ ✅ │ ❌ │ Must not fault, but may block │
192 * └────────────┴───────┴───────┴───────┴──────────────────────────────────────┘
193 *
194 */
195
196/* ────────────────────────────────────────────────────────────────────────── */
197
198/* Extract base behavior (mask out flags) */
199static inline enum alloc_behavior alloc_behavior_base(enum alloc_behavior raw) {
200 return raw & ALLOC_BEHAVIOR_MASK;
201}
202
203/* Does this behavior allow page faults? */
204static inline bool alloc_behavior_may_fault(enum alloc_behavior raw) {
205 switch (alloc_behavior_base(raw)) {
206 case ALLOC_BEHAVIOR_ATOMIC:
207 case ALLOC_BEHAVIOR_FAULT_SAFE: return false;
208 default: return true;
209 }
210}
211
212/* Does this behavior allow blocking or waiting? */
213static inline bool alloc_behavior_may_block(enum alloc_behavior raw) {
214 switch (alloc_behavior_base(raw)) {
215 case ALLOC_BEHAVIOR_ATOMIC:
216 case ALLOC_BEHAVIOR_NO_WAIT: return false;
217 default: return true;
218 }
219}
220
221/* Is this behavior ISR-safe? */
222static inline bool alloc_behavior_is_isr_safe(enum alloc_behavior raw) {
223 return alloc_behavior_base(raw) == ALLOC_BEHAVIOR_ATOMIC;
224}
225
226/* Does this behavior skip reclamation (e.g., GC, slab draining)? */
227static inline bool alloc_behavior_no_reclaim(enum alloc_behavior raw) {
228 return alloc_behavior_base(raw) == ALLOC_BEHAVIOR_NO_RECLAIM ||
229 alloc_behavior_base(raw) == ALLOC_BEHAVIOR_ATOMIC;
230}
231
232/* Fast hint: should this allocation prefer short paths? */
233static inline bool alloc_behavior_is_fast(enum alloc_behavior raw) {
234 return (raw & ALLOC_BEHAVIOR_FLAG_FAST);
235}
236
237static inline bool alloc_flag_behavior_verify(enum alloc_flags f,
238 enum alloc_behavior behavior) {
239 bool may_fault = alloc_behavior_may_fault(raw: behavior);
240 bool flag_requires_residency = !(f & ALLOC_FLAG_PAGEABLE);
241 bool flag_can_fault = (f & ALLOC_FLAG_MOVABLE) || (f & ALLOC_FLAG_PAGEABLE);
242
243 /* Non-faulting behavior cannot tolerate pageable or movable allocations */
244 if (!may_fault && flag_can_fault)
245 return false;
246
247 /* ISR-safe behavior must use nonpageable memory */
248 if (alloc_behavior_is_isr_safe(raw: behavior) && !flag_requires_residency)
249 return false;
250
251 return true;
252}
253
254static inline void alloc_request_sanitize(enum alloc_flags *f,
255 enum alloc_behavior *b) {
256 if (!alloc_flag_behavior_verify(f: *f, behavior: *b)) {
257 /* Force safety first */
258 log(LOG_SITE(slab), LOG_HANDLE(slab_flags), LOG_WARN,
259 "Allocation flag discrepancy");
260 if (alloc_behavior_is_isr_safe(raw: *b) || !alloc_behavior_may_fault(raw: *b)) {
261 *f &= ~(ALLOC_FLAG_PAGEABLE | ALLOC_FLAG_MOVABLE);
262 *f |= ALLOC_FLAG_NONPAGEABLE;
263 }
264 }
265}
266
267void *kmalloc_new(size_t size, enum alloc_flags flags,
268 enum alloc_behavior behavior);
269void kfree_new(void *ptr, enum alloc_behavior behavior);
270void *kmalloc_from_domain(domain_id_t domain, size_t size);
271
272void *kmalloc_internal(size_t size, enum alloc_flags flags,
273 enum alloc_behavior behavior);
274void *krealloc_internal(void *ptr, size_t size, enum alloc_flags flags,
275 enum alloc_behavior behavior);
276void kfree_internal(void *ptr, enum alloc_behavior behavior);
277size_t ksize(void *ptr);
278void *kmalloc_aligned_internal(size_t size, size_t align,
279 enum alloc_flags flags,
280 enum alloc_behavior behavior);
281void kfree_aligned_internal(void *ptr, enum alloc_behavior behavior);
282void kfree_defer_irq(void *ptr);
283void *kmalloc_pages(size_t n_pages, enum alloc_flags flags);
284bool kmalloc_ptr_in_slab_validate(void *ptr);
285