1/* @title: Allocator API */
2#pragma once
3#include <console/printf.h>
4#include <log.h>
5#include <mem/alloc_api_internal.h>
6#include <stdbool.h>
7#include <stddef.h>
8#include <stdint.h>
9
10LOG_SITE_EXTERN(slab);
11LOG_HANDLE_EXTERN(slab_flags);
12
13/*
14 * TL;DR: ALLOCATION FLAGS TELL THE ALLOCATOR WHAT KIND OF MEMORY
15 * YOU WANT, ALLOCATION BEHAVIORS TELL THE ALLOCATOR
16 * WHAT IT IS ALLOWED TO DO TO GET THAT MEMORY.
17 */
18
19/* ─────────────────────────── ALLOC FLAGS ─────────────────────────── */
20
21#define ALLOC_LOCALITY_SHIFT 9
22#define ALLOC_CLASS_SHIFT 12
23#define ALLOC_CLASS_MASK 0xF
24
25/* The larger the locality, the closer it must be */
26#define ALLOC_LOCALITY_MAX 7
27#define ALLOC_LOCALITY_MIN 0
28#define ALLOC_LOCALITY_MASK 0x7
29
30#define ALLOC_LOCALITY_FROM_FLAGS(flags) \
31 (((flags) >> ALLOC_LOCALITY_SHIFT) & ALLOC_LOCALITY_MASK)
32
33#define ALLOC_LOCALITY_TO_FLAGS(locality) \
34 (((locality) & ALLOC_LOCALITY_MASK) << ALLOC_LOCALITY_SHIFT)
35
36#define ALLOC_FLAG_TEST(flags, mask) (flags & mask)
37#define ALLOC_FLAG_CLASS(flags) \
38 ((flags >> ALLOC_CLASS_SHIFT) & ALLOC_CLASS_MASK)
39
40/* alloc_flags: 16 bit bitflags
41 *
42 * ┌───────────────────────────┐
43 * Bits │ 15..12 11..8 7..4 3..0 │
44 * Use │ %%%% ###A A*Zc MPFC │
45 * └───────────────────────────┘
46 *
47 * C - "Prefer cache alignment"
48 *
49 * F - "Allow flexible NUMA locality"
50 *
51 * P - "Allow memory to be pageable"
52 *
53 * M - "Allow memory to be movable"
54 *
55 * c - Physically contiguous - applies only to page_alloc
56 *
57 * Z - Zero on alloc, memory is zeroed
58 *
59 * ### - Locality bits
60 *
61 * A - Unused (available)
62 * * - Unused (unavailable)
63 * %%%% - Allocation class bits
64 *
65 */
66
67/* Flags define properties regarding
68 * the memory the allocator will return */
69enum alloc_flags : uint16_t {
70 /* Cache alignment */
71 ALLOC_FLAG_PREFER_CACHE_ALIGNED = (1 << 0),
72 ALLOC_FLAG_NO_CACHE_ALIGN = 0,
73
74 /* Flexible locality */
75 ALLOC_FLAG_FLEXIBLE_LOCALITY = (1 << 1),
76 ALLOC_FLAG_STRICT_LOCALITY = 0,
77
78 /* Pageable */
79 ALLOC_FLAG_PAGEABLE = (1 << 2),
80 ALLOC_FLAG_NONPAGEABLE = 0,
81
82 /* Movable */
83 ALLOC_FLAG_MOVABLE = (1 << 3),
84 ALLOC_FLAG_NONMOVABLE = 0,
85
86 /* Contiguous - applies only to page_alloc() */
87 ALLOC_FLAG_CONTIGUOUS = (1 << 4),
88 ALLOC_FLAG_NONCONTIGUOUS = 0,
89
90 /* Zero on alloc */
91 ALLOC_FLAG_ZERO_ON_ALLOC = (1 << 5),
92 ALLOC_FLAG_NON_ZERO = 0,
93
94 /* Allocation classes */
95 ALLOC_FLAG_CLASS_DEFAULT = (1 << ALLOC_CLASS_SHIFT),
96 ALLOC_FLAG_CLASS_INTERLEAVED = (2 << ALLOC_CLASS_SHIFT),
97 ALLOC_FLAG_CLASS_HIGH_BANDWIDTH = (3 << ALLOC_CLASS_SHIFT),
98};
99
100#define ALLOC_FLAGS_NONE 0
101#define ALLOC_FLAGS_UNAVAILABLE_BITS 0x40
102#define ALLOC_FLAGS_DEFAULT \
103 (ALLOC_FLAG_CLASS_DEFAULT | ALLOC_FLAG_FLEXIBLE_LOCALITY | \
104 ALLOC_FLAG_NONMOVABLE | ALLOC_FLAG_NONPAGEABLE | \
105 ALLOC_FLAG_NO_CACHE_ALIGN | ALLOC_LOCALITY_TO_FLAGS(ALLOC_LOCALITY_MIN))
106#define ALLOC_FLAGS_ZERO (ALLOC_FLAGS_DEFAULT | ALLOC_FLAG_ZERO_ON_ALLOC)
107
108#define ALLOC_FLAGS_PAGEABLE \
109 ALLOC_FLAG_PAGEABLE | ALLOC_FLAG_CLASS_DEFAULT | \
110 ALLOC_FLAG_FLEXIBLE_LOCALITY
111
112static inline bool alloc_flags_valid(enum alloc_flags flags) {
113 /* If an unavailable bit is set, it is not valid */
114 return !(flags & ALLOC_FLAGS_UNAVAILABLE_BITS);
115}
116
117/* ─────────────────────────── ALLOC BEHAVIORS ─────────────────────────── */
118
119#define ALLOC_BEHAVIOR_FLAG_SHIFT 4
120#define ALLOC_BEHAVIOR_MASK (0xF)
121#define ALLOC_BEHAVIOR_AVAILABLE_SHIFT 12
122
123/* alloc_behavior: 16 bits for a behavior and flags
124 *
125 * ┌───────────────────────────┐
126 * Bits │ 15..12 11..8 7..4 3..0 │
127 * Use │ AAAA **** **MF %%%% │
128 * └───────────────────────────┘
129 *
130 * %%%% - Allocation behavior bits
131 *
132 * F - "Prefer fast allocation" -- may fail fast/early
133 * M - Perform "minimal" allocation -- e.g. do not bother with
134 * new slab cache construction
135 * A - Unused (Available)
136 * * - Unused (Unavailable)
137 *
138 */
139
140/* Behaviors define what the allocator is
141 * allowed to do in a given invocation. */
142
143/* Allocation behavior restricts flags. If non-faulting behaviors
144 * are selected, then the allocator cannot allocate pageable memory */
145enum alloc_behavior : uint16_t {
146 ALLOC_BEHAVIOR_NORMAL,
147 ALLOC_BEHAVIOR_ATOMIC,
148 ALLOC_BEHAVIOR_NO_WAIT,
149 ALLOC_BEHAVIOR_NO_RECLAIM,
150 ALLOC_BEHAVIOR_FAULT_SAFE,
151 ALLOC_BEHAVIOR_FLAG_FAST = 1 << ALLOC_BEHAVIOR_FLAG_SHIFT,
152
153 /* Used by various allocators to prevent
154 * recursion in things they interact with */
155 ALLOC_BEHAVIOR_FLAG_MINIMAL = 1 << (ALLOC_BEHAVIOR_FLAG_SHIFT + 1),
156
157};
158#define ALLOC_BEHAVIOR_DEFAULT ALLOC_BEHAVIOR_NORMAL
159
160/* ────────────────────────────────────────────────────────────────────────── */
161
162/* Allocation Behavior Semantics
163 *
164 * Behaviors define what the allocator is ALLOWED to do
165 * Behaviors allow/forbid blocking, faulting, and use in ISRs
166 *
167 * ┌───────────────────────┐
168 * │ Allowed Behaviors │
169 * ┌────────────┼───────┬───────┬───────┼──────────────────────────────────────┐
170 * │ Behavior │ Fault │ Block │ ISR │ Comments │
171 * ├────────────┼───────┼───────┼───────┼──────────────────────────────────────┤
172 * │ NORMAL │ ✅ │ ✅ │ ❌ │ General-purpose + unrestricted │
173 * ├────────────┼───────┼───────┼───────┼──────────────────────────────────────┤
174 * │ ATOMIC │ ❌ │ ❌ │ ✅ │ For ISRs or hard contexts. Only uses │
175 * │ │ │ │ │ preresident, nonpageable memory │
176 * ├────────────┼───────┼───────┼───────┼──────────────────────────────────────┤
177 * │ NO_WAIT │ ✅ │ ❌ │ ❌ │ Nonblocking but can fault. For soft │
178 * │ │ │ │ │ realtime / fastpath code │
179 * ├────────────┼───────┼───────┼───────┼──────────────────────────────────────┤
180 * │ NO_RECLAIM │ ✅ │ ✅ │ ❌ │ May block but cannot trigger GC or │
181 * │ │ │ │ │ reclaim. For paging or lowmem code │
182 * ├────────────┼───────┼───────┼───────┼──────────────────────────────────────┤
183 * │ FAULT_SAFE │ ❌ │ ✅ │ ❌ │ Must not fault, but may block │
184 * └────────────┴───────┴───────┴───────┴──────────────────────────────────────┘
185 *
186 */
187
188/* ────────────────────────────────────────────────────────────────────────── */
189
190/* Extract base behavior (mask out flags) */
191static inline enum alloc_behavior alloc_behavior_base(enum alloc_behavior raw) {
192 return raw & ALLOC_BEHAVIOR_MASK;
193}
194
195/* Does this behavior allow page faults? */
196static inline bool alloc_behavior_may_fault(enum alloc_behavior raw) {
197 switch (alloc_behavior_base(raw)) {
198 case ALLOC_BEHAVIOR_ATOMIC:
199 case ALLOC_BEHAVIOR_FAULT_SAFE: return false;
200 default: return true;
201 }
202}
203
204/* Does this behavior allow blocking or waiting? */
205static inline bool alloc_behavior_may_block(enum alloc_behavior raw) {
206 switch (alloc_behavior_base(raw)) {
207 case ALLOC_BEHAVIOR_ATOMIC:
208 case ALLOC_BEHAVIOR_NO_WAIT: return false;
209 default: return true;
210 }
211}
212
213/* Is this behavior ISR-safe? */
214static inline bool alloc_behavior_is_isr_safe(enum alloc_behavior raw) {
215 return alloc_behavior_base(raw) == ALLOC_BEHAVIOR_ATOMIC;
216}
217
218/* Does this behavior skip reclamation (e.g., GC, slab draining)? */
219static inline bool alloc_behavior_no_reclaim(enum alloc_behavior raw) {
220 return alloc_behavior_base(raw) == ALLOC_BEHAVIOR_NO_RECLAIM ||
221 alloc_behavior_base(raw) == ALLOC_BEHAVIOR_ATOMIC;
222}
223
224/* Fast hint: should this allocation prefer short paths? */
225static inline bool alloc_behavior_is_fast(enum alloc_behavior raw) {
226 return (raw & ALLOC_BEHAVIOR_FLAG_FAST);
227}
228
229static inline bool alloc_flag_behavior_verify(enum alloc_flags f,
230 enum alloc_behavior behavior) {
231 bool may_fault = alloc_behavior_may_fault(raw: behavior);
232 bool flag_requires_residency = (f & ALLOC_FLAG_NONPAGEABLE);
233 bool flag_can_fault = (f & ALLOC_FLAG_MOVABLE) || (f & ALLOC_FLAG_PAGEABLE);
234
235 /* Non-faulting behavior cannot tolerate pageable or movable allocations */
236 if (!may_fault && flag_can_fault)
237 return false;
238
239 /* ISR-safe behavior must use nonpageable memory */
240 if (alloc_behavior_is_isr_safe(raw: behavior) && !flag_requires_residency)
241 return false;
242
243 return true;
244}
245
246static inline void alloc_request_sanitize(enum alloc_flags *f,
247 enum alloc_behavior *b) {
248 if (!alloc_flag_behavior_verify(f: *f, behavior: *b)) {
249 /* Force safety first */
250 log(LOG_SITE(slab), LOG_HANDLE(slab_flags), LOG_WARN,
251 "Allocation flag discrepancy");
252 if (alloc_behavior_is_isr_safe(raw: *b) || !alloc_behavior_may_fault(raw: *b)) {
253 *f &= ~(ALLOC_FLAG_PAGEABLE | ALLOC_FLAG_MOVABLE);
254 *f |= ALLOC_FLAG_NONPAGEABLE;
255 }
256 }
257}
258
259void *kmalloc_new(size_t size, enum alloc_flags flags,
260 enum alloc_behavior behavior);
261void kfree_new(void *ptr, enum alloc_behavior behavior);
262void *kmalloc_from_domain(size_t domain, size_t size);
263
264void *kmalloc_internal(size_t size, enum alloc_flags flags,
265 enum alloc_behavior behavior);
266void *krealloc_internal(void *ptr, size_t size, enum alloc_flags flags,
267 enum alloc_behavior behavior);
268void kfree_internal(void *ptr, enum alloc_behavior behavior);
269size_t ksize(void *ptr);
270void *kmalloc_aligned_internal(size_t size, size_t align,
271 enum alloc_flags flags,
272 enum alloc_behavior behavior);
273void kfree_aligned_internal(void *ptr, enum alloc_behavior behavior);
274void kfree_defer_irq(void *ptr);
275