1/* @title: Folio */
2#pragma once
3#include <compiler.h>
4#include <math/pow.h>
5#include <mem/alloc.h>
6#include <mem/hhdm.h>
7#include <mem/page.h>
8#include <sch/irql.h>
9#include <stdatomic.h>
10#include <structures/list.h>
11#include <types/refcount.h>
12#include <types/types.h>
13
14struct anon_vma;
15struct file_vma_range;
16struct vma_range;
17
18/* folio_flags: 32-bit bitflags
19 *
20 * ┌────────────────────────────────────┐
21 * Bits │ 31..8 7..4 3..0 │
22 * Use │ (reserved) ..LU DMAO │
23 * └────────────────────────────────────┘
24 *
25 * O - On an LRU / reclaim list
26 * A - Anon: mapping is an anon_vma (clear => file_vma_range)
27 * M - Mapped at least once (mapcount has been nonzero)
28 * D - Dirty: needs writeback
29 * U - Uptodate: contents valid
30 * L - Locked: I/O in flight
31 *
32 * Reserved bits fill in as the pager arrives (writeback, referenced,
33 * reclaim, mlocked, swapbacked, ...). Width is uint32_t for headroom.
34 */
35enum folio_flags : uint32_t {
36 FOLIO_FLAGS_NONE = 0,
37 FOLIO_FLAG_ON_LRU = (1u << 0),
38 FOLIO_FLAG_ANON = (1u << 1),
39 FOLIO_FLAG_MAPPED = (1u << 2),
40 FOLIO_FLAG_DIRTY = (1u << 3),
41 FOLIO_FLAG_UPTODATE = (1u << 4),
42 FOLIO_FLAG_LOCKED = (1u << 5),
43};
44
45enum folio_tag : uintptr_t {
46 FOLIO_TAG_ADDRESS_SPACE = 0,
47 FOLIO_TAG_ANON = 1,
48};
49
50#define FOLIO_TAG_BITS 0x7
51
52struct folio {
53 struct page *base_page;
54 _Atomic enum folio_flags flags;
55 uint8_t order; /* 2^order pages */
56 refcount_t refcount;
57 mapcount_t mapcount;
58 void *mapping; /* tagged, anon_vma or file_vma_range */
59 pgoff_t index;
60 struct list_head lru; /* reclaim linkage */
61};
62uint32_t page_get_folio_index(struct page *p);
63bool page_is_folio_head(struct page *p);
64
65struct folio *folio_alloc_folio_struct();
66void folio_free_folio_struct(struct folio *f);
67
68struct folio *folio_alloc_internal(uint8_t order, enum alloc_flags flags,
69 enum alloc_behavior bh);
70void folio_free(struct folio *folio);
71
72/* page <-> folio backptrs, give every struct page
73 * folio at alloc, clear them at free */
74void folio_bind_pages(struct folio *f);
75void folio_unbind_pages(struct folio *f);
76
77/* Operates on the data pages */
78void folio_zero(struct folio *f);
79void folio_copy(const struct folio *src, struct folio *dst);
80
81/* LRU */
82void folio_add_lru(struct folio *f);
83void folio_del_lru(struct folio *f);
84
85/* Some struct page functions here for folio */
86static inline void page_set_folio(struct page *p, struct folio *f) {
87 page_set_tag(p, tag: PAGE_TAG_FOLIO);
88 page_set_payload(p, payload: (uintptr_t) f);
89}
90
91static inline struct folio *page_get_folio(const struct page *p) {
92 kassert(page_get_tag(p) == PAGE_TAG_FOLIO);
93 return (struct folio *) page_get_payload(p);
94}
95
96static inline void page_clear_folio(struct page *p) {
97 kassert(page_get_tag(p) == PAGE_TAG_FOLIO);
98 page_set_payload(p, payload: 0);
99 page_set_tag(p, tag: PAGE_TAG_NONE);
100}
101
102static inline bool folio_mapped(const struct folio *f) {
103 return atomic_load(&f->mapcount) > 0;
104}
105
106static inline unsigned long folio_nr_pages(const struct folio *f) {
107 return pow2(n: f->order);
108}
109
110static inline void folio_set_tag(struct folio *f, enum folio_tag t) {
111 f->mapping = (void *) ((uintptr_t) f->mapping & ~FOLIO_TAG_BITS);
112 f->mapping = (void *) ((uintptr_t) f->mapping | t);
113}
114
115static inline enum folio_tag folio_get_tag(const struct folio *f) {
116 return (enum folio_tag)((uintptr_t) f->mapping & FOLIO_TAG_BITS);
117}
118
119static inline bool folio_test_flag(const struct folio *f, enum folio_flags m) {
120 return atomic_load_explicit(&f->flags, memory_order_acquire) & m;
121}
122
123static inline void folio_set_flag(struct folio *f, enum folio_flags m) {
124 atomic_fetch_or_explicit(&f->flags, m, memory_order_release);
125}
126
127static inline void folio_clear_flag(struct folio *f, enum folio_flags m) {
128 atomic_fetch_and_explicit(&f->flags, (enum folio_flags) ~(uint32_t) m,
129 memory_order_release);
130}
131
132static inline bool folio_test_set_flag(struct folio *f, enum folio_flags m) {
133 return atomic_fetch_or_explicit(&f->flags, m, memory_order_acq_rel) & m;
134}
135
136static inline bool folio_get(struct folio *f) {
137 return refcount_inc_not_zero(rc: &f->refcount);
138}
139
140static inline void folio_put(struct folio *f) {
141 if (refcount_dec_and_test(rc: &f->refcount))
142 folio_free(folio: f);
143}
144
145/* Nth page */
146static inline struct page *folio_get_page(const struct folio *f, size_t n) {
147 pfn_t nth_pfn = page_get_pfn(bp: f->base_page) + n;
148 return page_for_pfn(pfn: nth_pfn);
149}
150
151static inline pfn_t folio_get_pfn(const struct folio *f) {
152 return page_get_pfn(bp: f->base_page);
153}
154
155/* reverse of folio_page(): a mapped frame -> its folio, via the page
156 * backpointer. The fault/rmap paths take a PTE's phys and land back here. */
157static inline struct folio *folio_from_page(const struct page *p) {
158 return p ? page_get_folio(p) : NULL;
159}
160
161static inline struct folio *folio_from_paddr(paddr_t pa) {
162 return folio_from_page(p: page_for_paddr(paddr: pa));
163}
164
165static inline paddr_t folio_get_paddr(const struct folio *f) {
166 return page_get_paddr(bp: f->base_page);
167}
168
169/* HHDM map of base */
170static inline vaddr_t folio_get_vaddr(const struct folio *f) {
171 return hhdm_paddr_to_vaddr(p: folio_get_paddr(f));
172}
173
174static inline paddr_t folio_get_paddr_for(const struct folio *f, size_t n) {
175 return folio_get_paddr(f) + n * PAGE_SIZE;
176}
177
178/* HHDM map of page n */
179static inline vaddr_t folio_get_vaddr_for(const struct folio *f, size_t n) {
180 return hhdm_paddr_to_vaddr(p: folio_get_paddr_for(f, n));
181}
182
183/* state */
184static inline void folio_mark_dirty(struct folio *f) {
185 folio_set_flag(f, m: FOLIO_FLAG_DIRTY);
186}
187
188static inline void folio_mark_uptodate(struct folio *f) {
189 folio_set_flag(f, m: FOLIO_FLAG_UPTODATE);
190}
191
192static inline void folio_set_anon(struct folio *f, struct anon_vma *av,
193 pgoff_t index) {
194 kassert(!folio_test_set_flag(f, FOLIO_FLAG_ANON));
195 f->mapping = av;
196 f->index = index;
197 folio_set_tag(f, t: FOLIO_TAG_ANON);
198}
199
200static inline bool folio_is_anon(const struct folio *f) {
201 /* check tag, no atomic load */
202 return folio_get_tag(f) == FOLIO_TAG_ANON;
203}
204
205static inline bool folio_is_file_vma_range(const struct folio *f) {
206 return folio_get_tag(f) == FOLIO_TAG_ADDRESS_SPACE;
207}
208
209static inline struct anon_vma *folio_get_anon_vma(const struct folio *f) {
210 kassert(folio_is_anon(f));
211 return (struct anon_vma *) ((uintptr_t) f->mapping & ~FOLIO_TAG_BITS);
212}
213
214static inline struct file_vma_range *
215folio_get_file_vma_range(const struct folio *f) {
216 kassert(folio_is_file_vma_range(f));
217 return f->mapping;
218}
219
220static inline bool folio_trylock(struct folio *f) {
221 return !folio_test_set_flag(f, m: FOLIO_FLAG_LOCKED);
222}
223
224static inline void folio_lock(struct folio *f) {
225 while (true) {
226 if (folio_trylock(f))
227 break;
228
229 while (folio_test_flag(f, m: FOLIO_FLAG_LOCKED))
230 cpu_relax();
231 }
232}
233
234static inline void folio_unlock(struct folio *f) {
235 folio_clear_flag(f, m: FOLIO_FLAG_LOCKED);
236}
237
238/* mapcount, number of PTEs pointing at this folio */
239static inline void folio_mapcount_inc(struct folio *f) {
240 refcount_inc(rc: &f->mapcount); /* Using refcount functions is fine and safe */
241}
242
243static inline bool
244folio_mapcount_dec(struct folio *f) { /* true if dropped to 0 */
245 return refcount_dec_and_test(rc: &f->mapcount);
246}
247
248#define folio_for_each_page_struct(f, p) \
249 for (size_t __i = 0; \
250 __i < folio_nr_pages(f) && ((p) = folio_get_page((f), __i), true); \
251 __i++)
252
253#define folio_for_each_page_paddr(f, p) \
254 for (size_t __i = 0; __i < folio_nr_pages(f) && \
255 ((p) = folio_get_paddr_for((f), __i), true); \
256 __i++)
257
258#define folio_for_each_page_vaddr(f, p) \
259 for (size_t __i = 0; __i < folio_nr_pages(f) && \
260 ((p) = folio_get_vaddr_for((f), __i), true); \
261 __i++)
262
263#define folio_alloc_1(sz) \
264 folio_alloc_internal((sz), ALLOC_FLAGS_DEFAULT, ALLOC_BEHAVIOR_DEFAULT)
265#define folio_alloc_2(sz, fl) \
266 folio_alloc_internal((sz), (fl), ALLOC_BEHAVIOR_DEFAULT)
267#define folio_alloc_3(sz, fl, bh) folio_alloc_internal((sz), (fl), (bh))
268#define folio_alloc(...) \
269 _DISPATCH(folio_alloc, PP_NARG(__VA_ARGS__))(__VA_ARGS__)
270