1#include <math/units.h>
2#include <mem/address_range.h>
3#include <mem/asan.h>
4#include <mem/hhdm.h>
5#include <mem/page.h>
6#include <mem/page_alloc.h>
7#include <mem/page_fault.h>
8#include <mem/pmm.h>
9#include <mem/vas.h>
10
11ALLOC_FLAG_EX_REGISTER(PG, CONTIGUOUS);
12static bool page_alloc_pf_valid(struct page_fault_info *pfi);
13
14static struct page_fault_handler_ops page_alloc_pfho = {
15 .alloc_pages = NULL,
16 .update_after_map = NULL,
17 .is_valid_fault = page_alloc_pf_valid,
18};
19
20static struct page_fault_handler page_alloc_pfh = {
21 .ops = &page_alloc_pfho,
22};
23
24static struct vas *page_alloc_vas = NULL;
25ADDRESS_RANGE_DEFINE(page_alloc, .align = PAGE_SIZE,
26 .flags = ADDRESS_RANGE_DYNAMIC, .size = TIB(8),
27 .page_fault_handler = &page_alloc_pfh);
28
29void page_alloc_init(void) {
30 page_alloc_vas = vas_bootstrap_from(ar: &ADDRESS_RANGE(page_alloc));
31}
32
33static void page_alloc_vas_release(vaddr_t virt, size_t page_count,
34 size_t nr_mapped) {
35 for (size_t i = 0; i < nr_mapped; i++) {
36 vaddr_t vaddr = virt + i * PAGE_SIZE;
37 paddr_t phys = (paddr_t) vmm_get_phys(vaddr);
38 vmm_unmap_page(vaddr);
39
40 if (phys != PADDR_MAX)
41 pmm_free_page(addr: phys);
42 }
43
44 vas_free(vas: page_alloc_vas, addr: virt, size: page_count * PAGE_SIZE);
45}
46
47static void *page_alloc_vas_mapped_pages(size_t page_count,
48 enum alloc_flags flags,
49 bool demand_paged) {
50 vaddr_t virt = vas_alloc(vas: page_alloc_vas, size: page_count * PAGE_SIZE, PAGE_SIZE);
51 if (!virt)
52 return NULL;
53
54 page_flags_t page_flags = PAGE_PRESENT | PAGE_WRITE | PAGE_XD;
55 bool zero = flags & ALLOC_FLAG_ZERO_ON_ALLOC;
56
57 for (size_t i = 0; i < page_count; i++) {
58 vaddr_t vaddr = virt + i * PAGE_SIZE;
59
60 if (!demand_paged || !zero) {
61 uintptr_t phys = pmm_alloc_page(flags);
62 if (!phys) {
63 page_alloc_vas_release(virt, page_count, nr_mapped: i);
64 return NULL;
65 }
66
67 if (vmm_map_page(vaddr, phys, .page_flags = page_flags) < 0) {
68 pmm_free_page(addr: phys);
69 page_alloc_vas_release(virt, page_count, nr_mapped: i);
70 return NULL;
71 }
72 } else {
73 enum err e =
74 vmm_mark_demand_page(vaddr, DEMAND_PAGE_FLAG_ZERO_MEMORY |
75 DEMAND_PAGE_FLAG_WRITABLE);
76 if (e < 0) {
77 page_alloc_vas_release(virt, page_count, nr_mapped: i);
78 return NULL;
79 }
80 }
81 }
82
83 return (void *) virt;
84}
85
86/* Must be in vas, that's the only check */
87static bool page_alloc_pf_valid(struct page_fault_info *pfi) {
88 return vas_vaddr_is_allocated(vas: page_alloc_vas, addr: pfi->addr);
89}
90
91void *page_alloc_full(size_t page_count, struct alloc_params params) {
92 void *ret;
93 if (page_count == 1 || params.flags & ALLOC_FLAG_EX(PG, CONTIGUOUS)) {
94 paddr_t phys = pmm_alloc_pages(page_count);
95 if (!phys)
96 return NULL;
97
98 ret = hhdm_paddr_to_ptr(p: phys);
99 } else {
100 ret = page_alloc_vas_mapped_pages(page_count, flags: params.flags, false);
101 }
102
103#ifdef DEBUG_ASAN
104 if (ret)
105 asan_unpoison(ret, page_count * PAGE_SIZE);
106#endif
107
108 return ret;
109}
110
111void *page_alloc_demand_full(size_t page_count, struct alloc_params params) {
112 void *ret = page_alloc_vas_mapped_pages(page_count, flags: params.flags, true);
113
114#ifdef DEBUG_ASAN
115 /* NOTE: these pages are not yet backed; the shadow write here assumes the
116 * shadow itself is mapped for this VA range. */
117 if (ret)
118 asan_unpoison(ret, page_count * PAGE_SIZE);
119#endif
120
121 return ret;
122}
123
124void page_free_full(void *ptr, size_t page_count, enum alloc_behavior b) {
125 cc_ignore(b);
126
127#ifdef DEBUG_ASAN
128 if (ptr)
129 asan_poison(ptr, page_count * PAGE_SIZE);
130#endif
131
132 if (hhdm_ptr_in_range(ptr)) {
133 pmm_free_pages(addr: hhdm_ptr_to_paddr(ptr), count: page_count);
134 } else {
135 page_alloc_vas_release(virt: (vaddr_t) ptr, page_count, nr_mapped: page_count);
136 }
137}
138
139bool page_alloc_vaddr_in_vas(vaddr_t vaddr) {
140 return vas_vaddr_in_vas(vas: page_alloc_vas, addr: vaddr) ||
141 hhdm_vaddr_in_range(vaddr);
142}
143