1#include <math/kb_mb_gb_tb.h>
2#include <mem/address_range.h>
3#include <mem/asan.h>
4#include <mem/hhdm.h>
5#include <mem/page.h>
6#include <mem/page_alloc.h>
7#include <mem/page_fault.h>
8#include <mem/pmm.h>
9#include <mem/vas.h>
10
11static bool page_alloc_pf_valid(struct page_fault_info *pfi);
12
13static struct page_fault_handler_ops page_alloc_pfho = {
14 .alloc_pages = NULL,
15 .update_after_map = NULL,
16 .is_valid_fault = page_alloc_pf_valid,
17};
18
19static struct page_fault_handler page_alloc_pfh = {
20 .ops = &page_alloc_pfho,
21};
22
23static struct vas *page_alloc_vas = NULL;
24ADDRESS_RANGE_DECLARE(page_alloc, .align = PAGE_SIZE,
25 .flags = ADDRESS_RANGE_DYNAMIC, .size = TB(8),
26 .page_fault_handler = &page_alloc_pfh);
27
28void page_alloc_init() {
29 page_alloc_vas = vas_bootstrap_from(ar: &ADDRESS_RANGE(page_alloc));
30}
31
32static void *page_alloc_vas_mapped_pages(size_t n_pages, enum alloc_flags flags,
33 bool demand_paged) {
34 vaddr_t virt = vas_alloc(vas: page_alloc_vas, size: n_pages * PAGE_SIZE, PAGE_SIZE);
35 if (!virt)
36 return NULL;
37
38 uintptr_t phys_pages[n_pages];
39 uint64_t allocated = 0;
40
41 page_flags_t page_flags = PAGE_PRESENT | PAGE_WRITE | PAGE_XD;
42 bool zero = flags & ALLOC_FLAG_ZERO_ON_ALLOC;
43
44 for (uint64_t i = 0; i < n_pages; i++) {
45 if (!demand_paged || !zero) {
46 uintptr_t phys = pmm_alloc_page(flags);
47 if (!phys) {
48 for (uint64_t j = 0; j < allocated; j++)
49 pmm_free_page(addr: phys_pages[j]);
50 return NULL;
51 }
52
53 enum errno e = vmm_map_page(virt + i * PAGE_SIZE, phys, page_flags);
54 if (e < 0) {
55 pmm_free_page(addr: phys);
56 for (uint64_t j = 0; j < allocated; j++)
57 pmm_free_page(addr: phys_pages[j]);
58
59 return NULL;
60 }
61
62 phys_pages[allocated++] = phys;
63 } else {
64 enum errno e = vmm_mark_demand_page(virt + i * PAGE_SIZE,
65 DEMAND_PAGE_FLAG_ZERO_MEMORY |
66 DEMAND_PAGE_FLAG_WRITABLE);
67 if (e < 0) {
68 for (size_t i = 0; i < allocated; i++) {
69 vaddr_t v = virt + i * PAGE_SIZE;
70 vmm_unmap_page(v);
71 }
72
73 return NULL;
74 }
75 allocated++;
76 }
77 }
78
79 return (void *) virt;
80}
81
82/* Must be in vas, that's the only check */
83static bool page_alloc_pf_valid(struct page_fault_info *pfi) {
84 return vas_vaddr_is_allocated(vas: page_alloc_vas, addr: pfi->addr);
85}
86
87void *page_alloc_internal(size_t n_pages, enum alloc_flags flags,
88 enum alloc_behavior bh) {
89 void *ret;
90 if (n_pages == 1 || flags & ALLOC_FLAG_CONTIGUOUS) {
91 paddr_t phys = pmm_alloc_pages(n_pages);
92 if (!phys)
93 return NULL;
94
95 ret = hhdm_paddr_to_ptr(p: phys);
96 } else {
97 ret = page_alloc_vas_mapped_pages(n_pages, flags, false);
98 }
99
100#ifdef DEBUG_ASAN
101 if (ret)
102 asan_unpoison(ret, n_pages * PAGE_SIZE);
103#endif
104 return ret;
105}
106
107void *page_alloc_demand_internal(size_t n_pages, enum alloc_flags flags,
108 enum alloc_behavior bh) {
109 void *ret = page_alloc_vas_mapped_pages(n_pages, flags, true);
110
111#ifdef DEBUG_ASAN
112 /* NOTE: these pages are not yet backed; the shadow write here assumes the
113 * shadow itself is mapped for this VA range. */
114 if (ret)
115 asan_unpoison(ret, n_pages * PAGE_SIZE);
116#endif
117 return ret;
118}
119
120void page_free_internal(void *ptr, size_t n_pages, enum alloc_behavior b) {
121#ifdef DEBUG_ASAN
122 if (ptr)
123 asan_poison(ptr, n_pages * PAGE_SIZE);
124#endif
125
126 if (hhdm_ptr_in_range(ptr)) {
127 pmm_free_pages(addr: hhdm_ptr_to_paddr(ptr), count: n_pages);
128 } else {
129 vaddr_t virt = (vaddr_t) ptr;
130 for (uint32_t i = 0; i < n_pages; i++) {
131 vaddr_t vaddr = virt + i * PAGE_SIZE;
132 paddr_t phys = (paddr_t) vmm_get_phys(vaddr, VMM_FLAG_NONE);
133 vmm_unmap_page(vaddr);
134
135 if (phys != PADDR_MAX)
136 pmm_free_page(addr: phys);
137 }
138
139 vas_free(vas: page_alloc_vas, addr: virt, size: n_pages * PAGE_SIZE);
140 }
141}
142
143bool page_alloc_vaddr_in_vas(vaddr_t vaddr) {
144 return vas_vaddr_in_vas(vas: page_alloc_vas, vaddr) ||
145 hhdm_vaddr_in_range(vaddr);
146}
147