1#include <acpi/lapic.h>
2#include <cmdline.h>
3#include <console/printf.h>
4#include <global.h>
5#include <irq/idt.h>
6#include <limine.h>
7#include <linker/symbols.h>
8#include <mem/address_range.h>
9#include <mem/asan.h>
10#include <mem/demand_page.h>
11#include <mem/hhdm.h>
12#include <mem/page_table.h>
13#include <mem/pmm.h>
14#include <mem/tlb.h>
15#include <mem/vmm.h>
16#include <sch/sched.h>
17#include <smp/smp.h>
18#include <stdbool.h>
19#include <stdint.h>
20#include <string.h>
21#include <sync/spinlock.h>
22
23#include "mem/slab/internal.h"
24
25#define KERNEL_PML4_START_INDEX 256
26
27static inline bool in_use(pte_t pte) {
28 return pte_in_use(pte: (pte_atomic_t *) &pte);
29}
30
31static void vmm_mem_cmdline_callback(const char *str,
32 struct cmdline_entry *ent);
33
34enum pt_level {
35 PT_LEVEL_PML4 = 0,
36 PT_LEVEL_PDPT = 1,
37 PT_LEVEL_PD = 2,
38 PT_LEVEL_PT = 3,
39};
40
41struct pt_deferred_free {
42 paddr_t phys;
43 uint64_t epoch;
44 struct pt_deferred_free *next;
45};
46
47struct pt_walk {
48 struct page_table *tables[PT_LEVELS];
49 pte_t *entries[PT_LEVELS - 1];
50 enum irql irqls[PT_LEVELS - 1];
51 int depth;
52};
53
54CMDLINE_ENTRY_DECLARE(mem,
55 .desc = "Cap on physical memory the allocator will use",
56 .arg = "<hex bytes>",
57 .callback = vmm_mem_cmdline_callback,
58 .default_val = "0x700000000000",
59 .flags = CMDLINE_ENTRY_FLAGS_NONE, .value = NULL);
60
61ADDRESS_RANGE_DECLARE(hhdm, .base = 0xFFFF800000000000ULL,
62 /* default size: from the base up to the slab heap */
63 .size = SLAB_HEAP_START - 0xFFFF800000000000ULL);
64
65ADDRESS_RANGE_DECLARE(kernel, .base = 0xffffffff80000000,
66 .size = 0); /* Filled in at boot */
67
68bool hhdm_vaddr_in_range(vaddr_t vaddr) {
69 return address_range_addr_in_range(ar: &ADDRESS_RANGE(hhdm), vaddr);
70}
71
72bool hhdm_paddr_in_range(paddr_t paddr) {
73 return paddr < ADDRESS_RANGE(hhdm).size;
74}
75
76bool hhdm_ptr_in_range(void *ptr) {
77 return hhdm_vaddr_in_range(vaddr: (vaddr_t) ptr);
78}
79
80static struct pt_deferred_free *pt_free_list;
81static struct spinlock pt_free_lock = SPINLOCK_INIT;
82static struct page_table *kernel_pml4 = NULL;
83static uintptr_t vmm_map_top = VMM_MAP_BASE;
84
85static long string_to_int(const char *str) {
86 char *endptr;
87
88 if (str[0] == '0' && (str[1] == 'b' || str[1] == 'B')) {
89 return strtol(nptr: str + 2, endptr: &endptr, base: 2);
90 }
91
92 return strtol(nptr: str, endptr: &endptr, base: 0);
93}
94
95static void vmm_mem_cmdline_callback(const char *str,
96 struct cmdline_entry *ent) {
97 ADDRESS_RANGE(hhdm).size = string_to_int(str);
98}
99
100static inline struct page_table *alloc_pt(void) {
101 paddr_t phys = pmm_alloc_page();
102 if (!phys)
103 return NULL;
104
105 void *virt = hhdm_paddr_to_ptr(p: phys);
106 memset(virt, 0, PAGE_SIZE);
107 return virt;
108}
109
110static enum errno pte_init(pte_t *entry, uint64_t flags) {
111 struct page_table *new_table = alloc_pt();
112 if (!new_table)
113 return ERR_NO_MEM;
114
115 uintptr_t new_table_phys = hhdm_ptr_to_paddr(ptr: new_table);
116 *entry = new_table_phys | PAGE_PRESENT | PAGE_WRITE | PTE_LOCK_BIT | flags;
117 return ERR_OK;
118}
119
120enum irql pte_lock(pte_t *pt) {
121 return pte_lock_irql(pte: (void *) pt);
122}
123
124void pte_unlock(pte_t *pt, enum irql irql) {
125 pte_unlock_irql(pte: (void *) pt, old_irql: irql);
126}
127
128static void barrier_and_shootdown(enum vmm_flags flags, vaddr_t virt) {
129 memory_barrier();
130 invlpg(virt);
131
132 if (!(flags & VMM_FLAG_NO_TLB_SHOOTDOWN))
133 tlb_shootdown(addr: virt, true);
134}
135
136static inline uint64_t pt_index(uintptr_t virt, int level) {
137 return (virt >> (PT_SHIFT_L4 - level * PT_STRIDE)) & PT_INDEX_MASK;
138}
139
140static inline struct page_table *pt_next_table(pte_t entry) {
141 return hhdm_paddr_to_ptr(p: entry & PAGE_PHYS_MASK);
142}
143
144static inline void pt_walk_enter(void) {
145 if (global.current_bootstage >= BOOTSTAGE_MID_MP) {
146 uint64_t e =
147 atomic_load_explicit(&global.pt_epoch, memory_order_acquire);
148 atomic_store_explicit(&smp_core()->pt_seen_epoch, e,
149 memory_order_release);
150 }
151}
152
153static inline void pt_walk_exit(void) {
154 if (global.current_bootstage >= BOOTSTAGE_MID_MP) {
155 atomic_store_explicit(&smp_core()->pt_seen_epoch, UINT64_MAX,
156 memory_order_release);
157 }
158}
159
160static void enqueue_pt_free(paddr_t phys) {
161 if (global.current_bootstage < BOOTSTAGE_MID_MP)
162 return pmm_free_page(addr: phys);
163
164 struct pt_deferred_free *n =
165 (struct pt_deferred_free *) hhdm_paddr_to_ptr(p: phys);
166
167 uint64_t e =
168 atomic_fetch_add_explicit(&global.pt_epoch, 1, memory_order_acq_rel) +
169 1;
170 n->phys = phys;
171 n->epoch = e;
172
173 enum irql irql = spin_lock_irq_disable(lock: &pt_free_lock);
174 n->next = pt_free_list;
175 pt_free_list = n;
176 spin_unlock(lock: &pt_free_lock, old: irql);
177}
178
179void vmm_reclaim_page_tables(void) {
180 if (global.current_bootstage < BOOTSTAGE_LATE)
181 return;
182
183 if (smp_core()->reclaiming_page_tables)
184 return;
185
186 kassert(irql_get() == IRQL_DISPATCH_LEVEL);
187 smp_core()->reclaiming_page_tables = true;
188
189 uint64_t min_epoch = UINT64_MAX;
190 struct core *cpu;
191 for_each_cpu_struct(cpu) {
192 uint64_t seen =
193 atomic_load_explicit(&cpu->pt_seen_epoch, memory_order_acquire);
194 if (seen < min_epoch)
195 min_epoch = seen;
196 }
197
198 enum irql irql = IRQL_NONE;
199 if (!spin_trylock_irq_disable(lock: &pt_free_lock, out: &irql))
200 goto out;
201
202 struct pt_deferred_free *to_free = NULL;
203 struct pt_deferred_free **pp = &pt_free_list;
204 while (*pp) {
205 struct pt_deferred_free *n = *pp;
206 if (n->epoch >= min_epoch) {
207 pp = &n->next;
208 } else {
209 *pp = n->next;
210 n->next = to_free;
211 to_free = n;
212 }
213 }
214
215 spin_unlock(lock: &pt_free_lock, old: irql);
216
217 while (to_free) {
218 struct pt_deferred_free *n = to_free;
219 paddr_t phys = n->phys;
220 to_free = n->next;
221 pmm_free_pages(addr: phys, count: 1);
222 }
223
224out:
225 smp_core()->reclaiming_page_tables = false;
226}
227
228uintptr_t vmm_make_user_pml4(void) {
229 struct page_table *user_pml4 = alloc_pt();
230 if (!user_pml4) {
231 panic("Failed to allocate user pml4");
232 }
233
234 for (int i = KERNEL_PML4_START_INDEX; i < PT_ENTRIES; i++) {
235 user_pml4->entries[i] = kernel_pml4->entries[i];
236 }
237
238 return hhdm_ptr_to_paddr(ptr: user_pml4);
239}
240
241/* Leaf frames are not touched here, this gets rid of structural page tables */
242static void vmm_free_user_subtree(struct page_table *pdpt) {
243 for (int i3 = 0; i3 < PT_ENTRIES; i3++) {
244 pte_t e3 = pdpt->entries[i3];
245 if (!in_use(pte: e3) || (e3 & PAGE_PAGE_SIZE))
246 continue;
247
248 struct page_table *pd = pt_next_table(entry: e3);
249 for (int i2 = 0; i2 < PT_ENTRIES; i2++) {
250 pte_t e2 = pd->entries[i2];
251 if (!in_use(pte: e2) || (e2 & PAGE_2MB_page))
252 continue;
253
254 struct page_table *pt = pt_next_table(entry: e2);
255 enqueue_pt_free(phys: hhdm_ptr_to_paddr(ptr: pt));
256 }
257 enqueue_pt_free(phys: hhdm_ptr_to_paddr(ptr: pd));
258 }
259}
260
261void vmm_unmap_all_user_pages(struct page_table *pml4, enum vmm_flags vflags) {
262 (void) vflags;
263
264 for (int i4 = 0; i4 < KERNEL_PML4_START_INDEX; i4++) {
265 pte_t e4 = pml4->entries[i4];
266 if (!in_use(pte: e4))
267 continue;
268
269 struct page_table *pdpt = pt_next_table(entry: e4);
270 vmm_free_user_subtree(pdpt);
271 enqueue_pt_free(phys: hhdm_ptr_to_paddr(ptr: pdpt));
272 pml4->entries[i4] = 0;
273 }
274}
275
276void vmm_init(struct limine_memmap_response *memmap,
277 struct limine_executable_address_response *xa) {
278 kernel_pml4 = alloc_pt();
279 if (!kernel_pml4)
280 panic("Could not allocate space for kernel PML4");
281
282 uintptr_t kernel_pml4_phys = hhdm_ptr_to_paddr(ptr: kernel_pml4);
283
284 uint64_t kernel_phys_start = xa->physical_base;
285 uint64_t kernel_virt_start = xa->virtual_base;
286 uint64_t kernel_virt_end = (uint64_t) &__kernel_virt_end;
287 uint64_t kernel_size = kernel_virt_end - kernel_virt_start;
288 ADDRESS_RANGE(kernel).size = kernel_size;
289
290 paddr_t dummy_phys = pmm_alloc_page();
291 uint8_t *dummy_virt = hhdm_paddr_to_ptr(p: dummy_phys);
292 memset(dummy_virt, 0xFF, PAGE_SIZE);
293
294 enum errno e;
295
296 /* We leave the kernel writable to do boot time patching, but
297 * TODO: lock the kernel down and make it RO later on */
298 for (uint64_t i = 0; i < kernel_size; i += PAGE_SIZE) {
299 e = vmm_map_page(kernel_virt_start + i, kernel_phys_start + i,
300 PAGE_WRITE | PAGE_PRESENT, VMM_FLAG_NONE);
301 if (e < 0)
302 panic("Error %s whilst mapping kernel", errno_to_str(e));
303 }
304
305#ifdef DEBUG_ASAN
306 for (uintptr_t addr = kernel_virt_start; addr < kernel_virt_end;
307 addr += PAGE_SIZE) {
308 uint64_t shadow_addr = ASAN_SHADOW_OFFSET + (addr >> ASAN_SHADOW_SCALE);
309 shadow_addr = PAGE_ALIGN_DOWN(shadow_addr);
310 vmm_map_page(shadow_addr, dummy_phys, PAGE_PRESENT | PAGE_WRITE,
311 VMM_FLAG_MODIFY_LEAF);
312 }
313#endif
314
315 for (uint64_t i = 0; i < memmap->entry_count; i++) {
316 struct limine_memmap_entry *entry = memmap->entries[i];
317 if (entry->type == LIMINE_MEMMAP_BAD_MEMORY ||
318 entry->type == LIMINE_MEMMAP_RESERVED ||
319 entry->type == LIMINE_MEMMAP_ACPI_NVS) {
320 continue;
321 }
322
323 uint64_t base = entry->base;
324 uint64_t len = entry->length;
325 uint64_t end = base + len;
326 uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_XD;
327
328 if (entry->type == LIMINE_MEMMAP_FRAMEBUFFER) {
329 flags |= PAGE_WRITETHROUGH;
330 }
331
332 uint64_t phys = base;
333 while (phys < end) {
334 uint64_t virt = hhdm_paddr_to_vaddr(p: phys);
335
336 bool can_use_2mb = ((phys % PAGE_2MB) == 0) &&
337 ((virt % PAGE_2MB) == 0) &&
338 ((end - phys) >= PAGE_2MB);
339
340 if (can_use_2mb) {
341 e = vmm_map_page(virt, phys, flags, VMM_FLAG_NONE,
342 VMM_MAP_PAGE_SIZE_2MB);
343 phys += PAGE_2MB;
344 } else {
345 e = vmm_map_page(virt, phys, flags);
346 phys += PAGE_SIZE;
347 }
348 if (e < 0)
349 panic("Error %s whilst mapping kernel", errno_to_str(e));
350 }
351 }
352
353 asm volatile("mov %0, %%cr3" : : "r"(kernel_pml4_phys) : "memory");
354}
355
356static inline bool vmm_is_table_empty(struct page_table *table) {
357 for (int i = 0; i < PT_ENTRIES; i++) {
358 if (in_use(pte: table->entries[i]))
359 return false;
360 }
361 return true;
362}
363
364static inline int map_leaf_level(enum vmm_map_page_size sz) {
365 switch (sz) {
366 case VMM_MAP_PAGE_SIZE_1GB: return PT_LEVEL_PDPT;
367
368 case VMM_MAP_PAGE_SIZE_2MB: return PT_LEVEL_PD;
369
370 default: return PT_LEVEL_PT;
371 }
372}
373
374static inline size_t map_page_bytes(enum vmm_map_page_size sz) {
375 switch (sz) {
376 case VMM_MAP_PAGE_SIZE_1GB: return PAGE_1GB;
377 case VMM_MAP_PAGE_SIZE_2MB: return PAGE_2MB;
378 default: return PAGE_SIZE;
379 }
380}
381
382static inline pte_t build_leaf_pte(paddr_t phys, uint64_t flags,
383 enum vmm_map_page_size sz,
384 enum vmm_flags vflags) {
385 uint64_t extra_flags = (vflags & VMM_FLAG_MODIFY_LEAF) ? 0 : PAGE_PRESENT;
386 if (sz != VMM_MAP_PAGE_SIZE_4KB && !(vflags & VMM_FLAG_MODIFY_LEAF))
387 extra_flags |= PAGE_2MB_page;
388
389 flags |= extra_flags;
390 pte_t leaf = (phys & PAGE_PHYS_MASK) | flags;
391 return leaf;
392}
393
394static enum errno vmm_pt_apply(struct vmm_map_request *rq) {
395 bool reclaim = rq->is_unmap_internal;
396 vaddr_t virt = rq->virt;
397 if (virt == 0)
398 panic("CANNOT MAP PAGE 0x0!!!");
399
400 struct page_table *pml4 = rq->pml4 ? rq->pml4 : kernel_pml4;
401 enum vmm_flags vflags = rq->vmm_flags;
402 enum vmm_map_page_size sz = rq->page_size;
403 int leaf_level = map_leaf_level(sz);
404 bool want_huge = sz != VMM_MAP_PAGE_SIZE_4KB;
405
406 bool clear = vflags & VMM_FLAG_CLEAR_LEAF;
407 bool modify = vflags & VMM_FLAG_MODIFY_LEAF;
408 bool handle_exist = vflags & VMM_FLAG_HANDLE_PTE_EXISTING;
409
410 uint64_t user_flag =
411 ((vflags & VMM_FLAG_USER) && !modify) ? PAGE_USER_ALLOWED : 0;
412 uint64_t flags = rq->page_flags | user_flag;
413
414 size_t bytes = map_page_bytes(sz);
415 if (!IS_ALIGNED(virt, bytes) || (!clear && !IS_ALIGNED(rq->phys, bytes)))
416 panic("vmm_pt_apply: huge mapping not naturally aligned");
417
418 pt_walk_enter();
419 enum errno err = ERR_OK;
420 struct page_table *tables[PT_LEVELS];
421 enum irql irqls[PT_LEVELS - 1];
422 pte_t *entries[PT_LEVELS - 1];
423 paddr_t to_free[PT_LEVELS - 1] = {0};
424 int free_count = 0;
425
426 tables[0] = pml4;
427
428 int level = 0;
429 for (level = 0; level < leaf_level; level++) {
430
431#pragma GCC diagnostic push
432#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
433
434 pte_t *entry = &tables[level]->entries[pt_index(virt, level)];
435
436#pragma GCC diagnostic pop
437
438 entries[level] = entry;
439 irqls[level] = pte_lock(pt: entry);
440
441 if (!in_use(pte: *entry)) {
442 /* clear/unmap never builds tables: no table here means no leaf */
443 if (clear) {
444 level++;
445 goto out;
446 }
447 if ((err = pte_init(entry, flags: user_flag)) < 0) {
448 level++;
449 goto out;
450 }
451 }
452
453 /* present huge leaf where table was expected is a size mismatch
454 *
455 * only meaningful when present as non-present can have payload there */
456 kassert(!((*entry & PAGE_PRESENT) && (*entry & PAGE_2MB_page)));
457 tables[level + 1] = pt_next_table(entry: *entry);
458 }
459
460#pragma GCC diagnostic push
461#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
462
463 pte_t *last_entry =
464 &tables[leaf_level]->entries[pt_index(virt, level: leaf_level)];
465
466#pragma GCC diagnostic pop
467
468 enum irql last_irql = pte_lock(pt: last_entry);
469
470 bool was_present = *last_entry & PAGE_PRESENT;
471
472 /* page tables must match the caller's claims, PS bit is only meaningful
473 * for present bits as tagged non-present PTEs use bit 7 */
474 if (was_present) {
475 kassert((bool) (*last_entry & PAGE_2MB_page) == want_huge);
476 if (handle_exist) {
477 err = ERR_EXIST;
478 pte_unlock(pt: last_entry, irql: last_irql);
479 goto out;
480 }
481 }
482
483 if (clear) {
484 if (was_present)
485 barrier_and_shootdown(flags: vflags, virt);
486 *last_entry = PTE_LOCK_BIT; /* zero all but the held lock bit */
487 } else {
488 if (in_use(pte: *last_entry) && !modify)
489 panic(
490 "vmm_pt_apply: leaf in use without MODIFY_LEAF (double map?)");
491
492 if (was_present)
493 barrier_and_shootdown(flags: vflags, virt);
494
495 *last_entry =
496 build_leaf_pte(phys: rq->phys, flags, sz, vflags) | PTE_LOCK_BIT;
497 }
498
499 pte_unlock(pt: last_entry, irql: last_irql);
500
501 /* used in unmap */
502 if (reclaim) {
503 for (int up = leaf_level; up > 0; up--) {
504 if (!vmm_is_table_empty(table: tables[up]))
505 break;
506 to_free[free_count++] = hhdm_ptr_to_paddr(ptr: tables[up]);
507 *entries[up - 1] = PTE_LOCK_BIT;
508 }
509 }
510
511out:
512 for (int i = level - 1; i >= 0; i--)
513 pte_unlock(pt: entries[i], irql: irqls[i]);
514
515 for (int i = 0; i < free_count; i++)
516 enqueue_pt_free(phys: to_free[i]);
517
518 pt_walk_exit();
519 return err;
520}
521
522enum errno vmm_map_page_full(struct vmm_map_request *rq) {
523 return vmm_pt_apply(rq);
524}
525
526/* Tear down a single leaf and reclaim every page table it leaves empty. */
527void vmm_unmap_page_full(struct vmm_map_request *rq) {
528 struct vmm_map_request req = *rq;
529 req.vmm_flags |= VMM_FLAG_CLEAR_LEAF;
530 req.is_unmap_internal = true;
531 (void) vmm_pt_apply(rq: &req);
532}
533
534enum errno vmm_map_page_internal(vaddr_t virt, paddr_t phys, page_flags_t flags,
535 enum vmm_flags vflags,
536 enum vmm_map_page_size size) {
537 struct vmm_map_request rq = {
538 .virt = virt,
539 .phys = phys,
540 .page_flags = flags,
541 .vmm_flags = vflags,
542 .page_size = size,
543 };
544 return vmm_map_page_full(rq: &rq);
545}
546
547enum errno vmm_map_page_user_internal(struct page_table *pml4, vaddr_t virt,
548 paddr_t phys, page_flags_t flags,
549 enum vmm_flags vflags,
550 enum vmm_map_page_size size) {
551 struct vmm_map_request rq = {
552 .pml4 = pml4,
553 .virt = virt,
554 .phys = phys,
555 .page_flags = flags,
556 .vmm_flags = vflags | VMM_FLAG_USER,
557 .page_size = size,
558 };
559 return vmm_map_page_full(rq: &rq);
560}
561
562enum errno vmm_mark_demand_page_internal(vaddr_t virt,
563 enum demand_page_flags flags,
564 enum vmm_map_page_size size) {
565 struct pte_tagged ptag = {
566 .type = PTE_TAG_TYPE_DEMAND_PAGED,
567 .payload = flags,
568 };
569
570 uint64_t packed = pte_tagged_pack(pt: &ptag);
571
572 struct vmm_map_request rq = {
573 .pml4 = kernel_pml4,
574 .virt = virt,
575 .phys = 0,
576 .page_flags = packed,
577 .vmm_flags = VMM_FLAG_MODIFY_LEAF,
578 .page_size = size,
579 };
580
581 return vmm_map_page_full(rq: &rq);
582}
583
584enum errno vmm_map_demand_page_internal(vaddr_t virt, paddr_t phys,
585 enum demand_page_flags flags,
586 enum vmm_map_page_size size) {
587 uint64_t pflags = PAGE_PRESENT;
588 if (flags & DEMAND_PAGE_FLAG_WRITABLE)
589 pflags |= PAGE_WRITE;
590
591 if (flags & DEMAND_PAGE_FLAG_XD)
592 pflags |= PAGE_XD;
593
594 struct vmm_map_request rq = {
595 .pml4 = kernel_pml4,
596 .virt = virt,
597 .phys = phys,
598 .page_flags = pflags,
599 .vmm_flags = VMM_FLAG_MODIFY_LEAF | VMM_FLAG_HANDLE_PTE_EXISTING |
600 VMM_FLAG_NO_TLB_SHOOTDOWN,
601 .page_size = size,
602 };
603
604 return vmm_map_page_full(rq: &rq);
605}
606
607enum errno vmm_mark_demand_page_user_internal(struct page_table *pml4,
608 vaddr_t virt,
609 enum demand_page_flags flags,
610 enum vmm_map_page_size size) {
611 struct pte_tagged ptag = {
612 .type = PTE_TAG_TYPE_DEMAND_PAGED,
613 .payload = flags,
614 };
615
616 uint64_t packed = pte_tagged_pack(pt: &ptag);
617
618 struct vmm_map_request rq = {
619 .pml4 = pml4,
620 .virt = virt,
621 .phys = 0,
622 .page_flags = packed,
623
624 /* USER here is merely nominal, map_page_full ignores it */
625 .vmm_flags = VMM_FLAG_USER | VMM_FLAG_MODIFY_LEAF,
626 .page_size = size,
627 };
628
629 return vmm_map_page_full(rq: &rq);
630}
631
632void vmm_unmap_page_internal(vaddr_t virt, enum vmm_flags vflags,
633 enum vmm_map_page_size size) {
634 struct vmm_map_request rq = {
635 .virt = virt,
636 .vmm_flags = vflags,
637 .page_size = size,
638 };
639 vmm_unmap_page_full(rq: &rq);
640}
641
642static pte_t vmm_walk_leaf(struct page_table *root, vaddr_t virt,
643 int *out_level) {
644 pt_walk_enter();
645
646 struct page_table *table = root;
647 uint64_t snap = 0;
648 int level;
649
650 for (level = 0; level < PT_LEVEL_PT; level++) {
651 uint64_t index = pt_index(virt, level);
652
653#pragma GCC diagnostic push
654#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
655
656 /* assert so that clang inlines this instead of creating a call to
657 * __atomic_load (from c lib, breaks under -nostdlib), gcc inlines */
658 snap = atomic_load_explicit((_Atomic pte_t *) __builtin_assume_aligned(
659 &table->entries[index], sizeof(pte_t)),
660 memory_order_acquire);
661
662#pragma GCC diagnostic pop
663
664 if (!(snap & PAGE_PRESENT)) {
665 snap = 0;
666 goto out;
667 }
668
669 if (snap & PAGE_2MB_page)
670 goto out;
671
672 table = pt_next_table(entry: snap);
673 }
674
675#pragma GCC diagnostic push
676#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
677
678 snap = atomic_load_explicit(
679 (_Atomic pte_t *) __builtin_assume_aligned(
680 &table->entries[pt_index(virt, level: PT_LEVEL_PT)], sizeof(pte_t)),
681 memory_order_acquire);
682
683#pragma GCC diagnostic pop
684
685out:
686 pt_walk_exit();
687
688 if (out_level)
689 *out_level = level;
690
691 return snap;
692}
693
694paddr_t vmm_get_phys_internal(uintptr_t virt, enum vmm_flags vflags) {
695 (void) vflags;
696
697 int level;
698 uint64_t snap = vmm_walk_leaf(root: kernel_pml4, virt, out_level: &level);
699
700 if (!(snap & PAGE_PRESENT))
701 return (uintptr_t) -1;
702
703 if (level == PT_LEVEL_PDPT)
704 return (snap & PAGE_PHYS_MASK) + (virt & (PAGE_1GB - 1));
705
706 if (level == PT_LEVEL_PD)
707 return (snap & PAGE_2MB_PHYS_MASK) + (virt & (PAGE_2MB - 1));
708
709 return (snap & PAGE_PHYS_MASK) + (virt & 0xFFF);
710}
711
712pte_t vmm_get_leaf_pte_internal(vaddr_t virt, enum vmm_flags vflags) {
713 (void) vflags;
714 return vmm_walk_leaf(root: kernel_pml4, virt, NULL);
715}
716
717void *vmm_map(paddr_t paddr, vaddr_t vaddr, uint64_t len, uint64_t flags,
718 enum vmm_flags vflags) {
719 if (len == 0)
720 return NULL;
721
722 uintptr_t phys_start = PAGE_ALIGN_DOWN(paddr);
723 uintptr_t offset = paddr - phys_start;
724
725 uint64_t total_len = len + offset;
726 uint64_t total_pages = (total_len + PAGE_SIZE - 1) / PAGE_SIZE;
727
728 enum errno e = ERR_OK;
729 uint64_t mapped = 0;
730
731 for (; mapped < total_pages; mapped++) {
732 e = vmm_map_page(vaddr + mapped * PAGE_SIZE,
733 phys_start + mapped * PAGE_SIZE,
734 PAGE_PRESENT | PAGE_WRITE | flags, vflags);
735 if (e < 0)
736 goto unwind;
737 }
738
739 return (void *) (vaddr + offset);
740
741unwind:
742 for (uint64_t i = 0; i < mapped; i++)
743 vmm_unmap_page(vaddr + i * PAGE_SIZE, vflags);
744
745 return NULL;
746}
747
748void vmm_unmap(void *addr, uint64_t len, enum vmm_flags vflags) {
749 uintptr_t virt_addr = (uintptr_t) addr;
750 uintptr_t page_offset = virt_addr & (PAGE_SIZE - 1);
751 uintptr_t aligned_virt = PAGE_ALIGN_DOWN(virt_addr);
752
753 uint64_t total_len = len + page_offset;
754 uint64_t total_pages = PAGES_NEEDED_FOR(total_len);
755
756 for (uint64_t i = 0; i < total_pages; i++) {
757 vmm_unmap_page(aligned_virt + i * PAGE_SIZE, vflags);
758 }
759}
760
761void *vmm_map_bump_internal(uintptr_t addr, uint64_t len, uint64_t flags,
762 enum vmm_flags vflags) {
763 if (global.current_bootstage >= BOOTSTAGE_LATE)
764 log_warn_once("vmm_map_bump called after BOOTSTAGE_LATE...");
765
766 if (len == 0)
767 return NULL;
768
769 uintptr_t phys_start = PAGE_ALIGN_DOWN(addr);
770 uintptr_t offset = addr - phys_start;
771
772 uint64_t total_len = len + offset;
773 uint64_t total_pages = (total_len + PAGE_SIZE - 1) / PAGE_SIZE;
774
775 uint64_t span = total_pages * PAGE_SIZE;
776 if (total_pages != 0 && span / PAGE_SIZE != total_pages)
777 return NULL;
778 if (vmm_map_top > VMM_MAP_LIMIT || span > VMM_MAP_LIMIT - vmm_map_top)
779 return NULL;
780
781 uintptr_t virt_start = vmm_map_top;
782 vmm_map_top += span;
783
784 enum errno e = ERR_OK;
785 uint64_t mapped = 0;
786
787 for (; mapped < total_pages; mapped++) {
788 e = vmm_map_page(virt_start + mapped * PAGE_SIZE,
789 phys_start + mapped * PAGE_SIZE,
790 PAGE_PRESENT | PAGE_WRITE | flags, vflags);
791 if (e < 0)
792 goto unwind;
793 }
794
795 return (void *) (virt_start + offset);
796
797unwind:
798 for (uint64_t i = 0; i < mapped; i++)
799 vmm_unmap_page(virt_start + i * PAGE_SIZE, vflags);
800
801 vmm_map_top = virt_start;
802
803 return NULL;
804}
805
806void vmm_unmap_virt(void *addr, uint64_t len, enum vmm_flags vflags) {
807 uintptr_t virt_addr = (uintptr_t) addr;
808 uintptr_t page_offset = virt_addr & (PAGE_SIZE - 1);
809 uintptr_t aligned_virt = PAGE_ALIGN_DOWN(virt_addr);
810
811 uint64_t total_len = len + page_offset;
812 uint64_t total_pages = PAGES_NEEDED_FOR(total_len);
813
814 for (uint64_t i = 0; i < total_pages; i++) {
815 vmm_unmap_page(aligned_virt + i * PAGE_SIZE, vflags);
816 }
817}
818
819struct page_table *vmm_phys_to_pml4(paddr_t paddr) {
820 return (struct page_table *) hhdm_paddr_to_ptr(p: paddr);
821}
822