1#include <acpi/lapic.h>
2#include <cmdline.h>
3#include <compiler.h>
4#include <console/printf.h>
5#include <global.h>
6#include <irq/idt.h>
7#include <limine.h>
8#include <linker/symbols.h>
9#include <mem/address_range.h>
10#include <mem/asan.h>
11#include <mem/demand_page.h>
12#include <mem/hhdm.h>
13#include <mem/page_table.h>
14#include <mem/pmm.h>
15#include <mem/tlb.h>
16#include <mem/vmm.h>
17#include <sch/sched.h>
18#include <smp/smp.h>
19#include <stdbool.h>
20#include <stdint.h>
21#include <string.h>
22#include <sync/spinlock.h>
23
24#include "mem/slab/internal.h"
25
26#define KERNEL_PML4_START_INDEX 256
27
28static paddr_t text_phys_start;
29static paddr_t text_phys_end;
30
31bool vmm_phys_is_kernel_text(paddr_t phys) {
32 return text_phys_end && phys >= text_phys_start && phys < text_phys_end;
33}
34
35static inline bool in_use(pte_t pte) {
36 return pte_in_use(pte: (pte_atomic_t *) &pte);
37}
38
39enum pt_level {
40 PT_LEVEL_PML4 = 0,
41 PT_LEVEL_PDPT = 1,
42 PT_LEVEL_PD = 2,
43 PT_LEVEL_PT = 3,
44};
45
46struct pt_deferred_free {
47 paddr_t phys;
48 uint64_t epoch;
49 struct pt_deferred_free *next;
50};
51
52struct pt_walk {
53 struct page_table *tables[PT_LEVELS];
54 pte_t *entries[PT_LEVELS - 1];
55 enum irql irqls[PT_LEVELS - 1];
56 int depth;
57};
58
59ADDRESS_RANGE_DECLARE(hhdm, .base = 0xFFFF800000000000ULL,
60 /* default size: from the base up to the slab heap */
61 .size = SLAB_HEAP_START - 0xFFFF800000000000ULL);
62
63ADDRESS_RANGE_DECLARE(kernel, .base = 0xffffffff80000000,
64 .size = 0); /* Filled in at boot */
65
66CMDLINE_DECLARE_VAR(mem, ADDRESS_RANGE(hhdm).size,
67 .desc = "Cap on physical memory the allocator will use",
68 .arg = "<hex bytes>", .default_val = "0x700000000000");
69
70bool hhdm_vaddr_in_range(vaddr_t vaddr) {
71 return address_range_addr_in_range(ar: &ADDRESS_RANGE(hhdm), vaddr);
72}
73
74bool hhdm_paddr_in_range(paddr_t paddr) {
75 return paddr < ADDRESS_RANGE(hhdm).size;
76}
77
78bool hhdm_ptr_in_range(void *ptr) {
79 return hhdm_vaddr_in_range(vaddr: (vaddr_t) ptr);
80}
81
82static struct pt_deferred_free *pt_free_list;
83static struct spinlock pt_free_lock = SPINLOCK_INIT;
84static struct page_table *kernel_pml4 = NULL;
85static _Atomic uintptr_t vmm_map_top = VMM_MAP_BASE;
86static void vmm_unmap_aliased(vaddr_t virt, size_t len, enum vmm_flags vflags);
87
88static inline struct page_table *alloc_pt(void) {
89 paddr_t phys = pmm_alloc_page();
90 if (!phys)
91 return NULL;
92
93 void *virt = hhdm_paddr_to_ptr(p: phys);
94 memset(virt, 0, PAGE_SIZE);
95 return virt;
96}
97
98static enum errno pte_init(pte_t *entry, uint64_t flags) {
99 struct page_table *new_table = alloc_pt();
100 if (!new_table)
101 return ERR_NO_MEM;
102
103 uintptr_t new_table_phys = hhdm_ptr_to_paddr(ptr: new_table);
104 *entry = new_table_phys | PAGE_PRESENT | PAGE_WRITE | PTE_LOCK_BIT | flags;
105 return ERR_OK;
106}
107
108enum irql pte_lock(pte_t *pt) {
109 return pte_lock_irql(pte: (void *) pt);
110}
111
112void pte_unlock(pte_t *pt, enum irql irql) {
113 pte_unlock_irql(pte: (void *) pt, old_irql: irql);
114}
115
116static void barrier_and_shootdown(enum vmm_flags flags, vaddr_t virt) {
117 memory_barrier();
118 invlpg(virt);
119
120 if (!(flags & VMM_FLAG_NO_TLB_SHOOTDOWN))
121 tlb_shootdown(addr: virt, true);
122}
123
124static inline uint64_t pt_index(uintptr_t virt, int level) {
125 return (virt >> (PT_SHIFT_L4 - level * PT_STRIDE)) & PT_INDEX_MASK;
126}
127
128static inline struct page_table *pt_next_table(pte_t entry) {
129 return hhdm_paddr_to_ptr(p: entry & PAGE_PHYS_MASK);
130}
131
132/* Bytes of address space covered by an entry at `level`
133 *
134 * PML4 entries span 512GB, PDPT 1GB, PD 2MB, PT 4KB */
135static inline uint64_t pt_level_granule(int level) {
136 return 1ULL << (PT_SHIFT_L4 - level * PT_STRIDE);
137}
138
139static inline void pt_walk_enter(void) {
140 if (global.current_bootstage >= BOOTSTAGE_MID_MP) {
141 uint64_t e =
142 atomic_load_explicit(&global.pt_epoch, memory_order_acquire);
143 atomic_store_explicit(&smp_core(cond: TOPC_IRQL)->pt_seen_epoch, e,
144 memory_order_release);
145 }
146}
147
148static inline void pt_walk_exit(void) {
149 if (global.current_bootstage >= BOOTSTAGE_MID_MP) {
150 atomic_store_explicit(&smp_core(cond: TOPC_IRQL)->pt_seen_epoch, UINT64_MAX,
151 memory_order_release);
152 }
153}
154
155static void enqueue_pt_free(paddr_t phys) {
156 if (global.current_bootstage < BOOTSTAGE_MID_MP)
157 return pmm_free_page(addr: phys);
158
159 struct pt_deferred_free *n =
160 (struct pt_deferred_free *) hhdm_paddr_to_ptr(p: phys);
161
162 uint64_t e =
163 atomic_fetch_add_explicit(&global.pt_epoch, 1, memory_order_acq_rel) +
164 1;
165 n->phys = phys;
166 n->epoch = e;
167
168 enum irql irql = spin_lock_irq_disable(&pt_free_lock);
169 n->next = pt_free_list;
170 pt_free_list = n;
171 spin_unlock(&pt_free_lock, irql);
172}
173
174void vmm_reclaim_page_tables(void) {
175 if (global.current_bootstage < BOOTSTAGE_LATE)
176 return;
177
178 if (smp_read(TOPC_IRQL, reclaiming_page_tables))
179 return;
180
181 kassert(irql_get() == IRQL_DISPATCH_LEVEL);
182 smp_write(TOPC_IRQL, reclaiming_page_tables, true);
183
184 uint64_t min_epoch = UINT64_MAX;
185 struct core *cpu;
186 for_each_cpu_struct(cpu) {
187 uint64_t seen =
188 atomic_load_explicit(&cpu->pt_seen_epoch, memory_order_acquire);
189 if (seen < min_epoch)
190 min_epoch = seen;
191 }
192
193 enum irql irql = IRQL_NONE;
194 if (!spin_trylock_irq_disable(&pt_free_lock, &irql))
195 goto out;
196
197 struct pt_deferred_free *to_free = NULL;
198 struct pt_deferred_free **pp = &pt_free_list;
199 while (*pp) {
200 struct pt_deferred_free *n = *pp;
201 if (n->epoch >= min_epoch) {
202 pp = &n->next;
203 } else {
204 *pp = n->next;
205 n->next = to_free;
206 to_free = n;
207 }
208 }
209
210 spin_unlock(&pt_free_lock, irql);
211
212 while (to_free) {
213 struct pt_deferred_free *n = to_free;
214 paddr_t phys = n->phys;
215 to_free = n->next;
216 pmm_free_pages(addr: phys, count: 1);
217 }
218
219out:
220 smp_write(TOPC_IRQL, reclaiming_page_tables, false);
221}
222
223uintptr_t vmm_make_user_pml4(void) {
224 struct page_table *user_pml4 = alloc_pt();
225 if (!user_pml4) {
226 panic("Failed to allocate user pml4");
227 }
228
229 for (int i = KERNEL_PML4_START_INDEX; i < PT_ENTRIES; i++) {
230 user_pml4->entries[i] = kernel_pml4->entries[i];
231 }
232
233 return hhdm_ptr_to_paddr(ptr: user_pml4);
234}
235
236/* Leaf frames are not touched here, this gets rid of structural page tables */
237static void vmm_free_user_subtree(struct page_table *pdpt) {
238 for (int i3 = 0; i3 < PT_ENTRIES; i3++) {
239 pte_t e3 = pdpt->entries[i3];
240 if (!in_use(pte: e3) || (e3 & PAGE_HUGE))
241 continue;
242
243 struct page_table *pd = pt_next_table(entry: e3);
244 for (int i2 = 0; i2 < PT_ENTRIES; i2++) {
245 pte_t e2 = pd->entries[i2];
246 if (!in_use(pte: e2) || (e2 & PAGE_HUGE))
247 continue;
248
249 struct page_table *pt = pt_next_table(entry: e2);
250 enqueue_pt_free(phys: hhdm_ptr_to_paddr(ptr: pt));
251 }
252 enqueue_pt_free(phys: hhdm_ptr_to_paddr(ptr: pd));
253 }
254}
255
256void vmm_unmap_all_user_pages(struct page_table *pml4, enum vmm_flags vflags) {
257 (void) vflags;
258
259 for (int i4 = 0; i4 < KERNEL_PML4_START_INDEX; i4++) {
260 pte_t e4 = pml4->entries[i4];
261 if (!in_use(pte: e4))
262 continue;
263
264 struct page_table *pdpt = pt_next_table(entry: e4);
265 vmm_free_user_subtree(pdpt);
266 enqueue_pt_free(phys: hhdm_ptr_to_paddr(ptr: pdpt));
267 pml4->entries[i4] = 0;
268 }
269}
270
271void vmm_init(struct limine_memmap_response *memmap,
272 struct limine_executable_address_response *xa) {
273 kernel_pml4 = alloc_pt();
274 if (!kernel_pml4)
275 panic("Could not allocate space for kernel PML4");
276
277 uintptr_t kernel_pml4_phys = hhdm_ptr_to_paddr(ptr: kernel_pml4);
278
279 uint64_t kernel_phys_start = xa->physical_base;
280 uint64_t kernel_virt_start = xa->virtual_base;
281 uint64_t kernel_virt_end = (uint64_t) &__kernel_virt_end;
282 uint64_t kernel_size = kernel_virt_end - kernel_virt_start;
283 ADDRESS_RANGE(kernel).size = kernel_size;
284
285 paddr_t dummy_phys = pmm_alloc_page();
286 uint8_t *dummy_virt = hhdm_paddr_to_ptr(p: dummy_phys);
287 memset(dummy_virt, 0xFF, PAGE_SIZE);
288
289 enum errno e;
290
291 uint64_t text_start = (uint64_t) &__stext;
292 uint64_t text_end = (uint64_t) &__etext;
293 text_phys_start = kernel_phys_start + (text_start - kernel_virt_start);
294 text_phys_end = kernel_phys_start + (text_end - kernel_virt_start);
295
296 for (uint64_t i = 0; i < kernel_size; i += PAGE_SIZE) {
297 uint64_t virt = kernel_virt_start + i;
298 uint64_t flags = PAGE_PRESENT;
299
300 if (virt < text_start || virt >= text_end)
301 flags |= PAGE_WRITE;
302
303 e = vmm_map_page(virt, kernel_phys_start + i, flags, VMM_FLAG_NONE);
304 if (e < 0)
305 panic("Error %s whilst mapping kernel", errno_to_str(e));
306 }
307
308#ifdef DEBUG_ASAN
309 for (uintptr_t addr = kernel_virt_start; addr < kernel_virt_end;
310 addr += PAGE_SIZE) {
311 uint64_t shadow_addr = ASAN_SHADOW_OFFSET + (addr >> ASAN_SHADOW_SCALE);
312 shadow_addr = PAGE_ALIGN_DOWN(shadow_addr);
313 vmm_map_page(shadow_addr, dummy_phys, PAGE_PRESENT | PAGE_WRITE,
314 VMM_FLAG_MODIFY_LEAF);
315 }
316#endif
317
318 for (uint64_t i = 0; i < memmap->entry_count; i++) {
319 struct limine_memmap_entry *entry = memmap->entries[i];
320 if (entry->type == LIMINE_MEMMAP_BAD_MEMORY ||
321 entry->type == LIMINE_MEMMAP_RESERVED ||
322 entry->type == LIMINE_MEMMAP_ACPI_NVS) {
323 continue;
324 }
325
326 uint64_t base = entry->base;
327 uint64_t len = entry->length;
328 uint64_t end = base + len;
329 uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_XD;
330
331 if (entry->type == LIMINE_MEMMAP_FRAMEBUFFER) {
332 flags |= PAGE_WRITETHROUGH;
333 }
334
335 uint64_t phys = base;
336 while (phys < end) {
337 uint64_t virt = hhdm_paddr_to_vaddr(p: phys);
338
339 bool overlaps_text =
340 phys < text_phys_end && phys + PAGE_2MB > text_phys_start;
341
342 bool can_use_2mb = ((phys % PAGE_2MB) == 0) &&
343 ((virt % PAGE_2MB) == 0) &&
344 ((end - phys) >= PAGE_2MB) && !overlaps_text;
345
346 if (can_use_2mb) {
347 e = vmm_map_page(virt, phys, flags, VMM_FLAG_NONE,
348 VMM_MAP_PAGE_SIZE_2MB);
349 phys += PAGE_2MB;
350 } else {
351 uint64_t page_flags = flags;
352
353 if (phys >= text_phys_start && phys < text_phys_end)
354 page_flags &= ~PAGE_WRITE;
355
356 e = vmm_map_page(virt, phys, page_flags);
357 phys += PAGE_SIZE;
358 }
359 if (e < 0)
360 panic("Error %s whilst mapping kernel", errno_to_str(e));
361 }
362 }
363
364 asm volatile("mov %0, %%cr3" : : "r"(kernel_pml4_phys) : "memory");
365}
366
367static inline bool vmm_is_table_empty(struct page_table *table) {
368 for (int i = 0; i < PT_ENTRIES; i++) {
369 if (in_use(pte: table->entries[i]))
370 return false;
371 }
372 return true;
373}
374
375static inline int map_leaf_level(enum vmm_map_page_size sz) {
376 switch (sz) {
377 case VMM_MAP_PAGE_SIZE_1GB: return PT_LEVEL_PDPT;
378
379 case VMM_MAP_PAGE_SIZE_2MB: return PT_LEVEL_PD;
380
381 default: return PT_LEVEL_PT;
382 }
383}
384
385static inline size_t map_page_bytes(enum vmm_map_page_size sz) {
386 switch (sz) {
387 case VMM_MAP_PAGE_SIZE_1GB: return PAGE_1GB;
388 case VMM_MAP_PAGE_SIZE_2MB: return PAGE_2MB;
389 default: return PAGE_SIZE;
390 }
391}
392
393static inline pte_t build_leaf_pte(paddr_t phys, uint64_t flags,
394 enum vmm_map_page_size sz,
395 enum vmm_flags vflags) {
396 uint64_t extra_flags = (vflags & VMM_FLAG_MODIFY_LEAF) ? 0 : PAGE_PRESENT;
397 if (sz != VMM_MAP_PAGE_SIZE_4KB && !(vflags & VMM_FLAG_MODIFY_LEAF))
398 extra_flags |= PAGE_HUGE;
399
400 flags |= extra_flags;
401 pte_t leaf = (phys & PAGE_PHYS_MASK) | flags;
402 return leaf;
403}
404
405static enum errno vmm_pt_apply(struct vmm_map_request *rq) {
406 bool reclaim = rq->is_unmap_internal;
407 vaddr_t virt = rq->virt;
408 if (virt == 0)
409 panic("CANNOT MAP PAGE 0x0!!!");
410
411 struct page_table *pml4 = rq->pml4 ? rq->pml4 : kernel_pml4;
412 enum vmm_flags vflags = rq->vmm_flags;
413 enum vmm_map_page_size sz = rq->page_size;
414 int leaf_level = map_leaf_level(sz);
415 bool want_huge = sz != VMM_MAP_PAGE_SIZE_4KB;
416
417 bool clear = vflags & VMM_FLAG_CLEAR_LEAF;
418 bool modify = vflags & VMM_FLAG_MODIFY_LEAF;
419 bool handle_exist = vflags & VMM_FLAG_HANDLE_PTE_EXISTING;
420
421 uint64_t user_flag =
422 ((vflags & VMM_FLAG_USER) && !modify) ? PAGE_USER_ALLOWED : 0;
423 uint64_t flags = rq->page_flags | user_flag;
424
425 size_t bytes = map_page_bytes(sz);
426 if (!IS_ALIGNED(virt, bytes) || (!clear && !IS_ALIGNED(rq->phys, bytes)))
427 panic("vmm_pt_apply: huge mapping not naturally aligned");
428
429 /* Walk stays on one core from start to finish: pt_walk_enter/exit
430 * stamp per-core epoch, and shootdown at `out` runs
431 * after all pte_lock's are dropped */
432 enum irql walk_irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
433
434 pt_walk_enter();
435 enum errno err = ERR_OK;
436 struct page_table *tables[PT_LEVELS];
437 enum irql irqls[PT_LEVELS - 1];
438 pte_t *entries[PT_LEVELS - 1];
439 paddr_t to_free[PT_LEVELS - 1] = {0};
440 int free_count = 0;
441 bool shootdown = false;
442
443 tables[0] = pml4;
444
445 int level = 0;
446 for (level = 0; level < leaf_level; level++) {
447
448#pragma GCC diagnostic push
449#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
450
451 pte_t *entry = &tables[level]->entries[pt_index(virt, level)];
452
453#pragma GCC diagnostic pop
454
455 entries[level] = entry;
456 irqls[level] = pte_lock(pt: entry);
457
458 if (!in_use(pte: *entry)) {
459 /* clear/unmap never builds tables: no table here means no leaf */
460 if (clear) {
461 level++;
462 goto out;
463 }
464 if ((err = pte_init(entry, flags: user_flag)) < 0) {
465 level++;
466 goto out;
467 }
468 }
469
470 /* present huge leaf where table was expected is a size mismatch
471 *
472 * only meaningful when present as non-present can have payload there */
473 kassert(!((*entry & PAGE_PRESENT) && (*entry & PAGE_HUGE)));
474
475 /* We can't descend through a shared table to modify a leaf, since that
476 * would edit every other range aliasing with this table */
477 kassert(!pte_is_shared(*entry));
478
479 tables[level + 1] = pt_next_table(entry: *entry);
480 }
481
482#pragma GCC diagnostic push
483#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
484
485 pte_t *last_entry =
486 &tables[leaf_level]->entries[pt_index(virt, level: leaf_level)];
487
488#pragma GCC diagnostic pop
489
490 enum irql last_irql = pte_lock(pt: last_entry);
491
492 bool was_present = *last_entry & PAGE_PRESENT;
493
494 /* page tables must match the caller's claims, PS bit is only meaningful
495 * for present bits as tagged non-present PTEs use bit 7 */
496 if (was_present) {
497 kassert((*last_entry & PAGE_HUGE) == want_huge);
498 if (handle_exist) {
499 err = ERR_EXIST;
500 pte_unlock(pt: last_entry, irql: last_irql);
501 goto out;
502 }
503 }
504
505 if (clear) {
506 atomic_store_explicit((pte_atomic_t *) last_entry, PTE_LOCK_BIT,
507 memory_order_release);
508 } else {
509 if (in_use(pte: *last_entry) && !modify)
510 panic(
511 "vmm_pt_apply: leaf in use without MODIFY_LEAF (double map?)");
512
513 atomic_store_explicit((pte_atomic_t *) last_entry,
514 build_leaf_pte(phys: rq->phys, flags, sz, vflags) |
515 PTE_LOCK_BIT,
516 memory_order_release);
517 }
518
519 /* Publish new PTE first */
520 shootdown = was_present;
521
522 pte_unlock(pt: last_entry, irql: last_irql);
523
524 /* used in unmap */
525 if (reclaim) {
526 for (int up = leaf_level; up > 0; up--) {
527 if (!vmm_is_table_empty(table: tables[up]))
528 break;
529
530 /* A shared table is referenced by entries this walk can't see,
531 * so freeing it here would pull the rug from every other range */
532 if (pte_is_shared(pte: *entries[up - 1]))
533 break;
534
535 to_free[free_count++] = hhdm_ptr_to_paddr(ptr: tables[up]);
536 *entries[up - 1] = PTE_LOCK_BIT;
537 }
538
539 /* Detaching table retries a present upper entry */
540 if (free_count)
541 shootdown = true;
542 }
543
544out:
545 for (int i = level - 1; i >= 0; i--)
546 pte_unlock(pt: entries[i], irql: irqls[i]);
547
548 /* Shootdown after all locks dropped, ASAN causes issues otherwise */
549 if (shootdown)
550 barrier_and_shootdown(flags: vflags, virt);
551
552 for (int i = 0; i < free_count; i++)
553 enqueue_pt_free(phys: to_free[i]);
554
555 pt_walk_exit();
556 irql_lower(old_level: walk_irql);
557 return err;
558}
559
560static struct page_table *alloc_uniform_pt(uint64_t entry) {
561 struct page_table *t = alloc_pt();
562 if (!t)
563 return NULL;
564
565 for (int i = 0; i < PT_ENTRIES; i++)
566 t->entries[i] = entry;
567
568 return t;
569}
570
571/* Point an entry at an already built subtree */
572static enum errno alias_install_one(struct page_table *pml4, vaddr_t virt,
573 int parent_level, uint64_t install) {
574 struct page_table *tables[PT_LEVELS];
575 pte_t *entries[PT_LEVELS - 1];
576 enum irql irqls[PT_LEVELS - 1];
577 enum errno err = ERR_OK;
578 int level = 0;
579
580 tables[0] = pml4;
581
582 for (level = 0; level < parent_level; level++) {
583
584#pragma GCC diagnostic push
585#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
586
587 pte_t *entry = &tables[level]->entries[pt_index(virt, level)];
588
589#pragma GCC diagnostic pop
590
591 entries[level] = entry;
592 irqls[level] = pte_lock(pt: entry);
593
594 if (!in_use(pte: *entry)) {
595 if ((err = pte_init(entry, flags: 0)) < 0) {
596 level++;
597 goto out;
598 }
599 }
600
601 kassert(!((*entry & PAGE_PRESENT) && (*entry & PAGE_HUGE)));
602 tables[level + 1] = pt_next_table(entry: *entry);
603 }
604
605#pragma GCC diagnostic push
606#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
607
608 pte_t *target =
609 &tables[parent_level]->entries[pt_index(virt, level: parent_level)];
610
611#pragma GCC diagnostic pop
612
613 enum irql tirql = pte_lock(pt: target);
614
615 if (in_use(pte: *target))
616 err = ERR_EXIST;
617 else
618 *target = install | PTE_LOCK_BIT;
619
620 pte_unlock(pt: target, irql: tirql);
621
622out:
623 for (int i = level - 1; i >= 0; i--)
624 pte_unlock(pt: entries[i], irql: irqls[i]);
625
626 return err;
627}
628
629/* Use shared subtrees to map every page in the len to one physical page */
630enum errno vmm_map_aliased(vaddr_t virt, size_t len, paddr_t phys,
631 page_flags_t leaf_flags, enum vmm_flags vflags) {
632 if (!len)
633 return ERR_OK;
634
635 if (!IS_PAGE_ALIGNED(phys))
636 return ERR_INVAL;
637
638 int parent_level = -1;
639 for (int lvl = PT_LEVEL_PML4; lvl <= PT_LEVEL_PD; lvl++) {
640 uint64_t granule = pt_level_granule(level: lvl);
641 if (IS_ALIGNED(virt, granule) && IS_ALIGNED(len, granule)) {
642 parent_level = lvl;
643 break;
644 }
645 }
646
647 if (parent_level < 0)
648 return ERR_INVAL;
649
650 struct page_table *child =
651 alloc_uniform_pt(entry: (phys & PAGE_PHYS_MASK) | leaf_flags | PAGE_PRESENT);
652 if (!child)
653 return ERR_NO_MEM;
654
655 struct page_table *built[PT_LEVELS];
656 int built_count = 0;
657 built[built_count++] = child;
658
659 for (int lvl = PT_LEVELS - 2; lvl > parent_level; lvl--) {
660 /* SHARED marks every entry on the aliased path, not just the one we
661 * install below: each of these points at a table reachable from every
662 * other entry in the alias, so descending through one to edit a leaf
663 * would rewrite the whole aliased range at once */
664 struct page_table *up =
665 alloc_uniform_pt(entry: hhdm_ptr_to_paddr(ptr: child) | PAGE_PRESENT |
666 PAGE_WRITE | PTE_SHARED_BIT);
667 if (!up) {
668 for (int i = 0; i < built_count; i++)
669 pmm_free_page(addr: hhdm_ptr_to_paddr(ptr: built[i]));
670 return ERR_NO_MEM;
671 }
672 built[built_count++] = up;
673 child = up;
674 }
675
676 uint64_t install =
677 hhdm_ptr_to_paddr(ptr: child) | PAGE_PRESENT | PAGE_WRITE | PTE_SHARED_BIT;
678 uint64_t granule = pt_level_granule(level: parent_level);
679 struct page_table *pml4 = kernel_pml4;
680
681 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
682 pt_walk_enter();
683
684 enum errno err = ERR_OK;
685 vaddr_t v = virt;
686 for (; v < virt + len; v += granule) {
687 if ((err = alias_install_one(pml4, virt: v, parent_level, install)) < 0)
688 break;
689 }
690
691 pt_walk_exit();
692
693 if (err < 0) {
694 for (vaddr_t u = virt; u < v; u += granule)
695 vmm_unmap_aliased(virt: u, len: granule, vflags);
696
697 for (int i = 0; i < built_count; i++)
698 pmm_free_page(addr: hhdm_ptr_to_paddr(ptr: built[i]));
699
700 irql_lower(old_level: irql);
701 return err;
702 }
703
704 irql_lower(old_level: irql);
705
706 /* Fresh mappings over non-present entries, so no stale translation can
707 * exist, although previously speculative walks may have cached */
708 memory_barrier();
709 if (!(vflags & VMM_FLAG_NO_TLB_SHOOTDOWN))
710 tlb_shootdown(addr: virt, true);
711
712 return ERR_OK;
713}
714
715/* Drop aliased entries without modifying the shared subtree they point at
716 *
717 * Tables are leaked to caller's bookkeeping, they may still be referenced
718 * by other ranges, and this layer can't know */
719static void vmm_unmap_aliased(vaddr_t virt, size_t len, enum vmm_flags vflags) {
720 if (!len)
721 return;
722
723 pt_walk_enter();
724
725 for (vaddr_t v = virt; v < virt + len;) {
726 struct page_table *table = kernel_pml4;
727 int level = 0;
728 uint64_t granule = pt_level_granule(level: PT_LEVEL_PD);
729
730 for (; level <= PT_LEVEL_PD; level++) {
731
732#pragma GCC diagnostic push
733#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
734
735 pte_t *entry = &table->entries[pt_index(virt: v, level)];
736
737#pragma GCC diagnostic pop
738
739 enum irql irql = pte_lock(pt: entry);
740 pte_t val = *entry;
741
742 if (pte_is_shared(pte: val)) {
743 *entry = PTE_LOCK_BIT;
744 pte_unlock(pt: entry, irql);
745 granule = pt_level_granule(level);
746 barrier_and_shootdown(flags: vflags, virt: v);
747 goto next;
748 }
749
750 pte_unlock(pt: entry, irql);
751
752 if (!(val & PAGE_PRESENT) || (val & PAGE_HUGE))
753 break;
754
755 table = pt_next_table(entry: val);
756 }
757
758 next:
759 v += granule;
760 }
761
762 pt_walk_exit();
763}
764
765/* Private copy of a shared table
766 *
767 * The source is immutable when vmm_map_aliased builds it: each path into
768 * it becomes SHARED and nothing descends through a SHARED entry to edit
769 * leaves, so plain copies can't cause races with writes. The lock bit
770 * is masked anyways, since a lock copied into a table no one else can
771 * see would never get released */
772static struct page_table *pt_clone_shared(struct page_table *src) {
773 struct page_table *dst = alloc_pt();
774 if (!dst)
775 return NULL;
776
777 for (int i = 0; i < PT_ENTRIES; i++)
778 dst->entries[i] = src->entries[i] & ~PTE_LOCK_BIT;
779
780 return dst;
781}
782
783/* Give `virt` a private walk down to the parent of a `leaf_size` leaf,
784 * allowing following maps to write that leaf without touching every other
785 * range that aliases the same tables
786 *
787 * Every SHARED entry on the path is replaced by a private copy of its child,
788 * so the other 511 entries in the copy still point into the alias,
789 * meaning unrelated addresses resolve through the shared tables */
790enum errno vmm_unshare_path(vaddr_t virt, enum vmm_map_page_size leaf_size,
791 enum vmm_flags vflags) {
792 int leaf_level = map_leaf_level(sz: leaf_size);
793
794 struct page_table *tables[PT_LEVELS];
795 pte_t *entries[PT_LEVELS - 1];
796 enum irql irqls[PT_LEVELS - 1];
797 enum errno err = ERR_OK;
798 bool unshared = false;
799 int level = 0;
800
801 tables[0] = kernel_pml4;
802
803 enum irql outer = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
804 pt_walk_enter();
805
806 for (level = 0; level < leaf_level; level++) {
807
808#pragma GCC diagnostic push
809#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
810
811 pte_t *entry = &tables[level]->entries[pt_index(virt, level)];
812
813#pragma GCC diagnostic pop
814
815 entries[level] = entry;
816 irqls[level] = pte_lock(pt: entry);
817
818 if (!in_use(pte: *entry)) {
819 level++;
820 goto out;
821 }
822
823 kassert(!((*entry & PAGE_PRESENT) && (*entry & PAGE_HUGE)));
824
825 if (pte_is_shared(pte: *entry)) {
826 struct page_table *priv = pt_clone_shared(src: pt_next_table(entry: *entry));
827 if (!priv) {
828 err = ERR_NO_MEM;
829 level++;
830 goto out;
831 }
832
833 /* Swap the target, keep flags, drop SHARED */
834 pte_t cur = atomic_load_explicit((pte_atomic_t *) entry,
835 memory_order_acquire);
836 atomic_store_explicit(
837 (pte_atomic_t *) entry,
838 (hhdm_ptr_to_paddr(ptr: priv) & PAGE_PHYS_MASK) |
839 (cur & ~(PAGE_PHYS_MASK | PTE_SHARED_BIT)),
840 memory_order_release);
841 unshared = true;
842 }
843
844 tables[level + 1] = pt_next_table(entry: *entry);
845 }
846
847out:
848 for (int i = level - 1; i >= 0; i--)
849 pte_unlock(pt: entries[i], irql: irqls[i]);
850
851 /* After locks, IPI cores that may be spinning, since they can't answer
852 * until we let go */
853 if (unshared)
854 barrier_and_shootdown(flags: vflags, virt);
855
856 pt_walk_exit();
857
858 irql_lower(old_level: outer);
859
860 return err;
861}
862
863enum errno vmm_map_page_full(struct vmm_map_request *rq) {
864 if (unlikely(text_phys_end && (rq->page_flags & PAGE_WRITE) &&
865 rq->phys < text_phys_end &&
866 rq->phys + map_page_bytes(rq->page_size) > text_phys_start))
867 panic(
868 "writable alias of kernel text: virt 0x%lx phys 0x%lx flags 0x%lx",
869 (uint64_t) rq->virt, (uint64_t) rq->phys,
870 (uint64_t) rq->page_flags);
871
872 return vmm_pt_apply(rq);
873}
874
875/* Tear down a single leaf and reclaim every page table it leaves empty. */
876void vmm_unmap_page_full(struct vmm_map_request *rq) {
877 struct vmm_map_request req = *rq;
878 req.vmm_flags |= VMM_FLAG_CLEAR_LEAF;
879 req.is_unmap_internal = true;
880 (void) vmm_pt_apply(rq: &req);
881}
882
883enum errno vmm_map_page_internal(vaddr_t virt, paddr_t phys, page_flags_t flags,
884 enum vmm_flags vflags,
885 enum vmm_map_page_size size) {
886 struct vmm_map_request rq = {
887 .virt = virt,
888 .phys = phys,
889 .page_flags = flags,
890 .vmm_flags = vflags,
891 .page_size = size,
892 };
893 return vmm_map_page_full(rq: &rq);
894}
895
896enum errno vmm_map_page_user_internal(struct page_table *pml4, vaddr_t virt,
897 paddr_t phys, page_flags_t flags,
898 enum vmm_flags vflags,
899 enum vmm_map_page_size size) {
900 struct vmm_map_request rq = {
901 .pml4 = pml4,
902 .virt = virt,
903 .phys = phys,
904 .page_flags = flags,
905 .vmm_flags = vflags | VMM_FLAG_USER,
906 .page_size = size,
907 };
908 return vmm_map_page_full(rq: &rq);
909}
910
911enum errno vmm_mark_demand_page_internal(vaddr_t virt,
912 enum demand_page_flags flags,
913 enum vmm_map_page_size size) {
914 struct pte_tagged ptag = {
915 .type = PTE_TAG_TYPE_DEMAND_PAGED,
916 .payload = flags,
917 };
918
919 uint64_t packed = pte_tagged_pack(pt: &ptag);
920
921 struct vmm_map_request rq = {
922 .pml4 = kernel_pml4,
923 .virt = virt,
924 .phys = 0,
925 .page_flags = packed,
926 .vmm_flags = VMM_FLAG_MODIFY_LEAF,
927 .page_size = size,
928 };
929
930 return vmm_map_page_full(rq: &rq);
931}
932
933enum errno vmm_map_demand_page_internal(vaddr_t virt, paddr_t phys,
934 enum demand_page_flags flags,
935 enum vmm_map_page_size size) {
936 uint64_t pflags = PAGE_PRESENT;
937 if (flags & DEMAND_PAGE_FLAG_WRITABLE)
938 pflags |= PAGE_WRITE;
939
940 if (flags & DEMAND_PAGE_FLAG_XD)
941 pflags |= PAGE_XD;
942
943 struct vmm_map_request rq = {
944 .pml4 = kernel_pml4,
945 .virt = virt,
946 .phys = phys,
947 .page_flags = pflags,
948 .vmm_flags = VMM_FLAG_MODIFY_LEAF | VMM_FLAG_HANDLE_PTE_EXISTING |
949 VMM_FLAG_NO_TLB_SHOOTDOWN,
950 .page_size = size,
951 };
952
953 return vmm_map_page_full(rq: &rq);
954}
955
956enum errno vmm_mark_demand_page_user_internal(struct page_table *pml4,
957 vaddr_t virt,
958 enum demand_page_flags flags,
959 enum vmm_map_page_size size) {
960 struct pte_tagged ptag = {
961 .type = PTE_TAG_TYPE_DEMAND_PAGED,
962 .payload = flags,
963 };
964
965 uint64_t packed = pte_tagged_pack(pt: &ptag);
966
967 struct vmm_map_request rq = {
968 .pml4 = pml4,
969 .virt = virt,
970 .phys = 0,
971 .page_flags = packed,
972
973 /* USER here is merely nominal, map_page_full ignores it */
974 .vmm_flags = VMM_FLAG_USER | VMM_FLAG_MODIFY_LEAF,
975 .page_size = size,
976 };
977
978 return vmm_map_page_full(rq: &rq);
979}
980
981void vmm_unmap_page_internal(vaddr_t virt, enum vmm_flags vflags,
982 enum vmm_map_page_size size) {
983 struct vmm_map_request rq = {
984 .virt = virt,
985 .vmm_flags = vflags,
986 .page_size = size,
987 };
988 vmm_unmap_page_full(rq: &rq);
989}
990
991static pte_t vmm_walk_leaf(struct page_table *root, vaddr_t virt,
992 int *out_level) {
993 enum irql irql = irql_raise(new_level: IRQL_HIGH_LEVEL);
994 pt_walk_enter();
995
996 struct page_table *table = root;
997 uint64_t snap = 0;
998 int level;
999
1000 for (level = 0; level < PT_LEVEL_PT; level++) {
1001 uint64_t index = pt_index(virt, level);
1002
1003#pragma GCC diagnostic push
1004#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
1005
1006 /* assert so that clang inlines this instead of creating a call to
1007 * __atomic_load (from c lib, breaks under -nostdlib), gcc inlines */
1008 snap = atomic_load_explicit((_Atomic pte_t *) __builtin_assume_aligned(
1009 &table->entries[index], sizeof(pte_t)),
1010 memory_order_acquire);
1011
1012#pragma GCC diagnostic pop
1013
1014 if (!(snap & PAGE_PRESENT)) {
1015 snap = 0;
1016 goto out;
1017 }
1018
1019 if (snap & PAGE_HUGE)
1020 goto out;
1021
1022 table = pt_next_table(entry: snap);
1023 }
1024
1025#pragma GCC diagnostic push
1026#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
1027
1028 snap = atomic_load_explicit(
1029 (_Atomic pte_t *) __builtin_assume_aligned(
1030 &table->entries[pt_index(virt, level: PT_LEVEL_PT)], sizeof(pte_t)),
1031 memory_order_acquire);
1032
1033#pragma GCC diagnostic pop
1034
1035out:
1036 pt_walk_exit();
1037
1038 if (out_level)
1039 *out_level = level;
1040
1041 irql_lower(old_level: irql);
1042
1043 return snap;
1044}
1045
1046paddr_t vmm_get_phys_internal(uintptr_t virt, enum vmm_flags vflags) {
1047 (void) vflags;
1048
1049 int level;
1050 uint64_t snap = vmm_walk_leaf(root: kernel_pml4, virt, out_level: &level);
1051
1052 if (!(snap & PAGE_PRESENT))
1053 return (uintptr_t) -1;
1054
1055 if (level == PT_LEVEL_PDPT)
1056 return (snap & PAGE_1GB_PHYS_MASK) + (virt & (PAGE_1GB - 1));
1057
1058 if (level == PT_LEVEL_PD)
1059 return (snap & PAGE_2MB_PHYS_MASK) + (virt & (PAGE_2MB - 1));
1060
1061 return (snap & PAGE_PHYS_MASK) + (virt & 0xFFF);
1062}
1063
1064pte_t vmm_get_leaf_pte_internal(vaddr_t virt, enum vmm_flags vflags) {
1065 (void) vflags;
1066 return vmm_walk_leaf(root: kernel_pml4, virt, NULL);
1067}
1068
1069void *vmm_map(paddr_t paddr, vaddr_t vaddr, uint64_t len, uint64_t flags,
1070 enum vmm_flags vflags) {
1071 if (len == 0)
1072 return NULL;
1073
1074 uintptr_t phys_start = PAGE_ALIGN_DOWN(paddr);
1075 uintptr_t offset = paddr - phys_start;
1076
1077 uint64_t total_len = len + offset;
1078 uint64_t total_pages = (total_len + PAGE_SIZE - 1) / PAGE_SIZE;
1079
1080 enum errno e = ERR_OK;
1081 uint64_t mapped = 0;
1082
1083 for (; mapped < total_pages; mapped++) {
1084 e = vmm_map_page(vaddr + mapped * PAGE_SIZE,
1085 phys_start + mapped * PAGE_SIZE,
1086 PAGE_PRESENT | PAGE_WRITE | flags, vflags);
1087 if (e < 0)
1088 goto unwind;
1089 }
1090
1091 return (void *) (vaddr + offset);
1092
1093unwind:
1094 for (uint64_t i = 0; i < mapped; i++)
1095 vmm_unmap_page(vaddr + i * PAGE_SIZE, vflags);
1096
1097 return NULL;
1098}
1099
1100void vmm_unmap(void *addr, uint64_t len, enum vmm_flags vflags) {
1101 uintptr_t virt_addr = (uintptr_t) addr;
1102 uintptr_t page_offset = virt_addr & (PAGE_SIZE - 1);
1103 uintptr_t aligned_virt = PAGE_ALIGN_DOWN(virt_addr);
1104
1105 uint64_t total_len = len + page_offset;
1106 uint64_t total_pages = PAGES_NEEDED_FOR(total_len);
1107
1108 for (uint64_t i = 0; i < total_pages; i++) {
1109 vmm_unmap_page(aligned_virt + i * PAGE_SIZE, vflags);
1110 }
1111}
1112
1113void *vmm_map_bump_internal(uintptr_t addr, uint64_t len, uint64_t flags,
1114 enum vmm_flags vflags) {
1115 if (global.current_bootstage >= BOOTSTAGE_LATE)
1116 log_warn_once("vmm_map_bump called after BOOTSTAGE_LATE...");
1117
1118 if (len == 0)
1119 return NULL;
1120
1121 uintptr_t phys_start = PAGE_ALIGN_DOWN(addr);
1122 uintptr_t offset = addr - phys_start;
1123
1124 uint64_t total_len = len + offset;
1125 uint64_t total_pages = (total_len + PAGE_SIZE - 1) / PAGE_SIZE;
1126
1127 uint64_t span = total_pages * PAGE_SIZE;
1128 if (total_pages != 0 && span / PAGE_SIZE != total_pages)
1129 return NULL;
1130
1131 uintptr_t virt_start =
1132 atomic_load_explicit(&vmm_map_top, memory_order_relaxed);
1133 do {
1134 if (virt_start > VMM_MAP_LIMIT || span > VMM_MAP_LIMIT - virt_start)
1135 return NULL;
1136 } while (!atomic_compare_exchange_weak_explicit(
1137 &vmm_map_top, &virt_start, virt_start + span, memory_order_acq_rel,
1138 memory_order_relaxed));
1139
1140 enum errno e = ERR_OK;
1141 uint64_t mapped = 0;
1142
1143 for (; mapped < total_pages; mapped++) {
1144 e = vmm_map_page(virt_start + mapped * PAGE_SIZE,
1145 phys_start + mapped * PAGE_SIZE,
1146 PAGE_PRESENT | PAGE_WRITE | flags, vflags);
1147 if (e < 0)
1148 goto unwind;
1149 }
1150
1151 return (void *) (virt_start + offset);
1152
1153unwind:
1154 for (uint64_t i = 0; i < mapped; i++)
1155 vmm_unmap_page(virt_start + i * PAGE_SIZE, vflags);
1156
1157 uintptr_t expected = virt_start + span;
1158 atomic_compare_exchange_strong_explicit(&vmm_map_top, &expected, virt_start,
1159 memory_order_acq_rel,
1160 memory_order_relaxed);
1161
1162 return NULL;
1163}
1164
1165void vmm_unmap_virt(void *addr, uint64_t len, enum vmm_flags vflags) {
1166 uintptr_t virt_addr = (uintptr_t) addr;
1167 uintptr_t page_offset = virt_addr & (PAGE_SIZE - 1);
1168 uintptr_t aligned_virt = PAGE_ALIGN_DOWN(virt_addr);
1169
1170 uint64_t total_len = len + page_offset;
1171 uint64_t total_pages = PAGES_NEEDED_FOR(total_len);
1172
1173 for (uint64_t i = 0; i < total_pages; i++) {
1174 vmm_unmap_page(aligned_virt + i * PAGE_SIZE, vflags);
1175 }
1176}
1177
1178struct page_table *vmm_phys_to_pml4(paddr_t paddr) {
1179 return (struct page_table *) hhdm_paddr_to_ptr(p: paddr);
1180}
1181