1#include <cmdline.h>
2#include <compiler/core.h>
3#include <compiler/intrinsic.h>
4#include <console/printf.h>
5#include <global.h>
6#include <irq/idt.h>
7#include <limine.h>
8#include <linker/symbols.h>
9#include <math/bit.h>
10#include <mem/address_range.h>
11#include <mem/asan.h>
12#include <mem/demand_page.h>
13#include <mem/hhdm.h>
14#include <mem/page_table.h>
15#include <mem/pmm.h>
16#include <mem/tlb.h>
17#include <mem/vmm.h>
18#include <sch/sched.h>
19#include <smp/smp.h>
20#include <stdbool.h>
21#include <stdint.h>
22#include <string.h>
23#include <sync/spinlock.h>
24
25#include "mem/slab/internal.h"
26
27#define KERNEL_PML4_START_INDEX 256
28
29static paddr_t text_phys_start;
30static paddr_t text_phys_end;
31
32bool vmm_phys_is_kernel_text(paddr_t phys) {
33 return text_phys_end && phys >= text_phys_start && phys < text_phys_end;
34}
35
36static inline bool in_use(pte_t pte) {
37 return pte_in_use(pte: (pte_atomic_t *) &pte);
38}
39
40enum pt_level {
41 PT_LEVEL_PML4 = 0,
42 PT_LEVEL_PDPT = 1,
43 PT_LEVEL_PD = 2,
44 PT_LEVEL_PT = 3,
45};
46
47struct pt_deferred_free {
48 paddr_t phys;
49 uint64_t epoch;
50 struct pt_deferred_free *next;
51};
52
53struct pt_walk {
54 struct page_table *tables[PT_LEVELS];
55 pte_t *entries[PT_LEVELS - 1];
56 enum irql irqls[PT_LEVELS - 1];
57 int depth;
58};
59
60ADDRESS_RANGE_DEFINE(hhdm, .base = 0xFFFF800000000000ULL,
61 /* default size: from the base up to the slab heap */
62 .size = SLAB_HEAP_START - 0xFFFF800000000000ULL);
63
64ADDRESS_RANGE_DEFINE(kernel, .base = 0xffffffff80000000,
65 .size = 0); /* Filled in at boot */
66
67CMDLINE_DEFINE_VAR(mem, ADDRESS_RANGE(hhdm).size,
68 .desc = "Cap on physical memory the allocator will use",
69 .arg = "<hex bytes>", .default_val = "0x700000000000");
70
71bool hhdm_vaddr_in_range(vaddr_t vaddr) {
72 return address_range_addr_in_range(ar: &ADDRESS_RANGE(hhdm), vaddr);
73}
74
75bool hhdm_paddr_in_range(paddr_t paddr) {
76 return paddr < ADDRESS_RANGE(hhdm).size;
77}
78
79bool hhdm_ptr_in_range(void *ptr) {
80 return hhdm_vaddr_in_range(vaddr: (vaddr_t) ptr);
81}
82
83static struct pt_deferred_free *pt_free_list;
84static struct spinlock pt_free_lock = SPINLOCK_INIT;
85static struct page_table *kernel_pml4 = NULL;
86static atomic_uintptr_t vmm_map_top = VMM_MAP_BASE;
87static void vmm_unmap_aliased(vaddr_t virt, size_t len, enum tlb_mode mode);
88
89static inline struct page_table *alloc_pt(void) {
90 paddr_t phys = pmm_alloc_page();
91 if (!phys)
92 return NULL;
93
94 void *virt = hhdm_paddr_to_ptr(p: phys);
95 memset(virt, 0, PAGE_SIZE);
96 return virt;
97}
98
99static enum err pte_init(pte_t *entry, uint64_t flags) {
100 struct page_table *new_table = alloc_pt();
101 if (!new_table)
102 return ERR_NO_MEM;
103
104 uintptr_t new_table_phys = hhdm_ptr_to_paddr(ptr: new_table);
105 *entry = new_table_phys | PAGE_PRESENT | PAGE_WRITE | PTE_LOCK_BIT | flags;
106 return ERR_OK;
107}
108
109static enum irql pte_lock(pte_t *pt) TSA_NO_ANALYSIS {
110 return pte_lock_irql(pte: (void *) pt);
111}
112
113static void pte_unlock(pte_t *pt, enum irql irql) TSA_NO_ANALYSIS {
114 pte_unlock_irql(pte: (void *) pt, old_irql: irql);
115}
116
117static void barrier_and_shootdown(enum tlb_mode mode, vaddr_t virt) {
118 cpu_memory_barrier();
119 tlb_invlpg(virt);
120
121 if (mode != TLB_MODE_NONE)
122 tlb_shootdown(.mode = mode, .payload.addr = virt);
123}
124
125static inline uint64_t pt_index(uintptr_t virt, int level) {
126 return (virt >> (PT_SHIFT_L4 - level * PT_STRIDE)) & PT_INDEX_MASK;
127}
128
129static inline struct page_table *pt_next_table(pte_t entry) {
130 return hhdm_paddr_to_ptr(p: entry & PAGE_PHYS_MASK);
131}
132
133/* Bytes of address space covered by an entry at `level`
134 *
135 * PML4 entries span 512GB, PDPT 1GB, PD 2MB, PT 4KB */
136static inline uint64_t pt_level_granule(int level) {
137 return BIT(PT_SHIFT_L4 - level * PT_STRIDE);
138}
139
140static inline void pt_walk_enter(void) {
141 if (global.current_bootstage >= BOOTSTAGE_MID_MP) {
142 uint64_t e = atomic_load_acq(&global.pt_epoch);
143 atomic_store_release(&smp_core(TOPC_IRQL)->pt_seen_epoch, e);
144 }
145}
146
147static inline void pt_walk_exit(void) {
148 if (global.current_bootstage >= BOOTSTAGE_MID_MP) {
149 atomic_store_release(&smp_core(TOPC_IRQL)->pt_seen_epoch, UINT64_MAX);
150 }
151}
152
153static void enqueue_pt_free(paddr_t phys) {
154 if (global.current_bootstage < BOOTSTAGE_MID_MP)
155 return pmm_free_page(addr: phys);
156
157 struct pt_deferred_free *n =
158 (struct pt_deferred_free *) hhdm_paddr_to_ptr(p: phys);
159
160 uint64_t e = atomic_inc_return_new_acq_rel(&global.pt_epoch);
161 n->phys = phys;
162 n->epoch = e;
163
164 enum irql irql = spin_lock_high(&pt_free_lock);
165 n->next = pt_free_list;
166 pt_free_list = n;
167 spin_unlock(&pt_free_lock, irql);
168}
169
170void vmm_reclaim_page_tables(void) {
171 if (global.current_bootstage < BOOTSTAGE_LATE)
172 return;
173
174 if (smp_read(TOPC_IRQL, reclaiming_page_tables))
175 return;
176
177 enforce(irql_get() == IRQL_DISPATCH_LEVEL, INVALID_CONTEXT);
178 smp_write(TOPC_IRQL, reclaiming_page_tables, true);
179
180 uint64_t min_epoch = UINT64_MAX;
181 struct core *cpu;
182 for_each_cpu_struct(cpu) {
183 uint64_t seen = atomic_load_acq(&cpu->pt_seen_epoch);
184 if (seen < min_epoch)
185 min_epoch = seen;
186 }
187
188 enum irql irql = IRQL_NONE;
189 if (!spin_trylock_high(&pt_free_lock, &irql))
190 goto out;
191
192 struct pt_deferred_free *to_free = NULL;
193 struct pt_deferred_free **pp = &pt_free_list;
194 while (*pp) {
195 struct pt_deferred_free *n = *pp;
196 if (n->epoch >= min_epoch) {
197 pp = &n->next;
198 } else {
199 *pp = n->next;
200 n->next = to_free;
201 to_free = n;
202 }
203 }
204
205 spin_unlock(&pt_free_lock, irql);
206
207 while (to_free) {
208 struct pt_deferred_free *n = to_free;
209 paddr_t phys = n->phys;
210 to_free = n->next;
211 pmm_free_pages(phys, 1);
212 }
213
214out:
215 smp_write(TOPC_IRQL, reclaiming_page_tables, false);
216}
217
218uintptr_t vmm_make_user_pml4(void) {
219 struct page_table *user_pml4 = alloc_pt();
220 if (!user_pml4) {
221 panic(OUT_OF_MEMORY, "user pml4");
222 }
223
224 for (int i = KERNEL_PML4_START_INDEX; i < PT_ENTRIES; i++) {
225 user_pml4->entries[i] = kernel_pml4->entries[i];
226 }
227
228 return hhdm_ptr_to_paddr(ptr: user_pml4);
229}
230
231/* Leaf frames are not touched here, this gets rid of structural page tables */
232static void vmm_free_user_subtree(struct page_table *pdpt) {
233 for (int i3 = 0; i3 < PT_ENTRIES; i3++) {
234 pte_t e3 = pdpt->entries[i3];
235 if (!in_use(pte: e3) || (e3 & PAGE_HUGE))
236 continue;
237
238 struct page_table *pd = pt_next_table(entry: e3);
239 for (int i2 = 0; i2 < PT_ENTRIES; i2++) {
240 pte_t e2 = pd->entries[i2];
241 if (!in_use(pte: e2) || (e2 & PAGE_HUGE))
242 continue;
243
244 struct page_table *pt = pt_next_table(entry: e2);
245 enqueue_pt_free(phys: hhdm_ptr_to_paddr(ptr: pt));
246 }
247 enqueue_pt_free(phys: hhdm_ptr_to_paddr(ptr: pd));
248 }
249}
250
251void vmm_unmap_all_user_pages_full(struct page_table *pml4,
252 struct vmm_request rq) {
253 cc_ignore(rq);
254
255 for (int i4 = 0; i4 < KERNEL_PML4_START_INDEX; i4++) {
256 pte_t e4 = pml4->entries[i4];
257 if (!in_use(pte: e4))
258 continue;
259
260 struct page_table *pdpt = pt_next_table(entry: e4);
261 vmm_free_user_subtree(pdpt);
262 enqueue_pt_free(phys: hhdm_ptr_to_paddr(ptr: pdpt));
263 pml4->entries[i4] = 0;
264 }
265}
266
267void vmm_init(struct limine_memmap_response *memmap,
268 struct limine_executable_address_response *xa) {
269 kernel_pml4 = alloc_pt();
270 if (!kernel_pml4)
271 panic(OUT_OF_MEMORY, "kernel PML4");
272
273 uintptr_t kernel_pml4_phys = hhdm_ptr_to_paddr(ptr: kernel_pml4);
274
275 uint64_t kernel_phys_start = xa->physical_base;
276 uint64_t kernel_virt_start = xa->virtual_base;
277 uint64_t kernel_virt_end = (uint64_t) &__kernel_virt_end;
278 uint64_t kernel_size = kernel_virt_end - kernel_virt_start;
279 ADDRESS_RANGE(kernel).size = kernel_size;
280
281 paddr_t dummy_phys = pmm_alloc_page();
282 uint8_t *dummy_virt = hhdm_paddr_to_ptr(p: dummy_phys);
283 memset(dummy_virt, 0xFF, PAGE_SIZE);
284
285 enum err e;
286
287 uint64_t text_start = (uint64_t) &__stext;
288 uint64_t text_end = (uint64_t) &__etext;
289 text_phys_start = kernel_phys_start + (text_start - kernel_virt_start);
290 text_phys_end = kernel_phys_start + (text_end - kernel_virt_start);
291
292 for (uint64_t i = 0; i < kernel_size; i += PAGE_SIZE) {
293 uint64_t virt = kernel_virt_start + i;
294 uint64_t flags = PAGE_PRESENT;
295
296 if (virt < text_start || virt >= text_end)
297 flags |= PAGE_WRITE;
298
299 e = vmm_map_page(virt, kernel_phys_start + i, .page_flags = flags);
300 if (e < 0)
301 panic(INIT_FAILED, "mapping kernel: %s", errno_to_str(e));
302 }
303
304#ifdef DEBUG_ASAN
305 for (uintptr_t addr = kernel_virt_start; addr < kernel_virt_end;
306 addr += PAGE_SIZE) {
307 uint64_t shadow_addr = ASAN_SHADOW_OFFSET + (addr >> ASAN_SHADOW_SCALE);
308 shadow_addr = PAGE_ALIGN_DOWN(shadow_addr);
309 vmm_map_page(shadow_addr, dummy_phys,
310 .page_flags = PAGE_PRESENT | PAGE_WRITE,
311 .vmm_flags = VMM_FLAG_MODIFY_LEAF);
312 }
313#endif
314
315 for (uint64_t i = 0; i < memmap->entry_count; i++) {
316 struct limine_memmap_entry *entry = memmap->entries[i];
317 if (entry->type == LIMINE_MEMMAP_BAD_MEMORY ||
318 entry->type == LIMINE_MEMMAP_RESERVED ||
319 entry->type == LIMINE_MEMMAP_ACPI_NVS) {
320 continue;
321 }
322
323 uint64_t base = entry->base;
324 uint64_t len = entry->length;
325 uint64_t end = base + len;
326 uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_XD;
327
328 if (entry->type == LIMINE_MEMMAP_FRAMEBUFFER) {
329 flags |= PAGE_WRITETHROUGH;
330 }
331
332 uint64_t phys = base;
333 while (phys < end) {
334 uint64_t virt = hhdm_paddr_to_vaddr(p: phys);
335
336 bool overlaps_text =
337 phys < text_phys_end && phys + PAGE_2MB > text_phys_start;
338
339 bool can_use_2mb = ((phys % PAGE_2MB) == 0) &&
340 ((virt % PAGE_2MB) == 0) &&
341 ((end - phys) >= PAGE_2MB) && !overlaps_text;
342
343 if (can_use_2mb) {
344 e = vmm_map_page(virt, phys, .page_flags = flags,
345 .page_size = VMM_MAP_PAGE_SIZE_2MB);
346 phys += PAGE_2MB;
347 } else {
348 uint64_t page_flags = flags;
349
350 if (phys >= text_phys_start && phys < text_phys_end)
351 page_flags &= ~PAGE_WRITE;
352
353 e = vmm_map_page(virt, phys, .page_flags = page_flags);
354 phys += PAGE_SIZE;
355 }
356 if (e < 0)
357 panic(INIT_FAILED, "mapping kernel: %s", errno_to_str(e));
358 }
359 }
360
361 asm volatile("mov %0, %%cr3" : : "r"(kernel_pml4_phys) : "memory");
362}
363
364static inline bool vmm_is_table_empty(struct page_table *table) {
365 for (int i = 0; i < PT_ENTRIES; i++) {
366 if (in_use(pte: table->entries[i]))
367 return false;
368 }
369 return true;
370}
371
372static inline int map_leaf_level(enum vmm_map_page_size sz) {
373 switch (sz) {
374 case VMM_MAP_PAGE_SIZE_1GB: return PT_LEVEL_PDPT;
375
376 case VMM_MAP_PAGE_SIZE_2MB: return PT_LEVEL_PD;
377
378 default: return PT_LEVEL_PT;
379 }
380}
381
382static inline size_t map_page_bytes(enum vmm_map_page_size sz) {
383 switch (sz) {
384 case VMM_MAP_PAGE_SIZE_1GB: return PAGE_1GB;
385 case VMM_MAP_PAGE_SIZE_2MB: return PAGE_2MB;
386 default: return PAGE_SIZE;
387 }
388}
389
390static inline pte_t build_leaf_pte(paddr_t phys, uint64_t flags,
391 enum vmm_map_page_size sz,
392 enum vmm_flags vflags) {
393 uint64_t extra_flags = (vflags & VMM_FLAG_MODIFY_LEAF) ? 0 : PAGE_PRESENT;
394 if (sz != VMM_MAP_PAGE_SIZE_4KB && !(vflags & VMM_FLAG_MODIFY_LEAF))
395 extra_flags |= PAGE_HUGE;
396
397 flags |= extra_flags;
398 pte_t leaf = (phys & PAGE_PHYS_MASK) | flags;
399 return leaf;
400}
401
402static enum err vmm_pt_apply(struct vmm_map_request *rq) {
403 bool reclaim = rq->is_unmap_internal;
404 vaddr_t virt = rq->virt;
405 if (virt == 0)
406 panic("CANNOT MAP PAGE 0x0!!!");
407
408 struct page_table *pml4 = rq->pml4 ? rq->pml4 : kernel_pml4;
409 enum vmm_flags vflags = rq->rq.vmm_flags;
410 enum vmm_map_page_size sz = rq->rq.page_size;
411 int leaf_level = map_leaf_level(sz);
412 bool want_huge = sz != VMM_MAP_PAGE_SIZE_4KB;
413
414 bool clear = vflags & VMM_FLAG_CLEAR_LEAF;
415 bool modify = vflags & VMM_FLAG_MODIFY_LEAF;
416 bool handle_exist = vflags & VMM_FLAG_HANDLE_PTE_EXISTING;
417
418 uint64_t user_flag =
419 ((vflags & VMM_FLAG_USER) && !modify) ? PAGE_USER_ALLOWED : 0;
420 uint64_t flags = rq->rq.page_flags | user_flag;
421
422 size_t bytes = map_page_bytes(sz);
423 if (!IS_ALIGNED(virt, bytes) || (!clear && !IS_ALIGNED(rq->phys, bytes)))
424 panic("vmm_pt_apply: huge mapping not naturally aligned");
425
426 /* Walk stays on one core from start to finish: pt_walk_enter/exit
427 * stamp per-core epoch, and shootdown at `out` runs
428 * after all pte_lock's are dropped */
429 enum irql walk_irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
430
431 pt_walk_enter();
432 enum err err = ERR_OK;
433 struct page_table *tables[PT_LEVELS];
434 enum irql irqls[PT_LEVELS - 1];
435 pte_t *entries[PT_LEVELS - 1];
436 paddr_t to_free[PT_LEVELS - 1] = {0};
437 int free_count = 0;
438 bool shootdown = false;
439
440 tables[0] = pml4;
441
442 int level = 0;
443 for (level = 0; level < leaf_level; level++) {
444
445#pragma GCC diagnostic push
446#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
447
448 pte_t *entry = &tables[level]->entries[pt_index(virt, level)];
449
450#pragma GCC diagnostic pop
451
452 entries[level] = entry;
453 irqls[level] = pte_lock(pt: entry);
454
455 if (!in_use(pte: *entry)) {
456 /* clear/unmap never builds tables: no table here means no leaf */
457 if (clear) {
458 level++;
459 goto out;
460 }
461 if ((err = pte_init(entry, flags: user_flag)) < 0) {
462 level++;
463 goto out;
464 }
465 }
466
467 /* present huge leaf where table was expected is a size mismatch
468 *
469 * only meaningful when present as non-present can have payload there */
470 enforce(!((*entry & PAGE_PRESENT) && (*entry & PAGE_HUGE)));
471
472 /* We can't descend through a shared table to modify a leaf, since that
473 * would edit every other range aliasing with this table */
474 enforce(!pte_is_shared(*entry));
475
476 tables[level + 1] = pt_next_table(entry: *entry);
477 }
478
479#pragma GCC diagnostic push
480#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
481
482 pte_t *last_entry =
483 &tables[leaf_level]->entries[pt_index(virt, level: leaf_level)];
484
485#pragma GCC diagnostic pop
486
487 enum irql last_irql = pte_lock(pt: last_entry);
488
489 bool was_present = *last_entry & PAGE_PRESENT;
490
491 /* page tables must match the caller's claims, PS bit is only meaningful
492 * for present bits as tagged non-present PTEs use bit 7 */
493 if (was_present) {
494 enforce((*last_entry & PAGE_HUGE) == want_huge);
495 if (handle_exist) {
496 err = ERR_EXIST;
497 pte_unlock(pt: last_entry, irql: last_irql);
498 goto out;
499 }
500 }
501
502 if (clear) {
503 atomic_store_release((pte_atomic_t *) last_entry, PTE_LOCK_BIT);
504 } else {
505 if (in_use(pte: *last_entry) && !modify)
506 panic(DUPLICATE, "leaf in use without MODIFY_LEAF");
507
508 atomic_store_release((pte_atomic_t *) last_entry,
509 build_leaf_pte(rq->phys, flags, sz, vflags) |
510 PTE_LOCK_BIT);
511 }
512
513 /* Publish new PTE first */
514 shootdown = was_present;
515
516 pte_unlock(pt: last_entry, irql: last_irql);
517
518 /* used in unmap */
519 if (reclaim) {
520 for (int up = leaf_level; up > 0; up--) {
521 if (!vmm_is_table_empty(table: tables[up]))
522 break;
523
524 /* A shared table is referenced by entries this walk can't see,
525 * so freeing it here would pull the rug from every other range */
526 if (pte_is_shared(pte: *entries[up - 1]))
527 break;
528
529 to_free[free_count++] = hhdm_ptr_to_paddr(ptr: tables[up]);
530 *entries[up - 1] = PTE_LOCK_BIT;
531 }
532
533 /* Detaching table retries a present upper entry */
534 if (free_count)
535 shootdown = true;
536 }
537
538out:
539 for (int i = level - 1; i >= 0; i--)
540 pte_unlock(pt: entries[i], irql: irqls[i]);
541
542 /* Shootdown after all locks dropped, ASAN causes issues otherwise */
543 if (shootdown)
544 barrier_and_shootdown(mode: rq->rq.tlb_mode, virt);
545
546 for (int i = 0; i < free_count; i++)
547 enqueue_pt_free(phys: to_free[i]);
548
549 pt_walk_exit();
550 irql_lower(old_level: walk_irql);
551 return err;
552}
553
554static struct page_table *alloc_uniform_pt(uint64_t entry) {
555 struct page_table *t = alloc_pt();
556 if (!t)
557 return NULL;
558
559 for (int i = 0; i < PT_ENTRIES; i++)
560 t->entries[i] = entry;
561
562 return t;
563}
564
565/* Point an entry at an already built subtree */
566static enum err alias_install_one(struct page_table *pml4, vaddr_t virt,
567 int parent_level, uint64_t install) {
568 struct page_table *tables[PT_LEVELS];
569 pte_t *entries[PT_LEVELS - 1];
570 enum irql irqls[PT_LEVELS - 1];
571 enum err err = ERR_OK;
572 int level = 0;
573
574 tables[0] = pml4;
575
576 for (level = 0; level < parent_level; level++) {
577
578#pragma GCC diagnostic push
579#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
580
581 pte_t *entry = &tables[level]->entries[pt_index(virt, level)];
582
583#pragma GCC diagnostic pop
584
585 entries[level] = entry;
586 irqls[level] = pte_lock(pt: entry);
587
588 if (!in_use(pte: *entry)) {
589 if ((err = pte_init(entry, flags: 0)) < 0) {
590 level++;
591 goto out;
592 }
593 }
594
595 enforce(!((*entry & PAGE_PRESENT) && (*entry & PAGE_HUGE)));
596 tables[level + 1] = pt_next_table(entry: *entry);
597 }
598
599#pragma GCC diagnostic push
600#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
601
602 pte_t *target =
603 &tables[parent_level]->entries[pt_index(virt, level: parent_level)];
604
605#pragma GCC diagnostic pop
606
607 enum irql tirql = pte_lock(pt: target);
608
609 if (in_use(pte: *target))
610 err = ERR_EXIST;
611 else
612 *target = install | PTE_LOCK_BIT;
613
614 pte_unlock(pt: target, irql: tirql);
615
616out:
617 for (int i = level - 1; i >= 0; i--)
618 pte_unlock(pt: entries[i], irql: irqls[i]);
619
620 return err;
621}
622
623/* Use shared subtrees to map every page in the len to one physical page */
624enum err vmm_map_aliased_full(vaddr_t virt, size_t len, paddr_t phys,
625 struct vmm_request rq) {
626 page_flags_t leaf_flags = rq.page_flags;
627 if (!len)
628 return ERR_OK;
629
630 if (!IS_PAGE_ALIGNED(phys))
631 return ERR_INVAL;
632
633 int parent_level = -1;
634 for (int lvl = PT_LEVEL_PML4; lvl <= PT_LEVEL_PD; lvl++) {
635 uint64_t granule = pt_level_granule(level: lvl);
636 if (IS_ALIGNED(virt, granule) && IS_ALIGNED(len, granule)) {
637 parent_level = lvl;
638 break;
639 }
640 }
641
642 if (parent_level < 0)
643 return ERR_INVAL;
644
645 struct page_table *child =
646 alloc_uniform_pt(entry: (phys & PAGE_PHYS_MASK) | leaf_flags | PAGE_PRESENT);
647 if (!child)
648 return ERR_NO_MEM;
649
650 struct page_table *built[PT_LEVELS];
651 int built_count = 0;
652 built[built_count++] = child;
653
654 for (int lvl = PT_LEVELS - 2; lvl > parent_level; lvl--) {
655 /* SHARED marks every entry on the aliased path, not just the one we
656 * install below: each of these points at a table reachable from every
657 * other entry in the alias, so descending through one to edit a leaf
658 * would rewrite the whole aliased range at once */
659 struct page_table *up =
660 alloc_uniform_pt(entry: hhdm_ptr_to_paddr(ptr: child) | PAGE_PRESENT |
661 PAGE_WRITE | PTE_SHARED_BIT);
662 if (!up) {
663 for (int i = 0; i < built_count; i++)
664 pmm_free_page(addr: hhdm_ptr_to_paddr(ptr: built[i]));
665 return ERR_NO_MEM;
666 }
667 built[built_count++] = up;
668 child = up;
669 }
670
671 uint64_t install =
672 hhdm_ptr_to_paddr(ptr: child) | PAGE_PRESENT | PAGE_WRITE | PTE_SHARED_BIT;
673 uint64_t granule = pt_level_granule(level: parent_level);
674 struct page_table *pml4 = kernel_pml4;
675
676 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
677 pt_walk_enter();
678
679 enum err err = ERR_OK;
680 vaddr_t v = virt;
681 for (; v < virt + len; v += granule) {
682 if ((err = alias_install_one(pml4, virt: v, parent_level, install)) < 0)
683 break;
684 }
685
686 pt_walk_exit();
687
688 if (err < 0) {
689 for (vaddr_t u = virt; u < v; u += granule)
690 vmm_unmap_aliased(virt: u, len: granule, mode: rq.tlb_mode);
691
692 for (int i = 0; i < built_count; i++)
693 pmm_free_page(addr: hhdm_ptr_to_paddr(ptr: built[i]));
694
695 irql_lower(old_level: irql);
696 return err;
697 }
698
699 irql_lower(old_level: irql);
700
701 /* Fresh mappings over non-present entries, so no stale translation can
702 * exist, although previously speculative walks may have cached */
703 cpu_memory_barrier();
704 tlb_shootdown(.mode = rq.tlb_mode, .payload.addr = virt);
705
706 return ERR_OK;
707}
708
709static void vmm_unmap_aliased(vaddr_t virt, size_t len, enum tlb_mode mode) {
710 if (!len)
711 return;
712
713 pt_walk_enter();
714
715 for (vaddr_t v = virt; v < virt + len;) {
716 struct page_table *table = kernel_pml4;
717 int level = 0;
718 uint64_t granule = pt_level_granule(level: PT_LEVEL_PD);
719
720 for (; level <= PT_LEVEL_PD; level++) {
721
722#pragma GCC diagnostic push
723#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
724
725 pte_t *entry = &table->entries[pt_index(virt: v, level)];
726
727#pragma GCC diagnostic pop
728
729 enum irql irql = pte_lock(pt: entry);
730 pte_t val = *entry;
731
732 if (pte_is_shared(pte: val)) {
733 *entry = PTE_LOCK_BIT;
734 pte_unlock(pt: entry, irql);
735 granule = pt_level_granule(level);
736 barrier_and_shootdown(mode, virt: v);
737 goto next;
738 }
739
740 pte_unlock(pt: entry, irql);
741
742 if (!(val & PAGE_PRESENT) || (val & PAGE_HUGE))
743 break;
744
745 table = pt_next_table(entry: val);
746 }
747
748 next:
749 v += granule;
750 }
751
752 pt_walk_exit();
753}
754
755/* Private copy of a shared table
756 *
757 * The source is immutable when vmm_map_aliased builds it: each path into
758 * it becomes SHARED and nothing descends through a SHARED entry to edit
759 * leaves, so plain copies can't cause races with writes. The lock bit
760 * is masked anyways, since a lock copied into a table no one else can
761 * see would never get released */
762static struct page_table *pt_clone_shared(struct page_table *src) {
763 struct page_table *dst = alloc_pt();
764 if (!dst)
765 return NULL;
766
767 for (int i = 0; i < PT_ENTRIES; i++)
768 dst->entries[i] = src->entries[i] & ~PTE_LOCK_BIT;
769
770 return dst;
771}
772
773/* Give `virt` a private walk down to the parent of a `leaf_size` leaf,
774 * allowing following maps to write that leaf without touching every other
775 * range that aliases the same tables
776 *
777 * Every SHARED entry on the path is replaced by a private copy of its child,
778 * so the other 511 entries in the copy still point into the alias,
779 * meaning unrelated addresses resolve through the shared tables */
780enum err vmm_unshare_path_full(vaddr_t virt, struct vmm_request rq) {
781 int leaf_level = map_leaf_level(sz: rq.page_size);
782
783 struct page_table *tables[PT_LEVELS];
784 pte_t *entries[PT_LEVELS - 1];
785 enum irql irqls[PT_LEVELS - 1];
786 enum err err = ERR_OK;
787 bool unshared = false;
788 int level = 0;
789
790 tables[0] = kernel_pml4;
791
792 enum irql outer = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
793 pt_walk_enter();
794
795 for (level = 0; level < leaf_level; level++) {
796
797#pragma GCC diagnostic push
798#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
799
800 pte_t *entry = &tables[level]->entries[pt_index(virt, level)];
801
802#pragma GCC diagnostic pop
803
804 entries[level] = entry;
805 irqls[level] = pte_lock(pt: entry);
806
807 if (!in_use(pte: *entry)) {
808 level++;
809 goto out;
810 }
811
812 enforce(!((*entry & PAGE_PRESENT) && (*entry & PAGE_HUGE)));
813
814 if (pte_is_shared(pte: *entry)) {
815 struct page_table *priv = pt_clone_shared(src: pt_next_table(entry: *entry));
816 if (!priv) {
817 err = ERR_NO_MEM;
818 level++;
819 goto out;
820 }
821
822 /* Swap the target, keep flags, drop SHARED */
823 pte_t cur = atomic_load_acq((pte_atomic_t *) entry);
824 atomic_store_release(
825 (pte_atomic_t *) entry,
826 (hhdm_ptr_to_paddr(priv) & PAGE_PHYS_MASK) |
827 (cur & ~(PAGE_PHYS_MASK | PTE_SHARED_BIT)));
828 unshared = true;
829 }
830
831 tables[level + 1] = pt_next_table(entry: *entry);
832 }
833
834out:
835 for (int i = level - 1; i >= 0; i--)
836 pte_unlock(pt: entries[i], irql: irqls[i]);
837
838 /* After locks, IPI cores that may be spinning, since they can't answer
839 * until we let go */
840 if (unshared)
841 barrier_and_shootdown(mode: rq.tlb_mode, virt);
842
843 pt_walk_exit();
844
845 irql_lower(old_level: outer);
846
847 return err;
848}
849
850static enum err vmm_map_page_apply(struct vmm_map_request *rq) {
851 if (cc_unlikely(text_phys_end && (rq->rq.page_flags & PAGE_WRITE) &&
852 rq->phys < text_phys_end &&
853 rq->phys + map_page_bytes(rq->rq.page_size) >
854 text_phys_start))
855 panic(
856 "writable alias of kernel text: virt 0x%lx phys 0x%lx flags 0x%lx",
857 (uint64_t) rq->virt, (uint64_t) rq->phys,
858 (uint64_t) rq->rq.page_flags);
859
860 return vmm_pt_apply(rq);
861}
862
863/* Tear down a single leaf and reclaim every page table it leaves empty. */
864void vmm_unmap_page_full(vaddr_t virt, struct vmm_request rq) {
865 struct vmm_map_request req = {
866 .virt = virt,
867 .rq = rq,
868 .is_unmap_internal = true,
869 };
870 req.rq.vmm_flags |= VMM_FLAG_CLEAR_LEAF;
871 (void) vmm_pt_apply(rq: &req);
872}
873
874enum err vmm_map_page_full(vaddr_t virt, paddr_t phys, struct vmm_request rq) {
875 struct vmm_map_request req = {
876 .virt = virt,
877 .phys = phys,
878 .rq = rq,
879 };
880 return vmm_map_page_apply(rq: &req);
881}
882
883enum err vmm_map_page_user_full(struct page_table *pml4, vaddr_t virt,
884 paddr_t phys, struct vmm_request rq) {
885 struct vmm_map_request req = {
886 .pml4 = pml4,
887 .virt = virt,
888 .phys = phys,
889 .rq = rq,
890 };
891 req.rq.vmm_flags |= VMM_FLAG_USER;
892 return vmm_map_page_apply(rq: &req);
893}
894
895enum err vmm_mark_demand_page_full(vaddr_t virt, enum demand_page_flags flags,
896 struct vmm_request rq) {
897 struct pte_tagged ptag = {
898 .type = PTE_TAG_TYPE_DEMAND_PAGED,
899 .payload = flags,
900 };
901
902 struct vmm_map_request req = {
903 .pml4 = kernel_pml4,
904 .virt = virt,
905 .phys = 0,
906 .rq = rq,
907 };
908 req.rq.page_flags = pte_tagged_pack(pt: &ptag);
909 req.rq.vmm_flags |= VMM_FLAG_MODIFY_LEAF;
910
911 return vmm_map_page_apply(rq: &req);
912}
913
914enum err vmm_map_demand_page_full(vaddr_t virt, paddr_t phys,
915 enum demand_page_flags flags,
916 struct vmm_request rq) {
917 uint64_t pflags = PAGE_PRESENT;
918 if (flags & DEMAND_PAGE_FLAG_WRITABLE)
919 pflags |= PAGE_WRITE;
920
921 if (flags & DEMAND_PAGE_FLAG_XD)
922 pflags |= PAGE_XD;
923
924 struct vmm_map_request req = {
925 .pml4 = kernel_pml4,
926 .virt = virt,
927 .phys = phys,
928 .rq = rq,
929 };
930 req.rq.page_flags = pflags;
931 req.rq.vmm_flags |= VMM_FLAG_MODIFY_LEAF | VMM_FLAG_HANDLE_PTE_EXISTING;
932
933 return vmm_map_page_apply(rq: &req);
934}
935
936enum err vmm_mark_demand_page_user_full(struct page_table *pml4, vaddr_t virt,
937 enum demand_page_flags flags,
938 struct vmm_request rq) {
939 struct pte_tagged ptag = {
940 .type = PTE_TAG_TYPE_DEMAND_PAGED,
941 .payload = flags,
942 };
943
944 struct vmm_map_request req = {
945 .pml4 = pml4,
946 .virt = virt,
947 .phys = 0,
948 .rq = rq,
949 };
950 req.rq.page_flags = pte_tagged_pack(pt: &ptag);
951
952 /* USER here is merely nominal, map_page_apply ignores it */
953 req.rq.vmm_flags |= VMM_FLAG_USER | VMM_FLAG_MODIFY_LEAF;
954
955 return vmm_map_page_apply(rq: &req);
956}
957
958static pte_t vmm_walk_leaf(struct page_table *root, vaddr_t virt,
959 int *out_level) {
960 enum irql irql = irql_raise(new_level: IRQL_HIGH_LEVEL);
961 pt_walk_enter();
962
963 struct page_table *table = root;
964 uint64_t snap = 0;
965 int level;
966
967 for (level = 0; level < PT_LEVEL_PT; level++) {
968 uint64_t index = pt_index(virt, level);
969
970#pragma GCC diagnostic push
971#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
972
973 /* assert so that clang inlines this instead of creating a call to
974 * __atomic_load (from c lib, breaks under -nostdlib), gcc inlines */
975 snap = atomic_load_acq((atomic(pte_t) *) ci_assume_aligned(
976 &table->entries[index], sizeof(pte_t)));
977
978#pragma GCC diagnostic pop
979
980 if (!(snap & PAGE_PRESENT)) {
981 snap = 0;
982 goto out;
983 }
984
985 if (snap & PAGE_HUGE)
986 goto out;
987
988 table = pt_next_table(entry: snap);
989 }
990
991#pragma GCC diagnostic push
992#pragma GCC diagnostic ignored "-Waddress-of-packed-member"
993
994 snap = atomic_load_acq((atomic(pte_t) *) ci_assume_aligned(
995 &table->entries[pt_index(virt, PT_LEVEL_PT)], sizeof(pte_t)));
996
997#pragma GCC diagnostic pop
998
999out:
1000 pt_walk_exit();
1001
1002 if (out_level)
1003 *out_level = level;
1004
1005 irql_lower(old_level: irql);
1006
1007 return snap;
1008}
1009
1010paddr_t vmm_get_phys_full(vaddr_t virt, struct vmm_request rq) {
1011 cc_ignore(rq);
1012
1013 int level;
1014 uint64_t snap = vmm_walk_leaf(root: kernel_pml4, virt, out_level: &level);
1015
1016 if (!(snap & PAGE_PRESENT))
1017 return (uintptr_t) -1;
1018
1019 if (level == PT_LEVEL_PDPT)
1020 return (snap & PAGE_1GB_PHYS_MASK) + (virt & (PAGE_1GB - 1));
1021
1022 if (level == PT_LEVEL_PD)
1023 return (snap & PAGE_2MB_PHYS_MASK) + (virt & (PAGE_2MB - 1));
1024
1025 return (snap & PAGE_PHYS_MASK) + (virt & 0xFFF);
1026}
1027
1028pte_t vmm_get_leaf_pte_full(vaddr_t virt, struct vmm_request rq) {
1029 cc_ignore(rq);
1030 return vmm_walk_leaf(root: kernel_pml4, virt, NULL);
1031}
1032
1033static enum err vmm_map_pages(vaddr_t virt, paddr_t phys, uint64_t pages,
1034 struct vmm_request rq) {
1035 struct vmm_request page_rq = rq;
1036 page_rq.page_flags |= PAGE_PRESENT | PAGE_WRITE;
1037
1038 for (uint64_t i = 0; i < pages; i++) {
1039 enum err e = vmm_map_page_full(virt: virt + i * PAGE_SIZE,
1040 phys: phys + i * PAGE_SIZE, rq: page_rq);
1041 if (e < 0) {
1042 for (uint64_t j = 0; j < i; j++)
1043 vmm_unmap_page_full(virt: virt + j * PAGE_SIZE, rq);
1044 return e;
1045 }
1046 }
1047 return ERR_OK;
1048}
1049
1050void *vmm_map_full(paddr_t paddr, vaddr_t vaddr, uint64_t len,
1051 struct vmm_request rq) {
1052 if (len == 0)
1053 return NULL;
1054
1055 uintptr_t phys_start = PAGE_ALIGN_DOWN(paddr);
1056 uintptr_t offset = paddr - phys_start;
1057 uint64_t total_pages = PAGES_NEEDED_FOR(len + offset);
1058
1059 if (vmm_map_pages(virt: vaddr, phys: phys_start, pages: total_pages, rq) < 0)
1060 return NULL;
1061
1062 return (void *) (vaddr + offset);
1063}
1064
1065void vmm_unmap_full(void *addr, uint64_t len, struct vmm_request rq) {
1066 uintptr_t virt_addr = (uintptr_t) addr;
1067 uintptr_t page_offset = virt_addr & (PAGE_SIZE - 1);
1068 uintptr_t aligned_virt = PAGE_ALIGN_DOWN(virt_addr);
1069
1070 uint64_t total_len = len + page_offset;
1071 uint64_t total_pages = PAGES_NEEDED_FOR(total_len);
1072
1073 for (uint64_t i = 0; i < total_pages; i++) {
1074 vmm_unmap_page_full(virt: aligned_virt + i * PAGE_SIZE, rq);
1075 }
1076}
1077
1078void *vmm_map_bump_full(uintptr_t addr, uint64_t len, struct vmm_request rq) {
1079 if (global.current_bootstage >= BOOTSTAGE_LATE)
1080 log_msg_once(LOG_WARN, "vmm_map_bump called after BOOTSTAGE_LATE...");
1081
1082 if (len == 0)
1083 return NULL;
1084
1085 uintptr_t phys_start = PAGE_ALIGN_DOWN(addr);
1086 uintptr_t offset = addr - phys_start;
1087
1088 uint64_t total_len = len + offset;
1089 uint64_t total_pages = PAGES_NEEDED_FOR(total_len);
1090
1091 uint64_t span = total_pages * PAGE_SIZE;
1092 if (total_pages != 0 && span / PAGE_SIZE != total_pages)
1093 return NULL;
1094
1095 uintptr_t virt_start = atomic_load_rlx(&vmm_map_top);
1096 do {
1097 if (virt_start > VMM_MAP_LIMIT || span > VMM_MAP_LIMIT - virt_start)
1098 return NULL;
1099 } while (!atomic_cas_weak(&vmm_map_top, &virt_start, virt_start + span,
1100 mo_acq_rel, mo_rlx));
1101
1102 if (vmm_map_pages(virt: virt_start, phys: phys_start, pages: total_pages, rq) < 0) {
1103 uintptr_t expected = virt_start + span;
1104 atomic_cas_strong(&vmm_map_top, &expected, virt_start, mo_acq_rel,
1105 mo_rlx);
1106 return NULL;
1107 }
1108
1109 return (void *) (virt_start + offset);
1110}
1111
1112struct page_table *vmm_phys_to_pml4(paddr_t paddr) {
1113 return (struct page_table *) hhdm_paddr_to_ptr(p: paddr);
1114}
1115