1#include <errno.h>
2#include <kassert.h>
3#include <mem/anon_vma.h>
4#include <mem/avc.h>
5#include <mem/fixed_size_alloc.h>
6#include <mem/mm.h>
7#include <mem/vma_range.h>
8#include <smp/perdomain.h>
9#include <structures/list.h>
10#include <types/refcount.h>
11
12FIXED_SIZE_RANGE_PERDOMAIN_DECLARE(vma_range,
13 .obj_size = sizeof(struct vma_range),
14 .obj_align = _Alignof(struct vma_range));
15
16void vma_range_init(struct vma_range *vma_range, struct mm *mm, vaddr_t start,
17 vaddr_t end, enum vma_range_protection prot) {
18 /* fold [start, end) into interval node */
19 vma_range->mm_node.interval.low = start;
20 vma_range->mm_node.interval.high = end - 1;
21
22 /* convention: object space offset == virtual page index, so
23 * vma_range_address() is an identity over [start, end) and makes pgoff
24 * survive fork unchanged (child keeping same VA layout), the folio's
25 * `index` resolves to the right VA in every mm that ends up mapping it */
26 vma_range->pgoff = start >> PAGE_4K_SHIFT;
27 vma_range->prot = prot;
28 vma_range->anon_vma = NULL;
29 vma_range->mm = mm;
30 INIT_LIST_HEAD(list: &vma_range->anon_vma_chain);
31}
32
33struct vma_range *vma_range_alloc(struct mm *mm, vaddr_t start, vaddr_t end,
34 enum vma_range_protection prot) {
35 struct vma_range *vma_range = FSR_PERDOMAIN_ALLOC(vma_range);
36 if (!vma_range)
37 return NULL;
38
39 vma_range_init(vma_range, mm, start, end, prot);
40 return vma_range;
41}
42
43void vma_range_free(struct vma_range *vma_range) {
44 kassert(list_empty(&vma_range->anon_vma_chain));
45 FSR_PERDOMAIN_FREE(vma_range, vma_range);
46}
47
48/* first write fualt on an anon VMA without anon_vma, get a fresh
49 * anon_vma + AVC that puts the VMA in the tree so rmap can find the pages
50 */
51
52/* NOTE: relies on caller holding mm->lock, when faults run concurrently
53 * under read lock, needs double checked allocation against a per-mm lock */
54enum errno vma_range_anon_prepare(struct vma_range *vma_range) {
55 if (vma_range->anon_vma)
56 return ERR_OK;
57
58 struct anon_vma *av = anon_vma_alloc();
59 if (!av)
60 return ERR_NO_MEM;
61
62 struct anon_vma_chain *avc = avc_alloc();
63 if (!avc) {
64 anon_vma_free(av); /* never linked, refcount still 0 */
65 return ERR_NO_MEM;
66 }
67
68 refcount_inc(rc: &av->refcount); /* the AVC about to link pins this object */
69 vma_range->anon_vma = av;
70 avc_link(vma_range, av, avc);
71 return ERR_OK;
72}
73
74enum errno vma_range_set_prot(struct vma_range *vma_range,
75 enum vma_range_protection prot) {
76 vma_range->prot = prot;
77 /* TODO: walk [start,end) and rewrite the PTE permission bits to
78 * match, then TLB-shootdown the range. Until page tables are wired, already
79 * mapped pages keep their old permissions and only new faults see `prot`.
80 */
81 return ERR_OK;
82}
83
84struct vma_range *vma_range_find(struct mm *mm, vaddr_t addr) {
85 struct vma_range *v = mm_vma_range_find(mm, addr);
86 return (v && vma_range_start(vma_range: v) <= addr) ? v : NULL;
87}
88
89struct vma_range *vma_range_find_intersection(struct mm *mm, vaddr_t s,
90 vaddr_t e) {
91 return mm_vma_range_find_intersection(mm, s, e);
92}
93
94struct vma_range *vma_range_next(struct vma_range *vma_range) {
95 struct rbit_node *n = rbit_next(node: &vma_range->mm_node);
96 return n ? rbit_entry(n, struct vma_range, mm_node) : NULL;
97}
98
99struct vma_range *vma_range_prev(struct vma_range *vma_range) {
100 struct rbit_node *n = rbit_prev(node: &vma_range->mm_node);
101 return n ? rbit_entry(n, struct vma_range, mm_node) : NULL;
102}
103
104/* split `vma_range` at `addr`, keeping the low half in `vma_range`
105 *
106 * returned VMA is the high half, both staying in mm->vma_range_tree
107 *
108 * caller holds mm->lock */
109struct vma_range *vma_range_split(struct vma_range *vma_range, vaddr_t addr) {
110 if (!IS_PAGE_ALIGNED(addr))
111 return NULL;
112
113 if (addr <= vma_range_start(vma_range) || addr >= vma_range_end(vma_range))
114 return NULL;
115
116 struct mm *mm = vma_range->mm;
117 vaddr_t old_start = vma_range_start(vma_range);
118 vaddr_t old_end = vma_range_end(vma_range);
119
120 struct vma_range *new = vma_range_alloc(mm, start: addr, end: old_end, prot: vma_range->prot);
121 if (!new)
122 return NULL;
123
124 new->pgoff = vma_range->pgoff + ((addr - old_start) >> PAGE_4K_SHIFT);
125
126 /* mirror anon linkage onto the new half */
127 if (vma_range->anon_vma) {
128 new->anon_vma = vma_range->anon_vma;
129 if (anon_vma_clone(dst: new, src: vma_range) != ERR_OK) {
130 new->anon_vma = NULL;
131 vma_range_free(vma_range: new);
132 return NULL;
133 }
134 }
135
136 /* shrink original to low half, re-keying in every tree it lives in
137 *
138 * interval.low is unchaged */
139 mm_vma_range_remove(mm, vma_range);
140 vma_range->mm_node.interval.high = addr - 1;
141
142 struct anon_vma_chain *avc;
143 list_for_each_entry(avc, &vma_range->anon_vma_chain, same_vma_range)
144 avc_rekey(avc);
145
146 mm_vma_range_insert(mm, vma_range);
147 mm_vma_range_insert(mm, vma_range: new);
148 return new;
149}
150