1#include <global.h>
2#include <math/align.h>
3#include <mem/alloc.h>
4#include <mem/alloc_or_die.h>
5#include <mem/fixed_size_alloc.h>
6#include <mem/hhdm.h>
7#include <mem/pmm.h>
8#include <string.h>
9
10static inline size_t fixed_size_header_size(struct fixed_size_range *fsr) {
11 return ALIGN_UP(sizeof(struct fixed_size_page_hdr), fsr->attrs.obj_align);
12}
13
14static inline size_t fixed_size_per_page(struct fixed_size_range *fsr) {
15 return (PAGE_SIZE - fixed_size_header_size(fsr)) / fsr->full_node_size;
16}
17
18static inline struct fixed_size_page_hdr *fixed_size_page_of(void *o) {
19 return (struct fixed_size_page_hdr *) ALIGN_DOWN((uintptr_t) o, PAGE_SIZE);
20}
21
22static inline struct fixed_size_node *fsn_for_obj(struct fixed_size_range *fsr,
23 void *obj) {
24 return (struct fixed_size_node *) (obj + fsr->full_node_size -
25 sizeof(struct fixed_size_node));
26}
27
28static inline void *obj_for_fsn(struct fixed_size_range *fsr,
29 struct fixed_size_node *fsn) {
30 return (void *) ((uintptr_t) fsn -
31 (fsr->full_node_size - sizeof(struct fixed_size_node)));
32}
33
34static inline void *fixed_size_obj_n(struct fixed_size_range *fsr,
35 struct fixed_size_page_hdr *hdr,
36 uint32_t n) {
37 uint8_t *objs = (uint8_t *) ((uintptr_t) hdr + fixed_size_header_size(fsr));
38 return (objs + fsr->full_node_size * n);
39}
40
41static void fixed_size_drop_page(struct fixed_size_range *fsr,
42 struct fixed_size_page_hdr *hdr) {
43 SPINLOCK_ASSERT_HELD(&fsr->lock);
44 for (uint32_t i = 0; i < hdr->total; i++) {
45 void *obj = fixed_size_obj_n(fsr, hdr, n: i);
46 struct fixed_size_node *fsn = fsn_for_obj(fsr, obj);
47
48 if (fsr->attrs.deinit_obj)
49 fsr->attrs.deinit_obj(obj);
50
51 list_del_init(entry: &fsn->list_node);
52 }
53
54 list_del(entry: &hdr->page_list);
55 fsr->empty_pages--;
56 pmm_free_page(addr: hhdm_vaddr_to_paddr(v: (uintptr_t) hdr));
57}
58
59static bool fixed_size_refill(struct fixed_size_range *fsr) {
60 SPINLOCK_ASSERT_HELD(&fsr->lock);
61 uintptr_t phys = pmm_alloc_page();
62 if (!phys)
63 return false;
64
65 uintptr_t virt = hhdm_paddr_to_vaddr(p: phys);
66
67 struct fixed_size_page_hdr *hdr = (struct fixed_size_page_hdr *) virt;
68 hdr->total = fixed_size_per_page(fsr);
69 hdr->free_count = fixed_size_per_page(fsr);
70 INIT_LIST_HEAD(list: &hdr->page_list);
71 list_add_tail(new: &hdr->page_list, head: &fsr->fl_pages);
72 hdr->domain = fsr->domain;
73 fsr->empty_pages++;
74
75 for (uint32_t i = 0; i < hdr->total; i++) {
76 void *obj = fixed_size_obj_n(fsr, hdr, n: i);
77 struct fixed_size_node *fsn = fsn_for_obj(fsr, obj);
78 INIT_LIST_HEAD(list: &fsn->list_node);
79 list_add_tail(new: &fsn->list_node, head: &fsr->freelist);
80
81 if (fsr->attrs.init_obj) {
82 memset(obj, 0, fsr->attrs.obj_size);
83 fsr->attrs.init_obj(obj);
84 }
85 }
86
87 return true;
88}
89
90void *fixed_size_alloc(struct fixed_size_range *fsr) {
91 enum irql irql = spin_lock(lock: &fsr->lock);
92 if (list_empty(head: &fsr->freelist)) {
93 if (!fixed_size_refill(fsr)) {
94 spin_unlock(lock: &fsr->lock, old: irql);
95 return NULL;
96 }
97 }
98
99 struct list_head *pop = list_pop_front_init(head: &fsr->freelist);
100 struct fixed_size_node *fsn = (struct fixed_size_node *) pop;
101 void *obj = obj_for_fsn(fsr, fsn);
102
103 struct fixed_size_page_hdr *hdr = fixed_size_page_of(o: obj);
104
105 if (hdr->free_count == hdr->total)
106 fsr->empty_pages--;
107
108 hdr->free_count--;
109
110 spin_unlock(lock: &fsr->lock, old: irql);
111 return obj;
112}
113
114static void fixed_size_free_internal(struct fixed_size_range *fsr, void *obj) {
115 enum irql irql = spin_lock(lock: &fsr->lock);
116 struct fixed_size_page_hdr *hdr = fixed_size_page_of(o: obj);
117 struct fixed_size_node *fsn = fsn_for_obj(fsr, obj);
118
119 hdr->free_count++;
120 list_add_tail(new: &fsn->list_node, head: &fsr->freelist);
121
122 if (hdr->free_count != hdr->total) {
123 spin_unlock(lock: &fsr->lock, old: irql);
124 return;
125 }
126
127 fsr->empty_pages++;
128 if (fsr->empty_pages > FIXED_SIZE_KEEP_EMPTY_PAGES)
129 fixed_size_drop_page(fsr, hdr);
130
131 spin_unlock(lock: &fsr->lock, old: irql);
132}
133
134/* Wrapper around internal for domain FSRs, finds the right fsr */
135void fixed_size_free(struct fixed_size_range *fsr, void *obj) {
136 struct fixed_size_page_hdr *hdr = fixed_size_page_of(o: obj);
137 ssize_t domain = hdr->domain;
138 if (!fsr->perdomain_fsrs) {
139 kassert(domain == -1);
140
141 /* Just one */
142 return fixed_size_free_internal(fsr, obj);
143 }
144
145 kassert(fsr->perdomain_fsrs[domain]->domain == domain);
146 return fixed_size_free(fsr: fsr->perdomain_fsrs[domain], obj);
147}
148
149void fixed_size_reclaim_freelist_pages(struct fixed_size_range *fsr) {
150 struct list_head *pos, *tmp;
151
152 enum irql irql = spin_lock(lock: &fsr->lock);
153 list_for_each_safe(pos, tmp, &fsr->fl_pages) {
154 struct fixed_size_page_hdr *hdr =
155 container_of(pos, struct fixed_size_page_hdr, page_list);
156
157 if (hdr->free_count < hdr->total)
158 continue;
159
160 fixed_size_drop_page(fsr, hdr);
161 }
162
163 spin_unlock(lock: &fsr->lock, old: irql);
164}
165
166void fixed_size_range_init(struct fixed_size_range *fsr,
167 struct fixed_size_range_attributes *attrs) {
168 kassert(attrs->obj_size && attrs->obj_align, "Fill the fields out");
169 spinlock_init(lock: &fsr->lock);
170 fsr->attrs = *attrs;
171 INIT_LIST_HEAD(list: &fsr->freelist);
172 INIT_LIST_HEAD(list: &fsr->fl_pages);
173 fsr->empty_pages = 0;
174 fsr->full_node_size = ALIGN_UP(
175 sizeof(struct fixed_size_node) + attrs->obj_size, attrs->obj_align);
176
177 /* These are instantiated after init in the macro for perdomain */
178 fsr->perdomain_fsrs = NULL;
179 fsr->domain = -1;
180}
181
182struct fixed_size_range *
183fixed_size_range_create(struct fixed_size_range_attributes *attrs) {
184 struct fixed_size_range *fsr;
185 kassert(attrs);
186 if (attrs->bootstrap_mode) {
187 paddr_t phys = alloc_or_die(pmm_alloc_page());
188 vaddr_t virt = hhdm_paddr_to_vaddr(p: phys);
189 fsr = (void *) virt;
190 } else {
191 fsr = kmalloc(sizeof(struct fixed_size_range));
192 if (!fsr)
193 return NULL;
194 }
195
196 fixed_size_range_init(fsr, attrs);
197 return fsr;
198}
199