1#include <global.h>
2#include <math/align.h>
3#include <mem/alloc.h>
4#include <mem/alloc_or_die.h>
5#include <mem/fixed_size_alloc.h>
6#include <mem/hhdm.h>
7#include <mem/pmm.h>
8#include <string.h>
9
10static inline size_t fixed_size_header_size(struct fixed_size_range *fsr) {
11 return ALIGN_UP(sizeof(struct fixed_size_page_hdr), fsr->attrs.obj_align);
12}
13
14static inline size_t fixed_size_per_page(struct fixed_size_range *fsr) {
15 return (PAGE_SIZE - fixed_size_header_size(fsr)) / fsr->full_node_size;
16}
17
18static inline struct fixed_size_node *fsn_for_obj(struct fixed_size_range *fsr,
19 void *obj) {
20 return (struct fixed_size_node *) (obj + fsr->full_node_size -
21 sizeof(struct fixed_size_node));
22}
23
24static inline void *obj_for_fsn(struct fixed_size_range *fsr,
25 struct fixed_size_node *fsn) {
26 return (void *) ((uintptr_t) fsn -
27 (fsr->full_node_size - sizeof(struct fixed_size_node)));
28}
29
30static inline void *fixed_size_obj_n(struct fixed_size_range *fsr,
31 struct fixed_size_page_hdr *hdr,
32 uint32_t n) {
33 uint8_t *objs = (uint8_t *) ((uintptr_t) hdr + fixed_size_header_size(fsr));
34 return (objs + fsr->full_node_size * n);
35}
36
37static void fixed_size_drop_page(struct fixed_size_range *fsr,
38 struct fixed_size_page_hdr *hdr) {
39 SPINLOCK_ASSERT_HELD(&fsr->lock);
40 for (uint32_t i = 0; i < hdr->total; i++) {
41 void *obj = fixed_size_obj_n(fsr, hdr, n: i);
42 struct fixed_size_node *fsn = fsn_for_obj(fsr, obj);
43
44 if (fsr->attrs.deinit_obj)
45 fsr->attrs.deinit_obj(obj);
46
47 list_del_init(entry: &fsn->list_node);
48 }
49
50 list_del(entry: &hdr->page_list);
51 fsr->empty_pages--;
52 pmm_free_page(addr: hhdm_vaddr_to_paddr(v: (uintptr_t) hdr));
53}
54
55static bool fixed_size_refill(struct fixed_size_range *fsr) {
56 SPINLOCK_ASSERT_HELD(&fsr->lock);
57 uintptr_t phys = pmm_alloc_page();
58 if (!phys)
59 return false;
60
61 uintptr_t virt = hhdm_paddr_to_vaddr(p: phys);
62
63 struct fixed_size_page_hdr *hdr = (struct fixed_size_page_hdr *) virt;
64 hdr->total = fixed_size_per_page(fsr);
65 hdr->free_count = fixed_size_per_page(fsr);
66 INIT_LIST_HEAD(list: &hdr->page_list);
67 list_add_tail(new: &hdr->page_list, head: &fsr->fl_pages);
68 hdr->domain = fsr->domain;
69 fsr->empty_pages++;
70
71 for (uint32_t i = 0; i < hdr->total; i++) {
72 void *obj = fixed_size_obj_n(fsr, hdr, n: i);
73 struct fixed_size_node *fsn = fsn_for_obj(fsr, obj);
74 INIT_LIST_HEAD(list: &fsn->list_node);
75 list_add_tail(new: &fsn->list_node, head: &fsr->freelist);
76
77 if (fsr->attrs.init_obj) {
78 memset(obj, 0, fsr->attrs.obj_size);
79 fsr->attrs.init_obj(obj);
80 }
81 }
82
83 return true;
84}
85
86void *fixed_size_alloc(struct fixed_size_range *fsr) {
87 enum irql irql = spin_lock(&fsr->lock);
88 if (list_empty(head: &fsr->freelist)) {
89 if (!fixed_size_refill(fsr)) {
90 spin_unlock(&fsr->lock, irql);
91 return NULL;
92 }
93 }
94
95 struct list_head *pop = list_pop_front_init(head: &fsr->freelist);
96 struct fixed_size_node *fsn = (struct fixed_size_node *) pop;
97 void *obj = obj_for_fsn(fsr, fsn);
98
99 struct fixed_size_page_hdr *hdr = fixed_size_page_of(o: obj);
100
101 if (hdr->free_count == hdr->total)
102 fsr->empty_pages--;
103
104 hdr->free_count--;
105
106 spin_unlock(&fsr->lock, irql);
107 return obj;
108}
109
110static void fixed_size_free_internal(struct fixed_size_range *fsr, void *obj) {
111 enum irql irql = spin_lock(&fsr->lock);
112 struct fixed_size_page_hdr *hdr = fixed_size_page_of(o: obj);
113 struct fixed_size_node *fsn = fsn_for_obj(fsr, obj);
114
115 hdr->free_count++;
116 list_add_tail(new: &fsn->list_node, head: &fsr->freelist);
117
118 if (hdr->free_count != hdr->total) {
119 spin_unlock(&fsr->lock, irql);
120 return;
121 }
122
123 fsr->empty_pages++;
124 if (fsr->empty_pages > FIXED_SIZE_KEEP_EMPTY_PAGES)
125 fixed_size_drop_page(fsr, hdr);
126
127 spin_unlock(&fsr->lock, irql);
128}
129
130/* Wrapper around internal for domain FSRs, finds the right fsr */
131void fixed_size_free(struct fixed_size_range *fsr, void *obj) {
132 struct fixed_size_page_hdr *hdr = fixed_size_page_of(o: obj);
133 domain_id_t domain = hdr->domain;
134 if (!fsr->perdomain_fsrs) {
135 kassert(domain == DOMAIN_ID_NONE);
136
137 /* Just one */
138 return fixed_size_free_internal(fsr, obj);
139 }
140
141 kassert(fsr->perdomain_fsrs[domain]->domain == domain);
142 return fixed_size_free_internal(fsr: fsr->perdomain_fsrs[domain], obj);
143}
144
145void fixed_size_reclaim_freelist_pages(struct fixed_size_range *fsr) {
146 struct list_head *pos, *tmp;
147
148 enum irql irql = spin_lock(&fsr->lock);
149 list_for_each_safe(pos, tmp, &fsr->fl_pages) {
150 struct fixed_size_page_hdr *hdr =
151 container_of(pos, struct fixed_size_page_hdr, page_list);
152
153 if (hdr->free_count < hdr->total)
154 continue;
155
156 fixed_size_drop_page(fsr, hdr);
157 }
158
159 spin_unlock(&fsr->lock, irql);
160}
161
162void fixed_size_range_init(struct fixed_size_range *fsr,
163 struct fixed_size_range_attributes *attrs) {
164 kassert(attrs->obj_size && attrs->obj_align, "Fill the fields out");
165 spinlock_init(&fsr->lock);
166 fsr->attrs = *attrs;
167 INIT_LIST_HEAD(list: &fsr->freelist);
168 INIT_LIST_HEAD(list: &fsr->fl_pages);
169 fsr->empty_pages = 0;
170 fsr->full_node_size = ALIGN_UP(
171 sizeof(struct fixed_size_node) + attrs->obj_size, attrs->obj_align);
172
173 /* These are instantiated after init in the macro for perdomain */
174 fsr->perdomain_fsrs = NULL;
175 fsr->domain = DOMAIN_ID_NONE;
176}
177
178struct fixed_size_range *
179fixed_size_range_create(struct fixed_size_range_attributes *attrs) {
180 struct fixed_size_range *fsr;
181 kassert(attrs);
182 if (attrs->bootstrap_mode) {
183 paddr_t phys = alloc_or_die(pmm_alloc_page());
184 vaddr_t virt = hhdm_paddr_to_vaddr(p: phys);
185 fsr = (void *) virt;
186 } else {
187 fsr = kmalloc(sizeof(struct fixed_size_range));
188 if (!fsr)
189 return NULL;
190 }
191
192 fixed_size_range_init(fsr, attrs);
193 return fsr;
194}
195