1#include <mem/alloc_or_die.h>
2#include <mem/domain.h>
3#include <mem/slab.h>
4#include <smp/domain.h>
5#include <thread/daemon.h>
6
7#include "internal.h"
8#include "mem/domain/internal.h"
9
10void slab_domain_build_locality_lists(struct slab_domain *sdom) {
11 struct domain_buddy *buddy = sdom->domain->domain_buddy;
12 struct domain_zonelist *zl = &buddy->zonelist;
13
14 for (int i = 0; i < SLAB_TYPE_COUNT; i++) {
15 sdom->zonelists[i].count = zl->count;
16 sdom->zonelists[i].entries = kmalloc_or_die(
17 sizeof(struct slab_cache_ref) * zl->count, ALLOC_FLAGS_ZERO);
18 for (size_t j = 0; j < zl->count; j++) {
19 struct domain_zonelist_entry *zent = &zl->entries[j];
20 struct domain_buddy *bd = zent->domain;
21 size_t idx = bd - global.domain_buddies;
22 struct slab_domain *remote_sdom = global.domains[idx]->slab_domain;
23 sdom->zonelists[i].entries[j] = (struct slab_cache_ref){
24 .caches = remote_sdom->caches[i],
25 .type = i,
26 .locality = zent->distance,
27 .domain = remote_sdom,
28 };
29 }
30 }
31
32 sdom->zonelist_entry_count = zl->count;
33}
34
35void slab_init_caches(struct slab_caches *caches, enum slab_type type) {
36 for (size_t i = 0; i < slab_global.num_sizes; i++) {
37 struct slab_cache *cache = &caches->caches[i];
38 cache->type = type;
39 slab_cache_init(order: i, cache, ssc: &slab_global.class_sizes[i]);
40 }
41}
42
43void slab_domain_link_caches(struct slab_domain *domain,
44 struct slab_caches *caches) {
45 for (size_t i = 0; i < slab_global.num_sizes; i++) {
46 caches->caches[i].parent_domain = domain;
47 caches->caches[i].parent = caches;
48 }
49}
50
51static void slab_domain_init_cache(struct slab_domain *dom,
52 enum slab_type type) {
53 struct slab_caches *caches =
54 kmalloc_or_die(sizeof(struct slab_caches), ALLOC_FLAGS_ZERO);
55
56 dom->caches[type] = caches;
57 dom->caches[type]->caches = alloc_or_die(slab_caches_alloc());
58 slab_init_caches(caches, type);
59 slab_domain_link_caches(domain: dom, caches);
60}
61
62void slab_domain_init_caches(struct slab_domain *dom) {
63 for (int i = 0; i < SLAB_TYPE_COUNT; i++) {
64 slab_domain_init_cache(dom, type: i);
65 }
66}
67
68static size_t slab_bucket_reset(struct stat_bucket *bucket) {
69 struct slab_domain_bucket *db = bucket->private;
70 struct slab_domain *sd = bucket->parent->private;
71 struct slab_domain_bucket *agg = &sd->aggregate;
72 size_t stats_len =
73 sizeof(struct slab_domain_bucket) / sizeof(atomic_size_t);
74
75 atomic_size_t *parent_stats = (atomic_size_t *) agg;
76 atomic_size_t *bucket_stats = (atomic_size_t *) db;
77
78 /* Subtract this bucket's values from the parent */
79 for (size_t i = 0; i < stats_len; i++) {
80 size_t val = bucket_stats[i];
81 atomic_size_t *parent = &parent_stats[i];
82
83 /* This would underflow anyways... */
84 if (*parent < val) {
85 atomic_store(parent, 0);
86 } else {
87 atomic_fetch_sub(parent, val);
88 }
89 }
90
91 memset(db, 0, sizeof(struct slab_domain_bucket));
92 return 0;
93}
94
95void slab_domain_init_stats(struct slab_domain *domain) {
96 domain->stats = stat_series_create(SLAB_STAT_SERIES_CAPACITY,
97 SLAB_STAT_SERIES_BUCKET_US,
98 bucket_reset: slab_bucket_reset, private: domain);
99
100 domain->stats->private = domain;
101 domain->buckets =
102 kmalloc(sizeof(struct slab_domain_bucket) * SLAB_STAT_SERIES_CAPACITY,
103 ALLOC_FLAGS_ZERO);
104
105 if (!domain->stats || !domain->stats->buckets || !domain->buckets)
106 panic("Failed to create domain stat series");
107
108 struct stat_bucket *iter;
109 stat_series_for_each(domain->stats, iter) {
110 iter->private = &domain->buckets[__i];
111 }
112}
113
114static struct slab_cache *slab_domain_cache_for_slab(struct slab *slab) {
115 struct domain *d =
116 domain_for_addr(vmm_get_phys((vaddr_t) slab, VMM_FLAG_NONE));
117 d = d ? d : global.domains[0];
118
119 size_t o = slab->parent_cache->order;
120
121 return &d->slab_domain->caches[SLAB_TYPE_NONPAGEABLE]->caches[o];
122}
123
124void slab_domain_move_slabs(void) {
125 for (size_t i = 0; i < slab_global.num_sizes; i++) {
126 struct slab_cache *c = &slab_global.caches.caches[i];
127 for (size_t j = 0; j < SLAB_STANDARD_STATE_COUNT; j++) {
128 struct slab *slab, *tmp;
129 list_for_each_entry_safe(slab, tmp, &c->slabs[j], list) {
130 struct slab_cache *dest = slab_domain_cache_for_slab(slab);
131 enum irql irql = spin_lock(lock: &c->lock);
132 slab_list_del(slab);
133 spin_unlock(lock: &c->lock, old: irql);
134
135 slab_list_add(cache: dest, slab);
136 }
137 }
138 }
139}
140
141void slab_domain_init(void) {
142 for (size_t i = 0; i < global.domain_count; i++) {
143 struct domain *domain = global.domains[i];
144 struct slab_domain *sdomain =
145 kmalloc_or_die(sizeof(struct slab_domain), ALLOC_FLAGS_ZERO);
146
147 sdomain->domain = domain;
148 domain->slab_domain = sdomain;
149
150 slab_gc_init(dom: sdomain);
151 slab_free_queue_init(domain: sdomain, q: &sdomain->free_queue,
152 SLAB_FREE_QUEUE_CAPACITY);
153 slab_domain_percpu_init(domain: sdomain);
154 slab_domain_init_caches(dom: sdomain);
155 slab_domain_init_stats(domain: sdomain);
156 }
157
158 for (size_t i = 0; i < global.domain_count; i++)
159 slab_domain_build_locality_lists(sdom: global.domains[i]->slab_domain);
160
161 slab_domain_move_slabs();
162 slab_switch_to_domain_allocations();
163}
164
165void slab_domain_init_late() {
166 for (size_t i = 0; i < global.domain_count; i++) {
167 struct slab_domain *sd = global.domains[i]->slab_domain;
168 slab_domain_init_workqueue(domain: sd);
169 slab_domain_init_daemon(domain: sd);
170 }
171}
172