1#include <mem/domain.h>
2#include <mem/nofail.h>
3#include <mem/slab.h>
4#include <smp/domain.h>
5#include <thread/daemon.h>
6
7#include "internal.h"
8#include "mem/domain/internal.h"
9
10static void slab_domain_build_locality_lists(struct slab_domain *sdom) {
11 struct domain_buddy *buddy = sdom->domain->domain_buddy;
12 struct domain_zonelist *zl = &buddy->zonelist;
13
14 for (int i = 0; i < SLAB_TYPE_COUNT; i++) {
15 sdom->zonelists[i].count = zl->count;
16 sdom->zonelists[i].entries =
17 nofail_vnews(sdom->zonelists[i].entries, zl->count, ALLOC_ZERO);
18 for (size_t j = 0; j < zl->count; j++) {
19 struct domain_zonelist_entry *zent = &zl->entries[j];
20 struct domain_buddy *bd = zent->domain;
21 size_t idx = bd - global.domain_buddies;
22 struct slab_domain *remote_sdom = global.domains[idx]->slab_domain;
23 sdom->zonelists[i].entries[j] = (struct slab_cache_ref){
24 .caches = remote_sdom->caches[i],
25 .type = i,
26 .locality = zent->distance,
27 .domain = remote_sdom,
28 };
29 }
30 }
31
32 sdom->zonelist_entry_count = zl->count;
33}
34
35static void slab_init_caches(struct slab_caches *caches, enum slab_type type) {
36 for (size_t i = 0; i < slab_global.num_sizes; i++) {
37 struct slab_cache *cache = &caches->caches[i];
38 cache->type = type;
39 slab_cache_init(order: i, cache, ssc: &slab_global.class_sizes[i]);
40 }
41}
42
43static void slab_domain_link_caches(struct slab_domain *domain,
44 struct slab_caches *caches) {
45 for (size_t i = 0; i < slab_global.num_sizes; i++) {
46 caches->caches[i].parent_domain = domain;
47 caches->caches[i].parent = caches;
48 }
49}
50
51static void slab_domain_init_cache(struct slab_domain *dom,
52 enum slab_type type) {
53 struct slab_caches *caches = nofail_vnew(caches, ALLOC_ZERO);
54
55 dom->caches[type] = caches;
56 dom->caches[type]->caches = nofail(slab_caches_alloc());
57 slab_init_caches(caches, type);
58 slab_domain_link_caches(domain: dom, caches);
59}
60
61static void slab_domain_init_caches(struct slab_domain *dom) {
62 for (int i = 0; i < SLAB_TYPE_COUNT; i++) {
63 slab_domain_init_cache(dom, type: i);
64 }
65}
66
67static size_t slab_bucket_reset(struct stat_bucket *bucket) {
68 struct slab_domain_bucket *db = bucket->private;
69 struct slab_domain *sd = bucket->parent->private;
70 struct slab_domain_bucket *agg = &sd->aggregate;
71 size_t stats_len =
72 sizeof(struct slab_domain_bucket) / sizeof(atomic_size_t);
73
74 atomic_size_t *parent_stats = (atomic_size_t *) agg;
75 atomic_size_t *bucket_stats = (atomic_size_t *) db;
76
77 /* Subtract this bucket's values from the parent */
78 for (size_t i = 0; i < stats_len; i++) {
79 size_t val = atomic_load_rlx(&bucket_stats[i]);
80 atomic_size_t *parent = &parent_stats[i];
81
82 /* This would underflow anyways... */
83 if (atomic_load_rlx(parent) < val) {
84 atomic_store(parent, 0);
85 } else {
86 atomic_fetch_sub(parent, val);
87 }
88 }
89
90 memset(db, 0, sizeof(struct slab_domain_bucket));
91 return 0;
92}
93
94static void slab_domain_init_stats(struct slab_domain *domain) {
95 domain->stats = stat_series_create(SLAB_STAT_SERIES_CAPACITY,
96 SLAB_STAT_SERIES_BUCKET_US,
97 bucket_reset: slab_bucket_reset, private: domain);
98
99 domain->stats->private = domain;
100 domain->buckets =
101 vnews(domain->buckets, SLAB_STAT_SERIES_CAPACITY, ALLOC_ZERO);
102
103 if (!domain->stats || !domain->stats->buckets || !domain->buckets)
104 panic(OUT_OF_MEMORY, "domain stat series");
105
106 for (size_t i = 0; i < domain->stats->nbuckets; i++) {
107 domain->stats->buckets[i].private = &domain->buckets[i];
108 }
109}
110
111static struct slab_cache *slab_domain_cache_for_slab(struct slab *slab) {
112 struct domain *d = domain_for_addr(vmm_get_phys((vaddr_t) slab));
113 d = d ? d : global.domains[0];
114
115 size_t o = slab->parent_cache->order;
116
117 return &d->slab_domain->caches[SLAB_TYPE_NONPAGEABLE]->caches[o];
118}
119
120static void slab_domain_move_slabs(void) {
121 for (size_t i = 0; i < slab_global.num_sizes; i++) {
122 struct slab_cache *c = &slab_global.caches.caches[i];
123 for (size_t j = 0; j < SLAB_STANDARD_STATE_COUNT; j++) {
124 struct slab *slab, *tmp;
125 list_for_each_entry_safe(slab, tmp, &c->slabs[j], list) {
126 struct slab_cache *dest = slab_domain_cache_for_slab(slab);
127 enum irql irql = spin_lock(&c->lock);
128 slab_list_del(slab);
129 spin_unlock(&c->lock, irql);
130
131 slab_list_add(cache: dest, slab);
132 }
133 }
134 }
135}
136
137void slab_domain_init(void) {
138 for (size_t i = 0; i < global.domain_count; i++) {
139 struct domain *domain = global.domains[i];
140 struct slab_domain *sdomain = nofail_vnew(sdomain, ALLOC_ZERO);
141
142 sdomain->domain = domain;
143 domain->slab_domain = sdomain;
144
145 slab_gc_init(dom: sdomain);
146 slab_free_queue_init(domain: sdomain, q: &sdomain->free_queue,
147 SLAB_FREE_QUEUE_CAPACITY);
148 slab_domain_percpu_init(domain: sdomain);
149 slab_domain_init_caches(dom: sdomain);
150 slab_domain_init_stats(domain: sdomain);
151 }
152
153 for (size_t i = 0; i < global.domain_count; i++)
154 slab_domain_build_locality_lists(sdom: global.domains[i]->slab_domain);
155
156 slab_domain_move_slabs();
157 slab_switch_to_domain_allocations();
158 slab_global.domains_enabled = true;
159}
160
161void slab_domain_init_late(void) {
162 for (size_t i = 0; i < global.domain_count; i++) {
163 struct slab_domain *sd = global.domains[i]->slab_domain;
164 slab_domain_init_workqueue(domain: sd);
165 slab_domain_init_daemon(domain: sd);
166 }
167}
168