1#include <console/printf.h>
2#include <global.h>
3#include <log.h>
4#include <mem/alloc.h>
5#include <mem/nofail.h>
6#include <mem/numa.h>
7#include <mem/page.h>
8#include <sch/sched.h>
9#include <smp/domain.h>
10#include <stdbool.h>
11#include <sync/spinlock.h>
12
13LOG_SITE_DEFINE_PRINT(domain);
14LOG_HANDLE_DEFINE_PRINT(domain);
15
16#define domain_log(lvl, fmt, ...) \
17 log(LOG_SITE(domain), LOG_HANDLE(domain), lvl, fmt, ##__VA_ARGS__)
18
19#define domain_err(fmt, ...) domain_log(LOG_ERROR, fmt, ##__VA_ARGS__)
20#define domain_warn(fmt, ...) domain_log(LOG_WARN, fmt, ##__VA_ARGS__)
21#define domain_info(fmt, ...) domain_log(LOG_INFO, fmt, ##__VA_ARGS__)
22#define domain_debug(fmt, ...) domain_log(LOG_DEBUG, fmt, ##__VA_ARGS__)
23#define domain_trace(fmt, ...) domain_log(LOG_TRACE, fmt, ##__VA_ARGS__)
24
25static void init_global_domain(uint64_t domain_count) {
26 global.domain_count = domain_count;
27 global.domains = nofail_vnews(global.domains, domain_count, ALLOC_ZERO);
28
29 for (size_t i = 0; i < domain_count; i++) {
30
31 /* We align this up to the page so that they can all be
32 * migrated later on to pages on each domain... */
33 global.domains[i] = nofail(valloc_pages(
34 PAGES_NEEDED_FOR(sizeof(struct domain)), ALLOC_FLAGS_ZERO));
35
36 global.domains[i]->id = i;
37 }
38}
39
40/* Map 1:1 with NUMA nodes */
41static void construct_domains_from_numa_nodes(void) {
42 init_global_domain(domain_count: global.numa_node_count);
43 for (size_t i = 0; i < global.numa_node_count; i++) {
44 struct numa_node *nn = &global.numa_nodes[i];
45 struct domain *cd = global.domains[i];
46 cd->num_cores = cpu_mask_popcount(m: &nn->cpus);
47 cpu_mask_init(m: &cd->cpu_mask, nbits: global.core_count);
48 cpu_mask_copy(dst: &cd->cpu_mask, src: &nn->cpus);
49 cd->associated_node = nn;
50 cd->cores = nofail_vnews(cd->cores, cd->num_cores, ALLOC_ZERO);
51 }
52}
53
54static void construct_domains_from_cores(void) {
55 size_t domain_count = global.core_count / CORES_PER_DOMAIN;
56 size_t remainder = global.core_count % CORES_PER_DOMAIN;
57
58 if (remainder > 0)
59 domain_count++; /* one extra for leftover cores */
60
61 init_global_domain(domain_count);
62
63 for (size_t i = 0; i < domain_count; i++) {
64 struct domain *cd = global.domains[i];
65
66 cd->associated_node = NULL;
67
68 /* Decide how many cores this domain should get */
69 size_t cores_this_domain = CORES_PER_DOMAIN;
70 if (i == domain_count - 1 && remainder > 0)
71 cores_this_domain = remainder; /* last one gets leftovers */
72
73 cd->num_cores = cores_this_domain;
74 cpu_mask_init(m: &cd->cpu_mask, nbits: global.core_count);
75
76 for (size_t j = 0; j < cores_this_domain; j++) {
77 size_t core_index = i * CORES_PER_DOMAIN + j;
78 if (core_index >= global.core_count)
79 break;
80
81 cpu_mask_set(m: &cd->cpu_mask, cpu: core_index);
82 }
83 cd->cores = nofail_vnews(cd->cores, cores_this_domain, ALLOC_ZERO);
84 }
85}
86
87void domain_dump(void) {
88 domain_info("Domains (%zu total)", global.domain_count);
89
90 for (size_t i = 0; i < global.domain_count; i++) {
91 struct domain *cd = global.domains[i];
92 if (cd->associated_node) {
93 domain_info(" Domain %zu: Cores = %zu, NUMA node = %zu", i,
94 cd->num_cores, cd->associated_node->topo->id);
95 } else {
96 domain_info(" Domain %zu: Cores = %zu, NUMA node = <none>", i,
97 cd->num_cores);
98 }
99
100 for (size_t j = 0; j < cd->num_cores; j++) {
101 if (cd->cores && cd->cores[j]) {
102 struct core *c = cd->cores[j];
103 domain_info(" Core %zu", c->id);
104 } else {
105 domain_info(" Core <NULL>");
106 }
107 }
108 }
109}
110
111/* If NUMA is present, domains map 1:1 with
112 * NUMA nodes. If not, we just group cores into
113 * groups of CORES_PER_DOMAIN
114 * and construct domains from them. */
115void domain_init(void) {
116 if (global.numa_node_count > 1) {
117 construct_domains_from_numa_nodes();
118 } else {
119 construct_domains_from_cores();
120 }
121
122 /* NOTE: This is THE exception that we make */
123 global.cores[0]->domain = global.domains[0]; // NOLINT(*ArrayBound)
124}
125
126static void construct_domains_after_smp(void) {
127 if (global.numa_node_count > 1) {
128 for (size_t i = 0; i < global.domain_count; i++) {
129 struct domain *cd = global.domains[i];
130 struct numa_node *nn = &global.numa_nodes[i];
131 struct cpu_mask nm = nn->cpus;
132 size_t j;
133 size_t k = 0;
134 cpu_mask_for_each(j, nm) {
135 cd->cores[k++] = global.cores[j];
136 }
137 }
138 } else {
139 size_t core_index = 0;
140 for (size_t i = 0; i < global.domain_count; i++) {
141 struct domain *cd = global.domains[i];
142
143 for (size_t j = 0; j < cd->num_cores; j++) {
144 global.cores[core_index]->domain = cd;
145 cd->cores[j] = global.cores[core_index++];
146 }
147 }
148 }
149}
150
151void domain_init_after_smp(void) {
152 construct_domains_after_smp();
153 for (size_t i = 0; i < global.domain_count; i++) {
154 struct domain *domain = global.domains[i];
155
156 for (size_t j = 0; j < domain->num_cores; j++) {
157 domain->cores[j]->domain_cpu_id = j;
158 domain->cores[j]->domain = domain;
159 }
160 }
161}
162
163void domain_set_cpu_mask(struct cpu_mask *mask, struct domain *domain) {
164 for (size_t i = 0; i < domain->num_cores; i++)
165 cpu_mask_set(m: mask, cpu: domain->cores[i]->id);
166}
167
168/* Create a CPU mask that has all the bits for the domain set */
169struct cpu_mask *domain_create_cpu_mask(struct domain *domain) {
170 struct cpu_mask *ret = cpu_mask_create();
171 if (!ret)
172 goto err;
173
174 cpu_mask_init(m: ret, nbits: global.core_count);
175
176 domain_set_cpu_mask(mask: ret, domain);
177
178 return ret;
179err:
180 if (ret)
181 vfree(ret);
182
183 return NULL;
184}
185
186static void domains_move(void *a, void *b) {
187 cc_ignore(a, b);
188 for (size_t i = 0; i < global.domain_count; i++) {
189 struct domain *domain = global.domains[i];
190 movealloc(domain->id, domain);
191 }
192}
193
194bool domain_idle(struct domain *domain) {
195 for (size_t i = 0; i < domain->num_cores; i++)
196 if (!scheduler_core_idle(c: domain->cores[i]))
197 return false;
198
199 return true;
200}
201
202numa_node_t numa_node_for_cpu(cpu_id_t cpu) {
203 for (numa_node_t i = 0; i < global.numa_node_count; i++) {
204 struct numa_node *nn = &global.numa_nodes[i];
205 if (cpu_mask_test(m: &nn->cpus, cpu))
206 return i;
207 }
208
209 return 0;
210}
211
212domain_id_t domain_for_cpu(cpu_id_t cpu) {
213 struct domain *d;
214 domain_for_each_domain(d) {
215 if (cpu_mask_test(m: &d->cpu_mask, cpu))
216 return d->id;
217 }
218
219 unreachable();
220}
221
222MOVEALLOC_REGISTER_CALL(domain_move, domains_move, /* a = */ NULL,
223 /* b = */ NULL);
224
225void domain_caller_verify(enum topology_caller caller) {
226 struct topology_contract tct = {
227 .caller = caller,
228 .scope = TOPOLOGY_LEVEL_EX_DOMAIN,
229 };
230
231 bool valid = topology_contract_verify(c: tct);
232 enforce(valid, "Caller 0x%x was not satisfied", caller);
233}
234