1#include <console/printf.h>
2#include <global.h>
3#include <log.h>
4#include <mem/alloc.h>
5#include <mem/alloc_or_die.h>
6#include <mem/numa.h>
7#include <mem/page.h>
8#include <sch/sched.h>
9#include <smp/domain.h>
10#include <stdbool.h>
11#include <sync/spinlock.h>
12
13LOG_SITE_DECLARE_PRINT(domain);
14LOG_HANDLE_DECLARE_PRINT(domain);
15
16#define domain_log(lvl, fmt, ...) \
17 log(LOG_SITE(domain), LOG_HANDLE(domain), lvl, fmt, ##__VA_ARGS__)
18
19#define domain_err(fmt, ...) domain_log(LOG_ERROR, fmt, ##__VA_ARGS__)
20#define domain_warn(fmt, ...) domain_log(LOG_WARN, fmt, ##__VA_ARGS__)
21#define domain_info(fmt, ...) domain_log(LOG_INFO, fmt, ##__VA_ARGS__)
22#define domain_debug(fmt, ...) domain_log(LOG_DEBUG, fmt, ##__VA_ARGS__)
23#define domain_trace(fmt, ...) domain_log(LOG_TRACE, fmt, ##__VA_ARGS__)
24
25static void init_global_domain(uint64_t domain_count) {
26 global.domain_count = domain_count;
27 global.domains = alloc_or_die(
28 kmalloc(sizeof(struct domain *) * domain_count, ALLOC_FLAGS_ZERO));
29
30 for (size_t i = 0; i < domain_count; i++) {
31
32 /* We align this up to the page so that they can all be
33 * migrated later on to pages on each domain... */
34 global.domains[i] = alloc_or_die(kmalloc_pages(
35 PAGES_NEEDED_FOR(sizeof(struct domain)), ALLOC_FLAGS_ZERO));
36
37 global.domains[i]->id = i;
38 }
39}
40
41/* Map 1:1 with NUMA nodes */
42static void construct_domains_from_numa_nodes(void) {
43 init_global_domain(domain_count: global.numa_node_count);
44 for (size_t i = 0; i < global.numa_node_count; i++) {
45 struct numa_node *nn = &global.numa_nodes[i];
46 struct domain *cd = global.domains[i];
47 cd->num_cores = cpu_mask_popcount(m: &nn->cpus);
48 alloc_or_die(cpu_mask_init(&cd->cpu_mask, global.core_count));
49 cpu_mask_copy(dst: &cd->cpu_mask, src: &nn->cpus);
50 cd->associated_node = nn;
51 cd->cores = kmalloc_or_die(sizeof(struct core *) * cd->num_cores,
52 ALLOC_FLAGS_ZERO);
53 }
54}
55
56static void construct_domains_from_cores(void) {
57 size_t n_domains = global.core_count / CORES_PER_DOMAIN;
58 size_t remainder = global.core_count % CORES_PER_DOMAIN;
59
60 if (remainder > 0)
61 n_domains++; /* one extra for leftover cores */
62
63 init_global_domain(domain_count: n_domains);
64
65 for (size_t i = 0; i < n_domains; i++) {
66 struct domain *cd = global.domains[i];
67
68 cd->associated_node = NULL;
69
70 /* Decide how many cores this domain should get */
71 size_t cores_this_domain = CORES_PER_DOMAIN;
72 if (i == n_domains - 1 && remainder > 0)
73 cores_this_domain = remainder; /* last one gets leftovers */
74
75 cd->num_cores = cores_this_domain;
76 alloc_or_die(cpu_mask_init(&cd->cpu_mask, global.core_count));
77
78 for (size_t j = 0; j < cores_this_domain; j++) {
79 size_t core_index = i * CORES_PER_DOMAIN + j;
80 if (core_index >= global.core_count)
81 break;
82
83 cpu_mask_set(m: &cd->cpu_mask, cpu: core_index);
84 }
85 cd->cores = kmalloc_or_die(sizeof(struct core *) * cores_this_domain,
86 ALLOC_FLAGS_ZERO);
87 }
88}
89
90void domain_dump(void) {
91 domain_info("Domains (%zu total)", global.domain_count);
92
93 for (size_t i = 0; i < global.domain_count; i++) {
94 struct domain *cd = global.domains[i];
95 if (cd->associated_node) {
96 domain_info(" Domain %zu: Cores = %zu, NUMA node = %zu", i,
97 cd->num_cores, cd->associated_node->topo->id);
98 } else {
99 domain_info(" Domain %zu: Cores = %zu, NUMA node = <none>", i,
100 cd->num_cores);
101 }
102
103 for (size_t j = 0; j < cd->num_cores; j++) {
104 if (cd->cores && cd->cores[j]) {
105 struct core *c = cd->cores[j];
106 domain_info(" Core %zu", c->id);
107 } else {
108 domain_info(" Core <NULL>");
109 }
110 }
111 }
112}
113
114/* If NUMA is present, domains map 1:1 with
115 * NUMA nodes. If not, we just group cores into
116 * groups of CORES_PER_DOMAIN
117 * and construct domains from them. */
118void domain_init(void) {
119 if (global.numa_node_count > 1) {
120 construct_domains_from_numa_nodes();
121 } else {
122 construct_domains_from_cores();
123 }
124
125 /* NOTE: This is THE exception that we make */
126 global.cores[0]->domain = global.domains[0];
127}
128
129static void construct_domains_after_smp() {
130 if (global.numa_node_count > 1) {
131 for (size_t i = 0; i < global.domain_count; i++) {
132 struct domain *cd = global.domains[i];
133 struct numa_node *nn = &global.numa_nodes[i];
134 struct cpu_mask nm = nn->cpus;
135 size_t j;
136 size_t k = 0;
137 cpu_mask_for_each(j, nm) {
138 cd->cores[k++] = global.cores[j];
139 }
140 }
141 } else {
142 size_t core_index = 0;
143 for (size_t i = 0; i < global.domain_count; i++) {
144 struct domain *cd = global.domains[i];
145
146 for (size_t j = 0; j < cd->num_cores; j++) {
147 global.cores[core_index]->domain = cd;
148 cd->cores[j] = global.cores[core_index++];
149 }
150 }
151 }
152}
153
154void domain_init_after_smp() {
155 construct_domains_after_smp();
156 for (size_t i = 0; i < global.domain_count; i++) {
157 struct domain *domain = global.domains[i];
158
159 for (size_t j = 0; j < domain->num_cores; j++) {
160 domain->cores[j]->domain_cpu_id = j;
161 domain->cores[j]->domain = domain;
162 }
163 }
164}
165
166void domain_set_cpu_mask(struct cpu_mask *mask, struct domain *domain) {
167 for (size_t i = 0; i < domain->num_cores; i++)
168 cpu_mask_set(m: mask, cpu: domain->cores[i]->id);
169}
170
171/* Create a CPU mask that has all the bits for the domain set */
172struct cpu_mask *domain_create_cpu_mask(struct domain *domain) {
173 struct cpu_mask *ret = cpu_mask_create();
174 if (!ret)
175 goto err;
176
177 if (!cpu_mask_init(m: ret, nbits: global.core_count))
178 goto err;
179
180 domain_set_cpu_mask(mask: ret, domain);
181
182 return ret;
183err:
184 if (ret)
185 kfree(ret);
186
187 return NULL;
188}
189
190static void domains_move(void *a, void *b) {
191 (void) a, (void) b;
192 for (size_t i = 0; i < global.domain_count; i++) {
193 struct domain *domain = global.domains[i];
194 movealloc(domain->id, domain);
195 }
196}
197
198bool domain_idle(struct domain *domain) {
199 for (size_t i = 0; i < domain->num_cores; i++)
200 if (!scheduler_core_idle(c: domain->cores[i]))
201 return false;
202
203 return true;
204}
205
206numa_node_t numa_node_for_cpu(cpu_id_t cpu) {
207 for (numa_node_t i = 0; i < global.numa_node_count; i++) {
208 struct numa_node *nn = &global.numa_nodes[i];
209 if (cpu_mask_test(m: &nn->cpus, cpu))
210 return i;
211 }
212
213 return 0;
214}
215
216domain_id_t domain_for_cpu(cpu_id_t cpu) {
217 struct domain *d;
218 domain_for_each_domain(d) {
219 if (cpu_mask_test(m: &d->cpu_mask, cpu))
220 return d->id;
221 }
222
223 unreachable();
224}
225
226MOVEALLOC_REGISTER_CALL(domain_move, domains_move, /* a = */ NULL,
227 /* b = */ NULL);
228
229void domain_caller_verify(enum topology_caller caller) {
230 struct topology_contract tct = {
231 .caller = caller,
232 .scope = TOPOLOGY_LEVEL_DOMAIN,
233 };
234
235 bool valid = topology_contract_verify(c: tct);
236 kassert(valid, "Caller 0x%lx was not satisfied", caller);
237}
238