1#include <console/printf.h>
2#include <global.h>
3#include <log.h>
4#include <mem/alloc.h>
5#include <mem/alloc_or_die.h>
6#include <mem/numa.h>
7#include <sch/sched.h>
8#include <smp/domain.h>
9#include <stdbool.h>
10#include <sync/spinlock.h>
11
12LOG_SITE_DECLARE_DEFAULT(domain);
13LOG_HANDLE_DECLARE_DEFAULT(domain);
14
15#define domain_log(lvl, fmt, ...) \
16 log(LOG_SITE(domain), LOG_HANDLE(domain), lvl, fmt, ##__VA_ARGS__)
17
18#define domain_err(fmt, ...) domain_log(LOG_ERROR, fmt, ##__VA_ARGS__)
19#define domain_warn(fmt, ...) domain_log(LOG_WARN, fmt, ##__VA_ARGS__)
20#define domain_info(fmt, ...) domain_log(LOG_INFO, fmt, ##__VA_ARGS__)
21#define domain_debug(fmt, ...) domain_log(LOG_DEBUG, fmt, ##__VA_ARGS__)
22#define domain_trace(fmt, ...) domain_log(LOG_TRACE, fmt, ##__VA_ARGS__)
23
24static void init_global_domain(uint64_t domain_count) {
25 global.domain_count = domain_count;
26 global.domains = alloc_or_die(
27 kmalloc(sizeof(struct domain *) * domain_count, ALLOC_FLAGS_ZERO));
28
29 for (size_t i = 0; i < domain_count; i++) {
30
31 /* We align this up to the page so that they can all be
32 * migrated later on to pages on each domain... */
33 global.domains[i] = alloc_or_die(
34 kmalloc(PAGE_ALIGN_UP(sizeof(struct domain)), ALLOC_FLAGS_ZERO));
35
36 global.domains[i]->id = i;
37 }
38}
39
40/* Map 1:1 with NUMA nodes */
41static void construct_domains_from_numa_nodes(void) {
42 init_global_domain(domain_count: global.numa_node_count);
43 for (size_t i = 0; i < global.numa_node_count; i++) {
44 struct numa_node *nn = &global.numa_nodes[i];
45 struct domain *cd = global.domains[i];
46 cd->num_cores = cpu_mask_popcount(m: &nn->cpus);
47 cd->associated_node = nn;
48 cd->cores =
49 kmalloc(sizeof(struct core *) * cd->num_cores, ALLOC_FLAGS_ZERO);
50 }
51}
52
53static void construct_domains_after_smp() {
54 if (global.numa_node_count > 1) {
55 for (size_t i = 0; i < global.domain_count; i++) {
56 struct domain *cd = global.domains[i];
57 struct numa_node *nn = &global.numa_nodes[i];
58 struct cpu_mask nm = nn->cpus;
59 size_t j;
60 size_t k = 0;
61 cpu_mask_for_each(j, nm) {
62 cd->cores[k++] = global.cores[j];
63 }
64 }
65 } else {
66 size_t core_index = 0;
67 for (size_t i = 0; i < global.domain_count; i++) {
68 struct domain *cd = global.domains[i];
69
70 for (size_t j = 0; j < cd->num_cores; j++) {
71 global.cores[core_index]->domain = cd;
72 cd->cores[j] = global.cores[core_index++];
73 }
74 }
75 }
76}
77
78static void construct_domains_from_cores(void) {
79 size_t n_domains = global.core_count / CORES_PER_DOMAIN;
80 size_t remainder = global.core_count % CORES_PER_DOMAIN;
81
82 if (remainder > 0)
83 n_domains++; /* one extra for leftover cores */
84
85 init_global_domain(domain_count: n_domains);
86
87 for (size_t i = 0; i < n_domains; i++) {
88 struct domain *cd = global.domains[i];
89
90 cd->associated_node = NULL;
91
92 /* Decide how many cores this domain should get */
93 size_t cores_this_domain = CORES_PER_DOMAIN;
94 if (i == n_domains - 1 && remainder > 0)
95 cores_this_domain = remainder; /* last one gets leftovers */
96
97 cd->num_cores = cores_this_domain;
98 cd->cores = kmalloc_or_die(sizeof(struct core *) * cores_this_domain,
99 ALLOC_FLAGS_ZERO);
100 }
101}
102
103void domain_dump(void) {
104 domain_info("Domains (%zu total)", global.domain_count);
105
106 for (size_t i = 0; i < global.domain_count; i++) {
107 struct domain *cd = global.domains[i];
108 if (cd->associated_node) {
109 domain_info(" Domain %zu: Cores = %zu, NUMA node = %zu", i,
110 cd->num_cores, cd->associated_node->topo->id);
111 } else {
112 domain_info(" Domain %zu: Cores = %zu, NUMA node = <none>", i,
113 cd->num_cores);
114 }
115
116 for (size_t j = 0; j < cd->num_cores; j++) {
117 if (cd->cores && cd->cores[j]) {
118 struct core *c = cd->cores[j];
119 domain_info(" Core %zu", c->id);
120 } else {
121 domain_info(" Core <NULL>");
122 }
123 }
124 }
125}
126
127/* If NUMA is present, domains map 1:1 with
128 * NUMA nodes. If not, we just group cores into
129 * groups of CORES_PER_DOMAIN
130 * and construct domains from them. */
131void domain_init(void) {
132 if (global.numa_node_count > 1) {
133 construct_domains_from_numa_nodes();
134 } else {
135 construct_domains_from_cores();
136 }
137
138 /* NOTE: This is THE exception that we make */
139 global.cores[0]->domain = global.domains[0];
140}
141
142void domain_init_after_smp() {
143 construct_domains_after_smp();
144 for (size_t i = 0; i < global.domain_count; i++) {
145 struct domain *domain = global.domains[i];
146 alloc_or_die(cpu_mask_init(&domain->cpu_mask, global.core_count));
147
148 for (size_t j = 0; j < domain->num_cores; j++) {
149 domain->cores[j]->domain_cpu_id = j;
150 domain->cores[j]->domain = domain;
151 cpu_mask_set(m: &domain->cpu_mask, cpu: domain->cores[j]->id);
152 }
153 }
154}
155
156void domain_set_cpu_mask(struct cpu_mask *mask, struct domain *domain) {
157 for (size_t i = 0; i < domain->num_cores; i++)
158 cpu_mask_set(m: mask, cpu: domain->cores[i]->id);
159}
160
161/* Create a CPU mask that has all the bits for the domain set */
162struct cpu_mask *domain_create_cpu_mask(struct domain *domain) {
163 struct cpu_mask *ret = cpu_mask_create();
164 if (!ret)
165 goto err;
166
167 if (!cpu_mask_init(m: ret, nbits: global.core_count))
168 goto err;
169
170 domain_set_cpu_mask(mask: ret, domain);
171
172 return ret;
173err:
174 if (ret)
175 kfree(ret);
176
177 return NULL;
178}
179
180static void domains_move(void *a, void *b) {
181 (void) a, (void) b;
182 for (size_t i = 0; i < global.domain_count; i++) {
183 struct domain *domain = global.domains[i];
184 movealloc(domain->id, domain);
185 }
186}
187
188bool domain_idle(struct domain *domain) {
189 for (size_t i = 0; i < domain->num_cores; i++)
190 if (!scheduler_core_idle(c: domain->cores[i]))
191 return false;
192
193 return true;
194}
195
196size_t domain_for_core(size_t cpu) {
197 for (size_t i = 0; i < global.numa_node_count; i++) {
198 struct numa_node *nn = &global.numa_nodes[i];
199 if (cpu_mask_test(m: &nn->cpus, cpu))
200 return i;
201 }
202
203 panic("unreachable!");
204}
205
206MOVEALLOC_REGISTER_CALL(domain_move, domains_move, /* a = */ NULL,
207 /* b = */ NULL);
208