1#include <acpi/lapic.h>
2#include <kassert.h>
3#include <sch/sched.h>
4#include <smp/core.h>
5#include <stdatomic.h>
6#include <stdint.h>
7#include <thread/dpc.h>
8
9static struct dpc *dpc_steal_queue(struct dpc_queue *dq) {
10 struct dpc *list =
11 atomic_exchange_explicit(&dq->head, NULL, memory_order_acquire);
12
13 if (!list)
14 return NULL;
15
16 struct dpc *rev = NULL;
17
18 while (list) {
19 struct dpc *next =
20 atomic_load_explicit(&list->next, memory_order_relaxed);
21 atomic_store_explicit(&list->next, rev, memory_order_relaxed);
22 rev = list;
23 list = next;
24 }
25
26 return rev;
27}
28
29static void dpc_execute_all_in_queue(struct dpc_queue *dq) {
30 while (true) {
31 struct dpc *it = dpc_steal_queue(dq);
32 if (!it)
33 break;
34
35 while (it) {
36 atomic_store_explicit(&it->enqueued, false, memory_order_release);
37 it->func(it, it->ctx);
38 it = atomic_load_explicit(&it->next, memory_order_relaxed);
39 }
40 }
41}
42
43void dpc_run_local(void) {
44 struct core *me = smp_core();
45 if (me->in_resched)
46 return;
47
48 if (atomic_exchange(&me->executing_dpcs, true))
49 return;
50
51 size_t cpu = me->id;
52 enum dpc_event recent = me->dpc_event;
53 struct dpc_cpu *dc = &global.dpc_data[cpu];
54
55 dpc_execute_all_in_queue(dq: &dc->queues[recent]);
56
57 if (recent != DPC_NONE)
58 dpc_execute_all_in_queue(dq: &dc->queues[DPC_NONE]);
59
60 /* all clear */
61 me->dpc_event = DPC_NONE;
62 atomic_store(&me->executing_dpcs, false);
63}
64
65static void dpc_queue_enqueue(struct dpc_queue *dq, struct dpc *d) {
66 while (true) {
67 struct dpc *old_head =
68 atomic_load_explicit(&dq->head, memory_order_acquire);
69 atomic_store_explicit(&d->next, old_head, memory_order_relaxed);
70 if (atomic_compare_exchange_weak_explicit(&dq->head, &old_head, d,
71 memory_order_release,
72 memory_order_relaxed)) {
73 break;
74 }
75 cpu_relax();
76 }
77}
78
79bool dpc_enqueue_on_cpu(size_t cpu, struct dpc *d, enum dpc_event e) {
80 kassert(d);
81
82 if (atomic_exchange_explicit(&d->enqueued, true, memory_order_acq_rel))
83 return false;
84
85 struct dpc_cpu *dc = &global.dpc_data[cpu];
86
87 /* Clear next pointer then push via CAS loop */
88 atomic_store_explicit(&d->next, NULL, memory_order_relaxed);
89
90 struct dpc_queue *dq = &dc->queues[e];
91 dpc_queue_enqueue(dq, d);
92
93 scheduler_force_resched(sched: global.schedulers[cpu]);
94
95 return true;
96}
97
98/* Convenience: enqueue on current cpu */
99bool dpc_enqueue_local(struct dpc *d, enum dpc_event e) {
100 bool ret = dpc_enqueue_on_cpu(cpu: smp_core_id(), d, e);
101 return ret;
102}
103
104void dpc_init_percpu(void) {
105 global.dpc_data =
106 kmalloc(sizeof(struct dpc_cpu) * global.core_count, ALLOC_FLAGS_ZERO);
107 size_t i;
108 for_each_cpu_id(i) {
109 for (size_t j = 0; j < DPC_EVENT_MAX; j++) {
110 atomic_store_explicit(&global.dpc_data[i].queues[j].head, NULL,
111 memory_order_relaxed);
112 }
113 }
114}
115
116struct dpc *dpc_init(struct dpc *d, dpc_func_t fn, void *ctx) {
117 d->func = fn;
118 d->ctx = ctx;
119 atomic_store_explicit(&d->next, NULL, memory_order_relaxed);
120 atomic_store_explicit(&d->enqueued, false, memory_order_relaxed);
121 return d;
122}
123
124/* DPC creation helpers */
125struct dpc *dpc_create(dpc_func_t fn, void *ctx) {
126 struct dpc *d = kmalloc(sizeof(*d));
127 if (!d)
128 return NULL;
129
130 return dpc_init(d, fn, ctx);
131}
132