1#include <acpi/lapic.h>
2#include <kassert.h>
3#include <mem/alloc.h>
4#include <sch/sched.h>
5#include <smp/core.h>
6#include <stdatomic.h>
7#include <stdint.h>
8#include <thread/dpc.h>
9
10static struct dpc *dpc_steal_queue(struct dpc_queue *dq) {
11 struct dpc *list =
12 atomic_exchange_explicit(&dq->head, NULL, memory_order_acquire);
13
14 if (!list)
15 return NULL;
16
17 struct dpc *rev = NULL;
18
19 while (list) {
20 struct dpc *next =
21 atomic_load_explicit(&list->next, memory_order_relaxed);
22 atomic_store_explicit(&list->next, rev, memory_order_relaxed);
23 rev = list;
24 list = next;
25 }
26
27 return rev;
28}
29
30static void dpc_execute_all_in_queue(struct dpc_queue *dq) {
31 while (true) {
32 struct dpc *it = dpc_steal_queue(dq);
33 if (!it)
34 break;
35
36 while (it) {
37 struct dpc *next =
38 atomic_load_explicit(&it->next, memory_order_relaxed);
39 atomic_store_explicit(&it->enqueued, false, memory_order_release);
40 it->func(it->ctx);
41 it = next;
42 }
43 }
44}
45
46void dpc_drain_local(void) {
47 struct core *me = smp_core(cond: TOPC_IRQL);
48 if (me->in_resched)
49 return;
50
51 /* Recursion guard */
52 if (atomic_exchange(&me->executing_dpcs, true))
53 return;
54
55 kassert(irql_get() == IRQL_DISPATCH_LEVEL);
56
57 size_t cpu = me->id;
58 struct dpc_cpu *dc = &global.dpc_data[cpu];
59
60 do {
61 dpc_execute_all_in_queue(dq: &dc->queue);
62 } while (atomic_load_explicit(&dc->queue.head, memory_order_relaxed) !=
63 NULL);
64
65 atomic_store(&me->executing_dpcs, false);
66}
67
68void dpc_run_local(void) {
69 enum irql irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
70 dpc_drain_local();
71
72 /* The raise is only used to satisfy the IRQL requirement here,
73 * and the matching lower shouldn't reschedule and recurse,
74 * as scheduler_yield() will lower its IRQL too */
75 irql_lower_no_resched(old_level: irql);
76}
77
78void dpc_run_dpcs_from_irq(void) {
79 dpc_run_local();
80}
81
82static void dpc_queue_enqueue(struct dpc_queue *dq, struct dpc *d) {
83 while (true) {
84 struct dpc *old_head =
85 atomic_load_explicit(&dq->head, memory_order_acquire);
86 atomic_store_explicit(&d->next, old_head, memory_order_relaxed);
87 if (atomic_compare_exchange_weak_explicit(&dq->head, &old_head, d,
88 memory_order_release,
89 memory_order_relaxed)) {
90 break;
91 }
92 cpu_relax();
93 }
94}
95
96bool dpc_enqueue_on_cpu(size_t cpu, struct dpc *d) {
97 kassert(d);
98
99 if (atomic_exchange_explicit(&d->enqueued, true, memory_order_acq_rel))
100 return false;
101
102 struct dpc_cpu *dc = &global.dpc_data[cpu];
103
104 /* Clear next pointer then push via CAS loop */
105 atomic_store_explicit(&d->next, NULL, memory_order_relaxed);
106
107 struct dpc_queue *dq = &dc->queue;
108 dpc_queue_enqueue(dq, d);
109
110 scheduler_force_run_dpcs(cpu);
111
112 return true;
113}
114
115/* Convenience: enqueue on current cpu */
116bool dpc_enqueue_local(struct dpc *d) {
117 /* Snapshot it */
118 bool ret = dpc_enqueue_on_cpu(cpu: smp_id_raw(), d);
119 return ret;
120}
121
122void dpc_init_percpu(void) {
123 global.dpc_data =
124 kmalloc(sizeof(struct dpc_cpu) * global.core_count, ALLOC_FLAGS_ZERO);
125 size_t i;
126 for_each_cpu_id(i) {
127 atomic_store_explicit(&global.dpc_data[i].queue.head, NULL,
128 memory_order_relaxed);
129 }
130}
131
132struct dpc *dpc_init(struct dpc *d, dpc_func_t fn, void *ctx) {
133 d->func = fn;
134 d->ctx = ctx;
135 atomic_store_explicit(&d->next, NULL, memory_order_relaxed);
136 atomic_store_explicit(&d->enqueued, false, memory_order_relaxed);
137 return d;
138}
139
140/* DPC creation helpers */
141struct dpc *dpc_create(dpc_func_t fn, void *ctx) {
142 struct dpc *d = kmalloc(sizeof(*d));
143 if (!d)
144 return NULL;
145
146 return dpc_init(d, fn, ctx);
147}
148