1/* @title: Assembly Routines */
2#pragma once
3#include <compiler/core.h>
4#include <compiler/intrinsic.h>
5#include <stdbool.h>
6#include <stdint.h>
7#include <tsa.h>
8
9//
10//
11//
12//
13//
14// =========| IN - BYTE, WORD, LONG, SB, SW, SL |=========
15//
16//
17//
18//
19//
20
21static inline uint8_t inb(uint16_t port) {
22 uint8_t ret;
23 asm volatile("inb %1, %0" : "=a"(ret) : "Nd"(port));
24 return ret;
25}
26
27static inline uint16_t inw(uint16_t port) {
28 uint16_t ret;
29 asm volatile("inw %1, %0" : "=a"(ret) : "Nd"(port));
30 return ret;
31}
32
33static inline uint32_t inl(uint16_t port) {
34 uint32_t ret;
35 asm volatile("inl %1, %0" : "=a"(ret) : "Nd"(port));
36 return ret;
37}
38
39static inline void insb(uint16_t port, void *addr, uint32_t count) {
40 asm volatile("rep insb" : "+D"(addr), "+c"(count) : "d"(port) : "memory");
41}
42
43static inline void insw(uint16_t port, void *addr, uint32_t count) {
44 asm volatile("rep insw" : "+D"(addr), "+c"(count) : "d"(port) : "memory");
45}
46
47static inline void insl(uint16_t port, void *addr, uint32_t count) {
48 asm volatile("rep insl" : "+D"(addr), "+c"(count) : "d"(port) : "memory");
49}
50
51//
52//
53//
54//
55//
56// ========| OUT - BYTE, WORD, LONG, SB, SW, SL |========
57//
58//
59//
60//
61//
62
63static inline void outb(uint16_t port, uint8_t value) {
64 asm volatile("outb %0, %1" : : "a"(value), "Nd"(port));
65}
66
67static inline void outw(uint16_t port, uint16_t value) {
68 asm volatile("outw %1, %0" ::"dN"(port), "a"(value));
69}
70
71static inline void outl(uint16_t port, uint32_t value) {
72 asm volatile("outl %0, %1" : : "a"(value), "Nd"(port));
73}
74
75static inline void outsw(uint16_t port, const void *addr, uint32_t count) {
76 asm volatile("rep outsw" : "+S"(addr), "+c"(count) : "d"(port) : "memory");
77}
78
79static inline void outsb(uint16_t port, const void *addr, uint32_t count) {
80 asm volatile("rep outsb" : "+S"(addr), "+c"(count) : "d"(port) : "memory");
81}
82
83static inline void outsl(uint16_t port, const void *addr, uint32_t count) {
84 asm volatile("rep outsl" : "+S"(addr), "+c"(count) : "d"(port) : "memory");
85}
86
87//
88//
89//
90//
91//
92// =============================| MISC |==============================
93//
94//
95//
96//
97//
98
99static inline void cr8_write(uint64_t cr8) {
100 asm volatile("mov %0, %%cr8" ::"r"(cr8));
101}
102
103static inline uint64_t rdtsc(void) {
104 uint32_t lo, hi;
105 asm volatile("rdtsc" : "=a"(lo), "=d"(hi));
106 return ((uint64_t) hi << 32) | lo;
107}
108
109static inline uint64_t rdtsc_ordered(void) {
110 uint32_t lo, hi;
111 asm volatile("lfence\n\t"
112 "rdtsc"
113 : "=a"(lo), "=d"(hi));
114 return ((uint64_t) hi << 32) | lo;
115}
116
117static inline void cpuid_count(uint32_t leaf, uint32_t subleaf, uint32_t *eax,
118 uint32_t *ebx, uint32_t *ecx, uint32_t *edx) {
119 asm volatile("cpuid"
120 : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx)
121 : "a"(leaf), "c"(subleaf));
122}
123
124static inline uint64_t cr0_read(void) {
125 uint64_t cr0;
126 asm volatile("mov %%cr0, %0" : "=r"(cr0));
127 return cr0;
128}
129
130static inline void cr0_write(uint64_t cr0) {
131 asm volatile("mov %0, %%cr0" : : "r"(cr0) : "memory");
132}
133
134static inline uint64_t cr4_read(void) {
135 uint64_t cr4;
136 asm volatile("mov %%cr4, %0" : "=r"(cr4));
137 return cr4;
138}
139
140static inline void cr4_write(uint64_t cr4) {
141 asm volatile("mov %0, %%cr4" : : "r"(cr4));
142}
143
144static inline bool irqs_enabled(void) {
145 unsigned long flags;
146 asm volatile("pushf\n\t"
147 "pop %0\n\t"
148 : "=r"(flags)
149 :
150 :);
151 return (flags & (1 << 9)) != 0;
152}
153
154#define MSR_GS_BASE 0xC0000101
155
156static inline void wrmsr(uint32_t msr, uint64_t value) {
157 uint32_t lo = value & 0xFFFFFFFF;
158 uint32_t hi = value >> 32;
159 asm volatile("wrmsr" ::"c"(msr), "a"(lo), "d"(hi) : "memory");
160}
161
162static inline uint64_t rdmsr(uint32_t msr) {
163 uint64_t lo, hi;
164 asm volatile("rdmsr" : "=a"(lo), "=d"(hi) : "c"(msr));
165 return (hi << 32U) | lo;
166}
167
168static inline void io_port_wait(void) {
169 outb(port: 0x80, value: 0);
170}
171
172TSA_REENTRANT_CAPABILITY_DECLARE("irqs", IRQS_DISABLED);
173
174static inline void irq_enable(void) TSA_RELEASES_IRQS TSA_NO_ANALYSIS {
175 asm volatile("sti");
176}
177
178static inline void irq_disable(void) TSA_ACQUIRES_IRQS TSA_NO_ANALYSIS {
179 asm volatile("cli");
180}
181
182static inline void irq_enable_untracked(void) {
183 asm volatile("sti");
184}
185
186static inline bool irq_disable_save(void) TSA_ACQUIRES_IRQS TSA_NO_ANALYSIS {
187 bool enabled = irqs_enabled();
188 irq_disable();
189 return enabled;
190}
191
192static inline void
193irq_restore(bool was_enabled) TSA_RELEASES_IRQS TSA_NO_ANALYSIS {
194 if (was_enabled)
195 irq_enable();
196}
197
198static inline cc_always_inline cc_unused void
199irq_save_guard_exit(bool *was_enabled) TSA_RELEASES_IRQS TSA_NO_ANALYSIS {
200 irq_restore(was_enabled: *was_enabled);
201}
202
203#define irq_save_guard() \
204 cc_cleanup(irq_save_guard_exit) bool PP_CONCAT( \
205 irq_save_guard_, __COUNTER__) = irq_disable_save()
206
207#define irq_disable_guard() irq_save_guard()
208
209static inline void tlb_invlpg(uint64_t virt) {
210 asm volatile("invlpg (%0)" : : "r"(virt) : "memory");
211}
212
213static inline uint64_t cr2_read(void) {
214 uint64_t cr2;
215 asm volatile("mov %%cr2, %0" : "=r"(cr2));
216 return cr2;
217}
218
219static inline uint64_t cr3_read(void) {
220 uint64_t cr3;
221 asm volatile("mov %%cr3, %0" : "=r"(cr3));
222 return cr3;
223}
224
225static inline void cr3_write(uint64_t cr3) {
226 asm volatile("mov %0, %%cr3" ::"r"(cr3));
227}
228
229static inline void tlb_flush(void) {
230 uint64_t cr3 = cr3_read();
231 cr3_write(cr3);
232}
233
234static inline void cpu_pause(void) {
235 asm volatile("pause");
236}
237
238static inline void cpu_halt(void) {
239 asm volatile("hlt");
240}
241
242static inline void cpu_freeze(void) {
243 asm volatile("cli; hlt");
244}
245
246/* sti; hlt: ends the caller's interrupts-off region */
247static inline void cpu_idle(void) TSA_RELEASES_IRQS TSA_NO_ANALYSIS {
248 asm volatile("sti\n\thlt" ::: "memory");
249}
250
251static inline int bit_clz8(uint8_t a) {
252 return ci_clz(a);
253}
254
255static inline void cpu_memory_barrier(void) {
256 asm volatile("mfence" ::: "memory");
257}
258
259static inline void cpu_full_fence(void) {
260 asm volatile("mfence; lfence" ::: "memory");
261}
262
263static inline uint64_t rdtsc_after_stores(void) {
264 uint32_t lo, hi;
265 asm volatile("mfence; lfence; rdtsc" : "=a"(lo), "=d"(hi)::"memory");
266 return ((uint64_t) hi << 32) | lo;
267}
268
269static inline uint64_t rdtsc_before_loads(void) {
270 uint32_t lo, hi;
271 asm volatile("rdtsc; lfence" : "=a"(lo), "=d"(hi)::"memory");
272 return ((uint64_t) hi << 32) | lo;
273}
274