1#include <acpi/hpet.h>
2#include <asm.h>
3#include <log.h>
4#include <mem/alloc_or_die.h>
5#include <stdatomic.h>
6#include <stdbool.h>
7#include <stdint.h>
8#include <time/clock.h>
9#include <time/names.h>
10#include <types/freq.h>
11#include <types/types.h>
12
13#include "internal.h"
14#include <mem/alloc.h>
15
16struct tsc_sync_mailbox {
17 _Atomic uint32_t stage;
18 _Atomic uint64_t ap_tsc;
19};
20
21static struct tsc_sync_mailbox *mailboxes;
22static bool tsc_use_tsc_for_timekeeping = false;
23
24#define TSC_SYNC_ROUNDS 100
25#define TSC_MAX_ALLOWED_WARP_CYCLES 200
26
27static struct clock *tsc_clock_inst = NULL;
28
29static uint64_t tsc_clock_read(struct clock *clk) {
30 (void) clk;
31 return rdtsc_ordered();
32}
33
34static bool tsc_has_invariant(void) {
35 uint32_t eax, ebx, ecx, edx;
36 cpuid_count(leaf: 0x80000000, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
37 if (eax < 0x80000007)
38 return false;
39
40 cpuid_count(leaf: 0x80000007, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
41 return (edx & (1 << 8)) != 0; /* Invariant TSC flag */
42}
43
44freq_hz_t tsc_calibrate_hpet(void) {
45 uint64_t start_tsc = rdtsc_ordered();
46 time_us_t start_us = hpet_timestamp_us();
47 time_us_t target_us = start_us + 20000; /* 20 ms calibration window */
48
49 while (hpet_timestamp_us() < target_us)
50 cpu_relax();
51
52 uint64_t end_tsc = rdtsc_ordered();
53 time_us_t end_us = hpet_timestamp_us();
54
55 uint64_t delta_tsc = end_tsc - start_tsc;
56 time_us_t delta_us = end_us - start_us;
57
58 if (delta_us == 0)
59 return 0;
60
61 return (delta_tsc * 1000000ULL) / delta_us;
62}
63
64/* TODO: a state machine with explicit values would be better */
65bool tsc_sync_check_bsp(cpu_id_t ap_cpu) {
66 int64_t max_warp = 0;
67 uint64_t min_rtt = UINT64_MAX;
68 int64_t best_offset = 0;
69
70 for (int i = 0; i < TSC_SYNC_ROUNDS; i++) {
71 atomic_store_explicit(&mailboxes[ap_cpu].stage, 1,
72 memory_order_release);
73 while (atomic_load_explicit(&mailboxes[ap_cpu].stage,
74 memory_order_acquire) != 2)
75 cpu_relax();
76
77 /* Sample T0, signal AP to sample its TSC */
78 uint64_t t0 = rdtsc_ordered();
79 atomic_store_explicit(&mailboxes[ap_cpu].stage, 3,
80 memory_order_release);
81
82 while (atomic_load_explicit(&mailboxes[ap_cpu].stage,
83 memory_order_acquire) != 4)
84 cpu_relax();
85
86 uint64_t t1 = rdtsc_ordered();
87 uint64_t t_ap = atomic_load_explicit(&mailboxes[ap_cpu].ap_tsc,
88 memory_order_relaxed);
89
90 uint64_t rtt = t1 - t0;
91 int64_t warp = 0;
92
93 if (t_ap < t0) {
94 warp = (int64_t) (t0 - t_ap);
95 } else if (t_ap > t1) {
96 warp = (int64_t) (t_ap - t1);
97 }
98
99 if (warp > max_warp)
100 max_warp = warp;
101
102 if (rtt < min_rtt) {
103 min_rtt = rtt;
104 best_offset = (int64_t) t_ap - (int64_t) (t0 + rtt / 2);
105 }
106 }
107
108 atomic_store_explicit(&mailboxes[ap_cpu].stage, 0, memory_order_release);
109
110 if (max_warp > TSC_MAX_ALLOWED_WARP_CYCLES) {
111 log_msg(LOG_WARN,
112 "TSC sync check failed for CPU %zu (max warp: %ld cycles, min "
113 "RTT: %lu cycles)",
114 ap_cpu, max_warp, min_rtt);
115 if (tsc_clock_inst) {
116 tsc_clock_inst->flags |= CLOCK_FLAG_UNSTABLE;
117 tsc_clock_inst->rating = CLOCK_RATING_UNSUITABLE;
118 }
119 return false;
120 }
121
122 log_msg(LOG_INFO,
123 "TSC sync check OK for CPU %zu (warp: %ld cycles, offset: %ld "
124 "cycles, min RTT: %lu cycles)",
125 ap_cpu, max_warp, best_offset, min_rtt);
126 return true;
127}
128
129void tsc_sync_check_ap(cpu_id_t self) {
130 for (int i = 0; i < TSC_SYNC_ROUNDS; i++) {
131 while (atomic_load_explicit(&mailboxes[self].stage,
132 memory_order_acquire) != 1)
133 cpu_relax();
134
135 atomic_store_explicit(&mailboxes[self].stage, 2, memory_order_release);
136
137 while (atomic_load_explicit(&mailboxes[self].stage,
138 memory_order_acquire) != 3)
139 cpu_relax();
140
141 uint64_t ap = rdtsc_ordered();
142 atomic_store_explicit(&mailboxes[self].ap_tsc, ap,
143 memory_order_relaxed);
144 atomic_store_explicit(&mailboxes[self].stage, 4, memory_order_release);
145 }
146}
147
148void tsc_mailboxes_init(void) {
149 mailboxes =
150 kmalloc_or_die(sizeof(struct tsc_sync_mailbox) * global.core_count);
151}
152
153void tsc_sync_check_all_aps(void) {
154 size_t ok = 0;
155 for (cpu_id_t i = 1; i < global.core_count; i++)
156 ok += tsc_sync_check_bsp(ap_cpu: i);
157
158 if (ok == global.core_count - 1)
159 tsc_use_tsc_for_timekeeping = true;
160}
161
162bool tsc_should_use_tsc(void) {
163 return tsc_use_tsc_for_timekeeping;
164}
165
166struct clock *tsc_clock_init(freq_hz_t freq_hz) {
167 if (freq_hz == 0)
168 freq_hz = tsc_calibrate_hpet();
169
170 struct clock *clk = alloc_or_die(clock_create(CLOCK_NAME_TSC));
171
172 clk->read = tsc_clock_read;
173 clk->frequency_khz = HZ_TO_KHZ(freq_hz);
174 clk->mult = clock_frequency_to_mult(clock: clk);
175 clk->state = CLOCK_STATE_ON;
176 clk->flags = CLOCK_FLAG_HRES | CLOCK_FLAG_TIMESTAMP_SOURCE;
177
178 if (tsc_has_invariant()) {
179 clk->rating = CLOCK_RATING_BEST;
180 } else {
181 clk->rating = CLOCK_RATING_UNSUITABLE;
182 clk->flags |= CLOCK_FLAG_UNSTABLE;
183 }
184
185 tsc_clock_inst = clk;
186 clock_register(c: clk);
187
188 return clk;
189}
190