1#include <acpi/hpet.h>
2#include <asm.h>
3#include <atomic.h>
4#include <log.h>
5#include <math/bit.h>
6#include <mem/alloc.h>
7#include <mem/nofail.h>
8#include <smp/percpu.h>
9#include <stdbool.h>
10#include <stdint.h>
11#include <time/clock.h>
12#include <time/names.h>
13#include <time/tsc.h>
14#include <types/freq.h>
15#include <types/types.h>
16
17#include "internal.h"
18
19struct tsc_sync_mailbox {
20 atomic_uint32_t stage;
21 atomic_uint64_t ap_tsc;
22};
23
24struct tsc_globals tsc_global = {
25 .use_tsc = false, .max_warp = 0, .min_rtt = UINT64_MAX};
26
27PERCPU_DEFINE(struct tsc_sync_mailbox, mailboxes);
28
29#define TSC_SYNC_ROUNDS 100
30#define TSC_MAX_ALLOWED_WARP_CYCLES 200
31
32static struct clock *tsc_clock_inst = NULL;
33
34uint64_t tsc_clock_read(struct clock *clk) {
35 cc_ignore(clk);
36 return rdtsc_ordered();
37}
38
39bool tsc_has_invariant(void) {
40 uint32_t eax, ebx, ecx, edx;
41 cpuid_count(leaf: 0x80000000, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
42 if (eax < 0x80000007)
43 return false;
44
45 cpuid_count(leaf: 0x80000007, subleaf: 0, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
46 return BIT_TEST(edx, 8); /* Invariant TSC flag */
47}
48
49freq_hz_t tsc_calibrate_hpet(void) {
50 uint64_t start_tsc = rdtsc_ordered();
51 time_us_t start_us = hpet_timestamp_us();
52 time_us_t target_us = start_us + 20000; /* 20 ms calibration window */
53
54 while (hpet_timestamp_us() < target_us)
55 cpu_pause();
56
57 uint64_t end_tsc = rdtsc_ordered();
58 time_us_t end_us = hpet_timestamp_us();
59
60 uint64_t delta_tsc = end_tsc - start_tsc;
61 time_us_t delta_us = end_us - start_us;
62
63 if (delta_us == 0)
64 return 0;
65
66 return (delta_tsc * 1000000ULL) / delta_us;
67}
68
69/* TODO: a state machine with explicit values would be better */
70bool tsc_sync_check_bsp(cpu_id_t ap_cpu) {
71 int64_t best_offset = 0;
72
73 struct tsc_sync_mailbox *mbox = PERCPU_PTR_FOR(mailboxes, ap_cpu);
74 for (int i = 0; i < TSC_SYNC_ROUNDS; i++) {
75
76 atomic_store_release(&mbox->stage, 1);
77 while (atomic_load_acq(&mbox->stage) != 2)
78 cpu_pause();
79
80 /* Sample T0, signal AP to sample its TSC */
81 uint64_t t0 = rdtsc_ordered();
82 atomic_store_release(&mbox->stage, 3);
83
84 while (atomic_load_acq(&mbox->stage) != 4)
85 cpu_pause();
86
87 uint64_t t1 = rdtsc_ordered();
88 uint64_t t_ap = atomic_load_rlx(&mbox->ap_tsc);
89
90 uint64_t rtt = t1 - t0;
91 int64_t warp = 0;
92
93 if (t_ap < t0) {
94 warp = (int64_t) (t0 - t_ap);
95 } else if (t_ap > t1) {
96 warp = (int64_t) (t_ap - t1);
97 }
98
99 if (warp > tsc_global.max_warp)
100 tsc_global.max_warp = warp;
101
102 if (rtt < tsc_global.min_rtt) {
103 tsc_global.min_rtt = rtt;
104 best_offset = (int64_t) t_ap - (int64_t) (t0 + rtt / 2);
105 }
106 }
107
108 atomic_store_release(&mbox->stage, 0);
109
110 if (tsc_global.max_warp > TSC_MAX_ALLOWED_WARP_CYCLES) {
111 log_msg(LOG_WARN,
112 "TSC sync check failed for CPU %zu (max warp: %ld cycles, min "
113 "RTT: %lu cycles)",
114 ap_cpu, tsc_global.max_warp, tsc_global.min_rtt);
115 if (tsc_clock_inst) {
116 tsc_clock_inst->flags |= CLOCK_FLAG_UNSTABLE;
117 tsc_clock_inst->rating = CLOCK_RATING_UNSUITABLE;
118 }
119 return false;
120 }
121
122 log_msg(LOG_INFO,
123 "TSC sync check OK for CPU %zu (warp: %ld cycles, offset: %ld "
124 "cycles, min RTT: %lu cycles)",
125 ap_cpu, tsc_global.max_warp, best_offset, tsc_global.min_rtt);
126 return true;
127}
128
129void tsc_sync_check_ap(cpu_id_t self) {
130 struct tsc_sync_mailbox *mbox = PERCPU_PTR_FOR(mailboxes, self);
131 for (int i = 0; i < TSC_SYNC_ROUNDS; i++) {
132 while (atomic_load_acq(&mbox->stage) != 1)
133 cpu_pause();
134
135 atomic_store_release(&mbox->stage, 2);
136
137 while (atomic_load_acq(&mbox->stage) != 3)
138 cpu_pause();
139
140 uint64_t ap = rdtsc_ordered();
141 atomic_store_rlx(&mbox->ap_tsc, ap);
142 atomic_store_release(&mbox->stage, 4);
143 }
144}
145
146void tsc_sync_check_all_aps(void) {
147 size_t ok = 0;
148 for (cpu_id_t i = 1; i < global.core_count; i++)
149 ok += tsc_sync_check_bsp(ap_cpu: i);
150
151 if (ok == global.core_count - 1)
152 tsc_global.use_tsc = true;
153}
154
155bool tsc_should_use_tsc(void) {
156 return tsc_global.use_tsc;
157}
158
159struct clock *tsc_clock_init(freq_hz_t freq_hz) {
160 if (freq_hz == 0)
161 freq_hz = tsc_calibrate_hpet();
162
163 struct clock *clk = nofail(clock_create(CLOCK_NAME_TSC));
164
165 clk->read = tsc_clock_read;
166 clk->frequency_khz = HZ_TO_KHZ(freq_hz);
167 clk->mult = clock_frequency_to_mult(clock: clk);
168 clk->state = CLOCK_STATE_ON;
169 clk->flags = CLOCK_FLAG_HRES | CLOCK_FLAG_TIMESTAMP_SOURCE;
170
171 if (tsc_has_invariant()) {
172 clk->rating = CLOCK_RATING_BEST;
173 } else {
174 clk->rating = CLOCK_RATING_UNSUITABLE;
175 clk->flags |= CLOCK_FLAG_UNSTABLE;
176 }
177
178 tsc_clock_inst = clk;
179 clock_register(c: clk);
180
181 return clk;
182}
183