1#include <mem/tlb.h>
2
3#define TLB_STAMP_GUARD_SLACK 1000 /* I just picked a number */
4enum tlb_stamp_source {
5 TLB_STAMP_GEN, /* One global counter, fallback */
6 TLB_STAMP_TSC
7};
8
9struct tlb_queue_entry {
10 struct tlb_payload payload;
11};
12
13MPMC_FIFO_RING_DECLARE(tlb_queue, struct tlb_queue_entry);
14
15struct tlb_ring {
16 struct tlb_queue queue;
17 tlb_stamp_t stamp_latest;
18
19 atomic_uint64_t done_pos; /* every queue slot < this has been invalidated */
20 atomic_uint64_t flush_req; /* bumped when the queue full */
21 atomic_uint64_t flush_done; /* last flush_req covered by a full flush */
22};
23
24struct tlb_domain {
25 struct domain *domain;
26 size_t ipi_cost; /* How many IPIs does it take to
27 * bother everyone in here? */
28};
29
30struct tlb_cpu {
31 struct tlb_ring eager_ring;
32 struct tlb_ring lazy_ring;
33 tlb_stamp_t flush_stamp;
34 cpu_id_t id;
35};
36
37struct tlb_globals {
38 enum tlb_stamp_source stamp_source;
39 atomic_uint64_t stamp_gen;
40 uint64_t stamp_guard; /* Give leeway */
41};
42
43extern struct tlb_globals tlb_global;
44static inline tlb_stamp_t tlb_stamp_publish(void) {
45 if (tlb_global.stamp_source == TLB_STAMP_TSC)
46 return rdtsc_after_stores();
47
48 return atomic_fetch_add_acq_rel(&tlb_global.stamp_gen, 1);
49}
50
51static inline tlb_stamp_t tlb_stamp_read(void) {
52 if (tlb_global.stamp_source == TLB_STAMP_TSC)
53 return rdtsc_before_loads();
54
55 return atomic_load_acq(&tlb_global.stamp_gen);
56}
57
58/* Is `req` past `seen` ? */
59static inline bool tlb_stamp_past(tlb_stamp_t seen, tlb_stamp_t req) {
60 return seen > req + tlb_global.stamp_guard;
61}
62