1/* @title: TLB */
2#pragma once
3#include <atomic.h>
4#include <compiler/diagnostic.h>
5#include <math/range.h>
6#include <stdint.h>
7#include <structures/cpu_mask.h>
8#include <structures/mpmc_fifo_ring.h>
9#include <types/types.h>
10
11/* per-cpu */
12#define TLB_QUEUE_SIZE 64
13
14/*
15 * The way our TLB shootdowns work is that we have a two tiered queue:
16 *
17 * (1) Eager
18 * (2) Lazy
19 *
20 * We group by domains, which are NUMA nodes when available, or grouped by
21 * LLC/arbitrary when on UMA. On x86 (currently the only supported architecture),
22 * we use x2APIC cluster IPI batching to reduce IPI costs, and precompute
23 * the IPI costs for a given domain to determine whether a domain IPI
24 * should be chosen, or individual per-CPU IPIs.
25 *
26 * An issue with lazy TLB invalidations, however, is that thread migrations can
27 * affect coherency. For instance, if thread A is running on CPU 0, and it
28 * invalidates all relevant lazy queues, and uses that premise to then
29 * read some virtual memory, if it gets migrated to CPU 1, the
30 * TLB could end up stale.
31 *
32 * To mitigate this, we introduce an idea of "timestamping" TLB invalidations.
33 * This is done via the TSC on x86, and the idea is that upon an invalidation
34 * of the lazy queue, you record a timestamp for when that happened, on the
35 * current thread.
36 *
37 */
38
39typedef uint64_t tlb_stamp_t;
40
41/* A type describes what an actual operation is */
42enum tlb_payload_type {
43 TLB_PAYLOAD_PAGE, /* Invalidate just this one page */
44 TLB_PAYLOAD_RANGE, /* Invalidate this range */
45 TLB_PAYLOAD_FLUSH /* Completely flush everything */
46};
47
48/* Modes specify how to deliver. SYNC and ASYNC are eager */
49enum tlb_mode {
50 TLB_MODE_SYNC, /* Send the IPI out, wait for responses */
51 TLB_MODE_ASYNC, /* Fire and forget */
52 TLB_MODE_LAZY, /* Tell the other CPUs about it, but no enqueues */
53 TLB_MODE_NONE /* Local only: no remote CPUs are told (caller batches) */
54};
55
56struct tlb_payload {
57 enum tlb_payload_type type;
58 union {
59 vaddr_t addr;
60 struct {
61 RANGE_DEFINE(vaddr_t, range);
62 size_t stride;
63 };
64 };
65};
66
67struct tlb_request {
68 enum tlb_mode mode;
69 struct cpu_mask targets;
70 struct tlb_payload payload;
71};
72
73void tlb_init(void);
74enum irq_result tlb_shootdown_isr(struct irq_context *rsp, void *ctx);
75void tlb_shootdown_full(struct tlb_request rq);
76void tlb_invalidate_lazy(void);
77
78#define tlb_shootdown(...) \
79 tlb_shootdown_full(cc_wno_override_init_expr( \
80 struct tlb_request, \
81 ((struct tlb_request) {.mode = TLB_MODE_SYNC, \
82 .targets = CPU_MASK_INIT_ALL, \
83 .payload.type = TLB_PAYLOAD_PAGE, \
84 ##__VA_ARGS__})))
85