1/* @title: Page Table */
2#pragma once
3#include <asm.h>
4#include <irq/irq.h>
5#include <mem/page.h>
6#include <sch/irql.h>
7#include <stdatomic.h>
8#include <stdint.h>
9
10#define PT_LEVELS 4
11
12#define PT_SHIFT_L1 12
13#define PT_SHIFT_L2 21
14#define PT_SHIFT_L3 30
15#define PT_SHIFT_L4 39
16
17#define PT_STRIDE 9
18
19#define PTE_LOCK_SHIFT 9
20#define PTE_AVAIL2_SHIFT 10
21#define PTE_LOCK_BIT ((uint64_t) 1 << PTE_LOCK_SHIFT)
22#define PTE_AVAIL2_BIT ((uint64_t) 1 << PTE_AVAIL2_SHIFT)
23
24/* Packed layout:
25 *
26 * bit 0 P = 0
27 * bits 1-2 type
28 * bits 3-8 payload low (6 bits)
29 * bit 9 LOCK (not payload)
30 * bit 10 AVAIL2 (not payload)
31 * bits 11-63 payload high (53 bits)
32 */
33#define PTE_TAGGED_TYPE_SHIFT (PAGE_PRESENT_SHIFT + 1)
34#define PTE_TAGGED_TYPE_MASK 0x3ULL
35
36#define PTE_TAGGED_PAYLOAD_LOW_BITS 6
37#define PTE_TAGGED_PAYLOAD_LOW_SHIFT 3 /* packed position of the low chunk */
38#define PTE_TAGGED_PAYLOAD_HIGH_SHIFT \
39 11 /* packed position of the high chunk \
40 */
41#define PTE_TAGGED_PAYLOAD_LOW_MASK ((1ULL << PTE_TAGGED_PAYLOAD_LOW_BITS) - 1)
42#define PTE_TAGGED_PAYLOAD_BITS \
43 (PTE_TAGGED_PAYLOAD_LOW_BITS + (64 - PTE_TAGGED_PAYLOAD_HIGH_SHIFT))
44
45#define PTE_TAGGED_GET_TYPE(pte) \
46 (((pte) >> PTE_TAGGED_TYPE_SHIFT) & PTE_TAGGED_TYPE_MASK)
47#define PTE_TAGGED_SET_TYPE(type) ((uint64_t) (type) << PTE_TAGGED_TYPE_SHIFT)
48
49typedef _Atomic uint64_t pte_atomic_t;
50
51/* Packed into the low X bits */
52enum pte_tag_type : uint8_t {
53 PTE_TAG_TYPE_NONE = 0,
54 PTE_TAG_TYPE_DEMAND_PAGED = 1,
55};
56
57/* This is a bit like a tagged pointer, just
58 * that the payload moseys around LOCK_BIT and AVAIL2,
59 * and the type sits right above PRESENT
60 *
61 * The layout is effectively
62 *
63 * [ payload higher ] [ LOCK + AVAIL2 ] [ payload lower ] [ type ] [ P = 0 ]
64 */
65struct pte_tagged {
66 enum pte_tag_type type;
67 uint64_t payload; /* NOTE: higher 5 bits MUST be zero */
68};
69
70static inline void pte_tagged_check(struct pte_tagged *pt) {
71 kassert(pt->type != PTE_TAG_TYPE_NONE);
72 kassert((pt->payload >> PTE_TAGGED_PAYLOAD_BITS) == 0);
73}
74
75static inline uint64_t pte_tagged_pack(struct pte_tagged *pt) {
76 /* We leave LOCK + AVAIL2 zeroed here, whoever uses this must OR it into a
77 * PTE that already holds those bits, probably atomically */
78 uint64_t low = pt->payload & PTE_TAGGED_PAYLOAD_LOW_MASK;
79 uint64_t high = pt->payload >> PTE_TAGGED_PAYLOAD_LOW_BITS;
80
81 return PTE_TAGGED_SET_TYPE(pt->type) |
82 (low << PTE_TAGGED_PAYLOAD_LOW_SHIFT) |
83 (high << PTE_TAGGED_PAYLOAD_HIGH_SHIFT);
84}
85
86static inline struct pte_tagged pte_tagged_unpack(pte_t pte) {
87 struct pte_tagged pt;
88 pt.type = PTE_TAGGED_GET_TYPE(pte);
89
90 uint64_t low =
91 (pte >> PTE_TAGGED_PAYLOAD_LOW_SHIFT) & PTE_TAGGED_PAYLOAD_LOW_MASK;
92 uint64_t high = pte >> PTE_TAGGED_PAYLOAD_HIGH_SHIFT;
93
94 pt.payload = (high << PTE_TAGGED_PAYLOAD_LOW_BITS) | low;
95 return pt;
96}
97
98static inline bool pte_locked(pte_atomic_t *pte) {
99 return atomic_load_explicit(pte, memory_order_relaxed) & PTE_LOCK_BIT;
100}
101
102static inline uint64_t pte_read(pte_atomic_t *pte) {
103 return atomic_load_explicit(pte, memory_order_relaxed);
104}
105
106static inline uint64_t pte_or(pte_atomic_t *pte, uint64_t val) {
107 return atomic_fetch_or_explicit(pte, val, memory_order_acq_rel);
108}
109
110static inline void pte_unlock_internal(pte_atomic_t *pte) {
111 kassert(
112 atomic_fetch_and_explicit(pte, ~PTE_LOCK_BIT, memory_order_release) &
113 PTE_LOCK_BIT);
114}
115
116static inline void pte_lock_internal(pte_atomic_t *pte) {
117 for (;;) {
118 uint64_t old = atomic_load_explicit(pte, memory_order_relaxed);
119
120 if (old & PTE_LOCK_BIT) {
121 cpu_relax();
122 continue;
123 }
124
125 if (atomic_compare_exchange_weak_explicit(pte, &old, old | PTE_LOCK_BIT,
126 memory_order_acquire,
127 memory_order_relaxed))
128 return;
129
130 cpu_relax();
131 }
132}
133
134static inline enum irql pte_lock_irql(pte_atomic_t *pte) {
135 enum irql old_irql = irql_raise(new_level: IRQL_DISPATCH_LEVEL);
136 pte_lock_internal(pte);
137 return old_irql;
138}
139
140static inline void pte_unlock_irql(pte_atomic_t *pte, enum irql old_irql) {
141 pte_unlock_internal(pte);
142 irql_lower(old_level: old_irql);
143}
144
145static inline bool pte_has_tag(pte_atomic_t *pte) {
146 kassert(!(pte_read(pte) & PAGE_PRESENT));
147 return (pte_read(pte) & ~(PAGE_PRESENT | PTE_LOCK_BIT | PTE_AVAIL2_BIT)) !=
148 0;
149}
150
151static inline uint64_t pte_tag_pte(pte_atomic_t *pte, struct pte_tagged *ptag) {
152 kassert(pte_locked(pte));
153 kassert(!pte_has_tag(pte));
154 pte_tagged_check(pt: ptag);
155 uint64_t packed = pte_tagged_pack(pt: ptag);
156 return pte_or(pte, val: packed);
157}
158
159static inline struct pte_tagged pte_read_tag(pte_atomic_t *pte) {
160 kassert(pte_has_tag(pte));
161 return pte_tagged_unpack(pte: pte_read(pte));
162}
163
164static inline bool pte_in_use(pte_atomic_t *pte) {
165 if (pte_read(pte) & PAGE_PRESENT)
166 return true;
167
168 return pte_has_tag(pte);
169}
170