1#pragma once
2#include <atomic.h>
3#include <compiler/core.h>
4#include <mem/fixed_size_alloc.h>
5#include <mem/vas.h>
6#include <structures/list.h>
7#include <structures/rbt.h>
8
9/* Byte-granular bins preserve bootstrap array allocs */
10#define VAS_BIN_COUNT 64
11#define VAS_OWNER_GLOBAL CPU_ID_MAX
12
13#define VAS_MAG_CLASSES 4
14#define VAS_MAG_CAPACITY 16
15#define VAS_MAG_REFILL 4
16#define VAS_MAG_BYTE_LIMIT (8ULL << 20)
17
18enum vas_segment_type {
19 VAS_SEG_FREE,
20 VAS_SEG_BUSY,
21 VAS_SEG_IMPORTED,
22 VAS_SEG_CACHED,
23};
24
25/* Slots outlive tags, and only a successful token claim
26 * can allow a segment read, with page aligned
27 * addresses leaving low bits for the state */
28enum vas_mag_state { VAS_MAG_LIVE = 1, VAS_MAG_CACHED, VAS_MAG_CLAIMED };
29#define VAS_MAG_STATE_MASK 3UL
30
31static inline uintptr_t vas_token_make(vaddr_t addr, enum vas_mag_state state) {
32 return ct_raw(addr) | state;
33}
34
35struct vas_mag_slot {
36 atomic_uintptr_t token;
37 struct vas_segment *segment;
38};
39
40struct vas_magazine {
41 struct vas_mag_slot slots[VAS_MAG_CAPACITY];
42 /* Owner CPU only, at DISPATCH, prefer most recently freed */
43 uint32_t recent;
44};
45
46struct vas_segment {
47 vaddr_t start;
48 size_t length;
49
50 /* Zero in global, local segments can coalesce */
51 vaddr_t span_start;
52 atomic(enum vas_segment_type) type;
53 struct vas_mag_slot *mag_slot; /* Immutable while LIVE/CACHED */
54 struct rbt_node node;
55 struct list_node seg_node;
56 struct list_node bin_node;
57};
58
59struct vas_arena {
60 struct spinlock lock;
61 struct rbt tree;
62 struct list_head all_segs;
63 struct list_head free_bins[VAS_BIN_COUNT];
64 uint64_t bin_mask;
65 struct fixed_size_range fsr;
66 size_t total_free;
67 struct vas_magazine magazines[VAS_MAG_CLASSES];
68
69#ifdef TEST_ENABLED
70 /* TODO: Use injection sites and bring this out of TEST_ENABLED */
71 ssize_t tag_alloc_budget;
72 uint64_t mag_alloc_hits, mag_free_hits; /* Owner CPU only */
73 uint64_t requests[VAS_MAG_CLASSES + 1], alignments[64];
74#endif
75};
76
77struct vas {
78 struct vas_arena global;
79 vaddr_t base;
80 vaddr_t limit;
81 vaddr_t map_base;
82 size_t owner_count;
83 size_t bootstrap_pages; /* Zero for non-bootstrap ones */
84 struct vas_arena *local;
85 atomic(cpu_id_t) * chunk_owner;
86
87 /* Charged whilst enrolled, including LIVE slots,
88 * this is a bound on retained VA */
89 atomic_size_t mag_reserved_bytes;
90
91#ifdef TEST_ENABLED
92 bool magazines_disabled;
93#endif
94};
95