1#pragma once
2#include <mem/fixed_size_alloc.h>
3#include <mem/vas.h>
4#include <stdatomic.h>
5#include <structures/list.h>
6#include <structures/rbt.h>
7
8/* Byte-granular bins preserve bootstrap array allocs */
9#define VAS_BIN_COUNT 64
10#define VAS_OWNER_GLOBAL CPU_ID_MAX
11
12#define VAS_MAG_CLASSES 4
13#define VAS_MAG_CAPACITY 16
14#define VAS_MAG_REFILL 4
15#define VAS_MAG_BYTE_LIMIT (8ULL << 20)
16
17enum vas_segment_type {
18 VAS_SEG_FREE,
19 VAS_SEG_BUSY,
20 VAS_SEG_IMPORTED,
21 VAS_SEG_CACHED,
22};
23
24/* Slots outlive tags, and only a successful token claim
25 * can allow a segment read, with page aligned
26 * addresses leaving low bits for the state */
27enum vas_mag_state { VAS_MAG_LIVE = 1, VAS_MAG_CACHED, VAS_MAG_CLAIMED };
28#define VAS_MAG_STATE_MASK 3UL
29struct vas_mag_slot {
30 _Atomic uintptr_t token;
31 struct vas_segment *segment;
32};
33
34struct vas_magazine {
35 struct vas_mag_slot slots[VAS_MAG_CAPACITY];
36 /* Owner CPU only, at DISPATCH, prefer most recently freed */
37 uint32_t recent;
38};
39
40struct vas_segment {
41 vaddr_t start;
42 size_t length;
43
44 /* Zero in global, local segments can coalesce */
45 vaddr_t span_start;
46 _Atomic enum vas_segment_type type;
47 struct vas_mag_slot *mag_slot; /* Immutable while LIVE/CACHED */
48 struct rbt_node node;
49 struct list_head seg_node;
50 struct list_head bin_node;
51};
52
53struct vas_arena {
54 struct spinlock lock;
55 struct rbt tree;
56 struct list_head all_segs;
57 struct list_head free_bins[VAS_BIN_COUNT];
58 uint64_t bin_mask;
59 struct fixed_size_range fsr;
60 size_t total_free;
61 struct vas_magazine magazines[VAS_MAG_CLASSES];
62#ifdef TEST_ENABLED
63 /* TODO: Use injection sites and bring this out of TEST_ENABLED */
64 ssize_t tag_alloc_budget;
65 uint64_t mag_alloc_hits, mag_free_hits; /* Owner CPU only */
66 uint64_t requests[VAS_MAG_CLASSES + 1], alignments[64];
67#endif
68};
69
70struct vas {
71 struct vas_arena global;
72 vaddr_t base;
73 vaddr_t limit;
74 vaddr_t map_base;
75 size_t owner_count;
76 size_t bootstrap_pages; /* Zero for non-bootstrap ones */
77 struct vas_arena *local;
78 _Atomic cpu_id_t *chunk_owner;
79
80 /* Charged whilst enrolled, including LIVE slots,
81 * this is a bound on retained VA */
82 _Atomic size_t mag_reserved_bytes;
83#ifdef TEST_ENABLED
84 bool magazines_disabled;
85#endif
86};
87