1/* @title: Fixed Size Allocation */
2#pragma once
3#include <compiler/core.h>
4#include <mem/alloc.h>
5#include <mem/nofail.h>
6#include <mem/page.h>
7#include <smp/perdomain.h>
8#include <structures/list.h>
9#include <sync/spinlock.h>
10
11/* Number of fully-empty freelist backing pages to keep cached per local tree
12 * before reclaiming on free
13 *
14 * Provides hysteresis so a workload that oscillates across a page boundary
15 * does not pmm_alloc/pmm_free on every transition */
16#define FIXED_SIZE_KEEP_EMPTY_PAGES 1
17
18/* The idea here:
19 *
20 * memory is laid out as
21 *
22 * [ data ] [ potential empty ] [ fixed_size_node ]
23 *
24 * essentially, the objects have their own alignment, so each
25 * of these "full nodes" comes with the data first, then the
26 * empty zone. the fixed size node trails to link it all up
27 */
28struct fixed_size_node {
29 struct list_node list_node;
30};
31static_assert(offsetof(struct fixed_size_node, list_node) == 0);
32
33struct fixed_size_page_hdr {
34 domain_id_t domain; /* DOMAIN_ID_NONE means it isn't perdomain */
35 size_t free_count;
36 size_t total;
37 struct list_node page_list;
38};
39
40struct fixed_size_range_attributes {
41 size_t obj_size;
42 size_t obj_align;
43 void (*init_obj)(void *);
44 void (*deinit_obj)(void *);
45 bool bootstrap_mode : 1;
46};
47
48struct fixed_size_range {
49 struct fixed_size_range_attributes attrs;
50 struct spinlock lock;
51 struct list_head freelist; /* fixed_size_node list_node */
52 struct list_head fl_pages; /* fixed_size_page_hdr page_list */
53 size_t empty_pages;
54 size_t full_node_size; /* node size + object size */
55 struct fixed_size_range **perdomain_fsrs;
56 domain_id_t domain;
57};
58
59static_assert(sizeof(struct fixed_size_range) < PAGE_SIZE);
60
61void *fixed_size_alloc(struct fixed_size_range *fsr) cw_alloc();
62void fixed_size_free(struct fixed_size_range *fsr, void *obj);
63void fixed_size_reclaim_freelist_pages(struct fixed_size_range *fsr);
64struct fixed_size_range *
65fixed_size_range_create(struct fixed_size_range_attributes *attrs);
66void fixed_size_range_init(struct fixed_size_range *fsr,
67 struct fixed_size_range_attributes *attrs);
68
69#define FIXED_SIZE_RANGE_PERDOMAIN_DEFINE(name, ...) \
70 static cc_unused bool __fsr_##name##_enabled = false; \
71 static void __##name##_fsr_init(struct fixed_size_range *fsr_, \
72 size_t domain_) { \
73 static struct fixed_size_range **__perdomain_fsrs_##name = NULL; \
74 if (!__perdomain_fsrs_##name) \
75 __perdomain_fsrs_##name = \
76 nofail_vnews(__perdomain_fsrs_##name, global.domain_count); \
77 \
78 struct fixed_size_range_attributes __fsr_attrs = {__VA_ARGS__}; \
79 fixed_size_range_init(fsr_, &__fsr_attrs); \
80 __perdomain_fsrs_##name[domain_] = fsr_; \
81 fsr_->perdomain_fsrs = __perdomain_fsrs_##name; \
82 fsr_->domain = domain_; \
83 if (domain_ == global.domain_count - 1) \
84 __fsr_##name##_enabled = true; \
85 } \
86 PERDOMAIN_DEFINE(struct fixed_size_range, __##name##_fsr, \
87 __##name##_fsr_init)
88
89#define FSR_PERDOMAIN_ENABLED(name) __fsr_##name##_enabled
90#define FSR_PERDOMAIN(name) PERDOMAIN(__##name##_fsr)
91
92/* _NONE here: domains are an optimization anyways */
93#define FSR_PERDOMAIN_THIS(name) PERDOMAIN_PTR(TOPC_NONE, __##name##_fsr)
94#define FSR_PERDOMAIN_ALLOC(name) fixed_size_alloc(FSR_PERDOMAIN_THIS(name))
95#define FSR_PERDOMAIN_FREE(name, obj) \
96 fixed_size_free(FSR_PERDOMAIN_THIS(name), (obj))
97
98static inline struct fixed_size_page_hdr *fixed_size_page_of(void *o) {
99 return (struct fixed_size_page_hdr *) PAGE_ALIGN_DOWN(o);
100}
101