1#include "internal.h"
2
3/* BG and MAX both have unlimited scans. This is because they
4 * are never called from kmalloc/kfree, and we can dilly dally
5 * a bit due to the absence of a need for speed. */
6#define SLAB_GC_AGG_BG_SCAN_MAX SIZE_MAX
7#define SLAB_GC_AGG_RECLAIM_SCAN_MAX 32
8#define SLAB_GC_AGG_STANDARD_SCAN_MAX 64
9#define SLAB_GC_AGG_LOW_MEM_SCAN_MAX 1024
10#define SLAB_GC_AGG_EMERGENCY_SCAN_MAX 4096
11#define SLAB_GC_AGG_MAX_SCAN_MAX SIZE_MAX
12static const size_t gc_agg_scan_max[SLAB_GC_FLAG_AGG_COUNT] = {
13 [SLAB_GC_FLAG_AGG_BG] = SLAB_GC_AGG_BG_SCAN_MAX,
14 [SLAB_GC_FLAG_AGG_RECLAIM] = SLAB_GC_AGG_RECLAIM_SCAN_MAX,
15 [SLAB_GC_FLAG_AGG_STANDARD] = SLAB_GC_AGG_STANDARD_SCAN_MAX,
16 [SLAB_GC_FLAG_AGG_LOW_MEM] = SLAB_GC_AGG_LOW_MEM_SCAN_MAX,
17 [SLAB_GC_FLAG_AGG_EMERGENCY] = SLAB_GC_AGG_EMERGENCY_SCAN_MAX,
18 [SLAB_GC_FLAG_AGG_MAX] = SLAB_GC_AGG_MAX_SCAN_MAX,
19};
20
21/* BG and MAX once again are allowed to scan everything because
22 * they are always executed from contexts where they can dawdle for days */
23#define SLAB_GC_AGG_BG_SCAN_PCT 100
24#define SLAB_GC_AGG_RECLAIM_SCAN_PCT 2
25#define SLAB_GC_AGG_STANDARD_SCAN_PCT 5
26#define SLAB_GC_AGG_LOW_MEM_SCAN_PCT 15
27#define SLAB_GC_AGG_EMERGENCY_SCAN_PCT 25
28#define SLAB_GC_AGG_MAX_SCAN_PCT 100
29static const size_t gc_agg_scan_pct[SLAB_GC_FLAG_AGG_COUNT] = {
30 [SLAB_GC_FLAG_AGG_BG] = SLAB_GC_AGG_BG_SCAN_PCT,
31 [SLAB_GC_FLAG_AGG_RECLAIM] = SLAB_GC_AGG_RECLAIM_SCAN_PCT,
32 [SLAB_GC_FLAG_AGG_STANDARD] = SLAB_GC_AGG_STANDARD_SCAN_PCT,
33 [SLAB_GC_FLAG_AGG_LOW_MEM] = SLAB_GC_AGG_LOW_MEM_SCAN_PCT,
34 [SLAB_GC_FLAG_AGG_EMERGENCY] = SLAB_GC_AGG_EMERGENCY_SCAN_PCT,
35 [SLAB_GC_FLAG_AGG_MAX] = SLAB_GC_AGG_MAX_SCAN_PCT,
36};
37
38#define SLAB_GC_AGG_BG_AGE_FACTOR_PCT 150
39#define SLAB_GC_AGG_RECLAIM_AGE_FACTOR_PCT 100
40#define SLAB_GC_AGG_STANDARD_AGE_FACTOR_PCT 125
41#define SLAB_GC_AGG_LOW_MEM_AGE_FACTOR_PCT 150
42#define SLAB_GC_AGG_EMERGENCY_AGE_FACTOR_PCT 200
43#define SLAB_GC_AGG_MAX_AGE_FACTOR_PCT 300
44static const size_t gc_agg_age_factor_pct[SLAB_GC_FLAG_AGG_COUNT] = {
45 [SLAB_GC_FLAG_AGG_BG] = SLAB_GC_AGG_BG_AGE_FACTOR_PCT,
46 [SLAB_GC_FLAG_AGG_RECLAIM] = SLAB_GC_AGG_RECLAIM_AGE_FACTOR_PCT,
47 [SLAB_GC_FLAG_AGG_STANDARD] = SLAB_GC_AGG_STANDARD_AGE_FACTOR_PCT,
48 [SLAB_GC_FLAG_AGG_LOW_MEM] = SLAB_GC_AGG_LOW_MEM_AGE_FACTOR_PCT,
49 [SLAB_GC_FLAG_AGG_EMERGENCY] = SLAB_GC_AGG_EMERGENCY_AGE_FACTOR_PCT,
50 [SLAB_GC_FLAG_AGG_MAX] = SLAB_GC_AGG_MAX_AGE_FACTOR_PCT,
51};
52
53#define SLAB_GC_AGG_BG_SIZE_FACTOR_PCT 100
54#define SLAB_GC_AGG_RECLAIM_SIZE_FACTOR_PCT 200
55#define SLAB_GC_AGG_STANDARD_SIZE_FACTOR_PCT 225
56#define SLAB_GC_AGG_LOW_MEM_SIZE_FACTOR_PCT 200
57#define SLAB_GC_AGG_EMERGENCY_SIZE_FACTOR_PCT 250
58#define SLAB_GC_AGG_MAX_SIZE_FACTOR_PCT 300
59static const size_t gc_agg_size_factor_pct[SLAB_GC_FLAG_AGG_COUNT] = {
60 [SLAB_GC_FLAG_AGG_BG] = SLAB_GC_AGG_BG_SIZE_FACTOR_PCT,
61 [SLAB_GC_FLAG_AGG_RECLAIM] = SLAB_GC_AGG_RECLAIM_SIZE_FACTOR_PCT,
62 [SLAB_GC_FLAG_AGG_STANDARD] = SLAB_GC_AGG_STANDARD_SIZE_FACTOR_PCT,
63 [SLAB_GC_FLAG_AGG_LOW_MEM] = SLAB_GC_AGG_LOW_MEM_SIZE_FACTOR_PCT,
64 [SLAB_GC_FLAG_AGG_EMERGENCY] = SLAB_GC_AGG_EMERGENCY_SIZE_FACTOR_PCT,
65 [SLAB_GC_FLAG_AGG_MAX] = SLAB_GC_AGG_MAX_SIZE_FACTOR_PCT,
66};
67
68#define SLAB_GC_AGG_BG_RECYCLE_PENALTY_PCT 75
69#define SLAB_GC_AGG_RECLAIM_RECYCLE_PENALTY_PCT 50
70#define SLAB_GC_AGG_STANDARD_RECYCLE_PENALTY_PCT 50
71#define SLAB_GC_AGG_LOW_MEM_RECYCLE_PENALTY_PCT 25
72#define SLAB_GC_AGG_EMERGENCY_RECYCLE_PENALTY_PCT 0
73#define SLAB_GC_AGG_MAX_RECYCLE_PENALTY_PCT 0
74static const size_t gc_agg_recycle_penalty_pct[SLAB_GC_FLAG_AGG_COUNT] = {
75 [SLAB_GC_FLAG_AGG_BG] = SLAB_GC_AGG_BG_RECYCLE_PENALTY_PCT,
76 [SLAB_GC_FLAG_AGG_RECLAIM] = SLAB_GC_AGG_RECLAIM_RECYCLE_PENALTY_PCT,
77 [SLAB_GC_FLAG_AGG_STANDARD] = SLAB_GC_AGG_STANDARD_RECYCLE_PENALTY_PCT,
78 [SLAB_GC_FLAG_AGG_LOW_MEM] = SLAB_GC_AGG_LOW_MEM_RECYCLE_PENALTY_PCT,
79 [SLAB_GC_FLAG_AGG_EMERGENCY] = SLAB_GC_AGG_EMERGENCY_RECYCLE_PENALTY_PCT,
80 [SLAB_GC_FLAG_AGG_MAX] = SLAB_GC_AGG_MAX_RECYCLE_PENALTY_PCT,
81};
82
83#define SLAB_GC_AGG_BG_MAX_UNFIT 1
84#define SLAB_GC_AGG_RECLAIM_MAX_UNFIT 32
85#define SLAB_GC_AGG_STANDARD_MAX_UNFIT 16
86#define SLAB_GC_AGG_LOW_MEM_MAX_UNFIT 8
87#define SLAB_GC_AGG_EMERGENCY_MAX_UNFIT 4
88#define SLAB_GC_AGG_MAX_MAX_UNFIT 1
89static const size_t gc_agg_max_unfit_slabs[SLAB_GC_FLAG_AGG_COUNT] = {
90 [SLAB_GC_FLAG_AGG_BG] = SLAB_GC_AGG_BG_MAX_UNFIT,
91 [SLAB_GC_FLAG_AGG_RECLAIM] = SLAB_GC_AGG_RECLAIM_MAX_UNFIT,
92 [SLAB_GC_FLAG_AGG_STANDARD] = SLAB_GC_AGG_STANDARD_MAX_UNFIT,
93 [SLAB_GC_FLAG_AGG_LOW_MEM] = SLAB_GC_AGG_LOW_MEM_MAX_UNFIT,
94 [SLAB_GC_FLAG_AGG_EMERGENCY] = SLAB_GC_AGG_EMERGENCY_MAX_UNFIT,
95 [SLAB_GC_FLAG_AGG_MAX] = SLAB_GC_AGG_MAX_MAX_UNFIT,
96};
97