1#include "internal.h"
2
3/* Free_queue function naming semantics:
4 *
5 * "Draining" is removing elements one by one, and each element
6 * tries to get put on a per-core cache's mags. The free queue elements
7 * that don't fit are freed from the slab cache or re-enqueued
8 *
9 * "Flushing" is when the elements are all freed
10 * to the slab cache/page allocator */
11
12void slab_free_queue_init(struct slab_domain *domain, struct slab_free_queue *q,
13 size_t capacity) {
14 if (!mpmc_queue_init(q: &q->mpmc, capacity)) {
15 panic("Could not allocate slab free queue slots!");
16 }
17
18 q->parent = domain;
19 q->count = 0;
20}
21
22bool slab_free_queue_ringbuffer_enqueue(struct slab_free_queue *q,
23 vaddr_t addr) {
24 if (mpmc_queue_enqueue_uintptr(q: &q->mpmc, val: addr)) {
25 SLAB_FREE_QUEUE_INC_COUNT(q);
26 return true;
27 }
28 return false;
29}
30
31vaddr_t slab_free_queue_ringbuffer_dequeue(struct slab_free_queue *q) {
32 uintptr_t addr = 0;
33 if (mpmc_queue_dequeue_uintptr(q: &q->mpmc, out_val: &addr)) {
34 SLAB_FREE_QUEUE_DEC_COUNT(q);
35 return (vaddr_t) addr;
36 }
37 return 0x0;
38}
39
40bool slab_free_queue_enqueue(struct slab_free_queue *q, vaddr_t addr) {
41 return slab_free_queue_ringbuffer_enqueue(q, addr);
42}
43
44vaddr_t slab_free_queue_dequeue(struct slab_free_queue *q) {
45 return slab_free_queue_ringbuffer_dequeue(q);
46}
47
48static void slab_free_queue_free(struct slab_domain *d, void *ptr,
49 enum alloc_behavior bh) {
50 int32_t class = slab_size_to_index(size: ksize(ptr));
51 bool fits_in_slab = class >= 0;
52
53 if (fits_in_slab)
54 return slab_free(domain: d, obj: ptr);
55
56 struct slab_page_hdr *header = slab_page_hdr_for_addr(ptr);
57 return slab_free_page_hdr(hdr: header, bh);
58}
59
60size_t slab_free_queue_drain(struct slab_percpu_cache *cache,
61 struct slab_free_queue *queue, size_t target,
62 enum alloc_behavior bh) {
63 kassert(cache == slab_percpu_cache_local());
64 size_t drained_to_magazine = 0; /* Return value */
65 size_t addrs_dequeued = 0; /* Used to check against `target` */
66
67 while (true) {
68 if (addrs_dequeued >= target)
69 break;
70
71 /* Drain an element from our free_queue */
72 vaddr_t addr = slab_free_queue_dequeue(q: queue);
73 if (!addr)
74 break;
75
76 addrs_dequeued++;
77
78 /* What class? */
79 int32_t class = slab_size_to_index(size: slab_allocation_size(addr));
80 if (class < 0)
81 goto flush;
82
83 /* Magazines only cache nonpageable addresses */
84 struct slab *slab = slab_for_ptr(ptr: (void *) addr);
85 if (slab_is_pageable(s: slab))
86 goto flush;
87
88 /* Push it onto the magazine */
89 enum slab_magazine_type mtype =
90 slab_is_zeroed(s: slab) ? SLAB_MAGAZINE_ZERO : SLAB_MAGAZINE_NORMAL;
91
92 if (mtype == SLAB_MAGAZINE_ZERO)
93 memset((void *) addr, 0, slab->parent_cache->obj_size);
94
95 struct slab_magazine *mag = &cache->mags[mtype][class];
96 if (!slab_magazine_push(mag, obj: addr))
97 goto flush;
98
99 /* Success - pushed onto magazine */
100 drained_to_magazine++;
101 continue;
102
103 flush:
104 slab_free_queue_free(d: cache->domain, ptr: (void *) addr, bh);
105 }
106
107 return drained_to_magazine;
108}
109
110size_t slab_free_queue_flush(struct slab_domain *domain,
111 struct slab_free_queue *queue,
112 enum alloc_behavior bh) {
113 size_t total_freed = 0;
114
115 /* Drain the ringbuffer one element at a time */
116 while (true) {
117 vaddr_t addr = slab_free_queue_ringbuffer_dequeue(q: queue);
118 if (addr == 0x0)
119 break;
120
121 /* kfree_pages will put page backed allocations in here too,
122 * so we send them to THIS free function, which sorts by class */
123 slab_free_queue_free(d: domain, ptr: (void *) addr, bh);
124 total_freed++;
125 }
126 return total_freed;
127}
128
129size_t slab_free_queue_get_target_drain(struct slab_domain *domain,
130 size_t pct) {
131 size_t slab_domain_cpus = domain->domain->num_cores;
132 size_t total_fq_elems = SLAB_FREE_QUEUE_GET_COUNT(&domain->free_queue);
133 size_t portion = slab_domain_cpus / SLAB_PERCPU_REFILL_PER_CORE_WEIGHT;
134 if (portion == 0)
135 portion = 1;
136
137 return (total_fq_elems / portion) * pct / 100;
138}
139
140size_t slab_free_queue_drain_limited(struct slab_percpu_cache *pc,
141 struct slab_domain *dom, size_t pct,
142 enum alloc_behavior bh) {
143 size_t target = slab_free_queue_get_target_drain(domain: dom, pct);
144
145 /* This will also fill up the magazines for other orders. We set the target
146 * to prevent overly aggressive stealing from the free_queue into our
147 * percpu cache to allow other CPUs in our domain to get their fair share of
148 * what remains in the free_queue in the event that they must also refill */
149 return slab_free_queue_drain(cache: pc, queue: &dom->free_queue, target, bh);
150}
151