1#include <block/block.h>
2#include <block/sched.h>
3#include <console/printf.h>
4#include <mem/alloc.h>
5#include <sync/spinlock.h>
6#include <thread/workqueue.h>
7
8static void try_rq_reorder(struct bio_scheduler *sched) {
9 struct block_device *disk = sched->disk;
10 if (bdev_skip_reorder(disk))
11 return;
12
13 disk->ops->reorder(disk);
14}
15
16static void bio_sched_tick(void *ctx, void *unused) {
17 (void) unused;
18 struct bio_scheduler *sched = ctx;
19
20 mutex_lock(mutex: &sched->lock);
21
22 bio_sched_boost_starved(sched);
23 try_rq_reorder(sched);
24 bio_sched_try_early_dispatch(sched);
25
26 if (!sched_is_empty(sched)) {
27 mutex_unlock(mutex: &sched->lock);
28 defer_enqueue(func: bio_sched_tick, WORK_ARGS(sched, NULL),
29 delay_ms: sched->disk->ops->tick_ms);
30 } else {
31 sched->defer_pending = false;
32 mutex_unlock(mutex: &sched->lock);
33 }
34}
35
36static bool try_early_submit(struct bio_scheduler *sched,
37 struct bio_request *req) {
38 /* disk does not support/need IO scheduling */
39 if (submit_if_skip_sched(sched, req))
40 return true;
41
42 if (submit_if_urgent(sched, req))
43 return true;
44
45 return false;
46}
47
48void bio_sched_enqueue(struct block_device *disk, struct bio_request *req) {
49 kassert(req->disk == disk);
50
51 struct bio_scheduler *sched = disk->scheduler;
52
53 if (try_early_submit(sched, req))
54 return;
55
56 mutex_lock(mutex: &sched->lock);
57
58 bio_sched_enqueue_internal(sched, req);
59
60 bio_sched_try_early_dispatch(sched);
61 bio_sched_boost_starved(sched);
62
63 bio_sched_try_coalesce(sched);
64
65 try_rq_reorder(sched);
66
67 if (!sched->defer_pending) {
68 sched->defer_pending = true;
69 mutex_unlock(mutex: &sched->lock);
70 defer_enqueue(func: bio_sched_tick, WORK_ARGS(sched, NULL),
71 delay_ms: disk->ops->tick_ms);
72 } else {
73 mutex_unlock(mutex: &sched->lock);
74 }
75}
76
77void bio_sched_dequeue(struct block_device *disk, struct bio_request *req,
78 bool already_locked) {
79 struct bio_scheduler *sched = disk->scheduler;
80 if (!already_locked)
81 mutex_lock(mutex: &sched->lock);
82
83 bio_sched_dequeue_internal(sched, req);
84
85 if (!already_locked)
86 mutex_unlock(mutex: &sched->lock);
87}
88
89struct bio_scheduler *bio_sched_create(struct block_device *disk,
90 struct bio_scheduler_ops *ops) {
91 struct bio_scheduler *sched =
92 kmalloc(sizeof(struct bio_scheduler), ALLOC_FLAGS_ZERO);
93 if (!sched)
94 panic("Could not allocate space for block device IO scheduler");
95
96 for (size_t i = 0; i < BIO_SCHED_LEVELS; i++) {
97 INIT_LIST_HEAD(list: &sched->queues[i].list);
98 }
99 sched->disk = disk;
100 disk->ops = ops;
101 mutex_init(mtx: &sched->lock);
102 return sched;
103}
104