1#include <block/block.h>
2#include <block/sched.h>
3#include <console/printf.h>
4#include <mem/alloc.h>
5#include <sync/spinlock.h>
6#include <thread/workqueue.h>
7
8static void try_rq_reorder(struct bio_scheduler *sched) {
9 struct block_device *disk = sched->disk;
10 if (bdev_skip_reorder(disk))
11 return;
12
13 disk->ops->reorder(disk);
14}
15
16static void bio_sched_tick(void *ctx, void *unused) {
17 (void) unused;
18 struct bio_scheduler *sched = ctx;
19
20 mutex_lock(&sched->lock);
21
22 bio_sched_boost_starved(sched);
23 try_rq_reorder(sched);
24 bio_sched_try_early_dispatch(sched);
25
26 if (!sched_is_empty(sched)) {
27 mutex_unlock(&sched->lock);
28 delayed_work_schedule(dwork: &sched->tick_work, delay_ms: sched->disk->ops->tick_ms);
29 } else {
30 sched->defer_pending = false;
31 mutex_unlock(&sched->lock);
32 }
33}
34
35static bool try_early_submit(struct bio_scheduler *sched,
36 struct bio_request *req) {
37 /* disk does not support/need IO scheduling */
38 if (submit_if_skip_sched(sched, req))
39 return true;
40
41 if (submit_if_urgent(sched, req))
42 return true;
43
44 return false;
45}
46
47void bio_sched_enqueue(struct block_device *disk, struct bio_request *req) {
48 kassert(req->disk == disk);
49
50 struct bio_scheduler *sched = disk->scheduler;
51
52 if (try_early_submit(sched, req))
53 return;
54
55 mutex_lock(&sched->lock);
56
57 bio_sched_enqueue_internal(sched, req);
58
59 bio_sched_try_early_dispatch(sched);
60 bio_sched_boost_starved(sched);
61
62 bio_sched_try_coalesce(sched);
63
64 try_rq_reorder(sched);
65
66 if (!sched->defer_pending) {
67 sched->defer_pending = true;
68 mutex_unlock(&sched->lock);
69 delayed_work_schedule(dwork: &sched->tick_work, delay_ms: disk->ops->tick_ms);
70 } else {
71 mutex_unlock(&sched->lock);
72 }
73}
74
75void bio_sched_dequeue(struct block_device *disk, struct bio_request *req,
76 bool already_locked) {
77 struct bio_scheduler *sched = disk->scheduler;
78 if (!already_locked)
79 mutex_lock(&sched->lock);
80
81 bio_sched_dequeue_internal(sched, req);
82
83 if (!already_locked)
84 mutex_unlock(&sched->lock);
85}
86
87struct bio_scheduler *bio_sched_create(struct block_device *disk,
88 struct bio_scheduler_ops *ops) {
89 struct bio_scheduler *sched =
90 kmalloc(sizeof(struct bio_scheduler), ALLOC_FLAGS_ZERO);
91 if (!sched)
92 panic("Could not allocate space for block device IO scheduler");
93
94 for (size_t i = 0; i < BIO_SCHED_LEVELS; i++) {
95 INIT_LIST_HEAD(list: &sched->queues[i].list);
96 }
97 sched->disk = disk;
98 disk->ops = ops;
99 mutex_init(&sched->lock);
100 delayed_work_init(dwork: &sched->tick_work, fn: bio_sched_tick,
101 WORK_ARGS(sched, NULL));
102 return sched;
103}
104