1#include <block/block.h>
2#include <block/sched.h>
3#include <console/printf.h>
4#include <mem/alloc.h>
5#include <sync/lock_chk_assert.h>
6#include <sync/spinlock.h>
7#include <thread/workqueue.h>
8
9static void try_rq_reorder(struct bio_scheduler *sched) {
10 struct block_device *disk = sched->disk;
11 if (bdev_skip_reorder(disk))
12 return;
13
14 disk->ops->reorder(disk);
15}
16
17static void bio_sched_tick(void *ctx, void *unused) {
18 cc_ignore(unused);
19 struct bio_scheduler *sched = ctx;
20
21 mutex_lock(&sched->lock);
22
23 bio_sched_boost_starved(sched);
24 try_rq_reorder(sched);
25 bio_sched_try_early_dispatch(sched);
26
27 if (!sched_is_empty(sched)) {
28 mutex_unlock(&sched->lock);
29 delayed_work_schedule(dwork: &sched->tick_work, delay_ms: sched->disk->ops->tick_ms);
30 } else {
31 sched->defer_pending = false;
32 mutex_unlock(&sched->lock);
33 }
34}
35
36static bool try_early_submit(struct bio_scheduler *sched,
37 struct bio_request *req) {
38 /* disk does not support/need IO scheduling */
39 if (submit_if_skip_sched(sched, req))
40 return true;
41
42 if (submit_if_urgent(sched, req))
43 return true;
44
45 return false;
46}
47
48void bio_sched_enqueue(struct block_device *disk, struct bio_request *req) {
49 enforce_precond(req->disk == disk);
50
51 struct bio_scheduler *sched = disk->scheduler;
52
53 if (try_early_submit(sched, req))
54 return;
55
56 mutex_lock(&sched->lock);
57
58 bio_sched_enqueue_internal(sched, req);
59
60 bio_sched_try_early_dispatch(sched);
61 bio_sched_boost_starved(sched);
62
63 bio_sched_try_coalesce(sched);
64
65 try_rq_reorder(sched);
66
67 if (!sched->defer_pending) {
68 sched->defer_pending = true;
69 mutex_unlock(&sched->lock);
70 delayed_work_schedule(dwork: &sched->tick_work, delay_ms: disk->ops->tick_ms);
71 } else {
72 mutex_unlock(&sched->lock);
73 }
74}
75
76void bio_sched_dequeue(struct block_device *disk, struct bio_request *req) {
77 struct bio_scheduler *sched = disk->scheduler;
78 mutex_lock(&sched->lock);
79 bio_sched_dequeue_internal(sched, req);
80 mutex_unlock(&sched->lock);
81}
82
83struct bio_scheduler *bio_sched_create(struct block_device *disk,
84 struct bio_scheduler_ops *ops) {
85 struct bio_scheduler *sched = vnew(sched, ALLOC_ZERO);
86 if (!sched)
87 panic(OUT_OF_MEMORY, "block device IO scheduler");
88
89 for (size_t i = 0; i < BIO_SCHED_LEVELS; i++) {
90 list_head_init(h: &sched->queues[i].list);
91 }
92 sched->disk = disk;
93 disk->ops = ops;
94 mutex_init(&sched->lock);
95 delayed_work_init(dwork: &sched->tick_work, fn: bio_sched_tick,
96 WORK_ARGS(sched, NULL));
97 return sched;
98}
99