1#include <block/block.h>
2#include <block/sched.h>
3#include <console/printf.h>
4#include <mem/alloc.h>
5#include <stdint.h>
6#include <structures/dll.h>
7#include <sync/spinlock.h>
8#include <thread/workqueue.h>
9
10/* enqueuing skips enqueuing if the req is URGENT */
11
12static inline bool should_early_dispatch(struct bio_scheduler *sched) {
13 return sched->total_requests > sched->disk->ops->dispatch_threshold;
14}
15
16static bool try_dispatch_queue_head(struct bio_scheduler *sched,
17 struct bio_rqueue *q)
18 TSA_CALLER_HOLDS(&sched->lock) {
19 if (list_head_empty(h: &q->list))
20 return false;
21
22 struct bio_request *head =
23 list_first_entry(&q->list, struct bio_request, list);
24
25 if (head) {
26 bio_sched_dequeue_internal(sched, req: head);
27 sched->disk->submit_bio_async(sched->disk, head);
28 return true;
29 }
30 return false;
31}
32
33static void dispatch_queue(struct block_device *disk, struct bio_rqueue *q) {
34 struct mutex *lock = &disk->scheduler->lock;
35 mutex_lock(lock);
36
37 /* Move the entire list out of the queue under lock */
38 struct list_head tmp_list;
39 list_head_init(h: &tmp_list);
40 list_head_splice_back_init(dst: &tmp_list, src: &q->list);
41
42 /* Reset the queue */
43 list_head_init(h: &q->list);
44 disk->scheduler->total_requests -= q->request_count;
45 q->request_count = 0;
46
47 mutex_unlock(lock);
48
49 /* Dispatch requests from the copied list */
50 struct bio_request *req, *tmp;
51 list_for_each_entry_safe(req, tmp, &tmp_list, list) {
52 list_node_delete(n: &req->list); /* remove from tmp_list */
53 disk->submit_bio_async(disk, req);
54 }
55}
56
57static void do_early_dispatch(struct bio_scheduler *sched)
58 TSA_CALLER_HOLDS(&sched->lock) {
59 for (int prio = 0; prio < BIO_SCHED_LEVELS; prio++)
60 if (try_dispatch_queue_head(sched, q: &sched->queues[prio]))
61 return;
62}
63
64void bio_sched_try_early_dispatch(struct bio_scheduler *sched) {
65 MUTEX_ASSERT_HELD(&sched->lock);
66 if (should_early_dispatch(sched))
67 do_early_dispatch(sched);
68}
69
70void bio_sched_dispatch_partial(struct block_device *d,
71 enum bio_request_priority p) {
72 /* no one in urgent queue */
73 for (int32_t i = BIO_RQ_HIGH; i >= (int32_t) p; i--) {
74 dispatch_queue(disk: d, q: &d->scheduler->queues[i]);
75 }
76}
77
78void bio_sched_dispatch_all(struct block_device *d) {
79 bio_sched_dispatch_partial(d, p: BIO_RQ_BACKGROUND);
80}
81