1#include <block/block.h>
2#include <block/sched.h>
3#include <console/printf.h>
4#include <mem/alloc.h>
5#include <stdint.h>
6#include <structures/dll.h>
7#include <sync/spinlock.h>
8#include <thread/workqueue.h>
9
10/* enqueuing skips enqueuing if the req is URGENT */
11
12static inline bool should_early_dispatch(struct bio_scheduler *sched) {
13 return sched->total_requests > sched->disk->ops->dispatch_threshold;
14}
15
16static bool try_dispatch_queue_head(struct bio_scheduler *sched,
17 struct bio_rqueue *q) {
18 if (list_empty(head: &q->list))
19 return false;
20
21 struct bio_request *head =
22 list_first_entry(&q->list, struct bio_request, list);
23
24 if (head) {
25 bio_sched_dequeue_internal(sched, req: head);
26 sched->disk->submit_bio_async(sched->disk, head);
27 return true;
28 }
29 return false;
30}
31
32static void dispatch_queue(struct block_device *disk, struct bio_rqueue *q) {
33 struct mutex *lock = &disk->scheduler->lock;
34 mutex_lock(mutex: lock);
35
36 /* Move the entire list out of the queue under lock */
37 struct list_head tmp_list;
38 INIT_LIST_HEAD(list: &tmp_list);
39 if (!list_empty(head: &q->list)) {
40 tmp_list.next = q->list.next;
41 tmp_list.prev = q->list.prev;
42 tmp_list.next->prev = &tmp_list;
43 tmp_list.prev->next = &tmp_list;
44 }
45
46 /* Reset the queue */
47 INIT_LIST_HEAD(list: &q->list);
48 disk->scheduler->total_requests -= q->request_count;
49 q->request_count = 0;
50
51 mutex_unlock(mutex: lock);
52
53 /* Dispatch requests from the copied list */
54 struct bio_request *req, *tmp;
55 list_for_each_entry_safe(req, tmp, &tmp_list, list) {
56 list_del(entry: &req->list); /* remove from tmp_list */
57 disk->submit_bio_async(disk, req);
58 }
59}
60
61static void do_early_dispatch(struct bio_scheduler *sched) {
62 for (int prio = 0; prio < BIO_SCHED_LEVELS; prio++)
63 if (try_dispatch_queue_head(sched, q: &sched->queues[prio]))
64 return;
65}
66
67void bio_sched_try_early_dispatch(struct bio_scheduler *sched) {
68 MUTEX_ASSERT_HELD(&sched->lock);
69 if (should_early_dispatch(sched))
70 do_early_dispatch(sched);
71}
72
73void bio_sched_dispatch_partial(struct block_device *d,
74 enum bio_request_priority p) {
75 /* no one in urgent queue */
76 for (uint32_t i = BIO_RQ_HIGH; i > p; i--) {
77 dispatch_queue(disk: d, q: &d->scheduler->queues[i]);
78 }
79}
80
81void bio_sched_dispatch_all(struct block_device *d) {
82 bio_sched_dispatch_partial(d, p: BIO_RQ_BACKGROUND);
83}
84