summaryrefslogtreecommitdiff
path: root/block/blk-mq-sched.c
diff options
context:
space:
mode:
authorMing Lei <ming.lei@redhat.com>2021-01-22 05:33:12 +0300
committerMartin K. Petersen <martin.petersen@oracle.com>2021-03-05 01:36:59 +0300
commit2a5a24aa83382a88c43d18a901fab66e6ffe1199 (patch)
tree85d9a1aa8d76bb25fe9fdc56f9a1b74f575ce485 /block/blk-mq-sched.c
parentd022d18c045fc2ccf92d0f14cf80f98eb0a8e119 (diff)
downloadlinux-2a5a24aa83382a88c43d18a901fab66e6ffe1199.tar.xz
scsi: blk-mq: Return budget token from .get_budget callback
SCSI uses a global atomic variable to track queue depth for each LUN/request queue. This doesn't scale well when there are lots of CPU cores and the disk is very fast. It has been observed that IOPS is affected a lot by tracking queue depth via sdev->device_busy in the I/O path. Return budget token from .get_budget callback. The budget token can be passed to driver so that we can replace the atomic variable with sbitmap_queue and alleviate the scaling problems that way. Link: https://lore.kernel.org/r/20210122023317.687987-9-ming.lei@redhat.com Cc: Omar Sandoval <osandov@fb.com> Cc: Kashyap Desai <kashyap.desai@broadcom.com> Cc: Sumanesh Samanta <sumanesh.samanta@broadcom.com> Cc: Ewan D. Milne <emilne@redhat.com> Tested-by: Sumanesh Samanta <sumanesh.samanta@broadcom.com> Reviewed-by: Hannes Reinecke <hare@suse.de> Signed-off-by: Ming Lei <ming.lei@redhat.com> Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
Diffstat (limited to 'block/blk-mq-sched.c')
-rw-r--r--block/blk-mq-sched.c17
1 files changed, 13 insertions, 4 deletions
diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index ddb65e9e6fd9..712b7f0afd63 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -131,6 +131,7 @@ static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
do {
struct request *rq;
+ int budget_token;
if (e->type->ops.has_work && !e->type->ops.has_work(hctx))
break;
@@ -140,12 +141,13 @@ static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
break;
}
- if (!blk_mq_get_dispatch_budget(q))
+ budget_token = blk_mq_get_dispatch_budget(q);
+ if (budget_token < 0)
break;
rq = e->type->ops.dispatch_request(hctx);
if (!rq) {
- blk_mq_put_dispatch_budget(q);
+ blk_mq_put_dispatch_budget(q, budget_token);
/*
* We're releasing without dispatching. Holding the
* budget could have blocked any "hctx"s with the
@@ -157,6 +159,8 @@ static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
break;
}
+ blk_mq_set_rq_budget_token(rq, budget_token);
+
/*
* Now this rq owns the budget which has to be released
* if this rq won't be queued to driver via .queue_rq()
@@ -230,6 +234,8 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)
struct request *rq;
do {
+ int budget_token;
+
if (!list_empty_careful(&hctx->dispatch)) {
ret = -EAGAIN;
break;
@@ -238,12 +244,13 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)
if (!sbitmap_any_bit_set(&hctx->ctx_map))
break;
- if (!blk_mq_get_dispatch_budget(q))
+ budget_token = blk_mq_get_dispatch_budget(q);
+ if (budget_token < 0)
break;
rq = blk_mq_dequeue_from_ctx(hctx, ctx);
if (!rq) {
- blk_mq_put_dispatch_budget(q);
+ blk_mq_put_dispatch_budget(q, budget_token);
/*
* We're releasing without dispatching. Holding the
* budget could have blocked any "hctx"s with the
@@ -255,6 +262,8 @@ static int blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)
break;
}
+ blk_mq_set_rq_budget_token(rq, budget_token);
+
/*
* Now this rq owns the budget which has to be released
* if this rq won't be queued to driver via .queue_rq()