summaryrefslogtreecommitdiff
path: root/block/blk-mq.c
diff options
context:
space:
mode:
authorChristoph Hellwig <hch@lst.de>2020-06-11 09:44:49 +0300
committerJens Axboe <axboe@kernel.dk>2020-06-24 18:15:57 +0300
commit963395269c758641e1cb7208f3bdce6824ea608d (patch)
tree4a301bef2a2ddcaa786764d21cd9c65c375cc5cf /block/blk-mq.c
parent4c8fc19686dc761f600833fc9b8fa390eaf73dd5 (diff)
downloadlinux-963395269c758641e1cb7208f3bdce6824ea608d.tar.xz
blk-mq: factor out a blk_mq_complete_need_ipi helper
Add a helper to decide if we can complete locally or need an IPI. Reviewed-by: Daniel Wagner <dwagner@suse.de> Signed-off-by: Christoph Hellwig <hch@lst.de> Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'block/blk-mq.c')
-rw-r--r--block/blk-mq.c39
1 files changed, 21 insertions, 18 deletions
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 95125bfe779b..961635b40999 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -654,6 +654,24 @@ static void __blk_mq_complete_request_remote(void *data)
__blk_mq_complete_request(data);
}
+static inline bool blk_mq_complete_need_ipi(struct request *rq)
+{
+ int cpu = raw_smp_processor_id();
+
+ if (!IS_ENABLED(CONFIG_SMP) ||
+ !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
+ return false;
+
+ /* same CPU or cache domain? Complete locally */
+ if (cpu == rq->mq_ctx->cpu ||
+ (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
+ cpus_share_cache(cpu, rq->mq_ctx->cpu)))
+ return false;
+
+ /* don't try to IPI to an offline CPU */
+ return cpu_online(rq->mq_ctx->cpu);
+}
+
/**
* blk_mq_complete_request - end I/O on a request
* @rq: the request being processed
@@ -663,11 +681,6 @@ static void __blk_mq_complete_request_remote(void *data)
**/
void blk_mq_complete_request(struct request *rq)
{
- struct blk_mq_ctx *ctx = rq->mq_ctx;
- struct request_queue *q = rq->q;
- bool shared = false;
- int cpu;
-
WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
/*
@@ -675,25 +688,15 @@ void blk_mq_complete_request(struct request *rq)
* to redirect the completion.
*/
if (rq->cmd_flags & REQ_HIPRI) {
- q->mq_ops->complete(rq);
- return;
- }
-
- if (!IS_ENABLED(CONFIG_SMP) ||
- !test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags)) {
- __blk_mq_complete_request(rq);
+ rq->q->mq_ops->complete(rq);
return;
}
- cpu = raw_smp_processor_id();
- if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags))
- shared = cpus_share_cache(cpu, ctx->cpu);
-
- if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
+ if (blk_mq_complete_need_ipi(rq)) {
rq->csd.func = __blk_mq_complete_request_remote;
rq->csd.info = rq;
rq->csd.flags = 0;
- smp_call_function_single_async(ctx->cpu, &rq->csd);
+ smp_call_function_single_async(rq->mq_ctx->cpu, &rq->csd);
} else {
__blk_mq_complete_request(rq);
}