summaryrefslogtreecommitdiff
path: root/block
diff options
context:
space:
mode:
authorJens Axboe <axboe@fb.com>2014-05-29 19:53:32 +0400
committerJens Axboe <axboe@fb.com>2014-05-29 19:53:32 +0400
commit05f1dd5315217398fc8d122bdee80f96a9f21274 (patch)
treef1decfafa5f0d974d2f9c707a514bf68b9e2cddb /block
parent4d92a9beb39d80a7d8ff7c04ae12a10290105ae5 (diff)
downloadlinux-05f1dd5315217398fc8d122bdee80f96a9f21274.tar.xz
block: add queue flag for disabling SG merging
If devices are not SG starved, we waste a lot of time potentially collapsing SG segments. Enough that 1.5% of the CPU time goes to this, at only 400K IOPS. Add a queue flag, QUEUE_FLAG_NO_SG_MERGE, which just returns the number of vectors in a bio instead of looping over all segments and checking for collapsible ones. Add a BLK_MQ_F_SG_MERGE flag so that drivers can opt-in on the sg merging, if they so desire. Signed-off-by: Jens Axboe <axboe@fb.com>
Diffstat (limited to 'block')
-rw-r--r--block/blk-merge.c28
-rw-r--r--block/blk-mq.c3
2 files changed, 24 insertions, 7 deletions
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 6c583f9c5b65..b3bf0df0f4c2 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -13,7 +13,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
struct bio *bio)
{
struct bio_vec bv, bvprv = { NULL };
- int cluster, high, highprv = 1;
+ int cluster, high, highprv = 1, no_sg_merge;
unsigned int seg_size, nr_phys_segs;
struct bio *fbio, *bbio;
struct bvec_iter iter;
@@ -35,12 +35,21 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
cluster = blk_queue_cluster(q);
seg_size = 0;
nr_phys_segs = 0;
+ no_sg_merge = test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags);
+ high = 0;
for_each_bio(bio) {
bio_for_each_segment(bv, bio, iter) {
/*
+ * If SG merging is disabled, each bio vector is
+ * a segment
+ */
+ if (no_sg_merge)
+ goto new_segment;
+
+ /*
* the trick here is making sure that a high page is
- * never considered part of another segment, since that
- * might change with the bounce page.
+ * never considered part of another segment, since
+ * that might change with the bounce page.
*/
high = page_to_pfn(bv.bv_page) > queue_bounce_pfn(q);
if (!high && !highprv && cluster) {
@@ -84,11 +93,16 @@ void blk_recalc_rq_segments(struct request *rq)
void blk_recount_segments(struct request_queue *q, struct bio *bio)
{
- struct bio *nxt = bio->bi_next;
+ if (test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags))
+ bio->bi_phys_segments = bio->bi_vcnt;
+ else {
+ struct bio *nxt = bio->bi_next;
+
+ bio->bi_next = NULL;
+ bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio);
+ bio->bi_next = nxt;
+ }
- bio->bi_next = NULL;
- bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio);
- bio->bi_next = nxt;
bio->bi_flags |= (1 << BIO_SEG_VALID);
}
EXPORT_SYMBOL(blk_recount_segments);
diff --git a/block/blk-mq.c b/block/blk-mq.c
index f27fe44230c2..f98d977fd150 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1829,6 +1829,9 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
q->mq_ops = set->ops;
q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
+ if (!(set->flags & BLK_MQ_F_SG_MERGE))
+ q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
+
q->sg_reserved_size = INT_MAX;
INIT_WORK(&q->requeue_work, blk_mq_requeue_work);