summaryrefslogtreecommitdiff
path: root/include
diff options
context:
space:
mode:
authorJens Axboe <axboe@fb.com>2015-04-18 01:15:18 +0300
committerJens Axboe <axboe@fb.com>2015-05-05 22:32:47 +0300
commitc4cf5261f8bffd9de132b50660a69148e7575bd6 (patch)
treec3ff2d0d2db5f7392e8651b4f0d2958895faf8b2 /include
parentd9cee5d4f66ef36f69b0108dedbad7f7009bb6a8 (diff)
downloadlinux-c4cf5261f8bffd9de132b50660a69148e7575bd6.tar.xz
bio: skip atomic inc/dec of ->bi_remaining for non-chains
Struct bio has an atomic ref count for chained bio's, and we use this to know when to end IO on the bio. However, most bio's are not chained, so we don't need to always introduce this atomic operation as part of ending IO. Add a helper to elevate the bi_remaining count, and flag the bio as now actually needing the decrement at end_io time. Rename the field to __bi_remaining to catch any current users of this doing the incrementing manually. For high IOPS workloads, this reduces the overhead of bio_endio() substantially. Tested-by: Robert Elliott <elliott@hp.com> Acked-by: Kent Overstreet <kent.overstreet@gmail.com> Reviewed-by: Jan Kara <jack@suse.cz> Signed-off-by: Jens Axboe <axboe@fb.com>
Diffstat (limited to 'include')
-rw-r--r--include/linux/bio.h11
-rw-r--r--include/linux/blk_types.h3
2 files changed, 13 insertions, 1 deletions
diff --git a/include/linux/bio.h b/include/linux/bio.h
index da3a127c9958..8bfe9eee6d1a 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -645,6 +645,17 @@ static inline struct bio *bio_list_get(struct bio_list *bl)
}
/*
+ * Increment chain count for the bio. Make sure the CHAIN flag update
+ * is visible before the raised count.
+ */
+static inline void bio_inc_remaining(struct bio *bio)
+{
+ bio->bi_flags |= (1 << BIO_CHAIN);
+ smp_mb__before_atomic();
+ atomic_inc(&bio->__bi_remaining);
+}
+
+/*
* bio_set is used to allow other portions of the IO system to
* allocate their own private memory pools for bio and iovec structures.
* These memory pools in turn all allocate from the bio_slab
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index a1b25e35ea5f..8b07e0603887 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -65,7 +65,7 @@ struct bio {
unsigned int bi_seg_front_size;
unsigned int bi_seg_back_size;
- atomic_t bi_remaining;
+ atomic_t __bi_remaining;
bio_end_io_t *bi_end_io;
@@ -122,6 +122,7 @@ struct bio {
#define BIO_NULL_MAPPED 8 /* contains invalid user pages */
#define BIO_QUIET 9 /* Make BIO Quiet */
#define BIO_SNAP_STABLE 10 /* bio data must be snapshotted during write */
+#define BIO_CHAIN 11 /* chained bio, ->bi_remaining in effect */
/*
* Flags starting here get preserved by bio_reset() - this includes