summaryrefslogtreecommitdiff
path: root/include/linux/bio.h
diff options
context:
space:
mode:
authorJens Axboe <axboe@fb.com>2015-04-18 01:23:59 +0300
committerJens Axboe <axboe@fb.com>2015-05-05 22:32:49 +0300
commitdac56212e8127dbc0bff7be35c508bc280213309 (patch)
treea1a066d7f6deb7d66403137b8edafb394ce0ca6a /include/linux/bio.h
parentc4cf5261f8bffd9de132b50660a69148e7575bd6 (diff)
downloadlinux-dac56212e8127dbc0bff7be35c508bc280213309.tar.xz
bio: skip atomic inc/dec of ->bi_cnt for most use cases
Struct bio has a reference count that controls when it can be freed. Most uses cases is allocating the bio, which then returns with a single reference to it, doing IO, and then dropping that single reference. We can remove this atomic_dec_and_test() in the completion path, if nobody else is holding a reference to the bio. If someone does call bio_get() on the bio, then we flag the bio as now having valid count and that we must properly honor the reference count when it's being put. Tested-by: Robert Elliott <elliott@hp.com> Signed-off-by: Jens Axboe <axboe@fb.com>
Diffstat (limited to 'include/linux/bio.h')
-rw-r--r--include/linux/bio.h16
1 files changed, 15 insertions, 1 deletions
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 8bfe9eee6d1a..7486ea103f6e 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -290,7 +290,21 @@ static inline unsigned bio_segments(struct bio *bio)
* returns. and then bio would be freed memory when if (bio->bi_flags ...)
* runs
*/
-#define bio_get(bio) atomic_inc(&(bio)->bi_cnt)
+static inline void bio_get(struct bio *bio)
+{
+ bio->bi_flags |= (1 << BIO_REFFED);
+ smp_mb__before_atomic();
+ atomic_inc(&bio->__bi_cnt);
+}
+
+static inline void bio_cnt_set(struct bio *bio, unsigned int count)
+{
+ if (count != 1) {
+ bio->bi_flags |= (1 << BIO_REFFED);
+ smp_mb__before_atomic();
+ }
+ atomic_set(&bio->__bi_cnt, count);
+}
enum bip_flags {
BIP_BLOCK_INTEGRITY = 1 << 0, /* block layer owns integrity data */