summaryrefslogtreecommitdiff
path: root/include/linux/blk-cgroup.h
diff options
context:
space:
mode:
authorTejun Heo <tj@kernel.org>2015-08-19 00:55:21 +0300
committerJens Axboe <axboe@fb.com>2015-08-19 01:49:17 +0300
commite6269c44546755094979ab53609e6e203a68c8ff (patch)
treefc083e47b6ea842e5d260084542a2ed5f76a86ea /include/linux/blk-cgroup.h
parentae11889636111199dbcf47283b4167f578b69472 (diff)
downloadlinux-e6269c44546755094979ab53609e6e203a68c8ff.tar.xz
blkcg: add blkg_[rw]stat->aux_cnt and replace cfq_group->dead_stats with it
cgroup stats are local to each cgroup and doesn't propagate to ancestors by default. When recursive stats are necessary, the sum is calculated over all the descendants. This initially was for backward compatibility to support both group-local and recursive stats but this mode of operation makes general sense as stat update is much hotter thafn reporting those stats. This however ends up losing recursive stats when a child is removed. To work around this, cfq-iosched adds its stats to its parent cfq_group->dead_stats which is summed up together when calculating recursive stats. It's planned that the core stats will be moved to blkcg_gq, so we want to move the mechanism for keeping track of the stats of dead children from cfq to blkcg core. This patch adds blkg_[rw]stat->aux_cnt which are atomic64_t's keeping track of auxiliary counts which are excluded when reading local counts but included for recursive. blkg_[rw]stat_merge() which were used by cfq to implement dead_stats are replaced by blkg_[rw]stat_add_aux(), and cfq now forwards stats of a dead cgroup to the aux counts of parent->stats instead of separate ->dead_stats. This will also help making blkg_[rw]stats per-cpu. Signed-off-by: Tejun Heo <tj@kernel.org> Cc: Vivek Goyal <vgoyal@redhat.com> Signed-off-by: Jens Axboe <axboe@fb.com>
Diffstat (limited to 'include/linux/blk-cgroup.h')
-rw-r--r--include/linux/blk-cgroup.h46
1 files changed, 33 insertions, 13 deletions
diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h
index 4d1659c7f84b..e8092276af58 100644
--- a/include/linux/blk-cgroup.h
+++ b/include/linux/blk-cgroup.h
@@ -53,14 +53,20 @@ struct blkcg {
#endif
};
+/*
+ * blkg_[rw]stat->aux_cnt is excluded for local stats but included for
+ * recursive. Used to carry stats of dead children.
+ */
struct blkg_stat {
struct u64_stats_sync syncp;
uint64_t cnt;
+ atomic64_t aux_cnt;
};
struct blkg_rwstat {
struct u64_stats_sync syncp;
uint64_t cnt[BLKG_RWSTAT_NR];
+ atomic64_t aux_cnt[BLKG_RWSTAT_NR];
};
/*
@@ -483,6 +489,7 @@ struct request_list *__blk_queue_next_rl(struct request_list *rl,
static inline void blkg_stat_init(struct blkg_stat *stat)
{
u64_stats_init(&stat->syncp);
+ atomic64_set(&stat->aux_cnt, 0);
}
/**
@@ -504,8 +511,9 @@ static inline void blkg_stat_add(struct blkg_stat *stat, uint64_t val)
* blkg_stat_read - read the current value of a blkg_stat
* @stat: blkg_stat to read
*
- * Read the current value of @stat. This function can be called without
- * synchroniztion and takes care of u64 atomicity.
+ * Read the current value of @stat. The returned value doesn't include the
+ * aux count. This function can be called without synchroniztion and takes
+ * care of u64 atomicity.
*/
static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
{
@@ -527,23 +535,31 @@ static inline uint64_t blkg_stat_read(struct blkg_stat *stat)
static inline void blkg_stat_reset(struct blkg_stat *stat)
{
stat->cnt = 0;
+ atomic64_set(&stat->aux_cnt, 0);
}
/**
- * blkg_stat_merge - merge a blkg_stat into another
+ * blkg_stat_add_aux - add a blkg_stat into another's aux count
* @to: the destination blkg_stat
* @from: the source
*
- * Add @from's count to @to.
+ * Add @from's count including the aux one to @to's aux count.
*/
-static inline void blkg_stat_merge(struct blkg_stat *to, struct blkg_stat *from)
+static inline void blkg_stat_add_aux(struct blkg_stat *to,
+ struct blkg_stat *from)
{
- blkg_stat_add(to, blkg_stat_read(from));
+ atomic64_add(blkg_stat_read(from) + atomic64_read(&from->aux_cnt),
+ &to->aux_cnt);
}
static inline void blkg_rwstat_init(struct blkg_rwstat *rwstat)
{
+ int i;
+
u64_stats_init(&rwstat->syncp);
+
+ for (i = 0; i < BLKG_RWSTAT_NR; i++)
+ atomic64_set(&rwstat->aux_cnt[i], 0);
}
/**
@@ -614,26 +630,30 @@ static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
*/
static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
{
+ int i;
+
memset(rwstat->cnt, 0, sizeof(rwstat->cnt));
+
+ for (i = 0; i < BLKG_RWSTAT_NR; i++)
+ atomic64_set(&rwstat->aux_cnt[i], 0);
}
/**
- * blkg_rwstat_merge - merge a blkg_rwstat into another
+ * blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count
* @to: the destination blkg_rwstat
* @from: the source
*
- * Add @from's counts to @to.
+ * Add @from's count including the aux one to @to's aux count.
*/
-static inline void blkg_rwstat_merge(struct blkg_rwstat *to,
- struct blkg_rwstat *from)
+static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
+ struct blkg_rwstat *from)
{
struct blkg_rwstat v = blkg_rwstat_read(from);
int i;
- u64_stats_update_begin(&to->syncp);
for (i = 0; i < BLKG_RWSTAT_NR; i++)
- to->cnt[i] += v.cnt[i];
- u64_stats_update_end(&to->syncp);
+ atomic64_add(v.cnt[i] + atomic64_read(&from->aux_cnt[i]),
+ &to->aux_cnt[i]);
}
#ifdef CONFIG_BLK_DEV_THROTTLING