summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorShakeel Butt <shakeelb@google.com>2018-04-06 02:21:57 +0300
committerLinus Torvalds <torvalds@linux-foundation.org>2018-04-06 07:36:24 +0300
commitf9e13c0a5a33d1eaec374d6d4dab53a4f72756a0 (patch)
tree25020c64a2d657fe1fd0d002c5b59352e4a3e870
parent1ba586de22909f48db78682ee791e0213aba73ae (diff)
downloadlinux-f9e13c0a5a33d1eaec374d6d4dab53a4f72756a0.tar.xz
slab, slub: skip unnecessary kasan_cache_shutdown()
The kasan quarantine is designed to delay freeing slab objects to catch use-after-free. The quarantine can be large (several percent of machine memory size). When kmem_caches are deleted related objects are flushed from the quarantine but this requires scanning the entire quarantine which can be very slow. We have seen the kernel busily working on this while holding slab_mutex and badly affecting cache_reaper, slabinfo readers and memcg kmem cache creations. It can easily reproduced by following script: yes . | head -1000000 | xargs stat > /dev/null for i in `seq 1 10`; do seq 500 | (cd /cg/memory && xargs mkdir) seq 500 | xargs -I{} sh -c 'echo $BASHPID > \ /cg/memory/{}/tasks && exec stat .' > /dev/null seq 500 | (cd /cg/memory && xargs rmdir) done The busy stack: kasan_cache_shutdown shutdown_cache memcg_destroy_kmem_caches mem_cgroup_css_free css_free_rwork_fn process_one_work worker_thread kthread ret_from_fork This patch is based on the observation that if the kmem_cache to be destroyed is empty then there should not be any objects of this cache in the quarantine. Without the patch the script got stuck for couple of hours. With the patch the script completed within a second. Link: http://lkml.kernel.org/r/20180327230603.54721-1-shakeelb@google.com Signed-off-by: Shakeel Butt <shakeelb@google.com> Reviewed-by: Andrew Morton <akpm@linux-foundation.org> Acked-by: Andrey Ryabinin <aryabinin@virtuozzo.com> Acked-by: Christoph Lameter <cl@linux.com> Cc: Vladimir Davydov <vdavydov.dev@gmail.com> Cc: Alexander Potapenko <glider@google.com> Cc: Greg Thelen <gthelen@google.com> Cc: Dmitry Vyukov <dvyukov@google.com> Cc: Pekka Enberg <penberg@kernel.org> Cc: David Rientjes <rientjes@google.com> Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
-rw-r--r--mm/kasan/kasan.c3
-rw-r--r--mm/slab.c12
-rw-r--r--mm/slab.h1
-rw-r--r--mm/slub.c11
4 files changed, 26 insertions, 1 deletions
diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c
index f7a5e1d1ba87..bc0e68f7dc75 100644
--- a/mm/kasan/kasan.c
+++ b/mm/kasan/kasan.c
@@ -382,7 +382,8 @@ void kasan_cache_shrink(struct kmem_cache *cache)
void kasan_cache_shutdown(struct kmem_cache *cache)
{
- quarantine_remove_cache(cache);
+ if (!__kmem_cache_empty(cache))
+ quarantine_remove_cache(cache);
}
size_t kasan_metadata_size(struct kmem_cache *cache)
diff --git a/mm/slab.c b/mm/slab.c
index fb106e8277b7..e3a9b8e23306 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -2291,6 +2291,18 @@ out:
return nr_freed;
}
+bool __kmem_cache_empty(struct kmem_cache *s)
+{
+ int node;
+ struct kmem_cache_node *n;
+
+ for_each_kmem_cache_node(s, node, n)
+ if (!list_empty(&n->slabs_full) ||
+ !list_empty(&n->slabs_partial))
+ return false;
+ return true;
+}
+
int __kmem_cache_shrink(struct kmem_cache *cachep)
{
int ret = 0;
diff --git a/mm/slab.h b/mm/slab.h
index e8981e811c45..68bdf498da3b 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -166,6 +166,7 @@ static inline slab_flags_t kmem_cache_flags(unsigned int object_size,
SLAB_TEMPORARY | \
SLAB_ACCOUNT)
+bool __kmem_cache_empty(struct kmem_cache *);
int __kmem_cache_shutdown(struct kmem_cache *);
void __kmem_cache_release(struct kmem_cache *);
int __kmem_cache_shrink(struct kmem_cache *);
diff --git a/mm/slub.c b/mm/slub.c
index f48d942a487e..4fb037c98782 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3696,6 +3696,17 @@ static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
discard_slab(s, page);
}
+bool __kmem_cache_empty(struct kmem_cache *s)
+{
+ int node;
+ struct kmem_cache_node *n;
+
+ for_each_kmem_cache_node(s, node, n)
+ if (n->nr_partial || slabs_node(s, node))
+ return false;
+ return true;
+}
+
/*
* Release all resources used by a slab cache.
*/