diff options
Diffstat (limited to 'mm/slub.c')
| -rw-r--r-- | mm/slub.c | 85 |
1 files changed, 81 insertions, 4 deletions
diff --git a/mm/slub.c b/mm/slub.c index 7ec0a965c6a3..7f4bc7027ed5 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -214,11 +214,13 @@ enum track_item { TRACK_ALLOC, TRACK_FREE }; static int sysfs_slab_add(struct kmem_cache *); static int sysfs_slab_alias(struct kmem_cache *, const char *); static void memcg_propagate_slab_attrs(struct kmem_cache *s); +static void sysfs_slab_remove(struct kmem_cache *s); #else static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) { return 0; } static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { } +static inline void sysfs_slab_remove(struct kmem_cache *s) { } #endif static inline void stat(const struct kmem_cache *s, enum stat_item si) @@ -1630,6 +1632,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) flags &= ~GFP_SLAB_BUG_MASK; pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n", invalid_mask, &invalid_mask, flags, &flags); + dump_stack(); } return allocate_slab(s, @@ -3686,6 +3689,7 @@ int __kmem_cache_shutdown(struct kmem_cache *s) if (n->nr_partial || slabs_node(s, node)) return 1; } + sysfs_slab_remove(s); return 0; } @@ -3952,6 +3956,42 @@ int __kmem_cache_shrink(struct kmem_cache *s) return ret; } +#ifdef CONFIG_MEMCG +static void kmemcg_cache_deact_after_rcu(struct kmem_cache *s) +{ + /* + * Called with all the locks held after a sched RCU grace period. + * Even if @s becomes empty after shrinking, we can't know that @s + * doesn't have allocations already in-flight and thus can't + * destroy @s until the associated memcg is released. + * + * However, let's remove the sysfs files for empty caches here. + * Each cache has a lot of interface files which aren't + * particularly useful for empty draining caches; otherwise, we can + * easily end up with millions of unnecessary sysfs files on + * systems which have a lot of memory and transient cgroups. + */ + if (!__kmem_cache_shrink(s)) + sysfs_slab_remove(s); +} + +void __kmemcg_cache_deactivate(struct kmem_cache *s) +{ + /* + * Disable empty slabs caching. Used to avoid pinning offline + * memory cgroups by kmem pages that can be freed. + */ + s->cpu_partial = 0; + s->min_partial = 0; + + /* + * s->cpu_partial is checked locklessly (see put_cpu_partial), so + * we have to make sure the change is visible before shrinking. + */ + slab_deactivate_memcg_cache_rcu_sched(s, kmemcg_cache_deact_after_rcu); +} +#endif + static int slab_mem_going_offline_callback(void *arg) { struct kmem_cache *s; @@ -4108,6 +4148,7 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) } slab_init_memcg_params(s); list_add(&s->list, &slab_caches); + memcg_link_cache(s); return s; } @@ -4667,6 +4708,22 @@ enum slab_stat_type { #define SO_OBJECTS (1 << SL_OBJECTS) #define SO_TOTAL (1 << SL_TOTAL) +#ifdef CONFIG_MEMCG +static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON); + +static int __init setup_slub_memcg_sysfs(char *str) +{ + int v; + + if (get_option(&str, &v) > 0) + memcg_sysfs_enabled = v; + + return 1; +} + +__setup("slub_memcg_sysfs=", setup_slub_memcg_sysfs); +#endif + static ssize_t show_slab_objects(struct kmem_cache *s, char *buf, unsigned long flags) { @@ -5570,8 +5627,14 @@ static int sysfs_slab_add(struct kmem_cache *s) { int err; const char *name; + struct kset *kset = cache_kset(s); int unmergeable = slab_unmergeable(s); + if (!kset) { + kobject_init(&s->kobj, &slab_ktype); + return 0; + } + if (unmergeable) { /* * Slabcache can never be merged so we can use the name proper. @@ -5588,7 +5651,7 @@ static int sysfs_slab_add(struct kmem_cache *s) name = create_unique_id(s); } - s->kobj.kset = cache_kset(s); + s->kobj.kset = kset; err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); if (err) goto out; @@ -5598,7 +5661,7 @@ static int sysfs_slab_add(struct kmem_cache *s) goto out_del_kobj; #ifdef CONFIG_MEMCG - if (is_root_cache(s)) { + if (is_root_cache(s) && memcg_sysfs_enabled) { s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj); if (!s->memcg_kset) { err = -ENOMEM; @@ -5621,7 +5684,7 @@ out_del_kobj: goto out; } -void sysfs_slab_remove(struct kmem_cache *s) +static void sysfs_slab_remove(struct kmem_cache *s) { if (slab_state < FULL) /* @@ -5630,12 +5693,26 @@ void sysfs_slab_remove(struct kmem_cache *s) */ return; + if (!s->kobj.state_in_sysfs) + /* + * For a memcg cache, this may be called during + * deactivation and again on shutdown. Remove only once. + * A cache is never shut down before deactivation is + * complete, so no need to worry about synchronization. + */ + return; + #ifdef CONFIG_MEMCG kset_unregister(s->memcg_kset); #endif kobject_uevent(&s->kobj, KOBJ_REMOVE); kobject_del(&s->kobj); - kobject_put(&s->kobj); +} + +void sysfs_slab_release(struct kmem_cache *s) +{ + if (slab_state >= FULL) + kobject_put(&s->kobj); } /* |