From e2db1a9aa3814960a56583df39ea71e36d802278 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Thu, 25 Feb 2021 17:19:59 -0800 Subject: kasan, mm: optimize kmalloc poisoning For allocations from kmalloc caches, kasan_kmalloc() always follows kasan_slab_alloc(). Currenly, both of them unpoison the whole object, which is unnecessary. This patch provides separate implementations for both annotations: kasan_slab_alloc() unpoisons the whole object, and kasan_kmalloc() only poisons the redzone. For generic KASAN, the redzone start might not be aligned to KASAN_GRANULE_SIZE. Therefore, the poisoning is split in two parts: kasan_poison_last_granule() poisons the unaligned part, and then kasan_poison() poisons the rest. This patch also clarifies alignment guarantees of each of the poisoning functions and drops the unnecessary round_up() call for redzone_end. With this change, the early SLUB cache annotation needs to be changed to kasan_slab_alloc(), as kasan_kmalloc() doesn't unpoison objects now. The number of poisoned bytes for objects in this cache stays the same, as kmem_cache_node->object_size is equal to sizeof(struct kmem_cache_node). Link: https://lkml.kernel.org/r/7e3961cb52be380bc412860332063f5f7ce10d13.1612546384.git.andreyknvl@google.com Signed-off-by: Andrey Konovalov Reviewed-by: Marco Elver Cc: Alexander Potapenko Cc: Andrey Ryabinin Cc: Branislav Rankov Cc: Catalin Marinas Cc: Dmitry Vyukov Cc: Evgenii Stepanov Cc: Kevin Brodsky Cc: Peter Collingbourne Cc: Vincenzo Frascino Cc: Will Deacon Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/kasan/shadow.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'mm/kasan/shadow.c') diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index 1372a2fc0ca9..1ed7817e4ee6 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -69,10 +69,6 @@ void *memcpy(void *dest, const void *src, size_t len) return __memcpy(dest, src, len); } -/* - * Poisons the shadow memory for 'size' bytes starting from 'addr'. - * Memory addresses should be aligned to KASAN_GRANULE_SIZE. - */ void kasan_poison(const void *address, size_t size, u8 value) { void *shadow_start, *shadow_end; @@ -83,12 +79,12 @@ void kasan_poison(const void *address, size_t size, u8 value) * addresses to this function. */ address = kasan_reset_tag(address); - size = round_up(size, KASAN_GRANULE_SIZE); /* Skip KFENCE memory if called explicitly outside of sl*b. */ if (is_kfence_address(address)) return; + size = round_up(size, KASAN_GRANULE_SIZE); shadow_start = kasan_mem_to_shadow(address); shadow_end = kasan_mem_to_shadow(address + size); @@ -96,6 +92,16 @@ void kasan_poison(const void *address, size_t size, u8 value) } EXPORT_SYMBOL(kasan_poison); +#ifdef CONFIG_KASAN_GENERIC +void kasan_poison_last_granule(const void *address, size_t size) +{ + if (size & KASAN_GRANULE_MASK) { + u8 *shadow = (u8 *)kasan_mem_to_shadow(address + size); + *shadow = size & KASAN_GRANULE_MASK; + } +} +#endif + void kasan_unpoison(const void *address, size_t size) { u8 tag = get_tag(address); @@ -115,16 +121,12 @@ void kasan_unpoison(const void *address, size_t size) if (is_kfence_address(address)) return; + /* Unpoison round_up(size, KASAN_GRANULE_SIZE) bytes. */ kasan_poison(address, size, tag); - if (size & KASAN_GRANULE_MASK) { - u8 *shadow = (u8 *)kasan_mem_to_shadow(address + size); - - if (IS_ENABLED(CONFIG_KASAN_SW_TAGS)) - *shadow = tag; - else /* CONFIG_KASAN_GENERIC */ - *shadow = size & KASAN_GRANULE_MASK; - } + /* Partially poison the last granule for the generic mode. */ + if (IS_ENABLED(CONFIG_KASAN_GENERIC)) + kasan_poison_last_granule(address, size); } #ifdef CONFIG_MEMORY_HOTPLUG -- cgit v1.2.3