summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJesper Dangaard Brouer <brouer@redhat.com>2021-04-30 09:01:51 +0300
committerLinus Torvalds <torvalds@linux-foundation.org>2021-04-30 21:20:43 +0300
commitce76f9a1d9a21c2633dcd2a5605f923286e16e1d (patch)
treeac88cdbe2dcb1b12d204bbe2fb0401d98b61d81b
parent0f87d9d30f21390dd71114f30e63038980e6eb3f (diff)
downloadlinux-ce76f9a1d9a21c2633dcd2a5605f923286e16e1d.tar.xz
mm/page_alloc: optimize code layout for __alloc_pages_bulk
Looking at perf-report and ASM-code for __alloc_pages_bulk() it is clear that the code activated is suboptimal. The compiler guesses wrong and places unlikely code at the beginning. Due to the use of WARN_ON_ONCE() macro the UD2 asm instruction is added to the code, which confuse the I-cache prefetcher in the CPU. [mgorman@techsingularity.net: minor changes and rebasing] Link: https://lkml.kernel.org/r/20210325114228.27719-5-mgorman@techsingularity.net Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com> Signed-off-by: Mel Gorman <mgorman@techsingularity.net> Reviewed-by: Alexander Lobakin <alobakin@pm.me> Acked-By: Vlastimil Babka <vbabka@suse.cz> Cc: Alexander Duyck <alexander.duyck@gmail.com> Cc: Christoph Hellwig <hch@infradead.org> Cc: Chuck Lever <chuck.lever@oracle.com> Cc: David Miller <davem@davemloft.net> Cc: Ilias Apalodimas <ilias.apalodimas@linaro.org> Cc: Matthew Wilcox (Oracle) <willy@infradead.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
-rw-r--r--mm/page_alloc.c6
1 files changed, 3 insertions, 3 deletions
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index e240704b5d39..3f2291695b19 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -5042,7 +5042,7 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
unsigned int alloc_flags = ALLOC_WMARK_LOW;
int nr_populated = 0;
- if (WARN_ON_ONCE(nr_pages <= 0))
+ if (unlikely(nr_pages <= 0))
return 0;
/*
@@ -5089,7 +5089,7 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
* If there are no allowed local zones that meets the watermarks then
* try to allocate a single page and reclaim if necessary.
*/
- if (!zone)
+ if (unlikely(!zone))
goto failed;
/* Attempt the batch allocation */
@@ -5107,7 +5107,7 @@ unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
page = __rmqueue_pcplist(zone, ac.migratetype, alloc_flags,
pcp, pcp_list);
- if (!page) {
+ if (unlikely(!page)) {
/* Try and get at least one page */
if (!nr_populated)
goto failed_irq;