summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorUros Bizjak <ubizjak@gmail.com>2023-01-18 18:07:03 +0300
committerAndrew Morton <akpm@linux-foundation.org>2023-02-03 09:50:05 +0300
commit030c6ff6498881ac842c7f6da5eec181202b29f4 (patch)
treef08c6b2f70ed980d645fb671455023fb0e7e59c3
parentf3296f80c8065b26a27eb8ab686d0c1d67880524 (diff)
downloadlinux-030c6ff6498881ac842c7f6da5eec181202b29f4.tar.xz
lib/genalloc: use try_cmpxchg in {set,clear}_bits_ll
Use try_cmpxchg instead of cmpxchg (*ptr, old, new) == old in {set,clear}_bits_ll. x86 CMPXCHG instruction returns success in ZF flag, so this change saves a compare after cmpxchg (and related move instruction in front of cmpxchg). Also, try_cmpxchg implicitly assigns old *ptr value to "old" when cmpxchg fails. Note that the value from *ptr should be read using READ_ONCE to prevent the compiler from merging, refetching or reordering the read. The patch also declares these two functions inline, to ensure inlining. No functional change intended. Link: https://lkml.kernel.org/r/20230118150703.4024-1-ubizjak@gmail.com Signed-off-by: Uros Bizjak <ubizjak@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
-rw-r--r--lib/genalloc.c18
1 files changed, 8 insertions, 10 deletions
diff --git a/lib/genalloc.c b/lib/genalloc.c
index 00fc50d0a640..0c883d6fbd44 100644
--- a/lib/genalloc.c
+++ b/lib/genalloc.c
@@ -40,32 +40,30 @@ static inline size_t chunk_size(const struct gen_pool_chunk *chunk)
return chunk->end_addr - chunk->start_addr + 1;
}
-static int set_bits_ll(unsigned long *addr, unsigned long mask_to_set)
+static inline int
+set_bits_ll(unsigned long *addr, unsigned long mask_to_set)
{
- unsigned long val, nval;
+ unsigned long val = READ_ONCE(*addr);
- nval = *addr;
do {
- val = nval;
if (val & mask_to_set)
return -EBUSY;
cpu_relax();
- } while ((nval = cmpxchg(addr, val, val | mask_to_set)) != val);
+ } while (!try_cmpxchg(addr, &val, val | mask_to_set));
return 0;
}
-static int clear_bits_ll(unsigned long *addr, unsigned long mask_to_clear)
+static inline int
+clear_bits_ll(unsigned long *addr, unsigned long mask_to_clear)
{
- unsigned long val, nval;
+ unsigned long val = READ_ONCE(*addr);
- nval = *addr;
do {
- val = nval;
if ((val & mask_to_clear) != mask_to_clear)
return -EBUSY;
cpu_relax();
- } while ((nval = cmpxchg(addr, val, val & ~mask_to_clear)) != val);
+ } while (!try_cmpxchg(addr, &val, val & ~mask_to_clear));
return 0;
}