summaryrefslogtreecommitdiff
path: root/arch/s390/lib
diff options
context:
space:
mode:
authorMartin Schwidefsky <schwidefsky@de.ibm.com>2015-01-14 19:52:33 +0300
committerMartin Schwidefsky <schwidefsky@de.ibm.com>2015-01-23 17:17:04 +0300
commit2c72a44ecdf2a7ceac73844226e97ed2d5dd1e82 (patch)
treea8475e1252115402a27b4c9a822943782085b151 /arch/s390/lib
parent55b5eb75e7ccdfe94b6ea1be6bba0c21149abecf (diff)
downloadlinux-2c72a44ecdf2a7ceac73844226e97ed2d5dd1e82.tar.xz
s390/spinlock: add compare-and-delay to lock wait loops
Add the compare-and-delay instruction to the spin-lock and rw-lock retry loops. A CPU executing the compare-and-delay instruction stops until the lock value has changed. This is done to make the locking code for contended locks to behave better in regard to the multi- hreading facility. A thread of a core executing a compare-and-delay will allow the other threads of a core to get a larger share of the core resources. Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Diffstat (limited to 'arch/s390/lib')
-rw-r--r--arch/s390/lib/spinlock.c52
1 files changed, 45 insertions, 7 deletions
diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c
index 034a35a3e9c1..d6c9991f7797 100644
--- a/arch/s390/lib/spinlock.c
+++ b/arch/s390/lib/spinlock.c
@@ -12,7 +12,15 @@
#include <linux/smp.h>
#include <asm/io.h>
-int spin_retry = 1000;
+int spin_retry = -1;
+
+static int __init spin_retry_init(void)
+{
+ if (spin_retry < 0)
+ spin_retry = MACHINE_HAS_CAD ? 10 : 1000;
+ return 0;
+}
+early_initcall(spin_retry_init);
/**
* spin_retry= parameter
@@ -24,6 +32,11 @@ static int __init spin_retry_setup(char *str)
}
__setup("spin_retry=", spin_retry_setup);
+static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old)
+{
+ asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock));
+}
+
void arch_spin_lock_wait(arch_spinlock_t *lp)
{
unsigned int cpu = SPINLOCK_LOCKVAL;
@@ -46,6 +59,8 @@ void arch_spin_lock_wait(arch_spinlock_t *lp)
/* Loop for a while on the lock value. */
count = spin_retry;
do {
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&lp->lock, owner);
owner = ACCESS_ONCE(lp->lock);
} while (owner && count-- > 0);
if (!owner)
@@ -84,6 +99,8 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
/* Loop for a while on the lock value. */
count = spin_retry;
do {
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&lp->lock, owner);
owner = ACCESS_ONCE(lp->lock);
} while (owner && count-- > 0);
if (!owner)
@@ -100,11 +117,19 @@ EXPORT_SYMBOL(arch_spin_lock_wait_flags);
int arch_spin_trylock_retry(arch_spinlock_t *lp)
{
+ unsigned int cpu = SPINLOCK_LOCKVAL;
+ unsigned int owner;
int count;
- for (count = spin_retry; count > 0; count--)
- if (arch_spin_trylock_once(lp))
- return 1;
+ for (count = spin_retry; count > 0; count--) {
+ owner = ACCESS_ONCE(lp->lock);
+ /* Try to get the lock if it is free. */
+ if (!owner) {
+ if (_raw_compare_and_swap(&lp->lock, 0, cpu))
+ return 1;
+ } else if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&lp->lock, owner);
+ }
return 0;
}
EXPORT_SYMBOL(arch_spin_trylock_retry);
@@ -126,8 +151,11 @@ void _raw_read_lock_wait(arch_rwlock_t *rw)
}
old = ACCESS_ONCE(rw->lock);
owner = ACCESS_ONCE(rw->owner);
- if ((int) old < 0)
+ if ((int) old < 0) {
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&rw->lock, old);
continue;
+ }
if (_raw_compare_and_swap(&rw->lock, old, old + 1))
return;
}
@@ -141,8 +169,11 @@ int _raw_read_trylock_retry(arch_rwlock_t *rw)
while (count-- > 0) {
old = ACCESS_ONCE(rw->lock);
- if ((int) old < 0)
+ if ((int) old < 0) {
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&rw->lock, old);
continue;
+ }
if (_raw_compare_and_swap(&rw->lock, old, old + 1))
return 1;
}
@@ -173,6 +204,8 @@ void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev)
}
if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
break;
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&rw->lock, old);
}
}
EXPORT_SYMBOL(_raw_write_lock_wait);
@@ -201,6 +234,8 @@ void _raw_write_lock_wait(arch_rwlock_t *rw)
smp_rmb();
if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
break;
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&rw->lock, old);
}
}
EXPORT_SYMBOL(_raw_write_lock_wait);
@@ -214,8 +249,11 @@ int _raw_write_trylock_retry(arch_rwlock_t *rw)
while (count-- > 0) {
old = ACCESS_ONCE(rw->lock);
- if (old)
+ if (old) {
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&rw->lock, old);
continue;
+ }
if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
return 1;
}