summaryrefslogtreecommitdiff
path: root/arch/s390/include/asm/atomic_ops.h
diff options
context:
space:
mode:
authorMartin Schwidefsky <schwidefsky@de.ibm.com>2017-03-24 19:32:23 +0300
committerMartin Schwidefsky <schwidefsky@de.ibm.com>2017-09-28 08:29:44 +0300
commiteb3b7b848fb3dd00f7a57d633d4ae4d194aa7865 (patch)
treeab510aeb7e5aedc07d1cd0e60c5d430c74e02bd8 /arch/s390/include/asm/atomic_ops.h
parentb96f7d881ad94203e997cd2aa7112d4a06d121ef (diff)
downloadlinux-eb3b7b848fb3dd00f7a57d633d4ae4d194aa7865.tar.xz
s390/rwlock: introduce rwlock wait queueing
Like the common queued rwlock code the s390 implementation uses the queued spinlock code on a spinlock_t embedded in the rwlock_t to achieve the queueing. The encoding of the rwlock_t differs though, the counter field in the rwlock_t is split into two parts. The upper two bytes hold the write bit and the write wait counter, the lower two bytes hold the read counter. The arch_read_lock operation works exactly like the common qrwlock but the enqueue operation for a writer follows a diffent logic. After the failed inline try to get the rwlock in write, the writer first increases the write wait counter, acquires the wait spin_lock for the queueing, and then loops until there are no readers and the write bit is zero. Without the write wait counter a CPU that just released the rwlock could immediately reacquire the lock in the inline code, bypassing all outstanding read and write waiters. For s390 this would cause massive imbalances in favour of writers in case of a contended rwlock. Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Diffstat (limited to 'arch/s390/include/asm/atomic_ops.h')
-rw-r--r--arch/s390/include/asm/atomic_ops.h32
1 files changed, 21 insertions, 11 deletions
diff --git a/arch/s390/include/asm/atomic_ops.h b/arch/s390/include/asm/atomic_ops.h
index ba6d29412344..daec181fa1f7 100644
--- a/arch/s390/include/asm/atomic_ops.h
+++ b/arch/s390/include/asm/atomic_ops.h
@@ -39,19 +39,24 @@ __ATOMIC_OPS(__atomic64_xor, long, "laxg")
#undef __ATOMIC_OPS
#undef __ATOMIC_OP
-static inline void __atomic_add_const(int val, int *ptr)
-{
- asm volatile(
- " asi %[ptr],%[val]\n"
- : [ptr] "+Q" (*ptr) : [val] "i" (val) : "cc");
+#define __ATOMIC_CONST_OP(op_name, op_type, op_string, op_barrier) \
+static inline void op_name(op_type val, op_type *ptr) \
+{ \
+ asm volatile( \
+ op_string " %[ptr],%[val]\n" \
+ op_barrier \
+ : [ptr] "+Q" (*ptr) : [val] "i" (val) : "cc", "memory");\
}
-static inline void __atomic64_add_const(long val, long *ptr)
-{
- asm volatile(
- " agsi %[ptr],%[val]\n"
- : [ptr] "+Q" (*ptr) : [val] "i" (val) : "cc");
-}
+#define __ATOMIC_CONST_OPS(op_name, op_type, op_string) \
+ __ATOMIC_CONST_OP(op_name, op_type, op_string, "\n") \
+ __ATOMIC_CONST_OP(op_name##_barrier, op_type, op_string, "bcr 14,0\n")
+
+__ATOMIC_CONST_OPS(__atomic_add_const, int, "asi")
+__ATOMIC_CONST_OPS(__atomic64_add_const, long, "agsi")
+
+#undef __ATOMIC_CONST_OPS
+#undef __ATOMIC_CONST_OP
#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
@@ -107,6 +112,11 @@ __ATOMIC64_OPS(__atomic64_xor, "xgr")
#undef __ATOMIC64_OPS
+#define __atomic_add_const(val, ptr) __atomic_add(val, ptr)
+#define __atomic_add_const_barrier(val, ptr) __atomic_add(val, ptr)
+#define __atomic64_add_const(val, ptr) __atomic64_add(val, ptr)
+#define __atomic64_add_const_barrier(val, ptr) __atomic64_add(val, ptr)
+
#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
static inline int __atomic_cmpxchg(int *ptr, int old, int new)