s390/spinlock: add compare-and-delay to lock wait loops
Add the compare-and-delay instruction to the spin-lock and rw-lock
retry loops. A CPU executing the compare-and-delay instruction stops
until the lock value has changed. This is done to make the locking
code for contended locks to behave better in regard to the multi-
hreading facility. A thread of a core executing a compare-and-delay
will allow the other threads of a core to get a larger share of the
core resources.
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c
index 034a35a..d6c9991 100644
--- a/arch/s390/lib/spinlock.c
+++ b/arch/s390/lib/spinlock.c
@@ -12,7 +12,15 @@
#include <linux/smp.h>
#include <asm/io.h>
-int spin_retry = 1000;
+int spin_retry = -1;
+
+static int __init spin_retry_init(void)
+{
+ if (spin_retry < 0)
+ spin_retry = MACHINE_HAS_CAD ? 10 : 1000;
+ return 0;
+}
+early_initcall(spin_retry_init);
/**
* spin_retry= parameter
@@ -24,6 +32,11 @@
}
__setup("spin_retry=", spin_retry_setup);
+static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old)
+{
+ asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock));
+}
+
void arch_spin_lock_wait(arch_spinlock_t *lp)
{
unsigned int cpu = SPINLOCK_LOCKVAL;
@@ -46,6 +59,8 @@
/* Loop for a while on the lock value. */
count = spin_retry;
do {
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&lp->lock, owner);
owner = ACCESS_ONCE(lp->lock);
} while (owner && count-- > 0);
if (!owner)
@@ -84,6 +99,8 @@
/* Loop for a while on the lock value. */
count = spin_retry;
do {
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&lp->lock, owner);
owner = ACCESS_ONCE(lp->lock);
} while (owner && count-- > 0);
if (!owner)
@@ -100,11 +117,19 @@
int arch_spin_trylock_retry(arch_spinlock_t *lp)
{
+ unsigned int cpu = SPINLOCK_LOCKVAL;
+ unsigned int owner;
int count;
- for (count = spin_retry; count > 0; count--)
- if (arch_spin_trylock_once(lp))
- return 1;
+ for (count = spin_retry; count > 0; count--) {
+ owner = ACCESS_ONCE(lp->lock);
+ /* Try to get the lock if it is free. */
+ if (!owner) {
+ if (_raw_compare_and_swap(&lp->lock, 0, cpu))
+ return 1;
+ } else if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&lp->lock, owner);
+ }
return 0;
}
EXPORT_SYMBOL(arch_spin_trylock_retry);
@@ -126,8 +151,11 @@
}
old = ACCESS_ONCE(rw->lock);
owner = ACCESS_ONCE(rw->owner);
- if ((int) old < 0)
+ if ((int) old < 0) {
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&rw->lock, old);
continue;
+ }
if (_raw_compare_and_swap(&rw->lock, old, old + 1))
return;
}
@@ -141,8 +169,11 @@
while (count-- > 0) {
old = ACCESS_ONCE(rw->lock);
- if ((int) old < 0)
+ if ((int) old < 0) {
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&rw->lock, old);
continue;
+ }
if (_raw_compare_and_swap(&rw->lock, old, old + 1))
return 1;
}
@@ -173,6 +204,8 @@
}
if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
break;
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&rw->lock, old);
}
}
EXPORT_SYMBOL(_raw_write_lock_wait);
@@ -201,6 +234,8 @@
smp_rmb();
if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
break;
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&rw->lock, old);
}
}
EXPORT_SYMBOL(_raw_write_lock_wait);
@@ -214,8 +249,11 @@
while (count-- > 0) {
old = ACCESS_ONCE(rw->lock);
- if (old)
+ if (old) {
+ if (MACHINE_HAS_CAD)
+ _raw_compare_and_delay(&rw->lock, old);
continue;
+ }
if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
return 1;
}
