Skip to content

Commit

Permalink
s390/spinlock,rwlock: always to a load-and-test first
Browse files Browse the repository at this point in the history 
In case a lock is contended it is better to do a load-and-test first
before trying to get the lock with compare-and-swap. This helps to avoid
unnecessary cache invalidations of the cacheline for the lock if the
CPU has to wait for the lock. For an uncontended lock doing the
compare-and-swap directly is a bit better, if the CPU does not have the
cacheline in its cache yet the compare-and-swap will get it read-write
immediately while a load-and-test would get it read-only first.

Always to the load-and-test first to avoid the cacheline invalidations
for the contended case outweight the potential read-only to read-write
cacheline upgrade for the uncontended case.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
  • Loading branch information
Martin Schwidefsky committed May 20, 2014
1 parent 2bf29df commit bae8f56
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 33 deletions.
50 changes: 30 additions & 20 deletions arch/s390/include/asm/spinlock.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@ static inline int arch_spin_is_locked(arch_spinlock_t *lp)

static inline int arch_spin_trylock_once(arch_spinlock_t *lp)
{
return _raw_compare_and_swap(&lp->lock, 0, SPINLOCK_LOCKVAL);
barrier();
return likely(arch_spin_value_unlocked(*lp) &&
_raw_compare_and_swap(&lp->lock, 0, SPINLOCK_LOCKVAL));
}

static inline int arch_spin_tryrelease_once(arch_spinlock_t *lp)
Expand All @@ -69,20 +71,20 @@ static inline int arch_spin_tryrelease_once(arch_spinlock_t *lp)

static inline void arch_spin_lock(arch_spinlock_t *lp)
{
if (unlikely(!arch_spin_trylock_once(lp)))
if (!arch_spin_trylock_once(lp))
arch_spin_lock_wait(lp);
}

static inline void arch_spin_lock_flags(arch_spinlock_t *lp,
unsigned long flags)
{
if (unlikely(!arch_spin_trylock_once(lp)))
if (!arch_spin_trylock_once(lp))
arch_spin_lock_wait_flags(lp, flags);
}

static inline int arch_spin_trylock(arch_spinlock_t *lp)
{
if (unlikely(!arch_spin_trylock_once(lp)))
if (!arch_spin_trylock_once(lp))
return arch_spin_trylock_retry(lp);
return 1;
}
Expand Down Expand Up @@ -128,19 +130,29 @@ extern void _raw_write_lock_wait(arch_rwlock_t *lp);
extern void _raw_write_lock_wait_flags(arch_rwlock_t *lp, unsigned long flags);
extern int _raw_write_trylock_retry(arch_rwlock_t *lp);

static inline int arch_read_trylock_once(arch_rwlock_t *rw)
{
unsigned int old = ACCESS_ONCE(rw->lock);
return likely((int) old >= 0 &&
_raw_compare_and_swap(&rw->lock, old, old + 1));
}

static inline int arch_write_trylock_once(arch_rwlock_t *rw)
{
unsigned int old = ACCESS_ONCE(rw->lock);
return likely(old == 0 &&
_raw_compare_and_swap(&rw->lock, 0, 0x80000000));
}

static inline void arch_read_lock(arch_rwlock_t *rw)
{
unsigned int old;
old = rw->lock & 0x7fffffffU;
if (!_raw_compare_and_swap(&rw->lock, old, old + 1))
if (!arch_read_trylock_once(rw))
_raw_read_lock_wait(rw);
}

static inline void arch_read_lock_flags(arch_rwlock_t *rw, unsigned long flags)
{
unsigned int old;
old = rw->lock & 0x7fffffffU;
if (!_raw_compare_and_swap(&rw->lock, old, old + 1))
if (!arch_read_trylock_once(rw))
_raw_read_lock_wait_flags(rw, flags);
}

Expand All @@ -155,13 +167,13 @@ static inline void arch_read_unlock(arch_rwlock_t *rw)

static inline void arch_write_lock(arch_rwlock_t *rw)
{
if (unlikely(!_raw_compare_and_swap(&rw->lock, 0, 0x80000000)))
if (!arch_write_trylock_once(rw))
_raw_write_lock_wait(rw);
}

static inline void arch_write_lock_flags(arch_rwlock_t *rw, unsigned long flags)
{
if (unlikely(!_raw_compare_and_swap(&rw->lock, 0, 0x80000000)))
if (!arch_write_trylock_once(rw))
_raw_write_lock_wait_flags(rw, flags);
}

Expand All @@ -172,18 +184,16 @@ static inline void arch_write_unlock(arch_rwlock_t *rw)

static inline int arch_read_trylock(arch_rwlock_t *rw)
{
unsigned int old;
old = rw->lock & 0x7fffffffU;
if (likely(_raw_compare_and_swap(&rw->lock, old, old + 1)))
return 1;
return _raw_read_trylock_retry(rw);
if (!arch_read_trylock_once(rw))
return _raw_read_trylock_retry(rw);
return 1;
}

static inline int arch_write_trylock(arch_rwlock_t *rw)
{
if (likely(_raw_compare_and_swap(&rw->lock, 0, 0x80000000)))
return 1;
return _raw_write_trylock_retry(rw);
if (!arch_write_trylock_once(rw))
return _raw_write_trylock_retry(rw);
return 1;
}

#define arch_read_relax(lock) cpu_relax()
Expand Down
29 changes: 16 additions & 13 deletions arch/s390/lib/spinlock.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,9 @@ int arch_spin_trylock_retry(arch_spinlock_t *lp)
{
int count;

for (count = spin_retry; count > 0; count--) {
if (arch_spin_is_locked(lp))
continue;
for (count = spin_retry; count > 0; count--)
if (arch_spin_trylock_once(lp))
return 1;
}
return 0;
}
EXPORT_SYMBOL(arch_spin_trylock_retry);
Expand All @@ -120,9 +117,9 @@ void _raw_read_lock_wait(arch_rwlock_t *rw)
smp_yield();
count = spin_retry;
}
if (!arch_read_can_lock(rw))
old = ACCESS_ONCE(rw->lock);
if ((int) old < 0)
continue;
old = rw->lock & 0x7fffffffU;
if (_raw_compare_and_swap(&rw->lock, old, old + 1))
return;
}
Expand All @@ -140,9 +137,9 @@ void _raw_read_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
smp_yield();
count = spin_retry;
}
if (!arch_read_can_lock(rw))
old = ACCESS_ONCE(rw->lock);
if ((int) old < 0)
continue;
old = rw->lock & 0x7fffffffU;
local_irq_disable();
if (_raw_compare_and_swap(&rw->lock, old, old + 1))
return;
Expand All @@ -156,9 +153,9 @@ int _raw_read_trylock_retry(arch_rwlock_t *rw)
int count = spin_retry;

while (count-- > 0) {
if (!arch_read_can_lock(rw))
old = ACCESS_ONCE(rw->lock);
if ((int) old < 0)
continue;
old = rw->lock & 0x7fffffffU;
if (_raw_compare_and_swap(&rw->lock, old, old + 1))
return 1;
}
Expand All @@ -168,14 +165,16 @@ EXPORT_SYMBOL(_raw_read_trylock_retry);

void _raw_write_lock_wait(arch_rwlock_t *rw)
{
unsigned int old;
int count = spin_retry;

while (1) {
if (count-- <= 0) {
smp_yield();
count = spin_retry;
}
if (!arch_write_can_lock(rw))
old = ACCESS_ONCE(rw->lock);
if (old)
continue;
if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
return;
Expand All @@ -185,6 +184,7 @@ EXPORT_SYMBOL(_raw_write_lock_wait);

void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
{
unsigned int old;
int count = spin_retry;

local_irq_restore(flags);
Expand All @@ -193,7 +193,8 @@ void _raw_write_lock_wait_flags(arch_rwlock_t *rw, unsigned long flags)
smp_yield();
count = spin_retry;
}
if (!arch_write_can_lock(rw))
old = ACCESS_ONCE(rw->lock);
if (old)
continue;
local_irq_disable();
if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
Expand All @@ -204,10 +205,12 @@ EXPORT_SYMBOL(_raw_write_lock_wait_flags);

int _raw_write_trylock_retry(arch_rwlock_t *rw)
{
unsigned int old;
int count = spin_retry;

while (count-- > 0) {
if (!arch_write_can_lock(rw))
old = ACCESS_ONCE(rw->lock);
if (old)
continue;
if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
return 1;
Expand Down

0 comments on commit bae8f56

Please sign in to comment.