Skip to content

Commit

Permalink
[MIPS] Cleanup memory barriers for weakly ordered systems.
Browse files Browse the repository at this point in the history
Also the R4000 / R4600 LL/SC instructions imply a sync so no explicit sync
needed.

Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
  • Loading branch information
Ralf Baechle committed Dec 4, 2006
1 parent 08f57f7 commit 0004a9d
Show file tree
Hide file tree
Showing 8 changed files with 225 additions and 212 deletions.
4 changes: 4 additions & 0 deletions arch/mips/Kconfig
Original file line number Diff line number Diff line change
Expand Up @@ -1277,6 +1277,7 @@ config CPU_RM9000
select CPU_SUPPORTS_32BIT_KERNEL
select CPU_SUPPORTS_64BIT_KERNEL
select CPU_SUPPORTS_HIGHMEM
select WEAK_ORDERING

config CPU_SB1
bool "SB1"
Expand All @@ -1285,6 +1286,7 @@ config CPU_SB1
select CPU_SUPPORTS_32BIT_KERNEL
select CPU_SUPPORTS_64BIT_KERNEL
select CPU_SUPPORTS_HIGHMEM
select WEAK_ORDERING

endchoice

Expand Down Expand Up @@ -1345,6 +1347,8 @@ config SYS_HAS_CPU_RM9000
config SYS_HAS_CPU_SB1
bool

config WEAK_ORDERING
bool
endmenu

#
Expand Down
6 changes: 3 additions & 3 deletions arch/mips/kernel/smp.c
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ int smp_call_function (void (*func) (void *info), void *info, int retry,

spin_lock(&smp_call_lock);
call_data = &data;
mb();
smp_mb();

/* Send a message to all other CPUs and wait for them to respond */
for_each_online_cpu(i)
Expand Down Expand Up @@ -204,7 +204,7 @@ void smp_call_function_interrupt(void)
* Notify initiating CPU that I've grabbed the data and am
* about to execute the function.
*/
mb();
smp_mb();
atomic_inc(&call_data->started);

/*
Expand All @@ -215,7 +215,7 @@ void smp_call_function_interrupt(void)
irq_exit();

if (wait) {
mb();
smp_mb();
atomic_inc(&call_data->finished);
}
}
Expand Down
37 changes: 25 additions & 12 deletions include/asm-mips/atomic.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#define _ASM_ATOMIC_H

#include <linux/irqflags.h>
#include <asm/barrier.h>
#include <asm/cpu-features.h>
#include <asm/war.h>

Expand Down Expand Up @@ -130,6 +131,8 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
{
unsigned long result;

smp_mb();

if (cpu_has_llsc && R10000_LLSC_WAR) {
unsigned long temp;

Expand All @@ -140,7 +143,6 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
" sc %0, %2 \n"
" beqzl %0, 1b \n"
" addu %0, %1, %3 \n"
" sync \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
Expand All @@ -155,7 +157,6 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
" sc %0, %2 \n"
" beqz %0, 1b \n"
" addu %0, %1, %3 \n"
" sync \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
Expand All @@ -170,13 +171,17 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
local_irq_restore(flags);
}

smp_mb();

return result;
}

static __inline__ int atomic_sub_return(int i, atomic_t * v)
{
unsigned long result;

smp_mb();

if (cpu_has_llsc && R10000_LLSC_WAR) {
unsigned long temp;

Expand All @@ -187,7 +192,6 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
" sc %0, %2 \n"
" beqzl %0, 1b \n"
" subu %0, %1, %3 \n"
" sync \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
Expand All @@ -202,7 +206,6 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
" sc %0, %2 \n"
" beqz %0, 1b \n"
" subu %0, %1, %3 \n"
" sync \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
Expand All @@ -217,6 +220,8 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
local_irq_restore(flags);
}

smp_mb();

return result;
}

Expand All @@ -232,6 +237,8 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
{
unsigned long result;

smp_mb();

if (cpu_has_llsc && R10000_LLSC_WAR) {
unsigned long temp;

Expand All @@ -245,7 +252,6 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
" beqzl %0, 1b \n"
" subu %0, %1, %3 \n"
" .set reorder \n"
" sync \n"
"1: \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
Expand All @@ -264,7 +270,6 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
" beqz %0, 1b \n"
" subu %0, %1, %3 \n"
" .set reorder \n"
" sync \n"
"1: \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
Expand All @@ -281,6 +286,8 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
local_irq_restore(flags);
}

smp_mb();

return result;
}

Expand Down Expand Up @@ -484,6 +491,8 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
{
unsigned long result;

smp_mb();

if (cpu_has_llsc && R10000_LLSC_WAR) {
unsigned long temp;

Expand All @@ -494,7 +503,6 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
" scd %0, %2 \n"
" beqzl %0, 1b \n"
" addu %0, %1, %3 \n"
" sync \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
Expand All @@ -509,7 +517,6 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
" scd %0, %2 \n"
" beqz %0, 1b \n"
" addu %0, %1, %3 \n"
" sync \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
Expand All @@ -524,13 +531,17 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
local_irq_restore(flags);
}

smp_mb();

return result;
}

static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
{
unsigned long result;

smp_mb();

if (cpu_has_llsc && R10000_LLSC_WAR) {
unsigned long temp;

Expand All @@ -541,7 +552,6 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
" scd %0, %2 \n"
" beqzl %0, 1b \n"
" subu %0, %1, %3 \n"
" sync \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
Expand All @@ -556,7 +566,6 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
" scd %0, %2 \n"
" beqz %0, 1b \n"
" subu %0, %1, %3 \n"
" sync \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
Expand All @@ -571,6 +580,8 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
local_irq_restore(flags);
}

smp_mb();

return result;
}

Expand All @@ -586,6 +597,8 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
{
unsigned long result;

smp_mb();

if (cpu_has_llsc && R10000_LLSC_WAR) {
unsigned long temp;

Expand All @@ -599,7 +612,6 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
" beqzl %0, 1b \n"
" dsubu %0, %1, %3 \n"
" .set reorder \n"
" sync \n"
"1: \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
Expand All @@ -618,7 +630,6 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
" beqz %0, 1b \n"
" dsubu %0, %1, %3 \n"
" .set reorder \n"
" sync \n"
"1: \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
Expand All @@ -635,6 +646,8 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
local_irq_restore(flags);
}

smp_mb();

return result;
}

Expand Down
132 changes: 132 additions & 0 deletions include/asm-mips/barrier.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/*
* This file is subject to the terms and conditions of the GNU General Public
* License. See the file "COPYING" in the main directory of this archive
* for more details.
*
* Copyright (C) 2006 by Ralf Baechle (ralf@linux-mips.org)
*/
#ifndef __ASM_BARRIER_H
#define __ASM_BARRIER_H

/*
* read_barrier_depends - Flush all pending reads that subsequents reads
* depend on.
*
* No data-dependent reads from memory-like regions are ever reordered
* over this barrier. All reads preceding this primitive are guaranteed
* to access memory (but not necessarily other CPUs' caches) before any
* reads following this primitive that depend on the data return by
* any of the preceding reads. This primitive is much lighter weight than
* rmb() on most CPUs, and is never heavier weight than is
* rmb().
*
* These ordering constraints are respected by both the local CPU
* and the compiler.
*
* Ordering is not guaranteed by anything other than these primitives,
* not even by data dependencies. See the documentation for
* memory_barrier() for examples and URLs to more information.
*
* For example, the following code would force ordering (the initial
* value of "a" is zero, "b" is one, and "p" is "&a"):
*
* <programlisting>
* CPU 0 CPU 1
*
* b = 2;
* memory_barrier();
* p = &b; q = p;
* read_barrier_depends();
* d = *q;
* </programlisting>
*
* because the read of "*q" depends on the read of "p" and these
* two reads are separated by a read_barrier_depends(). However,
* the following code, with the same initial values for "a" and "b":
*
* <programlisting>
* CPU 0 CPU 1
*
* a = 2;
* memory_barrier();
* b = 3; y = b;
* read_barrier_depends();
* x = a;
* </programlisting>
*
* does not enforce ordering, since there is no data dependency between
* the read of "a" and the read of "b". Therefore, on some CPUs, such
* as Alpha, "y" could be set to 3 and "x" to 0. Use rmb()
* in cases like this where there are no data dependencies.
*/

#define read_barrier_depends() do { } while(0)
#define smp_read_barrier_depends() do { } while(0)

#ifdef CONFIG_CPU_HAS_SYNC
#define __sync() \
__asm__ __volatile__( \
".set push\n\t" \
".set noreorder\n\t" \
".set mips2\n\t" \
"sync\n\t" \
".set pop" \
: /* no output */ \
: /* no input */ \
: "memory")
#else
#define __sync() do { } while(0)
#endif

#define __fast_iob() \
__asm__ __volatile__( \
".set push\n\t" \
".set noreorder\n\t" \
"lw $0,%0\n\t" \
"nop\n\t" \
".set pop" \
: /* no output */ \
: "m" (*(int *)CKSEG1) \
: "memory")

#define fast_wmb() __sync()
#define fast_rmb() __sync()
#define fast_mb() __sync()
#define fast_iob() \
do { \
__sync(); \
__fast_iob(); \
} while (0)

#ifdef CONFIG_CPU_HAS_WB

#include <asm/wbflush.h>

#define wmb() fast_wmb()
#define rmb() fast_rmb()
#define mb() wbflush()
#define iob() wbflush()

#else /* !CONFIG_CPU_HAS_WB */

#define wmb() fast_wmb()
#define rmb() fast_rmb()
#define mb() fast_mb()
#define iob() fast_iob()

#endif /* !CONFIG_CPU_HAS_WB */

#if defined(CONFIG_WEAK_ORDERING) && defined(CONFIG_SMP)
#define __WEAK_ORDERING_MB " sync \n"
#else
#define __WEAK_ORDERING_MB " \n"
#endif

#define smp_mb() __asm__ __volatile__(__WEAK_ORDERING_MB : : :"memory")
#define smp_rmb() __asm__ __volatile__(__WEAK_ORDERING_MB : : :"memory")
#define smp_wmb() __asm__ __volatile__(__WEAK_ORDERING_MB : : :"memory")

#define set_mb(var, value) \
do { var = value; smp_mb(); } while (0)

#endif /* __ASM_BARRIER_H */
Loading

0 comments on commit 0004a9d

Please sign in to comment.