Skip to content

Commit

Permalink
MIPS: Get rid of branches to .subsections.
Browse files Browse the repository at this point in the history 
It was a nice optimization - on paper at least.  In practice it results in
branches that may exceed the maximum legal range for a branch.  We can
fight that problem with -ffunction-sections but -ffunction-sections again
is incompatible with -pg used by the function tracer.

By rewriting the loop around all simple LL/SC blocks to C we reduce the
amount of inline assembler and at the same time allow GCC to often fill
the branch delay slots with something sensible or whatever else clever
optimization it may have up in its sleeve.

With this optimization gone we also no longer need -ffunction-sections,
so drop it.

This optimization was originally introduced in 2.6.21, commit
5999eca25c1fd4b9b9aca7833b04d10fe4bc877d (linux-mips.org) rsp.
f65e4fa (kernel.org).

Original fix for the issues which caused me to pull this optimization by
Paul Gortmaker <paul.gortmaker@windriver.com>.

Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
  • Loading branch information
Ralf Baechle committed Oct 29, 2010
1 parent 18cb657 commit 7837314
Show file tree
Hide file tree
Showing 5 changed files with 243 additions and 297 deletions.
3 changes: 0 additions & 3 deletions arch/mips/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,6 @@ ifneq ($(SUBARCH),$(ARCH))
endif
endif

ifndef CONFIG_FUNCTION_TRACER
cflags-y := -ffunction-sections
endif
ifdef CONFIG_FUNCTION_GRAPH_TRACER
ifndef KBUILD_MCOUNT_RA_ADDRESS
ifeq ($(call cc-option-yn,-mmcount-ra-address), y)
Expand Down
208 changes: 96 additions & 112 deletions arch/mips/include/asm/atomic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,18 +64,16 @@ static __inline__ void atomic_add(int i, atomic_t * v)
} else if (kernel_uses_llsc) {
int temp;

__asm__ __volatile__(
" .set mips3 \n"
"1: ll %0, %1 # atomic_add \n"
" addu %0, %2 \n"
" sc %0, %1 \n"
" beqz %0, 2f \n"
" .subsection 2 \n"
"2: b 1b \n"
" .previous \n"
" .set mips0 \n"
: "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter));
do {
__asm__ __volatile__(
" .set mips3 \n"
" ll %0, %1 # atomic_add \n"
" addu %0, %2 \n"
" sc %0, %1 \n"
" .set mips0 \n"
: "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter));
} while (unlikely(!temp));
} else {
unsigned long flags;

Expand Down Expand Up @@ -109,18 +107,16 @@ static __inline__ void atomic_sub(int i, atomic_t * v)
} else if (kernel_uses_llsc) {
int temp;

__asm__ __volatile__(
" .set mips3 \n"
"1: ll %0, %1 # atomic_sub \n"
" subu %0, %2 \n"
" sc %0, %1 \n"
" beqz %0, 2f \n"
" .subsection 2 \n"
"2: b 1b \n"
" .previous \n"
" .set mips0 \n"
: "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter));
do {
__asm__ __volatile__(
" .set mips3 \n"
" ll %0, %1 # atomic_sub \n"
" subu %0, %2 \n"
" sc %0, %1 \n"
" .set mips0 \n"
: "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter));
} while (unlikely(!temp));
} else {
unsigned long flags;

Expand Down Expand Up @@ -156,20 +152,19 @@ static __inline__ int atomic_add_return(int i, atomic_t * v)
} else if (kernel_uses_llsc) {
int temp;

__asm__ __volatile__(
" .set mips3 \n"
"1: ll %1, %2 # atomic_add_return \n"
" addu %0, %1, %3 \n"
" sc %0, %2 \n"
" beqz %0, 2f \n"
" addu %0, %1, %3 \n"
" .subsection 2 \n"
"2: b 1b \n"
" .previous \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
: "memory");
do {
__asm__ __volatile__(
" .set mips3 \n"
" ll %1, %2 # atomic_add_return \n"
" addu %0, %1, %3 \n"
" sc %0, %2 \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
: "memory");
} while (unlikely(!result));

result = temp + i;
} else {
unsigned long flags;

Expand Down Expand Up @@ -205,23 +200,24 @@ static __inline__ int atomic_sub_return(int i, atomic_t * v)
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
: "memory");

result = temp - i;
} else if (kernel_uses_llsc) {
int temp;

__asm__ __volatile__(
" .set mips3 \n"
"1: ll %1, %2 # atomic_sub_return \n"
" subu %0, %1, %3 \n"
" sc %0, %2 \n"
" beqz %0, 2f \n"
" subu %0, %1, %3 \n"
" .subsection 2 \n"
"2: b 1b \n"
" .previous \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
: "memory");
do {
__asm__ __volatile__(
" .set mips3 \n"
" ll %1, %2 # atomic_sub_return \n"
" subu %0, %1, %3 \n"
" sc %0, %2 \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
: "memory");
} while (unlikely(!result));

result = temp - i;
} else {
unsigned long flags;

Expand Down Expand Up @@ -279,12 +275,9 @@ static __inline__ int atomic_sub_if_positive(int i, atomic_t * v)
" bltz %0, 1f \n"
" sc %0, %2 \n"
" .set noreorder \n"
" beqz %0, 2f \n"
" beqz %0, 1b \n"
" subu %0, %1, %3 \n"
" .set reorder \n"
" .subsection 2 \n"
"2: b 1b \n"
" .previous \n"
"1: \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
Expand Down Expand Up @@ -443,18 +436,16 @@ static __inline__ void atomic64_add(long i, atomic64_t * v)
} else if (kernel_uses_llsc) {
long temp;

__asm__ __volatile__(
" .set mips3 \n"
"1: lld %0, %1 # atomic64_add \n"
" daddu %0, %2 \n"
" scd %0, %1 \n"
" beqz %0, 2f \n"
" .subsection 2 \n"
"2: b 1b \n"
" .previous \n"
" .set mips0 \n"
: "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter));
do {
__asm__ __volatile__(
" .set mips3 \n"
" lld %0, %1 # atomic64_add \n"
" daddu %0, %2 \n"
" scd %0, %1 \n"
" .set mips0 \n"
: "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter));
} while (unlikely(!temp));
} else {
unsigned long flags;

Expand Down Expand Up @@ -488,18 +479,16 @@ static __inline__ void atomic64_sub(long i, atomic64_t * v)
} else if (kernel_uses_llsc) {
long temp;

__asm__ __volatile__(
" .set mips3 \n"
"1: lld %0, %1 # atomic64_sub \n"
" dsubu %0, %2 \n"
" scd %0, %1 \n"
" beqz %0, 2f \n"
" .subsection 2 \n"
"2: b 1b \n"
" .previous \n"
" .set mips0 \n"
: "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter));
do {
__asm__ __volatile__(
" .set mips3 \n"
" lld %0, %1 # atomic64_sub \n"
" dsubu %0, %2 \n"
" scd %0, %1 \n"
" .set mips0 \n"
: "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter));
} while (unlikely(!temp));
} else {
unsigned long flags;

Expand Down Expand Up @@ -535,20 +524,19 @@ static __inline__ long atomic64_add_return(long i, atomic64_t * v)
} else if (kernel_uses_llsc) {
long temp;

__asm__ __volatile__(
" .set mips3 \n"
"1: lld %1, %2 # atomic64_add_return \n"
" daddu %0, %1, %3 \n"
" scd %0, %2 \n"
" beqz %0, 2f \n"
" daddu %0, %1, %3 \n"
" .subsection 2 \n"
"2: b 1b \n"
" .previous \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
: "memory");
do {
__asm__ __volatile__(
" .set mips3 \n"
" lld %1, %2 # atomic64_add_return \n"
" daddu %0, %1, %3 \n"
" scd %0, %2 \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
: "memory");
} while (unlikely(!result));

result = temp + i;
} else {
unsigned long flags;

Expand Down Expand Up @@ -587,20 +575,19 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v)
} else if (kernel_uses_llsc) {
long temp;

__asm__ __volatile__(
" .set mips3 \n"
"1: lld %1, %2 # atomic64_sub_return \n"
" dsubu %0, %1, %3 \n"
" scd %0, %2 \n"
" beqz %0, 2f \n"
" dsubu %0, %1, %3 \n"
" .subsection 2 \n"
"2: b 1b \n"
" .previous \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
: "memory");
do {
__asm__ __volatile__(
" .set mips3 \n"
" lld %1, %2 # atomic64_sub_return \n"
" dsubu %0, %1, %3 \n"
" scd %0, %2 \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
: "Ir" (i), "m" (v->counter)
: "memory");
} while (unlikely(!result));

result = temp - i;
} else {
unsigned long flags;

Expand Down Expand Up @@ -658,12 +645,9 @@ static __inline__ long atomic64_sub_if_positive(long i, atomic64_t * v)
" bltz %0, 1f \n"
" scd %0, %2 \n"
" .set noreorder \n"
" beqz %0, 2f \n"
" beqz %0, 1b \n"
" dsubu %0, %1, %3 \n"
" .set reorder \n"
" .subsection 2 \n"
"2: b 1b \n"
" .previous \n"
"1: \n"
" .set mips0 \n"
: "=&r" (result), "=&r" (temp), "=m" (v->counter)
Expand Down
Loading

0 comments on commit 7837314

Please sign in to comment.