Skip to content

Commit

Permalink
arm64: atomics: fix grossly inconsistent asm constraints for exclusives
Browse files Browse the repository at this point in the history 
Our uses of inline asm constraints for atomic operations are fairly
wild and varied. We basically need to guarantee the following:

  1. Any instructions with barrier implications
     (load-acquire/store-release) have a "memory" clobber

  2. When performing exclusive accesses, the addresing mode is generated
     using the "Q" constraint

  3. Atomic blocks which use the condition flags, have a "cc" clobber

This patch addresses these concerns which, as well as fixing the
semantics of the code, stops GCC complaining about impossible asm
constraints.

Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
  • Loading branch information
Will Deacon authored and Catalin Marinas committed Feb 11, 2013
1 parent c0e01d5 commit 3a0310e
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 143 deletions.
132 changes: 66 additions & 66 deletions arch/arm64/include/asm/atomic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,12 @@ static inline void atomic_add(int i, atomic_t *v)
int result;

asm volatile("// atomic_add\n"
"1: ldxr %w0, [%3]\n"
" add %w0, %w0, %w4\n"
" stxr %w1, %w0, [%3]\n"
"1: ldxr %w0, %2\n"
" add %w0, %w0, %w3\n"
" stxr %w1, %w0, %2\n"
" cbnz %w1, 1b"
: "=&r" (result), "=&r" (tmp), "+o" (v->counter)
: "r" (&v->counter), "Ir" (i)
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
: "Ir" (i)
: "cc");
}

Expand All @@ -64,13 +64,13 @@ static inline int atomic_add_return(int i, atomic_t *v)
int result;

asm volatile("// atomic_add_return\n"
"1: ldaxr %w0, [%3]\n"
" add %w0, %w0, %w4\n"
" stlxr %w1, %w0, [%3]\n"
"1: ldaxr %w0, %2\n"
" add %w0, %w0, %w3\n"
" stlxr %w1, %w0, %2\n"
" cbnz %w1, 1b"
: "=&r" (result), "=&r" (tmp), "+o" (v->counter)
: "r" (&v->counter), "Ir" (i)
: "cc");
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
: "Ir" (i)
: "cc", "memory");

return result;
}
Expand All @@ -81,12 +81,12 @@ static inline void atomic_sub(int i, atomic_t *v)
int result;

asm volatile("// atomic_sub\n"
"1: ldxr %w0, [%3]\n"
" sub %w0, %w0, %w4\n"
" stxr %w1, %w0, [%3]\n"
"1: ldxr %w0, %2\n"
" sub %w0, %w0, %w3\n"
" stxr %w1, %w0, %2\n"
" cbnz %w1, 1b"
: "=&r" (result), "=&r" (tmp), "+o" (v->counter)
: "r" (&v->counter), "Ir" (i)
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
: "Ir" (i)
: "cc");
}

Expand All @@ -96,13 +96,13 @@ static inline int atomic_sub_return(int i, atomic_t *v)
int result;

asm volatile("// atomic_sub_return\n"
"1: ldaxr %w0, [%3]\n"
" sub %w0, %w0, %w4\n"
" stlxr %w1, %w0, [%3]\n"
"1: ldaxr %w0, %2\n"
" sub %w0, %w0, %w3\n"
" stlxr %w1, %w0, %2\n"
" cbnz %w1, 1b"
: "=&r" (result), "=&r" (tmp), "+o" (v->counter)
: "r" (&v->counter), "Ir" (i)
: "cc");
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
: "Ir" (i)
: "cc", "memory");

return result;
}
Expand All @@ -113,15 +113,15 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
int oldval;

asm volatile("// atomic_cmpxchg\n"
"1: ldaxr %w1, [%3]\n"
" cmp %w1, %w4\n"
"1: ldaxr %w1, %2\n"
" cmp %w1, %w3\n"
" b.ne 2f\n"
" stlxr %w0, %w5, [%3]\n"
" stlxr %w0, %w4, %2\n"
" cbnz %w0, 1b\n"
"2:"
: "=&r" (tmp), "=&r" (oldval), "+o" (ptr->counter)
: "r" (&ptr->counter), "Ir" (old), "r" (new)
: "cc");
: "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter)
: "Ir" (old), "r" (new)
: "cc", "memory");

return oldval;
}
Expand All @@ -131,12 +131,12 @@ static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
unsigned long tmp, tmp2;

asm volatile("// atomic_clear_mask\n"
"1: ldxr %0, [%3]\n"
" bic %0, %0, %4\n"
" stxr %w1, %0, [%3]\n"
"1: ldxr %0, %2\n"
" bic %0, %0, %3\n"
" stxr %w1, %0, %2\n"
" cbnz %w1, 1b"
: "=&r" (tmp), "=&r" (tmp2), "+o" (*addr)
: "r" (addr), "Ir" (mask)
: "=&r" (tmp), "=&r" (tmp2), "+Q" (*addr)
: "Ir" (mask)
: "cc");
}

Expand Down Expand Up @@ -182,12 +182,12 @@ static inline void atomic64_add(u64 i, atomic64_t *v)
unsigned long tmp;

asm volatile("// atomic64_add\n"
"1: ldxr %0, [%3]\n"
" add %0, %0, %4\n"
" stxr %w1, %0, [%3]\n"
"1: ldxr %0, %2\n"
" add %0, %0, %3\n"
" stxr %w1, %0, %2\n"
" cbnz %w1, 1b"
: "=&r" (result), "=&r" (tmp), "+o" (v->counter)
: "r" (&v->counter), "Ir" (i)
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
: "Ir" (i)
: "cc");
}

Expand All @@ -197,13 +197,13 @@ static inline long atomic64_add_return(long i, atomic64_t *v)
unsigned long tmp;

asm volatile("// atomic64_add_return\n"
"1: ldaxr %0, [%3]\n"
" add %0, %0, %4\n"
" stlxr %w1, %0, [%3]\n"
"1: ldaxr %0, %2\n"
" add %0, %0, %3\n"
" stlxr %w1, %0, %2\n"
" cbnz %w1, 1b"
: "=&r" (result), "=&r" (tmp), "+o" (v->counter)
: "r" (&v->counter), "Ir" (i)
: "cc");
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
: "Ir" (i)
: "cc", "memory");

return result;
}
Expand All @@ -214,12 +214,12 @@ static inline void atomic64_sub(u64 i, atomic64_t *v)
unsigned long tmp;

asm volatile("// atomic64_sub\n"
"1: ldxr %0, [%3]\n"
" sub %0, %0, %4\n"
" stxr %w1, %0, [%3]\n"
"1: ldxr %0, %2\n"
" sub %0, %0, %3\n"
" stxr %w1, %0, %2\n"
" cbnz %w1, 1b"
: "=&r" (result), "=&r" (tmp), "+o" (v->counter)
: "r" (&v->counter), "Ir" (i)
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
: "Ir" (i)
: "cc");
}

Expand All @@ -229,13 +229,13 @@ static inline long atomic64_sub_return(long i, atomic64_t *v)
unsigned long tmp;

asm volatile("// atomic64_sub_return\n"
"1: ldaxr %0, [%3]\n"
" sub %0, %0, %4\n"
" stlxr %w1, %0, [%3]\n"
"1: ldaxr %0, %2\n"
" sub %0, %0, %3\n"
" stlxr %w1, %0, %2\n"
" cbnz %w1, 1b"
: "=&r" (result), "=&r" (tmp), "+o" (v->counter)
: "r" (&v->counter), "Ir" (i)
: "cc");
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
: "Ir" (i)
: "cc", "memory");

return result;
}
Expand All @@ -246,15 +246,15 @@ static inline long atomic64_cmpxchg(atomic64_t *ptr, long old, long new)
unsigned long res;

asm volatile("// atomic64_cmpxchg\n"
"1: ldaxr %1, [%3]\n"
" cmp %1, %4\n"
"1: ldaxr %1, %2\n"
" cmp %1, %3\n"
" b.ne 2f\n"
" stlxr %w0, %5, [%3]\n"
" stlxr %w0, %4, %2\n"
" cbnz %w0, 1b\n"
"2:"
: "=&r" (res), "=&r" (oldval), "+o" (ptr->counter)
: "r" (&ptr->counter), "Ir" (old), "r" (new)
: "cc");
: "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter)
: "Ir" (old), "r" (new)
: "cc", "memory");

return oldval;
}
Expand All @@ -267,15 +267,15 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)
unsigned long tmp;

asm volatile("// atomic64_dec_if_positive\n"
"1: ldaxr %0, [%3]\n"
"1: ldaxr %0, %2\n"
" subs %0, %0, #1\n"
" b.mi 2f\n"
" stlxr %w1, %0, [%3]\n"
" stlxr %w1, %0, %2\n"
" cbnz %w1, 1b\n"
"2:"
: "=&r" (result), "=&r" (tmp), "+o" (v->counter)
: "r" (&v->counter)
: "cc");
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
:
: "cc", "memory");

return result;
}
Expand Down
74 changes: 37 additions & 37 deletions arch/arm64/include/asm/cmpxchg.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,39 +29,39 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size
switch (size) {
case 1:
asm volatile("// __xchg1\n"
"1: ldaxrb %w0, [%3]\n"
" stlxrb %w1, %w2, [%3]\n"
"1: ldaxrb %w0, %2\n"
" stlxrb %w1, %w3, %2\n"
" cbnz %w1, 1b\n"
: "=&r" (ret), "=&r" (tmp)
: "r" (x), "r" (ptr)
: "memory", "cc");
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr)
: "r" (x)
: "cc", "memory");
break;
case 2:
asm volatile("// __xchg2\n"
"1: ldaxrh %w0, [%3]\n"
" stlxrh %w1, %w2, [%3]\n"
"1: ldaxrh %w0, %2\n"
" stlxrh %w1, %w3, %2\n"
" cbnz %w1, 1b\n"
: "=&r" (ret), "=&r" (tmp)
: "r" (x), "r" (ptr)
: "memory", "cc");
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr)
: "r" (x)
: "cc", "memory");
break;
case 4:
asm volatile("// __xchg4\n"
"1: ldaxr %w0, [%3]\n"
" stlxr %w1, %w2, [%3]\n"
"1: ldaxr %w0, %2\n"
" stlxr %w1, %w3, %2\n"
" cbnz %w1, 1b\n"
: "=&r" (ret), "=&r" (tmp)
: "r" (x), "r" (ptr)
: "memory", "cc");
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr)
: "r" (x)
: "cc", "memory");
break;
case 8:
asm volatile("// __xchg8\n"
"1: ldaxr %0, [%3]\n"
" stlxr %w1, %2, [%3]\n"
"1: ldaxr %0, %2\n"
" stlxr %w1, %3, %2\n"
" cbnz %w1, 1b\n"
: "=&r" (ret), "=&r" (tmp)
: "r" (x), "r" (ptr)
: "memory", "cc");
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr)
: "r" (x)
: "cc", "memory");
break;
default:
BUILD_BUG();
Expand All @@ -82,59 +82,59 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
case 1:
do {
asm volatile("// __cmpxchg1\n"
" ldxrb %w1, [%2]\n"
" ldxrb %w1, %2\n"
" mov %w0, #0\n"
" cmp %w1, %w3\n"
" b.ne 1f\n"
" stxrb %w0, %w4, [%2]\n"
" stxrb %w0, %w4, %2\n"
"1:\n"
: "=&r" (res), "=&r" (oldval)
: "r" (ptr), "Ir" (old), "r" (new)
: "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr)
: "Ir" (old), "r" (new)
: "cc");
} while (res);
break;

case 2:
do {
asm volatile("// __cmpxchg2\n"
" ldxrh %w1, [%2]\n"
" ldxrh %w1, %2\n"
" mov %w0, #0\n"
" cmp %w1, %w3\n"
" b.ne 1f\n"
" stxrh %w0, %w4, [%2]\n"
" stxrh %w0, %w4, %2\n"
"1:\n"
: "=&r" (res), "=&r" (oldval)
: "r" (ptr), "Ir" (old), "r" (new)
: "memory", "cc");
: "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr)
: "Ir" (old), "r" (new)
: "cc");
} while (res);
break;

case 4:
do {
asm volatile("// __cmpxchg4\n"
" ldxr %w1, [%2]\n"
" ldxr %w1, %2\n"
" mov %w0, #0\n"
" cmp %w1, %w3\n"
" b.ne 1f\n"
" stxr %w0, %w4, [%2]\n"
" stxr %w0, %w4, %2\n"
"1:\n"
: "=&r" (res), "=&r" (oldval)
: "r" (ptr), "Ir" (old), "r" (new)
: "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr)
: "Ir" (old), "r" (new)
: "cc");
} while (res);
break;

case 8:
do {
asm volatile("// __cmpxchg8\n"
" ldxr %1, [%2]\n"
" ldxr %1, %2\n"
" mov %w0, #0\n"
" cmp %1, %3\n"
" b.ne 1f\n"
" stxr %w0, %4, [%2]\n"
" stxr %w0, %4, %2\n"
"1:\n"
: "=&r" (res), "=&r" (oldval)
: "r" (ptr), "Ir" (old), "r" (new)
: "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr)
: "Ir" (old), "r" (new)
: "cc");
} while (res);
break;
Expand Down
2 changes: 1 addition & 1 deletion arch/arm64/include/asm/futex.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
" .popsection\n" \
: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \
: "r" (oparg), "Ir" (-EFAULT) \
: "cc")
: "cc", "memory")

static inline int
futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr)
Expand Down
Loading

0 comments on commit 3a0310e

Please sign in to comment.