Skip to content

Commit

Permalink
instrumentation: Wire up cmpxchg128()
Browse files Browse the repository at this point in the history 
Wire up the cmpxchg128 family in the atomic wrapper scripts.

These provide the generic cmpxchg128 family of functions from the
arch_ prefixed version, adding explicit instrumentation where needed.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20230531132323.519237070@infradead.org
  • Loading branch information
Peter Zijlstra committed Jun 5, 2023
1 parent b23e139 commit 8c8b096
Show file tree
Hide file tree
Showing 4 changed files with 183 additions and 6 deletions.
95 changes: 94 additions & 1 deletion include/linux/atomic/atomic-arch-fallback.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,29 @@

#endif /* arch_cmpxchg64_relaxed */

#ifndef arch_cmpxchg128_relaxed
#define arch_cmpxchg128_acquire arch_cmpxchg128
#define arch_cmpxchg128_release arch_cmpxchg128
#define arch_cmpxchg128_relaxed arch_cmpxchg128
#else /* arch_cmpxchg128_relaxed */

#ifndef arch_cmpxchg128_acquire
#define arch_cmpxchg128_acquire(...) \
__atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
#endif

#ifndef arch_cmpxchg128_release
#define arch_cmpxchg128_release(...) \
__atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
#endif

#ifndef arch_cmpxchg128
#define arch_cmpxchg128(...) \
__atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
#endif

#endif /* arch_cmpxchg128_relaxed */

#ifndef arch_try_cmpxchg_relaxed
#ifdef arch_try_cmpxchg
#define arch_try_cmpxchg_acquire arch_try_cmpxchg
Expand Down Expand Up @@ -217,6 +240,76 @@

#endif /* arch_try_cmpxchg64_relaxed */

#ifndef arch_try_cmpxchg128_relaxed
#ifdef arch_try_cmpxchg128
#define arch_try_cmpxchg128_acquire arch_try_cmpxchg128
#define arch_try_cmpxchg128_release arch_try_cmpxchg128
#define arch_try_cmpxchg128_relaxed arch_try_cmpxchg128
#endif /* arch_try_cmpxchg128 */

#ifndef arch_try_cmpxchg128
#define arch_try_cmpxchg128(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg128((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg128 */

#ifndef arch_try_cmpxchg128_acquire
#define arch_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg128_acquire((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg128_acquire */

#ifndef arch_try_cmpxchg128_release
#define arch_try_cmpxchg128_release(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg128_release((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg128_release */

#ifndef arch_try_cmpxchg128_relaxed
#define arch_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
({ \
typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
___r = arch_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
if (unlikely(___r != ___o)) \
*___op = ___r; \
likely(___r == ___o); \
})
#endif /* arch_try_cmpxchg128_relaxed */

#else /* arch_try_cmpxchg128_relaxed */

#ifndef arch_try_cmpxchg128_acquire
#define arch_try_cmpxchg128_acquire(...) \
__atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
#endif

#ifndef arch_try_cmpxchg128_release
#define arch_try_cmpxchg128_release(...) \
__atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
#endif

#ifndef arch_try_cmpxchg128
#define arch_try_cmpxchg128(...) \
__atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
#endif

#endif /* arch_try_cmpxchg128_relaxed */

#ifndef arch_try_cmpxchg_local
#define arch_try_cmpxchg_local(_ptr, _oldp, _new) \
({ \
Expand Down Expand Up @@ -2668,4 +2761,4 @@ arch_atomic64_dec_if_positive(atomic64_t *v)
#endif

#endif /* _LINUX_ATOMIC_FALLBACK_H */
// ad2e2b4d168dbc60a73922616047a9bfa446af36
// 52dfc6fe4a2e7234bbd2aa3e16a377c1db793a53
86 changes: 85 additions & 1 deletion include/linux/atomic/atomic-instrumented.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2034,6 +2034,36 @@ atomic_long_dec_if_positive(atomic_long_t *v)
arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg128(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg128(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg128_acquire(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg128_acquire(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg128_release(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
kcsan_release(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg128_release(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg128_relaxed(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg128_relaxed(__ai_ptr, __VA_ARGS__); \
})

#define try_cmpxchg(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
Expand Down Expand Up @@ -2110,6 +2140,44 @@ atomic_long_dec_if_positive(atomic_long_t *v)
arch_try_cmpxchg64_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg128(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_mb(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg128(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg128_acquire(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg128_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg128_release(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
kcsan_release(); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg128_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg128_relaxed(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg128_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define cmpxchg_local(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
Expand All @@ -2124,6 +2192,13 @@ atomic_long_dec_if_positive(atomic_long_t *v)
arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
})

#define cmpxchg128_local(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
arch_cmpxchg128_local(__ai_ptr, __VA_ARGS__); \
})

#define sync_cmpxchg(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
Expand All @@ -2150,6 +2225,15 @@ atomic_long_dec_if_positive(atomic_long_t *v)
arch_try_cmpxchg64_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define try_cmpxchg128_local(ptr, oldp, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
typeof(oldp) __ai_oldp = (oldp); \
instrument_atomic_read_write(__ai_ptr, sizeof(*__ai_ptr)); \
instrument_read_write(__ai_oldp, sizeof(*__ai_oldp)); \
arch_try_cmpxchg128_local(__ai_ptr, __ai_oldp, __VA_ARGS__); \
})

#define cmpxchg_double(ptr, ...) \
({ \
typeof(ptr) __ai_ptr = (ptr); \
Expand All @@ -2167,4 +2251,4 @@ atomic_long_dec_if_positive(atomic_long_t *v)
})

#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
// 6b513a42e1a1b5962532a019b7fc91eaa044ad5e
// 82d1be694fab30414527d0877c29fa75ed5a0b74
4 changes: 2 additions & 2 deletions scripts/atomic/gen-atomic-fallback.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,11 +217,11 @@ cat << EOF
EOF

for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64"; do
for xchg in "arch_xchg" "arch_cmpxchg" "arch_cmpxchg64" "arch_cmpxchg128"; do
gen_xchg_fallbacks "${xchg}"
done

for cmpxchg in "cmpxchg" "cmpxchg64"; do
for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do
gen_try_cmpxchg_fallbacks "${cmpxchg}"
done

Expand Down
4 changes: 2 additions & 2 deletions scripts/atomic/gen-atomic-instrumented.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,14 +166,14 @@ grep '^[a-z]' "$1" | while read name meta args; do
done


for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg" "try_cmpxchg64"; do
for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128" "try_cmpxchg" "try_cmpxchg64" "try_cmpxchg128"; do
for order in "" "_acquire" "_release" "_relaxed"; do
gen_xchg "${xchg}" "${order}" ""
printf "\n"
done
done

for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" ; do
for xchg in "cmpxchg_local" "cmpxchg64_local" "cmpxchg128_local" "sync_cmpxchg" "try_cmpxchg_local" "try_cmpxchg64_local" "try_cmpxchg128_local"; do
gen_xchg "${xchg}" "" ""
printf "\n"
done
Expand Down

0 comments on commit 8c8b096

Please sign in to comment.