Skip to content

Commit

Permalink
locking/atomic: simplify non-atomic wrappers
Browse files Browse the repository at this point in the history 
Since the non-atomic arch_*() bitops use plain accesses, they are
implicitly instrumnted by the compiler, and we work around this in the
instrumented wrappers to avoid double instrumentation.

It's simpler to avoid the wrappers entirely, and use the preprocessor to
alias the arch_*() bitops to their regular versions, removing the need
for checks in the instrumented wrappers.

Suggested-by: Marco Elver <elver@google.com>
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Marco Elver <elver@google.com>
Link: https://lore.kernel.org/r/20210721155813.17082-1-mark.rutland@arm.com
  • Loading branch information
Mark Rutland authored and Peter Zijlstra committed Aug 4, 2021
1 parent cf3ee3c commit 9248e52
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 23 deletions.
21 changes: 7 additions & 14 deletions include/asm-generic/bitops/instrumented-non-atomic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@
*/
static inline void __set_bit(long nr, volatile unsigned long *addr)
{
if (!__is_defined(arch___set_bit_uses_plain_access))
instrument_write(addr + BIT_WORD(nr), sizeof(long));
instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___set_bit(nr, addr);
}

Expand All @@ -40,8 +39,7 @@ static inline void __set_bit(long nr, volatile unsigned long *addr)
*/
static inline void __clear_bit(long nr, volatile unsigned long *addr)
{
if (!__is_defined(arch___clear_bit_uses_plain_access))
instrument_write(addr + BIT_WORD(nr), sizeof(long));
instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___clear_bit(nr, addr);
}

Expand All @@ -56,8 +54,7 @@ static inline void __clear_bit(long nr, volatile unsigned long *addr)
*/
static inline void __change_bit(long nr, volatile unsigned long *addr)
{
if (!__is_defined(arch___change_bit_uses_plain_access))
instrument_write(addr + BIT_WORD(nr), sizeof(long));
instrument_write(addr + BIT_WORD(nr), sizeof(long));
arch___change_bit(nr, addr);
}

Expand Down Expand Up @@ -95,8 +92,7 @@ static inline void __instrument_read_write_bitop(long nr, volatile unsigned long
*/
static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
{
if (!__is_defined(arch___test_and_set_bit_uses_plain_access))
__instrument_read_write_bitop(nr, addr);
__instrument_read_write_bitop(nr, addr);
return arch___test_and_set_bit(nr, addr);
}

Expand All @@ -110,8 +106,7 @@ static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
*/
static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
{
if (!__is_defined(arch___test_and_clear_bit_uses_plain_access))
__instrument_read_write_bitop(nr, addr);
__instrument_read_write_bitop(nr, addr);
return arch___test_and_clear_bit(nr, addr);
}

Expand All @@ -125,8 +120,7 @@ static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
*/
static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
{
if (!__is_defined(arch___test_and_change_bit_uses_plain_access))
__instrument_read_write_bitop(nr, addr);
__instrument_read_write_bitop(nr, addr);
return arch___test_and_change_bit(nr, addr);
}

Expand All @@ -137,8 +131,7 @@ static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
*/
static inline bool test_bit(long nr, const volatile unsigned long *addr)
{
if (!__is_defined(arch_test_bit_uses_plain_access))
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long));
return arch_test_bit(nr, addr);
}

Expand Down
16 changes: 7 additions & 9 deletions include/asm-generic/bitops/non-atomic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ arch___set_bit(int nr, volatile unsigned long *addr)

*p |= mask;
}
#define arch___set_bit_uses_plain_access
#define __set_bit arch___set_bit

static __always_inline void
arch___clear_bit(int nr, volatile unsigned long *addr)
Expand All @@ -31,7 +31,7 @@ arch___clear_bit(int nr, volatile unsigned long *addr)

*p &= ~mask;
}
#define arch___clear_bit_uses_plain_access
#define __clear_bit arch___clear_bit

/**
* arch___change_bit - Toggle a bit in memory
Expand All @@ -50,7 +50,7 @@ void arch___change_bit(int nr, volatile unsigned long *addr)

*p ^= mask;
}
#define arch___change_bit_uses_plain_access
#define __change_bit arch___change_bit

/**
* arch___test_and_set_bit - Set a bit and return its old value
Expand All @@ -71,7 +71,7 @@ arch___test_and_set_bit(int nr, volatile unsigned long *addr)
*p = old | mask;
return (old & mask) != 0;
}
#define arch___test_and_set_bit_uses_plain_access
#define __test_and_set_bit arch___test_and_set_bit

/**
* arch___test_and_clear_bit - Clear a bit and return its old value
Expand All @@ -92,7 +92,7 @@ arch___test_and_clear_bit(int nr, volatile unsigned long *addr)
*p = old & ~mask;
return (old & mask) != 0;
}
#define arch___test_and_clear_bit_uses_plain_access
#define __test_and_clear_bit arch___test_and_clear_bit

/* WARNING: non atomic and it can be reordered! */
static __always_inline int
Expand All @@ -105,7 +105,7 @@ arch___test_and_change_bit(int nr, volatile unsigned long *addr)
*p = old ^ mask;
return (old & mask) != 0;
}
#define arch___test_and_change_bit_uses_plain_access
#define __test_and_change_bit arch___test_and_change_bit

/**
* arch_test_bit - Determine whether a bit is set
Expand All @@ -117,8 +117,6 @@ arch_test_bit(int nr, const volatile unsigned long *addr)
{
return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
}
#define arch_test_bit_uses_plain_access

#include <asm-generic/bitops/instrumented-non-atomic.h>
#define test_bit arch_test_bit

#endif /* _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ */

0 comments on commit 9248e52

Please sign in to comment.