Skip to content

Commit

Permalink
percpu: Add {raw,this}_cpu_try_cmpxchg()
Browse files Browse the repository at this point in the history 
Add the try_cmpxchg() form to the per-cpu ops.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Link: https://lore.kernel.org/r/20230531132323.587480729@infradead.org
  • Loading branch information
Peter Zijlstra committed Jun 5, 2023
1 parent 8c8b096 commit c5c0ba9
Show file tree
Hide file tree
Showing 2 changed files with 128 additions and 4 deletions.
113 changes: 109 additions & 4 deletions include/asm-generic/percpu.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,16 +89,37 @@ do { \
__ret; \
})

#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
#define __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, _cmpxchg) \
({ \
typeof(pcp) __val, __old = *(ovalp); \
__val = _cmpxchg(pcp, __old, nval); \
if (__val != __old) \
*(ovalp) = __val; \
__val == __old; \
})

#define raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
({ \
typeof(pcp) *__p = raw_cpu_ptr(&(pcp)); \
typeof(pcp) __ret; \
__ret = *__p; \
if (__ret == (oval)) \
typeof(pcp) __val = *__p, __old = *(ovalp); \
bool __ret; \
if (__val == __old) { \
*__p = nval; \
__ret = true; \
} else { \
*(ovalp) = __val; \
__ret = false; \
} \
__ret; \
})

#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
({ \
typeof(pcp) __old = (oval); \
raw_cpu_generic_try_cmpxchg(pcp, &__old, nval); \
__old; \
})

#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
({ \
typeof(pcp1) *__p1 = raw_cpu_ptr(&(pcp1)); \
Expand Down Expand Up @@ -170,6 +191,16 @@ do { \
__ret; \
})

#define this_cpu_generic_try_cmpxchg(pcp, ovalp, nval) \
({ \
bool __ret; \
unsigned long __flags; \
raw_local_irq_save(__flags); \
__ret = raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval); \
raw_local_irq_restore(__flags); \
__ret; \
})

#define this_cpu_generic_cmpxchg(pcp, oval, nval) \
({ \
typeof(pcp) __ret; \
Expand Down Expand Up @@ -282,6 +313,43 @@ do { \
#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
#endif

#ifndef raw_cpu_try_cmpxchg_1
#ifdef raw_cpu_cmpxchg_1
#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_1)
#else
#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
#endif
#endif
#ifndef raw_cpu_try_cmpxchg_2
#ifdef raw_cpu_cmpxchg_2
#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_2)
#else
#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
#endif
#endif
#ifndef raw_cpu_try_cmpxchg_4
#ifdef raw_cpu_cmpxchg_4
#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_4)
#else
#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
#endif
#endif
#ifndef raw_cpu_try_cmpxchg_8
#ifdef raw_cpu_cmpxchg_8
#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_8)
#else
#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
#endif
#endif

#ifndef raw_cpu_cmpxchg_1
#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
raw_cpu_generic_cmpxchg(pcp, oval, nval)
Expand Down Expand Up @@ -407,6 +475,43 @@ do { \
#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
#endif

#ifndef this_cpu_try_cmpxchg_1
#ifdef this_cpu_cmpxchg_1
#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_1)
#else
#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
#endif
#endif
#ifndef this_cpu_try_cmpxchg_2
#ifdef this_cpu_cmpxchg_2
#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_2)
#else
#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
#endif
#endif
#ifndef this_cpu_try_cmpxchg_4
#ifdef this_cpu_cmpxchg_4
#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_4)
#else
#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
#endif
#endif
#ifndef this_cpu_try_cmpxchg_8
#ifdef this_cpu_cmpxchg_8
#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_8)
#else
#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
#endif
#endif

#ifndef this_cpu_cmpxchg_1
#define this_cpu_cmpxchg_1(pcp, oval, nval) \
this_cpu_generic_cmpxchg(pcp, oval, nval)
Expand Down
19 changes: 19 additions & 0 deletions include/linux/percpu-defs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,21 @@ static __always_inline void __this_cpu_preempt_check(const char *op) { }
pscr2_ret__; \
})

#define __pcpu_size_call_return2bool(stem, variable, ...) \
({ \
bool pscr2_ret__; \
__verify_pcpu_ptr(&(variable)); \
switch(sizeof(variable)) { \
case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
default: \
__bad_size_call_parameter(); break; \
} \
pscr2_ret__; \
})

/*
* Special handling for cmpxchg_double. cmpxchg_double is passed two
* percpu variables. The first has to be aligned to a double word
Expand Down Expand Up @@ -426,6 +441,8 @@ do { \
#define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
#define raw_cpu_cmpxchg(pcp, oval, nval) \
__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
#define raw_cpu_try_cmpxchg(pcp, ovalp, nval) \
__pcpu_size_call_return2bool(raw_cpu_try_cmpxchg_, pcp, ovalp, nval)
#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)

Expand Down Expand Up @@ -513,6 +530,8 @@ do { \
#define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
#define this_cpu_cmpxchg(pcp, oval, nval) \
__pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
#define this_cpu_try_cmpxchg(pcp, ovalp, nval) \
__pcpu_size_call_return2bool(this_cpu_try_cmpxchg_, pcp, ovalp, nval)
#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
__pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)

Expand Down

0 comments on commit c5c0ba9

Please sign in to comment.