Skip to content

Commit

Permalink
percpu-refcount: make percpu_ref based on longs instead of ints
Browse files Browse the repository at this point in the history 
percpu_ref is currently based on ints and the number of refs it can
cover is (1 << 31).  This makes it impossible to use a percpu_ref to
count memory objects or pages on 64bit machines as it may overflow.
This forces those users to somehow aggregate the references before
contributing to the percpu_ref which is often cumbersome and sometimes
challenging to get the same level of performance as using the
percpu_ref directly.

While using ints for the percpu counters makes them pack tighter on
64bit machines, the possible gain from using ints instead of longs is
extremely small compared to the overall gain from per-cpu operation.
This patch makes percpu_ref based on longs so that it can be used to
directly count memory objects or pages.

Signed-off-by: Tejun Heo <tj@kernel.org>
Cc: Kent Overstreet <kmo@daterainc.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
  • Loading branch information
Tejun Heo committed Sep 20, 2014
1 parent 4843c33 commit e625305
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 30 deletions.
24 changes: 12 additions & 12 deletions include/linux/percpu-refcount.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ struct percpu_ref;
typedef void (percpu_ref_func_t)(struct percpu_ref *);

struct percpu_ref {
atomic_t count;
atomic_long_t count;
/*
* The low bit of the pointer indicates whether the ref is in percpu
* mode; if set, then get/put will manipulate the atomic_t.
Expand Down Expand Up @@ -97,7 +97,7 @@ static inline void percpu_ref_kill(struct percpu_ref *ref)
* branches as it can't assume that @ref->pcpu_count is not NULL.
*/
static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
unsigned __percpu **pcpu_countp)
unsigned long __percpu **pcpu_countp)
{
unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr);

Expand All @@ -107,7 +107,7 @@ static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
if (unlikely(pcpu_ptr & PCPU_REF_DEAD))
return false;

*pcpu_countp = (unsigned __percpu *)pcpu_ptr;
*pcpu_countp = (unsigned long __percpu *)pcpu_ptr;
return true;
}

Expand All @@ -119,14 +119,14 @@ static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
*/
static inline void percpu_ref_get(struct percpu_ref *ref)
{
unsigned __percpu *pcpu_count;
unsigned long __percpu *pcpu_count;

rcu_read_lock_sched();

if (__pcpu_ref_alive(ref, &pcpu_count))
this_cpu_inc(*pcpu_count);
else
atomic_inc(&ref->count);
atomic_long_inc(&ref->count);

rcu_read_unlock_sched();
}
Expand All @@ -142,7 +142,7 @@ static inline void percpu_ref_get(struct percpu_ref *ref)
*/
static inline bool percpu_ref_tryget(struct percpu_ref *ref)
{
unsigned __percpu *pcpu_count;
unsigned long __percpu *pcpu_count;
int ret = false;

rcu_read_lock_sched();
Expand All @@ -151,7 +151,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
this_cpu_inc(*pcpu_count);
ret = true;
} else {
ret = atomic_inc_not_zero(&ref->count);
ret = atomic_long_inc_not_zero(&ref->count);
}

rcu_read_unlock_sched();
Expand All @@ -175,7 +175,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
*/
static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
{
unsigned __percpu *pcpu_count;
unsigned long __percpu *pcpu_count;
int ret = false;

rcu_read_lock_sched();
Expand All @@ -199,13 +199,13 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
*/
static inline void percpu_ref_put(struct percpu_ref *ref)
{
unsigned __percpu *pcpu_count;
unsigned long __percpu *pcpu_count;

rcu_read_lock_sched();

if (__pcpu_ref_alive(ref, &pcpu_count))
this_cpu_dec(*pcpu_count);
else if (unlikely(atomic_dec_and_test(&ref->count)))
else if (unlikely(atomic_long_dec_and_test(&ref->count)))
ref->release(ref);

rcu_read_unlock_sched();
Expand All @@ -219,11 +219,11 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
*/
static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
{
unsigned __percpu *pcpu_count;
unsigned long __percpu *pcpu_count;

if (__pcpu_ref_alive(ref, &pcpu_count))
return false;
return !atomic_read(&ref->count);
return !atomic_long_read(&ref->count);
}

#endif
37 changes: 19 additions & 18 deletions lib/percpu-refcount.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,15 @@
* works.
*
* Converting to non percpu mode is done with some RCUish stuff in
* percpu_ref_kill. Additionally, we need a bias value so that the atomic_t
* can't hit 0 before we've added up all the percpu refs.
* percpu_ref_kill. Additionally, we need a bias value so that the
* atomic_long_t can't hit 0 before we've added up all the percpu refs.
*/

#define PCPU_COUNT_BIAS (1U << 31)
#define PCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1))

static unsigned __percpu *pcpu_count_ptr(struct percpu_ref *ref)
static unsigned long __percpu *pcpu_count_ptr(struct percpu_ref *ref)
{
return (unsigned __percpu *)(ref->pcpu_count_ptr & ~PCPU_REF_DEAD);
return (unsigned long __percpu *)(ref->pcpu_count_ptr & ~PCPU_REF_DEAD);
}

/**
Expand All @@ -43,17 +43,17 @@ static unsigned __percpu *pcpu_count_ptr(struct percpu_ref *ref)
* @gfp: allocation mask to use
*
* Initializes the refcount in single atomic counter mode with a refcount of 1;
* analagous to atomic_set(ref, 1).
* analagous to atomic_long_set(ref, 1).
*
* Note that @release must not sleep - it may potentially be called from RCU
* callback context by percpu_ref_kill().
*/
int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release,
gfp_t gfp)
{
atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS);
atomic_long_set(&ref->count, 1 + PCPU_COUNT_BIAS);

ref->pcpu_count_ptr = (unsigned long)alloc_percpu_gfp(unsigned, gfp);
ref->pcpu_count_ptr = (unsigned long)alloc_percpu_gfp(unsigned long, gfp);
if (!ref->pcpu_count_ptr)
return -ENOMEM;

Expand All @@ -75,13 +75,13 @@ EXPORT_SYMBOL_GPL(percpu_ref_init);
*/
void percpu_ref_reinit(struct percpu_ref *ref)
{
unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref);
int cpu;

BUG_ON(!pcpu_count);
WARN_ON(!percpu_ref_is_zero(ref));

atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS);
atomic_long_set(&ref->count, 1 + PCPU_COUNT_BIAS);

/*
* Restore per-cpu operation. smp_store_release() is paired with
Expand Down Expand Up @@ -109,7 +109,7 @@ EXPORT_SYMBOL_GPL(percpu_ref_reinit);
*/
void percpu_ref_exit(struct percpu_ref *ref)
{
unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref);

if (pcpu_count) {
free_percpu(pcpu_count);
Expand All @@ -121,14 +121,15 @@ EXPORT_SYMBOL_GPL(percpu_ref_exit);
static void percpu_ref_kill_rcu(struct rcu_head *rcu)
{
struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);
unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
unsigned count = 0;
unsigned long __percpu *pcpu_count = pcpu_count_ptr(ref);
unsigned long count = 0;
int cpu;

for_each_possible_cpu(cpu)
count += *per_cpu_ptr(pcpu_count, cpu);

pr_debug("global %i pcpu %i", atomic_read(&ref->count), (int) count);
pr_debug("global %ld pcpu %ld",
atomic_long_read(&ref->count), (long)count);

/*
* It's crucial that we sum the percpu counters _before_ adding the sum
Expand All @@ -143,11 +144,11 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu)
* time is equivalent and saves us atomic operations:
*/

atomic_add((int) count - PCPU_COUNT_BIAS, &ref->count);
atomic_long_add((long)count - PCPU_COUNT_BIAS, &ref->count);

WARN_ONCE(atomic_read(&ref->count) <= 0,
"percpu ref (%pf) <= 0 (%i) after killed",
ref->release, atomic_read(&ref->count));
WARN_ONCE(atomic_long_read(&ref->count) <= 0,
"percpu ref (%pf) <= 0 (%ld) after killed",
ref->release, atomic_long_read(&ref->count));

/* @ref is viewed as dead on all CPUs, send out kill confirmation */
if (ref->confirm_kill)
Expand Down

0 comments on commit e625305

Please sign in to comment.