Skip to content

Commit

Permalink
Merge branch 'rhashtable-Fix-unprotected-RCU-dereference-in-__rht_ptr'
Browse files Browse the repository at this point in the history 
Herbert Xu says:

====================
rhashtable: Fix unprotected RCU dereference in __rht_ptr

This patch series fixes an unprotected dereference in __rht_ptr.
The first patch is a minimal fix that does not use the correct
RCU markings but is suitable for backport, and the second patch
cleans up the RCU markings.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
David S. Miller committed Jul 29, 2020
2 parents 19016d9 + ce9b362 commit a7ef23e
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 57 deletions.
69 changes: 31 additions & 38 deletions include/linux/rhashtable.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ struct bucket_table {

struct lockdep_map dep_map;

struct rhash_lock_head *buckets[] ____cacheline_aligned_in_smp;
struct rhash_lock_head __rcu *buckets[] ____cacheline_aligned_in_smp;
};

/*
Expand Down Expand Up @@ -261,13 +261,12 @@ void rhashtable_free_and_destroy(struct rhashtable *ht,
void *arg);
void rhashtable_destroy(struct rhashtable *ht);

struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl,
unsigned int hash);
struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl,
unsigned int hash);
struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht,
struct bucket_table *tbl,
unsigned int hash);
struct rhash_lock_head __rcu **rht_bucket_nested(
const struct bucket_table *tbl, unsigned int hash);
struct rhash_lock_head __rcu **__rht_bucket_nested(
const struct bucket_table *tbl, unsigned int hash);
struct rhash_lock_head __rcu **rht_bucket_nested_insert(
struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash);

#define rht_dereference(p, ht) \
rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht))
Expand All @@ -284,21 +283,21 @@ struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht,
#define rht_entry(tpos, pos, member) \
({ tpos = container_of(pos, typeof(*tpos), member); 1; })

static inline struct rhash_lock_head *const *rht_bucket(
static inline struct rhash_lock_head __rcu *const *rht_bucket(
const struct bucket_table *tbl, unsigned int hash)
{
return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) :
&tbl->buckets[hash];
}

static inline struct rhash_lock_head **rht_bucket_var(
static inline struct rhash_lock_head __rcu **rht_bucket_var(
struct bucket_table *tbl, unsigned int hash)
{
return unlikely(tbl->nest) ? __rht_bucket_nested(tbl, hash) :
&tbl->buckets[hash];
}

static inline struct rhash_lock_head **rht_bucket_insert(
static inline struct rhash_lock_head __rcu **rht_bucket_insert(
struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)
{
return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) :
Expand All @@ -325,15 +324,15 @@ static inline struct rhash_lock_head **rht_bucket_insert(
*/

static inline void rht_lock(struct bucket_table *tbl,
struct rhash_lock_head **bkt)
struct rhash_lock_head __rcu **bkt)
{
local_bh_disable();
bit_spin_lock(0, (unsigned long *)bkt);
lock_map_acquire(&tbl->dep_map);
}

static inline void rht_lock_nested(struct bucket_table *tbl,
struct rhash_lock_head **bucket,
struct rhash_lock_head __rcu **bucket,
unsigned int subclass)
{
local_bh_disable();
Expand All @@ -342,18 +341,18 @@ static inline void rht_lock_nested(struct bucket_table *tbl,
}

static inline void rht_unlock(struct bucket_table *tbl,
struct rhash_lock_head **bkt)
struct rhash_lock_head __rcu **bkt)
{
lock_map_release(&tbl->dep_map);
bit_spin_unlock(0, (unsigned long *)bkt);
local_bh_enable();
}

static inline struct rhash_head __rcu *__rht_ptr(
struct rhash_lock_head *const *bkt)
static inline struct rhash_head *__rht_ptr(
struct rhash_lock_head *p, struct rhash_lock_head __rcu *const *bkt)
{
return (struct rhash_head __rcu *)
((unsigned long)*bkt & ~BIT(0) ?:
return (struct rhash_head *)
((unsigned long)p & ~BIT(0) ?:
(unsigned long)RHT_NULLS_MARKER(bkt));
}

Expand All @@ -365,47 +364,41 @@ static inline struct rhash_head __rcu *__rht_ptr(
* access is guaranteed, such as when destroying the table.
*/
static inline struct rhash_head *rht_ptr_rcu(
struct rhash_lock_head *const *bkt)
struct rhash_lock_head __rcu *const *bkt)
{
struct rhash_head __rcu *p = __rht_ptr(bkt);

return rcu_dereference(p);
return __rht_ptr(rcu_dereference(*bkt), bkt);
}

static inline struct rhash_head *rht_ptr(
struct rhash_lock_head *const *bkt,
struct rhash_lock_head __rcu *const *bkt,
struct bucket_table *tbl,
unsigned int hash)
{
return rht_dereference_bucket(__rht_ptr(bkt), tbl, hash);
return __rht_ptr(rht_dereference_bucket(*bkt, tbl, hash), bkt);
}

static inline struct rhash_head *rht_ptr_exclusive(
struct rhash_lock_head *const *bkt)
struct rhash_lock_head __rcu *const *bkt)
{
return rcu_dereference_protected(__rht_ptr(bkt), 1);
return __rht_ptr(rcu_dereference_protected(*bkt, 1), bkt);
}

static inline void rht_assign_locked(struct rhash_lock_head **bkt,
static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt,
struct rhash_head *obj)
{
struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt;

if (rht_is_a_nulls(obj))
obj = NULL;
rcu_assign_pointer(*p, (void *)((unsigned long)obj | BIT(0)));
rcu_assign_pointer(*bkt, (void *)((unsigned long)obj | BIT(0)));
}

static inline void rht_assign_unlock(struct bucket_table *tbl,
struct rhash_lock_head **bkt,
struct rhash_lock_head __rcu **bkt,
struct rhash_head *obj)
{
struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt;

if (rht_is_a_nulls(obj))
obj = NULL;
lock_map_release(&tbl->dep_map);
rcu_assign_pointer(*p, obj);
rcu_assign_pointer(*bkt, (void *)obj);
preempt_enable();
__release(bitlock);
local_bh_enable();
Expand Down Expand Up @@ -593,7 +586,7 @@ static inline struct rhash_head *__rhashtable_lookup(
.ht = ht,
.key = key,
};
struct rhash_lock_head *const *bkt;
struct rhash_lock_head __rcu *const *bkt;
struct bucket_table *tbl;
struct rhash_head *he;
unsigned int hash;
Expand Down Expand Up @@ -709,7 +702,7 @@ static inline void *__rhashtable_insert_fast(
.ht = ht,
.key = key,
};
struct rhash_lock_head **bkt;
struct rhash_lock_head __rcu **bkt;
struct rhash_head __rcu **pprev;
struct bucket_table *tbl;
struct rhash_head *head;
Expand Down Expand Up @@ -995,7 +988,7 @@ static inline int __rhashtable_remove_fast_one(
struct rhash_head *obj, const struct rhashtable_params params,
bool rhlist)
{
struct rhash_lock_head **bkt;
struct rhash_lock_head __rcu **bkt;
struct rhash_head __rcu **pprev;
struct rhash_head *he;
unsigned int hash;
Expand Down Expand Up @@ -1147,7 +1140,7 @@ static inline int __rhashtable_replace_fast(
struct rhash_head *obj_old, struct rhash_head *obj_new,
const struct rhashtable_params params)
{
struct rhash_lock_head **bkt;
struct rhash_lock_head __rcu **bkt;
struct rhash_head __rcu **pprev;
struct rhash_head *he;
unsigned int hash;
Expand Down
35 changes: 16 additions & 19 deletions lib/rhashtable.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@

union nested_table {
union nested_table __rcu *table;
struct rhash_lock_head *bucket;
struct rhash_lock_head __rcu *bucket;
};

static u32 head_hashfn(struct rhashtable *ht,
Expand Down Expand Up @@ -222,7 +222,7 @@ static struct bucket_table *rhashtable_last_table(struct rhashtable *ht,
}

static int rhashtable_rehash_one(struct rhashtable *ht,
struct rhash_lock_head **bkt,
struct rhash_lock_head __rcu **bkt,
unsigned int old_hash)
{
struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
Expand Down Expand Up @@ -275,7 +275,7 @@ static int rhashtable_rehash_chain(struct rhashtable *ht,
unsigned int old_hash)
{
struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht);
struct rhash_lock_head **bkt = rht_bucket_var(old_tbl, old_hash);
struct rhash_lock_head __rcu **bkt = rht_bucket_var(old_tbl, old_hash);
int err;

if (!bkt)
Expand Down Expand Up @@ -485,7 +485,7 @@ static int rhashtable_insert_rehash(struct rhashtable *ht,
}

static void *rhashtable_lookup_one(struct rhashtable *ht,
struct rhash_lock_head **bkt,
struct rhash_lock_head __rcu **bkt,
struct bucket_table *tbl, unsigned int hash,
const void *key, struct rhash_head *obj)
{
Expand Down Expand Up @@ -535,12 +535,10 @@ static void *rhashtable_lookup_one(struct rhashtable *ht,
return ERR_PTR(-ENOENT);
}

static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht,
struct rhash_lock_head **bkt,
struct bucket_table *tbl,
unsigned int hash,
struct rhash_head *obj,
void *data)
static struct bucket_table *rhashtable_insert_one(
struct rhashtable *ht, struct rhash_lock_head __rcu **bkt,
struct bucket_table *tbl, unsigned int hash, struct rhash_head *obj,
void *data)
{
struct bucket_table *new_tbl;
struct rhash_head *head;
Expand Down Expand Up @@ -591,7 +589,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key,
{
struct bucket_table *new_tbl;
struct bucket_table *tbl;
struct rhash_lock_head **bkt;
struct rhash_lock_head __rcu **bkt;
unsigned int hash;
void *data;

Expand Down Expand Up @@ -1173,8 +1171,8 @@ void rhashtable_destroy(struct rhashtable *ht)
}
EXPORT_SYMBOL_GPL(rhashtable_destroy);

struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl,
unsigned int hash)
struct rhash_lock_head __rcu **__rht_bucket_nested(
const struct bucket_table *tbl, unsigned int hash)
{
const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
unsigned int index = hash & ((1 << tbl->nest) - 1);
Expand Down Expand Up @@ -1202,20 +1200,19 @@ struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl,
}
EXPORT_SYMBOL_GPL(__rht_bucket_nested);

struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl,
unsigned int hash)
struct rhash_lock_head __rcu **rht_bucket_nested(
const struct bucket_table *tbl, unsigned int hash)
{
static struct rhash_lock_head *rhnull;
static struct rhash_lock_head __rcu *rhnull;

if (!rhnull)
INIT_RHT_NULLS_HEAD(rhnull);
return __rht_bucket_nested(tbl, hash) ?: &rhnull;
}
EXPORT_SYMBOL_GPL(rht_bucket_nested);

struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht,
struct bucket_table *tbl,
unsigned int hash)
struct rhash_lock_head __rcu **rht_bucket_nested_insert(
struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash)
{
const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *));
unsigned int index = hash & ((1 << tbl->nest) - 1);
Expand Down

0 comments on commit a7ef23e

Please sign in to comment.