From ba6306e3f648a857ae52ddcabc2859542fd2f94c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 16 May 2019 15:19:46 +0800 Subject: [PATCH 1/2] rhashtable: Remove RCU marking from rhash_lock_head The opaque type rhash_lock_head should not be marked with __rcu because it can never be dereferenced. We should apply the RCU marking when we turn it into a pointer which can be dereferenced. This patch does exactly that. This fixes a number of sparse warnings as well as getting rid of some unnecessary RCU checking. Signed-off-by: Herbert Xu Signed-off-by: David S. Miller --- include/linux/rhashtable.h | 58 +++++++++++++++++++++----------------- lib/rhashtable.c | 28 +++++++++--------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h index f7714d3b46bd0..9f8bc06d41368 100644 --- a/include/linux/rhashtable.h +++ b/include/linux/rhashtable.h @@ -84,7 +84,7 @@ struct bucket_table { struct lockdep_map dep_map; - struct rhash_lock_head __rcu *buckets[] ____cacheline_aligned_in_smp; + struct rhash_lock_head *buckets[] ____cacheline_aligned_in_smp; }; /* @@ -261,13 +261,13 @@ void rhashtable_free_and_destroy(struct rhashtable *ht, void *arg); void rhashtable_destroy(struct rhashtable *ht); -struct rhash_lock_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, - unsigned int hash); -struct rhash_lock_head __rcu **__rht_bucket_nested(const struct bucket_table *tbl, - unsigned int hash); -struct rhash_lock_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, - struct bucket_table *tbl, - unsigned int hash); +struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl, + unsigned int hash); +struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl, + unsigned int hash); +struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht, + struct bucket_table *tbl, + unsigned int hash); #define rht_dereference(p, ht) \ rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht)) @@ -284,21 +284,21 @@ struct rhash_lock_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, #define rht_entry(tpos, pos, member) \ ({ tpos = container_of(pos, typeof(*tpos), member); 1; }) -static inline struct rhash_lock_head __rcu *const *rht_bucket( +static inline struct rhash_lock_head *const *rht_bucket( const struct bucket_table *tbl, unsigned int hash) { return unlikely(tbl->nest) ? rht_bucket_nested(tbl, hash) : &tbl->buckets[hash]; } -static inline struct rhash_lock_head __rcu **rht_bucket_var( +static inline struct rhash_lock_head **rht_bucket_var( struct bucket_table *tbl, unsigned int hash) { return unlikely(tbl->nest) ? __rht_bucket_nested(tbl, hash) : &tbl->buckets[hash]; } -static inline struct rhash_lock_head __rcu **rht_bucket_insert( +static inline struct rhash_lock_head **rht_bucket_insert( struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash) { return unlikely(tbl->nest) ? rht_bucket_nested_insert(ht, tbl, hash) : @@ -349,6 +349,12 @@ static inline void rht_unlock(struct bucket_table *tbl, local_bh_enable(); } +static inline struct rhash_head __rcu *__rht_ptr( + struct rhash_lock_head *const *bkt) +{ + return (struct rhash_head __rcu *)((unsigned long)*bkt & ~BIT(0)); +} + /* * Where 'bkt' is a bucket and might be locked: * rht_ptr() dereferences that pointer and clears the lock bit. @@ -356,30 +362,30 @@ static inline void rht_unlock(struct bucket_table *tbl, * access is guaranteed, such as when destroying the table. */ static inline struct rhash_head *rht_ptr( - struct rhash_lock_head __rcu * const *bkt, + struct rhash_lock_head *const *bkt, struct bucket_table *tbl, unsigned int hash) { - const struct rhash_lock_head *p = - rht_dereference_bucket_rcu(*bkt, tbl, hash); + struct rhash_head __rcu *p = __rht_ptr(bkt); - if ((((unsigned long)p) & ~BIT(0)) == 0) + if (!p) return RHT_NULLS_MARKER(bkt); - return (void *)(((unsigned long)p) & ~BIT(0)); + + return rht_dereference_bucket_rcu(p, tbl, hash); } static inline struct rhash_head *rht_ptr_exclusive( - struct rhash_lock_head __rcu * const *bkt) + struct rhash_lock_head *const *bkt) { - const struct rhash_lock_head *p = - rcu_dereference_protected(*bkt, 1); + struct rhash_head __rcu *p = __rht_ptr(bkt); if (!p) return RHT_NULLS_MARKER(bkt); - return (void *)(((unsigned long)p) & ~BIT(0)); + + return rcu_dereference_protected(p, 1); } -static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt, +static inline void rht_assign_locked(struct rhash_lock_head **bkt, struct rhash_head *obj) { struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt; @@ -390,7 +396,7 @@ static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt, } static inline void rht_assign_unlock(struct bucket_table *tbl, - struct rhash_lock_head __rcu **bkt, + struct rhash_lock_head **bkt, struct rhash_head *obj) { struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt; @@ -587,7 +593,7 @@ static inline struct rhash_head *__rhashtable_lookup( .ht = ht, .key = key, }; - struct rhash_lock_head __rcu * const *bkt; + struct rhash_lock_head *const *bkt; struct bucket_table *tbl; struct rhash_head *he; unsigned int hash; @@ -703,7 +709,7 @@ static inline void *__rhashtable_insert_fast( .ht = ht, .key = key, }; - struct rhash_lock_head __rcu **bkt; + struct rhash_lock_head **bkt; struct rhash_head __rcu **pprev; struct bucket_table *tbl; struct rhash_head *head; @@ -989,7 +995,7 @@ static inline int __rhashtable_remove_fast_one( struct rhash_head *obj, const struct rhashtable_params params, bool rhlist) { - struct rhash_lock_head __rcu **bkt; + struct rhash_lock_head **bkt; struct rhash_head __rcu **pprev; struct rhash_head *he; unsigned int hash; @@ -1141,7 +1147,7 @@ static inline int __rhashtable_replace_fast( struct rhash_head *obj_old, struct rhash_head *obj_new, const struct rhashtable_params params) { - struct rhash_lock_head __rcu **bkt; + struct rhash_lock_head **bkt; struct rhash_head __rcu **pprev; struct rhash_head *he; unsigned int hash; diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 6529fe1b45c13..7708699a5b96d 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -34,7 +34,7 @@ union nested_table { union nested_table __rcu *table; - struct rhash_lock_head __rcu *bucket; + struct rhash_lock_head *bucket; }; static u32 head_hashfn(struct rhashtable *ht, @@ -216,7 +216,7 @@ static struct bucket_table *rhashtable_last_table(struct rhashtable *ht, } static int rhashtable_rehash_one(struct rhashtable *ht, - struct rhash_lock_head __rcu **bkt, + struct rhash_lock_head **bkt, unsigned int old_hash) { struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); @@ -269,7 +269,7 @@ static int rhashtable_rehash_chain(struct rhashtable *ht, unsigned int old_hash) { struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); - struct rhash_lock_head __rcu **bkt = rht_bucket_var(old_tbl, old_hash); + struct rhash_lock_head **bkt = rht_bucket_var(old_tbl, old_hash); int err; if (!bkt) @@ -478,7 +478,7 @@ static int rhashtable_insert_rehash(struct rhashtable *ht, } static void *rhashtable_lookup_one(struct rhashtable *ht, - struct rhash_lock_head __rcu **bkt, + struct rhash_lock_head **bkt, struct bucket_table *tbl, unsigned int hash, const void *key, struct rhash_head *obj) { @@ -529,7 +529,7 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, } static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht, - struct rhash_lock_head __rcu **bkt, + struct rhash_lock_head **bkt, struct bucket_table *tbl, unsigned int hash, struct rhash_head *obj, @@ -584,7 +584,7 @@ static void *rhashtable_try_insert(struct rhashtable *ht, const void *key, { struct bucket_table *new_tbl; struct bucket_table *tbl; - struct rhash_lock_head __rcu **bkt; + struct rhash_lock_head **bkt; unsigned int hash; void *data; @@ -1166,8 +1166,8 @@ void rhashtable_destroy(struct rhashtable *ht) } EXPORT_SYMBOL_GPL(rhashtable_destroy); -struct rhash_lock_head __rcu **__rht_bucket_nested(const struct bucket_table *tbl, - unsigned int hash) +struct rhash_lock_head **__rht_bucket_nested(const struct bucket_table *tbl, + unsigned int hash) { const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); unsigned int index = hash & ((1 << tbl->nest) - 1); @@ -1195,10 +1195,10 @@ struct rhash_lock_head __rcu **__rht_bucket_nested(const struct bucket_table *tb } EXPORT_SYMBOL_GPL(__rht_bucket_nested); -struct rhash_lock_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, - unsigned int hash) +struct rhash_lock_head **rht_bucket_nested(const struct bucket_table *tbl, + unsigned int hash) { - static struct rhash_lock_head __rcu *rhnull; + static struct rhash_lock_head *rhnull; if (!rhnull) INIT_RHT_NULLS_HEAD(rhnull); @@ -1206,9 +1206,9 @@ struct rhash_lock_head __rcu **rht_bucket_nested(const struct bucket_table *tbl, } EXPORT_SYMBOL_GPL(rht_bucket_nested); -struct rhash_lock_head __rcu **rht_bucket_nested_insert(struct rhashtable *ht, - struct bucket_table *tbl, - unsigned int hash) +struct rhash_lock_head **rht_bucket_nested_insert(struct rhashtable *ht, + struct bucket_table *tbl, + unsigned int hash) { const unsigned int shift = PAGE_SHIFT - ilog2(sizeof(void *)); unsigned int index = hash & ((1 << tbl->nest) - 1); From e9458a4e337db565191e3f1cee71bb553802def4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 16 May 2019 15:19:48 +0800 Subject: [PATCH 2/2] rhashtable: Fix cmpxchg RCU warnings As cmpxchg is a non-RCU mechanism it will cause sparse warnings when we use it for RCU. This patch adds explicit casts to silence those warnings. This should probably be moved to RCU itself in future. Signed-off-by: Herbert Xu Signed-off-by: David S. Miller --- lib/rhashtable.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 7708699a5b96d..935ec80f213ff 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -131,7 +131,7 @@ static union nested_table *nested_table_alloc(struct rhashtable *ht, INIT_RHT_NULLS_HEAD(ntbl[i].bucket); } - if (cmpxchg(prev, NULL, ntbl) == NULL) + if (cmpxchg((union nested_table **)prev, NULL, ntbl) == NULL) return ntbl; /* Raced with another thread. */ kfree(ntbl); @@ -296,7 +296,8 @@ static int rhashtable_rehash_attach(struct rhashtable *ht, * rcu_assign_pointer(). */ - if (cmpxchg(&old_tbl->future_tbl, NULL, new_tbl) != NULL) + if (cmpxchg((struct bucket_table **)&old_tbl->future_tbl, NULL, + new_tbl) != NULL) return -EEXIST; return 0;