From 5e290dace0d6f777463f92449ec95d38c3405330 Mon Sep 17 00:00:00 2001 From: Yishai Hadas Date: Wed, 29 Aug 2012 15:14:35 +0000 Subject: [PATCH] --- yaml --- r: 322936 b: refs/heads/master c: dd03e73481564fe57655d37154f84a0af00cb449 h: refs/heads/master v: v3 --- [refs] | 2 +- trunk/drivers/infiniband/ulp/ipoib/ipoib.h | 5 +- .../drivers/infiniband/ulp/ipoib/ipoib_main.c | 93 ++++++++++--------- .../infiniband/ulp/ipoib/ipoib_multicast.c | 2 + .../drivers/net/ethernet/mellanox/mlx4/icm.c | 30 +++--- .../drivers/net/ethernet/mellanox/mlx4/icm.h | 10 +- 6 files changed, 76 insertions(+), 66 deletions(-) diff --git a/[refs] b/[refs] index eb772642b73a..24515d02d462 100644 --- a/[refs] +++ b/[refs] @@ -1,2 +1,2 @@ --- -refs/heads/master: b5120a6e11e90d98d8a752545ac60bfa1ea95f1a +refs/heads/master: dd03e73481564fe57655d37154f84a0af00cb449 diff --git a/trunk/drivers/infiniband/ulp/ipoib/ipoib.h b/trunk/drivers/infiniband/ulp/ipoib/ipoib.h index 0af216d21f87..ca43901ed861 100644 --- a/trunk/drivers/infiniband/ulp/ipoib/ipoib.h +++ b/trunk/drivers/infiniband/ulp/ipoib/ipoib.h @@ -262,10 +262,7 @@ struct ipoib_ethtool_st { u16 max_coalesced_frames; }; -struct ipoib_neigh_table; - struct ipoib_neigh_hash { - struct ipoib_neigh_table *ntbl; struct ipoib_neigh __rcu **buckets; struct rcu_head rcu; u32 mask; @@ -274,9 +271,9 @@ struct ipoib_neigh_hash { struct ipoib_neigh_table { struct ipoib_neigh_hash __rcu *htbl; + rwlock_t rwlock; atomic_t entries; struct completion flushed; - struct completion deleted; }; /* diff --git a/trunk/drivers/infiniband/ulp/ipoib/ipoib_main.c b/trunk/drivers/infiniband/ulp/ipoib/ipoib_main.c index 1e19b5ae7c47..3e2085a3ee47 100644 --- a/trunk/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/trunk/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -546,15 +546,15 @@ static void neigh_add_path(struct sk_buff *skb, u8 *daddr, struct ipoib_neigh *neigh; unsigned long flags; - spin_lock_irqsave(&priv->lock, flags); neigh = ipoib_neigh_alloc(daddr, dev); if (!neigh) { - spin_unlock_irqrestore(&priv->lock, flags); ++dev->stats.tx_dropped; dev_kfree_skb_any(skb); return; } + spin_lock_irqsave(&priv->lock, flags); + path = __path_find(dev, daddr + 4); if (!path) { path = path_rec_create(dev, daddr + 4); @@ -863,10 +863,10 @@ static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv) if (test_bit(IPOIB_STOP_NEIGH_GC, &priv->flags)) return; - spin_lock_irqsave(&priv->lock, flags); + write_lock_bh(&ntbl->rwlock); htbl = rcu_dereference_protected(ntbl->htbl, - lockdep_is_held(&priv->lock)); + lockdep_is_held(&ntbl->rwlock)); if (!htbl) goto out_unlock; @@ -883,14 +883,16 @@ static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv) struct ipoib_neigh __rcu **np = &htbl->buckets[i]; while ((neigh = rcu_dereference_protected(*np, - lockdep_is_held(&priv->lock))) != NULL) { + lockdep_is_held(&ntbl->rwlock))) != NULL) { /* was the neigh idle for two GC periods */ if (time_after(neigh_obsolete, neigh->alive)) { rcu_assign_pointer(*np, rcu_dereference_protected(neigh->hnext, - lockdep_is_held(&priv->lock))); + lockdep_is_held(&ntbl->rwlock))); /* remove from path/mc list */ + spin_lock_irqsave(&priv->lock, flags); list_del(&neigh->list); + spin_unlock_irqrestore(&priv->lock, flags); call_rcu(&neigh->rcu, ipoib_neigh_reclaim); } else { np = &neigh->hnext; @@ -900,7 +902,7 @@ static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv) } out_unlock: - spin_unlock_irqrestore(&priv->lock, flags); + write_unlock_bh(&ntbl->rwlock); } static void ipoib_reap_neigh(struct work_struct *work) @@ -945,8 +947,10 @@ struct ipoib_neigh *ipoib_neigh_alloc(u8 *daddr, struct ipoib_neigh *neigh; u32 hash_val; + write_lock_bh(&ntbl->rwlock); + htbl = rcu_dereference_protected(ntbl->htbl, - lockdep_is_held(&priv->lock)); + lockdep_is_held(&ntbl->rwlock)); if (!htbl) { neigh = NULL; goto out_unlock; @@ -957,10 +961,10 @@ struct ipoib_neigh *ipoib_neigh_alloc(u8 *daddr, */ hash_val = ipoib_addr_hash(htbl, daddr); for (neigh = rcu_dereference_protected(htbl->buckets[hash_val], - lockdep_is_held(&priv->lock)); + lockdep_is_held(&ntbl->rwlock)); neigh != NULL; neigh = rcu_dereference_protected(neigh->hnext, - lockdep_is_held(&priv->lock))) { + lockdep_is_held(&ntbl->rwlock))) { if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) { /* found, take one ref on behalf of the caller */ if (!atomic_inc_not_zero(&neigh->refcnt)) { @@ -983,11 +987,12 @@ struct ipoib_neigh *ipoib_neigh_alloc(u8 *daddr, /* put in hash */ rcu_assign_pointer(neigh->hnext, rcu_dereference_protected(htbl->buckets[hash_val], - lockdep_is_held(&priv->lock))); + lockdep_is_held(&ntbl->rwlock))); rcu_assign_pointer(htbl->buckets[hash_val], neigh); atomic_inc(&ntbl->entries); out_unlock: + write_unlock_bh(&ntbl->rwlock); return neigh; } @@ -1035,29 +1040,35 @@ void ipoib_neigh_free(struct ipoib_neigh *neigh) struct ipoib_neigh *n; u32 hash_val; + write_lock_bh(&ntbl->rwlock); + htbl = rcu_dereference_protected(ntbl->htbl, - lockdep_is_held(&priv->lock)); + lockdep_is_held(&ntbl->rwlock)); if (!htbl) - return; + goto out_unlock; hash_val = ipoib_addr_hash(htbl, neigh->daddr); np = &htbl->buckets[hash_val]; for (n = rcu_dereference_protected(*np, - lockdep_is_held(&priv->lock)); + lockdep_is_held(&ntbl->rwlock)); n != NULL; n = rcu_dereference_protected(*np, - lockdep_is_held(&priv->lock))) { + lockdep_is_held(&ntbl->rwlock))) { if (n == neigh) { /* found */ rcu_assign_pointer(*np, rcu_dereference_protected(neigh->hnext, - lockdep_is_held(&priv->lock))); + lockdep_is_held(&ntbl->rwlock))); call_rcu(&neigh->rcu, ipoib_neigh_reclaim); - return; + goto out_unlock; } else { np = &n->hnext; } } + +out_unlock: + write_unlock_bh(&ntbl->rwlock); + } static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv) @@ -1069,6 +1080,7 @@ static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv) clear_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags); ntbl->htbl = NULL; + rwlock_init(&ntbl->rwlock); htbl = kzalloc(sizeof(*htbl), GFP_KERNEL); if (!htbl) return -ENOMEM; @@ -1083,7 +1095,6 @@ static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv) htbl->mask = (size - 1); htbl->buckets = buckets; ntbl->htbl = htbl; - htbl->ntbl = ntbl; atomic_set(&ntbl->entries, 0); /* start garbage collection */ @@ -1100,11 +1111,9 @@ static void neigh_hash_free_rcu(struct rcu_head *head) struct ipoib_neigh_hash, rcu); struct ipoib_neigh __rcu **buckets = htbl->buckets; - struct ipoib_neigh_table *ntbl = htbl->ntbl; kfree(buckets); kfree(htbl); - complete(&ntbl->deleted); } void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid) @@ -1116,10 +1125,10 @@ void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid) int i; /* remove all neigh connected to a given path or mcast */ - spin_lock_irqsave(&priv->lock, flags); + write_lock_bh(&ntbl->rwlock); htbl = rcu_dereference_protected(ntbl->htbl, - lockdep_is_held(&priv->lock)); + lockdep_is_held(&ntbl->rwlock)); if (!htbl) goto out_unlock; @@ -1129,14 +1138,16 @@ void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid) struct ipoib_neigh __rcu **np = &htbl->buckets[i]; while ((neigh = rcu_dereference_protected(*np, - lockdep_is_held(&priv->lock))) != NULL) { + lockdep_is_held(&ntbl->rwlock))) != NULL) { /* delete neighs belong to this parent */ if (!memcmp(gid, neigh->daddr + 4, sizeof (union ib_gid))) { rcu_assign_pointer(*np, rcu_dereference_protected(neigh->hnext, - lockdep_is_held(&priv->lock))); + lockdep_is_held(&ntbl->rwlock))); /* remove from parent list */ + spin_lock_irqsave(&priv->lock, flags); list_del(&neigh->list); + spin_unlock_irqrestore(&priv->lock, flags); call_rcu(&neigh->rcu, ipoib_neigh_reclaim); } else { np = &neigh->hnext; @@ -1145,7 +1156,7 @@ void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid) } } out_unlock: - spin_unlock_irqrestore(&priv->lock, flags); + write_unlock_bh(&ntbl->rwlock); } static void ipoib_flush_neighs(struct ipoib_dev_priv *priv) @@ -1153,44 +1164,37 @@ static void ipoib_flush_neighs(struct ipoib_dev_priv *priv) struct ipoib_neigh_table *ntbl = &priv->ntbl; struct ipoib_neigh_hash *htbl; unsigned long flags; - int i, wait_flushed = 0; - - init_completion(&priv->ntbl.flushed); + int i; - spin_lock_irqsave(&priv->lock, flags); + write_lock_bh(&ntbl->rwlock); htbl = rcu_dereference_protected(ntbl->htbl, - lockdep_is_held(&priv->lock)); + lockdep_is_held(&ntbl->rwlock)); if (!htbl) goto out_unlock; - wait_flushed = atomic_read(&priv->ntbl.entries); - if (!wait_flushed) - goto free_htbl; - for (i = 0; i < htbl->size; i++) { struct ipoib_neigh *neigh; struct ipoib_neigh __rcu **np = &htbl->buckets[i]; while ((neigh = rcu_dereference_protected(*np, - lockdep_is_held(&priv->lock))) != NULL) { + lockdep_is_held(&ntbl->rwlock))) != NULL) { rcu_assign_pointer(*np, rcu_dereference_protected(neigh->hnext, - lockdep_is_held(&priv->lock))); + lockdep_is_held(&ntbl->rwlock))); /* remove from path/mc list */ + spin_lock_irqsave(&priv->lock, flags); list_del(&neigh->list); + spin_unlock_irqrestore(&priv->lock, flags); call_rcu(&neigh->rcu, ipoib_neigh_reclaim); } } -free_htbl: rcu_assign_pointer(ntbl->htbl, NULL); call_rcu(&htbl->rcu, neigh_hash_free_rcu); out_unlock: - spin_unlock_irqrestore(&priv->lock, flags); - if (wait_flushed) - wait_for_completion(&priv->ntbl.flushed); + write_unlock_bh(&ntbl->rwlock); } static void ipoib_neigh_hash_uninit(struct net_device *dev) @@ -1199,7 +1203,7 @@ static void ipoib_neigh_hash_uninit(struct net_device *dev) int stopped; ipoib_dbg(priv, "ipoib_neigh_hash_uninit\n"); - init_completion(&priv->ntbl.deleted); + init_completion(&priv->ntbl.flushed); set_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags); /* Stop GC if called at init fail need to cancel work */ @@ -1207,9 +1211,10 @@ static void ipoib_neigh_hash_uninit(struct net_device *dev) if (!stopped) cancel_delayed_work(&priv->neigh_reap_task); - ipoib_flush_neighs(priv); - - wait_for_completion(&priv->ntbl.deleted); + if (atomic_read(&priv->ntbl.entries)) { + ipoib_flush_neighs(priv); + wait_for_completion(&priv->ntbl.flushed); + } } diff --git a/trunk/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/trunk/drivers/infiniband/ulp/ipoib/ipoib_multicast.c index 75367249f447..13f4aa7593c8 100644 --- a/trunk/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +++ b/trunk/drivers/infiniband/ulp/ipoib/ipoib_multicast.c @@ -707,7 +707,9 @@ void ipoib_mcast_send(struct net_device *dev, u8 *daddr, struct sk_buff *skb) neigh = ipoib_neigh_get(dev, daddr); spin_lock_irqsave(&priv->lock, flags); if (!neigh) { + spin_unlock_irqrestore(&priv->lock, flags); neigh = ipoib_neigh_alloc(daddr, dev); + spin_lock_irqsave(&priv->lock, flags); if (neigh) { kref_get(&mcast->ah->ref); neigh->ah = mcast->ah; diff --git a/trunk/drivers/net/ethernet/mellanox/mlx4/icm.c b/trunk/drivers/net/ethernet/mellanox/mlx4/icm.c index daf417923661..31d02649be41 100644 --- a/trunk/drivers/net/ethernet/mellanox/mlx4/icm.c +++ b/trunk/drivers/net/ethernet/mellanox/mlx4/icm.c @@ -227,9 +227,10 @@ int mlx4_UNMAP_ICM_AUX(struct mlx4_dev *dev) MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); } -int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj) +int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj) { - int i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size); + u32 i = (obj & (table->num_obj - 1)) / + (MLX4_TABLE_CHUNK_SIZE / table->obj_size); int ret = 0; mutex_lock(&table->mutex); @@ -262,16 +263,18 @@ int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj) return ret; } -void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj) +void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj) { - int i; + u32 i; + u64 offset; i = (obj & (table->num_obj - 1)) / (MLX4_TABLE_CHUNK_SIZE / table->obj_size); mutex_lock(&table->mutex); if (--table->icm[i]->refcount == 0) { - mlx4_UNMAP_ICM(dev, table->virt + i * MLX4_TABLE_CHUNK_SIZE, + offset = (u64) i * MLX4_TABLE_CHUNK_SIZE; + mlx4_UNMAP_ICM(dev, table->virt + offset, MLX4_TABLE_CHUNK_SIZE / MLX4_ICM_PAGE_SIZE); mlx4_free_icm(dev, table->icm[i], table->coherent); table->icm[i] = NULL; @@ -280,9 +283,11 @@ void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj) mutex_unlock(&table->mutex); } -void *mlx4_table_find(struct mlx4_icm_table *table, int obj, dma_addr_t *dma_handle) +void *mlx4_table_find(struct mlx4_icm_table *table, u32 obj, + dma_addr_t *dma_handle) { - int idx, offset, dma_offset, i; + int offset, dma_offset, i; + u64 idx; struct mlx4_icm_chunk *chunk; struct mlx4_icm *icm; struct page *page = NULL; @@ -292,7 +297,7 @@ void *mlx4_table_find(struct mlx4_icm_table *table, int obj, dma_addr_t *dma_han mutex_lock(&table->mutex); - idx = (obj & (table->num_obj - 1)) * table->obj_size; + idx = (u64) (obj & (table->num_obj - 1)) * table->obj_size; icm = table->icm[idx / MLX4_TABLE_CHUNK_SIZE]; dma_offset = offset = idx % MLX4_TABLE_CHUNK_SIZE; @@ -326,10 +331,11 @@ void *mlx4_table_find(struct mlx4_icm_table *table, int obj, dma_addr_t *dma_han } int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - int start, int end) + u32 start, u32 end) { int inc = MLX4_TABLE_CHUNK_SIZE / table->obj_size; - int i, err; + int err; + u32 i; for (i = start; i <= end; i += inc) { err = mlx4_table_get(dev, table, i); @@ -349,9 +355,9 @@ int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, } void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - int start, int end) + u32 start, u32 end) { - int i; + u32 i; for (i = start; i <= end; i += MLX4_TABLE_CHUNK_SIZE / table->obj_size) mlx4_table_put(dev, table, i); diff --git a/trunk/drivers/net/ethernet/mellanox/mlx4/icm.h b/trunk/drivers/net/ethernet/mellanox/mlx4/icm.h index a67744f53506..dee67fa39107 100644 --- a/trunk/drivers/net/ethernet/mellanox/mlx4/icm.h +++ b/trunk/drivers/net/ethernet/mellanox/mlx4/icm.h @@ -71,17 +71,17 @@ struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages, gfp_t gfp_mask, int coherent); void mlx4_free_icm(struct mlx4_dev *dev, struct mlx4_icm *icm, int coherent); -int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj); -void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, int obj); +int mlx4_table_get(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj); +void mlx4_table_put(struct mlx4_dev *dev, struct mlx4_icm_table *table, u32 obj); int mlx4_table_get_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - int start, int end); + u32 start, u32 end); void mlx4_table_put_range(struct mlx4_dev *dev, struct mlx4_icm_table *table, - int start, int end); + u32 start, u32 end); int mlx4_init_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table, u64 virt, int obj_size, u32 nobj, int reserved, int use_lowmem, int use_coherent); void mlx4_cleanup_icm_table(struct mlx4_dev *dev, struct mlx4_icm_table *table); -void *mlx4_table_find(struct mlx4_icm_table *table, int obj, dma_addr_t *dma_handle); +void *mlx4_table_find(struct mlx4_icm_table *table, u32 obj, dma_addr_t *dma_handle); static inline void mlx4_icm_first(struct mlx4_icm *icm, struct mlx4_icm_iter *iter)