From 02512482321c531df4abf73943529f8b44d869e2 Mon Sep 17 00:00:00 2001 From: Ido Shamay Date: Fri, 21 Feb 2014 12:39:17 +0200 Subject: [PATCH 1/2] net/mlx4: Set number of RX rings in a utility function mlx4_en_add() is too long. Moving set number of RX rings to a utiltity function to improve readability and modulization of the code. Signed-off-by: Ido Shamay Signed-off-by: Amir Vadai Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx4/en_main.c | 15 ++----------- drivers/net/ethernet/mellanox/mlx4/en_rx.c | 22 ++++++++++++++++++++ drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 2 +- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_main.c b/drivers/net/ethernet/mellanox/mlx4/en_main.c index d357bf5a46860..fa2f6e76f69bb 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_main.c @@ -274,19 +274,8 @@ static void *mlx4_en_add(struct mlx4_dev *dev) if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) mlx4_en_init_timestamp(mdev); - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { - if (!dev->caps.comp_pool) { - mdev->profile.prof[i].rx_ring_num = - rounddown_pow_of_two(max_t(int, MIN_RX_RINGS, - min_t(int, - dev->caps.num_comp_vectors, - DEF_RX_RINGS))); - } else { - mdev->profile.prof[i].rx_ring_num = rounddown_pow_of_two( - min_t(int, dev->caps.comp_pool/ - dev->caps.num_ports - 1 , MAX_MSIX_P_PORT - 1)); - } - } + /* Set default number of RX rings*/ + mlx4_en_set_num_rx_rings(mdev); /* Create our own workqueue for reset/multicast tasks * Note: we cannot use the shared workqueue because of deadlocks caused diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index 890922c1c8eea..85434d73c9bce 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -318,6 +318,28 @@ static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv, } } +void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev) +{ + int i; + int num_of_eqs; + struct mlx4_dev *dev = mdev->dev; + + mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { + if (!dev->caps.comp_pool) + num_of_eqs = max_t(int, MIN_RX_RINGS, + min_t(int, + dev->caps.num_comp_vectors, + DEF_RX_RINGS)); + else + num_of_eqs = min_t(int, MAX_MSIX_P_PORT, + dev->caps.comp_pool/ + dev->caps.num_ports) - 1; + + mdev->profile.prof[i].rx_ring_num = + rounddown_pow_of_two(num_of_eqs); + } +} + int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, u32 size, u16 stride, int node) diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h index 9ca223bc90fc4..3c25c7bf3dba6 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h @@ -737,7 +737,7 @@ int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, int cq, int user_prio); void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring); - +void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev); int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring **pring, u32 size, u16 stride, int node); From bb2146bc883e86b835e30644757a6d4a649a7ce8 Mon Sep 17 00:00:00 2001 From: Ido Shamay Date: Fri, 21 Feb 2014 12:39:18 +0200 Subject: [PATCH 2/2] net/mlx4: Fix limiting number of IRQ's instead of RSS queues This fix a performance bug introduced by commit 90b1ebe "mlx4: set maximal number of default RSS queues", which limits the numbers of IRQs opened by core module. The limit should be on the number of queues in the indirection table - rx_rings, and not on the number of IRQ's. Also, limiting on mlx4_core initialization instead of in mlx4_en, prevented using "ethtool -L" to utilize all the CPU's, when performance mode is prefered, since limiting this number to 8 reduces overall packet rate by 15%-50% in multiple TCP streams applications. For example, after running ethtool -L rx 16 Packet rate Before the fix 897799 After the fix 1142070 Results were obtained using netperf: S=200 ; ( for i in $(seq 1 $S) ; do ( \ netperf -H 11.7.13.55 -t TCP_RR -l 30 &) ; \ wait ; done | grep "1 1" | awk '{SUM+=$6} END {print SUM}' ) CC: Yuval Mintz Signed-off-by: Ido Shamay Signed-off-by: Amir Vadai Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx4/en_rx.c | 5 ++++- drivers/net/ethernet/mellanox/mlx4/main.c | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index 85434d73c9bce..8afb72ec957d8 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -322,6 +322,7 @@ void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev) { int i; int num_of_eqs; + int num_rx_rings; struct mlx4_dev *dev = mdev->dev; mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { @@ -335,8 +336,10 @@ void mlx4_en_set_num_rx_rings(struct mlx4_en_dev *mdev) dev->caps.comp_pool/ dev->caps.num_ports) - 1; + num_rx_rings = min_t(int, num_of_eqs, + netif_get_num_default_rss_queues()); mdev->profile.prof[i].rx_ring_num = - rounddown_pow_of_two(num_of_eqs); + rounddown_pow_of_two(num_rx_rings); } } diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index 218b759c506e6..979ea4364efbc 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -41,7 +41,6 @@ #include #include #include -#include #include #include @@ -1974,7 +1973,7 @@ static void mlx4_enable_msi_x(struct mlx4_dev *dev) struct mlx4_priv *priv = mlx4_priv(dev); struct msix_entry *entries; int nreq = min_t(int, dev->caps.num_ports * - min_t(int, netif_get_num_default_rss_queues() + 1, + min_t(int, num_online_cpus() + 1, MAX_MSIX_P_PORT) + MSIX_LEGACY_SZ, MAX_MSIX); int i;