From ea645f97bcec90a818dc6915d84cd0f752d9ccfd Mon Sep 17 00:00:00 2001 From: Roi Dayan Date: Thu, 27 Oct 2022 11:35:12 +0300 Subject: [PATCH 01/15] net/mlx5: Bridge, Use debug instead of warn if entry doesn't exists There is no need for the warn if entry already removed. Use debug print like in the update flow. Also update the messages so user can identify if the it's from the update flow or remove flow. Signed-off-by: Roi Dayan Reviewed-by: Vlad Buslov Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c index 4fbff7bcc1556..b176648d1343a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c @@ -1722,7 +1722,7 @@ void mlx5_esw_bridge_fdb_update_used(struct net_device *dev, u16 vport_num, u16 entry = mlx5_esw_bridge_fdb_lookup(bridge, fdb_info->addr, fdb_info->vid); if (!entry) { esw_debug(br_offloads->esw->dev, - "FDB entry with specified key not found (MAC=%pM,vid=%u,vport=%u)\n", + "FDB update entry with specified key not found (MAC=%pM,vid=%u,vport=%u)\n", fdb_info->addr, fdb_info->vid, vport_num); return; } @@ -1775,9 +1775,9 @@ void mlx5_esw_bridge_fdb_remove(struct net_device *dev, u16 vport_num, u16 esw_o bridge = port->bridge; entry = mlx5_esw_bridge_fdb_lookup(bridge, fdb_info->addr, fdb_info->vid); if (!entry) { - esw_warn(esw->dev, - "FDB entry with specified key not found (MAC=%pM,vid=%u,vport=%u)\n", - fdb_info->addr, fdb_info->vid, vport_num); + esw_debug(esw->dev, + "FDB remove entry with specified key not found (MAC=%pM,vid=%u,vport=%u)\n", + fdb_info->addr, fdb_info->vid, vport_num); return; } From d23b928befda3524f12d8da29921ea066d72bde7 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Mon, 31 Oct 2022 08:01:04 +0000 Subject: [PATCH 02/15] net/mlx5: Fix spelling mistake "destoy" -> "destroy" There is a spelling mistake in an error message. Fix it. Signed-off-by: Colin Ian King Reviewed-by: Leon Romanovsky Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c index eb81759244d50..9c3dfd68f8dfa 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_table.c @@ -292,7 +292,7 @@ int mlx5dr_table_destroy(struct mlx5dr_table *tbl) mlx5dr_dbg_tbl_del(tbl); ret = dr_table_destroy_sw_owned_tbl(tbl); if (ret) - mlx5dr_err(tbl->dmn, "Failed to destoy sw owned table\n"); + mlx5dr_err(tbl->dmn, "Failed to destroy sw owned table\n"); dr_table_uninit(tbl); From 71b75f0e02ed160ccbb4fe609036abe83aa32054 Mon Sep 17 00:00:00 2001 From: Moshe Shemesh Date: Mon, 8 Aug 2022 20:02:59 +0300 Subject: [PATCH 03/15] net/mlx5: Unregister traps on driver unload flow Before this patch, devlink traps are registered only on full driver probe and unregistered on driver removal. As devlink traps are not usable once driver functionality is unloaded, it should be unrgeistered also on flows that unload the driver and then registered when loaded back, e.g. devlink reload flow. Signed-off-by: Moshe Shemesh Reviewed-by: Aya Levin Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/devlink.c | 11 ++--------- drivers/net/ethernet/mellanox/mlx5/core/devlink.h | 2 ++ drivers/net/ethernet/mellanox/mlx5/core/main.c | 8 ++++++++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index 66c6a7017695d..cc2ae427dcb05 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -840,7 +840,7 @@ static const struct devlink_trap_group mlx5_trap_groups_arr[] = { DEVLINK_TRAP_GROUP_GENERIC(L2_DROPS, 0), }; -static int mlx5_devlink_traps_register(struct devlink *devlink) +int mlx5_devlink_traps_register(struct devlink *devlink) { struct mlx5_core_dev *core_dev = devlink_priv(devlink); int err; @@ -862,7 +862,7 @@ static int mlx5_devlink_traps_register(struct devlink *devlink) return err; } -static void mlx5_devlink_traps_unregister(struct devlink *devlink) +void mlx5_devlink_traps_unregister(struct devlink *devlink) { devl_traps_unregister(devlink, mlx5_traps_arr, ARRAY_SIZE(mlx5_traps_arr)); devl_trap_groups_unregister(devlink, mlx5_trap_groups_arr, @@ -889,17 +889,11 @@ int mlx5_devlink_register(struct devlink *devlink) if (err) goto max_uc_list_err; - err = mlx5_devlink_traps_register(devlink); - if (err) - goto traps_reg_err; - if (!mlx5_core_is_mp_slave(dev)) devlink_set_features(devlink, DEVLINK_F_RELOAD); return 0; -traps_reg_err: - mlx5_devlink_max_uc_list_param_unregister(devlink); max_uc_list_err: mlx5_devlink_auxdev_params_unregister(devlink); auxdev_reg_err: @@ -910,7 +904,6 @@ int mlx5_devlink_register(struct devlink *devlink) void mlx5_devlink_unregister(struct devlink *devlink) { - mlx5_devlink_traps_unregister(devlink); mlx5_devlink_max_uc_list_param_unregister(devlink); mlx5_devlink_auxdev_params_unregister(devlink); devlink_params_unregister(devlink, mlx5_devlink_params, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h index 30bf4882779b7..fd033df248560 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h @@ -30,6 +30,8 @@ void mlx5_devlink_trap_report(struct mlx5_core_dev *dev, int trap_id, struct sk_ int mlx5_devlink_trap_get_num_active(struct mlx5_core_dev *dev); int mlx5_devlink_traps_get_action(struct mlx5_core_dev *dev, int trap_id, enum devlink_trap_action *action); +int mlx5_devlink_traps_register(struct devlink *devlink); +void mlx5_devlink_traps_unregister(struct devlink *devlink); struct devlink *mlx5_devlink_alloc(struct device *dev); void mlx5_devlink_free(struct devlink *devlink); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 283c4cc28944d..9e6da51b74812 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1306,8 +1306,15 @@ static int mlx5_load(struct mlx5_core_dev *dev) mlx5_sf_dev_table_create(dev); + err = mlx5_devlink_traps_register(priv_to_devlink(dev)); + if (err) + goto err_traps_reg; + return 0; +err_traps_reg: + mlx5_sf_dev_table_destroy(dev); + mlx5_sriov_detach(dev); err_sriov: mlx5_lag_remove_mdev(dev); mlx5_ec_cleanup(dev); @@ -1336,6 +1343,7 @@ static int mlx5_load(struct mlx5_core_dev *dev) static void mlx5_unload(struct mlx5_core_dev *dev) { + mlx5_devlink_traps_unregister(priv_to_devlink(dev)); mlx5_sf_dev_table_destroy(dev); mlx5_sriov_detach(dev); mlx5_eswitch_disable(dev->priv.eswitch); From dd3dd7263cdeac601d0ecc5f04d87eee0a8277aa Mon Sep 17 00:00:00 2001 From: Eli Cohen Date: Wed, 21 Sep 2022 13:33:29 +0300 Subject: [PATCH 04/15] net/mlx5: Expose vhca_id to debugfs hca_id is an identifier of an mlx5_core instance within the hardware. This identifier may be required for troubleshooting. Expose it to debugfs. Example: $ cat /sys/kernel/debug/mlx5/mlx5_core.sf.2/vhca_id 0x12 Signed-off-by: Eli Cohen Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/main.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 9e6da51b74812..6d7c102861ea4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1588,6 +1588,16 @@ static int mlx5_hca_caps_alloc(struct mlx5_core_dev *dev) return -ENOMEM; } +static int vhca_id_show(struct seq_file *file, void *priv) +{ + struct mlx5_core_dev *dev = file->private; + + seq_printf(file, "0x%x\n", MLX5_CAP_GEN(dev, vhca_id)); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(vhca_id); + int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) { struct mlx5_priv *priv = &dev->priv; @@ -1612,6 +1622,7 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) priv->numa_node = dev_to_node(mlx5_core_dma_dev(dev)); priv->dbg.dbg_root = debugfs_create_dir(dev_name(dev->device), mlx5_debugfs_root); + debugfs_create_file("vhca_id", 0400, priv->dbg.dbg_root, dev, &vhca_id_fops); INIT_LIST_HEAD(&priv->traps); err = mlx5_tout_init(dev); From 60551e95a864f2ba82b98defcf85306599341918 Mon Sep 17 00:00:00 2001 From: Anisse Astier Date: Mon, 31 Oct 2022 17:56:04 +0100 Subject: [PATCH 05/15] net/mlx5e: remove unused list in arfs This is never used, and probably something that was intended to be used before per-protocol hash tables were chosen instead. Signed-off-by: Anisse Astier Reviewed-by: Leon Romanovsky Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c index 0ae1865086ff1..bed0c2d043e70 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c @@ -57,7 +57,6 @@ struct mlx5e_arfs_tables { struct arfs_table arfs_tables[ARFS_NUM_TYPES]; /* Protect aRFS rules list */ spinlock_t arfs_lock; - struct list_head rules; int last_filter_id; struct workqueue_struct *wq; }; @@ -376,7 +375,6 @@ int mlx5e_arfs_create_tables(struct mlx5e_flow_steering *fs, return -ENOMEM; spin_lock_init(&arfs->arfs_lock); - INIT_LIST_HEAD(&arfs->rules); arfs->wq = create_singlethread_workqueue("mlx5e_arfs"); if (!arfs->wq) goto err; From 9458108040b3c0980a02308ddf30568b9823349c Mon Sep 17 00:00:00 2001 From: Gal Pressman Date: Sun, 4 Sep 2022 13:29:26 +0300 Subject: [PATCH 06/15] net/mlx5e: Use clamp operation instead of open coding it Replace the min/max operations with a single clamp. Signed-off-by: Gal Pressman Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_tc.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 5a6aa61ec82af..3782f00972922 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -1060,12 +1060,9 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, hash_hairpin_info(peer_id, match_prio)); mutex_unlock(&tc->hairpin_tbl_lock); - params.log_data_size = 16; - params.log_data_size = min_t(u8, params.log_data_size, - MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz)); - params.log_data_size = max_t(u8, params.log_data_size, - MLX5_CAP_GEN(priv->mdev, log_min_hairpin_wq_data_sz)); - + params.log_data_size = clamp_t(u8, 16, + MLX5_CAP_GEN(priv->mdev, log_min_hairpin_wq_data_sz), + MLX5_CAP_GEN(priv->mdev, log_max_hairpin_wq_data_sz)); params.log_num_packets = params.log_data_size - MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(priv->mdev); params.log_num_packets = min_t(u8, params.log_num_packets, From 2c925db0a7d69b404d6bfe4c037935c2d367913d Mon Sep 17 00:00:00 2001 From: Ofer Levi Date: Tue, 9 Feb 2021 17:48:11 +0200 Subject: [PATCH 07/15] net/mlx5e: Support enhanced CQE compression CQE compression feature improves performance by reducing PCI bandwidth bottleneck on CQEs write. Enhanced CQE compression introduced in ConnectX-6 and it aims to reduce CPU utilization of SW side packets decompression by eliminating the need to rewrite ownership bit, which is likely to cost a cache-miss, is replaced by validity byte handled solely by HW. Another advantage of the enhanced feature is that session packets are available to SW as soon as a single CQE slot is filled, instead of waiting for session to close, this improves packet latency from NIC to host. Performance: Following are tested scenarios and reults comparing basic and enahnced CQE compression. setup: IXIA 100GbE connected directly to port 0 and port 1 of ConnectX-6 Dx 100GbE dual port. Case #1 RX only, single flow goes to single queue: IRQ rate reduced by ~ 30%, CPU utilization improved by 2%. Case #2 IP forwarding from port 1 to port 0 single flow goes to single queue: Avg latency improved from 60us to 21us, frame loss improved from 0.5% to 0.0%. Case #3 IP forwarding from port 1 to port 0 Max Throughput IXIA sends 100%, 8192 UDP flows, goes to 24 queues: Enhanced is equal or slightly better than basic. Testing the basic compression feature with this patch shows there is no perfrormance degradation of the basic compression feature. Signed-off-by: Ofer Levi Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en.h | 2 + .../ethernet/mellanox/mlx5/core/en/params.c | 10 +- .../net/ethernet/mellanox/mlx5/core/en_main.c | 8 + .../net/ethernet/mellanox/mlx5/core/en_rx.c | 150 +++++++++++++++--- drivers/net/ethernet/mellanox/mlx5/core/wq.h | 17 ++ include/linux/mlx5/device.h | 6 + 6 files changed, 170 insertions(+), 23 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 26a23047f1f3b..ff5b302531d51 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -344,6 +344,7 @@ enum { MLX5E_RQ_STATE_CSUM_FULL, /* cqe_csum_full hw bit is set */ MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, /* set when mini_cqe_resp_stride_index cap is used */ MLX5E_RQ_STATE_SHAMPO, /* set when SHAMPO cap is used */ + MLX5E_RQ_STATE_MINI_CQE_ENHANCED, /* set when enhanced mini_cqe_cap is used */ }; struct mlx5e_cq { @@ -370,6 +371,7 @@ struct mlx5e_cq_decomp { u8 mini_arr_idx; u16 left; u16 wqe_counter; + bool last_cqe_title; } ____cacheline_aligned_in_smp; enum mlx5e_dma_map_type { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 29dd3a04c1545..1a2de9bc65382 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -608,13 +608,15 @@ void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE : MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE; - mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n", + mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d %s)\n", params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ, params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ? BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, NULL)) : BIT(params->log_rq_mtu_frames), BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)), - MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)); + MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS), + MLX5_CAP_GEN(mdev, enhanced_cqe_compression) ? + "enhanced" : "basic"); } void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params) @@ -852,6 +854,10 @@ static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev, if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { MLX5_SET(cqc, cqc, mini_cqe_res_format, hw_stridx ? MLX5_CQE_FORMAT_CSUM_STRIDX : MLX5_CQE_FORMAT_CSUM); + MLX5_SET(cqc, cqc, cqe_compression_layout, + MLX5_CAP_GEN(mdev, enhanced_cqe_compression) ? + MLX5_CQE_COMPRESS_LAYOUT_ENHANCED : + MLX5_CQE_COMPRESS_LAYOUT_BASIC); MLX5_SET(cqc, cqc, cqe_comp_en, 1); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 1669c7d7f285b..c462b76743b64 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -1205,6 +1205,13 @@ int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state); + /* For enhanced CQE compression packet processing. decompress + * session according to the enhanced layout. + */ + if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) && + MLX5_CAP_GEN(mdev, enhanced_cqe_compression)) + __set_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state); + return 0; err_destroy_rq: @@ -1895,6 +1902,7 @@ static int mlx5e_alloc_cq_common(struct mlx5e_priv *priv, struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); cqe->op_own = 0xf1; + cqe->validity_iteration_count = 0xff; } cq->mdev = mdev; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index a61a43fc8d5c5..b1ea0b995d9c9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -89,6 +89,25 @@ static inline void mlx5e_read_cqe_slot(struct mlx5_cqwq *wq, memcpy(data, mlx5_cqwq_get_wqe(wq, ci), sizeof(struct mlx5_cqe64)); } +static void mlx5e_read_enhanced_title_slot(struct mlx5e_rq *rq, + struct mlx5_cqe64 *cqe) +{ + struct mlx5e_cq_decomp *cqd = &rq->cqd; + struct mlx5_cqe64 *title = &cqd->title; + + memcpy(title, cqe, sizeof(struct mlx5_cqe64)); + + if (likely(test_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state))) + return; + + if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) + cqd->wqe_counter = mpwrq_get_cqe_stride_index(title) + + mpwrq_get_cqe_consumed_strides(title); + else + cqd->wqe_counter = + mlx5_wq_cyc_ctr2ix(&rq->wqe.wq, be16_to_cpu(title->wqe_counter) + 1); +} + static inline void mlx5e_read_title_slot(struct mlx5e_rq *rq, struct mlx5_cqwq *wq, u32 cqcc) @@ -175,6 +194,38 @@ static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq *rq, cqd->title.rss_hash_result = 0; } +static u32 mlx5e_decompress_enhanced_cqe(struct mlx5e_rq *rq, + struct mlx5_cqwq *wq, + struct mlx5_cqe64 *cqe, + int budget_rem) +{ + struct mlx5e_cq_decomp *cqd = &rq->cqd; + u32 cqcc, left; + u32 i; + + left = get_cqe_enhanced_num_mini_cqes(cqe); + /* Here we avoid breaking the cqe compression session in the middle + * in case budget is not sufficient to handle all of it. In this case + * we return work_done == budget_rem to give 'busy' napi indication. + */ + if (unlikely(left > budget_rem)) + return budget_rem; + + cqcc = wq->cc; + cqd->mini_arr_idx = 0; + memcpy(cqd->mini_arr, cqe, sizeof(struct mlx5_cqe64)); + for (i = 0; i < left; i++, cqd->mini_arr_idx++, cqcc++) { + mlx5e_decompress_cqe_no_hash(rq, wq, cqcc); + INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq, + mlx5e_handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq_shampo, + rq, &cqd->title); + } + wq->cc = cqcc; + rq->stats->cqe_compress_pkts += left; + + return left; +} + static inline u32 mlx5e_decompress_cqes_cont(struct mlx5e_rq *rq, struct mlx5_cqwq *wq, int update_owner_only, @@ -220,7 +271,7 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq, rq, &cqd->title); cqd->mini_arr_idx++; - return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem) - 1; + return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem); } static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq, struct page *page) @@ -2211,45 +2262,102 @@ static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cq mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index); } -int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) +static int mlx5e_rx_cq_process_enhanced_cqe_comp(struct mlx5e_rq *rq, + struct mlx5_cqwq *cqwq, + int budget_rem) { - struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq); - struct mlx5_cqwq *cqwq = &cq->wq; - struct mlx5_cqe64 *cqe; + struct mlx5_cqe64 *cqe, *title_cqe = NULL; + struct mlx5e_cq_decomp *cqd = &rq->cqd; int work_done = 0; - if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) - return 0; + cqe = mlx5_cqwq_get_cqe_enahnced_comp(cqwq); + if (!cqe) + return work_done; - if (rq->cqd.left) { - work_done += mlx5e_decompress_cqes_cont(rq, cqwq, 0, budget); - if (work_done >= budget) - goto out; + if (cqd->last_cqe_title && + (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED)) { + rq->stats->cqe_compress_blks++; + cqd->last_cqe_title = false; } - cqe = mlx5_cqwq_get_cqe(cqwq); - if (!cqe) { - if (unlikely(work_done)) - goto out; - return 0; + do { + if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) { + if (title_cqe) { + mlx5e_read_enhanced_title_slot(rq, title_cqe); + title_cqe = NULL; + rq->stats->cqe_compress_blks++; + } + work_done += + mlx5e_decompress_enhanced_cqe(rq, cqwq, cqe, + budget_rem - work_done); + continue; + } + title_cqe = cqe; + mlx5_cqwq_pop(cqwq); + + INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq, + mlx5e_handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq_shampo, + rq, cqe); + work_done++; + } while (work_done < budget_rem && + (cqe = mlx5_cqwq_get_cqe_enahnced_comp(cqwq))); + + /* last cqe might be title on next poll bulk */ + if (title_cqe) { + mlx5e_read_enhanced_title_slot(rq, title_cqe); + cqd->last_cqe_title = true; } - do { + return work_done; +} + +static int mlx5e_rx_cq_process_basic_cqe_comp(struct mlx5e_rq *rq, + struct mlx5_cqwq *cqwq, + int budget_rem) +{ + struct mlx5_cqe64 *cqe; + int work_done = 0; + + if (rq->cqd.left) + work_done += mlx5e_decompress_cqes_cont(rq, cqwq, 0, budget_rem); + + while (work_done < budget_rem && (cqe = mlx5_cqwq_get_cqe(cqwq))) { if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) { work_done += mlx5e_decompress_cqes_start(rq, cqwq, - budget - work_done); + budget_rem - work_done); continue; } mlx5_cqwq_pop(cqwq); - INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq, mlx5e_handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq_shampo, rq, cqe); - } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(cqwq))); + work_done++; + } + + return work_done; +} + +int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) +{ + struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq); + struct mlx5_cqwq *cqwq = &cq->wq; + int work_done; + + if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) + return 0; + + if (test_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state)) + work_done = mlx5e_rx_cq_process_enhanced_cqe_comp(rq, cqwq, + budget); + else + work_done = mlx5e_rx_cq_process_basic_cqe_comp(rq, cqwq, + budget); + + if (work_done == 0) + return 0; -out: if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) && rq->hw_gro_data->skb) mlx5e_shampo_flush_skb(rq, NULL, false); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wq.h b/drivers/net/ethernet/mellanox/mlx5/core/wq.h index 4d629e5ddbc74..e4ef1d24a3ada 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wq.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/wq.h @@ -243,6 +243,23 @@ static inline struct mlx5_cqe64 *mlx5_cqwq_get_cqe(struct mlx5_cqwq *wq) return cqe; } +static inline +struct mlx5_cqe64 *mlx5_cqwq_get_cqe_enahnced_comp(struct mlx5_cqwq *wq) +{ + u8 sw_validity_iteration_count = mlx5_cqwq_get_wrap_cnt(wq) & 0xff; + u32 ci = mlx5_cqwq_get_ci(wq); + struct mlx5_cqe64 *cqe; + + cqe = mlx5_cqwq_get_wqe(wq, ci); + if (cqe->validity_iteration_count != sw_validity_iteration_count) + return NULL; + + /* ensure cqe content is read after cqe ownership bit/validity byte */ + dma_rmb(); + + return cqe; +} + static inline u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq) { return (u32)wq->fbc.sz_m1 + 1; diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 1ff91cb79ded5..eb3fac30488bb 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -882,6 +882,12 @@ static inline u8 get_cqe_opcode(struct mlx5_cqe64 *cqe) return cqe->op_own >> 4; } +static inline u8 get_cqe_enhanced_num_mini_cqes(struct mlx5_cqe64 *cqe) +{ + /* num_of_mini_cqes is zero based */ + return get_cqe_opcode(cqe) + 1; +} + static inline u8 get_cqe_lro_tcppsh(struct mlx5_cqe64 *cqe) { return (cqe->lro.tcppsh_abort_dupack >> 6) & 1; From 38438d39a9a001df8926f1b2b3087764dbbdc1aa Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 18 May 2022 11:46:35 +0300 Subject: [PATCH 08/15] net/mlx5e: Move params kernel log print to probe function Params info print was meant to be printed on load. With time, new calls to mlx5e_init_rq_type_params and mlx5e_build_rq_params were added, mistakenly printing the params once again. Move the print to were it belongs, in mlx5e_probe. Signed-off-by: Tariq Toukan Reviewed-by: Gal Pressman Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/en/params.c | 10 ---------- .../net/ethernet/mellanox/mlx5/core/en/params.h | 14 ++++++++++++++ drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 1 + 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 1a2de9bc65382..9ec9662d1d0b8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -607,16 +607,6 @@ void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, params->log_rq_mtu_frames = is_kdump_kernel() ? MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE : MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE; - - mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d %s)\n", - params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ, - params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ? - BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, NULL)) : - BIT(params->log_rq_mtu_frames), - BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)), - MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS), - MLX5_CAP_GEN(mdev, enhanced_cqe_compression) ? - "enhanced" : "basic"); } void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index 034debd140bce..c9be6eb88012d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -154,4 +154,18 @@ int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params); int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params); +static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev, + struct mlx5e_params *params) +{ + mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d %s)\n", + params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ, + params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ? + BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, NULL)) : + BIT(params->log_rq_mtu_frames), + BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)), + MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS), + MLX5_CAP_GEN(mdev, enhanced_cqe_compression) ? + "enhanced" : "basic"); +}; + #endif /* __MLX5_EN_PARAMS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index c462b76743b64..5df04e0025896 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -5955,6 +5955,7 @@ static int mlx5e_probe(struct auxiliary_device *adev, mlx5e_dcbnl_init_app(priv); mlx5_uplink_netdev_set(mdev, netdev); + mlx5e_params_print_info(mdev, &priv->channels.params); return 0; err_resume: From e74ae1faeb71a0c264102a0aad66d65e5152a568 Mon Sep 17 00:00:00 2001 From: Guy Truzman Date: Thu, 29 Sep 2022 15:12:51 +0300 Subject: [PATCH 09/15] net/mlx5e: Add error flow when failing update_rx Up until now, return value of update_rx was ignored. Therefore, flow continues even if it fails. Add error flow in case of update_rx fails in mlx5e_open_locked, mlx5i_open and mlx5i_pkey_open. Signed-off-by: Guy Truzman Reviewed-by: Aya Levin Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 7 ++++++- drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c | 7 ++++++- drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c | 6 +++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 5df04e0025896..14bd86e368d50 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -3069,7 +3069,10 @@ int mlx5e_open_locked(struct net_device *netdev) if (err) goto err_clear_state_opened_flag; - priv->profile->update_rx(priv); + err = priv->profile->update_rx(priv); + if (err) + goto err_close_channels; + mlx5e_selq_apply(&priv->selq); mlx5e_activate_priv_channels(priv); mlx5e_apply_traps(priv, true); @@ -3079,6 +3082,8 @@ int mlx5e_open_locked(struct net_device *netdev) mlx5e_queue_update_stats(priv); return 0; +err_close_channels: + mlx5e_close_channels(&priv->channels); err_clear_state_opened_flag: clear_bit(MLX5E_STATE_OPENED, &priv->state); mlx5e_selq_cancel(&priv->selq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c index 4e3a75496dd9a..7c5c500fd215e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c @@ -561,12 +561,17 @@ static int mlx5i_open(struct net_device *netdev) if (err) goto err_remove_fs_underlay_qp; - epriv->profile->update_rx(epriv); + err = epriv->profile->update_rx(epriv); + if (err) + goto err_close_channels; + mlx5e_activate_priv_channels(epriv); mutex_unlock(&epriv->state_lock); return 0; +err_close_channels: + mlx5e_close_channels(&epriv->channels); err_remove_fs_underlay_qp: mlx5_fs_remove_rx_underlay_qpn(mdev, ipriv->qpn); err_reset_qp: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c index 0227a521d301e..4d9c9e49645c9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c @@ -221,12 +221,16 @@ static int mlx5i_pkey_open(struct net_device *netdev) mlx5_core_warn(mdev, "opening child channels failed, %d\n", err); goto err_clear_state_opened_flag; } - epriv->profile->update_rx(epriv); + err = epriv->profile->update_rx(epriv); + if (err) + goto err_close_channels; mlx5e_activate_priv_channels(epriv); mutex_unlock(&epriv->state_lock); return 0; +err_close_channels: + mlx5e_close_channels(&epriv->channels); err_clear_state_opened_flag: mlx5e_destroy_tis(mdev, epriv->tisn[0][0]); err_remove_rx_uderlay_qp: From 989722906166173c703c1d73a26a0167fdad4136 Mon Sep 17 00:00:00 2001 From: Roi Dayan Date: Wed, 21 Sep 2022 09:17:15 +0300 Subject: [PATCH 10/15] net/mlx5e: TC, Remove redundant WARN_ON() The case where the packet is not offloaded and needs to be restored to slow path and couldn't find expected tunnel information should not dump a call trace to the user. there is a debug call. Signed-off-by: Roi Dayan Reviewed-by: Maor Dickman Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c index fac7e3ff26747..b08339d986d5f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c @@ -690,7 +690,6 @@ static bool mlx5e_restore_tunnel(struct mlx5e_priv *priv, struct sk_buff *skb, err = mapping_find(uplink_priv->tunnel_mapping, tun_id, &key); if (err) { - WARN_ON_ONCE(true); netdev_dbg(priv->netdev, "Couldn't find tunnel for tun_id: %d, err: %d\n", tun_id, err); From 1f74399fd1edea2eefd8d340f5d1f7c5f5edec66 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 14 Sep 2022 11:02:34 +0300 Subject: [PATCH 11/15] net/mlx5e: kTLS, Remove unused work field Work field in struct mlx5e_async_ctx is not used. Remove it. Signed-off-by: Tariq Toukan Reviewed-by: Gal Pressman Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 2e0335246967b..f8d9708feb7ca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -126,7 +126,6 @@ mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) struct mlx5e_async_ctx { struct mlx5_async_work context; struct mlx5_async_ctx async_ctx; - struct work_struct work; struct mlx5e_ktls_offload_context_tx *priv_tx; struct completion complete; int err; From 4d78a2ebbd2b40b4c14440b7417fc473120e15d6 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Mon, 12 Sep 2022 21:43:18 +0300 Subject: [PATCH 12/15] net/mlx5e: kTLS, Remove unnecessary per-callback completion Waiting on a completion object for each callback before cleaning up their async contexts is not necessary, as this is already implied in the mlx5_cmd_cleanup_async_ctx() API. Signed-off-by: Tariq Toukan Reviewed-by: Gal Pressman Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index f8d9708feb7ca..fcaa26847c8a1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -127,7 +127,6 @@ struct mlx5e_async_ctx { struct mlx5_async_work context; struct mlx5_async_ctx async_ctx; struct mlx5e_ktls_offload_context_tx *priv_tx; - struct completion complete; int err; union { u32 out_create[MLX5_ST_SZ_DW(create_tis_out)]; @@ -148,7 +147,6 @@ static struct mlx5e_async_ctx *mlx5e_bulk_async_init(struct mlx5_core_dev *mdev, struct mlx5e_async_ctx *async = &bulk_async[i]; mlx5_cmd_init_async_ctx(mdev, &async->async_ctx); - init_completion(&async->complete); } return bulk_async; @@ -175,12 +173,10 @@ static void create_tis_callback(int status, struct mlx5_async_work *context) if (status) { async->err = status; priv_tx->create_err = 1; - goto out; + return; } priv_tx->tisn = MLX5_GET(create_tis_out, async->out_create, tisn); -out: - complete(&async->complete); } static void destroy_tis_callback(int status, struct mlx5_async_work *context) @@ -189,7 +185,6 @@ static void destroy_tis_callback(int status, struct mlx5_async_work *context) container_of(context, struct mlx5e_async_ctx, context); struct mlx5e_ktls_offload_context_tx *priv_tx = async->priv_tx; - complete(&async->complete); kfree(priv_tx); } @@ -231,7 +226,6 @@ static void mlx5e_tls_priv_tx_cleanup(struct mlx5e_ktls_offload_context_tx *priv struct mlx5e_async_ctx *async) { if (priv_tx->create_err) { - complete(&async->complete); kfree(priv_tx); return; } @@ -259,11 +253,6 @@ static void mlx5e_tls_priv_tx_list_cleanup(struct mlx5_core_dev *mdev, i++; } - for (i = 0; i < size; i++) { - struct mlx5e_async_ctx *async = &bulk_async[i]; - - wait_for_completion(&async->complete); - } mlx5e_bulk_async_cleanup(bulk_async, size); } @@ -310,7 +299,6 @@ static void create_work(struct work_struct *work) for (j = 0; j < i; j++) { struct mlx5e_async_ctx *async = &bulk_async[j]; - wait_for_completion(&async->complete); if (!err && async->err) err = async->err; } From 341361533011945f0a7da85632662a5879233853 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 14 Sep 2022 10:35:04 +0300 Subject: [PATCH 13/15] net/mlx5e: kTLS, Use a single async context object per a callback bulk A single async context object is sufficient to wait for the completions of many callbacks. Switch to using one instance per a bulk of commands. Signed-off-by: Tariq Toukan Reviewed-by: Gal Pressman Signed-off-by: Saeed Mahameed --- .../mellanox/mlx5/core/en_accel/ktls_tx.c | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index fcaa26847c8a1..78072bf93f3f7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -125,7 +125,7 @@ mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) /* struct for callback API management */ struct mlx5e_async_ctx { struct mlx5_async_work context; - struct mlx5_async_ctx async_ctx; + struct mlx5_async_ctx *async_ctx; struct mlx5e_ktls_offload_context_tx *priv_tx; int err; union { @@ -134,33 +134,33 @@ struct mlx5e_async_ctx { }; }; -static struct mlx5e_async_ctx *mlx5e_bulk_async_init(struct mlx5_core_dev *mdev, int n) +struct mlx5e_bulk_async_ctx { + struct mlx5_async_ctx async_ctx; + DECLARE_FLEX_ARRAY(struct mlx5e_async_ctx, arr); +}; + +static struct mlx5e_bulk_async_ctx *mlx5e_bulk_async_init(struct mlx5_core_dev *mdev, int n) { - struct mlx5e_async_ctx *bulk_async; + struct mlx5e_bulk_async_ctx *bulk_async; + int sz; int i; - bulk_async = kvcalloc(n, sizeof(struct mlx5e_async_ctx), GFP_KERNEL); + sz = struct_size(bulk_async, arr, n); + bulk_async = kvzalloc(sz, GFP_KERNEL); if (!bulk_async) return NULL; - for (i = 0; i < n; i++) { - struct mlx5e_async_ctx *async = &bulk_async[i]; + mlx5_cmd_init_async_ctx(mdev, &bulk_async->async_ctx); - mlx5_cmd_init_async_ctx(mdev, &async->async_ctx); - } + for (i = 0; i < n; i++) + bulk_async->arr[i].async_ctx = &bulk_async->async_ctx; return bulk_async; } -static void mlx5e_bulk_async_cleanup(struct mlx5e_async_ctx *bulk_async, int n) +static void mlx5e_bulk_async_cleanup(struct mlx5e_bulk_async_ctx *bulk_async) { - int i; - - for (i = 0; i < n; i++) { - struct mlx5e_async_ctx *async = &bulk_async[i]; - - mlx5_cmd_cleanup_async_ctx(&async->async_ctx); - } + mlx5_cmd_cleanup_async_ctx(&bulk_async->async_ctx); kvfree(bulk_async); } @@ -208,7 +208,7 @@ mlx5e_tls_priv_tx_init(struct mlx5_core_dev *mdev, struct mlx5e_tls_sw_stats *sw goto err_out; } else { async->priv_tx = priv_tx; - err = mlx5e_ktls_create_tis_cb(mdev, &async->async_ctx, + err = mlx5e_ktls_create_tis_cb(mdev, async->async_ctx, async->out_create, sizeof(async->out_create), create_tis_callback, &async->context); if (err) @@ -231,7 +231,7 @@ static void mlx5e_tls_priv_tx_cleanup(struct mlx5e_ktls_offload_context_tx *priv } async->priv_tx = priv_tx; mlx5e_ktls_destroy_tis_cb(priv_tx->mdev, priv_tx->tisn, - &async->async_ctx, + async->async_ctx, async->out_destroy, sizeof(async->out_destroy), destroy_tis_callback, &async->context); } @@ -240,7 +240,7 @@ static void mlx5e_tls_priv_tx_list_cleanup(struct mlx5_core_dev *mdev, struct list_head *list, int size) { struct mlx5e_ktls_offload_context_tx *obj, *n; - struct mlx5e_async_ctx *bulk_async; + struct mlx5e_bulk_async_ctx *bulk_async; int i; bulk_async = mlx5e_bulk_async_init(mdev, size); @@ -249,11 +249,11 @@ static void mlx5e_tls_priv_tx_list_cleanup(struct mlx5_core_dev *mdev, i = 0; list_for_each_entry_safe(obj, n, list, list_node) { - mlx5e_tls_priv_tx_cleanup(obj, &bulk_async[i]); + mlx5e_tls_priv_tx_cleanup(obj, &bulk_async->arr[i]); i++; } - mlx5e_bulk_async_cleanup(bulk_async, size); + mlx5e_bulk_async_cleanup(bulk_async); } /* Recycling pool API */ @@ -279,7 +279,7 @@ static void create_work(struct work_struct *work) struct mlx5e_tls_tx_pool *pool = container_of(work, struct mlx5e_tls_tx_pool, create_work); struct mlx5e_ktls_offload_context_tx *obj; - struct mlx5e_async_ctx *bulk_async; + struct mlx5e_bulk_async_ctx *bulk_async; LIST_HEAD(local_list); int i, j, err = 0; @@ -288,7 +288,7 @@ static void create_work(struct work_struct *work) return; for (i = 0; i < MLX5E_TLS_TX_POOL_BULK; i++) { - obj = mlx5e_tls_priv_tx_init(pool->mdev, pool->sw_stats, &bulk_async[i]); + obj = mlx5e_tls_priv_tx_init(pool->mdev, pool->sw_stats, &bulk_async->arr[i]); if (IS_ERR(obj)) { err = PTR_ERR(obj); break; @@ -297,13 +297,13 @@ static void create_work(struct work_struct *work) } for (j = 0; j < i; j++) { - struct mlx5e_async_ctx *async = &bulk_async[j]; + struct mlx5e_async_ctx *async = &bulk_async->arr[j]; if (!err && async->err) err = async->err; } atomic64_add(i, &pool->sw_stats->tx_tls_pool_alloc); - mlx5e_bulk_async_cleanup(bulk_async, MLX5E_TLS_TX_POOL_BULK); + mlx5e_bulk_async_cleanup(bulk_async); if (err) goto err_out; From 05bb74c29d84dc6d26f57f5ed3970ab8aeb13551 Mon Sep 17 00:00:00 2001 From: Oz Shlomo Date: Mon, 31 Oct 2022 09:00:30 +0000 Subject: [PATCH 14/15] net/mlx5e: CT, optimize pre_ct table lookup The pre_ct table realizes in hardware the act_ct cache logic, bypassing the CT table if the ct state was already set by a previous ct lookup. As such, the pre_ct table will always miss for chain 0 filters. Optimize the pre_ct table lookup for rules installed on chain 0. Signed-off-by: Oz Shlomo Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/en/tc_ct.c | 89 ++++++++++++------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index 864ce0c393e61..a69849e0deed7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -1774,35 +1774,42 @@ mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft) /* We translate the tc filter with CT action to the following HW model: * - * +---------------------+ - * + ft prio (tc chain) + - * + original match + - * +---------------------+ - * | set chain miss mapping - * | set fte_id - * | set tunnel_id - * | do decap - * v - * +---------------------+ - * + pre_ct/pre_ct_nat + if matches +-------------------------+ - * + zone+nat match +---------------->+ post_act (see below) + - * +---------------------+ set zone +-------------------------+ - * | set zone - * v - * +--------------------+ - * + CT (nat or no nat) + - * + tuple + zone match + - * +--------------------+ - * | set mark - * | set labels_id - * | set established - * | set zone_restore - * | do nat (if needed) - * v - * +--------------+ - * + post_act + original filter actions - * + fte_id match +------------------------> - * +--------------+ + * +---------------------+ + * + ft prio (tc chain) + + * + original match + + * +---------------------+ + * | set chain miss mapping + * | set fte_id + * | set tunnel_id + * | do decap + * | + * +-------------+ + * | Chain 0 | + * | optimization| + * | v + * | +---------------------+ + * | + pre_ct/pre_ct_nat + if matches +----------------------+ + * | + zone+nat match +---------------->+ post_act (see below) + + * | +---------------------+ set zone +----------------------+ + * | | + * +-------------+ set zone + * | + * v + * +--------------------+ + * + CT (nat or no nat) + + * + tuple + zone match + + * +--------------------+ + * | set mark + * | set labels_id + * | set established + * | set zone_restore + * | do nat (if needed) + * v + * +--------------+ + * + post_act + original filter actions + * + fte_id match +------------------------> + * +--------------+ + * */ static struct mlx5_flow_handle * __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv, @@ -1818,6 +1825,7 @@ __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_flow *ct_flow; int chain_mapping = 0, err; struct mlx5_ct_ft *ft; + u16 zone; ct_flow = kzalloc(sizeof(*ct_flow), GFP_KERNEL); if (!ct_flow) { @@ -1884,6 +1892,25 @@ __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv, } } + /* Change original rule point to ct table + * Chain 0 sets the zone and jumps to ct table + * Other chains jump to pre_ct table to align with act_ct cached logic + */ + pre_ct_attr->dest_chain = 0; + if (!attr->chain) { + zone = ft->zone & MLX5_CT_ZONE_MASK; + err = mlx5e_tc_match_to_reg_set(priv->mdev, pre_mod_acts, ct_priv->ns_type, + ZONE_TO_REG, zone); + if (err) { + ct_dbg("Failed to set zone register mapping"); + goto err_mapping; + } + + pre_ct_attr->dest_ft = nat ? ct_priv->ct_nat : ct_priv->ct; + } else { + pre_ct_attr->dest_ft = nat ? ft->pre_ct_nat.ft : ft->pre_ct.ft; + } + mod_hdr = mlx5_modify_header_alloc(priv->mdev, ct_priv->ns_type, pre_mod_acts->num_actions, pre_mod_acts->actions); @@ -1893,10 +1920,6 @@ __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv, goto err_mapping; } pre_ct_attr->modify_hdr = mod_hdr; - - /* Change original rule point to ct table */ - pre_ct_attr->dest_chain = 0; - pre_ct_attr->dest_ft = nat ? ft->pre_ct_nat.ft : ft->pre_ct.ft; ct_flow->pre_ct_rule = mlx5_tc_rule_insert(priv, orig_spec, pre_ct_attr); if (IS_ERR(ct_flow->pre_ct_rule)) { From e07c4924a77dbf21bf1973411149784cfa5d3b27 Mon Sep 17 00:00:00 2001 From: Saeed Mahameed Date: Wed, 26 Oct 2022 11:26:31 +0100 Subject: [PATCH 15/15] net/mlx5e: ethtool: get_link_ext_stats for PHY down events Implement ethtool_op get_link_ext_stats for PHY down events Signed-off-by: Saeed Mahameed Reviewed-by: Gal Pressman --- .../ethernet/mellanox/mlx5/core/en_ethtool.c | 1 + .../net/ethernet/mellanox/mlx5/core/en_stats.c | 17 +++++++++++++++++ .../net/ethernet/mellanox/mlx5/core/en_stats.h | 2 ++ 3 files changed, 20 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 24aa25da482b5..d9397ffb396bc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -2463,4 +2463,5 @@ const struct ethtool_ops mlx5e_ethtool_ops = { .get_eth_mac_stats = mlx5e_get_eth_mac_stats, .get_eth_ctrl_stats = mlx5e_get_eth_ctrl_stats, .get_rmon_stats = mlx5e_get_rmon_stats, + .get_link_ext_stats = mlx5e_get_link_ext_stats }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 03c1841970f14..70c4ea3841d72 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -1241,6 +1241,23 @@ static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(phy) mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0); } +void mlx5e_get_link_ext_stats(struct net_device *dev, + struct ethtool_link_ext_stats *stats) +{ + struct mlx5e_priv *priv = netdev_priv(dev); + u32 out[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; + u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; + int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); + + MLX5_SET(ppcnt_reg, in, local_port, 1); + MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP); + mlx5_core_access_reg(priv->mdev, in, sz, out, + MLX5_ST_SZ_BYTES(ppcnt_reg), MLX5_REG_PPCNT, 0, 0); + + stats->link_down_events = MLX5_GET(ppcnt_reg, out, + counter_set.phys_layer_cntrs.link_down_events); +} + static int fec_num_lanes(struct mlx5_core_dev *dev) { u32 out[MLX5_ST_SZ_DW(pmlp_reg)] = {}; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index 9f781085be471..cbc831ca646b7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -126,6 +126,8 @@ void mlx5e_stats_eth_ctrl_get(struct mlx5e_priv *priv, void mlx5e_stats_rmon_get(struct mlx5e_priv *priv, struct ethtool_rmon_stats *rmon, const struct ethtool_rmon_hist_range **ranges); +void mlx5e_get_link_ext_stats(struct net_device *dev, + struct ethtool_link_ext_stats *stats); /* Concrete NIC Stats */