From 95484760f03d684da522a15682776a090db738da Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Fri, 26 Aug 2022 18:06:49 +0200 Subject: [PATCH 1/4] mlxsw: cmd: Edit the comment of 'max_lag' field in CONFIG_PROFILE Starting from Spectrum-4, the maximum number of LAG IDs can be configured by software via CONFIG_PROFILE command during driver initialization. Edit the comment of 'max_lag' field to mention that this field is reserved in Spectrum-1/2/3 and describe firmware behavior. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Reviewed-by: Petr Machata Signed-off-by: Petr Machata Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/cmd.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlxsw/cmd.h b/drivers/net/ethernet/mellanox/mlxsw/cmd.h index 60232fb8ccd7a..09bef04b11d16 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/cmd.h +++ b/drivers/net/ethernet/mellanox/mlxsw/cmd.h @@ -703,6 +703,9 @@ MLXSW_ITEM32(cmd_mbox, config_profile, max_vepa_channels, 0x10, 0, 8); /* cmd_mbox_config_profile_max_lag * Maximum number of LAG IDs requested. + * Reserved when Spectrum-1/2/3, supported from Spectrum-4 and above. + * For Spectrum-4, firmware sets 128 for values between 1-128 and 256 for values + * between 129-256. */ MLXSW_ITEM32(cmd_mbox, config_profile, max_lag, 0x14, 0, 16); From eb907e9779ca48e3f3fe3796ed88de017dd59414 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Fri, 26 Aug 2022 18:06:50 +0200 Subject: [PATCH 2/4] mlxsw: Support configuring 'max_lag' via CONFIG_PROFILE In the device, LAG identifiers are stored in the port group table (PGT). During initialization, firmware reserves a certain amount of entries at the beginning of this table for LAG identifiers. In Spectrum-4, the size of the PGT table did not increase, but the maximum number of LAG identifiers was doubled, leaving less room for others entries (e.g., flood entries) that also reside in the PGT. Therefore, in order to avoid a regression and as long as there is no explicit requirement to support 256 LAGs, mlxsw driver will configure the firmware to allocate the same amount of LAG entries (128) as in Spectrum-{2,3}. This configuration is done using 'max_lag' field in CONFIG_PROFILE command. Extend 'struct mlxsw_config_profile' to support 'max_lag' field and configure firmware accordingly. A next patch will adjust Spectrum-4 to configure 'max_lag' field. Signed-off-by: Amit Cohen Reviewed-by: Petr Machata Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/core.h | 2 ++ drivers/net/ethernet/mellanox/mlxsw/pci.c | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index 9de9fa24f27c7..383c423c3ef8d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -295,6 +295,7 @@ struct mlxsw_swid_config { struct mlxsw_config_profile { u16 used_max_vepa_channels:1, + used_max_lag:1, used_max_mid:1, used_max_pgt:1, used_max_system_port:1, @@ -310,6 +311,7 @@ struct mlxsw_config_profile { used_kvd_sizes:1, used_cqe_time_stamp_type:1; u8 max_vepa_channels; + u16 max_lag; u16 max_mid; u16 max_pgt; u16 max_system_port; diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 50527adc5b5ad..c968309657dd1 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -1187,6 +1187,11 @@ static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox, mlxsw_cmd_mbox_config_profile_max_vepa_channels_set( mbox, profile->max_vepa_channels); } + if (profile->used_max_lag) { + mlxsw_cmd_mbox_config_profile_set_max_lag_set(mbox, 1); + mlxsw_cmd_mbox_config_profile_max_lag_set(mbox, + profile->max_lag); + } if (profile->used_max_mid) { mlxsw_cmd_mbox_config_profile_set_max_mid_set( mbox, 1); From cf735d4c9bab8398259a3635fcd698726a2f1bbe Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Fri, 26 Aug 2022 18:06:51 +0200 Subject: [PATCH 3/4] mlxsw: Add a helper function for getting maximum LAG ID Currently the driver queries the maximum supported LAG ID from firmware. This will not be accurate anymore once the driver will configure 'max_lag' via CONFIG_PROFILE command. For resource query, firmware returns the maximum LAG ID which is supported by hardware. Software can configure firmware to do not allocate entries for all the supported LAGs, and to limit LAG IDs. In this case, the resource query will not return the actual maximum LAG ID. Add a helper function for getting this value. In case that 'max_lag' field was set during initialization, return the value which was used, otherwise, query firmware for the maximum supported ID. Signed-off-by: Amit Cohen Reviewed-by: Petr Machata Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/core.c | 25 ++++++++++++++++--- drivers/net/ethernet/mellanox/mlxsw/core.h | 2 ++ .../net/ethernet/mellanox/mlxsw/spectrum.c | 20 +++++++++------ 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index afbe046b35a0b..1d14b1d8c5005 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -186,6 +186,23 @@ unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core) } EXPORT_SYMBOL(mlxsw_core_max_ports); +int mlxsw_core_max_lag(struct mlxsw_core *mlxsw_core, u16 *p_max_lag) +{ + struct mlxsw_driver *driver = mlxsw_core->driver; + + if (driver->profile->used_max_lag) { + *p_max_lag = driver->profile->max_lag; + return 0; + } + + if (!MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG)) + return -EIO; + + *p_max_lag = MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG); + return 0; +} +EXPORT_SYMBOL(mlxsw_core_max_lag); + void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core) { return mlxsw_core->driver_priv; @@ -2099,6 +2116,7 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, struct mlxsw_core *mlxsw_core; struct mlxsw_driver *mlxsw_driver; size_t alloc_size; + u16 max_lag; int err; mlxsw_driver = mlxsw_core_driver_get(device_kind); @@ -2140,10 +2158,9 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, if (err) goto err_ports_init; - if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG) && - MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) { - alloc_size = sizeof(*mlxsw_core->lag.mapping) * - MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG) * + err = mlxsw_core_max_lag(mlxsw_core, &max_lag); + if (!err && MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) { + alloc_size = sizeof(*mlxsw_core->lag.mapping) * max_lag * MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS); mlxsw_core->lag.mapping = kzalloc(alloc_size, GFP_KERNEL); if (!mlxsw_core->lag.mapping) { diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index 383c423c3ef8d..ca0c3d2bee6b3 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -35,6 +35,8 @@ struct mlxsw_fw_rev; unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core); +int mlxsw_core_max_lag(struct mlxsw_core *mlxsw_core, u16 *p_max_lag); + void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core); struct mlxsw_linecards *mlxsw_core_linecards(struct mlxsw_core *mlxsw_core); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 30c7b0e157218..c71a040502799 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -2691,6 +2691,7 @@ static void mlxsw_sp_traps_fini(struct mlxsw_sp *mlxsw_sp) static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp) { char slcr_pl[MLXSW_REG_SLCR_LEN]; + u16 max_lag; u32 seed; int err; @@ -2709,12 +2710,14 @@ static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp) if (err) return err; - if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LAG) || - !MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LAG_MEMBERS)) + err = mlxsw_core_max_lag(mlxsw_sp->core, &max_lag); + if (err) + return err; + + if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LAG_MEMBERS)) return -EIO; - mlxsw_sp->lags = kcalloc(MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LAG), - sizeof(struct mlxsw_sp_upper), + mlxsw_sp->lags = kcalloc(max_lag, sizeof(struct mlxsw_sp_upper), GFP_KERNEL); if (!mlxsw_sp->lags) return -ENOMEM; @@ -4263,10 +4266,13 @@ static int mlxsw_sp_lag_index_get(struct mlxsw_sp *mlxsw_sp, { struct mlxsw_sp_upper *lag; int free_lag_id = -1; - u64 max_lag; - int i; + u16 max_lag; + int err, i; + + err = mlxsw_core_max_lag(mlxsw_sp->core, &max_lag); + if (err) + return err; - max_lag = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LAG); for (i = 0; i < max_lag; i++) { lag = mlxsw_sp_lag_get(mlxsw_sp, i); if (lag->ref_count) { From c503d8ae48f2443345ba82ef66a9dfc22389a66f Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Fri, 26 Aug 2022 18:06:52 +0200 Subject: [PATCH 4/4] mlxsw: spectrum: Add a copy of 'struct mlxsw_config_profile' for Spectrum-4 Starting from Spectrum-4, the maximum number of LAG IDs can be configured by software via CONFIG_PROFILE command during driver initialization. Add a dedicated instance of 'struct mlxsw_config_profile' for Spectrum-4 and set the 'max_lag' field to 128, which is the same amount of LAG entries as in Spectrum-{2,3}. Without this configuration, firmware reserves 256 (the value of 'cap_max_lag' resource) entries at beginning of PGT table for LAG identifiers, which means that less entries in PGT will be available. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Reviewed-by: Petr Machata Signed-off-by: Petr Machata Signed-off-by: Jakub Kicinski --- .../net/ethernet/mellanox/mlxsw/spectrum.c | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index c71a040502799..5bcf5bceff710 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -3512,6 +3512,33 @@ static const struct mlxsw_config_profile mlxsw_sp2_config_profile = { .cqe_time_stamp_type = MLXSW_CMD_MBOX_CONFIG_PROFILE_CQE_TIME_STAMP_TYPE_UTC, }; +/* Reduce number of LAGs from full capacity (256) to the maximum supported LAGs + * in Spectrum-2/3, to avoid regression in number of free entries in the PGT + * table. + */ +#define MLXSW_SP4_CONFIG_PROFILE_MAX_LAG 128 + +static const struct mlxsw_config_profile mlxsw_sp4_config_profile = { + .used_max_lag = 1, + .max_lag = MLXSW_SP4_CONFIG_PROFILE_MAX_LAG, + .used_flood_mode = 1, + .flood_mode = MLXSW_CMD_MBOX_CONFIG_PROFILE_FLOOD_MODE_CONTROLLED, + .used_max_ib_mc = 1, + .max_ib_mc = 0, + .used_max_pkey = 1, + .max_pkey = 0, + .used_ubridge = 1, + .ubridge = 1, + .swid_config = { + { + .used_type = 1, + .type = MLXSW_PORT_SWID_TYPE_ETH, + } + }, + .used_cqe_time_stamp_type = 1, + .cqe_time_stamp_type = MLXSW_CMD_MBOX_CONFIG_PROFILE_CQE_TIME_STAMP_TYPE_UTC, +}; + static void mlxsw_sp_resource_size_params_prepare(struct mlxsw_core *mlxsw_core, struct devlink_resource_size_params *kvd_size_params, @@ -4042,7 +4069,7 @@ static struct mlxsw_driver mlxsw_sp4_driver = { .params_unregister = mlxsw_sp2_params_unregister, .ptp_transmitted = mlxsw_sp_ptp_transmitted, .txhdr_len = MLXSW_TXHDR_LEN, - .profile = &mlxsw_sp2_config_profile, + .profile = &mlxsw_sp4_config_profile, .sdq_supports_cqe_v2 = true, };