From 3d7a6f677905ea74c060709a0c5fe7b3df1eaae3 Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Wed, 15 Sep 2021 13:13:05 +0300 Subject: [PATCH 01/10] mlxsw: core: Initialize switch driver last Commit 961cf99a074f ("mlxsw: core: Re-order initialization sequence") changed the initialization sequence so that the switch driver (e.g., mlxsw_spectrum) is initialized before registration with the hwmon and thermal subsystems. This was done in order to avoid situations where hwmon/thermal code uses features not supported by current firmware version, which is only validated as part of switch driver initialization. Later, commit b79cb787ac70 ("mlxsw: Move fw flashing code into core.c") moved firmware validation and flashing code from the switch driver to mlxsw_core so that it is performed before driver initialization. Therefore, change the initialization sequence back to its original form. In addition to being more straightforward, it will allow us to simplify parts of the code in subsequent patches and future patchsets. Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/core.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index 98420db90ea1e..44803746ded54 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -1995,12 +1995,6 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, if (err) goto err_health_init; - if (mlxsw_driver->init) { - err = mlxsw_driver->init(mlxsw_core, mlxsw_bus_info, extack); - if (err) - goto err_driver_init; - } - err = mlxsw_hwmon_init(mlxsw_core, mlxsw_bus_info, &mlxsw_core->hwmon); if (err) goto err_hwmon_init; @@ -2014,6 +2008,12 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, if (err) goto err_env_init; + if (mlxsw_driver->init) { + err = mlxsw_driver->init(mlxsw_core, mlxsw_bus_info, extack); + if (err) + goto err_driver_init; + } + mlxsw_core->is_initialized = true; devlink_params_publish(devlink); @@ -2022,14 +2022,13 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, return 0; +err_driver_init: + mlxsw_env_fini(mlxsw_core->env); err_env_init: mlxsw_thermal_fini(mlxsw_core->thermal); err_thermal_init: mlxsw_hwmon_fini(mlxsw_core->hwmon); err_hwmon_init: - if (mlxsw_core->driver->fini) - mlxsw_core->driver->fini(mlxsw_core); -err_driver_init: mlxsw_core_health_fini(mlxsw_core); err_health_init: err_fw_rev_validate: @@ -2101,11 +2100,11 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, devlink_params_unpublish(devlink); mlxsw_core->is_initialized = false; + if (mlxsw_core->driver->fini) + mlxsw_core->driver->fini(mlxsw_core); mlxsw_env_fini(mlxsw_core->env); mlxsw_thermal_fini(mlxsw_core->thermal); mlxsw_hwmon_fini(mlxsw_core->hwmon); - if (mlxsw_core->driver->fini) - mlxsw_core->driver->fini(mlxsw_core); mlxsw_core_health_fini(mlxsw_core); if (!reload) mlxsw_core_params_unregister(mlxsw_core); From 25a91f835a7b0964455594d11253060bf5b480b4 Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Wed, 15 Sep 2021 13:13:06 +0300 Subject: [PATCH 02/10] mlxsw: core: Remove mlxsw_core_is_initialized() After the previous patch, the switch driver is always initialized last, making this function redundant. Remove it. Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/core.c | 8 -------- drivers/net/ethernet/mellanox/mlxsw/core.h | 1 - drivers/net/ethernet/mellanox/mlxsw/core_env.c | 6 ------ 3 files changed, 15 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index 44803746ded54..8c634976ca773 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -90,7 +90,6 @@ struct mlxsw_core { struct devlink_health_reporter *fw_fatal; } health; struct mlxsw_env *env; - bool is_initialized; /* Denotes if core was already initialized. */ unsigned long driver_priv[]; /* driver_priv has to be always the last item */ }; @@ -2014,7 +2013,6 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, goto err_driver_init; } - mlxsw_core->is_initialized = true; devlink_params_publish(devlink); if (!reload) @@ -2099,7 +2097,6 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, } devlink_params_unpublish(devlink); - mlxsw_core->is_initialized = false; if (mlxsw_core->driver->fini) mlxsw_core->driver->fini(mlxsw_core); mlxsw_env_fini(mlxsw_core->env); @@ -2938,11 +2935,6 @@ struct mlxsw_env *mlxsw_core_env(const struct mlxsw_core *mlxsw_core) return mlxsw_core->env; } -bool mlxsw_core_is_initialized(const struct mlxsw_core *mlxsw_core) -{ - return mlxsw_core->is_initialized; -} - static void mlxsw_core_buf_dump_dbg(struct mlxsw_core *mlxsw_core, const char *buf, size_t size) { diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index d21981cc04ca7..12023a550007b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -249,7 +249,6 @@ mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core, u8 local_port); bool mlxsw_core_port_is_xm(const struct mlxsw_core *mlxsw_core, u8 local_port); struct mlxsw_env *mlxsw_core_env(const struct mlxsw_core *mlxsw_core); -bool mlxsw_core_is_initialized(const struct mlxsw_core *mlxsw_core); int mlxsw_core_schedule_dw(struct delayed_work *dwork, unsigned long delay); bool mlxsw_core_schedule_work(struct work_struct *work); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.c b/drivers/net/ethernet/mellanox/mlxsw/core_env.c index 3713c45cfa1ed..27e721f96b3bb 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.c @@ -667,12 +667,6 @@ mlxsw_env_module_overheat_counter_get(struct mlxsw_core *mlxsw_core, u8 module, { struct mlxsw_env *mlxsw_env = mlxsw_core_env(mlxsw_core); - /* Prevent switch driver from accessing uninitialized data. */ - if (!mlxsw_core_is_initialized(mlxsw_core)) { - *p_counter = 0; - return 0; - } - if (WARN_ON_ONCE(module >= mlxsw_env->module_count)) return -EINVAL; From 163f3d2dd01cf9b8f58b56fe94ce517e468385a9 Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Wed, 15 Sep 2021 13:13:07 +0300 Subject: [PATCH 03/10] mlxsw: core_env: Defer handling of module temperature warning events Module temperature events are currently handled in softIRQ context, requiring the 'module_info_lock' to be a spin lock. In future patchsets we will need to be able to hold the lock while sleeping. Therefore, defer handling of these events using a work queue so that the next patch will be able to convert the lock to a mutex. Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlxsw/core_env.c | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.c b/drivers/net/ethernet/mellanox/mlxsw/core_env.c index 27e721f96b3bb..27eba0a0c91cf 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.c @@ -482,20 +482,30 @@ static int mlxsw_env_module_temp_event_enable(struct mlxsw_core *mlxsw_core, return 0; } -static void mlxsw_env_mtwe_event_func(const struct mlxsw_reg_info *reg, - char *mtwe_pl, void *priv) +struct mlxsw_env_module_temp_warn_event { + struct mlxsw_env *mlxsw_env; + char mtwe_pl[MLXSW_REG_MTWE_LEN]; + struct work_struct work; +}; + +static void mlxsw_env_mtwe_event_work(struct work_struct *work) { - struct mlxsw_env *mlxsw_env = priv; + struct mlxsw_env_module_temp_warn_event *event; + struct mlxsw_env *mlxsw_env; int i, sensor_warning; bool is_overheat; + event = container_of(work, struct mlxsw_env_module_temp_warn_event, + work); + mlxsw_env = event->mlxsw_env; + for (i = 0; i < mlxsw_env->module_count; i++) { /* 64-127 of sensor_index are mapped to the port modules * sequentially (module 0 is mapped to sensor_index 64, * module 1 to sensor_index 65 and so on) */ sensor_warning = - mlxsw_reg_mtwe_sensor_warning_get(mtwe_pl, + mlxsw_reg_mtwe_sensor_warning_get(event->mtwe_pl, i + MLXSW_REG_MTMP_MODULE_INDEX_MIN); spin_lock(&mlxsw_env->module_info_lock); is_overheat = @@ -524,10 +534,29 @@ static void mlxsw_env_mtwe_event_func(const struct mlxsw_reg_info *reg, spin_unlock(&mlxsw_env->module_info_lock); } } + + kfree(event); +} + +static void +mlxsw_env_mtwe_listener_func(const struct mlxsw_reg_info *reg, char *mtwe_pl, + void *priv) +{ + struct mlxsw_env_module_temp_warn_event *event; + struct mlxsw_env *mlxsw_env = priv; + + event = kmalloc(sizeof(*event), GFP_ATOMIC); + if (!event) + return; + + event->mlxsw_env = mlxsw_env; + memcpy(event->mtwe_pl, mtwe_pl, MLXSW_REG_MTWE_LEN); + INIT_WORK(&event->work, mlxsw_env_mtwe_event_work); + mlxsw_core_schedule_work(&event->work); } static const struct mlxsw_listener mlxsw_env_temp_warn_listener = - MLXSW_EVENTL(mlxsw_env_mtwe_event_func, MTWE, MTWE); + MLXSW_EVENTL(mlxsw_env_mtwe_listener_func, MTWE, MTWE); static int mlxsw_env_temp_warn_event_register(struct mlxsw_core *mlxsw_core) { From bd6e43f5953d39d5cb06041d5ae3e9e5470336ef Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Wed, 15 Sep 2021 13:13:08 +0300 Subject: [PATCH 04/10] mlxsw: core_env: Convert 'module_info_lock' to a mutex After the previous patch, the lock is always taken in process context so it can be converted to a mutex. It is needed for future changes where we will need to be able to sleep when holding the lock. Convert the lock to a mutex. Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlxsw/core_env.c | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.c b/drivers/net/ethernet/mellanox/mlxsw/core_env.c index 27eba0a0c91cf..543f401cb5c6a 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.c @@ -5,6 +5,7 @@ #include #include #include +#include #include "core.h" #include "core_env.h" @@ -19,7 +20,7 @@ struct mlxsw_env_module_info { struct mlxsw_env { struct mlxsw_core *core; u8 module_count; - spinlock_t module_info_lock; /* Protects 'module_info'. */ + struct mutex module_info_lock; /* Protects 'module_info'. */ struct mlxsw_env_module_info module_info[]; }; @@ -507,7 +508,7 @@ static void mlxsw_env_mtwe_event_work(struct work_struct *work) sensor_warning = mlxsw_reg_mtwe_sensor_warning_get(event->mtwe_pl, i + MLXSW_REG_MTMP_MODULE_INDEX_MIN); - spin_lock(&mlxsw_env->module_info_lock); + mutex_lock(&mlxsw_env->module_info_lock); is_overheat = mlxsw_env->module_info[i].is_overheat; @@ -517,13 +518,13 @@ static void mlxsw_env_mtwe_event_work(struct work_struct *work) * warning OR current state in "no warning" and MTWE * does not report warning. */ - spin_unlock(&mlxsw_env->module_info_lock); + mutex_unlock(&mlxsw_env->module_info_lock); continue; } else if (is_overheat && !sensor_warning) { /* MTWE reports "no warning", turn is_overheat off. */ mlxsw_env->module_info[i].is_overheat = false; - spin_unlock(&mlxsw_env->module_info_lock); + mutex_unlock(&mlxsw_env->module_info_lock); } else { /* Current state is "no warning" and MTWE reports * "warning", increase the counter and turn is_overheat @@ -531,7 +532,7 @@ static void mlxsw_env_mtwe_event_work(struct work_struct *work) */ mlxsw_env->module_info[i].is_overheat = true; mlxsw_env->module_info[i].module_overheat_counter++; - spin_unlock(&mlxsw_env->module_info_lock); + mutex_unlock(&mlxsw_env->module_info_lock); } } @@ -597,9 +598,9 @@ static void mlxsw_env_pmpe_event_work(struct work_struct *work) work); mlxsw_env = event->mlxsw_env; - spin_lock_bh(&mlxsw_env->module_info_lock); + mutex_lock(&mlxsw_env->module_info_lock); mlxsw_env->module_info[event->module].is_overheat = false; - spin_unlock_bh(&mlxsw_env->module_info_lock); + mutex_unlock(&mlxsw_env->module_info_lock); err = mlxsw_env_module_has_temp_sensor(mlxsw_env->core, event->module, &has_temp_sensor); @@ -699,9 +700,9 @@ mlxsw_env_module_overheat_counter_get(struct mlxsw_core *mlxsw_core, u8 module, if (WARN_ON_ONCE(module >= mlxsw_env->module_count)) return -EINVAL; - spin_lock_bh(&mlxsw_env->module_info_lock); + mutex_lock(&mlxsw_env->module_info_lock); *p_counter = mlxsw_env->module_info[module].module_overheat_counter; - spin_unlock_bh(&mlxsw_env->module_info_lock); + mutex_unlock(&mlxsw_env->module_info_lock); return 0; } @@ -725,7 +726,7 @@ int mlxsw_env_init(struct mlxsw_core *mlxsw_core, struct mlxsw_env **p_env) if (!env) return -ENOMEM; - spin_lock_init(&env->module_info_lock); + mutex_init(&env->module_info_lock); env->core = mlxsw_core; env->module_count = module_count; *p_env = env; @@ -755,6 +756,7 @@ int mlxsw_env_init(struct mlxsw_core *mlxsw_core, struct mlxsw_env **p_env) err_module_plug_event_register: mlxsw_env_temp_warn_event_unregister(env); err_temp_warn_event_register: + mutex_destroy(&env->module_info_lock); kfree(env); return err; } @@ -765,5 +767,6 @@ void mlxsw_env_fini(struct mlxsw_env *env) /* Make sure there is no more event work scheduled. */ mlxsw_core_flush_owq(); mlxsw_env_temp_warn_event_unregister(env); + mutex_destroy(&env->module_info_lock); kfree(env); } From 06277ca23868d3e1ce9d09650f2cf14abc6b69be Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Wed, 15 Sep 2021 13:13:09 +0300 Subject: [PATCH 05/10] mlxsw: spectrum: Do not return an error in ndo_stop() The return value is not checked by the networking stack. Allows us to simplify a later patch. Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/spectrum.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 9cbc893c2545a..976d7e1ecb145 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -577,7 +577,8 @@ static int mlxsw_sp_port_stop(struct net_device *dev) struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); netif_stop_queue(dev); - return mlxsw_sp_port_admin_status_set(mlxsw_sp_port, false); + mlxsw_sp_port_admin_status_set(mlxsw_sp_port, false); + return 0; } static netdev_tx_t mlxsw_sp_port_xmit(struct sk_buff *skb, From 196bff2927a753b7472b2c48f93bed2c17f93a47 Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Wed, 15 Sep 2021 13:13:10 +0300 Subject: [PATCH 06/10] mlxsw: spectrum: Do not return an error in mlxsw_sp_port_module_unmap() The return value is never checked. Allows us to simplify a later patch. Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/spectrum.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 976d7e1ecb145..cea411884b054 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -551,13 +551,13 @@ mlxsw_sp_port_module_map(struct mlxsw_sp *mlxsw_sp, u8 local_port, return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pmlp), pmlp_pl); } -static int mlxsw_sp_port_module_unmap(struct mlxsw_sp *mlxsw_sp, u8 local_port) +static void mlxsw_sp_port_module_unmap(struct mlxsw_sp *mlxsw_sp, u8 local_port) { char pmlp_pl[MLXSW_REG_PMLP_LEN]; mlxsw_reg_pmlp_pack(pmlp_pl, local_port); mlxsw_reg_pmlp_width_set(pmlp_pl, 0); - return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pmlp), pmlp_pl); + mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pmlp), pmlp_pl); } static int mlxsw_sp_port_open(struct net_device *dev) From 896f399be0780549703d23fabe6e6bf40c661475 Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Wed, 15 Sep 2021 13:13:11 +0300 Subject: [PATCH 07/10] mlxsw: Track per-module port status In the common port module core, track the number of logical ports that are mapped to the port module and the number of logical ports using it that are administratively up. This will be used by later patches to potentially veto and control certain operations on the module, such as reset and setting its power mode. Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlxsw/core_env.c | 56 +++++++++++++++++++ .../net/ethernet/mellanox/mlxsw/core_env.h | 9 +++ drivers/net/ethernet/mellanox/mlxsw/minimal.c | 20 ++++++- .../net/ethernet/mellanox/mlxsw/spectrum.c | 37 ++++++++++-- 4 files changed, 113 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.c b/drivers/net/ethernet/mellanox/mlxsw/core_env.c index 543f401cb5c6a..c7b7254061ee3 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.c @@ -15,6 +15,8 @@ struct mlxsw_env_module_info { u64 module_overheat_counter; bool is_overheat; + int num_ports_mapped; + int num_ports_up; }; struct mlxsw_env { @@ -708,6 +710,60 @@ mlxsw_env_module_overheat_counter_get(struct mlxsw_core *mlxsw_core, u8 module, } EXPORT_SYMBOL(mlxsw_env_module_overheat_counter_get); +void mlxsw_env_module_port_map(struct mlxsw_core *mlxsw_core, u8 module) +{ + struct mlxsw_env *mlxsw_env = mlxsw_core_env(mlxsw_core); + + if (WARN_ON_ONCE(module >= mlxsw_env->module_count)) + return; + + mutex_lock(&mlxsw_env->module_info_lock); + mlxsw_env->module_info[module].num_ports_mapped++; + mutex_unlock(&mlxsw_env->module_info_lock); +} +EXPORT_SYMBOL(mlxsw_env_module_port_map); + +void mlxsw_env_module_port_unmap(struct mlxsw_core *mlxsw_core, u8 module) +{ + struct mlxsw_env *mlxsw_env = mlxsw_core_env(mlxsw_core); + + if (WARN_ON_ONCE(module >= mlxsw_env->module_count)) + return; + + mutex_lock(&mlxsw_env->module_info_lock); + mlxsw_env->module_info[module].num_ports_mapped--; + mutex_unlock(&mlxsw_env->module_info_lock); +} +EXPORT_SYMBOL(mlxsw_env_module_port_unmap); + +int mlxsw_env_module_port_up(struct mlxsw_core *mlxsw_core, u8 module) +{ + struct mlxsw_env *mlxsw_env = mlxsw_core_env(mlxsw_core); + + if (WARN_ON_ONCE(module >= mlxsw_env->module_count)) + return -EINVAL; + + mutex_lock(&mlxsw_env->module_info_lock); + mlxsw_env->module_info[module].num_ports_up++; + mutex_unlock(&mlxsw_env->module_info_lock); + + return 0; +} +EXPORT_SYMBOL(mlxsw_env_module_port_up); + +void mlxsw_env_module_port_down(struct mlxsw_core *mlxsw_core, u8 module) +{ + struct mlxsw_env *mlxsw_env = mlxsw_core_env(mlxsw_core); + + if (WARN_ON_ONCE(module >= mlxsw_env->module_count)) + return; + + mutex_lock(&mlxsw_env->module_info_lock); + mlxsw_env->module_info[module].num_ports_up--; + mutex_unlock(&mlxsw_env->module_info_lock); +} +EXPORT_SYMBOL(mlxsw_env_module_port_down); + int mlxsw_env_init(struct mlxsw_core *mlxsw_core, struct mlxsw_env **p_env) { char mgpir_pl[MLXSW_REG_MGPIR_LEN]; diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.h b/drivers/net/ethernet/mellanox/mlxsw/core_env.h index 0bf5bd0f8a7ee..ba9269f12cb8c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.h @@ -27,6 +27,15 @@ mlxsw_env_get_module_eeprom_by_page(struct mlxsw_core *mlxsw_core, u8 module, int mlxsw_env_module_overheat_counter_get(struct mlxsw_core *mlxsw_core, u8 module, u64 *p_counter); + +void mlxsw_env_module_port_map(struct mlxsw_core *mlxsw_core, u8 module); + +void mlxsw_env_module_port_unmap(struct mlxsw_core *mlxsw_core, u8 module); + +int mlxsw_env_module_port_up(struct mlxsw_core *mlxsw_core, u8 module); + +void mlxsw_env_module_port_down(struct mlxsw_core *mlxsw_core, u8 module); + int mlxsw_env_init(struct mlxsw_core *core, struct mlxsw_env **p_env); void mlxsw_env_fini(struct mlxsw_env *env); diff --git a/drivers/net/ethernet/mellanox/mlxsw/minimal.c b/drivers/net/ethernet/mellanox/mlxsw/minimal.c index d9d56c44e994f..a3eca0b56bbef 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/minimal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/minimal.c @@ -54,8 +54,20 @@ static int mlxsw_m_base_mac_get(struct mlxsw_m *mlxsw_m) return 0; } -static int mlxsw_m_port_dummy_open_stop(struct net_device *dev) +static int mlxsw_m_port_open(struct net_device *dev) { + struct mlxsw_m_port *mlxsw_m_port = netdev_priv(dev); + struct mlxsw_m *mlxsw_m = mlxsw_m_port->mlxsw_m; + + return mlxsw_env_module_port_up(mlxsw_m->core, mlxsw_m_port->module); +} + +static int mlxsw_m_port_stop(struct net_device *dev) +{ + struct mlxsw_m_port *mlxsw_m_port = netdev_priv(dev); + struct mlxsw_m *mlxsw_m = mlxsw_m_port->mlxsw_m; + + mlxsw_env_module_port_down(mlxsw_m->core, mlxsw_m_port->module); return 0; } @@ -70,8 +82,8 @@ mlxsw_m_port_get_devlink_port(struct net_device *dev) } static const struct net_device_ops mlxsw_m_port_netdev_ops = { - .ndo_open = mlxsw_m_port_dummy_open_stop, - .ndo_stop = mlxsw_m_port_dummy_open_stop, + .ndo_open = mlxsw_m_port_open, + .ndo_stop = mlxsw_m_port_stop, .ndo_get_devlink_port = mlxsw_m_port_get_devlink_port, }; @@ -266,6 +278,7 @@ static int mlxsw_m_port_module_map(struct mlxsw_m *mlxsw_m, u8 local_port, if (WARN_ON_ONCE(module >= max_ports)) return -EINVAL; + mlxsw_env_module_port_map(mlxsw_m->core, module); mlxsw_m->module_to_port[module] = ++mlxsw_m->max_ports; return 0; @@ -274,6 +287,7 @@ static int mlxsw_m_port_module_map(struct mlxsw_m *mlxsw_m, u8 local_port, static void mlxsw_m_port_module_unmap(struct mlxsw_m *mlxsw_m, u8 module) { mlxsw_m->module_to_port[module] = -1; + mlxsw_env_module_port_unmap(mlxsw_m->core, module); } static int mlxsw_m_ports_create(struct mlxsw_m *mlxsw_m) diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index cea411884b054..0e81ae723bc85 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -539,7 +539,9 @@ mlxsw_sp_port_module_map(struct mlxsw_sp *mlxsw_sp, u8 local_port, const struct mlxsw_sp_port_mapping *port_mapping) { char pmlp_pl[MLXSW_REG_PMLP_LEN]; - int i; + int i, err; + + mlxsw_env_module_port_map(mlxsw_sp->core, port_mapping->module); mlxsw_reg_pmlp_pack(pmlp_pl, local_port); mlxsw_reg_pmlp_width_set(pmlp_pl, port_mapping->width); @@ -548,36 +550,58 @@ mlxsw_sp_port_module_map(struct mlxsw_sp *mlxsw_sp, u8 local_port, mlxsw_reg_pmlp_tx_lane_set(pmlp_pl, i, port_mapping->lane + i); /* Rx & Tx */ } - return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pmlp), pmlp_pl); + err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pmlp), pmlp_pl); + if (err) + goto err_pmlp_write; + return 0; + +err_pmlp_write: + mlxsw_env_module_port_unmap(mlxsw_sp->core, port_mapping->module); + return err; } -static void mlxsw_sp_port_module_unmap(struct mlxsw_sp *mlxsw_sp, u8 local_port) +static void mlxsw_sp_port_module_unmap(struct mlxsw_sp *mlxsw_sp, u8 local_port, + u8 module) { char pmlp_pl[MLXSW_REG_PMLP_LEN]; mlxsw_reg_pmlp_pack(pmlp_pl, local_port); mlxsw_reg_pmlp_width_set(pmlp_pl, 0); mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pmlp), pmlp_pl); + mlxsw_env_module_port_unmap(mlxsw_sp->core, module); } static int mlxsw_sp_port_open(struct net_device *dev) { struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; int err; - err = mlxsw_sp_port_admin_status_set(mlxsw_sp_port, true); + err = mlxsw_env_module_port_up(mlxsw_sp->core, + mlxsw_sp_port->mapping.module); if (err) return err; + err = mlxsw_sp_port_admin_status_set(mlxsw_sp_port, true); + if (err) + goto err_port_admin_status_set; netif_start_queue(dev); return 0; + +err_port_admin_status_set: + mlxsw_env_module_port_down(mlxsw_sp->core, + mlxsw_sp_port->mapping.module); + return err; } static int mlxsw_sp_port_stop(struct net_device *dev) { struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; netif_stop_queue(dev); mlxsw_sp_port_admin_status_set(mlxsw_sp_port, false); + mlxsw_env_module_port_down(mlxsw_sp->core, + mlxsw_sp_port->mapping.module); return 0; } @@ -1747,13 +1771,14 @@ static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port, mlxsw_sp_port_swid_set(mlxsw_sp, local_port, MLXSW_PORT_SWID_DISABLED_PORT); err_port_swid_set: - mlxsw_sp_port_module_unmap(mlxsw_sp, local_port); + mlxsw_sp_port_module_unmap(mlxsw_sp, local_port, port_mapping->module); return err; } static void mlxsw_sp_port_remove(struct mlxsw_sp *mlxsw_sp, u8 local_port) { struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp->ports[local_port]; + u8 module = mlxsw_sp_port->mapping.module; cancel_delayed_work_sync(&mlxsw_sp_port->periodic_hw_stats.update_dw); cancel_delayed_work_sync(&mlxsw_sp_port->ptp.shaper_dw); @@ -1775,7 +1800,7 @@ static void mlxsw_sp_port_remove(struct mlxsw_sp *mlxsw_sp, u8 local_port) mlxsw_core_port_fini(mlxsw_sp->core, local_port); mlxsw_sp_port_swid_set(mlxsw_sp, local_port, MLXSW_PORT_SWID_DISABLED_PORT); - mlxsw_sp_port_module_unmap(mlxsw_sp, local_port); + mlxsw_sp_port_module_unmap(mlxsw_sp, local_port, module); } static int mlxsw_sp_cpu_port_create(struct mlxsw_sp *mlxsw_sp) From ef23841bb94ab525363e9161847894df70ee49ad Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Wed, 15 Sep 2021 13:13:12 +0300 Subject: [PATCH 08/10] mlxsw: reg: Add fields to PMAOS register The Ports Module Administrative and Operational Status (PMAOS) register configures and retrieves the per-module status. Extend it with fields required to support various module settings such as reset and power mode. Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/reg.h | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h index 8d87f3cc57113..667856f007288 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/reg.h +++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h @@ -5681,6 +5681,14 @@ static inline void mlxsw_reg_pspa_pack(char *payload, u8 swid, u8 local_port) MLXSW_REG_DEFINE(pmaos, MLXSW_REG_PMAOS_ID, MLXSW_REG_PMAOS_LEN); +/* reg_pmaos_rst + * Module reset toggle. + * Note: Setting reset while module is plugged-in will result in transition to + * "initializing" operational state. + * Access: OP + */ +MLXSW_ITEM32(reg, pmaos, rst, 0x00, 31, 1); + /* reg_pmaos_slot_index * Slot index. * Access: Index @@ -5693,6 +5701,24 @@ MLXSW_ITEM32(reg, pmaos, slot_index, 0x00, 24, 4); */ MLXSW_ITEM32(reg, pmaos, module, 0x00, 16, 8); +enum mlxsw_reg_pmaos_admin_status { + MLXSW_REG_PMAOS_ADMIN_STATUS_ENABLED = 1, + MLXSW_REG_PMAOS_ADMIN_STATUS_DISABLED = 2, + /* If the module is active and then unplugged, or experienced an error + * event, the operational status should go to "disabled" and can only + * be enabled upon explicit enable command. + */ + MLXSW_REG_PMAOS_ADMIN_STATUS_ENABLED_ONCE = 3, +}; + +/* reg_pmaos_admin_status + * Module administrative state (the desired state of the module). + * Note: To disable a module, all ports associated with the port must be + * administatively down first. + * Access: RW + */ +MLXSW_ITEM32(reg, pmaos, admin_status, 0x00, 8, 4); + /* reg_pmaos_ase * Admin state update enable. * If this bit is set, admin state will be updated based on admin_state field. From 8f4ebdb0a2744557a95810787126eb0555e7424b Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Wed, 15 Sep 2021 13:13:13 +0300 Subject: [PATCH 09/10] mlxsw: Make PMAOS pack function more generic The PMAOS register has enable bits (e.g., PMAOS.ee) that allow changing only a subset of the fields, which is exactly what subsequent patches will need to do. Instead of passing multiple arguments to its pack function, only pass the module index and let the rest be set by the different callers. No functional changes intended. Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/core_env.c | 6 ++++-- drivers/net/ethernet/mellanox/mlxsw/reg.h | 5 +---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.c b/drivers/net/ethernet/mellanox/mlxsw/core_env.c index c7b7254061ee3..a474629643aa0 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.c @@ -684,8 +684,10 @@ mlxsw_env_module_oper_state_event_enable(struct mlxsw_core *mlxsw_core, for (i = 0; i < module_count; i++) { char pmaos_pl[MLXSW_REG_PMAOS_LEN]; - mlxsw_reg_pmaos_pack(pmaos_pl, i, - MLXSW_REG_PMAOS_E_GENERATE_EVENT); + mlxsw_reg_pmaos_pack(pmaos_pl, i); + mlxsw_reg_pmaos_e_set(pmaos_pl, + MLXSW_REG_PMAOS_E_GENERATE_EVENT); + mlxsw_reg_pmaos_ee_set(pmaos_pl, true); err = mlxsw_reg_write(mlxsw_core, MLXSW_REG(pmaos), pmaos_pl); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h index 667856f007288..6c96e124e4c8d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/reg.h +++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h @@ -5747,13 +5747,10 @@ enum mlxsw_reg_pmaos_e { */ MLXSW_ITEM32(reg, pmaos, e, 0x04, 0, 2); -static inline void mlxsw_reg_pmaos_pack(char *payload, u8 module, - enum mlxsw_reg_pmaos_e e) +static inline void mlxsw_reg_pmaos_pack(char *payload, u8 module) { MLXSW_REG_ZERO(pmaos, payload); mlxsw_reg_pmaos_module_set(payload, module); - mlxsw_reg_pmaos_e_set(payload, e); - mlxsw_reg_pmaos_ee_set(payload, true); } /* PPLR - Port Physical Loopback Register From 49fd3b645de8c2cc124c284b2be8985c50e76ee0 Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Wed, 15 Sep 2021 13:13:14 +0300 Subject: [PATCH 10/10] mlxsw: Add support for transceiver modules reset Implement support for ethtool_ops::reset in order to reset transceiver modules. The module backing the netdev is reset when the 'ETH_RESET_PHY' flag is set. After a successful reset, the flag is cleared by the driver and other flags are ignored. This is in accordance with the interface documentation: "The reset() operation must clear the flags for the components which were actually reset. On successful return, the flags indicate the components which were not reset, either because they do not exist in the hardware or because they cannot be reset independently. The driver must never reset any components that were not requested." Reset is useful in order to allow a module to transition out of a fault state. From section 6.3.2.12 in CMIS 5.0: "Except for a power cycle, the only exit path from the ModuleFault state is to perform a module reset by taking an action that causes the ResetS transition signal to become TRUE (see Table 6-11)". An error is returned when the netdev is administratively up: # ip link set dev swp11 up # ethtool --reset swp11 phy ETHTOOL_RESET 0x40 Cannot issue ETHTOOL_RESET: Invalid argument # ip link set dev swp11 down # ethtool --reset swp11 phy ETHTOOL_RESET 0x40 Components reset: 0x40 An error is returned when the module is shared by multiple ports (split ports) and the "phy-shared" flag is not set: # devlink port split swp11 count 4 # ethtool --reset swp11s0 phy ETHTOOL_RESET 0x40 Cannot issue ETHTOOL_RESET: Invalid argument # ethtool --reset swp11s0 phy-shared ETHTOOL_RESET 0x400000 Components reset: 0x400000 # devlink port unsplit swp11s0 # ethtool --reset swp11 phy ETHTOOL_RESET 0x40 Components reset: 0x40 An error is also returned when one of the ports using the module is administratively up: # devlink port split swp11 count 4 # ip link set dev swp11s1 up # ethtool --reset swp11s0 phy-shared ETHTOOL_RESET 0x400000 Cannot issue ETHTOOL_RESET: Invalid argument # ip link set dev swp11s1 down # ethtool --reset swp11s0 phy-shared ETHTOOL_RESET 0x400000 Components reset: 0x400000 Reset is performed by writing to the "rst" bit of the PMAOS register, which instructs the firmware to assert the reset signal connected to the module for a fixed amount of time. Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlxsw/core_env.c | 53 +++++++++++++++++++ .../net/ethernet/mellanox/mlxsw/core_env.h | 4 ++ drivers/net/ethernet/mellanox/mlxsw/minimal.c | 10 ++++ .../mellanox/mlxsw/spectrum_ethtool.c | 10 ++++ 4 files changed, 77 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.c b/drivers/net/ethernet/mellanox/mlxsw/core_env.c index a474629643aa0..9e367174743d9 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.c @@ -392,6 +392,59 @@ mlxsw_env_get_module_eeprom_by_page(struct mlxsw_core *mlxsw_core, u8 module, } EXPORT_SYMBOL(mlxsw_env_get_module_eeprom_by_page); +static int mlxsw_env_module_reset(struct mlxsw_core *mlxsw_core, u8 module) +{ + char pmaos_pl[MLXSW_REG_PMAOS_LEN]; + + mlxsw_reg_pmaos_pack(pmaos_pl, module); + mlxsw_reg_pmaos_rst_set(pmaos_pl, true); + + return mlxsw_reg_write(mlxsw_core, MLXSW_REG(pmaos), pmaos_pl); +} + +int mlxsw_env_reset_module(struct net_device *netdev, + struct mlxsw_core *mlxsw_core, u8 module, u32 *flags) +{ + struct mlxsw_env *mlxsw_env = mlxsw_core_env(mlxsw_core); + u32 req = *flags; + int err; + + if (!(req & ETH_RESET_PHY) && + !(req & (ETH_RESET_PHY << ETH_RESET_SHARED_SHIFT))) + return 0; + + if (WARN_ON_ONCE(module >= mlxsw_env->module_count)) + return -EINVAL; + + mutex_lock(&mlxsw_env->module_info_lock); + + if (mlxsw_env->module_info[module].num_ports_up) { + netdev_err(netdev, "Cannot reset module when ports using it are administratively up\n"); + err = -EINVAL; + goto out; + } + + if (mlxsw_env->module_info[module].num_ports_mapped > 1 && + !(req & (ETH_RESET_PHY << ETH_RESET_SHARED_SHIFT))) { + netdev_err(netdev, "Cannot reset module without \"phy-shared\" flag when shared by multiple ports\n"); + err = -EINVAL; + goto out; + } + + err = mlxsw_env_module_reset(mlxsw_core, module); + if (err) { + netdev_err(netdev, "Failed to reset module\n"); + goto out; + } + + *flags &= ~(ETH_RESET_PHY | (ETH_RESET_PHY << ETH_RESET_SHARED_SHIFT)); + +out: + mutex_unlock(&mlxsw_env->module_info_lock); + return err; +} +EXPORT_SYMBOL(mlxsw_env_reset_module); + static int mlxsw_env_module_has_temp_sensor(struct mlxsw_core *mlxsw_core, u8 module, bool *p_has_temp_sensor) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_env.h b/drivers/net/ethernet/mellanox/mlxsw/core_env.h index ba9269f12cb8c..c486397f5dfec 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_env.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core_env.h @@ -24,6 +24,10 @@ mlxsw_env_get_module_eeprom_by_page(struct mlxsw_core *mlxsw_core, u8 module, const struct ethtool_module_eeprom *page, struct netlink_ext_ack *extack); +int mlxsw_env_reset_module(struct net_device *netdev, + struct mlxsw_core *mlxsw_core, u8 module, + u32 *flags); + int mlxsw_env_module_overheat_counter_get(struct mlxsw_core *mlxsw_core, u8 module, u64 *p_counter); diff --git a/drivers/net/ethernet/mellanox/mlxsw/minimal.c b/drivers/net/ethernet/mellanox/mlxsw/minimal.c index a3eca0b56bbef..9644e9c486b80 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/minimal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/minimal.c @@ -136,11 +136,21 @@ mlxsw_m_get_module_eeprom_by_page(struct net_device *netdev, page, extack); } +static int mlxsw_m_reset(struct net_device *netdev, u32 *flags) +{ + struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev); + struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core; + + return mlxsw_env_reset_module(netdev, core, mlxsw_m_port->module, + flags); +} + static const struct ethtool_ops mlxsw_m_port_ethtool_ops = { .get_drvinfo = mlxsw_m_module_get_drvinfo, .get_module_info = mlxsw_m_get_module_info, .get_module_eeprom = mlxsw_m_get_module_eeprom, .get_module_eeprom_by_page = mlxsw_m_get_module_eeprom_by_page, + .reset = mlxsw_m_reset, }; static int diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c index 267590a0eee78..06f1645561c69 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c @@ -1197,6 +1197,15 @@ mlxsw_sp_get_rmon_stats(struct net_device *dev, *ranges = mlxsw_rmon_ranges; } +static int mlxsw_sp_reset(struct net_device *dev, u32 *flags) +{ + struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev); + struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; + u8 module = mlxsw_sp_port->mapping.module; + + return mlxsw_env_reset_module(dev, mlxsw_sp->core, module, flags); +} + const struct ethtool_ops mlxsw_sp_port_ethtool_ops = { .cap_link_lanes_supported = true, .get_drvinfo = mlxsw_sp_port_get_drvinfo, @@ -1218,6 +1227,7 @@ const struct ethtool_ops mlxsw_sp_port_ethtool_ops = { .get_eth_mac_stats = mlxsw_sp_get_eth_mac_stats, .get_eth_ctrl_stats = mlxsw_sp_get_eth_ctrl_stats, .get_rmon_stats = mlxsw_sp_get_rmon_stats, + .reset = mlxsw_sp_reset, }; struct mlxsw_sp1_port_link_mode {