Skip to content

Commit

Permalink
{net/RDMA}/mlx5: introduce lag_for_each_peer
Browse files Browse the repository at this point in the history 
Introduce a generic APIs to iterate over all the devices which are part
of the LAG. This API replace mlx5_lag_get_peer_mdev() which retrieve
only a single peer device from the lag.

Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Mark Bloch <mbloch@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
  • Loading branch information
Shay Drory authored and Saeed Mahameed committed Jun 7, 2023
1 parent 962825e commit 222dd18
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 51 deletions.
98 changes: 62 additions & 36 deletions drivers/infiniband/hw/mlx5/ib_rep.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,45 +30,65 @@ mlx5_ib_set_vport_rep(struct mlx5_core_dev *dev,

static void mlx5_ib_register_peer_vport_reps(struct mlx5_core_dev *mdev);

static void mlx5_ib_num_ports_update(struct mlx5_core_dev *dev, u32 *num_ports)
{
struct mlx5_core_dev *peer_dev;
int i;

mlx5_lag_for_each_peer_mdev(dev, peer_dev, i) {
u32 peer_num_ports = mlx5_eswitch_get_total_vports(peer_dev);

if (mlx5_lag_is_mpesw(peer_dev))
*num_ports += peer_num_ports;
else
/* Only 1 ib port is the representor for all uplinks */
*num_ports += peer_num_ports - 1;
}
}

static int
mlx5_ib_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
{
u32 num_ports = mlx5_eswitch_get_total_vports(dev);
struct mlx5_core_dev *lag_master = dev;
const struct mlx5_ib_profile *profile;
struct mlx5_core_dev *peer_dev;
struct mlx5_ib_dev *ibdev;
int second_uplink = false;
u32 peer_num_ports;
int new_uplink = false;
int vport_index;
int ret;
int i;

vport_index = rep->vport_index;

if (mlx5_lag_is_shared_fdb(dev)) {
peer_dev = mlx5_lag_get_peer_mdev(dev);
peer_num_ports = mlx5_eswitch_get_total_vports(peer_dev);
if (mlx5_lag_is_master(dev)) {
if (mlx5_lag_is_mpesw(dev))
num_ports += peer_num_ports;
else
num_ports += peer_num_ports - 1;

mlx5_ib_num_ports_update(dev, &num_ports);
} else {
if (rep->vport == MLX5_VPORT_UPLINK) {
if (!mlx5_lag_is_mpesw(dev))
return 0;
second_uplink = true;
new_uplink = true;
}
mlx5_lag_for_each_peer_mdev(dev, peer_dev, i) {
u32 peer_n_ports = mlx5_eswitch_get_total_vports(peer_dev);

if (mlx5_lag_is_master(peer_dev))
lag_master = peer_dev;
else if (!mlx5_lag_is_mpesw(dev))
/* Only 1 ib port is the representor for all uplinks */
peer_n_ports--;

vport_index += peer_num_ports;
dev = peer_dev;
if (mlx5_get_dev_index(peer_dev) < mlx5_get_dev_index(dev))
vport_index += peer_n_ports;
}
}
}

if (rep->vport == MLX5_VPORT_UPLINK && !second_uplink)
if (rep->vport == MLX5_VPORT_UPLINK && !new_uplink)
profile = &raw_eth_profile;
else
return mlx5_ib_set_vport_rep(dev, rep, vport_index);
return mlx5_ib_set_vport_rep(lag_master, rep, vport_index);

ibdev = ib_alloc_device(mlx5_ib_dev, ib_dev);
if (!ibdev)
Expand All @@ -85,17 +105,17 @@ mlx5_ib_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
vport_index = rep->vport_index;
ibdev->port[vport_index].rep = rep;
ibdev->port[vport_index].roce.netdev =
mlx5_ib_get_rep_netdev(dev->priv.eswitch, rep->vport);
ibdev->mdev = dev;
mlx5_ib_get_rep_netdev(lag_master->priv.eswitch, rep->vport);
ibdev->mdev = lag_master;
ibdev->num_ports = num_ports;

ret = __mlx5_ib_add(ibdev, profile);
if (ret)
goto fail_add;

rep->rep_data[REP_IB].priv = ibdev;
if (mlx5_lag_is_shared_fdb(dev))
mlx5_ib_register_peer_vport_reps(dev);
if (mlx5_lag_is_shared_fdb(lag_master))
mlx5_ib_register_peer_vport_reps(lag_master);

return 0;

Expand All @@ -118,23 +138,27 @@ mlx5_ib_vport_rep_unload(struct mlx5_eswitch_rep *rep)
struct mlx5_ib_dev *dev = mlx5_ib_rep_to_dev(rep);
int vport_index = rep->vport_index;
struct mlx5_ib_port *port;
int i;

if (WARN_ON(!mdev))
return;

if (!dev)
return;

if (mlx5_lag_is_shared_fdb(mdev) &&
!mlx5_lag_is_master(mdev)) {
struct mlx5_core_dev *peer_mdev;

if (rep->vport == MLX5_VPORT_UPLINK && !mlx5_lag_is_mpesw(mdev))
return;
peer_mdev = mlx5_lag_get_peer_mdev(mdev);
vport_index += mlx5_eswitch_get_total_vports(peer_mdev);
for (i = 0; i < dev->num_ports; i++) {
if (dev->port[i].rep == rep)
break;
}
if (WARN_ON(i == dev->num_ports))
return;
vport_index = i;
}

if (!dev)
return;

port = &dev->port[vport_index];
write_lock(&port->roce.netdev_lock);
port->roce.netdev = NULL;
Expand All @@ -143,16 +167,18 @@ mlx5_ib_vport_rep_unload(struct mlx5_eswitch_rep *rep)
port->rep = NULL;

if (rep->vport == MLX5_VPORT_UPLINK) {
struct mlx5_core_dev *peer_mdev;
struct mlx5_eswitch *esw;

if (mlx5_lag_is_shared_fdb(mdev) && !mlx5_lag_is_master(mdev))
return;

if (mlx5_lag_is_shared_fdb(mdev)) {
peer_mdev = mlx5_lag_get_peer_mdev(mdev);
esw = peer_mdev->priv.eswitch;
mlx5_eswitch_unregister_vport_reps(esw, REP_IB);
struct mlx5_core_dev *peer_mdev;
struct mlx5_eswitch *esw;

mlx5_lag_for_each_peer_mdev(mdev, peer_mdev, i) {
esw = peer_mdev->priv.eswitch;
mlx5_eswitch_unregister_vport_reps(esw, REP_IB);
}
}
__mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
}
Expand All @@ -166,14 +192,14 @@ static const struct mlx5_eswitch_rep_ops rep_ops = {

static void mlx5_ib_register_peer_vport_reps(struct mlx5_core_dev *mdev)
{
struct mlx5_core_dev *peer_mdev = mlx5_lag_get_peer_mdev(mdev);
struct mlx5_core_dev *peer_mdev;
struct mlx5_eswitch *esw;
int i;

if (!peer_mdev)
return;

esw = peer_mdev->priv.eswitch;
mlx5_eswitch_register_vport_reps(esw, &rep_ops, REP_IB);
mlx5_lag_for_each_peer_mdev(mdev, peer_mdev, i) {
esw = peer_mdev->priv.eswitch;
mlx5_eswitch_register_vport_reps(esw, &rep_ops, REP_IB);
}
}

struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw,
Expand Down
24 changes: 15 additions & 9 deletions drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,16 +244,22 @@ static int mlx5_cmd_update_root_ft(struct mlx5_flow_root_namespace *ns,
ft->type == FS_FT_FDB &&
mlx5_lag_is_shared_fdb(dev) &&
mlx5_lag_is_master(dev)) {
err = mlx5_cmd_set_slave_root_fdb(dev,
mlx5_lag_get_peer_mdev(dev),
!disconnect, (!disconnect) ?
ft->id : 0);
if (err && !disconnect) {
MLX5_SET(set_flow_table_root_in, in, op_mod, 0);
MLX5_SET(set_flow_table_root_in, in, table_id,
ns->root_ft->id);
mlx5_cmd_exec_in(dev, set_flow_table_root, in);
struct mlx5_core_dev *peer_dev;
int i;

mlx5_lag_for_each_peer_mdev(dev, peer_dev, i) {
err = mlx5_cmd_set_slave_root_fdb(dev, peer_dev, !disconnect,
(!disconnect) ? ft->id : 0);
if (err && !disconnect) {
MLX5_SET(set_flow_table_root_in, in, op_mod, 0);
MLX5_SET(set_flow_table_root_in, in, table_id,
ns->root_ft->id);
mlx5_cmd_exec_in(dev, set_flow_table_root, in);
}
if (err)
break;
}

}

return err;
Expand Down
21 changes: 16 additions & 5 deletions drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1519,26 +1519,37 @@ u8 mlx5_lag_get_num_ports(struct mlx5_core_dev *dev)
}
EXPORT_SYMBOL(mlx5_lag_get_num_ports);

struct mlx5_core_dev *mlx5_lag_get_peer_mdev(struct mlx5_core_dev *dev)
struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int *i)
{
struct mlx5_core_dev *peer_dev = NULL;
struct mlx5_lag *ldev;
unsigned long flags;
int idx;

spin_lock_irqsave(&lag_lock, flags);
ldev = mlx5_lag_dev(dev);
if (!ldev)
goto unlock;

peer_dev = ldev->pf[MLX5_LAG_P1].dev == dev ?
ldev->pf[MLX5_LAG_P2].dev :
ldev->pf[MLX5_LAG_P1].dev;
if (*i == ldev->ports)
goto unlock;
for (idx = *i; idx < ldev->ports; idx++)
if (ldev->pf[idx].dev != dev)
break;

if (idx == ldev->ports) {
*i = idx;
goto unlock;
}
*i = idx + 1;

peer_dev = ldev->pf[idx].dev;

unlock:
spin_unlock_irqrestore(&lag_lock, flags);
return peer_dev;
}
EXPORT_SYMBOL(mlx5_lag_get_peer_mdev);
EXPORT_SYMBOL(mlx5_lag_get_next_peer_mdev);

int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev,
u64 *values,
Expand Down
8 changes: 7 additions & 1 deletion include/linux/mlx5/driver.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1174,7 +1174,13 @@ int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev,
u64 *values,
int num_counters,
size_t *offsets);
struct mlx5_core_dev *mlx5_lag_get_peer_mdev(struct mlx5_core_dev *dev);
struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int *i);

#define mlx5_lag_for_each_peer_mdev(dev, peer, i) \
for (i = 0, peer = mlx5_lag_get_next_peer_mdev(dev, &i); \
peer; \
peer = mlx5_lag_get_next_peer_mdev(dev, &i))

u8 mlx5_lag_get_num_ports(struct mlx5_core_dev *dev);
struct mlx5_uars_page *mlx5_get_uars_page(struct mlx5_core_dev *mdev);
void mlx5_put_uars_page(struct mlx5_core_dev *mdev, struct mlx5_uars_page *up);
Expand Down

0 comments on commit 222dd18

Please sign in to comment.