From 7c35ef4525c3255c747b935fe5af9ef4d1fbbf93 Mon Sep 17 00:00:00 2001 From: Or Gerlitz Date: Thu, 21 May 2015 15:14:05 +0300 Subject: [PATCH 1/6] net/mlx4_core: Enhance the MAD_IFC wrapper to convert VF port to physical Single port VFs always provide port = 1 (even if the actual physical port used is port 2). As such, we need to convert the port provided by the VF to the physical port before calling into the firmware. It turns out that the Linux mlx4 VF RoCE driver maintains a copy of the GID table and hence this change became critical only for single ported IB VFs, but it could be needed for other RoCE VF drivers too. Fixes: 449fc48866f7 ('net/mlx4: Adapt code for N-Port VF') Signed-off-by: Or Gerlitz Signed-off-by: Jack Morgenstein Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx4/cmd.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/cmd.c b/drivers/net/ethernet/mellanox/mlx4/cmd.c index 4f7dc044601e2..153fb15769a86 100644 --- a/drivers/net/ethernet/mellanox/mlx4/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx4/cmd.c @@ -877,7 +877,7 @@ static int mlx4_MAD_IFC_wrapper(struct mlx4_dev *dev, int slave, { struct ib_smp *smp = inbox->buf; u32 index; - u8 port; + u8 port, slave_port; u8 opcode_modifier; u16 *table; int err; @@ -889,7 +889,8 @@ static int mlx4_MAD_IFC_wrapper(struct mlx4_dev *dev, int slave, __be32 slave_cap_mask; __be64 slave_node_guid; - port = vhcr->in_modifier; + slave_port = vhcr->in_modifier; + port = mlx4_slave_convert_port(dev, slave, slave_port); /* network-view bit is for driver use only, and should not be passed to FW */ opcode_modifier = vhcr->op_modifier & ~0x8; /* clear netw view bit */ @@ -925,8 +926,9 @@ static int mlx4_MAD_IFC_wrapper(struct mlx4_dev *dev, int slave, if (smp->attr_id == IB_SMP_ATTR_PORT_INFO) { /*get the slave specific caps:*/ /*do the command */ + smp->attr_mod = cpu_to_be32(port); err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, - vhcr->in_modifier, opcode_modifier, + port, opcode_modifier, vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); /* modify the response for slaves */ if (!err && slave != mlx4_master_func_num(dev)) { @@ -970,7 +972,7 @@ static int mlx4_MAD_IFC_wrapper(struct mlx4_dev *dev, int slave, } if (smp->attr_id == IB_SMP_ATTR_NODE_INFO) { err = mlx4_cmd_box(dev, inbox->dma, outbox->dma, - vhcr->in_modifier, opcode_modifier, + port, opcode_modifier, vhcr->op, MLX4_CMD_TIME_CLASS_C, MLX4_CMD_NATIVE); if (!err) { slave_node_guid = mlx4_get_slave_node_guid(dev, slave); From 430910b1b93292d3a724c91123bb15bd32df1b5b Mon Sep 17 00:00:00 2001 From: Or Gerlitz Date: Thu, 21 May 2015 15:14:06 +0300 Subject: [PATCH 2/6] IB/mlx4: Convert slave port before building address-handle When multiplexling a MAD sent from VF, we should convert the port used by the guest to send the packet to the actual physical port which will be used to transmit the packet, before building the relevant address-handle (AH). This is needed under VPI for single ported VFs, since the code that builds the AH (mlx4_ib_query_ah()) makes decisions based on the input port. If we use the port number provided by the guest, it might have different protocol vs. the one this packat has to go from, and hence the result could be wrong. So far, the conversion was done after the AH was built and it worked for single ported Eth VFs which were not enabled under VPI. When adding support for single ported IB VFs and VPI, we hit that. Fixes: 449fc48866f7 ('net/mlx4: Adapt code for N-Port VF') Signed-off-by: Or Gerlitz Signed-off-by: David S. Miller --- drivers/infiniband/hw/mlx4/mad.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/mad.c b/drivers/infiniband/hw/mlx4/mad.c index 9cd2b002d7ae5..ad6a8818608d1 100644 --- a/drivers/infiniband/hw/mlx4/mad.c +++ b/drivers/infiniband/hw/mlx4/mad.c @@ -1365,14 +1365,17 @@ static void mlx4_ib_multiplex_mad(struct mlx4_ib_demux_pv_ctx *ctx, struct ib_wc * stadard address handle by decoding the tunnelled mlx4_ah fields */ memcpy(&ah.av, &tunnel->hdr.av, sizeof (struct mlx4_av)); ah.ibah.device = ctx->ib_dev; + + port = be32_to_cpu(ah.av.ib.port_pd) >> 24; + port = mlx4_slave_convert_port(dev->dev, slave, port); + if (port < 0) + return; + ah.av.ib.port_pd = cpu_to_be32(port << 24 | (be32_to_cpu(ah.av.ib.port_pd) & 0xffffff)); + mlx4_ib_query_ah(&ah.ibah, &ah_attr); if (ah_attr.ah_flags & IB_AH_GRH) fill_in_real_sgid_index(dev, slave, ctx->port, &ah_attr); - port = mlx4_slave_convert_port(dev->dev, slave, ah_attr.port_num); - if (port < 0) - return; - ah_attr.port_num = port; memcpy(ah_attr.dmac, tunnel->hdr.mac, 6); ah_attr.vlan_id = be16_to_cpu(tunnel->hdr.vlan); /* if slave have default vlan use it */ From 74d4943fbb8c2fd32440a0bb4a6a01290d15709b Mon Sep 17 00:00:00 2001 From: Or Gerlitz Date: Thu, 21 May 2015 15:14:07 +0300 Subject: [PATCH 3/6] net/mlx4_core: Modify port values when generting EQEs for VFs As part of enabling single ported VFs over IB ports we need to handle some of the flows for generting EQ events for VFs which don't come into play under Eth ports. This mainly includes port management events derived from changes of the phyiscal port (lid change, client re-register, down/up, etc), VF pkey table changes and VF guid changes initiated by the IB driver. (1) make sure that events are generated only for VFs sitting on the relevant physical port (under the ALL_SLAVES flow). (2) before generating the event, convert from physical (one or two) to VF port (always equals one). Signed-off-by: Or Gerlitz Signed-off-by: Jack Morgenstein Signed-off-by: David S. Miller --- drivers/infiniband/hw/mlx4/alias_GUID.c | 7 ++++++- drivers/net/ethernet/mellanox/mlx4/eq.c | 24 ++++++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/drivers/infiniband/hw/mlx4/alias_GUID.c b/drivers/infiniband/hw/mlx4/alias_GUID.c index 0f00204d2ecea..21cb41a60fe80 100644 --- a/drivers/infiniband/hw/mlx4/alias_GUID.c +++ b/drivers/infiniband/hw/mlx4/alias_GUID.c @@ -189,7 +189,7 @@ void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, { int i; u64 guid_indexes; - int slave_id; + int slave_id, slave_port; enum slave_port_state new_state; enum slave_port_state prev_state; __be64 tmp_cur_ag, form_cache_ag; @@ -217,6 +217,11 @@ void mlx4_ib_notify_slaves_on_guid_change(struct mlx4_ib_dev *dev, slave_id = (block_num * NUM_ALIAS_GUID_IN_REC) + i ; if (slave_id >= dev->dev->persist->num_vfs + 1) return; + + slave_port = mlx4_phys_to_slave_port(dev->dev, slave_id, port_num); + if (slave_port < 0) /* this port isn't available for the VF */ + continue; + tmp_cur_ag = *(__be64 *)&p_data[i * GUID_REC_SIZE]; form_cache_ag = get_cached_alias_guid(dev, port_num, (NUM_ALIAS_GUID_IN_REC * block_num) + i); diff --git a/drivers/net/ethernet/mellanox/mlx4/eq.c b/drivers/net/ethernet/mellanox/mlx4/eq.c index 2619c9fbf42df..80bcd648c5e0e 100644 --- a/drivers/net/ethernet/mellanox/mlx4/eq.c +++ b/drivers/net/ethernet/mellanox/mlx4/eq.c @@ -145,7 +145,7 @@ void mlx4_gen_slave_eqe(struct work_struct *work) struct mlx4_slave_event_eq *slave_eq = &mfunc->master.slave_eq; struct mlx4_eqe *eqe; u8 slave; - int i; + int i, phys_port, slave_port; for (eqe = next_slave_event_eqe(slave_eq); eqe; eqe = next_slave_event_eqe(slave_eq)) { @@ -154,9 +154,20 @@ void mlx4_gen_slave_eqe(struct work_struct *work) /* All active slaves need to receive the event */ if (slave == ALL_SLAVES) { for (i = 0; i <= dev->persist->num_vfs; i++) { + phys_port = 0; + if (eqe->type == MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT && + eqe->subtype == MLX4_DEV_PMC_SUBTYPE_PORT_INFO) { + phys_port = eqe->event.port_mgmt_change.port; + slave_port = mlx4_phys_to_slave_port(dev, i, phys_port); + if (slave_port < 0) /* VF doesn't have this port */ + continue; + eqe->event.port_mgmt_change.port = slave_port; + } if (mlx4_GEN_EQE(dev, i, eqe)) mlx4_warn(dev, "Failed to generate event for slave %d\n", i); + if (phys_port) + eqe->event.port_mgmt_change.port = phys_port; } } else { if (mlx4_GEN_EQE(dev, slave, eqe)) @@ -224,7 +235,7 @@ int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, u8 port) eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT; eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE; - eqe.event.port_mgmt_change.port = port; + eqe.event.port_mgmt_change.port = mlx4_phys_to_slave_port(dev, slave, port); return mlx4_GEN_EQE(dev, slave, &eqe); } @@ -241,7 +252,7 @@ int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, u8 port) eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT; eqe.subtype = MLX4_DEV_PMC_SUBTYPE_GUID_INFO; - eqe.event.port_mgmt_change.port = port; + eqe.event.port_mgmt_change.port = mlx4_phys_to_slave_port(dev, slave, port); return mlx4_GEN_EQE(dev, slave, &eqe); } @@ -251,6 +262,7 @@ int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port, u8 port_subtype_change) { struct mlx4_eqe eqe; + u8 slave_port = mlx4_phys_to_slave_port(dev, slave, port); /*don't send if we don't have the that slave */ if (dev->persist->num_vfs < slave) @@ -259,7 +271,7 @@ int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave, u8 port, eqe.type = MLX4_EVENT_TYPE_PORT_CHANGE; eqe.subtype = port_subtype_change; - eqe.event.port_change.port = cpu_to_be32(port << 28); + eqe.event.port_change.port = cpu_to_be32(slave_port << 28); mlx4_dbg(dev, "%s: sending: %d to slave: %d on port: %d\n", __func__, port_subtype_change, slave, port); @@ -589,6 +601,10 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq) if (SLAVE_PORT_GEN_EVENT_DOWN == gen_event) { if (i == mlx4_master_func_num(dev)) continue; + eqe->event.port_change.port = + cpu_to_be32( + (be32_to_cpu(eqe->event.port_change.port) & 0xFFFFFFF) + | (mlx4_phys_to_slave_port(dev, i, port) << 28)); mlx4_slave_event(dev, i, eqe); } } From f40e99e927c00356025c70adf2060eaaf35151a6 Mon Sep 17 00:00:00 2001 From: Or Gerlitz Date: Thu, 21 May 2015 15:14:08 +0300 Subject: [PATCH 4/6] net/mlx4_core: Adjust the schedule queue port for single ported IB VFs Some VF drivers flow set the schedule queue in the QP context but without setting none of OPTPAR_SCHED_QUEUE or OPTPAR_PRIMARY_ADDR_PATH. To allow for such non-modified drivers to function as single ported IB VFs, we must adjust the schedule queue port whenever being set, e.g as currently done for single ported Eth VFs. Signed-off-by: Or Gerlitz Signed-off-by: Jack Morgenstein Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx4/resource_tracker.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c index bafe2180cf0c4..82d562b90865e 100644 --- a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c +++ b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c @@ -3526,8 +3526,8 @@ static int adjust_qp_sched_queue(struct mlx4_dev *dev, int slave, pri_sched_queue = (qpc->pri_path.sched_queue & ~(1 << 6)) | ((port & 1) << 6); - if (optpar & MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH || - mlx4_is_eth(dev, port + 1)) { + if (optpar & (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH | MLX4_QP_OPTPAR_SCHED_QUEUE) || + qpc->pri_path.sched_queue || mlx4_is_eth(dev, port + 1)) { qpc->pri_path.sched_queue = pri_sched_queue; } From e5dfbf9a7904b3e21e665bed923eda49ced732d4 Mon Sep 17 00:00:00 2001 From: Or Gerlitz Date: Thu, 21 May 2015 15:14:09 +0300 Subject: [PATCH 5/6] net/mlx4_core: Adjust the schedule queue port in reset-to-init too It's legal for drivers to provide the QP port through the QPC schedule-queue field on the reset-to-init QP state change. Add adjusting of the schedule queue port in the SRIOV wrapper for that operation too. Signed-off-by: Or Gerlitz Signed-off-by: Jack Morgenstein Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx4/resource_tracker.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c index 82d562b90865e..15ec081816588 100644 --- a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c +++ b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c @@ -2703,6 +2703,10 @@ static void adjust_proxy_tun_qkey(struct mlx4_dev *dev, struct mlx4_vhcr *vhcr, context->qkey = cpu_to_be32(qkey); } +static int adjust_qp_sched_queue(struct mlx4_dev *dev, int slave, + struct mlx4_qp_context *qpc, + struct mlx4_cmd_mailbox *inbox); + int mlx4_RST2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, struct mlx4_cmd_mailbox *inbox, @@ -2725,6 +2729,10 @@ int mlx4_RST2INIT_QP_wrapper(struct mlx4_dev *dev, int slave, struct res_srq *srq; int local_qpn = be32_to_cpu(qpc->local_qpn) & 0xffffff; + err = adjust_qp_sched_queue(dev, slave, qpc, inbox); + if (err) + return err; + err = qp_res_start_move_to(dev, slave, qpn, RES_QP_HW, &qp, 0); if (err) return err; From be9b9eca257138b65c2864482a9edf6e2a8cedb6 Mon Sep 17 00:00:00 2001 From: Or Gerlitz Date: Thu, 21 May 2015 15:14:10 +0300 Subject: [PATCH 6/6] net/mlx4_core: Enable single ported IB VFs Remove the limitation that disallows configuring single ported VFs in the presence of IB ports, after addressing the issues that prevented that to work. SMI (QP0) requests/responses are still not supported for single ported IB VFs. Signed-off-by: Or Gerlitz Signed-off-by: Jack Morgenstein Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx4/cmd.c | 11 +++++++++++ drivers/net/ethernet/mellanox/mlx4/main.c | 12 ------------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/cmd.c b/drivers/net/ethernet/mellanox/mlx4/cmd.c index 153fb15769a86..77610452358b0 100644 --- a/drivers/net/ethernet/mellanox/mlx4/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx4/cmd.c @@ -3194,6 +3194,12 @@ int mlx4_vf_set_enable_smi_admin(struct mlx4_dev *dev, int slave, int port, int enabled) { struct mlx4_priv *priv = mlx4_priv(dev); + struct mlx4_active_ports actv_ports = mlx4_get_active_ports( + &priv->dev, slave); + int min_port = find_first_bit(actv_ports.ports, + priv->dev.caps.num_ports) + 1; + int max_port = min_port - 1 + + bitmap_weight(actv_ports.ports, priv->dev.caps.num_ports); if (slave == mlx4_master_func_num(dev)) return 0; @@ -3203,6 +3209,11 @@ int mlx4_vf_set_enable_smi_admin(struct mlx4_dev *dev, int slave, int port, enabled < 0 || enabled > 1) return -EINVAL; + if (min_port == max_port && dev->caps.num_ports > 1) { + mlx4_info(dev, "SMI access disallowed for single ported VFs\n"); + return -EPROTONOSUPPORT; + } + priv->mfunc.master.vf_admin[slave].enable_smi[port] = enabled; return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index ced5ecab5aa75..70d33f6e2a41a 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -2988,18 +2988,6 @@ static int mlx4_load_one(struct pci_dev *pdev, int pci_dev_data, /* In master functions, the communication channel must be initialized * after obtaining its address from fw */ if (mlx4_is_master(dev)) { - int ib_ports = 0; - - mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) - ib_ports++; - - if (ib_ports && - (num_vfs_argc > 1 || probe_vfs_argc > 1)) { - mlx4_err(dev, - "Invalid syntax of num_vfs/probe_vfs with IB port - single port VFs syntax is only supported when all ports are configured as ethernet\n"); - err = -EINVAL; - goto err_close; - } if (dev->caps.num_ports < 2 && num_vfs_argc > 1) { err = -EINVAL;