Skip to content

Commit

Permalink
mlx4_core: Added FW commands and their wrappers for supporting SRIOV
Browse files Browse the repository at this point in the history 
The following commands are added here:
1. QUERY_FUNC_CAP and its wrapper.  This function is used by VFs when
   they start up to receive configuration information from the PF, such
   as resource quotas for this VF, which ports should be used (currently
   two), what protocol is running on the port (currently Ethernet ONLY,
   or port not active).

2. QUERY_PORT and its wrapper. Previously, this FW command was invoked directly
   by the ETH driver (en_port.c) using mlx4_cmd_box. Virtualization is now
   required here (the VF's MAC address must be substituted for the PFs
   MAC address returned by the FW). We changed the invocation
   in the ETH driver to use mlx4_QUERY_PORT, and added the wrapper.

3. QUERY_HCA. Used by the VF to determine how the HCA was initialized.
   For now, we need only the multicast table member entry size
   (log2_mc_table_entry_sz, in the ConnectX PRM).  No wrapper is needed
   here, because the data may be passed as is to the VF without modification).

   In this command, we have added a GLOBAL_CAPS field for passing required
   configuration information from FW to a VF (this field is to allow safely
                   adding new SRIOV capabilities which require support in VF drivers, too).
   Bits will set here by FW in response to PF-driver configuration commands which
   will activate as yet undefined new SRIOV features. The VF will test to see that
   all required capabilities indicated by this field are supported (i.e., if a bit
   is set and the VF driver does not recognize that bit, it must abort
   its initialization).  Currently, no bits are set.

4. Added a CLOSE_PORT wrapper.  The PF context needs to keep track of how many VF contexts
   have the port open.  The PF context will not actually issue the FW close port command
   until the last port user issues a CLOSE_PORT request.

Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: Yevgeny Petrilin <yevgenyp@mellanox.co.il>
Signed-off-by: Marcel Apfelbaum <marcela@mellanox.co.il>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Marcel Apfelbaum authored and David S. Miller committed Dec 13, 2011
1 parent e8f081a commit 5cc914f
Show file tree
Hide file tree
Showing 3 changed files with 335 additions and 0 deletions.
291 changes: 291 additions & 0 deletions drivers/net/ethernet/mellanox/mlx4/fw.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
* SOFTWARE.
*/

#include <linux/etherdevice.h>
#include <linux/mlx4/cmd.h>
#include <linux/module.h>
#include <linux/cache.h>
Expand Down Expand Up @@ -145,6 +146,179 @@ int mlx4_MOD_STAT_CFG(struct mlx4_dev *dev, struct mlx4_mod_stat_cfg *cfg)
return err;
}

int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
struct mlx4_vhcr *vhcr,
struct mlx4_cmd_mailbox *inbox,
struct mlx4_cmd_mailbox *outbox,
struct mlx4_cmd_info *cmd)
{
u8 field;
u32 size;
int err = 0;

#define QUERY_FUNC_CAP_FLAGS_OFFSET 0x0
#define QUERY_FUNC_CAP_NUM_PORTS_OFFSET 0x1
#define QUERY_FUNC_CAP_FUNCTION_OFFSET 0x3
#define QUERY_FUNC_CAP_PF_BHVR_OFFSET 0x4
#define QUERY_FUNC_CAP_QP_QUOTA_OFFSET 0x10
#define QUERY_FUNC_CAP_CQ_QUOTA_OFFSET 0x14
#define QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET 0x18
#define QUERY_FUNC_CAP_MPT_QUOTA_OFFSET 0x20
#define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET 0x24
#define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET 0x28
#define QUERY_FUNC_CAP_MAX_EQ_OFFSET 0x2c
#define QUERY_FUNC_CAP_RESERVED_EQ_OFFSET 0X30

#define QUERY_FUNC_CAP_PHYS_PORT_OFFSET 0x3
#define QUERY_FUNC_CAP_ETH_PROPS_OFFSET 0xc

if (vhcr->op_modifier == 1) {
field = vhcr->in_modifier;
MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_PHYS_PORT_OFFSET);

field = 0; /* ensure fvl bit is not set */
MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_ETH_PROPS_OFFSET);
} else if (vhcr->op_modifier == 0) {
field = 1 << 7; /* enable only ethernet interface */
MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS_OFFSET);

field = slave;
MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FUNCTION_OFFSET);

field = dev->caps.num_ports;
MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_NUM_PORTS_OFFSET);

size = 0; /* no PF behavious is set for now */
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_PF_BHVR_OFFSET);

size = dev->caps.num_qps;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP_QUOTA_OFFSET);

size = dev->caps.num_srqs;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET);

size = dev->caps.num_cqs;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET);

size = dev->caps.num_eqs;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MAX_EQ_OFFSET);

size = dev->caps.reserved_eqs;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET);

size = dev->caps.num_mpts;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET);

size = dev->caps.num_mtt_segs * dev->caps.mtts_per_seg;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET);

size = dev->caps.num_mgms + dev->caps.num_amgms;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET);

} else
err = -EINVAL;

return err;
}

int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, struct mlx4_func_cap *func_cap)
{
struct mlx4_cmd_mailbox *mailbox;
u32 *outbox;
u8 field;
u32 size;
int i;
int err = 0;


mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox))
return PTR_ERR(mailbox);

err = mlx4_cmd_box(dev, 0, mailbox->dma, 0, 0, MLX4_CMD_QUERY_FUNC_CAP,
MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
if (err)
goto out;

outbox = mailbox->buf;

MLX4_GET(field, outbox, QUERY_FUNC_CAP_FLAGS_OFFSET);
if (!(field & (1 << 7))) {
mlx4_err(dev, "The host doesn't support eth interface\n");
err = -EPROTONOSUPPORT;
goto out;
}

MLX4_GET(field, outbox, QUERY_FUNC_CAP_FUNCTION_OFFSET);
func_cap->function = field;

MLX4_GET(field, outbox, QUERY_FUNC_CAP_NUM_PORTS_OFFSET);
func_cap->num_ports = field;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_PF_BHVR_OFFSET);
func_cap->pf_context_behaviour = size;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP_QUOTA_OFFSET);
func_cap->qp_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET);
func_cap->srq_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET);
func_cap->cq_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MAX_EQ_OFFSET);
func_cap->max_eq = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET);
func_cap->reserved_eq = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET);
func_cap->mpt_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET);
func_cap->mtt_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET);
func_cap->mcg_quota = size & 0xFFFFFF;

for (i = 1; i <= func_cap->num_ports; ++i) {
err = mlx4_cmd_box(dev, 0, mailbox->dma, i, 1,
MLX4_CMD_QUERY_FUNC_CAP,
MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
if (err)
goto out;

MLX4_GET(field, outbox, QUERY_FUNC_CAP_ETH_PROPS_OFFSET);
if (field & (1 << 7)) {
mlx4_err(dev, "VLAN is enforced on this port\n");
err = -EPROTONOSUPPORT;
goto out;
}

if (field & (1 << 6)) {
mlx4_err(dev, "Force mac is enabled on this port\n");
err = -EPROTONOSUPPORT;
goto out;
}

MLX4_GET(field, outbox, QUERY_FUNC_CAP_PHYS_PORT_OFFSET);
func_cap->physical_port[i] = field;
}

/* All other resources are allocated by the master, but we still report
* 'num' and 'reserved' capabilities as follows:
* - num remains the maximum resource index
* - 'num - reserved' is the total available objects of a resource, but
* resource indices may be less than 'reserved'
* TODO: set per-resource quotas */

out:
mlx4_free_cmd_mailbox(dev, mailbox);

return err;
}

int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
{
struct mlx4_cmd_mailbox *mailbox;
Expand Down Expand Up @@ -471,6 +645,54 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
return err;
}

int mlx4_QUERY_PORT_wrapper(struct mlx4_dev *dev, int slave,
struct mlx4_vhcr *vhcr,
struct mlx4_cmd_mailbox *inbox,
struct mlx4_cmd_mailbox *outbox,
struct mlx4_cmd_info *cmd)
{
u64 def_mac;
u8 port_type;
int err;

err = mlx4_cmd_box(dev, 0, outbox->dma, vhcr->in_modifier, 0,
MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B,
MLX4_CMD_NATIVE);

if (!err && dev->caps.function != slave) {
/* set slave default_mac address */
MLX4_GET(def_mac, outbox->buf, QUERY_PORT_MAC_OFFSET);
def_mac += slave << 8;
MLX4_PUT(outbox->buf, def_mac, QUERY_PORT_MAC_OFFSET);

/* get port type - currently only eth is enabled */
MLX4_GET(port_type, outbox->buf,
QUERY_PORT_SUPPORTED_TYPE_OFFSET);

/* disable ib */
port_type &= 0xFE;

/* check eth is enabled for this port */
if (!(port_type & 2))
mlx4_dbg(dev, "QUERY PORT: eth not supported by host");

MLX4_PUT(outbox->buf, port_type,
QUERY_PORT_SUPPORTED_TYPE_OFFSET);
}

return err;
}

static int mlx4_QUERY_PORT(struct mlx4_dev *dev, void *ptr, u8 port)
{
struct mlx4_cmd_mailbox *outbox = ptr;

return mlx4_cmd_box(dev, 0, outbox->dma, port, 0,
MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B,
MLX4_CMD_WRAPPED);
}
EXPORT_SYMBOL_GPL(mlx4_QUERY_PORT);

int mlx4_map_cmd(struct mlx4_dev *dev, u16 op, struct mlx4_icm *icm, u64 virt)
{
struct mlx4_cmd_mailbox *mailbox;
Expand Down Expand Up @@ -584,6 +806,7 @@ int mlx4_QUERY_FW(struct mlx4_dev *dev)

#define QUERY_FW_OUT_SIZE 0x100
#define QUERY_FW_VER_OFFSET 0x00
#define QUERY_FW_PPF_ID 0x09
#define QUERY_FW_CMD_IF_REV_OFFSET 0x0a
#define QUERY_FW_MAX_CMD_OFFSET 0x0f
#define QUERY_FW_ERR_START_OFFSET 0x30
Expand All @@ -594,6 +817,9 @@ int mlx4_QUERY_FW(struct mlx4_dev *dev)
#define QUERY_FW_CLR_INT_BASE_OFFSET 0x20
#define QUERY_FW_CLR_INT_BAR_OFFSET 0x28

#define QUERY_FW_COMM_BASE_OFFSET 0x40
#define QUERY_FW_COMM_BAR_OFFSET 0x48

mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox))
return PTR_ERR(mailbox);
Expand All @@ -613,6 +839,9 @@ int mlx4_QUERY_FW(struct mlx4_dev *dev)
((fw_ver & 0xffff0000ull) >> 16) |
((fw_ver & 0x0000ffffull) << 16);

MLX4_GET(lg, outbox, QUERY_FW_PPF_ID);
dev->caps.function = lg;

MLX4_GET(cmd_if_rev, outbox, QUERY_FW_CMD_IF_REV_OFFSET);
if (cmd_if_rev < MLX4_COMMAND_INTERFACE_MIN_REV ||
cmd_if_rev > MLX4_COMMAND_INTERFACE_MAX_REV) {
Expand Down Expand Up @@ -654,6 +883,11 @@ int mlx4_QUERY_FW(struct mlx4_dev *dev)
MLX4_GET(fw->clr_int_bar, outbox, QUERY_FW_CLR_INT_BAR_OFFSET);
fw->clr_int_bar = (fw->clr_int_bar >> 6) * 2;

MLX4_GET(fw->comm_base, outbox, QUERY_FW_COMM_BASE_OFFSET);
MLX4_GET(fw->comm_bar, outbox, QUERY_FW_COMM_BAR_OFFSET);
fw->comm_bar = (fw->comm_bar >> 6) * 2;
mlx4_dbg(dev, "Communication vector bar:%d offset:0x%llx\n",
fw->comm_bar, fw->comm_base);
mlx4_dbg(dev, "FW size %d KB\n", fw->fw_pages >> 2);

/*
Expand Down Expand Up @@ -748,6 +982,7 @@ int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param)
#define INIT_HCA_LOG_SRQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x2f)
#define INIT_HCA_CQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x30)
#define INIT_HCA_LOG_CQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x37)
#define INIT_HCA_EQE_CQE_OFFSETS (INIT_HCA_QPC_OFFSET + 0x38)
#define INIT_HCA_ALTC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x40)
#define INIT_HCA_AUXC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x50)
#define INIT_HCA_EQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x60)
Expand Down Expand Up @@ -849,6 +1084,35 @@ int mlx4_INIT_HCA(struct mlx4_dev *dev, struct mlx4_init_hca_param *param)
return err;
}

int mlx4_INIT_PORT_wrapper(struct mlx4_dev *dev, int slave,
struct mlx4_vhcr *vhcr,
struct mlx4_cmd_mailbox *inbox,
struct mlx4_cmd_mailbox *outbox,
struct mlx4_cmd_info *cmd)
{
struct mlx4_priv *priv = mlx4_priv(dev);
int port = vhcr->in_modifier;
int err;

if (priv->mfunc.master.slave_state[slave].init_port_mask & (1 << port))
return 0;

if (dev->caps.port_mask[port] == MLX4_PORT_TYPE_IB)
return -ENODEV;

/* Enable port only if it was previously disabled */
if (!priv->mfunc.master.init_port_ref[port]) {
err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_INIT_PORT,
MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE);
if (err)
return err;
priv->mfunc.master.slave_state[slave].init_port_mask |=
(1 << port);
}
++priv->mfunc.master.init_port_ref[port];
return 0;
}

int mlx4_INIT_PORT(struct mlx4_dev *dev, int port)
{
struct mlx4_cmd_mailbox *mailbox;
Expand Down Expand Up @@ -903,6 +1167,33 @@ int mlx4_INIT_PORT(struct mlx4_dev *dev, int port)
}
EXPORT_SYMBOL_GPL(mlx4_INIT_PORT);

int mlx4_CLOSE_PORT_wrapper(struct mlx4_dev *dev, int slave,
struct mlx4_vhcr *vhcr,
struct mlx4_cmd_mailbox *inbox,
struct mlx4_cmd_mailbox *outbox,
struct mlx4_cmd_info *cmd)
{
struct mlx4_priv *priv = mlx4_priv(dev);
int port = vhcr->in_modifier;
int err;

if (!(priv->mfunc.master.slave_state[slave].init_port_mask &
(1 << port)))
return 0;

if (dev->caps.port_mask[port] == MLX4_PORT_TYPE_IB)
return -ENODEV;
if (priv->mfunc.master.init_port_ref[port] == 1) {
err = mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, 1000,
MLX4_CMD_NATIVE);
if (err)
return err;
}
priv->mfunc.master.slave_state[slave].init_port_mask &= ~(1 << port);
--priv->mfunc.master.init_port_ref[port];
return 0;
}

int mlx4_CLOSE_PORT(struct mlx4_dev *dev, int port)
{
return mlx4_cmd(dev, 0, port, 0, MLX4_CMD_CLOSE_PORT, 1000,
Expand Down
Loading

0 comments on commit 5cc914f

Please sign in to comment.