Skip to content

Commit

Permalink
Merge branch 'mlx4'
Browse files Browse the repository at this point in the history 
Or Gerlitz says:

====================
Mellanox driver updates

This patch set from Jack Morgenstein does the following:

1. Fix MAC/VLAN SRIOV implementation, and add wrapper functions for VLAN allocation
   and de-allocation (patches 1-6).

2. Implements resource quotas when running under SRIOV (patches 7-10).
   Patch 7 is a small bug fix, and patches 8-10 implement the quotas.

Quotas are implemented per resource type for VFs and the PF, to prevent
any entity from simply grabbing all the resources for itself and leaving
the other entities unable to obtain such resources.

The series is against net-next commit ba48650 "ipv6: remove the unnecessary statement in find_match()"

changes from V0:
 - dropped the 1st patch which needs to go to -stable and hence through net,
   not net-next
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
David S. Miller committed Nov 4, 2013
2 parents 3b4c5cb + 146f3ef commit 5a6e55c
Show file tree
Hide file tree
Showing 11 changed files with 675 additions and 120 deletions.
8 changes: 4 additions & 4 deletions drivers/infiniband/hw/mlx4/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,18 +177,18 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,

props->max_mr_size = ~0ull;
props->page_size_cap = dev->dev->caps.page_size_cap;
props->max_qp = dev->dev->caps.num_qps - dev->dev->caps.reserved_qps;
props->max_qp = dev->dev->quotas.qp;
props->max_qp_wr = dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE;
props->max_sge = min(dev->dev->caps.max_sq_sg,
dev->dev->caps.max_rq_sg);
props->max_cq = dev->dev->caps.num_cqs - dev->dev->caps.reserved_cqs;
props->max_cq = dev->dev->quotas.cq;
props->max_cqe = dev->dev->caps.max_cqes;
props->max_mr = dev->dev->caps.num_mpts - dev->dev->caps.reserved_mrws;
props->max_mr = dev->dev->quotas.mpt;
props->max_pd = dev->dev->caps.num_pds - dev->dev->caps.reserved_pds;
props->max_qp_rd_atom = dev->dev->caps.max_qp_dest_rdma;
props->max_qp_init_rd_atom = dev->dev->caps.max_qp_init_rdma;
props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
props->max_srq = dev->dev->caps.num_srqs - dev->dev->caps.reserved_srqs;
props->max_srq = dev->dev->quotas.srq;
props->max_srq_wr = dev->dev->caps.max_srq_wqes - 1;
props->max_srq_sge = dev->dev->caps.max_srq_sge;
props->max_fast_reg_page_list_len = MLX4_MAX_FAST_REG_PAGES;
Expand Down
3 changes: 2 additions & 1 deletion drivers/net/ethernet/mellanox/mlx4/cmd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1687,7 +1687,7 @@ static void mlx4_master_deactivate_admin_state(struct mlx4_priv *priv, int slave
vp_oper = &priv->mfunc.master.vf_oper[slave].vport[port];
if (NO_INDX != vp_oper->vlan_idx) {
__mlx4_unregister_vlan(&priv->dev,
port, vp_oper->vlan_idx);
port, vp_oper->state.default_vlan);
vp_oper->vlan_idx = NO_INDX;
}
if (NO_INDX != vp_oper->mac_idx) {
Expand Down Expand Up @@ -1718,6 +1718,7 @@ static void mlx4_master_do_cmd(struct mlx4_dev *dev, int slave, u8 cmd,
if (cmd == MLX4_COMM_CMD_RESET) {
mlx4_warn(dev, "Received reset from slave:%d\n", slave);
slave_state[slave].active = false;
slave_state[slave].old_vlan_api = false;
mlx4_master_deactivate_admin_state(priv, slave);
for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i) {
slave_state[slave].event_eq[i].eqn = -1;
Expand Down
6 changes: 1 addition & 5 deletions drivers/net/ethernet/mellanox/mlx4/en_netdev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,18 +417,14 @@ static int mlx4_en_vlan_rx_kill_vid(struct net_device *dev,
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_en_dev *mdev = priv->mdev;
int err;
int idx;

en_dbg(HW, priv, "Killing VID:%d\n", vid);

clear_bit(vid, priv->active_vlans);

/* Remove VID from port VLAN filter */
mutex_lock(&mdev->state_lock);
if (!mlx4_find_cached_vlan(mdev->dev, priv->port, vid, &idx))
mlx4_unregister_vlan(mdev->dev, priv->port, idx);
else
en_dbg(HW, priv, "could not find vid %d in cache\n", vid);
mlx4_unregister_vlan(mdev->dev, priv->port, vid);

if (mdev->device_up && priv->port_up) {
err = mlx4_SET_VLAN_FLTR(mdev->dev, priv);
Expand Down
99 changes: 71 additions & 28 deletions drivers/net/ethernet/mellanox/mlx4/fw.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
struct mlx4_cmd_mailbox *outbox,
struct mlx4_cmd_info *cmd)
{
struct mlx4_priv *priv = mlx4_priv(dev);
u8 field;
u32 size;
int err = 0;
Expand All @@ -185,18 +186,26 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
#define QUERY_FUNC_CAP_NUM_PORTS_OFFSET 0x1
#define QUERY_FUNC_CAP_PF_BHVR_OFFSET 0x4
#define QUERY_FUNC_CAP_FMR_OFFSET 0x8
#define QUERY_FUNC_CAP_QP_QUOTA_OFFSET 0x10
#define QUERY_FUNC_CAP_CQ_QUOTA_OFFSET 0x14
#define QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET 0x18
#define QUERY_FUNC_CAP_MPT_QUOTA_OFFSET 0x20
#define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET 0x24
#define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET 0x28
#define QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP 0x10
#define QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP 0x14
#define QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP 0x18
#define QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP 0x20
#define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP 0x24
#define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP 0x28
#define QUERY_FUNC_CAP_MAX_EQ_OFFSET 0x2c
#define QUERY_FUNC_CAP_RESERVED_EQ_OFFSET 0x30

#define QUERY_FUNC_CAP_QP_QUOTA_OFFSET 0x50
#define QUERY_FUNC_CAP_CQ_QUOTA_OFFSET 0x54
#define QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET 0x58
#define QUERY_FUNC_CAP_MPT_QUOTA_OFFSET 0x60
#define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET 0x64
#define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET 0x68

#define QUERY_FUNC_CAP_FMR_FLAG 0x80
#define QUERY_FUNC_CAP_FLAG_RDMA 0x40
#define QUERY_FUNC_CAP_FLAG_ETH 0x80
#define QUERY_FUNC_CAP_FLAG_QUOTAS 0x10

/* when opcode modifier = 1 */
#define QUERY_FUNC_CAP_PHYS_PORT_OFFSET 0x3
Expand Down Expand Up @@ -237,8 +246,9 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP1_PROXY);

} else if (vhcr->op_modifier == 0) {
/* enable rdma and ethernet interfaces */
field = (QUERY_FUNC_CAP_FLAG_ETH | QUERY_FUNC_CAP_FLAG_RDMA);
/* enable rdma and ethernet interfaces, and new quota locations */
field = (QUERY_FUNC_CAP_FLAG_ETH | QUERY_FUNC_CAP_FLAG_RDMA |
QUERY_FUNC_CAP_FLAG_QUOTAS);
MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS_OFFSET);

field = dev->caps.num_ports;
Expand All @@ -250,29 +260,40 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave,
field = 0; /* protected FMR support not available as yet */
MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FMR_OFFSET);

size = dev->caps.num_qps;
size = priv->mfunc.master.res_tracker.res_alloc[RES_QP].quota[slave];
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP_QUOTA_OFFSET);
size = dev->caps.num_qps;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP);

size = dev->caps.num_srqs;
size = priv->mfunc.master.res_tracker.res_alloc[RES_SRQ].quota[slave];
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET);
size = dev->caps.num_srqs;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP);

size = dev->caps.num_cqs;
size = priv->mfunc.master.res_tracker.res_alloc[RES_CQ].quota[slave];
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET);
size = dev->caps.num_cqs;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP);

size = dev->caps.num_eqs;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MAX_EQ_OFFSET);

size = dev->caps.reserved_eqs;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET);

size = dev->caps.num_mpts;
size = priv->mfunc.master.res_tracker.res_alloc[RES_MPT].quota[slave];
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET);
size = dev->caps.num_mpts;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP);

size = dev->caps.num_mtts;
size = priv->mfunc.master.res_tracker.res_alloc[RES_MTT].quota[slave];
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET);
size = dev->caps.num_mtts;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP);

size = dev->caps.num_mgms + dev->caps.num_amgms;
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET);
MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP);

} else
err = -EINVAL;
Expand All @@ -287,7 +308,7 @@ int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u32 gen_or_port,
u32 *outbox;
u8 field, op_modifier;
u32 size;
int err = 0;
int err = 0, quotas = 0;

op_modifier = !!gen_or_port; /* 0 = general, 1 = logical port */

Expand All @@ -311,36 +332,58 @@ int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u32 gen_or_port,
goto out;
}
func_cap->flags = field;
quotas = !!(func_cap->flags & QUERY_FUNC_CAP_FLAG_QUOTAS);

MLX4_GET(field, outbox, QUERY_FUNC_CAP_NUM_PORTS_OFFSET);
func_cap->num_ports = field;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_PF_BHVR_OFFSET);
func_cap->pf_context_behaviour = size;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP_QUOTA_OFFSET);
func_cap->qp_quota = size & 0xFFFFFF;
if (quotas) {
MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP_QUOTA_OFFSET);
func_cap->qp_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET);
func_cap->srq_quota = size & 0xFFFFFF;
MLX4_GET(size, outbox, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET);
func_cap->srq_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET);
func_cap->cq_quota = size & 0xFFFFFF;
MLX4_GET(size, outbox, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET);
func_cap->cq_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET);
func_cap->mpt_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET);
func_cap->mtt_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET);
func_cap->mcg_quota = size & 0xFFFFFF;

} else {
MLX4_GET(size, outbox, QUERY_FUNC_CAP_QP_QUOTA_OFFSET_DEP);
func_cap->qp_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_SRQ_QUOTA_OFFSET_DEP);
func_cap->srq_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_CQ_QUOTA_OFFSET_DEP);
func_cap->cq_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET_DEP);
func_cap->mpt_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET_DEP);
func_cap->mtt_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP);
func_cap->mcg_quota = size & 0xFFFFFF;
}
MLX4_GET(size, outbox, QUERY_FUNC_CAP_MAX_EQ_OFFSET);
func_cap->max_eq = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET);
func_cap->reserved_eq = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MPT_QUOTA_OFFSET);
func_cap->mpt_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MTT_QUOTA_OFFSET);
func_cap->mtt_quota = size & 0xFFFFFF;

MLX4_GET(size, outbox, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET);
func_cap->mcg_quota = size & 0xFFFFFF;
goto out;
}

Expand Down
32 changes: 22 additions & 10 deletions drivers/net/ethernet/mellanox/mlx4/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -562,13 +562,17 @@ static int mlx4_slave_cap(struct mlx4_dev *dev)
}

dev->caps.num_ports = func_cap.num_ports;
dev->caps.num_qps = func_cap.qp_quota;
dev->caps.num_srqs = func_cap.srq_quota;
dev->caps.num_cqs = func_cap.cq_quota;
dev->caps.num_eqs = func_cap.max_eq;
dev->caps.reserved_eqs = func_cap.reserved_eq;
dev->caps.num_mpts = func_cap.mpt_quota;
dev->caps.num_mtts = func_cap.mtt_quota;
dev->quotas.qp = func_cap.qp_quota;
dev->quotas.srq = func_cap.srq_quota;
dev->quotas.cq = func_cap.cq_quota;
dev->quotas.mpt = func_cap.mpt_quota;
dev->quotas.mtt = func_cap.mtt_quota;
dev->caps.num_qps = 1 << hca_param.log_num_qps;
dev->caps.num_srqs = 1 << hca_param.log_num_srqs;
dev->caps.num_cqs = 1 << hca_param.log_num_cqs;
dev->caps.num_mpts = 1 << hca_param.log_mpt_sz;
dev->caps.num_eqs = func_cap.max_eq;
dev->caps.reserved_eqs = func_cap.reserved_eq;
dev->caps.num_pds = MLX4_NUM_PDS;
dev->caps.num_mgms = 0;
dev->caps.num_amgms = 0;
Expand Down Expand Up @@ -2102,9 +2106,15 @@ static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data)
"aborting.\n");
return err;
}
if (num_vfs > MLX4_MAX_NUM_VF) {
printk(KERN_ERR "There are more VF's (%d) than allowed(%d)\n",
num_vfs, MLX4_MAX_NUM_VF);

/* Due to requirement that all VFs and the PF are *guaranteed* 2 MACS
* per port, we must limit the number of VFs to 63 (since their are
* 128 MACs)
*/
if (num_vfs >= MLX4_MAX_NUM_VF) {
dev_err(&pdev->dev,
"Requested more VF's (%d) than allowed (%d)\n",
num_vfs, MLX4_MAX_NUM_VF - 1);
return -EINVAL;
}

Expand Down Expand Up @@ -2322,6 +2332,8 @@ static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data)
if (err)
goto err_steer;

mlx4_init_quotas(dev);

for (port = 1; port <= dev->caps.num_ports; port++) {
err = mlx4_init_port_info(dev, port);
if (err)
Expand Down
21 changes: 20 additions & 1 deletion drivers/net/ethernet/mellanox/mlx4/mlx4.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,7 @@ struct mlx4_slave_state {
u8 last_cmd;
u8 init_port_mask;
bool active;
bool old_vlan_api;
u8 function;
dma_addr_t vhcr_dma;
u16 mtu[MLX4_MAX_PORTS + 1];
Expand Down Expand Up @@ -503,12 +504,28 @@ struct slave_list {
struct list_head res_list[MLX4_NUM_OF_RESOURCE_TYPE];
};

struct resource_allocator {
spinlock_t alloc_lock; /* protect quotas */
union {
int res_reserved;
int res_port_rsvd[MLX4_MAX_PORTS];
};
union {
int res_free;
int res_port_free[MLX4_MAX_PORTS];
};
int *quota;
int *allocated;
int *guaranteed;
};

struct mlx4_resource_tracker {
spinlock_t lock;
/* tree for each resources */
struct rb_root res_tree[MLX4_NUM_OF_RESOURCE_TYPE];
/* num_of_slave's lists, one per slave */
struct slave_list *slave_list;
struct resource_allocator res_alloc[MLX4_NUM_OF_RESOURCE_TYPE];
};

#define SLAVE_EVENT_EQ_SIZE 128
Expand Down Expand Up @@ -1111,7 +1128,7 @@ int mlx4_change_port_types(struct mlx4_dev *dev,

void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table);
void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table);
void __mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, int index);
void __mlx4_unregister_vlan(struct mlx4_dev *dev, u8 port, u16 vlan);
int __mlx4_register_vlan(struct mlx4_dev *dev, u8 port, u16 vlan, int *index);

int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port, int pkey_tbl_sz);
Expand Down Expand Up @@ -1252,4 +1269,6 @@ static inline spinlock_t *mlx4_tlock(struct mlx4_dev *dev)

void mlx4_vf_immed_vlan_work_handler(struct work_struct *_work);

void mlx4_init_quotas(struct mlx4_dev *dev);

#endif /* MLX4_H */
6 changes: 3 additions & 3 deletions drivers/net/ethernet/mellanox/mlx4/mr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -755,14 +755,14 @@ int mlx4_init_mr_table(struct mlx4_dev *dev)
struct mlx4_mr_table *mr_table = &priv->mr_table;
int err;

if (!is_power_of_2(dev->caps.num_mpts))
return -EINVAL;

/* Nothing to do for slaves - all MR handling is forwarded
* to the master */
if (mlx4_is_slave(dev))
return 0;

if (!is_power_of_2(dev->caps.num_mpts))
return -EINVAL;

err = mlx4_bitmap_init(&mr_table->mpt_bitmap, dev->caps.num_mpts,
~0, dev->caps.reserved_mrws, 0);
if (err)
Expand Down
Loading

0 comments on commit 5a6e55c

Please sign in to comment.