Skip to content

Commit

Permalink
(IB, net)/mlx4: Add resource utilization support
Browse files Browse the repository at this point in the history 
Adding visibility of resource usage of QPs, CQs and counters used by
virtual functions. This feature will be used to give the PF administrator
more data while debugging VF status. Usage info was added to ALLOC_RES
command, to notify the PF if the resource which is being reserved or
allocated for the VF will be used by kernel driver or by user verbs.

Updated reservation and allocation functions of QP, CQ and counter with
additional usage parameter.

Signed-off-by: Moshe Shemesh <moshe@mellanox.com>
Signed-off-by: Leon Romanovsky <leon@kernel.org>
Signed-off-by: Doug Ledford <dledford@redhat.com>
  • Loading branch information
Moshe Shemesh authored and Doug Ledford committed Jul 24, 2017
1 parent ea30b96 commit f330187
Show file tree
Hide file tree
Showing 11 changed files with 45 additions and 20 deletions.
2 changes: 2 additions & 0 deletions drivers/infiniband/hw/mlx4/cq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
goto err_mtt;

uar = &to_mucontext(context)->uar;
cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS;
} else {
err = mlx4_db_alloc(dev->dev, &cq->db, 1);
if (err)
Expand All @@ -233,6 +234,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
goto err_db;

uar = &dev->priv_uar;
cq->mcq.usage = MLX4_RES_USAGE_DRIVER;
}

if (dev->eq_table)
Expand Down
6 changes: 4 additions & 2 deletions drivers/infiniband/hw/mlx4/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2779,7 +2779,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
allocated = 0;
if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i + 1) ==
IB_LINK_LAYER_ETHERNET) {
err = mlx4_counter_alloc(ibdev->dev, &counter_index);
err = mlx4_counter_alloc(ibdev->dev, &counter_index,
MLX4_RES_USAGE_DRIVER);
/* if failed to allocate a new counter, use default */
if (err)
counter_index =
Expand Down Expand Up @@ -2834,7 +2835,8 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
ibdev->steer_qpn_count = MLX4_IB_UC_MAX_NUM_QPS;
err = mlx4_qp_reserve_range(dev, ibdev->steer_qpn_count,
MLX4_IB_UC_STEER_QPN_ALIGN,
&ibdev->steer_qpn_base, 0);
&ibdev->steer_qpn_base, 0,
MLX4_RES_USAGE_DRIVER);
if (err)
goto err_counter;

Expand Down
13 changes: 9 additions & 4 deletions drivers/infiniband/hw/mlx4/qp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -769,6 +769,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
if (err)
goto err_mtt;
}
qp->mqp.usage = MLX4_RES_USAGE_USER_VERBS;
} else {
err = set_rq_size(dev, &init_attr->cap, !!pd->uobject,
qp_has_rq(init_attr), qp, 0);
Expand Down Expand Up @@ -841,6 +842,7 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
err = -ENOMEM;
goto err_wrid;
}
qp->mqp.usage = MLX4_RES_USAGE_DRIVER;
}

if (sqpn) {
Expand All @@ -860,13 +862,14 @@ static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
(init_attr->cap.max_send_wr ?
MLX4_RESERVE_ETH_BF_QP : 0) |
(init_attr->cap.max_recv_wr ?
MLX4_RESERVE_A0_QP : 0));
MLX4_RESERVE_A0_QP : 0),
qp->mqp.usage);
else
if (qp->flags & MLX4_IB_QP_NETIF)
err = mlx4_ib_steer_qp_alloc(dev, 1, &qpn);
else
err = mlx4_qp_reserve_range(dev->dev, 1, 1,
&qpn, 0);
&qpn, 0, qp->mqp.usage);
if (err)
goto err_proxy;
}
Expand Down Expand Up @@ -1218,7 +1221,9 @@ static struct ib_qp *_mlx4_ib_create_qp(struct ib_pd *pd,
if (udata)
return ERR_PTR(-EINVAL);
if (init_attr->create_flags & MLX4_IB_QP_CREATE_ROCE_V2_GSI) {
int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev, 1, 1, &sqpn, 0);
int res = mlx4_qp_reserve_range(to_mdev(pd->device)->dev,
1, 1, &sqpn, 0,
MLX4_RES_USAGE_DRIVER);

if (res)
return ERR_PTR(res);
Expand Down Expand Up @@ -1581,7 +1586,7 @@ static int create_qp_lb_counter(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
!(dev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_LB_SRC_CHK))
return 0;

err = mlx4_counter_alloc(dev->dev, &tmp_idx);
err = mlx4_counter_alloc(dev->dev, &tmp_idx, MLX4_RES_USAGE_DRIVER);
if (err)
return err;

Expand Down
7 changes: 4 additions & 3 deletions drivers/net/ethernet/mellanox/mlx4/cq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,13 +241,14 @@ int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
return err;
}

static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn, u8 usage)
{
u32 in_modifier = RES_CQ | (((u32)usage & 3) << 30);
u64 out_param;
int err;

if (mlx4_is_mfunc(dev)) {
err = mlx4_cmd_imm(dev, 0, &out_param, RES_CQ,
err = mlx4_cmd_imm(dev, 0, &out_param, in_modifier,
RES_OP_RESERVE_AND_MAP, MLX4_CMD_ALLOC_RES,
MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
if (err)
Expand Down Expand Up @@ -303,7 +304,7 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent,

cq->vector = vector;

err = mlx4_cq_alloc_icm(dev, &cq->cqn);
err = mlx4_cq_alloc_icm(dev, &cq->cqn, cq->usage);
if (err)
return err;

Expand Down
1 change: 1 addition & 0 deletions drivers/net/ethernet/mellanox/mlx4/en_cq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
(cq->type == RX && priv->hwtstamp_config.rx_filter))
timestamp_en = 1;

cq->mcq.usage = MLX4_RES_USAGE_DRIVER;
err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt,
&mdev->priv_uar, cq->wqres.db.dma, &cq->mcq,
cq->vector, 0, timestamp_en);
Expand Down
3 changes: 2 additions & 1 deletion drivers/net/ethernet/mellanox/mlx4/en_netdev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -651,7 +651,8 @@ static int mlx4_en_get_qp(struct mlx4_en_priv *priv)
return 0;
}

err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP);
err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP,
MLX4_RES_USAGE_DRIVER);
en_dbg(DRV, priv, "Reserved qp %d\n", *qpn);
if (err) {
en_err(priv, "Failed to reserve qp for mac registration\n");
Expand Down
6 changes: 4 additions & 2 deletions drivers/net/ethernet/mellanox/mlx4/en_rx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1081,7 +1081,8 @@ int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv)
u32 qpn;

err = mlx4_qp_reserve_range(priv->mdev->dev, 1, 1, &qpn,
MLX4_RESERVE_A0_QP);
MLX4_RESERVE_A0_QP,
MLX4_RES_USAGE_DRIVER);
if (err) {
en_err(priv, "Failed reserving drop qpn\n");
return err;
Expand Down Expand Up @@ -1127,7 +1128,8 @@ int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
flags = priv->rx_ring_num == 1 ? MLX4_RESERVE_A0_QP : 0;
err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
priv->rx_ring_num,
&rss_map->base_qpn, flags);
&rss_map->base_qpn, flags,
MLX4_RES_USAGE_DRIVER);
if (err) {
en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
return err;
Expand Down
3 changes: 2 additions & 1 deletion drivers/net/ethernet/mellanox/mlx4/en_tx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,8 @@ int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv,
(unsigned long long) ring->sp_wqres.buf.direct.map);

err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &ring->qpn,
MLX4_RESERVE_ETH_BF_QP);
MLX4_RESERVE_ETH_BF_QP,
MLX4_RES_USAGE_DRIVER);
if (err) {
en_err(priv, "failed reserving qp for TX ring\n");
goto err_hwq_res;
Expand Down
7 changes: 4 additions & 3 deletions drivers/net/ethernet/mellanox/mlx4/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2475,7 +2475,7 @@ static int mlx4_allocate_default_counters(struct mlx4_dev *dev)
priv->def_counter[port] = -1;

for (port = 0; port < dev->caps.num_ports; port++) {
err = mlx4_counter_alloc(dev, &idx);
err = mlx4_counter_alloc(dev, &idx, MLX4_RES_USAGE_DRIVER);

if (!err || err == -ENOSPC) {
priv->def_counter[port] = idx;
Expand Down Expand Up @@ -2517,13 +2517,14 @@ int __mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)
return 0;
}

int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)
int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx, u8 usage)
{
u32 in_modifier = RES_COUNTER | (((u32)usage & 3) << 30);
u64 out_param;
int err;

if (mlx4_is_mfunc(dev)) {
err = mlx4_cmd_imm(dev, 0, &out_param, RES_COUNTER,
err = mlx4_cmd_imm(dev, 0, &out_param, in_modifier,
RES_OP_RESERVE, MLX4_CMD_ALLOC_RES,
MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
if (!err)
Expand Down
5 changes: 3 additions & 2 deletions drivers/net/ethernet/mellanox/mlx4/qp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,8 +245,9 @@ int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
}

int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
int *base, u8 flags)
int *base, u8 flags, u8 usage)
{
u32 in_modifier = RES_QP | (((u32)usage & 3) << 30);
u64 in_param = 0;
u64 out_param;
int err;
Expand All @@ -258,7 +259,7 @@ int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
set_param_l(&in_param, (((u32)flags) << 24) | (u32)cnt);
set_param_h(&in_param, align);
err = mlx4_cmd_imm(dev, in_param, &out_param,
RES_QP, RES_OP_RESERVE,
in_modifier, RES_OP_RESERVE,
MLX4_CMD_ALLOC_RES,
MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
if (err)
Expand Down
12 changes: 10 additions & 2 deletions include/linux/mlx4/device.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,12 @@ enum mlx4_steer_type {
MLX4_NUM_STEERS
};

enum mlx4_resource_usage {
MLX4_RES_USAGE_NONE,
MLX4_RES_USAGE_DRIVER,
MLX4_RES_USAGE_USER_VERBS,
};

enum {
MLX4_NUM_FEXCH = 64 * 1024,
};
Expand Down Expand Up @@ -748,6 +754,7 @@ struct mlx4_cq {
} tasklet_ctx;
int reset_notify_added;
struct list_head reset_notify;
u8 usage;
};

struct mlx4_qp {
Expand All @@ -757,6 +764,7 @@ struct mlx4_qp {

atomic_t refcount;
struct completion free;
u8 usage;
};

struct mlx4_srq {
Expand Down Expand Up @@ -1120,7 +1128,7 @@ int mlx4_cq_alloc(struct mlx4_dev *dev, int nent, struct mlx4_mtt *mtt,
unsigned vector, int collapsed, int timestamp_en);
void mlx4_cq_free(struct mlx4_dev *dev, struct mlx4_cq *cq);
int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
int *base, u8 flags);
int *base, u8 flags, u8 usage);
void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt);

int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp);
Expand Down Expand Up @@ -1417,7 +1425,7 @@ int mlx4_get_phys_port_id(struct mlx4_dev *dev);
int mlx4_wol_read(struct mlx4_dev *dev, u64 *config, int port);
int mlx4_wol_write(struct mlx4_dev *dev, u64 config, int port);

int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx);
int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx, u8 usage);
void mlx4_counter_free(struct mlx4_dev *dev, u32 idx);
int mlx4_get_default_counter_index(struct mlx4_dev *dev, int port);

Expand Down

0 comments on commit f330187

Please sign in to comment.