Skip to content

Commit

Permalink
RDMA/bnxt_re: Refactor NQ allocation
Browse files Browse the repository at this point in the history 
Move NQ related data structures from rdev to a new structure
named "struct bnxt_re_nq_record" by keeping a pointer to in
the rdev structure. Allocate the memory for it dynamically.
This change is needed for subsequent patches in the series.

Also, removed the nq_task variable from rdev structure as it
is redundant and no longer used.

This change would help to reduce the size of the driver private
structure as well.

Reviewed-by: Chandramohan Akula <chandramohan.akula@broadcom.com>
Signed-off-by: Kalesh AP <kalesh-anakkur.purayil@broadcom.com>
Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com>
Link: https://patch.msgid.link/1731577748-1804-3-git-send-email-selvin.xavier@broadcom.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>
  • Loading branch information
Kalesh AP authored and Leon Romanovsky committed Nov 14, 2024
1 parent 65ecee1 commit 30b8713
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 33 deletions.
13 changes: 7 additions & 6 deletions drivers/infiniband/hw/bnxt_re/bnxt_re.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,11 @@ struct bnxt_re_pacing {
#define BNXT_RE_GRC_FIFO_REG_BASE 0x2000

#define BNXT_RE_MIN_MSIX 2
#define BNXT_RE_MAX_MSIX BNXT_MAX_ROCE_MSIX
struct bnxt_re_nq_record {
struct bnxt_qplib_nq nq[BNXT_RE_MAX_MSIX];
int num_msix;
};

#define MAX_CQ_HASH_BITS (16)
#define MAX_SRQ_HASH_BITS (16)
Expand Down Expand Up @@ -183,21 +188,17 @@ struct bnxt_re_dev {
unsigned int version, major, minor;
struct bnxt_qplib_chip_ctx *chip_ctx;
struct bnxt_en_dev *en_dev;
int num_msix;

int id;

struct delayed_work worker;
u8 cur_prio_map;

/* FP Notification Queue (CQ & SRQ) */
struct tasklet_struct nq_task;

/* RCFW Channel */
struct bnxt_qplib_rcfw rcfw;

/* NQ */
struct bnxt_qplib_nq nq[BNXT_MAX_ROCE_MSIX];
/* NQ record */
struct bnxt_re_nq_record *nqr;

/* Device Resources */
struct bnxt_qplib_dev_attr dev_attr;
Expand Down
6 changes: 3 additions & 3 deletions drivers/infiniband/hw/bnxt_re/ib_verbs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1872,8 +1872,8 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
srq->qplib_srq.wqe_size = bnxt_re_get_rwqe_size(dev_attr->max_srq_sges);
srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
srq->srq_limit = srq_init_attr->attr.srq_limit;
srq->qplib_srq.eventq_hw_ring_id = rdev->nq[0].ring_id;
nq = &rdev->nq[0];
srq->qplib_srq.eventq_hw_ring_id = rdev->nqr->nq[0].ring_id;
nq = &rdev->nqr->nq[0];

if (udata) {
rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
Expand Down Expand Up @@ -3122,7 +3122,7 @@ int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
* used for getting the NQ index.
*/
nq_alloc_cnt = atomic_inc_return(&rdev->nq_alloc_cnt);
nq = &rdev->nq[nq_alloc_cnt % (rdev->num_msix - 1)];
nq = &rdev->nqr->nq[nq_alloc_cnt % (rdev->nqr->num_msix - 1)];
cq->qplib_cq.max_wqe = entries;
cq->qplib_cq.cnq_hw_ring_id = nq->ring_id;
cq->qplib_cq.nq = nq;
Expand Down
74 changes: 50 additions & 24 deletions drivers/infiniband/hw/bnxt_re/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,8 +326,8 @@ static void bnxt_re_stop_irq(void *handle)
rdev = en_info->rdev;
rcfw = &rdev->rcfw;

for (indx = BNXT_RE_NQ_IDX; indx < rdev->num_msix; indx++) {
nq = &rdev->nq[indx - 1];
for (indx = BNXT_RE_NQ_IDX; indx < rdev->nqr->num_msix; indx++) {
nq = &rdev->nqr->nq[indx - 1];
bnxt_qplib_nq_stop_irq(nq, false);
}

Expand Down Expand Up @@ -362,7 +362,7 @@ static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
/* Vectors may change after restart, so update with new vectors
* in device sctructure.
*/
for (indx = 0; indx < rdev->num_msix; indx++)
for (indx = 0; indx < rdev->nqr->num_msix; indx++)
rdev->en_dev->msix_entries[indx].vector = ent[indx].vector;

rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,
Expand All @@ -371,8 +371,8 @@ static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
ibdev_warn(&rdev->ibdev, "Failed to reinit CREQ\n");
return;
}
for (indx = BNXT_RE_NQ_IDX ; indx < rdev->num_msix; indx++) {
nq = &rdev->nq[indx - 1];
for (indx = BNXT_RE_NQ_IDX ; indx < rdev->nqr->num_msix; indx++) {
nq = &rdev->nqr->nq[indx - 1];
rc = bnxt_qplib_nq_start_irq(nq, indx - 1,
msix_ent[indx].vector, false);
if (rc) {
Expand Down Expand Up @@ -1206,7 +1206,7 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)

addrconf_addr_eui48((u8 *)&ibdev->node_guid, rdev->netdev->dev_addr);

ibdev->num_comp_vectors = rdev->num_msix - 1;
ibdev->num_comp_vectors = rdev->nqr->num_msix - 1;
ibdev->dev.parent = &rdev->en_dev->pdev->dev;
ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;

Expand Down Expand Up @@ -1551,8 +1551,8 @@ static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)
{
int i;

for (i = 1; i < rdev->num_msix; i++)
bnxt_qplib_disable_nq(&rdev->nq[i - 1]);
for (i = 1; i < rdev->nqr->num_msix; i++)
bnxt_qplib_disable_nq(&rdev->nqr->nq[i - 1]);

if (rdev->qplib_res.rcfw)
bnxt_qplib_cleanup_res(&rdev->qplib_res);
Expand All @@ -1566,9 +1566,9 @@ static int bnxt_re_init_res(struct bnxt_re_dev *rdev)

bnxt_qplib_init_res(&rdev->qplib_res);

for (i = 1; i < rdev->num_msix ; i++) {
for (i = 1; i < rdev->nqr->num_msix ; i++) {
db_offt = rdev->en_dev->msix_entries[i].db_offset;
rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq[i - 1],
rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nqr->nq[i - 1],
i - 1, rdev->en_dev->msix_entries[i].vector,
db_offt, &bnxt_re_cqn_handler,
&bnxt_re_srqn_handler);
Expand All @@ -1582,20 +1582,22 @@ static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
return 0;
fail:
for (i = num_vec_enabled; i >= 0; i--)
bnxt_qplib_disable_nq(&rdev->nq[i]);
bnxt_qplib_disable_nq(&rdev->nqr->nq[i]);
return rc;
}

static void bnxt_re_free_nq_res(struct bnxt_re_dev *rdev)
{
struct bnxt_qplib_nq *nq;
u8 type;
int i;

for (i = 0; i < rdev->num_msix - 1; i++) {
for (i = 0; i < rdev->nqr->num_msix - 1; i++) {
type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, type);
bnxt_qplib_free_nq(&rdev->nq[i]);
rdev->nq[i].res = NULL;
nq = &rdev->nqr->nq[i];
bnxt_re_net_ring_free(rdev, nq->ring_id, type);
bnxt_qplib_free_nq(nq);
nq->res = NULL;
}
}

Expand Down Expand Up @@ -1637,20 +1639,20 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
if (rc)
goto dealloc_res;

for (i = 0; i < rdev->num_msix - 1; i++) {
for (i = 0; i < rdev->nqr->num_msix - 1; i++) {
struct bnxt_qplib_nq *nq;

nq = &rdev->nq[i];
nq = &rdev->nqr->nq[i];
nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
rc = bnxt_qplib_alloc_nq(&rdev->qplib_res, &rdev->nq[i]);
rc = bnxt_qplib_alloc_nq(&rdev->qplib_res, nq);
if (rc) {
ibdev_err(&rdev->ibdev, "Alloc Failed NQ%d rc:%#x",
i, rc);
goto free_nq;
}
type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr;
rattr.pages = nq->hwq.pbl[rdev->nq[i].hwq.level].pg_count;
rattr.pages = nq->hwq.pbl[rdev->nqr->nq[i].hwq.level].pg_count;
rattr.type = type;
rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
rattr.depth = BNXT_QPLIB_NQE_MAX_CNT - 1;
Expand All @@ -1660,7 +1662,7 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
ibdev_err(&rdev->ibdev,
"Failed to allocate NQ fw id with rc = 0x%x",
rc);
bnxt_qplib_free_nq(&rdev->nq[i]);
bnxt_qplib_free_nq(nq);
goto free_nq;
}
num_vec_created++;
Expand All @@ -1669,8 +1671,8 @@ static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
free_nq:
for (i = num_vec_created - 1; i >= 0; i--) {
type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, type);
bnxt_qplib_free_nq(&rdev->nq[i]);
bnxt_re_net_ring_free(rdev, rdev->nqr->nq[i].ring_id, type);
bnxt_qplib_free_nq(&rdev->nqr->nq[i]);
}
bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
&rdev->dpi_privileged);
Expand Down Expand Up @@ -1865,6 +1867,21 @@ static int bnxt_re_ib_init(struct bnxt_re_dev *rdev)
return rc;
}

static int bnxt_re_alloc_nqr_mem(struct bnxt_re_dev *rdev)
{
rdev->nqr = kzalloc(sizeof(*rdev->nqr), GFP_KERNEL);
if (!rdev->nqr)
return -ENOMEM;

return 0;
}

static void bnxt_re_free_nqr_mem(struct bnxt_re_dev *rdev)
{
kfree(rdev->nqr);
rdev->nqr = NULL;
}

static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type)
{
u8 type;
Expand Down Expand Up @@ -1894,11 +1911,12 @@ static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type)
bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
}

rdev->num_msix = 0;
rdev->nqr->num_msix = 0;

if (rdev->pacing.dbr_pacing)
bnxt_re_deinitialize_dbr_pacing(rdev);

bnxt_re_free_nqr_mem(rdev);
bnxt_re_destroy_chip_ctx(rdev);
if (op_type == BNXT_RE_COMPLETE_REMOVE) {
if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags))
Expand Down Expand Up @@ -1946,7 +1964,6 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type)
}
ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n",
rdev->en_dev->ulp_tbl->msix_requested);
rdev->num_msix = rdev->en_dev->ulp_tbl->msix_requested;

rc = bnxt_re_setup_chip_ctx(rdev);
if (rc) {
Expand All @@ -1956,6 +1973,15 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type)
return -EINVAL;
}

rc = bnxt_re_alloc_nqr_mem(rdev);
if (rc) {
bnxt_re_destroy_chip_ctx(rdev);
bnxt_unregister_dev(rdev->en_dev);
clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
return rc;
}
rdev->nqr->num_msix = rdev->en_dev->ulp_tbl->msix_requested;

/* Check whether VF or PF */
bnxt_re_get_sriov_func_type(rdev);

Expand Down

0 comments on commit 30b8713

Please sign in to comment.