From d38718a525a3017586daebc3cda22c25cd04d14c Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:14 +0200 Subject: [PATCH 01/15] mlxsw: pci: Move mlxsw_pci_eq_{init, fini}() Move mlxsw_pci_eq_{init, fini}() after mlxsw_pci_eq_tasklet() as a next patch will setup the tasklet as part of initialization. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/7ae120a02e1c490084daae7e684a0d40b7cce4e7.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 72 +++++++++++------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index af99bf17eb36d..efb9472a20697 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -765,42 +765,6 @@ static u8 mlxsw_pci_cq_elem_size(const struct mlxsw_pci_queue *q) MLXSW_PCI_CQE01_SIZE; } -static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, - struct mlxsw_pci_queue *q) -{ - int i; - int err; - - q->consumer_counter = 0; - - for (i = 0; i < q->count; i++) { - char *elem = mlxsw_pci_queue_elem_get(q, i); - - mlxsw_pci_eqe_owner_set(elem, 1); - } - - mlxsw_cmd_mbox_sw2hw_eq_int_msix_set(mbox, 1); /* MSI-X used */ - mlxsw_cmd_mbox_sw2hw_eq_st_set(mbox, 1); /* armed */ - mlxsw_cmd_mbox_sw2hw_eq_log_eq_size_set(mbox, ilog2(q->count)); - for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) { - dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i); - - mlxsw_cmd_mbox_sw2hw_eq_pa_set(mbox, i, mapaddr); - } - err = mlxsw_cmd_sw2hw_eq(mlxsw_pci->core, mbox, q->num); - if (err) - return err; - mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); - mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); - return 0; -} - -static void mlxsw_pci_eq_fini(struct mlxsw_pci *mlxsw_pci, - struct mlxsw_pci_queue *q) -{ - mlxsw_cmd_hw2sw_eq(mlxsw_pci->core, q->num); -} - static void mlxsw_pci_eq_cmd_event(struct mlxsw_pci *mlxsw_pci, char *eqe) { mlxsw_pci->cmd.comp.status = mlxsw_pci_eqe_cmd_status_get(eqe); @@ -877,6 +841,42 @@ static void mlxsw_pci_eq_tasklet(struct tasklet_struct *t) } } +static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, + struct mlxsw_pci_queue *q) +{ + int i; + int err; + + q->consumer_counter = 0; + + for (i = 0; i < q->count; i++) { + char *elem = mlxsw_pci_queue_elem_get(q, i); + + mlxsw_pci_eqe_owner_set(elem, 1); + } + + mlxsw_cmd_mbox_sw2hw_eq_int_msix_set(mbox, 1); /* MSI-X used */ + mlxsw_cmd_mbox_sw2hw_eq_st_set(mbox, 1); /* armed */ + mlxsw_cmd_mbox_sw2hw_eq_log_eq_size_set(mbox, ilog2(q->count)); + for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) { + dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i); + + mlxsw_cmd_mbox_sw2hw_eq_pa_set(mbox, i, mapaddr); + } + err = mlxsw_cmd_sw2hw_eq(mlxsw_pci->core, mbox, q->num); + if (err) + return err; + mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); + mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); + return 0; +} + +static void mlxsw_pci_eq_fini(struct mlxsw_pci *mlxsw_pci, + struct mlxsw_pci_queue *q) +{ + mlxsw_cmd_hw2sw_eq(mlxsw_pci->core, q->num); +} + struct mlxsw_pci_queue_ops { const char *name; enum mlxsw_pci_queue_type type; From f46de9f0e70c9e518a995194aea15ecaedffc0fa Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:15 +0200 Subject: [PATCH 02/15] mlxsw: pci: Move mlxsw_pci_cq_{init, fini}() Move mlxsw_pci_cq_{init, fini}() after mlxsw_pci_cq_tasklet() as a next patch will setup the tasklet as part of initialization. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/25196cb5baf5acf6ec1e956203790e018ba8e306.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 86 +++++++++++------------ 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index efb9472a20697..3fec3909d3c7e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -470,49 +470,6 @@ static void mlxsw_pci_cq_pre_init(struct mlxsw_pci *mlxsw_pci, q->u.cq.v = MLXSW_PCI_CQE_V1; } -static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, - struct mlxsw_pci_queue *q) -{ - int i; - int err; - - q->consumer_counter = 0; - - for (i = 0; i < q->count; i++) { - char *elem = mlxsw_pci_queue_elem_get(q, i); - - mlxsw_pci_cqe_owner_set(q->u.cq.v, elem, 1); - } - - if (q->u.cq.v == MLXSW_PCI_CQE_V1) - mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox, - MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_1); - else if (q->u.cq.v == MLXSW_PCI_CQE_V2) - mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox, - MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_2); - - mlxsw_cmd_mbox_sw2hw_cq_c_eqn_set(mbox, MLXSW_PCI_EQ_COMP_NUM); - mlxsw_cmd_mbox_sw2hw_cq_st_set(mbox, 0); - mlxsw_cmd_mbox_sw2hw_cq_log_cq_size_set(mbox, ilog2(q->count)); - for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) { - dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i); - - mlxsw_cmd_mbox_sw2hw_cq_pa_set(mbox, i, mapaddr); - } - err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num); - if (err) - return err; - mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); - mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); - return 0; -} - -static void mlxsw_pci_cq_fini(struct mlxsw_pci *mlxsw_pci, - struct mlxsw_pci_queue *q) -{ - mlxsw_cmd_hw2sw_cq(mlxsw_pci->core, q->num); -} - static unsigned int mlxsw_pci_read32_off(struct mlxsw_pci *mlxsw_pci, ptrdiff_t off) { @@ -753,6 +710,49 @@ static void mlxsw_pci_cq_tasklet(struct tasklet_struct *t) mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); } +static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, + struct mlxsw_pci_queue *q) +{ + int i; + int err; + + q->consumer_counter = 0; + + for (i = 0; i < q->count; i++) { + char *elem = mlxsw_pci_queue_elem_get(q, i); + + mlxsw_pci_cqe_owner_set(q->u.cq.v, elem, 1); + } + + if (q->u.cq.v == MLXSW_PCI_CQE_V1) + mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox, + MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_1); + else if (q->u.cq.v == MLXSW_PCI_CQE_V2) + mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox, + MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_2); + + mlxsw_cmd_mbox_sw2hw_cq_c_eqn_set(mbox, MLXSW_PCI_EQ_COMP_NUM); + mlxsw_cmd_mbox_sw2hw_cq_st_set(mbox, 0); + mlxsw_cmd_mbox_sw2hw_cq_log_cq_size_set(mbox, ilog2(q->count)); + for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) { + dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i); + + mlxsw_cmd_mbox_sw2hw_cq_pa_set(mbox, i, mapaddr); + } + err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num); + if (err) + return err; + mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); + mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); + return 0; +} + +static void mlxsw_pci_cq_fini(struct mlxsw_pci *mlxsw_pci, + struct mlxsw_pci_queue *q) +{ + mlxsw_cmd_hw2sw_cq(mlxsw_pci->core, q->num); +} + static u16 mlxsw_pci_cq_elem_count(const struct mlxsw_pci_queue *q) { return q->u.cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_COUNT : From fb29028ae7181fbe268d94f6b69d172da4e3640f Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:16 +0200 Subject: [PATCH 03/15] mlxsw: pci: Do not setup tasklet from operation Currently, the structure 'mlxsw_pci_queue_ops' holds a pointer to the callback function of tasklet. This is used only for EQ and CQ. mlxsw driver will use NAPI in a following patch set, so CQ will not use tasklet anymore. As preparation, remove this pointer from the shared operation structure and setup the tasklet as part of queue initialization. For now, setup tasklet for EQ and CQ. Later, CQ code will be changed. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/a326cae5fc1ad085a1a063c004983de6fe389414.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 3fec3909d3c7e..cd871dbfeb028 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -742,6 +742,7 @@ static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num); if (err) return err; + tasklet_setup(&q->tasklet, mlxsw_pci_cq_tasklet); mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); return 0; @@ -866,6 +867,7 @@ static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, err = mlxsw_cmd_sw2hw_eq(mlxsw_pci->core, mbox, q->num); if (err) return err; + tasklet_setup(&q->tasklet, mlxsw_pci_eq_tasklet); mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); return 0; @@ -886,7 +888,6 @@ struct mlxsw_pci_queue_ops { struct mlxsw_pci_queue *q); void (*fini)(struct mlxsw_pci *mlxsw_pci, struct mlxsw_pci_queue *q); - void (*tasklet)(struct tasklet_struct *t); u16 (*elem_count_f)(const struct mlxsw_pci_queue *q); u8 (*elem_size_f)(const struct mlxsw_pci_queue *q); u16 elem_count; @@ -914,7 +915,6 @@ static const struct mlxsw_pci_queue_ops mlxsw_pci_cq_ops = { .pre_init = mlxsw_pci_cq_pre_init, .init = mlxsw_pci_cq_init, .fini = mlxsw_pci_cq_fini, - .tasklet = mlxsw_pci_cq_tasklet, .elem_count_f = mlxsw_pci_cq_elem_count, .elem_size_f = mlxsw_pci_cq_elem_size }; @@ -923,7 +923,6 @@ static const struct mlxsw_pci_queue_ops mlxsw_pci_eq_ops = { .type = MLXSW_PCI_QUEUE_TYPE_EQ, .init = mlxsw_pci_eq_init, .fini = mlxsw_pci_eq_fini, - .tasklet = mlxsw_pci_eq_tasklet, .elem_count = MLXSW_PCI_EQE_COUNT, .elem_size = MLXSW_PCI_EQE_SIZE }; @@ -948,9 +947,6 @@ static int mlxsw_pci_queue_init(struct mlxsw_pci *mlxsw_pci, char *mbox, q->type = q_ops->type; q->pci = mlxsw_pci; - if (q_ops->tasklet) - tasklet_setup(&q->tasklet, q_ops->tasklet); - mem_item->size = MLXSW_PCI_AQ_SIZE; mem_item->buf = dma_alloc_coherent(&mlxsw_pci->pdev->dev, mem_item->size, &mem_item->mapaddr, From 38b124cb4ee5d3567cd138af33bbd6fefbfdb703 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:17 +0200 Subject: [PATCH 04/15] mlxsw: pci: Arm CQ doorbell regardless of number of completions Currently, as part of mlxsw_pci_cq_tasklet(), we check if any item was handled, and only in such case we arm doorbell. This is unlikely case, as we schedule tasklet only for CQs that we get an event for them, which means that they contain completions to handle. Remove this check, which is supposed to be true always, and even if it is false, it is not a mistake to ring the doorbell. We can warn on such case, but it is not really worth to add a check which will be run for each CQ handling when we do not expect to reach it and it does not point to logic error that should be handled. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/f8efa481bfe7bebb9f93bb803f44ab7da77f53e6.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index cd871dbfeb028..7994451111445 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -706,8 +706,8 @@ static void mlxsw_pci_cq_tasklet(struct tasklet_struct *t) if (++items == credits) break; } - if (items) - mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); + + mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); } static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, From 57beea8e5667241638c62568ce9a2cb7391cfd4a Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:18 +0200 Subject: [PATCH 05/15] mlxsw: pci: Remove unused counters The structure 'mlxsw_pci_queue' stores several counters which were consumed via debugfs. Since commit 9a32562becd9 ("mlxsw: Remove debugfs interface"), these counters are not used. Remove them. This makes the 'union u' and 'struct eq' redundant. Maintain 'struct cq' as it will be extended later. Replace increasing 'q->u.eq.ev_other_count' with WARN_ON_ONCE(), as it is used in an unreasonable case of receiving event in EQ which is not EQ0 or EQ1. When the queues are initialized, we check number of event queues and fail with the print "Unsupported number of queues" in case that the driver tries to initialize more than two queues. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/ee9e658800aa0390e08342100bc27daff4c176c0.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 49 +++++++++-------------- 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 7994451111445..ec18fe0c21172 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -80,18 +80,9 @@ struct mlxsw_pci_queue { enum mlxsw_pci_queue_type type; struct tasklet_struct tasklet; /* queue processing tasklet */ struct mlxsw_pci *pci; - union { - struct { - u32 comp_sdq_count; - u32 comp_rdq_count; - enum mlxsw_pci_cqe_v v; - } cq; - struct { - u32 ev_cmd_count; - u32 ev_comp_count; - u32 ev_other_count; - } eq; - } u; + struct { + enum mlxsw_pci_cqe_v v; + } cq; }; struct mlxsw_pci_queue_type_group { @@ -462,12 +453,12 @@ static void mlxsw_pci_rdq_fini(struct mlxsw_pci *mlxsw_pci, static void mlxsw_pci_cq_pre_init(struct mlxsw_pci *mlxsw_pci, struct mlxsw_pci_queue *q) { - q->u.cq.v = mlxsw_pci->max_cqe_ver; + q->cq.v = mlxsw_pci->max_cqe_ver; - if (q->u.cq.v == MLXSW_PCI_CQE_V2 && + if (q->cq.v == MLXSW_PCI_CQE_V2 && q->num < mlxsw_pci->num_sdq_cqs && !mlxsw_core_sdq_supports_cqe_v2(mlxsw_pci->core)) - q->u.cq.v = MLXSW_PCI_CQE_V1; + q->cq.v = MLXSW_PCI_CQE_V1; } static unsigned int mlxsw_pci_read32_off(struct mlxsw_pci *mlxsw_pci, @@ -663,7 +654,7 @@ static char *mlxsw_pci_cq_sw_cqe_get(struct mlxsw_pci_queue *q) elem_info = mlxsw_pci_queue_elem_info_consumer_get(q); elem = elem_info->elem; - owner_bit = mlxsw_pci_cqe_owner_get(q->u.cq.v, elem); + owner_bit = mlxsw_pci_cqe_owner_get(q->cq.v, elem); if (mlxsw_pci_elem_hw_owned(q, owner_bit)) return NULL; q->consumer_counter++; @@ -681,8 +672,8 @@ static void mlxsw_pci_cq_tasklet(struct tasklet_struct *t) while ((cqe = mlxsw_pci_cq_sw_cqe_get(q))) { u16 wqe_counter = mlxsw_pci_cqe_wqe_counter_get(cqe); - u8 sendq = mlxsw_pci_cqe_sr_get(q->u.cq.v, cqe); - u8 dqn = mlxsw_pci_cqe_dqn_get(q->u.cq.v, cqe); + u8 sendq = mlxsw_pci_cqe_sr_get(q->cq.v, cqe); + u8 dqn = mlxsw_pci_cqe_dqn_get(q->cq.v, cqe); char ncqe[MLXSW_PCI_CQE_SIZE_MAX]; memcpy(ncqe, cqe, q->elem_size); @@ -693,15 +684,13 @@ static void mlxsw_pci_cq_tasklet(struct tasklet_struct *t) sdq = mlxsw_pci_sdq_get(mlxsw_pci, dqn); mlxsw_pci_cqe_sdq_handle(mlxsw_pci, sdq, - wqe_counter, q->u.cq.v, ncqe); - q->u.cq.comp_sdq_count++; + wqe_counter, q->cq.v, ncqe); } else { struct mlxsw_pci_queue *rdq; rdq = mlxsw_pci_rdq_get(mlxsw_pci, dqn); mlxsw_pci_cqe_rdq_handle(mlxsw_pci, rdq, - wqe_counter, q->u.cq.v, ncqe); - q->u.cq.comp_rdq_count++; + wqe_counter, q->cq.v, ncqe); } if (++items == credits) break; @@ -721,13 +710,13 @@ static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, for (i = 0; i < q->count; i++) { char *elem = mlxsw_pci_queue_elem_get(q, i); - mlxsw_pci_cqe_owner_set(q->u.cq.v, elem, 1); + mlxsw_pci_cqe_owner_set(q->cq.v, elem, 1); } - if (q->u.cq.v == MLXSW_PCI_CQE_V1) + if (q->cq.v == MLXSW_PCI_CQE_V1) mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox, MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_1); - else if (q->u.cq.v == MLXSW_PCI_CQE_V2) + else if (q->cq.v == MLXSW_PCI_CQE_V2) mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox, MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_2); @@ -756,13 +745,13 @@ static void mlxsw_pci_cq_fini(struct mlxsw_pci *mlxsw_pci, static u16 mlxsw_pci_cq_elem_count(const struct mlxsw_pci_queue *q) { - return q->u.cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_COUNT : - MLXSW_PCI_CQE01_COUNT; + return q->cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_COUNT : + MLXSW_PCI_CQE01_COUNT; } static u8 mlxsw_pci_cq_elem_size(const struct mlxsw_pci_queue *q) { - return q->u.cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_SIZE : + return q->cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_SIZE : MLXSW_PCI_CQE01_SIZE; } @@ -815,16 +804,14 @@ static void mlxsw_pci_eq_tasklet(struct tasklet_struct *t) switch (q->num) { case MLXSW_PCI_EQ_ASYNC_NUM: mlxsw_pci_eq_cmd_event(mlxsw_pci, eqe); - q->u.eq.ev_cmd_count++; break; case MLXSW_PCI_EQ_COMP_NUM: cqn = mlxsw_pci_eqe_cqn_get(eqe); set_bit(cqn, active_cqns); cq_handle = true; - q->u.eq.ev_comp_count++; break; default: - q->u.eq.ev_other_count++; + WARN_ON_ONCE(1); } if (++items == credits) break; From 29ad2a990648c5e6ec93101cb3719dc820d363e9 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:19 +0200 Subject: [PATCH 06/15] mlxsw: pci: Make style changes in mlxsw_pci_eq_tasklet() This function will be used later only for EQ1. As preparation, reorder variables to reverse xmas tree and return earlier when it is possible, to simplify the code. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/2412d6c135b2a6aedb4484f5d8baab3aecd7b9ae.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index ec18fe0c21172..5688c14f7426b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -783,15 +783,14 @@ static char *mlxsw_pci_eq_sw_eqe_get(struct mlxsw_pci_queue *q) static void mlxsw_pci_eq_tasklet(struct tasklet_struct *t) { + unsigned long active_cqns[BITS_TO_LONGS(MLXSW_PCI_CQS_MAX)]; struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet); struct mlxsw_pci *mlxsw_pci = q->pci; - u8 cq_count = mlxsw_pci_cq_count(mlxsw_pci); - unsigned long active_cqns[BITS_TO_LONGS(MLXSW_PCI_CQS_MAX)]; - char *eqe; - u8 cqn; + int credits = q->count >> 1; bool cq_handle = false; + u8 cqn, cq_count; int items = 0; - int credits = q->count >> 1; + char *eqe; memset(&active_cqns, 0, sizeof(active_cqns)); @@ -816,13 +815,17 @@ static void mlxsw_pci_eq_tasklet(struct tasklet_struct *t) if (++items == credits) break; } - if (items) { - mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); - mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); - } + + if (!items) + return; + + mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); + mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); if (!cq_handle) return; + + cq_count = mlxsw_pci_cq_count(mlxsw_pci); for_each_set_bit(cqn, active_cqns, cq_count) { q = mlxsw_pci_cq_get(mlxsw_pci, cqn); mlxsw_pci_queue_tasklet_schedule(q); From d4b3930b19f7339c3fb98fc3ed71388b0180e7a7 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:20 +0200 Subject: [PATCH 07/15] mlxsw: pci: Poll command interface for each cmd_exec() Command interface is used for configuring and querying FW when EMADs are not available. During the time that the driver sets up the asynchronous queues, it polls the command interface for getting completions. Then, there is a short period when asynchronous queues work, but EMADs are not available (marked in the code as nopoll = true). During this time, we send commands via command interface, but we do not poll it, as we can get an interrupt for the completion. Completions of command interface are received from HW in EQ0 (event queue 0). The usage of EQ0 instead of polling is done only 4 times during initialization and one time during tear down, but it makes an overhead during lifetime of the driver. For each interrupt, we have to check if we get events in EQ0 or EQ1 and handle them. This is really ineffective, especially because of the fact that EQ0 is used only as part of driver init/fini. Instead, we can poll command interface for each call of cmd_exec(). It means that when we send a command via command interface (as EMADs are not available), we will poll it, regardless of availability of the asynchronous queues. This will allow us to configure later only EQ1 and simplify the flow. Remove 'nopoll' indication and change mlxsw_pci_cmd_exec() to poll till answer/timeout regardless of queues' state. For now, completions are handled also by EQ0, but it will be removed in next patch. Additional cleanups will be added in next patches. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/e674c70380ceda953e0e45a77334c5d22e69938f.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 48 ++++++++--------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 5688c14f7426b..2ee397922936b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -111,7 +111,6 @@ struct mlxsw_pci { struct mlxsw_pci_mem_item out_mbox; struct mlxsw_pci_mem_item in_mbox; struct mutex lock; /* Lock access to command registers */ - bool nopoll; wait_queue_head_t wait; bool wait_done; struct { @@ -1105,8 +1104,6 @@ static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox) goto err_rdqs_init; } - /* We have to poll in command interface until queues are initialized */ - mlxsw_pci->cmd.nopoll = true; return 0; err_rdqs_init: @@ -1120,7 +1117,6 @@ static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox) static void mlxsw_pci_aqs_fini(struct mlxsw_pci *mlxsw_pci) { - mlxsw_pci->cmd.nopoll = false; mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_rdq_ops); mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_sdq_ops); mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_cq_ops); @@ -1848,9 +1844,9 @@ static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod, { struct mlxsw_pci *mlxsw_pci = bus_priv; dma_addr_t in_mapaddr = 0, out_mapaddr = 0; - bool evreq = mlxsw_pci->cmd.nopoll; unsigned long timeout = msecs_to_jiffies(MLXSW_PCI_CIR_TIMEOUT_MSECS); bool *p_wait_done = &mlxsw_pci->cmd.wait_done; + unsigned long end; int err; *p_status = MLXSW_CMD_STATUS_OK; @@ -1879,28 +1875,20 @@ static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod, wmb(); /* all needs to be written before we write control register */ mlxsw_pci_write32(mlxsw_pci, CIR_CTRL, MLXSW_PCI_CIR_CTRL_GO_BIT | - (evreq ? MLXSW_PCI_CIR_CTRL_EVREQ_BIT : 0) | (opcode_mod << MLXSW_PCI_CIR_CTRL_OPCODE_MOD_SHIFT) | opcode); - if (!evreq) { - unsigned long end; - - end = jiffies + timeout; - do { - u32 ctrl = mlxsw_pci_read32(mlxsw_pci, CIR_CTRL); + end = jiffies + timeout; + do { + u32 ctrl = mlxsw_pci_read32(mlxsw_pci, CIR_CTRL); - if (!(ctrl & MLXSW_PCI_CIR_CTRL_GO_BIT)) { - *p_wait_done = true; - *p_status = ctrl >> MLXSW_PCI_CIR_CTRL_STATUS_SHIFT; - break; - } - cond_resched(); - } while (time_before(jiffies, end)); - } else { - wait_event_timeout(mlxsw_pci->cmd.wait, *p_wait_done, timeout); - *p_status = mlxsw_pci->cmd.comp.status; - } + if (!(ctrl & MLXSW_PCI_CIR_CTRL_GO_BIT)) { + *p_wait_done = true; + *p_status = ctrl >> MLXSW_PCI_CIR_CTRL_STATUS_SHIFT; + break; + } + cond_resched(); + } while (time_before(jiffies, end)); err = 0; if (*p_wait_done) { @@ -1917,14 +1905,12 @@ static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod, */ __be32 tmp; - if (!evreq) { - tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci, - CIR_OUT_PARAM_HI)); - memcpy(out_mbox, &tmp, sizeof(tmp)); - tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci, - CIR_OUT_PARAM_LO)); - memcpy(out_mbox + sizeof(tmp), &tmp, sizeof(tmp)); - } + tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci, + CIR_OUT_PARAM_HI)); + memcpy(out_mbox, &tmp, sizeof(tmp)); + tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci, + CIR_OUT_PARAM_LO)); + memcpy(out_mbox + sizeof(tmp), &tmp, sizeof(tmp)); } else if (!err && out_mbox) { memcpy(out_mbox, mlxsw_pci->cmd.out_mbox.buf, out_mbox_size); } From 7bc6a3098c3821b50e9ef54e22a1a9d701f6fe00 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:21 +0200 Subject: [PATCH 08/15] mlxsw: pci: Rename MLXSW_PCI_EQS_COUNT Currently we use MLXSW_PCI_EQS_COUNT event queues. A next patch will change the driver to initialize only EQ1, as EQ0 is not required anymore when we poll command interface. Rename the macro to MLXSW_PCI_EQS_MAX as later we will not initialize the maximum supported EQs, this value represents the maximum and a new macro will be added to represent the actual used queues. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/b08df430b62f23ca1aa3aaa257896d2d95aa7691.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 4 ++-- drivers/net/ethernet/mellanox/mlxsw/pci_hw.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 2ee397922936b..4ee8d71f0697e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -1059,7 +1059,7 @@ static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox) if (num_sdqs + num_rdqs > num_cqs || num_sdqs < MLXSW_PCI_SDQS_MIN || - num_cqs > MLXSW_PCI_CQS_MAX || num_eqs != MLXSW_PCI_EQS_COUNT) { + num_cqs > MLXSW_PCI_CQS_MAX || num_eqs != MLXSW_PCI_EQS_MAX) { dev_err(&pdev->dev, "Unsupported number of queues\n"); return -EINVAL; } @@ -1416,7 +1416,7 @@ static irqreturn_t mlxsw_pci_eq_irq_handler(int irq, void *dev_id) struct mlxsw_pci_queue *q; int i; - for (i = 0; i < MLXSW_PCI_EQS_COUNT; i++) { + for (i = 0; i < MLXSW_PCI_EQS_MAX; i++) { q = mlxsw_pci_eq_get(mlxsw_pci, i); mlxsw_pci_queue_tasklet_schedule(q); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h index 7cdf0ce24f288..32a4f436d24da 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h +++ b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h @@ -42,7 +42,7 @@ ((offset) + (type_offset) + (num) * 4) #define MLXSW_PCI_CQS_MAX 96 -#define MLXSW_PCI_EQS_COUNT 2 +#define MLXSW_PCI_EQS_MAX 2 #define MLXSW_PCI_EQ_ASYNC_NUM 0 #define MLXSW_PCI_EQ_COMP_NUM 1 From 6fc280a365151d09c748eba1a90ea12e4615317e Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:22 +0200 Subject: [PATCH 09/15] mlxsw: pci: Use only one event queue The device supports two event queues. EQ0 is used for command interface completion events. EQ1 is used for completion events of RDQ or SDQ. Currently, for each EQE (event queue element), we check the queue number and handle accordingly. More than that, for each interrupt we schedule tasklets for both EQs. This is really ineffective, especially because of the fact that EQ0 is used only as part of driver init/fini, when EMADs are not available. There is no point to schedule the tasklet for it and check each EQE. A previous patch changed the code to poll command interface for each use of it. It means that now there is no real reason to use EQ0, as we poll the command interface. Initialize only one event queue and use it as EQ1 (this is determined by queue number). Then, for each interrupt we can schedule the tasklet only for one queue and we do not have to check the queue number. This simplifies the code and should improve performance. Note that polling command interface is ok as we use it only as part of driver init/fini. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/23d764f5c032e4c363b98590b746a4b32d2bf900.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 54 ++++++-------------- drivers/net/ethernet/mellanox/mlxsw/pci_hw.h | 2 +- 2 files changed, 16 insertions(+), 40 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 4ee8d71f0697e..6b8596f2feb9a 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -223,10 +223,10 @@ static struct mlxsw_pci_queue *mlxsw_pci_cq_get(struct mlxsw_pci *mlxsw_pci, return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ, q_num); } -static struct mlxsw_pci_queue *mlxsw_pci_eq_get(struct mlxsw_pci *mlxsw_pci, - u8 q_num) +static struct mlxsw_pci_queue *mlxsw_pci_eq_get(struct mlxsw_pci *mlxsw_pci) { - return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_EQ, q_num); + /* There is only one EQ at index 0. */ + return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_EQ, 0); } static void __mlxsw_pci_queue_doorbell_set(struct mlxsw_pci *mlxsw_pci, @@ -754,16 +754,6 @@ static u8 mlxsw_pci_cq_elem_size(const struct mlxsw_pci_queue *q) MLXSW_PCI_CQE01_SIZE; } -static void mlxsw_pci_eq_cmd_event(struct mlxsw_pci *mlxsw_pci, char *eqe) -{ - mlxsw_pci->cmd.comp.status = mlxsw_pci_eqe_cmd_status_get(eqe); - mlxsw_pci->cmd.comp.out_param = - ((u64) mlxsw_pci_eqe_cmd_out_param_h_get(eqe)) << 32 | - mlxsw_pci_eqe_cmd_out_param_l_get(eqe); - mlxsw_pci->cmd.wait_done = true; - wake_up(&mlxsw_pci->cmd.wait); -} - static char *mlxsw_pci_eq_sw_eqe_get(struct mlxsw_pci_queue *q) { struct mlxsw_pci_queue_elem_info *elem_info; @@ -786,7 +776,6 @@ static void mlxsw_pci_eq_tasklet(struct tasklet_struct *t) struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet); struct mlxsw_pci *mlxsw_pci = q->pci; int credits = q->count >> 1; - bool cq_handle = false; u8 cqn, cq_count; int items = 0; char *eqe; @@ -794,23 +783,9 @@ static void mlxsw_pci_eq_tasklet(struct tasklet_struct *t) memset(&active_cqns, 0, sizeof(active_cqns)); while ((eqe = mlxsw_pci_eq_sw_eqe_get(q))) { + cqn = mlxsw_pci_eqe_cqn_get(eqe); + set_bit(cqn, active_cqns); - /* Command interface completion events are always received on - * queue MLXSW_PCI_EQ_ASYNC_NUM (EQ0) and completion events - * are mapped to queue MLXSW_PCI_EQ_COMP_NUM (EQ1). - */ - switch (q->num) { - case MLXSW_PCI_EQ_ASYNC_NUM: - mlxsw_pci_eq_cmd_event(mlxsw_pci, eqe); - break; - case MLXSW_PCI_EQ_COMP_NUM: - cqn = mlxsw_pci_eqe_cqn_get(eqe); - set_bit(cqn, active_cqns); - cq_handle = true; - break; - default: - WARN_ON_ONCE(1); - } if (++items == credits) break; } @@ -821,9 +796,6 @@ static void mlxsw_pci_eq_tasklet(struct tasklet_struct *t) mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); - if (!cq_handle) - return; - cq_count = mlxsw_pci_cq_count(mlxsw_pci); for_each_set_bit(cqn, active_cqns, cq_count) { q = mlxsw_pci_cq_get(mlxsw_pci, cqn); @@ -837,6 +809,13 @@ static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, int i; int err; + /* We expect to initialize only one EQ, which gets num=0 as it is + * located at index zero. We use the EQ as EQ1, so set the number for + * future use. + */ + WARN_ON_ONCE(q->num); + q->num = MLXSW_PCI_EQ_COMP_NUM; + q->consumer_counter = 0; for (i = 0; i < q->count; i++) { @@ -1077,7 +1056,7 @@ static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox) mlxsw_pci->num_sdq_cqs = num_sdqs; err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_eq_ops, - num_eqs); + MLXSW_PCI_EQS_COUNT); if (err) { dev_err(&pdev->dev, "Failed to initialize event queues\n"); return err; @@ -1414,12 +1393,9 @@ static irqreturn_t mlxsw_pci_eq_irq_handler(int irq, void *dev_id) { struct mlxsw_pci *mlxsw_pci = dev_id; struct mlxsw_pci_queue *q; - int i; - for (i = 0; i < MLXSW_PCI_EQS_MAX; i++) { - q = mlxsw_pci_eq_get(mlxsw_pci, i); - mlxsw_pci_queue_tasklet_schedule(q); - } + q = mlxsw_pci_eq_get(mlxsw_pci); + mlxsw_pci_queue_tasklet_schedule(q); return IRQ_HANDLED; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h index 32a4f436d24da..6bed495dcf0ff 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h +++ b/drivers/net/ethernet/mellanox/mlxsw/pci_hw.h @@ -43,7 +43,7 @@ #define MLXSW_PCI_CQS_MAX 96 #define MLXSW_PCI_EQS_MAX 2 -#define MLXSW_PCI_EQ_ASYNC_NUM 0 +#define MLXSW_PCI_EQS_COUNT 1 #define MLXSW_PCI_EQ_COMP_NUM 1 #define MLXSW_PCI_SDQS_MIN 2 /* EMAD and control traffic */ From 2c200863fcc78ec1797b0a7a36303e9466b0617d Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:23 +0200 Subject: [PATCH 10/15] mlxsw: pci: Remove unused wait queue The previous patch changed the code to do not handle command interface from event queue. With this change the wait queue is not used anymore. Remove it and 'wait_done' variable. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/f3af6a5a9dabd97d2920cefe475c6aa57767f504.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 6b8596f2feb9a..6ca782f37c2fd 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -111,8 +110,6 @@ struct mlxsw_pci { struct mlxsw_pci_mem_item out_mbox; struct mlxsw_pci_mem_item in_mbox; struct mutex lock; /* Lock access to command registers */ - wait_queue_head_t wait; - bool wait_done; struct { u8 status; u64 out_param; @@ -1821,8 +1818,8 @@ static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod, struct mlxsw_pci *mlxsw_pci = bus_priv; dma_addr_t in_mapaddr = 0, out_mapaddr = 0; unsigned long timeout = msecs_to_jiffies(MLXSW_PCI_CIR_TIMEOUT_MSECS); - bool *p_wait_done = &mlxsw_pci->cmd.wait_done; unsigned long end; + bool wait_done; int err; *p_status = MLXSW_CMD_STATUS_OK; @@ -1846,7 +1843,7 @@ static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod, mlxsw_pci_write32(mlxsw_pci, CIR_IN_MODIFIER, in_mod); mlxsw_pci_write32(mlxsw_pci, CIR_TOKEN, 0); - *p_wait_done = false; + wait_done = false; wmb(); /* all needs to be written before we write control register */ mlxsw_pci_write32(mlxsw_pci, CIR_CTRL, @@ -1859,7 +1856,7 @@ static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod, u32 ctrl = mlxsw_pci_read32(mlxsw_pci, CIR_CTRL); if (!(ctrl & MLXSW_PCI_CIR_CTRL_GO_BIT)) { - *p_wait_done = true; + wait_done = true; *p_status = ctrl >> MLXSW_PCI_CIR_CTRL_STATUS_SHIFT; break; } @@ -1867,7 +1864,7 @@ static int mlxsw_pci_cmd_exec(void *bus_priv, u16 opcode, u8 opcode_mod, } while (time_before(jiffies, end)); err = 0; - if (*p_wait_done) { + if (wait_done) { if (*p_status) err = -EIO; } else { @@ -1965,7 +1962,6 @@ static int mlxsw_pci_cmd_init(struct mlxsw_pci *mlxsw_pci) int err; mutex_init(&mlxsw_pci->cmd.lock); - init_waitqueue_head(&mlxsw_pci->cmd.wait); err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.in_mbox); if (err) From a0639236d42060c8f0e994ddf87fbb0f3cb2b048 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:24 +0200 Subject: [PATCH 11/15] mlxsw: pci: Make style change in mlxsw_pci_cq_tasklet() This function will be broken into several functions later. As preparation, reorder variables to reverse xmas tree. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/7170a8f4429ecb5a539b0374c621697778ff8363.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 6ca782f37c2fd..84bad681021e1 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -662,9 +662,9 @@ static void mlxsw_pci_cq_tasklet(struct tasklet_struct *t) { struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet); struct mlxsw_pci *mlxsw_pci = q->pci; - char *cqe; - int items = 0; int credits = q->count >> 1; + int items = 0; + char *cqe; while ((cqe = mlxsw_pci_cq_sw_cqe_get(q))) { u16 wqe_counter = mlxsw_pci_cqe_wqe_counter_get(cqe); From 1df7d871e3491de2f53afafedef8383185a3f613 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:25 +0200 Subject: [PATCH 12/15] mlxsw: pci: Break mlxsw_pci_cq_tasklet() into tasklets per queue type Completion queues are used for completions of RDQ or SDQ. Each completion queue is used for one DQ. The first CQs are used for SDQs and the rest are used for RDQs. Currently, for each CQE (completion queue element), we check 'sr' value (send/receive) to know if it is completion of RDQ or SDQ. Actually, we do not really have to check it, as according to the queue number we know if it handles completions of Rx or Tx. Break the tasklet into two - one for Rx (RDQ) and one for Tx (SDQ). Then, setup the appropriate tasklet for each queue as part of queue initialization. Use 'sr' value for unlikely case that we get completion with type that we do not expect. Call WARN_ON_ONCE() only after checking the value, to avoid calling this method for each completion. A next patch set will use NAPI to handle events, then we will have a separate poll method for Rx and Tx. This change is a preparation for NAPI usage. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/50fbc366f8de54cb5dc72a7c4f394333ef71f1d0.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 86 +++++++++++++++++++---- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 84bad681021e1..c10e6f22f8184 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -35,6 +35,11 @@ enum mlxsw_pci_queue_type { #define MLXSW_PCI_QUEUE_TYPE_COUNT 4 +enum mlxsw_pci_cq_type { + MLXSW_PCI_CQ_SDQ, + MLXSW_PCI_CQ_RDQ, +}; + static const u16 mlxsw_pci_doorbell_type_offset[] = { MLXSW_PCI_DOORBELL_SDQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_SDQ */ MLXSW_PCI_DOORBELL_RDQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_RDQ */ @@ -658,7 +663,7 @@ static char *mlxsw_pci_cq_sw_cqe_get(struct mlxsw_pci_queue *q) return elem; } -static void mlxsw_pci_cq_tasklet(struct tasklet_struct *t) +static void mlxsw_pci_cq_rx_tasklet(struct tasklet_struct *t) { struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet); struct mlxsw_pci *mlxsw_pci = q->pci; @@ -671,23 +676,54 @@ static void mlxsw_pci_cq_tasklet(struct tasklet_struct *t) u8 sendq = mlxsw_pci_cqe_sr_get(q->cq.v, cqe); u8 dqn = mlxsw_pci_cqe_dqn_get(q->cq.v, cqe); char ncqe[MLXSW_PCI_CQE_SIZE_MAX]; + struct mlxsw_pci_queue *rdq; + + if (unlikely(sendq)) { + WARN_ON_ONCE(1); + continue; + } memcpy(ncqe, cqe, q->elem_size); mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); - if (sendq) { - struct mlxsw_pci_queue *sdq; + rdq = mlxsw_pci_rdq_get(mlxsw_pci, dqn); + mlxsw_pci_cqe_rdq_handle(mlxsw_pci, rdq, + wqe_counter, q->cq.v, ncqe); + + if (++items == credits) + break; + } - sdq = mlxsw_pci_sdq_get(mlxsw_pci, dqn); - mlxsw_pci_cqe_sdq_handle(mlxsw_pci, sdq, - wqe_counter, q->cq.v, ncqe); - } else { - struct mlxsw_pci_queue *rdq; + mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); +} - rdq = mlxsw_pci_rdq_get(mlxsw_pci, dqn); - mlxsw_pci_cqe_rdq_handle(mlxsw_pci, rdq, - wqe_counter, q->cq.v, ncqe); +static void mlxsw_pci_cq_tx_tasklet(struct tasklet_struct *t) +{ + struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet); + struct mlxsw_pci *mlxsw_pci = q->pci; + int credits = q->count >> 1; + int items = 0; + char *cqe; + + while ((cqe = mlxsw_pci_cq_sw_cqe_get(q))) { + u16 wqe_counter = mlxsw_pci_cqe_wqe_counter_get(cqe); + u8 sendq = mlxsw_pci_cqe_sr_get(q->cq.v, cqe); + u8 dqn = mlxsw_pci_cqe_dqn_get(q->cq.v, cqe); + char ncqe[MLXSW_PCI_CQE_SIZE_MAX]; + struct mlxsw_pci_queue *sdq; + + if (unlikely(!sendq)) { + WARN_ON_ONCE(1); + continue; } + + memcpy(ncqe, cqe, q->elem_size); + mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); + + sdq = mlxsw_pci_sdq_get(mlxsw_pci, dqn); + mlxsw_pci_cqe_sdq_handle(mlxsw_pci, sdq, + wqe_counter, q->cq.v, ncqe); + if (++items == credits) break; } @@ -695,6 +731,32 @@ static void mlxsw_pci_cq_tasklet(struct tasklet_struct *t) mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); } +static enum mlxsw_pci_cq_type +mlxsw_pci_cq_type(const struct mlxsw_pci *mlxsw_pci, + const struct mlxsw_pci_queue *q) +{ + /* Each CQ is mapped to one DQ. The first 'num_sdq_cqs' queues are used + * for SDQs and the rest are used for RDQs. + */ + if (q->num < mlxsw_pci->num_sdq_cqs) + return MLXSW_PCI_CQ_SDQ; + + return MLXSW_PCI_CQ_RDQ; +} + +static void mlxsw_pci_cq_tasklet_setup(struct mlxsw_pci_queue *q, + enum mlxsw_pci_cq_type cq_type) +{ + switch (cq_type) { + case MLXSW_PCI_CQ_SDQ: + tasklet_setup(&q->tasklet, mlxsw_pci_cq_tx_tasklet); + break; + case MLXSW_PCI_CQ_RDQ: + tasklet_setup(&q->tasklet, mlxsw_pci_cq_rx_tasklet); + break; + } +} + static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, struct mlxsw_pci_queue *q) { @@ -727,7 +789,7 @@ static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num); if (err) return err; - tasklet_setup(&q->tasklet, mlxsw_pci_cq_tasklet); + mlxsw_pci_cq_tasklet_setup(q, mlxsw_pci_cq_type(mlxsw_pci, q)); mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); return 0; From 0cd1453b7e55064d06b49eebe34ffb43748ba12e Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:26 +0200 Subject: [PATCH 13/15] mlxsw: pci: Remove mlxsw_pci_sdq_count() The number of SDQs is stored as part of 'mlxsw_pci' structure. In some cases, the driver uses this value and in some cases it calls mlxsw_pci_sdq_count() to get the value. Align the code to use the stored value. This simplifies the code and makes it clearer that the value is always the same. Rename 'mlxsw_pci->num_sdq_cqs' to 'mlxsw_pci->num_sdqs' as now it is used not only in CQ context. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/0c8788506d9af35d589dbf64be35a508fd63d681.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index c10e6f22f8184..592e93a530d05 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -123,7 +123,7 @@ struct mlxsw_pci { struct mlxsw_bus_info bus_info; const struct pci_device_id *id; enum mlxsw_pci_cqe_v max_cqe_ver; /* Maximal supported CQE version */ - u8 num_sdq_cqs; /* Number of CQs used for SDQs */ + u8 num_sdqs; /* Number of SDQs */ bool skip_reset; }; @@ -188,11 +188,6 @@ static u8 __mlxsw_pci_queue_count(struct mlxsw_pci *mlxsw_pci, return queue_group->count; } -static u8 mlxsw_pci_sdq_count(struct mlxsw_pci *mlxsw_pci) -{ - return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_SDQ); -} - static u8 mlxsw_pci_cq_count(struct mlxsw_pci *mlxsw_pci) { return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ); @@ -391,7 +386,7 @@ static int mlxsw_pci_rdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, struct mlxsw_pci_queue *q) { struct mlxsw_pci_queue_elem_info *elem_info; - u8 sdq_count = mlxsw_pci_sdq_count(mlxsw_pci); + u8 sdq_count = mlxsw_pci->num_sdqs; int i; int err; @@ -457,7 +452,7 @@ static void mlxsw_pci_cq_pre_init(struct mlxsw_pci *mlxsw_pci, q->cq.v = mlxsw_pci->max_cqe_ver; if (q->cq.v == MLXSW_PCI_CQE_V2 && - q->num < mlxsw_pci->num_sdq_cqs && + q->num < mlxsw_pci->num_sdqs && !mlxsw_core_sdq_supports_cqe_v2(mlxsw_pci->core)) q->cq.v = MLXSW_PCI_CQE_V1; } @@ -735,10 +730,10 @@ static enum mlxsw_pci_cq_type mlxsw_pci_cq_type(const struct mlxsw_pci *mlxsw_pci, const struct mlxsw_pci_queue *q) { - /* Each CQ is mapped to one DQ. The first 'num_sdq_cqs' queues are used + /* Each CQ is mapped to one DQ. The first 'num_sdqs' queues are used * for SDQs and the rest are used for RDQs. */ - if (q->num < mlxsw_pci->num_sdq_cqs) + if (q->num < mlxsw_pci->num_sdqs) return MLXSW_PCI_CQ_SDQ; return MLXSW_PCI_CQ_RDQ; @@ -1112,7 +1107,7 @@ static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox) return -EINVAL; } - mlxsw_pci->num_sdq_cqs = num_sdqs; + mlxsw_pci->num_sdqs = num_sdqs; err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_eq_ops, MLXSW_PCI_EQS_COUNT); @@ -1780,7 +1775,7 @@ static struct mlxsw_pci_queue * mlxsw_pci_sdq_pick(struct mlxsw_pci *mlxsw_pci, const struct mlxsw_tx_info *tx_info) { - u8 ctl_sdq_count = mlxsw_pci_sdq_count(mlxsw_pci) - 1; + u8 ctl_sdq_count = mlxsw_pci->num_sdqs - 1; u8 sdqn; if (tx_info->is_emad) { From 82238f0ddb4628af28e021e2ee07c7b7d97f2eb2 Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:27 +0200 Subject: [PATCH 14/15] mlxsw: pci: Remove mlxsw_pci_cq_count() Currently, for each interrupt we call mlxsw_pci_cq_count() to determine the number of CQs. This call makes additional two function's calls. This can be removed by storing this value as part of structure 'mlxsw_pci', as we already do for number of SDQs. Remove the function and __mlxsw_pci_queue_count() which is now not used and store the value instead. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/f08ad113e8160678f3c8d401382a696c6c7f44c7.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 592e93a530d05..094b9d752a02e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -123,6 +123,7 @@ struct mlxsw_pci { struct mlxsw_bus_info bus_info; const struct pci_device_id *id; enum mlxsw_pci_cqe_v max_cqe_ver; /* Maximal supported CQE version */ + u8 num_cqs; /* Number of CQs */ u8 num_sdqs; /* Number of SDQs */ bool skip_reset; }; @@ -179,20 +180,6 @@ mlxsw_pci_queue_type_group_get(struct mlxsw_pci *mlxsw_pci, return &mlxsw_pci->queues[q_type]; } -static u8 __mlxsw_pci_queue_count(struct mlxsw_pci *mlxsw_pci, - enum mlxsw_pci_queue_type q_type) -{ - struct mlxsw_pci_queue_type_group *queue_group; - - queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_type); - return queue_group->count; -} - -static u8 mlxsw_pci_cq_count(struct mlxsw_pci *mlxsw_pci) -{ - return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ); -} - static struct mlxsw_pci_queue * __mlxsw_pci_queue_get(struct mlxsw_pci *mlxsw_pci, enum mlxsw_pci_queue_type q_type, u8 q_num) @@ -850,7 +837,7 @@ static void mlxsw_pci_eq_tasklet(struct tasklet_struct *t) mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); - cq_count = mlxsw_pci_cq_count(mlxsw_pci); + cq_count = mlxsw_pci->num_cqs; for_each_set_bit(cqn, active_cqns, cq_count) { q = mlxsw_pci_cq_get(mlxsw_pci, cqn); mlxsw_pci_queue_tasklet_schedule(q); @@ -1107,6 +1094,7 @@ static int mlxsw_pci_aqs_init(struct mlxsw_pci *mlxsw_pci, char *mbox) return -EINVAL; } + mlxsw_pci->num_cqs = num_cqs; mlxsw_pci->num_sdqs = num_sdqs; err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_eq_ops, From 77c6e27df9e50dfe7deb5a0cfe8c30175b88d89a Mon Sep 17 00:00:00 2001 From: Amit Cohen Date: Tue, 2 Apr 2024 15:54:28 +0200 Subject: [PATCH 15/15] mlxsw: pci: Store DQ pointer as part of CQ structure Currently, for each completion, we check the number of descriptor queue and take it via mlxsw_pci_{sdq,rdq}_get(). This is inefficient, the DQ should be the same for all the completions in CQ, as each CQ handles only one DQ - SDQ or RDQ. This mapping is handled as part of DQ initialization via mlxsw_cmd_mbox_sw2hw_dq_cq_set(). Instead, as part of DQ initialization, set DQ pointer in the appropriate CQ structure. When we handle completions, warn in case that the DQ number that we expect is different from the number we get in the CQE. Call WARN_ON_ONCE() only after checking the value, to avoid calling this method for each completion. Signed-off-by: Amit Cohen Reviewed-by: Ido Schimmel Signed-off-by: Petr Machata Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/a5b2559cd6d532c120f3194f89a1e257110318f1.1712062203.git.petrm@nvidia.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlxsw/pci.c | 41 ++++++++++++++++------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index 094b9d752a02e..4d617057af25c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -86,6 +86,7 @@ struct mlxsw_pci_queue { struct mlxsw_pci *pci; struct { enum mlxsw_pci_cqe_v v; + struct mlxsw_pci_queue *dq; } cq; }; @@ -194,13 +195,6 @@ static struct mlxsw_pci_queue *mlxsw_pci_sdq_get(struct mlxsw_pci *mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_SDQ, q_num); } -static struct mlxsw_pci_queue *mlxsw_pci_rdq_get(struct mlxsw_pci *mlxsw_pci, - u8 q_num) -{ - return __mlxsw_pci_queue_get(mlxsw_pci, - MLXSW_PCI_QUEUE_TYPE_RDQ, q_num); -} - static struct mlxsw_pci_queue *mlxsw_pci_cq_get(struct mlxsw_pci *mlxsw_pci, u8 q_num) { @@ -265,7 +259,9 @@ static dma_addr_t __mlxsw_pci_queue_page_get(struct mlxsw_pci_queue *q, static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, struct mlxsw_pci_queue *q) { + struct mlxsw_pci_queue *cq; int tclass; + u8 cq_num; int lp; int i; int err; @@ -278,7 +274,8 @@ static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, MLXSW_CMD_MBOX_SW2HW_DQ_SDQ_LP_WQE; /* Set CQ of same number of this SDQ. */ - mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, q->num); + cq_num = q->num; + mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, cq_num); mlxsw_cmd_mbox_sw2hw_dq_sdq_lp_set(mbox, lp); mlxsw_cmd_mbox_sw2hw_dq_sdq_tclass_set(mbox, tclass); mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */ @@ -291,6 +288,9 @@ static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, err = mlxsw_cmd_sw2hw_sdq(mlxsw_pci->core, mbox, q->num); if (err) return err; + + cq = mlxsw_pci_cq_get(mlxsw_pci, cq_num); + cq->cq.dq = q; mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q); return 0; } @@ -374,6 +374,8 @@ static int mlxsw_pci_rdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, { struct mlxsw_pci_queue_elem_info *elem_info; u8 sdq_count = mlxsw_pci->num_sdqs; + struct mlxsw_pci_queue *cq; + u8 cq_num; int i; int err; @@ -383,7 +385,8 @@ static int mlxsw_pci_rdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, /* Set CQ of same number of this RDQ with base * above SDQ count as the lower ones are assigned to SDQs. */ - mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, sdq_count + q->num); + cq_num = sdq_count + q->num; + mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, cq_num); mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */ for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) { dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i); @@ -395,6 +398,9 @@ static int mlxsw_pci_rdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, if (err) return err; + cq = mlxsw_pci_cq_get(mlxsw_pci, cq_num); + cq->cq.dq = q; + mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q); for (i = 0; i < q->count; i++) { @@ -415,6 +421,7 @@ static int mlxsw_pci_rdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, elem_info = mlxsw_pci_queue_elem_info_get(q, i); mlxsw_pci_rdq_skb_free(mlxsw_pci, elem_info); } + cq->cq.dq = NULL; mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num); return err; @@ -648,6 +655,7 @@ static char *mlxsw_pci_cq_sw_cqe_get(struct mlxsw_pci_queue *q) static void mlxsw_pci_cq_rx_tasklet(struct tasklet_struct *t) { struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet); + struct mlxsw_pci_queue *rdq = q->cq.dq; struct mlxsw_pci *mlxsw_pci = q->pci; int credits = q->count >> 1; int items = 0; @@ -658,17 +666,20 @@ static void mlxsw_pci_cq_rx_tasklet(struct tasklet_struct *t) u8 sendq = mlxsw_pci_cqe_sr_get(q->cq.v, cqe); u8 dqn = mlxsw_pci_cqe_dqn_get(q->cq.v, cqe); char ncqe[MLXSW_PCI_CQE_SIZE_MAX]; - struct mlxsw_pci_queue *rdq; if (unlikely(sendq)) { WARN_ON_ONCE(1); continue; } + if (unlikely(dqn != rdq->num)) { + WARN_ON_ONCE(1); + continue; + } + memcpy(ncqe, cqe, q->elem_size); mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); - rdq = mlxsw_pci_rdq_get(mlxsw_pci, dqn); mlxsw_pci_cqe_rdq_handle(mlxsw_pci, rdq, wqe_counter, q->cq.v, ncqe); @@ -682,6 +693,7 @@ static void mlxsw_pci_cq_rx_tasklet(struct tasklet_struct *t) static void mlxsw_pci_cq_tx_tasklet(struct tasklet_struct *t) { struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet); + struct mlxsw_pci_queue *sdq = q->cq.dq; struct mlxsw_pci *mlxsw_pci = q->pci; int credits = q->count >> 1; int items = 0; @@ -692,17 +704,20 @@ static void mlxsw_pci_cq_tx_tasklet(struct tasklet_struct *t) u8 sendq = mlxsw_pci_cqe_sr_get(q->cq.v, cqe); u8 dqn = mlxsw_pci_cqe_dqn_get(q->cq.v, cqe); char ncqe[MLXSW_PCI_CQE_SIZE_MAX]; - struct mlxsw_pci_queue *sdq; if (unlikely(!sendq)) { WARN_ON_ONCE(1); continue; } + if (unlikely(dqn != sdq->num)) { + WARN_ON_ONCE(1); + continue; + } + memcpy(ncqe, cqe, q->elem_size); mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); - sdq = mlxsw_pci_sdq_get(mlxsw_pci, dqn); mlxsw_pci_cqe_sdq_handle(mlxsw_pci, sdq, wqe_counter, q->cq.v, ncqe);