From 10a0176e4e6eb6243c4b1c55e50372e03139d592 Mon Sep 17 00:00:00 2001 From: "Mintz, Yuval" Date: Fri, 7 Apr 2017 11:04:57 +0300 Subject: [PATCH 1/5] qede: Update receive statistic once per NAPI Currently, each time an ingress packet is passed to networking stack the driver increments a per-queue SW statistic. As we want to have additional fields in the first cache-line of the Rx-queue struct, change flow so this statistic would be updated once per NAPI run. We will later push the statistic to a different cache line. Signed-off-by: Yuval Mintz Signed-off-by: David S. Miller --- drivers/net/ethernet/qlogic/qede/qede_fp.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index 1e65038c8fc06..c77e6972ab48a 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -624,7 +624,6 @@ static inline void qede_skb_receive(struct qede_dev *edev, __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); napi_gro_receive(&fp->napi, skb); - rxq->rcv_pkts++; } static void qede_set_gro_params(struct qede_dev *edev, @@ -884,9 +883,9 @@ static inline void qede_tpa_cont(struct qede_dev *edev, "Strange - TPA cont with more than a single len_list entry\n"); } -static void qede_tpa_end(struct qede_dev *edev, - struct qede_fastpath *fp, - struct eth_fast_path_rx_tpa_end_cqe *cqe) +static int qede_tpa_end(struct qede_dev *edev, + struct qede_fastpath *fp, + struct eth_fast_path_rx_tpa_end_cqe *cqe) { struct qede_rx_queue *rxq = fp->rxq; struct qede_agg_info *tpa_info; @@ -934,11 +933,12 @@ static void qede_tpa_end(struct qede_dev *edev, tpa_info->state = QEDE_AGG_STATE_NONE; - return; + return 1; err: tpa_info->state = QEDE_AGG_STATE_NONE; dev_kfree_skb_any(tpa_info->skb); tpa_info->skb = NULL; + return 0; } static u8 qede_check_notunn_csum(u16 flag) @@ -1178,8 +1178,7 @@ static int qede_rx_process_tpa_cqe(struct qede_dev *edev, qede_tpa_cont(edev, rxq, &cqe->fast_path_tpa_cont); return 0; case ETH_RX_CQE_TYPE_TPA_END: - qede_tpa_end(edev, fp, &cqe->fast_path_tpa_end); - return 1; + return qede_tpa_end(edev, fp, &cqe->fast_path_tpa_end); default: return 0; } @@ -1229,7 +1228,7 @@ static int qede_rx_process_cqe(struct qede_dev *edev, /* Run eBPF program if one is attached */ if (xdp_prog) if (!qede_rx_xdp(edev, fp, rxq, xdp_prog, bd, fp_cqe)) - return 1; + return 0; /* If this is an error packet then drop it */ flags = cqe->fast_path_regular.pars_flags.flags; @@ -1290,8 +1289,8 @@ static int qede_rx_int(struct qede_fastpath *fp, int budget) { struct qede_rx_queue *rxq = fp->rxq; struct qede_dev *edev = fp->edev; + int work_done = 0, rcv_pkts = 0; u16 hw_comp_cons, sw_comp_cons; - int work_done = 0; hw_comp_cons = le16_to_cpu(*rxq->hw_cons_ptr); sw_comp_cons = qed_chain_get_cons_idx(&rxq->rx_comp_ring); @@ -1305,12 +1304,14 @@ static int qede_rx_int(struct qede_fastpath *fp, int budget) /* Loop to complete all indicated BDs */ while ((sw_comp_cons != hw_comp_cons) && (work_done < budget)) { - qede_rx_process_cqe(edev, fp, rxq); + rcv_pkts += qede_rx_process_cqe(edev, fp, rxq); qed_chain_recycle_consumed(&rxq->rx_comp_ring); sw_comp_cons = qed_chain_get_cons_idx(&rxq->rx_comp_ring); work_done++; } + rxq->rcv_pkts += rcv_pkts; + /* Allocate replacement buffers */ while (rxq->num_rx_buffers - rxq->filled_buffers) if (qede_alloc_rx_buffer(rxq, false)) From 89e1afc44765d8b9f509d15df096494f14463e17 Mon Sep 17 00:00:00 2001 From: "Mintz, Yuval" Date: Fri, 7 Apr 2017 11:04:58 +0300 Subject: [PATCH 2/5] qede: Correct XDP forward unmapping Driver is currently using dma_unmap_single() with the address it passed to device for the purpose of forwarding, but the XDP transmission buffer was originally a page allocated for the rx-queue. The mapped address is likely to differ from the original mapped address due to the placement offset. This difference is going to get even bigger once we support headroom. Cache the original mapped address of the page, and use it for unmapping of the buffer when completion arrives for the XDP forwarded packet. Signed-off-by: Yuval Mintz Signed-off-by: David S. Miller --- drivers/net/ethernet/qlogic/qede/qede.h | 9 +++++++-- drivers/net/ethernet/qlogic/qede/qede_fp.c | 17 +++++++++-------- drivers/net/ethernet/qlogic/qede/qede_main.c | 8 ++++---- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h index e73a4a5165ee7..7ab2201a43b26 100644 --- a/drivers/net/ethernet/qlogic/qede/qede.h +++ b/drivers/net/ethernet/qlogic/qede/qede.h @@ -349,6 +349,11 @@ struct sw_tx_bd { #define QEDE_TSO_SPLIT_BD BIT(0) }; +struct sw_tx_xdp { + struct page *page; + dma_addr_t mapping; +}; + struct qede_tx_queue { u8 is_xdp; bool is_legacy; @@ -372,11 +377,11 @@ struct qede_tx_queue { #define QEDE_TXQ_IDX_TO_XDP(edev, idx) ((idx) + QEDE_MAX_TSS_CNT(edev)) /* Regular Tx requires skb + metadata for release purpose, - * while XDP requires only the pages themselves. + * while XDP requires the pages and the mapped address. */ union { struct sw_tx_bd *skbs; - struct page **pages; + struct sw_tx_xdp *xdp; } sw_tx_ring; struct qed_chain tx_pbl; diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index c77e6972ab48a..c61cfcfbbd56e 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -360,7 +360,8 @@ static int qede_xdp_xmit(struct qede_dev *edev, struct qede_fastpath *fp, metadata->mapping + padding, length, PCI_DMA_TODEVICE); - txq->sw_tx_ring.pages[idx] = metadata->data; + txq->sw_tx_ring.xdp[idx].page = metadata->data; + txq->sw_tx_ring.xdp[idx].mapping = metadata->mapping; txq->sw_tx_prod++; /* Mark the fastpath for future XDP doorbell */ @@ -384,19 +385,19 @@ int qede_txq_has_work(struct qede_tx_queue *txq) static void qede_xdp_tx_int(struct qede_dev *edev, struct qede_tx_queue *txq) { - struct eth_tx_1st_bd *bd; - u16 hw_bd_cons; + u16 hw_bd_cons, idx; hw_bd_cons = le16_to_cpu(*txq->hw_cons_ptr); barrier(); while (hw_bd_cons != qed_chain_get_cons_idx(&txq->tx_pbl)) { - bd = (struct eth_tx_1st_bd *)qed_chain_consume(&txq->tx_pbl); + qed_chain_consume(&txq->tx_pbl); + idx = txq->sw_tx_cons & NUM_TX_BDS_MAX; - dma_unmap_single(&edev->pdev->dev, BD_UNMAP_ADDR(bd), - PAGE_SIZE, DMA_BIDIRECTIONAL); - __free_page(txq->sw_tx_ring.pages[txq->sw_tx_cons & - NUM_TX_BDS_MAX]); + dma_unmap_page(&edev->pdev->dev, + txq->sw_tx_ring.xdp[idx].mapping, + PAGE_SIZE, DMA_BIDIRECTIONAL); + __free_page(txq->sw_tx_ring.xdp[idx].page); txq->sw_tx_cons++; txq->xmit_pkts++; diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index abd99109e5328..fa62c37dac7a2 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -1251,7 +1251,7 @@ static void qede_free_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq) { /* Free the parallel SW ring */ if (txq->is_xdp) - kfree(txq->sw_tx_ring.pages); + kfree(txq->sw_tx_ring.xdp); else kfree(txq->sw_tx_ring.skbs); @@ -1269,9 +1269,9 @@ static int qede_alloc_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq) /* Allocate the parallel driver ring for Tx buffers */ if (txq->is_xdp) { - size = sizeof(*txq->sw_tx_ring.pages) * TX_RING_SIZE; - txq->sw_tx_ring.pages = kzalloc(size, GFP_KERNEL); - if (!txq->sw_tx_ring.pages) + size = sizeof(*txq->sw_tx_ring.xdp) * TX_RING_SIZE; + txq->sw_tx_ring.xdp = kzalloc(size, GFP_KERNEL); + if (!txq->sw_tx_ring.xdp) goto err; } else { size = sizeof(*txq->sw_tx_ring.skbs) * TX_RING_SIZE; From 40b8c45492efc97d9d397475bf475d497fbb9174 Mon Sep 17 00:00:00 2001 From: "Mintz, Yuval" Date: Fri, 7 Apr 2017 11:04:59 +0300 Subject: [PATCH 3/5] qede: Prevent VFs from using XDP Current implementation of VFs is very tight in regard to queue resources. VFs support for XDP would require quite a bit of additional infrastructure in qede and qed [sharing of queue-zones between queues, more VF cids, mapping of the doorbell bar, etc.]. For now, prevent XDP programs from being attached to VFs. Signed-off-by: Yuval Mintz Signed-off-by: David S. Miller --- drivers/net/ethernet/qlogic/qede/qede_filter.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/ethernet/qlogic/qede/qede_filter.c b/drivers/net/ethernet/qlogic/qede/qede_filter.c index 107c3fda4792c..ecd5b3ab783f6 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_filter.c +++ b/drivers/net/ethernet/qlogic/qede/qede_filter.c @@ -537,6 +537,11 @@ int qede_xdp(struct net_device *dev, struct netdev_xdp *xdp) { struct qede_dev *edev = netdev_priv(dev); + if (IS_VF(edev)) { + DP_NOTICE(edev, "VFs don't support XDP\n"); + return -EOPNOTSUPP; + } + switch (xdp->command) { case XDP_SETUP_PROG: return qede_xdp_set(edev, xdp->prog); From 15ed8a47ff0571dd268e37002511993b47e996bd Mon Sep 17 00:00:00 2001 From: "Mintz, Yuval" Date: Fri, 7 Apr 2017 11:05:00 +0300 Subject: [PATCH 4/5] qede: Add support for ingress headroom Driver currently doesn't support any headroom; The only 'available' space it has in the head of the buffer is due to the placement offset. In order to allow [later] support of XDP adjustment of headroom, modify the the ingress flow to properly handle a scenario where the packets would have such. Signed-off-by: Yuval Mintz Signed-off-by: David S. Miller --- drivers/net/ethernet/qlogic/qede/qede.h | 13 ++++++----- drivers/net/ethernet/qlogic/qede/qede_fp.c | 23 +++++++++++--------- drivers/net/ethernet/qlogic/qede/qede_main.c | 5 +++-- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h index 7ab2201a43b26..5e7ad25db8adb 100644 --- a/drivers/net/ethernet/qlogic/qede/qede.h +++ b/drivers/net/ethernet/qlogic/qede/qede.h @@ -313,21 +313,24 @@ struct qede_rx_queue { u8 data_direction; u8 rxq_id; + /* Used once per each NAPI run */ + u16 num_rx_buffers; + + u16 rx_headroom; + u32 rx_buf_size; u32 rx_buf_seg_size; - u64 rcv_pkts; - struct sw_rx_data *sw_rx_ring; struct qed_chain rx_bd_ring; struct qed_chain rx_comp_ring ____cacheline_aligned; - /* Used once per each NAPI run */ - u16 num_rx_buffers; - /* GRO */ struct qede_agg_info tpa_info[ETH_TPA_MAX_AGGS_NUM]; + /* Used once per each NAPI run */ + u64 rcv_pkts; + u64 rx_hw_errors; u64 rx_alloc_errors; u64 rx_ip_frags; diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index c61cfcfbbd56e..e382d4bdf4303 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -87,7 +87,8 @@ int qede_alloc_rx_buffer(struct qede_rx_queue *rxq, bool allow_lazy) rx_bd = (struct eth_rx_bd *)qed_chain_produce(&rxq->rx_bd_ring); WARN_ON(!rx_bd); rx_bd->addr.hi = cpu_to_le32(upper_32_bits(mapping)); - rx_bd->addr.lo = cpu_to_le32(lower_32_bits(mapping)); + rx_bd->addr.lo = cpu_to_le32(lower_32_bits(mapping) + + rxq->rx_headroom); rxq->sw_rx_prod++; rxq->filled_buffers++; @@ -509,7 +510,8 @@ static inline void qede_reuse_page(struct qede_rx_queue *rxq, new_mapping = curr_prod->mapping + curr_prod->page_offset; rx_bd_prod->addr.hi = cpu_to_le32(upper_32_bits(new_mapping)); - rx_bd_prod->addr.lo = cpu_to_le32(lower_32_bits(new_mapping)); + rx_bd_prod->addr.lo = cpu_to_le32(lower_32_bits(new_mapping) + + rxq->rx_headroom); rxq->sw_rx_prod++; curr_cons->data = NULL; @@ -991,13 +993,14 @@ static bool qede_rx_xdp(struct qede_dev *edev, struct qede_rx_queue *rxq, struct bpf_prog *prog, struct sw_rx_data *bd, - struct eth_fast_path_rx_reg_cqe *cqe) + struct eth_fast_path_rx_reg_cqe *cqe, + u16 data_offset) { u16 len = le16_to_cpu(cqe->len_on_first_bd); struct xdp_buff xdp; enum xdp_action act; - xdp.data = page_address(bd->data) + cqe->placement_offset; + xdp.data = page_address(bd->data) + data_offset; xdp.data_end = xdp.data + len; /* Queues always have a full reset currently, so for the time @@ -1026,7 +1029,7 @@ static bool qede_rx_xdp(struct qede_dev *edev, /* Now if there's a transmission problem, we'd still have to * throw current buffer, as replacement was already allocated. */ - if (qede_xdp_xmit(edev, fp, bd, cqe->placement_offset, len)) { + if (qede_xdp_xmit(edev, fp, bd, data_offset, len)) { dma_unmap_page(rxq->dev, bd->mapping, PAGE_SIZE, DMA_BIDIRECTIONAL); __free_page(bd->data); @@ -1053,7 +1056,7 @@ static struct sk_buff *qede_rx_allocate_skb(struct qede_dev *edev, struct sw_rx_data *bd, u16 len, u16 pad) { - unsigned int offset = bd->page_offset; + unsigned int offset = bd->page_offset + pad; struct skb_frag_struct *frag; struct page *page = bd->data; unsigned int pull_len; @@ -1070,7 +1073,7 @@ static struct sk_buff *qede_rx_allocate_skb(struct qede_dev *edev, */ if (len + pad <= edev->rx_copybreak) { memcpy(skb_put(skb, len), - page_address(page) + pad + offset, len); + page_address(page) + offset, len); qede_reuse_page(rxq, bd); goto out; } @@ -1078,7 +1081,7 @@ static struct sk_buff *qede_rx_allocate_skb(struct qede_dev *edev, frag = &skb_shinfo(skb)->frags[0]; skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - page, pad + offset, len, rxq->rx_buf_seg_size); + page, offset, len, rxq->rx_buf_seg_size); va = skb_frag_address(frag); pull_len = eth_get_headlen(va, QEDE_RX_HDR_SIZE); @@ -1224,11 +1227,11 @@ static int qede_rx_process_cqe(struct qede_dev *edev, fp_cqe = &cqe->fast_path_regular; len = le16_to_cpu(fp_cqe->len_on_first_bd); - pad = fp_cqe->placement_offset; + pad = fp_cqe->placement_offset + rxq->rx_headroom; /* Run eBPF program if one is attached */ if (xdp_prog) - if (!qede_rx_xdp(edev, fp, rxq, xdp_prog, bd, fp_cqe)) + if (!qede_rx_xdp(edev, fp, rxq, xdp_prog, bd, fp_cqe, pad)) return 0; /* If this is an error packet then drop it */ diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index fa62c37dac7a2..91c3078c1ae45 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -1188,8 +1188,9 @@ static int qede_alloc_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq) rxq->rx_buf_size = NET_IP_ALIGN + ETH_OVERHEAD + edev->ndev->mtu; - if (rxq->rx_buf_size > PAGE_SIZE) - rxq->rx_buf_size = PAGE_SIZE; + /* Make sure that the headroom and payload fit in a single page */ + if (rxq->rx_buf_size + rxq->rx_headroom > PAGE_SIZE) + rxq->rx_buf_size = PAGE_SIZE - rxq->rx_headroom; /* Segment size to spilt a page in multiple equal parts, * unless XDP is used in which case we'd use the entire page. From 059eeb07e175086db1f84c1d8d29bb9aa8057797 Mon Sep 17 00:00:00 2001 From: "Mintz, Yuval" Date: Fri, 7 Apr 2017 11:05:01 +0300 Subject: [PATCH 5/5] qede: Support XDP adjustment of headers In case an XDP program is attached, reserve XDP_PACKET_HEADROOM bytes at the beginning of the packet for the program to play with. Modify the XDP logic in the driver to fill-in the missing bits and re-calculate offsets and length after the program has finished running to properly reflect the current status of the packet. We can then go and remove the limitation of not supporting XDP programs where xdp_adjust_head is set. Signed-off-by: Yuval Mintz Signed-off-by: David S. Miller --- drivers/net/ethernet/qlogic/qede/qede_filter.c | 5 ----- drivers/net/ethernet/qlogic/qede/qede_fp.c | 17 +++++++++++------ drivers/net/ethernet/qlogic/qede/qede_main.c | 1 + 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/net/ethernet/qlogic/qede/qede_filter.c b/drivers/net/ethernet/qlogic/qede/qede_filter.c index ecd5b3ab783f6..b00a4fce44b7b 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_filter.c +++ b/drivers/net/ethernet/qlogic/qede/qede_filter.c @@ -520,11 +520,6 @@ static int qede_xdp_set(struct qede_dev *edev, struct bpf_prog *prog) { struct qede_reload_args args; - if (prog && prog->xdp_adjust_head) { - DP_ERR(edev, "Does not support bpf_xdp_adjust_head()\n"); - return -EOPNOTSUPP; - } - /* If we're called, there was already a bpf reference increment */ args.func = &qede_xdp_reload_func; args.u.new_prog = prog; diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index e382d4bdf4303..961b1d36b9eb8 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -994,14 +994,14 @@ static bool qede_rx_xdp(struct qede_dev *edev, struct bpf_prog *prog, struct sw_rx_data *bd, struct eth_fast_path_rx_reg_cqe *cqe, - u16 data_offset) + u16 *data_offset, u16 *len) { - u16 len = le16_to_cpu(cqe->len_on_first_bd); struct xdp_buff xdp; enum xdp_action act; - xdp.data = page_address(bd->data) + data_offset; - xdp.data_end = xdp.data + len; + xdp.data_hard_start = page_address(bd->data); + xdp.data = xdp.data_hard_start + *data_offset; + xdp.data_end = xdp.data + *len; /* Queues always have a full reset currently, so for the time * being until there's atomic program replace just mark read @@ -1011,6 +1011,10 @@ static bool qede_rx_xdp(struct qede_dev *edev, act = bpf_prog_run_xdp(prog, &xdp); rcu_read_unlock(); + /* Recalculate, as XDP might have changed the headers */ + *data_offset = xdp.data - xdp.data_hard_start; + *len = xdp.data_end - xdp.data; + if (act == XDP_PASS) return true; @@ -1029,7 +1033,7 @@ static bool qede_rx_xdp(struct qede_dev *edev, /* Now if there's a transmission problem, we'd still have to * throw current buffer, as replacement was already allocated. */ - if (qede_xdp_xmit(edev, fp, bd, data_offset, len)) { + if (qede_xdp_xmit(edev, fp, bd, *data_offset, *len)) { dma_unmap_page(rxq->dev, bd->mapping, PAGE_SIZE, DMA_BIDIRECTIONAL); __free_page(bd->data); @@ -1231,7 +1235,8 @@ static int qede_rx_process_cqe(struct qede_dev *edev, /* Run eBPF program if one is attached */ if (xdp_prog) - if (!qede_rx_xdp(edev, fp, rxq, xdp_prog, bd, fp_cqe, pad)) + if (!qede_rx_xdp(edev, fp, rxq, xdp_prog, bd, fp_cqe, + &pad, &len)) return 0; /* If this is an error packet then drop it */ diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index 91c3078c1ae45..8c2baf8b2a087 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -1187,6 +1187,7 @@ static int qede_alloc_mem_rxq(struct qede_dev *edev, struct qede_rx_queue *rxq) rxq->num_rx_buffers = edev->q_num_rx_buffers; rxq->rx_buf_size = NET_IP_ALIGN + ETH_OVERHEAD + edev->ndev->mtu; + rxq->rx_headroom = edev->xdp_prog ? XDP_PACKET_HEADROOM : 0; /* Make sure that the headroom and payload fit in a single page */ if (rxq->rx_buf_size + rxq->rx_headroom > PAGE_SIZE)