Skip to content

Commit

Permalink
Merge branch 'add-af_xdp-support-for-cn10k'
Browse files Browse the repository at this point in the history 
Suman Ghosh says:

====================
Add af_xdp support for cn10k

This patchset includes changes to support AF_XDP for cn10k chipsets. Both
non-zero copy and zero copy will be supported after these changes. Also,
the RSS will be reconfigured once a particular receive queue is
added/removed to/from AF_XDP support.

Patch #1: octeontx2-pf: use xdp_return_frame() to free xdp buffers

Patch #2: octeontx2-pf: Add AF_XDP non-zero copy support

Patch #3: octeontx2-pf: AF_XDP zero copy receive support

Patch #4: octeontx2-pf: Reconfigure RSS table after enabling AF_XDP
zerocopy on rx queue

Patch #5: octeontx2-pf: Prepare for AF_XDP transmit

Patch #6: octeontx2-pf: AF_XDP zero copy transmit support
====================

Link: https://patch.msgid.link/20250213053141.2833254-1-sumang@marvell.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
  • Loading branch information
Paolo Abeni committed Feb 18, 2025
2 parents 0f375d9 + 53616af commit b4cb730
Show file tree
Hide file tree
Showing 12 changed files with 554 additions and 92 deletions.
2 changes: 1 addition & 1 deletion drivers/net/ethernet/marvell/octeontx2/nic/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ obj-$(CONFIG_RVU_ESWITCH) += rvu_rep.o

rvu_nicpf-y := otx2_pf.o otx2_common.o otx2_txrx.o otx2_ethtool.o \
otx2_flows.o otx2_tc.o cn10k.o otx2_dmac_flt.o \
otx2_devlink.o qos_sq.o qos.o
otx2_devlink.o qos_sq.o qos.o otx2_xsk.o
rvu_nicvf-y := otx2_vf.o
rvu_rep-y := rep.o

Expand Down
7 changes: 6 additions & 1 deletion drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,12 @@ int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
struct otx2_nic *pfvf = dev;
int cnt = cq->pool_ptrs;
u64 ptrs[NPA_MAX_BURST];
struct otx2_pool *pool;
dma_addr_t bufptr;
int num_ptrs = 1;

pool = &pfvf->qset.pool[cq->cq_idx];

/* Refill pool with new buffers */
while (cq->pool_ptrs) {
if (otx2_alloc_buffer(pfvf, cq, &bufptr)) {
Expand All @@ -124,7 +127,9 @@ int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
break;
}
cq->pool_ptrs--;
ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM;
ptrs[num_ptrs] = pool->xsk_pool ?
(u64)bufptr : (u64)bufptr + OTX2_HEAD_ROOM;

num_ptrs++;
if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) {
__cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
Expand Down
122 changes: 94 additions & 28 deletions drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include "otx2_common.h"
#include "otx2_struct.h"
#include "cn10k.h"
#include "otx2_xsk.h"

static bool otx2_is_pfc_enabled(struct otx2_nic *pfvf)
{
Expand Down Expand Up @@ -330,6 +331,10 @@ int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id)
rss_ctx = rss->rss_ctx[ctx_id];
/* Get memory to put this msg */
for (idx = 0; idx < rss->rss_size; idx++) {
/* Ignore the queue if AF_XDP zero copy is enabled */
if (test_bit(rss_ctx->ind_tbl[idx], pfvf->af_xdp_zc_qidx))
continue;

aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
if (!aq) {
/* The shared memory buffer can be full.
Expand Down Expand Up @@ -549,10 +554,13 @@ static int otx2_alloc_pool_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
}

static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
dma_addr_t *dma)
dma_addr_t *dma, int qidx, int idx)
{
u8 *buf;

if (pool->xsk_pool)
return otx2_xsk_pool_alloc_buf(pfvf, pool, dma, idx);

if (pool->page_pool)
return otx2_alloc_pool_buf(pfvf, pool, dma);

Expand All @@ -571,20 +579,21 @@ static int __otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
}

int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
dma_addr_t *dma)
dma_addr_t *dma, int qidx, int idx)
{
int ret;

local_bh_disable();
ret = __otx2_alloc_rbuf(pfvf, pool, dma);
ret = __otx2_alloc_rbuf(pfvf, pool, dma, qidx, idx);
local_bh_enable();
return ret;
}

int otx2_alloc_buffer(struct otx2_nic *pfvf, struct otx2_cq_queue *cq,
dma_addr_t *dma)
{
if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma)))
if (unlikely(__otx2_alloc_rbuf(pfvf, cq->rbpool, dma,
cq->cq_idx, cq->pool_ptrs - 1)))
return -ENOMEM;
return 0;
}
Expand Down Expand Up @@ -884,7 +893,7 @@ void otx2_sqb_flush(struct otx2_nic *pfvf)
#define RQ_PASS_LVL_AURA (255 - ((95 * 256) / 100)) /* RED when 95% is full */
#define RQ_DROP_LVL_AURA (255 - ((99 * 256) / 100)) /* Drop when 99% is full */

static int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura)
int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura)
{
struct otx2_qset *qset = &pfvf->qset;
struct nix_aq_enq_req *aq;
Expand Down Expand Up @@ -1028,6 +1037,10 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura)

sq->stats.bytes = 0;
sq->stats.pkts = 0;
/* Attach XSK_BUFF_POOL to XDP queue */
if (qidx > pfvf->hw.xdp_queues)
otx2_attach_xsk_buff(pfvf, sq, (qidx - pfvf->hw.xdp_queues));


chan_offset = qidx % pfvf->hw.tx_chan_cnt;
err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, chan_offset, sqb_aura);
Expand All @@ -1041,12 +1054,13 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura)

}

static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
{
struct otx2_qset *qset = &pfvf->qset;
int err, pool_id, non_xdp_queues;
struct nix_aq_enq_req *aq;
struct otx2_cq_queue *cq;
struct otx2_pool *pool;

cq = &qset->cq[qidx];
cq->cq_idx = qidx;
Expand All @@ -1055,8 +1069,20 @@ static int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx)
cq->cq_type = CQ_RX;
cq->cint_idx = qidx;
cq->cqe_cnt = qset->rqe_cnt;
if (pfvf->xdp_prog)
if (pfvf->xdp_prog) {
xdp_rxq_info_reg(&cq->xdp_rxq, pfvf->netdev, qidx, 0);
pool = &qset->pool[qidx];
if (pool->xsk_pool) {
xdp_rxq_info_reg_mem_model(&cq->xdp_rxq,
MEM_TYPE_XSK_BUFF_POOL,
NULL);
xsk_pool_set_rxq_info(pool->xsk_pool, &cq->xdp_rxq);
} else if (pool->page_pool) {
xdp_rxq_info_reg_mem_model(&cq->xdp_rxq,
MEM_TYPE_PAGE_POOL,
pool->page_pool);
}
}
} else if (qidx < non_xdp_queues) {
cq->cq_type = CQ_TX;
cq->cint_idx = qidx - pfvf->hw.rx_queues;
Expand Down Expand Up @@ -1275,9 +1301,10 @@ void otx2_free_bufs(struct otx2_nic *pfvf, struct otx2_pool *pool,

pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
page = virt_to_head_page(phys_to_virt(pa));

if (pool->page_pool) {
page_pool_put_full_page(pool->page_pool, page, true);
} else if (pool->xsk_pool) {
/* Note: No way of identifying xdp_buff */
} else {
dma_unmap_page_attrs(pfvf->dev, iova, size,
DMA_FROM_DEVICE,
Expand All @@ -1292,6 +1319,7 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type)
int pool_id, pool_start = 0, pool_end = 0, size = 0;
struct otx2_pool *pool;
u64 iova;
int idx;

if (type == AURA_NIX_SQ) {
pool_start = otx2_get_pool_idx(pfvf, type, 0);
Expand All @@ -1306,16 +1334,21 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type)

/* Free SQB and RQB pointers from the aura pool */
for (pool_id = pool_start; pool_id < pool_end; pool_id++) {
iova = otx2_aura_allocptr(pfvf, pool_id);
pool = &pfvf->qset.pool[pool_id];
iova = otx2_aura_allocptr(pfvf, pool_id);
while (iova) {
if (type == AURA_NIX_RQ)
iova -= OTX2_HEAD_ROOM;

otx2_free_bufs(pfvf, pool, iova, size);

iova = otx2_aura_allocptr(pfvf, pool_id);
}

for (idx = 0 ; idx < pool->xdp_cnt; idx++) {
if (!pool->xdp[idx])
continue;

xsk_buff_free(pool->xdp[idx]);
}
}
}

Expand All @@ -1332,7 +1365,8 @@ void otx2_aura_pool_free(struct otx2_nic *pfvf)
qmem_free(pfvf->dev, pool->stack);
qmem_free(pfvf->dev, pool->fc_addr);
page_pool_destroy(pool->page_pool);
pool->page_pool = NULL;
devm_kfree(pfvf->dev, pool->xdp);
pool->xsk_pool = NULL;
}
devm_kfree(pfvf->dev, pfvf->qset.pool);
pfvf->qset.pool = NULL;
Expand Down Expand Up @@ -1419,6 +1453,7 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id,
int stack_pages, int numptrs, int buf_size, int type)
{
struct page_pool_params pp_params = { 0 };
struct xsk_buff_pool *xsk_pool;
struct npa_aq_enq_req *aq;
struct otx2_pool *pool;
int err;
Expand Down Expand Up @@ -1462,21 +1497,35 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id,
aq->ctype = NPA_AQ_CTYPE_POOL;
aq->op = NPA_AQ_INSTOP_INIT;

if (type != AURA_NIX_RQ) {
pool->page_pool = NULL;
if (type != AURA_NIX_RQ)
return 0;

if (!test_bit(pool_id, pfvf->af_xdp_zc_qidx)) {
pp_params.order = get_order(buf_size);
pp_params.flags = PP_FLAG_DMA_MAP;
pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs);
pp_params.nid = NUMA_NO_NODE;
pp_params.dev = pfvf->dev;
pp_params.dma_dir = DMA_FROM_DEVICE;
pool->page_pool = page_pool_create(&pp_params);
if (IS_ERR(pool->page_pool)) {
netdev_err(pfvf->netdev, "Creation of page pool failed\n");
return PTR_ERR(pool->page_pool);
}
return 0;
}

pp_params.order = get_order(buf_size);
pp_params.flags = PP_FLAG_DMA_MAP;
pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs);
pp_params.nid = NUMA_NO_NODE;
pp_params.dev = pfvf->dev;
pp_params.dma_dir = DMA_FROM_DEVICE;
pool->page_pool = page_pool_create(&pp_params);
if (IS_ERR(pool->page_pool)) {
netdev_err(pfvf->netdev, "Creation of page pool failed\n");
return PTR_ERR(pool->page_pool);
/* Set XSK pool to support AF_XDP zero-copy */
xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, pool_id);
if (xsk_pool) {
pool->xsk_pool = xsk_pool;
pool->xdp_cnt = numptrs;
pool->xdp = devm_kcalloc(pfvf->dev,
numptrs, sizeof(struct xdp_buff *), GFP_KERNEL);
if (IS_ERR(pool->xdp)) {
netdev_err(pfvf->netdev, "Creation of xsk pool failed\n");
return PTR_ERR(pool->xdp);
}
}

return 0;
Expand Down Expand Up @@ -1537,9 +1586,18 @@ int otx2_sq_aura_pool_init(struct otx2_nic *pfvf)
}

for (ptr = 0; ptr < num_sqbs; ptr++) {
err = otx2_alloc_rbuf(pfvf, pool, &bufptr);
if (err)
err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr);
if (err) {
if (pool->xsk_pool) {
ptr--;
while (ptr >= 0) {
xsk_buff_free(pool->xdp[ptr]);
ptr--;
}
}
goto err_mem;
}

pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr);
sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr;
}
Expand Down Expand Up @@ -1589,11 +1647,19 @@ int otx2_rq_aura_pool_init(struct otx2_nic *pfvf)
/* Allocate pointers and free them to aura/pool */
for (pool_id = 0; pool_id < hw->rqpool_cnt; pool_id++) {
pool = &pfvf->qset.pool[pool_id];

for (ptr = 0; ptr < num_ptrs; ptr++) {
err = otx2_alloc_rbuf(pfvf, pool, &bufptr);
if (err)
err = otx2_alloc_rbuf(pfvf, pool, &bufptr, pool_id, ptr);
if (err) {
if (pool->xsk_pool) {
while (ptr)
xsk_buff_free(pool->xdp[--ptr]);
}
return -ENOMEM;
}

pfvf->hw_ops->aura_freeptr(pfvf, pool_id,
pool->xsk_pool ? bufptr :
bufptr + OTX2_HEAD_ROOM);
}
}
Expand Down
17 changes: 15 additions & 2 deletions drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include <linux/time64.h>
#include <linux/dim.h>
#include <uapi/linux/if_macsec.h>
#include <net/page_pool/helpers.h>

#include <mbox.h>
#include <npc.h>
Expand Down Expand Up @@ -128,6 +129,12 @@ enum otx2_errcodes_re {
ERRCODE_IL4_CSUM = 0x22,
};

enum otx2_xdp_action {
OTX2_XDP_TX = BIT(0),
OTX2_XDP_REDIRECT = BIT(1),
OTX2_AF_XDP_FRAME = BIT(2),
};

struct otx2_dev_stats {
u64 rx_bytes;
u64 rx_frames;
Expand Down Expand Up @@ -531,6 +538,8 @@ struct otx2_nic {

/* Inline ipsec */
struct cn10k_ipsec ipsec;
/* af_xdp zero-copy */
unsigned long *af_xdp_zc_qidx;
};

static inline bool is_otx2_lbkvf(struct pci_dev *pdev)
Expand Down Expand Up @@ -1002,7 +1011,7 @@ void otx2_txschq_free_one(struct otx2_nic *pfvf, u16 lvl, u16 schq);
void otx2_free_pending_sqe(struct otx2_nic *pfvf);
void otx2_sqb_flush(struct otx2_nic *pfvf);
int otx2_alloc_rbuf(struct otx2_nic *pfvf, struct otx2_pool *pool,
dma_addr_t *dma);
dma_addr_t *dma, int qidx, int idx);
int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable);
void otx2_ctx_disable(struct mbox *mbox, int type, bool npa);
int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable);
Expand Down Expand Up @@ -1032,6 +1041,8 @@ void otx2_pfaf_mbox_destroy(struct otx2_nic *pf);
void otx2_disable_mbox_intr(struct otx2_nic *pf);
void otx2_disable_napi(struct otx2_nic *pf);
irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq);
int otx2_rq_init(struct otx2_nic *pfvf, u16 qidx, u16 lpb_aura);
int otx2_cq_init(struct otx2_nic *pfvf, u16 qidx);

/* RSS configuration APIs*/
int otx2_rss_init(struct otx2_nic *pfvf);
Expand Down Expand Up @@ -1094,7 +1105,8 @@ int otx2_del_macfilter(struct net_device *netdev, const u8 *mac);
int otx2_add_macfilter(struct net_device *netdev, const u8 *mac);
int otx2_enable_rxvlan(struct otx2_nic *pf, bool enable);
int otx2_install_rxvlan_offload_flow(struct otx2_nic *pfvf);
bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx);
bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, struct xdp_frame *xdpf,
u64 iova, int len, u16 qidx, u16 flags);
u16 otx2_get_max_mtu(struct otx2_nic *pfvf);
int otx2_handle_ntuple_tc_features(struct net_device *netdev,
netdev_features_t features);
Expand Down Expand Up @@ -1175,4 +1187,5 @@ static inline int mcam_entry_cmp(const void *a, const void *b)
dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf,
struct sk_buff *skb, int seg, int *len);
void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg);
int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx);
#endif /* OTX2_COMMON_H */
6 changes: 5 additions & 1 deletion drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -910,8 +910,12 @@ static int otx2_get_rxfh(struct net_device *dev,
return -ENOENT;

if (indir) {
for (idx = 0; idx < rss->rss_size; idx++)
for (idx = 0; idx < rss->rss_size; idx++) {
/* Ignore if the rx queue is AF_XDP zero copy enabled */
if (test_bit(rss_ctx->ind_tbl[idx], pfvf->af_xdp_zc_qidx))
continue;
indir[idx] = rss_ctx->ind_tbl[idx];
}
}
if (rxfh->key)
memcpy(rxfh->key, rss->key, sizeof(rss->key));
Expand Down
Loading

0 comments on commit b4cb730

Please sign in to comment.