Skip to content

Commit

Permalink
tsnep: Add XDP socket zero-copy TX support
Browse files Browse the repository at this point in the history 
Send and complete XSK pool frames within TX NAPI context. NAPI context
is triggered by ndo_xsk_wakeup.

Test results with A53 1.2GHz:

xdpsock txonly copy mode, 64 byte frames:
                   pps            pkts           1.00
tx                 284,409        11,398,144
Two CPUs with 100% and 10% utilization.

xdpsock txonly zero-copy mode, 64 byte frames:
                   pps            pkts           1.00
tx                 511,929        5,890,368
Two CPUs with 100% and 1% utilization.

xdpsock l2fwd copy mode, 64 byte frames:
                   pps            pkts           1.00
rx                 248,985        7,315,885
tx                 248,921        7,315,885
Two CPUs with 100% and 10% utilization.

xdpsock l2fwd zero-copy mode, 64 byte frames:
                   pps            pkts           1.00
rx                 254,735        3,039,456
tx                 254,735        3,039,456
Two CPUs with 100% and 4% utilization.

Packet rate increases and CPU utilization is reduced in both cases.

Signed-off-by: Gerhard Engleder <gerhard@engleder-embedded.com>
Reviewed-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
  • Loading branch information
Gerhard Engleder authored and Jakub Kicinski committed Apr 25, 2023
1 parent 3fc2333 commit cd275c2
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 11 deletions.
2 changes: 2 additions & 0 deletions drivers/net/ethernet/engleder/tsnep.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ struct tsnep_tx_entry {
union {
struct sk_buff *skb;
struct xdp_frame *xdpf;
bool zc;
};
size_t len;
DEFINE_DMA_UNMAP_ADDR(dma);
Expand All @@ -88,6 +89,7 @@ struct tsnep_tx {
int read;
u32 owner_counter;
int increment_owner_counter;
struct xsk_buff_pool *xsk_pool;

u32 packets;
u32 bytes;
Expand Down
130 changes: 119 additions & 11 deletions drivers/net/ethernet/engleder/tsnep_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@
#define TSNEP_TX_TYPE_SKB_FRAG BIT(1)
#define TSNEP_TX_TYPE_XDP_TX BIT(2)
#define TSNEP_TX_TYPE_XDP_NDO BIT(3)
#define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO)
#define TSNEP_TX_TYPE_XSK BIT(4)

#define TSNEP_XDP_TX BIT(0)
#define TSNEP_XDP_REDIRECT BIT(1)
Expand Down Expand Up @@ -322,13 +324,47 @@ static void tsnep_tx_init(struct tsnep_tx *tx)
tx->increment_owner_counter = TSNEP_RING_SIZE - 1;
}

static void tsnep_tx_enable(struct tsnep_tx *tx)
{
struct netdev_queue *nq;

nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);

__netif_tx_lock_bh(nq);
netif_tx_wake_queue(nq);
__netif_tx_unlock_bh(nq);
}

static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi)
{
struct netdev_queue *nq;
u32 val;

nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);

__netif_tx_lock_bh(nq);
netif_tx_stop_queue(nq);
__netif_tx_unlock_bh(nq);

/* wait until TX is done in hardware */
readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val,
((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000,
1000000);

/* wait until TX is also done in software */
while (READ_ONCE(tx->read) != tx->write) {
napi_schedule(napi);
napi_synchronize(napi);
}
}

static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length,
bool last)
{
struct tsnep_tx_entry *entry = &tx->entry[index];

entry->properties = 0;
/* xdpf is union with skb */
/* xdpf and zc are union with skb */
if (entry->skb) {
entry->properties = length & TSNEP_DESC_LENGTH_MASK;
entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
Expand Down Expand Up @@ -646,10 +682,69 @@ static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter,
return xmit;
}

static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx)
{
struct tsnep_tx_entry *entry;
dma_addr_t dma;

entry = &tx->entry[tx->write];
entry->zc = true;

dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr);
xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len);

entry->type = TSNEP_TX_TYPE_XSK;
entry->len = xdpd->len;

entry->desc->tx = __cpu_to_le64(dma);

return xdpd->len;
}

static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd,
struct tsnep_tx *tx)
{
int length;

length = tsnep_xdp_tx_map_zc(xdpd, tx);

tsnep_tx_activate(tx, tx->write, length, true);
tx->write = (tx->write + 1) & TSNEP_RING_MASK;
}

static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx)
{
int desc_available = tsnep_tx_desc_available(tx);
struct xdp_desc *descs = tx->xsk_pool->tx_descs;
int batch, i;

/* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
* will be available for normal TX path and queue is stopped there if
* necessary
*/
if (desc_available <= (MAX_SKB_FRAGS + 1))
return;
desc_available -= MAX_SKB_FRAGS + 1;

batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available);
for (i = 0; i < batch; i++)
tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx);

if (batch) {
/* descriptor properties shall be valid before hardware is
* notified
*/
dma_wmb();

tsnep_xdp_xmit_flush(tx);
}
}

static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
{
struct tsnep_tx_entry *entry;
struct netdev_queue *nq;
int xsk_frames = 0;
int budget = 128;
int length;
int count;
Expand All @@ -676,7 +771,7 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
if ((entry->type & TSNEP_TX_TYPE_SKB) &&
skb_shinfo(entry->skb)->nr_frags > 0)
count += skb_shinfo(entry->skb)->nr_frags;
else if (!(entry->type & TSNEP_TX_TYPE_SKB) &&
else if ((entry->type & TSNEP_TX_TYPE_XDP) &&
xdp_frame_has_frags(entry->xdpf))
count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags;

Expand Down Expand Up @@ -705,9 +800,11 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)

if (entry->type & TSNEP_TX_TYPE_SKB)
napi_consume_skb(entry->skb, napi_budget);
else
else if (entry->type & TSNEP_TX_TYPE_XDP)
xdp_return_frame_rx_napi(entry->xdpf);
/* xdpf is union with skb */
else
xsk_frames++;
/* xdpf and zc are union with skb */
entry->skb = NULL;

tx->read = (tx->read + count) & TSNEP_RING_MASK;
Expand All @@ -718,6 +815,14 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
budget--;
} while (likely(budget));

if (tx->xsk_pool) {
if (xsk_frames)
xsk_tx_completed(tx->xsk_pool, xsk_frames);
if (xsk_uses_need_wakeup(tx->xsk_pool))
xsk_set_tx_need_wakeup(tx->xsk_pool);
tsnep_xdp_xmit_zc(tx);
}

if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) &&
netif_tx_queue_stopped(nq)) {
netif_tx_wake_queue(nq);
Expand Down Expand Up @@ -765,12 +870,6 @@ static int tsnep_tx_open(struct tsnep_tx *tx)

static void tsnep_tx_close(struct tsnep_tx *tx)
{
u32 val;

readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val,
((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000,
1000000);

tsnep_tx_ring_cleanup(tx);
}

Expand Down Expand Up @@ -1786,12 +1885,18 @@ static void tsnep_queue_enable(struct tsnep_queue *queue)
napi_enable(&queue->napi);
tsnep_enable_irq(queue->adapter, queue->irq_mask);

if (queue->tx)
tsnep_tx_enable(queue->tx);

if (queue->rx)
tsnep_rx_enable(queue->rx);
}

static void tsnep_queue_disable(struct tsnep_queue *queue)
{
if (queue->tx)
tsnep_tx_disable(queue->tx, &queue->napi);

napi_disable(&queue->napi);
tsnep_disable_irq(queue->adapter, queue->irq_mask);

Expand Down Expand Up @@ -1908,6 +2013,7 @@ int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool)
if (running)
tsnep_queue_disable(queue);

queue->tx->xsk_pool = pool;
queue->rx->xsk_pool = pool;

if (running) {
Expand All @@ -1928,6 +2034,7 @@ void tsnep_disable_xsk(struct tsnep_queue *queue)
tsnep_rx_free_zc(queue->rx);

queue->rx->xsk_pool = NULL;
queue->tx->xsk_pool = NULL;

if (running) {
tsnep_rx_reopen(queue->rx);
Expand Down Expand Up @@ -2438,7 +2545,8 @@ static int tsnep_probe(struct platform_device *pdev)

netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
NETDEV_XDP_ACT_NDO_XMIT |
NETDEV_XDP_ACT_NDO_XMIT_SG;
NETDEV_XDP_ACT_NDO_XMIT_SG |
NETDEV_XDP_ACT_XSK_ZEROCOPY;

/* carrier off reporting is important to ethtool even BEFORE open */
netif_carrier_off(netdev);
Expand Down

0 comments on commit cd275c2

Please sign in to comment.