Skip to content

Commit

Permalink
virtio-net: switch to use build_skb() for small buffer
Browse files Browse the repository at this point in the history 
This patch switch to use build_skb() for small buffer which can have
better performance for both TCP and XDP (since we can work at page
before skb creation). It also remove lots of XDP codes since both
mergeable and small buffer use page frag during refill now.

                       Before   | After
XDP_DROP(xdp1) 64B  :  11.1Mpps | 14.4Mpps

Tested with xdp1/xdp2/xdp_ip_tx_tunnel and netperf.

Signed-off-by: Jason Wang <jasowang@redhat.com>
Acked-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Jason Wang authored and David S. Miller committed Feb 21, 2017
1 parent ca4ef45 commit f6b1020
Showing 1 changed file with 63 additions and 75 deletions.
138 changes: 63 additions & 75 deletions drivers/net/virtio_net.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ module_param(gso, bool, 0444);
#define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN)
#define GOOD_COPY_LEN 128

#define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)

/* Amount of XDP headroom to prepend to packets for use by xdp_adjust_head */
#define VIRTIO_XDP_HEADROOM 256

Expand Down Expand Up @@ -343,11 +345,10 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,

static bool virtnet_xdp_xmit(struct virtnet_info *vi,
struct receive_queue *rq,
struct xdp_buff *xdp,
void *data)
struct xdp_buff *xdp)
{
struct virtio_net_hdr_mrg_rxbuf *hdr;
unsigned int num_sg, len;
unsigned int len;
struct send_queue *sq;
unsigned int qp;
void *xdp_sent;
Expand All @@ -358,89 +359,74 @@ static bool virtnet_xdp_xmit(struct virtnet_info *vi,

/* Free up any pending old buffers before queueing new ones. */
while ((xdp_sent = virtqueue_get_buf(sq->vq, &len)) != NULL) {
if (vi->mergeable_rx_bufs) {
struct page *sent_page = virt_to_head_page(xdp_sent);
struct page *sent_page = virt_to_head_page(xdp_sent);

put_page(sent_page);
} else { /* small buffer */
struct sk_buff *skb = xdp_sent;

kfree_skb(skb);
}
put_page(sent_page);
}

if (vi->mergeable_rx_bufs) {
xdp->data -= sizeof(struct virtio_net_hdr_mrg_rxbuf);
/* Zero header and leave csum up to XDP layers */
hdr = xdp->data;
memset(hdr, 0, vi->hdr_len);

num_sg = 1;
sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);
} else { /* small buffer */
struct sk_buff *skb = data;
xdp->data -= vi->hdr_len;
/* Zero header and leave csum up to XDP layers */
hdr = xdp->data;
memset(hdr, 0, vi->hdr_len);

/* Zero header and leave csum up to XDP layers */
hdr = skb_vnet_hdr(skb);
memset(hdr, 0, vi->hdr_len);
sg_init_one(sq->sg, xdp->data, xdp->data_end - xdp->data);

num_sg = 2;
sg_init_table(sq->sg, 2);
sg_set_buf(sq->sg, hdr, vi->hdr_len);
skb_to_sgvec(skb, sq->sg + 1,
xdp->data - xdp->data_hard_start,
xdp->data_end - xdp->data);
}
err = virtqueue_add_outbuf(sq->vq, sq->sg, num_sg,
data, GFP_ATOMIC);
err = virtqueue_add_outbuf(sq->vq, sq->sg, 1, xdp->data, GFP_ATOMIC);
if (unlikely(err)) {
if (vi->mergeable_rx_bufs) {
struct page *page = virt_to_head_page(xdp->data);
struct page *page = virt_to_head_page(xdp->data);

put_page(page);
} else /* small buffer */
kfree_skb(data);
/* On error abort to avoid unnecessary kick */
put_page(page);
return false;
}

virtqueue_kick(sq->vq);
return true;
}

static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
{
return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
}

static struct sk_buff *receive_small(struct net_device *dev,
struct virtnet_info *vi,
struct receive_queue *rq,
void *buf, unsigned int len)
{
struct sk_buff * skb = buf;
struct sk_buff *skb;
struct bpf_prog *xdp_prog;

unsigned int xdp_headroom = virtnet_get_headroom(vi);
unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom;
unsigned int headroom = vi->hdr_len + header_offset;
unsigned int buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
unsigned int delta = 0;
len -= vi->hdr_len;

rcu_read_lock();
xdp_prog = rcu_dereference(rq->xdp_prog);
if (xdp_prog) {
struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset;
struct xdp_buff xdp;
void *orig_data;
u32 act;

if (unlikely(hdr->hdr.gso_type || hdr->hdr.flags))
goto err_xdp;

xdp.data_hard_start = skb->data;
xdp.data = skb->data + VIRTIO_XDP_HEADROOM;
xdp.data_hard_start = buf + VIRTNET_RX_PAD + vi->hdr_len;
xdp.data = xdp.data_hard_start + xdp_headroom;
xdp.data_end = xdp.data + len;
orig_data = xdp.data;
act = bpf_prog_run_xdp(xdp_prog, &xdp);

switch (act) {
case XDP_PASS:
/* Recalculate length in case bpf program changed it */
__skb_pull(skb, xdp.data - xdp.data_hard_start);
len = xdp.data_end - xdp.data;
delta = orig_data - xdp.data;
break;
case XDP_TX:
if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp, skb)))
if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp)))
trace_xdp_exception(vi->dev, xdp_prog, act);
rcu_read_unlock();
goto xdp_xmit;
Expand All @@ -454,13 +440,25 @@ static struct sk_buff *receive_small(struct net_device *dev,
}
rcu_read_unlock();

skb_trim(skb, len);
skb = build_skb(buf, buflen);
if (!skb) {
put_page(virt_to_head_page(buf));
goto err;
}
skb_reserve(skb, headroom - delta);
skb_put(skb, len + delta);
if (!delta) {
buf += header_offset;
memcpy(skb_vnet_hdr(skb), buf, vi->hdr_len);
} /* keep zeroed vnet hdr since packet was changed by bpf */

err:
return skb;

err_xdp:
rcu_read_unlock();
dev->stats.rx_dropped++;
kfree_skb(skb);
put_page(virt_to_head_page(buf));
xdp_xmit:
return NULL;
}
Expand Down Expand Up @@ -621,7 +619,7 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
}
break;
case XDP_TX:
if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp, data)))
if (unlikely(!virtnet_xdp_xmit(vi, rq, &xdp)))
trace_xdp_exception(vi->dev, xdp_prog, act);
ewma_pkt_len_add(&rq->mrg_avg_pkt_len, len);
if (unlikely(xdp_page != page))
Expand Down Expand Up @@ -737,7 +735,7 @@ static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
} else if (vi->big_packets) {
give_pages(rq, buf);
} else {
dev_kfree_skb(buf);
put_page(virt_to_head_page(buf));
}
return 0;
}
Expand Down Expand Up @@ -780,34 +778,28 @@ static int receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
return 0;
}

static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
{
return vi->xdp_queue_pairs ? VIRTIO_XDP_HEADROOM : 0;
}

static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
gfp_t gfp)
{
int headroom = GOOD_PACKET_LEN + virtnet_get_headroom(vi);
struct page_frag *alloc_frag = &rq->alloc_frag;
char *buf;
unsigned int xdp_headroom = virtnet_get_headroom(vi);
struct sk_buff *skb;
struct virtio_net_hdr_mrg_rxbuf *hdr;
int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom;
int err;

skb = __netdev_alloc_skb_ip_align(vi->dev, headroom, gfp);
if (unlikely(!skb))
len = SKB_DATA_ALIGN(len) +
SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
if (unlikely(!skb_page_frag_refill(len, alloc_frag, gfp)))
return -ENOMEM;

skb_put(skb, headroom);

hdr = skb_vnet_hdr(skb);
sg_init_table(rq->sg, 2);
sg_set_buf(rq->sg, hdr, vi->hdr_len);
skb_to_sgvec(skb, rq->sg + 1, xdp_headroom, skb->len - xdp_headroom);

err = virtqueue_add_inbuf(rq->vq, rq->sg, 2, skb, gfp);
buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
get_page(alloc_frag->page);
alloc_frag->offset += len;
sg_init_one(rq->sg, buf + VIRTNET_RX_PAD + xdp_headroom,
vi->hdr_len + GOOD_PACKET_LEN);
err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, buf, gfp);
if (err < 0)
dev_kfree_skb(skb);
put_page(virt_to_head_page(buf));

return err;
}
Expand Down Expand Up @@ -1994,10 +1986,6 @@ static void free_receive_page_frags(struct virtnet_info *vi)

static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
{
/* For small receive mode always use kfree_skb variants */
if (!vi->mergeable_rx_bufs)
return false;

if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
return false;
else if (q < vi->curr_queue_pairs)
Expand Down Expand Up @@ -2032,7 +2020,7 @@ static void free_unused_bufs(struct virtnet_info *vi)
} else if (vi->big_packets) {
give_pages(&vi->rq[i], buf);
} else {
dev_kfree_skb(buf);
put_page(virt_to_head_page(buf));
}
}
}
Expand Down

0 comments on commit f6b1020

Please sign in to comment.