Skip to content

Commit

Permalink
i40e/i40evf: Break i40e_fetch_rx_buffer up to allow for reuse of frag…
Browse files Browse the repository at this point in the history 
… code

This patch is meant to clean up the code in preparation for us adding
support for build_skb.  Specifically we deconstruct i40e_fetch_buffer into
several functions so that those functions can later be reused when we add a
path for build_skb.

Specifically with this change we split out the code for adding a page to an
exiting skb.

Change-ID: Iab1efbab6b8b97cb60ab9fdd0be1d37a056a154d
Signed-off-by: Alexander Duyck <alexander.h.duyck@intel.com>
Tested-by: Andrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
  • Loading branch information
Alexander Duyck authored and Jeff Kirsher committed Mar 29, 2017
1 parent a0cfc31 commit fa2343e
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 146 deletions.
138 changes: 65 additions & 73 deletions drivers/net/ethernet/intel/i40e/i40e_txrx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1687,61 +1687,23 @@ static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer)
* @size: packet length from rx_desc
*
* This function will add the data contained in rx_buffer->page to the skb.
* This is done either through a direct copy if the data in the buffer is
* less than the skb header size, otherwise it will just attach the page as
* a frag to the skb.
* It will just attach the page as a frag to the skb.
*
* The function will then update the page offset if necessary and return
* true if the buffer can be reused by the adapter.
* The function will then update the page offset.
**/
static void i40e_add_rx_frag(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *rx_buffer,
struct sk_buff *skb,
unsigned int size)
{
struct page *page = rx_buffer->page;
unsigned char *va = page_address(page) + rx_buffer->page_offset;
#if (PAGE_SIZE < 8192)
unsigned int truesize = I40E_RXBUFFER_2048;
#else
unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
unsigned int truesize = SKB_DATA_ALIGN(size);
#endif
unsigned int pull_len;

if (unlikely(skb_is_nonlinear(skb)))
goto add_tail_frag;

/* will the data fit in the skb we allocated? if so, just
* copy it as it is pretty small anyway
*/
if (size <= I40E_RX_HDR_SIZE) {
memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));

/* page is to be freed, increase pagecnt_bias instead of
* decreasing page count.
*/
rx_buffer->pagecnt_bias++;
return;
}

/* we need the header to contain the greater of either
* ETH_HLEN or 60 bytes if the skb->len is less than
* 60 for skb_pad.
*/
pull_len = eth_get_headlen(va, I40E_RX_HDR_SIZE);

/* align pull length to size of long to optimize
* memcpy performance
*/
memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long)));

/* update all of the pointers */
va += pull_len;
size -= pull_len;

add_tail_frag:
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
(unsigned long)va & ~PAGE_MASK, size, truesize);
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
rx_buffer->page_offset, size, truesize);

/* page is being used so we must update the page offset */
#if (PAGE_SIZE < 8192)
Expand Down Expand Up @@ -1781,45 +1743,66 @@ static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring,
}

/**
* i40e_fetch_rx_buffer - Allocate skb and populate it
* i40e_construct_skb - Allocate skb and populate it
* @rx_ring: rx descriptor ring to transact packets on
* @rx_buffer: rx buffer to pull data from
* @size: size of buffer to add to skb
*
* This function allocates an skb on the fly, and populates it with the page
* data from the current receive descriptor, taking care to set up the skb
* correctly, as well as handling calling the page recycle function if
* necessary.
* This function allocates an skb. It then populates it with the page
* data from the current receive descriptor, taking care to set up the
* skb correctly.
*/
static inline
struct sk_buff *i40e_fetch_rx_buffer(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *rx_buffer,
struct sk_buff *skb,
unsigned int size)
static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *rx_buffer,
unsigned int size)
{
if (likely(!skb)) {
void *page_addr = page_address(rx_buffer->page) +
rx_buffer->page_offset;
void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
#if (PAGE_SIZE < 8192)
unsigned int truesize = I40E_RXBUFFER_2048;
#else
unsigned int truesize = SKB_DATA_ALIGN(size);
#endif
unsigned int headlen;
struct sk_buff *skb;

/* prefetch first cache line of first page */
prefetch(page_addr);
/* prefetch first cache line of first page */
prefetch(va);
#if L1_CACHE_BYTES < 128
prefetch(page_addr + L1_CACHE_BYTES);
prefetch(va + L1_CACHE_BYTES);
#endif

/* allocate a skb to store the frags */
skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
I40E_RX_HDR_SIZE,
GFP_ATOMIC | __GFP_NOWARN);
if (unlikely(!skb)) {
rx_ring->rx_stats.alloc_buff_failed++;
rx_buffer->pagecnt_bias++;
return NULL;
}
}
/* allocate a skb to store the frags */
skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
I40E_RX_HDR_SIZE,
GFP_ATOMIC | __GFP_NOWARN);
if (unlikely(!skb))
return NULL;

/* Determine available headroom for copy */
headlen = size;
if (headlen > I40E_RX_HDR_SIZE)
headlen = eth_get_headlen(va, I40E_RX_HDR_SIZE);

/* pull page into skb */
i40e_add_rx_frag(rx_ring, rx_buffer, skb, size);
/* align pull length to size of long to optimize memcpy performance */
memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));

/* update all of the pointers */
size -= headlen;
if (size) {
skb_add_rx_frag(skb, 0, rx_buffer->page,
rx_buffer->page_offset + headlen,
size, truesize);

/* buffer is used by skb, update page_offset */
#if (PAGE_SIZE < 8192)
rx_buffer->page_offset ^= truesize;
#else
rx_buffer->page_offset += truesize;
#endif
} else {
/* buffer is unused, reset bias back to rx_buffer */
rx_buffer->pagecnt_bias++;
}

return skb;
}
Expand Down Expand Up @@ -1944,9 +1927,18 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)

rx_buffer = i40e_get_rx_buffer(rx_ring, size);

skb = i40e_fetch_rx_buffer(rx_ring, rx_buffer, skb, size);
if (!skb)
/* retrieve a buffer from the ring */
if (skb)
i40e_add_rx_frag(rx_ring, rx_buffer, skb, size);
else
skb = i40e_construct_skb(rx_ring, rx_buffer, size);

/* exit if we failed to retrieve a buffer */
if (!skb) {
rx_ring->rx_stats.alloc_buff_failed++;
rx_buffer->pagecnt_bias++;
break;
}

i40e_put_rx_buffer(rx_ring, rx_buffer);
cleaned_count++;
Expand Down
138 changes: 65 additions & 73 deletions drivers/net/ethernet/intel/i40evf/i40e_txrx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1045,61 +1045,23 @@ static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer)
* @size: packet length from rx_desc
*
* This function will add the data contained in rx_buffer->page to the skb.
* This is done either through a direct copy if the data in the buffer is
* less than the skb header size, otherwise it will just attach the page as
* a frag to the skb.
* It will just attach the page as a frag to the skb.
*
* The function will then update the page offset if necessary and return
* true if the buffer can be reused by the adapter.
* The function will then update the page offset.
**/
static void i40e_add_rx_frag(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *rx_buffer,
struct sk_buff *skb,
unsigned int size)
{
struct page *page = rx_buffer->page;
unsigned char *va = page_address(page) + rx_buffer->page_offset;
#if (PAGE_SIZE < 8192)
unsigned int truesize = I40E_RXBUFFER_2048;
#else
unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
unsigned int truesize = SKB_DATA_ALIGN(size);
#endif
unsigned int pull_len;

if (unlikely(skb_is_nonlinear(skb)))
goto add_tail_frag;

/* will the data fit in the skb we allocated? if so, just
* copy it as it is pretty small anyway
*/
if (size <= I40E_RX_HDR_SIZE) {
memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));

/* page is to be freed, increase pagecnt_bias instead of
* decreasing page count.
*/
rx_buffer->pagecnt_bias++;
return;
}

/* we need the header to contain the greater of either
* ETH_HLEN or 60 bytes if the skb->len is less than
* 60 for skb_pad.
*/
pull_len = eth_get_headlen(va, I40E_RX_HDR_SIZE);

/* align pull length to size of long to optimize
* memcpy performance
*/
memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long)));

/* update all of the pointers */
va += pull_len;
size -= pull_len;

add_tail_frag:
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
(unsigned long)va & ~PAGE_MASK, size, truesize);
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
rx_buffer->page_offset, size, truesize);

/* page is being used so we must update the page offset */
#if (PAGE_SIZE < 8192)
Expand Down Expand Up @@ -1139,45 +1101,66 @@ static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring,
}

/**
* i40evf_fetch_rx_buffer - Allocate skb and populate it
* i40e_construct_skb - Allocate skb and populate it
* @rx_ring: rx descriptor ring to transact packets on
* @rx_buffer: rx buffer to pull data from
* @size: size of buffer to add to skb
*
* This function allocates an skb on the fly, and populates it with the page
* data from the current receive descriptor, taking care to set up the skb
* correctly, as well as handling calling the page recycle function if
* necessary.
* This function allocates an skb. It then populates it with the page
* data from the current receive descriptor, taking care to set up the
* skb correctly.
*/
static inline
struct sk_buff *i40evf_fetch_rx_buffer(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *rx_buffer,
struct sk_buff *skb,
unsigned int size)
static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
struct i40e_rx_buffer *rx_buffer,
unsigned int size)
{
if (likely(!skb)) {
void *page_addr = page_address(rx_buffer->page) +
rx_buffer->page_offset;
void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
#if (PAGE_SIZE < 8192)
unsigned int truesize = I40E_RXBUFFER_2048;
#else
unsigned int truesize = SKB_DATA_ALIGN(size);
#endif
unsigned int headlen;
struct sk_buff *skb;

/* prefetch first cache line of first page */
prefetch(page_addr);
/* prefetch first cache line of first page */
prefetch(va);
#if L1_CACHE_BYTES < 128
prefetch(page_addr + L1_CACHE_BYTES);
prefetch(va + L1_CACHE_BYTES);
#endif

/* allocate a skb to store the frags */
skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
I40E_RX_HDR_SIZE,
GFP_ATOMIC | __GFP_NOWARN);
if (unlikely(!skb)) {
rx_ring->rx_stats.alloc_buff_failed++;
rx_buffer->pagecnt_bias++;
return NULL;
}
}
/* allocate a skb to store the frags */
skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
I40E_RX_HDR_SIZE,
GFP_ATOMIC | __GFP_NOWARN);
if (unlikely(!skb))
return NULL;

/* Determine available headroom for copy */
headlen = size;
if (headlen > I40E_RX_HDR_SIZE)
headlen = eth_get_headlen(va, I40E_RX_HDR_SIZE);

/* pull page into skb */
i40e_add_rx_frag(rx_ring, rx_buffer, skb, size);
/* align pull length to size of long to optimize memcpy performance */
memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));

/* update all of the pointers */
size -= headlen;
if (size) {
skb_add_rx_frag(skb, 0, rx_buffer->page,
rx_buffer->page_offset + headlen,
size, truesize);

/* buffer is used by skb, update page_offset */
#if (PAGE_SIZE < 8192)
rx_buffer->page_offset ^= truesize;
#else
rx_buffer->page_offset += truesize;
#endif
} else {
/* buffer is unused, reset bias back to rx_buffer */
rx_buffer->pagecnt_bias++;
}

return skb;
}
Expand Down Expand Up @@ -1297,9 +1280,18 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)

rx_buffer = i40e_get_rx_buffer(rx_ring, size);

skb = i40evf_fetch_rx_buffer(rx_ring, rx_buffer, skb, size);
if (!skb)
/* retrieve a buffer from the ring */
if (skb)
i40e_add_rx_frag(rx_ring, rx_buffer, skb, size);
else
skb = i40e_construct_skb(rx_ring, rx_buffer, size);

/* exit if we failed to retrieve a buffer */
if (!skb) {
rx_ring->rx_stats.alloc_buff_failed++;
rx_buffer->pagecnt_bias++;
break;
}

i40e_put_rx_buffer(rx_ring, rx_buffer);
cleaned_count++;
Expand Down

0 comments on commit fa2343e

Please sign in to comment.