Skip to content

Commit

Permalink
sfc: allocate more RX buffers per page
Browse files Browse the repository at this point in the history 
Allocating 2 buffers per page is insanely inefficient when MTU is 1500
and PAGE_SIZE is 64K (as it usually is on POWER).  Allocate as many as
we can fit, and choose the refill batch size at run-time so that we
still always use a whole page at once.

[bwh: Fix loop condition to allow for compound pages; rebase]
Signed-off-by: Ben Hutchings <bhutchings@solarflare.com>
  • Loading branch information
Daniel Pieczko authored and Ben Hutchings committed Mar 7, 2013
1 parent 179ea7f commit 1648a23
Show file tree
Hide file tree
Showing 4 changed files with 56 additions and 45 deletions.
18 changes: 11 additions & 7 deletions drivers/net/ethernet/sfc/efx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -643,25 +643,29 @@ static void efx_start_datapath(struct efx_nic *efx)
if (rx_buf_len <= PAGE_SIZE) {
efx->rx_scatter = false;
efx->rx_buffer_order = 0;
if (rx_buf_len <= PAGE_SIZE / 2)
efx->rx_buffer_truesize = PAGE_SIZE / 2;
else
efx->rx_buffer_truesize = PAGE_SIZE;
} else if (efx->type->can_rx_scatter) {
BUILD_BUG_ON(sizeof(struct efx_rx_page_state) +
EFX_PAGE_IP_ALIGN + EFX_RX_USR_BUF_SIZE >
PAGE_SIZE / 2);
efx->rx_scatter = true;
efx->rx_dma_len = EFX_RX_USR_BUF_SIZE;
efx->rx_buffer_order = 0;
efx->rx_buffer_truesize = PAGE_SIZE / 2;
} else {
efx->rx_scatter = false;
efx->rx_buffer_order = get_order(rx_buf_len);
efx->rx_buffer_truesize = PAGE_SIZE << efx->rx_buffer_order;
}

efx->rx_bufs_per_page = (rx_buf_len <= PAGE_SIZE / 2) ? 2 : 1;
efx_rx_config_page_split(efx);
if (efx->rx_buffer_order)
netif_dbg(efx, drv, efx->net_dev,
"RX buf len=%u; page order=%u batch=%u\n",
efx->rx_dma_len, efx->rx_buffer_order,
efx->rx_pages_per_batch);
else
netif_dbg(efx, drv, efx->net_dev,
"RX buf len=%u step=%u bpp=%u; page batch=%u\n",
efx->rx_dma_len, efx->rx_page_buf_step,
efx->rx_bufs_per_page, efx->rx_pages_per_batch);

/* RX filters also have scatter-enabled flags */
if (efx->rx_scatter != old_rx_scatter)
Expand Down
1 change: 1 addition & 0 deletions drivers/net/ethernet/sfc/efx.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ extern int efx_setup_tc(struct net_device *net_dev, u8 num_tc);
extern unsigned int efx_tx_max_skb_descs(struct efx_nic *efx);

/* RX */
extern void efx_rx_config_page_split(struct efx_nic *efx);
extern int efx_probe_rx_queue(struct efx_rx_queue *rx_queue);
extern void efx_remove_rx_queue(struct efx_rx_queue *rx_queue);
extern void efx_init_rx_queue(struct efx_rx_queue *rx_queue);
Expand Down
2 changes: 2 additions & 0 deletions drivers/net/ethernet/sfc/net_driver.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -825,7 +825,9 @@ struct efx_nic {
unsigned int rx_dma_len;
unsigned int rx_buffer_order;
unsigned int rx_buffer_truesize;
unsigned int rx_page_buf_step;
unsigned int rx_bufs_per_page;
unsigned int rx_pages_per_batch;
u8 rx_hash_key[40];
u32 rx_indir_table[128];
bool rx_scatter;
Expand Down
80 changes: 42 additions & 38 deletions drivers/net/ethernet/sfc/rx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,15 @@
#include "selftest.h"
#include "workarounds.h"

/* Number of RX descriptors pushed at once. */
#define EFX_RX_BATCH 8
/* Preferred number of descriptors to fill at once */
#define EFX_RX_PREFERRED_BATCH 8U

/* Number of RX buffers to recycle pages for. When creating the RX page recycle
* ring, this number is divided by the number of buffers per page to calculate
* the number of pages to store in the RX page recycle ring.
*/
#define EFX_RECYCLE_RING_SIZE_IOMMU 4096
#define EFX_RECYCLE_RING_SIZE_NOIOMMU (2 * EFX_RX_BATCH)

/* Maximum length for an RX descriptor sharing a page */
#define EFX_RX_HALF_PAGE ((PAGE_SIZE >> 1) - sizeof(struct efx_rx_page_state) \
- EFX_PAGE_IP_ALIGN)
#define EFX_RECYCLE_RING_SIZE_NOIOMMU (2 * EFX_RX_PREFERRED_BATCH)

/* Size of buffer allocated for skb header area. */
#define EFX_SKB_HEADERS 64u
Expand Down Expand Up @@ -95,6 +91,19 @@ static inline void efx_sync_rx_buffer(struct efx_nic *efx,
DMA_FROM_DEVICE);
}

void efx_rx_config_page_split(struct efx_nic *efx)
{
efx->rx_page_buf_step = ALIGN(efx->rx_dma_len + EFX_PAGE_IP_ALIGN,
L1_CACHE_BYTES);
efx->rx_bufs_per_page = efx->rx_buffer_order ? 1 :
((PAGE_SIZE - sizeof(struct efx_rx_page_state)) /
efx->rx_page_buf_step);
efx->rx_buffer_truesize = (PAGE_SIZE << efx->rx_buffer_order) /
efx->rx_bufs_per_page;
efx->rx_pages_per_batch = DIV_ROUND_UP(EFX_RX_PREFERRED_BATCH,
efx->rx_bufs_per_page);
}

/* Check the RX page recycle ring for a page that can be reused. */
static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue)
{
Expand Down Expand Up @@ -134,10 +143,10 @@ static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue)
*
* @rx_queue: Efx RX queue
*
* This allocates memory for EFX_RX_BATCH receive buffers, maps them for DMA,
* and populates struct efx_rx_buffers for each one. Return a negative error
* code or 0 on success. If a single page can be split between two buffers,
* then the page will either be inserted fully, or not at at all.
* This allocates a batch of pages, maps them for DMA, and populates
* struct efx_rx_buffers for each one. Return a negative error code or
* 0 on success. If a single page can be used for multiple buffers,
* then the page will either be inserted fully, or not at all.
*/
static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue)
{
Expand All @@ -149,10 +158,8 @@ static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue)
dma_addr_t dma_addr;
unsigned index, count;

/* We can split a page between two buffers */
BUILD_BUG_ON(EFX_RX_BATCH & 1);

for (count = 0; count < EFX_RX_BATCH; ++count) {
count = 0;
do {
page = efx_reuse_page(rx_queue);
if (page == NULL) {
page = alloc_pages(__GFP_COLD | __GFP_COMP | GFP_ATOMIC,
Expand All @@ -174,32 +181,26 @@ static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue)
state = page_address(page);
dma_addr = state->dma_addr;
}
get_page(page);

dma_addr += sizeof(struct efx_rx_page_state);
page_offset = sizeof(struct efx_rx_page_state);

split:
index = rx_queue->added_count & rx_queue->ptr_mask;
rx_buf = efx_rx_buffer(rx_queue, index);
rx_buf->dma_addr = dma_addr + EFX_PAGE_IP_ALIGN;
rx_buf->page = page;
rx_buf->page_offset = page_offset + EFX_PAGE_IP_ALIGN;
rx_buf->len = efx->rx_dma_len;
++rx_queue->added_count;

if ((~count & 1) && (efx->rx_dma_len <= EFX_RX_HALF_PAGE)) {
/* Use the second half of the page */
get_page(page);
do {
index = rx_queue->added_count & rx_queue->ptr_mask;
rx_buf = efx_rx_buffer(rx_queue, index);
rx_buf->dma_addr = dma_addr + EFX_PAGE_IP_ALIGN;
rx_buf->page = page;
rx_buf->page_offset = page_offset + EFX_PAGE_IP_ALIGN;
rx_buf->len = efx->rx_dma_len;
rx_buf->flags = 0;
dma_addr += (PAGE_SIZE >> 1);
page_offset += (PAGE_SIZE >> 1);
++count;
goto split;
}
++rx_queue->added_count;
get_page(page);
dma_addr += efx->rx_page_buf_step;
page_offset += efx->rx_page_buf_step;
} while (page_offset + efx->rx_page_buf_step <= PAGE_SIZE);

rx_buf->flags = EFX_RX_BUF_LAST_IN_PAGE;
}
} while (++count < efx->rx_pages_per_batch);

return 0;
}
Expand Down Expand Up @@ -307,7 +308,8 @@ static void efx_recycle_rx_buffers(struct efx_channel *channel,
*/
void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue)
{
unsigned fill_level;
struct efx_nic *efx = rx_queue->efx;
unsigned int fill_level, batch_size;
int space, rc = 0;

/* Calculate current fill level, and exit if we don't need to fill */
Expand All @@ -322,8 +324,9 @@ void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue)
rx_queue->min_fill = fill_level;
}

batch_size = efx->rx_pages_per_batch * efx->rx_bufs_per_page;
space = rx_queue->max_fill - fill_level;
EFX_BUG_ON_PARANOID(space < EFX_RX_BATCH);
EFX_BUG_ON_PARANOID(space < batch_size);

netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
"RX queue %d fast-filling descriptor ring from"
Expand All @@ -340,7 +343,7 @@ void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue)
efx_schedule_slow_fill(rx_queue);
goto out;
}
} while ((space -= EFX_RX_BATCH) >= EFX_RX_BATCH);
} while ((space -= batch_size) >= batch_size);

netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
"RX queue %d fast-filled descriptor ring "
Expand Down Expand Up @@ -708,7 +711,8 @@ void efx_init_rx_queue(struct efx_rx_queue *rx_queue)

/* Initialise limit fields */
max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM;
max_trigger = max_fill - EFX_RX_BATCH;
max_trigger =
max_fill - efx->rx_pages_per_batch * efx->rx_bufs_per_page;
if (rx_refill_threshold != 0) {
trigger = max_fill * min(rx_refill_threshold, 100U) / 100U;
if (trigger > max_trigger)
Expand Down

0 comments on commit 1648a23

Please sign in to comment.