Skip to content

Commit

Permalink
gve: RX path for DQO-QPL
Browse files Browse the repository at this point in the history 
The RX path allocates the QPL page pool at queue creation, and
tries to reuse these pages through page recycling. This patch
ensures that on refill no non-QPL pages are posted to the device.

When the driver is running low on free buffers, an ondemand
allocation step kicks in that allocates a non-qpl page for
SKB business to free up the QPL page in use.

gve_try_recycle_buf was moved to gve_rx_append_frags so that driver does
not attempt to mark buffer as used if a non-qpl page was allocated
ondemand.

Signed-off-by: Rushil Gupta <rushilg@google.com>
Reviewed-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: Praveen Kaligineedi <pkaligineedi@google.com>
Signed-off-by: Bailey Forrest <bcf@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Rushil Gupta authored and David S. Miller committed Aug 6, 2023
1 parent a6fb8d5 commit e7075ab
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 18 deletions.
6 changes: 6 additions & 0 deletions drivers/net/ethernet/google/gve/gve.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,12 @@ struct gve_rx_ring {

/* qpl assigned to this queue */
struct gve_queue_page_list *qpl;

/* index into queue page list */
u32 next_qpl_page_idx;

/* track number of used buffers */
u16 used_buf_states_cnt;
} dqo;
};

Expand Down
126 changes: 108 additions & 18 deletions drivers/net/ethernet/google/gve/gve_rx_dqo.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,13 @@ static int gve_buf_ref_cnt(struct gve_rx_buf_state_dqo *bs)
}

static void gve_free_page_dqo(struct gve_priv *priv,
struct gve_rx_buf_state_dqo *bs)
struct gve_rx_buf_state_dqo *bs,
bool free_page)
{
page_ref_sub(bs->page_info.page, bs->page_info.pagecnt_bias - 1);
gve_free_page(&priv->pdev->dev, bs->page_info.page, bs->addr,
DMA_FROM_DEVICE);
if (free_page)
gve_free_page(&priv->pdev->dev, bs->page_info.page, bs->addr,
DMA_FROM_DEVICE);
bs->page_info.page = NULL;
}

Expand Down Expand Up @@ -130,12 +132,20 @@ gve_get_recycled_buf_state(struct gve_rx_ring *rx)
*/
for (i = 0; i < 5; i++) {
buf_state = gve_dequeue_buf_state(rx, &rx->dqo.used_buf_states);
if (gve_buf_ref_cnt(buf_state) == 0)
if (gve_buf_ref_cnt(buf_state) == 0) {
rx->dqo.used_buf_states_cnt--;
return buf_state;
}

gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
}

/* For QPL, we cannot allocate any new buffers and must
* wait for the existing ones to be available.
*/
if (rx->dqo.qpl)
return NULL;

/* If there are no free buf states discard an entry from
* `used_buf_states` so it can be used.
*/
Expand All @@ -144,23 +154,39 @@ gve_get_recycled_buf_state(struct gve_rx_ring *rx)
if (gve_buf_ref_cnt(buf_state) == 0)
return buf_state;

gve_free_page_dqo(rx->gve, buf_state);
gve_free_page_dqo(rx->gve, buf_state, true);
gve_free_buf_state(rx, buf_state);
}

return NULL;
}

static int gve_alloc_page_dqo(struct gve_priv *priv,
static int gve_alloc_page_dqo(struct gve_rx_ring *rx,
struct gve_rx_buf_state_dqo *buf_state)
{
int err;
struct gve_priv *priv = rx->gve;
u32 idx;

err = gve_alloc_page(priv, &priv->pdev->dev, &buf_state->page_info.page,
&buf_state->addr, DMA_FROM_DEVICE, GFP_ATOMIC);
if (err)
return err;
if (!rx->dqo.qpl) {
int err;

err = gve_alloc_page(priv, &priv->pdev->dev,
&buf_state->page_info.page,
&buf_state->addr,
DMA_FROM_DEVICE, GFP_ATOMIC);
if (err)
return err;
} else {
idx = rx->dqo.next_qpl_page_idx;
if (idx >= priv->rx_pages_per_qpl) {
net_err_ratelimited("%s: Out of QPL pages\n",
priv->dev->name);
return -ENOMEM;
}
buf_state->page_info.page = rx->dqo.qpl->pages[idx];
buf_state->addr = rx->dqo.qpl->page_buses[idx];
rx->dqo.next_qpl_page_idx++;
}
buf_state->page_info.page_offset = 0;
buf_state->page_info.page_address =
page_address(buf_state->page_info.page);
Expand Down Expand Up @@ -195,9 +221,13 @@ static void gve_rx_free_ring_dqo(struct gve_priv *priv, int idx)

for (i = 0; i < rx->dqo.num_buf_states; i++) {
struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i];

/* Only free page for RDA. QPL pages are freed in gve_main. */
if (bs->page_info.page)
gve_free_page_dqo(priv, bs);
gve_free_page_dqo(priv, bs, !rx->dqo.qpl);
}
if (rx->dqo.qpl) {
gve_unassign_qpl(priv, rx->dqo.qpl->id);
rx->dqo.qpl = NULL;
}

if (rx->dqo.bufq.desc_ring) {
Expand Down Expand Up @@ -229,7 +259,8 @@ static int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int idx)
int i;

const u32 buffer_queue_slots =
priv->options_dqo_rda.rx_buff_ring_entries;
priv->queue_format == GVE_DQO_RDA_FORMAT ?
priv->options_dqo_rda.rx_buff_ring_entries : priv->rx_desc_cnt;
const u32 completion_queue_slots = priv->rx_desc_cnt;

netif_dbg(priv, drv, priv->dev, "allocating rx ring DQO\n");
Expand All @@ -243,7 +274,9 @@ static int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int idx)
rx->ctx.skb_head = NULL;
rx->ctx.skb_tail = NULL;

rx->dqo.num_buf_states = min_t(s16, S16_MAX, buffer_queue_slots * 4);
rx->dqo.num_buf_states = priv->queue_format == GVE_DQO_RDA_FORMAT ?
min_t(s16, S16_MAX, buffer_queue_slots * 4) :
priv->rx_pages_per_qpl;
rx->dqo.buf_states = kvcalloc(rx->dqo.num_buf_states,
sizeof(rx->dqo.buf_states[0]),
GFP_KERNEL);
Expand Down Expand Up @@ -275,6 +308,13 @@ static int gve_rx_alloc_ring_dqo(struct gve_priv *priv, int idx)
if (!rx->dqo.bufq.desc_ring)
goto err;

if (priv->queue_format != GVE_DQO_RDA_FORMAT) {
rx->dqo.qpl = gve_assign_rx_qpl(priv, rx->q_num);
if (!rx->dqo.qpl)
goto err;
rx->dqo.next_qpl_page_idx = 0;
}

rx->q_resources = dma_alloc_coherent(hdev, sizeof(*rx->q_resources),
&rx->q_resources_bus, GFP_KERNEL);
if (!rx->q_resources)
Expand Down Expand Up @@ -352,7 +392,7 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx)
if (unlikely(!buf_state))
break;

if (unlikely(gve_alloc_page_dqo(priv, buf_state))) {
if (unlikely(gve_alloc_page_dqo(rx, buf_state))) {
u64_stats_update_begin(&rx->statss);
rx->rx_buf_alloc_fail++;
u64_stats_update_end(&rx->statss);
Expand Down Expand Up @@ -415,6 +455,7 @@ static void gve_try_recycle_buf(struct gve_priv *priv, struct gve_rx_ring *rx,

mark_used:
gve_enqueue_buf_state(rx, &rx->dqo.used_buf_states, buf_state);
rx->dqo.used_buf_states_cnt++;
}

static void gve_rx_skb_csum(struct sk_buff *skb,
Expand Down Expand Up @@ -475,6 +516,43 @@ static void gve_rx_free_skb(struct gve_rx_ring *rx)
rx->ctx.skb_tail = NULL;
}

static bool gve_rx_should_trigger_copy_ondemand(struct gve_rx_ring *rx)
{
if (!rx->dqo.qpl)
return false;
if (rx->dqo.used_buf_states_cnt <
(rx->dqo.num_buf_states -
GVE_DQO_QPL_ONDEMAND_ALLOC_THRESHOLD))
return false;
return true;
}

static int gve_rx_copy_ondemand(struct gve_rx_ring *rx,
struct gve_rx_buf_state_dqo *buf_state,
u16 buf_len)
{
struct page *page = alloc_page(GFP_ATOMIC);
int num_frags;

if (!page)
return -ENOMEM;

memcpy(page_address(page),
buf_state->page_info.page_address +
buf_state->page_info.page_offset,
buf_len);
num_frags = skb_shinfo(rx->ctx.skb_tail)->nr_frags;
skb_add_rx_frag(rx->ctx.skb_tail, num_frags, page,
0, buf_len, PAGE_SIZE);

u64_stats_update_begin(&rx->statss);
rx->rx_frag_alloc_cnt++;
u64_stats_update_end(&rx->statss);
/* Return unused buffer. */
gve_enqueue_buf_state(rx, &rx->dqo.recycled_buf_states, buf_state);
return 0;
}

/* Chains multi skbs for single rx packet.
* Returns 0 if buffer is appended, -1 otherwise.
*/
Expand Down Expand Up @@ -502,12 +580,20 @@ static int gve_rx_append_frags(struct napi_struct *napi,
rx->ctx.skb_head->truesize += priv->data_buffer_size_dqo;
}

/* Trigger ondemand page allocation if we are running low on buffers */
if (gve_rx_should_trigger_copy_ondemand(rx))
return gve_rx_copy_ondemand(rx, buf_state, buf_len);

skb_add_rx_frag(rx->ctx.skb_tail, num_frags,
buf_state->page_info.page,
buf_state->page_info.page_offset,
buf_len, priv->data_buffer_size_dqo);
gve_dec_pagecnt_bias(&buf_state->page_info);

/* Advances buffer page-offset if page is partially used.
* Marks buffer as used if page is full.
*/
gve_try_recycle_buf(priv, rx, buf_state);
return 0;
}

Expand Down Expand Up @@ -561,8 +647,6 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
priv)) != 0) {
goto error;
}

gve_try_recycle_buf(priv, rx, buf_state);
return 0;
}

Expand All @@ -588,6 +672,12 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
goto error;
rx->ctx.skb_tail = rx->ctx.skb_head;

if (gve_rx_should_trigger_copy_ondemand(rx)) {
if (gve_rx_copy_ondemand(rx, buf_state, buf_len) < 0)
goto error;
return 0;
}

skb_add_rx_frag(rx->ctx.skb_head, 0, buf_state->page_info.page,
buf_state->page_info.page_offset, buf_len,
priv->data_buffer_size_dqo);
Expand Down

0 comments on commit e7075ab

Please sign in to comment.