Skip to content

Commit

Permalink
be2net: add support for ndo_busy_poll
Browse files Browse the repository at this point in the history 
Includes:
- ndo_busy_poll implementation
- Locking between napi and busy_poll
- Fix rx_post_starvation (replenish rx-queues in out-of-mememory scenario)
  logic to accomodate busy_poll.

v2 changes:
[Eric D.'s comment] call alloc_pages() with GFP_ATOMIC even in ndo_busy_poll
context as it is not allowed to sleep.

Signed-off-by: Sathya Perla <sathya.perla@emulex.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Sathya Perla authored and David S. Miller committed Oct 27, 2013
1 parent 4d961a1 commit 6384a4d
Show file tree
Hide file tree
Showing 2 changed files with 187 additions and 18 deletions.
122 changes: 122 additions & 0 deletions drivers/net/ethernet/emulex/benet/be.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,19 @@ struct be_eq_obj {
u16 spurious_intr;
struct napi_struct napi;
struct be_adapter *adapter;

#ifdef CONFIG_NET_RX_BUSY_POLL
#define BE_EQ_IDLE 0
#define BE_EQ_NAPI 1 /* napi owns this EQ */
#define BE_EQ_POLL 2 /* poll owns this EQ */
#define BE_EQ_LOCKED (BE_EQ_NAPI | BE_EQ_POLL)
#define BE_EQ_NAPI_YIELD 4 /* napi yielded this EQ */
#define BE_EQ_POLL_YIELD 8 /* poll yielded this EQ */
#define BE_EQ_YIELD (BE_EQ_NAPI_YIELD | BE_EQ_POLL_YIELD)
#define BE_EQ_USER_PEND (BE_EQ_POLL | BE_EQ_POLL_YIELD)
unsigned int state;
spinlock_t lock; /* lock to serialize napi and busy-poll */
#endif /* CONFIG_NET_RX_BUSY_POLL */
} ____cacheline_aligned_in_smp;

struct be_aic_obj { /* Adaptive interrupt coalescing (AIC) info */
Expand All @@ -212,6 +225,11 @@ struct be_aic_obj { /* Adaptive interrupt coalescing (AIC) info */
u64 tx_reqs_prev; /* Used to calculate TX pps */
};

enum {
NAPI_POLLING,
BUSY_POLLING
};

struct be_mcc_obj {
struct be_queue_info q;
struct be_queue_info cq;
Expand Down Expand Up @@ -561,6 +579,10 @@ extern const struct ethtool_ops be_ethtool_ops;
for (i = 0, eqo = &adapter->eq_obj[i]; i < adapter->num_evt_qs; \
i++, eqo++)

#define for_all_rx_queues_on_eq(adapter, eqo, rxo, i) \
for (i = eqo->idx, rxo = &adapter->rx_obj[i]; i < adapter->num_rx_qs;\
i += adapter->num_evt_qs, rxo += adapter->num_evt_qs)

#define is_mcc_eqo(eqo) (eqo->idx == 0)
#define mcc_eqo(adapter) (&adapter->eq_obj[0])

Expand Down Expand Up @@ -711,6 +733,106 @@ static inline int qnq_async_evt_rcvd(struct be_adapter *adapter)
return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD;
}

#ifdef CONFIG_NET_RX_BUSY_POLL
static inline bool be_lock_napi(struct be_eq_obj *eqo)
{
bool status = true;

spin_lock(&eqo->lock); /* BH is already disabled */
if (eqo->state & BE_EQ_LOCKED) {
WARN_ON(eqo->state & BE_EQ_NAPI);
eqo->state |= BE_EQ_NAPI_YIELD;
status = false;
} else {
eqo->state = BE_EQ_NAPI;
}
spin_unlock(&eqo->lock);
return status;
}

static inline void be_unlock_napi(struct be_eq_obj *eqo)
{
spin_lock(&eqo->lock); /* BH is already disabled */

WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD));
eqo->state = BE_EQ_IDLE;

spin_unlock(&eqo->lock);
}

static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
{
bool status = true;

spin_lock_bh(&eqo->lock);
if (eqo->state & BE_EQ_LOCKED) {
eqo->state |= BE_EQ_POLL_YIELD;
status = false;
} else {
eqo->state |= BE_EQ_POLL;
}
spin_unlock_bh(&eqo->lock);
return status;
}

static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
{
spin_lock_bh(&eqo->lock);

WARN_ON(eqo->state & (BE_EQ_NAPI));
eqo->state = BE_EQ_IDLE;

spin_unlock_bh(&eqo->lock);
}

static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
{
spin_lock_init(&eqo->lock);
eqo->state = BE_EQ_IDLE;
}

static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
{
local_bh_disable();

/* It's enough to just acquire napi lock on the eqo to stop
* be_busy_poll() from processing any queueus.
*/
while (!be_lock_napi(eqo))
mdelay(1);

local_bh_enable();
}

#else /* CONFIG_NET_RX_BUSY_POLL */

static inline bool be_lock_napi(struct be_eq_obj *eqo)
{
return true;
}

static inline void be_unlock_napi(struct be_eq_obj *eqo)
{
}

static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
{
return false;
}

static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
{
}

static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
{
}

static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
{
}
#endif /* CONFIG_NET_RX_BUSY_POLL */

void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm,
u16 num_popped);
void be_link_status_update(struct be_adapter *adapter, u8 link_status);
Expand Down
83 changes: 65 additions & 18 deletions drivers/net/ethernet/emulex/benet/be_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <asm/div64.h>
#include <linux/aer.h>
#include <linux/if_bridge.h>
#include <net/busy_poll.h>

MODULE_VERSION(DRV_VER);
MODULE_DEVICE_TABLE(pci, be_dev_ids);
Expand Down Expand Up @@ -1556,7 +1557,7 @@ static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
}

/* Process the RX completion indicated by rxcp when GRO is disabled */
static void be_rx_compl_process(struct be_rx_obj *rxo,
static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
struct be_rx_compl_info *rxcp)
{
struct be_adapter *adapter = rxo->adapter;
Expand All @@ -1581,7 +1582,7 @@ static void be_rx_compl_process(struct be_rx_obj *rxo,
skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
if (netdev->features & NETIF_F_RXHASH)
skb->rxhash = rxcp->rss_hash;

skb_mark_napi_id(skb, napi);

if (rxcp->vlanf)
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
Expand Down Expand Up @@ -1639,6 +1640,7 @@ static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
if (adapter->netdev->features & NETIF_F_RXHASH)
skb->rxhash = rxcp->rss_hash;
skb_mark_napi_id(skb, napi);

if (rxcp->vlanf)
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
Expand Down Expand Up @@ -1819,6 +1821,8 @@ static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp)

if (posted) {
atomic_add(posted, &rxq->used);
if (rxo->rx_post_starved)
rxo->rx_post_starved = false;
be_rxq_notify(adapter, rxq->id, posted);
} else if (atomic_read(&rxq->used) == 0) {
/* Let be_worker replenish when memory is available */
Expand Down Expand Up @@ -2021,6 +2025,7 @@ static void be_evt_queues_destroy(struct be_adapter *adapter)
if (eqo->q.created) {
be_eq_clean(eqo);
be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
napi_hash_del(&eqo->napi);
netif_napi_del(&eqo->napi);
}
be_queue_free(adapter, &eqo->q);
Expand All @@ -2040,6 +2045,7 @@ static int be_evt_queues_create(struct be_adapter *adapter)
for_all_evt_queues(adapter, eqo, i) {
netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
BE_NAPI_WEIGHT);
napi_hash_add(&eqo->napi);
aic = &adapter->aic_obj[i];
eqo->adapter = adapter;
eqo->tx_budget = BE_TX_BUDGET;
Expand Down Expand Up @@ -2262,7 +2268,7 @@ static inline bool do_gro(struct be_rx_compl_info *rxcp)
}

static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
int budget)
int budget, int polling)
{
struct be_adapter *adapter = rxo->adapter;
struct be_queue_info *rx_cq = &rxo->cq;
Expand Down Expand Up @@ -2293,18 +2299,24 @@ static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
goto loop_continue;
}

if (do_gro(rxcp))
/* Don't do gro when we're busy_polling */
if (do_gro(rxcp) && polling != BUSY_POLLING)
be_rx_compl_process_gro(rxo, napi, rxcp);
else
be_rx_compl_process(rxo, rxcp);
be_rx_compl_process(rxo, napi, rxcp);

loop_continue:
be_rx_stats_update(rxo, rxcp);
}

if (work_done) {
be_cq_notify(adapter, rx_cq->id, true, work_done);

if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM)
/* When an rx-obj gets into post_starved state, just
* let be_worker do the posting.
*/
if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
!rxo->rx_post_starved)
be_post_rx_frags(rxo, GFP_ATOMIC);
}

Expand Down Expand Up @@ -2349,6 +2361,7 @@ int be_poll(struct napi_struct *napi, int budget)
struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
struct be_adapter *adapter = eqo->adapter;
int max_work = 0, work, i, num_evts;
struct be_rx_obj *rxo;
bool tx_done;

num_evts = events_get(eqo);
Expand All @@ -2361,13 +2374,18 @@ int be_poll(struct napi_struct *napi, int budget)
max_work = budget;
}

/* This loop will iterate twice for EQ0 in which
* completions of the last RXQ (default one) are also processed
* For other EQs the loop iterates only once
*/
for (i = eqo->idx; i < adapter->num_rx_qs; i += adapter->num_evt_qs) {
work = be_process_rx(&adapter->rx_obj[i], napi, budget);
max_work = max(work, max_work);
if (be_lock_napi(eqo)) {
/* This loop will iterate twice for EQ0 in which
* completions of the last RXQ (default one) are also processed
* For other EQs the loop iterates only once
*/
for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
max_work = max(work, max_work);
}
be_unlock_napi(eqo);
} else {
max_work = budget;
}

if (is_mcc_eqo(eqo))
Expand All @@ -2383,6 +2401,28 @@ int be_poll(struct napi_struct *napi, int budget)
return max_work;
}

#ifdef CONFIG_NET_RX_BUSY_POLL
static int be_busy_poll(struct napi_struct *napi)
{
struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
struct be_adapter *adapter = eqo->adapter;
struct be_rx_obj *rxo;
int i, work = 0;

if (!be_lock_busy_poll(eqo))
return LL_FLUSH_BUSY;

for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
if (work)
break;
}

be_unlock_busy_poll(eqo);
return work;
}
#endif

void be_detect_error(struct be_adapter *adapter)
{
u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
Expand Down Expand Up @@ -2614,9 +2654,11 @@ static int be_close(struct net_device *netdev)

be_roce_dev_close(adapter);

if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
for_all_evt_queues(adapter, eqo, i)
for_all_evt_queues(adapter, eqo, i) {
if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
napi_disable(&eqo->napi);
be_disable_busy_poll(eqo);
}
adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
}

Expand Down Expand Up @@ -2727,6 +2769,7 @@ static int be_open(struct net_device *netdev)

for_all_evt_queues(adapter, eqo, i) {
napi_enable(&eqo->napi);
be_enable_busy_poll(eqo);
be_eq_notify(adapter, eqo->q.id, true, false, 0);
}
adapter->flags |= BE_FLAGS_NAPI_ENABLED;
Expand Down Expand Up @@ -3989,6 +4032,9 @@ static const struct net_device_ops be_netdev_ops = {
#endif
.ndo_bridge_setlink = be_ndo_bridge_setlink,
.ndo_bridge_getlink = be_ndo_bridge_getlink,
#ifdef CONFIG_NET_RX_BUSY_POLL
.ndo_busy_poll = be_busy_poll
#endif
};

static void be_netdev_init(struct net_device *netdev)
Expand Down Expand Up @@ -4376,10 +4422,11 @@ static void be_worker(struct work_struct *work)
be_cmd_get_die_temperature(adapter);

for_all_rx_queues(adapter, rxo, i) {
if (rxo->rx_post_starved) {
rxo->rx_post_starved = false;
/* Replenish RX-queues starved due to memory
* allocation failures.
*/
if (rxo->rx_post_starved)
be_post_rx_frags(rxo, GFP_KERNEL);
}
}

be_eqd_update(adapter);
Expand Down

0 comments on commit 6384a4d

Please sign in to comment.