Skip to content

Commit

Permalink
atl1c: move tx cleanup processing out of interrupt
Browse files Browse the repository at this point in the history
Tx queue cleanup happens in interrupt handler on same core as rx queue
processing. Both can take considerable amount of processing in high
packet-per-second scenarios.

Sending big amounts of packets can stall the rx processing which is
unfair and also can lead to out-of-memory condition since
__dev_kfree_skb_irq queues the skbs for later kfree in softirq which
is not allowed to happen with heavy load in interrupt handler.

This puts tx cleanup in its own napi and enables threaded napi to
allow the rx/tx queue processing to happen on different cores.

The ability to sustain equal amounts of tx/rx traffic increased:
from 280Kpps to 1130Kpps on Threadripper 3960X with upcoming
Mikrotik 10/25G NIC,
from 520Kpps to 850Kpps on Intel i3-3320 with Mikrotik RB44Ge adapter.

Signed-off-by: Gatis Peisenieks <gatis@mikrotik.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Gatis Peisenieks authored and David S. Miller committed Apr 16, 2021
1 parent 2576e5d commit a1150a0
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 10 deletions.
2 changes: 2 additions & 0 deletions drivers/net/ethernet/atheros/atl1c/atl1c.h
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,7 @@ struct atl1c_hw {
u16 phy_id1;
u16 phy_id2;

spinlock_t intr_mask_lock; /* protect the intr_mask */
u32 intr_mask;

u8 preamble_len;
Expand Down Expand Up @@ -506,6 +507,7 @@ struct atl1c_adapter {
struct net_device *netdev;
struct pci_dev *pdev;
struct napi_struct napi;
struct napi_struct tx_napi;
struct page *rx_page;
unsigned int rx_page_offset;
unsigned int rx_frag_size;
Expand Down
44 changes: 34 additions & 10 deletions drivers/net/ethernet/atheros/atl1c/atl1c_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -813,6 +813,7 @@ static int atl1c_sw_init(struct atl1c_adapter *adapter)
atl1c_set_rxbufsize(adapter, adapter->netdev);
atomic_set(&adapter->irq_sem, 1);
spin_lock_init(&adapter->mdio_lock);
spin_lock_init(&adapter->hw.intr_mask_lock);
set_bit(__AT_DOWN, &adapter->flags);

return 0;
Expand Down Expand Up @@ -1530,20 +1531,19 @@ static inline void atl1c_clear_phy_int(struct atl1c_adapter *adapter)
spin_unlock(&adapter->mdio_lock);
}

static bool atl1c_clean_tx_irq(struct atl1c_adapter *adapter,
enum atl1c_trans_queue type)
static int atl1c_clean_tx(struct napi_struct *napi, int budget)
{
struct atl1c_tpd_ring *tpd_ring = &adapter->tpd_ring[type];
struct atl1c_adapter *adapter =
container_of(napi, struct atl1c_adapter, tx_napi);
struct atl1c_tpd_ring *tpd_ring = &adapter->tpd_ring[atl1c_trans_normal];
struct atl1c_buffer *buffer_info;
struct pci_dev *pdev = adapter->pdev;
u16 next_to_clean = atomic_read(&tpd_ring->next_to_clean);
u16 hw_next_to_clean;
u16 reg;
unsigned int total_bytes = 0, total_packets = 0;
unsigned long flags;

reg = type == atl1c_trans_high ? REG_TPD_PRI1_CIDX : REG_TPD_PRI0_CIDX;

AT_READ_REGW(&adapter->hw, reg, &hw_next_to_clean);
AT_READ_REGW(&adapter->hw, REG_TPD_PRI0_CIDX, &hw_next_to_clean);

while (next_to_clean != hw_next_to_clean) {
buffer_info = &tpd_ring->buffer_info[next_to_clean];
Expand All @@ -1564,7 +1564,15 @@ static bool atl1c_clean_tx_irq(struct atl1c_adapter *adapter,
netif_wake_queue(adapter->netdev);
}

return true;
if (total_packets < budget) {
napi_complete_done(napi, total_packets);
spin_lock_irqsave(&adapter->hw.intr_mask_lock, flags);
adapter->hw.intr_mask |= ISR_TX_PKT;
AT_WRITE_REG(&adapter->hw, REG_IMR, adapter->hw.intr_mask);
spin_unlock_irqrestore(&adapter->hw.intr_mask_lock, flags);
return total_packets;
}
return budget;
}

/**
Expand Down Expand Up @@ -1599,13 +1607,22 @@ static irqreturn_t atl1c_intr(int irq, void *data)
AT_WRITE_REG(hw, REG_ISR, status | ISR_DIS_INT);
if (status & ISR_RX_PKT) {
if (likely(napi_schedule_prep(&adapter->napi))) {
spin_lock(&hw->intr_mask_lock);
hw->intr_mask &= ~ISR_RX_PKT;
AT_WRITE_REG(hw, REG_IMR, hw->intr_mask);
spin_unlock(&hw->intr_mask_lock);
__napi_schedule(&adapter->napi);
}
}
if (status & ISR_TX_PKT)
atl1c_clean_tx_irq(adapter, atl1c_trans_normal);
if (status & ISR_TX_PKT) {
if (napi_schedule_prep(&adapter->tx_napi)) {
spin_lock(&hw->intr_mask_lock);
hw->intr_mask &= ~ISR_TX_PKT;
AT_WRITE_REG(hw, REG_IMR, hw->intr_mask);
spin_unlock(&hw->intr_mask_lock);
__napi_schedule(&adapter->tx_napi);
}
}

handled = IRQ_HANDLED;
/* check if PCIE PHY Link down */
Expand Down Expand Up @@ -1876,6 +1893,7 @@ static int atl1c_clean(struct napi_struct *napi, int budget)
struct atl1c_adapter *adapter =
container_of(napi, struct atl1c_adapter, napi);
int work_done = 0;
unsigned long flags;

/* Keep link state information with original netdev */
if (!netif_carrier_ok(adapter->netdev))
Expand All @@ -1886,8 +1904,10 @@ static int atl1c_clean(struct napi_struct *napi, int budget)
if (work_done < budget) {
quit_polling:
napi_complete_done(napi, work_done);
spin_lock_irqsave(&adapter->hw.intr_mask_lock, flags);
adapter->hw.intr_mask |= ISR_RX_PKT;
AT_WRITE_REG(&adapter->hw, REG_IMR, adapter->hw.intr_mask);
spin_unlock_irqrestore(&adapter->hw.intr_mask_lock, flags);
}
return work_done;
}
Expand Down Expand Up @@ -2325,6 +2345,7 @@ static int atl1c_up(struct atl1c_adapter *adapter)
atl1c_check_link_status(adapter);
clear_bit(__AT_DOWN, &adapter->flags);
napi_enable(&adapter->napi);
napi_enable(&adapter->tx_napi);
atl1c_irq_enable(adapter);
netif_start_queue(netdev);
return err;
Expand All @@ -2345,6 +2366,7 @@ static void atl1c_down(struct atl1c_adapter *adapter)
set_bit(__AT_DOWN, &adapter->flags);
netif_carrier_off(netdev);
napi_disable(&adapter->napi);
napi_disable(&adapter->tx_napi);
atl1c_irq_disable(adapter);
atl1c_free_irq(adapter);
/* disable ASPM if device inactive */
Expand Down Expand Up @@ -2593,7 +2615,9 @@ static int atl1c_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
adapter->mii.mdio_write = atl1c_mdio_write;
adapter->mii.phy_id_mask = 0x1f;
adapter->mii.reg_num_mask = MDIO_CTRL_REG_MASK;
dev_set_threaded(netdev, true);
netif_napi_add(netdev, &adapter->napi, atl1c_clean, 64);
netif_napi_add(netdev, &adapter->tx_napi, atl1c_clean_tx, 64);
timer_setup(&adapter->phy_config_timer, atl1c_phy_config, 0);
/* setup the private structure */
err = atl1c_sw_init(adapter);
Expand Down

0 comments on commit a1150a0

Please sign in to comment.