Skip to content

Commit

Permalink
IB/IPoIB: Separate control from HW operation on ipoib_open/stop ndo
Browse files Browse the repository at this point in the history 
This patch is preparing the netdev part at the IPoIB driver to be able
to use the ipoib_options.

It deals with the two flows from the .ndo: ipoib_open and ipoib_stop.

The code is rearranged as follows:
 * All operations which deal with the hardware resources, (for example
   change QP state, post-receive etc.) are performed in one place.
 * All operations that are control oriented (like restart multicast task,
   start the reap_ah etc.) are performed in separate place.

The functions that deal with the hardware resources now located at
__ipoib_ib_dev_open for the ipoib_open flow and __ipoib_ib_dev_stop
for ipoib_stop.

Signed-off-by: Erez Shitrit <erezsh@mellanox.com>
Reviewed-by: Alex Vesker <valex@mellanox.com>
Signed-off-by: Leon Romanovsky <leon@kernel.org>
Signed-off-by: Doug Ledford <dledford@redhat.com>
  • Loading branch information
Erez Shitrit authored and Doug Ledford committed Apr 20, 2017
1 parent 515ed4f commit 7ce1a3e
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 103 deletions.
2 changes: 1 addition & 1 deletion drivers/infiniband/ulp/ipoib/ipoib.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -503,7 +503,7 @@ void ipoib_dev_uninit_default(struct net_device *dev);
int ipoib_ib_dev_open(struct net_device *dev);
void ipoib_ib_dev_up(struct net_device *dev);
void ipoib_ib_dev_down(struct net_device *dev);
void ipoib_ib_dev_stop(struct net_device *dev);
int ipoib_ib_dev_stop_default(struct net_device *dev);
void ipoib_pkey_dev_check_presence(struct net_device *dev);

int ipoib_dev_init(struct net_device *dev, struct ib_device *ca, int port);
Expand Down
230 changes: 129 additions & 101 deletions drivers/infiniband/ulp/ipoib/ipoib_ib.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -692,24 +692,115 @@ static void ipoib_stop_ah(struct net_device *dev)
ipoib_flush_ah(dev);
}

void ipoib_ib_tx_timer_func(unsigned long ctx)
static int recvs_pending(struct net_device *dev)
{
drain_tx_cq((struct net_device *)ctx);
struct ipoib_dev_priv *priv = netdev_priv(dev);
int pending = 0;
int i;

for (i = 0; i < ipoib_recvq_size; ++i)
if (priv->rx_ring[i].skb)
++pending;

return pending;
}

int ipoib_ib_dev_open(struct net_device *dev)
int ipoib_ib_dev_stop_default(struct net_device *dev)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
int ret;
struct ib_qp_attr qp_attr;
unsigned long begin;
struct ipoib_tx_buf *tx_req;
int i;

ipoib_pkey_dev_check_presence(dev);
if (test_and_clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))
napi_disable(&priv->napi);

if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
ipoib_warn(priv, "P_Key 0x%04x is %s\n", priv->pkey,
(!(priv->pkey & 0x7fff) ? "Invalid" : "not found"));
return -1;
ipoib_cm_dev_stop(dev);

/*
* Move our QP to the error state and then reinitialize in
* when all work requests have completed or have been flushed.
*/
qp_attr.qp_state = IB_QPS_ERR;
if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
ipoib_warn(priv, "Failed to modify QP to ERROR state\n");

/* Wait for all sends and receives to complete */
begin = jiffies;

while (priv->tx_head != priv->tx_tail || recvs_pending(dev)) {
if (time_after(jiffies, begin + 5 * HZ)) {
ipoib_warn(priv,
"timing out; %d sends %d receives not completed\n",
priv->tx_head - priv->tx_tail,
recvs_pending(dev));

/*
* assume the HW is wedged and just free up
* all our pending work requests.
*/
while ((int)priv->tx_tail - (int)priv->tx_head < 0) {
tx_req = &priv->tx_ring[priv->tx_tail &
(ipoib_sendq_size - 1)];
ipoib_dma_unmap_tx(priv, tx_req);
dev_kfree_skb_any(tx_req->skb);
++priv->tx_tail;
--priv->tx_outstanding;
}

for (i = 0; i < ipoib_recvq_size; ++i) {
struct ipoib_rx_buf *rx_req;

rx_req = &priv->rx_ring[i];
if (!rx_req->skb)
continue;
ipoib_ud_dma_unmap_rx(priv,
priv->rx_ring[i].mapping);
dev_kfree_skb_any(rx_req->skb);
rx_req->skb = NULL;
}

goto timeout;
}

ipoib_drain_cq(dev);

msleep(1);
}

ipoib_dbg(priv, "All sends and receives done.\n");

timeout:
del_timer_sync(&priv->poll_timer);
qp_attr.qp_state = IB_QPS_RESET;
if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
ipoib_warn(priv, "Failed to modify QP to RESET state\n");

ib_req_notify_cq(priv->recv_cq, IB_CQ_NEXT_COMP);

return 0;
}

int ipoib_ib_dev_stop(struct net_device *dev)
{
ipoib_ib_dev_stop_default(dev);

ipoib_flush_ah(dev);

return 0;
}

void ipoib_ib_tx_timer_func(unsigned long ctx)
{
drain_tx_cq((struct net_device *)ctx);
}

int ipoib_ib_dev_open_default(struct net_device *dev)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
int ret;

ret = ipoib_init_qp(dev);
if (ret) {
ipoib_warn(priv, "ipoib_init_qp returned %d\n", ret);
Expand All @@ -728,10 +819,6 @@ int ipoib_ib_dev_open(struct net_device *dev)
goto dev_stop;
}

clear_bit(IPOIB_STOP_REAPER, &priv->flags);
queue_delayed_work(priv->wq, &priv->ah_reap_task,
round_jiffies_relative(HZ));

if (!test_and_set_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))
napi_enable(&priv->napi);

Expand All @@ -743,6 +830,35 @@ int ipoib_ib_dev_open(struct net_device *dev)
return -1;
}

int ipoib_ib_dev_open(struct net_device *dev)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);

ipoib_pkey_dev_check_presence(dev);

if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags)) {
ipoib_warn(priv, "P_Key 0x%04x is %s\n", priv->pkey,
(!(priv->pkey & 0x7fff) ? "Invalid" : "not found"));
return -1;
}

clear_bit(IPOIB_STOP_REAPER, &priv->flags);
queue_delayed_work(priv->wq, &priv->ah_reap_task,
round_jiffies_relative(HZ));

if (ipoib_ib_dev_open_default(dev)) {
pr_warn("%s: Failed to open dev\n", dev->name);
goto stop_ah_reap;
}

return 0;

stop_ah_reap:
set_bit(IPOIB_STOP_REAPER, &priv->flags);
cancel_delayed_work(&priv->ah_reap_task);
return -1;
}

void ipoib_pkey_dev_check_presence(struct net_device *dev)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
Expand Down Expand Up @@ -786,19 +902,6 @@ void ipoib_ib_dev_down(struct net_device *dev)
ipoib_flush_paths(dev);
}

static int recvs_pending(struct net_device *dev)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
int pending = 0;
int i;

for (i = 0; i < ipoib_recvq_size; ++i)
if (priv->rx_ring[i].skb)
++pending;

return pending;
}

void ipoib_drain_cq(struct net_device *dev)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
Expand Down Expand Up @@ -838,81 +941,6 @@ void ipoib_drain_cq(struct net_device *dev)
local_bh_enable();
}

void ipoib_ib_dev_stop(struct net_device *dev)
{
struct ipoib_dev_priv *priv = netdev_priv(dev);
struct ib_qp_attr qp_attr;
unsigned long begin;
struct ipoib_tx_buf *tx_req;
int i;

if (test_and_clear_bit(IPOIB_FLAG_INITIALIZED, &priv->flags))
napi_disable(&priv->napi);

ipoib_cm_dev_stop(dev);

/*
* Move our QP to the error state and then reinitialize in
* when all work requests have completed or have been flushed.
*/
qp_attr.qp_state = IB_QPS_ERR;
if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
ipoib_warn(priv, "Failed to modify QP to ERROR state\n");

/* Wait for all sends and receives to complete */
begin = jiffies;

while (priv->tx_head != priv->tx_tail || recvs_pending(dev)) {
if (time_after(jiffies, begin + 5 * HZ)) {
ipoib_warn(priv, "timing out; %d sends %d receives not completed\n",
priv->tx_head - priv->tx_tail, recvs_pending(dev));

/*
* assume the HW is wedged and just free up
* all our pending work requests.
*/
while ((int) priv->tx_tail - (int) priv->tx_head < 0) {
tx_req = &priv->tx_ring[priv->tx_tail &
(ipoib_sendq_size - 1)];
ipoib_dma_unmap_tx(priv, tx_req);
dev_kfree_skb_any(tx_req->skb);
++priv->tx_tail;
--priv->tx_outstanding;
}

for (i = 0; i < ipoib_recvq_size; ++i) {
struct ipoib_rx_buf *rx_req;

rx_req = &priv->rx_ring[i];
if (!rx_req->skb)
continue;
ipoib_ud_dma_unmap_rx(priv,
priv->rx_ring[i].mapping);
dev_kfree_skb_any(rx_req->skb);
rx_req->skb = NULL;
}

goto timeout;
}

ipoib_drain_cq(dev);

msleep(1);
}

ipoib_dbg(priv, "All sends and receives done.\n");

timeout:
del_timer_sync(&priv->poll_timer);
qp_attr.qp_state = IB_QPS_RESET;
if (ib_modify_qp(priv->qp, &qp_attr, IB_QP_STATE))
ipoib_warn(priv, "Failed to modify QP to RESET state\n");

ipoib_flush_ah(dev);

ib_req_notify_cq(priv->recv_cq, IB_CQ_NEXT_COMP);
}

/*
* Takes whatever value which is in pkey index 0 and updates priv->pkey
* returns 0 if the pkey value was changed.
Expand Down
2 changes: 1 addition & 1 deletion drivers/infiniband/ulp/ipoib/ipoib_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ static int ipoib_stop(struct net_device *dev)
netif_stop_queue(dev);

ipoib_ib_dev_down(dev);
ipoib_ib_dev_stop(dev);
ipoib_ib_dev_stop_default(dev);

if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
struct ipoib_dev_priv *cpriv;
Expand Down

0 comments on commit 7ce1a3e

Please sign in to comment.