Skip to content

Commit

Permalink
net/ibmvnic: unlock rtnl_lock in reset so linkwatch_event can run
Browse files Browse the repository at this point in the history 
Commit a5681e2 ("net/ibmnvic: Fix deadlock problem in reset")
made the change to hold the RTNL lock during a reset to avoid deadlock
but linkwatch_event is fired during the reset and needs the RTNL lock.
That keeps linkwatch_event process from proceeding until the reset
is complete. The reset process cannot tolerate the linkwatch_event
processing after reset completes, so release the RTNL lock during the
process to allow a chance for linkwatch_event to run during reset.
This does not guarantee that the linkwatch_event will be processed as
soon as link state changes, but is an improvement over the current code
where linkwatch_event processing is always delayed, which prevents
transmissions on the device from being deactivated leading transmit
watchdog timer to time-out.

Release the RTNL lock before link state change and re-acquire after
the link state change to allow linkwatch_event to grab the RTNL lock
and run during the reset.

Fixes: a5681e2 ("net/ibmnvic: Fix deadlock problem in reset")
Signed-off-by: Juliet Kim <julietk@linux.vnet.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Juliet Kim authored and David S. Miller committed Sep 25, 2019
1 parent 5aafeb7 commit b27507b
Show file tree
Hide file tree
Showing 2 changed files with 157 additions and 68 deletions.
224 changes: 156 additions & 68 deletions drivers/net/ethernet/ibm/ibmvnic.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1723,6 +1723,86 @@ static int ibmvnic_set_mac(struct net_device *netdev, void *p)
return rc;
}

/**
* do_change_param_reset returns zero if we are able to keep processing reset
* events, or non-zero if we hit a fatal error and must halt.
*/
static int do_change_param_reset(struct ibmvnic_adapter *adapter,
struct ibmvnic_rwi *rwi,
u32 reset_state)
{
struct net_device *netdev = adapter->netdev;
int i, rc;

netdev_dbg(adapter->netdev, "Change param resetting driver (%d)\n",
rwi->reset_reason);

netif_carrier_off(netdev);
adapter->reset_reason = rwi->reset_reason;

ibmvnic_cleanup(netdev);

if (reset_state == VNIC_OPEN) {
rc = __ibmvnic_close(netdev);
if (rc)
return rc;
}

release_resources(adapter);
release_sub_crqs(adapter, 1);
release_crq_queue(adapter);

adapter->state = VNIC_PROBED;

rc = init_crq_queue(adapter);

if (rc) {
netdev_err(adapter->netdev,
"Couldn't initialize crq. rc=%d\n", rc);
return rc;
}

rc = ibmvnic_reset_init(adapter);
if (rc)
return IBMVNIC_INIT_FAILED;

/* If the adapter was in PROBE state prior to the reset,
* exit here.
*/
if (reset_state == VNIC_PROBED)
return 0;

rc = ibmvnic_login(netdev);
if (rc) {
adapter->state = reset_state;
return rc;
}

rc = init_resources(adapter);
if (rc)
return rc;

ibmvnic_disable_irqs(adapter);

adapter->state = VNIC_CLOSED;

if (reset_state == VNIC_CLOSED)
return 0;

rc = __ibmvnic_open(netdev);
if (rc)
return IBMVNIC_OPEN_FAILED;

/* refresh device's multicast list */
ibmvnic_set_multi(netdev);

/* kick napi */
for (i = 0; i < adapter->req_rx_queues; i++)
napi_schedule(&adapter->napi[i]);

return 0;
}

/**
* do_reset returns zero if we are able to keep processing reset events, or
* non-zero if we hit a fatal error and must halt.
Expand All @@ -1738,6 +1818,8 @@ static int do_reset(struct ibmvnic_adapter *adapter,
netdev_dbg(adapter->netdev, "Re-setting driver (%d)\n",
rwi->reset_reason);

rtnl_lock();

netif_carrier_off(netdev);
adapter->reset_reason = rwi->reset_reason;

Expand All @@ -1751,16 +1833,25 @@ static int do_reset(struct ibmvnic_adapter *adapter,
if (reset_state == VNIC_OPEN &&
adapter->reset_reason != VNIC_RESET_MOBILITY &&
adapter->reset_reason != VNIC_RESET_FAILOVER) {
rc = __ibmvnic_close(netdev);
adapter->state = VNIC_CLOSING;

/* Release the RTNL lock before link state change and
* re-acquire after the link state change to allow
* linkwatch_event to grab the RTNL lock and run during
* a reset.
*/
rtnl_unlock();
rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
rtnl_lock();
if (rc)
return rc;
}
goto out;

if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM ||
adapter->wait_for_reset) {
release_resources(adapter);
release_sub_crqs(adapter, 1);
release_crq_queue(adapter);
if (adapter->state != VNIC_CLOSING) {
rc = -1;
goto out;
}

adapter->state = VNIC_CLOSED;
}

if (adapter->reset_reason != VNIC_RESET_NON_FATAL) {
Expand All @@ -1769,9 +1860,7 @@ static int do_reset(struct ibmvnic_adapter *adapter,
*/
adapter->state = VNIC_PROBED;

if (adapter->wait_for_reset) {
rc = init_crq_queue(adapter);
} else if (adapter->reset_reason == VNIC_RESET_MOBILITY) {
if (adapter->reset_reason == VNIC_RESET_MOBILITY) {
rc = ibmvnic_reenable_crq_queue(adapter);
release_sub_crqs(adapter, 1);
} else {
Expand All @@ -1783,69 +1872,66 @@ static int do_reset(struct ibmvnic_adapter *adapter,
if (rc) {
netdev_err(adapter->netdev,
"Couldn't initialize crq. rc=%d\n", rc);
return rc;
goto out;
}

rc = ibmvnic_reset_init(adapter);
if (rc)
return IBMVNIC_INIT_FAILED;
if (rc) {
rc = IBMVNIC_INIT_FAILED;
goto out;
}

/* If the adapter was in PROBE state prior to the reset,
* exit here.
*/
if (reset_state == VNIC_PROBED)
return 0;
if (reset_state == VNIC_PROBED) {
rc = 0;
goto out;
}

rc = ibmvnic_login(netdev);
if (rc) {
adapter->state = reset_state;
return rc;
goto out;
}

if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM ||
adapter->wait_for_reset) {
rc = init_resources(adapter);
if (rc)
return rc;
} else if (adapter->req_rx_queues != old_num_rx_queues ||
adapter->req_tx_queues != old_num_tx_queues ||
adapter->req_rx_add_entries_per_subcrq !=
old_num_rx_slots ||
adapter->req_tx_entries_per_subcrq !=
old_num_tx_slots) {
if (adapter->req_rx_queues != old_num_rx_queues ||
adapter->req_tx_queues != old_num_tx_queues ||
adapter->req_rx_add_entries_per_subcrq !=
old_num_rx_slots ||
adapter->req_tx_entries_per_subcrq !=
old_num_tx_slots) {
release_rx_pools(adapter);
release_tx_pools(adapter);
release_napi(adapter);
release_vpd_data(adapter);

rc = init_resources(adapter);
if (rc)
return rc;
goto out;

} else {
rc = reset_tx_pools(adapter);
if (rc)
return rc;
goto out;

rc = reset_rx_pools(adapter);
if (rc)
return rc;
goto out;
}
ibmvnic_disable_irqs(adapter);
}
adapter->state = VNIC_CLOSED;

if (reset_state == VNIC_CLOSED)
return 0;
if (reset_state == VNIC_CLOSED) {
rc = 0;
goto out;
}

rc = __ibmvnic_open(netdev);
if (rc) {
if (list_empty(&adapter->rwi_list))
adapter->state = VNIC_CLOSED;
else
adapter->state = reset_state;

return 0;
rc = IBMVNIC_OPEN_FAILED;
goto out;
}

/* refresh device's multicast list */
Expand All @@ -1855,11 +1941,15 @@ static int do_reset(struct ibmvnic_adapter *adapter,
for (i = 0; i < adapter->req_rx_queues; i++)
napi_schedule(&adapter->napi[i]);

if (adapter->reset_reason != VNIC_RESET_FAILOVER &&
adapter->reset_reason != VNIC_RESET_CHANGE_PARAM)
if (adapter->reset_reason != VNIC_RESET_FAILOVER)
call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, netdev);

return 0;
rc = 0;

out:
rtnl_unlock();

return rc;
}

static int do_hard_reset(struct ibmvnic_adapter *adapter,
Expand Down Expand Up @@ -1919,14 +2009,8 @@ static int do_hard_reset(struct ibmvnic_adapter *adapter,
return 0;

rc = __ibmvnic_open(netdev);
if (rc) {
if (list_empty(&adapter->rwi_list))
adapter->state = VNIC_CLOSED;
else
adapter->state = reset_state;

return 0;
}
if (rc)
return IBMVNIC_OPEN_FAILED;

return 0;
}
Expand Down Expand Up @@ -1965,20 +2049,11 @@ static void __ibmvnic_reset(struct work_struct *work)
{
struct ibmvnic_rwi *rwi;
struct ibmvnic_adapter *adapter;
bool we_lock_rtnl = false;
u32 reset_state;
int rc = 0;

adapter = container_of(work, struct ibmvnic_adapter, ibmvnic_reset);

/* netif_set_real_num_xx_queues needs to take rtnl lock here
* unless wait_for_reset is set, in which case the rtnl lock
* has already been taken before initializing the reset
*/
if (!adapter->wait_for_reset) {
rtnl_lock();
we_lock_rtnl = true;
}
reset_state = adapter->state;

rwi = get_next_rwi(adapter);
Expand All @@ -1990,22 +2065,39 @@ static void __ibmvnic_reset(struct work_struct *work)
break;
}

if (adapter->force_reset_recovery) {
adapter->force_reset_recovery = false;
rc = do_hard_reset(adapter, rwi, reset_state);
if (rwi->reset_reason == VNIC_RESET_CHANGE_PARAM) {
/* CHANGE_PARAM requestor holds rtnl_lock */
rc = do_change_param_reset(adapter, rwi, reset_state);
} else if (adapter->force_reset_recovery) {
/* Transport event occurred during previous reset */
if (adapter->wait_for_reset) {
/* Previous was CHANGE_PARAM; caller locked */
adapter->force_reset_recovery = false;
rc = do_hard_reset(adapter, rwi, reset_state);
} else {
rtnl_lock();
adapter->force_reset_recovery = false;
rc = do_hard_reset(adapter, rwi, reset_state);
rtnl_unlock();
}
} else {
rc = do_reset(adapter, rwi, reset_state);
}
kfree(rwi);
if (rc && rc != IBMVNIC_INIT_FAILED &&
if (rc == IBMVNIC_OPEN_FAILED) {
if (list_empty(&adapter->rwi_list))
adapter->state = VNIC_CLOSED;
else
adapter->state = reset_state;
rc = 0;
} else if (rc && rc != IBMVNIC_INIT_FAILED &&
!adapter->force_reset_recovery)
break;

rwi = get_next_rwi(adapter);
}

if (adapter->wait_for_reset) {
adapter->wait_for_reset = false;
adapter->reset_done_rc = rc;
complete(&adapter->reset_done);
}
Expand All @@ -2016,8 +2108,6 @@ static void __ibmvnic_reset(struct work_struct *work)
}

adapter->resetting = false;
if (we_lock_rtnl)
rtnl_unlock();
}

static int ibmvnic_reset(struct ibmvnic_adapter *adapter,
Expand Down Expand Up @@ -2078,8 +2168,6 @@ static int ibmvnic_reset(struct ibmvnic_adapter *adapter,

return 0;
err:
if (adapter->wait_for_reset)
adapter->wait_for_reset = false;
return -ret;
}

Expand Down
1 change: 1 addition & 0 deletions drivers/net/ethernet/ibm/ibmvnic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#define IBMVNIC_INVALID_MAP -1
#define IBMVNIC_STATS_TIMEOUT 1
#define IBMVNIC_INIT_FAILED 2
#define IBMVNIC_OPEN_FAILED 3

/* basic structures plus 100 2k buffers */
#define IBMVNIC_IO_ENTITLEMENT_DEFAULT 610305
Expand Down

0 comments on commit b27507b

Please sign in to comment.