From bd0b6723139416fd22acd5849b93edbd32d561f3 Mon Sep 17 00:00:00 2001 From: John Allen Date: Fri, 17 Mar 2017 17:13:40 -0500 Subject: [PATCH 1/4] ibmvnic: Move login and queue negotiation into ibmvnic_open VNIC server expects LINK_STATE_UP to be sent within 30s of the login. If we exceed the timeout, VNIC server will attempt to fail over. Since time between probe and open of the device is indeterminate, move login and queue negotiation into ibmvnic open so we can guarantee that login and sending LINK_STATE_UP occur within the 30s window. Signed-off-by: John Allen Signed-off-by: David S. Miller --- drivers/net/ethernet/ibm/ibmvnic.c | 88 +++++++++++++++--------------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 5f11b4dc95d2d..61d9d4045b4c8 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -110,6 +110,9 @@ static int ibmvnic_poll(struct napi_struct *napi, int data); static void send_map_query(struct ibmvnic_adapter *adapter); static void send_request_map(struct ibmvnic_adapter *, dma_addr_t, __be32, u8); static void send_request_unmap(struct ibmvnic_adapter *, u8); +static void send_login(struct ibmvnic_adapter *adapter); +static void send_cap_queries(struct ibmvnic_adapter *adapter); +static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter); struct ibmvnic_stat { char name[ETH_GSTRING_LEN]; @@ -371,14 +374,51 @@ static void free_rx_pool(struct ibmvnic_adapter *adapter, static int ibmvnic_open(struct net_device *netdev) { struct ibmvnic_adapter *adapter = netdev_priv(netdev); + unsigned long timeout = msecs_to_jiffies(30000); struct device *dev = &adapter->vdev->dev; struct ibmvnic_tx_pool *tx_pool; union ibmvnic_crq crq; int rxadd_subcrqs; u64 *size_array; int tx_subcrqs; + int rc = 0; int i, j; + do { + if (adapter->renegotiate) { + adapter->renegotiate = false; + release_sub_crqs_no_irqs(adapter); + + reinit_completion(&adapter->init_done); + send_cap_queries(adapter); + if (!wait_for_completion_timeout(&adapter->init_done, + timeout)) { + dev_err(dev, "Capabilities query timeout\n"); + return -1; + } + } + + reinit_completion(&adapter->init_done); + send_login(adapter); + if (!wait_for_completion_timeout(&adapter->init_done, + timeout)) { + dev_err(dev, "Login timeout\n"); + return -1; + } + } while (adapter->renegotiate); + + rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues); + if (rc) { + dev_err(dev, "failed to set the number of tx queues\n"); + return -1; + } + + rc = init_sub_crq_irqs(adapter); + if (rc) { + dev_err(dev, "failed to initialize sub crq irqs\n"); + return -1; + } + rxadd_subcrqs = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs); tx_subcrqs = @@ -508,6 +548,7 @@ static int ibmvnic_open(struct net_device *netdev) for (i = 0; i < adapter->req_rx_queues; i++) napi_disable(&adapter->napi[i]); alloc_napi_failed: + release_sub_crqs(adapter); return -ENOMEM; } @@ -3419,8 +3460,7 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq, dma_unmap_single(dev, adapter->ip_offload_ctrl_tok, sizeof(adapter->ip_offload_ctrl), DMA_TO_DEVICE); - /* We're done with the queries, perform the login */ - send_login(adapter); + complete(&adapter->init_done); break; case REQUEST_RAS_COMP_NUM_RSP: netdev_dbg(netdev, "Got Request RAS Comp Num Response\n"); @@ -3700,26 +3740,6 @@ static void handle_crq_init_rsp(struct work_struct *work) goto task_failed; } - do { - if (adapter->renegotiate) { - adapter->renegotiate = false; - release_sub_crqs_no_irqs(adapter); - - reinit_completion(&adapter->init_done); - send_cap_queries(adapter); - if (!wait_for_completion_timeout(&adapter->init_done, - timeout)) { - dev_err(dev, "Passive init timeout\n"); - goto task_failed; - } - } - } while (adapter->renegotiate); - rc = init_sub_crq_irqs(adapter); - - if (rc) - goto task_failed; - - netdev->real_num_tx_queues = adapter->req_tx_queues; netdev->mtu = adapter->req_mtu - ETH_HLEN; if (adapter->failover) { @@ -3840,39 +3860,17 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) if (!wait_for_completion_timeout(&adapter->init_done, timeout)) return 0; - do { - if (adapter->renegotiate) { - adapter->renegotiate = false; - release_sub_crqs_no_irqs(adapter); - - reinit_completion(&adapter->init_done); - send_cap_queries(adapter); - if (!wait_for_completion_timeout(&adapter->init_done, - timeout)) - return 0; - } - } while (adapter->renegotiate); - - rc = init_sub_crq_irqs(adapter); - if (rc) { - dev_err(&dev->dev, "failed to initialize sub crq irqs\n"); - goto free_debugfs; - } - - netdev->real_num_tx_queues = adapter->req_tx_queues; netdev->mtu = adapter->req_mtu - ETH_HLEN; rc = register_netdev(netdev); if (rc) { dev_err(&dev->dev, "failed to register netdev rc=%d\n", rc); - goto free_sub_crqs; + goto free_debugfs; } dev_info(&dev->dev, "ibmvnic registered\n"); return 0; -free_sub_crqs: - release_sub_crqs(adapter); free_debugfs: if (adapter->debugfs_dir && !IS_ERR(adapter->debugfs_dir)) debugfs_remove_recursive(adapter->debugfs_dir); From a57a5d25a56bd9625722d17fecacf21cbe9818d8 Mon Sep 17 00:00:00 2001 From: John Allen Date: Fri, 17 Mar 2017 17:13:41 -0500 Subject: [PATCH 2/4] ibmvnic: Move login to its own routine Move the code that handles login and renegotiation of ibmvnic capabilities to its own routine. Signed-off-by: Nathan Fontenot Signed-off-by: David S. Miller --- drivers/net/ethernet/ibm/ibmvnic.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 61d9d4045b4c8..04f8feb0f2a35 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -371,18 +371,11 @@ static void free_rx_pool(struct ibmvnic_adapter *adapter, pool->rx_buff = NULL; } -static int ibmvnic_open(struct net_device *netdev) +static int ibmvnic_login(struct net_device *netdev) { struct ibmvnic_adapter *adapter = netdev_priv(netdev); unsigned long timeout = msecs_to_jiffies(30000); struct device *dev = &adapter->vdev->dev; - struct ibmvnic_tx_pool *tx_pool; - union ibmvnic_crq crq; - int rxadd_subcrqs; - u64 *size_array; - int tx_subcrqs; - int rc = 0; - int i, j; do { if (adapter->renegotiate) { @@ -407,6 +400,25 @@ static int ibmvnic_open(struct net_device *netdev) } } while (adapter->renegotiate); + return 0; +} + +static int ibmvnic_open(struct net_device *netdev) +{ + struct ibmvnic_adapter *adapter = netdev_priv(netdev); + struct device *dev = &adapter->vdev->dev; + struct ibmvnic_tx_pool *tx_pool; + union ibmvnic_crq crq; + int rxadd_subcrqs; + u64 *size_array; + int tx_subcrqs; + int rc = 0; + int i, j; + + rc = ibmvnic_login(netdev); + if (rc) + return rc; + rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues); if (rc) { dev_err(dev, "failed to set the number of tx queues\n"); From f6ef6408e840f6e1a7c016d6dd1b4341bc1023f6 Mon Sep 17 00:00:00 2001 From: John Allen Date: Fri, 17 Mar 2017 17:13:42 -0500 Subject: [PATCH 3/4] ibmvnic: Move ibmvnic adapter intialization to its own routine The intialization of the ibmvnic driver with respect to the virtual server it connects to should be moved to its own routine. This will alolow the driver to initiate this process from places outside of the drivers probe routine. Signed-off-by: Nathan Fontenot Signed-off-by: David S. Miller --- drivers/net/ethernet/ibm/ibmvnic.c | 110 ++++++++++++++++------------- 1 file changed, 60 insertions(+), 50 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 04f8feb0f2a35..42274bf4bc095 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -3783,14 +3783,65 @@ static void handle_crq_init_rsp(struct work_struct *work) dev_err(dev, "Passive initialization was not successful\n"); } -static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) +static int ibmvnic_init(struct ibmvnic_adapter *adapter) { + struct device *dev = &adapter->vdev->dev; unsigned long timeout = msecs_to_jiffies(30000); + struct dentry *ent; + char buf[17]; /* debugfs name buf */ + int rc; + + rc = ibmvnic_init_crq_queue(adapter); + if (rc) { + dev_err(dev, "Couldn't initialize crq. rc=%d\n", rc); + return rc; + } + + adapter->stats_token = dma_map_single(dev, &adapter->stats, + sizeof(struct ibmvnic_statistics), + DMA_FROM_DEVICE); + if (dma_mapping_error(dev, adapter->stats_token)) { + ibmvnic_release_crq_queue(adapter); + dev_err(dev, "Couldn't map stats buffer\n"); + return -ENOMEM; + } + + snprintf(buf, sizeof(buf), "ibmvnic_%x", adapter->vdev->unit_address); + ent = debugfs_create_dir(buf, NULL); + if (!ent || IS_ERR(ent)) { + dev_info(dev, "debugfs create directory failed\n"); + adapter->debugfs_dir = NULL; + } else { + adapter->debugfs_dir = ent; + ent = debugfs_create_file("dump", S_IRUGO, + adapter->debugfs_dir, + adapter->netdev, &ibmvnic_dump_ops); + if (!ent || IS_ERR(ent)) { + dev_info(dev, "debugfs create dump file failed\n"); + adapter->debugfs_dump = NULL; + } else { + adapter->debugfs_dump = ent; + } + } + + init_completion(&adapter->init_done); + ibmvnic_send_crq_init(adapter); + if (!wait_for_completion_timeout(&adapter->init_done, timeout)) { + dev_err(dev, "Initialization sequence timed out\n"); + if (adapter->debugfs_dir && !IS_ERR(adapter->debugfs_dir)) + debugfs_remove_recursive(adapter->debugfs_dir); + ibmvnic_release_crq_queue(adapter); + return -1; + } + + return 0; +} + +static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) +{ struct ibmvnic_adapter *adapter; struct net_device *netdev; unsigned char *mac_addr_p; - struct dentry *ent; - char buf[17]; /* debugfs name buf */ int rc; dev_dbg(&dev->dev, "entering ibmvnic_probe for UA 0x%x\n", @@ -3828,69 +3879,28 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) spin_lock_init(&adapter->stats_lock); - rc = ibmvnic_init_crq_queue(adapter); - if (rc) { - dev_err(&dev->dev, "Couldn't initialize crq. rc=%d\n", rc); - goto free_netdev; - } - INIT_LIST_HEAD(&adapter->errors); INIT_LIST_HEAD(&adapter->inflight); spin_lock_init(&adapter->error_list_lock); spin_lock_init(&adapter->inflight_lock); - adapter->stats_token = dma_map_single(&dev->dev, &adapter->stats, - sizeof(struct ibmvnic_statistics), - DMA_FROM_DEVICE); - if (dma_mapping_error(&dev->dev, adapter->stats_token)) { - if (!firmware_has_feature(FW_FEATURE_CMO)) - dev_err(&dev->dev, "Couldn't map stats buffer\n"); - rc = -ENOMEM; - goto free_crq; - } - - snprintf(buf, sizeof(buf), "ibmvnic_%x", dev->unit_address); - ent = debugfs_create_dir(buf, NULL); - if (!ent || IS_ERR(ent)) { - dev_info(&dev->dev, "debugfs create directory failed\n"); - adapter->debugfs_dir = NULL; - } else { - adapter->debugfs_dir = ent; - ent = debugfs_create_file("dump", S_IRUGO, adapter->debugfs_dir, - netdev, &ibmvnic_dump_ops); - if (!ent || IS_ERR(ent)) { - dev_info(&dev->dev, - "debugfs create dump file failed\n"); - adapter->debugfs_dump = NULL; - } else { - adapter->debugfs_dump = ent; - } + rc = ibmvnic_init(adapter); + if (rc) { + free_netdev(netdev); + return rc; } - init_completion(&adapter->init_done); - ibmvnic_send_crq_init(adapter); - if (!wait_for_completion_timeout(&adapter->init_done, timeout)) - return 0; - netdev->mtu = adapter->req_mtu - ETH_HLEN; rc = register_netdev(netdev); if (rc) { dev_err(&dev->dev, "failed to register netdev rc=%d\n", rc); - goto free_debugfs; + free_netdev(netdev); + return rc; } dev_info(&dev->dev, "ibmvnic registered\n"); return 0; - -free_debugfs: - if (adapter->debugfs_dir && !IS_ERR(adapter->debugfs_dir)) - debugfs_remove_recursive(adapter->debugfs_dir); -free_crq: - ibmvnic_release_crq_queue(adapter); -free_netdev: - free_netdev(netdev); - return rc; } static int ibmvnic_remove(struct vio_dev *dev) From ea5509f53ce81662eb409c514086734d1ce16207 Mon Sep 17 00:00:00 2001 From: John Allen Date: Fri, 17 Mar 2017 17:13:43 -0500 Subject: [PATCH 4/4] ibmvnic: Correct ibmvnic handling of device open/close When closing the ibmvnic device we need to release the resources used in communicating to the virtual I/O server. These need to be re-negotiated with the server at open time. This patch moves the releasing of resources a separate routine and updates the open and close handlers to release all resources at close and re-negotiate and allocate these resources at open. Signed-off-by: Nathan Fontenot Signed-off-by: David S. Miller --- drivers/net/ethernet/ibm/ibmvnic.c | 114 +++++++++++++++++------------ drivers/net/ethernet/ibm/ibmvnic.h | 1 + 2 files changed, 67 insertions(+), 48 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 42274bf4bc095..30e1699649b82 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -113,6 +113,8 @@ static void send_request_unmap(struct ibmvnic_adapter *, u8); static void send_login(struct ibmvnic_adapter *adapter); static void send_cap_queries(struct ibmvnic_adapter *adapter); static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter); +static int ibmvnic_init(struct ibmvnic_adapter *); +static void ibmvnic_release_crq_queue(struct ibmvnic_adapter *); struct ibmvnic_stat { char name[ETH_GSTRING_LEN]; @@ -415,6 +417,12 @@ static int ibmvnic_open(struct net_device *netdev) int rc = 0; int i, j; + if (adapter->is_closed) { + rc = ibmvnic_init(adapter); + if (rc) + return rc; + } + rc = ibmvnic_login(netdev); if (rc) return rc; @@ -525,6 +533,7 @@ static int ibmvnic_open(struct net_device *netdev) ibmvnic_send_crq(adapter, &crq); netif_tx_start_all_queues(netdev); + adapter->is_closed = false; return 0; @@ -564,21 +573,12 @@ static int ibmvnic_open(struct net_device *netdev) return -ENOMEM; } -static int ibmvnic_close(struct net_device *netdev) +static void ibmvnic_release_resources(struct ibmvnic_adapter *adapter) { - struct ibmvnic_adapter *adapter = netdev_priv(netdev); struct device *dev = &adapter->vdev->dev; - union ibmvnic_crq crq; + int tx_scrqs, rx_scrqs; int i; - adapter->closing = true; - - for (i = 0; i < adapter->req_rx_queues; i++) - napi_disable(&adapter->napi[i]); - - if (!adapter->failover) - netif_tx_stop_all_queues(netdev); - if (adapter->bounce_buffer) { if (!dma_mapping_error(dev, adapter->bounce_buffer_dma)) { dma_unmap_single(&adapter->vdev->dev, @@ -591,33 +591,70 @@ static int ibmvnic_close(struct net_device *netdev) adapter->bounce_buffer = NULL; } - memset(&crq, 0, sizeof(crq)); - crq.logical_link_state.first = IBMVNIC_CRQ_CMD; - crq.logical_link_state.cmd = LOGICAL_LINK_STATE; - crq.logical_link_state.link_state = IBMVNIC_LOGICAL_LNK_DN; - ibmvnic_send_crq(adapter, &crq); + tx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs); + for (i = 0; i < tx_scrqs; i++) { + struct ibmvnic_tx_pool *tx_pool = &adapter->tx_pool[i]; - for (i = 0; i < be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs); - i++) { - kfree(adapter->tx_pool[i].tx_buff); - free_long_term_buff(adapter, - &adapter->tx_pool[i].long_term_buff); - kfree(adapter->tx_pool[i].free_map); + kfree(tx_pool->tx_buff); + free_long_term_buff(adapter, &tx_pool->long_term_buff); + kfree(tx_pool->free_map); } kfree(adapter->tx_pool); adapter->tx_pool = NULL; - for (i = 0; i < be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs); - i++) { - free_rx_pool(adapter, &adapter->rx_pool[i]); - free_long_term_buff(adapter, - &adapter->rx_pool[i].long_term_buff); + rx_scrqs = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs); + for (i = 0; i < rx_scrqs; i++) { + struct ibmvnic_rx_pool *rx_pool = &adapter->rx_pool[i]; + + free_rx_pool(adapter, rx_pool); + free_long_term_buff(adapter, &rx_pool->long_term_buff); } kfree(adapter->rx_pool); adapter->rx_pool = NULL; - adapter->closing = false; + release_sub_crqs(adapter); + ibmvnic_release_crq_queue(adapter); + + if (adapter->debugfs_dir && !IS_ERR(adapter->debugfs_dir)) + debugfs_remove_recursive(adapter->debugfs_dir); + + if (adapter->stats_token) + dma_unmap_single(dev, adapter->stats_token, + sizeof(struct ibmvnic_statistics), + DMA_FROM_DEVICE); + + if (adapter->ras_comps) + dma_free_coherent(dev, adapter->ras_comp_num * + sizeof(struct ibmvnic_fw_component), + adapter->ras_comps, adapter->ras_comps_tok); + + kfree(adapter->ras_comp_int); +} + +static int ibmvnic_close(struct net_device *netdev) +{ + struct ibmvnic_adapter *adapter = netdev_priv(netdev); + union ibmvnic_crq crq; + int i; + + adapter->closing = true; + + for (i = 0; i < adapter->req_rx_queues; i++) + napi_disable(&adapter->napi[i]); + + if (!adapter->failover) + netif_tx_stop_all_queues(netdev); + memset(&crq, 0, sizeof(crq)); + crq.logical_link_state.first = IBMVNIC_CRQ_CMD; + crq.logical_link_state.cmd = LOGICAL_LINK_STATE; + crq.logical_link_state.link_state = IBMVNIC_LOGICAL_LNK_DN; + ibmvnic_send_crq(adapter, &crq); + + ibmvnic_release_resources(adapter); + + adapter->is_closed = true; + adapter->closing = false; return 0; } @@ -3891,6 +3928,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) } netdev->mtu = adapter->req_mtu - ETH_HLEN; + adapter->is_closed = false; rc = register_netdev(netdev); if (rc) { @@ -3906,28 +3944,8 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) static int ibmvnic_remove(struct vio_dev *dev) { struct net_device *netdev = dev_get_drvdata(&dev->dev); - struct ibmvnic_adapter *adapter = netdev_priv(netdev); unregister_netdev(netdev); - - release_sub_crqs(adapter); - - ibmvnic_release_crq_queue(adapter); - - if (adapter->debugfs_dir && !IS_ERR(adapter->debugfs_dir)) - debugfs_remove_recursive(adapter->debugfs_dir); - - dma_unmap_single(&dev->dev, adapter->stats_token, - sizeof(struct ibmvnic_statistics), DMA_FROM_DEVICE); - - if (adapter->ras_comps) - dma_free_coherent(&dev->dev, - adapter->ras_comp_num * - sizeof(struct ibmvnic_fw_component), - adapter->ras_comps, adapter->ras_comps_tok); - - kfree(adapter->ras_comp_int); - free_netdev(netdev); dev_set_drvdata(&dev->dev, NULL); diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h index 1993b42666f73..10ad259208cb0 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.h +++ b/drivers/net/ethernet/ibm/ibmvnic.h @@ -1052,4 +1052,5 @@ struct ibmvnic_adapter { struct work_struct ibmvnic_xport; struct tasklet_struct tasklet; bool failover; + bool is_closed; };