From 82654b6b8ef8b93ee87a97fc562f87f081fc2f91 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Fri, 2 Jun 2017 16:32:08 +0800 Subject: [PATCH 1/7] nvme: fix hang in remove path We need to start admin queues too in nvme_kill_queues() for avoiding hang in remove path[1]. This patch is very similar with 806f026f9b901eaf(nvme: use blk_mq_start_hw_queues() in nvme_kill_queues()). [1] hang stack trace [] blk_execute_rq+0x56/0x80 [] __nvme_submit_sync_cmd+0x89/0xf0 [] nvme_set_features+0x5e/0x90 [] nvme_configure_apst+0x166/0x200 [] nvme_set_latency_tolerance+0x35/0x50 [] apply_constraint+0xb1/0xc0 [] dev_pm_qos_constraints_destroy+0xf4/0x1f0 [] dpm_sysfs_remove+0x2a/0x60 [] device_del+0x101/0x320 [] device_unregister+0x1a/0x60 [] device_destroy+0x3c/0x50 [] nvme_uninit_ctrl+0x45/0xa0 [] nvme_remove+0x78/0x110 [] pci_device_remove+0x39/0xb0 [] device_release_driver_internal+0x155/0x210 [] device_release_driver+0x12/0x20 [] nvme_remove_dead_ctrl_work+0x6b/0x70 [] process_one_work+0x18c/0x3a0 [] worker_thread+0x4e/0x3b0 [] kthread+0x109/0x140 [] ret_from_fork+0x2c/0x40 [] 0xffffffffffffffff Fixes: c5552fde102fc("nvme: Enable autonomous power state transitions") Reported-by: Rakesh Pandit Tested-by: Rakesh Pandit Reviewed-by: Sagi Grimberg Signed-off-by: Ming Lei Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index a60926410438b..0f9cc0c55e153 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2438,6 +2438,10 @@ void nvme_kill_queues(struct nvme_ctrl *ctrl) struct nvme_ns *ns; mutex_lock(&ctrl->namespaces_mutex); + + /* Forcibly start all queues to avoid having stuck requests */ + blk_mq_start_hw_queues(ctrl->admin_q); + list_for_each_entry(ns, &ctrl->namespaces, list) { /* * Revalidating a dead namespace sets capacity to 0. This will From 82b057caefaff2a891f821a617d939f46e03e844 Mon Sep 17 00:00:00 2001 From: Rakesh Pandit Date: Mon, 5 Jun 2017 14:43:11 +0300 Subject: [PATCH 2/7] nvme-pci: fix multiple ctrl removal scheduling Commit c5f6ce97c1210 tries to address multiple resets but fails as work_busy doesn't involve any synchronization and can fail. This is reproducible easily as can be seen by WARNING below which is triggered with line: WARN_ON(dev->ctrl.state == NVME_CTRL_RESETTING) Allowing multiple resets can result in multiple controller removal as well if different conditions inside nvme_reset_work fail and which might deadlock on device_release_driver. [ 480.327007] WARNING: CPU: 3 PID: 150 at drivers/nvme/host/pci.c:1900 nvme_reset_work+0x36c/0xec0 [ 480.327008] Modules linked in: rfcomm fuse nf_conntrack_netbios_ns nf_conntrack_broadcast... [ 480.327044] btusb videobuf2_core ghash_clmulni_intel snd_hwdep cfg80211 acer_wmi hci_uart.. [ 480.327065] CPU: 3 PID: 150 Comm: kworker/u16:2 Not tainted 4.12.0-rc1+ #13 [ 480.327065] Hardware name: Acer Predator G9-591/Mustang_SLS, BIOS V1.10 03/03/2016 [ 480.327066] Workqueue: nvme nvme_reset_work [ 480.327067] task: ffff880498ad8000 task.stack: ffffc90002218000 [ 480.327068] RIP: 0010:nvme_reset_work+0x36c/0xec0 [ 480.327069] RSP: 0018:ffffc9000221bdb8 EFLAGS: 00010246 [ 480.327070] RAX: 0000000000460000 RBX: ffff880498a98128 RCX: dead000000000200 [ 480.327070] RDX: 0000000000000001 RSI: ffff8804b1028020 RDI: ffff880498a98128 [ 480.327071] RBP: ffffc9000221be50 R08: 0000000000000000 R09: 0000000000000000 [ 480.327071] R10: ffffc90001963ce8 R11: 000000000000020d R12: ffff880498a98000 [ 480.327072] R13: ffff880498a53500 R14: ffff880498a98130 R15: ffff880498a98128 [ 480.327072] FS: 0000000000000000(0000) GS:ffff8804c1cc0000(0000) knlGS:0000000000000000 [ 480.327073] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 480.327074] CR2: 00007ffcf3c37f78 CR3: 0000000001e09000 CR4: 00000000003406e0 [ 480.327074] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 480.327075] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 480.327075] Call Trace: [ 480.327079] ? __switch_to+0x227/0x400 [ 480.327081] process_one_work+0x18c/0x3a0 [ 480.327082] worker_thread+0x4e/0x3b0 [ 480.327084] kthread+0x109/0x140 [ 480.327085] ? process_one_work+0x3a0/0x3a0 [ 480.327087] ? kthread_park+0x60/0x60 [ 480.327102] ret_from_fork+0x2c/0x40 [ 480.327103] Code: e8 5a dc ff ff 85 c0 41 89 c1 0f..... This patch addresses the problem by using state of controller to decide whether reset should be queued or not as state change is synchronizated using controller spinlock. Also cancel_work_sync is used to make sure remove cancels the reset_work and waits for it to finish. This patch also changes return value from -ENODEV to more appropriate -EBUSY if nvme_reset fails to change state. Fixes: c5f6ce97c1210 ("nvme: don't schedule multiple resets") Signed-off-by: Rakesh Pandit Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/host/pci.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index d52701df72457..951042a375d6b 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1367,7 +1367,7 @@ static bool nvme_should_reset(struct nvme_dev *dev, u32 csts) bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO); /* If there is a reset ongoing, we shouldn't reset again. */ - if (work_busy(&dev->reset_work)) + if (dev->ctrl.state == NVME_CTRL_RESETTING) return false; /* We shouldn't reset unless the controller is on fatal error state @@ -1903,7 +1903,7 @@ static void nvme_reset_work(struct work_struct *work) bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL); int result = -ENODEV; - if (WARN_ON(dev->ctrl.state == NVME_CTRL_RESETTING)) + if (WARN_ON(dev->ctrl.state != NVME_CTRL_RESETTING)) goto out; /* @@ -1913,9 +1913,6 @@ static void nvme_reset_work(struct work_struct *work) if (dev->ctrl.ctrl_config & NVME_CC_ENABLE) nvme_dev_disable(dev, false); - if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) - goto out; - result = nvme_pci_enable(dev); if (result) goto out; @@ -2009,8 +2006,8 @@ static int nvme_reset(struct nvme_dev *dev) { if (!dev->ctrl.admin_q || blk_queue_dying(dev->ctrl.admin_q)) return -ENODEV; - if (work_busy(&dev->reset_work)) - return -ENODEV; + if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING)) + return -EBUSY; if (!queue_work(nvme_workq, &dev->reset_work)) return -EBUSY; return 0; @@ -2136,6 +2133,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (result) goto release_pools; + nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_RESETTING); dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev)); queue_work(nvme_workq, &dev->reset_work); @@ -2179,6 +2177,7 @@ static void nvme_remove(struct pci_dev *pdev) nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING); + cancel_work_sync(&dev->reset_work); pci_set_drvdata(pdev, NULL); if (!pci_device_is_present(pdev)) { From e818a5b487fea20494b0e48548c1085634abdc0d Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Mon, 5 Jun 2017 20:35:56 +0300 Subject: [PATCH 3/7] nvme-rdma: fast fail incoming requests while we reconnect When we encounter an transport/controller errors, error recovery kicks in which performs: 1. stops io/admin queues 2. moves transport queues out of LIVE state 3. fast fail pending io 4. schedule periodic reconnects. But we also need to fast fail incoming IO taht enters after we already scheduled. Given that our queue is not LIVE anymore, simply restart the request queues to fail in .queue_rq Reported-by: Alex Turin Reported-by: shahar.salzman Signed-off-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Cc: stable@vger.kernel.org --- drivers/nvme/host/rdma.c | 44 ++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 28bd255c144dc..24397d306d532 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -753,28 +753,26 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work) if (ret) goto requeue; - blk_mq_start_stopped_hw_queues(ctrl->ctrl.admin_q, true); - ret = nvmf_connect_admin_queue(&ctrl->ctrl); if (ret) - goto stop_admin_q; + goto requeue; set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags); ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->cap); if (ret) - goto stop_admin_q; + goto requeue; nvme_start_keep_alive(&ctrl->ctrl); if (ctrl->queue_count > 1) { ret = nvme_rdma_init_io_queues(ctrl); if (ret) - goto stop_admin_q; + goto requeue; ret = nvme_rdma_connect_io_queues(ctrl); if (ret) - goto stop_admin_q; + goto requeue; } changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE); @@ -782,7 +780,6 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work) ctrl->ctrl.opts->nr_reconnects = 0; if (ctrl->queue_count > 1) { - nvme_start_queues(&ctrl->ctrl); nvme_queue_scan(&ctrl->ctrl); nvme_queue_async_events(&ctrl->ctrl); } @@ -791,8 +788,6 @@ static void nvme_rdma_reconnect_ctrl_work(struct work_struct *work) return; -stop_admin_q: - blk_mq_stop_hw_queues(ctrl->ctrl.admin_q); requeue: dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n", ctrl->ctrl.opts->nr_reconnects); @@ -823,6 +818,13 @@ static void nvme_rdma_error_recovery_work(struct work_struct *work) blk_mq_tagset_busy_iter(&ctrl->admin_tag_set, nvme_cancel_request, &ctrl->ctrl); + /* + * queues are not a live anymore, so restart the queues to fail fast + * new IO + */ + blk_mq_start_stopped_hw_queues(ctrl->ctrl.admin_q, true); + nvme_start_queues(&ctrl->ctrl); + nvme_rdma_reconnect_or_remove(ctrl); } @@ -1433,7 +1435,7 @@ nvme_rdma_timeout(struct request *rq, bool reserved) /* * We cannot accept any other command until the Connect command has completed. */ -static inline bool nvme_rdma_queue_is_ready(struct nvme_rdma_queue *queue, +static inline int nvme_rdma_queue_is_ready(struct nvme_rdma_queue *queue, struct request *rq) { if (unlikely(!test_bit(NVME_RDMA_Q_LIVE, &queue->flags))) { @@ -1441,11 +1443,22 @@ static inline bool nvme_rdma_queue_is_ready(struct nvme_rdma_queue *queue, if (!blk_rq_is_passthrough(rq) || cmd->common.opcode != nvme_fabrics_command || - cmd->fabrics.fctype != nvme_fabrics_type_connect) - return false; + cmd->fabrics.fctype != nvme_fabrics_type_connect) { + /* + * reconnecting state means transport disruption, which + * can take a long time and even might fail permanently, + * so we can't let incoming I/O be requeued forever. + * fail it fast to allow upper layers a chance to + * failover. + */ + if (queue->ctrl->ctrl.state == NVME_CTRL_RECONNECTING) + return -EIO; + else + return -EAGAIN; + } } - return true; + return 0; } static int nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, @@ -1463,8 +1476,9 @@ static int nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, WARN_ON_ONCE(rq->tag < 0); - if (!nvme_rdma_queue_is_ready(queue, rq)) - return BLK_MQ_RQ_QUEUE_BUSY; + ret = nvme_rdma_queue_is_ready(queue, rq); + if (unlikely(ret)) + goto err; dev = queue->device->dev; ib_dma_sync_single_for_cpu(dev, sqe->dma, From f874d5d079ec35158fa1a1509554c9d641bd5770 Mon Sep 17 00:00:00 2001 From: James Smart Date: Thu, 1 Jun 2017 22:54:21 -0700 Subject: [PATCH 4/7] nvme-fc: on lldd/transport io error, terminate association Per FC-NVME, when lldd or transport detects an i/o error, the connection must be terminated, which in turn requires the association to be termianted. Currently the transport simply creates a nvme completion status of transport error and returns the io. The FC-NVME spec makes the mandate as initiator and host, depending on the error, can get out of sync on outstanding io counts (sqhd/sqtail). Implement the association teardown on lldd or transport detected errors. Signed-off-by: James Smart Signed-off-by: Christoph Hellwig Reviewed-by: Sagi Grimberg --- drivers/nvme/host/fc.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 5b14cbefb7240..2edae54688e88 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -1139,6 +1139,7 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl) /* *********************** NVME Ctrl Routines **************************** */ static void __nvme_fc_final_op_cleanup(struct request *rq); +static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg); static int nvme_fc_reinit_request(void *data, struct request *rq) @@ -1265,7 +1266,7 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) struct nvme_command *sqe = &op->cmd_iu.sqe; __le16 status = cpu_to_le16(NVME_SC_SUCCESS << 1); union nvme_result result; - bool complete_rq; + bool complete_rq, terminate_assoc = true; /* * WARNING: @@ -1294,6 +1295,14 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) * fabricate a CQE, the following fields will not be set as they * are not referenced: * cqe.sqid, cqe.sqhd, cqe.command_id + * + * Failure or error of an individual i/o, in a transport + * detected fashion unrelated to the nvme completion status, + * potentially cause the initiator and target sides to get out + * of sync on SQ head/tail (aka outstanding io count allowed). + * Per FC-NVME spec, failure of an individual command requires + * the connection to be terminated, which in turn requires the + * association to be terminated. */ fc_dma_sync_single_for_cpu(ctrl->lport->dev, op->fcp_req.rspdma, @@ -1359,6 +1368,8 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) goto done; } + terminate_assoc = false; + done: if (op->flags & FCOP_FLAGS_AEN) { nvme_complete_async_event(&queue->ctrl->ctrl, status, &result); @@ -1366,7 +1377,7 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) atomic_set(&op->state, FCPOP_STATE_IDLE); op->flags = FCOP_FLAGS_AEN; /* clear other flags */ nvme_fc_ctrl_put(ctrl); - return; + goto check_error; } complete_rq = __nvme_fc_fcpop_chk_teardowns(ctrl, op); @@ -1379,6 +1390,10 @@ nvme_fc_fcpio_done(struct nvmefc_fcp_req *req) nvme_end_request(rq, status, result); } else __nvme_fc_final_op_cleanup(rq); + +check_error: + if (terminate_assoc) + nvme_fc_error_recovery(ctrl, "transport detected io error"); } static int From 24b7f0592f738a1127c72dbf5b72a83997dd6997 Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 5 Jun 2017 15:03:42 -0700 Subject: [PATCH 5/7] nvme-fc: fix missing put reference on controller create failure The failure case, of a create controller request, called nvme_uninit_ctrl() but didn't do a put to allow the nvme controller to be deleted. Signed-off-by: James Smart Signed-off-by: Christoph Hellwig --- drivers/nvme/host/fc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 2edae54688e88..92964cef0f4be 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2806,6 +2806,7 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, ctrl->ctrl.opts = NULL; /* initiate nvme ctrl ref counting teardown */ nvme_uninit_ctrl(&ctrl->ctrl); + nvme_put_ctrl(&ctrl->ctrl); /* as we're past the point where we transition to the ref * counting teardown path, if we return a bad pointer here, From da87591bea92204fcb921bac927666eb7141908e Mon Sep 17 00:00:00 2001 From: Kai-Heng Feng Date: Wed, 7 Jun 2017 15:25:42 +0800 Subject: [PATCH 6/7] nvme: only consider exit latency when choosing useful non-op power states When a NVMe is in non-op states, the latency is exlat. The latency will be enlat + exlat only when the NVMe tries to transit from operational state right atfer it begins to transit to non-operational state, which should be a rare case. Therefore, as Andy Lutomirski suggests, use exlat only when deciding power states to trainsit to. Signed-off-by: Kai-Heng Feng Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 0f9cc0c55e153..c07d8d4e18c91 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1342,7 +1342,7 @@ static void nvme_configure_apst(struct nvme_ctrl *ctrl) * transitioning between power states. Therefore, when running * in any given state, we will enter the next lower-power * non-operational state after waiting 50 * (enlat + exlat) - * microseconds, as long as that state's total latency is under + * microseconds, as long as that state's exit latency is under * the requested maximum latency. * * We will not autonomously enter any non-operational state for @@ -1387,7 +1387,7 @@ static void nvme_configure_apst(struct nvme_ctrl *ctrl) * lowest-power state, not the number of states. */ for (state = (int)ctrl->npss; state >= 0; state--) { - u64 total_latency_us, transition_ms; + u64 total_latency_us, exit_latency_us, transition_ms; if (target) table->entries[state] = target; @@ -1408,12 +1408,15 @@ static void nvme_configure_apst(struct nvme_ctrl *ctrl) NVME_PS_FLAGS_NON_OP_STATE)) continue; - total_latency_us = - (u64)le32_to_cpu(ctrl->psd[state].entry_lat) + - + le32_to_cpu(ctrl->psd[state].exit_lat); - if (total_latency_us > ctrl->ps_max_latency_us) + exit_latency_us = + (u64)le32_to_cpu(ctrl->psd[state].exit_lat); + if (exit_latency_us > ctrl->ps_max_latency_us) continue; + total_latency_us = + exit_latency_us + + le32_to_cpu(ctrl->psd[state].entry_lat); + /* * This state is good. Use it as the APST idle * target for higher power states. From 9947d6a09cd71937dade2fc14640e4843ae19802 Mon Sep 17 00:00:00 2001 From: Kai-Heng Feng Date: Wed, 7 Jun 2017 15:25:43 +0800 Subject: [PATCH 7/7] nvme: relax APST default max latency to 100ms Christoph Hellwig suggests we should to make APST work out of the box. Hence relax the the default max latency to make them able to enter deepest power state on default. Here are id-ctrl excerpts from two high latency NVMes: vid : 0x14a4 ssvid : 0x1b4b mn : CX2-GB1024-Q11 NVMe LITEON 1024GB ps 3 : mp:0.1000W non-operational enlat:5000 exlat:5000 rrt:3 rrl:3 rwt:3 rwl:3 idle_power:- active_power:- ps 4 : mp:0.0100W non-operational enlat:50000 exlat:100000 rrt:4 rrl:4 rwt:4 rwl:4 idle_power:- active_power:- vid : 0x15b7 ssvid : 0x1b4b mn : A400 NVMe SanDisk 512GB ps 3 : mp:0.0500W non-operational enlat:51000 exlat:10000 rrt:0 rrl:0 rwt:0 rwl:0 idle_power:- active_power:- ps 4 : mp:0.0055W non-operational enlat:1000000 exlat:100000 rrt:0 rrl:0 rwt:0 rwl:0 idle_power:- active_power:- Signed-off-by: Kai-Heng Feng Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index c07d8d4e18c91..903d5813023a9 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -56,7 +56,7 @@ MODULE_PARM_DESC(max_retries, "max number of retries a command may have"); static int nvme_char_major; module_param(nvme_char_major, int, 0); -static unsigned long default_ps_max_latency_us = 25000; +static unsigned long default_ps_max_latency_us = 100000; module_param(default_ps_max_latency_us, ulong, 0644); MODULE_PARM_DESC(default_ps_max_latency_us, "max power saving latency for new devices; use PM QOS to change per device");