From 6e2d15f59b1cc6ed613b94e0969335a7868f04ca Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Tue, 10 Oct 2023 17:20:42 +0800 Subject: [PATCH 01/10] scsi: scsi_debug: Create scsi_debug directory in the debugfs filesystem Create directory scsi_debug in the root of the debugfs filesystem. Prepare to add interface for manage error injection. Acked-by: Douglas Gilbert Signed-off-by: Wenchao Hao Link: https://lore.kernel.org/r/20231010092051.608007-2-haowenchao2@huawei.com Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index 9c0af50501f9a..562a48f53cda2 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -41,6 +41,7 @@ #include #include #include +#include #include @@ -862,6 +863,8 @@ static const int device_qfull_result = static const int condition_met_result = SAM_STAT_CONDITION_MET; +static struct dentry *sdebug_debugfs_root; + /* Only do the extra work involved in logical block provisioning if one or * more of the lbpu, lbpws or lbpws10 parameters are given and we are doing @@ -7011,6 +7014,10 @@ static int __init scsi_debug_init(void) goto driver_unreg; } + sdebug_debugfs_root = debugfs_create_dir("scsi_debug", NULL); + if (IS_ERR_OR_NULL(sdebug_debugfs_root)) + pr_info("%s: failed to create initial debugfs directory\n", __func__); + for (k = 0; k < hosts_to_add; k++) { if (want_store && k == 0) { ret = sdebug_add_host_helper(idx); @@ -7057,6 +7064,7 @@ static void __exit scsi_debug_exit(void) sdebug_erase_all_stores(false); xa_destroy(per_store_ap); + debugfs_remove(sdebug_debugfs_root); } device_initcall(scsi_debug_init); From a9996d722b1197b625acfa350dd2849e35ad6092 Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Tue, 10 Oct 2023 17:20:43 +0800 Subject: [PATCH 02/10] scsi: scsi_debug: Add interface to manage error injection for a single device This new facility uses the debugfs pseudo file system which is typically mounted under the /sys/kernel/debug directory and requires root permissions to access. The interface file is found at /sys/kernel/debug/scsi_debug//error where identifies the device (logical unit (LU)) to inject errors on. For the following description the ${error} environment variable is assumed to be set to/sys/kernel/debug/scsi_debug/1:0:0:0/error where 1:0:0:0 is a pseudo device (LU) owned by the scsi_debug driver. Rules are written to ${error} in the normal sysfs fashion (e.g. 'echo "0 -2 0x12" > ${error}'). More than one rule can be active on a device at a time and inactive rules (i.e. those whose error count is 0) remain in the rule listing. The existing rules can be read with 'cat ${error}' with oneline output for each rule. The interface format is line-by-line, each line is an error injection rule. Each rule contains integers separated by spaces, the first three columns correspond to "Error code", "Error count" and "SCSI command", other columns depend on Error code. General rule format: +--------+------+-------------------------------------------------------+ | Column | Type | Description | +--------+------+-------------------------------------------------------+ | 1 | u8 | Error code | | | | 0: timeout SCSI command | | | | 1: fail queuecommand, make queuecommand return | | | | given value | | | | 2: fail command, finish command with SCSI status, | | | | sense key and ASC/ASCQ values | | | | 3: make abort commands for specific command fail | | | | 4: make reset lun for specific command fail | +--------+------+-------------------------------------------------------+ | 2 | s32 | Error count | | | | 0: this rule will be ignored | | | | positive: the rule will always take effect | | | | negative: the rule takes effect n times where -n is | | | | the value given. Ignored after n times | +--------+------+-------------------------------------------------------+ | 3 | x8 | SCSI command opcode, 0xff for all commands | +--------+------+-------------------------------------------------------+ | ... | xxx | Error type specific fields | +--------+------+-------------------------------------------------------+ Notes: - When multiple error inject rules are added for the same SCSI command, the one with smaller error code will take effect (and the others will be ignored). - If the same error (i.e. same Error code and SCSI command) is added, the older one will be overwritten.. - Currently, the basic types are (u8/u16/u32/u64/s8/s16/s32/s64) and the hexadecimal types (x8/x16/x32/x64). - Where a hexadecimal value is expected (e.g. Column 3: SCSI command opcode) the "0x" prefix is optional on the value (e.g. the INQUIRY opcode can be given as '0x12' or '12'). - When the Error count is negative, reading ${error} will show that value incrementing, stopping when it gets to 0. Acked-by: Douglas Gilbert Signed-off-by: Wenchao Hao Link: https://lore.kernel.org/r/20231010092051.608007-3-haowenchao2@huawei.com Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 214 +++++++++++++++++++++++++++++++++++++- 1 file changed, 210 insertions(+), 4 deletions(-) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index 562a48f53cda2..46f1ee647b15f 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -286,6 +286,42 @@ struct sdeb_zone_state { /* ZBC: per zone state */ sector_t z_wp; }; +enum sdebug_err_type { + ERR_TMOUT_CMD = 0, /* make specific scsi command timeout */ + ERR_FAIL_QUEUE_CMD = 1, /* make specific scsi command's */ + /* queuecmd return failed */ + ERR_FAIL_CMD = 2, /* make specific scsi command's */ + /* queuecmd return succeed but */ + /* with errors set in scsi_cmnd */ +}; + +struct sdebug_err_inject { + int type; + struct list_head list; + int cnt; + unsigned char cmd; + struct rcu_head rcu; + + union { + /* + * For ERR_FAIL_QUEUE_CMD + */ + int queuecmd_ret; + + /* + * For ERR_FAIL_CMD + */ + struct { + unsigned char host_byte; + unsigned char driver_byte; + unsigned char status_byte; + unsigned char sense_key; + unsigned char asc; + unsigned char asq; + }; + }; +}; + struct sdebug_dev_info { struct list_head dev_list; unsigned int channel; @@ -311,6 +347,10 @@ struct sdebug_dev_info { unsigned int max_open; ktime_t create_ts; /* time since bootup that this device was created */ struct sdeb_zone_state *zstate; + + struct dentry *debugfs_entry; + struct spinlock list_lock; + struct list_head inject_err_list; }; struct sdebug_host_info { @@ -865,6 +905,143 @@ static const int condition_met_result = SAM_STAT_CONDITION_MET; static struct dentry *sdebug_debugfs_root; +static void sdebug_err_free(struct rcu_head *head) +{ + struct sdebug_err_inject *inject = + container_of(head, typeof(*inject), rcu); + + kfree(inject); +} + +static void sdebug_err_add(struct scsi_device *sdev, struct sdebug_err_inject *new) +{ + struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdev->hostdata; + struct sdebug_err_inject *err; + + spin_lock(&devip->list_lock); + list_for_each_entry_rcu(err, &devip->inject_err_list, list) { + if (err->type == new->type && err->cmd == new->cmd) { + list_del_rcu(&err->list); + call_rcu(&err->rcu, sdebug_err_free); + } + } + + list_add_tail_rcu(&new->list, &devip->inject_err_list); + spin_unlock(&devip->list_lock); +} + +static int sdebug_error_show(struct seq_file *m, void *p) +{ + struct scsi_device *sdev = (struct scsi_device *)m->private; + struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdev->hostdata; + struct sdebug_err_inject *err; + + seq_puts(m, "Type\tCount\tCommand\n"); + + rcu_read_lock(); + list_for_each_entry_rcu(err, &devip->inject_err_list, list) { + switch (err->type) { + case ERR_TMOUT_CMD: + seq_printf(m, "%d\t%d\t0x%x\n", err->type, err->cnt, + err->cmd); + break; + + case ERR_FAIL_QUEUE_CMD: + seq_printf(m, "%d\t%d\t0x%x\t0x%x\n", err->type, + err->cnt, err->cmd, err->queuecmd_ret); + break; + + case ERR_FAIL_CMD: + seq_printf(m, "%d\t%d\t0x%x\t0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", + err->type, err->cnt, err->cmd, + err->host_byte, err->driver_byte, + err->status_byte, err->sense_key, + err->asc, err->asq); + break; + } + } + rcu_read_unlock(); + + return 0; +} + +static int sdebug_error_open(struct inode *inode, struct file *file) +{ + return single_open(file, sdebug_error_show, inode->i_private); +} + +static ssize_t sdebug_error_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + char *buf; + unsigned int inject_type; + struct sdebug_err_inject *inject; + struct scsi_device *sdev = (struct scsi_device *)file->f_inode->i_private; + + buf = kmalloc(count, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + if (copy_from_user(buf, ubuf, count)) { + kfree(buf); + return -EFAULT; + } + + if (sscanf(buf, "%d", &inject_type) != 1) { + kfree(buf); + return -EINVAL; + } + + inject = kzalloc(sizeof(struct sdebug_err_inject), GFP_KERNEL); + if (!inject) { + kfree(buf); + return -ENOMEM; + } + + switch (inject_type) { + case ERR_TMOUT_CMD: + if (sscanf(buf, "%d %d %hhx", &inject->type, &inject->cnt, + &inject->cmd) != 3) + goto out_error; + break; + + case ERR_FAIL_QUEUE_CMD: + if (sscanf(buf, "%d %d %hhx %x", &inject->type, &inject->cnt, + &inject->cmd, &inject->queuecmd_ret) != 4) + goto out_error; + break; + + case ERR_FAIL_CMD: + if (sscanf(buf, "%d %d %hhx %hhx %hhx %hhx %hhx %hhx %hhx", + &inject->type, &inject->cnt, &inject->cmd, + &inject->host_byte, &inject->driver_byte, + &inject->status_byte, &inject->sense_key, + &inject->asc, &inject->asq) != 9) + goto out_error; + break; + + default: + goto out_error; + break; + } + + kfree(buf); + sdebug_err_add(sdev, inject); + + return count; + +out_error: + kfree(buf); + kfree(inject); + return -EINVAL; +} + +static const struct file_operations sdebug_error_fops = { + .open = sdebug_error_open, + .read = seq_read, + .write = sdebug_error_write, + .release = single_release, +}; /* Only do the extra work involved in logical block provisioning if one or * more of the lbpu, lbpws or lbpws10 parameters are given and we are doing @@ -5099,6 +5276,8 @@ static struct sdebug_dev_info *sdebug_device_create( } devip->create_ts = ktime_get_boottime(); atomic_set(&devip->stopped, (sdeb_tur_ms_to_ready > 0 ? 2 : 0)); + spin_lock_init(&devip->list_lock); + INIT_LIST_HEAD(&devip->inject_err_list); list_add_tail(&devip->dev_list, &sdbg_host->dev_info_list); } return devip; @@ -5144,6 +5323,7 @@ static int scsi_debug_slave_alloc(struct scsi_device *sdp) if (sdebug_verbose) pr_info("slave_alloc <%u %u %u %llu>\n", sdp->host->host_no, sdp->channel, sdp->id, sdp->lun); + return 0; } @@ -5151,6 +5331,7 @@ static int scsi_debug_slave_configure(struct scsi_device *sdp) { struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdp->hostdata; + struct dentry *dentry; if (sdebug_verbose) pr_info("slave_configure <%u %u %u %llu>\n", @@ -5166,6 +5347,19 @@ static int scsi_debug_slave_configure(struct scsi_device *sdp) if (sdebug_no_uld) sdp->no_uld_attach = 1; config_cdb_len(sdp); + + devip->debugfs_entry = debugfs_create_dir(dev_name(&sdp->sdev_dev), + sdebug_debugfs_root); + if (IS_ERR_OR_NULL(devip->debugfs_entry)) + pr_info("%s: failed to create debugfs directory for device %s\n", + __func__, dev_name(&sdp->sdev_gendev)); + + dentry = debugfs_create_file("error", 0600, devip->debugfs_entry, sdp, + &sdebug_error_fops); + if (IS_ERR_OR_NULL(dentry)) + pr_info("%s: failed to create error file for device %s\n", + __func__, dev_name(&sdp->sdev_gendev)); + return 0; } @@ -5173,15 +5367,27 @@ static void scsi_debug_slave_destroy(struct scsi_device *sdp) { struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdp->hostdata; + struct sdebug_err_inject *err; if (sdebug_verbose) pr_info("slave_destroy <%u %u %u %llu>\n", sdp->host->host_no, sdp->channel, sdp->id, sdp->lun); - if (devip) { - /* make this slot available for re-use */ - devip->used = false; - sdp->hostdata = NULL; + + if (!devip) + return; + + spin_lock(&devip->list_lock); + list_for_each_entry_rcu(err, &devip->inject_err_list, list) { + list_del_rcu(&err->list); + call_rcu(&err->rcu, sdebug_err_free); } + spin_unlock(&devip->list_lock); + + debugfs_remove(devip->debugfs_entry); + + /* make this slot available for re-use */ + devip->used = false; + sdp->hostdata = NULL; } /* Returns true if we require the queued memory to be freed by the caller. */ From 962d77cd4c852e6a34ffd44fc5f32ba678e02633 Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Tue, 10 Oct 2023 17:20:44 +0800 Subject: [PATCH 03/10] scsi: scsi_debug: Define grammar to remove added error injection The grammar to remove error injection is a line with fixed 3 columns separated by spaces. First column is fixed to "-". It tells this is a removal operation. Second column is the error code to match. Third column is the scsi command to match. For example the following command would remove timeout injection of inquiry command: echo "- 0 0x12" > /sys/kernel/debug/scsi_debug/0:0:0:1/error Acked-by: Douglas Gilbert Signed-off-by: Wenchao Hao Link: https://lore.kernel.org/r/20231010092051.608007-4-haowenchao2@huawei.com Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index 46f1ee647b15f..0d5a179dbd970 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -930,6 +930,34 @@ static void sdebug_err_add(struct scsi_device *sdev, struct sdebug_err_inject *n spin_unlock(&devip->list_lock); } +static int sdebug_err_remove(struct scsi_device *sdev, const char *buf, size_t count) +{ + struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdev->hostdata; + struct sdebug_err_inject *err; + int type; + unsigned char cmd; + + if (sscanf(buf, "- %d %hhx", &type, &cmd) != 2) { + kfree(buf); + return -EINVAL; + } + + spin_lock(&devip->list_lock); + list_for_each_entry_rcu(err, &devip->inject_err_list, list) { + if (err->type == type && err->cmd == cmd) { + list_del_rcu(&err->list); + call_rcu(&err->rcu, sdebug_err_free); + spin_unlock(&devip->list_lock); + kfree(buf); + return count; + } + } + spin_unlock(&devip->list_lock); + + kfree(buf); + return -EINVAL; +} + static int sdebug_error_show(struct seq_file *m, void *p) { struct scsi_device *sdev = (struct scsi_device *)m->private; @@ -987,6 +1015,9 @@ static ssize_t sdebug_error_write(struct file *file, const char __user *ubuf, return -EFAULT; } + if (buf[0] == '-') + return sdebug_err_remove(sdev, buf, count); + if (sscanf(buf, "%d", &inject_type) != 1) { kfree(buf); return -EINVAL; From 32be8b6e22eb76a08c66414593ef02d5eb151be7 Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Tue, 10 Oct 2023 17:20:45 +0800 Subject: [PATCH 04/10] scsi: scsi_debug: Time out command if the error is injected If a timeout error is injected, return 0 from scsi_debug_queuecommand to make the command time out. Time out SCSI command format: +--------+------+-------------------------------------------------------+ | Column | Type | Description | +--------+------+-------------------------------------------------------+ | 1 | u8 | Error type, fixed to 0x0 | +--------+------+-------------------------------------------------------+ | 2 | s32 | Error count | | | | 0: this rule will be ignored | | | | positive: the rule will always take effect | | | | negative: the rule takes effect n times where -n is | | | | the value given. Ignored after n times | +--------+------+-------------------------------------------------------+ | 3 | x8 | SCSI command opcode, 0xff for all commands | +--------+------+-------------------------------------------------------+ Examples: error=/sys/kernel/debug/scsi_debug/0:0:0:1/error echo "0 -10 0x12" > ${error} will make the device's inquiry command time out 10 times. echo "0 1 0x12" > ${error} will make the device's inquiry time out each time it is invoked on this device. Acked-by: Douglas Gilbert Signed-off-by: Wenchao Hao Link: https://lore.kernel.org/r/20231010092051.608007-5-haowenchao2@huawei.com Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index 0d5a179dbd970..9081c9d228821 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -7741,6 +7741,34 @@ static int sdebug_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num) return num_entries; } +static int sdebug_timeout_cmd(struct scsi_cmnd *cmnd) +{ + struct scsi_device *sdp = cmnd->device; + struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdp->hostdata; + struct sdebug_err_inject *err; + unsigned char *cmd = cmnd->cmnd; + int ret = 0; + + if (devip == NULL) + return 0; + + rcu_read_lock(); + list_for_each_entry_rcu(err, &devip->inject_err_list, list) { + if (err->type == ERR_TMOUT_CMD && + (err->cmd == cmd[0] || err->cmd == 0xff)) { + ret = !!err->cnt; + if (err->cnt < 0) + err->cnt++; + + rcu_read_unlock(); + return ret; + } + } + rcu_read_unlock(); + + return 0; +} + static int scsi_debug_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scp) { @@ -7799,6 +7827,12 @@ static int scsi_debug_queuecommand(struct Scsi_Host *shost, if (NULL == devip) goto err_out; } + + if (sdebug_timeout_cmd(scp)) { + scmd_printk(KERN_INFO, scp, "timeout command 0x%x\n", opcode); + return 0; + } + if (unlikely(inject_now && !atomic_read(&sdeb_inject_pending))) atomic_set(&sdeb_inject_pending, 1); From 33bccf55c20b66dfca6644c8dc9b396cec82e85c Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Tue, 10 Oct 2023 17:20:46 +0800 Subject: [PATCH 05/10] scsi: scsi_debug: Return failed value if error is injected If a fail queuecommand error is injected, return the failed value defined in the rule from queuecommand. Make queuecommand return format: +--------+------+-------------------------------------------------------+ | Column | Type | Description | +--------+------+-------------------------------------------------------+ | 1 | u8 | Error type, fixed to 0x1 | +--------+------+-------------------------------------------------------+ | 2 | s32 | Error count | | | | 0: this rule will be ignored | | | | positive: the rule will always take effect | | | | negative: the rule takes effect n times where -n is | | | | the value given. Ignored after n times | +--------+------+-------------------------------------------------------+ | 3 | x8 | SCSI command opcode, 0xff for all commands | +--------+------+-------------------------------------------------------+ | 4 | x32 | The queuecommand() return value we want | +--------+------+-------------------------------------------------------+ Examples: error=/sys/kernel/debug/scsi_debug/0:0:0:1/error echo "1 1 0x12 0x1055" > ${error} will make each INQUIRY command sent to that device return 0x1055 (SCSI_MLQUEUE_HOST_BUSY). Acked-by: Douglas Gilbert Signed-off-by: Wenchao Hao Link: https://lore.kernel.org/r/20231010092051.608007-6-haowenchao2@huawei.com Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index 9081c9d228821..efdd63dd6d25d 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -7769,6 +7769,34 @@ static int sdebug_timeout_cmd(struct scsi_cmnd *cmnd) return 0; } +static int sdebug_fail_queue_cmd(struct scsi_cmnd *cmnd) +{ + struct scsi_device *sdp = cmnd->device; + struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdp->hostdata; + struct sdebug_err_inject *err; + unsigned char *cmd = cmnd->cmnd; + int ret = 0; + + if (devip == NULL) + return 0; + + rcu_read_lock(); + list_for_each_entry_rcu(err, &devip->inject_err_list, list) { + if (err->type == ERR_FAIL_QUEUE_CMD && + (err->cmd == cmd[0] || err->cmd == 0xff)) { + ret = err->cnt ? err->queuecmd_ret : 0; + if (err->cnt < 0) + err->cnt++; + + rcu_read_unlock(); + return ret; + } + } + rcu_read_unlock(); + + return 0; +} + static int scsi_debug_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scp) { @@ -7788,6 +7816,7 @@ static int scsi_debug_queuecommand(struct Scsi_Host *shost, u8 opcode = cmd[0]; bool has_wlun_rl; bool inject_now; + int ret = 0; scsi_set_resid(scp, 0); if (sdebug_statistics) { @@ -7833,6 +7862,13 @@ static int scsi_debug_queuecommand(struct Scsi_Host *shost, return 0; } + ret = sdebug_fail_queue_cmd(scp); + if (ret) { + scmd_printk(KERN_INFO, scp, "fail queue command 0x%x with 0x%x\n", + opcode, ret); + return ret; + } + if (unlikely(inject_now && !atomic_read(&sdeb_inject_pending))) atomic_set(&sdeb_inject_pending, 1); From 33592274321eab2e35e2fdf01857c722431fcf8f Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Tue, 10 Oct 2023 17:20:47 +0800 Subject: [PATCH 06/10] scsi: scsi_debug: Set command result and sense data if error is injected If a fail command error is injected, set the command's status and sense data then finish this SCSI command. Set SCSI command's status and sense data format: +--------+------+-------------------------------------------------------+ | Column | Type | Description | +--------+------+-------------------------------------------------------+ | 1 | u8 | Error type, fixed to 0x2 | +--------+------+-------------------------------------------------------+ | 2 | s32 | Error Count | | | | 0: the rule will be ignored | | | | positive: the rule will always take effect | | | | negative: the rule takes effect n times where -n is | | | | the value given. Ignored after n times | +--------+------+-------------------------------------------------------+ | 3 | x8 | SCSI command opcode, 0xff for all commands | +--------+------+-------------------------------------------------------+ | 4 | x8 | Host byte in scsi_cmd::status | | | | [scsi_cmd::status has 32 bits holding these 3 bytes] | +--------+------+-------------------------------------------------------+ | 5 | x8 | Driver byte in scsi_cmd::status | +--------+------+-------------------------------------------------------+ | 6 | x8 | SCSI Status byte in scsi_cmd::status | +--------+------+-------------------------------------------------------+ | 7 | x8 | SCSI Sense Key in scsi_cmnd | +--------+------+-------------------------------------------------------+ | 8 | x8 | SCSI ASC in scsi_cmnd | +--------+------+-------------------------------------------------------+ | 9 | x8 | SCSI ASCQ in scsi_cmnd | +--------+------+-------------------------------------------------------+ Examples: error=/sys/kernel/debug/scsi_debug/0:0:0:1/error echo "2 -10 0x88 0 0 0x2 0x3 0x11 0x0" >${error} will make device's read command return with media error with additional sense of "Unrecovered read error" (UNC): Acked-by: Douglas Gilbert Signed-off-by: Wenchao Hao Link: https://lore.kernel.org/r/20231010092051.608007-7-haowenchao2@huawei.com Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 53 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index efdd63dd6d25d..d15d007f13178 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -7797,6 +7797,48 @@ static int sdebug_fail_queue_cmd(struct scsi_cmnd *cmnd) return 0; } +static int sdebug_fail_cmd(struct scsi_cmnd *cmnd, int *retval, + struct sdebug_err_inject *info) +{ + struct scsi_device *sdp = cmnd->device; + struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdp->hostdata; + struct sdebug_err_inject *err; + unsigned char *cmd = cmnd->cmnd; + int ret = 0; + int result; + + if (devip == NULL) + return 0; + + rcu_read_lock(); + list_for_each_entry_rcu(err, &devip->inject_err_list, list) { + if (err->type == ERR_FAIL_CMD && + (err->cmd == cmd[0] || err->cmd == 0xff)) { + if (!err->cnt) { + rcu_read_unlock(); + return 0; + } + + ret = !!err->cnt; + rcu_read_unlock(); + goto out_handle; + } + } + rcu_read_unlock(); + + return 0; + +out_handle: + if (err->cnt < 0) + err->cnt++; + mk_sense_buffer(cmnd, err->sense_key, err->asc, err->asq); + result = err->status_byte | err->host_byte << 16 | err->driver_byte << 24; + *info = *err; + *retval = schedule_resp(cmnd, devip, result, NULL, 0, 0); + + return ret; +} + static int scsi_debug_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scp) { @@ -7817,6 +7859,7 @@ static int scsi_debug_queuecommand(struct Scsi_Host *shost, bool has_wlun_rl; bool inject_now; int ret = 0; + struct sdebug_err_inject err; scsi_set_resid(scp, 0); if (sdebug_statistics) { @@ -7869,6 +7912,16 @@ static int scsi_debug_queuecommand(struct Scsi_Host *shost, return ret; } + if (sdebug_fail_cmd(scp, &ret, &err)) { + scmd_printk(KERN_INFO, scp, + "fail command 0x%x with hostbyte=0x%x, " + "driverbyte=0x%x, statusbyte=0x%x, " + "sense_key=0x%x, asc=0x%x, asq=0x%x\n", + opcode, err.host_byte, err.driver_byte, + err.status_byte, err.sense_key, err.asc, err.asq); + return ret; + } + if (unlikely(inject_now && !atomic_read(&sdeb_inject_pending))) atomic_set(&sdeb_inject_pending, 1); From 5551ce928805ba790db0fa0a895e207d5d05717d Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Tue, 10 Oct 2023 17:20:48 +0800 Subject: [PATCH 07/10] scsi: scsi_debug: Add new error injection type: Abort Failed Add error injection type 3 to make scsi_debug_abort() return FAILED. Fail abort command format: +--------+------+-------------------------------------------------------+ | Column | Type | Description | +--------+------+-------------------------------------------------------+ | 1 | u8 | Error type, fixed to 0x3 | +--------+------+-------------------------------------------------------+ | 2 | s32 | Error count | | | | 0: this rule will be ignored | | | | positive: the rule will always take effect | | | | negative: the rule takes effect n times where -n is | | | | the value given. Ignored after n times | +--------+------+-------------------------------------------------------+ | 3 | x8 | SCSI command opcode, 0xff for all commands | +--------+------+-------------------------------------------------------+ Examples: error=/sys/kernel/debug/scsi_debug/0:0:0:1/error echo "3 -10 0x12" > ${error} will make the device return FAILED when aborting inquiry command 10 times. Signed-off-by: Wenchao Hao Link: https://lore.kernel.org/r/20231010092051.608007-8-haowenchao2@huawei.com Tested-by: Douglas Gilbert Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index d15d007f13178..89f47ded1e238 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -293,6 +293,8 @@ enum sdebug_err_type { ERR_FAIL_CMD = 2, /* make specific scsi command's */ /* queuecmd return succeed but */ /* with errors set in scsi_cmnd */ + ERR_ABORT_CMD_FAILED = 3, /* control return FAILED from */ + /* scsi_debug_abort() */ }; struct sdebug_err_inject { @@ -970,6 +972,7 @@ static int sdebug_error_show(struct seq_file *m, void *p) list_for_each_entry_rcu(err, &devip->inject_err_list, list) { switch (err->type) { case ERR_TMOUT_CMD: + case ERR_ABORT_CMD_FAILED: seq_printf(m, "%d\t%d\t0x%x\n", err->type, err->cnt, err->cmd); break; @@ -1031,6 +1034,7 @@ static ssize_t sdebug_error_write(struct file *file, const char __user *ubuf, switch (inject_type) { case ERR_TMOUT_CMD: + case ERR_ABORT_CMD_FAILED: if (sscanf(buf, "%d %d %hhx", &inject->type, &inject->cnt, &inject->cmd) != 3) goto out_error; @@ -5512,9 +5516,39 @@ static void stop_all_queued(void) mutex_unlock(&sdebug_host_list_mutex); } +static int sdebug_fail_abort(struct scsi_cmnd *cmnd) +{ + struct scsi_device *sdp = cmnd->device; + struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdp->hostdata; + struct sdebug_err_inject *err; + unsigned char *cmd = cmnd->cmnd; + int ret = 0; + + if (devip == NULL) + return 0; + + rcu_read_lock(); + list_for_each_entry_rcu(err, &devip->inject_err_list, list) { + if (err->type == ERR_ABORT_CMD_FAILED && + (err->cmd == cmd[0] || err->cmd == 0xff)) { + ret = !!err->cnt; + if (err->cnt < 0) + err->cnt++; + + rcu_read_unlock(); + return ret; + } + } + rcu_read_unlock(); + + return 0; +} + static int scsi_debug_abort(struct scsi_cmnd *SCpnt) { bool ok = scsi_debug_abort_cmnd(SCpnt); + u8 *cmd = SCpnt->cmnd; + u8 opcode = cmd[0]; ++num_aborts; @@ -5523,6 +5557,12 @@ static int scsi_debug_abort(struct scsi_cmnd *SCpnt) "%s: command%s found\n", __func__, ok ? "" : " not"); + if (sdebug_fail_abort(SCpnt)) { + scmd_printk(KERN_INFO, SCpnt, "fail abort command 0x%x\n", + opcode); + return FAILED; + } + return SUCCESS; } From 0267811625e13a7743eeb6072b57509bf909f484 Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Tue, 10 Oct 2023 17:20:49 +0800 Subject: [PATCH 08/10] scsi: scsi_debug: Add new error injection type: Reset LUN failed Add error injection type 4 to make scsi_debug_device_reset() return FAILED. Fail abort command format: +--------+------+-------------------------------------------------------+ | Column | Type | Description | +--------+------+-------------------------------------------------------+ | 1 | u8 | Error type, fixed to 0x4 | +--------+------+-------------------------------------------------------+ | 2 | s32 | Error count | | | | 0: this rule will be ignored | | | | positive: the rule will always take effect | | | | negative: the rule takes effect n times where -n is | | | | the value given. Ignored after n times | +--------+------+-------------------------------------------------------+ | 3 | x8 | SCSI command opcode, 0xff for all commands | +--------+------+-------------------------------------------------------+ Examples: error=/sys/kernel/debug/scsi_debug/0:0:0:1/error echo "4 -10 0x12" > ${error} will make the device return FAILED when trying to reset LUN with inquiry command 10 times. error=/sys/kernel/debug/scsi_debug/0:0:0:1/error echo "4 -10 0xff" > ${error} will make the device return FAILED when trying to reset LUN 10 times. Usually we do not care about what command it is when trying to perform reset LUN, so 0xff could be applied. Signed-off-by: Wenchao Hao Link: https://lore.kernel.org/r/20231010092051.608007-9-haowenchao2@huawei.com Tested-by: Douglas Gilbert Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index 89f47ded1e238..e54ba0bfa0d62 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -295,6 +295,8 @@ enum sdebug_err_type { /* with errors set in scsi_cmnd */ ERR_ABORT_CMD_FAILED = 3, /* control return FAILED from */ /* scsi_debug_abort() */ + ERR_LUN_RESET_FAILED = 4, /* control return FAILED from */ + /* scsi_debug_device_reseLUN_RESET_FAILEDt() */ }; struct sdebug_err_inject { @@ -973,6 +975,7 @@ static int sdebug_error_show(struct seq_file *m, void *p) switch (err->type) { case ERR_TMOUT_CMD: case ERR_ABORT_CMD_FAILED: + case ERR_LUN_RESET_FAILED: seq_printf(m, "%d\t%d\t0x%x\n", err->type, err->cnt, err->cmd); break; @@ -1035,6 +1038,7 @@ static ssize_t sdebug_error_write(struct file *file, const char __user *ubuf, switch (inject_type) { case ERR_TMOUT_CMD: case ERR_ABORT_CMD_FAILED: + case ERR_LUN_RESET_FAILED: if (sscanf(buf, "%d %d %hhx", &inject->type, &inject->cnt, &inject->cmd) != 3) goto out_error; @@ -5586,10 +5590,40 @@ static void scsi_debug_stop_all_queued(struct scsi_device *sdp) scsi_debug_stop_all_queued_iter, sdp); } +static int sdebug_fail_lun_reset(struct scsi_cmnd *cmnd) +{ + struct scsi_device *sdp = cmnd->device; + struct sdebug_dev_info *devip = (struct sdebug_dev_info *)sdp->hostdata; + struct sdebug_err_inject *err; + unsigned char *cmd = cmnd->cmnd; + int ret = 0; + + if (devip == NULL) + return 0; + + rcu_read_lock(); + list_for_each_entry_rcu(err, &devip->inject_err_list, list) { + if (err->type == ERR_LUN_RESET_FAILED && + (err->cmd == cmd[0] || err->cmd == 0xff)) { + ret = !!err->cnt; + if (err->cnt < 0) + err->cnt++; + + rcu_read_unlock(); + return ret; + } + } + rcu_read_unlock(); + + return 0; +} + static int scsi_debug_device_reset(struct scsi_cmnd *SCpnt) { struct scsi_device *sdp = SCpnt->device; struct sdebug_dev_info *devip = sdp->hostdata; + u8 *cmd = SCpnt->cmnd; + u8 opcode = cmd[0]; ++num_dev_resets; @@ -5600,6 +5634,11 @@ static int scsi_debug_device_reset(struct scsi_cmnd *SCpnt) if (devip) set_bit(SDEBUG_UA_POR, devip->uas_bm); + if (sdebug_fail_lun_reset(SCpnt)) { + scmd_printk(KERN_INFO, SCpnt, "fail lun reset 0x%x\n", opcode); + return FAILED; + } + return SUCCESS; } From f084fe52c640775de51056670f568ec7104b922a Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Tue, 10 Oct 2023 17:20:50 +0800 Subject: [PATCH 09/10] scsi: scsi_debug: Add debugfs interface to fail target reset The interface is found at /sys/kernel/debug/scsi_debug/target/fail_reset where identifies the target to inject errors on. It's a simple bool type interface which would make this target's reset fail if set to 'Y'. Signed-off-by: Wenchao Hao Link: https://lore.kernel.org/r/20231010092051.608007-10-haowenchao2@huawei.com Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 114 +++++++++++++++++++++++++++++++++++++- 1 file changed, 113 insertions(+), 1 deletion(-) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index e54ba0bfa0d62..2743eb36c58ef 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -42,6 +42,7 @@ #include #include #include +#include #include @@ -357,6 +358,11 @@ struct sdebug_dev_info { struct list_head inject_err_list; }; +struct sdebug_target_info { + bool reset_fail; + struct dentry *debugfs_entry; +}; + struct sdebug_host_info { struct list_head host_list; int si_idx; /* sdeb_store_info (per host) xarray index */ @@ -1082,6 +1088,91 @@ static const struct file_operations sdebug_error_fops = { .release = single_release, }; +static int sdebug_target_reset_fail_show(struct seq_file *m, void *p) +{ + struct scsi_target *starget = (struct scsi_target *)m->private; + struct sdebug_target_info *targetip = + (struct sdebug_target_info *)starget->hostdata; + + if (targetip) + seq_printf(m, "%c\n", targetip->reset_fail ? 'Y' : 'N'); + + return 0; +} + +static int sdebug_target_reset_fail_open(struct inode *inode, struct file *file) +{ + return single_open(file, sdebug_target_reset_fail_show, inode->i_private); +} + +static ssize_t sdebug_target_reset_fail_write(struct file *file, + const char __user *ubuf, size_t count, loff_t *ppos) +{ + int ret; + struct scsi_target *starget = + (struct scsi_target *)file->f_inode->i_private; + struct sdebug_target_info *targetip = + (struct sdebug_target_info *)starget->hostdata; + + if (targetip) { + ret = kstrtobool_from_user(ubuf, count, &targetip->reset_fail); + return ret < 0 ? ret : count; + } + return -ENODEV; +} + +static const struct file_operations sdebug_target_reset_fail_fops = { + .open = sdebug_target_reset_fail_open, + .read = seq_read, + .write = sdebug_target_reset_fail_write, + .release = single_release, +}; + +static int sdebug_target_alloc(struct scsi_target *starget) +{ + struct sdebug_target_info *targetip; + struct dentry *dentry; + + targetip = kzalloc(sizeof(struct sdebug_target_info), GFP_KERNEL); + if (!targetip) + return -ENOMEM; + + targetip->debugfs_entry = debugfs_create_dir(dev_name(&starget->dev), + sdebug_debugfs_root); + if (IS_ERR_OR_NULL(targetip->debugfs_entry)) + pr_info("%s: failed to create debugfs directory for target %s\n", + __func__, dev_name(&starget->dev)); + + debugfs_create_file("fail_reset", 0600, targetip->debugfs_entry, starget, + &sdebug_target_reset_fail_fops); + if (IS_ERR_OR_NULL(dentry)) + pr_info("%s: failed to create fail_reset file for target %s\n", + __func__, dev_name(&starget->dev)); + + starget->hostdata = targetip; + + return 0; +} + +static void sdebug_tartget_cleanup_async(void *data, async_cookie_t cookie) +{ + struct sdebug_target_info *targetip = data; + + debugfs_remove(targetip->debugfs_entry); + kfree(targetip); +} + +static void sdebug_target_destroy(struct scsi_target *starget) +{ + struct sdebug_target_info *targetip; + + targetip = (struct sdebug_target_info *)starget->hostdata; + if (targetip) { + starget->hostdata = NULL; + async_schedule(sdebug_tartget_cleanup_async, targetip); + } +} + /* Only do the extra work involved in logical block provisioning if one or * more of the lbpu, lbpws or lbpws10 parameters are given and we are doing * real reads and writes (i.e. not skipping them for speed). @@ -5642,11 +5733,25 @@ static int scsi_debug_device_reset(struct scsi_cmnd *SCpnt) return SUCCESS; } +static int sdebug_fail_target_reset(struct scsi_cmnd *cmnd) +{ + struct scsi_target *starget = scsi_target(cmnd->device); + struct sdebug_target_info *targetip = + (struct sdebug_target_info *)starget->hostdata; + + if (targetip) + return targetip->reset_fail; + + return 0; +} + static int scsi_debug_target_reset(struct scsi_cmnd *SCpnt) { struct scsi_device *sdp = SCpnt->device; struct sdebug_host_info *sdbg_host = shost_to_sdebug_host(sdp->host); struct sdebug_dev_info *devip; + u8 *cmd = SCpnt->cmnd; + u8 opcode = cmd[0]; int k = 0; ++num_target_resets; @@ -5664,6 +5769,12 @@ static int scsi_debug_target_reset(struct scsi_cmnd *SCpnt) sdev_printk(KERN_INFO, sdp, "%s: %d device(s) found in target\n", __func__, k); + if (sdebug_fail_target_reset(SCpnt)) { + scmd_printk(KERN_INFO, SCpnt, "fail target reset 0x%x\n", + opcode); + return FAILED; + } + return SUCCESS; } @@ -8119,7 +8230,6 @@ static int sdebug_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) return 0; } - static struct scsi_host_template sdebug_driver_template = { .show_info = scsi_debug_show_info, .write_info = scsi_debug_write_info, @@ -8149,6 +8259,8 @@ static struct scsi_host_template sdebug_driver_template = { .track_queue_depth = 1, .cmd_size = sizeof(struct sdebug_scsi_cmd), .init_cmd_priv = sdebug_init_cmd_priv, + .target_alloc = sdebug_target_alloc, + .target_destroy = sdebug_target_destroy, }; static int sdebug_driver_probe(struct device *dev) From 573c2d066eb950dd9bd6e8735d3a859bbc21b3cc Mon Sep 17 00:00:00 2001 From: Wenchao Hao Date: Tue, 10 Oct 2023 17:20:51 +0800 Subject: [PATCH 10/10] scsi: scsi_debug: Add param to control sdev's allow_restart Add new module param "allow_restart" to control scsi_device's allow_restart flag. This flag determines if EH is triggered after a command completes with sense_key 0x6, ASC 0x4 and ASCQ 0x2. EH would be triggered if allow_restart=1 in this condition. The new param can be used with the error injection capability to test how commands completing with sense_key 0x6, ASC 0x4 and ASCQ 0x2 are handled. Signed-off-by: Wenchao Hao Link: https://lore.kernel.org/r/20231010092051.608007-11-haowenchao2@huawei.com Tested-by: Douglas Gilbert Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index 2743eb36c58ef..67922e2c4c191 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -843,6 +843,7 @@ static bool have_dif_prot; static bool write_since_sync; static bool sdebug_statistics = DEF_STATISTICS; static bool sdebug_wp; +static bool sdebug_allow_restart; /* Following enum: 0: no zbc, def; 1: host aware; 2: host managed */ static enum blk_zoned_model sdeb_zbc_model = BLK_ZONED_NONE; static char *sdeb_zbc_model_s; @@ -5478,6 +5479,9 @@ static int scsi_debug_slave_configure(struct scsi_device *sdp) sdp->no_uld_attach = 1; config_cdb_len(sdp); + if (sdebug_allow_restart) + sdp->allow_restart = 1; + devip->debugfs_entry = debugfs_create_dir(dev_name(&sdp->sdev_dev), sdebug_debugfs_root); if (IS_ERR_OR_NULL(devip->debugfs_entry)) @@ -6202,6 +6206,7 @@ module_param_named(zone_cap_mb, sdeb_zbc_zone_cap_mb, int, S_IRUGO); module_param_named(zone_max_open, sdeb_zbc_max_open, int, S_IRUGO); module_param_named(zone_nr_conv, sdeb_zbc_nr_conv, int, S_IRUGO); module_param_named(zone_size_mb, sdeb_zbc_zone_size_mb, int, S_IRUGO); +module_param_named(allow_restart, sdebug_allow_restart, bool, S_IRUGO | S_IWUSR); MODULE_AUTHOR("Eric Youngdale + Douglas Gilbert"); MODULE_DESCRIPTION("SCSI debug adapter driver"); @@ -6274,6 +6279,7 @@ MODULE_PARM_DESC(zone_cap_mb, "Zone capacity in MiB (def=zone size)"); MODULE_PARM_DESC(zone_max_open, "Maximum number of open zones; [0] for no limit (def=auto)"); MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones (def=1)"); MODULE_PARM_DESC(zone_size_mb, "Zone size in MiB (def=auto)"); +MODULE_PARM_DESC(allow_restart, "Set scsi_device's allow_restart flag(def=0)"); #define SDEBUG_INFO_LEN 256 static char sdebug_info[SDEBUG_INFO_LEN];