Skip to content

Commit

Permalink
scsi: lpfc: Move SCSI and NVME Stats to hardware queue structures
Browse files Browse the repository at this point in the history 
Many io statistics were being sampled and saved using adapter-based data
structures. This was creating a lot of contention and cache thrashing in
the I/O path.

Move the statistics to the hardware queue data structures.  Given the
per-queue data structures, use of atomic types is lessened.

Add new sysfs and debugfs stat routines to collate the per hardware queue
values and report at an adapter level.

Signed-off-by: Dick Kennedy <dick.kennedy@broadcom.com>
Signed-off-by: James Smart <jsmart2021@gmail.com>
Reviewed-by: Hannes Reinecke <hare@suse.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
  • Loading branch information
James Smart authored and Martin K. Petersen committed Feb 6, 2019
1 parent 63df6d6 commit 4c47efc
Show file tree
Hide file tree
Showing 10 changed files with 304 additions and 103 deletions.
9 changes: 1 addition & 8 deletions drivers/scsi/lpfc/lpfc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -479,6 +479,7 @@ struct lpfc_vport {
struct dentry *debug_disc_trc;
struct dentry *debug_nodelist;
struct dentry *debug_nvmestat;
struct dentry *debug_scsistat;
struct dentry *debug_nvmektime;
struct dentry *debug_cpucheck;
struct dentry *vport_debugfs_root;
Expand Down Expand Up @@ -946,14 +947,6 @@ struct lpfc_hba {
struct timer_list eratt_poll;
uint32_t eratt_poll_interval;

/*
* stat counters
*/
atomic_t fc4ScsiInputRequests;
atomic_t fc4ScsiOutputRequests;
atomic_t fc4ScsiControlRequests;
atomic_t fc4ScsiIoCmpls;

uint64_t bg_guard_err_cnt;
uint64_t bg_apptag_err_cnt;
uint64_t bg_reftag_err_cnt;
Expand Down
68 changes: 59 additions & 9 deletions drivers/scsi/lpfc/lpfc_attr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,6 @@
#define LPFC_MIN_MRQ_POST 512
#define LPFC_MAX_MRQ_POST 2048

#define LPFC_MAX_NVME_INFO_TMP_LEN 100
#define LPFC_NVME_INFO_MORE_STR "\nCould be more info...\n"

/*
* Write key size should be multiple of 4. If write key is changed
* make sure that library write key is also changed.
Expand Down Expand Up @@ -155,7 +152,7 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
struct lpfc_nvme_rport *rport;
struct lpfc_nodelist *ndlp;
struct nvme_fc_remote_port *nrport;
struct lpfc_nvme_ctrl_stat *cstat;
struct lpfc_fc4_ctrl_stat *cstat;
uint64_t data1, data2, data3;
uint64_t totin, totout, tot;
char *statep;
Expand Down Expand Up @@ -457,12 +454,12 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
totin = 0;
totout = 0;
for (i = 0; i < phba->cfg_hdw_queue; i++) {
cstat = &lport->cstat[i];
tot = atomic_read(&cstat->fc4NvmeIoCmpls);
cstat = &phba->sli4_hba.hdwq[i].nvme_cstat;
tot = cstat->io_cmpls;
totin += tot;
data1 = atomic_read(&cstat->fc4NvmeInputRequests);
data2 = atomic_read(&cstat->fc4NvmeOutputRequests);
data3 = atomic_read(&cstat->fc4NvmeControlRequests);
data1 = cstat->input_requests;
data2 = cstat->output_requests;
data3 = cstat->control_requests;
totout += (data1 + data2 + data3);
}
scnprintf(tmp, sizeof(tmp),
Expand Down Expand Up @@ -508,6 +505,57 @@ lpfc_nvme_info_show(struct device *dev, struct device_attribute *attr,
return len;
}

static ssize_t
lpfc_scsi_stat_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct Scsi_Host *shost = class_to_shost(dev);
struct lpfc_vport *vport = shost_priv(shost);
struct lpfc_hba *phba = vport->phba;
int len;
struct lpfc_fc4_ctrl_stat *cstat;
u64 data1, data2, data3;
u64 tot, totin, totout;
int i;
char tmp[LPFC_MAX_SCSI_INFO_TMP_LEN] = {0};

if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) ||
(phba->sli_rev != LPFC_SLI_REV4))
return 0;

scnprintf(buf, PAGE_SIZE, "SCSI HDWQ Statistics\n");

totin = 0;
totout = 0;
for (i = 0; i < phba->cfg_hdw_queue; i++) {
cstat = &phba->sli4_hba.hdwq[i].scsi_cstat;
tot = cstat->io_cmpls;
totin += tot;
data1 = cstat->input_requests;
data2 = cstat->output_requests;
data3 = cstat->control_requests;
totout += (data1 + data2 + data3);

scnprintf(tmp, sizeof(tmp), "HDWQ (%d): Rd %016llx Wr %016llx "
"IO %016llx ", i, data1, data2, data3);
if (strlcat(buf, tmp, PAGE_SIZE) >= PAGE_SIZE)
goto buffer_done;

scnprintf(tmp, sizeof(tmp), "Cmpl %016llx OutIO %016llx\n",
tot, ((data1 + data2 + data3) - tot));
if (strlcat(buf, tmp, PAGE_SIZE) >= PAGE_SIZE)
goto buffer_done;
}
scnprintf(tmp, sizeof(tmp), "Total FCP Cmpl %016llx Issue %016llx "
"OutIO %016llx\n", totin, totout, totout - totin);
strlcat(buf, tmp, PAGE_SIZE);

buffer_done:
len = strnlen(buf, PAGE_SIZE);

return len;
}

static ssize_t
lpfc_bg_info_show(struct device *dev, struct device_attribute *attr,
char *buf)
Expand Down Expand Up @@ -2573,6 +2621,7 @@ lpfc_##attr##_store(struct device *dev, struct device_attribute *attr, \


static DEVICE_ATTR(nvme_info, 0444, lpfc_nvme_info_show, NULL);
static DEVICE_ATTR(scsi_stat, 0444, lpfc_scsi_stat_show, NULL);
static DEVICE_ATTR(bg_info, S_IRUGO, lpfc_bg_info_show, NULL);
static DEVICE_ATTR(bg_guard_err, S_IRUGO, lpfc_bg_guard_err_show, NULL);
static DEVICE_ATTR(bg_apptag_err, S_IRUGO, lpfc_bg_apptag_err_show, NULL);
Expand Down Expand Up @@ -5642,6 +5691,7 @@ LPFC_ATTR_RW(enable_dpp, 1, 0, 1, "Enable Direct Packet Push");

struct device_attribute *lpfc_hba_attrs[] = {
&dev_attr_nvme_info,
&dev_attr_scsi_stat,
&dev_attr_bg_info,
&dev_attr_bg_guard_err,
&dev_attr_bg_apptag_err,
Expand Down
158 changes: 150 additions & 8 deletions drivers/scsi/lpfc/lpfc_debugfs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -840,7 +840,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
struct lpfc_nvmet_tgtport *tgtp;
struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
struct nvme_fc_local_port *localport;
struct lpfc_nvme_ctrl_stat *cstat;
struct lpfc_fc4_ctrl_stat *cstat;
struct lpfc_nvme_lport *lport;
uint64_t data1, data2, data3;
uint64_t tot, totin, totout;
Expand Down Expand Up @@ -979,7 +979,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
return len;

len += snprintf(buf + len, size - len,
"\nNVME Lport Statistics\n");
"\nNVME HDWQ Statistics\n");

len += snprintf(buf + len, size - len,
"LS: Xmt %016x Cmpl %016x\n",
Expand All @@ -993,20 +993,20 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
totin = 0;
totout = 0;
for (i = 0; i < phba->cfg_hdw_queue; i++) {
cstat = &lport->cstat[i];
tot = atomic_read(&cstat->fc4NvmeIoCmpls);
cstat = &phba->sli4_hba.hdwq[i].nvme_cstat;
tot = cstat->io_cmpls;
totin += tot;
data1 = atomic_read(&cstat->fc4NvmeInputRequests);
data2 = atomic_read(&cstat->fc4NvmeOutputRequests);
data3 = atomic_read(&cstat->fc4NvmeControlRequests);
data1 = cstat->input_requests;
data2 = cstat->output_requests;
data3 = cstat->control_requests;
totout += (data1 + data2 + data3);

/* Limit to 32, debugfs display buffer limitation */
if (i >= 32)
continue;

len += snprintf(buf + len, PAGE_SIZE - len,
"FCP (%d): Rd %016llx Wr %016llx "
"HDWQ (%d): Rd %016llx Wr %016llx "
"IO %016llx ",
i, data1, data2, data3);
len += snprintf(buf + len, PAGE_SIZE - len,
Expand Down Expand Up @@ -1046,6 +1046,66 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size)
return len;
}

/**
* lpfc_debugfs_scsistat_data - Dump target node list to a buffer
* @vport: The vport to gather target node info from.
* @buf: The buffer to dump log into.
* @size: The maximum amount of data to process.
*
* Description:
* This routine dumps the SCSI statistics associated with @vport
*
* Return Value:
* This routine returns the amount of bytes that were dumped into @buf and will
* not exceed @size.
**/
static int
lpfc_debugfs_scsistat_data(struct lpfc_vport *vport, char *buf, int size)
{
int len;
struct lpfc_hba *phba = vport->phba;
struct lpfc_fc4_ctrl_stat *cstat;
u64 data1, data2, data3;
u64 tot, totin, totout;
int i;
char tmp[LPFC_MAX_SCSI_INFO_TMP_LEN] = {0};

if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) ||
(phba->sli_rev != LPFC_SLI_REV4))
return 0;

scnprintf(buf, size, "SCSI HDWQ Statistics\n");

totin = 0;
totout = 0;
for (i = 0; i < phba->cfg_hdw_queue; i++) {
cstat = &phba->sli4_hba.hdwq[i].scsi_cstat;
tot = cstat->io_cmpls;
totin += tot;
data1 = cstat->input_requests;
data2 = cstat->output_requests;
data3 = cstat->control_requests;
totout += (data1 + data2 + data3);

scnprintf(tmp, sizeof(tmp), "HDWQ (%d): Rd %016llx Wr %016llx "
"IO %016llx ", i, data1, data2, data3);
if (strlcat(buf, tmp, size) >= size)
goto buffer_done;

scnprintf(tmp, sizeof(tmp), "Cmpl %016llx OutIO %016llx\n",
tot, ((data1 + data2 + data3) - tot));
if (strlcat(buf, tmp, size) >= size)
goto buffer_done;
}
scnprintf(tmp, sizeof(tmp), "Total FCP Cmpl %016llx Issue %016llx "
"OutIO %016llx\n", totin, totout, totout - totin);
strlcat(buf, tmp, size);

buffer_done:
len = strnlen(buf, size);

return len;
}

/**
* lpfc_debugfs_nvmektime_data - Dump target node list to a buffer
Expand Down Expand Up @@ -2211,6 +2271,64 @@ lpfc_debugfs_nvmestat_write(struct file *file, const char __user *buf,
return nbytes;
}

static int
lpfc_debugfs_scsistat_open(struct inode *inode, struct file *file)
{
struct lpfc_vport *vport = inode->i_private;
struct lpfc_debug *debug;
int rc = -ENOMEM;

debug = kmalloc(sizeof(*debug), GFP_KERNEL);
if (!debug)
goto out;

/* Round to page boundary */
debug->buffer = kzalloc(LPFC_SCSISTAT_SIZE, GFP_KERNEL);
if (!debug->buffer) {
kfree(debug);
goto out;
}

debug->len = lpfc_debugfs_scsistat_data(vport, debug->buffer,
LPFC_SCSISTAT_SIZE);

debug->i_private = inode->i_private;
file->private_data = debug;

rc = 0;
out:
return rc;
}

static ssize_t
lpfc_debugfs_scsistat_write(struct file *file, const char __user *buf,
size_t nbytes, loff_t *ppos)
{
struct lpfc_debug *debug = file->private_data;
struct lpfc_vport *vport = (struct lpfc_vport *)debug->i_private;
struct lpfc_hba *phba = vport->phba;
char mybuf[6] = {0};
int i;

/* Protect copy from user */
if (!access_ok(buf, nbytes))
return -EFAULT;

if (copy_from_user(mybuf, buf, (nbytes >= sizeof(mybuf)) ?
(sizeof(mybuf) - 1) : nbytes))
return -EFAULT;

if ((strncmp(&mybuf[0], "reset", strlen("reset")) == 0) ||
(strncmp(&mybuf[0], "zero", strlen("zero")) == 0)) {
for (i = 0; i < phba->cfg_hdw_queue; i++) {
memset(&phba->sli4_hba.hdwq[i].scsi_cstat, 0,
sizeof(phba->sli4_hba.hdwq[i].scsi_cstat));
}
}

return nbytes;
}

static int
lpfc_debugfs_nvmektime_open(struct inode *inode, struct file *file)
{
Expand Down Expand Up @@ -4972,6 +5090,16 @@ static const struct file_operations lpfc_debugfs_op_nvmestat = {
.release = lpfc_debugfs_release,
};

#undef lpfc_debugfs_op_scsistat
static const struct file_operations lpfc_debugfs_op_scsistat = {
.owner = THIS_MODULE,
.open = lpfc_debugfs_scsistat_open,
.llseek = lpfc_debugfs_lseek,
.read = lpfc_debugfs_read,
.write = lpfc_debugfs_scsistat_write,
.release = lpfc_debugfs_release,
};

#undef lpfc_debugfs_op_nvmektime
static const struct file_operations lpfc_debugfs_op_nvmektime = {
.owner = THIS_MODULE,
Expand Down Expand Up @@ -5612,6 +5740,17 @@ lpfc_debugfs_initialize(struct lpfc_vport *vport)
vport->vport_debugfs_root,
vport, &lpfc_debugfs_op_nvmestat);

snprintf(name, sizeof(name), "scsistat");
vport->debug_scsistat =
debugfs_create_file(name, 0644,
vport->vport_debugfs_root,
vport, &lpfc_debugfs_op_scsistat);
if (!vport->debug_scsistat) {
lpfc_printf_vlog(vport, KERN_ERR, LOG_INIT,
"0811 Cannot create debugfs scsistat\n");
goto debug_failed;
}

snprintf(name, sizeof(name), "nvmektime");
vport->debug_nvmektime =
debugfs_create_file(name, 0644,
Expand Down Expand Up @@ -5750,6 +5889,9 @@ lpfc_debugfs_terminate(struct lpfc_vport *vport)
debugfs_remove(vport->debug_nvmestat); /* nvmestat */
vport->debug_nvmestat = NULL;

debugfs_remove(vport->debug_scsistat); /* scsistat */
vport->debug_scsistat = NULL;

debugfs_remove(vport->debug_nvmektime); /* nvmektime */
vport->debug_nvmektime = NULL;

Expand Down
3 changes: 3 additions & 0 deletions drivers/scsi/lpfc/lpfc_debugfs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@
#define LPFC_CPUCHECK_SIZE 8192
#define LPFC_NVMEIO_TRC_SIZE 8192

/* scsistat output buffer size */
#define LPFC_SCSISTAT_SIZE 8192

#define LPFC_DEBUG_OUT_LINE_SZ 80

/*
Expand Down
Loading

0 comments on commit 4c47efc

Please sign in to comment.