Skip to content

Commit

Permalink
scsi: cxlflash: Add hardware queues attribute
Browse files Browse the repository at this point in the history 
As staging for supporting multiple hardware queues, add an attribute to show
and set the current number of hardware queues for the host. Support specifying
a hard limit or a CPU affinitized value. This will allow the number of
hardware queues to be tuned by a system administrator.

Signed-off-by: Matthew R. Ochs <mrochs@linux.vnet.ibm.com>
Signed-off-by: Uma Krishnan <ukrishn@linux.vnet.ibm.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
  • Loading branch information
Matthew R. Ochs authored and Martin K. Petersen committed Apr 14, 2017
1 parent bfc0bab commit 3065267
Show file tree
Hide file tree
Showing 2 changed files with 106 additions and 16 deletions.
10 changes: 7 additions & 3 deletions drivers/scsi/cxlflash/common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ extern const struct file_operations cxlflash_cxl_fops;
/* SQ for master issued cmds */
#define NUM_SQ_ENTRY CXLFLASH_MAX_CMDS

#define CXLFLASH_NUM_HWQS 1
/* Hardware queue definitions */
#define CXLFLASH_DEF_HWQS 1
#define CXLFLASH_MAX_HWQS 8
#define PRIMARY_HWQ 0


Expand Down Expand Up @@ -201,7 +203,7 @@ struct hwq {
} __aligned(cache_line_size());

struct afu {
struct hwq hwqs[CXLFLASH_NUM_HWQS];
struct hwq hwqs[CXLFLASH_MAX_HWQS];
int (*send_cmd)(struct afu *, struct afu_cmd *);
void (*context_reset)(struct afu_cmd *);

Expand All @@ -211,6 +213,8 @@ struct afu {
atomic_t cmds_active; /* Number of currently active AFU commands */
u64 hb;
u32 internal_lun; /* User-desired LUN mode for this AFU */
u32 num_hwqs; /* Number of hardware queues */
u32 desired_hwqs; /* Desired h/w queues, effective on AFU reset */

char version[16];
u64 interface_version;
Expand All @@ -221,7 +225,7 @@ struct afu {

static inline struct hwq *get_hwq(struct afu *afu, u32 index)
{
WARN_ON(index >= CXLFLASH_NUM_HWQS);
WARN_ON(index >= CXLFLASH_MAX_HWQS);

return &afu->hwqs[index];
}
Expand Down
112 changes: 99 additions & 13 deletions drivers/scsi/cxlflash/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -566,7 +566,7 @@ static void stop_afu(struct cxlflash_cfg *cfg)
ssleep(1);

if (afu_is_irqpoll_enabled(afu)) {
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
for (i = 0; i < afu->num_hwqs; i++) {
hwq = get_hwq(afu, i);

irq_poll_disable(&hwq->irqpoll);
Expand Down Expand Up @@ -676,13 +676,13 @@ static void term_afu(struct cxlflash_cfg *cfg)
* 2) Unmap the problem state area
* 3) Stop each master context
*/
for (k = CXLFLASH_NUM_HWQS - 1; k >= 0; k--)
for (k = cfg->afu->num_hwqs - 1; k >= 0; k--)
term_intr(cfg, UNMAP_THREE, k);

if (cfg->afu)
stop_afu(cfg);

for (k = CXLFLASH_NUM_HWQS - 1; k >= 0; k--)
for (k = cfg->afu->num_hwqs - 1; k >= 0; k--)
term_mc(cfg, k);

dev_dbg(dev, "%s: returning\n", __func__);
Expand Down Expand Up @@ -823,6 +823,7 @@ static int alloc_mem(struct cxlflash_cfg *cfg)
goto out;
}
cfg->afu->parent = cfg;
cfg->afu->desired_hwqs = CXLFLASH_DEF_HWQS;
cfg->afu->afu_map = NULL;
out:
return rc;
Expand Down Expand Up @@ -1116,7 +1117,7 @@ static void afu_err_intr_init(struct afu *afu)
/* IOARRIN yet), so there is nothing to clear. */

/* set LISN#, it is always sent to the context that wrote IOARRIN */
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
for (i = 0; i < afu->num_hwqs; i++) {
hwq = get_hwq(afu, i);

writeq_be(SISL_MSI_SYNC_ERROR, &hwq->host_map->ctx_ctrl);
Expand Down Expand Up @@ -1551,7 +1552,7 @@ static void init_pcr(struct cxlflash_cfg *cfg)
}

/* Copy frequently used fields into hwq */
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
for (i = 0; i < afu->num_hwqs; i++) {
hwq = get_hwq(afu, i);

hwq->ctx_hndl = (u16) cxl_process_element(hwq->ctx);
Expand Down Expand Up @@ -1586,7 +1587,7 @@ static int init_global(struct cxlflash_cfg *cfg)
}

/* Set up RRQ and SQ in HWQ for master issued cmds */
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
for (i = 0; i < afu->num_hwqs; i++) {
hwq = get_hwq(afu, i);
hmap = hwq->host_map;

Expand Down Expand Up @@ -1640,7 +1641,7 @@ static int init_global(struct cxlflash_cfg *cfg)
/* Set up master's own CTX_CAP to allow real mode, host translation */
/* tables, afu cmds and read/write GSCSI cmds. */
/* First, unlock ctx_cap write by reading mbox */
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
for (i = 0; i < afu->num_hwqs; i++) {
hwq = get_hwq(afu, i);

(void)readq_be(&hwq->ctrl_map->mbox_r); /* unlock ctx_cap */
Expand Down Expand Up @@ -1670,7 +1671,7 @@ static int start_afu(struct cxlflash_cfg *cfg)
init_pcr(cfg);

/* Initialize each HWQ */
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
for (i = 0; i < afu->num_hwqs; i++) {
hwq = get_hwq(afu, i);

/* After an AFU reset, RRQ entries are stale, clear them */
Expand Down Expand Up @@ -1888,7 +1889,8 @@ static int init_afu(struct cxlflash_cfg *cfg)

cxl_perst_reloads_same_image(cfg->cxl_afu, true);

for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
afu->num_hwqs = afu->desired_hwqs;
for (i = 0; i < afu->num_hwqs; i++) {
rc = init_mc(cfg, i);
if (rc) {
dev_err(dev, "%s: init_mc failed rc=%d index=%d\n",
Expand Down Expand Up @@ -1939,7 +1941,7 @@ static int init_afu(struct cxlflash_cfg *cfg)
}

afu_err_intr_init(cfg->afu);
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
for (i = 0; i < afu->num_hwqs; i++) {
hwq = get_hwq(afu, i);

spin_lock_init(&hwq->rrin_slock);
Expand All @@ -1953,7 +1955,7 @@ static int init_afu(struct cxlflash_cfg *cfg)
return rc;

err1:
for (i = CXLFLASH_NUM_HWQS - 1; i >= 0; i--) {
for (i = afu->num_hwqs - 1; i >= 0; i--) {
term_intr(cfg, UNMAP_THREE, i);
term_mc(cfg, i);
}
Expand Down Expand Up @@ -2550,7 +2552,7 @@ static ssize_t irqpoll_weight_store(struct device *dev,
}

if (afu_is_irqpoll_enabled(afu)) {
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
for (i = 0; i < afu->num_hwqs; i++) {
hwq = get_hwq(afu, i);

irq_poll_disable(&hwq->irqpoll);
Expand All @@ -2560,7 +2562,7 @@ static ssize_t irqpoll_weight_store(struct device *dev,
afu->irqpoll_weight = weight;

if (weight > 0) {
for (i = 0; i < CXLFLASH_NUM_HWQS; i++) {
for (i = 0; i < afu->num_hwqs; i++) {
hwq = get_hwq(afu, i);

irq_poll_init(&hwq->irqpoll, weight, cxlflash_irqpoll);
Expand All @@ -2570,6 +2572,88 @@ static ssize_t irqpoll_weight_store(struct device *dev,
return count;
}

/**
* num_hwqs_show() - presents the number of hardware queues for the host
* @dev: Generic device associated with the host.
* @attr: Device attribute representing the number of hardware queues.
* @buf: Buffer of length PAGE_SIZE to report back the number of hardware
* queues in ASCII.
*
* Return: The size of the ASCII string returned in @buf.
*/
static ssize_t num_hwqs_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev));
struct afu *afu = cfg->afu;

return scnprintf(buf, PAGE_SIZE, "%u\n", afu->num_hwqs);
}

/**
* num_hwqs_store() - sets the number of hardware queues for the host
* @dev: Generic device associated with the host.
* @attr: Device attribute representing the number of hardware queues.
* @buf: Buffer of length PAGE_SIZE containing the number of hardware
* queues in ASCII.
* @count: Length of data resizing in @buf.
*
* n > 0: num_hwqs = n
* n = 0: num_hwqs = num_online_cpus()
* n < 0: num_online_cpus() / abs(n)
*
* Return: The size of the ASCII string returned in @buf.
*/
static ssize_t num_hwqs_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct cxlflash_cfg *cfg = shost_priv(class_to_shost(dev));
struct afu *afu = cfg->afu;
int rc;
int nhwqs, num_hwqs;

rc = kstrtoint(buf, 10, &nhwqs);
if (rc)
return -EINVAL;

if (nhwqs >= 1)
num_hwqs = nhwqs;
else if (nhwqs == 0)
num_hwqs = num_online_cpus();
else
num_hwqs = num_online_cpus() / abs(nhwqs);

afu->desired_hwqs = min(num_hwqs, CXLFLASH_MAX_HWQS);
WARN_ON_ONCE(afu->desired_hwqs == 0);

retry:
switch (cfg->state) {
case STATE_NORMAL:
cfg->state = STATE_RESET;
drain_ioctls(cfg);
cxlflash_mark_contexts_error(cfg);
rc = afu_reset(cfg);
if (rc)
cfg->state = STATE_FAILTERM;
else
cfg->state = STATE_NORMAL;
wake_up_all(&cfg->reset_waitq);
break;
case STATE_RESET:
wait_event(cfg->reset_waitq, cfg->state != STATE_RESET);
if (cfg->state == STATE_NORMAL)
goto retry;
default:
/* Ideally should not happen */
dev_err(dev, "%s: Device is not ready, state=%d\n",
__func__, cfg->state);
break;
}

return count;
}

/**
* mode_show() - presents the current mode of the device
* @dev: Generic device associated with the device.
Expand Down Expand Up @@ -2601,6 +2685,7 @@ static DEVICE_ATTR_RO(port1_lun_table);
static DEVICE_ATTR_RO(port2_lun_table);
static DEVICE_ATTR_RO(port3_lun_table);
static DEVICE_ATTR_RW(irqpoll_weight);
static DEVICE_ATTR_RW(num_hwqs);

static struct device_attribute *cxlflash_host_attrs[] = {
&dev_attr_port0,
Expand All @@ -2614,6 +2699,7 @@ static struct device_attribute *cxlflash_host_attrs[] = {
&dev_attr_port2_lun_table,
&dev_attr_port3_lun_table,
&dev_attr_irqpoll_weight,
&dev_attr_num_hwqs,
NULL
};

Expand Down

0 comments on commit 3065267

Please sign in to comment.