Skip to content

Commit

Permalink
IB/srp: allow sg_tablesize to be set for each target
Browse files Browse the repository at this point in the history 
Different configurations of target software allow differing max sizes of
the command IU. Allowing this to be changed per-target allows all
targets on an initiator to get an optimal setting.

We deprecate srp_sg_tablesize and replace it with cmd_sg_entries in
preparation for allowing more indirect descriptors than can fit in the
IU.

Signed-off-by: David Dillow <dillowda@ornl.gov>
  • Loading branch information
David Dillow committed Mar 15, 2011
1 parent 961e0be commit 4924864
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 24 deletions.
81 changes: 57 additions & 24 deletions drivers/infiniband/ulp/srp/ib_srp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,16 @@ MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator "
"v" DRV_VERSION " (" DRV_RELDATE ")");
MODULE_LICENSE("Dual BSD/GPL");

static int srp_sg_tablesize = SRP_DEF_SG_TABLESIZE;
static int srp_max_iu_len;
static unsigned int srp_sg_tablesize;
static unsigned int cmd_sg_entries;
static int topspin_workarounds = 1;

module_param(srp_sg_tablesize, int, 0444);
MODULE_PARM_DESC(srp_sg_tablesize,
"Max number of gather/scatter entries per I/O (default is 12, max 255)");
module_param(srp_sg_tablesize, uint, 0444);
MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");

static int topspin_workarounds = 1;
module_param(cmd_sg_entries, uint, 0444);
MODULE_PARM_DESC(cmd_sg_entries,
"Default number of gather/scatter entries in the SRP command (default is 12, max 255)");

module_param(topspin_workarounds, int, 0444);
MODULE_PARM_DESC(topspin_workarounds,
Expand Down Expand Up @@ -364,7 +366,7 @@ static int srp_send_req(struct srp_target_port *target)

req->priv.opcode = SRP_LOGIN_REQ;
req->priv.tag = 0;
req->priv.req_it_iu_len = cpu_to_be32(srp_max_iu_len);
req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len);
req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
SRP_BUF_FORMAT_INDIRECT);
/*
Expand Down Expand Up @@ -1125,7 +1127,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
spin_unlock_irqrestore(&target->lock, flags);

dev = target->srp_host->srp_dev->dev;
ib_dma_sync_single_for_cpu(dev, iu->dma, srp_max_iu_len,
ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
DMA_TO_DEVICE);

scmnd->result = 0;
Expand All @@ -1149,7 +1151,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
goto err_iu;
}

ib_dma_sync_single_for_device(dev, iu->dma, srp_max_iu_len,
ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
DMA_TO_DEVICE);

if (srp_post_send(target, iu, len)) {
Expand Down Expand Up @@ -1189,7 +1191,7 @@ static int srp_alloc_iu_bufs(struct srp_target_port *target)

for (i = 0; i < SRP_SQ_SIZE; ++i) {
target->tx_ring[i] = srp_alloc_iu(target->srp_host,
srp_max_iu_len,
target->max_iu_len,
GFP_KERNEL, DMA_TO_DEVICE);
if (!target->tx_ring[i])
goto err;
Expand Down Expand Up @@ -1645,6 +1647,14 @@ static ssize_t show_local_ib_device(struct device *dev,
return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name);
}

static ssize_t show_cmd_sg_entries(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct srp_target_port *target = host_to_target(class_to_shost(dev));

return sprintf(buf, "%u\n", target->cmd_sg_cnt);
}

static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL);
static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL);
static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL);
Expand All @@ -1655,6 +1665,7 @@ static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL);
static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL);
static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL);
static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL);

static struct device_attribute *srp_host_attrs[] = {
&dev_attr_id_ext,
Expand All @@ -1667,6 +1678,7 @@ static struct device_attribute *srp_host_attrs[] = {
&dev_attr_zero_req_lim,
&dev_attr_local_ib_port,
&dev_attr_local_ib_device,
&dev_attr_cmd_sg_entries,
NULL
};

Expand All @@ -1679,6 +1691,7 @@ static struct scsi_host_template srp_template = {
.eh_abort_handler = srp_abort,
.eh_device_reset_handler = srp_reset_device,
.eh_host_reset_handler = srp_reset_host,
.sg_tablesize = SRP_DEF_SG_TABLESIZE,
.can_queue = SRP_CMD_SQ_SIZE,
.this_id = -1,
.cmd_per_lun = SRP_CMD_SQ_SIZE,
Expand Down Expand Up @@ -1750,6 +1763,7 @@ enum {
SRP_OPT_MAX_CMD_PER_LUN = 1 << 6,
SRP_OPT_IO_CLASS = 1 << 7,
SRP_OPT_INITIATOR_EXT = 1 << 8,
SRP_OPT_CMD_SG_ENTRIES = 1 << 9,
SRP_OPT_ALL = (SRP_OPT_ID_EXT |
SRP_OPT_IOC_GUID |
SRP_OPT_DGID |
Expand All @@ -1767,6 +1781,7 @@ static const match_table_t srp_opt_tokens = {
{ SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" },
{ SRP_OPT_IO_CLASS, "io_class=%x" },
{ SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" },
{ SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" },
{ SRP_OPT_ERR, NULL }
};

Expand Down Expand Up @@ -1894,6 +1909,14 @@ static int srp_parse_options(const char *buf, struct srp_target_port *target)
kfree(p);
break;

case SRP_OPT_CMD_SG_ENTRIES:
if (match_int(args, &token) || token < 1 || token > 255) {
printk(KERN_WARNING PFX "bad max cmd_sg_entries parameter '%s'\n", p);
goto out;
}
target->cmd_sg_cnt = token;
break;

default:
printk(KERN_WARNING PFX "unknown parameter or missing value "
"'%s' in target creation request\n", p);
Expand Down Expand Up @@ -1932,17 +1955,18 @@ static ssize_t srp_create_target(struct device *dev,
if (!target_host)
return -ENOMEM;

target_host->transportt = ib_srp_transport_template;
target_host->transportt = ib_srp_transport_template;
target_host->max_lun = SRP_MAX_LUN;
target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;

target = host_to_target(target_host);

target->io_class = SRP_REV16A_IB_IO_CLASS;
target->scsi_host = target_host;
target->srp_host = host;
target->lkey = host->srp_dev->mr->lkey;
target->rkey = host->srp_dev->mr->rkey;
target->io_class = SRP_REV16A_IB_IO_CLASS;
target->scsi_host = target_host;
target->srp_host = host;
target->lkey = host->srp_dev->mr->lkey;
target->rkey = host->srp_dev->mr->rkey;
target->cmd_sg_cnt = cmd_sg_entries;

spin_lock_init(&target->lock);
INIT_LIST_HEAD(&target->free_tx);
Expand All @@ -1956,6 +1980,11 @@ static ssize_t srp_create_target(struct device *dev,
if (ret)
goto err;

target_host->sg_tablesize = target->cmd_sg_cnt;
target->max_iu_len = sizeof (struct srp_cmd) +
sizeof (struct srp_indirect_buf) +
target->cmd_sg_cnt * sizeof (struct srp_direct_buf);

ib_query_gid(host->srp_dev->dev, host->port, 0, &target->path.sgid);

shost_printk(KERN_DEBUG, target->scsi_host, PFX
Expand Down Expand Up @@ -2217,21 +2246,25 @@ static int __init srp_init_module(void)

BUILD_BUG_ON(FIELD_SIZEOF(struct ib_wc, wr_id) < sizeof(void *));

if (srp_sg_tablesize > 255) {
printk(KERN_WARNING PFX "Clamping srp_sg_tablesize to 255\n");
srp_sg_tablesize = 255;
if (srp_sg_tablesize) {
printk(KERN_WARNING PFX "srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
if (!cmd_sg_entries)
cmd_sg_entries = srp_sg_tablesize;
}

if (!cmd_sg_entries)
cmd_sg_entries = SRP_DEF_SG_TABLESIZE;

if (cmd_sg_entries > 255) {
printk(KERN_WARNING PFX "Clamping cmd_sg_entries to 255\n");
cmd_sg_entries = 255;
}

ib_srp_transport_template =
srp_attach_transport(&ib_srp_transport_functions);
if (!ib_srp_transport_template)
return -ENOMEM;

srp_template.sg_tablesize = srp_sg_tablesize;
srp_max_iu_len = (sizeof (struct srp_cmd) +
sizeof (struct srp_indirect_buf) +
srp_sg_tablesize * 16);

ret = class_register(&srp_class);
if (ret) {
printk(KERN_ERR PFX "couldn't register class infiniband_srp\n");
Expand Down
2 changes: 2 additions & 0 deletions drivers/infiniband/ulp/srp/ib_srp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,8 @@ struct srp_target_port {
u32 lkey;
u32 rkey;
enum srp_target_state state;
unsigned int max_iu_len;
unsigned int cmd_sg_cnt;

/* Everything above this point is used in the hot path of
* command processing. Try to keep them packed into cachelines.
Expand Down

0 comments on commit 4924864

Please sign in to comment.