Skip to content

Commit

Permalink
RDMA/srp: Rework handling of the maximum information unit length
Browse files Browse the repository at this point in the history 
Move the maximum initiator to target information unit length parameter
from struct srp_target_port into struct srp_rdma_ch. This patch does
not change any functionality but makes the next patch easier to read.

Cc: Sergey Gorenko <sergeygo@mellanox.com>
Cc: Max Gurtovoy <maxg@mellanox.com>
Cc: Laurence Oberman <loberman@redhat.com>
Signed-off-by: Bart Van Assche <bvanassche@acm.org>
Signed-off-by: Doug Ledford <dledford@redhat.com>
  • Loading branch information
Bart Van Assche authored and Doug Ledford committed Dec 19, 2018
1 parent 4f6d498 commit 513d564
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 14 deletions.
40 changes: 27 additions & 13 deletions drivers/infiniband/ulp/srp/ib_srp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -823,7 +823,8 @@ static u8 srp_get_subnet_timeout(struct srp_host *host)
return subnet_timeout;
}

static int srp_send_req(struct srp_rdma_ch *ch, bool multich)
static int srp_send_req(struct srp_rdma_ch *ch, uint32_t max_iu_len,
bool multich)
{
struct srp_target_port *target = ch->target;
struct {
Expand Down Expand Up @@ -852,7 +853,7 @@ static int srp_send_req(struct srp_rdma_ch *ch, bool multich)

req->ib_req.opcode = SRP_LOGIN_REQ;
req->ib_req.tag = 0;
req->ib_req.req_it_iu_len = cpu_to_be32(target->max_iu_len);
req->ib_req.req_it_iu_len = cpu_to_be32(max_iu_len);
req->ib_req.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
SRP_BUF_FORMAT_INDIRECT);
req->ib_req.req_flags = (multich ? SRP_MULTICHAN_MULTI :
Expand Down Expand Up @@ -1145,7 +1146,8 @@ static int srp_connected_ch(struct srp_target_port *target)
return c;
}

static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)
static int srp_connect_ch(struct srp_rdma_ch *ch, uint32_t max_iu_len,
bool multich)
{
struct srp_target_port *target = ch->target;
int ret;
Expand All @@ -1158,7 +1160,7 @@ static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich)

while (1) {
init_completion(&ch->done);
ret = srp_send_req(ch, multich);
ret = srp_send_req(ch, max_iu_len, multich);
if (ret)
goto out;
ret = wait_for_completion_interruptible(&ch->done);
Expand Down Expand Up @@ -1344,6 +1346,16 @@ static void srp_terminate_io(struct srp_rport *rport)
}
}

/* Calculate maximum initiator to target information unit length. */
static uint32_t srp_max_it_iu_len(int cmd_sg_cnt)
{
uint32_t max_iu_len = sizeof(struct srp_cmd) + SRP_MAX_ADD_CDB_LEN +
sizeof(struct srp_indirect_buf) +
cmd_sg_cnt * sizeof(struct srp_direct_buf);

return max_iu_len;
}

/*
* It is up to the caller to ensure that srp_rport_reconnect() calls are
* serialized and that no concurrent srp_queuecommand(), srp_abort(),
Expand All @@ -1357,6 +1369,7 @@ static int srp_rport_reconnect(struct srp_rport *rport)
{
struct srp_target_port *target = rport->lld_data;
struct srp_rdma_ch *ch;
uint32_t max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt);
int i, j, ret = 0;
bool multich = false;

Expand Down Expand Up @@ -1402,7 +1415,7 @@ static int srp_rport_reconnect(struct srp_rport *rport)
ch = &target->ch[i];
if (ret)
break;
ret = srp_connect_ch(ch, multich);
ret = srp_connect_ch(ch, max_iu_len, multich);
multich = true;
}

Expand Down Expand Up @@ -2316,7 +2329,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)

req = &ch->req_ring[idx];
dev = target->srp_host->srp_dev->dev;
ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len,
ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_it_iu_len,
DMA_TO_DEVICE);

scmnd->host_scribble = (void *) req;
Expand Down Expand Up @@ -2353,7 +2366,7 @@ static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
goto err_iu;
}

ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len,
ib_dma_sync_single_for_device(dev, iu->dma, ch->max_it_iu_len,
DMA_TO_DEVICE);

if (srp_post_send(ch, iu, len)) {
Expand Down Expand Up @@ -2421,7 +2434,7 @@ static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)

for (i = 0; i < target->queue_size; ++i) {
ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
target->max_iu_len,
ch->max_it_iu_len,
GFP_KERNEL, DMA_TO_DEVICE);
if (!ch->tx_ring[i])
goto err;
Expand Down Expand Up @@ -2487,6 +2500,9 @@ static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
if (lrsp->opcode == SRP_LOGIN_RSP) {
ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
ch->req_lim = be32_to_cpu(lrsp->req_lim_delta);
ch->max_it_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt);
WARN_ON_ONCE(ch->max_it_iu_len >
be32_to_cpu(lrsp->max_it_iu_len));

/*
* Reserve credits for task management so we don't
Expand Down Expand Up @@ -3734,6 +3750,7 @@ static ssize_t srp_create_target(struct device *dev,
int ret, node_idx, node, cpu, i;
unsigned int max_sectors_per_mr, mr_per_cmd = 0;
bool multich = false;
uint32_t max_iu_len;

target_host = scsi_host_alloc(&srp_template,
sizeof (struct srp_target_port));
Expand Down Expand Up @@ -3839,10 +3856,7 @@ static ssize_t srp_create_target(struct device *dev,
target->mr_per_cmd = mr_per_cmd;
target->indirect_size = target->sg_tablesize *
sizeof (struct srp_direct_buf);
target->max_iu_len = sizeof (struct srp_cmd) +
SRP_MAX_ADD_CDB_LEN +
sizeof (struct srp_indirect_buf) +
target->cmd_sg_cnt * sizeof (struct srp_direct_buf);
max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt);

INIT_WORK(&target->tl_err_work, srp_tl_err_work);
INIT_WORK(&target->remove_work, srp_remove_work);
Expand Down Expand Up @@ -3897,7 +3911,7 @@ static ssize_t srp_create_target(struct device *dev,
if (ret)
goto err_disconnect;

ret = srp_connect_ch(ch, multich);
ret = srp_connect_ch(ch, max_iu_len, multich);
if (ret) {
char dst[64];

Expand Down
3 changes: 2 additions & 1 deletion drivers/infiniband/ulp/srp/ib_srp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ struct srp_request {
/**
* struct srp_rdma_ch
* @comp_vector: Completion vector used by this RDMA channel.
* @max_it_iu_len: Maximum initiator-to-target information unit length.
* @max_ti_iu_len: Maximum target-to-initiator information unit length.
*/
struct srp_rdma_ch {
Expand All @@ -149,6 +150,7 @@ struct srp_rdma_ch {
struct ib_fmr_pool *fmr_pool;
struct srp_fr_pool *fr_pool;
};
uint32_t max_it_iu_len;
uint32_t max_ti_iu_len;

/* Everything above this point is used in the hot path of
Expand Down Expand Up @@ -197,7 +199,6 @@ struct srp_target_port {
u32 ch_count;
u32 lkey;
enum srp_target_state state;
unsigned int max_iu_len;
unsigned int cmd_sg_cnt;
unsigned int indirect_size;
bool allow_ext_sg;
Expand Down

0 comments on commit 513d564

Please sign in to comment.