Skip to content

Commit

Permalink
IB/hfi1: Work request processing for fast register mr and invalidate
Browse files Browse the repository at this point in the history 
In order to support extended memory management support, add send side
processing of work requests of type IB_WR_REG_MR, IB_WR_LOCAL_INV, and
IB_WR_SEND_WITH_INV. The first two are local operations and are supported
for both RC and UC. Send with invalidate is only supported for RC because
the corresponding IB opcodes are not defined for UC.

Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Reviewed-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Jianxin Xiong <jianxin.xiong@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
  • Loading branch information
Jianxin Xiong authored and Doug Ledford committed Aug 2, 2016
1 parent a2df0c8 commit 0db3dfa
Show file tree
Hide file tree
Showing 4 changed files with 98 additions and 3 deletions.
46 changes: 44 additions & 2 deletions drivers/infiniband/hw/hfi1/rc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,7 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
char newreq;
int middle = 0;
int delta;
int err;

ps->s_txreq = get_txreq(ps->dev, qp);
if (IS_ERR(ps->s_txreq))
Expand Down Expand Up @@ -477,6 +478,35 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
qp->s_flags |= RVT_S_WAIT_FENCE;
goto bail;
}
/*
* Local operations are processed immediately
* after all prior requests have completed
*/
if (wqe->wr.opcode == IB_WR_REG_MR ||
wqe->wr.opcode == IB_WR_LOCAL_INV) {
if (qp->s_last != qp->s_cur)
goto bail;
if (++qp->s_cur == qp->s_size)
qp->s_cur = 0;
if (++qp->s_tail == qp->s_size)
qp->s_tail = 0;
if (wqe->wr.opcode == IB_WR_REG_MR)
err = rvt_fast_reg_mr(
qp, wqe->reg_wr.mr,
wqe->reg_wr.key,
wqe->reg_wr.access);
else
err = rvt_invalidate_rkey(
qp,
wqe->wr.ex.invalidate_rkey);
hfi1_send_complete(qp, wqe,
err ? IB_WC_LOC_PROT_ERR
: IB_WC_SUCCESS);
atomic_dec(&qp->local_ops_pending);
qp->s_hdrwords = 0;
goto done_free_tx;
}

newreq = 1;
qp->s_psn = wqe->psn;
}
Expand All @@ -491,6 +521,7 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
switch (wqe->wr.opcode) {
case IB_WR_SEND:
case IB_WR_SEND_WITH_IMM:
case IB_WR_SEND_WITH_INV:
/* If no credit, return. */
if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {
Expand All @@ -504,11 +535,17 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
}
if (wqe->wr.opcode == IB_WR_SEND) {
qp->s_state = OP(SEND_ONLY);
} else {
} else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
/* Immediate data comes after the BTH */
ohdr->u.imm_data = wqe->wr.ex.imm_data;
hwords += 1;
} else {
qp->s_state = OP(SEND_ONLY_WITH_INVALIDATE);
/* Invalidate rkey comes after the BTH */
ohdr->u.ieth = cpu_to_be32(
wqe->wr.ex.invalidate_rkey);
hwords += 1;
}
if (wqe->wr.send_flags & IB_SEND_SOLICITED)
bth0 |= IB_BTH_SOLICITED;
Expand Down Expand Up @@ -671,11 +708,16 @@ int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
}
if (wqe->wr.opcode == IB_WR_SEND) {
qp->s_state = OP(SEND_LAST);
} else {
} else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
/* Immediate data comes after the BTH */
ohdr->u.imm_data = wqe->wr.ex.imm_data;
hwords += 1;
} else {
qp->s_state = OP(SEND_LAST_WITH_INVALIDATE);
/* invalidate data comes after the BTH */
ohdr->u.ieth = cpu_to_be32(wqe->wr.ex.invalidate_rkey);
hwords += 1;
}
if (wqe->wr.send_flags & IB_SEND_SOLICITED)
bth0 |= IB_BTH_SOLICITED;
Expand Down
26 changes: 26 additions & 0 deletions drivers/infiniband/hw/hfi1/ruc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,7 @@ static void ruc_loopback(struct rvt_qp *sqp)
int ret;
int copy_last = 0;
u32 to;
int local_ops = 0;

rcu_read_lock();

Expand Down Expand Up @@ -440,11 +441,32 @@ static void ruc_loopback(struct rvt_qp *sqp)
sqp->s_sge.num_sge = wqe->wr.num_sge;
sqp->s_len = wqe->length;
switch (wqe->wr.opcode) {
case IB_WR_REG_MR:
if (rvt_fast_reg_mr(sqp, wqe->reg_wr.mr, wqe->reg_wr.key,
wqe->reg_wr.access))
send_status = IB_WC_LOC_PROT_ERR;
local_ops = 1;
goto send_comp;

case IB_WR_LOCAL_INV:
if (rvt_invalidate_rkey(sqp, wqe->wr.ex.invalidate_rkey))
send_status = IB_WC_LOC_PROT_ERR;
local_ops = 1;
goto send_comp;

case IB_WR_SEND_WITH_INV:
if (!rvt_invalidate_rkey(qp, wqe->wr.ex.invalidate_rkey)) {
wc.wc_flags = IB_WC_WITH_INVALIDATE;
wc.ex.invalidate_rkey = wqe->wr.ex.invalidate_rkey;
}
goto send;

case IB_WR_SEND_WITH_IMM:
wc.wc_flags = IB_WC_WITH_IMM;
wc.ex.imm_data = wqe->wr.ex.imm_data;
/* FALLTHROUGH */
case IB_WR_SEND:
send:
ret = hfi1_rvt_get_rwqe(qp, 0);
if (ret < 0)
goto op_err;
Expand Down Expand Up @@ -583,6 +605,10 @@ static void ruc_loopback(struct rvt_qp *sqp)
flush_send:
sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
hfi1_send_complete(sqp, wqe, send_status);
if (local_ops) {
atomic_dec(&sqp->local_ops_pending);
local_ops = 0;
}
goto again;

rnr_nak:
Expand Down
24 changes: 24 additions & 0 deletions drivers/infiniband/hw/hfi1/uc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
u32 len;
u32 pmtu = qp->pmtu;
int middle = 0;
int err;

ps->s_txreq = get_txreq(ps->dev, qp);
if (IS_ERR(ps->s_txreq))
Expand Down Expand Up @@ -118,6 +119,29 @@ int hfi1_make_uc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
clear_ahg(qp);
goto bail;
}
/*
* Local operations are processed immediately
* after all prior requests have completed.
*/
if (wqe->wr.opcode == IB_WR_REG_MR ||
wqe->wr.opcode == IB_WR_LOCAL_INV) {
if (qp->s_last != qp->s_cur)
goto bail;
if (++qp->s_cur == qp->s_size)
qp->s_cur = 0;
if (wqe->wr.opcode == IB_WR_REG_MR)
err = rvt_fast_reg_mr(qp, wqe->reg_wr.mr,
wqe->reg_wr.key,
wqe->reg_wr.access);
else
err = rvt_invalidate_rkey(
qp, wqe->wr.ex.invalidate_rkey);
hfi1_send_complete(qp, wqe, err ? IB_WC_LOC_PROT_ERR
: IB_WC_SUCCESS);
atomic_dec(&qp->local_ops_pending);
qp->s_hdrwords = 0;
goto done_free_tx;
}
/*
* Start a new request.
*/
Expand Down
5 changes: 4 additions & 1 deletion drivers/infiniband/hw/hfi1/verbs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,10 @@ const enum ib_wc_opcode ib_hfi1_wc_opcode[] = {
[IB_WR_SEND_WITH_IMM] = IB_WC_SEND,
[IB_WR_RDMA_READ] = IB_WC_RDMA_READ,
[IB_WR_ATOMIC_CMP_AND_SWP] = IB_WC_COMP_SWAP,
[IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD
[IB_WR_ATOMIC_FETCH_AND_ADD] = IB_WC_FETCH_ADD,
[IB_WR_SEND_WITH_INV] = IB_WC_SEND,
[IB_WR_LOCAL_INV] = IB_WC_LOCAL_INV,
[IB_WR_REG_MR] = IB_WC_REG_MR
};

/*
Expand Down

0 comments on commit 0db3dfa

Please sign in to comment.