Skip to content

Commit

Permalink
RDMA/uverbs: Export XRC TGT QPs to user space
Browse files Browse the repository at this point in the history 
Allow user space to operate on XRC TGT QPs the same way as other types
of QPs, with one notable exception: since XRC TGT QPs may be shared
among multiple processes, the XRC TGT QP is allowed to exist beyond the
lifetime of the creating process.

The process that creates the QP is allowed to destroy it, but if the
process exits without destroying the QP, then the QP will be left bound
to the lifetime of the XRCD.

TGT QPs are not associated with CQs or a PD.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
Signed-off-by: Roland Dreier <roland@purestorage.com>
  • Loading branch information
Sean Hefty authored and Roland Dreier committed Oct 13, 2011
1 parent 9977f4f commit b93f3c1
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 40 deletions.
105 changes: 67 additions & 38 deletions drivers/infiniband/core/uverbs_cmd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1370,8 +1370,11 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
struct ib_uverbs_create_qp_resp resp;
struct ib_udata udata;
struct ib_uqp_object *obj;
struct ib_pd *pd;
struct ib_cq *scq, *rcq = NULL;
struct ib_device *device;
struct ib_pd *pd = NULL;
struct ib_xrcd *xrcd = NULL;
struct ib_uobject *uninitialized_var(xrcd_uobj);
struct ib_cq *scq = NULL, *rcq = NULL;
struct ib_srq *srq = NULL;
struct ib_qp *qp;
struct ib_qp_init_attr attr;
Expand All @@ -1394,36 +1397,47 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_key);
down_write(&obj->uevent.uobject.mutex);

pd = idr_read_pd(cmd.pd_handle, file->ucontext);
scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, 0);
if (!pd || !scq) {
ret = -EINVAL;
goto err_put;
}

if (cmd.qp_type == IB_QPT_XRC_INI) {
cmd.max_recv_wr = cmd.max_recv_sge = 0;
if (cmd.qp_type == IB_QPT_XRC_TGT) {
xrcd = idr_read_xrcd(cmd.pd_handle, file->ucontext, &xrcd_uobj);
if (!xrcd) {
ret = -EINVAL;
goto err_put;
}
device = xrcd->device;
} else {
if (cmd.is_srq) {
srq = idr_read_srq(cmd.srq_handle, file->ucontext);
if (!srq || srq->srq_type != IB_SRQT_BASIC) {
pd = idr_read_pd(cmd.pd_handle, file->ucontext);
scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, 0);
if (!pd || !scq) {
ret = -EINVAL;
goto err_put;
}

if (cmd.qp_type == IB_QPT_XRC_INI) {
cmd.max_recv_wr = cmd.max_recv_sge = 0;
} else {
if (cmd.is_srq) {
srq = idr_read_srq(cmd.srq_handle, file->ucontext);
if (!srq || srq->srq_type != IB_SRQT_BASIC) {
ret = -EINVAL;
goto err_put;
}
}
rcq = (cmd.recv_cq_handle == cmd.send_cq_handle) ?
scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext, 1);
if (!rcq) {
ret = -EINVAL;
goto err_put;
}
}
rcq = (cmd.recv_cq_handle == cmd.send_cq_handle) ?
scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext, 1);
if (!rcq) {
ret = -EINVAL;
goto err_put;
}
device = pd->device;
}

attr.event_handler = ib_uverbs_qp_event_handler;
attr.qp_context = file;
attr.send_cq = scq;
attr.recv_cq = rcq;
attr.srq = srq;
attr.xrcd = xrcd;
attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
attr.qp_type = cmd.qp_type;
attr.create_flags = 0;
Expand All @@ -1438,27 +1452,33 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
INIT_LIST_HEAD(&obj->uevent.event_list);
INIT_LIST_HEAD(&obj->mcast_list);

qp = pd->device->create_qp(pd, &attr, &udata);
if (cmd.qp_type == IB_QPT_XRC_TGT)
qp = ib_create_qp(pd, &attr);
else
qp = device->create_qp(pd, &attr, &udata);

if (IS_ERR(qp)) {
ret = PTR_ERR(qp);
goto err_put;
}

qp->device = pd->device;
qp->pd = pd;
qp->send_cq = attr.send_cq;
qp->recv_cq = attr.recv_cq;
qp->srq = attr.srq;
qp->uobject = &obj->uevent.uobject;
qp->event_handler = attr.event_handler;
qp->qp_context = attr.qp_context;
qp->qp_type = attr.qp_type;
atomic_inc(&pd->usecnt);
atomic_inc(&attr.send_cq->usecnt);
if (attr.recv_cq)
atomic_inc(&attr.recv_cq->usecnt);
if (attr.srq)
atomic_inc(&attr.srq->usecnt);
if (cmd.qp_type != IB_QPT_XRC_TGT) {
qp->device = device;
qp->pd = pd;
qp->send_cq = attr.send_cq;
qp->recv_cq = attr.recv_cq;
qp->srq = attr.srq;
qp->event_handler = attr.event_handler;
qp->qp_context = attr.qp_context;
qp->qp_type = attr.qp_type;
atomic_inc(&pd->usecnt);
atomic_inc(&attr.send_cq->usecnt);
if (attr.recv_cq)
atomic_inc(&attr.recv_cq->usecnt);
if (attr.srq)
atomic_inc(&attr.srq->usecnt);
}
qp->uobject = &obj->uevent.uobject;

obj->uevent.uobject.object = qp;
ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject);
Expand All @@ -1480,8 +1500,12 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
goto err_copy;
}

put_pd_read(pd);
put_cq_read(scq);
if (xrcd)
put_xrcd_read(xrcd_uobj);
if (pd)
put_pd_read(pd);
if (scq)
put_cq_read(scq);
if (rcq && rcq != scq)
put_cq_read(rcq);
if (srq)
Expand All @@ -1504,6 +1528,8 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
ib_destroy_qp(qp);

err_put:
if (xrcd)
put_xrcd_read(xrcd_uobj);
if (pd)
put_pd_read(pd);
if (scq)
Expand Down Expand Up @@ -1623,6 +1649,9 @@ static int modify_qp_mask(enum ib_qp_type qp_type, int mask)
switch (qp_type) {
case IB_QPT_XRC_INI:
return mask & ~(IB_QP_MAX_DEST_RD_ATOMIC | IB_QP_MIN_RNR_TIMER);
case IB_QPT_XRC_TGT:
return mask & ~(IB_QP_MAX_QP_RD_ATOMIC | IB_QP_RETRY_CNT |
IB_QP_RNR_RETRY);
default:
return mask;
}
Expand Down
8 changes: 6 additions & 2 deletions drivers/infiniband/core/uverbs_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,8 +206,12 @@ static int ib_uverbs_cleanup_ucontext(struct ib_uverbs_file *file,
container_of(uobj, struct ib_uqp_object, uevent.uobject);

idr_remove_uobj(&ib_uverbs_qp_idr, uobj);
ib_uverbs_detach_umcast(qp, uqp);
ib_destroy_qp(qp);
if (qp->qp_type == IB_QPT_XRC_TGT) {
ib_release_qp(qp);
} else {
ib_uverbs_detach_umcast(qp, uqp);
ib_destroy_qp(qp);
}
ib_uverbs_release_uevent(file, &uqp->uevent);
kfree(uqp);
}
Expand Down

0 comments on commit b93f3c1

Please sign in to comment.