Skip to content

Commit

Permalink
IB/ipath: Optimize completion queue entry insertion and polling
Browse files Browse the repository at this point in the history 
The code to add an entry to the completion queue stored the QPN which is
needed for the user level verbs view of the completion queue entry but
the kernel struct ib_wc contains a pointer to the QP instead of a QPN.
When the kernel polled for a completion queue entry, the QPN was lookup
up and the QP pointer recovered. This patch stores the CQE differently
based on whether the CQ is a kernel CQ or a user CQ thus avoiding the
QPN to QP lookup overhead.

Signed-off-by: Ralph Campbell <ralph.campbell@qlogic.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>
  • Loading branch information
Ralph Campbell authored and Roland Dreier committed Oct 10, 2007
1 parent d42b01b commit 6cff2fa
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 47 deletions.
94 changes: 48 additions & 46 deletions drivers/infiniband/hw/ipath/ipath_cq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,22 +76,25 @@ void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int solicited)
}
return;
}
wc->queue[head].wr_id = entry->wr_id;
wc->queue[head].status = entry->status;
wc->queue[head].opcode = entry->opcode;
wc->queue[head].vendor_err = entry->vendor_err;
wc->queue[head].byte_len = entry->byte_len;
wc->queue[head].imm_data = (__u32 __force)entry->imm_data;
wc->queue[head].qp_num = entry->qp->qp_num;
wc->queue[head].src_qp = entry->src_qp;
wc->queue[head].wc_flags = entry->wc_flags;
wc->queue[head].pkey_index = entry->pkey_index;
wc->queue[head].slid = entry->slid;
wc->queue[head].sl = entry->sl;
wc->queue[head].dlid_path_bits = entry->dlid_path_bits;
wc->queue[head].port_num = entry->port_num;
/* Make sure queue entry is written before the head index. */
smp_wmb();
if (cq->ip) {
wc->uqueue[head].wr_id = entry->wr_id;
wc->uqueue[head].status = entry->status;
wc->uqueue[head].opcode = entry->opcode;
wc->uqueue[head].vendor_err = entry->vendor_err;
wc->uqueue[head].byte_len = entry->byte_len;
wc->uqueue[head].imm_data = (__u32 __force)entry->imm_data;
wc->uqueue[head].qp_num = entry->qp->qp_num;
wc->uqueue[head].src_qp = entry->src_qp;
wc->uqueue[head].wc_flags = entry->wc_flags;
wc->uqueue[head].pkey_index = entry->pkey_index;
wc->uqueue[head].slid = entry->slid;
wc->uqueue[head].sl = entry->sl;
wc->uqueue[head].dlid_path_bits = entry->dlid_path_bits;
wc->uqueue[head].port_num = entry->port_num;
/* Make sure entry is written before the head index. */
smp_wmb();
} else
wc->kqueue[head] = *entry;
wc->head = next;

if (cq->notify == IB_CQ_NEXT_COMP ||
Expand Down Expand Up @@ -130,38 +133,23 @@ int ipath_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
int npolled;
u32 tail;

/* The kernel can only poll a kernel completion queue */
if (cq->ip) {
npolled = -EINVAL;
goto bail;
}

spin_lock_irqsave(&cq->lock, flags);

wc = cq->queue;
tail = wc->tail;
if (tail > (u32) cq->ibcq.cqe)
tail = (u32) cq->ibcq.cqe;
for (npolled = 0; npolled < num_entries; ++npolled, ++entry) {
struct ipath_qp *qp;

if (tail == wc->head)
break;
/* Make sure entry is read after head index is read. */
smp_rmb();
qp = ipath_lookup_qpn(&to_idev(cq->ibcq.device)->qp_table,
wc->queue[tail].qp_num);
entry->qp = &qp->ibqp;
if (atomic_dec_and_test(&qp->refcount))
wake_up(&qp->wait);

entry->wr_id = wc->queue[tail].wr_id;
entry->status = wc->queue[tail].status;
entry->opcode = wc->queue[tail].opcode;
entry->vendor_err = wc->queue[tail].vendor_err;
entry->byte_len = wc->queue[tail].byte_len;
entry->imm_data = wc->queue[tail].imm_data;
entry->src_qp = wc->queue[tail].src_qp;
entry->wc_flags = wc->queue[tail].wc_flags;
entry->pkey_index = wc->queue[tail].pkey_index;
entry->slid = wc->queue[tail].slid;
entry->sl = wc->queue[tail].sl;
entry->dlid_path_bits = wc->queue[tail].dlid_path_bits;
entry->port_num = wc->queue[tail].port_num;
/* The kernel doesn't need a RMB since it has the lock. */
*entry = wc->kqueue[tail];
if (tail >= cq->ibcq.cqe)
tail = 0;
else
Expand All @@ -171,6 +159,7 @@ int ipath_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)

spin_unlock_irqrestore(&cq->lock, flags);

bail:
return npolled;
}

Expand Down Expand Up @@ -215,6 +204,7 @@ struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int entries, int comp_vec
struct ipath_cq *cq;
struct ipath_cq_wc *wc;
struct ib_cq *ret;
u32 sz;

if (entries < 1 || entries > ib_ipath_max_cqes) {
ret = ERR_PTR(-EINVAL);
Expand All @@ -235,7 +225,12 @@ struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int entries, int comp_vec
* We need to use vmalloc() in order to support mmap and large
* numbers of entries.
*/
wc = vmalloc_user(sizeof(*wc) + sizeof(struct ib_wc) * entries);
sz = sizeof(*wc);
if (udata && udata->outlen >= sizeof(__u64))
sz += sizeof(struct ib_uverbs_wc) * (entries + 1);
else
sz += sizeof(struct ib_wc) * (entries + 1);
wc = vmalloc_user(sz);
if (!wc) {
ret = ERR_PTR(-ENOMEM);
goto bail_cq;
Expand All @@ -247,9 +242,8 @@ struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int entries, int comp_vec
*/
if (udata && udata->outlen >= sizeof(__u64)) {
int err;
u32 s = sizeof *wc + sizeof(struct ib_wc) * entries;

cq->ip = ipath_create_mmap_info(dev, s, context, wc);
cq->ip = ipath_create_mmap_info(dev, sz, context, wc);
if (!cq->ip) {
ret = ERR_PTR(-ENOMEM);
goto bail_wc;
Expand Down Expand Up @@ -380,6 +374,7 @@ int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
struct ipath_cq_wc *wc;
u32 head, tail, n;
int ret;
u32 sz;

if (cqe < 1 || cqe > ib_ipath_max_cqes) {
ret = -EINVAL;
Expand All @@ -389,7 +384,12 @@ int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
/*
* Need to use vmalloc() if we want to support large #s of entries.
*/
wc = vmalloc_user(sizeof(*wc) + sizeof(struct ib_wc) * cqe);
sz = sizeof(*wc);
if (udata && udata->outlen >= sizeof(__u64))
sz += sizeof(struct ib_uverbs_wc) * (cqe + 1);
else
sz += sizeof(struct ib_wc) * (cqe + 1);
wc = vmalloc_user(sz);
if (!wc) {
ret = -ENOMEM;
goto bail;
Expand Down Expand Up @@ -430,7 +430,10 @@ int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
goto bail;
}
for (n = 0; tail != head; n++) {
wc->queue[n] = old_wc->queue[tail];
if (cq->ip)
wc->uqueue[n] = old_wc->uqueue[tail];
else
wc->kqueue[n] = old_wc->kqueue[tail];
if (tail == (u32) cq->ibcq.cqe)
tail = 0;
else
Expand All @@ -447,9 +450,8 @@ int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
if (cq->ip) {
struct ipath_ibdev *dev = to_idev(ibcq->device);
struct ipath_mmap_info *ip = cq->ip;
u32 s = sizeof *wc + sizeof(struct ib_wc) * cqe;

ipath_update_mmap_info(dev, ip, s, wc);
ipath_update_mmap_info(dev, ip, sz, wc);
spin_lock_irq(&dev->pending_lock);
if (list_empty(&ip->pending_mmaps))
list_add(&ip->pending_mmaps, &dev->pending_mmaps);
Expand Down
6 changes: 5 additions & 1 deletion drivers/infiniband/hw/ipath/ipath_verbs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,11 @@ struct ipath_mmap_info {
struct ipath_cq_wc {
u32 head; /* index of next entry to fill */
u32 tail; /* index of next ib_poll_cq() entry */
struct ib_uverbs_wc queue[1]; /* this is actually size ibcq.cqe + 1 */
union {
/* these are actually size ibcq.cqe + 1 */
struct ib_uverbs_wc uqueue[0];
struct ib_wc kqueue[0];
};
};

/*
Expand Down

0 comments on commit 6cff2fa

Please sign in to comment.