Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history 
yaml
---
r: 88547
b: refs/heads/master
c: bbf8eed
h: refs/heads/master
i:
  88545: 72c20b7
  88543: 0294ea8
v: v3
  • Loading branch information
Vladimir Sokolovsky authored and Roland Dreier committed Apr 17, 2008
1 parent 2812dc8 commit e8804af
Show file tree
Hide file tree
Showing 6 changed files with 301 additions and 34 deletions.
2 changes: 1 addition & 1 deletion [refs]
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: 3fdcb97f0b8d8a29117dc36acd0b15965d2a2160
refs/heads/master: bbf8eed1a0f8949f7385146624f736f829992a70
292 changes: 259 additions & 33 deletions trunk/drivers/infiniband/hw/mlx4/cq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,81 @@ int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
return mlx4_cq_modify(dev->dev, &mcq->mcq, cq_count, cq_period);
}

static int mlx4_ib_alloc_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int nent)
{
int err;

err = mlx4_buf_alloc(dev->dev, nent * sizeof(struct mlx4_cqe),
PAGE_SIZE * 2, &buf->buf);

if (err)
goto out;

err = mlx4_mtt_init(dev->dev, buf->buf.npages, buf->buf.page_shift,
&buf->mtt);
if (err)
goto err_buf;

err = mlx4_buf_write_mtt(dev->dev, &buf->mtt, &buf->buf);
if (err)
goto err_mtt;

return 0;

err_mtt:
mlx4_mtt_cleanup(dev->dev, &buf->mtt);

err_buf:
mlx4_buf_free(dev->dev, nent * sizeof(struct mlx4_cqe),
&buf->buf);

out:
return err;
}

static void mlx4_ib_free_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *buf, int cqe)
{
mlx4_buf_free(dev->dev, (cqe + 1) * sizeof(struct mlx4_cqe), &buf->buf);
}

static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_ucontext *context,
struct mlx4_ib_cq_buf *buf, struct ib_umem **umem,
u64 buf_addr, int cqe)
{
int err;

*umem = ib_umem_get(context, buf_addr, cqe * sizeof (struct mlx4_cqe),
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(*umem))
return PTR_ERR(*umem);

err = mlx4_mtt_init(dev->dev, ib_umem_page_count(*umem),
ilog2((*umem)->page_size), &buf->mtt);
if (err)
goto err_buf;

err = mlx4_ib_umem_write_mtt(dev, &buf->mtt, *umem);
if (err)
goto err_mtt;

return 0;

err_mtt:
mlx4_mtt_cleanup(dev->dev, &buf->mtt);

err_buf:
ib_umem_release(*umem);

return err;
}

struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector,
struct ib_ucontext *context,
struct ib_udata *udata)
{
struct mlx4_ib_dev *dev = to_mdev(ibdev);
struct mlx4_ib_cq *cq;
struct mlx4_uar *uar;
int buf_size;
int err;

if (entries < 1 || entries > dev->dev->caps.max_cqes)
Expand All @@ -112,8 +179,10 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector

entries = roundup_pow_of_two(entries + 1);
cq->ibcq.cqe = entries - 1;
buf_size = entries * sizeof (struct mlx4_cqe);
mutex_init(&cq->resize_mutex);
spin_lock_init(&cq->lock);
cq->resize_buf = NULL;
cq->resize_umem = NULL;

if (context) {
struct mlx4_ib_create_cq ucmd;
Expand All @@ -123,21 +192,10 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector
goto err_cq;
}

cq->umem = ib_umem_get(context, ucmd.buf_addr, buf_size,
IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(cq->umem)) {
err = PTR_ERR(cq->umem);
goto err_cq;
}

err = mlx4_mtt_init(dev->dev, ib_umem_page_count(cq->umem),
ilog2(cq->umem->page_size), &cq->buf.mtt);
err = mlx4_ib_get_cq_umem(dev, context, &cq->buf, &cq->umem,
ucmd.buf_addr, entries);
if (err)
goto err_buf;

err = mlx4_ib_umem_write_mtt(dev, &cq->buf.mtt, cq->umem);
if (err)
goto err_mtt;
goto err_cq;

err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr,
&cq->db);
Expand All @@ -155,19 +213,9 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector
*cq->mcq.set_ci_db = 0;
*cq->mcq.arm_db = 0;

if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, &cq->buf.buf)) {
err = -ENOMEM;
goto err_db;
}

err = mlx4_mtt_init(dev->dev, cq->buf.buf.npages, cq->buf.buf.page_shift,
&cq->buf.mtt);
err = mlx4_ib_alloc_cq_buf(dev, &cq->buf, entries);
if (err)
goto err_buf;

err = mlx4_buf_write_mtt(dev->dev, &cq->buf.mtt, &cq->buf.buf);
if (err)
goto err_mtt;
goto err_db;

uar = &dev->priv_uar;
}
Expand Down Expand Up @@ -195,12 +243,10 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector
err_mtt:
mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);

err_buf:
if (context)
ib_umem_release(cq->umem);
else
mlx4_buf_free(dev->dev, entries * sizeof (struct mlx4_cqe),
&cq->buf.buf);
mlx4_ib_free_cq_buf(dev, &cq->buf, entries);

err_db:
if (!context)
Expand All @@ -212,6 +258,170 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, int entries, int vector
return ERR_PTR(err);
}

static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,
int entries)
{
int err;

if (cq->resize_buf)
return -EBUSY;

cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC);
if (!cq->resize_buf)
return -ENOMEM;

err = mlx4_ib_alloc_cq_buf(dev, &cq->resize_buf->buf, entries);
if (err) {
kfree(cq->resize_buf);
cq->resize_buf = NULL;
return err;
}

cq->resize_buf->cqe = entries - 1;

return 0;
}

static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,
int entries, struct ib_udata *udata)
{
struct mlx4_ib_resize_cq ucmd;
int err;

if (cq->resize_umem)
return -EBUSY;

if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
return -EFAULT;

cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC);
if (!cq->resize_buf)
return -ENOMEM;

err = mlx4_ib_get_cq_umem(dev, cq->umem->context, &cq->resize_buf->buf,
&cq->resize_umem, ucmd.buf_addr, entries);
if (err) {
kfree(cq->resize_buf);
cq->resize_buf = NULL;
return err;
}

cq->resize_buf->cqe = entries - 1;

return 0;
}

static int mlx4_ib_get_outstanding_cqes(struct mlx4_ib_cq *cq)
{
u32 i;

i = cq->mcq.cons_index;
while (get_sw_cqe(cq, i & cq->ibcq.cqe))
++i;

return i - cq->mcq.cons_index;
}

static void mlx4_ib_cq_resize_copy_cqes(struct mlx4_ib_cq *cq)
{
struct mlx4_cqe *cqe;
int i;

i = cq->mcq.cons_index;
cqe = get_cqe(cq, i & cq->ibcq.cqe);
while ((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) != MLX4_CQE_OPCODE_RESIZE) {
memcpy(get_cqe_from_buf(&cq->resize_buf->buf,
(i + 1) & cq->resize_buf->cqe),
get_cqe(cq, i & cq->ibcq.cqe), sizeof(struct mlx4_cqe));
cqe = get_cqe(cq, ++i & cq->ibcq.cqe);
}
++cq->mcq.cons_index;
}

int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata)
{
struct mlx4_ib_dev *dev = to_mdev(ibcq->device);
struct mlx4_ib_cq *cq = to_mcq(ibcq);
int outst_cqe;
int err;

mutex_lock(&cq->resize_mutex);

if (entries < 1 || entries > dev->dev->caps.max_cqes) {
err = -EINVAL;
goto out;
}

entries = roundup_pow_of_two(entries + 1);
if (entries == ibcq->cqe + 1) {
err = 0;
goto out;
}

if (ibcq->uobject) {
err = mlx4_alloc_resize_umem(dev, cq, entries, udata);
if (err)
goto out;
} else {
/* Can't be smaller then the number of outstanding CQEs */
outst_cqe = mlx4_ib_get_outstanding_cqes(cq);
if (entries < outst_cqe + 1) {
err = 0;
goto out;
}

err = mlx4_alloc_resize_buf(dev, cq, entries);
if (err)
goto out;
}

err = mlx4_cq_resize(dev->dev, &cq->mcq, entries, &cq->resize_buf->buf.mtt);
if (err)
goto err_buf;

if (ibcq->uobject) {
cq->buf = cq->resize_buf->buf;
cq->ibcq.cqe = cq->resize_buf->cqe;
ib_umem_release(cq->umem);
cq->umem = cq->resize_umem;

kfree(cq->resize_buf);
cq->resize_buf = NULL;
cq->resize_umem = NULL;
} else {
spin_lock_irq(&cq->lock);
if (cq->resize_buf) {
mlx4_ib_cq_resize_copy_cqes(cq);
mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
cq->buf = cq->resize_buf->buf;
cq->ibcq.cqe = cq->resize_buf->cqe;

kfree(cq->resize_buf);
cq->resize_buf = NULL;
}
spin_unlock_irq(&cq->lock);
}

goto out;

err_buf:
if (!ibcq->uobject)
mlx4_ib_free_cq_buf(dev, &cq->resize_buf->buf,
cq->resize_buf->cqe);

kfree(cq->resize_buf);
cq->resize_buf = NULL;

if (cq->resize_umem) {
ib_umem_release(cq->resize_umem);
cq->resize_umem = NULL;
}

out:
mutex_unlock(&cq->resize_mutex);
return err;
}

int mlx4_ib_destroy_cq(struct ib_cq *cq)
{
struct mlx4_ib_dev *dev = to_mdev(cq->device);
Expand All @@ -224,8 +434,7 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq)
mlx4_ib_db_unmap_user(to_mucontext(cq->uobject->context), &mcq->db);
ib_umem_release(mcq->umem);
} else {
mlx4_buf_free(dev->dev, (cq->cqe + 1) * sizeof (struct mlx4_cqe),
&mcq->buf.buf);
mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe + 1);
mlx4_ib_db_free(dev, &mcq->db);
}

Expand Down Expand Up @@ -332,6 +541,7 @@ static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,
u32 g_mlpath_rqpn;
u16 wqe_ctr;

repoll:
cqe = next_cqe_sw(cq);
if (!cqe)
return -EAGAIN;
Expand All @@ -354,6 +564,22 @@ static int mlx4_ib_poll_one(struct mlx4_ib_cq *cq,
return -EINVAL;
}

/* Resize CQ in progress */
if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) == MLX4_CQE_OPCODE_RESIZE)) {
if (cq->resize_buf) {
struct mlx4_ib_dev *dev = to_mdev(cq->ibcq.device);

mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
cq->buf = cq->resize_buf->buf;
cq->ibcq.cqe = cq->resize_buf->cqe;

kfree(cq->resize_buf);
cq->resize_buf = NULL;
}

goto repoll;
}

if (!*cur_qp ||
(be32_to_cpu(cqe->my_qpn) & 0xffffff) != (*cur_qp)->mqp.qpn) {
/*
Expand Down
2 changes: 2 additions & 0 deletions trunk/drivers/infiniband/hw/mlx4/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
(1ull << IB_USER_VERBS_CMD_DEREG_MR) |
(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
(1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
(1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
(1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
(1ull << IB_USER_VERBS_CMD_CREATE_QP) |
(1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
Expand Down Expand Up @@ -610,6 +611,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
ibdev->ib_dev.post_recv = mlx4_ib_post_recv;
ibdev->ib_dev.create_cq = mlx4_ib_create_cq;
ibdev->ib_dev.modify_cq = mlx4_ib_modify_cq;
ibdev->ib_dev.resize_cq = mlx4_ib_resize_cq;
ibdev->ib_dev.destroy_cq = mlx4_ib_destroy_cq;
ibdev->ib_dev.poll_cq = mlx4_ib_poll_cq;
ibdev->ib_dev.req_notify_cq = mlx4_ib_arm_cq;
Expand Down
Loading

0 comments on commit e8804af

Please sign in to comment.