Skip to content

Commit

Permalink
[PATCH] IB/ipath: enforce device resource limits
Browse files Browse the repository at this point in the history 
These limits are somewhat artificial in that we don't actually have any
device limits.  However, the verbs layer expects that such limits exist
and are enforced, so we make up arbitrary (but sensible) limits.

Signed-off-by: Robert Walsh <robert.walsh@qlogic.com>
Signed-off-by: Bryan O'Sullivan <bryan.osullivan@qlogic.com>
Cc: "Michael S. Tsirkin" <mst@mellanox.co.il>
Cc: Roland Dreier <rolandd@cisco.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
  • Loading branch information
Bryan O'Sullivan authored and Linus Torvalds committed Jul 1, 2006
1 parent e8a88f0 commit fe62546
Show file tree
Hide file tree
Showing 6 changed files with 204 additions and 33 deletions.
15 changes: 15 additions & 0 deletions drivers/infiniband/hw/ipath/ipath_cq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,10 +158,21 @@ struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int entries,
struct ib_ucontext *context,
struct ib_udata *udata)
{
struct ipath_ibdev *dev = to_idev(ibdev);
struct ipath_cq *cq;
struct ib_wc *wc;
struct ib_cq *ret;

if (entries > ib_ipath_max_cqes) {
ret = ERR_PTR(-EINVAL);
goto bail;
}

if (dev->n_cqs_allocated == ib_ipath_max_cqs) {
ret = ERR_PTR(-ENOMEM);
goto bail;
}

/*
* Need to use vmalloc() if we want to support large #s of
* entries.
Expand Down Expand Up @@ -197,6 +208,8 @@ struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int entries,

ret = &cq->ibcq;

dev->n_cqs_allocated++;

bail:
return ret;
}
Expand All @@ -211,9 +224,11 @@ struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int entries,
*/
int ipath_destroy_cq(struct ib_cq *ibcq)
{
struct ipath_ibdev *dev = to_idev(ibcq->device);
struct ipath_cq *cq = to_icq(ibcq);

tasklet_kill(&cq->comptask);
dev->n_cqs_allocated--;
vfree(cq->queue);
kfree(cq);

Expand Down
6 changes: 4 additions & 2 deletions drivers/infiniband/hw/ipath/ipath_qp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -661,8 +661,10 @@ struct ib_qp *ipath_create_qp(struct ib_pd *ibpd,
size_t sz;
struct ib_qp *ret;

if (init_attr->cap.max_send_sge > 255 ||
init_attr->cap.max_recv_sge > 255) {
if (init_attr->cap.max_send_sge > ib_ipath_max_sges ||
init_attr->cap.max_recv_sge > ib_ipath_max_sges ||
init_attr->cap.max_send_wr > ib_ipath_max_qp_wrs ||
init_attr->cap.max_recv_wr > ib_ipath_max_qp_wrs) {
ret = ERR_PTR(-ENOMEM);
goto bail;
}
Expand Down
47 changes: 35 additions & 12 deletions drivers/infiniband/hw/ipath/ipath_srq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,23 @@ struct ib_srq *ipath_create_srq(struct ib_pd *ibpd,
struct ib_srq_init_attr *srq_init_attr,
struct ib_udata *udata)
{
struct ipath_ibdev *dev = to_idev(ibpd->device);
struct ipath_srq *srq;
u32 sz;
struct ib_srq *ret;

if (srq_init_attr->attr.max_sge < 1) {
if (dev->n_srqs_allocated == ib_ipath_max_srqs) {
ret = ERR_PTR(-ENOMEM);
goto bail;
}

if (srq_init_attr->attr.max_wr == 0) {
ret = ERR_PTR(-EINVAL);
goto bail;
}

if ((srq_init_attr->attr.max_sge > ib_ipath_max_srq_sges) ||
(srq_init_attr->attr.max_wr > ib_ipath_max_srq_wrs)) {
ret = ERR_PTR(-EINVAL);
goto bail;
}
Expand Down Expand Up @@ -165,6 +177,8 @@ struct ib_srq *ipath_create_srq(struct ib_pd *ibpd,

ret = &srq->ibsrq;

dev->n_srqs_allocated++;

bail:
return ret;
}
Expand All @@ -182,24 +196,26 @@ int ipath_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
unsigned long flags;
int ret;

if (attr_mask & IB_SRQ_LIMIT) {
spin_lock_irqsave(&srq->rq.lock, flags);
srq->limit = attr->srq_limit;
spin_unlock_irqrestore(&srq->rq.lock, flags);
}
if (attr_mask & IB_SRQ_MAX_WR) {
u32 size = attr->max_wr + 1;
struct ipath_rwqe *wq, *p;
u32 n;
u32 sz;
if (attr_mask & IB_SRQ_MAX_WR)
if ((attr->max_wr > ib_ipath_max_srq_wrs) ||
(attr->max_sge > srq->rq.max_sge)) {
ret = -EINVAL;
goto bail;
}

if (attr->max_sge < srq->rq.max_sge) {
if (attr_mask & IB_SRQ_LIMIT)
if (attr->srq_limit >= srq->rq.size) {
ret = -EINVAL;
goto bail;
}

if (attr_mask & IB_SRQ_MAX_WR) {
struct ipath_rwqe *wq, *p;
u32 sz, size, n;

sz = sizeof(struct ipath_rwqe) +
attr->max_sge * sizeof(struct ipath_sge);
size = attr->max_wr + 1;
wq = vmalloc(size * sz);
if (!wq) {
ret = -ENOMEM;
Expand Down Expand Up @@ -243,6 +259,11 @@ int ipath_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
spin_unlock_irqrestore(&srq->rq.lock, flags);
}

if (attr_mask & IB_SRQ_LIMIT) {
spin_lock_irqsave(&srq->rq.lock, flags);
srq->limit = attr->srq_limit;
spin_unlock_irqrestore(&srq->rq.lock, flags);
}
ret = 0;

bail:
Expand All @@ -266,7 +287,9 @@ int ipath_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
int ipath_destroy_srq(struct ib_srq *ibsrq)
{
struct ipath_srq *srq = to_isrq(ibsrq);
struct ipath_ibdev *dev = to_idev(ibsrq->device);

dev->n_srqs_allocated--;
vfree(srq->rq.wq);
kfree(srq);

Expand Down
109 changes: 96 additions & 13 deletions drivers/infiniband/hw/ipath/ipath_verbs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,59 @@ unsigned int ib_ipath_debug; /* debug mask */
module_param_named(debug, ib_ipath_debug, uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(debug, "Verbs debug mask");

static unsigned int ib_ipath_max_pds = 0xFFFF;
module_param_named(max_pds, ib_ipath_max_pds, uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(max_pds,
"Maximum number of protection domains to support");

static unsigned int ib_ipath_max_ahs = 0xFFFF;
module_param_named(max_ahs, ib_ipath_max_ahs, uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(max_ahs, "Maximum number of address handles to support");

unsigned int ib_ipath_max_cqes = 0x2FFFF;
module_param_named(max_cqes, ib_ipath_max_cqes, uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(max_cqes,
"Maximum number of completion queue entries to support");

unsigned int ib_ipath_max_cqs = 0x1FFFF;
module_param_named(max_cqs, ib_ipath_max_cqs, uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(max_cqs, "Maximum number of completion queues to support");

unsigned int ib_ipath_max_qp_wrs = 0x3FFF;
module_param_named(max_qp_wrs, ib_ipath_max_qp_wrs, uint,
S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(max_qp_wrs, "Maximum number of QP WRs to support");

unsigned int ib_ipath_max_sges = 0x60;
module_param_named(max_sges, ib_ipath_max_sges, uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(max_sges, "Maximum number of SGEs to support");

unsigned int ib_ipath_max_mcast_grps = 16384;
module_param_named(max_mcast_grps, ib_ipath_max_mcast_grps, uint,
S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(max_mcast_grps,
"Maximum number of multicast groups to support");

unsigned int ib_ipath_max_mcast_qp_attached = 16;
module_param_named(max_mcast_qp_attached, ib_ipath_max_mcast_qp_attached,
uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(max_mcast_qp_attached,
"Maximum number of attached QPs to support");

unsigned int ib_ipath_max_srqs = 1024;
module_param_named(max_srqs, ib_ipath_max_srqs, uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(max_srqs, "Maximum number of SRQs to support");

unsigned int ib_ipath_max_srq_sges = 128;
module_param_named(max_srq_sges, ib_ipath_max_srq_sges,
uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(max_srq_sges, "Maximum number of SRQ SGEs to support");

unsigned int ib_ipath_max_srq_wrs = 0x1FFFF;
module_param_named(max_srq_wrs, ib_ipath_max_srq_wrs,
uint, S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(max_srq_wrs, "Maximum number of SRQ WRs support");

MODULE_LICENSE("GPL");
MODULE_AUTHOR("QLogic <support@pathscale.com>");
MODULE_DESCRIPTION("QLogic InfiniPath driver");
Expand Down Expand Up @@ -581,24 +634,25 @@ static int ipath_query_device(struct ib_device *ibdev,
props->sys_image_guid = dev->sys_image_guid;

props->max_mr_size = ~0ull;
props->max_qp = 0xffff;
props->max_qp_wr = 0xffff;
props->max_sge = 255;
props->max_cq = 0xffff;
props->max_cqe = 0xffff;
props->max_mr = 0xffff;
props->max_pd = 0xffff;
props->max_qp = dev->qp_table.max;
props->max_qp_wr = ib_ipath_max_qp_wrs;
props->max_sge = ib_ipath_max_sges;
props->max_cq = ib_ipath_max_cqs;
props->max_ah = ib_ipath_max_ahs;
props->max_cqe = ib_ipath_max_cqes;
props->max_mr = dev->lk_table.max;
props->max_pd = ib_ipath_max_pds;
props->max_qp_rd_atom = 1;
props->max_qp_init_rd_atom = 1;
/* props->max_res_rd_atom */
props->max_srq = 0xffff;
props->max_srq_wr = 0xffff;
props->max_srq_sge = 255;
props->max_srq = ib_ipath_max_srqs;
props->max_srq_wr = ib_ipath_max_srq_wrs;
props->max_srq_sge = ib_ipath_max_srq_sges;
/* props->local_ca_ack_delay */
props->atomic_cap = IB_ATOMIC_HCA;
props->max_pkeys = ipath_layer_get_npkeys(dev->dd);
props->max_mcast_grp = 0xffff;
props->max_mcast_qp_attach = 0xffff;
props->max_mcast_grp = ib_ipath_max_mcast_grps;
props->max_mcast_qp_attach = ib_ipath_max_mcast_qp_attached;
props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
props->max_mcast_grp;

Expand Down Expand Up @@ -741,15 +795,30 @@ static struct ib_pd *ipath_alloc_pd(struct ib_device *ibdev,
struct ib_ucontext *context,
struct ib_udata *udata)
{
struct ipath_ibdev *dev = to_idev(ibdev);
struct ipath_pd *pd;
struct ib_pd *ret;

/*
* This is actually totally arbitrary. Some correctness tests
* assume there's a maximum number of PDs that can be allocated.
* We don't actually have this limit, but we fail the test if
* we allow allocations of more than we report for this value.
*/

if (dev->n_pds_allocated == ib_ipath_max_pds) {
ret = ERR_PTR(-ENOMEM);
goto bail;
}

pd = kmalloc(sizeof *pd, GFP_KERNEL);
if (!pd) {
ret = ERR_PTR(-ENOMEM);
goto bail;
}

dev->n_pds_allocated++;

/* ib_alloc_pd() will initialize pd->ibpd. */
pd->user = udata != NULL;

Expand All @@ -762,6 +831,9 @@ static struct ib_pd *ipath_alloc_pd(struct ib_device *ibdev,
static int ipath_dealloc_pd(struct ib_pd *ibpd)
{
struct ipath_pd *pd = to_ipd(ibpd);
struct ipath_ibdev *dev = to_idev(ibpd->device);

dev->n_pds_allocated--;

kfree(pd);

Expand All @@ -780,6 +852,12 @@ static struct ib_ah *ipath_create_ah(struct ib_pd *pd,
{
struct ipath_ah *ah;
struct ib_ah *ret;
struct ipath_ibdev *dev = to_idev(pd->device);

if (dev->n_ahs_allocated == ib_ipath_max_ahs) {
ret = ERR_PTR(-ENOMEM);
goto bail;
}

/* A multicast address requires a GRH (see ch. 8.4.1). */
if (ah_attr->dlid >= IPS_MULTICAST_LID_BASE &&
Expand All @@ -794,7 +872,7 @@ static struct ib_ah *ipath_create_ah(struct ib_pd *pd,
goto bail;
}

if (ah_attr->port_num != 1 ||
if (ah_attr->port_num < 1 ||
ah_attr->port_num > pd->device->phys_port_cnt) {
ret = ERR_PTR(-EINVAL);
goto bail;
Expand All @@ -806,6 +884,8 @@ static struct ib_ah *ipath_create_ah(struct ib_pd *pd,
goto bail;
}

dev->n_ahs_allocated++;

/* ib_create_ah() will initialize ah->ibah. */
ah->attr = *ah_attr;

Expand All @@ -823,8 +903,11 @@ static struct ib_ah *ipath_create_ah(struct ib_pd *pd,
*/
static int ipath_destroy_ah(struct ib_ah *ibah)
{
struct ipath_ibdev *dev = to_idev(ibah->device);
struct ipath_ah *ah = to_iah(ibah);

dev->n_ahs_allocated--;

kfree(ah);

return 0;
Expand Down
24 changes: 24 additions & 0 deletions drivers/infiniband/hw/ipath/ipath_verbs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ struct ipath_mcast {
struct list_head qp_list;
wait_queue_head_t wait;
atomic_t refcount;
int n_attached;
};

/* Memory region */
Expand Down Expand Up @@ -432,6 +433,11 @@ struct ipath_ibdev {
__be64 sys_image_guid; /* in network order */
__be64 gid_prefix; /* in network order */
__be64 mkey;
u32 n_pds_allocated; /* number of PDs allocated for device */
u32 n_ahs_allocated; /* number of AHs allocated for device */
u32 n_cqs_allocated; /* number of CQs allocated for device */
u32 n_srqs_allocated; /* number of SRQs allocated for device */
u32 n_mcast_grps_allocated; /* number of mcast groups allocated */
u64 ipath_sword; /* total dwords sent (sample result) */
u64 ipath_rword; /* total dwords received (sample result) */
u64 ipath_spkts; /* total packets sent (sample result) */
Expand Down Expand Up @@ -697,6 +703,24 @@ extern const int ib_ipath_state_ops[];

extern unsigned int ib_ipath_lkey_table_size;

extern unsigned int ib_ipath_max_cqes;

extern unsigned int ib_ipath_max_cqs;

extern unsigned int ib_ipath_max_qp_wrs;

extern unsigned int ib_ipath_max_sges;

extern unsigned int ib_ipath_max_mcast_grps;

extern unsigned int ib_ipath_max_mcast_qp_attached;

extern unsigned int ib_ipath_max_srqs;

extern unsigned int ib_ipath_max_srq_sges;

extern unsigned int ib_ipath_max_srq_wrs;

extern const u32 ib_ipath_rnr_table[];

#endif /* IPATH_VERBS_H */
Loading

0 comments on commit fe62546

Please sign in to comment.