Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history 
yaml
---
r: 94900
b: refs/heads/master
c: d227fa7
h: refs/heads/master
v: v3
  • Loading branch information
Stefan Roscher authored and Roland Dreier committed Apr 29, 2008
1 parent ff2b433 commit c9ad366
Show file tree
Hide file tree
Showing 5 changed files with 75 additions and 5 deletions.
2 changes: 1 addition & 1 deletion [refs]
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: f56bcd8013566d4ad4759ae5fc85a6660e4655c7
refs/heads/master: d227fa7288adebe5ba37fa8e4a589c977d4e4a34
5 changes: 5 additions & 0 deletions trunk/drivers/infiniband/hw/ehca/ehca_classes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ struct ehca_av;
#include "ehca_irq.h"

#define EHCA_EQE_CACHE_SIZE 20
#define EHCA_MAX_NUM_QUEUES 0xffff

struct ehca_eqe_cache_entry {
struct ehca_eqe *eqe;
Expand Down Expand Up @@ -127,6 +128,8 @@ struct ehca_shca {
/* MR pgsize: bit 0-3 means 4K, 64K, 1M, 16M respectively */
u32 hca_cap_mr_pgsize;
int max_mtu;
atomic_t num_cqs;
atomic_t num_qps;
};

struct ehca_pd {
Expand Down Expand Up @@ -344,6 +347,8 @@ extern int ehca_use_hp_mr;
extern int ehca_scaling_code;
extern int ehca_lock_hcalls;
extern int ehca_nr_ports;
extern int ehca_max_cq;
extern int ehca_max_qp;

struct ipzu_queue_resp {
u32 qe_size; /* queue entry size */
Expand Down
11 changes: 11 additions & 0 deletions trunk/drivers/infiniband/hw/ehca/ehca_cq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,19 @@ struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe, int comp_vector,
if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
return ERR_PTR(-EINVAL);

if (!atomic_add_unless(&shca->num_cqs, 1, ehca_max_cq)) {
ehca_err(device, "Unable to create CQ, max number of %i "
"CQs reached.", ehca_max_cq);
ehca_err(device, "To increase the maximum number of CQs "
"use the number_of_cqs module parameter.\n");
return ERR_PTR(-ENOSPC);
}

my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL);
if (!my_cq) {
ehca_err(device, "Out of memory for ehca_cq struct device=%p",
device);
atomic_dec(&shca->num_cqs);
return ERR_PTR(-ENOMEM);
}

Expand Down Expand Up @@ -305,6 +314,7 @@ struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe, int comp_vector,
create_cq_exit1:
kmem_cache_free(cq_cache, my_cq);

atomic_dec(&shca->num_cqs);
return cq;
}

Expand Down Expand Up @@ -359,6 +369,7 @@ int ehca_destroy_cq(struct ib_cq *cq)
ipz_queue_dtor(NULL, &my_cq->ipz_queue);
kmem_cache_free(cq_cache, my_cq);

atomic_dec(&shca->num_cqs);
return 0;
}

Expand Down
36 changes: 34 additions & 2 deletions trunk/drivers/infiniband/hw/ehca/ehca_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ int ehca_port_act_time = 30;
int ehca_static_rate = -1;
int ehca_scaling_code = 0;
int ehca_lock_hcalls = -1;
int ehca_max_cq = -1;
int ehca_max_qp = -1;

module_param_named(open_aqp1, ehca_open_aqp1, bool, S_IRUGO);
module_param_named(debug_level, ehca_debug_level, int, S_IRUGO);
Expand All @@ -79,6 +81,8 @@ module_param_named(poll_all_eqs, ehca_poll_all_eqs, bool, S_IRUGO);
module_param_named(static_rate, ehca_static_rate, int, S_IRUGO);
module_param_named(scaling_code, ehca_scaling_code, bool, S_IRUGO);
module_param_named(lock_hcalls, ehca_lock_hcalls, bool, S_IRUGO);
module_param_named(number_of_cqs, ehca_max_cq, int, S_IRUGO);
module_param_named(number_of_qps, ehca_max_qp, int, S_IRUGO);

MODULE_PARM_DESC(open_aqp1,
"Open AQP1 on startup (default: no)");
Expand All @@ -104,6 +108,12 @@ MODULE_PARM_DESC(scaling_code,
MODULE_PARM_DESC(lock_hcalls,
"Serialize all hCalls made by the driver "
"(default: autodetect)");
MODULE_PARM_DESC(number_of_cqs,
"Max number of CQs which can be allocated "
"(default: autodetect)");
MODULE_PARM_DESC(number_of_qps,
"Max number of QPs which can be allocated "
"(default: autodetect)");

DEFINE_RWLOCK(ehca_qp_idr_lock);
DEFINE_RWLOCK(ehca_cq_idr_lock);
Expand Down Expand Up @@ -355,6 +365,25 @@ static int ehca_sense_attributes(struct ehca_shca *shca)
if (rblock->memory_page_size_supported & pgsize_map[i])
shca->hca_cap_mr_pgsize |= pgsize_map[i + 1];

/* Set maximum number of CQs and QPs to calculate EQ size */
if (ehca_max_qp == -1)
ehca_max_qp = min_t(int, rblock->max_qp, EHCA_MAX_NUM_QUEUES);
else if (ehca_max_qp < 1 || ehca_max_qp > rblock->max_qp) {
ehca_gen_err("Requested number of QPs is out of range (1 - %i) "
"specified by HW", rblock->max_qp);
ret = -EINVAL;
goto sense_attributes1;
}

if (ehca_max_cq == -1)
ehca_max_cq = min_t(int, rblock->max_cq, EHCA_MAX_NUM_QUEUES);
else if (ehca_max_cq < 1 || ehca_max_cq > rblock->max_cq) {
ehca_gen_err("Requested number of CQs is out of range (1 - %i) "
"specified by HW", rblock->max_cq);
ret = -EINVAL;
goto sense_attributes1;
}

/* query max MTU from first port -- it's the same for all ports */
port = (struct hipz_query_port *)rblock;
h_ret = hipz_h_query_port(shca->ipz_hca_handle, 1, port);
Expand Down Expand Up @@ -684,7 +713,7 @@ static int __devinit ehca_probe(struct of_device *dev,
struct ehca_shca *shca;
const u64 *handle;
struct ib_pd *ibpd;
int ret, i;
int ret, i, eq_size;

handle = of_get_property(dev->node, "ibm,hca-handle", NULL);
if (!handle) {
Expand All @@ -705,6 +734,8 @@ static int __devinit ehca_probe(struct of_device *dev,
return -ENOMEM;
}
mutex_init(&shca->modify_mutex);
atomic_set(&shca->num_cqs, 0);
atomic_set(&shca->num_qps, 0);
for (i = 0; i < ARRAY_SIZE(shca->sport); i++)
spin_lock_init(&shca->sport[i].mod_sqp_lock);

Expand All @@ -724,8 +755,9 @@ static int __devinit ehca_probe(struct of_device *dev,
goto probe1;
}

eq_size = 2 * ehca_max_cq + 4 * ehca_max_qp;
/* create event queues */
ret = ehca_create_eq(shca, &shca->eq, EHCA_EQ, 2048);
ret = ehca_create_eq(shca, &shca->eq, EHCA_EQ, eq_size);
if (ret) {
ehca_err(&shca->ib_device, "Cannot create EQ.");
goto probe1;
Expand Down
26 changes: 24 additions & 2 deletions trunk/drivers/infiniband/hw/ehca/ehca_qp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,8 +421,18 @@ static struct ehca_qp *internal_create_qp(
u32 swqe_size = 0, rwqe_size = 0, ib_qp_num;
unsigned long flags;

if (init_attr->create_flags)
if (!atomic_add_unless(&shca->num_qps, 1, ehca_max_qp)) {
ehca_err(pd->device, "Unable to create QP, max number of %i "
"QPs reached.", ehca_max_qp);
ehca_err(pd->device, "To increase the maximum number of QPs "
"use the number_of_qps module parameter.\n");
return ERR_PTR(-ENOSPC);
}

if (init_attr->create_flags) {
atomic_dec(&shca->num_qps);
return ERR_PTR(-EINVAL);
}

memset(&parms, 0, sizeof(parms));
qp_type = init_attr->qp_type;
Expand All @@ -431,6 +441,7 @@ static struct ehca_qp *internal_create_qp(
init_attr->sq_sig_type != IB_SIGNAL_ALL_WR) {
ehca_err(pd->device, "init_attr->sg_sig_type=%x not allowed",
init_attr->sq_sig_type);
atomic_dec(&shca->num_qps);
return ERR_PTR(-EINVAL);
}

Expand All @@ -455,6 +466,7 @@ static struct ehca_qp *internal_create_qp(

if (is_llqp && has_srq) {
ehca_err(pd->device, "LLQPs can't have an SRQ");
atomic_dec(&shca->num_qps);
return ERR_PTR(-EINVAL);
}

Expand All @@ -466,6 +478,7 @@ static struct ehca_qp *internal_create_qp(
ehca_err(pd->device, "no more than three SGEs "
"supported for SRQ pd=%p max_sge=%x",
pd, init_attr->cap.max_recv_sge);
atomic_dec(&shca->num_qps);
return ERR_PTR(-EINVAL);
}
}
Expand All @@ -477,6 +490,7 @@ static struct ehca_qp *internal_create_qp(
qp_type != IB_QPT_SMI &&
qp_type != IB_QPT_GSI) {
ehca_err(pd->device, "wrong QP Type=%x", qp_type);
atomic_dec(&shca->num_qps);
return ERR_PTR(-EINVAL);
}

Expand All @@ -490,13 +504,15 @@ static struct ehca_qp *internal_create_qp(
"or max_rq_wr=%x for RC LLQP",
init_attr->cap.max_send_wr,
init_attr->cap.max_recv_wr);
atomic_dec(&shca->num_qps);
return ERR_PTR(-EINVAL);
}
break;
case IB_QPT_UD:
if (!EHCA_BMASK_GET(HCA_CAP_UD_LL_QP, shca->hca_cap)) {
ehca_err(pd->device, "UD LLQP not supported "
"by this adapter");
atomic_dec(&shca->num_qps);
return ERR_PTR(-ENOSYS);
}
if (!(init_attr->cap.max_send_sge <= 5
Expand All @@ -508,20 +524,22 @@ static struct ehca_qp *internal_create_qp(
"or max_recv_sge=%x for UD LLQP",
init_attr->cap.max_send_sge,
init_attr->cap.max_recv_sge);
atomic_dec(&shca->num_qps);
return ERR_PTR(-EINVAL);
} else if (init_attr->cap.max_send_wr > 255) {
ehca_err(pd->device,
"Invalid Number of "
"max_send_wr=%x for UD QP_TYPE=%x",
init_attr->cap.max_send_wr, qp_type);
atomic_dec(&shca->num_qps);
return ERR_PTR(-EINVAL);
}
break;
default:
ehca_err(pd->device, "unsupported LL QP Type=%x",
qp_type);
atomic_dec(&shca->num_qps);
return ERR_PTR(-EINVAL);
break;
}
} else {
int max_sge = (qp_type == IB_QPT_UD || qp_type == IB_QPT_SMI
Expand All @@ -533,6 +551,7 @@ static struct ehca_qp *internal_create_qp(
"send_sge=%x recv_sge=%x max_sge=%x",
init_attr->cap.max_send_sge,
init_attr->cap.max_recv_sge, max_sge);
atomic_dec(&shca->num_qps);
return ERR_PTR(-EINVAL);
}
}
Expand All @@ -543,6 +562,7 @@ static struct ehca_qp *internal_create_qp(
my_qp = kmem_cache_zalloc(qp_cache, GFP_KERNEL);
if (!my_qp) {
ehca_err(pd->device, "pd=%p not enough memory to alloc qp", pd);
atomic_dec(&shca->num_qps);
return ERR_PTR(-ENOMEM);
}

Expand Down Expand Up @@ -823,6 +843,7 @@ static struct ehca_qp *internal_create_qp(

create_qp_exit0:
kmem_cache_free(qp_cache, my_qp);
atomic_dec(&shca->num_qps);
return ERR_PTR(ret);
}

Expand Down Expand Up @@ -1948,6 +1969,7 @@ static int internal_destroy_qp(struct ib_device *dev, struct ehca_qp *my_qp,
if (HAS_SQ(my_qp))
ipz_queue_dtor(my_pd, &my_qp->ipz_squeue);
kmem_cache_free(qp_cache, my_qp);
atomic_dec(&shca->num_qps);
return 0;
}

Expand Down

0 comments on commit c9ad366

Please sign in to comment.