Skip to content

Commit

Permalink
---
Browse files Browse the repository at this point in the history 
yaml
---
r: 192470
b: refs/heads/master
c: 5e80ba8
h: refs/heads/master
v: v3
  • Loading branch information
Vladimir Sokolovsky authored and Roland Dreier committed Apr 21, 2010
1 parent 4e9fdbf commit 7ca4942
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 72 deletions.
2 changes: 1 addition & 1 deletion [refs]
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
---
refs/heads/master: 9fda1ac5fa09c49e9148f85be14f55e2bb856c0f
refs/heads/master: 5e80ba8ff0bd33ff4af2365969a231cbdb98cafb
9 changes: 4 additions & 5 deletions trunk/drivers/infiniband/ulp/iser/iscsi_iser.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ iscsi_iser_conn_destroy(struct iscsi_cls_conn *cls_conn)
*/
if (ib_conn) {
ib_conn->iser_conn = NULL;
iser_conn_put(ib_conn, 1); /* deref iscsi/ib conn unbinding */
iser_conn_put(ib_conn);
}
}

Expand Down Expand Up @@ -357,12 +357,11 @@ iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
/* binds the iSER connection retrieved from the previously
* connected ep_handle to the iSCSI layer connection. exchanges
* connection pointers */
iser_err("binding iscsi/iser conn %p %p to ib_conn %p\n",
conn, conn->dd_data, ib_conn);
iser_err("binding iscsi conn %p to iser_conn %p\n",conn,ib_conn);
iser_conn = conn->dd_data;
ib_conn->iser_conn = iser_conn;
iser_conn->ib_conn = ib_conn;
iser_conn_get(ib_conn); /* ref iscsi/ib conn binding */
iser_conn_get(ib_conn);
return 0;
}

Expand All @@ -383,7 +382,7 @@ iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
* There is no unbind event so the stop callback
* must release the ref from the bind.
*/
iser_conn_put(ib_conn, 1); /* deref iscsi/ib conn unbinding */
iser_conn_put(ib_conn);
}
iser_conn->ib_conn = NULL;
}
Expand Down
4 changes: 2 additions & 2 deletions trunk/drivers/infiniband/ulp/iser/iscsi_iser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,6 @@ struct iser_device {
struct ib_cq *tx_cq;
struct ib_mr *mr;
struct tasklet_struct cq_tasklet;
struct ib_event_handler event_handler;
struct list_head ig_list; /* entry in ig devices list */
int refcount;
};
Expand All @@ -247,6 +246,7 @@ struct iser_conn {
struct rdma_cm_id *cma_id; /* CMA ID */
struct ib_qp *qp; /* QP */
struct ib_fmr_pool *fmr_pool; /* pool of IB FMRs */
int disc_evt_flag; /* disconn event delivered */
wait_queue_head_t wait; /* waitq for conn/disconn */
int post_recv_buf_count; /* posted rx count */
atomic_t post_send_buf_count; /* posted tx count */
Expand Down Expand Up @@ -320,7 +320,7 @@ void iser_conn_init(struct iser_conn *ib_conn);

void iser_conn_get(struct iser_conn *ib_conn);

int iser_conn_put(struct iser_conn *ib_conn, int destroy_cma_id_allowed);
void iser_conn_put(struct iser_conn *ib_conn);

void iser_conn_terminate(struct iser_conn *ib_conn);

Expand Down
115 changes: 51 additions & 64 deletions trunk/drivers/infiniband/ulp/iser/iser_verbs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,6 @@ static void iser_qp_event_callback(struct ib_event *cause, void *context)
iser_err("got qp event %d\n",cause->event);
}

static void iser_event_handler(struct ib_event_handler *handler,
struct ib_event *event)
{
iser_err("async event %d on device %s port %d\n", event->event,
event->device->name, event->element.port_num);
}

/**
* iser_create_device_ib_res - creates Protection Domain (PD), Completion
* Queue (CQ), DMA Memory Region (DMA MR) with the device associated with
Expand Down Expand Up @@ -103,15 +96,8 @@ static int iser_create_device_ib_res(struct iser_device *device)
if (IS_ERR(device->mr))
goto dma_mr_err;

INIT_IB_EVENT_HANDLER(&device->event_handler, device->ib_device,
iser_event_handler);
if (ib_register_event_handler(&device->event_handler))
goto handler_err;

return 0;

handler_err:
ib_dereg_mr(device->mr);
dma_mr_err:
tasklet_kill(&device->cq_tasklet);
cq_arm_err:
Expand All @@ -134,7 +120,7 @@ static void iser_free_device_ib_res(struct iser_device *device)
BUG_ON(device->mr == NULL);

tasklet_kill(&device->cq_tasklet);
(void)ib_unregister_event_handler(&device->event_handler);

(void)ib_dereg_mr(device->mr);
(void)ib_destroy_cq(device->tx_cq);
(void)ib_destroy_cq(device->rx_cq);
Expand Down Expand Up @@ -163,8 +149,10 @@ static int iser_create_ib_conn_res(struct iser_conn *ib_conn)
device = ib_conn->device;

ib_conn->login_buf = kmalloc(ISER_RX_LOGIN_SIZE, GFP_KERNEL);
if (!ib_conn->login_buf)
goto out_err;
if (!ib_conn->login_buf) {
goto alloc_err;
ret = -ENOMEM;
}

ib_conn->login_dma = ib_dma_map_single(ib_conn->device->ib_device,
(void *)ib_conn->login_buf, ISER_RX_LOGIN_SIZE,
Expand All @@ -173,9 +161,10 @@ static int iser_create_ib_conn_res(struct iser_conn *ib_conn)
ib_conn->page_vec = kmalloc(sizeof(struct iser_page_vec) +
(sizeof(u64) * (ISCSI_ISER_SG_TABLESIZE +1)),
GFP_KERNEL);
if (!ib_conn->page_vec)
goto out_err;

if (!ib_conn->page_vec) {
ret = -ENOMEM;
goto alloc_err;
}
ib_conn->page_vec->pages = (u64 *) (ib_conn->page_vec + 1);

params.page_shift = SHIFT_4K;
Expand All @@ -195,8 +184,7 @@ static int iser_create_ib_conn_res(struct iser_conn *ib_conn)
ib_conn->fmr_pool = ib_create_fmr_pool(device->pd, &params);
if (IS_ERR(ib_conn->fmr_pool)) {
ret = PTR_ERR(ib_conn->fmr_pool);
ib_conn->fmr_pool = NULL;
goto out_err;
goto fmr_pool_err;
}

memset(&init_attr, 0, sizeof init_attr);
Expand All @@ -214,15 +202,20 @@ static int iser_create_ib_conn_res(struct iser_conn *ib_conn)

ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr);
if (ret)
goto out_err;
goto qp_err;

ib_conn->qp = ib_conn->cma_id->qp;
iser_err("setting conn %p cma_id %p: fmr_pool %p qp %p\n",
ib_conn, ib_conn->cma_id,
ib_conn->fmr_pool, ib_conn->cma_id->qp);
return ret;

out_err:
qp_err:
(void)ib_destroy_fmr_pool(ib_conn->fmr_pool);
fmr_pool_err:
kfree(ib_conn->page_vec);
kfree(ib_conn->login_buf);
alloc_err:
iser_err("unable to alloc mem or create resource, err %d\n", ret);
return ret;
}
Expand All @@ -231,7 +224,7 @@ static int iser_create_ib_conn_res(struct iser_conn *ib_conn)
* releases the FMR pool, QP and CMA ID objects, returns 0 on success,
* -1 on failure
*/
static int iser_free_ib_conn_res(struct iser_conn *ib_conn, int can_destroy_id)
static int iser_free_ib_conn_res(struct iser_conn *ib_conn)
{
BUG_ON(ib_conn == NULL);

Expand All @@ -246,8 +239,7 @@ static int iser_free_ib_conn_res(struct iser_conn *ib_conn, int can_destroy_id)
if (ib_conn->qp != NULL)
rdma_destroy_qp(ib_conn->cma_id);

/* if cma handler context, the caller acts s.t the cma destroy the id */
if (ib_conn->cma_id != NULL && can_destroy_id)
if (ib_conn->cma_id != NULL)
rdma_destroy_id(ib_conn->cma_id);

ib_conn->fmr_pool = NULL;
Expand Down Expand Up @@ -325,7 +317,7 @@ static int iser_conn_state_comp_exch(struct iser_conn *ib_conn,
/**
* Frees all conn objects and deallocs conn descriptor
*/
static void iser_conn_release(struct iser_conn *ib_conn, int can_destroy_id)
static void iser_conn_release(struct iser_conn *ib_conn)
{
struct iser_device *device = ib_conn->device;

Expand All @@ -335,11 +327,13 @@ static void iser_conn_release(struct iser_conn *ib_conn, int can_destroy_id)
list_del(&ib_conn->conn_list);
mutex_unlock(&ig.connlist_mutex);
iser_free_rx_descriptors(ib_conn);
iser_free_ib_conn_res(ib_conn, can_destroy_id);
iser_free_ib_conn_res(ib_conn);
ib_conn->device = NULL;
/* on EVENT_ADDR_ERROR there's no device yet for this conn */
if (device != NULL)
iser_device_try_release(device);
if (ib_conn->iser_conn)
ib_conn->iser_conn->ib_conn = NULL;
iscsi_destroy_endpoint(ib_conn->ep);
}

Expand All @@ -348,13 +342,10 @@ void iser_conn_get(struct iser_conn *ib_conn)
atomic_inc(&ib_conn->refcount);
}

int iser_conn_put(struct iser_conn *ib_conn, int can_destroy_id)
void iser_conn_put(struct iser_conn *ib_conn)
{
if (atomic_dec_and_test(&ib_conn->refcount)) {
iser_conn_release(ib_conn, can_destroy_id);
return 1;
}
return 0;
if (atomic_dec_and_test(&ib_conn->refcount))
iser_conn_release(ib_conn);
}

/**
Expand All @@ -378,20 +369,19 @@ void iser_conn_terminate(struct iser_conn *ib_conn)
wait_event_interruptible(ib_conn->wait,
ib_conn->state == ISER_CONN_DOWN);

iser_conn_put(ib_conn, 1); /* deref ib conn deallocate */
iser_conn_put(ib_conn);
}

static int iser_connect_error(struct rdma_cm_id *cma_id)
static void iser_connect_error(struct rdma_cm_id *cma_id)
{
struct iser_conn *ib_conn;
ib_conn = (struct iser_conn *)cma_id->context;

ib_conn->state = ISER_CONN_DOWN;
wake_up_interruptible(&ib_conn->wait);
return iser_conn_put(ib_conn, 0); /* deref ib conn's cma id */
}

static int iser_addr_handler(struct rdma_cm_id *cma_id)
static void iser_addr_handler(struct rdma_cm_id *cma_id)
{
struct iser_device *device;
struct iser_conn *ib_conn;
Expand All @@ -400,7 +390,8 @@ static int iser_addr_handler(struct rdma_cm_id *cma_id)
device = iser_device_find_by_ib_device(cma_id);
if (!device) {
iser_err("device lookup/creation failed\n");
return iser_connect_error(cma_id);
iser_connect_error(cma_id);
return;
}

ib_conn = (struct iser_conn *)cma_id->context;
Expand All @@ -409,13 +400,11 @@ static int iser_addr_handler(struct rdma_cm_id *cma_id)
ret = rdma_resolve_route(cma_id, 1000);
if (ret) {
iser_err("resolve route failed: %d\n", ret);
return iser_connect_error(cma_id);
iser_connect_error(cma_id);
}

return 0;
}

static int iser_route_handler(struct rdma_cm_id *cma_id)
static void iser_route_handler(struct rdma_cm_id *cma_id)
{
struct rdma_conn_param conn_param;
int ret;
Expand All @@ -436,9 +425,9 @@ static int iser_route_handler(struct rdma_cm_id *cma_id)
goto failure;
}

return 0;
return;
failure:
return iser_connect_error(cma_id);
iser_connect_error(cma_id);
}

static void iser_connected_handler(struct rdma_cm_id *cma_id)
Expand All @@ -450,12 +439,12 @@ static void iser_connected_handler(struct rdma_cm_id *cma_id)
wake_up_interruptible(&ib_conn->wait);
}

static int iser_disconnected_handler(struct rdma_cm_id *cma_id)
static void iser_disconnected_handler(struct rdma_cm_id *cma_id)
{
struct iser_conn *ib_conn;
int ret;

ib_conn = (struct iser_conn *)cma_id->context;
ib_conn->disc_evt_flag = 1;

/* getting here when the state is UP means that the conn is being *
* terminated asynchronously from the iSCSI layer's perspective. */
Expand All @@ -470,24 +459,20 @@ static int iser_disconnected_handler(struct rdma_cm_id *cma_id)
ib_conn->state = ISER_CONN_DOWN;
wake_up_interruptible(&ib_conn->wait);
}

ret = iser_conn_put(ib_conn, 0); /* deref ib conn's cma id */
return ret;
}

static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
{
int ret = 0;

iser_err("event %d status %d conn %p id %p\n",
event->event, event->status, cma_id->context, cma_id);
iser_err("event %d conn %p id %p\n",event->event,cma_id->context,cma_id);

switch (event->event) {
case RDMA_CM_EVENT_ADDR_RESOLVED:
ret = iser_addr_handler(cma_id);
iser_addr_handler(cma_id);
break;
case RDMA_CM_EVENT_ROUTE_RESOLVED:
ret = iser_route_handler(cma_id);
iser_route_handler(cma_id);
break;
case RDMA_CM_EVENT_ESTABLISHED:
iser_connected_handler(cma_id);
Expand All @@ -497,12 +482,13 @@ static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *eve
case RDMA_CM_EVENT_CONNECT_ERROR:
case RDMA_CM_EVENT_UNREACHABLE:
case RDMA_CM_EVENT_REJECTED:
ret = iser_connect_error(cma_id);
iser_err("event: %d, error: %d\n", event->event, event->status);
iser_connect_error(cma_id);
break;
case RDMA_CM_EVENT_DISCONNECTED:
case RDMA_CM_EVENT_DEVICE_REMOVAL:
case RDMA_CM_EVENT_ADDR_CHANGE:
ret = iser_disconnected_handler(cma_id);
iser_disconnected_handler(cma_id);
break;
default:
iser_err("Unexpected RDMA CM event (%d)\n", event->event);
Expand All @@ -517,7 +503,7 @@ void iser_conn_init(struct iser_conn *ib_conn)
init_waitqueue_head(&ib_conn->wait);
ib_conn->post_recv_buf_count = 0;
atomic_set(&ib_conn->post_send_buf_count, 0);
atomic_set(&ib_conn->refcount, 1); /* ref ib conn allocation */
atomic_set(&ib_conn->refcount, 1);
INIT_LIST_HEAD(&ib_conn->conn_list);
spin_lock_init(&ib_conn->lock);
}
Expand Down Expand Up @@ -545,7 +531,6 @@ int iser_connect(struct iser_conn *ib_conn,

ib_conn->state = ISER_CONN_PENDING;

iser_conn_get(ib_conn); /* ref ib conn's cma id */
ib_conn->cma_id = rdma_create_id(iser_cma_handler,
(void *)ib_conn,
RDMA_PS_TCP);
Expand Down Expand Up @@ -583,7 +568,7 @@ int iser_connect(struct iser_conn *ib_conn,
addr_failure:
ib_conn->state = ISER_CONN_DOWN;
connect_failure:
iser_conn_release(ib_conn, 1);
iser_conn_release(ib_conn);
return err;
}

Expand Down Expand Up @@ -752,10 +737,12 @@ static void iser_handle_comp_error(struct iser_tx_desc *desc,
iscsi_conn_failure(ib_conn->iser_conn->iscsi_conn,
ISCSI_ERR_CONN_FAILED);

/* no more non completed posts to the QP, complete the
* termination process w.o worrying on disconnect event */
ib_conn->state = ISER_CONN_DOWN;
wake_up_interruptible(&ib_conn->wait);
/* complete the termination process if disconnect event was delivered *
* note there are no more non completed posts to the QP */
if (ib_conn->disc_evt_flag) {
ib_conn->state = ISER_CONN_DOWN;
wake_up_interruptible(&ib_conn->wait);
}
}
}

Expand Down
Loading

0 comments on commit 7ca4942

Please sign in to comment.