Skip to content

Commit

Permalink
net/smc: adapt SMC client code to use the LLC flow
Browse files Browse the repository at this point in the history 
Change the code that processes the SMC client part of connection
establishment to use the LLC flow framework (CONFIRM_LINK request
messages).

Signed-off-by: Karsten Graul <kgraul@linux.ibm.com>
Reviewed-by: Ursula Braun <ubraun@linux.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Karsten Graul authored and David S. Miller committed Apr 30, 2020
1 parent 4667bb4 commit 0fb0b02
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 74 deletions.
69 changes: 45 additions & 24 deletions net/smc/af_smc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,22 +382,24 @@ static int smcr_lgr_reg_rmbs(struct smc_link_group *lgr,
static int smcr_clnt_conf_first_link(struct smc_sock *smc)
{
struct smc_link *link = smc->conn.lnk;
int rest;
struct smc_llc_qentry *qentry;
int rc;

link->lgr->type = SMC_LGR_SINGLE;

/* receive CONFIRM LINK request from server over RoCE fabric */
rest = wait_for_completion_interruptible_timeout(
&link->llc_confirm,
SMC_LLC_WAIT_FIRST_TIME);
if (rest <= 0) {
qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
SMC_LLC_CONFIRM_LINK);
if (!qentry) {
struct smc_clc_msg_decline dclc;

rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
}

if (link->llc_confirm_rc)
rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ);
smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
if (rc)
return SMC_CLC_DECL_RMBE_EC;

rc = smc_ib_modify_qp_rts(link);
Expand All @@ -409,31 +411,30 @@ static int smcr_clnt_conf_first_link(struct smc_sock *smc)
if (smcr_link_reg_rmb(link, smc->conn.rmb_desc, false))
return SMC_CLC_DECL_ERR_REGRMB;

/* confirm_rkey is implicit on 1st contact */
smc->conn.rmb_desc->is_conf_rkey = true;

/* send CONFIRM LINK response over RoCE fabric */
rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
if (rc < 0)
return SMC_CLC_DECL_TIMEOUT_CL;

/* receive ADD LINK request from server over RoCE fabric */
rest = wait_for_completion_interruptible_timeout(&link->llc_add,
SMC_LLC_WAIT_TIME);
if (rest <= 0) {
smc_llc_link_active(link);

/* optional 2nd link, receive ADD LINK request from server */
qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
SMC_LLC_ADD_LINK);
if (!qentry) {
struct smc_clc_msg_decline dclc;

rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_AL : rc;
if (rc == -EAGAIN)
rc = 0; /* no DECLINE received, go with one link */
return rc;
}

/* send add link reject message, only one link supported for now */
rc = smc_llc_send_add_link(link,
link->smcibdev->mac[link->ibport - 1],
link->gid, SMC_LLC_RESP);
if (rc < 0)
return SMC_CLC_DECL_TIMEOUT_AL;

smc_llc_link_active(link);

smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
/* tbd: call smc_llc_cli_add_link(link, qentry); */
return 0;
}

Expand Down Expand Up @@ -613,8 +614,8 @@ static int smc_connect_rdma(struct smc_sock *smc,
struct smc_clc_msg_accept_confirm *aclc,
struct smc_init_info *ini)
{
int i, reason_code = 0;
struct smc_link *link;
int reason_code = 0;

ini->is_smcd = false;
ini->ib_lcl = &aclc->lcl;
Expand All @@ -627,10 +628,28 @@ static int smc_connect_rdma(struct smc_sock *smc,
mutex_unlock(&smc_client_lgr_pending);
return reason_code;
}
link = smc->conn.lnk;

smc_conn_save_peer_info(smc, aclc);

if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
link = smc->conn.lnk;
} else {
/* set link that was assigned by server */
link = NULL;
for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
struct smc_link *l = &smc->conn.lgr->lnk[i];

if (l->peer_qpn == ntoh24(aclc->qpn)) {
link = l;
break;
}
}
if (!link)
return smc_connect_abort(smc, SMC_CLC_DECL_NOSRVLINK,
ini->cln_first_contact);
smc->conn.lnk = link;
}

/* create send buffer and rmb */
if (smc_buf_create(smc, false))
return smc_connect_abort(smc, SMC_CLC_DECL_MEM,
Expand Down Expand Up @@ -666,7 +685,9 @@ static int smc_connect_rdma(struct smc_sock *smc,

if (ini->cln_first_contact == SMC_FIRST_CONTACT) {
/* QP confirmation over RoCE fabric */
smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
reason_code = smcr_clnt_conf_first_link(smc);
smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
if (reason_code)
return smc_connect_abort(smc, reason_code,
ini->cln_first_contact);
Expand Down
1 change: 1 addition & 0 deletions net/smc/smc_clc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
#define SMC_CLC_DECL_GETVLANERR 0x03080000 /* err to get vlan id of ip device*/
#define SMC_CLC_DECL_ISMVLANERR 0x03090000 /* err to reg vlan id on ism dev */
#define SMC_CLC_DECL_NOACTLINK 0x030a0000 /* no active smc-r link in lgr */
#define SMC_CLC_DECL_NOSRVLINK 0x030b0000 /* SMC-R link from srv not found */
#define SMC_CLC_DECL_SYNCERR 0x04000000 /* synchronization error */
#define SMC_CLC_DECL_PEERDECL 0x05000000 /* peer declined during handshake */
#define SMC_CLC_DECL_INTERR 0x09990000 /* internal error */
Expand Down
3 changes: 0 additions & 3 deletions net/smc/smc_core.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,6 @@ struct smc_link {
struct smc_link_group *lgr; /* parent link group */

enum smc_link_state state; /* state of link */
struct completion llc_confirm; /* wait for rx of conf link */
int llc_confirm_rc; /* rc from confirm link msg */
struct completion llc_add; /* wait for rx of add link */
struct delayed_work llc_testlink_wrk; /* testlink worker */
struct completion llc_testlink_resp; /* wait for rx of testlink */
int llc_testlink_time; /* testlink interval */
Expand Down
72 changes: 25 additions & 47 deletions net/smc/smc_llc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,47 +528,6 @@ static int smc_llc_send_message(struct smc_link *link, void *llcbuf)

/********************************* receive ***********************************/

static void smc_llc_rx_confirm_link(struct smc_link *link,
struct smc_llc_msg_confirm_link *llc)
{
struct smc_link_group *lgr = smc_get_lgr(link);
int conf_rc = 0;

/* RMBE eyecatchers are not supported */
if (!(llc->hd.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
conf_rc = ENOTSUPP;

if (lgr->role == SMC_CLNT &&
link->state == SMC_LNK_ACTIVATING) {
link->llc_confirm_rc = conf_rc;
link->link_id = llc->link_num;
complete(&link->llc_confirm);
}
}

static void smc_llc_rx_add_link(struct smc_link *link,
struct smc_llc_msg_add_link *llc)
{
struct smc_link_group *lgr = smc_get_lgr(link);

if (link->state == SMC_LNK_ACTIVATING) {
complete(&link->llc_add);
return;
}

if (lgr->role == SMC_SERV) {
smc_llc_prep_add_link(llc, link,
link->smcibdev->mac[link->ibport - 1],
link->gid, SMC_LLC_REQ);

} else {
smc_llc_prep_add_link(llc, link,
link->smcibdev->mac[link->ibport - 1],
link->gid, SMC_LLC_RESP);
}
smc_llc_send_message(link, llc);
}

static void smc_llc_rx_delete_link(struct smc_link *link,
struct smc_llc_msg_del_link *llc)
{
Expand Down Expand Up @@ -657,6 +616,7 @@ static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
{
union smc_llc_msg *llc = &qentry->msg;
struct smc_link *link = qentry->link;
struct smc_link_group *lgr = link->lgr;

if (!smc_link_usable(link))
goto out;
Expand All @@ -665,11 +625,31 @@ static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
case SMC_LLC_TEST_LINK:
smc_llc_rx_test_link(link, &llc->test_link);
break;
case SMC_LLC_CONFIRM_LINK:
smc_llc_rx_confirm_link(link, &llc->confirm_link);
break;
case SMC_LLC_ADD_LINK:
smc_llc_rx_add_link(link, &llc->add_link);
if (list_empty(&lgr->list))
goto out; /* lgr is terminating */
if (lgr->role == SMC_CLNT) {
if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) {
/* a flow is waiting for this message */
smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
qentry);
wake_up_interruptible(&lgr->llc_waiter);
} else if (smc_llc_flow_start(&lgr->llc_flow_lcl,
qentry)) {
/* tbd: schedule_work(&lgr->llc_add_link_work); */
}
} else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
/* as smc server, handle client suggestion */
/* tbd: schedule_work(&lgr->llc_add_link_work); */
}
return;
case SMC_LLC_CONFIRM_LINK:
if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
/* a flow is waiting for this message */
smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
wake_up_interruptible(&lgr->llc_waiter);
return;
}
break;
case SMC_LLC_DELETE_LINK:
smc_llc_rx_delete_link(link, &llc->delete_link);
Expand Down Expand Up @@ -857,8 +837,6 @@ void smc_llc_lgr_clear(struct smc_link_group *lgr)

int smc_llc_link_init(struct smc_link *link)
{
init_completion(&link->llc_confirm);
init_completion(&link->llc_add);
init_completion(&link->llc_confirm_rkey_resp);
init_completion(&link->llc_delete_rkey_resp);
mutex_init(&link->llc_delete_rkey_mutex);
Expand Down

0 comments on commit 0fb0b02

Please sign in to comment.