Skip to content

Commit

Permalink
Merge branch 'smc-next'
Browse files Browse the repository at this point in the history 
Ursula Braun says:

====================
net/smc: patches 2018-07-25

here are 4 more patches for SMC: The first one is just a small
code cleanup in preparation for patch 2. Patch 2 switches to the
use of the vlan-gid for VLAN traffic. Patch 3 improves diagnosis
when creating SMC connections. Patch 4 improves synchronization
between local and remote link groups.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
David S. Miller committed Jul 26, 2018
2 parents f537530 + 0d18a0c commit b9a9ad7
Show file tree
Hide file tree
Showing 15 changed files with 228 additions and 144 deletions.
6 changes: 6 additions & 0 deletions include/uapi/linux/smc_diag.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ enum {
SMC_DIAG_LGRINFO,
SMC_DIAG_SHUTDOWN,
SMC_DIAG_DMBINFO,
SMC_DIAG_FALLBACK,
__SMC_DIAG_MAX,
};

Expand Down Expand Up @@ -92,6 +93,11 @@ struct smc_diag_lgrinfo {
__u8 role;
};

struct smc_diag_fallback {
__u32 reason;
__u32 peer_diagnosis;
};

struct smcd_diag_dmbinfo { /* SMC-D Socket internals */
__u32 linkid; /* Link identifier */
__u64 peer_gid; /* Peer GID */
Expand Down
86 changes: 45 additions & 41 deletions net/smc/af_smc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,20 +344,17 @@ static int smc_clnt_conf_first_link(struct smc_sock *smc)

rc = smc_ib_modify_qp_rts(link);
if (rc)
return SMC_CLC_DECL_INTERR;
return SMC_CLC_DECL_ERR_RDYLNK;

smc_wr_remember_qp_attr(link);

if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
return SMC_CLC_DECL_INTERR;
return SMC_CLC_DECL_ERR_REGRMB;

/* send CONFIRM LINK response over RoCE fabric */
rc = smc_llc_send_confirm_link(link,
link->smcibdev->mac[link->ibport - 1],
&link->smcibdev->gid[link->ibport - 1],
SMC_LLC_RESP);
rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
if (rc < 0)
return SMC_CLC_DECL_TCL;
return SMC_CLC_DECL_TIMEOUT_CL;

/* receive ADD LINK request from server over RoCE fabric */
rest = wait_for_completion_interruptible_timeout(&link->llc_add,
Expand All @@ -373,10 +370,9 @@ static int smc_clnt_conf_first_link(struct smc_sock *smc)
/* send add link reject message, only one link supported for now */
rc = smc_llc_send_add_link(link,
link->smcibdev->mac[link->ibport - 1],
&link->smcibdev->gid[link->ibport - 1],
SMC_LLC_RESP);
link->gid, SMC_LLC_RESP);
if (rc < 0)
return SMC_CLC_DECL_TCL;
return SMC_CLC_DECL_TIMEOUT_AL;

smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);

Expand Down Expand Up @@ -428,9 +424,10 @@ static void smc_link_save_peer_info(struct smc_link *link,
}

/* fall back during connect */
static int smc_connect_fallback(struct smc_sock *smc)
static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
{
smc->use_fallback = true;
smc->fallback_rsn = reason_code;
smc_copy_sock_settings_to_clc(smc);
if (smc->sk.sk_state == SMC_INIT)
smc->sk.sk_state = SMC_ACTIVE;
Expand All @@ -447,15 +444,15 @@ static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
sock_put(&smc->sk); /* passive closing */
return reason_code;
}
if (reason_code != SMC_CLC_DECL_REPLY) {
if (reason_code != SMC_CLC_DECL_PEERDECL) {
rc = smc_clc_send_decline(smc, reason_code);
if (rc < 0) {
if (smc->sk.sk_state == SMC_INIT)
sock_put(&smc->sk); /* passive closing */
return rc;
}
}
return smc_connect_fallback(smc);
return smc_connect_fallback(smc, reason_code);
}

/* abort connecting */
Expand All @@ -472,15 +469,16 @@ static int smc_connect_abort(struct smc_sock *smc, int reason_code,
/* check if there is a rdma device available for this connection. */
/* called for connect and listen */
static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev,
u8 *ibport)
u8 *ibport, unsigned short vlan_id, u8 gid[])
{
int reason_code = 0;

/* PNET table look up: search active ib_device and port
* within same PNETID that also contains the ethernet device
* used for the internal TCP socket
*/
smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport);
smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport, vlan_id,
gid);
if (!(*ibdev))
reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */

Expand Down Expand Up @@ -526,12 +524,12 @@ static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
static int smc_connect_clc(struct smc_sock *smc, int smc_type,
struct smc_clc_msg_accept_confirm *aclc,
struct smc_ib_device *ibdev, u8 ibport,
struct smcd_dev *ismdev)
u8 gid[], struct smcd_dev *ismdev)
{
int rc = 0;

/* do inband token exchange */
rc = smc_clc_send_proposal(smc, smc_type, ibdev, ibport, ismdev);
rc = smc_clc_send_proposal(smc, smc_type, ibdev, ibport, gid, ismdev);
if (rc)
return rc;
/* receive SMC Accept CLC message */
Expand Down Expand Up @@ -571,20 +569,20 @@ static int smc_connect_rdma(struct smc_sock *smc,
smc_link_save_peer_info(link, aclc);

if (smc_rmb_rtoken_handling(&smc->conn, aclc))
return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
local_contact);

smc_close_init(smc);
smc_rx_init(smc);

if (local_contact == SMC_FIRST_CONTACT) {
if (smc_ib_ready_link(link))
return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
local_contact);
} else {
if (!smc->conn.rmb_desc->reused &&
smc_reg_rmb(link, smc->conn.rmb_desc, true))
return smc_connect_abort(smc, SMC_CLC_DECL_INTERR,
return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
local_contact);
}
smc_rmb_sync_sg_for_device(&smc->conn);
Expand Down Expand Up @@ -653,6 +651,7 @@ static int __smc_connect(struct smc_sock *smc)
struct smc_clc_msg_accept_confirm aclc;
struct smc_ib_device *ibdev;
struct smcd_dev *ismdev;
u8 gid[SMC_GID_SIZE];
unsigned short vlan;
int smc_type;
int rc = 0;
Expand All @@ -661,11 +660,11 @@ static int __smc_connect(struct smc_sock *smc)
sock_hold(&smc->sk); /* sock put in passive closing */

if (smc->use_fallback)
return smc_connect_fallback(smc);
return smc_connect_fallback(smc, smc->fallback_rsn);

/* if peer has not signalled SMC-capability, fall back */
if (!tcp_sk(smc->clcsock->sk)->syn_smc)
return smc_connect_fallback(smc);
return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);

/* IPSec connections opt out of SMC-R optimizations */
if (using_ipsec(smc))
Expand All @@ -684,7 +683,7 @@ static int __smc_connect(struct smc_sock *smc)
}

/* check if there is a rdma device available */
if (!smc_check_rdma(smc, &ibdev, &ibport)) {
if (!smc_check_rdma(smc, &ibdev, &ibport, vlan, gid)) {
/* RDMA is supported for this connection */
rdma_supported = true;
if (ism_supported)
Expand All @@ -695,10 +694,10 @@ static int __smc_connect(struct smc_sock *smc)

/* if neither ISM nor RDMA are supported, fallback */
if (!rdma_supported && !ism_supported)
return smc_connect_decline_fallback(smc, SMC_CLC_DECL_CNFERR);
return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);

/* perform CLC handshake */
rc = smc_connect_clc(smc, smc_type, &aclc, ibdev, ibport, ismdev);
rc = smc_connect_clc(smc, smc_type, &aclc, ibdev, ibport, gid, ismdev);
if (rc) {
smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
return smc_connect_decline_fallback(smc, rc);
Expand All @@ -710,7 +709,7 @@ static int __smc_connect(struct smc_sock *smc)
else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
rc = smc_connect_ism(smc, &aclc, ismdev);
else
rc = SMC_CLC_DECL_CNFERR;
rc = SMC_CLC_DECL_MODEUNSUPP;
if (rc) {
smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
return smc_connect_decline_fallback(smc, rc);
Expand Down Expand Up @@ -948,15 +947,12 @@ static int smc_serv_conf_first_link(struct smc_sock *smc)
link = &lgr->lnk[SMC_SINGLE_LINK];

if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
return SMC_CLC_DECL_INTERR;
return SMC_CLC_DECL_ERR_REGRMB;

/* send CONFIRM LINK request to client over the RoCE fabric */
rc = smc_llc_send_confirm_link(link,
link->smcibdev->mac[link->ibport - 1],
&link->smcibdev->gid[link->ibport - 1],
SMC_LLC_REQ);
rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
if (rc < 0)
return SMC_CLC_DECL_TCL;
return SMC_CLC_DECL_TIMEOUT_CL;

/* receive CONFIRM LINK response from client over the RoCE fabric */
rest = wait_for_completion_interruptible_timeout(
Expand All @@ -976,10 +972,9 @@ static int smc_serv_conf_first_link(struct smc_sock *smc)
/* send ADD LINK request to client over the RoCE fabric */
rc = smc_llc_send_add_link(link,
link->smcibdev->mac[link->ibport - 1],
&link->smcibdev->gid[link->ibport - 1],
SMC_LLC_REQ);
link->gid, SMC_LLC_REQ);
if (rc < 0)
return SMC_CLC_DECL_TCL;
return SMC_CLC_DECL_TIMEOUT_AL;

/* receive ADD LINK response from client over the RoCE fabric */
rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
Expand Down Expand Up @@ -1054,7 +1049,8 @@ static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
}
smc_conn_free(&new_smc->conn);
new_smc->use_fallback = true;
if (reason_code && reason_code != SMC_CLC_DECL_REPLY) {
new_smc->fallback_rsn = reason_code;
if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
if (smc_clc_send_decline(new_smc, reason_code) < 0) {
smc_listen_out_err(new_smc);
return;
Expand Down Expand Up @@ -1145,7 +1141,7 @@ static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
if (local_contact != SMC_FIRST_CONTACT) {
if (!new_smc->conn.rmb_desc->reused) {
if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
return SMC_CLC_DECL_INTERR;
return SMC_CLC_DECL_ERR_REGRMB;
}
}
smc_rmb_sync_sg_for_device(&new_smc->conn);
Expand All @@ -1165,13 +1161,13 @@ static void smc_listen_rdma_finish(struct smc_sock *new_smc,
smc_link_save_peer_info(link, cclc);

if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
reason_code = SMC_CLC_DECL_INTERR;
reason_code = SMC_CLC_DECL_ERR_RTOK;
goto decline;
}

if (local_contact == SMC_FIRST_CONTACT) {
if (smc_ib_ready_link(link)) {
reason_code = SMC_CLC_DECL_INTERR;
reason_code = SMC_CLC_DECL_ERR_RDYLNK;
goto decline;
}
/* QP confirmation over RoCE fabric */
Expand Down Expand Up @@ -1199,6 +1195,7 @@ static void smc_listen_work(struct work_struct *work)
struct smcd_dev *ismdev;
u8 buf[SMC_CLC_MAX_LEN];
int local_contact = 0;
unsigned short vlan;
int reason_code = 0;
int rc = 0;
u8 ibport;
Expand All @@ -1211,6 +1208,7 @@ static void smc_listen_work(struct work_struct *work)
/* check if peer is smc capable */
if (!tcp_sk(newclcsock->sk)->syn_smc) {
new_smc->use_fallback = true;
new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
smc_listen_out_connected(new_smc);
return;
}
Expand Down Expand Up @@ -1247,14 +1245,16 @@ static void smc_listen_work(struct work_struct *work)
/* check if RDMA is available */
if (!ism_supported &&
((pclc->hdr.path != SMC_TYPE_R && pclc->hdr.path != SMC_TYPE_B) ||
smc_check_rdma(new_smc, &ibdev, &ibport) ||
smc_vlan_by_tcpsk(new_smc->clcsock, &vlan) ||
smc_check_rdma(new_smc, &ibdev, &ibport, vlan, NULL) ||
smc_listen_rdma_check(new_smc, pclc) ||
smc_listen_rdma_init(new_smc, pclc, ibdev, ibport,
&local_contact) ||
smc_listen_rdma_reg(new_smc, local_contact))) {
/* SMC not supported, decline */
mutex_unlock(&smc_create_lgr_pending);
smc_listen_decline(new_smc, SMC_CLC_DECL_CNFERR, local_contact);
smc_listen_decline(new_smc, SMC_CLC_DECL_MODEUNSUPP,
local_contact);
return;
}

Expand Down Expand Up @@ -1301,6 +1301,7 @@ static void smc_tcp_listen_work(struct work_struct *work)

new_smc->listen_smc = lsmc;
new_smc->use_fallback = lsmc->use_fallback;
new_smc->fallback_rsn = lsmc->fallback_rsn;
sock_hold(lsk); /* sock_put in smc_listen_work */
INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
smc_copy_sock_settings_to_smc(new_smc);
Expand Down Expand Up @@ -1455,6 +1456,7 @@ static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
if (msg->msg_flags & MSG_FASTOPEN) {
if (sk->sk_state == SMC_INIT) {
smc->use_fallback = true;
smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
} else {
rc = -EINVAL;
goto out;
Expand Down Expand Up @@ -1652,6 +1654,7 @@ static int smc_setsockopt(struct socket *sock, int level, int optname,
/* option not supported by SMC */
if (sk->sk_state == SMC_INIT) {
smc->use_fallback = true;
smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
} else {
if (!smc->use_fallback)
rc = -EINVAL;
Expand Down Expand Up @@ -1889,6 +1892,7 @@ static int smc_create(struct net *net, struct socket *sock, int protocol,
/* create internal TCP socket for CLC handshake and fallback */
smc = smc_sk(sk);
smc->use_fallback = false; /* assume rdma capability first */
smc->fallback_rsn = 0;
rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
&smc->clcsock);
if (rc) {
Expand Down
2 changes: 2 additions & 0 deletions net/smc/smc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,8 @@ struct smc_sock { /* smc sock container */
struct list_head accept_q; /* sockets to be accepted */
spinlock_t accept_q_lock; /* protects accept_q */
bool use_fallback; /* fallback to tcp */
int fallback_rsn; /* reason for fallback */
u32 peer_diagnosis; /* decline reason from peer */
int sockopt_defer_accept;
/* sockopt TCP_DEFER_ACCEPT
* value
Expand Down
16 changes: 9 additions & 7 deletions net/smc/smc_clc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,11 @@ int smc_clc_wait_msg(struct smc_sock *smc, void *buf, int buflen,
goto out;
}
if (clcm->type == SMC_CLC_DECLINE) {
reason_code = SMC_CLC_DECL_REPLY;
struct smc_clc_msg_decline *dclc;

dclc = (struct smc_clc_msg_decline *)clcm;
reason_code = SMC_CLC_DECL_PEERDECL;
smc->peer_diagnosis = ntohl(dclc->peer_diagnosis);
if (((struct smc_clc_msg_decline *)buf)->hdr.flag) {
smc->conn.lgr->sync_err = 1;
smc_lgr_terminate(smc->conn.lgr);
Expand Down Expand Up @@ -378,7 +382,7 @@ int smc_clc_send_decline(struct smc_sock *smc, u32 peer_diag_info)

/* send CLC PROPOSAL message across internal TCP socket */
int smc_clc_send_proposal(struct smc_sock *smc, int smc_type,
struct smc_ib_device *ibdev, u8 ibport,
struct smc_ib_device *ibdev, u8 ibport, u8 gid[],
struct smcd_dev *ismdev)
{
struct smc_clc_ipv6_prefix ipv6_prfx[SMC_CLC_MAX_V6_PREFIX];
Expand Down Expand Up @@ -409,7 +413,7 @@ int smc_clc_send_proposal(struct smc_sock *smc, int smc_type,
/* add SMC-R specifics */
memcpy(pclc.lcl.id_for_peer, local_systemid,
sizeof(local_systemid));
memcpy(&pclc.lcl.gid, &ibdev->gid[ibport - 1], SMC_GID_SIZE);
memcpy(&pclc.lcl.gid, gid, SMC_GID_SIZE);
memcpy(&pclc.lcl.mac, &ibdev->mac[ibport - 1], ETH_ALEN);
pclc.iparea_offset = htons(0);
}
Expand Down Expand Up @@ -492,8 +496,7 @@ int smc_clc_send_confirm(struct smc_sock *smc)
cclc.hdr.length = htons(SMCR_CLC_ACCEPT_CONFIRM_LEN);
memcpy(cclc.lcl.id_for_peer, local_systemid,
sizeof(local_systemid));
memcpy(&cclc.lcl.gid, &link->smcibdev->gid[link->ibport - 1],
SMC_GID_SIZE);
memcpy(&cclc.lcl.gid, link->gid, SMC_GID_SIZE);
memcpy(&cclc.lcl.mac, &link->smcibdev->mac[link->ibport - 1],
ETH_ALEN);
hton24(cclc.qpn, link->roce_qp->qp_num);
Expand Down Expand Up @@ -566,8 +569,7 @@ int smc_clc_send_accept(struct smc_sock *new_smc, int srv_first_contact)
link = &conn->lgr->lnk[SMC_SINGLE_LINK];
memcpy(aclc.lcl.id_for_peer, local_systemid,
sizeof(local_systemid));
memcpy(&aclc.lcl.gid, &link->smcibdev->gid[link->ibport - 1],
SMC_GID_SIZE);
memcpy(&aclc.lcl.gid, link->gid, SMC_GID_SIZE);
memcpy(&aclc.lcl.mac, link->smcibdev->mac[link->ibport - 1],
ETH_ALEN);
hton24(aclc.qpn, link->roce_qp->qp_num);
Expand Down
Loading

0 comments on commit b9a9ad7

Please sign in to comment.