Skip to content

Commit

Permalink
RDMA/cm: fix loopback address support
Browse files Browse the repository at this point in the history 
The RDMA CM is intended to support the use of a loopback address
when establishing a connection; however, the behavior of the CM
when loopback addresses are used is confusing and does not always
work, depending on whether loopback was specified by the server,
the client, or both.

The defined behavior of rdma_bind_addr is to associate an RDMA
device with an rdma_cm_id, as long as the user specified a non-
zero address.  (ie they weren't just trying to reserve a port)
Currently, if the loopback address is passed to rdam_bind_addr,
no device is associated with the rdma_cm_id.  Fix this.

If a loopback address is specified by the client as the destination
address for a connection, it will fail to establish a connection.
This is true even if the server is listing across all addresses or
on the loopback address itself.  The issue is that the server tries
to translate the IP address carried in the REQ message to a local
net_device address, which fails.  The translation is not needed in
this case, since the REQ carries the actual HW address that should
be used.

Finally, cleanup loopback support to be more transport neutral.
Replace separate calls to get/set the sgid and dgid from the
device address to a single call that behaves correctly depending
on the format of the device address.  And support both IPv4 and
IPv6 address formats.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>

[ Fixed RDS build by s/ib_addr_get/rdma_addr_get/  - Roland ]

Signed-off-by: Roland Dreier <rolandd@cisco.com>
  • Loading branch information
Sean Hefty authored and Roland Dreier committed Nov 19, 2009
1 parent c4315d8 commit 6f8372b
Show file tree
Hide file tree
Showing 5 changed files with 61 additions and 63 deletions.
77 changes: 43 additions & 34 deletions drivers/infiniband/core/cma.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,17 +330,7 @@ static int cma_acquire_dev(struct rdma_id_private *id_priv)
union ib_gid gid;
int ret = -ENODEV;

switch (dev_addr->dev_type) {
case ARPHRD_INFINIBAND:
ib_addr_get_sgid(dev_addr, &gid);
break;
case ARPHRD_ETHER:
iw_addr_get_sgid(dev_addr, &gid);
break;
default:
return -ENODEV;
}

rdma_addr_get_sgid(dev_addr, &gid);
list_for_each_entry(cma_dev, &dev_list, list) {
ret = ib_find_cached_gid(cma_dev->device, &gid,
&id_priv->id.port_num, NULL);
Expand Down Expand Up @@ -1032,11 +1022,17 @@ static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
if (rt->num_paths == 2)
rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;

ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
&id->route.addr.dev_addr);
if (ret)
goto destroy_id;
if (cma_any_addr((struct sockaddr *) &rt->addr.src_addr)) {
rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
ib_addr_set_pkey(&rt->addr.dev_addr, rt->path_rec[0].pkey);
} else {
ret = rdma_translate_ip((struct sockaddr *) &rt->addr.src_addr,
&rt->addr.dev_addr);
if (ret)
goto destroy_id;
}
rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);

id_priv = container_of(id, struct rdma_id_private, id);
id_priv->state = CMA_CONNECT;
Expand Down Expand Up @@ -1071,10 +1067,12 @@ static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
cma_save_net_info(&id->route.addr, &listen_id->route.addr,
ip_ver, port, src, dst);

ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
&id->route.addr.dev_addr);
if (ret)
goto err;
if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) {
ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
&id->route.addr.dev_addr);
if (ret)
goto err;
}

id_priv = container_of(id, struct rdma_id_private, id);
id_priv->state = CMA_CONNECT;
Expand Down Expand Up @@ -1565,8 +1563,8 @@ static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
struct sockaddr_in6 *sin6;

memset(&path_rec, 0, sizeof path_rec);
ib_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
ib_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
rdma_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
rdma_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
path_rec.numb_path = 1;
path_rec.reversible = 1;
Expand Down Expand Up @@ -1781,7 +1779,11 @@ static int cma_bind_loopback(struct rdma_id_private *id_priv)
if (ret)
goto out;

ib_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
id_priv->id.route.addr.dev_addr.dev_type =
(rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB) ?
ARPHRD_INFINIBAND : ARPHRD_ETHER;

rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
id_priv->id.port_num = p;
cma_attach_to_dev(id_priv, cma_dev);
Expand Down Expand Up @@ -1839,7 +1841,7 @@ static void addr_handler(int status, struct sockaddr *src_addr,
static int cma_resolve_loopback(struct rdma_id_private *id_priv)
{
struct cma_work *work;
struct sockaddr_in *src_in, *dst_in;
struct sockaddr *src, *dst;
union ib_gid gid;
int ret;

Expand All @@ -1853,14 +1855,19 @@ static int cma_resolve_loopback(struct rdma_id_private *id_priv)
goto err;
}

ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
ib_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);

if (cma_zero_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)) {
src_in = (struct sockaddr_in *)&id_priv->id.route.addr.src_addr;
dst_in = (struct sockaddr_in *)&id_priv->id.route.addr.dst_addr;
src_in->sin_family = dst_in->sin_family;
src_in->sin_addr.s_addr = dst_in->sin_addr.s_addr;
src = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
if (cma_zero_addr(src)) {
dst = (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
if ((src->sa_family = dst->sa_family) == AF_INET) {
((struct sockaddr_in *) src)->sin_addr.s_addr =
((struct sockaddr_in *) dst)->sin_addr.s_addr;
} else {
ipv6_addr_copy(&((struct sockaddr_in6 *) src)->sin6_addr,
&((struct sockaddr_in6 *) dst)->sin6_addr);
}
}

work->id = id_priv;
Expand Down Expand Up @@ -2089,7 +2096,9 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND))
return -EINVAL;

if (!cma_any_addr(addr)) {
if (cma_loopback_addr(addr)) {
ret = cma_bind_loopback(id_priv);
} else if (!cma_zero_addr(addr)) {
ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
if (ret)
goto err1;
Expand All @@ -2108,7 +2117,7 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)

return 0;
err2:
if (!cma_any_addr(addr)) {
if (id_priv->cma_dev) {
mutex_lock(&lock);
cma_detach_from_dev(id_priv);
mutex_unlock(&lock);
Expand Down Expand Up @@ -2721,7 +2730,7 @@ static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
if (id_priv->id.ps == RDMA_PS_UDP)
rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
ib_addr_get_sgid(dev_addr, &rec.port_gid);
rdma_addr_get_sgid(dev_addr, &rec.port_gid);
rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
rec.join_state = 1;

Expand Down
8 changes: 4 additions & 4 deletions drivers/infiniband/core/ucma.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -563,10 +563,10 @@ static void ucma_copy_ib_route(struct rdma_ucm_query_route_resp *resp,
switch (route->num_paths) {
case 0:
dev_addr = &route->addr.dev_addr;
ib_addr_get_dgid(dev_addr,
(union ib_gid *) &resp->ib_route[0].dgid);
ib_addr_get_sgid(dev_addr,
(union ib_gid *) &resp->ib_route[0].sgid);
rdma_addr_get_dgid(dev_addr,
(union ib_gid *) &resp->ib_route[0].dgid);
rdma_addr_get_sgid(dev_addr,
(union ib_gid *) &resp->ib_route[0].sgid);
resp->ib_route[0].pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
break;
case 2:
Expand Down
31 changes: 10 additions & 21 deletions include/rdma/ib_addr.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,40 +122,29 @@ static inline void ib_addr_get_mgid(struct rdma_dev_addr *dev_addr,
memcpy(gid, dev_addr->broadcast + 4, sizeof *gid);
}

static inline void ib_addr_get_sgid(struct rdma_dev_addr *dev_addr,
union ib_gid *gid)
static inline int rdma_addr_gid_offset(struct rdma_dev_addr *dev_addr)
{
memcpy(gid, dev_addr->src_dev_addr + 4, sizeof *gid);
return dev_addr->dev_type == ARPHRD_INFINIBAND ? 4 : 0;
}

static inline void ib_addr_set_sgid(struct rdma_dev_addr *dev_addr,
union ib_gid *gid)
static inline void rdma_addr_get_sgid(struct rdma_dev_addr *dev_addr, union ib_gid *gid)
{
memcpy(dev_addr->src_dev_addr + 4, gid, sizeof *gid);
memcpy(gid, dev_addr->src_dev_addr + rdma_addr_gid_offset(dev_addr), sizeof *gid);
}

static inline void ib_addr_get_dgid(struct rdma_dev_addr *dev_addr,
union ib_gid *gid)
static inline void rdma_addr_set_sgid(struct rdma_dev_addr *dev_addr, union ib_gid *gid)
{
memcpy(gid, dev_addr->dst_dev_addr + 4, sizeof *gid);
memcpy(dev_addr->src_dev_addr + rdma_addr_gid_offset(dev_addr), gid, sizeof *gid);
}

static inline void ib_addr_set_dgid(struct rdma_dev_addr *dev_addr,
union ib_gid *gid)
static inline void rdma_addr_get_dgid(struct rdma_dev_addr *dev_addr, union ib_gid *gid)
{
memcpy(dev_addr->dst_dev_addr + 4, gid, sizeof *gid);
memcpy(gid, dev_addr->dst_dev_addr + rdma_addr_gid_offset(dev_addr), sizeof *gid);
}

static inline void iw_addr_get_sgid(struct rdma_dev_addr *dev_addr,
union ib_gid *gid)
{
memcpy(gid, dev_addr->src_dev_addr, sizeof *gid);
}

static inline void iw_addr_get_dgid(struct rdma_dev_addr *dev_addr,
union ib_gid *gid)
static inline void rdma_addr_set_dgid(struct rdma_dev_addr *dev_addr, union ib_gid *gid)
{
memcpy(gid, dev_addr->dst_dev_addr, sizeof *gid);
memcpy(dev_addr->dst_dev_addr + rdma_addr_gid_offset(dev_addr), gid, sizeof *gid);
}

#endif /* IB_ADDR_H */
4 changes: 2 additions & 2 deletions net/rds/ib.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,8 @@ static int rds_ib_conn_info_visitor(struct rds_connection *conn,
ic = conn->c_transport_data;
dev_addr = &ic->i_cm_id->route.addr.dev_addr;

ib_addr_get_sgid(dev_addr, (union ib_gid *) &iinfo->src_gid);
ib_addr_get_dgid(dev_addr, (union ib_gid *) &iinfo->dst_gid);
rdma_addr_get_sgid(dev_addr, (union ib_gid *) &iinfo->src_gid);
rdma_addr_get_dgid(dev_addr, (union ib_gid *) &iinfo->dst_gid);

rds_ibdev = ib_get_client_data(ic->i_cm_id->device, &rds_ib_client);
iinfo->max_send_wr = ic->i_send_ring.w_nr;
Expand Down
4 changes: 2 additions & 2 deletions net/rds/iw.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,8 @@ static int rds_iw_conn_info_visitor(struct rds_connection *conn,
ic = conn->c_transport_data;
dev_addr = &ic->i_cm_id->route.addr.dev_addr;

ib_addr_get_sgid(dev_addr, (union ib_gid *) &iinfo->src_gid);
ib_addr_get_dgid(dev_addr, (union ib_gid *) &iinfo->dst_gid);
rdma_addr_get_sgid(dev_addr, (union ib_gid *) &iinfo->src_gid);
rdma_addr_get_dgid(dev_addr, (union ib_gid *) &iinfo->dst_gid);

rds_iwdev = ib_get_client_data(ic->i_cm_id->device, &rds_iw_client);
iinfo->max_send_wr = ic->i_send_ring.w_nr;
Expand Down

0 comments on commit 6f8372b

Please sign in to comment.