Skip to content

Commit

Permalink
mlx4_core: Add link type autosensing
Browse files Browse the repository at this point in the history 
When a port's link is down (except to driver restart) and the port is
configured for auto sensing, we try to sense port link type (Ethernet
or InfiniBand) in order to determine how to initialize the port.  If
the port type needs to be changed, all mlx4 for the device interfaces
are unregistered and then registered again with the new port
types.  Sensing is done with intervals of 3 seconds.

Signed-off-by: Yevgeny Petrilin <yevgenyp@mellanox.co.il>
Signed-off-by: Roland Dreier <rolandd@cisco.com>
  • Loading branch information
Yevgeny Petrilin authored and Roland Dreier committed Mar 19, 2009
1 parent 793730b commit 27bf91d
Show file tree
Hide file tree
Showing 8 changed files with 277 additions and 51 deletions.
2 changes: 1 addition & 1 deletion drivers/net/mlx4/Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
obj-$(CONFIG_MLX4_CORE) += mlx4_core.o

mlx4_core-y := alloc.o catas.o cmd.o cq.o eq.o fw.o icm.o intf.o main.o mcg.o \
mr.o pd.o port.o profile.o qp.o reset.o srq.o
mr.o pd.o port.o profile.o qp.o reset.o sense.o srq.o

obj-$(CONFIG_MLX4_EN) += mlx4_en.o

Expand Down
16 changes: 2 additions & 14 deletions drivers/net/mlx4/catas.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ enum {
static DEFINE_SPINLOCK(catas_lock);

static LIST_HEAD(catas_list);
static struct workqueue_struct *catas_wq;
static struct work_struct catas_work;

static int internal_err_reset = 1;
Expand Down Expand Up @@ -77,7 +76,7 @@ static void poll_catas(unsigned long dev_ptr)
list_add(&priv->catas_err.list, &catas_list);
spin_unlock(&catas_lock);

queue_work(catas_wq, &catas_work);
queue_work(mlx4_wq, &catas_work);
}
} else
mod_timer(&priv->catas_err.timer,
Expand Down Expand Up @@ -146,18 +145,7 @@ void mlx4_stop_catas_poll(struct mlx4_dev *dev)
spin_unlock_irq(&catas_lock);
}

int __init mlx4_catas_init(void)
void __init mlx4_catas_init(void)
{
INIT_WORK(&catas_work, catas_reset);

catas_wq = create_singlethread_workqueue("mlx4_err");
if (!catas_wq)
return -ENOMEM;

return 0;
}

void mlx4_catas_cleanup(void)
{
destroy_workqueue(catas_wq);
}
16 changes: 11 additions & 5 deletions drivers/net/mlx4/eq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
int cqn;
int eqes_found = 0;
int set_ci = 0;
int port;

while ((eqe = next_eqe_sw(eq))) {
/*
Expand Down Expand Up @@ -203,11 +204,16 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
break;

case MLX4_EVENT_TYPE_PORT_CHANGE:
mlx4_dispatch_event(dev,
eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_ACTIVE ?
MLX4_DEV_EVENT_PORT_UP :
MLX4_DEV_EVENT_PORT_DOWN,
be32_to_cpu(eqe->event.port_change.port) >> 28);
port = be32_to_cpu(eqe->event.port_change.port) >> 28;
if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) {
mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_DOWN,
port);
mlx4_priv(dev)->sense.do_sense_port[port] = 1;
} else {
mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP,
port);
mlx4_priv(dev)->sense.do_sense_port[port] = 0;
}
break;

case MLX4_EVENT_TYPE_CQ_ERROR:
Expand Down
104 changes: 77 additions & 27 deletions drivers/net/mlx4/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver");
MODULE_LICENSE("Dual BSD/GPL");
MODULE_VERSION(DRV_VERSION);

struct workqueue_struct *mlx4_wq;

#ifdef CONFIG_MLX4_DEBUG

int mlx4_debug_level = 0;
Expand Down Expand Up @@ -98,24 +100,23 @@ module_param_named(use_prio, use_prio, bool, 0444);
MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports "
"(0/1, default 0)");

static int mlx4_check_port_params(struct mlx4_dev *dev,
enum mlx4_port_type *port_type)
int mlx4_check_port_params(struct mlx4_dev *dev,
enum mlx4_port_type *port_type)
{
int i;

for (i = 0; i < dev->caps.num_ports - 1; i++) {
if (port_type[i] != port_type[i+1] &&
!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
mlx4_err(dev, "Only same port types supported "
"on this HCA, aborting.\n");
return -EINVAL;
if (port_type[i] != port_type[i + 1]) {
if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
mlx4_err(dev, "Only same port types supported "
"on this HCA, aborting.\n");
return -EINVAL;
}
if (port_type[i] == MLX4_PORT_TYPE_ETH &&
port_type[i + 1] == MLX4_PORT_TYPE_IB)
return -EINVAL;
}
}
if ((port_type[0] == MLX4_PORT_TYPE_ETH) &&
(port_type[1] == MLX4_PORT_TYPE_IB)) {
mlx4_err(dev, "eth-ib configuration is not supported.\n");
return -EINVAL;
}

for (i = 0; i < dev->caps.num_ports; i++) {
if (!(port_type[i] & dev->caps.supported_type[i+1])) {
Expand Down Expand Up @@ -225,6 +226,9 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
dev->caps.port_type[i] = MLX4_PORT_TYPE_IB;
else
dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH;
dev->caps.possible_type[i] = dev->caps.port_type[i];
mlx4_priv(dev)->sense.sense_allowed[i] =
dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO;

if (dev->caps.log_num_macs > dev_cap->log_max_macs[i]) {
dev->caps.log_num_macs = dev_cap->log_max_macs[i];
Expand Down Expand Up @@ -263,14 +267,16 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
* Change the port configuration of the device.
* Every user of this function must hold the port mutex.
*/
static int mlx4_change_port_types(struct mlx4_dev *dev,
enum mlx4_port_type *port_types)
int mlx4_change_port_types(struct mlx4_dev *dev,
enum mlx4_port_type *port_types)
{
int err = 0;
int change = 0;
int port;

for (port = 0; port < dev->caps.num_ports; port++) {
/* Change the port type only if the new type is different
* from the current, and not set to Auto */
if (port_types[port] != dev->caps.port_type[port + 1]) {
change = 1;
dev->caps.port_type[port + 1] = port_types[port];
Expand Down Expand Up @@ -302,10 +308,17 @@ static ssize_t show_port_type(struct device *dev,
struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
port_attr);
struct mlx4_dev *mdev = info->dev;
char type[8];

sprintf(type, "%s",
(mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB) ?
"ib" : "eth");
if (mdev->caps.possible_type[info->port] == MLX4_PORT_TYPE_AUTO)
sprintf(buf, "auto (%s)\n", type);
else
sprintf(buf, "%s\n", type);

return sprintf(buf, "%s\n",
mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB ?
"ib" : "eth");
return strlen(buf);
}

static ssize_t set_port_type(struct device *dev,
Expand All @@ -317,33 +330,64 @@ static ssize_t set_port_type(struct device *dev,
struct mlx4_dev *mdev = info->dev;
struct mlx4_priv *priv = mlx4_priv(mdev);
enum mlx4_port_type types[MLX4_MAX_PORTS];
enum mlx4_port_type new_types[MLX4_MAX_PORTS];
int i;
int err = 0;

if (!strcmp(buf, "ib\n"))
info->tmp_type = MLX4_PORT_TYPE_IB;
else if (!strcmp(buf, "eth\n"))
info->tmp_type = MLX4_PORT_TYPE_ETH;
else if (!strcmp(buf, "auto\n"))
info->tmp_type = MLX4_PORT_TYPE_AUTO;
else {
mlx4_err(mdev, "%s is not supported port type\n", buf);
return -EINVAL;
}

mlx4_stop_sense(mdev);
mutex_lock(&priv->port_mutex);
for (i = 0; i < mdev->caps.num_ports; i++)
/* Possible type is always the one that was delivered */
mdev->caps.possible_type[info->port] = info->tmp_type;

for (i = 0; i < mdev->caps.num_ports; i++) {
types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type :
mdev->caps.port_type[i+1];
mdev->caps.possible_type[i+1];
if (types[i] == MLX4_PORT_TYPE_AUTO)
types[i] = mdev->caps.port_type[i+1];
}

err = mlx4_check_port_params(mdev, types);
if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
for (i = 1; i <= mdev->caps.num_ports; i++) {
if (mdev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) {
mdev->caps.possible_type[i] = mdev->caps.port_type[i];
err = -EINVAL;
}
}
}
if (err) {
mlx4_err(mdev, "Auto sensing is not supported on this HCA. "
"Set only 'eth' or 'ib' for both ports "
"(should be the same)\n");
goto out;
}

mlx4_do_sense_ports(mdev, new_types, types);

err = mlx4_check_port_params(mdev, new_types);
if (err)
goto out;

for (i = 1; i <= mdev->caps.num_ports; i++)
priv->port[i].tmp_type = 0;
/* We are about to apply the changes after the configuration
* was verified, no need to remember the temporary types
* any more */
for (i = 0; i < mdev->caps.num_ports; i++)
priv->port[i + 1].tmp_type = 0;

err = mlx4_change_port_types(mdev, types);
err = mlx4_change_port_types(mdev, new_types);

out:
mlx4_start_sense(mdev);
mutex_unlock(&priv->port_mutex);
return err ? err : count;
}
Expand Down Expand Up @@ -1117,6 +1161,9 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
if (err)
goto err_port;

mlx4_sense_init(dev);
mlx4_start_sense(dev);

pci_set_drvdata(pdev, dev);

return 0;
Expand Down Expand Up @@ -1182,6 +1229,7 @@ static void mlx4_remove_one(struct pci_dev *pdev)
int p;

if (dev) {
mlx4_stop_sense(dev);
mlx4_unregister_device(dev);

for (p = 1; p <= dev->caps.num_ports; p++) {
Expand Down Expand Up @@ -1266,9 +1314,11 @@ static int __init mlx4_init(void)
if (mlx4_verify_params())
return -EINVAL;

ret = mlx4_catas_init();
if (ret)
return ret;
mlx4_catas_init();

mlx4_wq = create_singlethread_workqueue("mlx4");
if (!mlx4_wq)
return -ENOMEM;

ret = pci_register_driver(&mlx4_driver);
return ret < 0 ? ret : 0;
Expand All @@ -1277,7 +1327,7 @@ static int __init mlx4_init(void)
static void __exit mlx4_cleanup(void)
{
pci_unregister_driver(&mlx4_driver);
mlx4_catas_cleanup();
destroy_workqueue(mlx4_wq);
}

module_init(mlx4_init);
Expand Down
27 changes: 25 additions & 2 deletions drivers/net/mlx4/mlx4.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
#include <linux/mutex.h>
#include <linux/radix-tree.h>
#include <linux/timer.h>
#include <linux/workqueue.h>

#include <linux/mlx4/device.h>
#include <linux/mlx4/driver.h>
Expand Down Expand Up @@ -276,6 +277,13 @@ struct mlx4_port_info {
struct mlx4_vlan_table vlan_table;
};

struct mlx4_sense {
struct mlx4_dev *dev;
u8 do_sense_port[MLX4_MAX_PORTS + 1];
u8 sense_allowed[MLX4_MAX_PORTS + 1];
struct delayed_work sense_poll;
};

struct mlx4_priv {
struct mlx4_dev dev;

Expand Down Expand Up @@ -305,6 +313,7 @@ struct mlx4_priv {
struct mlx4_uar driver_uar;
void __iomem *kar;
struct mlx4_port_info port[MLX4_MAX_PORTS + 1];
struct mlx4_sense sense;
struct mutex port_mutex;
};

Expand All @@ -313,6 +322,10 @@ static inline struct mlx4_priv *mlx4_priv(struct mlx4_dev *dev)
return container_of(dev, struct mlx4_priv, dev);
}

#define MLX4_SENSE_RANGE (HZ * 3)

extern struct workqueue_struct *mlx4_wq;

u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap);
void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj);
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align);
Expand Down Expand Up @@ -346,8 +359,7 @@ void mlx4_cleanup_mcg_table(struct mlx4_dev *dev);

void mlx4_start_catas_poll(struct mlx4_dev *dev);
void mlx4_stop_catas_poll(struct mlx4_dev *dev);
int mlx4_catas_init(void);
void mlx4_catas_cleanup(void);
void mlx4_catas_init(void);
int mlx4_restart_one(struct pci_dev *pdev);
int mlx4_register_device(struct mlx4_dev *dev);
void mlx4_unregister_device(struct mlx4_dev *dev);
Expand Down Expand Up @@ -379,6 +391,17 @@ void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type);

void mlx4_handle_catas_err(struct mlx4_dev *dev);

void mlx4_do_sense_ports(struct mlx4_dev *dev,
enum mlx4_port_type *stype,
enum mlx4_port_type *defaults);
void mlx4_start_sense(struct mlx4_dev *dev);
void mlx4_stop_sense(struct mlx4_dev *dev);
void mlx4_sense_init(struct mlx4_dev *dev);
int mlx4_check_port_params(struct mlx4_dev *dev,
enum mlx4_port_type *port_type);
int mlx4_change_port_types(struct mlx4_dev *dev,
enum mlx4_port_type *port_types);

void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table);
void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table);

Expand Down
Loading

0 comments on commit 27bf91d

Please sign in to comment.