Skip to content

Commit

Permalink
net/mlx5_core: Add pci error handlers to mlx5_core driver
Browse files Browse the repository at this point in the history 
This patch implement the pci_error_handlers for mlx5_core which allow the
driver to recover from PCI error.

Once an error is detected in the PCI, the mlx5_pci_err_detected is called
and it:
1) Marks the device to be in 'Internal Error' state.
2) Dispatches an event to the mlx5_ib to flush all the outstanding cqes
with error.
3) Returns all the on going commands with error.
4) Unloads the driver.

Afterwards, the FW is reset and mlx5_pci_slot_reset is called and it
enables the device and restore it's pci state.

If the later succeeds, mlx5_pci_resume is called, and it loads the SW
stack.

Signed-off-by: Majd Dibbiny <majd@mellanox.com>
Signed-off-by: Eli Cohen <eli@mellanox.com>
Signed-off-by: Or Gerlitz <ogerlitz@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
Majd Dibbiny authored and David S. Miller committed Oct 15, 2015
1 parent fd76ee4 commit 89d44f0
Show file tree
Hide file tree
Showing 6 changed files with 451 additions and 12 deletions.
170 changes: 170 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/cmd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,8 +256,154 @@ static void dump_buf(void *buf, int size, int data_only, int offset)

enum {
MLX5_DRIVER_STATUS_ABORTED = 0xfe,
MLX5_DRIVER_SYND = 0xbadd00de,
};

static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
u32 *synd, u8 *status)
{
*synd = 0;
*status = 0;

switch (op) {
case MLX5_CMD_OP_TEARDOWN_HCA:
case MLX5_CMD_OP_DISABLE_HCA:
case MLX5_CMD_OP_MANAGE_PAGES:
case MLX5_CMD_OP_DESTROY_MKEY:
case MLX5_CMD_OP_DESTROY_EQ:
case MLX5_CMD_OP_DESTROY_CQ:
case MLX5_CMD_OP_DESTROY_QP:
case MLX5_CMD_OP_DESTROY_PSV:
case MLX5_CMD_OP_DESTROY_SRQ:
case MLX5_CMD_OP_DESTROY_XRC_SRQ:
case MLX5_CMD_OP_DESTROY_DCT:
case MLX5_CMD_OP_DEALLOC_Q_COUNTER:
case MLX5_CMD_OP_DEALLOC_PD:
case MLX5_CMD_OP_DEALLOC_UAR:
case MLX5_CMD_OP_DETTACH_FROM_MCG:
case MLX5_CMD_OP_DEALLOC_XRCD:
case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN:
case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT:
case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY:
case MLX5_CMD_OP_DESTROY_TIR:
case MLX5_CMD_OP_DESTROY_SQ:
case MLX5_CMD_OP_DESTROY_RQ:
case MLX5_CMD_OP_DESTROY_RMP:
case MLX5_CMD_OP_DESTROY_TIS:
case MLX5_CMD_OP_DESTROY_RQT:
case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
case MLX5_CMD_OP_DESTROY_FLOW_GROUP:
case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY:
return MLX5_CMD_STAT_OK;

case MLX5_CMD_OP_QUERY_HCA_CAP:
case MLX5_CMD_OP_QUERY_ADAPTER:
case MLX5_CMD_OP_INIT_HCA:
case MLX5_CMD_OP_ENABLE_HCA:
case MLX5_CMD_OP_QUERY_PAGES:
case MLX5_CMD_OP_SET_HCA_CAP:
case MLX5_CMD_OP_QUERY_ISSI:
case MLX5_CMD_OP_SET_ISSI:
case MLX5_CMD_OP_CREATE_MKEY:
case MLX5_CMD_OP_QUERY_MKEY:
case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS:
case MLX5_CMD_OP_PAGE_FAULT_RESUME:
case MLX5_CMD_OP_CREATE_EQ:
case MLX5_CMD_OP_QUERY_EQ:
case MLX5_CMD_OP_GEN_EQE:
case MLX5_CMD_OP_CREATE_CQ:
case MLX5_CMD_OP_QUERY_CQ:
case MLX5_CMD_OP_MODIFY_CQ:
case MLX5_CMD_OP_CREATE_QP:
case MLX5_CMD_OP_RST2INIT_QP:
case MLX5_CMD_OP_INIT2RTR_QP:
case MLX5_CMD_OP_RTR2RTS_QP:
case MLX5_CMD_OP_RTS2RTS_QP:
case MLX5_CMD_OP_SQERR2RTS_QP:
case MLX5_CMD_OP_2ERR_QP:
case MLX5_CMD_OP_2RST_QP:
case MLX5_CMD_OP_QUERY_QP:
case MLX5_CMD_OP_SQD_RTS_QP:
case MLX5_CMD_OP_INIT2INIT_QP:
case MLX5_CMD_OP_CREATE_PSV:
case MLX5_CMD_OP_CREATE_SRQ:
case MLX5_CMD_OP_QUERY_SRQ:
case MLX5_CMD_OP_ARM_RQ:
case MLX5_CMD_OP_CREATE_XRC_SRQ:
case MLX5_CMD_OP_QUERY_XRC_SRQ:
case MLX5_CMD_OP_ARM_XRC_SRQ:
case MLX5_CMD_OP_CREATE_DCT:
case MLX5_CMD_OP_DRAIN_DCT:
case MLX5_CMD_OP_QUERY_DCT:
case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
case MLX5_CMD_OP_QUERY_VPORT_STATE:
case MLX5_CMD_OP_MODIFY_VPORT_STATE:
case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT:
case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT:
case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
case MLX5_CMD_OP_SET_ROCE_ADDRESS:
case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT:
case MLX5_CMD_OP_QUERY_HCA_VPORT_GID:
case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY:
case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
case MLX5_CMD_OP_ALLOC_Q_COUNTER:
case MLX5_CMD_OP_QUERY_Q_COUNTER:
case MLX5_CMD_OP_ALLOC_PD:
case MLX5_CMD_OP_ALLOC_UAR:
case MLX5_CMD_OP_CONFIG_INT_MODERATION:
case MLX5_CMD_OP_ACCESS_REG:
case MLX5_CMD_OP_ATTACH_TO_MCG:
case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
case MLX5_CMD_OP_MAD_IFC:
case MLX5_CMD_OP_QUERY_MAD_DEMUX:
case MLX5_CMD_OP_SET_MAD_DEMUX:
case MLX5_CMD_OP_NOP:
case MLX5_CMD_OP_ALLOC_XRCD:
case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
case MLX5_CMD_OP_QUERY_CONG_STATUS:
case MLX5_CMD_OP_MODIFY_CONG_STATUS:
case MLX5_CMD_OP_QUERY_CONG_PARAMS:
case MLX5_CMD_OP_MODIFY_CONG_PARAMS:
case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
case MLX5_CMD_OP_CREATE_TIR:
case MLX5_CMD_OP_MODIFY_TIR:
case MLX5_CMD_OP_QUERY_TIR:
case MLX5_CMD_OP_CREATE_SQ:
case MLX5_CMD_OP_MODIFY_SQ:
case MLX5_CMD_OP_QUERY_SQ:
case MLX5_CMD_OP_CREATE_RQ:
case MLX5_CMD_OP_MODIFY_RQ:
case MLX5_CMD_OP_QUERY_RQ:
case MLX5_CMD_OP_CREATE_RMP:
case MLX5_CMD_OP_MODIFY_RMP:
case MLX5_CMD_OP_QUERY_RMP:
case MLX5_CMD_OP_CREATE_TIS:
case MLX5_CMD_OP_MODIFY_TIS:
case MLX5_CMD_OP_QUERY_TIS:
case MLX5_CMD_OP_CREATE_RQT:
case MLX5_CMD_OP_MODIFY_RQT:
case MLX5_CMD_OP_QUERY_RQT:
case MLX5_CMD_OP_CREATE_FLOW_TABLE:
case MLX5_CMD_OP_QUERY_FLOW_TABLE:
case MLX5_CMD_OP_CREATE_FLOW_GROUP:
case MLX5_CMD_OP_QUERY_FLOW_GROUP:
case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
*status = MLX5_DRIVER_STATUS_ABORTED;
*synd = MLX5_DRIVER_SYND;
return -EIO;
default:
mlx5_core_err(dev, "Unknown FW command (%d)\n", op);
return -EINVAL;
}
}

const char *mlx5_command_str(int command)
{
switch (command) {
Expand Down Expand Up @@ -592,6 +738,16 @@ static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
return err;
}

static __be32 *get_synd_ptr(struct mlx5_outbox_hdr *out)
{
return &out->syndrome;
}

static u8 *get_status_ptr(struct mlx5_outbox_hdr *out)
{
return &out->status;
}

/* Notes:
* 1. Callback functions may not sleep
* 2. page queue commands do not support asynchrous completion
Expand Down Expand Up @@ -1200,6 +1356,11 @@ static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,
return msg;
}

static u16 opcode_from_in(struct mlx5_inbox_hdr *in)
{
return be16_to_cpu(in->opcode);
}

static int is_manage_pages(struct mlx5_inbox_hdr *in)
{
return be16_to_cpu(in->opcode) == MLX5_CMD_OP_MANAGE_PAGES;
Expand All @@ -1214,6 +1375,15 @@ static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
gfp_t gfp;
int err;
u8 status = 0;
u32 drv_synd;

if (pci_channel_offline(dev->pdev) ||
dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
err = mlx5_internal_err_ret_value(dev, opcode_from_in(in), &drv_synd, &status);
*get_synd_ptr(out) = cpu_to_be32(drv_synd);
*get_status_ptr(out) = status;
return err;
}

pages_queue = is_manage_pages(in);
gfp = callback ? GFP_ATOMIC : GFP_KERNEL;
Expand Down
72 changes: 72 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/health.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include <linux/module.h>
#include <linux/random.h>
#include <linux/vmalloc.h>
#include <linux/hardirq.h>
#include <linux/mlx5/driver.h>
#include <linux/mlx5/cmd.h>
#include "mlx5_core.h"
Expand Down Expand Up @@ -68,6 +69,29 @@ static u8 get_nic_interface(struct mlx5_core_dev *dev)
return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3;
}

static void trigger_cmd_completions(struct mlx5_core_dev *dev)
{
unsigned long flags;
u64 vector;

/* wait for pending handlers to complete */
synchronize_irq(dev->priv.msix_arr[MLX5_EQ_VEC_CMD].vector);
spin_lock_irqsave(&dev->cmd.alloc_lock, flags);
vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1);
if (!vector)
goto no_trig;

vector |= MLX5_TRIGGERED_CMD_COMP;
spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);

mlx5_core_dbg(dev, "vector 0x%llx\n", vector);
mlx5_cmd_comp_handler(dev, vector);
return;

no_trig:
spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
}

static int in_fatal(struct mlx5_core_dev *dev)
{
struct mlx5_core_health *health = &dev->priv.health;
Expand All @@ -82,6 +106,43 @@ static int in_fatal(struct mlx5_core_dev *dev)
return 0;
}

void mlx5_enter_error_state(struct mlx5_core_dev *dev)
{
if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
return;

mlx5_core_err(dev, "start\n");
if (pci_channel_offline(dev->pdev) || in_fatal(dev))
dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;

mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 0);
mlx5_core_err(dev, "end\n");
}

static void mlx5_handle_bad_state(struct mlx5_core_dev *dev)
{
u8 nic_interface = get_nic_interface(dev);

switch (nic_interface) {
case MLX5_NIC_IFC_FULL:
mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n");
break;

case MLX5_NIC_IFC_DISABLED:
mlx5_core_warn(dev, "starting teardown\n");
break;

case MLX5_NIC_IFC_NO_DRAM_NIC:
mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n");
break;
default:
mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n",
nic_interface);
}

mlx5_disable_device(dev);
}

static void health_care(struct work_struct *work)
{
struct mlx5_core_health *health;
Expand All @@ -92,6 +153,7 @@ static void health_care(struct work_struct *work)
priv = container_of(health, struct mlx5_priv, health);
dev = container_of(priv, struct mlx5_core_dev, priv);
mlx5_core_warn(dev, "handling bad device here\n");
mlx5_handle_bad_state(dev);
}

static const char *hsynd_str(u8 synd)
Expand Down Expand Up @@ -147,6 +209,10 @@ static void print_health_info(struct mlx5_core_dev *dev)
u32 fw;
int i;

/* If the syndrom is 0, the device is OK and no need to print buffer */
if (!ioread8(&h->synd))
return;

for (i = 0; i < ARRAY_SIZE(h->assert_var); i++)
dev_err(&dev->pdev->dev, "assert_var[%d] 0x%08x\n", i, ioread32be(h->assert_var + i));

Expand Down Expand Up @@ -178,6 +244,12 @@ static void poll_health(unsigned long data)
struct mlx5_core_health *health = &dev->priv.health;
u32 count;

if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
trigger_cmd_completions(dev);
mod_timer(&health->timer, get_next_poll_jiffies());
return;
}

count = ioread32be(health->health_counter);
if (count == health->prev)
++health->miss_counter;
Expand Down
Loading

0 comments on commit 89d44f0

Please sign in to comment.