Skip to content

Commit

Permalink
Merge branch 'ionic-fw-recovery'
Browse files Browse the repository at this point in the history 
Shannon Nelson says:

====================
ionic: updates for stable FW recovery

Recent FW work has tightened up timings in its error recovery
handling and uncovered weaknesses in the driver's responses,
so this is a set of updates primarily for better handling of
the firmware's recovery mechanisms.
====================

Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
David S. Miller committed Jan 25, 2022
2 parents ca73b68 + 36197d8 commit 8a0de61
Show file tree
Hide file tree
Showing 9 changed files with 409 additions and 202 deletions.
7 changes: 6 additions & 1 deletion drivers/net/ethernet/pensando/ionic/ionic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ struct ionic_lif;
#define PCI_DEVICE_ID_PENSANDO_IONIC_ETH_PF 0x1002
#define PCI_DEVICE_ID_PENSANDO_IONIC_ETH_VF 0x1003

#define DEVCMD_TIMEOUT 10
#define DEVCMD_TIMEOUT 5
#define IONIC_ADMINQ_TIME_SLICE msecs_to_jiffies(100)

#define IONIC_PHC_UPDATE_NS 10000000000 /* 10s in nanoseconds */
Expand Down Expand Up @@ -78,6 +78,9 @@ void ionic_adminq_netdev_err_print(struct ionic_lif *lif, u8 opcode,
u8 status, int err);

int ionic_dev_cmd_wait(struct ionic *ionic, unsigned long max_wait);
int ionic_dev_cmd_wait_nomsg(struct ionic *ionic, unsigned long max_wait);
void ionic_dev_cmd_dev_err_print(struct ionic *ionic, u8 opcode, u8 status,
int err);
int ionic_set_dma_mask(struct ionic *ionic);
int ionic_setup(struct ionic *ionic);

Expand All @@ -89,4 +92,6 @@ int ionic_port_identify(struct ionic *ionic);
int ionic_port_init(struct ionic *ionic);
int ionic_port_reset(struct ionic *ionic);

const char *ionic_vf_attr_to_str(enum ionic_vf_attr attr);

#endif /* _IONIC_H_ */
17 changes: 10 additions & 7 deletions drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ void ionic_bus_unmap_dbpage(struct ionic *ionic, void __iomem *page)

static void ionic_vf_dealloc_locked(struct ionic *ionic)
{
struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR };
struct ionic_vf *v;
dma_addr_t dma = 0;
int i;

if (!ionic->vfs)
Expand All @@ -120,9 +120,8 @@ static void ionic_vf_dealloc_locked(struct ionic *ionic)
v = &ionic->vfs[i];

if (v->stats_pa) {
(void)ionic_set_vf_config(ionic, i,
IONIC_VF_ATTR_STATSADDR,
(u8 *)&dma);
vfc.stats_pa = 0;
(void)ionic_set_vf_config(ionic, i, &vfc);
dma_unmap_single(ionic->dev, v->stats_pa,
sizeof(v->stats), DMA_FROM_DEVICE);
v->stats_pa = 0;
Expand All @@ -143,6 +142,7 @@ static void ionic_vf_dealloc(struct ionic *ionic)

static int ionic_vf_alloc(struct ionic *ionic, int num_vfs)
{
struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_STATSADDR };
struct ionic_vf *v;
int err = 0;
int i;
Expand All @@ -166,9 +166,10 @@ static int ionic_vf_alloc(struct ionic *ionic, int num_vfs)
}

ionic->num_vfs++;

/* ignore failures from older FW, we just won't get stats */
(void)ionic_set_vf_config(ionic, i, IONIC_VF_ATTR_STATSADDR,
(u8 *)&v->stats_pa);
vfc.stats_pa = cpu_to_le64(v->stats_pa);
(void)ionic_set_vf_config(ionic, i, &vfc);
}

out:
Expand Down Expand Up @@ -331,6 +332,9 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
goto err_out_deregister_lifs;
}

mod_timer(&ionic->watchdog_timer,
round_jiffies(jiffies + ionic->watchdog_period));

return 0;

err_out_deregister_lifs:
Expand All @@ -348,7 +352,6 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
err_out_reset:
ionic_reset(ionic);
err_out_teardown:
del_timer_sync(&ionic->watchdog_timer);
pci_clear_master(pdev);
/* Don't fail the probe for these errors, keep
* the hw interface around for inspection
Expand Down
162 changes: 95 additions & 67 deletions drivers/net/ethernet/pensando/ionic/ionic_dev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ static void ionic_watchdog_cb(struct timer_list *t)
!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
ionic_link_status_check_request(lif, CAN_NOT_SLEEP);

if (test_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state)) {
if (test_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state) &&
!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
work = kzalloc(sizeof(*work), GFP_ATOMIC);
if (!work) {
netdev_err(lif->netdev, "rxmode change dropped\n");
Expand All @@ -46,6 +47,24 @@ static void ionic_watchdog_cb(struct timer_list *t)
}
}

static void ionic_watchdog_init(struct ionic *ionic)
{
struct ionic_dev *idev = &ionic->idev;

timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0);
ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ;

/* set times to ensure the first check will proceed */
atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ);
idev->last_hb_time = jiffies - 2 * ionic->watchdog_period;
/* init as ready, so no transition if the first check succeeds */
idev->last_fw_hb = 0;
idev->fw_hb_ready = true;
idev->fw_status_ready = true;
idev->fw_generation = IONIC_FW_STS_F_GENERATION &
ioread8(&idev->dev_info_regs->fw_status);
}

void ionic_init_devinfo(struct ionic *ionic)
{
struct ionic_dev *idev = &ionic->idev;
Expand Down Expand Up @@ -109,21 +128,7 @@ int ionic_dev_setup(struct ionic *ionic)
return -EFAULT;
}

timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0);
ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ;

/* set times to ensure the first check will proceed */
atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ);
idev->last_hb_time = jiffies - 2 * ionic->watchdog_period;
/* init as ready, so no transition if the first check succeeds */
idev->last_fw_hb = 0;
idev->fw_hb_ready = true;
idev->fw_status_ready = true;
idev->fw_generation = IONIC_FW_STS_F_GENERATION &
ioread8(&idev->dev_info_regs->fw_status);

mod_timer(&ionic->watchdog_timer,
round_jiffies(jiffies + ionic->watchdog_period));
ionic_watchdog_init(ionic);

idev->db_pages = bar->vaddr;
idev->phy_db_pages = bar->bus_addr;
Expand All @@ -132,10 +137,21 @@ int ionic_dev_setup(struct ionic *ionic)
}

/* Devcmd Interface */
bool ionic_is_fw_running(struct ionic_dev *idev)
{
u8 fw_status = ioread8(&idev->dev_info_regs->fw_status);

/* firmware is useful only if the running bit is set and
* fw_status != 0xff (bad PCI read)
*/
return (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING);
}

int ionic_heartbeat_check(struct ionic *ionic)
{
struct ionic_dev *idev = &ionic->idev;
unsigned long check_time, last_check_time;
struct ionic_dev *idev = &ionic->idev;
struct ionic_lif *lif = ionic->lif;
bool fw_status_ready = true;
bool fw_hb_ready;
u8 fw_generation;
Expand All @@ -155,13 +171,10 @@ int ionic_heartbeat_check(struct ionic *ionic)
goto do_check_time;
}

/* firmware is useful only if the running bit is set and
* fw_status != 0xff (bad PCI read)
* If fw_status is not ready don't bother with the generation.
*/
fw_status = ioread8(&idev->dev_info_regs->fw_status);

if (fw_status == 0xff || !(fw_status & IONIC_FW_STS_F_RUNNING)) {
/* If fw_status is not ready don't bother with the generation */
if (!ionic_is_fw_running(idev)) {
fw_status_ready = false;
} else {
fw_generation = fw_status & IONIC_FW_STS_F_GENERATION;
Expand All @@ -176,31 +189,41 @@ int ionic_heartbeat_check(struct ionic *ionic)
* the down, the next watchdog will see the fw is up
* and the generation value stable, so will trigger
* the fw-up activity.
*
* If we had already moved to FW_RESET from a RESET event,
* it is possible that we never saw the fw_status go to 0,
* so we fake the current idev->fw_status_ready here to
* force the transition and get FW up again.
*/
fw_status_ready = false;
if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
idev->fw_status_ready = false; /* go to running */
else
fw_status_ready = false; /* go to down */
}
}

/* is this a transition? */
if (fw_status_ready != idev->fw_status_ready) {
struct ionic_lif *lif = ionic->lif;
bool trigger = false;

idev->fw_status_ready = fw_status_ready;

if (!fw_status_ready) {
dev_info(ionic->dev, "FW stopped %u\n", fw_status);
if (lif && !test_bit(IONIC_LIF_F_FW_RESET, lif->state))
trigger = true;
} else {
dev_info(ionic->dev, "FW running %u\n", fw_status);
if (lif && test_bit(IONIC_LIF_F_FW_RESET, lif->state))
trigger = true;
if (!fw_status_ready && lif &&
!test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
!test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
dev_info(ionic->dev, "FW stopped 0x%02x\n", fw_status);
trigger = true;

} else if (fw_status_ready && lif &&
test_bit(IONIC_LIF_F_FW_RESET, lif->state) &&
!test_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) {
dev_info(ionic->dev, "FW running 0x%02x\n", fw_status);
trigger = true;
}

if (trigger) {
struct ionic_deferred_work *work;

idev->fw_status_ready = fw_status_ready;

work = kzalloc(sizeof(*work), GFP_ATOMIC);
if (work) {
work->type = IONIC_DW_TYPE_LIF_RESET;
Expand All @@ -210,12 +233,14 @@ int ionic_heartbeat_check(struct ionic *ionic)
}
}

if (!fw_status_ready)
if (!idev->fw_status_ready)
return -ENXIO;

/* wait at least one watchdog period since the last heartbeat */
/* Because of some variability in the actual FW heartbeat, we
* wait longer than the DEVCMD_TIMEOUT before checking again.
*/
last_check_time = idev->last_hb_time;
if (time_before(check_time, last_check_time + ionic->watchdog_period))
if (time_before(check_time, last_check_time + DEVCMD_TIMEOUT * 2 * HZ))
return 0;

fw_hb = ioread32(&idev->dev_info_regs->fw_heartbeat);
Expand Down Expand Up @@ -392,60 +417,63 @@ void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type)
}

/* VF commands */
int ionic_set_vf_config(struct ionic *ionic, int vf, u8 attr, u8 *data)
int ionic_set_vf_config(struct ionic *ionic, int vf,
struct ionic_vf_setattr_cmd *vfc)
{
union ionic_dev_cmd cmd = {
.vf_setattr.opcode = IONIC_CMD_VF_SETATTR,
.vf_setattr.attr = attr,
.vf_setattr.attr = vfc->attr,
.vf_setattr.vf_index = cpu_to_le16(vf),
};
int err;

memcpy(cmd.vf_setattr.pad, vfc->pad, sizeof(vfc->pad));

mutex_lock(&ionic->dev_cmd_lock);
ionic_dev_cmd_go(&ionic->idev, &cmd);
err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
mutex_unlock(&ionic->dev_cmd_lock);

return err;
}

int ionic_dev_cmd_vf_getattr(struct ionic *ionic, int vf, u8 attr,
struct ionic_vf_getattr_comp *comp)
{
union ionic_dev_cmd cmd = {
.vf_getattr.opcode = IONIC_CMD_VF_GETATTR,
.vf_getattr.attr = attr,
.vf_getattr.vf_index = cpu_to_le16(vf),
};
int err;

if (vf >= ionic->num_vfs)
return -EINVAL;

switch (attr) {
case IONIC_VF_ATTR_SPOOFCHK:
cmd.vf_setattr.spoofchk = *data;
dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
__func__, vf, *data);
break;
case IONIC_VF_ATTR_TRUST:
cmd.vf_setattr.trust = *data;
dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
__func__, vf, *data);
break;
case IONIC_VF_ATTR_LINKSTATE:
cmd.vf_setattr.linkstate = *data;
dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
__func__, vf, *data);
break;
case IONIC_VF_ATTR_MAC:
ether_addr_copy(cmd.vf_setattr.macaddr, data);
dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
__func__, vf, data);
break;
case IONIC_VF_ATTR_VLAN:
cmd.vf_setattr.vlanid = cpu_to_le16(*(u16 *)data);
dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
__func__, vf, *(u16 *)data);
break;
case IONIC_VF_ATTR_RATE:
cmd.vf_setattr.maxrate = cpu_to_le32(*(u32 *)data);
dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
__func__, vf, *(u32 *)data);
break;
case IONIC_VF_ATTR_STATSADDR:
cmd.vf_setattr.stats_pa = cpu_to_le64(*(u64 *)data);
dev_dbg(ionic->dev, "%s: vf %d stats_pa 0x%08llx\n",
__func__, vf, *(u64 *)data);
break;
default:
return -EINVAL;
}

mutex_lock(&ionic->dev_cmd_lock);
ionic_dev_cmd_go(&ionic->idev, &cmd);
err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
err = ionic_dev_cmd_wait_nomsg(ionic, DEVCMD_TIMEOUT);
memcpy_fromio(comp, &ionic->idev.dev_cmd_regs->comp.vf_getattr,
sizeof(*comp));
mutex_unlock(&ionic->dev_cmd_lock);

if (err && comp->status != IONIC_RC_ENOSUPP)
ionic_dev_cmd_dev_err_print(ionic, cmd.vf_getattr.opcode,
comp->status, err);

return err;
}

Expand Down
6 changes: 5 additions & 1 deletion drivers/net/ethernet/pensando/ionic/ionic_dev.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,10 @@ void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable);
void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type);
void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type);

int ionic_set_vf_config(struct ionic *ionic, int vf, u8 attr, u8 *data);
int ionic_set_vf_config(struct ionic *ionic, int vf,
struct ionic_vf_setattr_cmd *vfc);
int ionic_dev_cmd_vf_getattr(struct ionic *ionic, int vf, u8 attr,
struct ionic_vf_getattr_comp *comp);
void ionic_dev_cmd_queue_identify(struct ionic_dev *idev,
u16 lif_type, u8 qtype, u8 qver);
void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver);
Expand Down Expand Up @@ -353,5 +356,6 @@ void ionic_q_rewind(struct ionic_queue *q, struct ionic_desc_info *start);
void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
unsigned int stop_index);
int ionic_heartbeat_check(struct ionic *ionic);
bool ionic_is_fw_running(struct ionic_dev *idev);

#endif /* _IONIC_DEV_H_ */
Loading

0 comments on commit 8a0de61

Please sign in to comment.