Skip to content

Commit

Permalink
net/mlx5: Introduce vhca state event notifier
Browse files Browse the repository at this point in the history
vhca state events indicates change in the state of the vhca that may
occur due to a SF allocation, deallocation or enabling/disabling the
SF HCA.

Introduce vhca state event handler which will be used by SF devlink
port manager and SF hardware id allocator in subsequent patches
to act on the event.

This enables single entity to subscribe, query and rearm the event
for a function.

Signed-off-by: Parav Pandit <parav@nvidia.com>
Reviewed-by: Vu Pham <vuhuong@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>
  • Loading branch information
Parav Pandit authored and Saeed Mahameed committed Jan 22, 2021
1 parent a556dde commit f3196bb
Show file tree
Hide file tree
Showing 12 changed files with 422 additions and 0 deletions.
9 changes: 9 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/Kconfig
Original file line number Diff line number Diff line change
Expand Up @@ -203,3 +203,12 @@ config MLX5_SW_STEERING
default y
help
Build support for software-managed steering in the NIC.

config MLX5_SF
bool "Mellanox Technologies subfunction device support using auxiliary device"
depends on MLX5_CORE && MLX5_CORE_EN
default n
help
Build support for subfuction device in the NIC. A Mellanox subfunction
device can support RDMA, netdevice and vdpa device.
It is similar to a SRIOV VF but it doesn't require SRIOV support.
4 changes: 4 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,7 @@ mlx5_core-$(CONFIG_MLX5_SW_STEERING) += steering/dr_domain.o steering/dr_table.o
steering/dr_ste_v0.o \
steering/dr_cmd.o steering/dr_fw.o \
steering/dr_action.o steering/fs_dr.o
#
# SF device
#
mlx5_core-$(CONFIG_MLX5_SF) += sf/vhca_event.o
4 changes: 4 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/cmd.c
Original file line number Diff line number Diff line change
Expand Up @@ -464,6 +464,8 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
case MLX5_CMD_OP_ALLOC_MEMIC:
case MLX5_CMD_OP_MODIFY_XRQ:
case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
case MLX5_CMD_OP_QUERY_VHCA_STATE:
case MLX5_CMD_OP_MODIFY_VHCA_STATE:
*status = MLX5_DRIVER_STATUS_ABORTED;
*synd = MLX5_DRIVER_SYND;
return -EIO;
Expand Down Expand Up @@ -657,6 +659,8 @@ const char *mlx5_command_str(int command)
MLX5_COMMAND_STR_CASE(DESTROY_UMEM);
MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR);
MLX5_COMMAND_STR_CASE(MODIFY_XRQ);
MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE);
MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE);
default: return "unknown command opcode";
}
}
Expand Down
3 changes: 3 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/eq.c
Original file line number Diff line number Diff line change
Expand Up @@ -595,6 +595,9 @@ static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4])
async_event_mask |=
(1ull << MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED);

if (MLX5_CAP_GEN_MAX(dev, vhca_state))
async_event_mask |= (1ull << MLX5_EVENT_TYPE_VHCA_STATE_CHANGE);

mask[0] = async_event_mask;

if (MLX5_CAP_GEN(dev, event_cap))
Expand Down
7 changes: 7 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/events.c
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ static const char *eqe_type_str(u8 type)
return "MLX5_EVENT_TYPE_CMD";
case MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED:
return "MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED";
case MLX5_EVENT_TYPE_VHCA_STATE_CHANGE:
return "MLX5_EVENT_TYPE_VHCA_STATE_CHANGE";
case MLX5_EVENT_TYPE_PAGE_REQUEST:
return "MLX5_EVENT_TYPE_PAGE_REQUEST";
case MLX5_EVENT_TYPE_PAGE_FAULT:
Expand Down Expand Up @@ -403,3 +405,8 @@ int mlx5_notifier_call_chain(struct mlx5_events *events, unsigned int event, voi
{
return atomic_notifier_call_chain(&events->nh, event, data);
}

void mlx5_events_work_enqueue(struct mlx5_core_dev *dev, struct work_struct *work)
{
queue_work(dev->priv.events->wq, work);
}
16 changes: 16 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/main.c
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
#include "ecpf.h"
#include "lib/hv_vhca.h"
#include "diag/rsc_dump.h"
#include "sf/vhca_event.h"

MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver");
Expand Down Expand Up @@ -567,6 +568,8 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx)
if (MLX5_CAP_GEN_MAX(dev, mkey_by_name))
MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1);

mlx5_vhca_state_cap_handle(dev, set_hca_cap);

return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
}

Expand Down Expand Up @@ -884,6 +887,12 @@ static int mlx5_init_once(struct mlx5_core_dev *dev)
goto err_eswitch_cleanup;
}

err = mlx5_vhca_event_init(dev);
if (err) {
mlx5_core_err(dev, "Failed to init vhca event notifier %d\n", err);
goto err_fpga_cleanup;
}

dev->dm = mlx5_dm_create(dev);
if (IS_ERR(dev->dm))
mlx5_core_warn(dev, "Failed to init device memory%d\n", err);
Expand All @@ -894,6 +903,8 @@ static int mlx5_init_once(struct mlx5_core_dev *dev)

return 0;

err_fpga_cleanup:
mlx5_fpga_cleanup(dev);
err_eswitch_cleanup:
mlx5_eswitch_cleanup(dev->priv.eswitch);
err_sriov_cleanup:
Expand Down Expand Up @@ -925,6 +936,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
mlx5_hv_vhca_destroy(dev->hv_vhca);
mlx5_fw_tracer_destroy(dev->tracer);
mlx5_dm_cleanup(dev);
mlx5_vhca_event_cleanup(dev);
mlx5_fpga_cleanup(dev);
mlx5_eswitch_cleanup(dev->priv.eswitch);
mlx5_sriov_cleanup(dev);
Expand Down Expand Up @@ -1129,6 +1141,8 @@ static int mlx5_load(struct mlx5_core_dev *dev)
goto err_sriov;
}

mlx5_vhca_event_start(dev);

err = mlx5_ec_init(dev);
if (err) {
mlx5_core_err(dev, "Failed to init embedded CPU\n");
Expand All @@ -1146,6 +1160,7 @@ static int mlx5_load(struct mlx5_core_dev *dev)
err_sriov:
mlx5_ec_cleanup(dev);
err_ec:
mlx5_vhca_event_stop(dev);
mlx5_cleanup_fs(dev);
err_fs:
mlx5_accel_tls_cleanup(dev);
Expand Down Expand Up @@ -1173,6 +1188,7 @@ static void mlx5_unload(struct mlx5_core_dev *dev)
{
mlx5_sriov_detach(dev);
mlx5_ec_cleanup(dev);
mlx5_vhca_event_stop(dev);
mlx5_cleanup_fs(dev);
mlx5_accel_ipsec_cleanup(dev);
mlx5_accel_tls_cleanup(dev);
Expand Down
2 changes: 2 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
Original file line number Diff line number Diff line change
Expand Up @@ -259,4 +259,6 @@ void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state);

void mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup);
int mlx5_load_one(struct mlx5_core_dev *dev, bool boot);

void mlx5_events_work_enqueue(struct mlx5_core_dev *dev, struct work_struct *work);
#endif /* __MLX5_CORE_H__ */
82 changes: 82 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/sf/mlx5_ifc_vhca_event.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2020 Mellanox Technologies Ltd */

#ifndef __MLX5_IFC_VHCA_EVENT_H__
#define __MLX5_IFC_VHCA_EVENT_H__

enum mlx5_ifc_vhca_state {
MLX5_VHCA_STATE_INVALID = 0x0,
MLX5_VHCA_STATE_ALLOCATED = 0x1,
MLX5_VHCA_STATE_ACTIVE = 0x2,
MLX5_VHCA_STATE_IN_USE = 0x3,
MLX5_VHCA_STATE_TEARDOWN_REQUEST = 0x4,
};

struct mlx5_ifc_vhca_state_context_bits {
u8 arm_change_event[0x1];
u8 reserved_at_1[0xb];
u8 vhca_state[0x4];
u8 reserved_at_10[0x10];

u8 sw_function_id[0x20];

u8 reserved_at_40[0x80];
};

struct mlx5_ifc_query_vhca_state_out_bits {
u8 status[0x8];
u8 reserved_at_8[0x18];

u8 syndrome[0x20];

u8 reserved_at_40[0x40];

struct mlx5_ifc_vhca_state_context_bits vhca_state_context;
};

struct mlx5_ifc_query_vhca_state_in_bits {
u8 opcode[0x10];
u8 uid[0x10];

u8 reserved_at_20[0x10];
u8 op_mod[0x10];

u8 embedded_cpu_function[0x1];
u8 reserved_at_41[0xf];
u8 function_id[0x10];

u8 reserved_at_60[0x20];
};

struct mlx5_ifc_vhca_state_field_select_bits {
u8 reserved_at_0[0x1e];
u8 sw_function_id[0x1];
u8 arm_change_event[0x1];
};

struct mlx5_ifc_modify_vhca_state_out_bits {
u8 status[0x8];
u8 reserved_at_8[0x18];

u8 syndrome[0x20];

u8 reserved_at_40[0x40];
};

struct mlx5_ifc_modify_vhca_state_in_bits {
u8 opcode[0x10];
u8 uid[0x10];

u8 reserved_at_20[0x10];
u8 op_mod[0x10];

u8 embedded_cpu_function[0x1];
u8 reserved_at_41[0xf];
u8 function_id[0x10];

struct mlx5_ifc_vhca_state_field_select_bits vhca_state_field_select;

struct mlx5_ifc_vhca_state_context_bits vhca_state_context;
};

#endif
45 changes: 45 additions & 0 deletions drivers/net/ethernet/mellanox/mlx5/core/sf/sf.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/* Copyright (c) 2020 Mellanox Technologies Ltd */

#ifndef __MLX5_SF_H__
#define __MLX5_SF_H__

#include <linux/mlx5/driver.h>

static inline u16 mlx5_sf_start_function_id(const struct mlx5_core_dev *dev)
{
return MLX5_CAP_GEN(dev, sf_base_id);
}

#ifdef CONFIG_MLX5_SF

static inline bool mlx5_sf_supported(const struct mlx5_core_dev *dev)
{
return MLX5_CAP_GEN(dev, sf);
}

static inline u16 mlx5_sf_max_functions(const struct mlx5_core_dev *dev)
{
if (!mlx5_sf_supported(dev))
return 0;
if (MLX5_CAP_GEN(dev, max_num_sf))
return MLX5_CAP_GEN(dev, max_num_sf);
else
return 1 << MLX5_CAP_GEN(dev, log_max_sf);
}

#else

static inline bool mlx5_sf_supported(const struct mlx5_core_dev *dev)
{
return false;
}

static inline u16 mlx5_sf_max_functions(const struct mlx5_core_dev *dev)
{
return 0;
}

#endif

#endif
Loading

0 comments on commit f3196bb

Please sign in to comment.