From c673b6772abcc4161b80014b5f5ff01bad703f0b Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Mon, 14 Mar 2022 00:22:40 +0200 Subject: [PATCH 01/15] net/mlx5: Header file for crypto Take crypto API out of the generic mlx5.h header into a dedicated header. Signed-off-by: Tariq Toukan Signed-off-by: Jianbo Liu Signed-off-by: Saeed Mahameed --- .../mlx5/core/en_accel/ipsec_offload.c | 2 +- .../mellanox/mlx5/core/en_accel/ktls.c | 1 + .../mellanox/mlx5/core/en_accel/macsec.c | 2 +- .../ethernet/mellanox/mlx5/core/lib/crypto.c | 2 +- .../ethernet/mellanox/mlx5/core/lib/crypto.h | 19 +++++++++++++++++++ .../ethernet/mellanox/mlx5/core/lib/mlx5.h | 12 ------------ 6 files changed, 23 insertions(+), 15 deletions(-) create mode 100644 drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 2461462b7b991..57ac0f663fcdb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -4,7 +4,7 @@ #include "mlx5_core.h" #include "en.h" #include "ipsec.h" -#include "lib/mlx5.h" +#include "lib/crypto.h" enum { MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index eb5b09f81decb..d36788119b8bf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -4,6 +4,7 @@ #include #include "en.h" #include "lib/mlx5.h" +#include "lib/crypto.h" #include "en_accel/ktls.h" #include "en_accel/ktls_utils.h" #include "en_accel/fs_tcp.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index 7f6b940830b31..08d0929e82603 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -7,7 +7,7 @@ #include "en.h" #include "lib/aso.h" -#include "lib/mlx5.h" +#include "lib/crypto.h" #include "en_accel/macsec.h" #include "en_accel/macsec_fs.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index e995f8378df73..2521f31d36b78 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -2,7 +2,7 @@ // Copyright (c) 2019 Mellanox Technologies. #include "mlx5_core.h" -#include "lib/mlx5.h" +#include "lib/crypto.h" int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, void *key, u32 sz_bytes, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h new file mode 100644 index 0000000000000..cb85da9e39645 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_LIB_CRYPTO_H__ +#define __MLX5_LIB_CRYPTO_H__ + +enum { + MLX5_ACCEL_OBJ_TLS_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS, + MLX5_ACCEL_OBJ_IPSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC, + MLX5_ACCEL_OBJ_MACSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_MACSEC, +}; + +int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, + void *key, u32 sz_bytes, + u32 key_type, u32 *p_key_id); + +void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id); + +#endif /* __MLX5_LIB_CRYPTO_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h index 032adb21ad4b6..55bd7c4c021e7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h @@ -79,18 +79,6 @@ struct mlx5_pme_stats { void mlx5_get_pme_stats(struct mlx5_core_dev *dev, struct mlx5_pme_stats *stats); int mlx5_notifier_call_chain(struct mlx5_events *events, unsigned int event, void *data); -/* Crypto */ -enum { - MLX5_ACCEL_OBJ_TLS_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS, - MLX5_ACCEL_OBJ_IPSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC, - MLX5_ACCEL_OBJ_MACSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_MACSEC, -}; - -int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, - void *key, u32 sz_bytes, - u32 key_type, u32 *p_key_id); -void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id); - static inline struct net *mlx5_core_net(struct mlx5_core_dev *dev) { return devlink_net(priv_to_devlink(dev)); From 4744c7ad2299108bc3ecbea574cffb052863ba26 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Wed, 6 Jul 2022 13:06:30 +0000 Subject: [PATCH 02/15] net/mlx5: Add IFC bits for general obj create param Before this patch, the log_obj_range was defined inside general_obj_in_cmd_hdr to support bulk allocation. However, we need to modify/query one of the object in the bulk in later patch, so change those fields to param bits for parameters specific for cmd header, and add general_obj_create_param according to what was updated in spec. We will also add general_obj_query_param for modify/query later. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c | 6 ++++-- include/linux/mlx5/mlx5_ifc.h | 11 ++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c index 7758a425bfa82..8218c892b1617 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c @@ -204,13 +204,15 @@ mlx5e_flow_meter_create_aso_obj(struct mlx5e_flow_meters *flow_meters, int *obj_ u32 in[MLX5_ST_SZ_DW(create_flow_meter_aso_obj_in)] = {}; u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; struct mlx5_core_dev *mdev = flow_meters->mdev; - void *obj; + void *obj, *param; int err; MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO); - MLX5_SET(general_obj_in_cmd_hdr, in, log_obj_range, flow_meters->log_granularity); + param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); + MLX5_SET(general_obj_create_param, param, log_obj_range, + flow_meters->log_granularity); obj = MLX5_ADDR_OF(create_flow_meter_aso_obj_in, in, flow_meter_aso_obj); MLX5_SET(flow_meter_aso_obj, obj, meter_aso_access_pd, flow_meters->pdn); diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 0b102c651fe2d..17e293ceb6251 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -6196,6 +6196,13 @@ struct mlx5_ifc_match_definer_bits { }; }; +struct mlx5_ifc_general_obj_create_param_bits { + u8 alias_object[0x1]; + u8 reserved_at_1[0x2]; + u8 log_obj_range[0x5]; + u8 reserved_at_8[0x18]; +}; + struct mlx5_ifc_general_obj_in_cmd_hdr_bits { u8 opcode[0x10]; u8 uid[0x10]; @@ -6205,9 +6212,7 @@ struct mlx5_ifc_general_obj_in_cmd_hdr_bits { u8 obj_id[0x20]; - u8 reserved_at_60[0x3]; - u8 log_obj_range[0x5]; - u8 reserved_at_68[0x18]; + struct mlx5_ifc_general_obj_create_param_bits op_param; }; struct mlx5_ifc_general_obj_out_cmd_hdr_bits { From 9a0ed4f2bfe220e9389cc86d7f9b284119037eb8 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Thu, 17 Feb 2022 09:29:35 +0000 Subject: [PATCH 03/15] net/mlx5: Add IFC bits and enums for crypto key Add and extend structure layouts and defines for fast crypto key update. This is a prerequisite to support bulk creation, key modification and destruction, software wrapped DEK, and SYNC_CRYPTO command. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- include/linux/mlx5/mlx5_ifc.h | 146 ++++++++++++++++++++++++++++++++-- 1 file changed, 140 insertions(+), 6 deletions(-) diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 17e293ceb6251..7143b65f9f4aa 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -306,6 +306,7 @@ enum { MLX5_CMD_OP_SYNC_STEERING = 0xb00, MLX5_CMD_OP_QUERY_VHCA_STATE = 0xb0d, MLX5_CMD_OP_MODIFY_VHCA_STATE = 0xb0e, + MLX5_CMD_OP_SYNC_CRYPTO = 0xb12, MLX5_CMD_OP_MAX }; @@ -1112,6 +1113,30 @@ struct mlx5_ifc_sync_steering_out_bits { u8 reserved_at_40[0x40]; }; +struct mlx5_ifc_sync_crypto_in_bits { + u8 opcode[0x10]; + u8 uid[0x10]; + + u8 reserved_at_20[0x10]; + u8 op_mod[0x10]; + + u8 reserved_at_40[0x20]; + + u8 reserved_at_60[0x10]; + u8 crypto_type[0x10]; + + u8 reserved_at_80[0x80]; +}; + +struct mlx5_ifc_sync_crypto_out_bits { + u8 status[0x8]; + u8 reserved_at_8[0x18]; + + u8 syndrome[0x20]; + + u8 reserved_at_40[0x40]; +}; + struct mlx5_ifc_device_mem_cap_bits { u8 memic[0x1]; u8 reserved_at_1[0x1f]; @@ -1768,7 +1793,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 ats[0x1]; u8 reserved_at_462[0x1]; u8 log_max_uctx[0x5]; - u8 reserved_at_468[0x2]; + u8 reserved_at_468[0x1]; + u8 crypto[0x1]; u8 ipsec_offload[0x1]; u8 log_max_umem[0x5]; u8 max_num_eqs[0x10]; @@ -3351,6 +3377,30 @@ struct mlx5_ifc_shampo_cap_bits { u8 reserved_at_40[0x7c0]; }; +struct mlx5_ifc_crypto_cap_bits { + u8 reserved_at_0[0x3]; + u8 synchronize_dek[0x1]; + u8 int_kek_manual[0x1]; + u8 int_kek_auto[0x1]; + u8 reserved_at_6[0x1a]; + + u8 reserved_at_20[0x3]; + u8 log_dek_max_alloc[0x5]; + u8 reserved_at_28[0x3]; + u8 log_max_num_deks[0x5]; + u8 reserved_at_30[0x10]; + + u8 reserved_at_40[0x20]; + + u8 reserved_at_60[0x3]; + u8 log_dek_granularity[0x5]; + u8 reserved_at_68[0x3]; + u8 log_max_num_int_kek[0x5]; + u8 sw_wrapped_dek[0x10]; + + u8 reserved_at_80[0x780]; +}; + union mlx5_ifc_hca_cap_union_bits { struct mlx5_ifc_cmd_hca_cap_bits cmd_hca_cap; struct mlx5_ifc_cmd_hca_cap_2_bits cmd_hca_cap_2; @@ -3371,6 +3421,7 @@ union mlx5_ifc_hca_cap_union_bits { struct mlx5_ifc_virtio_emulation_cap_bits virtio_emulation_cap; struct mlx5_ifc_shampo_cap_bits shampo_cap; struct mlx5_ifc_macsec_cap_bits macsec_cap; + struct mlx5_ifc_crypto_cap_bits crypto_cap; u8 reserved_at_0[0x8000]; }; @@ -6203,6 +6254,11 @@ struct mlx5_ifc_general_obj_create_param_bits { u8 reserved_at_8[0x18]; }; +struct mlx5_ifc_general_obj_query_param_bits { + u8 alias_object[0x1]; + u8 obj_offset[0x1f]; +}; + struct mlx5_ifc_general_obj_in_cmd_hdr_bits { u8 opcode[0x10]; u8 uid[0x10]; @@ -6212,7 +6268,10 @@ struct mlx5_ifc_general_obj_in_cmd_hdr_bits { u8 obj_id[0x20]; - struct mlx5_ifc_general_obj_create_param_bits op_param; + union { + struct mlx5_ifc_general_obj_create_param_bits create; + struct mlx5_ifc_general_obj_query_param_bits query; + } op_param; }; struct mlx5_ifc_general_obj_out_cmd_hdr_bits { @@ -11707,6 +11766,7 @@ enum { MLX5_GENERAL_OBJECT_TYPES_SAMPLER = 0x20, MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO = 0x24, MLX5_GENERAL_OBJECT_TYPES_MACSEC = 0x27, + MLX5_GENERAL_OBJECT_TYPES_INT_KEK = 0x47, }; enum { @@ -11886,10 +11946,44 @@ struct mlx5_ifc_query_macsec_obj_out_bits { struct mlx5_ifc_macsec_offload_obj_bits macsec_object; }; +struct mlx5_ifc_wrapped_dek_bits { + u8 gcm_iv[0x60]; + + u8 reserved_at_60[0x20]; + + u8 const0[0x1]; + u8 key_size[0x1]; + u8 reserved_at_82[0x2]; + u8 key2_invalid[0x1]; + u8 reserved_at_85[0x3]; + u8 pd[0x18]; + + u8 key_purpose[0x5]; + u8 reserved_at_a5[0x13]; + u8 kek_id[0x8]; + + u8 reserved_at_c0[0x40]; + + u8 key1[0x8][0x20]; + + u8 key2[0x8][0x20]; + + u8 reserved_at_300[0x40]; + + u8 const1[0x1]; + u8 reserved_at_341[0x1f]; + + u8 reserved_at_360[0x20]; + + u8 auth_tag[0x80]; +}; + struct mlx5_ifc_encryption_key_obj_bits { u8 modify_field_select[0x40]; - u8 reserved_at_40[0x14]; + u8 state[0x8]; + u8 sw_wrapped[0x1]; + u8 reserved_at_49[0xb]; u8 key_size[0x4]; u8 reserved_at_58[0x4]; u8 key_type[0x4]; @@ -11897,10 +11991,17 @@ struct mlx5_ifc_encryption_key_obj_bits { u8 reserved_at_60[0x8]; u8 pd[0x18]; - u8 reserved_at_80[0x180]; - u8 key[8][0x20]; + u8 reserved_at_80[0x100]; + + u8 opaque[0x40]; + + u8 reserved_at_1c0[0x40]; - u8 reserved_at_300[0x500]; + u8 key[8][0x80]; + + u8 sw_wrapped_dek[8][0x80]; + + u8 reserved_at_a00[0x600]; }; struct mlx5_ifc_create_encryption_key_in_bits { @@ -11908,6 +12009,11 @@ struct mlx5_ifc_create_encryption_key_in_bits { struct mlx5_ifc_encryption_key_obj_bits encryption_key_object; }; +struct mlx5_ifc_modify_encryption_key_in_bits { + struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr; + struct mlx5_ifc_encryption_key_obj_bits encryption_key_object; +}; + enum { MLX5_FLOW_METER_MODE_BYTES_IP_LENGTH = 0x0, MLX5_FLOW_METER_MODE_BYTES_CALC_WITH_L2 = 0x1, @@ -11963,6 +12069,34 @@ struct mlx5_ifc_create_flow_meter_aso_obj_in_bits { struct mlx5_ifc_flow_meter_aso_obj_bits flow_meter_aso_obj; }; +struct mlx5_ifc_int_kek_obj_bits { + u8 modify_field_select[0x40]; + + u8 state[0x8]; + u8 auto_gen[0x1]; + u8 reserved_at_49[0xb]; + u8 key_size[0x4]; + u8 reserved_at_58[0x8]; + + u8 reserved_at_60[0x8]; + u8 pd[0x18]; + + u8 reserved_at_80[0x180]; + u8 key[8][0x80]; + + u8 reserved_at_600[0x200]; +}; + +struct mlx5_ifc_create_int_kek_obj_in_bits { + struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr; + struct mlx5_ifc_int_kek_obj_bits int_kek_object; +}; + +struct mlx5_ifc_create_int_kek_obj_out_bits { + struct mlx5_ifc_general_obj_out_cmd_hdr_bits general_obj_out_cmd_hdr; + struct mlx5_ifc_int_kek_obj_bits int_kek_object; +}; + struct mlx5_ifc_sampler_obj_bits { u8 modify_field_select[0x40]; From 60c8972d2cccba19ce5a993bb8221e609702ef0d Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Tue, 22 Nov 2022 06:09:22 +0000 Subject: [PATCH 04/15] net/mlx5: Change key type to key purpose Change the naming of key type in DEK fields and macros, to be consistent with the device spec. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h | 6 +++--- include/linux/mlx5/mlx5_ifc.h | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index 2521f31d36b78..7614595c54169 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -41,7 +41,7 @@ int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, memcpy(key_p, key, sz_bytes); MLX5_SET(encryption_key_obj, obj, key_size, general_obj_key_size); - MLX5_SET(encryption_key_obj, obj, key_type, key_type); + MLX5_SET(encryption_key_obj, obj, key_purpose, key_type); MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h index cb85da9e39645..0a5a7dc9fa05e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h @@ -5,9 +5,9 @@ #define __MLX5_LIB_CRYPTO_H__ enum { - MLX5_ACCEL_OBJ_TLS_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS, - MLX5_ACCEL_OBJ_IPSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC, - MLX5_ACCEL_OBJ_MACSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_MACSEC, + MLX5_ACCEL_OBJ_TLS_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_TLS, + MLX5_ACCEL_OBJ_IPSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_IPSEC, + MLX5_ACCEL_OBJ_MACSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_MACSEC, }; int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 7143b65f9f4aa..1b6201bb04c17 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -11986,7 +11986,7 @@ struct mlx5_ifc_encryption_key_obj_bits { u8 reserved_at_49[0xb]; u8 key_size[0x4]; u8 reserved_at_58[0x4]; - u8 key_type[0x4]; + u8 key_purpose[0x4]; u8 reserved_at_60[0x8]; u8 pd[0x18]; @@ -12135,9 +12135,9 @@ enum { }; enum { - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS = 0x1, - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC = 0x2, - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_MACSEC = 0x4, + MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_TLS = 0x1, + MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_IPSEC = 0x2, + MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_MACSEC = 0x4, }; struct mlx5_ifc_tls_static_params_bits { From fe298bdf6f654d4ad93069db9fe93b3b6632f4f1 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Mon, 23 May 2022 04:10:02 +0000 Subject: [PATCH 05/15] net/mlx5: Prepare for fast crypto key update if hardware supports it Add CAP for crypto offload, do the simple initialization if hardware supports it. Currently set log_dek_obj_range to 12, so 4k DEKs will be created in one bulk allocation. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/en_common.c | 10 ++++++ drivers/net/ethernet/mellanox/mlx5/core/fw.c | 6 ++++ .../ethernet/mellanox/mlx5/core/lib/crypto.c | 36 +++++++++++++++++++ .../ethernet/mellanox/mlx5/core/lib/crypto.h | 2 ++ .../net/ethernet/mellanox/mlx5/core/main.c | 1 + include/linux/mlx5/device.h | 4 +++ include/linux/mlx5/driver.h | 2 ++ 7 files changed, 61 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c index 68f19324db93c..4c9a3210600c2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c @@ -31,6 +31,7 @@ */ #include "en.h" +#include "lib/crypto.h" /* mlx5e global resources should be placed in this file. * Global resources are common to all the netdevices created on the same nic. @@ -104,6 +105,13 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev) INIT_LIST_HEAD(&res->td.tirs_list); mutex_init(&res->td.list_lock); + mdev->mlx5e_res.dek_priv = mlx5_crypto_dek_init(mdev); + if (IS_ERR(mdev->mlx5e_res.dek_priv)) { + mlx5_core_err(mdev, "crypto dek init failed, %ld\n", + PTR_ERR(mdev->mlx5e_res.dek_priv)); + mdev->mlx5e_res.dek_priv = NULL; + } + return 0; err_destroy_mkey: @@ -119,6 +127,8 @@ void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev) { struct mlx5e_hw_objs *res = &mdev->mlx5e_res.hw_objs; + mlx5_crypto_dek_cleanup(mdev->mlx5e_res.dek_priv); + mdev->mlx5e_res.dek_priv = NULL; mlx5_free_bfreg(mdev, &res->bfreg); mlx5_core_destroy_mkey(mdev, res->mkey); mlx5_core_dealloc_transport_domain(mdev, res->td.tdn); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index f34e758a2f1f6..7bb7be01225a8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -267,6 +267,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev) return err; } + if (MLX5_CAP_GEN(dev, crypto)) { + err = mlx5_core_get_caps(dev, MLX5_CAP_CRYPTO); + if (err) + return err; + } + if (MLX5_CAP_GEN(dev, shampo)) { err = mlx5_core_get_caps(dev, MLX5_CAP_DEV_SHAMPO); if (err) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index 7614595c54169..02bc365efade4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -4,6 +4,11 @@ #include "mlx5_core.h" #include "lib/crypto.h" +struct mlx5_crypto_dek_priv { + struct mlx5_core_dev *mdev; + int log_dek_obj_range; +}; + int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, void *key, u32 sz_bytes, u32 key_type, u32 *p_key_id) @@ -71,3 +76,34 @@ void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); } + +void mlx5_crypto_dek_cleanup(struct mlx5_crypto_dek_priv *dek_priv) +{ + if (!dek_priv) + return; + + kfree(dek_priv); +} + +struct mlx5_crypto_dek_priv *mlx5_crypto_dek_init(struct mlx5_core_dev *mdev) +{ + struct mlx5_crypto_dek_priv *dek_priv; + + if (!MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc)) + return NULL; + + dek_priv = kzalloc(sizeof(*dek_priv), GFP_KERNEL); + if (!dek_priv) + return ERR_PTR(-ENOMEM); + + dek_priv->mdev = mdev; + dek_priv->log_dek_obj_range = min_t(int, 12, + MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc)); + + mlx5_core_dbg(mdev, "Crypto DEK enabled, %d deks per alloc (max %d), total %d\n", + 1 << dek_priv->log_dek_obj_range, + 1 << MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc), + 1 << MLX5_CAP_CRYPTO(mdev, log_max_num_deks)); + + return dek_priv; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h index 0a5a7dc9fa05e..5968536047caf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h @@ -16,4 +16,6 @@ int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id); +struct mlx5_crypto_dek_priv *mlx5_crypto_dek_init(struct mlx5_core_dev *mdev); +void mlx5_crypto_dek_cleanup(struct mlx5_crypto_dek_priv *dek_priv); #endif /* __MLX5_LIB_CRYPTO_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 8823f20d2122b..9441588ac524a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1555,6 +1555,7 @@ static const int types[] = { MLX5_CAP_DEV_SHAMPO, MLX5_CAP_MACSEC, MLX5_CAP_ADV_VIRTUALIZATION, + MLX5_CAP_CRYPTO, }; static void mlx5_hca_caps_free(struct mlx5_core_dev *dev) diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 29d4b201c7b26..bc531bd9804ff 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -1204,6 +1204,7 @@ enum mlx5_cap_type { MLX5_CAP_VDPA_EMULATION = 0x13, MLX5_CAP_DEV_EVENT = 0x14, MLX5_CAP_IPSEC, + MLX5_CAP_CRYPTO = 0x1a, MLX5_CAP_DEV_SHAMPO = 0x1d, MLX5_CAP_MACSEC = 0x1f, MLX5_CAP_GENERAL_2 = 0x20, @@ -1460,6 +1461,9 @@ enum mlx5_qcam_feature_groups { #define MLX5_CAP_IPSEC(mdev, cap)\ MLX5_GET(ipsec_cap, (mdev)->caps.hca[MLX5_CAP_IPSEC]->cur, cap) +#define MLX5_CAP_CRYPTO(mdev, cap)\ + MLX5_GET(crypto_cap, (mdev)->caps.hca[MLX5_CAP_CRYPTO]->cur, cap) + #define MLX5_CAP_DEV_SHAMPO(mdev, cap)\ MLX5_GET(shampo_cap, mdev->caps.hca_cur[MLX5_CAP_DEV_SHAMPO], cap) diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 44167760ff29e..cd529e051b4dd 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -516,6 +516,7 @@ struct mlx5_vhca_state_notifier; struct mlx5_sf_dev_table; struct mlx5_sf_hw_table; struct mlx5_sf_table; +struct mlx5_crypto_dek_priv; struct mlx5_rate_limit { u32 rate; @@ -673,6 +674,7 @@ struct mlx5e_resources { } hw_objs; struct devlink_port dl_port; struct net_device *uplink_netdev; + struct mlx5_crypto_dek_priv *dek_priv; }; enum mlx5_sw_icm_type { From 55f0d6d200612ea01f20acabafe4c5c31c728250 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Mon, 8 Aug 2022 02:47:17 +0000 Subject: [PATCH 06/15] net/mlx5: Add const to the key pointer of encryption key creation Change key pointer to const void *, as there is no need to change the key content. This is also to avoid modifying the key by mistake. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c | 2 +- drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index d36788119b8bf..f80d6fce28d2d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -13,8 +13,8 @@ int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, struct tls_crypto_info *crypto_info, u32 *p_key_id) { + const void *key; u32 sz_bytes; - void *key; switch (crypto_info->cipher_type) { case TLS_CIPHER_AES_GCM_128: { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index 02bc365efade4..bc2a72491e101 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -10,7 +10,7 @@ struct mlx5_crypto_dek_priv { }; int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, - void *key, u32 sz_bytes, + const void *key, u32 sz_bytes, u32 key_type, u32 *p_key_id) { u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h index 5968536047caf..ee3ed8c863d19 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h @@ -11,7 +11,7 @@ enum { }; int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, - void *key, u32 sz_bytes, + const void *key, u32 sz_bytes, u32 key_type, u32 *p_key_id); void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id); From 942192541675b80222e98b4eb410695c5ab619a3 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Thu, 28 Jul 2022 03:59:29 +0000 Subject: [PATCH 07/15] net/mlx5: Refactor the encryption key creation Move the common code to general functions which can be used by fast update encryption key in later patches. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/lib/crypto.c | 77 +++++++++++++------ 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index bc2a72491e101..81fe5c3763a53 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -9,50 +9,79 @@ struct mlx5_crypto_dek_priv { int log_dek_obj_range; }; +static int mlx5_crypto_dek_get_key_sz(struct mlx5_core_dev *mdev, + u32 sz_bytes, u8 *key_sz_p) +{ + u32 sz_bits = sz_bytes * BITS_PER_BYTE; + + switch (sz_bits) { + case 128: + *key_sz_p = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128; + break; + case 256: + *key_sz_p = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256; + break; + default: + mlx5_core_err(mdev, "Crypto offload error, invalid key size (%u bits)\n", + sz_bits); + return -EINVAL; + } + + return 0; +} + +static int mlx5_crypto_dek_fill_key(struct mlx5_core_dev *mdev, u8 *key_obj, + const void *key, u32 sz_bytes) +{ + void *dst; + u8 key_sz; + int err; + + err = mlx5_crypto_dek_get_key_sz(mdev, sz_bytes, &key_sz); + if (err) + return err; + + MLX5_SET(encryption_key_obj, key_obj, key_size, key_sz); + + if (sz_bytes == 16) + /* For key size of 128b the MSBs are reserved. */ + dst = MLX5_ADDR_OF(encryption_key_obj, key_obj, key[1]); + else + dst = MLX5_ADDR_OF(encryption_key_obj, key_obj, key); + + memcpy(dst, key, sz_bytes); + + return 0; +} + int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, const void *key, u32 sz_bytes, u32 key_type, u32 *p_key_id) { u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; - u32 sz_bits = sz_bytes * BITS_PER_BYTE; - u8 general_obj_key_size; u64 general_obj_types; - void *obj, *key_p; + void *obj; int err; - obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); - key_p = MLX5_ADDR_OF(encryption_key_obj, obj, key); - general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types); if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY)) return -EINVAL; - switch (sz_bits) { - case 128: - general_obj_key_size = - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_128; - key_p += sz_bytes; - break; - case 256: - general_obj_key_size = - MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_KEY_SIZE_256; - break; - default: - return -EINVAL; - } - - memcpy(key_p, key, sz_bytes); - - MLX5_SET(encryption_key_obj, obj, key_size, general_obj_key_size); - MLX5_SET(encryption_key_obj, obj, key_purpose, key_type); MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); + + obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); + MLX5_SET(encryption_key_obj, obj, key_purpose, key_type); MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); + err = mlx5_crypto_dek_fill_key(mdev, obj, key, sz_bytes); + if (err) + return err; + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); if (!err) *p_key_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); From 204369e718e9f276bdc8677e8dbb648d3298735f Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Wed, 20 Jul 2022 02:36:05 +0000 Subject: [PATCH 08/15] net/mlx5: Add new APIs for fast update encryption key New APIs are added to support fast update DEKs. As a pool is created for each key purpose (type), one pair of pool APIs to get/put pool. Anotehr pair of DEKs APIs is to get DEK object from pool and update it with user key, or free it back to the pool. As The bulk allocation and destruction will be supported in later patches, old implementation is used here. To support these APIs, pool and dek structs are defined first. Only small number of fields are stored in them. For example, key_purpose and refcnt in pool struct, DEK object id in dek struct. More fields will be added to these structs in later patches, for example, the different bulk lists for pool struct, the bulk pointer dek struct belongs to, and a list_entry for the list in a pool, which is used to save keys waiting for being freed while other thread is doing sync. Besides the creation and destruction interfaces, new one is also added to get obj id. Currently these APIs are planned to used by TLS only. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/lib/crypto.c | 90 +++++++++++++++++-- .../ethernet/mellanox/mlx5/core/lib/crypto.h | 13 +++ 2 files changed, 98 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index 81fe5c3763a53..d1b4cc9907567 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -4,11 +4,28 @@ #include "mlx5_core.h" #include "lib/crypto.h" +#define MLX5_CRYPTO_DEK_POOLS_NUM (MLX5_ACCEL_OBJ_TYPE_KEY_NUM - 1) +#define type2idx(type) ((type) - 1) + +struct mlx5_crypto_dek_pool { + struct mlx5_core_dev *mdev; + u32 key_purpose; +}; + struct mlx5_crypto_dek_priv { struct mlx5_core_dev *mdev; int log_dek_obj_range; }; +struct mlx5_crypto_dek { + u32 obj_id; +}; + +u32 mlx5_crypto_dek_get_id(struct mlx5_crypto_dek *dek) +{ + return dek->obj_id; +} + static int mlx5_crypto_dek_get_key_sz(struct mlx5_core_dev *mdev, u32 sz_bytes, u8 *key_sz_p) { @@ -54,9 +71,9 @@ static int mlx5_crypto_dek_fill_key(struct mlx5_core_dev *mdev, u8 *key_obj, return 0; } -int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, - const void *key, u32 sz_bytes, - u32 key_type, u32 *p_key_id) +static int mlx5_crypto_create_dek_key(struct mlx5_core_dev *mdev, + const void *key, u32 sz_bytes, + u32 key_purpose, u32 *p_key_id) { u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; @@ -75,7 +92,7 @@ int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); - MLX5_SET(encryption_key_obj, obj, key_purpose, key_type); + MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); err = mlx5_crypto_dek_fill_key(mdev, obj, key, sz_bytes); @@ -92,7 +109,7 @@ int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, return err; } -void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) +static void mlx5_crypto_destroy_dek_key(struct mlx5_core_dev *mdev, u32 key_id) { u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; @@ -106,6 +123,69 @@ void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); } +int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, + const void *key, u32 sz_bytes, + u32 key_type, u32 *p_key_id) +{ + return mlx5_crypto_create_dek_key(mdev, key, sz_bytes, key_type, p_key_id); +} + +void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) +{ + mlx5_crypto_destroy_dek_key(mdev, key_id); +} + +struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_pool, + const void *key, u32 sz_bytes) +{ + struct mlx5_core_dev *mdev = dek_pool->mdev; + u32 key_purpose = dek_pool->key_purpose; + struct mlx5_crypto_dek *dek; + int err; + + dek = kzalloc(sizeof(*dek), GFP_KERNEL); + if (!dek) + return ERR_PTR(-ENOMEM); + + err = mlx5_crypto_create_dek_key(mdev, key, sz_bytes, + key_purpose, &dek->obj_id); + if (err) { + kfree(dek); + return ERR_PTR(err); + } + + return dek; +} + +void mlx5_crypto_dek_destroy(struct mlx5_crypto_dek_pool *dek_pool, + struct mlx5_crypto_dek *dek) +{ + struct mlx5_core_dev *mdev = dek_pool->mdev; + + mlx5_crypto_destroy_dek_key(mdev, dek->obj_id); + kfree(dek); +} + +struct mlx5_crypto_dek_pool * +mlx5_crypto_dek_pool_create(struct mlx5_core_dev *mdev, int key_purpose) +{ + struct mlx5_crypto_dek_pool *pool; + + pool = kzalloc(sizeof(*pool), GFP_KERNEL); + if (!pool) + return ERR_PTR(-ENOMEM); + + pool->mdev = mdev; + pool->key_purpose = key_purpose; + + return pool; +} + +void mlx5_crypto_dek_pool_destroy(struct mlx5_crypto_dek_pool *pool) +{ + kfree(pool); +} + void mlx5_crypto_dek_cleanup(struct mlx5_crypto_dek_priv *dek_priv) { if (!dek_priv) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h index ee3ed8c863d19..c819c047bb9cb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.h @@ -8,6 +8,7 @@ enum { MLX5_ACCEL_OBJ_TLS_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_TLS, MLX5_ACCEL_OBJ_IPSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_IPSEC, MLX5_ACCEL_OBJ_MACSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_PURPOSE_MACSEC, + MLX5_ACCEL_OBJ_TYPE_KEY_NUM, }; int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, @@ -16,6 +17,18 @@ int mlx5_create_encryption_key(struct mlx5_core_dev *mdev, void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id); +struct mlx5_crypto_dek_pool; +struct mlx5_crypto_dek; + +struct mlx5_crypto_dek_pool *mlx5_crypto_dek_pool_create(struct mlx5_core_dev *mdev, + int key_purpose); +void mlx5_crypto_dek_pool_destroy(struct mlx5_crypto_dek_pool *pool); +struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_pool, + const void *key, u32 sz_bytes); +void mlx5_crypto_dek_destroy(struct mlx5_crypto_dek_pool *dek_pool, + struct mlx5_crypto_dek *dek); +u32 mlx5_crypto_dek_get_id(struct mlx5_crypto_dek *dek); + struct mlx5_crypto_dek_priv *mlx5_crypto_dek_init(struct mlx5_core_dev *mdev); void mlx5_crypto_dek_cleanup(struct mlx5_crypto_dek_priv *dek_priv); #endif /* __MLX5_LIB_CRYPTO_H__ */ From 7a5b72c2a8e4f2292958b96423905948da583628 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Wed, 20 Jul 2022 02:43:40 +0000 Subject: [PATCH 09/15] net/mlx5: Add support SYNC_CRYPTO command Add support for SYNC_CRYPTO command. For now, it is executed only when initializing DEK, but needed when reusing keys in later patch. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/cmd.c | 3 ++ .../ethernet/mellanox/mlx5/core/lib/crypto.c | 36 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index 382d02f6619c7..b00e33ed05e91 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -104,6 +104,7 @@ static bool mlx5_cmd_is_throttle_opcode(u16 op) case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT: case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT: case MLX5_CMD_OP_QUERY_GENERAL_OBJECT: + case MLX5_CMD_OP_SYNC_CRYPTO: return true; } return false; @@ -523,6 +524,7 @@ static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op, case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE: case MLX5_CMD_OP_SAVE_VHCA_STATE: case MLX5_CMD_OP_LOAD_VHCA_STATE: + case MLX5_CMD_OP_SYNC_CRYPTO: *status = MLX5_DRIVER_STATUS_ABORTED; *synd = MLX5_DRIVER_SYND; return -ENOLINK; @@ -725,6 +727,7 @@ const char *mlx5_command_str(int command) MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE); MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE); MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE); + MLX5_COMMAND_STR_CASE(SYNC_CRYPTO); default: return "unknown command opcode"; } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index d1b4cc9907567..ce29251484c04 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -7,6 +7,10 @@ #define MLX5_CRYPTO_DEK_POOLS_NUM (MLX5_ACCEL_OBJ_TYPE_KEY_NUM - 1) #define type2idx(type) ((type) - 1) +enum { + MLX5_CRYPTO_DEK_ALL_TYPE = BIT(0), +}; + struct mlx5_crypto_dek_pool { struct mlx5_core_dev *mdev; u32 key_purpose; @@ -71,6 +75,28 @@ static int mlx5_crypto_dek_fill_key(struct mlx5_core_dev *mdev, u8 *key_obj, return 0; } +static int mlx5_crypto_cmd_sync_crypto(struct mlx5_core_dev *mdev, + int crypto_type) +{ + u32 in[MLX5_ST_SZ_DW(sync_crypto_in)] = {}; + int err; + + mlx5_core_dbg(mdev, + "Execute SYNC_CRYPTO command with crypto_type(0x%x)\n", + crypto_type); + + MLX5_SET(sync_crypto_in, in, opcode, MLX5_CMD_OP_SYNC_CRYPTO); + MLX5_SET(sync_crypto_in, in, crypto_type, crypto_type); + + err = mlx5_cmd_exec_in(mdev, sync_crypto, in); + if (err) + mlx5_core_err(mdev, + "Failed to exec sync crypto, type=%d, err=%d\n", + crypto_type, err); + + return err; +} + static int mlx5_crypto_create_dek_key(struct mlx5_core_dev *mdev, const void *key, u32 sz_bytes, u32 key_purpose, u32 *p_key_id) @@ -197,6 +223,7 @@ void mlx5_crypto_dek_cleanup(struct mlx5_crypto_dek_priv *dek_priv) struct mlx5_crypto_dek_priv *mlx5_crypto_dek_init(struct mlx5_core_dev *mdev) { struct mlx5_crypto_dek_priv *dek_priv; + int err; if (!MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc)) return NULL; @@ -209,10 +236,19 @@ struct mlx5_crypto_dek_priv *mlx5_crypto_dek_init(struct mlx5_core_dev *mdev) dek_priv->log_dek_obj_range = min_t(int, 12, MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc)); + /* sync all types of objects */ + err = mlx5_crypto_cmd_sync_crypto(mdev, MLX5_CRYPTO_DEK_ALL_TYPE); + if (err) + goto err_sync_crypto; + mlx5_core_dbg(mdev, "Crypto DEK enabled, %d deks per alloc (max %d), total %d\n", 1 << dek_priv->log_dek_obj_range, 1 << MLX5_CAP_CRYPTO(mdev, log_dek_max_alloc), 1 << MLX5_CAP_CRYPTO(mdev, log_max_num_deks)); return dek_priv; + +err_sync_crypto: + kfree(dek_priv); + return ERR_PTR(err); } From 4d570c7117ddfa96da96bc8917effcfa906e6bd1 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Wed, 20 Jul 2022 03:57:37 +0000 Subject: [PATCH 10/15] net/mlx5: Add bulk allocation and modify_dek operation To support fast update of keys into hardware, we optimize firmware to achieve the maximum rate. The approach is to create DEK objects in bulk, and update each of them with modify command. This patch supports bulk allocation and modify_dek commands for new firmware. However, as log_obj_range is 0 for now, only one DEK obj is allocated each time, and then updated with user key by modify_dek. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/lib/crypto.c | 85 ++++++++++++++++++- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index ce29251484c04..a7b863859d50a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -97,6 +97,72 @@ static int mlx5_crypto_cmd_sync_crypto(struct mlx5_core_dev *mdev, return err; } +static int mlx5_crypto_create_dek_bulk(struct mlx5_core_dev *mdev, + u32 key_purpose, int log_obj_range, + u32 *obj_id) +{ + u32 in[MLX5_ST_SZ_DW(create_encryption_key_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + void *obj, *param; + int err; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, + MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); + param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); + MLX5_SET(general_obj_create_param, param, log_obj_range, log_obj_range); + + obj = MLX5_ADDR_OF(create_encryption_key_in, in, encryption_key_object); + MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); + MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); + mlx5_core_dbg(mdev, "DEK objects created, bulk=%d, obj_id=%d\n", + 1 << log_obj_range, *obj_id); + + return 0; +} + +static int mlx5_crypto_modify_dek_key(struct mlx5_core_dev *mdev, + const void *key, u32 sz_bytes, u32 key_purpose, + u32 obj_id, u32 obj_offset) +{ + u32 in[MLX5_ST_SZ_DW(modify_encryption_key_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + void *obj, *param; + int err; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, + MLX5_CMD_OP_MODIFY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_ENCRYPTION_KEY); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); + + param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param); + MLX5_SET(general_obj_query_param, param, obj_offset, obj_offset); + + obj = MLX5_ADDR_OF(modify_encryption_key_in, in, encryption_key_object); + MLX5_SET64(encryption_key_obj, obj, modify_field_select, 1); + MLX5_SET(encryption_key_obj, obj, key_purpose, key_purpose); + MLX5_SET(encryption_key_obj, obj, pd, mdev->mlx5e_res.hw_objs.pdn); + + err = mlx5_crypto_dek_fill_key(mdev, obj, key, sz_bytes); + if (err) + return err; + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + + /* avoid leaking key on the stack */ + memzero_explicit(in, sizeof(in)); + + return err; +} + static int mlx5_crypto_create_dek_key(struct mlx5_core_dev *mdev, const void *key, u32 sz_bytes, u32 key_purpose, u32 *p_key_id) @@ -164,6 +230,7 @@ void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_pool, const void *key, u32 sz_bytes) { + struct mlx5_crypto_dek_priv *dek_priv = dek_pool->mdev->mlx5e_res.dek_priv; struct mlx5_core_dev *mdev = dek_pool->mdev; u32 key_purpose = dek_pool->key_purpose; struct mlx5_crypto_dek *dek; @@ -173,8 +240,22 @@ struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_ if (!dek) return ERR_PTR(-ENOMEM); - err = mlx5_crypto_create_dek_key(mdev, key, sz_bytes, - key_purpose, &dek->obj_id); + if (!dek_priv) { + err = mlx5_crypto_create_dek_key(mdev, key, sz_bytes, + key_purpose, &dek->obj_id); + goto out; + } + + err = mlx5_crypto_create_dek_bulk(mdev, key_purpose, 0, &dek->obj_id); + if (err) + goto out; + + err = mlx5_crypto_modify_dek_key(mdev, key, sz_bytes, key_purpose, + dek->obj_id, 0); + if (err) + mlx5_crypto_destroy_dek_key(mdev, dek->obj_id); + +out: if (err) { kfree(dek); return ERR_PTR(err); From c6e7d8171045e1425c2a813b8e04d86a94d4e7bc Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Wed, 20 Jul 2022 07:20:01 +0000 Subject: [PATCH 11/15] net/mlx5: Use bulk allocation for fast update encryption key We create a pool for each key type. For the pool, there is a struct to store the info for all DEK objects of one bulk allocation. As we use crypto->log_dek_obj_range, which is set to 12 in previous patch, for the log_obj_range of bulk allocation, 4096 DEKs are allocated in one time. To trace the state of all the keys in a bulk, two bitmaps are created. The need_sync bitmap is used to indicate the available state of the corresponding key. If the bit is 0, it can be used (available) as it either is newly created by FW, or SYNC_CRYPTO is executed and bit is reset after it is freed by upper layer user (this is the case to be handled in later patch). Otherwise, the key need to be synced. The in_use bitmap is used to indicate the key is being used, and reset when user free it. When ktls, ipsec or macsec need a key from a bulk, it get one with need_sync bit 0, then set both need_sync and in_used bit to 1. When user free a key, only in_use bit is reset to 0. So, for the combinations of (need_sync, in_use) of one DEK object, - (0,0) means the key is ready for use, - (1,1) means the key is currently being used by a user, - (1,0) means the key is freed, and waiting for being synced, - (0,1) is invalid state. There are two lists in each pool, partial_list and full_list, according to the number for available DEKs in a bulk. When user need a key, it get a bulk, either from partial list, or create new one from FW. Then the bulk is put in the different pool's lists according to the num of avail deks it has. If there is no avail deks, and all of them are be freed by users, for now, the bulk is destroyed. To speed up the bitmap search, a variable (avail_start) is added to indicate where to start to search need_sync bitmap for available key. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/lib/crypto.c | 215 +++++++++++++++++- 1 file changed, 208 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index a7b863859d50a..5c92ac5d95ee3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -14,6 +14,28 @@ enum { struct mlx5_crypto_dek_pool { struct mlx5_core_dev *mdev; u32 key_purpose; + int num_deks; /* the total number of keys in this pool */ + int avail_deks; /* the number of available keys in this pool */ + int in_use_deks; /* the number of being used keys in this pool */ + struct mutex lock; /* protect the following lists, and the bulks */ + struct list_head partial_list; /* some of keys are available */ + struct list_head full_list; /* no available keys */ +}; + +struct mlx5_crypto_dek_bulk { + struct mlx5_core_dev *mdev; + int base_obj_id; + int avail_start; /* the bit to start search */ + int num_deks; /* the total number of keys in a bulk */ + int avail_deks; /* the number of keys available, with need_sync bit 0 */ + int in_use_deks; /* the number of keys being used, with in_use bit 1 */ + struct list_head entry; + + /* 0: not being used by any user, 1: otherwise */ + unsigned long *in_use; + + /* The bits are set when they are used, and initialized to 0 */ + unsigned long *need_sync; }; struct mlx5_crypto_dek_priv { @@ -22,6 +44,7 @@ struct mlx5_crypto_dek_priv { }; struct mlx5_crypto_dek { + struct mlx5_crypto_dek_bulk *bulk; u32 obj_id; }; @@ -227,13 +250,166 @@ void mlx5_destroy_encryption_key(struct mlx5_core_dev *mdev, u32 key_id) mlx5_crypto_destroy_dek_key(mdev, key_id); } +static struct mlx5_crypto_dek_bulk * +mlx5_crypto_dek_bulk_create(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek_priv *dek_priv = pool->mdev->mlx5e_res.dek_priv; + struct mlx5_core_dev *mdev = pool->mdev; + struct mlx5_crypto_dek_bulk *bulk; + int num_deks, base_obj_id; + int err; + + bulk = kzalloc(sizeof(*bulk), GFP_KERNEL); + if (!bulk) + return ERR_PTR(-ENOMEM); + + num_deks = 1 << dek_priv->log_dek_obj_range; + bulk->need_sync = bitmap_zalloc(num_deks, GFP_KERNEL); + if (!bulk->need_sync) { + err = -ENOMEM; + goto err_out; + } + + bulk->in_use = bitmap_zalloc(num_deks, GFP_KERNEL); + if (!bulk->in_use) { + err = -ENOMEM; + goto err_out; + } + + err = mlx5_crypto_create_dek_bulk(mdev, pool->key_purpose, + dek_priv->log_dek_obj_range, + &base_obj_id); + if (err) + goto err_out; + + bulk->base_obj_id = base_obj_id; + bulk->num_deks = num_deks; + bulk->avail_deks = num_deks; + bulk->mdev = mdev; + + return bulk; + +err_out: + bitmap_free(bulk->in_use); + bitmap_free(bulk->need_sync); + kfree(bulk); + return ERR_PTR(err); +} + +static struct mlx5_crypto_dek_bulk * +mlx5_crypto_dek_pool_add_bulk(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek_bulk *bulk; + + bulk = mlx5_crypto_dek_bulk_create(pool); + if (IS_ERR(bulk)) + return bulk; + + pool->avail_deks += bulk->num_deks; + pool->num_deks += bulk->num_deks; + list_add(&bulk->entry, &pool->partial_list); + + return bulk; +} + +static void mlx5_crypto_dek_bulk_free(struct mlx5_crypto_dek_bulk *bulk) +{ + mlx5_crypto_destroy_dek_key(bulk->mdev, bulk->base_obj_id); + bitmap_free(bulk->need_sync); + bitmap_free(bulk->in_use); + kfree(bulk); +} + +static void mlx5_crypto_dek_pool_remove_bulk(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek_bulk *bulk) +{ + pool->num_deks -= bulk->num_deks; + pool->avail_deks -= bulk->avail_deks; + pool->in_use_deks -= bulk->in_use_deks; + list_del(&bulk->entry); + mlx5_crypto_dek_bulk_free(bulk); +} + +static struct mlx5_crypto_dek_bulk * +mlx5_crypto_dek_pool_pop(struct mlx5_crypto_dek_pool *pool, u32 *obj_offset) +{ + struct mlx5_crypto_dek_bulk *bulk; + int pos; + + mutex_lock(&pool->lock); + bulk = list_first_entry_or_null(&pool->partial_list, + struct mlx5_crypto_dek_bulk, entry); + + if (bulk) { + pos = find_next_zero_bit(bulk->need_sync, bulk->num_deks, + bulk->avail_start); + if (pos == bulk->num_deks) { + mlx5_core_err(pool->mdev, "Wrong DEK bulk avail_start.\n"); + pos = find_first_zero_bit(bulk->need_sync, bulk->num_deks); + } + WARN_ON(pos == bulk->num_deks); + } else { + bulk = mlx5_crypto_dek_pool_add_bulk(pool); + if (IS_ERR(bulk)) + goto out; + pos = 0; + } + + *obj_offset = pos; + bitmap_set(bulk->need_sync, pos, 1); + bitmap_set(bulk->in_use, pos, 1); + bulk->in_use_deks++; + bulk->avail_deks--; + if (!bulk->avail_deks) { + list_move(&bulk->entry, &pool->full_list); + bulk->avail_start = bulk->num_deks; + } else { + bulk->avail_start = pos + 1; + } + pool->avail_deks--; + pool->in_use_deks++; + +out: + mutex_unlock(&pool->lock); + return bulk; +} + +static int mlx5_crypto_dek_pool_push(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek *dek) +{ + struct mlx5_crypto_dek_bulk *bulk = dek->bulk; + int obj_offset; + bool old_val; + int err = 0; + + mutex_lock(&pool->lock); + obj_offset = dek->obj_id - bulk->base_obj_id; + old_val = test_and_clear_bit(obj_offset, bulk->in_use); + WARN_ON_ONCE(!old_val); + if (!old_val) { + err = -ENOENT; + goto out_free; + } + pool->in_use_deks--; + bulk->in_use_deks--; + if (!bulk->avail_deks && !bulk->in_use_deks) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk); + +out_free: + mutex_unlock(&pool->lock); + kfree(dek); + return err; +} + struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_pool, const void *key, u32 sz_bytes) { struct mlx5_crypto_dek_priv *dek_priv = dek_pool->mdev->mlx5e_res.dek_priv; struct mlx5_core_dev *mdev = dek_pool->mdev; u32 key_purpose = dek_pool->key_purpose; + struct mlx5_crypto_dek_bulk *bulk; struct mlx5_crypto_dek *dek; + int obj_offset; int err; dek = kzalloc(sizeof(*dek), GFP_KERNEL); @@ -246,14 +422,20 @@ struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_ goto out; } - err = mlx5_crypto_create_dek_bulk(mdev, key_purpose, 0, &dek->obj_id); - if (err) + bulk = mlx5_crypto_dek_pool_pop(dek_pool, &obj_offset); + if (IS_ERR(bulk)) { + err = PTR_ERR(bulk); goto out; + } + dek->bulk = bulk; + dek->obj_id = bulk->base_obj_id + obj_offset; err = mlx5_crypto_modify_dek_key(mdev, key, sz_bytes, key_purpose, - dek->obj_id, 0); - if (err) - mlx5_crypto_destroy_dek_key(mdev, dek->obj_id); + bulk->base_obj_id, obj_offset); + if (err) { + mlx5_crypto_dek_pool_push(dek_pool, dek); + return ERR_PTR(err); + } out: if (err) { @@ -267,10 +449,15 @@ struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_ void mlx5_crypto_dek_destroy(struct mlx5_crypto_dek_pool *dek_pool, struct mlx5_crypto_dek *dek) { + struct mlx5_crypto_dek_priv *dek_priv = dek_pool->mdev->mlx5e_res.dek_priv; struct mlx5_core_dev *mdev = dek_pool->mdev; - mlx5_crypto_destroy_dek_key(mdev, dek->obj_id); - kfree(dek); + if (!dek_priv) { + mlx5_crypto_destroy_dek_key(mdev, dek->obj_id); + kfree(dek); + } else { + mlx5_crypto_dek_pool_push(dek_pool, dek); + } } struct mlx5_crypto_dek_pool * @@ -285,11 +472,25 @@ mlx5_crypto_dek_pool_create(struct mlx5_core_dev *mdev, int key_purpose) pool->mdev = mdev; pool->key_purpose = key_purpose; + mutex_init(&pool->lock); + INIT_LIST_HEAD(&pool->partial_list); + INIT_LIST_HEAD(&pool->full_list); + return pool; } void mlx5_crypto_dek_pool_destroy(struct mlx5_crypto_dek_pool *pool) { + struct mlx5_crypto_dek_bulk *bulk, *tmp; + + list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk); + + list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk); + + mutex_destroy(&pool->lock); + kfree(pool); } From 709f07fe1a595dae3c891e18f8dca297402f95ea Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Wed, 16 Mar 2022 02:06:22 +0000 Subject: [PATCH 12/15] net/mlx5: Reuse DEKs after executing SYNC_CRYPTO command To fast update encryption keys, those freed keys with need_sync bit 1 and in_use bit 0 in a bulk, can be recycled. The keys are cached internally by the NIC, so invalidating internal NIC caches by SYNC_CRYPTO command is required before reusing them. A threshold in driver is added to avoid invalidating for every update. Only when the number of DEKs, which need to be synced, is over this threshold, the sync process will start. Besides, it is done in system workqueue. After SYNC_CRYPTO command is executed successfully, the bitmaps of each bulk must be reset accordingly, so that the freed DEKs can be reused. From the analysis in previous patch, the number of reused DEKs can be calculated by hweight_long(need_sync XOR in_use), and the need_sync bits can be reset by simply copying from in_use bits. Two more list (avail_list and sync_list) are added for each pool. The avail_list is for a bulk when all bits in need_sync are reset after sync. If there is no avail deks, and all are be freed by users, the bulk is moved to sync_list, instead of being destroyed in previous patch, and waiting for the invalidation. While syncing, they are simply reset need_sync bits, and moved to avail_list. Besides, add a wait_for_free list for the to-be-free DEKs. It is to avoid this corner case: when thread A is done with SYNC_CRYPTO but just before starting to reset the bitmaps, thread B is alloc dek, and free it immediately. It's obvious that this DEK can't be reused this time, so put it to waiting list, and do free after bulk bitmaps reset is finished. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/lib/crypto.c | 192 +++++++++++++++++- 1 file changed, 183 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index 5c92ac5d95ee3..8f8c18f806014 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -7,6 +7,22 @@ #define MLX5_CRYPTO_DEK_POOLS_NUM (MLX5_ACCEL_OBJ_TYPE_KEY_NUM - 1) #define type2idx(type) ((type) - 1) +#define MLX5_CRYPTO_DEK_POOL_SYNC_THRESH 128 + +/* calculate the num of DEKs, which are freed by any user + * (for example, TLS) after last revalidation in a pool or a bulk. + */ +#define MLX5_CRYPTO_DEK_CALC_FREED(a) \ + ({ typeof(a) _a = (a); \ + _a->num_deks - _a->avail_deks - _a->in_use_deks; }) + +#define MLX5_CRYPTO_DEK_POOL_CALC_FREED(pool) MLX5_CRYPTO_DEK_CALC_FREED(pool) +#define MLX5_CRYPTO_DEK_BULK_CALC_FREED(bulk) MLX5_CRYPTO_DEK_CALC_FREED(bulk) + +#define MLX5_CRYPTO_DEK_BULK_IDLE(bulk) \ + ({ typeof(bulk) _bulk = (bulk); \ + _bulk->avail_deks == _bulk->num_deks; }) + enum { MLX5_CRYPTO_DEK_ALL_TYPE = BIT(0), }; @@ -20,6 +36,16 @@ struct mlx5_crypto_dek_pool { struct mutex lock; /* protect the following lists, and the bulks */ struct list_head partial_list; /* some of keys are available */ struct list_head full_list; /* no available keys */ + struct list_head avail_list; /* all keys are available to use */ + + /* No in-used keys, and all need to be synced. + * These bulks will be put to avail list after sync. + */ + struct list_head sync_list; + + bool syncing; + struct list_head wait_for_free; + struct work_struct sync_work; }; struct mlx5_crypto_dek_bulk { @@ -34,7 +60,10 @@ struct mlx5_crypto_dek_bulk { /* 0: not being used by any user, 1: otherwise */ unsigned long *in_use; - /* The bits are set when they are used, and initialized to 0 */ + /* The bits are set when they are used, and reset after crypto_sync + * is executed. So, the value 0 means the key is newly created, or not + * used after sync, and 1 means it is in use, or freed but not synced + */ unsigned long *need_sync; }; @@ -45,6 +74,7 @@ struct mlx5_crypto_dek_priv { struct mlx5_crypto_dek { struct mlx5_crypto_dek_bulk *bulk; + struct list_head entry; u32 obj_id; }; @@ -349,9 +379,16 @@ mlx5_crypto_dek_pool_pop(struct mlx5_crypto_dek_pool *pool, u32 *obj_offset) } WARN_ON(pos == bulk->num_deks); } else { - bulk = mlx5_crypto_dek_pool_add_bulk(pool); - if (IS_ERR(bulk)) - goto out; + bulk = list_first_entry_or_null(&pool->avail_list, + struct mlx5_crypto_dek_bulk, + entry); + if (bulk) { + list_move(&bulk->entry, &pool->partial_list); + } else { + bulk = mlx5_crypto_dek_pool_add_bulk(pool); + if (IS_ERR(bulk)) + goto out; + } pos = 0; } @@ -374,15 +411,20 @@ mlx5_crypto_dek_pool_pop(struct mlx5_crypto_dek_pool *pool, u32 *obj_offset) return bulk; } -static int mlx5_crypto_dek_pool_push(struct mlx5_crypto_dek_pool *pool, - struct mlx5_crypto_dek *dek) +static bool mlx5_crypto_dek_need_sync(struct mlx5_crypto_dek_pool *pool) +{ + return !pool->syncing && + MLX5_CRYPTO_DEK_POOL_CALC_FREED(pool) > MLX5_CRYPTO_DEK_POOL_SYNC_THRESH; +} + +static int mlx5_crypto_dek_free_locked(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek *dek) { struct mlx5_crypto_dek_bulk *bulk = dek->bulk; int obj_offset; bool old_val; int err = 0; - mutex_lock(&pool->lock); obj_offset = dek->obj_id - bulk->base_obj_id; old_val = test_and_clear_bit(obj_offset, bulk->in_use); WARN_ON_ONCE(!old_val); @@ -393,14 +435,132 @@ static int mlx5_crypto_dek_pool_push(struct mlx5_crypto_dek_pool *pool, pool->in_use_deks--; bulk->in_use_deks--; if (!bulk->avail_deks && !bulk->in_use_deks) - mlx5_crypto_dek_pool_remove_bulk(pool, bulk); + list_move(&bulk->entry, &pool->sync_list); + + if (mlx5_crypto_dek_need_sync(pool) && schedule_work(&pool->sync_work)) + pool->syncing = true; out_free: - mutex_unlock(&pool->lock); kfree(dek); return err; } +static int mlx5_crypto_dek_pool_push(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek *dek) +{ + int err = 0; + + mutex_lock(&pool->lock); + if (pool->syncing) + list_add(&dek->entry, &pool->wait_for_free); + else + err = mlx5_crypto_dek_free_locked(pool, dek); + mutex_unlock(&pool->lock); + + return err; +} + +/* Update the bits for a bulk while sync, and avail_next for search. + * As the combinations of (need_sync, in_use) of one DEK are + * - (0,0) means the key is ready for use, + * - (1,1) means the key is currently being used by a user, + * - (1,0) means the key is freed, and waiting for being synced, + * - (0,1) is invalid state. + * the number of revalidated DEKs can be calculated by + * hweight_long(need_sync XOR in_use), and the need_sync bits can be reset + * by simply copying from in_use bits. + */ +static void mlx5_crypto_dek_bulk_reset_synced(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek_bulk *bulk) +{ + unsigned long *need_sync = bulk->need_sync; + unsigned long *in_use = bulk->in_use; + int i, freed, reused, avail_next; + bool first = true; + + freed = MLX5_CRYPTO_DEK_BULK_CALC_FREED(bulk); + + for (i = 0; freed && i < BITS_TO_LONGS(bulk->num_deks); + i++, need_sync++, in_use++) { + reused = hweight_long((*need_sync) ^ (*in_use)); + if (!reused) + continue; + + bulk->avail_deks += reused; + pool->avail_deks += reused; + *need_sync = *in_use; + if (first) { + avail_next = i * BITS_PER_TYPE(long); + if (bulk->avail_start > avail_next) + bulk->avail_start = avail_next; + first = false; + } + + freed -= reused; + } +} + +static void mlx5_crypto_dek_pool_free_wait_keys(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek *dek, *next; + + list_for_each_entry_safe(dek, next, &pool->wait_for_free, entry) { + list_del(&dek->entry); + mlx5_crypto_dek_free_locked(pool, dek); + } +} + +/* For all the bulks in each list, reset the bits while sync. + * Move them to different lists according to the number of available DEKs. + * And free DEKs in the waiting list at the end of this func. + */ +static void mlx5_crypto_dek_pool_reset_synced(struct mlx5_crypto_dek_pool *pool) +{ + struct mlx5_crypto_dek_bulk *bulk, *tmp; + + list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) { + mlx5_crypto_dek_bulk_reset_synced(pool, bulk); + if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk)) + list_move(&bulk->entry, &pool->avail_list); + } + + list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) { + mlx5_crypto_dek_bulk_reset_synced(pool, bulk); + + if (!bulk->avail_deks) + continue; + + if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk)) + list_move(&bulk->entry, &pool->avail_list); + else + list_move(&bulk->entry, &pool->partial_list); + } + + list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry) { + memset(bulk->need_sync, 0, BITS_TO_BYTES(bulk->num_deks)); + bulk->avail_start = 0; + bulk->avail_deks = bulk->num_deks; + pool->avail_deks += bulk->num_deks; + } + list_splice_init(&pool->sync_list, &pool->avail_list); + + mlx5_crypto_dek_pool_free_wait_keys(pool); +} + +static void mlx5_crypto_dek_sync_work_fn(struct work_struct *work) +{ + struct mlx5_crypto_dek_pool *pool = + container_of(work, struct mlx5_crypto_dek_pool, sync_work); + int err; + + err = mlx5_crypto_cmd_sync_crypto(pool->mdev, BIT(pool->key_purpose)); + mutex_lock(&pool->lock); + if (!err) + mlx5_crypto_dek_pool_reset_synced(pool); + pool->syncing = false; + mutex_unlock(&pool->lock); +} + struct mlx5_crypto_dek *mlx5_crypto_dek_create(struct mlx5_crypto_dek_pool *dek_pool, const void *key, u32 sz_bytes) { @@ -473,8 +633,12 @@ mlx5_crypto_dek_pool_create(struct mlx5_core_dev *mdev, int key_purpose) pool->key_purpose = key_purpose; mutex_init(&pool->lock); + INIT_LIST_HEAD(&pool->avail_list); INIT_LIST_HEAD(&pool->partial_list); INIT_LIST_HEAD(&pool->full_list); + INIT_LIST_HEAD(&pool->sync_list); + INIT_LIST_HEAD(&pool->wait_for_free); + INIT_WORK(&pool->sync_work, mlx5_crypto_dek_sync_work_fn); return pool; } @@ -483,9 +647,19 @@ void mlx5_crypto_dek_pool_destroy(struct mlx5_crypto_dek_pool *pool) { struct mlx5_crypto_dek_bulk *bulk, *tmp; + cancel_work_sync(&pool->sync_work); + + mlx5_crypto_dek_pool_free_wait_keys(pool); + + list_for_each_entry_safe(bulk, tmp, &pool->avail_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk); + list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) mlx5_crypto_dek_pool_remove_bulk(pool, bulk); + list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry) + mlx5_crypto_dek_pool_remove_bulk(pool, bulk); + list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) mlx5_crypto_dek_pool_remove_bulk(pool, bulk); From 12a9e1b73db00d38d0953c2fd45bd2854c34880a Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Mon, 25 Apr 2022 03:17:47 +0000 Subject: [PATCH 13/15] net/mlx5: Add async garbage collector for DEK bulk After invalidation, the idle bulk with all DEKs available for use, is destroyed, to free keys and mem. To get better performance, the firmware destruction operation is done asynchronously. So idle bulks are enqueued in destroy_list first, then destroyed in system workqueue. This will improve performance, as the destruction doesn't need to hold pool's mutex. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/lib/crypto.c | 74 ++++++++++++++++--- 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index 8f8c18f806014..e078530ef37de 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -46,6 +46,10 @@ struct mlx5_crypto_dek_pool { bool syncing; struct list_head wait_for_free; struct work_struct sync_work; + + spinlock_t destroy_lock; /* protect destroy_list */ + struct list_head destroy_list; + struct work_struct destroy_work; }; struct mlx5_crypto_dek_bulk { @@ -351,13 +355,15 @@ static void mlx5_crypto_dek_bulk_free(struct mlx5_crypto_dek_bulk *bulk) } static void mlx5_crypto_dek_pool_remove_bulk(struct mlx5_crypto_dek_pool *pool, - struct mlx5_crypto_dek_bulk *bulk) + struct mlx5_crypto_dek_bulk *bulk, + bool delay) { pool->num_deks -= bulk->num_deks; pool->avail_deks -= bulk->avail_deks; pool->in_use_deks -= bulk->in_use_deks; list_del(&bulk->entry); - mlx5_crypto_dek_bulk_free(bulk); + if (!delay) + mlx5_crypto_dek_bulk_free(bulk); } static struct mlx5_crypto_dek_bulk * @@ -500,6 +506,23 @@ static void mlx5_crypto_dek_bulk_reset_synced(struct mlx5_crypto_dek_pool *pool, } } +static void mlx5_crypto_dek_bulk_handle_avail(struct mlx5_crypto_dek_pool *pool, + struct mlx5_crypto_dek_bulk *bulk, + struct list_head *destroy_list) +{ + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, true); + list_add(&bulk->entry, destroy_list); +} + +static void mlx5_crypto_dek_pool_splice_destroy_list(struct mlx5_crypto_dek_pool *pool, + struct list_head *list, + struct list_head *head) +{ + spin_lock(&pool->destroy_lock); + list_splice_init(list, head); + spin_unlock(&pool->destroy_lock); +} + static void mlx5_crypto_dek_pool_free_wait_keys(struct mlx5_crypto_dek_pool *pool) { struct mlx5_crypto_dek *dek, *next; @@ -512,16 +535,18 @@ static void mlx5_crypto_dek_pool_free_wait_keys(struct mlx5_crypto_dek_pool *poo /* For all the bulks in each list, reset the bits while sync. * Move them to different lists according to the number of available DEKs. + * Destrory all the idle bulks for now. * And free DEKs in the waiting list at the end of this func. */ static void mlx5_crypto_dek_pool_reset_synced(struct mlx5_crypto_dek_pool *pool) { struct mlx5_crypto_dek_bulk *bulk, *tmp; + LIST_HEAD(destroy_list); list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) { mlx5_crypto_dek_bulk_reset_synced(pool, bulk); if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk)) - list_move(&bulk->entry, &pool->avail_list); + mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list); } list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) { @@ -531,7 +556,7 @@ static void mlx5_crypto_dek_pool_reset_synced(struct mlx5_crypto_dek_pool *pool) continue; if (MLX5_CRYPTO_DEK_BULK_IDLE(bulk)) - list_move(&bulk->entry, &pool->avail_list); + mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list); else list_move(&bulk->entry, &pool->partial_list); } @@ -541,10 +566,16 @@ static void mlx5_crypto_dek_pool_reset_synced(struct mlx5_crypto_dek_pool *pool) bulk->avail_start = 0; bulk->avail_deks = bulk->num_deks; pool->avail_deks += bulk->num_deks; + mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list); } - list_splice_init(&pool->sync_list, &pool->avail_list); mlx5_crypto_dek_pool_free_wait_keys(pool); + + if (!list_empty(&destroy_list)) { + mlx5_crypto_dek_pool_splice_destroy_list(pool, &destroy_list, + &pool->destroy_list); + schedule_work(&pool->destroy_work); + } } static void mlx5_crypto_dek_sync_work_fn(struct work_struct *work) @@ -620,6 +651,25 @@ void mlx5_crypto_dek_destroy(struct mlx5_crypto_dek_pool *dek_pool, } } +static void mlx5_crypto_dek_free_destroy_list(struct list_head *destroy_list) +{ + struct mlx5_crypto_dek_bulk *bulk, *tmp; + + list_for_each_entry_safe(bulk, tmp, destroy_list, entry) + mlx5_crypto_dek_bulk_free(bulk); +} + +static void mlx5_crypto_dek_destroy_work_fn(struct work_struct *work) +{ + struct mlx5_crypto_dek_pool *pool = + container_of(work, struct mlx5_crypto_dek_pool, destroy_work); + LIST_HEAD(destroy_list); + + mlx5_crypto_dek_pool_splice_destroy_list(pool, &pool->destroy_list, + &destroy_list); + mlx5_crypto_dek_free_destroy_list(&destroy_list); +} + struct mlx5_crypto_dek_pool * mlx5_crypto_dek_pool_create(struct mlx5_core_dev *mdev, int key_purpose) { @@ -639,6 +689,9 @@ mlx5_crypto_dek_pool_create(struct mlx5_core_dev *mdev, int key_purpose) INIT_LIST_HEAD(&pool->sync_list); INIT_LIST_HEAD(&pool->wait_for_free); INIT_WORK(&pool->sync_work, mlx5_crypto_dek_sync_work_fn); + spin_lock_init(&pool->destroy_lock); + INIT_LIST_HEAD(&pool->destroy_list); + INIT_WORK(&pool->destroy_work, mlx5_crypto_dek_destroy_work_fn); return pool; } @@ -648,20 +701,23 @@ void mlx5_crypto_dek_pool_destroy(struct mlx5_crypto_dek_pool *pool) struct mlx5_crypto_dek_bulk *bulk, *tmp; cancel_work_sync(&pool->sync_work); + cancel_work_sync(&pool->destroy_work); mlx5_crypto_dek_pool_free_wait_keys(pool); list_for_each_entry_safe(bulk, tmp, &pool->avail_list, entry) - mlx5_crypto_dek_pool_remove_bulk(pool, bulk); + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); list_for_each_entry_safe(bulk, tmp, &pool->full_list, entry) - mlx5_crypto_dek_pool_remove_bulk(pool, bulk); + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry) - mlx5_crypto_dek_pool_remove_bulk(pool, bulk); + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); list_for_each_entry_safe(bulk, tmp, &pool->partial_list, entry) - mlx5_crypto_dek_pool_remove_bulk(pool, bulk); + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, false); + + mlx5_crypto_dek_free_destroy_list(&pool->destroy_list); mutex_destroy(&pool->lock); From 8a6fa6df61ffc44681727ee05f051fd6df420a81 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Mon, 25 Apr 2022 03:17:47 +0000 Subject: [PATCH 14/15] net/mlx5: Keep only one bulk of full available DEKs One bulk with full available keys is left undestroyed, to service the possible requests from users quickly. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/lib/crypto.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c index e078530ef37de..3a94b8f8031ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/crypto.c @@ -506,12 +506,19 @@ static void mlx5_crypto_dek_bulk_reset_synced(struct mlx5_crypto_dek_pool *pool, } } -static void mlx5_crypto_dek_bulk_handle_avail(struct mlx5_crypto_dek_pool *pool, +/* Return true if the bulk is reused, false if destroyed with delay */ +static bool mlx5_crypto_dek_bulk_handle_avail(struct mlx5_crypto_dek_pool *pool, struct mlx5_crypto_dek_bulk *bulk, struct list_head *destroy_list) { + if (list_empty(&pool->avail_list)) { + list_move(&bulk->entry, &pool->avail_list); + return true; + } + mlx5_crypto_dek_pool_remove_bulk(pool, bulk, true); list_add(&bulk->entry, destroy_list); + return false; } static void mlx5_crypto_dek_pool_splice_destroy_list(struct mlx5_crypto_dek_pool *pool, @@ -535,7 +542,7 @@ static void mlx5_crypto_dek_pool_free_wait_keys(struct mlx5_crypto_dek_pool *poo /* For all the bulks in each list, reset the bits while sync. * Move them to different lists according to the number of available DEKs. - * Destrory all the idle bulks for now. + * Destrory all the idle bulks, except one for quick service. * And free DEKs in the waiting list at the end of this func. */ static void mlx5_crypto_dek_pool_reset_synced(struct mlx5_crypto_dek_pool *pool) @@ -562,11 +569,12 @@ static void mlx5_crypto_dek_pool_reset_synced(struct mlx5_crypto_dek_pool *pool) } list_for_each_entry_safe(bulk, tmp, &pool->sync_list, entry) { - memset(bulk->need_sync, 0, BITS_TO_BYTES(bulk->num_deks)); - bulk->avail_start = 0; bulk->avail_deks = bulk->num_deks; pool->avail_deks += bulk->num_deks; - mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list); + if (mlx5_crypto_dek_bulk_handle_avail(pool, bulk, &destroy_list)) { + memset(bulk->need_sync, 0, BITS_TO_BYTES(bulk->num_deks)); + bulk->avail_start = 0; + } } mlx5_crypto_dek_pool_free_wait_keys(pool); From f741db1a5171ebb93289258e64e69c2a780e3103 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Mon, 28 Nov 2022 00:55:16 +0000 Subject: [PATCH 15/15] net/mlx5e: kTLS, Improve connection rate by using fast update encryption key As the fast DEK update is fully implemented, use it for kTLS to get better performance. TIS pool was already supported to recycle the TISes. With this series and TIS pool, TLS CPS is improved by 9x higher, from 11k/s to 101k/s. Signed-off-by: Jianbo Liu Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../mellanox/mlx5/core/en_accel/ktls.c | 26 ++++++++++++------- .../mellanox/mlx5/core/en_accel/ktls.h | 11 +++++--- .../mellanox/mlx5/core/en_accel/ktls_rx.c | 21 ++++++++------- .../mellanox/mlx5/core/en_accel/ktls_tx.c | 17 +++++++----- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c index f80d6fce28d2d..cf704f106b7c2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c @@ -9,9 +9,8 @@ #include "en_accel/ktls_utils.h" #include "en_accel/fs_tcp.h" -int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, - struct tls_crypto_info *crypto_info, - u32 *p_key_id) +struct mlx5_crypto_dek *mlx5_ktls_create_key(struct mlx5_crypto_dek_pool *dek_pool, + struct tls_crypto_info *crypto_info) { const void *key; u32 sz_bytes; @@ -34,17 +33,16 @@ int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, break; } default: - return -EINVAL; + return ERR_PTR(-EINVAL); } - return mlx5_create_encryption_key(mdev, key, sz_bytes, - MLX5_ACCEL_OBJ_TLS_KEY, - p_key_id); + return mlx5_crypto_dek_create(dek_pool, key, sz_bytes); } -void mlx5_ktls_destroy_key(struct mlx5_core_dev *mdev, u32 key_id) +void mlx5_ktls_destroy_key(struct mlx5_crypto_dek_pool *dek_pool, + struct mlx5_crypto_dek *dek) { - mlx5_destroy_encryption_key(mdev, key_id); + mlx5_crypto_dek_destroy(dek_pool, dek); } static int mlx5e_ktls_add(struct net_device *netdev, struct sock *sk, @@ -190,6 +188,7 @@ static void mlx5e_tls_debugfs_init(struct mlx5e_tls *tls, int mlx5e_ktls_init(struct mlx5e_priv *priv) { + struct mlx5_crypto_dek_pool *dek_pool; struct mlx5e_tls *tls; if (!mlx5e_is_ktls_device(priv->mdev)) @@ -198,9 +197,15 @@ int mlx5e_ktls_init(struct mlx5e_priv *priv) tls = kzalloc(sizeof(*tls), GFP_KERNEL); if (!tls) return -ENOMEM; + tls->mdev = priv->mdev; + dek_pool = mlx5_crypto_dek_pool_create(priv->mdev, MLX5_ACCEL_OBJ_TLS_KEY); + if (IS_ERR(dek_pool)) { + kfree(tls); + return PTR_ERR(dek_pool); + } + tls->dek_pool = dek_pool; priv->tls = tls; - priv->tls->mdev = priv->mdev; mlx5e_tls_debugfs_init(tls, priv->dfs_root); @@ -217,6 +222,7 @@ void mlx5e_ktls_cleanup(struct mlx5e_priv *priv) debugfs_remove_recursive(tls->debugfs.dfs); tls->debugfs.dfs = NULL; + mlx5_crypto_dek_pool_destroy(tls->dek_pool); kfree(priv->tls); priv->tls = NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index fccf995ee16d7..f11075e676586 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -10,10 +10,12 @@ #include "en.h" #ifdef CONFIG_MLX5_EN_TLS -int mlx5_ktls_create_key(struct mlx5_core_dev *mdev, - struct tls_crypto_info *crypto_info, - u32 *p_key_id); -void mlx5_ktls_destroy_key(struct mlx5_core_dev *mdev, u32 key_id); +#include "lib/crypto.h" + +struct mlx5_crypto_dek *mlx5_ktls_create_key(struct mlx5_crypto_dek_pool *dek_pool, + struct tls_crypto_info *crypto_info); +void mlx5_ktls_destroy_key(struct mlx5_crypto_dek_pool *dek_pool, + struct mlx5_crypto_dek *dek); static inline bool mlx5e_is_ktls_device(struct mlx5_core_dev *mdev) { @@ -83,6 +85,7 @@ struct mlx5e_tls { struct mlx5e_tls_sw_stats sw_stats; struct workqueue_struct *rx_wq; struct mlx5e_tls_tx_pool *tx_pool; + struct mlx5_crypto_dek_pool *dek_pool; struct mlx5e_tls_debugfs debugfs; }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c index 3e54834747ce6..4be770443b0cd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c @@ -50,7 +50,7 @@ struct mlx5e_ktls_offload_context_rx { struct mlx5e_tls_sw_stats *sw_stats; struct completion add_ctx; struct mlx5e_tir tir; - u32 key_id; + struct mlx5_crypto_dek *dek; u32 rxq; DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS); @@ -148,7 +148,8 @@ post_static_params(struct mlx5e_icosq *sq, wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info, mlx5e_tir_get_tirn(&priv_rx->tir), - priv_rx->key_id, priv_rx->resync.seq, false, + mlx5_crypto_dek_get_id(priv_rx->dek), + priv_rx->resync.seq, false, TLS_OFFLOAD_CTX_DIR_RX); wi = (struct mlx5e_icosq_wqe_info) { .wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS, @@ -610,20 +611,22 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, struct mlx5e_ktls_offload_context_rx *priv_rx; struct mlx5e_ktls_rx_resync_ctx *resync; struct tls_context *tls_ctx; - struct mlx5_core_dev *mdev; + struct mlx5_crypto_dek *dek; struct mlx5e_priv *priv; int rxq, err; tls_ctx = tls_get_ctx(sk); priv = netdev_priv(netdev); - mdev = priv->mdev; priv_rx = kzalloc(sizeof(*priv_rx), GFP_KERNEL); if (unlikely(!priv_rx)) return -ENOMEM; - err = mlx5_ktls_create_key(mdev, crypto_info, &priv_rx->key_id); - if (err) + dek = mlx5_ktls_create_key(priv->tls->dek_pool, crypto_info); + if (IS_ERR(dek)) { + err = PTR_ERR(dek); goto err_create_key; + } + priv_rx->dek = dek; INIT_LIST_HEAD(&priv_rx->list); spin_lock_init(&priv_rx->lock); @@ -673,7 +676,7 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, err_post_wqes: mlx5e_tir_destroy(&priv_rx->tir); err_create_tir: - mlx5_ktls_destroy_key(mdev, priv_rx->key_id); + mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek); err_create_key: kfree(priv_rx); return err; @@ -683,11 +686,9 @@ void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx) { struct mlx5e_ktls_offload_context_rx *priv_rx; struct mlx5e_ktls_rx_resync_ctx *resync; - struct mlx5_core_dev *mdev; struct mlx5e_priv *priv; priv = netdev_priv(netdev); - mdev = priv->mdev; priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx); set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags); @@ -707,7 +708,7 @@ void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx) mlx5e_accel_fs_del_sk(priv_rx->rule.rule); mlx5e_tir_destroy(&priv_rx->tir); - mlx5_ktls_destroy_key(mdev, priv_rx->key_id); + mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek); /* priv_rx should normally be freed here, but if there is an outstanding * GET_PSV, deallocation will be delayed until the CQE for GET_PSV is * processed. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index 6db27062b765f..e80b43b7aac94 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -98,7 +98,7 @@ struct mlx5e_ktls_offload_context_tx { struct tls_offload_context_tx *tx_ctx; struct mlx5_core_dev *mdev; struct mlx5e_tls_sw_stats *sw_stats; - u32 key_id; + struct mlx5_crypto_dek *dek; u8 create_err : 1; }; @@ -457,6 +457,7 @@ int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, struct mlx5e_ktls_offload_context_tx *priv_tx; struct mlx5e_tls_tx_pool *pool; struct tls_context *tls_ctx; + struct mlx5_crypto_dek *dek; struct mlx5e_priv *priv; int err; @@ -468,9 +469,12 @@ int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, if (IS_ERR(priv_tx)) return PTR_ERR(priv_tx); - err = mlx5_ktls_create_key(pool->mdev, crypto_info, &priv_tx->key_id); - if (err) + dek = mlx5_ktls_create_key(priv->tls->dek_pool, crypto_info); + if (IS_ERR(dek)) { + err = PTR_ERR(dek); goto err_create_key; + } + priv_tx->dek = dek; priv_tx->expected_seq = start_offload_tcp_sn; switch (crypto_info->cipher_type) { @@ -512,7 +516,7 @@ void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx) pool = priv->tls->tx_pool; atomic64_inc(&priv_tx->sw_stats->tx_tls_del); - mlx5_ktls_destroy_key(priv_tx->mdev, priv_tx->key_id); + mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_tx->dek); pool_push(pool, priv_tx); } @@ -551,8 +555,9 @@ post_static_params(struct mlx5e_txqsq *sq, pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info, - priv_tx->tisn, priv_tx->key_id, 0, fence, - TLS_OFFLOAD_CTX_DIR_TX); + priv_tx->tisn, + mlx5_crypto_dek_get_id(priv_tx->dek), + 0, fence, TLS_OFFLOAD_CTX_DIR_TX); tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); sq->pc += num_wqebbs; }