From d0b7e990f760ec9a614fbe5f89a5cede4335a7bb Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:32 +0200 Subject: [PATCH 01/12] devlink: move linecard struct into linecard.c Instead of exposing linecard struct, expose a simple helper to get the linecard index, which is all is needed outside linecard.c. Move the linecard struct to linecard.c and keep it private similar to the rest of the devlink objects. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- net/devlink/devl_internal.h | 14 +------------- net/devlink/linecard.c | 19 +++++++++++++++++++ net/devlink/port.c | 4 ++-- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/net/devlink/devl_internal.h b/net/devlink/devl_internal.h index f6b5fea2e13cd..1b05c2c09e273 100644 --- a/net/devlink/devl_internal.h +++ b/net/devlink/devl_internal.h @@ -206,19 +206,7 @@ int devlink_rate_nodes_check(struct devlink *devlink, u16 mode, struct netlink_ext_ack *extack); /* Linecards */ -struct devlink_linecard { - struct list_head list; - struct devlink *devlink; - unsigned int index; - const struct devlink_linecard_ops *ops; - void *priv; - enum devlink_linecard_state state; - struct mutex state_lock; /* Protects state */ - const char *type; - struct devlink_linecard_type *types; - unsigned int types_count; - struct devlink *nested_devlink; -}; +unsigned int devlink_linecard_index(struct devlink_linecard *linecard); /* Devlink nl cmds */ int devlink_nl_cmd_reload(struct sk_buff *skb, struct genl_info *info); diff --git a/net/devlink/linecard.c b/net/devlink/linecard.c index 85c32c314b0f9..a0210ba56f2de 100644 --- a/net/devlink/linecard.c +++ b/net/devlink/linecard.c @@ -6,6 +6,25 @@ #include "devl_internal.h" +struct devlink_linecard { + struct list_head list; + struct devlink *devlink; + unsigned int index; + const struct devlink_linecard_ops *ops; + void *priv; + enum devlink_linecard_state state; + struct mutex state_lock; /* Protects state */ + const char *type; + struct devlink_linecard_type *types; + unsigned int types_count; + struct devlink *nested_devlink; +}; + +unsigned int devlink_linecard_index(struct devlink_linecard *linecard) +{ + return linecard->index; +} + static struct devlink_linecard * devlink_linecard_get_by_index(struct devlink *devlink, unsigned int linecard_index) diff --git a/net/devlink/port.c b/net/devlink/port.c index 4763b42885fbd..7b300a322ed90 100644 --- a/net/devlink/port.c +++ b/net/devlink/port.c @@ -483,7 +483,7 @@ static int devlink_nl_port_fill(struct sk_buff *msg, goto nla_put_failure; if (devlink_port->linecard && nla_put_u32(msg, DEVLINK_ATTR_LINECARD_INDEX, - devlink_port->linecard->index)) + devlink_linecard_index(devlink_port->linecard))) goto nla_put_failure; genlmsg_end(msg, hdr); @@ -1420,7 +1420,7 @@ static int __devlink_port_phys_port_name_get(struct devlink_port *devlink_port, case DEVLINK_PORT_FLAVOUR_PHYSICAL: if (devlink_port->linecard) n = snprintf(name, len, "l%u", - devlink_port->linecard->index); + devlink_linecard_index(devlink_port->linecard)); if (n < len) n += snprintf(name + n, len - n, "p%u", attrs->phys.port_number); From 85b47dc40bbc72ad68ff9d43bf750290b3c40c2b Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:33 +0200 Subject: [PATCH 02/12] net/mlx5: Disable eswitch as the first thing in mlx5_unload() The eswitch disable call does removal of all representors. Do that before clearing the SF device table and maintain the same flow as during SF devlink port removal, where the representor is removed before the actual SF is removed. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx5/core/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 15561965d2afa..d17c9c31b165d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1405,9 +1405,9 @@ static int mlx5_load(struct mlx5_core_dev *dev) static void mlx5_unload(struct mlx5_core_dev *dev) { + mlx5_eswitch_disable(dev->priv.eswitch); mlx5_devlink_traps_unregister(priv_to_devlink(dev)); mlx5_sf_dev_table_destroy(dev); - mlx5_eswitch_disable(dev->priv.eswitch); mlx5_sriov_detach(dev); mlx5_lag_remove_mdev(dev); mlx5_ec_cleanup(dev); From 602d61e307ac9eabb1b3acdec16b3bec78b75e04 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:34 +0200 Subject: [PATCH 03/12] net/mlx5: Lift reload limitation when SFs are present Historically, the shared devlink_mutex prevented devlink instances from being registered/unregistered during another devlink instance reload operation. However, devlink_muxex is gone for some time now, this limitation is no longer needed. Lift it. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx5/core/devlink.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index af8460bb257b9..3e064234f6fe9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -138,7 +138,6 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change, { struct mlx5_core_dev *dev = devlink_priv(devlink); struct pci_dev *pdev = dev->pdev; - bool sf_dev_allocated; int ret = 0; if (mlx5_dev_is_lightweight(dev)) { @@ -148,16 +147,6 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change, return 0; } - sf_dev_allocated = mlx5_sf_dev_allocated(dev); - if (sf_dev_allocated) { - /* Reload results in deleting SF device which further results in - * unregistering devlink instance while holding devlink_mutext. - * Hence, do not support reload. - */ - NL_SET_ERR_MSG_MOD(extack, "reload is unsupported when SFs are allocated"); - return -EOPNOTSUPP; - } - if (mlx5_lag_is_active(dev)) { NL_SET_ERR_MSG_MOD(extack, "reload is unsupported in Lag mode"); return -EOPNOTSUPP; From ad99637ac92dc18b979e6fa26eb440f38c0c6b55 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:35 +0200 Subject: [PATCH 04/12] devlink: put netnsid to nested handle If netns of devlink instance and nested devlink instance differs, put netnsid attr to indicate that. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- net/devlink/linecard.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/net/devlink/linecard.c b/net/devlink/linecard.c index a0210ba56f2de..f95abdc93c662 100644 --- a/net/devlink/linecard.c +++ b/net/devlink/linecard.c @@ -65,7 +65,8 @@ devlink_linecard_get_from_info(struct devlink *devlink, struct genl_info *info) return devlink_linecard_get_from_attrs(devlink, info->attrs); } -static int devlink_nl_put_nested_handle(struct sk_buff *msg, struct devlink *devlink) +static int devlink_nl_put_nested_handle(struct sk_buff *msg, struct net *net, + struct devlink *devlink) { struct nlattr *nested_attr; @@ -74,6 +75,13 @@ static int devlink_nl_put_nested_handle(struct sk_buff *msg, struct devlink *dev return -EMSGSIZE; if (devlink_nl_put_handle(msg, devlink)) goto nla_put_failure; + if (!net_eq(net, devlink_net(devlink))) { + int id = peernet2id_alloc(net, devlink_net(devlink), + GFP_KERNEL); + + if (nla_put_s32(msg, DEVLINK_ATTR_NETNS_ID, id)) + return -EMSGSIZE; + } nla_nest_end(msg, nested_attr); return 0; @@ -131,7 +139,8 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, } if (linecard->nested_devlink && - devlink_nl_put_nested_handle(msg, linecard->nested_devlink)) + devlink_nl_put_nested_handle(msg, devlink_net(devlink), + linecard->nested_devlink)) goto nla_put_failure; genlmsg_end(msg, hdr); From af1f1400af02e5a069d86ae7001b563c99395ea2 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:36 +0200 Subject: [PATCH 05/12] devlink: move devlink_nl_put_nested_handle() into netlink.c As the next patch is going to call this helper out of the linecard.c, move to netlink.c. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- net/devlink/devl_internal.h | 2 ++ net/devlink/linecard.c | 26 -------------------------- net/devlink/netlink.c | 26 ++++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/net/devlink/devl_internal.h b/net/devlink/devl_internal.h index 1b05c2c09e273..fbf00de1accfe 100644 --- a/net/devlink/devl_internal.h +++ b/net/devlink/devl_internal.h @@ -145,6 +145,8 @@ devlink_nl_put_handle(struct sk_buff *msg, struct devlink *devlink) return 0; } +int devlink_nl_put_nested_handle(struct sk_buff *msg, struct net *net, + struct devlink *devlink); int devlink_nl_msg_reply_and_new(struct sk_buff **msg, struct genl_info *info); /* Notify */ diff --git a/net/devlink/linecard.c b/net/devlink/linecard.c index f95abdc93c662..688e89daee6a9 100644 --- a/net/devlink/linecard.c +++ b/net/devlink/linecard.c @@ -65,32 +65,6 @@ devlink_linecard_get_from_info(struct devlink *devlink, struct genl_info *info) return devlink_linecard_get_from_attrs(devlink, info->attrs); } -static int devlink_nl_put_nested_handle(struct sk_buff *msg, struct net *net, - struct devlink *devlink) -{ - struct nlattr *nested_attr; - - nested_attr = nla_nest_start(msg, DEVLINK_ATTR_NESTED_DEVLINK); - if (!nested_attr) - return -EMSGSIZE; - if (devlink_nl_put_handle(msg, devlink)) - goto nla_put_failure; - if (!net_eq(net, devlink_net(devlink))) { - int id = peernet2id_alloc(net, devlink_net(devlink), - GFP_KERNEL); - - if (nla_put_s32(msg, DEVLINK_ATTR_NETNS_ID, id)) - return -EMSGSIZE; - } - - nla_nest_end(msg, nested_attr); - return 0; - -nla_put_failure: - nla_nest_cancel(msg, nested_attr); - return -EMSGSIZE; -} - struct devlink_linecard_type { const char *type; const void *priv; diff --git a/net/devlink/netlink.c b/net/devlink/netlink.c index fc3e7c029a3be..48b5cfc2842fa 100644 --- a/net/devlink/netlink.c +++ b/net/devlink/netlink.c @@ -82,6 +82,32 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { [DEVLINK_ATTR_REGION_DIRECT] = { .type = NLA_FLAG }, }; +int devlink_nl_put_nested_handle(struct sk_buff *msg, struct net *net, + struct devlink *devlink) +{ + struct nlattr *nested_attr; + + nested_attr = nla_nest_start(msg, DEVLINK_ATTR_NESTED_DEVLINK); + if (!nested_attr) + return -EMSGSIZE; + if (devlink_nl_put_handle(msg, devlink)) + goto nla_put_failure; + if (!net_eq(net, devlink_net(devlink))) { + int id = peernet2id_alloc(net, devlink_net(devlink), + GFP_KERNEL); + + if (nla_put_s32(msg, DEVLINK_ATTR_NETNS_ID, id)) + return -EMSGSIZE; + } + + nla_nest_end(msg, nested_attr); + return 0; + +nla_put_failure: + nla_nest_cancel(msg, nested_attr); + return -EMSGSIZE; +} + int devlink_nl_msg_reply_and_new(struct sk_buff **msg, struct genl_info *info) { int err; From 1c2197c47a93d0ea36e73e437271c7cbcc0e1ceb Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:37 +0200 Subject: [PATCH 06/12] devlink: extend devlink_nl_put_nested_handle() with attrtype arg As the next patch is going to call this helper with need to fill another type of nested attribute, pass it over function arg. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- net/devlink/devl_internal.h | 2 +- net/devlink/linecard.c | 3 ++- net/devlink/netlink.c | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/net/devlink/devl_internal.h b/net/devlink/devl_internal.h index fbf00de1accfe..53449dbd65456 100644 --- a/net/devlink/devl_internal.h +++ b/net/devlink/devl_internal.h @@ -146,7 +146,7 @@ devlink_nl_put_handle(struct sk_buff *msg, struct devlink *devlink) } int devlink_nl_put_nested_handle(struct sk_buff *msg, struct net *net, - struct devlink *devlink); + struct devlink *devlink, int attrtype); int devlink_nl_msg_reply_and_new(struct sk_buff **msg, struct genl_info *info); /* Notify */ diff --git a/net/devlink/linecard.c b/net/devlink/linecard.c index 688e89daee6a9..36170f466878a 100644 --- a/net/devlink/linecard.c +++ b/net/devlink/linecard.c @@ -114,7 +114,8 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, if (linecard->nested_devlink && devlink_nl_put_nested_handle(msg, devlink_net(devlink), - linecard->nested_devlink)) + linecard->nested_devlink, + DEVLINK_ATTR_NESTED_DEVLINK)) goto nla_put_failure; genlmsg_end(msg, hdr); diff --git a/net/devlink/netlink.c b/net/devlink/netlink.c index 48b5cfc2842fa..499304d9de497 100644 --- a/net/devlink/netlink.c +++ b/net/devlink/netlink.c @@ -83,11 +83,11 @@ static const struct nla_policy devlink_nl_policy[DEVLINK_ATTR_MAX + 1] = { }; int devlink_nl_put_nested_handle(struct sk_buff *msg, struct net *net, - struct devlink *devlink) + struct devlink *devlink, int attrtype) { struct nlattr *nested_attr; - nested_attr = nla_nest_start(msg, DEVLINK_ATTR_NESTED_DEVLINK); + nested_attr = nla_nest_start(msg, attrtype); if (!nested_attr) return -EMSGSIZE; if (devlink_nl_put_handle(msg, devlink)) From c137743bce02b18c1537d4681aa515f7b80bf0a8 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:38 +0200 Subject: [PATCH 07/12] devlink: introduce object and nested devlink relationship infra It is a bit tricky to maintain relationship between devlink objects and nested devlink instances due to following aspects: 1) Locking. It is necessary to lock the devlink instance that contains the object first, only after that to lock the nested instance. 2) Lifetimes. Objects (e.g devlink port) may be removed before the nested devlink instance. 3) Notifications. If nested instance changes (e.g. gets registered/unregistered) the nested-in object needs to send appropriate notifications. Resolve this by introducing an xarray that holds 1:1 relationships between devlink object and related nested devlink instance. Use that xarray index to get the object/nested devlink instance on the other side. Provide necessary helpers: devlink_rel_nested_in_add/clear() to add and clear the relationship. devlink_rel_nested_in_notify() to call the nested-in object to send notifications during nested instance register/unregister/netns change. devlink_rel_devlink_handle_put() to be used by nested-in object fill function to fill the nested handle. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- net/devlink/core.c | 215 ++++++++++++++++++++++++++++++++++++ net/devlink/dev.c | 1 + net/devlink/devl_internal.h | 17 +++ 3 files changed, 233 insertions(+) diff --git a/net/devlink/core.c b/net/devlink/core.c index 6cec4afb01fbd..2a98ff9a2f6b5 100644 --- a/net/devlink/core.c +++ b/net/devlink/core.c @@ -16,6 +16,219 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report); DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC); +static struct devlink *devlinks_xa_get(unsigned long index) +{ + struct devlink *devlink; + + rcu_read_lock(); + devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED); + if (!devlink || !devlink_try_get(devlink)) + devlink = NULL; + rcu_read_unlock(); + return devlink; +} + +/* devlink_rels xarray contains 1:1 relationships between + * devlink object and related nested devlink instance. + * The xarray index is used to get the nested object from + * the nested-in object code. + */ +static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1); + +#define DEVLINK_REL_IN_USE XA_MARK_0 + +struct devlink_rel { + u32 index; + refcount_t refcount; + u32 devlink_index; + struct { + u32 devlink_index; + u32 obj_index; + devlink_rel_notify_cb_t *notify_cb; + devlink_rel_cleanup_cb_t *cleanup_cb; + struct work_struct notify_work; + } nested_in; +}; + +static void devlink_rel_free(struct devlink_rel *rel) +{ + xa_erase(&devlink_rels, rel->index); + kfree(rel); +} + +static void __devlink_rel_get(struct devlink_rel *rel) +{ + refcount_inc(&rel->refcount); +} + +static void __devlink_rel_put(struct devlink_rel *rel) +{ + if (refcount_dec_and_test(&rel->refcount)) + devlink_rel_free(rel); +} + +static void devlink_rel_nested_in_notify_work(struct work_struct *work) +{ + struct devlink_rel *rel = container_of(work, struct devlink_rel, + nested_in.notify_work); + struct devlink *devlink; + + devlink = devlinks_xa_get(rel->nested_in.devlink_index); + if (!devlink) + goto rel_put; + if (!devl_trylock(devlink)) { + devlink_put(devlink); + goto reschedule_work; + } + if (!devl_is_registered(devlink)) { + devl_unlock(devlink); + devlink_put(devlink); + goto rel_put; + } + if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE)) + rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index); + rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index); + devl_unlock(devlink); + devlink_put(devlink); + +rel_put: + __devlink_rel_put(rel); + return; + +reschedule_work: + schedule_work(&rel->nested_in.notify_work); +} + +static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel) +{ + __devlink_rel_get(rel); + schedule_work(&rel->nested_in.notify_work); +} + +static struct devlink_rel *devlink_rel_alloc(void) +{ + struct devlink_rel *rel; + static u32 next; + int err; + + rel = kzalloc(sizeof(*rel), GFP_KERNEL); + if (!rel) + return ERR_PTR(-ENOMEM); + + err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel, + xa_limit_32b, &next, GFP_KERNEL); + if (err) { + kfree(rel); + return ERR_PTR(err); + } + + refcount_set(&rel->refcount, 1); + INIT_WORK(&rel->nested_in.notify_work, + &devlink_rel_nested_in_notify_work); + return rel; +} + +static void devlink_rel_put(struct devlink *devlink) +{ + struct devlink_rel *rel = devlink->rel; + + if (!rel) + return; + xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE); + devlink_rel_nested_in_notify_work_schedule(rel); + __devlink_rel_put(rel); + devlink->rel = NULL; +} + +void devlink_rel_nested_in_clear(u32 rel_index) +{ + xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE); +} + +int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index, + u32 obj_index, devlink_rel_notify_cb_t *notify_cb, + devlink_rel_cleanup_cb_t *cleanup_cb, + struct devlink *devlink) +{ + struct devlink_rel *rel = devlink_rel_alloc(); + + ASSERT_DEVLINK_NOT_REGISTERED(devlink); + + if (IS_ERR(rel)) + return PTR_ERR(rel); + + rel->devlink_index = devlink->index; + rel->nested_in.devlink_index = devlink_index; + rel->nested_in.obj_index = obj_index; + rel->nested_in.notify_cb = notify_cb; + rel->nested_in.cleanup_cb = cleanup_cb; + *rel_index = rel->index; + xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE); + devlink->rel = rel; + return 0; +} + +void devlink_rel_nested_in_notify(struct devlink *devlink) +{ + struct devlink_rel *rel = devlink->rel; + + if (!rel) + return; + devlink_rel_nested_in_notify_work_schedule(rel); +} + +static struct devlink_rel *devlink_rel_find(unsigned long rel_index) +{ + return xa_find(&devlink_rels, &rel_index, rel_index, + DEVLINK_REL_IN_USE); +} + +static struct devlink *devlink_rel_devlink_get_lock(u32 rel_index) +{ + struct devlink *devlink; + struct devlink_rel *rel; + u32 devlink_index; + + if (!rel_index) + return NULL; + xa_lock(&devlink_rels); + rel = devlink_rel_find(rel_index); + if (rel) + devlink_index = rel->devlink_index; + xa_unlock(&devlink_rels); + if (!rel) + return NULL; + devlink = devlinks_xa_get(devlink_index); + if (!devlink) + return NULL; + devl_lock(devlink); + if (!devl_is_registered(devlink)) { + devl_unlock(devlink); + devlink_put(devlink); + return NULL; + } + return devlink; +} + +int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink, + u32 rel_index, int attrtype, + bool *msg_updated) +{ + struct net *net = devlink_net(devlink); + struct devlink *rel_devlink; + int err; + + rel_devlink = devlink_rel_devlink_get_lock(rel_index); + if (!rel_devlink) + return 0; + err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype); + devl_unlock(rel_devlink); + devlink_put(rel_devlink); + if (!err && msg_updated) + *msg_updated = true; + return err; +} + void *devlink_priv(struct devlink *devlink) { return &devlink->priv; @@ -142,6 +355,7 @@ int devl_register(struct devlink *devlink) xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED); devlink_notify_register(devlink); + devlink_rel_nested_in_notify(devlink); return 0; } @@ -166,6 +380,7 @@ void devl_unregister(struct devlink *devlink) devlink_notify_unregister(devlink); xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED); + devlink_rel_put(devlink); } EXPORT_SYMBOL_GPL(devl_unregister); diff --git a/net/devlink/dev.c b/net/devlink/dev.c index bba4ace7d22ba..3ae26d9088ab3 100644 --- a/net/devlink/dev.c +++ b/net/devlink/dev.c @@ -372,6 +372,7 @@ static void devlink_reload_netns_change(struct devlink *devlink, devlink_notify_unregister(devlink); write_pnet(&devlink->_net, dest_net); devlink_notify_register(devlink); + devlink_rel_nested_in_notify(devlink); } int devlink_reload(struct devlink *devlink, struct net *dest_net, diff --git a/net/devlink/devl_internal.h b/net/devlink/devl_internal.h index 53449dbd65456..4cb534aff44db 100644 --- a/net/devlink/devl_internal.h +++ b/net/devlink/devl_internal.h @@ -17,6 +17,8 @@ #include "netlink_gen.h" +struct devlink_rel; + #define DEVLINK_REGISTERED XA_MARK_1 #define DEVLINK_RELOAD_STATS_ARRAY_SIZE \ @@ -55,6 +57,7 @@ struct devlink { u8 reload_failed:1; refcount_t refcount; struct rcu_work rwork; + struct devlink_rel *rel; char priv[] __aligned(NETDEV_ALIGN); }; @@ -92,6 +95,20 @@ static inline bool devl_is_registered(struct devlink *devlink) return xa_get_mark(&devlinks, devlink->index, DEVLINK_REGISTERED); } +typedef void devlink_rel_notify_cb_t(struct devlink *devlink, u32 obj_index); +typedef void devlink_rel_cleanup_cb_t(struct devlink *devlink, u32 obj_index, + u32 rel_index); + +void devlink_rel_nested_in_clear(u32 rel_index); +int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index, + u32 obj_index, devlink_rel_notify_cb_t *notify_cb, + devlink_rel_cleanup_cb_t *cleanup_cb, + struct devlink *devlink); +void devlink_rel_nested_in_notify(struct devlink *devlink); +int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink, + u32 rel_index, int attrtype, + bool *msg_updated); + /* Netlink */ #define DEVLINK_NL_FLAG_NEED_PORT BIT(0) #define DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT BIT(1) From 0b7a2721e36c11313f8b0f251a508d25a872cd28 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:39 +0200 Subject: [PATCH 08/12] devlink: expose peer SF devlink instance Introduce a new helper devl_port_fn_devlink_set() to be used by driver assigning a devlink instance to the peer devlink port function. Expose this to user over new netlink attribute nested under port function nest to expose devlink handle related to the port function. This is particularly helpful for user to understand the relationship between devlink instances created for SFs and the port functions they belong to. Note that caller of devlink_port_notify() needs to hold devlink instance lock, put the assertion to devl_port_fn_devlink_set() to make this requirement explicit. Also note the limitations that only allow to make this assignment for registered objects. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- include/net/devlink.h | 3 +++ include/uapi/linux/devlink.h | 1 + net/devlink/port.c | 51 ++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/include/net/devlink.h b/include/net/devlink.h index 29fd1b4ee6548..2655ab6101ecf 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -150,6 +150,7 @@ struct devlink_port { struct devlink_rate *devlink_rate; struct devlink_linecard *linecard; + u32 rel_index; }; struct devlink_port_new_attrs { @@ -1697,6 +1698,8 @@ void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 contro void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 controller, u16 pf, u32 sf, bool external); +int devl_port_fn_devlink_set(struct devlink_port *devlink_port, + struct devlink *fn_devlink); struct devlink_rate * devl_rate_node_create(struct devlink *devlink, void *priv, char *node_name, struct devlink_rate *parent); diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h index 03875e078be84..cd4b82458d1bc 100644 --- a/include/uapi/linux/devlink.h +++ b/include/uapi/linux/devlink.h @@ -680,6 +680,7 @@ enum devlink_port_function_attr { DEVLINK_PORT_FN_ATTR_STATE, /* u8 */ DEVLINK_PORT_FN_ATTR_OPSTATE, /* u8 */ DEVLINK_PORT_FN_ATTR_CAPS, /* bitfield32 */ + DEVLINK_PORT_FN_ATTR_DEVLINK, /* nested */ __DEVLINK_PORT_FUNCTION_ATTR_MAX, DEVLINK_PORT_FUNCTION_ATTR_MAX = __DEVLINK_PORT_FUNCTION_ATTR_MAX - 1 diff --git a/net/devlink/port.c b/net/devlink/port.c index 7b300a322ed90..4e90032424489 100644 --- a/net/devlink/port.c +++ b/net/devlink/port.c @@ -428,6 +428,13 @@ devlink_nl_port_function_attrs_put(struct sk_buff *msg, struct devlink_port *por if (err) goto out; err = devlink_port_fn_state_fill(port, msg, extack, &msg_updated); + if (err) + goto out; + err = devlink_rel_devlink_handle_put(msg, port->devlink, + port->rel_index, + DEVLINK_PORT_FN_ATTR_DEVLINK, + &msg_updated); + out: if (err || !msg_updated) nla_nest_cancel(msg, function_attr); @@ -1392,6 +1399,50 @@ void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 contro } EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_sf_set); +static void devlink_port_rel_notify_cb(struct devlink *devlink, u32 port_index) +{ + struct devlink_port *devlink_port; + + devlink_port = devlink_port_get_by_index(devlink, port_index); + if (!devlink_port) + return; + devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_NEW); +} + +static void devlink_port_rel_cleanup_cb(struct devlink *devlink, u32 port_index, + u32 rel_index) +{ + struct devlink_port *devlink_port; + + devlink_port = devlink_port_get_by_index(devlink, port_index); + if (devlink_port && devlink_port->rel_index == rel_index) + devlink_port->rel_index = 0; +} + +/** + * devl_port_fn_devlink_set - Attach peer devlink + * instance to port function. + * @devlink_port: devlink port + * @fn_devlink: devlink instance to attach + */ +int devl_port_fn_devlink_set(struct devlink_port *devlink_port, + struct devlink *fn_devlink) +{ + ASSERT_DEVLINK_PORT_REGISTERED(devlink_port); + + if (WARN_ON(devlink_port->attrs.flavour != DEVLINK_PORT_FLAVOUR_PCI_SF || + devlink_port->attrs.pci_sf.external)) + return -EINVAL; + + return devlink_rel_nested_in_add(&devlink_port->rel_index, + devlink_port->devlink->index, + devlink_port->index, + devlink_port_rel_notify_cb, + devlink_port_rel_cleanup_cb, + fn_devlink); +} +EXPORT_SYMBOL_GPL(devl_port_fn_devlink_set); + /** * devlink_port_linecard_set - Link port with a linecard * From ac5f395685bd16ca30c1c834dcbf8b555605ccae Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:40 +0200 Subject: [PATCH 09/12] net/mlx5: SF, Implement peer devlink set for SF representor devlink port Benefit from the existence of internal mlx5 notifier and extend it by event MLX5_DRIVER_EVENT_SF_PEER_DEVLINK. Use this event from SF auxiliary device probe/remove functions to pass the registered SF devlink instance to the SF representor. Process the new event in SF representor code and call devl_port_fn_devlink_set() to do the assignments. Implement this in work to avoid possible deadlock when probe/remove function of SF may be called with devlink instance lock held during devlink reload. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- .../ethernet/mellanox/mlx5/core/sf/dev/dev.h | 6 ++++ .../mellanox/mlx5/core/sf/dev/driver.c | 26 ++++++++++++++ .../ethernet/mellanox/mlx5/core/sf/devlink.c | 34 +++++++++++++++++++ include/linux/mlx5/device.h | 1 + 4 files changed, 67 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h index 2a66a427ef15a..b99131e95e37e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/dev.h @@ -19,6 +19,12 @@ struct mlx5_sf_dev { u16 fn_id; }; +struct mlx5_sf_peer_devlink_event_ctx { + u16 fn_id; + struct devlink *devlink; + int err; +}; + void mlx5_sf_dev_table_create(struct mlx5_core_dev *dev); void mlx5_sf_dev_table_destroy(struct mlx5_core_dev *dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c index 8fe82f1191bb9..169c2c68ed5c2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c @@ -8,6 +8,20 @@ #include "dev.h" #include "devlink.h" +static int mlx5_core_peer_devlink_set(struct mlx5_sf_dev *sf_dev, struct devlink *devlink) +{ + struct mlx5_sf_peer_devlink_event_ctx event_ctx = { + .fn_id = sf_dev->fn_id, + .devlink = devlink, + }; + int ret; + + ret = mlx5_blocking_notifier_call_chain(sf_dev->parent_mdev, + MLX5_DRIVER_EVENT_SF_PEER_DEVLINK, + &event_ctx); + return ret == NOTIFY_OK ? event_ctx.err : 0; +} + static int mlx5_sf_dev_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev); @@ -54,9 +68,21 @@ static int mlx5_sf_dev_probe(struct auxiliary_device *adev, const struct auxilia mlx5_core_warn(mdev, "mlx5_init_one err=%d\n", err); goto init_one_err; } + + err = mlx5_core_peer_devlink_set(sf_dev, devlink); + if (err) { + mlx5_core_warn(mdev, "mlx5_core_peer_devlink_set err=%d\n", err); + goto peer_devlink_set_err; + } + devlink_register(devlink); return 0; +peer_devlink_set_err: + if (mlx5_dev_is_lightweight(sf_dev->mdev)) + mlx5_uninit_one_light(sf_dev->mdev); + else + mlx5_uninit_one(sf_dev->mdev); init_one_err: iounmap(mdev->iseg); remap_err: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c index e34a8f88c518c..964a5b1876f30 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c @@ -28,6 +28,7 @@ struct mlx5_sf_table { struct mutex sf_state_lock; /* Serializes sf state among user cmds & vhca event handler. */ struct notifier_block esw_nb; struct notifier_block vhca_nb; + struct notifier_block mdev_nb; }; static struct mlx5_sf * @@ -511,6 +512,35 @@ static int mlx5_sf_esw_event(struct notifier_block *nb, unsigned long event, voi return 0; } +static int mlx5_sf_mdev_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct mlx5_sf_table *table = container_of(nb, struct mlx5_sf_table, mdev_nb); + struct mlx5_sf_peer_devlink_event_ctx *event_ctx = data; + int ret = NOTIFY_DONE; + struct mlx5_sf *sf; + + if (event != MLX5_DRIVER_EVENT_SF_PEER_DEVLINK) + return NOTIFY_DONE; + + table = mlx5_sf_table_try_get(table->dev); + if (!table) + return NOTIFY_DONE; + + mutex_lock(&table->sf_state_lock); + sf = mlx5_sf_lookup_by_function_id(table, event_ctx->fn_id); + if (!sf) + goto out; + + event_ctx->err = devl_port_fn_devlink_set(&sf->dl_port.dl_port, + event_ctx->devlink); + + ret = NOTIFY_OK; +out: + mutex_unlock(&table->sf_state_lock); + mlx5_sf_table_put(table); + return ret; +} + static bool mlx5_sf_table_supported(const struct mlx5_core_dev *dev) { return dev->priv.eswitch && MLX5_ESWITCH_MANAGER(dev) && @@ -544,6 +574,9 @@ int mlx5_sf_table_init(struct mlx5_core_dev *dev) if (err) goto vhca_err; + table->mdev_nb.notifier_call = mlx5_sf_mdev_event; + mlx5_blocking_notifier_register(dev, &table->mdev_nb); + return 0; vhca_err: @@ -562,6 +595,7 @@ void mlx5_sf_table_cleanup(struct mlx5_core_dev *dev) if (!table) return; + mlx5_blocking_notifier_unregister(dev, &table->mdev_nb); mlx5_vhca_event_notifier_unregister(table->dev, &table->vhca_nb); mlx5_esw_event_notifier_unregister(dev->priv.eswitch, &table->esw_nb); WARN_ON(refcount_read(&table->refcount)); diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 4d5be378fa8cc..8fbe22de16ef5 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -366,6 +366,7 @@ enum mlx5_driver_event { MLX5_DRIVER_EVENT_UPLINK_NETDEV, MLX5_DRIVER_EVENT_MACSEC_SA_ADDED, MLX5_DRIVER_EVENT_MACSEC_SA_DELETED, + MLX5_DRIVER_EVENT_SF_PEER_DEVLINK, }; enum { From 9473bc0119e7e7630d7c1c7c3816c290a6f3ae19 Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:41 +0200 Subject: [PATCH 10/12] devlink: convert linecard nested devlink to new rel infrastructure Benefit from the newly introduced rel infrastructure, treat the linecard nested devlink instances in the same way as port function instances. Convert the code to use the rel infrastructure. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- .../mellanox/mlxsw/core_linecard_dev.c | 9 ++-- include/net/devlink.h | 4 +- net/devlink/linecard.c | 47 ++++++++++++++----- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_linecard_dev.c b/drivers/net/ethernet/mellanox/mlxsw/core_linecard_dev.c index af37e650a8ad0..e8d6fe35bf361 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_linecard_dev.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_linecard_dev.c @@ -132,6 +132,7 @@ static int mlxsw_linecard_bdev_probe(struct auxiliary_device *adev, struct mlxsw_linecard *linecard = linecard_bdev->linecard; struct mlxsw_linecard_dev *linecard_dev; struct devlink *devlink; + int err; devlink = devlink_alloc(&mlxsw_linecard_dev_devlink_ops, sizeof(*linecard_dev), &adev->dev); @@ -141,8 +142,12 @@ static int mlxsw_linecard_bdev_probe(struct auxiliary_device *adev, linecard_dev->linecard = linecard_bdev->linecard; linecard_bdev->linecard_dev = linecard_dev; + err = devlink_linecard_nested_dl_set(linecard->devlink_linecard, devlink); + if (err) { + devlink_free(devlink); + return err; + } devlink_register(devlink); - devlink_linecard_nested_dl_set(linecard->devlink_linecard, devlink); return 0; } @@ -151,9 +156,7 @@ static void mlxsw_linecard_bdev_remove(struct auxiliary_device *adev) struct mlxsw_linecard_bdev *linecard_bdev = container_of(adev, struct mlxsw_linecard_bdev, adev); struct devlink *devlink = priv_to_devlink(linecard_bdev->linecard_dev); - struct mlxsw_linecard *linecard = linecard_bdev->linecard; - devlink_linecard_nested_dl_set(linecard->devlink_linecard, NULL); devlink_unregister(devlink); devlink_free(devlink); } diff --git a/include/net/devlink.h b/include/net/devlink.h index 2655ab6101ecf..0dfcd7d7fa18a 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -1720,8 +1720,8 @@ void devlink_linecard_provision_clear(struct devlink_linecard *linecard); void devlink_linecard_provision_fail(struct devlink_linecard *linecard); void devlink_linecard_activate(struct devlink_linecard *linecard); void devlink_linecard_deactivate(struct devlink_linecard *linecard); -void devlink_linecard_nested_dl_set(struct devlink_linecard *linecard, - struct devlink *nested_devlink); +int devlink_linecard_nested_dl_set(struct devlink_linecard *linecard, + struct devlink *nested_devlink); int devl_sb_register(struct devlink *devlink, unsigned int sb_index, u32 size, u16 ingress_pools_count, u16 egress_pools_count, u16 ingress_tc_count, diff --git a/net/devlink/linecard.c b/net/devlink/linecard.c index 36170f466878a..9ff1813f88c55 100644 --- a/net/devlink/linecard.c +++ b/net/devlink/linecard.c @@ -17,7 +17,7 @@ struct devlink_linecard { const char *type; struct devlink_linecard_type *types; unsigned int types_count; - struct devlink *nested_devlink; + u32 rel_index; }; unsigned int devlink_linecard_index(struct devlink_linecard *linecard) @@ -112,10 +112,10 @@ static int devlink_nl_linecard_fill(struct sk_buff *msg, nla_nest_end(msg, attr); } - if (linecard->nested_devlink && - devlink_nl_put_nested_handle(msg, devlink_net(devlink), - linecard->nested_devlink, - DEVLINK_ATTR_NESTED_DEVLINK)) + if (devlink_rel_devlink_handle_put(msg, devlink, + linecard->rel_index, + DEVLINK_ATTR_NESTED_DEVLINK, + NULL)) goto nla_put_failure; genlmsg_end(msg, hdr); @@ -524,7 +524,6 @@ EXPORT_SYMBOL_GPL(devlink_linecard_provision_set); void devlink_linecard_provision_clear(struct devlink_linecard *linecard) { mutex_lock(&linecard->state_lock); - WARN_ON(linecard->nested_devlink); linecard->state = DEVLINK_LINECARD_STATE_UNPROVISIONED; linecard->type = NULL; devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); @@ -543,7 +542,6 @@ EXPORT_SYMBOL_GPL(devlink_linecard_provision_clear); void devlink_linecard_provision_fail(struct devlink_linecard *linecard) { mutex_lock(&linecard->state_lock); - WARN_ON(linecard->nested_devlink); linecard->state = DEVLINK_LINECARD_STATE_PROVISIONING_FAILED; devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); mutex_unlock(&linecard->state_lock); @@ -591,6 +589,27 @@ void devlink_linecard_deactivate(struct devlink_linecard *linecard) } EXPORT_SYMBOL_GPL(devlink_linecard_deactivate); +static void devlink_linecard_rel_notify_cb(struct devlink *devlink, + u32 linecard_index) +{ + struct devlink_linecard *linecard; + + linecard = devlink_linecard_get_by_index(devlink, linecard_index); + if (!linecard) + return; + devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); +} + +static void devlink_linecard_rel_cleanup_cb(struct devlink *devlink, + u32 linecard_index, u32 rel_index) +{ + struct devlink_linecard *linecard; + + linecard = devlink_linecard_get_by_index(devlink, linecard_index); + if (linecard && linecard->rel_index == rel_index) + linecard->rel_index = 0; +} + /** * devlink_linecard_nested_dl_set - Attach/detach nested devlink * instance to linecard. @@ -598,12 +617,14 @@ EXPORT_SYMBOL_GPL(devlink_linecard_deactivate); * @linecard: devlink linecard * @nested_devlink: devlink instance to attach or NULL to detach */ -void devlink_linecard_nested_dl_set(struct devlink_linecard *linecard, - struct devlink *nested_devlink) +int devlink_linecard_nested_dl_set(struct devlink_linecard *linecard, + struct devlink *nested_devlink) { - mutex_lock(&linecard->state_lock); - linecard->nested_devlink = nested_devlink; - devlink_linecard_notify(linecard, DEVLINK_CMD_LINECARD_NEW); - mutex_unlock(&linecard->state_lock); + return devlink_rel_nested_in_add(&linecard->rel_index, + linecard->devlink->index, + linecard->index, + devlink_linecard_rel_notify_cb, + devlink_linecard_rel_cleanup_cb, + nested_devlink); } EXPORT_SYMBOL_GPL(devlink_linecard_nested_dl_set); From c5e1bf8a51cfe5060e91c7533098e329c0118f6d Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:42 +0200 Subject: [PATCH 11/12] devlink: introduce possibility to expose info about nested devlinks In mlx5, there is a devlink instance created for PCI device. Also, one separate devlink instance is created for auxiliary device that represents the netdev of uplink port. This relation is currently invisible to the devlink user. Benefit from the rel infrastructure and allow for nested devlink instance to set the relationship for the nested-in devlink instance. Note that there may be many nested instances, therefore use xarray to hold the list of rel_indexes for individual nested instances. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- include/net/devlink.h | 2 ++ net/devlink/core.c | 2 ++ net/devlink/dev.c | 49 +++++++++++++++++++++++++++++++++++++ net/devlink/devl_internal.h | 1 + 4 files changed, 54 insertions(+) diff --git a/include/net/devlink.h b/include/net/devlink.h index 0dfcd7d7fa18a..fad8e36e3d988 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -1921,6 +1921,8 @@ devlink_health_reporter_state_update(struct devlink_health_reporter *reporter, void devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter); +int devl_nested_devlink_set(struct devlink *devlink, + struct devlink *nested_devlink); bool devlink_is_reload_failed(const struct devlink *devlink); void devlink_remote_reload_actions_performed(struct devlink *devlink, enum devlink_reload_limit limit, diff --git a/net/devlink/core.c b/net/devlink/core.c index 2a98ff9a2f6b5..bcbbb952569f5 100644 --- a/net/devlink/core.c +++ b/net/devlink/core.c @@ -430,6 +430,7 @@ struct devlink *devlink_alloc_ns(const struct devlink_ops *ops, xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC); xa_init_flags(&devlink->params, XA_FLAGS_ALLOC); xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC); + xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC); write_pnet(&devlink->_net, net); INIT_LIST_HEAD(&devlink->rate_list); INIT_LIST_HEAD(&devlink->linecard_list); @@ -476,6 +477,7 @@ void devlink_free(struct devlink *devlink) WARN_ON(!list_empty(&devlink->linecard_list)); WARN_ON(!xa_empty(&devlink->ports)); + xa_destroy(&devlink->nested_rels); xa_destroy(&devlink->snapshot_ids); xa_destroy(&devlink->params); xa_destroy(&devlink->ports); diff --git a/net/devlink/dev.c b/net/devlink/dev.c index 3ae26d9088ab3..dc8039ca2b388 100644 --- a/net/devlink/dev.c +++ b/net/devlink/dev.c @@ -138,6 +138,23 @@ devlink_reload_stats_put(struct sk_buff *msg, struct devlink *devlink, bool is_r return -EMSGSIZE; } +static int devlink_nl_nested_fill(struct sk_buff *msg, struct devlink *devlink) +{ + unsigned long rel_index; + void *unused; + int err; + + xa_for_each(&devlink->nested_rels, rel_index, unused) { + err = devlink_rel_devlink_handle_put(msg, devlink, + rel_index, + DEVLINK_ATTR_NESTED_DEVLINK, + NULL); + if (err) + return err; + } + return 0; +} + static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink, enum devlink_command cmd, u32 portid, u32 seq, int flags) @@ -164,6 +181,10 @@ static int devlink_nl_fill(struct sk_buff *msg, struct devlink *devlink, goto dev_stats_nest_cancel; nla_nest_end(msg, dev_stats); + + if (devlink_nl_nested_fill(msg, devlink)) + goto nla_put_failure; + genlmsg_end(msg, hdr); return 0; @@ -230,6 +251,34 @@ int devlink_nl_get_dumpit(struct sk_buff *msg, struct netlink_callback *cb) return devlink_nl_dumpit(msg, cb, devlink_nl_get_dump_one); } +static void devlink_rel_notify_cb(struct devlink *devlink, u32 obj_index) +{ + devlink_notify(devlink, DEVLINK_CMD_NEW); +} + +static void devlink_rel_cleanup_cb(struct devlink *devlink, u32 obj_index, + u32 rel_index) +{ + xa_erase(&devlink->nested_rels, rel_index); +} + +int devl_nested_devlink_set(struct devlink *devlink, + struct devlink *nested_devlink) +{ + u32 rel_index; + int err; + + err = devlink_rel_nested_in_add(&rel_index, devlink->index, 0, + devlink_rel_notify_cb, + devlink_rel_cleanup_cb, + nested_devlink); + if (err) + return err; + return xa_insert(&devlink->nested_rels, rel_index, + xa_mk_value(0), GFP_KERNEL); +} +EXPORT_SYMBOL_GPL(devl_nested_devlink_set); + void devlink_notify_register(struct devlink *devlink) { devlink_notify(devlink, DEVLINK_CMD_NEW); diff --git a/net/devlink/devl_internal.h b/net/devlink/devl_internal.h index 4cb534aff44db..741d1bf1bec88 100644 --- a/net/devlink/devl_internal.h +++ b/net/devlink/devl_internal.h @@ -58,6 +58,7 @@ struct devlink { refcount_t refcount; struct rcu_work rwork; struct devlink_rel *rel; + struct xarray nested_rels; char priv[] __aligned(NETDEV_ALIGN); }; From 6c75258cc220eb11c31ec8cd23ddf7e1232077bd Mon Sep 17 00:00:00 2001 From: Jiri Pirko Date: Wed, 13 Sep 2023 09:12:43 +0200 Subject: [PATCH 12/12] net/mlx5e: Set en auxiliary devlink instance as nested Benefit from the previous commit introducing exposure of devlink instances relationship and set the nested instance for en auxiliary device. Signed-off-by: Jiri Pirko Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c index c6b6e290fd79d..0b1ac6e5c8900 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c @@ -12,11 +12,19 @@ struct mlx5e_dev *mlx5e_create_devlink(struct device *dev, { struct mlx5e_dev *mlx5e_dev; struct devlink *devlink; + int err; devlink = devlink_alloc_ns(&mlx5e_devlink_ops, sizeof(*mlx5e_dev), devlink_net(priv_to_devlink(mdev)), dev); if (!devlink) return ERR_PTR(-ENOMEM); + + err = devl_nested_devlink_set(priv_to_devlink(mdev), devlink); + if (err) { + devlink_free(devlink); + return ERR_PTR(err); + } + devlink_register(devlink); return devlink_priv(devlink); }