[thirdparty/kernel/stable.git] / drivers / net / ethernet / mellanox / mlx5 / core / lib / sd.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */

#include "lib/sd.h"
#include "mlx5_core.h"
#include "lib/mlx5.h"
#include "fs_cmd.h"
#include <linux/mlx5/vport.h>

#define sd_info(__dev, format, ...) \
	dev_info((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__)
#define sd_warn(__dev, format, ...) \
	dev_warn((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__)

struct mlx5_sd {
	u32 group_id;
	u8 host_buses;
	struct mlx5_devcom_comp_dev *devcom;
	bool primary;
	union {
		struct { /* primary */
			struct mlx5_core_dev *secondaries[MLX5_SD_MAX_GROUP_SZ - 1];
			struct mlx5_flow_table *tx_ft;
		};
		struct { /* secondary */
			struct mlx5_core_dev *primary_dev;
			u32 alias_obj_id;
		};
	};
};

static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev)
{
	struct mlx5_sd *sd = mlx5_get_sd(dev);

	if (!sd)
		return 1;

	return sd->host_buses;
}

static struct mlx5_core_dev *mlx5_sd_get_primary(struct mlx5_core_dev *dev)
{
	struct mlx5_sd *sd = mlx5_get_sd(dev);

	if (!sd)
		return dev;

	return sd->primary ? dev : sd->primary_dev;
}

struct mlx5_core_dev *
mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx)
{
	struct mlx5_sd *sd;

	if (idx == 0)
		return primary;

	if (idx >= mlx5_sd_get_host_buses(primary))
		return NULL;

	sd = mlx5_get_sd(primary);
	return sd->secondaries[idx - 1];
}

int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix)
{
	return ch_ix % mlx5_sd_get_host_buses(dev);
}

int mlx5_sd_ch_ix_get_vec_ix(struct mlx5_core_dev *dev, int ch_ix)
{
	return ch_ix / mlx5_sd_get_host_buses(dev);
}

struct mlx5_core_dev *mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev *primary, int ch_ix)
{
	int mdev_idx = mlx5_sd_ch_ix_get_dev_ix(primary, ch_ix);

	return mlx5_sd_primary_get_peer(primary, mdev_idx);
}

static bool ft_create_alias_supported(struct mlx5_core_dev *dev)
{
	u64 obj_allowed = MLX5_CAP_GEN_2_64(dev, allowed_object_for_other_vhca_access);
	u32 obj_supp = MLX5_CAP_GEN_2(dev, cross_vhca_object_to_object_supported);

	if (!(obj_supp &
	    MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_ROOT_TO_REMOTE_FLOW_TABLE))
		return false;

	if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE))
		return false;

	return true;
}

static bool mlx5_sd_is_supported(struct mlx5_core_dev *dev, u8 host_buses)
{
	/* Feature is currently implemented for PFs only */
	if (!mlx5_core_is_pf(dev))
		return false;

	/* Honor the SW implementation limit */
	if (host_buses > MLX5_SD_MAX_GROUP_SZ)
		return false;

	/* Disconnect secondaries from the network */
	if (!MLX5_CAP_GEN(dev, eswitch_manager))
		return false;
	if (!MLX5_CAP_GEN(dev, silent_mode))
		return false;

	/* RX steering from primary to secondaries */
	if (!MLX5_CAP_GEN(dev, cross_vhca_rqt))
		return false;
	if (host_buses > MLX5_CAP_GEN_2(dev, max_rqt_vhca_id))
		return false;

	/* TX steering from secondaries to primary */
	if (!ft_create_alias_supported(dev))
		return false;
	if (!MLX5_CAP_FLOWTABLE_NIC_TX(dev, reset_root_to_default))
		return false;

	return true;
}

static int mlx5_query_sd(struct mlx5_core_dev *dev, bool *sdm,
			 u8 *host_buses, u8 *sd_group)
{
	u32 out[MLX5_ST_SZ_DW(mpir_reg)];
	int err;

	err = mlx5_query_mpir_reg(dev, out);
	if (err)
		return err;

	err = mlx5_query_nic_vport_sd_group(dev, sd_group);
	if (err)
		return err;

	*sdm = MLX5_GET(mpir_reg, out, sdm);
	*host_buses = MLX5_GET(mpir_reg, out, host_buses);

	return 0;
}

static u32 mlx5_sd_group_id(struct mlx5_core_dev *dev, u8 sd_group)
{
	return (u32)((MLX5_CAP_GEN(dev, native_port_num) << 8) | sd_group);
}

static int sd_init(struct mlx5_core_dev *dev)
{
	u8 host_buses, sd_group;
	struct mlx5_sd *sd;
	u32 group_id;
	bool sdm;
	int err;

	err = mlx5_query_sd(dev, &sdm, &host_buses, &sd_group);
	if (err)
		return err;

	if (!sdm)
		return 0;

	if (!sd_group)
		return 0;

	group_id = mlx5_sd_group_id(dev, sd_group);

	if (!mlx5_sd_is_supported(dev, host_buses)) {
		sd_warn(dev, "can't support requested netdev combining for group id 0x%x), skipping\n",
			group_id);
		return 0;
	}

	sd = kzalloc(sizeof(*sd), GFP_KERNEL);
	if (!sd)
		return -ENOMEM;

	sd->host_buses = host_buses;
	sd->group_id = group_id;

	mlx5_set_sd(dev, sd);

	return 0;
}

static void sd_cleanup(struct mlx5_core_dev *dev)
{
	struct mlx5_sd *sd = mlx5_get_sd(dev);

	mlx5_set_sd(dev, NULL);
	kfree(sd);
}

static int sd_register(struct mlx5_core_dev *dev)
{
	struct mlx5_devcom_comp_dev *devcom, *pos;
	struct mlx5_core_dev *peer, *primary;
	struct mlx5_sd *sd, *primary_sd;
	int err, i;

	sd = mlx5_get_sd(dev);
	devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP,
						sd->group_id, NULL, dev);
	if (!devcom)
		return -ENOMEM;

	sd->devcom = devcom;

	if (mlx5_devcom_comp_get_size(devcom) != sd->host_buses)
		return 0;

	mlx5_devcom_comp_lock(devcom);
	mlx5_devcom_comp_set_ready(devcom, true);
	mlx5_devcom_comp_unlock(devcom);

	if (!mlx5_devcom_for_each_peer_begin(devcom)) {
		err = -ENODEV;
		goto err_devcom_unreg;
	}

	primary = dev;
	mlx5_devcom_for_each_peer_entry(devcom, peer, pos)
		if (peer->pdev->bus->number < primary->pdev->bus->number)
			primary = peer;

	primary_sd = mlx5_get_sd(primary);
	primary_sd->primary = true;
	i = 0;
	/* loop the secondaries */
	mlx5_devcom_for_each_peer_entry(primary_sd->devcom, peer, pos) {
		struct mlx5_sd *peer_sd = mlx5_get_sd(peer);

		primary_sd->secondaries[i++] = peer;
		peer_sd->primary = false;
		peer_sd->primary_dev = primary;
	}

	mlx5_devcom_for_each_peer_end(devcom);
	return 0;

err_devcom_unreg:
	mlx5_devcom_comp_lock(sd->devcom);
	mlx5_devcom_comp_set_ready(sd->devcom, false);
	mlx5_devcom_comp_unlock(sd->devcom);
	mlx5_devcom_unregister_component(sd->devcom);
	return err;
}

static void sd_unregister(struct mlx5_core_dev *dev)
{
	struct mlx5_sd *sd = mlx5_get_sd(dev);

	mlx5_devcom_comp_lock(sd->devcom);
	mlx5_devcom_comp_set_ready(sd->devcom, false);
	mlx5_devcom_comp_unlock(sd->devcom);
	mlx5_devcom_unregister_component(sd->devcom);
}

static int sd_cmd_set_primary(struct mlx5_core_dev *primary, u8 *alias_key)
{
	struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {};
	struct mlx5_sd *sd = mlx5_get_sd(primary);
	struct mlx5_flow_table_attr ft_attr = {};
	struct mlx5_flow_namespace *nic_ns;
	struct mlx5_flow_table *ft;
	int err;

	nic_ns = mlx5_get_flow_namespace(primary, MLX5_FLOW_NAMESPACE_EGRESS);
	if (!nic_ns)
		return -EOPNOTSUPP;

	ft = mlx5_create_flow_table(nic_ns, &ft_attr);
	if (IS_ERR(ft)) {
		err = PTR_ERR(ft);
		return err;
	}
	sd->tx_ft = ft;
	memcpy(allow_attr.access_key, alias_key, ACCESS_KEY_LEN);
	allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
	allow_attr.obj_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id;

	err = mlx5_cmd_allow_other_vhca_access(primary, &allow_attr);
	if (err) {
		mlx5_core_err(primary, "Failed to allow other vhca access err=%d\n",
			      err);
		mlx5_destroy_flow_table(ft);
		return err;
	}

	return 0;
}

static void sd_cmd_unset_primary(struct mlx5_core_dev *primary)
{
	struct mlx5_sd *sd = mlx5_get_sd(primary);

	mlx5_destroy_flow_table(sd->tx_ft);
}

static int sd_secondary_create_alias_ft(struct mlx5_core_dev *secondary,
					struct mlx5_core_dev *primary,
					struct mlx5_flow_table *ft,
					u32 *obj_id, u8 *alias_key)
{
	u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id;
	u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(primary, vhca_id);
	struct mlx5_cmd_alias_obj_create_attr alias_attr = {};
	int ret;

	memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN);
	alias_attr.obj_id = aliased_object_id;
	alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
	alias_attr.vhca_id = vhca_id_to_be_accessed;
	ret = mlx5_cmd_alias_obj_create(secondary, &alias_attr, obj_id);
	if (ret) {
		mlx5_core_err(secondary, "Failed to create alias object err=%d\n",
			      ret);
		return ret;
	}

	return 0;
}

static void sd_secondary_destroy_alias_ft(struct mlx5_core_dev *secondary)
{
	struct mlx5_sd *sd = mlx5_get_sd(secondary);

	mlx5_cmd_alias_obj_destroy(secondary, sd->alias_obj_id,
				   MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
}

static int sd_cmd_set_secondary(struct mlx5_core_dev *secondary,
				struct mlx5_core_dev *primary,
				u8 *alias_key)
{
	struct mlx5_sd *primary_sd = mlx5_get_sd(primary);
	struct mlx5_sd *sd = mlx5_get_sd(secondary);
	int err;

	err = mlx5_fs_cmd_set_l2table_entry_silent(secondary, 1);
	if (err)
		return err;

	err = sd_secondary_create_alias_ft(secondary, primary, primary_sd->tx_ft,
					   &sd->alias_obj_id, alias_key);
	if (err)
		goto err_unset_silent;

	err = mlx5_fs_cmd_set_tx_flow_table_root(secondary, sd->alias_obj_id, false);
	if (err)
		goto err_destroy_alias_ft;

	return 0;

err_destroy_alias_ft:
	sd_secondary_destroy_alias_ft(secondary);
err_unset_silent:
	mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0);
	return err;
}

static void sd_cmd_unset_secondary(struct mlx5_core_dev *secondary)
{
	mlx5_fs_cmd_set_tx_flow_table_root(secondary, 0, true);
	sd_secondary_destroy_alias_ft(secondary);
	mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0);
}

int mlx5_sd_init(struct mlx5_core_dev *dev)
{
	struct mlx5_core_dev *primary, *pos, *to;
	struct mlx5_sd *sd = mlx5_get_sd(dev);
	u8 alias_key[ACCESS_KEY_LEN];
	int err, i;

	err = sd_init(dev);
	if (err)
		return err;

	sd = mlx5_get_sd(dev);
	if (!sd)
		return 0;

	err = sd_register(dev);
	if (err)
		goto err_sd_cleanup;

	if (!mlx5_devcom_comp_is_ready(sd->devcom))
		return 0;

	primary = mlx5_sd_get_primary(dev);

	for (i = 0; i < ACCESS_KEY_LEN; i++)
		alias_key[i] = get_random_u8();

	err = sd_cmd_set_primary(primary, alias_key);
	if (err)
		goto err_sd_unregister;

	mlx5_sd_for_each_secondary(i, primary, pos) {
		err = sd_cmd_set_secondary(pos, primary, alias_key);
		if (err)
			goto err_unset_secondaries;
	}

	return 0;

err_unset_secondaries:
	to = pos;
	mlx5_sd_for_each_secondary_to(i, primary, to, pos)
		sd_cmd_unset_secondary(pos);
	sd_cmd_unset_primary(primary);
err_sd_unregister:
	sd_unregister(dev);
err_sd_cleanup:
	sd_cleanup(dev);
	return err;
}

void mlx5_sd_cleanup(struct mlx5_core_dev *dev)
{
	struct mlx5_sd *sd = mlx5_get_sd(dev);
	struct mlx5_core_dev *primary, *pos;
	int i;

	if (!sd)
		return;

	if (!mlx5_devcom_comp_is_ready(sd->devcom))
		goto out;

	primary = mlx5_sd_get_primary(dev);
	mlx5_sd_for_each_secondary(i, primary, pos)
		sd_cmd_unset_secondary(pos);
	sd_cmd_unset_primary(primary);
out:
	sd_unregister(dev);
	sd_cleanup(dev);
}

struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev,
					  struct auxiliary_device *adev,
					  int idx)
{
	struct mlx5_sd *sd = mlx5_get_sd(dev);
	struct mlx5_core_dev *primary;

	if (!sd)
		return adev;

	if (!mlx5_devcom_comp_is_ready(sd->devcom))
		return NULL;

	primary = mlx5_sd_get_primary(dev);
	if (dev == primary)
		return adev;

	return &primary->priv.adev[idx]->adev;
}
Commit	Line	Data
4a04a31f TT	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	/* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
	3
	4	#include "lib/sd.h"
	5	#include "mlx5_core.h"
63b9ce94	6	#include "lib/mlx5.h"
605fcce3	7	#include "fs_cmd.h"
63b9ce94	8	#include <linux/mlx5/vport.h>
4a04a31f TT	9
	10	#define sd_info(__dev, format, ...) \
	11	dev_info((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__)
	12	#define sd_warn(__dev, format, ...) \
	13	dev_warn((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__)
	14
	15	struct mlx5_sd {
63b9ce94 TT	16	u32 group_id;
63b9ce94 TT	17	u8 host_buses;
a45af9a9 TT	18	struct mlx5_devcom_comp_dev *devcom;
	19	bool primary;
	20	union {
	21	struct { /* primary */
	22	struct mlx5_core_dev *secondaries[MLX5_SD_MAX_GROUP_SZ - 1];
605fcce3	23	struct mlx5_flow_table *tx_ft;
a45af9a9 TT	24	};
	25	struct { /* secondary */
	26	struct mlx5_core_dev *primary_dev;
605fcce3	27	u32 alias_obj_id;
a45af9a9 TT	28	};
a45af9a9 TT	29	};
4a04a31f TT	30	};
	31
	32	static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev)
	33	{
63b9ce94 TT	34	struct mlx5_sd *sd = mlx5_get_sd(dev);
	35
	36	if (!sd)
	37	return 1;
	38
	39	return sd->host_buses;
4a04a31f TT	40	}
4a04a31f TT	41
a45af9a9 TT	42	static struct mlx5_core_dev mlx5_sd_get_primary(struct mlx5_core_dev dev)
	43	{
	44	struct mlx5_sd *sd = mlx5_get_sd(dev);
	45
	46	if (!sd)
	47	return dev;
	48
	49	return sd->primary ? dev : sd->primary_dev;
	50	}
	51
4a04a31f TT	52	struct mlx5_core_dev *
	53	mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx)
	54	{
a45af9a9 TT	55	struct mlx5_sd *sd;
a45af9a9 TT	56
4a04a31f TT	57	if (idx == 0)
	58	return primary;
	59
a45af9a9 TT	60	if (idx >= mlx5_sd_get_host_buses(primary))
	61	return NULL;
	62
	63	sd = mlx5_get_sd(primary);
	64	return sd->secondaries[idx - 1];
4a04a31f TT	65	}
	66
	67	int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix)
	68	{
	69	return ch_ix % mlx5_sd_get_host_buses(dev);
	70	}
	71
	72	int mlx5_sd_ch_ix_get_vec_ix(struct mlx5_core_dev *dev, int ch_ix)
	73	{
	74	return ch_ix / mlx5_sd_get_host_buses(dev);
	75	}
	76
	77	struct mlx5_core_dev mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev primary, int ch_ix)
	78	{
	79	int mdev_idx = mlx5_sd_ch_ix_get_dev_ix(primary, ch_ix);
	80
	81	return mlx5_sd_primary_get_peer(primary, mdev_idx);
	82	}
	83
605fcce3 TT	84	static bool ft_create_alias_supported(struct mlx5_core_dev *dev)
	85	{
	86	u64 obj_allowed = MLX5_CAP_GEN_2_64(dev, allowed_object_for_other_vhca_access);
	87	u32 obj_supp = MLX5_CAP_GEN_2(dev, cross_vhca_object_to_object_supported);
	88
	89	if (!(obj_supp &
	90	MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_ROOT_TO_REMOTE_FLOW_TABLE))
	91	return false;
	92
	93	if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE))
	94	return false;
	95
	96	return true;
	97	}
	98
63b9ce94 TT	99	static bool mlx5_sd_is_supported(struct mlx5_core_dev *dev, u8 host_buses)
	100	{
	101	/* Feature is currently implemented for PFs only */
	102	if (!mlx5_core_is_pf(dev))
	103	return false;
	104
	105	/* Honor the SW implementation limit */
	106	if (host_buses > MLX5_SD_MAX_GROUP_SZ)
	107	return false;
	108
605fcce3 TT	109	/* Disconnect secondaries from the network */
	110	if (!MLX5_CAP_GEN(dev, eswitch_manager))
	111	return false;
	112	if (!MLX5_CAP_GEN(dev, silent_mode))
	113	return false;
	114
	115	/* RX steering from primary to secondaries */
	116	if (!MLX5_CAP_GEN(dev, cross_vhca_rqt))
	117	return false;
	118	if (host_buses > MLX5_CAP_GEN_2(dev, max_rqt_vhca_id))
	119	return false;
	120
	121	/* TX steering from secondaries to primary */
	122	if (!ft_create_alias_supported(dev))
	123	return false;
	124	if (!MLX5_CAP_FLOWTABLE_NIC_TX(dev, reset_root_to_default))
	125	return false;
	126
63b9ce94 TT	127	return true;
	128	}
	129
	130	static int mlx5_query_sd(struct mlx5_core_dev dev, bool sdm,
	131	u8 host_buses, u8 sd_group)
	132	{
	133	u32 out[MLX5_ST_SZ_DW(mpir_reg)];
	134	int err;
	135
	136	err = mlx5_query_mpir_reg(dev, out);
	137	if (err)
	138	return err;
	139
	140	err = mlx5_query_nic_vport_sd_group(dev, sd_group);
	141	if (err)
	142	return err;
	143
	144	*sdm = MLX5_GET(mpir_reg, out, sdm);
	145	*host_buses = MLX5_GET(mpir_reg, out, host_buses);
	146
	147	return 0;
	148	}
	149
	150	static u32 mlx5_sd_group_id(struct mlx5_core_dev *dev, u8 sd_group)
	151	{
	152	return (u32)((MLX5_CAP_GEN(dev, native_port_num) << 8) \| sd_group);
	153	}
	154
	155	static int sd_init(struct mlx5_core_dev *dev)
	156	{
	157	u8 host_buses, sd_group;
	158	struct mlx5_sd *sd;
	159	u32 group_id;
	160	bool sdm;
	161	int err;
	162
	163	err = mlx5_query_sd(dev, &sdm, &host_buses, &sd_group);
	164	if (err)
	165	return err;
	166
	167	if (!sdm)
	168	return 0;
	169
	170	if (!sd_group)
	171	return 0;
	172
	173	group_id = mlx5_sd_group_id(dev, sd_group);
	174
	175	if (!mlx5_sd_is_supported(dev, host_buses)) {
	176	sd_warn(dev, "can't support requested netdev combining for group id 0x%x), skipping\n",
	177	group_id);
	178	return 0;
	179	}
	180
	181	sd = kzalloc(sizeof(*sd), GFP_KERNEL);
	182	if (!sd)
	183	return -ENOMEM;
	184
	185	sd->host_buses = host_buses;
	186	sd->group_id = group_id;
	187
	188	mlx5_set_sd(dev, sd);
	189
	190	return 0;
191	}
192
193	static void sd_cleanup(struct mlx5_core_dev *dev)
194	{
195	struct mlx5_sd *sd = mlx5_get_sd(dev);
196
197	mlx5_set_sd(dev, NULL);
198	kfree(sd);
199	}
200
a45af9a9 TT	201	static int sd_register(struct mlx5_core_dev *dev)
	202	{
	203	struct mlx5_devcom_comp_dev devcom, pos;
	204	struct mlx5_core_dev peer, primary;
	205	struct mlx5_sd sd, primary_sd;
	206	int err, i;
	207
	208	sd = mlx5_get_sd(dev);
	209	devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP,
	210	sd->group_id, NULL, dev);
	211	if (!devcom)
	212	return -ENOMEM;
	213
	214	sd->devcom = devcom;
	215
	216	if (mlx5_devcom_comp_get_size(devcom) != sd->host_buses)
	217	return 0;
	218
	219	mlx5_devcom_comp_lock(devcom);
	220	mlx5_devcom_comp_set_ready(devcom, true);
	221	mlx5_devcom_comp_unlock(devcom);
	222
	223	if (!mlx5_devcom_for_each_peer_begin(devcom)) {
	224	err = -ENODEV;
	225	goto err_devcom_unreg;
	226	}
	227
	228	primary = dev;
	229	mlx5_devcom_for_each_peer_entry(devcom, peer, pos)
	230	if (peer->pdev->bus->number < primary->pdev->bus->number)
	231	primary = peer;
	232
	233	primary_sd = mlx5_get_sd(primary);
	234	primary_sd->primary = true;
	235	i = 0;
	236	/* loop the secondaries */
	237	mlx5_devcom_for_each_peer_entry(primary_sd->devcom, peer, pos) {
	238	struct mlx5_sd *peer_sd = mlx5_get_sd(peer);
	239
	240	primary_sd->secondaries[i++] = peer;
	241	peer_sd->primary = false;
	242	peer_sd->primary_dev = primary;
	243	}
	244
	245	mlx5_devcom_for_each_peer_end(devcom);
	246	return 0;
	247
	248	err_devcom_unreg:
	249	mlx5_devcom_comp_lock(sd->devcom);
	250	mlx5_devcom_comp_set_ready(sd->devcom, false);
	251	mlx5_devcom_comp_unlock(sd->devcom);
	252	mlx5_devcom_unregister_component(sd->devcom);
	253	return err;
	254	}
	255
	256	static void sd_unregister(struct mlx5_core_dev *dev)
	257	{
	258	struct mlx5_sd *sd = mlx5_get_sd(dev);
	259
	260	mlx5_devcom_comp_lock(sd->devcom);
	261	mlx5_devcom_comp_set_ready(sd->devcom, false);
	262	mlx5_devcom_comp_unlock(sd->devcom);
	263	mlx5_devcom_unregister_component(sd->devcom);
	264	}
265
605fcce3 TT	266	static int sd_cmd_set_primary(struct mlx5_core_dev primary, u8 alias_key)
	267	{
	268	struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {};
	269	struct mlx5_sd *sd = mlx5_get_sd(primary);
	270	struct mlx5_flow_table_attr ft_attr = {};
	271	struct mlx5_flow_namespace *nic_ns;
	272	struct mlx5_flow_table *ft;
	273	int err;
	274
	275	nic_ns = mlx5_get_flow_namespace(primary, MLX5_FLOW_NAMESPACE_EGRESS);
	276	if (!nic_ns)
	277	return -EOPNOTSUPP;
	278
	279	ft = mlx5_create_flow_table(nic_ns, &ft_attr);
	280	if (IS_ERR(ft)) {
	281	err = PTR_ERR(ft);
	282	return err;
	283	}
	284	sd->tx_ft = ft;
	285	memcpy(allow_attr.access_key, alias_key, ACCESS_KEY_LEN);
	286	allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
	287	allow_attr.obj_id = (ft->type << FT_ID_FT_TYPE_OFFSET) \| ft->id;
	288
	289	err = mlx5_cmd_allow_other_vhca_access(primary, &allow_attr);
	290	if (err) {
	291	mlx5_core_err(primary, "Failed to allow other vhca access err=%d\n",
	292	err);
	293	mlx5_destroy_flow_table(ft);
	294	return err;
	295	}
	296
	297	return 0;
	298	}
	299
	300	static void sd_cmd_unset_primary(struct mlx5_core_dev *primary)
	301	{
	302	struct mlx5_sd *sd = mlx5_get_sd(primary);
	303
	304	mlx5_destroy_flow_table(sd->tx_ft);
	305	}
	306
	307	static int sd_secondary_create_alias_ft(struct mlx5_core_dev *secondary,
	308	struct mlx5_core_dev *primary,
	309	struct mlx5_flow_table *ft,
	310	u32 obj_id, u8 alias_key)
	311	{
	312	u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) \| ft->id;
	313	u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(primary, vhca_id);
	314	struct mlx5_cmd_alias_obj_create_attr alias_attr = {};
	315	int ret;
	316
	317	memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN);
	318	alias_attr.obj_id = aliased_object_id;
	319	alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
	320	alias_attr.vhca_id = vhca_id_to_be_accessed;
	321	ret = mlx5_cmd_alias_obj_create(secondary, &alias_attr, obj_id);
	322	if (ret) {
	323	mlx5_core_err(secondary, "Failed to create alias object err=%d\n",
	324	ret);
	325	return ret;
	326	}
	327
	328	return 0;
	329	}
330
331	static void sd_secondary_destroy_alias_ft(struct mlx5_core_dev *secondary)
332	{
333	struct mlx5_sd *sd = mlx5_get_sd(secondary);
334
335	mlx5_cmd_alias_obj_destroy(secondary, sd->alias_obj_id,
336	MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
337	}
338
339	static int sd_cmd_set_secondary(struct mlx5_core_dev *secondary,
340	struct mlx5_core_dev *primary,
341	u8 *alias_key)
342	{
343	struct mlx5_sd *primary_sd = mlx5_get_sd(primary);
344	struct mlx5_sd *sd = mlx5_get_sd(secondary);
345	int err;
346
347	err = mlx5_fs_cmd_set_l2table_entry_silent(secondary, 1);
348	if (err)
349	return err;
350
351	err = sd_secondary_create_alias_ft(secondary, primary, primary_sd->tx_ft,
352	&sd->alias_obj_id, alias_key);
353	if (err)
354	goto err_unset_silent;
355
356	err = mlx5_fs_cmd_set_tx_flow_table_root(secondary, sd->alias_obj_id, false);
357	if (err)
358	goto err_destroy_alias_ft;
359
360	return 0;
361
362	err_destroy_alias_ft:
363	sd_secondary_destroy_alias_ft(secondary);
364	err_unset_silent:
365	mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0);
366	return err;
367	}
368
369	static void sd_cmd_unset_secondary(struct mlx5_core_dev *secondary)
370	{
371	mlx5_fs_cmd_set_tx_flow_table_root(secondary, 0, true);
372	sd_secondary_destroy_alias_ft(secondary);
373	mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0);
374	}
375
4a04a31f TT	376	int mlx5_sd_init(struct mlx5_core_dev *dev)
4a04a31f TT	377	{
605fcce3	378	struct mlx5_core_dev primary, pos, *to;
a45af9a9	379	struct mlx5_sd *sd = mlx5_get_sd(dev);
605fcce3 TT	380	u8 alias_key[ACCESS_KEY_LEN];
605fcce3 TT	381	int err, i;
63b9ce94 TT	382
	383	err = sd_init(dev);
	384	if (err)
	385	return err;
	386
a45af9a9 TT	387	sd = mlx5_get_sd(dev);
	388	if (!sd)
	389	return 0;
	390
	391	err = sd_register(dev);
	392	if (err)
	393	goto err_sd_cleanup;
	394
605fcce3 TT	395	if (!mlx5_devcom_comp_is_ready(sd->devcom))
	396	return 0;
	397
	398	primary = mlx5_sd_get_primary(dev);
	399
	400	for (i = 0; i < ACCESS_KEY_LEN; i++)
	401	alias_key[i] = get_random_u8();
	402
	403	err = sd_cmd_set_primary(primary, alias_key);
	404	if (err)
	405	goto err_sd_unregister;
	406
	407	mlx5_sd_for_each_secondary(i, primary, pos) {
	408	err = sd_cmd_set_secondary(pos, primary, alias_key);
	409	if (err)
	410	goto err_unset_secondaries;
	411	}
	412
4a04a31f	413	return 0;
a45af9a9	414
605fcce3 TT	415	err_unset_secondaries:
	416	to = pos;
	417	mlx5_sd_for_each_secondary_to(i, primary, to, pos)
	418	sd_cmd_unset_secondary(pos);
	419	sd_cmd_unset_primary(primary);
	420	err_sd_unregister:
	421	sd_unregister(dev);
a45af9a9 TT	422	err_sd_cleanup:
	423	sd_cleanup(dev);
	424	return err;
4a04a31f TT	425	}
	426
	427	void mlx5_sd_cleanup(struct mlx5_core_dev *dev)
	428	{
63b9ce94	429	struct mlx5_sd *sd = mlx5_get_sd(dev);
605fcce3 TT	430	struct mlx5_core_dev primary, pos;
605fcce3 TT	431	int i;
63b9ce94 TT	432
	433	if (!sd)
	434	return;
	435
605fcce3 TT	436	if (!mlx5_devcom_comp_is_ready(sd->devcom))
	437	goto out;
	438
	439	primary = mlx5_sd_get_primary(dev);
	440	mlx5_sd_for_each_secondary(i, primary, pos)
	441	sd_cmd_unset_secondary(pos);
	442	sd_cmd_unset_primary(primary);
	443	out:
a45af9a9	444	sd_unregister(dev);
63b9ce94	445	sd_cleanup(dev);
4a04a31f TT	446	}
	447
	448	struct auxiliary_device mlx5_sd_get_adev(struct mlx5_core_dev dev,
	449	struct auxiliary_device *adev,
	450	int idx)
	451	{
a45af9a9 TT	452	struct mlx5_sd *sd = mlx5_get_sd(dev);
	453	struct mlx5_core_dev *primary;
	454
	455	if (!sd)
	456	return adev;
	457
	458	if (!mlx5_devcom_comp_is_ready(sd->devcom))
	459	return NULL;
	460
	461	primary = mlx5_sd_get_primary(dev);
	462	if (dev == primary)
	463	return adev;
	464
	465	return &primary->priv.adev[idx]->adev;
4a04a31f	466	}