drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
 /* Copyright (c) 2021 Mellanox Technologies. */

 #include <linux/etherdevice.h>
 #include <linux/idr.h>
 #include <linux/mlx5/driver.h>
 #include <linux/mlx5/mlx5_ifc.h>
 #include <linux/mlx5/vport.h>
 #include <linux/mlx5/fs.h>
 #include "mlx5_core.h"
 #include "eswitch.h"
 #include "en.h"
 #include "en_tc.h"
 #include "fs_core.h"
 #include "esw/indir_table.h"
 #include "lib/fs_chains.h"

 #define MLX5_ESW_INDIR_TABLE_SIZE 128
 #define MLX5_ESW_INDIR_TABLE_RECIRC_IDX_MAX (MLX5_ESW_INDIR_TABLE_SIZE - 2)
 #define MLX5_ESW_INDIR_TABLE_FWD_IDX (MLX5_ESW_INDIR_TABLE_SIZE - 1)

 struct mlx5_esw_indir_table_rule {
 	struct list_head list;
 	struct mlx5_flow_handle *handle;
 	union {
 		__be32 v4;
 		struct in6_addr v6;
 	} dst_ip;
 	u32 vni;
 	struct mlx5_modify_hdr *mh;
 	refcount_t refcnt;
 };

 struct mlx5_esw_indir_table_entry {
 	struct hlist_node hlist;
 	struct mlx5_flow_table *ft;
 	struct mlx5_flow_group *recirc_grp;
 	struct mlx5_flow_group *fwd_grp;
 	struct mlx5_flow_handle *fwd_rule;
 	struct list_head recirc_rules;
 	int recirc_cnt;
 	int fwd_ref;

 	u16 vport;
 	u8 ip_version;
 };

 struct mlx5_esw_indir_table {
 	struct mutex lock; /* protects table */
 	DECLARE_HASHTABLE(table, 8);
 };

 struct mlx5_esw_indir_table *
 mlx5_esw_indir_table_init(void)
 {
 	struct mlx5_esw_indir_table *indir = kvzalloc(sizeof(*indir), GFP_KERNEL);

 	if (!indir)
 		return ERR_PTR(-ENOMEM);

 	mutex_init(&indir->lock);
 	hash_init(indir->table);
 	return indir;
 }

 void
 mlx5_esw_indir_table_destroy(struct mlx5_esw_indir_table *indir)
 {
 	mutex_destroy(&indir->lock);
 	kvfree(indir);
 }

 bool
 mlx5_esw_indir_table_needed(struct mlx5_eswitch *esw,
 			    struct mlx5_flow_attr *attr,
 			    u16 vport_num,
 			    struct mlx5_core_dev *dest_mdev)
 {
 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;

 	/* Use indirect table for all IP traffic from UL to VF with vport
 	 * destination when source rewrite flag is set.
 	 */
 	return esw_attr->in_rep->vport == MLX5_VPORT_UPLINK &&
 		mlx5_eswitch_is_vf_vport(esw, vport_num) &&
 		esw->dev == dest_mdev &&
 		attr->ip_version &&
 		attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE;
 }

 u16
 mlx5_esw_indir_table_decap_vport(struct mlx5_flow_attr *attr)
 {
 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;

 	return esw_attr->rx_tun_attr ? esw_attr->rx_tun_attr->decap_vport : 0;
 }

 static struct mlx5_esw_indir_table_rule *
 mlx5_esw_indir_table_rule_lookup(struct mlx5_esw_indir_table_entry *e,
 				 struct mlx5_esw_flow_attr *attr)
 {
 	struct mlx5_esw_indir_table_rule *rule;

 	list_for_each_entry(rule, &e->recirc_rules, list)
 		if (rule->vni == attr->rx_tun_attr->vni &&
 		    !memcmp(&rule->dst_ip, &attr->rx_tun_attr->dst_ip,
 			    sizeof(attr->rx_tun_attr->dst_ip)))
 			goto found;
 	return NULL;

 found:
 	refcount_inc(&rule->refcnt);
 	return rule;
 }

 static int mlx5_esw_indir_table_rule_get(struct mlx5_eswitch *esw,
 					 struct mlx5_flow_attr *attr,
 					 struct mlx5_flow_spec *spec,
 					 struct mlx5_esw_indir_table_entry *e)
 {
 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
 	struct mlx5_fs_chains *chains = esw_chains(esw);
 	struct mlx5e_tc_mod_hdr_acts mod_acts = {};
 	struct mlx5_flow_destination dest = {};
 	struct mlx5_esw_indir_table_rule *rule;
 	struct mlx5_flow_act flow_act = {};
 	struct mlx5_flow_spec *rule_spec;
 	struct mlx5_flow_handle *handle;
 	int err = 0;
 	u32 data;

 	rule = mlx5_esw_indir_table_rule_lookup(e, esw_attr);
 	if (rule)
 		return 0;

 	if (e->recirc_cnt == MLX5_ESW_INDIR_TABLE_RECIRC_IDX_MAX)
 		return -EINVAL;

 	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
 	if (!rule_spec)
 		return -ENOMEM;

 	rule = kzalloc(sizeof(*rule), GFP_KERNEL);
 	if (!rule) {
 		err = -ENOMEM;
 		goto out;
 	}

 	rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS |
 					   MLX5_MATCH_MISC_PARAMETERS |
 					   MLX5_MATCH_MISC_PARAMETERS_2;
 	if (MLX5_CAP_FLOWTABLE_NIC_RX(esw->dev, ft_field_support.outer_ip_version)) {
 		MLX5_SET(fte_match_param, rule_spec->match_criteria,
 			 outer_headers.ip_version, 0xf);
 		MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.ip_version,
 			 attr->ip_version);
 	} else if (attr->ip_version) {
 		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
 				 outer_headers.ethertype);
 		MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.ethertype,
 			 (attr->ip_version == 4 ? ETH_P_IP : ETH_P_IPV6));
 	} else {
 		err = -EOPNOTSUPP;
 		goto err_ethertype;
 	}

 	if (attr->ip_version == 4) {
 		MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
 				 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 		MLX5_SET(fte_match_param, rule_spec->match_value,
 			 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
 			 ntohl(esw_attr->rx_tun_attr->dst_ip.v4));
 	} else if (attr->ip_version == 6) {
 		int len = sizeof(struct in6_addr);

 		memset(MLX5_ADDR_OF(fte_match_param, rule_spec->match_criteria,
 				    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 		       0xff, len);
 		memcpy(MLX5_ADDR_OF(fte_match_param, rule_spec->match_value,
 				    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 		       &esw_attr->rx_tun_attr->dst_ip.v6, len);
 	}

 	MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
 			 misc_parameters.vxlan_vni);
 	MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters.vxlan_vni,
 		 MLX5_GET(fte_match_param, spec->match_value, misc_parameters.vxlan_vni));

 	MLX5_SET(fte_match_param, rule_spec->match_criteria,
 		 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
 	MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters_2.metadata_reg_c_0,
 		 mlx5_eswitch_get_vport_metadata_for_match(esw_attr->in_mdev->priv.eswitch,
 							   MLX5_VPORT_UPLINK));

 	/* Modify flow source to recirculate packet */
 	data = mlx5_eswitch_get_vport_metadata_for_set(esw, esw_attr->rx_tun_attr->decap_vport);
 	err = mlx5e_tc_match_to_reg_set(esw->dev, &mod_acts, MLX5_FLOW_NAMESPACE_FDB,
 					VPORT_TO_REG, data);
 	if (err)
 		goto err_mod_hdr_regc0;

 	err = mlx5e_tc_match_to_reg_set(esw->dev, &mod_acts, MLX5_FLOW_NAMESPACE_FDB,
 					TUNNEL_TO_REG, ESW_TUN_SLOW_TABLE_GOTO_VPORT);
 	if (err)
 		goto err_mod_hdr_regc1;

 	flow_act.modify_hdr = mlx5_modify_header_alloc(esw->dev, MLX5_FLOW_NAMESPACE_FDB,
 						       mod_acts.num_actions, mod_acts.actions);
 	if (IS_ERR(flow_act.modify_hdr)) {
 		err = PTR_ERR(flow_act.modify_hdr);
 		goto err_mod_hdr_alloc;
 	}

 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
 	flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL | FLOW_ACT_NO_APPEND;
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	dest.ft = mlx5_chains_get_table(chains, 0, 1, 0);
 	if (IS_ERR(dest.ft)) {
 		err = PTR_ERR(dest.ft);
 		goto err_table;
 	}
 	handle = mlx5_add_flow_rules(e->ft, rule_spec, &flow_act, &dest, 1);
 	if (IS_ERR(handle)) {
 		err = PTR_ERR(handle);
 		goto err_handle;
 	}

 	dealloc_mod_hdr_actions(&mod_acts);
 	rule->handle = handle;
 	rule->vni = esw_attr->rx_tun_attr->vni;
 	rule->mh = flow_act.modify_hdr;
 	memcpy(&rule->dst_ip, &esw_attr->rx_tun_attr->dst_ip,
 	       sizeof(esw_attr->rx_tun_attr->dst_ip));
 	refcount_set(&rule->refcnt, 1);
 	list_add(&rule->list, &e->recirc_rules);
 	e->recirc_cnt++;
 	goto out;

 err_handle:
 	mlx5_chains_put_table(chains, 0, 1, 0);
 err_table:
 	mlx5_modify_header_dealloc(esw->dev, flow_act.modify_hdr);
 err_mod_hdr_alloc:
 err_mod_hdr_regc1:
 	dealloc_mod_hdr_actions(&mod_acts);
 err_mod_hdr_regc0:
 err_ethertype:
 	kfree(rule);
 out:
 	kvfree(rule_spec);
 	return err;
 }

 static void mlx5_esw_indir_table_rule_put(struct mlx5_eswitch *esw,
 					  struct mlx5_flow_attr *attr,
 					  struct mlx5_esw_indir_table_entry *e)
 {
 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
 	struct mlx5_fs_chains *chains = esw_chains(esw);
 	struct mlx5_esw_indir_table_rule *rule;

 	list_for_each_entry(rule, &e->recirc_rules, list)
 		if (rule->vni == esw_attr->rx_tun_attr->vni &&
 		    !memcmp(&rule->dst_ip, &esw_attr->rx_tun_attr->dst_ip,
 			    sizeof(esw_attr->rx_tun_attr->dst_ip)))
 			goto found;

 	return;

 found:
 	if (!refcount_dec_and_test(&rule->refcnt))
 		return;

 	mlx5_del_flow_rules(rule->handle);
 	mlx5_chains_put_table(chains, 0, 1, 0);
 	mlx5_modify_header_dealloc(esw->dev, rule->mh);
 	list_del(&rule->list);
 	kfree(rule);
 	e->recirc_cnt--;
 }

 static int mlx5_create_indir_recirc_group(struct mlx5_eswitch *esw,
 					  struct mlx5_flow_attr *attr,
 					  struct mlx5_flow_spec *spec,
 					  struct mlx5_esw_indir_table_entry *e)
 {
 	int err = 0, inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 	u32 *in, *match;

 	in = kvzalloc(inlen, GFP_KERNEL);
 	if (!in)
 		return -ENOMEM;

 	MLX5_SET(create_flow_group_in, in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS |
 		 MLX5_MATCH_MISC_PARAMETERS | MLX5_MATCH_MISC_PARAMETERS_2);
 	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);

 	if (MLX5_CAP_FLOWTABLE_NIC_RX(esw->dev, ft_field_support.outer_ip_version))
 		MLX5_SET(fte_match_param, match, outer_headers.ip_version, 0xf);
 	else
 		MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.ethertype);

 	if (attr->ip_version == 4) {
 		MLX5_SET_TO_ONES(fte_match_param, match,
 				 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
 	} else if (attr->ip_version == 6) {
 		memset(MLX5_ADDR_OF(fte_match_param, match,
 				    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
 		       0xff, sizeof(struct in6_addr));
 	} else {
 		err = -EOPNOTSUPP;
 		goto out;
 	}

 	MLX5_SET_TO_ONES(fte_match_param, match, misc_parameters.vxlan_vni);
 	MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0,
 		 mlx5_eswitch_get_vport_metadata_mask());
 	MLX5_SET(create_flow_group_in, in, start_flow_index, 0);
 	MLX5_SET(create_flow_group_in, in, end_flow_index, MLX5_ESW_INDIR_TABLE_RECIRC_IDX_MAX);
 	e->recirc_grp = mlx5_create_flow_group(e->ft, in);
 	if (IS_ERR(e->recirc_grp)) {
 		err = PTR_ERR(e->recirc_grp);
 		goto out;
 	}

 	INIT_LIST_HEAD(&e->recirc_rules);
 	e->recirc_cnt = 0;

 out:
 	kvfree(in);
 	return err;
 }

 static int mlx5_create_indir_fwd_group(struct mlx5_eswitch *esw,
 				       struct mlx5_esw_indir_table_entry *e)
 {
 	int err = 0, inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 	struct mlx5_flow_destination dest = {};
 	struct mlx5_flow_act flow_act = {};
 	struct mlx5_flow_spec *spec;
 	u32 *in;

 	in = kvzalloc(inlen, GFP_KERNEL);
 	if (!in)
 		return -ENOMEM;

 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 	if (!spec) {
 		kvfree(in);
 		return -ENOMEM;
 	}

 	/* Hold one entry */
 	MLX5_SET(create_flow_group_in, in, start_flow_index, MLX5_ESW_INDIR_TABLE_FWD_IDX);
 	MLX5_SET(create_flow_group_in, in, end_flow_index, MLX5_ESW_INDIR_TABLE_FWD_IDX);
 	e->fwd_grp = mlx5_create_flow_group(e->ft, in);
 	if (IS_ERR(e->fwd_grp)) {
 		err = PTR_ERR(e->fwd_grp);
 		goto err_out;
 	}

 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
 	dest.vport.num = e->vport;
 	dest.vport.vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id);
 	dest.vport.flags = MLX5_FLOW_DEST_VPORT_VHCA_ID;
 	e->fwd_rule = mlx5_add_flow_rules(e->ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR(e->fwd_rule)) {
 		mlx5_destroy_flow_group(e->fwd_grp);
 		err = PTR_ERR(e->fwd_rule);
 	}

 err_out:
 	kvfree(spec);
 	kvfree(in);
 	return err;
 }

 static struct mlx5_esw_indir_table_entry *
 mlx5_esw_indir_table_entry_create(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
 				  struct mlx5_flow_spec *spec, u16 vport, bool decap)
 {
 	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5_flow_namespace *root_ns;
 	struct mlx5_esw_indir_table_entry *e;
 	struct mlx5_flow_table *ft;
 	int err = 0;

 	root_ns = mlx5_get_flow_namespace(esw->dev, MLX5_FLOW_NAMESPACE_FDB);
 	if (!root_ns)
 		return ERR_PTR(-ENOENT);

 	e = kzalloc(sizeof(*e), GFP_KERNEL);
 	if (!e)
 		return ERR_PTR(-ENOMEM);

 	ft_attr.prio = FDB_TC_OFFLOAD;
 	ft_attr.max_fte = MLX5_ESW_INDIR_TABLE_SIZE;
 	ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
 	ft_attr.level = 1;

 	ft = mlx5_create_flow_table(root_ns, &ft_attr);
 	if (IS_ERR(ft)) {
 		err = PTR_ERR(ft);
 		goto tbl_err;
 	}
 	e->ft = ft;
 	e->vport = vport;
 	e->ip_version = attr->ip_version;
 	e->fwd_ref = !decap;

 	err = mlx5_create_indir_recirc_group(esw, attr, spec, e);
 	if (err)
 		goto recirc_grp_err;

 	if (decap) {
 		err = mlx5_esw_indir_table_rule_get(esw, attr, spec, e);
 		if (err)
 			goto recirc_rule_err;
 	}

 	err = mlx5_create_indir_fwd_group(esw, e);
 	if (err)
 		goto fwd_grp_err;

 	hash_add(esw->fdb_table.offloads.indir->table, &e->hlist,
 		 vport << 16 | attr->ip_version);

 	return e;

 fwd_grp_err:
 	if (decap)
 		mlx5_esw_indir_table_rule_put(esw, attr, e);
 recirc_rule_err:
 	mlx5_destroy_flow_group(e->recirc_grp);
 recirc_grp_err:
 	mlx5_destroy_flow_table(e->ft);
 tbl_err:
 	kfree(e);
 	return ERR_PTR(err);
 }

 static struct mlx5_esw_indir_table_entry *
 mlx5_esw_indir_table_entry_lookup(struct mlx5_eswitch *esw, u16 vport, u8 ip_version)
 {
 	struct mlx5_esw_indir_table_entry *e;
 	u32 key = vport << 16 | ip_version;

 	hash_for_each_possible(esw->fdb_table.offloads.indir->table, e, hlist, key)
 		if (e->vport == vport && e->ip_version == ip_version)
 			return e;

 	return NULL;
 }

 struct mlx5_flow_table *mlx5_esw_indir_table_get(struct mlx5_eswitch *esw,
 						 struct mlx5_flow_attr *attr,
 						 struct mlx5_flow_spec *spec,
 						 u16 vport, bool decap)
 {
 	struct mlx5_esw_indir_table_entry *e;
 	int err;

 	mutex_lock(&esw->fdb_table.offloads.indir->lock);
 	e = mlx5_esw_indir_table_entry_lookup(esw, vport, attr->ip_version);
 	if (e) {
 		if (!decap) {
 			e->fwd_ref++;
 		} else {
 			err = mlx5_esw_indir_table_rule_get(esw, attr, spec, e);
 			if (err)
 				goto out_err;
 		}
 	} else {
 		e = mlx5_esw_indir_table_entry_create(esw, attr, spec, vport, decap);
 		if (IS_ERR(e)) {
 			err = PTR_ERR(e);
 			esw_warn(esw->dev, "Failed to create indirection table, err %d.\n", err);
 			goto out_err;
 		}
 	}
 	mutex_unlock(&esw->fdb_table.offloads.indir->lock);
 	return e->ft;

 out_err:
 	mutex_unlock(&esw->fdb_table.offloads.indir->lock);
 	return ERR_PTR(err);
 }

 void mlx5_esw_indir_table_put(struct mlx5_eswitch *esw,
 			      struct mlx5_flow_attr *attr,
 			      u16 vport, bool decap)
 {
 	struct mlx5_esw_indir_table_entry *e;

 	mutex_lock(&esw->fdb_table.offloads.indir->lock);
 	e = mlx5_esw_indir_table_entry_lookup(esw, vport, attr->ip_version);
 	if (!e)
 		goto out;

 	if (!decap)
 		e->fwd_ref--;
 	else
 		mlx5_esw_indir_table_rule_put(esw, attr, e);

 	if (e->fwd_ref || e->recirc_cnt)
 		goto out;

 	hash_del(&e->hlist);
 	mlx5_destroy_flow_group(e->recirc_grp);
 	mlx5_del_flow_rules(e->fwd_rule);
 	mlx5_destroy_flow_group(e->fwd_grp);
 	mlx5_destroy_flow_table(e->ft);
 	kfree(e);
 out:
 	mutex_unlock(&esw->fdb_table.offloads.indir->lock);
 }
	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	/* Copyright (c) 2021 Mellanox Technologies. */

	#include <linux/etherdevice.h>
	#include <linux/idr.h>
	#include <linux/mlx5/driver.h>
	#include <linux/mlx5/mlx5_ifc.h>
	#include <linux/mlx5/vport.h>
	#include <linux/mlx5/fs.h>
	#include "mlx5_core.h"
	#include "eswitch.h"
	#include "en.h"
	#include "en_tc.h"
	#include "fs_core.h"
	#include "esw/indir_table.h"
	#include "lib/fs_chains.h"

	#define MLX5_ESW_INDIR_TABLE_SIZE 128
	#define MLX5_ESW_INDIR_TABLE_RECIRC_IDX_MAX (MLX5_ESW_INDIR_TABLE_SIZE - 2)
	#define MLX5_ESW_INDIR_TABLE_FWD_IDX (MLX5_ESW_INDIR_TABLE_SIZE - 1)

	struct mlx5_esw_indir_table_rule {
	struct list_head list;
	struct mlx5_flow_handle *handle;
	union {
	__be32 v4;
	struct in6_addr v6;
	} dst_ip;
	u32 vni;
	struct mlx5_modify_hdr *mh;
	refcount_t refcnt;
	};

	struct mlx5_esw_indir_table_entry {
	struct hlist_node hlist;
	struct mlx5_flow_table *ft;
	struct mlx5_flow_group *recirc_grp;
	struct mlx5_flow_group *fwd_grp;
	struct mlx5_flow_handle *fwd_rule;
	struct list_head recirc_rules;
	int recirc_cnt;
	int fwd_ref;

	u16 vport;
	u8 ip_version;
	};

	struct mlx5_esw_indir_table {
	struct mutex lock; /* protects table */
	DECLARE_HASHTABLE(table, 8);
	};

	struct mlx5_esw_indir_table *
	mlx5_esw_indir_table_init(void)
	{
	struct mlx5_esw_indir_table indir = kvzalloc(sizeof(indir), GFP_KERNEL);

	if (!indir)
	return ERR_PTR(-ENOMEM);

	mutex_init(&indir->lock);
	hash_init(indir->table);
	return indir;
	}

	void
	mlx5_esw_indir_table_destroy(struct mlx5_esw_indir_table *indir)
	{
	mutex_destroy(&indir->lock);
	kvfree(indir);
	}

	bool
	mlx5_esw_indir_table_needed(struct mlx5_eswitch *esw,
	struct mlx5_flow_attr *attr,
	u16 vport_num,
	struct mlx5_core_dev *dest_mdev)
	{
	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;

	/* Use indirect table for all IP traffic from UL to VF with vport
	* destination when source rewrite flag is set.
	*/
	return esw_attr->in_rep->vport == MLX5_VPORT_UPLINK &&
	mlx5_eswitch_is_vf_vport(esw, vport_num) &&
	esw->dev == dest_mdev &&
	attr->ip_version &&
	attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE;
	}

	u16
	mlx5_esw_indir_table_decap_vport(struct mlx5_flow_attr *attr)
	{
	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;

	return esw_attr->rx_tun_attr ? esw_attr->rx_tun_attr->decap_vport : 0;
	}

	static struct mlx5_esw_indir_table_rule *
	mlx5_esw_indir_table_rule_lookup(struct mlx5_esw_indir_table_entry *e,
	struct mlx5_esw_flow_attr *attr)
	{
	struct mlx5_esw_indir_table_rule *rule;

	list_for_each_entry(rule, &e->recirc_rules, list)
	if (rule->vni == attr->rx_tun_attr->vni &&
	!memcmp(&rule->dst_ip, &attr->rx_tun_attr->dst_ip,
	sizeof(attr->rx_tun_attr->dst_ip)))
	goto found;
	return NULL;

	found:
	refcount_inc(&rule->refcnt);
	return rule;
	}

	static int mlx5_esw_indir_table_rule_get(struct mlx5_eswitch *esw,
	struct mlx5_flow_attr *attr,
	struct mlx5_flow_spec *spec,
	struct mlx5_esw_indir_table_entry *e)
	{
	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
	struct mlx5_fs_chains *chains = esw_chains(esw);
	struct mlx5e_tc_mod_hdr_acts mod_acts = {};
	struct mlx5_flow_destination dest = {};
	struct mlx5_esw_indir_table_rule *rule;
	struct mlx5_flow_act flow_act = {};
	struct mlx5_flow_spec *rule_spec;
	struct mlx5_flow_handle *handle;
	int err = 0;
	u32 data;

	rule = mlx5_esw_indir_table_rule_lookup(e, esw_attr);
	if (rule)
	return 0;

	if (e->recirc_cnt == MLX5_ESW_INDIR_TABLE_RECIRC_IDX_MAX)
	return -EINVAL;

	rule_spec = kvzalloc(sizeof(*rule_spec), GFP_KERNEL);
	if (!rule_spec)
	return -ENOMEM;

	rule = kzalloc(sizeof(*rule), GFP_KERNEL);
	if (!rule) {
	err = -ENOMEM;
	goto out;
	}

	rule_spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS \|
	MLX5_MATCH_MISC_PARAMETERS \|
	MLX5_MATCH_MISC_PARAMETERS_2;
	if (MLX5_CAP_FLOWTABLE_NIC_RX(esw->dev, ft_field_support.outer_ip_version)) {
	MLX5_SET(fte_match_param, rule_spec->match_criteria,
	outer_headers.ip_version, 0xf);
	MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.ip_version,
	attr->ip_version);
	} else if (attr->ip_version) {
	MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
	outer_headers.ethertype);
	MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.ethertype,
	(attr->ip_version == 4 ? ETH_P_IP : ETH_P_IPV6));
	} else {
	err = -EOPNOTSUPP;
	goto err_ethertype;
	}

	if (attr->ip_version == 4) {
	MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
	outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
	MLX5_SET(fte_match_param, rule_spec->match_value,
	outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
	ntohl(esw_attr->rx_tun_attr->dst_ip.v4));
	} else if (attr->ip_version == 6) {
	int len = sizeof(struct in6_addr);

	memset(MLX5_ADDR_OF(fte_match_param, rule_spec->match_criteria,
	outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
	0xff, len);
	memcpy(MLX5_ADDR_OF(fte_match_param, rule_spec->match_value,
	outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
	&esw_attr->rx_tun_attr->dst_ip.v6, len);
	}

	MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria,
	misc_parameters.vxlan_vni);
	MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters.vxlan_vni,
	MLX5_GET(fte_match_param, spec->match_value, misc_parameters.vxlan_vni));

	MLX5_SET(fte_match_param, rule_spec->match_criteria,
	misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
	MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters_2.metadata_reg_c_0,
	mlx5_eswitch_get_vport_metadata_for_match(esw_attr->in_mdev->priv.eswitch,
	MLX5_VPORT_UPLINK));

	/* Modify flow source to recirculate packet */
	data = mlx5_eswitch_get_vport_metadata_for_set(esw, esw_attr->rx_tun_attr->decap_vport);
	err = mlx5e_tc_match_to_reg_set(esw->dev, &mod_acts, MLX5_FLOW_NAMESPACE_FDB,
	VPORT_TO_REG, data);
	if (err)
	goto err_mod_hdr_regc0;

	err = mlx5e_tc_match_to_reg_set(esw->dev, &mod_acts, MLX5_FLOW_NAMESPACE_FDB,
	TUNNEL_TO_REG, ESW_TUN_SLOW_TABLE_GOTO_VPORT);
	if (err)
	goto err_mod_hdr_regc1;

	flow_act.modify_hdr = mlx5_modify_header_alloc(esw->dev, MLX5_FLOW_NAMESPACE_FDB,
	mod_acts.num_actions, mod_acts.actions);
	if (IS_ERR(flow_act.modify_hdr)) {
	err = PTR_ERR(flow_act.modify_hdr);
	goto err_mod_hdr_alloc;
	}

	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST \| MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
	flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL \| FLOW_ACT_NO_APPEND;
	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	dest.ft = mlx5_chains_get_table(chains, 0, 1, 0);
	if (IS_ERR(dest.ft)) {
	err = PTR_ERR(dest.ft);
	goto err_table;
	}
	handle = mlx5_add_flow_rules(e->ft, rule_spec, &flow_act, &dest, 1);
	if (IS_ERR(handle)) {
	err = PTR_ERR(handle);
	goto err_handle;
	}

	dealloc_mod_hdr_actions(&mod_acts);
	rule->handle = handle;
	rule->vni = esw_attr->rx_tun_attr->vni;
	rule->mh = flow_act.modify_hdr;
	memcpy(&rule->dst_ip, &esw_attr->rx_tun_attr->dst_ip,
	sizeof(esw_attr->rx_tun_attr->dst_ip));
	refcount_set(&rule->refcnt, 1);
	list_add(&rule->list, &e->recirc_rules);
	e->recirc_cnt++;
	goto out;

	err_handle:
	mlx5_chains_put_table(chains, 0, 1, 0);
	err_table:
	mlx5_modify_header_dealloc(esw->dev, flow_act.modify_hdr);
	err_mod_hdr_alloc:
	err_mod_hdr_regc1:
	dealloc_mod_hdr_actions(&mod_acts);
	err_mod_hdr_regc0:
	err_ethertype:
	kfree(rule);
	out:
	kvfree(rule_spec);
	return err;
	}

	static void mlx5_esw_indir_table_rule_put(struct mlx5_eswitch *esw,
	struct mlx5_flow_attr *attr,
	struct mlx5_esw_indir_table_entry *e)
	{
	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
	struct mlx5_fs_chains *chains = esw_chains(esw);
	struct mlx5_esw_indir_table_rule *rule;

	list_for_each_entry(rule, &e->recirc_rules, list)
	if (rule->vni == esw_attr->rx_tun_attr->vni &&
	!memcmp(&rule->dst_ip, &esw_attr->rx_tun_attr->dst_ip,
	sizeof(esw_attr->rx_tun_attr->dst_ip)))
	goto found;

	return;

	found:
	if (!refcount_dec_and_test(&rule->refcnt))
	return;

	mlx5_del_flow_rules(rule->handle);
	mlx5_chains_put_table(chains, 0, 1, 0);
	mlx5_modify_header_dealloc(esw->dev, rule->mh);
	list_del(&rule->list);
	kfree(rule);
	e->recirc_cnt--;
	}

	static int mlx5_create_indir_recirc_group(struct mlx5_eswitch *esw,
	struct mlx5_flow_attr *attr,
	struct mlx5_flow_spec *spec,
	struct mlx5_esw_indir_table_entry *e)
	{
	int err = 0, inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	u32 in, match;

	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in)
	return -ENOMEM;

	MLX5_SET(create_flow_group_in, in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS \|
	MLX5_MATCH_MISC_PARAMETERS \| MLX5_MATCH_MISC_PARAMETERS_2);
	match = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);

	if (MLX5_CAP_FLOWTABLE_NIC_RX(esw->dev, ft_field_support.outer_ip_version))
	MLX5_SET(fte_match_param, match, outer_headers.ip_version, 0xf);
	else
	MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.ethertype);

	if (attr->ip_version == 4) {
	MLX5_SET_TO_ONES(fte_match_param, match,
	outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
	} else if (attr->ip_version == 6) {
	memset(MLX5_ADDR_OF(fte_match_param, match,
	outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
	0xff, sizeof(struct in6_addr));
	} else {
	err = -EOPNOTSUPP;
	goto out;
	}

	MLX5_SET_TO_ONES(fte_match_param, match, misc_parameters.vxlan_vni);
	MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0,
	mlx5_eswitch_get_vport_metadata_mask());
	MLX5_SET(create_flow_group_in, in, start_flow_index, 0);
	MLX5_SET(create_flow_group_in, in, end_flow_index, MLX5_ESW_INDIR_TABLE_RECIRC_IDX_MAX);
	e->recirc_grp = mlx5_create_flow_group(e->ft, in);
	if (IS_ERR(e->recirc_grp)) {
	err = PTR_ERR(e->recirc_grp);
	goto out;
	}

	INIT_LIST_HEAD(&e->recirc_rules);
	e->recirc_cnt = 0;

	out:
	kvfree(in);
	return err;
	}

	static int mlx5_create_indir_fwd_group(struct mlx5_eswitch *esw,
	struct mlx5_esw_indir_table_entry *e)
	{
	int err = 0, inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	struct mlx5_flow_destination dest = {};
	struct mlx5_flow_act flow_act = {};
	struct mlx5_flow_spec *spec;
	u32 *in;

	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in)
	return -ENOMEM;

	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	if (!spec) {
	kvfree(in);
	return -ENOMEM;
	}

	/* Hold one entry */
	MLX5_SET(create_flow_group_in, in, start_flow_index, MLX5_ESW_INDIR_TABLE_FWD_IDX);
	MLX5_SET(create_flow_group_in, in, end_flow_index, MLX5_ESW_INDIR_TABLE_FWD_IDX);
	e->fwd_grp = mlx5_create_flow_group(e->ft, in);
	if (IS_ERR(e->fwd_grp)) {
	err = PTR_ERR(e->fwd_grp);
	goto err_out;
	}

	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
	dest.vport.num = e->vport;
	dest.vport.vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id);
	dest.vport.flags = MLX5_FLOW_DEST_VPORT_VHCA_ID;
	e->fwd_rule = mlx5_add_flow_rules(e->ft, spec, &flow_act, &dest, 1);
	if (IS_ERR(e->fwd_rule)) {
	mlx5_destroy_flow_group(e->fwd_grp);
	err = PTR_ERR(e->fwd_rule);
	}

	err_out:
	kvfree(spec);
	kvfree(in);
	return err;
	}

	static struct mlx5_esw_indir_table_entry *
	mlx5_esw_indir_table_entry_create(struct mlx5_eswitch esw, struct mlx5_flow_attr attr,
	struct mlx5_flow_spec *spec, u16 vport, bool decap)
	{
	struct mlx5_flow_table_attr ft_attr = {};
	struct mlx5_flow_namespace *root_ns;
	struct mlx5_esw_indir_table_entry *e;
	struct mlx5_flow_table *ft;
	int err = 0;

	root_ns = mlx5_get_flow_namespace(esw->dev, MLX5_FLOW_NAMESPACE_FDB);
	if (!root_ns)
	return ERR_PTR(-ENOENT);

	e = kzalloc(sizeof(*e), GFP_KERNEL);
	if (!e)
	return ERR_PTR(-ENOMEM);

	ft_attr.prio = FDB_TC_OFFLOAD;
	ft_attr.max_fte = MLX5_ESW_INDIR_TABLE_SIZE;
	ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
	ft_attr.level = 1;

	ft = mlx5_create_flow_table(root_ns, &ft_attr);
	if (IS_ERR(ft)) {
	err = PTR_ERR(ft);
	goto tbl_err;
	}
	e->ft = ft;
	e->vport = vport;
	e->ip_version = attr->ip_version;
	e->fwd_ref = !decap;

	err = mlx5_create_indir_recirc_group(esw, attr, spec, e);
	if (err)
	goto recirc_grp_err;

	if (decap) {
	err = mlx5_esw_indir_table_rule_get(esw, attr, spec, e);
	if (err)
	goto recirc_rule_err;
	}

	err = mlx5_create_indir_fwd_group(esw, e);
	if (err)
	goto fwd_grp_err;

	hash_add(esw->fdb_table.offloads.indir->table, &e->hlist,
	vport << 16 \| attr->ip_version);

	return e;

	fwd_grp_err:
	if (decap)
	mlx5_esw_indir_table_rule_put(esw, attr, e);
	recirc_rule_err:
	mlx5_destroy_flow_group(e->recirc_grp);
	recirc_grp_err:
	mlx5_destroy_flow_table(e->ft);
	tbl_err:
	kfree(e);
	return ERR_PTR(err);
	}

	static struct mlx5_esw_indir_table_entry *
	mlx5_esw_indir_table_entry_lookup(struct mlx5_eswitch *esw, u16 vport, u8 ip_version)
	{
	struct mlx5_esw_indir_table_entry *e;
	u32 key = vport << 16 \| ip_version;

	hash_for_each_possible(esw->fdb_table.offloads.indir->table, e, hlist, key)
	if (e->vport == vport && e->ip_version == ip_version)
	return e;

	return NULL;
	}

	struct mlx5_flow_table mlx5_esw_indir_table_get(struct mlx5_eswitch esw,
	struct mlx5_flow_attr *attr,
	struct mlx5_flow_spec *spec,
	u16 vport, bool decap)
	{
	struct mlx5_esw_indir_table_entry *e;
	int err;

	mutex_lock(&esw->fdb_table.offloads.indir->lock);
	e = mlx5_esw_indir_table_entry_lookup(esw, vport, attr->ip_version);
	if (e) {
	if (!decap) {
	e->fwd_ref++;
	} else {
	err = mlx5_esw_indir_table_rule_get(esw, attr, spec, e);
	if (err)
	goto out_err;
	}
	} else {
	e = mlx5_esw_indir_table_entry_create(esw, attr, spec, vport, decap);
	if (IS_ERR(e)) {
	err = PTR_ERR(e);
	esw_warn(esw->dev, "Failed to create indirection table, err %d.\n", err);
	goto out_err;
	}
	}
	mutex_unlock(&esw->fdb_table.offloads.indir->lock);
	return e->ft;

	out_err:
	mutex_unlock(&esw->fdb_table.offloads.indir->lock);
	return ERR_PTR(err);
	}

	void mlx5_esw_indir_table_put(struct mlx5_eswitch *esw,
	struct mlx5_flow_attr *attr,
	u16 vport, bool decap)
	{
	struct mlx5_esw_indir_table_entry *e;

	mutex_lock(&esw->fdb_table.offloads.indir->lock);
	e = mlx5_esw_indir_table_entry_lookup(esw, vport, attr->ip_version);
	if (!e)
	goto out;

	if (!decap)
	e->fwd_ref--;
	else
	mlx5_esw_indir_table_rule_put(esw, attr, e);

	if (e->fwd_ref \|\| e->recirc_cnt)
	goto out;

	hash_del(&e->hlist);
	mlx5_destroy_flow_group(e->recirc_grp);
	mlx5_del_flow_rules(e->fwd_rule);
	mlx5_destroy_flow_group(e->fwd_grp);
	mlx5_destroy_flow_table(e->ft);
	kfree(e);
	out:
	mutex_unlock(&esw->fdb_table.offloads.indir->lock);
	}