drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
 // Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES.

 #include <linux/ip.h>
 #include <linux/ipv6.h>
 #include <linux/tcp.h>
 #include <linux/mlx5/fs.h>
 #include <linux/mlx5/driver.h>
 #include "mlx5_core.h"
 #include "lib/fs_ttc.h"

 #define MLX5_TTC_NUM_GROUPS	3
 #define MLX5_TTC_GROUP1_SIZE	(BIT(3) + MLX5_NUM_TUNNEL_TT)
 #define MLX5_TTC_GROUP2_SIZE	 BIT(1)
 #define MLX5_TTC_GROUP3_SIZE	 BIT(0)
 #define MLX5_TTC_TABLE_SIZE	(MLX5_TTC_GROUP1_SIZE +\
 				 MLX5_TTC_GROUP2_SIZE +\
 				 MLX5_TTC_GROUP3_SIZE)

 #define MLX5_INNER_TTC_NUM_GROUPS	3
 #define MLX5_INNER_TTC_GROUP1_SIZE	BIT(3)
 #define MLX5_INNER_TTC_GROUP2_SIZE	BIT(1)
 #define MLX5_INNER_TTC_GROUP3_SIZE	BIT(0)
 #define MLX5_INNER_TTC_TABLE_SIZE	(MLX5_INNER_TTC_GROUP1_SIZE +\
 					 MLX5_INNER_TTC_GROUP2_SIZE +\
 					 MLX5_INNER_TTC_GROUP3_SIZE)

 /* L3/L4 traffic type classifier */
 struct mlx5_ttc_table {
 	int num_groups;
 	struct mlx5_flow_table *t;
 	struct mlx5_flow_group **g;
 	struct mlx5_ttc_rule rules[MLX5_NUM_TT];
 	struct mlx5_flow_handle *tunnel_rules[MLX5_NUM_TUNNEL_TT];
 };

 struct mlx5_flow_table *mlx5_get_ttc_flow_table(struct mlx5_ttc_table *ttc)
 {
 	return ttc->t;
 }

 static void mlx5_cleanup_ttc_rules(struct mlx5_ttc_table *ttc)
 {
 	int i;

 	for (i = 0; i < MLX5_NUM_TT; i++) {
 		if (!IS_ERR_OR_NULL(ttc->rules[i].rule)) {
 			mlx5_del_flow_rules(ttc->rules[i].rule);
 			ttc->rules[i].rule = NULL;
 		}
 	}

 	for (i = 0; i < MLX5_NUM_TUNNEL_TT; i++) {
 		if (!IS_ERR_OR_NULL(ttc->tunnel_rules[i])) {
 			mlx5_del_flow_rules(ttc->tunnel_rules[i]);
 			ttc->tunnel_rules[i] = NULL;
 		}
 	}
 }

 struct mlx5_etype_proto {
 	u16 etype;
 	u8 proto;
 };

 static struct mlx5_etype_proto ttc_rules[] = {
 	[MLX5_TT_IPV4_TCP] = {
 		.etype = ETH_P_IP,
 		.proto = IPPROTO_TCP,
 	},
 	[MLX5_TT_IPV6_TCP] = {
 		.etype = ETH_P_IPV6,
 		.proto = IPPROTO_TCP,
 	},
 	[MLX5_TT_IPV4_UDP] = {
 		.etype = ETH_P_IP,
 		.proto = IPPROTO_UDP,
 	},
 	[MLX5_TT_IPV6_UDP] = {
 		.etype = ETH_P_IPV6,
 		.proto = IPPROTO_UDP,
 	},
 	[MLX5_TT_IPV4_IPSEC_AH] = {
 		.etype = ETH_P_IP,
 		.proto = IPPROTO_AH,
 	},
 	[MLX5_TT_IPV6_IPSEC_AH] = {
 		.etype = ETH_P_IPV6,
 		.proto = IPPROTO_AH,
 	},
 	[MLX5_TT_IPV4_IPSEC_ESP] = {
 		.etype = ETH_P_IP,
 		.proto = IPPROTO_ESP,
 	},
 	[MLX5_TT_IPV6_IPSEC_ESP] = {
 		.etype = ETH_P_IPV6,
 		.proto = IPPROTO_ESP,
 	},
 	[MLX5_TT_IPV4] = {
 		.etype = ETH_P_IP,
 		.proto = 0,
 	},
 	[MLX5_TT_IPV6] = {
 		.etype = ETH_P_IPV6,
 		.proto = 0,
 	},
 	[MLX5_TT_ANY] = {
 		.etype = 0,
 		.proto = 0,
 	},
 };

 static struct mlx5_etype_proto ttc_tunnel_rules[] = {
 	[MLX5_TT_IPV4_GRE] = {
 		.etype = ETH_P_IP,
 		.proto = IPPROTO_GRE,
 	},
 	[MLX5_TT_IPV6_GRE] = {
 		.etype = ETH_P_IPV6,
 		.proto = IPPROTO_GRE,
 	},
 	[MLX5_TT_IPV4_IPIP] = {
 		.etype = ETH_P_IP,
 		.proto = IPPROTO_IPIP,
 	},
 	[MLX5_TT_IPV6_IPIP] = {
 		.etype = ETH_P_IPV6,
 		.proto = IPPROTO_IPIP,
 	},
 	[MLX5_TT_IPV4_IPV6] = {
 		.etype = ETH_P_IP,
 		.proto = IPPROTO_IPV6,
 	},
 	[MLX5_TT_IPV6_IPV6] = {
 		.etype = ETH_P_IPV6,
 		.proto = IPPROTO_IPV6,
 	},

 };

 u8 mlx5_get_proto_by_tunnel_type(enum mlx5_tunnel_types tt)
 {
 	return ttc_tunnel_rules[tt].proto;
 }

 static bool mlx5_tunnel_proto_supported_rx(struct mlx5_core_dev *mdev,
 					   u8 proto_type)
 {
 	switch (proto_type) {
 	case IPPROTO_GRE:
 		return MLX5_CAP_ETH(mdev, tunnel_stateless_gre);
 	case IPPROTO_IPIP:
 	case IPPROTO_IPV6:
 		return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) ||
 			MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_rx));
 	default:
 		return false;
 	}
 }

 static bool mlx5_tunnel_any_rx_proto_supported(struct mlx5_core_dev *mdev)
 {
 	int tt;

 	for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
 		if (mlx5_tunnel_proto_supported_rx(mdev,
 						   ttc_tunnel_rules[tt].proto))
 			return true;
 	}
 	return false;
 }

 bool mlx5_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev)
 {
 	return (mlx5_tunnel_any_rx_proto_supported(mdev) &&
 		MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
 					  ft_field_support.inner_ip_version));
 }

 static u8 mlx5_etype_to_ipv(u16 ethertype)
 {
 	if (ethertype == ETH_P_IP)
 		return 4;

 	if (ethertype == ETH_P_IPV6)
 		return 6;

 	return 0;
 }

 static struct mlx5_flow_handle *
 mlx5_generate_ttc_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft,
 		       struct mlx5_flow_destination *dest, u16 etype, u8 proto)
 {
 	int match_ipv_outer =
 		MLX5_CAP_FLOWTABLE_NIC_RX(dev,
 					  ft_field_support.outer_ip_version);
 	MLX5_DECLARE_FLOW_ACT(flow_act);
 	struct mlx5_flow_handle *rule;
 	struct mlx5_flow_spec *spec;
 	int err = 0;
 	u8 ipv;

 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 	if (!spec)
 		return ERR_PTR(-ENOMEM);

 	if (proto) {
 		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
 		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, proto);
 	}

 	ipv = mlx5_etype_to_ipv(etype);
 	if (match_ipv_outer && ipv) {
 		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
 		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ipv);
 	} else if (etype) {
 		spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
 		MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, etype);
 	}

 	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
 		mlx5_core_err(dev, "%s: add rule failed\n", __func__);
 	}

 	kvfree(spec);
 	return err ? ERR_PTR(err) : rule;
 }

 static int mlx5_generate_ttc_table_rules(struct mlx5_core_dev *dev,
 					 struct ttc_params *params,
 					 struct mlx5_ttc_table *ttc)
 {
 	struct mlx5_flow_handle **trules;
 	struct mlx5_ttc_rule *rules;
 	struct mlx5_flow_table *ft;
 	int tt;
 	int err;

 	ft = ttc->t;
 	rules = ttc->rules;
 	for (tt = 0; tt < MLX5_NUM_TT; tt++) {
 		struct mlx5_ttc_rule *rule = &rules[tt];

 		if (test_bit(tt, params->ignore_dests))
 			continue;
 		rule->rule = mlx5_generate_ttc_rule(dev, ft, &params->dests[tt],
 						    ttc_rules[tt].etype,
 						    ttc_rules[tt].proto);
 		if (IS_ERR(rule->rule)) {
 			err = PTR_ERR(rule->rule);
 			rule->rule = NULL;
 			goto del_rules;
 		}
 		rule->default_dest = params->dests[tt];
 	}

 	if (!params->inner_ttc || !mlx5_tunnel_inner_ft_supported(dev))
 		return 0;

 	trules    = ttc->tunnel_rules;
 	for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
 		if (!mlx5_tunnel_proto_supported_rx(dev,
 						    ttc_tunnel_rules[tt].proto))
 			continue;
 		if (test_bit(tt, params->ignore_tunnel_dests))
 			continue;
 		trules[tt] = mlx5_generate_ttc_rule(dev, ft,
 						    &params->tunnel_dests[tt],
 						    ttc_tunnel_rules[tt].etype,
 						    ttc_tunnel_rules[tt].proto);
 		if (IS_ERR(trules[tt])) {
 			err = PTR_ERR(trules[tt]);
 			trules[tt] = NULL;
 			goto del_rules;
 		}
 	}

 	return 0;

 del_rules:
 	mlx5_cleanup_ttc_rules(ttc);
 	return err;
 }

 static int mlx5_create_ttc_table_groups(struct mlx5_ttc_table *ttc,
 					bool use_ipv)
 {
 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 	int ix = 0;
 	u32 *in;
 	int err;
 	u8 *mc;

 	ttc->g = kcalloc(MLX5_TTC_NUM_GROUPS, sizeof(*ttc->g), GFP_KERNEL);
 	if (!ttc->g)
 		return -ENOMEM;
 	in = kvzalloc(inlen, GFP_KERNEL);
 	if (!in) {
 		kfree(ttc->g);
 		ttc->g = NULL;
 		return -ENOMEM;
 	}

 	/* L4 Group */
 	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
 	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
 	if (use_ipv)
 		MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_version);
 	else
 		MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
 	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
 	MLX5_SET_CFG(in, start_flow_index, ix);
 	ix += MLX5_TTC_GROUP1_SIZE;
 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
 	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
 	if (IS_ERR(ttc->g[ttc->num_groups]))
 		goto err;
 	ttc->num_groups++;

 	/* L3 Group */
 	MLX5_SET(fte_match_param, mc, outer_headers.ip_protocol, 0);
 	MLX5_SET_CFG(in, start_flow_index, ix);
 	ix += MLX5_TTC_GROUP2_SIZE;
 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
 	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
 	if (IS_ERR(ttc->g[ttc->num_groups]))
 		goto err;
 	ttc->num_groups++;

 	/* Any Group */
 	memset(in, 0, inlen);
 	MLX5_SET_CFG(in, start_flow_index, ix);
 	ix += MLX5_TTC_GROUP3_SIZE;
 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
 	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
 	if (IS_ERR(ttc->g[ttc->num_groups]))
 		goto err;
 	ttc->num_groups++;

 	kvfree(in);
 	return 0;

 err:
 	err = PTR_ERR(ttc->g[ttc->num_groups]);
 	ttc->g[ttc->num_groups] = NULL;
 	kvfree(in);

 	return err;
 }

 static struct mlx5_flow_handle *
 mlx5_generate_inner_ttc_rule(struct mlx5_core_dev *dev,
 			     struct mlx5_flow_table *ft,
 			     struct mlx5_flow_destination *dest,
 			     u16 etype, u8 proto)
 {
 	MLX5_DECLARE_FLOW_ACT(flow_act);
 	struct mlx5_flow_handle *rule;
 	struct mlx5_flow_spec *spec;
 	int err = 0;
 	u8 ipv;

 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 	if (!spec)
 		return ERR_PTR(-ENOMEM);

 	ipv = mlx5_etype_to_ipv(etype);
 	if (etype && ipv) {
 		spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
 		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, inner_headers.ip_version);
 		MLX5_SET(fte_match_param, spec->match_value, inner_headers.ip_version, ipv);
 	}

 	if (proto) {
 		spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
 		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, inner_headers.ip_protocol);
 		MLX5_SET(fte_match_param, spec->match_value, inner_headers.ip_protocol, proto);
 	}

 	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
 		mlx5_core_err(dev, "%s: add inner TTC rule failed\n", __func__);
 	}

 	kvfree(spec);
 	return err ? ERR_PTR(err) : rule;
 }

 static int mlx5_generate_inner_ttc_table_rules(struct mlx5_core_dev *dev,
 					       struct ttc_params *params,
 					       struct mlx5_ttc_table *ttc)
 {
 	struct mlx5_ttc_rule *rules;
 	struct mlx5_flow_table *ft;
 	int err;
 	int tt;

 	ft = ttc->t;
 	rules = ttc->rules;

 	for (tt = 0; tt < MLX5_NUM_TT; tt++) {
 		struct mlx5_ttc_rule *rule = &rules[tt];

 		rule->rule = mlx5_generate_inner_ttc_rule(dev, ft,
 							  &params->dests[tt],
 							  ttc_rules[tt].etype,
 							  ttc_rules[tt].proto);
 		if (IS_ERR(rule->rule)) {
 			err = PTR_ERR(rule->rule);
 			rule->rule = NULL;
 			goto del_rules;
 		}
 		rule->default_dest = params->dests[tt];
 	}

 	return 0;

 del_rules:

 	mlx5_cleanup_ttc_rules(ttc);
 	return err;
 }

 static int mlx5_create_inner_ttc_table_groups(struct mlx5_ttc_table *ttc)
 {
 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 	int ix = 0;
 	u32 *in;
 	int err;
 	u8 *mc;

 	ttc->g = kcalloc(MLX5_INNER_TTC_NUM_GROUPS, sizeof(*ttc->g),
 			 GFP_KERNEL);
 	if (!ttc->g)
 		return -ENOMEM;
 	in = kvzalloc(inlen, GFP_KERNEL);
 	if (!in) {
 		kfree(ttc->g);
 		ttc->g = NULL;
 		return -ENOMEM;
 	}

 	/* L4 Group */
 	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
 	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol);
 	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_version);
 	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
 	MLX5_SET_CFG(in, start_flow_index, ix);
 	ix += MLX5_INNER_TTC_GROUP1_SIZE;
 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
 	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
 	if (IS_ERR(ttc->g[ttc->num_groups]))
 		goto err;
 	ttc->num_groups++;

 	/* L3 Group */
 	MLX5_SET(fte_match_param, mc, inner_headers.ip_protocol, 0);
 	MLX5_SET_CFG(in, start_flow_index, ix);
 	ix += MLX5_INNER_TTC_GROUP2_SIZE;
 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
 	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
 	if (IS_ERR(ttc->g[ttc->num_groups]))
 		goto err;
 	ttc->num_groups++;

 	/* Any Group */
 	memset(in, 0, inlen);
 	MLX5_SET_CFG(in, start_flow_index, ix);
 	ix += MLX5_INNER_TTC_GROUP3_SIZE;
 	MLX5_SET_CFG(in, end_flow_index, ix - 1);
 	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
 	if (IS_ERR(ttc->g[ttc->num_groups]))
 		goto err;
 	ttc->num_groups++;

 	kvfree(in);
 	return 0;

 err:
 	err = PTR_ERR(ttc->g[ttc->num_groups]);
 	ttc->g[ttc->num_groups] = NULL;
 	kvfree(in);

 	return err;
 }

 struct mlx5_ttc_table *mlx5_create_inner_ttc_table(struct mlx5_core_dev *dev,
 						   struct ttc_params *params)
 {
 	struct mlx5_ttc_table *ttc;
 	int err;

 	ttc = kvzalloc(sizeof(*ttc), GFP_KERNEL);
 	if (!ttc)
 		return ERR_PTR(-ENOMEM);

 	WARN_ON_ONCE(params->ft_attr.max_fte);
 	params->ft_attr.max_fte = MLX5_INNER_TTC_TABLE_SIZE;
 	ttc->t = mlx5_create_flow_table(params->ns, &params->ft_attr);
 	if (IS_ERR(ttc->t)) {
 		err = PTR_ERR(ttc->t);
 		kvfree(ttc);
 		return ERR_PTR(err);
 	}

 	err = mlx5_create_inner_ttc_table_groups(ttc);
 	if (err)
 		goto destroy_ft;

 	err = mlx5_generate_inner_ttc_table_rules(dev, params, ttc);
 	if (err)
 		goto destroy_ft;

 	return ttc;

 destroy_ft:
 	mlx5_destroy_ttc_table(ttc);
 	return ERR_PTR(err);
 }

 void mlx5_destroy_ttc_table(struct mlx5_ttc_table *ttc)
 {
 	int i;

 	mlx5_cleanup_ttc_rules(ttc);
 	for (i = ttc->num_groups - 1; i >= 0; i--) {
 		if (!IS_ERR_OR_NULL(ttc->g[i]))
 			mlx5_destroy_flow_group(ttc->g[i]);
 		ttc->g[i] = NULL;
 	}

 	kfree(ttc->g);
 	mlx5_destroy_flow_table(ttc->t);
 	kvfree(ttc);
 }

 struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev,
 					     struct ttc_params *params)
 {
 	bool match_ipv_outer =
 		MLX5_CAP_FLOWTABLE_NIC_RX(dev,
 					  ft_field_support.outer_ip_version);
 	struct mlx5_ttc_table *ttc;
 	int err;

 	ttc = kvzalloc(sizeof(*ttc), GFP_KERNEL);
 	if (!ttc)
 		return ERR_PTR(-ENOMEM);

 	WARN_ON_ONCE(params->ft_attr.max_fte);
 	params->ft_attr.max_fte = MLX5_TTC_TABLE_SIZE;
 	ttc->t = mlx5_create_flow_table(params->ns, &params->ft_attr);
 	if (IS_ERR(ttc->t)) {
 		err = PTR_ERR(ttc->t);
 		kvfree(ttc);
 		return ERR_PTR(err);
 	}

 	err = mlx5_create_ttc_table_groups(ttc, match_ipv_outer);
 	if (err)
 		goto destroy_ft;

 	err = mlx5_generate_ttc_table_rules(dev, params, ttc);
 	if (err)
 		goto destroy_ft;

 	return ttc;

 destroy_ft:
 	mlx5_destroy_ttc_table(ttc);
 	return ERR_PTR(err);
 }

 int mlx5_ttc_fwd_dest(struct mlx5_ttc_table *ttc, enum mlx5_traffic_types type,
 		      struct mlx5_flow_destination *new_dest)
 {
 	return mlx5_modify_rule_destination(ttc->rules[type].rule, new_dest,
 					    NULL);
 }

 struct mlx5_flow_destination
 mlx5_ttc_get_default_dest(struct mlx5_ttc_table *ttc,
 			  enum mlx5_traffic_types type)
 {
 	struct mlx5_flow_destination *dest = &ttc->rules[type].default_dest;

 	WARN_ONCE(dest->type != MLX5_FLOW_DESTINATION_TYPE_TIR,
 		  "TTC[%d] default dest is not setup yet", type);

 	return *dest;
 }

 int mlx5_ttc_fwd_default_dest(struct mlx5_ttc_table *ttc,
 			      enum mlx5_traffic_types type)
 {
 	struct mlx5_flow_destination dest = mlx5_ttc_get_default_dest(ttc, type);

 	return mlx5_ttc_fwd_dest(ttc, type, &dest);
 }
	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	// Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES.

	#include <linux/ip.h>
	#include <linux/ipv6.h>
	#include <linux/tcp.h>
	#include <linux/mlx5/fs.h>
	#include <linux/mlx5/driver.h>
	#include "mlx5_core.h"
	#include "lib/fs_ttc.h"

	#define MLX5_TTC_NUM_GROUPS 3
	#define MLX5_TTC_GROUP1_SIZE (BIT(3) + MLX5_NUM_TUNNEL_TT)
	#define MLX5_TTC_GROUP2_SIZE BIT(1)
	#define MLX5_TTC_GROUP3_SIZE BIT(0)
	#define MLX5_TTC_TABLE_SIZE (MLX5_TTC_GROUP1_SIZE +\
	MLX5_TTC_GROUP2_SIZE +\
	MLX5_TTC_GROUP3_SIZE)

	#define MLX5_INNER_TTC_NUM_GROUPS 3
	#define MLX5_INNER_TTC_GROUP1_SIZE BIT(3)
	#define MLX5_INNER_TTC_GROUP2_SIZE BIT(1)
	#define MLX5_INNER_TTC_GROUP3_SIZE BIT(0)
	#define MLX5_INNER_TTC_TABLE_SIZE (MLX5_INNER_TTC_GROUP1_SIZE +\
	MLX5_INNER_TTC_GROUP2_SIZE +\
	MLX5_INNER_TTC_GROUP3_SIZE)

	/* L3/L4 traffic type classifier */
	struct mlx5_ttc_table {
	int num_groups;
	struct mlx5_flow_table *t;
	struct mlx5_flow_group **g;
	struct mlx5_ttc_rule rules[MLX5_NUM_TT];
	struct mlx5_flow_handle *tunnel_rules[MLX5_NUM_TUNNEL_TT];
	};

	struct mlx5_flow_table mlx5_get_ttc_flow_table(struct mlx5_ttc_table ttc)
	{
	return ttc->t;
	}

	static void mlx5_cleanup_ttc_rules(struct mlx5_ttc_table *ttc)
	{
	int i;

	for (i = 0; i < MLX5_NUM_TT; i++) {
	if (!IS_ERR_OR_NULL(ttc->rules[i].rule)) {
	mlx5_del_flow_rules(ttc->rules[i].rule);
	ttc->rules[i].rule = NULL;
	}
	}

	for (i = 0; i < MLX5_NUM_TUNNEL_TT; i++) {
	if (!IS_ERR_OR_NULL(ttc->tunnel_rules[i])) {
	mlx5_del_flow_rules(ttc->tunnel_rules[i]);
	ttc->tunnel_rules[i] = NULL;
	}
	}
	}

	struct mlx5_etype_proto {
	u16 etype;
	u8 proto;
	};

	static struct mlx5_etype_proto ttc_rules[] = {
	[MLX5_TT_IPV4_TCP] = {
	.etype = ETH_P_IP,
	.proto = IPPROTO_TCP,
	},
	[MLX5_TT_IPV6_TCP] = {
	.etype = ETH_P_IPV6,
	.proto = IPPROTO_TCP,
	},
	[MLX5_TT_IPV4_UDP] = {
	.etype = ETH_P_IP,
	.proto = IPPROTO_UDP,
	},
	[MLX5_TT_IPV6_UDP] = {
	.etype = ETH_P_IPV6,
	.proto = IPPROTO_UDP,
	},
	[MLX5_TT_IPV4_IPSEC_AH] = {
	.etype = ETH_P_IP,
	.proto = IPPROTO_AH,
	},
	[MLX5_TT_IPV6_IPSEC_AH] = {
	.etype = ETH_P_IPV6,
	.proto = IPPROTO_AH,
	},
	[MLX5_TT_IPV4_IPSEC_ESP] = {
	.etype = ETH_P_IP,
	.proto = IPPROTO_ESP,
	},
	[MLX5_TT_IPV6_IPSEC_ESP] = {
	.etype = ETH_P_IPV6,
	.proto = IPPROTO_ESP,
	},
	[MLX5_TT_IPV4] = {
	.etype = ETH_P_IP,
	.proto = 0,
	},
	[MLX5_TT_IPV6] = {
	.etype = ETH_P_IPV6,
	.proto = 0,
	},
	[MLX5_TT_ANY] = {
	.etype = 0,
	.proto = 0,
	},
	};

	static struct mlx5_etype_proto ttc_tunnel_rules[] = {
	[MLX5_TT_IPV4_GRE] = {
	.etype = ETH_P_IP,
	.proto = IPPROTO_GRE,
	},
	[MLX5_TT_IPV6_GRE] = {
	.etype = ETH_P_IPV6,
	.proto = IPPROTO_GRE,
	},
	[MLX5_TT_IPV4_IPIP] = {
	.etype = ETH_P_IP,
	.proto = IPPROTO_IPIP,
	},
	[MLX5_TT_IPV6_IPIP] = {
	.etype = ETH_P_IPV6,
	.proto = IPPROTO_IPIP,
	},
	[MLX5_TT_IPV4_IPV6] = {
	.etype = ETH_P_IP,
	.proto = IPPROTO_IPV6,
	},
	[MLX5_TT_IPV6_IPV6] = {
	.etype = ETH_P_IPV6,
	.proto = IPPROTO_IPV6,
	},

	};

	u8 mlx5_get_proto_by_tunnel_type(enum mlx5_tunnel_types tt)
	{
	return ttc_tunnel_rules[tt].proto;
	}

	static bool mlx5_tunnel_proto_supported_rx(struct mlx5_core_dev *mdev,
	u8 proto_type)
	{
	switch (proto_type) {
	case IPPROTO_GRE:
	return MLX5_CAP_ETH(mdev, tunnel_stateless_gre);
	case IPPROTO_IPIP:
	case IPPROTO_IPV6:
	return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) \|\|
	MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_rx));
	default:
	return false;
	}
	}

	static bool mlx5_tunnel_any_rx_proto_supported(struct mlx5_core_dev *mdev)
	{
	int tt;

	for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
	if (mlx5_tunnel_proto_supported_rx(mdev,
	ttc_tunnel_rules[tt].proto))
	return true;
	}
	return false;
	}

	bool mlx5_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev)
	{
	return (mlx5_tunnel_any_rx_proto_supported(mdev) &&
	MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
	ft_field_support.inner_ip_version));
	}

	static u8 mlx5_etype_to_ipv(u16 ethertype)
	{
	if (ethertype == ETH_P_IP)
	return 4;

	if (ethertype == ETH_P_IPV6)
	return 6;

	return 0;
	}

	static struct mlx5_flow_handle *
	mlx5_generate_ttc_rule(struct mlx5_core_dev dev, struct mlx5_flow_table ft,
	struct mlx5_flow_destination *dest, u16 etype, u8 proto)
	{
	int match_ipv_outer =
	MLX5_CAP_FLOWTABLE_NIC_RX(dev,
	ft_field_support.outer_ip_version);
	MLX5_DECLARE_FLOW_ACT(flow_act);
	struct mlx5_flow_handle *rule;
	struct mlx5_flow_spec *spec;
	int err = 0;
	u8 ipv;

	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	if (!spec)
	return ERR_PTR(-ENOMEM);

	if (proto) {
	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, proto);
	}

	ipv = mlx5_etype_to_ipv(etype);
	if (match_ipv_outer && ipv) {
	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ipv);
	} else if (etype) {
	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, etype);
	}

	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1);
	if (IS_ERR(rule)) {
	err = PTR_ERR(rule);
	mlx5_core_err(dev, "%s: add rule failed\n", __func__);
	}

	kvfree(spec);
	return err ? ERR_PTR(err) : rule;
	}

	static int mlx5_generate_ttc_table_rules(struct mlx5_core_dev *dev,
	struct ttc_params *params,
	struct mlx5_ttc_table *ttc)
	{
	struct mlx5_flow_handle **trules;
	struct mlx5_ttc_rule *rules;
	struct mlx5_flow_table *ft;
	int tt;
	int err;

	ft = ttc->t;
	rules = ttc->rules;
	for (tt = 0; tt < MLX5_NUM_TT; tt++) {
	struct mlx5_ttc_rule *rule = &rules[tt];

	if (test_bit(tt, params->ignore_dests))
	continue;
	rule->rule = mlx5_generate_ttc_rule(dev, ft, &params->dests[tt],
	ttc_rules[tt].etype,
	ttc_rules[tt].proto);
	if (IS_ERR(rule->rule)) {
	err = PTR_ERR(rule->rule);
	rule->rule = NULL;
	goto del_rules;
	}
	rule->default_dest = params->dests[tt];
	}

	if (!params->inner_ttc \|\| !mlx5_tunnel_inner_ft_supported(dev))
	return 0;

	trules = ttc->tunnel_rules;
	for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
	if (!mlx5_tunnel_proto_supported_rx(dev,
	ttc_tunnel_rules[tt].proto))
	continue;
	if (test_bit(tt, params->ignore_tunnel_dests))
	continue;
	trules[tt] = mlx5_generate_ttc_rule(dev, ft,
	&params->tunnel_dests[tt],
	ttc_tunnel_rules[tt].etype,
	ttc_tunnel_rules[tt].proto);
	if (IS_ERR(trules[tt])) {
	err = PTR_ERR(trules[tt]);
	trules[tt] = NULL;
	goto del_rules;
	}
	}

	return 0;

	del_rules:
	mlx5_cleanup_ttc_rules(ttc);
	return err;
	}

	static int mlx5_create_ttc_table_groups(struct mlx5_ttc_table *ttc,
	bool use_ipv)
	{
	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	int ix = 0;
	u32 *in;
	int err;
	u8 *mc;

	ttc->g = kcalloc(MLX5_TTC_NUM_GROUPS, sizeof(*ttc->g), GFP_KERNEL);
	if (!ttc->g)
	return -ENOMEM;
	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in) {
	kfree(ttc->g);
	ttc->g = NULL;
	return -ENOMEM;
	}

	/* L4 Group */
	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
	if (use_ipv)
	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_version);
	else
	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_TTC_GROUP1_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
	goto err;
	ttc->num_groups++;

	/* L3 Group */
	MLX5_SET(fte_match_param, mc, outer_headers.ip_protocol, 0);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_TTC_GROUP2_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
	goto err;
	ttc->num_groups++;

	/* Any Group */
	memset(in, 0, inlen);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_TTC_GROUP3_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
	goto err;
	ttc->num_groups++;

	kvfree(in);
	return 0;

	err:
	err = PTR_ERR(ttc->g[ttc->num_groups]);
	ttc->g[ttc->num_groups] = NULL;
	kvfree(in);

	return err;
	}

	static struct mlx5_flow_handle *
	mlx5_generate_inner_ttc_rule(struct mlx5_core_dev *dev,
	struct mlx5_flow_table *ft,
	struct mlx5_flow_destination *dest,
	u16 etype, u8 proto)
	{
	MLX5_DECLARE_FLOW_ACT(flow_act);
	struct mlx5_flow_handle *rule;
	struct mlx5_flow_spec *spec;
	int err = 0;
	u8 ipv;

	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	if (!spec)
	return ERR_PTR(-ENOMEM);

	ipv = mlx5_etype_to_ipv(etype);
	if (etype && ipv) {
	spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, inner_headers.ip_version);
	MLX5_SET(fte_match_param, spec->match_value, inner_headers.ip_version, ipv);
	}

	if (proto) {
	spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, inner_headers.ip_protocol);
	MLX5_SET(fte_match_param, spec->match_value, inner_headers.ip_protocol, proto);
	}

	rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1);
	if (IS_ERR(rule)) {
	err = PTR_ERR(rule);
	mlx5_core_err(dev, "%s: add inner TTC rule failed\n", __func__);
	}

	kvfree(spec);
	return err ? ERR_PTR(err) : rule;
	}

	static int mlx5_generate_inner_ttc_table_rules(struct mlx5_core_dev *dev,
	struct ttc_params *params,
	struct mlx5_ttc_table *ttc)
	{
	struct mlx5_ttc_rule *rules;
	struct mlx5_flow_table *ft;
	int err;
	int tt;

	ft = ttc->t;
	rules = ttc->rules;

	for (tt = 0; tt < MLX5_NUM_TT; tt++) {
	struct mlx5_ttc_rule *rule = &rules[tt];

	rule->rule = mlx5_generate_inner_ttc_rule(dev, ft,
	&params->dests[tt],
	ttc_rules[tt].etype,
	ttc_rules[tt].proto);
	if (IS_ERR(rule->rule)) {
	err = PTR_ERR(rule->rule);
	rule->rule = NULL;
	goto del_rules;
	}
	rule->default_dest = params->dests[tt];
	}

	return 0;

	del_rules:

	mlx5_cleanup_ttc_rules(ttc);
	return err;
	}

	static int mlx5_create_inner_ttc_table_groups(struct mlx5_ttc_table *ttc)
	{
	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	int ix = 0;
	u32 *in;
	int err;
	u8 *mc;

	ttc->g = kcalloc(MLX5_INNER_TTC_NUM_GROUPS, sizeof(*ttc->g),
	GFP_KERNEL);
	if (!ttc->g)
	return -ENOMEM;
	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in) {
	kfree(ttc->g);
	ttc->g = NULL;
	return -ENOMEM;
	}

	/* L4 Group */
	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_protocol);
	MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_version);
	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_INNER_TTC_GROUP1_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
	goto err;
	ttc->num_groups++;

	/* L3 Group */
	MLX5_SET(fte_match_param, mc, inner_headers.ip_protocol, 0);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_INNER_TTC_GROUP2_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
	goto err;
	ttc->num_groups++;

	/* Any Group */
	memset(in, 0, inlen);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5_INNER_TTC_GROUP3_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
	if (IS_ERR(ttc->g[ttc->num_groups]))
	goto err;
	ttc->num_groups++;

	kvfree(in);
	return 0;

	err:
	err = PTR_ERR(ttc->g[ttc->num_groups]);
	ttc->g[ttc->num_groups] = NULL;
	kvfree(in);

	return err;
	}

	struct mlx5_ttc_table mlx5_create_inner_ttc_table(struct mlx5_core_dev dev,
	struct ttc_params *params)
	{
	struct mlx5_ttc_table *ttc;
	int err;

	ttc = kvzalloc(sizeof(*ttc), GFP_KERNEL);
	if (!ttc)
	return ERR_PTR(-ENOMEM);

	WARN_ON_ONCE(params->ft_attr.max_fte);
	params->ft_attr.max_fte = MLX5_INNER_TTC_TABLE_SIZE;
	ttc->t = mlx5_create_flow_table(params->ns, &params->ft_attr);
	if (IS_ERR(ttc->t)) {
	err = PTR_ERR(ttc->t);
	kvfree(ttc);
	return ERR_PTR(err);
	}

	err = mlx5_create_inner_ttc_table_groups(ttc);
	if (err)
	goto destroy_ft;

	err = mlx5_generate_inner_ttc_table_rules(dev, params, ttc);
	if (err)
	goto destroy_ft;

	return ttc;

	destroy_ft:
	mlx5_destroy_ttc_table(ttc);
	return ERR_PTR(err);
	}

	void mlx5_destroy_ttc_table(struct mlx5_ttc_table *ttc)
	{
	int i;

	mlx5_cleanup_ttc_rules(ttc);
	for (i = ttc->num_groups - 1; i >= 0; i--) {
	if (!IS_ERR_OR_NULL(ttc->g[i]))
	mlx5_destroy_flow_group(ttc->g[i]);
	ttc->g[i] = NULL;
	}

	kfree(ttc->g);
	mlx5_destroy_flow_table(ttc->t);
	kvfree(ttc);
	}

	struct mlx5_ttc_table mlx5_create_ttc_table(struct mlx5_core_dev dev,
	struct ttc_params *params)
	{
	bool match_ipv_outer =
	MLX5_CAP_FLOWTABLE_NIC_RX(dev,
	ft_field_support.outer_ip_version);
	struct mlx5_ttc_table *ttc;
	int err;

	ttc = kvzalloc(sizeof(*ttc), GFP_KERNEL);
	if (!ttc)
	return ERR_PTR(-ENOMEM);

	WARN_ON_ONCE(params->ft_attr.max_fte);
	params->ft_attr.max_fte = MLX5_TTC_TABLE_SIZE;
	ttc->t = mlx5_create_flow_table(params->ns, &params->ft_attr);
	if (IS_ERR(ttc->t)) {
	err = PTR_ERR(ttc->t);
	kvfree(ttc);
	return ERR_PTR(err);
	}

	err = mlx5_create_ttc_table_groups(ttc, match_ipv_outer);
	if (err)
	goto destroy_ft;

	err = mlx5_generate_ttc_table_rules(dev, params, ttc);
	if (err)
	goto destroy_ft;

	return ttc;

	destroy_ft:
	mlx5_destroy_ttc_table(ttc);
	return ERR_PTR(err);
	}

	int mlx5_ttc_fwd_dest(struct mlx5_ttc_table *ttc, enum mlx5_traffic_types type,
	struct mlx5_flow_destination *new_dest)
	{
	return mlx5_modify_rule_destination(ttc->rules[type].rule, new_dest,
	NULL);
	}

	struct mlx5_flow_destination
	mlx5_ttc_get_default_dest(struct mlx5_ttc_table *ttc,
	enum mlx5_traffic_types type)
	{
	struct mlx5_flow_destination *dest = &ttc->rules[type].default_dest;

	WARN_ONCE(dest->type != MLX5_FLOW_DESTINATION_TYPE_TIR,
	"TTC[%d] default dest is not setup yet", type);

	return *dest;
	}

	int mlx5_ttc_fwd_default_dest(struct mlx5_ttc_table *ttc,
	enum mlx5_traffic_types type)
	{
	struct mlx5_flow_destination dest = mlx5_ttc_get_default_dest(ttc, type);

	return mlx5_ttc_fwd_dest(ttc, type, &dest);
	}