drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */

#include <linux/netdevice.h>
#include "en_accel/fs_tcp.h"
#include "fs_core.h"

enum accel_fs_tcp_type {
	ACCEL_FS_IPV4_TCP,
	ACCEL_FS_IPV6_TCP,
	ACCEL_FS_TCP_NUM_TYPES,
};

struct mlx5e_accel_fs_tcp {
	struct mlx5e_flow_table tables[ACCEL_FS_TCP_NUM_TYPES];
	struct mlx5_flow_handle *default_rules[ACCEL_FS_TCP_NUM_TYPES];
};

static enum mlx5_traffic_types fs_accel2tt(enum accel_fs_tcp_type i)
{
	switch (i) {
	case ACCEL_FS_IPV4_TCP:
		return MLX5_TT_IPV4_TCP;
	default: /* ACCEL_FS_IPV6_TCP */
		return MLX5_TT_IPV6_TCP;
	}
}

static void accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct sock *sk)
{
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4);
	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
			    outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4),
	       &inet_sk(sk)->inet_daddr, 4);
	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
			    outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
	       &inet_sk(sk)->inet_rcv_saddr, 4);
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
			 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
			 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
}

#if IS_ENABLED(CONFIG_IPV6)
static void accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct sock *sk)
{
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6);
	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
			    outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
	       &sk->sk_v6_daddr, 16);
	memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
			    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
	       &inet6_sk(sk)->saddr, 16);
	memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
			    outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
	       0xff, 16);
	memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
			    outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
	       0xff, 16);
}
#endif

void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule)
{
	mlx5_del_flow_rules(rule);
}

struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv,
					       struct sock *sk, u32 tirn,
					       uint32_t flow_tag)
{
	struct mlx5_flow_destination dest = {};
	struct mlx5e_flow_table *ft = NULL;
	struct mlx5e_accel_fs_tcp *fs_tcp;
	MLX5_DECLARE_FLOW_ACT(flow_act);
	struct mlx5_flow_handle *flow;
	struct mlx5_flow_spec *spec;

	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
	if (!spec)
		return ERR_PTR(-ENOMEM);

	fs_tcp = priv->fs.accel_tcp;

	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;

	switch (sk->sk_family) {
	case AF_INET:
		accel_fs_tcp_set_ipv4_flow(spec, sk);
		ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP];
		mlx5e_dbg(HW, priv, "%s flow is %pI4:%d -> %pI4:%d\n", __func__,
			  &inet_sk(sk)->inet_rcv_saddr,
			  inet_sk(sk)->inet_sport,
			  &inet_sk(sk)->inet_daddr,
			  inet_sk(sk)->inet_dport);
		break;
#if IS_ENABLED(CONFIG_IPV6)
	case AF_INET6:
		if (!sk->sk_ipv6only &&
		    ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
			accel_fs_tcp_set_ipv4_flow(spec, sk);
			ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP];
		} else {
			accel_fs_tcp_set_ipv6_flow(spec, sk);
			ft = &fs_tcp->tables[ACCEL_FS_IPV6_TCP];
		}
		break;
#endif
	default:
		break;
	}

	if (!ft) {
		flow = ERR_PTR(-EINVAL);
		goto out;
	}

	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
			 outer_headers.tcp_dport);
	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
			 outer_headers.tcp_sport);
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport,
		 ntohs(inet_sk(sk)->inet_sport));
	MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport,
		 ntohs(inet_sk(sk)->inet_dport));

	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
	dest.tir_num = tirn;
	if (flow_tag != MLX5_FS_DEFAULT_FLOW_TAG) {
		spec->flow_context.flow_tag = flow_tag;
		spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
	}

	flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1);

	if (IS_ERR(flow))
		netdev_err(priv->netdev, "mlx5_add_flow_rules() failed, flow is %ld\n",
			   PTR_ERR(flow));

out:
	kvfree(spec);
	return flow;
}

static int accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv,
					 enum accel_fs_tcp_type type)
{
	struct mlx5e_flow_table *accel_fs_t;
	struct mlx5_flow_destination dest;
	struct mlx5e_accel_fs_tcp *fs_tcp;
	MLX5_DECLARE_FLOW_ACT(flow_act);
	struct mlx5_flow_handle *rule;
	int err = 0;

	fs_tcp = priv->fs.accel_tcp;
	accel_fs_t = &fs_tcp->tables[type];

	dest = mlx5_ttc_get_default_dest(priv->fs.ttc, fs_accel2tt(type));
	rule = mlx5_add_flow_rules(accel_fs_t->t, NULL, &flow_act, &dest, 1);
	if (IS_ERR(rule)) {
		err = PTR_ERR(rule);
		netdev_err(priv->netdev,
			   "%s: add default rule failed, accel_fs type=%d, err %d\n",
			   __func__, type, err);
		return err;
	}

	fs_tcp->default_rules[type] = rule;
	return 0;
}

#define MLX5E_ACCEL_FS_TCP_NUM_GROUPS	(2)
#define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE	(BIT(16) - 1)
#define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE	(BIT(0))
#define MLX5E_ACCEL_FS_TCP_TABLE_SIZE	(MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\
					 MLX5E_ACCEL_FS_TCP_GROUP2_SIZE)
static int accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft,
				      enum accel_fs_tcp_type type)
{
	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
	void *outer_headers_c;
	int ix = 0;
	u32 *in;
	int err;
	u8 *mc;

	ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
	in = kvzalloc(inlen, GFP_KERNEL);
	if  (!in || !ft->g) {
		kfree(ft->g);
		kvfree(in);
		return -ENOMEM;
	}

	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
	outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
	MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version);

	switch (type) {
	case ACCEL_FS_IPV4_TCP:
	case ACCEL_FS_IPV6_TCP:
		MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport);
		MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport);
		break;
	default:
		err = -EINVAL;
		goto out;
	}

	switch (type) {
	case ACCEL_FS_IPV4_TCP:
		MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
				 src_ipv4_src_ipv6.ipv4_layout.ipv4);
		MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
				 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
		break;
	case ACCEL_FS_IPV6_TCP:
		memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
				    src_ipv4_src_ipv6.ipv6_layout.ipv6),
		       0xff, 16);
		memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
				    dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
		       0xff, 16);
		break;
	default:
		err = -EINVAL;
		goto out;
	}

	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
	if (IS_ERR(ft->g[ft->num_groups]))
		goto err;
	ft->num_groups++;

	/* Default Flow Group */
	memset(in, 0, inlen);
	MLX5_SET_CFG(in, start_flow_index, ix);
	ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE;
	MLX5_SET_CFG(in, end_flow_index, ix - 1);
	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
	if (IS_ERR(ft->g[ft->num_groups]))
		goto err;
	ft->num_groups++;

	kvfree(in);
	return 0;

err:
	err = PTR_ERR(ft->g[ft->num_groups]);
	ft->g[ft->num_groups] = NULL;
out:
	kvfree(in);

	return err;
}

static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, enum accel_fs_tcp_type type)
{
	struct mlx5e_flow_table *ft = &priv->fs.accel_tcp->tables[type];
	struct mlx5_flow_table_attr ft_attr = {};
	int err;

	ft->num_groups = 0;

	ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE;
	ft_attr.level = MLX5E_ACCEL_FS_TCP_FT_LEVEL;
	ft_attr.prio = MLX5E_NIC_PRIO;

	ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
	if (IS_ERR(ft->t)) {
		err = PTR_ERR(ft->t);
		ft->t = NULL;
		return err;
	}

	netdev_dbg(priv->netdev, "Created fs accel table id %u level %u\n",
		   ft->t->id, ft->t->level);

	err = accel_fs_tcp_create_groups(ft, type);
	if (err)
		goto err;

	err = accel_fs_tcp_add_default_rule(priv, type);
	if (err)
		goto err;

	return 0;
err:
	mlx5e_destroy_flow_table(ft);
	return err;
}

static int accel_fs_tcp_disable(struct mlx5e_priv *priv)
{
	int err, i;

	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
		/* Modify ttc rules destination to point back to the indir TIRs */
		err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, fs_accel2tt(i));
		if (err) {
			netdev_err(priv->netdev,
				   "%s: modify ttc[%d] default destination failed, err(%d)\n",
				   __func__, fs_accel2tt(i), err);
			return err;
		}
	}

	return 0;
}

static int accel_fs_tcp_enable(struct mlx5e_priv *priv)
{
	struct mlx5_flow_destination dest = {};
	int err, i;

	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
		dest.ft = priv->fs.accel_tcp->tables[i].t;

		/* Modify ttc rules destination to point on the accel_fs FTs */
		err = mlx5_ttc_fwd_dest(priv->fs.ttc, fs_accel2tt(i), &dest);
		if (err) {
			netdev_err(priv->netdev,
				   "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
				   __func__, fs_accel2tt(i), err);
			return err;
		}
	}
	return 0;
}

static void accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i)
{
	struct mlx5e_accel_fs_tcp *fs_tcp;

	fs_tcp = priv->fs.accel_tcp;
	if (IS_ERR_OR_NULL(fs_tcp->tables[i].t))
		return;

	mlx5_del_flow_rules(fs_tcp->default_rules[i]);
	mlx5e_destroy_flow_table(&fs_tcp->tables[i]);
	fs_tcp->tables[i].t = NULL;
}

void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv)
{
	int i;

	if (!priv->fs.accel_tcp)
		return;

	accel_fs_tcp_disable(priv);

	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++)
		accel_fs_tcp_destroy_table(priv, i);

	kfree(priv->fs.accel_tcp);
	priv->fs.accel_tcp = NULL;
}

int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv)
{
	int i, err;

	if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version))
		return -EOPNOTSUPP;

	priv->fs.accel_tcp = kzalloc(sizeof(*priv->fs.accel_tcp), GFP_KERNEL);
	if (!priv->fs.accel_tcp)
		return -ENOMEM;

	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
		err = accel_fs_tcp_create_table(priv, i);
		if (err)
			goto err_destroy_tables;
	}

	err = accel_fs_tcp_enable(priv);
	if (err)
		goto err_destroy_tables;

	return 0;

err_destroy_tables:
	while (--i >= 0)
		accel_fs_tcp_destroy_table(priv, i);

	kfree(priv->fs.accel_tcp);
	priv->fs.accel_tcp = NULL;
	return err;
}