drivers/net/ethernet/mellanox/mlx5/core/accel/ipsec_offload.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */

#include "mlx5_core.h"
#include "ipsec_offload.h"
#include "lib/mlx5.h"
#include "en_accel/ipsec_fs.h"

#define MLX5_IPSEC_DEV_BASIC_CAPS (MLX5_ACCEL_IPSEC_CAP_DEVICE | MLX5_ACCEL_IPSEC_CAP_IPV6 | \
				   MLX5_ACCEL_IPSEC_CAP_LSO)

struct mlx5_ipsec_sa_ctx {
	struct rhash_head hash;
	u32 enc_key_id;
	u32 ipsec_obj_id;
	/* hw ctx */
	struct mlx5_core_dev *dev;
	struct mlx5_ipsec_esp_xfrm *mxfrm;
};

struct mlx5_ipsec_esp_xfrm {
	/* reference counter of SA ctx */
	struct mlx5_ipsec_sa_ctx *sa_ctx;
	struct mutex lock; /* protects mlx5_ipsec_esp_xfrm */
	struct mlx5_accel_esp_xfrm accel_xfrm;
};

static u32 mlx5_ipsec_offload_device_caps(struct mlx5_core_dev *mdev)
{
	u32 caps = MLX5_IPSEC_DEV_BASIC_CAPS;

	if (!mlx5_is_ipsec_device(mdev))
		return 0;

	if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) ||
	    !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt))
		return 0;

	if (MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) &&
	    MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt))
		caps |= MLX5_ACCEL_IPSEC_CAP_ESP;

	if (MLX5_CAP_IPSEC(mdev, ipsec_esn)) {
		caps |= MLX5_ACCEL_IPSEC_CAP_ESN;
		caps |= MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN;
	}

	/* We can accommodate up to 2^24 different IPsec objects
	 * because we use up to 24 bit in flow table metadata
	 * to hold the IPsec Object unique handle.
	 */
	WARN_ON_ONCE(MLX5_CAP_IPSEC(mdev, log_max_ipsec_offload) > 24);
	return caps;
}

static int
mlx5_ipsec_offload_esp_validate_xfrm_attrs(struct mlx5_core_dev *mdev,
					   const struct mlx5_accel_esp_xfrm_attrs *attrs)
{
	if (attrs->replay_type != MLX5_ACCEL_ESP_REPLAY_NONE) {
		mlx5_core_err(mdev, "Cannot offload xfrm states with anti replay (replay_type = %d)\n",
			      attrs->replay_type);
		return -EOPNOTSUPP;
	}

	if (attrs->keymat_type != MLX5_ACCEL_ESP_KEYMAT_AES_GCM) {
		mlx5_core_err(mdev, "Only aes gcm keymat is supported (keymat_type = %d)\n",
			      attrs->keymat_type);
		return -EOPNOTSUPP;
	}

	if (attrs->keymat.aes_gcm.iv_algo !=
	    MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ) {
		mlx5_core_err(mdev, "Only iv sequence algo is supported (iv_algo = %d)\n",
			      attrs->keymat.aes_gcm.iv_algo);
		return -EOPNOTSUPP;
	}

	if (attrs->keymat.aes_gcm.key_len != 128 &&
	    attrs->keymat.aes_gcm.key_len != 256) {
		mlx5_core_err(mdev, "Cannot offload xfrm states with key length other than 128/256 bit (key length = %d)\n",
			      attrs->keymat.aes_gcm.key_len);
		return -EOPNOTSUPP;
	}

	if ((attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) &&
	    !MLX5_CAP_IPSEC(mdev, ipsec_esn)) {
		mlx5_core_err(mdev, "Cannot offload xfrm states with ESN triggered\n");
		return -EOPNOTSUPP;
	}

	return 0;
}

static struct mlx5_accel_esp_xfrm *
mlx5_ipsec_offload_esp_create_xfrm(struct mlx5_core_dev *mdev,
				   const struct mlx5_accel_esp_xfrm_attrs *attrs,
				   u32 flags)
{
	struct mlx5_ipsec_esp_xfrm *mxfrm;
	int err = 0;

	err = mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs);
	if (err)
		return ERR_PTR(err);

	mxfrm = kzalloc(sizeof(*mxfrm), GFP_KERNEL);
	if (!mxfrm)
		return ERR_PTR(-ENOMEM);

	mutex_init(&mxfrm->lock);
	memcpy(&mxfrm->accel_xfrm.attrs, attrs,
	       sizeof(mxfrm->accel_xfrm.attrs));

	return &mxfrm->accel_xfrm;
}

static void mlx5_ipsec_offload_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm)
{
	struct mlx5_ipsec_esp_xfrm *mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm,
							 accel_xfrm);

	/* assuming no sa_ctx are connected to this xfrm_ctx */
	WARN_ON(mxfrm->sa_ctx);
	kfree(mxfrm);
}

struct mlx5_ipsec_obj_attrs {
	const struct aes_gcm_keymat *aes_gcm;
	u32 accel_flags;
	u32 esn_msb;
	u32 enc_key_id;
};

static int mlx5_create_ipsec_obj(struct mlx5_core_dev *mdev,
				 struct mlx5_ipsec_obj_attrs *attrs,
				 u32 *ipsec_id)
{
	const struct aes_gcm_keymat *aes_gcm = attrs->aes_gcm;
	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
	u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {};
	void *obj, *salt_p, *salt_iv_p;
	int err;

	obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object);

	/* salt and seq_iv */
	salt_p = MLX5_ADDR_OF(ipsec_obj, obj, salt);
	memcpy(salt_p, &aes_gcm->salt, sizeof(aes_gcm->salt));

	switch (aes_gcm->icv_len) {
	case 64:
		MLX5_SET(ipsec_obj, obj, icv_length,
			 MLX5_IPSEC_OBJECT_ICV_LEN_8B);
		break;
	case 96:
		MLX5_SET(ipsec_obj, obj, icv_length,
			 MLX5_IPSEC_OBJECT_ICV_LEN_12B);
		break;
	case 128:
		MLX5_SET(ipsec_obj, obj, icv_length,
			 MLX5_IPSEC_OBJECT_ICV_LEN_16B);
		break;
	default:
		return -EINVAL;
	}
	salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv);
	memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv));
	/* esn */
	if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) {
		MLX5_SET(ipsec_obj, obj, esn_en, 1);
		MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb);
		if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP)
			MLX5_SET(ipsec_obj, obj, esn_overlap, 1);
	}

	MLX5_SET(ipsec_obj, obj, dekn, attrs->enc_key_id);

	/* general object fields set */
	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
		 MLX5_GENERAL_OBJECT_TYPES_IPSEC);

	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	if (!err)
		*ipsec_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);

	return err;
}

static void mlx5_destroy_ipsec_obj(struct mlx5_core_dev *mdev, u32 ipsec_id)
{
	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];

	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
		 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
		 MLX5_GENERAL_OBJECT_TYPES_IPSEC);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id);

	mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
}

static void *mlx5_ipsec_offload_create_sa_ctx(struct mlx5_core_dev *mdev,
					      struct mlx5_accel_esp_xfrm *accel_xfrm,
					      const __be32 saddr[4], const __be32 daddr[4],
					      const __be32 spi, bool is_ipv6, u32 *hw_handle)
{
	struct mlx5_accel_esp_xfrm_attrs *xfrm_attrs = &accel_xfrm->attrs;
	struct aes_gcm_keymat *aes_gcm = &xfrm_attrs->keymat.aes_gcm;
	struct mlx5_ipsec_obj_attrs ipsec_attrs = {};
	struct mlx5_ipsec_esp_xfrm *mxfrm;
	struct mlx5_ipsec_sa_ctx *sa_ctx;
	int err;

	/* alloc SA context */
	sa_ctx = kzalloc(sizeof(*sa_ctx), GFP_KERNEL);
	if (!sa_ctx)
		return ERR_PTR(-ENOMEM);

	sa_ctx->dev = mdev;

	mxfrm = container_of(accel_xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm);
	mutex_lock(&mxfrm->lock);
	sa_ctx->mxfrm = mxfrm;

	/* key */
	err = mlx5_create_encryption_key(mdev, aes_gcm->aes_key,
					 aes_gcm->key_len / BITS_PER_BYTE,
					 MLX5_ACCEL_OBJ_IPSEC_KEY,
					 &sa_ctx->enc_key_id);
	if (err) {
		mlx5_core_dbg(mdev, "Failed to create encryption key (err = %d)\n", err);
		goto err_sa_ctx;
	}

	ipsec_attrs.aes_gcm = aes_gcm;
	ipsec_attrs.accel_flags = accel_xfrm->attrs.flags;
	ipsec_attrs.esn_msb = accel_xfrm->attrs.esn;
	ipsec_attrs.enc_key_id = sa_ctx->enc_key_id;
	err = mlx5_create_ipsec_obj(mdev, &ipsec_attrs,
				    &sa_ctx->ipsec_obj_id);
	if (err) {
		mlx5_core_dbg(mdev, "Failed to create IPsec object (err = %d)\n", err);
		goto err_enc_key;
	}

	*hw_handle = sa_ctx->ipsec_obj_id;
	mxfrm->sa_ctx = sa_ctx;
	mutex_unlock(&mxfrm->lock);

	return sa_ctx;

err_enc_key:
	mlx5_destroy_encryption_key(mdev, sa_ctx->enc_key_id);
err_sa_ctx:
	mutex_unlock(&mxfrm->lock);
	kfree(sa_ctx);
	return ERR_PTR(err);
}

static void mlx5_ipsec_offload_delete_sa_ctx(void *context)
{
	struct mlx5_ipsec_sa_ctx *sa_ctx = (struct mlx5_ipsec_sa_ctx *)context;
	struct mlx5_ipsec_esp_xfrm *mxfrm = sa_ctx->mxfrm;

	mutex_lock(&mxfrm->lock);
	mlx5_destroy_ipsec_obj(sa_ctx->dev, sa_ctx->ipsec_obj_id);
	mlx5_destroy_encryption_key(sa_ctx->dev, sa_ctx->enc_key_id);
	kfree(sa_ctx);
	mxfrm->sa_ctx = NULL;
	mutex_unlock(&mxfrm->lock);
}

static int mlx5_ipsec_offload_init(struct mlx5_core_dev *mdev)
{
	return 0;
}

static int mlx5_modify_ipsec_obj(struct mlx5_core_dev *mdev,
				 struct mlx5_ipsec_obj_attrs *attrs,
				 u32 ipsec_id)
{
	u32 in[MLX5_ST_SZ_DW(modify_ipsec_obj_in)] = {};
	u32 out[MLX5_ST_SZ_DW(query_ipsec_obj_out)];
	u64 modify_field_select = 0;
	u64 general_obj_types;
	void *obj;
	int err;

	if (!(attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED))
		return 0;

	general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
	if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
		return -EINVAL;

	/* general object fields set */
	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id);
	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	if (err) {
		mlx5_core_err(mdev, "Query IPsec object failed (Object id %d), err = %d\n",
			      ipsec_id, err);
		return err;
	}

	obj = MLX5_ADDR_OF(query_ipsec_obj_out, out, ipsec_object);
	modify_field_select = MLX5_GET64(ipsec_obj, obj, modify_field_select);

	/* esn */
	if (!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP) ||
	    !(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB))
		return -EOPNOTSUPP;

	obj = MLX5_ADDR_OF(modify_ipsec_obj_in, in, ipsec_object);
	MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb);
	if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP)
		MLX5_SET(ipsec_obj, obj, esn_overlap, 1);

	/* general object fields set */
	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);

	return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
}

static int mlx5_ipsec_offload_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm,
					      const struct mlx5_accel_esp_xfrm_attrs *attrs)
{
	struct mlx5_ipsec_obj_attrs ipsec_attrs = {};
	struct mlx5_core_dev *mdev = xfrm->mdev;
	struct mlx5_ipsec_esp_xfrm *mxfrm;

	int err = 0;

	if (!memcmp(&xfrm->attrs, attrs, sizeof(xfrm->attrs)))
		return 0;

	if (mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs))
		return -EOPNOTSUPP;

	mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm);

	mutex_lock(&mxfrm->lock);

	if (!mxfrm->sa_ctx)
		/* Not bound xfrm, change only sw attrs */
		goto change_sw_xfrm_attrs;

	/* need to add find and replace in ipsec_rhash_sa the sa_ctx */
	/* modify device with new hw_sa */
	ipsec_attrs.accel_flags = attrs->flags;
	ipsec_attrs.esn_msb = attrs->esn;
	err = mlx5_modify_ipsec_obj(mdev,
				    &ipsec_attrs,
				    mxfrm->sa_ctx->ipsec_obj_id);

change_sw_xfrm_attrs:
	if (!err)
		memcpy(&xfrm->attrs, attrs, sizeof(xfrm->attrs));

	mutex_unlock(&mxfrm->lock);
	return err;
}

static const struct mlx5_accel_ipsec_ops ipsec_offload_ops = {
	.device_caps = mlx5_ipsec_offload_device_caps,
	.create_hw_context = mlx5_ipsec_offload_create_sa_ctx,
	.free_hw_context = mlx5_ipsec_offload_delete_sa_ctx,
	.init = mlx5_ipsec_offload_init,
	.esp_create_xfrm = mlx5_ipsec_offload_esp_create_xfrm,
	.esp_destroy_xfrm = mlx5_ipsec_offload_esp_destroy_xfrm,
	.esp_modify_xfrm = mlx5_ipsec_offload_esp_modify_xfrm,
};

const struct mlx5_accel_ipsec_ops *mlx5_ipsec_offload_ops(struct mlx5_core_dev *mdev)
{
	if (!mlx5_ipsec_offload_device_caps(mdev))
		return NULL;

	return &ipsec_offload_ops;
}