drivers/net/ethernet/mellanox/mlx5/core/transobj.c - linux - Git at Google

 /*
  * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
  * General Public License (GPL) Version 2, available from the file
  * COPYING in the main directory of this source tree, or the
  * OpenIB.org BSD license below:
  *
  *     Redistribution and use in source and binary forms, with or
  *     without modification, are permitted provided that the following
  *     conditions are met:
  *
  *      - Redistributions of source code must retain the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer.
  *
  *      - Redistributions in binary form must reproduce the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer in the documentation and/or other materials
  *        provided with the distribution.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */

 #include <linux/mlx5/driver.h>
 #include "mlx5_core.h"
 #include <linux/mlx5/transobj.h>

 int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
 {
 	u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {};
 	u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {};
 	int err;

 	MLX5_SET(alloc_transport_domain_in, in, opcode,
 		 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);

 	err = mlx5_cmd_exec_inout(dev, alloc_transport_domain, in, out);
 	if (!err)
 		*tdn = MLX5_GET(alloc_transport_domain_out, out,
 				transport_domain);

 	return err;
 }
 EXPORT_SYMBOL(mlx5_core_alloc_transport_domain);

 void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
 {
 	u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {};

 	MLX5_SET(dealloc_transport_domain_in, in, opcode,
 		 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
 	MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
 	mlx5_cmd_exec_in(dev, dealloc_transport_domain, in);
 }
 EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain);

 int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn)
 {
 	u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {};
 	int err;

 	MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);
 	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (!err)
 		*rqn = MLX5_GET(create_rq_out, out, rqn);

 	return err;
 }
 EXPORT_SYMBOL(mlx5_core_create_rq);

 int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in)
 {
 	MLX5_SET(modify_rq_in, in, rqn, rqn);
 	MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);

 	return mlx5_cmd_exec_in(dev, modify_rq, in);
 }
 EXPORT_SYMBOL(mlx5_core_modify_rq);

 void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)
 {
 	u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {};

 	MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);
 	MLX5_SET(destroy_rq_in, in, rqn, rqn);
 	mlx5_cmd_exec_in(dev, destroy_rq, in);
 }
 EXPORT_SYMBOL(mlx5_core_destroy_rq);

 int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out)
 {
 	u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {};

 	MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ);
 	MLX5_SET(query_rq_in, in, rqn, rqn);

 	return mlx5_cmd_exec_inout(dev, query_rq, in, out);
 }
 EXPORT_SYMBOL(mlx5_core_query_rq);

 int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn)
 {
 	u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {};
 	int err;

 	MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
 	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (!err)
 		*sqn = MLX5_GET(create_sq_out, out, sqn);

 	return err;
 }

 int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in)
 {
 	MLX5_SET(modify_sq_in, in, sqn, sqn);
 	MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);
 	return mlx5_cmd_exec_in(dev, modify_sq, in);
 }
 EXPORT_SYMBOL(mlx5_core_modify_sq);

 void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)
 {
 	u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {};

 	MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);
 	MLX5_SET(destroy_sq_in, in, sqn, sqn);
 	mlx5_cmd_exec_in(dev, destroy_sq, in);
 }

 int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out)
 {
 	u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {};

 	MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ);
 	MLX5_SET(query_sq_in, in, sqn, sqn);
 	return mlx5_cmd_exec_inout(dev, query_sq, in, out);
 }
 EXPORT_SYMBOL(mlx5_core_query_sq);

 int mlx5_core_query_sq_state(struct mlx5_core_dev *dev, u32 sqn, u8 *state)
 {
 	void *out;
 	void *sqc;
 	int inlen;
 	int err;

 	inlen = MLX5_ST_SZ_BYTES(query_sq_out);
 	out = kvzalloc(inlen, GFP_KERNEL);
 	if (!out)
 		return -ENOMEM;

 	err = mlx5_core_query_sq(dev, sqn, out);
 	if (err)
 		goto out;

 	sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context);
 	*state = MLX5_GET(sqc, sqc, state);

 out:
 	kvfree(out);
 	return err;
 }
 EXPORT_SYMBOL_GPL(mlx5_core_query_sq_state);

 int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, u32 *tirn)
 {
 	u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
 	int err;

 	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
 	err = mlx5_cmd_exec_inout(dev, create_tir, in, out);
 	if (!err)
 		*tirn = MLX5_GET(create_tir_out, out, tirn);

 	return err;
 }
 EXPORT_SYMBOL(mlx5_core_create_tir);

 int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in)
 {
 	MLX5_SET(modify_tir_in, in, tirn, tirn);
 	MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR);
 	return mlx5_cmd_exec_in(dev, modify_tir, in);
 }

 void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)
 {
 	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};

 	MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
 	MLX5_SET(destroy_tir_in, in, tirn, tirn);
 	mlx5_cmd_exec_in(dev, destroy_tir, in);
 }
 EXPORT_SYMBOL(mlx5_core_destroy_tir);

 int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, u32 *tisn)
 {
 	u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {};
 	int err;

 	MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
 	err = mlx5_cmd_exec_inout(dev, create_tis, in, out);
 	if (!err)
 		*tisn = MLX5_GET(create_tis_out, out, tisn);

 	return err;
 }
 EXPORT_SYMBOL(mlx5_core_create_tis);

 int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in)
 {
 	MLX5_SET(modify_tis_in, in, tisn, tisn);
 	MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS);

 	return mlx5_cmd_exec_in(dev, modify_tis, in);
 }
 EXPORT_SYMBOL(mlx5_core_modify_tis);

 void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
 {
 	u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {};

 	MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
 	MLX5_SET(destroy_tis_in, in, tisn, tisn);
 	mlx5_cmd_exec_in(dev, destroy_tis, in);
 }
 EXPORT_SYMBOL(mlx5_core_destroy_tis);

 int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
 			 u32 *rqtn)
 {
 	u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {};
 	int err;

 	MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
 	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 	if (!err)
 		*rqtn = MLX5_GET(create_rqt_out, out, rqtn);

 	return err;
 }
 EXPORT_SYMBOL(mlx5_core_create_rqt);

 int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
 			 int inlen)
 {
 	u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {};

 	MLX5_SET(modify_rqt_in, in, rqtn, rqtn);
 	MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
 	return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
 }

 void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)
 {
 	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {};

 	MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
 	MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
 	mlx5_cmd_exec_in(dev, destroy_rqt, in);
 }
 EXPORT_SYMBOL(mlx5_core_destroy_rqt);

 static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev,
 				  struct mlx5_hairpin_params *params, u32 *rqn)
 {
 	u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0};
 	void *rqc, *wq;

 	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 	wq  = MLX5_ADDR_OF(rqc, rqc, wq);

 	MLX5_SET(rqc, rqc, hairpin, 1);
 	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
 	MLX5_SET(rqc, rqc, counter_set_id, params->q_counter);

 	MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
 	MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);

 	return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn);
 }

 static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev,
 				  struct mlx5_hairpin_params *params, u32 *sqn)
 {
 	u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0};
 	void *sqc, *wq;

 	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 	wq  = MLX5_ADDR_OF(sqc, sqc, wq);

 	MLX5_SET(sqc, sqc, hairpin, 1);
 	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);

 	MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
 	MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);

 	return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn);
 }

 static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp,
 				      struct mlx5_hairpin_params *params)
 {
 	int i, j, err;

 	for (i = 0; i < hp->num_channels; i++) {
 		err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]);
 		if (err)
 			goto out_err_rq;
 	}

 	for (i = 0; i < hp->num_channels; i++) {
 		err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]);
 		if (err)
 			goto out_err_sq;
 	}

 	return 0;

 out_err_sq:
 	for (j = 0; j < i; j++)
 		mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]);
 	i = hp->num_channels;
 out_err_rq:
 	for (j = 0; j < i; j++)
 		mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]);
 	return err;
 }

 static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp)
 {
 	int i;

 	for (i = 0; i < hp->num_channels; i++) {
 		mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]);
 		if (!hp->peer_gone)
 			mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);
 	}
 }

 static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn,
 				  int curr_state, int next_state,
 				  u16 peer_vhca, u32 peer_sq)
 {
 	u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {};
 	void *rqc;

 	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);

 	if (next_state == MLX5_RQC_STATE_RDY) {
 		MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq);
 		MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca);
 	}

 	MLX5_SET(modify_rq_in, in, rq_state, curr_state);
 	MLX5_SET(rqc, rqc, state, next_state);

 	return mlx5_core_modify_rq(func_mdev, rqn, in);
 }

 static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn,
 				  int curr_state, int next_state,
 				  u16 peer_vhca, u32 peer_rq)
 {
 	u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0};
 	void *sqc;

 	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);

 	if (next_state == MLX5_SQC_STATE_RDY) {
 		MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq);
 		MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca);
 	}

 	MLX5_SET(modify_sq_in, in, sq_state, curr_state);
 	MLX5_SET(sqc, sqc, state, next_state);

 	return mlx5_core_modify_sq(peer_mdev, sqn, in);
 }

 static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp)
 {
 	int i, j, err;

 	/* set peer SQs */
 	for (i = 0; i < hp->num_channels; i++) {
 		err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i],
 					     MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
 					     MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]);
 		if (err)
 			goto err_modify_sq;
 	}

 	/* set func RQs */
 	for (i = 0; i < hp->num_channels; i++) {
 		err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i],
 					     MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY,
 					     MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]);
 		if (err)
 			goto err_modify_rq;
 	}

 	return 0;

 err_modify_rq:
 	for (j = 0; j < i; j++)
 		mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY,
 				       MLX5_RQC_STATE_RST, 0, 0);
 	i = hp->num_channels;
 err_modify_sq:
 	for (j = 0; j < i; j++)
 		mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY,
 				       MLX5_SQC_STATE_RST, 0, 0);
 	return err;
 }

 static void mlx5_hairpin_unpair_peer_sq(struct mlx5_hairpin *hp)
 {
 	int i;

 	for (i = 0; i < hp->num_channels; i++)
 		mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY,
 				       MLX5_SQC_STATE_RST, 0, 0);
 }

 static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp)
 {
 	int i;

 	/* unset func RQs */
 	for (i = 0; i < hp->num_channels; i++)
 		mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY,
 				       MLX5_RQC_STATE_RST, 0, 0);
 	/* unset peer SQs */
 	if (!hp->peer_gone)
 		mlx5_hairpin_unpair_peer_sq(hp);
 }

 struct mlx5_hairpin *
 mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev,
 			 struct mlx5_core_dev *peer_mdev,
 			 struct mlx5_hairpin_params *params)
 {
 	struct mlx5_hairpin *hp;
 	int size, err;

 	size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32);
 	hp = kzalloc(size, GFP_KERNEL);
 	if (!hp)
 		return ERR_PTR(-ENOMEM);

 	hp->func_mdev = func_mdev;
 	hp->peer_mdev = peer_mdev;
 	hp->num_channels = params->num_channels;

 	hp->rqn = (void *)hp + sizeof(*hp);
 	hp->sqn = hp->rqn + params->num_channels;

 	/* alloc and pair func --> peer hairpin */
 	err = mlx5_hairpin_create_queues(hp, params);
 	if (err)
 		goto err_create_queues;

 	err = mlx5_hairpin_pair_queues(hp);
 	if (err)
 		goto err_pair_queues;

 	return hp;

 err_pair_queues:
 	mlx5_hairpin_destroy_queues(hp);
 err_create_queues:
 	kfree(hp);
 	return ERR_PTR(err);
 }

 void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp)
 {
 	mlx5_hairpin_unpair_queues(hp);
 	mlx5_hairpin_destroy_queues(hp);
 	kfree(hp);
 }

 void mlx5_core_hairpin_clear_dead_peer(struct mlx5_hairpin *hp)
 {
 	int i;

 	mlx5_hairpin_unpair_peer_sq(hp);

 	/* destroy peer SQ */
 	for (i = 0; i < hp->num_channels; i++)
 		mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);

 	hp->peer_gone = true;
 }
	/*
	* Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
	*
	* This software is available to you under a choice of one of two
	* licenses. You may choose to be licensed under the terms of the GNU
	* General Public License (GPL) Version 2, available from the file
	* COPYING in the main directory of this source tree, or the
	* OpenIB.org BSD license below:
	*
	* Redistribution and use in source and binary forms, with or
	* without modification, are permitted provided that the following
	* conditions are met:
	*
	* - Redistributions of source code must retain the above
	* copyright notice, this list of conditions and the following
	* disclaimer.
	*
	* - Redistributions in binary form must reproduce the above
	* copyright notice, this list of conditions and the following
	* disclaimer in the documentation and/or other materials
	* provided with the distribution.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	* SOFTWARE.
	*/

	#include <linux/mlx5/driver.h>
	#include "mlx5_core.h"
	#include <linux/mlx5/transobj.h>

	int mlx5_core_alloc_transport_domain(struct mlx5_core_dev dev, u32 tdn)
	{
	u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {};
	u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {};
	int err;

	MLX5_SET(alloc_transport_domain_in, in, opcode,
	MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);

	err = mlx5_cmd_exec_inout(dev, alloc_transport_domain, in, out);
	if (!err)
	*tdn = MLX5_GET(alloc_transport_domain_out, out,
	transport_domain);

	return err;
	}
	EXPORT_SYMBOL(mlx5_core_alloc_transport_domain);

	void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
	{
	u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {};

	MLX5_SET(dealloc_transport_domain_in, in, opcode,
	MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
	MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
	mlx5_cmd_exec_in(dev, dealloc_transport_domain, in);
	}
	EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain);

	int mlx5_core_create_rq(struct mlx5_core_dev dev, u32 in, int inlen, u32 *rqn)
	{
	u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {};
	int err;

	MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);
	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	if (!err)
	*rqn = MLX5_GET(create_rq_out, out, rqn);

	return err;
	}
	EXPORT_SYMBOL(mlx5_core_create_rq);

	int mlx5_core_modify_rq(struct mlx5_core_dev dev, u32 rqn, u32 in)
	{
	MLX5_SET(modify_rq_in, in, rqn, rqn);
	MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);

	return mlx5_cmd_exec_in(dev, modify_rq, in);
	}
	EXPORT_SYMBOL(mlx5_core_modify_rq);

	void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)
	{
	u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {};

	MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);
	MLX5_SET(destroy_rq_in, in, rqn, rqn);
	mlx5_cmd_exec_in(dev, destroy_rq, in);
	}
	EXPORT_SYMBOL(mlx5_core_destroy_rq);

	int mlx5_core_query_rq(struct mlx5_core_dev dev, u32 rqn, u32 out)
	{
	u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {};

	MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ);
	MLX5_SET(query_rq_in, in, rqn, rqn);

	return mlx5_cmd_exec_inout(dev, query_rq, in, out);
	}
	EXPORT_SYMBOL(mlx5_core_query_rq);

	int mlx5_core_create_sq(struct mlx5_core_dev dev, u32 in, int inlen, u32 *sqn)
	{
	u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {};
	int err;

	MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	if (!err)
	*sqn = MLX5_GET(create_sq_out, out, sqn);

	return err;
	}

	int mlx5_core_modify_sq(struct mlx5_core_dev dev, u32 sqn, u32 in)
	{
	MLX5_SET(modify_sq_in, in, sqn, sqn);
	MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);
	return mlx5_cmd_exec_in(dev, modify_sq, in);
	}
	EXPORT_SYMBOL(mlx5_core_modify_sq);

	void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)
	{
	u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {};

	MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);
	MLX5_SET(destroy_sq_in, in, sqn, sqn);
	mlx5_cmd_exec_in(dev, destroy_sq, in);
	}

	int mlx5_core_query_sq(struct mlx5_core_dev dev, u32 sqn, u32 out)
	{
	u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {};

	MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ);
	MLX5_SET(query_sq_in, in, sqn, sqn);
	return mlx5_cmd_exec_inout(dev, query_sq, in, out);
	}
	EXPORT_SYMBOL(mlx5_core_query_sq);

	int mlx5_core_query_sq_state(struct mlx5_core_dev dev, u32 sqn, u8 state)
	{
	void *out;
	void *sqc;
	int inlen;
	int err;

	inlen = MLX5_ST_SZ_BYTES(query_sq_out);
	out = kvzalloc(inlen, GFP_KERNEL);
	if (!out)
	return -ENOMEM;

	err = mlx5_core_query_sq(dev, sqn, out);
	if (err)
	goto out;

	sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context);
	*state = MLX5_GET(sqc, sqc, state);

	out:
	kvfree(out);
	return err;
	}
	EXPORT_SYMBOL_GPL(mlx5_core_query_sq_state);

	int mlx5_core_create_tir(struct mlx5_core_dev dev, u32 in, u32 *tirn)
	{
	u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
	int err;

	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
	err = mlx5_cmd_exec_inout(dev, create_tir, in, out);
	if (!err)
	*tirn = MLX5_GET(create_tir_out, out, tirn);

	return err;
	}
	EXPORT_SYMBOL(mlx5_core_create_tir);

	int mlx5_core_modify_tir(struct mlx5_core_dev dev, u32 tirn, u32 in)
	{
	MLX5_SET(modify_tir_in, in, tirn, tirn);
	MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR);
	return mlx5_cmd_exec_in(dev, modify_tir, in);
	}

	void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)
	{
	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {};

	MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
	MLX5_SET(destroy_tir_in, in, tirn, tirn);
	mlx5_cmd_exec_in(dev, destroy_tir, in);
	}
	EXPORT_SYMBOL(mlx5_core_destroy_tir);

	int mlx5_core_create_tis(struct mlx5_core_dev dev, u32 in, u32 *tisn)
	{
	u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {};
	int err;

	MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
	err = mlx5_cmd_exec_inout(dev, create_tis, in, out);
	if (!err)
	*tisn = MLX5_GET(create_tis_out, out, tisn);

	return err;
	}
	EXPORT_SYMBOL(mlx5_core_create_tis);

	int mlx5_core_modify_tis(struct mlx5_core_dev dev, u32 tisn, u32 in)
	{
	MLX5_SET(modify_tis_in, in, tisn, tisn);
	MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS);

	return mlx5_cmd_exec_in(dev, modify_tis, in);
	}
	EXPORT_SYMBOL(mlx5_core_modify_tis);

	void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
	{
	u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {};

	MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
	MLX5_SET(destroy_tis_in, in, tisn, tisn);
	mlx5_cmd_exec_in(dev, destroy_tis, in);
	}
	EXPORT_SYMBOL(mlx5_core_destroy_tis);

	int mlx5_core_create_rqt(struct mlx5_core_dev dev, u32 in, int inlen,
	u32 *rqtn)
	{
	u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {};
	int err;

	MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	if (!err)
	*rqtn = MLX5_GET(create_rqt_out, out, rqtn);

	return err;
	}
	EXPORT_SYMBOL(mlx5_core_create_rqt);

	int mlx5_core_modify_rqt(struct mlx5_core_dev dev, u32 rqtn, u32 in,
	int inlen)
	{
	u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {};

	MLX5_SET(modify_rqt_in, in, rqtn, rqtn);
	MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
	return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	}

	void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)
	{
	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {};

	MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
	MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
	mlx5_cmd_exec_in(dev, destroy_rqt, in);
	}
	EXPORT_SYMBOL(mlx5_core_destroy_rqt);

	static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev,
	struct mlx5_hairpin_params params, u32 rqn)
	{
	u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0};
	void rqc, wq;

	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
	wq = MLX5_ADDR_OF(rqc, rqc, wq);

	MLX5_SET(rqc, rqc, hairpin, 1);
	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
	MLX5_SET(rqc, rqc, counter_set_id, params->q_counter);

	MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
	MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);

	return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn);
	}

	static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev,
	struct mlx5_hairpin_params params, u32 sqn)
	{
	u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0};
	void sqc, wq;

	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
	wq = MLX5_ADDR_OF(sqc, sqc, wq);

	MLX5_SET(sqc, sqc, hairpin, 1);
	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);

	MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
	MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);

	return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn);
	}

	static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp,
	struct mlx5_hairpin_params *params)
	{
	int i, j, err;

	for (i = 0; i < hp->num_channels; i++) {
	err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]);
	if (err)
	goto out_err_rq;
	}

	for (i = 0; i < hp->num_channels; i++) {
	err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]);
	if (err)
	goto out_err_sq;
	}

	return 0;

	out_err_sq:
	for (j = 0; j < i; j++)
	mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]);
	i = hp->num_channels;
	out_err_rq:
	for (j = 0; j < i; j++)
	mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]);
	return err;
	}

	static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp)
	{
	int i;

	for (i = 0; i < hp->num_channels; i++) {
	mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]);
	if (!hp->peer_gone)
	mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);
	}
	}

	static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn,
	int curr_state, int next_state,
	u16 peer_vhca, u32 peer_sq)
	{
	u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {};
	void *rqc;

	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);

	if (next_state == MLX5_RQC_STATE_RDY) {
	MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq);
	MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca);
	}

	MLX5_SET(modify_rq_in, in, rq_state, curr_state);
	MLX5_SET(rqc, rqc, state, next_state);

	return mlx5_core_modify_rq(func_mdev, rqn, in);
	}

	static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn,
	int curr_state, int next_state,
	u16 peer_vhca, u32 peer_rq)
	{
	u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0};
	void *sqc;

	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);

	if (next_state == MLX5_SQC_STATE_RDY) {
	MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq);
	MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca);
	}

	MLX5_SET(modify_sq_in, in, sq_state, curr_state);
	MLX5_SET(sqc, sqc, state, next_state);

	return mlx5_core_modify_sq(peer_mdev, sqn, in);
	}

	static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp)
	{
	int i, j, err;

	/* set peer SQs */
	for (i = 0; i < hp->num_channels; i++) {
	err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i],
	MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
	MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]);
	if (err)
	goto err_modify_sq;
	}

	/* set func RQs */
	for (i = 0; i < hp->num_channels; i++) {
	err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i],
	MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY,
	MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]);
	if (err)
	goto err_modify_rq;
	}

	return 0;

	err_modify_rq:
	for (j = 0; j < i; j++)
	mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY,
	MLX5_RQC_STATE_RST, 0, 0);
	i = hp->num_channels;
	err_modify_sq:
	for (j = 0; j < i; j++)
	mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY,
	MLX5_SQC_STATE_RST, 0, 0);
	return err;
	}

	static void mlx5_hairpin_unpair_peer_sq(struct mlx5_hairpin *hp)
	{
	int i;

	for (i = 0; i < hp->num_channels; i++)
	mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY,
	MLX5_SQC_STATE_RST, 0, 0);
	}

	static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp)
	{
	int i;

	/* unset func RQs */
	for (i = 0; i < hp->num_channels; i++)
	mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY,
	MLX5_RQC_STATE_RST, 0, 0);
	/* unset peer SQs */
	if (!hp->peer_gone)
	mlx5_hairpin_unpair_peer_sq(hp);
	}

	struct mlx5_hairpin *
	mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev,
	struct mlx5_core_dev *peer_mdev,
	struct mlx5_hairpin_params *params)
	{
	struct mlx5_hairpin *hp;
	int size, err;

	size = sizeof(hp) + params->num_channels 2 * sizeof(u32);
	hp = kzalloc(size, GFP_KERNEL);
	if (!hp)
	return ERR_PTR(-ENOMEM);

	hp->func_mdev = func_mdev;
	hp->peer_mdev = peer_mdev;
	hp->num_channels = params->num_channels;

	hp->rqn = (void )hp + sizeof(hp);
	hp->sqn = hp->rqn + params->num_channels;

	/* alloc and pair func --> peer hairpin */
	err = mlx5_hairpin_create_queues(hp, params);
	if (err)
	goto err_create_queues;

	err = mlx5_hairpin_pair_queues(hp);
	if (err)
	goto err_pair_queues;

	return hp;

	err_pair_queues:
	mlx5_hairpin_destroy_queues(hp);
	err_create_queues:
	kfree(hp);
	return ERR_PTR(err);
	}

	void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp)
	{
	mlx5_hairpin_unpair_queues(hp);
	mlx5_hairpin_destroy_queues(hp);
	kfree(hp);
	}

	void mlx5_core_hairpin_clear_dead_peer(struct mlx5_hairpin *hp)
	{
	int i;

	mlx5_hairpin_unpair_peer_sq(hp);

	/* destroy peer SQ */
	for (i = 0; i < hp->num_channels; i++)
	mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);

	hp->peer_gone = true;
	}