drivers/media/platform/qcom/venus/hfi_venus.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
 * Copyright (C) 2017 Linaro Ltd.
 */

#include <linux/delay.h>
#include <linux/device.h>
#include <linux/dma-mapping.h>
#include <linux/interrupt.h>
#include <linux/iopoll.h>
#include <linux/kernel.h>
#include <linux/slab.h>

#include "core.h"
#include "hfi_cmds.h"
#include "hfi_msgs.h"
#include "hfi_venus.h"
#include "hfi_venus_io.h"
#include "firmware.h"

#define HFI_MASK_QHDR_TX_TYPE		0xff000000
#define HFI_MASK_QHDR_RX_TYPE		0x00ff0000
#define HFI_MASK_QHDR_PRI_TYPE		0x0000ff00
#define HFI_MASK_QHDR_ID_TYPE		0x000000ff

#define HFI_HOST_TO_CTRL_CMD_Q		0
#define HFI_CTRL_TO_HOST_MSG_Q		1
#define HFI_CTRL_TO_HOST_DBG_Q		2
#define HFI_MASK_QHDR_STATUS		0x000000ff

#define IFACEQ_NUM			3
#define IFACEQ_CMD_IDX			0
#define IFACEQ_MSG_IDX			1
#define IFACEQ_DBG_IDX			2
#define IFACEQ_MAX_BUF_COUNT		50
#define IFACEQ_MAX_PARALLEL_CLNTS	16
#define IFACEQ_DFLT_QHDR		0x01010000

#define POLL_INTERVAL_US		50

#define IFACEQ_MAX_PKT_SIZE		1024
#define IFACEQ_MED_PKT_SIZE		768
#define IFACEQ_MIN_PKT_SIZE		8
#define IFACEQ_VAR_SMALL_PKT_SIZE	100
#define IFACEQ_VAR_LARGE_PKT_SIZE	512
#define IFACEQ_VAR_HUGE_PKT_SIZE	(1024 * 12)

struct hfi_queue_table_header {
	u32 version;
	u32 size;
	u32 qhdr0_offset;
	u32 qhdr_size;
	u32 num_q;
	u32 num_active_q;
};

struct hfi_queue_header {
	u32 status;
	u32 start_addr;
	u32 type;
	u32 q_size;
	u32 pkt_size;
	u32 pkt_drop_cnt;
	u32 rx_wm;
	u32 tx_wm;
	u32 rx_req;
	u32 tx_req;
	u32 rx_irq_status;
	u32 tx_irq_status;
	u32 read_idx;
	u32 write_idx;
};

#define IFACEQ_TABLE_SIZE	\
	(sizeof(struct hfi_queue_table_header) +	\
	 sizeof(struct hfi_queue_header) * IFACEQ_NUM)

#define IFACEQ_QUEUE_SIZE	(IFACEQ_MAX_PKT_SIZE *	\
	IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)

#define IFACEQ_GET_QHDR_START_ADDR(ptr, i)	\
	(void *)(((ptr) + sizeof(struct hfi_queue_table_header)) +	\
		((i) * sizeof(struct hfi_queue_header)))

#define QDSS_SIZE		SZ_4K
#define SFR_SIZE		SZ_4K
#define QUEUE_SIZE		\
	(IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))

#define ALIGNED_QDSS_SIZE	ALIGN(QDSS_SIZE, SZ_4K)
#define ALIGNED_SFR_SIZE	ALIGN(SFR_SIZE, SZ_4K)
#define ALIGNED_QUEUE_SIZE	ALIGN(QUEUE_SIZE, SZ_4K)
#define SHARED_QSIZE		ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \
				      ALIGNED_QDSS_SIZE, SZ_1M)

struct mem_desc {
	dma_addr_t da;	/* device address */
	void *kva;	/* kernel virtual address */
	u32 size;
	unsigned long attrs;
};

struct iface_queue {
	struct hfi_queue_header *qhdr;
	struct mem_desc qmem;
};

enum venus_state {
	VENUS_STATE_DEINIT = 1,
	VENUS_STATE_INIT,
};

struct venus_hfi_device {
	struct venus_core *core;
	u32 irq_status;
	u32 last_packet_type;
	bool power_enabled;
	bool suspended;
	enum venus_state state;
	/* serialize read / write to the shared memory */
	struct mutex lock;
	struct completion pwr_collapse_prep;
	struct completion release_resource;
	struct mem_desc ifaceq_table;
	struct mem_desc sfr;
	struct iface_queue queues[IFACEQ_NUM];
	u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
	u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
};

static bool venus_pkt_debug;
int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
static bool venus_sys_idle_indicator;
static bool venus_fw_low_power_mode = true;
static int venus_hw_rsp_timeout = 1000;
static bool venus_fw_coverage;

static void venus_set_state(struct venus_hfi_device *hdev,
			    enum venus_state state)
{
	mutex_lock(&hdev->lock);
	hdev->state = state;
	mutex_unlock(&hdev->lock);
}

static bool venus_is_valid_state(struct venus_hfi_device *hdev)
{
	return hdev->state != VENUS_STATE_DEINIT;
}

static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)
{
	size_t pkt_size = *(u32 *)packet;

	if (!venus_pkt_debug)
		return;

	print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,
		       pkt_size, true);
}

static int venus_write_queue(struct venus_hfi_device *hdev,
			     struct iface_queue *queue,
			     void *packet, u32 *rx_req)
{
	struct hfi_queue_header *qhdr;
	u32 dwords, new_wr_idx;
	u32 empty_space, rd_idx, wr_idx, qsize;
	u32 *wr_ptr;

	if (!queue->qmem.kva)
		return -EINVAL;

	qhdr = queue->qhdr;
	if (!qhdr)
		return -EINVAL;

	venus_dump_packet(hdev, packet);

	dwords = (*(u32 *)packet) >> 2;
	if (!dwords)
		return -EINVAL;

	rd_idx = qhdr->read_idx;
	wr_idx = qhdr->write_idx;
	qsize = qhdr->q_size;
	/* ensure rd/wr indices's are read from memory */
	rmb();

	if (wr_idx >= rd_idx)
		empty_space = qsize - (wr_idx - rd_idx);
	else
		empty_space = rd_idx - wr_idx;

	if (empty_space <= dwords) {
		qhdr->tx_req = 1;
		/* ensure tx_req is updated in memory */
		wmb();
		return -ENOSPC;
	}

	qhdr->tx_req = 0;
	/* ensure tx_req is updated in memory */
	wmb();

	new_wr_idx = wr_idx + dwords;
	wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));
	if (new_wr_idx < qsize) {
		memcpy(wr_ptr, packet, dwords << 2);
	} else {
		size_t len;

		new_wr_idx -= qsize;
		len = (dwords - new_wr_idx) << 2;
		memcpy(wr_ptr, packet, len);
		memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);
	}

	/* make sure packet is written before updating the write index */
	wmb();

	qhdr->write_idx = new_wr_idx;
	*rx_req = qhdr->rx_req ? 1 : 0;

	/* make sure write index is updated before an interrupt is raised */
	mb();

	return 0;
}

static int venus_read_queue(struct venus_hfi_device *hdev,
			    struct iface_queue *queue, void *pkt, u32 *tx_req)
{
	struct hfi_queue_header *qhdr;
	u32 dwords, new_rd_idx;
	u32 rd_idx, wr_idx, type, qsize;
	u32 *rd_ptr;
	u32 recv_request = 0;
	int ret = 0;

	if (!queue->qmem.kva)
		return -EINVAL;

	qhdr = queue->qhdr;
	if (!qhdr)
		return -EINVAL;

	type = qhdr->type;
	rd_idx = qhdr->read_idx;
	wr_idx = qhdr->write_idx;
	qsize = qhdr->q_size;

	/* make sure data is valid before using it */
	rmb();

	/*
	 * Do not set receive request for debug queue, if set, Venus generates
	 * interrupt for debug messages even when there is no response message
	 * available. In general debug queue will not become full as it is being
	 * emptied out for every interrupt from Venus. Venus will anyway
	 * generates interrupt if it is full.
	 */
	if (type & HFI_CTRL_TO_HOST_MSG_Q)
		recv_request = 1;

	if (rd_idx == wr_idx) {
		qhdr->rx_req = recv_request;
		*tx_req = 0;
		/* update rx_req field in memory */
		wmb();
		return -ENODATA;
	}

	rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));
	dwords = *rd_ptr >> 2;
	if (!dwords)
		return -EINVAL;

	new_rd_idx = rd_idx + dwords;
	if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {
		if (new_rd_idx < qsize) {
			memcpy(pkt, rd_ptr, dwords << 2);
		} else {
			size_t len;

			new_rd_idx -= qsize;
			len = (dwords - new_rd_idx) << 2;
			memcpy(pkt, rd_ptr, len);
			memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);
		}
	} else {
		/* bad packet received, dropping */
		new_rd_idx = qhdr->write_idx;
		ret = -EBADMSG;
	}

	/* ensure the packet is read before updating read index */
	rmb();

	qhdr->read_idx = new_rd_idx;
	/* ensure updating read index */
	wmb();

	rd_idx = qhdr->read_idx;
	wr_idx = qhdr->write_idx;
	/* ensure rd/wr indices are read from memory */
	rmb();

	if (rd_idx != wr_idx)
		qhdr->rx_req = 0;
	else
		qhdr->rx_req = recv_request;

	*tx_req = qhdr->tx_req ? 1 : 0;

	/* ensure rx_req is stored to memory and tx_req is loaded from memory */
	mb();

	venus_dump_packet(hdev, pkt);

	return ret;
}

static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,
		       u32 size)
{
	struct device *dev = hdev->core->dev;

	desc->attrs = DMA_ATTR_WRITE_COMBINE;
	desc->size = ALIGN(size, SZ_4K);

	desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,
				    desc->attrs);
	if (!desc->kva)
		return -ENOMEM;

	return 0;
}

static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)
{
	struct device *dev = hdev->core->dev;

	dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);
}

static void venus_set_registers(struct venus_hfi_device *hdev)
{
	const struct venus_resources *res = hdev->core->res;
	const struct reg_val *tbl = res->reg_tbl;
	unsigned int count = res->reg_tbl_size;
	unsigned int i;

	for (i = 0; i < count; i++)
		writel(tbl[i].value, hdev->core->base + tbl[i].reg);
}

static void venus_soft_int(struct venus_hfi_device *hdev)
{
	void __iomem *cpu_ic_base = hdev->core->cpu_ic_base;
	u32 clear_bit;

	if (IS_V6(hdev->core))
		clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6);
	else
		clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT);

	writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT);
}

static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,
					 void *pkt, bool sync)
{
	struct device *dev = hdev->core->dev;
	struct hfi_pkt_hdr *cmd_packet;
	struct iface_queue *queue;
	u32 rx_req;
	int ret;

	if (!venus_is_valid_state(hdev))
		return -EINVAL;

	cmd_packet = (struct hfi_pkt_hdr *)pkt;
	hdev->last_packet_type = cmd_packet->pkt_type;

	queue = &hdev->queues[IFACEQ_CMD_IDX];

	ret = venus_write_queue(hdev, queue, pkt, &rx_req);
	if (ret) {
		dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);
		return ret;
	}

	if (sync) {
		/*
		 * Inform video hardware to raise interrupt for synchronous
		 * commands
		 */
		queue = &hdev->queues[IFACEQ_MSG_IDX];
		queue->qhdr->rx_req = 1;
		/* ensure rx_req is updated in memory */
		wmb();
	}

	if (rx_req)
		venus_soft_int(hdev);

	return 0;
}

static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync)
{
	int ret;

	mutex_lock(&hdev->lock);
	ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync);
	mutex_unlock(&hdev->lock);

	return ret;
}

static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,
				       u32 size, u32 addr, void *cookie)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);
	struct hfi_sys_set_resource_pkt *pkt;
	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
	int ret;

	if (id == VIDC_RESOURCE_NONE)
		return 0;

	pkt = (struct hfi_sys_set_resource_pkt *)packet;

	ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);
	if (ret)
		return ret;

	ret = venus_iface_cmdq_write(hdev, pkt, false);
	if (ret)
		return ret;

	return 0;
}

static int venus_boot_core(struct venus_hfi_device *hdev)
{
	struct device *dev = hdev->core->dev;
	static const unsigned int max_tries = 100;
	u32 ctrl_status = 0, mask_val;
	unsigned int count = 0;
	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
	void __iomem *wrapper_base = hdev->core->wrapper_base;
	int ret = 0;

	writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT);
	if (IS_V6(hdev->core)) {
		mask_val = readl(wrapper_base + WRAPPER_INTR_MASK);
		mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 |
			      WRAPPER_INTR_MASK_A2HCPU_MASK);
	} else {
		mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK;
	}
	writel(mask_val, wrapper_base + WRAPPER_INTR_MASK);
	writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3);

	while (!ctrl_status && count < max_tries) {
		ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
		if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {
			dev_err(dev, "invalid setting for UC_REGION\n");
			ret = -EINVAL;
			break;
		}

		usleep_range(500, 1000);
		count++;
	}

	if (count >= max_tries)
		ret = -ETIMEDOUT;

	if (IS_V6(hdev->core)) {
		writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6);
		writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6);
	}

	return ret;
}

static u32 venus_hwversion(struct venus_hfi_device *hdev)
{
	struct device *dev = hdev->core->dev;
	void __iomem *wrapper_base = hdev->core->wrapper_base;
	u32 ver;
	u32 major, minor, step;

	ver = readl(wrapper_base + WRAPPER_HW_VERSION);
	major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;
	major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;
	minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;
	minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
	step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;

	dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step);

	return major;
}

static int venus_run(struct venus_hfi_device *hdev)
{
	struct device *dev = hdev->core->dev;
	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
	int ret;

	/*
	 * Re-program all of the registers that get reset as a result of
	 * regulator_disable() and _enable()
	 */
	venus_set_registers(hdev);

	writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR);
	writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE);
	writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2);
	writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1);
	if (hdev->sfr.da)
		writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR);

	ret = venus_boot_core(hdev);
	if (ret) {
		dev_err(dev, "failed to reset venus core\n");
		return ret;
	}

	venus_hwversion(hdev);

	return 0;
}

static int venus_halt_axi(struct venus_hfi_device *hdev)
{
	void __iomem *wrapper_base = hdev->core->wrapper_base;
	void __iomem *vbif_base = hdev->core->vbif_base;
	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
	void __iomem *aon_base = hdev->core->aon_base;
	struct device *dev = hdev->core->dev;
	u32 val;
	u32 mask_val;
	int ret;

	if (IS_V6(hdev->core)) {
		writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6);

		if (hdev->core->res->num_vpp_pipes == 1)
			goto skip_aon_mvp_noc;

		writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL);
		ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS,
					 val,
					 val & BIT(0),
					 POLL_INTERVAL_US,
					 VBIF_AXI_HALT_ACK_TIMEOUT_US);
		if (ret)
			return -ETIMEDOUT;

skip_aon_mvp_noc:
		mask_val = (BIT(2) | BIT(1) | BIT(0));
		writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);

		writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
		ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6,
					 val,
					 val == 0,
					 POLL_INTERVAL_US,
					 VBIF_AXI_HALT_ACK_TIMEOUT_US);

		if (ret) {
			dev_err(dev, "DBLP Release: lpi_status %x\n", val);
			return -ETIMEDOUT;
		}
		return 0;
	}

	if (IS_V4(hdev->core)) {
		val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT);
		val |= WRAPPER_CPU_AXI_HALT_HALT;
		writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT);

		ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS,
					 val,
					 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,
					 POLL_INTERVAL_US,
					 VBIF_AXI_HALT_ACK_TIMEOUT_US);
		if (ret) {
			dev_err(dev, "AXI bus port halt timeout\n");
			return ret;
		}

		return 0;
	}

	/* Halt AXI and AXI IMEM VBIF Access */
	val = readl(vbif_base + VBIF_AXI_HALT_CTRL0);
	val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;
	writel(val, vbif_base + VBIF_AXI_HALT_CTRL0);

	/* Request for AXI bus port halt */
	ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val,
				 val & VBIF_AXI_HALT_CTRL1_HALT_ACK,
				 POLL_INTERVAL_US,
				 VBIF_AXI_HALT_ACK_TIMEOUT_US);
	if (ret) {
		dev_err(dev, "AXI bus port halt timeout\n");
		return ret;
	}

	return 0;
}

static int venus_power_off(struct venus_hfi_device *hdev)
{
	int ret;

	if (!hdev->power_enabled)
		return 0;

	ret = venus_set_hw_state_suspend(hdev->core);
	if (ret)
		return ret;

	ret = venus_halt_axi(hdev);
	if (ret)
		return ret;

	hdev->power_enabled = false;

	return 0;
}

static int venus_power_on(struct venus_hfi_device *hdev)
{
	int ret;

	if (hdev->power_enabled)
		return 0;

	ret = venus_set_hw_state_resume(hdev->core);
	if (ret)
		goto err;

	ret = venus_run(hdev);
	if (ret)
		goto err_suspend;

	hdev->power_enabled = true;

	return 0;

err_suspend:
	venus_set_hw_state_suspend(hdev->core);
err:
	hdev->power_enabled = false;
	return ret;
}

static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,
					void *pkt)
{
	struct iface_queue *queue;
	u32 tx_req;
	int ret;

	if (!venus_is_valid_state(hdev))
		return -EINVAL;

	queue = &hdev->queues[IFACEQ_MSG_IDX];

	ret = venus_read_queue(hdev, queue, pkt, &tx_req);
	if (ret)
		return ret;

	if (tx_req)
		venus_soft_int(hdev);

	return 0;
}

static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)
{
	int ret;

	mutex_lock(&hdev->lock);
	ret = venus_iface_msgq_read_nolock(hdev, pkt);
	mutex_unlock(&hdev->lock);

	return ret;
}

static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,
					void *pkt)
{
	struct iface_queue *queue;
	u32 tx_req;
	int ret;

	ret = venus_is_valid_state(hdev);
	if (!ret)
		return -EINVAL;

	queue = &hdev->queues[IFACEQ_DBG_IDX];

	ret = venus_read_queue(hdev, queue, pkt, &tx_req);
	if (ret)
		return ret;

	if (tx_req)
		venus_soft_int(hdev);

	return 0;
}

static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)
{
	int ret;

	if (!pkt)
		return -EINVAL;

	mutex_lock(&hdev->lock);
	ret = venus_iface_dbgq_read_nolock(hdev, pkt);
	mutex_unlock(&hdev->lock);

	return ret;
}

static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)
{
	qhdr->status = 1;
	qhdr->type = IFACEQ_DFLT_QHDR;
	qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;
	qhdr->pkt_size = 0;
	qhdr->rx_wm = 1;
	qhdr->tx_wm = 1;
	qhdr->rx_req = 1;
	qhdr->tx_req = 0;
	qhdr->rx_irq_status = 0;
	qhdr->tx_irq_status = 0;
	qhdr->read_idx = 0;
	qhdr->write_idx = 0;
}

static void venus_interface_queues_release(struct venus_hfi_device *hdev)
{
	mutex_lock(&hdev->lock);

	venus_free(hdev, &hdev->ifaceq_table);
	venus_free(hdev, &hdev->sfr);

	memset(hdev->queues, 0, sizeof(hdev->queues));
	memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));
	memset(&hdev->sfr, 0, sizeof(hdev->sfr));

	mutex_unlock(&hdev->lock);
}

static int venus_interface_queues_init(struct venus_hfi_device *hdev)
{
	struct hfi_queue_table_header *tbl_hdr;
	struct iface_queue *queue;
	struct hfi_sfr *sfr;
	struct mem_desc desc = {0};
	unsigned int offset;
	unsigned int i;
	int ret;

	ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);
	if (ret)
		return ret;

	hdev->ifaceq_table = desc;
	offset = IFACEQ_TABLE_SIZE;

	for (i = 0; i < IFACEQ_NUM; i++) {
		queue = &hdev->queues[i];
		queue->qmem.da = desc.da + offset;
		queue->qmem.kva = desc.kva + offset;
		queue->qmem.size = IFACEQ_QUEUE_SIZE;
		offset += queue->qmem.size;
		queue->qhdr =
			IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);

		venus_set_qhdr_defaults(queue->qhdr);

		queue->qhdr->start_addr = queue->qmem.da;

		if (i == IFACEQ_CMD_IDX)
			queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
		else if (i == IFACEQ_MSG_IDX)
			queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
		else if (i == IFACEQ_DBG_IDX)
			queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
	}

	tbl_hdr = hdev->ifaceq_table.kva;
	tbl_hdr->version = 0;
	tbl_hdr->size = IFACEQ_TABLE_SIZE;
	tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
	tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
	tbl_hdr->num_q = IFACEQ_NUM;
	tbl_hdr->num_active_q = IFACEQ_NUM;

	/*
	 * Set receive request to zero on debug queue as there is no
	 * need of interrupt from video hardware for debug messages
	 */
	queue = &hdev->queues[IFACEQ_DBG_IDX];
	queue->qhdr->rx_req = 0;

	ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);
	if (ret) {
		hdev->sfr.da = 0;
	} else {
		hdev->sfr = desc;
		sfr = hdev->sfr.kva;
		sfr->buf_size = ALIGNED_SFR_SIZE;
	}

	/* ensure table and queue header structs are settled in memory */
	wmb();

	return 0;
}

static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)
{
	struct hfi_sys_set_property_pkt *pkt;
	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
	int ret;

	pkt = (struct hfi_sys_set_property_pkt *)packet;

	pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);

	ret = venus_iface_cmdq_write(hdev, pkt, false);
	if (ret)
		return ret;

	return 0;
}

static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)
{
	struct hfi_sys_set_property_pkt *pkt;
	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
	int ret;

	pkt = (struct hfi_sys_set_property_pkt *)packet;

	pkt_sys_coverage_config(pkt, mode);

	ret = venus_iface_cmdq_write(hdev, pkt, false);
	if (ret)
		return ret;

	return 0;
}

static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,
				      bool enable)
{
	struct hfi_sys_set_property_pkt *pkt;
	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
	int ret;

	if (!enable)
		return 0;

	pkt = (struct hfi_sys_set_property_pkt *)packet;

	pkt_sys_idle_indicator(pkt, enable);

	ret = venus_iface_cmdq_write(hdev, pkt, false);
	if (ret)
		return ret;

	return 0;
}

static int venus_sys_set_power_control(struct venus_hfi_device *hdev,
				       bool enable)
{
	struct hfi_sys_set_property_pkt *pkt;
	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
	int ret;

	pkt = (struct hfi_sys_set_property_pkt *)packet;

	pkt_sys_power_control(pkt, enable);

	ret = venus_iface_cmdq_write(hdev, pkt, false);
	if (ret)
		return ret;

	return 0;
}

static int venus_get_queue_size(struct venus_hfi_device *hdev,
				unsigned int index)
{
	struct hfi_queue_header *qhdr;

	if (index >= IFACEQ_NUM)
		return -EINVAL;

	qhdr = hdev->queues[index].qhdr;
	if (!qhdr)
		return -EINVAL;

	return abs(qhdr->read_idx - qhdr->write_idx);
}

static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)
{
	struct device *dev = hdev->core->dev;
	int ret;

	ret = venus_sys_set_debug(hdev, venus_fw_debug);
	if (ret)
		dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);

	/*
	 * Idle indicator is disabled by default on some 4xx firmware versions,
	 * enable it explicitly in order to make suspend functional by checking
	 * WFI (wait-for-interrupt) bit.
	 */
	if (IS_V4(hdev->core) || IS_V6(hdev->core))
		venus_sys_idle_indicator = true;

	ret = venus_sys_set_idle_message(hdev, venus_sys_idle_indicator);
	if (ret)
		dev_warn(dev, "setting idle response ON failed (%d)\n", ret);

	ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);
	if (ret)
		dev_warn(dev, "setting hw power collapse ON failed (%d)\n",
			 ret);

	return ret;
}

static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	struct hfi_session_pkt pkt;

	pkt_session_cmd(&pkt, pkt_type, inst);

	return venus_iface_cmdq_write(hdev, &pkt, sync);
}

static void venus_flush_debug_queue(struct venus_hfi_device *hdev)
{
	struct device *dev = hdev->core->dev;
	void *packet = hdev->dbg_buf;

	while (!venus_iface_dbgq_read(hdev, packet)) {
		struct hfi_msg_sys_coverage_pkt *pkt = packet;

		if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
			struct hfi_msg_sys_debug_pkt *pkt = packet;

			dev_dbg(dev, VDBGFW "%s", pkt->msg_data);
		}
	}
}

static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,
					bool wait)
{
	unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);
	struct hfi_sys_pc_prep_pkt pkt;
	int ret;

	init_completion(&hdev->pwr_collapse_prep);

	pkt_sys_pc_prep(&pkt);

	ret = venus_iface_cmdq_write(hdev, &pkt, false);
	if (ret)
		return ret;

	if (!wait)
		return 0;

	ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);
	if (!ret) {
		venus_flush_debug_queue(hdev);
		return -ETIMEDOUT;
	}

	return 0;
}

static int venus_are_queues_empty(struct venus_hfi_device *hdev)
{
	int ret1, ret2;

	ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);
	if (ret1 < 0)
		return ret1;

	ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);
	if (ret2 < 0)
		return ret2;

	if (!ret1 && !ret2)
		return 1;

	return 0;
}

static void venus_sfr_print(struct venus_hfi_device *hdev)
{
	struct device *dev = hdev->core->dev;
	struct hfi_sfr *sfr = hdev->sfr.kva;
	void *p;

	if (!sfr)
		return;

	p = memchr(sfr->data, '\0', sfr->buf_size);
	/*
	 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates
	 * that Venus is in the process of crashing.
	 */
	if (!p)
		sfr->data[sfr->buf_size - 1] = '\0';

	dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);
}

static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,
					void *packet)
{
	struct hfi_msg_event_notify_pkt *event_pkt = packet;

	if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)
		return;

	venus_set_state(hdev, VENUS_STATE_DEINIT);

	venus_sfr_print(hdev);
}

static irqreturn_t venus_isr_thread(struct venus_core *core)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);
	const struct venus_resources *res;
	void *pkt;
	u32 msg_ret;

	if (!hdev)
		return IRQ_NONE;

	res = hdev->core->res;
	pkt = hdev->pkt_buf;


	while (!venus_iface_msgq_read(hdev, pkt)) {
		msg_ret = hfi_process_msg_packet(core, pkt);
		switch (msg_ret) {
		case HFI_MSG_EVENT_NOTIFY:
			venus_process_msg_sys_error(hdev, pkt);
			break;
		case HFI_MSG_SYS_INIT:
			venus_hfi_core_set_resource(core, res->vmem_id,
						    res->vmem_size,
						    res->vmem_addr,
						    hdev);
			break;
		case HFI_MSG_SYS_RELEASE_RESOURCE:
			complete(&hdev->release_resource);
			break;
		case HFI_MSG_SYS_PC_PREP:
			complete(&hdev->pwr_collapse_prep);
			break;
		default:
			break;
		}
	}

	venus_flush_debug_queue(hdev);

	return IRQ_HANDLED;
}

static irqreturn_t venus_isr(struct venus_core *core)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);
	u32 status;
	void __iomem *cpu_cs_base;
	void __iomem *wrapper_base;

	if (!hdev)
		return IRQ_NONE;

	cpu_cs_base = hdev->core->cpu_cs_base;
	wrapper_base = hdev->core->wrapper_base;

	status = readl(wrapper_base + WRAPPER_INTR_STATUS);
	if (IS_V6(core)) {
		if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
		    status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 ||
		    status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
			hdev->irq_status = status;
	} else {
		if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
		    status & WRAPPER_INTR_STATUS_A2HWD_MASK ||
		    status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
			hdev->irq_status = status;
	}
	writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR);
	if (!IS_V6(core))
		writel(status, wrapper_base + WRAPPER_INTR_CLEAR);

	return IRQ_WAKE_THREAD;
}

static int venus_core_init(struct venus_core *core)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);
	struct device *dev = core->dev;
	struct hfi_sys_get_property_pkt version_pkt;
	struct hfi_sys_init_pkt pkt;
	int ret;

	pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);

	venus_set_state(hdev, VENUS_STATE_INIT);

	ret = venus_iface_cmdq_write(hdev, &pkt, false);
	if (ret)
		return ret;

	pkt_sys_image_version(&version_pkt);

	ret = venus_iface_cmdq_write(hdev, &version_pkt, false);
	if (ret)
		dev_warn(dev, "failed to send image version pkt to fw\n");

	ret = venus_sys_set_default_properties(hdev);
	if (ret)
		return ret;

	return 0;
}

static int venus_core_deinit(struct venus_core *core)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);

	venus_set_state(hdev, VENUS_STATE_DEINIT);
	hdev->suspended = true;
	hdev->power_enabled = false;

	return 0;
}

static int venus_core_ping(struct venus_core *core, u32 cookie)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);
	struct hfi_sys_ping_pkt pkt;

	pkt_sys_ping(&pkt, cookie);

	return venus_iface_cmdq_write(hdev, &pkt, false);
}

static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);
	struct hfi_sys_test_ssr_pkt pkt;
	int ret;

	ret = pkt_sys_ssr_cmd(&pkt, trigger_type);
	if (ret)
		return ret;

	return venus_iface_cmdq_write(hdev, &pkt, false);
}

static int venus_session_init(struct venus_inst *inst, u32 session_type,
			      u32 codec)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	struct hfi_session_init_pkt pkt;
	int ret;

	ret = venus_sys_set_debug(hdev, venus_fw_debug);
	if (ret)
		goto err;

	ret = pkt_session_init(&pkt, inst, session_type, codec);
	if (ret)
		goto err;

	ret = venus_iface_cmdq_write(hdev, &pkt, true);
	if (ret)
		goto err;

	return 0;

err:
	venus_flush_debug_queue(hdev);
	return ret;
}

static int venus_session_end(struct venus_inst *inst)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	struct device *dev = hdev->core->dev;

	if (venus_fw_coverage) {
		if (venus_sys_set_coverage(hdev, venus_fw_coverage))
			dev_warn(dev, "fw coverage msg ON failed\n");
	}

	return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true);
}

static int venus_session_abort(struct venus_inst *inst)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);

	venus_flush_debug_queue(hdev);

	return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true);
}

static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	struct hfi_session_flush_pkt pkt;
	int ret;

	ret = pkt_session_flush(&pkt, inst, flush_mode);
	if (ret)
		return ret;

	return venus_iface_cmdq_write(hdev, &pkt, true);
}

static int venus_session_start(struct venus_inst *inst)
{
	return venus_session_cmd(inst, HFI_CMD_SESSION_START, true);
}

static int venus_session_stop(struct venus_inst *inst)
{
	return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true);
}

static int venus_session_continue(struct venus_inst *inst)
{
	return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false);
}

static int venus_session_etb(struct venus_inst *inst,
			     struct hfi_frame_data *in_frame)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	u32 session_type = inst->session_type;
	int ret;

	if (session_type == VIDC_SESSION_TYPE_DEC) {
		struct hfi_session_empty_buffer_compressed_pkt pkt;

		ret = pkt_session_etb_decoder(&pkt, inst, in_frame);
		if (ret)
			return ret;

		ret = venus_iface_cmdq_write(hdev, &pkt, false);
	} else if (session_type == VIDC_SESSION_TYPE_ENC) {
		struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;

		ret = pkt_session_etb_encoder(&pkt, inst, in_frame);
		if (ret)
			return ret;

		ret = venus_iface_cmdq_write(hdev, &pkt, false);
	} else {
		ret = -EINVAL;
	}

	return ret;
}

static int venus_session_ftb(struct venus_inst *inst,
			     struct hfi_frame_data *out_frame)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	struct hfi_session_fill_buffer_pkt pkt;
	int ret;

	ret = pkt_session_ftb(&pkt, inst, out_frame);
	if (ret)
		return ret;

	return venus_iface_cmdq_write(hdev, &pkt, false);
}

static int venus_session_set_buffers(struct venus_inst *inst,
				     struct hfi_buffer_desc *bd)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	struct hfi_session_set_buffers_pkt *pkt;
	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
	int ret;

	if (bd->buffer_type == HFI_BUFFER_INPUT)
		return 0;

	pkt = (struct hfi_session_set_buffers_pkt *)packet;

	ret = pkt_session_set_buffers(pkt, inst, bd);
	if (ret)
		return ret;

	return venus_iface_cmdq_write(hdev, pkt, false);
}

static int venus_session_unset_buffers(struct venus_inst *inst,
				       struct hfi_buffer_desc *bd)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	struct hfi_session_release_buffer_pkt *pkt;
	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
	int ret;

	if (bd->buffer_type == HFI_BUFFER_INPUT)
		return 0;

	pkt = (struct hfi_session_release_buffer_pkt *)packet;

	ret = pkt_session_unset_buffers(pkt, inst, bd);
	if (ret)
		return ret;

	return venus_iface_cmdq_write(hdev, pkt, true);
}

static int venus_session_load_res(struct venus_inst *inst)
{
	return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true);
}

static int venus_session_release_res(struct venus_inst *inst)
{
	return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true);
}

static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
				       u32 seq_hdr_len)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	struct hfi_session_parse_sequence_header_pkt *pkt;
	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
	int ret;

	pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;

	ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);
	if (ret)
		return ret;

	ret = venus_iface_cmdq_write(hdev, pkt, false);
	if (ret)
		return ret;

	return 0;
}

static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
				     u32 seq_hdr_len)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	struct hfi_session_get_sequence_header_pkt *pkt;
	u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
	int ret;

	pkt = (struct hfi_session_get_sequence_header_pkt *)packet;

	ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);
	if (ret)
		return ret;

	return venus_iface_cmdq_write(hdev, pkt, false);
}

static int venus_session_set_property(struct venus_inst *inst, u32 ptype,
				      void *pdata)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	struct hfi_session_set_property_pkt *pkt;
	u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
	int ret;

	pkt = (struct hfi_session_set_property_pkt *)packet;

	ret = pkt_session_set_property(pkt, inst, ptype, pdata);
	if (ret == -ENOTSUPP)
		return 0;
	if (ret)
		return ret;

	return venus_iface_cmdq_write(hdev, pkt, false);
}

static int venus_session_get_property(struct venus_inst *inst, u32 ptype)
{
	struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
	struct hfi_session_get_property_pkt pkt;
	int ret;

	ret = pkt_session_get_property(&pkt, inst, ptype);
	if (ret)
		return ret;

	return venus_iface_cmdq_write(hdev, &pkt, true);
}

static int venus_resume(struct venus_core *core)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);
	int ret = 0;

	mutex_lock(&hdev->lock);

	if (!hdev->suspended)
		goto unlock;

	ret = venus_power_on(hdev);

unlock:
	if (!ret)
		hdev->suspended = false;

	mutex_unlock(&hdev->lock);

	return ret;
}

static int venus_suspend_1xx(struct venus_core *core)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);
	struct device *dev = core->dev;
	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
	u32 ctrl_status;
	int ret;

	if (!hdev->power_enabled || hdev->suspended)
		return 0;

	mutex_lock(&hdev->lock);
	ret = venus_is_valid_state(hdev);
	mutex_unlock(&hdev->lock);

	if (!ret) {
		dev_err(dev, "bad state, cannot suspend\n");
		return -EINVAL;
	}

	ret = venus_prepare_power_collapse(hdev, true);
	if (ret) {
		dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
		return ret;
	}

	mutex_lock(&hdev->lock);

	if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {
		mutex_unlock(&hdev->lock);
		return -EINVAL;
	}

	ret = venus_are_queues_empty(hdev);
	if (ret < 0 || !ret) {
		mutex_unlock(&hdev->lock);
		return -EINVAL;
	}

	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
	if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {
		mutex_unlock(&hdev->lock);
		return -EINVAL;
	}

	ret = venus_power_off(hdev);
	if (ret) {
		mutex_unlock(&hdev->lock);
		return ret;
	}

	hdev->suspended = true;

	mutex_unlock(&hdev->lock);

	return 0;
}

static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)
{
	void __iomem *wrapper_base = hdev->core->wrapper_base;
	void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
	u32 ctrl_status, cpu_status;

	if (IS_V6(hdev->core))
		cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
	else
		cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);

	if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
	    ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
		return true;

	return false;
}

static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)
{
	void __iomem *wrapper_base = hdev->core->wrapper_base;
	void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
	u32 ctrl_status, cpu_status;

	if (IS_V6(hdev->core))
		cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
	else
		cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);

	if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
	    ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
		return true;

	return false;
}

static int venus_suspend_3xx(struct venus_core *core)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);
	struct device *dev = core->dev;
	void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
	u32 ctrl_status;
	bool val;
	int ret;

	if (!hdev->power_enabled || hdev->suspended)
		return 0;

	mutex_lock(&hdev->lock);
	ret = venus_is_valid_state(hdev);
	mutex_unlock(&hdev->lock);

	if (!ret) {
		dev_err(dev, "bad state, cannot suspend\n");
		return -EINVAL;
	}

	ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
	if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
		goto power_off;

	/*
	 * Power collapse sequence for Venus 3xx and 4xx versions:
	 * 1. Check for ARM9 and video core to be idle by checking WFI bit
	 *    (bit 0) in CPU status register and by checking Idle (bit 30) in
	 *    Control status register for video core.
	 * 2. Send a command to prepare for power collapse.
	 * 3. Check for WFI and PC_READY bits.
	 */
	ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,
				 1500, 100 * 1500);
	if (ret)
		return ret;

	ret = venus_prepare_power_collapse(hdev, false);
	if (ret) {
		dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
		return ret;
	}

	ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,
				 1500, 100 * 1500);
	if (ret)
		return ret;

power_off:
	mutex_lock(&hdev->lock);

	ret = venus_power_off(hdev);
	if (ret) {
		dev_err(dev, "venus_power_off (%d)\n", ret);
		mutex_unlock(&hdev->lock);
		return ret;
	}

	hdev->suspended = true;

	mutex_unlock(&hdev->lock);

	return 0;
}

static int venus_suspend(struct venus_core *core)
{
	if (IS_V3(core) || IS_V4(core) || IS_V6(core))
		return venus_suspend_3xx(core);

	return venus_suspend_1xx(core);
}

static const struct hfi_ops venus_hfi_ops = {
	.core_init			= venus_core_init,
	.core_deinit			= venus_core_deinit,
	.core_ping			= venus_core_ping,
	.core_trigger_ssr		= venus_core_trigger_ssr,

	.session_init			= venus_session_init,
	.session_end			= venus_session_end,
	.session_abort			= venus_session_abort,
	.session_flush			= venus_session_flush,
	.session_start			= venus_session_start,
	.session_stop			= venus_session_stop,
	.session_continue		= venus_session_continue,
	.session_etb			= venus_session_etb,
	.session_ftb			= venus_session_ftb,
	.session_set_buffers		= venus_session_set_buffers,
	.session_unset_buffers		= venus_session_unset_buffers,
	.session_load_res		= venus_session_load_res,
	.session_release_res		= venus_session_release_res,
	.session_parse_seq_hdr		= venus_session_parse_seq_hdr,
	.session_get_seq_hdr		= venus_session_get_seq_hdr,
	.session_set_property		= venus_session_set_property,
	.session_get_property		= venus_session_get_property,

	.resume				= venus_resume,
	.suspend			= venus_suspend,

	.isr				= venus_isr,
	.isr_thread			= venus_isr_thread,
};

void venus_hfi_destroy(struct venus_core *core)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);

	core->priv = NULL;
	venus_interface_queues_release(hdev);
	mutex_destroy(&hdev->lock);
	kfree(hdev);
	core->ops = NULL;
}

int venus_hfi_create(struct venus_core *core)
{
	struct venus_hfi_device *hdev;
	int ret;

	hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
	if (!hdev)
		return -ENOMEM;

	mutex_init(&hdev->lock);

	hdev->core = core;
	hdev->suspended = true;
	core->priv = hdev;
	core->ops = &venus_hfi_ops;

	ret = venus_interface_queues_init(hdev);
	if (ret)
		goto err_kfree;

	return 0;

err_kfree:
	kfree(hdev);
	core->priv = NULL;
	core->ops = NULL;
	return ret;
}

void venus_hfi_queues_reinit(struct venus_core *core)
{
	struct venus_hfi_device *hdev = to_hfi_priv(core);
	struct hfi_queue_table_header *tbl_hdr;
	struct iface_queue *queue;
	struct hfi_sfr *sfr;
	unsigned int i;

	mutex_lock(&hdev->lock);

	for (i = 0; i < IFACEQ_NUM; i++) {
		queue = &hdev->queues[i];
		queue->qhdr =
			IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);

		venus_set_qhdr_defaults(queue->qhdr);

		queue->qhdr->start_addr = queue->qmem.da;

		if (i == IFACEQ_CMD_IDX)
			queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
		else if (i == IFACEQ_MSG_IDX)
			queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
		else if (i == IFACEQ_DBG_IDX)
			queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
	}

	tbl_hdr = hdev->ifaceq_table.kva;
	tbl_hdr->version = 0;
	tbl_hdr->size = IFACEQ_TABLE_SIZE;
	tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
	tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
	tbl_hdr->num_q = IFACEQ_NUM;
	tbl_hdr->num_active_q = IFACEQ_NUM;

	/*
	 * Set receive request to zero on debug queue as there is no
	 * need of interrupt from video hardware for debug messages
	 */
	queue = &hdev->queues[IFACEQ_DBG_IDX];
	queue->qhdr->rx_req = 0;

	sfr = hdev->sfr.kva;
	sfr->buf_size = ALIGNED_SFR_SIZE;

	/* ensure table and queue header structs are settled in memory */
	wmb();

	mutex_unlock(&hdev->lock);
}