drivers/dma/dw-edma/dw-hdma-v0-core.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (c) 2023 Cai Huoqing
  * Synopsys DesignWare HDMA v0 core
  */

 #include <linux/bitfield.h>
 #include <linux/irqreturn.h>
 #include <linux/io-64-nonatomic-lo-hi.h>

 #include "dw-edma-core.h"
 #include "dw-hdma-v0-core.h"
 #include "dw-hdma-v0-regs.h"
 #include "dw-hdma-v0-debugfs.h"

 enum dw_hdma_control {
 	DW_HDMA_V0_CB					= BIT(0),
 	DW_HDMA_V0_TCB					= BIT(1),
 	DW_HDMA_V0_LLP					= BIT(2),
 	DW_HDMA_V0_LWIE					= BIT(3),
 	DW_HDMA_V0_RWIE					= BIT(4),
 	DW_HDMA_V0_CCS					= BIT(8),
 	DW_HDMA_V0_LLE					= BIT(9),
 };

 static inline struct dw_hdma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
 {
 	return dw->chip->reg_base;
 }

 static inline struct dw_hdma_v0_ch_regs __iomem *
 __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
 {
 	if (dir == EDMA_DIR_WRITE)
 		return &(__dw_regs(dw)->ch[ch].wr);
 	else
 		return &(__dw_regs(dw)->ch[ch].rd);
 }

 #define SET_CH_32(dw, dir, ch, name, value) \
 	writel(value, &(__dw_ch_regs(dw, dir, ch)->name))

 #define GET_CH_32(dw, dir, ch, name) \
 	readl(&(__dw_ch_regs(dw, dir, ch)->name))

 #define SET_BOTH_CH_32(dw, ch, name, value) \
 	do {					\
 		writel(value, &(__dw_ch_regs(dw, EDMA_DIR_WRITE, ch)->name));	\
 		writel(value, &(__dw_ch_regs(dw, EDMA_DIR_READ, ch)->name));	\
 	} while (0)

 /* HDMA management callbacks */
 static void dw_hdma_v0_core_off(struct dw_edma *dw)
 {
 	int id;

 	for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) {
 		SET_BOTH_CH_32(dw, id, int_setup,
 			       HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
 		SET_BOTH_CH_32(dw, id, int_clear,
 			       HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
 		SET_BOTH_CH_32(dw, id, ch_en, 0);
 	}
 }

 static u16 dw_hdma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
 {
 	/*
 	 * The HDMA IP have no way to know the number of hardware channels
 	 * available, we set it to maximum channels and let the platform
 	 * set the right number of channels.
 	 */
 	return HDMA_V0_MAX_NR_CH;
 }

 static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;
 	u32 tmp;

 	tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK,
 			GET_CH_32(dw, chan->id, chan->dir, ch_stat));

 	if (tmp == 1)
 		return DMA_IN_PROGRESS;
 	else if (tmp == 3)
 		return DMA_COMPLETE;
 	else
 		return DMA_ERROR;
 }

 static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;

 	SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_STOP_INT_MASK);
 }

 static void dw_hdma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;

 	SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_ABORT_INT_MASK);
 }

 static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;

 	return GET_CH_32(dw, chan->dir, chan->id, int_stat);
 }

 static irqreturn_t
 dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
 			   dw_edma_handler_t done, dw_edma_handler_t abort)
 {
 	struct dw_edma *dw = dw_irq->dw;
 	unsigned long total, pos, val;
 	irqreturn_t ret = IRQ_NONE;
 	struct dw_edma_chan *chan;
 	unsigned long off, mask;

 	if (dir == EDMA_DIR_WRITE) {
 		total = dw->wr_ch_cnt;
 		off = 0;
 		mask = dw_irq->wr_mask;
 	} else {
 		total = dw->rd_ch_cnt;
 		off = dw->wr_ch_cnt;
 		mask = dw_irq->rd_mask;
 	}

 	for_each_set_bit(pos, &mask, total) {
 		chan = &dw->chan[pos + off];

 		val = dw_hdma_v0_core_status_int(chan);
 		if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
 			dw_hdma_v0_core_clear_done_int(chan);
 			done(chan);

 			ret = IRQ_HANDLED;
 		}

 		if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) {
 			dw_hdma_v0_core_clear_abort_int(chan);
 			abort(chan);

 			ret = IRQ_HANDLED;
 		}
 	}

 	return ret;
 }

 static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
 				     u32 control, u32 size, u64 sar, u64 dar)
 {
 	ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);

 	if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
 		struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;

 		lli->control = control;
 		lli->transfer_size = size;
 		lli->sar.reg = sar;
 		lli->dar.reg = dar;
 	} else {
 		struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;

 		writel(control, &lli->control);
 		writel(size, &lli->transfer_size);
 		writeq(sar, &lli->sar.reg);
 		writeq(dar, &lli->dar.reg);
 	}
 }

 static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk,
 				     int i, u32 control, u64 pointer)
 {
 	ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);

 	if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
 		struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;

 		llp->control = control;
 		llp->llp.reg = pointer;
 	} else {
 		struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;

 		writel(control, &llp->control);
 		writeq(pointer, &llp->llp.reg);
 	}
 }

 static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
 {
 	struct dw_edma_burst *child;
 	u32 control = 0, i = 0;

 	if (chunk->cb)
 		control = DW_HDMA_V0_CB;

 	list_for_each_entry(child, &chunk->burst->list, list)
 		dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz,
 					 child->sar, child->dar);

 	control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB;
 	if (!chunk->cb)
 		control |= DW_HDMA_V0_CB;

 	dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
 }

 static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
 {
 	/*
 	 * In case of remote HDMA engine setup, the DW PCIe RP/EP internal
 	 * configuration registers and application memory are normally accessed
 	 * over different buses. Ensure LL-data reaches the memory before the
 	 * doorbell register is toggled by issuing the dummy-read from the remote
 	 * LL memory in a hope that the MRd TLP will return only after the
 	 * last MWr TLP is completed
 	 */
 	if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
 		readl(chunk->ll_region.vaddr.io);
 }

 static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
 {
 	struct dw_edma_chan *chan = chunk->chan;
 	struct dw_edma *dw = chan->dw;
 	u32 tmp;

 	dw_hdma_v0_core_write_chunk(chunk);

 	if (first) {
 		/* Enable engine */
 		SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
 		/* Interrupt unmask - stop, abort */
 		tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup);
 		tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
 		/* Interrupt enable - stop, abort */
 		tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN;
 		if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL))
 			tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN;
 		SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);
 		/* Channel control */
 		SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN);
 		/* Linked list */
 		/* llp is not aligned on 64bit -> keep 32bit accesses */
 		SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
 			  lower_32_bits(chunk->ll_region.paddr));
 		SET_CH_32(dw, chan->dir, chan->id, llp.msb,
 			  upper_32_bits(chunk->ll_region.paddr));
 	}
 	/* Set consumer cycle */
 	SET_CH_32(dw, chan->dir, chan->id, cycle_sync,
 		  HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT);

 	dw_hdma_v0_sync_ll_data(chunk);

 	/* Doorbell */
 	SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START);
 }

 static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
 {
 	struct dw_edma *dw = chan->dw;

 	/* MSI done addr - low, high */
 	SET_CH_32(dw, chan->dir, chan->id, msi_stop.lsb, chan->msi.address_lo);
 	SET_CH_32(dw, chan->dir, chan->id, msi_stop.msb, chan->msi.address_hi);
 	/* MSI abort addr - low, high */
 	SET_CH_32(dw, chan->dir, chan->id, msi_abort.lsb, chan->msi.address_lo);
 	SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi);
 	/* config MSI data */
 	SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data);
 }

 /* HDMA debugfs callbacks */
 static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw)
 {
 	dw_hdma_v0_debugfs_on(dw);
 }

 static const struct dw_edma_core_ops dw_hdma_v0_core = {
 	.off = dw_hdma_v0_core_off,
 	.ch_count = dw_hdma_v0_core_ch_count,
 	.ch_status = dw_hdma_v0_core_ch_status,
 	.handle_int = dw_hdma_v0_core_handle_int,
 	.start = dw_hdma_v0_core_start,
 	.ch_config = dw_hdma_v0_core_ch_config,
 	.debugfs_on = dw_hdma_v0_core_debugfs_on,
 };

 void dw_hdma_v0_core_register(struct dw_edma *dw)
 {
 	dw->core = &dw_hdma_v0_core;
 }
	// SPDX-License-Identifier: GPL-2.0
	/*
	* Copyright (c) 2023 Cai Huoqing
	* Synopsys DesignWare HDMA v0 core
	*/

	#include <linux/bitfield.h>
	#include <linux/irqreturn.h>
	#include <linux/io-64-nonatomic-lo-hi.h>

	#include "dw-edma-core.h"
	#include "dw-hdma-v0-core.h"
	#include "dw-hdma-v0-regs.h"
	#include "dw-hdma-v0-debugfs.h"

	enum dw_hdma_control {
	DW_HDMA_V0_CB = BIT(0),
	DW_HDMA_V0_TCB = BIT(1),
	DW_HDMA_V0_LLP = BIT(2),
	DW_HDMA_V0_LWIE = BIT(3),
	DW_HDMA_V0_RWIE = BIT(4),
	DW_HDMA_V0_CCS = BIT(8),
	DW_HDMA_V0_LLE = BIT(9),
	};

	static inline struct dw_hdma_v0_regs __iomem __dw_regs(struct dw_edma dw)
	{
	return dw->chip->reg_base;
	}

	static inline struct dw_hdma_v0_ch_regs __iomem *
	__dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
	{
	if (dir == EDMA_DIR_WRITE)
	return &(__dw_regs(dw)->ch[ch].wr);
	else
	return &(__dw_regs(dw)->ch[ch].rd);
	}

	#define SET_CH_32(dw, dir, ch, name, value) \
	writel(value, &(__dw_ch_regs(dw, dir, ch)->name))

	#define GET_CH_32(dw, dir, ch, name) \
	readl(&(__dw_ch_regs(dw, dir, ch)->name))

	#define SET_BOTH_CH_32(dw, ch, name, value) \
	do { \
	writel(value, &(__dw_ch_regs(dw, EDMA_DIR_WRITE, ch)->name)); \
	writel(value, &(__dw_ch_regs(dw, EDMA_DIR_READ, ch)->name)); \
	} while (0)

	/* HDMA management callbacks */
	static void dw_hdma_v0_core_off(struct dw_edma *dw)
	{
	int id;

	for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) {
	SET_BOTH_CH_32(dw, id, int_setup,
	HDMA_V0_STOP_INT_MASK \| HDMA_V0_ABORT_INT_MASK);
	SET_BOTH_CH_32(dw, id, int_clear,
	HDMA_V0_STOP_INT_MASK \| HDMA_V0_ABORT_INT_MASK);
	SET_BOTH_CH_32(dw, id, ch_en, 0);
	}
	}

	static u16 dw_hdma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
	{
	/*
	* The HDMA IP have no way to know the number of hardware channels
	* available, we set it to maximum channels and let the platform
	* set the right number of channels.
	*/
	return HDMA_V0_MAX_NR_CH;
	}

	static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan)
	{
	struct dw_edma *dw = chan->dw;
	u32 tmp;

	tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK,
	GET_CH_32(dw, chan->id, chan->dir, ch_stat));

	if (tmp == 1)
	return DMA_IN_PROGRESS;
	else if (tmp == 3)
	return DMA_COMPLETE;
	else
	return DMA_ERROR;
	}

	static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan)
	{
	struct dw_edma *dw = chan->dw;

	SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_STOP_INT_MASK);
	}

	static void dw_hdma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
	{
	struct dw_edma *dw = chan->dw;

	SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_ABORT_INT_MASK);
	}

	static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan)
	{
	struct dw_edma *dw = chan->dw;

	return GET_CH_32(dw, chan->dir, chan->id, int_stat);
	}

	static irqreturn_t
	dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
	dw_edma_handler_t done, dw_edma_handler_t abort)
	{
	struct dw_edma *dw = dw_irq->dw;
	unsigned long total, pos, val;
	irqreturn_t ret = IRQ_NONE;
	struct dw_edma_chan *chan;
	unsigned long off, mask;

	if (dir == EDMA_DIR_WRITE) {
	total = dw->wr_ch_cnt;
	off = 0;
	mask = dw_irq->wr_mask;
	} else {
	total = dw->rd_ch_cnt;
	off = dw->wr_ch_cnt;
	mask = dw_irq->rd_mask;
	}

	for_each_set_bit(pos, &mask, total) {
	chan = &dw->chan[pos + off];

	val = dw_hdma_v0_core_status_int(chan);
	if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
	dw_hdma_v0_core_clear_done_int(chan);
	done(chan);

	ret = IRQ_HANDLED;
	}

	if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) {
	dw_hdma_v0_core_clear_abort_int(chan);
	abort(chan);

	ret = IRQ_HANDLED;
	}
	}

	return ret;
	}

	static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
	u32 control, u32 size, u64 sar, u64 dar)
	{
	ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);

	if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
	struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;

	lli->control = control;
	lli->transfer_size = size;
	lli->sar.reg = sar;
	lli->dar.reg = dar;
	} else {
	struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;

	writel(control, &lli->control);
	writel(size, &lli->transfer_size);
	writeq(sar, &lli->sar.reg);
	writeq(dar, &lli->dar.reg);
	}
	}

	static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk,
	int i, u32 control, u64 pointer)
	{
	ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);

	if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
	struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;

	llp->control = control;
	llp->llp.reg = pointer;
	} else {
	struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;

	writel(control, &llp->control);
	writeq(pointer, &llp->llp.reg);
	}
	}

	static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
	{
	struct dw_edma_burst *child;
	u32 control = 0, i = 0;

	if (chunk->cb)
	control = DW_HDMA_V0_CB;

	list_for_each_entry(child, &chunk->burst->list, list)
	dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz,
	child->sar, child->dar);

	control = DW_HDMA_V0_LLP \| DW_HDMA_V0_TCB;
	if (!chunk->cb)
	control \|= DW_HDMA_V0_CB;

	dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
	}

	static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
	{
	/*
	* In case of remote HDMA engine setup, the DW PCIe RP/EP internal
	* configuration registers and application memory are normally accessed
	* over different buses. Ensure LL-data reaches the memory before the
	* doorbell register is toggled by issuing the dummy-read from the remote
	* LL memory in a hope that the MRd TLP will return only after the
	* last MWr TLP is completed
	*/
	if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
	readl(chunk->ll_region.vaddr.io);
	}

	static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
	{
	struct dw_edma_chan *chan = chunk->chan;
	struct dw_edma *dw = chan->dw;
	u32 tmp;

	dw_hdma_v0_core_write_chunk(chunk);

	if (first) {
	/* Enable engine */
	SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
	/* Interrupt unmask - stop, abort */
	tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup);
	tmp &= ~(HDMA_V0_STOP_INT_MASK \| HDMA_V0_ABORT_INT_MASK);
	/* Interrupt enable - stop, abort */
	tmp \|= HDMA_V0_LOCAL_STOP_INT_EN \| HDMA_V0_LOCAL_ABORT_INT_EN;
	if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL))
	tmp \|= HDMA_V0_REMOTE_STOP_INT_EN \| HDMA_V0_REMOTE_ABORT_INT_EN;
	SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);
	/* Channel control */
	SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN);
	/* Linked list */
	/* llp is not aligned on 64bit -> keep 32bit accesses */
	SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
	lower_32_bits(chunk->ll_region.paddr));
	SET_CH_32(dw, chan->dir, chan->id, llp.msb,
	upper_32_bits(chunk->ll_region.paddr));
	}
	/* Set consumer cycle */
	SET_CH_32(dw, chan->dir, chan->id, cycle_sync,
	HDMA_V0_CONSUMER_CYCLE_STAT \| HDMA_V0_CONSUMER_CYCLE_BIT);

	dw_hdma_v0_sync_ll_data(chunk);

	/* Doorbell */
	SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START);
	}

	static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
	{
	struct dw_edma *dw = chan->dw;

	/* MSI done addr - low, high */
	SET_CH_32(dw, chan->dir, chan->id, msi_stop.lsb, chan->msi.address_lo);
	SET_CH_32(dw, chan->dir, chan->id, msi_stop.msb, chan->msi.address_hi);
	/* MSI abort addr - low, high */
	SET_CH_32(dw, chan->dir, chan->id, msi_abort.lsb, chan->msi.address_lo);
	SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi);
	/* config MSI data */
	SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data);
	}

	/* HDMA debugfs callbacks */
	static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw)
	{
	dw_hdma_v0_debugfs_on(dw);
	}

	static const struct dw_edma_core_ops dw_hdma_v0_core = {
	.off = dw_hdma_v0_core_off,
	.ch_count = dw_hdma_v0_core_ch_count,
	.ch_status = dw_hdma_v0_core_ch_status,
	.handle_int = dw_hdma_v0_core_handle_int,
	.start = dw_hdma_v0_core_start,
	.ch_config = dw_hdma_v0_core_ch_config,
	.debugfs_on = dw_hdma_v0_core_debugfs_on,
	};

	void dw_hdma_v0_core_register(struct dw_edma *dw)
	{
	dw->core = &dw_hdma_v0_core;
	}