drivers/net/ethernet/mediatek/mtk_wed_wo.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2022 MediaTek Inc.
 *
 * Author: Lorenzo Bianconi <lorenzo@kernel.org>
 *	   Sujuan Chen <sujuan.chen@mediatek.com>
 */

#include <linux/kernel.h>
#include <linux/dma-mapping.h>
#include <linux/of_platform.h>
#include <linux/interrupt.h>
#include <linux/of_address.h>
#include <linux/mfd/syscon.h>
#include <linux/of_irq.h>
#include <linux/bitfield.h>

#include "mtk_wed.h"
#include "mtk_wed_regs.h"
#include "mtk_wed_wo.h"

static u32
mtk_wed_mmio_r32(struct mtk_wed_wo *wo, u32 reg)
{
	u32 val;

	if (regmap_read(wo->mmio.regs, reg, &val))
		val = ~0;

	return val;
}

static void
mtk_wed_mmio_w32(struct mtk_wed_wo *wo, u32 reg, u32 val)
{
	regmap_write(wo->mmio.regs, reg, val);
}

static u32
mtk_wed_wo_get_isr(struct mtk_wed_wo *wo)
{
	u32 val = mtk_wed_mmio_r32(wo, MTK_WED_WO_CCIF_RCHNUM);

	return val & MTK_WED_WO_CCIF_RCHNUM_MASK;
}

static void
mtk_wed_wo_set_isr(struct mtk_wed_wo *wo, u32 mask)
{
	mtk_wed_mmio_w32(wo, MTK_WED_WO_CCIF_IRQ0_MASK, mask);
}

static void
mtk_wed_wo_set_ack(struct mtk_wed_wo *wo, u32 mask)
{
	mtk_wed_mmio_w32(wo, MTK_WED_WO_CCIF_ACK, mask);
}

static void
mtk_wed_wo_set_isr_mask(struct mtk_wed_wo *wo, u32 mask, u32 val, bool set)
{
	unsigned long flags;

	spin_lock_irqsave(&wo->mmio.lock, flags);
	wo->mmio.irq_mask &= ~mask;
	wo->mmio.irq_mask |= val;
	if (set)
		mtk_wed_wo_set_isr(wo, wo->mmio.irq_mask);
	spin_unlock_irqrestore(&wo->mmio.lock, flags);
}

static void
mtk_wed_wo_irq_enable(struct mtk_wed_wo *wo, u32 mask)
{
	mtk_wed_wo_set_isr_mask(wo, 0, mask, false);
	tasklet_schedule(&wo->mmio.irq_tasklet);
}

static void
mtk_wed_wo_irq_disable(struct mtk_wed_wo *wo, u32 mask)
{
	mtk_wed_wo_set_isr_mask(wo, mask, 0, true);
}

static void
mtk_wed_wo_kickout(struct mtk_wed_wo *wo)
{
	mtk_wed_mmio_w32(wo, MTK_WED_WO_CCIF_BUSY, 1 << MTK_WED_WO_TXCH_NUM);
	mtk_wed_mmio_w32(wo, MTK_WED_WO_CCIF_TCHNUM, MTK_WED_WO_TXCH_NUM);
}

static void
mtk_wed_wo_queue_kick(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q,
		      u32 val)
{
	wmb();
	mtk_wed_mmio_w32(wo, q->regs.cpu_idx, val);
}

static void *
mtk_wed_wo_dequeue(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q, u32 *len,
		   bool flush)
{
	int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
	int index = (q->tail + 1) % q->n_desc;
	struct mtk_wed_wo_queue_entry *entry;
	struct mtk_wed_wo_queue_desc *desc;
	void *buf;

	if (!q->queued)
		return NULL;

	if (flush)
		q->desc[index].ctrl |= cpu_to_le32(MTK_WED_WO_CTL_DMA_DONE);
	else if (!(q->desc[index].ctrl & cpu_to_le32(MTK_WED_WO_CTL_DMA_DONE)))
		return NULL;

	q->tail = index;
	q->queued--;

	desc = &q->desc[index];
	entry = &q->entry[index];
	buf = entry->buf;
	if (len)
		*len = FIELD_GET(MTK_WED_WO_CTL_SD_LEN0,
				 le32_to_cpu(READ_ONCE(desc->ctrl)));
	if (buf)
		dma_unmap_single(wo->hw->dev, entry->addr, buf_len,
				 DMA_FROM_DEVICE);
	entry->buf = NULL;

	return buf;
}

static int
mtk_wed_wo_queue_refill(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q,
			bool rx)
{
	enum dma_data_direction dir = rx ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
	int n_buf = 0;

	spin_lock_bh(&q->lock);
	while (q->queued < q->n_desc) {
		struct mtk_wed_wo_queue_entry *entry;
		dma_addr_t addr;
		void *buf;

		buf = page_frag_alloc(&q->cache, q->buf_size, GFP_ATOMIC);
		if (!buf)
			break;

		addr = dma_map_single(wo->hw->dev, buf, q->buf_size, dir);
		if (unlikely(dma_mapping_error(wo->hw->dev, addr))) {
			skb_free_frag(buf);
			break;
		}

		q->head = (q->head + 1) % q->n_desc;
		entry = &q->entry[q->head];
		entry->addr = addr;
		entry->len = q->buf_size;
		q->entry[q->head].buf = buf;

		if (rx) {
			struct mtk_wed_wo_queue_desc *desc = &q->desc[q->head];
			u32 ctrl = MTK_WED_WO_CTL_LAST_SEC0 |
				   FIELD_PREP(MTK_WED_WO_CTL_SD_LEN0,
					      entry->len);

			WRITE_ONCE(desc->buf0, cpu_to_le32(addr));
			WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
		}
		q->queued++;
		n_buf++;
	}
	spin_unlock_bh(&q->lock);

	return n_buf;
}

static void
mtk_wed_wo_rx_complete(struct mtk_wed_wo *wo)
{
	mtk_wed_wo_set_ack(wo, MTK_WED_WO_RXCH_INT_MASK);
	mtk_wed_wo_irq_enable(wo, MTK_WED_WO_RXCH_INT_MASK);
}

static void
mtk_wed_wo_rx_run_queue(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q)
{
	for (;;) {
		struct mtk_wed_mcu_hdr *hdr;
		struct sk_buff *skb;
		void *data;
		u32 len;

		data = mtk_wed_wo_dequeue(wo, q, &len, false);
		if (!data)
			break;

		skb = build_skb(data, q->buf_size);
		if (!skb) {
			skb_free_frag(data);
			continue;
		}

		__skb_put(skb, len);
		if (mtk_wed_mcu_check_msg(wo, skb)) {
			dev_kfree_skb(skb);
			continue;
		}

		hdr = (struct mtk_wed_mcu_hdr *)skb->data;
		if (hdr->flag & cpu_to_le16(MTK_WED_WARP_CMD_FLAG_RSP))
			mtk_wed_mcu_rx_event(wo, skb);
		else
			mtk_wed_mcu_rx_unsolicited_event(wo, skb);
	}

	if (mtk_wed_wo_queue_refill(wo, q, true)) {
		u32 index = (q->head - 1) % q->n_desc;

		mtk_wed_wo_queue_kick(wo, q, index);
	}
}

static irqreturn_t
mtk_wed_wo_irq_handler(int irq, void *data)
{
	struct mtk_wed_wo *wo = data;

	mtk_wed_wo_set_isr(wo, 0);
	tasklet_schedule(&wo->mmio.irq_tasklet);

	return IRQ_HANDLED;
}

static void mtk_wed_wo_irq_tasklet(struct tasklet_struct *t)
{
	struct mtk_wed_wo *wo = from_tasklet(wo, t, mmio.irq_tasklet);
	u32 intr, mask;

	/* disable interrupts */
	mtk_wed_wo_set_isr(wo, 0);

	intr = mtk_wed_wo_get_isr(wo);
	intr &= wo->mmio.irq_mask;
	mask = intr & (MTK_WED_WO_RXCH_INT_MASK | MTK_WED_WO_EXCEPTION_INT_MASK);
	mtk_wed_wo_irq_disable(wo, mask);

	if (intr & MTK_WED_WO_RXCH_INT_MASK) {
		mtk_wed_wo_rx_run_queue(wo, &wo->q_rx);
		mtk_wed_wo_rx_complete(wo);
	}
}

/* mtk wed wo hw queues */

static int
mtk_wed_wo_queue_alloc(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q,
		       int n_desc, int buf_size, int index,
		       struct mtk_wed_wo_queue_regs *regs)
{
	spin_lock_init(&q->lock);
	q->regs = *regs;
	q->n_desc = n_desc;
	q->buf_size = buf_size;

	q->desc = dmam_alloc_coherent(wo->hw->dev, n_desc * sizeof(*q->desc),
				      &q->desc_dma, GFP_KERNEL);
	if (!q->desc)
		return -ENOMEM;

	q->entry = devm_kzalloc(wo->hw->dev, n_desc * sizeof(*q->entry),
				GFP_KERNEL);
	if (!q->entry)
		return -ENOMEM;

	return 0;
}

static void
mtk_wed_wo_queue_free(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q)
{
	mtk_wed_mmio_w32(wo, q->regs.cpu_idx, 0);
	dma_free_coherent(wo->hw->dev, q->n_desc * sizeof(*q->desc), q->desc,
			  q->desc_dma);
}

static void
mtk_wed_wo_queue_tx_clean(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q)
{
	struct page *page;
	int i;

	spin_lock_bh(&q->lock);
	for (i = 0; i < q->n_desc; i++) {
		struct mtk_wed_wo_queue_entry *entry = &q->entry[i];

		dma_unmap_single(wo->hw->dev, entry->addr, entry->len,
				 DMA_TO_DEVICE);
		skb_free_frag(entry->buf);
		entry->buf = NULL;
	}
	spin_unlock_bh(&q->lock);

	if (!q->cache.va)
		return;

	page = virt_to_page(q->cache.va);
	__page_frag_cache_drain(page, q->cache.pagecnt_bias);
	memset(&q->cache, 0, sizeof(q->cache));
}

static void
mtk_wed_wo_queue_rx_clean(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q)
{
	struct page *page;

	spin_lock_bh(&q->lock);
	for (;;) {
		void *buf = mtk_wed_wo_dequeue(wo, q, NULL, true);

		if (!buf)
			break;

		skb_free_frag(buf);
	}
	spin_unlock_bh(&q->lock);

	if (!q->cache.va)
		return;

	page = virt_to_page(q->cache.va);
	__page_frag_cache_drain(page, q->cache.pagecnt_bias);
	memset(&q->cache, 0, sizeof(q->cache));
}

static void
mtk_wed_wo_queue_reset(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q)
{
	mtk_wed_mmio_w32(wo, q->regs.cpu_idx, 0);
	mtk_wed_mmio_w32(wo, q->regs.desc_base, q->desc_dma);
	mtk_wed_mmio_w32(wo, q->regs.ring_size, q->n_desc);
}

int mtk_wed_wo_queue_tx_skb(struct mtk_wed_wo *wo, struct mtk_wed_wo_queue *q,
			    struct sk_buff *skb)
{
	struct mtk_wed_wo_queue_entry *entry;
	struct mtk_wed_wo_queue_desc *desc;
	int ret = 0, index;
	u32 ctrl;

	spin_lock_bh(&q->lock);

	q->tail = mtk_wed_mmio_r32(wo, q->regs.dma_idx);
	index = (q->head + 1) % q->n_desc;
	if (q->tail == index) {
		ret = -ENOMEM;
		goto out;
	}

	entry = &q->entry[index];
	if (skb->len > entry->len) {
		ret = -ENOMEM;
		goto out;
	}

	desc = &q->desc[index];
	q->head = index;

	dma_sync_single_for_cpu(wo->hw->dev, entry->addr, skb->len,
				DMA_TO_DEVICE);
	memcpy(entry->buf, skb->data, skb->len);
	dma_sync_single_for_device(wo->hw->dev, entry->addr, skb->len,
				   DMA_TO_DEVICE);

	ctrl = FIELD_PREP(MTK_WED_WO_CTL_SD_LEN0, skb->len) |
	       MTK_WED_WO_CTL_LAST_SEC0 | MTK_WED_WO_CTL_DMA_DONE;
	WRITE_ONCE(desc->buf0, cpu_to_le32(entry->addr));
	WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));

	mtk_wed_wo_queue_kick(wo, q, q->head);
	mtk_wed_wo_kickout(wo);
out:
	spin_unlock_bh(&q->lock);

	dev_kfree_skb(skb);

	return ret;
}

static int
mtk_wed_wo_exception_init(struct mtk_wed_wo *wo)
{
	return 0;
}

static int
mtk_wed_wo_hardware_init(struct mtk_wed_wo *wo)
{
	struct mtk_wed_wo_queue_regs regs;
	struct device_node *np;
	int ret;

	np = of_parse_phandle(wo->hw->node, "mediatek,wo-ccif", 0);
	if (!np)
		return -ENODEV;

	wo->mmio.regs = syscon_regmap_lookup_by_phandle(np, NULL);
	if (IS_ERR(wo->mmio.regs)) {
		ret = PTR_ERR(wo->mmio.regs);
		goto error_put;
	}

	wo->mmio.irq = irq_of_parse_and_map(np, 0);
	wo->mmio.irq_mask = MTK_WED_WO_ALL_INT_MASK;
	spin_lock_init(&wo->mmio.lock);
	tasklet_setup(&wo->mmio.irq_tasklet, mtk_wed_wo_irq_tasklet);

	ret = devm_request_irq(wo->hw->dev, wo->mmio.irq,
			       mtk_wed_wo_irq_handler, IRQF_TRIGGER_HIGH,
			       KBUILD_MODNAME, wo);
	if (ret)
		goto error;

	regs.desc_base = MTK_WED_WO_CCIF_DUMMY1;
	regs.ring_size = MTK_WED_WO_CCIF_DUMMY2;
	regs.dma_idx = MTK_WED_WO_CCIF_SHADOW4;
	regs.cpu_idx = MTK_WED_WO_CCIF_DUMMY3;

	ret = mtk_wed_wo_queue_alloc(wo, &wo->q_tx, MTK_WED_WO_RING_SIZE,
				     MTK_WED_WO_CMD_LEN, MTK_WED_WO_TXCH_NUM,
				     &regs);
	if (ret)
		goto error;

	mtk_wed_wo_queue_refill(wo, &wo->q_tx, false);
	mtk_wed_wo_queue_reset(wo, &wo->q_tx);

	regs.desc_base = MTK_WED_WO_CCIF_DUMMY5;
	regs.ring_size = MTK_WED_WO_CCIF_DUMMY6;
	regs.dma_idx = MTK_WED_WO_CCIF_SHADOW8;
	regs.cpu_idx = MTK_WED_WO_CCIF_DUMMY7;

	ret = mtk_wed_wo_queue_alloc(wo, &wo->q_rx, MTK_WED_WO_RING_SIZE,
				     MTK_WED_WO_CMD_LEN, MTK_WED_WO_RXCH_NUM,
				     &regs);
	if (ret)
		goto error;

	mtk_wed_wo_queue_refill(wo, &wo->q_rx, true);
	mtk_wed_wo_queue_reset(wo, &wo->q_rx);

	/* rx queue irqmask */
	mtk_wed_wo_set_isr(wo, wo->mmio.irq_mask);

	return 0;

error:
	devm_free_irq(wo->hw->dev, wo->mmio.irq, wo);
error_put:
	of_node_put(np);
	return ret;
}

static void
mtk_wed_wo_hw_deinit(struct mtk_wed_wo *wo)
{
	/* disable interrupts */
	mtk_wed_wo_set_isr(wo, 0);

	tasklet_disable(&wo->mmio.irq_tasklet);

	disable_irq(wo->mmio.irq);
	devm_free_irq(wo->hw->dev, wo->mmio.irq, wo);

	mtk_wed_wo_queue_tx_clean(wo, &wo->q_tx);
	mtk_wed_wo_queue_rx_clean(wo, &wo->q_rx);
	mtk_wed_wo_queue_free(wo, &wo->q_tx);
	mtk_wed_wo_queue_free(wo, &wo->q_rx);
}

int mtk_wed_wo_init(struct mtk_wed_hw *hw)
{
	struct mtk_wed_wo *wo;
	int ret;

	wo = devm_kzalloc(hw->dev, sizeof(*wo), GFP_KERNEL);
	if (!wo)
		return -ENOMEM;

	hw->wed_wo = wo;
	wo->hw = hw;

	ret = mtk_wed_wo_hardware_init(wo);
	if (ret)
		return ret;

	ret = mtk_wed_mcu_init(wo);
	if (ret)
		return ret;

	return mtk_wed_wo_exception_init(wo);
}

void mtk_wed_wo_deinit(struct mtk_wed_hw *hw)
{
	struct mtk_wed_wo *wo = hw->wed_wo;

	mtk_wed_wo_hw_deinit(wo);
}