// SPDX-License-Identifier: GPL-2.0-only
/*
 * CPU-agnostic ARM page table allocator.
 * Host-specific functions. The rest is in io-pgtable-arm-common.c.
 *
 * Copyright (C) 2014 ARM Limited
 *
 * Author: Will Deacon <will.deacon@arm.com>
 */

#define pr_fmt(fmt)	"arm-lpae io-pgtable: " fmt

#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/io-pgtable-arm.h>
#include <linux/kernel.h>
#include <linux/sizes.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/dma-mapping.h>

#include <asm/barrier.h>

bool selftest_running = false;

static dma_addr_t __arm_lpae_dma_addr(void *pages)
{
	return (dma_addr_t)virt_to_phys(pages);
}

void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, struct io_pgtable_cfg *cfg,
			     struct io_pgtable *iop)
{
	struct device *dev = cfg->iommu_dev;
	int order = get_order(size);
	struct page *p;
	dma_addr_t dma;
	void *pages;

	VM_BUG_ON((gfp & __GFP_HIGHMEM));
	p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order);
	if (!p)
		return NULL;

	pages = page_address(p);
	if (!cfg->coherent_walk) {
		dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE);
		if (dma_mapping_error(dev, dma))
			goto out_free;
		/*
		 * We depend on the IOMMU being able to work with any physical
		 * address directly, so if the DMA layer suggests otherwise by
		 * translating or truncating them, that bodes very badly...
		 */
		if (dma != virt_to_phys(pages))
			goto out_unmap;
	}

	return pages;

out_unmap:
	dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
	dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
out_free:
	__free_pages(p, order);
	return NULL;
}

void __arm_lpae_free_pages(void *pages, size_t size, struct io_pgtable_cfg *cfg)
{
	if (!cfg->coherent_walk)
		dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
				 size, DMA_TO_DEVICE);
	free_pages((unsigned long)pages, get_order(size));
}

void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
			 struct io_pgtable_cfg *cfg)
{
	dma_sync_single_for_device(cfg->iommu_dev, __arm_lpae_dma_addr(ptep),
				   sizeof(*ptep) * num_entries, DMA_TO_DEVICE);
}

static void arm_lpae_free_pgtable(struct io_pgtable *iop)
{
	struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(iop->ops);

	__arm_lpae_free_pgtable(data, data->start_level, iop->pgd, false);
	kfree(data);
}

int arm_64_lpae_alloc_pgtable_s1(struct io_pgtable *iop,
				 struct io_pgtable_cfg *cfg, void *cookie)
{
	struct arm_lpae_io_pgtable *data;

	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	if (arm_lpae_init_pgtable_s1(cfg, data))
		goto out_free_data;

	/* Looking good; allocate a pgd */
	iop->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
					  GFP_KERNEL, cfg, iop);
	if (!iop->pgd)
		goto out_free_data;

	/* Ensure the empty pgd is visible before any actual TTBR write */
	wmb();

	cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(iop->pgd);
	iop->ops = &data->iop.ops;
	return 0;

out_free_data:
	kfree(data);
	return -EINVAL;
}

static int arm_64_lpae_configure_s1(struct io_pgtable_cfg *cfg, size_t *pgd_size)
{
	int ret;
	struct arm_lpae_io_pgtable data = {};

	ret = arm_lpae_init_pgtable_s1(cfg, &data);
	if (ret)
		return ret;
	*pgd_size = sizeof(arm_lpae_iopte) << data.pgd_bits;
	return 0;
}

int arm_64_lpae_alloc_pgtable_s2(struct io_pgtable *iop,
				 struct io_pgtable_cfg *cfg, void *cookie)
{
	struct arm_lpae_io_pgtable *data;

	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	if (arm_lpae_init_pgtable_s2(cfg, data))
		goto out_free_data;

	/* Allocate pgd pages */
	iop->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
					  GFP_KERNEL, cfg, iop);
	if (!iop->pgd)
		goto out_free_data;

	/* Ensure the empty pgd is visible before any actual TTBR write */
	wmb();

	/* VTTBR */
	cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(iop->pgd);
	iop->ops = &data->iop.ops;
	return 0;

out_free_data:
	kfree(data);
	return -EINVAL;
}

static int arm_64_lpae_configure_s2(struct io_pgtable_cfg *cfg, size_t *pgd_size)
{
	int ret;
	struct arm_lpae_io_pgtable data = {};

	ret = arm_lpae_init_pgtable_s2(cfg, &data);
	if (ret)
		return ret;
	*pgd_size = sizeof(arm_lpae_iopte) << data.pgd_bits;
	return 0;
}

int arm_32_lpae_alloc_pgtable_s1(struct io_pgtable *iop,
				 struct io_pgtable_cfg *cfg, void *cookie)
{
	if (cfg->ias > 32 || cfg->oas > 40)
		return -EINVAL;

	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
	return arm_64_lpae_alloc_pgtable_s1(iop, cfg, cookie);
}

int arm_32_lpae_alloc_pgtable_s2(struct io_pgtable *iop,
				 struct io_pgtable_cfg *cfg, void *cookie)
{
	if (cfg->ias > 40 || cfg->oas > 40)
		return -EINVAL;

	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);
	return arm_64_lpae_alloc_pgtable_s2(iop, cfg, cookie);
}

int arm_mali_lpae_alloc_pgtable(struct io_pgtable *iop,
				struct io_pgtable_cfg *cfg, void *cookie)
{
	struct arm_lpae_io_pgtable *data;

	/* No quirks for Mali (hopefully) */
	if (cfg->quirks)
		return -EINVAL;

	if (cfg->ias > 48 || cfg->oas > 40)
		return -EINVAL;

	cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G);

	data = kzalloc(sizeof(*data), GFP_KERNEL);
	if (!data)
		return -ENOMEM;

	if (arm_lpae_init_pgtable(cfg, data))
		goto out_free_data;

	/* Mali seems to need a full 4-level table regardless of IAS */
	if (data->start_level > 0) {
		data->start_level = 0;
		data->pgd_bits = 0;
	}
	/*
	 * MEMATTR: Mali has no actual notion of a non-cacheable type, so the
	 * best we can do is mimic the out-of-tree driver and hope that the
	 * "implementation-defined caching policy" is good enough. Similarly,
	 * we'll use it for the sake of a valid attribute for our 'device'
	 * index, although callers should never request that in practice.
	 */
	cfg->arm_mali_lpae_cfg.memattr =
		(ARM_MALI_LPAE_MEMATTR_IMP_DEF
		 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) |
		(ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC
		 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) |
		(ARM_MALI_LPAE_MEMATTR_IMP_DEF
		 << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV));

	iop->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL,
					  cfg, iop);
	if (!iop->pgd)
		goto out_free_data;

	/* Ensure the empty pgd is visible before TRANSTAB can be written */
	wmb();

	cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(iop->pgd) |
					  ARM_MALI_LPAE_TTBR_READ_INNER |
					  ARM_MALI_LPAE_TTBR_ADRMODE_TABLE;
	if (cfg->coherent_walk)
		cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER;

	iop->ops = &data->iop.ops;
	return 0;

out_free_data:
	kfree(data);
	return -EINVAL;
}

struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = {
	.alloc		= arm_64_lpae_alloc_pgtable_s1,
	.free		= arm_lpae_free_pgtable,
	.configure	= arm_64_lpae_configure_s1,
};

struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = {
	.alloc		= arm_64_lpae_alloc_pgtable_s2,
	.free		= arm_lpae_free_pgtable,
	.configure	= arm_64_lpae_configure_s2,
};

struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = {
	.alloc		= arm_32_lpae_alloc_pgtable_s1,
	.free		= arm_lpae_free_pgtable,
};

struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = {
	.alloc		= arm_32_lpae_alloc_pgtable_s2,
	.free		= arm_lpae_free_pgtable,
};

struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = {
	.alloc		= arm_mali_lpae_alloc_pgtable,
	.free		= arm_lpae_free_pgtable,
};

#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST

static struct io_pgtable_cfg *cfg_cookie __initdata;

static void __init dummy_tlb_flush_all(void *cookie)
{
	WARN_ON(cookie != cfg_cookie);
}

static void __init dummy_tlb_flush(unsigned long iova, size_t size,
				   size_t granule, void *cookie)
{
	WARN_ON(cookie != cfg_cookie);
	WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
}

static void __init dummy_tlb_add_page(struct iommu_iotlb_gather *gather,
				      unsigned long iova, size_t granule,
				      void *cookie)
{
	dummy_tlb_flush(iova, granule, granule, cookie);
}

static const struct iommu_flush_ops dummy_tlb_ops __initconst = {
	.tlb_flush_all	= dummy_tlb_flush_all,
	.tlb_flush_walk	= dummy_tlb_flush,
	.tlb_add_page	= dummy_tlb_add_page,
};

static void __init arm_lpae_dump_ops(struct io_pgtable *iop)
{
	struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(iop->ops);
	struct io_pgtable_cfg *cfg = &data->iop.cfg;

	pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n",
		cfg->pgsize_bitmap, cfg->ias);
	pr_err("data: %d levels, 0x%zx pgd_size, %u pg_shift, %u bits_per_level, pgd @ %p\n",
		ARM_LPAE_MAX_LEVELS - data->start_level, ARM_LPAE_PGD_SIZE(data),
		ilog2(ARM_LPAE_GRANULE(data)), data->bits_per_level, iop->pgd);
}

#define __FAIL(iop, i)	({						\
		WARN(1, "selftest: test failed for fmt idx %d\n", (i));	\
		arm_lpae_dump_ops(iop);					\
		selftest_running = false;				\
		-EFAULT;						\
})

static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
{
	static const enum io_pgtable_fmt fmts[] __initconst = {
		ARM_64_LPAE_S1,
		ARM_64_LPAE_S2,
	};

	int i, j, ret;
	unsigned long iova;
	size_t size, mapped;
	struct io_pgtable iop;

	selftest_running = true;

	for (i = 0; i < ARRAY_SIZE(fmts); ++i) {
		cfg_cookie = cfg;
		cfg->fmt = fmts[i];
		ret = alloc_io_pgtable_ops(&iop, cfg, cfg);
		if (ret) {
			pr_err("selftest: failed to allocate io pgtable ops\n");
			return ret;
		}

		/*
		 * Initial sanity checks.
		 * Empty page tables shouldn't provide any translations.
		 */
		if (iopt_iova_to_phys(&iop, 42))
			return __FAIL(&iop, i);

		if (iopt_iova_to_phys(&iop, SZ_1G + 42))
			return __FAIL(&iop, i);

		if (iopt_iova_to_phys(&iop, SZ_2G + 42))
			return __FAIL(&iop, i);

		/*
		 * Distinct mappings of different granule sizes.
		 */
		iova = 0;
		for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
			size = 1UL << j;

			if (iopt_map_pages(&iop, iova, iova, size, 1,
					   IOMMU_READ | IOMMU_WRITE |
					   IOMMU_NOEXEC | IOMMU_CACHE,
					   GFP_KERNEL, &mapped))
				return __FAIL(&iop, i);

			/* Overlapping mappings */
			if (!iopt_map_pages(&iop, iova, iova + size, size, 1,
					    IOMMU_READ | IOMMU_NOEXEC,
					    GFP_KERNEL, &mapped))
				return __FAIL(&iop, i);

			if (iopt_iova_to_phys(&iop, iova + 42) != (iova + 42))
				return __FAIL(&iop, i);

			iova += SZ_1G;
		}

		/* Partial unmap */
		size = 1UL << __ffs(cfg->pgsize_bitmap);
		if (iopt_unmap_pages(&iop, SZ_1G + size, size, 1, NULL) != size)
			return __FAIL(&iop, i);

		/* Remap of partial unmap */
		if (iopt_map_pages(&iop, SZ_1G + size, size, size, 1,
				   IOMMU_READ, GFP_KERNEL, &mapped))
			return __FAIL(&iop, i);

		if (iopt_iova_to_phys(&iop, SZ_1G + size + 42) != (size + 42))
			return __FAIL(&iop, i);

		/* Full unmap */
		iova = 0;
		for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
			size = 1UL << j;

			if (iopt_unmap_pages(&iop, iova, size, 1, NULL) != size)
				return __FAIL(&iop, i);

			if (iopt_iova_to_phys(&iop, iova + 42))
				return __FAIL(&iop, i);

			/* Remap full block */
			if (iopt_map_pages(&iop, iova, iova, size, 1,
					   IOMMU_WRITE, GFP_KERNEL, &mapped))
				return __FAIL(&iop, i);

			if (iopt_iova_to_phys(&iop, iova + 42) != (iova + 42))
				return __FAIL(&iop, i);

			iova += SZ_1G;
		}

		free_io_pgtable_ops(&iop);
	}

	selftest_running = false;
	return 0;
}

static int __init arm_lpae_do_selftests(void)
{
	static const unsigned long pgsize[] __initconst = {
		SZ_4K | SZ_2M | SZ_1G,
		SZ_16K | SZ_32M,
		SZ_64K | SZ_512M,
	};

	static const unsigned int ias[] __initconst = {
		32, 36, 40, 42, 44, 48,
	};

	int i, j, pass = 0, fail = 0;
	struct device dev;
	struct io_pgtable_cfg cfg = {
		.tlb = &dummy_tlb_ops,
		.oas = 48,
		.coherent_walk = true,
		.iommu_dev = &dev,
	};

	/* __arm_lpae_alloc_pages() merely needs dev_to_node() to work */
	set_dev_node(&dev, NUMA_NO_NODE);

	for (i = 0; i < ARRAY_SIZE(pgsize); ++i) {
		for (j = 0; j < ARRAY_SIZE(ias); ++j) {
			cfg.pgsize_bitmap = pgsize[i];
			cfg.ias = ias[j];
			pr_info("selftest: pgsize_bitmap 0x%08lx, IAS %u\n",
				pgsize[i], ias[j]);
			if (arm_lpae_run_tests(&cfg))
				fail++;
			else
				pass++;
		}
	}

	pr_info("selftest: completed with %d PASS %d FAIL\n", pass, fail);
	return fail ? -EFAULT : 0;
}
subsys_initcall(arm_lpae_do_selftests);
#endif