drivers/iommu/io-pgtable-arm.h - linux - Git at Google

 /* SPDX-License-Identifier: GPL-2.0-only */
 #ifndef IO_PGTABLE_ARM_H_
 #define IO_PGTABLE_ARM_H_

 #include <linux/io-pgtable.h>

 typedef u64 arm_lpae_iopte;

 struct arm_lpae_io_pgtable {
 	struct io_pgtable	iop;

 	int			pgd_bits;
 	int			start_level;
 	int			bits_per_level;

 	void			*pgd;
 };

 #define ARM_LPAE_MAX_ADDR_BITS		52
 #define ARM_LPAE_S2_MAX_CONCAT_PAGES	16
 #define ARM_LPAE_MAX_LEVELS		4

 /* Struct accessors */
 #define io_pgtable_to_data(x)						\
 	container_of((x), struct arm_lpae_io_pgtable, iop)

 #define io_pgtable_ops_to_data(x)					\
 	io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))

 /*
  * Calculate the right shift amount to get to the portion describing level l
  * in a virtual address mapped by the pagetable in d.
  */
 #define ARM_LPAE_LVL_SHIFT(l,d)						\
 	(((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) +		\
 	ilog2(sizeof(arm_lpae_iopte)))

 #define ARM_LPAE_GRANULE(d)						\
 	(sizeof(arm_lpae_iopte) << (d)->bits_per_level)
 #define ARM_LPAE_PGD_SIZE(d)						\
 	(sizeof(arm_lpae_iopte) << (d)->pgd_bits)

 #define ARM_LPAE_PTES_PER_TABLE(d)					\
 	(ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte)))

 /*
  * Calculate the index at level l used to map virtual address a using the
  * pagetable in d.
  */
 #define ARM_LPAE_PGD_IDX(l,d)						\
 	((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)

 #define ARM_LPAE_LVL_IDX(a,l,d)						\
 	(((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) &			\
 	 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))

 /* Calculate the block/page mapping size at level l for pagetable in d. */
 #define ARM_LPAE_BLOCK_SIZE(l,d)	(1ULL << ARM_LPAE_LVL_SHIFT(l,d))

 /* Page table bits */
 #define ARM_LPAE_PTE_TYPE_SHIFT		0
 #define ARM_LPAE_PTE_TYPE_MASK		0x3

 #define ARM_LPAE_PTE_TYPE_BLOCK		1
 #define ARM_LPAE_PTE_TYPE_TABLE		3
 #define ARM_LPAE_PTE_TYPE_PAGE		3

 #define ARM_LPAE_PTE_ADDR_MASK		GENMASK_ULL(47,12)

 #define ARM_LPAE_PTE_NSTABLE		(((arm_lpae_iopte)1) << 63)
 #define ARM_LPAE_PTE_XN			(((arm_lpae_iopte)3) << 53)
 #define ARM_LPAE_PTE_DBM		(((arm_lpae_iopte)1) << 51)
 #define ARM_LPAE_PTE_AF			(((arm_lpae_iopte)1) << 10)
 #define ARM_LPAE_PTE_SH_NS		(((arm_lpae_iopte)0) << 8)
 #define ARM_LPAE_PTE_SH_OS		(((arm_lpae_iopte)2) << 8)
 #define ARM_LPAE_PTE_SH_IS		(((arm_lpae_iopte)3) << 8)
 #define ARM_LPAE_PTE_NS			(((arm_lpae_iopte)1) << 5)
 #define ARM_LPAE_PTE_VALID		(((arm_lpae_iopte)1) << 0)

 #define ARM_LPAE_PTE_ATTR_LO_MASK	(((arm_lpae_iopte)0x3ff) << 2)
 /* Ignore the contiguous bit for block splitting */
 #define ARM_LPAE_PTE_ATTR_HI_MASK	(ARM_LPAE_PTE_XN | ARM_LPAE_PTE_DBM)
 #define ARM_LPAE_PTE_ATTR_MASK		(ARM_LPAE_PTE_ATTR_LO_MASK |	\
 					 ARM_LPAE_PTE_ATTR_HI_MASK)
 /* Software bit for solving coherency races */
 #define ARM_LPAE_PTE_SW_SYNC		(((arm_lpae_iopte)1) << 55)

 /* Stage-1 PTE */
 #define ARM_LPAE_PTE_AP_UNPRIV		(((arm_lpae_iopte)1) << 6)
 #define ARM_LPAE_PTE_AP_RDONLY_BIT	7
 #define ARM_LPAE_PTE_AP_RDONLY		(((arm_lpae_iopte)1) << \
 					   ARM_LPAE_PTE_AP_RDONLY_BIT)
 #define ARM_LPAE_PTE_AP_WR_CLEAN_MASK	(ARM_LPAE_PTE_AP_RDONLY | \
 					 ARM_LPAE_PTE_DBM)
 #define ARM_LPAE_PTE_ATTRINDX_SHIFT	2
 #define ARM_LPAE_PTE_nG			(((arm_lpae_iopte)1) << 11)

 /* Stage-2 PTE */
 #define ARM_LPAE_PTE_HAP_FAULT		(((arm_lpae_iopte)0) << 6)
 #define ARM_LPAE_PTE_HAP_READ		(((arm_lpae_iopte)1) << 6)
 #define ARM_LPAE_PTE_HAP_WRITE		(((arm_lpae_iopte)2) << 6)
 /*
  * For !FWB these code to:
  *  1111 = Normal outer write back cachable / Inner Write Back Cachable
  *         Permit S1 to override
  *  0101 = Normal Non-cachable / Inner Non-cachable
  *  0001 = Device / Device-nGnRE
  * For S2FWB these code:
  *  0110 Force Normal Write Back
  *  0101 Normal* is forced Normal-NC, Device unchanged
  *  0001 Force Device-nGnRE
  */
 #define ARM_LPAE_PTE_MEMATTR_FWB_WB	(((arm_lpae_iopte)0x6) << 2)
 #define ARM_LPAE_PTE_MEMATTR_OIWB	(((arm_lpae_iopte)0xf) << 2)
 #define ARM_LPAE_PTE_MEMATTR_NC		(((arm_lpae_iopte)0x5) << 2)
 #define ARM_LPAE_PTE_MEMATTR_DEV	(((arm_lpae_iopte)0x1) << 2)

 /* Register bits */
 #define ARM_LPAE_VTCR_SL0_MASK		0x3

 #define ARM_LPAE_TCR_T0SZ_SHIFT		0

 #define ARM_LPAE_VTCR_PS_SHIFT		16
 #define ARM_LPAE_VTCR_PS_MASK		0x7

 #define ARM_LPAE_MAIR_ATTR_SHIFT(n)	((n) << 3)
 #define ARM_LPAE_MAIR_ATTR_MASK		0xff
 #define ARM_LPAE_MAIR_ATTR_DEVICE	0x04
 #define ARM_LPAE_MAIR_ATTR_NC		0x44
 #define ARM_LPAE_MAIR_ATTR_INC_OWBRWA	0xf4
 #define ARM_LPAE_MAIR_ATTR_WBRWA	0xff
 #define ARM_LPAE_MAIR_ATTR_IDX_NC	0
 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE	1
 #define ARM_LPAE_MAIR_ATTR_IDX_DEV	2
 #define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE	3

 #define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
 #define ARM_MALI_LPAE_TTBR_READ_INNER	BIT(2)
 #define ARM_MALI_LPAE_TTBR_SHARE_OUTER	BIT(4)

 #define ARM_MALI_LPAE_MEMATTR_IMP_DEF	0x88ULL
 #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL

 #define ARM_LPAE_TCR_TG0_4K            0
 #define ARM_LPAE_TCR_TG0_64K           1
 #define ARM_LPAE_TCR_TG0_16K           2

 #define ARM_LPAE_TCR_TG1_16K           1
 #define ARM_LPAE_TCR_TG1_4K            2
 #define ARM_LPAE_TCR_TG1_64K           3

 #define ARM_LPAE_TCR_SH_NS             0
 #define ARM_LPAE_TCR_SH_OS             2
 #define ARM_LPAE_TCR_SH_IS             3

 #define ARM_LPAE_TCR_RGN_NC            0
 #define ARM_LPAE_TCR_RGN_WBWA          1
 #define ARM_LPAE_TCR_RGN_WT            2
 #define ARM_LPAE_TCR_RGN_WB            3

 #define ARM_LPAE_TCR_PS_32_BIT         0x0ULL
 #define ARM_LPAE_TCR_PS_36_BIT         0x1ULL
 #define ARM_LPAE_TCR_PS_40_BIT         0x2ULL
 #define ARM_LPAE_TCR_PS_42_BIT         0x3ULL
 #define ARM_LPAE_TCR_PS_44_BIT         0x4ULL
 #define ARM_LPAE_TCR_PS_48_BIT         0x5ULL
 #define ARM_LPAE_TCR_PS_52_BIT         0x6ULL

 /* IOPTE accessors */
 #define iopte_type(pte)					\
 	(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)

 #define iopte_prot(pte)	((pte) & ARM_LPAE_PTE_ATTR_MASK)

 static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
 			      enum io_pgtable_fmt fmt)
 {
 	if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
 		return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE;

 	return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK;
 }

 static inline bool iopte_table(arm_lpae_iopte pte, int lvl)
 {
 	if (lvl == (ARM_LPAE_MAX_LEVELS - 1))
 		return false;
 	return iopte_type(pte) == ARM_LPAE_PTE_TYPE_TABLE;
 }

 #ifdef __KVM_NVHE_HYPERVISOR__
 #include <nvhe/memory.h>
 #define __arm_lpae_virt_to_phys	hyp_virt_to_phys
 #define __arm_lpae_phys_to_virt	hyp_phys_to_virt

 struct io_pgtable *kvm_arm_io_pgtable_alloc(struct io_pgtable_cfg *cfg,
 					    void *cookie,
 					    enum io_pgtable_fmt fmt,
 					    int *out_ret);
 #else
 #define __arm_lpae_virt_to_phys	__pa
 #define __arm_lpae_phys_to_virt	__va
 #endif

 static inline phys_addr_t iopte_to_paddr(arm_lpae_iopte pte,
 					 struct arm_lpae_io_pgtable *data)
 {
 	u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK;

 	if (ARM_LPAE_GRANULE(data) < SZ_64K)
 		return paddr;

 	/* Rotate the packed high-order bits back to the top */
 	return (paddr | (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4);
 }

 #define iopte_deref(pte, d) __arm_lpae_phys_to_virt(iopte_to_paddr(pte, d))

 static inline arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr,
 					    struct arm_lpae_io_pgtable *data)
 {
 	arm_lpae_iopte pte = paddr;

 	/* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */
 	return (pte | (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK;
 }

 /* Generic functions */
 int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
 		       phys_addr_t paddr, size_t pgsize, size_t pgcount,
 		       int iommu_prot, gfp_t gfp, size_t *mapped);
 size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova,
 			    size_t pgsize, size_t pgcount,
 			    struct iommu_iotlb_gather *gather);

 /* Host/hyp-specific functions */
 void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, struct io_pgtable_cfg *cfg, void *cookie);
 void __arm_lpae_free_pages(void *pages, size_t size, struct io_pgtable_cfg *cfg, void *cookie);
 void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
 			 struct io_pgtable_cfg *cfg);
 void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
 			     arm_lpae_iopte *ptep);
 void arm_lpae_split_blk(void);
 int arm_lpae_init_pgtable(struct io_pgtable_cfg *cfg,
 			  struct arm_lpae_io_pgtable *data);
 int arm_lpae_init_pgtable_s1(struct io_pgtable_cfg *cfg,
 			     struct arm_lpae_io_pgtable *data,
 			     void *cookie);
 int arm_lpae_init_pgtable_s2(struct io_pgtable_cfg *cfg,
 			     struct arm_lpae_io_pgtable *data,
 			     void *cookie);
 #endif /* IO_PGTABLE_H_ */
	/* SPDX-License-Identifier: GPL-2.0-only */
	#ifndef IO_PGTABLE_ARM_H_
	#define IO_PGTABLE_ARM_H_

	#include <linux/io-pgtable.h>

	typedef u64 arm_lpae_iopte;

	struct arm_lpae_io_pgtable {
	struct io_pgtable iop;

	int pgd_bits;
	int start_level;
	int bits_per_level;

	void *pgd;
	};

	#define ARM_LPAE_MAX_ADDR_BITS 52
	#define ARM_LPAE_S2_MAX_CONCAT_PAGES 16
	#define ARM_LPAE_MAX_LEVELS 4

	/* Struct accessors */
	#define io_pgtable_to_data(x) \
	container_of((x), struct arm_lpae_io_pgtable, iop)

	#define io_pgtable_ops_to_data(x) \
	io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))

	/*
	* Calculate the right shift amount to get to the portion describing level l
	* in a virtual address mapped by the pagetable in d.
	*/
	#define ARM_LPAE_LVL_SHIFT(l,d) \
	(((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \
	ilog2(sizeof(arm_lpae_iopte)))

	#define ARM_LPAE_GRANULE(d) \
	(sizeof(arm_lpae_iopte) << (d)->bits_per_level)
	#define ARM_LPAE_PGD_SIZE(d) \
	(sizeof(arm_lpae_iopte) << (d)->pgd_bits)

	#define ARM_LPAE_PTES_PER_TABLE(d) \
	(ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte)))

	/*
	* Calculate the index at level l used to map virtual address a using the
	* pagetable in d.
	*/
	#define ARM_LPAE_PGD_IDX(l,d) \
	((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)

	#define ARM_LPAE_LVL_IDX(a,l,d) \
	(((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \
	((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))

	/* Calculate the block/page mapping size at level l for pagetable in d. */
	#define ARM_LPAE_BLOCK_SIZE(l,d) (1ULL << ARM_LPAE_LVL_SHIFT(l,d))

	/* Page table bits */
	#define ARM_LPAE_PTE_TYPE_SHIFT 0
	#define ARM_LPAE_PTE_TYPE_MASK 0x3

	#define ARM_LPAE_PTE_TYPE_BLOCK 1
	#define ARM_LPAE_PTE_TYPE_TABLE 3
	#define ARM_LPAE_PTE_TYPE_PAGE 3

	#define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12)

	#define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63)
	#define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53)
	#define ARM_LPAE_PTE_DBM (((arm_lpae_iopte)1) << 51)
	#define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10)
	#define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8)
	#define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8)
	#define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8)
	#define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5)
	#define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0)

	#define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2)
	/* Ignore the contiguous bit for block splitting */
	#define ARM_LPAE_PTE_ATTR_HI_MASK (ARM_LPAE_PTE_XN \| ARM_LPAE_PTE_DBM)
	#define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK \| \
	ARM_LPAE_PTE_ATTR_HI_MASK)
	/* Software bit for solving coherency races */
	#define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55)

	/* Stage-1 PTE */
	#define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6)
	#define ARM_LPAE_PTE_AP_RDONLY_BIT 7
	#define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)1) << \
	ARM_LPAE_PTE_AP_RDONLY_BIT)
	#define ARM_LPAE_PTE_AP_WR_CLEAN_MASK (ARM_LPAE_PTE_AP_RDONLY \| \
	ARM_LPAE_PTE_DBM)
	#define ARM_LPAE_PTE_ATTRINDX_SHIFT 2
	#define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11)

	/* Stage-2 PTE */
	#define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6)
	#define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6)
	#define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6)
	/*
	* For !FWB these code to:
	* 1111 = Normal outer write back cachable / Inner Write Back Cachable
	* Permit S1 to override
	* 0101 = Normal Non-cachable / Inner Non-cachable
	* 0001 = Device / Device-nGnRE
	* For S2FWB these code:
	* 0110 Force Normal Write Back
	* 0101 Normal* is forced Normal-NC, Device unchanged
	* 0001 Force Device-nGnRE
	*/
	#define ARM_LPAE_PTE_MEMATTR_FWB_WB (((arm_lpae_iopte)0x6) << 2)
	#define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2)
	#define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2)
	#define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2)

	/* Register bits */
	#define ARM_LPAE_VTCR_SL0_MASK 0x3

	#define ARM_LPAE_TCR_T0SZ_SHIFT 0

	#define ARM_LPAE_VTCR_PS_SHIFT 16
	#define ARM_LPAE_VTCR_PS_MASK 0x7

	#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)
	#define ARM_LPAE_MAIR_ATTR_MASK 0xff
	#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04
	#define ARM_LPAE_MAIR_ATTR_NC 0x44
	#define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4
	#define ARM_LPAE_MAIR_ATTR_WBRWA 0xff
	#define ARM_LPAE_MAIR_ATTR_IDX_NC 0
	#define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1
	#define ARM_LPAE_MAIR_ATTR_IDX_DEV 2
	#define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3

	#define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
	#define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2)
	#define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4)

	#define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL
	#define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL

	#define ARM_LPAE_TCR_TG0_4K 0
	#define ARM_LPAE_TCR_TG0_64K 1
	#define ARM_LPAE_TCR_TG0_16K 2

	#define ARM_LPAE_TCR_TG1_16K 1
	#define ARM_LPAE_TCR_TG1_4K 2
	#define ARM_LPAE_TCR_TG1_64K 3

	#define ARM_LPAE_TCR_SH_NS 0
	#define ARM_LPAE_TCR_SH_OS 2
	#define ARM_LPAE_TCR_SH_IS 3

	#define ARM_LPAE_TCR_RGN_NC 0
	#define ARM_LPAE_TCR_RGN_WBWA 1
	#define ARM_LPAE_TCR_RGN_WT 2
	#define ARM_LPAE_TCR_RGN_WB 3

	#define ARM_LPAE_TCR_PS_32_BIT 0x0ULL
	#define ARM_LPAE_TCR_PS_36_BIT 0x1ULL
	#define ARM_LPAE_TCR_PS_40_BIT 0x2ULL
	#define ARM_LPAE_TCR_PS_42_BIT 0x3ULL
	#define ARM_LPAE_TCR_PS_44_BIT 0x4ULL
	#define ARM_LPAE_TCR_PS_48_BIT 0x5ULL
	#define ARM_LPAE_TCR_PS_52_BIT 0x6ULL

	/* IOPTE accessors */
	#define iopte_type(pte) \
	(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)

	#define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK)

	static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
	enum io_pgtable_fmt fmt)
	{
	if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
	return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE;

	return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK;
	}

	static inline bool iopte_table(arm_lpae_iopte pte, int lvl)
	{
	if (lvl == (ARM_LPAE_MAX_LEVELS - 1))
	return false;
	return iopte_type(pte) == ARM_LPAE_PTE_TYPE_TABLE;
	}

	#ifdef __KVM_NVHE_HYPERVISOR__
	#include <nvhe/memory.h>
	#define __arm_lpae_virt_to_phys hyp_virt_to_phys
	#define __arm_lpae_phys_to_virt hyp_phys_to_virt

	struct io_pgtable kvm_arm_io_pgtable_alloc(struct io_pgtable_cfg cfg,
	void *cookie,
	enum io_pgtable_fmt fmt,
	int *out_ret);
	#else
	#define __arm_lpae_virt_to_phys __pa
	#define __arm_lpae_phys_to_virt __va
	#endif

	static inline phys_addr_t iopte_to_paddr(arm_lpae_iopte pte,
	struct arm_lpae_io_pgtable *data)
	{
	u64 paddr = pte & ARM_LPAE_PTE_ADDR_MASK;

	if (ARM_LPAE_GRANULE(data) < SZ_64K)
	return paddr;

	/* Rotate the packed high-order bits back to the top */
	return (paddr \| (paddr << (48 - 12))) & (ARM_LPAE_PTE_ADDR_MASK << 4);
	}

	#define iopte_deref(pte, d) __arm_lpae_phys_to_virt(iopte_to_paddr(pte, d))

	static inline arm_lpae_iopte paddr_to_iopte(phys_addr_t paddr,
	struct arm_lpae_io_pgtable *data)
	{
	arm_lpae_iopte pte = paddr;

	/* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */
	return (pte \| (pte >> (48 - 12))) & ARM_LPAE_PTE_ADDR_MASK;
	}

	/* Generic functions */
	int arm_lpae_map_pages(struct io_pgtable_ops *ops, unsigned long iova,
	phys_addr_t paddr, size_t pgsize, size_t pgcount,
	int iommu_prot, gfp_t gfp, size_t *mapped);
	size_t arm_lpae_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova,
	size_t pgsize, size_t pgcount,
	struct iommu_iotlb_gather *gather);

	/* Host/hyp-specific functions */
	void __arm_lpae_alloc_pages(size_t size, gfp_t gfp, struct io_pgtable_cfg cfg, void *cookie);
	void __arm_lpae_free_pages(void pages, size_t size, struct io_pgtable_cfg cfg, void *cookie);
	void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
	struct io_pgtable_cfg *cfg);
	void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
	arm_lpae_iopte *ptep);
	void arm_lpae_split_blk(void);
	int arm_lpae_init_pgtable(struct io_pgtable_cfg *cfg,
	struct arm_lpae_io_pgtable *data);
	int arm_lpae_init_pgtable_s1(struct io_pgtable_cfg *cfg,
	struct arm_lpae_io_pgtable *data,
	void *cookie);
	int arm_lpae_init_pgtable_s2(struct io_pgtable_cfg *cfg,
	struct arm_lpae_io_pgtable *data,
	void *cookie);
	#endif /* IO_PGTABLE_H_ */