| /* SPDX-License-Identifier: GPL-2.0-only */ |
| /* |
| * IOMMU API for ARM architected SMMU implementations. |
| * |
| * Copyright (C) 2013 ARM Limited |
| * |
| * Author: Will Deacon <will.deacon@arm.com> |
| */ |
| |
| #ifndef _ARM_SMMU_H |
| #define _ARM_SMMU_H |
| |
| #include <linux/atomic.h> |
| #include <linux/bitfield.h> |
| #include <linux/bits.h> |
| #include <linux/clk.h> |
| #include <linux/device.h> |
| #include <linux/io-64-nonatomic-hi-lo.h> |
| #include <linux/io-pgtable.h> |
| #include <linux/iommu.h> |
| #include <linux/mutex.h> |
| #include <linux/spinlock.h> |
| #include <linux/types.h> |
| |
| /* Configuration registers */ |
| #define ARM_SMMU_GR0_sCR0 0x0 |
| #define ARM_SMMU_sCR0_VMID16EN BIT(31) |
| #define ARM_SMMU_sCR0_BSU GENMASK(15, 14) |
| #define ARM_SMMU_sCR0_FB BIT(13) |
| #define ARM_SMMU_sCR0_PTM BIT(12) |
| #define ARM_SMMU_sCR0_VMIDPNE BIT(11) |
| #define ARM_SMMU_sCR0_USFCFG BIT(10) |
| #define ARM_SMMU_sCR0_GCFGFIE BIT(5) |
| #define ARM_SMMU_sCR0_GCFGFRE BIT(4) |
| #define ARM_SMMU_sCR0_EXIDENABLE BIT(3) |
| #define ARM_SMMU_sCR0_GFIE BIT(2) |
| #define ARM_SMMU_sCR0_GFRE BIT(1) |
| #define ARM_SMMU_sCR0_CLIENTPD BIT(0) |
| |
| /* Auxiliary Configuration register */ |
| #define ARM_SMMU_GR0_sACR 0x10 |
| |
| /* Identification registers */ |
| #define ARM_SMMU_GR0_ID0 0x20 |
| #define ARM_SMMU_ID0_S1TS BIT(30) |
| #define ARM_SMMU_ID0_S2TS BIT(29) |
| #define ARM_SMMU_ID0_NTS BIT(28) |
| #define ARM_SMMU_ID0_SMS BIT(27) |
| #define ARM_SMMU_ID0_ATOSNS BIT(26) |
| #define ARM_SMMU_ID0_PTFS_NO_AARCH32 BIT(25) |
| #define ARM_SMMU_ID0_PTFS_NO_AARCH32S BIT(24) |
| #define ARM_SMMU_ID0_NUMIRPT GENMASK(23, 16) |
| #define ARM_SMMU_ID0_CTTW BIT(14) |
| #define ARM_SMMU_ID0_NUMSIDB GENMASK(12, 9) |
| #define ARM_SMMU_ID0_EXIDS BIT(8) |
| #define ARM_SMMU_ID0_NUMSMRG GENMASK(7, 0) |
| |
| #define ARM_SMMU_GR0_ID1 0x24 |
| #define ARM_SMMU_ID1_PAGESIZE BIT(31) |
| #define ARM_SMMU_ID1_NUMPAGENDXB GENMASK(30, 28) |
| #define ARM_SMMU_ID1_NUMS2CB GENMASK(23, 16) |
| #define ARM_SMMU_ID1_NUMCB GENMASK(7, 0) |
| |
| #define ARM_SMMU_GR0_ID2 0x28 |
| #define ARM_SMMU_ID2_VMID16 BIT(15) |
| #define ARM_SMMU_ID2_PTFS_64K BIT(14) |
| #define ARM_SMMU_ID2_PTFS_16K BIT(13) |
| #define ARM_SMMU_ID2_PTFS_4K BIT(12) |
| #define ARM_SMMU_ID2_UBS GENMASK(11, 8) |
| #define ARM_SMMU_ID2_OAS GENMASK(7, 4) |
| #define ARM_SMMU_ID2_IAS GENMASK(3, 0) |
| |
| #define ARM_SMMU_GR0_ID3 0x2c |
| #define ARM_SMMU_GR0_ID4 0x30 |
| #define ARM_SMMU_GR0_ID5 0x34 |
| #define ARM_SMMU_GR0_ID6 0x38 |
| |
| #define ARM_SMMU_GR0_ID7 0x3c |
| #define ARM_SMMU_ID7_MAJOR GENMASK(7, 4) |
| #define ARM_SMMU_ID7_MINOR GENMASK(3, 0) |
| |
| #define ARM_SMMU_GR0_sGFSR 0x48 |
| #define ARM_SMMU_sGFSR_USF BIT(1) |
| |
| #define ARM_SMMU_GR0_sGFSYNR0 0x50 |
| #define ARM_SMMU_GR0_sGFSYNR1 0x54 |
| #define ARM_SMMU_GR0_sGFSYNR2 0x58 |
| |
| /* Global TLB invalidation */ |
| #define ARM_SMMU_GR0_TLBIVMID 0x64 |
| #define ARM_SMMU_GR0_TLBIALLNSNH 0x68 |
| #define ARM_SMMU_GR0_TLBIALLH 0x6c |
| #define ARM_SMMU_GR0_sTLBGSYNC 0x70 |
| |
| #define ARM_SMMU_GR0_sTLBGSTATUS 0x74 |
| #define ARM_SMMU_sTLBGSTATUS_GSACTIVE BIT(0) |
| |
| /* Stream mapping registers */ |
| #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2)) |
| #define ARM_SMMU_SMR_VALID BIT(31) |
| #define ARM_SMMU_SMR_MASK GENMASK(31, 16) |
| #define ARM_SMMU_SMR_ID GENMASK(15, 0) |
| |
| #define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2)) |
| #define ARM_SMMU_S2CR_PRIVCFG GENMASK(25, 24) |
| enum arm_smmu_s2cr_privcfg { |
| S2CR_PRIVCFG_DEFAULT, |
| S2CR_PRIVCFG_DIPAN, |
| S2CR_PRIVCFG_UNPRIV, |
| S2CR_PRIVCFG_PRIV, |
| }; |
| #define ARM_SMMU_S2CR_TYPE GENMASK(17, 16) |
| enum arm_smmu_s2cr_type { |
| S2CR_TYPE_TRANS, |
| S2CR_TYPE_BYPASS, |
| S2CR_TYPE_FAULT, |
| }; |
| #define ARM_SMMU_S2CR_EXIDVALID BIT(10) |
| #define ARM_SMMU_S2CR_CBNDX GENMASK(7, 0) |
| |
| /* Context bank attribute registers */ |
| #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2)) |
| #define ARM_SMMU_CBAR_IRPTNDX GENMASK(31, 24) |
| #define ARM_SMMU_CBAR_TYPE GENMASK(17, 16) |
| enum arm_smmu_cbar_type { |
| CBAR_TYPE_S2_TRANS, |
| CBAR_TYPE_S1_TRANS_S2_BYPASS, |
| CBAR_TYPE_S1_TRANS_S2_FAULT, |
| CBAR_TYPE_S1_TRANS_S2_TRANS, |
| }; |
| #define ARM_SMMU_CBAR_S1_MEMATTR GENMASK(15, 12) |
| #define ARM_SMMU_CBAR_S1_MEMATTR_WB 0xf |
| #define ARM_SMMU_CBAR_S1_BPSHCFG GENMASK(9, 8) |
| #define ARM_SMMU_CBAR_S1_BPSHCFG_NSH 3 |
| #define ARM_SMMU_CBAR_VMID GENMASK(7, 0) |
| |
| #define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2)) |
| |
| #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) |
| #define ARM_SMMU_CBA2R_VMID16 GENMASK(31, 16) |
| #define ARM_SMMU_CBA2R_VA64 BIT(0) |
| |
| #define ARM_SMMU_CB_SCTLR 0x0 |
| #define ARM_SMMU_SCTLR_S1_ASIDPNE BIT(12) |
| #define ARM_SMMU_SCTLR_CFCFG BIT(7) |
| #define ARM_SMMU_SCTLR_CFIE BIT(6) |
| #define ARM_SMMU_SCTLR_CFRE BIT(5) |
| #define ARM_SMMU_SCTLR_E BIT(4) |
| #define ARM_SMMU_SCTLR_AFE BIT(2) |
| #define ARM_SMMU_SCTLR_TRE BIT(1) |
| #define ARM_SMMU_SCTLR_M BIT(0) |
| |
| #define ARM_SMMU_CB_ACTLR 0x4 |
| |
| #define ARM_SMMU_CB_RESUME 0x8 |
| #define ARM_SMMU_RESUME_TERMINATE BIT(0) |
| |
| #define ARM_SMMU_CB_TCR2 0x10 |
| #define ARM_SMMU_TCR2_SEP GENMASK(17, 15) |
| #define ARM_SMMU_TCR2_SEP_UPSTREAM 0x7 |
| #define ARM_SMMU_TCR2_AS BIT(4) |
| #define ARM_SMMU_TCR2_PASIZE GENMASK(3, 0) |
| |
| #define ARM_SMMU_CB_TTBR0 0x20 |
| #define ARM_SMMU_CB_TTBR1 0x28 |
| #define ARM_SMMU_TTBRn_ASID GENMASK_ULL(63, 48) |
| |
| #define ARM_SMMU_CB_TCR 0x30 |
| #define ARM_SMMU_TCR_EAE BIT(31) |
| #define ARM_SMMU_TCR_EPD1 BIT(23) |
| #define ARM_SMMU_TCR_TG0 GENMASK(15, 14) |
| #define ARM_SMMU_TCR_SH0 GENMASK(13, 12) |
| #define ARM_SMMU_TCR_ORGN0 GENMASK(11, 10) |
| #define ARM_SMMU_TCR_IRGN0 GENMASK(9, 8) |
| #define ARM_SMMU_TCR_T0SZ GENMASK(5, 0) |
| |
| #define ARM_SMMU_VTCR_RES1 BIT(31) |
| #define ARM_SMMU_VTCR_PS GENMASK(18, 16) |
| #define ARM_SMMU_VTCR_TG0 ARM_SMMU_TCR_TG0 |
| #define ARM_SMMU_VTCR_SH0 ARM_SMMU_TCR_SH0 |
| #define ARM_SMMU_VTCR_ORGN0 ARM_SMMU_TCR_ORGN0 |
| #define ARM_SMMU_VTCR_IRGN0 ARM_SMMU_TCR_IRGN0 |
| #define ARM_SMMU_VTCR_SL0 GENMASK(7, 6) |
| #define ARM_SMMU_VTCR_T0SZ ARM_SMMU_TCR_T0SZ |
| |
| #define ARM_SMMU_CB_CONTEXTIDR 0x34 |
| #define ARM_SMMU_CB_S1_MAIR0 0x38 |
| #define ARM_SMMU_CB_S1_MAIR1 0x3c |
| |
| #define ARM_SMMU_CB_PAR 0x50 |
| #define ARM_SMMU_CB_PAR_F BIT(0) |
| |
| #define ARM_SMMU_CB_FSR 0x58 |
| #define ARM_SMMU_FSR_MULTI BIT(31) |
| #define ARM_SMMU_FSR_SS BIT(30) |
| #define ARM_SMMU_FSR_UUT BIT(8) |
| #define ARM_SMMU_FSR_ASF BIT(7) |
| #define ARM_SMMU_FSR_TLBLKF BIT(6) |
| #define ARM_SMMU_FSR_TLBMCF BIT(5) |
| #define ARM_SMMU_FSR_EF BIT(4) |
| #define ARM_SMMU_FSR_PF BIT(3) |
| #define ARM_SMMU_FSR_AFF BIT(2) |
| #define ARM_SMMU_FSR_TF BIT(1) |
| |
| #define ARM_SMMU_FSR_IGN (ARM_SMMU_FSR_AFF | \ |
| ARM_SMMU_FSR_ASF | \ |
| ARM_SMMU_FSR_TLBMCF | \ |
| ARM_SMMU_FSR_TLBLKF) |
| |
| #define ARM_SMMU_FSR_FAULT (ARM_SMMU_FSR_MULTI | \ |
| ARM_SMMU_FSR_SS | \ |
| ARM_SMMU_FSR_UUT | \ |
| ARM_SMMU_FSR_EF | \ |
| ARM_SMMU_FSR_PF | \ |
| ARM_SMMU_FSR_TF | \ |
| ARM_SMMU_FSR_IGN) |
| |
| #define ARM_SMMU_CB_FAR 0x60 |
| |
| #define ARM_SMMU_CB_FSYNR0 0x68 |
| #define ARM_SMMU_FSYNR0_WNR BIT(4) |
| |
| #define ARM_SMMU_CB_S1_TLBIVA 0x600 |
| #define ARM_SMMU_CB_S1_TLBIASID 0x610 |
| #define ARM_SMMU_CB_S1_TLBIVAL 0x620 |
| #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630 |
| #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638 |
| #define ARM_SMMU_CB_TLBSYNC 0x7f0 |
| #define ARM_SMMU_CB_TLBSTATUS 0x7f4 |
| #define ARM_SMMU_CB_ATS1PR 0x800 |
| |
| #define ARM_SMMU_CB_ATSR 0x8f0 |
| #define ARM_SMMU_ATSR_ACTIVE BIT(0) |
| |
| |
| /* Maximum number of context banks per SMMU */ |
| #define ARM_SMMU_MAX_CBS 128 |
| |
| |
| /* Shared driver definitions */ |
| enum arm_smmu_arch_version { |
| ARM_SMMU_V1, |
| ARM_SMMU_V1_64K, |
| ARM_SMMU_V2, |
| }; |
| |
| enum arm_smmu_implementation { |
| GENERIC_SMMU, |
| ARM_MMU500, |
| CAVIUM_SMMUV2, |
| QCOM_SMMUV2, |
| }; |
| |
| struct arm_smmu_device { |
| struct device *dev; |
| |
| void __iomem *base; |
| unsigned int numpage; |
| unsigned int pgshift; |
| |
| #define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0) |
| #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1) |
| #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2) |
| #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3) |
| #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4) |
| #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5) |
| #define ARM_SMMU_FEAT_VMID16 (1 << 6) |
| #define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7) |
| #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8) |
| #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9) |
| #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10) |
| #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11) |
| #define ARM_SMMU_FEAT_EXIDS (1 << 12) |
| u32 features; |
| |
| enum arm_smmu_arch_version version; |
| enum arm_smmu_implementation model; |
| const struct arm_smmu_impl *impl; |
| |
| u32 num_context_banks; |
| u32 num_s2_context_banks; |
| DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS); |
| struct arm_smmu_cb *cbs; |
| atomic_t irptndx; |
| |
| u32 num_mapping_groups; |
| u16 streamid_mask; |
| u16 smr_mask_mask; |
| struct arm_smmu_smr *smrs; |
| struct arm_smmu_s2cr *s2crs; |
| struct mutex stream_map_mutex; |
| |
| unsigned long va_size; |
| unsigned long ipa_size; |
| unsigned long pa_size; |
| unsigned long pgsize_bitmap; |
| |
| u32 num_global_irqs; |
| u32 num_context_irqs; |
| unsigned int *irqs; |
| struct clk_bulk_data *clks; |
| int num_clks; |
| |
| spinlock_t global_sync_lock; |
| |
| /* IOMMU core code handle */ |
| struct iommu_device iommu; |
| }; |
| |
| enum arm_smmu_context_fmt { |
| ARM_SMMU_CTX_FMT_NONE, |
| ARM_SMMU_CTX_FMT_AARCH64, |
| ARM_SMMU_CTX_FMT_AARCH32_L, |
| ARM_SMMU_CTX_FMT_AARCH32_S, |
| }; |
| |
| struct arm_smmu_cfg { |
| u8 cbndx; |
| u8 irptndx; |
| union { |
| u16 asid; |
| u16 vmid; |
| }; |
| enum arm_smmu_cbar_type cbar; |
| enum arm_smmu_context_fmt fmt; |
| }; |
| #define ARM_SMMU_INVALID_IRPTNDX 0xff |
| |
| enum arm_smmu_domain_stage { |
| ARM_SMMU_DOMAIN_S1 = 0, |
| ARM_SMMU_DOMAIN_S2, |
| ARM_SMMU_DOMAIN_NESTED, |
| ARM_SMMU_DOMAIN_BYPASS, |
| }; |
| |
| struct arm_smmu_domain { |
| struct arm_smmu_device *smmu; |
| struct io_pgtable_ops *pgtbl_ops; |
| const struct iommu_flush_ops *flush_ops; |
| struct arm_smmu_cfg cfg; |
| enum arm_smmu_domain_stage stage; |
| bool non_strict; |
| struct mutex init_mutex; /* Protects smmu pointer */ |
| spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */ |
| struct iommu_domain domain; |
| }; |
| |
| static inline u32 arm_smmu_lpae_tcr(struct io_pgtable_cfg *cfg) |
| { |
| return ARM_SMMU_TCR_EPD1 | |
| FIELD_PREP(ARM_SMMU_TCR_TG0, cfg->arm_lpae_s1_cfg.tcr.tg) | |
| FIELD_PREP(ARM_SMMU_TCR_SH0, cfg->arm_lpae_s1_cfg.tcr.sh) | |
| FIELD_PREP(ARM_SMMU_TCR_ORGN0, cfg->arm_lpae_s1_cfg.tcr.orgn) | |
| FIELD_PREP(ARM_SMMU_TCR_IRGN0, cfg->arm_lpae_s1_cfg.tcr.irgn) | |
| FIELD_PREP(ARM_SMMU_TCR_T0SZ, cfg->arm_lpae_s1_cfg.tcr.tsz); |
| } |
| |
| static inline u32 arm_smmu_lpae_tcr2(struct io_pgtable_cfg *cfg) |
| { |
| return FIELD_PREP(ARM_SMMU_TCR2_PASIZE, cfg->arm_lpae_s1_cfg.tcr.ips) | |
| FIELD_PREP(ARM_SMMU_TCR2_SEP, ARM_SMMU_TCR2_SEP_UPSTREAM); |
| } |
| |
| static inline u32 arm_smmu_lpae_vtcr(struct io_pgtable_cfg *cfg) |
| { |
| return ARM_SMMU_VTCR_RES1 | |
| FIELD_PREP(ARM_SMMU_VTCR_PS, cfg->arm_lpae_s2_cfg.vtcr.ps) | |
| FIELD_PREP(ARM_SMMU_VTCR_TG0, cfg->arm_lpae_s2_cfg.vtcr.tg) | |
| FIELD_PREP(ARM_SMMU_VTCR_SH0, cfg->arm_lpae_s2_cfg.vtcr.sh) | |
| FIELD_PREP(ARM_SMMU_VTCR_ORGN0, cfg->arm_lpae_s2_cfg.vtcr.orgn) | |
| FIELD_PREP(ARM_SMMU_VTCR_IRGN0, cfg->arm_lpae_s2_cfg.vtcr.irgn) | |
| FIELD_PREP(ARM_SMMU_VTCR_SL0, cfg->arm_lpae_s2_cfg.vtcr.sl) | |
| FIELD_PREP(ARM_SMMU_VTCR_T0SZ, cfg->arm_lpae_s2_cfg.vtcr.tsz); |
| } |
| |
| /* Implementation details, yay! */ |
| struct arm_smmu_impl { |
| u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset); |
| void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset, |
| u32 val); |
| u64 (*read_reg64)(struct arm_smmu_device *smmu, int page, int offset); |
| void (*write_reg64)(struct arm_smmu_device *smmu, int page, int offset, |
| u64 val); |
| int (*cfg_probe)(struct arm_smmu_device *smmu); |
| int (*reset)(struct arm_smmu_device *smmu); |
| int (*init_context)(struct arm_smmu_domain *smmu_domain); |
| void (*tlb_sync)(struct arm_smmu_device *smmu, int page, int sync, |
| int status); |
| int (*def_domain_type)(struct device *dev); |
| }; |
| |
| static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n) |
| { |
| return smmu->base + (n << smmu->pgshift); |
| } |
| |
| static inline u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset) |
| { |
| if (smmu->impl && unlikely(smmu->impl->read_reg)) |
| return smmu->impl->read_reg(smmu, page, offset); |
| return readl_relaxed(arm_smmu_page(smmu, page) + offset); |
| } |
| |
| static inline void arm_smmu_writel(struct arm_smmu_device *smmu, int page, |
| int offset, u32 val) |
| { |
| if (smmu->impl && unlikely(smmu->impl->write_reg)) |
| smmu->impl->write_reg(smmu, page, offset, val); |
| else |
| writel_relaxed(val, arm_smmu_page(smmu, page) + offset); |
| } |
| |
| static inline u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset) |
| { |
| if (smmu->impl && unlikely(smmu->impl->read_reg64)) |
| return smmu->impl->read_reg64(smmu, page, offset); |
| return readq_relaxed(arm_smmu_page(smmu, page) + offset); |
| } |
| |
| static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page, |
| int offset, u64 val) |
| { |
| if (smmu->impl && unlikely(smmu->impl->write_reg64)) |
| smmu->impl->write_reg64(smmu, page, offset, val); |
| else |
| writeq_relaxed(val, arm_smmu_page(smmu, page) + offset); |
| } |
| |
| #define ARM_SMMU_GR0 0 |
| #define ARM_SMMU_GR1 1 |
| #define ARM_SMMU_CB(s, n) ((s)->numpage + (n)) |
| |
| #define arm_smmu_gr0_read(s, o) \ |
| arm_smmu_readl((s), ARM_SMMU_GR0, (o)) |
| #define arm_smmu_gr0_write(s, o, v) \ |
| arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v)) |
| |
| #define arm_smmu_gr1_read(s, o) \ |
| arm_smmu_readl((s), ARM_SMMU_GR1, (o)) |
| #define arm_smmu_gr1_write(s, o, v) \ |
| arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v)) |
| |
| #define arm_smmu_cb_read(s, n, o) \ |
| arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o)) |
| #define arm_smmu_cb_write(s, n, o, v) \ |
| arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v)) |
| #define arm_smmu_cb_readq(s, n, o) \ |
| arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o)) |
| #define arm_smmu_cb_writeq(s, n, o, v) \ |
| arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v)) |
| |
| struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu); |
| struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu); |
| |
| int arm_mmu500_reset(struct arm_smmu_device *smmu); |
| |
| #endif /* _ARM_SMMU_H */ |
