ANDROID: KVM: arm64: Introduce kvm_iommu_ops host_stage2_idmap_complete
This callback is called whenever a series of update have been done to
the IOMMU idmap. This is typically useful to invalidate several TLB
entries in one go instead of doing it on a per-idmap operation for the
sglist based host to guest donation.
The driver may not provide this callback.
Bug: 357781595
Bug: 361026567
Change-Id: Ib677d6738bc16eadf33c4df0405b1967a1f1dc45
Signed-off-by: Vincent Donnefort <vdonnefort@google.com>
diff --git a/arch/arm64/include/asm/kvm_hypevents.h b/arch/arm64/include/asm/kvm_hypevents.h
index 20aee22..24438e9 100644
--- a/arch/arm64/include/asm/kvm_hypevents.h
+++ b/arch/arm64/include/asm/kvm_hypevents.h
@@ -165,4 +165,15 @@ HYP_EVENT(iommu_idmap,
),
HE_PRINTK("from=0x%llx to=0x%llx prot=0x%x", __entry->from, __entry->to, __entry->prot)
);
+
+HYP_EVENT(iommu_idmap_complete,
+ HE_PROTO(bool map),
+ HE_STRUCT(
+ he_field(bool, map)
+ ),
+ HE_ASSIGN(
+ __entry->map = map;
+ ),
+ HE_PRINTK("map=%d", __entry->map)
+);
#endif /* __ARM64_KVM_HYPEVENTS_H_ */
diff --git a/arch/arm64/kvm/hyp/include/nvhe/iommu.h b/arch/arm64/kvm/hyp/include/nvhe/iommu.h
index 64a6d1a..b9ed791 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/iommu.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/iommu.h
@@ -34,6 +34,7 @@ struct kvm_iommu_ops {
phys_addr_t (*iova_to_phys)(struct kvm_hyp_iommu_domain *domain, unsigned long iova);
void (*iotlb_sync)(struct kvm_hyp_iommu_domain *domain,
struct iommu_iotlb_gather *gather);
+ void (*host_stage2_idmap_complete)(bool map);
};
int kvm_iommu_init(void *pool_base, size_t nr_pages, struct kvm_iommu_ops *ops);
@@ -58,6 +59,7 @@ size_t kvm_iommu_unmap_pages(pkvm_handle_t domain_id, unsigned long iova,
size_t pgsize, size_t pgcount);
phys_addr_t kvm_iommu_iova_to_phys(pkvm_handle_t domain_id, unsigned long iova);
bool kvm_iommu_host_dabt_handler(struct user_pt_regs *regs, u64 esr, u64 addr);
+void kvm_iommu_host_stage2_idmap_complete(bool map);
/* Flags for memory allocation for IOMMU drivers */
#define IOMMU_PAGE_NOCACHE BIT(0)
diff --git a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c
index 5d4c566..694ac86 100644
--- a/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c
+++ b/arch/arm64/kvm/hyp/nvhe/iommu/iommu.c
@@ -435,3 +435,13 @@ bool kvm_iommu_host_dabt_handler(struct user_pt_regs *regs, u64 esr, u64 addr)
}
return false;
}
+
+void kvm_iommu_host_stage2_idmap_complete(bool map)
+{
+ if (!kvm_idmap_initialized ||
+ !kvm_iommu_ops->host_stage2_idmap_complete)
+ return;
+
+ trace_iommu_idmap_complete(map);
+ kvm_iommu_ops->host_stage2_idmap_complete(map);
+}
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index 08f3237..4ae70f4 100644
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -708,6 +708,7 @@ static int __host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_i
if (update_iommu) {
prot = owner_id == PKVM_ID_HOST ? PKVM_HOST_MEM_PROT : 0;
kvm_iommu_host_stage2_idmap(addr, addr + size, prot);
+ kvm_iommu_host_stage2_idmap_complete(!!prot);
}
if (!is_memory)
return 0;
@@ -997,6 +998,7 @@ static int __host_set_page_state_range(u64 addr, u64 size,
if (ret)
return ret;
kvm_iommu_host_stage2_idmap(addr, addr + size, PKVM_HOST_MEM_PROT);
+ kvm_iommu_host_stage2_idmap_complete(true);
}
__host_update_page_state(addr, size, state);
@@ -1418,8 +1420,10 @@ int module_change_host_page_prot(u64 pfn, enum kvm_pgtable_prot prot,
} else {
ret = host_stage2_idmap_locked(
addr, size, prot);
- if (update_iommu)
+ if (update_iommu) {
kvm_iommu_host_stage2_idmap(addr, addr + size, prot);
+ kvm_iommu_host_stage2_idmap_complete(!!prot);
+ }
}
if (WARN_ON(ret) || !page || !prot)
@@ -1540,14 +1544,19 @@ int __pkvm_host_split_guest(u64 gfn, u64 size, struct pkvm_hyp_vcpu *vcpu)
return ret;
}
-static int __host_donate_guest(struct pkvm_hyp_vcpu *vcpu, u64 phys, u64 ipa, size_t size)
+static int __host_set_owner_guest(struct pkvm_hyp_vcpu *vcpu, u64 phys, u64 ipa,
+ size_t size, bool is_memory)
{
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
u64 nr_pages = size >> PAGE_SHIFT;
- enum kvm_pgtable_prot prot;
int ret;
- WARN_ON(host_stage2_set_owner_locked(phys, size, PKVM_ID_GUEST));
+ /*
+ * update_iommu=false, the caller must do the update _before_ this function is called. This
+ * intends to protect pvmfw loading.
+ */
+ WARN_ON(__host_stage2_set_owner_locked(phys, size, PKVM_ID_GUEST,
+ is_memory, 0, false));
psci_mem_protect_inc(nr_pages);
if (pkvm_ipa_range_has_pvmfw(vm, ipa, ipa + size)) {
ret = pkvm_load_pvmfw_pages(vm, ipa, phys, size);
@@ -1557,10 +1566,6 @@ static int __host_donate_guest(struct pkvm_hyp_vcpu *vcpu, u64 phys, u64 ipa, si
}
}
- prot = pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED);
- WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys, prot,
- &vcpu->vcpu.arch.stage2_mc, 0));
-
return 0;
}
@@ -1569,6 +1574,8 @@ int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu, u64 n
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
u64 phys = hyp_pfn_to_phys(pfn);
u64 ipa = hyp_pfn_to_phys(gfn);
+ enum kvm_pgtable_prot prot;
+ bool is_memory;
size_t size;
int ret;
@@ -1585,7 +1592,16 @@ int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu, u64 n
if (ret)
goto unlock;
- WARN_ON(__host_donate_guest(vcpu, phys, ipa, size));
+ is_memory = addr_is_memory(phys);
+ if (is_memory) {
+ kvm_iommu_host_stage2_idmap(phys, phys + size, 0);
+ kvm_iommu_host_stage2_idmap_complete(false);
+ }
+ WARN_ON(__host_set_owner_guest(vcpu, phys, ipa, size, is_memory));
+
+ prot = pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED);
+ WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys, prot,
+ &vcpu->vcpu.arch.stage2_mc, 0));
unlock:
guest_unlock_component(vm);
@@ -1662,12 +1678,29 @@ int __pkvm_host_donate_sglist_guest(struct pkvm_hyp_vcpu *vcpu)
goto unlock;
}
+ if (is_memory) {
+ for_each_hyp_ppage(ppage) {
+ size_t size = PAGE_SIZE << ppage->order;
+ u64 phys = hyp_pfn_to_phys(ppage->pfn);
+
+ kvm_iommu_host_stage2_idmap(phys, phys + size, 0);
+ }
+
+ kvm_iommu_host_stage2_idmap_complete(false);
+ }
+
for_each_hyp_ppage(ppage) {
size_t size = PAGE_SIZE << ppage->order;
u64 phys = hyp_pfn_to_phys(ppage->pfn);
u64 ipa = hyp_pfn_to_phys(ppage->gfn);
+ enum kvm_pgtable_prot prot;
- WARN_ON(__host_donate_guest(vcpu, phys, ipa, size));
+ /* Now the sglist is unmapped from the IOMMUs, we can load pvmfw */
+ WARN_ON(__host_set_owner_guest(vcpu, phys, ipa, size, is_memory));
+
+ prot = pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED);
+ WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys, prot,
+ &vcpu->vcpu.arch.stage2_mc, 0));
}
unlock:
