| // SPDX-License-Identifier: GPL-2.0 |
| /* |
| * Secure pages management: Migration of pages between normal and secure |
| * memory of KVM guests. |
| * |
| * Copyright 2018 Bharata B Rao, IBM Corp. <bharata@linux.ibm.com> |
| */ |
| |
| /* |
| * A pseries guest can be run as secure guest on Ultravisor-enabled |
| * POWER platforms. On such platforms, this driver will be used to manage |
| * the movement of guest pages between the normal memory managed by |
| * hypervisor (HV) and secure memory managed by Ultravisor (UV). |
| * |
| * The page-in or page-out requests from UV will come to HV as hcalls and |
| * HV will call back into UV via ultracalls to satisfy these page requests. |
| * |
| * Private ZONE_DEVICE memory equal to the amount of secure memory |
| * available in the platform for running secure guests is hotplugged. |
| * Whenever a page belonging to the guest becomes secure, a page from this |
| * private device memory is used to represent and track that secure page |
| * on the HV side. Some pages (like virtio buffers, VPA pages etc) are |
| * shared between UV and HV. However such pages aren't represented by |
| * device private memory and mappings to shared memory exist in both |
| * UV and HV page tables. |
| */ |
| |
| /* |
| * Notes on locking |
| * |
| * kvm->arch.uvmem_lock is a per-guest lock that prevents concurrent |
| * page-in and page-out requests for the same GPA. Concurrent accesses |
| * can either come via UV (guest vCPUs requesting for same page) |
| * or when HV and guest simultaneously access the same page. |
| * This mutex serializes the migration of page from HV(normal) to |
| * UV(secure) and vice versa. So the serialization points are around |
| * migrate_vma routines and page-in/out routines. |
| * |
| * Per-guest mutex comes with a cost though. Mainly it serializes the |
| * fault path as page-out can occur when HV faults on accessing secure |
| * guest pages. Currently UV issues page-in requests for all the guest |
| * PFNs one at a time during early boot (UV_ESM uvcall), so this is |
| * not a cause for concern. Also currently the number of page-outs caused |
| * by HV touching secure pages is very very low. If an when UV supports |
| * overcommitting, then we might see concurrent guest driven page-outs. |
| * |
| * Locking order |
| * |
| * 1. kvm->srcu - Protects KVM memslots |
| * 2. kvm->mm->mmap_lock - find_vma, migrate_vma_pages and helpers, ksm_madvise |
| * 3. kvm->arch.uvmem_lock - protects read/writes to uvmem slots thus acting |
| * as sync-points for page-in/out |
| */ |
| |
| /* |
| * Notes on page size |
| * |
| * Currently UV uses 2MB mappings internally, but will issue H_SVM_PAGE_IN |
| * and H_SVM_PAGE_OUT hcalls in PAGE_SIZE(64K) granularity. HV tracks |
| * secure GPAs at 64K page size and maintains one device PFN for each |
| * 64K secure GPA. UV_PAGE_IN and UV_PAGE_OUT calls by HV are also issued |
| * for 64K page at a time. |
| * |
| * HV faulting on secure pages: When HV touches any secure page, it |
| * faults and issues a UV_PAGE_OUT request with 64K page size. Currently |
| * UV splits and remaps the 2MB page if necessary and copies out the |
| * required 64K page contents. |
| * |
| * Shared pages: Whenever guest shares a secure page, UV will split and |
| * remap the 2MB page if required and issue H_SVM_PAGE_IN with 64K page size. |
| * |
| * HV invalidating a page: When a regular page belonging to secure |
| * guest gets unmapped, HV informs UV with UV_PAGE_INVAL of 64K |
| * page size. Using 64K page size is correct here because any non-secure |
| * page will essentially be of 64K page size. Splitting by UV during sharing |
| * and page-out ensures this. |
| * |
| * Page fault handling: When HV handles page fault of a page belonging |
| * to secure guest, it sends that to UV with a 64K UV_PAGE_IN request. |
| * Using 64K size is correct here too as UV would have split the 2MB page |
| * into 64k mappings and would have done page-outs earlier. |
| * |
| * In summary, the current secure pages handling code in HV assumes |
| * 64K page size and in fact fails any page-in/page-out requests of |
| * non-64K size upfront. If and when UV starts supporting multiple |
| * page-sizes, we need to break this assumption. |
| */ |
| |
| #include <linux/pagemap.h> |
| #include <linux/migrate.h> |
| #include <linux/kvm_host.h> |
| #include <linux/ksm.h> |
| #include <linux/of.h> |
| #include <linux/memremap.h> |
| #include <asm/ultravisor.h> |
| #include <asm/mman.h> |
| #include <asm/kvm_ppc.h> |
| #include <asm/kvm_book3s_uvmem.h> |
| |
| static struct dev_pagemap kvmppc_uvmem_pgmap; |
| static unsigned long *kvmppc_uvmem_bitmap; |
| static DEFINE_SPINLOCK(kvmppc_uvmem_bitmap_lock); |
| |
| /* |
| * States of a GFN |
| * --------------- |
| * The GFN can be in one of the following states. |
| * |
| * (a) Secure - The GFN is secure. The GFN is associated with |
| * a Secure VM, the contents of the GFN is not accessible |
| * to the Hypervisor. This GFN can be backed by a secure-PFN, |
| * or can be backed by a normal-PFN with contents encrypted. |
| * The former is true when the GFN is paged-in into the |
| * ultravisor. The latter is true when the GFN is paged-out |
| * of the ultravisor. |
| * |
| * (b) Shared - The GFN is shared. The GFN is associated with a |
| * a secure VM. The contents of the GFN is accessible to |
| * Hypervisor. This GFN is backed by a normal-PFN and its |
| * content is un-encrypted. |
| * |
| * (c) Normal - The GFN is a normal. The GFN is associated with |
| * a normal VM. The contents of the GFN is accessible to |
| * the Hypervisor. Its content is never encrypted. |
| * |
| * States of a VM. |
| * --------------- |
| * |
| * Normal VM: A VM whose contents are always accessible to |
| * the hypervisor. All its GFNs are normal-GFNs. |
| * |
| * Secure VM: A VM whose contents are not accessible to the |
| * hypervisor without the VM's consent. Its GFNs are |
| * either Shared-GFN or Secure-GFNs. |
| * |
| * Transient VM: A Normal VM that is transitioning to secure VM. |
| * The transition starts on successful return of |
| * H_SVM_INIT_START, and ends on successful return |
| * of H_SVM_INIT_DONE. This transient VM, can have GFNs |
| * in any of the three states; i.e Secure-GFN, Shared-GFN, |
| * and Normal-GFN. The VM never executes in this state |
| * in supervisor-mode. |
| * |
| * Memory slot State. |
| * ----------------------------- |
| * The state of a memory slot mirrors the state of the |
| * VM the memory slot is associated with. |
| * |
| * VM State transition. |
| * -------------------- |
| * |
| * A VM always starts in Normal Mode. |
| * |
| * H_SVM_INIT_START moves the VM into transient state. During this |
| * time the Ultravisor may request some of its GFNs to be shared or |
| * secured. So its GFNs can be in one of the three GFN states. |
| * |
| * H_SVM_INIT_DONE moves the VM entirely from transient state to |
| * secure-state. At this point any left-over normal-GFNs are |
| * transitioned to Secure-GFN. |
| * |
| * H_SVM_INIT_ABORT moves the transient VM back to normal VM. |
| * All its GFNs are moved to Normal-GFNs. |
| * |
| * UV_TERMINATE transitions the secure-VM back to normal-VM. All |
| * the secure-GFN and shared-GFNs are tranistioned to normal-GFN |
| * Note: The contents of the normal-GFN is undefined at this point. |
| * |
| * GFN state implementation: |
| * ------------------------- |
| * |
| * Secure GFN is associated with a secure-PFN; also called uvmem_pfn, |
| * when the GFN is paged-in. Its pfn[] has KVMPPC_GFN_UVMEM_PFN flag |
| * set, and contains the value of the secure-PFN. |
| * It is associated with a normal-PFN; also called mem_pfn, when |
| * the GFN is pagedout. Its pfn[] has KVMPPC_GFN_MEM_PFN flag set. |
| * The value of the normal-PFN is not tracked. |
| * |
| * Shared GFN is associated with a normal-PFN. Its pfn[] has |
| * KVMPPC_UVMEM_SHARED_PFN flag set. The value of the normal-PFN |
| * is not tracked. |
| * |
| * Normal GFN is associated with normal-PFN. Its pfn[] has |
| * no flag set. The value of the normal-PFN is not tracked. |
| * |
| * Life cycle of a GFN |
| * -------------------- |
| * |
| * -------------------------------------------------------------- |
| * | | Share | Unshare | SVM |H_SVM_INIT_DONE| |
| * | |operation |operation | abort/ | | |
| * | | | | terminate | | |
| * ------------------------------------------------------------- |
| * | | | | | | |
| * | Secure | Shared | Secure |Normal |Secure | |
| * | | | | | | |
| * | Shared | Shared | Secure |Normal |Shared | |
| * | | | | | | |
| * | Normal | Shared | Secure |Normal |Secure | |
| * -------------------------------------------------------------- |
| * |
| * Life cycle of a VM |
| * -------------------- |
| * |
| * -------------------------------------------------------------------- |
| * | | start | H_SVM_ |H_SVM_ |H_SVM_ |UV_SVM_ | |
| * | | VM |INIT_START|INIT_DONE|INIT_ABORT |TERMINATE | |
| * | | | | | | | |
| * --------- ---------------------------------------------------------- |
| * | | | | | | | |
| * | Normal | Normal | Transient|Error |Error |Normal | |
| * | | | | | | | |
| * | Secure | Error | Error |Error |Error |Normal | |
| * | | | | | | | |
| * |Transient| N/A | Error |Secure |Normal |Normal | |
| * -------------------------------------------------------------------- |
| */ |
| |
| #define KVMPPC_GFN_UVMEM_PFN (1UL << 63) |
| #define KVMPPC_GFN_MEM_PFN (1UL << 62) |
| #define KVMPPC_GFN_SHARED (1UL << 61) |
| #define KVMPPC_GFN_SECURE (KVMPPC_GFN_UVMEM_PFN | KVMPPC_GFN_MEM_PFN) |
| #define KVMPPC_GFN_FLAG_MASK (KVMPPC_GFN_SECURE | KVMPPC_GFN_SHARED) |
| #define KVMPPC_GFN_PFN_MASK (~KVMPPC_GFN_FLAG_MASK) |
| |
| struct kvmppc_uvmem_slot { |
| struct list_head list; |
| unsigned long nr_pfns; |
| unsigned long base_pfn; |
| unsigned long *pfns; |
| }; |
| struct kvmppc_uvmem_page_pvt { |
| struct kvm *kvm; |
| unsigned long gpa; |
| bool skip_page_out; |
| bool remove_gfn; |
| }; |
| |
| bool kvmppc_uvmem_available(void) |
| { |
| /* |
| * If kvmppc_uvmem_bitmap != NULL, then there is an ultravisor |
| * and our data structures have been initialized successfully. |
| */ |
| return !!kvmppc_uvmem_bitmap; |
| } |
| |
| int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot) |
| { |
| struct kvmppc_uvmem_slot *p; |
| |
| p = kzalloc(sizeof(*p), GFP_KERNEL); |
| if (!p) |
| return -ENOMEM; |
| p->pfns = vcalloc(slot->npages, sizeof(*p->pfns)); |
| if (!p->pfns) { |
| kfree(p); |
| return -ENOMEM; |
| } |
| p->nr_pfns = slot->npages; |
| p->base_pfn = slot->base_gfn; |
| |
| mutex_lock(&kvm->arch.uvmem_lock); |
| list_add(&p->list, &kvm->arch.uvmem_pfns); |
| mutex_unlock(&kvm->arch.uvmem_lock); |
| |
| return 0; |
| } |
| |
| /* |
| * All device PFNs are already released by the time we come here. |
| */ |
| void kvmppc_uvmem_slot_free(struct kvm *kvm, const struct kvm_memory_slot *slot) |
| { |
| struct kvmppc_uvmem_slot *p, *next; |
| |
| mutex_lock(&kvm->arch.uvmem_lock); |
| list_for_each_entry_safe(p, next, &kvm->arch.uvmem_pfns, list) { |
| if (p->base_pfn == slot->base_gfn) { |
| vfree(p->pfns); |
| list_del(&p->list); |
| kfree(p); |
| break; |
| } |
| } |
| mutex_unlock(&kvm->arch.uvmem_lock); |
| } |
| |
| static void kvmppc_mark_gfn(unsigned long gfn, struct kvm *kvm, |
| unsigned long flag, unsigned long uvmem_pfn) |
| { |
| struct kvmppc_uvmem_slot *p; |
| |
| list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) { |
| if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) { |
| unsigned long index = gfn - p->base_pfn; |
| |
| if (flag == KVMPPC_GFN_UVMEM_PFN) |
| p->pfns[index] = uvmem_pfn | flag; |
| else |
| p->pfns[index] = flag; |
| return; |
| } |
| } |
| } |
| |
| /* mark the GFN as secure-GFN associated with @uvmem pfn device-PFN. */ |
| static void kvmppc_gfn_secure_uvmem_pfn(unsigned long gfn, |
| unsigned long uvmem_pfn, struct kvm *kvm) |
| { |
| kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_UVMEM_PFN, uvmem_pfn); |
| } |
| |
| /* mark the GFN as secure-GFN associated with a memory-PFN. */ |
| static void kvmppc_gfn_secure_mem_pfn(unsigned long gfn, struct kvm *kvm) |
| { |
| kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_MEM_PFN, 0); |
| } |
| |
| /* mark the GFN as a shared GFN. */ |
| static void kvmppc_gfn_shared(unsigned long gfn, struct kvm *kvm) |
| { |
| kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_SHARED, 0); |
| } |
| |
| /* mark the GFN as a non-existent GFN. */ |
| static void kvmppc_gfn_remove(unsigned long gfn, struct kvm *kvm) |
| { |
| kvmppc_mark_gfn(gfn, kvm, 0, 0); |
| } |
| |
| /* return true, if the GFN is a secure-GFN backed by a secure-PFN */ |
| static bool kvmppc_gfn_is_uvmem_pfn(unsigned long gfn, struct kvm *kvm, |
| unsigned long *uvmem_pfn) |
| { |
| struct kvmppc_uvmem_slot *p; |
| |
| list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) { |
| if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) { |
| unsigned long index = gfn - p->base_pfn; |
| |
| if (p->pfns[index] & KVMPPC_GFN_UVMEM_PFN) { |
| if (uvmem_pfn) |
| *uvmem_pfn = p->pfns[index] & |
| KVMPPC_GFN_PFN_MASK; |
| return true; |
| } else |
| return false; |
| } |
| } |
| return false; |
| } |
| |
| /* |
| * starting from *gfn search for the next available GFN that is not yet |
| * transitioned to a secure GFN. return the value of that GFN in *gfn. If a |
| * GFN is found, return true, else return false |
| * |
| * Must be called with kvm->arch.uvmem_lock held. |
| */ |
| static bool kvmppc_next_nontransitioned_gfn(const struct kvm_memory_slot *memslot, |
| struct kvm *kvm, unsigned long *gfn) |
| { |
| struct kvmppc_uvmem_slot *p = NULL, *iter; |
| bool ret = false; |
| unsigned long i; |
| |
| list_for_each_entry(iter, &kvm->arch.uvmem_pfns, list) |
| if (*gfn >= iter->base_pfn && *gfn < iter->base_pfn + iter->nr_pfns) { |
| p = iter; |
| break; |
| } |
| if (!p) |
| return ret; |
| /* |
| * The code below assumes, one to one correspondence between |
| * kvmppc_uvmem_slot and memslot. |
| */ |
| for (i = *gfn; i < p->base_pfn + p->nr_pfns; i++) { |
| unsigned long index = i - p->base_pfn; |
| |
| if (!(p->pfns[index] & KVMPPC_GFN_FLAG_MASK)) { |
| *gfn = i; |
| ret = true; |
| break; |
| } |
| } |
| return ret; |
| } |
| |
| static int kvmppc_memslot_page_merge(struct kvm *kvm, |
| const struct kvm_memory_slot *memslot, bool merge) |
| { |
| unsigned long gfn = memslot->base_gfn; |
| unsigned long end, start = gfn_to_hva(kvm, gfn); |
| unsigned long vm_flags; |
| int ret = 0; |
| struct vm_area_struct *vma; |
| int merge_flag = (merge) ? MADV_MERGEABLE : MADV_UNMERGEABLE; |
| |
| if (kvm_is_error_hva(start)) |
| return H_STATE; |
| |
| end = start + (memslot->npages << PAGE_SHIFT); |
| |
| mmap_write_lock(kvm->mm); |
| do { |
| vma = find_vma_intersection(kvm->mm, start, end); |
| if (!vma) { |
| ret = H_STATE; |
| break; |
| } |
| /* Copy vm_flags to avoid partial modifications in ksm_madvise */ |
| vm_flags = vma->vm_flags; |
| ret = ksm_madvise(vma, vma->vm_start, vma->vm_end, |
| merge_flag, &vm_flags); |
| if (ret) { |
| ret = H_STATE; |
| break; |
| } |
| vm_flags_reset(vma, vm_flags); |
| start = vma->vm_end; |
| } while (end > vma->vm_end); |
| |
| mmap_write_unlock(kvm->mm); |
| return ret; |
| } |
| |
| static void __kvmppc_uvmem_memslot_delete(struct kvm *kvm, |
| const struct kvm_memory_slot *memslot) |
| { |
| uv_unregister_mem_slot(kvm->arch.lpid, memslot->id); |
| kvmppc_uvmem_slot_free(kvm, memslot); |
| kvmppc_memslot_page_merge(kvm, memslot, true); |
| } |
| |
| static int __kvmppc_uvmem_memslot_create(struct kvm *kvm, |
| const struct kvm_memory_slot *memslot) |
| { |
| int ret = H_PARAMETER; |
| |
| if (kvmppc_memslot_page_merge(kvm, memslot, false)) |
| return ret; |
| |
| if (kvmppc_uvmem_slot_init(kvm, memslot)) |
| goto out1; |
| |
| ret = uv_register_mem_slot(kvm->arch.lpid, |
| memslot->base_gfn << PAGE_SHIFT, |
| memslot->npages * PAGE_SIZE, |
| 0, memslot->id); |
| if (ret < 0) { |
| ret = H_PARAMETER; |
| goto out; |
| } |
| return 0; |
| out: |
| kvmppc_uvmem_slot_free(kvm, memslot); |
| out1: |
| kvmppc_memslot_page_merge(kvm, memslot, true); |
| return ret; |
| } |
| |
| unsigned long kvmppc_h_svm_init_start(struct kvm *kvm) |
| { |
| struct kvm_memslots *slots; |
| struct kvm_memory_slot *memslot, *m; |
| int ret = H_SUCCESS; |
| int srcu_idx, bkt; |
| |
| kvm->arch.secure_guest = KVMPPC_SECURE_INIT_START; |
| |
| if (!kvmppc_uvmem_bitmap) |
| return H_UNSUPPORTED; |
| |
| /* Only radix guests can be secure guests */ |
| if (!kvm_is_radix(kvm)) |
| return H_UNSUPPORTED; |
| |
| /* NAK the transition to secure if not enabled */ |
| if (!kvm->arch.svm_enabled) |
| return H_AUTHORITY; |
| |
| srcu_idx = srcu_read_lock(&kvm->srcu); |
| |
| /* register the memslot */ |
| slots = kvm_memslots(kvm); |
| kvm_for_each_memslot(memslot, bkt, slots) { |
| ret = __kvmppc_uvmem_memslot_create(kvm, memslot); |
| if (ret) |
| break; |
| } |
| |
| if (ret) { |
| slots = kvm_memslots(kvm); |
| kvm_for_each_memslot(m, bkt, slots) { |
| if (m == memslot) |
| break; |
| __kvmppc_uvmem_memslot_delete(kvm, memslot); |
| } |
| } |
| |
| srcu_read_unlock(&kvm->srcu, srcu_idx); |
| return ret; |
| } |
| |
| /* |
| * Provision a new page on HV side and copy over the contents |
| * from secure memory using UV_PAGE_OUT uvcall. |
| * Caller must held kvm->arch.uvmem_lock. |
| */ |
| static int __kvmppc_svm_page_out(struct vm_area_struct *vma, |
| unsigned long start, |
| unsigned long end, unsigned long page_shift, |
| struct kvm *kvm, unsigned long gpa, struct page *fault_page) |
| { |
| unsigned long src_pfn, dst_pfn = 0; |
| struct migrate_vma mig = { 0 }; |
| struct page *dpage, *spage; |
| struct kvmppc_uvmem_page_pvt *pvt; |
| unsigned long pfn; |
| int ret = U_SUCCESS; |
| |
| memset(&mig, 0, sizeof(mig)); |
| mig.vma = vma; |
| mig.start = start; |
| mig.end = end; |
| mig.src = &src_pfn; |
| mig.dst = &dst_pfn; |
| mig.pgmap_owner = &kvmppc_uvmem_pgmap; |
| mig.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE; |
| mig.fault_page = fault_page; |
| |
| /* The requested page is already paged-out, nothing to do */ |
| if (!kvmppc_gfn_is_uvmem_pfn(gpa >> page_shift, kvm, NULL)) |
| return ret; |
| |
| ret = migrate_vma_setup(&mig); |
| if (ret) |
| return -1; |
| |
| spage = migrate_pfn_to_page(*mig.src); |
| if (!spage || !(*mig.src & MIGRATE_PFN_MIGRATE)) |
| goto out_finalize; |
| |
| if (!is_zone_device_page(spage)) |
| goto out_finalize; |
| |
| dpage = alloc_page_vma(GFP_HIGHUSER, vma, start); |
| if (!dpage) { |
| ret = -1; |
| goto out_finalize; |
| } |
| |
| lock_page(dpage); |
| pvt = spage->zone_device_data; |
| pfn = page_to_pfn(dpage); |
| |
| /* |
| * This function is used in two cases: |
| * - When HV touches a secure page, for which we do UV_PAGE_OUT |
| * - When a secure page is converted to shared page, we *get* |
| * the page to essentially unmap the device page. In this |
| * case we skip page-out. |
| */ |
| if (!pvt->skip_page_out) |
| ret = uv_page_out(kvm->arch.lpid, pfn << page_shift, |
| gpa, 0, page_shift); |
| |
| if (ret == U_SUCCESS) |
| *mig.dst = migrate_pfn(pfn); |
| else { |
| unlock_page(dpage); |
| __free_page(dpage); |
| goto out_finalize; |
| } |
| |
| migrate_vma_pages(&mig); |
| |
| out_finalize: |
| migrate_vma_finalize(&mig); |
| return ret; |
| } |
| |
| static inline int kvmppc_svm_page_out(struct vm_area_struct *vma, |
| unsigned long start, unsigned long end, |
| unsigned long page_shift, |
| struct kvm *kvm, unsigned long gpa, |
| struct page *fault_page) |
| { |
| int ret; |
| |
| mutex_lock(&kvm->arch.uvmem_lock); |
| ret = __kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa, |
| fault_page); |
| mutex_unlock(&kvm->arch.uvmem_lock); |
| |
| return ret; |
| } |
| |
| /* |
| * Drop device pages that we maintain for the secure guest |
| * |
| * We first mark the pages to be skipped from UV_PAGE_OUT when there |
| * is HV side fault on these pages. Next we *get* these pages, forcing |
| * fault on them, do fault time migration to replace the device PTEs in |
| * QEMU page table with normal PTEs from newly allocated pages. |
| */ |
| void kvmppc_uvmem_drop_pages(const struct kvm_memory_slot *slot, |
| struct kvm *kvm, bool skip_page_out) |
| { |
| int i; |
| struct kvmppc_uvmem_page_pvt *pvt; |
| struct page *uvmem_page; |
| struct vm_area_struct *vma = NULL; |
| unsigned long uvmem_pfn, gfn; |
| unsigned long addr; |
| |
| mmap_read_lock(kvm->mm); |
| |
| addr = slot->userspace_addr; |
| |
| gfn = slot->base_gfn; |
| for (i = slot->npages; i; --i, ++gfn, addr += PAGE_SIZE) { |
| |
| /* Fetch the VMA if addr is not in the latest fetched one */ |
| if (!vma || addr >= vma->vm_end) { |
| vma = vma_lookup(kvm->mm, addr); |
| if (!vma) { |
| pr_err("Can't find VMA for gfn:0x%lx\n", gfn); |
| break; |
| } |
| } |
| |
| mutex_lock(&kvm->arch.uvmem_lock); |
| |
| if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) { |
| uvmem_page = pfn_to_page(uvmem_pfn); |
| pvt = uvmem_page->zone_device_data; |
| pvt->skip_page_out = skip_page_out; |
| pvt->remove_gfn = true; |
| |
| if (__kvmppc_svm_page_out(vma, addr, addr + PAGE_SIZE, |
| PAGE_SHIFT, kvm, pvt->gpa, NULL)) |
| pr_err("Can't page out gpa:0x%lx addr:0x%lx\n", |
| pvt->gpa, addr); |
| } else { |
| /* Remove the shared flag if any */ |
| kvmppc_gfn_remove(gfn, kvm); |
| } |
| |
| mutex_unlock(&kvm->arch.uvmem_lock); |
| } |
| |
| mmap_read_unlock(kvm->mm); |
| } |
| |
| unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm) |
| { |
| int srcu_idx, bkt; |
| struct kvm_memory_slot *memslot; |
| |
| /* |
| * Expect to be called only after INIT_START and before INIT_DONE. |
| * If INIT_DONE was completed, use normal VM termination sequence. |
| */ |
| if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)) |
| return H_UNSUPPORTED; |
| |
| if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE) |
| return H_STATE; |
| |
| srcu_idx = srcu_read_lock(&kvm->srcu); |
| |
| kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm)) |
| kvmppc_uvmem_drop_pages(memslot, kvm, false); |
| |
| srcu_read_unlock(&kvm->srcu, srcu_idx); |
| |
| kvm->arch.secure_guest = 0; |
| uv_svm_terminate(kvm->arch.lpid); |
| |
| return H_PARAMETER; |
| } |
| |
| /* |
| * Get a free device PFN from the pool |
| * |
| * Called when a normal page is moved to secure memory (UV_PAGE_IN). Device |
| * PFN will be used to keep track of the secure page on HV side. |
| * |
| * Called with kvm->arch.uvmem_lock held |
| */ |
| static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm) |
| { |
| struct page *dpage = NULL; |
| unsigned long bit, uvmem_pfn; |
| struct kvmppc_uvmem_page_pvt *pvt; |
| unsigned long pfn_last, pfn_first; |
| |
| pfn_first = kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT; |
| pfn_last = pfn_first + |
| (range_len(&kvmppc_uvmem_pgmap.range) >> PAGE_SHIFT); |
| |
| spin_lock(&kvmppc_uvmem_bitmap_lock); |
| bit = find_first_zero_bit(kvmppc_uvmem_bitmap, |
| pfn_last - pfn_first); |
| if (bit >= (pfn_last - pfn_first)) |
| goto out; |
| bitmap_set(kvmppc_uvmem_bitmap, bit, 1); |
| spin_unlock(&kvmppc_uvmem_bitmap_lock); |
| |
| pvt = kzalloc(sizeof(*pvt), GFP_KERNEL); |
| if (!pvt) |
| goto out_clear; |
| |
| uvmem_pfn = bit + pfn_first; |
| kvmppc_gfn_secure_uvmem_pfn(gpa >> PAGE_SHIFT, uvmem_pfn, kvm); |
| |
| pvt->gpa = gpa; |
| pvt->kvm = kvm; |
| |
| dpage = pfn_to_page(uvmem_pfn); |
| dpage->zone_device_data = pvt; |
| zone_device_page_init(dpage); |
| return dpage; |
| out_clear: |
| spin_lock(&kvmppc_uvmem_bitmap_lock); |
| bitmap_clear(kvmppc_uvmem_bitmap, bit, 1); |
| out: |
| spin_unlock(&kvmppc_uvmem_bitmap_lock); |
| return NULL; |
| } |
| |
| /* |
| * Alloc a PFN from private device memory pool. If @pagein is true, |
| * copy page from normal memory to secure memory using UV_PAGE_IN uvcall. |
| */ |
| static int kvmppc_svm_page_in(struct vm_area_struct *vma, |
| unsigned long start, |
| unsigned long end, unsigned long gpa, struct kvm *kvm, |
| unsigned long page_shift, |
| bool pagein) |
| { |
| unsigned long src_pfn, dst_pfn = 0; |
| struct migrate_vma mig = { 0 }; |
| struct page *spage; |
| unsigned long pfn; |
| struct page *dpage; |
| int ret = 0; |
| |
| memset(&mig, 0, sizeof(mig)); |
| mig.vma = vma; |
| mig.start = start; |
| mig.end = end; |
| mig.src = &src_pfn; |
| mig.dst = &dst_pfn; |
| mig.flags = MIGRATE_VMA_SELECT_SYSTEM; |
| |
| ret = migrate_vma_setup(&mig); |
| if (ret) |
| return ret; |
| |
| if (!(*mig.src & MIGRATE_PFN_MIGRATE)) { |
| ret = -1; |
| goto out_finalize; |
| } |
| |
| dpage = kvmppc_uvmem_get_page(gpa, kvm); |
| if (!dpage) { |
| ret = -1; |
| goto out_finalize; |
| } |
| |
| if (pagein) { |
| pfn = *mig.src >> MIGRATE_PFN_SHIFT; |
| spage = migrate_pfn_to_page(*mig.src); |
| if (spage) { |
| ret = uv_page_in(kvm->arch.lpid, pfn << page_shift, |
| gpa, 0, page_shift); |
| if (ret) |
| goto out_finalize; |
| } |
| } |
| |
| *mig.dst = migrate_pfn(page_to_pfn(dpage)); |
| migrate_vma_pages(&mig); |
| out_finalize: |
| migrate_vma_finalize(&mig); |
| return ret; |
| } |
| |
| static int kvmppc_uv_migrate_mem_slot(struct kvm *kvm, |
| const struct kvm_memory_slot *memslot) |
| { |
| unsigned long gfn = memslot->base_gfn; |
| struct vm_area_struct *vma; |
| unsigned long start, end; |
| int ret = 0; |
| |
| mmap_read_lock(kvm->mm); |
| mutex_lock(&kvm->arch.uvmem_lock); |
| while (kvmppc_next_nontransitioned_gfn(memslot, kvm, &gfn)) { |
| ret = H_STATE; |
| start = gfn_to_hva(kvm, gfn); |
| if (kvm_is_error_hva(start)) |
| break; |
| |
| end = start + (1UL << PAGE_SHIFT); |
| vma = find_vma_intersection(kvm->mm, start, end); |
| if (!vma || vma->vm_start > start || vma->vm_end < end) |
| break; |
| |
| ret = kvmppc_svm_page_in(vma, start, end, |
| (gfn << PAGE_SHIFT), kvm, PAGE_SHIFT, false); |
| if (ret) { |
| ret = H_STATE; |
| break; |
| } |
| |
| /* relinquish the cpu if needed */ |
| cond_resched(); |
| } |
| mutex_unlock(&kvm->arch.uvmem_lock); |
| mmap_read_unlock(kvm->mm); |
| return ret; |
| } |
| |
| unsigned long kvmppc_h_svm_init_done(struct kvm *kvm) |
| { |
| struct kvm_memslots *slots; |
| struct kvm_memory_slot *memslot; |
| int srcu_idx, bkt; |
| long ret = H_SUCCESS; |
| |
| if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)) |
| return H_UNSUPPORTED; |
| |
| /* migrate any unmoved normal pfn to device pfns*/ |
| srcu_idx = srcu_read_lock(&kvm->srcu); |
| slots = kvm_memslots(kvm); |
| kvm_for_each_memslot(memslot, bkt, slots) { |
| ret = kvmppc_uv_migrate_mem_slot(kvm, memslot); |
| if (ret) { |
| /* |
| * The pages will remain transitioned. |
| * Its the callers responsibility to |
| * terminate the VM, which will undo |
| * all state of the VM. Till then |
| * this VM is in a erroneous state. |
| * Its KVMPPC_SECURE_INIT_DONE will |
| * remain unset. |
| */ |
| ret = H_STATE; |
| goto out; |
| } |
| } |
| |
| kvm->arch.secure_guest |= KVMPPC_SECURE_INIT_DONE; |
| pr_info("LPID %d went secure\n", kvm->arch.lpid); |
| |
| out: |
| srcu_read_unlock(&kvm->srcu, srcu_idx); |
| return ret; |
| } |
| |
| /* |
| * Shares the page with HV, thus making it a normal page. |
| * |
| * - If the page is already secure, then provision a new page and share |
| * - If the page is a normal page, share the existing page |
| * |
| * In the former case, uses dev_pagemap_ops.migrate_to_ram handler |
| * to unmap the device page from QEMU's page tables. |
| */ |
| static unsigned long kvmppc_share_page(struct kvm *kvm, unsigned long gpa, |
| unsigned long page_shift) |
| { |
| |
| int ret = H_PARAMETER; |
| struct page *uvmem_page; |
| struct kvmppc_uvmem_page_pvt *pvt; |
| unsigned long pfn; |
| unsigned long gfn = gpa >> page_shift; |
| int srcu_idx; |
| unsigned long uvmem_pfn; |
| |
| srcu_idx = srcu_read_lock(&kvm->srcu); |
| mutex_lock(&kvm->arch.uvmem_lock); |
| if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) { |
| uvmem_page = pfn_to_page(uvmem_pfn); |
| pvt = uvmem_page->zone_device_data; |
| pvt->skip_page_out = true; |
| /* |
| * do not drop the GFN. It is a valid GFN |
| * that is transitioned to a shared GFN. |
| */ |
| pvt->remove_gfn = false; |
| } |
| |
| retry: |
| mutex_unlock(&kvm->arch.uvmem_lock); |
| pfn = gfn_to_pfn(kvm, gfn); |
| if (is_error_noslot_pfn(pfn)) |
| goto out; |
| |
| mutex_lock(&kvm->arch.uvmem_lock); |
| if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) { |
| uvmem_page = pfn_to_page(uvmem_pfn); |
| pvt = uvmem_page->zone_device_data; |
| pvt->skip_page_out = true; |
| pvt->remove_gfn = false; /* it continues to be a valid GFN */ |
| kvm_release_pfn_clean(pfn); |
| goto retry; |
| } |
| |
| if (!uv_page_in(kvm->arch.lpid, pfn << page_shift, gpa, 0, |
| page_shift)) { |
| kvmppc_gfn_shared(gfn, kvm); |
| ret = H_SUCCESS; |
| } |
| kvm_release_pfn_clean(pfn); |
| mutex_unlock(&kvm->arch.uvmem_lock); |
| out: |
| srcu_read_unlock(&kvm->srcu, srcu_idx); |
| return ret; |
| } |
| |
| /* |
| * H_SVM_PAGE_IN: Move page from normal memory to secure memory. |
| * |
| * H_PAGE_IN_SHARED flag makes the page shared which means that the same |
| * memory in is visible from both UV and HV. |
| */ |
| unsigned long kvmppc_h_svm_page_in(struct kvm *kvm, unsigned long gpa, |
| unsigned long flags, |
| unsigned long page_shift) |
| { |
| unsigned long start, end; |
| struct vm_area_struct *vma; |
| int srcu_idx; |
| unsigned long gfn = gpa >> page_shift; |
| int ret; |
| |
| if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)) |
| return H_UNSUPPORTED; |
| |
| if (page_shift != PAGE_SHIFT) |
| return H_P3; |
| |
| if (flags & ~H_PAGE_IN_SHARED) |
| return H_P2; |
| |
| if (flags & H_PAGE_IN_SHARED) |
| return kvmppc_share_page(kvm, gpa, page_shift); |
| |
| ret = H_PARAMETER; |
| srcu_idx = srcu_read_lock(&kvm->srcu); |
| mmap_read_lock(kvm->mm); |
| |
| start = gfn_to_hva(kvm, gfn); |
| if (kvm_is_error_hva(start)) |
| goto out; |
| |
| mutex_lock(&kvm->arch.uvmem_lock); |
| /* Fail the page-in request of an already paged-in page */ |
| if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL)) |
| goto out_unlock; |
| |
| end = start + (1UL << page_shift); |
| vma = find_vma_intersection(kvm->mm, start, end); |
| if (!vma || vma->vm_start > start || vma->vm_end < end) |
| goto out_unlock; |
| |
| if (kvmppc_svm_page_in(vma, start, end, gpa, kvm, page_shift, |
| true)) |
| goto out_unlock; |
| |
| ret = H_SUCCESS; |
| |
| out_unlock: |
| mutex_unlock(&kvm->arch.uvmem_lock); |
| out: |
| mmap_read_unlock(kvm->mm); |
| srcu_read_unlock(&kvm->srcu, srcu_idx); |
| return ret; |
| } |
| |
| |
| /* |
| * Fault handler callback that gets called when HV touches any page that |
| * has been moved to secure memory, we ask UV to give back the page by |
| * issuing UV_PAGE_OUT uvcall. |
| * |
| * This eventually results in dropping of device PFN and the newly |
| * provisioned page/PFN gets populated in QEMU page tables. |
| */ |
| static vm_fault_t kvmppc_uvmem_migrate_to_ram(struct vm_fault *vmf) |
| { |
| struct kvmppc_uvmem_page_pvt *pvt = vmf->page->zone_device_data; |
| |
| if (kvmppc_svm_page_out(vmf->vma, vmf->address, |
| vmf->address + PAGE_SIZE, PAGE_SHIFT, |
| pvt->kvm, pvt->gpa, vmf->page)) |
| return VM_FAULT_SIGBUS; |
| else |
| return 0; |
| } |
| |
| /* |
| * Release the device PFN back to the pool |
| * |
| * Gets called when secure GFN tranistions from a secure-PFN |
| * to a normal PFN during H_SVM_PAGE_OUT. |
| * Gets called with kvm->arch.uvmem_lock held. |
| */ |
| static void kvmppc_uvmem_page_free(struct page *page) |
| { |
| unsigned long pfn = page_to_pfn(page) - |
| (kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT); |
| struct kvmppc_uvmem_page_pvt *pvt; |
| |
| spin_lock(&kvmppc_uvmem_bitmap_lock); |
| bitmap_clear(kvmppc_uvmem_bitmap, pfn, 1); |
| spin_unlock(&kvmppc_uvmem_bitmap_lock); |
| |
| pvt = page->zone_device_data; |
| page->zone_device_data = NULL; |
| if (pvt->remove_gfn) |
| kvmppc_gfn_remove(pvt->gpa >> PAGE_SHIFT, pvt->kvm); |
| else |
| kvmppc_gfn_secure_mem_pfn(pvt->gpa >> PAGE_SHIFT, pvt->kvm); |
| kfree(pvt); |
| } |
| |
| static const struct dev_pagemap_ops kvmppc_uvmem_ops = { |
| .page_free = kvmppc_uvmem_page_free, |
| .migrate_to_ram = kvmppc_uvmem_migrate_to_ram, |
| }; |
| |
| /* |
| * H_SVM_PAGE_OUT: Move page from secure memory to normal memory. |
| */ |
| unsigned long |
| kvmppc_h_svm_page_out(struct kvm *kvm, unsigned long gpa, |
| unsigned long flags, unsigned long page_shift) |
| { |
| unsigned long gfn = gpa >> page_shift; |
| unsigned long start, end; |
| struct vm_area_struct *vma; |
| int srcu_idx; |
| int ret; |
| |
| if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)) |
| return H_UNSUPPORTED; |
| |
| if (page_shift != PAGE_SHIFT) |
| return H_P3; |
| |
| if (flags) |
| return H_P2; |
| |
| ret = H_PARAMETER; |
| srcu_idx = srcu_read_lock(&kvm->srcu); |
| mmap_read_lock(kvm->mm); |
| start = gfn_to_hva(kvm, gfn); |
| if (kvm_is_error_hva(start)) |
| goto out; |
| |
| end = start + (1UL << page_shift); |
| vma = find_vma_intersection(kvm->mm, start, end); |
| if (!vma || vma->vm_start > start || vma->vm_end < end) |
| goto out; |
| |
| if (!kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa, NULL)) |
| ret = H_SUCCESS; |
| out: |
| mmap_read_unlock(kvm->mm); |
| srcu_read_unlock(&kvm->srcu, srcu_idx); |
| return ret; |
| } |
| |
| int kvmppc_send_page_to_uv(struct kvm *kvm, unsigned long gfn) |
| { |
| unsigned long pfn; |
| int ret = U_SUCCESS; |
| |
| pfn = gfn_to_pfn(kvm, gfn); |
| if (is_error_noslot_pfn(pfn)) |
| return -EFAULT; |
| |
| mutex_lock(&kvm->arch.uvmem_lock); |
| if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL)) |
| goto out; |
| |
| ret = uv_page_in(kvm->arch.lpid, pfn << PAGE_SHIFT, gfn << PAGE_SHIFT, |
| 0, PAGE_SHIFT); |
| out: |
| kvm_release_pfn_clean(pfn); |
| mutex_unlock(&kvm->arch.uvmem_lock); |
| return (ret == U_SUCCESS) ? RESUME_GUEST : -EFAULT; |
| } |
| |
| int kvmppc_uvmem_memslot_create(struct kvm *kvm, const struct kvm_memory_slot *new) |
| { |
| int ret = __kvmppc_uvmem_memslot_create(kvm, new); |
| |
| if (!ret) |
| ret = kvmppc_uv_migrate_mem_slot(kvm, new); |
| |
| return ret; |
| } |
| |
| void kvmppc_uvmem_memslot_delete(struct kvm *kvm, const struct kvm_memory_slot *old) |
| { |
| __kvmppc_uvmem_memslot_delete(kvm, old); |
| } |
| |
| static u64 kvmppc_get_secmem_size(void) |
| { |
| struct device_node *np; |
| int i, len; |
| const __be32 *prop; |
| u64 size = 0; |
| |
| /* |
| * First try the new ibm,secure-memory nodes which supersede the |
| * secure-memory-ranges property. |
| * If we found some, no need to read the deprecated ones. |
| */ |
| for_each_compatible_node(np, NULL, "ibm,secure-memory") { |
| prop = of_get_property(np, "reg", &len); |
| if (!prop) |
| continue; |
| size += of_read_number(prop + 2, 2); |
| } |
| if (size) |
| return size; |
| |
| np = of_find_compatible_node(NULL, NULL, "ibm,uv-firmware"); |
| if (!np) |
| goto out; |
| |
| prop = of_get_property(np, "secure-memory-ranges", &len); |
| if (!prop) |
| goto out_put; |
| |
| for (i = 0; i < len / (sizeof(*prop) * 4); i++) |
| size += of_read_number(prop + (i * 4) + 2, 2); |
| |
| out_put: |
| of_node_put(np); |
| out: |
| return size; |
| } |
| |
| int kvmppc_uvmem_init(void) |
| { |
| int ret = 0; |
| unsigned long size; |
| struct resource *res; |
| void *addr; |
| unsigned long pfn_last, pfn_first; |
| |
| size = kvmppc_get_secmem_size(); |
| if (!size) { |
| /* |
| * Don't fail the initialization of kvm-hv module if |
| * the platform doesn't export ibm,uv-firmware node. |
| * Let normal guests run on such PEF-disabled platform. |
| */ |
| pr_info("KVMPPC-UVMEM: No support for secure guests\n"); |
| goto out; |
| } |
| |
| res = request_free_mem_region(&iomem_resource, size, "kvmppc_uvmem"); |
| if (IS_ERR(res)) { |
| ret = PTR_ERR(res); |
| goto out; |
| } |
| |
| kvmppc_uvmem_pgmap.type = MEMORY_DEVICE_PRIVATE; |
| kvmppc_uvmem_pgmap.range.start = res->start; |
| kvmppc_uvmem_pgmap.range.end = res->end; |
| kvmppc_uvmem_pgmap.nr_range = 1; |
| kvmppc_uvmem_pgmap.ops = &kvmppc_uvmem_ops; |
| /* just one global instance: */ |
| kvmppc_uvmem_pgmap.owner = &kvmppc_uvmem_pgmap; |
| addr = memremap_pages(&kvmppc_uvmem_pgmap, NUMA_NO_NODE); |
| if (IS_ERR(addr)) { |
| ret = PTR_ERR(addr); |
| goto out_free_region; |
| } |
| |
| pfn_first = res->start >> PAGE_SHIFT; |
| pfn_last = pfn_first + (resource_size(res) >> PAGE_SHIFT); |
| kvmppc_uvmem_bitmap = bitmap_zalloc(pfn_last - pfn_first, GFP_KERNEL); |
| if (!kvmppc_uvmem_bitmap) { |
| ret = -ENOMEM; |
| goto out_unmap; |
| } |
| |
| pr_info("KVMPPC-UVMEM: Secure Memory size 0x%lx\n", size); |
| return ret; |
| out_unmap: |
| memunmap_pages(&kvmppc_uvmem_pgmap); |
| out_free_region: |
| release_mem_region(res->start, size); |
| out: |
| return ret; |
| } |
| |
| void kvmppc_uvmem_free(void) |
| { |
| if (!kvmppc_uvmem_bitmap) |
| return; |
| |
| memunmap_pages(&kvmppc_uvmem_pgmap); |
| release_mem_region(kvmppc_uvmem_pgmap.range.start, |
| range_len(&kvmppc_uvmem_pgmap.range)); |
| bitmap_free(kvmppc_uvmem_bitmap); |
| } |