BOOM
Change-Id: I40fc21fc467527c5f14382eb04e4d27dc62b03df
diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index d268685..8648be8 100644
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -892,49 +892,6 @@ void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)
trace_host_mem_abort(esr, addr);
}
-struct pkvm_mem_transition {
- u64 nr_pages;
-
- struct {
- enum pkvm_component_id id;
- /* Address in the initiator's address space */
- u64 addr;
-
- union {
- struct {
- /* Address in the completer's address space */
- u64 completer_addr;
- } host;
- struct {
- u64 completer_addr;
- } hyp;
- struct {
- struct pkvm_hyp_vcpu *hyp_vcpu;
- } guest;
- };
- } initiator;
-
- struct {
- enum pkvm_component_id id;
-
- union {
- struct {
- struct pkvm_hyp_vcpu *hyp_vcpu;
- phys_addr_t phys;
- } guest;
- };
- } completer;
-};
-
-struct pkvm_mem_share {
- const struct pkvm_mem_transition tx;
- const enum kvm_pgtable_prot completer_prot;
-};
-
-struct pkvm_mem_donation {
- const struct pkvm_mem_transition tx;
-};
-
struct check_walk_data {
enum pkvm_page_state desired;
enum pkvm_page_state (*get_page_state)(kvm_pte_t pte, u64 addr);
@@ -1007,127 +964,6 @@ static int __host_set_page_state_range(u64 addr, u64 size,
return host_stage2_idmap_locked(addr, size, prot, true);
}
-static int host_request_owned_transition(u64 *completer_addr,
- const struct pkvm_mem_transition *tx)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
- u64 addr = tx->initiator.addr;
-
- *completer_addr = tx->initiator.host.completer_addr;
- return __host_check_page_state_range(addr, size, PKVM_PAGE_OWNED);
-}
-
-static int host_request_unshare(u64 *completer_addr,
- const struct pkvm_mem_transition *tx)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
- u64 addr = tx->initiator.addr;
-
- *completer_addr = tx->initiator.host.completer_addr;
- return __host_check_page_state_range(addr, size, PKVM_PAGE_SHARED_OWNED);
-}
-
-static int host_initiate_share(u64 *completer_addr,
- const struct pkvm_mem_transition *tx)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
- u64 addr = tx->initiator.addr;
-
- *completer_addr = tx->initiator.host.completer_addr;
- return __host_set_page_state_range(addr, size, PKVM_PAGE_SHARED_OWNED);
-}
-
-static int host_initiate_unshare(u64 *completer_addr,
- const struct pkvm_mem_transition *tx)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
- u64 addr = tx->initiator.addr;
-
- *completer_addr = tx->initiator.host.completer_addr;
- return __host_set_page_state_range(addr, size, PKVM_PAGE_OWNED);
-}
-
-static int host_initiate_donation(u64 *completer_addr,
- const struct pkvm_mem_transition *tx)
-{
- enum pkvm_component_id owner_id = tx->completer.id;
- u64 size = tx->nr_pages * PAGE_SIZE;
-
- *completer_addr = tx->initiator.host.completer_addr;
- return host_stage2_set_owner_locked(tx->initiator.addr, size, owner_id);
-}
-
-static bool __host_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx)
-{
- return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) ||
- tx->initiator.id != PKVM_ID_HYP);
-}
-
-static int __host_ack_transition(u64 addr, const struct pkvm_mem_transition *tx,
- enum pkvm_page_state state)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
-
- if (__host_ack_skip_pgtable_check(tx))
- return 0;
-
- return __host_check_page_state_range(addr, size, state);
-}
-
-static int host_ack_share(u64 addr, const struct pkvm_mem_transition *tx,
- enum kvm_pgtable_prot perms)
-{
- if (perms != PKVM_HOST_MEM_PROT)
- return -EPERM;
-
- return __host_ack_transition(addr, tx, PKVM_NOPAGE);
-}
-
-static int host_ack_donation(u64 addr, const struct pkvm_mem_transition *tx)
-{
- return __host_ack_transition(addr, tx, PKVM_NOPAGE);
-}
-
-static int host_ack_unshare(u64 addr, const struct pkvm_mem_transition *tx)
-{
- return __host_ack_transition(addr, tx, PKVM_PAGE_SHARED_BORROWED);
-}
-
-static int host_complete_share(u64 addr, const struct pkvm_mem_transition *tx,
- enum kvm_pgtable_prot perms)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
- int err;
-
- err = __host_set_page_state_range(addr, size, PKVM_PAGE_SHARED_BORROWED);
- if (err)
- return err;
-
- if (tx->initiator.id == PKVM_ID_GUEST)
- psci_mem_protect_dec(tx->nr_pages);
-
- return 0;
-}
-
-static int host_complete_unshare(u64 addr, const struct pkvm_mem_transition *tx)
-{
- enum pkvm_component_id owner_id = tx->initiator.id;
- u64 size = tx->nr_pages * PAGE_SIZE;
-
- if (tx->initiator.id == PKVM_ID_GUEST)
- psci_mem_protect_inc(tx->nr_pages);
-
- return host_stage2_set_owner_locked(addr, size, owner_id);
-}
-
-static int host_complete_donation(u64 addr, const struct pkvm_mem_transition *tx)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
- enum pkvm_component_id host_id = tx->completer.id;
-
- return host_stage2_set_owner_locked(addr, size, host_id);
-}
-
static enum pkvm_page_state hyp_get_page_state(kvm_pte_t pte, u64 addr)
{
enum pkvm_page_state state = 0;
@@ -1155,94 +991,11 @@ static int __hyp_check_page_state_range(u64 addr, u64 size,
return check_page_state_range(&pkvm_pgtable, addr, size, &d);
}
-static int hyp_request_donation(u64 *completer_addr,
- const struct pkvm_mem_transition *tx)
+static int __hyp_set_page_state_range(u64 addr, u64 size,
+ enum pkvm_page_state state)
{
- u64 size = tx->nr_pages * PAGE_SIZE;
- u64 addr = tx->initiator.addr;
-
- *completer_addr = tx->initiator.hyp.completer_addr;
- return __hyp_check_page_state_range(addr, size, PKVM_PAGE_OWNED);
-}
-
-static int hyp_initiate_donation(u64 *completer_addr,
- const struct pkvm_mem_transition *tx)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
- int ret;
-
- *completer_addr = tx->initiator.hyp.completer_addr;
- ret = kvm_pgtable_hyp_unmap(&pkvm_pgtable, tx->initiator.addr, size);
- return (ret != size) ? -EFAULT : 0;
-}
-
-static bool __hyp_ack_skip_pgtable_check(const struct pkvm_mem_transition *tx)
-{
- return !(IS_ENABLED(CONFIG_NVHE_EL2_DEBUG) ||
- tx->initiator.id != PKVM_ID_HOST);
-}
-
-static int hyp_ack_share(u64 addr, const struct pkvm_mem_transition *tx,
- enum kvm_pgtable_prot perms)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
-
- if (perms != PAGE_HYP)
- return -EPERM;
-
- if (__hyp_ack_skip_pgtable_check(tx))
- return 0;
-
- return __hyp_check_page_state_range(addr, size, PKVM_NOPAGE);
-}
-
-static int hyp_ack_unshare(u64 addr, const struct pkvm_mem_transition *tx)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
-
- if (tx->initiator.id == PKVM_ID_HOST && hyp_page_count((void *)addr))
- return -EBUSY;
-
- if (__hyp_ack_skip_pgtable_check(tx))
- return 0;
-
- return __hyp_check_page_state_range(addr, size,
- PKVM_PAGE_SHARED_BORROWED);
-}
-
-static int hyp_ack_donation(u64 addr, const struct pkvm_mem_transition *tx)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
-
- if (__hyp_ack_skip_pgtable_check(tx))
- return 0;
-
- return __hyp_check_page_state_range(addr, size, PKVM_NOPAGE);
-}
-
-static int hyp_complete_share(u64 addr, const struct pkvm_mem_transition *tx,
- enum kvm_pgtable_prot perms)
-{
- void *start = (void *)addr, *end = start + (tx->nr_pages * PAGE_SIZE);
- enum kvm_pgtable_prot prot;
-
- prot = pkvm_mkstate(perms, PKVM_PAGE_SHARED_BORROWED);
- return pkvm_create_mappings_locked(start, end, prot);
-}
-
-static int hyp_complete_unshare(u64 addr, const struct pkvm_mem_transition *tx)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
- int ret = kvm_pgtable_hyp_unmap(&pkvm_pgtable, addr, size);
-
- return (ret != size) ? -EFAULT : 0;
-}
-
-static int hyp_complete_donation(u64 addr,
- const struct pkvm_mem_transition *tx)
-{
- void *start = (void *)addr, *end = start + (tx->nr_pages * PAGE_SIZE);
- enum kvm_pgtable_prot prot = pkvm_mkstate(PAGE_HYP, PKVM_PAGE_OWNED);
+ void *start = (void *)addr, *end = start + size;
+ enum kvm_pgtable_prot prot = pkvm_mkstate(PAGE_HYP, state);
return pkvm_create_mappings_locked(start, end, prot);
}
@@ -1275,59 +1028,31 @@ static int __guest_check_page_state_range(struct pkvm_hyp_vcpu *vcpu, u64 addr,
return check_page_state_range(&vm->pgt, addr, size, &d);
}
-static int guest_ack_share(u64 addr, const struct pkvm_mem_transition *tx,
- enum kvm_pgtable_prot perms)
+static int __guest_complete_share(struct pkvm_hyp_vcpu *vcpu, u64 phys, u64 ipa, u64 size,
+ enum kvm_pgtable_prot prot)
{
- u64 size = tx->nr_pages * PAGE_SIZE;
-
- if (perms != KVM_PGTABLE_PROT_RWX)
- return -EPERM;
-
- return __guest_check_page_state_range(tx->completer.guest.hyp_vcpu,
- addr, size, PKVM_NOPAGE);
-}
-
-static int guest_ack_donation(u64 addr, const struct pkvm_mem_transition *tx)
-{
- u64 size = tx->nr_pages * PAGE_SIZE;
-
- return __guest_check_page_state_range(tx->completer.guest.hyp_vcpu,
- addr, size, PKVM_NOPAGE);
-}
-
-static int guest_complete_share(u64 addr, const struct pkvm_mem_transition *tx,
- enum kvm_pgtable_prot perms)
-{
- struct pkvm_hyp_vcpu *vcpu = tx->completer.guest.hyp_vcpu;
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
- u64 size = tx->nr_pages * PAGE_SIZE;
- enum kvm_pgtable_prot prot;
- prot = pkvm_mkstate(perms, PKVM_PAGE_SHARED_BORROWED);
- return kvm_pgtable_stage2_map(&vm->pgt, addr, size, tx->completer.guest.phys,
+ prot = pkvm_mkstate(KVM_PGTABLE_PROT_RWX, prot);
+ return kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys,
prot, &vcpu->vcpu.arch.pkvm_memcache);
}
-static int guest_complete_donation(u64 addr, const struct pkvm_mem_transition *tx)
+static int __guest_complete_donation(struct pkvm_hyp_vcpu *vcpu, u64 phys, u64 ipa, u64 size)
{
enum kvm_pgtable_prot prot = pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED);
- struct pkvm_hyp_vcpu *vcpu = tx->completer.guest.hyp_vcpu;
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
- phys_addr_t phys = tx->completer.guest.phys;
- u64 size = tx->nr_pages * PAGE_SIZE;
int err;
- if (tx->initiator.id == PKVM_ID_HOST)
- psci_mem_protect_inc(tx->nr_pages);
+ psci_mem_protect_inc(size >> PAGE_SHIFT);
- if (pkvm_ipa_range_has_pvmfw(vm, addr, addr + size)) {
+ if (pkvm_ipa_range_has_pvmfw(vm, ipa, ipa + size)) {
if (WARN_ON(!pkvm_hyp_vcpu_is_protected(vcpu))) {
err = -EPERM;
goto err_undo_psci;
}
- WARN_ON(tx->initiator.id != PKVM_ID_HOST);
- err = pkvm_load_pvmfw_pages(vm, addr, phys, size);
+ err = pkvm_load_pvmfw_pages(vm, ipa, phys, size);
if (err)
goto err_undo_psci;
}
@@ -1336,52 +1061,28 @@ static int guest_complete_donation(u64 addr, const struct pkvm_mem_transition *t
* If this fails, we effectively leak the pages since they're now
* owned by the guest but not mapped into its stage-2 page-table.
*/
- return kvm_pgtable_stage2_map(&vm->pgt, addr, size, phys, prot,
+ return kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys, prot,
&vcpu->vcpu.arch.pkvm_memcache);
err_undo_psci:
- if (tx->initiator.id == PKVM_ID_HOST)
- psci_mem_protect_dec(tx->nr_pages);
+ psci_mem_protect_dec(size >> PAGE_SHIFT);
return err;
}
-static int __guest_get_completer_addr(u64 *completer_addr, phys_addr_t phys,
- const struct pkvm_mem_transition *tx)
+static int __guest_request_transition(struct pkvm_hyp_vcpu *vcpu, u64 ipa,
+ enum pkvm_page_state desired, u64 *phys)
{
- switch (tx->completer.id) {
- case PKVM_ID_HOST:
- *completer_addr = phys;
- break;
- case PKVM_ID_HYP:
- *completer_addr = (u64)__hyp_va(phys);
- break;
- default:
- return -EINVAL;
- }
-
- return 0;
-}
-
-static int __guest_request_page_transition(u64 *completer_addr,
- const struct pkvm_mem_transition *tx,
- enum pkvm_page_state desired)
-{
- struct pkvm_hyp_vcpu *vcpu = tx->initiator.guest.hyp_vcpu;
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
enum pkvm_page_state state;
- phys_addr_t phys;
kvm_pte_t pte;
u32 level;
int ret;
- if (tx->nr_pages != 1)
- return -E2BIG;
-
- ret = kvm_pgtable_get_leaf(&vm->pgt, tx->initiator.addr, &pte, &level);
+ ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
if (ret)
return ret;
- state = guest_get_page_state(pte, tx->initiator.addr);
+ state = guest_get_page_state(pte, ipa);
if (state == PKVM_NOPAGE)
return -EFAULT;
@@ -1397,393 +1098,52 @@ static int __guest_request_page_transition(u64 *completer_addr,
if (level != KVM_PGTABLE_MAX_LEVELS - 1)
return -EINVAL;
- phys = kvm_pte_to_phys(pte);
- if (!addr_is_allowed_memory(phys))
+ *phys = kvm_pte_to_phys(pte);
+ if (!addr_is_allowed_memory(*phys))
return -EINVAL;
- return __guest_get_completer_addr(completer_addr, phys, tx);
+ return 0;
}
-static int guest_request_share(u64 *completer_addr,
- const struct pkvm_mem_transition *tx)
+static int __guest_initiate_transition(struct pkvm_hyp_vcpu *vcpu, u64 ipa,
+ enum pkvm_page_state state)
{
- return __guest_request_page_transition(completer_addr, tx,
- PKVM_PAGE_OWNED);
-}
-
-static int guest_request_unshare(u64 *completer_addr,
- const struct pkvm_mem_transition *tx)
-{
- return __guest_request_page_transition(completer_addr, tx,
- PKVM_PAGE_SHARED_OWNED);
-}
-
-static int __guest_initiate_page_transition(u64 *completer_addr,
- const struct pkvm_mem_transition *tx,
- enum pkvm_page_state state)
-{
- struct pkvm_hyp_vcpu *vcpu = tx->initiator.guest.hyp_vcpu;
struct kvm_hyp_memcache *mc = &vcpu->vcpu.arch.pkvm_memcache;
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
- u64 size = tx->nr_pages * PAGE_SIZE;
- u64 addr = tx->initiator.addr;
enum kvm_pgtable_prot prot;
phys_addr_t phys;
kvm_pte_t pte;
int ret;
- ret = kvm_pgtable_get_leaf(&vm->pgt, addr, &pte, NULL);
+ ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, NULL);
if (ret)
return ret;
phys = kvm_pte_to_phys(pte);
prot = pkvm_mkstate(kvm_pgtable_stage2_pte_prot(pte), state);
- ret = kvm_pgtable_stage2_map(&vm->pgt, addr, size, phys, prot, mc);
- if (ret)
- return ret;
- return __guest_get_completer_addr(completer_addr, phys, tx);
-}
-
-static int guest_initiate_share(u64 *completer_addr,
- const struct pkvm_mem_transition *tx)
-{
- return __guest_initiate_page_transition(completer_addr, tx,
- PKVM_PAGE_SHARED_OWNED);
-}
-
-static int guest_initiate_unshare(u64 *completer_addr,
- const struct pkvm_mem_transition *tx)
-{
- return __guest_initiate_page_transition(completer_addr, tx,
- PKVM_PAGE_OWNED);
-}
-
-static int check_share(struct pkvm_mem_share *share)
-{
- const struct pkvm_mem_transition *tx = &share->tx;
- u64 completer_addr;
- int ret;
-
- switch (tx->initiator.id) {
- case PKVM_ID_HOST:
- ret = host_request_owned_transition(&completer_addr, tx);
- break;
- case PKVM_ID_GUEST:
- ret = guest_request_share(&completer_addr, tx);
- break;
- default:
- ret = -EINVAL;
- }
-
- if (ret)
- return ret;
-
- switch (tx->completer.id) {
- case PKVM_ID_HOST:
- ret = host_ack_share(completer_addr, tx, share->completer_prot);
- break;
- case PKVM_ID_HYP:
- ret = hyp_ack_share(completer_addr, tx, share->completer_prot);
- break;
- case PKVM_ID_GUEST:
- ret = guest_ack_share(completer_addr, tx, share->completer_prot);
- break;
- case PKVM_ID_FFA:
- /*
- * We only check the host; the secure side will check the other
- * end when we forward the FFA call.
- */
- ret = 0;
- break;
- default:
- ret = -EINVAL;
- }
-
- return ret;
-}
-
-static int __do_share(struct pkvm_mem_share *share)
-{
- const struct pkvm_mem_transition *tx = &share->tx;
- u64 completer_addr;
- int ret;
-
- switch (tx->initiator.id) {
- case PKVM_ID_HOST:
- ret = host_initiate_share(&completer_addr, tx);
- break;
- case PKVM_ID_GUEST:
- ret = guest_initiate_share(&completer_addr, tx);
- break;
- default:
- ret = -EINVAL;
- }
-
- if (ret)
- return ret;
-
- switch (tx->completer.id) {
- case PKVM_ID_HOST:
- ret = host_complete_share(completer_addr, tx, share->completer_prot);
- break;
- case PKVM_ID_HYP:
- ret = hyp_complete_share(completer_addr, tx, share->completer_prot);
- break;
- case PKVM_ID_GUEST:
- ret = guest_complete_share(completer_addr, tx, share->completer_prot);
- break;
- case PKVM_ID_FFA:
- /*
- * We're not responsible for any secure page-tables, so there's
- * nothing to do here.
- */
- ret = 0;
- break;
- default:
- ret = -EINVAL;
- }
-
- return ret;
-}
-
-/*
- * do_share():
- *
- * The page owner grants access to another component with a given set
- * of permissions.
- *
- * Initiator: OWNED => SHARED_OWNED
- * Completer: NOPAGE => SHARED_BORROWED
- */
-static int do_share(struct pkvm_mem_share *share)
-{
- int ret;
-
- ret = check_share(share);
- if (ret)
- return ret;
-
- return WARN_ON(__do_share(share));
-}
-
-static int check_unshare(struct pkvm_mem_share *share)
-{
- const struct pkvm_mem_transition *tx = &share->tx;
- u64 completer_addr;
- int ret;
-
- switch (tx->initiator.id) {
- case PKVM_ID_HOST:
- ret = host_request_unshare(&completer_addr, tx);
- break;
- case PKVM_ID_GUEST:
- ret = guest_request_unshare(&completer_addr, tx);
- break;
- default:
- ret = -EINVAL;
- }
-
- if (ret)
- return ret;
-
- switch (tx->completer.id) {
- case PKVM_ID_HOST:
- ret = host_ack_unshare(completer_addr, tx);
- break;
- case PKVM_ID_HYP:
- ret = hyp_ack_unshare(completer_addr, tx);
- break;
- case PKVM_ID_FFA:
- /* See check_share() */
- ret = 0;
- break;
- default:
- ret = -EINVAL;
- }
-
- return ret;
-}
-
-static int __do_unshare(struct pkvm_mem_share *share)
-{
- const struct pkvm_mem_transition *tx = &share->tx;
- u64 completer_addr;
- int ret;
-
- switch (tx->initiator.id) {
- case PKVM_ID_HOST:
- ret = host_initiate_unshare(&completer_addr, tx);
- break;
- case PKVM_ID_GUEST:
- ret = guest_initiate_unshare(&completer_addr, tx);
- break;
- default:
- ret = -EINVAL;
- }
-
- if (ret)
- return ret;
-
- switch (tx->completer.id) {
- case PKVM_ID_HOST:
- ret = host_complete_unshare(completer_addr, tx);
- break;
- case PKVM_ID_HYP:
- ret = hyp_complete_unshare(completer_addr, tx);
- break;
- case PKVM_ID_FFA:
- /* See __do_share() */
- ret = 0;
- break;
- default:
- ret = -EINVAL;
- }
-
- return ret;
-}
-
-/*
- * do_unshare():
- *
- * The page owner revokes access from another component for a range of
- * pages which were previously shared using do_share().
- *
- * Initiator: SHARED_OWNED => OWNED
- * Completer: SHARED_BORROWED => NOPAGE
- */
-static int do_unshare(struct pkvm_mem_share *share)
-{
- int ret;
-
- ret = check_unshare(share);
- if (ret)
- return ret;
-
- return WARN_ON(__do_unshare(share));
-}
-
-static int check_donation(struct pkvm_mem_donation *donation)
-{
- const struct pkvm_mem_transition *tx = &donation->tx;
- u64 completer_addr;
- int ret;
-
- switch (tx->initiator.id) {
- case PKVM_ID_HOST:
- ret = host_request_owned_transition(&completer_addr, tx);
- break;
- case PKVM_ID_HYP:
- ret = hyp_request_donation(&completer_addr, tx);
- break;
- default:
- ret = -EINVAL;
- }
-
- if (ret)
- return ret;
-
- switch (tx->completer.id) {
- case PKVM_ID_HOST:
- ret = host_ack_donation(completer_addr, tx);
- break;
- case PKVM_ID_HYP:
- ret = hyp_ack_donation(completer_addr, tx);
- break;
- case PKVM_ID_GUEST:
- ret = guest_ack_donation(completer_addr, tx);
- break;
- default:
- ret = -EINVAL;
- }
-
- return ret;
-}
-
-static int __do_donate(struct pkvm_mem_donation *donation)
-{
- const struct pkvm_mem_transition *tx = &donation->tx;
- u64 completer_addr;
- int ret;
-
- switch (tx->initiator.id) {
- case PKVM_ID_HOST:
- ret = host_initiate_donation(&completer_addr, tx);
- break;
- case PKVM_ID_HYP:
- ret = hyp_initiate_donation(&completer_addr, tx);
- break;
- default:
- ret = -EINVAL;
- }
-
- if (ret)
- return ret;
-
- switch (tx->completer.id) {
- case PKVM_ID_HOST:
- ret = host_complete_donation(completer_addr, tx);
- break;
- case PKVM_ID_HYP:
- ret = hyp_complete_donation(completer_addr, tx);
- break;
- case PKVM_ID_GUEST:
- ret = guest_complete_donation(completer_addr, tx);
- break;
- default:
- ret = -EINVAL;
- }
-
- return ret;
-}
-
-/*
- * do_donate():
- *
- * The page owner transfers ownership to another component, losing access
- * as a consequence.
- *
- * Initiator: OWNED => NOPAGE
- * Completer: NOPAGE => OWNED
- */
-static int do_donate(struct pkvm_mem_donation *donation)
-{
- int ret;
-
- ret = check_donation(donation);
- if (ret)
- return ret;
-
- return WARN_ON(__do_donate(donation));
+ return kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys, prot, mc);
}
int __pkvm_host_share_hyp(u64 pfn)
{
- int ret;
u64 host_addr = hyp_pfn_to_phys(pfn);
u64 hyp_addr = (u64)__hyp_va(host_addr);
- struct pkvm_mem_share share = {
- .tx = {
- .nr_pages = 1,
- .initiator = {
- .id = PKVM_ID_HOST,
- .addr = host_addr,
- .host = {
- .completer_addr = hyp_addr,
- },
- },
- .completer = {
- .id = PKVM_ID_HYP,
- },
- },
- .completer_prot = PAGE_HYP,
- };
+ int ret;
host_lock_component();
hyp_lock_component();
- ret = do_share(&share);
+ ret = __host_check_page_state_range(host_addr, PAGE_SIZE, PKVM_PAGE_OWNED);
+ if (ret)
+ goto unlock;
+ ret = __hyp_check_page_state_range(hyp_addr, PAGE_SIZE, PKVM_NOPAGE);
+ if (ret)
+ goto unlock;
+ WARN_ON(__host_set_page_state_range(host_addr, PAGE_SIZE, PKVM_PAGE_SHARED_OWNED));
+ WARN_ON(__hyp_set_page_state_range(hyp_addr, PAGE_SIZE, PKVM_PAGE_SHARED_BORROWED));
+unlock:
hyp_unlock_component();
host_unlock_component();
@@ -1792,30 +1152,24 @@ int __pkvm_host_share_hyp(u64 pfn)
int __pkvm_guest_share_host(struct pkvm_hyp_vcpu *vcpu, u64 ipa)
{
- int ret;
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
- struct pkvm_mem_share share = {
- .tx = {
- .nr_pages = 1,
- .initiator = {
- .id = PKVM_ID_GUEST,
- .addr = ipa,
- .guest = {
- .hyp_vcpu = vcpu,
- },
- },
- .completer = {
- .id = PKVM_ID_HOST,
- },
- },
- .completer_prot = PKVM_HOST_MEM_PROT,
- };
+ u64 phys;
+ int ret;
host_lock_component();
guest_lock_component(vm);
- ret = do_share(&share);
+ ret = __guest_request_transition(vcpu, ipa, PKVM_PAGE_OWNED, &phys);
+ if (ret)
+ goto unlock;
+ ret = __host_check_page_state_range(phys, PAGE_SIZE, PKVM_NOPAGE);
+ if (ret)
+ goto unlock;
+ WARN_ON(__guest_initiate_transition(vcpu, ipa, PKVM_PAGE_SHARED_OWNED));
+ WARN_ON(__host_set_page_state_range(phys, PAGE_SIZE, PKVM_PAGE_SHARED_BORROWED));
+ psci_mem_protect_dec(1);
+unlock:
guest_unlock_component(vm);
host_unlock_component();
@@ -1824,30 +1178,24 @@ int __pkvm_guest_share_host(struct pkvm_hyp_vcpu *vcpu, u64 ipa)
int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *vcpu, u64 ipa)
{
- int ret;
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
- struct pkvm_mem_share share = {
- .tx = {
- .nr_pages = 1,
- .initiator = {
- .id = PKVM_ID_GUEST,
- .addr = ipa,
- .guest = {
- .hyp_vcpu = vcpu,
- },
- },
- .completer = {
- .id = PKVM_ID_HOST,
- },
- },
- .completer_prot = PKVM_HOST_MEM_PROT,
- };
+ u64 phys;
+ int ret;
host_lock_component();
guest_lock_component(vm);
- ret = do_unshare(&share);
+ ret = __guest_request_transition(vcpu, ipa, PKVM_PAGE_SHARED_OWNED, &phys);
+ if (ret)
+ goto unlock;
+ ret = __host_check_page_state_range(phys, PAGE_SIZE, PKVM_PAGE_SHARED_BORROWED);
+ if (ret)
+ goto unlock;
+ psci_mem_protect_inc(1);
+ WARN_ON(__guest_initiate_transition(vcpu, ipa, PKVM_PAGE_OWNED));
+ WARN_ON(host_stage2_set_owner_locked(phys, PAGE_SIZE, PKVM_ID_GUEST));
+unlock:
guest_unlock_component(vm);
host_unlock_component();
@@ -1856,31 +1204,27 @@ int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *vcpu, u64 ipa)
int __pkvm_host_unshare_hyp(u64 pfn)
{
- int ret;
u64 host_addr = hyp_pfn_to_phys(pfn);
u64 hyp_addr = (u64)__hyp_va(host_addr);
- struct pkvm_mem_share share = {
- .tx = {
- .nr_pages = 1,
- .initiator = {
- .id = PKVM_ID_HOST,
- .addr = host_addr,
- .host = {
- .completer_addr = hyp_addr,
- },
- },
- .completer = {
- .id = PKVM_ID_HYP,
- },
- },
- .completer_prot = PAGE_HYP,
- };
+ int ret;
host_lock_component();
hyp_lock_component();
- ret = do_unshare(&share);
+ ret = __host_check_page_state_range(host_addr, PAGE_SIZE, PKVM_PAGE_SHARED_OWNED);
+ if (ret)
+ goto unlock;
+ ret = __hyp_check_page_state_range(hyp_addr, PAGE_SIZE, PKVM_PAGE_SHARED_BORROWED);
+ if (ret)
+ goto unlock;
+ if (hyp_page_count((void *)hyp_addr)) {
+ ret = -EBUSY;
+ goto unlock;
+ }
+ WARN_ON(__host_set_page_state_range(host_addr, PAGE_SIZE, PKVM_PAGE_OWNED));
+ WARN_ON(kvm_pgtable_hyp_unmap(&pkvm_pgtable, hyp_addr, PAGE_SIZE) != PAGE_SIZE);
+unlock:
hyp_unlock_component();
host_unlock_component();
@@ -1889,30 +1233,24 @@ int __pkvm_host_unshare_hyp(u64 pfn)
int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages)
{
- int ret;
u64 host_addr = hyp_pfn_to_phys(pfn);
u64 hyp_addr = (u64)__hyp_va(host_addr);
- struct pkvm_mem_donation donation = {
- .tx = {
- .nr_pages = nr_pages,
- .initiator = {
- .id = PKVM_ID_HOST,
- .addr = host_addr,
- .host = {
- .completer_addr = hyp_addr,
- },
- },
- .completer = {
- .id = PKVM_ID_HYP,
- },
- },
- };
+ u64 size = nr_pages * PAGE_SIZE;
+ int ret;
host_lock_component();
hyp_lock_component();
- ret = do_donate(&donation);
+ ret = __host_check_page_state_range(host_addr, size, PKVM_PAGE_OWNED);
+ if (ret)
+ goto unlock;
+ ret = __hyp_check_page_state_range(hyp_addr, size, PKVM_NOPAGE);
+ if (ret)
+ goto unlock;
+ WARN_ON(host_stage2_set_owner_locked(host_addr, size, PKVM_ID_HYP));
+ WARN_ON(__hyp_set_page_state_range(hyp_addr, PAGE_SIZE, PKVM_PAGE_OWNED));
+unlock:
hyp_unlock_component();
host_unlock_component();
@@ -1921,30 +1259,25 @@ int __pkvm_host_donate_hyp(u64 pfn, u64 nr_pages)
int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages)
{
- int ret;
u64 host_addr = hyp_pfn_to_phys(pfn);
u64 hyp_addr = (u64)__hyp_va(host_addr);
- struct pkvm_mem_donation donation = {
- .tx = {
- .nr_pages = nr_pages,
- .initiator = {
- .id = PKVM_ID_HYP,
- .addr = hyp_addr,
- .hyp = {
- .completer_addr = host_addr,
- },
- },
- .completer = {
- .id = PKVM_ID_HOST,
- },
- },
- };
+ u64 size = nr_pages * PAGE_SIZE;
+ int ret;
host_lock_component();
hyp_lock_component();
- ret = do_donate(&donation);
+ ret = __hyp_check_page_state_range(hyp_addr, size, PKVM_PAGE_OWNED);
+ if (ret)
+ goto unlock;
+ ret = __host_check_page_state_range(host_addr, size, PKVM_NOPAGE);
+ if (ret)
+ goto unlock;
+ WARN_ON(kvm_pgtable_hyp_unmap(&pkvm_pgtable, hyp_addr, PAGE_SIZE) != PAGE_SIZE);
+ WARN_ON(host_stage2_set_owner_locked(host_addr, size, PKVM_ID_HOST));
+
+unlock:
hyp_unlock_component();
host_unlock_component();
@@ -2062,73 +1395,52 @@ void hyp_unpin_shared_mem(void *from, void *to)
int __pkvm_host_share_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu)
{
- int ret;
- u64 host_addr = hyp_pfn_to_phys(pfn);
- u64 guest_addr = hyp_pfn_to_phys(gfn);
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
- struct pkvm_mem_share share = {
- .tx = {
- .nr_pages = 1,
- .initiator = {
- .id = PKVM_ID_HOST,
- .addr = host_addr,
- .host = {
- .completer_addr = guest_addr,
- },
- },
- .completer = {
- .id = PKVM_ID_GUEST,
- .guest = {
- .hyp_vcpu = vcpu,
- .phys = host_addr,
- },
- },
- },
- .completer_prot = KVM_PGTABLE_PROT_RWX,
- };
+ u64 guest_addr = hyp_pfn_to_phys(gfn);
+ u64 host_addr = hyp_pfn_to_phys(pfn);
+ int ret;
host_lock_component();
guest_lock_component(vm);
- ret = do_share(&share);
+ ret = __host_check_page_state_range(host_addr, PAGE_SIZE, PKVM_PAGE_OWNED);
+ if (ret)
+ goto unlock;
+ ret = __guest_check_page_state_range(vcpu, guest_addr, PAGE_SIZE, PKVM_NOPAGE);
+ if (ret)
+ goto unlock;
+ WARN_ON(__host_set_page_state_range(host_addr, PAGE_SIZE, PKVM_PAGE_SHARED_OWNED));
+ WARN_ON(__guest_complete_share(vcpu, host_addr, guest_addr, PAGE_SIZE,
+ PKVM_PAGE_SHARED_BORROWED));
+unlock:
guest_unlock_component(vm);
host_unlock_component();
return ret;
+
}
int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu)
{
- int ret;
- u64 host_addr = hyp_pfn_to_phys(pfn);
- u64 guest_addr = hyp_pfn_to_phys(gfn);
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
- struct pkvm_mem_donation donation = {
- .tx = {
- .nr_pages = 1,
- .initiator = {
- .id = PKVM_ID_HOST,
- .addr = host_addr,
- .host = {
- .completer_addr = guest_addr,
- },
- },
- .completer = {
- .id = PKVM_ID_GUEST,
- .guest = {
- .hyp_vcpu = vcpu,
- .phys = host_addr,
- },
- },
- },
- };
+ u64 guest_addr = hyp_pfn_to_phys(gfn);
+ u64 host_addr = hyp_pfn_to_phys(pfn);
+ int ret;
host_lock_component();
guest_lock_component(vm);
- ret = do_donate(&donation);
+ ret = __host_check_page_state_range(host_addr, PAGE_SIZE, PKVM_PAGE_OWNED);
+ if (ret)
+ goto unlock;
+ ret = __guest_check_page_state_range(vcpu, guest_addr, PAGE_SIZE, PKVM_NOPAGE);
+ if (ret)
+ goto unlock;
+ WARN_ON(host_stage2_set_owner_locked(host_addr, PAGE_SIZE, PKVM_ID_GUEST));
+ WARN_ON(__guest_complete_donation(vcpu, host_addr, guest_addr, PAGE_SIZE));
+unlock:
guest_unlock_component(vm);
host_unlock_component();
@@ -2137,46 +1449,29 @@ int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu)
int __pkvm_host_share_ffa(u64 pfn, u64 nr_pages)
{
+ u64 host_addr = hyp_pfn_to_phys(pfn);
+ u64 size = nr_pages * PAGE_SIZE;
int ret;
- struct pkvm_mem_share share = {
- .tx = {
- .nr_pages = nr_pages,
- .initiator = {
- .id = PKVM_ID_HOST,
- .addr = hyp_pfn_to_phys(pfn),
- },
- .completer = {
- .id = PKVM_ID_FFA,
- },
- },
- };
host_lock_component();
- ret = do_share(&share);
+ ret = __host_check_page_state_range(host_addr, size, PKVM_PAGE_OWNED);
+ if (!ret)
+ WARN_ON(__host_set_page_state_range(host_addr, size, PKVM_PAGE_SHARED_OWNED));
host_unlock_component();
return ret;
}
-
int __pkvm_host_unshare_ffa(u64 pfn, u64 nr_pages)
{
+ u64 host_addr = hyp_pfn_to_phys(pfn);
+ u64 size = nr_pages * PAGE_SIZE;
int ret;
- struct pkvm_mem_share share = {
- .tx = {
- .nr_pages = nr_pages,
- .initiator = {
- .id = PKVM_ID_HOST,
- .addr = hyp_pfn_to_phys(pfn),
- },
- .completer = {
- .id = PKVM_ID_FFA,
- },
- },
- };
host_lock_component();
- ret = do_unshare(&share);
+ ret = __host_check_page_state_range(host_addr, size, PKVM_PAGE_SHARED_OWNED);
+ if (!ret)
+ WARN_ON(__host_set_page_state_range(host_addr, size, PKVM_PAGE_OWNED));
host_unlock_component();
return ret;
