blob: 7449ef33f0819db43aae1283ac07dddd35ffc636 [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2016-20 Intel Corporation. */
#include <linux/lockdep.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/shmem_fs.h>
#include <linux/suspend.h>
#include <linux/sched/mm.h>
#include "arch.h"
#include "encl.h"
#include "encls.h"
#include "sgx.h"
/*
* ELDU: Load an EPC page as unblocked. For more info, see "OS Management of EPC
* Pages" in the SDM.
*/
static int __sgx_encl_eldu(struct sgx_encl_page *encl_page,
struct sgx_epc_page *epc_page,
struct sgx_epc_page *secs_page)
{
unsigned long va_offset = encl_page->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK;
struct sgx_encl *encl = encl_page->encl;
struct sgx_pageinfo pginfo;
struct sgx_backing b;
pgoff_t page_index;
int ret;
if (secs_page)
page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base);
else
page_index = PFN_DOWN(encl->size);
ret = sgx_encl_get_backing(encl, page_index, &b);
if (ret)
return ret;
pginfo.addr = encl_page->desc & PAGE_MASK;
pginfo.contents = (unsigned long)kmap_atomic(b.contents);
pginfo.metadata = (unsigned long)kmap_atomic(b.pcmd) +
b.pcmd_offset;
if (secs_page)
pginfo.secs = (u64)sgx_get_epc_virt_addr(secs_page);
else
pginfo.secs = 0;
ret = __eldu(&pginfo, sgx_get_epc_virt_addr(epc_page),
sgx_get_epc_virt_addr(encl_page->va_page->epc_page) + va_offset);
if (ret) {
if (encls_failed(ret))
ENCLS_WARN(ret, "ELDU");
ret = -EFAULT;
}
kunmap_atomic((void *)(unsigned long)(pginfo.metadata - b.pcmd_offset));
kunmap_atomic((void *)(unsigned long)pginfo.contents);
sgx_encl_put_backing(&b, false);
return ret;
}
static struct sgx_epc_page *sgx_encl_eldu(struct sgx_encl_page *encl_page,
struct sgx_epc_page *secs_page)
{
unsigned long va_offset = encl_page->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK;
struct sgx_encl *encl = encl_page->encl;
struct sgx_epc_page *epc_page;
int ret;
epc_page = sgx_alloc_epc_page(encl_page, false);
if (IS_ERR(epc_page))
return epc_page;
ret = __sgx_encl_eldu(encl_page, epc_page, secs_page);
if (ret) {
sgx_free_epc_page(epc_page);
return ERR_PTR(ret);
}
sgx_free_va_slot(encl_page->va_page, va_offset);
list_move(&encl_page->va_page->list, &encl->va_pages);
encl_page->desc &= ~SGX_ENCL_PAGE_VA_OFFSET_MASK;
encl_page->epc_page = epc_page;
return epc_page;
}
static struct sgx_encl_page *sgx_encl_load_page(struct sgx_encl *encl,
unsigned long addr,
unsigned long vm_flags)
{
unsigned long vm_prot_bits = vm_flags & (VM_READ | VM_WRITE | VM_EXEC);
struct sgx_epc_page *epc_page;
struct sgx_encl_page *entry;
entry = xa_load(&encl->page_array, PFN_DOWN(addr));
if (!entry)
return ERR_PTR(-EFAULT);
/*
* Verify that the faulted page has equal or higher build time
* permissions than the VMA permissions (i.e. the subset of {VM_READ,
* VM_WRITE, VM_EXECUTE} in vma->vm_flags).
*/
if ((entry->vm_max_prot_bits & vm_prot_bits) != vm_prot_bits)
return ERR_PTR(-EFAULT);
/* Entry successfully located. */
if (entry->epc_page) {
if (entry->desc & SGX_ENCL_PAGE_BEING_RECLAIMED)
return ERR_PTR(-EBUSY);
return entry;
}
if (!(encl->secs.epc_page)) {
epc_page = sgx_encl_eldu(&encl->secs, NULL);
if (IS_ERR(epc_page))
return ERR_CAST(epc_page);
}
epc_page = sgx_encl_eldu(entry, encl->secs.epc_page);
if (IS_ERR(epc_page))
return ERR_CAST(epc_page);
encl->secs_child_cnt++;
sgx_mark_page_reclaimable(entry->epc_page);
return entry;
}
static vm_fault_t sgx_vma_fault(struct vm_fault *vmf)
{
unsigned long addr = (unsigned long)vmf->address;
struct vm_area_struct *vma = vmf->vma;
struct sgx_encl_page *entry;
unsigned long phys_addr;
struct sgx_encl *encl;
vm_fault_t ret;
encl = vma->vm_private_data;
/*
* It's very unlikely but possible that allocating memory for the
* mm_list entry of a forked process failed in sgx_vma_open(). When
* this happens, vm_private_data is set to NULL.
*/
if (unlikely(!encl))
return VM_FAULT_SIGBUS;
mutex_lock(&encl->lock);
entry = sgx_encl_load_page(encl, addr, vma->vm_flags);
if (IS_ERR(entry)) {
mutex_unlock(&encl->lock);
if (PTR_ERR(entry) == -EBUSY)
return VM_FAULT_NOPAGE;
return VM_FAULT_SIGBUS;
}
phys_addr = sgx_get_epc_phys_addr(entry->epc_page);
ret = vmf_insert_pfn(vma, addr, PFN_DOWN(phys_addr));
if (ret != VM_FAULT_NOPAGE) {
mutex_unlock(&encl->lock);
return VM_FAULT_SIGBUS;
}
sgx_encl_test_and_clear_young(vma->vm_mm, entry);
mutex_unlock(&encl->lock);
return VM_FAULT_NOPAGE;
}
static void sgx_vma_open(struct vm_area_struct *vma)
{
struct sgx_encl *encl = vma->vm_private_data;
/*
* It's possible but unlikely that vm_private_data is NULL. This can
* happen in a grandchild of a process, when sgx_encl_mm_add() had
* failed to allocate memory in this callback.
*/
if (unlikely(!encl))
return;
if (sgx_encl_mm_add(encl, vma->vm_mm))
vma->vm_private_data = NULL;
}
/**
* sgx_encl_may_map() - Check if a requested VMA mapping is allowed
* @encl: an enclave pointer
* @start: lower bound of the address range, inclusive
* @end: upper bound of the address range, exclusive
* @vm_flags: VMA flags
*
* Iterate through the enclave pages contained within [@start, @end) to verify
* that the permissions requested by a subset of {VM_READ, VM_WRITE, VM_EXEC}
* do not contain any permissions that are not contained in the build time
* permissions of any of the enclave pages within the given address range.
*
* An enclave creator must declare the strongest permissions that will be
* needed for each enclave page. This ensures that mappings have the identical
* or weaker permissions than the earlier declared permissions.
*
* Return: 0 on success, -EACCES otherwise
*/
int sgx_encl_may_map(struct sgx_encl *encl, unsigned long start,
unsigned long end, unsigned long vm_flags)
{
unsigned long vm_prot_bits = vm_flags & (VM_READ | VM_WRITE | VM_EXEC);
struct sgx_encl_page *page;
unsigned long count = 0;
int ret = 0;
XA_STATE(xas, &encl->page_array, PFN_DOWN(start));
/*
* Disallow READ_IMPLIES_EXEC tasks as their VMA permissions might
* conflict with the enclave page permissions.
*/
if (current->personality & READ_IMPLIES_EXEC)
return -EACCES;
mutex_lock(&encl->lock);
xas_lock(&xas);
xas_for_each(&xas, page, PFN_DOWN(end - 1)) {
if (~page->vm_max_prot_bits & vm_prot_bits) {
ret = -EACCES;
break;
}
/* Reschedule on every XA_CHECK_SCHED iteration. */
if (!(++count % XA_CHECK_SCHED)) {
xas_pause(&xas);
xas_unlock(&xas);
mutex_unlock(&encl->lock);
cond_resched();
mutex_lock(&encl->lock);
xas_lock(&xas);
}
}
xas_unlock(&xas);
mutex_unlock(&encl->lock);
return ret;
}
static int sgx_vma_mprotect(struct vm_area_struct *vma, unsigned long start,
unsigned long end, unsigned long newflags)
{
return sgx_encl_may_map(vma->vm_private_data, start, end, newflags);
}
static int sgx_encl_debug_read(struct sgx_encl *encl, struct sgx_encl_page *page,
unsigned long addr, void *data)
{
unsigned long offset = addr & ~PAGE_MASK;
int ret;
ret = __edbgrd(sgx_get_epc_virt_addr(page->epc_page) + offset, data);
if (ret)
return -EIO;
return 0;
}
static int sgx_encl_debug_write(struct sgx_encl *encl, struct sgx_encl_page *page,
unsigned long addr, void *data)
{
unsigned long offset = addr & ~PAGE_MASK;
int ret;
ret = __edbgwr(sgx_get_epc_virt_addr(page->epc_page) + offset, data);
if (ret)
return -EIO;
return 0;
}
/*
* Load an enclave page to EPC if required, and take encl->lock.
*/
static struct sgx_encl_page *sgx_encl_reserve_page(struct sgx_encl *encl,
unsigned long addr,
unsigned long vm_flags)
{
struct sgx_encl_page *entry;
for ( ; ; ) {
mutex_lock(&encl->lock);
entry = sgx_encl_load_page(encl, addr, vm_flags);
if (PTR_ERR(entry) != -EBUSY)
break;
mutex_unlock(&encl->lock);
}
if (IS_ERR(entry))
mutex_unlock(&encl->lock);
return entry;
}
static int sgx_vma_access(struct vm_area_struct *vma, unsigned long addr,
void *buf, int len, int write)
{
struct sgx_encl *encl = vma->vm_private_data;
struct sgx_encl_page *entry = NULL;
char data[sizeof(unsigned long)];
unsigned long align;
int offset;
int cnt;
int ret = 0;
int i;
/*
* If process was forked, VMA is still there but vm_private_data is set
* to NULL.
*/
if (!encl)
return -EFAULT;
if (!test_bit(SGX_ENCL_DEBUG, &encl->flags))
return -EFAULT;
for (i = 0; i < len; i += cnt) {
entry = sgx_encl_reserve_page(encl, (addr + i) & PAGE_MASK,
vma->vm_flags);
if (IS_ERR(entry)) {
ret = PTR_ERR(entry);
break;
}
align = ALIGN_DOWN(addr + i, sizeof(unsigned long));
offset = (addr + i) & (sizeof(unsigned long) - 1);
cnt = sizeof(unsigned long) - offset;
cnt = min(cnt, len - i);
ret = sgx_encl_debug_read(encl, entry, align, data);
if (ret)
goto out;
if (write) {
memcpy(data + offset, buf + i, cnt);
ret = sgx_encl_debug_write(encl, entry, align, data);
if (ret)
goto out;
} else {
memcpy(buf + i, data + offset, cnt);
}
out:
mutex_unlock(&encl->lock);
if (ret)
break;
}
return ret < 0 ? ret : i;
}
const struct vm_operations_struct sgx_vm_ops = {
.fault = sgx_vma_fault,
.mprotect = sgx_vma_mprotect,
.open = sgx_vma_open,
.access = sgx_vma_access,
};
/**
* sgx_encl_release - Destroy an enclave instance
* @kref: address of a kref inside &sgx_encl
*
* Used together with kref_put(). Frees all the resources associated with the
* enclave and the instance itself.
*/
void sgx_encl_release(struct kref *ref)
{
struct sgx_encl *encl = container_of(ref, struct sgx_encl, refcount);
struct sgx_va_page *va_page;
struct sgx_encl_page *entry;
unsigned long index;
xa_for_each(&encl->page_array, index, entry) {
if (entry->epc_page) {
/*
* The page and its radix tree entry cannot be freed
* if the page is being held by the reclaimer.
*/
if (sgx_unmark_page_reclaimable(entry->epc_page))
continue;
sgx_free_epc_page(entry->epc_page);
encl->secs_child_cnt--;
entry->epc_page = NULL;
}
kfree(entry);
}
xa_destroy(&encl->page_array);
if (!encl->secs_child_cnt && encl->secs.epc_page) {
sgx_free_epc_page(encl->secs.epc_page);
encl->secs.epc_page = NULL;
}
while (!list_empty(&encl->va_pages)) {
va_page = list_first_entry(&encl->va_pages, struct sgx_va_page,
list);
list_del(&va_page->list);
sgx_free_epc_page(va_page->epc_page);
kfree(va_page);
}
if (encl->backing)
fput(encl->backing);
cleanup_srcu_struct(&encl->srcu);
WARN_ON_ONCE(!list_empty(&encl->mm_list));
/* Detect EPC page leak's. */
WARN_ON_ONCE(encl->secs_child_cnt);
WARN_ON_ONCE(encl->secs.epc_page);
kfree(encl);
}
/*
* 'mm' is exiting and no longer needs mmu notifications.
*/
static void sgx_mmu_notifier_release(struct mmu_notifier *mn,
struct mm_struct *mm)
{
struct sgx_encl_mm *encl_mm = container_of(mn, struct sgx_encl_mm, mmu_notifier);
struct sgx_encl_mm *tmp = NULL;
/*
* The enclave itself can remove encl_mm. Note, objects can't be moved
* off an RCU protected list, but deletion is ok.
*/
spin_lock(&encl_mm->encl->mm_lock);
list_for_each_entry(tmp, &encl_mm->encl->mm_list, list) {
if (tmp == encl_mm) {
list_del_rcu(&encl_mm->list);
break;
}
}
spin_unlock(&encl_mm->encl->mm_lock);
if (tmp == encl_mm) {
synchronize_srcu(&encl_mm->encl->srcu);
mmu_notifier_put(mn);
}
}
static void sgx_mmu_notifier_free(struct mmu_notifier *mn)
{
struct sgx_encl_mm *encl_mm = container_of(mn, struct sgx_encl_mm, mmu_notifier);
/* 'encl_mm' is going away, put encl_mm->encl reference: */
kref_put(&encl_mm->encl->refcount, sgx_encl_release);
kfree(encl_mm);
}
static const struct mmu_notifier_ops sgx_mmu_notifier_ops = {
.release = sgx_mmu_notifier_release,
.free_notifier = sgx_mmu_notifier_free,
};
static struct sgx_encl_mm *sgx_encl_find_mm(struct sgx_encl *encl,
struct mm_struct *mm)
{
struct sgx_encl_mm *encl_mm = NULL;
struct sgx_encl_mm *tmp;
int idx;
idx = srcu_read_lock(&encl->srcu);
list_for_each_entry_rcu(tmp, &encl->mm_list, list) {
if (tmp->mm == mm) {
encl_mm = tmp;
break;
}
}
srcu_read_unlock(&encl->srcu, idx);
return encl_mm;
}
int sgx_encl_mm_add(struct sgx_encl *encl, struct mm_struct *mm)
{
struct sgx_encl_mm *encl_mm;
int ret;
/*
* Even though a single enclave may be mapped into an mm more than once,
* each 'mm' only appears once on encl->mm_list. This is guaranteed by
* holding the mm's mmap lock for write before an mm can be added or
* remove to an encl->mm_list.
*/
mmap_assert_write_locked(mm);
/*
* It's possible that an entry already exists in the mm_list, because it
* is removed only on VFS release or process exit.
*/
if (sgx_encl_find_mm(encl, mm))
return 0;
encl_mm = kzalloc(sizeof(*encl_mm), GFP_KERNEL);
if (!encl_mm)
return -ENOMEM;
/* Grab a refcount for the encl_mm->encl reference: */
kref_get(&encl->refcount);
encl_mm->encl = encl;
encl_mm->mm = mm;
encl_mm->mmu_notifier.ops = &sgx_mmu_notifier_ops;
ret = __mmu_notifier_register(&encl_mm->mmu_notifier, mm);
if (ret) {
kfree(encl_mm);
return ret;
}
spin_lock(&encl->mm_lock);
list_add_rcu(&encl_mm->list, &encl->mm_list);
/* Pairs with smp_rmb() in sgx_reclaimer_block(). */
smp_wmb();
encl->mm_list_version++;
spin_unlock(&encl->mm_lock);
return 0;
}
static struct page *sgx_encl_get_backing_page(struct sgx_encl *encl,
pgoff_t index)
{
struct inode *inode = encl->backing->f_path.dentry->d_inode;
struct address_space *mapping = inode->i_mapping;
gfp_t gfpmask = mapping_gfp_mask(mapping);
return shmem_read_mapping_page_gfp(mapping, index, gfpmask);
}
/**
* sgx_encl_get_backing() - Pin the backing storage
* @encl: an enclave pointer
* @page_index: enclave page index
* @backing: data for accessing backing storage for the page
*
* Pin the backing storage pages for storing the encrypted contents and Paging
* Crypto MetaData (PCMD) of an enclave page.
*
* Return:
* 0 on success,
* -errno otherwise.
*/
int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index,
struct sgx_backing *backing)
{
pgoff_t pcmd_index = PFN_DOWN(encl->size) + 1 + (page_index >> 5);
struct page *contents;
struct page *pcmd;
contents = sgx_encl_get_backing_page(encl, page_index);
if (IS_ERR(contents))
return PTR_ERR(contents);
pcmd = sgx_encl_get_backing_page(encl, pcmd_index);
if (IS_ERR(pcmd)) {
put_page(contents);
return PTR_ERR(pcmd);
}
backing->page_index = page_index;
backing->contents = contents;
backing->pcmd = pcmd;
backing->pcmd_offset =
(page_index & (PAGE_SIZE / sizeof(struct sgx_pcmd) - 1)) *
sizeof(struct sgx_pcmd);
return 0;
}
/**
* sgx_encl_put_backing() - Unpin the backing storage
* @backing: data for accessing backing storage for the page
* @do_write: mark pages dirty
*/
void sgx_encl_put_backing(struct sgx_backing *backing, bool do_write)
{
if (do_write) {
set_page_dirty(backing->pcmd);
set_page_dirty(backing->contents);
}
put_page(backing->pcmd);
put_page(backing->contents);
}
static int sgx_encl_test_and_clear_young_cb(pte_t *ptep, unsigned long addr,
void *data)
{
pte_t pte;
int ret;
ret = pte_young(*ptep);
if (ret) {
pte = pte_mkold(*ptep);
set_pte_at((struct mm_struct *)data, addr, ptep, pte);
}
return ret;
}
/**
* sgx_encl_test_and_clear_young() - Test and reset the accessed bit
* @mm: mm_struct that is checked
* @page: enclave page to be tested for recent access
*
* Checks the Access (A) bit from the PTE corresponding to the enclave page and
* clears it.
*
* Return: 1 if the page has been recently accessed and 0 if not.
*/
int sgx_encl_test_and_clear_young(struct mm_struct *mm,
struct sgx_encl_page *page)
{
unsigned long addr = page->desc & PAGE_MASK;
struct sgx_encl *encl = page->encl;
struct vm_area_struct *vma;
int ret;
ret = sgx_encl_find(mm, addr, &vma);
if (ret)
return 0;
if (encl != vma->vm_private_data)
return 0;
ret = apply_to_page_range(vma->vm_mm, addr, PAGE_SIZE,
sgx_encl_test_and_clear_young_cb, vma->vm_mm);
if (ret < 0)
return 0;
return ret;
}
/**
* sgx_alloc_va_page() - Allocate a Version Array (VA) page
*
* Allocate a free EPC page and convert it to a Version Array (VA) page.
*
* Return:
* a VA page,
* -errno otherwise
*/
struct sgx_epc_page *sgx_alloc_va_page(void)
{
struct sgx_epc_page *epc_page;
int ret;
epc_page = sgx_alloc_epc_page(NULL, true);
if (IS_ERR(epc_page))
return ERR_CAST(epc_page);
ret = __epa(sgx_get_epc_virt_addr(epc_page));
if (ret) {
WARN_ONCE(1, "EPA returned %d (0x%x)", ret, ret);
sgx_free_epc_page(epc_page);
return ERR_PTR(-EFAULT);
}
return epc_page;
}
/**
* sgx_alloc_va_slot - allocate a VA slot
* @va_page: a &struct sgx_va_page instance
*
* Allocates a slot from a &struct sgx_va_page instance.
*
* Return: offset of the slot inside the VA page
*/
unsigned int sgx_alloc_va_slot(struct sgx_va_page *va_page)
{
int slot = find_first_zero_bit(va_page->slots, SGX_VA_SLOT_COUNT);
if (slot < SGX_VA_SLOT_COUNT)
set_bit(slot, va_page->slots);
return slot << 3;
}
/**
* sgx_free_va_slot - free a VA slot
* @va_page: a &struct sgx_va_page instance
* @offset: offset of the slot inside the VA page
*
* Frees a slot from a &struct sgx_va_page instance.
*/
void sgx_free_va_slot(struct sgx_va_page *va_page, unsigned int offset)
{
clear_bit(offset >> 3, va_page->slots);
}
/**
* sgx_va_page_full - is the VA page full?
* @va_page: a &struct sgx_va_page instance
*
* Return: true if all slots have been taken
*/
bool sgx_va_page_full(struct sgx_va_page *va_page)
{
int slot = find_first_zero_bit(va_page->slots, SGX_VA_SLOT_COUNT);
return slot == SGX_VA_SLOT_COUNT;
}