| // SPDX-License-Identifier: GPL-2.0 |
| /* |
| * Support for Medifield PNW Camera Imaging ISP subsystem. |
| * |
| * Copyright (c) 2010 Intel Corporation. All Rights Reserved. |
| * |
| * Copyright (c) 2010 Silicon Hive www.siliconhive.com. |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License version |
| * 2 as published by the Free Software Foundation. |
| * |
| * This program is distributed in the hope that it will be useful, |
| * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| * GNU General Public License for more details. |
| * |
| * |
| */ |
| /* |
| * ISP MMU management wrap code |
| */ |
| #include <linux/kernel.h> |
| #include <linux/types.h> |
| #include <linux/gfp.h> |
| #include <linux/mm.h> /* for GFP_ATOMIC */ |
| #include <linux/slab.h> /* for kmalloc */ |
| #include <linux/list.h> |
| #include <linux/io.h> |
| #include <linux/module.h> |
| #include <linux/moduleparam.h> |
| #include <linux/string.h> |
| #include <linux/errno.h> |
| #include <linux/sizes.h> |
| |
| #ifdef CONFIG_X86 |
| #include <asm/set_memory.h> |
| #endif |
| |
| #include "atomisp_internal.h" |
| #include "mmu/isp_mmu.h" |
| |
| /* |
| * 64-bit x86 processor physical address layout: |
| * 0 - 0x7fffffff DDR RAM (2GB) |
| * 0x80000000 - 0xffffffff MMIO (2GB) |
| * 0x100000000 - 0x3fffffffffff DDR RAM (64TB) |
| * So if the system has more than 2GB DDR memory, the lower 2GB occupies the |
| * physical address 0 - 0x7fffffff and the rest will start from 0x100000000. |
| * We have to make sure memory is allocated from the lower 2GB for devices |
| * that are only 32-bit capable(e.g. the ISP MMU). |
| * |
| * For any confusion, contact bin.gao@intel.com. |
| */ |
| #define NR_PAGES_2GB (SZ_2G / PAGE_SIZE) |
| |
| static void free_mmu_map(struct isp_mmu *mmu, unsigned int start_isp_virt, |
| unsigned int end_isp_virt); |
| |
| static unsigned int atomisp_get_pte(phys_addr_t pt, unsigned int idx) |
| { |
| unsigned int *pt_virt = phys_to_virt(pt); |
| |
| return *(pt_virt + idx); |
| } |
| |
| static void atomisp_set_pte(phys_addr_t pt, |
| unsigned int idx, unsigned int pte) |
| { |
| unsigned int *pt_virt = phys_to_virt(pt); |
| *(pt_virt + idx) = pte; |
| } |
| |
| static void *isp_pt_phys_to_virt(phys_addr_t phys) |
| { |
| return phys_to_virt(phys); |
| } |
| |
| static phys_addr_t isp_pte_to_pgaddr(struct isp_mmu *mmu, |
| unsigned int pte) |
| { |
| return mmu->driver->pte_to_phys(mmu, pte); |
| } |
| |
| static unsigned int isp_pgaddr_to_pte_valid(struct isp_mmu *mmu, |
| phys_addr_t phys) |
| { |
| unsigned int pte = mmu->driver->phys_to_pte(mmu, phys); |
| |
| return (unsigned int)(pte | ISP_PTE_VALID_MASK(mmu)); |
| } |
| |
| /* |
| * allocate a uncacheable page table. |
| * return physical address. |
| */ |
| static phys_addr_t alloc_page_table(struct isp_mmu *mmu) |
| { |
| int i; |
| phys_addr_t page; |
| void *virt; |
| |
| virt = (void *)__get_free_page(GFP_KERNEL | GFP_DMA32); |
| |
| if (!virt) |
| return (phys_addr_t)NULL_PAGE; |
| |
| /* |
| * we need a uncacheable page table. |
| */ |
| #ifdef CONFIG_X86 |
| set_memory_uc((unsigned long)virt, 1); |
| #endif |
| |
| page = virt_to_phys(virt); |
| |
| for (i = 0; i < 1024; i++) { |
| /* NEED CHECK */ |
| atomisp_set_pte(page, i, mmu->driver->null_pte); |
| } |
| |
| return page; |
| } |
| |
| static void free_page_table(struct isp_mmu *mmu, phys_addr_t page) |
| { |
| void *virt; |
| |
| page &= ISP_PAGE_MASK; |
| /* |
| * reset the page to write back before free |
| */ |
| virt = phys_to_virt(page); |
| |
| #ifdef CONFIG_X86 |
| set_memory_wb((unsigned long)virt, 1); |
| #endif |
| |
| free_page((unsigned long)virt); |
| } |
| |
| static void mmu_remap_error(struct isp_mmu *mmu, |
| phys_addr_t l1_pt, unsigned int l1_idx, |
| phys_addr_t l2_pt, unsigned int l2_idx, |
| unsigned int isp_virt, phys_addr_t old_phys, |
| phys_addr_t new_phys) |
| { |
| dev_err(atomisp_dev, "address remap:\n\n" |
| "\tL1 PT: virt = %p, phys = 0x%llx, idx = %d\n" |
| "\tL2 PT: virt = %p, phys = 0x%llx, idx = %d\n" |
| "\told: isp_virt = 0x%x, phys = 0x%llx\n" |
| "\tnew: isp_virt = 0x%x, phys = 0x%llx\n", |
| isp_pt_phys_to_virt(l1_pt), |
| (u64)l1_pt, l1_idx, |
| isp_pt_phys_to_virt(l2_pt), |
| (u64)l2_pt, l2_idx, isp_virt, |
| (u64)old_phys, isp_virt, |
| (u64)new_phys); |
| } |
| |
| static void mmu_unmap_l2_pte_error(struct isp_mmu *mmu, |
| phys_addr_t l1_pt, unsigned int l1_idx, |
| phys_addr_t l2_pt, unsigned int l2_idx, |
| unsigned int isp_virt, unsigned int pte) |
| { |
| dev_err(atomisp_dev, "unmap invalid L2 pte:\n\n" |
| "\tL1 PT: virt = %p, phys = 0x%llx, idx = %d\n" |
| "\tL2 PT: virt = %p, phys = 0x%llx, idx = %d\n" |
| "\tisp_virt = 0x%x, pte(page phys) = 0x%x\n", |
| isp_pt_phys_to_virt(l1_pt), |
| (u64)l1_pt, l1_idx, |
| isp_pt_phys_to_virt(l2_pt), |
| (u64)l2_pt, l2_idx, isp_virt, |
| pte); |
| } |
| |
| static void mmu_unmap_l1_pte_error(struct isp_mmu *mmu, |
| phys_addr_t l1_pt, unsigned int l1_idx, |
| unsigned int isp_virt, unsigned int pte) |
| { |
| dev_err(atomisp_dev, "unmap invalid L1 pte (L2 PT):\n\n" |
| "\tL1 PT: virt = %p, phys = 0x%llx, idx = %d\n" |
| "\tisp_virt = 0x%x, l1_pte(L2 PT) = 0x%x\n", |
| isp_pt_phys_to_virt(l1_pt), |
| (u64)l1_pt, l1_idx, (unsigned int)isp_virt, |
| pte); |
| } |
| |
| static void mmu_unmap_l1_pt_error(struct isp_mmu *mmu, unsigned int pte) |
| { |
| dev_err(atomisp_dev, "unmap invalid L1PT:\n\n" |
| "L1PT = 0x%x\n", (unsigned int)pte); |
| } |
| |
| /* |
| * Update L2 page table according to isp virtual address and page physical |
| * address |
| */ |
| static int mmu_l2_map(struct isp_mmu *mmu, phys_addr_t l1_pt, |
| unsigned int l1_idx, phys_addr_t l2_pt, |
| unsigned int start, unsigned int end, phys_addr_t phys) |
| { |
| unsigned int ptr; |
| unsigned int idx; |
| unsigned int pte; |
| |
| l2_pt &= ISP_PAGE_MASK; |
| |
| start = start & ISP_PAGE_MASK; |
| end = ISP_PAGE_ALIGN(end); |
| phys &= ISP_PAGE_MASK; |
| |
| ptr = start; |
| do { |
| idx = ISP_PTR_TO_L2_IDX(ptr); |
| |
| pte = atomisp_get_pte(l2_pt, idx); |
| |
| if (ISP_PTE_VALID(mmu, pte)) { |
| mmu_remap_error(mmu, l1_pt, l1_idx, |
| l2_pt, idx, ptr, pte, phys); |
| |
| /* free all mapped pages */ |
| free_mmu_map(mmu, start, ptr); |
| |
| return -EINVAL; |
| } |
| |
| pte = isp_pgaddr_to_pte_valid(mmu, phys); |
| |
| atomisp_set_pte(l2_pt, idx, pte); |
| mmu->l2_pgt_refcount[l1_idx]++; |
| ptr += (1U << ISP_L2PT_OFFSET); |
| phys += (1U << ISP_L2PT_OFFSET); |
| } while (ptr < end && idx < ISP_L2PT_PTES - 1); |
| |
| return 0; |
| } |
| |
| /* |
| * Update L1 page table according to isp virtual address and page physical |
| * address |
| */ |
| static int mmu_l1_map(struct isp_mmu *mmu, phys_addr_t l1_pt, |
| unsigned int start, unsigned int end, |
| phys_addr_t phys) |
| { |
| phys_addr_t l2_pt; |
| unsigned int ptr, l1_aligned; |
| unsigned int idx; |
| unsigned int l2_pte; |
| int ret; |
| |
| l1_pt &= ISP_PAGE_MASK; |
| |
| start = start & ISP_PAGE_MASK; |
| end = ISP_PAGE_ALIGN(end); |
| phys &= ISP_PAGE_MASK; |
| |
| ptr = start; |
| do { |
| idx = ISP_PTR_TO_L1_IDX(ptr); |
| |
| l2_pte = atomisp_get_pte(l1_pt, idx); |
| |
| if (!ISP_PTE_VALID(mmu, l2_pte)) { |
| l2_pt = alloc_page_table(mmu); |
| if (l2_pt == NULL_PAGE) { |
| dev_err(atomisp_dev, |
| "alloc page table fail.\n"); |
| |
| /* free all mapped pages */ |
| free_mmu_map(mmu, start, ptr); |
| |
| return -ENOMEM; |
| } |
| |
| l2_pte = isp_pgaddr_to_pte_valid(mmu, l2_pt); |
| |
| atomisp_set_pte(l1_pt, idx, l2_pte); |
| mmu->l2_pgt_refcount[idx] = 0; |
| } |
| |
| l2_pt = isp_pte_to_pgaddr(mmu, l2_pte); |
| |
| l1_aligned = (ptr & ISP_PAGE_MASK) + (1U << ISP_L1PT_OFFSET); |
| |
| if (l1_aligned < end) { |
| ret = mmu_l2_map(mmu, l1_pt, idx, |
| l2_pt, ptr, l1_aligned, phys); |
| phys += (l1_aligned - ptr); |
| ptr = l1_aligned; |
| } else { |
| ret = mmu_l2_map(mmu, l1_pt, idx, |
| l2_pt, ptr, end, phys); |
| phys += (end - ptr); |
| ptr = end; |
| } |
| |
| if (ret) { |
| dev_err(atomisp_dev, "setup mapping in L2PT fail.\n"); |
| |
| /* free all mapped pages */ |
| free_mmu_map(mmu, start, ptr); |
| |
| return -EINVAL; |
| } |
| } while (ptr < end && idx < ISP_L1PT_PTES); |
| |
| return 0; |
| } |
| |
| /* |
| * Update page table according to isp virtual address and page physical |
| * address |
| */ |
| static int mmu_map(struct isp_mmu *mmu, unsigned int isp_virt, |
| phys_addr_t phys, unsigned int pgnr) |
| { |
| unsigned int start, end; |
| phys_addr_t l1_pt; |
| int ret; |
| |
| mutex_lock(&mmu->pt_mutex); |
| if (!ISP_PTE_VALID(mmu, mmu->l1_pte)) { |
| /* |
| * allocate 1 new page for L1 page table |
| */ |
| l1_pt = alloc_page_table(mmu); |
| if (l1_pt == NULL_PAGE) { |
| dev_err(atomisp_dev, "alloc page table fail.\n"); |
| mutex_unlock(&mmu->pt_mutex); |
| return -ENOMEM; |
| } |
| |
| /* |
| * setup L1 page table physical addr to MMU |
| */ |
| mmu->base_address = l1_pt; |
| mmu->l1_pte = isp_pgaddr_to_pte_valid(mmu, l1_pt); |
| memset(mmu->l2_pgt_refcount, 0, sizeof(int) * ISP_L1PT_PTES); |
| } |
| |
| l1_pt = isp_pte_to_pgaddr(mmu, mmu->l1_pte); |
| |
| start = (isp_virt) & ISP_PAGE_MASK; |
| end = start + (pgnr << ISP_PAGE_OFFSET); |
| phys &= ISP_PAGE_MASK; |
| |
| ret = mmu_l1_map(mmu, l1_pt, start, end, phys); |
| |
| if (ret) |
| dev_err(atomisp_dev, "setup mapping in L1PT fail.\n"); |
| |
| mutex_unlock(&mmu->pt_mutex); |
| return ret; |
| } |
| |
| /* |
| * Free L2 page table according to isp virtual address and page physical |
| * address |
| */ |
| static void mmu_l2_unmap(struct isp_mmu *mmu, phys_addr_t l1_pt, |
| unsigned int l1_idx, phys_addr_t l2_pt, |
| unsigned int start, unsigned int end) |
| { |
| unsigned int ptr; |
| unsigned int idx; |
| unsigned int pte; |
| |
| l2_pt &= ISP_PAGE_MASK; |
| |
| start = start & ISP_PAGE_MASK; |
| end = ISP_PAGE_ALIGN(end); |
| |
| ptr = start; |
| do { |
| idx = ISP_PTR_TO_L2_IDX(ptr); |
| |
| pte = atomisp_get_pte(l2_pt, idx); |
| |
| if (!ISP_PTE_VALID(mmu, pte)) |
| mmu_unmap_l2_pte_error(mmu, l1_pt, l1_idx, |
| l2_pt, idx, ptr, pte); |
| |
| atomisp_set_pte(l2_pt, idx, mmu->driver->null_pte); |
| mmu->l2_pgt_refcount[l1_idx]--; |
| ptr += (1U << ISP_L2PT_OFFSET); |
| } while (ptr < end && idx < ISP_L2PT_PTES - 1); |
| |
| if (mmu->l2_pgt_refcount[l1_idx] == 0) { |
| free_page_table(mmu, l2_pt); |
| atomisp_set_pte(l1_pt, l1_idx, mmu->driver->null_pte); |
| } |
| } |
| |
| /* |
| * Free L1 page table according to isp virtual address and page physical |
| * address |
| */ |
| static void mmu_l1_unmap(struct isp_mmu *mmu, phys_addr_t l1_pt, |
| unsigned int start, unsigned int end) |
| { |
| phys_addr_t l2_pt; |
| unsigned int ptr, l1_aligned; |
| unsigned int idx; |
| unsigned int l2_pte; |
| |
| l1_pt &= ISP_PAGE_MASK; |
| |
| start = start & ISP_PAGE_MASK; |
| end = ISP_PAGE_ALIGN(end); |
| |
| ptr = start; |
| do { |
| idx = ISP_PTR_TO_L1_IDX(ptr); |
| |
| l2_pte = atomisp_get_pte(l1_pt, idx); |
| |
| if (!ISP_PTE_VALID(mmu, l2_pte)) { |
| mmu_unmap_l1_pte_error(mmu, l1_pt, idx, ptr, l2_pte); |
| continue; |
| } |
| |
| l2_pt = isp_pte_to_pgaddr(mmu, l2_pte); |
| |
| l1_aligned = (ptr & ISP_PAGE_MASK) + (1U << ISP_L1PT_OFFSET); |
| |
| if (l1_aligned < end) { |
| mmu_l2_unmap(mmu, l1_pt, idx, l2_pt, ptr, l1_aligned); |
| ptr = l1_aligned; |
| } else { |
| mmu_l2_unmap(mmu, l1_pt, idx, l2_pt, ptr, end); |
| ptr = end; |
| } |
| /* |
| * use the same L2 page next time, so we don't |
| * need to invalidate and free this PT. |
| */ |
| /* atomisp_set_pte(l1_pt, idx, NULL_PTE); */ |
| } while (ptr < end && idx < ISP_L1PT_PTES); |
| } |
| |
| /* |
| * Free page table according to isp virtual address and page physical |
| * address |
| */ |
| static void mmu_unmap(struct isp_mmu *mmu, unsigned int isp_virt, |
| unsigned int pgnr) |
| { |
| unsigned int start, end; |
| phys_addr_t l1_pt; |
| |
| mutex_lock(&mmu->pt_mutex); |
| if (!ISP_PTE_VALID(mmu, mmu->l1_pte)) { |
| mmu_unmap_l1_pt_error(mmu, mmu->l1_pte); |
| mutex_unlock(&mmu->pt_mutex); |
| return; |
| } |
| |
| l1_pt = isp_pte_to_pgaddr(mmu, mmu->l1_pte); |
| |
| start = (isp_virt) & ISP_PAGE_MASK; |
| end = start + (pgnr << ISP_PAGE_OFFSET); |
| |
| mmu_l1_unmap(mmu, l1_pt, start, end); |
| mutex_unlock(&mmu->pt_mutex); |
| } |
| |
| /* |
| * Free page tables according to isp start virtual address and end virtual |
| * address. |
| */ |
| static void free_mmu_map(struct isp_mmu *mmu, unsigned int start_isp_virt, |
| unsigned int end_isp_virt) |
| { |
| unsigned int pgnr; |
| unsigned int start, end; |
| |
| start = (start_isp_virt) & ISP_PAGE_MASK; |
| end = (end_isp_virt) & ISP_PAGE_MASK; |
| pgnr = (end - start) >> ISP_PAGE_OFFSET; |
| mmu_unmap(mmu, start, pgnr); |
| } |
| |
| int isp_mmu_map(struct isp_mmu *mmu, unsigned int isp_virt, |
| phys_addr_t phys, unsigned int pgnr) |
| { |
| return mmu_map(mmu, isp_virt, phys, pgnr); |
| } |
| |
| void isp_mmu_unmap(struct isp_mmu *mmu, unsigned int isp_virt, |
| unsigned int pgnr) |
| { |
| mmu_unmap(mmu, isp_virt, pgnr); |
| } |
| |
| static void isp_mmu_flush_tlb_range_default(struct isp_mmu *mmu, |
| unsigned int start, |
| unsigned int size) |
| { |
| isp_mmu_flush_tlb(mmu); |
| } |
| |
| /*MMU init for internal structure*/ |
| int isp_mmu_init(struct isp_mmu *mmu, struct isp_mmu_client *driver) |
| { |
| if (!mmu) /* error */ |
| return -EINVAL; |
| if (!driver) /* error */ |
| return -EINVAL; |
| |
| if (!driver->name) |
| dev_warn(atomisp_dev, "NULL name for MMU driver...\n"); |
| |
| mmu->driver = driver; |
| |
| if (!driver->tlb_flush_all) { |
| dev_err(atomisp_dev, "tlb_flush_all operation not provided.\n"); |
| return -EINVAL; |
| } |
| |
| if (!driver->tlb_flush_range) |
| driver->tlb_flush_range = isp_mmu_flush_tlb_range_default; |
| |
| if (!driver->pte_valid_mask) { |
| dev_err(atomisp_dev, "PTE_MASK is missing from mmu driver\n"); |
| return -EINVAL; |
| } |
| |
| mmu->l1_pte = driver->null_pte; |
| |
| mutex_init(&mmu->pt_mutex); |
| |
| return 0; |
| } |
| |
| /*Free L1 and L2 page table*/ |
| void isp_mmu_exit(struct isp_mmu *mmu) |
| { |
| unsigned int idx; |
| unsigned int pte; |
| phys_addr_t l1_pt, l2_pt; |
| |
| if (!mmu) |
| return; |
| |
| if (!ISP_PTE_VALID(mmu, mmu->l1_pte)) { |
| dev_warn(atomisp_dev, "invalid L1PT: pte = 0x%x\n", |
| (unsigned int)mmu->l1_pte); |
| return; |
| } |
| |
| l1_pt = isp_pte_to_pgaddr(mmu, mmu->l1_pte); |
| |
| for (idx = 0; idx < ISP_L1PT_PTES; idx++) { |
| pte = atomisp_get_pte(l1_pt, idx); |
| |
| if (ISP_PTE_VALID(mmu, pte)) { |
| l2_pt = isp_pte_to_pgaddr(mmu, pte); |
| |
| free_page_table(mmu, l2_pt); |
| } |
| } |
| |
| free_page_table(mmu, l1_pt); |
| } |