arch/nds32/mm/proc.c

// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2005-2017 Andes Technology Corporation

#include <linux/module.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <asm/nds32.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include <asm/cacheflush.h>
#include <asm/l2_cache.h>
#include <nds32_intrinsic.h>

#include <asm/cache_info.h>
extern struct cache_info L1_cache_info[2];

int va_kernel_present(unsigned long addr)
{
	p4d_t *p4d;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *ptep, pte;

	p4d = p4d_offset(pgd_offset_k(addr), addr);
	pud = pud_offset(p4d, addr);
	pmd = pmd_offset(pud, addr);
	if (!pmd_none(*pmd)) {
		ptep = pte_offset_map(pmd, addr);
		pte = *ptep;
		if (pte_present(pte))
			return pte;
	}
	return 0;
}

pte_t va_present(struct mm_struct * mm, unsigned long addr)
{
	pgd_t *pgd;
	p4d_t *p4d;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *ptep, pte;

	pgd = pgd_offset(mm, addr);
	if (!pgd_none(*pgd)) {
		p4d = p4d_offset(pgd, addr);
		if (!p4d_none(*p4d)) {
			pud = pud_offset(p4d, addr);
			if (!pud_none(*pud)) {
				pmd = pmd_offset(pud, addr);
				if (!pmd_none(*pmd)) {
					ptep = pte_offset_map(pmd, addr);
					pte = *ptep;
					if (pte_present(pte))
						return pte;
				}
			}
		}
	}
	return 0;

}

int va_readable(struct pt_regs *regs, unsigned long addr)
{
	struct mm_struct *mm = current->mm;
	pte_t pte;
	int ret = 0;

	if (user_mode(regs)) {
		/* user mode */
		pte = va_present(mm, addr);
		if (!pte && pte_read(pte))
			ret = 1;
	} else {
		/* superuser mode is always readable, so we can only
		 * check it is present or not*/
		return (! !va_kernel_present(addr));
	}
	return ret;
}

int va_writable(struct pt_regs *regs, unsigned long addr)
{
	struct mm_struct *mm = current->mm;
	pte_t pte;
	int ret = 0;

	if (user_mode(regs)) {
		/* user mode */
		pte = va_present(mm, addr);
		if (!pte && pte_write(pte))
			ret = 1;
	} else {
		/* superuser mode */
		pte = va_kernel_present(addr);
		if (!pte && pte_kernel_write(pte))
			ret = 1;
	}
	return ret;
}

/*
 * All
 */
void cpu_icache_inval_all(void)
{
	unsigned long end, line_size;

	line_size = L1_cache_info[ICACHE].line_size;
	end =
	    line_size * L1_cache_info[ICACHE].ways * L1_cache_info[ICACHE].sets;

	do {
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));
	} while (end > 0);
	__nds32__isb();
}

void cpu_dcache_inval_all(void)
{
	__nds32__cctl_l1d_invalall();
}

#ifdef CONFIG_CACHE_L2
void dcache_wb_all_level(void)
{
	unsigned long flags, cmd;
	local_irq_save(flags);
	__nds32__cctl_l1d_wball_alvl();
	/* Section 1: Ensure the section 2 & 3 program code execution after */
	__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);

	/* Section 2: Confirm the writeback all level is done in CPU and L2C */
	cmd = CCTL_CMD_L2_SYNC;
	L2_CMD_RDY();
	L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
	L2_CMD_RDY();

	/* Section 3: Writeback whole L2 cache */
	cmd = CCTL_ALL_CMD | CCTL_CMD_L2_IX_WB;
	L2_CMD_RDY();
	L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
	L2_CMD_RDY();
	__nds32__msync_all();
	local_irq_restore(flags);
}
EXPORT_SYMBOL(dcache_wb_all_level);
#endif

void cpu_dcache_wb_all(void)
{
	__nds32__cctl_l1d_wball_one_lvl();
	__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
}

void cpu_dcache_wbinval_all(void)
{
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
	unsigned long flags;
	local_irq_save(flags);
#endif
	cpu_dcache_wb_all();
	cpu_dcache_inval_all();
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
	local_irq_restore(flags);
#endif
}

/*
 * Page
 */
void cpu_icache_inval_page(unsigned long start)
{
	unsigned long line_size, end;

	line_size = L1_cache_info[ICACHE].line_size;
	end = start + PAGE_SIZE;

	do {
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));
	} while (end != start);
	__nds32__isb();
}

void cpu_dcache_inval_page(unsigned long start)
{
	unsigned long line_size, end;

	line_size = L1_cache_info[DCACHE].line_size;
	end = start + PAGE_SIZE;

	do {
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
	} while (end != start);
}

void cpu_dcache_wb_page(unsigned long start)
{
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
	unsigned long line_size, end;

	line_size = L1_cache_info[DCACHE].line_size;
	end = start + PAGE_SIZE;

	do {
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
		end -= line_size;
		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
	} while (end != start);
	__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
#endif
}

void cpu_dcache_wbinval_page(unsigned long start)
{
	unsigned long line_size, end;

	line_size = L1_cache_info[DCACHE].line_size;
	end = start + PAGE_SIZE;

	do {
		end -= line_size;
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
#endif
		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
		end -= line_size;
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
#endif
		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
		end -= line_size;
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
#endif
		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
		end -= line_size;
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));
#endif
		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));
	} while (end != start);
	__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
}

void cpu_cache_wbinval_page(unsigned long page, int flushi)
{
	cpu_dcache_wbinval_page(page);
	if (flushi)
		cpu_icache_inval_page(page);
}

/*
 * Range
 */
void cpu_icache_inval_range(unsigned long start, unsigned long end)
{
	unsigned long line_size;

	line_size = L1_cache_info[ICACHE].line_size;

	while (end > start) {
		__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (start));
		start += line_size;
	}
	__nds32__isb();
}

void cpu_dcache_inval_range(unsigned long start, unsigned long end)
{
	unsigned long line_size;

	line_size = L1_cache_info[DCACHE].line_size;

	while (end > start) {
		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (start));
		start += line_size;
	}
}

void cpu_dcache_wb_range(unsigned long start, unsigned long end)
{
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
	unsigned long line_size;

	line_size = L1_cache_info[DCACHE].line_size;

	while (end > start) {
		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (start));
		start += line_size;
	}
	__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
#endif
}

void cpu_dcache_wbinval_range(unsigned long start, unsigned long end)
{
	unsigned long line_size;

	line_size = L1_cache_info[DCACHE].line_size;

	while (end > start) {
#ifndef CONFIG_CPU_DCACHE_WRITETHROUGH
		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (start));
#endif
		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (start));
		start += line_size;
	}
	__nds32__cctlidx_read(NDS32_CCTL_L1D_IX_RWD,0);
}

void cpu_cache_wbinval_range(unsigned long start, unsigned long end, int flushi)
{
	unsigned long line_size, align_start, align_end;

	line_size = L1_cache_info[DCACHE].line_size;
	align_start = start & ~(line_size - 1);
	align_end = (end + line_size - 1) & ~(line_size - 1);
	cpu_dcache_wbinval_range(align_start, align_end);

	if (flushi) {
		line_size = L1_cache_info[ICACHE].line_size;
		align_start = start & ~(line_size - 1);
		align_end = (end + line_size - 1) & ~(line_size - 1);
		cpu_icache_inval_range(align_start, align_end);
	}
}

void cpu_cache_wbinval_range_check(struct vm_area_struct *vma,
				   unsigned long start, unsigned long end,
				   bool flushi, bool wbd)
{
	unsigned long line_size, t_start, t_end;

	if (!flushi && !wbd)
		return;
	line_size = L1_cache_info[DCACHE].line_size;
	start = start & ~(line_size - 1);
	end = (end + line_size - 1) & ~(line_size - 1);

	if ((end - start) > (8 * PAGE_SIZE)) {
		if (wbd)
			cpu_dcache_wbinval_all();
		if (flushi)
			cpu_icache_inval_all();
		return;
	}

	t_start = (start + PAGE_SIZE) & PAGE_MASK;
	t_end = ((end - 1) & PAGE_MASK);

	if ((start & PAGE_MASK) == t_end) {
		if (va_present(vma->vm_mm, start)) {
			if (wbd)
				cpu_dcache_wbinval_range(start, end);
			if (flushi)
				cpu_icache_inval_range(start, end);
		}
		return;
	}

	if (va_present(vma->vm_mm, start)) {
		if (wbd)
			cpu_dcache_wbinval_range(start, t_start);
		if (flushi)
			cpu_icache_inval_range(start, t_start);
	}

	if (va_present(vma->vm_mm, end - 1)) {
		if (wbd)
			cpu_dcache_wbinval_range(t_end, end);
		if (flushi)
			cpu_icache_inval_range(t_end, end);
	}

	while (t_start < t_end) {
		if (va_present(vma->vm_mm, t_start)) {
			if (wbd)
				cpu_dcache_wbinval_page(t_start);
			if (flushi)
				cpu_icache_inval_page(t_start);
		}
		t_start += PAGE_SIZE;
	}
}

#ifdef CONFIG_CACHE_L2
static inline void cpu_l2cache_op(unsigned long start, unsigned long end, unsigned long op)
{
	if (atl2c_base) {
		unsigned long p_start = __pa(start);
		unsigned long p_end = __pa(end);
		unsigned long cmd;
		unsigned long line_size;
		/* TODO Can Use PAGE Mode to optimize if range large than PAGE_SIZE */
		line_size = L2_CACHE_LINE_SIZE();
		p_start = p_start & (~(line_size - 1));
		p_end = (p_end + line_size - 1) & (~(line_size - 1));
		cmd =
		    (p_start & ~(line_size - 1)) | op |
		    CCTL_SINGLE_CMD;
		do {
			L2_CMD_RDY();
			L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
			cmd += line_size;
			p_start += line_size;
		} while (p_end > p_start);
		cmd = CCTL_CMD_L2_SYNC;
		L2_CMD_RDY();
		L2C_W_REG(L2_CCTL_CMD_OFF, cmd);
		L2_CMD_RDY();
	}
}
#else
#define cpu_l2cache_op(start,end,op) do { } while (0)
#endif
/*
 * DMA
 */
void cpu_dma_wb_range(unsigned long start, unsigned long end)
{
	unsigned long line_size;
	unsigned long flags;
	line_size = L1_cache_info[DCACHE].line_size;
	start = start & (~(line_size - 1));
	end = (end + line_size - 1) & (~(line_size - 1));
	if (unlikely(start == end))
		return;

	local_irq_save(flags);
	cpu_dcache_wb_range(start, end);
	cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_WB);
	__nds32__msync_all();
	local_irq_restore(flags);
}

void cpu_dma_inval_range(unsigned long start, unsigned long end)
{
	unsigned long line_size;
	unsigned long old_start = start;
	unsigned long old_end = end;
	unsigned long flags;
	line_size = L1_cache_info[DCACHE].line_size;
	start = start & (~(line_size - 1));
	end = (end + line_size - 1) & (~(line_size - 1));
	if (unlikely(start == end))
		return;
	local_irq_save(flags);
	if (start != old_start) {
		cpu_dcache_wbinval_range(start, start + line_size);
		cpu_l2cache_op(start, start + line_size, CCTL_CMD_L2_PA_WBINVAL);
	}
	if (end != old_end) {
		cpu_dcache_wbinval_range(end - line_size, end);
		cpu_l2cache_op(end - line_size, end, CCTL_CMD_L2_PA_WBINVAL);
	}
	cpu_dcache_inval_range(start, end);
	cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_INVAL);
	__nds32__msync_all();
	local_irq_restore(flags);

}

void cpu_dma_wbinval_range(unsigned long start, unsigned long end)
{
	unsigned long line_size;
	unsigned long flags;
	line_size = L1_cache_info[DCACHE].line_size;
	start = start & (~(line_size - 1));
	end = (end + line_size - 1) & (~(line_size - 1));
	if (unlikely(start == end))
		return;

	local_irq_save(flags);
	cpu_dcache_wbinval_range(start, end);
	cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_WBINVAL);
	__nds32__msync_all();
	local_irq_restore(flags);
}

void cpu_proc_init(void)
{
}

void cpu_proc_fin(void)
{
}

void cpu_do_idle(void)
{
	__nds32__standby_no_wake_grant();
}

void cpu_reset(unsigned long reset)
{
	u32 tmp;
	GIE_DISABLE();
	tmp = __nds32__mfsr(NDS32_SR_CACHE_CTL);
	tmp &= ~(CACHE_CTL_mskIC_EN | CACHE_CTL_mskDC_EN);
	__nds32__mtsr_isb(tmp, NDS32_SR_CACHE_CTL);
	cpu_dcache_wbinval_all();
	cpu_icache_inval_all();

	__asm__ __volatile__("jr.toff %0\n\t"::"r"(reset));
}

void cpu_switch_mm(struct mm_struct *mm)
{
	unsigned long cid;
	cid = __nds32__mfsr(NDS32_SR_TLB_MISC);
	cid = (cid & ~TLB_MISC_mskCID) | mm->context.id;
	__nds32__mtsr_dsb(cid, NDS32_SR_TLB_MISC);
	__nds32__mtsr_isb(__pa(mm->pgd), NDS32_SR_L1_PPTB);
}