| /* clear_page.S: UltraSparc optimized copy page. |
| * |
| * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com) |
| * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com) |
| */ |
| |
| #include <asm/visasm.h> |
| #include <asm/thread_info.h> |
| #include <asm/page.h> |
| #include <asm/pgtable.h> |
| #include <asm/spitfire.h> |
| #include <asm/head.h> |
| |
| /* What we used to do was lock a TLB entry into a specific |
| * TLB slot, clear the page with interrupts disabled, then |
| * restore the original TLB entry. This was great for |
| * disturbing the TLB as little as possible, but it meant |
| * we had to keep interrupts disabled for a long time. |
| * |
| * Now, we simply use the normal TLB loading mechanism, |
| * and this makes the cpu choose a slot all by itself. |
| * Then we do a normal TLB flush on exit. We need only |
| * disable preemption during the clear. |
| */ |
| |
| #define TTE_BITS_TOP (_PAGE_VALID | _PAGE_SZBITS) |
| #define TTE_BITS_BOTTOM (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W) |
| #define DCACHE_SIZE (PAGE_SIZE * 2) |
| |
| #if (PAGE_SHIFT == 13) || (PAGE_SHIFT == 19) |
| #define PAGE_SIZE_REM 0x80 |
| #elif (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22) |
| #define PAGE_SIZE_REM 0x100 |
| #else |
| #error Wrong PAGE_SHIFT specified |
| #endif |
| |
| #define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \ |
| fmovd %reg0, %f48; fmovd %reg1, %f50; \ |
| fmovd %reg2, %f52; fmovd %reg3, %f54; \ |
| fmovd %reg4, %f56; fmovd %reg5, %f58; \ |
| fmovd %reg6, %f60; fmovd %reg7, %f62; |
| |
| .text |
| |
| .align 32 |
| .globl copy_user_page |
| .type copy_user_page,#function |
| copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ |
| lduw [%g6 + TI_PRE_COUNT], %o4 |
| sethi %uhi(PAGE_OFFSET), %g2 |
| sethi %hi(PAGE_SIZE), %o3 |
| |
| sllx %g2, 32, %g2 |
| sethi %uhi(TTE_BITS_TOP), %g3 |
| |
| sllx %g3, 32, %g3 |
| sub %o0, %g2, %g1 ! dest paddr |
| |
| sub %o1, %g2, %g2 ! src paddr |
| or %g3, TTE_BITS_BOTTOM, %g3 |
| |
| and %o2, %o3, %o0 ! vaddr D-cache alias bit |
| or %g1, %g3, %g1 ! dest TTE data |
| |
| or %g2, %g3, %g2 ! src TTE data |
| sethi %hi(TLBTEMP_BASE), %o3 |
| |
| sethi %hi(DCACHE_SIZE), %o1 |
| add %o0, %o3, %o0 ! dest TTE vaddr |
| |
| add %o4, 1, %o2 |
| add %o0, %o1, %o1 ! src TTE vaddr |
| |
| /* Disable preemption. */ |
| mov TLB_TAG_ACCESS, %g3 |
| stw %o2, [%g6 + TI_PRE_COUNT] |
| |
| /* Load TLB entries. */ |
| rdpr %pstate, %o2 |
| wrpr %o2, PSTATE_IE, %pstate |
| stxa %o0, [%g3] ASI_DMMU |
| stxa %g1, [%g0] ASI_DTLB_DATA_IN |
| membar #Sync |
| stxa %o1, [%g3] ASI_DMMU |
| stxa %g2, [%g0] ASI_DTLB_DATA_IN |
| membar #Sync |
| wrpr %o2, 0x0, %pstate |
| |
| BRANCH_IF_ANY_CHEETAH(g3,o2,1f) |
| ba,pt %xcc, 9f |
| nop |
| |
| 1: |
| VISEntryHalf |
| membar #StoreLoad | #StoreStore | #LoadStore |
| sethi %hi((PAGE_SIZE/64)-2), %o2 |
| mov %o0, %g1 |
| prefetch [%o1 + 0x000], #one_read |
| or %o2, %lo((PAGE_SIZE/64)-2), %o2 |
| prefetch [%o1 + 0x040], #one_read |
| prefetch [%o1 + 0x080], #one_read |
| prefetch [%o1 + 0x0c0], #one_read |
| ldd [%o1 + 0x000], %f0 |
| prefetch [%o1 + 0x100], #one_read |
| ldd [%o1 + 0x008], %f2 |
| prefetch [%o1 + 0x140], #one_read |
| ldd [%o1 + 0x010], %f4 |
| prefetch [%o1 + 0x180], #one_read |
| fmovd %f0, %f16 |
| ldd [%o1 + 0x018], %f6 |
| fmovd %f2, %f18 |
| ldd [%o1 + 0x020], %f8 |
| fmovd %f4, %f20 |
| ldd [%o1 + 0x028], %f10 |
| fmovd %f6, %f22 |
| ldd [%o1 + 0x030], %f12 |
| fmovd %f8, %f24 |
| ldd [%o1 + 0x038], %f14 |
| fmovd %f10, %f26 |
| ldd [%o1 + 0x040], %f0 |
| 1: ldd [%o1 + 0x048], %f2 |
| fmovd %f12, %f28 |
| ldd [%o1 + 0x050], %f4 |
| fmovd %f14, %f30 |
| stda %f16, [%o0] ASI_BLK_P |
| ldd [%o1 + 0x058], %f6 |
| fmovd %f0, %f16 |
| ldd [%o1 + 0x060], %f8 |
| fmovd %f2, %f18 |
| ldd [%o1 + 0x068], %f10 |
| fmovd %f4, %f20 |
| ldd [%o1 + 0x070], %f12 |
| fmovd %f6, %f22 |
| ldd [%o1 + 0x078], %f14 |
| fmovd %f8, %f24 |
| ldd [%o1 + 0x080], %f0 |
| prefetch [%o1 + 0x180], #one_read |
| fmovd %f10, %f26 |
| subcc %o2, 1, %o2 |
| add %o0, 0x40, %o0 |
| bne,pt %xcc, 1b |
| add %o1, 0x40, %o1 |
| |
| ldd [%o1 + 0x048], %f2 |
| fmovd %f12, %f28 |
| ldd [%o1 + 0x050], %f4 |
| fmovd %f14, %f30 |
| stda %f16, [%o0] ASI_BLK_P |
| ldd [%o1 + 0x058], %f6 |
| fmovd %f0, %f16 |
| ldd [%o1 + 0x060], %f8 |
| fmovd %f2, %f18 |
| ldd [%o1 + 0x068], %f10 |
| fmovd %f4, %f20 |
| ldd [%o1 + 0x070], %f12 |
| fmovd %f6, %f22 |
| add %o0, 0x40, %o0 |
| ldd [%o1 + 0x078], %f14 |
| fmovd %f8, %f24 |
| fmovd %f10, %f26 |
| fmovd %f12, %f28 |
| fmovd %f14, %f30 |
| stda %f16, [%o0] ASI_BLK_P |
| membar #Sync |
| VISExitHalf |
| ba,pt %xcc, 5f |
| nop |
| |
| 9: |
| VISEntry |
| ldub [%g6 + TI_FAULT_CODE], %g3 |
| mov %o0, %g1 |
| cmp %g3, 0 |
| rd %asi, %g3 |
| be,a,pt %icc, 1f |
| wr %g0, ASI_BLK_P, %asi |
| wr %g0, ASI_BLK_COMMIT_P, %asi |
| 1: ldda [%o1] ASI_BLK_P, %f0 |
| add %o1, 0x40, %o1 |
| ldda [%o1] ASI_BLK_P, %f16 |
| add %o1, 0x40, %o1 |
| sethi %hi(PAGE_SIZE), %o2 |
| 1: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14) |
| ldda [%o1] ASI_BLK_P, %f32 |
| stda %f48, [%o0] %asi |
| add %o1, 0x40, %o1 |
| sub %o2, 0x40, %o2 |
| add %o0, 0x40, %o0 |
| TOUCH(f16, f18, f20, f22, f24, f26, f28, f30) |
| ldda [%o1] ASI_BLK_P, %f0 |
| stda %f48, [%o0] %asi |
| add %o1, 0x40, %o1 |
| sub %o2, 0x40, %o2 |
| add %o0, 0x40, %o0 |
| TOUCH(f32, f34, f36, f38, f40, f42, f44, f46) |
| ldda [%o1] ASI_BLK_P, %f16 |
| stda %f48, [%o0] %asi |
| sub %o2, 0x40, %o2 |
| add %o1, 0x40, %o1 |
| cmp %o2, PAGE_SIZE_REM |
| bne,pt %xcc, 1b |
| add %o0, 0x40, %o0 |
| #if (PAGE_SHIFT == 16) || (PAGE_SHIFT == 22) |
| TOUCH(f0, f2, f4, f6, f8, f10, f12, f14) |
| ldda [%o1] ASI_BLK_P, %f32 |
| stda %f48, [%o0] %asi |
| add %o1, 0x40, %o1 |
| sub %o2, 0x40, %o2 |
| add %o0, 0x40, %o0 |
| TOUCH(f16, f18, f20, f22, f24, f26, f28, f30) |
| ldda [%o1] ASI_BLK_P, %f0 |
| stda %f48, [%o0] %asi |
| add %o1, 0x40, %o1 |
| sub %o2, 0x40, %o2 |
| add %o0, 0x40, %o0 |
| membar #Sync |
| stda %f32, [%o0] %asi |
| add %o0, 0x40, %o0 |
| stda %f0, [%o0] %asi |
| #else |
| membar #Sync |
| stda %f0, [%o0] %asi |
| add %o0, 0x40, %o0 |
| stda %f16, [%o0] %asi |
| #endif |
| membar #Sync |
| wr %g3, 0x0, %asi |
| VISExit |
| |
| 5: |
| stxa %g0, [%g1] ASI_DMMU_DEMAP |
| membar #Sync |
| |
| sethi %hi(DCACHE_SIZE), %g2 |
| stxa %g0, [%g1 + %g2] ASI_DMMU_DEMAP |
| membar #Sync |
| |
| retl |
| stw %o4, [%g6 + TI_PRE_COUNT] |
| |
| .size copy_user_page, .-copy_user_page |
