| /* SPDX-License-Identifier: GPL-2.0-or-later */ |
| /* |
| * |
| * Copyright (C) IBM Corporation, 2012 |
| * |
| * Author: Anton Blanchard <anton@au.ibm.com> |
| */ |
| |
| #include <asm/ppc_asm.h> |
| #include <asm/linkage.h> |
| #include <asm/asm-offsets.h> |
| #include <asm/export.h> |
| |
| .section ".toc","aw" |
| PPC64_CACHES: |
| .tc ppc64_caches[TC],ppc64_caches |
| .section ".text" |
| |
| /** |
| * __arch_clear_user: - Zero a block of memory in user space, with less checking. |
| * @to: Destination address, in user space. |
| * @n: Number of bytes to zero. |
| * |
| * Zero a block of memory in user space. Caller must check |
| * the specified block with access_ok() before calling this function. |
| * |
| * Returns number of bytes that could not be cleared. |
| * On success, this will be zero. |
| */ |
| |
| .macro err1 |
| 100: |
| EX_TABLE(100b,.Ldo_err1) |
| .endm |
| |
| .macro err2 |
| 200: |
| EX_TABLE(200b,.Ldo_err2) |
| .endm |
| |
| .macro err3 |
| 300: |
| EX_TABLE(300b,.Ldo_err3) |
| .endm |
| |
| .Ldo_err1: |
| mr r3,r8 |
| |
| .Ldo_err2: |
| mtctr r4 |
| 1: |
| err3; stb r0,0(r3) |
| addi r3,r3,1 |
| addi r4,r4,-1 |
| bdnz 1b |
| |
| .Ldo_err3: |
| mr r3,r4 |
| blr |
| |
| _GLOBAL_TOC(__arch_clear_user) |
| cmpdi r4,32 |
| neg r6,r3 |
| li r0,0 |
| blt .Lshort_clear |
| mr r8,r3 |
| mtocrf 0x01,r6 |
| clrldi r6,r6,(64-3) |
| |
| /* Get the destination 8 byte aligned */ |
| bf cr7*4+3,1f |
| err1; stb r0,0(r3) |
| addi r3,r3,1 |
| |
| 1: bf cr7*4+2,2f |
| err1; sth r0,0(r3) |
| addi r3,r3,2 |
| |
| 2: bf cr7*4+1,3f |
| err1; stw r0,0(r3) |
| addi r3,r3,4 |
| |
| 3: sub r4,r4,r6 |
| |
| cmpdi r4,32 |
| cmpdi cr1,r4,512 |
| blt .Lshort_clear |
| bgt cr1,.Llong_clear |
| |
| .Lmedium_clear: |
| srdi r6,r4,5 |
| mtctr r6 |
| |
| /* Do 32 byte chunks */ |
| 4: |
| err2; std r0,0(r3) |
| err2; std r0,8(r3) |
| err2; std r0,16(r3) |
| err2; std r0,24(r3) |
| addi r3,r3,32 |
| addi r4,r4,-32 |
| bdnz 4b |
| |
| .Lshort_clear: |
| /* up to 31 bytes to go */ |
| cmpdi r4,16 |
| blt 6f |
| err2; std r0,0(r3) |
| err2; std r0,8(r3) |
| addi r3,r3,16 |
| addi r4,r4,-16 |
| |
| /* Up to 15 bytes to go */ |
| 6: mr r8,r3 |
| clrldi r4,r4,(64-4) |
| mtocrf 0x01,r4 |
| bf cr7*4+0,7f |
| err1; std r0,0(r3) |
| addi r3,r3,8 |
| |
| 7: bf cr7*4+1,8f |
| err1; stw r0,0(r3) |
| addi r3,r3,4 |
| |
| 8: bf cr7*4+2,9f |
| err1; sth r0,0(r3) |
| addi r3,r3,2 |
| |
| 9: bf cr7*4+3,10f |
| err1; stb r0,0(r3) |
| |
| 10: li r3,0 |
| blr |
| |
| .Llong_clear: |
| ld r5,PPC64_CACHES@toc(r2) |
| |
| bf cr7*4+0,11f |
| err2; std r0,0(r3) |
| addi r3,r3,8 |
| addi r4,r4,-8 |
| |
| /* Destination is 16 byte aligned, need to get it cache block aligned */ |
| 11: lwz r7,DCACHEL1LOGBLOCKSIZE(r5) |
| lwz r9,DCACHEL1BLOCKSIZE(r5) |
| |
| /* |
| * With worst case alignment the long clear loop takes a minimum |
| * of 1 byte less than 2 cachelines. |
| */ |
| sldi r10,r9,2 |
| cmpd r4,r10 |
| blt .Lmedium_clear |
| |
| neg r6,r3 |
| addi r10,r9,-1 |
| and. r5,r6,r10 |
| beq 13f |
| |
| srdi r6,r5,4 |
| mtctr r6 |
| mr r8,r3 |
| 12: |
| err1; std r0,0(r3) |
| err1; std r0,8(r3) |
| addi r3,r3,16 |
| bdnz 12b |
| |
| sub r4,r4,r5 |
| |
| 13: srd r6,r4,r7 |
| mtctr r6 |
| mr r8,r3 |
| 14: |
| err1; dcbz 0,r3 |
| add r3,r3,r9 |
| bdnz 14b |
| |
| and r4,r4,r10 |
| |
| cmpdi r4,32 |
| blt .Lshort_clear |
| b .Lmedium_clear |
| EXPORT_SYMBOL(__arch_clear_user) |
