| /* SPDX-License-Identifier: GPL-2.0-or-later */ | |
| /* | |
| * | |
| * Copyright (C) IBM Corporation, 2012 | |
| * | |
| * Author: Anton Blanchard <anton@au.ibm.com> | |
| */ | |
| #include <linux/export.h> | |
| #include <asm/ppc_asm.h> | |
| #include <asm/linkage.h> | |
| #include <asm/asm-offsets.h> | |
| /** | |
| * __arch_clear_user: - Zero a block of memory in user space, with less checking. | |
| * @to: Destination address, in user space. | |
| * @n: Number of bytes to zero. | |
| * | |
| * Zero a block of memory in user space. Caller must check | |
| * the specified block with access_ok() before calling this function. | |
| * | |
| * Returns number of bytes that could not be cleared. | |
| * On success, this will be zero. | |
| */ | |
| .macro err1 | |
| 100: | |
| EX_TABLE(100b,.Ldo_err1) | |
| .endm | |
| .macro err2 | |
| 200: | |
| EX_TABLE(200b,.Ldo_err2) | |
| .endm | |
| .macro err3 | |
| 300: | |
| EX_TABLE(300b,.Ldo_err3) | |
| .endm | |
| .Ldo_err1: | |
| mr r3,r8 | |
| .Ldo_err2: | |
| mtctr r4 | |
| 1: | |
| err3; stb r0,0(r3) | |
| addi r3,r3,1 | |
| addi r4,r4,-1 | |
| bdnz 1b | |
| .Ldo_err3: | |
| mr r3,r4 | |
| blr | |
| _GLOBAL_TOC(__arch_clear_user) | |
| cmpdi r4,32 | |
| neg r6,r3 | |
| li r0,0 | |
| blt .Lshort_clear | |
| mr r8,r3 | |
| mtocrf 0x01,r6 | |
| clrldi r6,r6,(64-3) | |
| /* Get the destination 8 byte aligned */ | |
| bf cr7*4+3,1f | |
| err1; stb r0,0(r3) | |
| addi r3,r3,1 | |
| 1: bf cr7*4+2,2f | |
| err1; sth r0,0(r3) | |
| addi r3,r3,2 | |
| 2: bf cr7*4+1,3f | |
| err1; stw r0,0(r3) | |
| addi r3,r3,4 | |
| 3: sub r4,r4,r6 | |
| cmpdi r4,32 | |
| cmpdi cr1,r4,512 | |
| blt .Lshort_clear | |
| bgt cr1,.Llong_clear | |
| .Lmedium_clear: | |
| srdi r6,r4,5 | |
| mtctr r6 | |
| /* Do 32 byte chunks */ | |
| 4: | |
| err2; std r0,0(r3) | |
| err2; std r0,8(r3) | |
| err2; std r0,16(r3) | |
| err2; std r0,24(r3) | |
| addi r3,r3,32 | |
| addi r4,r4,-32 | |
| bdnz 4b | |
| .Lshort_clear: | |
| /* up to 31 bytes to go */ | |
| cmpdi r4,16 | |
| blt 6f | |
| err2; std r0,0(r3) | |
| err2; std r0,8(r3) | |
| addi r3,r3,16 | |
| addi r4,r4,-16 | |
| /* Up to 15 bytes to go */ | |
| 6: mr r8,r3 | |
| clrldi r4,r4,(64-4) | |
| mtocrf 0x01,r4 | |
| bf cr7*4+0,7f | |
| err1; std r0,0(r3) | |
| addi r3,r3,8 | |
| 7: bf cr7*4+1,8f | |
| err1; stw r0,0(r3) | |
| addi r3,r3,4 | |
| 8: bf cr7*4+2,9f | |
| err1; sth r0,0(r3) | |
| addi r3,r3,2 | |
| 9: bf cr7*4+3,10f | |
| err1; stb r0,0(r3) | |
| 10: li r3,0 | |
| blr | |
| .Llong_clear: | |
| LOAD_REG_ADDR(r5, ppc64_caches) | |
| bf cr7*4+0,11f | |
| err2; std r0,0(r3) | |
| addi r3,r3,8 | |
| addi r4,r4,-8 | |
| /* Destination is 16 byte aligned, need to get it cache block aligned */ | |
| 11: lwz r7,DCACHEL1LOGBLOCKSIZE(r5) | |
| lwz r9,DCACHEL1BLOCKSIZE(r5) | |
| /* | |
| * With worst case alignment the long clear loop takes a minimum | |
| * of 1 byte less than 2 cachelines. | |
| */ | |
| sldi r10,r9,2 | |
| cmpd r4,r10 | |
| blt .Lmedium_clear | |
| neg r6,r3 | |
| addi r10,r9,-1 | |
| and. r5,r6,r10 | |
| beq 13f | |
| srdi r6,r5,4 | |
| mtctr r6 | |
| mr r8,r3 | |
| 12: | |
| err1; std r0,0(r3) | |
| err1; std r0,8(r3) | |
| addi r3,r3,16 | |
| bdnz 12b | |
| sub r4,r4,r5 | |
| 13: srd r6,r4,r7 | |
| mtctr r6 | |
| mr r8,r3 | |
| 14: | |
| err1; dcbz 0,r3 | |
| add r3,r3,r9 | |
| bdnz 14b | |
| and r4,r4,r10 | |
| cmpdi r4,32 | |
| blt .Lshort_clear | |
| b .Lmedium_clear | |
| EXPORT_SYMBOL(__arch_clear_user) |