| /* SPDX-License-Identifier: GPL-2.0 */ |
| |
| #include <linux/export.h> |
| #include <linux/linkage.h> |
| |
| SYM_FUNC_START(memmove) |
| /* |
| * void *memmove(void *dest_in, const void *src_in, size_t n) |
| * -mregparm=3 passes these in registers: |
| * dest_in: %eax |
| * src_in: %edx |
| * n: %ecx |
| * See also: arch/x86/entry/calling.h for description of the calling convention. |
| * |
| * n can remain in %ecx, but for `rep movsl`, we'll need dest in %edi and src |
| * in %esi. |
| */ |
| .set dest_in, %eax |
| .set dest, %edi |
| .set src_in, %edx |
| .set src, %esi |
| .set n, %ecx |
| .set tmp0, %edx |
| .set tmp0w, %dx |
| .set tmp1, %ebx |
| .set tmp1w, %bx |
| .set tmp2, %eax |
| .set tmp3b, %cl |
| |
| /* |
| * Save all callee-saved registers, because this function is going to clobber |
| * all of them: |
| */ |
| pushl %ebp |
| movl %esp, %ebp // set standard frame pointer |
| |
| pushl %ebx |
| pushl %edi |
| pushl %esi |
| pushl %eax // save 'dest_in' parameter [eax] as the return value |
| |
| movl src_in, src |
| movl dest_in, dest |
| |
| /* Handle more 16 bytes in loop */ |
| cmpl $0x10, n |
| jb .Lmove_16B |
| |
| /* Decide forward/backward copy mode */ |
| cmpl dest, src |
| jb .Lbackwards_header |
| |
| /* |
| * movs instruction have many startup latency |
| * so we handle small size by general register. |
| */ |
| cmpl $680, n |
| jb .Ltoo_small_forwards |
| /* movs instruction is only good for aligned case. */ |
| movl src, tmp0 |
| xorl dest, tmp0 |
| andl $0xff, tmp0 |
| jz .Lforward_movs |
| .Ltoo_small_forwards: |
| subl $0x10, n |
| |
| /* We gobble 16 bytes forward in each loop. */ |
| .Lmove_16B_forwards_loop: |
| subl $0x10, n |
| movl 0*4(src), tmp0 |
| movl 1*4(src), tmp1 |
| movl tmp0, 0*4(dest) |
| movl tmp1, 1*4(dest) |
| movl 2*4(src), tmp0 |
| movl 3*4(src), tmp1 |
| movl tmp0, 2*4(dest) |
| movl tmp1, 3*4(dest) |
| leal 0x10(src), src |
| leal 0x10(dest), dest |
| jae .Lmove_16B_forwards_loop |
| addl $0x10, n |
| jmp .Lmove_16B |
| |
| /* Handle data forward by movs. */ |
| .p2align 4 |
| .Lforward_movs: |
| movl -4(src, n), tmp0 |
| leal -4(dest, n), tmp1 |
| shrl $2, n |
| rep movsl |
| movl tmp0, (tmp1) |
| jmp .Ldone |
| |
| /* Handle data backward by movs. */ |
| .p2align 4 |
| .Lbackwards_movs: |
| movl (src), tmp0 |
| movl dest, tmp1 |
| leal -4(src, n), src |
| leal -4(dest, n), dest |
| shrl $2, n |
| std |
| rep movsl |
| movl tmp0,(tmp1) |
| cld |
| jmp .Ldone |
| |
| /* Start to prepare for backward copy. */ |
| .p2align 4 |
| .Lbackwards_header: |
| cmpl $680, n |
| jb .Ltoo_small_backwards |
| movl src, tmp0 |
| xorl dest, tmp0 |
| andl $0xff, tmp0 |
| jz .Lbackwards_movs |
| |
| /* Calculate copy position to tail. */ |
| .Ltoo_small_backwards: |
| addl n, src |
| addl n, dest |
| subl $0x10, n |
| |
| /* We gobble 16 bytes backward in each loop. */ |
| .Lmove_16B_backwards_loop: |
| subl $0x10, n |
| |
| movl -1*4(src), tmp0 |
| movl -2*4(src), tmp1 |
| movl tmp0, -1*4(dest) |
| movl tmp1, -2*4(dest) |
| movl -3*4(src), tmp0 |
| movl -4*4(src), tmp1 |
| movl tmp0, -3*4(dest) |
| movl tmp1, -4*4(dest) |
| leal -0x10(src), src |
| leal -0x10(dest), dest |
| jae .Lmove_16B_backwards_loop |
| /* Calculate copy position to head. */ |
| addl $0x10, n |
| subl n, src |
| subl n, dest |
| |
| /* Move data from 8 bytes to 15 bytes. */ |
| .p2align 4 |
| .Lmove_16B: |
| cmpl $8, n |
| jb .Lmove_8B |
| movl 0*4(src), tmp0 |
| movl 1*4(src), tmp1 |
| movl -2*4(src, n), tmp2 |
| movl -1*4(src, n), src |
| |
| movl tmp0, 0*4(dest) |
| movl tmp1, 1*4(dest) |
| movl tmp2, -2*4(dest, n) |
| movl src, -1*4(dest, n) |
| jmp .Ldone |
| |
| /* Move data from 4 bytes to 7 bytes. */ |
| .p2align 4 |
| .Lmove_8B: |
| cmpl $4, n |
| jb .Lmove_4B |
| movl 0*4(src), tmp0 |
| movl -1*4(src, n), tmp1 |
| movl tmp0, 0*4(dest) |
| movl tmp1, -1*4(dest, n) |
| jmp .Ldone |
| |
| /* Move data from 2 bytes to 3 bytes. */ |
| .p2align 4 |
| .Lmove_4B: |
| cmpl $2, n |
| jb .Lmove_1B |
| movw 0*2(src), tmp0w |
| movw -1*2(src, n), tmp1w |
| movw tmp0w, 0*2(dest) |
| movw tmp1w, -1*2(dest, n) |
| jmp .Ldone |
| |
| /* Move data for 1 byte. */ |
| .p2align 4 |
| .Lmove_1B: |
| cmpl $1, n |
| jb .Ldone |
| movb (src), tmp3b |
| movb tmp3b, (dest) |
| .p2align 4 |
| .Ldone: |
| popl dest_in // restore 'dest_in' [eax] as the return value |
| /* Restore all callee-saved registers: */ |
| popl %esi |
| popl %edi |
| popl %ebx |
| popl %ebp |
| |
| RET |
| SYM_FUNC_END(memmove) |
| EXPORT_SYMBOL(memmove) |
