Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | /* |
| 3 | * __put_user functions. |
| 4 | * |
| 5 | * (C) Copyright 2005 Linus Torvalds |
Glauber Costa | 5cbbc3b | 2008-06-24 17:40:14 -0300 | [diff] [blame] | 6 | * (C) Copyright 2005 Andi Kleen |
| 7 | * (C) Copyright 2008 Glauber Costa |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | * |
| 9 | * These functions have a non-standard call interface |
| 10 | * to make them more efficient, especially as they |
| 11 | * return an error value in addition to the "real" |
| 12 | * return value. |
| 13 | */ |
Masahiro Yamada | 94ea9c0 | 2023-08-06 23:59:56 +0900 | [diff] [blame] | 14 | #include <linux/export.h> |
Jan Beulich | 00e065e | 2007-05-02 19:27:05 +0200 | [diff] [blame] | 15 | #include <linux/linkage.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | #include <asm/thread_info.h> |
Glauber Costa | 5cbbc3b | 2008-06-24 17:40:14 -0300 | [diff] [blame] | 17 | #include <asm/errno.h> |
Glauber Costa | 2528de4 | 2008-06-24 17:36:31 -0300 | [diff] [blame] | 18 | #include <asm/asm.h> |
H. Peter Anvin | 63bcff2 | 2012-09-21 12:43:12 -0700 | [diff] [blame] | 19 | #include <asm/smap.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 20 | |
| 21 | /* |
| 22 | * __put_user_X |
| 23 | * |
| 24 | * Inputs: %eax[:%edx] contains the data |
| 25 | * %ecx contains the address |
| 26 | * |
Linus Torvalds | d55564c | 2020-04-08 13:36:49 -0700 | [diff] [blame] | 27 | * Outputs: %ecx is error code (0 or -EFAULT) |
| 28 | * |
| 29 | * Clobbers: %ebx needed for task pointer |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 30 | * |
| 31 | * These functions should not modify any other registers, |
| 32 | * as they get called from within inline assembly. |
| 33 | */ |
| 34 | |
Kirill A. Shutemov | b19b74b | 2023-03-12 14:25:56 +0300 | [diff] [blame] | 35 | .macro check_range size:req |
| 36 | .if IS_ENABLED(CONFIG_X86_64) |
| 37 | mov %rcx, %rbx |
| 38 | sar $63, %rbx |
| 39 | or %rbx, %rcx |
| 40 | .else |
| 41 | cmp $TASK_SIZE_MAX-\size+1, %ecx |
| 42 | jae .Lbad_put_user |
| 43 | .endif |
| 44 | .endm |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 45 | |
| 46 | .text |
Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 47 | SYM_FUNC_START(__put_user_1) |
Kirill A. Shutemov | b19b74b | 2023-03-12 14:25:56 +0300 | [diff] [blame] | 48 | check_range size=1 |
H. Peter Anvin | 63bcff2 | 2012-09-21 12:43:12 -0700 | [diff] [blame] | 49 | ASM_STAC |
Glauber Costa | 2528de4 | 2008-06-24 17:36:31 -0300 | [diff] [blame] | 50 | 1: movb %al,(%_ASM_CX) |
Linus Torvalds | d55564c | 2020-04-08 13:36:49 -0700 | [diff] [blame] | 51 | xor %ecx,%ecx |
Josh Poimboeuf | 82e844a | 2019-07-17 20:36:44 -0500 | [diff] [blame] | 52 | ASM_CLAC |
Peter Zijlstra | f94909c | 2021-12-04 14:43:40 +0100 | [diff] [blame] | 53 | RET |
Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 54 | SYM_FUNC_END(__put_user_1) |
Al Viro | 784d569 | 2016-01-11 11:04:34 -0500 | [diff] [blame] | 55 | EXPORT_SYMBOL(__put_user_1) |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 56 | |
| 57 | SYM_FUNC_START(__put_user_nocheck_1) |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 58 | ASM_STAC |
| 59 | 2: movb %al,(%_ASM_CX) |
| 60 | xor %ecx,%ecx |
| 61 | ASM_CLAC |
| 62 | RET |
| 63 | SYM_FUNC_END(__put_user_nocheck_1) |
Linus Torvalds | d55564c | 2020-04-08 13:36:49 -0700 | [diff] [blame] | 64 | EXPORT_SYMBOL(__put_user_nocheck_1) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | |
Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 66 | SYM_FUNC_START(__put_user_2) |
Kirill A. Shutemov | b19b74b | 2023-03-12 14:25:56 +0300 | [diff] [blame] | 67 | check_range size=2 |
H. Peter Anvin | 63bcff2 | 2012-09-21 12:43:12 -0700 | [diff] [blame] | 68 | ASM_STAC |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 69 | 3: movw %ax,(%_ASM_CX) |
Linus Torvalds | d55564c | 2020-04-08 13:36:49 -0700 | [diff] [blame] | 70 | xor %ecx,%ecx |
Josh Poimboeuf | 82e844a | 2019-07-17 20:36:44 -0500 | [diff] [blame] | 71 | ASM_CLAC |
Peter Zijlstra | f94909c | 2021-12-04 14:43:40 +0100 | [diff] [blame] | 72 | RET |
Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 73 | SYM_FUNC_END(__put_user_2) |
Al Viro | 784d569 | 2016-01-11 11:04:34 -0500 | [diff] [blame] | 74 | EXPORT_SYMBOL(__put_user_2) |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 75 | |
| 76 | SYM_FUNC_START(__put_user_nocheck_2) |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 77 | ASM_STAC |
| 78 | 4: movw %ax,(%_ASM_CX) |
| 79 | xor %ecx,%ecx |
| 80 | ASM_CLAC |
| 81 | RET |
| 82 | SYM_FUNC_END(__put_user_nocheck_2) |
Linus Torvalds | d55564c | 2020-04-08 13:36:49 -0700 | [diff] [blame] | 83 | EXPORT_SYMBOL(__put_user_nocheck_2) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 84 | |
Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 85 | SYM_FUNC_START(__put_user_4) |
Kirill A. Shutemov | b19b74b | 2023-03-12 14:25:56 +0300 | [diff] [blame] | 86 | check_range size=4 |
H. Peter Anvin | 63bcff2 | 2012-09-21 12:43:12 -0700 | [diff] [blame] | 87 | ASM_STAC |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 88 | 5: movl %eax,(%_ASM_CX) |
Linus Torvalds | d55564c | 2020-04-08 13:36:49 -0700 | [diff] [blame] | 89 | xor %ecx,%ecx |
Josh Poimboeuf | 82e844a | 2019-07-17 20:36:44 -0500 | [diff] [blame] | 90 | ASM_CLAC |
Peter Zijlstra | f94909c | 2021-12-04 14:43:40 +0100 | [diff] [blame] | 91 | RET |
Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 92 | SYM_FUNC_END(__put_user_4) |
Al Viro | 784d569 | 2016-01-11 11:04:34 -0500 | [diff] [blame] | 93 | EXPORT_SYMBOL(__put_user_4) |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 94 | |
| 95 | SYM_FUNC_START(__put_user_nocheck_4) |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 96 | ASM_STAC |
| 97 | 6: movl %eax,(%_ASM_CX) |
| 98 | xor %ecx,%ecx |
| 99 | ASM_CLAC |
| 100 | RET |
| 101 | SYM_FUNC_END(__put_user_nocheck_4) |
Linus Torvalds | d55564c | 2020-04-08 13:36:49 -0700 | [diff] [blame] | 102 | EXPORT_SYMBOL(__put_user_nocheck_4) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 103 | |
Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 104 | SYM_FUNC_START(__put_user_8) |
Kirill A. Shutemov | b19b74b | 2023-03-12 14:25:56 +0300 | [diff] [blame] | 105 | check_range size=8 |
H. Peter Anvin | 63bcff2 | 2012-09-21 12:43:12 -0700 | [diff] [blame] | 106 | ASM_STAC |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 107 | 7: mov %_ASM_AX,(%_ASM_CX) |
Glauber Costa | 5cbbc3b | 2008-06-24 17:40:14 -0300 | [diff] [blame] | 108 | #ifdef CONFIG_X86_32 |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 109 | 8: movl %edx,4(%_ASM_CX) |
Glauber Costa | 5cbbc3b | 2008-06-24 17:40:14 -0300 | [diff] [blame] | 110 | #endif |
Linus Torvalds | d55564c | 2020-04-08 13:36:49 -0700 | [diff] [blame] | 111 | xor %ecx,%ecx |
Josh Poimboeuf | 82e844a | 2019-07-17 20:36:44 -0500 | [diff] [blame] | 112 | ASM_CLAC |
| 113 | RET |
Jiri Slaby | 6dcc562 | 2019-10-11 13:51:04 +0200 | [diff] [blame] | 114 | SYM_FUNC_END(__put_user_8) |
Al Viro | 784d569 | 2016-01-11 11:04:34 -0500 | [diff] [blame] | 115 | EXPORT_SYMBOL(__put_user_8) |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 116 | |
| 117 | SYM_FUNC_START(__put_user_nocheck_8) |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 118 | ASM_STAC |
| 119 | 9: mov %_ASM_AX,(%_ASM_CX) |
| 120 | #ifdef CONFIG_X86_32 |
| 121 | 10: movl %edx,4(%_ASM_CX) |
| 122 | #endif |
| 123 | xor %ecx,%ecx |
| 124 | ASM_CLAC |
| 125 | RET |
| 126 | SYM_FUNC_END(__put_user_nocheck_8) |
Linus Torvalds | d55564c | 2020-04-08 13:36:49 -0700 | [diff] [blame] | 127 | EXPORT_SYMBOL(__put_user_nocheck_8) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 128 | |
Nadav Amit | 5516c89 | 2023-05-25 11:42:44 -0700 | [diff] [blame] | 129 | SYM_CODE_START_LOCAL(__put_user_handle_exception) |
Josh Poimboeuf | 82e844a | 2019-07-17 20:36:44 -0500 | [diff] [blame] | 130 | ASM_CLAC |
Jiri Slaby | 98ededb | 2019-09-06 09:55:50 +0200 | [diff] [blame] | 131 | .Lbad_put_user: |
Linus Torvalds | d55564c | 2020-04-08 13:36:49 -0700 | [diff] [blame] | 132 | movl $-EFAULT,%ecx |
Josh Poimboeuf | 82e844a | 2019-07-17 20:36:44 -0500 | [diff] [blame] | 133 | RET |
Nadav Amit | 5516c89 | 2023-05-25 11:42:44 -0700 | [diff] [blame] | 134 | SYM_CODE_END(__put_user_handle_exception) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 135 | |
Qiuxu Zhuo | 8eed4e0 | 2024-01-29 14:38:42 +0800 | [diff] [blame] | 136 | _ASM_EXTABLE_UA(1b, __put_user_handle_exception) |
| 137 | _ASM_EXTABLE_UA(2b, __put_user_handle_exception) |
| 138 | _ASM_EXTABLE_UA(3b, __put_user_handle_exception) |
| 139 | _ASM_EXTABLE_UA(4b, __put_user_handle_exception) |
| 140 | _ASM_EXTABLE_UA(5b, __put_user_handle_exception) |
| 141 | _ASM_EXTABLE_UA(6b, __put_user_handle_exception) |
| 142 | _ASM_EXTABLE_UA(7b, __put_user_handle_exception) |
| 143 | _ASM_EXTABLE_UA(9b, __put_user_handle_exception) |
Thomas Gleixner | cb85597 | 2022-09-15 13:11:16 +0200 | [diff] [blame] | 144 | #ifdef CONFIG_X86_32 |
Qiuxu Zhuo | 8eed4e0 | 2024-01-29 14:38:42 +0800 | [diff] [blame] | 145 | _ASM_EXTABLE_UA(8b, __put_user_handle_exception) |
| 146 | _ASM_EXTABLE_UA(10b, __put_user_handle_exception) |
Glauber Costa | 5cbbc3b | 2008-06-24 17:40:14 -0300 | [diff] [blame] | 147 | #endif |