Thomas Gleixner | 457c899 | 2019-05-19 13:08:55 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Ingo Molnar | c33fa9f | 2008-04-17 20:05:36 +0200 | [diff] [blame] | 2 | /* |
Christoph Hellwig | 3f0acb1 | 2020-06-08 21:34:11 -0700 | [diff] [blame] | 3 | * Access kernel or user memory without faulting. |
Ingo Molnar | c33fa9f | 2008-04-17 20:05:36 +0200 | [diff] [blame] | 4 | */ |
Paul Gortmaker | b95f1b31 | 2011-10-16 02:01:52 -0400 | [diff] [blame] | 5 | #include <linux/export.h> |
Ingo Molnar | c33fa9f | 2008-04-17 20:05:36 +0200 | [diff] [blame] | 6 | #include <linux/mm.h> |
David Howells | 7c7fcf7 | 2010-10-27 17:29:01 +0100 | [diff] [blame] | 7 | #include <linux/uaccess.h> |
Alexei Starovoitov | d319f34 | 2023-04-10 19:43:44 +0200 | [diff] [blame] | 8 | #include <asm/tlb.h> |
Ingo Molnar | c33fa9f | 2008-04-17 20:05:36 +0200 | [diff] [blame] | 9 | |
Christoph Hellwig | fe55731 | 2020-06-17 09:37:53 +0200 | [diff] [blame] | 10 | bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, |
| 11 | size_t size) |
Christoph Hellwig | eab0c60 | 2020-06-08 21:34:27 -0700 | [diff] [blame] | 12 | { |
| 13 | return true; |
| 14 | } |
| 15 | |
Christoph Hellwig | fe55731 | 2020-06-17 09:37:53 +0200 | [diff] [blame] | 16 | #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ |
Christoph Hellwig | b58294e | 2020-06-08 21:34:58 -0700 | [diff] [blame] | 17 | while (len >= sizeof(type)) { \ |
| 18 | __get_kernel_nofault(dst, src, type, err_label); \ |
| 19 | dst += sizeof(type); \ |
| 20 | src += sizeof(type); \ |
| 21 | len -= sizeof(type); \ |
| 22 | } |
| 23 | |
Christoph Hellwig | fe55731 | 2020-06-17 09:37:53 +0200 | [diff] [blame] | 24 | long copy_from_kernel_nofault(void *dst, const void *src, size_t size) |
Christoph Hellwig | b58294e | 2020-06-08 21:34:58 -0700 | [diff] [blame] | 25 | { |
Arnd Bergmann | 2423de2 | 2021-08-11 08:30:18 +0100 | [diff] [blame] | 26 | unsigned long align = 0; |
| 27 | |
| 28 | if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) |
| 29 | align = (unsigned long)dst | (unsigned long)src; |
| 30 | |
Christoph Hellwig | fe55731 | 2020-06-17 09:37:53 +0200 | [diff] [blame] | 31 | if (!copy_from_kernel_nofault_allowed(src, size)) |
Christoph Hellwig | 2a71e81 | 2020-06-08 21:35:04 -0700 | [diff] [blame] | 32 | return -ERANGE; |
Christoph Hellwig | b58294e | 2020-06-08 21:34:58 -0700 | [diff] [blame] | 33 | |
| 34 | pagefault_disable(); |
Arnd Bergmann | 2423de2 | 2021-08-11 08:30:18 +0100 | [diff] [blame] | 35 | if (!(align & 7)) |
| 36 | copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); |
| 37 | if (!(align & 3)) |
| 38 | copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); |
| 39 | if (!(align & 1)) |
| 40 | copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); |
Christoph Hellwig | fe55731 | 2020-06-17 09:37:53 +0200 | [diff] [blame] | 41 | copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); |
Christoph Hellwig | b58294e | 2020-06-08 21:34:58 -0700 | [diff] [blame] | 42 | pagefault_enable(); |
| 43 | return 0; |
| 44 | Efault: |
| 45 | pagefault_enable(); |
| 46 | return -EFAULT; |
| 47 | } |
Christoph Hellwig | fe55731 | 2020-06-17 09:37:53 +0200 | [diff] [blame] | 48 | EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); |
Christoph Hellwig | b58294e | 2020-06-08 21:34:58 -0700 | [diff] [blame] | 49 | |
Christoph Hellwig | fe55731 | 2020-06-17 09:37:53 +0200 | [diff] [blame] | 50 | #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ |
Christoph Hellwig | b58294e | 2020-06-08 21:34:58 -0700 | [diff] [blame] | 51 | while (len >= sizeof(type)) { \ |
| 52 | __put_kernel_nofault(dst, src, type, err_label); \ |
| 53 | dst += sizeof(type); \ |
| 54 | src += sizeof(type); \ |
| 55 | len -= sizeof(type); \ |
| 56 | } |
| 57 | |
Christoph Hellwig | fe55731 | 2020-06-17 09:37:53 +0200 | [diff] [blame] | 58 | long copy_to_kernel_nofault(void *dst, const void *src, size_t size) |
Christoph Hellwig | b58294e | 2020-06-08 21:34:58 -0700 | [diff] [blame] | 59 | { |
Arnd Bergmann | 2423de2 | 2021-08-11 08:30:18 +0100 | [diff] [blame] | 60 | unsigned long align = 0; |
| 61 | |
| 62 | if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) |
| 63 | align = (unsigned long)dst | (unsigned long)src; |
| 64 | |
Christoph Hellwig | b58294e | 2020-06-08 21:34:58 -0700 | [diff] [blame] | 65 | pagefault_disable(); |
Arnd Bergmann | 2423de2 | 2021-08-11 08:30:18 +0100 | [diff] [blame] | 66 | if (!(align & 7)) |
| 67 | copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); |
| 68 | if (!(align & 3)) |
| 69 | copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); |
| 70 | if (!(align & 1)) |
| 71 | copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); |
Christoph Hellwig | fe55731 | 2020-06-17 09:37:53 +0200 | [diff] [blame] | 72 | copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); |
Christoph Hellwig | b58294e | 2020-06-08 21:34:58 -0700 | [diff] [blame] | 73 | pagefault_enable(); |
| 74 | return 0; |
| 75 | Efault: |
| 76 | pagefault_enable(); |
| 77 | return -EFAULT; |
| 78 | } |
| 79 | |
| 80 | long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) |
| 81 | { |
| 82 | const void *src = unsafe_addr; |
| 83 | |
| 84 | if (unlikely(count <= 0)) |
| 85 | return 0; |
Christoph Hellwig | fe55731 | 2020-06-17 09:37:53 +0200 | [diff] [blame] | 86 | if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) |
Christoph Hellwig | 2a71e81 | 2020-06-08 21:35:04 -0700 | [diff] [blame] | 87 | return -ERANGE; |
Christoph Hellwig | b58294e | 2020-06-08 21:34:58 -0700 | [diff] [blame] | 88 | |
| 89 | pagefault_disable(); |
| 90 | do { |
| 91 | __get_kernel_nofault(dst, src, u8, Efault); |
| 92 | dst++; |
| 93 | src++; |
| 94 | } while (dst[-1] && src - unsafe_addr < count); |
| 95 | pagefault_enable(); |
| 96 | |
| 97 | dst[-1] = '\0'; |
| 98 | return src - unsafe_addr; |
| 99 | Efault: |
| 100 | pagefault_enable(); |
Alban Crequy | 8678ea0 | 2022-11-10 09:56:13 +0100 | [diff] [blame] | 101 | dst[0] = '\0'; |
Christoph Hellwig | b58294e | 2020-06-08 21:34:58 -0700 | [diff] [blame] | 102 | return -EFAULT; |
| 103 | } |
Masami Hiramatsu | 3d70818 | 2019-05-15 14:38:18 +0900 | [diff] [blame] | 104 | |
| 105 | /** |
Christoph Hellwig | c0ee37e | 2020-06-17 09:37:54 +0200 | [diff] [blame] | 106 | * copy_from_user_nofault(): safely attempt to read from a user-space location |
Christoph Hellwig | fc3562d7 | 2020-06-08 21:34:55 -0700 | [diff] [blame] | 107 | * @dst: pointer to the buffer that shall take the data |
| 108 | * @src: address to read from. This must be a user address. |
| 109 | * @size: size of the data chunk |
| 110 | * |
| 111 | * Safely read from user address @src to the buffer at @dst. If a kernel fault |
| 112 | * happens, handle that and return -EFAULT. |
| 113 | */ |
Christoph Hellwig | c0ee37e | 2020-06-17 09:37:54 +0200 | [diff] [blame] | 114 | long copy_from_user_nofault(void *dst, const void __user *src, size_t size) |
Christoph Hellwig | fc3562d7 | 2020-06-08 21:34:55 -0700 | [diff] [blame] | 115 | { |
| 116 | long ret = -EFAULT; |
Alexei Starovoitov | d319f34 | 2023-04-10 19:43:44 +0200 | [diff] [blame] | 117 | |
| 118 | if (!__access_ok(src, size)) |
| 119 | return ret; |
| 120 | |
| 121 | if (!nmi_uaccess_okay()) |
| 122 | return ret; |
| 123 | |
| 124 | pagefault_disable(); |
| 125 | ret = __copy_from_user_inatomic(dst, src, size); |
| 126 | pagefault_enable(); |
Christoph Hellwig | fc3562d7 | 2020-06-08 21:34:55 -0700 | [diff] [blame] | 127 | |
| 128 | if (ret) |
| 129 | return -EFAULT; |
| 130 | return 0; |
| 131 | } |
Christoph Hellwig | c0ee37e | 2020-06-17 09:37:54 +0200 | [diff] [blame] | 132 | EXPORT_SYMBOL_GPL(copy_from_user_nofault); |
Christoph Hellwig | fc3562d7 | 2020-06-08 21:34:55 -0700 | [diff] [blame] | 133 | |
| 134 | /** |
Christoph Hellwig | c0ee37e | 2020-06-17 09:37:54 +0200 | [diff] [blame] | 135 | * copy_to_user_nofault(): safely attempt to write to a user-space location |
Christoph Hellwig | fc3562d7 | 2020-06-08 21:34:55 -0700 | [diff] [blame] | 136 | * @dst: address to write to |
| 137 | * @src: pointer to the data that shall be written |
| 138 | * @size: size of the data chunk |
| 139 | * |
| 140 | * Safely write to address @dst from the buffer at @src. If a kernel fault |
| 141 | * happens, handle that and return -EFAULT. |
| 142 | */ |
Christoph Hellwig | c0ee37e | 2020-06-17 09:37:54 +0200 | [diff] [blame] | 143 | long copy_to_user_nofault(void __user *dst, const void *src, size_t size) |
Christoph Hellwig | fc3562d7 | 2020-06-08 21:34:55 -0700 | [diff] [blame] | 144 | { |
| 145 | long ret = -EFAULT; |
Christoph Hellwig | fc3562d7 | 2020-06-08 21:34:55 -0700 | [diff] [blame] | 146 | |
Christoph Hellwig | fc3562d7 | 2020-06-08 21:34:55 -0700 | [diff] [blame] | 147 | if (access_ok(dst, size)) { |
| 148 | pagefault_disable(); |
| 149 | ret = __copy_to_user_inatomic(dst, src, size); |
| 150 | pagefault_enable(); |
| 151 | } |
Christoph Hellwig | fc3562d7 | 2020-06-08 21:34:55 -0700 | [diff] [blame] | 152 | |
| 153 | if (ret) |
| 154 | return -EFAULT; |
| 155 | return 0; |
| 156 | } |
Christoph Hellwig | c0ee37e | 2020-06-17 09:37:54 +0200 | [diff] [blame] | 157 | EXPORT_SYMBOL_GPL(copy_to_user_nofault); |
Christoph Hellwig | fc3562d7 | 2020-06-08 21:34:55 -0700 | [diff] [blame] | 158 | |
| 159 | /** |
Christoph Hellwig | bd88bb5 | 2020-06-08 21:34:14 -0700 | [diff] [blame] | 160 | * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user |
Masami Hiramatsu | 3d70818 | 2019-05-15 14:38:18 +0900 | [diff] [blame] | 161 | * address. |
| 162 | * @dst: Destination address, in kernel space. This buffer must be at |
| 163 | * least @count bytes long. |
| 164 | * @unsafe_addr: Unsafe user address. |
| 165 | * @count: Maximum number of bytes to copy, including the trailing NUL. |
| 166 | * |
| 167 | * Copies a NUL-terminated string from unsafe user address to kernel buffer. |
| 168 | * |
| 169 | * On success, returns the length of the string INCLUDING the trailing NUL. |
| 170 | * |
| 171 | * If access fails, returns -EFAULT (some data may have been copied |
| 172 | * and the trailing NUL added). |
| 173 | * |
| 174 | * If @count is smaller than the length of the string, copies @count-1 bytes, |
| 175 | * sets the last byte of @dst buffer to NUL and returns @count. |
| 176 | */ |
Christoph Hellwig | bd88bb5 | 2020-06-08 21:34:14 -0700 | [diff] [blame] | 177 | long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, |
Masami Hiramatsu | 3d70818 | 2019-05-15 14:38:18 +0900 | [diff] [blame] | 178 | long count) |
| 179 | { |
Masami Hiramatsu | 3d70818 | 2019-05-15 14:38:18 +0900 | [diff] [blame] | 180 | long ret; |
| 181 | |
| 182 | if (unlikely(count <= 0)) |
| 183 | return 0; |
| 184 | |
Masami Hiramatsu | 3d70818 | 2019-05-15 14:38:18 +0900 | [diff] [blame] | 185 | pagefault_disable(); |
| 186 | ret = strncpy_from_user(dst, unsafe_addr, count); |
| 187 | pagefault_enable(); |
Masami Hiramatsu | 3d70818 | 2019-05-15 14:38:18 +0900 | [diff] [blame] | 188 | |
| 189 | if (ret >= count) { |
| 190 | ret = count; |
| 191 | dst[ret - 1] = '\0'; |
| 192 | } else if (ret > 0) { |
| 193 | ret++; |
| 194 | } |
| 195 | |
| 196 | return ret; |
| 197 | } |
| 198 | |
| 199 | /** |
Christoph Hellwig | 02dddb1 | 2020-06-08 21:34:20 -0700 | [diff] [blame] | 200 | * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. |
Masami Hiramatsu | 3d70818 | 2019-05-15 14:38:18 +0900 | [diff] [blame] | 201 | * @unsafe_addr: The string to measure. |
| 202 | * @count: Maximum count (including NUL) |
| 203 | * |
| 204 | * Get the size of a NUL-terminated string in user space without pagefault. |
| 205 | * |
| 206 | * Returns the size of the string INCLUDING the terminating NUL. |
| 207 | * |
| 208 | * If the string is too long, returns a number larger than @count. User |
| 209 | * has to check the return value against "> count". |
| 210 | * On exception (or invalid count), returns 0. |
| 211 | * |
| 212 | * Unlike strnlen_user, this can be used from IRQ handler etc. because |
| 213 | * it disables pagefaults. |
| 214 | */ |
Christoph Hellwig | 02dddb1 | 2020-06-08 21:34:20 -0700 | [diff] [blame] | 215 | long strnlen_user_nofault(const void __user *unsafe_addr, long count) |
Masami Hiramatsu | 3d70818 | 2019-05-15 14:38:18 +0900 | [diff] [blame] | 216 | { |
Masami Hiramatsu | 3d70818 | 2019-05-15 14:38:18 +0900 | [diff] [blame] | 217 | int ret; |
| 218 | |
Masami Hiramatsu | 3d70818 | 2019-05-15 14:38:18 +0900 | [diff] [blame] | 219 | pagefault_disable(); |
| 220 | ret = strnlen_user(unsafe_addr, count); |
| 221 | pagefault_enable(); |
Masami Hiramatsu | 3d70818 | 2019-05-15 14:38:18 +0900 | [diff] [blame] | 222 | |
| 223 | return ret; |
| 224 | } |
Christophe Leroy | ad7489d | 2022-03-22 14:47:49 -0700 | [diff] [blame] | 225 | |
| 226 | void __copy_overflow(int size, unsigned long count) |
| 227 | { |
| 228 | WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); |
| 229 | } |
| 230 | EXPORT_SYMBOL(__copy_overflow); |