Thomas Gleixner | 457c899 | 2019-05-19 13:08:55 +0100 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 2 | /* Copyright 2002 Andi Kleen */ |
Dave Jones | 038b0a6 | 2006-10-04 03:38:54 -0400 | [diff] [blame] | 3 | |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 4 | #include <linux/linkage.h> |
Tony Luck | cbf8b5a | 2016-03-14 15:33:39 -0700 | [diff] [blame] | 5 | #include <asm/errno.h> |
Borislav Petkov | cd4d09e | 2016-01-26 22:12:04 +0100 | [diff] [blame] | 6 | #include <asm/cpufeatures.h> |
Dan Williams | 5d8beee | 2018-05-01 06:49:45 -0700 | [diff] [blame] | 7 | #include <asm/mcsafe_test.h> |
Fenghua Yu | 101068c | 2011-05-17 15:29:16 -0700 | [diff] [blame] | 8 | #include <asm/alternative-asm.h> |
Al Viro | 784d569 | 2016-01-11 11:04:34 -0500 | [diff] [blame] | 9 | #include <asm/export.h> |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 10 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | /* |
Borislav Petkov | e0bc8d1 | 2015-02-04 15:36:49 +0100 | [diff] [blame] | 12 | * We build a jump to memcpy_orig by default which gets NOPped out on |
| 13 | * the majority of x86 CPUs which set REP_GOOD. In addition, CPUs which |
| 14 | * have the enhanced REP MOVSB/STOSB feature (ERMS), change those NOPs |
| 15 | * to a jmp to memcpy_erms which does the REP; MOVSB mem copy. |
| 16 | */ |
| 17 | |
| 18 | .weak memcpy |
| 19 | |
| 20 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | * memcpy - Copy a memory block. |
| 22 | * |
Ingo Molnar | f3b6eaf | 2009-03-12 12:20:17 +0100 | [diff] [blame] | 23 | * Input: |
| 24 | * rdi destination |
| 25 | * rsi source |
| 26 | * rdx count |
| 27 | * |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 28 | * Output: |
| 29 | * rax original destination |
Ingo Molnar | f3b6eaf | 2009-03-12 12:20:17 +0100 | [diff] [blame] | 30 | */ |
Borislav Petkov | e0bc8d1 | 2015-02-04 15:36:49 +0100 | [diff] [blame] | 31 | ENTRY(__memcpy) |
| 32 | ENTRY(memcpy) |
| 33 | ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \ |
| 34 | "jmp memcpy_erms", X86_FEATURE_ERMS |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | |
Ingo Molnar | f3b6eaf | 2009-03-12 12:20:17 +0100 | [diff] [blame] | 36 | movq %rdi, %rax |
Jan Beulich | 2ab5609 | 2012-01-26 15:50:55 +0000 | [diff] [blame] | 37 | movq %rdx, %rcx |
| 38 | shrq $3, %rcx |
Ingo Molnar | f3b6eaf | 2009-03-12 12:20:17 +0100 | [diff] [blame] | 39 | andl $7, %edx |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 40 | rep movsq |
Ingo Molnar | f3b6eaf | 2009-03-12 12:20:17 +0100 | [diff] [blame] | 41 | movl %edx, %ecx |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 42 | rep movsb |
| 43 | ret |
Borislav Petkov | e0bc8d1 | 2015-02-04 15:36:49 +0100 | [diff] [blame] | 44 | ENDPROC(memcpy) |
| 45 | ENDPROC(__memcpy) |
Al Viro | 784d569 | 2016-01-11 11:04:34 -0500 | [diff] [blame] | 46 | EXPORT_SYMBOL(memcpy) |
| 47 | EXPORT_SYMBOL(__memcpy) |
Jan Beulich | 8d379da | 2006-09-26 10:52:32 +0200 | [diff] [blame] | 48 | |
Fenghua Yu | 101068c | 2011-05-17 15:29:16 -0700 | [diff] [blame] | 49 | /* |
Borislav Petkov | e0bc8d1 | 2015-02-04 15:36:49 +0100 | [diff] [blame] | 50 | * memcpy_erms() - enhanced fast string memcpy. This is faster and |
| 51 | * simpler than memcpy. Use memcpy_erms when possible. |
Fenghua Yu | 101068c | 2011-05-17 15:29:16 -0700 | [diff] [blame] | 52 | */ |
Borislav Petkov | e0bc8d1 | 2015-02-04 15:36:49 +0100 | [diff] [blame] | 53 | ENTRY(memcpy_erms) |
Fenghua Yu | 101068c | 2011-05-17 15:29:16 -0700 | [diff] [blame] | 54 | movq %rdi, %rax |
Jan Beulich | 2ab5609 | 2012-01-26 15:50:55 +0000 | [diff] [blame] | 55 | movq %rdx, %rcx |
Fenghua Yu | 101068c | 2011-05-17 15:29:16 -0700 | [diff] [blame] | 56 | rep movsb |
| 57 | ret |
Borislav Petkov | e0bc8d1 | 2015-02-04 15:36:49 +0100 | [diff] [blame] | 58 | ENDPROC(memcpy_erms) |
Fenghua Yu | 101068c | 2011-05-17 15:29:16 -0700 | [diff] [blame] | 59 | |
Borislav Petkov | e0bc8d1 | 2015-02-04 15:36:49 +0100 | [diff] [blame] | 60 | ENTRY(memcpy_orig) |
Ingo Molnar | f3b6eaf | 2009-03-12 12:20:17 +0100 | [diff] [blame] | 61 | movq %rdi, %rax |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 62 | |
Jan Beulich | 2ab5609 | 2012-01-26 15:50:55 +0000 | [diff] [blame] | 63 | cmpq $0x20, %rdx |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 64 | jb .Lhandle_tail |
| 65 | |
| 66 | /* |
Bart Van Assche | 9de4966 | 2011-05-01 14:09:21 +0200 | [diff] [blame] | 67 | * We check whether memory false dependence could occur, |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 68 | * then jump to corresponding copy mode. |
| 69 | */ |
| 70 | cmp %dil, %sil |
| 71 | jl .Lcopy_backward |
Jan Beulich | 2ab5609 | 2012-01-26 15:50:55 +0000 | [diff] [blame] | 72 | subq $0x20, %rdx |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 73 | .Lcopy_forward_loop: |
| 74 | subq $0x20, %rdx |
| 75 | |
| 76 | /* |
| 77 | * Move in blocks of 4x8 bytes: |
| 78 | */ |
| 79 | movq 0*8(%rsi), %r8 |
| 80 | movq 1*8(%rsi), %r9 |
| 81 | movq 2*8(%rsi), %r10 |
| 82 | movq 3*8(%rsi), %r11 |
| 83 | leaq 4*8(%rsi), %rsi |
| 84 | |
| 85 | movq %r8, 0*8(%rdi) |
| 86 | movq %r9, 1*8(%rdi) |
| 87 | movq %r10, 2*8(%rdi) |
| 88 | movq %r11, 3*8(%rdi) |
| 89 | leaq 4*8(%rdi), %rdi |
| 90 | jae .Lcopy_forward_loop |
Jan Beulich | 2ab5609 | 2012-01-26 15:50:55 +0000 | [diff] [blame] | 91 | addl $0x20, %edx |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 92 | jmp .Lhandle_tail |
| 93 | |
| 94 | .Lcopy_backward: |
| 95 | /* |
| 96 | * Calculate copy position to tail. |
| 97 | */ |
| 98 | addq %rdx, %rsi |
| 99 | addq %rdx, %rdi |
| 100 | subq $0x20, %rdx |
| 101 | /* |
| 102 | * At most 3 ALU operations in one cycle, |
Andy Shevchenko | d50ba368 | 2013-04-15 12:06:10 +0300 | [diff] [blame] | 103 | * so append NOPS in the same 16 bytes trunk. |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 104 | */ |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 105 | .p2align 4 |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 106 | .Lcopy_backward_loop: |
| 107 | subq $0x20, %rdx |
| 108 | movq -1*8(%rsi), %r8 |
| 109 | movq -2*8(%rsi), %r9 |
| 110 | movq -3*8(%rsi), %r10 |
| 111 | movq -4*8(%rsi), %r11 |
| 112 | leaq -4*8(%rsi), %rsi |
| 113 | movq %r8, -1*8(%rdi) |
| 114 | movq %r9, -2*8(%rdi) |
| 115 | movq %r10, -3*8(%rdi) |
| 116 | movq %r11, -4*8(%rdi) |
| 117 | leaq -4*8(%rdi), %rdi |
| 118 | jae .Lcopy_backward_loop |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 119 | |
Ingo Molnar | f3b6eaf | 2009-03-12 12:20:17 +0100 | [diff] [blame] | 120 | /* |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 121 | * Calculate copy position to head. |
Ingo Molnar | f3b6eaf | 2009-03-12 12:20:17 +0100 | [diff] [blame] | 122 | */ |
Jan Beulich | 2ab5609 | 2012-01-26 15:50:55 +0000 | [diff] [blame] | 123 | addl $0x20, %edx |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 124 | subq %rdx, %rsi |
| 125 | subq %rdx, %rdi |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 126 | .Lhandle_tail: |
Jan Beulich | 2ab5609 | 2012-01-26 15:50:55 +0000 | [diff] [blame] | 127 | cmpl $16, %edx |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 128 | jb .Lless_16bytes |
Ingo Molnar | f3b6eaf | 2009-03-12 12:20:17 +0100 | [diff] [blame] | 129 | |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 130 | /* |
| 131 | * Move data from 16 bytes to 31 bytes. |
| 132 | */ |
| 133 | movq 0*8(%rsi), %r8 |
| 134 | movq 1*8(%rsi), %r9 |
| 135 | movq -2*8(%rsi, %rdx), %r10 |
| 136 | movq -1*8(%rsi, %rdx), %r11 |
| 137 | movq %r8, 0*8(%rdi) |
| 138 | movq %r9, 1*8(%rdi) |
| 139 | movq %r10, -2*8(%rdi, %rdx) |
| 140 | movq %r11, -1*8(%rdi, %rdx) |
| 141 | retq |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 142 | .p2align 4 |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 143 | .Lless_16bytes: |
Jan Beulich | 2ab5609 | 2012-01-26 15:50:55 +0000 | [diff] [blame] | 144 | cmpl $8, %edx |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 145 | jb .Lless_8bytes |
| 146 | /* |
| 147 | * Move data from 8 bytes to 15 bytes. |
| 148 | */ |
| 149 | movq 0*8(%rsi), %r8 |
| 150 | movq -1*8(%rsi, %rdx), %r9 |
| 151 | movq %r8, 0*8(%rdi) |
| 152 | movq %r9, -1*8(%rdi, %rdx) |
| 153 | retq |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 154 | .p2align 4 |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 155 | .Lless_8bytes: |
Jan Beulich | 2ab5609 | 2012-01-26 15:50:55 +0000 | [diff] [blame] | 156 | cmpl $4, %edx |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 157 | jb .Lless_3bytes |
| 158 | |
| 159 | /* |
| 160 | * Move data from 4 bytes to 7 bytes. |
| 161 | */ |
| 162 | movl (%rsi), %ecx |
| 163 | movl -4(%rsi, %rdx), %r8d |
| 164 | movl %ecx, (%rdi) |
| 165 | movl %r8d, -4(%rdi, %rdx) |
| 166 | retq |
| 167 | .p2align 4 |
| 168 | .Lless_3bytes: |
Jan Beulich | 9d8e227 | 2012-01-26 15:55:32 +0000 | [diff] [blame] | 169 | subl $1, %edx |
| 170 | jb .Lend |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 171 | /* |
| 172 | * Move data from 1 bytes to 3 bytes. |
| 173 | */ |
Jan Beulich | 9d8e227 | 2012-01-26 15:55:32 +0000 | [diff] [blame] | 174 | movzbl (%rsi), %ecx |
| 175 | jz .Lstore_1byte |
| 176 | movzbq 1(%rsi), %r8 |
| 177 | movzbq (%rsi, %rdx), %r9 |
| 178 | movb %r8b, 1(%rdi) |
| 179 | movb %r9b, (%rdi, %rdx) |
| 180 | .Lstore_1byte: |
| 181 | movb %cl, (%rdi) |
Andi Kleen | 7bcd3f3 | 2006-02-03 21:51:02 +0100 | [diff] [blame] | 182 | |
Ingo Molnar | f3b6eaf | 2009-03-12 12:20:17 +0100 | [diff] [blame] | 183 | .Lend: |
Ma Ling | 59daa70 | 2010-06-29 03:24:25 +0800 | [diff] [blame] | 184 | retq |
Borislav Petkov | e0bc8d1 | 2015-02-04 15:36:49 +0100 | [diff] [blame] | 185 | ENDPROC(memcpy_orig) |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 186 | |
| 187 | #ifndef CONFIG_UML |
Dan Williams | 5d8beee | 2018-05-01 06:49:45 -0700 | [diff] [blame] | 188 | |
| 189 | MCSAFE_TEST_CTL |
| 190 | |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 191 | /* |
Dan Williams | da7bc9c | 2018-05-03 17:06:11 -0700 | [diff] [blame] | 192 | * __memcpy_mcsafe - memory copy with machine check exception handling |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 193 | * Note that we only catch machine checks when reading the source addresses. |
| 194 | * Writes to target are posted and don't generate machine checks. |
| 195 | */ |
Dan Williams | da7bc9c | 2018-05-03 17:06:11 -0700 | [diff] [blame] | 196 | ENTRY(__memcpy_mcsafe) |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 197 | cmpl $8, %edx |
| 198 | /* Less than 8 bytes? Go to byte copy loop */ |
| 199 | jb .L_no_whole_words |
| 200 | |
| 201 | /* Check for bad alignment of source */ |
| 202 | testl $7, %esi |
| 203 | /* Already aligned */ |
| 204 | jz .L_8byte_aligned |
| 205 | |
| 206 | /* Copy one byte at a time until source is 8-byte aligned */ |
| 207 | movl %esi, %ecx |
| 208 | andl $7, %ecx |
| 209 | subl $8, %ecx |
| 210 | negl %ecx |
| 211 | subl %ecx, %edx |
Dan Williams | bd13154 | 2018-05-03 17:06:16 -0700 | [diff] [blame] | 212 | .L_read_leading_bytes: |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 213 | movb (%rsi), %al |
Dan Williams | 5d8beee | 2018-05-01 06:49:45 -0700 | [diff] [blame] | 214 | MCSAFE_TEST_SRC %rsi 1 .E_leading_bytes |
| 215 | MCSAFE_TEST_DST %rdi 1 .E_leading_bytes |
Dan Williams | bd13154 | 2018-05-03 17:06:16 -0700 | [diff] [blame] | 216 | .L_write_leading_bytes: |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 217 | movb %al, (%rdi) |
| 218 | incq %rsi |
| 219 | incq %rdi |
| 220 | decl %ecx |
Dan Williams | bd13154 | 2018-05-03 17:06:16 -0700 | [diff] [blame] | 221 | jnz .L_read_leading_bytes |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 222 | |
| 223 | .L_8byte_aligned: |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 224 | movl %edx, %ecx |
| 225 | andl $7, %edx |
| 226 | shrl $3, %ecx |
| 227 | jz .L_no_whole_words |
| 228 | |
Dan Williams | bd13154 | 2018-05-03 17:06:16 -0700 | [diff] [blame] | 229 | .L_read_words: |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 230 | movq (%rsi), %r8 |
Dan Williams | 5d8beee | 2018-05-01 06:49:45 -0700 | [diff] [blame] | 231 | MCSAFE_TEST_SRC %rsi 8 .E_read_words |
| 232 | MCSAFE_TEST_DST %rdi 8 .E_write_words |
Dan Williams | bd13154 | 2018-05-03 17:06:16 -0700 | [diff] [blame] | 233 | .L_write_words: |
Dan Williams | da7bc9c | 2018-05-03 17:06:11 -0700 | [diff] [blame] | 234 | movq %r8, (%rdi) |
| 235 | addq $8, %rsi |
| 236 | addq $8, %rdi |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 237 | decl %ecx |
Dan Williams | bd13154 | 2018-05-03 17:06:16 -0700 | [diff] [blame] | 238 | jnz .L_read_words |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 239 | |
| 240 | /* Any trailing bytes? */ |
| 241 | .L_no_whole_words: |
| 242 | andl %edx, %edx |
| 243 | jz .L_done_memcpy_trap |
| 244 | |
| 245 | /* Copy trailing bytes */ |
| 246 | movl %edx, %ecx |
Dan Williams | bd13154 | 2018-05-03 17:06:16 -0700 | [diff] [blame] | 247 | .L_read_trailing_bytes: |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 248 | movb (%rsi), %al |
Dan Williams | 5d8beee | 2018-05-01 06:49:45 -0700 | [diff] [blame] | 249 | MCSAFE_TEST_SRC %rsi 1 .E_trailing_bytes |
| 250 | MCSAFE_TEST_DST %rdi 1 .E_trailing_bytes |
Dan Williams | bd13154 | 2018-05-03 17:06:16 -0700 | [diff] [blame] | 251 | .L_write_trailing_bytes: |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 252 | movb %al, (%rdi) |
| 253 | incq %rsi |
| 254 | incq %rdi |
| 255 | decl %ecx |
Dan Williams | bd13154 | 2018-05-03 17:06:16 -0700 | [diff] [blame] | 256 | jnz .L_read_trailing_bytes |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 257 | |
Tony Luck | cbf8b5a | 2016-03-14 15:33:39 -0700 | [diff] [blame] | 258 | /* Copy successful. Return zero */ |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 259 | .L_done_memcpy_trap: |
Jan Beulich | a7bea83 | 2018-07-02 04:31:54 -0600 | [diff] [blame] | 260 | xorl %eax, %eax |
Peter Zijlstra | b69656f | 2019-04-03 09:39:45 +0200 | [diff] [blame] | 261 | .L_done: |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 262 | ret |
Dan Williams | da7bc9c | 2018-05-03 17:06:11 -0700 | [diff] [blame] | 263 | ENDPROC(__memcpy_mcsafe) |
| 264 | EXPORT_SYMBOL_GPL(__memcpy_mcsafe) |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 265 | |
| 266 | .section .fixup, "ax" |
Dan Williams | 60622d6 | 2018-05-03 17:06:21 -0700 | [diff] [blame] | 267 | /* |
| 268 | * Return number of bytes not copied for any failure. Note that |
| 269 | * there is no "tail" handling since the source buffer is 8-byte |
| 270 | * aligned and poison is cacheline aligned. |
| 271 | */ |
| 272 | .E_read_words: |
| 273 | shll $3, %ecx |
| 274 | .E_leading_bytes: |
| 275 | addl %edx, %ecx |
| 276 | .E_trailing_bytes: |
| 277 | mov %ecx, %eax |
Peter Zijlstra | b69656f | 2019-04-03 09:39:45 +0200 | [diff] [blame] | 278 | jmp .L_done |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 279 | |
Dan Williams | 12c8913 | 2018-05-03 17:06:26 -0700 | [diff] [blame] | 280 | /* |
| 281 | * For write fault handling, given the destination is unaligned, |
| 282 | * we handle faults on multi-byte writes with a byte-by-byte |
| 283 | * copy up to the write-protected page. |
| 284 | */ |
| 285 | .E_write_words: |
| 286 | shll $3, %ecx |
| 287 | addl %edx, %ecx |
| 288 | movl %ecx, %edx |
| 289 | jmp mcsafe_handle_tail |
| 290 | |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 291 | .previous |
| 292 | |
Dan Williams | 60622d6 | 2018-05-03 17:06:21 -0700 | [diff] [blame] | 293 | _ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes) |
| 294 | _ASM_EXTABLE_FAULT(.L_read_words, .E_read_words) |
| 295 | _ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes) |
Dan Williams | 12c8913 | 2018-05-03 17:06:26 -0700 | [diff] [blame] | 296 | _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes) |
| 297 | _ASM_EXTABLE(.L_write_words, .E_write_words) |
| 298 | _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes) |
Tony Luck | 92b0729 | 2016-02-18 11:47:26 -0800 | [diff] [blame] | 299 | #endif |