Linus Torvalds | 170d13c | 2019-01-04 17:52:49 -0800 | [diff] [blame] | 1 | #include <linux/string.h> |
| 2 | #include <linux/module.h> |
| 3 | #include <linux/io.h> |
| 4 | |
Linus Torvalds | c228d29 | 2019-01-31 11:10:20 -0800 | [diff] [blame] | 5 | #define movs(type,to,from) \ |
| 6 | asm volatile("movs" type:"=&D" (to), "=&S" (from):"0" (to), "1" (from):"memory") |
| 7 | |
Linus Torvalds | 170d13c | 2019-01-04 17:52:49 -0800 | [diff] [blame] | 8 | /* Originally from i386/string.h */ |
Linus Torvalds | c228d29 | 2019-01-31 11:10:20 -0800 | [diff] [blame] | 9 | static __always_inline void rep_movs(void *to, const void *from, size_t n) |
Linus Torvalds | 170d13c | 2019-01-04 17:52:49 -0800 | [diff] [blame] | 10 | { |
| 11 | unsigned long d0, d1, d2; |
| 12 | asm volatile("rep ; movsl\n\t" |
| 13 | "testb $2,%b4\n\t" |
| 14 | "je 1f\n\t" |
| 15 | "movsw\n" |
| 16 | "1:\ttestb $1,%b4\n\t" |
| 17 | "je 2f\n\t" |
| 18 | "movsb\n" |
| 19 | "2:" |
| 20 | : "=&c" (d0), "=&D" (d1), "=&S" (d2) |
| 21 | : "0" (n / 4), "q" (n), "1" ((long)to), "2" ((long)from) |
| 22 | : "memory"); |
| 23 | } |
| 24 | |
| 25 | void memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) |
| 26 | { |
Linus Torvalds | c228d29 | 2019-01-31 11:10:20 -0800 | [diff] [blame] | 27 | if (unlikely(!n)) |
| 28 | return; |
| 29 | |
| 30 | /* Align any unaligned source IO */ |
| 31 | if (unlikely(1 & (unsigned long)from)) { |
| 32 | movs("b", to, from); |
| 33 | n--; |
| 34 | } |
| 35 | if (n > 1 && unlikely(2 & (unsigned long)from)) { |
| 36 | movs("w", to, from); |
| 37 | n-=2; |
| 38 | } |
| 39 | rep_movs(to, (const void *)from, n); |
Linus Torvalds | 170d13c | 2019-01-04 17:52:49 -0800 | [diff] [blame] | 40 | } |
| 41 | EXPORT_SYMBOL(memcpy_fromio); |
| 42 | |
| 43 | void memcpy_toio(volatile void __iomem *to, const void *from, size_t n) |
| 44 | { |
Linus Torvalds | c228d29 | 2019-01-31 11:10:20 -0800 | [diff] [blame] | 45 | if (unlikely(!n)) |
| 46 | return; |
| 47 | |
| 48 | /* Align any unaligned destination IO */ |
| 49 | if (unlikely(1 & (unsigned long)to)) { |
| 50 | movs("b", to, from); |
| 51 | n--; |
| 52 | } |
| 53 | if (n > 1 && unlikely(2 & (unsigned long)to)) { |
| 54 | movs("w", to, from); |
| 55 | n-=2; |
| 56 | } |
| 57 | rep_movs((void *)to, (const void *) from, n); |
Linus Torvalds | 170d13c | 2019-01-04 17:52:49 -0800 | [diff] [blame] | 58 | } |
| 59 | EXPORT_SYMBOL(memcpy_toio); |
| 60 | |
| 61 | void memset_io(volatile void __iomem *a, int b, size_t c) |
| 62 | { |
| 63 | /* |
| 64 | * TODO: memset can mangle the IO patterns quite a bit. |
| 65 | * perhaps it would be better to use a dumb one: |
| 66 | */ |
| 67 | memset((void *)a, b, c); |
| 68 | } |
| 69 | EXPORT_SYMBOL(memset_io); |