Jeremy Fitzhardinge | e982638 | 2011-08-18 11:48:06 -0700 | [diff] [blame] | 1 | #ifndef ASM_X86_CMPXCHG_H |
| 2 | #define ASM_X86_CMPXCHG_H |
| 3 | |
Jeremy Fitzhardinge | 61e2cd0 | 2011-08-29 14:47:58 -0700 | [diff] [blame] | 4 | #include <linux/compiler.h> |
Jeremy Fitzhardinge | e982638 | 2011-08-18 11:48:06 -0700 | [diff] [blame] | 5 | #include <asm/alternative.h> /* Provides LOCK_PREFIX */ |
| 6 | |
Jeremy Fitzhardinge | 61e2cd0 | 2011-08-29 14:47:58 -0700 | [diff] [blame] | 7 | /* |
| 8 | * Non-existant functions to indicate usage errors at link time |
| 9 | * (or compile-time if the compiler implements __compiletime_error(). |
| 10 | */ |
| 11 | extern void __xchg_wrong_size(void) |
| 12 | __compiletime_error("Bad argument size for xchg"); |
| 13 | extern void __cmpxchg_wrong_size(void) |
| 14 | __compiletime_error("Bad argument size for cmpxchg"); |
| 15 | extern void __xadd_wrong_size(void) |
| 16 | __compiletime_error("Bad argument size for xadd"); |
Jeremy Fitzhardinge | e982638 | 2011-08-18 11:48:06 -0700 | [diff] [blame] | 17 | |
| 18 | /* |
| 19 | * Constants for operation sizes. On 32-bit, the 64-bit size it set to |
| 20 | * -1 because sizeof will never return -1, thereby making those switch |
| 21 | * case statements guaranteeed dead code which the compiler will |
| 22 | * eliminate, and allowing the "missing symbol in the default case" to |
| 23 | * indicate a usage error. |
| 24 | */ |
| 25 | #define __X86_CASE_B 1 |
| 26 | #define __X86_CASE_W 2 |
| 27 | #define __X86_CASE_L 4 |
| 28 | #ifdef CONFIG_64BIT |
| 29 | #define __X86_CASE_Q 8 |
| 30 | #else |
| 31 | #define __X86_CASE_Q -1 /* sizeof will never return -1 */ |
| 32 | #endif |
| 33 | |
| 34 | /* |
| 35 | * Note: no "lock" prefix even on SMP: xchg always implies lock anyway. |
| 36 | * Since this is generally used to protect other memory information, we |
| 37 | * use "asm volatile" and "memory" clobbers to prevent gcc from moving |
| 38 | * information around. |
| 39 | */ |
| 40 | #define __xchg(x, ptr, size) \ |
| 41 | ({ \ |
| 42 | __typeof(*(ptr)) __x = (x); \ |
| 43 | switch (size) { \ |
| 44 | case __X86_CASE_B: \ |
| 45 | { \ |
| 46 | volatile u8 *__ptr = (volatile u8 *)(ptr); \ |
| 47 | asm volatile("xchgb %0,%1" \ |
| 48 | : "=q" (__x), "+m" (*__ptr) \ |
| 49 | : "0" (__x) \ |
| 50 | : "memory"); \ |
| 51 | break; \ |
| 52 | } \ |
| 53 | case __X86_CASE_W: \ |
| 54 | { \ |
| 55 | volatile u16 *__ptr = (volatile u16 *)(ptr); \ |
| 56 | asm volatile("xchgw %0,%1" \ |
| 57 | : "=r" (__x), "+m" (*__ptr) \ |
| 58 | : "0" (__x) \ |
| 59 | : "memory"); \ |
| 60 | break; \ |
| 61 | } \ |
| 62 | case __X86_CASE_L: \ |
| 63 | { \ |
| 64 | volatile u32 *__ptr = (volatile u32 *)(ptr); \ |
| 65 | asm volatile("xchgl %0,%1" \ |
| 66 | : "=r" (__x), "+m" (*__ptr) \ |
| 67 | : "0" (__x) \ |
| 68 | : "memory"); \ |
| 69 | break; \ |
| 70 | } \ |
| 71 | case __X86_CASE_Q: \ |
| 72 | { \ |
| 73 | volatile u64 *__ptr = (volatile u64 *)(ptr); \ |
| 74 | asm volatile("xchgq %0,%1" \ |
| 75 | : "=r" (__x), "+m" (*__ptr) \ |
| 76 | : "0" (__x) \ |
| 77 | : "memory"); \ |
| 78 | break; \ |
| 79 | } \ |
| 80 | default: \ |
| 81 | __xchg_wrong_size(); \ |
| 82 | } \ |
| 83 | __x; \ |
| 84 | }) |
| 85 | |
| 86 | #define xchg(ptr, v) \ |
| 87 | __xchg((v), (ptr), sizeof(*ptr)) |
| 88 | |
| 89 | /* |
| 90 | * Atomic compare and exchange. Compare OLD with MEM, if identical, |
| 91 | * store NEW in MEM. Return the initial value in MEM. Success is |
| 92 | * indicated by comparing RETURN with OLD. |
| 93 | */ |
| 94 | #define __raw_cmpxchg(ptr, old, new, size, lock) \ |
| 95 | ({ \ |
| 96 | __typeof__(*(ptr)) __ret; \ |
| 97 | __typeof__(*(ptr)) __old = (old); \ |
| 98 | __typeof__(*(ptr)) __new = (new); \ |
| 99 | switch (size) { \ |
| 100 | case __X86_CASE_B: \ |
| 101 | { \ |
| 102 | volatile u8 *__ptr = (volatile u8 *)(ptr); \ |
| 103 | asm volatile(lock "cmpxchgb %2,%1" \ |
| 104 | : "=a" (__ret), "+m" (*__ptr) \ |
| 105 | : "q" (__new), "0" (__old) \ |
| 106 | : "memory"); \ |
| 107 | break; \ |
| 108 | } \ |
| 109 | case __X86_CASE_W: \ |
| 110 | { \ |
| 111 | volatile u16 *__ptr = (volatile u16 *)(ptr); \ |
| 112 | asm volatile(lock "cmpxchgw %2,%1" \ |
| 113 | : "=a" (__ret), "+m" (*__ptr) \ |
| 114 | : "r" (__new), "0" (__old) \ |
| 115 | : "memory"); \ |
| 116 | break; \ |
| 117 | } \ |
| 118 | case __X86_CASE_L: \ |
| 119 | { \ |
| 120 | volatile u32 *__ptr = (volatile u32 *)(ptr); \ |
| 121 | asm volatile(lock "cmpxchgl %2,%1" \ |
| 122 | : "=a" (__ret), "+m" (*__ptr) \ |
| 123 | : "r" (__new), "0" (__old) \ |
| 124 | : "memory"); \ |
| 125 | break; \ |
| 126 | } \ |
| 127 | case __X86_CASE_Q: \ |
| 128 | { \ |
| 129 | volatile u64 *__ptr = (volatile u64 *)(ptr); \ |
| 130 | asm volatile(lock "cmpxchgq %2,%1" \ |
| 131 | : "=a" (__ret), "+m" (*__ptr) \ |
| 132 | : "r" (__new), "0" (__old) \ |
| 133 | : "memory"); \ |
| 134 | break; \ |
| 135 | } \ |
| 136 | default: \ |
| 137 | __cmpxchg_wrong_size(); \ |
| 138 | } \ |
| 139 | __ret; \ |
| 140 | }) |
| 141 | |
| 142 | #define __cmpxchg(ptr, old, new, size) \ |
| 143 | __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX) |
| 144 | |
| 145 | #define __sync_cmpxchg(ptr, old, new, size) \ |
| 146 | __raw_cmpxchg((ptr), (old), (new), (size), "lock; ") |
| 147 | |
| 148 | #define __cmpxchg_local(ptr, old, new, size) \ |
| 149 | __raw_cmpxchg((ptr), (old), (new), (size), "") |
| 150 | |
Thomas Gleixner | 96a388d | 2007-10-11 11:20:03 +0200 | [diff] [blame] | 151 | #ifdef CONFIG_X86_32 |
| 152 | # include "cmpxchg_32.h" |
| 153 | #else |
| 154 | # include "cmpxchg_64.h" |
| 155 | #endif |
Jeremy Fitzhardinge | e982638 | 2011-08-18 11:48:06 -0700 | [diff] [blame] | 156 | |
| 157 | #ifdef __HAVE_ARCH_CMPXCHG |
| 158 | #define cmpxchg(ptr, old, new) \ |
| 159 | __cmpxchg((ptr), (old), (new), sizeof(*ptr)) |
| 160 | |
| 161 | #define sync_cmpxchg(ptr, old, new) \ |
| 162 | __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr)) |
| 163 | |
| 164 | #define cmpxchg_local(ptr, old, new) \ |
| 165 | __cmpxchg_local((ptr), (old), (new), sizeof(*ptr)) |
| 166 | #endif |
| 167 | |
Jeremy Fitzhardinge | 433b3520 | 2011-06-21 12:00:55 -0700 | [diff] [blame] | 168 | #define __xadd(ptr, inc, lock) \ |
| 169 | ({ \ |
| 170 | __typeof__ (*(ptr)) __ret = (inc); \ |
| 171 | switch (sizeof(*(ptr))) { \ |
| 172 | case __X86_CASE_B: \ |
| 173 | asm volatile (lock "xaddb %b0, %1\n" \ |
| 174 | : "+r" (__ret), "+m" (*(ptr)) \ |
| 175 | : : "memory", "cc"); \ |
| 176 | break; \ |
| 177 | case __X86_CASE_W: \ |
| 178 | asm volatile (lock "xaddw %w0, %1\n" \ |
| 179 | : "+r" (__ret), "+m" (*(ptr)) \ |
| 180 | : : "memory", "cc"); \ |
| 181 | break; \ |
| 182 | case __X86_CASE_L: \ |
| 183 | asm volatile (lock "xaddl %0, %1\n" \ |
| 184 | : "+r" (__ret), "+m" (*(ptr)) \ |
| 185 | : : "memory", "cc"); \ |
| 186 | break; \ |
| 187 | case __X86_CASE_Q: \ |
| 188 | asm volatile (lock "xaddq %q0, %1\n" \ |
| 189 | : "+r" (__ret), "+m" (*(ptr)) \ |
| 190 | : : "memory", "cc"); \ |
| 191 | break; \ |
| 192 | default: \ |
| 193 | __xadd_wrong_size(); \ |
| 194 | } \ |
| 195 | __ret; \ |
| 196 | }) |
| 197 | |
| 198 | /* |
| 199 | * xadd() adds "inc" to "*ptr" and atomically returns the previous |
| 200 | * value of "*ptr". |
| 201 | * |
| 202 | * xadd() is locked when multiple CPUs are online |
| 203 | * xadd_sync() is always locked |
| 204 | * xadd_local() is never locked |
| 205 | */ |
| 206 | #define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX) |
| 207 | #define xadd_sync(ptr, inc) __xadd((ptr), (inc), "lock; ") |
| 208 | #define xadd_local(ptr, inc) __xadd((ptr), (inc), "") |
| 209 | |
Jeremy Fitzhardinge | e982638 | 2011-08-18 11:48:06 -0700 | [diff] [blame] | 210 | #endif /* ASM_X86_CMPXCHG_H */ |