Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 1 | /* |
Dave Jones | 835c34a | 2007-10-12 21:10:53 -0400 | [diff] [blame] | 2 | * check TSC synchronization. |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 3 | * |
| 4 | * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar |
| 5 | * |
| 6 | * We check whether all boot CPUs have their TSC's synchronized, |
| 7 | * print a warning if not and turn off the TSC clock-source. |
| 8 | * |
| 9 | * The warp-check is point-to-point between two CPUs, the CPU |
| 10 | * initiating the bootup is the 'source CPU', the freshly booting |
| 11 | * CPU is the 'target CPU'. |
| 12 | * |
| 13 | * Only two CPUs may participate - they can enter in any order. |
| 14 | * ( The serial nature of the boot logic and the CPU hotplug lock |
| 15 | * protects against more than 2 CPUs entering this code. ) |
| 16 | */ |
| 17 | #include <linux/spinlock.h> |
| 18 | #include <linux/kernel.h> |
| 19 | #include <linux/init.h> |
| 20 | #include <linux/smp.h> |
| 21 | #include <linux/nmi.h> |
| 22 | #include <asm/tsc.h> |
| 23 | |
| 24 | /* |
| 25 | * Entry/exit counters that make sure that both CPUs |
| 26 | * run the measurement code at once: |
| 27 | */ |
| 28 | static __cpuinitdata atomic_t start_count; |
| 29 | static __cpuinitdata atomic_t stop_count; |
| 30 | |
| 31 | /* |
| 32 | * We use a raw spinlock in this exceptional case, because |
| 33 | * we want to have the fastest, inlined, non-debug version |
| 34 | * of a critical section, to be able to prove TSC time-warps: |
| 35 | */ |
Thomas Gleixner | edc35bd | 2009-12-03 12:38:57 +0100 | [diff] [blame] | 36 | static __cpuinitdata arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; |
Ingo Molnar | 643bec9 | 2009-05-07 09:12:50 +0200 | [diff] [blame] | 37 | |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 38 | static __cpuinitdata cycles_t last_tsc; |
| 39 | static __cpuinitdata cycles_t max_warp; |
| 40 | static __cpuinitdata int nr_warps; |
| 41 | |
| 42 | /* |
| 43 | * TSC-warp measurement loop running on both CPUs: |
| 44 | */ |
Suresh Siddha | b0e5c77 | 2012-02-06 18:32:20 -0800 | [diff] [blame] | 45 | static __cpuinit void check_tsc_warp(unsigned int timeout) |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 46 | { |
| 47 | cycles_t start, now, prev, end; |
| 48 | int i; |
| 49 | |
Venki Pallipadi | 93ce99e | 2008-11-17 14:43:58 -0800 | [diff] [blame] | 50 | rdtsc_barrier(); |
Andi Kleen | 6d63de8 | 2008-01-30 13:32:39 +0100 | [diff] [blame] | 51 | start = get_cycles(); |
Venki Pallipadi | 93ce99e | 2008-11-17 14:43:58 -0800 | [diff] [blame] | 52 | rdtsc_barrier(); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 53 | /* |
Suresh Siddha | b0e5c77 | 2012-02-06 18:32:20 -0800 | [diff] [blame] | 54 | * The measurement runs for 'timeout' msecs: |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 55 | */ |
Suresh Siddha | b0e5c77 | 2012-02-06 18:32:20 -0800 | [diff] [blame] | 56 | end = start + (cycles_t) tsc_khz * timeout; |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 57 | now = start; |
| 58 | |
| 59 | for (i = 0; ; i++) { |
| 60 | /* |
| 61 | * We take the global lock, measure TSC, save the |
| 62 | * previous TSC that was measured (possibly on |
| 63 | * another CPU) and update the previous TSC timestamp. |
| 64 | */ |
Thomas Gleixner | 0199c4e | 2009-12-02 20:01:25 +0100 | [diff] [blame] | 65 | arch_spin_lock(&sync_lock); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 66 | prev = last_tsc; |
Venki Pallipadi | 93ce99e | 2008-11-17 14:43:58 -0800 | [diff] [blame] | 67 | rdtsc_barrier(); |
Andi Kleen | 6d63de8 | 2008-01-30 13:32:39 +0100 | [diff] [blame] | 68 | now = get_cycles(); |
Venki Pallipadi | 93ce99e | 2008-11-17 14:43:58 -0800 | [diff] [blame] | 69 | rdtsc_barrier(); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 70 | last_tsc = now; |
Thomas Gleixner | 0199c4e | 2009-12-02 20:01:25 +0100 | [diff] [blame] | 71 | arch_spin_unlock(&sync_lock); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 72 | |
| 73 | /* |
| 74 | * Be nice every now and then (and also check whether |
Ingo Molnar | df43510 | 2008-01-30 13:33:23 +0100 | [diff] [blame] | 75 | * measurement is done [we also insert a 10 million |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 76 | * loops safety exit, so we dont lock up in case the |
| 77 | * TSC readout is totally broken]): |
| 78 | */ |
| 79 | if (unlikely(!(i & 7))) { |
Ingo Molnar | df43510 | 2008-01-30 13:33:23 +0100 | [diff] [blame] | 80 | if (now > end || i > 10000000) |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 81 | break; |
| 82 | cpu_relax(); |
| 83 | touch_nmi_watchdog(); |
| 84 | } |
| 85 | /* |
| 86 | * Outside the critical section we can now see whether |
| 87 | * we saw a time-warp of the TSC going backwards: |
| 88 | */ |
| 89 | if (unlikely(prev > now)) { |
Thomas Gleixner | 0199c4e | 2009-12-02 20:01:25 +0100 | [diff] [blame] | 90 | arch_spin_lock(&sync_lock); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 91 | max_warp = max(max_warp, prev - now); |
| 92 | nr_warps++; |
Thomas Gleixner | 0199c4e | 2009-12-02 20:01:25 +0100 | [diff] [blame] | 93 | arch_spin_unlock(&sync_lock); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 94 | } |
Ingo Molnar | ad8ca49 | 2008-01-30 13:33:24 +0100 | [diff] [blame] | 95 | } |
Arjan van de Ven | bde78a7 | 2008-07-08 09:51:56 -0700 | [diff] [blame] | 96 | WARN(!(now-start), |
| 97 | "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n", |
Ingo Molnar | ad8ca49 | 2008-01-30 13:33:24 +0100 | [diff] [blame] | 98 | now-start, end-start); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 99 | } |
| 100 | |
| 101 | /* |
Suresh Siddha | b0e5c77 | 2012-02-06 18:32:20 -0800 | [diff] [blame] | 102 | * If the target CPU coming online doesn't have any of its core-siblings |
| 103 | * online, a timeout of 20msec will be used for the TSC-warp measurement |
| 104 | * loop. Otherwise a smaller timeout of 2msec will be used, as we have some |
| 105 | * information about this socket already (and this information grows as we |
| 106 | * have more and more logical-siblings in that socket). |
| 107 | * |
| 108 | * Ideally we should be able to skip the TSC sync check on the other |
| 109 | * core-siblings, if the first logical CPU in a socket passed the sync test. |
| 110 | * But as the TSC is per-logical CPU and can potentially be modified wrongly |
| 111 | * by the bios, TSC sync test for smaller duration should be able |
| 112 | * to catch such errors. Also this will catch the condition where all the |
| 113 | * cores in the socket doesn't get reset at the same time. |
| 114 | */ |
| 115 | static inline unsigned int loop_timeout(int cpu) |
| 116 | { |
| 117 | return (cpumask_weight(cpu_core_mask(cpu)) > 1) ? 2 : 20; |
| 118 | } |
| 119 | |
| 120 | /* |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 121 | * Source CPU calls into this - it waits for the freshly booted |
| 122 | * target CPU to arrive and then starts the measurement: |
| 123 | */ |
| 124 | void __cpuinit check_tsc_sync_source(int cpu) |
| 125 | { |
| 126 | int cpus = 2; |
| 127 | |
| 128 | /* |
| 129 | * No need to check if we already know that the TSC is not |
| 130 | * synchronized: |
| 131 | */ |
| 132 | if (unsynchronized_tsc()) |
| 133 | return; |
| 134 | |
Suresh Siddha | 28a0018 | 2011-11-04 15:42:17 -0700 | [diff] [blame] | 135 | if (tsc_clocksource_reliable) { |
Mike Travis | 9b3660a | 2009-11-17 18:22:16 -0600 | [diff] [blame] | 136 | if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING) |
| 137 | pr_info( |
| 138 | "Skipped synchronization checks as TSC is reliable.\n"); |
Alok Kataria | eca0cd0 | 2008-10-31 12:01:58 -0700 | [diff] [blame] | 139 | return; |
| 140 | } |
| 141 | |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 142 | /* |
| 143 | * Reset it - in case this is a second bootup: |
| 144 | */ |
| 145 | atomic_set(&stop_count, 0); |
| 146 | |
| 147 | /* |
| 148 | * Wait for the target to arrive: |
| 149 | */ |
| 150 | while (atomic_read(&start_count) != cpus-1) |
| 151 | cpu_relax(); |
| 152 | /* |
| 153 | * Trigger the target to continue into the measurement too: |
| 154 | */ |
| 155 | atomic_inc(&start_count); |
| 156 | |
Suresh Siddha | b0e5c77 | 2012-02-06 18:32:20 -0800 | [diff] [blame] | 157 | check_tsc_warp(loop_timeout(cpu)); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 158 | |
| 159 | while (atomic_read(&stop_count) != cpus-1) |
| 160 | cpu_relax(); |
| 161 | |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 162 | if (nr_warps) { |
Mike Travis | 9b3660a | 2009-11-17 18:22:16 -0600 | [diff] [blame] | 163 | pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n", |
| 164 | smp_processor_id(), cpu); |
Ingo Molnar | 643bec9 | 2009-05-07 09:12:50 +0200 | [diff] [blame] | 165 | pr_warning("Measured %Ld cycles TSC warp between CPUs, " |
| 166 | "turning off TSC clock.\n", max_warp); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 167 | mark_tsc_unstable("check_tsc_sync_source failed"); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 168 | } else { |
Mike Travis | 9b3660a | 2009-11-17 18:22:16 -0600 | [diff] [blame] | 169 | pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n", |
| 170 | smp_processor_id(), cpu); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 171 | } |
| 172 | |
| 173 | /* |
Mike Galbraith | 4c6b8b4 | 2008-01-30 13:30:04 +0100 | [diff] [blame] | 174 | * Reset it - just in case we boot another CPU later: |
| 175 | */ |
| 176 | atomic_set(&start_count, 0); |
| 177 | nr_warps = 0; |
| 178 | max_warp = 0; |
| 179 | last_tsc = 0; |
| 180 | |
| 181 | /* |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 182 | * Let the target continue with the bootup: |
| 183 | */ |
| 184 | atomic_inc(&stop_count); |
| 185 | } |
| 186 | |
| 187 | /* |
| 188 | * Freshly booted CPUs call into this: |
| 189 | */ |
| 190 | void __cpuinit check_tsc_sync_target(void) |
| 191 | { |
| 192 | int cpus = 2; |
| 193 | |
Suresh Siddha | 28a0018 | 2011-11-04 15:42:17 -0700 | [diff] [blame] | 194 | if (unsynchronized_tsc() || tsc_clocksource_reliable) |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 195 | return; |
| 196 | |
| 197 | /* |
| 198 | * Register this CPU's participation and wait for the |
| 199 | * source CPU to start the measurement: |
| 200 | */ |
| 201 | atomic_inc(&start_count); |
| 202 | while (atomic_read(&start_count) != cpus) |
| 203 | cpu_relax(); |
| 204 | |
Suresh Siddha | b0e5c77 | 2012-02-06 18:32:20 -0800 | [diff] [blame] | 205 | check_tsc_warp(loop_timeout(smp_processor_id())); |
Thomas Gleixner | 250c227 | 2007-10-11 11:17:24 +0200 | [diff] [blame] | 206 | |
| 207 | /* |
| 208 | * Ok, we are done: |
| 209 | */ |
| 210 | atomic_inc(&stop_count); |
| 211 | |
| 212 | /* |
| 213 | * Wait for the source CPU to print stuff: |
| 214 | */ |
| 215 | while (atomic_read(&stop_count) != cpus) |
| 216 | cpu_relax(); |
| 217 | } |