blob: fc25e60a588458078e7d66a4ba8a7e995624b585 [file] [log] [blame]
Thomas Gleixner250c2272007-10-11 11:17:24 +02001/*
Dave Jones835c34a2007-10-12 21:10:53 -04002 * check TSC synchronization.
Thomas Gleixner250c2272007-10-11 11:17:24 +02003 *
4 * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar
5 *
6 * We check whether all boot CPUs have their TSC's synchronized,
7 * print a warning if not and turn off the TSC clock-source.
8 *
9 * The warp-check is point-to-point between two CPUs, the CPU
10 * initiating the bootup is the 'source CPU', the freshly booting
11 * CPU is the 'target CPU'.
12 *
13 * Only two CPUs may participate - they can enter in any order.
14 * ( The serial nature of the boot logic and the CPU hotplug lock
15 * protects against more than 2 CPUs entering this code. )
16 */
17#include <linux/spinlock.h>
18#include <linux/kernel.h>
19#include <linux/init.h>
20#include <linux/smp.h>
21#include <linux/nmi.h>
22#include <asm/tsc.h>
23
24/*
25 * Entry/exit counters that make sure that both CPUs
26 * run the measurement code at once:
27 */
28static __cpuinitdata atomic_t start_count;
29static __cpuinitdata atomic_t stop_count;
30
31/*
32 * We use a raw spinlock in this exceptional case, because
33 * we want to have the fastest, inlined, non-debug version
34 * of a critical section, to be able to prove TSC time-warps:
35 */
Thomas Gleixneredc35bd2009-12-03 12:38:57 +010036static __cpuinitdata arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Ingo Molnar643bec92009-05-07 09:12:50 +020037
Thomas Gleixner250c2272007-10-11 11:17:24 +020038static __cpuinitdata cycles_t last_tsc;
39static __cpuinitdata cycles_t max_warp;
40static __cpuinitdata int nr_warps;
41
42/*
43 * TSC-warp measurement loop running on both CPUs:
44 */
Suresh Siddhab0e5c772012-02-06 18:32:20 -080045static __cpuinit void check_tsc_warp(unsigned int timeout)
Thomas Gleixner250c2272007-10-11 11:17:24 +020046{
47 cycles_t start, now, prev, end;
48 int i;
49
Venki Pallipadi93ce99e2008-11-17 14:43:58 -080050 rdtsc_barrier();
Andi Kleen6d63de82008-01-30 13:32:39 +010051 start = get_cycles();
Venki Pallipadi93ce99e2008-11-17 14:43:58 -080052 rdtsc_barrier();
Thomas Gleixner250c2272007-10-11 11:17:24 +020053 /*
Suresh Siddhab0e5c772012-02-06 18:32:20 -080054 * The measurement runs for 'timeout' msecs:
Thomas Gleixner250c2272007-10-11 11:17:24 +020055 */
Suresh Siddhab0e5c772012-02-06 18:32:20 -080056 end = start + (cycles_t) tsc_khz * timeout;
Thomas Gleixner250c2272007-10-11 11:17:24 +020057 now = start;
58
59 for (i = 0; ; i++) {
60 /*
61 * We take the global lock, measure TSC, save the
62 * previous TSC that was measured (possibly on
63 * another CPU) and update the previous TSC timestamp.
64 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +010065 arch_spin_lock(&sync_lock);
Thomas Gleixner250c2272007-10-11 11:17:24 +020066 prev = last_tsc;
Venki Pallipadi93ce99e2008-11-17 14:43:58 -080067 rdtsc_barrier();
Andi Kleen6d63de82008-01-30 13:32:39 +010068 now = get_cycles();
Venki Pallipadi93ce99e2008-11-17 14:43:58 -080069 rdtsc_barrier();
Thomas Gleixner250c2272007-10-11 11:17:24 +020070 last_tsc = now;
Thomas Gleixner0199c4e2009-12-02 20:01:25 +010071 arch_spin_unlock(&sync_lock);
Thomas Gleixner250c2272007-10-11 11:17:24 +020072
73 /*
74 * Be nice every now and then (and also check whether
Ingo Molnardf435102008-01-30 13:33:23 +010075 * measurement is done [we also insert a 10 million
Thomas Gleixner250c2272007-10-11 11:17:24 +020076 * loops safety exit, so we dont lock up in case the
77 * TSC readout is totally broken]):
78 */
79 if (unlikely(!(i & 7))) {
Ingo Molnardf435102008-01-30 13:33:23 +010080 if (now > end || i > 10000000)
Thomas Gleixner250c2272007-10-11 11:17:24 +020081 break;
82 cpu_relax();
83 touch_nmi_watchdog();
84 }
85 /*
86 * Outside the critical section we can now see whether
87 * we saw a time-warp of the TSC going backwards:
88 */
89 if (unlikely(prev > now)) {
Thomas Gleixner0199c4e2009-12-02 20:01:25 +010090 arch_spin_lock(&sync_lock);
Thomas Gleixner250c2272007-10-11 11:17:24 +020091 max_warp = max(max_warp, prev - now);
92 nr_warps++;
Thomas Gleixner0199c4e2009-12-02 20:01:25 +010093 arch_spin_unlock(&sync_lock);
Thomas Gleixner250c2272007-10-11 11:17:24 +020094 }
Ingo Molnarad8ca492008-01-30 13:33:24 +010095 }
Arjan van de Venbde78a72008-07-08 09:51:56 -070096 WARN(!(now-start),
97 "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n",
Ingo Molnarad8ca492008-01-30 13:33:24 +010098 now-start, end-start);
Thomas Gleixner250c2272007-10-11 11:17:24 +020099}
100
101/*
Suresh Siddhab0e5c772012-02-06 18:32:20 -0800102 * If the target CPU coming online doesn't have any of its core-siblings
103 * online, a timeout of 20msec will be used for the TSC-warp measurement
104 * loop. Otherwise a smaller timeout of 2msec will be used, as we have some
105 * information about this socket already (and this information grows as we
106 * have more and more logical-siblings in that socket).
107 *
108 * Ideally we should be able to skip the TSC sync check on the other
109 * core-siblings, if the first logical CPU in a socket passed the sync test.
110 * But as the TSC is per-logical CPU and can potentially be modified wrongly
111 * by the bios, TSC sync test for smaller duration should be able
112 * to catch such errors. Also this will catch the condition where all the
113 * cores in the socket doesn't get reset at the same time.
114 */
115static inline unsigned int loop_timeout(int cpu)
116{
117 return (cpumask_weight(cpu_core_mask(cpu)) > 1) ? 2 : 20;
118}
119
120/*
Thomas Gleixner250c2272007-10-11 11:17:24 +0200121 * Source CPU calls into this - it waits for the freshly booted
122 * target CPU to arrive and then starts the measurement:
123 */
124void __cpuinit check_tsc_sync_source(int cpu)
125{
126 int cpus = 2;
127
128 /*
129 * No need to check if we already know that the TSC is not
130 * synchronized:
131 */
132 if (unsynchronized_tsc())
133 return;
134
Suresh Siddha28a00182011-11-04 15:42:17 -0700135 if (tsc_clocksource_reliable) {
Mike Travis9b3660a2009-11-17 18:22:16 -0600136 if (cpu == (nr_cpu_ids-1) || system_state != SYSTEM_BOOTING)
137 pr_info(
138 "Skipped synchronization checks as TSC is reliable.\n");
Alok Katariaeca0cd02008-10-31 12:01:58 -0700139 return;
140 }
141
Thomas Gleixner250c2272007-10-11 11:17:24 +0200142 /*
143 * Reset it - in case this is a second bootup:
144 */
145 atomic_set(&stop_count, 0);
146
147 /*
148 * Wait for the target to arrive:
149 */
150 while (atomic_read(&start_count) != cpus-1)
151 cpu_relax();
152 /*
153 * Trigger the target to continue into the measurement too:
154 */
155 atomic_inc(&start_count);
156
Suresh Siddhab0e5c772012-02-06 18:32:20 -0800157 check_tsc_warp(loop_timeout(cpu));
Thomas Gleixner250c2272007-10-11 11:17:24 +0200158
159 while (atomic_read(&stop_count) != cpus-1)
160 cpu_relax();
161
Thomas Gleixner250c2272007-10-11 11:17:24 +0200162 if (nr_warps) {
Mike Travis9b3660a2009-11-17 18:22:16 -0600163 pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n",
164 smp_processor_id(), cpu);
Ingo Molnar643bec92009-05-07 09:12:50 +0200165 pr_warning("Measured %Ld cycles TSC warp between CPUs, "
166 "turning off TSC clock.\n", max_warp);
Thomas Gleixner250c2272007-10-11 11:17:24 +0200167 mark_tsc_unstable("check_tsc_sync_source failed");
Thomas Gleixner250c2272007-10-11 11:17:24 +0200168 } else {
Mike Travis9b3660a2009-11-17 18:22:16 -0600169 pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n",
170 smp_processor_id(), cpu);
Thomas Gleixner250c2272007-10-11 11:17:24 +0200171 }
172
173 /*
Mike Galbraith4c6b8b42008-01-30 13:30:04 +0100174 * Reset it - just in case we boot another CPU later:
175 */
176 atomic_set(&start_count, 0);
177 nr_warps = 0;
178 max_warp = 0;
179 last_tsc = 0;
180
181 /*
Thomas Gleixner250c2272007-10-11 11:17:24 +0200182 * Let the target continue with the bootup:
183 */
184 atomic_inc(&stop_count);
185}
186
187/*
188 * Freshly booted CPUs call into this:
189 */
190void __cpuinit check_tsc_sync_target(void)
191{
192 int cpus = 2;
193
Suresh Siddha28a00182011-11-04 15:42:17 -0700194 if (unsynchronized_tsc() || tsc_clocksource_reliable)
Thomas Gleixner250c2272007-10-11 11:17:24 +0200195 return;
196
197 /*
198 * Register this CPU's participation and wait for the
199 * source CPU to start the measurement:
200 */
201 atomic_inc(&start_count);
202 while (atomic_read(&start_count) != cpus)
203 cpu_relax();
204
Suresh Siddhab0e5c772012-02-06 18:32:20 -0800205 check_tsc_warp(loop_timeout(smp_processor_id()));
Thomas Gleixner250c2272007-10-11 11:17:24 +0200206
207 /*
208 * Ok, we are done:
209 */
210 atomic_inc(&stop_count);
211
212 /*
213 * Wait for the source CPU to print stuff:
214 */
215 while (atomic_read(&stop_count) != cpus)
216 cpu_relax();
217}