john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 1 | #include <linux/kernel.h> |
| 2 | #include <linux/sched.h> |
| 3 | #include <linux/interrupt.h> |
| 4 | #include <linux/init.h> |
| 5 | #include <linux/clocksource.h> |
| 6 | #include <linux/time.h> |
| 7 | #include <linux/acpi.h> |
| 8 | #include <linux/cpufreq.h> |
Thomas Gleixner | d371698 | 2007-10-12 23:04:06 +0200 | [diff] [blame] | 9 | #include <linux/acpi_pmtmr.h> |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 10 | |
Thomas Gleixner | d371698 | 2007-10-12 23:04:06 +0200 | [diff] [blame] | 11 | #include <asm/hpet.h> |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 12 | #include <asm/timex.h> |
| 13 | |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 14 | static int notsc __initdata = 0; |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 15 | |
| 16 | unsigned int cpu_khz; /* TSC clocks / usec, not used here */ |
| 17 | EXPORT_SYMBOL(cpu_khz); |
Joerg Roedel | 6b37f5a | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 18 | unsigned int tsc_khz; |
| 19 | EXPORT_SYMBOL(tsc_khz); |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 20 | |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 21 | static unsigned int cyc2ns_scale __read_mostly; |
| 22 | |
Thomas Gleixner | 2f0798a | 2007-10-12 23:04:23 +0200 | [diff] [blame] | 23 | static inline void set_cyc2ns_scale(unsigned long khz) |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 24 | { |
| 25 | cyc2ns_scale = (NSEC_PER_MSEC << NS_SCALE) / khz; |
| 26 | } |
| 27 | |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 28 | static unsigned long long cycles_2_ns(unsigned long long cyc) |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 29 | { |
| 30 | return (cyc * cyc2ns_scale) >> NS_SCALE; |
| 31 | } |
| 32 | |
| 33 | unsigned long long sched_clock(void) |
| 34 | { |
| 35 | unsigned long a = 0; |
| 36 | |
| 37 | /* Could do CPU core sync here. Opteron can execute rdtsc speculatively, |
| 38 | * which means it is not completely exact and may not be monotonous |
| 39 | * between CPUs. But the errors should be too small to matter for |
| 40 | * scheduling purposes. |
| 41 | */ |
| 42 | |
| 43 | rdtscll(a); |
| 44 | return cycles_2_ns(a); |
| 45 | } |
| 46 | |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 47 | static int tsc_unstable; |
| 48 | |
Rusty Russell | d7e28ff | 2007-07-19 01:49:23 -0700 | [diff] [blame] | 49 | inline int check_tsc_unstable(void) |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 50 | { |
| 51 | return tsc_unstable; |
| 52 | } |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 53 | #ifdef CONFIG_CPU_FREQ |
| 54 | |
| 55 | /* Frequency scaling support. Adjust the TSC based timer when the cpu frequency |
| 56 | * changes. |
| 57 | * |
| 58 | * RED-PEN: On SMP we assume all CPUs run with the same frequency. It's |
| 59 | * not that important because current Opteron setups do not support |
| 60 | * scaling on SMP anyroads. |
| 61 | * |
| 62 | * Should fix up last_tsc too. Currently gettimeofday in the |
| 63 | * first tick after the change will be slightly wrong. |
| 64 | */ |
| 65 | |
Thomas Gleixner | 7ff9847 | 2007-07-21 17:10:13 +0200 | [diff] [blame] | 66 | static unsigned int ref_freq; |
| 67 | static unsigned long loops_per_jiffy_ref; |
| 68 | static unsigned long tsc_khz_ref; |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 69 | |
| 70 | static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, |
| 71 | void *data) |
| 72 | { |
| 73 | struct cpufreq_freqs *freq = data; |
| 74 | unsigned long *lpj, dummy; |
| 75 | |
Mike Travis | 92cb761 | 2007-10-19 20:35:04 +0200 | [diff] [blame] | 76 | if (cpu_has(&cpu_data(freq->cpu), X86_FEATURE_CONSTANT_TSC)) |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 77 | return 0; |
| 78 | |
| 79 | lpj = &dummy; |
| 80 | if (!(freq->flags & CPUFREQ_CONST_LOOPS)) |
| 81 | #ifdef CONFIG_SMP |
Mike Travis | 92cb761 | 2007-10-19 20:35:04 +0200 | [diff] [blame] | 82 | lpj = &cpu_data(freq->cpu).loops_per_jiffy; |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 83 | #else |
| 84 | lpj = &boot_cpu_data.loops_per_jiffy; |
| 85 | #endif |
| 86 | |
| 87 | if (!ref_freq) { |
| 88 | ref_freq = freq->old; |
| 89 | loops_per_jiffy_ref = *lpj; |
Joerg Roedel | 6b37f5a | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 90 | tsc_khz_ref = tsc_khz; |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 91 | } |
| 92 | if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || |
| 93 | (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) || |
| 94 | (val == CPUFREQ_RESUMECHANGE)) { |
| 95 | *lpj = |
| 96 | cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new); |
| 97 | |
Joerg Roedel | 6b37f5a | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 98 | tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new); |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 99 | if (!(freq->flags & CPUFREQ_CONST_LOOPS)) |
john stultz | 5a90cf2 | 2007-05-02 19:27:08 +0200 | [diff] [blame] | 100 | mark_tsc_unstable("cpufreq changes"); |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 101 | } |
| 102 | |
Joerg Roedel | 6b37f5a | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 103 | set_cyc2ns_scale(tsc_khz_ref); |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 104 | |
| 105 | return 0; |
| 106 | } |
| 107 | |
| 108 | static struct notifier_block time_cpufreq_notifier_block = { |
| 109 | .notifier_call = time_cpufreq_notifier |
| 110 | }; |
| 111 | |
| 112 | static int __init cpufreq_tsc(void) |
| 113 | { |
Thomas Gleixner | 7ff9847 | 2007-07-21 17:10:13 +0200 | [diff] [blame] | 114 | cpufreq_register_notifier(&time_cpufreq_notifier_block, |
| 115 | CPUFREQ_TRANSITION_NOTIFIER); |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 116 | return 0; |
| 117 | } |
| 118 | |
| 119 | core_initcall(cpufreq_tsc); |
| 120 | |
| 121 | #endif |
| 122 | |
Thomas Gleixner | d371698 | 2007-10-12 23:04:06 +0200 | [diff] [blame] | 123 | #define MAX_RETRIES 5 |
| 124 | #define SMI_TRESHOLD 50000 |
| 125 | |
| 126 | /* |
| 127 | * Read TSC and the reference counters. Take care of SMI disturbance |
| 128 | */ |
| 129 | static unsigned long __init tsc_read_refs(unsigned long *pm, |
| 130 | unsigned long *hpet) |
| 131 | { |
| 132 | unsigned long t1, t2; |
| 133 | int i; |
| 134 | |
| 135 | for (i = 0; i < MAX_RETRIES; i++) { |
| 136 | t1 = get_cycles_sync(); |
| 137 | if (hpet) |
| 138 | *hpet = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF; |
| 139 | else |
| 140 | *pm = acpi_pm_read_early(); |
| 141 | t2 = get_cycles_sync(); |
| 142 | if ((t2 - t1) < SMI_TRESHOLD) |
| 143 | return t2; |
| 144 | } |
| 145 | return ULONG_MAX; |
| 146 | } |
| 147 | |
| 148 | /** |
| 149 | * tsc_calibrate - calibrate the tsc on boot |
| 150 | */ |
| 151 | void __init tsc_calibrate(void) |
| 152 | { |
| 153 | unsigned long flags, tsc1, tsc2, tr1, tr2, pm1, pm2, hpet1, hpet2; |
| 154 | int hpet = is_hpet_enabled(); |
| 155 | |
| 156 | local_irq_save(flags); |
| 157 | |
| 158 | tsc1 = tsc_read_refs(&pm1, hpet ? &hpet1 : NULL); |
| 159 | |
| 160 | outb((inb(0x61) & ~0x02) | 0x01, 0x61); |
| 161 | |
| 162 | outb(0xb0, 0x43); |
| 163 | outb((CLOCK_TICK_RATE / (1000 / 50)) & 0xff, 0x42); |
| 164 | outb((CLOCK_TICK_RATE / (1000 / 50)) >> 8, 0x42); |
| 165 | tr1 = get_cycles_sync(); |
| 166 | while ((inb(0x61) & 0x20) == 0); |
| 167 | tr2 = get_cycles_sync(); |
| 168 | |
| 169 | tsc2 = tsc_read_refs(&pm2, hpet ? &hpet2 : NULL); |
| 170 | |
| 171 | local_irq_restore(flags); |
| 172 | |
| 173 | /* |
| 174 | * Preset the result with the raw and inaccurate PIT |
| 175 | * calibration value |
| 176 | */ |
| 177 | tsc_khz = (tr2 - tr1) / 50; |
| 178 | |
| 179 | /* hpet or pmtimer available ? */ |
| 180 | if (!hpet && !pm1 && !pm2) { |
| 181 | printk(KERN_INFO "TSC calibrated against PIT\n"); |
| 182 | return; |
| 183 | } |
| 184 | |
| 185 | /* Check, whether the sampling was disturbed by an SMI */ |
| 186 | if (tsc1 == ULONG_MAX || tsc2 == ULONG_MAX) { |
| 187 | printk(KERN_WARNING "TSC calibration disturbed by SMI, " |
| 188 | "using PIT calibration result\n"); |
| 189 | return; |
| 190 | } |
| 191 | |
| 192 | tsc2 = (tsc2 - tsc1) * 1000000L; |
| 193 | |
| 194 | if (hpet) { |
| 195 | printk(KERN_INFO "TSC calibrated against HPET\n"); |
| 196 | if (hpet2 < hpet1) |
| 197 | hpet2 += 0x100000000; |
| 198 | hpet2 -= hpet1; |
| 199 | tsc1 = (hpet2 * hpet_readl(HPET_PERIOD)) / 1000000; |
| 200 | } else { |
| 201 | printk(KERN_INFO "TSC calibrated against PM_TIMER\n"); |
| 202 | if (pm2 < pm1) |
| 203 | pm2 += ACPI_PM_OVRRUN; |
| 204 | pm2 -= pm1; |
| 205 | tsc1 = (pm2 * 1000000000) / PMTMR_TICKS_PER_SEC; |
| 206 | } |
| 207 | |
| 208 | tsc_khz = tsc2 / tsc1; |
Thomas Gleixner | 2f0798a | 2007-10-12 23:04:23 +0200 | [diff] [blame] | 209 | set_cyc2ns_scale(tsc_khz); |
Thomas Gleixner | d371698 | 2007-10-12 23:04:06 +0200 | [diff] [blame] | 210 | } |
| 211 | |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 212 | /* |
| 213 | * Make an educated guess if the TSC is trustworthy and synchronized |
| 214 | * over all CPUs. |
| 215 | */ |
| 216 | __cpuinit int unsynchronized_tsc(void) |
| 217 | { |
| 218 | if (tsc_unstable) |
| 219 | return 1; |
| 220 | |
| 221 | #ifdef CONFIG_SMP |
| 222 | if (apic_is_clustered_box()) |
| 223 | return 1; |
| 224 | #endif |
| 225 | /* Most intel systems have synchronized TSCs except for |
| 226 | multi node systems */ |
Thomas Gleixner | 7ff9847 | 2007-07-21 17:10:13 +0200 | [diff] [blame] | 227 | if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 228 | #ifdef CONFIG_ACPI |
| 229 | /* But TSC doesn't tick in C3 so don't use it there */ |
Thomas Gleixner | 7ff9847 | 2007-07-21 17:10:13 +0200 | [diff] [blame] | 230 | if (acpi_gbl_FADT.header.length > 0 && |
| 231 | acpi_gbl_FADT.C3latency < 1000) |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 232 | return 1; |
| 233 | #endif |
Thomas Gleixner | 7ff9847 | 2007-07-21 17:10:13 +0200 | [diff] [blame] | 234 | return 0; |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 235 | } |
| 236 | |
Thomas Gleixner | 7ff9847 | 2007-07-21 17:10:13 +0200 | [diff] [blame] | 237 | /* Assume multi socket systems are not synchronized */ |
| 238 | return num_present_cpus() > 1; |
john stultz | c37e7bb | 2007-02-16 01:28:19 -0800 | [diff] [blame] | 239 | } |
| 240 | |
| 241 | int __init notsc_setup(char *s) |
| 242 | { |
| 243 | notsc = 1; |
| 244 | return 1; |
| 245 | } |
| 246 | |
| 247 | __setup("notsc", notsc_setup); |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 248 | |
| 249 | |
| 250 | /* clock source code: */ |
| 251 | static cycle_t read_tsc(void) |
| 252 | { |
| 253 | cycle_t ret = (cycle_t)get_cycles_sync(); |
| 254 | return ret; |
| 255 | } |
| 256 | |
john stultz | 7460ed2 | 2007-02-16 01:28:21 -0800 | [diff] [blame] | 257 | static cycle_t __vsyscall_fn vread_tsc(void) |
| 258 | { |
| 259 | cycle_t ret = (cycle_t)get_cycles_sync(); |
| 260 | return ret; |
| 261 | } |
| 262 | |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 263 | static struct clocksource clocksource_tsc = { |
| 264 | .name = "tsc", |
| 265 | .rating = 300, |
| 266 | .read = read_tsc, |
| 267 | .mask = CLOCKSOURCE_MASK(64), |
| 268 | .shift = 22, |
| 269 | .flags = CLOCK_SOURCE_IS_CONTINUOUS | |
| 270 | CLOCK_SOURCE_MUST_VERIFY, |
john stultz | 7460ed2 | 2007-02-16 01:28:21 -0800 | [diff] [blame] | 271 | .vread = vread_tsc, |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 272 | }; |
| 273 | |
john stultz | 5a90cf2 | 2007-05-02 19:27:08 +0200 | [diff] [blame] | 274 | void mark_tsc_unstable(char *reason) |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 275 | { |
| 276 | if (!tsc_unstable) { |
| 277 | tsc_unstable = 1; |
john stultz | 5a90cf2 | 2007-05-02 19:27:08 +0200 | [diff] [blame] | 278 | printk("Marking TSC unstable due to %s\n", reason); |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 279 | /* Change only the rating, when not registered */ |
| 280 | if (clocksource_tsc.mult) |
| 281 | clocksource_change_rating(&clocksource_tsc, 0); |
| 282 | else |
| 283 | clocksource_tsc.rating = 0; |
| 284 | } |
| 285 | } |
| 286 | EXPORT_SYMBOL_GPL(mark_tsc_unstable); |
| 287 | |
john stultz | 6bb74df | 2007-03-05 00:30:50 -0800 | [diff] [blame] | 288 | void __init init_tsc_clocksource(void) |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 289 | { |
| 290 | if (!notsc) { |
Joerg Roedel | 6b37f5a | 2007-05-02 19:27:06 +0200 | [diff] [blame] | 291 | clocksource_tsc.mult = clocksource_khz2mult(tsc_khz, |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 292 | clocksource_tsc.shift); |
| 293 | if (check_tsc_unstable()) |
| 294 | clocksource_tsc.rating = 0; |
| 295 | |
john stultz | 6bb74df | 2007-03-05 00:30:50 -0800 | [diff] [blame] | 296 | clocksource_register(&clocksource_tsc); |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 297 | } |
john stultz | 1489939 | 2007-02-16 01:28:20 -0800 | [diff] [blame] | 298 | } |