Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 2 | /* |
| 3 | * Count register synchronisation. |
| 4 | * |
Tim Anderson | eb9b514 | 2009-06-17 16:40:34 -0700 | [diff] [blame] | 5 | * All CPUs will have their count registers synchronised to the CPU0 next time |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 6 | * value. This can cause a small timewarp for CPU0. All other CPU's should |
| 7 | * not have done anything significant (but they may have had interrupts |
| 8 | * enabled briefly - prom_smp_finish() should not be responsible for enabling |
| 9 | * interrupts...) |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 10 | */ |
| 11 | |
| 12 | #include <linux/kernel.h> |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 13 | #include <linux/irqflags.h> |
Tim Anderson | eb9b514 | 2009-06-17 16:40:34 -0700 | [diff] [blame] | 14 | #include <linux/cpumask.h> |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 15 | |
Tim Anderson | eb9b514 | 2009-06-17 16:40:34 -0700 | [diff] [blame] | 16 | #include <asm/r4k-timer.h> |
Arun Sharma | 60063497 | 2011-07-26 16:09:06 -0700 | [diff] [blame] | 17 | #include <linux/atomic.h> |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 18 | #include <asm/barrier.h> |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 19 | #include <asm/mipsregs.h> |
| 20 | |
Huacai Chen | db0dbd5 | 2016-01-21 21:09:51 +0800 | [diff] [blame] | 21 | static unsigned int initcount = 0; |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 22 | static atomic_t count_count_start = ATOMIC_INIT(0); |
| 23 | static atomic_t count_count_stop = ATOMIC_INIT(0); |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 24 | |
Ralf Baechle | 7034228 | 2013-01-22 12:59:30 +0100 | [diff] [blame] | 25 | #define COUNTON 100 |
Huacai Chen | db0dbd5 | 2016-01-21 21:09:51 +0800 | [diff] [blame] | 26 | #define NR_LOOPS 3 |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 27 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 28 | void synchronise_count_master(int cpu) |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 29 | { |
| 30 | int i; |
| 31 | unsigned long flags; |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 32 | |
Matt Redfearn | 4fb69af | 2017-02-02 13:22:04 +0000 | [diff] [blame] | 33 | pr_info("Synchronize counters for CPU %u: ", cpu); |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 34 | |
| 35 | local_irq_save(flags); |
| 36 | |
| 37 | /* |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 38 | * We loop a few times to get a primed instruction cache, |
| 39 | * then the last pass is more or less synchronised and |
| 40 | * the master and slaves each set their cycle counters to a known |
| 41 | * value all at once. This reduces the chance of having random offsets |
| 42 | * between the processors, and guarantees that the maximum |
| 43 | * delay between the cycle counters is never bigger than |
| 44 | * the latency of information-passing (cachelines) between |
| 45 | * two CPUs. |
| 46 | */ |
| 47 | |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 48 | for (i = 0; i < NR_LOOPS; i++) { |
Jayachandran C | cf9bfe5 | 2012-08-14 18:56:13 +0530 | [diff] [blame] | 49 | /* slaves loop on '!= 2' */ |
| 50 | while (atomic_read(&count_count_start) != 1) |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 51 | mb(); |
| 52 | atomic_set(&count_count_stop, 0); |
| 53 | smp_wmb(); |
| 54 | |
Huacai Chen | db0dbd5 | 2016-01-21 21:09:51 +0800 | [diff] [blame] | 55 | /* Let the slave writes its count register */ |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 56 | atomic_inc(&count_count_start); |
| 57 | |
Huacai Chen | db0dbd5 | 2016-01-21 21:09:51 +0800 | [diff] [blame] | 58 | /* Count will be initialised to current timer */ |
| 59 | if (i == 1) |
| 60 | initcount = read_c0_count(); |
| 61 | |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 62 | /* |
| 63 | * Everyone initialises count in the last loop: |
| 64 | */ |
| 65 | if (i == NR_LOOPS-1) |
| 66 | write_c0_count(initcount); |
| 67 | |
| 68 | /* |
Huacai Chen | db0dbd5 | 2016-01-21 21:09:51 +0800 | [diff] [blame] | 69 | * Wait for slave to leave the synchronization point: |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 70 | */ |
Jayachandran C | cf9bfe5 | 2012-08-14 18:56:13 +0530 | [diff] [blame] | 71 | while (atomic_read(&count_count_stop) != 1) |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 72 | mb(); |
| 73 | atomic_set(&count_count_start, 0); |
| 74 | smp_wmb(); |
| 75 | atomic_inc(&count_count_stop); |
| 76 | } |
| 77 | /* Arrange for an interrupt in a short while */ |
| 78 | write_c0_compare(read_c0_count() + COUNTON); |
| 79 | |
| 80 | local_irq_restore(flags); |
| 81 | |
| 82 | /* |
| 83 | * i386 code reported the skew here, but the |
| 84 | * count registers were almost certainly out of sync |
| 85 | * so no point in alarming people |
| 86 | */ |
Matt Redfearn | 4fb69af | 2017-02-02 13:22:04 +0000 | [diff] [blame] | 87 | pr_cont("done.\n"); |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 88 | } |
| 89 | |
Paul Gortmaker | 078a55f | 2013-06-18 13:38:59 +0000 | [diff] [blame] | 90 | void synchronise_count_slave(int cpu) |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 91 | { |
| 92 | int i; |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 93 | |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 94 | /* |
| 95 | * Not every cpu is online at the time this gets called, |
| 96 | * so we first wait for the master to say everyone is ready |
| 97 | */ |
| 98 | |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 99 | for (i = 0; i < NR_LOOPS; i++) { |
| 100 | atomic_inc(&count_count_start); |
Jayachandran C | cf9bfe5 | 2012-08-14 18:56:13 +0530 | [diff] [blame] | 101 | while (atomic_read(&count_count_start) != 2) |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 102 | mb(); |
| 103 | |
| 104 | /* |
| 105 | * Everyone initialises count in the last loop: |
| 106 | */ |
| 107 | if (i == NR_LOOPS-1) |
| 108 | write_c0_count(initcount); |
| 109 | |
| 110 | atomic_inc(&count_count_stop); |
Jayachandran C | cf9bfe5 | 2012-08-14 18:56:13 +0530 | [diff] [blame] | 111 | while (atomic_read(&count_count_stop) != 2) |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 112 | mb(); |
| 113 | } |
| 114 | /* Arrange for an interrupt in a short while */ |
| 115 | write_c0_compare(read_c0_count() + COUNTON); |
Ralf Baechle | 39b8d52 | 2008-04-28 17:14:26 +0100 | [diff] [blame] | 116 | } |
| 117 | #undef NR_LOOPS |