blob: 8e4b3c32fcf9d95911a8f26e3171d93759cb5949 [file] [log] [blame]
Thomas Gleixner35728b82018-10-31 19:21:09 +01001// SPDX-License-Identifier: GPL-2.0
john stultz85240702007-05-08 00:27:59 -07002/*
Thomas Gleixner58c5fc22018-10-31 19:21:08 +01003 * Kernel timekeeping code and accessor functions. Based on code from
4 * timer.c, moved in commit 8524070b7982.
john stultz85240702007-05-08 00:27:59 -07005 */
John Stultzd7b42022012-09-04 15:12:07 -04006#include <linux/timekeeper_internal.h>
john stultz85240702007-05-08 00:27:59 -07007#include <linux/module.h>
8#include <linux/interrupt.h>
9#include <linux/percpu.h>
10#include <linux/init.h>
11#include <linux/mm.h>
Ingo Molnar38b8d202017-02-08 18:51:31 +010012#include <linux/nmi.h>
Alexey Dobriyand43c36d2009-10-07 17:09:06 +040013#include <linux/sched.h>
Ingo Molnar4f177222017-02-08 08:45:17 +010014#include <linux/sched/loadavg.h>
Pavel Tatashin3eca9932018-07-19 16:55:34 -040015#include <linux/sched/clock.h>
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +010016#include <linux/syscore_ops.h>
john stultz85240702007-05-08 00:27:59 -070017#include <linux/clocksource.h>
18#include <linux/jiffies.h>
19#include <linux/time.h>
Jason A. Donenfeld13669922022-04-10 16:49:50 +020020#include <linux/timex.h>
john stultz85240702007-05-08 00:27:59 -070021#include <linux/tick.h>
Martin Schwidefsky75c51582009-08-14 15:47:30 +020022#include <linux/stop_machine.h>
Marcelo Tosattie0b306f2012-11-27 23:28:59 -020023#include <linux/pvclock_gtod.h>
Gideon Israel Dsouza52f5684c2014-04-07 15:39:20 -070024#include <linux/compiler.h>
Ondrej Mosnacek2d87a062019-04-10 11:14:19 +020025#include <linux/audit.h>
john stultz85240702007-05-08 00:27:59 -070026
Thomas Gleixnereb93e4d2013-02-21 22:51:36 +000027#include "tick-internal.h"
John Stultzaa6f9c592013-03-22 11:31:29 -070028#include "ntp_internal.h"
Colin Cross5c835452013-05-21 22:32:14 -070029#include "timekeeping_internal.h"
Martin Schwidefsky155ec602009-08-14 15:47:26 +020030
David Vrabel04397fe92013-06-27 11:35:45 +010031#define TK_CLEAR_NTP (1 << 0)
32#define TK_MIRROR (1 << 1)
David Vrabel780427f2013-06-27 11:35:46 +010033#define TK_CLOCK_WAS_SET (1 << 2)
David Vrabel04397fe92013-06-27 11:35:45 +010034
Miroslav Lichvarb061c7a2018-06-04 15:34:21 +020035enum timekeeping_adv_mode {
36 /* Update timekeeper when a tick has passed */
37 TK_ADV_TICK,
38
39 /* Update timekeeper on a direct frequency change */
40 TK_ADV_FREQ
41};
42
Linus Torvaldsb923f122020-08-14 14:26:08 -070043DEFINE_RAW_SPINLOCK(timekeeper_lock);
Ahmed S. Darwish025e82b2020-07-20 17:55:23 +020044
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +000045/*
46 * The most important data for readout fits into a single 64 byte
47 * cache line.
48 */
49static struct {
Ahmed S. Darwish025e82b2020-07-20 17:55:23 +020050 seqcount_raw_spinlock_t seq;
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +000051 struct timekeeper timekeeper;
Bart Van Asschece10a5b2018-11-28 15:43:09 -080052} tk_core ____cacheline_aligned = {
Ahmed S. Darwish025e82b2020-07-20 17:55:23 +020053 .seq = SEQCNT_RAW_SPINLOCK_ZERO(tk_core.seq, &timekeeper_lock),
Bart Van Asschece10a5b2018-11-28 15:43:09 -080054};
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +000055
Thomas Gleixner48cdc132013-02-21 22:51:40 +000056static struct timekeeper shadow_timekeeper;
Martin Schwidefsky155ec602009-08-14 15:47:26 +020057
Thomas Gleixner71419b32020-08-14 12:19:34 +020058/* flag for if timekeeping is suspended */
59int __read_mostly timekeeping_suspended;
60
Thomas Gleixner4396e052014-07-16 21:05:23 +000061/**
62 * struct tk_fast - NMI safe timekeeper
63 * @seq: Sequence counter for protecting updates. The lowest bit
64 * is the index for the tk_read_base array
65 * @base: tk_read_base array. Access is indexed by the lowest bit of
66 * @seq.
67 *
68 * See @update_fast_timekeeper() below.
69 */
70struct tk_fast {
Ahmed S. Darwish249d0532020-08-27 13:40:41 +020071 seqcount_latch_t seq;
Thomas Gleixner4396e052014-07-16 21:05:23 +000072 struct tk_read_base base[2];
73};
74
Prarit Bhargava5df32102017-08-28 08:21:53 -040075/* Suspend-time cycles value for halted fast timekeeper. */
76static u64 cycles_at_suspend;
77
78static u64 dummy_clock_read(struct clocksource *cs)
79{
Thomas Gleixner71419b32020-08-14 12:19:34 +020080 if (timekeeping_suspended)
81 return cycles_at_suspend;
82 return local_clock();
Prarit Bhargava5df32102017-08-28 08:21:53 -040083}
84
85static struct clocksource dummy_clock = {
86 .read = dummy_clock_read,
87};
88
Thomas Gleixner71419b32020-08-14 12:19:34 +020089/*
90 * Boot time initialization which allows local_clock() to be utilized
91 * during early boot when clocksources are not available. local_clock()
92 * returns nanoseconds already so no conversion is required, hence mult=1
93 * and shift=0. When the first proper clocksource is installed then
94 * the fast time keepers are updated with the correct values.
95 */
96#define FAST_TK_INIT \
97 { \
98 .clock = &dummy_clock, \
99 .mask = CLOCKSOURCE_MASK(64), \
100 .mult = 1, \
101 .shift = 0, \
102 }
103
Prarit Bhargava5df32102017-08-28 08:21:53 -0400104static struct tk_fast tk_fast_mono ____cacheline_aligned = {
Ahmed S. Darwish249d0532020-08-27 13:40:41 +0200105 .seq = SEQCNT_LATCH_ZERO(tk_fast_mono.seq),
Thomas Gleixner71419b32020-08-14 12:19:34 +0200106 .base[0] = FAST_TK_INIT,
107 .base[1] = FAST_TK_INIT,
Prarit Bhargava5df32102017-08-28 08:21:53 -0400108};
109
110static struct tk_fast tk_fast_raw ____cacheline_aligned = {
Ahmed S. Darwish249d0532020-08-27 13:40:41 +0200111 .seq = SEQCNT_LATCH_ZERO(tk_fast_raw.seq),
Thomas Gleixner71419b32020-08-14 12:19:34 +0200112 .base[0] = FAST_TK_INIT,
113 .base[1] = FAST_TK_INIT,
Prarit Bhargava5df32102017-08-28 08:21:53 -0400114};
Thomas Gleixner4396e052014-07-16 21:05:23 +0000115
John Stultz1e75fa82012-07-13 01:21:53 -0400116static inline void tk_normalize_xtime(struct timekeeper *tk)
117{
Peter Zijlstra876e7882015-03-19 10:09:06 +0100118 while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) {
119 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
John Stultz1e75fa82012-07-13 01:21:53 -0400120 tk->xtime_sec++;
121 }
John Stultzfc6eead72017-05-22 17:20:20 -0700122 while (tk->tkr_raw.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_raw.shift)) {
123 tk->tkr_raw.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
124 tk->raw_sec++;
125 }
John Stultz1e75fa82012-07-13 01:21:53 -0400126}
John Stultz8fcce542011-11-14 11:46:39 -0800127
Ondrej Mosnacek985e6952018-07-13 14:06:42 +0200128static inline struct timespec64 tk_xtime(const struct timekeeper *tk)
Thomas Gleixnerc905fae2014-07-16 21:04:05 +0000129{
130 struct timespec64 ts;
131
132 ts.tv_sec = tk->xtime_sec;
Peter Zijlstra876e7882015-03-19 10:09:06 +0100133 ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
Thomas Gleixnerc905fae2014-07-16 21:04:05 +0000134 return ts;
135}
136
John Stultz7d489d12014-07-16 21:04:01 +0000137static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts)
John Stultz1e75fa82012-07-13 01:21:53 -0400138{
139 tk->xtime_sec = ts->tv_sec;
Peter Zijlstra876e7882015-03-19 10:09:06 +0100140 tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift;
John Stultz1e75fa82012-07-13 01:21:53 -0400141}
142
John Stultz7d489d12014-07-16 21:04:01 +0000143static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts)
John Stultz1e75fa82012-07-13 01:21:53 -0400144{
145 tk->xtime_sec += ts->tv_sec;
Peter Zijlstra876e7882015-03-19 10:09:06 +0100146 tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift;
John Stultz784ffcb2012-08-21 20:30:46 -0400147 tk_normalize_xtime(tk);
John Stultz1e75fa82012-07-13 01:21:53 -0400148}
John Stultz8fcce542011-11-14 11:46:39 -0800149
John Stultz7d489d12014-07-16 21:04:01 +0000150static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm)
John Stultz6d0ef902012-07-27 14:48:12 -0400151{
John Stultz7d489d12014-07-16 21:04:01 +0000152 struct timespec64 tmp;
John Stultz6d0ef902012-07-27 14:48:12 -0400153
154 /*
155 * Verify consistency of: offset_real = -wall_to_monotonic
156 * before modifying anything
157 */
John Stultz7d489d12014-07-16 21:04:01 +0000158 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec,
John Stultz6d0ef902012-07-27 14:48:12 -0400159 -tk->wall_to_monotonic.tv_nsec);
Thomas Gleixner2456e852016-12-25 11:38:40 +0100160 WARN_ON_ONCE(tk->offs_real != timespec64_to_ktime(tmp));
John Stultz6d0ef902012-07-27 14:48:12 -0400161 tk->wall_to_monotonic = wtm;
John Stultz7d489d12014-07-16 21:04:01 +0000162 set_normalized_timespec64(&tmp, -wtm.tv_sec, -wtm.tv_nsec);
163 tk->offs_real = timespec64_to_ktime(tmp);
John Stultz04005f62013-12-10 17:13:35 -0800164 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0));
John Stultz6d0ef902012-07-27 14:48:12 -0400165}
166
Thomas Gleixner47da70d2014-07-16 21:05:00 +0000167static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta)
John Stultz6d0ef902012-07-27 14:48:12 -0400168{
Thomas Gleixnera3ed0e432018-04-25 15:33:38 +0200169 tk->offs_boot = ktime_add(tk->offs_boot, delta);
Thomas Gleixnerb99328a2019-08-22 13:00:15 +0200170 /*
171 * Timespec representation for VDSO update to avoid 64bit division
172 * on every update.
173 */
174 tk->monotonic_to_boot = ktime_to_timespec64(tk->offs_boot);
John Stultz6d0ef902012-07-27 14:48:12 -0400175}
176
John Stultzceea5e32017-06-08 16:44:20 -0700177/*
178 * tk_clock_read - atomic clocksource read() helper
179 *
180 * This helper is necessary to use in the read paths because, while the
Ahmed S. Darwish025e82b2020-07-20 17:55:23 +0200181 * seqcount ensures we don't return a bad value while structures are updated,
John Stultzceea5e32017-06-08 16:44:20 -0700182 * it doesn't protect from potential crashes. There is the possibility that
183 * the tkr's clocksource may change between the read reference, and the
184 * clock reference passed to the read function. This can cause crashes if
185 * the wrong clocksource is passed to the wrong read function.
186 * This isn't necessary to use when holding the timekeeper_lock or doing
187 * a read of the fast-timekeeper tkrs (which is protected by its own locking
188 * and update logic).
189 */
Ondrej Mosnacek985e6952018-07-13 14:06:42 +0200190static inline u64 tk_clock_read(const struct tk_read_base *tkr)
John Stultzceea5e32017-06-08 16:44:20 -0700191{
192 struct clocksource *clock = READ_ONCE(tkr->clock);
193
194 return clock->read(clock);
195}
196
John Stultz3c17ad12015-03-11 21:16:32 -0700197#ifdef CONFIG_DEBUG_TIMEKEEPING
John Stultz4ca22c22015-03-11 21:16:35 -0700198#define WARNING_FREQ (HZ*300) /* 5 minute rate-limiting */
John Stultz4ca22c22015-03-11 21:16:35 -0700199
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100200static void timekeeping_check_update(struct timekeeper *tk, u64 offset)
John Stultz3c17ad12015-03-11 21:16:32 -0700201{
202
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100203 u64 max_cycles = tk->tkr_mono.clock->max_cycles;
Peter Zijlstra876e7882015-03-19 10:09:06 +0100204 const char *name = tk->tkr_mono.clock->name;
John Stultz3c17ad12015-03-11 21:16:32 -0700205
206 if (offset > max_cycles) {
John Stultza558cd02015-03-11 21:16:33 -0700207 printk_deferred("WARNING: timekeeping: Cycle offset (%lld) is larger than allowed by the '%s' clock's max_cycles value (%lld): time overflow danger\n",
John Stultz3c17ad12015-03-11 21:16:32 -0700208 offset, name, max_cycles);
John Stultza558cd02015-03-11 21:16:33 -0700209 printk_deferred(" timekeeping: Your kernel is sick, but tries to cope by capping time updates\n");
John Stultz3c17ad12015-03-11 21:16:32 -0700210 } else {
211 if (offset > (max_cycles >> 1)) {
Masanari Iidafc4fa6e2015-12-13 15:26:11 +0900212 printk_deferred("INFO: timekeeping: Cycle offset (%lld) is larger than the '%s' clock's 50%% safety margin (%lld)\n",
John Stultz3c17ad12015-03-11 21:16:32 -0700213 offset, name, max_cycles >> 1);
214 printk_deferred(" timekeeping: Your kernel is still fine, but is feeling a bit nervous\n");
215 }
216 }
John Stultz4ca22c22015-03-11 21:16:35 -0700217
John Stultz57d05a92015-05-13 16:04:47 -0700218 if (tk->underflow_seen) {
219 if (jiffies - tk->last_warning > WARNING_FREQ) {
John Stultz4ca22c22015-03-11 21:16:35 -0700220 printk_deferred("WARNING: Underflow in clocksource '%s' observed, time update ignored.\n", name);
221 printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
222 printk_deferred(" Your kernel is probably still fine.\n");
John Stultz57d05a92015-05-13 16:04:47 -0700223 tk->last_warning = jiffies;
John Stultz4ca22c22015-03-11 21:16:35 -0700224 }
John Stultz57d05a92015-05-13 16:04:47 -0700225 tk->underflow_seen = 0;
John Stultz4ca22c22015-03-11 21:16:35 -0700226 }
227
John Stultz57d05a92015-05-13 16:04:47 -0700228 if (tk->overflow_seen) {
229 if (jiffies - tk->last_warning > WARNING_FREQ) {
John Stultz4ca22c22015-03-11 21:16:35 -0700230 printk_deferred("WARNING: Overflow in clocksource '%s' observed, time update capped.\n", name);
231 printk_deferred(" Please report this, consider using a different clocksource, if possible.\n");
232 printk_deferred(" Your kernel is probably still fine.\n");
John Stultz57d05a92015-05-13 16:04:47 -0700233 tk->last_warning = jiffies;
John Stultz4ca22c22015-03-11 21:16:35 -0700234 }
John Stultz57d05a92015-05-13 16:04:47 -0700235 tk->overflow_seen = 0;
John Stultz4ca22c22015-03-11 21:16:35 -0700236 }
John Stultz3c17ad12015-03-11 21:16:32 -0700237}
John Stultza558cd02015-03-11 21:16:33 -0700238
Ondrej Mosnacek985e6952018-07-13 14:06:42 +0200239static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
John Stultza558cd02015-03-11 21:16:33 -0700240{
John Stultz57d05a92015-05-13 16:04:47 -0700241 struct timekeeper *tk = &tk_core.timekeeper;
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100242 u64 now, last, mask, max, delta;
John Stultz4ca22c22015-03-11 21:16:35 -0700243 unsigned int seq;
John Stultza558cd02015-03-11 21:16:33 -0700244
John Stultz4ca22c22015-03-11 21:16:35 -0700245 /*
Ahmed S. Darwish025e82b2020-07-20 17:55:23 +0200246 * Since we're called holding a seqcount, the data may shift
John Stultz4ca22c22015-03-11 21:16:35 -0700247 * under us while we're doing the calculation. This can cause
248 * false positives, since we'd note a problem but throw the
Ahmed S. Darwish025e82b2020-07-20 17:55:23 +0200249 * results away. So nest another seqcount here to atomically
John Stultz4ca22c22015-03-11 21:16:35 -0700250 * grab the points we are checking with.
251 */
252 do {
253 seq = read_seqcount_begin(&tk_core.seq);
John Stultzceea5e32017-06-08 16:44:20 -0700254 now = tk_clock_read(tkr);
John Stultz4ca22c22015-03-11 21:16:35 -0700255 last = tkr->cycle_last;
256 mask = tkr->mask;
257 max = tkr->clock->max_cycles;
258 } while (read_seqcount_retry(&tk_core.seq, seq));
John Stultza558cd02015-03-11 21:16:33 -0700259
John Stultz4ca22c22015-03-11 21:16:35 -0700260 delta = clocksource_delta(now, last, mask);
John Stultza558cd02015-03-11 21:16:33 -0700261
John Stultz057b87e2015-03-11 21:16:34 -0700262 /*
263 * Try to catch underflows by checking if we are seeing small
264 * mask-relative negative values.
265 */
John Stultz4ca22c22015-03-11 21:16:35 -0700266 if (unlikely((~delta & mask) < (mask >> 3))) {
John Stultz57d05a92015-05-13 16:04:47 -0700267 tk->underflow_seen = 1;
John Stultz057b87e2015-03-11 21:16:34 -0700268 delta = 0;
John Stultz4ca22c22015-03-11 21:16:35 -0700269 }
John Stultz057b87e2015-03-11 21:16:34 -0700270
John Stultza558cd02015-03-11 21:16:33 -0700271 /* Cap delta value to the max_cycles values to avoid mult overflows */
John Stultz4ca22c22015-03-11 21:16:35 -0700272 if (unlikely(delta > max)) {
John Stultz57d05a92015-05-13 16:04:47 -0700273 tk->overflow_seen = 1;
John Stultza558cd02015-03-11 21:16:33 -0700274 delta = tkr->clock->max_cycles;
John Stultz4ca22c22015-03-11 21:16:35 -0700275 }
John Stultza558cd02015-03-11 21:16:33 -0700276
277 return delta;
278}
John Stultz3c17ad12015-03-11 21:16:32 -0700279#else
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100280static inline void timekeeping_check_update(struct timekeeper *tk, u64 offset)
John Stultz3c17ad12015-03-11 21:16:32 -0700281{
282}
Ondrej Mosnacek985e6952018-07-13 14:06:42 +0200283static inline u64 timekeeping_get_delta(const struct tk_read_base *tkr)
John Stultza558cd02015-03-11 21:16:33 -0700284{
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100285 u64 cycle_now, delta;
John Stultza558cd02015-03-11 21:16:33 -0700286
287 /* read clocksource */
John Stultzceea5e32017-06-08 16:44:20 -0700288 cycle_now = tk_clock_read(tkr);
John Stultza558cd02015-03-11 21:16:33 -0700289
290 /* calculate the delta since the last update_wall_time */
291 delta = clocksource_delta(cycle_now, tkr->cycle_last, tkr->mask);
292
293 return delta;
294}
John Stultz3c17ad12015-03-11 21:16:32 -0700295#endif
296
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200297/**
Yijing Wangd26e4fe2013-11-28 16:28:55 +0800298 * tk_setup_internals - Set up internals to use clocksource clock.
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200299 *
Yijing Wangd26e4fe2013-11-28 16:28:55 +0800300 * @tk: The target timekeeper to setup.
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200301 * @clock: Pointer to clocksource.
302 *
303 * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
304 * pair and interval request.
305 *
306 * Unless you're the timekeeping code, you should not be using this!
307 */
John Stultzf726a692012-07-13 01:21:57 -0400308static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200309{
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100310 u64 interval;
Kasper Pedersena386b5a2010-10-20 15:55:15 -0700311 u64 tmp, ntpinterval;
John Stultz1e75fa82012-07-13 01:21:53 -0400312 struct clocksource *old_clock;
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200313
Christopher S. Hall2c756fe2016-02-22 03:15:23 -0800314 ++tk->cs_was_changed_seq;
Peter Zijlstra876e7882015-03-19 10:09:06 +0100315 old_clock = tk->tkr_mono.clock;
316 tk->tkr_mono.clock = clock;
Peter Zijlstra876e7882015-03-19 10:09:06 +0100317 tk->tkr_mono.mask = clock->mask;
John Stultzceea5e32017-06-08 16:44:20 -0700318 tk->tkr_mono.cycle_last = tk_clock_read(&tk->tkr_mono);
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200319
Peter Zijlstra4a4ad802015-03-19 09:28:44 +0100320 tk->tkr_raw.clock = clock;
Peter Zijlstra4a4ad802015-03-19 09:28:44 +0100321 tk->tkr_raw.mask = clock->mask;
322 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
323
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200324 /* Do the ns -> cycle conversion first, using original mult */
325 tmp = NTP_INTERVAL_LENGTH;
326 tmp <<= clock->shift;
Kasper Pedersena386b5a2010-10-20 15:55:15 -0700327 ntpinterval = tmp;
Martin Schwidefsky0a544192009-08-14 15:47:28 +0200328 tmp += clock->mult/2;
329 do_div(tmp, clock->mult);
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200330 if (tmp == 0)
331 tmp = 1;
332
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100333 interval = (u64) tmp;
John Stultzf726a692012-07-13 01:21:57 -0400334 tk->cycle_interval = interval;
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200335
336 /* Go back from cycles -> shifted ns */
Thomas Gleixnercbd99e32016-12-08 20:49:36 +0000337 tk->xtime_interval = interval * clock->mult;
John Stultzf726a692012-07-13 01:21:57 -0400338 tk->xtime_remainder = ntpinterval - tk->xtime_interval;
John Stultz3d88d56c52017-06-08 16:44:21 -0700339 tk->raw_interval = interval * clock->mult;
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200340
John Stultz1e75fa82012-07-13 01:21:53 -0400341 /* if changing clocks, convert xtime_nsec shift units */
342 if (old_clock) {
343 int shift_change = clock->shift - old_clock->shift;
John Stultzfc6eead72017-05-22 17:20:20 -0700344 if (shift_change < 0) {
Peter Zijlstra876e7882015-03-19 10:09:06 +0100345 tk->tkr_mono.xtime_nsec >>= -shift_change;
John Stultzfc6eead72017-05-22 17:20:20 -0700346 tk->tkr_raw.xtime_nsec >>= -shift_change;
347 } else {
Peter Zijlstra876e7882015-03-19 10:09:06 +0100348 tk->tkr_mono.xtime_nsec <<= shift_change;
John Stultzfc6eead72017-05-22 17:20:20 -0700349 tk->tkr_raw.xtime_nsec <<= shift_change;
350 }
John Stultz1e75fa82012-07-13 01:21:53 -0400351 }
Peter Zijlstra4a4ad802015-03-19 09:28:44 +0100352
Peter Zijlstra876e7882015-03-19 10:09:06 +0100353 tk->tkr_mono.shift = clock->shift;
Peter Zijlstra4a4ad802015-03-19 09:28:44 +0100354 tk->tkr_raw.shift = clock->shift;
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200355
John Stultzf726a692012-07-13 01:21:57 -0400356 tk->ntp_error = 0;
357 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
John Stultz375f45b2014-04-23 20:53:29 -0700358 tk->ntp_tick = ntpinterval << tk->ntp_error_shift;
Martin Schwidefsky0a544192009-08-14 15:47:28 +0200359
360 /*
361 * The timekeeper keeps its own mult values for the currently
362 * active clocksource. These value will be adjusted via NTP
363 * to counteract clock drifting.
364 */
Peter Zijlstra876e7882015-03-19 10:09:06 +0100365 tk->tkr_mono.mult = clock->mult;
Peter Zijlstra4a4ad802015-03-19 09:28:44 +0100366 tk->tkr_raw.mult = clock->mult;
John Stultzdc491592013-12-06 17:25:21 -0800367 tk->ntp_err_mult = 0;
Miroslav Lichvar78b98e32018-03-09 10:42:48 -0800368 tk->skip_second_overflow = 0;
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200369}
john stultz85240702007-05-08 00:27:59 -0700370
Martin Schwidefsky2ba2a302009-08-14 15:47:29 +0200371/* Timekeeper helper functions. */
Stephen Warren7b1f6202012-11-07 17:58:54 -0700372
Ondrej Mosnacek985e6952018-07-13 14:06:42 +0200373static inline u64 timekeeping_delta_to_ns(const struct tk_read_base *tkr, u64 delta)
Martin Schwidefsky2ba2a302009-08-14 15:47:29 +0200374{
Thomas Gleixner9c164572016-12-08 20:49:32 +0000375 u64 nsec;
Martin Schwidefsky2ba2a302009-08-14 15:47:29 +0200376
Christopher S. Hall6bd58f02016-02-22 03:15:19 -0800377 nsec = delta * tkr->mult + tkr->xtime_nsec;
378 nsec >>= tkr->shift;
John Stultzf2a5a082012-07-13 01:21:55 -0400379
Arnd Bergmann77f6c0b2020-09-24 12:30:50 +0200380 return nsec;
Martin Schwidefsky2ba2a302009-08-14 15:47:29 +0200381}
382
Ondrej Mosnacek985e6952018-07-13 14:06:42 +0200383static inline u64 timekeeping_get_ns(const struct tk_read_base *tkr)
Christopher S. Hall6bd58f02016-02-22 03:15:19 -0800384{
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100385 u64 delta;
Christopher S. Hall6bd58f02016-02-22 03:15:19 -0800386
387 delta = timekeeping_get_delta(tkr);
388 return timekeeping_delta_to_ns(tkr, delta);
389}
390
Ondrej Mosnacek985e6952018-07-13 14:06:42 +0200391static inline u64 timekeeping_cycles_to_ns(const struct tk_read_base *tkr, u64 cycles)
Christopher S. Hall6bd58f02016-02-22 03:15:19 -0800392{
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100393 u64 delta;
Christopher S. Hall6bd58f02016-02-22 03:15:19 -0800394
395 /* calculate the delta since the last update_wall_time */
396 delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
397 return timekeeping_delta_to_ns(tkr, delta);
398}
399
Thomas Gleixner4396e052014-07-16 21:05:23 +0000400/**
401 * update_fast_timekeeper - Update the fast and NMI safe monotonic timekeeper.
Rafael J. Wysockiaffe3e82015-02-11 05:01:52 +0100402 * @tkr: Timekeeping readout base from which we take the update
Alex Shie025b032020-11-13 15:24:31 +0800403 * @tkf: Pointer to NMI safe timekeeper
Thomas Gleixner4396e052014-07-16 21:05:23 +0000404 *
405 * We want to use this from any context including NMI and tracing /
406 * instrumenting the timekeeping code itself.
407 *
Peter Zijlstra6695b922015-05-27 11:09:36 +0930408 * Employ the latch technique; see @raw_write_seqcount_latch.
Thomas Gleixner4396e052014-07-16 21:05:23 +0000409 *
410 * So if a NMI hits the update of base[0] then it will use base[1]
411 * which is still consistent. In the worst case this can result is a
412 * slightly wrong timestamp (a few nanoseconds). See
413 * @ktime_get_mono_fast_ns.
414 */
Ondrej Mosnacek985e6952018-07-13 14:06:42 +0200415static void update_fast_timekeeper(const struct tk_read_base *tkr,
416 struct tk_fast *tkf)
Thomas Gleixner4396e052014-07-16 21:05:23 +0000417{
Peter Zijlstra4498e742015-03-19 09:36:19 +0100418 struct tk_read_base *base = tkf->base;
Thomas Gleixner4396e052014-07-16 21:05:23 +0000419
420 /* Force readers off to base[1] */
Peter Zijlstra4498e742015-03-19 09:36:19 +0100421 raw_write_seqcount_latch(&tkf->seq);
Thomas Gleixner4396e052014-07-16 21:05:23 +0000422
423 /* Update base[0] */
Rafael J. Wysockiaffe3e82015-02-11 05:01:52 +0100424 memcpy(base, tkr, sizeof(*base));
Thomas Gleixner4396e052014-07-16 21:05:23 +0000425
426 /* Force readers back to base[0] */
Peter Zijlstra4498e742015-03-19 09:36:19 +0100427 raw_write_seqcount_latch(&tkf->seq);
Thomas Gleixner4396e052014-07-16 21:05:23 +0000428
429 /* Update base[1] */
430 memcpy(base + 1, base, sizeof(*base));
431}
432
Thomas Gleixner90be8d62022-04-15 11:19:38 +0200433static __always_inline u64 fast_tk_get_delta_ns(struct tk_read_base *tkr)
434{
435 u64 delta, cycles = tk_clock_read(tkr);
436
437 delta = clocksource_delta(cycles, tkr->cycle_last, tkr->mask);
438 return timekeeping_delta_to_ns(tkr, delta);
439}
440
Thomas Gleixnerc1ce4062020-11-15 21:09:31 +0100441static __always_inline u64 __ktime_get_fast_ns(struct tk_fast *tkf)
442{
443 struct tk_read_base *tkr;
444 unsigned int seq;
445 u64 now;
446
447 do {
448 seq = raw_read_seqcount_latch(&tkf->seq);
449 tkr = tkf->base + (seq & 0x01);
450 now = ktime_to_ns(tkr->base);
Thomas Gleixner90be8d62022-04-15 11:19:38 +0200451 now += fast_tk_get_delta_ns(tkr);
Thomas Gleixnerc1ce4062020-11-15 21:09:31 +0100452 } while (read_seqcount_latch_retry(&tkf->seq, seq));
453
454 return now;
455}
456
Thomas Gleixner4396e052014-07-16 21:05:23 +0000457/**
458 * ktime_get_mono_fast_ns - Fast NMI safe access to clock monotonic
459 *
460 * This timestamp is not guaranteed to be monotonic across an update.
461 * The timestamp is calculated by:
462 *
463 * now = base_mono + clock_delta * slope
464 *
465 * So if the update lowers the slope, readers who are forced to the
466 * not yet updated second array are still using the old steeper slope.
467 *
468 * tmono
469 * ^
470 * | o n
471 * | o n
472 * | u
473 * | o
474 * |o
475 * |12345678---> reader order
476 *
477 * o = old slope
478 * u = update
479 * n = new slope
480 *
481 * So reader 6 will observe time going backwards versus reader 5.
482 *
Thomas Gleixnerc1ce4062020-11-15 21:09:31 +0100483 * While other CPUs are likely to be able to observe that, the only way
Thomas Gleixner4396e052014-07-16 21:05:23 +0000484 * for a CPU local observation is when an NMI hits in the middle of
485 * the update. Timestamps taken from that NMI context might be ahead
486 * of the following timestamps. Callers need to be aware of that and
487 * deal with it.
488 */
Kurt Kanzenbach2c33d772022-04-28 08:24:32 +0200489u64 notrace ktime_get_mono_fast_ns(void)
Peter Zijlstra4498e742015-03-19 09:36:19 +0100490{
491 return __ktime_get_fast_ns(&tk_fast_mono);
492}
Thomas Gleixner4396e052014-07-16 21:05:23 +0000493EXPORT_SYMBOL_GPL(ktime_get_mono_fast_ns);
494
Thomas Gleixnerc1ce4062020-11-15 21:09:31 +0100495/**
496 * ktime_get_raw_fast_ns - Fast NMI safe access to clock monotonic raw
497 *
498 * Contrary to ktime_get_mono_fast_ns() this is always correct because the
499 * conversion factor is not affected by NTP/PTP correction.
500 */
Kurt Kanzenbach2c33d772022-04-28 08:24:32 +0200501u64 notrace ktime_get_raw_fast_ns(void)
Peter Zijlstraf09cb9a2015-03-19 09:39:08 +0100502{
503 return __ktime_get_fast_ns(&tk_fast_raw);
504}
505EXPORT_SYMBOL_GPL(ktime_get_raw_fast_ns);
506
Thomas Gleixnera3ed0e432018-04-25 15:33:38 +0200507/**
508 * ktime_get_boot_fast_ns - NMI safe and fast access to boot clock.
509 *
510 * To keep it NMI safe since we're accessing from tracing, we're not using a
511 * separate timekeeper with updates to monotonic clock and boot offset
Ahmed S. Darwish025e82b2020-07-20 17:55:23 +0200512 * protected with seqcounts. This has the following minor side effects:
Thomas Gleixnera3ed0e432018-04-25 15:33:38 +0200513 *
514 * (1) Its possible that a timestamp be taken after the boot offset is updated
515 * but before the timekeeper is updated. If this happens, the new boot offset
516 * is added to the old timekeeping making the clock appear to update slightly
517 * earlier:
518 * CPU 0 CPU 1
519 * timekeeping_inject_sleeptime64()
520 * __timekeeping_inject_sleeptime(tk, delta);
521 * timestamp();
522 * timekeeping_update(tk, TK_CLEAR_NTP...);
523 *
524 * (2) On 32-bit systems, the 64-bit boot offset (tk->offs_boot) may be
525 * partially updated. Since the tk->offs_boot update is a rare event, this
526 * should be a rare occurrence which postprocessing should be able to handle.
Thomas Gleixnerc1ce4062020-11-15 21:09:31 +0100527 *
528 * The caveats vs. timestamp ordering as documented for ktime_get_fast_ns()
529 * apply as well.
Thomas Gleixnera3ed0e432018-04-25 15:33:38 +0200530 */
531u64 notrace ktime_get_boot_fast_ns(void)
532{
533 struct timekeeper *tk = &tk_core.timekeeper;
534
Thomas Gleixnereff48492022-04-15 11:19:35 +0200535 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_boot)));
Thomas Gleixnera3ed0e432018-04-25 15:33:38 +0200536}
537EXPORT_SYMBOL_GPL(ktime_get_boot_fast_ns);
538
Kurt Kanzenbach3dc6ffa2022-04-14 11:18:03 +0200539/**
540 * ktime_get_tai_fast_ns - NMI safe and fast access to tai clock.
541 *
542 * The same limitations as described for ktime_get_boot_fast_ns() apply. The
543 * mono time and the TAI offset are not read atomically which may yield wrong
544 * readouts. However, an update of the TAI offset is an rare event e.g., caused
545 * by settime or adjtimex with an offset. The user of this function has to deal
546 * with the possibility of wrong timestamps in post processing.
547 */
548u64 notrace ktime_get_tai_fast_ns(void)
549{
550 struct timekeeper *tk = &tk_core.timekeeper;
551
552 return (ktime_get_mono_fast_ns() + ktime_to_ns(data_race(tk->offs_tai)));
553}
554EXPORT_SYMBOL_GPL(ktime_get_tai_fast_ns);
555
Thomas Gleixnere2d977c2020-08-14 12:19:35 +0200556static __always_inline u64 __ktime_get_real_fast(struct tk_fast *tkf, u64 *mono)
Thomas Gleixner4c3711d2017-08-31 17:12:48 +0200557{
558 struct tk_read_base *tkr;
Thomas Gleixnere2d977c2020-08-14 12:19:35 +0200559 u64 basem, baser, delta;
Thomas Gleixner4c3711d2017-08-31 17:12:48 +0200560 unsigned int seq;
Thomas Gleixner4c3711d2017-08-31 17:12:48 +0200561
562 do {
563 seq = raw_read_seqcount_latch(&tkf->seq);
564 tkr = tkf->base + (seq & 0x01);
Thomas Gleixnere2d977c2020-08-14 12:19:35 +0200565 basem = ktime_to_ns(tkr->base);
566 baser = ktime_to_ns(tkr->base_real);
Thomas Gleixner90be8d62022-04-15 11:19:38 +0200567 delta = fast_tk_get_delta_ns(tkr);
Ahmed S. Darwish249d0532020-08-27 13:40:41 +0200568 } while (read_seqcount_latch_retry(&tkf->seq, seq));
Thomas Gleixner4c3711d2017-08-31 17:12:48 +0200569
Thomas Gleixnere2d977c2020-08-14 12:19:35 +0200570 if (mono)
571 *mono = basem + delta;
572 return baser + delta;
Thomas Gleixner4c3711d2017-08-31 17:12:48 +0200573}
574
575/**
576 * ktime_get_real_fast_ns: - NMI safe and fast access to clock realtime.
Thomas Gleixnerc1ce4062020-11-15 21:09:31 +0100577 *
578 * See ktime_get_fast_ns() for documentation of the time stamp ordering.
Thomas Gleixner4c3711d2017-08-31 17:12:48 +0200579 */
580u64 ktime_get_real_fast_ns(void)
581{
Thomas Gleixnere2d977c2020-08-14 12:19:35 +0200582 return __ktime_get_real_fast(&tk_fast_mono, NULL);
Thomas Gleixner4c3711d2017-08-31 17:12:48 +0200583}
Arnd Bergmanndf270672017-11-10 16:25:04 +0100584EXPORT_SYMBOL_GPL(ktime_get_real_fast_ns);
Thomas Gleixner4c3711d2017-08-31 17:12:48 +0200585
Rafael J. Wysocki060407a2015-02-13 14:49:02 +0100586/**
Thomas Gleixnere2d977c2020-08-14 12:19:35 +0200587 * ktime_get_fast_timestamps: - NMI safe timestamps
588 * @snapshot: Pointer to timestamp storage
589 *
590 * Stores clock monotonic, boottime and realtime timestamps.
591 *
592 * Boot time is a racy access on 32bit systems if the sleep time injection
593 * happens late during resume and not in timekeeping_resume(). That could
594 * be avoided by expanding struct tk_read_base with boot offset for 32bit
595 * and adding more overhead to the update. As this is a hard to observe
596 * once per resume event which can be filtered with reasonable effort using
597 * the accurate mono/real timestamps, it's probably not worth the trouble.
598 *
599 * Aside of that it might be possible on 32 and 64 bit to observe the
600 * following when the sleep time injection happens late:
601 *
602 * CPU 0 CPU 1
603 * timekeeping_resume()
604 * ktime_get_fast_timestamps()
605 * mono, real = __ktime_get_real_fast()
606 * inject_sleep_time()
607 * update boot offset
608 * boot = mono + bootoffset;
609 *
610 * That means that boot time already has the sleep time adjustment, but
611 * real time does not. On the next readout both are in sync again.
612 *
613 * Preventing this for 64bit is not really feasible without destroying the
614 * careful cache layout of the timekeeper because the sequence count and
615 * struct tk_read_base would then need two cache lines instead of one.
616 *
Ingo Molnar4bf07f62021-03-22 22:39:03 +0100617 * Access to the time keeper clock source is disabled across the innermost
Thomas Gleixnere2d977c2020-08-14 12:19:35 +0200618 * steps of suspend/resume. The accessors still work, but the timestamps
619 * are frozen until time keeping is resumed which happens very early.
620 *
621 * For regular suspend/resume there is no observable difference vs. sched
622 * clock, but it might affect some of the nasty low level debug printks.
623 *
Ingo Molnar4bf07f62021-03-22 22:39:03 +0100624 * OTOH, access to sched clock is not guaranteed across suspend/resume on
Thomas Gleixnere2d977c2020-08-14 12:19:35 +0200625 * all systems either so it depends on the hardware in use.
626 *
627 * If that turns out to be a real problem then this could be mitigated by
628 * using sched clock in a similar way as during early boot. But it's not as
629 * trivial as on early boot because it needs some careful protection
630 * against the clock monotonic timestamp jumping backwards on resume.
631 */
632void ktime_get_fast_timestamps(struct ktime_timestamps *snapshot)
633{
634 struct timekeeper *tk = &tk_core.timekeeper;
635
636 snapshot->real = __ktime_get_real_fast(&tk_fast_mono, &snapshot->mono);
637 snapshot->boot = snapshot->mono + ktime_to_ns(data_race(tk->offs_boot));
638}
639
640/**
Rafael J. Wysocki060407a2015-02-13 14:49:02 +0100641 * halt_fast_timekeeper - Prevent fast timekeeper from accessing clocksource.
642 * @tk: Timekeeper to snapshot.
643 *
644 * It generally is unsafe to access the clocksource after timekeeping has been
645 * suspended, so take a snapshot of the readout base of @tk and use it as the
646 * fast timekeeper's readout base while suspended. It will return the same
647 * number of cycles every time until timekeeping is resumed at which time the
648 * proper readout base for the fast timekeeper will be restored automatically.
649 */
Ondrej Mosnacek985e6952018-07-13 14:06:42 +0200650static void halt_fast_timekeeper(const struct timekeeper *tk)
Rafael J. Wysocki060407a2015-02-13 14:49:02 +0100651{
652 static struct tk_read_base tkr_dummy;
Ondrej Mosnacek985e6952018-07-13 14:06:42 +0200653 const struct tk_read_base *tkr = &tk->tkr_mono;
Rafael J. Wysocki060407a2015-02-13 14:49:02 +0100654
655 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
John Stultzceea5e32017-06-08 16:44:20 -0700656 cycles_at_suspend = tk_clock_read(tkr);
657 tkr_dummy.clock = &dummy_clock;
Thomas Gleixner4c3711d2017-08-31 17:12:48 +0200658 tkr_dummy.base_real = tkr->base + tk->offs_real;
Peter Zijlstra4498e742015-03-19 09:36:19 +0100659 update_fast_timekeeper(&tkr_dummy, &tk_fast_mono);
Peter Zijlstraf09cb9a2015-03-19 09:39:08 +0100660
661 tkr = &tk->tkr_raw;
662 memcpy(&tkr_dummy, tkr, sizeof(tkr_dummy));
John Stultzceea5e32017-06-08 16:44:20 -0700663 tkr_dummy.clock = &dummy_clock;
Peter Zijlstraf09cb9a2015-03-19 09:39:08 +0100664 update_fast_timekeeper(&tkr_dummy, &tk_fast_raw);
Rafael J. Wysocki060407a2015-02-13 14:49:02 +0100665}
666
Marcelo Tosattie0b306f2012-11-27 23:28:59 -0200667static RAW_NOTIFIER_HEAD(pvclock_gtod_chain);
668
David Vrabel780427f2013-06-27 11:35:46 +0100669static void update_pvclock_gtod(struct timekeeper *tk, bool was_set)
Marcelo Tosattie0b306f2012-11-27 23:28:59 -0200670{
David Vrabel780427f2013-06-27 11:35:46 +0100671 raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk);
Marcelo Tosattie0b306f2012-11-27 23:28:59 -0200672}
673
674/**
675 * pvclock_gtod_register_notifier - register a pvclock timedata update listener
Alex Shif27f7c32020-11-13 15:24:32 +0800676 * @nb: Pointer to the notifier block to register
Marcelo Tosattie0b306f2012-11-27 23:28:59 -0200677 */
678int pvclock_gtod_register_notifier(struct notifier_block *nb)
679{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +0000680 struct timekeeper *tk = &tk_core.timekeeper;
Marcelo Tosattie0b306f2012-11-27 23:28:59 -0200681 unsigned long flags;
682 int ret;
683
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +0000684 raw_spin_lock_irqsave(&timekeeper_lock, flags);
Marcelo Tosattie0b306f2012-11-27 23:28:59 -0200685 ret = raw_notifier_chain_register(&pvclock_gtod_chain, nb);
David Vrabel780427f2013-06-27 11:35:46 +0100686 update_pvclock_gtod(tk, true);
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +0000687 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
Marcelo Tosattie0b306f2012-11-27 23:28:59 -0200688
689 return ret;
690}
691EXPORT_SYMBOL_GPL(pvclock_gtod_register_notifier);
692
693/**
694 * pvclock_gtod_unregister_notifier - unregister a pvclock
695 * timedata update listener
Alex Shif27f7c32020-11-13 15:24:32 +0800696 * @nb: Pointer to the notifier block to unregister
Marcelo Tosattie0b306f2012-11-27 23:28:59 -0200697 */
698int pvclock_gtod_unregister_notifier(struct notifier_block *nb)
699{
Marcelo Tosattie0b306f2012-11-27 23:28:59 -0200700 unsigned long flags;
701 int ret;
702
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +0000703 raw_spin_lock_irqsave(&timekeeper_lock, flags);
Marcelo Tosattie0b306f2012-11-27 23:28:59 -0200704 ret = raw_notifier_chain_unregister(&pvclock_gtod_chain, nb);
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +0000705 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
Marcelo Tosattie0b306f2012-11-27 23:28:59 -0200706
707 return ret;
708}
709EXPORT_SYMBOL_GPL(pvclock_gtod_unregister_notifier);
710
Thomas Gleixner7c032df2014-07-16 21:04:10 +0000711/*
John Stultz833f32d2015-06-11 15:54:55 -0700712 * tk_update_leap_state - helper to update the next_leap_ktime
713 */
714static inline void tk_update_leap_state(struct timekeeper *tk)
715{
716 tk->next_leap_ktime = ntp_get_next_leap();
Thomas Gleixner2456e852016-12-25 11:38:40 +0100717 if (tk->next_leap_ktime != KTIME_MAX)
John Stultz833f32d2015-06-11 15:54:55 -0700718 /* Convert to monotonic time */
719 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real);
720}
721
722/*
Thomas Gleixner7c032df2014-07-16 21:04:10 +0000723 * Update the ktime_t based scalar nsec members of the timekeeper
724 */
725static inline void tk_update_ktime_data(struct timekeeper *tk)
726{
Heena Sirwani9e3680b2014-10-29 16:01:16 +0530727 u64 seconds;
728 u32 nsec;
Thomas Gleixner7c032df2014-07-16 21:04:10 +0000729
730 /*
731 * The xtime based monotonic readout is:
732 * nsec = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec + now();
733 * The ktime based monotonic readout is:
734 * nsec = base_mono + now();
735 * ==> base_mono = (xtime_sec + wtm_sec) * 1e9 + wtm_nsec
736 */
Heena Sirwani9e3680b2014-10-29 16:01:16 +0530737 seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec);
738 nsec = (u32) tk->wall_to_monotonic.tv_nsec;
Peter Zijlstra876e7882015-03-19 10:09:06 +0100739 tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec);
Thomas Gleixnerf519b1a2014-07-16 21:05:04 +0000740
Heena Sirwani9e3680b2014-10-29 16:01:16 +0530741 /*
742 * The sum of the nanoseconds portions of xtime and
743 * wall_to_monotonic can be greater/equal one second. Take
744 * this into account before updating tk->ktime_sec.
745 */
Peter Zijlstra876e7882015-03-19 10:09:06 +0100746 nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
Heena Sirwani9e3680b2014-10-29 16:01:16 +0530747 if (nsec >= NSEC_PER_SEC)
748 seconds++;
749 tk->ktime_sec = seconds;
John Stultzfc6eead72017-05-22 17:20:20 -0700750
751 /* Update the monotonic raw base */
John Stultz0bcdc092017-08-25 15:57:04 -0700752 tk->tkr_raw.base = ns_to_ktime(tk->raw_sec * NSEC_PER_SEC);
Thomas Gleixner7c032df2014-07-16 21:04:10 +0000753}
754
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +0000755/* must hold timekeeper_lock */
David Vrabel04397fe92013-06-27 11:35:45 +0100756static void timekeeping_update(struct timekeeper *tk, unsigned int action)
Thomas Gleixnercc062682011-11-13 23:19:49 +0000757{
David Vrabel04397fe92013-06-27 11:35:45 +0100758 if (action & TK_CLEAR_NTP) {
John Stultzf726a692012-07-13 01:21:57 -0400759 tk->ntp_error = 0;
Thomas Gleixnercc062682011-11-13 23:19:49 +0000760 ntp_clear();
761 }
Thomas Gleixner48cdc132013-02-21 22:51:40 +0000762
John Stultz833f32d2015-06-11 15:54:55 -0700763 tk_update_leap_state(tk);
Thomas Gleixner7c032df2014-07-16 21:04:10 +0000764 tk_update_ktime_data(tk);
765
Thomas Gleixner9bf24192014-09-06 12:24:49 +0200766 update_vsyscall(tk);
767 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET);
768
Thomas Gleixner4c3711d2017-08-31 17:12:48 +0200769 tk->tkr_mono.base_real = tk->tkr_mono.base + tk->offs_real;
Peter Zijlstra4498e742015-03-19 09:36:19 +0100770 update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono);
Peter Zijlstraf09cb9a2015-03-19 09:39:08 +0100771 update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw);
Thomas Gleixner868a3e92015-04-14 21:08:37 +0000772
773 if (action & TK_CLOCK_WAS_SET)
774 tk->clock_was_set_seq++;
John Stultzd1518322015-06-11 15:54:53 -0700775 /*
776 * The mirroring of the data to the shadow-timekeeper needs
777 * to happen last here to ensure we don't over-write the
778 * timekeeper structure on the next update with stale data
779 */
780 if (action & TK_MIRROR)
781 memcpy(&shadow_timekeeper, &tk_core.timekeeper,
782 sizeof(tk_core.timekeeper));
Thomas Gleixnercc062682011-11-13 23:19:49 +0000783}
784
john stultz85240702007-05-08 00:27:59 -0700785/**
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200786 * timekeeping_forward_now - update clock to the current time
Alex Shi6e5a9192020-11-13 15:24:34 +0800787 * @tk: Pointer to the timekeeper to update
john stultz85240702007-05-08 00:27:59 -0700788 *
Roman Zippel9a055112008-08-20 16:37:28 -0700789 * Forward the current clock to update its state since the last call to
790 * update_wall_time(). This is useful before significant clock changes,
791 * as it avoids having to deal with this time offset explicitly.
john stultz85240702007-05-08 00:27:59 -0700792 */
John Stultzf726a692012-07-13 01:21:57 -0400793static void timekeeping_forward_now(struct timekeeper *tk)
john stultz85240702007-05-08 00:27:59 -0700794{
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +0100795 u64 cycle_now, delta;
john stultz85240702007-05-08 00:27:59 -0700796
John Stultzceea5e32017-06-08 16:44:20 -0700797 cycle_now = tk_clock_read(&tk->tkr_mono);
Peter Zijlstra876e7882015-03-19 10:09:06 +0100798 delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
799 tk->tkr_mono.cycle_last = cycle_now;
Peter Zijlstra4a4ad802015-03-19 09:28:44 +0100800 tk->tkr_raw.cycle_last = cycle_now;
john stultz85240702007-05-08 00:27:59 -0700801
Peter Zijlstra876e7882015-03-19 10:09:06 +0100802 tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult;
John Stultzfc6eead72017-05-22 17:20:20 -0700803 tk->tkr_raw.xtime_nsec += delta * tk->tkr_raw.mult;
804
John Stultzfc6eead72017-05-22 17:20:20 -0700805 tk_normalize_xtime(tk);
john stultz85240702007-05-08 00:27:59 -0700806}
807
808/**
Arnd Bergmannedca71f2018-04-27 15:40:13 +0200809 * ktime_get_real_ts64 - Returns the time of day in a timespec64.
john stultz85240702007-05-08 00:27:59 -0700810 * @ts: pointer to the timespec to be set
811 *
Arnd Bergmannedca71f2018-04-27 15:40:13 +0200812 * Returns the time of day in a timespec64 (WARN if suspended).
john stultz85240702007-05-08 00:27:59 -0700813 */
Arnd Bergmannedca71f2018-04-27 15:40:13 +0200814void ktime_get_real_ts64(struct timespec64 *ts)
john stultz85240702007-05-08 00:27:59 -0700815{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +0000816 struct timekeeper *tk = &tk_core.timekeeper;
Rasmus Villemoese1e41b62019-03-18 20:55:56 +0100817 unsigned int seq;
Thomas Gleixneracc89612016-12-08 20:49:34 +0000818 u64 nsecs;
john stultz85240702007-05-08 00:27:59 -0700819
Arnd Bergmannedca71f2018-04-27 15:40:13 +0200820 WARN_ON(timekeeping_suspended);
821
john stultz85240702007-05-08 00:27:59 -0700822 do {
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +0000823 seq = read_seqcount_begin(&tk_core.seq);
john stultz85240702007-05-08 00:27:59 -0700824
John Stultz4e250fd2012-07-27 14:48:13 -0400825 ts->tv_sec = tk->xtime_sec;
Peter Zijlstra876e7882015-03-19 10:09:06 +0100826 nsecs = timekeeping_get_ns(&tk->tkr_mono);
john stultz85240702007-05-08 00:27:59 -0700827
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +0000828 } while (read_seqcount_retry(&tk_core.seq, seq));
john stultz85240702007-05-08 00:27:59 -0700829
John Stultzec145ba2012-09-11 19:26:03 -0400830 ts->tv_nsec = 0;
Thomas Gleixnerd6d29892014-07-16 21:04:04 +0000831 timespec64_add_ns(ts, nsecs);
Kees Cook1e817fb2012-11-19 10:26:16 -0800832}
Arnd Bergmannedca71f2018-04-27 15:40:13 +0200833EXPORT_SYMBOL(ktime_get_real_ts64);
john stultz85240702007-05-08 00:27:59 -0700834
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200835ktime_t ktime_get(void)
836{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +0000837 struct timekeeper *tk = &tk_core.timekeeper;
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200838 unsigned int seq;
Thomas Gleixnera016a5b2014-07-16 21:04:12 +0000839 ktime_t base;
Thomas Gleixneracc89612016-12-08 20:49:34 +0000840 u64 nsecs;
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200841
842 WARN_ON(timekeeping_suspended);
843
844 do {
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +0000845 seq = read_seqcount_begin(&tk_core.seq);
Peter Zijlstra876e7882015-03-19 10:09:06 +0100846 base = tk->tkr_mono.base;
847 nsecs = timekeeping_get_ns(&tk->tkr_mono);
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200848
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +0000849 } while (read_seqcount_retry(&tk_core.seq, seq));
John Stultz24e4a8c2014-07-16 21:03:53 +0000850
Thomas Gleixnera016a5b2014-07-16 21:04:12 +0000851 return ktime_add_ns(base, nsecs);
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200852}
853EXPORT_SYMBOL_GPL(ktime_get);
854
Harald Geyer6374f912015-04-07 11:12:35 +0000855u32 ktime_get_resolution_ns(void)
856{
857 struct timekeeper *tk = &tk_core.timekeeper;
858 unsigned int seq;
859 u32 nsecs;
860
861 WARN_ON(timekeeping_suspended);
862
863 do {
864 seq = read_seqcount_begin(&tk_core.seq);
865 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift;
866 } while (read_seqcount_retry(&tk_core.seq, seq));
867
868 return nsecs;
869}
870EXPORT_SYMBOL_GPL(ktime_get_resolution_ns);
871
Thomas Gleixner0077dc62014-07-16 21:04:13 +0000872static ktime_t *offsets[TK_OFFS_MAX] = {
873 [TK_OFFS_REAL] = &tk_core.timekeeper.offs_real,
Thomas Gleixnera3ed0e432018-04-25 15:33:38 +0200874 [TK_OFFS_BOOT] = &tk_core.timekeeper.offs_boot,
Thomas Gleixner0077dc62014-07-16 21:04:13 +0000875 [TK_OFFS_TAI] = &tk_core.timekeeper.offs_tai,
876};
877
878ktime_t ktime_get_with_offset(enum tk_offsets offs)
879{
880 struct timekeeper *tk = &tk_core.timekeeper;
881 unsigned int seq;
882 ktime_t base, *offset = offsets[offs];
Thomas Gleixneracc89612016-12-08 20:49:34 +0000883 u64 nsecs;
Thomas Gleixner0077dc62014-07-16 21:04:13 +0000884
885 WARN_ON(timekeeping_suspended);
886
887 do {
888 seq = read_seqcount_begin(&tk_core.seq);
Peter Zijlstra876e7882015-03-19 10:09:06 +0100889 base = ktime_add(tk->tkr_mono.base, *offset);
890 nsecs = timekeeping_get_ns(&tk->tkr_mono);
Thomas Gleixner0077dc62014-07-16 21:04:13 +0000891
892 } while (read_seqcount_retry(&tk_core.seq, seq));
893
894 return ktime_add_ns(base, nsecs);
895
896}
897EXPORT_SYMBOL_GPL(ktime_get_with_offset);
898
Arnd Bergmannb9ff6042018-04-27 15:40:15 +0200899ktime_t ktime_get_coarse_with_offset(enum tk_offsets offs)
900{
901 struct timekeeper *tk = &tk_core.timekeeper;
902 unsigned int seq;
903 ktime_t base, *offset = offsets[offs];
Thomas Gleixnere3ff9c32019-06-13 21:40:45 +0200904 u64 nsecs;
Arnd Bergmannb9ff6042018-04-27 15:40:15 +0200905
906 WARN_ON(timekeeping_suspended);
907
908 do {
909 seq = read_seqcount_begin(&tk_core.seq);
910 base = ktime_add(tk->tkr_mono.base, *offset);
Thomas Gleixnere3ff9c32019-06-13 21:40:45 +0200911 nsecs = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
Arnd Bergmannb9ff6042018-04-27 15:40:15 +0200912
913 } while (read_seqcount_retry(&tk_core.seq, seq));
914
Jason A. Donenfeld0354c1a32019-06-21 22:32:47 +0200915 return ktime_add_ns(base, nsecs);
Arnd Bergmannb9ff6042018-04-27 15:40:15 +0200916}
917EXPORT_SYMBOL_GPL(ktime_get_coarse_with_offset);
918
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200919/**
Ingo Molnar4bf07f62021-03-22 22:39:03 +0100920 * ktime_mono_to_any() - convert monotonic time to any other time
Thomas Gleixner9a6b5192014-07-16 21:04:22 +0000921 * @tmono: time to convert.
922 * @offs: which offset to use
923 */
924ktime_t ktime_mono_to_any(ktime_t tmono, enum tk_offsets offs)
925{
926 ktime_t *offset = offsets[offs];
Rasmus Villemoese1e41b62019-03-18 20:55:56 +0100927 unsigned int seq;
Thomas Gleixner9a6b5192014-07-16 21:04:22 +0000928 ktime_t tconv;
929
930 do {
931 seq = read_seqcount_begin(&tk_core.seq);
932 tconv = ktime_add(tmono, *offset);
933 } while (read_seqcount_retry(&tk_core.seq, seq));
934
935 return tconv;
936}
937EXPORT_SYMBOL_GPL(ktime_mono_to_any);
938
939/**
Thomas Gleixnerf519b1a2014-07-16 21:05:04 +0000940 * ktime_get_raw - Returns the raw monotonic time in ktime_t format
941 */
942ktime_t ktime_get_raw(void)
943{
944 struct timekeeper *tk = &tk_core.timekeeper;
945 unsigned int seq;
946 ktime_t base;
Thomas Gleixneracc89612016-12-08 20:49:34 +0000947 u64 nsecs;
Thomas Gleixnerf519b1a2014-07-16 21:05:04 +0000948
949 do {
950 seq = read_seqcount_begin(&tk_core.seq);
Peter Zijlstra4a4ad802015-03-19 09:28:44 +0100951 base = tk->tkr_raw.base;
952 nsecs = timekeeping_get_ns(&tk->tkr_raw);
Thomas Gleixnerf519b1a2014-07-16 21:05:04 +0000953
954 } while (read_seqcount_retry(&tk_core.seq, seq));
955
956 return ktime_add_ns(base, nsecs);
957}
958EXPORT_SYMBOL_GPL(ktime_get_raw);
959
960/**
Thomas Gleixnerd6d29892014-07-16 21:04:04 +0000961 * ktime_get_ts64 - get the monotonic clock in timespec64 format
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200962 * @ts: pointer to timespec variable
963 *
964 * The function calculates the monotonic clock from the realtime
965 * clock and the wall_to_monotonic offset and stores the result
John Stultz5322e4c2014-11-07 13:13:04 -0800966 * in normalized timespec64 format in the variable pointed to by @ts.
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200967 */
Thomas Gleixnerd6d29892014-07-16 21:04:04 +0000968void ktime_get_ts64(struct timespec64 *ts)
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200969{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +0000970 struct timekeeper *tk = &tk_core.timekeeper;
Thomas Gleixnerd6d29892014-07-16 21:04:04 +0000971 struct timespec64 tomono;
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200972 unsigned int seq;
Thomas Gleixneracc89612016-12-08 20:49:34 +0000973 u64 nsec;
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200974
975 WARN_ON(timekeeping_suspended);
976
977 do {
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +0000978 seq = read_seqcount_begin(&tk_core.seq);
Thomas Gleixnerd6d29892014-07-16 21:04:04 +0000979 ts->tv_sec = tk->xtime_sec;
Peter Zijlstra876e7882015-03-19 10:09:06 +0100980 nsec = timekeeping_get_ns(&tk->tkr_mono);
John Stultz4e250fd2012-07-27 14:48:13 -0400981 tomono = tk->wall_to_monotonic;
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200982
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +0000983 } while (read_seqcount_retry(&tk_core.seq, seq));
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200984
Thomas Gleixnerd6d29892014-07-16 21:04:04 +0000985 ts->tv_sec += tomono.tv_sec;
986 ts->tv_nsec = 0;
987 timespec64_add_ns(ts, nsec + tomono.tv_nsec);
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200988}
Thomas Gleixnerd6d29892014-07-16 21:04:04 +0000989EXPORT_SYMBOL_GPL(ktime_get_ts64);
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200990
Heena Sirwani9e3680b2014-10-29 16:01:16 +0530991/**
992 * ktime_get_seconds - Get the seconds portion of CLOCK_MONOTONIC
993 *
994 * Returns the seconds portion of CLOCK_MONOTONIC with a single non
995 * serialized read. tk->ktime_sec is of type 'unsigned long' so this
996 * works on both 32 and 64 bit systems. On 32 bit systems the readout
997 * covers ~136 years of uptime which should be enough to prevent
998 * premature wrap arounds.
999 */
1000time64_t ktime_get_seconds(void)
1001{
1002 struct timekeeper *tk = &tk_core.timekeeper;
1003
1004 WARN_ON(timekeeping_suspended);
1005 return tk->ktime_sec;
1006}
1007EXPORT_SYMBOL_GPL(ktime_get_seconds);
1008
Heena Sirwanidbe7aa62014-10-29 16:01:50 +05301009/**
1010 * ktime_get_real_seconds - Get the seconds portion of CLOCK_REALTIME
1011 *
Chunguang Xuaba428a2020-12-01 17:52:31 +08001012 * Returns the wall clock seconds since 1970.
Heena Sirwanidbe7aa62014-10-29 16:01:50 +05301013 *
1014 * For 64bit systems the fast access to tk->xtime_sec is preserved. On
1015 * 32bit systems the access must be protected with the sequence
1016 * counter to provide "atomic" access to the 64bit tk->xtime_sec
1017 * value.
1018 */
1019time64_t ktime_get_real_seconds(void)
1020{
1021 struct timekeeper *tk = &tk_core.timekeeper;
1022 time64_t seconds;
1023 unsigned int seq;
1024
1025 if (IS_ENABLED(CONFIG_64BIT))
1026 return tk->xtime_sec;
1027
1028 do {
1029 seq = read_seqcount_begin(&tk_core.seq);
1030 seconds = tk->xtime_sec;
1031
1032 } while (read_seqcount_retry(&tk_core.seq, seq));
1033
1034 return seconds;
1035}
1036EXPORT_SYMBOL_GPL(ktime_get_real_seconds);
1037
DengChaodee36652015-12-13 12:24:18 +08001038/**
1039 * __ktime_get_real_seconds - The same as ktime_get_real_seconds
1040 * but without the sequence counter protect. This internal function
1041 * is called just when timekeeping lock is already held.
1042 */
Thomas Gleixner865d3a92020-04-21 21:22:36 +02001043noinstr time64_t __ktime_get_real_seconds(void)
DengChaodee36652015-12-13 12:24:18 +08001044{
1045 struct timekeeper *tk = &tk_core.timekeeper;
1046
1047 return tk->xtime_sec;
1048}
1049
Christopher S. Hall9da0f492016-02-22 03:15:20 -08001050/**
1051 * ktime_get_snapshot - snapshots the realtime/monotonic raw clocks with counter
1052 * @systime_snapshot: pointer to struct receiving the system time snapshot
1053 */
1054void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
1055{
1056 struct timekeeper *tk = &tk_core.timekeeper;
Rasmus Villemoese1e41b62019-03-18 20:55:56 +01001057 unsigned int seq;
Christopher S. Hall9da0f492016-02-22 03:15:20 -08001058 ktime_t base_raw;
1059 ktime_t base_real;
Thomas Gleixneracc89612016-12-08 20:49:34 +00001060 u64 nsec_raw;
1061 u64 nsec_real;
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +01001062 u64 now;
Christopher S. Hall9da0f492016-02-22 03:15:20 -08001063
Christopher S. Hallba266212016-02-22 03:15:21 -08001064 WARN_ON_ONCE(timekeeping_suspended);
1065
Christopher S. Hall9da0f492016-02-22 03:15:20 -08001066 do {
1067 seq = read_seqcount_begin(&tk_core.seq);
John Stultzceea5e32017-06-08 16:44:20 -07001068 now = tk_clock_read(&tk->tkr_mono);
Thomas Gleixnerb2c67cb2020-12-09 14:09:27 +08001069 systime_snapshot->cs_id = tk->tkr_mono.clock->id;
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001070 systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
1071 systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
Christopher S. Hall9da0f492016-02-22 03:15:20 -08001072 base_real = ktime_add(tk->tkr_mono.base,
1073 tk_core.timekeeper.offs_real);
1074 base_raw = tk->tkr_raw.base;
1075 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono, now);
1076 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw, now);
1077 } while (read_seqcount_retry(&tk_core.seq, seq));
1078
1079 systime_snapshot->cycles = now;
1080 systime_snapshot->real = ktime_add_ns(base_real, nsec_real);
1081 systime_snapshot->raw = ktime_add_ns(base_raw, nsec_raw);
1082}
1083EXPORT_SYMBOL_GPL(ktime_get_snapshot);
DengChaodee36652015-12-13 12:24:18 +08001084
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001085/* Scale base by mult/div checking for overflow */
1086static int scale64_check_overflow(u64 mult, u64 div, u64 *base)
1087{
1088 u64 tmp, rem;
1089
1090 tmp = div64_u64_rem(*base, div, &rem);
1091
1092 if (((int)sizeof(u64)*8 - fls64(mult) < fls64(tmp)) ||
1093 ((int)sizeof(u64)*8 - fls64(mult) < fls64(rem)))
1094 return -EOVERFLOW;
1095 tmp *= mult;
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001096
Wen Yang4cbbc3a2020-01-20 18:05:23 +08001097 rem = div64_u64(rem * mult, div);
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001098 *base = tmp + rem;
1099 return 0;
1100}
1101
1102/**
1103 * adjust_historical_crosststamp - adjust crosstimestamp previous to current interval
1104 * @history: Snapshot representing start of history
1105 * @partial_history_cycles: Cycle offset into history (fractional part)
1106 * @total_history_cycles: Total history length in cycles
1107 * @discontinuity: True indicates clock was set on history period
1108 * @ts: Cross timestamp that should be adjusted using
1109 * partial/total ratio
1110 *
1111 * Helper function used by get_device_system_crosststamp() to correct the
1112 * crosstimestamp corresponding to the start of the current interval to the
1113 * system counter value (timestamp point) provided by the driver. The
1114 * total_history_* quantities are the total history starting at the provided
1115 * reference point and ending at the start of the current interval. The cycle
1116 * count between the driver timestamp point and the start of the current
1117 * interval is partial_history_cycles.
1118 */
1119static int adjust_historical_crosststamp(struct system_time_snapshot *history,
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +01001120 u64 partial_history_cycles,
1121 u64 total_history_cycles,
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001122 bool discontinuity,
1123 struct system_device_crosststamp *ts)
1124{
1125 struct timekeeper *tk = &tk_core.timekeeper;
1126 u64 corr_raw, corr_real;
1127 bool interp_forward;
1128 int ret;
1129
1130 if (total_history_cycles == 0 || partial_history_cycles == 0)
1131 return 0;
1132
1133 /* Interpolate shortest distance from beginning or end of history */
Nicholas Mc Guire5fc63f92017-03-24 20:03:35 +01001134 interp_forward = partial_history_cycles > total_history_cycles / 2;
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001135 partial_history_cycles = interp_forward ?
1136 total_history_cycles - partial_history_cycles :
1137 partial_history_cycles;
1138
1139 /*
1140 * Scale the monotonic raw time delta by:
1141 * partial_history_cycles / total_history_cycles
1142 */
1143 corr_raw = (u64)ktime_to_ns(
1144 ktime_sub(ts->sys_monoraw, history->raw));
1145 ret = scale64_check_overflow(partial_history_cycles,
1146 total_history_cycles, &corr_raw);
1147 if (ret)
1148 return ret;
1149
1150 /*
1151 * If there is a discontinuity in the history, scale monotonic raw
1152 * correction by:
1153 * mult(real)/mult(raw) yielding the realtime correction
1154 * Otherwise, calculate the realtime correction similar to monotonic
1155 * raw calculation
1156 */
1157 if (discontinuity) {
1158 corr_real = mul_u64_u32_div
1159 (corr_raw, tk->tkr_mono.mult, tk->tkr_raw.mult);
1160 } else {
1161 corr_real = (u64)ktime_to_ns(
1162 ktime_sub(ts->sys_realtime, history->real));
1163 ret = scale64_check_overflow(partial_history_cycles,
1164 total_history_cycles, &corr_real);
1165 if (ret)
1166 return ret;
1167 }
1168
1169 /* Fixup monotonic raw and real time time values */
1170 if (interp_forward) {
1171 ts->sys_monoraw = ktime_add_ns(history->raw, corr_raw);
1172 ts->sys_realtime = ktime_add_ns(history->real, corr_real);
1173 } else {
1174 ts->sys_monoraw = ktime_sub_ns(ts->sys_monoraw, corr_raw);
1175 ts->sys_realtime = ktime_sub_ns(ts->sys_realtime, corr_real);
1176 }
1177
1178 return 0;
1179}
1180
1181/*
1182 * cycle_between - true if test occurs chronologically between before and after
1183 */
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +01001184static bool cycle_between(u64 before, u64 test, u64 after)
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001185{
1186 if (test > before && test < after)
1187 return true;
1188 if (test < before && before > after)
1189 return true;
1190 return false;
1191}
1192
john stultz85240702007-05-08 00:27:59 -07001193/**
Christopher S. Hall8006c242016-02-22 03:15:22 -08001194 * get_device_system_crosststamp - Synchronously capture system/device timestamp
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001195 * @get_time_fn: Callback to get simultaneous device time and
Christopher S. Hall8006c242016-02-22 03:15:22 -08001196 * system counter from the device driver
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001197 * @ctx: Context passed to get_time_fn()
1198 * @history_begin: Historical reference point used to interpolate system
1199 * time when counter provided by the driver is before the current interval
Christopher S. Hall8006c242016-02-22 03:15:22 -08001200 * @xtstamp: Receives simultaneously captured system and device time
1201 *
1202 * Reads a timestamp from a device and correlates it to system time
1203 */
1204int get_device_system_crosststamp(int (*get_time_fn)
1205 (ktime_t *device_time,
1206 struct system_counterval_t *sys_counterval,
1207 void *ctx),
1208 void *ctx,
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001209 struct system_time_snapshot *history_begin,
Christopher S. Hall8006c242016-02-22 03:15:22 -08001210 struct system_device_crosststamp *xtstamp)
1211{
1212 struct system_counterval_t system_counterval;
1213 struct timekeeper *tk = &tk_core.timekeeper;
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +01001214 u64 cycles, now, interval_start;
Ingo Molnar64362572016-03-08 11:09:53 +01001215 unsigned int clock_was_set_seq = 0;
Christopher S. Hall8006c242016-02-22 03:15:22 -08001216 ktime_t base_real, base_raw;
Thomas Gleixneracc89612016-12-08 20:49:34 +00001217 u64 nsec_real, nsec_raw;
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001218 u8 cs_was_changed_seq;
Rasmus Villemoese1e41b62019-03-18 20:55:56 +01001219 unsigned int seq;
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001220 bool do_interp;
Christopher S. Hall8006c242016-02-22 03:15:22 -08001221 int ret;
1222
1223 do {
1224 seq = read_seqcount_begin(&tk_core.seq);
1225 /*
1226 * Try to synchronously capture device time and a system
1227 * counter value calling back into the device driver
1228 */
1229 ret = get_time_fn(&xtstamp->device, &system_counterval, ctx);
1230 if (ret)
1231 return ret;
1232
1233 /*
1234 * Verify that the clocksource associated with the captured
1235 * system counter value is the same as the currently installed
1236 * timekeeper clocksource
1237 */
1238 if (tk->tkr_mono.clock != system_counterval.cs)
1239 return -ENODEV;
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001240 cycles = system_counterval.cycles;
1241
1242 /*
1243 * Check whether the system counter value provided by the
1244 * device driver is on the current timekeeping interval.
1245 */
John Stultzceea5e32017-06-08 16:44:20 -07001246 now = tk_clock_read(&tk->tkr_mono);
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001247 interval_start = tk->tkr_mono.cycle_last;
1248 if (!cycle_between(interval_start, cycles, now)) {
1249 clock_was_set_seq = tk->clock_was_set_seq;
1250 cs_was_changed_seq = tk->cs_was_changed_seq;
1251 cycles = interval_start;
1252 do_interp = true;
1253 } else {
1254 do_interp = false;
1255 }
Christopher S. Hall8006c242016-02-22 03:15:22 -08001256
1257 base_real = ktime_add(tk->tkr_mono.base,
1258 tk_core.timekeeper.offs_real);
1259 base_raw = tk->tkr_raw.base;
1260
1261 nsec_real = timekeeping_cycles_to_ns(&tk->tkr_mono,
1262 system_counterval.cycles);
1263 nsec_raw = timekeeping_cycles_to_ns(&tk->tkr_raw,
1264 system_counterval.cycles);
1265 } while (read_seqcount_retry(&tk_core.seq, seq));
1266
1267 xtstamp->sys_realtime = ktime_add_ns(base_real, nsec_real);
1268 xtstamp->sys_monoraw = ktime_add_ns(base_raw, nsec_raw);
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001269
1270 /*
1271 * Interpolate if necessary, adjusting back from the start of the
1272 * current interval
1273 */
1274 if (do_interp) {
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +01001275 u64 partial_history_cycles, total_history_cycles;
Christopher S. Hall2c756fe2016-02-22 03:15:23 -08001276 bool discontinuity;
1277
1278 /*
1279 * Check that the counter value occurs after the provided
1280 * history reference and that the history doesn't cross a
1281 * clocksource change
1282 */
1283 if (!history_begin ||
1284 !cycle_between(history_begin->cycles,
1285 system_counterval.cycles, cycles) ||
1286 history_begin->cs_was_changed_seq != cs_was_changed_seq)
1287 return -EINVAL;
1288 partial_history_cycles = cycles - system_counterval.cycles;
1289 total_history_cycles = cycles - history_begin->cycles;
1290 discontinuity =
1291 history_begin->clock_was_set_seq != clock_was_set_seq;
1292
1293 ret = adjust_historical_crosststamp(history_begin,
1294 partial_history_cycles,
1295 total_history_cycles,
1296 discontinuity, xtstamp);
1297 if (ret)
1298 return ret;
1299 }
1300
Christopher S. Hall8006c242016-02-22 03:15:22 -08001301 return 0;
1302}
1303EXPORT_SYMBOL_GPL(get_device_system_crosststamp);
1304
1305/**
pang.xunlei21f7eca2014-11-18 19:15:16 +08001306 * do_settimeofday64 - Sets the time of day.
1307 * @ts: pointer to the timespec64 variable containing the new time
john stultz85240702007-05-08 00:27:59 -07001308 *
1309 * Sets the time of day to the new time and update NTP and notify hrtimers
1310 */
pang.xunlei21f7eca2014-11-18 19:15:16 +08001311int do_settimeofday64(const struct timespec64 *ts)
john stultz85240702007-05-08 00:27:59 -07001312{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001313 struct timekeeper *tk = &tk_core.timekeeper;
pang.xunlei21f7eca2014-11-18 19:15:16 +08001314 struct timespec64 ts_delta, xt;
John Stultz92c1d3e2011-11-14 14:05:44 -08001315 unsigned long flags;
Wang YanQinge1d7ba82015-06-23 18:38:54 +08001316 int ret = 0;
john stultz85240702007-05-08 00:27:59 -07001317
Thomas Gleixner7a8e61f2019-03-23 11:36:19 +01001318 if (!timespec64_valid_settod(ts))
john stultz85240702007-05-08 00:27:59 -07001319 return -EINVAL;
1320
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001321 raw_spin_lock_irqsave(&timekeeper_lock, flags);
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001322 write_seqcount_begin(&tk_core.seq);
john stultz85240702007-05-08 00:27:59 -07001323
John Stultz4e250fd2012-07-27 14:48:13 -04001324 timekeeping_forward_now(tk);
john stultz85240702007-05-08 00:27:59 -07001325
John Stultz4e250fd2012-07-27 14:48:13 -04001326 xt = tk_xtime(tk);
Yu Liao4e8c11b62021-12-13 21:57:27 +08001327 ts_delta = timespec64_sub(*ts, xt);
John Stultz1e75fa82012-07-13 01:21:53 -04001328
Wang YanQinge1d7ba82015-06-23 18:38:54 +08001329 if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) {
1330 ret = -EINVAL;
1331 goto out;
1332 }
1333
John Stultz7d489d12014-07-16 21:04:01 +00001334 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta));
john stultz85240702007-05-08 00:27:59 -07001335
pang.xunlei21f7eca2014-11-18 19:15:16 +08001336 tk_set_xtime(tk, ts);
Wang YanQinge1d7ba82015-06-23 18:38:54 +08001337out:
David Vrabel780427f2013-06-27 11:35:46 +01001338 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
john stultz85240702007-05-08 00:27:59 -07001339
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001340 write_seqcount_end(&tk_core.seq);
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001341 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
john stultz85240702007-05-08 00:27:59 -07001342
Thomas Gleixner17a1b882021-07-13 15:39:53 +02001343 /* Signal hrtimers about time change */
1344 clock_was_set(CLOCK_SET_WALL);
john stultz85240702007-05-08 00:27:59 -07001345
Ondrej Mosnacek2d87a062019-04-10 11:14:19 +02001346 if (!ret)
1347 audit_tk_injoffset(ts_delta);
1348
Wang YanQinge1d7ba82015-06-23 18:38:54 +08001349 return ret;
john stultz85240702007-05-08 00:27:59 -07001350}
pang.xunlei21f7eca2014-11-18 19:15:16 +08001351EXPORT_SYMBOL(do_settimeofday64);
john stultz85240702007-05-08 00:27:59 -07001352
John Stultzc528f7c2011-02-01 13:52:17 +00001353/**
1354 * timekeeping_inject_offset - Adds or subtracts from the current time.
Alex Shi6e5a9192020-11-13 15:24:34 +08001355 * @ts: Pointer to the timespec variable containing the offset
John Stultzc528f7c2011-02-01 13:52:17 +00001356 *
1357 * Adds or subtracts an offset value from the current time.
1358 */
Ondrej Mosnacek985e6952018-07-13 14:06:42 +02001359static int timekeeping_inject_offset(const struct timespec64 *ts)
John Stultzc528f7c2011-02-01 13:52:17 +00001360{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001361 struct timekeeper *tk = &tk_core.timekeeper;
John Stultz92c1d3e2011-11-14 14:05:44 -08001362 unsigned long flags;
Arnd Bergmann1572fa02017-10-19 13:14:45 +02001363 struct timespec64 tmp;
John Stultz4e8b1452012-08-08 15:36:20 -04001364 int ret = 0;
John Stultzc528f7c2011-02-01 13:52:17 +00001365
Arnd Bergmann1572fa02017-10-19 13:14:45 +02001366 if (ts->tv_nsec < 0 || ts->tv_nsec >= NSEC_PER_SEC)
John Stultzc528f7c2011-02-01 13:52:17 +00001367 return -EINVAL;
1368
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001369 raw_spin_lock_irqsave(&timekeeper_lock, flags);
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001370 write_seqcount_begin(&tk_core.seq);
John Stultzc528f7c2011-02-01 13:52:17 +00001371
John Stultz4e250fd2012-07-27 14:48:13 -04001372 timekeeping_forward_now(tk);
John Stultzc528f7c2011-02-01 13:52:17 +00001373
John Stultz4e8b1452012-08-08 15:36:20 -04001374 /* Make sure the proposed value is valid */
Arnd Bergmann1572fa02017-10-19 13:14:45 +02001375 tmp = timespec64_add(tk_xtime(tk), *ts);
1376 if (timespec64_compare(&tk->wall_to_monotonic, ts) > 0 ||
Thomas Gleixner7a8e61f2019-03-23 11:36:19 +01001377 !timespec64_valid_settod(&tmp)) {
John Stultz4e8b1452012-08-08 15:36:20 -04001378 ret = -EINVAL;
1379 goto error;
1380 }
John Stultz1e75fa82012-07-13 01:21:53 -04001381
Arnd Bergmann1572fa02017-10-19 13:14:45 +02001382 tk_xtime_add(tk, ts);
1383 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *ts));
John Stultzc528f7c2011-02-01 13:52:17 +00001384
John Stultz4e8b1452012-08-08 15:36:20 -04001385error: /* even if we error out, we forwarded the time, so call update */
David Vrabel780427f2013-06-27 11:35:46 +01001386 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
John Stultzc528f7c2011-02-01 13:52:17 +00001387
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001388 write_seqcount_end(&tk_core.seq);
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001389 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
John Stultzc528f7c2011-02-01 13:52:17 +00001390
Thomas Gleixner17a1b882021-07-13 15:39:53 +02001391 /* Signal hrtimers about time change */
1392 clock_was_set(CLOCK_SET_WALL);
John Stultzc528f7c2011-02-01 13:52:17 +00001393
John Stultz4e8b1452012-08-08 15:36:20 -04001394 return ret;
John Stultzc528f7c2011-02-01 13:52:17 +00001395}
Arnd Bergmanne0956dc2017-10-19 13:14:44 +02001396
1397/*
1398 * Indicates if there is an offset between the system clock and the hardware
1399 * clock/persistent clock/rtc.
1400 */
1401int persistent_clock_is_local;
1402
1403/*
1404 * Adjust the time obtained from the CMOS to be UTC time instead of
1405 * local time.
1406 *
1407 * This is ugly, but preferable to the alternatives. Otherwise we
1408 * would either need to write a program to do it in /etc/rc (and risk
1409 * confusion if the program gets run more than once; it would also be
1410 * hard to make the program warp the clock precisely n hours) or
1411 * compile in the timezone information into the kernel. Bad, bad....
1412 *
1413 * - TYT, 1992-01-01
1414 *
1415 * The best thing to do is to keep the CMOS clock in universal time (UTC)
1416 * as real UNIX machines always do it. This avoids all headaches about
1417 * daylight saving times and warping kernel clocks.
1418 */
1419void timekeeping_warp_clock(void)
1420{
1421 if (sys_tz.tz_minuteswest != 0) {
Arnd Bergmann1572fa02017-10-19 13:14:45 +02001422 struct timespec64 adjust;
Arnd Bergmanne0956dc2017-10-19 13:14:44 +02001423
1424 persistent_clock_is_local = 1;
1425 adjust.tv_sec = sys_tz.tz_minuteswest * 60;
1426 adjust.tv_nsec = 0;
1427 timekeeping_inject_offset(&adjust);
1428 }
1429}
John Stultzc528f7c2011-02-01 13:52:17 +00001430
Alex Shi199d2802020-11-13 15:24:33 +08001431/*
Stephen Boyd40d9f822016-12-07 14:33:23 -08001432 * __timekeeping_set_tai_offset - Sets the TAI offset from UTC and monotonic
John Stultzcc244dda2012-05-03 12:30:07 -07001433 */
Fengguang Wudd5d70e82013-03-25 12:24:24 -07001434static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset)
John Stultzcc244dda2012-05-03 12:30:07 -07001435{
1436 tk->tai_offset = tai_offset;
John Stultz04005f62013-12-10 17:13:35 -08001437 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0));
John Stultzcc244dda2012-05-03 12:30:07 -07001438}
1439
Alex Shi199d2802020-11-13 15:24:33 +08001440/*
john stultz85240702007-05-08 00:27:59 -07001441 * change_clocksource - Swaps clocksources if a new one is available
1442 *
1443 * Accumulates current time interval and initializes new clocksource
1444 */
Martin Schwidefsky75c51582009-08-14 15:47:30 +02001445static int change_clocksource(void *data)
john stultz85240702007-05-08 00:27:59 -07001446{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001447 struct timekeeper *tk = &tk_core.timekeeper;
Niklas Söderlundd4c7c282021-02-11 14:43:18 +01001448 struct clocksource *new, *old = NULL;
John Stultzf695cf92012-03-14 16:38:15 -07001449 unsigned long flags;
Niklas Söderlundd4c7c282021-02-11 14:43:18 +01001450 bool change = false;
john stultz85240702007-05-08 00:27:59 -07001451
Martin Schwidefsky75c51582009-08-14 15:47:30 +02001452 new = (struct clocksource *) data;
john stultz85240702007-05-08 00:27:59 -07001453
Thomas Gleixner09ac3692013-04-25 20:31:44 +00001454 /*
1455 * If the cs is in module, get a module reference. Succeeds
1456 * for built-in code (owner == NULL) as well.
1457 */
1458 if (try_module_get(new->owner)) {
Niklas Söderlundd4c7c282021-02-11 14:43:18 +01001459 if (!new->enable || new->enable(new) == 0)
1460 change = true;
1461 else
Thomas Gleixner09ac3692013-04-25 20:31:44 +00001462 module_put(new->owner);
Martin Schwidefsky75c51582009-08-14 15:47:30 +02001463 }
Niklas Söderlundd4c7c282021-02-11 14:43:18 +01001464
1465 raw_spin_lock_irqsave(&timekeeper_lock, flags);
1466 write_seqcount_begin(&tk_core.seq);
1467
1468 timekeeping_forward_now(tk);
1469
1470 if (change) {
1471 old = tk->tkr_mono.clock;
1472 tk_setup_internals(tk, new);
1473 }
1474
David Vrabel780427f2013-06-27 11:35:46 +01001475 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
John Stultzf695cf92012-03-14 16:38:15 -07001476
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001477 write_seqcount_end(&tk_core.seq);
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001478 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
John Stultzf695cf92012-03-14 16:38:15 -07001479
Niklas Söderlundd4c7c282021-02-11 14:43:18 +01001480 if (old) {
1481 if (old->disable)
1482 old->disable(old);
1483
1484 module_put(old->owner);
1485 }
1486
Martin Schwidefsky75c51582009-08-14 15:47:30 +02001487 return 0;
1488}
john stultz85240702007-05-08 00:27:59 -07001489
Martin Schwidefsky75c51582009-08-14 15:47:30 +02001490/**
1491 * timekeeping_notify - Install a new clock source
1492 * @clock: pointer to the clock source
1493 *
1494 * This function is called from clocksource.c after a new, better clock
1495 * source has been registered. The caller holds the clocksource_mutex.
1496 */
Thomas Gleixnerba919d12013-04-25 20:31:44 +00001497int timekeeping_notify(struct clocksource *clock)
Martin Schwidefsky75c51582009-08-14 15:47:30 +02001498{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001499 struct timekeeper *tk = &tk_core.timekeeper;
John Stultz4e250fd2012-07-27 14:48:13 -04001500
Peter Zijlstra876e7882015-03-19 10:09:06 +01001501 if (tk->tkr_mono.clock == clock)
Thomas Gleixnerba919d12013-04-25 20:31:44 +00001502 return 0;
Martin Schwidefsky75c51582009-08-14 15:47:30 +02001503 stop_machine(change_clocksource, clock, NULL);
john stultz85240702007-05-08 00:27:59 -07001504 tick_clock_notify();
Peter Zijlstra876e7882015-03-19 10:09:06 +01001505 return tk->tkr_mono.clock == clock ? 0 : -1;
john stultz85240702007-05-08 00:27:59 -07001506}
Martin Schwidefsky75c51582009-08-14 15:47:30 +02001507
Thomas Gleixnera40f2622009-07-07 13:00:31 +02001508/**
Arnd Bergmannfb7fcc92018-04-27 15:40:14 +02001509 * ktime_get_raw_ts64 - Returns the raw monotonic time in a timespec
John Stultzcdba2ec2014-11-07 11:03:20 -08001510 * @ts: pointer to the timespec64 to be set
John Stultz2d422442008-08-20 16:37:30 -07001511 *
1512 * Returns the raw monotonic time (completely un-modified by ntp)
1513 */
Arnd Bergmannfb7fcc92018-04-27 15:40:14 +02001514void ktime_get_raw_ts64(struct timespec64 *ts)
John Stultz2d422442008-08-20 16:37:30 -07001515{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001516 struct timekeeper *tk = &tk_core.timekeeper;
Rasmus Villemoese1e41b62019-03-18 20:55:56 +01001517 unsigned int seq;
Thomas Gleixneracc89612016-12-08 20:49:34 +00001518 u64 nsecs;
John Stultz2d422442008-08-20 16:37:30 -07001519
1520 do {
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001521 seq = read_seqcount_begin(&tk_core.seq);
John Stultzfc6eead72017-05-22 17:20:20 -07001522 ts->tv_sec = tk->raw_sec;
Peter Zijlstra4a4ad802015-03-19 09:28:44 +01001523 nsecs = timekeeping_get_ns(&tk->tkr_raw);
John Stultz2d422442008-08-20 16:37:30 -07001524
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001525 } while (read_seqcount_retry(&tk_core.seq, seq));
John Stultz2d422442008-08-20 16:37:30 -07001526
John Stultzfc6eead72017-05-22 17:20:20 -07001527 ts->tv_nsec = 0;
1528 timespec64_add_ns(ts, nsecs);
John Stultz2d422442008-08-20 16:37:30 -07001529}
Arnd Bergmannfb7fcc92018-04-27 15:40:14 +02001530EXPORT_SYMBOL(ktime_get_raw_ts64);
John Stultzcdba2ec2014-11-07 11:03:20 -08001531
John Stultz2d422442008-08-20 16:37:30 -07001532
John Stultz2d422442008-08-20 16:37:30 -07001533/**
Li Zefancf4fc6c2008-02-08 04:19:24 -08001534 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
john stultz85240702007-05-08 00:27:59 -07001535 */
Li Zefancf4fc6c2008-02-08 04:19:24 -08001536int timekeeping_valid_for_hres(void)
john stultz85240702007-05-08 00:27:59 -07001537{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001538 struct timekeeper *tk = &tk_core.timekeeper;
Rasmus Villemoese1e41b62019-03-18 20:55:56 +01001539 unsigned int seq;
john stultz85240702007-05-08 00:27:59 -07001540 int ret;
1541
1542 do {
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001543 seq = read_seqcount_begin(&tk_core.seq);
john stultz85240702007-05-08 00:27:59 -07001544
Peter Zijlstra876e7882015-03-19 10:09:06 +01001545 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
john stultz85240702007-05-08 00:27:59 -07001546
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001547 } while (read_seqcount_retry(&tk_core.seq, seq));
john stultz85240702007-05-08 00:27:59 -07001548
1549 return ret;
1550}
1551
1552/**
Jon Hunter98962462009-08-18 12:45:10 -05001553 * timekeeping_max_deferment - Returns max time the clocksource can be deferred
Jon Hunter98962462009-08-18 12:45:10 -05001554 */
1555u64 timekeeping_max_deferment(void)
1556{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001557 struct timekeeper *tk = &tk_core.timekeeper;
Rasmus Villemoese1e41b62019-03-18 20:55:56 +01001558 unsigned int seq;
John Stultz70471f22011-11-14 12:48:10 -08001559 u64 ret;
John Stultz42e71e82012-07-13 01:21:51 -04001560
John Stultz70471f22011-11-14 12:48:10 -08001561 do {
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001562 seq = read_seqcount_begin(&tk_core.seq);
John Stultz70471f22011-11-14 12:48:10 -08001563
Peter Zijlstra876e7882015-03-19 10:09:06 +01001564 ret = tk->tkr_mono.clock->max_idle_ns;
John Stultz70471f22011-11-14 12:48:10 -08001565
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001566 } while (read_seqcount_retry(&tk_core.seq, seq));
John Stultz70471f22011-11-14 12:48:10 -08001567
1568 return ret;
Jon Hunter98962462009-08-18 12:45:10 -05001569}
1570
1571/**
Arnd Bergmann92661782018-08-14 14:15:23 +02001572 * read_persistent_clock64 - Return time from the persistent clock.
Alex Shi6e5a9192020-11-13 15:24:34 +08001573 * @ts: Pointer to the storage for the readout value
john stultz85240702007-05-08 00:27:59 -07001574 *
1575 * Weak dummy function for arches that do not yet support it.
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +02001576 * Reads the time from the battery backed persistent clock.
1577 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
john stultz85240702007-05-08 00:27:59 -07001578 *
1579 * XXX - Do be sure to remove it once all arches implement it.
1580 */
Arnd Bergmann92661782018-08-14 14:15:23 +02001581void __weak read_persistent_clock64(struct timespec64 *ts)
john stultz85240702007-05-08 00:27:59 -07001582{
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +02001583 ts->tv_sec = 0;
1584 ts->tv_nsec = 0;
john stultz85240702007-05-08 00:27:59 -07001585}
1586
Martin Schwidefsky23970e32009-08-14 15:47:32 +02001587/**
Pavel Tatashin3eca9932018-07-19 16:55:34 -04001588 * read_persistent_wall_and_boot_offset - Read persistent clock, and also offset
1589 * from the boot.
Martin Schwidefsky23970e32009-08-14 15:47:32 +02001590 *
1591 * Weak dummy function for arches that do not yet support it.
Alex Shi29efc462020-11-13 15:24:35 +08001592 * @wall_time: - current time as returned by persistent clock
1593 * @boot_offset: - offset that is defined as wall_time - boot_time
1594 *
Pavel Tatashin4b1b7f82018-07-19 16:55:35 -04001595 * The default function calculates offset based on the current value of
1596 * local_clock(). This way architectures that support sched_clock() but don't
1597 * support dedicated boot time clock will provide the best estimate of the
1598 * boot time.
Martin Schwidefsky23970e32009-08-14 15:47:32 +02001599 */
Pavel Tatashin3eca9932018-07-19 16:55:34 -04001600void __weak __init
1601read_persistent_wall_and_boot_offset(struct timespec64 *wall_time,
1602 struct timespec64 *boot_offset)
Martin Schwidefsky23970e32009-08-14 15:47:32 +02001603{
Pavel Tatashin3eca9932018-07-19 16:55:34 -04001604 read_persistent_clock64(wall_time);
Pavel Tatashin4b1b7f82018-07-19 16:55:35 -04001605 *boot_offset = ns_to_timespec64(local_clock());
Martin Schwidefsky23970e32009-08-14 15:47:32 +02001606}
1607
Mukesh Ojhaf473e5f2018-07-17 12:01:29 +05301608/*
1609 * Flag reflecting whether timekeeping_resume() has injected sleeptime.
1610 *
1611 * The flag starts of false and is only set when a suspend reaches
1612 * timekeeping_suspend(), timekeeping_resume() sets it to false when the
1613 * timekeeper clocksource is not stopping across suspend and has been
1614 * used to update sleep time. If the timekeeper clocksource has stopped
1615 * then the flag stays true and is used by the RTC resume code to decide
1616 * whether sleeptime must be injected and if so the flag gets false then.
1617 *
1618 * If a suspend fails before reaching timekeeping_resume() then the flag
1619 * stays false and prevents erroneous sleeptime injection.
1620 */
1621static bool suspend_timing_needed;
Xunlei Pang0fa88cb2015-04-01 20:34:38 -07001622
1623/* Flag for if there is a persistent clock on this platform */
1624static bool persistent_clock_exists;
1625
john stultz85240702007-05-08 00:27:59 -07001626/*
1627 * timekeeping_init - Initializes the clocksource and common timekeeping values
1628 */
1629void __init timekeeping_init(void)
1630{
Pavel Tatashin3eca9932018-07-19 16:55:34 -04001631 struct timespec64 wall_time, boot_offset, wall_to_mono;
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001632 struct timekeeper *tk = &tk_core.timekeeper;
Martin Schwidefsky155ec602009-08-14 15:47:26 +02001633 struct clocksource *clock;
john stultz85240702007-05-08 00:27:59 -07001634 unsigned long flags;
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +02001635
Pavel Tatashin3eca9932018-07-19 16:55:34 -04001636 read_persistent_wall_and_boot_offset(&wall_time, &boot_offset);
Thomas Gleixner7a8e61f2019-03-23 11:36:19 +01001637 if (timespec64_valid_settod(&wall_time) &&
Pavel Tatashin3eca9932018-07-19 16:55:34 -04001638 timespec64_to_ns(&wall_time) > 0) {
Xunlei Pang0fa88cb2015-04-01 20:34:38 -07001639 persistent_clock_exists = true;
Pavel Tatashin684ad532018-07-25 16:00:18 -04001640 } else if (timespec64_to_ns(&wall_time) != 0) {
Pavel Tatashin3eca9932018-07-19 16:55:34 -04001641 pr_warn("Persistent clock returned invalid value");
1642 wall_time = (struct timespec64){0};
John Stultz4e8b1452012-08-08 15:36:20 -04001643 }
john stultz85240702007-05-08 00:27:59 -07001644
Pavel Tatashin3eca9932018-07-19 16:55:34 -04001645 if (timespec64_compare(&wall_time, &boot_offset) < 0)
1646 boot_offset = (struct timespec64){0};
1647
1648 /*
1649 * We want set wall_to_mono, so the following is true:
1650 * wall time + wall_to_mono = boot time
1651 */
1652 wall_to_mono = timespec64_sub(boot_offset, wall_time);
1653
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001654 raw_spin_lock_irqsave(&timekeeper_lock, flags);
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001655 write_seqcount_begin(&tk_core.seq);
John Stultz06c017f2013-03-22 11:37:28 -07001656 ntp_init();
1657
Martin Schwidefskyf1b82742009-08-14 15:47:21 +02001658 clock = clocksource_default_clock();
Martin Schwidefskya0f7d482009-08-14 15:47:19 +02001659 if (clock->enable)
1660 clock->enable(clock);
John Stultz4e250fd2012-07-27 14:48:13 -04001661 tk_setup_internals(tk, clock);
john stultz85240702007-05-08 00:27:59 -07001662
Pavel Tatashin3eca9932018-07-19 16:55:34 -04001663 tk_set_xtime(tk, &wall_time);
John Stultzfc6eead72017-05-22 17:20:20 -07001664 tk->raw_sec = 0;
John Stultz1e75fa82012-07-13 01:21:53 -04001665
Pavel Tatashin3eca9932018-07-19 16:55:34 -04001666 tk_set_wall_to_mono(tk, wall_to_mono);
John Stultz6d0ef902012-07-27 14:48:12 -04001667
Thomas Gleixner56fd16c2015-10-16 15:50:22 +02001668 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
Thomas Gleixner48cdc132013-02-21 22:51:40 +00001669
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001670 write_seqcount_end(&tk_core.seq);
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001671 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
john stultz85240702007-05-08 00:27:59 -07001672}
1673
Xunlei Pang264bb3f2015-04-01 20:34:37 -07001674/* time in seconds when suspend began for persistent clock */
John Stultz7d489d12014-07-16 21:04:01 +00001675static struct timespec64 timekeeping_suspend_time;
john stultz85240702007-05-08 00:27:59 -07001676
1677/**
John Stultz304529b2011-04-01 14:32:09 -07001678 * __timekeeping_inject_sleeptime - Internal function to add sleep interval
Alex Shi6e5a9192020-11-13 15:24:34 +08001679 * @tk: Pointer to the timekeeper to be updated
1680 * @delta: Pointer to the delta value in timespec64 format
John Stultz304529b2011-04-01 14:32:09 -07001681 *
1682 * Takes a timespec offset measuring a suspend interval and properly
1683 * adds the sleep offset to the timekeeping variables.
1684 */
John Stultzf726a692012-07-13 01:21:57 -04001685static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
Ondrej Mosnacek985e6952018-07-13 14:06:42 +02001686 const struct timespec64 *delta)
John Stultz304529b2011-04-01 14:32:09 -07001687{
John Stultz7d489d12014-07-16 21:04:01 +00001688 if (!timespec64_valid_strict(delta)) {
John Stultz6d9bcb62014-06-04 16:11:43 -07001689 printk_deferred(KERN_WARNING
1690 "__timekeeping_inject_sleeptime: Invalid "
1691 "sleep delta value!\n");
John Stultzcb5de2f8d2011-06-01 18:18:09 -07001692 return;
1693 }
John Stultzf726a692012-07-13 01:21:57 -04001694 tk_xtime_add(tk, delta);
Thomas Gleixnera3ed0e432018-04-25 15:33:38 +02001695 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta));
Thomas Gleixner47da70d2014-07-16 21:05:00 +00001696 tk_update_sleep_time(tk, timespec64_to_ktime(*delta));
Colin Cross5c835452013-05-21 22:32:14 -07001697 tk_debug_account_sleep_time(delta);
John Stultz304529b2011-04-01 14:32:09 -07001698}
1699
Xunlei Pang7f298132015-04-01 20:34:35 -07001700#if defined(CONFIG_PM_SLEEP) && defined(CONFIG_RTC_HCTOSYS_DEVICE)
John Stultz304529b2011-04-01 14:32:09 -07001701/**
Xunlei Pang0fa88cb2015-04-01 20:34:38 -07001702 * We have three kinds of time sources to use for sleep time
1703 * injection, the preference order is:
1704 * 1) non-stop clocksource
1705 * 2) persistent clock (ie: RTC accessible when irqs are off)
1706 * 3) RTC
1707 *
1708 * 1) and 2) are used by timekeeping, 3) by RTC subsystem.
1709 * If system has neither 1) nor 2), 3) will be used finally.
1710 *
1711 *
1712 * If timekeeping has injected sleeptime via either 1) or 2),
1713 * 3) becomes needless, so in this case we don't need to call
1714 * rtc_resume(), and this is what timekeeping_rtc_skipresume()
1715 * means.
1716 */
1717bool timekeeping_rtc_skipresume(void)
1718{
Mukesh Ojhaf473e5f2018-07-17 12:01:29 +05301719 return !suspend_timing_needed;
Xunlei Pang0fa88cb2015-04-01 20:34:38 -07001720}
1721
1722/**
1723 * 1) can be determined whether to use or not only when doing
1724 * timekeeping_resume() which is invoked after rtc_suspend(),
1725 * so we can't skip rtc_suspend() surely if system has 1).
1726 *
1727 * But if system has 2), 2) will definitely be used, so in this
1728 * case we don't need to call rtc_suspend(), and this is what
1729 * timekeeping_rtc_skipsuspend() means.
1730 */
1731bool timekeeping_rtc_skipsuspend(void)
1732{
1733 return persistent_clock_exists;
1734}
1735
1736/**
pang.xunlei04d90892014-11-18 19:15:17 +08001737 * timekeeping_inject_sleeptime64 - Adds suspend interval to timeekeeping values
1738 * @delta: pointer to a timespec64 delta value
John Stultz304529b2011-04-01 14:32:09 -07001739 *
Xunlei Pang2ee96632015-04-01 20:34:22 -07001740 * This hook is for architectures that cannot support read_persistent_clock64
John Stultz304529b2011-04-01 14:32:09 -07001741 * because their RTC/persistent clock is only accessible when irqs are enabled.
Xunlei Pang0fa88cb2015-04-01 20:34:38 -07001742 * and also don't have an effective nonstop clocksource.
John Stultz304529b2011-04-01 14:32:09 -07001743 *
1744 * This function should only be called by rtc_resume(), and allows
1745 * a suspend offset to be injected into the timekeeping values.
1746 */
Ondrej Mosnacek985e6952018-07-13 14:06:42 +02001747void timekeeping_inject_sleeptime64(const struct timespec64 *delta)
John Stultz304529b2011-04-01 14:32:09 -07001748{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001749 struct timekeeper *tk = &tk_core.timekeeper;
John Stultz92c1d3e2011-11-14 14:05:44 -08001750 unsigned long flags;
John Stultz304529b2011-04-01 14:32:09 -07001751
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001752 raw_spin_lock_irqsave(&timekeeper_lock, flags);
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001753 write_seqcount_begin(&tk_core.seq);
John Stultz70471f22011-11-14 12:48:10 -08001754
Mukesh Ojhaf473e5f2018-07-17 12:01:29 +05301755 suspend_timing_needed = false;
1756
John Stultz4e250fd2012-07-27 14:48:13 -04001757 timekeeping_forward_now(tk);
John Stultz304529b2011-04-01 14:32:09 -07001758
pang.xunlei04d90892014-11-18 19:15:17 +08001759 __timekeeping_inject_sleeptime(tk, delta);
John Stultz304529b2011-04-01 14:32:09 -07001760
David Vrabel780427f2013-06-27 11:35:46 +01001761 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET);
John Stultz304529b2011-04-01 14:32:09 -07001762
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001763 write_seqcount_end(&tk_core.seq);
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001764 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
John Stultz304529b2011-04-01 14:32:09 -07001765
Thomas Gleixner17a1b882021-07-13 15:39:53 +02001766 /* Signal hrtimers about time change */
1767 clock_was_set(CLOCK_SET_WALL | CLOCK_SET_BOOT);
John Stultz304529b2011-04-01 14:32:09 -07001768}
Xunlei Pang7f298132015-04-01 20:34:35 -07001769#endif
John Stultz304529b2011-04-01 14:32:09 -07001770
John Stultz304529b2011-04-01 14:32:09 -07001771/**
john stultz85240702007-05-08 00:27:59 -07001772 * timekeeping_resume - Resumes the generic timekeeping subsystem.
john stultz85240702007-05-08 00:27:59 -07001773 */
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +01001774void timekeeping_resume(void)
john stultz85240702007-05-08 00:27:59 -07001775{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001776 struct timekeeper *tk = &tk_core.timekeeper;
Peter Zijlstra876e7882015-03-19 10:09:06 +01001777 struct clocksource *clock = tk->tkr_mono.clock;
John Stultz92c1d3e2011-11-14 14:05:44 -08001778 unsigned long flags;
John Stultz7d489d12014-07-16 21:04:01 +00001779 struct timespec64 ts_new, ts_delta;
Baolin Wang39232ed52018-07-17 15:55:16 +08001780 u64 cycle_now, nsec;
Mukesh Ojhaf473e5f2018-07-17 12:01:29 +05301781 bool inject_sleeptime = false;
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +02001782
Xunlei Pang2ee96632015-04-01 20:34:22 -07001783 read_persistent_clock64(&ts_new);
john stultz85240702007-05-08 00:27:59 -07001784
Rafael J. Wysockiadc78e62012-08-06 01:40:41 +02001785 clockevents_resume();
Thomas Gleixnerd10ff3f2007-05-14 11:10:02 +02001786 clocksource_resume();
1787
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001788 raw_spin_lock_irqsave(&timekeeper_lock, flags);
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001789 write_seqcount_begin(&tk_core.seq);
john stultz85240702007-05-08 00:27:59 -07001790
Feng Tange445cf12013-03-12 11:56:48 +08001791 /*
1792 * After system resumes, we need to calculate the suspended time and
1793 * compensate it for the OS time. There are 3 sources that could be
1794 * used: Nonstop clocksource during suspend, persistent clock and rtc
1795 * device.
1796 *
1797 * One specific platform may have 1 or 2 or all of them, and the
1798 * preference will be:
1799 * suspend-nonstop clocksource -> persistent clock -> rtc
1800 * The less preferred source will only be tried if there is no better
1801 * usable source. The rtc part is handled separately in rtc core code.
1802 */
John Stultzceea5e32017-06-08 16:44:20 -07001803 cycle_now = tk_clock_read(&tk->tkr_mono);
Baolin Wang39232ed52018-07-17 15:55:16 +08001804 nsec = clocksource_stop_suspend_timing(clock, cycle_now);
1805 if (nsec > 0) {
John Stultz7d489d12014-07-16 21:04:01 +00001806 ts_delta = ns_to_timespec64(nsec);
Mukesh Ojhaf473e5f2018-07-17 12:01:29 +05301807 inject_sleeptime = true;
John Stultz7d489d12014-07-16 21:04:01 +00001808 } else if (timespec64_compare(&ts_new, &timekeeping_suspend_time) > 0) {
1809 ts_delta = timespec64_sub(ts_new, timekeeping_suspend_time);
Mukesh Ojhaf473e5f2018-07-17 12:01:29 +05301810 inject_sleeptime = true;
john stultz85240702007-05-08 00:27:59 -07001811 }
Feng Tange445cf12013-03-12 11:56:48 +08001812
Mukesh Ojhaf473e5f2018-07-17 12:01:29 +05301813 if (inject_sleeptime) {
1814 suspend_timing_needed = false;
Feng Tange445cf12013-03-12 11:56:48 +08001815 __timekeeping_inject_sleeptime(tk, &ts_delta);
Mukesh Ojhaf473e5f2018-07-17 12:01:29 +05301816 }
Feng Tange445cf12013-03-12 11:56:48 +08001817
1818 /* Re-base the last cycle value */
Peter Zijlstra876e7882015-03-19 10:09:06 +01001819 tk->tkr_mono.cycle_last = cycle_now;
Peter Zijlstra4a4ad802015-03-19 09:28:44 +01001820 tk->tkr_raw.cycle_last = cycle_now;
1821
John Stultz4e250fd2012-07-27 14:48:13 -04001822 tk->ntp_error = 0;
john stultz85240702007-05-08 00:27:59 -07001823 timekeeping_suspended = 0;
David Vrabel780427f2013-06-27 11:35:46 +01001824 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001825 write_seqcount_end(&tk_core.seq);
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001826 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
john stultz85240702007-05-08 00:27:59 -07001827
1828 touch_softlockup_watchdog();
1829
Thomas Gleixnera761a672021-07-13 15:39:51 +02001830 /* Resume the clockevent device(s) and hrtimers */
Thomas Gleixner4ffee522015-03-25 13:09:16 +01001831 tick_resume();
Thomas Gleixnera761a672021-07-13 15:39:51 +02001832 /* Notify timerfd as resume is equivalent to clock_was_set() */
1833 timerfd_resume();
john stultz85240702007-05-08 00:27:59 -07001834}
1835
Rafael J. Wysocki124cf9112015-02-13 23:50:43 +01001836int timekeeping_suspend(void)
john stultz85240702007-05-08 00:27:59 -07001837{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001838 struct timekeeper *tk = &tk_core.timekeeper;
John Stultz92c1d3e2011-11-14 14:05:44 -08001839 unsigned long flags;
John Stultz7d489d12014-07-16 21:04:01 +00001840 struct timespec64 delta, delta_delta;
1841 static struct timespec64 old_delta;
Baolin Wang39232ed52018-07-17 15:55:16 +08001842 struct clocksource *curr_clock;
1843 u64 cycle_now;
john stultz85240702007-05-08 00:27:59 -07001844
Xunlei Pang2ee96632015-04-01 20:34:22 -07001845 read_persistent_clock64(&timekeeping_suspend_time);
Thomas Gleixner3be90952007-09-16 15:36:43 +02001846
Zoran Markovic0d6bd992013-05-17 11:24:05 -07001847 /*
1848 * On some systems the persistent_clock can not be detected at
1849 * timekeeping_init by its return value, so if we see a valid
1850 * value returned, update the persistent_clock_exists flag.
1851 */
1852 if (timekeeping_suspend_time.tv_sec || timekeeping_suspend_time.tv_nsec)
Xunlei Pang0fa88cb2015-04-01 20:34:38 -07001853 persistent_clock_exists = true;
Zoran Markovic0d6bd992013-05-17 11:24:05 -07001854
Mukesh Ojhaf473e5f2018-07-17 12:01:29 +05301855 suspend_timing_needed = true;
1856
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001857 raw_spin_lock_irqsave(&timekeeper_lock, flags);
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001858 write_seqcount_begin(&tk_core.seq);
John Stultz4e250fd2012-07-27 14:48:13 -04001859 timekeeping_forward_now(tk);
john stultz85240702007-05-08 00:27:59 -07001860 timekeeping_suspended = 1;
John Stultzcb332172011-05-31 22:53:23 -07001861
Baolin Wang39232ed52018-07-17 15:55:16 +08001862 /*
1863 * Since we've called forward_now, cycle_last stores the value
1864 * just read from the current clocksource. Save this to potentially
1865 * use in suspend timing.
1866 */
1867 curr_clock = tk->tkr_mono.clock;
1868 cycle_now = tk->tkr_mono.cycle_last;
1869 clocksource_start_suspend_timing(curr_clock, cycle_now);
1870
Xunlei Pang0fa88cb2015-04-01 20:34:38 -07001871 if (persistent_clock_exists) {
John Stultzcb332172011-05-31 22:53:23 -07001872 /*
Xunlei Pang264bb3f2015-04-01 20:34:37 -07001873 * To avoid drift caused by repeated suspend/resumes,
1874 * which each can add ~1 second drift error,
1875 * try to compensate so the difference in system time
1876 * and persistent_clock time stays close to constant.
John Stultzcb332172011-05-31 22:53:23 -07001877 */
Xunlei Pang264bb3f2015-04-01 20:34:37 -07001878 delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time);
1879 delta_delta = timespec64_sub(delta, old_delta);
1880 if (abs(delta_delta.tv_sec) >= 2) {
1881 /*
1882 * if delta_delta is too large, assume time correction
1883 * has occurred and set old_delta to the current delta.
1884 */
1885 old_delta = delta;
1886 } else {
1887 /* Otherwise try to adjust old_system to compensate */
1888 timekeeping_suspend_time =
1889 timespec64_add(timekeeping_suspend_time, delta_delta);
1890 }
John Stultzcb332172011-05-31 22:53:23 -07001891 }
John Stultz330a1612013-12-11 19:10:36 -08001892
1893 timekeeping_update(tk, TK_MIRROR);
Rafael J. Wysocki060407a2015-02-13 14:49:02 +01001894 halt_fast_timekeeper(tk);
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00001895 write_seqcount_end(&tk_core.seq);
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00001896 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
john stultz85240702007-05-08 00:27:59 -07001897
Thomas Gleixner4ffee522015-03-25 13:09:16 +01001898 tick_suspend();
Magnus Dammc54a42b2010-02-02 14:41:41 -08001899 clocksource_suspend();
Rafael J. Wysockiadc78e62012-08-06 01:40:41 +02001900 clockevents_suspend();
john stultz85240702007-05-08 00:27:59 -07001901
1902 return 0;
1903}
1904
1905/* sysfs resume/suspend bits for timekeeping */
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +01001906static struct syscore_ops timekeeping_syscore_ops = {
john stultz85240702007-05-08 00:27:59 -07001907 .resume = timekeeping_resume,
1908 .suspend = timekeeping_suspend,
john stultz85240702007-05-08 00:27:59 -07001909};
1910
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +01001911static int __init timekeeping_init_ops(void)
john stultz85240702007-05-08 00:27:59 -07001912{
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +01001913 register_syscore_ops(&timekeeping_syscore_ops);
1914 return 0;
john stultz85240702007-05-08 00:27:59 -07001915}
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +01001916device_initcall(timekeeping_init_ops);
john stultz85240702007-05-08 00:27:59 -07001917
1918/*
John Stultzdc491592013-12-06 17:25:21 -08001919 * Apply a multiplier adjustment to the timekeeper
john stultz85240702007-05-08 00:27:59 -07001920 */
John Stultzdc491592013-12-06 17:25:21 -08001921static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk,
1922 s64 offset,
Miroslav Lichvar78b98e32018-03-09 10:42:48 -08001923 s32 mult_adj)
john stultz85240702007-05-08 00:27:59 -07001924{
John Stultzdc491592013-12-06 17:25:21 -08001925 s64 interval = tk->cycle_interval;
john stultz85240702007-05-08 00:27:59 -07001926
Miroslav Lichvar78b98e32018-03-09 10:42:48 -08001927 if (mult_adj == 0) {
1928 return;
1929 } else if (mult_adj == -1) {
John Stultzdc491592013-12-06 17:25:21 -08001930 interval = -interval;
Miroslav Lichvar78b98e32018-03-09 10:42:48 -08001931 offset = -offset;
1932 } else if (mult_adj != 1) {
1933 interval *= mult_adj;
1934 offset *= mult_adj;
john stultz85240702007-05-08 00:27:59 -07001935 }
john stultz85240702007-05-08 00:27:59 -07001936
John Stultzc2bc1112011-10-27 18:12:42 -07001937 /*
1938 * So the following can be confusing.
1939 *
John Stultzdc491592013-12-06 17:25:21 -08001940 * To keep things simple, lets assume mult_adj == 1 for now.
John Stultzc2bc1112011-10-27 18:12:42 -07001941 *
John Stultzdc491592013-12-06 17:25:21 -08001942 * When mult_adj != 1, remember that the interval and offset values
John Stultzc2bc1112011-10-27 18:12:42 -07001943 * have been appropriately scaled so the math is the same.
1944 *
1945 * The basic idea here is that we're increasing the multiplier
1946 * by one, this causes the xtime_interval to be incremented by
1947 * one cycle_interval. This is because:
1948 * xtime_interval = cycle_interval * mult
1949 * So if mult is being incremented by one:
1950 * xtime_interval = cycle_interval * (mult + 1)
1951 * Its the same as:
1952 * xtime_interval = (cycle_interval * mult) + cycle_interval
1953 * Which can be shortened to:
1954 * xtime_interval += cycle_interval
1955 *
1956 * So offset stores the non-accumulated cycles. Thus the current
1957 * time (in shifted nanoseconds) is:
1958 * now = (offset * adj) + xtime_nsec
1959 * Now, even though we're adjusting the clock frequency, we have
1960 * to keep time consistent. In other words, we can't jump back
1961 * in time, and we also want to avoid jumping forward in time.
1962 *
1963 * So given the same offset value, we need the time to be the same
1964 * both before and after the freq adjustment.
1965 * now = (offset * adj_1) + xtime_nsec_1
1966 * now = (offset * adj_2) + xtime_nsec_2
1967 * So:
1968 * (offset * adj_1) + xtime_nsec_1 =
1969 * (offset * adj_2) + xtime_nsec_2
1970 * And we know:
1971 * adj_2 = adj_1 + 1
1972 * So:
1973 * (offset * adj_1) + xtime_nsec_1 =
1974 * (offset * (adj_1+1)) + xtime_nsec_2
1975 * (offset * adj_1) + xtime_nsec_1 =
1976 * (offset * adj_1) + offset + xtime_nsec_2
1977 * Canceling the sides:
1978 * xtime_nsec_1 = offset + xtime_nsec_2
1979 * Which gives us:
1980 * xtime_nsec_2 = xtime_nsec_1 - offset
Ingo Molnar4bf07f62021-03-22 22:39:03 +01001981 * Which simplifies to:
John Stultzc2bc1112011-10-27 18:12:42 -07001982 * xtime_nsec -= offset
John Stultzc2bc1112011-10-27 18:12:42 -07001983 */
Peter Zijlstra876e7882015-03-19 10:09:06 +01001984 if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) {
pang.xunlei6067dc52014-10-08 15:03:34 +08001985 /* NTP adjustment caused clocksource mult overflow */
1986 WARN_ON_ONCE(1);
1987 return;
1988 }
1989
Peter Zijlstra876e7882015-03-19 10:09:06 +01001990 tk->tkr_mono.mult += mult_adj;
John Stultzf726a692012-07-13 01:21:57 -04001991 tk->xtime_interval += interval;
Peter Zijlstra876e7882015-03-19 10:09:06 +01001992 tk->tkr_mono.xtime_nsec -= offset;
John Stultzdc491592013-12-06 17:25:21 -08001993}
John Stultz2a8c0882012-07-13 01:21:56 -04001994
John Stultzdc491592013-12-06 17:25:21 -08001995/*
John Stultzdc491592013-12-06 17:25:21 -08001996 * Adjust the timekeeper's multiplier to the correct frequency
1997 * and also to reduce the accumulated error value.
1998 */
1999static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
2000{
Miroslav Lichvar78b98e32018-03-09 10:42:48 -08002001 u32 mult;
John Stultzdc491592013-12-06 17:25:21 -08002002
Miroslav Lichvar78b98e32018-03-09 10:42:48 -08002003 /*
2004 * Determine the multiplier from the current NTP tick length.
2005 * Avoid expensive division when the tick length doesn't change.
2006 */
2007 if (likely(tk->ntp_tick == ntp_tick_length())) {
2008 mult = tk->tkr_mono.mult - tk->ntp_err_mult;
2009 } else {
2010 tk->ntp_tick = ntp_tick_length();
2011 mult = div64_u64((tk->ntp_tick >> tk->ntp_error_shift) -
2012 tk->xtime_remainder, tk->cycle_interval);
John Stultzdc491592013-12-06 17:25:21 -08002013 }
2014
Miroslav Lichvar78b98e32018-03-09 10:42:48 -08002015 /*
2016 * If the clock is behind the NTP time, increase the multiplier by 1
2017 * to catch up with it. If it's ahead and there was a remainder in the
2018 * tick division, the clock will slow down. Otherwise it will stay
2019 * ahead until the tick length changes to a non-divisible value.
2020 */
2021 tk->ntp_err_mult = tk->ntp_error > 0 ? 1 : 0;
2022 mult += tk->ntp_err_mult;
2023
2024 timekeeping_apply_adjustment(tk, offset, mult - tk->tkr_mono.mult);
2025
Peter Zijlstra876e7882015-03-19 10:09:06 +01002026 if (unlikely(tk->tkr_mono.clock->maxadj &&
2027 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult)
2028 > tk->tkr_mono.clock->maxadj))) {
John Stultzdc491592013-12-06 17:25:21 -08002029 printk_once(KERN_WARNING
2030 "Adjusting %s more than 11%% (%ld vs %ld)\n",
Peter Zijlstra876e7882015-03-19 10:09:06 +01002031 tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult,
2032 (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj);
John Stultzdc491592013-12-06 17:25:21 -08002033 }
2034
John Stultz2a8c0882012-07-13 01:21:56 -04002035 /*
2036 * It may be possible that when we entered this function, xtime_nsec
2037 * was very small. Further, if we're slightly speeding the clocksource
2038 * in the code above, its possible the required corrective factor to
2039 * xtime_nsec could cause it to underflow.
2040 *
Miroslav Lichvar78b98e32018-03-09 10:42:48 -08002041 * Now, since we have already accumulated the second and the NTP
2042 * subsystem has been notified via second_overflow(), we need to skip
2043 * the next update.
John Stultz2a8c0882012-07-13 01:21:56 -04002044 */
Peter Zijlstra876e7882015-03-19 10:09:06 +01002045 if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) {
Miroslav Lichvar78b98e32018-03-09 10:42:48 -08002046 tk->tkr_mono.xtime_nsec += (u64)NSEC_PER_SEC <<
2047 tk->tkr_mono.shift;
2048 tk->xtime_sec--;
2049 tk->skip_second_overflow = 1;
John Stultz2a8c0882012-07-13 01:21:56 -04002050 }
john stultz85240702007-05-08 00:27:59 -07002051}
2052
Alex Shi199d2802020-11-13 15:24:33 +08002053/*
John Stultz1f4f9482012-07-13 01:21:54 -04002054 * accumulate_nsecs_to_secs - Accumulates nsecs into secs
2055 *
Zhen Lei571af552015-08-25 14:42:53 +08002056 * Helper function that accumulates the nsecs greater than a second
John Stultz1f4f9482012-07-13 01:21:54 -04002057 * from the xtime_nsec field to the xtime_secs field.
2058 * It also calls into the NTP code to handle leapsecond processing.
John Stultz1f4f9482012-07-13 01:21:54 -04002059 */
David Vrabel780427f2013-06-27 11:35:46 +01002060static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk)
John Stultz1f4f9482012-07-13 01:21:54 -04002061{
Peter Zijlstra876e7882015-03-19 10:09:06 +01002062 u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift;
John Stultz5258d3f2013-12-11 20:07:49 -08002063 unsigned int clock_set = 0;
John Stultz1f4f9482012-07-13 01:21:54 -04002064
Peter Zijlstra876e7882015-03-19 10:09:06 +01002065 while (tk->tkr_mono.xtime_nsec >= nsecps) {
John Stultz1f4f9482012-07-13 01:21:54 -04002066 int leap;
2067
Peter Zijlstra876e7882015-03-19 10:09:06 +01002068 tk->tkr_mono.xtime_nsec -= nsecps;
John Stultz1f4f9482012-07-13 01:21:54 -04002069 tk->xtime_sec++;
2070
Miroslav Lichvar78b98e32018-03-09 10:42:48 -08002071 /*
2072 * Skip NTP update if this second was accumulated before,
2073 * i.e. xtime_nsec underflowed in timekeeping_adjust()
2074 */
2075 if (unlikely(tk->skip_second_overflow)) {
2076 tk->skip_second_overflow = 0;
2077 continue;
2078 }
2079
John Stultz1f4f9482012-07-13 01:21:54 -04002080 /* Figure out if its a leap sec and apply if needed */
2081 leap = second_overflow(tk->xtime_sec);
John Stultz6d0ef902012-07-27 14:48:12 -04002082 if (unlikely(leap)) {
John Stultz7d489d12014-07-16 21:04:01 +00002083 struct timespec64 ts;
John Stultz1f4f9482012-07-13 01:21:54 -04002084
John Stultz6d0ef902012-07-27 14:48:12 -04002085 tk->xtime_sec += leap;
2086
2087 ts.tv_sec = leap;
2088 ts.tv_nsec = 0;
2089 tk_set_wall_to_mono(tk,
John Stultz7d489d12014-07-16 21:04:01 +00002090 timespec64_sub(tk->wall_to_monotonic, ts));
John Stultz6d0ef902012-07-27 14:48:12 -04002091
John Stultzcc244dda2012-05-03 12:30:07 -07002092 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap);
2093
John Stultz5258d3f2013-12-11 20:07:49 -08002094 clock_set = TK_CLOCK_WAS_SET;
John Stultz6d0ef902012-07-27 14:48:12 -04002095 }
John Stultz1f4f9482012-07-13 01:21:54 -04002096 }
John Stultz5258d3f2013-12-11 20:07:49 -08002097 return clock_set;
John Stultz1f4f9482012-07-13 01:21:54 -04002098}
2099
Alex Shi199d2802020-11-13 15:24:33 +08002100/*
john stultza092ff02009-10-02 16:17:53 -07002101 * logarithmic_accumulation - shifted accumulation of cycles
2102 *
2103 * This functions accumulates a shifted interval of cycles into
Randy Dunlapb0294f32020-08-06 20:32:48 -07002104 * a shifted interval nanoseconds. Allows for O(log) accumulation
john stultza092ff02009-10-02 16:17:53 -07002105 * loop.
2106 *
2107 * Returns the unconsumed cycles.
2108 */
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +01002109static u64 logarithmic_accumulation(struct timekeeper *tk, u64 offset,
2110 u32 shift, unsigned int *clock_set)
john stultza092ff02009-10-02 16:17:53 -07002111{
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +01002112 u64 interval = tk->cycle_interval << shift;
John Stultz3d88d56c52017-06-08 16:44:21 -07002113 u64 snsec_per_sec;
john stultza092ff02009-10-02 16:17:53 -07002114
Zhen Lei571af552015-08-25 14:42:53 +08002115 /* If the offset is smaller than a shifted interval, do nothing */
Thomas Gleixner23a95372013-02-21 22:51:36 +00002116 if (offset < interval)
john stultza092ff02009-10-02 16:17:53 -07002117 return offset;
2118
2119 /* Accumulate one shifted interval */
Thomas Gleixner23a95372013-02-21 22:51:36 +00002120 offset -= interval;
Peter Zijlstra876e7882015-03-19 10:09:06 +01002121 tk->tkr_mono.cycle_last += interval;
Peter Zijlstra4a4ad802015-03-19 09:28:44 +01002122 tk->tkr_raw.cycle_last += interval;
john stultza092ff02009-10-02 16:17:53 -07002123
Peter Zijlstra876e7882015-03-19 10:09:06 +01002124 tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift;
John Stultz5258d3f2013-12-11 20:07:49 -08002125 *clock_set |= accumulate_nsecs_to_secs(tk);
john stultza092ff02009-10-02 16:17:53 -07002126
Jason Wesseldeda2e82010-08-09 14:20:09 -07002127 /* Accumulate raw time */
John Stultz3d88d56c52017-06-08 16:44:21 -07002128 tk->tkr_raw.xtime_nsec += tk->raw_interval << shift;
2129 snsec_per_sec = (u64)NSEC_PER_SEC << tk->tkr_raw.shift;
2130 while (tk->tkr_raw.xtime_nsec >= snsec_per_sec) {
2131 tk->tkr_raw.xtime_nsec -= snsec_per_sec;
John Stultzfc6eead72017-05-22 17:20:20 -07002132 tk->raw_sec++;
john stultza092ff02009-10-02 16:17:53 -07002133 }
2134
2135 /* Accumulate error between NTP and clock interval */
John Stultz375f45b2014-04-23 20:53:29 -07002136 tk->ntp_error += tk->ntp_tick << shift;
John Stultzf726a692012-07-13 01:21:57 -04002137 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
2138 (tk->ntp_error_shift + shift);
john stultza092ff02009-10-02 16:17:53 -07002139
2140 return offset;
2141}
2142
Miroslav Lichvarb061c7a2018-06-04 15:34:21 +02002143/*
2144 * timekeeping_advance - Updates the timekeeper to the current time and
2145 * current NTP tick length
john stultz85240702007-05-08 00:27:59 -07002146 */
Thomas Gleixner1b267792021-07-13 15:39:52 +02002147static bool timekeeping_advance(enum timekeeping_adv_mode mode)
john stultz85240702007-05-08 00:27:59 -07002148{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002149 struct timekeeper *real_tk = &tk_core.timekeeper;
Thomas Gleixner48cdc132013-02-21 22:51:40 +00002150 struct timekeeper *tk = &shadow_timekeeper;
Thomas Gleixnera5a1d1c2016-12-21 20:32:01 +01002151 u64 offset;
john stultza092ff02009-10-02 16:17:53 -07002152 int shift = 0, maxshift;
John Stultz5258d3f2013-12-11 20:07:49 -08002153 unsigned int clock_set = 0;
John Stultz70471f22011-11-14 12:48:10 -08002154 unsigned long flags;
2155
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00002156 raw_spin_lock_irqsave(&timekeeper_lock, flags);
john stultz85240702007-05-08 00:27:59 -07002157
2158 /* Make sure we're fully resumed: */
2159 if (unlikely(timekeeping_suspended))
John Stultz70471f22011-11-14 12:48:10 -08002160 goto out;
john stultz85240702007-05-08 00:27:59 -07002161
John Stultzceea5e32017-06-08 16:44:20 -07002162 offset = clocksource_delta(tk_clock_read(&tk->tkr_mono),
Peter Zijlstra876e7882015-03-19 10:09:06 +01002163 tk->tkr_mono.cycle_last, tk->tkr_mono.mask);
john stultz85240702007-05-08 00:27:59 -07002164
John Stultzbf2ac312012-08-21 20:30:49 -04002165 /* Check if there's really nothing to do */
Miroslav Lichvarb061c7a2018-06-04 15:34:21 +02002166 if (offset < real_tk->cycle_interval && mode == TK_ADV_TICK)
John Stultzbf2ac312012-08-21 20:30:49 -04002167 goto out;
2168
John Stultz3c17ad12015-03-11 21:16:32 -07002169 /* Do some additional sanity checking */
Stafford Hornea529bea2017-06-28 22:21:35 +09002170 timekeeping_check_update(tk, offset);
John Stultz3c17ad12015-03-11 21:16:32 -07002171
john stultza092ff02009-10-02 16:17:53 -07002172 /*
2173 * With NO_HZ we may have to accumulate many cycle_intervals
2174 * (think "ticks") worth of time at once. To do this efficiently,
2175 * we calculate the largest doubling multiple of cycle_intervals
Jim Cromie88b28ad2012-03-14 21:28:56 -06002176 * that is smaller than the offset. We then accumulate that
john stultza092ff02009-10-02 16:17:53 -07002177 * chunk in one go, and then try to consume the next smaller
2178 * doubled multiple.
john stultz85240702007-05-08 00:27:59 -07002179 */
John Stultz4e250fd2012-07-27 14:48:13 -04002180 shift = ilog2(offset) - ilog2(tk->cycle_interval);
john stultza092ff02009-10-02 16:17:53 -07002181 shift = max(0, shift);
Jim Cromie88b28ad2012-03-14 21:28:56 -06002182 /* Bound shift to one less than what overflows tick_length */
John Stultzea7cf492011-11-14 13:18:07 -08002183 maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
john stultza092ff02009-10-02 16:17:53 -07002184 shift = min(shift, maxshift);
John Stultz4e250fd2012-07-27 14:48:13 -04002185 while (offset >= tk->cycle_interval) {
John Stultz5258d3f2013-12-11 20:07:49 -08002186 offset = logarithmic_accumulation(tk, offset, shift,
2187 &clock_set);
John Stultz4e250fd2012-07-27 14:48:13 -04002188 if (offset < tk->cycle_interval<<shift)
John Stultz830ec042010-03-18 14:47:30 -07002189 shift--;
john stultz85240702007-05-08 00:27:59 -07002190 }
2191
Miroslav Lichvar78b98e32018-03-09 10:42:48 -08002192 /* Adjust the multiplier to correct NTP error */
John Stultz4e250fd2012-07-27 14:48:13 -04002193 timekeeping_adjust(tk, offset);
john stultz85240702007-05-08 00:27:59 -07002194
John Stultz6a867a32010-04-06 14:30:51 -07002195 /*
John Stultz6a867a32010-04-06 14:30:51 -07002196 * Finally, make sure that after the rounding
John Stultz1e75fa82012-07-13 01:21:53 -04002197 * xtime_nsec isn't larger than NSEC_PER_SEC
John Stultz6a867a32010-04-06 14:30:51 -07002198 */
John Stultz5258d3f2013-12-11 20:07:49 -08002199 clock_set |= accumulate_nsecs_to_secs(tk);
Linus Torvalds83f57a12009-12-22 14:10:37 -08002200
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002201 write_seqcount_begin(&tk_core.seq);
Thomas Gleixner48cdc132013-02-21 22:51:40 +00002202 /*
2203 * Update the real timekeeper.
2204 *
2205 * We could avoid this memcpy by switching pointers, but that
2206 * requires changes to all other timekeeper usage sites as
2207 * well, i.e. move the timekeeper pointer getter into the
2208 * spinlocked/seqcount protected sections. And we trade this
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002209 * memcpy under the tk_core.seq against one before we start
Thomas Gleixner48cdc132013-02-21 22:51:40 +00002210 * updating.
2211 */
John Stultz906c5552015-06-17 10:05:53 -07002212 timekeeping_update(tk, clock_set);
Thomas Gleixner48cdc132013-02-21 22:51:40 +00002213 memcpy(real_tk, tk, sizeof(*tk));
John Stultz906c5552015-06-17 10:05:53 -07002214 /* The memcpy must come last. Do not put anything here! */
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002215 write_seqcount_end(&tk_core.seq);
Thomas Gleixnerca4523c2013-02-21 22:51:40 +00002216out:
Thomas Gleixner9a7a71b2013-02-21 22:51:38 +00002217 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
Thomas Gleixner1b267792021-07-13 15:39:52 +02002218
2219 return !!clock_set;
john stultz85240702007-05-08 00:27:59 -07002220}
Tomas Janousek7c3f1a52007-07-15 23:39:41 -07002221
2222/**
Miroslav Lichvarb061c7a2018-06-04 15:34:21 +02002223 * update_wall_time - Uses the current clocksource to increment the wall time
2224 *
2225 */
2226void update_wall_time(void)
2227{
Thomas Gleixner1b267792021-07-13 15:39:52 +02002228 if (timekeeping_advance(TK_ADV_TICK))
2229 clock_was_set_delayed();
Miroslav Lichvarb061c7a2018-06-04 15:34:21 +02002230}
2231
2232/**
John Stultzd08c0cd2014-12-08 12:00:09 -08002233 * getboottime64 - Return the real time of system boot.
2234 * @ts: pointer to the timespec64 to be set
Tomas Janousek7c3f1a52007-07-15 23:39:41 -07002235 *
John Stultzd08c0cd2014-12-08 12:00:09 -08002236 * Returns the wall-time of boot in a timespec64.
Tomas Janousek7c3f1a52007-07-15 23:39:41 -07002237 *
2238 * This is based on the wall_to_monotonic offset and the total suspend
2239 * time. Calls to settimeofday will affect the value returned (which
2240 * basically means that however wrong your real time clock is at boot time,
2241 * you get the right time here).
2242 */
John Stultzd08c0cd2014-12-08 12:00:09 -08002243void getboottime64(struct timespec64 *ts)
Tomas Janousek7c3f1a52007-07-15 23:39:41 -07002244{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002245 struct timekeeper *tk = &tk_core.timekeeper;
Thomas Gleixnera3ed0e432018-04-25 15:33:38 +02002246 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot);
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +02002247
John Stultzd08c0cd2014-12-08 12:00:09 -08002248 *ts = ktime_to_timespec64(t);
Tomas Janousek7c3f1a52007-07-15 23:39:41 -07002249}
John Stultzd08c0cd2014-12-08 12:00:09 -08002250EXPORT_SYMBOL_GPL(getboottime64);
Tomas Janousek7c3f1a52007-07-15 23:39:41 -07002251
Arnd Bergmannfb7fcc92018-04-27 15:40:14 +02002252void ktime_get_coarse_real_ts64(struct timespec64 *ts)
john stultz2c6b47d2007-07-24 17:47:43 -07002253{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002254 struct timekeeper *tk = &tk_core.timekeeper;
Rasmus Villemoese1e41b62019-03-18 20:55:56 +01002255 unsigned int seq;
john stultz2c6b47d2007-07-24 17:47:43 -07002256
2257 do {
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002258 seq = read_seqcount_begin(&tk_core.seq);
Linus Torvalds83f57a12009-12-22 14:10:37 -08002259
Arnd Bergmannfb7fcc92018-04-27 15:40:14 +02002260 *ts = tk_xtime(tk);
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002261 } while (read_seqcount_retry(&tk_core.seq, seq));
john stultz2c6b47d2007-07-24 17:47:43 -07002262}
Arnd Bergmannfb7fcc92018-04-27 15:40:14 +02002263EXPORT_SYMBOL(ktime_get_coarse_real_ts64);
john stultzda15cfd2009-08-19 19:13:34 -07002264
Arnd Bergmannfb7fcc92018-04-27 15:40:14 +02002265void ktime_get_coarse_ts64(struct timespec64 *ts)
john stultzda15cfd2009-08-19 19:13:34 -07002266{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002267 struct timekeeper *tk = &tk_core.timekeeper;
John Stultz7d489d12014-07-16 21:04:01 +00002268 struct timespec64 now, mono;
Rasmus Villemoese1e41b62019-03-18 20:55:56 +01002269 unsigned int seq;
john stultzda15cfd2009-08-19 19:13:34 -07002270
2271 do {
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002272 seq = read_seqcount_begin(&tk_core.seq);
Linus Torvalds83f57a12009-12-22 14:10:37 -08002273
John Stultz4e250fd2012-07-27 14:48:13 -04002274 now = tk_xtime(tk);
2275 mono = tk->wall_to_monotonic;
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002276 } while (read_seqcount_retry(&tk_core.seq, seq));
john stultzda15cfd2009-08-19 19:13:34 -07002277
Arnd Bergmannfb7fcc92018-04-27 15:40:14 +02002278 set_normalized_timespec64(ts, now.tv_sec + mono.tv_sec,
john stultzda15cfd2009-08-19 19:13:34 -07002279 now.tv_nsec + mono.tv_nsec);
john stultzda15cfd2009-08-19 19:13:34 -07002280}
Arnd Bergmannfb7fcc92018-04-27 15:40:14 +02002281EXPORT_SYMBOL(ktime_get_coarse_ts64);
Torben Hohn871cf1e2011-01-27 15:58:55 +01002282
2283/*
John Stultzd6ad4182012-02-28 16:50:11 -08002284 * Must hold jiffies_lock
Torben Hohn871cf1e2011-01-27 15:58:55 +01002285 */
2286void do_timer(unsigned long ticks)
2287{
2288 jiffies_64 += ticks;
Paul Gortmaker46132e32020-07-01 14:34:18 -04002289 calc_global_load();
Torben Hohn871cf1e2011-01-27 15:58:55 +01002290}
Torben Hohn48cf76f72011-01-27 15:59:05 +01002291
2292/**
John Stultz76f41082014-07-16 21:03:52 +00002293 * ktime_get_update_offsets_now - hrtimer helper
Thomas Gleixner868a3e92015-04-14 21:08:37 +00002294 * @cwsseq: pointer to check and store the clock was set sequence number
Thomas Gleixnerf6c06ab2012-07-10 18:43:24 -04002295 * @offs_real: pointer to storage for monotonic -> realtime offset
Thomas Gleixnera3ed0e432018-04-25 15:33:38 +02002296 * @offs_boot: pointer to storage for monotonic -> boottime offset
Xie XiuQib7bc50e2013-10-18 09:13:30 +08002297 * @offs_tai: pointer to storage for monotonic -> clock tai offset
Thomas Gleixnerf6c06ab2012-07-10 18:43:24 -04002298 *
Thomas Gleixner868a3e92015-04-14 21:08:37 +00002299 * Returns current monotonic time and updates the offsets if the
2300 * sequence number in @cwsseq and timekeeper.clock_was_set_seq are
2301 * different.
2302 *
Xie XiuQib7bc50e2013-10-18 09:13:30 +08002303 * Called from hrtimer_interrupt() or retrigger_next_event()
Thomas Gleixnerf6c06ab2012-07-10 18:43:24 -04002304 */
Thomas Gleixner868a3e92015-04-14 21:08:37 +00002305ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
Thomas Gleixnera3ed0e432018-04-25 15:33:38 +02002306 ktime_t *offs_boot, ktime_t *offs_tai)
Thomas Gleixnerf6c06ab2012-07-10 18:43:24 -04002307{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002308 struct timekeeper *tk = &tk_core.timekeeper;
Thomas Gleixnerf6c06ab2012-07-10 18:43:24 -04002309 unsigned int seq;
Thomas Gleixnera37c0aa2014-07-16 21:04:19 +00002310 ktime_t base;
2311 u64 nsecs;
Thomas Gleixnerf6c06ab2012-07-10 18:43:24 -04002312
2313 do {
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002314 seq = read_seqcount_begin(&tk_core.seq);
Thomas Gleixnerf6c06ab2012-07-10 18:43:24 -04002315
Peter Zijlstra876e7882015-03-19 10:09:06 +01002316 base = tk->tkr_mono.base;
2317 nsecs = timekeeping_get_ns(&tk->tkr_mono);
John Stultz833f32d2015-06-11 15:54:55 -07002318 base = ktime_add_ns(base, nsecs);
2319
Thomas Gleixner868a3e92015-04-14 21:08:37 +00002320 if (*cwsseq != tk->clock_was_set_seq) {
2321 *cwsseq = tk->clock_was_set_seq;
2322 *offs_real = tk->offs_real;
Thomas Gleixnera3ed0e432018-04-25 15:33:38 +02002323 *offs_boot = tk->offs_boot;
Thomas Gleixner868a3e92015-04-14 21:08:37 +00002324 *offs_tai = tk->offs_tai;
2325 }
John Stultz833f32d2015-06-11 15:54:55 -07002326
2327 /* Handle leapsecond insertion adjustments */
Thomas Gleixner2456e852016-12-25 11:38:40 +01002328 if (unlikely(base >= tk->next_leap_ktime))
John Stultz833f32d2015-06-11 15:54:55 -07002329 *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0));
2330
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002331 } while (read_seqcount_retry(&tk_core.seq, seq));
Thomas Gleixnerf6c06ab2012-07-10 18:43:24 -04002332
John Stultz833f32d2015-06-11 15:54:55 -07002333 return base;
Thomas Gleixnerf6c06ab2012-07-10 18:43:24 -04002334}
Thomas Gleixnerf6c06ab2012-07-10 18:43:24 -04002335
Alex Shi199d2802020-11-13 15:24:33 +08002336/*
Arnd Bergmann1572fa02017-10-19 13:14:45 +02002337 * timekeeping_validate_timex - Ensures the timex is ok for use in do_adjtimex
Arnd Bergmanne0956dc2017-10-19 13:14:44 +02002338 */
Deepa Dinamaniead25412018-07-02 22:44:21 -07002339static int timekeeping_validate_timex(const struct __kernel_timex *txc)
Arnd Bergmanne0956dc2017-10-19 13:14:44 +02002340{
2341 if (txc->modes & ADJ_ADJTIME) {
2342 /* singleshot must not be used with any other mode bits */
2343 if (!(txc->modes & ADJ_OFFSET_SINGLESHOT))
2344 return -EINVAL;
2345 if (!(txc->modes & ADJ_OFFSET_READONLY) &&
2346 !capable(CAP_SYS_TIME))
2347 return -EPERM;
2348 } else {
2349 /* In order to modify anything, you gotta be super-user! */
2350 if (txc->modes && !capable(CAP_SYS_TIME))
2351 return -EPERM;
2352 /*
2353 * if the quartz is off by more than 10% then
2354 * something is VERY wrong!
2355 */
2356 if (txc->modes & ADJ_TICK &&
2357 (txc->tick < 900000/USER_HZ ||
2358 txc->tick > 1100000/USER_HZ))
2359 return -EINVAL;
2360 }
2361
2362 if (txc->modes & ADJ_SETOFFSET) {
2363 /* In order to inject time, you gotta be super-user! */
2364 if (!capable(CAP_SYS_TIME))
2365 return -EPERM;
2366
Arnd Bergmann1572fa02017-10-19 13:14:45 +02002367 /*
2368 * Validate if a timespec/timeval used to inject a time
Ingo Molnar4bf07f62021-03-22 22:39:03 +01002369 * offset is valid. Offsets can be positive or negative, so
Arnd Bergmann1572fa02017-10-19 13:14:45 +02002370 * we don't check tv_sec. The value of the timeval/timespec
2371 * is the sum of its fields,but *NOTE*:
2372 * The field tv_usec/tv_nsec must always be non-negative and
2373 * we can't have more nanoseconds/microseconds than a second.
2374 */
2375 if (txc->time.tv_usec < 0)
2376 return -EINVAL;
2377
Arnd Bergmanne0956dc2017-10-19 13:14:44 +02002378 if (txc->modes & ADJ_NANO) {
Arnd Bergmann1572fa02017-10-19 13:14:45 +02002379 if (txc->time.tv_usec >= NSEC_PER_SEC)
Arnd Bergmanne0956dc2017-10-19 13:14:44 +02002380 return -EINVAL;
Arnd Bergmanne0956dc2017-10-19 13:14:44 +02002381 } else {
Arnd Bergmann1572fa02017-10-19 13:14:45 +02002382 if (txc->time.tv_usec >= USEC_PER_SEC)
Arnd Bergmanne0956dc2017-10-19 13:14:44 +02002383 return -EINVAL;
2384 }
2385 }
2386
2387 /*
2388 * Check for potential multiplication overflows that can
2389 * only happen on 64-bit systems:
2390 */
2391 if ((txc->modes & ADJ_FREQUENCY) && (BITS_PER_LONG == 64)) {
2392 if (LLONG_MIN / PPM_SCALE > txc->freq)
2393 return -EINVAL;
2394 if (LLONG_MAX / PPM_SCALE < txc->freq)
2395 return -EINVAL;
2396 }
2397
2398 return 0;
2399}
2400
Jason A. Donenfeld13669922022-04-10 16:49:50 +02002401/**
2402 * random_get_entropy_fallback - Returns the raw clock source value,
2403 * used by random.c for platforms with no valid random_get_entropy().
2404 */
2405unsigned long random_get_entropy_fallback(void)
2406{
2407 struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono;
2408 struct clocksource *clock = READ_ONCE(tkr->clock);
2409
2410 if (unlikely(timekeeping_suspended || !clock))
2411 return 0;
2412 return clock->read(clock);
2413}
2414EXPORT_SYMBOL_GPL(random_get_entropy_fallback);
Arnd Bergmanne0956dc2017-10-19 13:14:44 +02002415
2416/**
John Stultzaa6f9c592013-03-22 11:31:29 -07002417 * do_adjtimex() - Accessor function to NTP __do_adjtimex function
2418 */
Deepa Dinamaniead25412018-07-02 22:44:21 -07002419int do_adjtimex(struct __kernel_timex *txc)
John Stultzaa6f9c592013-03-22 11:31:29 -07002420{
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002421 struct timekeeper *tk = &tk_core.timekeeper;
Ondrej Mosnacek7e8eda72019-04-10 11:14:20 +02002422 struct audit_ntp_data ad;
Thomas Gleixner1b267792021-07-13 15:39:52 +02002423 bool clock_set = false;
John Stultz7d489d12014-07-16 21:04:01 +00002424 struct timespec64 ts;
Thomas Gleixner1b267792021-07-13 15:39:52 +02002425 unsigned long flags;
John Stultz4e8f8b32013-04-10 12:41:49 -07002426 s32 orig_tai, tai;
John Stultze4085692013-03-22 12:08:52 -07002427 int ret;
2428
2429 /* Validate the data before disabling interrupts */
Arnd Bergmann1572fa02017-10-19 13:14:45 +02002430 ret = timekeeping_validate_timex(txc);
John Stultze4085692013-03-22 12:08:52 -07002431 if (ret)
2432 return ret;
2433
John Stultzcef90372013-03-22 15:04:13 -07002434 if (txc->modes & ADJ_SETOFFSET) {
Arnd Bergmann1572fa02017-10-19 13:14:45 +02002435 struct timespec64 delta;
John Stultzcef90372013-03-22 15:04:13 -07002436 delta.tv_sec = txc->time.tv_sec;
2437 delta.tv_nsec = txc->time.tv_usec;
2438 if (!(txc->modes & ADJ_NANO))
2439 delta.tv_nsec *= 1000;
2440 ret = timekeeping_inject_offset(&delta);
2441 if (ret)
2442 return ret;
Ondrej Mosnacek2d87a062019-04-10 11:14:19 +02002443
2444 audit_tk_injoffset(delta);
John Stultzcef90372013-03-22 15:04:13 -07002445 }
2446
Ondrej Mosnacek7e8eda72019-04-10 11:14:20 +02002447 audit_ntp_init(&ad);
2448
Arnd Bergmannd30faff2018-06-18 16:08:01 +02002449 ktime_get_real_ts64(&ts);
John Stultzaa6f9c592013-03-22 11:31:29 -07002450
John Stultz06c017f2013-03-22 11:37:28 -07002451 raw_spin_lock_irqsave(&timekeeper_lock, flags);
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002452 write_seqcount_begin(&tk_core.seq);
John Stultz06c017f2013-03-22 11:37:28 -07002453
John Stultz4e8f8b32013-04-10 12:41:49 -07002454 orig_tai = tai = tk->tai_offset;
Ondrej Mosnacek7e8eda72019-04-10 11:14:20 +02002455 ret = __do_adjtimex(txc, &ts, &tai, &ad);
John Stultz87ace392013-03-22 12:28:15 -07002456
John Stultz4e8f8b32013-04-10 12:41:49 -07002457 if (tai != orig_tai) {
2458 __timekeeping_set_tai_offset(tk, tai);
John Stultzf55c0762013-12-11 18:50:25 -08002459 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET);
Thomas Gleixner1b267792021-07-13 15:39:52 +02002460 clock_set = true;
John Stultz4e8f8b32013-04-10 12:41:49 -07002461 }
John Stultz833f32d2015-06-11 15:54:55 -07002462 tk_update_leap_state(tk);
2463
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002464 write_seqcount_end(&tk_core.seq);
John Stultz06c017f2013-03-22 11:37:28 -07002465 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
2466
Ondrej Mosnacek7e8eda72019-04-10 11:14:20 +02002467 audit_ntp_log(&ad);
2468
Miroslav Lichvarb061c7a2018-06-04 15:34:21 +02002469 /* Update the multiplier immediately if frequency was set directly */
2470 if (txc->modes & (ADJ_FREQUENCY | ADJ_TICK))
Thomas Gleixner1b267792021-07-13 15:39:52 +02002471 clock_set |= timekeeping_advance(TK_ADV_FREQ);
Miroslav Lichvarb061c7a2018-06-04 15:34:21 +02002472
Thomas Gleixner1b267792021-07-13 15:39:52 +02002473 if (clock_set)
Thomas Gleixner17a1b882021-07-13 15:39:53 +02002474 clock_was_set(CLOCK_REALTIME);
John Stultz6fdda9a2013-12-10 17:18:18 -08002475
John Stultz7bd36012013-09-11 16:50:56 -07002476 ntp_notify_cmos_timer();
2477
John Stultz87ace392013-03-22 12:28:15 -07002478 return ret;
2479}
John Stultzaa6f9c592013-03-22 11:31:29 -07002480
2481#ifdef CONFIG_NTP_PPS
2482/**
2483 * hardpps() - Accessor function to NTP __hardpps function
2484 */
Arnd Bergmann7ec88e42015-09-28 22:21:28 +02002485void hardpps(const struct timespec64 *phase_ts, const struct timespec64 *raw_ts)
John Stultzaa6f9c592013-03-22 11:31:29 -07002486{
John Stultz06c017f2013-03-22 11:37:28 -07002487 unsigned long flags;
2488
2489 raw_spin_lock_irqsave(&timekeeper_lock, flags);
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002490 write_seqcount_begin(&tk_core.seq);
John Stultz06c017f2013-03-22 11:37:28 -07002491
John Stultzaa6f9c592013-03-22 11:31:29 -07002492 __hardpps(phase_ts, raw_ts);
John Stultz06c017f2013-03-22 11:37:28 -07002493
Thomas Gleixner3fdb14f2014-07-16 21:04:07 +00002494 write_seqcount_end(&tk_core.seq);
John Stultz06c017f2013-03-22 11:37:28 -07002495 raw_spin_unlock_irqrestore(&timekeeper_lock, flags);
John Stultzaa6f9c592013-03-22 11:31:29 -07002496}
2497EXPORT_SYMBOL(hardpps);
Robert P. J. Daya2d81802017-09-08 16:17:19 -07002498#endif /* CONFIG_NTP_PPS */