blob: bad96b476eb6eb13c1d7f1774a1348506d0e5f10 [file] [log] [blame]
Jason Baronbf5438fc2010-09-17 11:09:00 -04001/*
2 * jump label support
3 *
4 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
Peter Zijlstra90eec102015-11-16 11:08:45 +01005 * Copyright (C) 2011 Peter Zijlstra
Jason Baronbf5438fc2010-09-17 11:09:00 -04006 *
7 */
Jason Baronbf5438fc2010-09-17 11:09:00 -04008#include <linux/memory.h>
9#include <linux/uaccess.h>
10#include <linux/module.h>
11#include <linux/list.h>
Jason Baronbf5438fc2010-09-17 11:09:00 -040012#include <linux/slab.h>
13#include <linux/sort.h>
14#include <linux/err.h>
Ingo Molnarc5905af2012-02-24 08:31:31 +010015#include <linux/static_key.h>
Andrew Jones851cf6e2013-08-09 19:51:57 +053016#include <linux/jump_label_ratelimit.h>
Jason Baron1f69bf92016-08-03 13:46:36 -070017#include <linux/bug.h>
Thomas Gleixnerf2545b2d2017-05-24 10:15:35 +020018#include <linux/cpu.h>
Josh Poimboeuf578ae442018-03-19 13:18:57 -050019#include <asm/sections.h>
Jason Baronbf5438fc2010-09-17 11:09:00 -040020
Jason Baronbf5438fc2010-09-17 11:09:00 -040021/* mutex to protect coming/going of the the jump_label table */
22static DEFINE_MUTEX(jump_label_mutex);
23
Jason Baron91bad2f2010-10-01 17:23:48 -040024void jump_label_lock(void)
25{
26 mutex_lock(&jump_label_mutex);
27}
28
29void jump_label_unlock(void)
30{
31 mutex_unlock(&jump_label_mutex);
32}
33
Jason Baronbf5438fc2010-09-17 11:09:00 -040034static int jump_label_cmp(const void *a, const void *b)
35{
36 const struct jump_entry *jea = a;
37 const struct jump_entry *jeb = b;
38
Ard Biesheuvel9ae033a2018-09-18 23:51:36 -070039 if (jump_entry_key(jea) < jump_entry_key(jeb))
Jason Baronbf5438fc2010-09-17 11:09:00 -040040 return -1;
41
Ard Biesheuvel9ae033a2018-09-18 23:51:36 -070042 if (jump_entry_key(jea) > jump_entry_key(jeb))
Jason Baronbf5438fc2010-09-17 11:09:00 -040043 return 1;
44
45 return 0;
46}
47
Ard Biesheuvel50ff18a2018-09-18 23:51:37 -070048static void jump_label_swap(void *a, void *b, int size)
49{
50 long delta = (unsigned long)a - (unsigned long)b;
51 struct jump_entry *jea = a;
52 struct jump_entry *jeb = b;
53 struct jump_entry tmp = *jea;
54
55 jea->code = jeb->code - delta;
56 jea->target = jeb->target - delta;
57 jea->key = jeb->key - delta;
58
59 jeb->code = tmp.code + delta;
60 jeb->target = tmp.target + delta;
61 jeb->key = tmp.key + delta;
62}
63
Jason Baronbf5438fc2010-09-17 11:09:00 -040064static void
Jason Barond430d3d2011-03-16 17:29:47 -040065jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
Jason Baronbf5438fc2010-09-17 11:09:00 -040066{
67 unsigned long size;
Ard Biesheuvel50ff18a2018-09-18 23:51:37 -070068 void *swapfn = NULL;
69
70 if (IS_ENABLED(CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE))
71 swapfn = jump_label_swap;
Jason Baronbf5438fc2010-09-17 11:09:00 -040072
73 size = (((unsigned long)stop - (unsigned long)start)
74 / sizeof(struct jump_entry));
Ard Biesheuvel50ff18a2018-09-18 23:51:37 -070075 sort(start, size, sizeof(struct jump_entry), jump_label_cmp, swapfn);
Jason Baronbf5438fc2010-09-17 11:09:00 -040076}
77
Peter Zijlstra706249c2015-07-24 15:06:37 +020078static void jump_label_update(struct static_key *key);
Peter Zijlstraa1efb012015-07-24 14:55:40 +020079
Jason Baron1f69bf92016-08-03 13:46:36 -070080/*
Masahiro Yamadae9666d12018-12-31 00:14:15 +090081 * There are similar definitions for the !CONFIG_JUMP_LABEL case in jump_label.h.
Jason Baron1f69bf92016-08-03 13:46:36 -070082 * The use of 'atomic_read()' requires atomic.h and its problematic for some
83 * kernel headers such as kernel.h and others. Since static_key_count() is not
Masahiro Yamadae9666d12018-12-31 00:14:15 +090084 * used in the branch statements as it is for the !CONFIG_JUMP_LABEL case its ok
Jason Baron1f69bf92016-08-03 13:46:36 -070085 * to have it be a function here. Similarly, for 'static_key_enable()' and
86 * 'static_key_disable()', which require bug.h. This should allow jump_label.h
Masahiro Yamadae9666d12018-12-31 00:14:15 +090087 * to be included from most/all places for CONFIG_JUMP_LABEL.
Jason Baron1f69bf92016-08-03 13:46:36 -070088 */
89int static_key_count(struct static_key *key)
90{
91 /*
92 * -1 means the first static_key_slow_inc() is in progress.
93 * static_key_enabled() must return true, so return 1 here.
94 */
95 int n = atomic_read(&key->enabled);
96
97 return n >= 0 ? n : 1;
98}
99EXPORT_SYMBOL_GPL(static_key_count);
100
Peter Zijlstrace48c1462018-01-22 22:53:28 +0100101void static_key_slow_inc_cpuslocked(struct static_key *key)
Jason Baronbf5438fc2010-09-17 11:09:00 -0400102{
Paolo Bonzini4c5ea0a2016-06-21 18:52:17 +0200103 int v, v1;
104
Borislav Petkov5cdda512017-10-18 17:24:28 +0200105 STATIC_KEY_CHECK_USE(key);
Peter Zijlstracb538262018-07-31 14:35:32 +0200106 lockdep_assert_cpus_held();
Paolo Bonzini4c5ea0a2016-06-21 18:52:17 +0200107
108 /*
109 * Careful if we get concurrent static_key_slow_inc() calls;
110 * later calls must wait for the first one to _finish_ the
111 * jump_label_update() process. At the same time, however,
112 * the jump_label_update() call below wants to see
113 * static_key_enabled(&key) for jumps to be updated properly.
114 *
115 * So give a special meaning to negative key->enabled: it sends
116 * static_key_slow_inc() down the slow path, and it is non-zero
117 * so it counts as "enabled" in jump_label_update(). Note that
118 * atomic_inc_unless_negative() checks >= 0, so roll our own.
119 */
120 for (v = atomic_read(&key->enabled); v > 0; v = v1) {
121 v1 = atomic_cmpxchg(&key->enabled, v, v + 1);
Marc Zyngier8b7b4122017-08-01 09:02:55 +0100122 if (likely(v1 == v))
Paolo Bonzini4c5ea0a2016-06-21 18:52:17 +0200123 return;
124 }
Jason Baronbf5438fc2010-09-17 11:09:00 -0400125
Jason Baron91bad2f2010-10-01 17:23:48 -0400126 jump_label_lock();
Paolo Bonzini4c5ea0a2016-06-21 18:52:17 +0200127 if (atomic_read(&key->enabled) == 0) {
128 atomic_set(&key->enabled, -1);
Peter Zijlstra706249c2015-07-24 15:06:37 +0200129 jump_label_update(key);
Peter Zijlstrad0646a62017-08-01 23:58:50 +0200130 /*
131 * Ensure that if the above cmpxchg loop observes our positive
132 * value, it must also observe all the text changes.
133 */
134 atomic_set_release(&key->enabled, 1);
Paolo Bonzini4c5ea0a2016-06-21 18:52:17 +0200135 } else {
136 atomic_inc(&key->enabled);
137 }
Jason Barond430d3d2011-03-16 17:29:47 -0400138 jump_label_unlock();
Marc Zyngier8b7b4122017-08-01 09:02:55 +0100139}
140
141void static_key_slow_inc(struct static_key *key)
142{
143 cpus_read_lock();
144 static_key_slow_inc_cpuslocked(key);
Thomas Gleixnerf2545b2d2017-05-24 10:15:35 +0200145 cpus_read_unlock();
Jason Barond430d3d2011-03-16 17:29:47 -0400146}
Ingo Molnarc5905af2012-02-24 08:31:31 +0100147EXPORT_SYMBOL_GPL(static_key_slow_inc);
Jason Barond430d3d2011-03-16 17:29:47 -0400148
Marc Zyngier5a405272017-08-01 09:02:56 +0100149void static_key_enable_cpuslocked(struct static_key *key)
Paolo Bonzini1dbb6702017-08-01 17:24:04 +0200150{
Borislav Petkov5cdda512017-10-18 17:24:28 +0200151 STATIC_KEY_CHECK_USE(key);
Peter Zijlstracb538262018-07-31 14:35:32 +0200152 lockdep_assert_cpus_held();
Marc Zyngier5a405272017-08-01 09:02:56 +0100153
Paolo Bonzini1dbb6702017-08-01 17:24:04 +0200154 if (atomic_read(&key->enabled) > 0) {
155 WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
156 return;
157 }
158
Paolo Bonzini1dbb6702017-08-01 17:24:04 +0200159 jump_label_lock();
160 if (atomic_read(&key->enabled) == 0) {
161 atomic_set(&key->enabled, -1);
162 jump_label_update(key);
Peter Zijlstrad0646a62017-08-01 23:58:50 +0200163 /*
164 * See static_key_slow_inc().
165 */
166 atomic_set_release(&key->enabled, 1);
Paolo Bonzini1dbb6702017-08-01 17:24:04 +0200167 }
168 jump_label_unlock();
Marc Zyngier5a405272017-08-01 09:02:56 +0100169}
170EXPORT_SYMBOL_GPL(static_key_enable_cpuslocked);
171
172void static_key_enable(struct static_key *key)
173{
174 cpus_read_lock();
175 static_key_enable_cpuslocked(key);
Paolo Bonzini1dbb6702017-08-01 17:24:04 +0200176 cpus_read_unlock();
177}
178EXPORT_SYMBOL_GPL(static_key_enable);
179
Marc Zyngier5a405272017-08-01 09:02:56 +0100180void static_key_disable_cpuslocked(struct static_key *key)
Paolo Bonzini1dbb6702017-08-01 17:24:04 +0200181{
Borislav Petkov5cdda512017-10-18 17:24:28 +0200182 STATIC_KEY_CHECK_USE(key);
Peter Zijlstracb538262018-07-31 14:35:32 +0200183 lockdep_assert_cpus_held();
Marc Zyngier5a405272017-08-01 09:02:56 +0100184
Paolo Bonzini1dbb6702017-08-01 17:24:04 +0200185 if (atomic_read(&key->enabled) != 1) {
186 WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
187 return;
188 }
189
Paolo Bonzini1dbb6702017-08-01 17:24:04 +0200190 jump_label_lock();
191 if (atomic_cmpxchg(&key->enabled, 1, 0))
192 jump_label_update(key);
193 jump_label_unlock();
Marc Zyngier5a405272017-08-01 09:02:56 +0100194}
195EXPORT_SYMBOL_GPL(static_key_disable_cpuslocked);
196
197void static_key_disable(struct static_key *key)
198{
199 cpus_read_lock();
200 static_key_disable_cpuslocked(key);
Paolo Bonzini1dbb6702017-08-01 17:24:04 +0200201 cpus_read_unlock();
202}
203EXPORT_SYMBOL_GPL(static_key_disable);
204
Peter Zijlstrace48c1462018-01-22 22:53:28 +0100205static void __static_key_slow_dec_cpuslocked(struct static_key *key,
Marc Zyngier8b7b4122017-08-01 09:02:55 +0100206 unsigned long rate_limit,
207 struct delayed_work *work)
Jason Barond430d3d2011-03-16 17:29:47 -0400208{
Peter Zijlstracb538262018-07-31 14:35:32 +0200209 lockdep_assert_cpus_held();
210
Paolo Bonzini4c5ea0a2016-06-21 18:52:17 +0200211 /*
212 * The negative count check is valid even when a negative
213 * key->enabled is in use by static_key_slow_inc(); a
214 * __static_key_slow_dec() before the first static_key_slow_inc()
215 * returns is unbalanced, because all other static_key_slow_inc()
216 * instances block while the update is in progress.
217 */
Jason Baronfadf0462012-02-21 15:02:53 -0500218 if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
219 WARN(atomic_read(&key->enabled) < 0,
220 "jump label: negative count!\n");
Jason Barond430d3d2011-03-16 17:29:47 -0400221 return;
Jason Baronfadf0462012-02-21 15:02:53 -0500222 }
Jason Barond430d3d2011-03-16 17:29:47 -0400223
Gleb Natapovb2029522011-11-27 17:59:09 +0200224 if (rate_limit) {
225 atomic_inc(&key->enabled);
226 schedule_delayed_work(work, rate_limit);
Ingo Molnarc5905af2012-02-24 08:31:31 +0100227 } else {
Peter Zijlstra706249c2015-07-24 15:06:37 +0200228 jump_label_update(key);
Ingo Molnarc5905af2012-02-24 08:31:31 +0100229 }
Jason Baron91bad2f2010-10-01 17:23:48 -0400230 jump_label_unlock();
Marc Zyngier8b7b4122017-08-01 09:02:55 +0100231}
232
233static void __static_key_slow_dec(struct static_key *key,
234 unsigned long rate_limit,
235 struct delayed_work *work)
236{
237 cpus_read_lock();
Peter Zijlstrace48c1462018-01-22 22:53:28 +0100238 __static_key_slow_dec_cpuslocked(key, rate_limit, work);
Thomas Gleixnerf2545b2d2017-05-24 10:15:35 +0200239 cpus_read_unlock();
Jason Baronbf5438fc2010-09-17 11:09:00 -0400240}
241
Gleb Natapovb2029522011-11-27 17:59:09 +0200242static void jump_label_update_timeout(struct work_struct *work)
243{
Ingo Molnarc5905af2012-02-24 08:31:31 +0100244 struct static_key_deferred *key =
245 container_of(work, struct static_key_deferred, work.work);
246 __static_key_slow_dec(&key->key, 0, NULL);
Gleb Natapovb2029522011-11-27 17:59:09 +0200247}
248
Ingo Molnarc5905af2012-02-24 08:31:31 +0100249void static_key_slow_dec(struct static_key *key)
Gleb Natapovb2029522011-11-27 17:59:09 +0200250{
Borislav Petkov5cdda512017-10-18 17:24:28 +0200251 STATIC_KEY_CHECK_USE(key);
Ingo Molnarc5905af2012-02-24 08:31:31 +0100252 __static_key_slow_dec(key, 0, NULL);
Gleb Natapovb2029522011-11-27 17:59:09 +0200253}
Ingo Molnarc5905af2012-02-24 08:31:31 +0100254EXPORT_SYMBOL_GPL(static_key_slow_dec);
Gleb Natapovb2029522011-11-27 17:59:09 +0200255
Peter Zijlstrace48c1462018-01-22 22:53:28 +0100256void static_key_slow_dec_cpuslocked(struct static_key *key)
257{
258 STATIC_KEY_CHECK_USE(key);
259 __static_key_slow_dec_cpuslocked(key, 0, NULL);
260}
261
Ingo Molnarc5905af2012-02-24 08:31:31 +0100262void static_key_slow_dec_deferred(struct static_key_deferred *key)
Gleb Natapovb2029522011-11-27 17:59:09 +0200263{
Borislav Petkov5cdda512017-10-18 17:24:28 +0200264 STATIC_KEY_CHECK_USE(key);
Ingo Molnarc5905af2012-02-24 08:31:31 +0100265 __static_key_slow_dec(&key->key, key->timeout, &key->work);
Gleb Natapovb2029522011-11-27 17:59:09 +0200266}
Ingo Molnarc5905af2012-02-24 08:31:31 +0100267EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
Gleb Natapovb2029522011-11-27 17:59:09 +0200268
David Matlackb6416e62016-12-16 14:30:35 -0800269void static_key_deferred_flush(struct static_key_deferred *key)
270{
Borislav Petkov5cdda512017-10-18 17:24:28 +0200271 STATIC_KEY_CHECK_USE(key);
David Matlackb6416e62016-12-16 14:30:35 -0800272 flush_delayed_work(&key->work);
273}
274EXPORT_SYMBOL_GPL(static_key_deferred_flush);
275
Ingo Molnarc5905af2012-02-24 08:31:31 +0100276void jump_label_rate_limit(struct static_key_deferred *key,
Gleb Natapovb2029522011-11-27 17:59:09 +0200277 unsigned long rl)
278{
Borislav Petkov5cdda512017-10-18 17:24:28 +0200279 STATIC_KEY_CHECK_USE(key);
Gleb Natapovb2029522011-11-27 17:59:09 +0200280 key->timeout = rl;
281 INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);
282}
Gleb Natapova181dc12012-08-05 15:58:29 +0300283EXPORT_SYMBOL_GPL(jump_label_rate_limit);
Gleb Natapovb2029522011-11-27 17:59:09 +0200284
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400285static int addr_conflict(struct jump_entry *entry, void *start, void *end)
286{
Ard Biesheuvel9ae033a2018-09-18 23:51:36 -0700287 if (jump_entry_code(entry) <= (unsigned long)end &&
288 jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400289 return 1;
290
291 return 0;
292}
293
Jason Barond430d3d2011-03-16 17:29:47 -0400294static int __jump_label_text_reserved(struct jump_entry *iter_start,
295 struct jump_entry *iter_stop, void *start, void *end)
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400296{
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400297 struct jump_entry *iter;
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400298
Jason Barond430d3d2011-03-16 17:29:47 -0400299 iter = iter_start;
300 while (iter < iter_stop) {
301 if (addr_conflict(iter, start, end))
302 return 1;
303 iter++;
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400304 }
Jason Barond430d3d2011-03-16 17:29:47 -0400305
306 return 0;
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400307}
308
Peter Zijlstra706249c2015-07-24 15:06:37 +0200309/*
Jeremy Fitzhardinge20284aa2011-10-03 11:01:46 -0700310 * Update code which is definitely not currently executing.
311 * Architectures which need heavyweight synchronization to modify
312 * running code can override this to make the non-live update case
313 * cheaper.
314 */
Peter Zijlstra9cdbe1c2011-12-06 17:27:29 +0100315void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry *entry,
Jeremy Fitzhardinge20284aa2011-10-03 11:01:46 -0700316 enum jump_label_type type)
317{
Peter Zijlstra706249c2015-07-24 15:06:37 +0200318 arch_jump_label_transform(entry, type);
Jason Barond430d3d2011-03-16 17:29:47 -0400319}
320
Peter Zijlstraa1efb012015-07-24 14:55:40 +0200321static inline struct jump_entry *static_key_entries(struct static_key *key)
322{
Jason Baron3821fd32017-02-03 15:42:24 -0500323 WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED);
324 return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK);
Peter Zijlstraa1efb012015-07-24 14:55:40 +0200325}
326
Peter Zijlstra706249c2015-07-24 15:06:37 +0200327static inline bool static_key_type(struct static_key *key)
328{
Jason Baron3821fd32017-02-03 15:42:24 -0500329 return key->type & JUMP_TYPE_TRUE;
330}
331
332static inline bool static_key_linked(struct static_key *key)
333{
334 return key->type & JUMP_TYPE_LINKED;
335}
336
337static inline void static_key_clear_linked(struct static_key *key)
338{
339 key->type &= ~JUMP_TYPE_LINKED;
340}
341
342static inline void static_key_set_linked(struct static_key *key)
343{
344 key->type |= JUMP_TYPE_LINKED;
Peter Zijlstra706249c2015-07-24 15:06:37 +0200345}
346
Jason Baron3821fd32017-02-03 15:42:24 -0500347/***
348 * A 'struct static_key' uses a union such that it either points directly
349 * to a table of 'struct jump_entry' or to a linked list of modules which in
350 * turn point to 'struct jump_entry' tables.
351 *
352 * The two lower bits of the pointer are used to keep track of which pointer
353 * type is in use and to store the initial branch direction, we use an access
354 * function which preserves these bits.
355 */
356static void static_key_set_entries(struct static_key *key,
357 struct jump_entry *entries)
358{
359 unsigned long type;
360
361 WARN_ON_ONCE((unsigned long)entries & JUMP_TYPE_MASK);
362 type = key->type & JUMP_TYPE_MASK;
363 key->entries = entries;
364 key->type |= type;
365}
366
Peter Zijlstra706249c2015-07-24 15:06:37 +0200367static enum jump_label_type jump_label_type(struct jump_entry *entry)
Ingo Molnarc5905af2012-02-24 08:31:31 +0100368{
Peter Zijlstra706249c2015-07-24 15:06:37 +0200369 struct static_key *key = jump_entry_key(entry);
Peter Zijlstraa1efb012015-07-24 14:55:40 +0200370 bool enabled = static_key_enabled(key);
Ard Biesheuvel9ae033a2018-09-18 23:51:36 -0700371 bool branch = jump_entry_is_branch(entry);
Ingo Molnarc5905af2012-02-24 08:31:31 +0100372
Peter Zijlstra11276d52015-07-24 15:09:55 +0200373 /* See the comment in linux/jump_label.h */
374 return enabled ^ branch;
Ingo Molnarc5905af2012-02-24 08:31:31 +0100375}
376
Peter Zijlstra706249c2015-07-24 15:06:37 +0200377static void __jump_label_update(struct static_key *key,
378 struct jump_entry *entry,
Ard Biesheuvel19483672018-09-18 23:51:42 -0700379 struct jump_entry *stop,
380 bool init)
Peter Zijlstra706249c2015-07-24 15:06:37 +0200381{
382 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
383 /*
Josh Poimboeufdc1dd182018-02-20 11:37:52 -0600384 * An entry->code of 0 indicates an entry which has been
385 * disabled because it was in an init text area.
Peter Zijlstra706249c2015-07-24 15:06:37 +0200386 */
Ard Biesheuvel19483672018-09-18 23:51:42 -0700387 if (init || !jump_entry_is_init(entry)) {
Ard Biesheuvel9ae033a2018-09-18 23:51:36 -0700388 if (kernel_text_address(jump_entry_code(entry)))
Josh Poimboeufdc1dd182018-02-20 11:37:52 -0600389 arch_jump_label_transform(entry, jump_label_type(entry));
390 else
Josh Poimboeufaf1d8302018-03-14 10:24:20 -0500391 WARN_ONCE(1, "can't patch jump_label at %pS",
Ard Biesheuvel9ae033a2018-09-18 23:51:36 -0700392 (void *)jump_entry_code(entry));
Josh Poimboeufdc1dd182018-02-20 11:37:52 -0600393 }
Peter Zijlstra706249c2015-07-24 15:06:37 +0200394 }
395}
396
Jeremy Fitzhardinge97ce2c82011-10-12 16:17:54 -0700397void __init jump_label_init(void)
Jason Barond430d3d2011-03-16 17:29:47 -0400398{
399 struct jump_entry *iter_start = __start___jump_table;
400 struct jump_entry *iter_stop = __stop___jump_table;
Ingo Molnarc5905af2012-02-24 08:31:31 +0100401 struct static_key *key = NULL;
Jason Barond430d3d2011-03-16 17:29:47 -0400402 struct jump_entry *iter;
403
Jason Baron1f69bf92016-08-03 13:46:36 -0700404 /*
405 * Since we are initializing the static_key.enabled field with
406 * with the 'raw' int values (to avoid pulling in atomic.h) in
407 * jump_label.h, let's make sure that is safe. There are only two
408 * cases to check since we initialize to 0 or 1.
409 */
410 BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0);
411 BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1);
412
Kevin Haoe3f91082016-07-23 14:42:37 +0530413 if (static_key_initialized)
414 return;
415
Thomas Gleixnerf2545b2d2017-05-24 10:15:35 +0200416 cpus_read_lock();
Jason Barond430d3d2011-03-16 17:29:47 -0400417 jump_label_lock();
418 jump_label_sort_entries(iter_start, iter_stop);
419
420 for (iter = iter_start; iter < iter_stop; iter++) {
Ingo Molnarc5905af2012-02-24 08:31:31 +0100421 struct static_key *iterk;
Jeremy Fitzhardinge37348802011-09-29 11:10:05 -0700422
Peter Zijlstra11276d52015-07-24 15:09:55 +0200423 /* rewrite NOPs */
424 if (jump_label_type(iter) == JUMP_LABEL_NOP)
425 arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
426
Ard Biesheuvel19483672018-09-18 23:51:42 -0700427 if (init_section_contains((void *)jump_entry_code(iter), 1))
428 jump_entry_set_init(iter);
429
Peter Zijlstra7dcfd912015-07-24 15:02:27 +0200430 iterk = jump_entry_key(iter);
Jeremy Fitzhardinge37348802011-09-29 11:10:05 -0700431 if (iterk == key)
Jason Barond430d3d2011-03-16 17:29:47 -0400432 continue;
433
Jeremy Fitzhardinge37348802011-09-29 11:10:05 -0700434 key = iterk;
Jason Baron3821fd32017-02-03 15:42:24 -0500435 static_key_set_entries(key, iter);
Jason Barond430d3d2011-03-16 17:29:47 -0400436 }
Hannes Frederic Sowac4b2c0c2013-10-19 21:48:53 +0200437 static_key_initialized = true;
Jason Barond430d3d2011-03-16 17:29:47 -0400438 jump_label_unlock();
Thomas Gleixnerf2545b2d2017-05-24 10:15:35 +0200439 cpus_read_unlock();
Jason Barond430d3d2011-03-16 17:29:47 -0400440}
Jason Barond430d3d2011-03-16 17:29:47 -0400441
442#ifdef CONFIG_MODULES
443
Peter Zijlstra11276d52015-07-24 15:09:55 +0200444static enum jump_label_type jump_label_init_type(struct jump_entry *entry)
445{
446 struct static_key *key = jump_entry_key(entry);
447 bool type = static_key_type(key);
Ard Biesheuvel9ae033a2018-09-18 23:51:36 -0700448 bool branch = jump_entry_is_branch(entry);
Peter Zijlstra11276d52015-07-24 15:09:55 +0200449
450 /* See the comment in linux/jump_label.h */
451 return type ^ branch;
452}
453
Ingo Molnarc5905af2012-02-24 08:31:31 +0100454struct static_key_mod {
455 struct static_key_mod *next;
Jason Barond430d3d2011-03-16 17:29:47 -0400456 struct jump_entry *entries;
457 struct module *mod;
458};
459
Jason Baron3821fd32017-02-03 15:42:24 -0500460static inline struct static_key_mod *static_key_mod(struct static_key *key)
461{
Borislav Petkov34e12b82018-09-09 13:42:52 +0200462 WARN_ON_ONCE(!static_key_linked(key));
Jason Baron3821fd32017-02-03 15:42:24 -0500463 return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);
464}
465
466/***
467 * key->type and key->next are the same via union.
468 * This sets key->next and preserves the type bits.
469 *
470 * See additional comments above static_key_set_entries().
471 */
472static void static_key_set_mod(struct static_key *key,
473 struct static_key_mod *mod)
474{
475 unsigned long type;
476
477 WARN_ON_ONCE((unsigned long)mod & JUMP_TYPE_MASK);
478 type = key->type & JUMP_TYPE_MASK;
479 key->next = mod;
480 key->type |= type;
481}
482
Jason Barond430d3d2011-03-16 17:29:47 -0400483static int __jump_label_mod_text_reserved(void *start, void *end)
484{
485 struct module *mod;
486
Rusty Russellbdc9f372016-07-27 12:17:35 +0930487 preempt_disable();
Jason Barond430d3d2011-03-16 17:29:47 -0400488 mod = __module_text_address((unsigned long)start);
Rusty Russellbdc9f372016-07-27 12:17:35 +0930489 WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
490 preempt_enable();
491
Jason Barond430d3d2011-03-16 17:29:47 -0400492 if (!mod)
493 return 0;
494
Jason Barond430d3d2011-03-16 17:29:47 -0400495
496 return __jump_label_text_reserved(mod->jump_entries,
497 mod->jump_entries + mod->num_jump_entries,
498 start, end);
499}
500
Peter Zijlstra706249c2015-07-24 15:06:37 +0200501static void __jump_label_mod_update(struct static_key *key)
Jason Barond430d3d2011-03-16 17:29:47 -0400502{
Peter Zijlstra706249c2015-07-24 15:06:37 +0200503 struct static_key_mod *mod;
Jason Barond430d3d2011-03-16 17:29:47 -0400504
Jason Baron3821fd32017-02-03 15:42:24 -0500505 for (mod = static_key_mod(key); mod; mod = mod->next) {
506 struct jump_entry *stop;
507 struct module *m;
Jiri Olsa7cbc5b82011-05-10 12:43:46 +0200508
Jason Baron3821fd32017-02-03 15:42:24 -0500509 /*
510 * NULL if the static_key is defined in a module
511 * that does not use it
512 */
513 if (!mod->entries)
514 continue;
515
516 m = mod->mod;
517 if (!m)
518 stop = __stop___jump_table;
519 else
520 stop = m->jump_entries + m->num_jump_entries;
Ard Biesheuvel19483672018-09-18 23:51:42 -0700521 __jump_label_update(key, mod->entries, stop,
Ard Biesheuvel77ac1c02018-10-01 10:13:24 +0200522 m && m->state == MODULE_STATE_COMING);
Jason Barond430d3d2011-03-16 17:29:47 -0400523 }
524}
525
526/***
527 * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
528 * @mod: module to patch
529 *
530 * Allow for run-time selection of the optimal nops. Before the module
531 * loads patch these with arch_get_jump_label_nop(), which is specified by
532 * the arch specific jump label code.
533 */
534void jump_label_apply_nops(struct module *mod)
535{
536 struct jump_entry *iter_start = mod->jump_entries;
537 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
538 struct jump_entry *iter;
539
540 /* if the module doesn't have jump label entries, just return */
541 if (iter_start == iter_stop)
542 return;
543
Peter Zijlstra11276d52015-07-24 15:09:55 +0200544 for (iter = iter_start; iter < iter_stop; iter++) {
545 /* Only write NOPs for arch_branch_static(). */
546 if (jump_label_init_type(iter) == JUMP_LABEL_NOP)
547 arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
548 }
Jason Barond430d3d2011-03-16 17:29:47 -0400549}
550
551static int jump_label_add_module(struct module *mod)
552{
553 struct jump_entry *iter_start = mod->jump_entries;
554 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
555 struct jump_entry *iter;
Ingo Molnarc5905af2012-02-24 08:31:31 +0100556 struct static_key *key = NULL;
Jason Baron3821fd32017-02-03 15:42:24 -0500557 struct static_key_mod *jlm, *jlm2;
Jason Barond430d3d2011-03-16 17:29:47 -0400558
559 /* if the module doesn't have jump label entries, just return */
560 if (iter_start == iter_stop)
561 return 0;
562
563 jump_label_sort_entries(iter_start, iter_stop);
564
565 for (iter = iter_start; iter < iter_stop; iter++) {
Ingo Molnarc5905af2012-02-24 08:31:31 +0100566 struct static_key *iterk;
567
Ard Biesheuvel19483672018-09-18 23:51:42 -0700568 if (within_module_init(jump_entry_code(iter), mod))
569 jump_entry_set_init(iter);
570
Peter Zijlstra7dcfd912015-07-24 15:02:27 +0200571 iterk = jump_entry_key(iter);
Ingo Molnarc5905af2012-02-24 08:31:31 +0100572 if (iterk == key)
Jason Barond430d3d2011-03-16 17:29:47 -0400573 continue;
574
Ingo Molnarc5905af2012-02-24 08:31:31 +0100575 key = iterk;
Ard Biesheuvel9ae033a2018-09-18 23:51:36 -0700576 if (within_module((unsigned long)key, mod)) {
Jason Baron3821fd32017-02-03 15:42:24 -0500577 static_key_set_entries(key, iter);
Jason Barond430d3d2011-03-16 17:29:47 -0400578 continue;
579 }
Ingo Molnarc5905af2012-02-24 08:31:31 +0100580 jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
Jason Barond430d3d2011-03-16 17:29:47 -0400581 if (!jlm)
582 return -ENOMEM;
Jason Baron3821fd32017-02-03 15:42:24 -0500583 if (!static_key_linked(key)) {
584 jlm2 = kzalloc(sizeof(struct static_key_mod),
585 GFP_KERNEL);
586 if (!jlm2) {
587 kfree(jlm);
588 return -ENOMEM;
589 }
590 preempt_disable();
591 jlm2->mod = __module_address((unsigned long)key);
592 preempt_enable();
593 jlm2->entries = static_key_entries(key);
594 jlm2->next = NULL;
595 static_key_set_mod(key, jlm2);
596 static_key_set_linked(key);
597 }
Jason Barond430d3d2011-03-16 17:29:47 -0400598 jlm->mod = mod;
599 jlm->entries = iter;
Jason Baron3821fd32017-02-03 15:42:24 -0500600 jlm->next = static_key_mod(key);
601 static_key_set_mod(key, jlm);
602 static_key_set_linked(key);
Jason Barond430d3d2011-03-16 17:29:47 -0400603
Peter Zijlstra11276d52015-07-24 15:09:55 +0200604 /* Only update if we've changed from our initial state */
605 if (jump_label_type(iter) != jump_label_init_type(iter))
Ard Biesheuvel19483672018-09-18 23:51:42 -0700606 __jump_label_update(key, iter, iter_stop, true);
Jason Barond430d3d2011-03-16 17:29:47 -0400607 }
608
609 return 0;
610}
611
612static void jump_label_del_module(struct module *mod)
613{
614 struct jump_entry *iter_start = mod->jump_entries;
615 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
616 struct jump_entry *iter;
Ingo Molnarc5905af2012-02-24 08:31:31 +0100617 struct static_key *key = NULL;
618 struct static_key_mod *jlm, **prev;
Jason Barond430d3d2011-03-16 17:29:47 -0400619
620 for (iter = iter_start; iter < iter_stop; iter++) {
Peter Zijlstra7dcfd912015-07-24 15:02:27 +0200621 if (jump_entry_key(iter) == key)
Jason Barond430d3d2011-03-16 17:29:47 -0400622 continue;
623
Peter Zijlstra7dcfd912015-07-24 15:02:27 +0200624 key = jump_entry_key(iter);
Jason Barond430d3d2011-03-16 17:29:47 -0400625
Ard Biesheuvel9ae033a2018-09-18 23:51:36 -0700626 if (within_module((unsigned long)key, mod))
Jason Barond430d3d2011-03-16 17:29:47 -0400627 continue;
628
Jason Baron3821fd32017-02-03 15:42:24 -0500629 /* No memory during module load */
630 if (WARN_ON(!static_key_linked(key)))
631 continue;
632
Jason Barond430d3d2011-03-16 17:29:47 -0400633 prev = &key->next;
Jason Baron3821fd32017-02-03 15:42:24 -0500634 jlm = static_key_mod(key);
Jason Barond430d3d2011-03-16 17:29:47 -0400635
636 while (jlm && jlm->mod != mod) {
637 prev = &jlm->next;
638 jlm = jlm->next;
639 }
640
Jason Baron3821fd32017-02-03 15:42:24 -0500641 /* No memory during module load */
642 if (WARN_ON(!jlm))
643 continue;
644
645 if (prev == &key->next)
646 static_key_set_mod(key, jlm->next);
647 else
Jason Barond430d3d2011-03-16 17:29:47 -0400648 *prev = jlm->next;
Jason Baron3821fd32017-02-03 15:42:24 -0500649
650 kfree(jlm);
651
652 jlm = static_key_mod(key);
653 /* if only one etry is left, fold it back into the static_key */
654 if (jlm->next == NULL) {
655 static_key_set_entries(key, jlm->entries);
656 static_key_clear_linked(key);
Jason Barond430d3d2011-03-16 17:29:47 -0400657 kfree(jlm);
658 }
659 }
660}
661
Jason Barond430d3d2011-03-16 17:29:47 -0400662static int
663jump_label_module_notify(struct notifier_block *self, unsigned long val,
664 void *data)
665{
666 struct module *mod = data;
667 int ret = 0;
668
Thomas Gleixnerf2545b2d2017-05-24 10:15:35 +0200669 cpus_read_lock();
670 jump_label_lock();
671
Jason Barond430d3d2011-03-16 17:29:47 -0400672 switch (val) {
673 case MODULE_STATE_COMING:
Jason Barond430d3d2011-03-16 17:29:47 -0400674 ret = jump_label_add_module(mod);
Jason Baron3821fd32017-02-03 15:42:24 -0500675 if (ret) {
Borislav Petkovda260fe2018-09-07 12:35:21 +0200676 WARN(1, "Failed to allocate memory: jump_label may not work properly.\n");
Jason Barond430d3d2011-03-16 17:29:47 -0400677 jump_label_del_module(mod);
Jason Baron3821fd32017-02-03 15:42:24 -0500678 }
Jason Barond430d3d2011-03-16 17:29:47 -0400679 break;
680 case MODULE_STATE_GOING:
Jason Barond430d3d2011-03-16 17:29:47 -0400681 jump_label_del_module(mod);
Jason Barond430d3d2011-03-16 17:29:47 -0400682 break;
Jason Barond430d3d2011-03-16 17:29:47 -0400683 }
684
Thomas Gleixnerf2545b2d2017-05-24 10:15:35 +0200685 jump_label_unlock();
686 cpus_read_unlock();
687
Jason Barond430d3d2011-03-16 17:29:47 -0400688 return notifier_from_errno(ret);
689}
690
Wei Yongjun885885f2016-06-17 17:19:40 +0000691static struct notifier_block jump_label_module_nb = {
Jason Barond430d3d2011-03-16 17:29:47 -0400692 .notifier_call = jump_label_module_notify,
693 .priority = 1, /* higher than tracepoints */
694};
695
696static __init int jump_label_init_module(void)
697{
698 return register_module_notifier(&jump_label_module_nb);
699}
700early_initcall(jump_label_init_module);
701
702#endif /* CONFIG_MODULES */
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400703
704/***
705 * jump_label_text_reserved - check if addr range is reserved
706 * @start: start text addr
707 * @end: end text addr
708 *
709 * checks if the text addr located between @start and @end
710 * overlaps with any of the jump label patch addresses. Code
711 * that wants to modify kernel text should first verify that
712 * it does not overlap with any of the jump label addresses.
Jason Baron91bad2f2010-10-01 17:23:48 -0400713 * Caller must hold jump_label_mutex.
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400714 *
715 * returns 1 if there is an overlap, 0 otherwise
716 */
717int jump_label_text_reserved(void *start, void *end)
718{
Jason Barond430d3d2011-03-16 17:29:47 -0400719 int ret = __jump_label_text_reserved(__start___jump_table,
720 __stop___jump_table, start, end);
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400721
Jason Barond430d3d2011-03-16 17:29:47 -0400722 if (ret)
723 return ret;
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400724
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400725#ifdef CONFIG_MODULES
Jason Barond430d3d2011-03-16 17:29:47 -0400726 ret = __jump_label_mod_text_reserved(start, end);
Jason Baron4c3ef6d2010-09-17 11:09:08 -0400727#endif
Jason Baronbf5438fc2010-09-17 11:09:00 -0400728 return ret;
729}
Jason Barond430d3d2011-03-16 17:29:47 -0400730
Peter Zijlstra706249c2015-07-24 15:06:37 +0200731static void jump_label_update(struct static_key *key)
Jason Barond430d3d2011-03-16 17:29:47 -0400732{
Ingo Molnarc5905af2012-02-24 08:31:31 +0100733 struct jump_entry *stop = __stop___jump_table;
Jason Baron3821fd32017-02-03 15:42:24 -0500734 struct jump_entry *entry;
Jason Baronbf5438fc2010-09-17 11:09:00 -0400735#ifdef CONFIG_MODULES
Peter Zijlstrabed831f2015-05-27 11:09:35 +0930736 struct module *mod;
Xiao Guangrong140fe3b2011-06-21 10:35:55 +0800737
Jason Baron3821fd32017-02-03 15:42:24 -0500738 if (static_key_linked(key)) {
739 __jump_label_mod_update(key);
740 return;
741 }
Xiao Guangrong140fe3b2011-06-21 10:35:55 +0800742
Peter Zijlstrabed831f2015-05-27 11:09:35 +0930743 preempt_disable();
744 mod = __module_address((unsigned long)key);
Xiao Guangrong140fe3b2011-06-21 10:35:55 +0800745 if (mod)
746 stop = mod->jump_entries + mod->num_jump_entries;
Peter Zijlstrabed831f2015-05-27 11:09:35 +0930747 preempt_enable();
Jason Barond430d3d2011-03-16 17:29:47 -0400748#endif
Jason Baron3821fd32017-02-03 15:42:24 -0500749 entry = static_key_entries(key);
Xiao Guangrong140fe3b2011-06-21 10:35:55 +0800750 /* if there are no users, entry can be NULL */
751 if (entry)
Ard Biesheuvel19483672018-09-18 23:51:42 -0700752 __jump_label_update(key, entry, stop,
753 system_state < SYSTEM_RUNNING);
Jason Baronbf5438fc2010-09-17 11:09:00 -0400754}
755
Peter Zijlstra1987c942015-07-27 18:32:09 +0200756#ifdef CONFIG_STATIC_KEYS_SELFTEST
757static DEFINE_STATIC_KEY_TRUE(sk_true);
758static DEFINE_STATIC_KEY_FALSE(sk_false);
759
760static __init int jump_label_test(void)
761{
762 int i;
763
764 for (i = 0; i < 2; i++) {
765 WARN_ON(static_key_enabled(&sk_true.key) != true);
766 WARN_ON(static_key_enabled(&sk_false.key) != false);
767
768 WARN_ON(!static_branch_likely(&sk_true));
769 WARN_ON(!static_branch_unlikely(&sk_true));
770 WARN_ON(static_branch_likely(&sk_false));
771 WARN_ON(static_branch_unlikely(&sk_false));
772
773 static_branch_disable(&sk_true);
774 static_branch_enable(&sk_false);
775
776 WARN_ON(static_key_enabled(&sk_true.key) == true);
777 WARN_ON(static_key_enabled(&sk_false.key) == false);
778
779 WARN_ON(static_branch_likely(&sk_true));
780 WARN_ON(static_branch_unlikely(&sk_true));
781 WARN_ON(!static_branch_likely(&sk_false));
782 WARN_ON(!static_branch_unlikely(&sk_false));
783
784 static_branch_enable(&sk_true);
785 static_branch_disable(&sk_false);
786 }
787
788 return 0;
789}
Jason Baron92ee46e2017-11-13 16:48:47 -0500790early_initcall(jump_label_test);
Peter Zijlstra1987c942015-07-27 18:32:09 +0200791#endif /* STATIC_KEYS_SELFTEST */