Paul E. McKenney | 00de9d7 | 2019-01-17 10:21:12 -0800 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0+ |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 2 | /* |
| 3 | * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition. |
| 4 | * |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 5 | * Copyright IBM Corporation, 2008 |
| 6 | * |
Paul E. McKenney | 00de9d7 | 2019-01-17 10:21:12 -0800 | [diff] [blame] | 7 | * Author: Paul E. McKenney <paulmck@linux.ibm.com> |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 8 | * |
| 9 | * For detailed explanation of Read-Copy Update mechanism see - |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 10 | * Documentation/RCU |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 11 | */ |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 12 | #include <linux/completion.h> |
| 13 | #include <linux/interrupt.h> |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 14 | #include <linux/notifier.h> |
Ingo Molnar | f9411eb | 2017-02-06 09:50:49 +0100 | [diff] [blame] | 15 | #include <linux/rcupdate_wait.h> |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 16 | #include <linux/kernel.h> |
Paul Gortmaker | 9984de1 | 2011-05-23 14:51:41 -0400 | [diff] [blame] | 17 | #include <linux/export.h> |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 18 | #include <linux/mutex.h> |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 19 | #include <linux/sched.h> |
| 20 | #include <linux/types.h> |
| 21 | #include <linux/init.h> |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 22 | #include <linux/time.h> |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 23 | #include <linux/cpu.h> |
Linus Torvalds | 268bb0c | 2011-05-20 12:50:29 -0700 | [diff] [blame] | 24 | #include <linux/prefetch.h> |
Joel Fernandes (Google) | 77a40f9 | 2019-08-30 12:36:32 -0400 | [diff] [blame] | 25 | #include <linux/slab.h> |
Uladzislau Rezki (Sony) | 64d1d06 | 2020-05-25 23:47:54 +0200 | [diff] [blame] | 26 | #include <linux/mm.h> |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 27 | |
Paul E. McKenney | 29c00b4 | 2011-06-17 15:53:19 -0700 | [diff] [blame] | 28 | #include "rcu.h" |
| 29 | |
Paul E. McKenney | 6d48152 | 2017-05-17 10:54:29 -0700 | [diff] [blame] | 30 | /* Global control variables for rcupdate callback mechanism. */ |
| 31 | struct rcu_ctrlblk { |
| 32 | struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */ |
| 33 | struct rcu_head **donetail; /* ->next pointer of last "done" CB. */ |
| 34 | struct rcu_head **curtail; /* ->next pointer of last CB. */ |
Paul E. McKenney | 0909fc2 | 2021-02-25 17:36:06 -0800 | [diff] [blame] | 35 | unsigned long gp_seq; /* Grace-period counter. */ |
Paul E. McKenney | 6d48152 | 2017-05-17 10:54:29 -0700 | [diff] [blame] | 36 | }; |
| 37 | |
| 38 | /* Definition for rcupdate control block. */ |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 39 | static struct rcu_ctrlblk rcu_ctrlblk = { |
| 40 | .donetail = &rcu_ctrlblk.rcucblist, |
| 41 | .curtail = &rcu_ctrlblk.rcucblist, |
Paul E. McKenney | 0909fc2 | 2021-02-25 17:36:06 -0800 | [diff] [blame] | 42 | .gp_seq = 0 - 300UL, |
Paul E. McKenney | 6d48152 | 2017-05-17 10:54:29 -0700 | [diff] [blame] | 43 | }; |
| 44 | |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 45 | void rcu_barrier(void) |
Ingo Molnar | f9411eb | 2017-02-06 09:50:49 +0100 | [diff] [blame] | 46 | { |
Joel Fernandes (Google) | 3cb278e7 | 2022-10-16 16:22:54 +0000 | [diff] [blame] | 47 | wait_rcu_gp(call_rcu_hurry); |
Ingo Molnar | f9411eb | 2017-02-06 09:50:49 +0100 | [diff] [blame] | 48 | } |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 49 | EXPORT_SYMBOL(rcu_barrier); |
Ingo Molnar | f9411eb | 2017-02-06 09:50:49 +0100 | [diff] [blame] | 50 | |
Paul E. McKenney | 65cfe35 | 2018-07-01 07:40:52 -0700 | [diff] [blame] | 51 | /* Record an rcu quiescent state. */ |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 52 | void rcu_qs(void) |
Ingo Molnar | f9411eb | 2017-02-06 09:50:49 +0100 | [diff] [blame] | 53 | { |
Eric Dumazet | b554d7de | 2011-04-28 07:23:45 +0200 | [diff] [blame] | 54 | unsigned long flags; |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 55 | |
Eric Dumazet | b554d7de | 2011-04-28 07:23:45 +0200 | [diff] [blame] | 56 | local_irq_save(flags); |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 57 | if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { |
| 58 | rcu_ctrlblk.donetail = rcu_ctrlblk.curtail; |
Cyrill Gorcunov | 18d7e40 | 2019-01-24 21:14:37 +0300 | [diff] [blame] | 59 | raise_softirq_irqoff(RCU_SOFTIRQ); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 60 | } |
Paul E. McKenney | 414c123 | 2022-04-13 15:17:25 -0700 | [diff] [blame] | 61 | WRITE_ONCE(rcu_ctrlblk.gp_seq, rcu_ctrlblk.gp_seq + 2); |
Eric Dumazet | b554d7de | 2011-04-28 07:23:45 +0200 | [diff] [blame] | 62 | local_irq_restore(flags); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 63 | } |
| 64 | |
| 65 | /* |
| 66 | * Check to see if the scheduling-clock interrupt came from an extended |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 67 | * quiescent state, and, if so, tell RCU about it. This function must |
| 68 | * be called from hardirq context. It is normally called from the |
| 69 | * scheduling-clock interrupt. |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 70 | */ |
Paul E. McKenney | c98cac6 | 2018-11-21 11:35:03 -0800 | [diff] [blame] | 71 | void rcu_sched_clock_irq(int user) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 72 | { |
Paul E. McKenney | c5bacd9 | 2018-07-20 14:18:23 -0700 | [diff] [blame] | 73 | if (user) { |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 74 | rcu_qs(); |
Paul E. McKenney | c5bacd9 | 2018-07-20 14:18:23 -0700 | [diff] [blame] | 75 | } else if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { |
| 76 | set_tsk_need_resched(current); |
| 77 | set_preempt_need_resched(); |
| 78 | } |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 79 | } |
| 80 | |
Joel Fernandes (Google) | 77a40f9 | 2019-08-30 12:36:32 -0400 | [diff] [blame] | 81 | /* |
| 82 | * Reclaim the specified callback, either by invoking it for non-kfree cases or |
| 83 | * freeing it directly (for kfree). Return true if kfreeing, false otherwise. |
| 84 | */ |
| 85 | static inline bool rcu_reclaim_tiny(struct rcu_head *head) |
| 86 | { |
| 87 | rcu_callback_t f; |
| 88 | unsigned long offset = (unsigned long)head->func; |
| 89 | |
| 90 | rcu_lock_acquire(&rcu_callback_map); |
Uladzislau Rezki (Sony) | c408b21 | 2020-05-25 23:47:55 +0200 | [diff] [blame] | 91 | if (__is_kvfree_rcu_offset(offset)) { |
| 92 | trace_rcu_invoke_kvfree_callback("", head, offset); |
Uladzislau Rezki (Sony) | 64d1d06 | 2020-05-25 23:47:54 +0200 | [diff] [blame] | 93 | kvfree((void *)head - offset); |
Joel Fernandes (Google) | 77a40f9 | 2019-08-30 12:36:32 -0400 | [diff] [blame] | 94 | rcu_lock_release(&rcu_callback_map); |
| 95 | return true; |
| 96 | } |
| 97 | |
| 98 | trace_rcu_invoke_callback("", head); |
| 99 | f = head->func; |
Zhen Lei | 2cbc482 | 2023-08-05 11:17:26 +0800 | [diff] [blame] | 100 | debug_rcu_head_callback(head); |
Joel Fernandes (Google) | 77a40f9 | 2019-08-30 12:36:32 -0400 | [diff] [blame] | 101 | WRITE_ONCE(head->func, (rcu_callback_t)0L); |
| 102 | f(head); |
| 103 | rcu_lock_release(&rcu_callback_map); |
| 104 | return false; |
| 105 | } |
| 106 | |
Paul E. McKenney | 65cfe35 | 2018-07-01 07:40:52 -0700 | [diff] [blame] | 107 | /* Invoke the RCU callbacks whose grace period has elapsed. */ |
| 108 | static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 109 | { |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 110 | struct rcu_head *next, *list; |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 111 | unsigned long flags; |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 112 | |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 113 | /* Move the ready-to-invoke callbacks to a local list. */ |
| 114 | local_irq_save(flags); |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 115 | if (rcu_ctrlblk.donetail == &rcu_ctrlblk.rcucblist) { |
Paul E. McKenney | 6e91f8c | 2015-05-11 11:13:05 -0700 | [diff] [blame] | 116 | /* No callbacks ready, so just leave. */ |
| 117 | local_irq_restore(flags); |
| 118 | return; |
| 119 | } |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 120 | list = rcu_ctrlblk.rcucblist; |
| 121 | rcu_ctrlblk.rcucblist = *rcu_ctrlblk.donetail; |
| 122 | *rcu_ctrlblk.donetail = NULL; |
| 123 | if (rcu_ctrlblk.curtail == rcu_ctrlblk.donetail) |
| 124 | rcu_ctrlblk.curtail = &rcu_ctrlblk.rcucblist; |
| 125 | rcu_ctrlblk.donetail = &rcu_ctrlblk.rcucblist; |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 126 | local_irq_restore(flags); |
| 127 | |
| 128 | /* Invoke the callbacks on the local list. */ |
| 129 | while (list) { |
| 130 | next = list->next; |
| 131 | prefetch(next); |
Mathieu Desnoyers | 551d55a | 2010-04-17 08:48:42 -0400 | [diff] [blame] | 132 | debug_rcu_head_unqueue(list); |
Paul E. McKenney | b2c0710 | 2010-09-09 13:40:39 -0700 | [diff] [blame] | 133 | local_bh_disable(); |
Joel Fernandes (Google) | 77a40f9 | 2019-08-30 12:36:32 -0400 | [diff] [blame] | 134 | rcu_reclaim_tiny(list); |
Paul E. McKenney | b2c0710 | 2010-09-09 13:40:39 -0700 | [diff] [blame] | 135 | local_bh_enable(); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 136 | list = next; |
| 137 | } |
| 138 | } |
| 139 | |
Paul E. McKenney | b2c0710 | 2010-09-09 13:40:39 -0700 | [diff] [blame] | 140 | /* |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 141 | * Wait for a grace period to elapse. But it is illegal to invoke |
Paul E. McKenney | 679d3f3 | 2018-07-07 18:12:26 -0700 | [diff] [blame] | 142 | * synchronize_rcu() from within an RCU read-side critical section. |
Paul E. McKenney | 7f45353 | 2022-04-14 11:49:58 -0700 | [diff] [blame] | 143 | * Therefore, any legal call to synchronize_rcu() is a quiescent state, |
| 144 | * and so on a UP system, synchronize_rcu() need do nothing, other than |
| 145 | * let the polled APIs know that another grace period elapsed. |
| 146 | * |
Paul E. McKenney | 65cfe35 | 2018-07-01 07:40:52 -0700 | [diff] [blame] | 147 | * (But Lai Jiangshan points out the benefits of doing might_sleep() |
| 148 | * to reduce latency.) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 149 | * |
| 150 | * Cool, huh? (Due to Josh Triplett.) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 151 | */ |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 152 | void synchronize_rcu(void) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 153 | { |
Paul E. McKenney | f78f5b9 | 2015-06-18 15:50:02 -0700 | [diff] [blame] | 154 | RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || |
| 155 | lock_is_held(&rcu_lock_map) || |
| 156 | lock_is_held(&rcu_sched_lock_map), |
Paul E. McKenney | 679d3f3 | 2018-07-07 18:12:26 -0700 | [diff] [blame] | 157 | "Illegal synchronize_rcu() in RCU read-side critical section"); |
Paul E. McKenney | 7f45353 | 2022-04-14 11:49:58 -0700 | [diff] [blame] | 158 | WRITE_ONCE(rcu_ctrlblk.gp_seq, rcu_ctrlblk.gp_seq + 2); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 159 | } |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 160 | EXPORT_SYMBOL_GPL(synchronize_rcu); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 161 | |
Zqiang | 6ca0292 | 2022-07-01 10:44:04 +0800 | [diff] [blame] | 162 | static void tiny_rcu_leak_callback(struct rcu_head *rhp) |
| 163 | { |
| 164 | } |
| 165 | |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 166 | /* |
Paul E. McKenney | 679d3f3 | 2018-07-07 18:12:26 -0700 | [diff] [blame] | 167 | * Post an RCU callback to be invoked after the end of an RCU grace |
Paul E. McKenney | 65cfe35 | 2018-07-01 07:40:52 -0700 | [diff] [blame] | 168 | * period. But since we have but one CPU, that would be after any |
| 169 | * quiescent state. |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 170 | */ |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 171 | void call_rcu(struct rcu_head *head, rcu_callback_t func) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 172 | { |
Zqiang | 6ca0292 | 2022-07-01 10:44:04 +0800 | [diff] [blame] | 173 | static atomic_t doublefrees; |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 174 | unsigned long flags; |
| 175 | |
Zqiang | 6ca0292 | 2022-07-01 10:44:04 +0800 | [diff] [blame] | 176 | if (debug_rcu_head_queue(head)) { |
| 177 | if (atomic_inc_return(&doublefrees) < 4) { |
| 178 | pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); |
| 179 | mem_dump_obj(head); |
| 180 | } |
| 181 | |
| 182 | if (!__is_kvfree_rcu_offset((unsigned long)head->func)) |
| 183 | WRITE_ONCE(head->func, tiny_rcu_leak_callback); |
| 184 | return; |
| 185 | } |
| 186 | |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 187 | head->func = func; |
| 188 | head->next = NULL; |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 189 | |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 190 | local_irq_save(flags); |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 191 | *rcu_ctrlblk.curtail = head; |
| 192 | rcu_ctrlblk.curtail = &head->next; |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 193 | local_irq_restore(flags); |
Lai Jiangshan | 5f6130f | 2014-12-09 17:53:34 +0800 | [diff] [blame] | 194 | |
| 195 | if (unlikely(is_idle_task(current))) { |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 196 | /* force scheduling for rcu_qs() */ |
Lai Jiangshan | 5f6130f | 2014-12-09 17:53:34 +0800 | [diff] [blame] | 197 | resched_cpu(0); |
| 198 | } |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 199 | } |
Paul E. McKenney | 709fdce | 2018-07-03 10:44:44 -0700 | [diff] [blame] | 200 | EXPORT_SYMBOL_GPL(call_rcu); |
Paul E. McKenney | 9dc5ad3 | 2013-03-27 10:11:15 -0700 | [diff] [blame] | 201 | |
Paul E. McKenney | 0909fc2 | 2021-02-25 17:36:06 -0800 | [diff] [blame] | 202 | /* |
Paul E. McKenney | 91a967f | 2022-07-28 15:37:05 -0700 | [diff] [blame] | 203 | * Store a grace-period-counter "cookie". For more information, |
| 204 | * see the Tree RCU header comment. |
| 205 | */ |
| 206 | void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) |
| 207 | { |
| 208 | rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; |
| 209 | } |
| 210 | EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); |
| 211 | |
| 212 | /* |
Paul E. McKenney | 0909fc2 | 2021-02-25 17:36:06 -0800 | [diff] [blame] | 213 | * Return a grace-period-counter "cookie". For more information, |
| 214 | * see the Tree RCU header comment. |
| 215 | */ |
| 216 | unsigned long get_state_synchronize_rcu(void) |
| 217 | { |
| 218 | return READ_ONCE(rcu_ctrlblk.gp_seq); |
| 219 | } |
| 220 | EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); |
| 221 | |
| 222 | /* |
| 223 | * Return a grace-period-counter "cookie" and ensure that a future grace |
| 224 | * period completes. For more information, see the Tree RCU header comment. |
| 225 | */ |
| 226 | unsigned long start_poll_synchronize_rcu(void) |
| 227 | { |
| 228 | unsigned long gp_seq = get_state_synchronize_rcu(); |
| 229 | |
| 230 | if (unlikely(is_idle_task(current))) { |
| 231 | /* force scheduling for rcu_qs() */ |
| 232 | resched_cpu(0); |
| 233 | } |
| 234 | return gp_seq; |
| 235 | } |
| 236 | EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); |
| 237 | |
| 238 | /* |
| 239 | * Return true if the grace period corresponding to oldstate has completed |
| 240 | * and false otherwise. For more information, see the Tree RCU header |
| 241 | * comment. |
| 242 | */ |
| 243 | bool poll_state_synchronize_rcu(unsigned long oldstate) |
| 244 | { |
Paul E. McKenney | 414c123 | 2022-04-13 15:17:25 -0700 | [diff] [blame] | 245 | return oldstate == RCU_GET_STATE_COMPLETED || READ_ONCE(rcu_ctrlblk.gp_seq) != oldstate; |
Paul E. McKenney | 0909fc2 | 2021-02-25 17:36:06 -0800 | [diff] [blame] | 246 | } |
| 247 | EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); |
| 248 | |
Johannes Berg | 800d6ac | 2022-05-27 17:07:45 +0200 | [diff] [blame] | 249 | #ifdef CONFIG_KASAN_GENERIC |
Uladzislau Rezki (Sony) | 04a522b | 2022-10-25 16:46:12 +0200 | [diff] [blame] | 250 | void kvfree_call_rcu(struct rcu_head *head, void *ptr) |
Johannes Berg | 800d6ac | 2022-05-27 17:07:45 +0200 | [diff] [blame] | 251 | { |
Uladzislau Rezki (Sony) | 04a522b | 2022-10-25 16:46:12 +0200 | [diff] [blame] | 252 | if (head) |
Johannes Berg | 800d6ac | 2022-05-27 17:07:45 +0200 | [diff] [blame] | 253 | kasan_record_aux_stack_noalloc(ptr); |
Johannes Berg | 800d6ac | 2022-05-27 17:07:45 +0200 | [diff] [blame] | 254 | |
Uladzislau Rezki (Sony) | 04a522b | 2022-10-25 16:46:12 +0200 | [diff] [blame] | 255 | __kvfree_call_rcu(head, ptr); |
Johannes Berg | 800d6ac | 2022-05-27 17:07:45 +0200 | [diff] [blame] | 256 | } |
| 257 | EXPORT_SYMBOL_GPL(kvfree_call_rcu); |
| 258 | #endif |
| 259 | |
Pranith Kumar | aa23c6fbc | 2014-09-19 11:32:29 -0400 | [diff] [blame] | 260 | void __init rcu_init(void) |
Paul E. McKenney | 9dc5ad3 | 2013-03-27 10:11:15 -0700 | [diff] [blame] | 261 | { |
| 262 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); |
Pranith Kumar | aa23c6fbc | 2014-09-19 11:32:29 -0400 | [diff] [blame] | 263 | rcu_early_boot_tests(); |
Paul E. McKenney | 9dc5ad3 | 2013-03-27 10:11:15 -0700 | [diff] [blame] | 264 | } |