Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License as published by |
| 6 | * the Free Software Foundation; either version 2 of the License, or |
| 7 | * (at your option) any later version. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, |
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | * GNU General Public License for more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License |
Paul E. McKenney | 87de1cf | 2013-12-03 10:02:52 -0800 | [diff] [blame] | 15 | * along with this program; if not, you can access it online at |
| 16 | * http://www.gnu.org/licenses/gpl-2.0.html. |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 17 | * |
| 18 | * Copyright IBM Corporation, 2008 |
| 19 | * |
| 20 | * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com> |
| 21 | * |
| 22 | * For detailed explanation of Read-Copy Update mechanism see - |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 23 | * Documentation/RCU |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 24 | */ |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 25 | #include <linux/completion.h> |
| 26 | #include <linux/interrupt.h> |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 27 | #include <linux/notifier.h> |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 28 | #include <linux/rcupdate.h> |
| 29 | #include <linux/kernel.h> |
Paul Gortmaker | 9984de1 | 2011-05-23 14:51:41 -0400 | [diff] [blame] | 30 | #include <linux/export.h> |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 31 | #include <linux/mutex.h> |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 32 | #include <linux/sched.h> |
| 33 | #include <linux/types.h> |
| 34 | #include <linux/init.h> |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 35 | #include <linux/time.h> |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 36 | #include <linux/cpu.h> |
Linus Torvalds | 268bb0c | 2011-05-20 12:50:29 -0700 | [diff] [blame] | 37 | #include <linux/prefetch.h> |
Steven Rostedt (Red Hat) | af658dc | 2015-04-29 14:36:05 -0400 | [diff] [blame] | 38 | #include <linux/trace_events.h> |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 39 | |
Paul E. McKenney | 29c00b4 | 2011-06-17 15:53:19 -0700 | [diff] [blame] | 40 | #include "rcu.h" |
| 41 | |
Paul E. McKenney | 4102ada | 2013-10-08 20:23:47 -0700 | [diff] [blame] | 42 | /* Forward declarations for tiny_plugin.h. */ |
Paul E. McKenney | 24278d1 | 2010-09-27 17:25:23 -0700 | [diff] [blame] | 43 | struct rcu_ctrlblk; |
Paul E. McKenney | 965a002 | 2011-06-18 09:55:39 -0700 | [diff] [blame] | 44 | static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp); |
| 45 | static void rcu_process_callbacks(struct softirq_action *unused); |
Paul E. McKenney | a57eb94 | 2010-06-29 16:49:16 -0700 | [diff] [blame] | 46 | static void __call_rcu(struct rcu_head *head, |
| 47 | void (*func)(struct rcu_head *rcu), |
| 48 | struct rcu_ctrlblk *rcp); |
| 49 | |
Paul E. McKenney | 4102ada | 2013-10-08 20:23:47 -0700 | [diff] [blame] | 50 | #include "tiny_plugin.h" |
Paul E. McKenney | 6bfc09e | 2012-10-19 12:49:17 -0700 | [diff] [blame] | 51 | |
Paul E. McKenney | cc6783f | 2013-09-06 17:39:49 -0700 | [diff] [blame] | 52 | #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 53 | |
| 54 | /* |
| 55 | * Test whether RCU thinks that the current CPU is idle. |
| 56 | */ |
Linus Torvalds | b29c830 | 2013-11-16 12:23:18 -0800 | [diff] [blame] | 57 | bool notrace __rcu_is_watching(void) |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 58 | { |
Lai Jiangshan | 5f6130f | 2014-12-09 17:53:34 +0800 | [diff] [blame] | 59 | return true; |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 60 | } |
Paul E. McKenney | 5c173eb | 2013-09-13 17:20:11 -0700 | [diff] [blame] | 61 | EXPORT_SYMBOL(__rcu_is_watching); |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 62 | |
Paul E. McKenney | cc6783f | 2013-09-06 17:39:49 -0700 | [diff] [blame] | 63 | #endif /* defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */ |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 64 | |
| 65 | /* |
Eric Dumazet | b554d7de | 2011-04-28 07:23:45 +0200 | [diff] [blame] | 66 | * Helper function for rcu_sched_qs() and rcu_bh_qs(). |
| 67 | * Also irqs are disabled to avoid confusion due to interrupt handlers |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 68 | * invoking call_rcu(). |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 69 | */ |
| 70 | static int rcu_qsctr_help(struct rcu_ctrlblk *rcp) |
| 71 | { |
Paul E. McKenney | 1496144 | 2013-04-16 07:49:22 -0700 | [diff] [blame] | 72 | RCU_TRACE(reset_cpu_stall_ticks(rcp)); |
Alexander Gordeev | 27153ac | 2015-02-11 15:42:37 +0100 | [diff] [blame] | 73 | if (rcp->donetail != rcp->curtail) { |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 74 | rcp->donetail = rcp->curtail; |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 75 | return 1; |
| 76 | } |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 77 | |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 78 | return 0; |
| 79 | } |
| 80 | |
| 81 | /* |
| 82 | * Record an rcu quiescent state. And an rcu_bh quiescent state while we |
| 83 | * are at it, given that any rcu quiescent state is also an rcu_bh |
| 84 | * quiescent state. Use "+" instead of "||" to defeat short circuiting. |
| 85 | */ |
Paul E. McKenney | 284a8c9 | 2014-08-14 16:38:46 -0700 | [diff] [blame] | 86 | void rcu_sched_qs(void) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 87 | { |
Eric Dumazet | b554d7de | 2011-04-28 07:23:45 +0200 | [diff] [blame] | 88 | unsigned long flags; |
| 89 | |
| 90 | local_irq_save(flags); |
Paul E. McKenney | 99652b5 | 2010-03-30 15:50:01 -0700 | [diff] [blame] | 91 | if (rcu_qsctr_help(&rcu_sched_ctrlblk) + |
| 92 | rcu_qsctr_help(&rcu_bh_ctrlblk)) |
Paul E. McKenney | 9dc5ad3 | 2013-03-27 10:11:15 -0700 | [diff] [blame] | 93 | raise_softirq(RCU_SOFTIRQ); |
Eric Dumazet | b554d7de | 2011-04-28 07:23:45 +0200 | [diff] [blame] | 94 | local_irq_restore(flags); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 95 | } |
| 96 | |
| 97 | /* |
| 98 | * Record an rcu_bh quiescent state. |
| 99 | */ |
Paul E. McKenney | 284a8c9 | 2014-08-14 16:38:46 -0700 | [diff] [blame] | 100 | void rcu_bh_qs(void) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 101 | { |
Eric Dumazet | b554d7de | 2011-04-28 07:23:45 +0200 | [diff] [blame] | 102 | unsigned long flags; |
| 103 | |
| 104 | local_irq_save(flags); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 105 | if (rcu_qsctr_help(&rcu_bh_ctrlblk)) |
Paul E. McKenney | 9dc5ad3 | 2013-03-27 10:11:15 -0700 | [diff] [blame] | 106 | raise_softirq(RCU_SOFTIRQ); |
Eric Dumazet | b554d7de | 2011-04-28 07:23:45 +0200 | [diff] [blame] | 107 | local_irq_restore(flags); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 108 | } |
| 109 | |
| 110 | /* |
| 111 | * Check to see if the scheduling-clock interrupt came from an extended |
Paul E. McKenney | 9b2e4f1 | 2011-09-30 12:10:22 -0700 | [diff] [blame] | 112 | * quiescent state, and, if so, tell RCU about it. This function must |
| 113 | * be called from hardirq context. It is normally called from the |
| 114 | * scheduling-clock interrupt. |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 115 | */ |
Paul E. McKenney | c3377c2d | 2014-10-21 07:53:02 -0700 | [diff] [blame] | 116 | void rcu_check_callbacks(int user) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 117 | { |
Paul E. McKenney | 1496144 | 2013-04-16 07:49:22 -0700 | [diff] [blame] | 118 | RCU_TRACE(check_cpu_stalls()); |
Alexander Gordeev | ca9558a | 2014-10-31 14:55:05 +0000 | [diff] [blame] | 119 | if (user) |
Paul E. McKenney | 284a8c9 | 2014-08-14 16:38:46 -0700 | [diff] [blame] | 120 | rcu_sched_qs(); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 121 | else if (!in_softirq()) |
Paul E. McKenney | 284a8c9 | 2014-08-14 16:38:46 -0700 | [diff] [blame] | 122 | rcu_bh_qs(); |
Paul E. McKenney | 8315f42 | 2014-06-27 13:42:20 -0700 | [diff] [blame] | 123 | if (user) |
| 124 | rcu_note_voluntary_context_switch(current); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 125 | } |
| 126 | |
| 127 | /* |
Paul E. McKenney | b2c0710 | 2010-09-09 13:40:39 -0700 | [diff] [blame] | 128 | * Invoke the RCU callbacks on the specified rcu_ctrlkblk structure |
| 129 | * whose grace period has elapsed. |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 130 | */ |
Paul E. McKenney | 965a002 | 2011-06-18 09:55:39 -0700 | [diff] [blame] | 131 | static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 132 | { |
Steven Rostedt (Red Hat) | e66c33d | 2013-07-12 16:50:28 -0400 | [diff] [blame] | 133 | const char *rn = NULL; |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 134 | struct rcu_head *next, *list; |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 135 | unsigned long flags; |
Paul E. McKenney | 9e571a8 | 2010-09-30 21:26:52 -0700 | [diff] [blame] | 136 | RCU_TRACE(int cb_count = 0); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 137 | |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 138 | /* Move the ready-to-invoke callbacks to a local list. */ |
| 139 | local_irq_save(flags); |
Paul E. McKenney | 6e91f8c | 2015-05-11 11:13:05 -0700 | [diff] [blame] | 140 | if (rcp->donetail == &rcp->rcucblist) { |
| 141 | /* No callbacks ready, so just leave. */ |
| 142 | local_irq_restore(flags); |
| 143 | return; |
| 144 | } |
Paul E. McKenney | 486e259 | 2012-01-06 14:11:30 -0800 | [diff] [blame] | 145 | RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, rcp->qlen, -1)); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 146 | list = rcp->rcucblist; |
| 147 | rcp->rcucblist = *rcp->donetail; |
| 148 | *rcp->donetail = NULL; |
| 149 | if (rcp->curtail == rcp->donetail) |
| 150 | rcp->curtail = &rcp->rcucblist; |
| 151 | rcp->donetail = &rcp->rcucblist; |
| 152 | local_irq_restore(flags); |
| 153 | |
| 154 | /* Invoke the callbacks on the local list. */ |
Paul E. McKenney | d4c08f2 | 2011-06-25 06:36:56 -0700 | [diff] [blame] | 155 | RCU_TRACE(rn = rcp->name); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 156 | while (list) { |
| 157 | next = list->next; |
| 158 | prefetch(next); |
Mathieu Desnoyers | 551d55a | 2010-04-17 08:48:42 -0400 | [diff] [blame] | 159 | debug_rcu_head_unqueue(list); |
Paul E. McKenney | b2c0710 | 2010-09-09 13:40:39 -0700 | [diff] [blame] | 160 | local_bh_disable(); |
Paul E. McKenney | d4c08f2 | 2011-06-25 06:36:56 -0700 | [diff] [blame] | 161 | __rcu_reclaim(rn, list); |
Paul E. McKenney | b2c0710 | 2010-09-09 13:40:39 -0700 | [diff] [blame] | 162 | local_bh_enable(); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 163 | list = next; |
Paul E. McKenney | 9e571a8 | 2010-09-30 21:26:52 -0700 | [diff] [blame] | 164 | RCU_TRACE(cb_count++); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 165 | } |
Paul E. McKenney | 9e571a8 | 2010-09-30 21:26:52 -0700 | [diff] [blame] | 166 | RCU_TRACE(rcu_trace_sub_qlen(rcp, cb_count)); |
Paul E. McKenney | 0d75292 | 2013-08-17 18:08:37 -0700 | [diff] [blame] | 167 | RCU_TRACE(trace_rcu_batch_end(rcp->name, |
| 168 | cb_count, 0, need_resched(), |
Paul E. McKenney | 4968c30 | 2011-12-07 16:32:40 -0800 | [diff] [blame] | 169 | is_idle_task(current), |
Paul E. McKenney | 9dc5ad3 | 2013-03-27 10:11:15 -0700 | [diff] [blame] | 170 | false)); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 171 | } |
| 172 | |
Paul E. McKenney | 965a002 | 2011-06-18 09:55:39 -0700 | [diff] [blame] | 173 | static void rcu_process_callbacks(struct softirq_action *unused) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 174 | { |
Paul E. McKenney | 965a002 | 2011-06-18 09:55:39 -0700 | [diff] [blame] | 175 | __rcu_process_callbacks(&rcu_sched_ctrlblk); |
| 176 | __rcu_process_callbacks(&rcu_bh_ctrlblk); |
Paul E. McKenney | b2c0710 | 2010-09-09 13:40:39 -0700 | [diff] [blame] | 177 | } |
| 178 | |
| 179 | /* |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 180 | * Wait for a grace period to elapse. But it is illegal to invoke |
| 181 | * synchronize_sched() from within an RCU read-side critical section. |
| 182 | * Therefore, any legal call to synchronize_sched() is a quiescent |
| 183 | * state, and so on a UP system, synchronize_sched() need do nothing. |
| 184 | * Ditto for synchronize_rcu_bh(). (But Lai Jiangshan points out the |
| 185 | * benefits of doing might_sleep() to reduce latency.) |
| 186 | * |
| 187 | * Cool, huh? (Due to Josh Triplett.) |
| 188 | * |
Paul E. McKenney | da848c4 | 2010-03-30 15:46:01 -0700 | [diff] [blame] | 189 | * But we want to make this a static inline later. The cond_resched() |
| 190 | * currently makes this problematic. |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 191 | */ |
| 192 | void synchronize_sched(void) |
| 193 | { |
Paul E. McKenney | f78f5b9 | 2015-06-18 15:50:02 -0700 | [diff] [blame] | 194 | RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || |
| 195 | lock_is_held(&rcu_lock_map) || |
| 196 | lock_is_held(&rcu_sched_lock_map), |
| 197 | "Illegal synchronize_sched() in RCU read-side critical section"); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 198 | cond_resched(); |
| 199 | } |
| 200 | EXPORT_SYMBOL_GPL(synchronize_sched); |
| 201 | |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 202 | /* |
| 203 | * Helper function for call_rcu() and call_rcu_bh(). |
| 204 | */ |
| 205 | static void __call_rcu(struct rcu_head *head, |
| 206 | void (*func)(struct rcu_head *rcu), |
| 207 | struct rcu_ctrlblk *rcp) |
| 208 | { |
| 209 | unsigned long flags; |
| 210 | |
Mathieu Desnoyers | 551d55a | 2010-04-17 08:48:42 -0400 | [diff] [blame] | 211 | debug_rcu_head_queue(head); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 212 | head->func = func; |
| 213 | head->next = NULL; |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 214 | |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 215 | local_irq_save(flags); |
| 216 | *rcp->curtail = head; |
| 217 | rcp->curtail = &head->next; |
Paul E. McKenney | 9e571a8 | 2010-09-30 21:26:52 -0700 | [diff] [blame] | 218 | RCU_TRACE(rcp->qlen++); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 219 | local_irq_restore(flags); |
Lai Jiangshan | 5f6130f | 2014-12-09 17:53:34 +0800 | [diff] [blame] | 220 | |
| 221 | if (unlikely(is_idle_task(current))) { |
| 222 | /* force scheduling for rcu_sched_qs() */ |
| 223 | resched_cpu(0); |
| 224 | } |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 225 | } |
| 226 | |
| 227 | /* |
Paul E. McKenney | a57eb94 | 2010-06-29 16:49:16 -0700 | [diff] [blame] | 228 | * Post an RCU callback to be invoked after the end of an RCU-sched grace |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 229 | * period. But since we have but one CPU, that would be after any |
| 230 | * quiescent state. |
| 231 | */ |
Paul E. McKenney | a57eb94 | 2010-06-29 16:49:16 -0700 | [diff] [blame] | 232 | void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 233 | { |
Paul E. McKenney | 99652b5 | 2010-03-30 15:50:01 -0700 | [diff] [blame] | 234 | __call_rcu(head, func, &rcu_sched_ctrlblk); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 235 | } |
Paul E. McKenney | a57eb94 | 2010-06-29 16:49:16 -0700 | [diff] [blame] | 236 | EXPORT_SYMBOL_GPL(call_rcu_sched); |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 237 | |
| 238 | /* |
| 239 | * Post an RCU bottom-half callback to be invoked after any subsequent |
| 240 | * quiescent state. |
| 241 | */ |
Ingo Molnar | 4ce5b90 | 2009-10-26 07:55:55 +0100 | [diff] [blame] | 242 | void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) |
Paul E. McKenney | 9b1d82f | 2009-10-25 19:03:50 -0700 | [diff] [blame] | 243 | { |
| 244 | __call_rcu(head, func, &rcu_bh_ctrlblk); |
| 245 | } |
| 246 | EXPORT_SYMBOL_GPL(call_rcu_bh); |
Paul E. McKenney | 9dc5ad3 | 2013-03-27 10:11:15 -0700 | [diff] [blame] | 247 | |
Pranith Kumar | aa23c6fbc | 2014-09-19 11:32:29 -0400 | [diff] [blame] | 248 | void __init rcu_init(void) |
Paul E. McKenney | 9dc5ad3 | 2013-03-27 10:11:15 -0700 | [diff] [blame] | 249 | { |
| 250 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); |
Paul E. McKenney | 630181c | 2014-12-23 21:33:14 -0800 | [diff] [blame] | 251 | RCU_TRACE(reset_cpu_stall_ticks(&rcu_sched_ctrlblk)); |
| 252 | RCU_TRACE(reset_cpu_stall_ticks(&rcu_bh_ctrlblk)); |
Pranith Kumar | aa23c6fbc | 2014-09-19 11:32:29 -0400 | [diff] [blame] | 253 | |
| 254 | rcu_early_boot_tests(); |
Paul E. McKenney | 9dc5ad3 | 2013-03-27 10:11:15 -0700 | [diff] [blame] | 255 | } |