blob: 909644abee67fc72a47566c063e1f7493c0bbf95 [file] [log] [blame]
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -04001// SPDX-License-Identifier: GPL-2.0+
2//
Paul E. McKenney8e4ec3d2020-06-17 11:33:54 -07003// Scalability test comparing RCU vs other mechanisms
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -04004// for acquiring references on objects.
5//
6// Copyright (C) Google, 2020.
7//
8// Author: Joel Fernandes <joel@joelfernandes.org>
9
10#define pr_fmt(fmt) fmt
11
12#include <linux/atomic.h>
13#include <linux/bitops.h>
14#include <linux/completion.h>
15#include <linux/cpu.h>
16#include <linux/delay.h>
17#include <linux/err.h>
18#include <linux/init.h>
19#include <linux/interrupt.h>
20#include <linux/kthread.h>
21#include <linux/kernel.h>
22#include <linux/mm.h>
23#include <linux/module.h>
24#include <linux/moduleparam.h>
25#include <linux/notifier.h>
26#include <linux/percpu.h>
27#include <linux/rcupdate.h>
Paul E. McKenney72bb7492020-06-02 08:34:41 -070028#include <linux/rcupdate_trace.h>
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -040029#include <linux/reboot.h>
30#include <linux/sched.h>
31#include <linux/spinlock.h>
32#include <linux/smp.h>
33#include <linux/stat.h>
34#include <linux/srcu.h>
35#include <linux/slab.h>
36#include <linux/torture.h>
37#include <linux/types.h>
38
39#include "rcu.h"
40
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -070041#define SCALE_FLAG "-ref-scale: "
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -040042
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -070043#define SCALEOUT(s, x...) \
44 pr_alert("%s" SCALE_FLAG s, scale_type, ## x)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -040045
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -070046#define VERBOSE_SCALEOUT(s, x...) \
Li Zhijianf71f22b2021-10-29 17:40:24 +080047 do { \
48 if (verbose) \
49 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
50 } while (0)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -040051
Paul E. McKenneye76506f2020-11-15 10:24:52 -080052static atomic_t verbose_batch_ctr;
53
54#define VERBOSE_SCALEOUT_BATCH(s, x...) \
55do { \
56 if (verbose && \
57 (verbose_batched <= 0 || \
Paul E. McKenney414c1162020-11-25 10:50:35 -080058 !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \
59 schedule_timeout_uninterruptible(1); \
Li Zhijianf71f22b2021-10-29 17:40:24 +080060 pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \
Paul E. McKenney414c1162020-11-25 10:50:35 -080061 } \
Paul E. McKenneye76506f2020-11-15 10:24:52 -080062} while (0)
63
Li Zhijianf71f22b2021-10-29 17:40:24 +080064#define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -040065
66MODULE_LICENSE("GPL");
67MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>");
68
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -070069static char *scale_type = "rcu";
70module_param(scale_type, charp, 0444);
71MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock.");
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -040072
73torture_param(int, verbose, 0, "Enable verbose debugging printk()s");
Paul E. McKenneye76506f2020-11-15 10:24:52 -080074torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s");
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -040075
Paul E. McKenney777a54c2020-05-25 14:16:44 -070076// Wait until there are multiple CPUs before starting test.
Paul E. McKenney8e4ec3d2020-06-17 11:33:54 -070077torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,
Paul E. McKenney777a54c2020-05-25 14:16:44 -070078 "Holdoff time before test start (s)");
79// Number of loops per experiment, all readers execute operations concurrently.
Paul E. McKenney4dd72a32020-05-29 13:11:26 -070080torture_param(long, loops, 10000, "Number of loops per experiment.");
Paul E. McKenney8fc28782020-05-25 15:48:38 -070081// Number of readers, with -1 defaulting to about 75% of the CPUs.
82torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");
83// Number of runs.
84torture_param(int, nruns, 30, "Number of experiments to run.");
Paul E. McKenney918b3512020-05-31 18:14:57 -070085// Reader delay in nanoseconds, 0 for no delay.
86torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -040087
88#ifdef MODULE
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -070089# define REFSCALE_SHUTDOWN 0
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -040090#else
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -070091# define REFSCALE_SHUTDOWN 1
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -040092#endif
93
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -070094torture_param(bool, shutdown, REFSCALE_SHUTDOWN,
95 "Shutdown at end of scalability tests.");
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -040096
97struct reader_task {
98 struct task_struct *task;
Paul E. McKenneyaf2789d2020-05-26 11:22:03 -070099 int start_reader;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400100 wait_queue_head_t wq;
101 u64 last_duration_ns;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400102};
103
104static struct task_struct *shutdown_task;
105static wait_queue_head_t shutdown_wq;
106
107static struct task_struct *main_task;
108static wait_queue_head_t main_wq;
109static int shutdown_start;
110
111static struct reader_task *reader_tasks;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400112
113// Number of readers that are part of the current experiment.
114static atomic_t nreaders_exp;
115
116// Use to wait for all threads to start.
117static atomic_t n_init;
Paul E. McKenney86e0da22020-05-26 11:40:52 -0700118static atomic_t n_started;
Paul E. McKenney2db0bda2020-05-26 12:34:57 -0700119static atomic_t n_warmedup;
120static atomic_t n_cooleddown;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400121
122// Track which experiment is currently running.
123static int exp_idx;
124
125// Operations vector for selecting different types of tests.
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700126struct ref_scale_ops {
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400127 void (*init)(void);
128 void (*cleanup)(void);
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700129 void (*readsection)(const int nloops);
Paul E. McKenney918b3512020-05-31 18:14:57 -0700130 void (*delaysection)(const int nloops, const int udl, const int ndl);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400131 const char *name;
132};
133
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700134static struct ref_scale_ops *cur_ops;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400135
Paul E. McKenney918b3512020-05-31 18:14:57 -0700136static void un_delay(const int udl, const int ndl)
137{
138 if (udl)
139 udelay(udl);
140 if (ndl)
141 ndelay(ndl);
142}
143
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700144static void ref_rcu_read_section(const int nloops)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400145{
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700146 int i;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400147
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700148 for (i = nloops; i >= 0; i--) {
149 rcu_read_lock();
150 rcu_read_unlock();
151 }
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400152}
153
Paul E. McKenney918b3512020-05-31 18:14:57 -0700154static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl)
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700155{
156 int i;
157
158 for (i = nloops; i >= 0; i--) {
159 rcu_read_lock();
Paul E. McKenney918b3512020-05-31 18:14:57 -0700160 un_delay(udl, ndl);
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700161 rcu_read_unlock();
162 }
163}
164
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700165static void rcu_sync_scale_init(void)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400166{
167}
168
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700169static struct ref_scale_ops rcu_ops = {
170 .init = rcu_sync_scale_init,
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700171 .readsection = ref_rcu_read_section,
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700172 .delaysection = ref_rcu_delay_section,
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400173 .name = "rcu"
174};
175
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700176// Definitions for SRCU ref scale testing.
177DEFINE_STATIC_SRCU(srcu_refctl_scale);
178static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400179
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700180static void srcu_ref_scale_read_section(const int nloops)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400181{
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700182 int i;
183 int idx;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400184
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700185 for (i = nloops; i >= 0; i--) {
186 idx = srcu_read_lock(srcu_ctlp);
187 srcu_read_unlock(srcu_ctlp, idx);
188 }
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400189}
190
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700191static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700192{
193 int i;
194 int idx;
195
196 for (i = nloops; i >= 0; i--) {
197 idx = srcu_read_lock(srcu_ctlp);
Paul E. McKenney918b3512020-05-31 18:14:57 -0700198 un_delay(udl, ndl);
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700199 srcu_read_unlock(srcu_ctlp, idx);
200 }
201}
202
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700203static struct ref_scale_ops srcu_ops = {
204 .init = rcu_sync_scale_init,
205 .readsection = srcu_ref_scale_read_section,
206 .delaysection = srcu_ref_scale_delay_section,
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400207 .name = "srcu"
208};
209
Paul E. McKenney5f654af2022-03-25 14:39:54 -0700210#ifdef CONFIG_TASKS_RCU
211
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700212// Definitions for RCU Tasks ref scale testing: Empty read markers.
Paul E. McKenneye13ef442020-06-03 11:56:34 -0700213// These definitions also work for RCU Rude readers.
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700214static void rcu_tasks_ref_scale_read_section(const int nloops)
Paul E. McKenneye13ef442020-06-03 11:56:34 -0700215{
216 int i;
217
218 for (i = nloops; i >= 0; i--)
219 continue;
220}
221
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700222static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
Paul E. McKenneye13ef442020-06-03 11:56:34 -0700223{
224 int i;
225
226 for (i = nloops; i >= 0; i--)
227 un_delay(udl, ndl);
228}
229
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700230static struct ref_scale_ops rcu_tasks_ops = {
231 .init = rcu_sync_scale_init,
232 .readsection = rcu_tasks_ref_scale_read_section,
233 .delaysection = rcu_tasks_ref_scale_delay_section,
Paul E. McKenneye13ef442020-06-03 11:56:34 -0700234 .name = "rcu-tasks"
235};
236
Paul E. McKenney5f654af2022-03-25 14:39:54 -0700237#define RCU_TASKS_OPS &rcu_tasks_ops,
238
239#else // #ifdef CONFIG_TASKS_RCU
240
241#define RCU_TASKS_OPS
242
243#endif // #else // #ifdef CONFIG_TASKS_RCU
244
Paul E. McKenneydec86782022-03-25 15:21:07 -0700245#ifdef CONFIG_TASKS_TRACE_RCU
246
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700247// Definitions for RCU Tasks Trace ref scale testing.
248static void rcu_trace_ref_scale_read_section(const int nloops)
Paul E. McKenney72bb7492020-06-02 08:34:41 -0700249{
250 int i;
251
252 for (i = nloops; i >= 0; i--) {
253 rcu_read_lock_trace();
254 rcu_read_unlock_trace();
255 }
256}
257
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700258static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl)
Paul E. McKenney72bb7492020-06-02 08:34:41 -0700259{
260 int i;
261
262 for (i = nloops; i >= 0; i--) {
263 rcu_read_lock_trace();
264 un_delay(udl, ndl);
265 rcu_read_unlock_trace();
266 }
267}
268
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700269static struct ref_scale_ops rcu_trace_ops = {
270 .init = rcu_sync_scale_init,
271 .readsection = rcu_trace_ref_scale_read_section,
272 .delaysection = rcu_trace_ref_scale_delay_section,
Paul E. McKenney72bb7492020-06-02 08:34:41 -0700273 .name = "rcu-trace"
274};
275
Paul E. McKenneydec86782022-03-25 15:21:07 -0700276#define RCU_TRACE_OPS &rcu_trace_ops,
277
278#else // #ifdef CONFIG_TASKS_TRACE_RCU
279
280#define RCU_TRACE_OPS
281
282#endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU
283
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400284// Definitions for reference count
285static atomic_t refcnt;
286
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700287static void ref_refcnt_section(const int nloops)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400288{
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700289 int i;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400290
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700291 for (i = nloops; i >= 0; i--) {
292 atomic_inc(&refcnt);
293 atomic_dec(&refcnt);
294 }
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400295}
296
Paul E. McKenney918b3512020-05-31 18:14:57 -0700297static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl)
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700298{
299 int i;
300
301 for (i = nloops; i >= 0; i--) {
302 atomic_inc(&refcnt);
Paul E. McKenney918b3512020-05-31 18:14:57 -0700303 un_delay(udl, ndl);
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700304 atomic_dec(&refcnt);
305 }
306}
307
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700308static struct ref_scale_ops refcnt_ops = {
309 .init = rcu_sync_scale_init,
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700310 .readsection = ref_refcnt_section,
311 .delaysection = ref_refcnt_delay_section,
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400312 .name = "refcnt"
313};
314
315// Definitions for rwlock
316static rwlock_t test_rwlock;
317
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700318static void ref_rwlock_init(void)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400319{
320 rwlock_init(&test_rwlock);
321}
322
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700323static void ref_rwlock_section(const int nloops)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400324{
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700325 int i;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400326
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700327 for (i = nloops; i >= 0; i--) {
328 read_lock(&test_rwlock);
329 read_unlock(&test_rwlock);
330 }
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400331}
332
Paul E. McKenney918b3512020-05-31 18:14:57 -0700333static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl)
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700334{
335 int i;
336
337 for (i = nloops; i >= 0; i--) {
338 read_lock(&test_rwlock);
Paul E. McKenney918b3512020-05-31 18:14:57 -0700339 un_delay(udl, ndl);
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700340 read_unlock(&test_rwlock);
341 }
342}
343
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700344static struct ref_scale_ops rwlock_ops = {
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700345 .init = ref_rwlock_init,
346 .readsection = ref_rwlock_section,
347 .delaysection = ref_rwlock_delay_section,
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400348 .name = "rwlock"
349};
350
351// Definitions for rwsem
352static struct rw_semaphore test_rwsem;
353
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700354static void ref_rwsem_init(void)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400355{
356 init_rwsem(&test_rwsem);
357}
358
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700359static void ref_rwsem_section(const int nloops)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400360{
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700361 int i;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400362
Paul E. McKenney75dd8ef2020-05-25 14:59:06 -0700363 for (i = nloops; i >= 0; i--) {
364 down_read(&test_rwsem);
365 up_read(&test_rwsem);
366 }
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400367}
368
Paul E. McKenney918b3512020-05-31 18:14:57 -0700369static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl)
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700370{
371 int i;
372
373 for (i = nloops; i >= 0; i--) {
374 down_read(&test_rwsem);
Paul E. McKenney918b3512020-05-31 18:14:57 -0700375 un_delay(udl, ndl);
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700376 up_read(&test_rwsem);
377 }
378}
379
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700380static struct ref_scale_ops rwsem_ops = {
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700381 .init = ref_rwsem_init,
382 .readsection = ref_rwsem_section,
383 .delaysection = ref_rwsem_delay_section,
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400384 .name = "rwsem"
385};
386
Paul E. McKenneye9b800d2021-03-10 18:02:36 -0800387// Definitions for global spinlock
388static DEFINE_SPINLOCK(test_lock);
389
390static void ref_lock_section(const int nloops)
391{
392 int i;
393
394 preempt_disable();
395 for (i = nloops; i >= 0; i--) {
396 spin_lock(&test_lock);
397 spin_unlock(&test_lock);
398 }
399 preempt_enable();
400}
401
402static void ref_lock_delay_section(const int nloops, const int udl, const int ndl)
403{
404 int i;
405
406 preempt_disable();
407 for (i = nloops; i >= 0; i--) {
408 spin_lock(&test_lock);
409 un_delay(udl, ndl);
410 spin_unlock(&test_lock);
411 }
412 preempt_enable();
413}
414
415static struct ref_scale_ops lock_ops = {
416 .readsection = ref_lock_section,
417 .delaysection = ref_lock_delay_section,
418 .name = "lock"
419};
420
421// Definitions for global irq-save spinlock
422
423static void ref_lock_irq_section(const int nloops)
424{
425 unsigned long flags;
426 int i;
427
428 preempt_disable();
429 for (i = nloops; i >= 0; i--) {
430 spin_lock_irqsave(&test_lock, flags);
431 spin_unlock_irqrestore(&test_lock, flags);
432 }
433 preempt_enable();
434}
435
436static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl)
437{
438 unsigned long flags;
439 int i;
440
441 preempt_disable();
442 for (i = nloops; i >= 0; i--) {
443 spin_lock_irqsave(&test_lock, flags);
444 un_delay(udl, ndl);
445 spin_unlock_irqrestore(&test_lock, flags);
446 }
447 preempt_enable();
448}
449
450static struct ref_scale_ops lock_irq_ops = {
451 .readsection = ref_lock_irq_section,
452 .delaysection = ref_lock_irq_delay_section,
453 .name = "lock-irq"
454};
455
456// Definitions acquire-release.
457static DEFINE_PER_CPU(unsigned long, test_acqrel);
458
459static void ref_acqrel_section(const int nloops)
460{
461 unsigned long x;
462 int i;
463
464 preempt_disable();
465 for (i = nloops; i >= 0; i--) {
466 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
467 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
468 }
469 preempt_enable();
470}
471
472static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl)
473{
474 unsigned long x;
475 int i;
476
477 preempt_disable();
478 for (i = nloops; i >= 0; i--) {
479 x = smp_load_acquire(this_cpu_ptr(&test_acqrel));
480 un_delay(udl, ndl);
481 smp_store_release(this_cpu_ptr(&test_acqrel), x + 1);
482 }
483 preempt_enable();
484}
485
486static struct ref_scale_ops acqrel_ops = {
487 .readsection = ref_acqrel_section,
488 .delaysection = ref_acqrel_delay_section,
489 .name = "acqrel"
490};
491
Paul E. McKenney25f6fa52021-05-03 17:04:57 -0700492static volatile u64 stopopts;
493
494static void ref_clock_section(const int nloops)
495{
496 u64 x = 0;
497 int i;
498
499 preempt_disable();
500 for (i = nloops; i >= 0; i--)
501 x += ktime_get_real_fast_ns();
502 preempt_enable();
503 stopopts = x;
504}
505
506static void ref_clock_delay_section(const int nloops, const int udl, const int ndl)
507{
508 u64 x = 0;
509 int i;
510
511 preempt_disable();
512 for (i = nloops; i >= 0; i--) {
513 x += ktime_get_real_fast_ns();
514 un_delay(udl, ndl);
515 }
516 preempt_enable();
517 stopopts = x;
518}
519
520static struct ref_scale_ops clock_ops = {
521 .readsection = ref_clock_section,
522 .delaysection = ref_clock_delay_section,
523 .name = "clock"
524};
525
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700526static void rcu_scale_one_reader(void)
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700527{
528 if (readdelay <= 0)
529 cur_ops->readsection(loops);
530 else
Paul E. McKenney918b3512020-05-31 18:14:57 -0700531 cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700532}
533
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400534// Reader kthread. Repeatedly does empty RCU read-side
535// critical section, minimizing update-side interference.
536static int
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700537ref_scale_reader(void *arg)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400538{
539 unsigned long flags;
540 long me = (long)arg;
541 struct reader_task *rt = &(reader_tasks[me]);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400542 u64 start;
543 s64 duration;
544
Paul E. McKenneye76506f2020-11-15 10:24:52 -0800545 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me);
Paul E. McKenney05bc2762021-06-10 09:24:43 -0700546 WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)));
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400547 set_user_nice(current, MAX_NICE);
548 atomic_inc(&n_init);
Paul E. McKenney777a54c2020-05-25 14:16:44 -0700549 if (holdoff)
550 schedule_timeout_interruptible(holdoff * HZ);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400551repeat:
Paul E. McKenney05bc2762021-06-10 09:24:43 -0700552 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id());
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400553
554 // Wait for signal that this reader can start.
Paul E. McKenneyaf2789d2020-05-26 11:22:03 -0700555 wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400556 torture_must_stop());
557
558 if (torture_must_stop())
559 goto end;
560
561 // Make sure that the CPU is affinitized appropriately during testing.
Paul E. McKenney05bc2762021-06-10 09:24:43 -0700562 WARN_ON_ONCE(raw_smp_processor_id() != me);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400563
Paul E. McKenneyaf2789d2020-05-26 11:22:03 -0700564 WRITE_ONCE(rt->start_reader, 0);
Paul E. McKenney86e0da22020-05-26 11:40:52 -0700565 if (!atomic_dec_return(&n_started))
566 while (atomic_read_acquire(&n_started))
567 cpu_relax();
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400568
Paul E. McKenneye76506f2020-11-15 10:24:52 -0800569 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx);
Paul E. McKenneyb864f892020-05-26 10:57:34 -0700570
Paul E. McKenney2db0bda2020-05-26 12:34:57 -0700571
572 // To reduce noise, do an initial cache-warming invocation, check
573 // in, and then keep warming until everyone has checked in.
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700574 rcu_scale_one_reader();
Paul E. McKenney2db0bda2020-05-26 12:34:57 -0700575 if (!atomic_dec_return(&n_warmedup))
576 while (atomic_read_acquire(&n_warmedup))
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700577 rcu_scale_one_reader();
Paul E. McKenney2db0bda2020-05-26 12:34:57 -0700578 // Also keep interrupts disabled. This also has the effect
579 // of preventing entries into slow path for rcu_read_unlock().
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400580 local_irq_save(flags);
581 start = ktime_get_mono_fast_ns();
582
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700583 rcu_scale_one_reader();
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400584
585 duration = ktime_get_mono_fast_ns() - start;
586 local_irq_restore(flags);
587
588 rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;
Paul E. McKenney2db0bda2020-05-26 12:34:57 -0700589 // To reduce runtime-skew noise, do maintain-load invocations until
590 // everyone is done.
591 if (!atomic_dec_return(&n_cooleddown))
592 while (atomic_read_acquire(&n_cooleddown))
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700593 rcu_scale_one_reader();
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400594
Paul E. McKenneyb864f892020-05-26 10:57:34 -0700595 if (atomic_dec_and_test(&nreaders_exp))
596 wake_up(&main_wq);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400597
Paul E. McKenneye76506f2020-11-15 10:24:52 -0800598 VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",
599 me, exp_idx, atomic_read(&nreaders_exp));
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400600
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400601 if (!torture_must_stop())
602 goto repeat;
603end:
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700604 torture_kthread_stopping("ref_scale_reader");
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400605 return 0;
606}
607
Paul E. McKenney29907502020-05-26 09:32:57 -0700608static void reset_readers(void)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400609{
610 int i;
611 struct reader_task *rt;
612
Paul E. McKenneydbf28ef2020-05-25 17:22:24 -0700613 for (i = 0; i < nreaders; i++) {
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400614 rt = &(reader_tasks[i]);
615
616 rt->last_duration_ns = 0;
617 }
618}
619
620// Print the results of each reader and return the sum of all their durations.
Paul E. McKenney29907502020-05-26 09:32:57 -0700621static u64 process_durations(int n)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400622{
623 int i;
624 struct reader_task *rt;
625 char buf1[64];
Paul E. McKenney2e90de72020-05-25 17:45:03 -0700626 char *buf;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400627 u64 sum = 0;
628
Li Zhijian9880eb82021-10-25 11:26:57 +0800629 buf = kmalloc(800 + 64, GFP_KERNEL);
Paul E. McKenney2e90de72020-05-25 17:45:03 -0700630 if (!buf)
631 return 0;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400632 buf[0] = 0;
633 sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",
634 exp_idx);
635
Paul E. McKenneydbf28ef2020-05-25 17:22:24 -0700636 for (i = 0; i < n && !torture_must_stop(); i++) {
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400637 rt = &(reader_tasks[i]);
638 sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns);
639
640 if (i % 5 == 0)
641 strcat(buf, "\n");
Li Zhijian9880eb82021-10-25 11:26:57 +0800642 if (strlen(buf) >= 800) {
643 pr_alert("%s", buf);
644 buf[0] = 0;
645 }
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400646 strcat(buf, buf1);
647
648 sum += rt->last_duration_ns;
649 }
Li Zhijian9880eb82021-10-25 11:26:57 +0800650 pr_alert("%s\n", buf);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400651
Paul E. McKenney2e90de72020-05-25 17:45:03 -0700652 kfree(buf);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400653 return sum;
654}
655
656// The main_func is the main orchestrator, it performs a bunch of
657// experiments. For every experiment, it orders all the readers
658// involved to start and waits for them to finish the experiment. It
659// then reads their timestamps and starts the next experiment. Each
660// experiment progresses from 1 concurrent reader to N of them at which
661// point all the timestamps are printed.
662static int main_func(void *arg)
663{
664 int exp, r;
665 char buf1[64];
Paul E. McKenneyf518f152020-05-25 17:32:56 -0700666 char *buf;
Paul E. McKenneydbf28ef2020-05-25 17:22:24 -0700667 u64 *result_avg;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400668
669 set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids));
670 set_user_nice(current, MAX_NICE);
671
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700672 VERBOSE_SCALEOUT("main_func task started");
Paul E. McKenneydbf28ef2020-05-25 17:22:24 -0700673 result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL);
Li Zhijian9880eb82021-10-25 11:26:57 +0800674 buf = kzalloc(800 + 64, GFP_KERNEL);
Paul E. McKenneyf518f152020-05-25 17:32:56 -0700675 if (!result_avg || !buf) {
Li Zhijian4feeb9d2021-10-25 11:26:58 +0800676 SCALEOUT_ERRSTRING("out of memory");
Li Zhijianc30c8762021-10-25 11:26:56 +0800677 goto oom_exit;
Paul E. McKenneyf518f152020-05-25 17:32:56 -0700678 }
Paul E. McKenney777a54c2020-05-25 14:16:44 -0700679 if (holdoff)
680 schedule_timeout_interruptible(holdoff * HZ);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400681
Paul E. McKenney96af8662020-05-27 16:46:56 -0700682 // Wait for all threads to start.
683 atomic_inc(&n_init);
684 while (atomic_read(&n_init) < nreaders + 1)
685 schedule_timeout_uninterruptible(1);
686
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400687 // Start exp readers up per experiment
Paul E. McKenneydbf28ef2020-05-25 17:22:24 -0700688 for (exp = 0; exp < nruns && !torture_must_stop(); exp++) {
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400689 if (torture_must_stop())
690 goto end;
691
Paul E. McKenneydbf28ef2020-05-25 17:22:24 -0700692 reset_readers();
693 atomic_set(&nreaders_exp, nreaders);
Paul E. McKenney86e0da22020-05-26 11:40:52 -0700694 atomic_set(&n_started, nreaders);
Paul E. McKenney2db0bda2020-05-26 12:34:57 -0700695 atomic_set(&n_warmedup, nreaders);
696 atomic_set(&n_cooleddown, nreaders);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400697
698 exp_idx = exp;
699
Paul E. McKenneydbf28ef2020-05-25 17:22:24 -0700700 for (r = 0; r < nreaders; r++) {
Paul E. McKenneyaf2789d2020-05-26 11:22:03 -0700701 smp_store_release(&reader_tasks[r].start_reader, 1);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400702 wake_up(&reader_tasks[r].wq);
703 }
704
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700705 VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers",
Paul E. McKenneydbf28ef2020-05-25 17:22:24 -0700706 nreaders);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400707
708 wait_event(main_wq,
709 !atomic_read(&nreaders_exp) || torture_must_stop());
710
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700711 VERBOSE_SCALEOUT("main_func: experiment ended");
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400712
713 if (torture_must_stop())
714 goto end;
715
Arnd Bergmann7c944d72020-05-29 14:36:26 -0700716 result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400717 }
718
719 // Print the average of all experiments
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700720 SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n");
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400721
Li Zhijian9880eb82021-10-25 11:26:57 +0800722 pr_alert("Runs\tTime(ns)\n");
Paul E. McKenneydbf28ef2020-05-25 17:22:24 -0700723 for (exp = 0; exp < nruns; exp++) {
Arnd Bergmann7c944d72020-05-29 14:36:26 -0700724 u64 avg;
725 u32 rem;
726
Arnd Bergmann7c944d72020-05-29 14:36:26 -0700727 avg = div_u64_rem(result_avg[exp], 1000, &rem);
728 sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400729 strcat(buf, buf1);
Li Zhijian9880eb82021-10-25 11:26:57 +0800730 if (strlen(buf) >= 800) {
731 pr_alert("%s", buf);
732 buf[0] = 0;
733 }
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400734 }
735
Li Zhijian9880eb82021-10-25 11:26:57 +0800736 pr_alert("%s", buf);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400737
Li Zhijianc30c8762021-10-25 11:26:56 +0800738oom_exit:
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400739 // This will shutdown everything including us.
740 if (shutdown) {
741 shutdown_start = 1;
742 wake_up(&shutdown_wq);
743 }
744
745 // Wait for torture to stop us
746 while (!torture_must_stop())
747 schedule_timeout_uninterruptible(1);
748
749end:
750 torture_kthread_stopping("main_func");
Paul E. McKenneyf518f152020-05-25 17:32:56 -0700751 kfree(result_avg);
752 kfree(buf);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400753 return 0;
754}
755
756static void
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700757ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400758{
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700759 pr_alert("%s" SCALE_FLAG
760 "--- %s: verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag,
Paul E. McKenneyb4d1e342020-05-28 16:37:35 -0700761 verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400762}
763
764static void
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700765ref_scale_cleanup(void)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400766{
767 int i;
768
769 if (torture_cleanup_begin())
770 return;
771
772 if (!cur_ops) {
773 torture_cleanup_end();
774 return;
775 }
776
777 if (reader_tasks) {
778 for (i = 0; i < nreaders; i++)
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700779 torture_stop_kthread("ref_scale_reader",
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400780 reader_tasks[i].task);
781 }
782 kfree(reader_tasks);
783
784 torture_stop_kthread("main_task", main_task);
785 kfree(main_task);
786
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700787 // Do scale-type-specific cleanup operations.
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400788 if (cur_ops->cleanup != NULL)
789 cur_ops->cleanup();
790
791 torture_cleanup_end();
792}
793
794// Shutdown kthread. Just waits to be awakened, then shuts down system.
795static int
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700796ref_scale_shutdown(void *arg)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400797{
798 wait_event(shutdown_wq, shutdown_start);
799
800 smp_mb(); // Wake before output.
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700801 ref_scale_cleanup();
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400802 kernel_power_off();
803
804 return -EINVAL;
805}
806
807static int __init
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700808ref_scale_init(void)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400809{
810 long i;
811 int firsterr = 0;
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700812 static struct ref_scale_ops *scale_ops[] = {
Paul E. McKenneydec86782022-03-25 15:21:07 -0700813 &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops,
Paul E. McKenney25f6fa52021-05-03 17:04:57 -0700814 &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops,
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400815 };
816
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700817 if (!torture_init_begin(scale_type, verbose))
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400818 return -EBUSY;
819
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700820 for (i = 0; i < ARRAY_SIZE(scale_ops); i++) {
821 cur_ops = scale_ops[i];
822 if (strcmp(scale_type, cur_ops->name) == 0)
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400823 break;
824 }
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700825 if (i == ARRAY_SIZE(scale_ops)) {
826 pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type);
827 pr_alert("rcu-scale types:");
828 for (i = 0; i < ARRAY_SIZE(scale_ops); i++)
829 pr_cont(" %s", scale_ops[i]->name);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400830 pr_cont("\n");
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400831 firsterr = -EINVAL;
832 cur_ops = NULL;
833 goto unwind;
834 }
835 if (cur_ops->init)
836 cur_ops->init();
837
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700838 ref_scale_print_module_parms(cur_ops, "Start of test");
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400839
840 // Shutdown task
841 if (shutdown) {
842 init_waitqueue_head(&shutdown_wq);
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700843 firsterr = torture_create_kthread(ref_scale_shutdown, NULL,
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400844 shutdown_task);
Paul E. McKenneyed60ad72021-08-05 15:57:12 -0700845 if (torture_init_error(firsterr))
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400846 goto unwind;
847 schedule_timeout_uninterruptible(1);
848 }
849
Paul E. McKenney8fc28782020-05-25 15:48:38 -0700850 // Reader tasks (default to ~75% of online CPUs).
851 if (nreaders < 0)
852 nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2);
Paul E. McKenney0c6d18d2020-08-27 09:58:19 -0700853 if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops))
854 loops = 1;
855 if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders))
856 nreaders = 1;
857 if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns))
858 nruns = 1;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400859 reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]),
860 GFP_KERNEL);
861 if (!reader_tasks) {
Li Zhijian4feeb9d2021-10-25 11:26:58 +0800862 SCALEOUT_ERRSTRING("out of memory");
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400863 firsterr = -ENOMEM;
864 goto unwind;
865 }
866
Li Zhijianf71f22b2021-10-29 17:40:24 +0800867 VERBOSE_SCALEOUT("Starting %d reader threads", nreaders);
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400868
869 for (i = 0; i < nreaders; i++) {
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700870 firsterr = torture_create_kthread(ref_scale_reader, (void *)i,
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400871 reader_tasks[i].task);
Paul E. McKenneyed60ad72021-08-05 15:57:12 -0700872 if (torture_init_error(firsterr))
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400873 goto unwind;
874
875 init_waitqueue_head(&(reader_tasks[i].wq));
876 }
877
878 // Main Task
879 init_waitqueue_head(&main_wq);
880 firsterr = torture_create_kthread(main_func, NULL, main_task);
Paul E. McKenneyed60ad72021-08-05 15:57:12 -0700881 if (torture_init_error(firsterr))
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400882 goto unwind;
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400883
884 torture_init_end();
885 return 0;
886
887unwind:
888 torture_init_end();
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700889 ref_scale_cleanup();
Paul E. McKenneybc80d352020-09-17 10:37:10 -0700890 if (shutdown) {
891 WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST));
892 kernel_power_off();
893 }
Joel Fernandes (Google)653ed642020-05-25 00:36:48 -0400894 return firsterr;
895}
896
Paul E. McKenney1fbeb3a2020-06-17 11:53:53 -0700897module_init(ref_scale_init);
898module_exit(ref_scale_cleanup);