Paul E. McKenney | 2e24ce8 | 2019-01-17 10:16:42 -0800 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0+ |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2 | /* |
Paul E. McKenney | 29766f1 | 2006-06-27 02:54:02 -0700 | [diff] [blame] | 3 | * Read-Copy Update module-based torture test facility |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 4 | * |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 5 | * Copyright (C) IBM Corporation, 2005, 2006 |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 6 | * |
Paul E. McKenney | 2e24ce8 | 2019-01-17 10:16:42 -0800 | [diff] [blame] | 7 | * Authors: Paul E. McKenney <paulmck@linux.ibm.com> |
Josh Triplett | e0198b29 | 2014-07-30 16:08:42 -0700 | [diff] [blame] | 8 | * Josh Triplett <josh@joshtriplett.org> |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 9 | * |
Mauro Carvalho Chehab | 43cb545 | 2020-04-21 19:04:06 +0200 | [diff] [blame] | 10 | * See also: Documentation/RCU/torture.rst |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 11 | */ |
Paul E. McKenney | 6050003 | 2018-05-15 12:25:05 -0700 | [diff] [blame] | 12 | |
| 13 | #define pr_fmt(fmt) fmt |
| 14 | |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 15 | #include <linux/types.h> |
| 16 | #include <linux/kernel.h> |
| 17 | #include <linux/init.h> |
| 18 | #include <linux/module.h> |
| 19 | #include <linux/kthread.h> |
| 20 | #include <linux/err.h> |
| 21 | #include <linux/spinlock.h> |
| 22 | #include <linux/smp.h> |
Paul E. McKenney | 9cf8fc6 | 2020-03-06 14:00:46 -0800 | [diff] [blame] | 23 | #include <linux/rcupdate_wait.h> |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 24 | #include <linux/interrupt.h> |
Ingo Molnar | 174cd4b | 2017-02-02 19:15:33 +0100 | [diff] [blame] | 25 | #include <linux/sched/signal.h> |
Ingo Molnar | ae7e81c | 2017-02-01 18:07:51 +0100 | [diff] [blame] | 26 | #include <uapi/linux/sched/types.h> |
Arun Sharma | 60063497 | 2011-07-26 16:09:06 -0700 | [diff] [blame] | 27 | #include <linux/atomic.h> |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 28 | #include <linux/bitops.h> |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 29 | #include <linux/completion.h> |
| 30 | #include <linux/moduleparam.h> |
| 31 | #include <linux/percpu.h> |
| 32 | #include <linux/notifier.h> |
Paul E. McKenney | 343e909 | 2008-12-15 16:13:07 -0800 | [diff] [blame] | 33 | #include <linux/reboot.h> |
Rafael J. Wysocki | 8314418 | 2007-07-17 04:03:35 -0700 | [diff] [blame] | 34 | #include <linux/freezer.h> |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 35 | #include <linux/cpu.h> |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 36 | #include <linux/delay.h> |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 37 | #include <linux/stat.h> |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 38 | #include <linux/srcu.h> |
Robert P. J. Day | 1aeb272 | 2008-04-29 00:59:25 -0700 | [diff] [blame] | 39 | #include <linux/slab.h> |
Paul E. McKenney | 5249453 | 2012-11-14 16:26:40 -0800 | [diff] [blame] | 40 | #include <linux/trace_clock.h> |
Harvey Harrison | f07767f | 2008-10-20 10:23:38 -0700 | [diff] [blame] | 41 | #include <asm/byteorder.h> |
Paul E. McKenney | 51b1130 | 2014-01-27 11:49:39 -0800 | [diff] [blame] | 42 | #include <linux/torture.h> |
Paul E. McKenney | 38706bc | 2014-08-18 21:12:17 -0700 | [diff] [blame] | 43 | #include <linux/vmalloc.h> |
Paul E. McKenney | 0032f4e | 2017-08-30 10:40:17 -0700 | [diff] [blame] | 44 | #include <linux/sched/debug.h> |
Joel Fernandes (Google) | 450efca | 2018-06-10 16:45:43 -0700 | [diff] [blame] | 45 | #include <linux/sched/sysctl.h> |
Paul E. McKenney | e0aff97 | 2018-10-01 17:40:54 -0700 | [diff] [blame] | 46 | #include <linux/oom.h> |
Paul E. McKenney | d38e6dc | 2019-07-28 12:00:48 -0700 | [diff] [blame] | 47 | #include <linux/tick.h> |
Paul E. McKenney | c1a76c0 | 2020-03-10 10:32:30 -0700 | [diff] [blame] | 48 | #include <linux/rcupdate_trace.h> |
Wander Lairson Costa | 5ff7c9f | 2021-11-10 11:37:45 -0300 | [diff] [blame] | 49 | #include <linux/nmi.h> |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 50 | |
Paul E. McKenney | 25c3632 | 2017-05-03 09:51:55 -0700 | [diff] [blame] | 51 | #include "rcu.h" |
| 52 | |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 53 | MODULE_LICENSE("GPL"); |
Paul E. McKenney | 2e24ce8 | 2019-01-17 10:16:42 -0800 | [diff] [blame] | 54 | MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com> and Josh Triplett <josh@joshtriplett.org>"); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 55 | |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 56 | /* Bits for ->extendables field, extendables param, and related definitions. */ |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 57 | #define RCUTORTURE_RDR_SHIFT_1 8 /* Put SRCU index in upper bits. */ |
| 58 | #define RCUTORTURE_RDR_MASK_1 (1 << RCUTORTURE_RDR_SHIFT_1) |
| 59 | #define RCUTORTURE_RDR_SHIFT_2 9 /* Put SRCU index in upper bits. */ |
| 60 | #define RCUTORTURE_RDR_MASK_2 (1 << RCUTORTURE_RDR_SHIFT_2) |
Paul E. McKenney | 2ceebc0 | 2018-07-06 15:16:12 -0700 | [diff] [blame] | 61 | #define RCUTORTURE_RDR_BH 0x01 /* Extend readers by disabling bh. */ |
| 62 | #define RCUTORTURE_RDR_IRQ 0x02 /* ... disabling interrupts. */ |
| 63 | #define RCUTORTURE_RDR_PREEMPT 0x04 /* ... disabling preemption. */ |
| 64 | #define RCUTORTURE_RDR_RBH 0x08 /* ... rcu_read_lock_bh(). */ |
| 65 | #define RCUTORTURE_RDR_SCHED 0x10 /* ... rcu_read_lock_sched(). */ |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 66 | #define RCUTORTURE_RDR_RCU_1 0x20 /* ... entering another RCU reader. */ |
| 67 | #define RCUTORTURE_RDR_RCU_2 0x40 /* ... entering another RCU reader. */ |
| 68 | #define RCUTORTURE_RDR_NBITS 7 /* Number of bits defined above. */ |
Paul E. McKenney | 2ceebc0 | 2018-07-06 15:16:12 -0700 | [diff] [blame] | 69 | #define RCUTORTURE_MAX_EXTEND \ |
| 70 | (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_IRQ | RCUTORTURE_RDR_PREEMPT | \ |
| 71 | RCUTORTURE_RDR_RBH | RCUTORTURE_RDR_SCHED) |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 72 | #define RCUTORTURE_RDR_MAX_LOOPS 0x7 /* Maximum reader extensions. */ |
| 73 | /* Must be power of two minus one. */ |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 74 | #define RCUTORTURE_RDR_MAX_SEGS (RCUTORTURE_RDR_MAX_LOOPS + 3) |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 75 | |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 76 | torture_param(int, extendables, RCUTORTURE_MAX_EXTEND, |
| 77 | "Extend readers by disabling bh (1), irqs (2), or preempt (4)"); |
Paul E. McKenney | cb506e1 | 2022-04-17 20:43:13 -0700 | [diff] [blame] | 78 | torture_param(int, fqs_duration, 0, "Duration of fqs bursts (us), 0 to disable"); |
Paul E. McKenney | 9e25022 | 2014-01-27 16:27:00 -0800 | [diff] [blame] | 79 | torture_param(int, fqs_holdoff, 0, "Holdoff time within fqs bursts (us)"); |
| 80 | torture_param(int, fqs_stutter, 3, "Wait time between fqs bursts (s)"); |
Paul E. McKenney | cb506e1 | 2022-04-17 20:43:13 -0700 | [diff] [blame] | 81 | torture_param(int, fwd_progress, 1, "Number of grace-period forward progress tasks (0 to disable)"); |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 82 | torture_param(int, fwd_progress_div, 4, "Fraction of CPU stall to wait"); |
Paul E. McKenney | cb506e1 | 2022-04-17 20:43:13 -0700 | [diff] [blame] | 83 | torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress tests (s)"); |
| 84 | torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()"); |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 85 | torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives"); |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 86 | torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives"); |
Paul E. McKenney | b6fe491 | 2022-08-04 13:46:05 -0700 | [diff] [blame] | 87 | torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives"); |
Paul E. McKenney | 8df13f0 | 2022-08-04 15:23:26 -0700 | [diff] [blame] | 88 | torture_param(bool, gp_cond_exp_full, false, |
| 89 | "Use conditional/async full-stateexpedited GP wait primitives"); |
Paul E. McKenney | 9e25022 | 2014-01-27 16:27:00 -0800 | [diff] [blame] | 90 | torture_param(bool, gp_exp, false, "Use expedited GP wait primitives"); |
Paul E. McKenney | cb506e1 | 2022-04-17 20:43:13 -0700 | [diff] [blame] | 91 | torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives"); |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 92 | torture_param(bool, gp_poll, false, "Use polling GP wait primitives"); |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 93 | torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives"); |
Paul E. McKenney | 76ea364 | 2022-08-02 17:04:54 -0700 | [diff] [blame] | 94 | torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives"); |
Paul E. McKenney | 6c502b1 | 2022-08-03 12:38:51 -0700 | [diff] [blame] | 95 | torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives"); |
Paul E. McKenney | f0bf8fa | 2014-03-21 16:17:56 -0700 | [diff] [blame] | 96 | torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives"); |
Paul E. McKenney | 9e25022 | 2014-01-27 16:27:00 -0800 | [diff] [blame] | 97 | torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers"); |
Paul E. McKenney | d685514 | 2020-08-11 10:33:39 -0700 | [diff] [blame] | 98 | torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); |
Paul E. McKenney | cb506e1 | 2022-04-17 20:43:13 -0700 | [diff] [blame] | 99 | torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing"); |
Paul E. McKenney | 9e25022 | 2014-01-27 16:27:00 -0800 | [diff] [blame] | 100 | torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); |
| 101 | torture_param(int, nreaders, -1, "Number of RCU reader threads"); |
Paul E. McKenney | cb506e1 | 2022-04-17 20:43:13 -0700 | [diff] [blame] | 102 | torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing"); |
Paul E. McKenney | 9e25022 | 2014-01-27 16:27:00 -0800 | [diff] [blame] | 103 | torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); |
Paul E. McKenney | cb506e1 | 2022-04-17 20:43:13 -0700 | [diff] [blame] | 104 | torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 105 | torture_param(int, nocbs_nthreads, 0, "Number of NOCB toggle threads, 0 to disable"); |
| 106 | torture_param(int, nocbs_toggle, 1000, "Time between toggling nocb state (ms)"); |
Paul E. McKenney | cb506e1 | 2022-04-17 20:43:13 -0700 | [diff] [blame] | 107 | torture_param(int, read_exit_delay, 13, "Delay between read-then-exit episodes (s)"); |
| 108 | torture_param(int, read_exit_burst, 16, "# of read-then-exit bursts per episode, zero to disable"); |
Paul E. McKenney | 9e25022 | 2014-01-27 16:27:00 -0800 | [diff] [blame] | 109 | torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); |
| 110 | torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); |
| 111 | torture_param(int, stall_cpu, 0, "Stall duration (s), zero to disable."); |
Paul E. McKenney | cb506e1 | 2022-04-17 20:43:13 -0700 | [diff] [blame] | 112 | torture_param(int, stall_cpu_holdoff, 10, "Time to wait before starting stall (s)."); |
| 113 | torture_param(bool, stall_no_softlockup, false, "Avoid softlockup warning during cpu stall."); |
Paul E. McKenney | 2b1516e | 2017-08-18 16:11:37 -0700 | [diff] [blame] | 114 | torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); |
Paul E. McKenney | 19a8ff9 | 2020-03-11 17:39:12 -0700 | [diff] [blame] | 115 | torture_param(int, stall_cpu_block, 0, "Sleep while stalling."); |
Paul E. McKenney | cb506e1 | 2022-04-17 20:43:13 -0700 | [diff] [blame] | 116 | torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s)."); |
| 117 | torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); |
Paul E. McKenney | 9e25022 | 2014-01-27 16:27:00 -0800 | [diff] [blame] | 118 | torture_param(int, stutter, 5, "Number of seconds to run/halt test"); |
| 119 | torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); |
Paul E. McKenney | cb506e1 | 2022-04-17 20:43:13 -0700 | [diff] [blame] | 120 | torture_param(int, test_boost_duration, 4, "Duration of each boost test, seconds."); |
| 121 | torture_param(int, test_boost_interval, 7, "Interval between boost tests, seconds."); |
| 122 | torture_param(bool, test_no_idle_hz, true, "Test support for tickless idle CPUs"); |
| 123 | torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); |
Paul E. McKenney | 9e25022 | 2014-01-27 16:27:00 -0800 | [diff] [blame] | 124 | |
Paul E. McKenney | b5daa8f | 2014-01-30 13:38:09 -0800 | [diff] [blame] | 125 | static char *torture_type = "rcu"; |
Josh Triplett | d6ad671 | 2007-03-06 01:42:13 -0800 | [diff] [blame] | 126 | module_param(torture_type, charp, 0444); |
Paul E. McKenney | c770c82 | 2018-07-07 10:28:07 -0700 | [diff] [blame] | 127 | MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, srcu, ...)"); |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 128 | |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 129 | static int nrealnocbers; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 130 | static int nrealreaders; |
| 131 | static struct task_struct *writer_task; |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 132 | static struct task_struct **fakewriter_tasks; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 133 | static struct task_struct **reader_tasks; |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 134 | static struct task_struct **nocb_tasks; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 135 | static struct task_struct *stats_task; |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 136 | static struct task_struct *fqs_task; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 137 | static struct task_struct *boost_tasks[NR_CPUS]; |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 138 | static struct task_struct *stall_task; |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 139 | static struct task_struct **fwd_prog_tasks; |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 140 | static struct task_struct **barrier_cbs_tasks; |
| 141 | static struct task_struct *barrier_task; |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 142 | static struct task_struct *read_exit_task; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 143 | |
| 144 | #define RCU_TORTURE_PIPE_LEN 10 |
| 145 | |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 146 | // Mailbox-like structure to check RCU global memory ordering. |
| 147 | struct rcu_torture_reader_check { |
| 148 | unsigned long rtc_myloops; |
| 149 | int rtc_chkrdr; |
| 150 | unsigned long rtc_chkloops; |
| 151 | int rtc_ready; |
| 152 | struct rcu_torture_reader_check *rtc_assigner; |
| 153 | } ____cacheline_internodealigned_in_smp; |
| 154 | |
| 155 | // Update-side data structure used to check RCU readers. |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 156 | struct rcu_torture { |
| 157 | struct rcu_head rtort_rcu; |
| 158 | int rtort_pipe_count; |
| 159 | struct list_head rtort_free; |
Paul E. McKenney | 996417d | 2005-11-18 01:10:50 -0800 | [diff] [blame] | 160 | int rtort_mbtest; |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 161 | struct rcu_torture_reader_check *rtort_chkp; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 162 | }; |
| 163 | |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 164 | static LIST_HEAD(rcu_torture_freelist); |
Paul E. McKenney | 0ddea0e | 2010-09-19 21:06:14 -0700 | [diff] [blame] | 165 | static struct rcu_torture __rcu *rcu_torture_current; |
Paul E. McKenney | 4a29865 | 2011-04-03 21:33:51 -0700 | [diff] [blame] | 166 | static unsigned long rcu_torture_current_version; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 167 | static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN]; |
| 168 | static DEFINE_SPINLOCK(rcu_torture_lock); |
Paul E. McKenney | 67522be | 2016-03-01 08:52:19 -0800 | [diff] [blame] | 169 | static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count); |
| 170 | static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 171 | static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1]; |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 172 | static struct rcu_torture_reader_check *rcu_torture_reader_mbchk; |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 173 | static atomic_t n_rcu_torture_alloc; |
| 174 | static atomic_t n_rcu_torture_alloc_fail; |
| 175 | static atomic_t n_rcu_torture_free; |
| 176 | static atomic_t n_rcu_torture_mberror; |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 177 | static atomic_t n_rcu_torture_mbchk_fail; |
| 178 | static atomic_t n_rcu_torture_mbchk_tries; |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 179 | static atomic_t n_rcu_torture_error; |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 180 | static long n_rcu_torture_barrier_error; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 181 | static long n_rcu_torture_boost_ktrerror; |
| 182 | static long n_rcu_torture_boost_rterror; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 183 | static long n_rcu_torture_boost_failure; |
| 184 | static long n_rcu_torture_boosts; |
Paul E. McKenney | 8da9a59 | 2018-05-22 11:17:51 -0700 | [diff] [blame] | 185 | static atomic_long_t n_rcu_torture_timers; |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 186 | static long n_barrier_attempts; |
Joel Fernandes (Google) | bf5b643 | 2018-06-19 15:14:19 -0700 | [diff] [blame] | 187 | static long n_barrier_successes; /* did rcu_barrier test succeed? */ |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 188 | static unsigned long n_read_exits; |
Josh Triplett | e303373 | 2006-10-04 02:17:14 -0700 | [diff] [blame] | 189 | static struct list_head rcu_torture_removed; |
Paul E. McKenney | 60013d5 | 2019-07-10 08:30:00 -0700 | [diff] [blame] | 190 | static unsigned long shutdown_jiffies; |
Joel Fernandes (Google) | 959954d | 2020-06-18 16:29:55 -0400 | [diff] [blame] | 191 | static unsigned long start_gp_seq; |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 192 | static atomic_long_t n_nocb_offload; |
| 193 | static atomic_long_t n_nocb_deoffload; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 194 | |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 195 | static int rcu_torture_writer_state; |
| 196 | #define RTWS_FIXED_DELAY 0 |
| 197 | #define RTWS_DELAY 1 |
| 198 | #define RTWS_REPLACE 2 |
| 199 | #define RTWS_DEF_FREE 3 |
| 200 | #define RTWS_EXP_SYNC 4 |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 201 | #define RTWS_COND_GET 5 |
Paul E. McKenney | b6fe491 | 2022-08-04 13:46:05 -0700 | [diff] [blame] | 202 | #define RTWS_COND_GET_FULL 6 |
| 203 | #define RTWS_COND_GET_EXP 7 |
Paul E. McKenney | 8df13f0 | 2022-08-04 15:23:26 -0700 | [diff] [blame] | 204 | #define RTWS_COND_GET_EXP_FULL 8 |
| 205 | #define RTWS_COND_SYNC 9 |
| 206 | #define RTWS_COND_SYNC_FULL 10 |
| 207 | #define RTWS_COND_SYNC_EXP 11 |
| 208 | #define RTWS_COND_SYNC_EXP_FULL 12 |
| 209 | #define RTWS_POLL_GET 13 |
| 210 | #define RTWS_POLL_GET_FULL 14 |
| 211 | #define RTWS_POLL_GET_EXP 15 |
| 212 | #define RTWS_POLL_GET_EXP_FULL 16 |
| 213 | #define RTWS_POLL_WAIT 17 |
| 214 | #define RTWS_POLL_WAIT_FULL 18 |
| 215 | #define RTWS_POLL_WAIT_EXP 19 |
| 216 | #define RTWS_POLL_WAIT_EXP_FULL 20 |
| 217 | #define RTWS_SYNC 21 |
| 218 | #define RTWS_STUTTER 22 |
| 219 | #define RTWS_STOPPING 23 |
Paul E. McKenney | 18aff33 | 2015-11-17 13:35:28 -0800 | [diff] [blame] | 220 | static const char * const rcu_torture_writer_state_names[] = { |
| 221 | "RTWS_FIXED_DELAY", |
| 222 | "RTWS_DELAY", |
| 223 | "RTWS_REPLACE", |
| 224 | "RTWS_DEF_FREE", |
| 225 | "RTWS_EXP_SYNC", |
| 226 | "RTWS_COND_GET", |
Paul E. McKenney | b6fe491 | 2022-08-04 13:46:05 -0700 | [diff] [blame] | 227 | "RTWS_COND_GET_FULL", |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 228 | "RTWS_COND_GET_EXP", |
Paul E. McKenney | 8df13f0 | 2022-08-04 15:23:26 -0700 | [diff] [blame] | 229 | "RTWS_COND_GET_EXP_FULL", |
Paul E. McKenney | 18aff33 | 2015-11-17 13:35:28 -0800 | [diff] [blame] | 230 | "RTWS_COND_SYNC", |
Paul E. McKenney | b6fe491 | 2022-08-04 13:46:05 -0700 | [diff] [blame] | 231 | "RTWS_COND_SYNC_FULL", |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 232 | "RTWS_COND_SYNC_EXP", |
Paul E. McKenney | 8df13f0 | 2022-08-04 15:23:26 -0700 | [diff] [blame] | 233 | "RTWS_COND_SYNC_EXP_FULL", |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 234 | "RTWS_POLL_GET", |
Paul E. McKenney | 76ea364 | 2022-08-02 17:04:54 -0700 | [diff] [blame] | 235 | "RTWS_POLL_GET_FULL", |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 236 | "RTWS_POLL_GET_EXP", |
Paul E. McKenney | 6c502b1 | 2022-08-03 12:38:51 -0700 | [diff] [blame] | 237 | "RTWS_POLL_GET_EXP_FULL", |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 238 | "RTWS_POLL_WAIT", |
Paul E. McKenney | 76ea364 | 2022-08-02 17:04:54 -0700 | [diff] [blame] | 239 | "RTWS_POLL_WAIT_FULL", |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 240 | "RTWS_POLL_WAIT_EXP", |
Paul E. McKenney | 6c502b1 | 2022-08-03 12:38:51 -0700 | [diff] [blame] | 241 | "RTWS_POLL_WAIT_EXP_FULL", |
Paul E. McKenney | 18aff33 | 2015-11-17 13:35:28 -0800 | [diff] [blame] | 242 | "RTWS_SYNC", |
| 243 | "RTWS_STUTTER", |
| 244 | "RTWS_STOPPING", |
| 245 | }; |
| 246 | |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 247 | /* Record reader segment types and duration for first failing read. */ |
| 248 | struct rt_read_seg { |
| 249 | int rt_readstate; |
| 250 | unsigned long rt_delay_jiffies; |
| 251 | unsigned long rt_delay_ms; |
| 252 | unsigned long rt_delay_us; |
| 253 | bool rt_preempted; |
| 254 | }; |
| 255 | static int err_segs_recorded; |
| 256 | static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS]; |
| 257 | static int rt_read_nsegs; |
| 258 | |
Paul E. McKenney | 18aff33 | 2015-11-17 13:35:28 -0800 | [diff] [blame] | 259 | static const char *rcu_torture_writer_state_getname(void) |
| 260 | { |
| 261 | unsigned int i = READ_ONCE(rcu_torture_writer_state); |
| 262 | |
| 263 | if (i >= ARRAY_SIZE(rcu_torture_writer_state_names)) |
| 264 | return "???"; |
| 265 | return rcu_torture_writer_state_names[i]; |
| 266 | } |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 267 | |
Steven Rostedt | e4aa0da | 2013-02-04 13:36:13 -0500 | [diff] [blame] | 268 | #ifdef CONFIG_RCU_TRACE |
| 269 | static u64 notrace rcu_trace_clock_local(void) |
| 270 | { |
| 271 | u64 ts = trace_clock_local(); |
Paul E. McKenney | a3b7b6c | 2017-06-23 16:07:17 -0700 | [diff] [blame] | 272 | |
| 273 | (void)do_div(ts, NSEC_PER_USEC); |
Steven Rostedt | e4aa0da | 2013-02-04 13:36:13 -0500 | [diff] [blame] | 274 | return ts; |
| 275 | } |
| 276 | #else /* #ifdef CONFIG_RCU_TRACE */ |
| 277 | static u64 notrace rcu_trace_clock_local(void) |
| 278 | { |
| 279 | return 0ULL; |
| 280 | } |
| 281 | #endif /* #else #ifdef CONFIG_RCU_TRACE */ |
| 282 | |
Paul E. McKenney | 60013d5 | 2019-07-10 08:30:00 -0700 | [diff] [blame] | 283 | /* |
| 284 | * Stop aggressive CPU-hog tests a bit before the end of the test in order |
| 285 | * to avoid interfering with test shutdown. |
| 286 | */ |
| 287 | static bool shutdown_time_arrived(void) |
| 288 | { |
| 289 | return shutdown_secs && time_after(jiffies, shutdown_jiffies - 30 * HZ); |
| 290 | } |
| 291 | |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 292 | static unsigned long boost_starttime; /* jiffies of next boost test start. */ |
Pranith Kumar | 58ade2d | 2014-06-11 16:39:43 -0400 | [diff] [blame] | 293 | static DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */ |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 294 | /* and boost task create/destroy. */ |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 295 | static atomic_t barrier_cbs_count; /* Barrier callbacks registered. */ |
Paul E. McKenney | c6ebcbb | 2012-05-28 19:21:41 -0700 | [diff] [blame] | 296 | static bool barrier_phase; /* Test phase. */ |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 297 | static atomic_t barrier_cbs_invoked; /* Barrier callbacks invoked. */ |
| 298 | static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */ |
| 299 | static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 300 | |
Paul E. McKenney | e22ef8d | 2021-12-17 12:33:53 -0800 | [diff] [blame] | 301 | static atomic_t rcu_fwd_cb_nodelay; /* Short rcu_torture_delay() delays. */ |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 302 | |
Paul E. McKenney | 343e909 | 2008-12-15 16:13:07 -0800 | [diff] [blame] | 303 | /* |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 304 | * Allocate an element from the rcu_tortures pool. |
| 305 | */ |
Adrian Bunk | 97a41e2 | 2006-01-08 01:02:17 -0800 | [diff] [blame] | 306 | static struct rcu_torture * |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 307 | rcu_torture_alloc(void) |
| 308 | { |
| 309 | struct list_head *p; |
| 310 | |
Ingo Molnar | adac166 | 2006-01-25 19:50:12 +0100 | [diff] [blame] | 311 | spin_lock_bh(&rcu_torture_lock); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 312 | if (list_empty(&rcu_torture_freelist)) { |
| 313 | atomic_inc(&n_rcu_torture_alloc_fail); |
Ingo Molnar | adac166 | 2006-01-25 19:50:12 +0100 | [diff] [blame] | 314 | spin_unlock_bh(&rcu_torture_lock); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 315 | return NULL; |
| 316 | } |
| 317 | atomic_inc(&n_rcu_torture_alloc); |
| 318 | p = rcu_torture_freelist.next; |
| 319 | list_del_init(p); |
Ingo Molnar | adac166 | 2006-01-25 19:50:12 +0100 | [diff] [blame] | 320 | spin_unlock_bh(&rcu_torture_lock); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 321 | return container_of(p, struct rcu_torture, rtort_free); |
| 322 | } |
| 323 | |
| 324 | /* |
| 325 | * Free an element to the rcu_tortures pool. |
| 326 | */ |
| 327 | static void |
| 328 | rcu_torture_free(struct rcu_torture *p) |
| 329 | { |
| 330 | atomic_inc(&n_rcu_torture_free); |
Ingo Molnar | adac166 | 2006-01-25 19:50:12 +0100 | [diff] [blame] | 331 | spin_lock_bh(&rcu_torture_lock); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 332 | list_add_tail(&p->rtort_free, &rcu_torture_freelist); |
Ingo Molnar | adac166 | 2006-01-25 19:50:12 +0100 | [diff] [blame] | 333 | spin_unlock_bh(&rcu_torture_lock); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 334 | } |
| 335 | |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 336 | /* |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 337 | * Operations vector for selecting different types of tests. |
| 338 | */ |
| 339 | |
| 340 | struct rcu_torture_ops { |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 341 | int ttype; |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 342 | void (*init)(void); |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 343 | void (*cleanup)(void); |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 344 | int (*readlock)(void); |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 345 | void (*read_delay)(struct torture_random_state *rrsp, |
| 346 | struct rt_read_seg *rtrsp); |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 347 | void (*readunlock)(int idx); |
Paul E. McKenney | a5c095e | 2021-03-13 20:05:31 -0800 | [diff] [blame] | 348 | int (*readlock_held)(void); |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 349 | unsigned long (*get_gp_seq)(void); |
Paul E. McKenney | d7219312 | 2018-05-15 15:24:41 -0700 | [diff] [blame] | 350 | unsigned long (*gp_diff)(unsigned long new, unsigned long old); |
Paul E. McKenney | 0acc512 | 2009-06-25 09:08:17 -0700 | [diff] [blame] | 351 | void (*deferred_free)(struct rcu_torture *p); |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 352 | void (*sync)(void); |
Paul E. McKenney | 2ec1f2d9 | 2013-06-12 15:12:21 -0700 | [diff] [blame] | 353 | void (*exp_sync)(void); |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 354 | unsigned long (*get_gp_state_exp)(void); |
| 355 | unsigned long (*start_gp_poll_exp)(void); |
Paul E. McKenney | 6c502b1 | 2022-08-03 12:38:51 -0700 | [diff] [blame] | 356 | void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp); |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 357 | bool (*poll_gp_state_exp)(unsigned long oldstate); |
| 358 | void (*cond_sync_exp)(unsigned long oldstate); |
Paul E. McKenney | 8df13f0 | 2022-08-04 15:23:26 -0700 | [diff] [blame] | 359 | void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp); |
Paul E. McKenney | fd56f64b | 2020-11-13 20:14:27 -0800 | [diff] [blame] | 360 | unsigned long (*get_gp_state)(void); |
Paul E. McKenney | 3fdefca | 2022-07-28 19:58:13 -0700 | [diff] [blame] | 361 | void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp); |
Paul E. McKenney | d0eac20 | 2022-04-13 16:14:02 -0700 | [diff] [blame] | 362 | unsigned long (*get_gp_completed)(void); |
Paul E. McKenney | 91a967f | 2022-07-28 15:37:05 -0700 | [diff] [blame] | 363 | void (*get_gp_completed_full)(struct rcu_gp_oldstate *rgosp); |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 364 | unsigned long (*start_gp_poll)(void); |
Paul E. McKenney | 76ea364 | 2022-08-02 17:04:54 -0700 | [diff] [blame] | 365 | void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp); |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 366 | bool (*poll_gp_state)(unsigned long oldstate); |
Paul E. McKenney | 91a967f | 2022-07-28 15:37:05 -0700 | [diff] [blame] | 367 | bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp); |
Paul E. McKenney | ed7d2f1 | 2022-08-01 16:57:17 -0700 | [diff] [blame] | 368 | bool (*poll_need_2gp)(bool poll, bool poll_full); |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 369 | void (*cond_sync)(unsigned long oldstate); |
Paul E. McKenney | b6fe491 | 2022-08-04 13:46:05 -0700 | [diff] [blame] | 370 | void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp); |
Boqun Feng | db3e8db | 2015-07-29 13:29:39 +0800 | [diff] [blame] | 371 | call_rcu_func_t call; |
Paul E. McKenney | 2326974 | 2008-05-12 21:21:05 +0200 | [diff] [blame] | 372 | void (*cb_barrier)(void); |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 373 | void (*fqs)(void); |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 374 | void (*stats)(void); |
Paul E. McKenney | 27c0f14 | 2020-09-15 17:08:03 -0700 | [diff] [blame] | 375 | void (*gp_kthread_dbg)(void); |
Paul E. McKenney | 0260b92 | 2021-04-08 13:01:14 -0700 | [diff] [blame] | 376 | bool (*check_boost_failed)(unsigned long gp_state, int *cpup); |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 377 | int (*stall_dur)(void); |
Paul E. McKenney | 613b00f | 2021-11-23 11:53:52 -0800 | [diff] [blame] | 378 | long cbflood_max; |
Paul E. McKenney | 0acc512 | 2009-06-25 09:08:17 -0700 | [diff] [blame] | 379 | int irq_capable; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 380 | int can_boost; |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 381 | int extendables; |
Paul E. McKenney | 5eabea5 | 2019-04-12 09:02:46 -0700 | [diff] [blame] | 382 | int slow_gps; |
Paul E. McKenney | 340170f | 2021-09-24 21:30:26 -0700 | [diff] [blame] | 383 | int no_pi_lock; |
Steven Rostedt (Red Hat) | e66c33d | 2013-07-12 16:50:28 -0400 | [diff] [blame] | 384 | const char *name; |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 385 | }; |
Paul E. McKenney | a71fca5 | 2009-09-18 10:28:19 -0700 | [diff] [blame] | 386 | |
| 387 | static struct rcu_torture_ops *cur_ops; |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 388 | |
| 389 | /* |
| 390 | * Definitions for rcu torture testing. |
| 391 | */ |
| 392 | |
Paul E. McKenney | a5c095e | 2021-03-13 20:05:31 -0800 | [diff] [blame] | 393 | static int torture_readlock_not_held(void) |
| 394 | { |
| 395 | return rcu_read_lock_bh_held() || rcu_read_lock_sched_held(); |
| 396 | } |
| 397 | |
Josh Triplett | a49a4af | 2006-09-29 01:59:30 -0700 | [diff] [blame] | 398 | static int rcu_torture_read_lock(void) __acquires(RCU) |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 399 | { |
| 400 | rcu_read_lock(); |
| 401 | return 0; |
| 402 | } |
| 403 | |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 404 | static void |
| 405 | rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 406 | { |
Paul E. McKenney | d0af39e | 2016-10-10 18:26:04 -0700 | [diff] [blame] | 407 | unsigned long started; |
| 408 | unsigned long completed; |
Josh Triplett | b8d57a7 | 2009-09-08 15:54:35 -0700 | [diff] [blame] | 409 | const unsigned long shortdelay_us = 200; |
Paul E. McKenney | 1e69676 | 2018-07-20 12:04:12 -0700 | [diff] [blame] | 410 | unsigned long longdelay_ms = 300; |
Paul E. McKenney | d0af39e | 2016-10-10 18:26:04 -0700 | [diff] [blame] | 411 | unsigned long long ts; |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 412 | |
Josh Triplett | b8d57a7 | 2009-09-08 15:54:35 -0700 | [diff] [blame] | 413 | /* We want a short delay sometimes to make a reader delay the grace |
| 414 | * period, and we want a long delay occasionally to trigger |
| 415 | * force_quiescent_state. */ |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 416 | |
Paul E. McKenney | e22ef8d | 2021-12-17 12:33:53 -0800 | [diff] [blame] | 417 | if (!atomic_read(&rcu_fwd_cb_nodelay) && |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 418 | !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 419 | started = cur_ops->get_gp_seq(); |
Paul E. McKenney | d0af39e | 2016-10-10 18:26:04 -0700 | [diff] [blame] | 420 | ts = rcu_trace_clock_local(); |
Paul E. McKenney | 1e69676 | 2018-07-20 12:04:12 -0700 | [diff] [blame] | 421 | if (preempt_count() & (SOFTIRQ_MASK | HARDIRQ_MASK)) |
| 422 | longdelay_ms = 5; /* Avoid triggering BH limits. */ |
Josh Triplett | b8d57a7 | 2009-09-08 15:54:35 -0700 | [diff] [blame] | 423 | mdelay(longdelay_ms); |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 424 | rtrsp->rt_delay_ms = longdelay_ms; |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 425 | completed = cur_ops->get_gp_seq(); |
Paul E. McKenney | d0af39e | 2016-10-10 18:26:04 -0700 | [diff] [blame] | 426 | do_trace_rcu_torture_read(cur_ops->name, NULL, ts, |
| 427 | started, completed); |
| 428 | } |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 429 | if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) { |
Josh Triplett | b8d57a7 | 2009-09-08 15:54:35 -0700 | [diff] [blame] | 430 | udelay(shortdelay_us); |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 431 | rtrsp->rt_delay_us = shortdelay_us; |
| 432 | } |
Paul E. McKenney | 51b1130 | 2014-01-27 11:49:39 -0800 | [diff] [blame] | 433 | if (!preempt_count() && |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 434 | !(torture_random(rrsp) % (nrealreaders * 500))) { |
Paul E. McKenney | cc1321c | 2017-10-16 11:05:03 -0700 | [diff] [blame] | 435 | torture_preempt_schedule(); /* QS only if preemptible. */ |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 436 | rtrsp->rt_preempted = true; |
| 437 | } |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 438 | } |
| 439 | |
Josh Triplett | a49a4af | 2006-09-29 01:59:30 -0700 | [diff] [blame] | 440 | static void rcu_torture_read_unlock(int idx) __releases(RCU) |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 441 | { |
| 442 | rcu_read_unlock(); |
| 443 | } |
| 444 | |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 445 | /* |
| 446 | * Update callback in the pipe. This should be invoked after a grace period. |
| 447 | */ |
| 448 | static bool |
| 449 | rcu_torture_pipe_update_one(struct rcu_torture *rp) |
| 450 | { |
| 451 | int i; |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 452 | struct rcu_torture_reader_check *rtrcp = READ_ONCE(rp->rtort_chkp); |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 453 | |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 454 | if (rtrcp) { |
| 455 | WRITE_ONCE(rp->rtort_chkp, NULL); |
| 456 | smp_store_release(&rtrcp->rtc_ready, 1); // Pair with smp_load_acquire(). |
| 457 | } |
Paul E. McKenney | 2024891 | 2019-12-21 10:41:48 -0800 | [diff] [blame] | 458 | i = READ_ONCE(rp->rtort_pipe_count); |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 459 | if (i > RCU_TORTURE_PIPE_LEN) |
| 460 | i = RCU_TORTURE_PIPE_LEN; |
| 461 | atomic_inc(&rcu_torture_wcount[i]); |
Paul E. McKenney | 2024891 | 2019-12-21 10:41:48 -0800 | [diff] [blame] | 462 | WRITE_ONCE(rp->rtort_pipe_count, i + 1); |
| 463 | if (rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) { |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 464 | rp->rtort_mbtest = 0; |
| 465 | return true; |
| 466 | } |
| 467 | return false; |
| 468 | } |
| 469 | |
| 470 | /* |
| 471 | * Update all callbacks in the pipe. Suitable for synchronous grace-period |
| 472 | * primitives. |
| 473 | */ |
| 474 | static void |
| 475 | rcu_torture_pipe_update(struct rcu_torture *old_rp) |
| 476 | { |
| 477 | struct rcu_torture *rp; |
| 478 | struct rcu_torture *rp1; |
| 479 | |
| 480 | if (old_rp) |
| 481 | list_add(&old_rp->rtort_free, &rcu_torture_removed); |
| 482 | list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) { |
| 483 | if (rcu_torture_pipe_update_one(rp)) { |
| 484 | list_del(&rp->rtort_free); |
| 485 | rcu_torture_free(rp); |
| 486 | } |
| 487 | } |
| 488 | } |
| 489 | |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 490 | static void |
| 491 | rcu_torture_cb(struct rcu_head *p) |
| 492 | { |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 493 | struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); |
| 494 | |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 495 | if (torture_must_stop_irq()) { |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 496 | /* Test is ending, just drop callbacks on the floor. */ |
| 497 | /* The next initialization will pick up the pieces. */ |
| 498 | return; |
| 499 | } |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 500 | if (rcu_torture_pipe_update_one(rp)) |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 501 | rcu_torture_free(rp); |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 502 | else |
Paul E. McKenney | 0acc512 | 2009-06-25 09:08:17 -0700 | [diff] [blame] | 503 | cur_ops->deferred_free(rp); |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 504 | } |
| 505 | |
Paul E. McKenney | 6b80da4 | 2014-11-21 14:19:26 -0800 | [diff] [blame] | 506 | static unsigned long rcu_no_completed(void) |
Paul E. McKenney | d9a3da0 | 2009-12-02 12:10:15 -0800 | [diff] [blame] | 507 | { |
| 508 | return 0; |
| 509 | } |
| 510 | |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 511 | static void rcu_torture_deferred_free(struct rcu_torture *p) |
| 512 | { |
| 513 | call_rcu(&p->rtort_rcu, rcu_torture_cb); |
| 514 | } |
| 515 | |
Paul E. McKenney | 2ec1f2d9 | 2013-06-12 15:12:21 -0700 | [diff] [blame] | 516 | static void rcu_sync_torture_init(void) |
| 517 | { |
| 518 | INIT_LIST_HEAD(&rcu_torture_removed); |
| 519 | } |
| 520 | |
Paul E. McKenney | ed7d2f1 | 2022-08-01 16:57:17 -0700 | [diff] [blame] | 521 | static bool rcu_poll_need_2gp(bool poll, bool poll_full) |
| 522 | { |
Paul E. McKenney | b3cdd0a | 2022-08-04 18:05:46 -0700 | [diff] [blame] | 523 | return poll; |
Paul E. McKenney | ed7d2f1 | 2022-08-01 16:57:17 -0700 | [diff] [blame] | 524 | } |
| 525 | |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 526 | static struct rcu_torture_ops rcu_ops = { |
Paul E. McKenney | 0260b92 | 2021-04-08 13:01:14 -0700 | [diff] [blame] | 527 | .ttype = RCU_FLAVOR, |
| 528 | .init = rcu_sync_torture_init, |
| 529 | .readlock = rcu_torture_read_lock, |
| 530 | .read_delay = rcu_read_delay, |
| 531 | .readunlock = rcu_torture_read_unlock, |
| 532 | .readlock_held = torture_readlock_not_held, |
| 533 | .get_gp_seq = rcu_get_gp_seq, |
| 534 | .gp_diff = rcu_seq_diff, |
| 535 | .deferred_free = rcu_torture_deferred_free, |
| 536 | .sync = synchronize_rcu, |
| 537 | .exp_sync = synchronize_rcu_expedited, |
| 538 | .get_gp_state = get_state_synchronize_rcu, |
Paul E. McKenney | 3fdefca | 2022-07-28 19:58:13 -0700 | [diff] [blame] | 539 | .get_gp_state_full = get_state_synchronize_rcu_full, |
Paul E. McKenney | d0eac20 | 2022-04-13 16:14:02 -0700 | [diff] [blame] | 540 | .get_gp_completed = get_completed_synchronize_rcu, |
Paul E. McKenney | 91a967f | 2022-07-28 15:37:05 -0700 | [diff] [blame] | 541 | .get_gp_completed_full = get_completed_synchronize_rcu_full, |
Paul E. McKenney | 0260b92 | 2021-04-08 13:01:14 -0700 | [diff] [blame] | 542 | .start_gp_poll = start_poll_synchronize_rcu, |
Paul E. McKenney | 76ea364 | 2022-08-02 17:04:54 -0700 | [diff] [blame] | 543 | .start_gp_poll_full = start_poll_synchronize_rcu_full, |
Paul E. McKenney | 0260b92 | 2021-04-08 13:01:14 -0700 | [diff] [blame] | 544 | .poll_gp_state = poll_state_synchronize_rcu, |
Paul E. McKenney | 91a967f | 2022-07-28 15:37:05 -0700 | [diff] [blame] | 545 | .poll_gp_state_full = poll_state_synchronize_rcu_full, |
Paul E. McKenney | ed7d2f1 | 2022-08-01 16:57:17 -0700 | [diff] [blame] | 546 | .poll_need_2gp = rcu_poll_need_2gp, |
Paul E. McKenney | 0260b92 | 2021-04-08 13:01:14 -0700 | [diff] [blame] | 547 | .cond_sync = cond_synchronize_rcu, |
Paul E. McKenney | b6fe491 | 2022-08-04 13:46:05 -0700 | [diff] [blame] | 548 | .cond_sync_full = cond_synchronize_rcu_full, |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 549 | .get_gp_state_exp = get_state_synchronize_rcu, |
| 550 | .start_gp_poll_exp = start_poll_synchronize_rcu_expedited, |
Paul E. McKenney | 6c502b1 | 2022-08-03 12:38:51 -0700 | [diff] [blame] | 551 | .start_gp_poll_exp_full = start_poll_synchronize_rcu_expedited_full, |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 552 | .poll_gp_state_exp = poll_state_synchronize_rcu, |
| 553 | .cond_sync_exp = cond_synchronize_rcu_expedited, |
Paul E. McKenney | 0260b92 | 2021-04-08 13:01:14 -0700 | [diff] [blame] | 554 | .call = call_rcu, |
| 555 | .cb_barrier = rcu_barrier, |
| 556 | .fqs = rcu_force_quiescent_state, |
| 557 | .stats = NULL, |
| 558 | .gp_kthread_dbg = show_rcu_gp_kthreads, |
| 559 | .check_boost_failed = rcu_check_boost_fail, |
| 560 | .stall_dur = rcu_jiffies_till_stall_check, |
| 561 | .irq_capable = 1, |
| 562 | .can_boost = IS_ENABLED(CONFIG_RCU_BOOST), |
| 563 | .extendables = RCUTORTURE_MAX_EXTEND, |
| 564 | .name = "rcu" |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 565 | }; |
| 566 | |
Paul E. McKenney | c32e066 | 2006-06-27 02:54:04 -0700 | [diff] [blame] | 567 | /* |
Paul E. McKenney | ff20e25 | 2014-02-06 08:45:56 -0800 | [diff] [blame] | 568 | * Don't even think about trying any of these in real life!!! |
| 569 | * The names includes "busted", and they really means it! |
| 570 | * The only purpose of these functions is to provide a buggy RCU |
| 571 | * implementation to make sure that rcutorture correctly emits |
| 572 | * buggy-RCU error messages. |
| 573 | */ |
| 574 | static void rcu_busted_torture_deferred_free(struct rcu_torture *p) |
| 575 | { |
| 576 | /* This is a deliberate bug for testing purposes only! */ |
| 577 | rcu_torture_cb(&p->rtort_rcu); |
| 578 | } |
| 579 | |
| 580 | static void synchronize_rcu_busted(void) |
| 581 | { |
| 582 | /* This is a deliberate bug for testing purposes only! */ |
| 583 | } |
| 584 | |
| 585 | static void |
Boqun Feng | b6a4ae7 | 2015-07-29 13:29:38 +0800 | [diff] [blame] | 586 | call_rcu_busted(struct rcu_head *head, rcu_callback_t func) |
Paul E. McKenney | ff20e25 | 2014-02-06 08:45:56 -0800 | [diff] [blame] | 587 | { |
| 588 | /* This is a deliberate bug for testing purposes only! */ |
| 589 | func(head); |
| 590 | } |
| 591 | |
| 592 | static struct rcu_torture_ops rcu_busted_ops = { |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 593 | .ttype = INVALID_RCU_FLAVOR, |
Paul E. McKenney | ff20e25 | 2014-02-06 08:45:56 -0800 | [diff] [blame] | 594 | .init = rcu_sync_torture_init, |
| 595 | .readlock = rcu_torture_read_lock, |
| 596 | .read_delay = rcu_read_delay, /* just reuse rcu's version. */ |
| 597 | .readunlock = rcu_torture_read_unlock, |
Paul E. McKenney | a5c095e | 2021-03-13 20:05:31 -0800 | [diff] [blame] | 598 | .readlock_held = torture_readlock_not_held, |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 599 | .get_gp_seq = rcu_no_completed, |
Paul E. McKenney | ff20e25 | 2014-02-06 08:45:56 -0800 | [diff] [blame] | 600 | .deferred_free = rcu_busted_torture_deferred_free, |
| 601 | .sync = synchronize_rcu_busted, |
| 602 | .exp_sync = synchronize_rcu_busted, |
| 603 | .call = call_rcu_busted, |
| 604 | .cb_barrier = NULL, |
| 605 | .fqs = NULL, |
| 606 | .stats = NULL, |
| 607 | .irq_capable = 1, |
Paul E. McKenney | b3c9831 | 2017-06-06 16:39:00 -0700 | [diff] [blame] | 608 | .name = "busted" |
Paul E. McKenney | ff20e25 | 2014-02-06 08:45:56 -0800 | [diff] [blame] | 609 | }; |
| 610 | |
| 611 | /* |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 612 | * Definitions for srcu torture testing. |
| 613 | */ |
| 614 | |
Lai Jiangshan | cda4dc8 | 2012-10-13 01:14:17 +0800 | [diff] [blame] | 615 | DEFINE_STATIC_SRCU(srcu_ctl); |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 616 | static struct srcu_struct srcu_ctld; |
| 617 | static struct srcu_struct *srcu_ctlp = &srcu_ctl; |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 618 | |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 619 | static int srcu_torture_read_lock(void) __acquires(srcu_ctlp) |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 620 | { |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 621 | return srcu_read_lock(srcu_ctlp); |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 622 | } |
| 623 | |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 624 | static void |
| 625 | srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 626 | { |
| 627 | long delay; |
| 628 | const long uspertick = 1000000 / HZ; |
| 629 | const long longdelay = 10; |
| 630 | |
| 631 | /* We want there to be long-running readers, but not all the time. */ |
| 632 | |
Paul E. McKenney | 51b1130 | 2014-01-27 11:49:39 -0800 | [diff] [blame] | 633 | delay = torture_random(rrsp) % |
| 634 | (nrealreaders * 2 * longdelay * uspertick); |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 635 | if (!delay && in_task()) { |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 636 | schedule_timeout_interruptible(longdelay); |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 637 | rtrsp->rt_delay_jiffies = longdelay; |
| 638 | } else { |
| 639 | rcu_read_delay(rrsp, rtrsp); |
| 640 | } |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 641 | } |
| 642 | |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 643 | static void srcu_torture_read_unlock(int idx) __releases(srcu_ctlp) |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 644 | { |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 645 | srcu_read_unlock(srcu_ctlp, idx); |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 646 | } |
| 647 | |
Paul E. McKenney | a5c095e | 2021-03-13 20:05:31 -0800 | [diff] [blame] | 648 | static int torture_srcu_read_lock_held(void) |
| 649 | { |
| 650 | return srcu_read_lock_held(srcu_ctlp); |
| 651 | } |
| 652 | |
Paul E. McKenney | 6b80da4 | 2014-11-21 14:19:26 -0800 | [diff] [blame] | 653 | static unsigned long srcu_torture_completed(void) |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 654 | { |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 655 | return srcu_batches_completed(srcu_ctlp); |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 656 | } |
| 657 | |
Lai Jiangshan | 9059c94 | 2012-03-19 16:12:14 +0800 | [diff] [blame] | 658 | static void srcu_torture_deferred_free(struct rcu_torture *rp) |
| 659 | { |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 660 | call_srcu(srcu_ctlp, &rp->rtort_rcu, rcu_torture_cb); |
Lai Jiangshan | 9059c94 | 2012-03-19 16:12:14 +0800 | [diff] [blame] | 661 | } |
| 662 | |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 663 | static void srcu_torture_synchronize(void) |
| 664 | { |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 665 | synchronize_srcu(srcu_ctlp); |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 666 | } |
| 667 | |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 668 | static unsigned long srcu_torture_get_gp_state(void) |
| 669 | { |
| 670 | return get_state_synchronize_srcu(srcu_ctlp); |
| 671 | } |
| 672 | |
| 673 | static unsigned long srcu_torture_start_gp_poll(void) |
| 674 | { |
| 675 | return start_poll_synchronize_srcu(srcu_ctlp); |
| 676 | } |
| 677 | |
| 678 | static bool srcu_torture_poll_gp_state(unsigned long oldstate) |
| 679 | { |
| 680 | return poll_state_synchronize_srcu(srcu_ctlp, oldstate); |
| 681 | } |
| 682 | |
Paul E. McKenney | e3f8d37 | 2012-05-08 10:21:50 -0700 | [diff] [blame] | 683 | static void srcu_torture_call(struct rcu_head *head, |
Boqun Feng | b6a4ae7 | 2015-07-29 13:29:38 +0800 | [diff] [blame] | 684 | rcu_callback_t func) |
Paul E. McKenney | e3f8d37 | 2012-05-08 10:21:50 -0700 | [diff] [blame] | 685 | { |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 686 | call_srcu(srcu_ctlp, head, func); |
Paul E. McKenney | e3f8d37 | 2012-05-08 10:21:50 -0700 | [diff] [blame] | 687 | } |
| 688 | |
| 689 | static void srcu_torture_barrier(void) |
| 690 | { |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 691 | srcu_barrier(srcu_ctlp); |
Paul E. McKenney | e3f8d37 | 2012-05-08 10:21:50 -0700 | [diff] [blame] | 692 | } |
| 693 | |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 694 | static void srcu_torture_stats(void) |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 695 | { |
Paul E. McKenney | 115a1a5 | 2017-05-22 13:31:03 -0700 | [diff] [blame] | 696 | srcu_torture_stats_print(srcu_ctlp, torture_type, TORTURE_FLAG); |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 697 | } |
| 698 | |
Paul E. McKenney | 2ec1f2d9 | 2013-06-12 15:12:21 -0700 | [diff] [blame] | 699 | static void srcu_torture_synchronize_expedited(void) |
| 700 | { |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 701 | synchronize_srcu_expedited(srcu_ctlp); |
Paul E. McKenney | 2ec1f2d9 | 2013-06-12 15:12:21 -0700 | [diff] [blame] | 702 | } |
| 703 | |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 704 | static struct rcu_torture_ops srcu_ops = { |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 705 | .ttype = SRCU_FLAVOR, |
Lai Jiangshan | cda4dc8 | 2012-10-13 01:14:17 +0800 | [diff] [blame] | 706 | .init = rcu_sync_torture_init, |
Paul E. McKenney | 0acc512 | 2009-06-25 09:08:17 -0700 | [diff] [blame] | 707 | .readlock = srcu_torture_read_lock, |
| 708 | .read_delay = srcu_read_delay, |
| 709 | .readunlock = srcu_torture_read_unlock, |
Paul E. McKenney | a5c095e | 2021-03-13 20:05:31 -0800 | [diff] [blame] | 710 | .readlock_held = torture_srcu_read_lock_held, |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 711 | .get_gp_seq = srcu_torture_completed, |
Lai Jiangshan | 9059c94 | 2012-03-19 16:12:14 +0800 | [diff] [blame] | 712 | .deferred_free = srcu_torture_deferred_free, |
Paul E. McKenney | 0acc512 | 2009-06-25 09:08:17 -0700 | [diff] [blame] | 713 | .sync = srcu_torture_synchronize, |
Paul E. McKenney | 2ec1f2d9 | 2013-06-12 15:12:21 -0700 | [diff] [blame] | 714 | .exp_sync = srcu_torture_synchronize_expedited, |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 715 | .get_gp_state = srcu_torture_get_gp_state, |
| 716 | .start_gp_poll = srcu_torture_start_gp_poll, |
| 717 | .poll_gp_state = srcu_torture_poll_gp_state, |
Paul E. McKenney | e3f8d37 | 2012-05-08 10:21:50 -0700 | [diff] [blame] | 718 | .call = srcu_torture_call, |
| 719 | .cb_barrier = srcu_torture_barrier, |
Paul E. McKenney | 0acc512 | 2009-06-25 09:08:17 -0700 | [diff] [blame] | 720 | .stats = srcu_torture_stats, |
Paul E. McKenney | 9c0f1c7 | 2022-01-27 20:29:10 -0800 | [diff] [blame] | 721 | .cbflood_max = 50000, |
Paul E. McKenney | 5e741fa | 2017-06-06 12:52:44 -0700 | [diff] [blame] | 722 | .irq_capable = 1, |
Paul E. McKenney | 340170f | 2021-09-24 21:30:26 -0700 | [diff] [blame] | 723 | .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), |
Paul E. McKenney | 0acc512 | 2009-06-25 09:08:17 -0700 | [diff] [blame] | 724 | .name = "srcu" |
Paul E. McKenney | b2896d2 | 2006-10-04 02:17:03 -0700 | [diff] [blame] | 725 | }; |
| 726 | |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 727 | static void srcu_torture_init(void) |
| 728 | { |
| 729 | rcu_sync_torture_init(); |
| 730 | WARN_ON(init_srcu_struct(&srcu_ctld)); |
| 731 | srcu_ctlp = &srcu_ctld; |
| 732 | } |
| 733 | |
| 734 | static void srcu_torture_cleanup(void) |
| 735 | { |
Paul E. McKenney | f5ad399 | 2019-02-13 13:54:37 -0800 | [diff] [blame] | 736 | cleanup_srcu_struct(&srcu_ctld); |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 737 | srcu_ctlp = &srcu_ctl; /* In case of a later rcutorture run. */ |
| 738 | } |
| 739 | |
| 740 | /* As above, but dynamically allocated. */ |
| 741 | static struct rcu_torture_ops srcud_ops = { |
| 742 | .ttype = SRCU_FLAVOR, |
| 743 | .init = srcu_torture_init, |
| 744 | .cleanup = srcu_torture_cleanup, |
| 745 | .readlock = srcu_torture_read_lock, |
| 746 | .read_delay = srcu_read_delay, |
| 747 | .readunlock = srcu_torture_read_unlock, |
Paul E. McKenney | a5c095e | 2021-03-13 20:05:31 -0800 | [diff] [blame] | 748 | .readlock_held = torture_srcu_read_lock_held, |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 749 | .get_gp_seq = srcu_torture_completed, |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 750 | .deferred_free = srcu_torture_deferred_free, |
| 751 | .sync = srcu_torture_synchronize, |
| 752 | .exp_sync = srcu_torture_synchronize_expedited, |
Paul E. McKenney | 599d97e | 2022-07-27 16:16:41 -0700 | [diff] [blame] | 753 | .get_gp_state = srcu_torture_get_gp_state, |
| 754 | .start_gp_poll = srcu_torture_start_gp_poll, |
| 755 | .poll_gp_state = srcu_torture_poll_gp_state, |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 756 | .call = srcu_torture_call, |
| 757 | .cb_barrier = srcu_torture_barrier, |
| 758 | .stats = srcu_torture_stats, |
Paul E. McKenney | 9c0f1c7 | 2022-01-27 20:29:10 -0800 | [diff] [blame] | 759 | .cbflood_max = 50000, |
Paul E. McKenney | 5e741fa | 2017-06-06 12:52:44 -0700 | [diff] [blame] | 760 | .irq_capable = 1, |
Paul E. McKenney | 340170f | 2021-09-24 21:30:26 -0700 | [diff] [blame] | 761 | .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 762 | .name = "srcud" |
| 763 | }; |
| 764 | |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 765 | /* As above, but broken due to inappropriate reader extension. */ |
| 766 | static struct rcu_torture_ops busted_srcud_ops = { |
| 767 | .ttype = SRCU_FLAVOR, |
| 768 | .init = srcu_torture_init, |
| 769 | .cleanup = srcu_torture_cleanup, |
| 770 | .readlock = srcu_torture_read_lock, |
| 771 | .read_delay = rcu_read_delay, |
| 772 | .readunlock = srcu_torture_read_unlock, |
Paul E. McKenney | a5c095e | 2021-03-13 20:05:31 -0800 | [diff] [blame] | 773 | .readlock_held = torture_srcu_read_lock_held, |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 774 | .get_gp_seq = srcu_torture_completed, |
| 775 | .deferred_free = srcu_torture_deferred_free, |
| 776 | .sync = srcu_torture_synchronize, |
| 777 | .exp_sync = srcu_torture_synchronize_expedited, |
| 778 | .call = srcu_torture_call, |
| 779 | .cb_barrier = srcu_torture_barrier, |
| 780 | .stats = srcu_torture_stats, |
| 781 | .irq_capable = 1, |
Paul E. McKenney | 340170f | 2021-09-24 21:30:26 -0700 | [diff] [blame] | 782 | .no_pi_lock = IS_ENABLED(CONFIG_TINY_SRCU), |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 783 | .extendables = RCUTORTURE_MAX_EXTEND, |
| 784 | .name = "busted_srcud" |
| 785 | }; |
| 786 | |
Josh Triplett | 4b6c2cc | 2006-10-04 02:17:16 -0700 | [diff] [blame] | 787 | /* |
Paul E. McKenney | 40c1278 | 2022-03-17 13:29:59 -0700 | [diff] [blame] | 788 | * Definitions for trivial CONFIG_PREEMPT=n-only torture testing. |
| 789 | * This implementation does not necessarily work well with CPU hotplug. |
| 790 | */ |
| 791 | |
| 792 | static void synchronize_rcu_trivial(void) |
| 793 | { |
| 794 | int cpu; |
| 795 | |
| 796 | for_each_online_cpu(cpu) { |
| 797 | rcutorture_sched_setaffinity(current->pid, cpumask_of(cpu)); |
| 798 | WARN_ON_ONCE(raw_smp_processor_id() != cpu); |
| 799 | } |
| 800 | } |
| 801 | |
| 802 | static int rcu_torture_read_lock_trivial(void) __acquires(RCU) |
| 803 | { |
| 804 | preempt_disable(); |
| 805 | return 0; |
| 806 | } |
| 807 | |
| 808 | static void rcu_torture_read_unlock_trivial(int idx) __releases(RCU) |
| 809 | { |
| 810 | preempt_enable(); |
| 811 | } |
| 812 | |
| 813 | static struct rcu_torture_ops trivial_ops = { |
| 814 | .ttype = RCU_TRIVIAL_FLAVOR, |
| 815 | .init = rcu_sync_torture_init, |
| 816 | .readlock = rcu_torture_read_lock_trivial, |
| 817 | .read_delay = rcu_read_delay, /* just reuse rcu's version. */ |
| 818 | .readunlock = rcu_torture_read_unlock_trivial, |
| 819 | .readlock_held = torture_readlock_not_held, |
| 820 | .get_gp_seq = rcu_no_completed, |
| 821 | .sync = synchronize_rcu_trivial, |
| 822 | .exp_sync = synchronize_rcu_trivial, |
| 823 | .fqs = NULL, |
| 824 | .stats = NULL, |
| 825 | .irq_capable = 1, |
| 826 | .name = "trivial" |
| 827 | }; |
| 828 | |
Paul E. McKenney | 3b6e1dd | 2022-03-17 15:18:27 -0700 | [diff] [blame] | 829 | #ifdef CONFIG_TASKS_RCU |
| 830 | |
Paul E. McKenney | 40c1278 | 2022-03-17 13:29:59 -0700 | [diff] [blame] | 831 | /* |
Paul E. McKenney | 69c6045 | 2014-07-01 11:59:36 -0700 | [diff] [blame] | 832 | * Definitions for RCU-tasks torture testing. |
| 833 | */ |
| 834 | |
| 835 | static int tasks_torture_read_lock(void) |
| 836 | { |
| 837 | return 0; |
| 838 | } |
| 839 | |
| 840 | static void tasks_torture_read_unlock(int idx) |
| 841 | { |
| 842 | } |
| 843 | |
| 844 | static void rcu_tasks_torture_deferred_free(struct rcu_torture *p) |
| 845 | { |
| 846 | call_rcu_tasks(&p->rtort_rcu, rcu_torture_cb); |
| 847 | } |
| 848 | |
Paul E. McKenney | 9cf8fc6 | 2020-03-06 14:00:46 -0800 | [diff] [blame] | 849 | static void synchronize_rcu_mult_test(void) |
| 850 | { |
| 851 | synchronize_rcu_mult(call_rcu_tasks, call_rcu); |
| 852 | } |
| 853 | |
Paul E. McKenney | 69c6045 | 2014-07-01 11:59:36 -0700 | [diff] [blame] | 854 | static struct rcu_torture_ops tasks_ops = { |
| 855 | .ttype = RCU_TASKS_FLAVOR, |
| 856 | .init = rcu_sync_torture_init, |
| 857 | .readlock = tasks_torture_read_lock, |
| 858 | .read_delay = rcu_read_delay, /* just reuse rcu's version. */ |
| 859 | .readunlock = tasks_torture_read_unlock, |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 860 | .get_gp_seq = rcu_no_completed, |
Paul E. McKenney | 69c6045 | 2014-07-01 11:59:36 -0700 | [diff] [blame] | 861 | .deferred_free = rcu_tasks_torture_deferred_free, |
| 862 | .sync = synchronize_rcu_tasks, |
Paul E. McKenney | 9cf8fc6 | 2020-03-06 14:00:46 -0800 | [diff] [blame] | 863 | .exp_sync = synchronize_rcu_mult_test, |
Paul E. McKenney | 69c6045 | 2014-07-01 11:59:36 -0700 | [diff] [blame] | 864 | .call = call_rcu_tasks, |
| 865 | .cb_barrier = rcu_barrier_tasks, |
Paul E. McKenney | 27c0f14 | 2020-09-15 17:08:03 -0700 | [diff] [blame] | 866 | .gp_kthread_dbg = show_rcu_tasks_classic_gp_kthread, |
Paul E. McKenney | 69c6045 | 2014-07-01 11:59:36 -0700 | [diff] [blame] | 867 | .fqs = NULL, |
| 868 | .stats = NULL, |
| 869 | .irq_capable = 1, |
Paul E. McKenney | 5eabea5 | 2019-04-12 09:02:46 -0700 | [diff] [blame] | 870 | .slow_gps = 1, |
Paul E. McKenney | 69c6045 | 2014-07-01 11:59:36 -0700 | [diff] [blame] | 871 | .name = "tasks" |
| 872 | }; |
| 873 | |
Paul E. McKenney | 3b6e1dd | 2022-03-17 15:18:27 -0700 | [diff] [blame] | 874 | #define TASKS_OPS &tasks_ops, |
Paul E. McKenney | c682db5 | 2019-04-19 07:38:27 -0700 | [diff] [blame] | 875 | |
Paul E. McKenney | 3b6e1dd | 2022-03-17 15:18:27 -0700 | [diff] [blame] | 876 | #else // #ifdef CONFIG_TASKS_RCU |
Paul E. McKenney | c682db5 | 2019-04-19 07:38:27 -0700 | [diff] [blame] | 877 | |
Paul E. McKenney | 3b6e1dd | 2022-03-17 15:18:27 -0700 | [diff] [blame] | 878 | #define TASKS_OPS |
Paul E. McKenney | c682db5 | 2019-04-19 07:38:27 -0700 | [diff] [blame] | 879 | |
Paul E. McKenney | 3b6e1dd | 2022-03-17 15:18:27 -0700 | [diff] [blame] | 880 | #endif // #else #ifdef CONFIG_TASKS_RCU |
Paul E. McKenney | c682db5 | 2019-04-19 07:38:27 -0700 | [diff] [blame] | 881 | |
Paul E. McKenney | c682db5 | 2019-04-19 07:38:27 -0700 | [diff] [blame] | 882 | |
Paul E. McKenney | 4c3f7b0 | 2022-03-17 16:16:45 -0700 | [diff] [blame] | 883 | #ifdef CONFIG_TASKS_RUDE_RCU |
Paul E. McKenney | c682db5 | 2019-04-19 07:38:27 -0700 | [diff] [blame] | 884 | |
Paul E. McKenney | 3d6e43c | 2020-03-03 15:02:50 -0800 | [diff] [blame] | 885 | /* |
| 886 | * Definitions for rude RCU-tasks torture testing. |
| 887 | */ |
| 888 | |
| 889 | static void rcu_tasks_rude_torture_deferred_free(struct rcu_torture *p) |
| 890 | { |
| 891 | call_rcu_tasks_rude(&p->rtort_rcu, rcu_torture_cb); |
| 892 | } |
| 893 | |
| 894 | static struct rcu_torture_ops tasks_rude_ops = { |
| 895 | .ttype = RCU_TASKS_RUDE_FLAVOR, |
| 896 | .init = rcu_sync_torture_init, |
| 897 | .readlock = rcu_torture_read_lock_trivial, |
| 898 | .read_delay = rcu_read_delay, /* just reuse rcu's version. */ |
| 899 | .readunlock = rcu_torture_read_unlock_trivial, |
| 900 | .get_gp_seq = rcu_no_completed, |
| 901 | .deferred_free = rcu_tasks_rude_torture_deferred_free, |
| 902 | .sync = synchronize_rcu_tasks_rude, |
| 903 | .exp_sync = synchronize_rcu_tasks_rude, |
| 904 | .call = call_rcu_tasks_rude, |
| 905 | .cb_barrier = rcu_barrier_tasks_rude, |
Paul E. McKenney | 27c0f14 | 2020-09-15 17:08:03 -0700 | [diff] [blame] | 906 | .gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread, |
Paul E. McKenney | 613b00f | 2021-11-23 11:53:52 -0800 | [diff] [blame] | 907 | .cbflood_max = 50000, |
Paul E. McKenney | 3d6e43c | 2020-03-03 15:02:50 -0800 | [diff] [blame] | 908 | .fqs = NULL, |
| 909 | .stats = NULL, |
| 910 | .irq_capable = 1, |
| 911 | .name = "tasks-rude" |
| 912 | }; |
| 913 | |
Paul E. McKenney | 4c3f7b0 | 2022-03-17 16:16:45 -0700 | [diff] [blame] | 914 | #define TASKS_RUDE_OPS &tasks_rude_ops, |
| 915 | |
| 916 | #else // #ifdef CONFIG_TASKS_RUDE_RCU |
| 917 | |
| 918 | #define TASKS_RUDE_OPS |
| 919 | |
| 920 | #endif // #else #ifdef CONFIG_TASKS_RUDE_RCU |
| 921 | |
| 922 | |
Paul E. McKenney | 40c1278 | 2022-03-17 13:29:59 -0700 | [diff] [blame] | 923 | #ifdef CONFIG_TASKS_TRACE_RCU |
| 924 | |
Paul E. McKenney | c1a76c0 | 2020-03-10 10:32:30 -0700 | [diff] [blame] | 925 | /* |
| 926 | * Definitions for tracing RCU-tasks torture testing. |
| 927 | */ |
| 928 | |
| 929 | static int tasks_tracing_torture_read_lock(void) |
| 930 | { |
| 931 | rcu_read_lock_trace(); |
| 932 | return 0; |
| 933 | } |
| 934 | |
| 935 | static void tasks_tracing_torture_read_unlock(int idx) |
| 936 | { |
| 937 | rcu_read_unlock_trace(); |
| 938 | } |
| 939 | |
| 940 | static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p) |
| 941 | { |
| 942 | call_rcu_tasks_trace(&p->rtort_rcu, rcu_torture_cb); |
| 943 | } |
| 944 | |
| 945 | static struct rcu_torture_ops tasks_tracing_ops = { |
| 946 | .ttype = RCU_TASKS_TRACING_FLAVOR, |
| 947 | .init = rcu_sync_torture_init, |
| 948 | .readlock = tasks_tracing_torture_read_lock, |
| 949 | .read_delay = srcu_read_delay, /* just reuse srcu's version. */ |
| 950 | .readunlock = tasks_tracing_torture_read_unlock, |
Paul E. McKenney | a5c095e | 2021-03-13 20:05:31 -0800 | [diff] [blame] | 951 | .readlock_held = rcu_read_lock_trace_held, |
Paul E. McKenney | c1a76c0 | 2020-03-10 10:32:30 -0700 | [diff] [blame] | 952 | .get_gp_seq = rcu_no_completed, |
| 953 | .deferred_free = rcu_tasks_tracing_torture_deferred_free, |
| 954 | .sync = synchronize_rcu_tasks_trace, |
| 955 | .exp_sync = synchronize_rcu_tasks_trace, |
| 956 | .call = call_rcu_tasks_trace, |
| 957 | .cb_barrier = rcu_barrier_tasks_trace, |
Paul E. McKenney | 27c0f14 | 2020-09-15 17:08:03 -0700 | [diff] [blame] | 958 | .gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread, |
Paul E. McKenney | 613b00f | 2021-11-23 11:53:52 -0800 | [diff] [blame] | 959 | .cbflood_max = 50000, |
Paul E. McKenney | c1a76c0 | 2020-03-10 10:32:30 -0700 | [diff] [blame] | 960 | .fqs = NULL, |
| 961 | .stats = NULL, |
| 962 | .irq_capable = 1, |
| 963 | .slow_gps = 1, |
| 964 | .name = "tasks-tracing" |
| 965 | }; |
| 966 | |
Paul E. McKenney | 40c1278 | 2022-03-17 13:29:59 -0700 | [diff] [blame] | 967 | #define TASKS_TRACING_OPS &tasks_tracing_ops, |
| 968 | |
| 969 | #else // #ifdef CONFIG_TASKS_TRACE_RCU |
| 970 | |
| 971 | #define TASKS_TRACING_OPS |
| 972 | |
| 973 | #endif // #else #ifdef CONFIG_TASKS_TRACE_RCU |
| 974 | |
| 975 | |
Paul E. McKenney | d7219312 | 2018-05-15 15:24:41 -0700 | [diff] [blame] | 976 | static unsigned long rcutorture_seq_diff(unsigned long new, unsigned long old) |
| 977 | { |
| 978 | if (!cur_ops->gp_diff) |
| 979 | return new - old; |
| 980 | return cur_ops->gp_diff(new, old); |
| 981 | } |
| 982 | |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 983 | /* |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 984 | * RCU torture priority-boost testing. Runs one real-time thread per |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 985 | * CPU for moderate bursts, repeatedly starting grace periods and waiting |
| 986 | * for them to complete. If a given grace period takes too long, we assume |
| 987 | * that priority inversion has occurred. |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 988 | */ |
| 989 | |
Joel Fernandes (Google) | 450efca | 2018-06-10 16:45:43 -0700 | [diff] [blame] | 990 | static int old_rt_runtime = -1; |
| 991 | |
| 992 | static void rcu_torture_disable_rt_throttle(void) |
| 993 | { |
| 994 | /* |
| 995 | * Disable RT throttling so that rcutorture's boost threads don't get |
| 996 | * throttled. Only possible if rcutorture is built-in otherwise the |
| 997 | * user should manually do this by setting the sched_rt_period_us and |
| 998 | * sched_rt_runtime sysctls. |
| 999 | */ |
| 1000 | if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime != -1) |
| 1001 | return; |
| 1002 | |
| 1003 | old_rt_runtime = sysctl_sched_rt_runtime; |
| 1004 | sysctl_sched_rt_runtime = -1; |
| 1005 | } |
| 1006 | |
| 1007 | static void rcu_torture_enable_rt_throttle(void) |
| 1008 | { |
| 1009 | if (!IS_BUILTIN(CONFIG_RCU_TORTURE_TEST) || old_rt_runtime == -1) |
| 1010 | return; |
| 1011 | |
| 1012 | sysctl_sched_rt_runtime = old_rt_runtime; |
| 1013 | old_rt_runtime = -1; |
| 1014 | } |
| 1015 | |
Paul E. McKenney | 063f5a4 | 2021-04-14 13:00:10 -0700 | [diff] [blame] | 1016 | static bool rcu_torture_boost_failed(unsigned long gp_state, unsigned long *start) |
Joel Fernandes (Google) | 3b745c8 | 2018-06-10 16:45:44 -0700 | [diff] [blame] | 1017 | { |
Paul E. McKenney | 0260b92 | 2021-04-08 13:01:14 -0700 | [diff] [blame] | 1018 | int cpu; |
Paul E. McKenney | 5e59fba | 2021-01-15 13:30:38 -0800 | [diff] [blame] | 1019 | static int dbg_done; |
Paul E. McKenney | 063f5a4 | 2021-04-14 13:00:10 -0700 | [diff] [blame] | 1020 | unsigned long end = jiffies; |
Paul E. McKenney | bcd4af4 | 2021-04-08 10:46:55 -0700 | [diff] [blame] | 1021 | bool gp_done; |
Paul E. McKenney | 0260b92 | 2021-04-08 13:01:14 -0700 | [diff] [blame] | 1022 | unsigned long j; |
| 1023 | static unsigned long last_persist; |
| 1024 | unsigned long lp; |
| 1025 | unsigned long mininterval = test_boost_duration * HZ - HZ / 2; |
Paul E. McKenney | 5e59fba | 2021-01-15 13:30:38 -0800 | [diff] [blame] | 1026 | |
Paul E. McKenney | 063f5a4 | 2021-04-14 13:00:10 -0700 | [diff] [blame] | 1027 | if (end - *start > mininterval) { |
Paul E. McKenney | 7b9dad7 | 2021-04-07 17:09:37 -0700 | [diff] [blame] | 1028 | // Recheck after checking time to avoid false positives. |
| 1029 | smp_mb(); // Time check before grace-period check. |
| 1030 | if (cur_ops->poll_gp_state(gp_state)) |
| 1031 | return false; // passed, though perhaps just barely |
Paul E. McKenney | 0260b92 | 2021-04-08 13:01:14 -0700 | [diff] [blame] | 1032 | if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, &cpu)) { |
| 1033 | // At most one persisted message per boost test. |
| 1034 | j = jiffies; |
| 1035 | lp = READ_ONCE(last_persist); |
| 1036 | if (time_after(j, lp + mininterval) && cmpxchg(&last_persist, lp, j) == lp) |
| 1037 | pr_info("Boost inversion persisted: No QS from CPU %d\n", cpu); |
| 1038 | return false; // passed on a technicality |
| 1039 | } |
Joel Fernandes (Google) | 3b745c8 | 2018-06-10 16:45:44 -0700 | [diff] [blame] | 1040 | VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed"); |
| 1041 | n_rcu_torture_boost_failure++; |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 1042 | if (!xchg(&dbg_done, 1) && cur_ops->gp_kthread_dbg) { |
| 1043 | pr_info("Boost inversion thread ->rt_priority %u gp_state %lu jiffies %lu\n", |
Paul E. McKenney | 063f5a4 | 2021-04-14 13:00:10 -0700 | [diff] [blame] | 1044 | current->rt_priority, gp_state, end - *start); |
Paul E. McKenney | 5e59fba | 2021-01-15 13:30:38 -0800 | [diff] [blame] | 1045 | cur_ops->gp_kthread_dbg(); |
Paul E. McKenney | bcd4af4 | 2021-04-08 10:46:55 -0700 | [diff] [blame] | 1046 | // Recheck after print to flag grace period ending during splat. |
| 1047 | gp_done = cur_ops->poll_gp_state(gp_state); |
| 1048 | pr_info("Boost inversion: GP %lu %s.\n", gp_state, |
| 1049 | gp_done ? "ended already" : "still pending"); |
Joel Fernandes (Google) | 3b745c8 | 2018-06-10 16:45:44 -0700 | [diff] [blame] | 1050 | |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 1051 | } |
Joel Fernandes (Google) | 3b745c8 | 2018-06-10 16:45:44 -0700 | [diff] [blame] | 1052 | |
Paul E. McKenney | 7b9dad7 | 2021-04-07 17:09:37 -0700 | [diff] [blame] | 1053 | return true; // failed |
Paul E. McKenney | 063f5a4 | 2021-04-14 13:00:10 -0700 | [diff] [blame] | 1054 | } else if (cur_ops->check_boost_failed && !cur_ops->check_boost_failed(gp_state, NULL)) { |
| 1055 | *start = jiffies; |
Joel Fernandes (Google) | 3b745c8 | 2018-06-10 16:45:44 -0700 | [diff] [blame] | 1056 | } |
| 1057 | |
Paul E. McKenney | 7b9dad7 | 2021-04-07 17:09:37 -0700 | [diff] [blame] | 1058 | return false; // passed |
Joel Fernandes (Google) | 3b745c8 | 2018-06-10 16:45:44 -0700 | [diff] [blame] | 1059 | } |
| 1060 | |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1061 | static int rcu_torture_boost(void *arg) |
| 1062 | { |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1063 | unsigned long endtime; |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 1064 | unsigned long gp_state; |
| 1065 | unsigned long gp_state_time; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1066 | unsigned long oldstarttime; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1067 | |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 1068 | VERBOSE_TOROUT_STRING("rcu_torture_boost started"); |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1069 | |
| 1070 | /* Set real-time priority. */ |
Peter Zijlstra | 8b70098 | 2020-04-22 13:10:04 +0200 | [diff] [blame] | 1071 | sched_set_fifo_low(current); |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1072 | |
| 1073 | /* Each pass through the following loop does one boost-test cycle. */ |
| 1074 | do { |
Paul E. McKenney | 5e59fba | 2021-01-15 13:30:38 -0800 | [diff] [blame] | 1075 | bool failed = false; // Test failed already in this test interval |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 1076 | bool gp_initiated = false; |
Joel Fernandes (Google) | 3b745c8 | 2018-06-10 16:45:44 -0700 | [diff] [blame] | 1077 | |
Joel Fernandes (Google) | 3b745c8 | 2018-06-10 16:45:44 -0700 | [diff] [blame] | 1078 | if (kthread_should_stop()) |
| 1079 | goto checkwait; |
| 1080 | |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1081 | /* Wait for the next test interval. */ |
Paul E. McKenney | a47f9f1 | 2021-12-13 11:05:07 -0800 | [diff] [blame] | 1082 | oldstarttime = READ_ONCE(boost_starttime); |
Paul E. McKenney | 3c80b40 | 2020-04-10 15:37:12 -0700 | [diff] [blame] | 1083 | while (time_before(jiffies, oldstarttime)) { |
Paul E. McKenney | 0e11c8e | 2013-01-10 16:21:07 -0800 | [diff] [blame] | 1084 | schedule_timeout_interruptible(oldstarttime - jiffies); |
Paul E. McKenney | ab1b788 | 2020-09-22 16:42:42 -0700 | [diff] [blame] | 1085 | if (stutter_wait("rcu_torture_boost")) |
| 1086 | sched_set_fifo_low(current); |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 1087 | if (torture_must_stop()) |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1088 | goto checkwait; |
| 1089 | } |
| 1090 | |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 1091 | // Do one boost-test interval. |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1092 | endtime = oldstarttime + test_boost_duration * HZ; |
Paul E. McKenney | 3c80b40 | 2020-04-10 15:37:12 -0700 | [diff] [blame] | 1093 | while (time_before(jiffies, endtime)) { |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 1094 | // Has current GP gone too long? |
| 1095 | if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) |
Paul E. McKenney | 063f5a4 | 2021-04-14 13:00:10 -0700 | [diff] [blame] | 1096 | failed = rcu_torture_boost_failed(gp_state, &gp_state_time); |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 1097 | // If we don't have a grace period in flight, start one. |
| 1098 | if (!gp_initiated || cur_ops->poll_gp_state(gp_state)) { |
| 1099 | gp_state = cur_ops->start_gp_poll(); |
| 1100 | gp_initiated = true; |
| 1101 | gp_state_time = jiffies; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1102 | } |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 1103 | if (stutter_wait("rcu_torture_boost")) { |
Paul E. McKenney | ab1b788 | 2020-09-22 16:42:42 -0700 | [diff] [blame] | 1104 | sched_set_fifo_low(current); |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 1105 | // If the grace period already ended, |
| 1106 | // we don't know when that happened, so |
| 1107 | // start over. |
| 1108 | if (cur_ops->poll_gp_state(gp_state)) |
| 1109 | gp_initiated = false; |
| 1110 | } |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 1111 | if (torture_must_stop()) |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1112 | goto checkwait; |
| 1113 | } |
| 1114 | |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 1115 | // In case the grace period extended beyond the end of the loop. |
| 1116 | if (gp_initiated && !failed && !cur_ops->poll_gp_state(gp_state)) |
Paul E. McKenney | 063f5a4 | 2021-04-14 13:00:10 -0700 | [diff] [blame] | 1117 | rcu_torture_boost_failed(gp_state, &gp_state_time); |
Joel Fernandes (Google) | 3b745c8 | 2018-06-10 16:45:44 -0700 | [diff] [blame] | 1118 | |
| 1119 | /* |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1120 | * Set the start time of the next test interval. |
| 1121 | * Yes, this is vulnerable to long delays, but such |
| 1122 | * delays simply cause a false negative for the next |
| 1123 | * interval. Besides, we are running at RT priority, |
| 1124 | * so delays should be relatively rare. |
| 1125 | */ |
Paul E. McKenney | a47f9f1 | 2021-12-13 11:05:07 -0800 | [diff] [blame] | 1126 | while (oldstarttime == READ_ONCE(boost_starttime) && !kthread_should_stop()) { |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1127 | if (mutex_trylock(&boost_mutex)) { |
Paul E. McKenney | 8c7ec02 | 2021-04-07 20:00:00 -0700 | [diff] [blame] | 1128 | if (oldstarttime == boost_starttime) { |
Paul E. McKenney | a47f9f1 | 2021-12-13 11:05:07 -0800 | [diff] [blame] | 1129 | WRITE_ONCE(boost_starttime, |
| 1130 | jiffies + test_boost_interval * HZ); |
Paul E. McKenney | 8c7ec02 | 2021-04-07 20:00:00 -0700 | [diff] [blame] | 1131 | n_rcu_torture_boosts++; |
| 1132 | } |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1133 | mutex_unlock(&boost_mutex); |
| 1134 | break; |
| 1135 | } |
| 1136 | schedule_timeout_uninterruptible(1); |
| 1137 | } |
| 1138 | |
| 1139 | /* Go do the stutter. */ |
Paul E. McKenney | ab1b788 | 2020-09-22 16:42:42 -0700 | [diff] [blame] | 1140 | checkwait: if (stutter_wait("rcu_torture_boost")) |
| 1141 | sched_set_fifo_low(current); |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 1142 | } while (!torture_must_stop()); |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1143 | |
| 1144 | /* Clean up and exit. */ |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 1145 | while (!kthread_should_stop()) { |
Paul E. McKenney | 7fafaac | 2014-01-31 17:37:28 -0800 | [diff] [blame] | 1146 | torture_shutdown_absorb("rcu_torture_boost"); |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1147 | schedule_timeout_uninterruptible(1); |
Paul E. McKenney | 7fafaac | 2014-01-31 17:37:28 -0800 | [diff] [blame] | 1148 | } |
Paul E. McKenney | 7fafaac | 2014-01-31 17:37:28 -0800 | [diff] [blame] | 1149 | torture_kthread_stopping("rcu_torture_boost"); |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 1150 | return 0; |
| 1151 | } |
| 1152 | |
| 1153 | /* |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 1154 | * RCU torture force-quiescent-state kthread. Repeatedly induces |
| 1155 | * bursts of calls to force_quiescent_state(), increasing the probability |
| 1156 | * of occurrence of some important types of race conditions. |
| 1157 | */ |
| 1158 | static int |
| 1159 | rcu_torture_fqs(void *arg) |
| 1160 | { |
| 1161 | unsigned long fqs_resume_time; |
| 1162 | int fqs_burst_remaining; |
Paul E. McKenney | ab1b788 | 2020-09-22 16:42:42 -0700 | [diff] [blame] | 1163 | int oldnice = task_nice(current); |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 1164 | |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 1165 | VERBOSE_TOROUT_STRING("rcu_torture_fqs task started"); |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 1166 | do { |
| 1167 | fqs_resume_time = jiffies + fqs_stutter * HZ; |
Paul E. McKenney | 3c80b40 | 2020-04-10 15:37:12 -0700 | [diff] [blame] | 1168 | while (time_before(jiffies, fqs_resume_time) && |
Paul E. McKenney | 93898fb | 2011-08-17 12:39:34 -0700 | [diff] [blame] | 1169 | !kthread_should_stop()) { |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 1170 | schedule_timeout_interruptible(1); |
| 1171 | } |
| 1172 | fqs_burst_remaining = fqs_duration; |
Paul E. McKenney | 93898fb | 2011-08-17 12:39:34 -0700 | [diff] [blame] | 1173 | while (fqs_burst_remaining > 0 && |
| 1174 | !kthread_should_stop()) { |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 1175 | cur_ops->fqs(); |
| 1176 | udelay(fqs_holdoff); |
| 1177 | fqs_burst_remaining -= fqs_holdoff; |
| 1178 | } |
Paul E. McKenney | ab1b788 | 2020-09-22 16:42:42 -0700 | [diff] [blame] | 1179 | if (stutter_wait("rcu_torture_fqs")) |
| 1180 | sched_set_normal(current, oldnice); |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 1181 | } while (!torture_must_stop()); |
Paul E. McKenney | 7fafaac | 2014-01-31 17:37:28 -0800 | [diff] [blame] | 1182 | torture_kthread_stopping("rcu_torture_fqs"); |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 1183 | return 0; |
| 1184 | } |
| 1185 | |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 1186 | // Used by writers to randomly choose from the available grace-period primitives. |
| 1187 | static int synctype[ARRAY_SIZE(rcu_torture_writer_state_names)] = { }; |
Paul E. McKenney | 18fbf30 | 2020-11-16 16:46:06 -0800 | [diff] [blame] | 1188 | static int nsynctypes; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 1189 | |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 1190 | /* |
Paul E. McKenney | 18fbf30 | 2020-11-16 16:46:06 -0800 | [diff] [blame] | 1191 | * Determine which grace-period primitives are available. |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 1192 | */ |
Paul E. McKenney | 18fbf30 | 2020-11-16 16:46:06 -0800 | [diff] [blame] | 1193 | static void rcu_torture_write_types(void) |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 1194 | { |
Paul E. McKenney | b6fe491 | 2022-08-04 13:46:05 -0700 | [diff] [blame] | 1195 | bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full; |
Paul E. McKenney | 8df13f0 | 2022-08-04 15:23:26 -0700 | [diff] [blame] | 1196 | bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp; |
| 1197 | bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll; |
| 1198 | bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 1199 | |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1200 | /* Initialize synctype[] array. If none set, take default. */ |
Paul E. McKenney | 5d7801f | 2022-08-23 12:11:12 -0700 | [diff] [blame] | 1201 | if (!gp_cond1 && |
| 1202 | !gp_cond_exp1 && |
| 1203 | !gp_cond_full1 && |
| 1204 | !gp_cond_exp_full1 && |
| 1205 | !gp_exp1 && |
| 1206 | !gp_poll_exp1 && |
| 1207 | !gp_poll_exp_full1 && |
| 1208 | !gp_normal1 && |
| 1209 | !gp_poll1 && |
| 1210 | !gp_poll_full1 && |
| 1211 | !gp_sync1) { |
| 1212 | gp_cond1 = true; |
| 1213 | gp_cond_exp1 = true; |
| 1214 | gp_cond_full1 = true; |
| 1215 | gp_cond_exp_full1 = true; |
| 1216 | gp_exp1 = true; |
| 1217 | gp_poll_exp1 = true; |
| 1218 | gp_poll_exp_full1 = true; |
| 1219 | gp_normal1 = true; |
| 1220 | gp_poll1 = true; |
| 1221 | gp_poll_full1 = true; |
| 1222 | gp_sync1 = true; |
| 1223 | } |
Paul E. McKenney | fd56f64b | 2020-11-13 20:14:27 -0800 | [diff] [blame] | 1224 | if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) { |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1225 | synctype[nsynctypes++] = RTWS_COND_GET; |
Paul E. McKenney | db0c1a8 | 2017-12-08 12:23:10 -0800 | [diff] [blame] | 1226 | pr_info("%s: Testing conditional GPs.\n", __func__); |
Paul E. McKenney | fd56f64b | 2020-11-13 20:14:27 -0800 | [diff] [blame] | 1227 | } else if (gp_cond && (!cur_ops->get_gp_state || !cur_ops->cond_sync)) { |
Paul E. McKenney | e0d31a34c | 2017-12-01 15:22:38 -0800 | [diff] [blame] | 1228 | pr_alert("%s: gp_cond without primitives.\n", __func__); |
Paul E. McKenney | db0c1a8 | 2017-12-08 12:23:10 -0800 | [diff] [blame] | 1229 | } |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 1230 | if (gp_cond_exp1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp) { |
| 1231 | synctype[nsynctypes++] = RTWS_COND_GET_EXP; |
| 1232 | pr_info("%s: Testing conditional expedited GPs.\n", __func__); |
| 1233 | } else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) { |
| 1234 | pr_alert("%s: gp_cond_exp without primitives.\n", __func__); |
| 1235 | } |
Paul E. McKenney | b6fe491 | 2022-08-04 13:46:05 -0700 | [diff] [blame] | 1236 | if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) { |
| 1237 | synctype[nsynctypes++] = RTWS_COND_GET_FULL; |
| 1238 | pr_info("%s: Testing conditional full-state GPs.\n", __func__); |
| 1239 | } else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) { |
| 1240 | pr_alert("%s: gp_cond_full without primitives.\n", __func__); |
| 1241 | } |
Paul E. McKenney | 8df13f0 | 2022-08-04 15:23:26 -0700 | [diff] [blame] | 1242 | if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) { |
| 1243 | synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL; |
| 1244 | pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__); |
| 1245 | } else if (gp_cond_exp_full && |
| 1246 | (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) { |
| 1247 | pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__); |
| 1248 | } |
Paul E. McKenney | db0c1a8 | 2017-12-08 12:23:10 -0800 | [diff] [blame] | 1249 | if (gp_exp1 && cur_ops->exp_sync) { |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1250 | synctype[nsynctypes++] = RTWS_EXP_SYNC; |
Paul E. McKenney | db0c1a8 | 2017-12-08 12:23:10 -0800 | [diff] [blame] | 1251 | pr_info("%s: Testing expedited GPs.\n", __func__); |
| 1252 | } else if (gp_exp && !cur_ops->exp_sync) { |
Paul E. McKenney | e0d31a34c | 2017-12-01 15:22:38 -0800 | [diff] [blame] | 1253 | pr_alert("%s: gp_exp without primitives.\n", __func__); |
Paul E. McKenney | db0c1a8 | 2017-12-08 12:23:10 -0800 | [diff] [blame] | 1254 | } |
| 1255 | if (gp_normal1 && cur_ops->deferred_free) { |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1256 | synctype[nsynctypes++] = RTWS_DEF_FREE; |
Paul E. McKenney | db0c1a8 | 2017-12-08 12:23:10 -0800 | [diff] [blame] | 1257 | pr_info("%s: Testing asynchronous GPs.\n", __func__); |
| 1258 | } else if (gp_normal && !cur_ops->deferred_free) { |
Paul E. McKenney | e0d31a34c | 2017-12-01 15:22:38 -0800 | [diff] [blame] | 1259 | pr_alert("%s: gp_normal without primitives.\n", __func__); |
Paul E. McKenney | db0c1a8 | 2017-12-08 12:23:10 -0800 | [diff] [blame] | 1260 | } |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 1261 | if (gp_poll1 && cur_ops->start_gp_poll && cur_ops->poll_gp_state) { |
| 1262 | synctype[nsynctypes++] = RTWS_POLL_GET; |
| 1263 | pr_info("%s: Testing polling GPs.\n", __func__); |
| 1264 | } else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) { |
| 1265 | pr_alert("%s: gp_poll without primitives.\n", __func__); |
| 1266 | } |
Paul E. McKenney | 76ea364 | 2022-08-02 17:04:54 -0700 | [diff] [blame] | 1267 | if (gp_poll_full1 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) { |
| 1268 | synctype[nsynctypes++] = RTWS_POLL_GET_FULL; |
| 1269 | pr_info("%s: Testing polling full-state GPs.\n", __func__); |
| 1270 | } else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) { |
| 1271 | pr_alert("%s: gp_poll_full without primitives.\n", __func__); |
| 1272 | } |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 1273 | if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) { |
| 1274 | synctype[nsynctypes++] = RTWS_POLL_GET_EXP; |
| 1275 | pr_info("%s: Testing polling expedited GPs.\n", __func__); |
| 1276 | } else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) { |
| 1277 | pr_alert("%s: gp_poll_exp without primitives.\n", __func__); |
| 1278 | } |
Paul E. McKenney | 6c502b1 | 2022-08-03 12:38:51 -0700 | [diff] [blame] | 1279 | if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) { |
| 1280 | synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL; |
| 1281 | pr_info("%s: Testing polling full-state expedited GPs.\n", __func__); |
| 1282 | } else if (gp_poll_exp_full && |
| 1283 | (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) { |
| 1284 | pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__); |
| 1285 | } |
Paul E. McKenney | db0c1a8 | 2017-12-08 12:23:10 -0800 | [diff] [blame] | 1286 | if (gp_sync1 && cur_ops->sync) { |
Paul E. McKenney | f0bf8fa | 2014-03-21 16:17:56 -0700 | [diff] [blame] | 1287 | synctype[nsynctypes++] = RTWS_SYNC; |
Paul E. McKenney | db0c1a8 | 2017-12-08 12:23:10 -0800 | [diff] [blame] | 1288 | pr_info("%s: Testing normal GPs.\n", __func__); |
| 1289 | } else if (gp_sync && !cur_ops->sync) { |
Paul E. McKenney | e0d31a34c | 2017-12-01 15:22:38 -0800 | [diff] [blame] | 1290 | pr_alert("%s: gp_sync without primitives.\n", __func__); |
Paul E. McKenney | db0c1a8 | 2017-12-08 12:23:10 -0800 | [diff] [blame] | 1291 | } |
Paul E. McKenney | 18fbf30 | 2020-11-16 16:46:06 -0800 | [diff] [blame] | 1292 | } |
| 1293 | |
| 1294 | /* |
Paul E. McKenney | ccb4222 | 2022-07-29 22:05:17 -0700 | [diff] [blame] | 1295 | * Do the specified rcu_torture_writer() synchronous grace period, |
| 1296 | * while also testing out the polled APIs. Note well that the single-CPU |
| 1297 | * grace-period optimizations must be accounted for. |
| 1298 | */ |
| 1299 | static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) |
| 1300 | { |
| 1301 | unsigned long cookie; |
| 1302 | struct rcu_gp_oldstate cookie_full; |
| 1303 | bool dopoll; |
| 1304 | bool dopoll_full; |
| 1305 | unsigned long r = torture_random(trsp); |
| 1306 | |
| 1307 | dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300); |
| 1308 | dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00); |
| 1309 | if (dopoll || dopoll_full) |
| 1310 | cpus_read_lock(); |
| 1311 | if (dopoll) |
| 1312 | cookie = cur_ops->get_gp_state(); |
| 1313 | if (dopoll_full) |
| 1314 | cur_ops->get_gp_state_full(&cookie_full); |
Paul E. McKenney | ed7d2f1 | 2022-08-01 16:57:17 -0700 | [diff] [blame] | 1315 | if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full)) |
Paul E. McKenney | ccb4222 | 2022-07-29 22:05:17 -0700 | [diff] [blame] | 1316 | sync(); |
| 1317 | sync(); |
| 1318 | WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie), |
| 1319 | "%s: Cookie check 3 failed %pS() online %*pbl.", |
| 1320 | __func__, sync, cpumask_pr_args(cpu_online_mask)); |
| 1321 | WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full), |
| 1322 | "%s: Cookie check 4 failed %pS() online %*pbl", |
| 1323 | __func__, sync, cpumask_pr_args(cpu_online_mask)); |
| 1324 | if (dopoll || dopoll_full) |
| 1325 | cpus_read_unlock(); |
| 1326 | } |
| 1327 | |
| 1328 | /* |
Paul E. McKenney | 18fbf30 | 2020-11-16 16:46:06 -0800 | [diff] [blame] | 1329 | * RCU torture writer kthread. Repeatedly substitutes a new structure |
| 1330 | * for that pointed to by rcu_torture_current, freeing the old structure |
| 1331 | * after a series of grace periods (the "pipeline"). |
| 1332 | */ |
| 1333 | static int |
| 1334 | rcu_torture_writer(void *arg) |
| 1335 | { |
| 1336 | bool boot_ended; |
| 1337 | bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); |
| 1338 | unsigned long cookie; |
Paul E. McKenney | 91a967f | 2022-07-28 15:37:05 -0700 | [diff] [blame] | 1339 | struct rcu_gp_oldstate cookie_full; |
Paul E. McKenney | 18fbf30 | 2020-11-16 16:46:06 -0800 | [diff] [blame] | 1340 | int expediting = 0; |
| 1341 | unsigned long gp_snap; |
Paul E. McKenney | 76ea364 | 2022-08-02 17:04:54 -0700 | [diff] [blame] | 1342 | struct rcu_gp_oldstate gp_snap_full; |
Paul E. McKenney | 18fbf30 | 2020-11-16 16:46:06 -0800 | [diff] [blame] | 1343 | int i; |
| 1344 | int idx; |
| 1345 | int oldnice = task_nice(current); |
| 1346 | struct rcu_torture *rp; |
| 1347 | struct rcu_torture *old_rp; |
| 1348 | static DEFINE_TORTURE_RANDOM(rand); |
| 1349 | bool stutter_waited; |
| 1350 | |
| 1351 | VERBOSE_TOROUT_STRING("rcu_torture_writer task started"); |
| 1352 | if (!can_expedite) |
| 1353 | pr_alert("%s" TORTURE_FLAG |
| 1354 | " GP expediting controlled from boot/sysfs for %s.\n", |
| 1355 | torture_type, cur_ops->name); |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1356 | if (WARN_ONCE(nsynctypes == 0, |
David Vernet | 80dcee6 | 2022-03-07 14:46:57 -0800 | [diff] [blame] | 1357 | "%s: No update-side primitives.\n", __func__)) { |
Paul E. McKenney | f0bf8fa | 2014-03-21 16:17:56 -0700 | [diff] [blame] | 1358 | /* |
| 1359 | * No updates primitives, so don't try updating. |
| 1360 | * The resulting test won't be testing much, hence the |
| 1361 | * above WARN_ONCE(). |
| 1362 | */ |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1363 | rcu_torture_writer_state = RTWS_STOPPING; |
| 1364 | torture_kthread_stopping("rcu_torture_writer"); |
David Vernet | 80dcee6 | 2022-03-07 14:46:57 -0800 | [diff] [blame] | 1365 | return 0; |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1366 | } |
| 1367 | |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 1368 | do { |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 1369 | rcu_torture_writer_state = RTWS_FIXED_DELAY; |
Paul E. McKenney | 1eba0ef | 2020-11-17 14:12:24 -0800 | [diff] [blame] | 1370 | torture_hrtimeout_us(500, 1000, &rand); |
Paul E. McKenney | a71fca5 | 2009-09-18 10:28:19 -0700 | [diff] [blame] | 1371 | rp = rcu_torture_alloc(); |
| 1372 | if (rp == NULL) |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 1373 | continue; |
| 1374 | rp->rtort_pipe_count = 0; |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 1375 | rcu_torture_writer_state = RTWS_DELAY; |
Paul E. McKenney | 51b1130 | 2014-01-27 11:49:39 -0800 | [diff] [blame] | 1376 | udelay(torture_random(&rand) & 0x3ff); |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 1377 | rcu_torture_writer_state = RTWS_REPLACE; |
Paul E. McKenney | 0ddea0e | 2010-09-19 21:06:14 -0700 | [diff] [blame] | 1378 | old_rp = rcu_dereference_check(rcu_torture_current, |
| 1379 | current == writer_task); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 1380 | rp->rtort_mbtest = 1; |
| 1381 | rcu_assign_pointer(rcu_torture_current, rp); |
Paul E. McKenney | 9b2619a | 2009-09-23 09:50:43 -0700 | [diff] [blame] | 1382 | smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */ |
Josh Triplett | c8e5b16 | 2007-05-08 00:33:20 -0700 | [diff] [blame] | 1383 | if (old_rp) { |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 1384 | i = old_rp->rtort_pipe_count; |
| 1385 | if (i > RCU_TORTURE_PIPE_LEN) |
| 1386 | i = RCU_TORTURE_PIPE_LEN; |
| 1387 | atomic_inc(&rcu_torture_wcount[i]); |
Paul E. McKenney | 2024891 | 2019-12-21 10:41:48 -0800 | [diff] [blame] | 1388 | WRITE_ONCE(old_rp->rtort_pipe_count, |
| 1389 | old_rp->rtort_pipe_count + 1); |
Paul E. McKenney | d594231 | 2022-08-01 17:33:24 -0700 | [diff] [blame] | 1390 | |
| 1391 | // Make sure readers block polled grace periods. |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 1392 | if (cur_ops->get_gp_state && cur_ops->poll_gp_state) { |
| 1393 | idx = cur_ops->readlock(); |
| 1394 | cookie = cur_ops->get_gp_state(); |
Paul E. McKenney | 37d6ade | 2022-08-02 10:22:12 -0700 | [diff] [blame] | 1395 | WARN_ONCE(cur_ops->poll_gp_state(cookie), |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 1396 | "%s: Cookie check 1 failed %s(%d) %lu->%lu\n", |
| 1397 | __func__, |
| 1398 | rcu_torture_writer_state_getname(), |
| 1399 | rcu_torture_writer_state, |
| 1400 | cookie, cur_ops->get_gp_state()); |
Paul E. McKenney | d0eac20 | 2022-04-13 16:14:02 -0700 | [diff] [blame] | 1401 | if (cur_ops->get_gp_completed) { |
| 1402 | cookie = cur_ops->get_gp_completed(); |
| 1403 | WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie)); |
| 1404 | } |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 1405 | cur_ops->readunlock(idx); |
| 1406 | } |
Paul E. McKenney | d594231 | 2022-08-01 17:33:24 -0700 | [diff] [blame] | 1407 | if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) { |
| 1408 | idx = cur_ops->readlock(); |
| 1409 | cur_ops->get_gp_state_full(&cookie_full); |
| 1410 | WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), |
| 1411 | "%s: Cookie check 5 failed %s(%d) online %*pbl\n", |
| 1412 | __func__, |
| 1413 | rcu_torture_writer_state_getname(), |
| 1414 | rcu_torture_writer_state, |
| 1415 | cpumask_pr_args(cpu_online_mask)); |
| 1416 | if (cur_ops->get_gp_completed_full) { |
| 1417 | cur_ops->get_gp_completed_full(&cookie_full); |
| 1418 | WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full)); |
| 1419 | } |
| 1420 | cur_ops->readunlock(idx); |
Paul E. McKenney | 91a967f | 2022-07-28 15:37:05 -0700 | [diff] [blame] | 1421 | } |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1422 | switch (synctype[torture_random(&rand) % nsynctypes]) { |
| 1423 | case RTWS_DEF_FREE: |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 1424 | rcu_torture_writer_state = RTWS_DEF_FREE; |
Paul E. McKenney | 2ec1f2d9 | 2013-06-12 15:12:21 -0700 | [diff] [blame] | 1425 | cur_ops->deferred_free(old_rp); |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1426 | break; |
| 1427 | case RTWS_EXP_SYNC: |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 1428 | rcu_torture_writer_state = RTWS_EXP_SYNC; |
Paul E. McKenney | ccb4222 | 2022-07-29 22:05:17 -0700 | [diff] [blame] | 1429 | do_rtws_sync(&rand, cur_ops->exp_sync); |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1430 | rcu_torture_pipe_update(old_rp); |
| 1431 | break; |
| 1432 | case RTWS_COND_GET: |
| 1433 | rcu_torture_writer_state = RTWS_COND_GET; |
Paul E. McKenney | fd56f64b | 2020-11-13 20:14:27 -0800 | [diff] [blame] | 1434 | gp_snap = cur_ops->get_gp_state(); |
Paul E. McKenney | ea31fd9 | 2020-11-17 11:32:54 -0800 | [diff] [blame] | 1435 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1436 | rcu_torture_writer_state = RTWS_COND_SYNC; |
| 1437 | cur_ops->cond_sync(gp_snap); |
| 1438 | rcu_torture_pipe_update(old_rp); |
| 1439 | break; |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 1440 | case RTWS_COND_GET_EXP: |
| 1441 | rcu_torture_writer_state = RTWS_COND_GET_EXP; |
| 1442 | gp_snap = cur_ops->get_gp_state_exp(); |
| 1443 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); |
| 1444 | rcu_torture_writer_state = RTWS_COND_SYNC_EXP; |
| 1445 | cur_ops->cond_sync_exp(gp_snap); |
| 1446 | rcu_torture_pipe_update(old_rp); |
| 1447 | break; |
Paul E. McKenney | b6fe491 | 2022-08-04 13:46:05 -0700 | [diff] [blame] | 1448 | case RTWS_COND_GET_FULL: |
| 1449 | rcu_torture_writer_state = RTWS_COND_GET_FULL; |
| 1450 | cur_ops->get_gp_state_full(&gp_snap_full); |
| 1451 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); |
| 1452 | rcu_torture_writer_state = RTWS_COND_SYNC_FULL; |
| 1453 | cur_ops->cond_sync_full(&gp_snap_full); |
| 1454 | rcu_torture_pipe_update(old_rp); |
| 1455 | break; |
Paul E. McKenney | 8df13f0 | 2022-08-04 15:23:26 -0700 | [diff] [blame] | 1456 | case RTWS_COND_GET_EXP_FULL: |
| 1457 | rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL; |
| 1458 | cur_ops->get_gp_state_full(&gp_snap_full); |
| 1459 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); |
| 1460 | rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL; |
| 1461 | cur_ops->cond_sync_exp_full(&gp_snap_full); |
| 1462 | rcu_torture_pipe_update(old_rp); |
| 1463 | break; |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 1464 | case RTWS_POLL_GET: |
| 1465 | rcu_torture_writer_state = RTWS_POLL_GET; |
| 1466 | gp_snap = cur_ops->start_gp_poll(); |
| 1467 | rcu_torture_writer_state = RTWS_POLL_WAIT; |
Paul E. McKenney | ea31fd9 | 2020-11-17 11:32:54 -0800 | [diff] [blame] | 1468 | while (!cur_ops->poll_gp_state(gp_snap)) |
| 1469 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, |
| 1470 | &rand); |
Paul E. McKenney | 0fd0548 | 2020-11-13 20:43:59 -0800 | [diff] [blame] | 1471 | rcu_torture_pipe_update(old_rp); |
| 1472 | break; |
Paul E. McKenney | 76ea364 | 2022-08-02 17:04:54 -0700 | [diff] [blame] | 1473 | case RTWS_POLL_GET_FULL: |
| 1474 | rcu_torture_writer_state = RTWS_POLL_GET_FULL; |
| 1475 | cur_ops->start_gp_poll_full(&gp_snap_full); |
| 1476 | rcu_torture_writer_state = RTWS_POLL_WAIT_FULL; |
| 1477 | while (!cur_ops->poll_gp_state_full(&gp_snap_full)) |
| 1478 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, |
| 1479 | &rand); |
| 1480 | rcu_torture_pipe_update(old_rp); |
| 1481 | break; |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 1482 | case RTWS_POLL_GET_EXP: |
| 1483 | rcu_torture_writer_state = RTWS_POLL_GET_EXP; |
| 1484 | gp_snap = cur_ops->start_gp_poll_exp(); |
| 1485 | rcu_torture_writer_state = RTWS_POLL_WAIT_EXP; |
| 1486 | while (!cur_ops->poll_gp_state_exp(gp_snap)) |
| 1487 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, |
| 1488 | &rand); |
| 1489 | rcu_torture_pipe_update(old_rp); |
| 1490 | break; |
Paul E. McKenney | 6c502b1 | 2022-08-03 12:38:51 -0700 | [diff] [blame] | 1491 | case RTWS_POLL_GET_EXP_FULL: |
| 1492 | rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL; |
| 1493 | cur_ops->start_gp_poll_exp_full(&gp_snap_full); |
| 1494 | rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL; |
| 1495 | while (!cur_ops->poll_gp_state_full(&gp_snap_full)) |
| 1496 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, |
| 1497 | &rand); |
| 1498 | rcu_torture_pipe_update(old_rp); |
| 1499 | break; |
Paul E. McKenney | f0bf8fa | 2014-03-21 16:17:56 -0700 | [diff] [blame] | 1500 | case RTWS_SYNC: |
| 1501 | rcu_torture_writer_state = RTWS_SYNC; |
Paul E. McKenney | ccb4222 | 2022-07-29 22:05:17 -0700 | [diff] [blame] | 1502 | do_rtws_sync(&rand, cur_ops->sync); |
Paul E. McKenney | f0bf8fa | 2014-03-21 16:17:56 -0700 | [diff] [blame] | 1503 | rcu_torture_pipe_update(old_rp); |
| 1504 | break; |
Paul E. McKenney | a48f3fa | 2014-03-18 15:57:41 -0700 | [diff] [blame] | 1505 | default: |
| 1506 | WARN_ON_ONCE(1); |
| 1507 | break; |
Paul E. McKenney | 2ec1f2d9 | 2013-06-12 15:12:21 -0700 | [diff] [blame] | 1508 | } |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 1509 | } |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 1510 | WRITE_ONCE(rcu_torture_current_version, |
| 1511 | rcu_torture_current_version + 1); |
Paul E. McKenney | 4bb3c5f | 2015-02-18 16:31:29 -0800 | [diff] [blame] | 1512 | /* Cycle through nesting levels of rcu_expedite_gp() calls. */ |
| 1513 | if (can_expedite && |
| 1514 | !(torture_random(&rand) & 0xff & (!!expediting - 1))) { |
| 1515 | WARN_ON_ONCE(expediting == 0 && rcu_gp_is_expedited()); |
| 1516 | if (expediting >= 0) |
| 1517 | rcu_expedite_gp(); |
| 1518 | else |
| 1519 | rcu_unexpedite_gp(); |
| 1520 | if (++expediting > 3) |
| 1521 | expediting = -expediting; |
Paul E. McKenney | f7c0e6a | 2017-12-08 11:37:24 -0800 | [diff] [blame] | 1522 | } else if (!can_expedite) { /* Disabled during boot, recheck. */ |
| 1523 | can_expedite = !rcu_gp_is_expedited() && |
| 1524 | !rcu_gp_is_normal(); |
Paul E. McKenney | 4bb3c5f | 2015-02-18 16:31:29 -0800 | [diff] [blame] | 1525 | } |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 1526 | rcu_torture_writer_state = RTWS_STUTTER; |
Paul E. McKenney | 12a910e | 2020-11-16 16:01:50 -0800 | [diff] [blame] | 1527 | boot_ended = rcu_inkernel_boot_has_ended(); |
Paul E. McKenney | ab1b788 | 2020-09-22 16:42:42 -0700 | [diff] [blame] | 1528 | stutter_waited = stutter_wait("rcu_torture_writer"); |
| 1529 | if (stutter_waited && |
Paul E. McKenney | e22ef8d | 2021-12-17 12:33:53 -0800 | [diff] [blame] | 1530 | !atomic_read(&rcu_fwd_cb_nodelay) && |
Paul E. McKenney | 3432d76 | 2019-04-15 14:50:05 -0700 | [diff] [blame] | 1531 | !cur_ops->slow_gps && |
Paul E. McKenney | 59ee032 | 2019-11-28 18:54:06 -0800 | [diff] [blame] | 1532 | !torture_must_stop() && |
Paul E. McKenney | 12a910e | 2020-11-16 16:01:50 -0800 | [diff] [blame] | 1533 | boot_ended) |
Paul E. McKenney | 474e59b | 2018-08-07 14:34:44 -0700 | [diff] [blame] | 1534 | for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) |
Paul E. McKenney | e8516c6 | 2019-04-09 11:06:32 -0700 | [diff] [blame] | 1535 | if (list_empty(&rcu_tortures[i].rtort_free) && |
| 1536 | rcu_access_pointer(rcu_torture_current) != |
Paul E. McKenney | 34aa34b | 2019-05-16 16:15:16 -0700 | [diff] [blame] | 1537 | &rcu_tortures[i]) { |
Anna-Maria Behnsen | 14c0017 | 2022-04-11 17:19:03 +0200 | [diff] [blame] | 1538 | tracing_off(); |
Paul E. McKenney | e8516c6 | 2019-04-09 11:06:32 -0700 | [diff] [blame] | 1539 | WARN(1, "%s: rtort_pipe_count: %d\n", __func__, rcu_tortures[i].rtort_pipe_count); |
Anna-Maria Behnsen | 14c0017 | 2022-04-11 17:19:03 +0200 | [diff] [blame] | 1540 | rcu_ftrace_dump(DUMP_ALL); |
Paul E. McKenney | 34aa34b | 2019-05-16 16:15:16 -0700 | [diff] [blame] | 1541 | } |
Paul E. McKenney | ab1b788 | 2020-09-22 16:42:42 -0700 | [diff] [blame] | 1542 | if (stutter_waited) |
| 1543 | sched_set_normal(current, oldnice); |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 1544 | } while (!torture_must_stop()); |
Paul E. McKenney | cae7cc6 | 2020-04-26 19:20:37 -0700 | [diff] [blame] | 1545 | rcu_torture_current = NULL; // Let stats task know that we are done. |
Paul E. McKenney | 4bb3c5f | 2015-02-18 16:31:29 -0800 | [diff] [blame] | 1546 | /* Reset expediting back to unexpedited. */ |
| 1547 | if (expediting > 0) |
| 1548 | expediting = -expediting; |
| 1549 | while (can_expedite && expediting++ < 0) |
| 1550 | rcu_unexpedite_gp(); |
| 1551 | WARN_ON_ONCE(can_expedite && rcu_gp_is_expedited()); |
Paul E. McKenney | f7c0e6a | 2017-12-08 11:37:24 -0800 | [diff] [blame] | 1552 | if (!can_expedite) |
| 1553 | pr_alert("%s" TORTURE_FLAG |
| 1554 | " Dynamic grace-period expediting was disabled.\n", |
| 1555 | torture_type); |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 1556 | rcu_torture_writer_state = RTWS_STOPPING; |
Paul E. McKenney | 7fafaac | 2014-01-31 17:37:28 -0800 | [diff] [blame] | 1557 | torture_kthread_stopping("rcu_torture_writer"); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 1558 | return 0; |
| 1559 | } |
| 1560 | |
| 1561 | /* |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 1562 | * RCU torture fake writer kthread. Repeatedly calls sync, with a random |
| 1563 | * delay between calls. |
| 1564 | */ |
| 1565 | static int |
| 1566 | rcu_torture_fakewriter(void *arg) |
| 1567 | { |
Paul E. McKenney | 682189a | 2020-11-16 17:10:39 -0800 | [diff] [blame] | 1568 | unsigned long gp_snap; |
Paul E. McKenney | 76ea364 | 2022-08-02 17:04:54 -0700 | [diff] [blame] | 1569 | struct rcu_gp_oldstate gp_snap_full; |
Paul E. McKenney | 51b1130 | 2014-01-27 11:49:39 -0800 | [diff] [blame] | 1570 | DEFINE_TORTURE_RANDOM(rand); |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 1571 | |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 1572 | VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); |
Linus Torvalds | 971eae7 | 2014-03-31 11:21:19 -0700 | [diff] [blame] | 1573 | set_user_nice(current, MAX_NICE); |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 1574 | |
David Vernet | 39b3cab | 2022-03-07 14:46:55 -0800 | [diff] [blame] | 1575 | if (WARN_ONCE(nsynctypes == 0, |
| 1576 | "%s: No update-side primitives.\n", __func__)) { |
| 1577 | /* |
| 1578 | * No updates primitives, so don't try updating. |
| 1579 | * The resulting test won't be testing much, hence the |
| 1580 | * above WARN_ONCE(). |
| 1581 | */ |
| 1582 | torture_kthread_stopping("rcu_torture_fakewriter"); |
| 1583 | return 0; |
| 1584 | } |
| 1585 | |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 1586 | do { |
Paul E. McKenney | 1eba0ef | 2020-11-17 14:12:24 -0800 | [diff] [blame] | 1587 | torture_hrtimeout_jiffies(torture_random(&rand) % 10, &rand); |
Paul E. McKenney | 72472a0 | 2012-05-29 17:50:51 -0700 | [diff] [blame] | 1588 | if (cur_ops->cb_barrier != NULL && |
Paul E. McKenney | 51b1130 | 2014-01-27 11:49:39 -0800 | [diff] [blame] | 1589 | torture_random(&rand) % (nfakewriters * 8) == 0) { |
Paul E. McKenney | 72472a0 | 2012-05-29 17:50:51 -0700 | [diff] [blame] | 1590 | cur_ops->cb_barrier(); |
Paul E. McKenney | 682189a | 2020-11-16 17:10:39 -0800 | [diff] [blame] | 1591 | } else { |
| 1592 | switch (synctype[torture_random(&rand) % nsynctypes]) { |
| 1593 | case RTWS_DEF_FREE: |
| 1594 | break; |
| 1595 | case RTWS_EXP_SYNC: |
Paul E. McKenney | 2ec1f2d9 | 2013-06-12 15:12:21 -0700 | [diff] [blame] | 1596 | cur_ops->exp_sync(); |
Paul E. McKenney | 682189a | 2020-11-16 17:10:39 -0800 | [diff] [blame] | 1597 | break; |
| 1598 | case RTWS_COND_GET: |
| 1599 | gp_snap = cur_ops->get_gp_state(); |
Paul E. McKenney | ea31fd9 | 2020-11-17 11:32:54 -0800 | [diff] [blame] | 1600 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); |
Paul E. McKenney | 682189a | 2020-11-16 17:10:39 -0800 | [diff] [blame] | 1601 | cur_ops->cond_sync(gp_snap); |
| 1602 | break; |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 1603 | case RTWS_COND_GET_EXP: |
| 1604 | gp_snap = cur_ops->get_gp_state_exp(); |
| 1605 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); |
| 1606 | cur_ops->cond_sync_exp(gp_snap); |
| 1607 | break; |
Paul E. McKenney | b6fe491 | 2022-08-04 13:46:05 -0700 | [diff] [blame] | 1608 | case RTWS_COND_GET_FULL: |
| 1609 | cur_ops->get_gp_state_full(&gp_snap_full); |
| 1610 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); |
| 1611 | cur_ops->cond_sync_full(&gp_snap_full); |
| 1612 | break; |
Paul E. McKenney | 8df13f0 | 2022-08-04 15:23:26 -0700 | [diff] [blame] | 1613 | case RTWS_COND_GET_EXP_FULL: |
| 1614 | cur_ops->get_gp_state_full(&gp_snap_full); |
| 1615 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand); |
| 1616 | cur_ops->cond_sync_exp_full(&gp_snap_full); |
| 1617 | break; |
Paul E. McKenney | 682189a | 2020-11-16 17:10:39 -0800 | [diff] [blame] | 1618 | case RTWS_POLL_GET: |
| 1619 | gp_snap = cur_ops->start_gp_poll(); |
| 1620 | while (!cur_ops->poll_gp_state(gp_snap)) { |
Paul E. McKenney | ea31fd9 | 2020-11-17 11:32:54 -0800 | [diff] [blame] | 1621 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, |
| 1622 | &rand); |
Paul E. McKenney | 682189a | 2020-11-16 17:10:39 -0800 | [diff] [blame] | 1623 | } |
| 1624 | break; |
Paul E. McKenney | 76ea364 | 2022-08-02 17:04:54 -0700 | [diff] [blame] | 1625 | case RTWS_POLL_GET_FULL: |
| 1626 | cur_ops->start_gp_poll_full(&gp_snap_full); |
| 1627 | while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { |
| 1628 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, |
| 1629 | &rand); |
| 1630 | } |
| 1631 | break; |
Paul E. McKenney | 11d62f0 | 2022-02-01 07:01:20 -0800 | [diff] [blame] | 1632 | case RTWS_POLL_GET_EXP: |
| 1633 | gp_snap = cur_ops->start_gp_poll_exp(); |
| 1634 | while (!cur_ops->poll_gp_state_exp(gp_snap)) { |
| 1635 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, |
| 1636 | &rand); |
| 1637 | } |
| 1638 | break; |
Paul E. McKenney | 6c502b1 | 2022-08-03 12:38:51 -0700 | [diff] [blame] | 1639 | case RTWS_POLL_GET_EXP_FULL: |
| 1640 | cur_ops->start_gp_poll_exp_full(&gp_snap_full); |
| 1641 | while (!cur_ops->poll_gp_state_full(&gp_snap_full)) { |
| 1642 | torture_hrtimeout_jiffies(torture_random(&rand) % 16, |
| 1643 | &rand); |
| 1644 | } |
| 1645 | break; |
Paul E. McKenney | 682189a | 2020-11-16 17:10:39 -0800 | [diff] [blame] | 1646 | case RTWS_SYNC: |
| 1647 | cur_ops->sync(); |
| 1648 | break; |
| 1649 | default: |
| 1650 | WARN_ON_ONCE(1); |
| 1651 | break; |
| 1652 | } |
Paul E. McKenney | 2ec1f2d9 | 2013-06-12 15:12:21 -0700 | [diff] [blame] | 1653 | } |
Paul E. McKenney | 628edaa | 2014-01-31 11:57:43 -0800 | [diff] [blame] | 1654 | stutter_wait("rcu_torture_fakewriter"); |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 1655 | } while (!torture_must_stop()); |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 1656 | |
Paul E. McKenney | 7fafaac | 2014-01-31 17:37:28 -0800 | [diff] [blame] | 1657 | torture_kthread_stopping("rcu_torture_fakewriter"); |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 1658 | return 0; |
| 1659 | } |
| 1660 | |
Paul E. McKenney | f34c8585 | 2017-07-20 15:27:32 -0700 | [diff] [blame] | 1661 | static void rcu_torture_timer_cb(struct rcu_head *rhp) |
| 1662 | { |
| 1663 | kfree(rhp); |
| 1664 | } |
| 1665 | |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 1666 | // Set up and carry out testing of RCU's global memory ordering |
| 1667 | static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, |
| 1668 | struct torture_random_state *trsp) |
| 1669 | { |
| 1670 | unsigned long loops; |
Paul E. McKenney | 1afb95f | 2020-12-19 07:34:35 -0800 | [diff] [blame] | 1671 | int noc = torture_num_online_cpus(); |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 1672 | int rdrchked; |
| 1673 | int rdrchker; |
| 1674 | struct rcu_torture_reader_check *rtrcp; // Me. |
| 1675 | struct rcu_torture_reader_check *rtrcp_assigner; // Assigned us to do checking. |
| 1676 | struct rcu_torture_reader_check *rtrcp_chked; // Reader being checked. |
| 1677 | struct rcu_torture_reader_check *rtrcp_chker; // Reader doing checking when not me. |
| 1678 | |
| 1679 | if (myid < 0) |
| 1680 | return; // Don't try this from timer handlers. |
| 1681 | |
| 1682 | // Increment my counter. |
| 1683 | rtrcp = &rcu_torture_reader_mbchk[myid]; |
| 1684 | WRITE_ONCE(rtrcp->rtc_myloops, rtrcp->rtc_myloops + 1); |
| 1685 | |
| 1686 | // Attempt to assign someone else some checking work. |
| 1687 | rdrchked = torture_random(trsp) % nrealreaders; |
| 1688 | rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; |
| 1689 | rdrchker = torture_random(trsp) % nrealreaders; |
| 1690 | rtrcp_chker = &rcu_torture_reader_mbchk[rdrchker]; |
| 1691 | if (rdrchked != myid && rdrchked != rdrchker && noc >= rdrchked && noc >= rdrchker && |
| 1692 | smp_load_acquire(&rtrcp->rtc_chkrdr) < 0 && // Pairs with smp_store_release below. |
| 1693 | !READ_ONCE(rtp->rtort_chkp) && |
| 1694 | !smp_load_acquire(&rtrcp_chker->rtc_assigner)) { // Pairs with smp_store_release below. |
| 1695 | rtrcp->rtc_chkloops = READ_ONCE(rtrcp_chked->rtc_myloops); |
| 1696 | WARN_ON_ONCE(rtrcp->rtc_chkrdr >= 0); |
| 1697 | rtrcp->rtc_chkrdr = rdrchked; |
| 1698 | WARN_ON_ONCE(rtrcp->rtc_ready); // This gets set after the grace period ends. |
| 1699 | if (cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, NULL, rtrcp) || |
| 1700 | cmpxchg_relaxed(&rtp->rtort_chkp, NULL, rtrcp)) |
| 1701 | (void)cmpxchg_relaxed(&rtrcp_chker->rtc_assigner, rtrcp, NULL); // Back out. |
| 1702 | } |
| 1703 | |
| 1704 | // If assigned some completed work, do it! |
| 1705 | rtrcp_assigner = READ_ONCE(rtrcp->rtc_assigner); |
| 1706 | if (!rtrcp_assigner || !smp_load_acquire(&rtrcp_assigner->rtc_ready)) |
| 1707 | return; // No work or work not yet ready. |
| 1708 | rdrchked = rtrcp_assigner->rtc_chkrdr; |
| 1709 | if (WARN_ON_ONCE(rdrchked < 0)) |
| 1710 | return; |
| 1711 | rtrcp_chked = &rcu_torture_reader_mbchk[rdrchked]; |
| 1712 | loops = READ_ONCE(rtrcp_chked->rtc_myloops); |
| 1713 | atomic_inc(&n_rcu_torture_mbchk_tries); |
| 1714 | if (ULONG_CMP_LT(loops, rtrcp_assigner->rtc_chkloops)) |
| 1715 | atomic_inc(&n_rcu_torture_mbchk_fail); |
| 1716 | rtrcp_assigner->rtc_chkloops = loops + ULONG_MAX / 2; |
| 1717 | rtrcp_assigner->rtc_ready = 0; |
| 1718 | smp_store_release(&rtrcp->rtc_assigner, NULL); // Someone else can assign us work. |
| 1719 | smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign. |
| 1720 | } |
| 1721 | |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 1722 | /* |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1723 | * Do one extension of an RCU read-side critical section using the |
| 1724 | * current reader state in readstate (set to zero for initial entry |
| 1725 | * to extended critical section), set the new state as specified by |
| 1726 | * newstate (set to zero for final exit from extended critical section), |
| 1727 | * and random-number-generator state in trsp. If this is neither the |
| 1728 | * beginning or end of the critical section and if there was actually a |
| 1729 | * change, do a ->read_delay(). |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 1730 | */ |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1731 | static void rcutorture_one_extend(int *readstate, int newstate, |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1732 | struct torture_random_state *trsp, |
| 1733 | struct rt_read_seg *rtrsp) |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 1734 | { |
Paul E. McKenney | 52b1fc3 | 2020-03-28 18:53:25 -0700 | [diff] [blame] | 1735 | unsigned long flags; |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 1736 | int idxnew1 = -1; |
| 1737 | int idxnew2 = -1; |
| 1738 | int idxold1 = *readstate; |
| 1739 | int idxold2 = idxold1; |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1740 | int statesnew = ~*readstate & newstate; |
| 1741 | int statesold = *readstate & ~newstate; |
| 1742 | |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 1743 | WARN_ON_ONCE(idxold2 < 0); |
| 1744 | WARN_ON_ONCE((idxold2 >> RCUTORTURE_RDR_SHIFT_2) > 1); |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1745 | rtrsp->rt_readstate = newstate; |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1746 | |
| 1747 | /* First, put new protection in place to avoid critical-section gap. */ |
| 1748 | if (statesnew & RCUTORTURE_RDR_BH) |
| 1749 | local_bh_disable(); |
Scott Wood | 71921a9 | 2021-08-20 09:42:36 +0200 | [diff] [blame] | 1750 | if (statesnew & RCUTORTURE_RDR_RBH) |
| 1751 | rcu_read_lock_bh(); |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1752 | if (statesnew & RCUTORTURE_RDR_IRQ) |
| 1753 | local_irq_disable(); |
| 1754 | if (statesnew & RCUTORTURE_RDR_PREEMPT) |
| 1755 | preempt_disable(); |
Paul E. McKenney | 2ceebc0 | 2018-07-06 15:16:12 -0700 | [diff] [blame] | 1756 | if (statesnew & RCUTORTURE_RDR_SCHED) |
| 1757 | rcu_read_lock_sched(); |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 1758 | if (statesnew & RCUTORTURE_RDR_RCU_1) |
| 1759 | idxnew1 = (cur_ops->readlock() & 0x1) << RCUTORTURE_RDR_SHIFT_1; |
| 1760 | if (statesnew & RCUTORTURE_RDR_RCU_2) |
| 1761 | idxnew2 = (cur_ops->readlock() & 0x1) << RCUTORTURE_RDR_SHIFT_2; |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1762 | |
Scott Wood | 71921a9 | 2021-08-20 09:42:36 +0200 | [diff] [blame] | 1763 | /* |
| 1764 | * Next, remove old protection, in decreasing order of strength |
| 1765 | * to avoid unlock paths that aren't safe in the stronger |
| 1766 | * context. Namely: BH can not be enabled with disabled interrupts. |
| 1767 | * Additionally PREEMPT_RT requires that BH is enabled in preemptible |
| 1768 | * context. |
| 1769 | */ |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1770 | if (statesold & RCUTORTURE_RDR_IRQ) |
| 1771 | local_irq_enable(); |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1772 | if (statesold & RCUTORTURE_RDR_PREEMPT) |
| 1773 | preempt_enable(); |
Paul E. McKenney | 2ceebc0 | 2018-07-06 15:16:12 -0700 | [diff] [blame] | 1774 | if (statesold & RCUTORTURE_RDR_SCHED) |
| 1775 | rcu_read_unlock_sched(); |
Scott Wood | 71921a9 | 2021-08-20 09:42:36 +0200 | [diff] [blame] | 1776 | if (statesold & RCUTORTURE_RDR_BH) |
| 1777 | local_bh_enable(); |
| 1778 | if (statesold & RCUTORTURE_RDR_RBH) |
| 1779 | rcu_read_unlock_bh(); |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 1780 | if (statesold & RCUTORTURE_RDR_RCU_2) { |
| 1781 | cur_ops->readunlock((idxold2 >> RCUTORTURE_RDR_SHIFT_2) & 0x1); |
| 1782 | WARN_ON_ONCE(idxnew2 != -1); |
| 1783 | idxold2 = 0; |
| 1784 | } |
| 1785 | if (statesold & RCUTORTURE_RDR_RCU_1) { |
Paul E. McKenney | 340170f | 2021-09-24 21:30:26 -0700 | [diff] [blame] | 1786 | bool lockit; |
Paul E. McKenney | 52b1fc3 | 2020-03-28 18:53:25 -0700 | [diff] [blame] | 1787 | |
Paul E. McKenney | 340170f | 2021-09-24 21:30:26 -0700 | [diff] [blame] | 1788 | lockit = !cur_ops->no_pi_lock && !statesnew && !(torture_random(trsp) & 0xffff); |
Paul E. McKenney | 52b1fc3 | 2020-03-28 18:53:25 -0700 | [diff] [blame] | 1789 | if (lockit) |
| 1790 | raw_spin_lock_irqsave(¤t->pi_lock, flags); |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 1791 | cur_ops->readunlock((idxold1 >> RCUTORTURE_RDR_SHIFT_1) & 0x1); |
| 1792 | WARN_ON_ONCE(idxnew1 != -1); |
| 1793 | idxold1 = 0; |
Paul E. McKenney | 52b1fc3 | 2020-03-28 18:53:25 -0700 | [diff] [blame] | 1794 | if (lockit) |
| 1795 | raw_spin_unlock_irqrestore(¤t->pi_lock, flags); |
| 1796 | } |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1797 | |
| 1798 | /* Delay if neither beginning nor end and there was a change. */ |
| 1799 | if ((statesnew || statesold) && *readstate && newstate) |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1800 | cur_ops->read_delay(trsp, rtrsp); |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1801 | |
| 1802 | /* Update the reader state. */ |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 1803 | if (idxnew1 == -1) |
| 1804 | idxnew1 = idxold1 & RCUTORTURE_RDR_MASK_1; |
| 1805 | WARN_ON_ONCE(idxnew1 < 0); |
| 1806 | if (WARN_ON_ONCE((idxnew1 >> RCUTORTURE_RDR_SHIFT_1) > 1)) |
| 1807 | pr_info("Unexpected idxnew1 value of %#x\n", idxnew1); |
| 1808 | if (idxnew2 == -1) |
| 1809 | idxnew2 = idxold2 & RCUTORTURE_RDR_MASK_2; |
| 1810 | WARN_ON_ONCE(idxnew2 < 0); |
| 1811 | WARN_ON_ONCE((idxnew2 >> RCUTORTURE_RDR_SHIFT_2) > 1); |
| 1812 | *readstate = idxnew1 | idxnew2 | newstate; |
| 1813 | WARN_ON_ONCE(*readstate < 0); |
| 1814 | if (WARN_ON_ONCE((*readstate >> RCUTORTURE_RDR_SHIFT_2) > 1)) |
| 1815 | pr_info("Unexpected idxnew2 value of %#x\n", idxnew2); |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1816 | } |
| 1817 | |
| 1818 | /* Return the biggest extendables mask given current RCU and boot parameters. */ |
| 1819 | static int rcutorture_extend_mask_max(void) |
| 1820 | { |
| 1821 | int mask; |
| 1822 | |
| 1823 | WARN_ON_ONCE(extendables & ~RCUTORTURE_MAX_EXTEND); |
| 1824 | mask = extendables & RCUTORTURE_MAX_EXTEND & cur_ops->extendables; |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 1825 | mask = mask | RCUTORTURE_RDR_RCU_1 | RCUTORTURE_RDR_RCU_2; |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1826 | return mask; |
| 1827 | } |
| 1828 | |
| 1829 | /* Return a random protection state mask, but with at least one bit set. */ |
| 1830 | static int |
| 1831 | rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) |
| 1832 | { |
| 1833 | int mask = rcutorture_extend_mask_max(); |
Paul E. McKenney | bf1bef5 | 2018-06-10 08:50:09 -0700 | [diff] [blame] | 1834 | unsigned long randmask1 = torture_random(trsp) >> 8; |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1835 | unsigned long randmask2 = randmask1 >> 3; |
Scott Wood | 71921a9 | 2021-08-20 09:42:36 +0200 | [diff] [blame] | 1836 | unsigned long preempts = RCUTORTURE_RDR_PREEMPT | RCUTORTURE_RDR_SCHED; |
| 1837 | unsigned long preempts_irq = preempts | RCUTORTURE_RDR_IRQ; |
| 1838 | unsigned long bhs = RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH; |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1839 | |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 1840 | WARN_ON_ONCE(mask >> RCUTORTURE_RDR_SHIFT_1); |
Paul E. McKenney | a3b0e1e5 | 2019-02-28 15:06:13 -0800 | [diff] [blame] | 1841 | /* Mostly only one bit (need preemption!), sometimes lots of bits. */ |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1842 | if (!(randmask1 & 0x7)) |
Paul E. McKenney | bf1bef5 | 2018-06-10 08:50:09 -0700 | [diff] [blame] | 1843 | mask = mask & randmask2; |
| 1844 | else |
| 1845 | mask = mask & (1 << (randmask2 % RCUTORTURE_RDR_NBITS)); |
Scott Wood | 71921a9 | 2021-08-20 09:42:36 +0200 | [diff] [blame] | 1846 | |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 1847 | // Can't have nested RCU reader without outer RCU reader. |
| 1848 | if (!(mask & RCUTORTURE_RDR_RCU_1) && (mask & RCUTORTURE_RDR_RCU_2)) { |
| 1849 | if (oldmask & RCUTORTURE_RDR_RCU_1) |
| 1850 | mask &= ~RCUTORTURE_RDR_RCU_2; |
| 1851 | else |
| 1852 | mask |= RCUTORTURE_RDR_RCU_1; |
| 1853 | } |
| 1854 | |
Scott Wood | 71921a9 | 2021-08-20 09:42:36 +0200 | [diff] [blame] | 1855 | /* |
| 1856 | * Can't enable bh w/irq disabled. |
| 1857 | */ |
| 1858 | if (mask & RCUTORTURE_RDR_IRQ) |
| 1859 | mask |= oldmask & bhs; |
| 1860 | |
| 1861 | /* |
| 1862 | * Ideally these sequences would be detected in debug builds |
| 1863 | * (regardless of RT), but until then don't stop testing |
| 1864 | * them on non-RT. |
| 1865 | */ |
| 1866 | if (IS_ENABLED(CONFIG_PREEMPT_RT)) { |
| 1867 | /* Can't modify BH in atomic context */ |
| 1868 | if (oldmask & preempts_irq) |
| 1869 | mask &= ~bhs; |
| 1870 | if ((oldmask | mask) & preempts_irq) |
| 1871 | mask |= oldmask & bhs; |
| 1872 | } |
| 1873 | |
Paul E. McKenney | 1c3d539 | 2021-09-22 20:49:12 -0700 | [diff] [blame] | 1874 | return mask ?: RCUTORTURE_RDR_RCU_1; |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1875 | } |
| 1876 | |
| 1877 | /* |
| 1878 | * Do a randomly selected number of extensions of an existing RCU read-side |
| 1879 | * critical section. |
| 1880 | */ |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1881 | static struct rt_read_seg * |
| 1882 | rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, |
| 1883 | struct rt_read_seg *rtrsp) |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1884 | { |
| 1885 | int i; |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1886 | int j; |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1887 | int mask = rcutorture_extend_mask_max(); |
| 1888 | |
| 1889 | WARN_ON_ONCE(!*readstate); /* -Existing- RCU read-side critsect! */ |
| 1890 | if (!((mask - 1) & mask)) |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1891 | return rtrsp; /* Current RCU reader not extendable. */ |
| 1892 | /* Bias towards larger numbers of loops. */ |
| 1893 | i = (torture_random(trsp) >> 3); |
| 1894 | i = ((i | (i >> 3)) & RCUTORTURE_RDR_MAX_LOOPS) + 1; |
| 1895 | for (j = 0; j < i; j++) { |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1896 | mask = rcutorture_extend_mask(*readstate, trsp); |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1897 | rcutorture_one_extend(readstate, mask, trsp, &rtrsp[j]); |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1898 | } |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1899 | return &rtrsp[j]; |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1900 | } |
| 1901 | |
| 1902 | /* |
Paul E. McKenney | 6b06aa7 | 2018-05-22 10:56:05 -0700 | [diff] [blame] | 1903 | * Do one read-side critical section, returning false if there was |
| 1904 | * no data to read. Can be invoked both from process context and |
| 1905 | * from a timer handler. |
| 1906 | */ |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 1907 | static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) |
Paul E. McKenney | 6b06aa7 | 2018-05-22 10:56:05 -0700 | [diff] [blame] | 1908 | { |
Paul E. McKenney | 967c298 | 2022-08-24 18:57:45 -0700 | [diff] [blame] | 1909 | bool checkpolling = !(torture_random(trsp) & 0xfff); |
Paul E. McKenney | bc480a6 | 2020-11-15 12:45:57 -0800 | [diff] [blame] | 1910 | unsigned long cookie; |
Paul E. McKenney | f4754ad | 2022-08-03 16:40:48 -0700 | [diff] [blame] | 1911 | struct rcu_gp_oldstate cookie_full; |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1912 | int i; |
Paul E. McKenney | 917963d | 2014-11-21 17:10:16 -0800 | [diff] [blame] | 1913 | unsigned long started; |
Paul E. McKenney | 6b80da4 | 2014-11-21 14:19:26 -0800 | [diff] [blame] | 1914 | unsigned long completed; |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1915 | int newstate; |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 1916 | struct rcu_torture *p; |
| 1917 | int pipe_count; |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1918 | int readstate = 0; |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1919 | struct rt_read_seg rtseg[RCUTORTURE_RDR_MAX_SEGS] = { { 0 } }; |
| 1920 | struct rt_read_seg *rtrsp = &rtseg[0]; |
| 1921 | struct rt_read_seg *rtrsp1; |
Paul E. McKenney | 5249453 | 2012-11-14 16:26:40 -0800 | [diff] [blame] | 1922 | unsigned long long ts; |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 1923 | |
Paul E. McKenney | 7752275 | 2020-06-11 16:43:14 -0700 | [diff] [blame] | 1924 | WARN_ON_ONCE(!rcu_is_watching()); |
Paul E. McKenney | 2397d07 | 2018-05-25 07:29:25 -0700 | [diff] [blame] | 1925 | newstate = rcutorture_extend_mask(readstate, trsp); |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1926 | rcutorture_one_extend(&readstate, newstate, trsp, rtrsp++); |
Paul E. McKenney | 967c298 | 2022-08-24 18:57:45 -0700 | [diff] [blame] | 1927 | if (checkpolling) { |
| 1928 | if (cur_ops->get_gp_state && cur_ops->poll_gp_state) |
| 1929 | cookie = cur_ops->get_gp_state(); |
| 1930 | if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) |
| 1931 | cur_ops->get_gp_state_full(&cookie_full); |
| 1932 | } |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 1933 | started = cur_ops->get_gp_seq(); |
Steven Rostedt | e4aa0da | 2013-02-04 13:36:13 -0500 | [diff] [blame] | 1934 | ts = rcu_trace_clock_local(); |
Paul E. McKenney | 632ee20 | 2010-02-22 17:04:45 -0800 | [diff] [blame] | 1935 | p = rcu_dereference_check(rcu_torture_current, |
Paul E. McKenney | a5c095e | 2021-03-13 20:05:31 -0800 | [diff] [blame] | 1936 | !cur_ops->readlock_held || cur_ops->readlock_held()); |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 1937 | if (p == NULL) { |
Paul E. McKenney | 6b06aa7 | 2018-05-22 10:56:05 -0700 | [diff] [blame] | 1938 | /* Wait for rcu_torture_writer to get underway */ |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1939 | rcutorture_one_extend(&readstate, 0, trsp, rtrsp); |
Paul E. McKenney | 6b06aa7 | 2018-05-22 10:56:05 -0700 | [diff] [blame] | 1940 | return false; |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 1941 | } |
| 1942 | if (p->rtort_mbtest == 0) |
| 1943 | atomic_inc(&n_rcu_torture_mberror); |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 1944 | rcu_torture_reader_do_mbchk(myid, p, trsp); |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1945 | rtrsp = rcutorture_loop_extend(&readstate, trsp, rtrsp); |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 1946 | preempt_disable(); |
Paul E. McKenney | 2024891 | 2019-12-21 10:41:48 -0800 | [diff] [blame] | 1947 | pipe_count = READ_ONCE(p->rtort_pipe_count); |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 1948 | if (pipe_count > RCU_TORTURE_PIPE_LEN) { |
| 1949 | /* Should not happen, but... */ |
| 1950 | pipe_count = RCU_TORTURE_PIPE_LEN; |
| 1951 | } |
Paul E. McKenney | 17ef2fe | 2018-04-27 11:39:34 -0700 | [diff] [blame] | 1952 | completed = cur_ops->get_gp_seq(); |
Paul E. McKenney | 5249453 | 2012-11-14 16:26:40 -0800 | [diff] [blame] | 1953 | if (pipe_count > 1) { |
Paul E. McKenney | 6b06aa7 | 2018-05-22 10:56:05 -0700 | [diff] [blame] | 1954 | do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu, |
| 1955 | ts, started, completed); |
Paul E. McKenney | 274529b | 2016-03-21 19:46:04 -0700 | [diff] [blame] | 1956 | rcu_ftrace_dump(DUMP_ALL); |
Paul E. McKenney | 5249453 | 2012-11-14 16:26:40 -0800 | [diff] [blame] | 1957 | } |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 1958 | __this_cpu_inc(rcu_torture_count[pipe_count]); |
Paul E. McKenney | d7219312 | 2018-05-15 15:24:41 -0700 | [diff] [blame] | 1959 | completed = rcutorture_seq_diff(completed, started); |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 1960 | if (completed > RCU_TORTURE_PIPE_LEN) { |
| 1961 | /* Should not happen, but... */ |
| 1962 | completed = RCU_TORTURE_PIPE_LEN; |
| 1963 | } |
Rusty Russell | dd17c8f | 2009-10-29 22:34:15 +0900 | [diff] [blame] | 1964 | __this_cpu_inc(rcu_torture_batch[completed]); |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 1965 | preempt_enable(); |
Paul E. McKenney | 967c298 | 2022-08-24 18:57:45 -0700 | [diff] [blame] | 1966 | if (checkpolling) { |
| 1967 | if (cur_ops->get_gp_state && cur_ops->poll_gp_state) |
| 1968 | WARN_ONCE(cur_ops->poll_gp_state(cookie), |
| 1969 | "%s: Cookie check 2 failed %s(%d) %lu->%lu\n", |
| 1970 | __func__, |
| 1971 | rcu_torture_writer_state_getname(), |
| 1972 | rcu_torture_writer_state, |
| 1973 | cookie, cur_ops->get_gp_state()); |
| 1974 | if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) |
| 1975 | WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full), |
| 1976 | "%s: Cookie check 6 failed %s(%d) online %*pbl\n", |
| 1977 | __func__, |
| 1978 | rcu_torture_writer_state_getname(), |
| 1979 | rcu_torture_writer_state, |
| 1980 | cpumask_pr_args(cpu_online_mask)); |
| 1981 | } |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1982 | rcutorture_one_extend(&readstate, 0, trsp, rtrsp); |
Paul E. McKenney | 902d82e6 | 2021-09-22 20:31:44 -0700 | [diff] [blame] | 1983 | WARN_ON_ONCE(readstate); |
Paul E. McKenney | d685514 | 2020-08-11 10:33:39 -0700 | [diff] [blame] | 1984 | // This next splat is expected behavior if leakpointer, especially |
| 1985 | // for CONFIG_RCU_STRICT_GRACE_PERIOD=y kernels. |
| 1986 | WARN_ON_ONCE(leakpointer && READ_ONCE(p->rtort_pipe_count) > 1); |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 1987 | |
| 1988 | /* If error or close call, record the sequence of reader protections. */ |
| 1989 | if ((pipe_count > 1 || completed > 1) && !xchg(&err_segs_recorded, 1)) { |
| 1990 | i = 0; |
| 1991 | for (rtrsp1 = &rtseg[0]; rtrsp1 < rtrsp; rtrsp1++) |
| 1992 | err_segs[i++] = *rtrsp1; |
| 1993 | rt_read_nsegs = i; |
| 1994 | } |
| 1995 | |
Paul E. McKenney | 6b06aa7 | 2018-05-22 10:56:05 -0700 | [diff] [blame] | 1996 | return true; |
| 1997 | } |
| 1998 | |
Paul E. McKenney | 3025520e | 2018-05-22 11:38:47 -0700 | [diff] [blame] | 1999 | static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand); |
| 2000 | |
Paul E. McKenney | 6b06aa7 | 2018-05-22 10:56:05 -0700 | [diff] [blame] | 2001 | /* |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2002 | * RCU torture reader from timer handler. Dereferences rcu_torture_current, |
| 2003 | * incrementing the corresponding element of the pipeline array. The |
| 2004 | * counter in the element should never be greater than 1, otherwise, the |
| 2005 | * RCU implementation is broken. |
| 2006 | */ |
| 2007 | static void rcu_torture_timer(struct timer_list *unused) |
| 2008 | { |
Paul E. McKenney | 8da9a59 | 2018-05-22 11:17:51 -0700 | [diff] [blame] | 2009 | atomic_long_inc(&n_rcu_torture_timers); |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 2010 | (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1); |
Paul E. McKenney | f34c8585 | 2017-07-20 15:27:32 -0700 | [diff] [blame] | 2011 | |
| 2012 | /* Test call_rcu() invocation from interrupt handler. */ |
| 2013 | if (cur_ops->call) { |
| 2014 | struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_NOWAIT); |
| 2015 | |
| 2016 | if (rhp) |
| 2017 | cur_ops->call(rhp, rcu_torture_timer_cb); |
| 2018 | } |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 2019 | } |
| 2020 | |
| 2021 | /* |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2022 | * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, |
| 2023 | * incrementing the corresponding element of the pipeline array. The |
| 2024 | * counter in the element should never be greater than 1, otherwise, the |
| 2025 | * RCU implementation is broken. |
| 2026 | */ |
| 2027 | static int |
| 2028 | rcu_torture_reader(void *arg) |
| 2029 | { |
Paul E. McKenney | 444da51 | 2018-07-04 14:14:42 -0700 | [diff] [blame] | 2030 | unsigned long lastsleep = jiffies; |
Paul E. McKenney | c04dd09 | 2018-07-23 14:16:47 -0700 | [diff] [blame] | 2031 | long myid = (long)arg; |
| 2032 | int mynumonline = myid; |
Paul E. McKenney | 51b1130 | 2014-01-27 11:49:39 -0800 | [diff] [blame] | 2033 | DEFINE_TORTURE_RANDOM(rand); |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 2034 | struct timer_list t; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2035 | |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 2036 | VERBOSE_TOROUT_STRING("rcu_torture_reader task started"); |
Linus Torvalds | 971eae7 | 2014-03-31 11:21:19 -0700 | [diff] [blame] | 2037 | set_user_nice(current, MAX_NICE); |
Paul E. McKenney | 0acc512 | 2009-06-25 09:08:17 -0700 | [diff] [blame] | 2038 | if (irqreader && cur_ops->irq_capable) |
Kees Cook | fd30b71 | 2017-10-22 17:58:54 -0700 | [diff] [blame] | 2039 | timer_setup_on_stack(&t, rcu_torture_timer, 0); |
Paul E. McKenney | d38e6dc | 2019-07-28 12:00:48 -0700 | [diff] [blame] | 2040 | tick_dep_set_task(current, TICK_DEP_BIT_RCU); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2041 | do { |
Paul E. McKenney | 0acc512 | 2009-06-25 09:08:17 -0700 | [diff] [blame] | 2042 | if (irqreader && cur_ops->irq_capable) { |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 2043 | if (!timer_pending(&t)) |
Paul E. McKenney | 6155fec | 2010-02-22 17:05:04 -0800 | [diff] [blame] | 2044 | mod_timer(&t, jiffies + 1); |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 2045 | } |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 2046 | if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop()) |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2047 | schedule_timeout_interruptible(HZ); |
Paul E. McKenney | d38e6dc | 2019-07-28 12:00:48 -0700 | [diff] [blame] | 2048 | if (time_after(jiffies, lastsleep) && !torture_must_stop()) { |
Paul E. McKenney | 1eba0ef | 2020-11-17 14:12:24 -0800 | [diff] [blame] | 2049 | torture_hrtimeout_us(500, 1000, &rand); |
Paul E. McKenney | 444da51 | 2018-07-04 14:14:42 -0700 | [diff] [blame] | 2050 | lastsleep = jiffies + 10; |
| 2051 | } |
Paul E. McKenney | 1afb95f | 2020-12-19 07:34:35 -0800 | [diff] [blame] | 2052 | while (torture_num_online_cpus() < mynumonline && !torture_must_stop()) |
Paul E. McKenney | c04dd09 | 2018-07-23 14:16:47 -0700 | [diff] [blame] | 2053 | schedule_timeout_interruptible(HZ / 5); |
Paul E. McKenney | 628edaa | 2014-01-31 11:57:43 -0800 | [diff] [blame] | 2054 | stutter_wait("rcu_torture_reader"); |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 2055 | } while (!torture_must_stop()); |
Thomas Gleixner | 424c1b6 | 2014-03-23 08:58:27 -0700 | [diff] [blame] | 2056 | if (irqreader && cur_ops->irq_capable) { |
Paul E. McKenney | 0729fbf | 2008-06-25 12:24:52 -0700 | [diff] [blame] | 2057 | del_timer_sync(&t); |
Thomas Gleixner | 424c1b6 | 2014-03-23 08:58:27 -0700 | [diff] [blame] | 2058 | destroy_timer_on_stack(&t); |
| 2059 | } |
Paul E. McKenney | d38e6dc | 2019-07-28 12:00:48 -0700 | [diff] [blame] | 2060 | tick_dep_clear_task(current, TICK_DEP_BIT_RCU); |
Paul E. McKenney | 7fafaac | 2014-01-31 17:37:28 -0800 | [diff] [blame] | 2061 | torture_kthread_stopping("rcu_torture_reader"); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2062 | return 0; |
| 2063 | } |
| 2064 | |
| 2065 | /* |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 2066 | * Randomly Toggle CPUs' callback-offload state. This uses hrtimers to |
| 2067 | * increase race probabilities and fuzzes the interval between toggling. |
| 2068 | */ |
| 2069 | static int rcu_nocb_toggle(void *arg) |
| 2070 | { |
| 2071 | int cpu; |
| 2072 | int maxcpu = -1; |
| 2073 | int oldnice = task_nice(current); |
| 2074 | long r; |
| 2075 | DEFINE_TORTURE_RANDOM(rand); |
| 2076 | ktime_t toggle_delay; |
| 2077 | unsigned long toggle_fuzz; |
| 2078 | ktime_t toggle_interval = ms_to_ktime(nocbs_toggle); |
| 2079 | |
| 2080 | VERBOSE_TOROUT_STRING("rcu_nocb_toggle task started"); |
| 2081 | while (!rcu_inkernel_boot_has_ended()) |
| 2082 | schedule_timeout_interruptible(HZ / 10); |
| 2083 | for_each_online_cpu(cpu) |
| 2084 | maxcpu = cpu; |
| 2085 | WARN_ON(maxcpu < 0); |
| 2086 | if (toggle_interval > ULONG_MAX) |
| 2087 | toggle_fuzz = ULONG_MAX >> 3; |
| 2088 | else |
| 2089 | toggle_fuzz = toggle_interval >> 3; |
| 2090 | if (toggle_fuzz <= 0) |
| 2091 | toggle_fuzz = NSEC_PER_USEC; |
| 2092 | do { |
| 2093 | r = torture_random(&rand); |
| 2094 | cpu = (r >> 4) % (maxcpu + 1); |
| 2095 | if (r & 0x1) { |
| 2096 | rcu_nocb_cpu_offload(cpu); |
| 2097 | atomic_long_inc(&n_nocb_offload); |
| 2098 | } else { |
| 2099 | rcu_nocb_cpu_deoffload(cpu); |
| 2100 | atomic_long_inc(&n_nocb_deoffload); |
| 2101 | } |
| 2102 | toggle_delay = torture_random(&rand) % toggle_fuzz + toggle_interval; |
| 2103 | set_current_state(TASK_INTERRUPTIBLE); |
| 2104 | schedule_hrtimeout(&toggle_delay, HRTIMER_MODE_REL); |
| 2105 | if (stutter_wait("rcu_nocb_toggle")) |
| 2106 | sched_set_normal(current, oldnice); |
| 2107 | } while (!torture_must_stop()); |
| 2108 | torture_kthread_stopping("rcu_nocb_toggle"); |
| 2109 | return 0; |
| 2110 | } |
| 2111 | |
| 2112 | /* |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2113 | * Print torture statistics. Caller must ensure that there is only |
| 2114 | * one call to this function at a given time!!! This is normally |
| 2115 | * accomplished by relying on the module system to only have one copy |
| 2116 | * of the module loaded, and then by giving the rcu_torture_stats |
| 2117 | * kthread full control (or the init/cleanup functions when rcu_torture_stats |
| 2118 | * thread is not running). |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2119 | */ |
Chen Gang | d100895 | 2013-11-07 10:30:25 +0800 | [diff] [blame] | 2120 | static void |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2121 | rcu_torture_stats_print(void) |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2122 | { |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2123 | int cpu; |
| 2124 | int i; |
| 2125 | long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; |
| 2126 | long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 }; |
Paul E. McKenney | 5396d31 | 2020-01-08 19:58:13 -0800 | [diff] [blame] | 2127 | struct rcu_torture *rtcp; |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 2128 | static unsigned long rtcv_snap = ULONG_MAX; |
Paul E. McKenney | 0032f4e | 2017-08-30 10:40:17 -0700 | [diff] [blame] | 2129 | static bool splatted; |
Paul E. McKenney | 4ffa669 | 2016-06-30 11:56:38 -0700 | [diff] [blame] | 2130 | struct task_struct *wtp; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2131 | |
KAMEZAWA Hiroyuki | 0a94502 | 2006-03-28 01:56:37 -0800 | [diff] [blame] | 2132 | for_each_possible_cpu(cpu) { |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2133 | for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { |
Paul E. McKenney | f042a43 | 2020-01-03 16:27:00 -0800 | [diff] [blame] | 2134 | pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]); |
| 2135 | batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2136 | } |
| 2137 | } |
Paul E. McKenney | 8c0666d | 2022-05-20 13:18:16 -0700 | [diff] [blame] | 2138 | for (i = RCU_TORTURE_PIPE_LEN; i >= 0; i--) { |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2139 | if (pipesummary[i] != 0) |
| 2140 | break; |
| 2141 | } |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2142 | |
| 2143 | pr_alert("%s%s ", torture_type, TORTURE_FLAG); |
Paul E. McKenney | 5396d31 | 2020-01-08 19:58:13 -0800 | [diff] [blame] | 2144 | rtcp = rcu_access_pointer(rcu_torture_current); |
Paul E. McKenney | 354ea05 | 2019-05-25 12:36:53 -0700 | [diff] [blame] | 2145 | pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ", |
Paul E. McKenney | 5396d31 | 2020-01-08 19:58:13 -0800 | [diff] [blame] | 2146 | rtcp, |
| 2147 | rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER", |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2148 | rcu_torture_current_version, |
| 2149 | list_empty(&rcu_torture_freelist), |
| 2150 | atomic_read(&n_rcu_torture_alloc), |
| 2151 | atomic_read(&n_rcu_torture_alloc_fail), |
| 2152 | atomic_read(&n_rcu_torture_free)); |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 2153 | pr_cont("rtmbe: %d rtmbkf: %d/%d rtbe: %ld rtbke: %ld rtbre: %ld ", |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2154 | atomic_read(&n_rcu_torture_mberror), |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 2155 | atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries), |
SeongJae Park | 472213a | 2016-08-13 15:54:35 +0900 | [diff] [blame] | 2156 | n_rcu_torture_barrier_error, |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2157 | n_rcu_torture_boost_ktrerror, |
| 2158 | n_rcu_torture_boost_rterror); |
| 2159 | pr_cont("rtbf: %ld rtb: %ld nt: %ld ", |
| 2160 | n_rcu_torture_boost_failure, |
| 2161 | n_rcu_torture_boosts, |
Paul E. McKenney | 8da9a59 | 2018-05-22 11:17:51 -0700 | [diff] [blame] | 2162 | atomic_long_read(&n_rcu_torture_timers)); |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2163 | torture_onoff_stats(); |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 2164 | pr_cont("barrier: %ld/%ld:%ld ", |
Paul E. McKenney | c9527be | 2020-02-18 13:41:02 -0800 | [diff] [blame] | 2165 | data_race(n_barrier_successes), |
| 2166 | data_race(n_barrier_attempts), |
| 2167 | data_race(n_rcu_torture_barrier_error)); |
Paul E. McKenney | f759081 | 2020-12-21 11:17:16 -0800 | [diff] [blame] | 2168 | pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic. |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 2169 | pr_cont("nocb-toggles: %ld:%ld\n", |
| 2170 | atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload)); |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2171 | |
| 2172 | pr_alert("%s%s ", torture_type, TORTURE_FLAG); |
Paul E. McKenney | 8b5ddf8 | 2019-08-14 12:02:40 -0700 | [diff] [blame] | 2173 | if (atomic_read(&n_rcu_torture_mberror) || |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 2174 | atomic_read(&n_rcu_torture_mbchk_fail) || |
Paul E. McKenney | 8b5ddf8 | 2019-08-14 12:02:40 -0700 | [diff] [blame] | 2175 | n_rcu_torture_barrier_error || n_rcu_torture_boost_ktrerror || |
| 2176 | n_rcu_torture_boost_rterror || n_rcu_torture_boost_failure || |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 2177 | i > 1) { |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2178 | pr_cont("%s", "!!! "); |
Paul E. McKenney | 996417d | 2005-11-18 01:10:50 -0800 | [diff] [blame] | 2179 | atomic_inc(&n_rcu_torture_error); |
Paul E. McKenney | 8b5ddf8 | 2019-08-14 12:02:40 -0700 | [diff] [blame] | 2180 | WARN_ON_ONCE(atomic_read(&n_rcu_torture_mberror)); |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 2181 | WARN_ON_ONCE(atomic_read(&n_rcu_torture_mbchk_fail)); |
Paul E. McKenney | 8b5ddf8 | 2019-08-14 12:02:40 -0700 | [diff] [blame] | 2182 | WARN_ON_ONCE(n_rcu_torture_barrier_error); // rcu_barrier() |
| 2183 | WARN_ON_ONCE(n_rcu_torture_boost_ktrerror); // no boost kthread |
| 2184 | WARN_ON_ONCE(n_rcu_torture_boost_rterror); // can't set RT prio |
Paul E. McKenney | 5e59fba | 2021-01-15 13:30:38 -0800 | [diff] [blame] | 2185 | WARN_ON_ONCE(n_rcu_torture_boost_failure); // boost failed (TIMER_SOFTIRQ RT prio?) |
Paul E. McKenney | 8b5ddf8 | 2019-08-14 12:02:40 -0700 | [diff] [blame] | 2186 | WARN_ON_ONCE(i > 1); // Too-short grace period |
Paul E. McKenney | 996417d | 2005-11-18 01:10:50 -0800 | [diff] [blame] | 2187 | } |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2188 | pr_cont("Reader Pipe: "); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2189 | for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2190 | pr_cont(" %ld", pipesummary[i]); |
| 2191 | pr_cont("\n"); |
| 2192 | |
| 2193 | pr_alert("%s%s ", torture_type, TORTURE_FLAG); |
| 2194 | pr_cont("Reader Batch: "); |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 2195 | for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2196 | pr_cont(" %ld", batchsummary[i]); |
| 2197 | pr_cont("\n"); |
| 2198 | |
| 2199 | pr_alert("%s%s ", torture_type, TORTURE_FLAG); |
| 2200 | pr_cont("Free-Block Circulation: "); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2201 | for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2202 | pr_cont(" %d", atomic_read(&rcu_torture_wcount[i])); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2203 | } |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2204 | pr_cont("\n"); |
| 2205 | |
Josh Triplett | c8e5b16 | 2007-05-08 00:33:20 -0700 | [diff] [blame] | 2206 | if (cur_ops->stats) |
Joe Perches | eea203f | 2014-07-14 09:16:15 -0400 | [diff] [blame] | 2207 | cur_ops->stats(); |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 2208 | if (rtcv_snap == rcu_torture_current_version && |
Paul E. McKenney | 5396d31 | 2020-01-08 19:58:13 -0800 | [diff] [blame] | 2209 | rcu_access_pointer(rcu_torture_current) && |
| 2210 | !rcu_stall_is_suppressed()) { |
Paul E. McKenney | 7f6733c | 2017-04-18 17:17:35 -0700 | [diff] [blame] | 2211 | int __maybe_unused flags = 0; |
Paul E. McKenney | aebc826 | 2018-05-01 06:42:51 -0700 | [diff] [blame] | 2212 | unsigned long __maybe_unused gp_seq = 0; |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 2213 | |
| 2214 | rcutorture_get_gp_data(cur_ops->ttype, |
Paul E. McKenney | aebc826 | 2018-05-01 06:42:51 -0700 | [diff] [blame] | 2215 | &flags, &gp_seq); |
Paul E. McKenney | 7f6733c | 2017-04-18 17:17:35 -0700 | [diff] [blame] | 2216 | srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, |
Paul E. McKenney | aebc826 | 2018-05-01 06:42:51 -0700 | [diff] [blame] | 2217 | &flags, &gp_seq); |
Paul E. McKenney | 4ffa669 | 2016-06-30 11:56:38 -0700 | [diff] [blame] | 2218 | wtp = READ_ONCE(writer_task); |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 2219 | pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n", |
Paul E. McKenney | 18aff33 | 2015-11-17 13:35:28 -0800 | [diff] [blame] | 2220 | rcu_torture_writer_state_getname(), |
Paul E. McKenney | aebc826 | 2018-05-01 06:42:51 -0700 | [diff] [blame] | 2221 | rcu_torture_writer_state, gp_seq, flags, |
Peter Zijlstra | 2f064a5 | 2021-06-11 10:28:17 +0200 | [diff] [blame] | 2222 | wtp == NULL ? ~0U : wtp->__state, |
Paul E. McKenney | 808de39 | 2017-06-19 10:03:22 -0700 | [diff] [blame] | 2223 | wtp == NULL ? -1 : (int)task_cpu(wtp)); |
Paul E. McKenney | 0032f4e | 2017-08-30 10:40:17 -0700 | [diff] [blame] | 2224 | if (!splatted && wtp) { |
| 2225 | sched_show_task(wtp); |
| 2226 | splatted = true; |
| 2227 | } |
Paul E. McKenney | 27c0f14 | 2020-09-15 17:08:03 -0700 | [diff] [blame] | 2228 | if (cur_ops->gp_kthread_dbg) |
| 2229 | cur_ops->gp_kthread_dbg(); |
Paul E. McKenney | 274529b | 2016-03-21 19:46:04 -0700 | [diff] [blame] | 2230 | rcu_ftrace_dump(DUMP_ALL); |
Paul E. McKenney | ad0dc7f | 2014-02-19 10:51:42 -0800 | [diff] [blame] | 2231 | } |
| 2232 | rtcv_snap = rcu_torture_current_version; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2233 | } |
| 2234 | |
| 2235 | /* |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2236 | * Periodically prints torture statistics, if periodic statistics printing |
| 2237 | * was specified via the stat_interval module parameter. |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2238 | */ |
| 2239 | static int |
| 2240 | rcu_torture_stats(void *arg) |
| 2241 | { |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 2242 | VERBOSE_TOROUT_STRING("rcu_torture_stats task started"); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2243 | do { |
| 2244 | schedule_timeout_interruptible(stat_interval * HZ); |
| 2245 | rcu_torture_stats_print(); |
Paul E. McKenney | f67a335 | 2014-01-29 07:40:27 -0800 | [diff] [blame] | 2246 | torture_shutdown_absorb("rcu_torture_stats"); |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 2247 | } while (!torture_must_stop()); |
Paul E. McKenney | 7fafaac | 2014-01-31 17:37:28 -0800 | [diff] [blame] | 2248 | torture_kthread_stopping("rcu_torture_stats"); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 2249 | return 0; |
| 2250 | } |
| 2251 | |
Paul E. McKenney | 7ab2bd3 | 2021-05-02 19:56:05 -0700 | [diff] [blame] | 2252 | /* Test mem_dump_obj() and friends. */ |
| 2253 | static void rcu_torture_mem_dump_obj(void) |
| 2254 | { |
| 2255 | struct rcu_head *rhp; |
| 2256 | struct kmem_cache *kcp; |
| 2257 | static int z; |
| 2258 | |
| 2259 | kcp = kmem_cache_create("rcuscale", 136, 8, SLAB_STORE_USER, NULL); |
Li Qiong | 1a5ca5e | 2022-06-12 14:48:25 +0800 | [diff] [blame] | 2260 | if (WARN_ON_ONCE(!kcp)) |
| 2261 | return; |
Paul E. McKenney | 7ab2bd3 | 2021-05-02 19:56:05 -0700 | [diff] [blame] | 2262 | rhp = kmem_cache_alloc(kcp, GFP_KERNEL); |
Li Qiong | 1a5ca5e | 2022-06-12 14:48:25 +0800 | [diff] [blame] | 2263 | if (WARN_ON_ONCE(!rhp)) { |
| 2264 | kmem_cache_destroy(kcp); |
| 2265 | return; |
| 2266 | } |
Paul E. McKenney | 7ab2bd3 | 2021-05-02 19:56:05 -0700 | [diff] [blame] | 2267 | pr_alert("mem_dump_obj() slab test: rcu_torture_stats = %px, &rhp = %px, rhp = %px, &z = %px\n", stats_task, &rhp, rhp, &z); |
| 2268 | pr_alert("mem_dump_obj(ZERO_SIZE_PTR):"); |
| 2269 | mem_dump_obj(ZERO_SIZE_PTR); |
| 2270 | pr_alert("mem_dump_obj(NULL):"); |
| 2271 | mem_dump_obj(NULL); |
| 2272 | pr_alert("mem_dump_obj(%px):", &rhp); |
| 2273 | mem_dump_obj(&rhp); |
| 2274 | pr_alert("mem_dump_obj(%px):", rhp); |
| 2275 | mem_dump_obj(rhp); |
| 2276 | pr_alert("mem_dump_obj(%px):", &rhp->func); |
| 2277 | mem_dump_obj(&rhp->func); |
| 2278 | pr_alert("mem_dump_obj(%px):", &z); |
| 2279 | mem_dump_obj(&z); |
| 2280 | kmem_cache_free(kcp, rhp); |
| 2281 | kmem_cache_destroy(kcp); |
| 2282 | rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); |
Li Qiong | 1a5ca5e | 2022-06-12 14:48:25 +0800 | [diff] [blame] | 2283 | if (WARN_ON_ONCE(!rhp)) |
| 2284 | return; |
Paul E. McKenney | 7ab2bd3 | 2021-05-02 19:56:05 -0700 | [diff] [blame] | 2285 | pr_alert("mem_dump_obj() kmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); |
| 2286 | pr_alert("mem_dump_obj(kmalloc %px):", rhp); |
| 2287 | mem_dump_obj(rhp); |
| 2288 | pr_alert("mem_dump_obj(kmalloc %px):", &rhp->func); |
| 2289 | mem_dump_obj(&rhp->func); |
| 2290 | kfree(rhp); |
| 2291 | rhp = vmalloc(4096); |
Li Qiong | 1a5ca5e | 2022-06-12 14:48:25 +0800 | [diff] [blame] | 2292 | if (WARN_ON_ONCE(!rhp)) |
| 2293 | return; |
Paul E. McKenney | 7ab2bd3 | 2021-05-02 19:56:05 -0700 | [diff] [blame] | 2294 | pr_alert("mem_dump_obj() vmalloc test: rcu_torture_stats = %px, &rhp = %px, rhp = %px\n", stats_task, &rhp, rhp); |
| 2295 | pr_alert("mem_dump_obj(vmalloc %px):", rhp); |
| 2296 | mem_dump_obj(rhp); |
| 2297 | pr_alert("mem_dump_obj(vmalloc %px):", &rhp->func); |
| 2298 | mem_dump_obj(&rhp->func); |
| 2299 | vfree(rhp); |
| 2300 | } |
| 2301 | |
Paul E. McKenney | eac45e5 | 2018-05-17 11:33:17 -0700 | [diff] [blame] | 2302 | static void |
Steven Rostedt (Red Hat) | e66c33d | 2013-07-12 16:50:28 -0400 | [diff] [blame] | 2303 | rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) |
Paul E. McKenney | 95c3832 | 2006-03-24 03:15:58 -0800 | [diff] [blame] | 2304 | { |
Paul E. McKenney | 2caa1e4 | 2012-08-09 16:30:45 -0700 | [diff] [blame] | 2305 | pr_alert("%s" TORTURE_FLAG |
| 2306 | "--- %s: nreaders=%d nfakewriters=%d " |
| 2307 | "stat_interval=%d verbose=%d test_no_idle_hz=%d " |
| 2308 | "shuffle_interval=%d stutter=%d irqreader=%d " |
| 2309 | "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " |
| 2310 | "test_boost=%d/%d test_boost_interval=%d " |
| 2311 | "test_boost_duration=%d shutdown_secs=%d " |
Paul E. McKenney | 2b1516e | 2017-08-18 16:11:37 -0700 | [diff] [blame] | 2312 | "stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d " |
Paul E. McKenney | 19a8ff9 | 2020-03-11 17:39:12 -0700 | [diff] [blame] | 2313 | "stall_cpu_block=%d " |
Paul E. McKenney | 67afeed | 2012-10-20 12:56:06 -0700 | [diff] [blame] | 2314 | "n_barrier_cbs=%d " |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 2315 | "onoff_interval=%d onoff_holdoff=%d " |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 2316 | "read_exit_delay=%d read_exit_burst=%d " |
| 2317 | "nocbs_nthreads=%d nocbs_toggle=%d\n", |
Paul E. McKenney | 2caa1e4 | 2012-08-09 16:30:45 -0700 | [diff] [blame] | 2318 | torture_type, tag, nrealreaders, nfakewriters, |
| 2319 | stat_interval, verbose, test_no_idle_hz, shuffle_interval, |
| 2320 | stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, |
| 2321 | test_boost, cur_ops->can_boost, |
| 2322 | test_boost_interval, test_boost_duration, shutdown_secs, |
Paul E. McKenney | 2b1516e | 2017-08-18 16:11:37 -0700 | [diff] [blame] | 2323 | stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff, |
Paul E. McKenney | 19a8ff9 | 2020-03-11 17:39:12 -0700 | [diff] [blame] | 2324 | stall_cpu_block, |
Paul E. McKenney | 67afeed | 2012-10-20 12:56:06 -0700 | [diff] [blame] | 2325 | n_barrier_cbs, |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 2326 | onoff_interval, onoff_holdoff, |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 2327 | read_exit_delay, read_exit_burst, |
| 2328 | nocbs_nthreads, nocbs_toggle); |
Paul E. McKenney | 95c3832 | 2006-03-24 03:15:58 -0800 | [diff] [blame] | 2329 | } |
| 2330 | |
Sebastian Andrzej Siewior | 0ffd374 | 2016-08-18 14:57:22 +0200 | [diff] [blame] | 2331 | static int rcutorture_booster_cleanup(unsigned int cpu) |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 2332 | { |
| 2333 | struct task_struct *t; |
| 2334 | |
| 2335 | if (boost_tasks[cpu] == NULL) |
Sebastian Andrzej Siewior | 0ffd374 | 2016-08-18 14:57:22 +0200 | [diff] [blame] | 2336 | return 0; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 2337 | mutex_lock(&boost_mutex); |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 2338 | t = boost_tasks[cpu]; |
| 2339 | boost_tasks[cpu] = NULL; |
Joel Fernandes (Google) | 450efca | 2018-06-10 16:45:43 -0700 | [diff] [blame] | 2340 | rcu_torture_enable_rt_throttle(); |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 2341 | mutex_unlock(&boost_mutex); |
| 2342 | |
| 2343 | /* This must be outside of the mutex, otherwise deadlock! */ |
Paul E. McKenney | 9c029b8 | 2014-02-04 11:47:08 -0800 | [diff] [blame] | 2344 | torture_stop_kthread(rcu_torture_boost, t); |
Sebastian Andrzej Siewior | 0ffd374 | 2016-08-18 14:57:22 +0200 | [diff] [blame] | 2345 | return 0; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 2346 | } |
| 2347 | |
Sebastian Andrzej Siewior | 0ffd374 | 2016-08-18 14:57:22 +0200 | [diff] [blame] | 2348 | static int rcutorture_booster_init(unsigned int cpu) |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 2349 | { |
| 2350 | int retval; |
| 2351 | |
| 2352 | if (boost_tasks[cpu] != NULL) |
| 2353 | return 0; /* Already created, nothing more to do. */ |
| 2354 | |
Frederic Weisbecker | 3002153 | 2022-06-10 15:03:57 +0200 | [diff] [blame] | 2355 | // Testing RCU priority boosting requires rcutorture do |
| 2356 | // some serious abuse. Counter this by running ksoftirqd |
| 2357 | // at higher priority. |
| 2358 | if (IS_BUILTIN(CONFIG_RCU_TORTURE_TEST)) { |
| 2359 | struct sched_param sp; |
| 2360 | struct task_struct *t; |
| 2361 | |
| 2362 | t = per_cpu(ksoftirqd, cpu); |
| 2363 | WARN_ON_ONCE(!t); |
| 2364 | sp.sched_priority = 2; |
| 2365 | sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); |
| 2366 | } |
| 2367 | |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 2368 | /* Don't allow time recalculation while creating a new task. */ |
| 2369 | mutex_lock(&boost_mutex); |
Joel Fernandes (Google) | 450efca | 2018-06-10 16:45:43 -0700 | [diff] [blame] | 2370 | rcu_torture_disable_rt_throttle(); |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 2371 | VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task"); |
Cai Huoqing | 3b9cb4b | 2022-01-14 14:03:02 -0800 | [diff] [blame] | 2372 | boost_tasks[cpu] = kthread_run_on_cpu(rcu_torture_boost, NULL, |
| 2373 | cpu, "rcu_torture_boost_%u"); |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 2374 | if (IS_ERR(boost_tasks[cpu])) { |
| 2375 | retval = PTR_ERR(boost_tasks[cpu]); |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 2376 | VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed"); |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 2377 | n_rcu_torture_boost_ktrerror++; |
| 2378 | boost_tasks[cpu] = NULL; |
| 2379 | mutex_unlock(&boost_mutex); |
| 2380 | return retval; |
| 2381 | } |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 2382 | mutex_unlock(&boost_mutex); |
| 2383 | return 0; |
| 2384 | } |
| 2385 | |
Paul E. McKenney | d5f546d | 2011-11-04 11:44:12 -0700 | [diff] [blame] | 2386 | /* |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 2387 | * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then |
| 2388 | * induces a CPU stall for the time specified by stall_cpu. |
| 2389 | */ |
Paul Gortmaker | 49fb4c6 | 2013-06-19 14:52:21 -0400 | [diff] [blame] | 2390 | static int rcu_torture_stall(void *args) |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 2391 | { |
Paul E. McKenney | 19a8ff9 | 2020-03-11 17:39:12 -0700 | [diff] [blame] | 2392 | int idx; |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 2393 | unsigned long stop_at; |
| 2394 | |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 2395 | VERBOSE_TOROUT_STRING("rcu_torture_stall task started"); |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 2396 | if (stall_cpu_holdoff > 0) { |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 2397 | VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff"); |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 2398 | schedule_timeout_interruptible(stall_cpu_holdoff * HZ); |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 2399 | VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff"); |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 2400 | } |
Paul E. McKenney | 55b2dcf | 2020-04-01 19:57:52 -0700 | [diff] [blame] | 2401 | if (!kthread_should_stop() && stall_gp_kthread > 0) { |
| 2402 | VERBOSE_TOROUT_STRING("rcu_torture_stall begin GP stall"); |
| 2403 | rcu_gp_set_torture_wait(stall_gp_kthread * HZ); |
| 2404 | for (idx = 0; idx < stall_gp_kthread + 2; idx++) { |
| 2405 | if (kthread_should_stop()) |
| 2406 | break; |
| 2407 | schedule_timeout_uninterruptible(HZ); |
| 2408 | } |
| 2409 | } |
| 2410 | if (!kthread_should_stop() && stall_cpu > 0) { |
| 2411 | VERBOSE_TOROUT_STRING("rcu_torture_stall begin CPU stall"); |
Arnd Bergmann | 622be33f | 2018-06-18 16:47:34 +0200 | [diff] [blame] | 2412 | stop_at = ktime_get_seconds() + stall_cpu; |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 2413 | /* RCU CPU stall is expected behavior in following code. */ |
Paul E. McKenney | 19a8ff9 | 2020-03-11 17:39:12 -0700 | [diff] [blame] | 2414 | idx = cur_ops->readlock(); |
Paul E. McKenney | 2b1516e | 2017-08-18 16:11:37 -0700 | [diff] [blame] | 2415 | if (stall_cpu_irqsoff) |
| 2416 | local_irq_disable(); |
Paul E. McKenney | 19a8ff9 | 2020-03-11 17:39:12 -0700 | [diff] [blame] | 2417 | else if (!stall_cpu_block) |
Paul E. McKenney | 2b1516e | 2017-08-18 16:11:37 -0700 | [diff] [blame] | 2418 | preempt_disable(); |
Stephen Zhang | 0a27fff | 2021-01-23 17:54:17 +0800 | [diff] [blame] | 2419 | pr_alert("%s start on CPU %d.\n", |
| 2420 | __func__, raw_smp_processor_id()); |
Arnd Bergmann | 622be33f | 2018-06-18 16:47:34 +0200 | [diff] [blame] | 2421 | while (ULONG_CMP_LT((unsigned long)ktime_get_seconds(), |
| 2422 | stop_at)) |
Paul E. McKenney | 59e8366 | 2021-05-16 21:17:27 -0700 | [diff] [blame] | 2423 | if (stall_cpu_block) { |
| 2424 | #ifdef CONFIG_PREEMPTION |
| 2425 | preempt_schedule(); |
| 2426 | #else |
Paul E. McKenney | 19a8ff9 | 2020-03-11 17:39:12 -0700 | [diff] [blame] | 2427 | schedule_timeout_uninterruptible(HZ); |
Paul E. McKenney | 59e8366 | 2021-05-16 21:17:27 -0700 | [diff] [blame] | 2428 | #endif |
Wander Lairson Costa | 5ff7c9f | 2021-11-10 11:37:45 -0300 | [diff] [blame] | 2429 | } else if (stall_no_softlockup) { |
| 2430 | touch_softlockup_watchdog(); |
Paul E. McKenney | 59e8366 | 2021-05-16 21:17:27 -0700 | [diff] [blame] | 2431 | } |
Paul E. McKenney | 2b1516e | 2017-08-18 16:11:37 -0700 | [diff] [blame] | 2432 | if (stall_cpu_irqsoff) |
| 2433 | local_irq_enable(); |
Paul E. McKenney | 19a8ff9 | 2020-03-11 17:39:12 -0700 | [diff] [blame] | 2434 | else if (!stall_cpu_block) |
Paul E. McKenney | 2b1516e | 2017-08-18 16:11:37 -0700 | [diff] [blame] | 2435 | preempt_enable(); |
Paul E. McKenney | 19a8ff9 | 2020-03-11 17:39:12 -0700 | [diff] [blame] | 2436 | cur_ops->readunlock(idx); |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 2437 | } |
Stephen Zhang | 0a27fff | 2021-01-23 17:54:17 +0800 | [diff] [blame] | 2438 | pr_alert("%s end.\n", __func__); |
Paul E. McKenney | f67a335 | 2014-01-29 07:40:27 -0800 | [diff] [blame] | 2439 | torture_shutdown_absorb("rcu_torture_stall"); |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 2440 | while (!kthread_should_stop()) |
| 2441 | schedule_timeout_interruptible(10 * HZ); |
| 2442 | return 0; |
| 2443 | } |
| 2444 | |
| 2445 | /* Spawn CPU-stall kthread, if stall_cpu specified. */ |
| 2446 | static int __init rcu_torture_stall_init(void) |
| 2447 | { |
Paul E. McKenney | 55b2dcf | 2020-04-01 19:57:52 -0700 | [diff] [blame] | 2448 | if (stall_cpu <= 0 && stall_gp_kthread <= 0) |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 2449 | return 0; |
Paul E. McKenney | 47cf29b | 2014-02-03 11:52:27 -0800 | [diff] [blame] | 2450 | return torture_create_kthread(rcu_torture_stall, NULL, stall_task); |
Paul E. McKenney | c13f375 | 2012-01-20 15:36:33 -0800 | [diff] [blame] | 2451 | } |
| 2452 | |
Paul E. McKenney | 9fdcb9a | 2018-07-19 13:36:00 -0700 | [diff] [blame] | 2453 | /* State structure for forward-progress self-propagating RCU callback. */ |
| 2454 | struct fwd_cb_state { |
| 2455 | struct rcu_head rh; |
| 2456 | int stop; |
| 2457 | }; |
| 2458 | |
| 2459 | /* |
| 2460 | * Forward-progress self-propagating RCU callback function. Because |
| 2461 | * callbacks run from softirq, this function is an implicit RCU read-side |
| 2462 | * critical section. |
| 2463 | */ |
| 2464 | static void rcu_torture_fwd_prog_cb(struct rcu_head *rhp) |
| 2465 | { |
| 2466 | struct fwd_cb_state *fcsp = container_of(rhp, struct fwd_cb_state, rh); |
| 2467 | |
| 2468 | if (READ_ONCE(fcsp->stop)) { |
| 2469 | WRITE_ONCE(fcsp->stop, 2); |
| 2470 | return; |
| 2471 | } |
| 2472 | cur_ops->call(&fcsp->rh, rcu_torture_fwd_prog_cb); |
| 2473 | } |
| 2474 | |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2475 | /* State for continuous-flood RCU callbacks. */ |
| 2476 | struct rcu_fwd_cb { |
| 2477 | struct rcu_head rh; |
| 2478 | struct rcu_fwd_cb *rfc_next; |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2479 | struct rcu_fwd *rfc_rfp; |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2480 | int rfc_gps; |
| 2481 | }; |
Paul E. McKenney | a289e60 | 2019-11-05 08:31:56 -0800 | [diff] [blame] | 2482 | |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2483 | #define MAX_FWD_CB_JIFFIES (8 * HZ) /* Maximum CB test duration. */ |
| 2484 | #define MIN_FWD_CB_LAUNDERS 3 /* This many CB invocations to count. */ |
| 2485 | #define MIN_FWD_CBS_LAUNDERED 100 /* Number of counted CBs. */ |
Paul E. McKenney | 2e57bf9 | 2018-10-05 16:43:09 -0700 | [diff] [blame] | 2486 | #define FWD_CBS_HIST_DIV 10 /* Histogram buckets/second. */ |
Paul E. McKenney | a289e60 | 2019-11-05 08:31:56 -0800 | [diff] [blame] | 2487 | #define N_LAUNDERS_HIST (2 * MAX_FWD_CB_JIFFIES / (HZ / FWD_CBS_HIST_DIV)) |
| 2488 | |
Paul E. McKenney | cd618d1 | 2019-01-08 13:41:26 -0800 | [diff] [blame] | 2489 | struct rcu_launder_hist { |
| 2490 | long n_launders; |
| 2491 | unsigned long launder_gp_seq; |
| 2492 | }; |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2493 | |
Paul E. McKenney | a289e60 | 2019-11-05 08:31:56 -0800 | [diff] [blame] | 2494 | struct rcu_fwd { |
| 2495 | spinlock_t rcu_fwd_lock; |
| 2496 | struct rcu_fwd_cb *rcu_fwd_cb_head; |
| 2497 | struct rcu_fwd_cb **rcu_fwd_cb_tail; |
| 2498 | long n_launders_cb; |
| 2499 | unsigned long rcu_fwd_startat; |
| 2500 | struct rcu_launder_hist n_launders_hist[N_LAUNDERS_HIST]; |
| 2501 | unsigned long rcu_launder_gp_seq_start; |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2502 | int rcu_fwd_id; |
Paul E. McKenney | a289e60 | 2019-11-05 08:31:56 -0800 | [diff] [blame] | 2503 | }; |
| 2504 | |
Paul E. McKenney | 57f6020 | 2020-07-20 08:34:07 -0700 | [diff] [blame] | 2505 | static DEFINE_MUTEX(rcu_fwd_mutex); |
Jason Yan | afbc157 | 2020-04-09 19:42:38 +0800 | [diff] [blame] | 2506 | static struct rcu_fwd *rcu_fwds; |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2507 | static unsigned long rcu_fwd_seq; |
Paul E. McKenney | 53b541f | 2021-11-23 13:51:11 -0800 | [diff] [blame] | 2508 | static atomic_long_t rcu_fwd_max_cbs; |
Jason Yan | afbc157 | 2020-04-09 19:42:38 +0800 | [diff] [blame] | 2509 | static bool rcu_fwd_emergency_stop; |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2510 | |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2511 | static void rcu_torture_fwd_cb_hist(struct rcu_fwd *rfp) |
Paul E. McKenney | 1a68275 | 2018-10-03 12:33:41 -0700 | [diff] [blame] | 2512 | { |
Paul E. McKenney | cd618d1 | 2019-01-08 13:41:26 -0800 | [diff] [blame] | 2513 | unsigned long gps; |
| 2514 | unsigned long gps_old; |
Paul E. McKenney | 1a68275 | 2018-10-03 12:33:41 -0700 | [diff] [blame] | 2515 | int i; |
| 2516 | int j; |
| 2517 | |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2518 | for (i = ARRAY_SIZE(rfp->n_launders_hist) - 1; i > 0; i--) |
| 2519 | if (rfp->n_launders_hist[i].n_launders > 0) |
Paul E. McKenney | 1a68275 | 2018-10-03 12:33:41 -0700 | [diff] [blame] | 2520 | break; |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2521 | pr_alert("%s: Callback-invocation histogram %d (duration %lu jiffies):", |
| 2522 | __func__, rfp->rcu_fwd_id, jiffies - rfp->rcu_fwd_startat); |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2523 | gps_old = rfp->rcu_launder_gp_seq_start; |
Paul E. McKenney | cd618d1 | 2019-01-08 13:41:26 -0800 | [diff] [blame] | 2524 | for (j = 0; j <= i; j++) { |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2525 | gps = rfp->n_launders_hist[j].launder_gp_seq; |
Paul E. McKenney | cd618d1 | 2019-01-08 13:41:26 -0800 | [diff] [blame] | 2526 | pr_cont(" %ds/%d: %ld:%ld", |
Paul E. McKenney | a289e60 | 2019-11-05 08:31:56 -0800 | [diff] [blame] | 2527 | j + 1, FWD_CBS_HIST_DIV, |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2528 | rfp->n_launders_hist[j].n_launders, |
Paul E. McKenney | cd618d1 | 2019-01-08 13:41:26 -0800 | [diff] [blame] | 2529 | rcutorture_seq_diff(gps, gps_old)); |
| 2530 | gps_old = gps; |
| 2531 | } |
Paul E. McKenney | 1a68275 | 2018-10-03 12:33:41 -0700 | [diff] [blame] | 2532 | pr_cont("\n"); |
| 2533 | } |
| 2534 | |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2535 | /* Callback function for continuous-flood RCU callbacks. */ |
| 2536 | static void rcu_torture_fwd_cb_cr(struct rcu_head *rhp) |
| 2537 | { |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2538 | unsigned long flags; |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2539 | int i; |
| 2540 | struct rcu_fwd_cb *rfcp = container_of(rhp, struct rcu_fwd_cb, rh); |
| 2541 | struct rcu_fwd_cb **rfcpp; |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2542 | struct rcu_fwd *rfp = rfcp->rfc_rfp; |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2543 | |
| 2544 | rfcp->rfc_next = NULL; |
| 2545 | rfcp->rfc_gps++; |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2546 | spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); |
| 2547 | rfcpp = rfp->rcu_fwd_cb_tail; |
| 2548 | rfp->rcu_fwd_cb_tail = &rfcp->rfc_next; |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2549 | WRITE_ONCE(*rfcpp, rfcp); |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2550 | WRITE_ONCE(rfp->n_launders_cb, rfp->n_launders_cb + 1); |
| 2551 | i = ((jiffies - rfp->rcu_fwd_startat) / (HZ / FWD_CBS_HIST_DIV)); |
| 2552 | if (i >= ARRAY_SIZE(rfp->n_launders_hist)) |
| 2553 | i = ARRAY_SIZE(rfp->n_launders_hist) - 1; |
| 2554 | rfp->n_launders_hist[i].n_launders++; |
| 2555 | rfp->n_launders_hist[i].launder_gp_seq = cur_ops->get_gp_seq(); |
| 2556 | spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2557 | } |
| 2558 | |
Paul E. McKenney | ab21f60 | 2019-04-14 18:30:22 -0700 | [diff] [blame] | 2559 | // Give the scheduler a chance, even on nohz_full CPUs. |
Paul E. McKenney | bd1bfc5 | 2019-06-22 14:35:59 -0700 | [diff] [blame] | 2560 | static void rcu_torture_fwd_prog_cond_resched(unsigned long iter) |
Paul E. McKenney | ab21f60 | 2019-04-14 18:30:22 -0700 | [diff] [blame] | 2561 | { |
Sebastian Andrzej Siewior | 90326f0 | 2019-10-15 21:18:14 +0200 | [diff] [blame] | 2562 | if (IS_ENABLED(CONFIG_PREEMPTION) && IS_ENABLED(CONFIG_NO_HZ_FULL)) { |
Paul E. McKenney | bd1bfc5 | 2019-06-22 14:35:59 -0700 | [diff] [blame] | 2563 | // Real call_rcu() floods hit userspace, so emulate that. |
| 2564 | if (need_resched() || (iter & 0xfff)) |
Paul E. McKenney | ab21f60 | 2019-04-14 18:30:22 -0700 | [diff] [blame] | 2565 | schedule(); |
Paul E. McKenney | d38e6dc | 2019-07-28 12:00:48 -0700 | [diff] [blame] | 2566 | return; |
Paul E. McKenney | ab21f60 | 2019-04-14 18:30:22 -0700 | [diff] [blame] | 2567 | } |
Paul E. McKenney | d38e6dc | 2019-07-28 12:00:48 -0700 | [diff] [blame] | 2568 | // No userspace emulation: CB invocation throttles call_rcu() |
| 2569 | cond_resched(); |
Paul E. McKenney | ab21f60 | 2019-04-14 18:30:22 -0700 | [diff] [blame] | 2570 | } |
| 2571 | |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2572 | /* |
| 2573 | * Free all callbacks on the rcu_fwd_cb_head list, either because the |
| 2574 | * test is over or because we hit an OOM event. |
| 2575 | */ |
Paul E. McKenney | 6764100 | 2019-11-06 08:20:20 -0800 | [diff] [blame] | 2576 | static unsigned long rcu_torture_fwd_prog_cbfree(struct rcu_fwd *rfp) |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2577 | { |
| 2578 | unsigned long flags; |
| 2579 | unsigned long freed = 0; |
| 2580 | struct rcu_fwd_cb *rfcp; |
| 2581 | |
| 2582 | for (;;) { |
Paul E. McKenney | 6764100 | 2019-11-06 08:20:20 -0800 | [diff] [blame] | 2583 | spin_lock_irqsave(&rfp->rcu_fwd_lock, flags); |
| 2584 | rfcp = rfp->rcu_fwd_cb_head; |
Paul E. McKenney | 140e53f | 2019-04-09 10:08:18 -0700 | [diff] [blame] | 2585 | if (!rfcp) { |
Paul E. McKenney | 6764100 | 2019-11-06 08:20:20 -0800 | [diff] [blame] | 2586 | spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2587 | break; |
Paul E. McKenney | 140e53f | 2019-04-09 10:08:18 -0700 | [diff] [blame] | 2588 | } |
Paul E. McKenney | 6764100 | 2019-11-06 08:20:20 -0800 | [diff] [blame] | 2589 | rfp->rcu_fwd_cb_head = rfcp->rfc_next; |
| 2590 | if (!rfp->rcu_fwd_cb_head) |
| 2591 | rfp->rcu_fwd_cb_tail = &rfp->rcu_fwd_cb_head; |
| 2592 | spin_unlock_irqrestore(&rfp->rcu_fwd_lock, flags); |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2593 | kfree(rfcp); |
| 2594 | freed++; |
Paul E. McKenney | bd1bfc5 | 2019-06-22 14:35:59 -0700 | [diff] [blame] | 2595 | rcu_torture_fwd_prog_cond_resched(freed); |
Paul E. McKenney | 79ba7ff | 2019-08-04 13:17:35 -0700 | [diff] [blame] | 2596 | if (tick_nohz_full_enabled()) { |
| 2597 | local_irq_save(flags); |
| 2598 | rcu_momentary_dyntick_idle(); |
| 2599 | local_irq_restore(flags); |
| 2600 | } |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2601 | } |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2602 | return freed; |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2603 | } |
| 2604 | |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2605 | /* Carry out need_resched()/cond_resched() forward-progress testing. */ |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2606 | static void rcu_torture_fwd_prog_nr(struct rcu_fwd *rfp, |
| 2607 | int *tested, int *tested_tries) |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2608 | { |
Paul E. McKenney | 119248b | 2018-07-18 15:39:37 -0700 | [diff] [blame] | 2609 | unsigned long cver; |
Paul E. McKenney | f4de46e | 2018-07-24 20:50:40 -0700 | [diff] [blame] | 2610 | unsigned long dur; |
Paul E. McKenney | 7c590fc | 2018-08-07 16:42:42 -0700 | [diff] [blame] | 2611 | struct fwd_cb_state fcs; |
Paul E. McKenney | 119248b | 2018-07-18 15:39:37 -0700 | [diff] [blame] | 2612 | unsigned long gps; |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2613 | int idx; |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2614 | int sd; |
| 2615 | int sd4; |
| 2616 | bool selfpropcb = false; |
| 2617 | unsigned long stopat; |
| 2618 | static DEFINE_TORTURE_RANDOM(trs); |
| 2619 | |
Paul E. McKenney | 05b7246 | 2021-12-16 15:36:02 -0800 | [diff] [blame] | 2620 | pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); |
Paul E. McKenney | a7eb937 | 2020-10-09 19:51:55 -0700 | [diff] [blame] | 2621 | if (!cur_ops->sync) |
| 2622 | return; // Cannot do need_resched() forward progress testing without ->sync. |
| 2623 | if (cur_ops->call && cur_ops->cb_barrier) { |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2624 | init_rcu_head_on_stack(&fcs.rh); |
| 2625 | selfpropcb = true; |
| 2626 | } |
| 2627 | |
| 2628 | /* Tight loop containing cond_resched(). */ |
Paul E. McKenney | e22ef8d | 2021-12-17 12:33:53 -0800 | [diff] [blame] | 2629 | atomic_inc(&rcu_fwd_cb_nodelay); |
Paul E. McKenney | e8516c6 | 2019-04-09 11:06:32 -0700 | [diff] [blame] | 2630 | cur_ops->sync(); /* Later readers see above write. */ |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2631 | if (selfpropcb) { |
| 2632 | WRITE_ONCE(fcs.stop, 0); |
| 2633 | cur_ops->call(&fcs.rh, rcu_torture_fwd_prog_cb); |
| 2634 | } |
| 2635 | cver = READ_ONCE(rcu_torture_current_version); |
| 2636 | gps = cur_ops->get_gp_seq(); |
| 2637 | sd = cur_ops->stall_dur() + 1; |
| 2638 | sd4 = (sd + fwd_progress_div - 1) / fwd_progress_div; |
| 2639 | dur = sd4 + torture_random(&trs) % (sd - sd4); |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2640 | WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); |
| 2641 | stopat = rfp->rcu_fwd_startat + dur; |
Paul E. McKenney | e0aff97 | 2018-10-01 17:40:54 -0700 | [diff] [blame] | 2642 | while (time_before(jiffies, stopat) && |
Paul E. McKenney | 60013d5 | 2019-07-10 08:30:00 -0700 | [diff] [blame] | 2643 | !shutdown_time_arrived() && |
Paul E. McKenney | e0aff97 | 2018-10-01 17:40:54 -0700 | [diff] [blame] | 2644 | !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2645 | idx = cur_ops->readlock(); |
| 2646 | udelay(10); |
| 2647 | cur_ops->readunlock(idx); |
| 2648 | if (!fwd_progress_need_resched || need_resched()) |
Paul E. McKenney | fbbd5e3 | 2019-08-15 11:43:53 -0700 | [diff] [blame] | 2649 | cond_resched(); |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2650 | } |
| 2651 | (*tested_tries)++; |
Paul E. McKenney | e0aff97 | 2018-10-01 17:40:54 -0700 | [diff] [blame] | 2652 | if (!time_before(jiffies, stopat) && |
Paul E. McKenney | 60013d5 | 2019-07-10 08:30:00 -0700 | [diff] [blame] | 2653 | !shutdown_time_arrived() && |
Paul E. McKenney | e0aff97 | 2018-10-01 17:40:54 -0700 | [diff] [blame] | 2654 | !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2655 | (*tested)++; |
| 2656 | cver = READ_ONCE(rcu_torture_current_version) - cver; |
| 2657 | gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); |
| 2658 | WARN_ON(!cver && gps < 2); |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2659 | pr_alert("%s: %d Duration %ld cver %ld gps %ld\n", __func__, |
| 2660 | rfp->rcu_fwd_id, dur, cver, gps); |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2661 | } |
| 2662 | if (selfpropcb) { |
| 2663 | WRITE_ONCE(fcs.stop, 1); |
| 2664 | cur_ops->sync(); /* Wait for running CB to complete. */ |
Paul E. McKenney | 05b7246 | 2021-12-16 15:36:02 -0800 | [diff] [blame] | 2665 | pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2666 | cur_ops->cb_barrier(); /* Wait for queued callbacks. */ |
| 2667 | } |
| 2668 | |
| 2669 | if (selfpropcb) { |
| 2670 | WARN_ON(READ_ONCE(fcs.stop) != 2); |
| 2671 | destroy_rcu_head_on_stack(&fcs.rh); |
| 2672 | } |
Paul E. McKenney | e8516c6 | 2019-04-09 11:06:32 -0700 | [diff] [blame] | 2673 | schedule_timeout_uninterruptible(HZ / 10); /* Let kthreads recover. */ |
Paul E. McKenney | e22ef8d | 2021-12-17 12:33:53 -0800 | [diff] [blame] | 2674 | atomic_dec(&rcu_fwd_cb_nodelay); |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2675 | } |
| 2676 | |
| 2677 | /* Carry out call_rcu() forward-progress testing. */ |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2678 | static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2679 | { |
| 2680 | unsigned long cver; |
Paul E. McKenney | 79ba7ff | 2019-08-04 13:17:35 -0700 | [diff] [blame] | 2681 | unsigned long flags; |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2682 | unsigned long gps; |
| 2683 | int i; |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2684 | long n_launders; |
| 2685 | long n_launders_cb_snap; |
| 2686 | long n_launders_sa; |
| 2687 | long n_max_cbs; |
| 2688 | long n_max_gps; |
| 2689 | struct rcu_fwd_cb *rfcp; |
| 2690 | struct rcu_fwd_cb *rfcpn; |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2691 | unsigned long stopat; |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2692 | unsigned long stoppedat; |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2693 | |
Paul E. McKenney | 05b7246 | 2021-12-16 15:36:02 -0800 | [diff] [blame] | 2694 | pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2695 | if (READ_ONCE(rcu_fwd_emergency_stop)) |
| 2696 | return; /* Get out of the way quickly, no GP wait! */ |
Paul E. McKenney | c682db5 | 2019-04-19 07:38:27 -0700 | [diff] [blame] | 2697 | if (!cur_ops->call) |
| 2698 | return; /* Can't do call_rcu() fwd prog without ->call. */ |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2699 | |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2700 | /* Loop continuously posting RCU callbacks. */ |
Paul E. McKenney | e22ef8d | 2021-12-17 12:33:53 -0800 | [diff] [blame] | 2701 | atomic_inc(&rcu_fwd_cb_nodelay); |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2702 | cur_ops->sync(); /* Later readers see above write. */ |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2703 | WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); |
| 2704 | stopat = rfp->rcu_fwd_startat + MAX_FWD_CB_JIFFIES; |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2705 | n_launders = 0; |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2706 | rfp->n_launders_cb = 0; // Hoist initialization for multi-kthread |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2707 | n_launders_sa = 0; |
| 2708 | n_max_cbs = 0; |
| 2709 | n_max_gps = 0; |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2710 | for (i = 0; i < ARRAY_SIZE(rfp->n_launders_hist); i++) |
| 2711 | rfp->n_launders_hist[i].n_launders = 0; |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2712 | cver = READ_ONCE(rcu_torture_current_version); |
| 2713 | gps = cur_ops->get_gp_seq(); |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2714 | rfp->rcu_launder_gp_seq_start = gps; |
Paul E. McKenney | d38e6dc | 2019-07-28 12:00:48 -0700 | [diff] [blame] | 2715 | tick_dep_set_task(current, TICK_DEP_BIT_RCU); |
Paul E. McKenney | e0aff97 | 2018-10-01 17:40:54 -0700 | [diff] [blame] | 2716 | while (time_before(jiffies, stopat) && |
Paul E. McKenney | 60013d5 | 2019-07-10 08:30:00 -0700 | [diff] [blame] | 2717 | !shutdown_time_arrived() && |
Paul E. McKenney | e0aff97 | 2018-10-01 17:40:54 -0700 | [diff] [blame] | 2718 | !READ_ONCE(rcu_fwd_emergency_stop) && !torture_must_stop()) { |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2719 | rfcp = READ_ONCE(rfp->rcu_fwd_cb_head); |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2720 | rfcpn = NULL; |
| 2721 | if (rfcp) |
| 2722 | rfcpn = READ_ONCE(rfcp->rfc_next); |
| 2723 | if (rfcpn) { |
| 2724 | if (rfcp->rfc_gps >= MIN_FWD_CB_LAUNDERS && |
| 2725 | ++n_max_gps >= MIN_FWD_CBS_LAUNDERED) |
| 2726 | break; |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2727 | rfp->rcu_fwd_cb_head = rfcpn; |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2728 | n_launders++; |
| 2729 | n_launders_sa++; |
Paul E. McKenney | 613b00f | 2021-11-23 11:53:52 -0800 | [diff] [blame] | 2730 | } else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) { |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2731 | rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL); |
| 2732 | if (WARN_ON_ONCE(!rfcp)) { |
| 2733 | schedule_timeout_interruptible(1); |
| 2734 | continue; |
| 2735 | } |
| 2736 | n_max_cbs++; |
| 2737 | n_launders_sa = 0; |
| 2738 | rfcp->rfc_gps = 0; |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2739 | rfcp->rfc_rfp = rfp; |
Paul E. McKenney | 613b00f | 2021-11-23 11:53:52 -0800 | [diff] [blame] | 2740 | } else { |
| 2741 | rfcp = NULL; |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2742 | } |
Paul E. McKenney | 613b00f | 2021-11-23 11:53:52 -0800 | [diff] [blame] | 2743 | if (rfcp) |
| 2744 | cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); |
Paul E. McKenney | bd1bfc5 | 2019-06-22 14:35:59 -0700 | [diff] [blame] | 2745 | rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs); |
Paul E. McKenney | 79ba7ff | 2019-08-04 13:17:35 -0700 | [diff] [blame] | 2746 | if (tick_nohz_full_enabled()) { |
| 2747 | local_irq_save(flags); |
| 2748 | rcu_momentary_dyntick_idle(); |
| 2749 | local_irq_restore(flags); |
| 2750 | } |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2751 | } |
| 2752 | stoppedat = jiffies; |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2753 | n_launders_cb_snap = READ_ONCE(rfp->n_launders_cb); |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2754 | cver = READ_ONCE(rcu_torture_current_version) - cver; |
| 2755 | gps = rcutorture_seq_diff(cur_ops->get_gp_seq(), gps); |
Paul E. McKenney | 05b7246 | 2021-12-16 15:36:02 -0800 | [diff] [blame] | 2756 | pr_alert("%s: Waiting for CBs: %pS() %d\n", __func__, cur_ops->cb_barrier, rfp->rcu_fwd_id); |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2757 | cur_ops->cb_barrier(); /* Wait for callbacks to be invoked. */ |
Paul E. McKenney | 6764100 | 2019-11-06 08:20:20 -0800 | [diff] [blame] | 2758 | (void)rcu_torture_fwd_prog_cbfree(rfp); |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2759 | |
Paul E. McKenney | 60013d5 | 2019-07-10 08:30:00 -0700 | [diff] [blame] | 2760 | if (!torture_must_stop() && !READ_ONCE(rcu_fwd_emergency_stop) && |
| 2761 | !shutdown_time_arrived()) { |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2762 | WARN_ON(n_max_gps < MIN_FWD_CBS_LAUNDERED); |
| 2763 | pr_alert("%s Duration %lu barrier: %lu pending %ld n_launders: %ld n_launders_sa: %ld n_max_gps: %ld n_max_cbs: %ld cver %ld gps %ld\n", |
| 2764 | __func__, |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2765 | stoppedat - rfp->rcu_fwd_startat, jiffies - stoppedat, |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2766 | n_launders + n_max_cbs - n_launders_cb_snap, |
| 2767 | n_launders, n_launders_sa, |
| 2768 | n_max_gps, n_max_cbs, cver, gps); |
Paul E. McKenney | 53b541f | 2021-11-23 13:51:11 -0800 | [diff] [blame] | 2769 | atomic_long_add(n_max_cbs, &rcu_fwd_max_cbs); |
Paul E. McKenney | 89440d2 | 2021-12-28 15:59:38 -0800 | [diff] [blame] | 2770 | mutex_lock(&rcu_fwd_mutex); // Serialize histograms. |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2771 | rcu_torture_fwd_cb_hist(rfp); |
Paul E. McKenney | 89440d2 | 2021-12-28 15:59:38 -0800 | [diff] [blame] | 2772 | mutex_unlock(&rcu_fwd_mutex); |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2773 | } |
Paul E. McKenney | e8516c6 | 2019-04-09 11:06:32 -0700 | [diff] [blame] | 2774 | schedule_timeout_uninterruptible(HZ); /* Let CBs drain. */ |
Paul E. McKenney | d38e6dc | 2019-07-28 12:00:48 -0700 | [diff] [blame] | 2775 | tick_dep_clear_task(current, TICK_DEP_BIT_RCU); |
Paul E. McKenney | e22ef8d | 2021-12-17 12:33:53 -0800 | [diff] [blame] | 2776 | atomic_dec(&rcu_fwd_cb_nodelay); |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2777 | } |
| 2778 | |
Paul E. McKenney | e0aff97 | 2018-10-01 17:40:54 -0700 | [diff] [blame] | 2779 | |
| 2780 | /* |
| 2781 | * OOM notifier, but this only prints diagnostic information for the |
| 2782 | * current forward-progress test. |
| 2783 | */ |
| 2784 | static int rcutorture_oom_notify(struct notifier_block *self, |
| 2785 | unsigned long notused, void *nfreed) |
| 2786 | { |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2787 | int i; |
| 2788 | long ncbs; |
Paul E. McKenney | 57f6020 | 2020-07-20 08:34:07 -0700 | [diff] [blame] | 2789 | struct rcu_fwd *rfp; |
Paul E. McKenney | 6764100 | 2019-11-06 08:20:20 -0800 | [diff] [blame] | 2790 | |
Paul E. McKenney | 57f6020 | 2020-07-20 08:34:07 -0700 | [diff] [blame] | 2791 | mutex_lock(&rcu_fwd_mutex); |
| 2792 | rfp = rcu_fwds; |
| 2793 | if (!rfp) { |
| 2794 | mutex_unlock(&rcu_fwd_mutex); |
| 2795 | return NOTIFY_OK; |
| 2796 | } |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2797 | WARN(1, "%s invoked upon OOM during forward-progress testing.\n", |
| 2798 | __func__); |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2799 | for (i = 0; i < fwd_progress; i++) { |
| 2800 | rcu_torture_fwd_cb_hist(&rfp[i]); |
| 2801 | rcu_fwd_progress_check(1 + (jiffies - READ_ONCE(rfp[i].rcu_fwd_startat)) / 2); |
| 2802 | } |
Paul E. McKenney | e0aff97 | 2018-10-01 17:40:54 -0700 | [diff] [blame] | 2803 | WRITE_ONCE(rcu_fwd_emergency_stop, true); |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2804 | smp_mb(); /* Emergency stop before free and wait to avoid hangs. */ |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2805 | ncbs = 0; |
| 2806 | for (i = 0; i < fwd_progress; i++) |
| 2807 | ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); |
| 2808 | pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); |
Zqiang | 48297a2 | 2022-07-31 18:53:56 +0800 | [diff] [blame] | 2809 | cur_ops->cb_barrier(); |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2810 | ncbs = 0; |
| 2811 | for (i = 0; i < fwd_progress; i++) |
| 2812 | ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); |
| 2813 | pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); |
Zqiang | 48297a2 | 2022-07-31 18:53:56 +0800 | [diff] [blame] | 2814 | cur_ops->cb_barrier(); |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2815 | ncbs = 0; |
| 2816 | for (i = 0; i < fwd_progress; i++) |
| 2817 | ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]); |
| 2818 | pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs); |
Paul E. McKenney | 2667ccc | 2018-10-05 09:09:49 -0700 | [diff] [blame] | 2819 | smp_mb(); /* Frees before return to avoid redoing OOM. */ |
| 2820 | (*(unsigned long *)nfreed)++; /* Forward progress CBs freed! */ |
| 2821 | pr_info("%s returning after OOM processing.\n", __func__); |
Paul E. McKenney | 57f6020 | 2020-07-20 08:34:07 -0700 | [diff] [blame] | 2822 | mutex_unlock(&rcu_fwd_mutex); |
Paul E. McKenney | e0aff97 | 2018-10-01 17:40:54 -0700 | [diff] [blame] | 2823 | return NOTIFY_OK; |
| 2824 | } |
| 2825 | |
| 2826 | static struct notifier_block rcutorture_oom_nb = { |
| 2827 | .notifier_call = rcutorture_oom_notify |
| 2828 | }; |
| 2829 | |
Paul E. McKenney | 6b3de7a | 2018-08-28 14:38:43 -0700 | [diff] [blame] | 2830 | /* Carry out grace-period forward-progress testing. */ |
| 2831 | static int rcu_torture_fwd_prog(void *args) |
| 2832 | { |
Paul E. McKenney | 53b541f | 2021-11-23 13:51:11 -0800 | [diff] [blame] | 2833 | bool firsttime = true; |
| 2834 | long max_cbs; |
Paul E. McKenney | ab1b788 | 2020-09-22 16:42:42 -0700 | [diff] [blame] | 2835 | int oldnice = task_nice(current); |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2836 | unsigned long oldseq = READ_ONCE(rcu_fwd_seq); |
Paul E. McKenney | 6b1b832 | 2019-11-05 09:08:58 -0800 | [diff] [blame] | 2837 | struct rcu_fwd *rfp = args; |
Paul E. McKenney | f4de46e | 2018-07-24 20:50:40 -0700 | [diff] [blame] | 2838 | int tested = 0; |
Paul E. McKenney | 152f4af | 2018-07-19 10:57:58 -0700 | [diff] [blame] | 2839 | int tested_tries = 0; |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2840 | |
| 2841 | VERBOSE_TOROUT_STRING("rcu_torture_fwd_progress task started"); |
Paul E. McKenney | 5ab7ab8 | 2018-09-21 18:08:09 -0700 | [diff] [blame] | 2842 | rcu_bind_current_to_nocb(); |
Paul E. McKenney | fecad50 | 2018-07-20 12:18:11 -0700 | [diff] [blame] | 2843 | if (!IS_ENABLED(CONFIG_SMP) || !IS_ENABLED(CONFIG_RCU_BOOST)) |
| 2844 | set_user_nice(current, MAX_NICE); |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2845 | do { |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2846 | if (!rfp->rcu_fwd_id) { |
| 2847 | schedule_timeout_interruptible(fwd_progress_holdoff * HZ); |
| 2848 | WRITE_ONCE(rcu_fwd_emergency_stop, false); |
Paul E. McKenney | 53b541f | 2021-11-23 13:51:11 -0800 | [diff] [blame] | 2849 | if (!firsttime) { |
| 2850 | max_cbs = atomic_long_xchg(&rcu_fwd_max_cbs, 0); |
| 2851 | pr_alert("%s n_max_cbs: %ld\n", __func__, max_cbs); |
| 2852 | } |
| 2853 | firsttime = false; |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2854 | WRITE_ONCE(rcu_fwd_seq, rcu_fwd_seq + 1); |
| 2855 | } else { |
Paul E. McKenney | 02b51a1 | 2021-12-17 15:05:05 -0800 | [diff] [blame] | 2856 | while (READ_ONCE(rcu_fwd_seq) == oldseq && !torture_must_stop()) |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2857 | schedule_timeout_interruptible(1); |
| 2858 | oldseq = READ_ONCE(rcu_fwd_seq); |
| 2859 | } |
| 2860 | pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); |
| 2861 | if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id) |
Paul E. McKenney | 4355080 | 2019-12-04 15:58:41 -0800 | [diff] [blame] | 2862 | rcu_torture_fwd_prog_cr(rfp); |
Paul E. McKenney | 613b00f | 2021-11-23 11:53:52 -0800 | [diff] [blame] | 2863 | if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) && |
| 2864 | (!IS_ENABLED(CONFIG_TINY_RCU) || |
| 2865 | (rcu_inkernel_boot_has_ended() && |
| 2866 | torture_num_online_cpus() > rfp->rcu_fwd_id))) |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2867 | rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries); |
Paul E. McKenney | 4871848 | 2018-08-15 15:32:51 -0700 | [diff] [blame] | 2868 | |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2869 | /* Avoid slow periods, better to test when busy. */ |
Paul E. McKenney | ab1b788 | 2020-09-22 16:42:42 -0700 | [diff] [blame] | 2870 | if (stutter_wait("rcu_torture_fwd_prog")) |
| 2871 | sched_set_normal(current, oldnice); |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2872 | } while (!torture_must_stop()); |
Paul E. McKenney | 152f4af | 2018-07-19 10:57:58 -0700 | [diff] [blame] | 2873 | /* Short runs might not contain a valid forward-progress attempt. */ |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2874 | if (!rfp->rcu_fwd_id) { |
| 2875 | WARN_ON(!tested && tested_tries >= 5); |
| 2876 | pr_alert("%s: tested %d tested_tries %d\n", __func__, tested, tested_tries); |
| 2877 | } |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2878 | torture_kthread_stopping("rcu_torture_fwd_prog"); |
| 2879 | return 0; |
| 2880 | } |
| 2881 | |
| 2882 | /* If forward-progress checking is requested and feasible, spawn the thread. */ |
| 2883 | static int __init rcu_torture_fwd_prog_init(void) |
| 2884 | { |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2885 | int i; |
| 2886 | int ret = 0; |
Paul E. McKenney | 5155be9 | 2019-11-06 08:35:08 -0800 | [diff] [blame] | 2887 | struct rcu_fwd *rfp; |
Paul E. McKenney | 6764100 | 2019-11-06 08:20:20 -0800 | [diff] [blame] | 2888 | |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2889 | if (!fwd_progress) |
| 2890 | return 0; /* Not requested, so don't do it. */ |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2891 | if (fwd_progress >= nr_cpu_ids) { |
| 2892 | VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Limiting fwd_progress to # CPUs.\n"); |
| 2893 | fwd_progress = nr_cpu_ids; |
| 2894 | } else if (fwd_progress < 0) { |
| 2895 | fwd_progress = nr_cpu_ids; |
| 2896 | } |
Paul E. McKenney | a7eb937 | 2020-10-09 19:51:55 -0700 | [diff] [blame] | 2897 | if ((!cur_ops->sync && !cur_ops->call) || |
Paul E. McKenney | 613b00f | 2021-11-23 11:53:52 -0800 | [diff] [blame] | 2898 | (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) || |
| 2899 | cur_ops == &rcu_busted_ops) { |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2900 | VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test"); |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2901 | fwd_progress = 0; |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2902 | return 0; |
| 2903 | } |
| 2904 | if (stall_cpu > 0) { |
| 2905 | VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, conflicts with CPU-stall testing"); |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2906 | fwd_progress = 0; |
Zhouyi Zhou | 3ac8587 | 2021-07-26 05:43:33 +0800 | [diff] [blame] | 2907 | if (IS_MODULE(CONFIG_RCU_TORTURE_TEST)) |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2908 | return -EINVAL; /* In module, can fail back to user. */ |
| 2909 | WARN_ON(1); /* Make sure rcutorture notices conflict. */ |
| 2910 | return 0; |
| 2911 | } |
| 2912 | if (fwd_progress_holdoff <= 0) |
| 2913 | fwd_progress_holdoff = 1; |
| 2914 | if (fwd_progress_div <= 0) |
| 2915 | fwd_progress_div = 4; |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2916 | rfp = kcalloc(fwd_progress, sizeof(*rfp), GFP_KERNEL); |
| 2917 | fwd_prog_tasks = kcalloc(fwd_progress, sizeof(*fwd_prog_tasks), GFP_KERNEL); |
| 2918 | if (!rfp || !fwd_prog_tasks) { |
| 2919 | kfree(rfp); |
| 2920 | kfree(fwd_prog_tasks); |
| 2921 | fwd_prog_tasks = NULL; |
| 2922 | fwd_progress = 0; |
Paul E. McKenney | 5155be9 | 2019-11-06 08:35:08 -0800 | [diff] [blame] | 2923 | return -ENOMEM; |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2924 | } |
| 2925 | for (i = 0; i < fwd_progress; i++) { |
| 2926 | spin_lock_init(&rfp[i].rcu_fwd_lock); |
| 2927 | rfp[i].rcu_fwd_cb_tail = &rfp[i].rcu_fwd_cb_head; |
| 2928 | rfp[i].rcu_fwd_id = i; |
| 2929 | } |
Paul E. McKenney | 57f6020 | 2020-07-20 08:34:07 -0700 | [diff] [blame] | 2930 | mutex_lock(&rcu_fwd_mutex); |
Paul E. McKenney | c8fa637 | 2020-07-19 14:40:31 -0700 | [diff] [blame] | 2931 | rcu_fwds = rfp; |
Paul E. McKenney | 57f6020 | 2020-07-20 08:34:07 -0700 | [diff] [blame] | 2932 | mutex_unlock(&rcu_fwd_mutex); |
Paul E. McKenney | 299c7d9 | 2020-07-22 10:45:12 -0700 | [diff] [blame] | 2933 | register_oom_notifier(&rcutorture_oom_nb); |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2934 | for (i = 0; i < fwd_progress; i++) { |
| 2935 | ret = torture_create_kthread(rcu_torture_fwd_prog, &rcu_fwds[i], fwd_prog_tasks[i]); |
| 2936 | if (ret) { |
| 2937 | fwd_progress = i; |
| 2938 | return ret; |
| 2939 | } |
| 2940 | } |
| 2941 | return 0; |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 2942 | } |
| 2943 | |
Paul E. McKenney | c8fa637 | 2020-07-19 14:40:31 -0700 | [diff] [blame] | 2944 | static void rcu_torture_fwd_prog_cleanup(void) |
| 2945 | { |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2946 | int i; |
Paul E. McKenney | c8fa637 | 2020-07-19 14:40:31 -0700 | [diff] [blame] | 2947 | struct rcu_fwd *rfp; |
| 2948 | |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2949 | if (!rcu_fwds || !fwd_prog_tasks) |
| 2950 | return; |
| 2951 | for (i = 0; i < fwd_progress; i++) |
| 2952 | torture_stop_kthread(rcu_torture_fwd_prog, fwd_prog_tasks[i]); |
| 2953 | unregister_oom_notifier(&rcutorture_oom_nb); |
Paul E. McKenney | 57f6020 | 2020-07-20 08:34:07 -0700 | [diff] [blame] | 2954 | mutex_lock(&rcu_fwd_mutex); |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2955 | rfp = rcu_fwds; |
Paul E. McKenney | c8fa637 | 2020-07-19 14:40:31 -0700 | [diff] [blame] | 2956 | rcu_fwds = NULL; |
Paul E. McKenney | 57f6020 | 2020-07-20 08:34:07 -0700 | [diff] [blame] | 2957 | mutex_unlock(&rcu_fwd_mutex); |
Paul E. McKenney | c8fa637 | 2020-07-19 14:40:31 -0700 | [diff] [blame] | 2958 | kfree(rfp); |
Paul E. McKenney | 82e3100 | 2021-11-22 20:55:18 -0800 | [diff] [blame] | 2959 | kfree(fwd_prog_tasks); |
| 2960 | fwd_prog_tasks = NULL; |
Paul E. McKenney | c8fa637 | 2020-07-19 14:40:31 -0700 | [diff] [blame] | 2961 | } |
| 2962 | |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 2963 | /* Callback function for RCU barrier testing. */ |
Rashika Kheria | b3b8a4d | 2014-02-27 17:16:57 +0530 | [diff] [blame] | 2964 | static void rcu_torture_barrier_cbf(struct rcu_head *rcu) |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 2965 | { |
| 2966 | atomic_inc(&barrier_cbs_invoked); |
| 2967 | } |
| 2968 | |
Paul E. McKenney | 50d4b62 | 2020-02-04 15:00:56 -0800 | [diff] [blame] | 2969 | /* IPI handler to get callback posted on desired CPU, if online. */ |
| 2970 | static void rcu_torture_barrier1cb(void *rcu_void) |
| 2971 | { |
| 2972 | struct rcu_head *rhp = rcu_void; |
| 2973 | |
| 2974 | cur_ops->call(rhp, rcu_torture_barrier_cbf); |
| 2975 | } |
| 2976 | |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 2977 | /* kthread function to register callbacks used to test RCU barriers. */ |
| 2978 | static int rcu_torture_barrier_cbs(void *arg) |
| 2979 | { |
| 2980 | long myid = (long)arg; |
Jules Irenge | 8f43d59 | 2020-06-01 19:45:48 +0100 | [diff] [blame] | 2981 | bool lastphase = false; |
Paul E. McKenney | 78e4bc3 | 2013-09-24 15:04:06 -0700 | [diff] [blame] | 2982 | bool newphase; |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 2983 | struct rcu_head rcu; |
| 2984 | |
| 2985 | init_rcu_head_on_stack(&rcu); |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 2986 | VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started"); |
Linus Torvalds | 971eae7 | 2014-03-31 11:21:19 -0700 | [diff] [blame] | 2987 | set_user_nice(current, MAX_NICE); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 2988 | do { |
| 2989 | wait_event(barrier_cbs_wq[myid], |
Paul E. McKenney | 78e4bc3 | 2013-09-24 15:04:06 -0700 | [diff] [blame] | 2990 | (newphase = |
Paul E. McKenney | 6c7ed42 | 2015-04-13 11:58:08 -0700 | [diff] [blame] | 2991 | smp_load_acquire(&barrier_phase)) != lastphase || |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 2992 | torture_must_stop()); |
Paul E. McKenney | 78e4bc3 | 2013-09-24 15:04:06 -0700 | [diff] [blame] | 2993 | lastphase = newphase; |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 2994 | if (torture_must_stop()) |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 2995 | break; |
Paul E. McKenney | 6c7ed42 | 2015-04-13 11:58:08 -0700 | [diff] [blame] | 2996 | /* |
| 2997 | * The above smp_load_acquire() ensures barrier_phase load |
Paul E. McKenney | aab0573 | 2016-05-02 12:20:51 -0700 | [diff] [blame] | 2998 | * is ordered before the following ->call(). |
Paul E. McKenney | 6c7ed42 | 2015-04-13 11:58:08 -0700 | [diff] [blame] | 2999 | */ |
Paul E. McKenney | 50d4b62 | 2020-02-04 15:00:56 -0800 | [diff] [blame] | 3000 | if (smp_call_function_single(myid, rcu_torture_barrier1cb, |
| 3001 | &rcu, 1)) { |
| 3002 | // IPI failed, so use direct call from current CPU. |
| 3003 | cur_ops->call(&rcu, rcu_torture_barrier_cbf); |
| 3004 | } |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3005 | if (atomic_dec_and_test(&barrier_cbs_count)) |
| 3006 | wake_up(&barrier_wq); |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 3007 | } while (!torture_must_stop()); |
Paul E. McKenney | 69c6045 | 2014-07-01 11:59:36 -0700 | [diff] [blame] | 3008 | if (cur_ops->cb_barrier != NULL) |
| 3009 | cur_ops->cb_barrier(); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3010 | destroy_rcu_head_on_stack(&rcu); |
Paul E. McKenney | 7fafaac | 2014-01-31 17:37:28 -0800 | [diff] [blame] | 3011 | torture_kthread_stopping("rcu_torture_barrier_cbs"); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3012 | return 0; |
| 3013 | } |
| 3014 | |
| 3015 | /* kthread function to drive and coordinate RCU barrier testing. */ |
| 3016 | static int rcu_torture_barrier(void *arg) |
| 3017 | { |
| 3018 | int i; |
| 3019 | |
Paul E. McKenney | 5ccf60f | 2014-01-29 07:25:25 -0800 | [diff] [blame] | 3020 | VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting"); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3021 | do { |
| 3022 | atomic_set(&barrier_cbs_invoked, 0); |
| 3023 | atomic_set(&barrier_cbs_count, n_barrier_cbs); |
Paul E. McKenney | 6c7ed42 | 2015-04-13 11:58:08 -0700 | [diff] [blame] | 3024 | /* Ensure barrier_phase ordered after prior assignments. */ |
| 3025 | smp_store_release(&barrier_phase, !barrier_phase); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3026 | for (i = 0; i < n_barrier_cbs; i++) |
| 3027 | wake_up(&barrier_cbs_wq[i]); |
| 3028 | wait_event(barrier_wq, |
| 3029 | atomic_read(&barrier_cbs_count) == 0 || |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 3030 | torture_must_stop()); |
| 3031 | if (torture_must_stop()) |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3032 | break; |
| 3033 | n_barrier_attempts++; |
Paul E. McKenney | 78e4bc3 | 2013-09-24 15:04:06 -0700 | [diff] [blame] | 3034 | cur_ops->cb_barrier(); /* Implies smp_mb() for wait_event(). */ |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3035 | if (atomic_read(&barrier_cbs_invoked) != n_barrier_cbs) { |
| 3036 | n_rcu_torture_barrier_error++; |
Paul E. McKenney | 7602de4a | 2014-12-17 18:39:54 -0800 | [diff] [blame] | 3037 | pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n", |
| 3038 | atomic_read(&barrier_cbs_invoked), |
| 3039 | n_barrier_cbs); |
Paul E. McKenney | 9470a18 | 2020-02-05 12:54:34 -0800 | [diff] [blame] | 3040 | WARN_ON(1); |
| 3041 | // Wait manually for the remaining callbacks |
| 3042 | i = 0; |
| 3043 | do { |
| 3044 | if (WARN_ON(i++ > HZ)) |
| 3045 | i = INT_MIN; |
| 3046 | schedule_timeout_interruptible(1); |
| 3047 | cur_ops->cb_barrier(); |
| 3048 | } while (atomic_read(&barrier_cbs_invoked) != |
| 3049 | n_barrier_cbs && |
| 3050 | !torture_must_stop()); |
| 3051 | smp_mb(); // Can't trust ordering if broken. |
| 3052 | if (!torture_must_stop()) |
| 3053 | pr_err("Recovered: barrier_cbs_invoked = %d\n", |
| 3054 | atomic_read(&barrier_cbs_invoked)); |
Joel Fernandes (Google) | bf5b643 | 2018-06-19 15:14:19 -0700 | [diff] [blame] | 3055 | } else { |
| 3056 | n_barrier_successes++; |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3057 | } |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3058 | schedule_timeout_interruptible(HZ / 10); |
Paul E. McKenney | 36970bb | 2014-01-30 15:49:29 -0800 | [diff] [blame] | 3059 | } while (!torture_must_stop()); |
Paul E. McKenney | 7fafaac | 2014-01-31 17:37:28 -0800 | [diff] [blame] | 3060 | torture_kthread_stopping("rcu_torture_barrier"); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3061 | return 0; |
| 3062 | } |
| 3063 | |
| 3064 | /* Initialize RCU barrier testing. */ |
| 3065 | static int rcu_torture_barrier_init(void) |
| 3066 | { |
| 3067 | int i; |
| 3068 | int ret; |
| 3069 | |
Paul E. McKenney | d9eba768 | 2015-05-14 15:35:43 -0700 | [diff] [blame] | 3070 | if (n_barrier_cbs <= 0) |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3071 | return 0; |
| 3072 | if (cur_ops->call == NULL || cur_ops->cb_barrier == NULL) { |
Paul E. McKenney | 2caa1e4 | 2012-08-09 16:30:45 -0700 | [diff] [blame] | 3073 | pr_alert("%s" TORTURE_FLAG |
| 3074 | " Call or barrier ops missing for %s,\n", |
| 3075 | torture_type, cur_ops->name); |
| 3076 | pr_alert("%s" TORTURE_FLAG |
| 3077 | " RCU barrier testing omitted from run.\n", |
| 3078 | torture_type); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3079 | return 0; |
| 3080 | } |
| 3081 | atomic_set(&barrier_cbs_count, 0); |
| 3082 | atomic_set(&barrier_cbs_invoked, 0); |
| 3083 | barrier_cbs_tasks = |
Paul E. McKenney | 68a675d | 2017-12-01 14:26:56 -0800 | [diff] [blame] | 3084 | kcalloc(n_barrier_cbs, sizeof(barrier_cbs_tasks[0]), |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3085 | GFP_KERNEL); |
| 3086 | barrier_cbs_wq = |
Paul E. McKenney | 68a675d | 2017-12-01 14:26:56 -0800 | [diff] [blame] | 3087 | kcalloc(n_barrier_cbs, sizeof(barrier_cbs_wq[0]), GFP_KERNEL); |
Sasha Levin | de5e643 | 2012-12-20 14:11:28 -0500 | [diff] [blame] | 3088 | if (barrier_cbs_tasks == NULL || !barrier_cbs_wq) |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3089 | return -ENOMEM; |
| 3090 | for (i = 0; i < n_barrier_cbs; i++) { |
| 3091 | init_waitqueue_head(&barrier_cbs_wq[i]); |
Paul E. McKenney | 47cf29b | 2014-02-03 11:52:27 -0800 | [diff] [blame] | 3092 | ret = torture_create_kthread(rcu_torture_barrier_cbs, |
| 3093 | (void *)(long)i, |
| 3094 | barrier_cbs_tasks[i]); |
| 3095 | if (ret) |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3096 | return ret; |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3097 | } |
Paul E. McKenney | 47cf29b | 2014-02-03 11:52:27 -0800 | [diff] [blame] | 3098 | return torture_create_kthread(rcu_torture_barrier, NULL, barrier_task); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3099 | } |
| 3100 | |
| 3101 | /* Clean up after RCU barrier testing. */ |
| 3102 | static void rcu_torture_barrier_cleanup(void) |
| 3103 | { |
| 3104 | int i; |
| 3105 | |
Paul E. McKenney | 9c029b8 | 2014-02-04 11:47:08 -0800 | [diff] [blame] | 3106 | torture_stop_kthread(rcu_torture_barrier, barrier_task); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3107 | if (barrier_cbs_tasks != NULL) { |
Paul E. McKenney | 9c029b8 | 2014-02-04 11:47:08 -0800 | [diff] [blame] | 3108 | for (i = 0; i < n_barrier_cbs; i++) |
| 3109 | torture_stop_kthread(rcu_torture_barrier_cbs, |
| 3110 | barrier_cbs_tasks[i]); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3111 | kfree(barrier_cbs_tasks); |
| 3112 | barrier_cbs_tasks = NULL; |
| 3113 | } |
| 3114 | if (barrier_cbs_wq != NULL) { |
| 3115 | kfree(barrier_cbs_wq); |
| 3116 | barrier_cbs_wq = NULL; |
| 3117 | } |
| 3118 | } |
| 3119 | |
Joel Fernandes (Google) | 4babd85 | 2018-06-19 15:14:18 -0700 | [diff] [blame] | 3120 | static bool rcu_torture_can_boost(void) |
| 3121 | { |
| 3122 | static int boost_warn_once; |
| 3123 | int prio; |
| 3124 | |
| 3125 | if (!(test_boost == 1 && cur_ops->can_boost) && test_boost != 2) |
| 3126 | return false; |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 3127 | if (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state) |
Paul E. McKenney | 5e59fba | 2021-01-15 13:30:38 -0800 | [diff] [blame] | 3128 | return false; |
Joel Fernandes (Google) | 4babd85 | 2018-06-19 15:14:18 -0700 | [diff] [blame] | 3129 | |
| 3130 | prio = rcu_get_gp_kthreads_prio(); |
| 3131 | if (!prio) |
| 3132 | return false; |
| 3133 | |
| 3134 | if (prio < 2) { |
Paul E. McKenney | ea6d962 | 2021-03-30 16:30:32 -0700 | [diff] [blame] | 3135 | if (boost_warn_once == 1) |
Joel Fernandes (Google) | 4babd85 | 2018-06-19 15:14:18 -0700 | [diff] [blame] | 3136 | return false; |
| 3137 | |
Joel Fernandes (Google) | bf5b643 | 2018-06-19 15:14:19 -0700 | [diff] [blame] | 3138 | pr_alert("%s: WARN: RCU kthread priority too low to test boosting. Skipping RCU boost test. Try passing rcutree.kthread_prio > 1 on the kernel command line.\n", KBUILD_MODNAME); |
Joel Fernandes (Google) | 4babd85 | 2018-06-19 15:14:18 -0700 | [diff] [blame] | 3139 | boost_warn_once = 1; |
| 3140 | return false; |
| 3141 | } |
| 3142 | |
| 3143 | return true; |
| 3144 | } |
| 3145 | |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 3146 | static bool read_exit_child_stop; |
| 3147 | static bool read_exit_child_stopped; |
| 3148 | static wait_queue_head_t read_exit_wq; |
| 3149 | |
| 3150 | // Child kthread which just does an rcutorture reader and exits. |
| 3151 | static int rcu_torture_read_exit_child(void *trsp_in) |
| 3152 | { |
| 3153 | struct torture_random_state *trsp = trsp_in; |
| 3154 | |
| 3155 | set_user_nice(current, MAX_NICE); |
| 3156 | // Minimize time between reading and exiting. |
| 3157 | while (!kthread_should_stop()) |
| 3158 | schedule_timeout_uninterruptible(1); |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 3159 | (void)rcu_torture_one_read(trsp, -1); |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 3160 | return 0; |
| 3161 | } |
| 3162 | |
| 3163 | // Parent kthread which creates and destroys read-exit child kthreads. |
| 3164 | static int rcu_torture_read_exit(void *unused) |
| 3165 | { |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 3166 | bool errexit = false; |
| 3167 | int i; |
| 3168 | struct task_struct *tsp; |
| 3169 | DEFINE_TORTURE_RANDOM(trs); |
| 3170 | |
| 3171 | // Allocate and initialize. |
| 3172 | set_user_nice(current, MAX_NICE); |
| 3173 | VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of test"); |
| 3174 | |
| 3175 | // Each pass through this loop does one read-exit episode. |
| 3176 | do { |
Paul E. McKenney | d984114 | 2022-04-27 11:46:02 -0700 | [diff] [blame] | 3177 | VERBOSE_TOROUT_STRING("rcu_torture_read_exit: Start of episode"); |
| 3178 | for (i = 0; i < read_exit_burst; i++) { |
| 3179 | if (READ_ONCE(read_exit_child_stop)) |
| 3180 | break; |
| 3181 | stutter_wait("rcu_torture_read_exit"); |
| 3182 | // Spawn child. |
| 3183 | tsp = kthread_run(rcu_torture_read_exit_child, |
| 3184 | &trs, "%s", "rcu_torture_read_exit_child"); |
| 3185 | if (IS_ERR(tsp)) { |
| 3186 | TOROUT_ERRSTRING("out of memory"); |
| 3187 | errexit = true; |
| 3188 | break; |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 3189 | } |
Paul E. McKenney | d984114 | 2022-04-27 11:46:02 -0700 | [diff] [blame] | 3190 | cond_resched(); |
| 3191 | kthread_stop(tsp); |
| 3192 | n_read_exits++; |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 3193 | } |
Paul E. McKenney | d984114 | 2022-04-27 11:46:02 -0700 | [diff] [blame] | 3194 | VERBOSE_TOROUT_STRING("rcu_torture_read_exit: End of episode"); |
| 3195 | rcu_barrier(); // Wait for task_struct free, avoid OOM. |
| 3196 | i = 0; |
| 3197 | for (; !errexit && !READ_ONCE(read_exit_child_stop) && i < read_exit_delay; i++) |
| 3198 | schedule_timeout_uninterruptible(HZ); |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 3199 | } while (!errexit && !READ_ONCE(read_exit_child_stop)); |
| 3200 | |
| 3201 | // Clean up and exit. |
| 3202 | smp_store_release(&read_exit_child_stopped, true); // After reaping. |
| 3203 | smp_mb(); // Store before wakeup. |
| 3204 | wake_up(&read_exit_wq); |
| 3205 | while (!torture_must_stop()) |
| 3206 | schedule_timeout_uninterruptible(1); |
| 3207 | torture_kthread_stopping("rcu_torture_read_exit"); |
| 3208 | return 0; |
| 3209 | } |
| 3210 | |
| 3211 | static int rcu_torture_read_exit_init(void) |
| 3212 | { |
| 3213 | if (read_exit_burst <= 0) |
Paul E. McKenney | fda8486 | 2021-08-03 17:42:25 -0700 | [diff] [blame] | 3214 | return 0; |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 3215 | init_waitqueue_head(&read_exit_wq); |
| 3216 | read_exit_child_stop = false; |
| 3217 | read_exit_child_stopped = false; |
| 3218 | return torture_create_kthread(rcu_torture_read_exit, NULL, |
| 3219 | read_exit_task); |
| 3220 | } |
| 3221 | |
| 3222 | static void rcu_torture_read_exit_cleanup(void) |
| 3223 | { |
| 3224 | if (!read_exit_task) |
| 3225 | return; |
| 3226 | WRITE_ONCE(read_exit_child_stop, true); |
| 3227 | smp_mb(); // Above write before wait. |
| 3228 | wait_event(read_exit_wq, smp_load_acquire(&read_exit_child_stopped)); |
| 3229 | torture_stop_kthread(rcutorture_read_exit, read_exit_task); |
| 3230 | } |
| 3231 | |
Sebastian Andrzej Siewior | 0ffd374 | 2016-08-18 14:57:22 +0200 | [diff] [blame] | 3232 | static enum cpuhp_state rcutor_hp; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 3233 | |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3234 | static void |
| 3235 | rcu_torture_cleanup(void) |
| 3236 | { |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 3237 | int firsttime; |
Paul E. McKenney | 034777d | 2018-04-19 08:43:11 -0700 | [diff] [blame] | 3238 | int flags = 0; |
Paul E. McKenney | aebc826 | 2018-05-01 06:42:51 -0700 | [diff] [blame] | 3239 | unsigned long gp_seq = 0; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3240 | int i; |
| 3241 | |
Davidlohr Bueso | d36a7a0 | 2014-09-11 20:40:21 -0700 | [diff] [blame] | 3242 | if (torture_cleanup_begin()) { |
Paul E. McKenney | 6f81bd6 | 2021-12-06 15:12:14 -0800 | [diff] [blame] | 3243 | if (cur_ops->cb_barrier != NULL) { |
| 3244 | pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); |
Paul E. McKenney | 343e909 | 2008-12-15 16:13:07 -0800 | [diff] [blame] | 3245 | cur_ops->cb_barrier(); |
Paul E. McKenney | 6f81bd6 | 2021-12-06 15:12:14 -0800 | [diff] [blame] | 3246 | } |
Paul E. McKenney | 99d6a2a | 2022-02-04 12:45:18 -0800 | [diff] [blame] | 3247 | rcu_gp_slow_unregister(NULL); |
Paul E. McKenney | 343e909 | 2008-12-15 16:13:07 -0800 | [diff] [blame] | 3248 | return; |
| 3249 | } |
Paul E. McKenney | b813afa | 2019-03-21 09:27:28 -0700 | [diff] [blame] | 3250 | if (!cur_ops) { |
| 3251 | torture_cleanup_end(); |
Paul E. McKenney | 99d6a2a | 2022-02-04 12:45:18 -0800 | [diff] [blame] | 3252 | rcu_gp_slow_unregister(NULL); |
Paul E. McKenney | b813afa | 2019-03-21 09:27:28 -0700 | [diff] [blame] | 3253 | return; |
| 3254 | } |
Paul E. McKenney | 3808dc9 | 2014-01-28 15:29:21 -0800 | [diff] [blame] | 3255 | |
Paul E. McKenney | 27c0f14 | 2020-09-15 17:08:03 -0700 | [diff] [blame] | 3256 | if (cur_ops->gp_kthread_dbg) |
| 3257 | cur_ops->gp_kthread_dbg(); |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 3258 | rcu_torture_read_exit_cleanup(); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3259 | rcu_torture_barrier_cleanup(); |
Paul E. McKenney | c8fa637 | 2020-07-19 14:40:31 -0700 | [diff] [blame] | 3260 | rcu_torture_fwd_prog_cleanup(); |
Paul E. McKenney | 9c029b8 | 2014-02-04 11:47:08 -0800 | [diff] [blame] | 3261 | torture_stop_kthread(rcu_torture_stall, stall_task); |
Paul E. McKenney | 9c029b8 | 2014-02-04 11:47:08 -0800 | [diff] [blame] | 3262 | torture_stop_kthread(rcu_torture_writer, writer_task); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3263 | |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 3264 | if (nocb_tasks) { |
| 3265 | for (i = 0; i < nrealnocbers; i++) |
| 3266 | torture_stop_kthread(rcu_nocb_toggle, nocb_tasks[i]); |
| 3267 | kfree(nocb_tasks); |
| 3268 | nocb_tasks = NULL; |
| 3269 | } |
| 3270 | |
Josh Triplett | c8e5b16 | 2007-05-08 00:33:20 -0700 | [diff] [blame] | 3271 | if (reader_tasks) { |
Paul E. McKenney | 9c029b8 | 2014-02-04 11:47:08 -0800 | [diff] [blame] | 3272 | for (i = 0; i < nrealreaders; i++) |
| 3273 | torture_stop_kthread(rcu_torture_reader, |
| 3274 | reader_tasks[i]); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3275 | kfree(reader_tasks); |
Paul E. McKenney | 293b93d | 2020-09-23 16:46:36 -0700 | [diff] [blame] | 3276 | reader_tasks = NULL; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3277 | } |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 3278 | kfree(rcu_torture_reader_mbchk); |
| 3279 | rcu_torture_reader_mbchk = NULL; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3280 | |
Josh Triplett | c8e5b16 | 2007-05-08 00:33:20 -0700 | [diff] [blame] | 3281 | if (fakewriter_tasks) { |
Paul E. McKenney | 293b93d | 2020-09-23 16:46:36 -0700 | [diff] [blame] | 3282 | for (i = 0; i < nfakewriters; i++) |
Paul E. McKenney | 9c029b8 | 2014-02-04 11:47:08 -0800 | [diff] [blame] | 3283 | torture_stop_kthread(rcu_torture_fakewriter, |
| 3284 | fakewriter_tasks[i]); |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 3285 | kfree(fakewriter_tasks); |
| 3286 | fakewriter_tasks = NULL; |
| 3287 | } |
| 3288 | |
Paul E. McKenney | aebc826 | 2018-05-01 06:42:51 -0700 | [diff] [blame] | 3289 | rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq); |
| 3290 | srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq); |
Joel Fernandes (Google) | 959954d | 2020-06-18 16:29:55 -0400 | [diff] [blame] | 3291 | pr_alert("%s: End-test grace-period state: g%ld f%#x total-gps=%ld\n", |
| 3292 | cur_ops->name, (long)gp_seq, flags, |
| 3293 | rcutorture_seq_diff(gp_seq, start_gp_seq)); |
Paul E. McKenney | 9c029b8 | 2014-02-04 11:47:08 -0800 | [diff] [blame] | 3294 | torture_stop_kthread(rcu_torture_stats, stats_task); |
| 3295 | torture_stop_kthread(rcu_torture_fqs, fqs_task); |
Paul E. McKenney | fd13fe1 | 2021-08-06 08:57:26 -0700 | [diff] [blame] | 3296 | if (rcu_torture_can_boost() && rcutor_hp >= 0) |
Sebastian Andrzej Siewior | 0ffd374 | 2016-08-18 14:57:22 +0200 | [diff] [blame] | 3297 | cpuhp_remove_state(rcutor_hp); |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 3298 | |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 3299 | /* |
Paul E. McKenney | 62a1a94 | 2018-07-07 18:12:26 -0700 | [diff] [blame] | 3300 | * Wait for all RCU callbacks to fire, then do torture-type-specific |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 3301 | * cleanup operations. |
| 3302 | */ |
Paul E. McKenney | 6f81bd6 | 2021-12-06 15:12:14 -0800 | [diff] [blame] | 3303 | if (cur_ops->cb_barrier != NULL) { |
| 3304 | pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); |
Paul E. McKenney | 2326974 | 2008-05-12 21:21:05 +0200 | [diff] [blame] | 3305 | cur_ops->cb_barrier(); |
Paul E. McKenney | 6f81bd6 | 2021-12-06 15:12:14 -0800 | [diff] [blame] | 3306 | } |
Paul E. McKenney | ca1d51e | 2015-04-14 12:28:22 -0700 | [diff] [blame] | 3307 | if (cur_ops->cleanup != NULL) |
| 3308 | cur_ops->cleanup(); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3309 | |
Paul E. McKenney | 7ab2bd3 | 2021-05-02 19:56:05 -0700 | [diff] [blame] | 3310 | rcu_torture_mem_dump_obj(); |
| 3311 | |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3312 | rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 3313 | |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 3314 | if (err_segs_recorded) { |
| 3315 | pr_alert("Failure/close-call rcutorture reader segments:\n"); |
| 3316 | if (rt_read_nsegs == 0) |
| 3317 | pr_alert("\t: No segments recorded!!!\n"); |
| 3318 | firsttime = 1; |
| 3319 | for (i = 0; i < rt_read_nsegs; i++) { |
| 3320 | pr_alert("\t%d: %#x ", i, err_segs[i].rt_readstate); |
| 3321 | if (err_segs[i].rt_delay_jiffies != 0) { |
| 3322 | pr_cont("%s%ldjiffies", firsttime ? "" : "+", |
| 3323 | err_segs[i].rt_delay_jiffies); |
| 3324 | firsttime = 0; |
| 3325 | } |
| 3326 | if (err_segs[i].rt_delay_ms != 0) { |
| 3327 | pr_cont("%s%ldms", firsttime ? "" : "+", |
| 3328 | err_segs[i].rt_delay_ms); |
| 3329 | firsttime = 0; |
| 3330 | } |
| 3331 | if (err_segs[i].rt_delay_us != 0) { |
| 3332 | pr_cont("%s%ldus", firsttime ? "" : "+", |
| 3333 | err_segs[i].rt_delay_us); |
| 3334 | firsttime = 0; |
| 3335 | } |
| 3336 | pr_cont("%s\n", |
| 3337 | err_segs[i].rt_preempted ? "preempted" : ""); |
| 3338 | |
| 3339 | } |
| 3340 | } |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3341 | if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 3342 | rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); |
Paul E. McKenney | 2e9e808 | 2014-01-28 15:58:22 -0800 | [diff] [blame] | 3343 | else if (torture_onoff_failures()) |
Paul E. McKenney | 091541b | 2012-01-10 12:51:14 -0800 | [diff] [blame] | 3344 | rcu_torture_print_module_parms(cur_ops, |
| 3345 | "End of test: RCU_HOTPLUG"); |
Paul E. McKenney | 95c3832 | 2006-03-24 03:15:58 -0800 | [diff] [blame] | 3346 | else |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 3347 | rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); |
Davidlohr Bueso | d36a7a0 | 2014-09-11 20:40:21 -0700 | [diff] [blame] | 3348 | torture_cleanup_end(); |
Paul E. McKenney | 99d6a2a | 2022-02-04 12:45:18 -0800 | [diff] [blame] | 3349 | rcu_gp_slow_unregister(&rcu_fwd_cb_nodelay); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3350 | } |
| 3351 | |
Paul E. McKenney | d2818df | 2013-04-23 17:05:42 -0700 | [diff] [blame] | 3352 | #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD |
| 3353 | static void rcu_torture_leak_cb(struct rcu_head *rhp) |
| 3354 | { |
| 3355 | } |
| 3356 | |
| 3357 | static void rcu_torture_err_cb(struct rcu_head *rhp) |
| 3358 | { |
| 3359 | /* |
| 3360 | * This -might- happen due to race conditions, but is unlikely. |
| 3361 | * The scenario that leads to this happening is that the |
| 3362 | * first of the pair of duplicate callbacks is queued, |
| 3363 | * someone else starts a grace period that includes that |
| 3364 | * callback, then the second of the pair must wait for the |
| 3365 | * next grace period. Unlikely, but can happen. If it |
| 3366 | * does happen, the debug-objects subsystem won't have splatted. |
| 3367 | */ |
Paul E. McKenney | e0d31a34c | 2017-12-01 15:22:38 -0800 | [diff] [blame] | 3368 | pr_alert("%s: duplicated callback was invoked.\n", KBUILD_MODNAME); |
Paul E. McKenney | d2818df | 2013-04-23 17:05:42 -0700 | [diff] [blame] | 3369 | } |
| 3370 | #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ |
| 3371 | |
| 3372 | /* |
| 3373 | * Verify that double-free causes debug-objects to complain, but only |
| 3374 | * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test |
| 3375 | * cannot be carried out. |
| 3376 | */ |
| 3377 | static void rcu_test_debug_objects(void) |
| 3378 | { |
| 3379 | #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD |
| 3380 | struct rcu_head rh1; |
| 3381 | struct rcu_head rh2; |
Paul E. McKenney | edf7b84 | 2020-12-02 17:52:07 -0800 | [diff] [blame] | 3382 | struct rcu_head *rhp = kmalloc(sizeof(*rhp), GFP_KERNEL); |
Paul E. McKenney | d2818df | 2013-04-23 17:05:42 -0700 | [diff] [blame] | 3383 | |
| 3384 | init_rcu_head_on_stack(&rh1); |
| 3385 | init_rcu_head_on_stack(&rh2); |
Paul E. McKenney | e0d31a34c | 2017-12-01 15:22:38 -0800 | [diff] [blame] | 3386 | pr_alert("%s: WARN: Duplicate call_rcu() test starting.\n", KBUILD_MODNAME); |
Paul E. McKenney | d2818df | 2013-04-23 17:05:42 -0700 | [diff] [blame] | 3387 | |
| 3388 | /* Try to queue the rh2 pair of callbacks for the same grace period. */ |
| 3389 | preempt_disable(); /* Prevent preemption from interrupting test. */ |
| 3390 | rcu_read_lock(); /* Make it impossible to finish a grace period. */ |
| 3391 | call_rcu(&rh1, rcu_torture_leak_cb); /* Start grace period. */ |
| 3392 | local_irq_disable(); /* Make it harder to start a new grace period. */ |
| 3393 | call_rcu(&rh2, rcu_torture_leak_cb); |
| 3394 | call_rcu(&rh2, rcu_torture_err_cb); /* Duplicate callback. */ |
Paul E. McKenney | edf7b84 | 2020-12-02 17:52:07 -0800 | [diff] [blame] | 3395 | if (rhp) { |
| 3396 | call_rcu(rhp, rcu_torture_leak_cb); |
| 3397 | call_rcu(rhp, rcu_torture_err_cb); /* Another duplicate callback. */ |
| 3398 | } |
Paul E. McKenney | d2818df | 2013-04-23 17:05:42 -0700 | [diff] [blame] | 3399 | local_irq_enable(); |
| 3400 | rcu_read_unlock(); |
| 3401 | preempt_enable(); |
| 3402 | |
| 3403 | /* Wait for them all to get done so we can safely return. */ |
| 3404 | rcu_barrier(); |
Paul E. McKenney | e0d31a34c | 2017-12-01 15:22:38 -0800 | [diff] [blame] | 3405 | pr_alert("%s: WARN: Duplicate call_rcu() test complete.\n", KBUILD_MODNAME); |
Paul E. McKenney | d2818df | 2013-04-23 17:05:42 -0700 | [diff] [blame] | 3406 | destroy_rcu_head_on_stack(&rh1); |
| 3407 | destroy_rcu_head_on_stack(&rh2); |
Zqiang | 98ea203 | 2022-04-27 15:15:20 +0800 | [diff] [blame] | 3408 | kfree(rhp); |
Paul E. McKenney | d2818df | 2013-04-23 17:05:42 -0700 | [diff] [blame] | 3409 | #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ |
Paul E. McKenney | e0d31a34c | 2017-12-01 15:22:38 -0800 | [diff] [blame] | 3410 | pr_alert("%s: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n", KBUILD_MODNAME); |
Paul E. McKenney | d2818df | 2013-04-23 17:05:42 -0700 | [diff] [blame] | 3411 | #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ |
| 3412 | } |
| 3413 | |
Paul E. McKenney | 3a6cb58 | 2018-12-10 09:44:52 -0800 | [diff] [blame] | 3414 | static void rcutorture_sync(void) |
| 3415 | { |
| 3416 | static unsigned long n; |
| 3417 | |
| 3418 | if (cur_ops->sync && !(++n & 0xfff)) |
| 3419 | cur_ops->sync(); |
| 3420 | } |
| 3421 | |
Josh Triplett | 6f8bc500 | 2007-05-08 00:25:24 -0700 | [diff] [blame] | 3422 | static int __init |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3423 | rcu_torture_init(void) |
| 3424 | { |
Paul E. McKenney | c04dd09 | 2018-07-23 14:16:47 -0700 | [diff] [blame] | 3425 | long i; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3426 | int cpu; |
| 3427 | int firsterr = 0; |
Joel Fernandes (Google) | 959954d | 2020-06-18 16:29:55 -0400 | [diff] [blame] | 3428 | int flags = 0; |
| 3429 | unsigned long gp_seq = 0; |
Paul E. McKenney | 2ec1f2d9 | 2013-06-12 15:12:21 -0700 | [diff] [blame] | 3430 | static struct rcu_torture_ops *torture_ops[] = { |
Paul E. McKenney | 40c1278 | 2022-03-17 13:29:59 -0700 | [diff] [blame] | 3431 | &rcu_ops, &rcu_busted_ops, &srcu_ops, &srcud_ops, &busted_srcud_ops, |
Paul E. McKenney | 4c3f7b0 | 2022-03-17 16:16:45 -0700 | [diff] [blame] | 3432 | TASKS_OPS TASKS_RUDE_OPS TASKS_TRACING_OPS |
Paul E. McKenney | 40c1278 | 2022-03-17 13:29:59 -0700 | [diff] [blame] | 3433 | &trivial_ops, |
Paul E. McKenney | 2ec1f2d9 | 2013-06-12 15:12:21 -0700 | [diff] [blame] | 3434 | }; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3435 | |
Paul E. McKenney | a2f2577 | 2017-11-21 20:19:17 -0800 | [diff] [blame] | 3436 | if (!torture_init_begin(torture_type, verbose)) |
Paul E. McKenney | 5228084 | 2014-04-07 09:14:11 -0700 | [diff] [blame] | 3437 | return -EBUSY; |
Paul E. McKenney | 343e909 | 2008-12-15 16:13:07 -0800 | [diff] [blame] | 3438 | |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3439 | /* Process args and tell the world that the torturer is on the job. */ |
Josh Triplett | ade5fb8 | 2007-05-08 00:33:22 -0700 | [diff] [blame] | 3440 | for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 3441 | cur_ops = torture_ops[i]; |
Josh Triplett | ade5fb8 | 2007-05-08 00:33:22 -0700 | [diff] [blame] | 3442 | if (strcmp(torture_type, cur_ops->name) == 0) |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 3443 | break; |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 3444 | } |
Josh Triplett | ade5fb8 | 2007-05-08 00:33:22 -0700 | [diff] [blame] | 3445 | if (i == ARRAY_SIZE(torture_ops)) { |
Paul E. McKenney | 2caa1e4 | 2012-08-09 16:30:45 -0700 | [diff] [blame] | 3446 | pr_alert("rcu-torture: invalid torture type: \"%s\"\n", |
| 3447 | torture_type); |
| 3448 | pr_alert("rcu-torture types:"); |
Paul E. McKenney | cf886c4 | 2009-10-25 19:03:54 -0700 | [diff] [blame] | 3449 | for (i = 0; i < ARRAY_SIZE(torture_ops); i++) |
Joe Perches | a753835 | 2018-05-14 13:27:33 -0700 | [diff] [blame] | 3450 | pr_cont(" %s", torture_ops[i]->name); |
| 3451 | pr_cont("\n"); |
Paul E. McKenney | 889d487 | 2015-08-24 11:37:58 -0700 | [diff] [blame] | 3452 | firsterr = -EINVAL; |
Paul E. McKenney | b813afa | 2019-03-21 09:27:28 -0700 | [diff] [blame] | 3453 | cur_ops = NULL; |
Paul E. McKenney | 889d487 | 2015-08-24 11:37:58 -0700 | [diff] [blame] | 3454 | goto unwind; |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 3455 | } |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 3456 | if (cur_ops->fqs == NULL && fqs_duration != 0) { |
Paul E. McKenney | 2caa1e4 | 2012-08-09 16:30:45 -0700 | [diff] [blame] | 3457 | pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 3458 | fqs_duration = 0; |
| 3459 | } |
Josh Triplett | c8e5b16 | 2007-05-08 00:33:20 -0700 | [diff] [blame] | 3460 | if (cur_ops->init) |
Paul E. McKenney | 889d487 | 2015-08-24 11:37:58 -0700 | [diff] [blame] | 3461 | cur_ops->init(); |
Paul E. McKenney | 72e9bb5 | 2006-06-27 02:54:03 -0700 | [diff] [blame] | 3462 | |
Paul E. McKenney | 64e4b43 | 2014-03-12 10:26:35 -0700 | [diff] [blame] | 3463 | if (nreaders >= 0) { |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3464 | nrealreaders = nreaders; |
Paul E. McKenney | 64e4b43 | 2014-03-12 10:26:35 -0700 | [diff] [blame] | 3465 | } else { |
Paul E. McKenney | 3838cc1 | 2015-03-12 13:55:48 -0700 | [diff] [blame] | 3466 | nrealreaders = num_online_cpus() - 2 - nreaders; |
Paul E. McKenney | 64e4b43 | 2014-03-12 10:26:35 -0700 | [diff] [blame] | 3467 | if (nrealreaders <= 0) |
| 3468 | nrealreaders = 1; |
| 3469 | } |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 3470 | rcu_torture_print_module_parms(cur_ops, "Start of test"); |
Joel Fernandes (Google) | 959954d | 2020-06-18 16:29:55 -0400 | [diff] [blame] | 3471 | rcutorture_get_gp_data(cur_ops->ttype, &flags, &gp_seq); |
| 3472 | srcutorture_get_gp_data(cur_ops->ttype, srcu_ctlp, &flags, &gp_seq); |
| 3473 | start_gp_seq = gp_seq; |
| 3474 | pr_alert("%s: Start-test grace-period state: g%ld f%#x\n", |
| 3475 | cur_ops->name, (long)gp_seq, flags); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3476 | |
| 3477 | /* Set up the freelist. */ |
| 3478 | |
| 3479 | INIT_LIST_HEAD(&rcu_torture_freelist); |
Ahmed S. Darwish | 788e770 | 2007-05-08 00:33:14 -0700 | [diff] [blame] | 3480 | for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) { |
Paul E. McKenney | 996417d | 2005-11-18 01:10:50 -0800 | [diff] [blame] | 3481 | rcu_tortures[i].rtort_mbtest = 0; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3482 | list_add_tail(&rcu_tortures[i].rtort_free, |
| 3483 | &rcu_torture_freelist); |
| 3484 | } |
| 3485 | |
| 3486 | /* Initialize the statistics so that each run gets its own numbers. */ |
| 3487 | |
| 3488 | rcu_torture_current = NULL; |
| 3489 | rcu_torture_current_version = 0; |
| 3490 | atomic_set(&n_rcu_torture_alloc, 0); |
| 3491 | atomic_set(&n_rcu_torture_alloc_fail, 0); |
| 3492 | atomic_set(&n_rcu_torture_free, 0); |
Paul E. McKenney | 996417d | 2005-11-18 01:10:50 -0800 | [diff] [blame] | 3493 | atomic_set(&n_rcu_torture_mberror, 0); |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 3494 | atomic_set(&n_rcu_torture_mbchk_fail, 0); |
| 3495 | atomic_set(&n_rcu_torture_mbchk_tries, 0); |
Paul E. McKenney | 996417d | 2005-11-18 01:10:50 -0800 | [diff] [blame] | 3496 | atomic_set(&n_rcu_torture_error, 0); |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3497 | n_rcu_torture_barrier_error = 0; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 3498 | n_rcu_torture_boost_ktrerror = 0; |
| 3499 | n_rcu_torture_boost_rterror = 0; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 3500 | n_rcu_torture_boost_failure = 0; |
| 3501 | n_rcu_torture_boosts = 0; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3502 | for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) |
| 3503 | atomic_set(&rcu_torture_wcount[i], 0); |
KAMEZAWA Hiroyuki | 0a94502 | 2006-03-28 01:56:37 -0800 | [diff] [blame] | 3504 | for_each_possible_cpu(cpu) { |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3505 | for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) { |
| 3506 | per_cpu(rcu_torture_count, cpu)[i] = 0; |
| 3507 | per_cpu(rcu_torture_batch, cpu)[i] = 0; |
| 3508 | } |
| 3509 | } |
Paul E. McKenney | c116dba | 2018-07-13 12:09:14 -0700 | [diff] [blame] | 3510 | err_segs_recorded = 0; |
| 3511 | rt_read_nsegs = 0; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3512 | |
| 3513 | /* Start up the kthreads. */ |
| 3514 | |
Paul E. McKenney | 18fbf30 | 2020-11-16 16:46:06 -0800 | [diff] [blame] | 3515 | rcu_torture_write_types(); |
Paul E. McKenney | 47cf29b | 2014-02-03 11:52:27 -0800 | [diff] [blame] | 3516 | firsterr = torture_create_kthread(rcu_torture_writer, NULL, |
| 3517 | writer_task); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3518 | if (torture_init_error(firsterr)) |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3519 | goto unwind; |
Paul E. McKenney | 4444d85 | 2015-05-14 15:42:40 -0700 | [diff] [blame] | 3520 | if (nfakewriters > 0) { |
Paul E. McKenney | 68a675d | 2017-12-01 14:26:56 -0800 | [diff] [blame] | 3521 | fakewriter_tasks = kcalloc(nfakewriters, |
Paul E. McKenney | 4444d85 | 2015-05-14 15:42:40 -0700 | [diff] [blame] | 3522 | sizeof(fakewriter_tasks[0]), |
| 3523 | GFP_KERNEL); |
| 3524 | if (fakewriter_tasks == NULL) { |
Li Zhijian | 81faa4f | 2021-11-03 16:30:28 +0800 | [diff] [blame] | 3525 | TOROUT_ERRSTRING("out of memory"); |
Paul E. McKenney | 4444d85 | 2015-05-14 15:42:40 -0700 | [diff] [blame] | 3526 | firsterr = -ENOMEM; |
| 3527 | goto unwind; |
| 3528 | } |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 3529 | } |
| 3530 | for (i = 0; i < nfakewriters; i++) { |
Paul E. McKenney | 47cf29b | 2014-02-03 11:52:27 -0800 | [diff] [blame] | 3531 | firsterr = torture_create_kthread(rcu_torture_fakewriter, |
| 3532 | NULL, fakewriter_tasks[i]); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3533 | if (torture_init_error(firsterr)) |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 3534 | goto unwind; |
Josh Triplett | b772e1d | 2006-10-04 02:17:13 -0700 | [diff] [blame] | 3535 | } |
Paul E. McKenney | 68a675d | 2017-12-01 14:26:56 -0800 | [diff] [blame] | 3536 | reader_tasks = kcalloc(nrealreaders, sizeof(reader_tasks[0]), |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3537 | GFP_KERNEL); |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 3538 | rcu_torture_reader_mbchk = kcalloc(nrealreaders, sizeof(*rcu_torture_reader_mbchk), |
| 3539 | GFP_KERNEL); |
| 3540 | if (!reader_tasks || !rcu_torture_reader_mbchk) { |
Li Zhijian | 81faa4f | 2021-11-03 16:30:28 +0800 | [diff] [blame] | 3541 | TOROUT_ERRSTRING("out of memory"); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3542 | firsterr = -ENOMEM; |
| 3543 | goto unwind; |
| 3544 | } |
| 3545 | for (i = 0; i < nrealreaders; i++) { |
Paul E. McKenney | 0050453 | 2020-10-29 15:08:57 -0700 | [diff] [blame] | 3546 | rcu_torture_reader_mbchk[i].rtc_chkrdr = -1; |
Paul E. McKenney | c04dd09 | 2018-07-23 14:16:47 -0700 | [diff] [blame] | 3547 | firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, |
Paul E. McKenney | 47cf29b | 2014-02-03 11:52:27 -0800 | [diff] [blame] | 3548 | reader_tasks[i]); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3549 | if (torture_init_error(firsterr)) |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3550 | goto unwind; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3551 | } |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 3552 | nrealnocbers = nocbs_nthreads; |
| 3553 | if (WARN_ON(nrealnocbers < 0)) |
| 3554 | nrealnocbers = 1; |
| 3555 | if (WARN_ON(nocbs_toggle < 0)) |
| 3556 | nocbs_toggle = HZ; |
| 3557 | if (nrealnocbers > 0) { |
| 3558 | nocb_tasks = kcalloc(nrealnocbers, sizeof(nocb_tasks[0]), GFP_KERNEL); |
| 3559 | if (nocb_tasks == NULL) { |
Li Zhijian | 81faa4f | 2021-11-03 16:30:28 +0800 | [diff] [blame] | 3560 | TOROUT_ERRSTRING("out of memory"); |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 3561 | firsterr = -ENOMEM; |
| 3562 | goto unwind; |
| 3563 | } |
| 3564 | } else { |
| 3565 | nocb_tasks = NULL; |
| 3566 | } |
| 3567 | for (i = 0; i < nrealnocbers; i++) { |
| 3568 | firsterr = torture_create_kthread(rcu_nocb_toggle, NULL, nocb_tasks[i]); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3569 | if (torture_init_error(firsterr)) |
Paul E. McKenney | 2c4319b | 2020-09-23 17:39:46 -0700 | [diff] [blame] | 3570 | goto unwind; |
| 3571 | } |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3572 | if (stat_interval > 0) { |
Paul E. McKenney | 47cf29b | 2014-02-03 11:52:27 -0800 | [diff] [blame] | 3573 | firsterr = torture_create_kthread(rcu_torture_stats, NULL, |
| 3574 | stats_task); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3575 | if (torture_init_error(firsterr)) |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3576 | goto unwind; |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3577 | } |
Paul E. McKenney | e8e255f | 2015-05-14 16:55:45 -0700 | [diff] [blame] | 3578 | if (test_no_idle_hz && shuffle_interval > 0) { |
Paul E. McKenney | 3808dc9 | 2014-01-28 15:29:21 -0800 | [diff] [blame] | 3579 | firsterr = torture_shuffle_init(shuffle_interval * HZ); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3580 | if (torture_init_error(firsterr)) |
Rusty Russell | 73d0a4b | 2009-03-30 22:05:16 -0600 | [diff] [blame] | 3581 | goto unwind; |
Srivatsa Vaddagiri | d84f520 | 2006-01-08 01:03:42 -0800 | [diff] [blame] | 3582 | } |
Paul E. McKenney | d120f65 | 2008-06-18 05:21:44 -0700 | [diff] [blame] | 3583 | if (stutter < 0) |
| 3584 | stutter = 0; |
| 3585 | if (stutter) { |
Paul E. McKenney | ff3bf92 | 2019-04-09 14:44:49 -0700 | [diff] [blame] | 3586 | int t; |
| 3587 | |
| 3588 | t = cur_ops->stall_dur ? cur_ops->stall_dur() : stutter * HZ; |
| 3589 | firsterr = torture_stutter_init(stutter * HZ, t); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3590 | if (torture_init_error(firsterr)) |
Paul E. McKenney | d120f65 | 2008-06-18 05:21:44 -0700 | [diff] [blame] | 3591 | goto unwind; |
Paul E. McKenney | d120f65 | 2008-06-18 05:21:44 -0700 | [diff] [blame] | 3592 | } |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 3593 | if (fqs_duration < 0) |
| 3594 | fqs_duration = 0; |
| 3595 | if (fqs_duration) { |
Paul E. McKenney | 628edaa | 2014-01-31 11:57:43 -0800 | [diff] [blame] | 3596 | /* Create the fqs thread */ |
Paul E. McKenney | d0d0606 | 2014-03-17 20:56:45 -0700 | [diff] [blame] | 3597 | firsterr = torture_create_kthread(rcu_torture_fqs, NULL, |
| 3598 | fqs_task); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3599 | if (torture_init_error(firsterr)) |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 3600 | goto unwind; |
Paul E. McKenney | bf66f18 | 2010-01-04 15:09:10 -0800 | [diff] [blame] | 3601 | } |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 3602 | if (test_boost_interval < 1) |
| 3603 | test_boost_interval = 1; |
| 3604 | if (test_boost_duration < 2) |
| 3605 | test_boost_duration = 2; |
Joel Fernandes (Google) | 4babd85 | 2018-06-19 15:14:18 -0700 | [diff] [blame] | 3606 | if (rcu_torture_can_boost()) { |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 3607 | |
| 3608 | boost_starttime = jiffies + test_boost_interval * HZ; |
Sebastian Andrzej Siewior | 0ffd374 | 2016-08-18 14:57:22 +0200 | [diff] [blame] | 3609 | |
| 3610 | firsterr = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "RCU_TORTURE", |
| 3611 | rcutorture_booster_init, |
| 3612 | rcutorture_booster_cleanup); |
Sebastian Andrzej Siewior | 0ffd374 | 2016-08-18 14:57:22 +0200 | [diff] [blame] | 3613 | rcutor_hp = firsterr; |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3614 | if (torture_init_error(firsterr)) |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3615 | goto unwind; |
Paul E. McKenney | 8e8be45 | 2010-09-02 16:16:14 -0700 | [diff] [blame] | 3616 | } |
Paul E. McKenney | 60013d5 | 2019-07-10 08:30:00 -0700 | [diff] [blame] | 3617 | shutdown_jiffies = jiffies + shutdown_secs * HZ; |
Paul E. McKenney | 01025eb | 2014-01-31 15:15:02 -0800 | [diff] [blame] | 3618 | firsterr = torture_shutdown_init(shutdown_secs, rcu_torture_cleanup); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3619 | if (torture_init_error(firsterr)) |
Paul E. McKenney | e991dbc | 2014-01-31 14:52:13 -0800 | [diff] [blame] | 3620 | goto unwind; |
Paul E. McKenney | 3a6cb58 | 2018-12-10 09:44:52 -0800 | [diff] [blame] | 3621 | firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, |
| 3622 | rcutorture_sync); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3623 | if (torture_init_error(firsterr)) |
Paul E. McKenney | 37e377d | 2012-02-17 22:12:18 -0800 | [diff] [blame] | 3624 | goto unwind; |
Paul E. McKenney | 01025eb | 2014-01-31 15:15:02 -0800 | [diff] [blame] | 3625 | firsterr = rcu_torture_stall_init(); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3626 | if (torture_init_error(firsterr)) |
Paul E. McKenney | 37e377d | 2012-02-17 22:12:18 -0800 | [diff] [blame] | 3627 | goto unwind; |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 3628 | firsterr = rcu_torture_fwd_prog_init(); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3629 | if (torture_init_error(firsterr)) |
Paul E. McKenney | 1b27291 | 2018-07-18 14:32:31 -0700 | [diff] [blame] | 3630 | goto unwind; |
Paul E. McKenney | 01025eb | 2014-01-31 15:15:02 -0800 | [diff] [blame] | 3631 | firsterr = rcu_torture_barrier_init(); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3632 | if (torture_init_error(firsterr)) |
Paul E. McKenney | fae4b54 | 2012-02-20 17:51:45 -0800 | [diff] [blame] | 3633 | goto unwind; |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 3634 | firsterr = rcu_torture_read_exit_init(); |
Paul E. McKenney | efeff6b | 2021-08-05 13:28:24 -0700 | [diff] [blame] | 3635 | if (torture_init_error(firsterr)) |
Paul E. McKenney | 4a5f133 | 2020-04-24 11:21:40 -0700 | [diff] [blame] | 3636 | goto unwind; |
Paul E. McKenney | d2818df | 2013-04-23 17:05:42 -0700 | [diff] [blame] | 3637 | if (object_debug) |
| 3638 | rcu_test_debug_objects(); |
Paul E. McKenney | b5daa8f | 2014-01-30 13:38:09 -0800 | [diff] [blame] | 3639 | torture_init_end(); |
Paul E. McKenney | 99d6a2a | 2022-02-04 12:45:18 -0800 | [diff] [blame] | 3640 | rcu_gp_slow_register(&rcu_fwd_cb_nodelay); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3641 | return 0; |
| 3642 | |
| 3643 | unwind: |
Paul E. McKenney | b5daa8f | 2014-01-30 13:38:09 -0800 | [diff] [blame] | 3644 | torture_init_end(); |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3645 | rcu_torture_cleanup(); |
Paul E. McKenney | 4994684 | 2020-09-18 13:30:33 -0700 | [diff] [blame] | 3646 | if (shutdown_secs) { |
| 3647 | WARN_ON(!IS_MODULE(CONFIG_RCU_TORTURE_TEST)); |
| 3648 | kernel_power_off(); |
| 3649 | } |
Paul E. McKenney | a241ec6 | 2005-10-30 15:03:12 -0800 | [diff] [blame] | 3650 | return firsterr; |
| 3651 | } |
| 3652 | |
| 3653 | module_init(rcu_torture_init); |
| 3654 | module_exit(rcu_torture_cleanup); |