Thomas Gleixner | 457c899 | 2019-05-19 13:08:55 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Joe Perches | a818e52 | 2020-06-04 16:50:20 -0700 | [diff] [blame] | 2 | #define pr_fmt(fmt) "%s: " fmt, __func__ |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 3 | |
| 4 | #include <linux/kernel.h> |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 5 | #include <linux/sched.h> |
| 6 | #include <linux/wait.h> |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 7 | #include <linux/slab.h> |
Paul E. McKenney | 3375efe | 2020-12-08 14:43:43 -0800 | [diff] [blame] | 8 | #include <linux/mm.h> |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 9 | #include <linux/percpu-refcount.h> |
| 10 | |
| 11 | /* |
| 12 | * Initially, a percpu refcount is just a set of percpu counters. Initially, we |
| 13 | * don't try to detect the ref hitting 0 - which means that get/put can just |
| 14 | * increment or decrement the local counter. Note that the counter on a |
| 15 | * particular cpu can (and will) wrap - this is fine, when we go to shutdown the |
| 16 | * percpu counters will all sum to the correct value |
| 17 | * |
Bogdan Sikora | bdb428c | 2015-12-27 14:58:23 +0100 | [diff] [blame] | 18 | * (More precisely: because modular arithmetic is commutative the sum of all the |
Tejun Heo | eecc16b | 2014-09-24 13:31:48 -0400 | [diff] [blame] | 19 | * percpu_count vars will be equal to what it would have been if all the gets |
| 20 | * and puts were done to a single integer, even if some of the percpu integers |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 21 | * overflow or underflow). |
| 22 | * |
| 23 | * The real trick to implementing percpu refcounts is shutdown. We can't detect |
| 24 | * the ref hitting 0 on every put - this would require global synchronization |
| 25 | * and defeat the whole purpose of using percpu refs. |
| 26 | * |
| 27 | * What we do is require the user to keep track of the initial refcount; we know |
| 28 | * the ref can't hit 0 before the user drops the initial ref, so as long as we |
| 29 | * convert to non percpu mode before the initial ref is dropped everything |
| 30 | * works. |
| 31 | * |
| 32 | * Converting to non percpu mode is done with some RCUish stuff in |
Tejun Heo | e625305 | 2014-09-20 01:27:25 -0400 | [diff] [blame] | 33 | * percpu_ref_kill. Additionally, we need a bias value so that the |
| 34 | * atomic_long_t can't hit 0 before we've added up all the percpu refs. |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 35 | */ |
| 36 | |
Tejun Heo | eecc16b | 2014-09-24 13:31:48 -0400 | [diff] [blame] | 37 | #define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1)) |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 38 | |
Tejun Heo | 33e465c | 2015-09-29 17:47:20 -0400 | [diff] [blame] | 39 | static DEFINE_SPINLOCK(percpu_ref_switch_lock); |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 40 | static DECLARE_WAIT_QUEUE_HEAD(percpu_ref_switch_waitq); |
| 41 | |
Tejun Heo | eecc16b | 2014-09-24 13:31:48 -0400 | [diff] [blame] | 42 | static unsigned long __percpu *percpu_count_ptr(struct percpu_ref *ref) |
Tejun Heo | eae7975d | 2014-06-28 08:10:13 -0400 | [diff] [blame] | 43 | { |
Tejun Heo | eecc16b | 2014-09-24 13:31:48 -0400 | [diff] [blame] | 44 | return (unsigned long __percpu *) |
Tejun Heo | 27344a9 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 45 | (ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC_DEAD); |
Tejun Heo | eae7975d | 2014-06-28 08:10:13 -0400 | [diff] [blame] | 46 | } |
| 47 | |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 48 | /** |
| 49 | * percpu_ref_init - initialize a percpu refcount |
Tejun Heo | ac89906 | 2013-06-12 20:43:06 -0700 | [diff] [blame] | 50 | * @ref: percpu_ref to initialize |
| 51 | * @release: function which will be called when refcount hits 0 |
Tejun Heo | 2aad2a8 | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 52 | * @flags: PERCPU_REF_INIT_* flags |
Tejun Heo | a34375e | 2014-09-08 09:51:30 +0900 | [diff] [blame] | 53 | * @gfp: allocation mask to use |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 54 | * |
Ira Weiny | 15617df | 2020-02-21 15:16:07 -0800 | [diff] [blame] | 55 | * Initializes @ref. @ref starts out in percpu mode with a refcount of 1 unless |
| 56 | * @flags contains PERCPU_REF_INIT_ATOMIC or PERCPU_REF_INIT_DEAD. These flags |
| 57 | * change the start state to atomic with the latter setting the initial refcount |
| 58 | * to 0. See the definitions of PERCPU_REF_INIT_* flags for flag behaviors. |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 59 | * |
| 60 | * Note that @release must not sleep - it may potentially be called from RCU |
| 61 | * callback context by percpu_ref_kill(). |
| 62 | */ |
Tejun Heo | a34375e | 2014-09-08 09:51:30 +0900 | [diff] [blame] | 63 | int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release, |
Tejun Heo | 2aad2a8 | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 64 | unsigned int flags, gfp_t gfp) |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 65 | { |
Tejun Heo | 27344a9 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 66 | size_t align = max_t(size_t, 1 << __PERCPU_REF_FLAG_BITS, |
| 67 | __alignof__(unsigned long)); |
Tejun Heo | 2aad2a8 | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 68 | unsigned long start_count = 0; |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 69 | struct percpu_ref_data *data; |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 70 | |
Tejun Heo | 27344a9 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 71 | ref->percpu_count_ptr = (unsigned long) |
| 72 | __alloc_percpu_gfp(sizeof(unsigned long), align, gfp); |
Tejun Heo | eecc16b | 2014-09-24 13:31:48 -0400 | [diff] [blame] | 73 | if (!ref->percpu_count_ptr) |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 74 | return -ENOMEM; |
| 75 | |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 76 | data = kzalloc(sizeof(*ref->data), gfp); |
| 77 | if (!data) { |
| 78 | free_percpu((void __percpu *)ref->percpu_count_ptr); |
Al Viro | a917143 | 2022-05-18 02:13:40 -0400 | [diff] [blame] | 79 | ref->percpu_count_ptr = 0; |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 80 | return -ENOMEM; |
| 81 | } |
| 82 | |
| 83 | data->force_atomic = flags & PERCPU_REF_INIT_ATOMIC; |
| 84 | data->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT; |
Tejun Heo | 1cae13e | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 85 | |
Roman Gushchin | 7d9ab9b | 2019-05-07 10:01:50 -0700 | [diff] [blame] | 86 | if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD)) { |
Tejun Heo | 2aad2a8 | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 87 | ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC; |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 88 | data->allow_reinit = true; |
Roman Gushchin | 7d9ab9b | 2019-05-07 10:01:50 -0700 | [diff] [blame] | 89 | } else { |
Tejun Heo | 2aad2a8 | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 90 | start_count += PERCPU_COUNT_BIAS; |
Roman Gushchin | 7d9ab9b | 2019-05-07 10:01:50 -0700 | [diff] [blame] | 91 | } |
Tejun Heo | 2aad2a8 | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 92 | |
| 93 | if (flags & PERCPU_REF_INIT_DEAD) |
| 94 | ref->percpu_count_ptr |= __PERCPU_REF_DEAD; |
| 95 | else |
| 96 | start_count++; |
| 97 | |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 98 | atomic_long_set(&data->count, start_count); |
Tejun Heo | 2aad2a8 | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 99 | |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 100 | data->release = release; |
| 101 | data->confirm_switch = NULL; |
| 102 | data->ref = ref; |
| 103 | ref->data = data; |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 104 | return 0; |
| 105 | } |
Matias Bjorling | 5e9dd37 | 2013-10-16 13:47:01 -0700 | [diff] [blame] | 106 | EXPORT_SYMBOL_GPL(percpu_ref_init); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 107 | |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 108 | static void __percpu_ref_exit(struct percpu_ref *ref) |
| 109 | { |
| 110 | unsigned long __percpu *percpu_count = percpu_count_ptr(ref); |
| 111 | |
| 112 | if (percpu_count) { |
| 113 | /* non-NULL confirm_switch indicates switching in progress */ |
Ming Lei | 7ea6bf2 | 2020-10-09 12:03:56 +0800 | [diff] [blame] | 114 | WARN_ON_ONCE(ref->data && ref->data->confirm_switch); |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 115 | free_percpu(percpu_count); |
| 116 | ref->percpu_count_ptr = __PERCPU_REF_ATOMIC_DEAD; |
| 117 | } |
| 118 | } |
| 119 | |
Tejun Heo | bc497bd | 2013-06-12 20:52:35 -0700 | [diff] [blame] | 120 | /** |
Tejun Heo | 9a1049d | 2014-06-28 08:10:14 -0400 | [diff] [blame] | 121 | * percpu_ref_exit - undo percpu_ref_init() |
| 122 | * @ref: percpu_ref to exit |
Tejun Heo | bc497bd | 2013-06-12 20:52:35 -0700 | [diff] [blame] | 123 | * |
Tejun Heo | 9a1049d | 2014-06-28 08:10:14 -0400 | [diff] [blame] | 124 | * This function exits @ref. The caller is responsible for ensuring that |
| 125 | * @ref is no longer in active use. The usual places to invoke this |
| 126 | * function from are the @ref->release() callback or in init failure path |
| 127 | * where percpu_ref_init() succeeded but other parts of the initialization |
| 128 | * of the embedding object failed. |
Tejun Heo | bc497bd | 2013-06-12 20:52:35 -0700 | [diff] [blame] | 129 | */ |
Tejun Heo | 9a1049d | 2014-06-28 08:10:14 -0400 | [diff] [blame] | 130 | void percpu_ref_exit(struct percpu_ref *ref) |
Tejun Heo | bc497bd | 2013-06-12 20:52:35 -0700 | [diff] [blame] | 131 | { |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 132 | struct percpu_ref_data *data = ref->data; |
| 133 | unsigned long flags; |
Tejun Heo | bc497bd | 2013-06-12 20:52:35 -0700 | [diff] [blame] | 134 | |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 135 | __percpu_ref_exit(ref); |
| 136 | |
| 137 | if (!data) |
| 138 | return; |
| 139 | |
| 140 | spin_lock_irqsave(&percpu_ref_switch_lock, flags); |
| 141 | ref->percpu_count_ptr |= atomic_long_read(&ref->data->count) << |
| 142 | __PERCPU_REF_FLAG_BITS; |
| 143 | ref->data = NULL; |
| 144 | spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); |
| 145 | |
| 146 | kfree(data); |
Tejun Heo | bc497bd | 2013-06-12 20:52:35 -0700 | [diff] [blame] | 147 | } |
Tejun Heo | 9a1049d | 2014-06-28 08:10:14 -0400 | [diff] [blame] | 148 | EXPORT_SYMBOL_GPL(percpu_ref_exit); |
Tejun Heo | bc497bd | 2013-06-12 20:52:35 -0700 | [diff] [blame] | 149 | |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 150 | static void percpu_ref_call_confirm_rcu(struct rcu_head *rcu) |
| 151 | { |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 152 | struct percpu_ref_data *data = container_of(rcu, |
| 153 | struct percpu_ref_data, rcu); |
| 154 | struct percpu_ref *ref = data->ref; |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 155 | |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 156 | data->confirm_switch(ref); |
| 157 | data->confirm_switch = NULL; |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 158 | wake_up_all(&percpu_ref_switch_waitq); |
| 159 | |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 160 | if (!data->allow_reinit) |
| 161 | __percpu_ref_exit(ref); |
Roman Gushchin | 7d9ab9b | 2019-05-07 10:01:50 -0700 | [diff] [blame] | 162 | |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 163 | /* drop ref from percpu_ref_switch_to_atomic() */ |
| 164 | percpu_ref_put(ref); |
| 165 | } |
| 166 | |
| 167 | static void percpu_ref_switch_to_atomic_rcu(struct rcu_head *rcu) |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 168 | { |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 169 | struct percpu_ref_data *data = container_of(rcu, |
| 170 | struct percpu_ref_data, rcu); |
| 171 | struct percpu_ref *ref = data->ref; |
Tejun Heo | eecc16b | 2014-09-24 13:31:48 -0400 | [diff] [blame] | 172 | unsigned long __percpu *percpu_count = percpu_count_ptr(ref); |
Paul E. McKenney | 3375efe | 2020-12-08 14:43:43 -0800 | [diff] [blame] | 173 | static atomic_t underflows; |
Tejun Heo | e625305 | 2014-09-20 01:27:25 -0400 | [diff] [blame] | 174 | unsigned long count = 0; |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 175 | int cpu; |
| 176 | |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 177 | for_each_possible_cpu(cpu) |
Tejun Heo | eecc16b | 2014-09-24 13:31:48 -0400 | [diff] [blame] | 178 | count += *per_cpu_ptr(percpu_count, cpu); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 179 | |
Joe Perches | a818e52 | 2020-06-04 16:50:20 -0700 | [diff] [blame] | 180 | pr_debug("global %lu percpu %lu\n", |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 181 | atomic_long_read(&data->count), count); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 182 | |
| 183 | /* |
| 184 | * It's crucial that we sum the percpu counters _before_ adding the sum |
| 185 | * to &ref->count; since gets could be happening on one cpu while puts |
| 186 | * happen on another, adding a single cpu's count could cause |
| 187 | * @ref->count to hit 0 before we've got a consistent value - but the |
| 188 | * sum of all the counts will be consistent and correct. |
| 189 | * |
| 190 | * Subtracting the bias value then has to happen _after_ adding count to |
| 191 | * &ref->count; we need the bias value to prevent &ref->count from |
| 192 | * reaching 0 before we add the percpu counts. But doing it at the same |
| 193 | * time is equivalent and saves us atomic operations: |
| 194 | */ |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 195 | atomic_long_add((long)count - PERCPU_COUNT_BIAS, &data->count); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 196 | |
Paul E. McKenney | 3375efe | 2020-12-08 14:43:43 -0800 | [diff] [blame] | 197 | if (WARN_ONCE(atomic_long_read(&data->count) <= 0, |
| 198 | "percpu ref (%ps) <= 0 (%ld) after switching to atomic", |
| 199 | data->release, atomic_long_read(&data->count)) && |
| 200 | atomic_inc_return(&underflows) < 4) { |
| 201 | pr_err("%s(): percpu_ref underflow", __func__); |
| 202 | mem_dump_obj(data); |
| 203 | } |
Kent Overstreet | 687b0ad | 2014-01-06 13:13:26 -0800 | [diff] [blame] | 204 | |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 205 | /* @ref is viewed as dead on all CPUs, send out switch confirmation */ |
| 206 | percpu_ref_call_confirm_rcu(rcu); |
| 207 | } |
Tejun Heo | dbece3a | 2013-06-13 19:23:53 -0700 | [diff] [blame] | 208 | |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 209 | static void percpu_ref_noop_confirm_switch(struct percpu_ref *ref) |
| 210 | { |
| 211 | } |
| 212 | |
| 213 | static void __percpu_ref_switch_to_atomic(struct percpu_ref *ref, |
| 214 | percpu_ref_func_t *confirm_switch) |
| 215 | { |
Tejun Heo | b2302c7 | 2015-09-29 17:47:17 -0400 | [diff] [blame] | 216 | if (ref->percpu_count_ptr & __PERCPU_REF_ATOMIC) { |
Tejun Heo | 1880835 | 2015-09-29 17:47:18 -0400 | [diff] [blame] | 217 | if (confirm_switch) |
Tejun Heo | b2302c7 | 2015-09-29 17:47:17 -0400 | [diff] [blame] | 218 | confirm_switch(ref); |
Tejun Heo | b2302c7 | 2015-09-29 17:47:17 -0400 | [diff] [blame] | 219 | return; |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 220 | } |
Tejun Heo | b2302c7 | 2015-09-29 17:47:17 -0400 | [diff] [blame] | 221 | |
| 222 | /* switching from percpu to atomic */ |
| 223 | ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC; |
| 224 | |
| 225 | /* |
| 226 | * Non-NULL ->confirm_switch is used to indicate that switching is |
| 227 | * in progress. Use noop one if unspecified. |
| 228 | */ |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 229 | ref->data->confirm_switch = confirm_switch ?: |
| 230 | percpu_ref_noop_confirm_switch; |
Tejun Heo | b2302c7 | 2015-09-29 17:47:17 -0400 | [diff] [blame] | 231 | |
| 232 | percpu_ref_get(ref); /* put after confirmation */ |
Joel Fernandes (Google) | 343a72e | 2022-10-16 16:22:58 +0000 | [diff] [blame] | 233 | call_rcu_hurry(&ref->data->rcu, |
| 234 | percpu_ref_switch_to_atomic_rcu); |
Tejun Heo | b2302c7 | 2015-09-29 17:47:17 -0400 | [diff] [blame] | 235 | } |
| 236 | |
| 237 | static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref) |
| 238 | { |
| 239 | unsigned long __percpu *percpu_count = percpu_count_ptr(ref); |
| 240 | int cpu; |
| 241 | |
| 242 | BUG_ON(!percpu_count); |
| 243 | |
| 244 | if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC)) |
| 245 | return; |
| 246 | |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 247 | if (WARN_ON_ONCE(!ref->data->allow_reinit)) |
Roman Gushchin | 7d9ab9b | 2019-05-07 10:01:50 -0700 | [diff] [blame] | 248 | return; |
| 249 | |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 250 | atomic_long_add(PERCPU_COUNT_BIAS, &ref->data->count); |
Tejun Heo | b2302c7 | 2015-09-29 17:47:17 -0400 | [diff] [blame] | 251 | |
| 252 | /* |
Paul E. McKenney | b393e8b | 2017-10-09 10:20:44 -0700 | [diff] [blame] | 253 | * Restore per-cpu operation. smp_store_release() is paired |
| 254 | * with READ_ONCE() in __ref_is_percpu() and guarantees that the |
| 255 | * zeroing is visible to all percpu accesses which can see the |
| 256 | * following __PERCPU_REF_ATOMIC clearing. |
Tejun Heo | b2302c7 | 2015-09-29 17:47:17 -0400 | [diff] [blame] | 257 | */ |
| 258 | for_each_possible_cpu(cpu) |
| 259 | *per_cpu_ptr(percpu_count, cpu) = 0; |
| 260 | |
| 261 | smp_store_release(&ref->percpu_count_ptr, |
| 262 | ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 263 | } |
| 264 | |
Tejun Heo | 3f49bdd | 2015-09-29 17:47:19 -0400 | [diff] [blame] | 265 | static void __percpu_ref_switch_mode(struct percpu_ref *ref, |
| 266 | percpu_ref_func_t *confirm_switch) |
| 267 | { |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 268 | struct percpu_ref_data *data = ref->data; |
| 269 | |
Tejun Heo | 33e465c | 2015-09-29 17:47:20 -0400 | [diff] [blame] | 270 | lockdep_assert_held(&percpu_ref_switch_lock); |
| 271 | |
Tejun Heo | 3f49bdd | 2015-09-29 17:47:19 -0400 | [diff] [blame] | 272 | /* |
| 273 | * If the previous ATOMIC switching hasn't finished yet, wait for |
| 274 | * its completion. If the caller ensures that ATOMIC switching |
| 275 | * isn't in progress, this function can be called from any context. |
Tejun Heo | 3f49bdd | 2015-09-29 17:47:19 -0400 | [diff] [blame] | 276 | */ |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 277 | wait_event_lock_irq(percpu_ref_switch_waitq, !data->confirm_switch, |
Tejun Heo | 33e465c | 2015-09-29 17:47:20 -0400 | [diff] [blame] | 278 | percpu_ref_switch_lock); |
Tejun Heo | 3f49bdd | 2015-09-29 17:47:19 -0400 | [diff] [blame] | 279 | |
Nikolay Borisov | 9e9da02 | 2021-05-11 16:16:33 +0300 | [diff] [blame] | 280 | if (data->force_atomic || percpu_ref_is_dying(ref)) |
Tejun Heo | 3f49bdd | 2015-09-29 17:47:19 -0400 | [diff] [blame] | 281 | __percpu_ref_switch_to_atomic(ref, confirm_switch); |
| 282 | else |
| 283 | __percpu_ref_switch_to_percpu(ref); |
| 284 | } |
| 285 | |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 286 | /** |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 287 | * percpu_ref_switch_to_atomic - switch a percpu_ref to atomic mode |
| 288 | * @ref: percpu_ref to switch to atomic mode |
| 289 | * @confirm_switch: optional confirmation callback |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 290 | * |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 291 | * There's no reason to use this function for the usual reference counting. |
| 292 | * Use percpu_ref_kill[_and_confirm](). |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 293 | * |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 294 | * Schedule switching of @ref to atomic mode. All its percpu counts will |
| 295 | * be collected to the main atomic counter. On completion, when all CPUs |
| 296 | * are guaraneed to be in atomic mode, @confirm_switch, which may not |
| 297 | * block, is invoked. This function may be invoked concurrently with all |
| 298 | * the get/put operations and can safely be mixed with kill and reinit |
Tejun Heo | 1cae13e | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 299 | * operations. Note that @ref will stay in atomic mode across kill/reinit |
| 300 | * cycles until percpu_ref_switch_to_percpu() is called. |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 301 | * |
Tejun Heo | 3f49bdd | 2015-09-29 17:47:19 -0400 | [diff] [blame] | 302 | * This function may block if @ref is in the process of switching to atomic |
| 303 | * mode. If the caller ensures that @ref is not in the process of |
| 304 | * switching to atomic mode, this function can be called from any context. |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 305 | */ |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 306 | void percpu_ref_switch_to_atomic(struct percpu_ref *ref, |
| 307 | percpu_ref_func_t *confirm_switch) |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 308 | { |
Tejun Heo | 33e465c | 2015-09-29 17:47:20 -0400 | [diff] [blame] | 309 | unsigned long flags; |
| 310 | |
| 311 | spin_lock_irqsave(&percpu_ref_switch_lock, flags); |
| 312 | |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 313 | ref->data->force_atomic = true; |
Tejun Heo | 3f49bdd | 2015-09-29 17:47:19 -0400 | [diff] [blame] | 314 | __percpu_ref_switch_mode(ref, confirm_switch); |
Tejun Heo | 33e465c | 2015-09-29 17:47:20 -0400 | [diff] [blame] | 315 | |
| 316 | spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); |
Kent Overstreet | 215e262 | 2013-05-31 15:26:45 -0700 | [diff] [blame] | 317 | } |
NeilBrown | 210f7cd | 2017-03-15 14:05:14 +1100 | [diff] [blame] | 318 | EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic); |
| 319 | |
| 320 | /** |
| 321 | * percpu_ref_switch_to_atomic_sync - switch a percpu_ref to atomic mode |
| 322 | * @ref: percpu_ref to switch to atomic mode |
| 323 | * |
| 324 | * Schedule switching the ref to atomic mode, and wait for the |
| 325 | * switch to complete. Caller must ensure that no other thread |
| 326 | * will switch back to percpu mode. |
| 327 | */ |
| 328 | void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref) |
| 329 | { |
| 330 | percpu_ref_switch_to_atomic(ref, NULL); |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 331 | wait_event(percpu_ref_switch_waitq, !ref->data->confirm_switch); |
NeilBrown | 210f7cd | 2017-03-15 14:05:14 +1100 | [diff] [blame] | 332 | } |
| 333 | EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic_sync); |
Tejun Heo | a223737 | 2014-09-24 13:31:48 -0400 | [diff] [blame] | 334 | |
Tejun Heo | f47ad45 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 335 | /** |
| 336 | * percpu_ref_switch_to_percpu - switch a percpu_ref to percpu mode |
| 337 | * @ref: percpu_ref to switch to percpu mode |
| 338 | * |
| 339 | * There's no reason to use this function for the usual reference counting. |
| 340 | * To re-use an expired ref, use percpu_ref_reinit(). |
| 341 | * |
| 342 | * Switch @ref to percpu mode. This function may be invoked concurrently |
| 343 | * with all the get/put operations and can safely be mixed with kill and |
Tejun Heo | 1cae13e | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 344 | * reinit operations. This function reverses the sticky atomic state set |
| 345 | * by PERCPU_REF_INIT_ATOMIC or percpu_ref_switch_to_atomic(). If @ref is |
| 346 | * dying or dead, the actual switching takes place on the following |
| 347 | * percpu_ref_reinit(). |
Tejun Heo | f47ad45 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 348 | * |
Tejun Heo | 3f49bdd | 2015-09-29 17:47:19 -0400 | [diff] [blame] | 349 | * This function may block if @ref is in the process of switching to atomic |
| 350 | * mode. If the caller ensures that @ref is not in the process of |
| 351 | * switching to atomic mode, this function can be called from any context. |
Tejun Heo | f47ad45 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 352 | */ |
| 353 | void percpu_ref_switch_to_percpu(struct percpu_ref *ref) |
| 354 | { |
Tejun Heo | 33e465c | 2015-09-29 17:47:20 -0400 | [diff] [blame] | 355 | unsigned long flags; |
| 356 | |
| 357 | spin_lock_irqsave(&percpu_ref_switch_lock, flags); |
| 358 | |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 359 | ref->data->force_atomic = false; |
Tejun Heo | 3f49bdd | 2015-09-29 17:47:19 -0400 | [diff] [blame] | 360 | __percpu_ref_switch_mode(ref, NULL); |
Tejun Heo | 33e465c | 2015-09-29 17:47:20 -0400 | [diff] [blame] | 361 | |
| 362 | spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); |
Tejun Heo | f47ad45 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 363 | } |
NeilBrown | 210f7cd | 2017-03-15 14:05:14 +1100 | [diff] [blame] | 364 | EXPORT_SYMBOL_GPL(percpu_ref_switch_to_percpu); |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 365 | |
| 366 | /** |
| 367 | * percpu_ref_kill_and_confirm - drop the initial ref and schedule confirmation |
| 368 | * @ref: percpu_ref to kill |
| 369 | * @confirm_kill: optional confirmation callback |
| 370 | * |
| 371 | * Equivalent to percpu_ref_kill() but also schedules kill confirmation if |
| 372 | * @confirm_kill is not NULL. @confirm_kill, which may not block, will be |
| 373 | * called after @ref is seen as dead from all CPUs at which point all |
| 374 | * further invocations of percpu_ref_tryget_live() will fail. See |
| 375 | * percpu_ref_tryget_live() for details. |
| 376 | * |
| 377 | * This function normally doesn't block and can be called from any context |
Tejun Heo | f47ad45 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 378 | * but it may block if @confirm_kill is specified and @ref is in the |
Tejun Heo | a2f5630 | 2015-09-29 17:47:16 -0400 | [diff] [blame] | 379 | * process of switching to atomic mode by percpu_ref_switch_to_atomic(). |
Tejun Heo | b3a5d11 | 2018-03-14 12:45:12 -0700 | [diff] [blame] | 380 | * |
| 381 | * There are no implied RCU grace periods between kill and release. |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 382 | */ |
| 383 | void percpu_ref_kill_and_confirm(struct percpu_ref *ref, |
| 384 | percpu_ref_func_t *confirm_kill) |
| 385 | { |
Tejun Heo | 33e465c | 2015-09-29 17:47:20 -0400 | [diff] [blame] | 386 | unsigned long flags; |
| 387 | |
| 388 | spin_lock_irqsave(&percpu_ref_switch_lock, flags); |
| 389 | |
Nikolay Borisov | 9e9da02 | 2021-05-11 16:16:33 +0300 | [diff] [blame] | 390 | WARN_ONCE(percpu_ref_is_dying(ref), |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 391 | "%s called more than once on %ps!", __func__, |
| 392 | ref->data->release); |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 393 | |
| 394 | ref->percpu_count_ptr |= __PERCPU_REF_DEAD; |
Tejun Heo | 3f49bdd | 2015-09-29 17:47:19 -0400 | [diff] [blame] | 395 | __percpu_ref_switch_mode(ref, confirm_kill); |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 396 | percpu_ref_put(ref); |
Tejun Heo | 33e465c | 2015-09-29 17:47:20 -0400 | [diff] [blame] | 397 | |
| 398 | spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); |
Tejun Heo | 490c79a | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 399 | } |
| 400 | EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm); |
Tejun Heo | f47ad45 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 401 | |
| 402 | /** |
Ming Lei | 2b0d3d3 | 2020-10-01 23:48:41 +0800 | [diff] [blame] | 403 | * percpu_ref_is_zero - test whether a percpu refcount reached zero |
| 404 | * @ref: percpu_ref to test |
| 405 | * |
| 406 | * Returns %true if @ref reached zero. |
| 407 | * |
| 408 | * This function is safe to call as long as @ref is between init and exit. |
| 409 | */ |
| 410 | bool percpu_ref_is_zero(struct percpu_ref *ref) |
| 411 | { |
| 412 | unsigned long __percpu *percpu_count; |
| 413 | unsigned long count, flags; |
| 414 | |
| 415 | if (__ref_is_percpu(ref, &percpu_count)) |
| 416 | return false; |
| 417 | |
| 418 | /* protect us from being destroyed */ |
| 419 | spin_lock_irqsave(&percpu_ref_switch_lock, flags); |
| 420 | if (ref->data) |
| 421 | count = atomic_long_read(&ref->data->count); |
| 422 | else |
| 423 | count = ref->percpu_count_ptr >> __PERCPU_REF_FLAG_BITS; |
| 424 | spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); |
| 425 | |
| 426 | return count == 0; |
| 427 | } |
| 428 | EXPORT_SYMBOL_GPL(percpu_ref_is_zero); |
| 429 | |
| 430 | /** |
Tejun Heo | f47ad45 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 431 | * percpu_ref_reinit - re-initialize a percpu refcount |
| 432 | * @ref: perpcu_ref to re-initialize |
| 433 | * |
| 434 | * Re-initialize @ref so that it's in the same state as when it finished |
Tejun Heo | 1cae13e | 2014-09-24 13:31:50 -0400 | [diff] [blame] | 435 | * percpu_ref_init() ignoring %PERCPU_REF_INIT_DEAD. @ref must have been |
| 436 | * initialized successfully and reached 0 but not exited. |
Tejun Heo | f47ad45 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 437 | * |
| 438 | * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while |
| 439 | * this function is in progress. |
| 440 | */ |
| 441 | void percpu_ref_reinit(struct percpu_ref *ref) |
| 442 | { |
Bart Van Assche | 18c9a6b | 2018-09-26 14:01:07 -0700 | [diff] [blame] | 443 | WARN_ON_ONCE(!percpu_ref_is_zero(ref)); |
| 444 | |
| 445 | percpu_ref_resurrect(ref); |
| 446 | } |
| 447 | EXPORT_SYMBOL_GPL(percpu_ref_reinit); |
| 448 | |
| 449 | /** |
| 450 | * percpu_ref_resurrect - modify a percpu refcount from dead to live |
| 451 | * @ref: perpcu_ref to resurrect |
| 452 | * |
| 453 | * Modify @ref so that it's in the same state as before percpu_ref_kill() was |
| 454 | * called. @ref must be dead but must not yet have exited. |
| 455 | * |
| 456 | * If @ref->release() frees @ref then the caller is responsible for |
| 457 | * guaranteeing that @ref->release() does not get called while this |
| 458 | * function is in progress. |
| 459 | * |
| 460 | * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while |
| 461 | * this function is in progress. |
| 462 | */ |
| 463 | void percpu_ref_resurrect(struct percpu_ref *ref) |
| 464 | { |
| 465 | unsigned long __percpu *percpu_count; |
Tejun Heo | 33e465c | 2015-09-29 17:47:20 -0400 | [diff] [blame] | 466 | unsigned long flags; |
| 467 | |
| 468 | spin_lock_irqsave(&percpu_ref_switch_lock, flags); |
| 469 | |
Nikolay Borisov | 9e9da02 | 2021-05-11 16:16:33 +0300 | [diff] [blame] | 470 | WARN_ON_ONCE(!percpu_ref_is_dying(ref)); |
Bart Van Assche | 18c9a6b | 2018-09-26 14:01:07 -0700 | [diff] [blame] | 471 | WARN_ON_ONCE(__ref_is_percpu(ref, &percpu_count)); |
Tejun Heo | f47ad45 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 472 | |
| 473 | ref->percpu_count_ptr &= ~__PERCPU_REF_DEAD; |
| 474 | percpu_ref_get(ref); |
Tejun Heo | 3f49bdd | 2015-09-29 17:47:19 -0400 | [diff] [blame] | 475 | __percpu_ref_switch_mode(ref, NULL); |
Tejun Heo | 33e465c | 2015-09-29 17:47:20 -0400 | [diff] [blame] | 476 | |
| 477 | spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); |
Tejun Heo | f47ad45 | 2014-09-24 13:31:49 -0400 | [diff] [blame] | 478 | } |
Bart Van Assche | 18c9a6b | 2018-09-26 14:01:07 -0700 | [diff] [blame] | 479 | EXPORT_SYMBOL_GPL(percpu_ref_resurrect); |