Cheah Kok Cheong | 08b21fb | 2017-12-21 19:35:30 +0800 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 2 | /* |
| 3 | * padata.c - generic interface to process data streams in parallel |
| 4 | * |
Daniel Jordan | bfcdcef8 | 2019-12-03 14:31:14 -0500 | [diff] [blame] | 5 | * See Documentation/core-api/padata.rst for more information. |
Steffen Klassert | 107f8bd | 2012-03-28 08:42:34 +0200 | [diff] [blame] | 6 | * |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 7 | * Copyright (C) 2008, 2009 secunet Security Networks AG |
| 8 | * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com> |
| 9 | * |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 10 | * Copyright (c) 2020 Oracle and/or its affiliates. |
| 11 | * Author: Daniel Jordan <daniel.m.jordan@oracle.com> |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 12 | */ |
| 13 | |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 14 | #include <linux/completion.h> |
Paul Gortmaker | 9984de1 | 2011-05-23 14:51:41 -0400 | [diff] [blame] | 15 | #include <linux/export.h> |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 16 | #include <linux/cpumask.h> |
| 17 | #include <linux/err.h> |
| 18 | #include <linux/cpu.h> |
| 19 | #include <linux/padata.h> |
| 20 | #include <linux/mutex.h> |
| 21 | #include <linux/sched.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 22 | #include <linux/slab.h> |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 23 | #include <linux/sysfs.h> |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 24 | #include <linux/rcupdate.h> |
| 25 | |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 26 | #define PADATA_WORK_ONSTACK 1 /* Work's memory is on stack */ |
| 27 | |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 28 | struct padata_work { |
| 29 | struct work_struct pw_work; |
| 30 | struct list_head pw_list; /* padata_free_works linkage */ |
| 31 | void *pw_data; |
| 32 | }; |
| 33 | |
| 34 | static DEFINE_SPINLOCK(padata_works_lock); |
| 35 | static struct padata_work *padata_works; |
| 36 | static LIST_HEAD(padata_free_works); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 37 | |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 38 | struct padata_mt_job_state { |
| 39 | spinlock_t lock; |
| 40 | struct completion completion; |
| 41 | struct padata_mt_job *job; |
| 42 | int nworks; |
| 43 | int nworks_fini; |
| 44 | unsigned long chunk_size; |
| 45 | }; |
| 46 | |
Herbert Xu | 07928d9 | 2019-11-19 13:17:31 +0800 | [diff] [blame] | 47 | static void padata_free_pd(struct parallel_data *pd); |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 48 | static void __init padata_mt_helper(struct work_struct *work); |
Herbert Xu | 07928d9 | 2019-11-19 13:17:31 +0800 | [diff] [blame] | 49 | |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 50 | static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index) |
| 51 | { |
| 52 | int cpu, target_cpu; |
| 53 | |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 54 | target_cpu = cpumask_first(pd->cpumask.pcpu); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 55 | for (cpu = 0; cpu < cpu_index; cpu++) |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 56 | target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 57 | |
| 58 | return target_cpu; |
| 59 | } |
| 60 | |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 61 | static int padata_cpu_hash(struct parallel_data *pd, unsigned int seq_nr) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 62 | { |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 63 | /* |
| 64 | * Hash the sequence numbers to the cpus by taking |
| 65 | * seq_nr mod. number of cpus in use. |
| 66 | */ |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 67 | int cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 68 | |
| 69 | return padata_index_to_cpu(pd, cpu_index); |
| 70 | } |
| 71 | |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 72 | static struct padata_work *padata_work_alloc(void) |
| 73 | { |
| 74 | struct padata_work *pw; |
| 75 | |
| 76 | lockdep_assert_held(&padata_works_lock); |
| 77 | |
| 78 | if (list_empty(&padata_free_works)) |
| 79 | return NULL; /* No more work items allowed to be queued. */ |
| 80 | |
| 81 | pw = list_first_entry(&padata_free_works, struct padata_work, pw_list); |
| 82 | list_del(&pw->pw_list); |
| 83 | return pw; |
| 84 | } |
| 85 | |
Nathan Chancellor | 0d24f1b | 2022-12-13 11:35:28 -0700 | [diff] [blame] | 86 | /* |
| 87 | * This function is marked __ref because this function may be optimized in such |
| 88 | * a way that it directly refers to work_fn's address, which causes modpost to |
| 89 | * complain when work_fn is marked __init. This scenario was observed with clang |
| 90 | * LTO, where padata_work_init() was optimized to refer directly to |
| 91 | * padata_mt_helper() because the calls to padata_work_init() with other work_fn |
| 92 | * values were eliminated or inlined. |
| 93 | */ |
| 94 | static void __ref padata_work_init(struct padata_work *pw, work_func_t work_fn, |
| 95 | void *data, int flags) |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 96 | { |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 97 | if (flags & PADATA_WORK_ONSTACK) |
| 98 | INIT_WORK_ONSTACK(&pw->pw_work, work_fn); |
| 99 | else |
| 100 | INIT_WORK(&pw->pw_work, work_fn); |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 101 | pw->pw_data = data; |
| 102 | } |
| 103 | |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 104 | static int __init padata_work_alloc_mt(int nworks, void *data, |
| 105 | struct list_head *head) |
| 106 | { |
| 107 | int i; |
| 108 | |
Herbert Xu | 58329c4 | 2024-04-03 17:36:18 +0800 | [diff] [blame] | 109 | spin_lock_bh(&padata_works_lock); |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 110 | /* Start at 1 because the current task participates in the job. */ |
| 111 | for (i = 1; i < nworks; ++i) { |
| 112 | struct padata_work *pw = padata_work_alloc(); |
| 113 | |
| 114 | if (!pw) |
| 115 | break; |
| 116 | padata_work_init(pw, padata_mt_helper, data, 0); |
| 117 | list_add(&pw->pw_list, head); |
| 118 | } |
Herbert Xu | 58329c4 | 2024-04-03 17:36:18 +0800 | [diff] [blame] | 119 | spin_unlock_bh(&padata_works_lock); |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 120 | |
| 121 | return i; |
| 122 | } |
| 123 | |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 124 | static void padata_work_free(struct padata_work *pw) |
| 125 | { |
| 126 | lockdep_assert_held(&padata_works_lock); |
| 127 | list_add(&pw->pw_list, &padata_free_works); |
| 128 | } |
| 129 | |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 130 | static void __init padata_works_free(struct list_head *works) |
| 131 | { |
| 132 | struct padata_work *cur, *next; |
| 133 | |
| 134 | if (list_empty(works)) |
| 135 | return; |
| 136 | |
Herbert Xu | 58329c4 | 2024-04-03 17:36:18 +0800 | [diff] [blame] | 137 | spin_lock_bh(&padata_works_lock); |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 138 | list_for_each_entry_safe(cur, next, works, pw_list) { |
| 139 | list_del(&cur->pw_list); |
| 140 | padata_work_free(cur); |
| 141 | } |
Herbert Xu | 58329c4 | 2024-04-03 17:36:18 +0800 | [diff] [blame] | 142 | spin_unlock_bh(&padata_works_lock); |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 143 | } |
| 144 | |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 145 | static void padata_parallel_worker(struct work_struct *parallel_work) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 146 | { |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 147 | struct padata_work *pw = container_of(parallel_work, struct padata_work, |
| 148 | pw_work); |
| 149 | struct padata_priv *padata = pw->pw_data; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 150 | |
| 151 | local_bh_disable(); |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 152 | padata->parallel(padata); |
| 153 | spin_lock(&padata_works_lock); |
| 154 | padata_work_free(pw); |
| 155 | spin_unlock(&padata_works_lock); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 156 | local_bh_enable(); |
| 157 | } |
| 158 | |
Steffen Klassert | 0198ffd | 2010-05-19 13:44:27 +1000 | [diff] [blame] | 159 | /** |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 160 | * padata_do_parallel - padata parallelization function |
| 161 | * |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 162 | * @ps: padatashell |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 163 | * @padata: object to be parallelized |
Daniel Jordan | e6ce0e0 | 2019-09-05 21:40:24 -0400 | [diff] [blame] | 164 | * @cb_cpu: pointer to the CPU that the serialization callback function should |
| 165 | * run on. If it's not in the serial cpumask of @pinst |
| 166 | * (i.e. cpumask.cbcpu), this function selects a fallback CPU and if |
| 167 | * none found, returns -EINVAL. |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 168 | * |
| 169 | * The parallelization callback function will run with BHs off. |
| 170 | * Note: Every object which is parallelized by padata_do_parallel |
| 171 | * must be seen by padata_do_serial. |
Daniel Jordan | bfcdcef8 | 2019-12-03 14:31:14 -0500 | [diff] [blame] | 172 | * |
| 173 | * Return: 0 on success or else negative error code. |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 174 | */ |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 175 | int padata_do_parallel(struct padata_shell *ps, |
Daniel Jordan | e6ce0e0 | 2019-09-05 21:40:24 -0400 | [diff] [blame] | 176 | struct padata_priv *padata, int *cb_cpu) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 177 | { |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 178 | struct padata_instance *pinst = ps->pinst; |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 179 | int i, cpu, cpu_index, err; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 180 | struct parallel_data *pd; |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 181 | struct padata_work *pw; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 182 | |
| 183 | rcu_read_lock_bh(); |
| 184 | |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 185 | pd = rcu_dereference_bh(ps->pd); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 186 | |
Steffen Klassert | 83f619f | 2010-07-07 15:32:02 +0200 | [diff] [blame] | 187 | err = -EINVAL; |
Steffen Klassert | 7424713 | 2010-07-20 08:51:25 +0200 | [diff] [blame] | 188 | if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 189 | goto out; |
| 190 | |
Daniel Jordan | e6ce0e0 | 2019-09-05 21:40:24 -0400 | [diff] [blame] | 191 | if (!cpumask_test_cpu(*cb_cpu, pd->cpumask.cbcpu)) { |
Yury Norov | 1c4cafd | 2022-01-23 10:38:52 -0800 | [diff] [blame] | 192 | if (cpumask_empty(pd->cpumask.cbcpu)) |
Daniel Jordan | e6ce0e0 | 2019-09-05 21:40:24 -0400 | [diff] [blame] | 193 | goto out; |
| 194 | |
| 195 | /* Select an alternate fallback CPU and notify the caller. */ |
| 196 | cpu_index = *cb_cpu % cpumask_weight(pd->cpumask.cbcpu); |
| 197 | |
| 198 | cpu = cpumask_first(pd->cpumask.cbcpu); |
| 199 | for (i = 0; i < cpu_index; i++) |
| 200 | cpu = cpumask_next(cpu, pd->cpumask.cbcpu); |
| 201 | |
| 202 | *cb_cpu = cpu; |
| 203 | } |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 204 | |
Lu Jialin | 8f4f68e | 2023-09-04 13:33:41 +0000 | [diff] [blame] | 205 | err = -EBUSY; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 206 | if ((pinst->flags & PADATA_RESET)) |
| 207 | goto out; |
| 208 | |
Xiyu Yang | d5ee8e7 | 2021-07-20 11:05:11 -0400 | [diff] [blame] | 209 | refcount_inc(&pd->refcnt); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 210 | padata->pd = pd; |
Daniel Jordan | e6ce0e0 | 2019-09-05 21:40:24 -0400 | [diff] [blame] | 211 | padata->cb_cpu = *cb_cpu; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 212 | |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 213 | spin_lock(&padata_works_lock); |
| 214 | padata->seq_nr = ++pd->seq_nr; |
| 215 | pw = padata_work_alloc(); |
| 216 | spin_unlock(&padata_works_lock); |
Daniel Jordan | 1b0df11 | 2020-09-02 13:07:56 -0400 | [diff] [blame] | 217 | |
Daniel Jordan | 34c3a47 | 2022-11-16 20:28:02 -0500 | [diff] [blame] | 218 | if (!pw) { |
| 219 | /* Maximum works limit exceeded, run in the current task. */ |
| 220 | padata->parallel(padata); |
| 221 | } |
| 222 | |
Daniel Jordan | 1b0df11 | 2020-09-02 13:07:56 -0400 | [diff] [blame] | 223 | rcu_read_unlock_bh(); |
| 224 | |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 225 | if (pw) { |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 226 | padata_work_init(pw, padata_parallel_worker, padata, 0); |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 227 | queue_work(pinst->parallel_wq, &pw->pw_work); |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 228 | } |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 229 | |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 230 | return 0; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 231 | out: |
| 232 | rcu_read_unlock_bh(); |
| 233 | |
| 234 | return err; |
| 235 | } |
| 236 | EXPORT_SYMBOL(padata_do_parallel); |
| 237 | |
Steffen Klassert | 0198ffd | 2010-05-19 13:44:27 +1000 | [diff] [blame] | 238 | /* |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 239 | * padata_find_next - Find the next object that needs serialization. |
Steffen Klassert | 0198ffd | 2010-05-19 13:44:27 +1000 | [diff] [blame] | 240 | * |
Daniel Jordan | bfcdcef8 | 2019-12-03 14:31:14 -0500 | [diff] [blame] | 241 | * Return: |
| 242 | * * A pointer to the control struct of the next object that needs |
| 243 | * serialization, if present in one of the percpu reorder queues. |
| 244 | * * NULL, if the next object that needs serialization will |
| 245 | * be parallel processed by another cpu and is not yet present in |
| 246 | * the cpu's reorder queue. |
Steffen Klassert | 0198ffd | 2010-05-19 13:44:27 +1000 | [diff] [blame] | 247 | */ |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 248 | static struct padata_priv *padata_find_next(struct parallel_data *pd, |
| 249 | bool remove_object) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 250 | { |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 251 | struct padata_priv *padata; |
| 252 | struct padata_list *reorder; |
Herbert Xu | 6fc4dbc | 2019-07-18 23:01:46 +0800 | [diff] [blame] | 253 | int cpu = pd->cpu; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 254 | |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 255 | reorder = per_cpu_ptr(pd->reorder_list, cpu); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 256 | |
Jason A. Donenfeld | de5540d | 2017-03-23 12:24:43 +0100 | [diff] [blame] | 257 | spin_lock(&reorder->lock); |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 258 | if (list_empty(&reorder->list)) { |
| 259 | spin_unlock(&reorder->lock); |
| 260 | return NULL; |
| 261 | } |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 262 | |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 263 | padata = list_entry(reorder->list.next, struct padata_priv, list); |
| 264 | |
| 265 | /* |
| 266 | * Checks the rare case where two or more parallel jobs have hashed to |
| 267 | * the same CPU and one of the later ones finishes first. |
| 268 | */ |
| 269 | if (padata->seq_nr != pd->processed) { |
| 270 | spin_unlock(&reorder->lock); |
| 271 | return NULL; |
| 272 | } |
| 273 | |
| 274 | if (remove_object) { |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 275 | list_del_init(&padata->list); |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 276 | ++pd->processed; |
| 277 | pd->cpu = cpumask_next_wrap(cpu, pd->cpumask.pcpu, -1, false); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 278 | } |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 279 | |
Jason A. Donenfeld | de5540d | 2017-03-23 12:24:43 +0100 | [diff] [blame] | 280 | spin_unlock(&reorder->lock); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 281 | return padata; |
| 282 | } |
| 283 | |
| 284 | static void padata_reorder(struct parallel_data *pd) |
| 285 | { |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 286 | struct padata_instance *pinst = pd->ps->pinst; |
Steffen Klassert | 3047817 | 2012-03-09 07:20:12 +0100 | [diff] [blame] | 287 | int cb_cpu; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 288 | struct padata_priv *padata; |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 289 | struct padata_serial_queue *squeue; |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 290 | struct padata_list *reorder; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 291 | |
Steffen Klassert | 0198ffd | 2010-05-19 13:44:27 +1000 | [diff] [blame] | 292 | /* |
| 293 | * We need to ensure that only one cpu can work on dequeueing of |
| 294 | * the reorder queue the time. Calculating in which percpu reorder |
| 295 | * queue the next object will arrive takes some time. A spinlock |
| 296 | * would be highly contended. Also it is not clear in which order |
| 297 | * the objects arrive to the reorder queues. So a cpu could wait to |
| 298 | * get the lock just to notice that there is nothing to do at the |
| 299 | * moment. Therefore we use a trylock and let the holder of the lock |
| 300 | * care for all the objects enqueued during the holdtime of the lock. |
| 301 | */ |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 302 | if (!spin_trylock_bh(&pd->lock)) |
Steffen Klassert | d46a5ac | 2010-05-19 13:43:14 +1000 | [diff] [blame] | 303 | return; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 304 | |
| 305 | while (1) { |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 306 | padata = padata_find_next(pd, true); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 307 | |
Steffen Klassert | 0198ffd | 2010-05-19 13:44:27 +1000 | [diff] [blame] | 308 | /* |
Jason A. Donenfeld | 69b3484 | 2017-04-12 10:40:19 +0200 | [diff] [blame] | 309 | * If the next object that needs serialization is parallel |
| 310 | * processed by another cpu and is still on it's way to the |
| 311 | * cpu's reorder queue, nothing to do for now. |
Steffen Klassert | 0198ffd | 2010-05-19 13:44:27 +1000 | [diff] [blame] | 312 | */ |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 313 | if (!padata) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 314 | break; |
| 315 | |
Steffen Klassert | 3047817 | 2012-03-09 07:20:12 +0100 | [diff] [blame] | 316 | cb_cpu = padata->cb_cpu; |
| 317 | squeue = per_cpu_ptr(pd->squeue, cb_cpu); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 318 | |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 319 | spin_lock(&squeue->serial.lock); |
| 320 | list_add_tail(&padata->list, &squeue->serial.list); |
| 321 | spin_unlock(&squeue->serial.lock); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 322 | |
Daniel Jordan | 45d153c | 2019-09-05 21:40:27 -0400 | [diff] [blame] | 323 | queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 324 | } |
| 325 | |
| 326 | spin_unlock_bh(&pd->lock); |
| 327 | |
Steffen Klassert | 0198ffd | 2010-05-19 13:44:27 +1000 | [diff] [blame] | 328 | /* |
| 329 | * The next object that needs serialization might have arrived to |
Herbert Xu | 6fc4dbc | 2019-07-18 23:01:46 +0800 | [diff] [blame] | 330 | * the reorder queues in the meantime. |
Daniel Jordan | cf144f8 | 2019-07-16 12:32:53 -0400 | [diff] [blame] | 331 | * |
Herbert Xu | 6fc4dbc | 2019-07-18 23:01:46 +0800 | [diff] [blame] | 332 | * Ensure reorder queue is read after pd->lock is dropped so we see |
| 333 | * new objects from another task in padata_do_serial. Pairs with |
Daniel Jordan | e04ec0d | 2020-06-08 17:26:52 -0400 | [diff] [blame] | 334 | * smp_mb in padata_do_serial. |
Steffen Klassert | 0198ffd | 2010-05-19 13:44:27 +1000 | [diff] [blame] | 335 | */ |
Daniel Jordan | cf144f8 | 2019-07-16 12:32:53 -0400 | [diff] [blame] | 336 | smp_mb(); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 337 | |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 338 | reorder = per_cpu_ptr(pd->reorder_list, pd->cpu); |
| 339 | if (!list_empty(&reorder->list) && padata_find_next(pd, false)) |
Daniel Jordan | 45d153c | 2019-09-05 21:40:27 -0400 | [diff] [blame] | 340 | queue_work(pinst->serial_wq, &pd->reorder_work); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 341 | } |
| 342 | |
Mathias Krause | cf5868c | 2017-09-08 20:57:10 +0200 | [diff] [blame] | 343 | static void invoke_padata_reorder(struct work_struct *work) |
| 344 | { |
Mathias Krause | cf5868c | 2017-09-08 20:57:10 +0200 | [diff] [blame] | 345 | struct parallel_data *pd; |
| 346 | |
| 347 | local_bh_disable(); |
Herbert Xu | 6fc4dbc | 2019-07-18 23:01:46 +0800 | [diff] [blame] | 348 | pd = container_of(work, struct parallel_data, reorder_work); |
Mathias Krause | cf5868c | 2017-09-08 20:57:10 +0200 | [diff] [blame] | 349 | padata_reorder(pd); |
| 350 | local_bh_enable(); |
| 351 | } |
| 352 | |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 353 | static void padata_serial_worker(struct work_struct *serial_work) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 354 | { |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 355 | struct padata_serial_queue *squeue; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 356 | struct parallel_data *pd; |
| 357 | LIST_HEAD(local_list); |
Herbert Xu | 07928d9 | 2019-11-19 13:17:31 +0800 | [diff] [blame] | 358 | int cnt; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 359 | |
| 360 | local_bh_disable(); |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 361 | squeue = container_of(serial_work, struct padata_serial_queue, work); |
| 362 | pd = squeue->pd; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 363 | |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 364 | spin_lock(&squeue->serial.lock); |
| 365 | list_replace_init(&squeue->serial.list, &local_list); |
| 366 | spin_unlock(&squeue->serial.lock); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 367 | |
Herbert Xu | 07928d9 | 2019-11-19 13:17:31 +0800 | [diff] [blame] | 368 | cnt = 0; |
| 369 | |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 370 | while (!list_empty(&local_list)) { |
| 371 | struct padata_priv *padata; |
| 372 | |
| 373 | padata = list_entry(local_list.next, |
| 374 | struct padata_priv, list); |
| 375 | |
| 376 | list_del_init(&padata->list); |
| 377 | |
| 378 | padata->serial(padata); |
Herbert Xu | 07928d9 | 2019-11-19 13:17:31 +0800 | [diff] [blame] | 379 | cnt++; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 380 | } |
| 381 | local_bh_enable(); |
Herbert Xu | 07928d9 | 2019-11-19 13:17:31 +0800 | [diff] [blame] | 382 | |
Xiyu Yang | d5ee8e7 | 2021-07-20 11:05:11 -0400 | [diff] [blame] | 383 | if (refcount_sub_and_test(cnt, &pd->refcnt)) |
Herbert Xu | 07928d9 | 2019-11-19 13:17:31 +0800 | [diff] [blame] | 384 | padata_free_pd(pd); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 385 | } |
| 386 | |
Steffen Klassert | 0198ffd | 2010-05-19 13:44:27 +1000 | [diff] [blame] | 387 | /** |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 388 | * padata_do_serial - padata serialization function |
| 389 | * |
| 390 | * @padata: object to be serialized. |
| 391 | * |
| 392 | * padata_do_serial must be called for every parallelized object. |
| 393 | * The serialization callback function will run with BHs off. |
| 394 | */ |
| 395 | void padata_do_serial(struct padata_priv *padata) |
| 396 | { |
Daniel Jordan | 065cf57 | 2019-07-19 15:04:44 -0400 | [diff] [blame] | 397 | struct parallel_data *pd = padata->pd; |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 398 | int hashed_cpu = padata_cpu_hash(pd, padata->seq_nr); |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 399 | struct padata_list *reorder = per_cpu_ptr(pd->reorder_list, hashed_cpu); |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 400 | struct padata_priv *cur; |
Daniel Jordan | 57ddfec | 2022-11-16 20:28:04 -0500 | [diff] [blame] | 401 | struct list_head *pos; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 402 | |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 403 | spin_lock(&reorder->lock); |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 404 | /* Sort in ascending order of sequence number. */ |
Daniel Jordan | 57ddfec | 2022-11-16 20:28:04 -0500 | [diff] [blame] | 405 | list_for_each_prev(pos, &reorder->list) { |
| 406 | cur = list_entry(pos, struct padata_priv, list); |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 407 | if (cur->seq_nr < padata->seq_nr) |
| 408 | break; |
Daniel Jordan | 57ddfec | 2022-11-16 20:28:04 -0500 | [diff] [blame] | 409 | } |
| 410 | list_add(&padata->list, pos); |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 411 | spin_unlock(&reorder->lock); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 412 | |
Daniel Jordan | cf144f8 | 2019-07-16 12:32:53 -0400 | [diff] [blame] | 413 | /* |
Herbert Xu | 6fc4dbc | 2019-07-18 23:01:46 +0800 | [diff] [blame] | 414 | * Ensure the addition to the reorder list is ordered correctly |
Daniel Jordan | cf144f8 | 2019-07-16 12:32:53 -0400 | [diff] [blame] | 415 | * with the trylock of pd->lock in padata_reorder. Pairs with smp_mb |
| 416 | * in padata_reorder. |
| 417 | */ |
Daniel Jordan | e04ec0d | 2020-06-08 17:26:52 -0400 | [diff] [blame] | 418 | smp_mb(); |
Daniel Jordan | cf144f8 | 2019-07-16 12:32:53 -0400 | [diff] [blame] | 419 | |
Herbert Xu | 6fc4dbc | 2019-07-18 23:01:46 +0800 | [diff] [blame] | 420 | padata_reorder(pd); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 421 | } |
| 422 | EXPORT_SYMBOL(padata_do_serial); |
| 423 | |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 424 | static int padata_setup_cpumasks(struct padata_instance *pinst) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 425 | { |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 426 | struct workqueue_attrs *attrs; |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 427 | int err; |
| 428 | |
| 429 | attrs = alloc_workqueue_attrs(); |
| 430 | if (!attrs) |
| 431 | return -ENOMEM; |
| 432 | |
| 433 | /* Restrict parallel_wq workers to pd->cpumask.pcpu. */ |
| 434 | cpumask_copy(attrs->cpumask, pinst->cpumask.pcpu); |
| 435 | err = apply_workqueue_attrs(pinst->parallel_wq, attrs); |
| 436 | free_workqueue_attrs(attrs); |
| 437 | |
| 438 | return err; |
| 439 | } |
| 440 | |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 441 | static void __init padata_mt_helper(struct work_struct *w) |
| 442 | { |
| 443 | struct padata_work *pw = container_of(w, struct padata_work, pw_work); |
| 444 | struct padata_mt_job_state *ps = pw->pw_data; |
| 445 | struct padata_mt_job *job = ps->job; |
| 446 | bool done; |
| 447 | |
| 448 | spin_lock(&ps->lock); |
| 449 | |
| 450 | while (job->size > 0) { |
| 451 | unsigned long start, size, end; |
| 452 | |
| 453 | start = job->start; |
| 454 | /* So end is chunk size aligned if enough work remains. */ |
| 455 | size = roundup(start + 1, ps->chunk_size) - start; |
| 456 | size = min(size, job->size); |
| 457 | end = start + size; |
| 458 | |
| 459 | job->start = end; |
| 460 | job->size -= size; |
| 461 | |
| 462 | spin_unlock(&ps->lock); |
| 463 | job->thread_fn(start, end, job->fn_arg); |
| 464 | spin_lock(&ps->lock); |
| 465 | } |
| 466 | |
| 467 | ++ps->nworks_fini; |
| 468 | done = (ps->nworks_fini == ps->nworks); |
| 469 | spin_unlock(&ps->lock); |
| 470 | |
| 471 | if (done) |
| 472 | complete(&ps->completion); |
| 473 | } |
| 474 | |
| 475 | /** |
| 476 | * padata_do_multithreaded - run a multithreaded job |
| 477 | * @job: Description of the job. |
| 478 | * |
| 479 | * See the definition of struct padata_mt_job for more details. |
| 480 | */ |
| 481 | void __init padata_do_multithreaded(struct padata_mt_job *job) |
| 482 | { |
| 483 | /* In case threads finish at different times. */ |
| 484 | static const unsigned long load_balance_factor = 4; |
| 485 | struct padata_work my_work, *pw; |
| 486 | struct padata_mt_job_state ps; |
| 487 | LIST_HEAD(works); |
Gang Li Subject: padata: dispatch works on | eb52286 | 2024-03-06 13:04:17 -0800 | [diff] [blame] | 488 | int nworks, nid; |
| 489 | static atomic_t last_used_nid __initdata; |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 490 | |
| 491 | if (job->size == 0) |
| 492 | return; |
| 493 | |
| 494 | /* Ensure at least one thread when size < min_chunk. */ |
Anthony Yznaga | f84155c | 2023-02-22 16:33:12 -0800 | [diff] [blame] | 495 | nworks = max(job->size / max(job->min_chunk, job->align), 1ul); |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 496 | nworks = min(nworks, job->max_threads); |
| 497 | |
| 498 | if (nworks == 1) { |
| 499 | /* Single thread, no coordination needed, cut to the chase. */ |
| 500 | job->thread_fn(job->start, job->start + job->size, job->fn_arg); |
| 501 | return; |
| 502 | } |
| 503 | |
| 504 | spin_lock_init(&ps.lock); |
| 505 | init_completion(&ps.completion); |
| 506 | ps.job = job; |
| 507 | ps.nworks = padata_work_alloc_mt(nworks, &ps, &works); |
| 508 | ps.nworks_fini = 0; |
| 509 | |
| 510 | /* |
| 511 | * Chunk size is the amount of work a helper does per call to the |
| 512 | * thread function. Load balance large jobs between threads by |
| 513 | * increasing the number of chunks, guarantee at least the minimum |
| 514 | * chunk size from the caller, and honor the caller's alignment. |
| 515 | */ |
| 516 | ps.chunk_size = job->size / (ps.nworks * load_balance_factor); |
| 517 | ps.chunk_size = max(ps.chunk_size, job->min_chunk); |
| 518 | ps.chunk_size = roundup(ps.chunk_size, job->align); |
| 519 | |
| 520 | list_for_each_entry(pw, &works, pw_list) |
Gang Li Subject: padata: dispatch works on | eb52286 | 2024-03-06 13:04:17 -0800 | [diff] [blame] | 521 | if (job->numa_aware) { |
| 522 | int old_node = atomic_read(&last_used_nid); |
| 523 | |
| 524 | do { |
| 525 | nid = next_node_in(old_node, node_states[N_CPU]); |
| 526 | } while (!atomic_try_cmpxchg(&last_used_nid, &old_node, nid)); |
| 527 | queue_work_node(nid, system_unbound_wq, &pw->pw_work); |
| 528 | } else { |
| 529 | queue_work(system_unbound_wq, &pw->pw_work); |
| 530 | } |
Daniel Jordan | 004ed42 | 2020-06-03 15:59:43 -0700 | [diff] [blame] | 531 | |
| 532 | /* Use the current thread, which saves starting a workqueue worker. */ |
| 533 | padata_work_init(&my_work, padata_mt_helper, &ps, PADATA_WORK_ONSTACK); |
| 534 | padata_mt_helper(&my_work.pw_work); |
| 535 | |
| 536 | /* Wait for all the helpers to finish. */ |
| 537 | wait_for_completion(&ps.completion); |
| 538 | |
| 539 | destroy_work_on_stack(&my_work.pw_work); |
| 540 | padata_works_free(&works); |
| 541 | } |
| 542 | |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 543 | static void __padata_list_init(struct padata_list *pd_list) |
| 544 | { |
| 545 | INIT_LIST_HEAD(&pd_list->list); |
| 546 | spin_lock_init(&pd_list->lock); |
| 547 | } |
| 548 | |
| 549 | /* Initialize all percpu queues used by serial workers */ |
| 550 | static void padata_init_squeues(struct parallel_data *pd) |
| 551 | { |
| 552 | int cpu; |
| 553 | struct padata_serial_queue *squeue; |
| 554 | |
| 555 | for_each_cpu(cpu, pd->cpumask.cbcpu) { |
| 556 | squeue = per_cpu_ptr(pd->squeue, cpu); |
| 557 | squeue->pd = pd; |
| 558 | __padata_list_init(&squeue->serial); |
| 559 | INIT_WORK(&squeue->work, padata_serial_worker); |
| 560 | } |
| 561 | } |
| 562 | |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 563 | /* Initialize per-CPU reorder lists */ |
| 564 | static void padata_init_reorder_list(struct parallel_data *pd) |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 565 | { |
Daniel Jordan | c51636a | 2019-09-05 21:40:29 -0400 | [diff] [blame] | 566 | int cpu; |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 567 | struct padata_list *list; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 568 | |
Daniel Jordan | c51636a | 2019-09-05 21:40:29 -0400 | [diff] [blame] | 569 | for_each_cpu(cpu, pd->cpumask.pcpu) { |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 570 | list = per_cpu_ptr(pd->reorder_list, cpu); |
| 571 | __padata_list_init(list); |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 572 | } |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 573 | } |
| 574 | |
| 575 | /* Allocate and initialize the internal cpumask dependend resources. */ |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 576 | static struct parallel_data *padata_alloc_pd(struct padata_shell *ps) |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 577 | { |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 578 | struct padata_instance *pinst = ps->pinst; |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 579 | struct parallel_data *pd; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 580 | |
| 581 | pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL); |
| 582 | if (!pd) |
| 583 | goto err; |
| 584 | |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 585 | pd->reorder_list = alloc_percpu(struct padata_list); |
| 586 | if (!pd->reorder_list) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 587 | goto err_free_pd; |
| 588 | |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 589 | pd->squeue = alloc_percpu(struct padata_serial_queue); |
| 590 | if (!pd->squeue) |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 591 | goto err_free_reorder_list; |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 592 | |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 593 | pd->ps = ps; |
Daniel Jordan | cec00e6e | 2020-07-14 16:13:53 -0400 | [diff] [blame] | 594 | |
| 595 | if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL)) |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 596 | goto err_free_squeue; |
Daniel Jordan | cec00e6e | 2020-07-14 16:13:53 -0400 | [diff] [blame] | 597 | if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL)) |
| 598 | goto err_free_pcpu; |
| 599 | |
Daniel Jordan | d69e037 | 2020-07-14 16:13:54 -0400 | [diff] [blame] | 600 | cpumask_and(pd->cpumask.pcpu, pinst->cpumask.pcpu, cpu_online_mask); |
| 601 | cpumask_and(pd->cpumask.cbcpu, pinst->cpumask.cbcpu, cpu_online_mask); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 602 | |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 603 | padata_init_reorder_list(pd); |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 604 | padata_init_squeues(pd); |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 605 | pd->seq_nr = -1; |
Xiyu Yang | d5ee8e7 | 2021-07-20 11:05:11 -0400 | [diff] [blame] | 606 | refcount_set(&pd->refcnt, 1); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 607 | spin_lock_init(&pd->lock); |
Daniel Jordan | ec9c7d1 | 2019-08-08 12:05:35 -0400 | [diff] [blame] | 608 | pd->cpu = cpumask_first(pd->cpumask.pcpu); |
Herbert Xu | 6fc4dbc | 2019-07-18 23:01:46 +0800 | [diff] [blame] | 609 | INIT_WORK(&pd->reorder_work, invoke_padata_reorder); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 610 | |
| 611 | return pd; |
| 612 | |
Daniel Jordan | cec00e6e | 2020-07-14 16:13:53 -0400 | [diff] [blame] | 613 | err_free_pcpu: |
| 614 | free_cpumask_var(pd->cpumask.pcpu); |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 615 | err_free_squeue: |
| 616 | free_percpu(pd->squeue); |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 617 | err_free_reorder_list: |
| 618 | free_percpu(pd->reorder_list); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 619 | err_free_pd: |
| 620 | kfree(pd); |
| 621 | err: |
| 622 | return NULL; |
| 623 | } |
| 624 | |
| 625 | static void padata_free_pd(struct parallel_data *pd) |
| 626 | { |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 627 | free_cpumask_var(pd->cpumask.pcpu); |
| 628 | free_cpumask_var(pd->cpumask.cbcpu); |
Daniel Jordan | f601c72 | 2020-07-14 16:13:56 -0400 | [diff] [blame] | 629 | free_percpu(pd->reorder_list); |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 630 | free_percpu(pd->squeue); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 631 | kfree(pd); |
| 632 | } |
| 633 | |
Steffen Klassert | 4c87917 | 2010-07-07 15:30:10 +0200 | [diff] [blame] | 634 | static void __padata_start(struct padata_instance *pinst) |
| 635 | { |
| 636 | pinst->flags |= PADATA_INIT; |
| 637 | } |
| 638 | |
Steffen Klassert | ee83655 | 2010-07-07 15:30:47 +0200 | [diff] [blame] | 639 | static void __padata_stop(struct padata_instance *pinst) |
| 640 | { |
| 641 | if (!(pinst->flags & PADATA_INIT)) |
| 642 | return; |
| 643 | |
| 644 | pinst->flags &= ~PADATA_INIT; |
| 645 | |
| 646 | synchronize_rcu(); |
Steffen Klassert | ee83655 | 2010-07-07 15:30:47 +0200 | [diff] [blame] | 647 | } |
| 648 | |
Lucas De Marchi | 25985ed | 2011-03-30 22:57:33 -0300 | [diff] [blame] | 649 | /* Replace the internal control structure with a new one. */ |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 650 | static int padata_replace_one(struct padata_shell *ps) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 651 | { |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 652 | struct parallel_data *pd_new; |
| 653 | |
| 654 | pd_new = padata_alloc_pd(ps); |
| 655 | if (!pd_new) |
| 656 | return -ENOMEM; |
| 657 | |
| 658 | ps->opd = rcu_dereference_protected(ps->pd, 1); |
| 659 | rcu_assign_pointer(ps->pd, pd_new); |
| 660 | |
| 661 | return 0; |
| 662 | } |
| 663 | |
Daniel Jordan | 894c9ef | 2019-12-03 14:31:10 -0500 | [diff] [blame] | 664 | static int padata_replace(struct padata_instance *pinst) |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 665 | { |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 666 | struct padata_shell *ps; |
Daniel Jordan | 41ccdbf | 2020-02-10 13:11:00 -0500 | [diff] [blame] | 667 | int err = 0; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 668 | |
| 669 | pinst->flags |= PADATA_RESET; |
| 670 | |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 671 | list_for_each_entry(ps, &pinst->pslist, list) { |
| 672 | err = padata_replace_one(ps); |
| 673 | if (err) |
| 674 | break; |
| 675 | } |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 676 | |
| 677 | synchronize_rcu(); |
| 678 | |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 679 | list_for_each_entry_continue_reverse(ps, &pinst->pslist, list) |
Xiyu Yang | d5ee8e7 | 2021-07-20 11:05:11 -0400 | [diff] [blame] | 680 | if (refcount_dec_and_test(&ps->opd->refcnt)) |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 681 | padata_free_pd(ps->opd); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 682 | |
| 683 | pinst->flags &= ~PADATA_RESET; |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 684 | |
| 685 | return err; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 686 | } |
| 687 | |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 688 | /* If cpumask contains no active cpu, we mark the instance as invalid. */ |
| 689 | static bool padata_validate_cpumask(struct padata_instance *pinst, |
| 690 | const struct cpumask *cpumask) |
| 691 | { |
Steffen Klassert | 13614e0 | 2012-03-28 08:43:21 +0200 | [diff] [blame] | 692 | if (!cpumask_intersects(cpumask, cpu_online_mask)) { |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 693 | pinst->flags |= PADATA_INVALID; |
| 694 | return false; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 695 | } |
| 696 | |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 697 | pinst->flags &= ~PADATA_INVALID; |
| 698 | return true; |
| 699 | } |
| 700 | |
Steffen Klassert | 65ff577 | 2010-07-27 07:15:06 +0200 | [diff] [blame] | 701 | static int __padata_set_cpumasks(struct padata_instance *pinst, |
| 702 | cpumask_var_t pcpumask, |
| 703 | cpumask_var_t cbcpumask) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 704 | { |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 705 | int valid; |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 706 | int err; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 707 | |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 708 | valid = padata_validate_cpumask(pinst, pcpumask); |
| 709 | if (!valid) { |
| 710 | __padata_stop(pinst); |
| 711 | goto out_replace; |
| 712 | } |
| 713 | |
| 714 | valid = padata_validate_cpumask(pinst, cbcpumask); |
Steffen Klassert | b89661d | 2010-07-20 08:49:20 +0200 | [diff] [blame] | 715 | if (!valid) |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 716 | __padata_stop(pinst); |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 717 | |
Steffen Klassert | b89661d | 2010-07-20 08:49:20 +0200 | [diff] [blame] | 718 | out_replace: |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 719 | cpumask_copy(pinst->cpumask.pcpu, pcpumask); |
| 720 | cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 721 | |
Daniel Jordan | 894c9ef | 2019-12-03 14:31:10 -0500 | [diff] [blame] | 722 | err = padata_setup_cpumasks(pinst) ?: padata_replace(pinst); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 723 | |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 724 | if (valid) |
| 725 | __padata_start(pinst); |
| 726 | |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 727 | return err; |
Steffen Klassert | 65ff577 | 2010-07-27 07:15:06 +0200 | [diff] [blame] | 728 | } |
| 729 | |
| 730 | /** |
Daniel Jordan | bfcdcef8 | 2019-12-03 14:31:14 -0500 | [diff] [blame] | 731 | * padata_set_cpumask - Sets specified by @cpumask_type cpumask to the value |
| 732 | * equivalent to @cpumask. |
Steffen Klassert | 65ff577 | 2010-07-27 07:15:06 +0200 | [diff] [blame] | 733 | * @pinst: padata instance |
| 734 | * @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding |
| 735 | * to parallel and serial cpumasks respectively. |
| 736 | * @cpumask: the cpumask to use |
Daniel Jordan | bfcdcef8 | 2019-12-03 14:31:14 -0500 | [diff] [blame] | 737 | * |
| 738 | * Return: 0 on success or negative error code |
Steffen Klassert | 65ff577 | 2010-07-27 07:15:06 +0200 | [diff] [blame] | 739 | */ |
| 740 | int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, |
| 741 | cpumask_var_t cpumask) |
| 742 | { |
| 743 | struct cpumask *serial_mask, *parallel_mask; |
| 744 | int err = -EINVAL; |
| 745 | |
Sebastian Andrzej Siewior | 80771c8 | 2021-08-03 16:16:10 +0200 | [diff] [blame] | 746 | cpus_read_lock(); |
Daniel Jordan | 38228e8 | 2019-12-03 14:31:11 -0500 | [diff] [blame] | 747 | mutex_lock(&pinst->lock); |
Steffen Klassert | 65ff577 | 2010-07-27 07:15:06 +0200 | [diff] [blame] | 748 | |
| 749 | switch (cpumask_type) { |
| 750 | case PADATA_CPU_PARALLEL: |
| 751 | serial_mask = pinst->cpumask.cbcpu; |
| 752 | parallel_mask = cpumask; |
| 753 | break; |
| 754 | case PADATA_CPU_SERIAL: |
| 755 | parallel_mask = pinst->cpumask.pcpu; |
| 756 | serial_mask = cpumask; |
| 757 | break; |
| 758 | default: |
| 759 | goto out; |
| 760 | } |
| 761 | |
| 762 | err = __padata_set_cpumasks(pinst, parallel_mask, serial_mask); |
| 763 | |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 764 | out: |
| 765 | mutex_unlock(&pinst->lock); |
Sebastian Andrzej Siewior | 80771c8 | 2021-08-03 16:16:10 +0200 | [diff] [blame] | 766 | cpus_read_unlock(); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 767 | |
| 768 | return err; |
| 769 | } |
| 770 | EXPORT_SYMBOL(padata_set_cpumask); |
| 771 | |
Arnd Bergmann | 19d795b | 2016-05-19 17:09:59 -0700 | [diff] [blame] | 772 | #ifdef CONFIG_HOTPLUG_CPU |
| 773 | |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 774 | static int __padata_add_cpu(struct padata_instance *pinst, int cpu) |
| 775 | { |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 776 | int err = 0; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 777 | |
Steffen Klassert | 13614e0 | 2012-03-28 08:43:21 +0200 | [diff] [blame] | 778 | if (cpumask_test_cpu(cpu, cpu_online_mask)) { |
Daniel Jordan | 894c9ef | 2019-12-03 14:31:10 -0500 | [diff] [blame] | 779 | err = padata_replace(pinst); |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 780 | |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 781 | if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) && |
| 782 | padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 783 | __padata_start(pinst); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 784 | } |
| 785 | |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 786 | return err; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 787 | } |
| 788 | |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 789 | static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) |
| 790 | { |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 791 | int err = 0; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 792 | |
Daniel Jordan | 894c9ef | 2019-12-03 14:31:10 -0500 | [diff] [blame] | 793 | if (!cpumask_test_cpu(cpu, cpu_online_mask)) { |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 794 | if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) || |
Steffen Klassert | b89661d | 2010-07-20 08:49:20 +0200 | [diff] [blame] | 795 | !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 796 | __padata_stop(pinst); |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 797 | |
Daniel Jordan | 894c9ef | 2019-12-03 14:31:10 -0500 | [diff] [blame] | 798 | err = padata_replace(pinst); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 799 | } |
| 800 | |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 801 | return err; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 802 | } |
| 803 | |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 804 | static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu) |
| 805 | { |
| 806 | return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) || |
| 807 | cpumask_test_cpu(cpu, pinst->cpumask.cbcpu); |
| 808 | } |
| 809 | |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 810 | static int padata_cpu_online(unsigned int cpu, struct hlist_node *node) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 811 | { |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 812 | struct padata_instance *pinst; |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 813 | int ret; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 814 | |
Daniel Jordan | 3c2214b | 2020-04-21 12:34:55 -0400 | [diff] [blame] | 815 | pinst = hlist_entry_safe(node, struct padata_instance, cpu_online_node); |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 816 | if (!pinst_has_cpu(pinst, cpu)) |
| 817 | return 0; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 818 | |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 819 | mutex_lock(&pinst->lock); |
| 820 | ret = __padata_add_cpu(pinst, cpu); |
| 821 | mutex_unlock(&pinst->lock); |
| 822 | return ret; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 823 | } |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 824 | |
Daniel Jordan | 894c9ef | 2019-12-03 14:31:10 -0500 | [diff] [blame] | 825 | static int padata_cpu_dead(unsigned int cpu, struct hlist_node *node) |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 826 | { |
| 827 | struct padata_instance *pinst; |
| 828 | int ret; |
| 829 | |
Daniel Jordan | 3c2214b | 2020-04-21 12:34:55 -0400 | [diff] [blame] | 830 | pinst = hlist_entry_safe(node, struct padata_instance, cpu_dead_node); |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 831 | if (!pinst_has_cpu(pinst, cpu)) |
| 832 | return 0; |
| 833 | |
| 834 | mutex_lock(&pinst->lock); |
| 835 | ret = __padata_remove_cpu(pinst, cpu); |
| 836 | mutex_unlock(&pinst->lock); |
| 837 | return ret; |
| 838 | } |
| 839 | |
| 840 | static enum cpuhp_state hp_online; |
Steffen Klassert | e2cb2f1 | 2010-04-29 14:40:10 +0200 | [diff] [blame] | 841 | #endif |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 842 | |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 843 | static void __padata_free(struct padata_instance *pinst) |
| 844 | { |
| 845 | #ifdef CONFIG_HOTPLUG_CPU |
Daniel Jordan | 3c2214b | 2020-04-21 12:34:55 -0400 | [diff] [blame] | 846 | cpuhp_state_remove_instance_nocalls(CPUHP_PADATA_DEAD, |
| 847 | &pinst->cpu_dead_node); |
| 848 | cpuhp_state_remove_instance_nocalls(hp_online, &pinst->cpu_online_node); |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 849 | #endif |
| 850 | |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 851 | WARN_ON(!list_empty(&pinst->pslist)); |
| 852 | |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 853 | free_cpumask_var(pinst->cpumask.pcpu); |
| 854 | free_cpumask_var(pinst->cpumask.cbcpu); |
Daniel Jordan | 45d153c | 2019-09-05 21:40:27 -0400 | [diff] [blame] | 855 | destroy_workqueue(pinst->serial_wq); |
| 856 | destroy_workqueue(pinst->parallel_wq); |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 857 | kfree(pinst); |
| 858 | } |
| 859 | |
| 860 | #define kobj2pinst(_kobj) \ |
| 861 | container_of(_kobj, struct padata_instance, kobj) |
| 862 | #define attr2pentry(_attr) \ |
| 863 | container_of(_attr, struct padata_sysfs_entry, attr) |
| 864 | |
| 865 | static void padata_sysfs_release(struct kobject *kobj) |
| 866 | { |
| 867 | struct padata_instance *pinst = kobj2pinst(kobj); |
| 868 | __padata_free(pinst); |
| 869 | } |
| 870 | |
| 871 | struct padata_sysfs_entry { |
| 872 | struct attribute attr; |
| 873 | ssize_t (*show)(struct padata_instance *, struct attribute *, char *); |
| 874 | ssize_t (*store)(struct padata_instance *, struct attribute *, |
| 875 | const char *, size_t); |
| 876 | }; |
| 877 | |
| 878 | static ssize_t show_cpumask(struct padata_instance *pinst, |
| 879 | struct attribute *attr, char *buf) |
| 880 | { |
| 881 | struct cpumask *cpumask; |
| 882 | ssize_t len; |
| 883 | |
| 884 | mutex_lock(&pinst->lock); |
| 885 | if (!strcmp(attr->name, "serial_cpumask")) |
| 886 | cpumask = pinst->cpumask.cbcpu; |
| 887 | else |
| 888 | cpumask = pinst->cpumask.pcpu; |
| 889 | |
Tejun Heo | 4497da6 | 2015-02-13 14:38:05 -0800 | [diff] [blame] | 890 | len = snprintf(buf, PAGE_SIZE, "%*pb\n", |
| 891 | nr_cpu_ids, cpumask_bits(cpumask)); |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 892 | mutex_unlock(&pinst->lock); |
Tejun Heo | 4497da6 | 2015-02-13 14:38:05 -0800 | [diff] [blame] | 893 | return len < PAGE_SIZE ? len : -EINVAL; |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 894 | } |
| 895 | |
| 896 | static ssize_t store_cpumask(struct padata_instance *pinst, |
| 897 | struct attribute *attr, |
| 898 | const char *buf, size_t count) |
| 899 | { |
| 900 | cpumask_var_t new_cpumask; |
| 901 | ssize_t ret; |
| 902 | int mask_type; |
| 903 | |
| 904 | if (!alloc_cpumask_var(&new_cpumask, GFP_KERNEL)) |
| 905 | return -ENOMEM; |
| 906 | |
| 907 | ret = bitmap_parse(buf, count, cpumask_bits(new_cpumask), |
| 908 | nr_cpumask_bits); |
| 909 | if (ret < 0) |
| 910 | goto out; |
| 911 | |
| 912 | mask_type = !strcmp(attr->name, "serial_cpumask") ? |
| 913 | PADATA_CPU_SERIAL : PADATA_CPU_PARALLEL; |
| 914 | ret = padata_set_cpumask(pinst, mask_type, new_cpumask); |
| 915 | if (!ret) |
| 916 | ret = count; |
| 917 | |
| 918 | out: |
| 919 | free_cpumask_var(new_cpumask); |
| 920 | return ret; |
| 921 | } |
| 922 | |
| 923 | #define PADATA_ATTR_RW(_name, _show_name, _store_name) \ |
| 924 | static struct padata_sysfs_entry _name##_attr = \ |
| 925 | __ATTR(_name, 0644, _show_name, _store_name) |
| 926 | #define PADATA_ATTR_RO(_name, _show_name) \ |
| 927 | static struct padata_sysfs_entry _name##_attr = \ |
| 928 | __ATTR(_name, 0400, _show_name, NULL) |
| 929 | |
| 930 | PADATA_ATTR_RW(serial_cpumask, show_cpumask, store_cpumask); |
| 931 | PADATA_ATTR_RW(parallel_cpumask, show_cpumask, store_cpumask); |
| 932 | |
| 933 | /* |
| 934 | * Padata sysfs provides the following objects: |
| 935 | * serial_cpumask [RW] - cpumask for serial workers |
| 936 | * parallel_cpumask [RW] - cpumask for parallel workers |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 937 | */ |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 938 | static struct attribute *padata_default_attrs[] = { |
| 939 | &serial_cpumask_attr.attr, |
| 940 | ¶llel_cpumask_attr.attr, |
| 941 | NULL, |
| 942 | }; |
Kimberly Brown | 2064fbc | 2019-04-01 22:51:47 -0400 | [diff] [blame] | 943 | ATTRIBUTE_GROUPS(padata_default); |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 944 | |
| 945 | static ssize_t padata_sysfs_show(struct kobject *kobj, |
| 946 | struct attribute *attr, char *buf) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 947 | { |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 948 | struct padata_instance *pinst; |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 949 | struct padata_sysfs_entry *pentry; |
| 950 | ssize_t ret = -EIO; |
| 951 | |
| 952 | pinst = kobj2pinst(kobj); |
| 953 | pentry = attr2pentry(attr); |
| 954 | if (pentry->show) |
| 955 | ret = pentry->show(pinst, attr, buf); |
| 956 | |
| 957 | return ret; |
| 958 | } |
| 959 | |
| 960 | static ssize_t padata_sysfs_store(struct kobject *kobj, struct attribute *attr, |
| 961 | const char *buf, size_t count) |
| 962 | { |
| 963 | struct padata_instance *pinst; |
| 964 | struct padata_sysfs_entry *pentry; |
| 965 | ssize_t ret = -EIO; |
| 966 | |
| 967 | pinst = kobj2pinst(kobj); |
| 968 | pentry = attr2pentry(attr); |
| 969 | if (pentry->show) |
| 970 | ret = pentry->store(pinst, attr, buf, count); |
| 971 | |
| 972 | return ret; |
| 973 | } |
| 974 | |
| 975 | static const struct sysfs_ops padata_sysfs_ops = { |
| 976 | .show = padata_sysfs_show, |
| 977 | .store = padata_sysfs_store, |
| 978 | }; |
| 979 | |
Thomas Weißschuh | 0bedc99 | 2023-02-17 03:17:49 +0000 | [diff] [blame] | 980 | static const struct kobj_type padata_attr_type = { |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 981 | .sysfs_ops = &padata_sysfs_ops, |
Kimberly Brown | 2064fbc | 2019-04-01 22:51:47 -0400 | [diff] [blame] | 982 | .default_groups = padata_default_groups, |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 983 | .release = padata_sysfs_release, |
| 984 | }; |
| 985 | |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 986 | /** |
Daniel Jordan | 3f25719 | 2020-07-14 16:13:55 -0400 | [diff] [blame] | 987 | * padata_alloc - allocate and initialize a padata instance |
Daniel Jordan | b128a30 | 2019-09-05 21:40:21 -0400 | [diff] [blame] | 988 | * @name: used to identify the instance |
Daniel Jordan | bfcdcef8 | 2019-12-03 14:31:14 -0500 | [diff] [blame] | 989 | * |
| 990 | * Return: new instance on success, NULL on error |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 991 | */ |
Daniel Jordan | 3f25719 | 2020-07-14 16:13:55 -0400 | [diff] [blame] | 992 | struct padata_instance *padata_alloc(const char *name) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 993 | { |
| 994 | struct padata_instance *pinst; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 995 | |
| 996 | pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL); |
| 997 | if (!pinst) |
| 998 | goto err; |
| 999 | |
Daniel Jordan | bfde23c | 2019-09-05 21:40:28 -0400 | [diff] [blame] | 1000 | pinst->parallel_wq = alloc_workqueue("%s_parallel", WQ_UNBOUND, 0, |
| 1001 | name); |
Daniel Jordan | 45d153c | 2019-09-05 21:40:27 -0400 | [diff] [blame] | 1002 | if (!pinst->parallel_wq) |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 1003 | goto err_free_inst; |
Daniel Jordan | b128a30 | 2019-09-05 21:40:21 -0400 | [diff] [blame] | 1004 | |
Sebastian Andrzej Siewior | 80771c8 | 2021-08-03 16:16:10 +0200 | [diff] [blame] | 1005 | cpus_read_lock(); |
Daniel Jordan | cc491d8 | 2019-09-05 21:40:26 -0400 | [diff] [blame] | 1006 | |
Daniel Jordan | 45d153c | 2019-09-05 21:40:27 -0400 | [diff] [blame] | 1007 | pinst->serial_wq = alloc_workqueue("%s_serial", WQ_MEM_RECLAIM | |
| 1008 | WQ_CPU_INTENSIVE, 1, name); |
| 1009 | if (!pinst->serial_wq) |
Daniel Jordan | cc491d8 | 2019-09-05 21:40:26 -0400 | [diff] [blame] | 1010 | goto err_put_cpus; |
Daniel Jordan | 45d153c | 2019-09-05 21:40:27 -0400 | [diff] [blame] | 1011 | |
| 1012 | if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL)) |
| 1013 | goto err_free_serial_wq; |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 1014 | if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) { |
| 1015 | free_cpumask_var(pinst->cpumask.pcpu); |
Daniel Jordan | 45d153c | 2019-09-05 21:40:27 -0400 | [diff] [blame] | 1016 | goto err_free_serial_wq; |
Steffen Klassert | 33e5445 | 2010-07-07 15:31:26 +0200 | [diff] [blame] | 1017 | } |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 1018 | |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 1019 | INIT_LIST_HEAD(&pinst->pslist); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 1020 | |
Daniel Jordan | 3f25719 | 2020-07-14 16:13:55 -0400 | [diff] [blame] | 1021 | cpumask_copy(pinst->cpumask.pcpu, cpu_possible_mask); |
| 1022 | cpumask_copy(pinst->cpumask.cbcpu, cpu_possible_mask); |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 1023 | |
| 1024 | if (padata_setup_cpumasks(pinst)) |
Daniel Jordan | d69e037 | 2020-07-14 16:13:54 -0400 | [diff] [blame] | 1025 | goto err_free_masks; |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 1026 | |
Daniel Jordan | bd25b488 | 2020-07-14 16:13:51 -0400 | [diff] [blame] | 1027 | __padata_start(pinst); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 1028 | |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 1029 | kobject_init(&pinst->kobj, &padata_attr_type); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 1030 | mutex_init(&pinst->lock); |
| 1031 | |
Richard Weinberger | b8b4a41 | 2013-08-23 13:12:33 +0200 | [diff] [blame] | 1032 | #ifdef CONFIG_HOTPLUG_CPU |
Daniel Jordan | 3c2214b | 2020-04-21 12:34:55 -0400 | [diff] [blame] | 1033 | cpuhp_state_add_instance_nocalls_cpuslocked(hp_online, |
| 1034 | &pinst->cpu_online_node); |
Daniel Jordan | 894c9ef | 2019-12-03 14:31:10 -0500 | [diff] [blame] | 1035 | cpuhp_state_add_instance_nocalls_cpuslocked(CPUHP_PADATA_DEAD, |
Daniel Jordan | 3c2214b | 2020-04-21 12:34:55 -0400 | [diff] [blame] | 1036 | &pinst->cpu_dead_node); |
Richard Weinberger | b8b4a41 | 2013-08-23 13:12:33 +0200 | [diff] [blame] | 1037 | #endif |
Daniel Jordan | cc491d8 | 2019-09-05 21:40:26 -0400 | [diff] [blame] | 1038 | |
Sebastian Andrzej Siewior | 80771c8 | 2021-08-03 16:16:10 +0200 | [diff] [blame] | 1039 | cpus_read_unlock(); |
Daniel Jordan | cc491d8 | 2019-09-05 21:40:26 -0400 | [diff] [blame] | 1040 | |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 1041 | return pinst; |
| 1042 | |
Dan Kruchinin | e15bacb | 2010-07-14 14:31:57 +0400 | [diff] [blame] | 1043 | err_free_masks: |
| 1044 | free_cpumask_var(pinst->cpumask.pcpu); |
| 1045 | free_cpumask_var(pinst->cpumask.cbcpu); |
Daniel Jordan | 45d153c | 2019-09-05 21:40:27 -0400 | [diff] [blame] | 1046 | err_free_serial_wq: |
| 1047 | destroy_workqueue(pinst->serial_wq); |
Daniel Jordan | cc491d8 | 2019-09-05 21:40:26 -0400 | [diff] [blame] | 1048 | err_put_cpus: |
Sebastian Andrzej Siewior | 80771c8 | 2021-08-03 16:16:10 +0200 | [diff] [blame] | 1049 | cpus_read_unlock(); |
Daniel Jordan | 45d153c | 2019-09-05 21:40:27 -0400 | [diff] [blame] | 1050 | destroy_workqueue(pinst->parallel_wq); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 1051 | err_free_inst: |
| 1052 | kfree(pinst); |
| 1053 | err: |
| 1054 | return NULL; |
| 1055 | } |
Daniel Jordan | 3f25719 | 2020-07-14 16:13:55 -0400 | [diff] [blame] | 1056 | EXPORT_SYMBOL(padata_alloc); |
Thomas Gleixner | 9596695 | 2017-05-24 10:15:17 +0200 | [diff] [blame] | 1057 | |
| 1058 | /** |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 1059 | * padata_free - free a padata instance |
| 1060 | * |
Daniel Jordan | bfcdcef8 | 2019-12-03 14:31:14 -0500 | [diff] [blame] | 1061 | * @pinst: padata instance to free |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 1062 | */ |
| 1063 | void padata_free(struct padata_instance *pinst) |
| 1064 | { |
Dan Kruchinin | 5e017dc | 2010-07-14 14:33:08 +0400 | [diff] [blame] | 1065 | kobject_put(&pinst->kobj); |
Steffen Klassert | 16295be | 2010-01-06 19:47:10 +1100 | [diff] [blame] | 1066 | } |
| 1067 | EXPORT_SYMBOL(padata_free); |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 1068 | |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 1069 | /** |
| 1070 | * padata_alloc_shell - Allocate and initialize padata shell. |
| 1071 | * |
| 1072 | * @pinst: Parent padata_instance object. |
Daniel Jordan | bfcdcef8 | 2019-12-03 14:31:14 -0500 | [diff] [blame] | 1073 | * |
| 1074 | * Return: new shell on success, NULL on error |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 1075 | */ |
| 1076 | struct padata_shell *padata_alloc_shell(struct padata_instance *pinst) |
| 1077 | { |
| 1078 | struct parallel_data *pd; |
| 1079 | struct padata_shell *ps; |
| 1080 | |
| 1081 | ps = kzalloc(sizeof(*ps), GFP_KERNEL); |
| 1082 | if (!ps) |
| 1083 | goto out; |
| 1084 | |
| 1085 | ps->pinst = pinst; |
| 1086 | |
Sebastian Andrzej Siewior | 80771c8 | 2021-08-03 16:16:10 +0200 | [diff] [blame] | 1087 | cpus_read_lock(); |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 1088 | pd = padata_alloc_pd(ps); |
Sebastian Andrzej Siewior | 80771c8 | 2021-08-03 16:16:10 +0200 | [diff] [blame] | 1089 | cpus_read_unlock(); |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 1090 | |
| 1091 | if (!pd) |
| 1092 | goto out_free_ps; |
| 1093 | |
| 1094 | mutex_lock(&pinst->lock); |
| 1095 | RCU_INIT_POINTER(ps->pd, pd); |
| 1096 | list_add(&ps->list, &pinst->pslist); |
| 1097 | mutex_unlock(&pinst->lock); |
| 1098 | |
| 1099 | return ps; |
| 1100 | |
| 1101 | out_free_ps: |
| 1102 | kfree(ps); |
| 1103 | out: |
| 1104 | return NULL; |
| 1105 | } |
| 1106 | EXPORT_SYMBOL(padata_alloc_shell); |
| 1107 | |
| 1108 | /** |
| 1109 | * padata_free_shell - free a padata shell |
| 1110 | * |
| 1111 | * @ps: padata shell to free |
| 1112 | */ |
| 1113 | void padata_free_shell(struct padata_shell *ps) |
| 1114 | { |
WangJinchao | 7ddc21e | 2023-10-16 09:15:21 +0800 | [diff] [blame] | 1115 | struct parallel_data *pd; |
| 1116 | |
Eric Biggers | 07b24c7 | 2020-02-25 20:59:22 -0800 | [diff] [blame] | 1117 | if (!ps) |
| 1118 | return; |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 1119 | |
Eric Biggers | 07b24c7 | 2020-02-25 20:59:22 -0800 | [diff] [blame] | 1120 | mutex_lock(&ps->pinst->lock); |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 1121 | list_del(&ps->list); |
WangJinchao | 7ddc21e | 2023-10-16 09:15:21 +0800 | [diff] [blame] | 1122 | pd = rcu_dereference_protected(ps->pd, 1); |
| 1123 | if (refcount_dec_and_test(&pd->refcnt)) |
| 1124 | padata_free_pd(pd); |
Eric Biggers | 07b24c7 | 2020-02-25 20:59:22 -0800 | [diff] [blame] | 1125 | mutex_unlock(&ps->pinst->lock); |
Herbert Xu | bbefa1d | 2019-11-26 15:58:45 +0800 | [diff] [blame] | 1126 | |
| 1127 | kfree(ps); |
| 1128 | } |
| 1129 | EXPORT_SYMBOL(padata_free_shell); |
| 1130 | |
Daniel Jordan | f1b192b | 2020-06-03 15:59:35 -0700 | [diff] [blame] | 1131 | void __init padata_init(void) |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 1132 | { |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 1133 | unsigned int i, possible_cpus; |
Daniel Jordan | f1b192b | 2020-06-03 15:59:35 -0700 | [diff] [blame] | 1134 | #ifdef CONFIG_HOTPLUG_CPU |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 1135 | int ret; |
| 1136 | |
| 1137 | ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "padata:online", |
Daniel Jordan | 894c9ef | 2019-12-03 14:31:10 -0500 | [diff] [blame] | 1138 | padata_cpu_online, NULL); |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 1139 | if (ret < 0) |
Daniel Jordan | f1b192b | 2020-06-03 15:59:35 -0700 | [diff] [blame] | 1140 | goto err; |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 1141 | hp_online = ret; |
Daniel Jordan | 894c9ef | 2019-12-03 14:31:10 -0500 | [diff] [blame] | 1142 | |
| 1143 | ret = cpuhp_setup_state_multi(CPUHP_PADATA_DEAD, "padata:dead", |
| 1144 | NULL, padata_cpu_dead); |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 1145 | if (ret < 0) |
| 1146 | goto remove_online_state; |
| 1147 | #endif |
| 1148 | |
| 1149 | possible_cpus = num_possible_cpus(); |
| 1150 | padata_works = kmalloc_array(possible_cpus, sizeof(struct padata_work), |
| 1151 | GFP_KERNEL); |
| 1152 | if (!padata_works) |
| 1153 | goto remove_dead_state; |
| 1154 | |
| 1155 | for (i = 0; i < possible_cpus; ++i) |
| 1156 | list_add(&padata_works[i].pw_list, &padata_free_works); |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 1157 | |
Daniel Jordan | f1b192b | 2020-06-03 15:59:35 -0700 | [diff] [blame] | 1158 | return; |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 1159 | |
| 1160 | remove_dead_state: |
| 1161 | #ifdef CONFIG_HOTPLUG_CPU |
| 1162 | cpuhp_remove_multi_state(CPUHP_PADATA_DEAD); |
| 1163 | remove_online_state: |
| 1164 | cpuhp_remove_multi_state(hp_online); |
Daniel Jordan | f1b192b | 2020-06-03 15:59:35 -0700 | [diff] [blame] | 1165 | err: |
Sebastian Andrzej Siewior | 30e9215 | 2016-09-06 19:04:49 +0200 | [diff] [blame] | 1166 | #endif |
Daniel Jordan | 4611ce2 | 2020-06-03 15:59:39 -0700 | [diff] [blame] | 1167 | pr_warn("padata: initialization failed\n"); |
Daniel Jordan | f1b192b | 2020-06-03 15:59:35 -0700 | [diff] [blame] | 1168 | } |