Thomas Gleixner | 457c899 | 2019-05-19 13:08:55 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Frederic Weisbecker | 7863406 | 2017-10-27 04:42:28 +0200 | [diff] [blame] | 2 | /* |
| 3 | * Housekeeping management. Manage the targets for routine code that can run on |
| 4 | * any CPU: unbound workqueues, timers, kthreads and any offloadable work. |
| 5 | * |
| 6 | * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker |
Frederic Weisbecker | 1bda3f8 | 2018-02-21 05:17:26 +0100 | [diff] [blame] | 7 | * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker |
Frederic Weisbecker | 7863406 | 2017-10-27 04:42:28 +0200 | [diff] [blame] | 8 | * |
| 9 | */ |
Frederic Weisbecker | 7863406 | 2017-10-27 04:42:28 +0200 | [diff] [blame] | 10 | |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 11 | enum hk_flags { |
| 12 | HK_FLAG_TIMER = BIT(HK_TYPE_TIMER), |
| 13 | HK_FLAG_RCU = BIT(HK_TYPE_RCU), |
| 14 | HK_FLAG_MISC = BIT(HK_TYPE_MISC), |
| 15 | HK_FLAG_SCHED = BIT(HK_TYPE_SCHED), |
| 16 | HK_FLAG_TICK = BIT(HK_TYPE_TICK), |
| 17 | HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN), |
| 18 | HK_FLAG_WQ = BIT(HK_TYPE_WQ), |
| 19 | HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ), |
| 20 | HK_FLAG_KTHREAD = BIT(HK_TYPE_KTHREAD), |
| 21 | }; |
| 22 | |
Ingo Molnar | dfcb245 | 2018-12-03 10:05:56 +0100 | [diff] [blame] | 23 | DEFINE_STATIC_KEY_FALSE(housekeeping_overridden); |
| 24 | EXPORT_SYMBOL_GPL(housekeeping_overridden); |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 25 | |
| 26 | struct housekeeping { |
| 27 | cpumask_var_t cpumasks[HK_TYPE_MAX]; |
| 28 | unsigned long flags; |
| 29 | }; |
| 30 | |
| 31 | static struct housekeeping housekeeping; |
Frederic Weisbecker | 7e56a1c | 2017-10-27 04:42:31 +0200 | [diff] [blame] | 32 | |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 33 | bool housekeeping_enabled(enum hk_type type) |
Wanpeng Li | 0c5f81d | 2019-07-06 09:26:51 +0800 | [diff] [blame] | 34 | { |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 35 | return !!(housekeeping.flags & BIT(type)); |
Wanpeng Li | 0c5f81d | 2019-07-06 09:26:51 +0800 | [diff] [blame] | 36 | } |
| 37 | EXPORT_SYMBOL_GPL(housekeeping_enabled); |
| 38 | |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 39 | int housekeeping_any_cpu(enum hk_type type) |
Frederic Weisbecker | 7e56a1c | 2017-10-27 04:42:31 +0200 | [diff] [blame] | 40 | { |
Wanpeng Li | e0e8d49 | 2019-06-28 16:51:41 +0800 | [diff] [blame] | 41 | int cpu; |
| 42 | |
| 43 | if (static_branch_unlikely(&housekeeping_overridden)) { |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 44 | if (housekeeping.flags & BIT(type)) { |
| 45 | cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id()); |
Wanpeng Li | e0e8d49 | 2019-06-28 16:51:41 +0800 | [diff] [blame] | 46 | if (cpu < nr_cpu_ids) |
| 47 | return cpu; |
| 48 | |
Oleg Nesterov | 5097cbc | 2024-04-11 16:39:05 +0200 | [diff] [blame] | 49 | cpu = cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask); |
| 50 | if (likely(cpu < nr_cpu_ids)) |
| 51 | return cpu; |
| 52 | /* |
| 53 | * Unless we have another problem this can only happen |
| 54 | * at boot time before start_secondary() brings the 1st |
| 55 | * housekeeping CPU up. |
| 56 | */ |
| 57 | WARN_ON_ONCE(system_state == SYSTEM_RUNNING || |
| 58 | type != HK_TYPE_TIMER); |
Wanpeng Li | e0e8d49 | 2019-06-28 16:51:41 +0800 | [diff] [blame] | 59 | } |
| 60 | } |
Frederic Weisbecker | 7e56a1c | 2017-10-27 04:42:31 +0200 | [diff] [blame] | 61 | return smp_processor_id(); |
| 62 | } |
| 63 | EXPORT_SYMBOL_GPL(housekeeping_any_cpu); |
| 64 | |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 65 | const struct cpumask *housekeeping_cpumask(enum hk_type type) |
Frederic Weisbecker | 7e56a1c | 2017-10-27 04:42:31 +0200 | [diff] [blame] | 66 | { |
Ingo Molnar | dfcb245 | 2018-12-03 10:05:56 +0100 | [diff] [blame] | 67 | if (static_branch_unlikely(&housekeeping_overridden)) |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 68 | if (housekeeping.flags & BIT(type)) |
| 69 | return housekeeping.cpumasks[type]; |
Frederic Weisbecker | 7e56a1c | 2017-10-27 04:42:31 +0200 | [diff] [blame] | 70 | return cpu_possible_mask; |
| 71 | } |
| 72 | EXPORT_SYMBOL_GPL(housekeeping_cpumask); |
| 73 | |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 74 | void housekeeping_affine(struct task_struct *t, enum hk_type type) |
Frederic Weisbecker | 7e56a1c | 2017-10-27 04:42:31 +0200 | [diff] [blame] | 75 | { |
Ingo Molnar | dfcb245 | 2018-12-03 10:05:56 +0100 | [diff] [blame] | 76 | if (static_branch_unlikely(&housekeeping_overridden)) |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 77 | if (housekeeping.flags & BIT(type)) |
| 78 | set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]); |
Frederic Weisbecker | 7e56a1c | 2017-10-27 04:42:31 +0200 | [diff] [blame] | 79 | } |
| 80 | EXPORT_SYMBOL_GPL(housekeeping_affine); |
| 81 | |
Frederic Weisbecker | 04d4e66 | 2022-02-07 16:59:06 +0100 | [diff] [blame] | 82 | bool housekeeping_test_cpu(int cpu, enum hk_type type) |
Frederic Weisbecker | 7e56a1c | 2017-10-27 04:42:31 +0200 | [diff] [blame] | 83 | { |
Ingo Molnar | dfcb245 | 2018-12-03 10:05:56 +0100 | [diff] [blame] | 84 | if (static_branch_unlikely(&housekeeping_overridden)) |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 85 | if (housekeeping.flags & BIT(type)) |
| 86 | return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]); |
Frederic Weisbecker | 7e56a1c | 2017-10-27 04:42:31 +0200 | [diff] [blame] | 87 | return true; |
| 88 | } |
| 89 | EXPORT_SYMBOL_GPL(housekeeping_test_cpu); |
Frederic Weisbecker | 7863406 | 2017-10-27 04:42:28 +0200 | [diff] [blame] | 90 | |
| 91 | void __init housekeeping_init(void) |
| 92 | { |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 93 | enum hk_type type; |
| 94 | |
| 95 | if (!housekeeping.flags) |
Frederic Weisbecker | 7863406 | 2017-10-27 04:42:28 +0200 | [diff] [blame] | 96 | return; |
| 97 | |
Ingo Molnar | dfcb245 | 2018-12-03 10:05:56 +0100 | [diff] [blame] | 98 | static_branch_enable(&housekeeping_overridden); |
Frederic Weisbecker | e179f5a | 2017-10-27 04:42:32 +0200 | [diff] [blame] | 99 | |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 100 | if (housekeeping.flags & HK_FLAG_TICK) |
Frederic Weisbecker | d84b313 | 2018-02-21 05:17:27 +0100 | [diff] [blame] | 101 | sched_tick_offload_init(); |
| 102 | |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 103 | for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) { |
| 104 | /* We need at least one CPU to handle housekeeping work */ |
| 105 | WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type])); |
| 106 | } |
Frederic Weisbecker | 7863406 | 2017-10-27 04:42:28 +0200 | [diff] [blame] | 107 | } |
Frederic Weisbecker | 6f1982f | 2017-10-27 04:42:36 +0200 | [diff] [blame] | 108 | |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 109 | static void __init housekeeping_setup_type(enum hk_type type, |
| 110 | cpumask_var_t housekeeping_staging) |
| 111 | { |
| 112 | |
| 113 | alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]); |
| 114 | cpumask_copy(housekeeping.cpumasks[type], |
| 115 | housekeeping_staging); |
| 116 | } |
| 117 | |
| 118 | static int __init housekeeping_setup(char *str, unsigned long flags) |
Frederic Weisbecker | 6f1982f | 2017-10-27 04:42:36 +0200 | [diff] [blame] | 119 | { |
Frederic Weisbecker | 6367b60 | 2022-02-07 16:59:07 +0100 | [diff] [blame] | 120 | cpumask_var_t non_housekeeping_mask, housekeeping_staging; |
Oleg Nesterov | 257bf89 | 2024-04-13 16:17:46 +0200 | [diff] [blame] | 121 | unsigned int first_cpu; |
Frederic Weisbecker | 0cd3e59 | 2022-02-07 16:59:08 +0100 | [diff] [blame] | 122 | int err = 0; |
Frederic Weisbecker | 6f1982f | 2017-10-27 04:42:36 +0200 | [diff] [blame] | 123 | |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 124 | if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) { |
Frederic Weisbecker | 65e53f8 | 2022-02-07 16:59:09 +0100 | [diff] [blame] | 125 | if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) { |
| 126 | pr_warn("Housekeeping: nohz unsupported." |
| 127 | " Build with CONFIG_NO_HZ_FULL\n"); |
| 128 | return 0; |
| 129 | } |
| 130 | } |
| 131 | |
Frederic Weisbecker | 6f1982f | 2017-10-27 04:42:36 +0200 | [diff] [blame] | 132 | alloc_bootmem_cpumask_var(&non_housekeeping_mask); |
Paul Gortmaker | 915a2bc | 2021-04-19 00:26:59 -0400 | [diff] [blame] | 133 | if (cpulist_parse(str, non_housekeeping_mask) < 0) { |
Frederic Weisbecker | edb9382 | 2017-10-27 04:42:37 +0200 | [diff] [blame] | 134 | pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n"); |
Frederic Weisbecker | 0cd3e59 | 2022-02-07 16:59:08 +0100 | [diff] [blame] | 135 | goto free_non_housekeeping_mask; |
Frederic Weisbecker | 6f1982f | 2017-10-27 04:42:36 +0200 | [diff] [blame] | 136 | } |
| 137 | |
Frederic Weisbecker | 6367b60 | 2022-02-07 16:59:07 +0100 | [diff] [blame] | 138 | alloc_bootmem_cpumask_var(&housekeeping_staging); |
| 139 | cpumask_andnot(housekeeping_staging, |
| 140 | cpu_possible_mask, non_housekeeping_mask); |
Frederic Weisbecker | 6f1982f | 2017-10-27 04:42:36 +0200 | [diff] [blame] | 141 | |
Oleg Nesterov | 257bf89 | 2024-04-13 16:17:46 +0200 | [diff] [blame] | 142 | first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging); |
| 143 | if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) { |
Frederic Weisbecker | 6367b60 | 2022-02-07 16:59:07 +0100 | [diff] [blame] | 144 | __cpumask_set_cpu(smp_processor_id(), housekeeping_staging); |
| 145 | __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask); |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 146 | if (!housekeeping.flags) { |
Nicholas Piggin | 9219565 | 2019-04-11 13:34:47 +1000 | [diff] [blame] | 147 | pr_warn("Housekeeping: must include one present CPU, " |
| 148 | "using boot CPU:%d\n", smp_processor_id()); |
Nicholas Piggin | 9219565 | 2019-04-11 13:34:47 +1000 | [diff] [blame] | 149 | } |
Frederic Weisbecker | 6367b60 | 2022-02-07 16:59:07 +0100 | [diff] [blame] | 150 | } |
| 151 | |
Oleg Nesterov | 257bf89 | 2024-04-13 16:17:46 +0200 | [diff] [blame] | 152 | if (cpumask_empty(non_housekeeping_mask)) |
| 153 | goto free_housekeeping_staging; |
| 154 | |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 155 | if (!housekeeping.flags) { |
| 156 | /* First setup call ("nohz_full=" or "isolcpus=") */ |
| 157 | enum hk_type type; |
| 158 | |
| 159 | for_each_set_bit(type, &flags, HK_TYPE_MAX) |
| 160 | housekeeping_setup_type(type, housekeeping_staging); |
Nicholas Piggin | 9219565 | 2019-04-11 13:34:47 +1000 | [diff] [blame] | 161 | } else { |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 162 | /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */ |
| 163 | enum hk_type type; |
| 164 | unsigned long iter_flags = flags & housekeeping.flags; |
| 165 | |
| 166 | for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) { |
| 167 | if (!cpumask_equal(housekeeping_staging, |
| 168 | housekeeping.cpumasks[type])) { |
| 169 | pr_warn("Housekeeping: nohz_full= must match isolcpus=\n"); |
| 170 | goto free_housekeeping_staging; |
| 171 | } |
Frederic Weisbecker | edb9382 | 2017-10-27 04:42:37 +0200 | [diff] [blame] | 172 | } |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 173 | |
| 174 | iter_flags = flags & ~housekeeping.flags; |
| 175 | |
| 176 | for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) |
| 177 | housekeeping_setup_type(type, housekeeping_staging); |
Frederic Weisbecker | edb9382 | 2017-10-27 04:42:37 +0200 | [diff] [blame] | 178 | } |
Frederic Weisbecker | 6367b60 | 2022-02-07 16:59:07 +0100 | [diff] [blame] | 179 | |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 180 | if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) |
Frederic Weisbecker | 65e53f8 | 2022-02-07 16:59:09 +0100 | [diff] [blame] | 181 | tick_nohz_full_setup(non_housekeeping_mask); |
Frederic Weisbecker | 6f1982f | 2017-10-27 04:42:36 +0200 | [diff] [blame] | 182 | |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 183 | housekeeping.flags |= flags; |
Frederic Weisbecker | 0cd3e59 | 2022-02-07 16:59:08 +0100 | [diff] [blame] | 184 | err = 1; |
Frederic Weisbecker | 6f1982f | 2017-10-27 04:42:36 +0200 | [diff] [blame] | 185 | |
Frederic Weisbecker | 0cd3e59 | 2022-02-07 16:59:08 +0100 | [diff] [blame] | 186 | free_housekeeping_staging: |
| 187 | free_bootmem_cpumask_var(housekeeping_staging); |
| 188 | free_non_housekeeping_mask: |
Frederic Weisbecker | 6f1982f | 2017-10-27 04:42:36 +0200 | [diff] [blame] | 189 | free_bootmem_cpumask_var(non_housekeeping_mask); |
| 190 | |
Frederic Weisbecker | 0cd3e59 | 2022-02-07 16:59:08 +0100 | [diff] [blame] | 191 | return err; |
Frederic Weisbecker | 6f1982f | 2017-10-27 04:42:36 +0200 | [diff] [blame] | 192 | } |
Frederic Weisbecker | edb9382 | 2017-10-27 04:42:37 +0200 | [diff] [blame] | 193 | |
| 194 | static int __init housekeeping_nohz_full_setup(char *str) |
| 195 | { |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 196 | unsigned long flags; |
Frederic Weisbecker | edb9382 | 2017-10-27 04:42:37 +0200 | [diff] [blame] | 197 | |
Marcelo Tosatti | 9cc5b86 | 2020-05-27 16:29:09 +0200 | [diff] [blame] | 198 | flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU | |
| 199 | HK_FLAG_MISC | HK_FLAG_KTHREAD; |
Frederic Weisbecker | edb9382 | 2017-10-27 04:42:37 +0200 | [diff] [blame] | 200 | |
| 201 | return housekeeping_setup(str, flags); |
| 202 | } |
Frederic Weisbecker | 6f1982f | 2017-10-27 04:42:36 +0200 | [diff] [blame] | 203 | __setup("nohz_full=", housekeeping_nohz_full_setup); |
Frederic Weisbecker | edb9382 | 2017-10-27 04:42:37 +0200 | [diff] [blame] | 204 | |
| 205 | static int __init housekeeping_isolcpus_setup(char *str) |
| 206 | { |
Frederic Weisbecker | ed3b362 | 2022-02-07 16:59:10 +0100 | [diff] [blame] | 207 | unsigned long flags = 0; |
Peter Xu | 3662daf | 2020-04-03 18:35:17 -0400 | [diff] [blame] | 208 | bool illegal = false; |
| 209 | char *par; |
| 210 | int len; |
Frederic Weisbecker | 150dfee | 2017-10-27 04:42:38 +0200 | [diff] [blame] | 211 | |
| 212 | while (isalpha(*str)) { |
| 213 | if (!strncmp(str, "nohz,", 5)) { |
| 214 | str += 5; |
| 215 | flags |= HK_FLAG_TICK; |
| 216 | continue; |
| 217 | } |
| 218 | |
| 219 | if (!strncmp(str, "domain,", 7)) { |
| 220 | str += 7; |
| 221 | flags |= HK_FLAG_DOMAIN; |
| 222 | continue; |
| 223 | } |
| 224 | |
Ming Lei | 11ea68f | 2020-01-20 17:16:25 +0800 | [diff] [blame] | 225 | if (!strncmp(str, "managed_irq,", 12)) { |
| 226 | str += 12; |
| 227 | flags |= HK_FLAG_MANAGED_IRQ; |
| 228 | continue; |
| 229 | } |
| 230 | |
Peter Xu | 3662daf | 2020-04-03 18:35:17 -0400 | [diff] [blame] | 231 | /* |
| 232 | * Skip unknown sub-parameter and validate that it is not |
| 233 | * containing an invalid character. |
| 234 | */ |
| 235 | for (par = str, len = 0; *str && *str != ','; str++, len++) { |
| 236 | if (!isalpha(*str) && *str != '_') |
| 237 | illegal = true; |
| 238 | } |
| 239 | |
| 240 | if (illegal) { |
| 241 | pr_warn("isolcpus: Invalid flag %.*s\n", len, par); |
| 242 | return 0; |
| 243 | } |
| 244 | |
| 245 | pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par); |
| 246 | str++; |
Frederic Weisbecker | 150dfee | 2017-10-27 04:42:38 +0200 | [diff] [blame] | 247 | } |
| 248 | |
| 249 | /* Default behaviour for isolcpus without flags */ |
| 250 | if (!flags) |
| 251 | flags |= HK_FLAG_DOMAIN; |
| 252 | |
| 253 | return housekeeping_setup(str, flags); |
Frederic Weisbecker | edb9382 | 2017-10-27 04:42:37 +0200 | [diff] [blame] | 254 | } |
| 255 | __setup("isolcpus=", housekeeping_isolcpus_setup); |