blob: 5891e715f00d028b0d2f4fd157e99f319b899a1d [file] [log] [blame]
Thomas Gleixner457c8992019-05-19 13:08:55 +01001// SPDX-License-Identifier: GPL-2.0-only
Frederic Weisbecker78634062017-10-27 04:42:28 +02002/*
3 * Housekeeping management. Manage the targets for routine code that can run on
4 * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5 *
6 * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
Frederic Weisbecker1bda3f82018-02-21 05:17:26 +01007 * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
Frederic Weisbecker78634062017-10-27 04:42:28 +02008 *
9 */
Frederic Weisbecker78634062017-10-27 04:42:28 +020010
Frederic Weisbecker04d4e662022-02-07 16:59:06 +010011enum hk_flags {
12 HK_FLAG_TIMER = BIT(HK_TYPE_TIMER),
13 HK_FLAG_RCU = BIT(HK_TYPE_RCU),
14 HK_FLAG_MISC = BIT(HK_TYPE_MISC),
15 HK_FLAG_SCHED = BIT(HK_TYPE_SCHED),
16 HK_FLAG_TICK = BIT(HK_TYPE_TICK),
17 HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN),
18 HK_FLAG_WQ = BIT(HK_TYPE_WQ),
19 HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ),
20 HK_FLAG_KTHREAD = BIT(HK_TYPE_KTHREAD),
21};
22
Ingo Molnardfcb2452018-12-03 10:05:56 +010023DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
24EXPORT_SYMBOL_GPL(housekeeping_overridden);
Frederic Weisbeckered3b3622022-02-07 16:59:10 +010025
26struct housekeeping {
27 cpumask_var_t cpumasks[HK_TYPE_MAX];
28 unsigned long flags;
29};
30
31static struct housekeeping housekeeping;
Frederic Weisbecker7e56a1c2017-10-27 04:42:31 +020032
Frederic Weisbecker04d4e662022-02-07 16:59:06 +010033bool housekeeping_enabled(enum hk_type type)
Wanpeng Li0c5f81d2019-07-06 09:26:51 +080034{
Frederic Weisbeckered3b3622022-02-07 16:59:10 +010035 return !!(housekeeping.flags & BIT(type));
Wanpeng Li0c5f81d2019-07-06 09:26:51 +080036}
37EXPORT_SYMBOL_GPL(housekeeping_enabled);
38
Frederic Weisbecker04d4e662022-02-07 16:59:06 +010039int housekeeping_any_cpu(enum hk_type type)
Frederic Weisbecker7e56a1c2017-10-27 04:42:31 +020040{
Wanpeng Lie0e8d492019-06-28 16:51:41 +080041 int cpu;
42
43 if (static_branch_unlikely(&housekeeping_overridden)) {
Frederic Weisbeckered3b3622022-02-07 16:59:10 +010044 if (housekeeping.flags & BIT(type)) {
45 cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
Wanpeng Lie0e8d492019-06-28 16:51:41 +080046 if (cpu < nr_cpu_ids)
47 return cpu;
48
Oleg Nesterov5097cbc2024-04-11 16:39:05 +020049 cpu = cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
50 if (likely(cpu < nr_cpu_ids))
51 return cpu;
52 /*
53 * Unless we have another problem this can only happen
54 * at boot time before start_secondary() brings the 1st
55 * housekeeping CPU up.
56 */
57 WARN_ON_ONCE(system_state == SYSTEM_RUNNING ||
58 type != HK_TYPE_TIMER);
Wanpeng Lie0e8d492019-06-28 16:51:41 +080059 }
60 }
Frederic Weisbecker7e56a1c2017-10-27 04:42:31 +020061 return smp_processor_id();
62}
63EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
64
Frederic Weisbecker04d4e662022-02-07 16:59:06 +010065const struct cpumask *housekeeping_cpumask(enum hk_type type)
Frederic Weisbecker7e56a1c2017-10-27 04:42:31 +020066{
Ingo Molnardfcb2452018-12-03 10:05:56 +010067 if (static_branch_unlikely(&housekeeping_overridden))
Frederic Weisbeckered3b3622022-02-07 16:59:10 +010068 if (housekeeping.flags & BIT(type))
69 return housekeeping.cpumasks[type];
Frederic Weisbecker7e56a1c2017-10-27 04:42:31 +020070 return cpu_possible_mask;
71}
72EXPORT_SYMBOL_GPL(housekeeping_cpumask);
73
Frederic Weisbecker04d4e662022-02-07 16:59:06 +010074void housekeeping_affine(struct task_struct *t, enum hk_type type)
Frederic Weisbecker7e56a1c2017-10-27 04:42:31 +020075{
Ingo Molnardfcb2452018-12-03 10:05:56 +010076 if (static_branch_unlikely(&housekeeping_overridden))
Frederic Weisbeckered3b3622022-02-07 16:59:10 +010077 if (housekeeping.flags & BIT(type))
78 set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
Frederic Weisbecker7e56a1c2017-10-27 04:42:31 +020079}
80EXPORT_SYMBOL_GPL(housekeeping_affine);
81
Frederic Weisbecker04d4e662022-02-07 16:59:06 +010082bool housekeeping_test_cpu(int cpu, enum hk_type type)
Frederic Weisbecker7e56a1c2017-10-27 04:42:31 +020083{
Ingo Molnardfcb2452018-12-03 10:05:56 +010084 if (static_branch_unlikely(&housekeeping_overridden))
Frederic Weisbeckered3b3622022-02-07 16:59:10 +010085 if (housekeeping.flags & BIT(type))
86 return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
Frederic Weisbecker7e56a1c2017-10-27 04:42:31 +020087 return true;
88}
89EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
Frederic Weisbecker78634062017-10-27 04:42:28 +020090
91void __init housekeeping_init(void)
92{
Frederic Weisbeckered3b3622022-02-07 16:59:10 +010093 enum hk_type type;
94
95 if (!housekeeping.flags)
Frederic Weisbecker78634062017-10-27 04:42:28 +020096 return;
97
Ingo Molnardfcb2452018-12-03 10:05:56 +010098 static_branch_enable(&housekeeping_overridden);
Frederic Weisbeckere179f5a2017-10-27 04:42:32 +020099
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100100 if (housekeeping.flags & HK_FLAG_TICK)
Frederic Weisbeckerd84b3132018-02-21 05:17:27 +0100101 sched_tick_offload_init();
102
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100103 for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
104 /* We need at least one CPU to handle housekeeping work */
105 WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
106 }
Frederic Weisbecker78634062017-10-27 04:42:28 +0200107}
Frederic Weisbecker6f1982f2017-10-27 04:42:36 +0200108
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100109static void __init housekeeping_setup_type(enum hk_type type,
110 cpumask_var_t housekeeping_staging)
111{
112
113 alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
114 cpumask_copy(housekeeping.cpumasks[type],
115 housekeeping_staging);
116}
117
118static int __init housekeeping_setup(char *str, unsigned long flags)
Frederic Weisbecker6f1982f2017-10-27 04:42:36 +0200119{
Frederic Weisbecker6367b602022-02-07 16:59:07 +0100120 cpumask_var_t non_housekeeping_mask, housekeeping_staging;
Oleg Nesterov257bf892024-04-13 16:17:46 +0200121 unsigned int first_cpu;
Frederic Weisbecker0cd3e592022-02-07 16:59:08 +0100122 int err = 0;
Frederic Weisbecker6f1982f2017-10-27 04:42:36 +0200123
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100124 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
Frederic Weisbecker65e53f82022-02-07 16:59:09 +0100125 if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
126 pr_warn("Housekeeping: nohz unsupported."
127 " Build with CONFIG_NO_HZ_FULL\n");
128 return 0;
129 }
130 }
131
Frederic Weisbecker6f1982f2017-10-27 04:42:36 +0200132 alloc_bootmem_cpumask_var(&non_housekeeping_mask);
Paul Gortmaker915a2bc2021-04-19 00:26:59 -0400133 if (cpulist_parse(str, non_housekeeping_mask) < 0) {
Frederic Weisbeckeredb93822017-10-27 04:42:37 +0200134 pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
Frederic Weisbecker0cd3e592022-02-07 16:59:08 +0100135 goto free_non_housekeeping_mask;
Frederic Weisbecker6f1982f2017-10-27 04:42:36 +0200136 }
137
Frederic Weisbecker6367b602022-02-07 16:59:07 +0100138 alloc_bootmem_cpumask_var(&housekeeping_staging);
139 cpumask_andnot(housekeeping_staging,
140 cpu_possible_mask, non_housekeeping_mask);
Frederic Weisbecker6f1982f2017-10-27 04:42:36 +0200141
Oleg Nesterov257bf892024-04-13 16:17:46 +0200142 first_cpu = cpumask_first_and(cpu_present_mask, housekeeping_staging);
143 if (first_cpu >= nr_cpu_ids || first_cpu >= setup_max_cpus) {
Frederic Weisbecker6367b602022-02-07 16:59:07 +0100144 __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
145 __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100146 if (!housekeeping.flags) {
Nicholas Piggin92195652019-04-11 13:34:47 +1000147 pr_warn("Housekeeping: must include one present CPU, "
148 "using boot CPU:%d\n", smp_processor_id());
Nicholas Piggin92195652019-04-11 13:34:47 +1000149 }
Frederic Weisbecker6367b602022-02-07 16:59:07 +0100150 }
151
Oleg Nesterov257bf892024-04-13 16:17:46 +0200152 if (cpumask_empty(non_housekeeping_mask))
153 goto free_housekeeping_staging;
154
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100155 if (!housekeeping.flags) {
156 /* First setup call ("nohz_full=" or "isolcpus=") */
157 enum hk_type type;
158
159 for_each_set_bit(type, &flags, HK_TYPE_MAX)
160 housekeeping_setup_type(type, housekeeping_staging);
Nicholas Piggin92195652019-04-11 13:34:47 +1000161 } else {
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100162 /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
163 enum hk_type type;
164 unsigned long iter_flags = flags & housekeeping.flags;
165
166 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
167 if (!cpumask_equal(housekeeping_staging,
168 housekeeping.cpumasks[type])) {
169 pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
170 goto free_housekeeping_staging;
171 }
Frederic Weisbeckeredb93822017-10-27 04:42:37 +0200172 }
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100173
174 iter_flags = flags & ~housekeeping.flags;
175
176 for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
177 housekeeping_setup_type(type, housekeeping_staging);
Frederic Weisbeckeredb93822017-10-27 04:42:37 +0200178 }
Frederic Weisbecker6367b602022-02-07 16:59:07 +0100179
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100180 if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
Frederic Weisbecker65e53f82022-02-07 16:59:09 +0100181 tick_nohz_full_setup(non_housekeeping_mask);
Frederic Weisbecker6f1982f2017-10-27 04:42:36 +0200182
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100183 housekeeping.flags |= flags;
Frederic Weisbecker0cd3e592022-02-07 16:59:08 +0100184 err = 1;
Frederic Weisbecker6f1982f2017-10-27 04:42:36 +0200185
Frederic Weisbecker0cd3e592022-02-07 16:59:08 +0100186free_housekeeping_staging:
187 free_bootmem_cpumask_var(housekeeping_staging);
188free_non_housekeeping_mask:
Frederic Weisbecker6f1982f2017-10-27 04:42:36 +0200189 free_bootmem_cpumask_var(non_housekeeping_mask);
190
Frederic Weisbecker0cd3e592022-02-07 16:59:08 +0100191 return err;
Frederic Weisbecker6f1982f2017-10-27 04:42:36 +0200192}
Frederic Weisbeckeredb93822017-10-27 04:42:37 +0200193
194static int __init housekeeping_nohz_full_setup(char *str)
195{
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100196 unsigned long flags;
Frederic Weisbeckeredb93822017-10-27 04:42:37 +0200197
Marcelo Tosatti9cc5b862020-05-27 16:29:09 +0200198 flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
199 HK_FLAG_MISC | HK_FLAG_KTHREAD;
Frederic Weisbeckeredb93822017-10-27 04:42:37 +0200200
201 return housekeeping_setup(str, flags);
202}
Frederic Weisbecker6f1982f2017-10-27 04:42:36 +0200203__setup("nohz_full=", housekeeping_nohz_full_setup);
Frederic Weisbeckeredb93822017-10-27 04:42:37 +0200204
205static int __init housekeeping_isolcpus_setup(char *str)
206{
Frederic Weisbeckered3b3622022-02-07 16:59:10 +0100207 unsigned long flags = 0;
Peter Xu3662daf2020-04-03 18:35:17 -0400208 bool illegal = false;
209 char *par;
210 int len;
Frederic Weisbecker150dfee2017-10-27 04:42:38 +0200211
212 while (isalpha(*str)) {
213 if (!strncmp(str, "nohz,", 5)) {
214 str += 5;
215 flags |= HK_FLAG_TICK;
216 continue;
217 }
218
219 if (!strncmp(str, "domain,", 7)) {
220 str += 7;
221 flags |= HK_FLAG_DOMAIN;
222 continue;
223 }
224
Ming Lei11ea68f2020-01-20 17:16:25 +0800225 if (!strncmp(str, "managed_irq,", 12)) {
226 str += 12;
227 flags |= HK_FLAG_MANAGED_IRQ;
228 continue;
229 }
230
Peter Xu3662daf2020-04-03 18:35:17 -0400231 /*
232 * Skip unknown sub-parameter and validate that it is not
233 * containing an invalid character.
234 */
235 for (par = str, len = 0; *str && *str != ','; str++, len++) {
236 if (!isalpha(*str) && *str != '_')
237 illegal = true;
238 }
239
240 if (illegal) {
241 pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
242 return 0;
243 }
244
245 pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
246 str++;
Frederic Weisbecker150dfee2017-10-27 04:42:38 +0200247 }
248
249 /* Default behaviour for isolcpus without flags */
250 if (!flags)
251 flags |= HK_FLAG_DOMAIN;
252
253 return housekeeping_setup(str, flags);
Frederic Weisbeckeredb93822017-10-27 04:42:37 +0200254}
255__setup("isolcpus=", housekeeping_isolcpus_setup);