blob: b2715988791ebe28c7f28aa9dbbb3086d2cecb84 [file] [log] [blame]
Jens Axboe771b53d02019-10-22 10:25:58 -06001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Basic worker thread pool for io_uring
4 *
5 * Copyright (C) 2019 Jens Axboe
6 *
7 */
8#include <linux/kernel.h>
9#include <linux/init.h>
10#include <linux/errno.h>
11#include <linux/sched/signal.h>
Jens Axboe771b53d02019-10-22 10:25:58 -060012#include <linux/percpu.h>
13#include <linux/slab.h>
Jens Axboe771b53d02019-10-22 10:25:58 -060014#include <linux/rculist_nulls.h>
Jens Axboe43c01fb2020-10-22 09:02:50 -060015#include <linux/cpu.h>
Eric W. Biederman355f8412022-02-09 12:47:08 -060016#include <linux/task_work.h>
Paul Moore5bd21822021-02-16 19:46:48 -050017#include <linux/audit.h>
Breno Leitaoda64d6d2023-03-10 12:11:07 -080018#include <linux/mmu_context.h>
Eugene Syromiatnikovdd47c102021-09-13 17:44:15 +020019#include <uapi/linux/io_uring.h>
Jens Axboe771b53d02019-10-22 10:25:58 -060020
21#include "io-wq.h"
Pavel Begunkova6b21fb2022-06-21 10:09:01 +010022#include "slist.h"
Pavel Begunkov024f15e2022-06-21 10:09:02 +010023#include "io_uring.h"
Jens Axboe771b53d02019-10-22 10:25:58 -060024
25#define WORKER_IDLE_TIMEOUT (5 * HZ)
26
27enum {
28 IO_WORKER_F_UP = 1, /* up and active */
29 IO_WORKER_F_RUNNING = 2, /* account as running */
30 IO_WORKER_F_FREE = 4, /* worker on free list */
Jens Axboe05c5f4e2021-09-01 13:01:17 -060031 IO_WORKER_F_BOUND = 8, /* is doing bounded work */
Jens Axboe771b53d02019-10-22 10:25:58 -060032};
33
34enum {
35 IO_WQ_BIT_EXIT = 0, /* wq exiting */
Jens Axboe771b53d02019-10-22 10:25:58 -060036};
37
38enum {
Jens Axboef95dc202021-08-31 13:57:32 -060039 IO_ACCT_STALLED_BIT = 0, /* stalled on hash */
Jens Axboe771b53d02019-10-22 10:25:58 -060040};
41
42/*
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -030043 * One for each thread in a wq pool
Jens Axboe771b53d02019-10-22 10:25:58 -060044 */
45struct io_worker {
46 refcount_t ref;
47 unsigned flags;
48 struct hlist_nulls_node nulls_node;
Jens Axboee61df662019-11-13 13:54:49 -070049 struct list_head all_list;
Jens Axboe771b53d02019-10-22 10:25:58 -060050 struct task_struct *task;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -030051 struct io_wq *wq;
Jens Axboe36c2f922019-11-13 09:43:34 -070052
Jens Axboe771b53d02019-10-22 10:25:58 -060053 struct io_wq_work *cur_work;
Jens Axboe361aee42022-01-18 19:23:51 -070054 struct io_wq_work *next_work;
Jens Axboe081b5822022-01-18 19:13:43 -070055 raw_spinlock_t lock;
Jens Axboe771b53d02019-10-22 10:25:58 -060056
Jens Axboeeb2de942021-02-23 19:59:06 -070057 struct completion ref_done;
58
Jens Axboed3e9f732021-08-04 08:37:25 -060059 unsigned long create_state;
60 struct callback_head create_work;
61 int create_index;
62
Jens Axboe3146cba2021-09-01 11:20:10 -060063 union {
64 struct rcu_head rcu;
65 struct work_struct work;
66 };
Jens Axboe771b53d02019-10-22 10:25:58 -060067};
68
Jens Axboe771b53d02019-10-22 10:25:58 -060069#if BITS_PER_LONG == 64
70#define IO_WQ_HASH_ORDER 6
71#else
72#define IO_WQ_HASH_ORDER 5
73#endif
74
Pavel Begunkov86f3cd12020-03-23 22:57:22 +030075#define IO_WQ_NR_HASH_BUCKETS (1u << IO_WQ_HASH_ORDER)
76
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -030077struct io_wq_acct {
Jens Axboec5def4a2019-11-07 11:41:16 -070078 unsigned nr_workers;
79 unsigned max_workers;
Jens Axboe685fe7f2021-03-08 09:37:51 -070080 int index;
Jens Axboec5def4a2019-11-07 11:41:16 -070081 atomic_t nr_running;
Hao Xu42abc952022-02-06 17:52:39 +080082 raw_spinlock_t lock;
Jens Axboef95dc202021-08-31 13:57:32 -060083 struct io_wq_work_list work_list;
84 unsigned long flags;
Jens Axboec5def4a2019-11-07 11:41:16 -070085};
86
87enum {
88 IO_WQ_ACCT_BOUND,
89 IO_WQ_ACCT_UNBOUND,
Jens Axboef95dc202021-08-31 13:57:32 -060090 IO_WQ_ACCT_NR,
Jens Axboec5def4a2019-11-07 11:41:16 -070091};
92
Jens Axboe771b53d02019-10-22 10:25:58 -060093/*
Jens Axboe771b53d02019-10-22 10:25:58 -060094 * Per io_wq state
95 */
96struct io_wq {
Jens Axboe771b53d02019-10-22 10:25:58 -060097 unsigned long state;
Jens Axboe771b53d02019-10-22 10:25:58 -060098
Pavel Begunkove9fd9392020-03-04 16:14:12 +030099 free_work_fn *free_work;
Pavel Begunkovf5fa38c2020-06-08 21:08:20 +0300100 io_wq_work_fn *do_work;
Jens Axboe7d723062019-11-12 22:31:31 -0700101
Jens Axboee9418942021-02-19 12:33:30 -0700102 struct io_wq_hash *hash;
103
Jens Axboefb3a1f62021-02-26 09:47:20 -0700104 atomic_t worker_refs;
105 struct completion worker_done;
106
Jens Axboe43c01fb2020-10-22 09:02:50 -0600107 struct hlist_node cpuhp_node;
Jens Axboe3bfe6102021-02-16 14:15:30 -0700108
Jens Axboe685fe7f2021-03-08 09:37:51 -0700109 struct task_struct *task;
Pavel Begunkovc7f405d2021-06-14 02:36:12 +0100110
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300111 struct io_wq_acct acct[IO_WQ_ACCT_NR];
112
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300113 /* lock protects access to elements below */
114 raw_spinlock_t lock;
115
116 struct hlist_nulls_head free_list;
117 struct list_head all_list;
118
119 struct wait_queue_entry wait;
120
121 struct io_wq_work *hash_tail[IO_WQ_NR_HASH_BUCKETS];
122
123 cpumask_var_t cpu_mask;
Jens Axboe771b53d02019-10-22 10:25:58 -0600124};
125
Jens Axboe43c01fb2020-10-22 09:02:50 -0600126static enum cpuhp_state io_wq_online;
127
Jens Axboef0127252021-03-03 15:47:04 -0700128struct io_cb_cancel_data {
129 work_cancel_fn *fn;
130 void *data;
131 int nr_running;
132 int nr_pending;
133 bool cancel_all;
134};
135
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300136static bool create_io_worker(struct io_wq *wq, int index);
137static void io_wq_dec_running(struct io_worker *worker);
138static bool io_acct_cancel_pending_work(struct io_wq *wq,
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300139 struct io_wq_acct *acct,
Jens Axboe3146cba2021-09-01 11:20:10 -0600140 struct io_cb_cancel_data *match);
Pavel Begunkov1d5f5ea2021-10-29 13:11:33 +0100141static void create_worker_cb(struct callback_head *cb);
Jens Axboe71a85382021-12-10 08:29:30 -0700142static void io_wq_cancel_tw_create(struct io_wq *wq);
Jens Axboef0127252021-03-03 15:47:04 -0700143
Jens Axboe771b53d02019-10-22 10:25:58 -0600144static bool io_worker_get(struct io_worker *worker)
145{
146 return refcount_inc_not_zero(&worker->ref);
147}
148
149static void io_worker_release(struct io_worker *worker)
150{
151 if (refcount_dec_and_test(&worker->ref))
Jens Axboeeb2de942021-02-23 19:59:06 -0700152 complete(&worker->ref_done);
Jens Axboe771b53d02019-10-22 10:25:58 -0600153}
154
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300155static inline struct io_wq_acct *io_get_acct(struct io_wq *wq, bool bound)
Pavel Begunkov8418f222021-03-22 01:58:28 +0000156{
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300157 return &wq->acct[bound ? IO_WQ_ACCT_BOUND : IO_WQ_ACCT_UNBOUND];
Pavel Begunkov8418f222021-03-22 01:58:28 +0000158}
159
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300160static inline struct io_wq_acct *io_work_get_acct(struct io_wq *wq,
161 struct io_wq_work *work)
Jens Axboec5def4a2019-11-07 11:41:16 -0700162{
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300163 return io_get_acct(wq, !(work->flags & IO_WQ_WORK_UNBOUND));
Jens Axboec5def4a2019-11-07 11:41:16 -0700164}
165
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300166static inline struct io_wq_acct *io_wq_get_acct(struct io_worker *worker)
Jens Axboec5def4a2019-11-07 11:41:16 -0700167{
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300168 return io_get_acct(worker->wq, worker->flags & IO_WORKER_F_BOUND);
Jens Axboec5def4a2019-11-07 11:41:16 -0700169}
170
Jens Axboe685fe7f2021-03-08 09:37:51 -0700171static void io_worker_ref_put(struct io_wq *wq)
172{
173 if (atomic_dec_and_test(&wq->worker_refs))
174 complete(&wq->worker_done);
175}
176
Pavel Begunkov1d5f5ea2021-10-29 13:11:33 +0100177static void io_worker_cancel_cb(struct io_worker *worker)
178{
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300179 struct io_wq_acct *acct = io_wq_get_acct(worker);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300180 struct io_wq *wq = worker->wq;
Pavel Begunkov1d5f5ea2021-10-29 13:11:33 +0100181
182 atomic_dec(&acct->nr_running);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300183 raw_spin_lock(&wq->lock);
Pavel Begunkov1d5f5ea2021-10-29 13:11:33 +0100184 acct->nr_workers--;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300185 raw_spin_unlock(&wq->lock);
Pavel Begunkov1d5f5ea2021-10-29 13:11:33 +0100186 io_worker_ref_put(wq);
187 clear_bit_unlock(0, &worker->create_state);
188 io_worker_release(worker);
189}
190
191static bool io_task_worker_match(struct callback_head *cb, void *data)
192{
193 struct io_worker *worker;
194
195 if (cb->func != create_worker_cb)
196 return false;
197 worker = container_of(cb, struct io_worker, create_work);
198 return worker == data;
199}
200
Jens Axboe771b53d02019-10-22 10:25:58 -0600201static void io_worker_exit(struct io_worker *worker)
202{
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300203 struct io_wq *wq = worker->wq;
Jens Axboe771b53d02019-10-22 10:25:58 -0600204
Pavel Begunkov1d5f5ea2021-10-29 13:11:33 +0100205 while (1) {
206 struct callback_head *cb = task_work_cancel_match(wq->task,
207 io_task_worker_match, worker);
208
209 if (!cb)
210 break;
211 io_worker_cancel_cb(worker);
212 }
Jens Axboe771b53d02019-10-22 10:25:58 -0600213
Pavel Begunkovc907e522021-10-23 12:13:55 +0100214 io_worker_release(worker);
Jens Axboeeb2de942021-02-23 19:59:06 -0700215 wait_for_completion(&worker->ref_done);
Jens Axboe771b53d02019-10-22 10:25:58 -0600216
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300217 raw_spin_lock(&wq->lock);
Jens Axboe83d6c392021-08-03 09:14:35 -0600218 if (worker->flags & IO_WORKER_F_FREE)
Jens Axboebf1daa42021-02-16 18:00:55 -0700219 hlist_nulls_del_rcu(&worker->nulls_node);
Jens Axboee61df662019-11-13 13:54:49 -0700220 list_del_rcu(&worker->all_list);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300221 raw_spin_unlock(&wq->lock);
222 io_wq_dec_running(worker);
Jens Axboe83d6c392021-08-03 09:14:35 -0600223 worker->flags = 0;
Hao Xu42abc952022-02-06 17:52:39 +0800224 preempt_disable();
Jens Axboe83d6c392021-08-03 09:14:35 -0600225 current->flags &= ~PF_IO_WORKER;
226 preempt_enable();
Jens Axboe771b53d02019-10-22 10:25:58 -0600227
YueHaibing364b05f2019-11-02 15:55:01 +0800228 kfree_rcu(worker, rcu);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300229 io_worker_ref_put(wq);
Jens Axboe46fe18b2021-03-04 12:39:36 -0700230 do_exit(0);
Jens Axboe771b53d02019-10-22 10:25:58 -0600231}
232
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300233static inline bool io_acct_run_queue(struct io_wq_acct *acct)
Jens Axboec5def4a2019-11-07 11:41:16 -0700234{
Hao Xue13fb1f2022-02-06 17:52:40 +0800235 bool ret = false;
236
237 raw_spin_lock(&acct->lock);
Jens Axboef95dc202021-08-31 13:57:32 -0600238 if (!wq_list_empty(&acct->work_list) &&
239 !test_bit(IO_ACCT_STALLED_BIT, &acct->flags))
Hao Xue13fb1f2022-02-06 17:52:40 +0800240 ret = true;
241 raw_spin_unlock(&acct->lock);
242
243 return ret;
Jens Axboec5def4a2019-11-07 11:41:16 -0700244}
245
246/*
247 * Check head of free list for an available worker. If one isn't available,
Jens Axboe685fe7f2021-03-08 09:37:51 -0700248 * caller must create one.
Jens Axboec5def4a2019-11-07 11:41:16 -0700249 */
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300250static bool io_wq_activate_free_worker(struct io_wq *wq,
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300251 struct io_wq_acct *acct)
Jens Axboec5def4a2019-11-07 11:41:16 -0700252 __must_hold(RCU)
253{
254 struct hlist_nulls_node *n;
255 struct io_worker *worker;
256
Jens Axboe83d6c392021-08-03 09:14:35 -0600257 /*
258 * Iterate free_list and see if we can find an idle worker to
259 * activate. If a given worker is on the free_list but in the process
260 * of exiting, keep trying.
261 */
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300262 hlist_nulls_for_each_entry_rcu(worker, n, &wq->free_list, nulls_node) {
Jens Axboe83d6c392021-08-03 09:14:35 -0600263 if (!io_worker_get(worker))
264 continue;
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300265 if (io_wq_get_acct(worker) != acct) {
Jens Axboef95dc202021-08-31 13:57:32 -0600266 io_worker_release(worker);
267 continue;
268 }
Jens Axboe83d6c392021-08-03 09:14:35 -0600269 if (wake_up_process(worker->task)) {
270 io_worker_release(worker);
271 return true;
272 }
Jens Axboec5def4a2019-11-07 11:41:16 -0700273 io_worker_release(worker);
Jens Axboec5def4a2019-11-07 11:41:16 -0700274 }
275
276 return false;
277}
278
279/*
280 * We need a worker. If we find a free one, we're good. If not, and we're
Jens Axboe685fe7f2021-03-08 09:37:51 -0700281 * below the max number of workers, create one.
Jens Axboec5def4a2019-11-07 11:41:16 -0700282 */
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300283static bool io_wq_create_worker(struct io_wq *wq, struct io_wq_acct *acct)
Jens Axboec5def4a2019-11-07 11:41:16 -0700284{
Jens Axboec5def4a2019-11-07 11:41:16 -0700285 /*
286 * Most likely an attempt to queue unbounded work on an io_wq that
287 * wasn't setup with any unbounded workers.
288 */
Pavel Begunkove6ab8992021-06-17 18:13:59 +0100289 if (unlikely(!acct->max_workers))
290 pr_warn_once("io-wq is not configured for unbound workers");
Jens Axboec5def4a2019-11-07 11:41:16 -0700291
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300292 raw_spin_lock(&wq->lock);
Pavel Begunkovbc369922021-10-19 20:31:26 +0100293 if (acct->nr_workers >= acct->max_workers) {
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300294 raw_spin_unlock(&wq->lock);
Hao Xu7a842fb2021-09-12 03:40:50 +0800295 return true;
Jens Axboe94ffb0a2021-08-30 11:55:22 -0600296 }
Hao Xu7a842fb2021-09-12 03:40:50 +0800297 acct->nr_workers++;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300298 raw_spin_unlock(&wq->lock);
Hao Xu7a842fb2021-09-12 03:40:50 +0800299 atomic_inc(&acct->nr_running);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300300 atomic_inc(&wq->worker_refs);
301 return create_io_worker(wq, acct->index);
Jens Axboec5def4a2019-11-07 11:41:16 -0700302}
303
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300304static void io_wq_inc_running(struct io_worker *worker)
Jens Axboec5def4a2019-11-07 11:41:16 -0700305{
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300306 struct io_wq_acct *acct = io_wq_get_acct(worker);
Jens Axboec5def4a2019-11-07 11:41:16 -0700307
308 atomic_inc(&acct->nr_running);
309}
310
Jens Axboe685fe7f2021-03-08 09:37:51 -0700311static void create_worker_cb(struct callback_head *cb)
312{
Jens Axboed3e9f732021-08-04 08:37:25 -0600313 struct io_worker *worker;
Jens Axboe685fe7f2021-03-08 09:37:51 -0700314 struct io_wq *wq;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300315
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300316 struct io_wq_acct *acct;
Jens Axboe05c5f4e2021-09-01 13:01:17 -0600317 bool do_create = false;
Jens Axboe685fe7f2021-03-08 09:37:51 -0700318
Jens Axboed3e9f732021-08-04 08:37:25 -0600319 worker = container_of(cb, struct io_worker, create_work);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300320 wq = worker->wq;
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300321 acct = &wq->acct[worker->create_index];
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300322 raw_spin_lock(&wq->lock);
323
Hao Xu49e7f0c2021-08-08 21:54:33 +0800324 if (acct->nr_workers < acct->max_workers) {
Hao Xu21698272021-08-05 18:05:38 +0800325 acct->nr_workers++;
Hao Xu49e7f0c2021-08-08 21:54:33 +0800326 do_create = true;
327 }
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300328 raw_spin_unlock(&wq->lock);
Hao Xu49e7f0c2021-08-08 21:54:33 +0800329 if (do_create) {
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300330 create_io_worker(wq, worker->create_index);
Hao Xu49e7f0c2021-08-08 21:54:33 +0800331 } else {
332 atomic_dec(&acct->nr_running);
333 io_worker_ref_put(wq);
334 }
Jens Axboed3e9f732021-08-04 08:37:25 -0600335 clear_bit_unlock(0, &worker->create_state);
336 io_worker_release(worker);
Jens Axboe685fe7f2021-03-08 09:37:51 -0700337}
338
Jens Axboe3146cba2021-09-01 11:20:10 -0600339static bool io_queue_worker_create(struct io_worker *worker,
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300340 struct io_wq_acct *acct,
Jens Axboe3146cba2021-09-01 11:20:10 -0600341 task_work_func_t func)
Jens Axboe685fe7f2021-03-08 09:37:51 -0700342{
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300343 struct io_wq *wq = worker->wq;
Jens Axboe685fe7f2021-03-08 09:37:51 -0700344
345 /* raced with exit, just ignore create call */
346 if (test_bit(IO_WQ_BIT_EXIT, &wq->state))
347 goto fail;
Jens Axboed3e9f732021-08-04 08:37:25 -0600348 if (!io_worker_get(worker))
349 goto fail;
350 /*
351 * create_state manages ownership of create_work/index. We should
352 * only need one entry per worker, as the worker going to sleep
353 * will trigger the condition, and waking will clear it once it
354 * runs the task_work.
355 */
356 if (test_bit(0, &worker->create_state) ||
357 test_and_set_bit_lock(0, &worker->create_state))
358 goto fail_release;
Jens Axboe685fe7f2021-03-08 09:37:51 -0700359
Jens Axboe71a85382021-12-10 08:29:30 -0700360 atomic_inc(&wq->worker_refs);
Jens Axboe3146cba2021-09-01 11:20:10 -0600361 init_task_work(&worker->create_work, func);
Jens Axboed3e9f732021-08-04 08:37:25 -0600362 worker->create_index = acct->index;
Jens Axboe71a85382021-12-10 08:29:30 -0700363 if (!task_work_add(wq->task, &worker->create_work, TWA_SIGNAL)) {
364 /*
365 * EXIT may have been set after checking it above, check after
366 * adding the task_work and remove any creation item if it is
367 * now set. wq exit does that too, but we can have added this
368 * work item after we canceled in io_wq_exit_workers().
369 */
370 if (test_bit(IO_WQ_BIT_EXIT, &wq->state))
371 io_wq_cancel_tw_create(wq);
372 io_worker_ref_put(wq);
Jens Axboe3146cba2021-09-01 11:20:10 -0600373 return true;
Jens Axboe71a85382021-12-10 08:29:30 -0700374 }
375 io_worker_ref_put(wq);
Jens Axboed3e9f732021-08-04 08:37:25 -0600376 clear_bit_unlock(0, &worker->create_state);
377fail_release:
378 io_worker_release(worker);
Jens Axboe685fe7f2021-03-08 09:37:51 -0700379fail:
380 atomic_dec(&acct->nr_running);
381 io_worker_ref_put(wq);
Jens Axboe3146cba2021-09-01 11:20:10 -0600382 return false;
Jens Axboe685fe7f2021-03-08 09:37:51 -0700383}
384
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300385static void io_wq_dec_running(struct io_worker *worker)
Jens Axboec5def4a2019-11-07 11:41:16 -0700386{
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300387 struct io_wq_acct *acct = io_wq_get_acct(worker);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300388 struct io_wq *wq = worker->wq;
Jens Axboec5def4a2019-11-07 11:41:16 -0700389
Jens Axboe685fe7f2021-03-08 09:37:51 -0700390 if (!(worker->flags & IO_WORKER_F_UP))
391 return;
392
Hao Xu42abc952022-02-06 17:52:39 +0800393 if (!atomic_dec_and_test(&acct->nr_running))
394 return;
Hao Xue13fb1f2022-02-06 17:52:40 +0800395 if (!io_acct_run_queue(acct))
Hao Xu42abc952022-02-06 17:52:39 +0800396 return;
Hao Xu42abc952022-02-06 17:52:39 +0800397
Hao Xu42abc952022-02-06 17:52:39 +0800398 atomic_inc(&acct->nr_running);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300399 atomic_inc(&wq->worker_refs);
Hao Xu42abc952022-02-06 17:52:39 +0800400 io_queue_worker_create(worker, acct, create_worker_cb);
Jens Axboec5def4a2019-11-07 11:41:16 -0700401}
402
Jens Axboe771b53d02019-10-22 10:25:58 -0600403/*
404 * Worker will start processing some work. Move it to the busy list, if
405 * it's currently on the freelist
406 */
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300407static void __io_worker_busy(struct io_wq *wq, struct io_worker *worker)
Jens Axboe771b53d02019-10-22 10:25:58 -0600408{
409 if (worker->flags & IO_WORKER_F_FREE) {
410 worker->flags &= ~IO_WORKER_F_FREE;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300411 raw_spin_lock(&wq->lock);
Jens Axboe771b53d02019-10-22 10:25:58 -0600412 hlist_nulls_del_init_rcu(&worker->nulls_node);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300413 raw_spin_unlock(&wq->lock);
Jens Axboe771b53d02019-10-22 10:25:58 -0600414 }
Jens Axboe771b53d02019-10-22 10:25:58 -0600415}
416
417/*
Jens Axboe07d99092023-03-27 13:10:21 -0600418 * No work, worker going to sleep. Move to freelist.
Jens Axboe771b53d02019-10-22 10:25:58 -0600419 */
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300420static void __io_worker_idle(struct io_wq *wq, struct io_worker *worker)
421 __must_hold(wq->lock)
Jens Axboe771b53d02019-10-22 10:25:58 -0600422{
423 if (!(worker->flags & IO_WORKER_F_FREE)) {
424 worker->flags |= IO_WORKER_F_FREE;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300425 hlist_nulls_add_head_rcu(&worker->nulls_node, &wq->free_list);
Jens Axboe771b53d02019-10-22 10:25:58 -0600426 }
Jens Axboe771b53d02019-10-22 10:25:58 -0600427}
428
Pavel Begunkov60cf46a2020-03-14 00:31:05 +0300429static inline unsigned int io_get_work_hash(struct io_wq_work *work)
430{
431 return work->flags >> IO_WQ_HASH_SHIFT;
432}
433
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300434static bool io_wait_on_hash(struct io_wq *wq, unsigned int hash)
Jens Axboee9418942021-02-19 12:33:30 -0700435{
Jens Axboed3e3c102021-11-11 17:32:53 -0700436 bool ret = false;
Jens Axboee9418942021-02-19 12:33:30 -0700437
Jens Axboe08bdbd32021-08-31 06:57:25 -0600438 spin_lock_irq(&wq->hash->wait.lock);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300439 if (list_empty(&wq->wait.entry)) {
440 __add_wait_queue(&wq->hash->wait, &wq->wait);
Jens Axboee9418942021-02-19 12:33:30 -0700441 if (!test_bit(hash, &wq->hash->map)) {
442 __set_current_state(TASK_RUNNING);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300443 list_del_init(&wq->wait.entry);
Jens Axboed3e3c102021-11-11 17:32:53 -0700444 ret = true;
Jens Axboee9418942021-02-19 12:33:30 -0700445 }
446 }
Jens Axboe08bdbd32021-08-31 06:57:25 -0600447 spin_unlock_irq(&wq->hash->wait.lock);
Jens Axboed3e3c102021-11-11 17:32:53 -0700448 return ret;
Jens Axboee9418942021-02-19 12:33:30 -0700449}
450
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300451static struct io_wq_work *io_get_next_work(struct io_wq_acct *acct,
Jens Axboe0242f642021-08-31 13:53:00 -0600452 struct io_worker *worker)
Hao Xu42abc952022-02-06 17:52:39 +0800453 __must_hold(acct->lock)
Jens Axboe771b53d02019-10-22 10:25:58 -0600454{
Jens Axboe6206f0e2019-11-26 11:59:32 -0700455 struct io_wq_work_node *node, *prev;
Pavel Begunkov86f3cd12020-03-23 22:57:22 +0300456 struct io_wq_work *work, *tail;
Jens Axboee9418942021-02-19 12:33:30 -0700457 unsigned int stall_hash = -1U;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300458 struct io_wq *wq = worker->wq;
Jens Axboe771b53d02019-10-22 10:25:58 -0600459
Jens Axboef95dc202021-08-31 13:57:32 -0600460 wq_list_for_each(node, prev, &acct->work_list) {
Jens Axboee9418942021-02-19 12:33:30 -0700461 unsigned int hash;
462
Jens Axboe6206f0e2019-11-26 11:59:32 -0700463 work = container_of(node, struct io_wq_work, list);
464
Jens Axboe771b53d02019-10-22 10:25:58 -0600465 /* not hashed, can run anytime */
Pavel Begunkov8766dd52020-03-14 00:31:04 +0300466 if (!io_wq_is_hashed(work)) {
Jens Axboef95dc202021-08-31 13:57:32 -0600467 wq_list_del(&acct->work_list, node, prev);
Jens Axboe771b53d02019-10-22 10:25:58 -0600468 return work;
469 }
470
Pavel Begunkov60cf46a2020-03-14 00:31:05 +0300471 hash = io_get_work_hash(work);
Jens Axboee9418942021-02-19 12:33:30 -0700472 /* all items with this hash lie in [work, tail] */
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300473 tail = wq->hash_tail[hash];
Jens Axboee9418942021-02-19 12:33:30 -0700474
475 /* hashed, can run if not already running */
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300476 if (!test_and_set_bit(hash, &wq->hash->map)) {
477 wq->hash_tail[hash] = NULL;
Jens Axboef95dc202021-08-31 13:57:32 -0600478 wq_list_cut(&acct->work_list, &tail->list, prev);
Jens Axboe771b53d02019-10-22 10:25:58 -0600479 return work;
480 }
Jens Axboee9418942021-02-19 12:33:30 -0700481 if (stall_hash == -1U)
482 stall_hash = hash;
483 /* fast forward to a next hash, for-each will fix up @prev */
484 node = &tail->list;
485 }
486
487 if (stall_hash != -1U) {
Jens Axboed3e3c102021-11-11 17:32:53 -0700488 bool unstalled;
489
Jens Axboe0242f642021-08-31 13:53:00 -0600490 /*
491 * Set this before dropping the lock to avoid racing with new
492 * work being added and clearing the stalled bit.
493 */
Jens Axboef95dc202021-08-31 13:57:32 -0600494 set_bit(IO_ACCT_STALLED_BIT, &acct->flags);
Hao Xu42abc952022-02-06 17:52:39 +0800495 raw_spin_unlock(&acct->lock);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300496 unstalled = io_wait_on_hash(wq, stall_hash);
Hao Xu42abc952022-02-06 17:52:39 +0800497 raw_spin_lock(&acct->lock);
Jens Axboed3e3c102021-11-11 17:32:53 -0700498 if (unstalled) {
499 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300500 if (wq_has_sleeper(&wq->hash->wait))
501 wake_up(&wq->hash->wait);
Jens Axboed3e3c102021-11-11 17:32:53 -0700502 }
Jens Axboe771b53d02019-10-22 10:25:58 -0600503 }
504
505 return NULL;
506}
507
Pavel Begunkovdc026a72020-03-04 16:14:09 +0300508static void io_assign_current_work(struct io_worker *worker,
509 struct io_wq_work *work)
510{
Pavel Begunkovd78298e2020-03-14 00:31:03 +0300511 if (work) {
Pavel Begunkov024f15e2022-06-21 10:09:02 +0100512 io_run_task_work();
Pavel Begunkovd78298e2020-03-14 00:31:03 +0300513 cond_resched();
514 }
Pavel Begunkovdc026a72020-03-04 16:14:09 +0300515
Jens Axboe081b5822022-01-18 19:13:43 -0700516 raw_spin_lock(&worker->lock);
Pavel Begunkovdc026a72020-03-04 16:14:09 +0300517 worker->cur_work = work;
Jens Axboe361aee42022-01-18 19:23:51 -0700518 worker->next_work = NULL;
Jens Axboe081b5822022-01-18 19:13:43 -0700519 raw_spin_unlock(&worker->lock);
Pavel Begunkovdc026a72020-03-04 16:14:09 +0300520}
521
Jens Axboe771b53d02019-10-22 10:25:58 -0600522static void io_worker_handle_work(struct io_worker *worker)
Jens Axboe771b53d02019-10-22 10:25:58 -0600523{
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300524 struct io_wq_acct *acct = io_wq_get_acct(worker);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300525 struct io_wq *wq = worker->wq;
Pavel Begunkovc60eb042021-04-08 01:54:42 +0100526 bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state);
Jens Axboe771b53d02019-10-22 10:25:58 -0600527
528 do {
Pavel Begunkov86f3cd12020-03-23 22:57:22 +0300529 struct io_wq_work *work;
Jens Axboe73031f762022-01-19 13:11:58 -0700530
Jens Axboe771b53d02019-10-22 10:25:58 -0600531 /*
Jens Axboe771b53d02019-10-22 10:25:58 -0600532 * If we got some work, mark us as busy. If we didn't, but
533 * the list isn't empty, it means we stalled on hashed work.
534 * Mark us stalled so we don't keep looking for work when we
535 * can't make progress, any work completion or insertion will
536 * clear the stalled flag.
537 */
Hao Xue13fb1f2022-02-06 17:52:40 +0800538 raw_spin_lock(&acct->lock);
Jens Axboef95dc202021-08-31 13:57:32 -0600539 work = io_get_next_work(acct, worker);
Hao Xu42abc952022-02-06 17:52:39 +0800540 raw_spin_unlock(&acct->lock);
Jens Axboe361aee42022-01-18 19:23:51 -0700541 if (work) {
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300542 __io_worker_busy(wq, worker);
Jens Axboe771b53d02019-10-22 10:25:58 -0600543
Jens Axboe361aee42022-01-18 19:23:51 -0700544 /*
545 * Make sure cancelation can find this, even before
546 * it becomes the active work. That avoids a window
547 * where the work has been removed from our general
548 * work list, but isn't yet discoverable as the
549 * current work item for this worker.
550 */
551 raw_spin_lock(&worker->lock);
552 worker->next_work = work;
553 raw_spin_unlock(&worker->lock);
Hao Xu42abc952022-02-06 17:52:39 +0800554 } else {
Jens Axboe771b53d02019-10-22 10:25:58 -0600555 break;
Hao Xu42abc952022-02-06 17:52:39 +0800556 }
Pavel Begunkov58e39312020-03-04 16:14:10 +0300557 io_assign_current_work(worker, work);
Jens Axboee9418942021-02-19 12:33:30 -0700558 __set_current_state(TASK_RUNNING);
Jens Axboe36c2f922019-11-13 09:43:34 -0700559
Pavel Begunkovdc026a72020-03-04 16:14:09 +0300560 /* handle a whole dependent link */
561 do {
Pavel Begunkov5280f7e2021-02-04 13:52:08 +0000562 struct io_wq_work *next_hashed, *linked;
Pavel Begunkovb089ed392020-07-25 14:42:00 +0300563 unsigned int hash = io_get_work_hash(work);
Hillf Dantonfd1c4bc2019-12-24 09:14:29 -0700564
Pavel Begunkov86f3cd12020-03-23 22:57:22 +0300565 next_hashed = wq_next_work(work);
Pavel Begunkovc60eb042021-04-08 01:54:42 +0100566
567 if (unlikely(do_kill) && (work->flags & IO_WQ_WORK_UNBOUND))
568 work->flags |= IO_WQ_WORK_CANCEL;
Pavel Begunkov5280f7e2021-02-04 13:52:08 +0000569 wq->do_work(work);
570 io_assign_current_work(worker, NULL);
Jens Axboe36c2f922019-11-13 09:43:34 -0700571
Pavel Begunkov5280f7e2021-02-04 13:52:08 +0000572 linked = wq->free_work(work);
Pavel Begunkov86f3cd12020-03-23 22:57:22 +0300573 work = next_hashed;
574 if (!work && linked && !io_wq_is_hashed(linked)) {
575 work = linked;
576 linked = NULL;
577 }
578 io_assign_current_work(worker, work);
Pavel Begunkov86f3cd12020-03-23 22:57:22 +0300579 if (linked)
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300580 io_wq_enqueue(wq, linked);
Pavel Begunkov86f3cd12020-03-23 22:57:22 +0300581
582 if (hash != -1U && !next_hashed) {
Jens Axboed3e3c102021-11-11 17:32:53 -0700583 /* serialize hash clear with wake_up() */
584 spin_lock_irq(&wq->hash->wait.lock);
Jens Axboee9418942021-02-19 12:33:30 -0700585 clear_bit(hash, &wq->hash->map);
Jens Axboef95dc202021-08-31 13:57:32 -0600586 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
Jens Axboed3e3c102021-11-11 17:32:53 -0700587 spin_unlock_irq(&wq->hash->wait.lock);
Jens Axboee9418942021-02-19 12:33:30 -0700588 if (wq_has_sleeper(&wq->hash->wait))
589 wake_up(&wq->hash->wait);
Pavel Begunkovdc026a72020-03-04 16:14:09 +0300590 }
Pavel Begunkov58e39312020-03-04 16:14:10 +0300591 } while (work);
Jens Axboe771b53d02019-10-22 10:25:58 -0600592 } while (1);
593}
594
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300595static int io_wq_worker(void *data)
Jens Axboe771b53d02019-10-22 10:25:58 -0600596{
597 struct io_worker *worker = data;
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300598 struct io_wq_acct *acct = io_wq_get_acct(worker);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300599 struct io_wq *wq = worker->wq;
Jens Axboe01e68ce2023-03-08 07:18:51 -0700600 bool exit_mask = false, last_timeout = false;
Jens Axboe46fe18b2021-03-04 12:39:36 -0700601 char buf[TASK_COMM_LEN];
Jens Axboe771b53d02019-10-22 10:25:58 -0600602
Jens Axboe46fe18b2021-03-04 12:39:36 -0700603 worker->flags |= (IO_WORKER_F_UP | IO_WORKER_F_RUNNING);
Jens Axboe46fe18b2021-03-04 12:39:36 -0700604
Jens Axboe685fe7f2021-03-08 09:37:51 -0700605 snprintf(buf, sizeof(buf), "iou-wrk-%d", wq->task->pid);
Jens Axboe46fe18b2021-03-04 12:39:36 -0700606 set_task_comm(current, buf);
Jens Axboe771b53d02019-10-22 10:25:58 -0600607
608 while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) {
Jens Axboe16efa4f2021-03-12 20:26:13 -0700609 long ret;
610
Jens Axboe506d95f2019-12-07 21:03:59 -0700611 set_current_state(TASK_INTERRUPTIBLE);
Hao Xue13fb1f2022-02-06 17:52:40 +0800612 while (io_acct_run_queue(acct))
Jens Axboe771b53d02019-10-22 10:25:58 -0600613 io_worker_handle_work(worker);
Hao Xue13fb1f2022-02-06 17:52:40 +0800614
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300615 raw_spin_lock(&wq->lock);
Jens Axboe01e68ce2023-03-08 07:18:51 -0700616 /*
617 * Last sleep timed out. Exit if we're not the last worker,
618 * or if someone modified our affinity.
619 */
620 if (last_timeout && (exit_mask || acct->nr_workers > 1)) {
Hao Xu767a65e2021-09-12 03:40:52 +0800621 acct->nr_workers--;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300622 raw_spin_unlock(&wq->lock);
Jens Axboe05c5f4e2021-09-01 13:01:17 -0600623 __set_current_state(TASK_RUNNING);
624 break;
625 }
626 last_timeout = false;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300627 __io_worker_idle(wq, worker);
628 raw_spin_unlock(&wq->lock);
Pavel Begunkov024f15e2022-06-21 10:09:02 +0100629 if (io_run_task_work())
Jens Axboe00ddff42021-03-21 07:06:56 -0600630 continue;
Jens Axboe16efa4f2021-03-12 20:26:13 -0700631 ret = schedule_timeout(WORKER_IDLE_TIMEOUT);
Jens Axboedbe1bdb2021-03-25 18:16:06 -0600632 if (signal_pending(current)) {
633 struct ksignal ksig;
634
635 if (!get_signal(&ksig))
636 continue;
Jens Axboe78f88762021-09-27 10:04:10 -0600637 break;
Jens Axboedbe1bdb2021-03-25 18:16:06 -0600638 }
Jens Axboe01e68ce2023-03-08 07:18:51 -0700639 if (!ret) {
640 last_timeout = true;
641 exit_mask = !cpumask_test_cpu(raw_smp_processor_id(),
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300642 wq->cpu_mask);
Jens Axboe01e68ce2023-03-08 07:18:51 -0700643 }
Jens Axboe771b53d02019-10-22 10:25:58 -0600644 }
645
Hao Xue13fb1f2022-02-06 17:52:40 +0800646 if (test_bit(IO_WQ_BIT_EXIT, &wq->state))
Pavel Begunkove5872272021-06-14 02:36:17 +0100647 io_worker_handle_work(worker);
Jens Axboe771b53d02019-10-22 10:25:58 -0600648
649 io_worker_exit(worker);
650 return 0;
651}
652
653/*
Jens Axboe771b53d02019-10-22 10:25:58 -0600654 * Called when a worker is scheduled in. Mark us as currently running.
655 */
656void io_wq_worker_running(struct task_struct *tsk)
657{
Eric W. Biedermane32cf5d2021-12-22 22:10:09 -0600658 struct io_worker *worker = tsk->worker_private;
Jens Axboe771b53d02019-10-22 10:25:58 -0600659
Jens Axboe3bfe6102021-02-16 14:15:30 -0700660 if (!worker)
661 return;
Jens Axboe771b53d02019-10-22 10:25:58 -0600662 if (!(worker->flags & IO_WORKER_F_UP))
663 return;
664 if (worker->flags & IO_WORKER_F_RUNNING)
665 return;
666 worker->flags |= IO_WORKER_F_RUNNING;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300667 io_wq_inc_running(worker);
Jens Axboe771b53d02019-10-22 10:25:58 -0600668}
669
670/*
671 * Called when worker is going to sleep. If there are no workers currently
Jens Axboe685fe7f2021-03-08 09:37:51 -0700672 * running and we have work pending, wake up a free one or create a new one.
Jens Axboe771b53d02019-10-22 10:25:58 -0600673 */
674void io_wq_worker_sleeping(struct task_struct *tsk)
675{
Eric W. Biedermane32cf5d2021-12-22 22:10:09 -0600676 struct io_worker *worker = tsk->worker_private;
Jens Axboe771b53d02019-10-22 10:25:58 -0600677
Jens Axboe3bfe6102021-02-16 14:15:30 -0700678 if (!worker)
679 return;
Jens Axboe771b53d02019-10-22 10:25:58 -0600680 if (!(worker->flags & IO_WORKER_F_UP))
681 return;
682 if (!(worker->flags & IO_WORKER_F_RUNNING))
683 return;
684
685 worker->flags &= ~IO_WORKER_F_RUNNING;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300686 io_wq_dec_running(worker);
Jens Axboe771b53d02019-10-22 10:25:58 -0600687}
688
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300689static void io_init_new_worker(struct io_wq *wq, struct io_worker *worker,
Jens Axboe3146cba2021-09-01 11:20:10 -0600690 struct task_struct *tsk)
Jens Axboe3bfe6102021-02-16 14:15:30 -0700691{
Eric W. Biedermane32cf5d2021-12-22 22:10:09 -0600692 tsk->worker_private = worker;
Jens Axboe46fe18b2021-03-04 12:39:36 -0700693 worker->task = tsk;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300694 set_cpus_allowed_ptr(tsk, wq->cpu_mask);
Jens Axboe46fe18b2021-03-04 12:39:36 -0700695
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300696 raw_spin_lock(&wq->lock);
697 hlist_nulls_add_head_rcu(&worker->nulls_node, &wq->free_list);
698 list_add_tail_rcu(&worker->all_list, &wq->all_list);
Jens Axboe46fe18b2021-03-04 12:39:36 -0700699 worker->flags |= IO_WORKER_F_FREE;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300700 raw_spin_unlock(&wq->lock);
Jens Axboe46fe18b2021-03-04 12:39:36 -0700701 wake_up_new_task(tsk);
Jens Axboe771b53d02019-10-22 10:25:58 -0600702}
703
Jens Axboe3146cba2021-09-01 11:20:10 -0600704static bool io_wq_work_match_all(struct io_wq_work *work, void *data)
705{
706 return true;
707}
708
709static inline bool io_should_retry_thread(long err)
710{
Jens Axboea226abc2021-12-02 19:40:15 -0700711 /*
712 * Prevent perpetual task_work retry, if the task (or its group) is
713 * exiting.
714 */
715 if (fatal_signal_pending(current))
716 return false;
717
Jens Axboe3146cba2021-09-01 11:20:10 -0600718 switch (err) {
719 case -EAGAIN:
720 case -ERESTARTSYS:
721 case -ERESTARTNOINTR:
722 case -ERESTARTNOHAND:
723 return true;
724 default:
725 return false;
726 }
727}
728
729static void create_worker_cont(struct callback_head *cb)
730{
731 struct io_worker *worker;
732 struct task_struct *tsk;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300733 struct io_wq *wq;
Jens Axboe3146cba2021-09-01 11:20:10 -0600734
735 worker = container_of(cb, struct io_worker, create_work);
736 clear_bit_unlock(0, &worker->create_state);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300737 wq = worker->wq;
738 tsk = create_io_thread(io_wq_worker, worker, NUMA_NO_NODE);
Jens Axboe3146cba2021-09-01 11:20:10 -0600739 if (!IS_ERR(tsk)) {
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300740 io_init_new_worker(wq, worker, tsk);
Jens Axboe3146cba2021-09-01 11:20:10 -0600741 io_worker_release(worker);
742 return;
743 } else if (!io_should_retry_thread(PTR_ERR(tsk))) {
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300744 struct io_wq_acct *acct = io_wq_get_acct(worker);
Jens Axboe3146cba2021-09-01 11:20:10 -0600745
746 atomic_dec(&acct->nr_running);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300747 raw_spin_lock(&wq->lock);
Jens Axboe3146cba2021-09-01 11:20:10 -0600748 acct->nr_workers--;
749 if (!acct->nr_workers) {
750 struct io_cb_cancel_data match = {
751 .fn = io_wq_work_match_all,
752 .cancel_all = true,
753 };
754
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300755 raw_spin_unlock(&wq->lock);
756 while (io_acct_cancel_pending_work(wq, acct, &match))
Hao Xu42abc952022-02-06 17:52:39 +0800757 ;
758 } else {
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300759 raw_spin_unlock(&wq->lock);
Jens Axboe3146cba2021-09-01 11:20:10 -0600760 }
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300761 io_worker_ref_put(wq);
Qiang.zhang66e70be2021-09-09 19:58:22 +0800762 kfree(worker);
Jens Axboe3146cba2021-09-01 11:20:10 -0600763 return;
764 }
765
766 /* re-create attempts grab a new worker ref, drop the existing one */
767 io_worker_release(worker);
768 schedule_work(&worker->work);
769}
770
771static void io_workqueue_create(struct work_struct *work)
772{
773 struct io_worker *worker = container_of(work, struct io_worker, work);
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300774 struct io_wq_acct *acct = io_wq_get_acct(worker);
Jens Axboe3146cba2021-09-01 11:20:10 -0600775
Bixuan Cui71e1cef2021-09-11 16:58:47 +0800776 if (!io_queue_worker_create(worker, acct, create_worker_cont))
Qiang.zhang66e70be2021-09-09 19:58:22 +0800777 kfree(worker);
Jens Axboe3146cba2021-09-01 11:20:10 -0600778}
779
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300780static bool create_io_worker(struct io_wq *wq, int index)
Jens Axboe3146cba2021-09-01 11:20:10 -0600781{
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -0300782 struct io_wq_acct *acct = &wq->acct[index];
Jens Axboe3146cba2021-09-01 11:20:10 -0600783 struct io_worker *worker;
784 struct task_struct *tsk;
785
786 __set_current_state(TASK_RUNNING);
787
Breno Leitaoda64d6d2023-03-10 12:11:07 -0800788 worker = kzalloc(sizeof(*worker), GFP_KERNEL);
Jens Axboe3146cba2021-09-01 11:20:10 -0600789 if (!worker) {
790fail:
791 atomic_dec(&acct->nr_running);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300792 raw_spin_lock(&wq->lock);
Jens Axboe3146cba2021-09-01 11:20:10 -0600793 acct->nr_workers--;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300794 raw_spin_unlock(&wq->lock);
Jens Axboe3146cba2021-09-01 11:20:10 -0600795 io_worker_ref_put(wq);
796 return false;
797 }
798
799 refcount_set(&worker->ref, 1);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300800 worker->wq = wq;
Jens Axboe081b5822022-01-18 19:13:43 -0700801 raw_spin_lock_init(&worker->lock);
Jens Axboe3146cba2021-09-01 11:20:10 -0600802 init_completion(&worker->ref_done);
803
804 if (index == IO_WQ_ACCT_BOUND)
805 worker->flags |= IO_WORKER_F_BOUND;
806
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300807 tsk = create_io_thread(io_wq_worker, worker, NUMA_NO_NODE);
Jens Axboe3146cba2021-09-01 11:20:10 -0600808 if (!IS_ERR(tsk)) {
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300809 io_init_new_worker(wq, worker, tsk);
Jens Axboe3146cba2021-09-01 11:20:10 -0600810 } else if (!io_should_retry_thread(PTR_ERR(tsk))) {
Qiang.zhang66e70be2021-09-09 19:58:22 +0800811 kfree(worker);
Jens Axboe3146cba2021-09-01 11:20:10 -0600812 goto fail;
813 } else {
814 INIT_WORK(&worker->work, io_workqueue_create);
815 schedule_work(&worker->work);
816 }
817
818 return true;
819}
820
Hillf Dantonc4068bf2020-09-26 21:26:55 +0800821/*
822 * Iterate the passed in list and call the specific function for each
823 * worker that isn't exiting
824 */
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300825static bool io_wq_for_each_worker(struct io_wq *wq,
Hillf Dantonc4068bf2020-09-26 21:26:55 +0800826 bool (*func)(struct io_worker *, void *),
827 void *data)
828{
829 struct io_worker *worker;
830 bool ret = false;
831
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300832 list_for_each_entry_rcu(worker, &wq->all_list, all_list) {
Hillf Dantonc4068bf2020-09-26 21:26:55 +0800833 if (io_worker_get(worker)) {
834 /* no task if node is/was offline */
835 if (worker->task)
836 ret = func(worker, data);
837 io_worker_release(worker);
838 if (ret)
839 break;
840 }
841 }
842
843 return ret;
844}
845
846static bool io_wq_worker_wake(struct io_worker *worker, void *data)
847{
Jens Axboe6cf58622022-04-25 19:49:01 -0600848 __set_notify_signal(worker->task);
Hillf Dantonc4068bf2020-09-26 21:26:55 +0800849 wake_up_process(worker->task);
850 return false;
851}
852
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300853static void io_run_cancel(struct io_wq_work *work, struct io_wq *wq)
Pavel Begunkovfc04c392020-03-01 19:18:19 +0300854{
855 do {
Pavel Begunkovfc04c392020-03-01 19:18:19 +0300856 work->flags |= IO_WQ_WORK_CANCEL;
Pavel Begunkov5280f7e2021-02-04 13:52:08 +0000857 wq->do_work(work);
858 work = wq->free_work(work);
Pavel Begunkovfc04c392020-03-01 19:18:19 +0300859 } while (work);
860}
861
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300862static void io_wq_insert_work(struct io_wq *wq, struct io_wq_work *work)
Pavel Begunkov86f3cd12020-03-23 22:57:22 +0300863{
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300864 struct io_wq_acct *acct = io_work_get_acct(wq, work);
Pavel Begunkov86f3cd12020-03-23 22:57:22 +0300865 unsigned int hash;
866 struct io_wq_work *tail;
867
868 if (!io_wq_is_hashed(work)) {
869append:
Jens Axboef95dc202021-08-31 13:57:32 -0600870 wq_list_add_tail(&work->list, &acct->work_list);
Pavel Begunkov86f3cd12020-03-23 22:57:22 +0300871 return;
872 }
873
874 hash = io_get_work_hash(work);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300875 tail = wq->hash_tail[hash];
876 wq->hash_tail[hash] = work;
Pavel Begunkov86f3cd12020-03-23 22:57:22 +0300877 if (!tail)
878 goto append;
879
Jens Axboef95dc202021-08-31 13:57:32 -0600880 wq_list_add_after(&work->list, &tail->list, &acct->work_list);
Pavel Begunkov86f3cd12020-03-23 22:57:22 +0300881}
882
Pavel Begunkov713b9822021-09-08 10:09:29 +0100883static bool io_wq_work_match_item(struct io_wq_work *work, void *data)
884{
885 return work == data;
886}
887
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300888void io_wq_enqueue(struct io_wq *wq, struct io_wq_work *work)
Jens Axboe771b53d02019-10-22 10:25:58 -0600889{
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300890 struct io_wq_acct *acct = io_work_get_acct(wq, work);
Hao Xu42abc952022-02-06 17:52:39 +0800891 struct io_cb_cancel_data match;
Jens Axboe94ffb0a2021-08-30 11:55:22 -0600892 unsigned work_flags = work->flags;
893 bool do_create;
Jens Axboe771b53d02019-10-22 10:25:58 -0600894
Jens Axboe991468d2021-07-23 11:53:54 -0600895 /*
896 * If io-wq is exiting for this task, or if the request has explicitly
897 * been marked as one that should not get executed, cancel it here.
898 */
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300899 if (test_bit(IO_WQ_BIT_EXIT, &wq->state) ||
Jens Axboe991468d2021-07-23 11:53:54 -0600900 (work->flags & IO_WQ_WORK_CANCEL)) {
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300901 io_run_cancel(work, wq);
Jens Axboe4fb6ac32021-02-25 10:17:09 -0700902 return;
903 }
904
Hao Xu42abc952022-02-06 17:52:39 +0800905 raw_spin_lock(&acct->lock);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300906 io_wq_insert_work(wq, work);
Jens Axboef95dc202021-08-31 13:57:32 -0600907 clear_bit(IO_ACCT_STALLED_BIT, &acct->flags);
Hao Xu42abc952022-02-06 17:52:39 +0800908 raw_spin_unlock(&acct->lock);
Jens Axboe94ffb0a2021-08-30 11:55:22 -0600909
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300910 raw_spin_lock(&wq->lock);
Jens Axboe94ffb0a2021-08-30 11:55:22 -0600911 rcu_read_lock();
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300912 do_create = !io_wq_activate_free_worker(wq, acct);
Jens Axboe94ffb0a2021-08-30 11:55:22 -0600913 rcu_read_unlock();
914
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300915 raw_spin_unlock(&wq->lock);
Jens Axboe771b53d02019-10-22 10:25:58 -0600916
Jens Axboe94ffb0a2021-08-30 11:55:22 -0600917 if (do_create && ((work_flags & IO_WQ_WORK_CONCURRENT) ||
Jens Axboe3146cba2021-09-01 11:20:10 -0600918 !atomic_read(&acct->nr_running))) {
919 bool did_create;
920
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300921 did_create = io_wq_create_worker(wq, acct);
Pavel Begunkov713b9822021-09-08 10:09:29 +0100922 if (likely(did_create))
923 return;
924
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300925 raw_spin_lock(&wq->lock);
Hao Xu42abc952022-02-06 17:52:39 +0800926 if (acct->nr_workers) {
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300927 raw_spin_unlock(&wq->lock);
Hao Xu42abc952022-02-06 17:52:39 +0800928 return;
Jens Axboe3146cba2021-09-01 11:20:10 -0600929 }
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300930 raw_spin_unlock(&wq->lock);
Hao Xu42abc952022-02-06 17:52:39 +0800931
932 /* fatal condition, failed to create the first worker */
933 match.fn = io_wq_work_match_item,
934 match.data = work,
935 match.cancel_all = false,
936
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300937 io_acct_cancel_pending_work(wq, acct, &match);
Jens Axboe3146cba2021-09-01 11:20:10 -0600938 }
Jens Axboe771b53d02019-10-22 10:25:58 -0600939}
940
Jens Axboe771b53d02019-10-22 10:25:58 -0600941/*
Pavel Begunkov8766dd52020-03-14 00:31:04 +0300942 * Work items that hash to the same value will not be done in parallel.
943 * Used to limit concurrent writes, generally hashed by inode.
Jens Axboe771b53d02019-10-22 10:25:58 -0600944 */
Pavel Begunkov8766dd52020-03-14 00:31:04 +0300945void io_wq_hash_work(struct io_wq_work *work, void *val)
Jens Axboe771b53d02019-10-22 10:25:58 -0600946{
Pavel Begunkov8766dd52020-03-14 00:31:04 +0300947 unsigned int bit;
Jens Axboe771b53d02019-10-22 10:25:58 -0600948
949 bit = hash_ptr(val, IO_WQ_HASH_ORDER);
950 work->flags |= (IO_WQ_WORK_HASHED | (bit << IO_WQ_HASH_SHIFT));
Jens Axboe771b53d02019-10-22 10:25:58 -0600951}
952
Jens Axboe361aee42022-01-18 19:23:51 -0700953static bool __io_wq_worker_cancel(struct io_worker *worker,
954 struct io_cb_cancel_data *match,
955 struct io_wq_work *work)
956{
957 if (work && match->fn(work, match->data)) {
958 work->flags |= IO_WQ_WORK_CANCEL;
Jens Axboe6cf58622022-04-25 19:49:01 -0600959 __set_notify_signal(worker->task);
Jens Axboe361aee42022-01-18 19:23:51 -0700960 return true;
961 }
962
963 return false;
964}
965
Pavel Begunkov2293b412020-03-07 01:15:39 +0300966static bool io_wq_worker_cancel(struct io_worker *worker, void *data)
Jens Axboe62755e32019-10-28 21:49:21 -0600967{
Pavel Begunkov2293b412020-03-07 01:15:39 +0300968 struct io_cb_cancel_data *match = data;
Jens Axboe62755e32019-10-28 21:49:21 -0600969
970 /*
971 * Hold the lock to avoid ->cur_work going out of scope, caller
Jens Axboe36c2f922019-11-13 09:43:34 -0700972 * may dereference the passed in work.
Jens Axboe62755e32019-10-28 21:49:21 -0600973 */
Jens Axboe081b5822022-01-18 19:13:43 -0700974 raw_spin_lock(&worker->lock);
Jens Axboe361aee42022-01-18 19:23:51 -0700975 if (__io_wq_worker_cancel(worker, match, worker->cur_work) ||
976 __io_wq_worker_cancel(worker, match, worker->next_work))
Pavel Begunkov4f26bda2020-06-15 10:24:03 +0300977 match->nr_running++;
Jens Axboe081b5822022-01-18 19:13:43 -0700978 raw_spin_unlock(&worker->lock);
Jens Axboe771b53d02019-10-22 10:25:58 -0600979
Pavel Begunkov4f26bda2020-06-15 10:24:03 +0300980 return match->nr_running && !match->cancel_all;
Jens Axboe771b53d02019-10-22 10:25:58 -0600981}
982
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300983static inline void io_wq_remove_pending(struct io_wq *wq,
Pavel Begunkov204361a2020-08-23 20:33:10 +0300984 struct io_wq_work *work,
985 struct io_wq_work_node *prev)
986{
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300987 struct io_wq_acct *acct = io_work_get_acct(wq, work);
Pavel Begunkov204361a2020-08-23 20:33:10 +0300988 unsigned int hash = io_get_work_hash(work);
989 struct io_wq_work *prev_work = NULL;
990
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300991 if (io_wq_is_hashed(work) && work == wq->hash_tail[hash]) {
Pavel Begunkov204361a2020-08-23 20:33:10 +0300992 if (prev)
993 prev_work = container_of(prev, struct io_wq_work, list);
994 if (prev_work && io_get_work_hash(prev_work) == hash)
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300995 wq->hash_tail[hash] = prev_work;
Pavel Begunkov204361a2020-08-23 20:33:10 +0300996 else
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -0300997 wq->hash_tail[hash] = NULL;
Pavel Begunkov204361a2020-08-23 20:33:10 +0300998 }
Jens Axboef95dc202021-08-31 13:57:32 -0600999 wq_list_del(&acct->work_list, &work->list, prev);
Pavel Begunkov204361a2020-08-23 20:33:10 +03001000}
1001
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001002static bool io_acct_cancel_pending_work(struct io_wq *wq,
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -03001003 struct io_wq_acct *acct,
Jens Axboe3146cba2021-09-01 11:20:10 -06001004 struct io_cb_cancel_data *match)
Jens Axboe771b53d02019-10-22 10:25:58 -06001005{
Jens Axboe6206f0e2019-11-26 11:59:32 -07001006 struct io_wq_work_node *node, *prev;
Jens Axboe771b53d02019-10-22 10:25:58 -06001007 struct io_wq_work *work;
Jens Axboe771b53d02019-10-22 10:25:58 -06001008
Hao Xu42abc952022-02-06 17:52:39 +08001009 raw_spin_lock(&acct->lock);
Jens Axboe3146cba2021-09-01 11:20:10 -06001010 wq_list_for_each(node, prev, &acct->work_list) {
1011 work = container_of(node, struct io_wq_work, list);
1012 if (!match->fn(work, match->data))
1013 continue;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001014 io_wq_remove_pending(wq, work, prev);
Hao Xu42abc952022-02-06 17:52:39 +08001015 raw_spin_unlock(&acct->lock);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001016 io_run_cancel(work, wq);
Jens Axboe3146cba2021-09-01 11:20:10 -06001017 match->nr_pending++;
1018 /* not safe to continue after unlock */
1019 return true;
1020 }
Hao Xu42abc952022-02-06 17:52:39 +08001021 raw_spin_unlock(&acct->lock);
Jens Axboe3146cba2021-09-01 11:20:10 -06001022
1023 return false;
1024}
1025
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001026static void io_wq_cancel_pending_work(struct io_wq *wq,
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -03001027 struct io_cb_cancel_data *match)
Jens Axboe3146cba2021-09-01 11:20:10 -06001028{
1029 int i;
Pavel Begunkov4f26bda2020-06-15 10:24:03 +03001030retry:
Jens Axboef95dc202021-08-31 13:57:32 -06001031 for (i = 0; i < IO_WQ_ACCT_NR; i++) {
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001032 struct io_wq_acct *acct = io_get_acct(wq, i == 0);
Pavel Begunkov4f26bda2020-06-15 10:24:03 +03001033
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001034 if (io_acct_cancel_pending_work(wq, acct, match)) {
Jens Axboe3146cba2021-09-01 11:20:10 -06001035 if (match->cancel_all)
1036 goto retry;
Jens Axboe36e4c582022-01-18 19:18:20 -07001037 break;
Jens Axboef95dc202021-08-31 13:57:32 -06001038 }
Jens Axboe771b53d02019-10-22 10:25:58 -06001039 }
Pavel Begunkovf4c26652020-06-15 10:24:02 +03001040}
Jens Axboe771b53d02019-10-22 10:25:58 -06001041
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001042static void io_wq_cancel_running_work(struct io_wq *wq,
Pavel Begunkovf4c26652020-06-15 10:24:02 +03001043 struct io_cb_cancel_data *match)
1044{
Jens Axboe771b53d02019-10-22 10:25:58 -06001045 rcu_read_lock();
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001046 io_wq_for_each_worker(wq, io_wq_worker_cancel, match);
Jens Axboe771b53d02019-10-22 10:25:58 -06001047 rcu_read_unlock();
Jens Axboe771b53d02019-10-22 10:25:58 -06001048}
1049
Pavel Begunkov2293b412020-03-07 01:15:39 +03001050enum io_wq_cancel io_wq_cancel_cb(struct io_wq *wq, work_cancel_fn *cancel,
Pavel Begunkov4f26bda2020-06-15 10:24:03 +03001051 void *data, bool cancel_all)
Pavel Begunkov2293b412020-03-07 01:15:39 +03001052{
1053 struct io_cb_cancel_data match = {
Pavel Begunkov4f26bda2020-06-15 10:24:03 +03001054 .fn = cancel,
1055 .data = data,
1056 .cancel_all = cancel_all,
Pavel Begunkov2293b412020-03-07 01:15:39 +03001057 };
Pavel Begunkov2293b412020-03-07 01:15:39 +03001058
Pavel Begunkovf4c26652020-06-15 10:24:02 +03001059 /*
1060 * First check pending list, if we're lucky we can just remove it
1061 * from there. CANCEL_OK means that the work is returned as-new,
1062 * no completion will be posted for it.
Jens Axboeefdf5182022-01-18 19:22:32 -07001063 *
1064 * Then check if a free (going busy) or busy worker has the work
Pavel Begunkovf4c26652020-06-15 10:24:02 +03001065 * currently running. If we find it there, we'll return CANCEL_RUNNING
1066 * as an indication that we attempt to signal cancellation. The
1067 * completion will run normally in this case.
Jens Axboeefdf5182022-01-18 19:22:32 -07001068 *
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001069 * Do both of these while holding the wq->lock, to ensure that
Jens Axboeefdf5182022-01-18 19:22:32 -07001070 * we'll find a work item regardless of state.
Pavel Begunkovf4c26652020-06-15 10:24:02 +03001071 */
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001072 io_wq_cancel_pending_work(wq, &match);
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001073 if (match.nr_pending && !match.cancel_all)
1074 return IO_WQ_CANCEL_OK;
Pavel Begunkovf4c26652020-06-15 10:24:02 +03001075
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001076 raw_spin_lock(&wq->lock);
1077 io_wq_cancel_running_work(wq, &match);
1078 raw_spin_unlock(&wq->lock);
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001079 if (match.nr_running && !match.cancel_all)
1080 return IO_WQ_CANCEL_RUNNING;
Pavel Begunkovf4c26652020-06-15 10:24:02 +03001081
Pavel Begunkov4f26bda2020-06-15 10:24:03 +03001082 if (match.nr_running)
1083 return IO_WQ_CANCEL_RUNNING;
1084 if (match.nr_pending)
1085 return IO_WQ_CANCEL_OK;
Pavel Begunkovf4c26652020-06-15 10:24:02 +03001086 return IO_WQ_CANCEL_NOTFOUND;
Pavel Begunkov2293b412020-03-07 01:15:39 +03001087}
1088
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001089static int io_wq_hash_wake(struct wait_queue_entry *wait, unsigned mode,
Jens Axboee9418942021-02-19 12:33:30 -07001090 int sync, void *key)
1091{
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001092 struct io_wq *wq = container_of(wait, struct io_wq, wait);
Jens Axboef95dc202021-08-31 13:57:32 -06001093 int i;
Jens Axboee9418942021-02-19 12:33:30 -07001094
1095 list_del_init(&wait->entry);
1096
1097 rcu_read_lock();
Jens Axboef95dc202021-08-31 13:57:32 -06001098 for (i = 0; i < IO_WQ_ACCT_NR; i++) {
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -03001099 struct io_wq_acct *acct = &wq->acct[i];
Jens Axboef95dc202021-08-31 13:57:32 -06001100
1101 if (test_and_clear_bit(IO_ACCT_STALLED_BIT, &acct->flags))
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001102 io_wq_activate_free_worker(wq, acct);
Jens Axboef95dc202021-08-31 13:57:32 -06001103 }
Jens Axboee9418942021-02-19 12:33:30 -07001104 rcu_read_unlock();
Jens Axboee9418942021-02-19 12:33:30 -07001105 return 1;
1106}
1107
Jens Axboe576a3472019-11-25 08:49:20 -07001108struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
Jens Axboe771b53d02019-10-22 10:25:58 -06001109{
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001110 int ret, i;
Jens Axboe771b53d02019-10-22 10:25:58 -06001111 struct io_wq *wq;
1112
Pavel Begunkovf5fa38c2020-06-08 21:08:20 +03001113 if (WARN_ON_ONCE(!data->free_work || !data->do_work))
Pavel Begunkove9fd9392020-03-04 16:14:12 +03001114 return ERR_PTR(-EINVAL);
Pavel Begunkove6ab8992021-06-17 18:13:59 +01001115 if (WARN_ON_ONCE(!bounded))
1116 return ERR_PTR(-EINVAL);
Pavel Begunkove9fd9392020-03-04 16:14:12 +03001117
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001118 wq = kzalloc(sizeof(struct io_wq), GFP_KERNEL);
Jens Axboe771b53d02019-10-22 10:25:58 -06001119 if (!wq)
1120 return ERR_PTR(-ENOMEM);
Jens Axboe43c01fb2020-10-22 09:02:50 -06001121 ret = cpuhp_state_add_instance_nocalls(io_wq_online, &wq->cpuhp_node);
1122 if (ret)
Pavel Begunkovc7f405d2021-06-14 02:36:12 +01001123 goto err_wq;
Jens Axboe771b53d02019-10-22 10:25:58 -06001124
Jens Axboee9418942021-02-19 12:33:30 -07001125 refcount_inc(&data->hash->refs);
1126 wq->hash = data->hash;
Pavel Begunkove9fd9392020-03-04 16:14:12 +03001127 wq->free_work = data->free_work;
Pavel Begunkovf5fa38c2020-06-08 21:08:20 +03001128 wq->do_work = data->do_work;
Jens Axboe7d723062019-11-12 22:31:31 -07001129
Jens Axboe43c01fb2020-10-22 09:02:50 -06001130 ret = -ENOMEM;
Jens Axboe771b53d02019-10-22 10:25:58 -06001131
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001132 if (!alloc_cpumask_var(&wq->cpu_mask, GFP_KERNEL))
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001133 goto err;
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001134 cpumask_copy(wq->cpu_mask, cpu_possible_mask);
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -03001135 wq->acct[IO_WQ_ACCT_BOUND].max_workers = bounded;
1136 wq->acct[IO_WQ_ACCT_UNBOUND].max_workers =
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001137 task_rlimit(current, RLIMIT_NPROC);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001138 INIT_LIST_HEAD(&wq->wait.entry);
1139 wq->wait.func = io_wq_hash_wake;
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001140 for (i = 0; i < IO_WQ_ACCT_NR; i++) {
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -03001141 struct io_wq_acct *acct = &wq->acct[i];
Jens Axboef95dc202021-08-31 13:57:32 -06001142
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001143 acct->index = i;
1144 atomic_set(&acct->nr_running, 0);
1145 INIT_WQ_LIST(&acct->work_list);
1146 raw_spin_lock_init(&acct->lock);
Jens Axboe771b53d02019-10-22 10:25:58 -06001147 }
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001148
1149 raw_spin_lock_init(&wq->lock);
1150 INIT_HLIST_NULLS_HEAD(&wq->free_list, 0);
1151 INIT_LIST_HEAD(&wq->all_list);
Jens Axboe771b53d02019-10-22 10:25:58 -06001152
Jens Axboe685fe7f2021-03-08 09:37:51 -07001153 wq->task = get_task_struct(data->task);
Jens Axboe685fe7f2021-03-08 09:37:51 -07001154 atomic_set(&wq->worker_refs, 1);
1155 init_completion(&wq->worker_done);
1156 return wq;
Jens Axboeb60fda62019-11-19 08:37:07 -07001157err:
Jens Axboedc7bbc92021-03-01 09:09:56 -07001158 io_wq_put_hash(data->hash);
Jens Axboe43c01fb2020-10-22 09:02:50 -06001159 cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node);
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001160
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001161 free_cpumask_var(wq->cpu_mask);
Jens Axboe43c01fb2020-10-22 09:02:50 -06001162err_wq:
Jens Axboeb60fda62019-11-19 08:37:07 -07001163 kfree(wq);
Jens Axboe771b53d02019-10-22 10:25:58 -06001164 return ERR_PTR(ret);
1165}
1166
Jens Axboec80ca472021-04-01 19:57:07 -06001167static bool io_task_work_match(struct callback_head *cb, void *data)
1168{
Jens Axboed3e9f732021-08-04 08:37:25 -06001169 struct io_worker *worker;
Jens Axboec80ca472021-04-01 19:57:07 -06001170
Jens Axboe3b33e3f2021-09-08 19:57:26 -06001171 if (cb->func != create_worker_cb && cb->func != create_worker_cont)
Jens Axboec80ca472021-04-01 19:57:07 -06001172 return false;
Jens Axboed3e9f732021-08-04 08:37:25 -06001173 worker = container_of(cb, struct io_worker, create_work);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001174 return worker->wq == data;
Jens Axboec80ca472021-04-01 19:57:07 -06001175}
1176
Pavel Begunkov17a91052021-05-23 15:48:39 +01001177void io_wq_exit_start(struct io_wq *wq)
1178{
1179 set_bit(IO_WQ_BIT_EXIT, &wq->state);
1180}
1181
Jens Axboe71a85382021-12-10 08:29:30 -07001182static void io_wq_cancel_tw_create(struct io_wq *wq)
Jens Axboeafcc4012021-02-26 13:48:19 -07001183{
Jens Axboe685fe7f2021-03-08 09:37:51 -07001184 struct callback_head *cb;
Jens Axboe685fe7f2021-03-08 09:37:51 -07001185
Jens Axboec80ca472021-04-01 19:57:07 -06001186 while ((cb = task_work_cancel_match(wq->task, io_task_work_match, wq)) != NULL) {
Jens Axboed3e9f732021-08-04 08:37:25 -06001187 struct io_worker *worker;
Jens Axboe685fe7f2021-03-08 09:37:51 -07001188
Jens Axboed3e9f732021-08-04 08:37:25 -06001189 worker = container_of(cb, struct io_worker, create_work);
Pavel Begunkov1d5f5ea2021-10-29 13:11:33 +01001190 io_worker_cancel_cb(worker);
Jens Axboee6db6f92023-01-08 10:39:17 -07001191 /*
1192 * Only the worker continuation helper has worker allocated and
1193 * hence needs freeing.
1194 */
1195 if (cb->func == create_worker_cont)
1196 kfree(worker);
Jens Axboeafcc4012021-02-26 13:48:19 -07001197 }
Jens Axboe71a85382021-12-10 08:29:30 -07001198}
1199
1200static void io_wq_exit_workers(struct io_wq *wq)
1201{
Jens Axboe71a85382021-12-10 08:29:30 -07001202 if (!wq->task)
1203 return;
1204
1205 io_wq_cancel_tw_create(wq);
Jens Axboe685fe7f2021-03-08 09:37:51 -07001206
1207 rcu_read_lock();
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001208 io_wq_for_each_worker(wq, io_wq_worker_wake, NULL);
Jens Axboe685fe7f2021-03-08 09:37:51 -07001209 rcu_read_unlock();
1210 io_worker_ref_put(wq);
1211 wait_for_completion(&wq->worker_done);
Zqiang3743c172021-05-26 13:08:26 +08001212
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001213 spin_lock_irq(&wq->hash->wait.lock);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001214 list_del_init(&wq->wait.entry);
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001215 spin_unlock_irq(&wq->hash->wait.lock);
1216
Jens Axboe685fe7f2021-03-08 09:37:51 -07001217 put_task_struct(wq->task);
1218 wq->task = NULL;
Jens Axboeafcc4012021-02-26 13:48:19 -07001219}
1220
Jens Axboe4fb6ac32021-02-25 10:17:09 -07001221static void io_wq_destroy(struct io_wq *wq)
Jens Axboe771b53d02019-10-22 10:25:58 -06001222{
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001223 struct io_cb_cancel_data match = {
1224 .fn = io_wq_work_match_all,
1225 .cancel_all = true,
1226 };
Jens Axboe771b53d02019-10-22 10:25:58 -06001227
Jens Axboe43c01fb2020-10-22 09:02:50 -06001228 cpuhp_state_remove_instance_nocalls(io_wq_online, &wq->cpuhp_node);
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001229 io_wq_cancel_pending_work(wq, &match);
1230 free_cpumask_var(wq->cpu_mask);
Jens Axboee9418942021-02-19 12:33:30 -07001231 io_wq_put_hash(wq->hash);
Jens Axboe771b53d02019-10-22 10:25:58 -06001232 kfree(wq);
Jens Axboe4fb6ac32021-02-25 10:17:09 -07001233}
1234
Jens Axboeafcc4012021-02-26 13:48:19 -07001235void io_wq_put_and_exit(struct io_wq *wq)
1236{
Pavel Begunkov17a91052021-05-23 15:48:39 +01001237 WARN_ON_ONCE(!test_bit(IO_WQ_BIT_EXIT, &wq->state));
1238
Jens Axboe685fe7f2021-03-08 09:37:51 -07001239 io_wq_exit_workers(wq);
Pavel Begunkov382cb032021-06-14 02:36:13 +01001240 io_wq_destroy(wq);
Jens Axboeafcc4012021-02-26 13:48:19 -07001241}
1242
Jens Axboe0e034962021-06-17 10:08:11 -06001243struct online_data {
1244 unsigned int cpu;
1245 bool online;
1246};
1247
Jens Axboe43c01fb2020-10-22 09:02:50 -06001248static bool io_wq_worker_affinity(struct io_worker *worker, void *data)
1249{
Jens Axboe0e034962021-06-17 10:08:11 -06001250 struct online_data *od = data;
Jens Axboe43c01fb2020-10-22 09:02:50 -06001251
Jens Axboe0e034962021-06-17 10:08:11 -06001252 if (od->online)
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001253 cpumask_set_cpu(od->cpu, worker->wq->cpu_mask);
Jens Axboe0e034962021-06-17 10:08:11 -06001254 else
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001255 cpumask_clear_cpu(od->cpu, worker->wq->cpu_mask);
Jens Axboe43c01fb2020-10-22 09:02:50 -06001256 return false;
1257}
1258
Jens Axboe0e034962021-06-17 10:08:11 -06001259static int __io_wq_cpu_online(struct io_wq *wq, unsigned int cpu, bool online)
1260{
1261 struct online_data od = {
1262 .cpu = cpu,
1263 .online = online
1264 };
Jens Axboe0e034962021-06-17 10:08:11 -06001265
1266 rcu_read_lock();
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001267 io_wq_for_each_worker(wq, io_wq_worker_affinity, &od);
Jens Axboe0e034962021-06-17 10:08:11 -06001268 rcu_read_unlock();
1269 return 0;
1270}
1271
Jens Axboe43c01fb2020-10-22 09:02:50 -06001272static int io_wq_cpu_online(unsigned int cpu, struct hlist_node *node)
1273{
1274 struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node);
Jens Axboe43c01fb2020-10-22 09:02:50 -06001275
Jens Axboe0e034962021-06-17 10:08:11 -06001276 return __io_wq_cpu_online(wq, cpu, true);
1277}
1278
1279static int io_wq_cpu_offline(unsigned int cpu, struct hlist_node *node)
1280{
1281 struct io_wq *wq = hlist_entry_safe(node, struct io_wq, cpuhp_node);
1282
1283 return __io_wq_cpu_online(wq, cpu, false);
Jens Axboe43c01fb2020-10-22 09:02:50 -06001284}
1285
Jens Axboefe764212021-06-17 10:19:54 -06001286int io_wq_cpu_affinity(struct io_wq *wq, cpumask_var_t mask)
1287{
Jens Axboefe764212021-06-17 10:19:54 -06001288 rcu_read_lock();
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001289 if (mask)
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001290 cpumask_copy(wq->cpu_mask, mask);
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001291 else
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001292 cpumask_copy(wq->cpu_mask, cpu_possible_mask);
Jens Axboefe764212021-06-17 10:19:54 -06001293 rcu_read_unlock();
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001294
Jens Axboefe764212021-06-17 10:19:54 -06001295 return 0;
1296}
1297
Jens Axboe2e480052021-08-27 11:33:19 -06001298/*
1299 * Set max number of unbounded workers, returns old value. If new_count is 0,
1300 * then just return the old value.
1301 */
1302int io_wq_max_workers(struct io_wq *wq, int *new_count)
1303{
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -03001304 struct io_wq_acct *acct;
Beld Zhang71c9ce22021-11-02 12:32:08 -06001305 int prev[IO_WQ_ACCT_NR];
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001306 int i;
Jens Axboe2e480052021-08-27 11:33:19 -06001307
Eugene Syromiatnikovdd47c102021-09-13 17:44:15 +02001308 BUILD_BUG_ON((int) IO_WQ_ACCT_BOUND != (int) IO_WQ_BOUND);
1309 BUILD_BUG_ON((int) IO_WQ_ACCT_UNBOUND != (int) IO_WQ_UNBOUND);
1310 BUILD_BUG_ON((int) IO_WQ_ACCT_NR != 2);
1311
Hao Xu86127bb2022-02-06 17:52:41 +08001312 for (i = 0; i < IO_WQ_ACCT_NR; i++) {
Jens Axboe2e480052021-08-27 11:33:19 -06001313 if (new_count[i] > task_rlimit(current, RLIMIT_NPROC))
1314 new_count[i] = task_rlimit(current, RLIMIT_NPROC);
1315 }
1316
Beld Zhang71c9ce22021-11-02 12:32:08 -06001317 for (i = 0; i < IO_WQ_ACCT_NR; i++)
1318 prev[i] = 0;
1319
Jens Axboe2e480052021-08-27 11:33:19 -06001320 rcu_read_lock();
Jens Axboe2e480052021-08-27 11:33:19 -06001321
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001322 raw_spin_lock(&wq->lock);
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001323 for (i = 0; i < IO_WQ_ACCT_NR; i++) {
Gabriel Krisman Bertazidfd63ba2023-03-21 22:16:27 -03001324 acct = &wq->acct[i];
Breno Leitaoda64d6d2023-03-10 12:11:07 -08001325 prev[i] = max_t(int, acct->max_workers, prev[i]);
1326 if (new_count[i])
1327 acct->max_workers = new_count[i];
Jens Axboe2e480052021-08-27 11:33:19 -06001328 }
Gabriel Krisman Bertazieb479432023-03-21 22:16:28 -03001329 raw_spin_unlock(&wq->lock);
Jens Axboe2e480052021-08-27 11:33:19 -06001330 rcu_read_unlock();
Beld Zhang71c9ce22021-11-02 12:32:08 -06001331
1332 for (i = 0; i < IO_WQ_ACCT_NR; i++)
1333 new_count[i] = prev[i];
1334
Jens Axboe2e480052021-08-27 11:33:19 -06001335 return 0;
1336}
1337
Jens Axboe43c01fb2020-10-22 09:02:50 -06001338static __init int io_wq_init(void)
1339{
1340 int ret;
1341
1342 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "io-wq/online",
Jens Axboe0e034962021-06-17 10:08:11 -06001343 io_wq_cpu_online, io_wq_cpu_offline);
Jens Axboe43c01fb2020-10-22 09:02:50 -06001344 if (ret < 0)
1345 return ret;
1346 io_wq_online = ret;
1347 return 0;
1348}
1349subsys_initcall(io_wq_init);