blob: 5d101c4053e05f2fddb8c656d543afb62fecd05b [file] [log] [blame]
Maarten Lankhorste9417592014-07-01 12:57:14 +02001/*
2 * Fence mechanism for dma-buf and to allow for asynchronous dma access
3 *
4 * Copyright (C) 2012 Canonical Ltd
5 * Copyright (C) 2012 Texas Instruments
6 *
7 * Authors:
8 * Rob Clark <robdclark@gmail.com>
9 * Maarten Lankhorst <maarten.lankhorst@canonical.com>
10 *
11 * This program is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU General Public License version 2 as published by
13 * the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but WITHOUT
16 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
18 * more details.
19 */
20
21#include <linux/slab.h>
22#include <linux/export.h>
23#include <linux/atomic.h>
Chris Wilsonf54d1862016-10-25 13:00:45 +010024#include <linux/dma-fence.h>
Ingo Molnar174cd4b2017-02-02 19:15:33 +010025#include <linux/sched/signal.h>
Maarten Lankhorste9417592014-07-01 12:57:14 +020026
27#define CREATE_TRACE_POINTS
Chris Wilsonf54d1862016-10-25 13:00:45 +010028#include <trace/events/dma_fence.h>
Maarten Lankhorste9417592014-07-01 12:57:14 +020029
Chris Wilsonf54d1862016-10-25 13:00:45 +010030EXPORT_TRACEPOINT_SYMBOL(dma_fence_emit);
Chris Wilson8c96c672017-01-24 11:57:58 +000031EXPORT_TRACEPOINT_SYMBOL(dma_fence_enable_signal);
Maarten Lankhorste9417592014-07-01 12:57:14 +020032
Thierry Redinge9f3b792014-08-08 12:42:32 +020033/*
Maarten Lankhorste9417592014-07-01 12:57:14 +020034 * fence context counter: each execution context should have its own
35 * fence context, this allows checking if fences belong to the same
36 * context or not. One device can have multiple separate contexts,
37 * and they're used if some engine can run independently of another.
38 */
Chris Wilsonf54d1862016-10-25 13:00:45 +010039static atomic64_t dma_fence_context_counter = ATOMIC64_INIT(0);
Maarten Lankhorste9417592014-07-01 12:57:14 +020040
41/**
Chris Wilsonf54d1862016-10-25 13:00:45 +010042 * dma_fence_context_alloc - allocate an array of fence contexts
Maarten Lankhorste9417592014-07-01 12:57:14 +020043 * @num: [in] amount of contexts to allocate
44 *
45 * This function will return the first index of the number of fences allocated.
46 * The fence context is used for setting fence->context to a unique number.
47 */
Chris Wilsonf54d1862016-10-25 13:00:45 +010048u64 dma_fence_context_alloc(unsigned num)
Maarten Lankhorste9417592014-07-01 12:57:14 +020049{
Daniel Vetter6ce31262017-07-20 14:51:07 +020050 WARN_ON(!num);
Chris Wilsonf54d1862016-10-25 13:00:45 +010051 return atomic64_add_return(num, &dma_fence_context_counter) - num;
Maarten Lankhorste9417592014-07-01 12:57:14 +020052}
Chris Wilsonf54d1862016-10-25 13:00:45 +010053EXPORT_SYMBOL(dma_fence_context_alloc);
Maarten Lankhorste9417592014-07-01 12:57:14 +020054
55/**
Chris Wilsonf54d1862016-10-25 13:00:45 +010056 * dma_fence_signal_locked - signal completion of a fence
Maarten Lankhorste9417592014-07-01 12:57:14 +020057 * @fence: the fence to signal
58 *
59 * Signal completion for software callbacks on a fence, this will unblock
Chris Wilsonf54d1862016-10-25 13:00:45 +010060 * dma_fence_wait() calls and run all the callbacks added with
61 * dma_fence_add_callback(). Can be called multiple times, but since a fence
Maarten Lankhorste9417592014-07-01 12:57:14 +020062 * can only go from unsignaled to signaled state, it will only be effective
63 * the first time.
64 *
Chris Wilsonf54d1862016-10-25 13:00:45 +010065 * Unlike dma_fence_signal, this function must be called with fence->lock held.
Maarten Lankhorste9417592014-07-01 12:57:14 +020066 */
Chris Wilsonf54d1862016-10-25 13:00:45 +010067int dma_fence_signal_locked(struct dma_fence *fence)
Maarten Lankhorste9417592014-07-01 12:57:14 +020068{
Chris Wilsonf54d1862016-10-25 13:00:45 +010069 struct dma_fence_cb *cur, *tmp;
Maarten Lankhorste9417592014-07-01 12:57:14 +020070 int ret = 0;
71
Rob Clark78010cd2016-10-24 15:57:10 -040072 lockdep_assert_held(fence->lock);
73
Maarten Lankhorste9417592014-07-01 12:57:14 +020074 if (WARN_ON(!fence))
75 return -EINVAL;
76
Chris Wilsonf54d1862016-10-25 13:00:45 +010077 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
Maarten Lankhorste9417592014-07-01 12:57:14 +020078 ret = -EINVAL;
79
80 /*
Chris Wilsonf54d1862016-10-25 13:00:45 +010081 * we might have raced with the unlocked dma_fence_signal,
Maarten Lankhorste9417592014-07-01 12:57:14 +020082 * still run through all callbacks
83 */
Chris Wilson76250f22017-02-14 12:40:01 +000084 } else {
85 fence->timestamp = ktime_get();
86 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
Chris Wilsonf54d1862016-10-25 13:00:45 +010087 trace_dma_fence_signaled(fence);
Chris Wilson76250f22017-02-14 12:40:01 +000088 }
Maarten Lankhorste9417592014-07-01 12:57:14 +020089
90 list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {
91 list_del_init(&cur->node);
92 cur->func(fence, cur);
93 }
94 return ret;
95}
Chris Wilsonf54d1862016-10-25 13:00:45 +010096EXPORT_SYMBOL(dma_fence_signal_locked);
Maarten Lankhorste9417592014-07-01 12:57:14 +020097
98/**
Chris Wilsonf54d1862016-10-25 13:00:45 +010099 * dma_fence_signal - signal completion of a fence
Maarten Lankhorste9417592014-07-01 12:57:14 +0200100 * @fence: the fence to signal
101 *
102 * Signal completion for software callbacks on a fence, this will unblock
Chris Wilsonf54d1862016-10-25 13:00:45 +0100103 * dma_fence_wait() calls and run all the callbacks added with
104 * dma_fence_add_callback(). Can be called multiple times, but since a fence
Maarten Lankhorste9417592014-07-01 12:57:14 +0200105 * can only go from unsignaled to signaled state, it will only be effective
106 * the first time.
107 */
Chris Wilsonf54d1862016-10-25 13:00:45 +0100108int dma_fence_signal(struct dma_fence *fence)
Maarten Lankhorste9417592014-07-01 12:57:14 +0200109{
110 unsigned long flags;
111
112 if (!fence)
113 return -EINVAL;
114
Chris Wilsonf54d1862016-10-25 13:00:45 +0100115 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
Maarten Lankhorste9417592014-07-01 12:57:14 +0200116 return -EINVAL;
117
Chris Wilson76250f22017-02-14 12:40:01 +0000118 fence->timestamp = ktime_get();
119 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
Chris Wilsonf54d1862016-10-25 13:00:45 +0100120 trace_dma_fence_signaled(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200121
Chris Wilsonf54d1862016-10-25 13:00:45 +0100122 if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags)) {
123 struct dma_fence_cb *cur, *tmp;
Maarten Lankhorste9417592014-07-01 12:57:14 +0200124
125 spin_lock_irqsave(fence->lock, flags);
126 list_for_each_entry_safe(cur, tmp, &fence->cb_list, node) {
127 list_del_init(&cur->node);
128 cur->func(fence, cur);
129 }
130 spin_unlock_irqrestore(fence->lock, flags);
131 }
132 return 0;
133}
Chris Wilsonf54d1862016-10-25 13:00:45 +0100134EXPORT_SYMBOL(dma_fence_signal);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200135
136/**
Chris Wilsonf54d1862016-10-25 13:00:45 +0100137 * dma_fence_wait_timeout - sleep until the fence gets signaled
Maarten Lankhorste9417592014-07-01 12:57:14 +0200138 * or until timeout elapses
139 * @fence: [in] the fence to wait on
140 * @intr: [in] if true, do an interruptible wait
141 * @timeout: [in] timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
142 *
143 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
144 * remaining timeout in jiffies on success. Other error values may be
145 * returned on custom implementations.
146 *
147 * Performs a synchronous wait on this fence. It is assumed the caller
148 * directly or indirectly (buf-mgr between reservation and committing)
149 * holds a reference to the fence, otherwise the fence might be
150 * freed before return, resulting in undefined behavior.
151 */
152signed long
Chris Wilsonf54d1862016-10-25 13:00:45 +0100153dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
Maarten Lankhorste9417592014-07-01 12:57:14 +0200154{
155 signed long ret;
156
157 if (WARN_ON(timeout < 0))
158 return -EINVAL;
159
Chris Wilsonf54d1862016-10-25 13:00:45 +0100160 trace_dma_fence_wait_start(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200161 ret = fence->ops->wait(fence, intr, timeout);
Chris Wilsonf54d1862016-10-25 13:00:45 +0100162 trace_dma_fence_wait_end(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200163 return ret;
164}
Chris Wilsonf54d1862016-10-25 13:00:45 +0100165EXPORT_SYMBOL(dma_fence_wait_timeout);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200166
Chris Wilsonf54d1862016-10-25 13:00:45 +0100167void dma_fence_release(struct kref *kref)
Maarten Lankhorste9417592014-07-01 12:57:14 +0200168{
Chris Wilsonf54d1862016-10-25 13:00:45 +0100169 struct dma_fence *fence =
170 container_of(kref, struct dma_fence, refcount);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200171
Chris Wilsonf54d1862016-10-25 13:00:45 +0100172 trace_dma_fence_destroy(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200173
Daniel Vetter6ce31262017-07-20 14:51:07 +0200174 WARN_ON(!list_empty(&fence->cb_list));
Maarten Lankhorste9417592014-07-01 12:57:14 +0200175
176 if (fence->ops->release)
177 fence->ops->release(fence);
178 else
Chris Wilsonf54d1862016-10-25 13:00:45 +0100179 dma_fence_free(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200180}
Chris Wilsonf54d1862016-10-25 13:00:45 +0100181EXPORT_SYMBOL(dma_fence_release);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200182
Chris Wilsonf54d1862016-10-25 13:00:45 +0100183void dma_fence_free(struct dma_fence *fence)
Maarten Lankhorste9417592014-07-01 12:57:14 +0200184{
Maarten Lankhorst3c3b1772014-07-01 12:58:00 +0200185 kfree_rcu(fence, rcu);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200186}
Chris Wilsonf54d1862016-10-25 13:00:45 +0100187EXPORT_SYMBOL(dma_fence_free);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200188
189/**
Chris Wilsonf54d1862016-10-25 13:00:45 +0100190 * dma_fence_enable_sw_signaling - enable signaling on fence
Maarten Lankhorste9417592014-07-01 12:57:14 +0200191 * @fence: [in] the fence to enable
192 *
193 * this will request for sw signaling to be enabled, to make the fence
194 * complete as soon as possible
195 */
Chris Wilsonf54d1862016-10-25 13:00:45 +0100196void dma_fence_enable_sw_signaling(struct dma_fence *fence)
Maarten Lankhorste9417592014-07-01 12:57:14 +0200197{
198 unsigned long flags;
199
Chris Wilsonf54d1862016-10-25 13:00:45 +0100200 if (!test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
201 &fence->flags) &&
202 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
203 trace_dma_fence_enable_signal(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200204
205 spin_lock_irqsave(fence->lock, flags);
206
207 if (!fence->ops->enable_signaling(fence))
Chris Wilsonf54d1862016-10-25 13:00:45 +0100208 dma_fence_signal_locked(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200209
210 spin_unlock_irqrestore(fence->lock, flags);
211 }
212}
Chris Wilsonf54d1862016-10-25 13:00:45 +0100213EXPORT_SYMBOL(dma_fence_enable_sw_signaling);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200214
215/**
Chris Wilsonf54d1862016-10-25 13:00:45 +0100216 * dma_fence_add_callback - add a callback to be called when the fence
Maarten Lankhorste9417592014-07-01 12:57:14 +0200217 * is signaled
218 * @fence: [in] the fence to wait on
219 * @cb: [in] the callback to register
220 * @func: [in] the function to call
221 *
Chris Wilsonf54d1862016-10-25 13:00:45 +0100222 * cb will be initialized by dma_fence_add_callback, no initialization
Maarten Lankhorste9417592014-07-01 12:57:14 +0200223 * by the caller is required. Any number of callbacks can be registered
224 * to a fence, but a callback can only be registered to one fence at a time.
225 *
226 * Note that the callback can be called from an atomic context. If
227 * fence is already signaled, this function will return -ENOENT (and
228 * *not* call the callback)
229 *
230 * Add a software callback to the fence. Same restrictions apply to
Chris Wilsonf54d1862016-10-25 13:00:45 +0100231 * refcount as it does to dma_fence_wait, however the caller doesn't need to
Maarten Lankhorste9417592014-07-01 12:57:14 +0200232 * keep a refcount to fence afterwards: when software access is enabled,
233 * the creator of the fence is required to keep the fence alive until
Chris Wilsonf54d1862016-10-25 13:00:45 +0100234 * after it signals with dma_fence_signal. The callback itself can be called
Maarten Lankhorste9417592014-07-01 12:57:14 +0200235 * from irq context.
236 *
Gustavo Padovanf642de12017-02-15 15:57:25 -0200237 * Returns 0 in case of success, -ENOENT if the fence is already signaled
238 * and -EINVAL in case of error.
Maarten Lankhorste9417592014-07-01 12:57:14 +0200239 */
Chris Wilsonf54d1862016-10-25 13:00:45 +0100240int dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *cb,
241 dma_fence_func_t func)
Maarten Lankhorste9417592014-07-01 12:57:14 +0200242{
243 unsigned long flags;
244 int ret = 0;
245 bool was_set;
246
247 if (WARN_ON(!fence || !func))
248 return -EINVAL;
249
Chris Wilsonf54d1862016-10-25 13:00:45 +0100250 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
Maarten Lankhorste9417592014-07-01 12:57:14 +0200251 INIT_LIST_HEAD(&cb->node);
252 return -ENOENT;
253 }
254
255 spin_lock_irqsave(fence->lock, flags);
256
Chris Wilsonf54d1862016-10-25 13:00:45 +0100257 was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
258 &fence->flags);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200259
Chris Wilsonf54d1862016-10-25 13:00:45 +0100260 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
Maarten Lankhorste9417592014-07-01 12:57:14 +0200261 ret = -ENOENT;
262 else if (!was_set) {
Chris Wilsonf54d1862016-10-25 13:00:45 +0100263 trace_dma_fence_enable_signal(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200264
265 if (!fence->ops->enable_signaling(fence)) {
Chris Wilsonf54d1862016-10-25 13:00:45 +0100266 dma_fence_signal_locked(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200267 ret = -ENOENT;
268 }
269 }
270
271 if (!ret) {
272 cb->func = func;
273 list_add_tail(&cb->node, &fence->cb_list);
274 } else
275 INIT_LIST_HEAD(&cb->node);
276 spin_unlock_irqrestore(fence->lock, flags);
277
278 return ret;
279}
Chris Wilsonf54d1862016-10-25 13:00:45 +0100280EXPORT_SYMBOL(dma_fence_add_callback);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200281
282/**
Chris Wilsond6c99f42017-01-04 14:12:21 +0000283 * dma_fence_get_status - returns the status upon completion
284 * @fence: [in] the dma_fence to query
285 *
286 * This wraps dma_fence_get_status_locked() to return the error status
287 * condition on a signaled fence. See dma_fence_get_status_locked() for more
288 * details.
289 *
290 * Returns 0 if the fence has not yet been signaled, 1 if the fence has
291 * been signaled without an error condition, or a negative error code
292 * if the fence has been completed in err.
293 */
294int dma_fence_get_status(struct dma_fence *fence)
295{
296 unsigned long flags;
297 int status;
298
299 spin_lock_irqsave(fence->lock, flags);
300 status = dma_fence_get_status_locked(fence);
301 spin_unlock_irqrestore(fence->lock, flags);
302
303 return status;
304}
305EXPORT_SYMBOL(dma_fence_get_status);
306
307/**
Chris Wilsonf54d1862016-10-25 13:00:45 +0100308 * dma_fence_remove_callback - remove a callback from the signaling list
Maarten Lankhorste9417592014-07-01 12:57:14 +0200309 * @fence: [in] the fence to wait on
310 * @cb: [in] the callback to remove
311 *
312 * Remove a previously queued callback from the fence. This function returns
Masanari Iidaf353d712014-10-22 00:00:14 +0900313 * true if the callback is successfully removed, or false if the fence has
Maarten Lankhorste9417592014-07-01 12:57:14 +0200314 * already been signaled.
315 *
316 * *WARNING*:
317 * Cancelling a callback should only be done if you really know what you're
318 * doing, since deadlocks and race conditions could occur all too easily. For
319 * this reason, it should only ever be done on hardware lockup recovery,
320 * with a reference held to the fence.
321 */
322bool
Chris Wilsonf54d1862016-10-25 13:00:45 +0100323dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb)
Maarten Lankhorste9417592014-07-01 12:57:14 +0200324{
325 unsigned long flags;
326 bool ret;
327
328 spin_lock_irqsave(fence->lock, flags);
329
330 ret = !list_empty(&cb->node);
331 if (ret)
332 list_del_init(&cb->node);
333
334 spin_unlock_irqrestore(fence->lock, flags);
335
336 return ret;
337}
Chris Wilsonf54d1862016-10-25 13:00:45 +0100338EXPORT_SYMBOL(dma_fence_remove_callback);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200339
340struct default_wait_cb {
Chris Wilsonf54d1862016-10-25 13:00:45 +0100341 struct dma_fence_cb base;
Maarten Lankhorste9417592014-07-01 12:57:14 +0200342 struct task_struct *task;
343};
344
345static void
Chris Wilsonf54d1862016-10-25 13:00:45 +0100346dma_fence_default_wait_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
Maarten Lankhorste9417592014-07-01 12:57:14 +0200347{
348 struct default_wait_cb *wait =
349 container_of(cb, struct default_wait_cb, base);
350
351 wake_up_state(wait->task, TASK_NORMAL);
352}
353
354/**
Chris Wilsonf54d1862016-10-25 13:00:45 +0100355 * dma_fence_default_wait - default sleep until the fence gets signaled
Maarten Lankhorste9417592014-07-01 12:57:14 +0200356 * or until timeout elapses
357 * @fence: [in] the fence to wait on
358 * @intr: [in] if true, do an interruptible wait
359 * @timeout: [in] timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
360 *
361 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or the
Alex Deucherbcc004b2016-11-07 16:16:13 -0500362 * remaining timeout in jiffies on success. If timeout is zero the value one is
363 * returned if the fence is already signaled for consistency with other
364 * functions taking a jiffies timeout.
Maarten Lankhorste9417592014-07-01 12:57:14 +0200365 */
366signed long
Chris Wilsonf54d1862016-10-25 13:00:45 +0100367dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
Maarten Lankhorste9417592014-07-01 12:57:14 +0200368{
369 struct default_wait_cb cb;
370 unsigned long flags;
Alex Deucherbcc004b2016-11-07 16:16:13 -0500371 signed long ret = timeout ? timeout : 1;
Maarten Lankhorste9417592014-07-01 12:57:14 +0200372 bool was_set;
373
Chris Wilsonf54d1862016-10-25 13:00:45 +0100374 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
Alex Deucherbcc004b2016-11-07 16:16:13 -0500375 return ret;
Maarten Lankhorste9417592014-07-01 12:57:14 +0200376
377 spin_lock_irqsave(fence->lock, flags);
378
379 if (intr && signal_pending(current)) {
380 ret = -ERESTARTSYS;
381 goto out;
382 }
383
Chris Wilsonf54d1862016-10-25 13:00:45 +0100384 was_set = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
385 &fence->flags);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200386
Chris Wilsonf54d1862016-10-25 13:00:45 +0100387 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
Maarten Lankhorste9417592014-07-01 12:57:14 +0200388 goto out;
389
390 if (!was_set) {
Chris Wilsonf54d1862016-10-25 13:00:45 +0100391 trace_dma_fence_enable_signal(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200392
393 if (!fence->ops->enable_signaling(fence)) {
Chris Wilsonf54d1862016-10-25 13:00:45 +0100394 dma_fence_signal_locked(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200395 goto out;
396 }
397 }
398
Andres Rodriguez03c0c5f2017-04-26 10:46:20 -0400399 if (!timeout) {
400 ret = 0;
401 goto out;
402 }
403
Chris Wilsonf54d1862016-10-25 13:00:45 +0100404 cb.base.func = dma_fence_default_wait_cb;
Maarten Lankhorste9417592014-07-01 12:57:14 +0200405 cb.task = current;
406 list_add(&cb.base.node, &fence->cb_list);
407
Chris Wilsonf54d1862016-10-25 13:00:45 +0100408 while (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && ret > 0) {
Maarten Lankhorste9417592014-07-01 12:57:14 +0200409 if (intr)
410 __set_current_state(TASK_INTERRUPTIBLE);
411 else
412 __set_current_state(TASK_UNINTERRUPTIBLE);
413 spin_unlock_irqrestore(fence->lock, flags);
414
415 ret = schedule_timeout(ret);
416
417 spin_lock_irqsave(fence->lock, flags);
418 if (ret > 0 && intr && signal_pending(current))
419 ret = -ERESTARTSYS;
420 }
421
422 if (!list_empty(&cb.base.node))
423 list_del(&cb.base.node);
424 __set_current_state(TASK_RUNNING);
425
426out:
427 spin_unlock_irqrestore(fence->lock, flags);
428 return ret;
429}
Chris Wilsonf54d1862016-10-25 13:00:45 +0100430EXPORT_SYMBOL(dma_fence_default_wait);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200431
Christian Königa5194352015-10-20 16:34:16 +0200432static bool
monk.liu7392b4b2016-11-04 16:16:09 -0400433dma_fence_test_signaled_any(struct dma_fence **fences, uint32_t count,
434 uint32_t *idx)
Christian Königa5194352015-10-20 16:34:16 +0200435{
436 int i;
437
438 for (i = 0; i < count; ++i) {
Chris Wilsonf54d1862016-10-25 13:00:45 +0100439 struct dma_fence *fence = fences[i];
monk.liu7392b4b2016-11-04 16:16:09 -0400440 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
441 if (idx)
442 *idx = i;
Christian Königa5194352015-10-20 16:34:16 +0200443 return true;
monk.liu7392b4b2016-11-04 16:16:09 -0400444 }
Christian Königa5194352015-10-20 16:34:16 +0200445 }
446 return false;
447}
448
449/**
Chris Wilsonf54d1862016-10-25 13:00:45 +0100450 * dma_fence_wait_any_timeout - sleep until any fence gets signaled
Christian Königa5194352015-10-20 16:34:16 +0200451 * or until timeout elapses
452 * @fences: [in] array of fences to wait on
453 * @count: [in] number of fences to wait on
454 * @intr: [in] if true, do an interruptible wait
455 * @timeout: [in] timeout value in jiffies, or MAX_SCHEDULE_TIMEOUT
monk.liu7392b4b2016-11-04 16:16:09 -0400456 * @idx: [out] the first signaled fence index, meaningful only on
457 * positive return
Christian Königa5194352015-10-20 16:34:16 +0200458 *
459 * Returns -EINVAL on custom fence wait implementation, -ERESTARTSYS if
460 * interrupted, 0 if the wait timed out, or the remaining timeout in jiffies
461 * on success.
462 *
463 * Synchronous waits for the first fence in the array to be signaled. The
464 * caller needs to hold a reference to all fences in the array, otherwise a
465 * fence might be freed before return, resulting in undefined behavior.
466 */
467signed long
Chris Wilsonf54d1862016-10-25 13:00:45 +0100468dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count,
monk.liu7392b4b2016-11-04 16:16:09 -0400469 bool intr, signed long timeout, uint32_t *idx)
Christian Königa5194352015-10-20 16:34:16 +0200470{
471 struct default_wait_cb *cb;
472 signed long ret = timeout;
473 unsigned i;
474
475 if (WARN_ON(!fences || !count || timeout < 0))
476 return -EINVAL;
477
478 if (timeout == 0) {
479 for (i = 0; i < count; ++i)
monk.liu7392b4b2016-11-04 16:16:09 -0400480 if (dma_fence_is_signaled(fences[i])) {
481 if (idx)
482 *idx = i;
Christian Königa5194352015-10-20 16:34:16 +0200483 return 1;
monk.liu7392b4b2016-11-04 16:16:09 -0400484 }
Christian Königa5194352015-10-20 16:34:16 +0200485
486 return 0;
487 }
488
489 cb = kcalloc(count, sizeof(struct default_wait_cb), GFP_KERNEL);
490 if (cb == NULL) {
491 ret = -ENOMEM;
492 goto err_free_cb;
493 }
494
495 for (i = 0; i < count; ++i) {
Chris Wilsonf54d1862016-10-25 13:00:45 +0100496 struct dma_fence *fence = fences[i];
Christian Königa5194352015-10-20 16:34:16 +0200497
Chris Wilsonf54d1862016-10-25 13:00:45 +0100498 if (fence->ops->wait != dma_fence_default_wait) {
Christian Königa5194352015-10-20 16:34:16 +0200499 ret = -EINVAL;
500 goto fence_rm_cb;
501 }
502
503 cb[i].task = current;
Chris Wilsonf54d1862016-10-25 13:00:45 +0100504 if (dma_fence_add_callback(fence, &cb[i].base,
505 dma_fence_default_wait_cb)) {
Christian Königa5194352015-10-20 16:34:16 +0200506 /* This fence is already signaled */
monk.liu7392b4b2016-11-04 16:16:09 -0400507 if (idx)
508 *idx = i;
Christian Königa5194352015-10-20 16:34:16 +0200509 goto fence_rm_cb;
510 }
511 }
512
513 while (ret > 0) {
514 if (intr)
515 set_current_state(TASK_INTERRUPTIBLE);
516 else
517 set_current_state(TASK_UNINTERRUPTIBLE);
518
monk.liu7392b4b2016-11-04 16:16:09 -0400519 if (dma_fence_test_signaled_any(fences, count, idx))
Christian Königa5194352015-10-20 16:34:16 +0200520 break;
521
522 ret = schedule_timeout(ret);
523
524 if (ret > 0 && intr && signal_pending(current))
525 ret = -ERESTARTSYS;
526 }
527
528 __set_current_state(TASK_RUNNING);
529
530fence_rm_cb:
531 while (i-- > 0)
Chris Wilsonf54d1862016-10-25 13:00:45 +0100532 dma_fence_remove_callback(fences[i], &cb[i].base);
Christian Königa5194352015-10-20 16:34:16 +0200533
534err_free_cb:
535 kfree(cb);
536
537 return ret;
538}
Chris Wilsonf54d1862016-10-25 13:00:45 +0100539EXPORT_SYMBOL(dma_fence_wait_any_timeout);
Christian Königa5194352015-10-20 16:34:16 +0200540
Maarten Lankhorste9417592014-07-01 12:57:14 +0200541/**
Chris Wilsonf54d1862016-10-25 13:00:45 +0100542 * dma_fence_init - Initialize a custom fence.
Maarten Lankhorste9417592014-07-01 12:57:14 +0200543 * @fence: [in] the fence to initialize
Chris Wilsonf54d1862016-10-25 13:00:45 +0100544 * @ops: [in] the dma_fence_ops for operations on this fence
Maarten Lankhorste9417592014-07-01 12:57:14 +0200545 * @lock: [in] the irqsafe spinlock to use for locking this fence
546 * @context: [in] the execution context this fence is run on
547 * @seqno: [in] a linear increasing sequence number for this context
548 *
549 * Initializes an allocated fence, the caller doesn't have to keep its
550 * refcount after committing with this fence, but it will need to hold a
Chris Wilsonf54d1862016-10-25 13:00:45 +0100551 * refcount again if dma_fence_ops.enable_signaling gets called. This can
Maarten Lankhorste9417592014-07-01 12:57:14 +0200552 * be used for other implementing other types of fence.
553 *
554 * context and seqno are used for easy comparison between fences, allowing
Chris Wilsonf54d1862016-10-25 13:00:45 +0100555 * to check which fence is later by simply using dma_fence_later.
Maarten Lankhorste9417592014-07-01 12:57:14 +0200556 */
557void
Chris Wilsonf54d1862016-10-25 13:00:45 +0100558dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
559 spinlock_t *lock, u64 context, unsigned seqno)
Maarten Lankhorste9417592014-07-01 12:57:14 +0200560{
561 BUG_ON(!lock);
562 BUG_ON(!ops || !ops->wait || !ops->enable_signaling ||
563 !ops->get_driver_name || !ops->get_timeline_name);
564
565 kref_init(&fence->refcount);
566 fence->ops = ops;
567 INIT_LIST_HEAD(&fence->cb_list);
568 fence->lock = lock;
569 fence->context = context;
570 fence->seqno = seqno;
571 fence->flags = 0UL;
Chris Wilsona009e972017-01-04 14:12:22 +0000572 fence->error = 0;
Maarten Lankhorste9417592014-07-01 12:57:14 +0200573
Chris Wilsonf54d1862016-10-25 13:00:45 +0100574 trace_dma_fence_init(fence);
Maarten Lankhorste9417592014-07-01 12:57:14 +0200575}
Chris Wilsonf54d1862016-10-25 13:00:45 +0100576EXPORT_SYMBOL(dma_fence_init);