blob: 8a08ffde31e7580b48db1a4e685c5bdb456b34a4 [file] [log] [blame] [edit]
// SPDX-License-Identifier: GPL-2.0-only
/*
* dma-fence-array: aggregate fences to be waited together
*
* Copyright (C) 2016 Collabora Ltd
* Copyright (C) 2016 Advanced Micro Devices, Inc.
* Authors:
* Gustavo Padovan <gustavo@padovan.org>
* Christian König <christian.koenig@amd.com>
*/
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/dma-fence-array.h>
#define PENDING_ERROR 1
static const char *dma_fence_array_get_driver_name(struct dma_fence *fence)
{
return "dma_fence_array";
}
static const char *dma_fence_array_get_timeline_name(struct dma_fence *fence)
{
return "unbound";
}
static void dma_fence_array_set_pending_error(struct dma_fence_array *array,
int error)
{
/*
* Propagate the first error reported by any of our fences, but only
* before we ourselves are signaled.
*/
if (error)
cmpxchg(&array->base.error, PENDING_ERROR, error);
}
static void dma_fence_array_clear_pending_error(struct dma_fence_array *array)
{
/* Clear the error flag if not actually set. */
cmpxchg(&array->base.error, PENDING_ERROR, 0);
}
static void irq_dma_fence_array_work(struct irq_work *wrk)
{
struct dma_fence_array *array = container_of(wrk, typeof(*array), work);
dma_fence_array_clear_pending_error(array);
dma_fence_signal(&array->base);
dma_fence_put(&array->base);
}
static void dma_fence_array_cb_func(struct dma_fence *f,
struct dma_fence_cb *cb)
{
struct dma_fence_array_cb *array_cb =
container_of(cb, struct dma_fence_array_cb, cb);
struct dma_fence_array *array = array_cb->array;
dma_fence_array_set_pending_error(array, f->error);
if (atomic_dec_and_test(&array->num_pending))
irq_work_queue(&array->work);
else
dma_fence_put(&array->base);
}
static bool dma_fence_array_enable_signaling(struct dma_fence *fence)
{
struct dma_fence_array *array = to_dma_fence_array(fence);
struct dma_fence_array_cb *cb = array->callbacks;
unsigned i;
for (i = 0; i < array->num_fences; ++i) {
cb[i].array = array;
/*
* As we may report that the fence is signaled before all
* callbacks are complete, we need to take an additional
* reference count on the array so that we do not free it too
* early. The core fence handling will only hold the reference
* until we signal the array as complete (but that is now
* insufficient).
*/
dma_fence_get(&array->base);
if (dma_fence_add_callback(array->fences[i], &cb[i].cb,
dma_fence_array_cb_func)) {
int error = array->fences[i]->error;
dma_fence_array_set_pending_error(array, error);
dma_fence_put(&array->base);
if (atomic_dec_and_test(&array->num_pending)) {
dma_fence_array_clear_pending_error(array);
return false;
}
}
}
return true;
}
static bool dma_fence_array_signaled(struct dma_fence *fence)
{
struct dma_fence_array *array = to_dma_fence_array(fence);
if (atomic_read(&array->num_pending) > 0)
return false;
dma_fence_array_clear_pending_error(array);
return true;
}
static void dma_fence_array_release(struct dma_fence *fence)
{
struct dma_fence_array *array = to_dma_fence_array(fence);
unsigned i;
for (i = 0; i < array->num_fences; ++i)
dma_fence_put(array->fences[i]);
kfree(array->fences);
dma_fence_free(fence);
}
static void dma_fence_array_set_deadline(struct dma_fence *fence,
ktime_t deadline)
{
struct dma_fence_array *array = to_dma_fence_array(fence);
unsigned i;
for (i = 0; i < array->num_fences; ++i)
dma_fence_set_deadline(array->fences[i], deadline);
}
const struct dma_fence_ops dma_fence_array_ops = {
.get_driver_name = dma_fence_array_get_driver_name,
.get_timeline_name = dma_fence_array_get_timeline_name,
.enable_signaling = dma_fence_array_enable_signaling,
.signaled = dma_fence_array_signaled,
.release = dma_fence_array_release,
.set_deadline = dma_fence_array_set_deadline,
};
EXPORT_SYMBOL(dma_fence_array_ops);
/**
* dma_fence_array_alloc - Allocate a custom fence array
* @num_fences: [in] number of fences to add in the array
*
* Return dma fence array on success, NULL on failure
*/
struct dma_fence_array *dma_fence_array_alloc(int num_fences)
{
struct dma_fence_array *array;
return kzalloc(struct_size(array, callbacks, num_fences), GFP_KERNEL);
}
EXPORT_SYMBOL(dma_fence_array_alloc);
/**
* dma_fence_array_init - Init a custom fence array
* @array: [in] dma fence array to arm
* @num_fences: [in] number of fences to add in the array
* @fences: [in] array containing the fences
* @context: [in] fence context to use
* @seqno: [in] sequence number to use
* @signal_on_any: [in] signal on any fence in the array
*
* Implementation of @dma_fence_array_create without allocation. Useful to init
* a preallocated dma fence array in the path of reclaim or dma fence signaling.
*/
void dma_fence_array_init(struct dma_fence_array *array,
int num_fences, struct dma_fence **fences,
u64 context, unsigned seqno,
bool signal_on_any)
{
WARN_ON(!num_fences || !fences);
array->num_fences = num_fences;
spin_lock_init(&array->lock);
dma_fence_init(&array->base, &dma_fence_array_ops, &array->lock,
context, seqno);
init_irq_work(&array->work, irq_dma_fence_array_work);
atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences);
array->fences = fences;
array->base.error = PENDING_ERROR;
/*
* dma_fence_array objects should never contain any other fence
* containers or otherwise we run into recursion and potential kernel
* stack overflow on operations on the dma_fence_array.
*
* The correct way of handling this is to flatten out the array by the
* caller instead.
*
* Enforce this here by checking that we don't create a dma_fence_array
* with any container inside.
*/
while (num_fences--)
WARN_ON(dma_fence_is_container(fences[num_fences]));
}
EXPORT_SYMBOL(dma_fence_array_init);
/**
* dma_fence_array_create - Create a custom fence array
* @num_fences: [in] number of fences to add in the array
* @fences: [in] array containing the fences
* @context: [in] fence context to use
* @seqno: [in] sequence number to use
* @signal_on_any: [in] signal on any fence in the array
*
* Allocate a dma_fence_array object and initialize the base fence with
* dma_fence_init().
* In case of error it returns NULL.
*
* The caller should allocate the fences array with num_fences size
* and fill it with the fences it wants to add to the object. Ownership of this
* array is taken and dma_fence_put() is used on each fence on release.
*
* If @signal_on_any is true the fence array signals if any fence in the array
* signals, otherwise it signals when all fences in the array signal.
*/
struct dma_fence_array *dma_fence_array_create(int num_fences,
struct dma_fence **fences,
u64 context, unsigned seqno,
bool signal_on_any)
{
struct dma_fence_array *array;
array = dma_fence_array_alloc(num_fences);
if (!array)
return NULL;
dma_fence_array_init(array, num_fences, fences,
context, seqno, signal_on_any);
return array;
}
EXPORT_SYMBOL(dma_fence_array_create);
/**
* dma_fence_match_context - Check if all fences are from the given context
* @fence: [in] fence or fence array
* @context: [in] fence context to check all fences against
*
* Checks the provided fence or, for a fence array, all fences in the array
* against the given context. Returns false if any fence is from a different
* context.
*/
bool dma_fence_match_context(struct dma_fence *fence, u64 context)
{
struct dma_fence_array *array = to_dma_fence_array(fence);
unsigned i;
if (!dma_fence_is_array(fence))
return fence->context == context;
for (i = 0; i < array->num_fences; i++) {
if (array->fences[i]->context != context)
return false;
}
return true;
}
EXPORT_SYMBOL(dma_fence_match_context);
struct dma_fence *dma_fence_array_first(struct dma_fence *head)
{
struct dma_fence_array *array;
if (!head)
return NULL;
array = to_dma_fence_array(head);
if (!array)
return head;
if (!array->num_fences)
return NULL;
return array->fences[0];
}
EXPORT_SYMBOL(dma_fence_array_first);
struct dma_fence *dma_fence_array_next(struct dma_fence *head,
unsigned int index)
{
struct dma_fence_array *array = to_dma_fence_array(head);
if (!array || index >= array->num_fences)
return NULL;
return array->fences[index];
}
EXPORT_SYMBOL(dma_fence_array_next);