drivers/gpu/drm/xe/xe_sync.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2021 Intel Corporation
 */

#include "xe_sync.h"

#include <linux/dma-fence-array.h>
#include <linux/kthread.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>

#include <drm/drm_print.h>
#include <drm/drm_syncobj.h>
#include <uapi/drm/xe_drm.h>

#include "xe_device.h"
#include "xe_exec_queue.h"
#include "xe_macros.h"
#include "xe_sched_job_types.h"

struct xe_user_fence {
	struct xe_device *xe;
	struct kref refcount;
	struct dma_fence_cb cb;
	struct work_struct worker;
	struct mm_struct *mm;
	u64 __user *addr;
	u64 value;
	int signalled;
};

static void user_fence_destroy(struct kref *kref)
{
	struct xe_user_fence *ufence = container_of(kref, struct xe_user_fence,
						 refcount);

	mmdrop(ufence->mm);
	kfree(ufence);
}

static void user_fence_get(struct xe_user_fence *ufence)
{
	kref_get(&ufence->refcount);
}

static void user_fence_put(struct xe_user_fence *ufence)
{
	kref_put(&ufence->refcount, user_fence_destroy);
}

static struct xe_user_fence *user_fence_create(struct xe_device *xe, u64 addr,
					       u64 value)
{
	struct xe_user_fence *ufence;
	u64 __user *ptr = u64_to_user_ptr(addr);
	u64 __maybe_unused prefetch_val;

	if (get_user(prefetch_val, ptr))
		return ERR_PTR(-EFAULT);

	ufence = kzalloc_obj(*ufence);
	if (!ufence)
		return ERR_PTR(-ENOMEM);

	ufence->xe = xe;
	kref_init(&ufence->refcount);
	ufence->addr = ptr;
	ufence->value = value;
	ufence->mm = current->mm;
	mmgrab(ufence->mm);

	return ufence;
}

static void user_fence_worker(struct work_struct *w)
{
	struct xe_user_fence *ufence = container_of(w, struct xe_user_fence, worker);

	WRITE_ONCE(ufence->signalled, 1);
	if (mmget_not_zero(ufence->mm)) {
		kthread_use_mm(ufence->mm);
		if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value)))
			XE_WARN_ON("Copy to user failed");
		kthread_unuse_mm(ufence->mm);
		mmput(ufence->mm);
	} else {
		drm_dbg(&ufence->xe->drm, "mmget_not_zero() failed, ufence wasn't signaled\n");
	}

	/*
	 * Wake up waiters only after updating the ufence state, allowing the UMD
	 * to safely reuse the same ufence without encountering -EBUSY errors.
	 */
	wake_up_all(&ufence->xe->ufence_wq);
	user_fence_put(ufence);
}

static void kick_ufence(struct xe_user_fence *ufence, struct dma_fence *fence)
{
	INIT_WORK(&ufence->worker, user_fence_worker);
	queue_work(ufence->xe->ordered_wq, &ufence->worker);
	dma_fence_put(fence);
}

static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
{
	struct xe_user_fence *ufence = container_of(cb, struct xe_user_fence, cb);

	kick_ufence(ufence, fence);
}

int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef,
			struct xe_sync_entry *sync,
			struct drm_xe_sync __user *sync_user,
			struct drm_syncobj *ufence_syncobj,
			u64 ufence_timeline_value,
			unsigned int flags)
{
	struct drm_xe_sync sync_in;
	int err;
	bool exec = flags & SYNC_PARSE_FLAG_EXEC;
	bool in_lr_mode = flags & SYNC_PARSE_FLAG_LR_MODE;
	bool disallow_user_fence = flags & SYNC_PARSE_FLAG_DISALLOW_USER_FENCE;
	bool signal;

	if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user)))
		return -EFAULT;

	if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) ||
	    XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1]))
		return -EINVAL;

	signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL;
	switch (sync_in.type) {
	case DRM_XE_SYNC_TYPE_SYNCOBJ:
		if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
			return -EOPNOTSUPP;

		if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
			return -EINVAL;

		sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
		if (XE_IOCTL_DBG(xe, !sync->syncobj))
			return -ENOENT;

		if (!signal) {
			sync->fence = drm_syncobj_fence_get(sync->syncobj);
			if (XE_IOCTL_DBG(xe, !sync->fence)) {
				err = -EINVAL;
				goto free_sync;
			}
		}
		break;

	case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ:
		if (XE_IOCTL_DBG(xe, in_lr_mode && signal))
			return -EOPNOTSUPP;

		if (XE_IOCTL_DBG(xe, upper_32_bits(sync_in.addr)))
			return -EINVAL;

		if (XE_IOCTL_DBG(xe, sync_in.timeline_value == 0))
			return -EINVAL;

		sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle);
		if (XE_IOCTL_DBG(xe, !sync->syncobj))
			return -ENOENT;

		if (signal) {
			sync->chain_fence = dma_fence_chain_alloc();
			if (!sync->chain_fence) {
				err = -ENOMEM;
				goto free_sync;
			}
		} else {
			sync->fence = drm_syncobj_fence_get(sync->syncobj);
			if (XE_IOCTL_DBG(xe, !sync->fence)) {
				err = -EINVAL;
				goto free_sync;
			}

			err = dma_fence_chain_find_seqno(&sync->fence,
							 sync_in.timeline_value);
			if (err)
				goto free_sync;
		}
		break;

	case DRM_XE_SYNC_TYPE_USER_FENCE:
		if (XE_IOCTL_DBG(xe, disallow_user_fence))
			return -EOPNOTSUPP;

		if (XE_IOCTL_DBG(xe, !signal))
			return -EOPNOTSUPP;

		if (XE_IOCTL_DBG(xe, sync_in.addr & 0x7))
			return -EINVAL;

		if (exec) {
			sync->addr = sync_in.addr;
		} else {
			sync->ufence_timeline_value = ufence_timeline_value;
			sync->ufence = user_fence_create(xe, sync_in.addr,
							 sync_in.timeline_value);
			if (XE_IOCTL_DBG(xe, IS_ERR(sync->ufence)))
				return PTR_ERR(sync->ufence);
			sync->ufence_chain_fence = dma_fence_chain_alloc();
			if (!sync->ufence_chain_fence) {
				err = -ENOMEM;
				goto free_sync;
			}
			sync->ufence_syncobj = ufence_syncobj;
		}

		break;

	default:
		return -EINVAL;
	}

	sync->type = sync_in.type;
	sync->flags = sync_in.flags;
	sync->timeline_value = sync_in.timeline_value;

	return 0;

free_sync:
	xe_sync_entry_cleanup(sync);
	return err;
}
ALLOW_ERROR_INJECTION(xe_sync_entry_parse, ERRNO);

int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job)
{
	if (sync->fence)
		return  drm_sched_job_add_dependency(&job->drm,
						     dma_fence_get(sync->fence));

	return 0;
}

/**
 * xe_sync_entry_wait() - Wait on in-sync
 * @sync: Sync object
 *
 * If the sync is in an in-sync, wait on the sync to signal.
 *
 * Return: 0 on success, -ERESTARTSYS on failure (interruption)
 */
int xe_sync_entry_wait(struct xe_sync_entry *sync)
{
	return xe_sync_needs_wait(sync) ?
		dma_fence_wait(sync->fence, true) : 0;
}

/**
 * xe_sync_needs_wait() - Sync needs a wait (input dma-fence not signaled)
 * @sync: Sync object
 *
 * Return: True if sync needs a wait, False otherwise
 */
bool xe_sync_needs_wait(struct xe_sync_entry *sync)
{
	return sync->fence &&
	       !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &sync->fence->flags);
}

void xe_sync_entry_signal(struct xe_sync_entry *sync, struct dma_fence *fence)
{
	if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL))
		return;

	if (sync->chain_fence) {
		drm_syncobj_add_point(sync->syncobj, sync->chain_fence,
				      fence, sync->timeline_value);
		/*
		 * The chain's ownership is transferred to the
		 * timeline.
		 */
		sync->chain_fence = NULL;
	} else if (sync->syncobj) {
		drm_syncobj_replace_fence(sync->syncobj, fence);
	} else if (sync->ufence) {
		int err;

		drm_syncobj_add_point(sync->ufence_syncobj,
				      sync->ufence_chain_fence,
				      fence, sync->ufence_timeline_value);
		sync->ufence_chain_fence = NULL;

		fence = drm_syncobj_fence_get(sync->ufence_syncobj);
		user_fence_get(sync->ufence);
		err = dma_fence_add_callback(fence, &sync->ufence->cb,
					     user_fence_cb);
		if (err == -ENOENT) {
			kick_ufence(sync->ufence, fence);
		} else if (err) {
			XE_WARN_ON("failed to add user fence");
			user_fence_put(sync->ufence);
			dma_fence_put(fence);
		}
	}
}

void xe_sync_entry_cleanup(struct xe_sync_entry *sync)
{
	if (sync->syncobj)
		drm_syncobj_put(sync->syncobj);
	dma_fence_put(sync->fence);
	dma_fence_chain_free(sync->chain_fence);
	dma_fence_chain_free(sync->ufence_chain_fence);
	if (!IS_ERR_OR_NULL(sync->ufence))
		user_fence_put(sync->ufence);
}

/**
 * xe_sync_in_fence_get() - Get a fence from syncs, exec queue, and VM
 * @sync: input syncs
 * @num_sync: number of syncs
 * @q: exec queue
 * @vm: VM
 *
 * Get a fence from syncs, exec queue, and VM. If syncs contain in-fences create
 * and return a composite fence of all in-fences + last fence. If no in-fences
 * return last fence on  input exec queue. Caller must drop reference to
 * returned fence.
 *
 * Return: fence on success, ERR_PTR(-ENOMEM) on failure
 */
struct dma_fence *
xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
		     struct xe_exec_queue *q, struct xe_vm *vm)
{
	struct dma_fence **fences = NULL;
	struct dma_fence_array *cf = NULL;
	struct dma_fence *fence;
	int i, num_fence = 0, current_fence = 0;

	lockdep_assert_held(&vm->lock);

	/* Reject in fences */
	for (i = 0; i < num_sync; ++i)
		if (sync[i].fence)
			return ERR_PTR(-EOPNOTSUPP);

	if (q->flags & EXEC_QUEUE_FLAG_VM) {
		struct xe_exec_queue *__q;
		struct xe_tile *tile;
		u8 id;

		for_each_tile(tile, vm->xe, id) {
			num_fence++;
			for_each_tlb_inval(i)
				num_fence++;
		}

		fences = kmalloc_objs(*fences, num_fence);
		if (!fences)
			return ERR_PTR(-ENOMEM);

		fences[current_fence++] =
			xe_exec_queue_last_fence_get(q, vm);
		for_each_tlb_inval(i)
			fences[current_fence++] =
				xe_exec_queue_tlb_inval_last_fence_get(q, vm, i);
		list_for_each_entry(__q, &q->multi_gt_list,
				    multi_gt_link) {
			fences[current_fence++] =
				xe_exec_queue_last_fence_get(__q, vm);
			for_each_tlb_inval(i)
				fences[current_fence++] =
					xe_exec_queue_tlb_inval_last_fence_get(__q, vm, i);
		}

		xe_assert(vm->xe, current_fence == num_fence);
		cf = dma_fence_array_create(num_fence, fences,
					    dma_fence_context_alloc(1),
					    1);
		if (!cf)
			goto err_out;

		return &cf->base;
	}

	fence = xe_exec_queue_last_fence_get(q, vm);
	return fence;

err_out:
	while (current_fence)
		dma_fence_put(fences[--current_fence]);
	kfree(fences);

	return ERR_PTR(-ENOMEM);
}

/**
 * __xe_sync_ufence_get() - Get user fence from user fence
 * @ufence: input user fence
 *
 * Get a user fence reference from user fence
 *
 * Return: xe_user_fence pointer with reference
 */
struct xe_user_fence *__xe_sync_ufence_get(struct xe_user_fence *ufence)
{
	user_fence_get(ufence);

	return ufence;
}

/**
 * xe_sync_ufence_get() - Get user fence from sync
 * @sync: input sync
 *
 * Get a user fence reference from sync.
 *
 * Return: xe_user_fence pointer with reference
 */
struct xe_user_fence *xe_sync_ufence_get(struct xe_sync_entry *sync)
{
	user_fence_get(sync->ufence);

	return sync->ufence;
}

/**
 * xe_sync_ufence_put() - Put user fence reference
 * @ufence: user fence reference
 *
 */
void xe_sync_ufence_put(struct xe_user_fence *ufence)
{
	user_fence_put(ufence);
}

/**
 * xe_sync_ufence_get_status() - Get user fence status
 * @ufence: user fence
 *
 * Return: 1 if signalled, 0 not signalled, <0 on error
 */
int xe_sync_ufence_get_status(struct xe_user_fence *ufence)
{
	return READ_ONCE(ufence->signalled);
}