drivers/gpu/drm/xe/xe_hw_engine_group.c - linux - Git at Google

 // SPDX-License-Identifier: MIT
 /*
  * Copyright © 2024 Intel Corporation
  */

 #include <drm/drm_managed.h>

 #include "xe_assert.h"
 #include "xe_device.h"
 #include "xe_exec_queue.h"
 #include "xe_gt.h"
 #include "xe_hw_engine_group.h"
 #include "xe_vm.h"

 static void
 hw_engine_group_free(struct drm_device *drm, void *arg)
 {
 	struct xe_hw_engine_group *group = arg;

 	destroy_workqueue(group->resume_wq);
 	kfree(group);
 }

 static void
 hw_engine_group_resume_lr_jobs_func(struct work_struct *w)
 {
 	struct xe_exec_queue *q;
 	struct xe_hw_engine_group *group = container_of(w, struct xe_hw_engine_group, resume_work);
 	int err;
 	enum xe_hw_engine_group_execution_mode previous_mode;

 	err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode);
 	if (err)
 		return;

 	if (previous_mode == EXEC_MODE_LR)
 		goto put;

 	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
 		if (!xe_vm_in_fault_mode(q->vm))
 			continue;

 		q->ops->resume(q);
 	}

 put:
 	xe_hw_engine_group_put(group);
 }

 static struct xe_hw_engine_group *
 hw_engine_group_alloc(struct xe_device *xe)
 {
 	struct xe_hw_engine_group *group;
 	int err;

 	group = kzalloc(sizeof(*group), GFP_KERNEL);
 	if (!group)
 		return ERR_PTR(-ENOMEM);

 	group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", 0, 0);
 	if (!group->resume_wq)
 		return ERR_PTR(-ENOMEM);

 	init_rwsem(&group->mode_sem);
 	INIT_WORK(&group->resume_work, hw_engine_group_resume_lr_jobs_func);
 	INIT_LIST_HEAD(&group->exec_queue_list);

 	err = drmm_add_action_or_reset(&xe->drm, hw_engine_group_free, group);
 	if (err)
 		return ERR_PTR(err);

 	return group;
 }

 /**
  * xe_hw_engine_setup_groups() - Setup the hw engine groups for the gt
  * @gt: The gt for which groups are setup
  *
  * Return: 0 on success, negative error code on error.
  */
 int xe_hw_engine_setup_groups(struct xe_gt *gt)
 {
 	struct xe_hw_engine *hwe;
 	enum xe_hw_engine_id id;
 	struct xe_hw_engine_group *group_rcs_ccs, *group_bcs, *group_vcs_vecs;
 	struct xe_device *xe = gt_to_xe(gt);
 	int err;

 	group_rcs_ccs = hw_engine_group_alloc(xe);
 	if (IS_ERR(group_rcs_ccs)) {
 		err = PTR_ERR(group_rcs_ccs);
 		goto err_group_rcs_ccs;
 	}

 	group_bcs = hw_engine_group_alloc(xe);
 	if (IS_ERR(group_bcs)) {
 		err = PTR_ERR(group_bcs);
 		goto err_group_bcs;
 	}

 	group_vcs_vecs = hw_engine_group_alloc(xe);
 	if (IS_ERR(group_vcs_vecs)) {
 		err = PTR_ERR(group_vcs_vecs);
 		goto err_group_vcs_vecs;
 	}

 	for_each_hw_engine(hwe, gt, id) {
 		switch (hwe->class) {
 		case XE_ENGINE_CLASS_COPY:
 			hwe->hw_engine_group = group_bcs;
 			break;
 		case XE_ENGINE_CLASS_RENDER:
 		case XE_ENGINE_CLASS_COMPUTE:
 			hwe->hw_engine_group = group_rcs_ccs;
 			break;
 		case XE_ENGINE_CLASS_VIDEO_DECODE:
 		case XE_ENGINE_CLASS_VIDEO_ENHANCE:
 			hwe->hw_engine_group = group_vcs_vecs;
 			break;
 		case XE_ENGINE_CLASS_OTHER:
 			break;
 		default:
 			drm_warn(&xe->drm, "NOT POSSIBLE");
 		}
 	}

 	return 0;

 err_group_vcs_vecs:
 	kfree(group_vcs_vecs);
 err_group_bcs:
 	kfree(group_bcs);
 err_group_rcs_ccs:
 	kfree(group_rcs_ccs);

 	return err;
 }

 /**
  * xe_hw_engine_group_add_exec_queue() - Add an exec queue to a hw engine group
  * @group: The hw engine group
  * @q: The exec_queue
  *
  * Return: 0 on success,
  *	    -EINTR if the lock could not be acquired
  */
 int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q)
 {
 	int err;
 	struct xe_device *xe = gt_to_xe(q->gt);

 	xe_assert(xe, group);
 	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM));
 	xe_assert(xe, q->vm);

 	if (xe_vm_in_preempt_fence_mode(q->vm))
 		return 0;

 	err = down_write_killable(&group->mode_sem);
 	if (err)
 		return err;

 	if (xe_vm_in_fault_mode(q->vm) && group->cur_mode == EXEC_MODE_DMA_FENCE) {
 		q->ops->suspend(q);
 		err = q->ops->suspend_wait(q);
 		if (err)
 			goto err_suspend;

 		xe_hw_engine_group_resume_faulting_lr_jobs(group);
 	}

 	list_add(&q->hw_engine_group_link, &group->exec_queue_list);
 	up_write(&group->mode_sem);

 	return 0;

 err_suspend:
 	up_write(&group->mode_sem);
 	return err;
 }

 /**
  * xe_hw_engine_group_del_exec_queue() - Delete an exec queue from a hw engine group
  * @group: The hw engine group
  * @q: The exec_queue
  */
 void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q)
 {
 	struct xe_device *xe = gt_to_xe(q->gt);

 	xe_assert(xe, group);
 	xe_assert(xe, q->vm);

 	down_write(&group->mode_sem);

 	if (!list_empty(&q->hw_engine_group_link))
 		list_del(&q->hw_engine_group_link);

 	up_write(&group->mode_sem);
 }

 /**
  * xe_hw_engine_group_resume_faulting_lr_jobs() - Asynchronously resume the hw engine group's
  * faulting LR jobs
  * @group: The hw engine group
  */
 void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group)
 {
 	queue_work(group->resume_wq, &group->resume_work);
 }

 /**
  * xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group
  * @group: The hw engine group
  *
  * Return: 0 on success, negative error code on error.
  */
 static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group)
 {
 	int err;
 	struct xe_exec_queue *q;
 	bool need_resume = false;

 	lockdep_assert_held_write(&group->mode_sem);

 	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
 		if (!xe_vm_in_fault_mode(q->vm))
 			continue;

 		need_resume = true;
 		q->ops->suspend(q);
 	}

 	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
 		if (!xe_vm_in_fault_mode(q->vm))
 			continue;

 		err = q->ops->suspend_wait(q);
 		if (err)
 			goto err_suspend;
 	}

 	if (need_resume)
 		xe_hw_engine_group_resume_faulting_lr_jobs(group);

 	return 0;

 err_suspend:
 	up_write(&group->mode_sem);
 	return err;
 }

 /**
  * xe_hw_engine_group_wait_for_dma_fence_jobs() - Wait for dma fence jobs to complete
  * @group: The hw engine group
  *
  * This function is not meant to be called directly from a user IOCTL as dma_fence_wait()
  * is not interruptible.
  *
  * Return: 0 on success,
  *	   -ETIME if waiting for one job failed
  */
 static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group *group)
 {
 	long timeout;
 	struct xe_exec_queue *q;
 	struct dma_fence *fence;

 	lockdep_assert_held_write(&group->mode_sem);

 	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
 		if (xe_vm_in_lr_mode(q->vm))
 			continue;

 		fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm);
 		timeout = dma_fence_wait(fence, false);
 		dma_fence_put(fence);

 		if (timeout < 0)
 			return -ETIME;
 	}

 	return 0;
 }

 static int switch_mode(struct xe_hw_engine_group *group)
 {
 	int err = 0;
 	enum xe_hw_engine_group_execution_mode new_mode;

 	lockdep_assert_held_write(&group->mode_sem);

 	switch (group->cur_mode) {
 	case EXEC_MODE_LR:
 		new_mode = EXEC_MODE_DMA_FENCE;
 		err = xe_hw_engine_group_suspend_faulting_lr_jobs(group);
 		break;
 	case EXEC_MODE_DMA_FENCE:
 		new_mode = EXEC_MODE_LR;
 		err = xe_hw_engine_group_wait_for_dma_fence_jobs(group);
 		break;
 	}

 	if (err)
 		return err;

 	group->cur_mode = new_mode;

 	return 0;
 }

 /**
  * xe_hw_engine_group_get_mode() - Get the group to execute in the new mode
  * @group: The hw engine group
  * @new_mode: The new execution mode
  * @previous_mode: Pointer to the previous mode provided for use by caller
  *
  * Return: 0 if successful, -EINTR if locking failed.
  */
 int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,
 				enum xe_hw_engine_group_execution_mode new_mode,
 				enum xe_hw_engine_group_execution_mode *previous_mode)
 __acquires(&group->mode_sem)
 {
 	int err = down_read_interruptible(&group->mode_sem);

 	if (err)
 		return err;

 	*previous_mode = group->cur_mode;

 	if (new_mode != group->cur_mode) {
 		up_read(&group->mode_sem);
 		err = down_write_killable(&group->mode_sem);
 		if (err)
 			return err;

 		if (new_mode != group->cur_mode) {
 			err = switch_mode(group);
 			if (err) {
 				up_write(&group->mode_sem);
 				return err;
 			}
 		}
 		downgrade_write(&group->mode_sem);
 	}

 	return err;
 }

 /**
  * xe_hw_engine_group_put() - Put the group
  * @group: The hw engine group
  */
 void xe_hw_engine_group_put(struct xe_hw_engine_group *group)
 __releases(&group->mode_sem)
 {
 	up_read(&group->mode_sem);
 }

 /**
  * xe_hw_engine_group_find_exec_mode() - Find the execution mode for this exec queue
  * @q: The exec_queue
  */
 enum xe_hw_engine_group_execution_mode
 xe_hw_engine_group_find_exec_mode(struct xe_exec_queue *q)
 {
 	if (xe_vm_in_fault_mode(q->vm))
 		return EXEC_MODE_LR;
 	else
 		return EXEC_MODE_DMA_FENCE;
 }
	// SPDX-License-Identifier: MIT
	/*
	* Copyright © 2024 Intel Corporation
	*/

	#include <drm/drm_managed.h>

	#include "xe_assert.h"
	#include "xe_device.h"
	#include "xe_exec_queue.h"
	#include "xe_gt.h"
	#include "xe_hw_engine_group.h"
	#include "xe_vm.h"

	static void
	hw_engine_group_free(struct drm_device drm, void arg)
	{
	struct xe_hw_engine_group *group = arg;

	destroy_workqueue(group->resume_wq);
	kfree(group);
	}

	static void
	hw_engine_group_resume_lr_jobs_func(struct work_struct *w)
	{
	struct xe_exec_queue *q;
	struct xe_hw_engine_group *group = container_of(w, struct xe_hw_engine_group, resume_work);
	int err;
	enum xe_hw_engine_group_execution_mode previous_mode;

	err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode);
	if (err)
	return;

	if (previous_mode == EXEC_MODE_LR)
	goto put;

	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
	if (!xe_vm_in_fault_mode(q->vm))
	continue;

	q->ops->resume(q);
	}

	put:
	xe_hw_engine_group_put(group);
	}

	static struct xe_hw_engine_group *
	hw_engine_group_alloc(struct xe_device *xe)
	{
	struct xe_hw_engine_group *group;
	int err;

	group = kzalloc(sizeof(*group), GFP_KERNEL);
	if (!group)
	return ERR_PTR(-ENOMEM);

	group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", 0, 0);
	if (!group->resume_wq)
	return ERR_PTR(-ENOMEM);

	init_rwsem(&group->mode_sem);
	INIT_WORK(&group->resume_work, hw_engine_group_resume_lr_jobs_func);
	INIT_LIST_HEAD(&group->exec_queue_list);

	err = drmm_add_action_or_reset(&xe->drm, hw_engine_group_free, group);
	if (err)
	return ERR_PTR(err);

	return group;
	}

	/**
	* xe_hw_engine_setup_groups() - Setup the hw engine groups for the gt
	* @gt: The gt for which groups are setup
	*
	* Return: 0 on success, negative error code on error.
	*/
	int xe_hw_engine_setup_groups(struct xe_gt *gt)
	{
	struct xe_hw_engine *hwe;
	enum xe_hw_engine_id id;
	struct xe_hw_engine_group group_rcs_ccs, group_bcs, *group_vcs_vecs;
	struct xe_device *xe = gt_to_xe(gt);
	int err;

	group_rcs_ccs = hw_engine_group_alloc(xe);
	if (IS_ERR(group_rcs_ccs)) {
	err = PTR_ERR(group_rcs_ccs);
	goto err_group_rcs_ccs;
	}

	group_bcs = hw_engine_group_alloc(xe);
	if (IS_ERR(group_bcs)) {
	err = PTR_ERR(group_bcs);
	goto err_group_bcs;
	}

	group_vcs_vecs = hw_engine_group_alloc(xe);
	if (IS_ERR(group_vcs_vecs)) {
	err = PTR_ERR(group_vcs_vecs);
	goto err_group_vcs_vecs;
	}

	for_each_hw_engine(hwe, gt, id) {
	switch (hwe->class) {
	case XE_ENGINE_CLASS_COPY:
	hwe->hw_engine_group = group_bcs;
	break;
	case XE_ENGINE_CLASS_RENDER:
	case XE_ENGINE_CLASS_COMPUTE:
	hwe->hw_engine_group = group_rcs_ccs;
	break;
	case XE_ENGINE_CLASS_VIDEO_DECODE:
	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
	hwe->hw_engine_group = group_vcs_vecs;
	break;
	case XE_ENGINE_CLASS_OTHER:
	break;
	default:
	drm_warn(&xe->drm, "NOT POSSIBLE");
	}
	}

	return 0;

	err_group_vcs_vecs:
	kfree(group_vcs_vecs);
	err_group_bcs:
	kfree(group_bcs);
	err_group_rcs_ccs:
	kfree(group_rcs_ccs);

	return err;
	}

	/**
	* xe_hw_engine_group_add_exec_queue() - Add an exec queue to a hw engine group
	* @group: The hw engine group
	* @q: The exec_queue
	*
	* Return: 0 on success,
	* -EINTR if the lock could not be acquired
	*/
	int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group group, struct xe_exec_queue q)
	{
	int err;
	struct xe_device *xe = gt_to_xe(q->gt);

	xe_assert(xe, group);
	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM));
	xe_assert(xe, q->vm);

	if (xe_vm_in_preempt_fence_mode(q->vm))
	return 0;

	err = down_write_killable(&group->mode_sem);
	if (err)
	return err;

	if (xe_vm_in_fault_mode(q->vm) && group->cur_mode == EXEC_MODE_DMA_FENCE) {
	q->ops->suspend(q);
	err = q->ops->suspend_wait(q);
	if (err)
	goto err_suspend;

	xe_hw_engine_group_resume_faulting_lr_jobs(group);
	}

	list_add(&q->hw_engine_group_link, &group->exec_queue_list);
	up_write(&group->mode_sem);

	return 0;

	err_suspend:
	up_write(&group->mode_sem);
	return err;
	}

	/**
	* xe_hw_engine_group_del_exec_queue() - Delete an exec queue from a hw engine group
	* @group: The hw engine group
	* @q: The exec_queue
	*/
	void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group group, struct xe_exec_queue q)
	{
	struct xe_device *xe = gt_to_xe(q->gt);

	xe_assert(xe, group);
	xe_assert(xe, q->vm);

	down_write(&group->mode_sem);

	if (!list_empty(&q->hw_engine_group_link))
	list_del(&q->hw_engine_group_link);

	up_write(&group->mode_sem);
	}

	/**
	* xe_hw_engine_group_resume_faulting_lr_jobs() - Asynchronously resume the hw engine group's
	* faulting LR jobs
	* @group: The hw engine group
	*/
	void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group)
	{
	queue_work(group->resume_wq, &group->resume_work);
	}

	/**
	* xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group
	* @group: The hw engine group
	*
	* Return: 0 on success, negative error code on error.
	*/
	static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group)
	{
	int err;
	struct xe_exec_queue *q;
	bool need_resume = false;

	lockdep_assert_held_write(&group->mode_sem);

	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
	if (!xe_vm_in_fault_mode(q->vm))
	continue;

	need_resume = true;
	q->ops->suspend(q);
	}

	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
	if (!xe_vm_in_fault_mode(q->vm))
	continue;

	err = q->ops->suspend_wait(q);
	if (err)
	goto err_suspend;
	}

	if (need_resume)
	xe_hw_engine_group_resume_faulting_lr_jobs(group);

	return 0;

	err_suspend:
	up_write(&group->mode_sem);
	return err;
	}

	/**
	* xe_hw_engine_group_wait_for_dma_fence_jobs() - Wait for dma fence jobs to complete
	* @group: The hw engine group
	*
	* This function is not meant to be called directly from a user IOCTL as dma_fence_wait()
	* is not interruptible.
	*
	* Return: 0 on success,
	* -ETIME if waiting for one job failed
	*/
	static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group *group)
	{
	long timeout;
	struct xe_exec_queue *q;
	struct dma_fence *fence;

	lockdep_assert_held_write(&group->mode_sem);

	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
	if (xe_vm_in_lr_mode(q->vm))
	continue;

	fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm);
	timeout = dma_fence_wait(fence, false);
	dma_fence_put(fence);

	if (timeout < 0)
	return -ETIME;
	}

	return 0;
	}

	static int switch_mode(struct xe_hw_engine_group *group)
	{
	int err = 0;
	enum xe_hw_engine_group_execution_mode new_mode;

	lockdep_assert_held_write(&group->mode_sem);

	switch (group->cur_mode) {
	case EXEC_MODE_LR:
	new_mode = EXEC_MODE_DMA_FENCE;
	err = xe_hw_engine_group_suspend_faulting_lr_jobs(group);
	break;
	case EXEC_MODE_DMA_FENCE:
	new_mode = EXEC_MODE_LR;
	err = xe_hw_engine_group_wait_for_dma_fence_jobs(group);
	break;
	}

	if (err)
	return err;

	group->cur_mode = new_mode;

	return 0;
	}

	/**
	* xe_hw_engine_group_get_mode() - Get the group to execute in the new mode
	* @group: The hw engine group
	* @new_mode: The new execution mode
	* @previous_mode: Pointer to the previous mode provided for use by caller
	*
	* Return: 0 if successful, -EINTR if locking failed.
	*/
	int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,
	enum xe_hw_engine_group_execution_mode new_mode,
	enum xe_hw_engine_group_execution_mode *previous_mode)
	__acquires(&group->mode_sem)
	{
	int err = down_read_interruptible(&group->mode_sem);

	if (err)
	return err;

	*previous_mode = group->cur_mode;

	if (new_mode != group->cur_mode) {
	up_read(&group->mode_sem);
	err = down_write_killable(&group->mode_sem);
	if (err)
	return err;

	if (new_mode != group->cur_mode) {
	err = switch_mode(group);
	if (err) {
	up_write(&group->mode_sem);
	return err;
	}
	}
	downgrade_write(&group->mode_sem);
	}

	return err;
	}

	/**
	* xe_hw_engine_group_put() - Put the group
	* @group: The hw engine group
	*/
	void xe_hw_engine_group_put(struct xe_hw_engine_group *group)
	__releases(&group->mode_sem)
	{
	up_read(&group->mode_sem);
	}

	/**
	* xe_hw_engine_group_find_exec_mode() - Find the execution mode for this exec queue
	* @q: The exec_queue
	*/
	enum xe_hw_engine_group_execution_mode
	xe_hw_engine_group_find_exec_mode(struct xe_exec_queue *q)
	{
	if (xe_vm_in_fault_mode(q->vm))
	return EXEC_MODE_LR;
	else
	return EXEC_MODE_DMA_FENCE;
	}