| // SPDX-License-Identifier: MIT | |
| /* | |
| * Copyright © 2024 Intel Corporation | |
| */ | |
| #include <drm/drm_managed.h> | |
| #include "xe_assert.h" | |
| #include "xe_device.h" | |
| #include "xe_exec_queue.h" | |
| #include "xe_gt.h" | |
| #include "xe_hw_engine_group.h" | |
| #include "xe_vm.h" | |
| static void | |
| hw_engine_group_free(struct drm_device *drm, void *arg) | |
| { | |
| struct xe_hw_engine_group *group = arg; | |
| destroy_workqueue(group->resume_wq); | |
| kfree(group); | |
| } | |
| static void | |
| hw_engine_group_resume_lr_jobs_func(struct work_struct *w) | |
| { | |
| struct xe_exec_queue *q; | |
| struct xe_hw_engine_group *group = container_of(w, struct xe_hw_engine_group, resume_work); | |
| int err; | |
| enum xe_hw_engine_group_execution_mode previous_mode; | |
| err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode); | |
| if (err) | |
| return; | |
| if (previous_mode == EXEC_MODE_LR) | |
| goto put; | |
| list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { | |
| if (!xe_vm_in_fault_mode(q->vm)) | |
| continue; | |
| q->ops->resume(q); | |
| } | |
| put: | |
| xe_hw_engine_group_put(group); | |
| } | |
| static struct xe_hw_engine_group * | |
| hw_engine_group_alloc(struct xe_device *xe) | |
| { | |
| struct xe_hw_engine_group *group; | |
| int err; | |
| group = kzalloc(sizeof(*group), GFP_KERNEL); | |
| if (!group) | |
| return ERR_PTR(-ENOMEM); | |
| group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", 0, 0); | |
| if (!group->resume_wq) | |
| return ERR_PTR(-ENOMEM); | |
| init_rwsem(&group->mode_sem); | |
| INIT_WORK(&group->resume_work, hw_engine_group_resume_lr_jobs_func); | |
| INIT_LIST_HEAD(&group->exec_queue_list); | |
| err = drmm_add_action_or_reset(&xe->drm, hw_engine_group_free, group); | |
| if (err) | |
| return ERR_PTR(err); | |
| return group; | |
| } | |
| /** | |
| * xe_hw_engine_setup_groups() - Setup the hw engine groups for the gt | |
| * @gt: The gt for which groups are setup | |
| * | |
| * Return: 0 on success, negative error code on error. | |
| */ | |
| int xe_hw_engine_setup_groups(struct xe_gt *gt) | |
| { | |
| struct xe_hw_engine *hwe; | |
| enum xe_hw_engine_id id; | |
| struct xe_hw_engine_group *group_rcs_ccs, *group_bcs, *group_vcs_vecs; | |
| struct xe_device *xe = gt_to_xe(gt); | |
| int err; | |
| group_rcs_ccs = hw_engine_group_alloc(xe); | |
| if (IS_ERR(group_rcs_ccs)) { | |
| err = PTR_ERR(group_rcs_ccs); | |
| goto err_group_rcs_ccs; | |
| } | |
| group_bcs = hw_engine_group_alloc(xe); | |
| if (IS_ERR(group_bcs)) { | |
| err = PTR_ERR(group_bcs); | |
| goto err_group_bcs; | |
| } | |
| group_vcs_vecs = hw_engine_group_alloc(xe); | |
| if (IS_ERR(group_vcs_vecs)) { | |
| err = PTR_ERR(group_vcs_vecs); | |
| goto err_group_vcs_vecs; | |
| } | |
| for_each_hw_engine(hwe, gt, id) { | |
| switch (hwe->class) { | |
| case XE_ENGINE_CLASS_COPY: | |
| hwe->hw_engine_group = group_bcs; | |
| break; | |
| case XE_ENGINE_CLASS_RENDER: | |
| case XE_ENGINE_CLASS_COMPUTE: | |
| hwe->hw_engine_group = group_rcs_ccs; | |
| break; | |
| case XE_ENGINE_CLASS_VIDEO_DECODE: | |
| case XE_ENGINE_CLASS_VIDEO_ENHANCE: | |
| hwe->hw_engine_group = group_vcs_vecs; | |
| break; | |
| case XE_ENGINE_CLASS_OTHER: | |
| break; | |
| default: | |
| drm_warn(&xe->drm, "NOT POSSIBLE"); | |
| } | |
| } | |
| return 0; | |
| err_group_vcs_vecs: | |
| kfree(group_vcs_vecs); | |
| err_group_bcs: | |
| kfree(group_bcs); | |
| err_group_rcs_ccs: | |
| kfree(group_rcs_ccs); | |
| return err; | |
| } | |
| /** | |
| * xe_hw_engine_group_add_exec_queue() - Add an exec queue to a hw engine group | |
| * @group: The hw engine group | |
| * @q: The exec_queue | |
| * | |
| * Return: 0 on success, | |
| * -EINTR if the lock could not be acquired | |
| */ | |
| int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q) | |
| { | |
| int err; | |
| struct xe_device *xe = gt_to_xe(q->gt); | |
| xe_assert(xe, group); | |
| xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM)); | |
| xe_assert(xe, q->vm); | |
| if (xe_vm_in_preempt_fence_mode(q->vm)) | |
| return 0; | |
| err = down_write_killable(&group->mode_sem); | |
| if (err) | |
| return err; | |
| if (xe_vm_in_fault_mode(q->vm) && group->cur_mode == EXEC_MODE_DMA_FENCE) { | |
| q->ops->suspend(q); | |
| err = q->ops->suspend_wait(q); | |
| if (err) | |
| goto err_suspend; | |
| xe_hw_engine_group_resume_faulting_lr_jobs(group); | |
| } | |
| list_add(&q->hw_engine_group_link, &group->exec_queue_list); | |
| up_write(&group->mode_sem); | |
| return 0; | |
| err_suspend: | |
| up_write(&group->mode_sem); | |
| return err; | |
| } | |
| /** | |
| * xe_hw_engine_group_del_exec_queue() - Delete an exec queue from a hw engine group | |
| * @group: The hw engine group | |
| * @q: The exec_queue | |
| */ | |
| void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q) | |
| { | |
| struct xe_device *xe = gt_to_xe(q->gt); | |
| xe_assert(xe, group); | |
| xe_assert(xe, q->vm); | |
| down_write(&group->mode_sem); | |
| if (!list_empty(&q->hw_engine_group_link)) | |
| list_del(&q->hw_engine_group_link); | |
| up_write(&group->mode_sem); | |
| } | |
| /** | |
| * xe_hw_engine_group_resume_faulting_lr_jobs() - Asynchronously resume the hw engine group's | |
| * faulting LR jobs | |
| * @group: The hw engine group | |
| */ | |
| void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group) | |
| { | |
| queue_work(group->resume_wq, &group->resume_work); | |
| } | |
| /** | |
| * xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group | |
| * @group: The hw engine group | |
| * | |
| * Return: 0 on success, negative error code on error. | |
| */ | |
| static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group) | |
| { | |
| int err; | |
| struct xe_exec_queue *q; | |
| bool need_resume = false; | |
| lockdep_assert_held_write(&group->mode_sem); | |
| list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { | |
| if (!xe_vm_in_fault_mode(q->vm)) | |
| continue; | |
| need_resume = true; | |
| q->ops->suspend(q); | |
| } | |
| list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { | |
| if (!xe_vm_in_fault_mode(q->vm)) | |
| continue; | |
| err = q->ops->suspend_wait(q); | |
| if (err) | |
| goto err_suspend; | |
| } | |
| if (need_resume) | |
| xe_hw_engine_group_resume_faulting_lr_jobs(group); | |
| return 0; | |
| err_suspend: | |
| up_write(&group->mode_sem); | |
| return err; | |
| } | |
| /** | |
| * xe_hw_engine_group_wait_for_dma_fence_jobs() - Wait for dma fence jobs to complete | |
| * @group: The hw engine group | |
| * | |
| * This function is not meant to be called directly from a user IOCTL as dma_fence_wait() | |
| * is not interruptible. | |
| * | |
| * Return: 0 on success, | |
| * -ETIME if waiting for one job failed | |
| */ | |
| static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group *group) | |
| { | |
| long timeout; | |
| struct xe_exec_queue *q; | |
| struct dma_fence *fence; | |
| lockdep_assert_held_write(&group->mode_sem); | |
| list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { | |
| if (xe_vm_in_lr_mode(q->vm)) | |
| continue; | |
| fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm); | |
| timeout = dma_fence_wait(fence, false); | |
| dma_fence_put(fence); | |
| if (timeout < 0) | |
| return -ETIME; | |
| } | |
| return 0; | |
| } | |
| static int switch_mode(struct xe_hw_engine_group *group) | |
| { | |
| int err = 0; | |
| enum xe_hw_engine_group_execution_mode new_mode; | |
| lockdep_assert_held_write(&group->mode_sem); | |
| switch (group->cur_mode) { | |
| case EXEC_MODE_LR: | |
| new_mode = EXEC_MODE_DMA_FENCE; | |
| err = xe_hw_engine_group_suspend_faulting_lr_jobs(group); | |
| break; | |
| case EXEC_MODE_DMA_FENCE: | |
| new_mode = EXEC_MODE_LR; | |
| err = xe_hw_engine_group_wait_for_dma_fence_jobs(group); | |
| break; | |
| } | |
| if (err) | |
| return err; | |
| group->cur_mode = new_mode; | |
| return 0; | |
| } | |
| /** | |
| * xe_hw_engine_group_get_mode() - Get the group to execute in the new mode | |
| * @group: The hw engine group | |
| * @new_mode: The new execution mode | |
| * @previous_mode: Pointer to the previous mode provided for use by caller | |
| * | |
| * Return: 0 if successful, -EINTR if locking failed. | |
| */ | |
| int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group, | |
| enum xe_hw_engine_group_execution_mode new_mode, | |
| enum xe_hw_engine_group_execution_mode *previous_mode) | |
| __acquires(&group->mode_sem) | |
| { | |
| int err = down_read_interruptible(&group->mode_sem); | |
| if (err) | |
| return err; | |
| *previous_mode = group->cur_mode; | |
| if (new_mode != group->cur_mode) { | |
| up_read(&group->mode_sem); | |
| err = down_write_killable(&group->mode_sem); | |
| if (err) | |
| return err; | |
| if (new_mode != group->cur_mode) { | |
| err = switch_mode(group); | |
| if (err) { | |
| up_write(&group->mode_sem); | |
| return err; | |
| } | |
| } | |
| downgrade_write(&group->mode_sem); | |
| } | |
| return err; | |
| } | |
| /** | |
| * xe_hw_engine_group_put() - Put the group | |
| * @group: The hw engine group | |
| */ | |
| void xe_hw_engine_group_put(struct xe_hw_engine_group *group) | |
| __releases(&group->mode_sem) | |
| { | |
| up_read(&group->mode_sem); | |
| } | |
| /** | |
| * xe_hw_engine_group_find_exec_mode() - Find the execution mode for this exec queue | |
| * @q: The exec_queue | |
| */ | |
| enum xe_hw_engine_group_execution_mode | |
| xe_hw_engine_group_find_exec_mode(struct xe_exec_queue *q) | |
| { | |
| if (xe_vm_in_fault_mode(q->vm)) | |
| return EXEC_MODE_LR; | |
| else | |
| return EXEC_MODE_DMA_FENCE; | |
| } |