drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c - linux - Git at Google

 // SPDX-License-Identifier: MIT
 /*
  * Copyright © 2023 Intel Corporation
  */

 #include "xe_gt_tlb_invalidation.h"

 #include "abi/guc_actions_abi.h"
 #include "xe_device.h"
 #include "xe_gt.h"
 #include "xe_gt_printk.h"
 #include "xe_guc.h"
 #include "xe_guc_ct.h"
 #include "xe_mmio.h"
 #include "xe_trace.h"
 #include "regs/xe_guc_regs.h"

 #define TLB_TIMEOUT	(HZ / 4)

 static void xe_gt_tlb_fence_timeout(struct work_struct *work)
 {
 	struct xe_gt *gt = container_of(work, struct xe_gt,
 					tlb_invalidation.fence_tdr.work);
 	struct xe_gt_tlb_invalidation_fence *fence, *next;

 	spin_lock_irq(&gt->tlb_invalidation.pending_lock);
 	list_for_each_entry_safe(fence, next,
 				 &gt->tlb_invalidation.pending_fences, link) {
 		s64 since_inval_ms = ktime_ms_delta(ktime_get(),
 						    fence->invalidation_time);

 		if (msecs_to_jiffies(since_inval_ms) < TLB_TIMEOUT)
 			break;

 		trace_xe_gt_tlb_invalidation_fence_timeout(fence);
 		xe_gt_err(gt, "TLB invalidation fence timeout, seqno=%d recv=%d",
 			  fence->seqno, gt->tlb_invalidation.seqno_recv);

 		list_del(&fence->link);
 		fence->base.error = -ETIME;
 		dma_fence_signal(&fence->base);
 		dma_fence_put(&fence->base);
 	}
 	if (!list_empty(&gt->tlb_invalidation.pending_fences))
 		queue_delayed_work(system_wq,
 				   &gt->tlb_invalidation.fence_tdr,
 				   TLB_TIMEOUT);
 	spin_unlock_irq(&gt->tlb_invalidation.pending_lock);
 }

 /**
  * xe_gt_tlb_invalidation_init - Initialize GT TLB invalidation state
  * @gt: graphics tile
  *
  * Initialize GT TLB invalidation state, purely software initialization, should
  * be called once during driver load.
  *
  * Return: 0 on success, negative error code on error.
  */
 int xe_gt_tlb_invalidation_init(struct xe_gt *gt)
 {
 	gt->tlb_invalidation.seqno = 1;
 	INIT_LIST_HEAD(&gt->tlb_invalidation.pending_fences);
 	spin_lock_init(&gt->tlb_invalidation.pending_lock);
 	spin_lock_init(&gt->tlb_invalidation.lock);
 	INIT_DELAYED_WORK(&gt->tlb_invalidation.fence_tdr,
 			  xe_gt_tlb_fence_timeout);

 	return 0;
 }

 static void
 __invalidation_fence_signal(struct xe_gt_tlb_invalidation_fence *fence)
 {
 	trace_xe_gt_tlb_invalidation_fence_signal(fence);
 	dma_fence_signal(&fence->base);
 	dma_fence_put(&fence->base);
 }

 static void
 invalidation_fence_signal(struct xe_gt_tlb_invalidation_fence *fence)
 {
 	list_del(&fence->link);
 	__invalidation_fence_signal(fence);
 }

 /**
  * xe_gt_tlb_invalidation_reset - Initialize GT TLB invalidation reset
  * @gt: graphics tile
  *
  * Signal any pending invalidation fences, should be called during a GT reset
  */
 void xe_gt_tlb_invalidation_reset(struct xe_gt *gt)
 {
 	struct xe_gt_tlb_invalidation_fence *fence, *next;
 	struct xe_guc *guc = &gt->uc.guc;
 	int pending_seqno;

 	/*
 	 * CT channel is already disabled at this point. No new TLB requests can
 	 * appear.
 	 */

 	mutex_lock(&gt->uc.guc.ct.lock);
 	spin_lock_irq(&gt->tlb_invalidation.pending_lock);
 	cancel_delayed_work(&gt->tlb_invalidation.fence_tdr);
 	/*
 	 * We might have various kworkers waiting for TLB flushes to complete
 	 * which are not tracked with an explicit TLB fence, however at this
 	 * stage that will never happen since the CT is already disabled, so
 	 * make sure we signal them here under the assumption that we have
 	 * completed a full GT reset.
 	 */
 	if (gt->tlb_invalidation.seqno == 1)
 		pending_seqno = TLB_INVALIDATION_SEQNO_MAX - 1;
 	else
 		pending_seqno = gt->tlb_invalidation.seqno - 1;
 	WRITE_ONCE(gt->tlb_invalidation.seqno_recv, pending_seqno);
 	wake_up_all(&guc->ct.wq);

 	list_for_each_entry_safe(fence, next,
 				 &gt->tlb_invalidation.pending_fences, link)
 		invalidation_fence_signal(fence);
 	spin_unlock_irq(&gt->tlb_invalidation.pending_lock);
 	mutex_unlock(&gt->uc.guc.ct.lock);
 }

 static bool tlb_invalidation_seqno_past(struct xe_gt *gt, int seqno)
 {
 	int seqno_recv = READ_ONCE(gt->tlb_invalidation.seqno_recv);

 	if (seqno - seqno_recv < -(TLB_INVALIDATION_SEQNO_MAX / 2))
 		return false;

 	if (seqno - seqno_recv > (TLB_INVALIDATION_SEQNO_MAX / 2))
 		return true;

 	return seqno_recv >= seqno;
 }

 static int send_tlb_invalidation(struct xe_guc *guc,
 				 struct xe_gt_tlb_invalidation_fence *fence,
 				 u32 *action, int len)
 {
 	struct xe_gt *gt = guc_to_gt(guc);
 	int seqno;
 	int ret;

 	/*
 	 * XXX: The seqno algorithm relies on TLB invalidation being processed
 	 * in order which they currently are, if that changes the algorithm will
 	 * need to be updated.
 	 */

 	mutex_lock(&guc->ct.lock);
 	seqno = gt->tlb_invalidation.seqno;
 	if (fence) {
 		fence->seqno = seqno;
 		trace_xe_gt_tlb_invalidation_fence_send(fence);
 	}
 	action[1] = seqno;
 	ret = xe_guc_ct_send_locked(&guc->ct, action, len,
 				    G2H_LEN_DW_TLB_INVALIDATE, 1);
 	if (!ret && fence) {
 		spin_lock_irq(&gt->tlb_invalidation.pending_lock);
 		/*
 		 * We haven't actually published the TLB fence as per
 		 * pending_fences, but in theory our seqno could have already
 		 * been written as we acquired the pending_lock. In such a case
 		 * we can just go ahead and signal the fence here.
 		 */
 		if (tlb_invalidation_seqno_past(gt, seqno)) {
 			__invalidation_fence_signal(fence);
 		} else {
 			fence->invalidation_time = ktime_get();
 			list_add_tail(&fence->link,
 				      &gt->tlb_invalidation.pending_fences);

 			if (list_is_singular(&gt->tlb_invalidation.pending_fences))
 				queue_delayed_work(system_wq,
 						   &gt->tlb_invalidation.fence_tdr,
 						   TLB_TIMEOUT);
 		}
 		spin_unlock_irq(&gt->tlb_invalidation.pending_lock);
 	} else if (ret < 0 && fence) {
 		__invalidation_fence_signal(fence);
 	}
 	if (!ret) {
 		gt->tlb_invalidation.seqno = (gt->tlb_invalidation.seqno + 1) %
 			TLB_INVALIDATION_SEQNO_MAX;
 		if (!gt->tlb_invalidation.seqno)
 			gt->tlb_invalidation.seqno = 1;
 		ret = seqno;
 	}
 	mutex_unlock(&guc->ct.lock);

 	return ret;
 }

 #define MAKE_INVAL_OP(type)	((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
 		XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \
 		XE_GUC_TLB_INVAL_FLUSH_CACHE)

 /**
  * xe_gt_tlb_invalidation_guc - Issue a TLB invalidation on this GT for the GuC
  * @gt: graphics tile
  *
  * Issue a TLB invalidation for the GuC. Completion of TLB is asynchronous and
  * caller can use seqno + xe_gt_tlb_invalidation_wait to wait for completion.
  *
  * Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,
  * negative error code on error.
  */
 static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt)
 {
 	u32 action[] = {
 		XE_GUC_ACTION_TLB_INVALIDATION,
 		0,  /* seqno, replaced in send_tlb_invalidation */
 		MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC),
 	};

 	return send_tlb_invalidation(&gt->uc.guc, NULL, action,
 				     ARRAY_SIZE(action));
 }

 /**
  * xe_gt_tlb_invalidation_ggtt - Issue a TLB invalidation on this GT for the GGTT
  * @gt: graphics tile
  *
  * Issue a TLB invalidation for the GGTT. Completion of TLB invalidation is
  * synchronous.
  *
  * Return: 0 on success, negative error code on error
  */
 int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt)
 {
 	struct xe_device *xe = gt_to_xe(gt);

 	if (xe_guc_ct_enabled(&gt->uc.guc.ct) &&
 	    gt->uc.guc.submission_state.enabled) {
 		int seqno;

 		seqno = xe_gt_tlb_invalidation_guc(gt);
 		if (seqno <= 0)
 			return seqno;

 		xe_gt_tlb_invalidation_wait(gt, seqno);
 	} else if (xe_device_uc_enabled(xe)) {
 		xe_gt_WARN_ON(gt, xe_force_wake_get(gt_to_fw(gt), XE_FW_GT));
 		if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) {
 			xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC1,
 					PVC_GUC_TLB_INV_DESC1_INVALIDATE);
 			xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC0,
 					PVC_GUC_TLB_INV_DESC0_VALID);
 		} else {
 			xe_mmio_write32(gt, GUC_TLB_INV_CR,
 					GUC_TLB_INV_CR_INVALIDATE);
 		}
 		xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
 	}

 	return 0;
 }

 /**
  * xe_gt_tlb_invalidation_vma - Issue a TLB invalidation on this GT for a VMA
  * @gt: graphics tile
  * @fence: invalidation fence which will be signal on TLB invalidation
  * completion, can be NULL
  * @vma: VMA to invalidate
  *
  * Issue a range based TLB invalidation if supported, if not fallback to a full
  * TLB invalidation. Completion of TLB is asynchronous and caller can either use
  * the invalidation fence or seqno + xe_gt_tlb_invalidation_wait to wait for
  * completion.
  *
  * Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,
  * negative error code on error.
  */
 int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
 			       struct xe_gt_tlb_invalidation_fence *fence,
 			       struct xe_vma *vma)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 #define MAX_TLB_INVALIDATION_LEN	7
 	u32 action[MAX_TLB_INVALIDATION_LEN];
 	int len = 0;

 	xe_gt_assert(gt, vma);

 	/* Execlists not supported */
 	if (gt_to_xe(gt)->info.force_execlist) {
 		if (fence)
 			__invalidation_fence_signal(fence);

 		return 0;
 	}

 	action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
 	action[len++] = 0; /* seqno, replaced in send_tlb_invalidation */
 	if (!xe->info.has_range_tlb_invalidation) {
 		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
 	} else {
 		u64 start = xe_vma_start(vma);
 		u64 length = xe_vma_size(vma);
 		u64 align, end;

 		if (length < SZ_4K)
 			length = SZ_4K;

 		/*
 		 * We need to invalidate a higher granularity if start address
 		 * is not aligned to length. When start is not aligned with
 		 * length we need to find the length large enough to create an
 		 * address mask covering the required range.
 		 */
 		align = roundup_pow_of_two(length);
 		start = ALIGN_DOWN(xe_vma_start(vma), align);
 		end = ALIGN(xe_vma_end(vma), align);
 		length = align;
 		while (start + length < end) {
 			length <<= 1;
 			start = ALIGN_DOWN(xe_vma_start(vma), length);
 		}

 		/*
 		 * Minimum invalidation size for a 2MB page that the hardware
 		 * expects is 16MB
 		 */
 		if (length >= SZ_2M) {
 			length = max_t(u64, SZ_16M, length);
 			start = ALIGN_DOWN(xe_vma_start(vma), length);
 		}

 		xe_gt_assert(gt, length >= SZ_4K);
 		xe_gt_assert(gt, is_power_of_2(length));
 		xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1, ilog2(SZ_2M) + 1)));
 		xe_gt_assert(gt, IS_ALIGNED(start, length));

 		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE);
 		action[len++] = xe_vma_vm(vma)->usm.asid;
 		action[len++] = lower_32_bits(start);
 		action[len++] = upper_32_bits(start);
 		action[len++] = ilog2(length) - ilog2(SZ_4K);
 	}

 	xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);

 	return send_tlb_invalidation(&gt->uc.guc, fence, action, len);
 }

 /**
  * xe_gt_tlb_invalidation_wait - Wait for TLB to complete
  * @gt: graphics tile
  * @seqno: seqno to wait which was returned from xe_gt_tlb_invalidation
  *
  * Wait for 200ms for a TLB invalidation to complete, in practice we always
  * should receive the TLB invalidation within 200ms.
  *
  * Return: 0 on success, -ETIME on TLB invalidation timeout
  */
 int xe_gt_tlb_invalidation_wait(struct xe_gt *gt, int seqno)
 {
 	struct xe_guc *guc = &gt->uc.guc;
 	int ret;

 	/* Execlists not supported */
 	if (gt_to_xe(gt)->info.force_execlist)
 		return 0;

 	/*
 	 * XXX: See above, this algorithm only works if seqno are always in
 	 * order
 	 */
 	ret = wait_event_timeout(guc->ct.wq,
 				 tlb_invalidation_seqno_past(gt, seqno),
 				 TLB_TIMEOUT);
 	if (!ret) {
 		struct drm_printer p = xe_gt_err_printer(gt);

 		xe_gt_err(gt, "TLB invalidation time'd out, seqno=%d, recv=%d\n",
 			  seqno, gt->tlb_invalidation.seqno_recv);
 		xe_guc_ct_print(&guc->ct, &p, true);
 		return -ETIME;
 	}

 	return 0;
 }

 /**
  * xe_guc_tlb_invalidation_done_handler - TLB invalidation done handler
  * @guc: guc
  * @msg: message indicating TLB invalidation done
  * @len: length of message
  *
  * Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any
  * invalidation fences for seqno. Algorithm for this depends on seqno being
  * received in-order and asserts this assumption.
  *
  * Return: 0 on success, -EPROTO for malformed messages.
  */
 int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
 {
 	struct xe_gt *gt = guc_to_gt(guc);
 	struct xe_gt_tlb_invalidation_fence *fence, *next;
 	unsigned long flags;

 	if (unlikely(len != 1))
 		return -EPROTO;

 	/*
 	 * This can also be run both directly from the IRQ handler and also in
 	 * process_g2h_msg(). Only one may process any individual CT message,
 	 * however the order they are processed here could result in skipping a
 	 * seqno. To handle that we just process all the seqnos from the last
 	 * seqno_recv up to and including the one in msg[0]. The delta should be
 	 * very small so there shouldn't be much of pending_fences we actually
 	 * need to iterate over here.
 	 *
 	 * From GuC POV we expect the seqnos to always appear in-order, so if we
 	 * see something later in the timeline we can be sure that anything
 	 * appearing earlier has already signalled, just that we have yet to
 	 * officially process the CT message like if racing against
 	 * process_g2h_msg().
 	 */
 	spin_lock_irqsave(&gt->tlb_invalidation.pending_lock, flags);
 	if (tlb_invalidation_seqno_past(gt, msg[0])) {
 		spin_unlock_irqrestore(&gt->tlb_invalidation.pending_lock, flags);
 		return 0;
 	}

 	/*
 	 * wake_up_all() and wait_event_timeout() already have the correct
 	 * barriers.
 	 */
 	WRITE_ONCE(gt->tlb_invalidation.seqno_recv, msg[0]);
 	wake_up_all(&guc->ct.wq);

 	list_for_each_entry_safe(fence, next,
 				 &gt->tlb_invalidation.pending_fences, link) {
 		trace_xe_gt_tlb_invalidation_fence_recv(fence);

 		if (!tlb_invalidation_seqno_past(gt, fence->seqno))
 			break;

 		invalidation_fence_signal(fence);
 	}

 	if (!list_empty(&gt->tlb_invalidation.pending_fences))
 		mod_delayed_work(system_wq,
 				 &gt->tlb_invalidation.fence_tdr,
 				 TLB_TIMEOUT);
 	else
 		cancel_delayed_work(&gt->tlb_invalidation.fence_tdr);

 	spin_unlock_irqrestore(&gt->tlb_invalidation.pending_lock, flags);

 	return 0;
 }
	// SPDX-License-Identifier: MIT
	/*
	* Copyright © 2023 Intel Corporation
	*/

	#include "xe_gt_tlb_invalidation.h"

	#include "abi/guc_actions_abi.h"
	#include "xe_device.h"
	#include "xe_gt.h"
	#include "xe_gt_printk.h"
	#include "xe_guc.h"
	#include "xe_guc_ct.h"
	#include "xe_mmio.h"
	#include "xe_trace.h"
	#include "regs/xe_guc_regs.h"

	#define TLB_TIMEOUT (HZ / 4)

	static void xe_gt_tlb_fence_timeout(struct work_struct *work)
	{
	struct xe_gt *gt = container_of(work, struct xe_gt,
	tlb_invalidation.fence_tdr.work);
	struct xe_gt_tlb_invalidation_fence fence, next;

	spin_lock_irq(&gt->tlb_invalidation.pending_lock);
	list_for_each_entry_safe(fence, next,
	&gt->tlb_invalidation.pending_fences, link) {
	s64 since_inval_ms = ktime_ms_delta(ktime_get(),
	fence->invalidation_time);

	if (msecs_to_jiffies(since_inval_ms) < TLB_TIMEOUT)
	break;

	trace_xe_gt_tlb_invalidation_fence_timeout(fence);
	xe_gt_err(gt, "TLB invalidation fence timeout, seqno=%d recv=%d",
	fence->seqno, gt->tlb_invalidation.seqno_recv);

	list_del(&fence->link);
	fence->base.error = -ETIME;
	dma_fence_signal(&fence->base);
	dma_fence_put(&fence->base);
	}
	if (!list_empty(&gt->tlb_invalidation.pending_fences))
	queue_delayed_work(system_wq,
	&gt->tlb_invalidation.fence_tdr,
	TLB_TIMEOUT);
	spin_unlock_irq(&gt->tlb_invalidation.pending_lock);
	}

	/**
	* xe_gt_tlb_invalidation_init - Initialize GT TLB invalidation state
	* @gt: graphics tile
	*
	* Initialize GT TLB invalidation state, purely software initialization, should
	* be called once during driver load.
	*
	* Return: 0 on success, negative error code on error.
	*/
	int xe_gt_tlb_invalidation_init(struct xe_gt *gt)
	{
	gt->tlb_invalidation.seqno = 1;
	INIT_LIST_HEAD(&gt->tlb_invalidation.pending_fences);
	spin_lock_init(&gt->tlb_invalidation.pending_lock);
	spin_lock_init(&gt->tlb_invalidation.lock);
	INIT_DELAYED_WORK(&gt->tlb_invalidation.fence_tdr,
	xe_gt_tlb_fence_timeout);

	return 0;
	}

	static void
	__invalidation_fence_signal(struct xe_gt_tlb_invalidation_fence *fence)
	{
	trace_xe_gt_tlb_invalidation_fence_signal(fence);
	dma_fence_signal(&fence->base);
	dma_fence_put(&fence->base);
	}

	static void
	invalidation_fence_signal(struct xe_gt_tlb_invalidation_fence *fence)
	{
	list_del(&fence->link);
	__invalidation_fence_signal(fence);
	}

	/**
	* xe_gt_tlb_invalidation_reset - Initialize GT TLB invalidation reset
	* @gt: graphics tile
	*
	* Signal any pending invalidation fences, should be called during a GT reset
	*/
	void xe_gt_tlb_invalidation_reset(struct xe_gt *gt)
	{
	struct xe_gt_tlb_invalidation_fence fence, next;
	struct xe_guc *guc = &gt->uc.guc;
	int pending_seqno;

	/*
	* CT channel is already disabled at this point. No new TLB requests can
	* appear.
	*/

	mutex_lock(&gt->uc.guc.ct.lock);
	spin_lock_irq(&gt->tlb_invalidation.pending_lock);
	cancel_delayed_work(&gt->tlb_invalidation.fence_tdr);
	/*
	* We might have various kworkers waiting for TLB flushes to complete
	* which are not tracked with an explicit TLB fence, however at this
	* stage that will never happen since the CT is already disabled, so
	* make sure we signal them here under the assumption that we have
	* completed a full GT reset.
	*/
	if (gt->tlb_invalidation.seqno == 1)
	pending_seqno = TLB_INVALIDATION_SEQNO_MAX - 1;
	else
	pending_seqno = gt->tlb_invalidation.seqno - 1;
	WRITE_ONCE(gt->tlb_invalidation.seqno_recv, pending_seqno);
	wake_up_all(&guc->ct.wq);

	list_for_each_entry_safe(fence, next,
	&gt->tlb_invalidation.pending_fences, link)
	invalidation_fence_signal(fence);
	spin_unlock_irq(&gt->tlb_invalidation.pending_lock);
	mutex_unlock(&gt->uc.guc.ct.lock);
	}

	static bool tlb_invalidation_seqno_past(struct xe_gt *gt, int seqno)
	{
	int seqno_recv = READ_ONCE(gt->tlb_invalidation.seqno_recv);

	if (seqno - seqno_recv < -(TLB_INVALIDATION_SEQNO_MAX / 2))
	return false;

	if (seqno - seqno_recv > (TLB_INVALIDATION_SEQNO_MAX / 2))
	return true;

	return seqno_recv >= seqno;
	}

	static int send_tlb_invalidation(struct xe_guc *guc,
	struct xe_gt_tlb_invalidation_fence *fence,
	u32 *action, int len)
	{
	struct xe_gt *gt = guc_to_gt(guc);
	int seqno;
	int ret;

	/*
	* XXX: The seqno algorithm relies on TLB invalidation being processed
	* in order which they currently are, if that changes the algorithm will
	* need to be updated.
	*/

	mutex_lock(&guc->ct.lock);
	seqno = gt->tlb_invalidation.seqno;
	if (fence) {
	fence->seqno = seqno;
	trace_xe_gt_tlb_invalidation_fence_send(fence);
	}
	action[1] = seqno;
	ret = xe_guc_ct_send_locked(&guc->ct, action, len,
	G2H_LEN_DW_TLB_INVALIDATE, 1);
	if (!ret && fence) {
	spin_lock_irq(&gt->tlb_invalidation.pending_lock);
	/*
	* We haven't actually published the TLB fence as per
	* pending_fences, but in theory our seqno could have already
	* been written as we acquired the pending_lock. In such a case
	* we can just go ahead and signal the fence here.
	*/
	if (tlb_invalidation_seqno_past(gt, seqno)) {
	__invalidation_fence_signal(fence);
	} else {
	fence->invalidation_time = ktime_get();
	list_add_tail(&fence->link,
	&gt->tlb_invalidation.pending_fences);

	if (list_is_singular(&gt->tlb_invalidation.pending_fences))
	queue_delayed_work(system_wq,
	&gt->tlb_invalidation.fence_tdr,
	TLB_TIMEOUT);
	}
	spin_unlock_irq(&gt->tlb_invalidation.pending_lock);
	} else if (ret < 0 && fence) {
	__invalidation_fence_signal(fence);
	}
	if (!ret) {
	gt->tlb_invalidation.seqno = (gt->tlb_invalidation.seqno + 1) %
	TLB_INVALIDATION_SEQNO_MAX;
	if (!gt->tlb_invalidation.seqno)
	gt->tlb_invalidation.seqno = 1;
	ret = seqno;
	}
	mutex_unlock(&guc->ct.lock);

	return ret;
	}

	#define MAKE_INVAL_OP(type) ((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) \| \
	XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT \| \
	XE_GUC_TLB_INVAL_FLUSH_CACHE)

	/**
	* xe_gt_tlb_invalidation_guc - Issue a TLB invalidation on this GT for the GuC
	* @gt: graphics tile
	*
	* Issue a TLB invalidation for the GuC. Completion of TLB is asynchronous and
	* caller can use seqno + xe_gt_tlb_invalidation_wait to wait for completion.
	*
	* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,
	* negative error code on error.
	*/
	static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt)
	{
	u32 action[] = {
	XE_GUC_ACTION_TLB_INVALIDATION,
	0, /* seqno, replaced in send_tlb_invalidation */
	MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC),
	};

	return send_tlb_invalidation(&gt->uc.guc, NULL, action,
	ARRAY_SIZE(action));
	}

	/**
	* xe_gt_tlb_invalidation_ggtt - Issue a TLB invalidation on this GT for the GGTT
	* @gt: graphics tile
	*
	* Issue a TLB invalidation for the GGTT. Completion of TLB invalidation is
	* synchronous.
	*
	* Return: 0 on success, negative error code on error
	*/
	int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt)
	{
	struct xe_device *xe = gt_to_xe(gt);

	if (xe_guc_ct_enabled(&gt->uc.guc.ct) &&
	gt->uc.guc.submission_state.enabled) {
	int seqno;

	seqno = xe_gt_tlb_invalidation_guc(gt);
	if (seqno <= 0)
	return seqno;

	xe_gt_tlb_invalidation_wait(gt, seqno);
	} else if (xe_device_uc_enabled(xe)) {
	xe_gt_WARN_ON(gt, xe_force_wake_get(gt_to_fw(gt), XE_FW_GT));
	if (xe->info.platform == XE_PVC \|\| GRAPHICS_VER(xe) >= 20) {
	xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC1,
	PVC_GUC_TLB_INV_DESC1_INVALIDATE);
	xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC0,
	PVC_GUC_TLB_INV_DESC0_VALID);
	} else {
	xe_mmio_write32(gt, GUC_TLB_INV_CR,
	GUC_TLB_INV_CR_INVALIDATE);
	}
	xe_force_wake_put(gt_to_fw(gt), XE_FW_GT);
	}

	return 0;
	}

	/**
	* xe_gt_tlb_invalidation_vma - Issue a TLB invalidation on this GT for a VMA
	* @gt: graphics tile
	* @fence: invalidation fence which will be signal on TLB invalidation
	* completion, can be NULL
	* @vma: VMA to invalidate
	*
	* Issue a range based TLB invalidation if supported, if not fallback to a full
	* TLB invalidation. Completion of TLB is asynchronous and caller can either use
	* the invalidation fence or seqno + xe_gt_tlb_invalidation_wait to wait for
	* completion.
	*
	* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success,
	* negative error code on error.
	*/
	int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,
	struct xe_gt_tlb_invalidation_fence *fence,
	struct xe_vma *vma)
	{
	struct xe_device *xe = gt_to_xe(gt);
	#define MAX_TLB_INVALIDATION_LEN 7
	u32 action[MAX_TLB_INVALIDATION_LEN];
	int len = 0;

	xe_gt_assert(gt, vma);

	/* Execlists not supported */
	if (gt_to_xe(gt)->info.force_execlist) {
	if (fence)
	__invalidation_fence_signal(fence);

	return 0;
	}

	action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
	action[len++] = 0; /* seqno, replaced in send_tlb_invalidation */
	if (!xe->info.has_range_tlb_invalidation) {
	action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
	} else {
	u64 start = xe_vma_start(vma);
	u64 length = xe_vma_size(vma);
	u64 align, end;

	if (length < SZ_4K)
	length = SZ_4K;

	/*
	* We need to invalidate a higher granularity if start address
	* is not aligned to length. When start is not aligned with
	* length we need to find the length large enough to create an
	* address mask covering the required range.
	*/
	align = roundup_pow_of_two(length);
	start = ALIGN_DOWN(xe_vma_start(vma), align);
	end = ALIGN(xe_vma_end(vma), align);
	length = align;
	while (start + length < end) {
	length <<= 1;
	start = ALIGN_DOWN(xe_vma_start(vma), length);
	}

	/*
	* Minimum invalidation size for a 2MB page that the hardware
	* expects is 16MB
	*/
	if (length >= SZ_2M) {
	length = max_t(u64, SZ_16M, length);
	start = ALIGN_DOWN(xe_vma_start(vma), length);
	}

	xe_gt_assert(gt, length >= SZ_4K);
	xe_gt_assert(gt, is_power_of_2(length));
	xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1, ilog2(SZ_2M) + 1)));
	xe_gt_assert(gt, IS_ALIGNED(start, length));

	action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_PAGE_SELECTIVE);
	action[len++] = xe_vma_vm(vma)->usm.asid;
	action[len++] = lower_32_bits(start);
	action[len++] = upper_32_bits(start);
	action[len++] = ilog2(length) - ilog2(SZ_4K);
	}

	xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);

	return send_tlb_invalidation(&gt->uc.guc, fence, action, len);
	}

	/**
	* xe_gt_tlb_invalidation_wait - Wait for TLB to complete
	* @gt: graphics tile
	* @seqno: seqno to wait which was returned from xe_gt_tlb_invalidation
	*
	* Wait for 200ms for a TLB invalidation to complete, in practice we always
	* should receive the TLB invalidation within 200ms.
	*
	* Return: 0 on success, -ETIME on TLB invalidation timeout
	*/
	int xe_gt_tlb_invalidation_wait(struct xe_gt *gt, int seqno)
	{
	struct xe_guc *guc = &gt->uc.guc;
	int ret;

	/* Execlists not supported */
	if (gt_to_xe(gt)->info.force_execlist)
	return 0;

	/*
	* XXX: See above, this algorithm only works if seqno are always in
	* order
	*/
	ret = wait_event_timeout(guc->ct.wq,
	tlb_invalidation_seqno_past(gt, seqno),
	TLB_TIMEOUT);
	if (!ret) {
	struct drm_printer p = xe_gt_err_printer(gt);

	xe_gt_err(gt, "TLB invalidation time'd out, seqno=%d, recv=%d\n",
	seqno, gt->tlb_invalidation.seqno_recv);
	xe_guc_ct_print(&guc->ct, &p, true);
	return -ETIME;
	}

	return 0;
	}

	/**
	* xe_guc_tlb_invalidation_done_handler - TLB invalidation done handler
	* @guc: guc
	* @msg: message indicating TLB invalidation done
	* @len: length of message
	*
	* Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any
	* invalidation fences for seqno. Algorithm for this depends on seqno being
	* received in-order and asserts this assumption.
	*
	* Return: 0 on success, -EPROTO for malformed messages.
	*/
	int xe_guc_tlb_invalidation_done_handler(struct xe_guc guc, u32 msg, u32 len)
	{
	struct xe_gt *gt = guc_to_gt(guc);
	struct xe_gt_tlb_invalidation_fence fence, next;
	unsigned long flags;

	if (unlikely(len != 1))
	return -EPROTO;

	/*
	* This can also be run both directly from the IRQ handler and also in
	* process_g2h_msg(). Only one may process any individual CT message,
	* however the order they are processed here could result in skipping a
	* seqno. To handle that we just process all the seqnos from the last
	* seqno_recv up to and including the one in msg[0]. The delta should be
	* very small so there shouldn't be much of pending_fences we actually
	* need to iterate over here.
	*
	* From GuC POV we expect the seqnos to always appear in-order, so if we
	* see something later in the timeline we can be sure that anything
	* appearing earlier has already signalled, just that we have yet to
	* officially process the CT message like if racing against
	* process_g2h_msg().
	*/
	spin_lock_irqsave(&gt->tlb_invalidation.pending_lock, flags);
	if (tlb_invalidation_seqno_past(gt, msg[0])) {
	spin_unlock_irqrestore(&gt->tlb_invalidation.pending_lock, flags);
	return 0;
	}

	/*
	* wake_up_all() and wait_event_timeout() already have the correct
	* barriers.
	*/
	WRITE_ONCE(gt->tlb_invalidation.seqno_recv, msg[0]);
	wake_up_all(&guc->ct.wq);

	list_for_each_entry_safe(fence, next,
	&gt->tlb_invalidation.pending_fences, link) {
	trace_xe_gt_tlb_invalidation_fence_recv(fence);

	if (!tlb_invalidation_seqno_past(gt, fence->seqno))
	break;

	invalidation_fence_signal(fence);
	}

	if (!list_empty(&gt->tlb_invalidation.pending_fences))
	mod_delayed_work(system_wq,
	&gt->tlb_invalidation.fence_tdr,
	TLB_TIMEOUT);
	else
	cancel_delayed_work(&gt->tlb_invalidation.fence_tdr);

	spin_unlock_irqrestore(&gt->tlb_invalidation.pending_lock, flags);

	return 0;
	}