Contributors: 13
Author Tokens Token Proportion Commits Commit Proportion
Matthew Brost 1538 89.21% 19 52.78%
Brian Nguyen 117 6.79% 3 8.33%
Nirmoy Das 30 1.74% 2 5.56%
Michal Wajdeczko 10 0.58% 3 8.33%
Francois Dugast 5 0.29% 1 2.78%
Matt Roper 5 0.29% 1 2.78%
Lucas De Marchi 5 0.29% 1 2.78%
Thomas Hellstrom 5 0.29% 1 2.78%
Radhakrishna Sripada 4 0.23% 1 2.78%
Matthew Auld 2 0.12% 1 2.78%
Tangudu Tilak Tirumalesh 1 0.06% 1 2.78%
Sanjay Yadav 1 0.06% 1 2.78%
Stuart Summers 1 0.06% 1 2.78%
Total 1724 36 


// SPDX-License-Identifier: MIT
/*
 * Copyright © 2025 Intel Corporation
 */

#include "abi/guc_actions_abi.h"

#include "xe_device.h"
#include "xe_exec_queue.h"
#include "xe_exec_queue_types.h"
#include "xe_gt_stats.h"
#include "xe_gt_types.h"
#include "xe_guc.h"
#include "xe_guc_ct.h"
#include "xe_guc_exec_queue_types.h"
#include "xe_guc_tlb_inval.h"
#include "xe_force_wake.h"
#include "xe_mmio.h"
#include "xe_sa.h"
#include "xe_tlb_inval.h"
#include "xe_vm.h"

#include "regs/xe_guc_regs.h"

/*
 * XXX: The seqno algorithm relies on TLB invalidation being processed in order
 * which they currently are by the GuC, if that changes the algorithm will need
 * to be updated.
 */

static int send_tlb_inval(struct xe_guc *guc, const u32 *action, int len)
{
	struct xe_gt *gt = guc_to_gt(guc);

	xe_gt_assert(gt, action[1]);	/* Seqno */

	xe_gt_stats_incr(gt, XE_GT_STATS_ID_TLB_INVAL, 1);
	return xe_guc_ct_send(&guc->ct, action, len,
			      G2H_LEN_DW_TLB_INVALIDATE, 1);
}

#define MAKE_INVAL_OP_FLUSH(type, flush_cache)	((type << XE_GUC_TLB_INVAL_TYPE_SHIFT) | \
		XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | \
		(flush_cache ? \
		XE_GUC_TLB_INVAL_FLUSH_CACHE : 0))

#define MAKE_INVAL_OP(type)	MAKE_INVAL_OP_FLUSH(type, true)

static int send_tlb_inval_all(struct xe_tlb_inval *tlb_inval, u32 seqno)
{
	struct xe_guc *guc = tlb_inval->private;
	u32 action[] = {
		XE_GUC_ACTION_TLB_INVALIDATION_ALL,
		seqno,
		MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL),
	};

	return send_tlb_inval(guc, action, ARRAY_SIZE(action));
}

static int send_tlb_inval_ggtt(struct xe_tlb_inval *tlb_inval, u32 seqno)
{
	struct xe_guc *guc = tlb_inval->private;
	struct xe_gt *gt = guc_to_gt(guc);
	struct xe_device *xe = guc_to_xe(guc);

	/*
	 * Returning -ECANCELED in this function is squashed at the caller and
	 * signals waiters.
	 */

	if (xe_guc_ct_enabled(&guc->ct) && guc->submission_state.enabled) {
		u32 action[] = {
			XE_GUC_ACTION_TLB_INVALIDATION,
			seqno,
			MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC),
		};

		return send_tlb_inval(guc, action, ARRAY_SIZE(action));
	} else if (xe_device_uc_enabled(xe) && !xe_device_wedged(xe)) {
		struct xe_mmio *mmio = &gt->mmio;

		if (IS_SRIOV_VF(xe))
			return -ECANCELED;

		CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FW_GT);
		if (xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20) {
			xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC1,
					PVC_GUC_TLB_INV_DESC1_INVALIDATE);
			xe_mmio_write32(mmio, PVC_GUC_TLB_INV_DESC0,
					PVC_GUC_TLB_INV_DESC0_VALID);
		} else {
			xe_mmio_write32(mmio, GUC_TLB_INV_CR,
					GUC_TLB_INV_CR_INVALIDATE);
		}
	}

	return -ECANCELED;
}

static int send_page_reclaim(struct xe_guc *guc, u32 seqno,
			     u64 gpu_addr)
{
	struct xe_gt *gt = guc_to_gt(guc);
	u32 action[] = {
		XE_GUC_ACTION_PAGE_RECLAMATION,
		seqno,
		lower_32_bits(gpu_addr),
		upper_32_bits(gpu_addr),
	};

	xe_gt_stats_incr(gt, XE_GT_STATS_ID_PRL_ISSUED_COUNT, 1);

	return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
			      G2H_LEN_DW_PAGE_RECLAMATION, 1);
}

static u64 normalize_invalidation_range(struct xe_gt *gt, u64 *start, u64 *end)
{
	u64 orig_start = *start;
	u64 length = *end - *start;
	u64 align;

	if (length < SZ_4K)
		length = SZ_4K;

	align = roundup_pow_of_two(length);
	*start = ALIGN_DOWN(*start, align);
	*end = ALIGN(*end, align);
	length = align;
	while (*start + length < *end) {
		length <<= 1;
		*start = ALIGN_DOWN(orig_start, length);
	}

	if (length >= SZ_2M) {
		length = max_t(u64, SZ_16M, length);
		*start = ALIGN_DOWN(orig_start, length);
	}

	xe_gt_assert(gt, length >= SZ_4K);
	xe_gt_assert(gt, is_power_of_2(length));
	xe_gt_assert(gt, !(length & GENMASK(ilog2(SZ_16M) - 1,
					    ilog2(SZ_2M) + 1)));
	xe_gt_assert(gt, IS_ALIGNED(*start, length));

	return length;
}

/*
 * Ensure that roundup_pow_of_two(length) doesn't overflow.
 * Note that roundup_pow_of_two() operates on unsigned long,
 * not on u64.
 */
#define MAX_RANGE_TLB_INVALIDATION_LENGTH (rounddown_pow_of_two(ULONG_MAX))

static int send_tlb_inval_ppgtt(struct xe_guc *guc, u32 seqno, u64 start,
				u64 end, u32 id, u32 type,
				struct drm_suballoc *prl_sa)
{
#define MAX_TLB_INVALIDATION_LEN	7
	struct xe_gt *gt = guc_to_gt(guc);
	struct xe_device *xe = guc_to_xe(guc);
	u32 action[MAX_TLB_INVALIDATION_LEN];
	u64 length = end - start;
	int len = 0, err;

	xe_gt_assert(gt, (type == XE_GUC_TLB_INVAL_PAGE_SELECTIVE &&
			  !xe->info.has_ctx_tlb_inval) ||
		     (type == XE_GUC_TLB_INVAL_PAGE_SELECTIVE_CTX &&
		      xe->info.has_ctx_tlb_inval));

	action[len++] = XE_GUC_ACTION_TLB_INVALIDATION;
	action[len++] = !prl_sa ? seqno : TLB_INVALIDATION_SEQNO_INVALID;
	if (!gt_to_xe(gt)->info.has_range_tlb_inval ||
	    length > MAX_RANGE_TLB_INVALIDATION_LENGTH) {
		action[len++] = MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL);
	} else {
		u64 normalize_len = normalize_invalidation_range(gt, &start,
								 &end);
		bool need_flush = !prl_sa &&
			seqno != TLB_INVALIDATION_SEQNO_INVALID;

		/* Flush on NULL case, Media is not required to modify flush due to no PPC so NOP */
		action[len++] = MAKE_INVAL_OP_FLUSH(type, need_flush);
		action[len++] = id;
		action[len++] = lower_32_bits(start);
		action[len++] = upper_32_bits(start);
		action[len++] = ilog2(normalize_len) - ilog2(SZ_4K);
	}

	xe_gt_assert(gt, len <= MAX_TLB_INVALIDATION_LEN);
#undef MAX_TLB_INVALIDATION_LEN

	err = send_tlb_inval(guc, action, len);
	if (!err && prl_sa) {
		xe_gt_assert(gt, seqno != TLB_INVALIDATION_SEQNO_INVALID);
		err = send_page_reclaim(guc, seqno, xe_sa_bo_gpu_addr(prl_sa));
	}
	return err;
}

static int send_tlb_inval_asid_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
				     u64 start, u64 end, u32 asid,
				     struct drm_suballoc *prl_sa)
{
	struct xe_guc *guc = tlb_inval->private;

	lockdep_assert_held(&tlb_inval->seqno_lock);

	if (guc_to_xe(guc)->info.force_execlist)
		return -ECANCELED;

	return send_tlb_inval_ppgtt(guc, seqno, start, end, asid,
				    XE_GUC_TLB_INVAL_PAGE_SELECTIVE, prl_sa);
}

static int send_tlb_inval_ctx_ppgtt(struct xe_tlb_inval *tlb_inval, u32 seqno,
				    u64 start, u64 end, u32 asid,
				    struct drm_suballoc *prl_sa)
{
	struct xe_guc *guc = tlb_inval->private;
	struct xe_device *xe = guc_to_xe(guc);
	struct xe_exec_queue *q, *next, *last_q = NULL;
	struct xe_vm *vm;
	LIST_HEAD(tlb_inval_list);
	int err = 0, id = guc_to_gt(guc)->info.id;

	lockdep_assert_held(&tlb_inval->seqno_lock);

	if (xe->info.force_execlist)
		return -ECANCELED;

	vm = xe_device_asid_to_vm(xe, asid);
	if (IS_ERR(vm))
		return PTR_ERR(vm);

	down_read(&vm->exec_queues.lock);

	/*
	 * XXX: Randomly picking a threshold for now. This will need to be
	 * tuned based on expected UMD queue counts and performance profiling.
	 */
#define EXEC_QUEUE_COUNT_FULL_THRESHOLD	8
	if (vm->exec_queues.count[id] >= EXEC_QUEUE_COUNT_FULL_THRESHOLD) {
		u32 action[] = {
			XE_GUC_ACTION_TLB_INVALIDATION,
			seqno,
			MAKE_INVAL_OP(XE_GUC_TLB_INVAL_FULL),
		};

		err = send_tlb_inval(guc, action, ARRAY_SIZE(action));
		goto err_unlock;
	}
#undef EXEC_QUEUE_COUNT_FULL_THRESHOLD

	/*
	 * Move exec queues to a temporary list to issue invalidations. A
	 * reference must be taken to prevent concurrent deregistrations.
	 *
	 * List modification is safe because we hold 'vm->exec_queues.lock' for
	 * reading, which prevents external modifications. Using a per-GT list
	 * is also safe since 'tlb_inval->seqno_lock' ensures no other GT users
	 * can enter this code path.
	 */
	list_for_each_entry_safe(q, next, &vm->exec_queues.list[id],
				 vm_exec_queue_link) {
		if (xe_exec_queue_get_unless_zero(q)) {
			last_q = q;
			list_move_tail(&q->vm_exec_queue_link, &tlb_inval_list);
		}
	}

	if (!last_q) {
		/*
		 * We can't break fence ordering for TLB invalidation jobs, if
		 * TLB invalidations are inflight issue a dummy invalidation to
		 * maintain ordering. Nor can we move safely the seqno_recv when
		 * returning -ECANCELED if TLB invalidations are in flight. Use
		 * GGTT invalidation as dummy invalidation given ASID
		 * invalidations are unsupported here.
		 */
		if (xe_tlb_inval_idle(tlb_inval))
			err = -ECANCELED;
		else
			err = send_tlb_inval_ggtt(tlb_inval, seqno);
		goto err_unlock;
	}

	list_for_each_entry_safe(q, next, &tlb_inval_list, vm_exec_queue_link) {
		struct drm_suballoc *__prl_sa = NULL;
		int __seqno = TLB_INVALIDATION_SEQNO_INVALID;
		u32 type = XE_GUC_TLB_INVAL_PAGE_SELECTIVE_CTX;

		xe_assert(xe, q->vm == vm);

		if (err)
			goto unref;

		if (last_q == q) {
			__prl_sa = prl_sa;
			__seqno = seqno;
		}

		err = send_tlb_inval_ppgtt(guc, __seqno, start, end,
					   q->guc->id, type, __prl_sa);

unref:
		/*
		 * Must always return exec queue to original list / drop
		 * reference
		 */
		list_move_tail(&q->vm_exec_queue_link,
			       &vm->exec_queues.list[id]);
		xe_exec_queue_put(q);
	}

err_unlock:
	up_read(&vm->exec_queues.lock);
	xe_vm_put(vm);

	return err;
}

static bool tlb_inval_initialized(struct xe_tlb_inval *tlb_inval)
{
	struct xe_guc *guc = tlb_inval->private;

	return xe_guc_ct_initialized(&guc->ct);
}

static void tlb_inval_flush(struct xe_tlb_inval *tlb_inval)
{
	struct xe_guc *guc = tlb_inval->private;

	LNL_FLUSH_WORK(&guc->ct.g2h_worker);
}

static long tlb_inval_timeout_delay(struct xe_tlb_inval *tlb_inval)
{
	struct xe_guc *guc = tlb_inval->private;

	/* this reflects what HW/GuC needs to process TLB inv request */
	const long hw_tlb_timeout = HZ / 4;

	/* this estimates actual delay caused by the CTB transport */
	long delay = xe_guc_ct_queue_proc_time_jiffies(&guc->ct);

	return hw_tlb_timeout + 2 * delay;
}

static const struct xe_tlb_inval_ops guc_tlb_inval_asid_ops = {
	.all = send_tlb_inval_all,
	.ggtt = send_tlb_inval_ggtt,
	.ppgtt = send_tlb_inval_asid_ppgtt,
	.initialized = tlb_inval_initialized,
	.flush = tlb_inval_flush,
	.timeout_delay = tlb_inval_timeout_delay,
};

static const struct xe_tlb_inval_ops guc_tlb_inval_ctx_ops = {
	.ggtt = send_tlb_inval_ggtt,
	.all = send_tlb_inval_all,
	.ppgtt = send_tlb_inval_ctx_ppgtt,
	.initialized = tlb_inval_initialized,
	.flush = tlb_inval_flush,
	.timeout_delay = tlb_inval_timeout_delay,
};

/**
 * xe_guc_tlb_inval_init_early() - Init GuC TLB invalidation early
 * @guc: GuC object
 * @tlb_inval: TLB invalidation client
 *
 * Initialize GuC TLB invalidation by setting back pointer in TLB invalidation
 * client to the GuC and setting GuC backend ops.
 */
void xe_guc_tlb_inval_init_early(struct xe_guc *guc,
				 struct xe_tlb_inval *tlb_inval)
{
	struct xe_device *xe = guc_to_xe(guc);

	tlb_inval->private = guc;

	if (xe->info.has_ctx_tlb_inval)
		tlb_inval->ops = &guc_tlb_inval_ctx_ops;
	else
		tlb_inval->ops = &guc_tlb_inval_asid_ops;
}

/**
 * xe_guc_tlb_inval_done_handler() - TLB invalidation done handler
 * @guc: guc
 * @msg: message indicating TLB invalidation done
 * @len: length of message
 *
 * Parse seqno of TLB invalidation, wake any waiters for seqno, and signal any
 * invalidation fences for seqno. Algorithm for this depends on seqno being
 * received in-order and asserts this assumption.
 *
 * Return: 0 on success, -EPROTO for malformed messages.
 */
int xe_guc_tlb_inval_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
{
	struct xe_gt *gt = guc_to_gt(guc);

	if (unlikely(len != 1))
		return -EPROTO;

	xe_tlb_inval_done_handler(&gt->tlb_inval, msg[0]);

	return 0;
}