Contributors: 11
Author Tokens Token Proportion Commits Commit Proportion
Matthew Brost 328 52.15% 1 2.94%
Lucas De Marchi 192 30.52% 18 52.94%
Matt Roper 63 10.02% 5 14.71%
Gustavo Sousa 26 4.13% 3 8.82%
Harish Chegondi 5 0.79% 1 2.94%
Marcin Bernatowicz 5 0.79% 1 2.94%
Thomas Hellstrom 3 0.48% 1 2.94%
Michal Wajdeczko 2 0.32% 1 2.94%
Shuicheng Lin 2 0.32% 1 2.94%
Nitin Gote 2 0.32% 1 2.94%
Matt Atwood 1 0.16% 1 2.94%
Total 629 34 


/* SPDX-License-Identifier: MIT */
/*
 * Copyright © 2022 Intel Corporation
 */

#ifndef _XE_RTP_H_
#define _XE_RTP_H_

#include <linux/types.h>
#include <linux/xarray.h>

#define _XE_RTP_INCLUDE_PRIVATE_HELPERS

#include "xe_rtp_helpers.h"
#include "xe_rtp_types.h"

#undef _XE_RTP_INCLUDE_PRIVATE_HELPERS

/*
 * Register table poke infrastructure
 */

struct xe_hw_engine;
struct xe_gt;
struct xe_reg_sr;

/*
 * Macros to encode rules to match against platform, IP version, stepping, etc.
 * Shouldn't be used directly - see XE_RTP_RULES()
 */
#define _XE_RTP_RULE_PLATFORM(plat__)						\
	{ .match_type = XE_RTP_MATCH_PLATFORM, .platform = plat__ }

#define _XE_RTP_RULE_SUBPLATFORM(plat__, sub__)					\
	{ .match_type = XE_RTP_MATCH_SUBPLATFORM,				\
	  .platform = plat__, .subplatform = sub__ }

#define _XE_RTP_RULE_PLATFORM_STEP(start__, end__)				\
	{ .match_type = XE_RTP_MATCH_PLATFORM_STEP,				\
	  .step_start = start__, .step_end = end__ }

#define _XE_RTP_RULE_GRAPHICS_STEP(start__, end__)				\
	{ .match_type = XE_RTP_MATCH_GRAPHICS_STEP,				\
	  .step_start = start__, .step_end = end__ }

#define _XE_RTP_RULE_MEDIA_STEP(start__, end__)					\
	{ .match_type = XE_RTP_MATCH_MEDIA_STEP,				\
	  .step_start = start__, .step_end = end__ }

#define _XE_RTP_RULE_ENGINE_CLASS(cls__)					\
	{ .match_type = XE_RTP_MATCH_ENGINE_CLASS,				\
	  .engine_class = (cls__) }

/**
 * XE_RTP_RULE_PLATFORM - Create rule matching platform
 * @plat_: platform to match
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_PLATFORM(plat_)						\
	_XE_RTP_RULE_PLATFORM(XE_##plat_)

/**
 * XE_RTP_RULE_SUBPLATFORM - Create rule matching platform and sub-platform
 * @plat_: platform to match
 * @sub_: sub-platform to match
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_SUBPLATFORM(plat_, sub_)					\
	_XE_RTP_RULE_SUBPLATFORM(XE_##plat_, XE_SUBPLATFORM_##plat_##_##sub_)

/**
 * XE_RTP_RULE_PLATFORM_STEP - Create rule matching platform-level stepping
 * @start_: First stepping matching the rule
 * @end_: First stepping that does not match the rule
 *
 * Note that the range matching this rule is [ @start_, @end_ ), i.e. inclusive
 * on the left, exclusive on the right.
 *
 * You need to make sure that proper support for reading platform-level stepping
 * information is present for the target platform before using this rule.
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_PLATFORM_STEP(start_, end_)					\
	_XE_RTP_RULE_PLATFORM_STEP(STEP_##start_, STEP_##end_)

/**
 * XE_RTP_RULE_GRAPHICS_STEP - Create rule matching graphics stepping
 * @start_: First stepping matching the rule
 * @end_: First stepping that does not match the rule
 *
 * Note that the range matching this rule is [ @start_, @end_ ), i.e. inclusive
 * on the left, exclusive on the right.
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_GRAPHICS_STEP(start_, end_)					\
	_XE_RTP_RULE_GRAPHICS_STEP(STEP_##start_, STEP_##end_)

/**
 * XE_RTP_RULE_MEDIA_STEP - Create rule matching media stepping
 * @start_: First stepping matching the rule
 * @end_: First stepping that does not match the rule
 *
 * Note that the range matching this rule is [ @start_, @end_ ), i.e. inclusive
 * on the left, exclusive on the right.
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_MEDIA_STEP(start_, end_)					\
	_XE_RTP_RULE_MEDIA_STEP(STEP_##start_, STEP_##end_)

/**
 * XE_RTP_RULE_ENGINE_CLASS - Create rule matching an engine class
 * @cls_: Engine class to match
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_ENGINE_CLASS(cls_)						\
	_XE_RTP_RULE_ENGINE_CLASS(XE_ENGINE_CLASS_##cls_)

/**
 * XE_RTP_RULE_FUNC - Create rule using callback function for match
 * @func__: Function to call to decide if rule matches
 *
 * This allows more complex checks to be performed. The ``XE_RTP``
 * infrastructure will simply call the function @func_ passed to decide if this
 * rule matches the device.
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_FUNC(func__)						\
	{ .match_type = XE_RTP_MATCH_FUNC,					\
	  .match_func = (func__) }

/**
 * XE_RTP_RULE_GRAPHICS_VERSION - Create rule matching graphics version
 * @ver__: Graphics IP version to match
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_GRAPHICS_VERSION(ver__)					\
	{ .match_type = XE_RTP_MATCH_GRAPHICS_VERSION,				\
	  .ver_start = ver__, }

/**
 * XE_RTP_RULE_GRAPHICS_VERSION_RANGE - Create rule matching a range of graphics version
 * @ver_start__: First graphics IP version to match
 * @ver_end__: Last graphics IP version to match
 *
 * Note that the range matching this rule is [ @ver_start__, @ver_end__ ], i.e.
 * inclusive on both sides
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_GRAPHICS_VERSION_RANGE(ver_start__, ver_end__)		\
	{ .match_type = XE_RTP_MATCH_GRAPHICS_VERSION_RANGE,			\
	  .ver_start = ver_start__, .ver_end = ver_end__, }

/**
 * XE_RTP_RULE_GRAPHICS_VERSION_ANY_GT - Create rule matching graphics version on any GT
 * @ver__: Graphics IP version to match
 *
 * Like XE_RTP_RULE_GRAPHICS_VERSION, but it matches even if the current GT
 * being checked is not of the graphics type. It allows to add RTP entries to
 * another GT when the device contains a Graphics IP with that version.
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_GRAPHICS_VERSION_ANY_GT(ver__)				\
	{ .match_type = XE_RTP_MATCH_GRAPHICS_VERSION_ANY_GT,			\
	  .ver_start = ver__, }

/**
 * XE_RTP_RULE_MEDIA_VERSION - Create rule matching media version
 * @ver__: Media IP version to match
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_MEDIA_VERSION(ver__)					\
	{ .match_type = XE_RTP_MATCH_MEDIA_VERSION,				\
	  .ver_start = ver__, }

/**
 * XE_RTP_RULE_MEDIA_VERSION_RANGE - Create rule matching a range of media version
 * @ver_start__: First media IP version to match
 * @ver_end__: Last media IP version to match
 *
 * Note that the range matching this rule is [ @ver_start__, @ver_end__ ], i.e.
 * inclusive on both sides
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_MEDIA_VERSION_RANGE(ver_start__, ver_end__)			\
	{ .match_type = XE_RTP_MATCH_MEDIA_VERSION_RANGE,			\
	  .ver_start = ver_start__, .ver_end = ver_end__, }

/**
 * XE_RTP_RULE_MEDIA_VERSION_ANY_GT - Create rule matching media version on any GT
 * @ver__: Media IP version to match
 *
 * Like XE_RTP_RULE_MEDIA_VERSION, but it matches even if the current GT being
 * checked is not of the media type. It allows to add RTP entries to another
 * GT when the device contains a Media IP with that version.
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_MEDIA_VERSION_ANY_GT(ver__)					\
	{ .match_type = XE_RTP_MATCH_MEDIA_VERSION_ANY_GT,			\
	  .ver_start = ver__, }

/**
 * XE_RTP_RULE_IS_INTEGRATED - Create a rule matching integrated graphics devices
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_IS_INTEGRATED						\
	{ .match_type = XE_RTP_MATCH_INTEGRATED }

/**
 * XE_RTP_RULE_IS_DISCRETE - Create a rule matching discrete graphics devices
 *
 * Refer to XE_RTP_RULES() for expected usage.
 */
#define XE_RTP_RULE_IS_DISCRETE							\
	{ .match_type = XE_RTP_MATCH_DISCRETE }

/**
 * XE_RTP_RULE_OR - Create an OR condition for rtp rules
 *
 * RTP rules are AND'ed when evaluated and all of them need to match.
 * XE_RTP_RULE_OR allows to create set of rules where any of them matching is
 * sufficient for the action to trigger. Example:
 *
 * .. code-block:: c
 *
 *	const struct xe_rtp_entry_sr entries[] = {
 *		...
 *		{ XE_RTP_NAME("test-entry"),
 *		  XE_RTP_RULES(PLATFORM(DG2), OR, PLATFORM(TIGERLAKE)),
 *		  ...
 *		},
 *		...
 *	};
 */
#define XE_RTP_RULE_OR								\
	{ .match_type = XE_RTP_MATCH_OR }

/**
 * XE_RTP_ACTION_WR - Helper to write a value to the register, overriding all
 *                    the bits
 * @reg_: Register
 * @val_: Value to set
 * @...: Additional fields to override in the struct xe_rtp_action entry
 *
 * The correspondent notation in bspec is:
 *
 *	REGNAME = VALUE
 */
#define XE_RTP_ACTION_WR(reg_, val_, ...)					\
	{ .reg = XE_RTP_DROP_CAST(reg_),					\
	  .clr_bits = ~0u, .set_bits = (val_),					\
	  .read_mask = (~0u), ##__VA_ARGS__ }

/**
 * XE_RTP_ACTION_SET - Set bits from @val_ in the register.
 * @reg_: Register
 * @val_: Bits to set in the register
 * @...: Additional fields to override in the struct xe_rtp_action entry
 *
 * For masked registers this translates to a single write, while for other
 * registers it's a RMW. The correspondent bspec notation is (example for bits 2
 * and 5, but could be any):
 *
 *	REGNAME[2] = 1
 *	REGNAME[5] = 1
 */
#define XE_RTP_ACTION_SET(reg_, val_, ...)					\
	{ .reg = XE_RTP_DROP_CAST(reg_),					\
	  .clr_bits = val_, .set_bits = val_,					\
	  .read_mask = val_, ##__VA_ARGS__ }

/**
 * XE_RTP_ACTION_CLR: Clear bits from @val_ in the register.
 * @reg_: Register
 * @val_: Bits to clear in the register
 * @...: Additional fields to override in the struct xe_rtp_action entry
 *
 * For masked registers this translates to a single write, while for other
 * registers it's a RMW. The correspondent bspec notation is (example for bits 2
 * and 5, but could be any):
 *
 *	REGNAME[2] = 0
 *	REGNAME[5] = 0
 */
#define XE_RTP_ACTION_CLR(reg_, val_, ...)					\
	{ .reg = XE_RTP_DROP_CAST(reg_),					\
	  .clr_bits = val_, .set_bits = 0,					\
	  .read_mask = val_, ##__VA_ARGS__ }

/**
 * XE_RTP_ACTION_FIELD_SET: Set a bit range
 * @reg_: Register
 * @mask_bits_: Mask of bits to be changed in the register, forming a field
 * @val_: Value to set in the field denoted by @mask_bits_
 * @...: Additional fields to override in the struct xe_rtp_action entry
 *
 * For masked registers this translates to a single write, while for other
 * registers it's a RMW. The correspondent bspec notation is:
 *
 *	REGNAME[<end>:<start>] = VALUE
 */
#define XE_RTP_ACTION_FIELD_SET(reg_, mask_bits_, val_, ...)			\
	{ .reg = XE_RTP_DROP_CAST(reg_),					\
	  .clr_bits = mask_bits_, .set_bits = val_,				\
	  .read_mask = mask_bits_, ##__VA_ARGS__ }

#define XE_RTP_ACTION_FIELD_SET_NO_READ_MASK(reg_, mask_bits_, val_, ...)	\
	{ .reg = XE_RTP_DROP_CAST(reg_),					\
	  .clr_bits = (mask_bits_), .set_bits = (val_),				\
	  .read_mask = 0, ##__VA_ARGS__ }

/**
 * XE_RTP_ACTION_WHITELIST - Add register to userspace whitelist
 * @reg_: Register
 * @val_: Whitelist-specific flags to set
 * @...: Additional fields to override in the struct xe_rtp_action entry
 *
 * Add a register to the whitelist, allowing userspace to modify the ster with
 * regular user privileges.
 */
#define XE_RTP_ACTION_WHITELIST(reg_, val_, ...)				\
	/* TODO fail build if ((flags) & ~(RING_FORCE_TO_NONPRIV_MASK_VALID)) */\
	{ .reg = XE_RTP_DROP_CAST(reg_),					\
	  .set_bits = val_,							\
	  .clr_bits = RING_FORCE_TO_NONPRIV_MASK_VALID,				\
	  ##__VA_ARGS__ }

/**
 * XE_RTP_NAME - Helper to set the name in xe_rtp_entry
 * @s_: Name describing this rule, often a HW-specific number
 *
 * TODO: maybe move this behind a debug config?
 */
#define XE_RTP_NAME(s_)	.name = (s_)

/**
 * XE_RTP_ENTRY_FLAG - Helper to add multiple flags to a struct xe_rtp_entry_sr
 * @...: Entry flags, without the ``XE_RTP_ENTRY_FLAG_`` prefix
 *
 * Helper to automatically add a ``XE_RTP_ENTRY_FLAG_`` prefix to the flags
 * when defining struct xe_rtp_entry entries. Example:
 *
 * .. code-block:: c
 *
 *	const struct xe_rtp_entry_sr wa_entries[] = {
 *		...
 *		{ XE_RTP_NAME("test-entry"),
 *		  ...
 *		  XE_RTP_ENTRY_FLAG(FOREACH_ENGINE),
 *		  ...
 *		},
 *		...
 *	};
 */
#define XE_RTP_ENTRY_FLAG(...)							\
	.flags = (XE_RTP_PASTE_FOREACH(ENTRY_FLAG_, BITWISE_OR, (__VA_ARGS__)))

/**
 * XE_RTP_ACTION_FLAG - Helper to add multiple flags to a struct xe_rtp_action
 * @...: Action flags, without the ``XE_RTP_ACTION_FLAG_`` prefix
 *
 * Helper to automatically add a ``XE_RTP_ACTION_FLAG_`` prefix to the flags
 * when defining struct xe_rtp_action entries. Example:
 *
 * .. code-block:: c
 *
 *	const struct xe_rtp_entry_sr wa_entries[] = {
 *		...
 *		{ XE_RTP_NAME("test-entry"),
 *		  ...
 *		  XE_RTP_ACTION_SET(..., XE_RTP_ACTION_FLAG(FOREACH_ENGINE)),
 *		  ...
 *		},
 *		...
 *	};
 */
#define XE_RTP_ACTION_FLAG(...)							\
	.flags = (XE_RTP_PASTE_FOREACH(ACTION_FLAG_, BITWISE_OR, (__VA_ARGS__)))

/**
 * XE_RTP_RULES - Helper to set multiple rules to a struct xe_rtp_entry_sr entry
 * @...: Rules
 *
 * When an RTP table is being processed, the rules of each entry are evaluated
 * to check if they match the target entity (platform, gt or hwe, depending on
 * the specific RTP table).
 *
 * The sequence of arguments of this macro must follow the following eBNF
 * grammar::
 *
 *            rules = disjunction;
 *      disjunction = conjunction, { "OR", conjunction };
 *      conjunction = single_rule, { single_rule };
 *				   (* the AND operator is implicit *)
 *      single_rule = ? GRAPHICS_VERSION(...), MEDIA_VERSION(...),
 *                      FUNC(...), etc ?
 *
 * Examples:
 *
 * .. code-block:: c
 *
 *	const struct xe_rtp_entry_sr wa_entries[] = {
 *		...
 *		{ XE_RTP_NAME("entry-a"),
 *		  // Match DG2-G10 with graphics steppings A0 up-to B0
 *		  // (exclusive).
 *		  XE_RTP_RULES(SUBPLATFORM(DG2, G10), GRAPHICS_STEP(A0, B0)),
 *		  ...
 *		},
 *		{ XE_RTP_NAME("entry-b"),
 *		  // Match graphics version 20 (all steppings) or graphics
 *		  // version 30 steppings A0 up-to B0 (exclusive).
 *		  XE_RTP_RULES(GRAPHICS_VERSION(2000), OR,
 *			       GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0))
 *		  ...
 *		},
 *		...
 *	};
 */
#define XE_RTP_RULES(...)							\
	.n_rules = COUNT_ARGS(__VA_ARGS__),					\
	.rules = (const struct xe_rtp_rule[]) {					\
		XE_RTP_PASTE_FOREACH(RULE_, COMMA, (__VA_ARGS__))	\
	}

/**
 * XE_RTP_ACTIONS - Helper to set multiple actions to a struct xe_rtp_entry_sr
 * @...: Actions to be taken
 *
 * At least one action is needed and up to 12 are supported. See XE_RTP_ACTION_*
 * for the possible actions. Example:
 *
 * .. code-block:: c
 *
 *	const struct xe_rtp_entry_sr wa_entries[] = {
 *		...
 *		{ XE_RTP_NAME("test-entry"),
 *		  XE_RTP_RULES(...),
 *		  XE_RTP_ACTIONS(SET(..), SET(...), CLR(...)),
 *		  ...
 *		},
 *		...
 *	};
 */
#define XE_RTP_ACTIONS(...)							\
	.n_actions = COUNT_ARGS(__VA_ARGS__),					\
	.actions = (const struct xe_rtp_action[]) {				\
		XE_RTP_PASTE_FOREACH(ACTION_, COMMA, (__VA_ARGS__))	\
	}

/*
 * Note: ARRAY_SIZE() cannot be used here because it expands through
 * __must_be_array() -> __BUILD_BUG_ON_ZERO_MSG() -> _Static_assert inside
 * sizeof(struct{}), which clang < 21 rejects when the compound literal
 * contains non-compile-time-constant initializers.
 */
#define XE_RTP_TABLE_SR(...) { \
	.entries = (const struct xe_rtp_entry_sr[]){__VA_ARGS__}, \
	.n_entries = sizeof((const struct xe_rtp_entry_sr[]){__VA_ARGS__}) / \
		sizeof(struct xe_rtp_entry_sr), \
}

#define XE_RTP_TABLE(...) { \
	.entries = (const struct xe_rtp_entry[]){__VA_ARGS__}, \
	.n_entries = sizeof((const struct xe_rtp_entry[]){__VA_ARGS__}) / \
		sizeof(struct xe_rtp_entry), \
}

#define XE_RTP_PROCESS_CTX_INITIALIZER(arg__) _Generic((arg__),							\
	struct xe_hw_engine * :	(struct xe_rtp_process_ctx){ { (void *)(arg__) }, XE_RTP_PROCESS_TYPE_ENGINE },	\
	struct xe_gt * :	(struct xe_rtp_process_ctx){ { (void *)(arg__) }, XE_RTP_PROCESS_TYPE_GT },	\
	struct xe_device * :	(struct xe_rtp_process_ctx){ { (void *)(arg__) }, XE_RTP_PROCESS_TYPE_DEVICE })

void xe_rtp_process_ctx_enable_active_tracking(struct xe_rtp_process_ctx *ctx,
					       unsigned long *active_entries,
					       size_t n_entries);

void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx,
			  const struct xe_rtp_table_sr *table,
			  struct xe_reg_sr *sr,
			  bool process_in_vf);

void xe_rtp_process(struct xe_rtp_process_ctx *ctx,
		    const struct xe_rtp_table *table);

/* Match functions to be used with XE_RTP_MATCH_FUNC */

/**
 * xe_rtp_match_always - Match RTP entry unconditionally
 * @xe: Device structure
 * @gt: GT structure
 * @hwe: Engine instance
 *
 * Returns: true, regardless of inputs
 */
bool xe_rtp_match_always(const struct xe_device *xe,
			 const struct xe_gt *gt,
			 const struct xe_hw_engine *hwe);

/**
 * xe_rtp_match_even_instance - Match if engine instance is even
 * @xe: Device structure
 * @gt: GT structure
 * @hwe: Engine instance
 *
 * Returns: true if engine instance is even, false otherwise
 */
bool xe_rtp_match_even_instance(const struct xe_device *xe,
				const struct xe_gt *gt,
				const struct xe_hw_engine *hwe);

/*
 * xe_rtp_match_first_render_or_compute - Match if it's first render or compute
 * engine in the GT
 *
 * @xe: Device structure
 * @gt: GT structure
 * @hwe: Engine instance
 *
 * Registers on the render reset domain need to have their values re-applied
 * when any of those engines are reset. Since the engines reset together, a
 * programming can be set to just one of them. For simplicity the first engine
 * of either render or compute class can be chosen.
 *
 * Returns: true if engine id is the first to match the render reset domain,
 * false otherwise.
 */
bool xe_rtp_match_first_render_or_compute(const struct xe_device *xe,
					  const struct xe_gt *gt,
					  const struct xe_hw_engine *hwe);

/*
 * xe_rtp_match_not_sriov_vf - Match when not on SR-IOV VF device
 *
 * @xe: Device structure
 * @gt: GT structure
 * @hwe: Engine instance
 *
 * Returns: true if device is not VF, false otherwise.
 */
bool xe_rtp_match_not_sriov_vf(const struct xe_device *xe,
			       const struct xe_gt *gt,
			       const struct xe_hw_engine *hwe);

bool xe_rtp_match_psmi_enabled(const struct xe_device *xe,
			       const struct xe_gt *gt,
			       const struct xe_hw_engine *hwe);

bool xe_rtp_match_gt_has_discontiguous_dss_groups(const struct xe_device *xe,
						  const struct xe_gt *gt,
						  const struct xe_hw_engine *hwe);

/**
 * xe_rtp_match_has_flat_ccs - Match when platform has FlatCCS compression
 * @xe: Device structure
 * @gt: GT structure
 * @hwe: Engine instance
 *
 * Returns: true if platform has FlatCCS compression, false otherwise
 */
bool xe_rtp_match_has_flat_ccs(const struct xe_device *xe,
			       const struct xe_gt *gt,
			       const struct xe_hw_engine *hwe);

/**
 * xe_rtp_match_has_msix - Match when platform has MSI-X
 * @xe: Device structure
 * @gt: GT structure
 * @hwe: Engine instance
 *
 * Returns: true if platform has MSI-X interrupt support
 */
bool xe_rtp_match_has_msix(const struct xe_device *xe,
			   const struct xe_gt *gt,
			   const struct xe_hw_engine *hwe);

#endif