| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Matthew Brost | 9776 | 63.16% | 61 | 34.46% |
| Niranjana Vishwanathapura | 2178 | 14.07% | 14 | 7.91% |
| Francois Dugast | 862 | 5.57% | 6 | 3.39% |
| Rodrigo Vivi | 734 | 4.74% | 7 | 3.95% |
| Daniele Ceraolo Spurio | 454 | 2.93% | 16 | 9.04% |
| Tomasz Lis | 340 | 2.20% | 8 | 4.52% |
| Michal Wajdeczko | 261 | 1.69% | 12 | 6.78% |
| Matthew Auld | 143 | 0.92% | 8 | 4.52% |
| Zhanjun Dong | 103 | 0.67% | 4 | 2.26% |
| Raag Jadav | 87 | 0.56% | 2 | 1.13% |
| Maarten Lankhorst | 83 | 0.54% | 2 | 1.13% |
| K V P, Satyanarayana | 70 | 0.45% | 2 | 1.13% |
| José Roberto de Souza | 61 | 0.39% | 2 | 1.13% |
| Tejas Upadhyay | 52 | 0.34% | 4 | 2.26% |
| Nirmoy Das | 46 | 0.30% | 2 | 1.13% |
| John Harrison | 44 | 0.28% | 4 | 2.26% |
| Matt Roper | 43 | 0.28% | 2 | 1.13% |
| Jonathan Cavitt | 31 | 0.20% | 2 | 1.13% |
| Lucas De Marchi | 30 | 0.19% | 3 | 1.69% |
| Bommu Krishnaiah | 15 | 0.10% | 1 | 0.56% |
| Brian Welty | 12 | 0.08% | 2 | 1.13% |
| Himal Prasad Ghimiray | 12 | 0.08% | 2 | 1.13% |
| Shuicheng Lin | 12 | 0.08% | 1 | 0.56% |
| Kees Cook | 11 | 0.07% | 1 | 0.56% |
| Tvrtko A. Ursulin | 9 | 0.06% | 2 | 1.13% |
| Umesh Nerlige Ramappa | 3 | 0.02% | 1 | 0.56% |
| Maíra Canal | 2 | 0.01% | 1 | 0.56% |
| Thomas Hellstrom | 1 | 0.01% | 1 | 0.56% |
| Sanjay Yadav | 1 | 0.01% | 1 | 0.56% |
| Colin Ian King | 1 | 0.01% | 1 | 0.56% |
| Nitin Gote | 1 | 0.01% | 1 | 0.56% |
| Lukasz Laguna | 1 | 0.01% | 1 | 0.56% |
| Total | 15479 | 177 |
// SPDX-License-Identifier: MIT /* * Copyright © 2022 Intel Corporation */ #include "xe_guc_submit.h" #include <linux/bitfield.h> #include <linux/bitmap.h> #include <linux/circ_buf.h> #include <linux/delay.h> #include <linux/dma-fence-array.h> #include <linux/math64.h> #include <drm/drm_managed.h> #include "abi/guc_actions_abi.h" #include "abi/guc_actions_slpc_abi.h" #include "abi/guc_klvs_abi.h" #include "xe_assert.h" #include "xe_bo.h" #include "xe_devcoredump.h" #include "xe_device.h" #include "xe_exec_queue.h" #include "xe_force_wake.h" #include "xe_gpu_scheduler.h" #include "xe_gt.h" #include "xe_gt_clock.h" #include "xe_gt_printk.h" #include "xe_guc.h" #include "xe_guc_capture.h" #include "xe_guc_ct.h" #include "xe_guc_exec_queue_types.h" #include "xe_guc_id_mgr.h" #include "xe_guc_klv_helpers.h" #include "xe_guc_submit_types.h" #include "xe_hw_engine.h" #include "xe_lrc.h" #include "xe_macros.h" #include "xe_map.h" #include "xe_mocs.h" #include "xe_pm.h" #include "xe_ring_ops_types.h" #include "xe_sched_job.h" #include "xe_trace.h" #include "xe_uc_fw.h" #include "xe_vm.h" #define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN 6 static int guc_submit_reset_prepare(struct xe_guc *guc); static struct xe_guc * exec_queue_to_guc(struct xe_exec_queue *q) { return &q->gt->uc.guc; } /* * Helpers for engine state, using an atomic as some of the bits can transition * as the same time (e.g. a suspend can be happning at the same time as schedule * engine done being processed). */ #define EXEC_QUEUE_STATE_REGISTERED (1 << 0) #define EXEC_QUEUE_STATE_ENABLED (1 << 1) #define EXEC_QUEUE_STATE_PENDING_ENABLE (1 << 2) #define EXEC_QUEUE_STATE_PENDING_DISABLE (1 << 3) #define EXEC_QUEUE_STATE_DESTROYED (1 << 4) #define EXEC_QUEUE_STATE_SUSPENDED (1 << 5) #define EXEC_QUEUE_STATE_RESET (1 << 6) #define EXEC_QUEUE_STATE_KILLED (1 << 7) #define EXEC_QUEUE_STATE_WEDGED (1 << 8) #define EXEC_QUEUE_STATE_BANNED (1 << 9) #define EXEC_QUEUE_STATE_PENDING_RESUME (1 << 10) #define EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND (1 << 11) static bool exec_queue_registered(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED; } static void set_exec_queue_registered(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); } static void clear_exec_queue_registered(struct xe_exec_queue *q) { atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); } static bool exec_queue_enabled(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_ENABLED; } static void set_exec_queue_enabled(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_ENABLED, &q->guc->state); } static void clear_exec_queue_enabled(struct xe_exec_queue *q) { atomic_and(~EXEC_QUEUE_STATE_ENABLED, &q->guc->state); } static bool exec_queue_pending_enable(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE; } static void set_exec_queue_pending_enable(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); } static void clear_exec_queue_pending_enable(struct xe_exec_queue *q) { atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); } static bool exec_queue_pending_disable(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE; } static void set_exec_queue_pending_disable(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); } static void clear_exec_queue_pending_disable(struct xe_exec_queue *q) { atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); } static bool exec_queue_destroyed(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED; } static void set_exec_queue_destroyed(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); } static void clear_exec_queue_destroyed(struct xe_exec_queue *q) { atomic_and(~EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); } static bool exec_queue_banned(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_BANNED; } static void set_exec_queue_banned(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state); } static bool exec_queue_suspended(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED; } static void set_exec_queue_suspended(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); } static void clear_exec_queue_suspended(struct xe_exec_queue *q) { atomic_and(~EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); } static bool exec_queue_reset(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET; } static void set_exec_queue_reset(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state); } static bool exec_queue_killed(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_KILLED; } static void set_exec_queue_killed(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_KILLED, &q->guc->state); } static bool exec_queue_wedged(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_WEDGED; } static void set_exec_queue_wedged(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_WEDGED, &q->guc->state); } static bool exec_queue_pending_resume(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_RESUME; } static void set_exec_queue_pending_resume(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state); } static void clear_exec_queue_pending_resume(struct xe_exec_queue *q) { atomic_and(~EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state); } static bool exec_queue_idle_skip_suspend(struct xe_exec_queue *q) { return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND; } static void set_exec_queue_idle_skip_suspend(struct xe_exec_queue *q) { atomic_or(EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state); } static void clear_exec_queue_idle_skip_suspend(struct xe_exec_queue *q) { atomic_and(~EXEC_QUEUE_STATE_IDLE_SKIP_SUSPEND, &q->guc->state); } static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) { return (atomic_read(&q->guc->state) & (EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_BANNED)); } static void guc_submit_sw_fini(struct drm_device *drm, void *arg) { struct xe_guc *guc = arg; struct xe_device *xe = guc_to_xe(guc); struct xe_gt *gt = guc_to_gt(guc); int ret; ret = wait_event_timeout(guc->submission_state.fini_wq, xa_empty(&guc->submission_state.exec_queue_lookup), HZ * 5); drain_workqueue(xe->destroy_wq); xe_gt_assert(gt, ret); xa_destroy(&guc->submission_state.exec_queue_lookup); } static void guc_submit_fini(void *arg) { struct xe_guc *guc = arg; /* Forcefully kill any remaining exec queues */ xe_guc_ct_stop(&guc->ct); guc_submit_reset_prepare(guc); xe_guc_softreset(guc); xe_guc_submit_stop(guc); xe_uc_fw_sanitize(&guc->fw); xe_guc_submit_pause_abort(guc); } static void guc_submit_wedged_fini(void *arg) { struct xe_guc *guc = arg; struct xe_exec_queue *q; unsigned long index; mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { if (exec_queue_wedged(q)) { mutex_unlock(&guc->submission_state.lock); xe_exec_queue_put(q); mutex_lock(&guc->submission_state.lock); } } mutex_unlock(&guc->submission_state.lock); } static const struct xe_exec_queue_ops guc_exec_queue_ops; static void primelockdep(struct xe_guc *guc) { if (!IS_ENABLED(CONFIG_LOCKDEP)) return; fs_reclaim_acquire(GFP_KERNEL); mutex_lock(&guc->submission_state.lock); mutex_unlock(&guc->submission_state.lock); fs_reclaim_release(GFP_KERNEL); } /** * xe_guc_submit_init() - Initialize GuC submission. * @guc: the &xe_guc to initialize * @num_ids: number of GuC context IDs to use * * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all * GuC context IDs supported by the GuC firmware should be used for submission. * * Only VF drivers will have to provide explicit number of GuC context IDs * that they can use for submission. * * Return: 0 on success or a negative error code on failure. */ int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) { struct xe_device *xe = guc_to_xe(guc); struct xe_gt *gt = guc_to_gt(guc); int err; err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock); if (err) return err; err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids); if (err) return err; gt->exec_queue_ops = &guc_exec_queue_ops; xa_init(&guc->submission_state.exec_queue_lookup); init_waitqueue_head(&guc->submission_state.fini_wq); primelockdep(guc); guc->submission_state.initialized = true; err = drmm_add_action_or_reset(&xe->drm, guc_submit_sw_fini, guc); if (err) return err; return devm_add_action_or_reset(xe->drm.dev, guc_submit_fini, guc); } /* * Given that we want to guarantee enough RCS throughput to avoid missing * frames, we set the yield policy to 20% of each 80ms interval. */ #define RC_YIELD_DURATION 80 /* in ms */ #define RC_YIELD_RATIO 20 /* in percent */ static u32 *emit_render_compute_yield_klv(u32 *emit) { *emit++ = PREP_GUC_KLV_TAG(SCHEDULING_POLICIES_RENDER_COMPUTE_YIELD); *emit++ = RC_YIELD_DURATION; *emit++ = RC_YIELD_RATIO; return emit; } #define SCHEDULING_POLICY_MAX_DWORDS 16 static int guc_init_global_schedule_policy(struct xe_guc *guc) { u32 data[SCHEDULING_POLICY_MAX_DWORDS]; u32 *emit = data; u32 count = 0; int ret; if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 1, 0)) return 0; *emit++ = XE_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV; if (CCS_INSTANCES(guc_to_gt(guc))) emit = emit_render_compute_yield_klv(emit); count = emit - data; if (count > 1) { xe_assert(guc_to_xe(guc), count <= SCHEDULING_POLICY_MAX_DWORDS); ret = xe_guc_ct_send_block(&guc->ct, data, count); if (ret < 0) { xe_gt_err(guc_to_gt(guc), "failed to enable GuC scheduling policies: %pe\n", ERR_PTR(ret)); return ret; } } return 0; } int xe_guc_submit_enable(struct xe_guc *guc) { int ret; ret = guc_init_global_schedule_policy(guc); if (ret) return ret; guc->submission_state.enabled = true; return 0; } void xe_guc_submit_disable(struct xe_guc *guc) { guc->submission_state.enabled = false; } static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count) { int i; lockdep_assert_held(&guc->submission_state.lock); for (i = 0; i < xa_count; ++i) xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i); xe_guc_id_mgr_release_locked(&guc->submission_state.idm, q->guc->id, q->width); if (xa_empty(&guc->submission_state.exec_queue_lookup)) wake_up(&guc->submission_state.fini_wq); } static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) { int ret; int i; /* * Must use GFP_NOWAIT as this lock is in the dma fence signalling path, * worse case user gets -ENOMEM on engine create and has to try again. * * FIXME: Have caller pre-alloc or post-alloc /w GFP_KERNEL to prevent * failure. */ lockdep_assert_held(&guc->submission_state.lock); ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm, q->width); if (ret < 0) return ret; q->guc->id = ret; for (i = 0; i < q->width; ++i) { ret = xa_err(xa_store(&guc->submission_state.exec_queue_lookup, q->guc->id + i, q, GFP_NOWAIT)); if (ret) goto err_release; } return 0; err_release: __release_guc_id(guc, q, i); return ret; } static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) { mutex_lock(&guc->submission_state.lock); __release_guc_id(guc, q, q->width); mutex_unlock(&guc->submission_state.lock); } struct exec_queue_policy { u32 count; struct guc_update_exec_queue_policy h2g; }; static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy) { size_t bytes = sizeof(policy->h2g.header) + (sizeof(policy->h2g.klv[0]) * policy->count); return bytes / sizeof(u32); } static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy, u16 guc_id) { policy->h2g.header.action = XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES; policy->h2g.header.guc_id = guc_id; policy->count = 0; } #define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \ static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \ u32 data) \ { \ XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \ \ policy->h2g.klv[policy->count].kl = \ FIELD_PREP(GUC_KLV_0_KEY, \ GUC_CONTEXT_POLICIES_KLV_ID_##id) | \ FIELD_PREP(GUC_KLV_0_LEN, 1); \ policy->h2g.klv[policy->count].value = data; \ policy->count++; \ } MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY) MAKE_EXEC_QUEUE_POLICY_ADD(slpc_exec_queue_freq_req, SLPM_GT_FREQUENCY) #undef MAKE_EXEC_QUEUE_POLICY_ADD static const int xe_exec_queue_prio_to_guc[] = { [XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL, [XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL, [XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH, [XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH, }; static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q) { struct exec_queue_policy policy; enum xe_exec_queue_priority prio = q->sched_props.priority; u32 timeslice_us = q->sched_props.timeslice_us; u32 slpc_exec_queue_freq_req = 0; u32 preempt_timeout_us = q->sched_props.preempt_timeout_us; xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q) && !xe_exec_queue_is_multi_queue_secondary(q)); if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY) slpc_exec_queue_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE; __guc_exec_queue_policy_start_klv(&policy, q->guc->id); __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]); __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us); __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us); __guc_exec_queue_policy_add_slpc_exec_queue_freq_req(&policy, slpc_exec_queue_freq_req); xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, __guc_exec_queue_policy_action_size(&policy), 0, 0); } static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q) { struct exec_queue_policy policy; xe_assert(guc_to_xe(guc), !xe_exec_queue_is_multi_queue_secondary(q)); __guc_exec_queue_policy_start_klv(&policy, q->guc->id); __guc_exec_queue_policy_add_preemption_timeout(&policy, 1); xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, __guc_exec_queue_policy_action_size(&policy), 0, 0); } static bool vf_recovery(struct xe_guc *guc) { return xe_gt_recovery_pending(guc_to_gt(guc)); } static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */ wake_up_all(&xe->ufence_wq); xe_sched_tdr_queue_imm(&q->guc->sched); } static void xe_guc_exec_queue_group_trigger_cleanup(struct xe_exec_queue *q) { struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); struct xe_exec_queue_group *group = q->multi_queue.group; struct xe_exec_queue *eq; xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q)); /* Group banned, skip timeout check in TDR */ WRITE_ONCE(group->banned, true); xe_guc_exec_queue_trigger_cleanup(primary); mutex_lock(&group->list_lock); list_for_each_entry(eq, &group->list, multi_queue.link) xe_guc_exec_queue_trigger_cleanup(eq); mutex_unlock(&group->list_lock); } static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q) { if (xe_exec_queue_is_multi_queue(q)) { struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); struct xe_exec_queue_group *group = q->multi_queue.group; struct xe_exec_queue *eq; /* Group banned, skip timeout check in TDR */ WRITE_ONCE(group->banned, true); set_exec_queue_reset(primary); if (!exec_queue_banned(primary)) xe_guc_exec_queue_trigger_cleanup(primary); mutex_lock(&group->list_lock); list_for_each_entry(eq, &group->list, multi_queue.link) { set_exec_queue_reset(eq); if (!exec_queue_banned(eq)) xe_guc_exec_queue_trigger_cleanup(eq); } mutex_unlock(&group->list_lock); } else { set_exec_queue_reset(q); if (!exec_queue_banned(q)) xe_guc_exec_queue_trigger_cleanup(q); } } static void set_exec_queue_group_banned(struct xe_exec_queue *q) { struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); struct xe_exec_queue_group *group = q->multi_queue.group; struct xe_exec_queue *eq; /* Ban all queues of the multi-queue group */ xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q)); set_exec_queue_banned(primary); mutex_lock(&group->list_lock); list_for_each_entry(eq, &group->list, multi_queue.link) set_exec_queue_banned(eq); mutex_unlock(&group->list_lock); } /* Helper for context registration H2G */ struct guc_ctxt_registration_info { u32 flags; u32 context_idx; u32 engine_class; u32 engine_submit_mask; u32 wq_desc_lo; u32 wq_desc_hi; u32 wq_base_lo; u32 wq_base_hi; u32 wq_size; u32 cgp_lo; u32 cgp_hi; u32 hwlrca_lo; u32 hwlrca_hi; }; #define parallel_read(xe_, map_, field_) \ xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ field_) #define parallel_write(xe_, map_, field_, val_) \ xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ field_, val_) /** * DOC: Multi Queue Group GuC interface * * The multi queue group coordination between KMD and GuC is through a software * construct called Context Group Page (CGP). The CGP is a KMD managed 4KB page * allocated in the global GTT. * * CGP format: * * +-----------+---------------------------+---------------------------------------------+ * | DWORD | Name | Description | * +-----------+---------------------------+---------------------------------------------+ * | 0 | Version | Bits [15:8]=Major ver, [7:0]=Minor ver | * +-----------+---------------------------+---------------------------------------------+ * | 1..15 | RESERVED | MBZ | * +-----------+---------------------------+---------------------------------------------+ * | 16 | KMD_QUEUE_UPDATE_MASK_DW0 | KMD queue mask for queues 31..0 | * +-----------+---------------------------+---------------------------------------------+ * | 17 | KMD_QUEUE_UPDATE_MASK_DW1 | KMD queue mask for queues 63..32 | * +-----------+---------------------------+---------------------------------------------+ * | 18..31 | RESERVED | MBZ | * +-----------+---------------------------+---------------------------------------------+ * | 32 | Q0CD_DW0 | Queue 0 context LRC descriptor lower DWORD | * +-----------+---------------------------+---------------------------------------------+ * | 33 | Q0ContextIndex | Context ID for Queue 0 | * +-----------+---------------------------+---------------------------------------------+ * | 34 | Q1CD_DW0 | Queue 1 context LRC descriptor lower DWORD | * +-----------+---------------------------+---------------------------------------------+ * | 35 | Q1ContextIndex | Context ID for Queue 1 | * +-----------+---------------------------+---------------------------------------------+ * | ... |... | ... | * +-----------+---------------------------+---------------------------------------------+ * | 158 | Q63CD_DW0 | Queue 63 context LRC descriptor lower DWORD | * +-----------+---------------------------+---------------------------------------------+ * | 159 | Q63ContextIndex | Context ID for Queue 63 | * +-----------+---------------------------+---------------------------------------------+ * | 160..1024 | RESERVED | MBZ | * +-----------+---------------------------+---------------------------------------------+ * * While registering Q0 with GuC, CGP is updated with Q0 entry and GuC is notified * through XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE H2G message which specifies * the CGP address. When the secondary queues are added to the group, the CGP is * updated with entry for that queue and GuC is notified through the H2G interface * XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC. GuC responds to these H2G messages * with a XE_GUC_ACTION_NOTIFY_MULTIQ_CONTEXT_CGP_SYNC_DONE G2H message. GuC also * sends a XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR notification for any * error in the CGP. Only one of these CGP update messages can be outstanding * (waiting for GuC response) at any time. The bits in KMD_QUEUE_UPDATE_MASK_DW* * fields indicate which queue entry is being updated in the CGP. * * The primary queue (Q0) represents the multi queue group context in GuC and * submission on any queue of the group must be through Q0 GuC interface only. * * As it is not required to register secondary queues with GuC, the secondary queue * context ids in the CGP are populated with Q0 context id. */ #define CGP_VERSION_MAJOR_SHIFT 8 static void xe_guc_exec_queue_group_cgp_update(struct xe_device *xe, struct xe_exec_queue *q) { struct xe_exec_queue_group *group = q->multi_queue.group; u32 guc_id = group->primary->guc->id; /* Currently implementing CGP version 1.0 */ xe_map_wr(xe, &group->cgp_bo->vmap, 0, u32, 1 << CGP_VERSION_MAJOR_SHIFT); xe_map_wr(xe, &group->cgp_bo->vmap, (32 + q->multi_queue.pos * 2) * sizeof(u32), u32, lower_32_bits(xe_lrc_descriptor(q->lrc[0]))); xe_map_wr(xe, &group->cgp_bo->vmap, (33 + q->multi_queue.pos * 2) * sizeof(u32), u32, guc_id); if (q->multi_queue.pos / 32) { xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), u32, BIT(q->multi_queue.pos % 32)); xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), u32, 0); } else { xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), u32, BIT(q->multi_queue.pos)); xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), u32, 0); } } static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc, struct xe_exec_queue *q, const u32 *action, u32 len) { struct xe_exec_queue_group *group = q->multi_queue.group; struct xe_device *xe = guc_to_xe(guc); long ret; /* * As all queues of a multi queue group use single drm scheduler * submit workqueue, CGP synchronization with GuC are serialized. * Hence, no locking is required here. * Wait for any pending CGP_SYNC_DONE response before updating the * CGP page and sending CGP_SYNC message. * * FIXME: Support VF migration */ ret = wait_event_timeout(guc->ct.wq, !READ_ONCE(group->sync_pending) || xe_guc_read_stopped(guc), HZ); if (!ret || xe_guc_read_stopped(guc)) { /* CGP_SYNC failed. Reset gt, cleanup the group */ xe_gt_warn(guc_to_gt(guc), "Wait for CGP_SYNC_DONE response failed!\n"); set_exec_queue_group_banned(q); xe_gt_reset_async(q->gt); xe_guc_exec_queue_group_trigger_cleanup(q); return; } xe_lrc_set_multi_queue_priority(q->lrc[0], q->multi_queue.priority); xe_guc_exec_queue_group_cgp_update(xe, q); WRITE_ONCE(group->sync_pending, true); xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1); } static void __register_exec_queue_group(struct xe_guc *guc, struct xe_exec_queue *q, struct guc_ctxt_registration_info *info) { #define MAX_MULTI_QUEUE_REG_SIZE (8) u32 action[MAX_MULTI_QUEUE_REG_SIZE]; int len = 0; action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE; action[len++] = info->flags; action[len++] = info->context_idx; action[len++] = info->engine_class; action[len++] = info->engine_submit_mask; action[len++] = 0; /* Reserved */ action[len++] = info->cgp_lo; action[len++] = info->cgp_hi; xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_REG_SIZE); #undef MAX_MULTI_QUEUE_REG_SIZE /* * The above XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE do expect a * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response * from guc. */ xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); } static void xe_guc_exec_queue_group_add(struct xe_guc *guc, struct xe_exec_queue *q) { #define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; int len = 0; xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_multi_queue_secondary(q)); action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; action[len++] = q->multi_queue.group->primary->guc->id; xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); #undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE /* * The above XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC do expect a * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response * from guc. */ xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); } static void __register_mlrc_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q, struct guc_ctxt_registration_info *info) { #define MAX_MLRC_REG_SIZE (13 + XE_HW_ENGINE_MAX_INSTANCE * 2) u32 action[MAX_MLRC_REG_SIZE]; int len = 0; int i; xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_parallel(q)); action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; action[len++] = info->flags; action[len++] = info->context_idx; action[len++] = info->engine_class; action[len++] = info->engine_submit_mask; action[len++] = info->wq_desc_lo; action[len++] = info->wq_desc_hi; action[len++] = info->wq_base_lo; action[len++] = info->wq_base_hi; action[len++] = info->wq_size; action[len++] = q->width; action[len++] = info->hwlrca_lo; action[len++] = info->hwlrca_hi; for (i = 1; i < q->width; ++i) { struct xe_lrc *lrc = q->lrc[i]; action[len++] = lower_32_bits(xe_lrc_descriptor(lrc)); action[len++] = upper_32_bits(xe_lrc_descriptor(lrc)); } /* explicitly checks some fields that we might fixup later */ xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo == action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_5_WQ_DESC_ADDR_LOWER]); xe_gt_assert(guc_to_gt(guc), info->wq_base_lo == action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_7_WQ_BUF_BASE_LOWER]); xe_gt_assert(guc_to_gt(guc), q->width == action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_10_NUM_CTXS]); xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo == action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_11_HW_LRC_ADDR]); xe_gt_assert(guc_to_gt(guc), len <= MAX_MLRC_REG_SIZE); #undef MAX_MLRC_REG_SIZE xe_guc_ct_send(&guc->ct, action, len, 0, 0); } static void __register_exec_queue(struct xe_guc *guc, struct guc_ctxt_registration_info *info) { u32 action[] = { XE_GUC_ACTION_REGISTER_CONTEXT, info->flags, info->context_idx, info->engine_class, info->engine_submit_mask, info->wq_desc_lo, info->wq_desc_hi, info->wq_base_lo, info->wq_base_hi, info->wq_size, info->hwlrca_lo, info->hwlrca_hi, }; /* explicitly checks some fields that we might fixup later */ xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo == action[XE_GUC_REGISTER_CONTEXT_DATA_5_WQ_DESC_ADDR_LOWER]); xe_gt_assert(guc_to_gt(guc), info->wq_base_lo == action[XE_GUC_REGISTER_CONTEXT_DATA_7_WQ_BUF_BASE_LOWER]); xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo == action[XE_GUC_REGISTER_CONTEXT_DATA_10_HW_LRC_ADDR]); xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); } static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); struct xe_lrc *lrc = q->lrc[0]; struct guc_ctxt_registration_info info; xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q)); xe_gt_assert(guc_to_gt(guc), ctx_type < GUC_CONTEXT_COUNT); memset(&info, 0, sizeof(info)); info.context_idx = q->guc->id; info.engine_class = xe_engine_class_to_guc_class(q->class); info.engine_submit_mask = q->logical_mask; info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc)); info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc)); info.flags = CONTEXT_REGISTRATION_FLAG_KMD | FIELD_PREP(CONTEXT_REGISTRATION_FLAG_TYPE, ctx_type); if (xe_exec_queue_is_multi_queue(q)) { struct xe_exec_queue_group *group = q->multi_queue.group; info.cgp_lo = xe_bo_ggtt_addr(group->cgp_bo); info.cgp_hi = 0; } if (xe_exec_queue_is_parallel(q)) { u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); struct iosys_map map = xe_lrc_parallel_map(lrc); info.wq_desc_lo = lower_32_bits(ggtt_addr + offsetof(struct guc_submit_parallel_scratch, wq_desc)); info.wq_desc_hi = upper_32_bits(ggtt_addr + offsetof(struct guc_submit_parallel_scratch, wq_desc)); info.wq_base_lo = lower_32_bits(ggtt_addr + offsetof(struct guc_submit_parallel_scratch, wq[0])); info.wq_base_hi = upper_32_bits(ggtt_addr + offsetof(struct guc_submit_parallel_scratch, wq[0])); info.wq_size = WQ_SIZE; q->guc->wqi_head = 0; q->guc->wqi_tail = 0; xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE); parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE); } set_exec_queue_registered(q); trace_xe_exec_queue_register(q); if (xe_exec_queue_is_multi_queue_primary(q)) __register_exec_queue_group(guc, q, &info); else if (xe_exec_queue_is_parallel(q)) __register_mlrc_exec_queue(guc, q, &info); else if (!xe_exec_queue_is_multi_queue_secondary(q)) __register_exec_queue(guc, &info); if (!xe_exec_queue_is_multi_queue_secondary(q)) init_policies(guc, q); if (xe_exec_queue_is_multi_queue_secondary(q)) xe_guc_exec_queue_group_add(guc, q); } static u32 wq_space_until_wrap(struct xe_exec_queue *q) { return (WQ_SIZE - q->guc->wqi_tail); } static inline void relaxed_ms_sleep(unsigned int delay_ms) { unsigned long min_us, max_us; if (!delay_ms) return; if (delay_ms > 20) { msleep(delay_ms); return; } min_us = mul_u32_u32(delay_ms, 1000); max_us = min_us + 500; usleep_range(min_us, max_us); } static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); unsigned int sleep_period_ms = 1, sleep_total_ms = 0; #define AVAILABLE_SPACE \ CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE) if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) { try_again: q->guc->wqi_head = parallel_read(xe, map, wq_desc.head); if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) { if (sleep_total_ms > 2000) { xe_gt_reset_async(q->gt); return -ENODEV; } msleep(sleep_period_ms); sleep_total_ms += sleep_period_ms; if (sleep_period_ms < 64) sleep_period_ms <<= 1; goto try_again; } } #undef AVAILABLE_SPACE return 0; } static int wq_noop_append(struct xe_exec_queue *q) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1; if (wq_wait_for_space(q, wq_space_until_wrap(q))) return -ENODEV; xe_gt_assert(guc_to_gt(guc), FIELD_FIT(WQ_LEN_MASK, len_dw)); parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)], FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | FIELD_PREP(WQ_LEN_MASK, len_dw)); q->guc->wqi_tail = 0; return 0; } static void wq_item_append(struct xe_exec_queue *q) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); #define WQ_HEADER_SIZE 4 /* Includes 1 LRC address too */ u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)]; u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32); u32 len_dw = (wqi_size / sizeof(u32)) - 1; int i = 0, j; if (wqi_size > wq_space_until_wrap(q)) { if (wq_noop_append(q)) return; } if (wq_wait_for_space(q, wqi_size)) return; wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | FIELD_PREP(WQ_LEN_MASK, len_dw); wqi[i++] = xe_lrc_descriptor(q->lrc[0]); wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) | FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64)); wqi[i++] = 0; for (j = 1; j < q->width; ++j) { struct xe_lrc *lrc = q->lrc[j]; wqi[i++] = lrc->ring.tail / sizeof(u64); } xe_gt_assert(guc_to_gt(guc), i == wqi_size / sizeof(u32)); iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch, wq[q->guc->wqi_tail / sizeof(u32)])); xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size); q->guc->wqi_tail += wqi_size; xe_gt_assert(guc_to_gt(guc), q->guc->wqi_tail <= WQ_SIZE); xe_device_wmb(xe); map = xe_lrc_parallel_map(q->lrc[0]); parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail); } #define RESUME_PENDING ~0x0ull static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_lrc *lrc = q->lrc[0]; u32 action[3]; u32 g2h_len = 0; u32 num_g2h = 0; int len = 0; bool extra_submit = false; xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); if (!job->restore_replay || job->last_replay) { if (xe_exec_queue_is_parallel(q)) wq_item_append(q); else if (!exec_queue_idle_skip_suspend(q)) xe_lrc_set_ring_tail(lrc, lrc->ring.tail); job->last_replay = false; } if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q)) return; /* * All queues in a multi-queue group will use the primary queue * of the group to interface with GuC. */ q = xe_exec_queue_multi_queue_primary(q); if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) { action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; action[len++] = q->guc->id; action[len++] = GUC_CONTEXT_ENABLE; g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; num_g2h = 1; if (xe_exec_queue_is_parallel(q)) extra_submit = true; q->guc->resume_time = RESUME_PENDING; set_exec_queue_pending_enable(q); set_exec_queue_enabled(q); trace_xe_exec_queue_scheduling_enable(q); } else { action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; action[len++] = q->guc->id; trace_xe_exec_queue_submit(q); } xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); if (extra_submit) { len = 0; action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; action[len++] = q->guc->id; trace_xe_exec_queue_submit(q); xe_guc_ct_send(&guc->ct, action, len, 0, 0); } } static struct dma_fence * guc_exec_queue_run_job(struct drm_sched_job *drm_job) { struct xe_sched_job *job = to_xe_sched_job(drm_job); struct xe_exec_queue *q = job->q; struct xe_guc *guc = exec_queue_to_guc(q); bool killed_or_banned_or_wedged = exec_queue_killed_or_banned_or_wedged(q); xe_gt_assert(guc_to_gt(guc), !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) || exec_queue_banned(q) || exec_queue_suspended(q)); trace_xe_sched_job_run(job); if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) { if (xe_exec_queue_is_multi_queue_secondary(q)) { struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); if (exec_queue_killed_or_banned_or_wedged(primary)) { killed_or_banned_or_wedged = true; goto run_job_out; } if (!exec_queue_registered(primary)) register_exec_queue(primary, GUC_CONTEXT_NORMAL); } if (!exec_queue_registered(q)) register_exec_queue(q, GUC_CONTEXT_NORMAL); if (!job->restore_replay) q->ring_ops->emit_job(job); submit_exec_queue(q, job); job->restore_replay = false; } run_job_out: return job->fence; } static void guc_exec_queue_free_job(struct drm_sched_job *drm_job) { struct xe_sched_job *job = to_xe_sched_job(drm_job); trace_xe_sched_job_free(job); xe_sched_job_put(job); } int xe_guc_read_stopped(struct xe_guc *guc) { return atomic_read(&guc->submission_state.stopped); } static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, u32 runnable_state); static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q); #define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \ u32 action[] = { \ XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \ q->guc->id, \ GUC_CONTEXT_##enable_disable, \ } static void disable_scheduling_deregister(struct xe_guc *guc, struct xe_exec_queue *q) { MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); int ret; if (!xe_exec_queue_is_multi_queue_secondary(q)) set_min_preemption_timeout(guc, q); smp_rmb(); ret = wait_event_timeout(guc->ct.wq, (!exec_queue_pending_enable(q) && !exec_queue_pending_disable(q)) || xe_guc_read_stopped(guc) || vf_recovery(guc), HZ * 5); if (!ret && !vf_recovery(guc)) { struct xe_gpu_scheduler *sched = &q->guc->sched; xe_gt_warn(q->gt, "Pending enable/disable failed to respond\n"); xe_sched_submission_start(sched); xe_gt_reset_async(q->gt); xe_sched_tdr_queue_imm(sched); return; } clear_exec_queue_enabled(q); set_exec_queue_pending_disable(q); set_exec_queue_destroyed(q); trace_xe_exec_queue_scheduling_disable(q); /* * Reserve space for both G2H here as the 2nd G2H is sent from a G2H * handler and we are not allowed to reserved G2H space in handlers. */ if (xe_exec_queue_is_multi_queue_secondary(q)) handle_multi_queue_secondary_sched_done(guc, q, 0); else xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), G2H_LEN_DW_SCHED_CONTEXT_MODE_SET + G2H_LEN_DW_DEREGISTER_CONTEXT, 2); } /** * xe_guc_submit_wedge() - Wedge GuC submission * @guc: the GuC object * * Save exec queue's registered with GuC state by taking a ref to each queue. * Register a DRMM handler to drop refs upon driver unload. */ void xe_guc_submit_wedge(struct xe_guc *guc) { struct xe_device *xe = guc_to_xe(guc); struct xe_gt *gt = guc_to_gt(guc); struct xe_exec_queue *q; unsigned long index; int err; xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); /* * If device is being wedged even before submission_state is * initialized, there's nothing to do here. */ if (!guc->submission_state.initialized) return; if (xe->wedged.mode == 2) { err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev, guc_submit_wedged_fini, guc); if (err) { xe_gt_err(gt, "Failed to register clean-up on wedged.mode=2; " "Although device is wedged.\n"); return; } mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) if (xe_exec_queue_get_unless_zero(q)) set_exec_queue_wedged(q); mutex_unlock(&guc->submission_state.lock); } else { /* Forcefully kill any remaining exec queues, signal fences */ guc_submit_reset_prepare(guc); xe_guc_submit_stop(guc); xe_guc_softreset(guc); xe_uc_fw_sanitize(&guc->fw); xe_guc_submit_pause_abort(guc); } } static bool guc_submit_hint_wedged(struct xe_guc *guc) { struct xe_device *xe = guc_to_xe(guc); if (xe->wedged.mode != XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) return false; if (xe_device_wedged(xe)) return true; xe_device_declare_wedged(xe); return true; } #define ADJUST_FIVE_PERCENT(__t) mul_u64_u32_div(__t, 105, 100) static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job) { struct xe_gt *gt = guc_to_gt(exec_queue_to_guc(q)); u32 ctx_timestamp, ctx_job_timestamp; u32 timeout_ms = q->sched_props.job_timeout_ms; u32 diff; u64 running_time_ms; if (!xe_sched_job_started(job)) { xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, not started", xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), q->guc->id); return xe_sched_invalidate_job(job, 2); } ctx_timestamp = lower_32_bits(xe_lrc_timestamp(q->lrc[0])); if (ctx_timestamp == job->sample_timestamp) { if (IS_SRIOV_VF(gt_to_xe(gt))) xe_gt_notice(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck", xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), q->guc->id); else xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck", xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), q->guc->id); return xe_sched_invalidate_job(job, 0); } job->sample_timestamp = ctx_timestamp; ctx_job_timestamp = xe_lrc_ctx_job_timestamp(q->lrc[0]); /* * Counter wraps at ~223s at the usual 19.2MHz, be paranoid catch * possible overflows with a high timeout. */ xe_gt_assert(gt, timeout_ms < 100 * MSEC_PER_SEC); diff = ctx_timestamp - ctx_job_timestamp; /* * Ensure timeout is within 5% to account for an GuC scheduling latency */ running_time_ms = ADJUST_FIVE_PERCENT(xe_gt_clock_interval_to_ms(gt, diff)); xe_gt_dbg(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, running_time_ms=%llu, timeout_ms=%u, diff=0x%08x", xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), q->guc->id, running_time_ms, timeout_ms, diff); return running_time_ms >= timeout_ms; } static void enable_scheduling(struct xe_exec_queue *q) { MAKE_SCHED_CONTEXT_ACTION(q, ENABLE); struct xe_guc *guc = exec_queue_to_guc(q); int ret; xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); set_exec_queue_pending_enable(q); set_exec_queue_enabled(q); trace_xe_exec_queue_scheduling_enable(q); if (xe_exec_queue_is_multi_queue_secondary(q)) handle_multi_queue_secondary_sched_done(guc, q, 1); else xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); ret = wait_event_timeout(guc->ct.wq, !exec_queue_pending_enable(q) || xe_guc_read_stopped(guc) || vf_recovery(guc), HZ * 5); if ((!ret && !vf_recovery(guc)) || xe_guc_read_stopped(guc)) { xe_gt_warn(guc_to_gt(guc), "Schedule enable failed to respond"); set_exec_queue_banned(q); xe_gt_reset_async(q->gt); xe_sched_tdr_queue_imm(&q->guc->sched); } } static void disable_scheduling(struct xe_exec_queue *q, bool immediate) { MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); struct xe_guc *guc = exec_queue_to_guc(q); xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); if (immediate && !xe_exec_queue_is_multi_queue_secondary(q)) set_min_preemption_timeout(guc, q); clear_exec_queue_enabled(q); set_exec_queue_pending_disable(q); trace_xe_exec_queue_scheduling_disable(q); if (xe_exec_queue_is_multi_queue_secondary(q)) handle_multi_queue_secondary_sched_done(guc, q, 0); else xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); } static enum drm_gpu_sched_stat guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) { struct xe_sched_job *job = to_xe_sched_job(drm_job); struct drm_sched_job *tmp_job; struct xe_exec_queue *q = job->q; struct xe_gpu_scheduler *sched = &q->guc->sched; struct xe_guc *guc = exec_queue_to_guc(q); const char *process_name = "no process"; struct xe_device *xe = guc_to_xe(guc); int err = -ETIME; pid_t pid = -1; bool wedged = false, skip_timeout_check; xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); /* * TDR has fired before free job worker. Common if exec queue * immediately closed after last fence signaled. Add back to pending * list so job can be freed and kick scheduler ensuring free job is not * lost. */ if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags) || vf_recovery(guc)) return DRM_GPU_SCHED_STAT_NO_HANG; /* Kill the run_job entry point */ xe_sched_submission_stop(sched); /* Must check all state after stopping scheduler */ skip_timeout_check = exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q); /* Skip timeout check if multi-queue group is banned */ if (xe_exec_queue_is_multi_queue(q) && READ_ONCE(q->multi_queue.group->banned)) skip_timeout_check = true; /* LR jobs can only get here if queue has been killed or hit an error */ if (xe_exec_queue_is_lr(q)) xe_gt_assert(guc_to_gt(guc), skip_timeout_check); /* * FIXME: In multi-queue scenario, the TDR must ensure that the whole * multi-queue group is off the HW before signaling the fences to avoid * possible memory corruptions. This means disabling scheduling on the * primary queue before or during the secondary queue's TDR. Need to * implement this in least obtrusive way. */ /* * If devcoredump not captured and GuC capture for the job is not ready * do manual capture first and decide later if we need to use it */ if (!exec_queue_killed(q) && !xe->devcoredump.captured && !xe_guc_capture_get_matching_and_lock(q)) { /* take force wake before engine register manual capture */ CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n"); xe_engine_snapshot_capture_for_queue(q); } /* * Check if job is actually timed out, if so restart job execution and TDR */ if (!skip_timeout_check && !check_timeout(q, job)) goto rearm; if (!exec_queue_killed(q)) wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); set_exec_queue_banned(q); /* Kick job / queue off hardware */ if (!wedged && (exec_queue_enabled(q) || exec_queue_pending_disable(q))) { int ret; if (exec_queue_reset(q)) err = -EIO; if (xe_uc_fw_is_running(&guc->fw)) { /* * Wait for any pending G2H to flush out before * modifying state */ ret = wait_event_timeout(guc->ct.wq, (!exec_queue_pending_enable(q) && !exec_queue_pending_disable(q)) || xe_guc_read_stopped(guc) || vf_recovery(guc), HZ * 5); if (vf_recovery(guc)) goto handle_vf_resume; if (!ret || xe_guc_read_stopped(guc)) goto trigger_reset; disable_scheduling(q, skip_timeout_check); } /* * Must wait for scheduling to be disabled before signalling * any fences, if GT broken the GT reset code should signal us. * * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault * error) messages which can cause the schedule disable to get * lost. If this occurs, trigger a GT reset to recover. */ smp_rmb(); ret = wait_event_timeout(guc->ct.wq, !xe_uc_fw_is_running(&guc->fw) || !exec_queue_pending_disable(q) || xe_guc_read_stopped(guc) || vf_recovery(guc), HZ * 5); if (vf_recovery(guc)) goto handle_vf_resume; if (!ret || xe_guc_read_stopped(guc)) { trigger_reset: if (!ret) xe_gt_warn(guc_to_gt(guc), "Schedule disable failed to respond, guc_id=%d", q->guc->id); xe_devcoredump(q, job, "Schedule disable failed to respond, guc_id=%d, ret=%d, guc_read=%d", q->guc->id, ret, xe_guc_read_stopped(guc)); xe_gt_reset_async(q->gt); xe_sched_tdr_queue_imm(sched); goto rearm; } } if (q->vm && q->vm->xef) { process_name = q->vm->xef->process_name; pid = q->vm->xef->pid; } if (!exec_queue_killed(q)) xe_gt_notice(guc_to_gt(guc), "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]", xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), q->guc->id, q->flags, process_name, pid); trace_xe_sched_job_timedout(job); if (!exec_queue_killed(q)) xe_devcoredump(q, job, "Timedout job - seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx", xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), q->guc->id, q->flags); /* * Kernel jobs should never fail, nor should VM jobs if they do * somethings has gone wrong and the GT needs a reset */ xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL, "Kernel-submitted job timed out\n"); xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q), "VM job timed out on non-killed execqueue\n"); if (!wedged && (q->flags & EXEC_QUEUE_FLAG_KERNEL || (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)))) { if (!xe_sched_invalidate_job(job, 2)) { xe_gt_reset_async(q->gt); goto rearm; } } /* Mark all outstanding jobs as bad, thus completing them */ xe_sched_job_set_error(job, err); drm_sched_for_each_pending_job(tmp_job, &sched->base, NULL) xe_sched_job_set_error(to_xe_sched_job(tmp_job), -ECANCELED); xe_sched_submission_start(sched); if (xe_exec_queue_is_multi_queue(q)) xe_guc_exec_queue_group_trigger_cleanup(q); else xe_guc_exec_queue_trigger_cleanup(q); /* * We want the job added back to the pending list so it gets freed; this * is what DRM_GPU_SCHED_STAT_NO_HANG does. */ return DRM_GPU_SCHED_STAT_NO_HANG; rearm: /* * XXX: Ideally want to adjust timeout based on current execution time * but there is not currently an easy way to do in DRM scheduler. With * some thought, do this in a follow up. */ xe_sched_submission_start(sched); handle_vf_resume: return DRM_GPU_SCHED_STAT_NO_HANG; } static void guc_exec_queue_fini(struct xe_exec_queue *q) { struct xe_guc_exec_queue *ge = q->guc; struct xe_guc *guc = exec_queue_to_guc(q); release_guc_id(guc, q); xe_sched_entity_fini(&ge->entity); xe_sched_fini(&ge->sched); /* * RCU free due sched being exported via DRM scheduler fences * (timeline name). */ kfree_rcu(ge, rcu); } static void __guc_exec_queue_destroy_async(struct work_struct *w) { struct xe_guc_exec_queue *ge = container_of(w, struct xe_guc_exec_queue, destroy_async); struct xe_exec_queue *q = ge->q; struct xe_guc *guc = exec_queue_to_guc(q); guard(xe_pm_runtime)(guc_to_xe(guc)); trace_xe_exec_queue_destroy(q); if (xe_exec_queue_is_multi_queue_secondary(q)) { struct xe_exec_queue_group *group = q->multi_queue.group; mutex_lock(&group->list_lock); list_del(&q->multi_queue.link); mutex_unlock(&group->list_lock); } /* Confirm no work left behind accessing device structures */ cancel_delayed_work_sync(&ge->sched.base.work_tdr); xe_exec_queue_fini(q); } static void guc_exec_queue_destroy_async(struct xe_exec_queue *q) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); INIT_WORK(&q->guc->destroy_async, __guc_exec_queue_destroy_async); /* We must block on kernel engines so slabs are empty on driver unload */ if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q)) __guc_exec_queue_destroy_async(&q->guc->destroy_async); else queue_work(xe->destroy_wq, &q->guc->destroy_async); } static void __guc_exec_queue_destroy(struct xe_guc *guc, struct xe_exec_queue *q) { /* * Might be done from within the GPU scheduler, need to do async as we * fini the scheduler when the engine is fini'd, the scheduler can't * complete fini within itself (circular dependency). Async resolves * this we and don't really care when everything is fini'd, just that it * is. */ guc_exec_queue_destroy_async(q); } static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg) { struct xe_exec_queue *q = msg->private_data; struct xe_guc *guc = exec_queue_to_guc(q); xe_gt_assert(guc_to_gt(guc), !(q->flags & EXEC_QUEUE_FLAG_PERMANENT)); trace_xe_exec_queue_cleanup_entity(q); /* * Expected state transitions for cleanup: * - If the exec queue is registered and GuC firmware is running, we must first * disable scheduling and deregister the queue to ensure proper teardown and * resource release in the GuC, then destroy the exec queue on driver side. * - If the GuC is already stopped (e.g., during driver unload or GPU reset), * we cannot expect a response for the deregister request. In this case, * it is safe to directly destroy the exec queue on driver side, as the GuC * will not process further requests and all resources must be cleaned up locally. */ if (exec_queue_registered(q) && xe_uc_fw_is_running(&guc->fw)) disable_scheduling_deregister(guc, q); else __guc_exec_queue_destroy(guc, q); } static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q) { return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q); } static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg) { struct xe_exec_queue *q = msg->private_data; struct xe_guc *guc = exec_queue_to_guc(q); if (guc_exec_queue_allowed_to_change_state(q)) init_policies(guc, q); kfree(msg); } static void __suspend_fence_signal(struct xe_exec_queue *q) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); if (!q->guc->suspend_pending) return; WRITE_ONCE(q->guc->suspend_pending, false); /* * We use a GuC shared wait queue for VFs because the VF resfix start * interrupt must be able to wake all instances of suspend_wait. This * prevents the VF migration worker from being starved during * scheduling. */ if (IS_SRIOV_VF(xe)) wake_up_all(&guc->ct.wq); else wake_up(&q->guc->suspend_wait); } static void suspend_fence_signal(struct xe_exec_queue *q) { struct xe_guc *guc = exec_queue_to_guc(q); xe_gt_assert(guc_to_gt(guc), exec_queue_suspended(q) || exec_queue_killed(q) || xe_guc_read_stopped(guc)); xe_gt_assert(guc_to_gt(guc), q->guc->suspend_pending); __suspend_fence_signal(q); } static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) { struct xe_exec_queue *q = msg->private_data; struct xe_guc *guc = exec_queue_to_guc(q); bool idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q); if (!idle_skip_suspend && guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) && exec_queue_enabled(q)) { wait_event(guc->ct.wq, vf_recovery(guc) || ((q->guc->resume_time != RESUME_PENDING || xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q))); if (!xe_guc_read_stopped(guc)) { s64 since_resume_ms = ktime_ms_delta(ktime_get(), q->guc->resume_time); s64 wait_ms = q->vm->preempt.min_run_period_ms - since_resume_ms; if (wait_ms > 0 && q->guc->resume_time) relaxed_ms_sleep(wait_ms); set_exec_queue_suspended(q); disable_scheduling(q, false); } } else if (q->guc->suspend_pending) { if (idle_skip_suspend) set_exec_queue_idle_skip_suspend(q); set_exec_queue_suspended(q); suspend_fence_signal(q); } } static void sched_context(struct xe_exec_queue *q) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_lrc *lrc = q->lrc[0]; u32 action[] = { XE_GUC_ACTION_SCHED_CONTEXT, q->guc->id, }; xe_gt_assert(guc_to_gt(guc), !xe_exec_queue_is_parallel(q)); xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); trace_xe_exec_queue_submit(q); xe_lrc_set_ring_tail(lrc, lrc->ring.tail); xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); } static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) { struct xe_exec_queue *q = msg->private_data; if (guc_exec_queue_allowed_to_change_state(q)) { clear_exec_queue_suspended(q); if (!exec_queue_enabled(q)) { if (exec_queue_idle_skip_suspend(q)) { struct xe_lrc *lrc = q->lrc[0]; clear_exec_queue_idle_skip_suspend(q); xe_lrc_set_ring_tail(lrc, lrc->ring.tail); } q->guc->resume_time = RESUME_PENDING; set_exec_queue_pending_resume(q); enable_scheduling(q); } else if (exec_queue_idle_skip_suspend(q)) { clear_exec_queue_idle_skip_suspend(q); sched_context(q); } } else { clear_exec_queue_suspended(q); clear_exec_queue_idle_skip_suspend(q); } } static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sched_msg *msg) { struct xe_exec_queue *q = msg->private_data; if (guc_exec_queue_allowed_to_change_state(q)) { #define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) struct xe_guc *guc = exec_queue_to_guc(q); struct xe_exec_queue_group *group = q->multi_queue.group; u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; int len = 0; action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; action[len++] = group->primary->guc->id; xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); #undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE xe_guc_exec_queue_group_cgp_sync(guc, q, action, len); } kfree(msg); } #define CLEANUP 1 /* Non-zero values to catch uninitialized msg */ #define SET_SCHED_PROPS 2 #define SUSPEND 3 #define RESUME 4 #define SET_MULTI_QUEUE_PRIORITY 5 #define OPCODE_MASK 0xf #define MSG_LOCKED BIT(8) #define MSG_HEAD BIT(9) static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) { struct xe_device *xe = guc_to_xe(exec_queue_to_guc(msg->private_data)); trace_xe_sched_msg_recv(msg); switch (msg->opcode) { case CLEANUP: __guc_exec_queue_process_msg_cleanup(msg); break; case SET_SCHED_PROPS: __guc_exec_queue_process_msg_set_sched_props(msg); break; case SUSPEND: __guc_exec_queue_process_msg_suspend(msg); break; case RESUME: __guc_exec_queue_process_msg_resume(msg); break; case SET_MULTI_QUEUE_PRIORITY: __guc_exec_queue_process_msg_set_multi_queue_priority(msg); break; default: XE_WARN_ON("Unknown message type"); } xe_pm_runtime_put(xe); } static const struct drm_sched_backend_ops drm_sched_ops = { .run_job = guc_exec_queue_run_job, .free_job = guc_exec_queue_free_job, .timedout_job = guc_exec_queue_timedout_job, }; static const struct xe_sched_backend_ops xe_sched_ops = { .process_msg = guc_exec_queue_process_msg, }; static int guc_exec_queue_init(struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched; struct xe_guc *guc = exec_queue_to_guc(q); struct workqueue_struct *submit_wq = NULL; struct xe_guc_exec_queue *ge; long timeout; int err, i; xe_gt_assert(guc_to_gt(guc), xe_device_uc_enabled(guc_to_xe(guc))); ge = kzalloc_obj(*ge); if (!ge) return -ENOMEM; q->guc = ge; ge->q = q; init_rcu_head(&ge->rcu); init_waitqueue_head(&ge->suspend_wait); for (i = 0; i < MAX_STATIC_MSG_TYPE; ++i) INIT_LIST_HEAD(&ge->static_msgs[i].link); timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT : msecs_to_jiffies(q->sched_props.job_timeout_ms); /* * Use primary queue's submit_wq for all secondary queues of a * multi queue group. This serialization avoids any locking around * CGP synchronization with GuC. */ if (xe_exec_queue_is_multi_queue_secondary(q)) { struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); submit_wq = primary->guc->sched.base.submit_wq; } err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, submit_wq, xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES, 64, timeout, guc_to_gt(guc)->ordered_wq, NULL, q->name, gt_to_xe(q->gt)->drm.dev); if (err) goto err_free; sched = &ge->sched; err = xe_sched_entity_init(&ge->entity, sched); if (err) goto err_sched; mutex_lock(&guc->submission_state.lock); err = alloc_guc_id(guc, q); if (err) goto err_entity; q->entity = &ge->entity; if (xe_guc_read_stopped(guc) || vf_recovery(guc)) xe_sched_stop(sched); mutex_unlock(&guc->submission_state.lock); xe_exec_queue_assign_name(q, q->guc->id); /* * Maintain secondary queues of the multi queue group in a list * for handling dependencies across the queues in the group. */ if (xe_exec_queue_is_multi_queue_secondary(q)) { struct xe_exec_queue_group *group = q->multi_queue.group; INIT_LIST_HEAD(&q->multi_queue.link); mutex_lock(&group->list_lock); list_add_tail(&q->multi_queue.link, &group->list); mutex_unlock(&group->list_lock); } if (xe_exec_queue_is_multi_queue(q)) trace_xe_exec_queue_create_multi_queue(q); else trace_xe_exec_queue_create(q); return 0; err_entity: mutex_unlock(&guc->submission_state.lock); xe_sched_entity_fini(&ge->entity); err_sched: xe_sched_fini(&ge->sched); err_free: kfree(ge); return err; } static void guc_exec_queue_kill(struct xe_exec_queue *q) { trace_xe_exec_queue_kill(q); set_exec_queue_killed(q); __suspend_fence_signal(q); xe_guc_exec_queue_trigger_cleanup(q); } static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg, u32 opcode) { xe_pm_runtime_get_noresume(guc_to_xe(exec_queue_to_guc(q))); INIT_LIST_HEAD(&msg->link); msg->opcode = opcode & OPCODE_MASK; msg->private_data = q; trace_xe_sched_msg_add(msg); if (opcode & MSG_HEAD) xe_sched_add_msg_head(&q->guc->sched, msg); else if (opcode & MSG_LOCKED) xe_sched_add_msg_locked(&q->guc->sched, msg); else xe_sched_add_msg(&q->guc->sched, msg); } static void guc_exec_queue_try_add_msg_head(struct xe_exec_queue *q, struct xe_sched_msg *msg, u32 opcode) { if (!list_empty(&msg->link)) return; guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED | MSG_HEAD); } static bool guc_exec_queue_try_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg, u32 opcode) { if (!list_empty(&msg->link)) return false; guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED); return true; } #define STATIC_MSG_CLEANUP 0 #define STATIC_MSG_SUSPEND 1 #define STATIC_MSG_RESUME 2 static void guc_exec_queue_destroy(struct xe_exec_queue *q) { struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && !exec_queue_wedged(q)) guc_exec_queue_add_msg(q, msg, CLEANUP); else __guc_exec_queue_destroy(exec_queue_to_guc(q), q); } static int guc_exec_queue_set_priority(struct xe_exec_queue *q, enum xe_exec_queue_priority priority) { struct xe_sched_msg *msg; if (q->sched_props.priority == priority || exec_queue_killed_or_banned_or_wedged(q)) return 0; msg = kmalloc_obj(*msg); if (!msg) return -ENOMEM; q->sched_props.priority = priority; guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); return 0; } static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) { struct xe_sched_msg *msg; if (q->sched_props.timeslice_us == timeslice_us || exec_queue_killed_or_banned_or_wedged(q)) return 0; msg = kmalloc_obj(*msg); if (!msg) return -ENOMEM; q->sched_props.timeslice_us = timeslice_us; guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); return 0; } static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, u32 preempt_timeout_us) { struct xe_sched_msg *msg; if (q->sched_props.preempt_timeout_us == preempt_timeout_us || exec_queue_killed_or_banned_or_wedged(q)) return 0; msg = kmalloc_obj(*msg); if (!msg) return -ENOMEM; q->sched_props.preempt_timeout_us = preempt_timeout_us; guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); return 0; } static int guc_exec_queue_set_multi_queue_priority(struct xe_exec_queue *q, enum xe_multi_queue_priority priority) { struct xe_sched_msg *msg; xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q)); if (q->multi_queue.priority == priority || exec_queue_killed_or_banned_or_wedged(q)) return 0; msg = kmalloc_obj(*msg); if (!msg) return -ENOMEM; q->multi_queue.priority = priority; guc_exec_queue_add_msg(q, msg, SET_MULTI_QUEUE_PRIORITY); return 0; } static int guc_exec_queue_suspend(struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND; if (exec_queue_killed_or_banned_or_wedged(q)) return -EINVAL; xe_sched_msg_lock(sched); if (guc_exec_queue_try_add_msg(q, msg, SUSPEND)) q->guc->suspend_pending = true; xe_sched_msg_unlock(sched); return 0; } static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); int ret; /* * Likely don't need to check exec_queue_killed() as we clear * suspend_pending upon kill but to be paranoid but races in which * suspend_pending is set after kill also check kill here. */ #define WAIT_COND \ (!READ_ONCE(q->guc->suspend_pending) || exec_queue_killed(q) || \ xe_guc_read_stopped(guc)) retry: if (IS_SRIOV_VF(xe)) ret = wait_event_interruptible_timeout(guc->ct.wq, WAIT_COND || vf_recovery(guc), HZ * 5); else ret = wait_event_interruptible_timeout(q->guc->suspend_wait, WAIT_COND, HZ * 5); if (vf_recovery(guc) && !xe_device_wedged((guc_to_xe(guc)))) return -EAGAIN; if (!ret) { xe_gt_warn(guc_to_gt(guc), "Suspend fence, guc_id=%d, failed to respond", q->guc->id); /* XXX: Trigger GT reset? */ return -ETIME; } else if (IS_SRIOV_VF(xe) && !WAIT_COND) { /* Corner case on RESFIX DONE where vf_recovery() changes */ goto retry; } #undef WAIT_COND return ret < 0 ? ret : 0; } static void guc_exec_queue_resume(struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME; struct xe_guc *guc = exec_queue_to_guc(q); xe_gt_assert(guc_to_gt(guc), !q->guc->suspend_pending); xe_sched_msg_lock(sched); guc_exec_queue_try_add_msg(q, msg, RESUME); xe_sched_msg_unlock(sched); } static bool guc_exec_queue_reset_status(struct xe_exec_queue *q) { if (xe_exec_queue_is_multi_queue_secondary(q) && guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q))) return true; return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q); } /* * All of these functions are an abstraction layer which other parts of Xe can * use to trap into the GuC backend. All of these functions, aside from init, * really shouldn't do much other than trap into the DRM scheduler which * synchronizes these operations. */ static const struct xe_exec_queue_ops guc_exec_queue_ops = { .init = guc_exec_queue_init, .kill = guc_exec_queue_kill, .fini = guc_exec_queue_fini, .destroy = guc_exec_queue_destroy, .set_priority = guc_exec_queue_set_priority, .set_timeslice = guc_exec_queue_set_timeslice, .set_preempt_timeout = guc_exec_queue_set_preempt_timeout, .set_multi_queue_priority = guc_exec_queue_set_multi_queue_priority, .suspend = guc_exec_queue_suspend, .suspend_wait = guc_exec_queue_suspend_wait, .resume = guc_exec_queue_resume, .reset_status = guc_exec_queue_reset_status, }; static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; bool do_destroy = false; /* Stop scheduling + flush any DRM scheduler operations */ xe_sched_submission_stop(sched); /* Clean up lost G2H + reset engine state */ if (exec_queue_registered(q)) { if (exec_queue_destroyed(q)) do_destroy = true; } if (q->guc->suspend_pending) { set_exec_queue_suspended(q); suspend_fence_signal(q); } atomic_and(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_BANNED | EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_DESTROYED | EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); q->guc->resume_time = 0; trace_xe_exec_queue_stop(q); /* * Ban any engine (aside from kernel and engines used for VM ops) with a * started but not complete job or if a job has gone through a GT reset * more than twice. */ if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) { struct xe_sched_job *job = xe_sched_first_pending_job(sched); bool ban = false; if (job) { if ((xe_sched_job_started(job) && !xe_sched_job_completed(job)) || xe_sched_invalidate_job(job, 2)) { trace_xe_sched_job_ban(job); ban = true; } } if (ban) { set_exec_queue_banned(q); xe_guc_exec_queue_trigger_cleanup(q); } } if (do_destroy) __guc_exec_queue_destroy(guc, q); } static int guc_submit_reset_prepare(struct xe_guc *guc) { int ret; /* * Using an atomic here rather than submission_state.lock as this * function can be called while holding the CT lock (engine reset * failure). submission_state.lock needs the CT lock to resubmit jobs. * Atomic is not ideal, but it works to prevent against concurrent reset * and releasing any TDRs waiting on guc->submission_state.stopped. */ ret = atomic_fetch_or(1, &guc->submission_state.stopped); smp_wmb(); wake_up_all(&guc->ct.wq); return ret; } int xe_guc_submit_reset_prepare(struct xe_guc *guc) { if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc))) return 0; if (!guc->submission_state.initialized) return 0; return guc_submit_reset_prepare(guc); } void xe_guc_submit_reset_wait(struct xe_guc *guc) { wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) || !xe_guc_read_stopped(guc)); } void xe_guc_submit_stop(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1); mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { /* Prevent redundant attempts to stop parallel queues */ if (q->guc->id != index) continue; guc_exec_queue_stop(guc, q); } mutex_unlock(&guc->submission_state.lock); /* * No one can enter the backend at this point, aside from new engine * creation which is protected by guc->submission_state.lock. */ } static void guc_exec_queue_revert_pending_state_change(struct xe_guc *guc, struct xe_exec_queue *q) { bool pending_enable, pending_disable, pending_resume; pending_enable = exec_queue_pending_enable(q); pending_resume = exec_queue_pending_resume(q); if (pending_enable && pending_resume) { q->guc->needs_resume = true; xe_gt_dbg(guc_to_gt(guc), "Replay RESUME - guc_id=%d", q->guc->id); } if (pending_enable && !pending_resume) { clear_exec_queue_registered(q); xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER - guc_id=%d", q->guc->id); } if (pending_enable) { clear_exec_queue_enabled(q); clear_exec_queue_pending_resume(q); clear_exec_queue_pending_enable(q); xe_gt_dbg(guc_to_gt(guc), "Replay ENABLE - guc_id=%d", q->guc->id); } if (exec_queue_destroyed(q) && exec_queue_registered(q)) { clear_exec_queue_destroyed(q); q->guc->needs_cleanup = true; xe_gt_dbg(guc_to_gt(guc), "Replay CLEANUP - guc_id=%d", q->guc->id); } pending_disable = exec_queue_pending_disable(q); if (pending_disable && exec_queue_suspended(q)) { clear_exec_queue_suspended(q); q->guc->needs_suspend = true; xe_gt_dbg(guc_to_gt(guc), "Replay SUSPEND - guc_id=%d", q->guc->id); } if (pending_disable) { if (!pending_enable) set_exec_queue_enabled(q); clear_exec_queue_pending_disable(q); xe_gt_dbg(guc_to_gt(guc), "Replay DISABLE - guc_id=%d", q->guc->id); } q->guc->resume_time = 0; } static void lrc_parallel_clear(struct xe_lrc *lrc) { struct xe_device *xe = gt_to_xe(lrc->gt); struct iosys_map map = xe_lrc_parallel_map(lrc); int i; for (i = 0; i < WQ_SIZE / sizeof(u32); ++i) parallel_write(xe, map, wq[i], FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | FIELD_PREP(WQ_LEN_MASK, 0)); } /* * This function is quite complex but only real way to ensure no state is lost * during VF resume flows. The function scans the queue state, make adjustments * as needed, and queues jobs / messages which replayed upon unpause. */ static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; struct xe_sched_job *job; int i; lockdep_assert_held(&guc->submission_state.lock); /* Stop scheduling + flush any DRM scheduler operations */ xe_sched_submission_stop(sched); cancel_delayed_work_sync(&sched->base.work_tdr); guc_exec_queue_revert_pending_state_change(guc, q); if (xe_exec_queue_is_parallel(q)) { /* Pairs with WRITE_ONCE in __xe_exec_queue_init */ struct xe_lrc *lrc = READ_ONCE(q->lrc[0]); /* * NOP existing WQ commands that may contain stale GGTT * addresses. These will be replayed upon unpause. The hardware * seems to get confused if the WQ head/tail pointers are * adjusted. */ if (lrc) lrc_parallel_clear(lrc); } job = xe_sched_first_pending_job(sched); if (job) { job->restore_replay = true; /* * Adjust software tail so jobs submitted overwrite previous * position in ring buffer with new GGTT addresses. */ for (i = 0; i < q->width; ++i) q->lrc[i]->ring.tail = job->ptrs[i].head; } } /** * xe_guc_submit_pause - Stop further runs of submission tasks on given GuC. * @guc: the &xe_guc struct instance whose scheduler is to be disabled */ void xe_guc_submit_pause(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) xe_sched_submission_stop(&q->guc->sched); mutex_unlock(&guc->submission_state.lock); } /** * xe_guc_submit_pause_vf - Stop further runs of submission tasks for VF. * @guc: the &xe_guc struct instance whose scheduler is to be disabled */ void xe_guc_submit_pause_vf(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); xe_gt_assert(guc_to_gt(guc), vf_recovery(guc)); mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { /* Prevent redundant attempts to stop parallel queues */ if (q->guc->id != index) continue; guc_exec_queue_pause(guc, q); } mutex_unlock(&guc->submission_state.lock); } static void guc_exec_queue_start(struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; if (!exec_queue_killed_or_banned_or_wedged(q)) { struct xe_sched_job *job = xe_sched_first_pending_job(sched); int i; trace_xe_exec_queue_resubmit(q); if (job) { for (i = 0; i < q->width; ++i) { /* * The GuC context is unregistered at this point * time, adjusting software ring tail ensures * jobs are rewritten in original placement, * adjusting LRC tail ensures the newly loaded * GuC / contexts only view the LRC tail * increasing as jobs are written out. */ q->lrc[i]->ring.tail = job->ptrs[i].head; xe_lrc_set_ring_tail(q->lrc[i], xe_lrc_ring_head(q->lrc[i])); } } xe_sched_resubmit_jobs(sched); } xe_sched_submission_start(sched); xe_sched_submission_resume_tdr(sched); } int xe_guc_submit_start(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1); mutex_lock(&guc->submission_state.lock); atomic_dec(&guc->submission_state.stopped); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { /* Prevent redundant attempts to start parallel queues */ if (q->guc->id != index) continue; guc_exec_queue_start(q); } mutex_unlock(&guc->submission_state.lock); wake_up_all(&guc->ct.wq); return 0; } static void guc_exec_queue_unpause_prepare(struct xe_guc *guc, struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; struct xe_sched_job *job = NULL; struct drm_sched_job *s_job; bool restore_replay = false; drm_sched_for_each_pending_job(s_job, &sched->base, NULL) { job = to_xe_sched_job(s_job); restore_replay |= job->restore_replay; if (restore_replay) { xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d", q->guc->id, xe_sched_job_seqno(job)); q->ring_ops->emit_job(job); job->restore_replay = true; } } if (job) job->last_replay = true; } /** * xe_guc_submit_unpause_prepare_vf - Prepare unpause submission tasks for VF. * @guc: the &xe_guc struct instance whose scheduler is to be prepared for unpause */ void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); xe_gt_assert(guc_to_gt(guc), vf_recovery(guc)); mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { /* Prevent redundant attempts to stop parallel queues */ if (q->guc->id != index) continue; guc_exec_queue_unpause_prepare(guc, q); } mutex_unlock(&guc->submission_state.lock); } static void guc_exec_queue_replay_pending_state_change(struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; struct xe_sched_msg *msg; if (q->guc->needs_cleanup) { msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; guc_exec_queue_add_msg(q, msg, CLEANUP); q->guc->needs_cleanup = false; } if (q->guc->needs_suspend) { msg = q->guc->static_msgs + STATIC_MSG_SUSPEND; xe_sched_msg_lock(sched); guc_exec_queue_try_add_msg_head(q, msg, SUSPEND); xe_sched_msg_unlock(sched); q->guc->needs_suspend = false; } /* * The resume must be in the message queue before the suspend as it is * not possible for a resume to be issued if a suspend pending is, but * the inverse is possible. */ if (q->guc->needs_resume) { msg = q->guc->static_msgs + STATIC_MSG_RESUME; xe_sched_msg_lock(sched); guc_exec_queue_try_add_msg_head(q, msg, RESUME); xe_sched_msg_unlock(sched); q->guc->needs_resume = false; } } static void guc_exec_queue_unpause(struct xe_guc *guc, struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; bool needs_tdr = exec_queue_killed_or_banned_or_wedged(q); lockdep_assert_held(&guc->submission_state.lock); xe_sched_resubmit_jobs(sched); guc_exec_queue_replay_pending_state_change(q); xe_sched_submission_start(sched); if (needs_tdr) xe_guc_exec_queue_trigger_cleanup(q); xe_sched_submission_resume_tdr(sched); } /** * xe_guc_submit_unpause - Allow further runs of submission tasks on given GuC. * @guc: the &xe_guc struct instance whose scheduler is to be enabled */ void xe_guc_submit_unpause(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) xe_sched_submission_start(&q->guc->sched); mutex_unlock(&guc->submission_state.lock); } /** * xe_guc_submit_unpause_vf - Allow further runs of submission tasks for VF. * @guc: the &xe_guc struct instance whose scheduler is to be enabled */ void xe_guc_submit_unpause_vf(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { /* * Prevent redundant attempts to stop parallel queues, or queues * created after resfix done. */ if (q->guc->id != index || !drm_sched_is_stopped(&q->guc->sched.base)) continue; guc_exec_queue_unpause(guc, q); } mutex_unlock(&guc->submission_state.lock); } /** * xe_guc_submit_pause_abort - Abort all paused submission task on given GuC. * @guc: the &xe_guc struct instance whose scheduler is to be aborted */ void xe_guc_submit_pause_abort(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { struct xe_gpu_scheduler *sched = &q->guc->sched; /* Prevent redundant attempts to stop parallel queues */ if (q->guc->id != index) continue; xe_sched_submission_start(sched); guc_exec_queue_kill(q); } mutex_unlock(&guc->submission_state.lock); } static struct xe_exec_queue * g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id) { struct xe_gt *gt = guc_to_gt(guc); struct xe_exec_queue *q; if (unlikely(guc_id >= GUC_ID_MAX)) { xe_gt_err(gt, "Invalid guc_id %u\n", guc_id); return NULL; } q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id); if (unlikely(!q)) { xe_gt_err(gt, "No exec queue found for guc_id %u\n", guc_id); return NULL; } xe_gt_assert(guc_to_gt(guc), guc_id >= q->guc->id); xe_gt_assert(guc_to_gt(guc), guc_id < (q->guc->id + q->width)); return q; } static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) { u32 action[] = { XE_GUC_ACTION_DEREGISTER_CONTEXT, q->guc->id, }; xe_gt_assert(guc_to_gt(guc), exec_queue_destroyed(q)); xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); trace_xe_exec_queue_deregister(q); if (xe_exec_queue_is_multi_queue_secondary(q)) handle_deregister_done(guc, q); else xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action)); } static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, u32 runnable_state) { trace_xe_exec_queue_scheduling_done(q); if (runnable_state == 1) { xe_gt_assert(guc_to_gt(guc), exec_queue_pending_enable(q)); q->guc->resume_time = ktime_get(); clear_exec_queue_pending_resume(q); clear_exec_queue_pending_enable(q); smp_wmb(); wake_up_all(&guc->ct.wq); } else { xe_gt_assert(guc_to_gt(guc), runnable_state == 0); xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q)); if (q->guc->suspend_pending) { suspend_fence_signal(q); clear_exec_queue_pending_disable(q); } else { if (exec_queue_banned(q)) { smp_wmb(); wake_up_all(&guc->ct.wq); } if (exec_queue_destroyed(q)) { /* * Make sure to clear the pending_disable only * after sampling the destroyed state. We want * to ensure we don't trigger the unregister too * early with something intending to only * disable scheduling. The caller doing the * destroy must wait for an ongoing * pending_disable before marking as destroyed. */ clear_exec_queue_pending_disable(q); deregister_exec_queue(guc, q); } else { clear_exec_queue_pending_disable(q); } } } } static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, u32 runnable_state) { /* Take CT lock here as handle_sched_done() do send a h2g message */ mutex_lock(&guc->ct.lock); handle_sched_done(guc, q, runnable_state); mutex_unlock(&guc->ct.lock); } int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_exec_queue *q; u32 guc_id, runnable_state; if (unlikely(len < 2)) return -EPROTO; guc_id = msg[0]; runnable_state = msg[1]; q = g2h_exec_queue_lookup(guc, guc_id); if (unlikely(!q)) return -EPROTO; if (unlikely(!exec_queue_pending_enable(q) && !exec_queue_pending_disable(q))) { xe_gt_err(guc_to_gt(guc), "SCHED_DONE: Unexpected engine state 0x%04x, guc_id=%d, runnable_state=%u", atomic_read(&q->guc->state), q->guc->id, runnable_state); return -EPROTO; } handle_sched_done(guc, q, runnable_state); return 0; } static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q) { trace_xe_exec_queue_deregister_done(q); clear_exec_queue_registered(q); __guc_exec_queue_destroy(guc, q); } int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_exec_queue *q; u32 guc_id; if (unlikely(len < 1)) return -EPROTO; guc_id = msg[0]; q = g2h_exec_queue_lookup(guc, guc_id); if (unlikely(!q)) return -EPROTO; if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) || exec_queue_pending_enable(q) || exec_queue_enabled(q)) { xe_gt_err(guc_to_gt(guc), "DEREGISTER_DONE: Unexpected engine state 0x%04x, guc_id=%d", atomic_read(&q->guc->state), q->guc->id); return -EPROTO; } handle_deregister_done(guc, q); return 0; } int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_gt *gt = guc_to_gt(guc); struct xe_exec_queue *q; u32 guc_id; if (unlikely(len < 1)) return -EPROTO; guc_id = msg[0]; q = g2h_exec_queue_lookup(guc, guc_id); if (unlikely(!q)) return -EPROTO; xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x", xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id, atomic_read(&q->guc->state)); trace_xe_exec_queue_reset(q); /* * A banned engine is a NOP at this point (came from * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel * jobs by setting timeout of the job to the minimum value kicking * guc_exec_queue_timedout_job. */ xe_guc_exec_queue_reset_trigger_cleanup(q); return 0; } /* * xe_guc_error_capture_handler - Handler of GuC captured message * @guc: The GuC object * @msg: Point to the message * @len: The message length * * When GuC captured data is ready, GuC will send message * XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION to host, this function will be * called 1st to check status before process the data comes with the message. * * Returns: error code. 0 if success */ int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len) { u32 status; if (unlikely(len != XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION_DATA_LEN)) return -EPROTO; status = msg[0] & XE_GUC_STATE_CAPTURE_EVENT_STATUS_MASK; if (status == XE_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE) xe_gt_warn(guc_to_gt(guc), "G2H-Error capture no space"); xe_guc_capture_process(guc); return 0; } int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_gt *gt = guc_to_gt(guc); struct xe_exec_queue *q; u32 guc_id; u32 type = XE_GUC_CAT_ERR_TYPE_INVALID; if (unlikely(!len || len > 2)) return -EPROTO; guc_id = msg[0]; if (len == 2) type = msg[1]; if (guc_id == GUC_ID_UNKNOWN) { /* * GuC uses GUC_ID_UNKNOWN if it can not map the CAT fault to any PF/VF * context. In such case only PF will be notified about that fault. */ xe_gt_err_ratelimited(gt, "Memory CAT error reported by GuC!\n"); return 0; } q = g2h_exec_queue_lookup(guc, guc_id); if (unlikely(!q)) return -EPROTO; /* * The type is HW-defined and changes based on platform, so we don't * decode it in the kernel and only check if it is valid. * See bspec 54047 and 72187 for details. */ if (type != XE_GUC_CAT_ERR_TYPE_INVALID) xe_gt_info(gt, "Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d", type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); else xe_gt_info(gt, "Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d", xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); trace_xe_exec_queue_memory_cat_error(q); /* Treat the same as engine reset */ xe_guc_exec_queue_reset_trigger_cleanup(q); return 0; } int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_gt *gt = guc_to_gt(guc); u8 guc_class, instance; u32 reason; if (unlikely(len != 3)) return -EPROTO; guc_class = msg[0]; instance = msg[1]; reason = msg[2]; /* Unexpected failure of a hardware feature, log an actual error */ xe_gt_err(gt, "GuC engine reset request failed on %d:%d because 0x%08X", guc_class, instance, reason); xe_gt_reset_async(gt); return 0; } int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_gt *gt = guc_to_gt(guc); struct xe_device *xe = guc_to_xe(guc); struct xe_exec_queue *q; u32 guc_id = msg[2]; if (unlikely(len != XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN)) { drm_err(&xe->drm, "Invalid length %u", len); return -EPROTO; } q = g2h_exec_queue_lookup(guc, guc_id); if (unlikely(!q)) return -EPROTO; xe_gt_dbg(gt, "CGP context error: [%s] err=0x%x, q0_id=0x%x LRCA=0x%x guc_id=0x%x", msg[0] & 1 ? "uc" : "kmd", msg[1], msg[2], msg[3], msg[4]); trace_xe_exec_queue_cgp_context_error(q); /* Treat the same as engine reset */ xe_guc_exec_queue_reset_trigger_cleanup(q); return 0; } /** * xe_guc_exec_queue_cgp_sync_done_handler - CGP synchronization done handler * @guc: guc * @msg: message indicating CGP sync done * @len: length of message * * Set multi queue group's sync_pending flag to false and wakeup anyone waiting * for CGP synchronization to complete. * * Return: 0 on success, -EPROTO for malformed messages. */ int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_device *xe = guc_to_xe(guc); struct xe_exec_queue *q; u32 guc_id = msg[0]; if (unlikely(len < 1)) { drm_err(&xe->drm, "Invalid CGP_SYNC_DONE length %u", len); return -EPROTO; } q = g2h_exec_queue_lookup(guc, guc_id); if (unlikely(!q)) return -EPROTO; if (!xe_exec_queue_is_multi_queue_primary(q)) { drm_err(&xe->drm, "Unexpected CGP_SYNC_DONE response"); return -EPROTO; } /* Wakeup the serialized cgp update wait */ WRITE_ONCE(q->multi_queue.group->sync_pending, false); xe_guc_ct_wake_waiters(&guc->ct); return 0; } static void guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q, struct xe_guc_submit_exec_queue_snapshot *snapshot) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); int i; snapshot->guc.wqi_head = q->guc->wqi_head; snapshot->guc.wqi_tail = q->guc->wqi_tail; snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head); snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail); snapshot->parallel.wq_desc.status = parallel_read(xe, map, wq_desc.wq_status); if (snapshot->parallel.wq_desc.head != snapshot->parallel.wq_desc.tail) { for (i = snapshot->parallel.wq_desc.head; i != snapshot->parallel.wq_desc.tail; i = (i + sizeof(u32)) % WQ_SIZE) snapshot->parallel.wq[i / sizeof(u32)] = parallel_read(xe, map, wq[i / sizeof(u32)]); } } static void guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, struct drm_printer *p) { int i; drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n", snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head); drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n", snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail); drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status); if (snapshot->parallel.wq_desc.head != snapshot->parallel.wq_desc.tail) { for (i = snapshot->parallel.wq_desc.head; i != snapshot->parallel.wq_desc.tail; i = (i + sizeof(u32)) % WQ_SIZE) drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32), snapshot->parallel.wq[i / sizeof(u32)]); } } /** * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine. * @q: faulty exec queue * * This can be printed out in a later stage like during dev_coredump * analysis. * * Returns: a GuC Submit Engine snapshot object that must be freed by the * caller, using `xe_guc_exec_queue_snapshot_free`. */ struct xe_guc_submit_exec_queue_snapshot * xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; struct xe_guc_submit_exec_queue_snapshot *snapshot; int i; snapshot = kzalloc_obj(*snapshot, GFP_ATOMIC); if (!snapshot) return NULL; snapshot->guc.id = q->guc->id; memcpy(&snapshot->name, &q->name, sizeof(snapshot->name)); snapshot->class = q->class; snapshot->logical_mask = q->logical_mask; snapshot->width = q->width; snapshot->refcount = kref_read(&q->refcount); snapshot->sched_timeout = sched->base.timeout; snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us; snapshot->sched_props.preempt_timeout_us = q->sched_props.preempt_timeout_us; snapshot->lrc = kmalloc_objs(struct xe_lrc_snapshot *, q->width, GFP_ATOMIC); if (snapshot->lrc) { for (i = 0; i < q->width; ++i) { struct xe_lrc *lrc = q->lrc[i]; snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc); } } snapshot->schedule_state = atomic_read(&q->guc->state); snapshot->exec_queue_flags = q->flags; snapshot->parallel_execution = xe_exec_queue_is_parallel(q); if (snapshot->parallel_execution) guc_exec_queue_wq_snapshot_capture(q, snapshot); if (xe_exec_queue_is_multi_queue(q)) { snapshot->multi_queue.valid = true; snapshot->multi_queue.primary = xe_exec_queue_multi_queue_primary(q)->guc->id; snapshot->multi_queue.pos = q->multi_queue.pos; } return snapshot; } /** * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine. * @snapshot: Previously captured snapshot of job. * * This captures some data that requires taking some locks, so it cannot be done in signaling path. */ void xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot) { int i; if (!snapshot || !snapshot->lrc) return; for (i = 0; i < snapshot->width; ++i) xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]); } /** * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot. * @snapshot: GuC Submit Engine snapshot object. * @p: drm_printer where it will be printed out. * * This function prints out a given GuC Submit Engine snapshot object. */ void xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, struct drm_printer *p) { int i; if (!snapshot) return; drm_printf(p, "GuC ID: %d\n", snapshot->guc.id); drm_printf(p, "\tName: %s\n", snapshot->name); drm_printf(p, "\tClass: %d\n", snapshot->class); drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask); drm_printf(p, "\tWidth: %d\n", snapshot->width); drm_printf(p, "\tRef: %d\n", snapshot->refcount); drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout); drm_printf(p, "\tTimeslice: %u (us)\n", snapshot->sched_props.timeslice_us); drm_printf(p, "\tPreempt timeout: %u (us)\n", snapshot->sched_props.preempt_timeout_us); for (i = 0; snapshot->lrc && i < snapshot->width; ++i) xe_lrc_snapshot_print(snapshot->lrc[i], p); drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state); drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags); if (snapshot->parallel_execution) guc_exec_queue_wq_snapshot_print(snapshot, p); if (snapshot->multi_queue.valid) { drm_printf(p, "\tMulti queue primary GuC ID: %d\n", snapshot->multi_queue.primary); drm_printf(p, "\tMulti queue position: %d\n", snapshot->multi_queue.pos); } } /** * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given * snapshot. * @snapshot: GuC Submit Engine snapshot object. * * This function free all the memory that needed to be allocated at capture * time. */ void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot) { int i; if (!snapshot) return; if (snapshot->lrc) { for (i = 0; i < snapshot->width; i++) xe_lrc_snapshot_free(snapshot->lrc[i]); kfree(snapshot->lrc); } kfree(snapshot); } static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p) { struct xe_guc_submit_exec_queue_snapshot *snapshot; snapshot = xe_guc_exec_queue_snapshot_capture(q); xe_guc_exec_queue_snapshot_print(snapshot, p); xe_guc_exec_queue_snapshot_free(snapshot); } /** * xe_guc_register_vf_exec_queue - Register exec queue for a given context type. * @q: Execution queue * @ctx_type: Type of the context * * This function registers the execution queue with the guc. Special context * types like GUC_CONTEXT_COMPRESSION_SAVE and GUC_CONTEXT_COMPRESSION_RESTORE * are only applicable for IGPU and in the VF. * Submits the execution queue to GUC after registering it. * * Returns - None. */ void xe_guc_register_vf_exec_queue(struct xe_exec_queue *q, int ctx_type) { struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); struct xe_gt *gt = guc_to_gt(guc); xe_gt_assert(gt, IS_SRIOV_VF(xe)); xe_gt_assert(gt, !IS_DGFX(xe)); xe_gt_assert(gt, ctx_type == GUC_CONTEXT_COMPRESSION_SAVE || ctx_type == GUC_CONTEXT_COMPRESSION_RESTORE); xe_gt_assert(gt, GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 23, 0)); register_exec_queue(q, ctx_type); enable_scheduling(q); } /** * xe_guc_submit_print - GuC Submit Print. * @guc: GuC. * @p: drm_printer where it will be printed out. * * This function capture and prints snapshots of **all** GuC Engines. */ void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) { struct xe_exec_queue *q; unsigned long index; if (!xe_device_uc_enabled(guc_to_xe(guc))) return; mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) guc_exec_queue_print(q, p); mutex_unlock(&guc->submission_state.lock); } /** * xe_guc_has_registered_mlrc_queues - check whether there are any MLRC queues * registered with the GuC * @guc: GuC. * * Return: true if any MLRC queue is registered with the GuC, false otherwise. */ bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc) { struct xe_exec_queue *q; unsigned long index; guard(mutex)(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) if (q->width > 1) return true; return false; } /** * xe_guc_contexts_hwsp_rebase - Re-compute GGTT references within all * exec queues registered to given GuC. * @guc: the &xe_guc struct instance * @scratch: scratch buffer to be used as temporary storage * * Returns: zero on success, negative error code on failure. */ int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch) { struct xe_exec_queue *q; unsigned long index; int err = 0; mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { /* Prevent redundant attempts to stop parallel queues */ if (q->guc->id != index) continue; err = xe_exec_queue_contexts_hwsp_rebase(q, scratch); if (err) break; } mutex_unlock(&guc->submission_state.lock); return err; }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1