Author | Tokens | Token Proportion | Commits | Commit Proportion |
---|---|---|---|---|
Matthew Brost | 609 | 33.59% | 1 | 2.38% |
Anshuman Gupta | 534 | 29.45% | 6 | 14.29% |
Rodrigo Vivi | 286 | 15.77% | 17 | 40.48% |
Matthew Auld | 184 | 10.15% | 5 | 11.90% |
Badal Nilawar | 69 | 3.81% | 1 | 2.38% |
Riana Tauro | 35 | 1.93% | 2 | 4.76% |
Maarten Lankhorst | 34 | 1.88% | 1 | 2.38% |
Himal Prasad Ghimiray | 26 | 1.43% | 1 | 2.38% |
Matt Roper | 24 | 1.32% | 3 | 7.14% |
Jani Nikula | 5 | 0.28% | 1 | 2.38% |
Ohad Sharabi | 3 | 0.17% | 1 | 2.38% |
Michał Winiarski | 2 | 0.11% | 1 | 2.38% |
Daniele Ceraolo Spurio | 1 | 0.06% | 1 | 2.38% |
Lucas De Marchi | 1 | 0.06% | 1 | 2.38% |
Total | 1813 | 42 |
// SPDX-License-Identifier: MIT /* * Copyright © 2022 Intel Corporation */ #include "xe_pm.h" #include <linux/pm_runtime.h> #include <drm/drm_managed.h> #include <drm/ttm/ttm_placement.h> #include "display/xe_display.h" #include "xe_bo.h" #include "xe_bo_evict.h" #include "xe_device.h" #include "xe_device_sysfs.h" #include "xe_ggtt.h" #include "xe_gt.h" #include "xe_guc.h" #include "xe_irq.h" #include "xe_pcode.h" #include "xe_wa.h" /** * DOC: Xe Power Management * * Xe PM implements the main routines for both system level suspend states and * for the opportunistic runtime suspend states. * * System Level Suspend (S-States) - In general this is OS initiated suspend * driven by ACPI for achieving S0ix (a.k.a. S2idle, freeze), S3 (suspend to ram), * S4 (disk). The main functions here are `xe_pm_suspend` and `xe_pm_resume`. They * are the main point for the suspend to and resume from these states. * * PCI Device Suspend (D-States) - This is the opportunistic PCIe device low power * state D3, controlled by the PCI subsystem and ACPI with the help from the * runtime_pm infrastructure. * PCI D3 is special and can mean D3hot, where Vcc power is on for keeping memory * alive and quicker low latency resume or D3Cold where Vcc power is off for * better power savings. * The Vcc control of PCI hierarchy can only be controlled at the PCI root port * level, while the device driver can be behind multiple bridges/switches and * paired with other devices. For this reason, the PCI subsystem cannot perform * the transition towards D3Cold. The lowest runtime PM possible from the PCI * subsystem is D3hot. Then, if all these paired devices in the same root port * are in D3hot, ACPI will assist here and run its own methods (_PR3 and _OFF) * to perform the transition from D3hot to D3cold. Xe may disallow this * transition by calling pci_d3cold_disable(root_pdev) before going to runtime * suspend. It will be based on runtime conditions such as VRAM usage for a * quick and low latency resume for instance. * * Runtime PM - This infrastructure provided by the Linux kernel allows the * device drivers to indicate when the can be runtime suspended, so the device * could be put at D3 (if supported), or allow deeper package sleep states * (PC-states), and/or other low level power states. Xe PM component provides * `xe_pm_runtime_suspend` and `xe_pm_runtime_resume` functions that PCI * subsystem will call before transition to/from runtime suspend. * * Also, Xe PM provides get and put functions that Xe driver will use to * indicate activity. In order to avoid locking complications with the memory * management, whenever possible, these get and put functions needs to be called * from the higher/outer levels. * The main cases that need to be protected from the outer levels are: IOCTL, * sysfs, debugfs, dma-buf sharing, GPU execution. * * This component is not responsible for GT idleness (RC6) nor GT frequency * management (RPS). */ #ifdef CONFIG_LOCKDEP static struct lockdep_map xe_pm_runtime_lockdep_map = { .name = "xe_pm_runtime_lockdep_map" }; #endif /** * xe_pm_suspend - Helper for System suspend, i.e. S0->S3 / S0->S2idle * @xe: xe device instance * * Return: 0 on success */ int xe_pm_suspend(struct xe_device *xe) { struct xe_gt *gt; u8 id; int err; drm_dbg(&xe->drm, "Suspending device\n"); for_each_gt(gt, xe, id) xe_gt_suspend_prepare(gt); xe_display_pm_suspend(xe, false); /* FIXME: Super racey... */ err = xe_bo_evict_all(xe); if (err) goto err; for_each_gt(gt, xe, id) { err = xe_gt_suspend(gt); if (err) { xe_display_pm_resume(xe, false); goto err; } } xe_irq_suspend(xe); xe_display_pm_suspend_late(xe); drm_dbg(&xe->drm, "Device suspended\n"); return 0; err: drm_dbg(&xe->drm, "Device suspend failed %d\n", err); return err; } /** * xe_pm_resume - Helper for System resume S3->S0 / S2idle->S0 * @xe: xe device instance * * Return: 0 on success */ int xe_pm_resume(struct xe_device *xe) { struct xe_tile *tile; struct xe_gt *gt; u8 id; int err; drm_dbg(&xe->drm, "Resuming device\n"); for_each_tile(tile, xe, id) xe_wa_apply_tile_workarounds(tile); err = xe_pcode_ready(xe, true); if (err) return err; xe_display_pm_resume_early(xe); /* * This only restores pinned memory which is the memory required for the * GT(s) to resume. */ err = xe_bo_restore_kernel(xe); if (err) goto err; xe_irq_resume(xe); for_each_gt(gt, xe, id) xe_gt_resume(gt); xe_display_pm_resume(xe, false); err = xe_bo_restore_user(xe); if (err) goto err; drm_dbg(&xe->drm, "Device resumed\n"); return 0; err: drm_dbg(&xe->drm, "Device resume failed %d\n", err); return err; } static bool xe_pm_pci_d3cold_capable(struct xe_device *xe) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); struct pci_dev *root_pdev; root_pdev = pcie_find_root_port(pdev); if (!root_pdev) return false; /* D3Cold requires PME capability */ if (!pci_pme_capable(root_pdev, PCI_D3cold)) { drm_dbg(&xe->drm, "d3cold: PME# not supported\n"); return false; } /* D3Cold requires _PR3 power resource */ if (!pci_pr3_present(root_pdev)) { drm_dbg(&xe->drm, "d3cold: ACPI _PR3 not present\n"); return false; } return true; } static void xe_pm_runtime_init(struct xe_device *xe) { struct device *dev = xe->drm.dev; /* * Disable the system suspend direct complete optimization. * We need to ensure that the regular device suspend/resume functions * are called since our runtime_pm cannot guarantee local memory * eviction for d3cold. * TODO: Check HDA audio dependencies claimed by i915, and then enforce * this option to integrated graphics as well. */ if (IS_DGFX(xe)) dev_pm_set_driver_flags(dev, DPM_FLAG_NO_DIRECT_COMPLETE); pm_runtime_use_autosuspend(dev); pm_runtime_set_autosuspend_delay(dev, 1000); pm_runtime_set_active(dev); pm_runtime_allow(dev); pm_runtime_mark_last_busy(dev); pm_runtime_put(dev); } int xe_pm_init_early(struct xe_device *xe) { int err; INIT_LIST_HEAD(&xe->mem_access.vram_userfault.list); err = drmm_mutex_init(&xe->drm, &xe->mem_access.vram_userfault.lock); if (err) return err; err = drmm_mutex_init(&xe->drm, &xe->d3cold.lock); if (err) return err; return 0; } /** * xe_pm_init - Initialize Xe Power Management * @xe: xe device instance * * This component is responsible for System and Device sleep states. * * Returns 0 for success, negative error code otherwise. */ int xe_pm_init(struct xe_device *xe) { int err; /* For now suspend/resume is only allowed with GuC */ if (!xe_device_uc_enabled(xe)) return 0; xe->d3cold.capable = xe_pm_pci_d3cold_capable(xe); if (xe->d3cold.capable) { err = xe_device_sysfs_init(xe); if (err) return err; err = xe_pm_set_vram_threshold(xe, DEFAULT_VRAM_THRESHOLD); if (err) return err; } xe_pm_runtime_init(xe); return 0; } /** * xe_pm_runtime_fini - Finalize Runtime PM * @xe: xe device instance */ void xe_pm_runtime_fini(struct xe_device *xe) { struct device *dev = xe->drm.dev; pm_runtime_get_sync(dev); pm_runtime_forbid(dev); } static void xe_pm_write_callback_task(struct xe_device *xe, struct task_struct *task) { WRITE_ONCE(xe->pm_callback_task, task); /* * Just in case it's somehow possible for our writes to be reordered to * the extent that something else re-uses the task written in * pm_callback_task. For example after returning from the callback, but * before the reordered write that resets pm_callback_task back to NULL. */ smp_mb(); /* pairs with xe_pm_read_callback_task */ } struct task_struct *xe_pm_read_callback_task(struct xe_device *xe) { smp_mb(); /* pairs with xe_pm_write_callback_task */ return READ_ONCE(xe->pm_callback_task); } /** * xe_pm_runtime_suspended - Check if runtime_pm state is suspended * @xe: xe device instance * * This does not provide any guarantee that the device is going to remain * suspended as it might be racing with the runtime state transitions. * It can be used only as a non-reliable assertion, to ensure that we are not in * the sleep state while trying to access some memory for instance. * * Returns true if PCI device is suspended, false otherwise. */ bool xe_pm_runtime_suspended(struct xe_device *xe) { return pm_runtime_suspended(xe->drm.dev); } /** * xe_pm_runtime_suspend - Prepare our device for D3hot/D3Cold * @xe: xe device instance * * Returns 0 for success, negative error code otherwise. */ int xe_pm_runtime_suspend(struct xe_device *xe) { struct xe_bo *bo, *on; struct xe_gt *gt; u8 id; int err = 0; /* Disable access_ongoing asserts and prevent recursive pm calls */ xe_pm_write_callback_task(xe, current); /* * The actual xe_pm_runtime_put() is always async underneath, so * exactly where that is called should makes no difference to us. However * we still need to be very careful with the locks that this callback * acquires and the locks that are acquired and held by any callers of * xe_runtime_pm_get(). We already have the matching annotation * on that side, but we also need it here. For example lockdep should be * able to tell us if the following scenario is in theory possible: * * CPU0 | CPU1 (kworker) * lock(A) | * | xe_pm_runtime_suspend() * | lock(A) * xe_pm_runtime_get() | * * This will clearly deadlock since rpm core needs to wait for * xe_pm_runtime_suspend() to complete, but here we are holding lock(A) * on CPU0 which prevents CPU1 making forward progress. With the * annotation here and in xe_pm_runtime_get() lockdep will see * the potential lock inversion and give us a nice splat. */ lock_map_acquire(&xe_pm_runtime_lockdep_map); /* * Applying lock for entire list op as xe_ttm_bo_destroy and xe_bo_move_notify * also checks and delets bo entry from user fault list. */ mutex_lock(&xe->mem_access.vram_userfault.lock); list_for_each_entry_safe(bo, on, &xe->mem_access.vram_userfault.list, vram_userfault_link) xe_bo_runtime_pm_release_mmap_offset(bo); mutex_unlock(&xe->mem_access.vram_userfault.lock); if (xe->d3cold.allowed) { xe_display_pm_suspend(xe, true); err = xe_bo_evict_all(xe); if (err) goto out; } for_each_gt(gt, xe, id) { err = xe_gt_suspend(gt); if (err) goto out; } xe_irq_suspend(xe); if (xe->d3cold.allowed) xe_display_pm_suspend_late(xe); out: if (err) xe_display_pm_resume(xe, true); lock_map_release(&xe_pm_runtime_lockdep_map); xe_pm_write_callback_task(xe, NULL); return err; } /** * xe_pm_runtime_resume - Waking up from D3hot/D3Cold * @xe: xe device instance * * Returns 0 for success, negative error code otherwise. */ int xe_pm_runtime_resume(struct xe_device *xe) { struct xe_gt *gt; u8 id; int err = 0; /* Disable access_ongoing asserts and prevent recursive pm calls */ xe_pm_write_callback_task(xe, current); lock_map_acquire(&xe_pm_runtime_lockdep_map); if (xe->d3cold.allowed) { err = xe_pcode_ready(xe, true); if (err) goto out; xe_display_pm_resume_early(xe); /* * This only restores pinned memory which is the memory * required for the GT(s) to resume. */ err = xe_bo_restore_kernel(xe); if (err) goto out; } xe_irq_resume(xe); for_each_gt(gt, xe, id) xe_gt_resume(gt); if (xe->d3cold.allowed) { xe_display_pm_resume(xe, true); err = xe_bo_restore_user(xe); if (err) goto out; } out: lock_map_release(&xe_pm_runtime_lockdep_map); xe_pm_write_callback_task(xe, NULL); return err; } /* * For places where resume is synchronous it can be quite easy to deadlock * if we are not careful. Also in practice it might be quite timing * sensitive to ever see the 0 -> 1 transition with the callers locks * held, so deadlocks might exist but are hard for lockdep to ever see. * With this in mind, help lockdep learn about the potentially scary * stuff that can happen inside the runtime_resume callback by acquiring * a dummy lock (it doesn't protect anything and gets compiled out on * non-debug builds). Lockdep then only needs to see the * xe_pm_runtime_lockdep_map -> runtime_resume callback once, and then can * hopefully validate all the (callers_locks) -> xe_pm_runtime_lockdep_map. * For example if the (callers_locks) are ever grabbed in the * runtime_resume callback, lockdep should give us a nice splat. */ static void pm_runtime_lockdep_prime(void) { lock_map_acquire(&xe_pm_runtime_lockdep_map); lock_map_release(&xe_pm_runtime_lockdep_map); } /** * xe_pm_runtime_get - Get a runtime_pm reference and resume synchronously * @xe: xe device instance */ void xe_pm_runtime_get(struct xe_device *xe) { pm_runtime_get_noresume(xe->drm.dev); if (xe_pm_read_callback_task(xe) == current) return; pm_runtime_lockdep_prime(); pm_runtime_resume(xe->drm.dev); } /** * xe_pm_runtime_put - Put the runtime_pm reference back and mark as idle * @xe: xe device instance */ void xe_pm_runtime_put(struct xe_device *xe) { if (xe_pm_read_callback_task(xe) == current) { pm_runtime_put_noidle(xe->drm.dev); } else { pm_runtime_mark_last_busy(xe->drm.dev); pm_runtime_put(xe->drm.dev); } } /** * xe_pm_runtime_get_ioctl - Get a runtime_pm reference before ioctl * @xe: xe device instance * * Returns: Any number greater than or equal to 0 for success, negative error * code otherwise. */ int xe_pm_runtime_get_ioctl(struct xe_device *xe) { if (WARN_ON(xe_pm_read_callback_task(xe) == current)) return -ELOOP; pm_runtime_lockdep_prime(); return pm_runtime_get_sync(xe->drm.dev); } /** * xe_pm_runtime_get_if_active - Get a runtime_pm reference if device active * @xe: xe device instance * * Return: True if device is awake (regardless the previous number of references) * and a new reference was taken, false otherwise. */ bool xe_pm_runtime_get_if_active(struct xe_device *xe) { return pm_runtime_get_if_active(xe->drm.dev) > 0; } /** * xe_pm_runtime_get_if_in_use - Get a new reference if device is active with previous ref taken * @xe: xe device instance * * Return: True if device is awake, a previous reference had been already taken, * and a new reference was now taken, false otherwise. */ bool xe_pm_runtime_get_if_in_use(struct xe_device *xe) { if (xe_pm_read_callback_task(xe) == current) { /* The device is awake, grab the ref and move on */ pm_runtime_get_noresume(xe->drm.dev); return true; } return pm_runtime_get_if_in_use(xe->drm.dev) > 0; } /** * xe_pm_runtime_get_noresume - Bump runtime PM usage counter without resuming * @xe: xe device instance * * This function should be used in inner places where it is surely already * protected by outer-bound callers of `xe_pm_runtime_get`. * It will warn if not protected. * The reference should be put back after this function regardless, since it * will always bump the usage counter, regardless. */ void xe_pm_runtime_get_noresume(struct xe_device *xe) { bool ref; ref = xe_pm_runtime_get_if_in_use(xe); if (drm_WARN(&xe->drm, !ref, "Missing outer runtime PM protection\n")) pm_runtime_get_noresume(xe->drm.dev); } /** * xe_pm_runtime_resume_and_get - Resume, then get a runtime_pm ref if awake. * @xe: xe device instance * * Returns: True if device is awake and the reference was taken, false otherwise. */ bool xe_pm_runtime_resume_and_get(struct xe_device *xe) { if (xe_pm_read_callback_task(xe) == current) { /* The device is awake, grab the ref and move on */ pm_runtime_get_noresume(xe->drm.dev); return true; } pm_runtime_lockdep_prime(); return pm_runtime_resume_and_get(xe->drm.dev) >= 0; } /** * xe_pm_assert_unbounded_bridge - Disable PM on unbounded pcie parent bridge * @xe: xe device instance */ void xe_pm_assert_unbounded_bridge(struct xe_device *xe) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); struct pci_dev *bridge = pci_upstream_bridge(pdev); if (!bridge) return; if (!bridge->driver) { drm_warn(&xe->drm, "unbounded parent pci bridge, device won't support any PM support.\n"); device_set_pm_not_required(&pdev->dev); } } /** * xe_pm_set_vram_threshold - Set a vram threshold for allowing/blocking D3Cold * @xe: xe device instance * @threshold: VRAM size in bites for the D3cold threshold * * Returns 0 for success, negative error code otherwise. */ int xe_pm_set_vram_threshold(struct xe_device *xe, u32 threshold) { struct ttm_resource_manager *man; u32 vram_total_mb = 0; int i; for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { man = ttm_manager_type(&xe->ttm, i); if (man) vram_total_mb += DIV_ROUND_UP_ULL(man->size, 1024 * 1024); } drm_dbg(&xe->drm, "Total vram %u mb\n", vram_total_mb); if (threshold > vram_total_mb) return -EINVAL; mutex_lock(&xe->d3cold.lock); xe->d3cold.vram_threshold = threshold; mutex_unlock(&xe->d3cold.lock); return 0; } /** * xe_pm_d3cold_allowed_toggle - Check conditions to toggle d3cold.allowed * @xe: xe device instance * * To be called during runtime_pm idle callback. * Check for all the D3Cold conditions ahead of runtime suspend. */ void xe_pm_d3cold_allowed_toggle(struct xe_device *xe) { struct ttm_resource_manager *man; u32 total_vram_used_mb = 0; u64 vram_used; int i; if (!xe->d3cold.capable) { xe->d3cold.allowed = false; return; } for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { man = ttm_manager_type(&xe->ttm, i); if (man) { vram_used = ttm_resource_manager_usage(man); total_vram_used_mb += DIV_ROUND_UP_ULL(vram_used, 1024 * 1024); } } mutex_lock(&xe->d3cold.lock); if (total_vram_used_mb < xe->d3cold.vram_threshold) xe->d3cold.allowed = true; else xe->d3cold.allowed = false; mutex_unlock(&xe->d3cold.lock); drm_dbg(&xe->drm, "d3cold: allowed=%s\n", str_yes_no(xe->d3cold.allowed)); }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1