Contributors: 11
Author Tokens Token Proportion Commits Commit Proportion
Shashank Sharma 323 50.95% 11 21.15%
Alex Deucher 90 14.20% 10 19.23%
Jesse Zhang 64 10.09% 6 11.54%
Sunil Khatri 64 10.09% 5 9.62%
Prike Liang 26 4.10% 5 9.62%
Christian König 25 3.94% 6 11.54%
Arunpravin Pannerslvam 13 2.05% 3 5.77%
Saleemkhan Jamadar 13 2.05% 1 1.92%
Lijo Lazar 7 1.10% 3 5.77%
Leo Liu 5 0.79% 1 1.92%
Andrey Grodzovsky 4 0.63% 1 1.92%
Total 634 52 


/* SPDX-License-Identifier: MIT */
/*
 * Copyright 2023 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 */

#ifndef AMDGPU_USERQ_H_
#define AMDGPU_USERQ_H_
#include "amdgpu_eviction_fence.h"

#define AMDGPU_MAX_USERQ_COUNT 512

#define to_ev_fence(f) container_of(f, struct amdgpu_eviction_fence, base)
#define uq_mgr_to_fpriv(u) container_of(u, struct amdgpu_fpriv, userq_mgr)
#define work_to_uq_mgr(w, name) container_of(w, struct amdgpu_userq_mgr, name)

enum amdgpu_userq_state {
	AMDGPU_USERQ_STATE_UNMAPPED = 0,
	AMDGPU_USERQ_STATE_MAPPED,
	AMDGPU_USERQ_STATE_PREEMPTED,
	AMDGPU_USERQ_STATE_HUNG,
	AMDGPU_USERQ_STATE_INVALID_VA,
};

struct amdgpu_mqd_prop;

struct amdgpu_userq_obj {
	void		 *cpu_ptr;
	uint64_t	 gpu_addr;
	struct amdgpu_bo *obj;
};

struct amdgpu_usermode_queue {
	int			queue_type;
	enum amdgpu_userq_state state;
	uint64_t		doorbell_handle;
	uint64_t		doorbell_index;
	uint64_t		flags;
	struct amdgpu_mqd_prop	*userq_prop;
	struct amdgpu_userq_mgr *userq_mgr;
	struct amdgpu_vm	*vm;
	struct amdgpu_userq_obj mqd;
	struct amdgpu_userq_obj	db_obj;
	struct amdgpu_userq_obj fw_obj;
	struct amdgpu_userq_obj wptr_obj;

	/**
	 * @fence_drv_lock: Protecting @fence_drv_xa.
	 */
	struct mutex		fence_drv_lock;

	/**
	 * @fence_drv_xa:
	 *
	 * References to the external fence drivers returned by wait_ioctl.
	 * Dropped on the next signaled dma_fence or queue destruction.
	 */
	struct xarray		fence_drv_xa;
	struct amdgpu_userq_fence_driver *fence_drv;
	struct dma_fence	*last_fence;
	u32			xcp_id;
	int			priority;
	struct dentry		*debugfs_queue;

	/**
	 * @hang_detect_work:
	 *
	 * Delayed work which runs when userq_fences time out.
	 */
	struct delayed_work	hang_detect_work;
	struct kref		refcount;

	union {
		struct {
			u64 queue_rb;
			u64 wptr;
			u64 rptr;
			u64 eop;
			u64 shadow;
			u64 csa;
		} va;
		u64 va_array[6];
	} userq_vas;
};

struct amdgpu_userq_funcs {
	int (*mqd_create)(struct amdgpu_usermode_queue *queue,
			  struct drm_amdgpu_userq_in *args);
	int (*mqd_update)(struct amdgpu_usermode_queue *queue,
			  struct drm_amdgpu_userq_in *args);
	void (*mqd_destroy)(struct amdgpu_usermode_queue *uq);
	int (*unmap)(struct amdgpu_usermode_queue *queue);
	int (*map)(struct amdgpu_usermode_queue *queue);
	int (*preempt)(struct amdgpu_usermode_queue *queue);
	int (*restore)(struct amdgpu_usermode_queue *queue);
	int (*detect_and_reset)(struct amdgpu_device *adev,
		  int queue_type);
};

/* Usermode queues for gfx */
struct amdgpu_userq_mgr {
	/**
	 * @userq_xa: Per-process user queue map (queue ID → queue)
	 * Key: queue_id (unique ID within the process's userq manager)
	 * Value: struct amdgpu_usermode_queue
	 */
	struct xarray			userq_xa;
	struct mutex			userq_mutex;
	struct amdgpu_device		*adev;
	struct delayed_work		resume_work;
	struct drm_file			*file;

	/**
	 * @reset_work:
	 *
	 * Reset work which is used when eviction fails.
	 */
	struct work_struct		reset_work;
	atomic_t                        userq_count[AMDGPU_RING_TYPE_MAX];
};

struct amdgpu_db_info {
	uint64_t doorbell_handle;
	uint32_t queue_type;
	uint32_t doorbell_offset;
	struct amdgpu_userq_obj	*db_obj;
};

struct amdgpu_usermode_queue *amdgpu_userq_get(struct amdgpu_userq_mgr *uq_mgr, u32 qid);
void amdgpu_userq_put(struct amdgpu_usermode_queue *queue);

int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);

int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
			  struct amdgpu_device *adev);

void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev);
void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);

void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr);

void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr,
				  struct amdgpu_eviction_fence_mgr *evf_mgr);

u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev);
bool amdgpu_userq_enabled(struct drm_device *dev);

int amdgpu_userq_suspend(struct amdgpu_device *adev);
int amdgpu_userq_resume(struct amdgpu_device *adev);

int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev,
						  u32 idx);
int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev,
						   u32 idx);
void amdgpu_userq_reset_work(struct work_struct *work);
void amdgpu_userq_pre_reset(struct amdgpu_device *adev);
int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost);
void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue);
void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell);

int amdgpu_userq_input_va_validate(struct amdgpu_device *adev,
				   struct amdgpu_usermode_queue *queue,
				   u64 addr, u64 expected_size, u64 *va_out);

void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev,
					struct amdgpu_bo_va_mapping *mapping);
#endif