Contributors: 5
Author Tokens Token Proportion Commits Commit Proportion
Lizhi Hou 3540 92.99% 27 77.14%
David Zhang 171 4.49% 3 8.57%
Max Zhen 69 1.81% 2 5.71%
Doruk Tan Ozturk 23 0.60% 2 5.71%
Kees Cook 4 0.11% 1 2.86%
Total 3807 35 


// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
 */

#include <drm/amdxdna_accel.h>
#include <drm/drm_device.h>
#include <drm/drm_drv.h>
#include <drm/drm_file.h>
#include <drm/drm_gem.h>
#include <drm/drm_gem_shmem_helper.h>
#include <drm/drm_print.h>
#include <drm/gpu_scheduler.h>
#include <linux/xarray.h>
#include <trace/events/amdxdna.h>

#include "amdxdna_ctx.h"
#include "amdxdna_gem.h"
#include "amdxdna_pci_drv.h"
#include "amdxdna_pm.h"

#define MAX_HWCTX_ID		255
#define MAX_ARG_COUNT		4095

struct amdxdna_fence {
	struct dma_fence	base;
	spinlock_t		lock; /* for base */
	struct amdxdna_hwctx	*hwctx;
};

static const char *amdxdna_fence_get_driver_name(struct dma_fence *fence)
{
	return KBUILD_MODNAME;
}

static const char *amdxdna_fence_get_timeline_name(struct dma_fence *fence)
{
	struct amdxdna_fence *xdna_fence;

	xdna_fence = container_of(fence, struct amdxdna_fence, base);

	return xdna_fence->hwctx->name;
}

static const struct dma_fence_ops fence_ops = {
	.get_driver_name = amdxdna_fence_get_driver_name,
	.get_timeline_name = amdxdna_fence_get_timeline_name,
};

static struct dma_fence *amdxdna_fence_create(struct amdxdna_hwctx *hwctx)
{
	struct amdxdna_fence *fence;

	fence = kzalloc_obj(*fence);
	if (!fence)
		return NULL;

	fence->hwctx = hwctx;
	spin_lock_init(&fence->lock);
	dma_fence_init(&fence->base, &fence_ops, &fence->lock, hwctx->id, 0);
	return &fence->base;
}

static void amdxdna_hwctx_release_expanded_heap(struct amdxdna_hwctx *hwctx)
{
	struct amdxdna_client *client = hwctx->client;
	struct amdxdna_gem_obj *heap;
	unsigned long heap_id;

	mutex_lock(&client->mm_lock);
	if (hwctx->last_attached_heap) {
		xa_for_each_range(&client->dev_heap_xa, heap_id, heap, 1,
				  hwctx->last_attached_heap) {
			amdxdna_gem_unpin(heap);
			drm_gem_object_put(to_gobj(heap));
		}
	}
	mutex_unlock(&client->mm_lock);
}

static void amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx *hwctx,
				      struct srcu_struct *ss)
{
	struct amdxdna_client *client = hwctx->client;
	struct amdxdna_dev *xdna = client->xdna;

	synchronize_srcu(ss);

	/* At this point, user is not able to submit new commands */
	xdna->dev_info->ops->hwctx_fini(hwctx);

	amdxdna_hwctx_release_expanded_heap(hwctx);
	kfree(hwctx->name);
	kfree(hwctx);
}

int amdxdna_hwctx_walk(struct amdxdna_client *client, void *arg,
		       int (*walk)(struct amdxdna_hwctx *hwctx, void *arg))
{
	struct amdxdna_hwctx *hwctx;
	unsigned long hwctx_id;
	int ret = 0, idx;

	idx = srcu_read_lock(&client->hwctx_srcu);
	amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
		ret = walk(hwctx, arg);
		if (ret)
			break;
	}
	srcu_read_unlock(&client->hwctx_srcu, idx);

	return ret;
}

void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size)
{
	struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo);
	u32 num_masks, count;

	if (!cmd)
		return NULL;

	if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN)
		num_masks = 0;
	else
		num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header);

	if (size) {
		count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header);
		if (unlikely(count <= num_masks ||
			     count * sizeof(u32) +
			     offsetof(struct amdxdna_cmd, data[0]) >
			     abo->mem.size)) {
			*size = 0;
			return NULL;
		}
		*size = (count - num_masks) * sizeof(u32);
	}
	return &cmd->data[num_masks];
}

u32 amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo)
{
	struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo);
	u32 num_masks, i;
	u32 *cu_mask;

	if (!cmd)
		return INVALID_CU_IDX;

	if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN)
		return INVALID_CU_IDX;

	num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header);
	cu_mask = cmd->data;
	for (i = 0; i < num_masks; i++) {
		if (cu_mask[i])
			return ffs(cu_mask[i]) - 1;
	}

	return INVALID_CU_IDX;
}

int amdxdna_cmd_set_error(struct amdxdna_gem_obj *abo,
			  struct amdxdna_sched_job *job, u32 cmd_idx,
			  enum ert_cmd_state error_state,
			  void *err_data, size_t size)
{
	struct amdxdna_client *client = job->hwctx->client;
	struct amdxdna_cmd *cmd = amdxdna_gem_vmap(abo);
	struct amdxdna_cmd_chain *cc = NULL;

	if (!cmd)
		return -ENOMEM;

	cmd->header &= ~AMDXDNA_CMD_STATE;
	cmd->header |= FIELD_PREP(AMDXDNA_CMD_STATE, error_state);

	if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN) {
		cc = amdxdna_cmd_get_payload(abo, NULL);
		cc->error_index = (cmd_idx < cc->command_count) ? cmd_idx : 0;
		abo = amdxdna_gem_get_obj(client, cc->data[0], AMDXDNA_BO_SHARE);
		if (!abo)
			return -EINVAL;
		cmd = amdxdna_gem_vmap(abo);
		if (!cmd)
			return -ENOMEM;
	}

	memset(cmd->data, 0xff, abo->mem.size - sizeof(*cmd));
	if (err_data)
		memcpy(cmd->data, err_data, min(size, abo->mem.size - sizeof(*cmd)));

	if (cc)
		amdxdna_gem_put_obj(abo);

	return 0;
}

/*
 * This should be called in close() and remove(). DO NOT call in other syscalls.
 * This guarantee that when hwctx and resources will be released, if user
 * doesn't call amdxdna_drm_destroy_hwctx_ioctl.
 */
void amdxdna_hwctx_remove_all(struct amdxdna_client *client)
{
	struct amdxdna_hwctx *hwctx;
	unsigned long hwctx_id;

	amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
		XDNA_DBG(client->xdna, "PID %d close HW context %d",
			 client->pid, hwctx->id);
		xa_erase(&client->hwctx_xa, hwctx->id);
		amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu);
	}
}

int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
	struct amdxdna_client *client = filp->driver_priv;
	struct amdxdna_drm_create_hwctx *args = data;
	struct amdxdna_dev *xdna = to_xdna_dev(dev);
	struct amdxdna_hwctx *hwctx;
	int ret, idx;

	if (args->ext || args->ext_flags)
		return -EINVAL;

	if (!xdna->dev_info->ops->hwctx_init)
		return -EOPNOTSUPP;

	hwctx = kzalloc_obj(*hwctx);
	if (!hwctx)
		return -ENOMEM;

	if (copy_from_user(&hwctx->qos, u64_to_user_ptr(args->qos_p), sizeof(hwctx->qos))) {
		XDNA_ERR(xdna, "Access QoS info failed");
		kfree(hwctx);
		return -EFAULT;
	}

	hwctx->client = client;
	hwctx->fw_ctx_id = -1;
	hwctx->num_tiles = args->num_tiles;
	hwctx->umq_bo_hdl = args->umq_bo;
	hwctx->doorbell_offset = AMDXDNA_INVALID_DOORBELL_OFFSET;
	hwctx->mem_size = args->mem_size;
	hwctx->max_opc = args->max_opc;

	guard(mutex)(&xdna->dev_lock);

	if (!drm_dev_enter(dev, &idx)) {
		ret = -ENODEV;
		goto free_hwctx;
	}

	ret = xdna->dev_info->ops->hwctx_init(hwctx);
	if (ret) {
		XDNA_ERR(xdna, "Init hwctx failed, ret %d", ret);
		goto release_expanded_heap;
	}

	hwctx->name = kasprintf(GFP_KERNEL, "hwctx.%d.%d", client->pid, hwctx->fw_ctx_id);
	if (!hwctx->name) {
		ret = -ENOMEM;
		goto fini_hwctx;
	}

	ret = xa_alloc_cyclic(&client->hwctx_xa, &hwctx->id, hwctx,
			      XA_LIMIT(AMDXDNA_INVALID_CTX_HANDLE + 1, MAX_HWCTX_ID),
			      &client->next_hwctxid, GFP_KERNEL);
	if (ret < 0) {
		XDNA_ERR(xdna, "Allocate hwctx ID failed, ret %d", ret);
		goto free_name;
	}

	args->handle = hwctx->id;
	args->syncobj_handle = hwctx->syncobj_hdl;
	args->umq_doorbell = hwctx->doorbell_offset;

	atomic64_set(&hwctx->job_submit_cnt, 0);
	atomic64_set(&hwctx->job_free_cnt, 0);
	XDNA_DBG(xdna, "PID %d create HW context %d, ret %d", client->pid, args->handle, ret);
	drm_dev_exit(idx);
	return 0;

free_name:
	kfree(hwctx->name);
fini_hwctx:
	xdna->dev_info->ops->hwctx_fini(hwctx);
release_expanded_heap:
	amdxdna_hwctx_release_expanded_heap(hwctx);
	drm_dev_exit(idx);
free_hwctx:
	kfree(hwctx);
	return ret;
}

int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
	struct amdxdna_client *client = filp->driver_priv;
	struct amdxdna_drm_destroy_hwctx *args = data;
	struct amdxdna_dev *xdna = to_xdna_dev(dev);
	struct amdxdna_hwctx *hwctx;
	int ret = 0, idx;

	if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad)))
		return -EINVAL;

	if (!drm_dev_enter(dev, &idx))
		return -ENODEV;

	mutex_lock(&xdna->client_lock);
	mutex_lock(&xdna->dev_lock);
	hwctx = xa_erase(&client->hwctx_xa, args->handle);
	if (!hwctx) {
		ret = -EINVAL;
		XDNA_DBG(xdna, "PID %d HW context %d not exist",
			 client->pid, args->handle);
		goto out;
	}

	/*
	 * The pushed jobs are handled by DRM scheduler during destroy.
	 * SRCU to synchronize with exec command ioctls.
	 */
	amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu);

	XDNA_DBG(xdna, "PID %d destroyed HW context %d", client->pid, args->handle);
out:
	mutex_unlock(&xdna->dev_lock);
	mutex_unlock(&xdna->client_lock);
	drm_dev_exit(idx);
	return ret;
}

int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
	struct amdxdna_client *client = filp->driver_priv;
	struct amdxdna_drm_config_hwctx *args = data;
	struct amdxdna_dev *xdna = to_xdna_dev(dev);
	struct amdxdna_hwctx *hwctx;
	u32 buf_size;
	void *buf;
	int ret;
	u64 val;

	if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad)))
		return -EINVAL;

	if (!xdna->dev_info->ops->hwctx_config)
		return -EOPNOTSUPP;

	val = args->param_val;
	buf_size = args->param_val_size;

	switch (args->param_type) {
	case DRM_AMDXDNA_HWCTX_CONFIG_CU:
		/* For those types that param_val is pointer */
		if (buf_size > PAGE_SIZE) {
			XDNA_ERR(xdna, "Config CU param buffer too large");
			return -E2BIG;
		}

		/* Hwctx needs to keep buf */
		buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
		if (!buf)
			return -ENOMEM;

		if (copy_from_user(buf, u64_to_user_ptr(val), buf_size)) {
			kfree(buf);
			return -EFAULT;
		}

		break;
	case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF:
	case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF:
		/* For those types that param_val is a value */
		buf = NULL;
		buf_size = 0;
		break;
	default:
		XDNA_DBG(xdna, "Unknown HW context config type %d", args->param_type);
		return -EINVAL;
	}

	ret = amdxdna_pm_resume_get(xdna);
	if (ret) {
		XDNA_ERR(xdna, "Resume failed, ret %d", ret);
		goto free_buf;
	}

	mutex_lock(&xdna->client_lock);
	mutex_lock(&xdna->dev_lock);
	hwctx = xa_load(&client->hwctx_xa, args->handle);
	if (!hwctx) {
		XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle);
		ret = -EINVAL;
		goto unlock;
	}

	ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size);

unlock:
	mutex_unlock(&xdna->dev_lock);
	mutex_unlock(&xdna->client_lock);
	amdxdna_pm_suspend_put(xdna);
free_buf:
	kfree(buf);
	return ret;
}

int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl)
{
	struct amdxdna_dev *xdna = client->xdna;
	struct amdxdna_hwctx *hwctx;
	struct amdxdna_gem_obj *abo;
	struct drm_gem_object *gobj;
	int ret;

	if (!xdna->dev_info->ops->hwctx_sync_debug_bo)
		return -EOPNOTSUPP;

	gobj = drm_gem_object_lookup(client->filp, debug_bo_hdl);
	if (!gobj)
		return -EINVAL;

	ret = amdxdna_pm_resume_get(xdna);
	if (ret) {
		XDNA_ERR(xdna, "Resume failed, ret %d", ret);
		goto put_obj;
	}

	abo = to_xdna_obj(gobj);
	mutex_lock(&xdna->client_lock);
	mutex_lock(&xdna->dev_lock);
	hwctx = xa_load(&client->hwctx_xa, abo->assigned_hwctx);
	if (!hwctx) {
		ret = -EINVAL;
		goto unlock;
	}

	ret = xdna->dev_info->ops->hwctx_sync_debug_bo(hwctx, debug_bo_hdl);

unlock:
	mutex_unlock(&xdna->dev_lock);
	mutex_unlock(&xdna->client_lock);
	amdxdna_pm_suspend_put(xdna);
put_obj:
	drm_gem_object_put(gobj);
	return ret;
}

static int amdxdna_hwctx_expand_heap(struct amdxdna_hwctx *hwctx)
{
	struct amdxdna_client *client = hwctx->client;
	struct amdxdna_dev *xdna = client->xdna;
	struct amdxdna_gem_obj *heap;
	unsigned long heap_id, nid;
	int ret = 0;

	nid = hwctx->last_attached_heap + 1;
	if (nid == client->dev_heap_nid)
		goto out;

	xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
			  nid, client->dev_heap_nid) {
		drm_gem_object_get(to_gobj(heap));
		ret = amdxdna_gem_pin(heap);
		if (ret) {
			drm_gem_object_put(to_gobj(heap));
			break;
		}

		ret = xdna->dev_info->ops->hwctx_heap_expand(hwctx, heap);
		if (ret) {
			amdxdna_gem_unpin(heap);
			drm_gem_object_put(to_gobj(heap));
			break;
		}

		hwctx->last_attached_heap = heap_id;
	}

out:
	return ret;
}

int amdxdna_update_heap(struct amdxdna_client *client, struct amdxdna_hwctx *hwctx)
{
	unsigned long hwctx_id;
	int ret;

	ret = amdxdna_pm_resume_get_locked(client->xdna);
	if (ret)
		return ret;

	mutex_lock(&client->mm_lock);

	if (hwctx) {
		ret = amdxdna_hwctx_expand_heap(hwctx);
	} else {
		amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
			ret = amdxdna_hwctx_expand_heap(hwctx);
			if (ret)
				break;
		}
	}
	mutex_unlock(&client->mm_lock);

	amdxdna_pm_suspend_put(client->xdna);

	return ret;
}

static void
amdxdna_arg_bos_put(struct amdxdna_sched_job *job)
{
	int i;

	for (i = 0; i < job->bo_cnt; i++) {
		if (!job->bos[i])
			break;
		drm_gem_object_put(job->bos[i]);
	}
}

static int
amdxdna_arg_bos_lookup(struct amdxdna_client *client,
		       struct amdxdna_sched_job *job,
		       u32 *bo_hdls, u32 bo_cnt)
{
	struct drm_gem_object *gobj;
	int i, ret;

	job->bo_cnt = bo_cnt;
	for (i = 0; i < job->bo_cnt; i++) {
		struct amdxdna_gem_obj *abo;

		gobj = drm_gem_object_lookup(client->filp, bo_hdls[i]);
		if (!gobj) {
			ret = -ENOENT;
			goto put_shmem_bo;
		}
		abo = to_xdna_obj(gobj);

		mutex_lock(&abo->lock);
		if (abo->pinned) {
			mutex_unlock(&abo->lock);
			job->bos[i] = gobj;
			continue;
		}

		ret = amdxdna_gem_pin_nolock(abo);
		if (ret) {
			mutex_unlock(&abo->lock);
			drm_gem_object_put(gobj);
			goto put_shmem_bo;
		}
		abo->pinned = true;
		mutex_unlock(&abo->lock);

		job->bos[i] = gobj;
	}

	return 0;

put_shmem_bo:
	amdxdna_arg_bos_put(job);
	return ret;
}

void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job)
{
	trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release");
	amdxdna_pm_suspend_put(job->hwctx->client->xdna);
	amdxdna_arg_bos_put(job);
	amdxdna_gem_put_obj(job->cmd_bo);
	dma_fence_put(job->fence);
	mmdrop(job->mm);
}

int amdxdna_cmd_submit(struct amdxdna_client *client,
		       struct amdxdna_drv_cmd *drv_cmd,
		       u32 cmd_bo_hdl, u32 *arg_bo_hdls, u32 arg_bo_cnt,
		       u32 hwctx_hdl, u64 *seq)
{
	struct amdxdna_dev *xdna = client->xdna;
	struct amdxdna_sched_job *job;
	struct amdxdna_hwctx *hwctx;
	int ret, idx;

	XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt);

	if (!xdna->dev_info->ops->cmd_submit)
		return -EOPNOTSUPP;

	job = kzalloc_flex(*job, bos, arg_bo_cnt);
	if (!job)
		return -ENOMEM;

	job->drv_cmd = drv_cmd;

	if (cmd_bo_hdl != AMDXDNA_INVALID_BO_HANDLE) {
		job->cmd_bo = amdxdna_gem_get_obj(client, cmd_bo_hdl, AMDXDNA_BO_SHARE);
		if (!job->cmd_bo) {
			XDNA_ERR(xdna, "Failed to get cmd bo from %d", cmd_bo_hdl);
			ret = -EINVAL;
			goto free_job;
		}
	} else if (!drv_cmd) {
		/*
		 * Only internal driver commands (drv_cmd != NULL) may omit a
		 * command BO. A user command submission with the invalid handle
		 * would leave job->cmd_bo NULL and later fault when the scheduler
		 * dereferences it in amdxdna_cmd_set_state().
		 */
		XDNA_DBG(xdna, "Command BO handle required for user submission");
		ret = -EINVAL;
		goto free_job;
	}

	ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt);
	if (ret) {
		XDNA_ERR(xdna, "Argument BOs lookup failed, ret %d", ret);
		goto cmd_put;
	}

	ret = amdxdna_pm_resume_get(xdna);
	if (ret) {
		XDNA_ERR(xdna, "Resume failed, ret %d", ret);
		goto put_bos;
	}

	idx = srcu_read_lock(&client->hwctx_srcu);
	hwctx = xa_load(&client->hwctx_xa, hwctx_hdl);
	if (!hwctx) {
		XDNA_DBG(xdna, "PID %d failed to get hwctx %d",
			 client->pid, hwctx_hdl);
		ret = -EINVAL;
		goto unlock_srcu;
	}

	job->hwctx = hwctx;
	job->mm = current->mm;
	mmgrab(job->mm);

	job->fence = amdxdna_fence_create(hwctx);
	if (!job->fence) {
		XDNA_ERR(xdna, "Failed to create fence");
		ret = -ENOMEM;
		goto unlock_srcu;
	}
	kref_init(&job->refcnt);

	ret = xdna->dev_info->ops->cmd_submit(hwctx, job, seq);
	if (ret)
		goto put_fence;

	/*
	 * The amdxdna_hwctx_destroy_rcu() will release hwctx and associated
	 * resource after synchronize_srcu(). The submitted jobs should be
	 * handled by the queue, for example DRM scheduler, in device layer.
	 * For here we can unlock SRCU.
	 */
	srcu_read_unlock(&client->hwctx_srcu, idx);
	trace_amdxdna_debug_point(hwctx->name, *seq, "job pushed");

	return 0;

put_fence:
	dma_fence_put(job->fence);
unlock_srcu:
	srcu_read_unlock(&client->hwctx_srcu, idx);
	amdxdna_pm_suspend_put(xdna);
put_bos:
	amdxdna_arg_bos_put(job);
cmd_put:
	amdxdna_gem_put_obj(job->cmd_bo);
free_job:
	if (job->mm)
		mmdrop(job->mm);
	kfree(job);
	return ret;
}

/*
 * The submit command ioctl submits a command to firmware. One firmware command
 * may contain multiple command BOs for processing as a whole.
 * The command sequence number is returned which can be used for wait command ioctl.
 */
static int amdxdna_drm_submit_execbuf(struct amdxdna_client *client,
				      struct amdxdna_drm_exec_cmd *args)
{
	struct amdxdna_dev *xdna = client->xdna;
	u32 *arg_bo_hdls = NULL;
	u32 cmd_bo_hdl;
	int ret;

	if (args->arg_count > MAX_ARG_COUNT) {
		XDNA_ERR(xdna, "Invalid arg bo count %d", args->arg_count);
		return -EINVAL;
	}

	/* Only support single command for now. */
	if (args->cmd_count != 1) {
		XDNA_ERR(xdna, "Invalid cmd bo count %d", args->cmd_count);
		return -EINVAL;
	}

	cmd_bo_hdl = (u32)args->cmd_handles;
	if (args->arg_count) {
		arg_bo_hdls = kcalloc(args->arg_count, sizeof(u32), GFP_KERNEL);
		if (!arg_bo_hdls)
			return -ENOMEM;
		ret = copy_from_user(arg_bo_hdls, u64_to_user_ptr(args->args),
				     args->arg_count * sizeof(u32));
		if (ret) {
			ret = -EFAULT;
			goto free_cmd_bo_hdls;
		}
	}

	ret = amdxdna_cmd_submit(client, NULL, cmd_bo_hdl, arg_bo_hdls,
				 args->arg_count, args->hwctx, &args->seq);
	if (ret)
		XDNA_DBG(xdna, "Submit cmds failed, ret %d", ret);

free_cmd_bo_hdls:
	kfree(arg_bo_hdls);
	if (!ret)
		XDNA_DBG(xdna, "Pushed cmd %lld to scheduler", args->seq);
	return ret;
}

int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
	struct amdxdna_client *client = filp->driver_priv;
	struct amdxdna_drm_exec_cmd *args = data;

	if (args->ext || args->ext_flags)
		return -EINVAL;

	trace_amdxdna_debug_point(current->comm, args->type, "job received");

	switch (args->type) {
	case AMDXDNA_CMD_SUBMIT_EXEC_BUF:
		return amdxdna_drm_submit_execbuf(client, args);
	}

	XDNA_ERR(client->xdna, "Invalid command type %d", args->type);
	return -EINVAL;
}

int amdxdna_drm_wait_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
	struct amdxdna_client *client = filp->driver_priv;
	struct amdxdna_dev *xdna = to_xdna_dev(dev);
	struct amdxdna_drm_wait_cmd *args = data;
	struct amdxdna_hwctx *hwctx;
	int ret, idx;

	XDNA_DBG(xdna, "PID %d ctx %d timeout set %d ms for cmd %llu",
		 client->pid, args->hwctx, args->timeout, args->seq);

	if (!xdna->dev_info->ops->cmd_wait)
		return -EOPNOTSUPP;

	idx = srcu_read_lock(&client->hwctx_srcu);
	hwctx = xa_load(&client->hwctx_xa, args->hwctx);
	if (!hwctx) {
		XDNA_DBG(xdna, "PID %d failed to get ctx %d", client->pid, args->hwctx);
		ret = -EINVAL;
		goto unlock_ctx_srcu;
	}

	ret = xdna->dev_info->ops->cmd_wait(hwctx, args->seq, args->timeout);

	XDNA_DBG(xdna, "PID %d ctx %d cmd %lld wait finished, ret %d",
		 client->pid, args->hwctx, args->seq, ret);

	trace_amdxdna_debug_point(current->comm, args->seq, "job returned to user");

unlock_ctx_srcu:
	srcu_read_unlock(&client->hwctx_srcu, idx);
	return ret;
}