cregit-Linux how code gets into the kernel

Release 4.11 drivers/gpu/drm/amd/scheduler/gpu_scheduler.c

/*
 * Copyright 2015 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 *
 */
#include <linux/kthread.h>
#include <linux/wait.h>
#include <linux/sched.h>
#include <uapi/linux/sched/types.h>
#include <drm/drmP.h>
#include "gpu_scheduler.h"


#define CREATE_TRACE_POINTS
#include "gpu_sched_trace.h"

static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity);
static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
static void amd_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);

/* Initialize a given run queue struct */

static void amd_sched_rq_init(struct amd_sched_rq *rq) { spin_lock_init(&rq->lock); INIT_LIST_HEAD(&rq->entities); rq->current_entity = NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Jammy Zhou2678.79%133.33%
Christian König721.21%266.67%
Total33100.00%3100.00%


static void amd_sched_rq_add_entity(struct amd_sched_rq *rq, struct amd_sched_entity *entity) { if (!list_empty(&entity->list)) return; spin_lock(&rq->lock); list_add_tail(&entity->list, &rq->entities); spin_unlock(&rq->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Jammy Zhou3154.39%125.00%
Christian König1424.56%250.00%
Chunming Zhou1221.05%125.00%
Total57100.00%4100.00%


static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq, struct amd_sched_entity *entity) { if (list_empty(&entity->list)) return; spin_lock(&rq->lock); list_del_init(&entity->list); if (rq->current_entity == entity) rq->current_entity = NULL; spin_unlock(&rq->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Jammy Zhou2843.08%125.00%
Christian König2640.00%250.00%
Chunming Zhou1116.92%125.00%
Total65100.00%4100.00%

/** * Select an entity which could provide a job to run * * @rq The run queue to check. * * Try to find a ready entity, returns NULL if none found. */
static struct amd_sched_entity * amd_sched_rq_select_entity(struct amd_sched_rq *rq) { struct amd_sched_entity *entity; spin_lock(&rq->lock); entity = rq->current_entity; if (entity) { list_for_each_entry_continue(entity, &rq->entities, list) { if (amd_sched_entity_is_ready(entity)) { rq->current_entity = entity; spin_unlock(&rq->lock); return entity; } } } list_for_each_entry(entity, &rq->entities, list) { if (amd_sched_entity_is_ready(entity)) { rq->current_entity = entity; spin_unlock(&rq->lock); return entity; } if (entity == rq->current_entity) break; } spin_unlock(&rq->lock); return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Christian König8363.85%480.00%
Jammy Zhou4736.15%120.00%
Total130100.00%5100.00%

/** * Init a context entity used by scheduler when submit to HW ring. * * @sched The pointer to the scheduler * @entity The pointer to a valid amd_sched_entity * @rq The run queue this entity belongs * @kernel If this is an entity for the kernel * @jobs The max number of jobs in the job queue * * return 0 if succeed. negative error code on failure */
int amd_sched_entity_init(struct amd_gpu_scheduler *sched, struct amd_sched_entity *entity, struct amd_sched_rq *rq, uint32_t jobs) { int r; if (!(sched && entity && rq)) return -EINVAL; memset(entity, 0, sizeof(struct amd_sched_entity)); INIT_LIST_HEAD(&entity->list); entity->rq = rq; entity->sched = sched; spin_lock_init(&entity->queue_lock); r = kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL); if (r) return r; atomic_set(&entity->fence_seq, 0); entity->fence_context = dma_fence_context_alloc(2); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Jammy Zhou8564.89%222.22%
Christian König3929.77%555.56%
Chunming Zhou64.58%111.11%
Chris Wilson10.76%111.11%
Total131100.00%9100.00%

/** * Query if entity is initialized * * @sched Pointer to scheduler instance * @entity The pointer to a valid scheduler entity * * return true if entity is initialized, false otherwise */
static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched, struct amd_sched_entity *entity) { return entity->sched == sched && entity->rq != NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Jammy Zhou2586.21%125.00%
Christian König413.79%375.00%
Total29100.00%4100.00%

/** * Check if entity is idle * * @entity The pointer to a valid scheduler entity * * Return true if entity don't has any unscheduled jobs. */
static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity) { rmb(); if (kfifo_is_empty(&entity->job_queue)) return true; return false; }

Contributors

PersonTokensPropCommitsCommitProp
Jammy Zhou2790.00%133.33%
Christian König310.00%266.67%
Total30100.00%3100.00%

/** * Check if entity is ready * * @entity The pointer to a valid scheduler entity * * Return true if entity could provide a job. */
static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity) { if (kfifo_is_empty(&entity->job_queue)) return false; if (ACCESS_ONCE(entity->dependency)) return false; return true; }

Contributors

PersonTokensPropCommitsCommitProp
Christian König39100.00%1100.00%
Total39100.00%1100.00%

/** * Destroy a context entity * * @sched Pointer to scheduler instance * @entity The pointer to a valid scheduler entity * * Cleanup and free the allocated resources. */
void amd_sched_entity_fini(struct amd_gpu_scheduler *sched, struct amd_sched_entity *entity) { struct amd_sched_rq *rq = entity->rq; if (!amd_sched_entity_is_initialized(sched, entity)) return; /** * The client will not queue more IBs during this fini, consume existing * queued IBs */ wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity)); amd_sched_rq_remove_entity(rq, entity); kfifo_free(&entity->job_queue); }

Contributors

PersonTokensPropCommitsCommitProp
Jammy Zhou5180.95%112.50%
Christian König1219.05%787.50%
Total63100.00%8100.00%


static void amd_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb) { struct amd_sched_entity *entity = container_of(cb, struct amd_sched_entity, cb); entity->dependency = NULL; dma_fence_put(f); amd_sched_wakeup(entity->sched); }

Contributors

PersonTokensPropCommitsCommitProp
Christian König4693.88%266.67%
Chris Wilson36.12%133.33%
Total49100.00%3100.00%


static void amd_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb) { struct amd_sched_entity *entity = container_of(cb, struct amd_sched_entity, cb); entity->dependency = NULL; dma_fence_put(f); }

Contributors

PersonTokensPropCommitsCommitProp
Monk Liu3992.86%150.00%
Chris Wilson37.14%150.00%
Total42100.00%2100.00%


static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity) { struct amd_gpu_scheduler *sched = entity->sched; struct dma_fence * fence = entity->dependency; struct amd_sched_fence *s_fence; if (fence->context == entity->fence_context) { /* We can ignore fences from ourself */ dma_fence_put(entity->dependency); return false; } s_fence = to_amd_sched_fence(fence); if (s_fence && s_fence->sched == sched) { /* * Fence is from the same scheduler, only need to wait for * it to be scheduled */ fence = dma_fence_get(&s_fence->scheduled); dma_fence_put(entity->dependency); entity->dependency = fence; if (!dma_fence_add_callback(fence, &entity->cb, amd_sched_entity_clear_dep)) return true; /* Ignore it when it is already scheduled */ dma_fence_put(fence); return false; } if (!dma_fence_add_callback(entity->dependency, &entity->cb, amd_sched_entity_wakeup)) return true; dma_fence_put(entity->dependency); return false; }

Contributors

PersonTokensPropCommitsCommitProp
Christian König14994.90%266.67%
Chris Wilson85.10%133.33%
Total157100.00%3100.00%


static struct amd_sched_job * amd_sched_entity_pop_job(struct amd_sched_entity *entity) { struct amd_gpu_scheduler *sched = entity->sched; struct amd_sched_job *sched_job; if (!kfifo_out_peek(&entity->job_queue, &sched_job, sizeof(sched_job))) return NULL; while ((entity->dependency = sched->ops->dependency(sched_job))) if (amd_sched_entity_add_dependency_cb(entity)) return NULL; return sched_job; }

Contributors

PersonTokensPropCommitsCommitProp
Christian König7493.67%583.33%
Junwei (Martin) Zhang56.33%116.67%
Total79100.00%6100.00%

/** * Helper to submit a job to the job queue * * @sched_job The pointer to job required to submit * * Returns true if we could submit the job. */
static bool amd_sched_entity_in(struct amd_sched_job *sched_job) { struct amd_gpu_scheduler *sched = sched_job->sched; struct amd_sched_entity *entity = sched_job->s_entity; bool added, first = false; spin_lock(&entity->queue_lock); added = kfifo_in(&entity->job_queue, &sched_job, sizeof(sched_job)) == sizeof(sched_job); if (added && kfifo_len(&entity->job_queue) == sizeof(sched_job)) first = true; spin_unlock(&entity->queue_lock); /* first job wakes up scheduler */ if (first) { /* Add the entity to the run queue */ amd_sched_rq_add_entity(entity->rq, entity); amd_sched_wakeup(sched); } return added; }

Contributors

PersonTokensPropCommitsCommitProp
Christian König4436.36%218.18%
Chunming Zhou3831.40%654.55%
Jammy Zhou2419.83%19.09%
Nicolai Hähnle97.44%19.09%
Junwei (Martin) Zhang64.96%19.09%
Total121100.00%11100.00%

/* job_finish is called after hw fence signaled, and * the job had already been deleted from ring_mirror_list */
static void amd_sched_job_finish(struct work_struct *work) { struct amd_sched_job *s_job = container_of(work, struct amd_sched_job, finish_work); struct amd_gpu_scheduler *sched = s_job->sched; /* remove job from ring_mirror_list */ spin_lock(&sched->job_list_lock); list_del_init(&s_job->node); if (sched->timeout != MAX_SCHEDULE_TIMEOUT) { struct amd_sched_job *next; spin_unlock(&sched->job_list_lock); cancel_delayed_work_sync(&s_job->work_tdr); spin_lock(&sched->job_list_lock); /* queue TDR for next job */ next = list_first_entry_or_null(&sched->ring_mirror_list, struct amd_sched_job, node); if (next) schedule_delayed_work(&next->work_tdr, sched->timeout); } spin_unlock(&sched->job_list_lock); sched->ops->free_job(s_job); }

Contributors

PersonTokensPropCommitsCommitProp
Monk Liu7755.00%233.33%
Christian König6345.00%466.67%
Total140100.00%6100.00%


static void amd_sched_job_finish_cb(struct dma_fence *f, struct dma_fence_cb *cb) { struct amd_sched_job *job = container_of(cb, struct amd_sched_job, finish_cb); schedule_work(&job->finish_work); }

Contributors

PersonTokensPropCommitsCommitProp
Christian König3692.31%133.33%
Chris Wilson25.13%133.33%
Monk Liu12.56%133.33%
Total39100.00%3100.00%


static void amd_sched_job_begin(struct amd_sched_job *s_job) { struct amd_gpu_scheduler *sched = s_job->sched; spin_lock(&sched->job_list_lock); list_add_tail(&s_job->node, &sched->ring_mirror_list); if (sched->timeout != MAX_SCHEDULE_TIMEOUT && list_first_entry_or_null(&sched->ring_mirror_list, struct amd_sched_job, node) == s_job) schedule_delayed_work(&s_job->work_tdr, sched->timeout); spin_unlock(&sched->job_list_lock); }

Contributors

PersonTokensPropCommitsCommitProp
Monk Liu5464.29%125.00%
Christian König3035.71%375.00%
Total84100.00%4100.00%


static void amd_sched_job_timedout(struct work_struct *work) { struct amd_sched_job *job = container_of(work, struct amd_sched_job, work_tdr.work); job->sched->ops->timedout_job(job); }

Contributors

PersonTokensPropCommitsCommitProp
Christian König39100.00%1100.00%
Total39100.00%1100.00%


void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched) { struct amd_sched_job *s_job; spin_lock(&sched->job_list_lock); list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) { if (dma_fence_remove_callback(s_job->s_fence->parent, &s_job->s_fence->cb)) { dma_fence_put(s_job->s_fence->parent); s_job->s_fence->parent = NULL; } } atomic_set(&sched->hw_rq_count, 0); spin_unlock(&sched->job_list_lock); }

Contributors

PersonTokensPropCommitsCommitProp
Chunming Zhou8697.73%266.67%
Chris Wilson22.27%133.33%
Total88100.00%3100.00%


void amd_sched_job_recovery(struct amd_gpu_scheduler *sched) { struct amd_sched_job *s_job, *tmp; int r; spin_lock(&sched->job_list_lock); s_job = list_first_entry_or_null(&sched->ring_mirror_list, struct amd_sched_job, node); if (s_job && sched->timeout != MAX_SCHEDULE_TIMEOUT) schedule_delayed_work(&s_job->work_tdr, sched->timeout); list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { struct amd_sched_fence *s_fence = s_job->s_fence; struct dma_fence *fence; spin_unlock(&sched->job_list_lock); fence = sched->ops->run_job(s_job); atomic_inc(&sched->hw_rq_count); if (fence) { s_fence->parent = dma_fence_get(fence); r = dma_fence_add_callback(fence, &s_fence->cb, amd_sched_process_job); if (r == -ENOENT) amd_sched_process_job(fence, &s_fence->cb); else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); dma_fence_put(fence); } else { DRM_ERROR("Failed to run job!\n"); amd_sched_process_job(NULL, &s_fence->cb); } spin_lock(&sched->job_list_lock); } spin_unlock(&sched->job_list_lock); }

Contributors

PersonTokensPropCommitsCommitProp
Chunming Zhou20695.37%360.00%
Christian König62.78%120.00%
Chris Wilson41.85%120.00%
Total216100.00%5100.00%

/** * Submit a job to the job queue * * @sched_job The pointer to job required to submit * * Returns 0 for success, negative error code otherwise. */
void amd_sched_entity_push_job(struct amd_sched_job *sched_job) { struct amd_sched_entity *entity = sched_job->s_entity; trace_amd_sched_job(sched_job); dma_fence_add_callback(&sched_job->s_fence->finished, &sched_job->finish_cb, amd_sched_job_finish_cb); wait_event(entity->sched->job_scheduled, amd_sched_entity_in(sched_job)); }

Contributors

PersonTokensPropCommitsCommitProp
Christian König3970.91%654.55%
Monk Liu1120.00%19.09%
Nicolai Hähnle23.64%19.09%
Chunming Zhou11.82%19.09%
Jammy Zhou11.82%19.09%
Chris Wilson11.82%19.09%
Total55100.00%11100.00%

/* init a sched_job with basic field */
int amd_sched_job_init(struct amd_sched_job *job, struct amd_gpu_scheduler *sched, struct amd_sched_entity *entity, void *owner) { job->sched = sched; job->s_entity = entity; job->s_fence = amd_sched_fence_create(entity, owner); if (!job->s_fence) return -ENOMEM; INIT_WORK(&job->finish_work, amd_sched_job_finish); INIT_LIST_HEAD(&job->node); INIT_DELAYED_WORK(&job->work_tdr, amd_sched_job_timedout); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Monk Liu6876.40%350.00%
Christian König2123.60%350.00%
Total89100.00%6100.00%

/** * Return ture if we can push more jobs to the hw. */
static bool amd_sched_ready(struct amd_gpu_scheduler *sched) { return atomic_read(&sched->hw_rq_count) < sched->hw_submission_limit; }

Contributors

PersonTokensPropCommitsCommitProp
Christian König24100.00%1100.00%
Total24100.00%1100.00%

/** * Wake up the scheduler when it is ready */
static void amd_sched_wakeup(struct amd_gpu_scheduler *sched) { if (amd_sched_ready(sched)) wake_up_interruptible(&sched->wake_up_worker); }

Contributors

PersonTokensPropCommitsCommitProp
Christian König26100.00%2100.00%
Total26100.00%2100.00%

/** * Select next entity to process */
static struct amd_sched_entity * amd_sched_select_entity(struct amd_gpu_scheduler *sched) { struct amd_sched_entity *entity; int i; if (!amd_sched_ready(sched)) return NULL; /* Kernel run queue has higher priority than normal run queue*/ for (i = 0; i < AMD_SCHED_MAX_PRIORITY; i++) { entity = amd_sched_rq_select_entity(&sched->sched_rq[i]); if (entity) break; } return entity; }

Contributors

PersonTokensPropCommitsCommitProp
Christian König4666.67%266.67%
Chunming Zhou2333.33%133.33%
Total69100.00%3100.00%


static void amd_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb) { struct amd_sched_fence *s_fence = container_of(cb, struct amd_sched_fence, cb); struct amd_gpu_scheduler *sched = s_fence->sched; atomic_dec(&sched->hw_rq_count); amd_sched_fence_finished(s_fence); trace_amd_sched_process_job(s_fence); dma_fence_put(&s_fence->finished); wake_up_interruptible(&sched->wake_up_worker); }

Contributors

PersonTokensPropCommitsCommitProp
Christian König5979.73%666.67%
Chunming Zhou1216.22%222.22%
Chris Wilson34.05%111.11%
Total74100.00%9100.00%


static bool amd_sched_blocked(struct amd_gpu_scheduler *sched) { if (kthread_should_park()) { kthread_parkme(); return true; } return false; }

Contributors

PersonTokensPropCommitsCommitProp
Chunming Zhou27100.00%1100.00%
Total27100.00%1100.00%


static int amd_sched_main(void *param) { struct sched_param sparam = {.sched_priority = 1}; struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param; int r, count; sched_setscheduler(current, SCHED_FIFO, &sparam); while (!kthread_should_stop()) { struct amd_sched_entity *entity = NULL; struct amd_sched_fence *s_fence; struct amd_sched_job *sched_job; struct dma_fence *fence; wait_event_interruptible(sched->wake_up_worker, (!amd_sched_blocked(sched) && (entity = amd_sched_select_entity(sched))) || kthread_should_stop()); if (!entity) continue; sched_job = amd_sched_entity_pop_job(entity); if (!sched_job) continue; s_fence = sched_job->s_fence; atomic_inc(&sched->hw_rq_count); amd_sched_job_begin(sched_job); fence = sched->ops->run_job(sched_job); amd_sched_fence_scheduled(s_fence); if (fence) { s_fence->parent = dma_fence_get(fence); r = dma_fence_add_callback(fence, &s_fence->cb, amd_sched_process_job); if (r == -ENOENT) amd_sched_process_job(fence, &s_fence->cb); else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); dma_fence_put(fence); } else { DRM_ERROR("Failed to run job!\n"); amd_sched_process_job(NULL, &s_fence->cb); } count = kfifo_out(&entity->job_queue, &sched_job, sizeof(sched_job)); WARN_ON(count != sizeof(sched_job)); wake_up(&sched->job_scheduled); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Christian König13648.75%1157.89%
Jammy Zhou8831.54%210.53%
Chunming Zhou3111.11%315.79%
Alex Deucher134.66%15.26%
Junwei (Martin) Zhang72.51%15.26%
Chris Wilson41.43%15.26%
Total279100.00%19100.00%

/** * Init a gpu scheduler instance * * @sched The pointer to the scheduler * @ops The backend operations for this scheduler. * @hw_submissions Number of hw submissions to do. * @name Name used for debugging * * Return 0 on success, otherwise error code. */
int amd_sched_init(struct amd_gpu_scheduler *sched, const struct amd_sched_backend_ops *ops, unsigned hw_submission, long timeout, const char *name) { int i; sched->ops = ops; sched->hw_submission_limit = hw_submission; sched->name = name; sched->timeout = timeout; for (i = 0; i < AMD_SCHED_MAX_PRIORITY; i++) amd_sched_rq_init(&sched->sched_rq[i]); init_waitqueue_head(&sched->wake_up_worker); init_waitqueue_head(&sched->job_scheduled); INIT_LIST_HEAD(&sched->ring_mirror_list); spin_lock_init(&sched->job_list_lock); atomic_set(&sched->hw_rq_count, 0); /* Each scheduler will run on a seperate kernel thread */ sched->thread = kthread_run(amd_sched_main, sched, sched->name); if (IS_ERR(sched->thread)) { DRM_ERROR("Failed to create scheduler for %s.\n", name); return PTR_ERR(sched->thread); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Jammy Zhou7243.64%214.29%
Christian König3621.82%642.86%
Chunming Zhou3118.79%321.43%
Monk Liu169.70%17.14%
Junwei (Martin) Zhang95.45%17.14%
Nils Wallménius10.61%17.14%
Total165100.00%14100.00%

/** * Destroy a gpu scheduler * * @sched The pointer to the scheduler */
void amd_sched_fini(struct amd_gpu_scheduler *sched) { if (sched->thread) kthread_stop(sched->thread); }

Contributors

PersonTokensPropCommitsCommitProp
Jammy Zhou1565.22%133.33%
Dave Airlie626.09%133.33%
Christian König28.70%133.33%
Total23100.00%3100.00%


Overall Contributors

PersonTokensPropCommitsCommitProp
Christian König113444.68%3650.70%
Jammy Zhou53821.20%45.63%
Chunming Zhou50319.82%1825.35%
Monk Liu26710.52%57.04%
Chris Wilson331.30%11.41%
Junwei (Martin) Zhang291.14%22.82%
Alex Deucher130.51%11.41%
Nicolai Hähnle110.43%11.41%
Dave Airlie60.24%11.41%
Ingo Molnar30.12%11.41%
Nils Wallménius10.04%11.41%
Total2538100.00%71100.00%
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.