Release 4.11 drivers/gpu/drm/i915/i915_sw_fence.c

Directory: drivers/gpu/drm/i915
/*
 * (C) Copyright 2016 Intel Corporation
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */

#include <linux/slab.h>
#include <linux/dma-fence.h>
#include <linux/reservation.h>

#include "i915_sw_fence.h"


#define I915_SW_FENCE_FLAG_ALLOC BIT(3) 
/* after WQ_FLAG_* for safety */

static DEFINE_SPINLOCK(i915_sw_fence_lock);

enum {
	
DEBUG_FENCE_IDLE = 0,
	
DEBUG_FENCE_NOTIFY,
};

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS



static void *i915_sw_fence_debug_hint(void *addr)
{
	return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 31 100.00% 1 100.00%
Total 31 100.00% 1 100.00%


static struct debug_obj_descr i915_sw_fence_debug_descr = {
	.name = "i915_sw_fence",
	.debug_hint = i915_sw_fence_debug_hint,
};



static inline void debug_fence_init(struct i915_sw_fence *fence)
{
	debug_object_init(fence, &i915_sw_fence_debug_descr);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
	debug_object_activate(fence, &i915_sw_fence_debug_descr);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 30 100.00% 1 100.00%
Total 30 100.00% 1 100.00%



static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void debug_fence_free(struct i915_sw_fence *fence)
{
	debug_object_free(fence, &i915_sw_fence_debug_descr);
	smp_wmb(); /* flush the change in state before reallocation */
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 24 100.00% 2 100.00%
Total 24 100.00% 2 100.00%



static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%

#else



static inline void debug_fence_init(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 17 100.00% 1 100.00%
Total 17 100.00% 1 100.00%



static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void debug_fence_free(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%

#endif



static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
				  enum i915_sw_fence_notify state)
{
	i915_sw_fence_notify_t fn;

	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	return fn(fence, state);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 39 100.00% 1 100.00%
Total 39 100.00% 1 100.00%

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS


void i915_sw_fence_fini(struct i915_sw_fence *fence)
{
	debug_fence_free(fence);
}
Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 15 100.00% 1 100.00%
Total 15 100.00% 1 100.00%

#endif



static void i915_sw_fence_release(struct kref *kref)
{
	struct i915_sw_fence *fence = container_of(kref, typeof(*fence), kref);

	WARN_ON(atomic_read(&fence->pending) > 0);
	debug_fence_destroy(fence);

	if (fence->flags & I915_SW_FENCE_MASK) {
		__i915_sw_fence_notify(fence, FENCE_FREE);
	} else {
		i915_sw_fence_fini(fence);
		kfree(fence);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 77 100.00% 2 100.00%
Total 77 100.00% 2 100.00%



static void i915_sw_fence_put(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);
	kref_put(&fence->kref, i915_sw_fence_release);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 26 100.00% 2 100.00%
Total 26 100.00% 2 100.00%



static struct i915_sw_fence *i915_sw_fence_get(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);
	kref_get(&fence->kref);
	return fence;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 29 100.00% 2 100.00%
Total 29 100.00% 2 100.00%



static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
					struct list_head *continuation)
{
	wait_queue_head_t *x = &fence->wait;
	wait_queue_t *pos, *next;
	unsigned long flags;

	debug_fence_deactivate(fence);
	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */

	/*
         * To prevent unbounded recursion as we traverse the graph of
         * i915_sw_fences, we move the task_list from this, the next ready
         * fence, to the tail of the original fence's task_list
         * (and so added to the list to be woken).
         */

	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	if (continuation) {
		list_for_each_entry_safe(pos, next, &x->task_list, task_list) {
			if (pos->func == autoremove_wake_function)
				pos->func(pos, TASK_NORMAL, 0, continuation);
			else
				list_move_tail(&pos->task_list, continuation);
		}
	} else {
		LIST_HEAD(extra);

		do {
			list_for_each_entry_safe(pos, next,
						 &x->task_list, task_list)
				pos->func(pos, TASK_NORMAL, 0, &extra);

			if (list_empty(&extra))
				break;

			list_splice_tail_init(&extra, &x->task_list);
		} while (1);
	}
	spin_unlock_irqrestore(&x->lock, flags);

	debug_fence_assert(fence);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 195 100.00% 2 100.00%
Total 195 100.00% 2 100.00%



static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
				     struct list_head *continuation)
{
	debug_fence_assert(fence);

	if (!atomic_dec_and_test(&fence->pending))
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);

	if (fence->flags & I915_SW_FENCE_MASK &&
	    __i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);

	__i915_sw_fence_wake_up_all(fence, continuation);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 76 100.00% 2 100.00%
Total 76 100.00% 2 100.00%



static void i915_sw_fence_complete(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);

	if (WARN_ON(i915_sw_fence_done(fence)))
		return;

	__i915_sw_fence_complete(fence, NULL);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 34 100.00% 2 100.00%
Total 34 100.00% 2 100.00%



static void i915_sw_fence_await(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);
	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 29 100.00% 2 100.00%
Total 29 100.00% 2 100.00%



void __i915_sw_fence_init(struct i915_sw_fence *fence,
			  i915_sw_fence_notify_t fn,
			  const char *name,
			  struct lock_class_key *key)
{
	BUG_ON((unsigned long)fn & ~I915_SW_FENCE_MASK);

	debug_fence_init(fence);

	__init_waitqueue_head(&fence->wait, name, key);
	kref_init(&fence->kref);
	atomic_set(&fence->pending, 1);
	fence->flags = (unsigned long)fn;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 80 100.00% 3 100.00%
Total 80 100.00% 3 100.00%



static void __i915_sw_fence_commit(struct i915_sw_fence *fence)
{
	i915_sw_fence_complete(fence);
	i915_sw_fence_put(fence);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 21 100.00% 2 100.00%
Total 21 100.00% 2 100.00%



void i915_sw_fence_commit(struct i915_sw_fence *fence)
{
	debug_fence_activate(fence);
	__i915_sw_fence_commit(fence);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void *key)
{
	list_del(&wq->task_list);
	__i915_sw_fence_complete(wq->private, key);
	i915_sw_fence_put(wq->private);
	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
		kfree(wq);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 60 100.00% 2 100.00%
Total 60 100.00% 2 100.00%



static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				    const struct i915_sw_fence * const signaler)
{
	wait_queue_t *wq;

	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return false;

	if (fence == signaler)
		return true;

	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		if (__i915_sw_fence_check_if_after(wq->private, signaler))
			return true;
	}

	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 82 100.00% 1 100.00%
Total 82 100.00% 1 100.00%



static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
{
	wait_queue_t *wq;

	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return;

	list_for_each_entry(wq, &fence->wait.task_list, task_list) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		__i915_sw_fence_clear_checked_bit(wq->private);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 55 100.00% 1 100.00%
Total 55 100.00% 1 100.00%



static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				  const struct i915_sw_fence * const signaler)
{
	unsigned long flags;
	bool err;

	if (!IS_ENABLED(CONFIG_I915_SW_FENCE_CHECK_DAG))
		return false;

	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	err = __i915_sw_fence_check_if_after(fence, signaler);
	__i915_sw_fence_clear_checked_bit(fence);
	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);

	return err;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 69 100.00% 1 100.00%
Total 69 100.00% 1 100.00%



static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
					  struct i915_sw_fence *signaler,
					  wait_queue_t *wq, gfp_t gfp)
{
	unsigned long flags;
	int pending;

	debug_fence_assert(fence);

	if (i915_sw_fence_done(signaler))
		return 0;

	debug_fence_assert(signaler);

	/* The dependency graph must be acyclic. */
	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
		return -EINVAL;

	pending = 0;
	if (!wq) {
		wq = kmalloc(sizeof(*wq), gfp);
		if (!wq) {
			if (!gfpflags_allow_blocking(gfp))
				return -ENOMEM;

			i915_sw_fence_wait(signaler);
			return 0;
		}

		pending |= I915_SW_FENCE_FLAG_ALLOC;
	}

	INIT_LIST_HEAD(&wq->task_list);
	wq->flags = pending;
	wq->func = i915_sw_fence_wake;
	wq->private = i915_sw_fence_get(fence);

	i915_sw_fence_await(fence);

	spin_lock_irqsave(&signaler->wait.lock, flags);
	if (likely(!i915_sw_fence_done(signaler))) {
		__add_wait_queue_tail(&signaler->wait, wq);
		pending = 1;
	} else {
		i915_sw_fence_wake(wq, 0, 0, NULL);
		pending = 0;
	}
	spin_unlock_irqrestore(&signaler->wait.lock, flags);

	return pending;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 228 100.00% 3 100.00%
Total 228 100.00% 3 100.00%



int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
				 struct i915_sw_fence *signaler,
				 wait_queue_t *wq)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 31 100.00% 1 100.00%
Total 31 100.00% 1 100.00%



int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
				     struct i915_sw_fence *signaler,
				     gfp_t gfp)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 30 100.00% 2 100.00%
Total 30 100.00% 2 100.00%


struct i915_sw_dma_fence_cb {
	
struct dma_fence_cb base;
	
struct i915_sw_fence *fence;
	
struct dma_fence *dma;
	
struct timer_list timer;
};



static void timer_i915_sw_fence_wake(unsigned long data)
{
	struct i915_sw_dma_fence_cb *cb = (struct i915_sw_dma_fence_cb *)data;

	printk(KERN_WARNING "asynchronous wait on fence %s:%s:%x timed out\n",
	       cb->dma->ops->get_driver_name(cb->dma),
	       cb->dma->ops->get_timeline_name(cb->dma),
	       cb->dma->seqno);
	dma_fence_put(cb->dma);
	cb->dma = NULL;

	__i915_sw_fence_commit(cb->fence);
	cb->timer.function = NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 88 100.00% 3 100.00%
Total 88 100.00% 3 100.00%



static void dma_i915_sw_fence_wake(struct dma_fence *dma,
				   struct dma_fence_cb *data)
{
	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);

	del_timer_sync(&cb->timer);
	if (cb->timer.function)
		__i915_sw_fence_commit(cb->fence);
	dma_fence_put(cb->dma);

	kfree(cb);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 69 100.00% 3 100.00%
Total 69 100.00% 3 100.00%



int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
				  struct dma_fence *dma,
				  unsigned long timeout,
				  gfp_t gfp)
{
	struct i915_sw_dma_fence_cb *cb;
	int ret;

	debug_fence_assert(fence);

	if (dma_fence_is_signaled(dma))
		return 0;

	cb = kmalloc(sizeof(*cb), gfp);
	if (!cb) {
		if (!gfpflags_allow_blocking(gfp))
			return -ENOMEM;

		return dma_fence_wait(dma, false);
	}

	cb->fence = i915_sw_fence_get(fence);
	i915_sw_fence_await(fence);

	cb->dma = NULL;
	__setup_timer(&cb->timer,
		      timer_i915_sw_fence_wake, (unsigned long)cb,
		      TIMER_IRQSAFE);
	if (timeout) {
		cb->dma = dma_fence_get(dma);
		mod_timer(&cb->timer, round_jiffies_up(jiffies + timeout));
	}

	ret = dma_fence_add_callback(dma, &cb->base, dma_i915_sw_fence_wake);
	if (ret == 0) {
		ret = 1;
	} else {
		dma_i915_sw_fence_wake(dma, &cb->base);
		if (ret == -ENOENT) /* fence already signaled */
			ret = 0;
	}

	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 207 100.00% 3 100.00%
Total 207 100.00% 3 100.00%



int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
				    struct reservation_object *resv,
				    const struct dma_fence_ops *exclude,
				    bool write,
				    unsigned long timeout,
				    gfp_t gfp)
{
	struct dma_fence *excl;
	int ret = 0, pending;

	debug_fence_assert(fence);

	if (write) {
		struct dma_fence **shared;
		unsigned int count, i;

		ret = reservation_object_get_fences_rcu(resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			if (shared[i]->ops == exclude)
				continue;

			pending = i915_sw_fence_await_dma_fence(fence,
								shared[i],
								timeout,
								gfp);
			if (pending < 0) {
				ret = pending;
				break;
			}

			ret |= pending;
		}

		for (i = 0; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);
	} else {
		excl = reservation_object_get_excl_rcu(resv);
	}

	if (ret >= 0 && excl && excl->ops != exclude) {
		pending = i915_sw_fence_await_dma_fence(fence,
							excl,
							timeout,
							gfp);
		if (pending < 0)
			ret = pending;
		else
			ret |= pending;
	}

	dma_fence_put(excl);

	return ret;
}
Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 237 100.00% 3 100.00%
Total 237 100.00% 3 100.00%

Overall Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 2150 100.00% 6 100.00%
Total 2150 100.00% 6 100.00%
Directory: drivers/gpu/drm/i915

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.
Person	Tokens	Prop	Commits	CommitProp
Chris Wilson	31	100.00%	1	100.00%
Total	31	100.00%	1	100.00%
cregit-Linux how code gets into the kernel

Release 4.11 drivers/gpu/drm/i915/i915_sw_fence.c

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Overall Contributors
