Release 4.17 drivers/gpu/drm/i915/i915_sw_fence.c

Directory: drivers/gpu/drm/i915
/*
 * (C) Copyright 2016 Intel Corporation
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */

#include <linux/slab.h>
#include <linux/dma-fence.h>
#include <linux/irq_work.h>
#include <linux/reservation.h>

#include "i915_sw_fence.h"
#include "i915_selftest.h"


#define I915_SW_FENCE_FLAG_ALLOC BIT(3) 
/* after WQ_FLAG_* for safety */

static DEFINE_SPINLOCK(i915_sw_fence_lock);


enum {
	
DEBUG_FENCE_IDLE = 0,
	
DEBUG_FENCE_NOTIFY,
};

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS



static void *i915_sw_fence_debug_hint(void *addr)
{
	return (void *)(((struct i915_sw_fence *)addr)->flags & I915_SW_FENCE_MASK);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 31 100.00% 1 100.00%
Total 31 100.00% 1 100.00%


static struct debug_obj_descr i915_sw_fence_debug_descr = {
	.name = "i915_sw_fence",
	.debug_hint = i915_sw_fence_debug_hint,
};



static inline void debug_fence_init(struct i915_sw_fence *fence)
{
	debug_object_init(fence, &i915_sw_fence_debug_descr);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
{
	debug_object_init_on_stack(fence, &i915_sw_fence_debug_descr);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
	debug_object_activate(fence, &i915_sw_fence_debug_descr);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
	debug_object_active_state(fence, &i915_sw_fence_debug_descr, old, new);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 30 100.00% 1 100.00%
Total 30 100.00% 1 100.00%



static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
	debug_object_deactivate(fence, &i915_sw_fence_debug_descr);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
	debug_object_destroy(fence, &i915_sw_fence_debug_descr);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void debug_fence_free(struct i915_sw_fence *fence)
{
	debug_object_free(fence, &i915_sw_fence_debug_descr);
	smp_wmb(); /* flush the change in state before reallocation */
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 24 100.00% 2 100.00%
Total 24 100.00% 2 100.00%



static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
	debug_object_assert_init(fence, &i915_sw_fence_debug_descr);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%

#else



static inline void debug_fence_init(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void debug_fence_init_onstack(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void debug_fence_activate(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void debug_fence_set_state(struct i915_sw_fence *fence,
					 int old, int new)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 17 100.00% 1 100.00%
Total 17 100.00% 1 100.00%



static inline void debug_fence_deactivate(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void debug_fence_destroy(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void debug_fence_free(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void debug_fence_assert(struct i915_sw_fence *fence)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%

#endif



static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
				  enum i915_sw_fence_notify state)
{
	i915_sw_fence_notify_t fn;

	fn = (i915_sw_fence_notify_t)(fence->flags & I915_SW_FENCE_MASK);
	return fn(fence, state);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 39 100.00% 1 100.00%
Total 39 100.00% 1 100.00%

#ifdef CONFIG_DRM_I915_SW_FENCE_DEBUG_OBJECTS


void i915_sw_fence_fini(struct i915_sw_fence *fence)
{
	debug_fence_free(fence);
}
Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 15 100.00% 1 100.00%
Total 15 100.00% 1 100.00%

#endif



static void __i915_sw_fence_wake_up_all(struct i915_sw_fence *fence,
					struct list_head *continuation)
{
	wait_queue_head_t *x = &fence->wait;
	wait_queue_entry_t *pos, *next;
	unsigned long flags;

	debug_fence_deactivate(fence);
	atomic_set_release(&fence->pending, -1); /* 0 -> -1 [done] */

	/*
         * To prevent unbounded recursion as we traverse the graph of
         * i915_sw_fences, we move the entry list from this, the next ready
         * fence, to the tail of the original fence's entry list
         * (and so added to the list to be woken).
         */

	spin_lock_irqsave_nested(&x->lock, flags, 1 + !!continuation);
	if (continuation) {
		list_for_each_entry_safe(pos, next, &x->head, entry) {
			if (pos->func == autoremove_wake_function)
				pos->func(pos, TASK_NORMAL, 0, continuation);
			else
				list_move_tail(&pos->entry, continuation);
		}
	} else {
		LIST_HEAD(extra);

		do {
			list_for_each_entry_safe(pos, next, &x->head, entry)
				pos->func(pos, TASK_NORMAL, 0, &extra);

			if (list_empty(&extra))
				break;

			list_splice_tail_init(&extra, &x->head);
		} while (1);
	}
	spin_unlock_irqrestore(&x->lock, flags);

	debug_fence_assert(fence);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 187 95.90% 2 50.00%
Ingo Molnar 8 4.10% 2 50.00%
Total 195 100.00% 4 100.00%



static void __i915_sw_fence_complete(struct i915_sw_fence *fence,
				     struct list_head *continuation)
{
	debug_fence_assert(fence);

	if (!atomic_dec_and_test(&fence->pending))
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_IDLE, DEBUG_FENCE_NOTIFY);

	if (__i915_sw_fence_notify(fence, FENCE_COMPLETE) != NOTIFY_DONE)
		return;

	debug_fence_set_state(fence, DEBUG_FENCE_NOTIFY, DEBUG_FENCE_IDLE);

	__i915_sw_fence_wake_up_all(fence, continuation);

	debug_fence_destroy(fence);
	__i915_sw_fence_notify(fence, FENCE_FREE);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 82 100.00% 3 100.00%
Total 82 100.00% 3 100.00%



static void i915_sw_fence_complete(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);

	if (WARN_ON(i915_sw_fence_done(fence)))
		return;

	__i915_sw_fence_complete(fence, NULL);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 34 100.00% 2 100.00%
Total 34 100.00% 2 100.00%



static void i915_sw_fence_await(struct i915_sw_fence *fence)
{
	debug_fence_assert(fence);
	WARN_ON(atomic_inc_return(&fence->pending) <= 1);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 29 100.00% 2 100.00%
Total 29 100.00% 2 100.00%



void __i915_sw_fence_init(struct i915_sw_fence *fence,
			  i915_sw_fence_notify_t fn,
			  const char *name,
			  struct lock_class_key *key)
{
	BUG_ON(!fn || (unsigned long)fn & ~I915_SW_FENCE_MASK);

	debug_fence_init(fence);

	__init_waitqueue_head(&fence->wait, name, key);
	atomic_set(&fence->pending, 1);
	fence->flags = (unsigned long)fn;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 75 100.00% 4 100.00%
Total 75 100.00% 4 100.00%



void i915_sw_fence_commit(struct i915_sw_fence *fence)
{
	debug_fence_activate(fence);
	i915_sw_fence_complete(fence);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 20 100.00% 2 100.00%
Total 20 100.00% 2 100.00%



static int i915_sw_fence_wake(wait_queue_entry_t *wq, unsigned mode, int flags, void *key)
{
	list_del(&wq->entry);
	__i915_sw_fence_complete(wq->private, key);

	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
		kfree(wq);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 51 96.23% 2 50.00%
Ingo Molnar 2 3.77% 2 50.00%
Total 53 100.00% 4 100.00%



static bool __i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				    const struct i915_sw_fence * const signaler)
{
	wait_queue_entry_t *wq;

	if (__test_and_set_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return false;

	if (fence == signaler)
		return true;

	list_for_each_entry(wq, &fence->wait.head, entry) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		if (__i915_sw_fence_check_if_after(wq->private, signaler))
			return true;
	}

	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 79 96.34% 1 33.33%
Ingo Molnar 3 3.66% 2 66.67%
Total 82 100.00% 3 100.00%



static void __i915_sw_fence_clear_checked_bit(struct i915_sw_fence *fence)
{
	wait_queue_entry_t *wq;

	if (!__test_and_clear_bit(I915_SW_FENCE_CHECKED_BIT, &fence->flags))
		return;

	list_for_each_entry(wq, &fence->wait.head, entry) {
		if (wq->func != i915_sw_fence_wake)
			continue;

		__i915_sw_fence_clear_checked_bit(wq->private);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 52 94.55% 1 33.33%
Ingo Molnar 3 5.45% 2 66.67%
Total 55 100.00% 3 100.00%



static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
				  const struct i915_sw_fence * const signaler)
{
	unsigned long flags;
	bool err;

	if (!IS_ENABLED(CONFIG_DRM_I915_SW_FENCE_CHECK_DAG))
		return false;

	spin_lock_irqsave(&i915_sw_fence_lock, flags);
	err = __i915_sw_fence_check_if_after(fence, signaler);
	__i915_sw_fence_clear_checked_bit(fence);
	spin_unlock_irqrestore(&i915_sw_fence_lock, flags);

	return err;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 69 100.00% 2 100.00%
Total 69 100.00% 2 100.00%



static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
					  struct i915_sw_fence *signaler,
					  wait_queue_entry_t *wq, gfp_t gfp)
{
	unsigned long flags;
	int pending;

	debug_fence_assert(fence);
	might_sleep_if(gfpflags_allow_blocking(gfp));

	if (i915_sw_fence_done(signaler))
		return 0;

	debug_fence_assert(signaler);

	/* The dependency graph must be acyclic. */
	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
		return -EINVAL;

	pending = 0;
	if (!wq) {
		wq = kmalloc(sizeof(*wq), gfp);
		if (!wq) {
			if (!gfpflags_allow_blocking(gfp))
				return -ENOMEM;

			i915_sw_fence_wait(signaler);
			return 0;
		}

		pending |= I915_SW_FENCE_FLAG_ALLOC;
	}

	INIT_LIST_HEAD(&wq->entry);
	wq->flags = pending;
	wq->func = i915_sw_fence_wake;
	wq->private = fence;

	i915_sw_fence_await(fence);

	spin_lock_irqsave(&signaler->wait.lock, flags);
	if (likely(!i915_sw_fence_done(signaler))) {
		__add_wait_queue_entry_tail(&signaler->wait, wq);
		pending = 1;
	} else {
		i915_sw_fence_wake(wq, 0, 0, NULL);
		pending = 0;
	}
	spin_unlock_irqrestore(&signaler->wait.lock, flags);

	return pending;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 230 98.71% 4 66.67%
Ingo Molnar 3 1.29% 2 33.33%
Total 233 100.00% 6 100.00%



int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
				 struct i915_sw_fence *signaler,
				 wait_queue_entry_t *wq)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 30 96.77% 1 50.00%
Ingo Molnar 1 3.23% 1 50.00%
Total 31 100.00% 2 100.00%



int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
				     struct i915_sw_fence *signaler,
				     gfp_t gfp)
{
	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 30 100.00% 1 100.00%
Total 30 100.00% 1 100.00%


struct i915_sw_dma_fence_cb {
	
struct dma_fence_cb base;
	
struct i915_sw_fence *fence;
};


struct i915_sw_dma_fence_cb_timer {
	
struct i915_sw_dma_fence_cb base;
	
struct dma_fence *dma;
	
struct timer_list timer;
	
struct irq_work work;
	
struct rcu_head rcu;
};



static void dma_i915_sw_fence_wake(struct dma_fence *dma,
				   struct dma_fence_cb *data)
{
	struct i915_sw_dma_fence_cb *cb = container_of(data, typeof(*cb), base);

	i915_sw_fence_complete(cb->fence);
	kfree(cb);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 46 100.00% 1 100.00%
Total 46 100.00% 1 100.00%



static void timer_i915_sw_fence_wake(struct timer_list *t)
{
	struct i915_sw_dma_fence_cb_timer *cb = from_timer(cb, t, timer);
	struct i915_sw_fence *fence;

	fence = xchg(&cb->base.fence, NULL);
	if (!fence)
		return;

	pr_warn("asynchronous wait on fence %s:%s:%x timed out\n",
		cb->dma->ops->get_driver_name(cb->dma),
		cb->dma->ops->get_timeline_name(cb->dma),
		cb->dma->seqno);

	i915_sw_fence_complete(fence);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 79 85.87% 4 66.67%
Kees Cook 12 13.04% 1 16.67%
Joe Perches 1 1.09% 1 16.67%
Total 92 100.00% 6 100.00%



static void dma_i915_sw_fence_wake_timer(struct dma_fence *dma,
					 struct dma_fence_cb *data)
{
	struct i915_sw_dma_fence_cb_timer *cb =
		container_of(data, typeof(*cb), base.base);
	struct i915_sw_fence *fence;

	fence = xchg(&cb->base.fence, NULL);
	if (fence)
		i915_sw_fence_complete(fence);

	irq_work_queue(&cb->work);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 72 100.00% 5 100.00%
Total 72 100.00% 5 100.00%



static void irq_i915_sw_fence_work(struct irq_work *wrk)
{
	struct i915_sw_dma_fence_cb_timer *cb =
		container_of(wrk, typeof(*cb), work);

	del_timer_sync(&cb->timer);
	dma_fence_put(cb->dma);

	kfree_rcu(cb, rcu);
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 51 100.00% 5 100.00%
Total 51 100.00% 5 100.00%



int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
				  struct dma_fence *dma,
				  unsigned long timeout,
				  gfp_t gfp)
{
	struct i915_sw_dma_fence_cb *cb;
	dma_fence_func_t func;
	int ret;

	debug_fence_assert(fence);
	might_sleep_if(gfpflags_allow_blocking(gfp));

	if (dma_fence_is_signaled(dma))
		return 0;

	cb = kmalloc(timeout ?
		     sizeof(struct i915_sw_dma_fence_cb_timer) :
		     sizeof(struct i915_sw_dma_fence_cb),
		     gfp);
	if (!cb) {
		if (!gfpflags_allow_blocking(gfp))
			return -ENOMEM;

		return dma_fence_wait(dma, false);
	}

	cb->fence = fence;
	i915_sw_fence_await(fence);

	func = dma_i915_sw_fence_wake;
	if (timeout) {
		struct i915_sw_dma_fence_cb_timer *timer =
			container_of(cb, typeof(*timer), base);

		timer->dma = dma_fence_get(dma);
		init_irq_work(&timer->work, irq_i915_sw_fence_work);

		timer_setup(&timer->timer,
			    timer_i915_sw_fence_wake, TIMER_IRQSAFE);
		mod_timer(&timer->timer, round_jiffies_up(jiffies + timeout));

		func = dma_i915_sw_fence_wake_timer;
	}

	ret = dma_fence_add_callback(dma, &cb->base, func);
	if (ret == 0) {
		ret = 1;
	} else {
		func(dma, &cb->base);
		if (ret == -ENOENT) /* fence already signaled */
			ret = 0;
	}

	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 247 100.00% 7 100.00%
Total 247 100.00% 7 100.00%



int i915_sw_fence_await_reservation(struct i915_sw_fence *fence,
				    struct reservation_object *resv,
				    const struct dma_fence_ops *exclude,
				    bool write,
				    unsigned long timeout,
				    gfp_t gfp)
{
	struct dma_fence *excl;
	int ret = 0, pending;

	debug_fence_assert(fence);
	might_sleep_if(gfpflags_allow_blocking(gfp));

	if (write) {
		struct dma_fence **shared;
		unsigned int count, i;

		ret = reservation_object_get_fences_rcu(resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			if (shared[i]->ops == exclude)
				continue;

			pending = i915_sw_fence_await_dma_fence(fence,
								shared[i],
								timeout,
								gfp);
			if (pending < 0) {
				ret = pending;
				break;
			}

			ret |= pending;
		}

		for (i = 0; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);
	} else {
		excl = reservation_object_get_excl_rcu(resv);
	}

	if (ret >= 0 && excl && excl->ops != exclude) {
		pending = i915_sw_fence_await_dma_fence(fence,
							excl,
							timeout,
							gfp);
		if (pending < 0)
			ret = pending;
		else
			ret |= pending;
	}

	dma_fence_put(excl);

	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 245 100.00% 4 100.00%
Total 245 100.00% 4 100.00%

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/lib_sw_fence.c"
#include "selftests/i915_sw_fence.c"
#endif
Overall Contributors
Person Tokens Prop Commits CommitProp
Chris Wilson 2183 98.51% 14 77.78%
Ingo Molnar 20 0.90% 2 11.11%
Kees Cook 12 0.54% 1 5.56%
Joe Perches 1 0.05% 1 5.56%
Total 2216 100.00% 18 100.00%
Directory: drivers/gpu/drm/i915

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.
Person	Tokens	Prop	Commits	CommitProp
Chris Wilson	31	100.00%	1	100.00%
Total	31	100.00%	1	100.00%
cregit-Linux how code gets into the kernel

Release 4.17 drivers/gpu/drm/i915/i915_sw_fence.c

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Overall Contributors
