Release 4.15 kernel/sched/rt.c

Directory: kernel/sched
// SPDX-License-Identifier: GPL-2.0
/*
 * Real-Time Scheduling Class (mapped to the SCHED_FIFO and SCHED_RR
 * policies)
 */

#include "sched.h"

#include <linux/slab.h>
#include <linux/irq_work.h>


int sched_rr_timeslice = RR_TIMESLICE;

int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE;

static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);


struct rt_bandwidth def_rt_bandwidth;



static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
{
	struct rt_bandwidth *rt_b =
		container_of(timer, struct rt_bandwidth, rt_period_timer);
	int idle = 0;
	int overrun;

	raw_spin_lock(&rt_b->rt_runtime_lock);
	for (;;) {
		overrun = hrtimer_forward_now(timer, rt_b->rt_period);
		if (!overrun)
			break;

		raw_spin_unlock(&rt_b->rt_runtime_lock);
		idle = do_sched_rt_period_timer(rt_b, overrun);
		raw_spin_lock(&rt_b->rt_runtime_lock);
	}
	if (idle)
		rt_b->rt_period_active = 0;
	raw_spin_unlock(&rt_b->rt_runtime_lock);

	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 117 100.00% 3 100.00%
Total 117 100.00% 3 100.00%



void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
{
	rt_b->rt_period = ns_to_ktime(period);
	rt_b->rt_runtime = runtime;

	raw_spin_lock_init(&rt_b->rt_runtime_lock);

	hrtimer_init(&rt_b->rt_period_timer,
			CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	rt_b->rt_period_timer.function = sched_rt_period_timer;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 59 100.00% 1 100.00%
Total 59 100.00% 1 100.00%



static void start_rt_bandwidth(struct rt_bandwidth *rt_b)
{
	if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
		return;

	raw_spin_lock(&rt_b->rt_runtime_lock);
	if (!rt_b->rt_period_active) {
		rt_b->rt_period_active = 1;
		/*
                 * SCHED_DEADLINE updates the bandwidth, as a run away
                 * RT task with a DL task could hog a CPU. But DL does
                 * not reset the period. If a deadline task was running
                 * without an RT task running, it can cause RT tasks to
                 * throttle when they start up. Kick the timer right away
                 * to update the period.
                 */
		hrtimer_forward_now(&rt_b->rt_period_timer, ns_to_ktime(0));
		hrtimer_start_expires(&rt_b->rt_period_timer, HRTIMER_MODE_ABS_PINNED);
	}
	raw_spin_unlock(&rt_b->rt_runtime_lock);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 74 93.67% 2 66.67%
Steven Rostedt 5 6.33% 1 33.33%
Total 79 100.00% 3 100.00%



void init_rt_rq(struct rt_rq *rt_rq)
{
	struct rt_prio_array *array;
	int i;

	array = &rt_rq->active;
	for (i = 0; i < MAX_RT_PRIO; i++) {
		INIT_LIST_HEAD(array->queue + i);
		__clear_bit(i, array->bitmap);
	}
	/* delimiter for bitsearch: */
	__set_bit(MAX_RT_PRIO, array->bitmap);

#if defined CONFIG_SMP
	rt_rq->highest_prio.curr = MAX_RT_PRIO;
	rt_rq->highest_prio.next = MAX_RT_PRIO;
	rt_rq->rt_nr_migratory = 0;
	rt_rq->overloaded = 0;
	plist_head_init(&rt_rq->pushable_tasks);
#endif /* CONFIG_SMP */
	/* We start is dequeued state, because no RT tasks are queued */
	rt_rq->rt_queued = 0;

	rt_rq->rt_time = 0;
	rt_rq->rt_throttled = 0;
	rt_rq->rt_runtime = 0;
	raw_spin_lock_init(&rt_rq->rt_runtime_lock);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 134 93.06% 1 33.33%
Kirill V Tkhai 7 4.86% 1 33.33%
Steven Rostedt 3 2.08% 1 33.33%
Total 144 100.00% 3 100.00%

#ifdef CONFIG_RT_GROUP_SCHED


static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b)
{
	hrtimer_cancel(&rt_b->rt_period_timer);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 19 100.00% 1 100.00%
Total 19 100.00% 1 100.00%


#define rt_entity_is_task(rt_se) (!(rt_se)->my_q)



static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
{
#ifdef CONFIG_SCHED_DEBUG
	WARN_ON_ONCE(!rt_entity_is_task(rt_se));
#endif
	return container_of(rt_se, struct task_struct, rt);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 39 100.00% 1 100.00%
Total 39 100.00% 1 100.00%



static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
{
	return rt_rq->rq;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 19 100.00% 1 100.00%
Total 19 100.00% 1 100.00%



static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
{
	return rt_se->rt_rq;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 19 100.00% 1 100.00%
Total 19 100.00% 1 100.00%



static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
{
	struct rt_rq *rt_rq = rt_se->rt_rq;

	return rt_rq->rq;
}

Contributors
Person Tokens Prop Commits CommitProp
Kirill V Tkhai 28 100.00% 1 100.00%
Total 28 100.00% 1 100.00%



void free_rt_sched_group(struct task_group *tg)
{
	int i;

	if (tg->rt_se)
		destroy_rt_bandwidth(&tg->rt_bandwidth);

	for_each_possible_cpu(i) {
		if (tg->rt_rq)
			kfree(tg->rt_rq[i]);
		if (tg->rt_se)
			kfree(tg->rt_se[i]);
	}

	kfree(tg->rt_rq);
	kfree(tg->rt_se);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 79 100.00% 1 100.00%
Total 79 100.00% 1 100.00%



void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
		struct sched_rt_entity *rt_se, int cpu,
		struct sched_rt_entity *parent)
{
	struct rq *rq = cpu_rq(cpu);

	rt_rq->highest_prio.curr = MAX_RT_PRIO;
	rt_rq->rt_nr_boosted = 0;
	rt_rq->rq = rq;
	rt_rq->tg = tg;

	tg->rt_rq[cpu] = rt_rq;
	tg->rt_se[cpu] = rt_se;

	if (!rt_se)
		return;

	if (!parent)
		rt_se->rt_rq = &rq->rt;
	else
		rt_se->rt_rq = parent->my_q;

	rt_se->my_q = rt_rq;
	rt_se->parent = parent;
	INIT_LIST_HEAD(&rt_se->run_list);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 131 100.00% 2 100.00%
Total 131 100.00% 2 100.00%



int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
{
	struct rt_rq *rt_rq;
	struct sched_rt_entity *rt_se;
	int i;

	tg->rt_rq = kzalloc(sizeof(rt_rq) * nr_cpu_ids, GFP_KERNEL);
	if (!tg->rt_rq)
		goto err;
	tg->rt_se = kzalloc(sizeof(rt_se) * nr_cpu_ids, GFP_KERNEL);
	if (!tg->rt_se)
		goto err;

	init_rt_bandwidth(&tg->rt_bandwidth,
			ktime_to_ns(def_rt_bandwidth.rt_period), 0);

	for_each_possible_cpu(i) {
		rt_rq = kzalloc_node(sizeof(struct rt_rq),
				     GFP_KERNEL, cpu_to_node(i));
		if (!rt_rq)
			goto err;

		rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
				     GFP_KERNEL, cpu_to_node(i));
		if (!rt_se)
			goto err_free_rq;

		init_rt_rq(rt_rq);
		rt_rq->rt_runtime = tg->rt_bandwidth.rt_runtime;
		init_tg_rt_entry(tg, rt_rq, rt_se, i, parent->rt_se[i]);
	}

	return 1;

err_free_rq:
	kfree(rt_rq);
err:
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 184 90.64% 2 66.67%
Gregory Haskins 19 9.36% 1 33.33%
Total 203 100.00% 3 100.00%

#else /* CONFIG_RT_GROUP_SCHED */


#define rt_entity_is_task(rt_se) (1)



static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)
{
	return container_of(rt_se, struct task_struct, rt);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 25 100.00% 1 100.00%
Total 25 100.00% 1 100.00%



static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
{
	return container_of(rt_rq, struct rq, rt);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 25 100.00% 1 100.00%
Total 25 100.00% 1 100.00%



static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se)
{
	struct task_struct *p = rt_task_of(rt_se);

	return task_rq(p);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 21 70.00% 1 50.00%
Kirill V Tkhai 9 30.00% 1 50.00%
Total 30 100.00% 2 100.00%



static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se)
{
	struct rq *rq = rq_of_rt_se(rt_se);

	return &rq->rt;
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 15 50.00% 1 50.00%
Kirill V Tkhai 15 50.00% 1 50.00%
Total 30 100.00% 2 100.00%



void free_rt_sched_group(struct task_group *tg) { }

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 9 100.00% 1 100.00%
Total 9 100.00% 1 100.00%



int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
{
	return 1;
}
Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 18 100.00% 1 100.00%
Total 18 100.00% 1 100.00%

#endif /* CONFIG_RT_GROUP_SCHED */

#ifdef CONFIG_SMP

static void pull_rt_task(struct rq *this_rq);



static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
{
	/* Try to pull RT tasks here if we lower this rq's prio */
	return rq->rt.highest_prio.curr > prev->prio;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 31 100.00% 1 100.00%
Total 31 100.00% 1 100.00%



static inline int rt_overloaded(struct rq *rq)
{
	return atomic_read(&rq->rd->rto_count);
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 15 65.22% 1 50.00%
Gregory Haskins 8 34.78% 1 50.00%
Total 23 100.00% 2 100.00%



static inline void rt_set_overload(struct rq *rq)
{
	if (!rq->online)
		return;

	cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);
	/*
         * Make sure the mask is visible before we set
         * the overload count. That is checked to determine
         * if we should look at the mask. It would be a shame
         * if we looked at the mask, but the mask was not
         * updated yet.
         *
         * Matched by the barrier in pull_rt_task().
         */
	smp_wmb();
	atomic_inc(&rq->rd->rto_count);
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 27 57.45% 1 20.00%
Gregory Haskins 17 36.17% 2 40.00%
Peter Zijlstra 2 4.26% 1 20.00%
Rusty Russell 1 2.13% 1 20.00%
Total 47 100.00% 5 100.00%



static inline void rt_clear_overload(struct rq *rq)
{
	if (!rq->online)
		return;

	/* the order here really doesn't matter */
	atomic_dec(&rq->rd->rto_count);
	cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 24 54.55% 1 20.00%
Gregory Haskins 19 43.18% 3 60.00%
Rusty Russell 1 2.27% 1 20.00%
Total 44 100.00% 5 100.00%



static void update_rt_migration(struct rt_rq *rt_rq)
{
	if (rt_rq->rt_nr_migratory && rt_rq->rt_nr_total > 1) {
		if (!rt_rq->overloaded) {
			rt_set_overload(rq_of_rt_rq(rt_rq));
			rt_rq->overloaded = 1;
		}
	} else if (rt_rq->overloaded) {
		rt_clear_overload(rq_of_rt_rq(rt_rq));
		rt_rq->overloaded = 0;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 70 98.59% 4 80.00%
Peter Zijlstra 1 1.41% 1 20.00%
Total 71 100.00% 5 100.00%



static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
{
	struct task_struct *p;

	if (!rt_entity_is_task(rt_se))
		return;

	p = rt_task_of(rt_se);
	rt_rq = &rq_of_rt_rq(rt_rq)->rt;

	rt_rq->rt_nr_total++;
	if (p->nr_cpus_allowed > 1)
		rt_rq->rt_nr_migratory++;

	update_rt_migration(rt_rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 47 67.14% 3 50.00%
Gregory Haskins 21 30.00% 2 33.33%
Ingo Molnar 2 2.86% 1 16.67%
Total 70 100.00% 6 100.00%



static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
{
	struct task_struct *p;

	if (!rt_entity_is_task(rt_se))
		return;

	p = rt_task_of(rt_se);
	rt_rq = &rq_of_rt_rq(rt_rq)->rt;

	rt_rq->rt_nr_total--;
	if (p->nr_cpus_allowed > 1)
		rt_rq->rt_nr_migratory--;

	update_rt_migration(rt_rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 37 52.86% 2 50.00%
Gregory Haskins 31 44.29% 1 25.00%
Ingo Molnar 2 2.86% 1 25.00%
Total 70 100.00% 4 100.00%



static inline int has_pushable_tasks(struct rq *rq)
{
	return !plist_head_empty(&rq->rt.pushable_tasks);
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 24 100.00% 1 100.00%
Total 24 100.00% 1 100.00%

static DEFINE_PER_CPU(struct callback_head, rt_push_head);
static DEFINE_PER_CPU(struct callback_head, rt_pull_head);

static void push_rt_tasks(struct rq *);
static void pull_rt_task(struct rq *);



static inline void queue_push_tasks(struct rq *rq)
{
	if (!has_pushable_tasks(rq))
		return;

	queue_balance_callback(rq, &per_cpu(rt_push_head, rq->cpu), push_rt_tasks);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 38 100.00% 3 100.00%
Total 38 100.00% 3 100.00%



static inline void queue_pull_task(struct rq *rq)
{
	queue_balance_callback(rq, &per_cpu(rt_pull_head, rq->cpu), pull_rt_task);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 29 100.00% 1 100.00%
Total 29 100.00% 1 100.00%



static void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
{
	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);
	plist_node_init(&p->pushable_tasks, p->prio);
	plist_add(&p->pushable_tasks, &rq->rt.pushable_tasks);

	/* Update the highest prio pushable task */
	if (p->prio < rq->rt.highest_prio.next)
		rq->rt.highest_prio.next = p->prio;
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 58 68.24% 2 66.67%
Steven Rostedt 27 31.76% 1 33.33%
Total 85 100.00% 3 100.00%



static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
{
	plist_del(&p->pushable_tasks, &rq->rt.pushable_tasks);

	/* Update the new highest prio pushable task */
	if (has_pushable_tasks(rq)) {
		p = plist_first_entry(&rq->rt.pushable_tasks,
				      struct task_struct, pushable_tasks);
		rq->rt.highest_prio.next = p->prio;
	} else
		rq->rt.highest_prio.next = MAX_RT_PRIO;
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 39 48.15% 1 33.33%
Gregory Haskins 30 37.04% 1 33.33%
Ingo Molnar 12 14.81% 1 33.33%
Total 81 100.00% 3 100.00%

#else



static inline void enqueue_pushable_task(struct rq *rq, struct task_struct *p)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 15 93.75% 2 66.67%
Peter Zijlstra 1 6.25% 1 33.33%
Total 16 100.00% 3 100.00%



static inline void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 15 93.75% 2 66.67%
Peter Zijlstra 1 6.25% 1 33.33%
Total 16 100.00% 3 100.00%


static inline

void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 11 68.75% 1 33.33%
Peter Zijlstra 5 31.25% 2 66.67%
Total 16 100.00% 3 100.00%


static inline

void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 9 56.25% 1 33.33%
Peter Zijlstra 7 43.75% 2 66.67%
Total 16 100.00% 3 100.00%



static inline bool need_pull_rt_task(struct rq *rq, struct task_struct *prev)
{
	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void pull_rt_task(struct rq *this_rq)
{
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 11 100.00% 2 100.00%
Total 11 100.00% 2 100.00%



static inline void queue_push_tasks(struct rq *rq)
{
}
Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 11 100.00% 2 100.00%
Total 11 100.00% 2 100.00%

#endif /* CONFIG_SMP */

static void enqueue_top_rt_rq(struct rt_rq *rt_rq);
static void dequeue_top_rt_rq(struct rt_rq *rt_rq);



static inline int on_rt_rq(struct sched_rt_entity *rt_se)
{
	return rt_se->on_rq;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 17 100.00% 2 100.00%
Total 17 100.00% 2 100.00%

#ifdef CONFIG_RT_GROUP_SCHED



static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
{
	if (!rt_rq->tg)
		return RUNTIME_INF;

	return rt_rq->rt_runtime;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 27 100.00% 3 100.00%
Total 27 100.00% 3 100.00%



static inline u64 sched_rt_period(struct rt_rq *rt_rq)
{
	return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 24 100.00% 3 100.00%
Total 24 100.00% 3 100.00%


typedef struct task_group *rt_rq_iter_t;



static inline struct task_group *next_task_group(struct task_group *tg)
{
	do {
		tg = list_entry_rcu(tg->list.next,
			typeof(struct task_group), list);
	} while (&tg->list != &task_groups && task_group_is_autogroup(tg));

	if (&tg->list == &task_groups)
		tg = NULL;

	return tg;
}

Contributors
Person Tokens Prop Commits CommitProp
Yong Zhang 69 100.00% 1 100.00%
Total 69 100.00% 1 100.00%


#define for_each_rt_rq(rt_rq, iter, rq)					\
	for (iter = container_of(&task_groups, typeof(*iter), list);    \
                (iter = next_task_group(iter)) &&                       \
                (rt_rq = iter->rt_rq[cpu_of(rq)]);)


#define for_each_sched_rt_entity(rt_se) \
	for (; rt_se; rt_se = rt_se->parent)



static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
{
	return rt_se->my_q;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 19 100.00% 1 100.00%
Total 19 100.00% 1 100.00%

static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);



static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
{
	struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;
	struct rq *rq = rq_of_rt_rq(rt_rq);
	struct sched_rt_entity *rt_se;

	int cpu = cpu_of(rq);

	rt_se = rt_rq->tg->rt_se[cpu];

	if (rt_rq->rt_nr_running) {
		if (!rt_se)
			enqueue_top_rt_rq(rt_rq);
		else if (!on_rt_rq(rt_se))
			enqueue_rt_entity(rt_se, 0);

		if (rt_rq->highest_prio.curr < curr->prio)
			resched_curr(rq);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 47 43.52% 4 36.36%
Kirill V Tkhai 23 21.30% 2 18.18%
Dario Faggioli 19 17.59% 1 9.09%
Yong Zhang 8 7.41% 1 9.09%
Balbir Singh 8 7.41% 1 9.09%
Gregory Haskins 2 1.85% 1 9.09%
Thomas Gleixner 1 0.93% 1 9.09%
Total 108 100.00% 11 100.00%



static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
{
	struct sched_rt_entity *rt_se;
	int cpu = cpu_of(rq_of_rt_rq(rt_rq));

	rt_se = rt_rq->tg->rt_se[cpu];

	if (!rt_se)
		dequeue_top_rt_rq(rt_rq);
	else if (on_rt_rq(rt_se))
		dequeue_rt_entity(rt_se, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 33 52.38% 3 50.00%
Balbir Singh 12 19.05% 1 16.67%
Kirill V Tkhai 10 15.87% 1 16.67%
Yong Zhang 8 12.70% 1 16.67%
Total 63 100.00% 6 100.00%



static inline int rt_rq_throttled(struct rt_rq *rt_rq)
{
	return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
}

Contributors
Person Tokens Prop Commits CommitProp
Kirill V Tkhai 22 100.00% 1 100.00%
Total 22 100.00% 1 100.00%



static int rt_se_boosted(struct sched_rt_entity *rt_se)
{
	struct rt_rq *rt_rq = group_rt_rq(rt_se);
	struct task_struct *p;

	if (rt_rq)
		return !!rt_rq->rt_nr_boosted;

	p = rt_task_of(rt_se);
	return p->prio != p->normal_prio;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 53 100.00% 1 100.00%
Total 53 100.00% 1 100.00%

#ifdef CONFIG_SMP


static inline const struct cpumask *sched_rt_period_mask(void)
{
	return this_rq()->rd->span;
}
Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 15 75.00% 1 33.33%
Rusty Russell 4 20.00% 1 33.33%
Nathan Zimmer 1 5.00% 1 33.33%
Total 20 100.00% 3 100.00%

#else


static inline const struct cpumask *sched_rt_period_mask(void)
{
	return cpu_online_mask;
}
Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 10 66.67% 1 50.00%
Rusty Russell 5 33.33% 1 50.00%
Total 15 100.00% 2 100.00%

#endif


static inline

struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
{
	return container_of(rt_b, struct task_group, rt_bandwidth)->rt_rq[cpu];
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 33 100.00% 1 100.00%
Total 33 100.00% 1 100.00%



static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
{
	return &rt_rq->tg->rt_bandwidth;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 22 100.00% 1 100.00%
Total 22 100.00% 1 100.00%

#else /* !CONFIG_RT_GROUP_SCHED */



static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
{
	return rt_rq->rt_runtime;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 17 100.00% 3 100.00%
Total 17 100.00% 3 100.00%



static inline u64 sched_rt_period(struct rt_rq *rt_rq)
{
	return ktime_to_ns(def_rt_bandwidth.rt_period);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 20 100.00% 3 100.00%
Total 20 100.00% 3 100.00%


typedef struct rt_rq *rt_rq_iter_t;


#define for_each_rt_rq(rt_rq, iter, rq) \
	for ((void) iter, rt_rq = &rq->rt; rt_rq; rt_rq = NULL)


#define for_each_sched_rt_entity(rt_se) \
	for (; rt_se; rt_se = NULL)



static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
{
	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 17 100.00% 1 100.00%
Total 17 100.00% 1 100.00%



static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
{
	struct rq *rq = rq_of_rt_rq(rt_rq);

	if (!rt_rq->rt_nr_running)
		return;

	enqueue_top_rt_rq(rt_rq);
	resched_curr(rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Kirill V Tkhai 17 42.50% 2 40.00%
John Blackwood 13 32.50% 1 20.00%
Peter Zijlstra 10 25.00% 2 40.00%
Total 40 100.00% 5 100.00%



static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
{
	dequeue_top_rt_rq(rt_rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 10 58.82% 2 66.67%
Kirill V Tkhai 7 41.18% 1 33.33%
Total 17 100.00% 3 100.00%



static inline int rt_rq_throttled(struct rt_rq *rt_rq)
{
	return rt_rq->rt_throttled;
}

Contributors
Person Tokens Prop Commits CommitProp
Kirill V Tkhai 17 100.00% 1 100.00%
Total 17 100.00% 1 100.00%



static inline const struct cpumask *sched_rt_period_mask(void)
{
	return cpu_online_mask;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 10 66.67% 1 50.00%
Rusty Russell 5 33.33% 1 50.00%
Total 15 100.00% 2 100.00%


static inline

struct rt_rq *sched_rt_period_rt_rq(struct rt_bandwidth *rt_b, int cpu)
{
	return &cpu_rq(cpu)->rt;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 26 100.00% 1 100.00%
Total 26 100.00% 1 100.00%



static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
{
	return &def_rt_bandwidth;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 18 100.00% 1 100.00%
Total 18 100.00% 1 100.00%

#endif /* CONFIG_RT_GROUP_SCHED */



bool sched_rt_bandwidth_account(struct rt_rq *rt_rq)
{
	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);

	return (hrtimer_active(&rt_b->rt_period_timer) ||
		rt_rq->rt_time < rt_b->rt_runtime);
}

Contributors
Person Tokens Prop Commits CommitProp
Juri Lelli 39 100.00% 1 100.00%
Total 39 100.00% 1 100.00%

#ifdef CONFIG_SMP
/*
 * We ran out of runtime, see if we can borrow some from our neighbours.
 */


static void do_balance_runtime(struct rt_rq *rt_rq)
{
	struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
	struct root_domain *rd = rq_of_rt_rq(rt_rq)->rd;
	int i, weight;
	u64 rt_period;

	weight = cpumask_weight(rd->span);

	raw_spin_lock(&rt_b->rt_runtime_lock);
	rt_period = ktime_to_ns(rt_b->rt_period);
	for_each_cpu(i, rd->span) {
		struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
		s64 diff;

		if (iter == rt_rq)
			continue;

		raw_spin_lock(&iter->rt_runtime_lock);
		/*
                 * Either all rqs have inf runtime and there's nothing to steal
                 * or __disable_runtime() below sets a specific rq to inf to
                 * indicate its been disabled and disalow stealing.
                 */
		if (iter->rt_runtime == RUNTIME_INF)
			goto next;

		/*
                 * From runqueues with spare time, take 1/n part of their
                 * spare time, but no more than our period.
                 */
		diff = iter->rt_runtime - iter->rt_time;
		if (diff > 0) {
			diff = div_u64((u64)diff, weight);
			if (rt_rq->rt_runtime + diff > rt_period)
				diff = rt_period - rt_rq->rt_runtime;
			iter->rt_runtime -= diff;
			rt_rq->rt_runtime += diff;
			if (rt_rq->rt_runtime == rt_period) {
				raw_spin_unlock(&iter->rt_runtime_lock);
				break;
			}
		}
next:
		raw_spin_unlock(&iter->rt_runtime_lock);
	}
	raw_spin_unlock(&rt_b->rt_runtime_lock);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 205 95.35% 5 55.56%
Thomas Gleixner 5 2.33% 1 11.11%
Rusty Russell 2 0.93% 1 11.11%
Shawn Bohrer 2 0.93% 1 11.11%
Juri Lelli 1 0.47% 1 11.11%
Total 215 100.00% 9 100.00%

/*
 * Ensure this RQ takes back all the runtime it lend to its neighbours.
 */


static void __disable_runtime(struct rq *rq)
{
	struct root_domain *rd = rq->rd;
	rt_rq_iter_t iter;
	struct rt_rq *rt_rq;

	if (unlikely(!scheduler_running))
		return;

	for_each_rt_rq(rt_rq, iter, rq) {
		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
		s64 want;
		int i;

		raw_spin_lock(&rt_b->rt_runtime_lock);
		raw_spin_lock(&rt_rq->rt_runtime_lock);
		/*
                 * Either we're all inf and nobody needs to borrow, or we're
                 * already disabled and thus have nothing to do, or we have
                 * exactly the right amount of runtime to take out.
                 */
		if (rt_rq->rt_runtime == RUNTIME_INF ||
				rt_rq->rt_runtime == rt_b->rt_runtime)
			goto balanced;
		raw_spin_unlock(&rt_rq->rt_runtime_lock);

		/*
                 * Calculate the difference between what we started out with
                 * and what we current have, that's the amount of runtime
                 * we lend and now have to reclaim.
                 */
		want = rt_b->rt_runtime - rt_rq->rt_runtime;

		/*
                 * Greedy reclaim, take back as much as we can.
                 */
		for_each_cpu(i, rd->span) {
			struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
			s64 diff;

			/*
                         * Can't reclaim from ourselves or disabled runqueues.
                         */
			if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)
				continue;

			raw_spin_lock(&iter->rt_runtime_lock);
			if (want > 0) {
				diff = min_t(s64, iter->rt_runtime, want);
				iter->rt_runtime -= diff;
				want -= diff;
			} else {
				iter->rt_runtime -= want;
				want -= want;
			}
			raw_spin_unlock(&iter->rt_runtime_lock);

			if (!want)
				break;
		}

		raw_spin_lock(&rt_rq->rt_runtime_lock);
		/*
                 * We cannot be left wanting - that would mean some runtime
                 * leaked out of the system.
                 */
		BUG_ON(want);
balanced:
		/*
                 * Disable all the borrow logic by pretending we have inf
                 * runtime - in which case borrowing doesn't make sense.
                 */
		rt_rq->rt_runtime = RUNTIME_INF;
		rt_rq->rt_throttled = 0;
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
		raw_spin_unlock(&rt_b->rt_runtime_lock);

		/* Make rt_rq available for pick_next_task() */
		sched_rt_rq_enqueue(rt_rq);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 246 90.11% 7 58.33%
Thomas Gleixner 8 2.93% 1 8.33%
Peter Boonstoppel 6 2.20% 1 8.33%
Cheng Xu 6 2.20% 1 8.33%
Kirill Tkhai 6 2.20% 1 8.33%
Rusty Russell 1 0.37% 1 8.33%
Total 273 100.00% 12 100.00%



static void __enable_runtime(struct rq *rq)
{
	rt_rq_iter_t iter;
	struct rt_rq *rt_rq;

	if (unlikely(!scheduler_running))
		return;

	/*
         * Reset each runqueue's bandwidth settings
         */
	for_each_rt_rq(rt_rq, iter, rq) {
		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);

		raw_spin_lock(&rt_b->rt_runtime_lock);
		raw_spin_lock(&rt_rq->rt_runtime_lock);
		rt_rq->rt_runtime = rt_b->rt_runtime;
		rt_rq->rt_time = 0;
		rt_rq->rt_throttled = 0;
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
		raw_spin_unlock(&rt_b->rt_runtime_lock);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 75 74.26% 4 40.00%
Dhaval Giani 8 7.92% 1 10.00%
Cheng Xu 6 5.94% 1 10.00%
Yanmin Zhang 6 5.94% 1 10.00%
Thomas Gleixner 4 3.96% 1 10.00%
Srivatsa Vaddagiri 1 0.99% 1 10.00%
Ingo Molnar 1 0.99% 1 10.00%
Total 101 100.00% 10 100.00%



static void balance_runtime(struct rt_rq *rt_rq)
{
	if (!sched_feat(RT_RUNTIME_SHARE))
		return;

	if (rt_rq->rt_time > rt_rq->rt_runtime) {
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
		do_balance_runtime(rt_rq);
		raw_spin_lock(&rt_rq->rt_runtime_lock);
	}
}
Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 48 90.57% 6 66.67%
Juri Lelli 2 3.77% 1 11.11%
Thomas Gleixner 2 3.77% 1 11.11%
Ingo Molnar 1 1.89% 1 11.11%
Total 53 100.00% 9 100.00%

#else /* !CONFIG_SMP */


static inline void balance_runtime(struct rt_rq *rt_rq) {}
Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 6 54.55% 1 25.00%
Peter Zijlstra 3 27.27% 2 50.00%
Juri Lelli 2 18.18% 1 25.00%
Total 11 100.00% 4 100.00%

#endif /* CONFIG_SMP */



static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
{
	int i, idle = 1, throttled = 0;
	const struct cpumask *span;

	span = sched_rt_period_mask();
#ifdef CONFIG_RT_GROUP_SCHED
	/*
         * FIXME: isolated CPUs should really leave the root task group,
         * whether they are isolcpus or were isolated via cpusets, lest
         * the timer run on a CPU which does not service all runqueues,
         * potentially leaving other CPUs indefinitely throttled.  If
         * isolation is really required, the user will turn the throttle
         * off to kill the perturbations it causes anyway.  Meanwhile,
         * this maintains functionality for boot and/or troubleshooting.
         */
	if (rt_b == &root_task_group.rt_bandwidth)
		span = cpu_online_mask;
#endif
	for_each_cpu(i, span) {
		int enqueue = 0;
		struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
		struct rq *rq = rq_of_rt_rq(rt_rq);
		int skip;

		/*
                 * When span == cpu_online_mask, taking each rq->lock
                 * can be time-consuming. Try to avoid it when possible.
                 */
		raw_spin_lock(&rt_rq->rt_runtime_lock);
		skip = !rt_rq->rt_time && !rt_rq->rt_nr_running;
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
		if (skip)
			continue;

		raw_spin_lock(&rq->lock);
		if (rt_rq->rt_time) {
			u64 runtime;

			raw_spin_lock(&rt_rq->rt_runtime_lock);
			if (rt_rq->rt_throttled)
				balance_runtime(rt_rq);
			runtime = rt_rq->rt_runtime;
			rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
			if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
				rt_rq->rt_throttled = 0;
				enqueue = 1;

				/*
                                 * When we're idle and a woken (rt) task is
                                 * throttled check_preempt_curr() will set
                                 * skip_update and the time between the wakeup
                                 * and this unthrottle will get accounted as
                                 * 'runtime'.
                                 */
				if (rt_rq->rt_nr_running && rq->curr == rq->idle)
					rq_clock_skip_update(rq, false);
			}
			if (rt_rq->rt_time || rt_rq->rt_nr_running)
				idle = 0;
			raw_spin_unlock(&rt_rq->rt_runtime_lock);
		} else if (rt_rq->rt_nr_running) {
			idle = 0;
			if (!rt_rq_throttled(rt_rq))
				enqueue = 1;
		}
		if (rt_rq->rt_throttled)
			throttled = 1;

		if (enqueue)
			sched_rt_rq_enqueue(rt_rq);
		raw_spin_unlock(&rq->lock);
	}

	if (!throttled && (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF))
		return 1;

	return idle;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 220 66.87% 7 38.89%
Dave Kleikamp 37 11.25% 1 5.56%
Mike Galbraith 35 10.64% 2 11.11%
Balbir Singh 14 4.26% 1 5.56%
Steven Rostedt 13 3.95% 3 16.67%
Rusty Russell 5 1.52% 1 5.56%
Thomas Gleixner 4 1.22% 2 11.11%
Gregory Haskins 1 0.30% 1 5.56%
Total 329 100.00% 18 100.00%



static inline int rt_se_prio(struct sched_rt_entity *rt_se)
{
#ifdef CONFIG_RT_GROUP_SCHED
	struct rt_rq *rt_rq = group_rt_rq(rt_se);

	if (rt_rq)
		return rt_rq->highest_prio.curr;
#endif

	return rt_task_of(rt_se)->prio;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 43 93.48% 2 50.00%
Gregory Haskins 2 4.35% 1 25.00%
Steven Rostedt 1 2.17% 1 25.00%
Total 46 100.00% 4 100.00%



static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
{
	u64 runtime = sched_rt_runtime(rt_rq);

	if (rt_rq->rt_throttled)
		return rt_rq_throttled(rt_rq);

	if (runtime >= sched_rt_period(rt_rq))
		return 0;

	balance_runtime(rt_rq);
	runtime = sched_rt_runtime(rt_rq);
	if (runtime == RUNTIME_INF)
		return 0;

	if (rt_rq->rt_time > runtime) {
		struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);

		/*
                 * Don't actually throttle groups that have no runtime assigned
                 * but accrue some time due to boosting.
                 */
		if (likely(rt_b->rt_runtime)) {
			rt_rq->rt_throttled = 1;
			printk_deferred_once("sched: RT throttling activated\n");
		} else {
			/*
                         * In case we did anyway, make it go away,
                         * replenishment is a joke, since it will replenish us
                         * with exactly 0 ns.
                         */
			rt_rq->rt_time = 0;
		}

		if (rt_rq_throttled(rt_rq)) {
			sched_rt_rq_dequeue(rt_rq);
			return 1;
		}
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 119 86.86% 9 60.00%
Steven Rostedt 11 8.03% 2 13.33%
Thomas Gleixner 4 2.92% 1 6.67%
John Stultz 1 0.73% 1 6.67%
Gregory Haskins 1 0.73% 1 6.67%
Hai Shan 1 0.73% 1 6.67%
Total 137 100.00% 15 100.00%

/*
 * Update the current task's runtime statistics. Skip current tasks that
 * are not in our scheduling class.
 */


static void update_curr_rt(struct rq *rq)
{
	struct task_struct *curr = rq->curr;
	struct sched_rt_entity *rt_se = &curr->rt;
	u64 delta_exec;

	if (curr->sched_class != &rt_sched_class)
		return;

	delta_exec = rq_clock_task(rq) - curr->se.exec_start;
	if (unlikely((s64)delta_exec <= 0))
		return;

	/* Kick cpufreq (see the comment in kernel/sched/sched.h). */
	cpufreq_update_util(rq, SCHED_CPUFREQ_RT);

	schedstat_set(curr->se.statistics.exec_max,
		      max(curr->se.statistics.exec_max, delta_exec));

	curr->se.sum_exec_runtime += delta_exec;
	account_group_exec_runtime(curr, delta_exec);

	curr->se.exec_start = rq_clock_task(rq);
	cgroup_account_cputime(curr, delta_exec);

	sched_rt_avg_update(rq, delta_exec);

	if (!rt_bandwidth_enabled())
		return;

	for_each_sched_rt_entity(rt_se) {
		struct rt_rq *rt_rq = rt_rq_of_se(rt_se);

		if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {
			raw_spin_lock(&rt_rq->rt_runtime_lock);
			rt_rq->rt_time += delta_exec;
			if (sched_rt_runtime_exceeded(rt_rq))
				resched_curr(rq);
			raw_spin_unlock(&rt_rq->rt_runtime_lock);
		}
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 76 36.36% 8 30.77%
Ingo Molnar 76 36.36% 3 11.54%
Dhaval Giani 9 4.31% 1 3.85%
Dimitri Sivanich 8 3.83% 1 3.85%
Frank Mayhar 7 3.35% 1 3.85%
Frédéric Weisbecker 6 2.87% 1 3.85%
Giedrius Rekasius 5 2.39% 1 3.85%
Lucas De Marchi 4 1.91% 1 3.85%
Wanpeng Li 4 1.91% 1 3.85%
Kirill V Tkhai 4 1.91% 2 7.69%
Rafael J. Wysocki 3 1.44% 1 3.85%
Thomas Gleixner 2 0.96% 1 3.85%
Steven Rostedt 2 0.96% 1 3.85%
Tejun Heo 1 0.48% 1 3.85%
Srivatsa Vaddagiri 1 0.48% 1 3.85%
Viresh Kumar 1 0.48% 1 3.85%
Total 209 100.00% 26 100.00%


static void

dequeue_top_rt_rq(struct rt_rq *rt_rq)
{
	struct rq *rq = rq_of_rt_rq(rt_rq);

	BUG_ON(&rq->rt != rt_rq);

	if (!rt_rq->rt_queued)
		return;

	BUG_ON(!rq->nr_running);

	sub_nr_running(rq, rt_rq->rt_nr_running);
	rt_rq->rt_queued = 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Kirill V Tkhai 62 100.00% 2 100.00%
Total 62 100.00% 2 100.00%


static void

enqueue_top_rt_rq(struct rt_rq *rt_rq)
{
	struct rq *rq = rq_of_rt_rq(rt_rq);

	BUG_ON(&rq->rt != rt_rq);

	if (rt_rq->rt_queued)
		return;
	if (rt_rq_throttled(rt_rq) || !rt_rq->rt_nr_running)
		return;

	add_nr_running(rq, rt_rq->rt_nr_running);
	rt_rq->rt_queued = 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Kirill V Tkhai 66 100.00% 2 100.00%
Total 66 100.00% 2 100.00%

#if defined CONFIG_SMP


static void

inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
{
	struct rq *rq = rq_of_rt_rq(rt_rq);

#ifdef CONFIG_RT_GROUP_SCHED
	/*
         * Change rq's cpupri only if rt_rq is the top queue.
         */
	if (&rq->rt != rt_rq)
		return;
#endif
	if (rq->online && prio < prev_prio)
		cpupri_set(&rq->rd->cpupri, rq->cpu, prio);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 26 37.68% 4 44.44%
Kirill V Tkhai 16 23.19% 1 11.11%
Peter Zijlstra 13 18.84% 2 22.22%
Ingo Molnar 10 14.49% 1 11.11%
Steven Rostedt 4 5.80% 1 11.11%
Total 69 100.00% 9 100.00%


static void

dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
{
	struct rq *rq = rq_of_rt_rq(rt_rq);

#ifdef CONFIG_RT_GROUP_SCHED
	/*
         * Change rq's cpupri only if rt_rq is the top queue.
         */
	if (&rq->rt != rt_rq)
		return;
#endif
	if (rq->online && rt_rq->highest_prio.curr != prev_prio)
		cpupri_set(&rq->rd->cpupri, rq->cpu, rt_rq->highest_prio.curr);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 55 71.43% 1 25.00%
Kirill V Tkhai 16 20.78% 1 25.00%
Peter Zijlstra 6 7.79% 2 50.00%
Total 77 100.00% 4 100.00%

#else /* CONFIG_SMP */


static inline

void inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}
Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 17 100.00% 1 100.00%
Total 17 100.00% 1 100.00%


static inline

void dec_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio) {}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 17 100.00% 1 100.00%
Total 17 100.00% 1 100.00%

#endif /* CONFIG_SMP */

#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED

static void

inc_rt_prio(struct rt_rq *rt_rq, int prio)
{
	int prev_prio = rt_rq->highest_prio.curr;

	if (prio < prev_prio)
		rt_rq->highest_prio.curr = prio;

	inc_rt_prio_smp(rt_rq, prio, prev_prio);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 46 100.00% 1 100.00%
Total 46 100.00% 1 100.00%


static void

dec_rt_prio(struct rt_rq *rt_rq, int prio)
{
	int prev_prio = rt_rq->highest_prio.curr;

	if (rt_rq->rt_nr_running) {

		WARN_ON(prio < prev_prio);

		/*
                 * This may have been our highest task, and therefore
                 * we may have some recomputation to do
                 */
		if (prio == prev_prio) {
			struct rt_prio_array *array = &rt_rq->active;

			rt_rq->highest_prio.curr =
				sched_find_first_bit(array->bitmap);
		}

	} else
		rt_rq->highest_prio.curr = MAX_RT_PRIO;

	dec_rt_prio_smp(rt_rq, prio, prev_prio);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 88 100.00% 1 100.00%
Total 88 100.00% 1 100.00%

#else



static inline void inc_rt_prio(struct rt_rq *rt_rq, int prio) {}
Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%



static inline void dec_rt_prio(struct rt_rq *rt_rq, int prio) {}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%

#endif /* CONFIG_SMP || CONFIG_RT_GROUP_SCHED */

#ifdef CONFIG_RT_GROUP_SCHED


static void

inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
{
	if (rt_se_boosted(rt_se))
		rt_rq->rt_nr_boosted++;

	if (rt_rq->tg)
		start_rt_bandwidth(&rt_rq->tg->rt_bandwidth);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 27 61.36% 1 25.00%
Peter Zijlstra 17 38.64% 3 75.00%
Total 44 100.00% 4 100.00%


static void

dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
{
	if (rt_se_boosted(rt_se))
		rt_rq->rt_nr_boosted--;

	WARN_ON(!rt_rq->rt_nr_running && rt_rq->rt_nr_boosted);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 28 70.00% 1 20.00%
Peter Zijlstra 10 25.00% 3 60.00%
Ingo Molnar 2 5.00% 1 20.00%
Total 40 100.00% 5 100.00%

#else /* CONFIG_RT_GROUP_SCHED */


static void

inc_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
{
	start_rt_bandwidth(&def_rt_bandwidth);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 15 68.18% 1 33.33%
Peter Zijlstra 7 31.82% 2 66.67%
Total 22 100.00% 3 100.00%


static inline

void dec_rt_group(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) {}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 13 81.25% 1 33.33%
Ingo Molnar 2 12.50% 1 33.33%
Steven Rostedt 1 6.25% 1 33.33%
Total 16 100.00% 3 100.00%

#endif /* CONFIG_RT_GROUP_SCHED */


static inline

unsigned int rt_se_nr_running(struct sched_rt_entity *rt_se)
{
	struct rt_rq *group_rq = group_rt_rq(rt_se);

	if (group_rq)
		return group_rq->rt_nr_running;
	else
		return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Kirill V Tkhai 34 94.44% 1 50.00%
Gregory Haskins 2 5.56% 1 50.00%
Total 36 100.00% 2 100.00%


static inline

unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se)
{
	struct rt_rq *group_rq = group_rt_rq(rt_se);
	struct task_struct *tsk;

	if (group_rq)
		return group_rq->rr_nr_running;

	tsk = rt_task_of(rt_se);

	return (tsk->policy == SCHED_RR) ? 1 : 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Frédéric Weisbecker 55 96.49% 1 50.00%
Kirill V Tkhai 2 3.51% 1 50.00%
Total 57 100.00% 2 100.00%


static inline

void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
{
	int prio = rt_se_prio(rt_se);

	WARN_ON(!rt_prio(prio));
	rt_rq->rt_nr_running += rt_se_nr_running(rt_se);
	rt_rq->rr_nr_running += rt_se_rr_nr_running(rt_se);

	inc_rt_prio(rt_rq, prio);
	inc_rt_migration(rt_se, rt_rq);
	inc_rt_group(rt_se, rt_rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 21 28.77% 4 36.36%
Peter Zijlstra 17 23.29% 2 18.18%
Steven Rostedt 14 19.18% 2 18.18%
Frédéric Weisbecker 11 15.07% 1 9.09%
Ingo Molnar 5 6.85% 1 9.09%
Kirill V Tkhai 5 6.85% 1 9.09%
Total 73 100.00% 11 100.00%


static inline

void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
{
	WARN_ON(!rt_prio(rt_se_prio(rt_se)));
	WARN_ON(!rt_rq->rt_nr_running);
	rt_rq->rt_nr_running -= rt_se_nr_running(rt_se);
	rt_rq->rr_nr_running -= rt_se_rr_nr_running(rt_se);

	dec_rt_prio(rt_rq, rt_se_prio(rt_se));
	dec_rt_migration(rt_se, rt_rq);
	dec_rt_group(rt_se, rt_rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 28 35.44% 2 25.00%
Gregory Haskins 15 18.99% 2 25.00%
Steven Rostedt 11 13.92% 1 12.50%
Ingo Molnar 11 13.92% 1 12.50%
Frédéric Weisbecker 9 11.39% 1 12.50%
Kirill V Tkhai 5 6.33% 1 12.50%
Total 79 100.00% 8 100.00%

/*
 * Change rt_se->run_list location unless SAVE && !MOVE
 *
 * assumes ENQUEUE/DEQUEUE flags match
 */


static inline bool move_entity(unsigned int flags)
{
	if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) == DEQUEUE_SAVE)
		return false;

	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 31 100.00% 1 100.00%
Total 31 100.00% 1 100.00%



static void __delist_rt_entity(struct sched_rt_entity *rt_se, struct rt_prio_array *array)
{
	list_del_init(&rt_se->run_list);

	if (list_empty(array->queue + rt_se_prio(rt_se)))
		__clear_bit(rt_se_prio(rt_se), array->bitmap);

	rt_se->on_list = 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 56 100.00% 1 100.00%
Total 56 100.00% 1 100.00%



static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
{
	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
	struct rt_prio_array *array = &rt_rq->active;
	struct rt_rq *group_rq = group_rt_rq(rt_se);
	struct list_head *queue = array->queue + rt_se_prio(rt_se);

	/*
         * Don't enqueue the group if its throttled, or when empty.
         * The latter is a consequence of the former when a child group
         * get throttled and the current group doesn't have any other
         * active members.
         */
	if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) {
		if (rt_se->on_list)
			__delist_rt_entity(rt_se, array);
		return;
	}

	if (move_entity(flags)) {
		WARN_ON_ONCE(rt_se->on_list);
		if (flags & ENQUEUE_HEAD)
			list_add(&rt_se->run_list, queue);
		else
			list_add_tail(&rt_se->run_list, queue);

		__set_bit(rt_se_prio(rt_se), array->bitmap);
		rt_se->on_list = 1;
	}
	rt_se->on_rq = 1;

	inc_rt_tasks(rt_se, rt_rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 113 68.07% 4 40.00%
Ingo Molnar 20 12.05% 1 10.00%
Thomas Gleixner 15 9.04% 1 10.00%
Gregory Haskins 8 4.82% 2 20.00%
Dmitry Adamushko 8 4.82% 1 10.00%
Steven Rostedt 2 1.20% 1 10.00%
Total 166 100.00% 10 100.00%



static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
{
	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
	struct rt_prio_array *array = &rt_rq->active;

	if (move_entity(flags)) {
		WARN_ON_ONCE(!rt_se->on_list);
		__delist_rt_entity(rt_se, array);
	}
	rt_se->on_rq = 0;

	dec_rt_tasks(rt_se, rt_rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 63 87.50% 4 66.67%
Steven Rostedt 9 12.50% 2 33.33%
Total 72 100.00% 6 100.00%

/*
 * Because the prio of an upper entry depends on the lower
 * entries, we must remove entries top - down.
 */


static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags)
{
	struct sched_rt_entity *back = NULL;

	for_each_sched_rt_entity(rt_se) {
		rt_se->back = back;
		back = rt_se;
	}

	dequeue_top_rt_rq(rt_rq_of_se(back));

	for (rt_se = back; rt_se; rt_se = rt_se->back) {
		if (on_rt_rq(rt_se))
			__dequeue_rt_entity(rt_se, flags);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 59 77.63% 4 57.14%
Ingo Molnar 8 10.53% 1 14.29%
Kirill V Tkhai 8 10.53% 1 14.29%
Dmitry Adamushko 1 1.32% 1 14.29%
Total 76 100.00% 7 100.00%



static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
{
	struct rq *rq = rq_of_rt_se(rt_se);

	dequeue_rt_stack(rt_se, flags);
	for_each_sched_rt_entity(rt_se)
		__enqueue_rt_entity(rt_se, flags);
	enqueue_top_rt_rq(&rq->rt);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 19 37.25% 2 40.00%
Kirill V Tkhai 18 35.29% 1 20.00%
Ingo Molnar 12 23.53% 1 20.00%
Thomas Gleixner 2 3.92% 1 20.00%
Total 51 100.00% 5 100.00%



static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
{
	struct rq *rq = rq_of_rt_se(rt_se);

	dequeue_rt_stack(rt_se, flags);

	for_each_sched_rt_entity(rt_se) {
		struct rt_rq *rt_rq = group_rt_rq(rt_se);

		if (rt_rq && rt_rq->rt_nr_running)
			__enqueue_rt_entity(rt_se, flags);
	}
	enqueue_top_rt_rq(&rq->rt);
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 43 60.56% 4 44.44%
Kirill V Tkhai 18 25.35% 1 11.11%
Steven Rostedt 9 12.68% 3 33.33%
Thomas Gleixner 1 1.41% 1 11.11%
Total 71 100.00% 9 100.00%

/*
 * Adding/removing a task to/from a priority array:
 */

static void

enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
{
	struct sched_rt_entity *rt_se = &p->rt;

	if (flags & ENQUEUE_WAKEUP)
		rt_se->timeout = 0;

	enqueue_rt_entity(rt_se, flags);

	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
		enqueue_pushable_task(rq, p);
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 21 29.58% 1 12.50%
Peter Zijlstra 20 28.17% 3 37.50%
Steven Rostedt 15 21.13% 1 12.50%
Ingo Molnar 14 19.72% 2 25.00%
Thomas Gleixner 1 1.41% 1 12.50%
Total 71 100.00% 8 100.00%



static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
{
	struct sched_rt_entity *rt_se = &p->rt;

	update_curr_rt(rq);
	dequeue_rt_entity(rt_se, flags);

	dequeue_pushable_task(rq, p);
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 21 43.75% 1 14.29%
Peter Zijlstra 10 20.83% 4 57.14%
Ingo Molnar 10 20.83% 1 14.29%
Gregory Haskins 7 14.58% 1 14.29%
Total 48 100.00% 7 100.00%

/*
 * Put task to the head or the end of the run list without the overhead of
 * dequeue followed by enqueue.
 */

static void

requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)
{
	if (on_rt_rq(rt_se)) {
		struct rt_prio_array *array = &rt_rq->active;
		struct list_head *queue = array->queue + rt_se_prio(rt_se);

		if (head)
			list_move(&rt_se->run_list, queue);
		else
			list_move_tail(&rt_se->run_list, queue);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Dmitry Adamushko 31 40.26% 2 22.22%
Peter Zijlstra 15 19.48% 2 22.22%
Steven Rostedt 15 19.48% 2 22.22%
Ingo Molnar 12 15.58% 2 22.22%
Gregory Haskins 4 5.19% 1 11.11%
Total 77 100.00% 9 100.00%



static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head)
{
	struct sched_rt_entity *rt_se = &p->rt;
	struct rt_rq *rt_rq;

	for_each_sched_rt_entity(rt_se) {
		rt_rq = rt_rq_of_se(rt_se);
		requeue_rt_entity(rt_rq, rt_se, head);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 39 69.64% 1 20.00%
Steven Rostedt 10 17.86% 2 40.00%
Dmitry Adamushko 5 8.93% 1 20.00%
Ingo Molnar 2 3.57% 1 20.00%
Total 56 100.00% 5 100.00%



static void yield_task_rt(struct rq *rq)
{
	requeue_task_rt(rq, rq->curr, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
Ingo Molnar 12 54.55% 1 25.00%
Dmitry Adamushko 5 22.73% 2 50.00%
Steven Rostedt 5 22.73% 1 25.00%
Total 22 100.00% 4 100.00%

#ifdef CONFIG_SMP
static int find_lowest_rq(struct task_struct *task);


static int

select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags)
{
	struct task_struct *curr;
	struct rq *rq;

	/* For anything but wake ups, just return the task_cpu */
	if (sd_flag != SD_BALANCE_WAKE && sd_flag != SD_BALANCE_FORK)
		goto out;

	rq = cpu_rq(cpu);

	rcu_read_lock();
	curr = READ_ONCE(rq->curr); /* unlocked access */

	/*
         * If the current task on @p's runqueue is an RT task, then
         * try to see if we can wake this RT task up on another
         * runqueue. Otherwise simply start this RT task
         * on its current runqueue.
         *
         * We want to avoid overloading runqueues. If the woken
         * task is a higher priority, then it will stay on this CPU
         * and the lower prio task should be moved to another CPU.
         * Even though this will probably make the lower prio task
         * lose its cache, we do not want to bounce a higher task
         * around just because it gave up its CPU, perhaps for a
         * lock?
         *
         * For equal prio tasks, we just let the scheduler sort it out.
         *
         * Otherwise, just let it ride on the affined RQ and the
         * post-schedule router will push the preempted task away
         *
         * This test is optimistic, if we get it wrong the load-balancer
         * will have to sort it out.
         */
	if (curr && unlikely(rt_task(curr)) &&
	    (curr->nr_cpus_allowed < 2 ||
	     curr->prio <= p->prio)) {
		int target = find_lowest_rq(p);

		/*
                 * Don't bother moving it if the destination CPU is
                 * not running a lower priority task.
                 */
		if (target != -1 &&
		    p->prio < cpu_rq(target)->rt.highest_prio.curr)
			cpu = target;
	}
	rcu_read_unlock();

out:
	return cpu;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 52 37.68% 5 26.32%
Steven Rostedt 42 30.43% 6 31.58%
Gregory Haskins 24 17.39% 4 21.05%
Tim Chen 16 11.59% 1 5.26%
Ingo Molnar 2 1.45% 1 5.26%
Shawn Bohrer 1 0.72% 1 5.26%
Jason Low 1 0.72% 1 5.26%
Total 138 100.00% 19 100.00%



static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
{
	/*
         * Current can't be migrated, useless to reschedule,
         * let's hope p can move out.
         */
	if (rq->curr->nr_cpus_allowed == 1 ||
	    !cpupri_find(&rq->rd->cpupri, rq->curr, NULL))
		return;

	/*
         * p is migratable, so let's not schedule it and
         * see if it is pushed or pulled somewhere else.
         */
	if (p->nr_cpus_allowed != 1
	    && cpupri_find(&rq->rd->cpupri, p, NULL))
		return;

	/*
         * There appears to be other cpus that can accept
         * current and none to run 'p', so lets reschedule
         * to try and push current away:
         */
	requeue_task_rt(rq, p, 1);
	resched_curr(rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Dmitry Adamushko 63 75.00% 1 20.00%
Wanpeng Li 12 14.29% 1 20.00%
Rusty Russell 4 4.76% 1 20.00%
Ingo Molnar 4 4.76% 1 20.00%
Kirill V Tkhai 1 1.19% 1 20.00%
Total 84 100.00% 5 100.00%

#endif /* CONFIG_SMP */

/*
 * Preempt the current task with a newly woken task if needed:
 */


static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flags)
{
	if (p->prio < rq->curr->prio) {
		resched_curr(rq);
		return;
	}

#ifdef CONFIG_SMP
	/*
         * If:
         *
         * - the newly woken task is of equal priority to the current task
         * - the newly woken task is non-migratable while current is migratable
         * - current will be preempted on the next reschedule
         *
         * we should check to see if current can readily move to a different
         * cpu.  If so, we will reschedule to allow the push logic to try
         * to move current somewhere else, making room for our non-migratable
         * task.
         */
	if (p->prio == rq->curr->prio && !test_tsk_need_resched(rq->curr))
		check_preempt_equal_prio(rq, p);
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
Ingo Molnar 32 44.44% 1 11.11%
Gregory Haskins 24 33.33% 2 22.22%
Hillf Danton 6 8.33% 1 11.11%
Peter Zijlstra 3 4.17% 2 22.22%
Dmitry Adamushko 3 4.17% 1 11.11%
Steven Rostedt 3 4.17% 1 11.11%
Kirill V Tkhai 1 1.39% 1 11.11%
Total 72 100.00% 9 100.00%



static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
						   struct rt_rq *rt_rq)
{
	struct rt_prio_array *array = &rt_rq->active;
	struct sched_rt_entity *next = NULL;
	struct list_head *queue;
	int idx;

	idx = sched_find_first_bit(array->bitmap);
	BUG_ON(idx >= MAX_RT_PRIO);

	queue = array->queue + idx;
	next = list_entry(queue->next, struct sched_rt_entity, run_list);

	return next;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 56 66.67% 1 50.00%
Ingo Molnar 28 33.33% 1 50.00%
Total 84 100.00% 2 100.00%



static struct task_struct *_pick_next_task_rt(struct rq *rq)
{
	struct sched_rt_entity *rt_se;
	struct task_struct *p;
	struct rt_rq *rt_rq  = &rq->rt;

	do {
		rt_se = pick_next_rt_entity(rq, rt_rq);
		BUG_ON(!rt_se);
		rt_rq = group_rt_rq(rt_se);
	} while (rt_rq);

	p = rt_task_of(rt_se);
	p->se.exec_start = rq_clock_task(rq);

	return p;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 54 64.29% 3 30.00%
Ingo Molnar 20 23.81% 2 20.00%
Gregory Haskins 3 3.57% 2 20.00%
Frédéric Weisbecker 3 3.57% 1 10.00%
Steven Rostedt 2 2.38% 1 10.00%
Dmitry Adamushko 2 2.38% 1 10.00%
Total 84 100.00% 10 100.00%


static struct task_struct *

pick_next_task_rt(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
{
	struct task_struct *p;
	struct rt_rq *rt_rq = &rq->rt;

	if (need_pull_rt_task(rq, prev)) {
		/*
                 * This is OK, because current is on_cpu, which avoids it being
                 * picked for load-balance and preemption/IRQs are still
                 * disabled avoiding further scheduler activity on it and we're
                 * being very careful to re-start the picking loop.
                 */
		rq_unpin_lock(rq, rf);
		pull_rt_task(rq);
		rq_repin_lock(rq, rf);
		/*
                 * pull_rt_task() can drop (and re-acquire) rq->lock; this
                 * means a dl or stop task can slip in, in which case we need
                 * to re-start task selection.
                 */
		if (unlikely((rq->stop && task_on_rq_queued(rq->stop)) ||
			     rq->dl.dl_nr_running))
			return RETRY_TASK;
	}

	/*
         * We may dequeue prev's rt_rq in put_prev_task().
         * So, we update time before rt_nr_running check.
         */
	if (prev->sched_class == &rt_sched_class)
		update_curr_rt(rq);

	if (!rt_rq->rt_queued)
		return NULL;

	put_prev_task(rq, prev);

	p = _pick_next_task_rt(rq);

	/* The running task is never eligible for pushing */
	dequeue_pushable_task(rq, p);

	queue_push_tasks(rq);

	return p;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 78 51.32% 7 50.00%
Gregory Haskins 37 24.34% 2 14.29%
Kirill Tkhai 18 11.84% 2 14.29%
Kirill V Tkhai 12 7.89% 2 14.29%
Matt Fleming 7 4.61% 1 7.14%
Total 152 100.00% 14 100.00%



static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
{
	update_curr_rt(rq);

	/*
         * The previous task needs to be made eligible for pushing
         * if it is still active
         */
	if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
		enqueue_pushable_task(rq, p);
}

Contributors
Person Tokens Prop Commits CommitProp
Ingo Molnar 23 51.11% 2 50.00%
Gregory Haskins 17 37.78% 1 25.00%
Peter Zijlstra 5 11.11% 1 25.00%
Total 45 100.00% 4 100.00%

#ifdef CONFIG_SMP

/* Only try algorithms three times */

#define RT_MAX_TRIES 3



static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
{
	if (!task_running(rq, p) &&
	    cpumask_test_cpu(cpu, &p->cpus_allowed))
		return 1;
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 41 91.11% 1 33.33%
Ingo Molnar 3 6.67% 1 33.33%
Rusty Russell 1 2.22% 1 33.33%
Total 45 100.00% 3 100.00%

/*
 * Return the highest pushable rq's task, which is suitable to be executed
 * on the cpu, NULL otherwise
 */


static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
{
	struct plist_head *head = &rq->rt.pushable_tasks;
	struct task_struct *p;

	if (!has_pushable_tasks(rq))
		return NULL;

	plist_for_each_entry(p, head, pushable_tasks) {
		if (pick_rt_task(rq, p, cpu))
			return p;
	}

	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 29 40.85% 3 37.50%
Kirill V Tkhai 23 32.39% 1 12.50%
Peter Zijlstra 16 22.54% 2 25.00%
Gregory Haskins 3 4.23% 2 25.00%
Total 71 100.00% 8 100.00%

static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);



static int find_lowest_rq(struct task_struct *task)
{
	struct sched_domain *sd;
	struct cpumask *lowest_mask = this_cpu_cpumask_var_ptr(local_cpu_mask);
	int this_cpu = smp_processor_id();
	int cpu      = task_cpu(task);

	/* Make sure the mask is initialized first */
	if (unlikely(!lowest_mask))
		return -1;

	if (task->nr_cpus_allowed == 1)
		return -1; /* No other targets possible */

	if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask))
		return -1; /* No targets found */

	/*
         * At this point we have built a mask of cpus representing the
         * lowest priority tasks in the system.  Now we want to elect
         * the best one based on our affinity and topology.
         *
         * We prioritize the last cpu that the task executed on since
         * it is most likely cache-hot in that location.
         */
	if (cpumask_test_cpu(cpu, lowest_mask))
		return cpu;

	/*
         * Otherwise, we consult the sched_domains span maps to figure
         * out which cpu is logically closest to our hot cache data.
         */
	if (!cpumask_test_cpu(this_cpu, lowest_mask))
		this_cpu = -1; /* Skip this_cpu opt if not among lowest */

	rcu_read_lock();
	for_each_domain(cpu, sd) {
		if (sd->flags & SD_WAKE_AFFINE) {
			int best_cpu;

			/*
                         * "this_cpu" is cheaper to preempt than a
                         * remote processor.
                         */
			if (this_cpu != -1 &&
			    cpumask_test_cpu(this_cpu, sched_domain_span(sd))) {
				rcu_read_unlock();
				return this_cpu;
			}

			best_cpu = cpumask_first_and(lowest_mask,
						     sched_domain_span(sd));
			if (best_cpu < nr_cpu_ids) {
				rcu_read_unlock();
				return best_cpu;
			}
		}
	}
	rcu_read_unlock();

	/*
         * And finally, if there were no matches within the domains
         * just give the caller *something* to work with from the compatible
         * locations.
         */
	if (this_cpu != -1)
		return this_cpu;

	cpu = cpumask_any(lowest_mask);
	if (cpu < nr_cpu_ids)
		return cpu;
	return -1;
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 139 60.17% 4 36.36%
Rusty Russell 60 25.97% 3 27.27%
Xiaotian Feng 16 6.93% 1 9.09%
Steven Rostedt 13 5.63% 1 9.09%
Ingo Molnar 2 0.87% 1 9.09%
Christoph Lameter 1 0.43% 1 9.09%
Total 231 100.00% 11 100.00%

/* Will lock the rq it finds */


static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
{
	struct rq *lowest_rq = NULL;
	int tries;
	int cpu;

	for (tries = 0; tries < RT_MAX_TRIES; tries++) {
		cpu = find_lowest_rq(task);

		if ((cpu == -1) || (cpu == rq->cpu))
			break;

		lowest_rq = cpu_rq(cpu);

		if (lowest_rq->rt.highest_prio.curr <= task->prio) {
			/*
                         * Target rq has tasks of equal or higher priority,
                         * retrying does not release any lock and is unlikely
                         * to yield a different result.
                         */
			lowest_rq = NULL;
			break;
		}

		/* if the prio of this runqueue changed, try again */
		if (double_lock_balance(rq, lowest_rq)) {
			/*
                         * We had to unlock the run queue. In
                         * the mean time, task could have
                         * migrated already or had its affinity changed.
                         * Also make sure that it wasn't scheduled on its rq.
                         */
			if (unlikely(task_rq(task) != rq ||
				     !cpumask_test_cpu(lowest_rq->cpu, &task->cpus_allowed) ||
				     task_running(rq, task) ||
				     !rt_task(task) ||
				     !task_on_rq_queued(task))) {

				double_unlock_balance(rq, lowest_rq);
				lowest_rq = NULL;
				break;
			}
		}

		/* If this rq is still suitable use it. */
		if (lowest_rq->rt.highest_prio.curr > task->prio)
			break;

		/* try again */
		double_unlock_balance(rq, lowest_rq);
		lowest_rq = NULL;
	}

	return lowest_rq;
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 84 41.58% 1 8.33%
Gregory Haskins 75 37.13% 3 25.00%
Tim Chen 22 10.89% 1 8.33%
Peter Zijlstra 6 2.97% 2 16.67%
Xunlei Pang 6 2.97% 1 8.33%
Ingo Molnar 5 2.48% 2 16.67%
Kirill Tkhai 3 1.49% 1 8.33%
Rusty Russell 1 0.50% 1 8.33%
Total 202 100.00% 12 100.00%



static struct task_struct *pick_next_pushable_task(struct rq *rq)
{
	struct task_struct *p;

	if (!has_pushable_tasks(rq))
		return NULL;

	p = plist_first_entry(&rq->rt.pushable_tasks,
			      struct task_struct, pushable_tasks);

	BUG_ON(rq->cpu != task_cpu(p));
	BUG_ON(task_current(rq, p));
	BUG_ON(p->nr_cpus_allowed <= 1);

	BUG_ON(!task_on_rq_queued(p));
	BUG_ON(!rt_task(p));

	return p;
}

Contributors
Person Tokens Prop Commits CommitProp
Gregory Haskins 89 90.82% 1 25.00%
Steven Rostedt 4 4.08% 1 25.00%
Kirill Tkhai 3 3.06% 1 25.00%
Ingo Molnar 2 2.04% 1 25.00%
Total 98 100.00% 4 100.00%

/*
 * If the current CPU has more than one RT task, see if the non
 * running task can migrate over to a CPU that is running a task
 * of lesser priority.
 */


static int push_rt_task(struct rq *rq)
{
	struct task_struct *next_task;
	struct rq *lowest_rq;
	int ret = 0;

	if (!rq->rt.overloaded)
		return 0;

	next_task = pick_next_pushable_task(rq);
	if (!next_task)
		return 0;

retry:
	if (unlikely(next_task == rq->curr)) {
		WARN_ON(1);
		return 0;
	}

	/*
         * It's possible that the next_task slipped in of
         * higher priority than current. If that's the case
         * just reschedule current.
         */
	if (unlikely(next_task->prio < rq->curr->prio)) {
		resched_curr(rq);
		return 0;
	}

	/* We might release rq lock */
	get_task_struct(next_task);

	/* find_lock_lowest_rq locks the rq if found */
	lowest_rq = find_lock_lowest_rq(next_task, rq);
	if (!lowest_rq) {
		struct task_struct *task;
		/*
                 * find_lock_lowest_rq releases rq->lock
                 * so it is possible that next_task has migrated.
                 *
                 * We need to make sure that the task is still on the same
                 * run-queue and is also still the next task eligible for
                 * pushing.
                 */
		task = pick_next_pushable_task(rq);
		if (task == next_task) {
			/*
                         * The task hasn't migrated, and is still the next
                         * eligible task, but we failed to find a run-queue
                         * to push it to.  Do not retry in this case, since
                         * other cpus will pull from us when ready.
                         */
			goto out;
		}

		if (!task)
			/* No more tasks, just exit */
			goto out;

		/*
                 * Something has shifted, try again.
                 */
		put_task_struct(next_task);
		next_task = task;
		goto retry;
	}

	deactivate_task(rq, next_task, 0);
	set_task_cpu(next_task, lowest_rq->cpu);
	activate_task(lowest_rq, next_task, 0);
	ret = 1;

	resched_curr(lowest_rq);

	double_unlock_balance(rq, lowest_rq);

out:
	put_task_struct(next_task);

	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 172 76.44% 2 22.22%
Gregory Haskins 36 16.00% 4 44.44%
Hillf Danton 12 5.33% 1 11.11%
Peter Zijlstra 3 1.33% 1 11.11%
Kirill V Tkhai 2 0.89% 1 11.11%
Total 225 100.00% 9 100.00%



static void push_rt_tasks(struct rq *rq)
{
	/* push_rt_task will return true if it moved an RT */
	while (push_rt_task(rq))
		;
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%

#ifdef HAVE_RT_PUSH_IPI

/*
 * When a high priority task schedules out from a CPU and a lower priority
 * task is scheduled in, a check is made to see if there's any RT tasks
 * on other CPUs that are waiting to run because a higher priority RT task
 * is currently running on its CPU. In this case, the CPU with multiple RT
 * tasks queued on it (overloaded) needs to be notified that a CPU has opened
 * up that may be able to run one of its non-running queued RT tasks.
 *
 * All CPUs with overloaded RT tasks need to be notified as there is currently
 * no way to know which of these CPUs have the highest priority task waiting
 * to run. Instead of trying to take a spinlock on each of these CPUs,
 * which has shown to cause large latency when done on machines with many
 * CPUs, sending an IPI to the CPUs to have them push off the overloaded
 * RT tasks waiting to run.
 *
 * Just sending an IPI to each of the CPUs is also an issue, as on large
 * count CPU machines, this can cause an IPI storm on a CPU, especially
 * if its the only CPU with multiple RT tasks queued, and a large number
 * of CPUs scheduling a lower priority task at the same time.
 *
 * Each root domain has its own irq work function that can iterate over
 * all CPUs with RT overloaded tasks. Since all CPUs with overloaded RT
 * tassk must be checked if there's one or many CPUs that are lowering
 * their priority, there's a single irq work iterator that will try to
 * push off RT tasks that are waiting to run.
 *
 * When a CPU schedules a lower priority task, it will kick off the
 * irq work iterator that will jump to each CPU with overloaded RT tasks.
 * As it only takes the first CPU that schedules a lower priority task
 * to start the process, the rto_start variable is incremented and if
 * the atomic result is one, then that CPU will try to take the rto_lock.
 * This prevents high contention on the lock as the process handles all
 * CPUs scheduling lower priority tasks.
 *
 * All CPUs that are scheduling a lower priority task will increment the
 * rt_loop_next variable. This will make sure that the irq work iterator
 * checks all RT overloaded CPUs whenever a CPU schedules a new lower
 * priority task, even if the iterator is in the middle of a scan. Incrementing
 * the rt_loop_next will cause the iterator to perform another scan.
 *
 */


static int rto_next_cpu(struct rq *rq)
{
	struct root_domain *rd = rq->rd;
	int next;
	int cpu;

	/*
         * When starting the IPI RT pushing, the rto_cpu is set to -1,
         * rt_next_cpu() will simply return the first CPU found in
         * the rto_mask.
         *
         * If rto_next_cpu() is called with rto_cpu is a valid cpu, it
         * will return the next CPU found in the rto_mask.
         *
         * If there are no more CPUs left in the rto_mask, then a check is made
         * against rto_loop and rto_loop_next. rto_loop is only updated with
         * the rto_lock held, but any CPU may increment the rto_loop_next
         * without any locking.
         */
	for (;;) {

		/* When rto_cpu is -1 this acts like cpumask_first() */
		cpu = cpumask_next(rd->rto_cpu, rd->rto_mask);

		rd->rto_cpu = cpu;

		if (cpu < nr_cpu_ids)
			return cpu;

		rd->rto_cpu = -1;

		/*
                 * ACQUIRE ensures we see the @rto_mask changes
                 * made prior to the @next value observed.
                 *
                 * Matches WMB in rt_set_overload().
                 */
		next = atomic_read_acquire(&rd->rto_loop_next);

		if (rd->rto_loop == next)
			break;

		rd->rto_loop = next;
	}

	return -1;
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 97 97.00% 4 80.00%
Ingo Molnar 3 3.00% 1 20.00%
Total 100 100.00% 5 100.00%



static inline bool rto_start_trylock(atomic_t *v)
{
	return !atomic_cmpxchg_acquire(v, 0, 1);
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 19 86.36% 2 50.00%
Gregory Haskins 2 9.09% 1 25.00%
Ingo Molnar 1 4.55% 1 25.00%
Total 22 100.00% 4 100.00%



static inline void rto_start_unlock(atomic_t *v)
{
	atomic_set_release(v, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 18 100.00% 3 100.00%
Total 18 100.00% 3 100.00%



static void tell_cpu_to_push(struct rq *rq)
{
	int cpu = -1;

	/* Keep the loop going if the IPI is currently active */
	atomic_inc(&rq->rd->rto_loop_next);

	/* Only one CPU can initiate a loop at a time */
	if (!rto_start_trylock(&rq->rd->rto_loop_start))
		return;

	raw_spin_lock(&rq->rd->rto_lock);

	/*
         * The rto_cpu is updated under the lock, if it has a valid cpu
         * then the IPI is still running and will continue due to the
         * update to loop_next, and nothing needs to be done here.
         * Otherwise it is finishing up and an ipi needs to be sent.
         */
	if (rq->rd->rto_cpu < 0)
		cpu = rto_next_cpu(rq);

	raw_spin_unlock(&rq->rd->rto_lock);

	rto_start_unlock(&rq->rd->rto_loop_start);

	if (cpu >= 0)
		irq_work_queue_on(&rq->rd->rto_push_work, cpu);
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 109 100.00% 2 100.00%
Total 109 100.00% 2 100.00%

/* Called from hardirq context */


void rto_push_irq_work_func(struct irq_work *work)
{
	struct rq *rq;
	int cpu;

	rq = this_rq();

	/*
         * We do not need to grab the lock to check for has_pushable_tasks.
         * When it gets updated, a check is made if a push is possible.
         */
	if (has_pushable_tasks(rq)) {
		raw_spin_lock(&rq->lock);
		push_rt_tasks(rq);
		raw_spin_unlock(&rq->lock);
	}

	raw_spin_lock(&rq->rd->rto_lock);

	/* Pass the IPI to the next rt overloaded queue */
	cpu = rto_next_cpu(rq);

	raw_spin_unlock(&rq->rd->rto_lock);

	if (cpu < 0)
		return;

	/* Try the next RT overloaded CPU */
	irq_work_queue_on(&rq->rd->rto_push_work, cpu);
}
Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 102 100.00% 2 100.00%
Total 102 100.00% 2 100.00%

#endif /* HAVE_RT_PUSH_IPI */



static void pull_rt_task(struct rq *this_rq)
{
	int this_cpu = this_rq->cpu, cpu;
	bool resched = false;
	struct task_struct *p;
	struct rq *src_rq;
	int rt_overload_count = rt_overloaded(this_rq);

	if (likely(!rt_overload_count))
		return;

	/*
         * Match the barrier from rt_set_overloaded; this guarantees that if we
         * see overloaded we must also see the rto_mask bit.
         */
	smp_rmb();

	/* If we are the only overloaded CPU do nothing */
	if (rt_overload_count == 1 &&
	    cpumask_test_cpu(this_rq->cpu, this_rq->rd->rto_mask))
		return;

#ifdef HAVE_RT_PUSH_IPI
	if (sched_feat(RT_PUSH_IPI)) {
		tell_cpu_to_push(this_rq);
		return;
	}
#endif

	for_each_cpu(cpu, this_rq->rd->rto_mask) {
		if (this_cpu == cpu)
			continue;

		src_rq = cpu_rq(cpu);

		/*
                 * Don't bother taking the src_rq->lock if the next highest
                 * task is known to be lower-priority than our current task.
                 * This may look racy, but if this value is about to go
                 * logically higher, the src_rq will push this task away.
                 * And if its going logically lower, we do not care
                 */
		if (src_rq->rt.highest_prio.next >=
		    this_rq->rt.highest_prio.curr)
			continue;

		/*
                 * We can potentially drop this_rq's lock in
                 * double_lock_balance, and another CPU could
                 * alter this_rq
                 */
		double_lock_balance(this_rq, src_rq);

		/*
                 * We can pull only a task, which is pushable
                 * on its rq, and no others.
                 */
		p = pick_highest_pushable_task(src_rq, this_cpu);

		/*
                 * Do we have an RT task that preempts
                 * the to-be-scheduled task?
                 */
		if (p && (p->prio < this_rq->rt.highest_prio.curr)) {
			WARN_ON(p == src_rq->curr);
			WARN_ON(!task_on_rq_queued(p));

			/*
                         * There's a chance that p is higher in priority
                         * than what's currently running on its cpu.
                         * This is just that p is wakeing up and hasn't
                         * had a chance to schedule. We only pull
                         * p if it is lower in priority than the
                         * current task on the run queue
                         */
			if (p->prio < src_rq->curr->prio)
				goto skip;

			resched = true;

			deactivate_task(src_rq, p, 0);
			set_task_cpu(p, this_cpu);
			activate_task(this_rq, p, 0);
			/*
                         * We continue with the search, just in
                         * case there's an even higher prio task
                         * in another runqueue. (low likelihood
                         * but possible)
                         */
		}
skip:
		double_unlock_balance(this_rq, src_rq);
	}

	if (resched)
		resched_curr(this_rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 202 77.69% 3 23.08%
Gregory Haskins 27 10.38% 3 23.08%
Peter Zijlstra 22 8.46% 2 15.38%
Kirill Tkhai 3 1.15% 1 7.69%
Kirill V Tkhai 2 0.77% 1 7.69%
Mike Galbraith 2 0.77% 1 7.69%
Lucas De Marchi 1 0.38% 1 7.69%
Rusty Russell 1 0.38% 1 7.69%
Total 260 100.00% 13 100.00%

/*
 * If we are not running and we are not going to reschedule soon, we should
 * try to push tasks away now
 */


static void task_woken_rt(struct rq *rq, struct task_struct *p)
{
	if (!task_running(rq, p) &&
	    !test_tsk_need_resched(rq->curr) &&
	    p->nr_cpus_allowed > 1 &&
	    (dl_task(rq->curr) || rt_task(rq->curr)) &&
	    (rq->curr->nr_cpus_allowed < 2 ||
	     rq->curr->prio <= p->prio))
		push_rt_tasks(rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 58 71.60% 3 30.00%
Juri Lelli 9 11.11% 1 10.00%
Gregory Haskins 8 9.88% 3 30.00%
Ingo Molnar 4 4.94% 1 10.00%
Shawn Bohrer 1 1.23% 1 10.00%
Peter Zijlstra 1 1.23% 1 10.00%
Total 81 100.00% 10 100.00%

/* Assumes rq->lock is held */


static void rq_online_rt(struct rq *rq)
{
	if (rq->rt.overloaded)
		rt_set_overload(rq);

	__enable_runtime(rq);

	cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio.curr);
}

Contributors
Person Tokens Prop Commits CommitProp
Ingo Molnar 23 45.10% 1 20.00%
Gregory Haskins 23 45.10% 3 60.00%
Peter Zijlstra 5 9.80% 1 20.00%
Total 51 100.00% 5 100.00%

/* Assumes rq->lock is held */


static void rq_offline_rt(struct rq *rq)
{
	if (rq->rt.overloaded)
		rt_clear_overload(rq);

	__disable_runtime(rq);

	cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);
}

Contributors
Person Tokens Prop Commits CommitProp
Ingo Molnar 23 51.11% 1 25.00%
Gregory Haskins 17 37.78% 2 50.00%
Peter Zijlstra 5 11.11% 1 25.00%
Total 45 100.00% 4 100.00%

/*
 * When switch from the rt queue, we bring ourselves to a position
 * that we might want to pull RT tasks from other runqueues.
 */


static void switched_from_rt(struct rq *rq, struct task_struct *p)
{
	/*
         * If there are other RT tasks then we will reschedule
         * and the scheduling of the other RT tasks will handle
         * the balancing. But if we are the last RT task
         * we may need to handle the pulling of RT tasks
         * now.
         */
	if (!task_on_rq_queued(p) || rq->rt.rt_nr_running)
		return;

	queue_pull_task(rq);
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 29 78.38% 1 20.00%
Kirill V Tkhai 3 8.11% 1 20.00%
Kirill Tkhai 3 8.11% 1 20.00%
Peter Zijlstra 2 5.41% 2 40.00%
Total 37 100.00% 5 100.00%



void __init init_sched_rt_class(void)
{
	unsigned int i;

	for_each_possible_cpu(i) {
		zalloc_cpumask_var_node(&per_cpu(local_cpu_mask, i),
					GFP_KERNEL, cpu_to_node(i));
	}
}
Contributors
Person Tokens Prop Commits CommitProp
Rusty Russell 27 75.00% 1 20.00%
Mike Travis 5 13.89% 1 20.00%
Peter Zijlstra 2 5.56% 1 20.00%
Yinghai Lu 1 2.78% 1 20.00%
Li Zefan 1 2.78% 1 20.00%
Total 36 100.00% 5 100.00%

#endif /* CONFIG_SMP */

/*
 * When switching a task to RT, we may overload the runqueue
 * with RT tasks. In this case we try to push them off to
 * other runqueues.
 */


static void switched_to_rt(struct rq *rq, struct task_struct *p)
{
	/*
         * If we are already running, then there's nothing
         * that needs to be done. But if we are not running
         * we may need to preempt the current running task.
         * If that current running task is also an RT task
         * then see if we can move to another run queue.
         */
	if (task_on_rq_queued(p) && rq->curr != p) {
#ifdef CONFIG_SMP
		if (p->nr_cpus_allowed > 1 && rq->rt.overloaded)
			queue_push_tasks(rq);
#endif /* CONFIG_SMP */
		if (p->prio < rq->curr->prio)
			resched_curr(rq);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 52 70.27% 1 12.50%
Peter Zijlstra 9 12.16% 2 25.00%
Kirill V Tkhai 5 6.76% 2 25.00%
Kirill Tkhai 3 4.05% 1 12.50%
Sebastian Andrzej Siewior 3 4.05% 1 12.50%
Ingo Molnar 2 2.70% 1 12.50%
Total 74 100.00% 8 100.00%

/*
 * Priority of the task has changed. This may cause
 * us to initiate a push or pull.
 */

static void

prio_changed_rt(struct rq *rq, struct task_struct *p, int oldprio)
{
	if (!task_on_rq_queued(p))
		return;

	if (rq->curr == p) {
#ifdef CONFIG_SMP
		/*
                 * If our priority decreases while running, we
                 * may need to pull tasks to this runqueue.
                 */
		if (oldprio < p->prio)
			queue_pull_task(rq);

		/*
                 * If there's a higher priority task waiting to run
                 * then reschedule.
                 */
		if (p->prio > rq->rt.highest_prio.curr)
			resched_curr(rq);
#else
		/* For UP simply resched on drop of prio */
		if (oldprio < p->prio)
			resched_curr(rq);
#endif /* CONFIG_SMP */
	} else {
		/*
                 * This task is not running, but if it is
                 * greater than the current running task
                 * then reschedule.
                 */
		if (p->prio < rq->curr->prio)
			resched_curr(rq);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Steven Rostedt 92 80.00% 1 16.67%
Peter Zijlstra 13 11.30% 2 33.33%
Kirill V Tkhai 5 4.35% 1 16.67%
Kirill Tkhai 3 2.61% 1 16.67%
Gregory Haskins 2 1.74% 1 16.67%
Total 115 100.00% 6 100.00%

#ifdef CONFIG_POSIX_TIMERS


static void watchdog(struct rq *rq, struct task_struct *p)
{
	unsigned long soft, hard;

	/* max may change after cur was read, this will be fixed next tick */
	soft = task_rlimit(p, RLIMIT_RTTIME);
	hard = task_rlimit_max(p, RLIMIT_RTTIME);

	if (soft != RLIM_INFINITY) {
		unsigned long next;

		if (p->rt.watchdog_stamp != jiffies) {
			p->rt.timeout++;
			p->rt.watchdog_stamp = jiffies;
		}

		next = DIV_ROUND_UP(min(soft, hard), USEC_PER_SEC/HZ);
		if (p->rt.timeout > next)
			p->cputime_expires.sched_exp = p->se.sum_exec_runtime;
	}
}
Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 86 72.88% 2 40.00%
Ying Xue 20 16.95% 1 20.00%
Jiri Slaby 9 7.63% 1 20.00%
Frank Mayhar 3 2.54% 1 20.00%
Total 118 100.00% 5 100.00%

#else


static inline void watchdog(struct rq *rq, struct task_struct *p) { }
Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 16 100.00% 1 100.00%
Total 16 100.00% 1 100.00%

#endif



static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
{
	struct sched_rt_entity *rt_se = &p->rt;

	update_curr_rt(rq);

	watchdog(rq, p);

	/*
         * RR tasks need a special form of timeslice management.
         * FIFO tasks have no timeslices.
         */
	if (p->policy != SCHED_RR)
		return;

	if (--p->rt.time_slice)
		return;

	p->rt.time_slice = sched_rr_timeslice;

	/*
         * Requeue to the end of queue if we (and all of our ancestors) are not
         * the only element on the queue
         */
	for_each_sched_rt_entity(rt_se) {
		if (rt_se->run_list.prev != rt_se->run_list.next) {
			requeue_task_rt(rq, p, 0);
			resched_curr(rq);
			return;
		}
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Ingo Molnar 46 42.99% 1 9.09%
Colin Cross 19 17.76% 1 9.09%
Dmitry Adamushko 19 17.76% 2 18.18%
Peter Zijlstra 19 17.76% 4 36.36%
Kirill Tkhai 2 1.87% 1 9.09%
Clark Williams 1 0.93% 1 9.09%
Li Bin 1 0.93% 1 9.09%
Total 107 100.00% 11 100.00%



static void set_curr_task_rt(struct rq *rq)
{
	struct task_struct *p = rq->curr;

	p->se.exec_start = rq_clock_task(rq);

	/* The running task is never eligible for pushing */
	dequeue_pushable_task(rq, p);
}

Contributors
Person Tokens Prop Commits CommitProp
Srivatsa Vaddagiri 28 71.79% 1 33.33%
Gregory Haskins 8 20.51% 1 33.33%
Frédéric Weisbecker 3 7.69% 1 33.33%
Total 39 100.00% 3 100.00%



static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
{
	/*
         * Time slice is 0 for SCHED_FIFO tasks
         */
	if (task->policy == SCHED_RR)
		return sched_rr_timeslice;
	else
		return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Peter Williams 26 78.79% 1 25.00%
Thomas Gleixner 5 15.15% 1 25.00%
Clark Williams 1 3.03% 1 25.00%
H Hartley Sweeten 1 3.03% 1 25.00%
Total 33 100.00% 4 100.00%


const struct sched_class rt_sched_class = {
	.next			= &fair_sched_class,
	.enqueue_task		= enqueue_task_rt,
	.dequeue_task		= dequeue_task_rt,
	.yield_task		= yield_task_rt,

	.check_preempt_curr	= check_preempt_curr_rt,

	.pick_next_task		= pick_next_task_rt,
	.put_prev_task		= put_prev_task_rt,

#ifdef CONFIG_SMP
	.select_task_rq		= select_task_rq_rt,

	.set_cpus_allowed       = set_cpus_allowed_common,
	.rq_online              = rq_online_rt,
	.rq_offline             = rq_offline_rt,
	.task_woken		= task_woken_rt,
	.switched_from		= switched_from_rt,
#endif

	.set_curr_task          = set_curr_task_rt,
	.task_tick		= task_tick_rt,

	.get_rr_interval	= get_rr_interval_rt,

	.prio_changed		= prio_changed_rt,
	.switched_to		= switched_to_rt,

	.update_curr		= update_curr_rt,
};

#ifdef CONFIG_RT_GROUP_SCHED
/*
 * Ensure that the real time constraints are schedulable.
 */
static DEFINE_MUTEX(rt_constraints_mutex);

/* Must be called with tasklist_lock held */


static inline int tg_has_rt_tasks(struct task_group *tg)
{
	struct task_struct *g, *p;

	/*
         * Autogroups do not have RT tasks; see autogroup_create().
         */
	if (task_group_is_autogroup(tg))
		return 0;

	for_each_process_thread(g, p) {
		if (rt_task(p) && task_group(p) == tg)
			return 1;
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 59 100.00% 1 100.00%
Total 59 100.00% 1 100.00%


struct rt_schedulable_data {
	
struct task_group *tg;
	
u64 rt_period;
	
u64 rt_runtime;
};



static int tg_rt_schedulable(struct task_group *tg, void *data)
{
	struct rt_schedulable_data *d = data;
	struct task_group *child;
	unsigned long total, sum = 0;
	u64 period, runtime;

	period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	runtime = tg->rt_bandwidth.rt_runtime;

	if (tg == d->tg) {
		period = d->rt_period;
		runtime = d->rt_runtime;
	}

	/*
         * Cannot have more runtime than the period.
         */
	if (runtime > period && runtime != RUNTIME_INF)
		return -EINVAL;

	/*
         * Ensure we don't starve existing RT tasks.
         */
	if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg))
		return -EBUSY;

	total = to_ratio(period, runtime);

	/*
         * Nobody can have more than the global setting allows.
         */
	if (total > to_ratio(global_rt_period(), global_rt_runtime()))
		return -EINVAL;

	/*
         * The sum of our children's runtime should not exceed our own.
         */
	list_for_each_entry_rcu(child, &tg->children, siblings) {
		period = ktime_to_ns(child->rt_bandwidth.rt_period);
		runtime = child->rt_bandwidth.rt_runtime;

		if (child == d->tg) {
			period = d->rt_period;
			runtime = d->rt_runtime;
		}

		sum += to_ratio(period, runtime);
	}

	if (sum > total)
		return -EINVAL;

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 215 100.00% 1 100.00%
Total 215 100.00% 1 100.00%



static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
{
	int ret;

	struct rt_schedulable_data data = {
		.tg = tg,
		.rt_period = period,
		.rt_runtime = runtime,
        };

	rcu_read_lock();
	ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data);
	rcu_read_unlock();

	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 62 100.00% 1 100.00%
Total 62 100.00% 1 100.00%



static int tg_set_rt_bandwidth(struct task_group *tg,
		u64 rt_period, u64 rt_runtime)
{
	int i, err = 0;

	/*
         * Disallowing the root group RT runtime is BAD, it would disallow the
         * kernel creating (and or operating) RT threads.
         */
	if (tg == &root_task_group && rt_runtime == 0)
		return -EINVAL;

	/* No period doesn't make any sense. */
	if (rt_period == 0)
		return -EINVAL;

	mutex_lock(&rt_constraints_mutex);
	read_lock(&tasklist_lock);
	err = __rt_schedulable(tg, rt_period, rt_runtime);
	if (err)
		goto unlock;

	raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock);
	tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period);
	tg->rt_bandwidth.rt_runtime = rt_runtime;

	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = tg->rt_rq[i];

		raw_spin_lock(&rt_rq->rt_runtime_lock);
		rt_rq->rt_runtime = rt_runtime;
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
	}
	raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock);
unlock:
	read_unlock(&tasklist_lock);
	mutex_unlock(&rt_constraints_mutex);

	return err;
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 177 100.00% 1 100.00%
Total 177 100.00% 1 100.00%



int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
{
	u64 rt_runtime, rt_period;

	rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
	rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
	if (rt_runtime_us < 0)
		rt_runtime = RUNTIME_INF;

	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 58 100.00% 1 100.00%
Total 58 100.00% 1 100.00%



long sched_group_rt_runtime(struct task_group *tg)
{
	u64 rt_runtime_us;

	if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF)
		return -1;

	rt_runtime_us = tg->rt_bandwidth.rt_runtime;
	do_div(rt_runtime_us, NSEC_PER_USEC);
	return rt_runtime_us;
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 45 100.00% 1 100.00%
Total 45 100.00% 1 100.00%



int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us)
{
	u64 rt_runtime, rt_period;

	rt_period = rt_period_us * NSEC_PER_USEC;
	rt_runtime = tg->rt_bandwidth.rt_runtime;

	return tg_set_rt_bandwidth(tg, rt_period, rt_runtime);
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 42 100.00% 1 100.00%
Total 42 100.00% 1 100.00%



long sched_group_rt_period(struct task_group *tg)
{
	u64 rt_period_us;

	rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period);
	do_div(rt_period_us, NSEC_PER_USEC);
	return rt_period_us;
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 34 100.00% 1 100.00%
Total 34 100.00% 1 100.00%



static int sched_rt_global_constraints(void)
{
	int ret = 0;

	mutex_lock(&rt_constraints_mutex);
	read_lock(&tasklist_lock);
	ret = __rt_schedulable(NULL, 0, 0);
	read_unlock(&tasklist_lock);
	mutex_unlock(&rt_constraints_mutex);

	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 51 100.00% 1 100.00%
Total 51 100.00% 1 100.00%



int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
{
	/* Don't accept realtime tasks when there is no way for them to run */
	if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0)
		return 0;

	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 37 100.00% 1 100.00%
Total 37 100.00% 1 100.00%

#else /* !CONFIG_RT_GROUP_SCHED */


static int sched_rt_global_constraints(void)
{
	unsigned long flags;
	int i;

	raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
	for_each_possible_cpu(i) {
		struct rt_rq *rt_rq = &cpu_rq(i)->rt;

		raw_spin_lock(&rt_rq->rt_runtime_lock);
		rt_rq->rt_runtime = global_rt_runtime();
		raw_spin_unlock(&rt_rq->rt_runtime_lock);
	}
	raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);

	return 0;
}
Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 80 100.00% 1 100.00%
Total 80 100.00% 1 100.00%

#endif /* CONFIG_RT_GROUP_SCHED */



static int sched_rt_global_validate(void)
{
	if (sysctl_sched_rt_period <= 0)
		return -EINVAL;

	if ((sysctl_sched_rt_runtime != RUNTIME_INF) &&
		(sysctl_sched_rt_runtime > sysctl_sched_rt_period))
		return -EINVAL;

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 39 100.00% 1 100.00%
Total 39 100.00% 1 100.00%



static void sched_rt_do_global(void)
{
	def_rt_bandwidth.rt_runtime = global_rt_runtime();
	def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period());
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 25 100.00% 1 100.00%
Total 25 100.00% 1 100.00%



int sched_rt_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int old_period, old_runtime;
	static DEFINE_MUTEX(mutex);
	int ret;

	mutex_lock(&mutex);
	old_period = sysctl_sched_rt_period;
	old_runtime = sysctl_sched_rt_runtime;

	ret = proc_dointvec(table, write, buffer, lenp, ppos);

	if (!ret && write) {
		ret = sched_rt_global_validate();
		if (ret)
			goto undo;

		ret = sched_dl_global_validate();
		if (ret)
			goto undo;

		ret = sched_rt_global_constraints();
		if (ret)
			goto undo;

		sched_rt_do_global();
		sched_dl_do_global();
	}
	if (0) {
undo:
		sysctl_sched_rt_period = old_period;
		sysctl_sched_rt_runtime = old_runtime;
	}
	mutex_unlock(&mutex);

	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 145 100.00% 1 100.00%
Total 145 100.00% 1 100.00%



int sched_rr_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int ret;
	static DEFINE_MUTEX(mutex);

	mutex_lock(&mutex);
	ret = proc_dointvec(table, write, buffer, lenp, ppos);
	/*
         * Make sure that internally we keep jiffies.
         * Also, writing zero resets the timeslice to default:
         */
	if (!ret && write) {
		sched_rr_timeslice =
			sysctl_sched_rr_timeslice <= 0 ? RR_TIMESLICE :
			msecs_to_jiffies(sysctl_sched_rr_timeslice);
	}
	mutex_unlock(&mutex);
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Nico Pitre 88 100.00% 1 100.00%
Total 88 100.00% 1 100.00%

#ifdef CONFIG_SCHED_DEBUG
extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq);



void print_rt_stats(struct seq_file *m, int cpu)
{
	rt_rq_iter_t iter;
	struct rt_rq *rt_rq;

	rcu_read_lock();
	for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))
		print_rt_rq(m, cpu, rt_rq);
	rcu_read_unlock();
}
Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 38 86.36% 1 50.00%
Cheng Xu 6 13.64% 1 50.00%
Total 44 100.00% 2 100.00%

#endif /* CONFIG_SCHED_DEBUG */
Overall Contributors
Person Tokens Prop Commits CommitProp
Peter Zijlstra 3854 38.00% 64 29.49%
Steven Rostedt 1557 15.35% 19 8.76%
Gregory Haskins 1426 14.06% 26 11.98%
Nico Pitre 1213 11.96% 2 0.92%
Ingo Molnar 494 4.87% 11 5.07%
Kirill V Tkhai 484 4.77% 12 5.53%
Dmitry Adamushko 137 1.35% 5 2.30%
Rusty Russell 119 1.17% 7 3.23%
Frédéric Weisbecker 87 0.86% 2 0.92%
Yong Zhang 86 0.85% 2 0.92%
Thomas Gleixner 55 0.54% 5 2.30%
Juri Lelli 53 0.52% 3 1.38%
Cheng Xu 53 0.52% 1 0.46%
Kirill Tkhai 44 0.43% 4 1.84%
Peter Williams 40 0.39% 3 1.38%
Tim Chen 38 0.37% 1 0.46%
Mike Galbraith 37 0.36% 3 1.38%
Dave Kleikamp 37 0.36% 1 0.46%
Balbir Singh 34 0.34% 1 0.46%
Srivatsa Vaddagiri 33 0.33% 2 0.92%
Dhaval Giani 22 0.22% 2 0.92%
Ying Xue 20 0.20% 1 0.46%
Colin Cross 19 0.19% 1 0.46%
Dario Faggioli 19 0.19% 1 0.46%
Hillf Danton 18 0.18% 2 0.92%
Xiaotian Feng 16 0.16% 1 0.46%
Wanpeng Li 16 0.16% 2 0.92%
John Blackwood 13 0.13% 1 0.46%
Shile Zhang 11 0.11% 1 0.46%
Frank Mayhar 10 0.10% 1 0.46%
Jiri Slaby 9 0.09% 1 0.46%
Dimitri Sivanich 8 0.08% 1 0.46%
Matt Fleming 7 0.07% 1 0.46%
Clark Williams 7 0.07% 1 0.46%
Peter Boonstoppel 6 0.06% 1 0.46%
Li Zefan 6 0.06% 2 0.92%
Yanmin Zhang 6 0.06% 1 0.46%
Xunlei Pang 6 0.06% 1 0.46%
Lucas De Marchi 5 0.05% 2 0.92%
Mike Travis 5 0.05% 1 0.46%
Giedrius Rekasius 5 0.05% 1 0.46%
Stanislaw Gruszka 5 0.05% 1 0.46%
Shawn Bohrer 4 0.04% 2 0.92%
Sebastian Andrzej Siewior 3 0.03% 1 0.46%
Rafael J. Wysocki 3 0.03% 1 0.46%
Hai Shan 1 0.01% 1 0.46%
Nathan Zimmer 1 0.01% 1 0.46%
H Hartley Sweeten 1 0.01% 1 0.46%
Jason Low 1 0.01% 1 0.46%
John Stultz 1 0.01% 1 0.46%
Greg Kroah-Hartman 1 0.01% 1 0.46%
Tejun Heo 1 0.01% 1 0.46%
Viresh Kumar 1 0.01% 1 0.46%
Yinghai Lu 1 0.01% 1 0.46%
Richard Weinberger 1 0.01% 1 0.46%
Li Bin 1 0.01% 1 0.46%
Christoph Lameter 1 0.01% 1 0.46%
Total 10142 100.00% 217 100.00%
Directory: kernel/sched

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.
Person	Tokens	Prop	Commits	CommitProp
Peter Zijlstra	117	100.00%	3	100.00%
Total	117	100.00%	3	100.00%
cregit-Linux how code gets into the kernel

Release 4.15 kernel/sched/rt.c

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors
