Release 4.15 kernel/sched/sched.h

/* SPDX-License-Identifier: GPL-2.0 */

#include <linux/sched.h>
#include <linux/sched/autogroup.h>
#include <linux/sched/sysctl.h>
#include <linux/sched/topology.h>
#include <linux/sched/rt.h>
#include <linux/sched/deadline.h>
#include <linux/sched/clock.h>
#include <linux/sched/wake_q.h>
#include <linux/sched/signal.h>
#include <linux/sched/numa_balancing.h>
#include <linux/sched/mm.h>
#include <linux/sched/cpufreq.h>
#include <linux/sched/stat.h>
#include <linux/sched/nohz.h>
#include <linux/sched/debug.h>
#include <linux/sched/hotplug.h>
#include <linux/sched/task.h>
#include <linux/sched/task_stack.h>
#include <linux/sched/cputime.h>
#include <linux/sched/init.h>

#include <linux/u64_stats_sync.h>
#include <linux/kernel_stat.h>
#include <linux/binfmts.h>
#include <linux/mutex.h>
#include <linux/spinlock.h>
#include <linux/stop_machine.h>
#include <linux/irq_work.h>
#include <linux/tick.h>
#include <linux/slab.h>
#include <linux/cgroup.h>

#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#endif

#include "cpupri.h"
#include "cpudeadline.h"

#ifdef CONFIG_SCHED_DEBUG

# define SCHED_WARN_ON(x)	WARN_ONCE(x, #x)
#else

# define SCHED_WARN_ON(x)	({ (void)(x), 0; })
#endif

struct rq;
struct cpuidle_state;

/* task_struct::on_rq states: */

#define TASK_ON_RQ_QUEUED	1

#define TASK_ON_RQ_MIGRATING	2

extern __read_mostly int scheduler_running;

extern unsigned long calc_load_update;
extern atomic_long_t calc_load_tasks;

extern void calc_global_load_tick(struct rq *this_rq);
extern long calc_load_fold_active(struct rq *this_rq, long adjust);

#ifdef CONFIG_SMP
extern void cpu_load_update_active(struct rq *this_rq);
#else


static inline void cpu_load_update_active(struct rq *this_rq) { }
Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
10
90.91%
1
50.00%
Frédéric Weisbecker
1
9.09%
1
50.00%
Total
11
100.00%
2
100.00%

#endif

/*
 * Helpers for converting nanosecond timing to jiffy resolution
 */

#define NS_TO_JIFFIES(TIME)	((unsigned long)(TIME) / (NSEC_PER_SEC / HZ))

/*
 * Increase resolution of nice-level calculations for 64-bit architectures.
 * The extra resolution improves shares distribution and load balancing of
 * low-weight task groups (eg. nice +19 on an autogroup), deeper taskgroup
 * hierarchies, especially on larger systems. This is not a user-visible change
 * and does not change the user-interface for setting shares/weights.
 *
 * We increase resolution only if we have enough bits to allow this increased
 * resolution (i.e. 64bit). The costs for increasing resolution when 32bit are
 * pretty high and the returns do not justify the increased costs.
 *
 * Really only required when CONFIG_FAIR_GROUP_SCHED is also set, but to
 * increase coverage and consistency always enable it on 64bit platforms.
 */
#ifdef CONFIG_64BIT

# define NICE_0_LOAD_SHIFT	(SCHED_FIXEDPOINT_SHIFT + SCHED_FIXEDPOINT_SHIFT)

# define scale_load(w)		((w) << SCHED_FIXEDPOINT_SHIFT)

# define scale_load_down(w)	((w) >> SCHED_FIXEDPOINT_SHIFT)
#else

# define NICE_0_LOAD_SHIFT	(SCHED_FIXEDPOINT_SHIFT)

# define scale_load(w)		(w)

# define scale_load_down(w)	(w)
#endif

/*
 * Task weight (visible to users) and its load (invisible to users) have
 * independent resolution, but they should be well calibrated. We use
 * scale_load() and scale_load_down(w) to convert between them. The
 * following must be true:
 *
 *  scale_load(sched_prio_to_weight[USER_PRIO(NICE_TO_PRIO(0))]) == NICE_0_LOAD
 *
 */

#define NICE_0_LOAD		(1L << NICE_0_LOAD_SHIFT)

/*
 * Single value that decides SCHED_DEADLINE internal math precision.
 * 10 -> just above 1us
 * 9  -> just above 0.5us
 */

#define DL_SCALE (10)

/*
 * These are the 'tuning knobs' of the scheduler:
 */

/*
 * single value that denotes runtime == period, ie unlimited time.
 */

#define RUNTIME_INF	((u64)~0ULL)



static inline int idle_policy(int policy)
{
	return policy == SCHED_IDLE;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Henrik Austad
15
100.00%
1
100.00%
Total
15
100.00%
1
100.00%



static inline int fair_policy(int policy)
{
	return policy == SCHED_NORMAL || policy == SCHED_BATCH;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dario Faggioli
19
100.00%
1
100.00%
Total
19
100.00%
1
100.00%



static inline int rt_policy(int policy)
{
	return policy == SCHED_FIFO || policy == SCHED_RR;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
18
94.74%
1
50.00%
Dario Faggioli
1
5.26%
1
50.00%
Total
19
100.00%
2
100.00%



static inline int dl_policy(int policy)
{
	return policy == SCHED_DEADLINE;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dario Faggioli
15
100.00%
1
100.00%
Total
15
100.00%
1
100.00%



static inline bool valid_policy(int policy)
{
	return idle_policy(policy) || fair_policy(policy) ||
		rt_policy(policy) || dl_policy(policy);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Henrik Austad
31
100.00%
1
100.00%
Total
31
100.00%
1
100.00%



static inline int task_has_rt_policy(struct task_struct *p)
{
	return rt_policy(p->policy);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
20
100.00%
1
100.00%
Total
20
100.00%
1
100.00%



static inline int task_has_dl_policy(struct task_struct *p)
{
	return dl_policy(p->policy);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dario Faggioli
20
100.00%
1
100.00%
Total
20
100.00%
1
100.00%

/*
 * Tells if entity @a should preempt entity @b.
 */

static inline bool

dl_entity_preempt(struct sched_dl_entity *a, struct sched_dl_entity *b)
{
	return dl_time_before(a->deadline, b->deadline);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dario Faggioli
29
100.00%
2
100.00%
Total
29
100.00%
2
100.00%

/*
 * This is the priority-queue data structure of the RT scheduling class:
 */

struct rt_prio_array {
	DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */
	
struct list_head queue[MAX_RT_PRIO];
};


struct rt_bandwidth {
	/* nests inside the rq lock: */
	
raw_spinlock_t		rt_runtime_lock;
	
ktime_t			rt_period;
	
u64			rt_runtime;
	
struct hrtimer		rt_period_timer;
	
unsigned int		rt_period_active;
};

void __dl_clear_params(struct task_struct *p);

/*
 * To keep the bandwidth of -deadline tasks and groups under control
 * we need some place where:
 *  - store the maximum -deadline bandwidth of the system (the group);
 *  - cache the fraction of that bandwidth that is currently allocated.
 *
 * This is all done in the data structure below. It is similar to the
 * one used for RT-throttling (rt_bandwidth), with the main difference
 * that, since here we are only interested in admission control, we
 * do not decrease any runtime while the group "executes", neither we
 * need a timer to replenish it.
 *
 * With respect to SMP, the bandwidth is given on a per-CPU basis,
 * meaning that:
 *  - dl_bw (< 100%) is the bandwidth of the system (group) on each CPU;
 *  - dl_total_bw array contains, in the i-eth element, the currently
 *    allocated bandwidth on the i-eth CPU.
 * Moreover, groups consume bandwidth on each CPU, while tasks only
 * consume bandwidth on the CPU they're running on.
 * Finally, dl_total_bw_cpu is used to cache the index of dl_total_bw
 * that will be shown the next time the proc or cgroup controls will
 * be red. It on its turn can be changed by writing on its own
 * control.
 */

struct dl_bandwidth {
	
raw_spinlock_t dl_runtime_lock;
	
u64 dl_runtime;
	
u64 dl_period;
};



static inline int dl_bandwidth_enabled(void)
{
	return sysctl_sched_rt_runtime >= 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dario Faggioli
13
92.86%
1
50.00%
Peter Zijlstra
1
7.14%
1
50.00%
Total
14
100.00%
2
100.00%


struct dl_bw {
	
raw_spinlock_t lock;
	

u64 bw, total_bw;
};

static inline void __dl_update(struct dl_bw *dl_b, s64 bw);


static inline

void __dl_sub(struct dl_bw *dl_b, u64 tsk_bw, int cpus)
{
	dl_b->total_bw -= tsk_bw;
	__dl_update(dl_b, (s32)tsk_bw / cpus);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Juri Lelli
20
55.56%
1
33.33%
Luca Abeni
15
41.67%
1
33.33%
Peter Zijlstra
1
2.78%
1
33.33%
Total
36
100.00%
3
100.00%


static inline

void __dl_add(struct dl_bw *dl_b, u64 tsk_bw, int cpus)
{
	dl_b->total_bw += tsk_bw;
	__dl_update(dl_b, -((s32)tsk_bw / cpus));
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Juri Lelli
21
53.85%
1
50.00%
Luca Abeni
18
46.15%
1
50.00%
Total
39
100.00%
2
100.00%


static inline

bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw)
{
	return dl_b->bw != -1 &&
	       dl_b->bw * cpus < dl_b->total_bw - old_bw + new_bw;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Juri Lelli
43
100.00%
1
100.00%
Total
43
100.00%
1
100.00%

void dl_change_utilization(struct task_struct *p, u64 new_bw);
extern void init_dl_bw(struct dl_bw *dl_b);
extern int sched_dl_global_validate(void);
extern void sched_dl_do_global(void);
extern int sched_dl_overflow(struct task_struct *p, int policy,
			     const struct sched_attr *attr);
extern void __setparam_dl(struct task_struct *p, const struct sched_attr *attr);
extern void __getparam_dl(struct task_struct *p, struct sched_attr *attr);
extern bool __checkparam_dl(const struct sched_attr *attr);
extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr);
extern int dl_task_can_attach(struct task_struct *p,
			      const struct cpumask *cs_cpus_allowed);
extern int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur,
					const struct cpumask *trial);
extern bool dl_cpu_busy(unsigned int cpu);

#ifdef CONFIG_CGROUP_SCHED

#include <linux/cgroup.h>

struct cfs_rq;
struct rt_rq;

extern struct list_head task_groups;


struct cfs_bandwidth {
#ifdef CONFIG_CFS_BANDWIDTH
	
raw_spinlock_t lock;
	
ktime_t period;
	

u64 quota, runtime;
	
s64 hierarchical_quota;
	
u64 runtime_expires;

	

int idle, period_active;
	

struct hrtimer period_timer, slack_timer;
	
struct list_head throttled_cfs_rq;

	/* statistics */
	

int nr_periods, nr_throttled;
	
u64 throttled_time;
#endif
};

/* task group related information */

struct task_group {
	
struct cgroup_subsys_state css;

#ifdef CONFIG_FAIR_GROUP_SCHED
	/* schedulable entities of this group on each cpu */
	
struct sched_entity **se;
	/* runqueue "owned" by this group on each cpu */
	
struct cfs_rq **cfs_rq;
	
unsigned long shares;

#ifdef	CONFIG_SMP
	/*
         * load_avg can be heavily contended at clock tick time, so put
         * it in its own cacheline separated from the fields above which
         * will also be accessed at each tick.
         */
	
atomic_long_t load_avg ____cacheline_aligned;
#endif
#endif

#ifdef CONFIG_RT_GROUP_SCHED
	
struct sched_rt_entity **rt_se;
	
struct rt_rq **rt_rq;

	
struct rt_bandwidth rt_bandwidth;
#endif

	
struct rcu_head rcu;
	
struct list_head list;

	
struct task_group *parent;
	
struct list_head siblings;
	
struct list_head children;

#ifdef CONFIG_SCHED_AUTOGROUP
	
struct autogroup *autogroup;
#endif

	
struct cfs_bandwidth cfs_bandwidth;
};

#ifdef CONFIG_FAIR_GROUP_SCHED

#define ROOT_TASK_GROUP_LOAD	NICE_0_LOAD

/*
 * A weight of 0 or 1 can cause arithmetics problems.
 * A weight of a cfs_rq is the sum of weights of which entities
 * are queued on this cfs_rq, so a weight of a entity should not be
 * too large, so as the shares value of a task group.
 * (The default weight is 1024 - so there's no practical
 *  limitation from this.)
 */

#define MIN_SHARES	(1UL <<  1)

#define MAX_SHARES	(1UL << 18)
#endif


typedef int (*tg_visitor)(struct task_group *, void *);

extern int walk_tg_tree_from(struct task_group *from,
			     tg_visitor down, tg_visitor up, void *data);

/*
 * Iterate the full tree, calling @down when first entering a node and @up when
 * leaving it for the final time.
 *
 * Caller must hold rcu_lock or sufficient equivalent.
 */


static inline int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
{
	return walk_tg_tree_from(&root_task_group, down, up, data);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
30
100.00%
1
100.00%
Total
30
100.00%
1
100.00%

extern int tg_nop(struct task_group *tg, void *data);

extern void free_fair_sched_group(struct task_group *tg);
extern int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent);
extern void online_fair_sched_group(struct task_group *tg);
extern void unregister_fair_sched_group(struct task_group *tg);
extern void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
			struct sched_entity *se, int cpu,
			struct sched_entity *parent);
extern void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b);

extern void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b);
extern void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b);
extern void unthrottle_cfs_rq(struct cfs_rq *cfs_rq);

extern void free_rt_sched_group(struct task_group *tg);
extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent);
extern void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
		struct sched_rt_entity *rt_se, int cpu,
		struct sched_rt_entity *parent);
extern int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us);
extern int sched_group_set_rt_period(struct task_group *tg, u64 rt_period_us);
extern long sched_group_rt_runtime(struct task_group *tg);
extern long sched_group_rt_period(struct task_group *tg);
extern int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk);

extern struct task_group *sched_create_group(struct task_group *parent);
extern void sched_online_group(struct task_group *tg,
			       struct task_group *parent);
extern void sched_destroy_group(struct task_group *tg);
extern void sched_offline_group(struct task_group *tg);

extern void sched_move_task(struct task_struct *tsk);

#ifdef CONFIG_FAIR_GROUP_SCHED
extern int sched_group_set_shares(struct task_group *tg, unsigned long shares);

#ifdef CONFIG_SMP
extern void set_task_rq_fair(struct sched_entity *se,
			     struct cfs_rq *prev, struct cfs_rq *next);
#else /* !CONFIG_SMP */


static inline void set_task_rq_fair(struct sched_entity *se,
			     struct cfs_rq *prev, struct cfs_rq *next) { }
Contributors
Person
Tokens
Prop
Commits
CommitProp
Byungchul Park
21
100.00%
1
100.00%
Total
21
100.00%
1
100.00%

#endif /* CONFIG_SMP */
#endif /* CONFIG_FAIR_GROUP_SCHED */

#else /* CONFIG_CGROUP_SCHED */


struct cfs_bandwidth { };

#endif	/* CONFIG_CGROUP_SCHED */

/* CFS-related fields in a runqueue */

struct cfs_rq {
	
struct load_weight load;
	
unsigned long runnable_weight;
	

unsigned int nr_running, h_nr_running;

	
u64 exec_clock;
	
u64 min_vruntime;
#ifndef CONFIG_64BIT
	
u64 min_vruntime_copy;
#endif

	
struct rb_root_cached tasks_timeline;

	/*
         * 'curr' points to currently running entity on this cfs_rq.
         * It is set to NULL otherwise (i.e when none are currently running).
         */
	



struct sched_entity *curr, *next, *last, *skip;

#ifdef	CONFIG_SCHED_DEBUG
	
unsigned int nr_spread_over;
#endif

#ifdef CONFIG_SMP
	/*
         * CFS load tracking
         */
	
struct sched_avg avg;
#ifndef CONFIG_64BIT
	
u64 load_last_update_time_copy;
#endif
	struct {
		
raw_spinlock_t	lock ____cacheline_aligned;
		
int		nr;
		
unsigned long	load_avg;
		
unsigned long	util_avg;
		
unsigned long	runnable_sum;
	
} removed;

#ifdef CONFIG_FAIR_GROUP_SCHED
	
unsigned long tg_load_avg_contrib;
	
long propagate;
	
long prop_runnable_sum;

	/*
         *   h_load = weight * f(tg)
         *
         * Where f(tg) is the recursive weight fraction assigned to
         * this group.
         */
	
unsigned long h_load;
	
u64 last_h_load_update;
	
struct sched_entity *h_load_next;
#endif /* CONFIG_FAIR_GROUP_SCHED */
#endif /* CONFIG_SMP */

#ifdef CONFIG_FAIR_GROUP_SCHED
	
struct rq *rq;	/* cpu runqueue to which this cfs_rq is attached */

	/*
         * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in
         * a hierarchy). Non-leaf lrqs hold other higher schedulable entities
         * (like users, containers etc.)
         *
         * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This
         * list is used during load balance.
         */
	
int on_list;
	
struct list_head leaf_cfs_rq_list;
	
struct task_group *tg;	/* group that "owns" this runqueue */

#ifdef CONFIG_CFS_BANDWIDTH
	
int runtime_enabled;
	
u64 runtime_expires;
	
s64 runtime_remaining;

	

u64 throttled_clock, throttled_clock_task;
	
u64 throttled_clock_task_time;
	

int throttled, throttle_count;
	
struct list_head throttled_list;
#endif /* CONFIG_CFS_BANDWIDTH */
#endif /* CONFIG_FAIR_GROUP_SCHED */
};



static inline int rt_bandwidth_enabled(void)
{
	return sysctl_sched_rt_runtime >= 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
14
100.00%
1
100.00%
Total
14
100.00%
1
100.00%

/* RT IPI pull logic requires IRQ_WORK */
#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_SMP)

# define HAVE_RT_PUSH_IPI
#endif

/* Real-Time classes' related field in a runqueue: */

struct rt_rq {
	
struct rt_prio_array active;
	
unsigned int rt_nr_running;
	
unsigned int rr_nr_running;
#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
	struct {
		
int curr; /* highest queued rt task prio */
#ifdef CONFIG_SMP
		
int next; /* next highest */
#endif
	
} highest_prio;
#endif
#ifdef CONFIG_SMP
	
unsigned long rt_nr_migratory;
	
unsigned long rt_nr_total;
	
int overloaded;
	
struct plist_head pushable_tasks;
#endif /* CONFIG_SMP */
	
int rt_queued;

	
int rt_throttled;
	
u64 rt_time;
	
u64 rt_runtime;
	/* Nests inside the rq lock: */
	
raw_spinlock_t rt_runtime_lock;

#ifdef CONFIG_RT_GROUP_SCHED
	
unsigned long rt_nr_boosted;

	
struct rq *rq;
	
struct task_group *tg;
#endif
};

/* Deadline class' related fields in a runqueue */

struct dl_rq {
	/* runqueue is an rbtree, ordered by deadline */
	
struct rb_root_cached root;

	
unsigned long dl_nr_running;

#ifdef CONFIG_SMP
	/*
         * Deadline values of the currently executing and the
         * earliest ready task on this rq. Caching these facilitates
         * the decision wether or not a ready but not running task
         * should migrate somewhere else.
         */
	struct {
		
u64 curr;
		
u64 next;
	
} earliest_dl;

	
unsigned long dl_nr_migratory;
	
int overloaded;

	/*
         * Tasks on this rq that can be pushed away. They are kept in
         * an rb-tree, ordered by tasks' deadlines, with caching
         * of the leftmost (earliest deadline) element.
         */
	
struct rb_root_cached pushable_dl_tasks_root;
#else
	
struct dl_bw dl_bw;
#endif
	/*
         * "Active utilization" for this runqueue: increased when a
         * task wakes up (becomes TASK_RUNNING) and decreased when a
         * task blocks
         */
	
u64 running_bw;

	/*
         * Utilization of the tasks "assigned" to this runqueue (including
         * the tasks that are in runqueue and the tasks that executed on this
         * CPU and blocked). Increased when a task moves to this runqueue, and
         * decreased when the task moves away (migrates, changes scheduling
         * policy, or terminates).
         * This is needed to compute the "inactive utilization" for the
         * runqueue (inactive utilization = this_bw - running_bw).
         */
	
u64 this_bw;
	
u64 extra_bw;

	/*
         * Inverse of the fraction of CPU utilization that can be reclaimed
         * by the GRUB algorithm.
         */
	
u64 bw_ratio;
};

#ifdef CONFIG_SMP



static inline bool sched_asym_prefer(int a, int b)
{
	return arch_asym_cpu_priority(a) > arch_asym_cpu_priority(b);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Tim Chen
24
100.00%
1
100.00%
Total
24
100.00%
1
100.00%

/*
 * We add the notion of a root-domain which will be used to define per-domain
 * variables. Each exclusive cpuset essentially defines an island domain by
 * fully partitioning the member cpus from any other cpuset. Whenever a new
 * exclusive cpuset is created, we also create and attach a new root-domain
 * object.
 *
 */

struct root_domain {
	
atomic_t refcount;
	
atomic_t rto_count;
	
struct rcu_head rcu;
	
cpumask_var_t span;
	
cpumask_var_t online;

	/* Indicate more than one runnable task for any CPU */
	
bool overload;

	/*
         * The bit corresponding to a CPU gets set here if such CPU has more
         * than one runnable -deadline task (as it is below for RT tasks).
         */
	
cpumask_var_t dlo_mask;
	
atomic_t dlo_count;
	
struct dl_bw dl_bw;
	
struct cpudl cpudl;

#ifdef HAVE_RT_PUSH_IPI
	/*
         * For IPI pull requests, loop across the rto_mask.
         */
	
struct irq_work rto_push_work;
	
raw_spinlock_t rto_lock;
	/* These are only updated and read within rto_lock */
	
int rto_loop;
	
int rto_cpu;
	/* These atomics are updated outside of a lock */
	
atomic_t rto_loop_next;
	
atomic_t rto_loop_start;
#endif
	/*
         * The "RT overload" flag: it gets set if a CPU has more than
         * one runnable RT task.
         */
	
cpumask_var_t rto_mask;
	
struct cpupri cpupri;

	
unsigned long max_cpu_capacity;
};

extern struct root_domain def_root_domain;
extern struct mutex sched_domains_mutex;

extern void init_defrootdomain(void);
extern int sched_init_domains(const struct cpumask *cpu_map);
extern void rq_attach_root(struct rq *rq, struct root_domain *rd);

#ifdef HAVE_RT_PUSH_IPI
extern void rto_push_irq_work_func(struct irq_work *work);
#endif
#endif /* CONFIG_SMP */

/*
 * This is the main, per-CPU runqueue data structure.
 *
 * Locking rule: those places that want to lock multiple runqueues
 * (such as the load balancing or the thread migration code), lock
 * acquire operations must be ordered by ascending &runqueue.
 */

struct rq {
	/* runqueue lock: */
	
raw_spinlock_t lock;

	/*
         * nr_running and cpu_load should be in the same cacheline because
         * remote CPUs use both these fields when doing load calculation.
         */
	
unsigned int nr_running;
#ifdef CONFIG_NUMA_BALANCING
	
unsigned int nr_numa_running;
	
unsigned int nr_preferred_running;
#endif
	
#define CPU_LOAD_IDX_MAX 5
	
unsigned long cpu_load[CPU_LOAD_IDX_MAX];
#ifdef CONFIG_NO_HZ_COMMON
#ifdef CONFIG_SMP
	
unsigned long last_load_update_tick;
#endif /* CONFIG_SMP */
	
unsigned long nohz_flags;
#endif /* CONFIG_NO_HZ_COMMON */
#ifdef CONFIG_NO_HZ_FULL
	
unsigned long last_sched_tick;
#endif
	/* capture load from *all* tasks on this cpu: */
	
struct load_weight load;
	
unsigned long nr_load_updates;
	
u64 nr_switches;

	
struct cfs_rq cfs;
	
struct rt_rq rt;
	
struct dl_rq dl;

#ifdef CONFIG_FAIR_GROUP_SCHED
	/* list of leaf cfs_rq on this cpu: */
	
struct list_head leaf_cfs_rq_list;
	
struct list_head *tmp_alone_branch;
#endif /* CONFIG_FAIR_GROUP_SCHED */

	/*
         * This is part of a global counter where only the total sum
         * over all CPUs matters. A task can increase this counter on
         * one CPU and if it got migrated afterwards it may decrease
         * it on another CPU. Always updated under the runqueue lock:
         */
	
unsigned long nr_uninterruptible;

	


struct task_struct *curr, *idle, *stop;
	
unsigned long next_balance;
	
struct mm_struct *prev_mm;

	
unsigned int clock_update_flags;
	
u64 clock;
	
u64 clock_task;

	
atomic_t nr_iowait;

#ifdef CONFIG_SMP
	
struct root_domain *rd;
	
struct sched_domain *sd;

	
unsigned long cpu_capacity;
	
unsigned long cpu_capacity_orig;

	
struct callback_head *balance_callback;

	
unsigned char idle_balance;
	/* For active balancing */
	
int active_balance;
	
int push_cpu;
	
struct cpu_stop_work active_balance_work;
	/* cpu of this runqueue: */
	
int cpu;
	
int online;

	
struct list_head cfs_tasks;

	
u64 rt_avg;
	
u64 age_stamp;
	
u64 idle_stamp;
	
u64 avg_idle;

	/* This is used to determine avg_idle's max value */
	
u64 max_idle_balance_cost;
#endif

#ifdef CONFIG_IRQ_TIME_ACCOUNTING
	
u64 prev_irq_time;
#endif
#ifdef CONFIG_PARAVIRT
	
u64 prev_steal_time;
#endif
#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
	
u64 prev_steal_time_rq;
#endif

	/* calc_load related fields */
	
unsigned long calc_load_update;
	
long calc_load_active;

#ifdef CONFIG_SCHED_HRTICK
#ifdef CONFIG_SMP
	
int hrtick_csd_pending;
	
call_single_data_t hrtick_csd;
#endif
	
struct hrtimer hrtick_timer;
#endif

#ifdef CONFIG_SCHEDSTATS
	/* latency stats */
	
struct sched_info rq_sched_info;
	
unsigned long long rq_cpu_time;
	/* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */

	/* sys_sched_yield() stats */
	
unsigned int yld_count;

	/* schedule() stats */
	
unsigned int sched_count;
	
unsigned int sched_goidle;

	/* try_to_wake_up() stats */
	
unsigned int ttwu_count;
	
unsigned int ttwu_local;
#endif

#ifdef CONFIG_SMP
	
struct llist_head wake_list;
#endif

#ifdef CONFIG_CPU_IDLE
	/* Must be inspected within a rcu lock section */
	
struct cpuidle_state *idle_state;
#endif
};



static inline int cpu_of(struct rq *rq)
{
#ifdef CONFIG_SMP
	return rq->cpu;
#else
	return 0;
#endif
}


Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
27
100.00%
1
100.00%
Total
27
100.00%
1
100.00%

#ifdef CONFIG_SCHED_SMT

extern struct static_key_false sched_smt_present;

extern void __update_idle_core(struct rq *rq);



static inline void update_idle_core(struct rq *rq)
{
	if (static_branch_unlikely(&sched_smt_present))
		__update_idle_core(rq);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
25
100.00%
1
100.00%
Total
25
100.00%
1
100.00%

#else


static inline void update_idle_core(struct rq *rq) { }
Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
11
100.00%
1
100.00%
Total
11
100.00%
1
100.00%

#endif

DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);


#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))

#define this_rq()		this_cpu_ptr(&runqueues)

#define task_rq(p)		cpu_rq(task_cpu(p))

#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)

#define raw_rq()		raw_cpu_ptr(&runqueues)



static inline u64 __rq_clock_broken(struct rq *rq)
{
	return READ_ONCE(rq->clock);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Peter Zijlstra
19
95.00%
1
50.00%
Jason Low
1
5.00%
1
50.00%
Total
20
100.00%
2
100.00%

/*
 * rq::clock_update_flags bits
 *
 * %RQCF_REQ_SKIP - will request skipping of clock update on the next
 *  call to __schedule(). This is an optimisation to avoid
 *  neighbouring rq clock updates.
 *
 * %RQCF_ACT_SKIP - is set from inside of __schedule() when skipping is
 *  in effect and calls to update_rq_clock() are being ignored.
 *
 * %RQCF_UPDATED - is a debug flag that indicates whether a call has been
 *  made to update_rq_clock() since the last time rq::lock was pinned.
 *
 * If inside of __schedule(), clock_update_flags will have been
 * shifted left (a left shift is a cheap operation for the fast path
 * to promote %RQCF_REQ_SKIP to %RQCF_ACT_SKIP), so you must use,
 *
 *      if (rq-clock_update_flags >= RQCF_UPDATED)
 *
 * to check if %RQCF_UPADTED is set. It'll never be shifted more than
 * one position though, because the next rq_unpin_lock() will shift it
 * back.
 */

#define RQCF_REQ_SKIP	0x01

#define RQCF_ACT_SKIP	0x02

#define RQCF_UPDATED	0x04



static inline void assert_clock_updated(struct rq *rq)
{
	/*
         * The only reason for not seeing a clock update since the
         * last rq_pin_lock() is if we're currently skipping updates.
         */
	SCHED_WARN_ON(rq->clock_update_flags < RQCF_ACT_SKIP);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Matt Fleming
22
100.00%
1
100.00%
Total
22
100.00%
1
100.00%



static inline u64 rq_clock(struct rq *rq)
{
	lockdep_assert_held(&rq->lock);
	assert_clock_updated(rq);

	return