Contributors: 2
Author Tokens Token Proportion Commits Commit Proportion
Gabriele Monaco 1469 99.46% 2 66.67%
Daniel Bristot de Oliveira 8 0.54% 1 33.33%
Total 1477 3 


// SPDX-License-Identifier: GPL-2.0
#include <linux/ftrace.h>
#include <linux/tracepoint.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/rv.h>
#include <rv/instrumentation.h>

#define MODULE_NAME "nomiss"

#include <uapi/linux/sched/types.h>
#include <trace/events/syscalls.h>
#include <trace/events/sched.h>
#include <trace/events/task.h>
#include <rv_trace.h>

#define RV_MON_TYPE RV_MON_PER_OBJ
#define HA_TIMER_TYPE HA_TIMER_WHEEL
/* The start condition is on sched_switch, it's dangerous to allocate there */
#define DA_SKIP_AUTO_ALLOC
typedef struct sched_dl_entity *monitor_target;
#include "nomiss.h"
#include <rv/ha_monitor.h>
#include <monitors/deadline/deadline.h>

/*
 * User configurable deadline threshold. If the total utilisation of deadline
 * tasks is larger than 1, they are only guaranteed bounded tardiness. See
 * Documentation/scheduler/sched-deadline.rst for more details.
 * The minimum tardiness without sched_feat(HRTICK_DL) is 1 tick to accommodate
 * for throttle enforced on the next tick.
 */
static u64 deadline_thresh = TICK_NSEC;
module_param(deadline_thresh, ullong, 0644);
#define DEADLINE_NS(ha_mon) (ha_get_target(ha_mon)->dl_deadline + deadline_thresh)

static u64 ha_get_env(struct ha_monitor *ha_mon, enum envs_nomiss env, u64 time_ns)
{
	if (env == clk_nomiss)
		return ha_get_clk_ns(ha_mon, env, time_ns);
	else if (env == is_constr_dl_nomiss)
		return !dl_is_implicit(ha_get_target(ha_mon));
	else if (env == is_defer_nomiss)
		return ha_get_target(ha_mon)->dl_defer;
	return ENV_INVALID_VALUE;
}

static void ha_reset_env(struct ha_monitor *ha_mon, enum envs_nomiss env, u64 time_ns)
{
	if (env == clk_nomiss)
		ha_reset_clk_ns(ha_mon, env, time_ns);
}

static inline bool ha_verify_invariants(struct ha_monitor *ha_mon,
					enum states curr_state, enum events event,
					enum states next_state, u64 time_ns)
{
	if (curr_state == ready_nomiss)
		return ha_check_invariant_ns(ha_mon, clk_nomiss, time_ns);
	else if (curr_state == running_nomiss)
		return ha_check_invariant_ns(ha_mon, clk_nomiss, time_ns);
	return true;
}

static inline void ha_convert_inv_guard(struct ha_monitor *ha_mon,
					enum states curr_state, enum events event,
					enum states next_state, u64 time_ns)
{
	if (curr_state == next_state)
		return;
	if (curr_state == ready_nomiss)
		ha_inv_to_guard(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns);
	else if (curr_state == running_nomiss)
		ha_inv_to_guard(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns);
}

static inline bool ha_verify_guards(struct ha_monitor *ha_mon,
				    enum states curr_state, enum events event,
				    enum states next_state, u64 time_ns)
{
	bool res = true;

	if (curr_state == ready_nomiss && event == dl_replenish_nomiss)
		ha_reset_env(ha_mon, clk_nomiss, time_ns);
	else if (curr_state == ready_nomiss && event == dl_throttle_nomiss)
		res = ha_get_env(ha_mon, is_defer_nomiss, time_ns) == 1ull;
	else if (curr_state == idle_nomiss && event == dl_replenish_nomiss)
		ha_reset_env(ha_mon, clk_nomiss, time_ns);
	else if (curr_state == running_nomiss && event == dl_replenish_nomiss)
		ha_reset_env(ha_mon, clk_nomiss, time_ns);
	else if (curr_state == sleeping_nomiss && event == dl_replenish_nomiss)
		ha_reset_env(ha_mon, clk_nomiss, time_ns);
	else if (curr_state == sleeping_nomiss && event == dl_throttle_nomiss)
		res = ha_get_env(ha_mon, is_constr_dl_nomiss, time_ns) == 1ull ||
		      ha_get_env(ha_mon, is_defer_nomiss, time_ns) == 1ull;
	else if (curr_state == throttled_nomiss && event == dl_replenish_nomiss)
		ha_reset_env(ha_mon, clk_nomiss, time_ns);
	return res;
}

static inline void ha_setup_invariants(struct ha_monitor *ha_mon,
				       enum states curr_state, enum events event,
				       enum states next_state, u64 time_ns)
{
	if (next_state == curr_state && event != dl_replenish_nomiss)
		return;
	if (next_state == ready_nomiss)
		ha_start_timer_ns(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns);
	else if (next_state == running_nomiss)
		ha_start_timer_ns(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns);
	else if (curr_state == ready_nomiss)
		ha_cancel_timer(ha_mon);
	else if (curr_state == running_nomiss)
		ha_cancel_timer(ha_mon);
}

static bool ha_verify_constraint(struct ha_monitor *ha_mon,
				 enum states curr_state, enum events event,
				 enum states next_state, u64 time_ns)
{
	if (!ha_verify_invariants(ha_mon, curr_state, event, next_state, time_ns))
		return false;

	ha_convert_inv_guard(ha_mon, curr_state, event, next_state, time_ns);

	if (!ha_verify_guards(ha_mon, curr_state, event, next_state, time_ns))
		return false;

	ha_setup_invariants(ha_mon, curr_state, event, next_state, time_ns);

	return true;
}

static void handle_dl_replenish(void *data, struct sched_dl_entity *dl_se,
				int cpu, u8 type)
{
	if (is_supported_type(type))
		da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_replenish_nomiss);
}

static void handle_dl_throttle(void *data, struct sched_dl_entity *dl_se,
			       int cpu, u8 type)
{
	if (is_supported_type(type))
		da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_throttle_nomiss);
}

static void handle_dl_server_stop(void *data, struct sched_dl_entity *dl_se,
				  int cpu, u8 type)
{
	/*
	 * This isn't the standard use of da_handle_start_run_event since this
	 * event cannot only occur from the initial state.
	 * It is fine to use here because it always brings to a known state and
	 * the fact we "pretend" the transition starts from the initial state
	 * has no side effect.
	 */
	if (is_supported_type(type))
		da_handle_start_run_event(EXPAND_ID(dl_se, cpu, type), dl_server_stop_nomiss);
}

static inline void handle_server_switch(struct task_struct *next, int cpu, u8 type)
{
	struct sched_dl_entity *dl_se = get_server(next, type);

	if (dl_se && is_idle_task(next))
		da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_server_idle_nomiss);
}

static void handle_sched_switch(void *data, bool preempt,
				struct task_struct *prev,
				struct task_struct *next,
				unsigned int prev_state)
{
	int cpu = task_cpu(next);

	if (prev_state != TASK_RUNNING && !preempt && prev->policy == SCHED_DEADLINE)
		da_handle_event(EXPAND_ID_TASK(prev), sched_switch_suspend_nomiss);
	if (next->policy == SCHED_DEADLINE)
		da_handle_start_run_event(EXPAND_ID_TASK(next), sched_switch_in_nomiss);

	/*
	 * The server is available in next only if the next task is boosted,
	 * otherwise we need to retrieve it.
	 * Here the server continues in the state running/armed until actually
	 * stopped, this works since we continue expecting a throttle.
	 */
	if (next->dl_server)
		da_handle_start_event(EXPAND_ID(next->dl_server, cpu,
						get_server_type(next)),
				      sched_switch_in_nomiss);
	else {
		handle_server_switch(next, cpu, DL_SERVER_FAIR);
		if (IS_ENABLED(CONFIG_SCHED_CLASS_EXT))
			handle_server_switch(next, cpu, DL_SERVER_EXT);
	}
}

static void handle_sys_enter(void *data, struct pt_regs *regs, long id)
{
	struct task_struct *p;
	int new_policy = -1;
	pid_t pid = 0;

	new_policy = extract_params(regs, id, &pid);
	if (new_policy < 0)
		return;
	guard(rcu)();
	p = pid ? find_task_by_vpid(pid) : current;
	if (unlikely(!p) || new_policy == p->policy)
		return;

	if (p->policy == SCHED_DEADLINE)
		da_reset(EXPAND_ID_TASK(p));
	else if (new_policy == SCHED_DEADLINE)
		da_create_or_get(EXPAND_ID_TASK(p));
}

static void handle_sched_wakeup(void *data, struct task_struct *tsk)
{
	if (tsk->policy == SCHED_DEADLINE)
		da_handle_event(EXPAND_ID_TASK(tsk), sched_wakeup_nomiss);
}

static int enable_nomiss(void)
{
	int retval;

	retval = ha_monitor_init();
	if (retval)
		return retval;

	retval = init_storage(false);
	if (retval)
		return retval;
	rv_attach_trace_probe("nomiss", sched_dl_replenish_tp, handle_dl_replenish);
	rv_attach_trace_probe("nomiss", sched_dl_throttle_tp, handle_dl_throttle);
	rv_attach_trace_probe("nomiss", sched_dl_server_stop_tp, handle_dl_server_stop);
	rv_attach_trace_probe("nomiss", sched_switch, handle_sched_switch);
	rv_attach_trace_probe("nomiss", sched_wakeup, handle_sched_wakeup);
	if (!should_skip_syscall_handle())
		rv_attach_trace_probe("nomiss", sys_enter, handle_sys_enter);
	rv_attach_trace_probe("nomiss", task_newtask, handle_newtask);
	rv_attach_trace_probe("nomiss", sched_process_exit, handle_exit);

	return 0;
}

static void disable_nomiss(void)
{
	rv_this.enabled = 0;

	/* Those are RCU writers, detach earlier hoping to close a bit faster */
	rv_detach_trace_probe("nomiss", task_newtask, handle_newtask);
	rv_detach_trace_probe("nomiss", sched_process_exit, handle_exit);
	if (!should_skip_syscall_handle())
		rv_detach_trace_probe("nomiss", sys_enter, handle_sys_enter);

	rv_detach_trace_probe("nomiss", sched_dl_replenish_tp, handle_dl_replenish);
	rv_detach_trace_probe("nomiss", sched_dl_throttle_tp, handle_dl_throttle);
	rv_detach_trace_probe("nomiss", sched_dl_server_stop_tp, handle_dl_server_stop);
	rv_detach_trace_probe("nomiss", sched_switch, handle_sched_switch);
	rv_detach_trace_probe("nomiss", sched_wakeup, handle_sched_wakeup);

	ha_monitor_destroy();
}

static struct rv_monitor rv_this = {
	.name = "nomiss",
	.description = "dl entities run to completion before their deadline.",
	.enable = enable_nomiss,
	.disable = disable_nomiss,
	.reset = da_monitor_reset_all,
	.enabled = 0,
};

static int __init register_nomiss(void)
{
	return rv_register_monitor(&rv_this, &rv_deadline);
}

static void __exit unregister_nomiss(void)
{
	rv_unregister_monitor(&rv_this);
}

module_init(register_nomiss);
module_exit(unregister_nomiss);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Gabriele Monaco <gmonaco@redhat.com>");
MODULE_DESCRIPTION("nomiss: dl entities run to completion before their deadline.");