| // SPDX-License-Identifier: GPL-2.0 | |
| #include <linux/ftrace.h> | |
| #include <linux/tracepoint.h> | |
| #include <linux/kernel.h> | |
| #include <linux/module.h> | |
| #include <linux/init.h> | |
| #include <linux/rv.h> | |
| #include <rv/instrumentation.h> | |
| #define MODULE_NAME "nomiss" | |
| #include <uapi/linux/sched/types.h> | |
| #include <trace/events/syscalls.h> | |
| #include <trace/events/sched.h> | |
| #include <trace/events/task.h> | |
| #include <rv_trace.h> | |
| #define RV_MON_TYPE RV_MON_PER_OBJ | |
| #define HA_TIMER_TYPE HA_TIMER_WHEEL | |
| /* The start condition is on sched_switch, it's dangerous to allocate there */ | |
| #define DA_SKIP_AUTO_ALLOC | |
| typedef struct sched_dl_entity *monitor_target; | |
| #include "nomiss.h" | |
| #include <rv/ha_monitor.h> | |
| #include <monitors/deadline/deadline.h> | |
| /* | |
| * User configurable deadline threshold. If the total utilisation of deadline | |
| * tasks is larger than 1, they are only guaranteed bounded tardiness. See | |
| * Documentation/scheduler/sched-deadline.rst for more details. | |
| * The minimum tardiness without sched_feat(HRTICK_DL) is 1 tick to accommodate | |
| * for throttle enforced on the next tick. | |
| */ | |
| static u64 deadline_thresh = TICK_NSEC; | |
| module_param(deadline_thresh, ullong, 0644); | |
| #define DEADLINE_NS(ha_mon) (ha_get_target(ha_mon)->dl_deadline + deadline_thresh) | |
| static u64 ha_get_env(struct ha_monitor *ha_mon, enum envs_nomiss env, u64 time_ns) | |
| { | |
| if (env == clk_nomiss) | |
| return ha_get_clk_ns(ha_mon, env, time_ns); | |
| else if (env == is_constr_dl_nomiss) | |
| return !dl_is_implicit(ha_get_target(ha_mon)); | |
| else if (env == is_defer_nomiss) | |
| return ha_get_target(ha_mon)->dl_defer; | |
| return ENV_INVALID_VALUE; | |
| } | |
| static void ha_reset_env(struct ha_monitor *ha_mon, enum envs_nomiss env, u64 time_ns) | |
| { | |
| if (env == clk_nomiss) | |
| ha_reset_clk_ns(ha_mon, env, time_ns); | |
| } | |
| static inline bool ha_verify_invariants(struct ha_monitor *ha_mon, | |
| enum states curr_state, enum events event, | |
| enum states next_state, u64 time_ns) | |
| { | |
| if (curr_state == ready_nomiss) | |
| return ha_check_invariant_ns(ha_mon, clk_nomiss, time_ns, DEADLINE_NS(ha_mon)); | |
| else if (curr_state == running_nomiss) | |
| return ha_check_invariant_ns(ha_mon, clk_nomiss, time_ns, DEADLINE_NS(ha_mon)); | |
| return true; | |
| } | |
| static inline bool ha_verify_guards(struct ha_monitor *ha_mon, | |
| enum states curr_state, enum events event, | |
| enum states next_state, u64 time_ns) | |
| { | |
| bool res = true; | |
| if (curr_state == ready_nomiss && event == dl_replenish_nomiss) | |
| ha_reset_env(ha_mon, clk_nomiss, time_ns); | |
| else if (curr_state == ready_nomiss && event == dl_throttle_nomiss) | |
| res = ha_get_env(ha_mon, is_defer_nomiss, time_ns) == 1ull; | |
| else if (curr_state == idle_nomiss && event == dl_replenish_nomiss) | |
| ha_reset_env(ha_mon, clk_nomiss, time_ns); | |
| else if (curr_state == running_nomiss && event == dl_replenish_nomiss) | |
| ha_reset_env(ha_mon, clk_nomiss, time_ns); | |
| else if (curr_state == sleeping_nomiss && event == dl_replenish_nomiss) | |
| ha_reset_env(ha_mon, clk_nomiss, time_ns); | |
| else if (curr_state == sleeping_nomiss && event == dl_throttle_nomiss) | |
| res = ha_get_env(ha_mon, is_constr_dl_nomiss, time_ns) == 1ull || | |
| ha_get_env(ha_mon, is_defer_nomiss, time_ns) == 1ull; | |
| else if (curr_state == throttled_nomiss && event == dl_replenish_nomiss) | |
| ha_reset_env(ha_mon, clk_nomiss, time_ns); | |
| return res; | |
| } | |
| static inline void ha_setup_invariants(struct ha_monitor *ha_mon, | |
| enum states curr_state, enum events event, | |
| enum states next_state, u64 time_ns) | |
| { | |
| if (next_state == curr_state && event != dl_replenish_nomiss) | |
| return; | |
| if (next_state == ready_nomiss) | |
| ha_start_timer_ns(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns); | |
| else if (next_state == running_nomiss) | |
| ha_start_timer_ns(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns); | |
| else if (curr_state == ready_nomiss) | |
| ha_cancel_timer(ha_mon); | |
| else if (curr_state == running_nomiss) | |
| ha_cancel_timer(ha_mon); | |
| } | |
| static bool ha_verify_constraint(struct ha_monitor *ha_mon, | |
| enum states curr_state, enum events event, | |
| enum states next_state, u64 time_ns) | |
| { | |
| if (!ha_verify_invariants(ha_mon, curr_state, event, next_state, time_ns)) | |
| return false; | |
| if (!ha_verify_guards(ha_mon, curr_state, event, next_state, time_ns)) | |
| return false; | |
| ha_setup_invariants(ha_mon, curr_state, event, next_state, time_ns); | |
| return true; | |
| } | |
| static void handle_dl_replenish(void *data, struct sched_dl_entity *dl_se, | |
| int cpu, u8 type) | |
| { | |
| if (is_supported_type(type)) | |
| da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_replenish_nomiss); | |
| } | |
| static void handle_dl_throttle(void *data, struct sched_dl_entity *dl_se, | |
| int cpu, u8 type) | |
| { | |
| if (is_supported_type(type)) | |
| da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_throttle_nomiss); | |
| } | |
| static void handle_dl_server_stop(void *data, struct sched_dl_entity *dl_se, | |
| int cpu, u8 type) | |
| { | |
| /* | |
| * This isn't the standard use of da_handle_start_run_event since this | |
| * event cannot only occur from the initial state. | |
| * It is fine to use here because it always brings to a known state and | |
| * the fact we "pretend" the transition starts from the initial state | |
| * has no side effect. | |
| */ | |
| if (is_supported_type(type)) | |
| da_handle_start_run_event(EXPAND_ID(dl_se, cpu, type), dl_server_stop_nomiss); | |
| } | |
| static inline void handle_server_switch(struct task_struct *next, int cpu, u8 type) | |
| { | |
| struct sched_dl_entity *dl_se = get_server(next, type); | |
| if (dl_se && is_idle_task(next)) | |
| da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_server_idle_nomiss); | |
| } | |
| static void handle_sched_switch(void *data, bool preempt, | |
| struct task_struct *prev, | |
| struct task_struct *next, | |
| unsigned int prev_state) | |
| { | |
| int cpu = task_cpu(next); | |
| if (prev_state != TASK_RUNNING && !preempt && prev->policy == SCHED_DEADLINE) | |
| da_handle_event(EXPAND_ID_TASK(prev), sched_switch_suspend_nomiss); | |
| if (next->policy == SCHED_DEADLINE) | |
| da_handle_start_run_event(EXPAND_ID_TASK(next), sched_switch_in_nomiss); | |
| /* | |
| * The server is available in next only if the next task is boosted, | |
| * otherwise we need to retrieve it. | |
| * Here the server continues in the state running/armed until actually | |
| * stopped, this works since we continue expecting a throttle. | |
| */ | |
| if (next->dl_server) | |
| da_handle_start_event(EXPAND_ID(next->dl_server, cpu, | |
| get_server_type(next)), | |
| sched_switch_in_nomiss); | |
| else { | |
| handle_server_switch(next, cpu, DL_SERVER_FAIR); | |
| if (IS_ENABLED(CONFIG_SCHED_CLASS_EXT)) | |
| handle_server_switch(next, cpu, DL_SERVER_EXT); | |
| } | |
| } | |
| static void handle_sys_enter(void *data, struct pt_regs *regs, long id) | |
| { | |
| struct task_struct *p; | |
| int new_policy = -1; | |
| pid_t pid = 0; | |
| new_policy = extract_params(regs, id, &pid); | |
| if (new_policy < 0) | |
| return; | |
| guard(rcu)(); | |
| p = pid ? find_task_by_vpid(pid) : current; | |
| if (unlikely(!p) || new_policy == p->policy) | |
| return; | |
| if (p->policy == SCHED_DEADLINE) | |
| da_reset(EXPAND_ID_TASK(p)); | |
| else if (new_policy == SCHED_DEADLINE) | |
| da_create_or_get(EXPAND_ID_TASK(p)); | |
| } | |
| static void handle_sched_wakeup(void *data, struct task_struct *tsk) | |
| { | |
| if (tsk->policy == SCHED_DEADLINE) | |
| da_handle_event(EXPAND_ID_TASK(tsk), sched_wakeup_nomiss); | |
| } | |
| static int enable_nomiss(void) | |
| { | |
| int retval; | |
| retval = ha_monitor_init(); | |
| if (retval) | |
| return retval; | |
| retval = init_storage(false); | |
| if (retval) | |
| return retval; | |
| rv_attach_trace_probe("nomiss", sched_dl_replenish_tp, handle_dl_replenish); | |
| rv_attach_trace_probe("nomiss", sched_dl_throttle_tp, handle_dl_throttle); | |
| rv_attach_trace_probe("nomiss", sched_dl_server_stop_tp, handle_dl_server_stop); | |
| rv_attach_trace_probe("nomiss", sched_switch, handle_sched_switch); | |
| rv_attach_trace_probe("nomiss", sched_wakeup, handle_sched_wakeup); | |
| if (!should_skip_syscall_handle()) | |
| rv_attach_trace_probe("nomiss", sys_enter, handle_sys_enter); | |
| rv_attach_trace_probe("nomiss", task_newtask, handle_newtask); | |
| rv_attach_trace_probe("nomiss", sched_process_exit, handle_exit); | |
| return 0; | |
| } | |
| static void disable_nomiss(void) | |
| { | |
| rv_this.enabled = 0; | |
| /* Those are RCU writers, detach earlier hoping to close a bit faster */ | |
| rv_detach_trace_probe("nomiss", task_newtask, handle_newtask); | |
| rv_detach_trace_probe("nomiss", sched_process_exit, handle_exit); | |
| if (!should_skip_syscall_handle()) | |
| rv_detach_trace_probe("nomiss", sys_enter, handle_sys_enter); | |
| rv_detach_trace_probe("nomiss", sched_dl_replenish_tp, handle_dl_replenish); | |
| rv_detach_trace_probe("nomiss", sched_dl_throttle_tp, handle_dl_throttle); | |
| rv_detach_trace_probe("nomiss", sched_dl_server_stop_tp, handle_dl_server_stop); | |
| rv_detach_trace_probe("nomiss", sched_switch, handle_sched_switch); | |
| rv_detach_trace_probe("nomiss", sched_wakeup, handle_sched_wakeup); | |
| ha_monitor_destroy(); | |
| } | |
| static struct rv_monitor rv_this = { | |
| .name = "nomiss", | |
| .description = "dl entities run to completion before their deadline.", | |
| .enable = enable_nomiss, | |
| .disable = disable_nomiss, | |
| .reset = da_monitor_reset_all, | |
| .enabled = 0, | |
| }; | |
| static int __init register_nomiss(void) | |
| { | |
| return rv_register_monitor(&rv_this, &rv_deadline); | |
| } | |
| static void __exit unregister_nomiss(void) | |
| { | |
| rv_unregister_monitor(&rv_this); | |
| } | |
| module_init(register_nomiss); | |
| module_exit(unregister_nomiss); | |
| MODULE_LICENSE("GPL"); | |
| MODULE_AUTHOR("Gabriele Monaco <gmonaco@redhat.com>"); | |
| MODULE_DESCRIPTION("nomiss: dl entities run to completion before their deadline."); | |
| #if IS_ENABLED(CONFIG_RV_MONITORS_KUNIT_TEST) | |
| #include <kunit/visibility.h> | |
| #include "nomiss_kunit.h" | |
| const struct rv_nomiss_ops rv_nomiss_ops = { | |
| .mon = RV_MON_OPS_INIT(), | |
| .deadline_thresh = &deadline_thresh, | |
| .handle_dl_replenish = handle_dl_replenish, | |
| .handle_dl_throttle = handle_dl_throttle, | |
| .handle_dl_server_stop = handle_dl_server_stop, | |
| .handle_sched_switch = handle_sched_switch, | |
| .handle_sched_wakeup = handle_sched_wakeup, | |
| .handle_sys_enter = handle_sys_enter, | |
| .handle_newtask = handle_newtask, | |
| }; | |
| EXPORT_SYMBOL_IF_KUNIT(rv_nomiss_ops); | |
| #endif |