From c848a4fd1ecdcb8d44ec2027690e5a30ee236776 Mon Sep 17 00:00:00 2001 From: Cruz Zhao Date: Wed, 26 May 2021 11:00:04 +0800 Subject: [PATCH 2741/2944] alinux: sched: fix the performence regression caused by update_rq_on_expel() fix #34359599 fix #34361563 fix #34362981 In order to prevent scheduling errors in the case of ipi failure, update_rq_on_expel() is called in function pick_next_task_fair(), but this will cause performence regression. In order to prevent errors while ensuring performence, we introduce an interval to call update_rq_on_expel(), the default value is 10, unit: ms. We also introdue a sched_feat: ID_LOOSE_EXPEL to control whether to call update_rq_on_expel() in function pick_next_task_fair() periodly, if on, update_rq_on_expel() will be called loosely, and the interval will be larger than sysctl_sched_expel_update_interval between calls, called everytime otherwise. The default value of ID_LOOSE_EXPEL is ON. Here follows the data of test will-it-scale-sched_yield-thread-15 before applying this patch: tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 1,0,0.00,1479767,95.80,1479767 tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 48,0,0.00,15057129,0.12,0 tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 96,0,0.00,14793466,0.14,0 Here follows the data of test will-it-scale-sched_yield-thread-15 after applying this patch with sysctl_sched_expel_update_interval = 10 and ID_LOOSE_EXPEL on: tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 1,0,0.00,1555869,93.80,1555869 tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 48,0,0.00,15592435,0.12,0 tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 96,0,0.00,15282931,0.15,0 We also tested it with sysbench, and here is the scenario: online task: sysbench cpu number of online tasks: 8 number of threads each online task: 8 bvt of online tasks: 2 offline task: sysbench mysql number of offline tasks: 4 number of threads each offline task: 4 bvt of offline task: -1 Here follows the data before applying this patch: online task latency: 2.06ms, online task events: 58434.90625 offline task latency: 132.75ms, offline task events: 904.4375 Here follows the data after appling this patch: online task latency: 1.99875ms, online task events: 60314.71875 offline task latency: 124.255ms, offline task events: 966.5625 As we can see from the result, this patch can resolve the performence regression to a certain degree. Signed-off-by: Cruz Zhao Acked-by: Michael Wang --- include/linux/sched/sysctl.h | 1 + kernel/sched/fair.c | 28 +++++++++++++++++++++++++++- kernel/sched/features.h | 1 + kernel/sched/sched.h | 1 + kernel/sysctl.c | 7 +++++++ 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 7748250..d975618b 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -67,6 +67,7 @@ int sched_proc_update_handler(struct ctl_table *table, int write, extern unsigned int sysctl_sched_idle_saver_wmark; #ifdef CONFIG_SCHED_SMT extern int sysctl_sched_expel_idle_balance_delay; +extern unsigned long sysctl_sched_expel_update_interval; #endif #endif diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c479408..9667222 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -141,6 +141,16 @@ int __weak arch_asym_cpu_priority(int cpu) * Default: -1, units: ms */ int sysctl_sched_expel_idle_balance_delay = -1; +/* + * In order to prevent scheduling errors in the case of ipi failure, + * update_rq_on_expel() is called in function pick_next_task_fair(), + * but this will cause performence regression. In order to prevent + * errors while ensuring performence, we introduce an interval to call + * update_rq_on_expel(). + * + * Default: 10, units: ms + */ +const_debug unsigned long sysctl_sched_expel_update_interval = 10; #endif #endif @@ -671,6 +681,14 @@ static inline void update_rq_on_expel(struct rq *rq) rq->on_expel = ret; } +static inline void loose_update_rq_on_expel(struct rq *rq) +{ + if (time_after(jiffies, rq->expel_next_update) && rq->nr_under_running) { + update_rq_on_expel(rq); + rq->expel_next_update = jiffies + sysctl_sched_expel_update_interval; + } +} + static inline bool rq_on_expel(struct rq *rq) { return rq->on_expel; @@ -750,6 +768,10 @@ static inline void update_rq_on_expel(struct rq *rq) { } +static inline void loose_update_rq_on_expel(struct rq *rq) +{ +} + static inline bool rq_on_expel(struct rq *rq) { return false; @@ -7948,7 +7970,11 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ if (!cfs_rq->nr_running) goto idle; - update_rq_on_expel(rq); + if (sched_feat(ID_LOOSE_EXPEL)) + loose_update_rq_on_expel(rq); + else + update_rq_on_expel(rq); + if (expellee_only(rq)) { /* * In order to mark CPU as IDLE, we need to call diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 92db87a..8603c81 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -95,4 +95,5 @@ SCHED_FEAT(ID_IDLE_AVG, true) SCHED_FEAT(ID_RESCUE_EXPELLEE, true) SCHED_FEAT(ID_EXPELLEE_NEVER_HOT, false) +SCHED_FEAT(ID_LOOSE_EXPEL, true) #endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 621d433..97be8dd 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -873,6 +873,7 @@ struct rq { u64 avg_id_idle; #ifdef CONFIG_SCHED_SMT unsigned long next_expel_ib; + unsigned long expel_next_update; #endif #endif diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 2e00f7c..455df53 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -536,6 +536,13 @@ static int sysrq_sysctl_handler(struct ctl_table *table, int write, .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "sched_expel_update_interval", + .data = &sysctl_sched_expel_update_interval, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_dointvec, + }, #endif #endif #ifdef CONFIG_PROVE_LOCKING -- 1.8.3.1