From e6f05bd1d63a84a9f0cb6e869f14f3428c4048a3 Mon Sep 17 00:00:00 2001 From: Cruz Zhao Date: Wed, 26 May 2021 11:00:04 +0800 Subject: [PATCH 2918/2944] alinux: sched: fix the performence regression caused by update_rq_on_expel() fix #34359599 fix #34361563 In order to prevent scheduling errors in the case of ipi failure, update_rq_on_expel() is called in function pick_next_task_fair(), but this will cause performence regression. In order to prevent errors while ensuring performence, we introduce an interval to call update_rq_on_expel(), the default value is 10, unit: jiffy. We also introdue a sched_feat: ID_LOOSE_EXPEL to control whether to call update_rq_on_expel() in function pick_next_task_fair() periodly, if on, update_rq_on_expel() will be called loosely, and the interval will be larger than sysctl_sched_expel_update_interval between calls, called everytime otherwise. The default value of ID_LOOSE_EXPEL is OFF. Here follows the data of test will-it-scale-sched_yield-thread-15 before applying this patch: will-it-scale-sched_yield-thread-15-1 tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 1,0,0.00,1479767,95.80,1479767 will-it-scale-sched_yield-thread-15-50% tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 48,0,0.00,15057129,0.12,0 will-it-scale-sched_yield-thread-15-100% tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 96,0,0.00,14793466,0.14,0 Here follows the data of test will-it-scale-sched_yield-thread-15 after applying this patch with sysctl_sched_expel_update_interval = 10 and ID_LOOSE_EXPEL on: will-it-scale-sched_yield-thread-15-1 tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 1,0,0.00,1566401,95.82,1566401 will-it-scale-sched_yield-thread-15-50% tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 48,0,0.00,15789744,0.12,0 will-it-scale-sched_yield-thread-15-100% tasks,processes,processes_idle,threads,threads_idle,linear 0,0,100,0,100,0 96,0,0.00,15612437,0.15,0 We also tested it with nginx + ffmpeg, and here is the scenario: online task: nginx number of online tasks: 2 bvt of online tasks: 2 offline task: ffmpeg number of offline tasks: 4 bvt of offline task: -1 Here follows the data before applying this patch: only online online+offline cpu usage: 7.79% 46.69% nginx rt: 0.84ms 0.92ms Here follows the data after appling this patch sched_expel_update_interval = 10(unit: jiffy) only online online+offline cpu usage: 7.91% 46.84% nginx rt: 0.82ms 0.89ms sched_expel_update_interval = 100(unit: jiffy) only online online+offline cpu usage: 7.81% 46.45% nginx rt: 0.81ms 0.87ms sched_expel_update_interval = 1000(unit: jiffy) only online online+offline cpu usage: 7.91% 46.19% nginx rt: 0.81ms 0.87ms As we can see from the result, this patch can resolve the performence regression to a certain degree. Signed-off-by: Cruz Zhao Acked-by: Michael Wang --- include/linux/sched/sysctl.h | 1 + kernel/sched/fair.c | 38 +++++++++++++++++++++++++++++++++++++- kernel/sched/features.h | 1 + kernel/sched/sched.h | 1 + kernel/sysctl.c | 7 +++++++ 5 files changed, 47 insertions(+), 1 deletion(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 7748250..d975618b 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -67,6 +67,7 @@ int sched_proc_update_handler(struct ctl_table *table, int write, extern unsigned int sysctl_sched_idle_saver_wmark; #ifdef CONFIG_SCHED_SMT extern int sysctl_sched_expel_idle_balance_delay; +extern unsigned long sysctl_sched_expel_update_interval; #endif #endif diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c479408..599d0f9 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -141,6 +141,16 @@ int __weak arch_asym_cpu_priority(int cpu) * Default: -1, units: ms */ int sysctl_sched_expel_idle_balance_delay = -1; +/* + * In order to prevent scheduling errors in the case of ipi failure, + * update_rq_on_expel() is called in function pick_next_task_fair(), + * but this will cause performence regression. In order to prevent + * errors while ensuring performence, we introduce an interval to call + * update_rq_on_expel(). + * + * Default: 10, units: jiffy + */ +const_debug unsigned long sysctl_sched_expel_update_interval = 10; #endif #endif @@ -671,6 +681,24 @@ static inline void update_rq_on_expel(struct rq *rq) rq->on_expel = ret; } +/* + * This funciton is called in pick_next_task_fair() if ID_LOOSE_EXPEL is + * on to improve the performence. But there is a risk to do so. + * Consider a scenario that if this cpu is executing pick_next_task_fair() + * and skip update_rq_on_expel(), a reschedule IPI is arriving but local + * irq is disabled, so an underclass task is picked to run on this cpu until + * reschedule, which will degrade the performence of highclass task. + */ +static inline void loose_update_rq_on_expel(struct rq *rq) +{ + if (rq->nr_under_running && + time_after(jiffies, rq->next_expel_update)) { + update_rq_on_expel(rq); + rq->next_expel_update = jiffies + + msecs_to_jiffies(sysctl_sched_expel_update_interval); + } +} + static inline bool rq_on_expel(struct rq *rq) { return rq->on_expel; @@ -750,6 +778,10 @@ static inline void update_rq_on_expel(struct rq *rq) { } +static inline void loose_update_rq_on_expel(struct rq *rq) +{ +} + static inline bool rq_on_expel(struct rq *rq) { return false; @@ -7948,7 +7980,11 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ if (!cfs_rq->nr_running) goto idle; - update_rq_on_expel(rq); + if (sched_feat(ID_LOOSE_EXPEL)) + loose_update_rq_on_expel(rq); + else + update_rq_on_expel(rq); + if (expellee_only(rq)) { /* * In order to mark CPU as IDLE, we need to call diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 92db87a..47a2d91 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -95,4 +95,5 @@ SCHED_FEAT(ID_IDLE_AVG, true) SCHED_FEAT(ID_RESCUE_EXPELLEE, true) SCHED_FEAT(ID_EXPELLEE_NEVER_HOT, false) +SCHED_FEAT(ID_LOOSE_EXPEL, false) #endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 621d433..3fcd95f 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -873,6 +873,7 @@ struct rq { u64 avg_id_idle; #ifdef CONFIG_SCHED_SMT unsigned long next_expel_ib; + unsigned long next_expel_update; #endif #endif diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 2e00f7c..455df53 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -536,6 +536,13 @@ static int sysrq_sysctl_handler(struct ctl_table *table, int write, .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "sched_expel_update_interval", + .data = &sysctl_sched_expel_update_interval, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_dointvec, + }, #endif #endif #ifdef CONFIG_PROVE_LOCKING -- 1.8.3.1