From 84c08b2e134a35fec6f484da519effdb8e2a32c3 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 5 Aug 2020 14:23:30 +0800 Subject: [PATCH 2691/2944] alinux: sched: introduce group identity 'idle saver' to #30665478 We found that underclass tasks can migrate frequently and occupy too much idle CPU, this cause a high 99th latency on highclass since the latency to wakeup on idle CPU are much lower than to preempt the underclass. Thus we introduce the IDLE_SAVER identity which on default activated for group with bvt -1, now when they want to wakeup on idle CPU, it require the average idle time to be above 'sysctl_sched_idle_saver_wmark' (default 1ms). For the underclass who want to occupy more idle CPU, eg. the vcpu threads, just deactive the identity by turn off the bit 3 in 'cpu.identity'. Also improved the logical in select_idle_core, now consider a core full of underclass as backup option for highcalss, and prevent underclass to take the idle core with expel or idle saver reason. Signed-off-by: Michael Wang Signed-off-by: Cruz Zhao Acked-by: Michael Wang --- include/linux/sched/sysctl.h | 1 + kernel/sched/fair.c | 92 +++++++++++++++++++++++++++++++++++--------- kernel/sysctl.c | 7 ++++ 3 files changed, 82 insertions(+), 18 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index d3b3a8b..7748250 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -64,6 +64,7 @@ int sched_proc_update_handler(struct ctl_table *table, int write, #ifdef CONFIG_GROUP_IDENTITY extern unsigned int sysctl_sched_bvt_place_epsilon; +extern unsigned int sysctl_sched_idle_saver_wmark; #ifdef CONFIG_SCHED_SMT extern int sysctl_sched_expel_idle_balance_delay; #endif diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 53a1f08..62aa5ff 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -119,6 +119,14 @@ int __weak arch_asym_cpu_priority(int cpu) */ unsigned int sysctl_sched_bvt_place_epsilon = 1000000UL; +/* + * The idle saver tasks will try to save idle CPU for others + * unless the average idle of CPU above the watermark. + * + * Default: 0 msec, units: nanoseconds + */ +unsigned int sysctl_sched_idle_saver_wmark; + #ifdef CONFIG_SCHED_SMT /* * When CPU is full of expellee and on expel, it actually can @@ -561,6 +569,7 @@ enum { #define ID_UNDERCLASS 0x0001 #define ID_HIGHCLASS 0x0002 #define ID_SMT_EXPELLER 0x0004 +#define ID_IDLE_SAVER 0x0008 #define IDENTITY_FLAGS_MASK 0x00ff static DEFINE_MUTEX(identity_mutex); @@ -600,6 +609,11 @@ static inline bool is_expeller(struct sched_entity *se) return test_identity(se, ID_SMT_EXPELLER); } +static inline bool is_idle_saver(struct sched_entity *se) +{ + return test_identity(se, ID_IDLE_SAVER); +} + static inline bool underclass_only(int cpu) { struct rq *rq = cpu_rq(cpu); @@ -784,6 +798,17 @@ static inline bool is_underclass_task(struct task_struct *p) return ret; } +static inline bool is_idle_saver_task(struct task_struct *p) +{ + bool ret; + + rcu_read_lock(); + ret = p->se.parent ? is_idle_saver(p->se.parent) : false; + rcu_read_unlock(); + + return ret; +} + static noinline bool id_can_migrate_task(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) { @@ -820,19 +845,28 @@ static inline bool is_underclass_task(struct task_struct *p) } static noinline bool -id_idle_cpu(struct task_struct *p, int cpu, bool expellee) +id_idle_cpu(struct task_struct *p, int cpu, bool expellee, bool *idle) { struct rq *rq = cpu_rq(cpu); bool need_expel = rq_on_expel(rq) && expellee; + bool is_saver = is_idle_saver_task(p); bool is_idle = available_idle_cpu(cpu); + if (idle) + *idle = is_idle; + if (need_expel) return false; - if (is_idle) - return true; /* CPU full of underclass is idle for highclass */ - return is_highclass_task(p) && underclass_only(cpu); + if (!is_idle) + return is_highclass_task(p) && underclass_only(cpu); + + if (!is_saver) + return true; + + /* Save idle CPU for others unless above watermark */ + return rq->avg_idle >= sysctl_sched_idle_saver_wmark; } static __always_inline void @@ -980,10 +1014,10 @@ int update_bvt_warp_ns(struct task_group *tg, s64 val) switch (val) { case TYPE_IDLE: - flags = ID_UNDERCLASS; + flags = ID_UNDERCLASS | ID_IDLE_SAVER; break; case TYPE_BATCH: - flags = ID_UNDERCLASS; + flags = ID_UNDERCLASS | ID_IDLE_SAVER; break; case TYPE_NORMAL: break; @@ -1499,9 +1533,13 @@ static inline bool expel_ib_disallow(struct rq *rq) } static inline bool -id_idle_cpu(struct task_struct *p, int cpu, bool expellee) +id_idle_cpu(struct task_struct *p, int cpu, bool expellee, bool *idle) { bool is_idle = available_idle_cpu(cpu); + + if (idle) + *idle = is_idle; + return is_idle; } @@ -7047,8 +7085,8 @@ void __update_idle_core(struct rq *rq) static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) { struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); - int core, cpu; - bool is_expellee; + int core, cpu, id_backup = -1; + bool is_expellee, do_clear = true; if (!static_branch_likely(&sched_smt_present)) return -1; @@ -7061,24 +7099,42 @@ static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int is_expellee = is_expellee_task(p); for_each_cpu_wrap(core, cpus, target) { bool idle = true; + bool id_idle = true; for_each_cpu(cpu, cpu_smt_mask(core)) { + bool is_idle = true; cpumask_clear_cpu(cpu, cpus); - if (!id_idle_cpu(p, cpu, is_expellee)) + if (!id_idle_cpu(p, cpu, is_expellee, &is_idle)) + id_idle = false; + + if (!is_idle) idle = false; } - if (idle) + if (idle && id_idle) return core; + + if (id_idle) + id_backup = core; + + /* + * This only happens when a CPU is idle but + * not suitable for underclass task, we + * should not clear the idle info since this + * is still a good idle core for others. + */ + if (idle) + do_clear = false; } /* * Failed to find an idle core; stop looking for one. */ - set_idle_cores(target, 0); + if (do_clear) + set_idle_cores(target, 0); - return -1; + return id_backup; } /* @@ -7096,7 +7152,7 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t for_each_cpu(cpu, cpu_smt_mask(target)) { if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) continue; - if (id_idle_cpu(p, cpu, is_expellee)) + if (id_idle_cpu(p, cpu, is_expellee, NULL)) return cpu; } @@ -7161,7 +7217,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t return -1; if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) continue; - if (id_idle_cpu(p, cpu, is_expellee)) + if (id_idle_cpu(p, cpu, is_expellee, NULL)) break; } @@ -7182,7 +7238,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) int i, recent_used_cpu; bool is_expellee = is_expellee_task(p); - if (id_idle_cpu(p, target, is_expellee)) + if (id_idle_cpu(p, target, is_expellee, NULL)) return target; /* @@ -7190,7 +7246,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) */ if (prev != target && cpus_share_cache(prev, target) && - id_idle_cpu(p, prev, is_expellee)) + id_idle_cpu(p, prev, is_expellee, NULL)) return prev; /* Check a recently used CPU as a potential idle candidate: */ @@ -7198,7 +7254,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (recent_used_cpu != prev && recent_used_cpu != target && cpus_share_cache(recent_used_cpu, target) && - id_idle_cpu(p, recent_used_cpu, is_expellee) && + id_idle_cpu(p, recent_used_cpu, is_expellee, NULL) && cpumask_test_cpu(p->recent_used_cpu, &p->cpus_allowed)) { /* * Replace recent_used_cpu with prev as it is a potential diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 282a625..2e00f7c 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -529,6 +529,13 @@ static int sysrq_sysctl_handler(struct ctl_table *table, int write, .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "sched_idle_saver_wmark", + .data = &sysctl_sched_idle_saver_wmark, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, #endif #endif #ifdef CONFIG_PROVE_LOCKING -- 1.8.3.1