From a4f07eb17f1319121f7d350eeb0133c58dcbd961 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 26 Aug 2020 16:56:24 +0800 Subject: [PATCH 2693/2944] alinux:sched: rescue the expellee on migration to #30665478 In case if the CPU is on expel, even the expellee dominated the CPU, it still can't run. In this case if the expel take long time, migrate the expellee to CPU not on expel could be helpful. While if the expel will finish soon, this become harmful, however since the testing show good benefit and it balance load anyway, let's just doit. Could be switch off by: echo NO_ID_RESCUE_EXPELLEE > /sys/kernel/debug/sched_features Another switch is ID_EXPELLEE_NEVER_HOT which is on default off, this skip task_hot check for expellee on migration check, we consider that as a risk but it do help some cases, so let's reserve an option. Also make CPU on expel the backup source for load balance when the destination CPU is not on expel, now we could disable sched_expel_idle_balance_delay since after expellee migrated, the CPU will be balanced anyway. Signed-off-by: Michael Wang Signed-off-by: Cruz Zhao Acked-by: Michael Wang --- kernel/sched/fair.c | 93 ++++++++++++++++++++++++++++++++++++++++--------- kernel/sched/features.h | 2 ++ 2 files changed, 78 insertions(+), 17 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 32ebfd6..d1038bb 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -138,9 +138,9 @@ int __weak arch_asym_cpu_priority(int cpu) * This knob help to control the minimum interval of expel idle * balance, -1 means disallow. * - * Default: 5 msec, units: ms + * Default: -1, units: ms */ -int sysctl_sched_expel_idle_balance_delay = 5; +int sysctl_sched_expel_idle_balance_delay = -1; #endif #endif @@ -661,7 +661,14 @@ static inline bool need_expel(int this_cpu) static inline void update_rq_on_expel(struct rq *rq) { - rq->on_expel = need_expel(rq->cpu); + bool ret = need_expel(rq->cpu); + + /* + * Write 'on_expel' as less as possible since + * it's really hot. + */ + if (ret != rq->on_expel) + rq->on_expel = ret; } static inline bool rq_on_expel(struct rq *rq) @@ -720,6 +727,14 @@ static inline bool is_expellee_task(struct task_struct *p) { return !__is_expel_immune(&p->se, true); } + +static inline unsigned long expel_score(struct rq *rq) +{ + if (!sched_feat(ID_RESCUE_EXPELLEE)) + return 0; + + return rq->cfs.nr_running - rq->nr_expel_immune; +} #else static inline bool expellee_only(struct rq *rq) { @@ -759,6 +774,11 @@ static inline bool is_expellee_task(struct task_struct *p) { return false; } + +static inline unsigned long expel_score(struct rq *rq) +{ + return 0; +} #endif static inline bool is_highclass_task(struct task_struct *p) @@ -826,22 +846,38 @@ static inline bool is_idle_seeker_task(struct task_struct *p) return ret; } -static noinline bool +static noinline int id_can_migrate_task(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) { /* Do not migrate the last highclass, try someone else */ if (is_highclass_task(p) && src_rq->nr_high_running < 2) goto bad_dst; + if (!is_expellee_task(p)) + return -1; + /* Do not migrate expellee task to CPU on expel */ - if (is_expellee_task(p) && rq_on_expel(dst_rq)) + if (rq_on_expel(dst_rq)) goto bad_dst; - return true; + if (sched_feat(ID_EXPELLEE_NEVER_HOT)) + goto good_dst; + + /* Pass if both are not on expel */ + if (!rq_on_expel(src_rq)) + return -1; + + if (!sched_feat(ID_RESCUE_EXPELLEE)) + return -1; + +good_dst: + /* Now take the rescue chance */ + schedstat_inc(p->se.statistics.nr_forced_migrations); + return 1; bad_dst: schedstat_inc(p->se.statistics.nr_failed_migrations_id); - return false; + return 0; } static noinline bool @@ -1576,10 +1612,15 @@ static inline bool expel_ib_disallow(struct rq *rq) return false; } -static bool +static inline unsigned long expel_score(struct rq *rq) +{ + return 0; +} + +static int id_can_migrate_task(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) { - return true; + return -1; } static inline bool @@ -8320,6 +8361,7 @@ static inline int migrate_degrades_locality(struct task_struct *p, static int can_migrate_task(struct task_struct *p, struct lb_env *env) { + int ret; int tsk_cache_hot; lockdep_assert_held(&env->src_rq->lock); @@ -8372,8 +8414,9 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) return 0; } - if (!id_can_migrate_task(p, env->src_rq, env->dst_rq)) - return 0; + ret = id_can_migrate_task(p, env->src_rq, env->dst_rq); + if (ret != -1) + return ret; /* * Aggressive migration if: @@ -8466,8 +8509,10 @@ static int detach_tasks(struct lb_env *env) * We don't want to steal all, otherwise we may be treated likewise, * which could at worst lead to a livelock crash. */ - if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) - break; + if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1) { + if (!expellee_only(env->src_rq)) + break; + } p = list_last_entry(tasks, struct task_struct, se.group_node); @@ -9714,11 +9759,13 @@ static struct rq *find_busiest_queue(struct lb_env *env, struct sched_group *group) { struct rq *busiest = NULL, *rq; + struct rq *backup = NULL; + unsigned long max_backup_score = 0; unsigned long busiest_load = 0, busiest_capacity = 1; int i; for_each_cpu_and(i, sched_group_span(group), env->cpus) { - unsigned long capacity, wl; + unsigned long capacity, wl, backup_score; enum fbq_type rt; rq = cpu_rq(i); @@ -9751,6 +9798,18 @@ static struct rq *find_busiest_queue(struct lb_env *env, wl = weighted_cpuload(rq); /* + * A CPU on expel is a good source for CPU not on, + * in case if we can't find candidate by compare + * load, find the backup by compare expel score. + */ + backup_score = expel_score(rq); + if (backup_score > max_backup_score && + rq_on_expel(rq) && !rq_on_expel(env->dst_rq)) { + backup = rq; + max_backup_score = backup_score; + } + + /* * When comparing with imbalance, use weighted_cpuload() * which is not scaled with the CPU capacity. */ @@ -9777,7 +9836,7 @@ static struct rq *find_busiest_queue(struct lb_env *env, } } - return busiest; + return busiest ? busiest : backup; } /* @@ -9915,7 +9974,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, env.src_rq = busiest; ld_moved = 0; - if (busiest->nr_running > 1) { + if (busiest->nr_running > 1 || expellee_only(busiest)) { /* * Attempt to move tasks. If find_busiest_group has found * an imbalance but busiest->nr_running <= 1, the group is @@ -10753,7 +10812,7 @@ static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK)) return false; - if (idle != CPU_IDLE) { + if (idle != CPU_IDLE && !expellee_only(this_rq)) { atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu)); return false; } diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 1302763..92db87a 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -93,4 +93,6 @@ #ifdef CONFIG_GROUP_IDENTITY SCHED_FEAT(ID_IDLE_AVG, true) +SCHED_FEAT(ID_RESCUE_EXPELLEE, true) +SCHED_FEAT(ID_EXPELLEE_NEVER_HOT, false) #endif -- 1.8.3.1