From bcc726c56a0f68069023953d1501ac56f54ddeb8 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 5 Aug 2020 15:47:49 +0800 Subject: [PATCH 2692/2944] alinux: sched: introduce 'idle seeker' and ID_IDLE_AVG to #30665478 The SIS_PROP will limit the search range of idle CPU which lead into the unstable latency of highclass tasks. Thus we introduced the IDLE_SEEKER identity which on default activated for group with bvt 2 or 1, now they will ignore the limitation caused by SIS_PROP, and get more chances to locate an real idle CPU in select_idle_cpu(). The identity could be deactive by turn off the bit 4 in 'cpu.identity' Currently we are using avg_idle as a knob to make sure underclass won't grab too much idle CPU, however, this is not very fair since other tasks may not even running. Thus we introduce the new sched feature ID_IDLE_AVG, the new knob avg_id_idle will stand for the average period of idle + underclass execution, by replacing this with avg_idle, underclass will save idle CPU only when other tasks are rapidly using it. Also the sched_debug entry will print the execution sum time of highclass and underclass now. Signed-off-by: Michael Wang Signed-off-by: Cruz Zhao Acked-by: Michael Wang --- kernel/sched/core.c | 2 ++ kernel/sched/debug.c | 5 +++ kernel/sched/fair.c | 91 ++++++++++++++++++++++++++++++++++++++++++++----- kernel/sched/features.h | 4 +++ kernel/sched/sched.h | 6 ++++ 5 files changed, 99 insertions(+), 9 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ed94c37..eb59fba 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1697,6 +1697,8 @@ static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags, rq->avg_idle = max; rq->idle_stamp = 0; + + update_id_idle_avg(rq, delta); } #endif } diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index f375a4a..c8eb30c 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -662,6 +662,8 @@ static void print_cpu(struct seq_file *m, int cpu) P(smt_expeller); P(on_expel); #endif + PN(high_exec_sum); + PN(under_exec_sum); #endif P(nr_switches); P(nr_load_updates); @@ -675,6 +677,9 @@ static void print_cpu(struct seq_file *m, int cpu) #ifdef CONFIG_SMP #define P64(n) SEQ_printf(m, " .%-30s: %Ld\n", #n, rq->n); +#ifdef CONFIG_GROUP_IDENTITY + P64(avg_id_idle); +#endif P64(avg_idle); P64(max_idle_balance_cost); #undef P64 diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 62aa5ff..32ebfd6 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -570,6 +570,7 @@ enum { #define ID_HIGHCLASS 0x0002 #define ID_SMT_EXPELLER 0x0004 #define ID_IDLE_SAVER 0x0008 +#define ID_IDLE_SEEKER 0x0010 #define IDENTITY_FLAGS_MASK 0x00ff static DEFINE_MUTEX(identity_mutex); @@ -614,6 +615,11 @@ static inline bool is_idle_saver(struct sched_entity *se) return test_identity(se, ID_IDLE_SAVER); } +static inline bool is_idle_seeker(struct sched_entity *se) +{ + return test_identity(se, ID_IDLE_SEEKER); +} + static inline bool underclass_only(int cpu) { struct rq *rq = cpu_rq(cpu); @@ -809,6 +815,17 @@ static inline bool is_idle_saver_task(struct task_struct *p) return ret; } +static inline bool is_idle_seeker_task(struct task_struct *p) +{ + bool ret; + + rcu_read_lock(); + ret = p->se.parent ? is_idle_seeker(p->se.parent) : false; + rcu_read_unlock(); + + return ret; +} + static noinline bool id_can_migrate_task(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) { @@ -851,6 +868,7 @@ static inline bool is_idle_saver_task(struct task_struct *p) bool need_expel = rq_on_expel(rq) && expellee; bool is_saver = is_idle_saver_task(p); bool is_idle = available_idle_cpu(cpu); + u64 avg_idle = rq->avg_idle; if (idle) *idle = is_idle; @@ -865,8 +883,11 @@ static inline bool is_idle_saver_task(struct task_struct *p) if (!is_saver) return true; + if (sched_feat(ID_IDLE_AVG)) + avg_idle = rq->avg_id_idle; + /* Save idle CPU for others unless above watermark */ - return rq->avg_idle >= sysctl_sched_idle_saver_wmark; + return avg_idle >= sysctl_sched_idle_saver_wmark; } static __always_inline void @@ -1022,10 +1043,10 @@ int update_bvt_warp_ns(struct task_group *tg, s64 val) case TYPE_NORMAL: break; case TYPE_LS: - flags = ID_HIGHCLASS; + flags = ID_HIGHCLASS | ID_IDLE_SEEKER; break; case TYPE_STRICT: - flags = ID_HIGHCLASS | ID_SMT_EXPELLER; + flags = ID_HIGHCLASS | ID_IDLE_SEEKER | ID_SMT_EXPELLER; break; default: return -ERANGE; @@ -1393,6 +1414,36 @@ static inline void update_under_min_vruntime(struct cfs_rq *cfs_rq) sync_min_vruntime(cfs_rq); } +/* + * Here we maintain the knob for idle saver, which is the + * average of idle + underclass execution time. + * + * Thus more the underclass running, higher the knob, more + * chance for underclass to get the idle CPU. + */ +void update_id_idle_avg(struct rq *rq, u64 delta) +{ + s64 diff; + u64 max = sysctl_sched_idle_saver_wmark; + + delta += rq->under_exec_sum - rq->under_exec_stamp; + diff = delta - rq->avg_id_idle; + rq->avg_id_idle += diff >> 3; + + if (rq->avg_id_idle > max) + rq->avg_id_idle = max; + + rq->under_exec_stamp = rq->under_exec_sum; +} + +static inline void id_update_exec(struct rq *rq, u64 delta_exec) +{ + if (is_underclass_task(rq->curr)) + rq->under_exec_sum += delta_exec; + else if (is_highclass_task(rq->curr)) + rq->high_exec_sum += delta_exec; +} + #ifdef CONFIG_SCHED_SMT void notify_smt_expeller(struct rq *rq, struct task_struct *p) { @@ -1491,6 +1542,11 @@ static inline bool is_highclass_task(struct task_struct *p) return false; } +static inline bool is_idle_seeker_task(struct task_struct *p) +{ + return false; +} + static inline bool is_expellee_task(struct task_struct *p) { return false; @@ -1619,6 +1675,9 @@ static inline void update_under_min_vruntime(struct cfs_rq *cfs_rq) { } +static inline void id_update_exec(struct rq *rq, u64 delta_exec) +{ +} #endif static void update_min_vruntime(struct cfs_rq *cfs_rq) @@ -1998,6 +2057,7 @@ static void update_curr(struct cfs_rq *cfs_rq) trace_sched_stat_runtime(curtask, delta_exec, curr->vruntime); cgroup_account_cputime(curtask, delta_exec); account_group_exec_runtime(curtask, delta_exec); + id_update_exec(rq_of(cfs_rq), delta_exec); cpu_stress_update(rq_of(cfs_rq), delta_exec); } @@ -7184,8 +7244,8 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t u64 avg_cost, avg_idle; u64 time, cost; s64 delta; - int cpu, nr = INT_MAX; - bool is_expellee; + int cpu, nr = INT_MAX, id_backup = -1; + bool is_seeker, is_expellee; this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); if (!this_sd) @@ -7201,7 +7261,8 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t if (sched_feat(SIS_AVG_CPU) && avg_idle < avg_cost) return -1; - if (sched_feat(SIS_PROP)) { + is_seeker = is_idle_seeker_task(p); + if (!is_seeker && sched_feat(SIS_PROP)) { u64 span_avg = sd->span_weight * avg_idle; if (span_avg > 4*avg_cost) nr = div_u64(span_avg, avg_cost); @@ -7213,12 +7274,24 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t is_expellee = is_expellee_task(p); for_each_cpu_wrap(cpu, sched_domain_span(sd), target) { + bool idle; + if (!--nr) return -1; if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) continue; - if (id_idle_cpu(p, cpu, is_expellee, NULL)) - break; + + /* + * Here is the best opportunity to locate a real + * idle CPU for seeker, so consider id idle cpu as + * a backup option, which will be pick only when + * failed to locate a real idle one. + */ + if (id_idle_cpu(p, cpu, is_expellee, &idle)) { + if (idle || !is_seeker) + break; + id_backup = cpu; + } } time = local_clock() - time; @@ -7226,7 +7299,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t delta = (s64)(time - cost) / 8; this_sd->avg_scan_cost += delta; - return cpu; + return (unsigned int)cpu < nr_cpumask_bits ? cpu : id_backup; } /* diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 3f9d64d..1302763 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -90,3 +90,7 @@ * UtilEstimation. Use estimated CPU utilization. */ SCHED_FEAT(UTIL_EST, true) + +#ifdef CONFIG_GROUP_IDENTITY +SCHED_FEAT(ID_IDLE_AVG, true) +#endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 8736455..c96c3e0 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -498,10 +498,12 @@ static inline void set_task_rq_fair(struct sched_entity *se, extern void notify_smt_expeller(struct rq *rq, struct task_struct *p); extern void handle_smt_expeller(void); extern unsigned int id_nr_invalid(struct rq *rq); +extern void update_id_idle_avg(struct rq *rq, u64 delta); #else static inline void notify_smt_expeller(struct rq *rq, struct task_struct *p) {} static inline void handle_smt_expeller(void) {} static inline unsigned int id_nr_invalid(struct rq *rq) { return 0; } +static inline void update_id_idle_avg(struct rq *rq, u64 delta) {} #endif /* CFS-related fields in a runqueue */ @@ -864,6 +866,10 @@ struct rq { bool smt_expeller; bool smt_expellee; bool on_expel; + u64 high_exec_sum; + u64 under_exec_sum; + u64 under_exec_stamp; + u64 avg_id_idle; #ifdef CONFIG_SCHED_SMT unsigned long next_expel_ib; #endif -- 1.8.3.1