From 5526cecbda006f96db013ab550bbbe6c5a7a1b5f Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Wed, 5 Aug 2020 13:11:21 +0800 Subject: [PATCH 2690/2944] alinux: sched: introduce group identity 'smt expeller' to #30665478 Concurrent execution on smt cpus could impact each other on performance since they sharing the hardware unit, an underclass task should not cause regression to the highclass task running on smt peers in this way. The legacy feature Noise Clean was supposed to address this issue, while it introduced too much overhead on throttle/unthrottle tasks which causing unstable latency in real world. As now we have all underclass tasks on separate tree, just hidden such tasks from pick_next_task() would be enough. Thus we introduced another identity SMT_EXPELLER, which will try to expel the underclass on smt peers when running. By apply the corresponding bit into 'group_identity', tasks willbecome expeller, when on executing, it's smt peers won't be able to pick the underclass tasks for running. Legacy 'bvt_warp_ns' 2 will have this identity on default. To support the new cgroup deployment, eg k8s, it requires the highclass parent to have both highclass and underclass child. In order to support expel in this case, we need to skip the highclass se, when and only when it's full of underclass tasks. We achieved this by isolate them from the rb-tree temporarily, until the end of expel, or it's no longer only contain underclass tasks, like throttle but cost far more less. Signed-off-by: Michael Wang Signed-off-by: Cruz Zhao Acked-by: Michael Wang --- include/linux/sched.h | 3 + include/linux/sched/sysctl.h | 3 + init/Kconfig | 2 + kernel/sched/core.c | 7 + kernel/sched/debug.c | 15 +- kernel/sched/fair.c | 724 +++++++++++++++++++++++++++++++++++++++++-- kernel/sched/loadavg.c | 2 + kernel/sched/sched.h | 22 +- kernel/sysctl.c | 9 + 9 files changed, 756 insertions(+), 31 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 1361fbb..b7f1a2e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -494,6 +494,9 @@ struct sched_entity { #ifdef CONFIG_GROUP_IDENTITY int id_flags; +#ifdef CONFIG_SCHED_SMT + struct list_head expel_node; +#endif #endif #ifdef CONFIG_SMP diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index bdb1b35..d3b3a8b 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -64,6 +64,9 @@ int sched_proc_update_handler(struct ctl_table *table, int write, #ifdef CONFIG_GROUP_IDENTITY extern unsigned int sysctl_sched_bvt_place_epsilon; +#ifdef CONFIG_SCHED_SMT +extern int sysctl_sched_expel_idle_balance_delay; +#endif #endif #ifdef CONFIG_SCHED_AUTOGROUP diff --git a/init/Kconfig b/init/Kconfig index 3dd9b31..6e3a1dc 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -812,6 +812,8 @@ config GROUP_IDENTITY also consider underclass only cpu as idle. ID_UNDERCLASS Tasks take penalty on wakeup. + ID_SMT_EXPELLER + expel the underclass tasks on smt-cpus when executing These identity allow the tasks of cgroup to perform special schedule behaviour. diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d163b65..ed94c37 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1769,6 +1769,7 @@ void sched_ttwu_pending(void) void scheduler_ipi(void) { + handle_smt_expeller(); /* * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting * TIF_NEED_RESCHED remotely (for the first time) will also send @@ -2131,6 +2132,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) #ifdef CONFIG_GROUP_IDENTITY p->se.id_flags = 0; +#ifdef CONFIG_SCHED_SMT + INIT_LIST_HEAD(&p->se.expel_node); +#endif #endif #ifdef CONFIG_SCHEDSTATS @@ -3452,6 +3456,9 @@ static void __sched notrace __schedule(bool preempt) } next = pick_next_task(rq, prev, &rf); + + notify_smt_expeller(rq, next); + clear_tsk_need_resched(prev); clear_preempt_need_resched(); diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index e1b4eef..f375a4a 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -503,7 +503,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) raw_spin_lock_irqsave(&rq->lock, flags); if (rb_first_cached(&cfs_rq->tasks_timeline)) - MIN_vruntime = (__pick_first_entity(cfs_rq))->vruntime; + MIN_vruntime = (debug_pick_first_entity(cfs_rq))->vruntime; last = __pick_last_entity(cfs_rq); if (last) max_vruntime = last->vruntime; @@ -526,6 +526,14 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->nr_spread_over); SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running); #ifdef CONFIG_GROUP_IDENTITY +#ifdef CONFIG_SCHED_SMT + SEQ_printf(m, " .%-30s: %d\n", "h_nr_expel_immune", + cfs_rq->h_nr_expel_immune); + SEQ_printf(m, " .%-30s: %lld.%06lu\n", "expel_start", + SPLIT_NS(cfs_rq->expel_start)); + SEQ_printf(m, " .%-30s: %lld.%06lu\n", "expel_spread", + SPLIT_NS(cfs_rq->expel_spread)); +#endif SEQ_printf(m, " .%-30s: %lld.%06lu\n", "min_under_vruntime", SPLIT_NS(cfs_rq->min_under_vruntime)); #endif @@ -649,6 +657,11 @@ static void print_cpu(struct seq_file *m, int cpu) #ifdef CONFIG_GROUP_IDENTITY P(nr_high_running); P(nr_under_running); +#ifdef CONFIG_SCHED_SMT + P(nr_expel_immune); + P(smt_expeller); + P(on_expel); +#endif #endif P(nr_switches); P(nr_load_updates); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ec4d9e8..53a1f08 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -118,6 +118,22 @@ int __weak arch_asym_cpu_priority(int cpu) * Default: 1 msec in the future, units: nanoseconds */ unsigned int sysctl_sched_bvt_place_epsilon = 1000000UL; + +#ifdef CONFIG_SCHED_SMT +/* + * When CPU is full of expellee and on expel, it actually can + * execute nothing, just like an idle CPU. + * + * Thus do idle balance in such cases may improve load balance, + * but will cause more damage to the expellee. + * + * This knob help to control the minimum interval of expel idle + * balance, -1 means disallow. + * + * Default: 5 msec, units: ms + */ +int sysctl_sched_expel_idle_balance_delay = 5; +#endif #endif /* @@ -544,6 +560,7 @@ enum { #define ID_NORMAL 0x0000 #define ID_UNDERCLASS 0x0001 #define ID_HIGHCLASS 0x0002 +#define ID_SMT_EXPELLER 0x0004 #define IDENTITY_FLAGS_MASK 0x00ff static DEFINE_MUTEX(identity_mutex); @@ -578,6 +595,11 @@ static inline bool is_highclass(struct sched_entity *se) return test_identity(se, ID_HIGHCLASS); } +static inline bool is_expeller(struct sched_entity *se) +{ + return test_identity(se, ID_SMT_EXPELLER); +} + static inline bool underclass_only(int cpu) { struct rq *rq = cpu_rq(cpu); @@ -586,6 +608,139 @@ static inline bool underclass_only(int cpu) rq->cfs.h_nr_running == rq->nr_under_running; } +#ifdef CONFIG_SCHED_SMT +/* + * This helper tell us if the cpu is on expel and full of expellee tasks. + * + * Such cpu may become idle with underclass enqueued for long time, + * thus we need to trigger nohz load balance for it when it's possible. + */ +static inline bool expellee_only(struct rq *rq) +{ + return rq->on_expel && !rq->nr_expel_immune && rq->cfs.h_nr_running; +} + +static inline bool need_expel(int this_cpu) +{ + int cpu; + + /* Not yet booted up */ + if (!cpu_smt_mask(this_cpu)) + return false; + + for_each_cpu(cpu, cpu_smt_mask(this_cpu)) { + if (cpu == this_cpu) + continue; + + if (cpu_rq(cpu)->smt_expeller) + return true; + } + + return false; +} + +static inline void update_rq_on_expel(struct rq *rq) +{ + rq->on_expel = need_expel(rq->cpu); +} + +static inline bool rq_on_expel(struct rq *rq) +{ + return rq->on_expel; +} + +static inline bool expel_ib_disallow(struct rq *rq) +{ + if (sysctl_sched_expel_idle_balance_delay < 0) + return true; + + if (time_before(jiffies, rq->next_expel_ib)) + return true; + + rq->next_expel_ib = jiffies + + msecs_to_jiffies(sysctl_sched_expel_idle_balance_delay); + + return false; +} + +static inline bool expellee_se(struct sched_entity *se) +{ + return se->my_q && !se->my_q->h_nr_expel_immune; +} + +/* + * The check on wakeup should not consider the case that + * highclass group full of underclass tasks, the wakee + * may change the situation. + */ +static inline bool __is_expel_immune(struct sched_entity *se, bool wakeup) +{ + bool ret = true; + + /* To expel if hierarchy contain underclass identity */ + rcu_read_lock(); + for_each_sched_entity(se) { + if (is_underclass(se) || + (!wakeup && expellee_se(se))) { + ret = false; + break; + } + } + rcu_read_unlock(); + + return ret; +} + +static inline bool is_expel_immune(struct sched_entity *se) +{ + return __is_expel_immune(se, false); +} + +static inline bool is_expellee_task(struct task_struct *p) +{ + return !__is_expel_immune(&p->se, true); +} +#else +static inline bool expellee_only(struct rq *rq) +{ + return false; +} + +static inline bool need_expel(int this_cpu) +{ + return false; +} + +static inline void update_rq_on_expel(struct rq *rq) +{ +} + +static inline bool rq_on_expel(struct rq *rq) +{ + return false; +} + +static inline bool expel_ib_disallow(struct rq *rq) +{ + return false; +} + +static inline bool expellee_se(struct sched_entity *se) +{ + return false; +} + +static inline bool is_expel_immune(struct sched_entity *se) +{ + return true; +} + +static inline bool is_expellee_task(struct task_struct *p) +{ + return false; +} +#endif + static inline bool is_highclass_task(struct task_struct *p) { bool ret; @@ -597,6 +752,28 @@ static inline bool is_highclass_task(struct task_struct *p) return ret; } +/* + * Underclass tasks should be hidden from the rq when cpu is on + * expel, this helper return true when se need expel on rq. + */ +static inline bool should_expel_se(struct rq *rq, struct sched_entity *se) +{ + return rq_on_expel(rq) && !is_expel_immune(se); +} + +static inline bool task_is_expeller(struct task_struct *p) +{ + bool ret = false; + + /* Check the identity of task group it belonged */ + rcu_read_lock(); + if (p->se.parent) + ret = is_expeller(p->se.parent); + rcu_read_unlock(); + + return ret; +} + static inline bool is_underclass_task(struct task_struct *p) { bool ret; @@ -614,6 +791,10 @@ static inline bool is_underclass_task(struct task_struct *p) if (is_highclass_task(p) && src_rq->nr_high_running < 2) goto bad_dst; + /* Do not migrate expellee task to CPU on expel */ + if (is_expellee_task(p) && rq_on_expel(dst_rq)) + goto bad_dst; + return true; bad_dst: @@ -624,18 +805,31 @@ static inline bool is_underclass_task(struct task_struct *p) static noinline bool id_wake_affine(struct task_struct *p, int this_cpu, int prev_cpu) { + struct rq *this_rq = cpu_rq(this_cpu); struct rq *prev_rq = cpu_rq(prev_cpu); /* Last highclass should stay */ if (is_highclass_task(p) && prev_rq->nr_high_running < 1) return false; + /* Do not pull underclass to the cpu on expel */ + if (is_expellee_task(p) && rq_on_expel(this_rq)) + return false; + return true; } -static noinline bool id_idle_cpu(struct task_struct *p, int cpu) +static noinline bool +id_idle_cpu(struct task_struct *p, int cpu, bool expellee) { - if (available_idle_cpu(cpu)) + struct rq *rq = cpu_rq(cpu); + bool need_expel = rq_on_expel(rq) && expellee; + bool is_idle = available_idle_cpu(cpu); + + if (need_expel) + return false; + + if (is_idle) return true; /* CPU full of underclass is idle for highclass */ return is_highclass_task(p) && underclass_only(cpu); @@ -658,8 +852,69 @@ static noinline bool id_idle_cpu(struct task_struct *p, int cpu) rq->nr_under_running += delta; } +#ifdef CONFIG_SCHED_SMT +static inline u64 get_expel_spread(struct cfs_rq *cfs_rq) +{ + return cfs_rq->expel_spread; +} + +static inline unsigned int get_h_nr_expel_immune(struct sched_entity *se) +{ + return se->my_q->h_nr_expel_immune; +} + +static inline void +update_nr_expel_immune(struct cfs_rq *cfs_rq, struct sched_entity *se, + bool *immune, long delta) +{ + if (!(*immune)) + return; + + *immune = !is_underclass(se); + if (*immune) { + cfs_rq->h_nr_expel_immune += delta; + if (!se->parent) + cfs_rq->rq->nr_expel_immune += delta; + } +} + +static inline void +hierarchy_update_nr_expel_immune(struct sched_entity *se, long delta) +{ + bool immune = true; + + for_each_sched_entity(se) + update_nr_expel_immune(cfs_rq_of(se), se, &immune, delta); +} +#else +static inline u64 get_expel_spread(struct cfs_rq *cfs_rq) +{ + return 0; +} + +static inline unsigned int get_h_nr_expel_immune(struct sched_entity *se) +{ + return 0; +} + +static inline void +update_nr_expel_immune(struct cfs_rq *cfs_rq, struct sched_entity *se, + bool *immune, long delta) +{ +} + +static inline void +hierarchy_update_nr_expel_immune(struct sched_entity *se, long delta) +{ +} +#endif + static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se); static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se); +static void update_min_vruntime(struct cfs_rq *cfs_rq); +static void update_curr(struct cfs_rq *cfs_rq); +static inline u64 +id_min_vruntime(struct cfs_rq *cfs_rq, struct sched_entity *se); static void __update_identity(struct task_group *tg, int flags) { @@ -670,32 +925,45 @@ static void __update_identity(struct task_group *tg, int flags) for_each_online_cpu(cpu) { bool on_rq; - unsigned int delta; + unsigned int delta, ei_delta; struct cfs_rq *cfs_rq; struct sched_entity *se; struct rq *rq = cpu_rq(cpu); rq_lock_irq(rq, &rf); + update_rq_clock(rq); se = tg->se[cpu]; cfs_rq = cfs_rq_of(se); delta = se->my_q->nr_tasks; + ei_delta = get_h_nr_expel_immune(se); on_rq = se->on_rq; if (on_rq) { if (se != cfs_rq->curr) __dequeue_entity(cfs_rq, se); + hierarchy_update_nr_expel_immune(se, -ei_delta); id_update_nr_running(tg, rq, -delta); + + update_curr(cfs_rq); + se->vruntime -= id_min_vruntime(cfs_rq, se); } se->id_flags = flags; if (on_rq) { + se->vruntime += id_min_vruntime(cfs_rq, se); + if (se != cfs_rq->curr) __enqueue_entity(cfs_rq, se); + hierarchy_update_nr_expel_immune(se, ei_delta); id_update_nr_running(tg, rq, delta); + + update_min_vruntime(cfs_rq); } + notify_smt_expeller(rq, rq->curr); + rq_unlock_irq(rq, &rf); } } @@ -723,7 +991,7 @@ int update_bvt_warp_ns(struct task_group *tg, s64 val) flags = ID_HIGHCLASS; break; case TYPE_STRICT: - flags = ID_HIGHCLASS; + flags = ID_HIGHCLASS | ID_SMT_EXPELLER; break; default: return -ERANGE; @@ -765,6 +1033,9 @@ static inline void identity_init_cfs_rq(struct cfs_rq *cfs_rq) { cfs_rq->under_timeline = RB_ROOT_CACHED; cfs_rq->min_under_vruntime = (u64)(-(1LL << 20)); +#ifdef CONFIG_SCHED_SMT + INIT_LIST_HEAD(&cfs_rq->expel_list); +#endif } static inline struct rb_root_cached * @@ -776,6 +1047,102 @@ static inline void identity_init_cfs_rq(struct cfs_rq *cfs_rq) return &cfs_rq->tasks_timeline; } +#ifdef CONFIG_SCHED_SMT +static inline void +update_expel_start(struct cfs_rq *cfs_rq, struct sched_entity *se) +{ + u64 min_under_vruntime; + + if (cfs_rq->expel_start) + return; + + /* + * Consider underclass as dequeued on expel, + * 'expel_start' record the penalty which will + * be applied to 'expel_spread' later. + */ + cfs_rq->expel_start = sysctl_sched_bvt_place_epsilon; + + min_under_vruntime = cfs_rq->min_under_vruntime + cfs_rq->expel_spread; + if (min_under_vruntime <= cfs_rq->min_vruntime) + return; + + cfs_rq->expel_start += min_under_vruntime - cfs_rq->min_vruntime; +} + +static inline void update_expel_spread(struct cfs_rq *cfs_rq) +{ + if (!cfs_rq->expel_start) + return; + + /* + * Consider underclass as enqueue after expel, + * 'expel_spread' record the vruntime lost on + * expel which applied to all the undercalss + * until min vruntime sync up. + */ + cfs_rq->expel_spread = cfs_rq->min_vruntime - + cfs_rq->min_under_vruntime + + cfs_rq->expel_start; + cfs_rq->expel_start = 0; +} + +static void +place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial); + +static inline void check_expellee_se(struct cfs_rq *cfs_rq) +{ + struct sched_entity *se, *tmp; + + list_for_each_entry_safe(se, tmp, &cfs_rq->expel_list, expel_node) { + if (rq_on_expel(rq_of(cfs_rq)) && expellee_se(se)) + continue; + + list_del_init(&se->expel_node); + place_entity(cfs_rq, se, 0); + __enqueue_entity(cfs_rq, se); + } +} + +static inline struct rb_node *skip_expellee_se(struct cfs_rq *cfs_rq) +{ + struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); + + while (left) { + struct sched_entity *se = + rb_entry(left, struct sched_entity, run_node); + + if (!expellee_se(se)) + break; + + left = rb_next(&se->run_node); + + __dequeue_entity(cfs_rq, se); + list_add_tail(&se->expel_node, &cfs_rq->expel_list); + } + + return left; +} +#else +static inline void +update_expel_start(struct cfs_rq *cfs_rq, struct sched_entity *se) +{ +} + +static inline void update_expel_spread(struct cfs_rq *cfs_rq) +{ +} + +static inline void check_expellee_se(struct cfs_rq *cfs_rq) +{ +} + +static inline struct rb_node *skip_expellee_se(struct cfs_rq *cfs_rq) +{ + return rb_first_cached(&cfs_rq->tasks_timeline); +} +#endif + static inline struct rb_node * id_rb_first_cached(struct cfs_rq *cfs_rq) { @@ -787,7 +1154,45 @@ static inline void identity_init_cfs_rq(struct cfs_rq *cfs_rq) &cfs_rq->under_timeline, }; - if (cfs_rq->min_under_vruntime < cfs_rq->min_vruntime) { + /* + * Usually only tasks on under tree need expel, but there + * is one special case, non-underclass cgroup with the + * underclass descendant. + * + * In this case, the non underclass se could contain only + * expellee, we should not pick such se, otherwise we endup + * to pick expellee and expel failed. + * + * So the tricky here is to skip such se and isolate them + * from the rb-tree, this is similar to throttle but cost + * far more less. + * + * So these se will be hidden just like underclass, but we + * still need to check them on each pick since any of them + * may become non-expellee when there is a non-expellee task + * enqueued, in such case we put them back to the rb-tree so + * they are no longer hidden. + */ + check_expellee_se(cfs_rq); + if (rq_on_expel(rq_of(cfs_rq))) + return skip_expellee_se(cfs_rq); + + /* + * When cpu is on expel, underclass tasks will be hidden + * from picking, however since they are still on runqueue, + * after expel they gain vruntime bonus if the min vruntime + * are not yet sync up. + * + * We address this by recording 'expel_spread' after expel, + * which is the vruntime penalty applied to all the underclass + * until sync up. + * + * Check update_expel_xxx() for more details. + */ + update_expel_spread(cfs_rq); + + if (cfs_rq->min_under_vruntime + get_expel_spread(cfs_rq) < + cfs_rq->min_vruntime) { roots[0] = &cfs_rq->under_timeline; roots[1] = &cfs_rq->tasks_timeline; } @@ -801,6 +1206,22 @@ static inline void identity_init_cfs_rq(struct cfs_rq *cfs_rq) return NULL; } +/* + * Consider the 'expel_spread' when checking priority according + * to vruntime. + */ +static inline u64 id_vruntime(struct sched_entity *se) +{ + return is_underclass(se) ? + se->vruntime + get_expel_spread(cfs_rq_of(se)) : se->vruntime; +} + +static inline int +id_entity_before(struct sched_entity *a, struct sched_entity *b) +{ + return (s64)(id_vruntime(a) - id_vruntime(b)) < 0; +} + static inline u64 id_min_vruntime(struct cfs_rq *cfs_rq, struct sched_entity *se) { @@ -834,8 +1255,19 @@ static inline void identity_init_cfs_rq(struct cfs_rq *cfs_rq) bool under_curr = is_underclass(curr); bool under_se = is_underclass(se); - if (under_curr == under_se) + if (under_curr == under_se) { +#ifdef CONFIG_SCHED_SMT + /* Full of expellee is also underclass when on expel */ + if (rq_on_expel(rq_of(cfs_rq_of(curr)))) { + bool expel_curr = expellee_se(curr); + bool expel_se = expellee_se(se); + + if (expel_curr != expel_se) + return expel_se ? -1 : 1; + } +#endif return 0; + } return under_se ? -1 : 1; } @@ -866,17 +1298,33 @@ static inline void identity_init_cfs_rq(struct cfs_rq *cfs_rq) */ static inline void sync_min_vruntime(struct cfs_rq *cfs_rq) { + bool sync_up = false; u64 effect_vruntime = max_vruntime(cfs_rq->min_vruntime, cfs_rq->min_under_vruntime); bool no_curr = (!cfs_rq->curr || !cfs_rq->curr->on_rq); if (!rb_first_cached(&cfs_rq->tasks_timeline) && - (no_curr || is_underclass(cfs_rq->curr))) + (no_curr || is_underclass(cfs_rq->curr))) { cfs_rq->min_vruntime = effect_vruntime; + sync_up = true; + } if (!rb_first_cached(&cfs_rq->under_timeline) && - (no_curr || !is_underclass(cfs_rq->curr))) + (no_curr || !is_underclass(cfs_rq->curr))) { cfs_rq->min_under_vruntime = effect_vruntime; + sync_up = true; + } + +#ifdef CONFIG_SCHED_SMT + /* + * Once min vruntime sync up, the expel penalty will be + * included, just clear it. + */ + if (sync_up) { + cfs_rq->expel_start = 0; + cfs_rq->expel_spread = 0; + } +#endif } static inline void update_under_min_vruntime(struct cfs_rq *cfs_rq) @@ -911,6 +1359,82 @@ static inline void update_under_min_vruntime(struct cfs_rq *cfs_rq) sync_min_vruntime(cfs_rq); } +#ifdef CONFIG_SCHED_SMT +void notify_smt_expeller(struct rq *rq, struct task_struct *p) +{ + int cpu, this_cpu = rq->cpu; + bool expeller = false; + bool expellee = false; + + if (p->sched_class == &fair_sched_class) { + expeller = task_is_expeller(p); + expellee = !is_expel_immune(&p->se); + } + + rq->smt_expellee = expellee; + + if (expeller == rq->smt_expeller) + return; + + rq->smt_expeller = expeller; + + for_each_cpu(cpu, cpu_smt_mask(this_cpu)) { + if (cpu == this_cpu) + continue; + + smp_send_reschedule(cpu); + } +} + +static inline bool expel_resched(struct rq *rq) +{ + /* Nothing to do for expeller */ + if (rq->smt_expeller) + return false; + + /* Stop the expel when started */ + if (rq->curr == rq->idle) + return !rq_on_expel(rq) && rq->cfs.nr_running; + + /* Start the expel when stopped */ + return rq_on_expel(rq) && rq->smt_expellee; +} + +void handle_smt_expeller(void) +{ + struct rq *rq = this_rq(); + + update_rq_on_expel(rq); + + /* Safe since 'current' can't be changed during IPI */ + if (expel_resched(rq) && !test_tsk_need_resched(rq->curr)) { + set_tsk_need_resched(rq->curr); + set_preempt_need_resched(); + } +} + +unsigned int id_nr_invalid(struct rq *rq) +{ + if (rq_on_expel(rq)) + return rq->cfs.h_nr_running - rq->nr_expel_immune; + + return 0; +} +#else +void notify_smt_expeller(struct rq *rq, struct task_struct *p) +{ +} + +void handle_smt_expeller(void) +{ +} + +unsigned int id_nr_invalid(struct rq *rq) +{ + return 0; +} +#endif + #else static inline bool is_underclass(struct sched_entity *curr) @@ -933,6 +1457,35 @@ static inline bool is_highclass_task(struct task_struct *p) return false; } +static inline bool is_expellee_task(struct task_struct *p) +{ + return false; +} + +static inline bool should_expel_se(struct rq *rq, struct sched_entity *se) +{ + return false; +} + +static inline bool expellee_only(struct rq *rq) +{ + return false; +} + +static inline void update_rq_on_expel(struct rq *rq) +{ +} + +static inline bool rq_on_expel(struct rq *rq) +{ + return false; +} + +static inline bool expel_ib_disallow(struct rq *rq) +{ + return false; +} + static bool id_can_migrate_task(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) { @@ -945,15 +1498,22 @@ static inline bool is_highclass_task(struct task_struct *p) return true; } -static inline bool id_idle_cpu(struct task_struct *p, int cpu) +static inline bool +id_idle_cpu(struct task_struct *p, int cpu, bool expellee) { - return available_idle_cpu(cpu); + bool is_idle = available_idle_cpu(cpu); + return is_idle; } static inline void identity_init_cfs_rq(struct cfs_rq *cfs_rq) { } +static inline void +update_expel_start(struct cfs_rq *cfs_rq, struct sched_entity *se) +{ +} + static inline struct rb_root_cached * id_rb_root(struct cfs_rq *cfs_rq, struct sched_entity *se) { @@ -966,6 +1526,17 @@ static inline void identity_init_cfs_rq(struct cfs_rq *cfs_rq) return rb_first_cached(&cfs_rq->tasks_timeline); } +static inline u64 id_vruntime(struct sched_entity *se) +{ + return se->vruntime; +} + +static inline int +id_entity_before(struct sched_entity *a, struct sched_entity *b) +{ + return entity_before(a, b); +} + static inline u64 id_min_vruntime(struct cfs_rq *cfs_rq, struct sched_entity *se) { @@ -995,6 +1566,17 @@ static inline void identity_init_cfs_rq(struct cfs_rq *cfs_rq) { } +static inline void +update_nr_expel_immune(struct cfs_rq *cfs_rq, struct sched_entity *se, + bool *immune, long delta) +{ +} + +static inline unsigned int get_h_nr_expel_immune(struct sched_entity *se) +{ + return 0; +} + static inline void update_under_min_vruntime(struct cfs_rq *cfs_rq) { } @@ -1073,9 +1655,27 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { +#if defined(CONFIG_GROUP_IDENTITY) && defined(CONFIG_SCHED_SMT) + /* Either on list or on rb-tree */ + if (!list_empty(&se->expel_node)) { + list_del_init(&se->expel_node); + return; + } +#endif rb_erase_cached(&se->run_node, id_rb_root(cfs_rq, se)); } +/* No tricy here */ +struct sched_entity *debug_pick_first_entity(struct cfs_rq *cfs_rq) +{ + struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); + + if (!left) + return NULL; + + return rb_entry(left, struct sched_entity, run_node); +} + struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) { struct rb_node *left = id_rb_first_cached(cfs_rq); @@ -4630,6 +5230,11 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) return; } + if (should_expel_se(rq_of(cfs_rq), curr)) { + resched_curr(rq_of(cfs_rq)); + return; + } + /* * Ensure that a task that missed wakeup preemption by a * narrow margin doesn't have to wait for a full slice. @@ -4638,8 +5243,12 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) if (is_highclass(curr) && delta_exec < sysctl_sched_min_granularity) return; + /* Must be on expel if no next se, and curr won't be expellee */ se = __pick_first_entity(cfs_rq); - delta = curr->vruntime - se->vruntime; + if (!se) + return; + + delta = id_vruntime(curr) - id_vruntime(se); if (delta < 0) return; @@ -4701,8 +5310,8 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) * If curr is set we have to see if its left of the leftmost entity * still in the tree, provided there was anything in the tree at all. */ - if (!left || (curr && entity_before(curr, left))) - left = curr; + if (!left || (curr && id_entity_before(curr, left))) + left = should_expel_se(rq_of(cfs_rq), curr) ? left : curr; se = left; /* ideally we run the leftmost entity */ @@ -4717,7 +5326,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) second = __pick_first_entity(cfs_rq); } else { second = __pick_next_entity(se); - if (!second || (curr && entity_before(curr, second))) + if (!second || (curr && id_entity_before(curr, second))) second = curr; } @@ -4739,6 +5348,9 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) clear_buddies(cfs_rq, se); + if (rq_on_expel(rq_of(cfs_rq))) + update_expel_start(cfs_rq, se); + return se; } @@ -5057,10 +5669,11 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) struct rq *rq = rq_of(cfs_rq); struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); struct sched_entity *se; - long task_delta, dequeue = 1; - bool empty; + long task_delta, dequeue = 1, ei_delta; + bool empty, immune = true; se = tg->se[cpu_of(rq_of(cfs_rq))]; + ei_delta = get_h_nr_expel_immune(se); /* freeze hierarchy runnable averages while throttled */ rcu_read_lock(); @@ -5079,6 +5692,7 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) dequeue_entity(qcfs_rq, se, DEQUEUE_SLEEP); } qcfs_rq->h_nr_running -= task_delta; + update_nr_expel_immune(qcfs_rq, se, &immune, -ei_delta); if (qcfs_rq->load.weight) dequeue = 0; @@ -5116,15 +5730,17 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) { + bool immune = true; struct task_group *tg = cfs_rq->tg; struct cfs_rq *bottom_cfs_rq = cfs_rq; struct rq *rq = rq_of(cfs_rq); struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); struct sched_entity *se; int enqueue = 1; - long task_delta; + long task_delta, ei_delta; se = tg->se[cpu_of(rq)]; + ei_delta = get_h_nr_expel_immune(se); cfs_rq->throttled = 0; @@ -5153,6 +5769,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) enqueue_entity(cfs_rq, se, ENQUEUE_WAKEUP); } cfs_rq->h_nr_running += task_delta; + update_nr_expel_immune(cfs_rq, se, &immune, ei_delta); if (cfs_rq_throttled(cfs_rq)) break; @@ -5754,6 +6371,7 @@ static inline void hrtick_update(struct rq *rq) static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) { + bool immune = true; struct cfs_rq *cfs_rq; struct sched_entity *se = &p->se; @@ -5796,6 +6414,7 @@ static inline void hrtick_update(struct rq *rq) break; } cfs_rq->h_nr_running++; + update_nr_expel_immune(cfs_rq, se, &immune, 1); flags = ENQUEUE_WAKEUP; } @@ -5803,6 +6422,7 @@ static inline void hrtick_update(struct rq *rq) for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running++; + update_nr_expel_immune(cfs_rq, se, &immune, 1); if (cfs_rq_throttled(cfs_rq)) break; @@ -5845,6 +6465,7 @@ static inline void hrtick_update(struct rq *rq) */ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) { + bool immune = true; struct cfs_rq *cfs_rq; struct sched_entity *se = &p->se; int task_sleep = flags & DEQUEUE_SLEEP; @@ -5870,6 +6491,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) break; } cfs_rq->h_nr_running--; + update_nr_expel_immune(cfs_rq, se, &immune, -1); /* Don't dequeue parent if it has other entities besides us */ if (cfs_rq->load.weight) { @@ -5889,6 +6511,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); cfs_rq->h_nr_running--; + update_nr_expel_immune(cfs_rq, se, &immune, -1); if (cfs_rq_throttled(cfs_rq)) break; @@ -6425,6 +7048,7 @@ static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int { struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask); int core, cpu; + bool is_expellee; if (!static_branch_likely(&sched_smt_present)) return -1; @@ -6434,12 +7058,14 @@ static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int cpumask_and(cpus, sched_domain_span(sd), &p->cpus_allowed); + is_expellee = is_expellee_task(p); for_each_cpu_wrap(core, cpus, target) { bool idle = true; for_each_cpu(cpu, cpu_smt_mask(core)) { + cpumask_clear_cpu(cpu, cpus); - if (!available_idle_cpu(cpu)) + if (!id_idle_cpu(p, cpu, is_expellee)) idle = false; } @@ -6461,14 +7087,16 @@ static int select_idle_core(struct task_struct *p, struct sched_domain *sd, int static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) { int cpu; + bool is_expellee; if (!static_branch_likely(&sched_smt_present)) return -1; + is_expellee = is_expellee_task(p); for_each_cpu(cpu, cpu_smt_mask(target)) { if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) continue; - if (id_idle_cpu(p, cpu)) + if (id_idle_cpu(p, cpu, is_expellee)) return cpu; } @@ -6501,6 +7129,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t u64 time, cost; s64 delta; int cpu, nr = INT_MAX; + bool is_expellee; this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc)); if (!this_sd) @@ -6526,12 +7155,13 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t time = local_clock(); + is_expellee = is_expellee_task(p); for_each_cpu_wrap(cpu, sched_domain_span(sd), target) { if (!--nr) return -1; if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) continue; - if (available_idle_cpu(cpu)) + if (id_idle_cpu(p, cpu, is_expellee)) break; } @@ -6550,8 +7180,9 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) { struct sched_domain *sd; int i, recent_used_cpu; + bool is_expellee = is_expellee_task(p); - if (id_idle_cpu(p, target)) + if (id_idle_cpu(p, target, is_expellee)) return target; /* @@ -6559,7 +7190,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) */ if (prev != target && cpus_share_cache(prev, target) && - id_idle_cpu(p, prev)) + id_idle_cpu(p, prev, is_expellee)) return prev; /* Check a recently used CPU as a potential idle candidate: */ @@ -6567,7 +7198,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (recent_used_cpu != prev && recent_used_cpu != target && cpus_share_cache(recent_used_cpu, target) && - id_idle_cpu(p, recent_used_cpu) && + id_idle_cpu(p, recent_used_cpu, is_expellee) && cpumask_test_cpu(p->recent_used_cpu, &p->cpus_allowed)) { /* * Replace recent_used_cpu with prev as it is a potential @@ -7089,11 +7720,27 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ struct sched_entity *se; struct task_struct *p; int new_tasks; + bool expel_pulled = false; again: if (!cfs_rq->nr_running) goto idle; + update_rq_on_expel(rq); + if (expellee_only(rq)) { + /* + * In order to mark CPU as IDLE, we need to call + * idle_balance(), while since we still have + * expellee on rq, we may pulled nothing and come + * back to here again, check to avoid deadloop. + */ + if (expel_ib_disallow(rq) || expel_pulled) + return NULL; + + expel_pulled = true; + goto idle; + } + #ifdef CONFIG_FAIR_GROUP_SCHED if (prev->sched_class != &fair_sched_class) goto simple; @@ -7133,6 +7780,14 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ if (!cfs_rq->nr_running) goto idle; + if (expellee_only(rq)) { + if (expel_ib_disallow(rq)) + return NULL; + + expel_pulled = true; + goto idle; + } + goto simple; } } @@ -9606,15 +10261,21 @@ static inline int on_null_domain(struct rq *rq) static inline int find_new_ilb(void) { - int ilb; + int ilb, ret = nr_cpu_ids; for_each_cpu_and(ilb, nohz.idle_cpus_mask, housekeeping_cpumask(HK_FLAG_MISC)) { if (idle_cpu(ilb)) return ilb; + /* + * The expellee only cpu is also some kind of idle, + * pick it if no other choice. + */ + if (expellee_only(cpu_rq(ilb))) + ret = ilb; } - return nr_cpu_ids; + return ret; } /* @@ -9687,7 +10348,7 @@ static void nohz_balancer_kick(struct rq *rq) if (time_before(now, nohz.next_balance)) goto out; - if (rq->nr_running >= 2) { + if (rq->nr_running >= 2 || expellee_only(rq)) { flags = NOHZ_KICK_MASK; goto out; } @@ -9881,7 +10542,10 @@ static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags, smp_mb(); for_each_cpu(balance_cpu, nohz.idle_cpus_mask) { - if (balance_cpu == this_cpu || !idle_cpu(balance_cpu)) + rq = cpu_rq(balance_cpu); + + if (balance_cpu == this_cpu || + (!idle_cpu(balance_cpu) && !expellee_only(rq))) continue; /* @@ -9894,8 +10558,6 @@ static bool _nohz_idle_balance(struct rq *this_rq, unsigned int flags, goto abort; } - rq = cpu_rq(balance_cpu); - has_blocked_load |= update_nohz_stats(rq, true); /* @@ -10723,6 +11385,10 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, #endif } +#if defined(CONFIG_GROUP_IDENTITY) && defined(CONFIG_SCHED_SMT) + INIT_LIST_HEAD(&se->expel_node); +#endif + se->my_q = cfs_rq; /* guarantee group entities always have weight */ update_load_set(&se->load, NICE_0_LOAD); diff --git a/kernel/sched/loadavg.c b/kernel/sched/loadavg.c index 2d1f906..54a212a 100644 --- a/kernel/sched/loadavg.c +++ b/kernel/sched/loadavg.c @@ -83,6 +83,8 @@ long calc_load_fold_active(struct rq *this_rq, long adjust) nr_active = this_rq->nr_running - adjust; nr_active += (long)this_rq->nr_uninterruptible; + nr_active -= id_nr_invalid(this_rq); + if (nr_active != this_rq->calc_load_active) { delta = nr_active - this_rq->calc_load_active; this_rq->calc_load_active = nr_active; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 2451eab..8736455 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -495,6 +495,13 @@ static inline void set_task_rq_fair(struct sched_entity *se, #ifdef CONFIG_GROUP_IDENTITY extern int update_identity(struct task_group *tg, s64 val); extern int update_bvt_warp_ns(struct task_group *tg, s64 val); +extern void notify_smt_expeller(struct rq *rq, struct task_struct *p); +extern void handle_smt_expeller(void); +extern unsigned int id_nr_invalid(struct rq *rq); +#else +static inline void notify_smt_expeller(struct rq *rq, struct task_struct *p) {} +static inline void handle_smt_expeller(void) {} +static inline unsigned int id_nr_invalid(struct rq *rq) { return 0; } #endif /* CFS-related fields in a runqueue */ @@ -515,6 +522,12 @@ struct cfs_rq { #ifdef CONFIG_GROUP_IDENTITY unsigned int nr_tasks; u64 min_under_vruntime; +#ifdef CONFIG_SCHED_SMT + u64 expel_spread; + u64 expel_start; + unsigned int h_nr_expel_immune; + struct list_head expel_list; +#endif struct rb_root_cached under_timeline; #endif @@ -847,6 +860,13 @@ struct rq { #ifdef CONFIG_GROUP_IDENTITY unsigned int nr_high_running; unsigned int nr_under_running; + unsigned int nr_expel_immune; + bool smt_expeller; + bool smt_expellee; + bool on_expel; +#ifdef CONFIG_SCHED_SMT + unsigned long next_expel_ib; +#endif #endif /* @@ -2109,7 +2129,7 @@ static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2) #endif -extern struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq); +extern struct sched_entity *debug_pick_first_entity(struct cfs_rq *cfs_rq); extern struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq); #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 3c048bf..282a625 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -521,6 +521,15 @@ static int sysrq_sysctl_handler(struct ctl_table *table, int write, .mode = 0644, .proc_handler = proc_dointvec, }, +#ifdef CONFIG_SCHED_SMT + { + .procname = "sched_expel_idle_balance_delay", + .data = &sysctl_sched_expel_idle_balance_delay, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif #endif #ifdef CONFIG_PROVE_LOCKING { -- 1.8.3.1