From 982358c4f1f0d20536d1829699b6741fc89c79d0 Mon Sep 17 00:00:00 2001 From: Xunlei Pang Date: Wed, 24 Jul 2019 13:46:31 +0800 Subject: [PATCH 2216/2944] alinux: sched: Maintain "nr_uninterruptible" in runqueue to #28143103 It's relatively easy to maintain nr_uninterruptible in scheduler compared to doing it in cpuacct, we assume that "cpu,cpuacct" are bound together, so that it can be used for per-cgroup load. This can help to calculate per-cgroup load average. Note that there's false-sharing between rq->idle and rq->prev_mm, also between rq->idle and rq->clock_update_flags. Mainly because rq->idle is read from CPUs other than this_cpu and read frequently. The other two are written frequently. To address the false-sharing problem and pefermance regression (-8% observed by hackbench-100%), we should place prev_mm and the sequential fields on the new cacheline. Signed-off-by: Xunlei Pang Signed-off-by: Yihao Wu Acked-by: Shanpei Chen --- kernel/sched/core.c | 28 +++++++++++++++++++++++++--- kernel/sched/fair.c | 11 +++++++++++ kernel/sched/rt.c | 14 ++++++++++++++ kernel/sched/sched.h | 15 ++++++++++++++- 4 files changed, 64 insertions(+), 4 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 1e179fa..aa7a769 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -754,18 +754,28 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags) p->sched_class->dequeue_task(rq, p, flags); } +static void update_nr_uninterruptible(struct task_struct *tsk, long inc) +{ + if (tsk->sched_class->update_nr_uninterruptible) + tsk->sched_class->update_nr_uninterruptible(tsk, inc); +} + void activate_task(struct rq *rq, struct task_struct *p, int flags) { - if (task_contributes_to_load(p)) + if (task_contributes_to_load(p)) { + update_nr_uninterruptible(p, -1); rq->nr_uninterruptible--; + } enqueue_task(rq, p, flags); } void deactivate_task(struct rq *rq, struct task_struct *p, int flags) { - if (task_contributes_to_load(p)) + if (task_contributes_to_load(p)) { + update_nr_uninterruptible(p, 1); rq->nr_uninterruptible++; + } dequeue_task(rq, p, flags); } @@ -1700,8 +1710,10 @@ static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags, lockdep_assert_held(&rq->lock); #ifdef CONFIG_SMP - if (p->sched_contributes_to_load) + if (p->sched_contributes_to_load) { + update_nr_uninterruptible(p, -1); rq->nr_uninterruptible--; + } if (wake_flags & WF_MIGRATED) en_flags |= ENQUEUE_MIGRATED; @@ -6382,8 +6394,18 @@ void sched_move_task(struct task_struct *tsk) if (running) put_prev_task(rq, tsk); + /* decrease old group */ + if ((!queued && task_contributes_to_load(tsk)) || + (tsk->state == TASK_WAKING && tsk->sched_contributes_to_load)) + update_nr_uninterruptible(tsk, -1); + sched_change_group(tsk, TASK_MOVE_GROUP); + /* increase new group after change */ + if ((!queued && task_contributes_to_load(tsk)) || + (tsk->state == TASK_WAKING && tsk->sched_contributes_to_load)) + update_nr_uninterruptible(tsk, 1); + if (queued) enqueue_task(rq, tsk, queue_flags); if (running) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 05a28f1..e46ee56 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10188,6 +10188,16 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task return rr_interval; } +#ifdef CONFIG_SCHED_SLI +static void update_nr_uninterruptible_fair(struct task_struct *p, long inc) +{ + struct sched_entity *se = &p->se; + + for_each_sched_entity(se) + cfs_rq_of(se)->nr_uninterruptible += inc; +} +#endif + /* * All the scheduling class methods: */ @@ -10231,6 +10241,7 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task #endif #ifdef CONFIG_SCHED_SLI + .update_nr_uninterruptible = update_nr_uninterruptible_fair, .update_nr_iowait = update_nr_iowait_fair, #endif }; diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 707f7fc..042fb33 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -2374,6 +2374,16 @@ static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task) return 0; } +#ifdef CONFIG_SCHED_SLI +static void update_nr_uninterruptible_rt(struct task_struct *p, long inc) +{ + struct sched_rt_entity *se = &p->rt; + + for_each_sched_rt_entity(se) + rt_rq_of_se(se)->nr_uninterruptible += inc; +} +#endif + const struct sched_class rt_sched_class = { .next = &fair_sched_class, .enqueue_task = enqueue_task_rt, @@ -2404,6 +2414,10 @@ static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task) .switched_to = switched_to_rt, .update_curr = update_curr_rt, + +#ifdef CONFIG_SCHED_SLI + .update_nr_uninterruptible = update_nr_uninterruptible_rt, +#endif }; #ifdef CONFIG_RT_GROUP_SCHED diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index db6a76f..ac0807d 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -576,6 +576,8 @@ struct cfs_rq { #endif /* CONFIG_CFS_BANDWIDTH */ #endif /* CONFIG_FAIR_GROUP_SCHED */ + unsigned long nr_uninterruptible; + ALI_HOTFIX_RESERVE(1) ALI_HOTFIX_RESERVE(2) ALI_HOTFIX_RESERVE(3) @@ -626,6 +628,8 @@ struct rt_rq { struct rq *rq; struct task_group *tg; #endif + + unsigned long nr_uninterruptible; }; static inline bool rt_rq_is_runnable(struct rt_rq *rt_rq) @@ -836,7 +840,14 @@ struct rq { struct task_struct *idle; struct task_struct *stop; unsigned long next_balance; - struct mm_struct *prev_mm; + + /* + * Frequent writing to prev_mm and clock_update_flags on local + * CPU causes cacheline containing idle to be invalidated on + * other CPUs. Put prev_mm and sequential fields on a new + * cacheline to fix it. + */ + struct mm_struct *prev_mm ____cacheline_aligned; unsigned int clock_update_flags; u64 clock; @@ -1670,6 +1681,8 @@ struct sched_class { #ifdef CONFIG_FAIR_GROUP_SCHED void (*task_change_group)(struct task_struct *p, int type); #endif + + void (*update_nr_uninterruptible)(struct task_struct *p, long inc); void (*update_nr_iowait)(struct task_struct *p, long inc); }; -- 1.8.3.1