From 36da4fe9851a3f8a0fa3973540df6935bc01d0c9 Mon Sep 17 00:00:00 2001
From: Xunlei Pang <xlpang@linux.alibaba.com>
Date: Wed, 24 Jul 2019 13:46:31 +0800
Subject: [PATCH 0950/2944] alinux: sched: Maintain "nr_uninterruptible" in
 runqueue

to #26424323

It's relatively easy to maintain nr_uninterruptible in scheduler
compared to doing it in cpuacct, we assume that "cpu,cpuacct" are
bound together, so that it can be used for per-cgroup load.

This will be needed to calculate per-cgroup load average later.

Reviewed-by: Michael Wang <yun.wang@linux.alibaba.com>
Signed-off-by: Xunlei Pang <xlpang@linux.alibaba.com>
Signed-off-by: Yihao Wu <wuyihao@linux.alibaba.com>
---
 include/linux/cgroup.h |  2 ++
 kernel/cgroup/cgroup.c |  6 ++++++
 kernel/sched/core.c    | 28 +++++++++++++++++++++++++---
 kernel/sched/fair.c    | 14 ++++++++++++++
 kernel/sched/rt.c      | 14 ++++++++++++++
 kernel/sched/sched.h   |  6 ++++++
 6 files changed, 67 insertions(+), 3 deletions(-)

diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index cbe792f..dbe0d5d 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -117,6 +117,8 @@ struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
 int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry);
 int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns,
 		     struct pid *pid, struct task_struct *tsk);
+extern struct cgroup_subsys_state *global_cgroup_css(struct cgroup *cgrp,
+						     int ssid);
 
 void cgroup_fork(struct task_struct *p);
 extern int cgroup_can_fork(struct task_struct *p);
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 447ec84..b275e6b 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -472,6 +472,12 @@ static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp,
 		return &cgrp->self;
 }
 
+struct cgroup_subsys_state *global_cgroup_css(struct cgroup *cgrp,
+						int ssid)
+{
+	return cgroup_css(cgrp, cgroup_subsys[(ssid)]);
+}
+
 /**
  * cgroup_tryget_css - try to get a cgroup's css for the specified subsystem
  * @cgrp: the cgroup of interest
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index d2760d7..bd27cba 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -744,18 +744,28 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
 	p->sched_class->dequeue_task(rq, p, flags);
 }
 
+static void update_nr_uninterruptible(struct task_struct *tsk, long inc)
+{
+	if (tsk->sched_class->update_nr_uninterruptible)
+		tsk->sched_class->update_nr_uninterruptible(tsk, inc);
+}
+
 void activate_task(struct rq *rq, struct task_struct *p, int flags)
 {
-	if (task_contributes_to_load(p))
+	if (task_contributes_to_load(p)) {
+		update_nr_uninterruptible(p, -1);
 		rq->nr_uninterruptible--;
+	}
 
 	enqueue_task(rq, p, flags);
 }
 
 void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
 {
-	if (task_contributes_to_load(p))
+	if (task_contributes_to_load(p)) {
+		update_nr_uninterruptible(p, 1);
 		rq->nr_uninterruptible++;
+	}
 
 	dequeue_task(rq, p, flags);
 }
@@ -1689,8 +1699,10 @@ static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
 	lockdep_assert_held(&rq->lock);
 
 #ifdef CONFIG_SMP
-	if (p->sched_contributes_to_load)
+	if (p->sched_contributes_to_load) {
+		update_nr_uninterruptible(p, -1);
 		rq->nr_uninterruptible--;
+	}
 
 	if (wake_flags & WF_MIGRATED)
 		en_flags |= ENQUEUE_MIGRATED;
@@ -6378,8 +6390,18 @@ void sched_move_task(struct task_struct *tsk)
 	if (running)
 		put_prev_task(rq, tsk);
 
+	/* decrease old group */
+	if ((!queued && task_contributes_to_load(tsk)) ||
+	    (tsk->state == TASK_WAKING && tsk->sched_contributes_to_load))
+		update_nr_uninterruptible(tsk, -1);
+
 	sched_change_group(tsk, TASK_MOVE_GROUP);
 
+	/* increase new group after change */
+	if ((!queued && task_contributes_to_load(tsk)) ||
+	    (tsk->state == TASK_WAKING && tsk->sched_contributes_to_load))
+		update_nr_uninterruptible(tsk, 1);
+
 	if (queued)
 		enqueue_task(rq, tsk, queue_flags);
 	if (running)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 6e12d6a..90295a1 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -10233,6 +10233,16 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task
 	return rr_interval;
 }
 
+#ifdef CONFIG_SCHED_SLI
+static void update_nr_uninterruptible_fair(struct task_struct *p, long inc)
+{
+	struct sched_entity *se = &p->se;
+
+	for_each_sched_entity(se)
+		cfs_rq_of(se)->nr_uninterruptible += inc;
+}
+#endif
+
 /*
  * All the scheduling class methods:
  */
@@ -10274,6 +10284,10 @@ static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task
 #ifdef CONFIG_FAIR_GROUP_SCHED
 	.task_change_group	= task_change_group_fair,
 #endif
+
+#ifdef CONFIG_SCHED_SLI
+	.update_nr_uninterruptible = update_nr_uninterruptible_fair,
+#endif
 };
 
 #ifdef CONFIG_SCHED_DEBUG
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index b980cc9..9f4e9b8 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -2374,6 +2374,16 @@ static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
 		return 0;
 }
 
+#ifdef CONFIG_SCHED_SLI
+static void update_nr_uninterruptible_rt(struct task_struct *p, long inc)
+{
+	struct sched_rt_entity *se = &p->rt;
+
+	for_each_sched_rt_entity(se)
+		rt_rq_of_se(se)->nr_uninterruptible += inc;
+}
+#endif
+
 const struct sched_class rt_sched_class = {
 	.next			= &fair_sched_class,
 	.enqueue_task		= enqueue_task_rt,
@@ -2404,6 +2414,10 @@ static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
 	.switched_to		= switched_to_rt,
 
 	.update_curr		= update_curr_rt,
+
+#ifdef CONFIG_SCHED_SLI
+	.update_nr_uninterruptible = update_nr_uninterruptible_rt,
+#endif
 };
 
 #ifdef CONFIG_RT_GROUP_SCHED
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 1d1431e..81ecabe 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -575,6 +575,8 @@ struct cfs_rq {
 #endif /* CONFIG_CFS_BANDWIDTH */
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 
+	unsigned long nr_uninterruptible;
+
 	ALI_HOTFIX_RESERVE(1)
 	ALI_HOTFIX_RESERVE(2)
 	ALI_HOTFIX_RESERVE(3)
@@ -625,6 +627,8 @@ struct rt_rq {
 	struct rq		*rq;
 	struct task_group	*tg;
 #endif
+
+	unsigned long nr_uninterruptible;
 };
 
 static inline bool rt_rq_is_runnable(struct rt_rq *rt_rq)
@@ -1671,6 +1675,8 @@ struct sched_class {
 #ifdef CONFIG_FAIR_GROUP_SCHED
 	void (*task_change_group)(struct task_struct *p, int type);
 #endif
+
+	void (*update_nr_uninterruptible)(struct task_struct *p, long inc);
 };
 
 static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
-- 
1.8.3.1

