From 9b415cc6e5cc2ac2b6a78d271f4e695b78839ebe Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Thu, 29 Oct 2020 15:33:11 +0800 Subject: [PATCH 2119/2944] sched/fair: Fix insertion in rq->leaf_cfs_rq_list fix #31060850 commit f6783319737f28e4436a69611853a5a098cbe974 upstream The algorithm used to order cfs_rq in rq->leaf_cfs_rq_list assumes that it will walk down to root the 1st time a cfs_rq is used and we will finish to add either a cfs_rq without parent or a cfs_rq with a parent that is already on the list. But this is not always true in presence of throttling. Because a cfs_rq can be throttled even if it has never been used but other CPUs of the cgroup have already used all the bandwdith, we are not sure to go down to the root and add all cfs_rq in the list. Ensure that all cfs_rq will be added in the list even if they are throttled. Acked-by: Michael Wang Signed-off-by: Cruz Zhao --- kernel/sched/fair.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 2a3618e..05a28f1 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -282,13 +282,13 @@ static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) return grp->my_q; } -static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) +static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) { struct rq *rq = rq_of(cfs_rq); int cpu = cpu_of(rq); if (cfs_rq->on_list) - return; + return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list; cfs_rq->on_list = 1; @@ -317,7 +317,7 @@ static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) * list. */ rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; - return; + return true; } if (!cfs_rq->tg->parent) { /* @@ -331,7 +331,7 @@ static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) * tmp_alone_branch to the beginning of the list. */ rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; - return; + return true; } /* @@ -346,7 +346,7 @@ static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) * of the branch */ rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list; - return; + return false; } static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) @@ -461,8 +461,9 @@ static inline struct cfs_rq *group_cfs_rq(struct sched_entity *grp) return NULL; } -static inline void list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) +static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq) { + return true; } static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq) @@ -5141,6 +5142,11 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq) #else /* CONFIG_CFS_BANDWIDTH */ +static inline bool cfs_bandwidth_used(void) +{ + return false; +} + static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq) { return rq_clock_task(rq_of(cfs_rq)); @@ -5303,6 +5309,20 @@ static inline void hrtick_update(struct rq *rq) if (!se) add_nr_running(rq, 1); + if (cfs_bandwidth_used()) { + /* + * When bandwidth control is enabled; the cfs_rq_throttled() + * breaks in the above iteration can result in incomplete + * leaf list maintenance, resulting in triggering the assertion + * below. + */ + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + + if (list_add_leaf_cfs_rq(cfs_rq)) + break; + } + } assert_list_leaf_cfs_rq(rq); -- 1.8.3.1