From 9e7b35d6d84e2361d56a113cf40139fe83f4ba37 Mon Sep 17 00:00:00 2001 From: Yihao Wu Date: Tue, 21 Apr 2020 12:09:45 +0800 Subject: [PATCH 0958/2944] alinux: sched: Introduce per-cgroup iowait accounting to #26424323 We account iowait when the cgroup's se is idle, and it has blocked task on the hierarchy of se->my_q. To achieve this, we also add cg_nr_running to track the hierarchical number of blocked tasks. We do it when a blocked task wakes up or a task is blocked. Signed-off-by: Yihao Wu Signed-off-by: Shanpei Chen Acked-by: Michael Wang --- include/linux/sched.h | 4 ++++ kernel/sched/core.c | 3 +++ kernel/sched/cpuacct.c | 28 ++++++++++++++++++++++++++-- kernel/sched/fair.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/sched.h | 7 +++++++ 5 files changed, 84 insertions(+), 2 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 07f374d..e7d7b99 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -468,9 +468,13 @@ struct sched_entity { u64 cg_idle_start; u64 cg_idle_sum; u64 cg_init_time; + u64 cg_nr_iowait; + u64 cg_iowait_sum; + u64 cg_iowait_start; u64 cg_ineffective_sum; u64 cg_ineffective_start; seqcount_t idle_seqcount; + spinlock_t iowait_lock; u64 nr_migrations; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 7510ec0..6b3e019 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2046,6 +2046,7 @@ static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) if (p->in_iowait) { delayacct_blkio_end(p); atomic_dec(&task_rq(p)->nr_iowait); + update_nr_iowait(p, -1); } cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); @@ -2060,6 +2061,7 @@ static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) if (p->in_iowait) { delayacct_blkio_end(p); atomic_dec(&task_rq(p)->nr_iowait); + update_nr_iowait(p, -1); } #endif /* CONFIG_SMP */ @@ -3436,6 +3438,7 @@ static void __sched notrace __schedule(bool preempt) if (prev->in_iowait) { atomic_inc(&rq->nr_iowait); + update_nr_iowait(prev, 1); delayacct_blkio_start(); } } diff --git a/kernel/sched/cpuacct.c b/kernel/sched/cpuacct.c index 52f0307..eaacf6e 100644 --- a/kernel/sched/cpuacct.c +++ b/kernel/sched/cpuacct.c @@ -447,6 +447,7 @@ static unsigned long ca_uninterruptible(struct cpuacct *ca, int cpu) void cgroup_idle_start(struct sched_entity *se) { + unsigned long flags; u64 clock; if (!schedstat_enabled()) @@ -457,12 +458,18 @@ void cgroup_idle_start(struct sched_entity *se) write_seqcount_begin(&se->idle_seqcount); __schedstat_set(se->cg_idle_start, clock); write_seqcount_end(&se->idle_seqcount); + + spin_lock_irqsave(&se->iowait_lock, flags); + if (schedstat_val(se->cg_nr_iowait)) + __schedstat_set(se->cg_iowait_start, clock); + spin_unlock_irqrestore(&se->iowait_lock, flags); } void cgroup_idle_end(struct sched_entity *se) { + unsigned long flags; u64 clock; - u64 idle_start; + u64 idle_start, iowait_start; if (!schedstat_enabled()) return; @@ -474,6 +481,14 @@ void cgroup_idle_end(struct sched_entity *se) __schedstat_add(se->cg_idle_sum, clock - idle_start); __schedstat_set(se->cg_idle_start, 0); write_seqcount_end(&se->idle_seqcount); + + spin_lock_irqsave(&se->iowait_lock, flags); + if (schedstat_val(se->cg_nr_iowait)) { + iowait_start = schedstat_val(se->cg_iowait_start); + __schedstat_add(se->cg_iowait_sum, clock - iowait_start); + __schedstat_set(se->cg_iowait_start, 0); + } + spin_unlock_irqrestore(&se->iowait_lock, flags); } void cpuacct_cpuset_changed(struct cgroup *cgrp, struct cpumask *deleted, @@ -561,8 +576,9 @@ static void __cpuacct_get_usage_result(struct cpuacct *ca, int cpu, res->softirq = kcpustat->cpustat[CPUTIME_SOFTIRQ]; if (se && schedstat_enabled()) { unsigned int seq; + unsigned long flags; u64 idle_start, ineff, ineff_start, elapse, complement; - u64 clock; + u64 clock, iowait_start; do { seq = read_seqcount_begin(&se->idle_seqcount); @@ -578,6 +594,13 @@ static void __cpuacct_get_usage_result(struct cpuacct *ca, int cpu, if (ineff_start) __schedstat_add(ineff, clock - ineff_start); + spin_lock_irqsave(&se->iowait_lock, flags); + res->iowait = schedstat_val(se->cg_iowait_sum); + iowait_start = schedstat_val(se->cg_iowait_start); + if (iowait_start) + __schedstat_add(res->iowait, clock - iowait_start); + spin_unlock_irqrestore(&se->iowait_lock, flags); + res->steal = 0; elapse = clock - schedstat_val(se->cg_init_time); @@ -585,6 +608,7 @@ static void __cpuacct_get_usage_result(struct cpuacct *ca, int cpu, if (elapse > complement) res->steal = elapse - complement; + res->idle -= res->iowait; } else { res->idle = res->iowait = res->steal = 0; } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c5cc33b..08671ef 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10032,6 +10032,44 @@ void init_cfs_rq(struct cfs_rq *cfs_rq) } #ifdef CONFIG_FAIR_GROUP_SCHED +#ifdef CONFIG_SCHED_SLI +static void update_nr_iowait_fair(struct task_struct *p, long inc) +{ + unsigned long flags; + struct sched_entity *se = p->se.parent; + u64 clock; + + if (!schedstat_enabled()) + return; + + clock = __rq_clock_broken(cpu_rq(p->cpu)); + + for_each_sched_entity(se) { + /* + * Avoid locking rq->lock from try_to_wakeup hot path, in + * the price of poor consistency among cgroup hierarchy, + * which we can tolerate. + * While accessing se->on_rq does need to hold rq->lock. We + * already do, because when inc==1, the caller is __schedule + * and task_move_group_fair + */ + spin_lock_irqsave(&se->iowait_lock, flags); + if (!se->on_rq && !schedstat_val(se->cg_nr_iowait) && inc > 0) + __schedstat_set(se->cg_iowait_start, clock); + if (schedstat_val(se->cg_iowait_start) > 0 && + schedstat_val(se->cg_nr_iowait) + inc == 0) { + __schedstat_add(se->cg_iowait_sum, clock - + schedstat_val(se->cg_iowait_start)); + __schedstat_set(se->cg_iowait_start, 0); + } + __schedstat_add(se->cg_nr_iowait, inc); + spin_unlock_irqrestore(&se->iowait_lock, flags); + } +} +#else +static void update_nr_iowait_fair(struct task_struct *p, long inc) {} +#endif + static void task_set_group_fair(struct task_struct *p) { struct sched_entity *se = &p->se; @@ -10042,6 +10080,8 @@ static void task_set_group_fair(struct task_struct *p) static void task_move_group_fair(struct task_struct *p) { + if (p->in_iowait) + update_nr_iowait_fair(p, -1); detach_task_cfs_rq(p); set_task_rq(p, task_cpu(p)); @@ -10050,6 +10090,8 @@ static void task_move_group_fair(struct task_struct *p) p->se.avg.last_update_time = 0; #endif attach_task_cfs_rq(p); + if (p->in_iowait) + update_nr_iowait_fair(p, 1); } static void task_change_group_fair(struct task_struct *p, int type) @@ -10196,6 +10238,7 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, update_load_set(&se->load, NICE_0_LOAD); se->parent = parent; seqcount_init(&se->idle_seqcount); + spin_lock_init(&se->iowait_lock); se->cg_idle_start = se->cg_init_time = cpu_clock(cpu); } @@ -10322,6 +10365,7 @@ static void update_nr_uninterruptible_fair(struct task_struct *p, long inc) #ifdef CONFIG_SCHED_SLI .update_nr_uninterruptible = update_nr_uninterruptible_fair, + .update_nr_iowait = update_nr_iowait_fair, #endif }; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 06a75e6..f1d1f31 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1677,6 +1677,7 @@ struct sched_class { #endif void (*update_nr_uninterruptible)(struct task_struct *p, long inc); + void (*update_nr_iowait)(struct task_struct *p, long inc); }; static inline void put_prev_task(struct rq *rq, struct task_struct *prev) @@ -1689,6 +1690,12 @@ static inline void set_curr_task(struct rq *rq, struct task_struct *curr) curr->sched_class->set_curr_task(rq); } +static inline void update_nr_iowait(struct task_struct *p, long inc) +{ + if (p->sched_class->update_nr_iowait) + p->sched_class->update_nr_iowait(p, inc); +} + #ifdef CONFIG_SMP #define sched_class_highest (&stop_sched_class) #else -- 1.8.3.1