From 827bf49be8a3b8940b9b45485a93b171e4483fa7 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Tue, 4 Aug 2020 16:28:18 +0800 Subject: [PATCH 2689/2944] alinux: sched: introduce per-cgroup identity to #30665478 In collocation environment, we deploy latency sensitive (LS) workloads and best effort (BE) workloads on the same machine, and borrowed vritual time(BVT) is the legacy feature to help ensure the schedule priority for LS. This patch is just porting the idea, however, in order to reduce the overhead, we introduced group identity feature which based on the idea of multiple rb-tree. There are now two rb-tree for each cfs_rq, and tasks with special identity will be queued into the corresponding tree, this help reduce the complexity of implementation and obviously reduce the overhead of following smt expeller(Noise Clean) feature. By write particular identity into the new cpu-cgroup entry named 'group_identity', tasks of this cgroup will perform the special schedule behavior, including: * ID_NORMAL Normal CFS tasks * ID_HIGHCLASS Tasks preempt normal & underclass tasks on wakeup, also consider underclass only cpu as idle. * ID_UNDERCLASS Tasks take penalty on wakeup. And the legacy entry 'bvt_warp_ns' was reserved for compatibility, the relationship with identity are: * 2 == ID_HIGHCLASS * 1 == ID_HIGHCLASS * 0 == ID_NORMAL * -1 == ID_UNDERCLASS * -2 == ID_UNDERCLASS Signed-off-by: Michael Wang Signed-off-by: Cruz Zhao Acked-by: Michael Wang --- include/linux/sched.h | 7 + include/linux/sched/sysctl.h | 4 + init/Kconfig | 16 ++ kernel/sched/core.c | 51 ++++ kernel/sched/debug.c | 11 + kernel/sched/fair.c | 605 ++++++++++++++++++++++++++++++++++++++++--- kernel/sched/sched.h | 21 ++ kernel/sysctl.c | 9 + 8 files changed, 692 insertions(+), 32 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 23b016d..1361fbb 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -431,6 +431,9 @@ struct sched_statistics { u64 nr_migrations_cold; u64 nr_failed_migrations_affine; u64 nr_failed_migrations_running; +#ifdef CONFIG_GROUP_IDENTITY + u64 nr_failed_migrations_id; +#endif u64 nr_failed_migrations_hot; u64 nr_forced_migrations; @@ -489,6 +492,10 @@ struct sched_entity { struct cfs_rq *my_q; #endif +#ifdef CONFIG_GROUP_IDENTITY + int id_flags; +#endif + #ifdef CONFIG_SMP /* * Per entity load average tracking. diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 0460712..bdb1b35 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -62,6 +62,10 @@ int sched_proc_update_handler(struct ctl_table *table, int write, extern unsigned int sysctl_sched_cfs_bw_burst_enabled; #endif +#ifdef CONFIG_GROUP_IDENTITY +extern unsigned int sysctl_sched_bvt_place_epsilon; +#endif + #ifdef CONFIG_SCHED_AUTOGROUP extern unsigned int sysctl_sched_autogroup_enabled; #endif diff --git a/init/Kconfig b/init/Kconfig index 376ba03..3dd9b31 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -799,6 +799,22 @@ config FAIR_GROUP_SCHED depends on CGROUP_SCHED default CGROUP_SCHED +config GROUP_IDENTITY + bool "Group identity for SCHED_OTHER" + depends on (CGROUP_SCHED && 64BIT && SMP) + default CGROUP_SCHED + help + This feature introduced identity for cgroup, including: + ID_NORMAL + Normal CFS tasks. + ID_HIGHCLASS + Tasks preempt normal & underclass tasks on wakeup, + also consider underclass only cpu as idle. + ID_UNDERCLASS + Tasks take penalty on wakeup. + These identity allow the tasks of cgroup to perform special + schedule behaviour. + config CFS_BANDWIDTH bool "CPU bandwidth provisioning for FAIR_GROUP_SCHED" depends on FAIR_GROUP_SCHED diff --git a/kernel/sched/core.c b/kernel/sched/core.c index aa7a769..d163b65 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2129,6 +2129,10 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->se.cfs_rq = NULL; #endif +#ifdef CONFIG_GROUP_IDENTITY + p->se.id_flags = 0; +#endif + #ifdef CONFIG_SCHEDSTATS /* Even if schedstat is disabled, there should not be garbage */ memset(&p->se.statistics, 0, sizeof(p->se.statistics)); @@ -6963,6 +6967,40 @@ static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, } #endif /* CONFIG_RT_GROUP_SCHED */ +#ifdef CONFIG_GROUP_IDENTITY +static int cpu_bvt_warp_ns_write_s64(struct cgroup_subsys_state *css, + struct cftype *cftype, s64 val) +{ + struct task_group *tg = css_tg(css); + + return update_bvt_warp_ns(tg, val); +} + +static s64 cpu_bvt_warp_ns_read_s64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct task_group *tg = css_tg(css); + + return tg->bvt_warp_ns; +} + +static int cpu_identity_write_s64(struct cgroup_subsys_state *css, + struct cftype *cftype, s64 val) +{ + struct task_group *tg = css_tg(css); + + return update_identity(tg, val); +} + +static s64 cpu_identity_read_s64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + struct task_group *tg = css_tg(css); + + return tg->id_flags; +} +#endif + static struct cftype cpu_legacy_files[] = { #ifdef CONFIG_FAIR_GROUP_SCHED { @@ -7009,6 +7047,19 @@ static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, .write_u64 = cpu_rt_period_write_uint, }, #endif +#ifdef CONFIG_GROUP_IDENTITY + /* legacy bvt interface */ + { + .name = "bvt_warp_ns", + .read_s64 = cpu_bvt_warp_ns_read_s64, + .write_s64 = cpu_bvt_warp_ns_write_s64, + }, + { + .name = "identity", + .read_s64 = cpu_identity_read_s64, + .write_s64 = cpu_identity_write_s64, + }, +#endif { } /* Terminate */ }; diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index fe1c5ec..e1b4eef 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -525,6 +525,10 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) SEQ_printf(m, " .%-30s: %d\n", "nr_spread_over", cfs_rq->nr_spread_over); SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running); +#ifdef CONFIG_GROUP_IDENTITY + SEQ_printf(m, " .%-30s: %lld.%06lu\n", "min_under_vruntime", + SPLIT_NS(cfs_rq->min_under_vruntime)); +#endif SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight); #ifdef CONFIG_SMP SEQ_printf(m, " .%-30s: %ld\n", "runnable_weight", cfs_rq->runnable_weight); @@ -642,6 +646,10 @@ static void print_cpu(struct seq_file *m, int cpu) SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rq->x)) P(nr_running); +#ifdef CONFIG_GROUP_IDENTITY + P(nr_high_running); + P(nr_under_running); +#endif P(nr_switches); P(nr_load_updates); P(nr_uninterruptible); @@ -906,6 +914,9 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, P_SCHEDSTAT(se.statistics.nr_migrations_cold); P_SCHEDSTAT(se.statistics.nr_failed_migrations_affine); P_SCHEDSTAT(se.statistics.nr_failed_migrations_running); +#ifdef CONFIG_GROUP_IDENTITY + P_SCHEDSTAT(se.statistics.nr_failed_migrations_id); +#endif P_SCHEDSTAT(se.statistics.nr_failed_migrations_hot); P_SCHEDSTAT(se.statistics.nr_forced_migrations); P_SCHEDSTAT(se.statistics.nr_wakeups); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3d38061..ec4d9e8 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -110,6 +110,16 @@ int __weak arch_asym_cpu_priority(int cpu) unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; #endif +#ifdef CONFIG_GROUP_IDENTITY +/* + * Waking batch tasks are placed sched_bvt_place_epsilon + * nanoseconds relative to min_vruntime. + * + * Default: 1 msec in the future, units: nanoseconds + */ +unsigned int sysctl_sched_bvt_place_epsilon = 1000000UL; +#endif + /* * The margin used when comparing utilization with CPU capacity: * util * margin < capacity * 1024 @@ -520,6 +530,477 @@ static inline int entity_before(struct sched_entity *a, return (s64)(a->vruntime - b->vruntime) < 0; } +#ifdef CONFIG_GROUP_IDENTITY + +/* legacy bvt type */ +enum { + TYPE_IDLE = -2, + TYPE_BATCH, + TYPE_NORMAL, + TYPE_LS, + TYPE_STRICT, +}; + +#define ID_NORMAL 0x0000 +#define ID_UNDERCLASS 0x0001 +#define ID_HIGHCLASS 0x0002 +#define IDENTITY_FLAGS_MASK 0x00ff + +static DEFINE_MUTEX(identity_mutex); +/* + * When we talking about identity, there are two viewpoint, that is + * level-view and top-view. + * + * Helpers like is_xxx() is for level-view, usually for the comparison + * of two se from the same level, their identity depends on the task + * group they standing for, and normal task will never got identity. + * + * Helpers like is_xxx_task() is for top-view, usually for the task + * scheduling decisions, to estimate the cpu situation for tasks. + * + * To be noticed, identity of task on top-view depends on the identity + * of it's group, for example we consider a task from the underclass + * group as a underclass task, depite of the fact that it may be the + * descendant of a highclass group. + */ +static inline bool test_identity(struct sched_entity *se, int flags) +{ + return se->id_flags & flags; +} + +static inline bool is_underclass(struct sched_entity *se) +{ + return test_identity(se, ID_UNDERCLASS); +} + +static inline bool is_highclass(struct sched_entity *se) +{ + return test_identity(se, ID_HIGHCLASS); +} + +static inline bool underclass_only(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + return rq->cfs.h_nr_running && + rq->cfs.h_nr_running == rq->nr_under_running; +} + +static inline bool is_highclass_task(struct task_struct *p) +{ + bool ret; + + rcu_read_lock(); + ret = p->se.parent ? is_highclass(p->se.parent) : false; + rcu_read_unlock(); + + return ret; +} + +static inline bool is_underclass_task(struct task_struct *p) +{ + bool ret; + + rcu_read_lock(); + ret = p->se.parent ? is_underclass(p->se.parent) : false; + rcu_read_unlock(); + return ret; +} + +static noinline bool +id_can_migrate_task(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) +{ + /* Do not migrate the last highclass, try someone else */ + if (is_highclass_task(p) && src_rq->nr_high_running < 2) + goto bad_dst; + + return true; + +bad_dst: + schedstat_inc(p->se.statistics.nr_failed_migrations_id); + return false; +} + +static noinline bool +id_wake_affine(struct task_struct *p, int this_cpu, int prev_cpu) +{ + struct rq *prev_rq = cpu_rq(prev_cpu); + + /* Last highclass should stay */ + if (is_highclass_task(p) && prev_rq->nr_high_running < 1) + return false; + + return true; +} + +static noinline bool id_idle_cpu(struct task_struct *p, int cpu) +{ + if (available_idle_cpu(cpu)) + return true; + /* CPU full of underclass is idle for highclass */ + return is_highclass_task(p) && underclass_only(cpu); +} + +static __always_inline void +id_update_nr_running(struct task_group *tg, struct rq *rq, long delta) +{ + struct sched_entity *se; + + if (!tg || tg == &root_task_group) + return; + + se = tg->se[rq->cpu]; + + if (is_highclass(se)) + rq->nr_high_running += delta; + + if (is_underclass(se)) + rq->nr_under_running += delta; +} + +static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se); +static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se); + +static void __update_identity(struct task_group *tg, int flags) +{ + int cpu; + struct rq_flags rf; + + tg->id_flags = flags; + + for_each_online_cpu(cpu) { + bool on_rq; + unsigned int delta; + struct cfs_rq *cfs_rq; + struct sched_entity *se; + struct rq *rq = cpu_rq(cpu); + + rq_lock_irq(rq, &rf); + + se = tg->se[cpu]; + cfs_rq = cfs_rq_of(se); + delta = se->my_q->nr_tasks; + on_rq = se->on_rq; + + if (on_rq) { + if (se != cfs_rq->curr) + __dequeue_entity(cfs_rq, se); + id_update_nr_running(tg, rq, -delta); + } + + se->id_flags = flags; + + if (on_rq) { + if (se != cfs_rq->curr) + __enqueue_entity(cfs_rq, se); + id_update_nr_running(tg, rq, delta); + } + + rq_unlock_irq(rq, &rf); + } +} + +int update_bvt_warp_ns(struct task_group *tg, s64 val) +{ + int flags = 0; + + /* + * We can't change the bvt type of the root cgroup. + */ + if (!tg->se[0]) + return -EINVAL; + + switch (val) { + case TYPE_IDLE: + flags = ID_UNDERCLASS; + break; + case TYPE_BATCH: + flags = ID_UNDERCLASS; + break; + case TYPE_NORMAL: + break; + case TYPE_LS: + flags = ID_HIGHCLASS; + break; + case TYPE_STRICT: + flags = ID_HIGHCLASS; + break; + default: + return -ERANGE; + } + + mutex_lock(&identity_mutex); + if (tg->bvt_warp_ns != val) { + __update_identity(tg, flags); + tg->bvt_warp_ns = val; + } + mutex_unlock(&identity_mutex); + + return 0; +} + +int update_identity(struct task_group *tg, s64 val) +{ + /* + * We can't change the flags of the root cgroup. + */ + if (!tg->se[0]) + return -EINVAL; + + if (val & ~IDENTITY_FLAGS_MASK) + return -ERANGE; + + if (val & ID_HIGHCLASS && val & ID_UNDERCLASS) + return -EINVAL; + + mutex_lock(&identity_mutex); + if (tg->id_flags != val) + __update_identity(tg, val); + mutex_unlock(&identity_mutex); + + return 0; +} + +static inline void identity_init_cfs_rq(struct cfs_rq *cfs_rq) +{ + cfs_rq->under_timeline = RB_ROOT_CACHED; + cfs_rq->min_under_vruntime = (u64)(-(1LL << 20)); +} + +static inline struct rb_root_cached * +id_rb_root(struct cfs_rq *cfs_rq, struct sched_entity *se) +{ + if (is_underclass(se)) + return &cfs_rq->under_timeline; + + return &cfs_rq->tasks_timeline; +} + +static inline struct rb_node * +id_rb_first_cached(struct cfs_rq *cfs_rq) +{ + int i; + struct rb_node *left; + + struct rb_root_cached *roots[2] = { + &cfs_rq->tasks_timeline, + &cfs_rq->under_timeline, + }; + + if (cfs_rq->min_under_vruntime < cfs_rq->min_vruntime) { + roots[0] = &cfs_rq->under_timeline; + roots[1] = &cfs_rq->tasks_timeline; + } + + for (i = 0; i < 2; i++) { + left = rb_first_cached(roots[i]); + if (left) + return left; + } + + return NULL; +} + +static inline u64 +id_min_vruntime(struct cfs_rq *cfs_rq, struct sched_entity *se) +{ + return is_underclass(se) ? + cfs_rq->min_under_vruntime : cfs_rq->min_vruntime; +} + +/* + * id_preempt_xxx() help to check if preempt should happen for identity + * reason, return value are consist with wakeup_preempt_entity(). + * + * -1 -- curr should preempt se for higher identity + * 1 -- se should preempt curr for higher identity + * 0 -- no preempt for same identity + * + * id_preempt_underclass() + * others will preempt underclass, called with no respect to the + * vruntime. + * + * id_preempt_highclass() + * highclass will preempt normal, called with the respect to the + * vruntime. + * + * id_preempt_all() + * higher identity will preempt lower, called with the respect to + * the vruntime. + */ +static inline int +id_preempt_underclass(struct sched_entity *curr, struct sched_entity *se) +{ + bool under_curr = is_underclass(curr); + bool under_se = is_underclass(se); + + if (under_curr == under_se) + return 0; + + return under_se ? -1 : 1; +} + +static inline int +id_preempt_highclass(struct sched_entity *curr, struct sched_entity *se) +{ + bool high_curr = is_highclass(curr); + bool high_se = is_highclass(se); + + if (high_curr == high_se) + return 0; + + return high_curr ? -1 : 1; +} + +static inline int +id_preempt_all(struct sched_entity *curr, struct sched_entity *se) +{ + int ret = id_preempt_highclass(curr, se); + + return ret == 0 ? id_preempt_underclass(curr, se) : ret; +} + +/* + * We must sync the empty tree's min vruntime up, otherwise it + * gain vruntime bonus just for being idle. + */ +static inline void sync_min_vruntime(struct cfs_rq *cfs_rq) +{ + u64 effect_vruntime = max_vruntime(cfs_rq->min_vruntime, + cfs_rq->min_under_vruntime); + bool no_curr = (!cfs_rq->curr || !cfs_rq->curr->on_rq); + + if (!rb_first_cached(&cfs_rq->tasks_timeline) && + (no_curr || is_underclass(cfs_rq->curr))) + cfs_rq->min_vruntime = effect_vruntime; + + if (!rb_first_cached(&cfs_rq->under_timeline) && + (no_curr || !is_underclass(cfs_rq->curr))) + cfs_rq->min_under_vruntime = effect_vruntime; +} + +static inline void update_under_min_vruntime(struct cfs_rq *cfs_rq) +{ + struct sched_entity *curr = cfs_rq->curr; + struct rb_node *leftmost = rb_first_cached(&cfs_rq->under_timeline); + + u64 vruntime = cfs_rq->min_under_vruntime; + + if (curr) { + if (curr->on_rq && is_underclass(curr)) + vruntime = curr->vruntime; + else + curr = NULL; + } + + if (leftmost) { /* non-empty tree */ + struct sched_entity *se; + + se = rb_entry(leftmost, struct sched_entity, run_node); + + if (!curr) + vruntime = se->vruntime; + else + vruntime = min_vruntime(vruntime, se->vruntime); + } + + /* ensure we never gain time by being placed backwards. */ + cfs_rq->min_under_vruntime = + max_vruntime(cfs_rq->min_under_vruntime, vruntime); + + sync_min_vruntime(cfs_rq); +} + +#else + +static inline bool is_underclass(struct sched_entity *curr) +{ + return false; +} + +static inline bool is_highclass(struct sched_entity *se) +{ + return false; +} + +static inline bool underclass_only(int cpu) +{ + return false; +} + +static inline bool is_highclass_task(struct task_struct *p) +{ + return false; +} + +static bool +id_can_migrate_task(struct task_struct *p, struct rq *src_rq, struct rq *dst_rq) +{ + return true; +} + +static inline bool +id_wake_affine(struct task_struct *p, int this_cpu, int prev_cpu) +{ + return true; +} + +static inline bool id_idle_cpu(struct task_struct *p, int cpu) +{ + return available_idle_cpu(cpu); +} + +static inline void identity_init_cfs_rq(struct cfs_rq *cfs_rq) +{ +} + +static inline struct rb_root_cached * +id_rb_root(struct cfs_rq *cfs_rq, struct sched_entity *se) +{ + return &cfs_rq->tasks_timeline; +} + +static inline struct rb_node * +id_rb_first_cached(struct cfs_rq *cfs_rq) +{ + return rb_first_cached(&cfs_rq->tasks_timeline); +} + +static inline u64 +id_min_vruntime(struct cfs_rq *cfs_rq, struct sched_entity *se) +{ + return cfs_rq->min_vruntime; +} + +static inline int +id_preempt_underclass(struct sched_entity *curr, struct sched_entity *se) +{ + return 0; +} + +static inline int +id_preempt_highclass(struct sched_entity *curr, struct sched_entity *se) +{ + return 0; +} + +static inline int +id_preempt_all(struct sched_entity *curr, struct sched_entity *se) +{ + return 0; +} + +static __always_inline void +id_update_nr_running(struct task_group *tg, struct rq *rq, long delta) +{ +} + +static inline void update_under_min_vruntime(struct cfs_rq *cfs_rq) +{ +} + +#endif + static void update_min_vruntime(struct cfs_rq *cfs_rq) { struct sched_entity *curr = cfs_rq->curr; @@ -528,7 +1009,7 @@ static void update_min_vruntime(struct cfs_rq *cfs_rq) u64 vruntime = cfs_rq->min_vruntime; if (curr) { - if (curr->on_rq) + if (curr->on_rq && !is_underclass(curr)) vruntime = curr->vruntime; else curr = NULL; @@ -550,6 +1031,8 @@ static void update_min_vruntime(struct cfs_rq *cfs_rq) smp_wmb(); cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime; #endif + + update_under_min_vruntime(cfs_rq); } /* @@ -557,11 +1040,15 @@ static void update_min_vruntime(struct cfs_rq *cfs_rq) */ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - struct rb_node **link = &cfs_rq->tasks_timeline.rb_root.rb_node; + struct rb_root_cached *root; + struct rb_node **link; struct rb_node *parent = NULL; struct sched_entity *entry; bool leftmost = true; + root = id_rb_root(cfs_rq, se); + link = &root->rb_root.rb_node; + /* * Find the right place in the rbtree: */ @@ -581,18 +1068,17 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) } rb_link_node(&se->run_node, parent, link); - rb_insert_color_cached(&se->run_node, - &cfs_rq->tasks_timeline, leftmost); + rb_insert_color_cached(&se->run_node, root, leftmost); } static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - rb_erase_cached(&se->run_node, &cfs_rq->tasks_timeline); + rb_erase_cached(&se->run_node, id_rb_root(cfs_rq, se)); } struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) { - struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline); + struct rb_node *left = id_rb_first_cached(cfs_rq); if (!left) return NULL; @@ -2804,6 +3290,9 @@ static inline void update_scan_period(struct task_struct *p, int new_cpu) account_numa_enqueue(rq, task_of(se)); list_add(&se->group_node, &rq->cfs_tasks); +#ifdef CONFIG_GROUP_IDENTITY + cfs_rq->nr_tasks++; +#endif } #endif cfs_rq->nr_running++; @@ -3875,7 +4364,7 @@ static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se) static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) { - u64 vruntime = cfs_rq->min_vruntime; + u64 vruntime = id_min_vruntime(cfs_rq, se); /* * The 'current' period is already promised to the current tasks, @@ -3900,6 +4389,16 @@ static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se) vruntime -= thresh; } +#ifdef CONFIG_GROUP_ENTITY + /* + * The runtime penalty for underclass wakee to make sure + * they won't interrupt other's execution too much. + */ + if (!initial && is_underclass(se)) + vruntime = cfs_rq->min_vruntime + + sysctl_sched_bvt_place_epsilon; +#endif + /* ensure we never gain time by being placed backwards. */ se->vruntime = max_vruntime(se->vruntime, vruntime); } @@ -3968,7 +4467,7 @@ static inline void check_schedstat_required(void) * update_curr(). */ if (renorm && curr) - se->vruntime += cfs_rq->min_vruntime; + se->vruntime += id_min_vruntime(cfs_rq, se); update_curr(cfs_rq); @@ -3979,7 +4478,7 @@ static inline void check_schedstat_required(void) * fairness detriment of existing tasks. */ if (renorm && !curr) - se->vruntime += cfs_rq->min_vruntime; + se->vruntime += id_min_vruntime(cfs_rq, se); /* * When enqueuing a sched_entity, we must: @@ -4092,7 +4591,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) * can move min_vruntime forward still more. */ if (!(flags & DEQUEUE_SLEEP)) - se->vruntime -= cfs_rq->min_vruntime; + se->vruntime -= id_min_vruntime(cfs_rq, se); /* return excess runtime on last dequeue */ return_cfs_rq_runtime(cfs_rq); @@ -4136,7 +4635,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) * narrow margin doesn't have to wait for a full slice. * This also mitigates buddy induced latencies under load. */ - if (delta_exec < sysctl_sched_min_granularity) + if (is_highclass(curr) && delta_exec < sysctl_sched_min_granularity) return; se = __pick_first_entity(cfs_rq); @@ -4145,7 +4644,7 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) if (delta < 0) return; - if (delta > ideal_runtime) + if (delta > ideal_runtime || id_preempt_all(curr, se) == 1) resched_curr(rq_of(cfs_rq)); } @@ -4554,17 +5053,18 @@ static int tg_throttle_down(struct task_group *tg, void *data) static void throttle_cfs_rq(struct cfs_rq *cfs_rq) { + struct task_group *tg = cfs_rq->tg; struct rq *rq = rq_of(cfs_rq); - struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); + struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); struct sched_entity *se; long task_delta, dequeue = 1; bool empty; - se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; + se = tg->se[cpu_of(rq_of(cfs_rq))]; /* freeze hierarchy runnable averages while throttled */ rcu_read_lock(); - walk_tg_tree_from(cfs_rq->tg, tg_throttle_down, tg_nop, (void *)rq); + walk_tg_tree_from(tg, tg_throttle_down, tg_nop, (void *)rq); rcu_read_unlock(); task_delta = cfs_rq->h_nr_running; @@ -4584,8 +5084,10 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) dequeue = 0; } - if (!se) + if (!se) { sub_nr_running(rq, task_delta); + id_update_nr_running(tg, rq, -task_delta); + } cfs_rq->throttled = 1; cfs_rq->throttled_clock = rq_clock(rq); @@ -4614,14 +5116,15 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) { + struct task_group *tg = cfs_rq->tg; struct cfs_rq *bottom_cfs_rq = cfs_rq; struct rq *rq = rq_of(cfs_rq); - struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); + struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); struct sched_entity *se; int enqueue = 1; long task_delta; - se = cfs_rq->tg->se[cpu_of(rq)]; + se = tg->se[cpu_of(rq)]; cfs_rq->throttled = 0; @@ -4633,7 +5136,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) raw_spin_unlock(&cfs_b->lock); /* update hierarchical throttle state */ - walk_tg_tree_from(cfs_rq->tg, tg_nop, tg_unthrottle_up, (void *)rq); + walk_tg_tree_from(tg, tg_nop, tg_unthrottle_up, (void *)rq); if (!cfs_rq->load.weight) return; @@ -4657,9 +5160,10 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) SCHED_WARN_ON(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list); - if (!se) + if (!se) { add_nr_running(rq, task_delta); - + id_update_nr_running(tg, rq, task_delta); + } /* Determine whether we need to wake up potentially idle CPU: */ if (rq->curr == rq->idle && rq->cfs.nr_running) resched_curr(rq); @@ -5307,8 +5811,11 @@ static inline void hrtick_update(struct rq *rq) update_cfs_group(se); } - if (!se) + if (!se) { add_nr_running(rq, 1); + id_update_nr_running(task_group(p), rq, 1); + } + if (cfs_bandwidth_used()) { /* * When bandwidth control is enabled; the cfs_rq_throttled() @@ -5390,8 +5897,10 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) update_cfs_group(se); } - if (!se) + if (!se) { sub_nr_running(rq, 1); + id_update_nr_running(task_group(p), rq, -1); + } util_est_dequeue(&rq->cfs, p, task_sleep); hrtick_update(rq); @@ -5959,7 +6468,7 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t for_each_cpu(cpu, cpu_smt_mask(target)) { if (!cpumask_test_cpu(cpu, &p->cpus_allowed)) continue; - if (available_idle_cpu(cpu)) + if (id_idle_cpu(p, cpu)) return cpu; } @@ -6042,13 +6551,15 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) struct sched_domain *sd; int i, recent_used_cpu; - if (available_idle_cpu(target)) + if (id_idle_cpu(p, target)) return target; /* * If the previous CPU is cache affine and idle, don't be stupid: */ - if (prev != target && cpus_share_cache(prev, target) && available_idle_cpu(prev)) + if (prev != target && + cpus_share_cache(prev, target) && + id_idle_cpu(p, prev)) return prev; /* Check a recently used CPU as a potential idle candidate: */ @@ -6056,7 +6567,7 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target) if (recent_used_cpu != prev && recent_used_cpu != target && cpus_share_cache(recent_used_cpu, target) && - available_idle_cpu(recent_used_cpu) && + id_idle_cpu(p, recent_used_cpu) && cpumask_test_cpu(p->recent_used_cpu, &p->cpus_allowed)) { /* * Replace recent_used_cpu with prev as it is a potential @@ -6272,10 +6783,15 @@ static int wake_cap(struct task_struct *p, int cpu, int prev_cpu) int want_affine = 0; int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING); + /* Endow LS task the ability to balance at fork */ + if ((sd_flag & SD_BALANCE_FORK) && is_highclass_task(p)) + sd_flag |= SD_BALANCE_WAKE; + if (sd_flag & SD_BALANCE_WAKE) { record_wakee(p); want_affine = !wake_wide(p) && !wake_cap(p, cpu, prev_cpu) - && cpumask_test_cpu(cpu, &p->cpus_allowed); + && cpumask_test_cpu(cpu, &p->cpus_allowed) + && id_wake_affine(p, cpu, prev_cpu); } rcu_read_lock(); @@ -6350,7 +6866,7 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) min_vruntime = cfs_rq->min_vruntime; #endif - se->vruntime -= min_vruntime; + se->vruntime -= id_min_vruntime(cfs_rq, se); } if (p->on_rq == TASK_ON_RQ_MIGRATING) { @@ -6425,11 +6941,24 @@ static unsigned long wakeup_gran(struct sched_entity *se) static int wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se) { + int ret; s64 gran, vdiff = curr->vruntime - se->vruntime; + /* + * Others always preempt underclass on wakeup, no + * respect to vruntime. + */ + ret = id_preempt_underclass(curr, se); + if (ret) + return ret; + if (vdiff <= 0) return -1; + ret = id_preempt_highclass(curr, se); + if (ret) + return ret; + gran = wakeup_gran(se); if (vdiff > gran) return 1; @@ -7059,6 +7588,9 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) return 0; } + if (!id_can_migrate_task(p, env->src_rq, env->dst_rq)) + return 0; + /* * Aggressive migration if: * 1) destination numa is preferred @@ -9719,7 +10251,7 @@ static void task_fork_fair(struct task_struct *p) resched_curr(rq); } - se->vruntime -= cfs_rq->min_vruntime; + se->vruntime -= id_min_vruntime(cfs_rq, se); rq_unlock(rq, &rf); } @@ -9839,7 +10371,7 @@ static void detach_task_cfs_rq(struct task_struct *p) * cause 'unlimited' sleep bonus. */ place_entity(cfs_rq, se, 0); - se->vruntime -= cfs_rq->min_vruntime; + se->vruntime -= id_min_vruntime(cfs_rq, se); } detach_entity_cfs_rq(se); @@ -9853,7 +10385,7 @@ static void attach_task_cfs_rq(struct task_struct *p) attach_entity_cfs_rq(se); if (!vruntime_normalized(p)) - se->vruntime += cfs_rq->min_vruntime; + se->vruntime += id_min_vruntime(cfs_rq, se); } static void switched_from_fair(struct rq *rq, struct task_struct *p) @@ -9906,6 +10438,7 @@ void init_cfs_rq(struct cfs_rq *cfs_rq) #ifdef CONFIG_SMP raw_spin_lock_init(&cfs_rq->removed.lock); #endif + identity_init_cfs_rq(cfs_rq); } #ifdef CONFIG_FAIR_GROUP_SCHED @@ -10088,6 +10621,11 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) tg->shares = NICE_0_LOAD; +#ifdef CONFIG_GROUP_IDENTITY + tg->bvt_warp_ns = parent->bvt_warp_ns; + tg->id_flags = parent->id_flags; +#endif + init_cfs_bandwidth(tg_cfs_bandwidth(tg)); for_each_possible_cpu(i) { @@ -10180,6 +10718,9 @@ void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq, } else { se->cfs_rq = parent->my_q; se->depth = parent->depth + 1; +#ifdef CONFIG_GROUP_IDENTITY + se->id_flags = parent->id_flags; +#endif } se->my_q = cfs_rq; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 09aa58f..2451eab 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -381,6 +381,11 @@ struct task_group { #endif #endif +#ifdef CONFIG_GROUP_IDENTITY + int bvt_warp_ns; + int id_flags; +#endif + #ifdef CONFIG_RT_GROUP_SCHED struct sched_rt_entity **rt_se; struct rt_rq **rt_rq; @@ -487,6 +492,11 @@ static inline void set_task_rq_fair(struct sched_entity *se, #endif /* CONFIG_CGROUP_SCHED */ +#ifdef CONFIG_GROUP_IDENTITY +extern int update_identity(struct task_group *tg, s64 val); +extern int update_bvt_warp_ns(struct task_group *tg, s64 val); +#endif + /* CFS-related fields in a runqueue */ struct cfs_rq { struct load_weight load; @@ -502,6 +512,12 @@ struct cfs_rq { struct rb_root_cached tasks_timeline; +#ifdef CONFIG_GROUP_IDENTITY + unsigned int nr_tasks; + u64 min_under_vruntime; + struct rb_root_cached under_timeline; +#endif + /* * 'curr' points to currently running entity on this cfs_rq. * It is set to NULL otherwise (i.e when none are currently running). @@ -828,6 +844,11 @@ struct rq { struct list_head *tmp_alone_branch; #endif /* CONFIG_FAIR_GROUP_SCHED */ +#ifdef CONFIG_GROUP_IDENTITY + unsigned int nr_high_running; + unsigned int nr_under_running; +#endif + /* * This is part of a global counter where only the total sum * over all CPUs matters. A task can increase this counter on diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 48cc89a..3c048bf 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -513,6 +513,15 @@ static int sysrq_sysctl_handler(struct ctl_table *table, int write, .extra2 = &one, }, #endif +#ifdef CONFIG_GROUP_IDENTITY + { + .procname = "sched_bvt_place_epsilon", + .data = &sysctl_sched_bvt_place_epsilon, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif #ifdef CONFIG_PROVE_LOCKING { .procname = "prove_locking", -- 1.8.3.1