From b2818621d5e7d88999bcab6fd5ef18f1b2224091 Mon Sep 17 00:00:00 2001 From: Michael Wang Date: Mon, 20 Jan 2020 17:24:11 +0800 Subject: [PATCH 2695/2944] alinux: sched/isolation: dynamical CPU isolation support fix #28231823 to #30665478 We have so many wild tasks under root cgroup, they come from anywhere and there are no good way to manage them properly. However, we don't want them to disturb our critical tasks, for example the IO process tasks in MOC environment. Currently we're using 'isolcpu' cmdline parameter to isolate CPUs, however this is a static config, so later when we want to release more CPU resources, reboot is required. This patch introduced a way to dynamically isolate CPUs from those wild tasks, give admin the capability of dynamical CPU resource arrangement. By 'echo CPULIST > /proc/dyn_isolcpus', the CPUs will be: * Isolated from the unbound userspace wild tasks * Get rid of schedule domain * Isolated from the unbound workqueue worker Reviewed-by: Shanpei Chen Signed-off-by: Michael Wang Signed-off-byy: Cruz Zhao Acked-by: Michael Wang --- include/linux/sched/isolation.h | 7 ++ kernel/cgroup/cpuset.c | 23 ++++- kernel/sched/isolation.c | 189 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 3 deletions(-) diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h index 4a6582c..30c84ba 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -40,6 +40,13 @@ static inline void housekeeping_affine(struct task_struct *t, static inline void housekeeping_init(void) { } #endif /* CONFIG_CPU_ISOLATION */ +#if defined(CONFIG_CPU_ISOLATION) && defined(CONFIG_CGROUP_SCHED) +DECLARE_STATIC_KEY_FALSE(dyn_isolcpus_enabled); +extern void wilds_cpus_allowed(struct cpumask *pmask); +#else +static inline void wilds_cpus_allowed(struct cpumask *pmask) {} +#endif + static inline bool housekeeping_cpu(int cpu, enum hk_flags flags) { #ifdef CONFIG_CPU_ISOLATION diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 290177b..e2a621d 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -1545,9 +1545,10 @@ static void cpuset_attach(struct cgroup_taskset *tset) mutex_lock(&cpuset_mutex); /* prepare for attach */ - if (cs == &top_cpuset) + if (cs == &top_cpuset) { cpumask_copy(cpus_attach, cpu_possible_mask); - else + wilds_cpus_allowed(cpus_attach); + } else guarantee_online_cpus(cs, cpus_attach); guarantee_online_mems(cs, &cpuset_attach_nodemask_to); @@ -2102,8 +2103,24 @@ static void cpuset_bind(struct cgroup_subsys_state *root_css) */ static void cpuset_fork(struct task_struct *task) { - if (task_css_is_root(task, cpuset_cgrp_id)) + if (task_css_is_root(task, cpuset_cgrp_id)) { + /* + * This is necessary since update_wilds_cpumask() + * could have missed the 'task', if it's parent is + * the last one on the iteratoration list, like: + * + * 1. 'task' dup old dyn_allowed from parent + * 2. update_wilds_cpumask() begin + * 3. new dyn_allowed applied to parent + * 4. update_wilds_cpumask() end + * 5. 'task' add into iteratoration list + * + * Fix this by redup current's allowed here if changed. + */ + if (!cpumask_equal(&task->cpus_allowed, ¤t->cpus_allowed)) + set_cpus_allowed_ptr(task, ¤t->cpus_allowed); return; + } set_cpus_allowed_ptr(task, ¤t->cpus_allowed); task->mems_allowed = current->mems_allowed; diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index e680218..0934cf5 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -22,11 +22,36 @@ int housekeeping_any_cpu(enum hk_flags flags) } EXPORT_SYMBOL_GPL(housekeeping_any_cpu); +#ifdef CONFIG_CGROUP_SCHED +/* + * dyn_allowed -- allowed CPUs for wild tasks. + * + * dyn_isolated -- isolated CPUs for wild tasks. + * + * dyn_possible -- possible CPUs for dynamical isolation. + */ +static cpumask_var_t dyn_allowed; +static cpumask_var_t dyn_isolated; +static cpumask_var_t dyn_possible; + +static bool dyn_isolcpus_ready; + +DEFINE_STATIC_KEY_FALSE(dyn_isolcpus_enabled); +EXPORT_SYMBOL_GPL(dyn_isolcpus_enabled); +#endif + const struct cpumask *housekeeping_cpumask(enum hk_flags flags) { +#ifdef CONFIG_CGROUP_SCHED + if (static_branch_unlikely(&dyn_isolcpus_enabled)) + if (flags & HK_FLAG_DOMAIN) + return dyn_allowed; +#endif + if (static_branch_unlikely(&housekeeping_overriden)) if (housekeeping_flags & flags) return housekeeping_mask; + return cpu_possible_mask; } EXPORT_SYMBOL_GPL(housekeeping_cpumask); @@ -41,6 +66,12 @@ void housekeeping_affine(struct task_struct *t, enum hk_flags flags) bool housekeeping_test_cpu(int cpu, enum hk_flags flags) { +#ifdef CONFIG_CGROUP_SCHED + if (static_branch_unlikely(&dyn_isolcpus_enabled)) + if (flags & HK_FLAG_DOMAIN) + return cpumask_test_cpu(cpu, dyn_allowed); +#endif + if (static_branch_unlikely(&housekeeping_overriden)) if (housekeeping_flags & flags) return cpumask_test_cpu(cpu, housekeeping_mask); @@ -48,8 +79,28 @@ bool housekeeping_test_cpu(int cpu, enum hk_flags flags) } EXPORT_SYMBOL_GPL(housekeeping_test_cpu); +#ifdef CONFIG_CGROUP_SCHED +static inline void free_dyn_masks(void) +{ + free_cpumask_var(dyn_allowed); + free_cpumask_var(dyn_isolated); + free_cpumask_var(dyn_possible); +} +#endif + void __init housekeeping_init(void) { +#ifdef CONFIG_CGROUP_SCHED + if (zalloc_cpumask_var(&dyn_allowed, GFP_KERNEL) && + zalloc_cpumask_var(&dyn_isolated, GFP_KERNEL) && + zalloc_cpumask_var(&dyn_possible, GFP_KERNEL)) { + cpumask_copy(dyn_allowed, cpu_possible_mask); + cpumask_copy(dyn_possible, cpu_possible_mask); + dyn_isolcpus_ready = true; + } else + free_dyn_masks(); +#endif + if (!housekeeping_flags) return; @@ -60,6 +111,12 @@ void __init housekeeping_init(void) /* We need at least one CPU to handle housekeeping work */ WARN_ON_ONCE(cpumask_empty(housekeeping_mask)); +#ifdef CONFIG_CGROUP_SCHED + if (housekeeping_flags & HK_FLAG_DOMAIN) { + cpumask_copy(dyn_allowed, housekeeping_mask); + cpumask_copy(dyn_possible, housekeeping_mask); + } +#endif } static int __init housekeeping_setup(char *str, enum hk_flags flags) @@ -151,3 +208,135 @@ static int __init housekeeping_isolcpus_setup(char *str) return housekeeping_setup(str, flags); } __setup("isolcpus=", housekeeping_isolcpus_setup); + +#ifdef CONFIG_CGROUP_SCHED +static int dyn_isolcpus_show(struct seq_file *s, void *p) +{ + seq_printf(s, "%*pbl\n", cpumask_pr_args(dyn_isolated)); + + return 0; +} + +static int dyn_isolcpus_open(struct inode *inode, struct file *file) +{ + return single_open(file, dyn_isolcpus_show, NULL); +} + +void wilds_cpus_allowed(struct cpumask *pmask) +{ + if (static_branch_unlikely(&dyn_isolcpus_enabled)) + cpumask_and(pmask, pmask, dyn_allowed); +} + +void update_wilds_cpumask(cpumask_var_t new_allowed, cpumask_var_t old_allowed) +{ + struct css_task_iter it; + struct task_struct *task; + struct task_group *tg = &root_task_group; + + css_task_iter_start(&tg->css, 0, &it); + while ((task = css_task_iter_next(&it))) { + if (task->flags & PF_KTHREAD) + continue; + + if (!cpumask_equal(&task->cpus_allowed, old_allowed)) + continue; + + set_cpus_allowed_ptr(task, new_allowed); + } + css_task_iter_end(&it); +} + +static DEFINE_MUTEX(dyn_isolcpus_mutex); + +static ssize_t write_dyn_isolcpus(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + int ret = count; + cpumask_var_t isolated; + cpumask_var_t new_allowed; + cpumask_var_t old_allowed; + + mutex_lock(&dyn_isolcpus_mutex); + + if (!zalloc_cpumask_var(&isolated, GFP_KERNEL)) { + ret = -ENOMEM; + goto out; + } + + if (!zalloc_cpumask_var(&new_allowed, GFP_KERNEL)) { + ret = -ENOMEM; + goto free_isolated; + } + + if (!zalloc_cpumask_var(&old_allowed, GFP_KERNEL)) { + ret = -ENOMEM; + goto free_new_allowed; + } + + if (cpumask_parselist_user(buf, count, isolated)) { + ret = -EINVAL; + goto free_all; + } + + if (!cpumask_subset(isolated, dyn_possible)) { + ret = -EINVAL; + goto free_all; + } + + /* At least reserve one for wild tasks to run */ + cpumask_andnot(new_allowed, dyn_possible, isolated); + if (!cpumask_intersects(new_allowed, cpu_online_mask)) { + ret = -EINVAL; + goto free_all; + } + + cpumask_copy(old_allowed, dyn_allowed); + cpumask_copy(dyn_allowed, new_allowed); + cpumask_copy(dyn_isolated, isolated); + + if (cpumask_empty(dyn_isolated)) + static_branch_disable(&dyn_isolcpus_enabled); + else + static_branch_enable(&dyn_isolcpus_enabled); + + update_wilds_cpumask(new_allowed, old_allowed); + + rebuild_sched_domains(); + workqueue_set_unbound_cpumask(new_allowed); + +free_all: + free_cpumask_var(old_allowed); +free_new_allowed: + free_cpumask_var(new_allowed); +free_isolated: + free_cpumask_var(isolated); +out: + mutex_unlock(&dyn_isolcpus_mutex); + + return ret; +} + +static const struct file_operations proc_dyn_isolcpus_operations = { + .open = dyn_isolcpus_open, + .read = seq_read, + .write = write_dyn_isolcpus, + .llseek = noop_llseek, +}; + +static int __init dyn_isolcpus_init(void) +{ + if (dyn_isolcpus_ready && + !proc_create("dyn_isolcpus", 0200, NULL, + &proc_dyn_isolcpus_operations)) { + dyn_isolcpus_ready = false; + free_dyn_masks(); + } + + if (!dyn_isolcpus_ready) + pr_err("Initialize Dynamical Isolation Failed\n"); + + return 0; +} +early_initcall(dyn_isolcpus_init); +#endif -- 1.8.3.1