From dc6cf8e45fc41e6bd7b13761c8f758ffac72f856 Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Tue, 27 Oct 2020 11:39:46 +0800 Subject: [PATCH 2536/2944] alinux: cgroup: use cache for kernfs node to #31531504 Under default hierarchy, which used only in v2 mode, files of different cgroups may be different because of subtree control. It is certain that Except default hierarchy, files of cgroup under same hierarchy are same. Therefore, to reduce time for creating cgroup, we can avoid to create these files and delete them repeatly. What we do is that, when removing a cgroup, moving the whole directory with files in it to an invisible place by function kernfs_rename(), and if a new cgroup is to be created, moving a directory back rather than creating a new directory and files in it. performance test: env: Intel(R) Xeon(R) CPU E5-2682 v4 @ 2.50GHz method: output time of creating/moving kernfs node, creating cgroup under each subsystem in batch and calculate average time. subsystem avg_time_without_cache avg_time_witch_cache cpu 9964.7ns 1353.9ns cpuacct 13777.2ns 1312.53ns memory 33946.6ns 1399.59ns Signed-off-by: Yi Tao Suggested-by: Shanpei Chen Acked-by: Michael Wang Acked-by: Xunlei Pang --- include/linux/cgroup-defs.h | 14 ++++++ kernel/cgroup/cgroup.c | 114 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 114 insertions(+), 14 deletions(-) diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index aaeb2de..5742889 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -463,6 +463,15 @@ struct cgroup { struct cgroup_file events_file; /* handle for "cgroup.events" */ /* + * When creating a new cgroup, kn may be allocated from cache with + * kernfs_nodes already exist for cgroup control files. If true, there + * is no need to call css_populate_dir to generate cgroup basic control + * files and subsys control files. + */ + bool kn_from_cache; + bool kn_to_cache; + + /* * The bitmask of subsystems enabled on the child cgroups. * ->subtree_control is the one configured through * "cgroup.subtree_control" while ->child_ss_mask is the effective @@ -547,6 +556,11 @@ struct cgroup { struct cgroup_root { struct kernfs_root *kf_root; + /* Dummy parent for kernfs_node */ + struct kernfs_root *orphanage; + + struct cache_header orphan_header; + /* The bitmask of subsystems attached to this hierarchy */ unsigned int subsys_mask; diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 2c47345..3c3ef09 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -220,7 +220,7 @@ static void css_task_iter_skip(struct css_task_iter *it, static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, struct cgroup_subsys *ss); static void css_release(struct percpu_ref *ref); -static void kill_css(struct cgroup_subsys_state *css); +static void kill_css(struct cgroup_subsys_state *css, bool kn_to_cache); static int cgroup_addrm_files(struct cgroup_subsys_state *css, struct cgroup *cgrp, struct cftype cfts[], bool is_add); @@ -1928,6 +1928,16 @@ void init_cgroup_root(struct cgroup_root *root, struct cgroup_sb_opts *opts) set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); } +static void cgroup_clean_cache(void **ptr) {} + +static void cgroup_release_cache(void **ptr) +{ + struct kernfs_node *kn = *ptr; + + kernfs_remove(kn); +} + + int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask, int ref_flags) { LIST_HEAD(tmp_links); @@ -1971,6 +1981,11 @@ int cgroup_setup_root(struct cgroup_root *root, u16 ss_mask, int ref_flags) KERNFS_ROOT_CREATE_DEACTIVATED | KERNFS_ROOT_SUPPORT_EXPORTOP, root_cgrp); + + root->orphanage = kernfs_create_root(NULL, 0, NULL); + init_cache_header(&root->orphan_header, DEFAULT_CACHE_SIZE, + cgroup_clean_cache, cgroup_release_cache); + if (IS_ERR(root->kf_root)) { ret = PTR_ERR(root->kf_root); goto exit_root_id; @@ -2997,9 +3012,11 @@ static int cgroup_apply_control_enable(struct cgroup *cgrp) } if (css_visible(css)) { - ret = css_populate_dir(css); - if (ret) - return ret; + if (!cgrp->kn_from_cache) { + ret = css_populate_dir(css); + if (ret) + return ret; + } } } } @@ -3038,7 +3055,8 @@ static void cgroup_apply_control_disable(struct cgroup *cgrp) if (css->parent && !(cgroup_ss_mask(dsct) & (1 << ss->id))) { - kill_css(css); + /* remove files directly in v2 mode */ + kill_css(css, false); } else if (!css_visible(css)) { css_clear_dir(css); if (ss->css_reset) @@ -4833,7 +4851,6 @@ static void css_free_rwork_fn(struct work_struct *work) /* cgroup free path */ atomic_dec(&cgrp->root->nr_cgrps); cgroup1_pidlist_destroy_all(cgrp); - cancel_work_sync(&cgrp->release_agent_work); if (cgroup_parent(cgrp)) { /* @@ -4847,6 +4864,9 @@ static void css_free_rwork_fn(struct work_struct *work) psi_cgroup_free(cgrp); if (cgroup_on_dfl(cgrp)) cgroup_rstat_exit(cgrp); + if (cgrp->kn_to_cache) + put_to_cache(&cgrp->root->orphan_header, + (void **)&cgrp->kn, 1); kfree(cgrp); } else { /* @@ -5188,6 +5208,27 @@ static bool cgroup_check_hierarchy_limits(struct cgroup *parent) return ret; } +static struct kernfs_node * +cgroup_create_kernfs(struct kernfs_node *parent, const char *name, umode_t mode, + struct cgroup *cgrp) +{ + struct kernfs_node *kn; + struct cgroup_root *root = cgrp->root; + + if (get_from_cache(&root->orphan_header, (void **)&kn, 1)) { + kernfs_rename(kn, parent, name); + kn->priv = (void *)cgrp; + cgrp->kn_from_cache = true; + return kn; + } + + kn = kernfs_create_dir(parent, name, mode, cgrp); + cgrp->kn_from_cache = false; + + return kn; + +} + int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode) { struct cgroup *parent, *cgrp; @@ -5214,7 +5255,7 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode) } /* create the directory */ - kn = kernfs_create_dir(parent->kn, name, mode, cgrp); + kn = cgroup_create_kernfs(parent->kn, name, mode, cgrp); if (IS_ERR(kn)) { ret = PTR_ERR(kn); goto out_destroy; @@ -5231,7 +5272,8 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode) if (ret) goto out_destroy; - ret = css_populate_dir(&cgrp->self); + if (!cgrp->kn_from_cache) + ret = css_populate_dir(&cgrp->self); if (ret) goto out_destroy; @@ -5297,7 +5339,7 @@ static void css_killed_ref_fn(struct percpu_ref *ref) * asynchronously once css_tryget_online() is guaranteed to fail and when * the reference count reaches zero, @css will be released. */ -static void kill_css(struct cgroup_subsys_state *css) +static void kill_css(struct cgroup_subsys_state *css, bool kn_to_cache) { lockdep_assert_held(&cgroup_mutex); @@ -5310,7 +5352,10 @@ static void kill_css(struct cgroup_subsys_state *css) * This must happen before css is disassociated with its cgroup. * See seq_css() for details. */ - css_clear_dir(css); + if (!kn_to_cache) + css_clear_dir(css); + else + css->flags &= ~CSS_VISIBLE; /* * Killing would put the base ref, but we need to keep it alive @@ -5331,6 +5376,26 @@ static void kill_css(struct cgroup_subsys_state *css) percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn); } +/* + * Moving kernfs_node to dummy parent require an unique name to guarantee + * success + */ +static int unique_id; + +static bool kn_to_cache_check(struct cgroup *cgrp) +{ + if (cgroup_on_dfl(cgrp)) + return false; + + if (IS_ERR(cgrp->kn) || !cgrp->kn) + return false; + + if (!cache_not_full(&cgrp->root->orphan_header)) + return false; + + return true; +} + /** * cgroup_destroy_locked - the first stage of cgroup destruction * @cgrp: cgroup to be destroyed @@ -5361,6 +5426,7 @@ static int cgroup_destroy_locked(struct cgroup *cgrp) struct cgroup *tcgrp, *parent = cgroup_parent(cgrp); struct cgroup_subsys_state *css; struct cgrp_cset_link *link; + char unique_name[16]; int ssid; lockdep_assert_held(&cgroup_mutex); @@ -5393,13 +5459,33 @@ static int cgroup_destroy_locked(struct cgroup *cgrp) link->cset->dead = true; spin_unlock_irq(&css_set_lock); + /* try to put kernfs_node to cache */ + if (kn_to_cache_check(cgrp)) + cgrp->kn_to_cache = true; + else + cgrp->kn_to_cache = false; + /* initiate massacre of all css's */ for_each_css(css, ssid, cgrp) - kill_css(css); + kill_css(css, cgrp->kn_to_cache); + + /* + * release_agent_work need path of kn, cancel it before moving kn. work + * may be running and waiting for cgroup_mutex. to avoid dead lock, + * release cgroup_mutex and acquire it later. + */ + mutex_unlock(&cgroup_mutex); + cancel_work_sync(&cgrp->release_agent_work); + mutex_lock(&cgroup_mutex); - /* clear and remove @cgrp dir, @cgrp has an extra ref on its kn */ - css_clear_dir(&cgrp->self); - kernfs_remove(cgrp->kn); + if (!cgrp->kn_to_cache) { + css_clear_dir(&cgrp->self); + kernfs_remove(cgrp->kn); + } else { + cgrp->self.flags &= ~CSS_VISIBLE; + snprintf(unique_name, sizeof(unique_name), "%d", unique_id++); + kernfs_rename(cgrp->kn, cgrp->root->orphanage->kn, unique_name); + } if (parent && cgroup_is_threaded(cgrp)) parent->nr_threaded_children--; -- 1.8.3.1