From 51b3509e23bd5127097c98cbf65e561f83c8d5a0 Mon Sep 17 00:00:00 2001 From: Yi Tao Date: Tue, 27 Oct 2020 11:25:06 +0800 Subject: [PATCH 2535/2944] alinux: sched: use cache when creating task_group to #31531504 Some members of task_group such as cfs_rq and rt_rq are related to how many cpus the machine has. When creating task_group, it costs lots of time to allocate memory for each cpu core in a loop. To reduce time we record the address of memory and reuse it when allcating instead of freeing directly. performance test: env: Intel(R) Xeon(R) CPU E5-2682 v4 @ 2.50GHz method: get system time at the beginning and end of function cpu_cgroup_css_alloc() then output the time. create cgroup under cpu subsystem in batch and calculate average time. avg time of cpu_cgroup_css_alloc() without cache: 121758ns with cache: 48121.8ns Signed-off-by: Yi Tao Suggested-by: Shanpei Chen Acked-by: Michael Wang --- kernel/sched/fair.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++----- kernel/sched/rt.c | 89 ++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 161 insertions(+), 16 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c30a005..3d38061 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9984,21 +9984,85 @@ static void task_change_group_fair(struct task_struct *p, int type) } } -void free_fair_sched_group(struct task_group *tg) +static void fair_sched_clean_up(void **ptr) { + struct cfs_rq **cfs_rq = ptr[0]; + struct sched_entity **se = ptr[1]; int i; - destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); + for_each_possible_cpu(i) { + memset(cfs_rq[i], 0, sizeof(struct cfs_rq)); + memset(se[i], 0, sizeof(struct sched_entity)); + init_cfs_rq(cfs_rq[i]); + init_entity_runnable_average(se[i]); + } + +} + +static bool fair_sched_check_integrity(struct task_group *tg) +{ + int i; + + if (!tg->cfs_rq || !tg->se) + return false; + + for_each_possible_cpu(i) + if (!tg->cfs_rq[i] || !tg->se[i]) + return false; + + return true; +} + +static void fair_sched_cache_init(struct task_group *tg, struct task_group *parent) +{ + int i; + + tg->shares = NICE_0_LOAD; + + init_cfs_bandwidth(tg_cfs_bandwidth(tg)); + + for_each_possible_cpu(i) + init_tg_cfs_entry(tg, tg->cfs_rq[i], tg->se[i], i, parent->se[i]); +} + +static void __free_fair_sched_group(void **ptr) +{ + struct cfs_rq **cfs_rq = ptr[0]; + struct sched_entity **se = ptr[1]; + int i; for_each_possible_cpu(i) { - if (tg->cfs_rq) - kfree(tg->cfs_rq[i]); - if (tg->se) - kfree(tg->se[i]); + if (cfs_rq) + kfree(cfs_rq[i]); + if (se) + kfree(se[i]); + } + kfree(cfs_rq); + kfree(se); +} + +CACHE_HEADER(fair_sched_cache_header, DEFAULT_CACHE_SIZE, + fair_sched_clean_up, __free_fair_sched_group); + +void free_fair_sched_group(struct task_group *tg) +{ + void *tmp[2]; + + destroy_cfs_bandwidth(tg_cfs_bandwidth(tg)); + + tmp[0] = (void *)tg->cfs_rq; + tmp[1] = (void *)tg->se; + + /* + * alloc_fair_sched_group() may fail and free a broken task_group, + * check integrity before putting to cache. + */ + if (fair_sched_check_integrity(tg)) { + if (put_to_cache(&fair_sched_cache_header, tmp, 2)) + return; } - kfree(tg->cfs_rq); - kfree(tg->se); + __free_fair_sched_group(tmp); } int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) @@ -10006,6 +10070,14 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) struct sched_entity *se; struct cfs_rq *cfs_rq; int i; + void *tmp[2]; + + if (get_from_cache(&fair_sched_cache_header, tmp, 2)) { + tg->cfs_rq = tmp[0]; + tg->se = tmp[1]; + fair_sched_cache_init(tg, parent); + return 1; + } tg->cfs_rq = kcalloc(nr_cpu_ids, sizeof(cfs_rq), GFP_KERNEL); if (!tg->cfs_rq) diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 042fb33..3d1792a 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -134,22 +134,87 @@ static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) return rt_rq->rq; } -void free_rt_sched_group(struct task_group *tg) +static void rt_sched_clean_up(void **ptr) +{ + struct rt_rq **rt_rq = ptr[0]; + struct sched_rt_entity **rt_se = ptr[1]; + int i; + + for_each_possible_cpu(i) { + memset(rt_rq[i], 0, sizeof(struct rt_rq)); + memset(rt_se[i], 0, sizeof(struct sched_rt_entity)); + init_rt_rq(rt_rq[i]); + } +} + +static bool rt_sched_check_integrity(struct task_group *tg) +{ + int i; + + if (!tg->rt_rq || !tg->rt_se) + return false; + + for_each_possible_cpu(i) { + if (!tg->rt_rq[i] || !tg->rt_se[i]) + return false; + } + + return true; +} + +static void rt_sched_cache_init(struct task_group *tg, struct task_group *parent) { int i; + init_rt_bandwidth(&tg->rt_bandwidth, + ktime_to_ns(def_rt_bandwidth.rt_period), 0); + + for_each_possible_cpu(i) { + tg->rt_rq[i]->rt_runtime = tg->rt_bandwidth.rt_runtime; + init_tg_rt_entry(tg, tg->rt_rq[i], tg->rt_se[i], i, parent->rt_se[i]); + } +} + +static void __free_rt_sched_group(void **ptr) +{ + struct rt_rq **rt_rq = ptr[0]; + struct sched_rt_entity **rt_se = ptr[1]; + int i; + + for_each_possible_cpu(i) { + if (rt_rq) + kfree(rt_rq[i]); + if (rt_se) + kfree(rt_se[i]); + } + + kfree(rt_rq); + kfree(rt_se); +} + +CACHE_HEADER(rt_sched_cache_header, DEFAULT_CACHE_SIZE, + rt_sched_clean_up, __free_rt_sched_group); + +void free_rt_sched_group(struct task_group *tg) +{ + void *tmp[2]; + if (tg->rt_se) destroy_rt_bandwidth(&tg->rt_bandwidth); - for_each_possible_cpu(i) { - if (tg->rt_rq) - kfree(tg->rt_rq[i]); - if (tg->rt_se) - kfree(tg->rt_se[i]); + tmp[0] = (void *)tg->rt_rq; + tmp[1] = (void *)tg->rt_se; + + /* + * alloc_rt_sched_group() may fail and free a broken task_group, + * check integrity before putting to cache. + */ + if (rt_sched_check_integrity(tg)) { + if (put_to_cache(&rt_sched_cache_header, tmp, 2)) + return; } - kfree(tg->rt_rq); - kfree(tg->rt_se); + __free_rt_sched_group(tmp); } void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq, @@ -184,6 +249,14 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) struct rt_rq *rt_rq; struct sched_rt_entity *rt_se; int i; + void *tmp[2]; + + if (get_from_cache(&rt_sched_cache_header, tmp, 2)) { + tg->rt_rq = tmp[0]; + tg->rt_se = tmp[1]; + rt_sched_cache_init(tg, parent); + return 1; + } tg->rt_rq = kcalloc(nr_cpu_ids, sizeof(rt_rq), GFP_KERNEL); if (!tg->rt_rq) -- 1.8.3.1