From a97cbf90404c681f48ced9b0ea0bb65a380ccfd8 Mon Sep 17 00:00:00 2001 From: Huaixin Chang Date: Tue, 5 May 2020 18:29:41 +0800 Subject: [PATCH 2116/2944] alinux: sched/fair: Introduce init buffer into CFS burst to #27440320 For CFS burst, cpu.cfs_burst_us is used to denote how much unused cputime a group can accumulate. However, users may want a much bigger buffer at first, and a smaller buffer at runtime. Thus, cpu.cfs_init_buffer_us is introduced to denote how much init cputime a group is granted at the very beginning. A group can consume cputime from init buffer without being throttled. When cputime from init buffer drops below cpu.cfs_burst_us, normal behaviour of CPU burst is restored. Also init buffer has no effect on its group tasks any more. Acked-by: Shanpei Chen Signed-off-by: Huaixin Chang --- kernel/sched/core.c | 71 +++++++++++++++++++++++++++++++++++++++++++++------- kernel/sched/fair.c | 4 ++- kernel/sched/sched.h | 2 ++ 3 files changed, 67 insertions(+), 10 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 9203019..1e179fa 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6528,7 +6528,7 @@ static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota, - u64 burst) + u64 burst, u64 init_buffer) { int i, ret = 0, runtime_enabled, runtime_was_enabled; struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; @@ -6562,7 +6562,7 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota, /* * Bound burst to defend burst against overflow during bandwidth shift. */ - if (burst > max_cfs_runtime) + if (burst > max_cfs_runtime || init_buffer > max_cfs_runtime) return -EINVAL; if (quota == RUNTIME_INF) @@ -6592,6 +6592,7 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota, cfs_b->quota = quota; cfs_b->burst = burst; cfs_b->buffer = buffer; + cfs_b->init_buffer = init_buffer; cfs_b->max_overrun = DIV_ROUND_UP_ULL(max_cfs_runtime, quota); cfs_b->runtime = cfs_b->quota; @@ -6608,6 +6609,8 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota, cfs_b->runtime = min(max_cfs_runtime, cfs_b->runtime); } + cfs_b->runtime = max(cfs_b->runtime, init_buffer); + cfs_b->current_buffer = max(cfs_b->buffer, init_buffer); cfs_b->previous_runtime = cfs_b->runtime; /* Restart the period timer (if active) to handle new period expiry: */ @@ -6640,10 +6643,11 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota, int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) { - u64 quota, period, burst; + u64 quota, period, burst, init_buffer; period = ktime_to_ns(tg->cfs_bandwidth.period); burst = tg->cfs_bandwidth.burst; + init_buffer = tg->cfs_bandwidth.init_buffer; if (cfs_quota_us < 0) quota = RUNTIME_INF; else if ((u64)cfs_quota_us <= U64_MAX / NSEC_PER_USEC) @@ -6651,7 +6655,7 @@ int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) else return -EINVAL; - return tg_set_cfs_bandwidth(tg, period, quota, burst); + return tg_set_cfs_bandwidth(tg, period, quota, burst, init_buffer); } long tg_get_cfs_quota(struct task_group *tg) @@ -6669,7 +6673,7 @@ long tg_get_cfs_quota(struct task_group *tg) int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) { - u64 quota, period, burst; + u64 quota, period, burst, init_buffer; if ((u64)cfs_period_us > U64_MAX / NSEC_PER_USEC) return -EINVAL; @@ -6677,8 +6681,9 @@ int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) period = (u64)cfs_period_us * NSEC_PER_USEC; quota = tg->cfs_bandwidth.quota; burst = tg->cfs_bandwidth.burst; + init_buffer = tg->cfs_bandwidth.init_buffer; - return tg_set_cfs_bandwidth(tg, period, quota, burst); + return tg_set_cfs_bandwidth(tg, period, quota, burst, init_buffer); } long tg_get_cfs_period(struct task_group *tg) @@ -6693,10 +6698,11 @@ long tg_get_cfs_period(struct task_group *tg) int tg_set_cfs_burst(struct task_group *tg, long cfs_burst_us) { - u64 quota, period, burst; + u64 quota, period, burst, init_buffer; period = ktime_to_ns(tg->cfs_bandwidth.period); quota = tg->cfs_bandwidth.quota; + init_buffer = tg->cfs_bandwidth.init_buffer; if (cfs_burst_us < 0) burst = RUNTIME_INF; else if ((u64)cfs_burst_us <= U64_MAX / NSEC_PER_USEC) @@ -6704,7 +6710,7 @@ int tg_set_cfs_burst(struct task_group *tg, long cfs_burst_us) else return -EINVAL; - return tg_set_cfs_bandwidth(tg, period, quota, burst); + return tg_set_cfs_bandwidth(tg, period, quota, burst, init_buffer); } long tg_get_cfs_burst(struct task_group *tg) @@ -6720,6 +6726,36 @@ long tg_get_cfs_burst(struct task_group *tg) return burst_us; } +int tg_set_cfs_init_buffer(struct task_group *tg, long cfs_init_buffer_us) +{ + u64 quota, period, burst, init_buffer; + + period = ktime_to_ns(tg->cfs_bandwidth.period); + quota = tg->cfs_bandwidth.quota; + burst = tg->cfs_bandwidth.burst; + if (cfs_init_buffer_us < 0) + init_buffer = RUNTIME_INF; + else if ((u64)cfs_init_buffer_us <= U64_MAX / NSEC_PER_USEC) + init_buffer = (u64)cfs_init_buffer_us * NSEC_PER_USEC; + else + return -EINVAL; + + return tg_set_cfs_bandwidth(tg, period, quota, burst, init_buffer); +} + +long tg_get_cfs_init_buffer(struct task_group *tg) +{ + u64 init_buffer_us; + + if (tg->cfs_bandwidth.init_buffer == RUNTIME_INF) + return -1; + + init_buffer_us = tg->cfs_bandwidth.init_buffer; + do_div(init_buffer_us, NSEC_PER_USEC); + + return init_buffer_us; +} + static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, struct cftype *cft) { @@ -6756,6 +6792,18 @@ static int cpu_cfs_burst_write_s64(struct cgroup_subsys_state *css, return tg_set_cfs_burst(css_tg(css), cfs_burst_us); } +static s64 cpu_cfs_init_buffer_read_s64(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return tg_get_cfs_init_buffer(css_tg(css)); +} + +static int cpu_cfs_init_buffer_write_s64(struct cgroup_subsys_state *css, + struct cftype *cftype, s64 cfs_init_buffer_us) +{ + return tg_set_cfs_init_buffer(css_tg(css), cfs_init_buffer_us); +} + struct cfs_schedulable_data { struct task_group *tg; u64 period, quota; @@ -6918,6 +6966,11 @@ static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, .write_s64 = cpu_cfs_burst_write_s64, }, { + .name = "cfs_init_buffer_us", + .read_s64 = cpu_cfs_init_buffer_read_s64, + .write_s64 = cpu_cfs_init_buffer_write_s64, + }, + { .name = "stat", .seq_show = cpu_cfs_stat_show, }, @@ -7073,7 +7126,7 @@ static ssize_t cpu_max_write(struct kernfs_open_file *of, ret = cpu_period_quota_parse(buf, &period, "a); if (!ret) - ret = tg_set_cfs_bandwidth(tg, period, quota, 0); + ret = tg_set_cfs_bandwidth(tg, period, quota, 0, 0); return ret ?: nbytes; } #endif diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 080d48d..6aebd8e1 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4393,10 +4393,11 @@ void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b, u64 overrun) cfs_b->nr_burst++; } + cfs_b->current_buffer = max(cfs_b->runtime, cfs_b->buffer); overrun = min(overrun, cfs_b->max_overrun); refill = cfs_b->quota * overrun; cfs_b->runtime += refill; - cfs_b->runtime = min(cfs_b->runtime, cfs_b->buffer); + cfs_b->runtime = min(cfs_b->runtime, cfs_b->current_buffer); cfs_b->previous_runtime = cfs_b->runtime; } @@ -5015,6 +5016,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) cfs_b->quota = RUNTIME_INF; cfs_b->period = ns_to_ktime(default_cfs_period()); cfs_b->burst = 0; + cfs_b->init_buffer = 0; cfs_b->buffer = RUNTIME_INF; INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f329aee..db6a76f 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -336,6 +336,8 @@ struct cfs_bandwidth { u64 quota; u64 runtime; u64 burst; + u64 init_buffer; + u64 current_buffer; u64 buffer; u64 max_overrun; u64 previous_runtime; -- 1.8.3.1