From 1f3a50b5d334b849c337cfe40698311e1436dd7b Mon Sep 17 00:00:00 2001 From: Ning Zhang Date: Tue, 11 May 2021 17:00:49 +0800 Subject: [PATCH 2756/2944] mm, thp: introduce thp reclaim threshold fix #34670772 In this patch, we add memory.thp_reclaim_ctrl for each memory cgroup to control thp reclaim. And we add the first controller "threshold" to set the reclaim threshold. The default value is 16, which means if a huge page contains over 16 zero subpages (estimated), the huge page can be split and the zero subpages can be reclaimed when the thp zero subpages reclaim is working. You can change this value by: echo "threshold $v" > /sys/fs/cgroup/memory/{memcg}/thp_reclaim_ctrl Signed-off-by: Ning Zhang Reviewed-by: Gang Deng Reviewed-by: Xu Yu --- Documentation/admin-guide/mm/transhuge.rst | 19 ++++++++-- include/linux/memcontrol.h | 3 ++ mm/huge_memory.c | 23 +++++++----- mm/memcontrol.c | 59 ++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 12 deletions(-) diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index af16a77..87427e4 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -467,11 +467,24 @@ disable If the mode is reclaim or swap, the new huge page will be add to a reclaim queue in mem_cgroup, and the queue would be scanned when memory reclaiming. -The huge page which contains at least 1/32 zero pages would be split(estimate -it by checking some discrete unsigned long values). - The queue stat can be checked like this:: cat /sys/fs/cgroup/memory/{memcg}/memory.thp_reclaim_stat Now, it only contains the queue length of each node. + +We also add a controller interface to set configs for thp reclaim:: + + /sys/fs/cgroup/memory/{memcg}/memory.thp_reclaim_ctrl + +threshold + means the huge page which contains at least threshold zero pages would + be split (estimate it by checking some discrete unsigned long values). + The default value of threshold is 16, and will inherit from it's parent. + The range of this value is (0, HPAGE_PMD_NR], which means the value must + be less than or equal to HPAGE_PMD_NR (512 in x86), and be greater than 0. + We can set reclaim threshold to be 8 by this:: + + echo "threshold 8" > memory.thp_reclaim_ctrl + +Only one of the configs mentioned above can be set at a time. diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 09ef8b2..9eaab10 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -242,6 +242,8 @@ struct memcg_padding { #define THP_RECLAIM_ZSR 2 #define THP_RECLAIM_MEMCG 3 /* For global configure*/ + +#define THP_RECLAIM_THRESHOLD_DEFAULT 16 #endif /* @@ -402,6 +404,7 @@ struct mem_cgroup { #ifdef CONFIG_TRANSPARENT_HUGEPAGE int thp_reclaim; + int thp_reclaim_threshold; #endif ALI_HOTFIX_RESERVE(1) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 7ee252c..7611bde 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3211,7 +3211,7 @@ static inline bool is_zero_page(struct page *page) * We'll split the huge page iff it contains at least 1/32 zeros, * estimate it by checking some discrete unsigned long values. */ -static bool hugepage_estimate_zero(struct page *page) +static bool hugepage_estimate_zero(struct page *page, int threshold) { unsigned int i, maybe_zero_pages = 0, offset = 0; void *addr; @@ -3220,12 +3220,16 @@ static bool hugepage_estimate_zero(struct page *page) addr = kmap(page); if (unlikely((offset + 1) * BYTES_PER_LONG > PAGE_SIZE)) offset = 0; - if (*(const unsigned long *)(addr + offset) == 0UL) - maybe_zero_pages++; + if (*(const unsigned long *)(addr + offset) == 0UL) { + if (++maybe_zero_pages == threshold) { + kunmap(page); + return true; + } + } kunmap(page); } - return (maybe_zero_pages << 5) >= HPAGE_PMD_NR; + return false; } static bool replace_zero_pte(struct page *page, struct vm_area_struct *vma, @@ -3429,7 +3433,7 @@ static unsigned long hugepage_reclaim_count(struct shrinker *shrink, * from lru list and locked. */ static struct page *get_reclaim_hugepage(struct hugepage_reclaim *hr_queue, - bool *empty, bool disable) + int threshold, bool *empty, bool disable) { struct page *page = NULL, *reclaim_page = NULL; unsigned long flags; @@ -3450,7 +3454,7 @@ static struct page *get_reclaim_hugepage(struct hugepage_reclaim *hr_queue, spin_unlock_irqrestore(&hr_queue->reclaim_queue_lock, flags); if (hugepage_can_reclaim(page) && - hugepage_estimate_zero(page) && + hugepage_estimate_zero(page, threshold) && !isolate_lru_page(page)) { __mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON, @@ -3487,7 +3491,7 @@ static unsigned long hugepage_reclaim_scan(struct shrinker *shrink, struct lruvec *lruvec = mem_cgroup_lruvec(sc->memcg, pgdat); struct hugepage_reclaim *hr_queue = NULL; struct page *page; - int thp_reclaim; + int thp_reclaim, threshold; unsigned long nr_to_scan, nr_scanned; unsigned long flags; bool empty; @@ -3507,6 +3511,7 @@ static unsigned long hugepage_reclaim_scan(struct shrinker *shrink, if (!hr_queue) return SHRINK_STOP; + threshold = READ_ONCE(sc->memcg->thp_reclaim_threshold); thp_reclaim = READ_ONCE(global_thp_reclaim); thp_reclaim = (thp_reclaim != THP_RECLAIM_MEMCG) ? thp_reclaim : READ_ONCE(sc->memcg->thp_reclaim); @@ -3518,8 +3523,8 @@ static unsigned long hugepage_reclaim_scan(struct shrinker *shrink, LIST_HEAD(keep_list); cond_resched(); - page = get_reclaim_hugepage(hr_queue, &empty, - thp_reclaim == THP_RECLAIM_DISABLE); + page = get_reclaim_hugepage(hr_queue, threshold, &empty, + thp_reclaim == THP_RECLAIM_DISABLE); if (empty) break; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 992c02c..8381a6c 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -72,6 +72,7 @@ #include #include "slab.h" #include +#include #include @@ -5835,6 +5836,57 @@ static int memcg_thp_reclaim_stat_show(struct seq_file *m, void *v) return 0; } + +static inline char *strsep_s(char **s, const char *ct) +{ + char *p; + + while ((p = strsep(s, ct))) { + if (*p) + return p; + } + return NULL; +} + +static int memcg_thp_reclaim_ctrl_show(struct seq_file *m, void *v) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); + int thp_reclaim_threshold = READ_ONCE(memcg->thp_reclaim_threshold); + + seq_printf(m, "threshold\t%d\n", thp_reclaim_threshold); + + return 0; +} + +static ssize_t memcg_thp_reclaim_ctrl_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); + int ret, threshold; + char *key, *value; + + key = strsep_s(&buf, " \t\n"); + if (!key) + return -EINVAL; + + if (!strcmp(key, "threshold")) { + value = strsep_s(&buf, " \t\n"); + if (!value) + return -EINVAL; + + ret = kstrtouint(value, 0, &threshold); + if (ret) + return ret; + + if (threshold > HPAGE_PMD_NR || threshold < 1) + return -EINVAL; + + xchg(&memcg->thp_reclaim_threshold, threshold); + } else + return -EINVAL; + + return nbytes; +} #endif static struct cftype mem_cgroup_legacy_files[] = { @@ -6113,6 +6165,11 @@ static int memcg_thp_reclaim_stat_show(struct seq_file *m, void *v) .name = "thp_reclaim_stat", .seq_show = memcg_thp_reclaim_stat_show, }, + { + .name = "thp_reclaim_ctrl", + .seq_show = memcg_thp_reclaim_ctrl_show, + .write = memcg_thp_reclaim_ctrl_write, + }, #endif { }, /* terminate */ }; @@ -6302,6 +6359,7 @@ static bool __mem_cgroup_init(struct mem_cgroup *memcg) #endif #ifdef CONFIG_TRANSPARENT_HUGEPAGE memcg->thp_reclaim = THP_RECLAIM_DISABLE; + memcg->thp_reclaim_threshold = THP_RECLAIM_THRESHOLD_DEFAULT; #endif kidled_memcg_init(memcg); idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); @@ -6469,6 +6527,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void) : 50; #ifdef CONFIG_TRANSPARENT_HUGEPAGE memcg->thp_reclaim = parent->thp_reclaim; + memcg->thp_reclaim_threshold = parent->thp_reclaim_threshold; #endif kidled_memcg_inherit_parent_buckets(parent, memcg); } -- 1.8.3.1