From 57456d1625ba9036968fa0be70a6036b88f2b2f4 Mon Sep 17 00:00:00 2001 From: Ning Zhang Date: Sat, 29 May 2021 21:41:06 +0800 Subject: [PATCH 2757/2944] mm, thp: introduce a controller to trigger thp reclaim fix #34670772 Add a new controller named "reclaim" for memory.thp_reclaim_ctrl to trigger thp reclaim immediately: echo "reclaim 1" > memory.thp_reclaim_ctrl echo "reclaim 2" > memory.thp_reclaim_ctrl "reclaim 1" means triggering reclaim only for current memcg. "reclaim 2" means triggering reclaim for current memcg and it's children memcgs. Signed-off-by: Ning Zhang Reviewed-by: Gang Deng Reviewed-by: Xu Yu --- Documentation/admin-guide/mm/transhuge.rst | 11 +++ include/linux/memcontrol.h | 1 + mm/huge_memory.c | 150 ++++++++++++++++++----------- mm/memcontrol.c | 28 +++++- 4 files changed, 135 insertions(+), 55 deletions(-) diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 87427e4..5333afa 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -487,4 +487,15 @@ threshold echo "threshold 8" > memory.thp_reclaim_ctrl +reclaim + triggers action immediately for the huge pages in the reclaim queue. + The action deponds on the thp reclaim config (reclaim, swap or disable, + disable means just remove the huge page from the queue). + This contronller has two value, 1 and 2. 1 means just reclaim the current + memcg, and 2 means reclaim the current memcg and all the children memcgs. + Like this:: + + echo "reclaim 1" > memory.thp_reclaim_ctrl + echo "reclaim 2" > memory.thp_reclaim_ctrl + Only one of the configs mentioned above can be set at a time. diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 9eaab10..eb602ae 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -925,6 +925,7 @@ void memcg_check_wmark_min_adj(struct task_struct *curr, extern int global_thp_reclaim; extern void set_hugepage_reclaim_shrinker_bit(struct mem_cgroup *memcg, int nid); +extern void reclaim_memcg_huge_pages(struct mem_cgroup *memcg); static inline struct list_head *hugepage_reclaim_list(struct page *page) { diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 7611bde..ae1b9c8 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -3483,17 +3483,108 @@ static struct page *get_reclaim_hugepage(struct hugepage_reclaim *hr_queue, } +static void reclaim_huge_page(struct lruvec *lruvec, struct page *page, int mode) +{ + struct pglist_data *pgdat = page_pgdat(page); + unsigned long flags; + LIST_HEAD(split_list); + LIST_HEAD(keep_list); + + switch (mode) { + case THP_RECLAIM_SWAP: + /* + * Putback the page to inactive lru list. + * Since there is no interface function to put the + * page to the tail of inactive lru list, we put it + * to the head of the list in order to less invasion. + */ + __ClearPageActive(page); + ClearPageReferenced(page); + unlock_page(page); + putback_lru_page(page); + mod_node_page_state(pgdat, NR_ISOLATED_ANON, + -HPAGE_PMD_NR); + break; + case THP_RECLAIM_ZSR: + /* + * Split the huge page and reclaim the zero subpages. + * And putback the non-zero subpages to the lru list. + */ + if (split_huge_page_to_list(page, &split_list)) { + unlock_page(page); + putback_lru_page(page); + mod_node_page_state(pgdat, NR_ISOLATED_ANON, + -HPAGE_PMD_NR); + break; + } + + unlock_page(page); + list_add_tail(&page->lru, &split_list); + reclaim_zero_subpages(&split_list, &keep_list); + + spin_lock_irqsave(&pgdat->lru_lock, flags); + putback_inactive_pages(lruvec, &keep_list); + spin_unlock_irqrestore(&pgdat->lru_lock, flags); + mod_node_page_state(pgdat, NR_ISOLATED_ANON, + -HPAGE_PMD_NR); + + mem_cgroup_uncharge_list(&keep_list); + free_unref_page_list(&keep_list); + break; + default: + WARN_ONCE(1, "To reclaim a huge page when thp reclaim is disable"); + break; + } +} + +static inline int get_reclaim_mode(struct mem_cgroup *memcg) +{ + int reclaim = READ_ONCE(global_thp_reclaim); + + return (reclaim != THP_RECLAIM_MEMCG) ? reclaim : + READ_ONCE(memcg->thp_reclaim); +} + +void reclaim_memcg_huge_pages(struct mem_cgroup *memcg) +{ + struct lruvec *lruvec; + struct hugepage_reclaim *hr_queue; + struct page *page; + int thp_reclaim, threshold, nid; + bool empty; + + threshold = READ_ONCE(memcg->thp_reclaim_threshold); + thp_reclaim = get_reclaim_mode(memcg); + + for_each_online_node(nid) { + lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); + hr_queue = &memcg->nodeinfo[nid]->hugepage_reclaim_queue; + while (1) { + cond_resched(); + page = get_reclaim_hugepage(hr_queue, threshold, + &empty, thp_reclaim == THP_RECLAIM_DISABLE); + + if (empty) + break; + + if (!page) + continue; + + reclaim_huge_page(lruvec, page, thp_reclaim); + } + + } +} + static unsigned long hugepage_reclaim_scan(struct shrinker *shrink, struct shrink_control *sc) { int nid = sc->nid; - struct pglist_data *pgdat = NODE_DATA(nid); - struct lruvec *lruvec = mem_cgroup_lruvec(sc->memcg, pgdat); + struct lruvec *lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(nid)); struct hugepage_reclaim *hr_queue = NULL; struct page *page; int thp_reclaim, threshold; unsigned long nr_to_scan, nr_scanned; - unsigned long flags; bool empty; #define MAX_SCAN_HUGEPAGES 32UL @@ -3512,16 +3603,11 @@ static unsigned long hugepage_reclaim_scan(struct shrinker *shrink, return SHRINK_STOP; threshold = READ_ONCE(sc->memcg->thp_reclaim_threshold); - thp_reclaim = READ_ONCE(global_thp_reclaim); - thp_reclaim = (thp_reclaim != THP_RECLAIM_MEMCG) ? thp_reclaim : - READ_ONCE(sc->memcg->thp_reclaim); + thp_reclaim = get_reclaim_mode(sc->memcg); nr_to_scan = min(sc->nr_to_scan, MAX_SCAN_HUGEPAGES); nr_scanned = 0; while (nr_to_scan) { - LIST_HEAD(split_list); - LIST_HEAD(keep_list); - cond_resched(); page = get_reclaim_hugepage(hr_queue, threshold, &empty, thp_reclaim == THP_RECLAIM_DISABLE); @@ -3534,51 +3620,7 @@ static unsigned long hugepage_reclaim_scan(struct shrinker *shrink, if (!page) continue; - switch (thp_reclaim) { - case THP_RECLAIM_SWAP: - /* - * Putback the page to inactive lru list. - * Since there is no interface function to put the - * page to the tail of inactive lru list, we put it - * to the head of the list in order to less invasion. - */ - __ClearPageActive(page); - ClearPageReferenced(page); - unlock_page(page); - putback_lru_page(page); - mod_node_page_state(pgdat, NR_ISOLATED_ANON, - -HPAGE_PMD_NR); - - continue; - - case THP_RECLAIM_ZSR: - /* - * Split the huge page and reclaim the zero subpages. - * And putback the non-zero subpages to the lru list. - */ - if (split_huge_page_to_list(page, &split_list)) { - unlock_page(page); - putback_lru_page(page); - mod_node_page_state(pgdat, NR_ISOLATED_ANON, - -HPAGE_PMD_NR); - continue; - } - - unlock_page(page); - list_add_tail(&page->lru, &split_list); - reclaim_zero_subpages(&split_list, &keep_list); - - spin_lock_irqsave(&pgdat->lru_lock, flags); - putback_inactive_pages(lruvec, &keep_list); - spin_unlock_irqrestore(&pgdat->lru_lock, flags); - mod_node_page_state(pgdat, NR_ISOLATED_ANON, - -HPAGE_PMD_NR); - - mem_cgroup_uncharge_list(&keep_list); - free_unref_page_list(&keep_list); - - break; - } + reclaim_huge_page(lruvec, page, thp_reclaim); } sc->nr_scanned = nr_scanned; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 8381a6c..fce1f01 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5858,11 +5858,13 @@ static int memcg_thp_reclaim_ctrl_show(struct seq_file *m, void *v) return 0; } +#define CTRL_RECLAIM_MEMCG 1 /* only relciam current memcg*/ +#define CTRL_RECLAIM_ALL 2 /* reclaim current memcg and all the child memcg */ static ssize_t memcg_thp_reclaim_ctrl_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); - int ret, threshold; + int ret, threshold, mode; char *key, *value; key = strsep_s(&buf, " \t\n"); @@ -5882,6 +5884,30 @@ static ssize_t memcg_thp_reclaim_ctrl_write(struct kernfs_open_file *of, return -EINVAL; xchg(&memcg->thp_reclaim_threshold, threshold); + } else if (!strcmp(key, "reclaim")) { + struct mem_cgroup *iter; + + value = strsep_s(&buf, " \t\n"); + if (!value) + return -EINVAL; + + ret = kstrtouint(value, 0, &mode); + if (ret) + return ret; + + switch (mode) { + case CTRL_RECLAIM_MEMCG: + reclaim_memcg_huge_pages(memcg); + break; + case CTRL_RECLAIM_ALL: + iter = mem_cgroup_iter(memcg, NULL, NULL); + do { + reclaim_memcg_huge_pages(iter); + } while ((iter = mem_cgroup_iter(memcg, iter, NULL))); + break; + default: + return -EINVAL; + } } else return -EINVAL; -- 1.8.3.1