From 57456d1625ba9036968fa0be70a6036b88f2b2f4 Mon Sep 17 00:00:00 2001
From: Ning Zhang <ningzhang@linux.alibaba.com>
Date: Sat, 29 May 2021 21:41:06 +0800
Subject: [PATCH 2757/2944] mm, thp: introduce a controller to trigger thp
 reclaim

fix #34670772

Add a new controller named "reclaim" for memory.thp_reclaim_ctrl
to trigger thp reclaim immediately:
	echo "reclaim 1" > memory.thp_reclaim_ctrl
	echo "reclaim 2" > memory.thp_reclaim_ctrl

"reclaim 1" means triggering reclaim only for current memcg.
"reclaim 2" means triggering reclaim for current memcg and it's
children memcgs.

Signed-off-by: Ning Zhang <ningzhang@linux.alibaba.com>
Reviewed-by: Gang Deng <gavin.dg@linux.alibaba.com>
Reviewed-by: Xu Yu <xuyu@linux.alibaba.com>
---
 Documentation/admin-guide/mm/transhuge.rst |  11 +++
 include/linux/memcontrol.h                 |   1 +
 mm/huge_memory.c                           | 150 ++++++++++++++++++-----------
 mm/memcontrol.c                            |  28 +++++-
 4 files changed, 135 insertions(+), 55 deletions(-)

diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst
index 87427e4..5333afa 100644
--- a/Documentation/admin-guide/mm/transhuge.rst
+++ b/Documentation/admin-guide/mm/transhuge.rst
@@ -487,4 +487,15 @@ threshold
 
         echo "threshold 8" > memory.thp_reclaim_ctrl
 
+reclaim
+        triggers action immediately for the huge pages in the reclaim queue.
+        The action deponds on the thp reclaim config (reclaim, swap or disable,
+        disable means just remove the huge page from the queue).
+        This contronller has two value, 1 and 2. 1 means just reclaim the current
+        memcg, and 2 means reclaim the current memcg and all the children memcgs.
+        Like this::
+
+        echo "reclaim 1" > memory.thp_reclaim_ctrl
+        echo "reclaim 2" > memory.thp_reclaim_ctrl
+
 Only one of the configs mentioned above can be set at a time.
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 9eaab10..eb602ae 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -925,6 +925,7 @@ void memcg_check_wmark_min_adj(struct task_struct *curr,
 extern int global_thp_reclaim;
 extern void set_hugepage_reclaim_shrinker_bit(struct mem_cgroup *memcg,
 					      int nid);
+extern void reclaim_memcg_huge_pages(struct mem_cgroup *memcg);
 
 static inline struct list_head *hugepage_reclaim_list(struct page *page)
 {
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 7611bde..ae1b9c8 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -3483,17 +3483,108 @@ static struct page *get_reclaim_hugepage(struct hugepage_reclaim *hr_queue,
 
 }
 
+static void reclaim_huge_page(struct lruvec *lruvec, struct page *page, int mode)
+{
+	struct pglist_data *pgdat = page_pgdat(page);
+	unsigned long flags;
+	LIST_HEAD(split_list);
+	LIST_HEAD(keep_list);
+
+	switch (mode) {
+	case THP_RECLAIM_SWAP:
+		/*
+		 * Putback the page to inactive lru list.
+		 * Since there is no interface function to put the
+		 * page to the tail of inactive lru list, we put it
+		 * to the head of the list in order to less invasion.
+		 */
+		__ClearPageActive(page);
+		ClearPageReferenced(page);
+		unlock_page(page);
+		putback_lru_page(page);
+		mod_node_page_state(pgdat, NR_ISOLATED_ANON,
+				    -HPAGE_PMD_NR);
+		break;
+	case THP_RECLAIM_ZSR:
+		/*
+		 * Split the huge page and reclaim the zero subpages.
+		 * And putback the non-zero subpages to the lru list.
+		 */
+		if (split_huge_page_to_list(page, &split_list)) {
+			unlock_page(page);
+			putback_lru_page(page);
+			mod_node_page_state(pgdat, NR_ISOLATED_ANON,
+					    -HPAGE_PMD_NR);
+			break;
+		}
+
+		unlock_page(page);
+		list_add_tail(&page->lru, &split_list);
+		reclaim_zero_subpages(&split_list, &keep_list);
+
+		spin_lock_irqsave(&pgdat->lru_lock, flags);
+		putback_inactive_pages(lruvec, &keep_list);
+		spin_unlock_irqrestore(&pgdat->lru_lock, flags);
+		mod_node_page_state(pgdat, NR_ISOLATED_ANON,
+				    -HPAGE_PMD_NR);
+
+		mem_cgroup_uncharge_list(&keep_list);
+		free_unref_page_list(&keep_list);
+		break;
+	default:
+		WARN_ONCE(1, "To reclaim a huge page when thp reclaim is disable");
+		break;
+	}
+}
+
+static inline int get_reclaim_mode(struct mem_cgroup *memcg)
+{
+	int reclaim = READ_ONCE(global_thp_reclaim);
+
+	return (reclaim != THP_RECLAIM_MEMCG) ? reclaim :
+					READ_ONCE(memcg->thp_reclaim);
+}
+
+void reclaim_memcg_huge_pages(struct mem_cgroup *memcg)
+{
+	struct lruvec *lruvec;
+	struct hugepage_reclaim *hr_queue;
+	struct page *page;
+	int thp_reclaim, threshold, nid;
+	bool empty;
+
+	threshold = READ_ONCE(memcg->thp_reclaim_threshold);
+	thp_reclaim = get_reclaim_mode(memcg);
+
+	for_each_online_node(nid) {
+		lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
+		hr_queue = &memcg->nodeinfo[nid]->hugepage_reclaim_queue;
+		while (1) {
+			cond_resched();
+			page = get_reclaim_hugepage(hr_queue, threshold,
+				&empty, thp_reclaim == THP_RECLAIM_DISABLE);
+
+			if (empty)
+				break;
+
+			if (!page)
+				continue;
+
+			reclaim_huge_page(lruvec, page, thp_reclaim);
+		}
+
+	}
+}
+
 static unsigned long hugepage_reclaim_scan(struct shrinker *shrink,
 		struct shrink_control *sc)
 {
 	int nid = sc->nid;
-	struct pglist_data *pgdat = NODE_DATA(nid);
-	struct lruvec *lruvec = mem_cgroup_lruvec(sc->memcg, pgdat);
+	struct lruvec *lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(nid));
 	struct hugepage_reclaim *hr_queue = NULL;
 	struct page *page;
 	int thp_reclaim, threshold;
 	unsigned long nr_to_scan, nr_scanned;
-	unsigned long flags;
 	bool empty;
 #define MAX_SCAN_HUGEPAGES 32UL
 
@@ -3512,16 +3603,11 @@ static unsigned long hugepage_reclaim_scan(struct shrinker *shrink,
 		return SHRINK_STOP;
 
 	threshold = READ_ONCE(sc->memcg->thp_reclaim_threshold);
-	thp_reclaim = READ_ONCE(global_thp_reclaim);
-	thp_reclaim = (thp_reclaim != THP_RECLAIM_MEMCG) ? thp_reclaim :
-			READ_ONCE(sc->memcg->thp_reclaim);
+	thp_reclaim = get_reclaim_mode(sc->memcg);
 	nr_to_scan = min(sc->nr_to_scan, MAX_SCAN_HUGEPAGES);
 	nr_scanned = 0;
 
 	while (nr_to_scan) {
-		LIST_HEAD(split_list);
-		LIST_HEAD(keep_list);
-
 		cond_resched();
 		page = get_reclaim_hugepage(hr_queue, threshold, &empty,
 					    thp_reclaim == THP_RECLAIM_DISABLE);
@@ -3534,51 +3620,7 @@ static unsigned long hugepage_reclaim_scan(struct shrinker *shrink,
 		if (!page)
 			continue;
 
-		switch (thp_reclaim) {
-		case THP_RECLAIM_SWAP:
-			/*
-			 * Putback the page to inactive lru list.
-			 * Since there is no interface function to put the
-			 * page to the tail of inactive lru list, we put it
-			 * to the head of the list in order to less invasion.
-			 */
-			__ClearPageActive(page);
-			ClearPageReferenced(page);
-			unlock_page(page);
-			putback_lru_page(page);
-			mod_node_page_state(pgdat, NR_ISOLATED_ANON,
-					    -HPAGE_PMD_NR);
-
-			continue;
-
-		case THP_RECLAIM_ZSR:
-			/*
-			 * Split the huge page and reclaim the zero subpages.
-			 * And putback the non-zero subpages to the lru list.
-			 */
-			if (split_huge_page_to_list(page, &split_list)) {
-				unlock_page(page);
-				putback_lru_page(page);
-				mod_node_page_state(pgdat, NR_ISOLATED_ANON,
-						    -HPAGE_PMD_NR);
-				continue;
-			}
-
-			unlock_page(page);
-			list_add_tail(&page->lru, &split_list);
-			reclaim_zero_subpages(&split_list, &keep_list);
-
-			spin_lock_irqsave(&pgdat->lru_lock, flags);
-			putback_inactive_pages(lruvec, &keep_list);
-			spin_unlock_irqrestore(&pgdat->lru_lock, flags);
-			mod_node_page_state(pgdat, NR_ISOLATED_ANON,
-					    -HPAGE_PMD_NR);
-
-			mem_cgroup_uncharge_list(&keep_list);
-			free_unref_page_list(&keep_list);
-
-			break;
-		}
+		reclaim_huge_page(lruvec, page, thp_reclaim);
 	}
 
 	sc->nr_scanned = nr_scanned;
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 8381a6c..fce1f01 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -5858,11 +5858,13 @@ static int memcg_thp_reclaim_ctrl_show(struct seq_file *m, void *v)
 	return 0;
 }
 
+#define CTRL_RECLAIM_MEMCG 1 /* only relciam current memcg*/
+#define CTRL_RECLAIM_ALL   2 /* reclaim current memcg and all the child memcg */
 static ssize_t memcg_thp_reclaim_ctrl_write(struct kernfs_open_file *of,
 					char *buf, size_t nbytes, loff_t off)
 {
 	struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
-	int ret, threshold;
+	int ret, threshold, mode;
 	char *key, *value;
 
 	key = strsep_s(&buf, " \t\n");
@@ -5882,6 +5884,30 @@ static ssize_t memcg_thp_reclaim_ctrl_write(struct kernfs_open_file *of,
 			return -EINVAL;
 
 		xchg(&memcg->thp_reclaim_threshold, threshold);
+	} else if (!strcmp(key, "reclaim")) {
+		struct mem_cgroup *iter;
+
+		value = strsep_s(&buf, " \t\n");
+		if (!value)
+			return -EINVAL;
+
+		ret = kstrtouint(value, 0, &mode);
+		if (ret)
+			return ret;
+
+		switch (mode) {
+		case CTRL_RECLAIM_MEMCG:
+			reclaim_memcg_huge_pages(memcg);
+			break;
+		case CTRL_RECLAIM_ALL:
+			iter = mem_cgroup_iter(memcg, NULL, NULL);
+			do {
+				reclaim_memcg_huge_pages(iter);
+			} while ((iter = mem_cgroup_iter(memcg, iter, NULL)));
+			break;
+		default:
+			return -EINVAL;
+		}
 	} else
 		return -EINVAL;
 
-- 
1.8.3.1

