From 62f8852885cc7f23063886d36fd36d94b48d3982 Mon Sep 17 00:00:00 2001 From: Ning Zhang Date: Tue, 15 Dec 2020 19:38:33 +0800 Subject: [PATCH 2754/2944] mm, thp: introduce thp zero subpages reclaim fix #34670772 Transparent huge pages could reduce the number of address translation misses, which could improve performance for the applications. But one concern is that THP may lead to memory bloat which may cause OOM. The reason is a huge page may contain some zero subpages which user didn't really access them. This patch introduces a mechanism to reclaim these zero subpages, it works when memory pressure is high. We'll estimate whether a huge page contains enough zero subpages at first, then try split it and reclaim. We store the huge page to a new list, and add a slab shrinker to scan the list when reclaiming. We add a interface 'thp_reclaim' to control on or off in memory cgroup: echo reclaim > memory.thp_reclaim to enable reclaim mode. echo swap > memory.thp_reclaim to enable swap mode. echo disable > memory.thp_reclaim to disable. Or you can configure it by the global interface: /sys/kernel/mm/transparent_hugepage/reclaim The default mode is inherited from it's parent: 1. The reclaim mode means the huge page will be split and the zero subpages will be reclaimed. 2. The swap mode means we just put the huge page in the tail of inactive lru list and use swap mechanism to swap it. Meanwhile, we add a interface thp_reclaim_stat to show the stat of each node in the memory cgroup: $cat memory.thp_reclaim_stat queue_length 10 Signed-off-by: Ning Zhang Reviewed-by: Xunlei Pang Reviewed-by: Alex Shi Reviewed-by: Gang Deng Reviewed-by: Xu Yu --- Documentation/admin-guide/mm/transhuge.rst | 52 ++++ include/linux/memcontrol.h | 100 ++++++ include/linux/mm.h | 2 + include/linux/mm_types.h | 8 + include/linux/mmzone.h | 6 + mm/huge_memory.c | 481 ++++++++++++++++++++++++++++- mm/memcontrol.c | 98 ++++++ mm/page_alloc.c | 6 + mm/vmscan.c | 3 +- 9 files changed, 753 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst index 2399fb0..af16a77 100644 --- a/Documentation/admin-guide/mm/transhuge.rst +++ b/Documentation/admin-guide/mm/transhuge.rst @@ -423,3 +423,55 @@ support enabled just fine as always. No difference can be noted in hugetlbfs other than there will be less overall fragmentation. All usual features belonging to hugetlbfs are preserved and unaffected. libhugetlbfs will also work fine as usual. + +THP zero subpages reclaim +========================= +THP may lead to memory bloat which may cause OOM. The reason is a huge +page may contain some zero subpages which users didn't really access them. +To avoid this, a mechanism to reclaim these zero subpages is introduced:: + + echo reclaim > /sys/fs/cgroup/memory/{memcg}/memory.thp_reclaim + echo swap > /sys/fs/cgroup/memory/{memcg}/memory.thp_reclaim + echo disable > /sys/fs/cgroup/memory/{memcg}/memory.thp_reclaim + +reclaim + means the huge page will be split and the zero subpages will be + reclaimed. + +swap + means we just put the huge page in the inactive lru list and use + swap mechanism to swap it. + +disable + means do nothing. + +The default mode is inherited from its parent. The default mode of root +memcg is disable. + +We also add a global interface, if you don't want to configure it by configuring +every memory cgroup, you can use this one:: + + /sys/kernel/mm/transparent_hugepage/reclaim + +memcg + The default mode. It means every mem cgroup will use their own configure. + +reclaim + means every mem cgroup will use reclaim. + +swap + means every mem cgroup will use swap. + +disable + means every mem cgroup will use disable. + +If the mode is reclaim or swap, the new huge page will be add to a reclaim +queue in mem_cgroup, and the queue would be scanned when memory reclaiming. +The huge page which contains at least 1/32 zero pages would be split(estimate +it by checking some discrete unsigned long values). + +The queue stat can be checked like this:: + + cat /sys/fs/cgroup/memory/{memcg}/memory.thp_reclaim_stat + +Now, it only contains the queue length of each node. diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 498d061..7420e7f 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -188,6 +188,7 @@ struct mem_cgroup_per_node { #ifdef CONFIG_TRANSPARENT_HUGEPAGE struct deferred_split deferred_split_queue; + struct hugepage_reclaim hugepage_reclaim_queue; #endif struct mem_cgroup *memcg; /* Back pointer, we cannot */ @@ -235,6 +236,14 @@ struct memcg_padding { #define MEMCG_PADDING(name) #endif +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define THP_RECLAIM_DISABLE 0 +#define THP_RECLAIM_SWAP 1 +#define THP_RECLAIM_ZSR 2 + +#define THP_RECLAIM_MEMCG 3 /* For global configure*/ +#endif + /* * The memory controller data structure. The memory controller controls both * page cache and RSS per cgroup. We would eventually like to provide @@ -391,6 +400,10 @@ struct mem_cgroup { /* memory latency stat */ struct mem_cgroup_lat_stat_cpu __percpu *lat_stat_cpu; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + int thp_reclaim; +#endif + ALI_HOTFIX_RESERVE(1) ALI_HOTFIX_RESERVE(2) ALI_HOTFIX_RESERVE(3) @@ -904,8 +917,95 @@ static inline bool is_wmark_ok(struct mem_cgroup *memcg, bool high) int memcg_get_wmark_min_adj(struct task_struct *curr); void memcg_check_wmark_min_adj(struct task_struct *curr, struct alloc_context *ac); + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +extern int global_thp_reclaim; +extern void set_hugepage_reclaim_shrinker_bit(struct mem_cgroup *memcg, + int nid); + +static inline struct list_head *hugepage_reclaim_list(struct page *page) +{ + return &page[3].hugepage_reclaim_list; +} + +static inline bool add_hugepage_to_queue(struct page *page) +{ + struct page *head = compound_head(page); + struct mem_cgroup *memcg = compound_head(page)->mem_cgroup; + int nid = page_to_nid(page); + struct hugepage_reclaim *hr_queue; + unsigned long flags; + int memcg_reclaim, global_reclaim; + bool ret = false; + + if (!memcg || !(PageTransHuge(head) && !PageHuge(head))) + return ret; + + global_reclaim = READ_ONCE(global_thp_reclaim); + if (global_reclaim == THP_RECLAIM_DISABLE) + return ret; + else if (global_reclaim == THP_RECLAIM_MEMCG) { + memcg_reclaim = READ_ONCE(memcg->thp_reclaim); + if (memcg_reclaim == THP_RECLAIM_DISABLE) + return ret; + } + + hr_queue = &memcg->nodeinfo[nid]->hugepage_reclaim_queue; + spin_lock_irqsave(&hr_queue->reclaim_queue_lock, flags); + if (list_empty(hugepage_reclaim_list(head))) { + list_add(hugepage_reclaim_list(head), + &hr_queue->reclaim_queue); + hr_queue->reclaim_queue_len++; + ret = true; + } + + if (unlikely(hr_queue->reclaim_queue_len == 1)) + set_hugepage_reclaim_shrinker_bit(memcg, page_to_nid(page)); + + spin_unlock_irqrestore(&hr_queue->reclaim_queue_lock, flags); + + return ret; +} + +static inline bool del_hugepage_from_queue(struct page *page) +{ + struct page *head = compound_head(page); + struct mem_cgroup *memcg = compound_head(page)->mem_cgroup; + int nid = page_to_nid(page); + struct hugepage_reclaim *hr_queue; + unsigned long flags; + bool ret = false; + + if (!memcg || !(PageTransHuge(head) && !PageHuge(head))) + return ret; + + hr_queue = &memcg->nodeinfo[nid]->hugepage_reclaim_queue; + spin_lock_irqsave(&hr_queue->reclaim_queue_lock, flags); + if (!list_empty(hugepage_reclaim_list(head))) { + list_del_init(hugepage_reclaim_list(head)); + hr_queue->reclaim_queue_len--; + ret = true; + } + spin_unlock_irqrestore(&hr_queue->reclaim_queue_lock, flags); + + return ret; +} +#endif + #else /* CONFIG_MEMCG */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline bool add_hugepage_to_queue(struct page *page) +{ + return false; +} + +static inline bool del_hugepage_from_queue(struct page *page) +{ + return false; +} +#endif + #define MEM_CGROUP_ID_SHIFT 0 #define MEM_CGROUP_ID_MAX 0 diff --git a/include/linux/mm.h b/include/linux/mm.h index 55b1751..958c482 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -173,6 +173,8 @@ extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *); extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *); +extern void putback_inactive_pages(struct lruvec *lruvec, + struct list_head *page_list); #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n)) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index b32ac16..78d4a8e 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -138,6 +138,14 @@ struct page { /* For both global and memcg */ struct list_head deferred_list; }; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct { /* Third tail page of compound page */ + unsigned long _compound_pad_3; /* compound_head */ + unsigned long _compound_pad_4; + /* For zero subpage reclaim */ + struct list_head hugepage_reclaim_list; + }; +#endif struct { /* Page table pages */ unsigned long _pt_pad_1; /* compound_head */ pgtable_t pmd_huge_pte; /* protected by page->ptl */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 8010d7a..6e9b01b 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -648,6 +648,12 @@ struct deferred_split { struct list_head split_queue; unsigned long split_queue_len; }; + +struct hugepage_reclaim { + spinlock_t reclaim_queue_lock; + struct list_head reclaim_queue; + unsigned long reclaim_queue_len; +}; #endif /* diff --git a/mm/huge_memory.c b/mm/huge_memory.c index e620fb1..7ee252c 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -61,6 +61,14 @@ static struct shrinker deferred_split_shrinker; +#ifdef CONFIG_MEMCG +/* + * Zero subpages reclaim for huge pages only works when memcg defined. + */ +int global_thp_reclaim; +static struct shrinker hugepage_reclaim_shrinker; +#endif + static atomic_t huge_zero_refcount; struct page *huge_zero_page __read_mostly; @@ -342,6 +350,48 @@ static ssize_t fast_cow_store(struct kobject *kobj, static struct kobj_attribute fast_cow_attr = __ATTR(fast_cow, 0644, fast_cow_show, fast_cow_store); +#ifdef CONFIG_MEMCG +static ssize_t reclaim_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + int thp_reclaim = READ_ONCE(global_thp_reclaim); + + if (thp_reclaim == THP_RECLAIM_MEMCG) + return sprintf(buf, "[memcg] reclaim swap disable\n"); + else if (thp_reclaim == THP_RECLAIM_ZSR) + return sprintf(buf, "memcg [reclaim] swap disable\n"); + else if (thp_reclaim == THP_RECLAIM_SWAP) + return sprintf(buf, "memcg reclaim [swap] disable\n"); + else + return sprintf(buf, "memcg reclaim swap [disable]\n"); +} + +static ssize_t reclaim_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + if (!memcmp("memcg", buf, + min(sizeof("memcg")-1, count))) + WRITE_ONCE(global_thp_reclaim, THP_RECLAIM_MEMCG); + else if (!memcmp("reclaim", buf, + min(sizeof("reclaim")-1, count))) + WRITE_ONCE(global_thp_reclaim, THP_RECLAIM_ZSR); + else if (!memcmp("swap", buf, + min(sizeof("swap")-1, count))) + WRITE_ONCE(global_thp_reclaim, THP_RECLAIM_SWAP); + else if (!memcmp("disable", buf, + min(sizeof("disable")-1, count))) + WRITE_ONCE(global_thp_reclaim, THP_RECLAIM_DISABLE); + else + return -EINVAL; + + return count; +} + +static struct kobj_attribute reclaim_attr = + __ATTR(reclaim, 0644, reclaim_show, reclaim_store); +#endif + static struct attribute *hugepage_attr[] = { &enabled_attr.attr, &defrag_attr.attr, @@ -354,6 +404,9 @@ static ssize_t fast_cow_store(struct kobject *kobj, &debug_cow_attr.attr, #endif &fast_cow_attr.attr, +#ifdef CONFIG_MEMCG + &reclaim_attr.attr, +#endif NULL, }; @@ -443,6 +496,12 @@ static int __init hugepage_init(void) err = register_shrinker(&deferred_split_shrinker); if (err) goto err_split_shrinker; +#ifdef CONFIG_MEMCG + err = register_shrinker(&hugepage_reclaim_shrinker); + if (err) + goto err_hugepage_reclaim; + global_thp_reclaim = THP_RECLAIM_MEMCG; +#endif /* * By default disable transparent hugepages on smaller systems, @@ -461,6 +520,10 @@ static int __init hugepage_init(void) return 0; err_khugepaged: unregister_shrinker(&deferred_split_shrinker); +#ifdef CONFIG_MEMCG +err_hugepage_reclaim: + unregister_shrinker(&hugepage_reclaim_shrinker); +#endif err_split_shrinker: unregister_shrinker(&huge_zero_page_shrinker); err_hzp_shrinker: @@ -539,6 +602,9 @@ void prep_transhuge_page(struct page *page) */ INIT_LIST_HEAD(page_deferred_list(page)); +#ifdef CONFIG_MEMCG + INIT_LIST_HEAD(hugepage_reclaim_list(page)); +#endif set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR); } @@ -2512,7 +2578,7 @@ static void __split_huge_page_tail(struct page *head, int tail, (1L << PG_dirty))); /* ->mapping in first tail page is compound_mapcount */ - VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING, + VM_BUG_ON_PAGE(tail > 3 && page_tail->mapping != TAIL_MAPPING, page_tail); page_tail->mapping = head->mapping; page_tail->index = head->index + tail; @@ -2740,6 +2806,8 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageCompound(page), page); + del_hugepage_from_queue(page); + if (PageWriteback(page)) return -EBUSY; @@ -2912,6 +2980,7 @@ void deferred_split_huge_page(struct page *page) memcg_set_shrinker_bit(memcg, page_to_nid(page), deferred_split_shrinker.id); #endif + del_hugepage_from_queue(page); } spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); } @@ -3112,3 +3181,413 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) update_mmu_cache_pmd(vma, address, pvmw->pmd); } #endif + +#ifdef CONFIG_MEMCG +void set_hugepage_reclaim_shrinker_bit(struct mem_cgroup *memcg, int nid) +{ + memcg_set_shrinker_bit(memcg, nid, hugepage_reclaim_shrinker.id); +} + +static inline bool is_zero_page(struct page *page) +{ + void *addr = kmap(page); + unsigned long *ul = (unsigned long *)addr; + bool ret = true; + int i; + +#define BYTES_PER_LONG (BITS_PER_LONG / BITS_PER_BYTE) + BUILD_BUG_ON(PAGE_SIZE % BYTES_PER_LONG); + for (i = 0; i < PAGE_SIZE; i += BYTES_PER_LONG, ul++) + if (*ul) { + ret = false; + break; + } + kunmap(page); + + return ret; +} + +/* + * We'll split the huge page iff it contains at least 1/32 zeros, + * estimate it by checking some discrete unsigned long values. + */ +static bool hugepage_estimate_zero(struct page *page) +{ + unsigned int i, maybe_zero_pages = 0, offset = 0; + void *addr; + + for (i = 0; i < HPAGE_PMD_NR; i++, page++, offset++) { + addr = kmap(page); + if (unlikely((offset + 1) * BYTES_PER_LONG > PAGE_SIZE)) + offset = 0; + if (*(const unsigned long *)(addr + offset) == 0UL) + maybe_zero_pages++; + kunmap(page); + } + + return (maybe_zero_pages << 5) >= HPAGE_PMD_NR; +} + +static bool replace_zero_pte(struct page *page, struct vm_area_struct *vma, + unsigned long addr, void *zero_page) +{ + struct page_vma_mapped_walk pvmw = { + .page = page, + .vma = vma, + .address = addr, + .flags = PVMW_SYNC | PVMW_MIGRATION, + }; + pte_t pte; + + VM_BUG_ON_PAGE(PageTail(page), page); + + while (page_vma_mapped_walk(&pvmw)) { + pte = pte_mkspecial( + pfn_pte(page_to_pfn((struct page *)zero_page), + vma->vm_page_prot)); + + /* + * We're replacing an anonymous page with a zero page, which is + * not anonymous. We need to do proper accounting otherwise we + * will get wrong values in /proc, and a BUG message in dmesg + * when tearing down the mm. + */ + dec_mm_counter(vma->vm_mm, MM_ANONPAGES); + set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); + + /* We ignore the dirty or referenced bit, it's acceptable. */ + + /* No need to invalidate - it was non-present before.*/ + update_mmu_cache(vma, pvmw.address, pvmw.pte); + } + + return true; +} + +static void replace_zero_ptes_locked(struct page *page) +{ + struct page *zero_page = ZERO_PAGE(0); + struct rmap_walk_control rwc = { + .rmap_one = replace_zero_pte, + .arg = zero_page, + }; + + rmap_walk_locked(page, &rwc); +} + +static bool replace_zero_page(struct page *page) +{ + struct anon_vma *anon_vma = NULL; + enum ttu_flags ttu_flags = TTU_RMAP_LOCKED | TTU_MIGRATION; + bool unmap_success; + bool ret = true; + + anon_vma = page_get_anon_vma(page); + if (!anon_vma) + return false; + + anon_vma_lock_write(anon_vma); + unmap_success = try_to_unmap(page, ttu_flags); + + if (!unmap_success) + ret = false; + else if (!is_zero_page(page)) { + /* remap the non-zero page */ + remove_migration_ptes(page, page, true); + ret = false; + } else + replace_zero_ptes_locked(page); + + anon_vma_unlock_write(anon_vma); + put_anon_vma(anon_vma); + + return ret; +} + +/* + * reclaim_zero_subpages - reclaim the zero subpages and putback the non-zero + * subpages. + * + * The non-zero subpages are putback to the keep_list, and will be putback to + * the lru list. + * + * Return the number of reclaimed zero subpages. + */ +static unsigned long reclaim_zero_subpages(struct list_head *list, + struct list_head *keep_list) +{ + LIST_HEAD(zero_list); + struct reclaim_state *reclaim_state = current->reclaim_state; + struct page *page; + unsigned long reclaimed = 0; + + while (!list_empty(list)) { + page = lru_to_page(list); + list_del_init(&page->lru); + if (is_zero_page(page)) { + if (!trylock_page(page)) + goto keep; + + if (!replace_zero_page(page)) { + unlock_page(page); + goto keep; + } + + __ClearPageActive(page); + unlock_page(page); + if (put_page_testzero(page)) { + list_add(&page->lru, &zero_list); + reclaimed++; + if (reclaim_state) + reclaim_state->reclaimed_slab++; + } + + /* someone may hold the zero page, we just skip it. */ + + continue; + } +keep: + list_add(&page->lru, keep_list); + } + + mem_cgroup_uncharge_list(&zero_list); + free_unref_page_list(&zero_list); + + return reclaimed; + +} + +static unsigned long hugepage_reclaim_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + int nid = sc->nid; + struct hugepage_reclaim *hr_queue; + + if (sc->memcg) { + hr_queue = &sc->memcg->nodeinfo[nid]->hugepage_reclaim_queue; + return READ_ONCE(hr_queue->reclaim_queue_len); + } + + return 0; +} + +#ifdef CONFIG_MMU +#define ZSR_IF_HAVE_PG_MLOCK(flag) (1UL << flag) +#else +#define ZSR_IF_HAVE_PG_MLOCK(flag) 0 +#endif +#ifdef CONFIG_ARCH_USES_PG_UNCACHED +#define ZSR_IF_HAVE_PG_UNCACHED(flag) (1UL << flag) +#else +#define ZSR_IF_HAVE_PG_UNCACHED(flag) 0 +#endif +#ifdef CONFIG_MEMORY_FAILURE +#define ZSR_IF_HAVE_PG_HWPOISON(flag) (1UL << flag) +#else +#define ZSR_IF_HAVE_PG_HWPOISON(flag) 0 +#endif + +/* Fiter unsupported page flags. */ +#define PAGE_FLAG_CHECK_AT_ZERO_SUBPAGE_RECLAIM \ + ((1UL << PG_error) | \ + (1UL << PG_owner_priv_1) | \ + (1UL << PG_arch_1) | \ + (1UL << PG_reserved) | \ + (1UL << PG_private) | \ + (1UL << PG_private_2) | \ + (1UL << PG_writeback) | \ + (1UL << PG_swapcache) | \ + (1UL << PG_mappedtodisk) | \ + (1UL << PG_reclaim) | \ + (1UL << PG_unevictable) | \ + ZSR_IF_HAVE_PG_MLOCK(PG_mlocked) | \ + ZSR_IF_HAVE_PG_UNCACHED(PG_uncached) | \ + ZSR_IF_HAVE_PG_HWPOISON(PG_hwpoison)) + +#define hugepage_can_reclaim(page) \ + (PageAnon(page) && !PageKsm(page) && PageTransHuge(page) && \ + !PageHuge(page) && \ + !(page->flags & PAGE_FLAG_CHECK_AT_ZERO_SUBPAGE_RECLAIM)) + +#define hr_list_to_page(head) \ + compound_head(list_entry((head)->prev, struct page,\ + hugepage_reclaim_list)) + +/* + * get_reclaim_hugepage - get one huge page from huge page reclaim queue + * @hugepage_reclaim: the huge page reclaim queue + * @empty: store the state if the queue is empty or not + * @disable: whethe the thp reclaim mode is disable or not + * + * If the queue is empty, then @empty is true, and hugepage_reclaim_scan + * will be stopped. + * + * if @disable is true, it means the thp reclaim is disable, and we just + * delete one huge page from the queue. + * + * Return the huge page can be reclaimed, and the huge page is isolated + * from lru list and locked. + */ +static struct page *get_reclaim_hugepage(struct hugepage_reclaim *hr_queue, + bool *empty, bool disable) +{ + struct page *page = NULL, *reclaim_page = NULL; + unsigned long flags; + + *empty = true; + spin_lock_irqsave(&hr_queue->reclaim_queue_lock, flags); + if (!list_empty(&hr_queue->reclaim_queue)) { + *empty = false; + page = hr_list_to_page(&hr_queue->reclaim_queue); + list_del_init(hugepage_reclaim_list(page)); + hr_queue->reclaim_queue_len--; + + if (disable) + goto unlock; + + if (hugepage_can_reclaim(page) && get_page_unless_zero(page)) { + if (trylock_page(page)) { + spin_unlock_irqrestore(&hr_queue->reclaim_queue_lock, + flags); + if (hugepage_can_reclaim(page) && + hugepage_estimate_zero(page) && + !isolate_lru_page(page)) { + __mod_node_page_state(page_pgdat(page), + NR_ISOLATED_ANON, + HPAGE_PMD_NR); + /* + * dec the reference added in + * isolate_lru_page + */ + page_ref_dec(page); + reclaim_page = page; + } else { + unlock_page(page); + put_page(page); + } + + return reclaim_page; + + } else + put_page(page); + } + } +unlock: + spin_unlock_irqrestore(&hr_queue->reclaim_queue_lock, flags); + + return reclaim_page; + +} + +static unsigned long hugepage_reclaim_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + int nid = sc->nid; + struct pglist_data *pgdat = NODE_DATA(nid); + struct lruvec *lruvec = mem_cgroup_lruvec(sc->memcg, pgdat); + struct hugepage_reclaim *hr_queue = NULL; + struct page *page; + int thp_reclaim; + unsigned long nr_to_scan, nr_scanned; + unsigned long flags; + bool empty; +#define MAX_SCAN_HUGEPAGES 32UL + + /* + * This is to avoid drop_slab (by 'echo 2 > /proc/sys/vm/drop_caches') + * to trigger ZSR. Since Page reclaim (kswapd or direct reclaim) will + * set the PF_MEMALLOC flag, and drop_slab not. + */ + if (!(current->flags & PF_MEMALLOC)) + return SHRINK_STOP; + + if (sc->memcg) + hr_queue = &sc->memcg->nodeinfo[nid]->hugepage_reclaim_queue; + + if (!hr_queue) + return SHRINK_STOP; + + thp_reclaim = READ_ONCE(global_thp_reclaim); + thp_reclaim = (thp_reclaim != THP_RECLAIM_MEMCG) ? thp_reclaim : + READ_ONCE(sc->memcg->thp_reclaim); + nr_to_scan = min(sc->nr_to_scan, MAX_SCAN_HUGEPAGES); + nr_scanned = 0; + + while (nr_to_scan) { + LIST_HEAD(split_list); + LIST_HEAD(keep_list); + + cond_resched(); + page = get_reclaim_hugepage(hr_queue, &empty, + thp_reclaim == THP_RECLAIM_DISABLE); + if (empty) + break; + + nr_to_scan--; + nr_scanned++; + + if (!page) + continue; + + switch (thp_reclaim) { + case THP_RECLAIM_SWAP: + /* + * Putback the page to inactive lru list. + * Since there is no interface function to put the + * page to the tail of inactive lru list, we put it + * to the head of the list in order to less invasion. + */ + __ClearPageActive(page); + ClearPageReferenced(page); + unlock_page(page); + putback_lru_page(page); + mod_node_page_state(pgdat, NR_ISOLATED_ANON, + -HPAGE_PMD_NR); + + continue; + + case THP_RECLAIM_ZSR: + /* + * Split the huge page and reclaim the zero subpages. + * And putback the non-zero subpages to the lru list. + */ + if (split_huge_page_to_list(page, &split_list)) { + unlock_page(page); + putback_lru_page(page); + mod_node_page_state(pgdat, NR_ISOLATED_ANON, + -HPAGE_PMD_NR); + continue; + } + + unlock_page(page); + list_add_tail(&page->lru, &split_list); + reclaim_zero_subpages(&split_list, &keep_list); + + spin_lock_irqsave(&pgdat->lru_lock, flags); + putback_inactive_pages(lruvec, &keep_list); + spin_unlock_irqrestore(&pgdat->lru_lock, flags); + mod_node_page_state(pgdat, NR_ISOLATED_ANON, + -HPAGE_PMD_NR); + + mem_cgroup_uncharge_list(&keep_list); + free_unref_page_list(&keep_list); + + break; + } + } + + sc->nr_scanned = nr_scanned; + if (!sc->nr_scanned && list_empty(&hr_queue->reclaim_queue)) + return SHRINK_STOP; + + return sc->nr_scanned; +} + +static struct shrinker hugepage_reclaim_shrinker = { + .count_objects = hugepage_reclaim_count, + .scan_objects = hugepage_reclaim_scan, + .seeks = DEFAULT_SEEKS, + .flags = SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE | + SHRINKER_NONSLAB, +}; +#endif diff --git a/mm/memcontrol.c b/mm/memcontrol.c index ea52973..992c02c 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3021,6 +3021,10 @@ static void commit_charge(struct page *page, struct mem_cgroup *memcg) * - exclusive reference */ page->mem_cgroup = memcg; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + add_hugepage_to_queue(page); +#endif } #ifdef CONFIG_MEMCG_KMEM @@ -5779,6 +5783,60 @@ static int memory_events_local_show(struct seq_file *m, void *v) return 0; } +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static int memcg_thp_reclaim_show(struct seq_file *m, void *v) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); + int thp_reclaim = READ_ONCE(memcg->thp_reclaim); + + if (thp_reclaim == THP_RECLAIM_ZSR) + seq_puts(m, "[reclaim] swap disable\n"); + else if (memcg->thp_reclaim == THP_RECLAIM_SWAP) + seq_puts(m, "reclaim [swap] disable\n"); + else + seq_puts(m, "reclaim swap [disable]\n"); + + return 0; +} + +static ssize_t memcg_thp_reclaim_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); + + buf = strstrip(buf); + if (!strcmp(buf, "reclaim")) + WRITE_ONCE(memcg->thp_reclaim, + THP_RECLAIM_ZSR); + else if (!strcmp(buf, "swap")) + WRITE_ONCE(memcg->thp_reclaim, THP_RECLAIM_SWAP); + else if (!strcmp(buf, "disable")) + WRITE_ONCE(memcg->thp_reclaim, THP_RECLAIM_DISABLE); + else + return -EINVAL; + + return nbytes; +} + +static int memcg_thp_reclaim_stat_show(struct seq_file *m, void *v) +{ + struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); + struct mem_cgroup_per_node *mz; + int node; + unsigned long len; + + seq_puts(m, "queue_length"); + for_each_node(node) { + mz = mem_cgroup_nodeinfo(memcg, node); + len = READ_ONCE(mz->hugepage_reclaim_queue.reclaim_queue_len); + seq_printf(m, " %8lu", len); + } + seq_puts(m, "\n"); + + return 0; +} +#endif + static struct cftype mem_cgroup_legacy_files[] = { { .name = "usage_in_bytes", @@ -6045,6 +6103,17 @@ static int memory_events_local_show(struct seq_file *m, void *v) .file_offset = offsetof(struct mem_cgroup, swap_events_file), .seq_show = swap_events_show, }, +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + { + .name = "thp_reclaim", + .seq_show = memcg_thp_reclaim_show, + .write = memcg_thp_reclaim_write, + }, + { + .name = "thp_reclaim_stat", + .seq_show = memcg_thp_reclaim_stat_show, + }, +#endif { }, /* terminate */ }; @@ -6198,6 +6267,10 @@ static void mem_cgroup_per_node_info_init(struct mem_cgroup *memcg, int node) spin_lock_init(&pn->deferred_split_queue.split_queue_lock); INIT_LIST_HEAD(&pn->deferred_split_queue.split_queue); pn->deferred_split_queue.split_queue_len = 0; + + spin_lock_init(&pn->hugepage_reclaim_queue.reclaim_queue_lock); + INIT_LIST_HEAD(&pn->hugepage_reclaim_queue.reclaim_queue); + pn->hugepage_reclaim_queue.reclaim_queue_len = 0; #endif } @@ -6227,6 +6300,9 @@ static bool __mem_cgroup_init(struct mem_cgroup *memcg) #ifdef CONFIG_CGROUP_WRITEBACK INIT_LIST_HEAD(&memcg->cgwb_list); #endif +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + memcg->thp_reclaim = THP_RECLAIM_DISABLE; +#endif kidled_memcg_init(memcg); idr_replace(&mem_cgroup_idr, memcg, memcg->id.id); @@ -6391,6 +6467,9 @@ static struct mem_cgroup *mem_cgroup_alloc(void) /* Default gap is 0.5% max limit */ memcg->wmark_scale_factor = parent->wmark_scale_factor ? : 50; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + memcg->thp_reclaim = parent->thp_reclaim; +#endif kidled_memcg_inherit_parent_buckets(parent, memcg); } if (parent && parent->use_hierarchy) { @@ -6708,6 +6787,9 @@ static int mem_cgroup_move_account(struct page *page, struct pglist_data *pgdat; unsigned int nr_pages = compound ? hpage_nr_pages(page) : 1; int ret; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + bool del_hugepage_queue; +#endif VM_BUG_ON(from == to); VM_BUG_ON_PAGE(PageLRU(page), page); @@ -6774,6 +6856,10 @@ static int mem_cgroup_move_account(struct page *page, __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages); } +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + del_hugepage_queue = del_hugepage_from_queue(page); +#endif + /* * All state has been migrated, let's switch to the new memcg. * @@ -6791,6 +6877,10 @@ static int mem_cgroup_move_account(struct page *page, page->mem_cgroup = to; /* caller should have done css_get */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (del_hugepage_queue) + add_hugepage_to_queue(page); +#endif __unlock_page_memcg(from); ret = 0; @@ -7862,6 +7952,10 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug) if (!page->mem_cgroup) return; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + del_hugepage_from_queue(page); +#endif + /* * Nobody should be changing or seriously looking at * page->mem_cgroup at this point, we have fully @@ -8307,6 +8401,10 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) VM_BUG_ON_PAGE(oldid, page); mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + del_hugepage_from_queue(page); +#endif + page->mem_cgroup = NULL; if (!mem_cgroup_is_root(memcg)) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 6324d9a..82e5eab 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1107,6 +1107,12 @@ static int free_tail_pages_check(struct page *head_page, struct page *page) * deferred_list.next -- ignore value. */ break; + case 3: + /* + * the third tail page: ->mapping is + * hugepage_reclaim_list.next -- ignore value. + */ + break; default: if (page->mapping != TAIL_MAPPING) { bad_page(page, "corrupted mapping in tail page", 0); diff --git a/mm/vmscan.c b/mm/vmscan.c index 0066520..6975d63 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -1808,8 +1808,7 @@ static int too_many_isolated(struct pglist_data *pgdat, int file, return isolated > inactive; } -static noinline_for_stack void -putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list) +void putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list) { struct pglist_data *pgdat = lruvec_pgdat(lruvec); LIST_HEAD(pages_to_free); -- 1.8.3.1