From d3b586512756116691cea8c9479f9d95b50b80da Mon Sep 17 00:00:00 2001 From: Joonsoo Kim Date: Tue, 11 Aug 2020 18:30:47 -0700 Subject: [PATCH 2382/2944] mm/swapcache: support to handle the shadow entries task #30476868 commit 3852f6768ede542ed48b9077bedf482c7ecb6327 upstream Workingset detection for anonymous page will be implemented in the following patch and it requires to store the shadow entries into the swapcache. This patch implements an infrastructure to store the shadow entry in the swapcache. In addition, upstream patch use an new radix tree based on xas. however, kernel-4.19 do not support that implementation. It is an big job to backport the patches to refactor radix tree's implementation, hence we need to adapt the change in the old radix tree. Signed-off-by: Joonsoo Kim Signed-off-by: Andrew Morton Acked-by: Johannes Weiner Cc: Hugh Dickins Cc: Matthew Wilcox Cc: Mel Gorman Cc: Michal Hocko Cc: Minchan Kim Cc: Vlastimil Babka Link: http://lkml.kernel.org/r/1595490560-15117-5-git-send-email-iamjoonsoo.kim@lge.com Signed-off-by: Linus Torvalds Signed-off-by: zhongjiang-ali Reviewed-by: Xunlei Pang --- include/linux/swap.h | 14 ++++-- mm/swap_state.c | 129 +++++++++++++++++++++++++++++++++++++++++++-------- mm/swapfile.c | 2 + mm/vmscan.c | 2 +- 4 files changed, 124 insertions(+), 23 deletions(-) diff --git a/include/linux/swap.h b/include/linux/swap.h index f9a4137..a4fb5cd 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -415,9 +415,12 @@ int generic_swapfile_activate(struct swap_info_struct *, struct file *, extern void show_swap_cache_info(void); extern int add_to_swap(struct page *page); extern int add_to_swap_cache(struct page *, swp_entry_t, gfp_t); -extern int __add_to_swap_cache(struct page *page, swp_entry_t entry); -extern void __delete_from_swap_cache(struct page *); +extern int __add_to_swap_cache(struct page *page, + swp_entry_t entry, void **shadowp); +extern void __delete_from_swap_cache(struct page *, void *shadow); extern void delete_from_swap_cache(struct page *); +extern void clear_shadow_from_swap_cache(int type, unsigned long begin, + unsigned long end); extern void free_page_and_swap_cache(struct page *); extern void free_pages_and_swap_cache(struct page **, int); extern struct page *lookup_swap_cache(swp_entry_t entry, @@ -576,7 +579,7 @@ static inline int add_to_swap_cache(struct page *page, swp_entry_t entry, return -1; } -static inline void __delete_from_swap_cache(struct page *page) +static inline void __delete_from_swap_cache(struct page *page, void *shadow) { } @@ -584,6 +587,11 @@ static inline void delete_from_swap_cache(struct page *page) { } +static inline void clear_shadow_from_swap_cache(int type, unsigned long begin, + unsigned long end) +{ +} + static inline int page_swapcount(struct page *page) { return 0; diff --git a/mm/swap_state.c b/mm/swap_state.c index 6bed727..2f6333b 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -107,11 +108,110 @@ void show_swap_cache_info(void) printk("Total swap = %lukB\n", total_swap_pages << (PAGE_SHIFT - 10)); } +static int swap_cache_tree_insert(struct address_space *mapping, + unsigned long index, struct page *page, void **shadowp) +{ + struct radix_tree_node *node; + void **slot; + int error; + + error = __radix_tree_create(&mapping->i_pages, index, 0, + &node, &slot); + if (error) + return error; + if (*slot) { + void *p; + + p = radix_tree_deref_slot_protected(slot, + &mapping->i_pages.xa_lock); + if (!radix_tree_exceptional_entry(p)) + return -EEXIST; + + mapping->nrexceptional--; + if (shadowp) + *shadowp = p; + } + __radix_tree_replace(&mapping->i_pages, node, slot, page, + workingset_lookup_update(mapping)); + return 0; +} + +static void swap_cache_tree_delete(struct page *page, void *shadowp) +{ + struct radix_tree_node *node; + struct address_space *mapping; + void **slot; + int i, nr = hpage_nr_pages(page); + pgoff_t idx; + swp_entry_t entry; + + entry.val = page_private(page); + mapping = swap_address_space(entry); + idx = swp_offset(entry); + for (i = 0; i < nr; i++) { + if (!__radix_tree_lookup(&mapping->i_pages, idx + i, + &node, &slot)) + continue; + __radix_tree_replace(&mapping->i_pages, node, slot, shadowp, + workingset_lookup_update(mapping)); + set_page_private(page + i, 0); + } + + if (shadowp) { + mapping->nrexceptional += nr; + /* + * Make sure the nrexceptional update is committed before + * the nrpages update so that final truncate racing + * with reclaim does not see both counters 0 at the + * same time and miss a shadow entry. + */ + smp_wmb(); + } + mapping->nrpages -= nr; +} + +void clear_shadow_from_swap_cache(int type, unsigned long begin, + unsigned long end) +{ + struct radix_tree_node *node; + unsigned long curr = begin; + void **slot; + int i; + + for (;;) { + swp_entry_t entry = swp_entry(type, curr); + struct address_space *address_space = swap_address_space(entry); + pgoff_t idx = swp_offset(entry); + + xa_lock_irq(&address_space->i_pages); + for (i = 0; i < (end - curr) && + i < round_up(curr, SWAP_ADDRESS_SPACE_PAGES); i++) { + if (!__radix_tree_lookup(&address_space->i_pages, idx + i, + &node, &slot)) + continue; + if (!radix_tree_exceptional_entry(*slot)) + continue; + __radix_tree_replace(&address_space->i_pages, node, slot, NULL, + workingset_lookup_update(address_space)); + address_space->nrexceptional--; + } + xa_unlock_irq(&address_space->i_pages); + + /* search the next swapcache until we meet end */ + curr >>= SWAP_ADDRESS_SPACE_SHIFT; + curr++; + curr <<= SWAP_ADDRESS_SPACE_SHIFT; + if (curr > end) + break; + } +} + /* * __add_to_swap_cache resembles add_to_page_cache_locked on swapper_space, * but sets SwapCache flag and private instead of mapping and index. */ -int __add_to_swap_cache(struct page *page, swp_entry_t entry) +int __add_to_swap_cache(struct page *page, + swp_entry_t entry, void **shadowp) { int error, i, nr = hpage_nr_pages(page); struct address_space *address_space; @@ -128,11 +228,12 @@ int __add_to_swap_cache(struct page *page, swp_entry_t entry) xa_lock_irq(&address_space->i_pages); for (i = 0; i < nr; i++) { set_page_private(page + i, entry.val + i); - error = radix_tree_insert(&address_space->i_pages, - idx + i, page + i); + error = swap_cache_tree_insert(address_space, + idx + i, page + i, shadowp); if (unlikely(error)) break; } + if (likely(!error)) { address_space->nrpages += nr; __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); @@ -164,7 +265,7 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask) error = radix_tree_maybe_preload_order(gfp_mask, compound_order(page)); if (!error) { - error = __add_to_swap_cache(page, entry); + error = __add_to_swap_cache(page, entry, NULL); radix_tree_preload_end(); } return error; @@ -174,26 +275,16 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask) * This must be called only on pages that have * been verified to be in the swap cache. */ -void __delete_from_swap_cache(struct page *page) +void __delete_from_swap_cache(struct page *page, void *shadow) { - struct address_space *address_space; - int i, nr = hpage_nr_pages(page); - swp_entry_t entry; - pgoff_t idx; + int nr = hpage_nr_pages(page); VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageSwapCache(page), page); VM_BUG_ON_PAGE(PageWriteback(page), page); - entry.val = page_private(page); - address_space = swap_address_space(entry); - idx = swp_offset(entry); - for (i = 0; i < nr; i++) { - radix_tree_delete(&address_space->i_pages, idx + i); - set_page_private(page + i, 0); - } + swap_cache_tree_delete(page, shadow); ClearPageSwapCache(page); - address_space->nrpages -= nr; __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr); ADD_CACHE_INFO(del_total, nr); } @@ -271,7 +362,7 @@ void delete_from_swap_cache(struct page *page) address_space = swap_address_space(entry); xa_lock_irq(&address_space->i_pages); - __delete_from_swap_cache(page); + __delete_from_swap_cache(page, NULL); xa_unlock_irq(&address_space->i_pages); put_swap_page(page, entry); @@ -458,7 +549,7 @@ struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, __SetPageSwapBacked(page); /* May fail (-ENOMEM) if radix node allocation failed. */ - if (__add_to_swap_cache(page, entry)) { + if (__add_to_swap_cache(page, entry, NULL)) { radix_tree_preload_end(); put_swap_page(page, entry); goto fail_unlock; diff --git a/mm/swapfile.c b/mm/swapfile.c index 2ec6608..a606341 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -663,6 +663,7 @@ static void add_to_avail_list(struct swap_info_struct *p) static void swap_range_free(struct swap_info_struct *si, unsigned long offset, unsigned int nr_entries) { + unsigned long begin = offset; unsigned long end = offset + nr_entries - 1; void (*swap_slot_free_notify)(struct block_device *, unsigned long); @@ -688,6 +689,7 @@ static void swap_range_free(struct swap_info_struct *si, unsigned long offset, swap_slot_free_notify(si->bdev, offset); offset++; } + clear_shadow_from_swap_cache(si->type, begin, end); } static int scan_swap_map_slots(struct swap_info_struct *si, diff --git a/mm/vmscan.c b/mm/vmscan.c index 13a02d9..fef02dd 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -904,7 +904,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, if (PageSwapCache(page)) { swp_entry_t swap = { .val = page_private(page) }; mem_cgroup_swapout(page, swap); - __delete_from_swap_cache(page); + __delete_from_swap_cache(page, NULL); xa_unlock_irqrestore(&mapping->i_pages, flags); put_swap_page(page, swap); workingset_eviction(page, target_memcg); -- 1.8.3.1