From ab4551efe9d9a6e513f1bf5f5c864055661e7f5e Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Wed, 3 Jun 2020 16:01:41 -0700 Subject: [PATCH 2334/2944] mm: memcontrol: convert page cache to a new mem_cgroup_charge() API task #30476868 commit 3fea5a499d57dec46043fcdb08e38eac1767bb0d upstream The try/commit/cancel protocol that memcg uses dates back to when pages used to be uncharged upon removal from the page cache, and thus couldn't be committed before the insertion had succeeded. Nowadays, pages are uncharged when they are physically freed; it doesn't matter whether the insertion was successful or not. For the page cache, the transaction dance has become unnecessary. Introduce a mem_cgroup_charge() function that simply charges a newly allocated page to a cgroup and sets up page->mem_cgroup in one single step. If the insertion fails, the caller doesn't have to do anything but free/put the page. Then switch the page cache over to this new API. Subsequent patches will also convert anon pages, but it needs a bit more prep work. Right now, memcg depends on page->mapping being already set up at the time of charging, so that it can maintain its own MEMCG_CACHE and MEMCG_RSS counters. For anon, page->mapping is set under the same pte lock under which the page is publishd, so a single charge point that can block doesn't work there just yet. The following prep patches will replace the private memcg counters with the generic vmstat counters, thus removing the page->mapping dependency, then complete the transition to the new single-point charge API and delete the old transactional scheme. v2: leave shmem swapcache when charging fails to avoid double IO (Joonsoo) v3: rebase on preceeding shmem simplification patch Signed-off-by: Johannes Weiner Signed-off-by: Andrew Morton Reviewed-by: Alex Shi Cc: Hugh Dickins Cc: Joonsoo Kim Cc: "Kirill A. Shutemov" Cc: Michal Hocko Cc: Roman Gushchin Cc: Shakeel Butt Cc: Balbir Singh Link: http://lkml.kernel.org/r/20200508183105.225460-6-hannes@cmpxchg.org Signed-off-by: Linus Torvalds Signed-off-by: zhongjiang-ali Reviewed-by: Xunlei Pang --- include/linux/memcontrol.h | 10 ++++++++ mm/filemap.c | 25 ++++++++---------- mm/memcontrol.c | 29 +++++++++++++++++++-- mm/shmem.c | 64 +++++++++++++++++----------------------------- 4 files changed, 72 insertions(+), 56 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index bd57e243..0376189 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -432,6 +432,10 @@ int mem_cgroup_try_charge_delay(struct page *page, struct mm_struct *mm, void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg, bool lrucare); void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg); + +int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask, + bool lrucare); + void mem_cgroup_uncharge(struct page *page); void mem_cgroup_uncharge_list(struct list_head *page_list); @@ -969,6 +973,12 @@ static inline void mem_cgroup_cancel_charge(struct page *page, { } +static inline int mem_cgroup_charge(struct page *page, struct mm_struct *mm, + gfp_t gfp_mask, bool lrucare) +{ + return 0; +} + static inline void mem_cgroup_uncharge(struct page *page) { } diff --git a/mm/filemap.c b/mm/filemap.c index e77c5d8..39189e0 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -854,30 +854,31 @@ static int __add_to_page_cache_locked(struct page *page, void **shadowp) { int huge = PageHuge(page); - struct mem_cgroup *memcg; int error; VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(PageSwapBacked(page), page); + get_page(page); + page->mapping = mapping; + page->index = offset; + if (!huge) { - error = mem_cgroup_try_charge(page, current->mm, - gfp_mask, &memcg); - if (error) + error = mem_cgroup_charge(page, current->mm, gfp_mask, false); + if (error) { + page->mapping = NULL; + put_page(page); return error; + } } error = radix_tree_maybe_preload(gfp_mask & GFP_RECLAIM_MASK); if (error) { - if (!huge) - mem_cgroup_cancel_charge(page, memcg); + page->mapping = NULL; + put_page(page); return error; } - get_page(page); - page->mapping = mapping; - page->index = offset; - xa_lock_irq(&mapping->i_pages); error = page_cache_tree_insert(mapping, page, shadowp); radix_tree_preload_end(); @@ -888,16 +889,12 @@ static int __add_to_page_cache_locked(struct page *page, if (!huge) __inc_node_page_state(page, NR_FILE_PAGES); xa_unlock_irq(&mapping->i_pages); - if (!huge) - mem_cgroup_commit_charge(page, memcg, false); trace_mm_filemap_add_to_page_cache(page); return 0; err_insert: page->mapping = NULL; /* Leave page->index set: truncation relies upon it */ xa_unlock_irq(&mapping->i_pages); - if (!huge) - mem_cgroup_cancel_charge(page, memcg); put_page(page); return error; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b6f37d2..5b9f176 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -7747,6 +7747,33 @@ void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg) cancel_charge(memcg, nr_pages); } +/** + * mem_cgroup_charge - charge a newly allocated page to a cgroup + * @page: page to charge + * @mm: mm context of the victim + * @gfp_mask: reclaim mode + * @lrucare: page might be on the LRU already + * + * Try to charge @page to the memcg that @mm belongs to, reclaiming + * pages according to @gfp_mask if necessary. + * + * Returns 0 on success. Otherwise, an error code is returned. + */ +int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask, + bool lrucare) +{ + struct mem_cgroup *memcg; + int ret; + + VM_BUG_ON_PAGE(!page->mapping, page); + + ret = mem_cgroup_try_charge(page, mm, gfp_mask, &memcg); + if (ret) + return ret; + mem_cgroup_commit_charge(page, memcg, lrucare); + return 0; +} + struct uncharge_gather { struct mem_cgroup *memcg; unsigned long pgpgout; @@ -7794,8 +7821,6 @@ static void uncharge_batch(const struct uncharge_gather *ug) static void uncharge_page(struct page *page, struct uncharge_gather *ug) { VM_BUG_ON_PAGE(PageLRU(page), page); - VM_BUG_ON_PAGE(page_count(page) && !is_zone_device_page(page) && - !PageHWPoison(page) , page); if (!page->mem_cgroup) return; diff --git a/mm/shmem.c b/mm/shmem.c index 35449f9..d697557 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -590,7 +590,8 @@ static inline bool is_huge_enabled(struct shmem_sb_info *sbinfo) */ static int shmem_add_to_page_cache(struct page *page, struct address_space *mapping, - pgoff_t index, void *expected, gfp_t gfp) + pgoff_t index, void *expected, gfp_t gfp, + struct mm_struct *charge_mm) { int error, nr = hpage_nr_pages(page); @@ -604,6 +605,14 @@ static int shmem_add_to_page_cache(struct page *page, page->mapping = mapping; page->index = index; + error = mem_cgroup_charge(page, charge_mm, gfp, PageSwapCache(page)); + if (error) { + page->mapping = NULL; + page_ref_sub(page, nr); + return error; + } + cgroup_throttle_swaprate(page, gfp); + xa_lock_irq(&mapping->i_pages); if (PageTransHuge(page)) { void __rcu **results; @@ -1204,7 +1213,7 @@ static int shmem_unuse_inode(struct shmem_inode_info *info, */ if (!error) error = shmem_add_to_page_cache(*pagep, mapping, index, - radswap, gfp); + radswap, gfp, current->mm); if (error != -ENOMEM) { /* * Truncation and eviction use free_swap_and_cache(), which @@ -1229,7 +1238,6 @@ int shmem_unuse(swp_entry_t swap, struct page *page) { struct list_head *this, *next; struct shmem_inode_info *info; - struct mem_cgroup *memcg; int error = 0; /* @@ -1244,7 +1252,7 @@ int shmem_unuse(swp_entry_t swap, struct page *page) * the shmem_swaplist_mutex which might hold up shmem_writepage(). * Charged back to the user (not to caller) when swap account is used. */ - error = mem_cgroup_try_charge_delay(page, current->mm, GFP_KERNEL, &memcg); + error = mem_cgroup_charge(page, current->mm, GFP_KERNEL, PageSwapCache(page)); if (error) goto out; /* No radix_tree_preload: swap entry keeps a place for page in tree */ @@ -1267,9 +1275,7 @@ int shmem_unuse(swp_entry_t swap, struct page *page) if (error) { if (error != -ENOMEM) error = 0; - mem_cgroup_cancel_charge(page, memcg); - } else - mem_cgroup_commit_charge(page, memcg, true); + } out: unlock_page(page); put_page(page); @@ -1632,7 +1638,6 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, struct shmem_inode_info *info = SHMEM_I(inode); struct shmem_sb_info *sbinfo; struct mm_struct *charge_mm; - struct mem_cgroup *memcg; struct page *page; swp_entry_t swap; enum sgp_type sgp_huge = sgp; @@ -1722,19 +1727,12 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, goto failed; } - error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg); + error = shmem_add_to_page_cache(page, mapping, index, + swp_to_radix_entry(swap), gfp, + charge_mm); if (error) goto failed; - error = shmem_add_to_page_cache(page, mapping, index, - swp_to_radix_entry(swap), gfp); - if (error) { - mem_cgroup_cancel_charge(page, memcg); - goto failed; - } - - mem_cgroup_commit_charge(page, memcg, true); - spin_lock_irq(&info->lock); info->swapped--; shmem_recalc_inode(inode); @@ -1814,21 +1812,16 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, if (sgp == SGP_WRITE) __SetPageReferenced(page); - error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg); - if (error) - goto unacct; error = radix_tree_maybe_preload_order(gfp & GFP_RECLAIM_MASK, compound_order(page)); if (!error) { error = shmem_add_to_page_cache(page, mapping, hindex, - NULL, gfp & GFP_RECLAIM_MASK); + NULL, gfp & GFP_RECLAIM_MASK, + charge_mm); radix_tree_preload_end(); } - if (error) { - mem_cgroup_cancel_charge(page, memcg); + if (error) goto unacct; - } - mem_cgroup_commit_charge(page, memcg, false); lru_cache_add_anon(page); spin_lock_irq(&info->lock); @@ -2243,7 +2236,6 @@ static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, struct address_space *mapping = inode->i_mapping; gfp_t gfp = mapping_gfp_mask(mapping); pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); - struct mem_cgroup *memcg; spinlock_t *ptl; void *page_kaddr; struct page *page; @@ -2293,20 +2285,14 @@ static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, if (unlikely(offset >= max_off)) goto out_release; - ret = mem_cgroup_try_charge_delay(page, dst_mm, gfp, &memcg); - if (ret) - goto out_release; - ret = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK); if (!ret) { ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL, - gfp & GFP_RECLAIM_MASK); + gfp & GFP_RECLAIM_MASK, dst_mm); radix_tree_preload_end(); } if (ret) - goto out_release_uncharge; - - mem_cgroup_commit_charge(page, memcg, false); + goto out_release; _dst_pte = mk_pte(page, dst_vma->vm_page_prot); if (dst_vma->vm_flags & VM_WRITE) @@ -2327,11 +2313,11 @@ static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, ret = -EFAULT; max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); if (unlikely(offset >= max_off)) - goto out_release_uncharge_unlock; + goto out_release_unlock; ret = -EEXIST; if (!pte_none(*dst_pte)) - goto out_release_uncharge_unlock; + goto out_release_unlock; lru_cache_add_anon(page); @@ -2352,12 +2338,10 @@ static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, ret = 0; out: return ret; -out_release_uncharge_unlock: +out_release_unlock: pte_unmap_unlock(dst_pte, ptl); ClearPageDirty(page); delete_from_page_cache(page); -out_release_uncharge: - mem_cgroup_cancel_charge(page, memcg); out_release: unlock_page(page); put_page(page); -- 1.8.3.1