From 892970b760e2b08e1508533e63704432951e78af Mon Sep 17 00:00:00 2001 From: Xu Yu Date: Wed, 25 Dec 2019 02:59:48 +0800 Subject: [PATCH 0878/2944] alinux: mm, memcg: add memsli procfs switch interface to #26424368 Since memsli also records latency histogram for swapout and swapin, which are NOT in the slow memory path, the overhead of memsli could be nonnegligible in some specific scenarios. For example, in scenarios with frequent swapping out and in, memsli could introduce overhead of ~1% of total run time of the synthetic testcase. This adds procfs interface for memsli switch. The memsli feature is enabled by default, and you can now disable it by: $ echo 0 > /proc/memsli/enabled Apparently, you can check current memsli switch status by: $ cat /proc/memsli/enabled Note that disabling memsli at runtime will NOT clear the existing latency histogram. You still need to manually reset the specified latency histogram(s) by echo 0 into the corresponding cgroup control file(s). Signed-off-by: Xu Yu Reviewed-by: Yang Shi Reviewed-by: Xunlei Pang --- include/linux/memcontrol.h | 11 ++++++ mm/memcontrol.c | 93 ++++++++++++++++++++++++++++++++++++++++++++-- mm/memory.c | 8 ++-- mm/page_alloc.c | 9 +++-- mm/shmem.c | 4 +- mm/vmscan.c | 4 +- 6 files changed, 114 insertions(+), 15 deletions(-) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 32c54b7..26e63f6 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -937,6 +937,8 @@ void memcg_check_wmark_min_adj(struct task_struct *curr, struct alloc_context *ac); extern void memcg_lat_stat_update(enum mem_lat_stat_item sidx, u64 duration); +extern void memcg_lat_stat_start(u64 *start); +extern u64 memcg_lat_stat_end(u64 start); #else /* CONFIG_MEMCG */ #define MEM_CGROUP_ID_SHIFT 0 @@ -1292,6 +1294,15 @@ static inline void memcg_lat_stat_update(enum mem_lat_stat_item sidx, u64 duration) { } + +static inline void memcg_lat_stat_start(u64 *start) +{ +} + +static inline u64 memcg_lat_stat_end(u64 start) +{ + return 0; +} #endif /* CONFIG_MEMCG */ /* idx can be of type enum memcg_stat_item or node_stat_item */ diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b10f3c4..f11f4eb 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -71,6 +71,7 @@ #include #include #include "slab.h" +#include #include @@ -89,6 +90,9 @@ /* Kernel memory accounting disabled? */ static bool cgroup_memory_nokmem; +/* Cgroup memory SLI disabled? */ +static DEFINE_STATIC_KEY_FALSE(cgroup_memory_nosli); + /* Whether the swap controller is active */ #ifdef CONFIG_MEMCG_SWAP int do_swap_account __read_mostly; @@ -2461,7 +2465,7 @@ void mem_cgroup_handle_over_high(void) return; memcg = get_mem_cgroup_from_mm(current->mm); - start = ktime_get_ns(); + memcg_lat_stat_start(&start); reclaim_high(memcg, nr_pages, GFP_KERNEL); current->memcg_nr_pages_over_high = 0; @@ -2532,7 +2536,7 @@ void mem_cgroup_handle_over_high(void) out: memcg_lat_stat_update(MEM_LAT_MEMCG_DIRECT_RECLAIM, - (ktime_get_ns() - start)); + memcg_lat_stat_end(start)); css_put(&memcg->css); } @@ -2608,11 +2612,11 @@ static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, memcg_memory_event(mem_over_limit, MEMCG_MAX); - start = ktime_get_ns(); + memcg_lat_stat_start(&start); nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages, gfp_mask, may_swap); memcg_lat_stat_update(MEM_LAT_MEMCG_DIRECT_RECLAIM, - (ktime_get_ns() - start)); + memcg_lat_stat_end(start)); if (mem_cgroup_margin(mem_over_limit) >= nr_pages) goto retry; @@ -4627,6 +4631,9 @@ void memcg_lat_stat_update(enum mem_lat_stat_item sidx, u64 duration) struct mem_cgroup *memcg, *iter; enum mem_lat_count_t cidx; + if (static_branch_unlikely(&cgroup_memory_nosli)) + return; + if (mem_cgroup_disabled()) return; @@ -4641,6 +4648,22 @@ void memcg_lat_stat_update(enum mem_lat_stat_item sidx, u64 duration) css_put(&memcg->css); } +void memcg_lat_stat_start(u64 *start) +{ + if (!static_branch_unlikely(&cgroup_memory_nosli) && + !mem_cgroup_disabled()) + *start = ktime_get_ns(); +} + +u64 memcg_lat_stat_end(u64 start) +{ + if (!static_branch_unlikely(&cgroup_memory_nosli) && + !mem_cgroup_disabled()) + return ktime_get_ns() - start; + else + return 0; +} + static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) { struct mem_cgroup_threshold_ary *t; @@ -7754,6 +7777,56 @@ static int __init enable_cgroup_writeback_v1(char *s) __setup("cgwb_v1", enable_cgroup_writeback_v1); #endif +static int memsli_enabled_show(struct seq_file *m, void *v) +{ + seq_printf(m, "%d\n", !static_key_enabled(&cgroup_memory_nosli)); + return 0; +} + +static int memsli_enabled_open(struct inode *inode, struct file *file) +{ + return single_open(file, memsli_enabled_show, NULL); +} + +static ssize_t memsli_enabled_write(struct file *file, const char __user *ubuf, + size_t count, loff_t *ppos) +{ + char val = -1; + int ret = count; + + if (count < 1 || *ppos) { + ret = -EINVAL; + goto out; + } + + if (copy_from_user(&val, ubuf, 1)) { + ret = -EFAULT; + goto out; + } + + switch (val) { + case '0': + static_branch_enable(&cgroup_memory_nosli); + break; + case '1': + static_branch_disable(&cgroup_memory_nosli); + break; + default: + ret = -EINVAL; + } + +out: + return ret; +} + +static const struct file_operations memsli_enabled_fops = { + .open = memsli_enabled_open, + .read = seq_read, + .write = memsli_enabled_write, + .llseek = seq_lseek, + .release = single_release, +}; + /* * subsys_initcall() for memory controller. * @@ -7765,6 +7838,7 @@ static int __init enable_cgroup_writeback_v1(char *s) static int __init mem_cgroup_init(void) { int cpu, node; + struct proc_dir_entry *memsli_dir, *memsli_enabled_file; memcg_wmark_wq = alloc_workqueue("memcg_wmark", WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_FREEZABLE, @@ -7773,6 +7847,17 @@ static int __init mem_cgroup_init(void) if (!memcg_wmark_wq) return -ENOMEM; + memsli_dir = proc_mkdir("memsli", NULL); + if (!memsli_dir) + return -ENOMEM; + + memsli_enabled_file = proc_create("enabled", 0600, + memsli_dir, &memsli_enabled_fops); + if (!memsli_enabled_file) { + remove_proc_entry("memsli", NULL); + return -ENOMEM; + } + #ifdef CONFIG_MEMCG_KMEM /* * Kmem cache creation is mostly done with the slab_mutex held, diff --git a/mm/memory.c b/mm/memory.c index 058756f..55a8fe1 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4042,11 +4042,13 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) } if (!pte_present(vmf->orig_pte)) { - u64 start = ktime_get_ns(); - vm_fault_t retval = do_swap_page(vmf); + vm_fault_t retval; + u64 start; + memcg_lat_stat_start(&start); + retval = do_swap_page(vmf); memcg_lat_stat_update(MEM_LAT_DIRECT_SWAPIN, - (ktime_get_ns() - start)); + memcg_lat_stat_end(start)); return retval; } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ee9f49b..a1d3dc3 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -3726,14 +3726,15 @@ void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...) return NULL; psi_memstall_enter(&pflags); - start = ktime_get_ns(); + memcg_lat_stat_start(&start); noreclaim_flag = memalloc_noreclaim_save(); *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, prio); memalloc_noreclaim_restore(noreclaim_flag); - memcg_lat_stat_update(MEM_LAT_DIRECT_COMPACT, (ktime_get_ns() - start)); + memcg_lat_stat_update(MEM_LAT_DIRECT_COMPACT, + memcg_lat_stat_end(start)); psi_memstall_leave(&pflags); if (*compact_result <= COMPACT_INACTIVE) @@ -3941,7 +3942,7 @@ void fs_reclaim_release(gfp_t gfp_mask) /* We now go into synchronous reclaim */ cpuset_memory_pressure_bump(); psi_memstall_enter(&pflags); - start = ktime_get_ns(); + memcg_lat_stat_start(&start); fs_reclaim_acquire(gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); reclaim_state.reclaimed_slab = 0; @@ -3954,7 +3955,7 @@ void fs_reclaim_release(gfp_t gfp_mask) memalloc_noreclaim_restore(noreclaim_flag); fs_reclaim_release(gfp_mask); memcg_lat_stat_update(MEM_LAT_GLOBAL_DIRECT_RECLAIM, - (ktime_get_ns() - start)); + memcg_lat_stat_end(start)); psi_memstall_leave(&pflags); cond_resched(); diff --git a/mm/shmem.c b/mm/shmem.c index e02f257..db3edee 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1695,10 +1695,10 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, count_memcg_event_mm(charge_mm, PGMAJFAULT); } /* Here we actually start the io */ - start = ktime_get_ns(); + memcg_lat_stat_start(&start); page = shmem_swapin(swap, gfp, info, index); memcg_lat_stat_update(MEM_LAT_DIRECT_SWAPIN, - (ktime_get_ns() - start)); + memcg_lat_stat_end(start)); if (!page) { error = -ENOMEM; goto failed; diff --git a/mm/vmscan.c b/mm/vmscan.c index 36d2928..8916d999 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -916,13 +916,13 @@ static pageout_t pageout(struct page *page, struct address_space *mapping, SetPageReclaim(page); if (!current_is_kswapd()) - start = ktime_get_ns(); + memcg_lat_stat_start(&start); res = mapping->a_ops->writepage(page, &wbc); if (!current_is_kswapd()) memcg_lat_stat_update(global_reclaim(sc) ? MEM_LAT_GLOBAL_DIRECT_SWAPOUT : MEM_LAT_MEMCG_DIRECT_SWAPOUT, - (ktime_get_ns() - start)); + memcg_lat_stat_end(start)); if (res < 0) handle_write_error(mapping, page, res); if (res == AOP_WRITEPAGE_ACTIVATE) { -- 1.8.3.1