mm/vmscan: protect the workingset on anonymous LRU

In current implementation, newly created or swap-in anonymous page is
started on active list.  Growing active list results in rebalancing
active/inactive list so old pages on active list are demoted to inactive
list.  Hence, the page on active list isn't protected at all.

Following is an example of this situation.

Assume that 50 hot pages on active list.  Numbers denote the number of
pages on active/inactive list (active | inactive).

1. 50 hot pages on active list
50(h) | 0

2. workload: 50 newly created (used-once) pages
50(uo) | 50(h)

3. workload: another 50 newly created (used-once) pages
50(uo) | 50(uo), swap-out 50(h)

This patch tries to fix this issue.  Like as file LRU, newly created or
swap-in anonymous pages will be inserted to the inactive list.  They are
promoted to active list if enough reference happens.  This simple
modification changes the above example as following.

1. 50 hot pages on active list
50(h) | 0

2. workload: 50 newly created (used-once) pages
50(h) | 50(uo)

3. workload: another 50 newly created (used-once) pages
50(h) | 50(uo), swap-out 50(uo)

As you can see, hot pages on active list would be protected.

Note that, this implementation has a drawback that the page cannot be
promoted and will be swapped-out if re-access interval is greater than the
size of inactive list but less than the size of total(active+inactive).
To solve this potential issue, following patch will apply workingset
detection similar to the one that's already applied to file LRU.

Signed-off-by: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Acked-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Hugh Dickins <hughd@google.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: Minchan Kim <minchan@kernel.org>
Link: http://lkml.kernel.org/r/1595490560-15117-3-git-send-email-iamjoonsoo.kim@lge.com
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/swap.h b/include/linux/swap.h
index 7eb59bc552a5bee63e30844d7b6f3d8855143f96..51ec9cdb92c0fae8d82756942e825baf716aa1a9 100644 (file)
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -352,7 +352,7 @@ extern void deactivate_page(struct page *page);
 extern void mark_page_lazyfree(struct page *page);
 extern void swap_setup(void);
 
-extern void lru_cache_add_active_or_unevictable(struct page *page,
+extern void lru_cache_add_inactive_or_unevictable(struct page *page,
                                                struct vm_area_struct *vma);
 
 /* linux/mm/vmscan.c */

diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
index 25de10c904e6e9a3c19b37cc82a3cb838de22234..49047d479c576352e36c1f77d7393409bbca6b77 100644 (file)
--- a/kernel/events/uprobes.c
+++ b/kernel/events/uprobes.c
@@ -184,7 +184,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
        if (new_page) {
                get_page(new_page);
                page_add_new_anon_rmap(new_page, vma, addr, false);
-               lru_cache_add_active_or_unevictable(new_page, vma);
+               lru_cache_add_inactive_or_unevictable(new_page, vma);
        } else
                /* no new page, just dec_mm_counter for old_page */
                dec_mm_counter(mm, MM_ANONPAGES);

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 206f52b36ffb9e5580eabf3d95b5294f88b18955..863c495776d7ca9221bfece8f245027771154e61 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -640,7 +640,7 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf,
                entry = mk_huge_pmd(page, vma->vm_page_prot);
                entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
                page_add_new_anon_rmap(page, vma, haddr, true);
-               lru_cache_add_active_or_unevictable(page, vma);
+               lru_cache_add_inactive_or_unevictable(page, vma);
                pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
                set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
                add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);

diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index b52bd46ad146df634ff9956a0467378f6b65b861..15a9af791014c5602d9939606503c174837da08d 100644 (file)
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1173,7 +1173,7 @@ static void collapse_huge_page(struct mm_struct *mm,
        spin_lock(pmd_ptl);
        BUG_ON(!pmd_none(*pmd));
        page_add_new_anon_rmap(new_page, vma, address, true);
-       lru_cache_add_active_or_unevictable(new_page, vma);
+       lru_cache_add_inactive_or_unevictable(new_page, vma);
        pgtable_trans_huge_deposit(mm, pmd, pgtable);
        set_pmd_at(mm, address, pmd, _pmd);
        update_mmu_cache_pmd(vma, address, pmd);

diff --git a/mm/memory.c b/mm/memory.c
index c39a13b09602cc60f013dbe503e14ba32d239b65..6fe8b5b22c575b9775347abd33c8951017b33672 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2715,7 +2715,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
                 */
                ptep_clear_flush_notify(vma, vmf->address, vmf->pte);
                page_add_new_anon_rmap(new_page, vma, vmf->address, false);
-               lru_cache_add_active_or_unevictable(new_page, vma);
+               lru_cache_add_inactive_or_unevictable(new_page, vma);
                /*
                 * We call the notify macro here because, when using secondary
                 * mmu page tables (such as kvm shadow page tables), we want the
@@ -3266,10 +3266,9 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
        /* ksm created a completely new copy */
        if (unlikely(page != swapcache && swapcache)) {
                page_add_new_anon_rmap(page, vma, vmf->address, false);
-               lru_cache_add_active_or_unevictable(page, vma);
+               lru_cache_add_inactive_or_unevictable(page, vma);
        } else {
                do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
-               activate_page(page);
        }
 
        swap_free(entry);
@@ -3414,7 +3413,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
 
        inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
        page_add_new_anon_rmap(page, vma, vmf->address, false);
-       lru_cache_add_active_or_unevictable(page, vma);
+       lru_cache_add_inactive_or_unevictable(page, vma);
 setpte:
        set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
 
@@ -3672,7 +3671,7 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct page *page)
        if (write && !(vma->vm_flags & VM_SHARED)) {
                inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
                page_add_new_anon_rmap(page, vma, vmf->address, false);
-               lru_cache_add_active_or_unevictable(page, vma);
+               lru_cache_add_inactive_or_unevictable(page, vma);
        } else {
                inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
                page_add_file_rmap(page, false);

diff --git a/mm/migrate.c b/mm/migrate.c
index d179657f8685c157b194fed756f40d9ad994ddb2..819e55130134cd7a1d4d7cd17ee1ab833924b324 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -2830,7 +2830,7 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
        inc_mm_counter(mm, MM_ANONPAGES);
        page_add_new_anon_rmap(page, vma, addr, false);
        if (!is_zone_device_page(page))
-               lru_cache_add_active_or_unevictable(page, vma);
+               lru_cache_add_inactive_or_unevictable(page, vma);
        get_page(page);
 
        if (flush) {

diff --git a/mm/swap.c b/mm/swap.c
index de257c0a89b1229cfa6aaa39c436a0c32e1c4043..9285e60c7d6e8949cf912b421c1ecd7951e560fb 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -476,23 +476,24 @@ void lru_cache_add(struct page *page)
 EXPORT_SYMBOL(lru_cache_add);
 
 /**
- * lru_cache_add_active_or_unevictable
+ * lru_cache_add_inactive_or_unevictable
  * @page:  the page to be added to LRU
  * @vma:   vma in which page is mapped for determining reclaimability
  *
- * Place @page on the active or unevictable LRU list, depending on its
+ * Place @page on the inactive or unevictable LRU list, depending on its
  * evictability.  Note that if the page is not evictable, it goes
  * directly back onto it's zone's unevictable list, it does NOT use a
  * per cpu pagevec.
  */
-void lru_cache_add_active_or_unevictable(struct page *page,
+void lru_cache_add_inactive_or_unevictable(struct page *page,
                                         struct vm_area_struct *vma)
 {
+       bool unevictable;
+
        VM_BUG_ON_PAGE(PageLRU(page), page);
 
-       if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED))
-               SetPageActive(page);
-       else if (!TestSetPageMlocked(page)) {
+       unevictable = (vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED;
+       if (unlikely(unevictable) && !TestSetPageMlocked(page)) {
                /*
                 * We use the irq-unsafe __mod_zone_page_stat because this
                 * counter is not modified from interrupt context, and the pte

diff --git a/mm/swapfile.c b/mm/swapfile.c
index 6c26916e95fd4a40400e8e9572dca81b52dab7e1..82183432fdd0ed083fa43f74985e4971e544c07f 100644 (file)
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -1915,7 +1915,7 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
                page_add_anon_rmap(page, vma, addr, false);
        } else { /* ksm created a completely new copy */
                page_add_new_anon_rmap(page, vma, addr, false);
-               lru_cache_add_active_or_unevictable(page, vma);
+               lru_cache_add_inactive_or_unevictable(page, vma);
        }
        swap_free(entry);
        /*

diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index b80419320c7d2160592e1842a19274f60dafa790..9a3d451402d7b222a86b8a5e9e9ab9020e1792a0 100644 (file)
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -123,7 +123,7 @@ static int mcopy_atomic_pte(struct mm_struct *dst_mm,
 
        inc_mm_counter(dst_mm, MM_ANONPAGES);
        page_add_new_anon_rmap(page, dst_vma, dst_addr, false);
-       lru_cache_add_active_or_unevictable(page, dst_vma);
+       lru_cache_add_inactive_or_unevictable(page, dst_vma);
 
        set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
 

diff --git a/mm/vmscan.c b/mm/vmscan.c
index e34fc04b7045c1e9cd23818d2ee66f041a2602dc..783cd7fdc61a34aef3f468b5701192aa1c7e2e8b 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -998,8 +998,6 @@ static enum page_references page_check_references(struct page *page,
                return PAGEREF_RECLAIM;
 
        if (referenced_ptes) {
-               if (PageSwapBacked(page))
-                       return PAGEREF_ACTIVATE;
                /*
                 * All mapped pages start out with page table
                 * references from the instantiating fault, so we need
@@ -1022,7 +1020,7 @@ static enum page_references page_check_references(struct page *page,
                /*
                 * Activate file-backed executable pages after first usage.
                 */
-               if (vm_flags & VM_EXEC)
+               if ((vm_flags & VM_EXEC) && !PageSwapBacked(page))
                        return PAGEREF_ACTIVATE;
 
                return PAGEREF_KEEP;