mm/damon/vaddr,paddr: support pageout prioritization

This makes the default monitoring primitives for virtual address spaces
and the physical address sapce to support memory regions prioritization
for 'PAGEOUT' DAMOS action.  It calculates hotness of each region as
weighted sum of 'nr_accesses' and 'age' of the region and get the
priority score as reverse of the hotness, so that cold regions can be
paged out first.

Link: https://lkml.kernel.org/r/20211019150731.16699-9-sj@kernel.org
Signed-off-by: SeongJae Park <sj@kernel.org>
Cc: Amit Shah <amit@kernel.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: David Woodhouse <dwmw@amazon.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Leonard Foerster <foersleo@amazon.de>
Cc: Marco Elver <elver@google.com>
Cc: Markus Boehme <markubo@amazon.de>
Cc: Shakeel Butt <shakeelb@google.com>
Cc: Shuah Khan <shuah@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/damon.h b/include/linux/damon.h
index 5d47ad9e3911bdca5bc8598d422fbd604b491b4d..1217566a0ebcff1c616d3ae8a7ccdeb7c8ac3574 100644 (file)
--- a/include/linux/damon.h
+++ b/include/linux/damon.h
@@ -421,6 +421,8 @@ bool damon_va_target_valid(void *t);
 void damon_va_cleanup(struct damon_ctx *ctx);
 int damon_va_apply_scheme(struct damon_ctx *context, struct damon_target *t,
                struct damon_region *r, struct damos *scheme);
+int damon_va_scheme_score(struct damon_ctx *context, struct damon_target *t,
+               struct damon_region *r, struct damos *scheme);
 void damon_va_set_primitives(struct damon_ctx *ctx);
 
 #endif /* CONFIG_DAMON_VADDR */
@@ -433,6 +435,8 @@ unsigned int damon_pa_check_accesses(struct damon_ctx *ctx);
 bool damon_pa_target_valid(void *t);
 int damon_pa_apply_scheme(struct damon_ctx *context, struct damon_target *t,
                struct damon_region *r, struct damos *scheme);
+int damon_pa_scheme_score(struct damon_ctx *context, struct damon_target *t,
+               struct damon_region *r, struct damos *scheme);
 void damon_pa_set_primitives(struct damon_ctx *ctx);
 
 #endif /* CONFIG_DAMON_PADDR */

diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c
index 957ada55de77b468eb9da016dfd0b60d28ef762d..a496d6f203d64c1aa6f7de2e50a39f18c8c61a1f 100644 (file)
--- a/mm/damon/paddr.c
+++ b/mm/damon/paddr.c
@@ -246,6 +246,19 @@ int damon_pa_apply_scheme(struct damon_ctx *ctx, struct damon_target *t,
        return 0;
 }
 
+int damon_pa_scheme_score(struct damon_ctx *context, struct damon_target *t,
+               struct damon_region *r, struct damos *scheme)
+{
+       switch (scheme->action) {
+       case DAMOS_PAGEOUT:
+               return damon_pageout_score(context, r, scheme);
+       default:
+               break;
+       }
+
+       return DAMOS_MAX_SCORE;
+}
+
 void damon_pa_set_primitives(struct damon_ctx *ctx)
 {
        ctx->primitive.init = NULL;
@@ -256,4 +269,5 @@ void damon_pa_set_primitives(struct damon_ctx *ctx)
        ctx->primitive.target_valid = damon_pa_target_valid;
        ctx->primitive.cleanup = NULL;
        ctx->primitive.apply_scheme = damon_pa_apply_scheme;
+       ctx->primitive.get_scheme_score = damon_pa_scheme_score;
 }

diff --git a/mm/damon/prmtv-common.c b/mm/damon/prmtv-common.c
index 7e62ee54fb5435c8df8a30528e3987d7eec57c52..92a04f5831d6bdab339e472c4fafa1da932c4630 100644 (file)
--- a/mm/damon/prmtv-common.c
+++ b/mm/damon/prmtv-common.c
@@ -85,3 +85,49 @@ void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr)
        put_page(page);
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 }
+
+#define DAMON_MAX_SUBSCORE     (100)
+#define DAMON_MAX_AGE_IN_LOG   (32)
+
+int damon_pageout_score(struct damon_ctx *c, struct damon_region *r,
+                       struct damos *s)
+{
+       unsigned int max_nr_accesses;
+       int freq_subscore;
+       unsigned int age_in_sec;
+       int age_in_log, age_subscore;
+       unsigned int freq_weight = s->quota.weight_nr_accesses;
+       unsigned int age_weight = s->quota.weight_age;
+       int hotness;
+
+       max_nr_accesses = c->aggr_interval / c->sample_interval;
+       freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
+
+       age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
+       for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
+                       age_in_log++, age_in_sec >>= 1)
+               ;
+
+       /* If frequency is 0, higher age means it's colder */
+       if (freq_subscore == 0)
+               age_in_log *= -1;
+
+       /*
+        * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
+        * Scale it to be in [0, 100] and set it as age subscore.
+        */
+       age_in_log += DAMON_MAX_AGE_IN_LOG;
+       age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
+               DAMON_MAX_AGE_IN_LOG / 2;
+
+       hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
+       if (freq_weight + age_weight)
+               hotness /= freq_weight + age_weight;
+       /*
+        * Transform it to fit in [0, DAMOS_MAX_SCORE]
+        */
+       hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
+
+       /* Return coldness of the region */
+       return DAMOS_MAX_SCORE - hotness;
+}

diff --git a/mm/damon/prmtv-common.h b/mm/damon/prmtv-common.h
index 7093d19e5d42868425a8620424955047307e9816..61f27037603e19ac29fec50ee232cf536f0cbb01 100644 (file)
--- a/mm/damon/prmtv-common.h
+++ b/mm/damon/prmtv-common.h
@@ -15,3 +15,6 @@ struct page *damon_get_page(unsigned long pfn);
 
 void damon_ptep_mkold(pte_t *pte, struct mm_struct *mm, unsigned long addr);
 void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr);
+
+int damon_pageout_score(struct damon_ctx *c, struct damon_region *r,
+                       struct damos *s);

diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
index 758501b8d97db6ed5d2004b37c7ed4129c029782..675cd8c7df9b875dc7d2ff0a3a41c38044fca949 100644 (file)
--- a/mm/damon/vaddr.c
+++ b/mm/damon/vaddr.c
@@ -633,6 +633,20 @@ int damon_va_apply_scheme(struct damon_ctx *ctx, struct damon_target *t,
        return damos_madvise(t, r, madv_action);
 }
 
+int damon_va_scheme_score(struct damon_ctx *context, struct damon_target *t,
+               struct damon_region *r, struct damos *scheme)
+{
+
+       switch (scheme->action) {
+       case DAMOS_PAGEOUT:
+               return damon_pageout_score(context, r, scheme);
+       default:
+               break;
+       }
+
+       return DAMOS_MAX_SCORE;
+}
+
 void damon_va_set_primitives(struct damon_ctx *ctx)
 {
        ctx->primitive.init = damon_va_init;
@@ -643,6 +657,7 @@ void damon_va_set_primitives(struct damon_ctx *ctx)
        ctx->primitive.target_valid = damon_va_target_valid;
        ctx->primitive.cleanup = NULL;
        ctx->primitive.apply_scheme = damon_va_apply_scheme;
+       ctx->primitive.get_scheme_score = damon_va_scheme_score;
 }
 
 #include "vaddr-test.h"