mm/damon/schemes: implement size quota for schemes application speed control

There could be arbitrarily large memory regions fulfilling the target
data access pattern of a DAMON-based operation scheme.  In the case,
applying the action of the scheme could incur too high overhead.  To
provide an intuitive way for avoiding it, this implements a feature
called size quota.  If the quota is set, DAMON tries to apply the action
only up to the given amount of memory regions within a given time
window.

Link: https://lkml.kernel.org/r/20211019150731.16699-3-sj@kernel.org
Signed-off-by: SeongJae Park <sj@kernel.org>
Cc: Amit Shah <amit@kernel.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Rientjes <rientjes@google.com>
Cc: David Woodhouse <dwmw@amazon.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Leonard Foerster <foersleo@amazon.de>
Cc: Marco Elver <elver@google.com>
Cc: Markus Boehme <markubo@amazon.de>
Cc: Shakeel Butt <shakeelb@google.com>
Cc: Shuah Khan <shuah@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/damon.h b/include/linux/damon.h
index 9a327bc787b522ed5e69f3047483e1f736995239..3a1ce9d9921c8138dc124b7ba46d64b9c5a9ab2f 100644 (file)
--- a/include/linux/damon.h
+++ b/include/linux/damon.h
@@ -89,6 +89,26 @@ enum damos_action {
        DAMOS_STAT,             /* Do nothing but only record the stat */
 };
 
+/**
+ * struct damos_quota - Controls the aggressiveness of the given scheme.
+ * @sz:                        Maximum bytes of memory that the action can be applied.
+ * @reset_interval:    Charge reset interval in milliseconds.
+ *
+ * To avoid consuming too much CPU time or IO resources for applying the
+ * &struct damos->action to large memory, DAMON allows users to set a size
+ * quota.  The quota can be set by writing non-zero values to &sz.  If the size
+ * quota is set, DAMON tries to apply the action only up to &sz bytes within
+ * &reset_interval.
+ */
+struct damos_quota {
+       unsigned long sz;
+       unsigned long reset_interval;
+
+/* private: For charging the quota */
+       unsigned long charged_sz;
+       unsigned long charged_from;
+};
+
 /**
  * struct damos - Represents a Data Access Monitoring-based Operation Scheme.
  * @min_sz_region:     Minimum size of target regions.
@@ -98,13 +118,20 @@ enum damos_action {
  * @min_age_region:    Minimum age of target regions.
  * @max_age_region:    Maximum age of target regions.
  * @action:            &damo_action to be applied to the target regions.
+ * @quota:             Control the aggressiveness of this scheme.
  * @stat_count:                Total number of regions that this scheme is applied.
  * @stat_sz:           Total size of regions that this scheme is applied.
  * @list:              List head for siblings.
  *
- * For each aggregation interval, DAMON applies @action to monitoring target
- * regions fit in the condition and updates the statistics.  Note that both
- * the minimums and the maximums are inclusive.
+ * For each aggregation interval, DAMON finds regions which fit in the
+ * condition (&min_sz_region, &max_sz_region, &min_nr_accesses,
+ * &max_nr_accesses, &min_age_region, &max_age_region) and applies &action to
+ * those.  To avoid consuming too much CPU time or IO resources for the
+ * &action, &quota is used.
+ *
+ * After applying the &action to each region, &stat_count and &stat_sz is
+ * updated to reflect the number of regions and total size of regions that the
+ * &action is applied.
  */
 struct damos {
        unsigned long min_sz_region;
@@ -114,6 +141,7 @@ struct damos {
        unsigned int min_age_region;
        unsigned int max_age_region;
        enum damos_action action;
+       struct damos_quota quota;
        unsigned long stat_count;
        unsigned long stat_sz;
        struct list_head list;
@@ -310,7 +338,7 @@ struct damos *damon_new_scheme(
                unsigned long min_sz_region, unsigned long max_sz_region,
                unsigned int min_nr_accesses, unsigned int max_nr_accesses,
                unsigned int min_age_region, unsigned int max_age_region,
-               enum damos_action action);
+               enum damos_action action, struct damos_quota *quota);
 void damon_add_scheme(struct damon_ctx *ctx, struct damos *s);
 void damon_destroy_scheme(struct damos *s);
 

diff --git a/mm/damon/core.c b/mm/damon/core.c
index 2f6785737902dd98d326ba9d041e18318ec8a274..cce14a0d5c725eb03dc9187cff671ce01c1b97f5 100644 (file)
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -89,7 +89,7 @@ struct damos *damon_new_scheme(
                unsigned long min_sz_region, unsigned long max_sz_region,
                unsigned int min_nr_accesses, unsigned int max_nr_accesses,
                unsigned int min_age_region, unsigned int max_age_region,
-               enum damos_action action)
+               enum damos_action action, struct damos_quota *quota)
 {
        struct damos *scheme;
 
@@ -107,6 +107,11 @@ struct damos *damon_new_scheme(
        scheme->stat_sz = 0;
        INIT_LIST_HEAD(&scheme->list);
 
+       scheme->quota.sz = quota->sz;
+       scheme->quota.reset_interval = quota->reset_interval;
+       scheme->quota.charged_sz = 0;
+       scheme->quota.charged_from = 0;
+
        return scheme;
 }
 
@@ -530,15 +535,25 @@ static void kdamond_reset_aggregated(struct damon_ctx *c)
        }
 }
 
+static void damon_split_region_at(struct damon_ctx *ctx,
+               struct damon_target *t, struct damon_region *r,
+               unsigned long sz_r);
+
 static void damon_do_apply_schemes(struct damon_ctx *c,
                                   struct damon_target *t,
                                   struct damon_region *r)
 {
        struct damos *s;
-       unsigned long sz;
 
        damon_for_each_scheme(s, c) {
-               sz = r->ar.end - r->ar.start;
+               struct damos_quota *quota = &s->quota;
+               unsigned long sz = r->ar.end - r->ar.start;
+
+               /* Check the quota */
+               if (quota->sz && quota->charged_sz >= quota->sz)
+                       continue;
+
+               /* Check the target regions condition */
                if (sz < s->min_sz_region || s->max_sz_region < sz)
                        continue;
                if (r->nr_accesses < s->min_nr_accesses ||
@@ -546,22 +561,51 @@ static void damon_do_apply_schemes(struct damon_ctx *c,
                        continue;
                if (r->age < s->min_age_region || s->max_age_region < r->age)
                        continue;
-               s->stat_count++;
-               s->stat_sz += sz;
-               if (c->primitive.apply_scheme)
+
+               /* Apply the scheme */
+               if (c->primitive.apply_scheme) {
+                       if (quota->sz && quota->charged_sz + sz > quota->sz) {
+                               sz = ALIGN_DOWN(quota->sz - quota->charged_sz,
+                                               DAMON_MIN_REGION);
+                               if (!sz)
+                                       goto update_stat;
+                               damon_split_region_at(c, t, r, sz);
+                       }
                        c->primitive.apply_scheme(c, t, r, s);
+                       quota->charged_sz += sz;
+               }
                if (s->action != DAMOS_STAT)
                        r->age = 0;
+
+update_stat:
+               s->stat_count++;
+               s->stat_sz += sz;
        }
 }
 
 static void kdamond_apply_schemes(struct damon_ctx *c)
 {
        struct damon_target *t;
-       struct damon_region *r;
+       struct damon_region *r, *next_r;
+       struct damos *s;
+
+       damon_for_each_scheme(s, c) {
+               struct damos_quota *quota = &s->quota;
+
+               if (!quota->sz)
+                       continue;
+
+               /* New charge window starts */
+               if (time_after_eq(jiffies, quota->charged_from +
+                                       msecs_to_jiffies(
+                                               quota->reset_interval))) {
+                       quota->charged_from = jiffies;
+                       quota->charged_sz = 0;
+               }
+       }
 
        damon_for_each_target(t, c) {
-               damon_for_each_region(r, t)
+               damon_for_each_region_safe(r, next_r, t)
                        damon_do_apply_schemes(c, t, r);
        }
 }

diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c
index c90988a20fa4f515fa6ecc0437f064b074c97828..a04bd50cc4c4e607f527ff41c928914a23a7ca02 100644 (file)
--- a/mm/damon/dbgfs.c
+++ b/mm/damon/dbgfs.c
@@ -188,6 +188,8 @@ static struct damos **str_to_schemes(const char *str, ssize_t len,
 
        *nr_schemes = 0;
        while (pos < len && *nr_schemes < max_nr_schemes) {
+               struct damos_quota quota = {};
+
                ret = sscanf(&str[pos], "%lu %lu %u %u %u %u %u%n",
                                &min_sz, &max_sz, &min_nr_a, &max_nr_a,
                                &min_age, &max_age, &action, &parsed);
@@ -200,7 +202,7 @@ static struct damos **str_to_schemes(const char *str, ssize_t len,
 
                pos += parsed;
                scheme = damon_new_scheme(min_sz, max_sz, min_nr_a, max_nr_a,
-                               min_age, max_age, action);
+                               min_age, max_age, action, &quota);
                if (!scheme)
                        goto fail;