test: Allow running tests multiple times

Some tests can have race conditions which are hard to detect on a single
one. Add a way to run tests more than once, to help with this.

Each individual test is run the requested number of times before moving
to the next test. If any runs failed, a message is shown.

This is most useful when running a single test, since running all tests
multiple times can take a while.

Signed-off-by: Simon Glass <sjg@chromium.org>

diff --git a/arch/sandbox/cpu/spl.c b/arch/sandbox/cpu/spl.c
index fe5d44d36edc4010fac4a74a683c8bf4aabc088b..1d49a9bd1020f32d1624aff771937f0bef1f9433 100644 (file)
--- a/arch/sandbox/cpu/spl.c
+++ b/arch/sandbox/cpu/spl.c
@@ -89,7 +89,7 @@ void spl_board_init(void)
                int ret;
 
                ret = ut_run_list("spl", NULL, tests, count,
-                                 state->select_unittests);
+                                 state->select_unittests, 1);
                /* continue execution into U-Boot */
        }
 }

diff --git a/doc/develop/tests_sandbox.rst b/doc/develop/tests_sandbox.rst
index 40cf8ecdd7f8c8012a92bb7e03ba46488b2c0a0e..8e42a32afb9acef99299e3b9802794424472db34 100644 (file)
--- a/doc/develop/tests_sandbox.rst
+++ b/doc/develop/tests_sandbox.rst
@@ -119,6 +119,30 @@ You can easily use gdb on these tests, without needing --gdbserver::
 You can then single-step and look at variables as needed.
 
 
+Running tests multiple times
+----------------------------
+
+Some tests can have race conditions which are hard to detect on a single
+one. It is possible to run each individual test multiple times, before moving
+to the next test, with the '-r' flag.
+
+This is most useful when running a single test, since running all tests
+multiple times can take a while.
+
+For example::
+
+   => ut dm -r1000 dm_test_rtc_set_get
+   ...
+   Test: dm_test_rtc_set_get: rtc.c (flat tree)
+   Test: dm_test_rtc_set_get: rtc.c
+   test/dm/rtc.c:257, dm_test_rtc_reset(): old_base_time == base_time: Expected 0x62e7453c (1659323708), got 0x62e7453d (1659323709)
+   Test: dm_test_rtc_set_get: rtc.c (flat tree)
+   Test: dm_test_rtc_set_get: rtc.c
+   Test: dm_test_rtc_set_get: rtc.c (flat tree)
+   ...
+   Test dm_test_rtc_reset failed 3 times
+
+
 Running sandbox_spl tests directly
 ----------------------------------
 

diff --git a/include/test/test.h b/include/test/test.h
index c888d68b1ed7240dde67a09f0424f8d1dfa5c838..2b68331b5468fa5a21ca3d89f1eaff69c44d1273 100644 (file)
--- a/include/test/test.h
+++ b/include/test/test.h
@@ -20,6 +20,7 @@
  * @testdev: Test device
  * @force_fail_alloc: Force all memory allocs to fail
  * @skip_post_probe: Skip uclass post-probe processing
+ * @runs_per_test: Number of times to run each test (typically 1)
  * @expect_str: Temporary string used to hold expected string value
  * @actual_str: Temporary string used to hold actual string value
  */
@@ -32,6 +33,7 @@ struct unit_test_state {
        struct udevice *testdev;
        int force_fail_alloc;
        int skip_post_probe;
+       int runs_per_test;
        char expect_str[512];
        char actual_str[512];
 };

diff --git a/include/test/ut.h b/include/test/ut.h
index 18740f5807c662eccbd1630492a86411415d9e3f..f7d1d18f7c17b9e77c588321ad095198d63c8d37 100644 (file)
--- a/include/test/ut.h
+++ b/include/test/ut.h
@@ -403,9 +403,10 @@ void test_set_state(struct unit_test_state *uts);
  * @count: Number of tests to run
  * @select_name: Name of a single test to run (from the list provided). If NULL
  *     then all tests are run
+ * @runs_per_test: Number of times to run each test (typically 1)
  * Return: 0 if all tests passed, -1 if any failed
  */
 int ut_run_list(const char *name, const char *prefix, struct unit_test *tests,
-               int count, const char *select_name);
+               int count, const char *select_name, int runs_per_test);
 
 #endif

diff --git a/test/cmd_ut.c b/test/cmd_ut.c
index 3789c6b784c03778af7fcdb58d3c14fc2edd24dc..11c219b48acc59e1dbbd3cdf75d243ace5b5865c 100644 (file)
--- a/test/cmd_ut.c
+++ b/test/cmd_ut.c
@@ -18,10 +18,17 @@ int cmd_ut_category(const char *name, const char *prefix,
                    struct unit_test *tests, int n_ents,
                    int argc, char *const argv[])
 {
+       int runs_per_text = 1;
        int ret;
 
+       if (argc > 1 && !strncmp("-r", argv[1], 2)) {
+               runs_per_text = dectoul(argv[1] + 2, NULL);
+               argv++;
+               argc++;
+       }
+
        ret = ut_run_list(name, prefix, tests, n_ents,
-                         argc > 1 ? argv[1] : NULL);
+                         argc > 1 ? argv[1] : NULL, runs_per_text);
 
        return ret ? CMD_RET_FAILURE : 0;
 }
@@ -168,7 +175,8 @@ static char ut_help_text[] =
 #ifdef CONFIG_CMD_LOADM
        "ut loadm [test-name]- test of parameters and load memory blob\n"
 #endif
-       ;
+       "All commands accept an optional [-r<runs>] flag before [test-name], to\n"
+       "run each test multiple times (<runs> is in decimal)";
 #endif /* CONFIG_SYS_LONGHELP */
 
 U_BOOT_CMD(

diff --git a/test/dm/test-dm.c b/test/dm/test-dm.c
index f5cda81bbfc06a88f36dab5090ff0f09d3b4a2cd..eb3581333b96a9e12ff73e344b081c10c2e135e0 100644 (file)
--- a/test/dm/test-dm.c
+++ b/test/dm/test-dm.c
@@ -29,13 +29,14 @@ DECLARE_GLOBAL_DATA_PTR;
  *     "fdt_pre_reloc"), or NULL to run all
  * Return: 0 if all tests passed, 1 if not
  */
-static int dm_test_run(const char *test_name)
+static int dm_test_run(const char *test_name, int runs_per_text)
 {
        struct unit_test *tests = UNIT_TEST_SUITE_START(dm_test);
        const int n_ents = UNIT_TEST_SUITE_COUNT(dm_test);
        int ret;
 
-       ret = ut_run_list("driver model", "dm_test_", tests, n_ents, test_name);
+       ret = ut_run_list("driver model", "dm_test_", tests, n_ents, test_name,
+                         runs_per_text);
 
        return ret ? CMD_RET_FAILURE : 0;
 }
@@ -43,9 +44,15 @@ static int dm_test_run(const char *test_name)
 int do_ut_dm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
 {
        const char *test_name = NULL;
+       int runs_per_text = 1;
 
+       if (argc > 1 && !strncmp("-r", argv[1], 2)) {
+               runs_per_text = dectoul(argv[1] + 2, NULL);
+               argv++;
+               argc++;
+       }
        if (argc > 1)
                test_name = argv[1];
 
-       return dm_test_run(test_name);
+       return dm_test_run(test_name, runs_per_text);
 }

diff --git a/test/test-main.c b/test/test-main.c
index 31837e57a8fb323a3cd835b4a495d51bb2375e0d..4f1c54b0f9ba211abc2403f22be6a367f52034e9 100644 (file)
--- a/test/test-main.c
+++ b/test/test-main.c
@@ -390,11 +390,17 @@ static int ut_run_tests(struct unit_test_state *uts, const char *prefix,
 
        for (test = tests; test < tests + count; test++) {
                const char *test_name = test->name;
-               int ret;
+               int ret, i, old_fail_count;
 
                if (!test_matches(prefix, test_name, select_name))
                        continue;
-               ret = ut_run_test_live_flat(uts, test, select_name);
+               old_fail_count = uts->fail_count;
+               for (i = 0; i < uts->runs_per_test; i++)
+                       ret = ut_run_test_live_flat(uts, test, select_name);
+               if (uts->fail_count != old_fail_count) {
+                       printf("Test %s failed %d times\n", select_name,
+                              uts->fail_count - old_fail_count);
+               }
                found++;
                if (ret == -EAGAIN)
                        continue;
@@ -408,7 +414,8 @@ static int ut_run_tests(struct unit_test_state *uts, const char *prefix,
 }
 
 int ut_run_list(const char *category, const char *prefix,
-               struct unit_test *tests, int count, const char *select_name)
+               struct unit_test *tests, int count, const char *select_name,
+               int runs_per_test)
 {
        struct unit_test_state uts = { .fail_count = 0 };
        bool has_dm_tests = false;
@@ -432,6 +439,7 @@ int ut_run_list(const char *category, const char *prefix,
                printf("Running %d %s tests\n", count, category);
 
        uts.of_root = gd_of_root();
+       uts.runs_per_test = runs_per_test;
        ret = ut_run_tests(&uts, prefix, tests, count, select_name);
 
        if (ret == -ENOENT)