test: Allow running tests multiple times
Some tests can have race conditions which are hard to detect on a single
one. Add a way to run tests more than once, to help with this.
Each individual test is run the requested number of times before moving
to the next test. If any runs failed, a message is shown.
This is most useful when running a single test, since running all tests
multiple times can take a while.
Signed-off-by: Simon Glass <sjg@chromium.org>
diff --git a/test/cmd_ut.c b/test/cmd_ut.c
index 3789c6b..11c219b 100644
--- a/test/cmd_ut.c
+++ b/test/cmd_ut.c
@@ -18,10 +18,17 @@
struct unit_test *tests, int n_ents,
int argc, char *const argv[])
{
+ int runs_per_text = 1;
int ret;
+ if (argc > 1 && !strncmp("-r", argv[1], 2)) {
+ runs_per_text = dectoul(argv[1] + 2, NULL);
+ argv++;
+ argc++;
+ }
+
ret = ut_run_list(name, prefix, tests, n_ents,
- argc > 1 ? argv[1] : NULL);
+ argc > 1 ? argv[1] : NULL, runs_per_text);
return ret ? CMD_RET_FAILURE : 0;
}
@@ -168,7 +175,8 @@
#ifdef CONFIG_CMD_LOADM
"ut loadm [test-name]- test of parameters and load memory blob\n"
#endif
- ;
+ "All commands accept an optional [-r<runs>] flag before [test-name], to\n"
+ "run each test multiple times (<runs> is in decimal)";
#endif /* CONFIG_SYS_LONGHELP */
U_BOOT_CMD(
diff --git a/test/dm/test-dm.c b/test/dm/test-dm.c
index f5cda81..eb35813 100644
--- a/test/dm/test-dm.c
+++ b/test/dm/test-dm.c
@@ -29,13 +29,14 @@
* "fdt_pre_reloc"), or NULL to run all
* Return: 0 if all tests passed, 1 if not
*/
-static int dm_test_run(const char *test_name)
+static int dm_test_run(const char *test_name, int runs_per_text)
{
struct unit_test *tests = UNIT_TEST_SUITE_START(dm_test);
const int n_ents = UNIT_TEST_SUITE_COUNT(dm_test);
int ret;
- ret = ut_run_list("driver model", "dm_test_", tests, n_ents, test_name);
+ ret = ut_run_list("driver model", "dm_test_", tests, n_ents, test_name,
+ runs_per_text);
return ret ? CMD_RET_FAILURE : 0;
}
@@ -43,9 +44,15 @@
int do_ut_dm(struct cmd_tbl *cmdtp, int flag, int argc, char *const argv[])
{
const char *test_name = NULL;
+ int runs_per_text = 1;
+ if (argc > 1 && !strncmp("-r", argv[1], 2)) {
+ runs_per_text = dectoul(argv[1] + 2, NULL);
+ argv++;
+ argc++;
+ }
if (argc > 1)
test_name = argv[1];
- return dm_test_run(test_name);
+ return dm_test_run(test_name, runs_per_text);
}
diff --git a/test/test-main.c b/test/test-main.c
index 31837e5..4f1c54b 100644
--- a/test/test-main.c
+++ b/test/test-main.c
@@ -390,11 +390,17 @@
for (test = tests; test < tests + count; test++) {
const char *test_name = test->name;
- int ret;
+ int ret, i, old_fail_count;
if (!test_matches(prefix, test_name, select_name))
continue;
- ret = ut_run_test_live_flat(uts, test, select_name);
+ old_fail_count = uts->fail_count;
+ for (i = 0; i < uts->runs_per_test; i++)
+ ret = ut_run_test_live_flat(uts, test, select_name);
+ if (uts->fail_count != old_fail_count) {
+ printf("Test %s failed %d times\n", select_name,
+ uts->fail_count - old_fail_count);
+ }
found++;
if (ret == -EAGAIN)
continue;
@@ -408,7 +414,8 @@
}
int ut_run_list(const char *category, const char *prefix,
- struct unit_test *tests, int count, const char *select_name)
+ struct unit_test *tests, int count, const char *select_name,
+ int runs_per_test)
{
struct unit_test_state uts = { .fail_count = 0 };
bool has_dm_tests = false;
@@ -432,6 +439,7 @@
printf("Running %d %s tests\n", count, category);
uts.of_root = gd_of_root();
+ uts.runs_per_test = runs_per_test;
ret = ut_run_tests(&uts, prefix, tests, count, select_name);
if (ret == -ENOENT)