From 63b065be80d57dfee4688ddc32b8344e922648d9 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 20 Nov 2018 01:06:04 +0300
Subject: [PATCH] regress: introduce TT_RETRIABLE

We have some tests that has false-positive due to real/CPU time bound,
but they are pretty generic and we do not want to skip them by default.

TT_RETRIABLE is the flag that will indicate tinytest to retry the test
in case of failure, use it to avoid next possible false-positives:
- real time-related
- CPU time-related
Since I guess it is better to see/grepping RETRYING messages over
ignoring completely failed builds.

No configuration switch for number of retries was done on purpose (only
3 retries and no more).

And this is how it looks BTW:
  $ gcc ../test/tinytest_demo.c ../test/tinytest.c
  $ ./a.out --verbose --no-fork
  demo/timeout_retry
  demo/timeout_retry:
    FAIL ../test/tinytest_demo.c:201: assert(i != 1): 1 vs 1
    [timeout_retry FAILED]

    [RETRYING timeout_retry (3)]
  demo/timeout_retry:
           OK ../test/tinytest_demo.c:201: assert(i != 1): 2 vs 1
           OK ../test/tinytest_demo.c:213: assert(t2-t1 >= 4): 5 vs 4
           OK ../test/tinytest_demo.c:215: assert(t2-t1 <= 6): 5 vs 6
  1 tests ok.  (0 skipped)
---
 test/tinytest.c      | 37 +++++++++++++++++++++++++++++--------
 test/tinytest.h      |  5 ++++-
 test/tinytest_demo.c | 30 ++++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+), 9 deletions(-)

diff --git a/test/tinytest.c b/test/tinytest.c
index 3a8e3310..a27a906a 100644
--- a/test/tinytest.c
+++ b/test/tinytest.c
@@ -253,15 +253,12 @@ testcase_run_one(const struct testgroup_t *group,
 	}
 
 	if (outcome == OK) {
-		++n_ok;
 		if (opt_verbosity>0 && !opt_forked)
 			puts(opt_verbosity==1?"OK":"");
 	} else if (outcome == SKIP) {
-		++n_skipped;
 		if (opt_verbosity>0 && !opt_forked)
 			puts("SKIPPED");
 	} else {
-		++n_bad;
 		if (!opt_forked)
 			printf("\n  [%s FAILED]\n", testcase->name);
 	}
@@ -428,11 +425,35 @@ tinytest_main(int c, const char **v, struct testgroup_t *groups)
 #endif
 
 	++in_tinytest_main;
-	for (i=0; groups[i].prefix; ++i)
-		for (j=0; groups[i].cases[j].name; ++j)
-			if (groups[i].cases[j].flags & TT_ENABLED_)
-				testcase_run_one(&groups[i],
-						 &groups[i].cases[j]);
+	for (i = 0; groups[i].prefix; ++i) {
+		struct testgroup_t *group = &groups[i];
+		for (j = 0; group->cases[j].name; ++j) {
+			struct testcase_t *testcase = &group->cases[j];
+			int test_attempts = 3;
+			int test_ret_err;
+
+			if (!(testcase->flags & TT_ENABLED_))
+				continue;
+
+			for (;;) {
+				test_ret_err = testcase_run_one(group, testcase);
+
+				if (test_ret_err == OK)
+					break;
+				if (!(testcase->flags & TT_RETRIABLE))
+					break;
+				printf("\n  [RETRYING %s (%i)]\n", testcase->name, test_attempts);
+				if (!test_attempts--)
+					break;
+			}
+
+			switch (test_ret_err) {
+				case OK:   ++n_ok;      break;
+				case SKIP: ++n_skipped; break;
+				default:   ++n_bad;     break;
+			}
+		}
+	}
 
 	--in_tinytest_main;
 
diff --git a/test/tinytest.h b/test/tinytest.h
index ed07b26b..d321dd46 100644
--- a/test/tinytest.h
+++ b/test/tinytest.h
@@ -34,8 +34,11 @@
 #define TT_ENABLED_  (1<<2)
 /** Flag for a test that's off by default. */
 #define TT_OFF_BY_DEFAULT  (1<<3)
+/** Flag for a test that should be runned again in case of failure (but not
+ * more then 3 times). */
+#define TT_RETRIABLE	(1<<4)
 /** If you add your own flags, make them start at this point. */
-#define TT_FIRST_USER_FLAG (1<<4)
+#define TT_FIRST_USER_FLAG (1<<5)
 
 typedef void (*testcase_fn)(void *);
 
diff --git a/test/tinytest_demo.c b/test/tinytest_demo.c
index f6bfd66a..123855ff 100644
--- a/test/tinytest_demo.c
+++ b/test/tinytest_demo.c
@@ -192,6 +192,32 @@ test_timeout(void *ptr)
 	;
 }
 
+void
+test_timeout_retry(void *ptr)
+{
+	static int i = 0;
+
+	++i;
+	tt_int_op(i, !=, 1);
+
+	time_t t1, t2;
+	(void)ptr;
+	t1 = time(NULL);
+#ifdef _WIN32
+	Sleep(5000);
+#else
+	sleep(5);
+#endif
+	t2 = time(NULL);
+
+	tt_int_op(t2-t1, >=, 4);
+
+	tt_int_op(t2-t1, <=, 6);
+
+ end:
+	;
+}
+
 /* ============================================================ */
 
 /* Now we need to make sure that our tests get invoked.	  First, you take
@@ -212,6 +238,10 @@ struct testcase_t demo_tests[] = {
 	 * can enable it manually by passing +demo/timeout at the command line.*/
 	{ "timeout", test_timeout, TT_OFF_BY_DEFAULT },
 
+	/* This test will be retried. (and it will not pass from the first
+	 * time) */
+	{ "timeout_retry", test_timeout_retry, TT_RETRIABLE },
+
 	/* The array has to end with END_OF_TESTCASES. */
 	END_OF_TESTCASES
 };
-- 
2.50.1