regress: introduce TT_RETRIABLE

We have some tests that has false-positive due to real/CPU time bound,
but they are pretty generic and we do not want to skip them by default.

TT_RETRIABLE is the flag that will indicate tinytest to retry the test
in case of failure, use it to avoid next possible false-positives:
- real time-related
- CPU time-related
Since I guess it is better to see/grepping RETRYING messages over
ignoring completely failed builds.

No configuration switch for number of retries was done on purpose (only
3 retries and no more).

And this is how it looks BTW:
  $ gcc ../test/tinytest_demo.c ../test/tinytest.c
  $ ./a.out --verbose --no-fork
  demo/timeout_retry
  demo/timeout_retry:
    FAIL ../test/tinytest_demo.c:201: assert(i != 1): 1 vs 1
    [timeout_retry FAILED]

    [RETRYING timeout_retry (3)]
  demo/timeout_retry:
           OK ../test/tinytest_demo.c:201: assert(i != 1): 2 vs 1
           OK ../test/tinytest_demo.c:213: assert(t2-t1 >= 4): 5 vs 4
           OK ../test/tinytest_demo.c:215: assert(t2-t1 <= 6): 5 vs 6
  1 tests ok.  (0 skipped)

diff --git a/test/tinytest.c b/test/tinytest.c
index 3a8e331055015365f72c861157becfc6ae5ceb47..a27a906a0ade7fbc8b1de89d5d506ba12f90edc9 100644 (file)
--- a/test/tinytest.c
+++ b/test/tinytest.c
@@ -253,15 +253,12 @@ testcase_run_one(const struct testgroup_t *group,
        }
 
        if (outcome == OK) {
-               ++n_ok;
                if (opt_verbosity>0 && !opt_forked)
                        puts(opt_verbosity==1?"OK":"");
        } else if (outcome == SKIP) {
-               ++n_skipped;
                if (opt_verbosity>0 && !opt_forked)
                        puts("SKIPPED");
        } else {
-               ++n_bad;
                if (!opt_forked)
                        printf("\n  [%s FAILED]\n", testcase->name);
        }
@@ -428,11 +425,35 @@ tinytest_main(int c, const char **v, struct testgroup_t *groups)
 #endif
 
        ++in_tinytest_main;
-       for (i=0; groups[i].prefix; ++i)
-               for (j=0; groups[i].cases[j].name; ++j)
-                       if (groups[i].cases[j].flags & TT_ENABLED_)
-                               testcase_run_one(&groups[i],
-                                                &groups[i].cases[j]);
+       for (i = 0; groups[i].prefix; ++i) {
+               struct testgroup_t *group = &groups[i];
+               for (j = 0; group->cases[j].name; ++j) {
+                       struct testcase_t *testcase = &group->cases[j];
+                       int test_attempts = 3;
+                       int test_ret_err;
+
+                       if (!(testcase->flags & TT_ENABLED_))
+                               continue;
+
+                       for (;;) {
+                               test_ret_err = testcase_run_one(group, testcase);
+
+                               if (test_ret_err == OK)
+                                       break;
+                               if (!(testcase->flags & TT_RETRIABLE))
+                                       break;
+                               printf("\n  [RETRYING %s (%i)]\n", testcase->name, test_attempts);
+                               if (!test_attempts--)
+                                       break;
+                       }
+
+                       switch (test_ret_err) {
+                               case OK:   ++n_ok;      break;
+                               case SKIP: ++n_skipped; break;
+                               default:   ++n_bad;     break;
+                       }
+               }
+       }
 
        --in_tinytest_main;
 

diff --git a/test/tinytest.h b/test/tinytest.h
index ed07b26bc006bd6f3c00ced89c0ba90198203466..d321dd4675423a778f2e3d836f95dd70bd8d9804 100644 (file)
--- a/test/tinytest.h
+++ b/test/tinytest.h
@@ -34,8 +34,11 @@
 #define TT_ENABLED_  (1<<2)
 /** Flag for a test that's off by default. */
 #define TT_OFF_BY_DEFAULT  (1<<3)
+/** Flag for a test that should be runned again in case of failure (but not
+ * more then 3 times). */
+#define TT_RETRIABLE   (1<<4)
 /** If you add your own flags, make them start at this point. */
-#define TT_FIRST_USER_FLAG (1<<4)
+#define TT_FIRST_USER_FLAG (1<<5)
 
 typedef void (*testcase_fn)(void *);
 

diff --git a/test/tinytest_demo.c b/test/tinytest_demo.c
index f6bfd66a1a1f13d623bb57344a14ad301d9e6bf5..123855fff7fe8233c314d19df37e31ef876794e1 100644 (file)
--- a/test/tinytest_demo.c
+++ b/test/tinytest_demo.c
@@ -192,6 +192,32 @@ test_timeout(void *ptr)
        ;
 }
 
+void
+test_timeout_retry(void *ptr)
+{
+       static int i = 0;
+
+       ++i;
+       tt_int_op(i, !=, 1);
+
+       time_t t1, t2;
+       (void)ptr;
+       t1 = time(NULL);
+#ifdef _WIN32
+       Sleep(5000);
+#else
+       sleep(5);
+#endif
+       t2 = time(NULL);
+
+       tt_int_op(t2-t1, >=, 4);
+
+       tt_int_op(t2-t1, <=, 6);
+
+ end:
+       ;
+}
+
 /* ============================================================ */
 
 /* Now we need to make sure that our tests get invoked.          First, you take
@@ -212,6 +238,10 @@ struct testcase_t demo_tests[] = {
         * can enable it manually by passing +demo/timeout at the command line.*/
        { "timeout", test_timeout, TT_OFF_BY_DEFAULT },
 
+       /* This test will be retried. (and it will not pass from the first
+        * time) */
+       { "timeout_retry", test_timeout_retry, TT_RETRIABLE },
+
        /* The array has to end with END_OF_TESTCASES. */
        END_OF_TESTCASES
 };