runtests: add -o to run test cases in scrambled order

... instead of numerical order.

Closes #1466

diff --git a/tests/runtests.1 b/tests/runtests.1
index 5576b2fd8d8e80f8b01d7efeac2773b4c52bc19f..02540c84be48cfbf917d132b92c43d88fea8e8cc 100644 (file)
--- a/tests/runtests.1
+++ b/tests/runtests.1
@@ -5,7 +5,7 @@
 .\" *                            | (__| |_| |  _ <| |___
 .\" *                             \___|\___/|_| \_\_____|
 .\" *
-.\" * Copyright (C) 1998 - 2013, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
 .\" *
 .\" * This software is licensed as described in the file COPYING, which
 .\" * you should have received as part of this distribution. The terms
@@ -71,6 +71,8 @@ detected. Useful for debugging.
 Lists all test case names.
 .IP "-n"
 Disable the check for and use of valgrind.
+.IP "-o"
+Run the tests in a scrambled, or randomized, order instead of sequentially.
 .IP "-p"
 Prints out all files in "log/" to stdout when a test case fails. Very
 practical when used in the automated and distributed tests since then the

diff --git a/tests/runtests.pl b/tests/runtests.pl
index e281e318cdf57101f06570d1d31a874c1c29470e..d3e4fbbccb10183d7585f8e7c377ee7fdbfc8968 100755 (executable)
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -194,7 +194,7 @@ my $pwd = getcwd();          # current working directory
 
 my $start;
 my $ftpchecktime=1; # time it took to verify our test FTP server
-
+my $scrambleorder;
 my $stunnel = checkcmd("stunnel4") || checkcmd("tstunnel") || checkcmd("stunnel");
 my $valgrind = checktestcmd("valgrind");
 my $valgrind_logfile="--logfile";
@@ -4870,6 +4870,10 @@ while(@ARGV) {
         # have the servers display protocol output
         $debugprotocol=1;
     }
+    elsif($ARGV[0] eq "-e") {
+        # run the tests cases event based if possible
+        $run_event_based=1;
+    }
     elsif ($ARGV[0] eq "-g") {
         # run this test with gdb
         $gdbthis=1;
@@ -4892,6 +4896,10 @@ while(@ARGV) {
         # no valgrind
         undef $valgrind;
     }
+    elsif ($ARGV[0] eq "-o") {
+        # execute in scrambled order
+        $scrambleorder=1;
+    }
     elsif($ARGV[0] =~ /^-t(.*)/) {
         # torture
         $torture=1;
@@ -4907,10 +4915,6 @@ while(@ARGV) {
         # continue anyway, even if a test fail
         $anyway=1;
     }
-    elsif($ARGV[0] eq "-e") {
-        # run the tests cases event based if possible
-        $run_event_based=1;
-    }
     elsif($ARGV[0] eq "-p") {
         $postmortem=1;
     }
@@ -4958,12 +4962,14 @@ Usage: runtests.pl [options] [test selection(s)]
   -bN      use base port number N for test servers (default $base)
   -c path  use this curl executable
   -d       display server debug info
+  -e       event-based execution
   -g       run the test case with gdb
   -gw      run the test case with gdb as a windowed application
   -h       this help text
   -k       keep stdout and stderr files present after tests
   -l       list all test case names/descriptions
   -n       no valgrind
+  -o       scrambled order
   -p       print log file contents when a test fails
   -r       run time statistics
   -rf      full run time statistics
@@ -5188,6 +5194,19 @@ else {
     $TESTCASES = $verified;
 }
 
+if($scrambleorder) {
+    # scramble the order of the test cases
+    my @rand;
+    while($TESTCASES) {
+        my @all = split(/ /, $TESTCASES);
+        my $r = rand @all;
+        push @rand, $all[$r];
+        $all[$r]="";
+        $TESTCASES = join(" ", @all);
+    }
+    $TESTCASES = join(" ", @rand);
+}
+
 #######################################################################
 # Start the command line log
 #