test: offer "automake" output and check for perl better

runtests.pl -am now uses the "PASS/FAIL: [desc]" output for each
executed test. You can run 'make test-am' in the root build directory to
invoke that. The reason for this output style is to better allow generic
test suite parsers to also grok our test output.

The test Makefile now also tests that perl was indeed found and that the
PERL variable points to an executable before it tries to run the main
test perl script runtests.pl,

diff --git a/Makefile.am b/Makefile.am
index 6ff1d51eaf4dd4f6028826402b278d14e0b8a54b..484e4df20ef0c5de8b046d610caae3114031d5de 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -81,6 +81,9 @@ test-full:
 test-torture:
        @(cd tests; $(MAKE) all torture-test)
 
+test-am:
+       @(cd tests; $(MAKE) all am-test)
+
 endif
 
 examples:

diff --git a/tests/Makefile.am b/tests/Makefile.am
index 42f89dcea8aa2c3ea5489cea1f8f87146aa4a354..d8c3e18abf176df2af3d671a5acc5347fb7fd070 100644 (file)
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -55,26 +55,31 @@ curl:
 
 if CROSSCOMPILING
 TEST = @echo "NOTICE: we can't run the tests when cross-compiling!"
-TEST_Q =
-TEST_F =
-TEST_T =
 else # if not cross-compiling:
 TEST = srcdir=$(srcdir) $(PERL) $(PERLFLAGS) $(srcdir)/runtests.pl
 TEST_Q = -a -s
+TEST_AM = -a -am
 TEST_F = -a -p -r
 TEST_T = -a -t
 endif
 
-test: all
+# make sure that PERL is pointing to an executable
+perlcheck:
+       @if ! test -x "$(PERL)"; then echo "No perl!"; exit 2; fi
+
+test: perhcheck all
        $(TEST)
 
-quiet-test: all
+quiet-test: perlcheck all
        $(TEST) $(TEST_Q)
 
-full-test: all
+am-test: perlcheck all
+       $(TEST) $(TEST_AM)
+
+full-test: perlcheck all
        $(TEST) $(TEST_F)
 
-torture-test: all
+torture-test: perlcheck all
        $(TEST) $(TEST_T)
 
 .1.html:

diff --git a/tests/runtests.pl b/tests/runtests.pl
index e263788afd62e6c87f830f9d68e8bf3d4a548f34..27fff4a954a1f1200faac2b919b3687fe4611b2b 100755 (executable)
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -262,6 +262,7 @@ my %oldenv;
 #
 
 my $short;
+my $automakestyle;
 my $verbose;
 my $debugprotocol;
 my $anyway;
@@ -2148,18 +2149,25 @@ sub filteroff {
 #
 sub compare {
     # filter off patterns _before_ this comparison!
-    my ($subject, $firstref, $secondref)=@_;
+    my ($testnum, $testname, $subject, $firstref, $secondref)=@_;
 
     my $result = compareparts($firstref, $secondref);
 
     if($result) {
+        # timestamp test result verification end
+        $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
+
         if(!$short) {
-            logmsg "\n $subject FAILED:\n";
+            logmsg "\n $testnum: $subject FAILED:\n";
             logmsg showdiff($LOGDIR, $firstref, $secondref);
         }
-        else {
+        elsif(!$automakestyle) {
             logmsg "FAILED\n";
         }
+        else {
+            # automakestyle
+            logmsg "FAIL: $testnum - $testname - $subject\n";
+        }
     }
     return $result;
 }
@@ -2874,7 +2882,7 @@ sub singletest {
         timestampskippedevents($testnum);
         return -1;
     }
-    logmsg sprintf("test %03d...", $testnum);
+    logmsg sprintf("test %03d...", $testnum) if(!$automakestyle);
 
     # extract the reply data
     my @reply = getpart("reply", "data");
@@ -2916,12 +2924,9 @@ sub singletest {
 
     # name of the test
     my @testname= getpart("client", "name");
-
-    if(!$short) {
-        my $name = $testname[0];
-        $name =~ s/\n//g;
-        logmsg "[$name]\n";
-    }
+    my $testname = $testname[0];
+    $testname =~ s/\n//g;
+    logmsg "[$testname]\n" if(!$short);
 
     if($listonly) {
         timestampskippedevents($testnum);
@@ -3333,10 +3338,8 @@ sub singletest {
             chomp($validstdout[$#validstdout]);
         }
 
-        $res = compare("stdout", \@actual, \@validstdout);
+        $res = compare($testnum, $testname, "stdout", \@actual, \@validstdout);
         if($res) {
-            # timestamp test result verification end
-            $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
             return 1;
         }
         $ok .= "s";
@@ -3357,10 +3360,8 @@ sub singletest {
             map s/\r\n/\n/g, @out;
         }
 
-        $res = compare("data", \@out, \@reply);
+        $res = compare($testnum, $testname, "data", \@out, \@reply);
         if ($res) {
-            # timestamp test result verification end
-            $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
             return 1;
         }
         $ok .= "d";
@@ -3372,10 +3373,8 @@ sub singletest {
     if(@upload) {
         # verify uploaded data
         my @out = loadarray("$LOGDIR/upload.$testnum");
-        $res = compare("upload", \@out, \@upload);
+        $res = compare($testnum, $testname, "upload", \@out, \@upload);
         if ($res) {
-            # timestamp test result verification end
-            $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
             return 1;
         }
         $ok .= "u";
@@ -3419,10 +3418,8 @@ sub singletest {
             }
         }
 
-        $res = compare("protocol", \@out, \@protstrip);
+        $res = compare($testnum, $testname, "protocol", \@out, \@protstrip);
         if($res) {
-            # timestamp test result verification end
-            $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
             return 1;
         }
 
@@ -3469,10 +3466,8 @@ sub singletest {
             }
         }
 
-        $res = compare("proxy", \@out, \@protstrip);
+        $res = compare($testnum, $testname, "proxy", \@out, \@protstrip);
         if($res) {
-            # timestamp test result verification end
-            $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
             return 1;
         }
 
@@ -3521,10 +3516,9 @@ sub singletest {
 
             @outfile = fixarray(@outfile);
 
-            $res = compare("output ($filename)", \@generated, \@outfile);
+            $res = compare($testnum, $testname, "output ($filename)",
+                           \@generated, \@outfile);
             if($res) {
-                # timestamp test result verification end
-                $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
                 return 1;
             }
 
@@ -3614,8 +3608,13 @@ sub singletest {
             }
             my @e = valgrindparse($srcdir, $feature{'SSL'}, "$LOGDIR/$vgfile");
             if(@e && $e[0]) {
-                logmsg " valgrind ERROR ";
-                logmsg @e;
+                if($automakestyle) {
+                    logmsg "FAIL: $testnum - $testname - valgrind\n";
+                }
+                else {
+                    logmsg " valgrind ERROR ";
+                    logmsg @e;
+                }
                 # timestamp test result verification end
                 $timevrfyend{$testnum} = Time::HiRes::time() if($timestats);
                 return 1;
@@ -3641,7 +3640,13 @@ sub singletest {
     my $left=sprintf("remaining: %02d:%02d",
                      $estleft/60,
                      $estleft%60);
-    logmsg sprintf("OK (%-3d out of %-3d, %s)\n", $count, $total, $left);
+
+    if(!$automakestyle) {
+        logmsg sprintf("OK (%-3d out of %-3d, %s)\n", $count, $total, $left);
+    }
+    else {
+        logmsg "PASS: $testnum - $testname\n";
+    }
 
     # the test succeeded, remove all log files
     if(!$keepoutfiles) {
@@ -4284,6 +4289,11 @@ while(@ARGV) {
         # short output
         $short=1;
     }
+    elsif($ARGV[0] eq "-am") {
+        # automake-style output
+        $short=1;
+        $automakestyle=1;
+    }
     elsif($ARGV[0] eq "-n") {
         # no valgrind
         undef $valgrind;
@@ -4360,6 +4370,7 @@ Usage: runtests.pl [options] [test selection(s)]
   -r       run time statistics
   -rf      full run time statistics
   -s       short output
+  -am      automake style output PASS/FAIL: [number] [name]
   -t[N]    torture (simulate memory alloc failures); N means fail Nth alloc
   -v       verbose output
   [num]    like "5 6 9" or " 5 to 22 " to run those tests only