tests: add stderr comparison to the test suite

The code is more or less copied from the stdout comparison code, maybe
some better reuse is possible.

test 1457 is adjusted to make the output actually match (by using --silent)
test 506 used <stderr> without actually needing it, so that <stderr> block is removed

Closes #3536

diff --git a/tests/data/test1457 b/tests/data/test1457
index aad6d43d43e6e6453a9cff50947926943fe1dd7a..43a5f35d3ed6addb61c963dd95ec99d1f038381d 100644 (file)
--- a/tests/data/test1457
+++ b/tests/data/test1457
@@ -29,7 +29,7 @@ http
 Check if %{stderr} and %{stdout} switch between stdout and stderr.
 </name>
 <command>
-http://%HOSTIP:%HTTPPORT/1457 --write-out 'line1%{stderr}line2%{stdout}line3'
+http://%HOSTIP:%HTTPPORT/1457 --silent --write-out 'line1%{stderr}line2%{stdout}line3'
 </command>
 </client>
 

diff --git a/tests/data/test506 b/tests/data/test506
index 30f4aa9c04f2dd651f645fbb2454affc2e2270c6..8f06e0e4f1db8001b6309f67ab838f6f2266ac28 100644 (file)
--- a/tests/data/test506
+++ b/tests/data/test506
@@ -228,9 +228,6 @@ lock:   share  [Pigs in space]: 94
 unlock: share  [Pigs in space]: 95
 GLOBAL_CLEANUP
 </stdout>
-<stderr>
-http://%HOSTIP:%HTTPPORT/506 
-</stderr>
 <file name="log/jar506" mode="text">
 # Netscape HTTP Cookie File
 # https://curl.haxx.se/docs/http-cookies.html

diff --git a/tests/runtests.pl b/tests/runtests.pl
index 6c8b903c30fb63e22524bdb17b44320977a3a2a4..66ab405f52a5a06b882a1c5b8d313123aee2b9cf 100755 (executable)
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -6,7 +6,7 @@
 #                            | (__| |_| |  _ <| |___
 #                             \___|\___/|_| \_\_____|
 #
-# Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+# Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
 #
 # This software is licensed as described in the file COPYING, which
 # you should have received as part of this distribution. The terms
@@ -3746,6 +3746,7 @@ sub singletest {
 
     # if this section exists, we verify that the stdout contained this:
     my @validstdout = fixarray ( getpart("verify", "stdout") );
+    my @validstderr = fixarray ( getpart("verify", "stderr") );
 
     # if this section exists, we verify upload
     my @upload = getpart("verify", "upload");
@@ -4229,6 +4230,57 @@ sub singletest {
         $ok .= "-"; # stdout not checked
     }
 
+    if (@validstderr) {
+        # verify redirected stderr
+        my @actual = loadarray($STDERR);
+
+        # what parts to cut off from stderr
+        my @stripfile = getpart("verify", "stripfile");
+
+        foreach my $strip (@stripfile) {
+            chomp $strip;
+            my @newgen;
+            for(@actual) {
+                eval $strip;
+                if($_) {
+                    push @newgen, $_;
+                }
+            }
+            # this is to get rid of array entries that vanished (zero
+            # length) because of replacements
+            @actual = @newgen;
+        }
+
+        # variable-replace in the stderr we have from the test case file
+        @validstderr = fixarray(@validstderr);
+
+        # get all attributes
+        my %hash = getpartattr("verify", "stderr");
+
+        # get the mode attribute
+        my $filemode=$hash{'mode'};
+        if($filemode && ($filemode eq "text") && $has_textaware) {
+            # text mode when running on windows: fix line endings
+            map s/\r\n/\n/g, @validstderr;
+            map s/\n/\r\n/g, @validstderr;
+        }
+
+        if($hash{'nonewline'}) {
+            # Yes, we must cut off the final newline from the final line
+            # of the protocol data
+            chomp($validstderr[$#validstderr]);
+        }
+
+        $res = compare($testnum, $testname, "stderr", \@actual, \@validstderr);
+        if($res) {
+            return 1;
+        }
+        $ok .= "r";
+    }
+    else {
+        $ok .= "-"; # stderr not checked
+    }
+
     if(@protocol) {
         # Verify the sent request
         my @out = loadarray($SERVERIN);