]> granicus.if.org Git - pdns/commitdiff
add 'fail.*' option to the regression-test
authorKees Monshouwer <mind04@monshouwer.org>
Sun, 21 Feb 2016 09:46:58 +0000 (10:46 +0100)
committermind04 <mind04@monshouwer.org>
Mon, 22 Feb 2016 08:53:05 +0000 (09:53 +0100)
A failing test must be brokem.

64 files changed:
regression-tests/runtests
regression-tests/tests/8bit-txt-unescaped/failing.ldap-simple [moved from regression-tests/tests/8bit-txt-unescaped/skip.ldap-simple with 100% similarity]
regression-tests/tests/8bit-txt-unescaped/failing.ldap-tree [moved from regression-tests/tests/8bit-txt-unescaped/skip.ldap-tree with 100% similarity]
regression-tests/tests/any-query/failing.ldap-simple [moved from regression-tests/tests/any-query/skip.ldap-simple with 100% similarity]
regression-tests/tests/any-query/failing.ldap-tree [moved from regression-tests/tests/any-query/skip.ldap-tree with 100% similarity]
regression-tests/tests/basic-eui48/failing.bind [moved from regression-tests/tests/basic-eui48/skip.bind with 100% similarity]
regression-tests/tests/basic-eui48/failing.mydns [moved from regression-tests/tests/basic-eui48/skip.mydns with 100% similarity]
regression-tests/tests/basic-eui48/failing.nsd [moved from regression-tests/tests/basic-eui48/skip.nsd with 100% similarity]
regression-tests/tests/basic-eui64/failing.bind [moved from regression-tests/tests/basic-eui64/skip.bind with 100% similarity]
regression-tests/tests/basic-eui64/failing.mydns [moved from regression-tests/tests/basic-eui64/skip.mydns with 100% similarity]
regression-tests/tests/basic-eui64/failing.nsd [moved from regression-tests/tests/basic-eui64/skip.nsd with 100% similarity]
regression-tests/tests/basic-loc/failing.mydns [moved from regression-tests/tests/basic-loc/skip.mydns with 100% similarity]
regression-tests/tests/basic-nonzone/failing.ldap-simple [moved from regression-tests/tests/basic-ns-resolution/skip.ldap-simple with 100% similarity]
regression-tests/tests/basic-nonzone/failing.ldap-tree [moved from regression-tests/tests/basic-ns-resolution/skip.ldap-tree with 100% similarity]
regression-tests/tests/basic-ns-resolution/failing.ldap-simple [moved from regression-tests/tests/cname-and-wildcard-but-no-correct-type/skip.oracle-slave-nsec3 with 100% similarity]
regression-tests/tests/basic-ns-resolution/failing.ldap-tree [moved from regression-tests/tests/cname-wildcard-chain/skip.oracle-nsec3 with 100% similarity]
regression-tests/tests/cname-and-wildcard-but-no-correct-type/failing.oracle-slave-nsec3 [moved from regression-tests/tests/dname-self/skip.mydns with 100% similarity]
regression-tests/tests/cname-to-unauth-any/failing.ldap-simple [moved from regression-tests/tests/dname/skip.mydns with 100% similarity]
regression-tests/tests/cname-to-unauth-any/failing.ldap-tree [moved from regression-tests/tests/double/skip.mydns with 100% similarity]
regression-tests/tests/cname-to-unauth/failing.ldap-simple [moved from regression-tests/tests/ds-at-secure-delegation/skip.mydns with 100% similarity]
regression-tests/tests/cname-to-unauth/failing.ldap-tree [moved from regression-tests/tests/ds-at-unsecure-delegation/skip.oracle-nsec with 100% similarity]
regression-tests/tests/cname-wildcard-chain/failing.oracle-nsec3 [moved from regression-tests/tests/ds-at-unsecure-zone-cut/skip.oracle-nsec with 100% similarity]
regression-tests/tests/dname-self/failing.mydns [moved from regression-tests/tests/ds-inside-delegation/skip.oracle-nsec with 100% similarity]
regression-tests/tests/dname/failing.ldap-simple [moved from regression-tests/tests/ds-inside-delegation/skip.oracle-nsec3 with 100% similarity]
regression-tests/tests/dname/failing.ldap-tree [moved from regression-tests/tests/ent-asterisk/skip.bind with 100% similarity]
regression-tests/tests/dname/failing.mydns [moved from regression-tests/tests/long-name/skip.oracle-slave-nsec3 with 100% similarity]
regression-tests/tests/double/failing.mydns [moved from regression-tests/tests/mx-case-sensitivy-with-ap/skip.ldap-simple with 100% similarity]
regression-tests/tests/ds-at-secure-delegation/description
regression-tests/tests/ds-at-secure-delegation/failing.mydns [moved from regression-tests/tests/mx-case-sensitivy-with-ap/skip.ldap-tree with 100% similarity]
regression-tests/tests/ds-at-unsecure-delegation/description
regression-tests/tests/ds-at-unsecure-delegation/failing.oracle-nsec [moved from regression-tests/tests/mx-with-simple-additional-processing/skip.ldap-simple with 100% similarity]
regression-tests/tests/ds-at-unsecure-zone-cut/failing.oracle-nsec [moved from regression-tests/tests/mx-with-simple-additional-processing/skip.ldap-tree with 100% similarity]
regression-tests/tests/ds-inside-delegation/failing.oracle-nsec [moved from regression-tests/tests/ns-with-identical-glue/skip.oracle-slave-nsec3 with 100% similarity]
regression-tests/tests/ds-inside-delegation/failing.oracle-nsec3 [moved from regression-tests/tests/nsec-glue-at-delegation/skip.oracle-nsec with 100% similarity]
regression-tests/tests/ent-asterisk/failing.bind [moved from regression-tests/tests/nsec-glue-at-delegation/skip.oracle-nsec3 with 100% similarity]
regression-tests/tests/external-cname-pointer/failing.ldap-simple [moved from regression-tests/tests/nsec-glue/skip.oracle-nsec with 100% similarity]
regression-tests/tests/external-cname-pointer/failing.ldap-tree [moved from regression-tests/tests/nsec3-hash-query/skip.oracle-nsec with 100% similarity]
regression-tests/tests/long-name/failing.oracle-slave-nsec3 [moved from regression-tests/tests/root-mx/skip.ldap-simple with 100% similarity]
regression-tests/tests/minimal-nxdomain/description
regression-tests/tests/mx-case-sensitivy-with-ap/failing.ldap-simple [moved from regression-tests/tests/root-mx/skip.ldap-tree with 100% similarity]
regression-tests/tests/mx-case-sensitivy-with-ap/failing.ldap-tree [moved from regression-tests/tests/root-srv/skip.ldap-simple with 100% similarity]
regression-tests/tests/mx-with-simple-additional-processing/failing.ldap-simple [moved from regression-tests/tests/root-srv/skip.ldap-tree with 100% similarity]
regression-tests/tests/mx-with-simple-additional-processing/failing.ldap-tree [moved from regression-tests/tests/secure-delegation-ds-ns/skip.oracle-nsec with 100% similarity]
regression-tests/tests/ns-with-identical-glue/failing.oracle-slave-nsec3 [moved from regression-tests/tests/secure-delegation-ds-ns/skip.oracle-nsec3 with 100% similarity]
regression-tests/tests/nsec-glue-at-delegation/failing.oracle-nsec [moved from regression-tests/tests/secure-delegation/skip.ldap-simple with 100% similarity]
regression-tests/tests/nsec-glue-at-delegation/failing.oracle-nsec3 [moved from regression-tests/tests/test-urc/skip.mydns with 100% similarity]
regression-tests/tests/nsec-glue/failing.oracle-nsec [moved from regression-tests/tests/verify-dnssec-zone/skip.oracle-nsec with 100% similarity]
regression-tests/tests/nsec3-hash-query/failing.oracle-nsec [moved from regression-tests/tests/verify-dnssec-zone/skip.oracle-nsec3 with 100% similarity]
regression-tests/tests/root-mx/failing.ldap-simple [moved from regression-tests/tests/very-long-txt/skip.bind with 100% similarity]
regression-tests/tests/root-mx/failing.ldap-tree [moved from regression-tests/tests/very-long-txt/skip.nsd with 100% similarity]
regression-tests/tests/root-srv/failing.ldap-simple [new file with mode: 0644]
regression-tests/tests/root-srv/failing.ldap-tree [new file with mode: 0644]
regression-tests/tests/secure-delegation-ds-ns/description
regression-tests/tests/secure-delegation-ds-ns/failing.oracle-nsec [new file with mode: 0644]
regression-tests/tests/secure-delegation-ds-ns/failing.oracle-nsec3 [new file with mode: 0644]
regression-tests/tests/secure-delegation/failing.ldap-simple [new file with mode: 0644]
regression-tests/tests/test-urc/failing.mydns [new file with mode: 0644]
regression-tests/tests/unknown-domain/failing.ldap-simple [new file with mode: 0644]
regression-tests/tests/unknown-domain/failing.ldap-tree [new file with mode: 0644]
regression-tests/tests/verify-dnssec-zone/failing.oracle-nsec [new file with mode: 0644]
regression-tests/tests/verify-dnssec-zone/failing.oracle-nsec3 [new file with mode: 0644]
regression-tests/tests/very-long-txt/failing.bind [new file with mode: 0644]
regression-tests/tests/very-long-txt/failing.mydns [moved from regression-tests/tests/very-long-txt/skip.mydns with 100% similarity]
regression-tests/tests/very-long-txt/failing.nsd [new file with mode: 0644]

index ccefb1ca301fd22723b8dffbced2f8047558860d..2ed6015f35aa1f329bc879d502c597c63d20eca6 100755 (executable)
@@ -66,6 +66,16 @@ do
                done
        fi
 
+       SHOULDFAIL=0
+       for reason in $skipreasons $context
+       do
+               if [ -e $a/failing.$reason ]
+               then
+                       SHOULDFAIL=1
+                       break
+               fi
+       done
+
        if [ "$spectest" != "" ] && [ "$spectest" != "$testname" ] && [ "$testname" != "00dnssec-grabkeys" ]
        then
                SKIPIT=1
@@ -95,13 +105,27 @@ do
                diff ${diffopts} $expected $a/real_result > $a/diff 2>&1
                if [ -s $a/diff ] 
                then
-                       result="        Failed test $a"
-                       echo $testname >> failed_tests
-                       failed=$[$failed+1]
+                       if [ $SHOULDFAIL = 0 ]
+                       then
+                               result="        Failed test $a"
+                               echo $testname >> failed_tests
+                               failed=$[$failed+1]
+                       else
+                               result="        Expected failure for reason $reason, test passed $a"
+                               echo $testname >> passed_tests
+                               passed=$[$passed+1]
+                       fi
                else
-                       result="        Passed test $a"
-                       echo $testname >> passed_tests
-                       passed=$[$passed+1]
+                       if [ $SHOULDFAIL = 0 ]
+                       then
+                               result="        Passed test $a"
+                               echo $testname >> passed_tests
+                               passed=$[$passed+1]
+                       else
+                               result="        Unexpected pass for reason $reason, test failed $a"
+                               echo $testname >> failed_tests
+                               failed=$[$failed+1]
+                       fi
                fi
        fi      
        echo "$result"
index 0b14d35839637ed3e1957d289f26026a713aa51d..ac02e4e5f9c4df3441206b3175c59d5c5776cea7 100644 (file)
@@ -1,4 +1,3 @@
 This test tries to resolve a DS question at a secure delegation.
 It was written specifically to verify that we do not sign NS records
 at secure delegations.
-
index 689afdf0ad0638f9219cc17d936a23b5539e385b..b2bf44fe3597e53c73d5ed3807af248199da6570 100644 (file)
@@ -1,2 +1 @@
 This test tries to resolve a DS question at an unsecure delegation.
-
index 7a2790e71e52b5884715aeaf8de131bcf6866c95..212c5e8bd6e4f429d3e1b7f8fc08a309098869c9 100644 (file)
@@ -1,2 +1,2 @@
 Minimal zone (only NS records) Make sure non-existent hosts generates a correct
-NSEC(3) denial.
\ No newline at end of file
+NSEC(3) denial.
diff --git a/regression-tests/tests/root-srv/failing.ldap-simple b/regression-tests/tests/root-srv/failing.ldap-simple
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/regression-tests/tests/root-srv/failing.ldap-tree b/regression-tests/tests/root-srv/failing.ldap-tree
new file mode 100644 (file)
index 0000000..e69de29
index 3ce51e86e86aa669f9f78a3d9c176544ce052169..1143a0ca09885c74555d5c7b907f23bdecf68220 100644 (file)
@@ -1,4 +1,3 @@
 This test checks the DS/NS response for a secure referral.
 It was written specifically to verify that we do not sign NS records
 at secure delegations.
-
diff --git a/regression-tests/tests/secure-delegation-ds-ns/failing.oracle-nsec b/regression-tests/tests/secure-delegation-ds-ns/failing.oracle-nsec
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/regression-tests/tests/secure-delegation-ds-ns/failing.oracle-nsec3 b/regression-tests/tests/secure-delegation-ds-ns/failing.oracle-nsec3
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/regression-tests/tests/secure-delegation/failing.ldap-simple b/regression-tests/tests/secure-delegation/failing.ldap-simple
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/regression-tests/tests/test-urc/failing.mydns b/regression-tests/tests/test-urc/failing.mydns
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/regression-tests/tests/unknown-domain/failing.ldap-simple b/regression-tests/tests/unknown-domain/failing.ldap-simple
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/regression-tests/tests/unknown-domain/failing.ldap-tree b/regression-tests/tests/unknown-domain/failing.ldap-tree
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/regression-tests/tests/verify-dnssec-zone/failing.oracle-nsec b/regression-tests/tests/verify-dnssec-zone/failing.oracle-nsec
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/regression-tests/tests/verify-dnssec-zone/failing.oracle-nsec3 b/regression-tests/tests/verify-dnssec-zone/failing.oracle-nsec3
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/regression-tests/tests/very-long-txt/failing.bind b/regression-tests/tests/very-long-txt/failing.bind
new file mode 100644 (file)
index 0000000..e69de29
diff --git a/regression-tests/tests/very-long-txt/failing.nsd b/regression-tests/tests/very-long-txt/failing.nsd
new file mode 100644 (file)
index 0000000..e69de29