From: Kees Monshouwer Date: Sun, 21 Feb 2016 09:46:58 +0000 (+0100) Subject: add 'fail.*' option to the regression-test X-Git-Tag: auth-4.0.0-alpha2~11^2~4 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=d6fbf4e292fd920621fe3abc39616265c2a17ca7;p=pdns add 'fail.*' option to the regression-test A failing test must be brokem. --- diff --git a/regression-tests/runtests b/regression-tests/runtests index ccefb1ca3..2ed6015f3 100755 --- a/regression-tests/runtests +++ b/regression-tests/runtests @@ -66,6 +66,16 @@ do done fi + SHOULDFAIL=0 + for reason in $skipreasons $context + do + if [ -e $a/failing.$reason ] + then + SHOULDFAIL=1 + break + fi + done + if [ "$spectest" != "" ] && [ "$spectest" != "$testname" ] && [ "$testname" != "00dnssec-grabkeys" ] then SKIPIT=1 @@ -95,13 +105,27 @@ do diff ${diffopts} $expected $a/real_result > $a/diff 2>&1 if [ -s $a/diff ] then - result=" Failed test $a" - echo $testname >> failed_tests - failed=$[$failed+1] + if [ $SHOULDFAIL = 0 ] + then + result=" Failed test $a" + echo $testname >> failed_tests + failed=$[$failed+1] + else + result=" Expected failure for reason $reason, test passed $a" + echo $testname >> passed_tests + passed=$[$passed+1] + fi else - result=" Passed test $a" - echo $testname >> passed_tests - passed=$[$passed+1] + if [ $SHOULDFAIL = 0 ] + then + result=" Passed test $a" + echo $testname >> passed_tests + passed=$[$passed+1] + else + result=" Unexpected pass for reason $reason, test failed $a" + echo $testname >> failed_tests + failed=$[$failed+1] + fi fi fi echo "$result" diff --git a/regression-tests/tests/8bit-txt-unescaped/skip.ldap-simple b/regression-tests/tests/8bit-txt-unescaped/failing.ldap-simple similarity index 100% rename from regression-tests/tests/8bit-txt-unescaped/skip.ldap-simple rename to regression-tests/tests/8bit-txt-unescaped/failing.ldap-simple diff --git a/regression-tests/tests/8bit-txt-unescaped/skip.ldap-tree b/regression-tests/tests/8bit-txt-unescaped/failing.ldap-tree similarity index 100% rename from regression-tests/tests/8bit-txt-unescaped/skip.ldap-tree rename to regression-tests/tests/8bit-txt-unescaped/failing.ldap-tree diff --git a/regression-tests/tests/any-query/skip.ldap-simple b/regression-tests/tests/any-query/failing.ldap-simple similarity index 100% rename from regression-tests/tests/any-query/skip.ldap-simple rename to regression-tests/tests/any-query/failing.ldap-simple diff --git a/regression-tests/tests/any-query/skip.ldap-tree b/regression-tests/tests/any-query/failing.ldap-tree similarity index 100% rename from regression-tests/tests/any-query/skip.ldap-tree rename to regression-tests/tests/any-query/failing.ldap-tree diff --git a/regression-tests/tests/basic-eui48/skip.bind b/regression-tests/tests/basic-eui48/failing.bind similarity index 100% rename from regression-tests/tests/basic-eui48/skip.bind rename to regression-tests/tests/basic-eui48/failing.bind diff --git a/regression-tests/tests/basic-eui48/skip.mydns b/regression-tests/tests/basic-eui48/failing.mydns similarity index 100% rename from regression-tests/tests/basic-eui48/skip.mydns rename to regression-tests/tests/basic-eui48/failing.mydns diff --git a/regression-tests/tests/basic-eui48/skip.nsd b/regression-tests/tests/basic-eui48/failing.nsd similarity index 100% rename from regression-tests/tests/basic-eui48/skip.nsd rename to regression-tests/tests/basic-eui48/failing.nsd diff --git a/regression-tests/tests/basic-eui64/skip.bind b/regression-tests/tests/basic-eui64/failing.bind similarity index 100% rename from regression-tests/tests/basic-eui64/skip.bind rename to regression-tests/tests/basic-eui64/failing.bind diff --git a/regression-tests/tests/basic-eui64/skip.mydns b/regression-tests/tests/basic-eui64/failing.mydns similarity index 100% rename from regression-tests/tests/basic-eui64/skip.mydns rename to regression-tests/tests/basic-eui64/failing.mydns diff --git a/regression-tests/tests/basic-eui64/skip.nsd b/regression-tests/tests/basic-eui64/failing.nsd similarity index 100% rename from regression-tests/tests/basic-eui64/skip.nsd rename to regression-tests/tests/basic-eui64/failing.nsd diff --git a/regression-tests/tests/basic-loc/skip.mydns b/regression-tests/tests/basic-loc/failing.mydns similarity index 100% rename from regression-tests/tests/basic-loc/skip.mydns rename to regression-tests/tests/basic-loc/failing.mydns diff --git a/regression-tests/tests/basic-ns-resolution/skip.ldap-simple b/regression-tests/tests/basic-nonzone/failing.ldap-simple similarity index 100% rename from regression-tests/tests/basic-ns-resolution/skip.ldap-simple rename to regression-tests/tests/basic-nonzone/failing.ldap-simple diff --git a/regression-tests/tests/basic-ns-resolution/skip.ldap-tree b/regression-tests/tests/basic-nonzone/failing.ldap-tree similarity index 100% rename from regression-tests/tests/basic-ns-resolution/skip.ldap-tree rename to regression-tests/tests/basic-nonzone/failing.ldap-tree diff --git a/regression-tests/tests/cname-and-wildcard-but-no-correct-type/skip.oracle-slave-nsec3 b/regression-tests/tests/basic-ns-resolution/failing.ldap-simple similarity index 100% rename from regression-tests/tests/cname-and-wildcard-but-no-correct-type/skip.oracle-slave-nsec3 rename to regression-tests/tests/basic-ns-resolution/failing.ldap-simple diff --git a/regression-tests/tests/cname-wildcard-chain/skip.oracle-nsec3 b/regression-tests/tests/basic-ns-resolution/failing.ldap-tree similarity index 100% rename from regression-tests/tests/cname-wildcard-chain/skip.oracle-nsec3 rename to regression-tests/tests/basic-ns-resolution/failing.ldap-tree diff --git a/regression-tests/tests/dname-self/skip.mydns b/regression-tests/tests/cname-and-wildcard-but-no-correct-type/failing.oracle-slave-nsec3 similarity index 100% rename from regression-tests/tests/dname-self/skip.mydns rename to regression-tests/tests/cname-and-wildcard-but-no-correct-type/failing.oracle-slave-nsec3 diff --git a/regression-tests/tests/dname/skip.mydns b/regression-tests/tests/cname-to-unauth-any/failing.ldap-simple similarity index 100% rename from regression-tests/tests/dname/skip.mydns rename to regression-tests/tests/cname-to-unauth-any/failing.ldap-simple diff --git a/regression-tests/tests/double/skip.mydns b/regression-tests/tests/cname-to-unauth-any/failing.ldap-tree similarity index 100% rename from regression-tests/tests/double/skip.mydns rename to regression-tests/tests/cname-to-unauth-any/failing.ldap-tree diff --git a/regression-tests/tests/ds-at-secure-delegation/skip.mydns b/regression-tests/tests/cname-to-unauth/failing.ldap-simple similarity index 100% rename from regression-tests/tests/ds-at-secure-delegation/skip.mydns rename to regression-tests/tests/cname-to-unauth/failing.ldap-simple diff --git a/regression-tests/tests/ds-at-unsecure-delegation/skip.oracle-nsec b/regression-tests/tests/cname-to-unauth/failing.ldap-tree similarity index 100% rename from regression-tests/tests/ds-at-unsecure-delegation/skip.oracle-nsec rename to regression-tests/tests/cname-to-unauth/failing.ldap-tree diff --git a/regression-tests/tests/ds-at-unsecure-zone-cut/skip.oracle-nsec b/regression-tests/tests/cname-wildcard-chain/failing.oracle-nsec3 similarity index 100% rename from regression-tests/tests/ds-at-unsecure-zone-cut/skip.oracle-nsec rename to regression-tests/tests/cname-wildcard-chain/failing.oracle-nsec3 diff --git a/regression-tests/tests/ds-inside-delegation/skip.oracle-nsec b/regression-tests/tests/dname-self/failing.mydns similarity index 100% rename from regression-tests/tests/ds-inside-delegation/skip.oracle-nsec rename to regression-tests/tests/dname-self/failing.mydns diff --git a/regression-tests/tests/ds-inside-delegation/skip.oracle-nsec3 b/regression-tests/tests/dname/failing.ldap-simple similarity index 100% rename from regression-tests/tests/ds-inside-delegation/skip.oracle-nsec3 rename to regression-tests/tests/dname/failing.ldap-simple diff --git a/regression-tests/tests/ent-asterisk/skip.bind b/regression-tests/tests/dname/failing.ldap-tree similarity index 100% rename from regression-tests/tests/ent-asterisk/skip.bind rename to regression-tests/tests/dname/failing.ldap-tree diff --git a/regression-tests/tests/long-name/skip.oracle-slave-nsec3 b/regression-tests/tests/dname/failing.mydns similarity index 100% rename from regression-tests/tests/long-name/skip.oracle-slave-nsec3 rename to regression-tests/tests/dname/failing.mydns diff --git a/regression-tests/tests/mx-case-sensitivy-with-ap/skip.ldap-simple b/regression-tests/tests/double/failing.mydns similarity index 100% rename from regression-tests/tests/mx-case-sensitivy-with-ap/skip.ldap-simple rename to regression-tests/tests/double/failing.mydns diff --git a/regression-tests/tests/ds-at-secure-delegation/description b/regression-tests/tests/ds-at-secure-delegation/description index 0b14d3583..ac02e4e5f 100644 --- a/regression-tests/tests/ds-at-secure-delegation/description +++ b/regression-tests/tests/ds-at-secure-delegation/description @@ -1,4 +1,3 @@ This test tries to resolve a DS question at a secure delegation. It was written specifically to verify that we do not sign NS records at secure delegations. - diff --git a/regression-tests/tests/mx-case-sensitivy-with-ap/skip.ldap-tree b/regression-tests/tests/ds-at-secure-delegation/failing.mydns similarity index 100% rename from regression-tests/tests/mx-case-sensitivy-with-ap/skip.ldap-tree rename to regression-tests/tests/ds-at-secure-delegation/failing.mydns diff --git a/regression-tests/tests/ds-at-unsecure-delegation/description b/regression-tests/tests/ds-at-unsecure-delegation/description index 689afdf0a..b2bf44fe3 100644 --- a/regression-tests/tests/ds-at-unsecure-delegation/description +++ b/regression-tests/tests/ds-at-unsecure-delegation/description @@ -1,2 +1 @@ This test tries to resolve a DS question at an unsecure delegation. - diff --git a/regression-tests/tests/mx-with-simple-additional-processing/skip.ldap-simple b/regression-tests/tests/ds-at-unsecure-delegation/failing.oracle-nsec similarity index 100% rename from regression-tests/tests/mx-with-simple-additional-processing/skip.ldap-simple rename to regression-tests/tests/ds-at-unsecure-delegation/failing.oracle-nsec diff --git a/regression-tests/tests/mx-with-simple-additional-processing/skip.ldap-tree b/regression-tests/tests/ds-at-unsecure-zone-cut/failing.oracle-nsec similarity index 100% rename from regression-tests/tests/mx-with-simple-additional-processing/skip.ldap-tree rename to regression-tests/tests/ds-at-unsecure-zone-cut/failing.oracle-nsec diff --git a/regression-tests/tests/ns-with-identical-glue/skip.oracle-slave-nsec3 b/regression-tests/tests/ds-inside-delegation/failing.oracle-nsec similarity index 100% rename from regression-tests/tests/ns-with-identical-glue/skip.oracle-slave-nsec3 rename to regression-tests/tests/ds-inside-delegation/failing.oracle-nsec diff --git a/regression-tests/tests/nsec-glue-at-delegation/skip.oracle-nsec b/regression-tests/tests/ds-inside-delegation/failing.oracle-nsec3 similarity index 100% rename from regression-tests/tests/nsec-glue-at-delegation/skip.oracle-nsec rename to regression-tests/tests/ds-inside-delegation/failing.oracle-nsec3 diff --git a/regression-tests/tests/nsec-glue-at-delegation/skip.oracle-nsec3 b/regression-tests/tests/ent-asterisk/failing.bind similarity index 100% rename from regression-tests/tests/nsec-glue-at-delegation/skip.oracle-nsec3 rename to regression-tests/tests/ent-asterisk/failing.bind diff --git a/regression-tests/tests/nsec-glue/skip.oracle-nsec b/regression-tests/tests/external-cname-pointer/failing.ldap-simple similarity index 100% rename from regression-tests/tests/nsec-glue/skip.oracle-nsec rename to regression-tests/tests/external-cname-pointer/failing.ldap-simple diff --git a/regression-tests/tests/nsec3-hash-query/skip.oracle-nsec b/regression-tests/tests/external-cname-pointer/failing.ldap-tree similarity index 100% rename from regression-tests/tests/nsec3-hash-query/skip.oracle-nsec rename to regression-tests/tests/external-cname-pointer/failing.ldap-tree diff --git a/regression-tests/tests/root-mx/skip.ldap-simple b/regression-tests/tests/long-name/failing.oracle-slave-nsec3 similarity index 100% rename from regression-tests/tests/root-mx/skip.ldap-simple rename to regression-tests/tests/long-name/failing.oracle-slave-nsec3 diff --git a/regression-tests/tests/minimal-nxdomain/description b/regression-tests/tests/minimal-nxdomain/description index 7a2790e71..212c5e8bd 100644 --- a/regression-tests/tests/minimal-nxdomain/description +++ b/regression-tests/tests/minimal-nxdomain/description @@ -1,2 +1,2 @@ Minimal zone (only NS records) Make sure non-existent hosts generates a correct -NSEC(3) denial. \ No newline at end of file +NSEC(3) denial. diff --git a/regression-tests/tests/root-mx/skip.ldap-tree b/regression-tests/tests/mx-case-sensitivy-with-ap/failing.ldap-simple similarity index 100% rename from regression-tests/tests/root-mx/skip.ldap-tree rename to regression-tests/tests/mx-case-sensitivy-with-ap/failing.ldap-simple diff --git a/regression-tests/tests/root-srv/skip.ldap-simple b/regression-tests/tests/mx-case-sensitivy-with-ap/failing.ldap-tree similarity index 100% rename from regression-tests/tests/root-srv/skip.ldap-simple rename to regression-tests/tests/mx-case-sensitivy-with-ap/failing.ldap-tree diff --git a/regression-tests/tests/root-srv/skip.ldap-tree b/regression-tests/tests/mx-with-simple-additional-processing/failing.ldap-simple similarity index 100% rename from regression-tests/tests/root-srv/skip.ldap-tree rename to regression-tests/tests/mx-with-simple-additional-processing/failing.ldap-simple diff --git a/regression-tests/tests/secure-delegation-ds-ns/skip.oracle-nsec b/regression-tests/tests/mx-with-simple-additional-processing/failing.ldap-tree similarity index 100% rename from regression-tests/tests/secure-delegation-ds-ns/skip.oracle-nsec rename to regression-tests/tests/mx-with-simple-additional-processing/failing.ldap-tree diff --git a/regression-tests/tests/secure-delegation-ds-ns/skip.oracle-nsec3 b/regression-tests/tests/ns-with-identical-glue/failing.oracle-slave-nsec3 similarity index 100% rename from regression-tests/tests/secure-delegation-ds-ns/skip.oracle-nsec3 rename to regression-tests/tests/ns-with-identical-glue/failing.oracle-slave-nsec3 diff --git a/regression-tests/tests/secure-delegation/skip.ldap-simple b/regression-tests/tests/nsec-glue-at-delegation/failing.oracle-nsec similarity index 100% rename from regression-tests/tests/secure-delegation/skip.ldap-simple rename to regression-tests/tests/nsec-glue-at-delegation/failing.oracle-nsec diff --git a/regression-tests/tests/test-urc/skip.mydns b/regression-tests/tests/nsec-glue-at-delegation/failing.oracle-nsec3 similarity index 100% rename from regression-tests/tests/test-urc/skip.mydns rename to regression-tests/tests/nsec-glue-at-delegation/failing.oracle-nsec3 diff --git a/regression-tests/tests/verify-dnssec-zone/skip.oracle-nsec b/regression-tests/tests/nsec-glue/failing.oracle-nsec similarity index 100% rename from regression-tests/tests/verify-dnssec-zone/skip.oracle-nsec rename to regression-tests/tests/nsec-glue/failing.oracle-nsec diff --git a/regression-tests/tests/verify-dnssec-zone/skip.oracle-nsec3 b/regression-tests/tests/nsec3-hash-query/failing.oracle-nsec similarity index 100% rename from regression-tests/tests/verify-dnssec-zone/skip.oracle-nsec3 rename to regression-tests/tests/nsec3-hash-query/failing.oracle-nsec diff --git a/regression-tests/tests/very-long-txt/skip.bind b/regression-tests/tests/root-mx/failing.ldap-simple similarity index 100% rename from regression-tests/tests/very-long-txt/skip.bind rename to regression-tests/tests/root-mx/failing.ldap-simple diff --git a/regression-tests/tests/very-long-txt/skip.nsd b/regression-tests/tests/root-mx/failing.ldap-tree similarity index 100% rename from regression-tests/tests/very-long-txt/skip.nsd rename to regression-tests/tests/root-mx/failing.ldap-tree diff --git a/regression-tests/tests/root-srv/failing.ldap-simple b/regression-tests/tests/root-srv/failing.ldap-simple new file mode 100644 index 000000000..e69de29bb diff --git a/regression-tests/tests/root-srv/failing.ldap-tree b/regression-tests/tests/root-srv/failing.ldap-tree new file mode 100644 index 000000000..e69de29bb diff --git a/regression-tests/tests/secure-delegation-ds-ns/description b/regression-tests/tests/secure-delegation-ds-ns/description index 3ce51e86e..1143a0ca0 100644 --- a/regression-tests/tests/secure-delegation-ds-ns/description +++ b/regression-tests/tests/secure-delegation-ds-ns/description @@ -1,4 +1,3 @@ This test checks the DS/NS response for a secure referral. It was written specifically to verify that we do not sign NS records at secure delegations. - diff --git a/regression-tests/tests/secure-delegation-ds-ns/failing.oracle-nsec b/regression-tests/tests/secure-delegation-ds-ns/failing.oracle-nsec new file mode 100644 index 000000000..e69de29bb diff --git a/regression-tests/tests/secure-delegation-ds-ns/failing.oracle-nsec3 b/regression-tests/tests/secure-delegation-ds-ns/failing.oracle-nsec3 new file mode 100644 index 000000000..e69de29bb diff --git a/regression-tests/tests/secure-delegation/failing.ldap-simple b/regression-tests/tests/secure-delegation/failing.ldap-simple new file mode 100644 index 000000000..e69de29bb diff --git a/regression-tests/tests/test-urc/failing.mydns b/regression-tests/tests/test-urc/failing.mydns new file mode 100644 index 000000000..e69de29bb diff --git a/regression-tests/tests/unknown-domain/failing.ldap-simple b/regression-tests/tests/unknown-domain/failing.ldap-simple new file mode 100644 index 000000000..e69de29bb diff --git a/regression-tests/tests/unknown-domain/failing.ldap-tree b/regression-tests/tests/unknown-domain/failing.ldap-tree new file mode 100644 index 000000000..e69de29bb diff --git a/regression-tests/tests/verify-dnssec-zone/failing.oracle-nsec b/regression-tests/tests/verify-dnssec-zone/failing.oracle-nsec new file mode 100644 index 000000000..e69de29bb diff --git a/regression-tests/tests/verify-dnssec-zone/failing.oracle-nsec3 b/regression-tests/tests/verify-dnssec-zone/failing.oracle-nsec3 new file mode 100644 index 000000000..e69de29bb diff --git a/regression-tests/tests/very-long-txt/failing.bind b/regression-tests/tests/very-long-txt/failing.bind new file mode 100644 index 000000000..e69de29bb diff --git a/regression-tests/tests/very-long-txt/skip.mydns b/regression-tests/tests/very-long-txt/failing.mydns similarity index 100% rename from regression-tests/tests/very-long-txt/skip.mydns rename to regression-tests/tests/very-long-txt/failing.mydns diff --git a/regression-tests/tests/very-long-txt/failing.nsd b/regression-tests/tests/very-long-txt/failing.nsd new file mode 100644 index 000000000..e69de29bb