The atofoo_test and hltq_test tests now display their own test error rate.

Display pass/fail count separately for sudo_conf and sudo_parseln tests.
Check stderr output for the sudo_conf test.

diff --git a/MANIFEST b/MANIFEST
index 6ef1633cad3ff1d2f0571adb513dc6d48e1d26d5..f5b45b81cf49cc28bbc8b6d99195b6f4c91be182 100644 (file)
--- a/MANIFEST
+++ b/MANIFEST
@@ -33,6 +33,7 @@ common/regress/sudo_conf/test3.in
 common/regress/sudo_conf/test3.out.ok
 common/regress/sudo_conf/test4.in
 common/regress/sudo_conf/test4.out.ok
+common/regress/sudo_conf/test5.err.ok
 common/regress/sudo_conf/test5.in
 common/regress/sudo_conf/test5.out.ok
 common/regress/sudo_conf/test6.in

diff --git a/common/Makefile.in b/common/Makefile.in
index d886dfe6331ca42db6c0c38d725f3ad72fc84418..6ff286769fe64024452949579bcd1cb62f64bec7 100644 (file)
--- a/common/Makefile.in
+++ b/common/Makefile.in
@@ -126,43 +126,59 @@ uninstall:
 cppcheck:
        cppcheck $(CPPCHECK_OPTS) -I$(incdir) -I$(top_builddir) -I$(top_srcdir) $(srcdir)/*.c
 
+# Note: some regress checks are run from srcdir for consistent error messages
 check: $(TEST_PROGS)
        @if test X"$(cross_compiling)" != X"yes"; then \
-           passed=0; failed=0; total=0; \
-           total=1; \
-           if ./atofoo_test; then \
-               passed=`expr $$passed + 1`; \
-           else \
-               failed=`expr $$failed + 1`; \
-           fi; \
-           if ./hltq_test; then \
-               passed=`expr $$passed + 1`; \
-           else \
-               failed=`expr $$failed + 1`; \
-           fi; \
+           rval=0; \
+           ./atofoo_test || rval=`expr $$rval + $$?`; \
+           ./hltq_test || rval=`expr $$rval + $$?`; \
+           build_dir=`pwd`; \
+           cd $(srcdir); \
            for dir in sudo_conf sudo_parseln; do \
-               mkdir -p regress/$$dir; \
-               for t in $(srcdir)/regress/$$dir/*.in; do \
+               passed=0; failed=0; total=0; \
+               mkdir -p $$build_dir/regress/$$dir; \
+               for t in regress/$$dir/*.in; do \
                    base=`basename $$t .in`; \
-                   out="regress/$$dir/$$base.out"; \
+                   out="$$build_dir/regress/$$dir/$${base}.out"; \
+                   out_ok="regress/$$dir/$${base}.out.ok"; \
+                   err="$$build_dir/regress/$$dir/$${base}.err"; \
+                   err_ok="regress/$$dir/$${base}.err.ok"; \
                    if test "$$dir" = "sudo_conf"; then \
-                       ./conf_test $$t >$$out; \
+                       $$build_dir/conf_test $$t >$$out 2>$$err; \
                    else \
-                       ./parseln_test <$$t >$$out; \
+                       $$build_dir/parseln_test <$$t >$$out 2>$$err; \
                    fi; \
-                   if cmp $$out $(srcdir)/$$out.ok >/dev/null; then \
+                   if cmp $$out $$out_ok >/dev/null; then \
                        passed=`expr $$passed + 1`; \
                        echo "$$dir/$$base: OK"; \
                    else \
                        failed=`expr $$failed + 1`; \
                        echo "$$dir/$$base: FAIL"; \
-                       diff $$out $(srcdir)/$$out.ok; \
+                       diff $$out $$out_ok || true; \
                    fi; \
                    total=`expr $$total + 1`; \
+                   if test -s $$err_ok; then \
+                       if cmp $$err $$err_ok >/dev/null; then \
+                           passed=`expr $$passed + 1`; \
+                           echo "$$dir/$$base (stderr): OK"; \
+                       else \
+                           failed=`expr $$failed + 1`; \
+                           echo "$$dir/$$base (stderr): FAIL"; \
+                           diff $$err $$err_ok || true; \
+                       fi; \
+                       total=`expr $$total + 1`; \
+                   elif test -s $$err; then \
+                       failed=`expr $$failed + 1`; \
+                       echo "$$dir/$$base (stderr): FAIL"; \
+                       cat $$err 1>&2; \
+                   fi; \
                done; \
+               if test $$failed -ne 0; then \
+                   rval=`expr $$rval + $$failed`; \
+               fi; \
+               echo "$$dir: $$passed/$$total tests passed; $$failed/$$total tests failed"; \
            done; \
-           echo "$$dir: $$passed/$$total tests passed; $$failed/$$total tests failed"; \
-           exit $$failed; \
+           exit $$rval; \
        fi
 
 clean:

diff --git a/common/regress/atofoo/atofoo_test.c b/common/regress/atofoo/atofoo_test.c
index 8c3f6d382d6a342d9545a8a66a391556ad62683b..512221c2a2299fc1adbc6b9a2f1619be01db6dc2 100644 (file)
--- a/common/regress/atofoo/atofoo_test.c
+++ b/common/regress/atofoo/atofoo_test.c
@@ -61,13 +61,14 @@ static struct atobool_data {
 };
 
 static int
-test_atobool(void)
+test_atobool(int *ntests)
 {
     struct atobool_data *d;
     int errors = 0;
     int value;
 
     for (d = atobool_data; d->bool_str != NULL; d++) {
+       (*ntests)++;
        value = atobool(d->bool_str);
        if (value != d->value) {
            warningx_nodebug("FAIL: %s != %d", d->bool_str, d->value);
@@ -94,7 +95,7 @@ static struct atoid_data {
 };
 
 static int
-test_atoid(void)
+test_atoid(int *ntests)
 {
     struct atoid_data *d;
     const char *errstr;
@@ -103,6 +104,7 @@ test_atoid(void)
     id_t value;
 
     for (d = atoid_data; d->idstr != NULL; d++) {
+       (*ntests)++;
        errstr = "some error";
        value = atoid(d->idstr, d->sep, &ep, &errstr);
        if (errstr != NULL) {
@@ -136,7 +138,7 @@ static struct atomode_data {
 };
 
 static int
-test_atomode(void)
+test_atomode(int *ntests)
 {
     struct atomode_data *d;
     const char *errstr;
@@ -144,6 +146,7 @@ test_atomode(void)
     mode_t mode;
 
     for (d = atomode_data; d->mode_str != NULL; d++) {
+       (*ntests)++;
        errstr = "some error";
        mode = atomode(d->mode_str, &errstr);
        if (errstr != NULL) {
@@ -167,12 +170,16 @@ int
 main(int argc, char *argv[])
 {
     int errors = 0;
+    int ntests = 0;
 
     initprogname(argc > 0 ? argv[0] : "atofoo");
 
-    errors += test_atobool();
-    errors += test_atoid();
-    errors += test_atomode();
+    errors += test_atobool(&ntests);
+    errors += test_atoid(&ntests);
+    errors += test_atomode(&ntests);
+
+    printf("%s: %d tests run, %d errors, %d%% success rate\n", getprogname(),
+       ntests, errors, (ntests - errors) * 100 / ntests);
 
     exit(errors);
 }

diff --git a/common/regress/sudo_conf/test5.err.ok b/common/regress/sudo_conf/test5.err.ok
new file mode 100644 (file)
index 0000000..f85dfc3
--- /dev/null
+++ b/common/regress/sudo_conf/test5.err.ok
@@ -0,0 +1 @@
+conf_test: invalid max groups `0' in regress/sudo_conf/test5.in, line 1

diff --git a/common/regress/tailq/hltq_test.c b/common/regress/tailq/hltq_test.c
index 42437e68bf943e1ae4b4f7a55faa5c2c522c31f3..fdbfb6050f3258d314e6fc09505abc5e9b22bffe 100644 (file)
--- a/common/regress/tailq/hltq_test.c
+++ b/common/regress/tailq/hltq_test.c
@@ -70,6 +70,7 @@ main(int argc, char *argv[])
     struct test_data *hltq;
     struct test_data_list tq;
     int errors = 0;
+    int ntests = 0;
 
     initprogname(argc > 0 ? argv[0] : "hltq_test");
 
@@ -83,14 +84,17 @@ main(int argc, char *argv[])
        warningx_nodebug("FAIL: HLTQ_FIRST(1 entry) doesn't return first element: got %p, expected %p", HLTQ_FIRST(&d1), &d1);
        errors++;
     }
+    ntests++;
     if (HLTQ_LAST(&d1, test_data, entries) != &d1) {
        warningx_nodebug("FAIL: HLTQ_LAST(1 entry) doesn't return first element: got %p, expected %p", HLTQ_LAST(&d1, test_data, entries), &d1);
        errors++;
     }
+    ntests++;
     if (HLTQ_PREV(&d1, test_data, entries) != NULL) {
        warningx_nodebug("FAIL: HLTQ_PREV(1 entry) doesn't return NULL: got %p", HLTQ_PREV(&d1, test_data, entries));
        errors++;
     }
+    ntests++;
 
     HLTQ_INIT(&d2, entries);
     d2.a = 2;
@@ -112,36 +116,44 @@ main(int argc, char *argv[])
        warningx_nodebug("FAIL: HLTQ_FIRST(3 entries) doesn't return first element: got %p, expected %p", HLTQ_FIRST(hltq), &d1);
        errors++;
     }
+    ntests++;
     if (HLTQ_LAST(hltq, test_data, entries) != &d3) {
        warningx_nodebug("FAIL: HLTQ_LAST(3 entries) doesn't return third element: got %p, expected %p", HLTQ_LAST(hltq, test_data, entries), &d3);
        errors++;
     }
+    ntests++;
 
     if (HLTQ_NEXT(&d1, entries) != &d2) {
        warningx_nodebug("FAIL: HLTQ_NEXT(&d1) doesn't return &d2: got %p, expected %p", HLTQ_NEXT(&d1, entries), &d2);
        errors++;
     }
+    ntests++;
     if (HLTQ_NEXT(&d2, entries) != &d3) {
        warningx_nodebug("FAIL: HLTQ_NEXT(&d2) doesn't return &d3: got %p, expected %p", HLTQ_NEXT(&d2, entries), &d3);
        errors++;
     }
+    ntests++;
     if (HLTQ_NEXT(&d3, entries) != NULL) {
        warningx_nodebug("FAIL: HLTQ_NEXT(&d3) doesn't return NULL: got %p", HLTQ_NEXT(&d3, entries));
        errors++;
     }
+    ntests++;
 
     if (HLTQ_PREV(&d1, test_data, entries) != NULL) {
        warningx_nodebug("FAIL: HLTQ_PREV(&d1) doesn't return NULL: got %p", HLTQ_PREV(&d1, test_data, entries));
        errors++;
     }
+    ntests++;
     if (HLTQ_PREV(&d2, test_data, entries) != &d1) {
        warningx_nodebug("FAIL: HLTQ_PREV(&d2) doesn't return &d1: got %p, expected %p", HLTQ_PREV(&d2, test_data, entries), &d1);
        errors++;
     }
+    ntests++;
     if (HLTQ_PREV(&d3, test_data, entries) != &d2) {
        warningx_nodebug("FAIL: HLTQ_PREV(&d3) doesn't return &d2: got %p, expected %p", HLTQ_PREV(&d3, test_data, entries), &d2);
        errors++;
     }
+    ntests++;
 
     /* Test conversion to TAILQ. */
     HLTQ_TO_TAILQ(&tq, hltq, entries);
@@ -150,36 +162,47 @@ main(int argc, char *argv[])
        warningx_nodebug("FAIL: TAILQ_FIRST(&tq) doesn't return first element: got %p, expected %p", TAILQ_FIRST(&tq), &d1);
        errors++;
     }
+    ntests++;
     if (TAILQ_LAST(&tq, test_data_list) != &d3) {
        warningx_nodebug("FAIL: TAILQ_LAST(&tq) doesn't return third element: got %p, expected %p", TAILQ_LAST(&tq, test_data_list), &d3);
        errors++;
     }
+    ntests++;
 
     if (TAILQ_NEXT(&d1, entries) != &d2) {
        warningx_nodebug("FAIL: TAILQ_NEXT(&d1) doesn't return &d2: got %p, expected %p", TAILQ_NEXT(&d1, entries), &d2);
        errors++;
     }
+    ntests++;
     if (TAILQ_NEXT(&d2, entries) != &d3) {
        warningx_nodebug("FAIL: TAILQ_NEXT(&d2) doesn't return &d3: got %p, expected %p", TAILQ_NEXT(&d2, entries), &d3);
        errors++;
     }
+    ntests++;
     if (TAILQ_NEXT(&d3, entries) != NULL) {
        warningx_nodebug("FAIL: TAILQ_NEXT(&d3) doesn't return NULL: got %p", TAILQ_NEXT(&d3, entries));
        errors++;
     }
+    ntests++;
 
     if (TAILQ_PREV(&d1, test_data_list, entries) != NULL) {
        warningx_nodebug("FAIL: TAILQ_PREV(&d1) doesn't return NULL: got %p", TAILQ_PREV(&d1, test_data_list, entries));
        errors++;
     }
+    ntests++;
     if (TAILQ_PREV(&d2, test_data_list, entries) != &d1) {
        warningx_nodebug("FAIL: TAILQ_PREV(&d2) doesn't return &d1: got %p, expected %p", TAILQ_PREV(&d2, test_data_list, entries), &d1);
        errors++;
     }
+    ntests++;
     if (TAILQ_PREV(&d3, test_data_list, entries) != &d2) {
        warningx_nodebug("FAIL: TAILQ_PREV(&d3) doesn't return &d2: got %p, expected %p", TAILQ_PREV(&d3, test_data_list, entries), &d2);
        errors++;
     }
+    ntests++;
+
+    printf("%s: %d tests run, %d errors, %d%% success rate\n", getprogname(),
+       ntests, errors, (ntests - errors) * 100 / ntests);
 
     exit(errors);
 }