run-tests improvements:

  Add XLEAK section, this is made necessary by the support
  for valgrind tools other than memcheck.
  XLEAK is to a leaking test what XFAIL is to a failing test.
  Tidy up summary a little to exclude Expected Fail and Expected Leak
  when there are no expected fail/leaks

diff --git a/run-tests.php b/run-tests.php
index df959914866cf8c2a94a0bfb4dc570d8682788ea..7abc4aedee1a061bf3e14a405a67116f1e63562f 100755 (executable)
--- a/run-tests.php
+++ b/run-tests.php
@@ -300,6 +300,7 @@ NO_PROC_OPEN_ERROR;
                'WARNED' => array(),
                'LEAKED' => array(),
                'XFAILED' => array(),
+               'XLEAKED' => array(),
                'SLOW' => array()
        );
 
@@ -566,7 +567,7 @@ Options:
                 with value 'bar').
 
     -g          Comma separated list of groups to show during test run
-                (possible values: PASS, FAIL, XFAIL, SKIP, BORK, WARN, LEAK, REDIRECT).
+                (possible values: PASS, FAIL, XFAIL, XLEAK, SKIP, BORK, WARN, LEAK, REDIRECT).
 
     -m          Test for memory leaks with Valgrind (equivalent to -M memcheck).
 
@@ -1313,7 +1314,7 @@ function run_all_tests($test_files, $env, $redir_tested = null)
                $test_idx++;
 
                if ($workerID) {
-                       $PHP_FAILED_TESTS = ['BORKED' => [], 'FAILED' => [], 'WARNED' => [], 'LEAKED' => [], 'XFAILED' => [], 'SLOW' => []];
+                       $PHP_FAILED_TESTS = ['BORKED' => [], 'FAILED' => [], 'WARNED' => [], 'LEAKED' => [], 'XFAILED' => [], 'XLEAKED' => [], 'SLOW' => []];
                        ob_start();
                }
 
@@ -1336,7 +1337,7 @@ function run_all_tests($test_files, $env, $redir_tested = null)
                        }
 
                        $test_results[$index] = $result;
-                       if ($failed_tests_file && ($result == 'XFAILED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) {
+                       if ($failed_tests_file && ($result == 'XFAILED' || $result == 'XLEAKED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) {
                                fwrite($failed_tests_file, "$index\n");
                        }
                        if ($result_tests_file) {
@@ -1594,7 +1595,7 @@ escape:
                                                        if (!is_array($name) && $result != 'REDIR') {
                                                                $test_results[$index] = $result;
 
-                                                               if ($failed_tests_file && ($result == 'XFAILED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) {
+                                                               if ($failed_tests_file && ($result == 'XFAILED' || $result == 'XLEAKED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) {
                                                                        fwrite($failed_tests_file, "$index\n");
                                                                }
                                                                if ($result_tests_file) {
@@ -1822,7 +1823,7 @@ TEST $file
                                'FILE', 'FILEEOF', 'FILE_EXTERNAL', 'REDIRECTTEST',
                                'CAPTURE_STDIO', 'STDIN', 'CGI', 'PHPDBG',
                                'INI', 'ENV', 'EXTENSIONS',
-                               'SKIPIF', 'XFAIL', 'CLEAN',
+                               'SKIPIF', 'XFAIL', 'XLEAK', 'CLEAN',
                                'CREDITS', 'DESCRIPTION', 'CONFLICTS',
                        ))) {
                                $bork_info = 'Unknown section "' . $section . '"';
@@ -2625,7 +2626,10 @@ COMMAND $cmd
                                if (isset($section_text['XFAIL'])) {
                                        $warn = true;
                                        $info = " (warn: XFAIL section but test passes)";
-                               } else {
+                               } else if (isset($section_text['XLEAK'])) {
+                                       $warn = true;
+                                       $info = " (warn: XLEAK section but test passes)";
+                } else {
                                        show_result("PASS", $tested, $tested_file, '', $temp_filenames);
                                        junit_mark_test_as('PASS', $shortname, $tested);
                                        return 'PASSED';
@@ -2651,7 +2655,10 @@ COMMAND $cmd
                                if (isset($section_text['XFAIL'])) {
                                        $warn = true;
                                        $info = " (warn: XFAIL section but test passes)";
-                               } else {
+                               } if (isset($section_text['XLEAK'])) {
+                                       $warn = true;
+                                       $info = " (warn: XLEAK section but test passes)";      
+                } else {
                                        show_result("PASS", $tested, $tested_file, '', $temp_filenames);
                                        junit_mark_test_as('PASS', $shortname, $tested);
                                        return 'PASSED';
@@ -2674,7 +2681,8 @@ COMMAND $cmd
        }
 
        if ($leaked) {
-               $restype[] = 'LEAK';
+        $restype[] = isset($section_text['XLEAK']) ? 
+                        'XLEAK' : 'LEAK';
        }
 
        if ($warn) {
@@ -2685,7 +2693,10 @@ COMMAND $cmd
                if (isset($section_text['XFAIL'])) {
                        $restype[] = 'XFAIL';
                        $info = '  XFAIL REASON: ' . rtrim($section_text['XFAIL']);
-               } else {
+               } else if ($section_text['XLEAK']) {
+            $restype[] = 'XLEAK';
+                       $info = '  XLEAK REASON: ' . rtrim($section_text['XLEAK']);
+        } else {
                        $restype[] = 'FAIL';
                }
        }
@@ -2962,7 +2973,8 @@ function compute_summary()
                'FAILED' => 0,
                'BORKED' => 0,
                'LEAKED' => 0,
-               'XFAILED' => 0
+               'XFAILED' => 0,
+               'XLEAKED' => 0
        );
 
        foreach ($test_results as $v) {
@@ -2987,10 +2999,11 @@ function get_summary($show_ext_summary, $show_html)
                $x_warned = (100.0 * $sum_results['WARNED']) / $x_total;
                $x_failed = (100.0 * $sum_results['FAILED']) / $x_total;
                $x_xfailed = (100.0 * $sum_results['XFAILED']) / $x_total;
+               $x_xleaked = (100.0 * $sum_results['XLEAKED']) / $x_total;
                $x_leaked = (100.0 * $sum_results['LEAKED']) / $x_total;
                $x_passed = (100.0 * $sum_results['PASSED']) / $x_total;
        } else {
-               $x_warned = $x_failed = $x_passed = $x_leaked = $x_xfailed = 0;
+               $x_warned = $x_failed = $x_passed = $x_leaked = $x_xfailed = $x_xleaked = 0;
        }
 
        $summary = '';
@@ -3021,12 +3034,20 @@ Tests borked    : ' . sprintf('%4d (%5.1f%%)', $sum_results['BORKED'], $percent_
        $summary .= '
 Tests skipped   : ' . sprintf('%4d (%5.1f%%)', $sum_results['SKIPPED'], $percent_results['SKIPPED']) . ' --------
 Tests warned    : ' . sprintf('%4d (%5.1f%%)', $sum_results['WARNED'], $percent_results['WARNED']) . ' ' . sprintf('(%5.1f%%)', $x_warned) . '
-Tests failed    : ' . sprintf('%4d (%5.1f%%)', $sum_results['FAILED'], $percent_results['FAILED']) . ' ' . sprintf('(%5.1f%%)', $x_failed) . '
+Tests failed    : ' . sprintf('%4d (%5.1f%%)', $sum_results['FAILED'], $percent_results['FAILED']) . ' ' . sprintf('(%5.1f%%)', $x_failed);
+
+    if ($sum_results['XFAILED']) {
+        $summary .= '
 Expected fail   : ' . sprintf('%4d (%5.1f%%)', $sum_results['XFAILED'], $percent_results['XFAILED']) . ' ' . sprintf('(%5.1f%%)', $x_xfailed);
+    }
 
        if ($valgrind) {
                $summary .= '
 Tests leaked    : ' . sprintf('%4d (%5.1f%%)', $sum_results['LEAKED'], $percent_results['LEAKED']) . ' ' . sprintf('(%5.1f%%)', $x_leaked);
+        if ($sum_results['XLEAKED']) {
+            $summary .= '
+Expected leak   : ' . sprintf('%4d (%5.1f%%)', $sum_results['XLEAKED'], $percent_results['XLEAKED']) . ' ' . sprintf('(%5.1f%%)', $x_xleaked);
+        }
        }
 
        $summary .= '
@@ -3115,6 +3136,19 @@ LEAKED TEST SUMMARY
                $failed_test_summary .= "=====================================================================\n";
        }
 
+       if (count($PHP_FAILED_TESTS['XLEAKED'])) {
+               $failed_test_summary .= '
+=====================================================================
+EXPECTED LEAK TEST SUMMARY
+---------------------------------------------------------------------
+';
+               foreach ($PHP_FAILED_TESTS['XLEAKED'] as $failed_test_data) {
+                       $failed_test_summary .= $failed_test_data['test_name'] . $failed_test_data['info'] . "\n";
+               }
+
+               $failed_test_summary .= "=====================================================================\n";
+       }
+
        if ($failed_test_summary && !getenv('NO_PHPTEST_SUMMARY')) {
                $summary .= $failed_test_summary;
        }
@@ -3390,13 +3424,13 @@ function junit_mark_test_as($type, $file_name, $test_name, $time = null, $messag
 
        if (is_array($type)) {
                $output_type = $type[0] . 'ED';
-               $temp = array_intersect(array('XFAIL', 'FAIL', 'WARN'), $type);
+               $temp = array_intersect(array('XFAIL', 'XLEAK', 'FAIL', 'WARN'), $type);
                $type = reset($temp);
        } else {
                $output_type = $type . 'ED';
        }
 
-       if ('PASS' == $type || 'XFAIL' == $type) {
+       if ('PASS' == $type || 'XFAIL' == $type || 'XLEAK' == $type) {
                junit_suite_record($suite, 'test_pass');
        } elseif ('BORK' == $type) {
                junit_suite_record($suite, 'test_error');