From e1151c9549cf5a7235c1609019f79bbb1bbcdbea Mon Sep 17 00:00:00 2001 From: Joe Watkins Date: Thu, 23 May 2019 10:55:25 +0200 Subject: [PATCH] run-tests improvements: Add XLEAK section, this is made necessary by the support for valgrind tools other than memcheck. XLEAK is to a leaking test what XFAIL is to a failing test. Tidy up summary a little to exclude Expected Fail and Expected Leak when there are no expected fail/leaks --- run-tests.php | 62 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/run-tests.php b/run-tests.php index df95991486..7abc4aedee 100755 --- a/run-tests.php +++ b/run-tests.php @@ -300,6 +300,7 @@ NO_PROC_OPEN_ERROR; 'WARNED' => array(), 'LEAKED' => array(), 'XFAILED' => array(), + 'XLEAKED' => array(), 'SLOW' => array() ); @@ -566,7 +567,7 @@ Options: with value 'bar'). -g Comma separated list of groups to show during test run - (possible values: PASS, FAIL, XFAIL, SKIP, BORK, WARN, LEAK, REDIRECT). + (possible values: PASS, FAIL, XFAIL, XLEAK, SKIP, BORK, WARN, LEAK, REDIRECT). -m Test for memory leaks with Valgrind (equivalent to -M memcheck). @@ -1313,7 +1314,7 @@ function run_all_tests($test_files, $env, $redir_tested = null) $test_idx++; if ($workerID) { - $PHP_FAILED_TESTS = ['BORKED' => [], 'FAILED' => [], 'WARNED' => [], 'LEAKED' => [], 'XFAILED' => [], 'SLOW' => []]; + $PHP_FAILED_TESTS = ['BORKED' => [], 'FAILED' => [], 'WARNED' => [], 'LEAKED' => [], 'XFAILED' => [], 'XLEAKED' => [], 'SLOW' => []]; ob_start(); } @@ -1336,7 +1337,7 @@ function run_all_tests($test_files, $env, $redir_tested = null) } $test_results[$index] = $result; - if ($failed_tests_file && ($result == 'XFAILED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) { + if ($failed_tests_file && ($result == 'XFAILED' || $result == 'XLEAKED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) { fwrite($failed_tests_file, "$index\n"); } if ($result_tests_file) { @@ -1594,7 +1595,7 @@ escape: if (!is_array($name) && $result != 'REDIR') { $test_results[$index] = $result; - if ($failed_tests_file && ($result == 'XFAILED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) { + if ($failed_tests_file && ($result == 'XFAILED' || $result == 'XLEAKED' || $result == 'FAILED' || $result == 'WARNED' || $result == 'LEAKED')) { fwrite($failed_tests_file, "$index\n"); } if ($result_tests_file) { @@ -1822,7 +1823,7 @@ TEST $file 'FILE', 'FILEEOF', 'FILE_EXTERNAL', 'REDIRECTTEST', 'CAPTURE_STDIO', 'STDIN', 'CGI', 'PHPDBG', 'INI', 'ENV', 'EXTENSIONS', - 'SKIPIF', 'XFAIL', 'CLEAN', + 'SKIPIF', 'XFAIL', 'XLEAK', 'CLEAN', 'CREDITS', 'DESCRIPTION', 'CONFLICTS', ))) { $bork_info = 'Unknown section "' . $section . '"'; @@ -2625,7 +2626,10 @@ COMMAND $cmd if (isset($section_text['XFAIL'])) { $warn = true; $info = " (warn: XFAIL section but test passes)"; - } else { + } else if (isset($section_text['XLEAK'])) { + $warn = true; + $info = " (warn: XLEAK section but test passes)"; + } else { show_result("PASS", $tested, $tested_file, '', $temp_filenames); junit_mark_test_as('PASS', $shortname, $tested); return 'PASSED'; @@ -2651,7 +2655,10 @@ COMMAND $cmd if (isset($section_text['XFAIL'])) { $warn = true; $info = " (warn: XFAIL section but test passes)"; - } else { + } if (isset($section_text['XLEAK'])) { + $warn = true; + $info = " (warn: XLEAK section but test passes)"; + } else { show_result("PASS", $tested, $tested_file, '', $temp_filenames); junit_mark_test_as('PASS', $shortname, $tested); return 'PASSED'; @@ -2674,7 +2681,8 @@ COMMAND $cmd } if ($leaked) { - $restype[] = 'LEAK'; + $restype[] = isset($section_text['XLEAK']) ? + 'XLEAK' : 'LEAK'; } if ($warn) { @@ -2685,7 +2693,10 @@ COMMAND $cmd if (isset($section_text['XFAIL'])) { $restype[] = 'XFAIL'; $info = ' XFAIL REASON: ' . rtrim($section_text['XFAIL']); - } else { + } else if ($section_text['XLEAK']) { + $restype[] = 'XLEAK'; + $info = ' XLEAK REASON: ' . rtrim($section_text['XLEAK']); + } else { $restype[] = 'FAIL'; } } @@ -2962,7 +2973,8 @@ function compute_summary() 'FAILED' => 0, 'BORKED' => 0, 'LEAKED' => 0, - 'XFAILED' => 0 + 'XFAILED' => 0, + 'XLEAKED' => 0 ); foreach ($test_results as $v) { @@ -2987,10 +2999,11 @@ function get_summary($show_ext_summary, $show_html) $x_warned = (100.0 * $sum_results['WARNED']) / $x_total; $x_failed = (100.0 * $sum_results['FAILED']) / $x_total; $x_xfailed = (100.0 * $sum_results['XFAILED']) / $x_total; + $x_xleaked = (100.0 * $sum_results['XLEAKED']) / $x_total; $x_leaked = (100.0 * $sum_results['LEAKED']) / $x_total; $x_passed = (100.0 * $sum_results['PASSED']) / $x_total; } else { - $x_warned = $x_failed = $x_passed = $x_leaked = $x_xfailed = 0; + $x_warned = $x_failed = $x_passed = $x_leaked = $x_xfailed = $x_xleaked = 0; } $summary = ''; @@ -3021,12 +3034,20 @@ Tests borked : ' . sprintf('%4d (%5.1f%%)', $sum_results['BORKED'], $percent_ $summary .= ' Tests skipped : ' . sprintf('%4d (%5.1f%%)', $sum_results['SKIPPED'], $percent_results['SKIPPED']) . ' -------- Tests warned : ' . sprintf('%4d (%5.1f%%)', $sum_results['WARNED'], $percent_results['WARNED']) . ' ' . sprintf('(%5.1f%%)', $x_warned) . ' -Tests failed : ' . sprintf('%4d (%5.1f%%)', $sum_results['FAILED'], $percent_results['FAILED']) . ' ' . sprintf('(%5.1f%%)', $x_failed) . ' +Tests failed : ' . sprintf('%4d (%5.1f%%)', $sum_results['FAILED'], $percent_results['FAILED']) . ' ' . sprintf('(%5.1f%%)', $x_failed); + + if ($sum_results['XFAILED']) { + $summary .= ' Expected fail : ' . sprintf('%4d (%5.1f%%)', $sum_results['XFAILED'], $percent_results['XFAILED']) . ' ' . sprintf('(%5.1f%%)', $x_xfailed); + } if ($valgrind) { $summary .= ' Tests leaked : ' . sprintf('%4d (%5.1f%%)', $sum_results['LEAKED'], $percent_results['LEAKED']) . ' ' . sprintf('(%5.1f%%)', $x_leaked); + if ($sum_results['XLEAKED']) { + $summary .= ' +Expected leak : ' . sprintf('%4d (%5.1f%%)', $sum_results['XLEAKED'], $percent_results['XLEAKED']) . ' ' . sprintf('(%5.1f%%)', $x_xleaked); + } } $summary .= ' @@ -3115,6 +3136,19 @@ LEAKED TEST SUMMARY $failed_test_summary .= "=====================================================================\n"; } + if (count($PHP_FAILED_TESTS['XLEAKED'])) { + $failed_test_summary .= ' +===================================================================== +EXPECTED LEAK TEST SUMMARY +--------------------------------------------------------------------- +'; + foreach ($PHP_FAILED_TESTS['XLEAKED'] as $failed_test_data) { + $failed_test_summary .= $failed_test_data['test_name'] . $failed_test_data['info'] . "\n"; + } + + $failed_test_summary .= "=====================================================================\n"; + } + if ($failed_test_summary && !getenv('NO_PHPTEST_SUMMARY')) { $summary .= $failed_test_summary; } @@ -3390,13 +3424,13 @@ function junit_mark_test_as($type, $file_name, $test_name, $time = null, $messag if (is_array($type)) { $output_type = $type[0] . 'ED'; - $temp = array_intersect(array('XFAIL', 'FAIL', 'WARN'), $type); + $temp = array_intersect(array('XFAIL', 'XLEAK', 'FAIL', 'WARN'), $type); $type = reset($temp); } else { $output_type = $type . 'ED'; } - if ('PASS' == $type || 'XFAIL' == $type) { + if ('PASS' == $type || 'XFAIL' == $type || 'XLEAK' == $type) { junit_suite_record($suite, 'test_pass'); } elseif ('BORK' == $type) { junit_suite_record($suite, 'test_error'); -- 2.40.0