From 0c696577eb50855c030d656064e3276842b864c7 Mon Sep 17 00:00:00 2001 From: Tyson Andre Date: Mon, 20 Jan 2020 16:53:37 -0500 Subject: [PATCH] Use smaller batch size in run-tests.php when appropriate - When valgrind is used, communication overhead is relatively small, so just use a batch size of 1. - If this is running a small enough number of tests, reduce the batch size to give batches to more workers. (Previously, if there were 90 tests and -j8, only 3 of 8 workers would get a batch of size 32 or less. After this change, the batch size is 12 or less) Closes GH-5098 --- run-tests.php | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/run-tests.php b/run-tests.php index d7dd69731c..9219e09185 100755 --- a/run-tests.php +++ b/run-tests.php @@ -1341,7 +1341,7 @@ function run_all_tests($test_files, $env, $redir_tested = null) /** The heart of parallel testing. */ function run_all_tests_parallel($test_files, $env, $redir_tested) { - global $workers, $test_idx, $test_cnt, $test_results, $failed_tests_file, $result_tests_file, $PHP_FAILED_TESTS, $shuffle, $SHOW_ONLY_GROUPS; + global $workers, $test_idx, $test_cnt, $test_results, $failed_tests_file, $result_tests_file, $PHP_FAILED_TESTS, $shuffle, $SHOW_ONLY_GROUPS, $valgrind; // The PHP binary running run-tests.php, and run-tests.php itself // This PHP executable is *not* necessarily the same as the tested version @@ -1415,6 +1415,7 @@ function run_all_tests_parallel($test_files, $env, $redir_tested) { } $sockPort = substr($sockName, $portPos + 1); $sockUri = "tcp://$sockHost:$sockPort"; + $totalFileCount = count($test_files); for ($i = 1; $i <= $workers; $i++) { $proc = proc_open( @@ -1548,8 +1549,14 @@ escape: $sequentialTests = []; } // Batch multiple tests to reduce communication overhead. + // - When valgrind is used, communication overhead is relatively small, + // so just use a batch size of 1. + // - If this is running a small enough number of tests, + // reduce the batch size to give batches to more workers. $files = []; - $batchSize = $shuffle ? 4 : 32; + $maxBatchSize = $valgrind ? 1 : ($shuffle ? 4 : 32); + $averageFilesPerWorker = max(1, (int)ceil($totalFileCount / count($workerProcs))); + $batchSize = min($maxBatchSize, $averageFilesPerWorker); while (count($files) <= $batchSize && $file = array_pop($test_files)) { foreach ($fileConflictsWith[$file] as $conflictKey) { if (isset($activeConflicts[$conflictKey])) { -- 2.40.0