Use smaller batch size in run-tests.php when appropriate

- When valgrind is used, communication overhead is relatively small,
  so just use a batch size of 1.
- If this is running a small enough number of tests,
  reduce the batch size to give batches to more workers.
  (Previously, if there were 90 tests and -j8,
  only 3 of 8 workers would get a batch of size 32 or less.
  After this change, the batch size is 12 or less)

Closes GH-5098

diff --git a/run-tests.php b/run-tests.php
index d7dd69731c55bde4829f612fa0d349811142b1ea..9219e09185e5c1709177fef333a48467ac32a45c 100755 (executable)
--- a/run-tests.php
+++ b/run-tests.php
@@ -1341,7 +1341,7 @@ function run_all_tests($test_files, $env, $redir_tested = null)
 
 /** The heart of parallel testing. */
 function run_all_tests_parallel($test_files, $env, $redir_tested) {
-       global $workers, $test_idx, $test_cnt, $test_results, $failed_tests_file, $result_tests_file, $PHP_FAILED_TESTS, $shuffle, $SHOW_ONLY_GROUPS;
+       global $workers, $test_idx, $test_cnt, $test_results, $failed_tests_file, $result_tests_file, $PHP_FAILED_TESTS, $shuffle, $SHOW_ONLY_GROUPS, $valgrind;
 
        // The PHP binary running run-tests.php, and run-tests.php itself
        // This PHP executable is *not* necessarily the same as the tested version
@@ -1415,6 +1415,7 @@ function run_all_tests_parallel($test_files, $env, $redir_tested) {
        }
        $sockPort = substr($sockName, $portPos + 1);
        $sockUri = "tcp://$sockHost:$sockPort";
+       $totalFileCount = count($test_files);
 
        for ($i = 1; $i <= $workers; $i++) {
                $proc = proc_open(
@@ -1548,8 +1549,14 @@ escape:
                                                                $sequentialTests = [];
                                                        }
                                                        // Batch multiple tests to reduce communication overhead.
+                                                       // - When valgrind is used, communication overhead is relatively small,
+                                                       //   so just use a batch size of 1.
+                                                       // - If this is running a small enough number of tests,
+                                                       //   reduce the batch size to give batches to more workers.
                                                        $files = [];
-                                                       $batchSize = $shuffle ? 4 : 32;
+                                                       $maxBatchSize = $valgrind ? 1 : ($shuffle ? 4 : 32);
+                                                       $averageFilesPerWorker = max(1, (int)ceil($totalFileCount / count($workerProcs)));
+                                                       $batchSize = min($maxBatchSize, $averageFilesPerWorker);
                                                        while (count($files) <= $batchSize && $file = array_pop($test_files)) {
                                                                foreach ($fileConflictsWith[$file] as $conflictKey) {
                                                                        if (isset($activeConflicts[$conflictKey])) {