From: Nikita Popov Date: Wed, 21 Oct 2020 13:01:47 +0000 (+0200) Subject: Add --repeat testing mode X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=1b3b430f478c785c88655533518a66b88ece060a;p=php Add --repeat testing mode This testing mode executes the test multiple times in the same process (but in different requests). It is primarily intended to catch tracing JIT bugs, but also catches state leaks across requests. Closes GH-6365. --- diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dabf35b3d2..1932b1cdee 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -103,3 +103,8 @@ jobs: configurationName: DEBUG_NTS_FILE_CACHE configurationParameters: '--enable-debug --disable-zts' timeoutInMinutes: 90 + - template: azure/job.yml + parameters: + configurationName: DEBUG_NTS_REPEAT + configurationParameters: '--enable-debug --disable-zts' + runTestsParameters: '--repeat 2' diff --git a/ext/opcache/tests/bug65915.phpt b/ext/opcache/tests/bug65915.phpt index c616c4fb5f..b62dbd8ea7 100644 --- a/ext/opcache/tests/bug65915.phpt +++ b/ext/opcache/tests/bug65915.phpt @@ -5,7 +5,11 @@ opcache.enable=1 opcache.enable_cli=1 opcache.file_cache_only=0 --SKIPIF-- - + --FILE-- + --FILE-- --FILE-- --FILE-- --FILE-- --FILE-- --FILE-- --FILE-- --FILE-- + --FILE-- --FILE-- --FILE-- --FILE-- 1 && isset($section_text['FILE_EXTERNAL'])) { + return skip_test($tested, $tested_file, $shortname, 'Test with FILE_EXTERNAL might not be repeatable'); + } + foreach (['FILE', 'EXPECT', 'EXPECTF', 'EXPECTREGEX'] as $prefix) { $key = $prefix . '_EXTERNAL'; @@ -1951,9 +1972,6 @@ TEST $file } fclose($fp); - $shortname = str_replace(TEST_PHP_SRCDIR . '/', '', $file); - $tested_file = $shortname; - if ($bork_info !== null) { show_result("BORK", $bork_info, $tested_file); $PHP_FAILED_TESTS['BORKED'][] = [ @@ -1983,8 +2001,6 @@ TEST $file $cmdRedirect = ''; } - $tested = trim($section_text['TEST']); - /* For GET/POST/PUT tests, check if cgi sapi is available and if it is, use it. */ if (array_key_exists('CGI', $section_text) || !empty($section_text['GET']) || !empty($section_text['POST']) || !empty($section_text['GZIP_POST']) || !empty($section_text['DEFLATE_POST']) || !empty($section_text['POST_RAW']) || !empty($section_text['PUT']) || !empty($section_text['COOKIE']) || !empty($section_text['EXPECTHEADERS'])) { if (isset($php_cgi)) { @@ -1999,13 +2015,12 @@ TEST $file } elseif (file_exists(dirname($php) . "/php-cgi")) { $php = realpath(dirname($php) . "/php-cgi") . ' -C '; } else { - show_result('SKIP', $tested, $tested_file, "reason: CGI not available"); - - junit_init_suite(junit_get_suitename_for($shortname)); - junit_mark_test_as('SKIP', $shortname, $tested, 0, 'CGI not available'); - return 'SKIPPED'; + return skip_test($tested, $tested_file, $shortname, 'CGI not available'); } } + if ($num_repeats > 1) { + return skip_test($tested, $tested_file, $shortname, 'CGI does not support --repeat'); + } $uses_cgi = true; } @@ -2023,11 +2038,22 @@ TEST $file // be run straight away. For example, EXTENSIONS, SKIPIF, CLEAN. $extra_options = '-rr'; } else { - show_result('SKIP', $tested, $tested_file, "reason: phpdbg not available"); + return skip_test($tested, $tested_file, $shortname, 'phpdbg not available'); + } + if ($num_repeats > 1) { + return skip_test($tested, $tested_file, $shortname, 'phpdbg does not support --repeat'); + } + } - junit_init_suite(junit_get_suitename_for($shortname)); - junit_mark_test_as('SKIP', $shortname, $tested, 0, 'phpdbg not available'); - return 'SKIPPED'; + if ($num_repeats > 1) { + if (array_key_exists('CLEAN', $section_text)) { + return skip_test($tested, $tested_file, $shortname, 'Test with CLEAN might not be repeatable'); + } + if (array_key_exists('STDIN', $section_text)) { + return skip_test($tested, $tested_file, $shortname, 'Test with STDIN might not be repeatable'); + } + if (array_key_exists('CAPTURE_STDIO', $section_text)) { + return skip_test($tested, $tested_file, $shortname, 'Test with CAPTURE_STDIO might not be repeatable'); } } @@ -2173,6 +2199,9 @@ TEST $file // even though all the files are re-created. $ini_settings['opcache.validate_timestamps'] = '0'; } + } else if ($num_repeats > 1) { + // Make sure warnings still show up on the second run. + $ini_settings['opcache.record_warnings'] = '1'; } // Any special ini settings @@ -2183,6 +2212,10 @@ TEST $file $replacement = IS_WINDOWS ? '"' . PHP_BINARY . ' -r \"while ($in = fgets(STDIN)) echo $in;\" > $1"' : 'tee $1 >/dev/null'; $section_text['INI'] = preg_replace('/{MAIL:(\S+)}/', $replacement, $section_text['INI']); settings2array(preg_split("/[\n\r]+/", $section_text['INI']), $ini_settings); + + if ($num_repeats > 1 && isset($ini_settings['opcache.opt_debug_level'])) { + return skip_test($tested, $tested_file, $shortname, 'opt_debug_level tests are not repeatable'); + } } $ini_settings = settings2params($ini_settings); @@ -2484,7 +2517,8 @@ TEST $file $env['CONTENT_TYPE'] = ''; $env['CONTENT_LENGTH'] = ''; - $cmd = "$php $pass_options $ini_settings -f \"$test_file\" $args$cmdRedirect"; + $repeat_option = $num_repeats > 1 ? "--repeat $num_repeats" : ""; + $cmd = "$php $pass_options $repeat_option $ini_settings -f \"$test_file\" $args$cmdRedirect"; } if ($valgrind) { @@ -2557,6 +2591,25 @@ COMMAND $cmd } } + if ($num_repeats > 1) { + // In repeat mode, retain the output before the first execution, + // and of the last execution. Do this early, because the trimming below + // makes the newline handling complicated. + $separator1 = "Executing for the first time...\n"; + $separator1_pos = strpos($out, $separator1); + if ($separator1_pos !== false) { + $separator2 = "Finished execution, repeating...\n"; + $separator2_pos = strrpos($out, $separator2); + if ($separator2_pos !== false) { + $out = substr($out, 0, $separator1_pos) + . substr($out, $separator2_pos + strlen($separator2)); + } else { + $out = substr($out, 0, $separator1_pos) + . substr($out, $separator1_pos + strlen($separator1)); + } + } + } + // Does the output match what is expected? $output = preg_replace("/\r\n/", "\n", trim($out)); diff --git a/sapi/cli/php_cli.c b/sapi/cli/php_cli.c index d28f5a5378..5092fb0ffd 100644 --- a/sapi/cli/php_cli.c +++ b/sapi/cli/php_cli.c @@ -172,6 +172,9 @@ const opt_struct OPTIONS[] = { {14, 1, "ri"}, {14, 1, "rextinfo"}, {15, 0, "ini"}, + /* Internal testing option -- may be changed or removed without notice, + * including in patch releases. */ + {16, 1, "repeat"}, {'-', 0, NULL} /* end of args */ }; @@ -529,7 +532,7 @@ static void php_cli_usage(char *argv0) static php_stream *s_in_process = NULL; -static void cli_register_file_handles(void) /* {{{ */ +static void cli_register_file_handles(zend_bool no_close) /* {{{ */ { php_stream *s_in, *s_out, *s_err; php_stream_context *sc_in=NULL, *sc_out=NULL, *sc_err=NULL; @@ -546,11 +549,11 @@ static void cli_register_file_handles(void) /* {{{ */ return; } -#if PHP_DEBUG - /* do not close stdout and stderr */ - s_out->flags |= PHP_STREAM_FLAG_NO_CLOSE; - s_err->flags |= PHP_STREAM_FLAG_NO_CLOSE; -#endif + if (no_close) { + s_in->flags |= PHP_STREAM_FLAG_NO_CLOSE; + s_out->flags |= PHP_STREAM_FLAG_NO_CLOSE; + s_err->flags |= PHP_STREAM_FLAG_NO_CLOSE; + } s_in_process = s_in; @@ -614,6 +617,8 @@ static int do_cli(int argc, char **argv) /* {{{ */ int interactive=0; const char *param_error=NULL; int hide_argv = 0; + int num_repeats = 1; + pid_t pid = getpid(); zend_try { @@ -839,6 +844,9 @@ static int do_cli(int argc, char **argv) /* {{{ */ case 15: behavior = PHP_MODE_SHOW_INI_CONFIG; break; + case 16: + num_repeats = atoi(php_optarg); + break; default: break; } @@ -873,6 +881,12 @@ static int do_cli(int argc, char **argv) /* {{{ */ fflush(stdout); } + if (num_repeats > 1) { + fprintf(stdout, "Executing for the first time...\n"); + fflush(stdout); + } + +do_repeat: /* only set script_file if not set already and not in direct mode and not at end of parameter list */ if (argc > php_optind && !script_file @@ -940,7 +954,7 @@ static int do_cli(int argc, char **argv) /* {{{ */ switch (behavior) { case PHP_MODE_STANDARD: if (strcmp(file_handle.filename, "Standard input code")) { - cli_register_file_handles(); + cli_register_file_handles(/* no_close */ PHP_DEBUG || num_repeats > 1); } if (interactive && cli_shell_callbacks.cli_shell_run) { @@ -975,7 +989,7 @@ static int do_cli(int argc, char **argv) /* {{{ */ } break; case PHP_MODE_CLI_DIRECT: - cli_register_file_handles(); + cli_register_file_handles(/* no_close */ PHP_DEBUG || num_repeats > 1); zend_eval_string_ex(exec_direct, NULL, "Command line code", 1); break; @@ -985,7 +999,7 @@ static int do_cli(int argc, char **argv) /* {{{ */ size_t len, index = 0; zval argn, argi; - cli_register_file_handles(); + cli_register_file_handles(/* no_close */ PHP_DEBUG || num_repeats > 1); if (exec_begin) { zend_eval_string_ex(exec_begin, NULL, "Command line begin code", 1); @@ -1112,6 +1126,12 @@ out: if (translated_path) { free(translated_path); } + /* Don't repeat fork()ed processes. */ + if (--num_repeats && pid == getpid()) { + fprintf(stdout, "Finished execution, repeating...\n"); + fflush(stdout); + goto do_repeat; + } return EG(exit_status); err: sapi_deactivate();