From f1c2b79f1dd7ffb57c8f13f1aad0b951d60a0745 Mon Sep 17 00:00:00 2001 From: NickSdot Date: Tue, 28 Jul 2026 20:29:29 +0700 Subject: [PATCH 1/8] Run tests in parallel by default --- README.md | 8 +- docs/source/miscellaneous/writing-tests.rst | 5 + ext/gd/tests/createfromwbmp2.phpt | 2 +- ext/gd/tests/createfromwbmp2_extern.phpt | 4 +- ext/zip/tests/oo_addglob_leak.phpt | 4 +- run-tests.php | 68 ++++++++++++- sapi/cli/tests/010-2.phpt | 2 +- sapi/cli/tests/010.phpt | 4 +- tests/run-test/automatic_worker_limit.phpt | 103 ++++++++++++++++++++ 9 files changed, 185 insertions(+), 15 deletions(-) create mode 100644 tests/run-test/automatic_worker_limit.phpt diff --git a/README.md b/README.md index d83203d74c0b..80618f2d6f94 100644 --- a/README.md +++ b/README.md @@ -97,15 +97,15 @@ can be determined using `nproc`. PHP ships with an extensive test suite, the command `make test` is used after successful compilation of the sources to run this test suite. -It is possible to run tests using multiple cores by setting `-jN` in -`TEST_PHP_ARGS` or `TESTS`: +Tests run in parallel by default, using up to 10 detected logical processors. +Set `-jN` in `TEST_PHP_ARGS` or `TESTS` to override the worker count: ```shell make TEST_PHP_ARGS=-j4 test ``` -Shall run `make test` with a maximum of 4 concurrent jobs: Generally the maximum -number of jobs should not exceed the number of cores available. +This runs `make test` with a maximum of 4 concurrent jobs. Alternatively, +use `-j1` to run tests sequentially. Use the `TEST_PHP_ARGS` or `TESTS` variable to test only specific directories: diff --git a/docs/source/miscellaneous/writing-tests.rst b/docs/source/miscellaneous/writing-tests.rst index 8e17674ae481..4339e042e6a4 100644 --- a/docs/source/miscellaneous/writing-tests.rst +++ b/docs/source/miscellaneous/writing-tests.rst @@ -192,6 +192,11 @@ When you are testing your test case it's really important to make sure that you temporary resources (eg files) that you used in the test. There is a special ``--CLEAN--`` section to help you do this — see `here <#clean>`_. +Tests run in parallel by default. Mutable resources such as files, directories, ports, database +objects, and IPC identifiers must therefore be unique to each test. Read-only fixtures may be +shared. If a resource cannot be isolated, declare the narrowest applicable conflict using +``--CONFLICTS--`` or a ``CONFLICTS`` file. + Another good check is to look at what lines of code in the PHP source your test case covers. This is easy to do, there are some instructions on the `PHP Wiki `_. diff --git a/ext/gd/tests/createfromwbmp2.phpt b/ext/gd/tests/createfromwbmp2.phpt index 4608c861323f..7007ff545d74 100644 --- a/ext/gd/tests/createfromwbmp2.phpt +++ b/ext/gd/tests/createfromwbmp2.phpt @@ -8,7 +8,7 @@ gd ?> --FILE-- "); diff --git a/ext/gd/tests/createfromwbmp2_extern.phpt b/ext/gd/tests/createfromwbmp2_extern.phpt index 68895f9a3570..711f2e8ca3ed 100644 --- a/ext/gd/tests/createfromwbmp2_extern.phpt +++ b/ext/gd/tests/createfromwbmp2_extern.phpt @@ -4,7 +4,7 @@ imagecreatefromwbmp with invalid wbmp gd --FILE-- "); @@ -41,4 +41,4 @@ unlink($filename); --EXPECTF-- Warning: imagecreatefromwbmp(): %croduct of memory allocation multiplication would exceed INT_MAX, failing operation gracefully%win %s on line %d -Warning: imagecreatefromwbmp(): "%s_tmp.wbmp" is not a valid WBMP file in %s on line %d +Warning: imagecreatefromwbmp(): "%s_tmp_createfromwbmp2_extern.wbmp" is not a valid WBMP file in %s on line %d diff --git a/ext/zip/tests/oo_addglob_leak.phpt b/ext/zip/tests/oo_addglob_leak.phpt index 9040c5565f84..be7f92dccb90 100644 --- a/ext/zip/tests/oo_addglob_leak.phpt +++ b/ext/zip/tests/oo_addglob_leak.phpt @@ -12,7 +12,7 @@ if(!defined("GLOB_BRACE")) die ('skip requires GLOB_BRACE'); $dirname = __DIR__ . '/'; include $dirname . 'utils.inc'; -$dirname = __DIR__ . '/__tmp_oo_addglob2/'; +$dirname = __DIR__ . '/__tmp_oo_addglob_leak/'; $file = $dirname . 'test.zip'; @mkdir($dirname); @@ -38,7 +38,7 @@ var_dump($zip->addGlob($dirname . 'bar.*', GLOB_BRACE, $options)); --EXPECTF-- array(1) { diff --git a/run-tests.php b/run-tests.php index 998e7e24c337..83a107e1325e 100755 --- a/run-tests.php +++ b/run-tests.php @@ -34,9 +34,9 @@ function show_usage(): void php run-tests.php [options] [files] [directories] Options: - -j Run up to simultaneous testing processes in parallel for - quicker testing on systems with multiple logical processors. - Note that this is experimental feature. + -j Run up to simultaneous testing processes. By default, + the worker count is detected automatically. Use -j1 to run + tests sequentially. -l Read the testfiles to be executed from . After the test has finished all failed tests are written to the same . @@ -351,6 +351,7 @@ function main(): void $shuffle = false; $bless = false; $workers = null; + $workersExplicit = false; $context_line_count = 3; $num_repeats = 1; $show_progress = true; @@ -412,6 +413,7 @@ function main(): void switch ($switch) { case 'j': + $workersExplicit = true; $workers = substr($argv[$i], 2); if ($workers == 0 || !preg_match('/^\d+$/', $workers)) { error("'$workers' is not a valid number of workers, try e.g. -j16 for 16 workers"); @@ -641,6 +643,19 @@ function main(): void } } + if (!$workersExplicit && (!$selected_tests || count($test_files) > 1)) { + $workers = get_default_worker_count(); + if ( + $workers !== null + && ($valgrind !== null || isset($environment['SKIP_ASAN'])) + ) { + $workers = min($workers, 2); + } + if ($workers !== null && !can_create_parallel_worker_socket()) { + $workers = null; + } + } + if ($online === null && !isset($environment['SKIP_ONLINE_TESTS'])) { $online = false; } @@ -800,6 +815,53 @@ function main(): void } } +function get_default_worker_count(): ?int +{ + if (IS_WINDOWS) { + $workerCount = getenv('NUMBER_OF_PROCESSORS'); + return is_string($workerCount) ? parse_default_worker_count($workerCount) : null; + } + + $commands = [ + 'nproc 2>/dev/null', + 'getconf _NPROCESSORS_ONLN 2>/dev/null', + 'getconf NPROCESSORS_ONLN 2>/dev/null', + 'sysctl -n hw.logicalcpu 2>/dev/null', + 'sysctl -n hw.ncpu 2>/dev/null', + ]; + foreach ($commands as $command) { + $workerCount = shell_exec($command); + if ( + is_string($workerCount) + && ($workerCount = parse_default_worker_count($workerCount)) !== null + ) { + return $workerCount; + } + } + + return null; +} + +function parse_default_worker_count(string $workerCount): ?int +{ + $workerCount = filter_var(trim($workerCount), FILTER_VALIDATE_INT, [ + 'options' => ['min_range' => 2], + ]); + + return $workerCount !== false ? min($workerCount, 10) : null; +} + +function can_create_parallel_worker_socket(): bool +{ + $socket = @stream_socket_server('tcp://127.0.0.1:0'); + if ($socket === false) { + return false; + } + + fclose($socket); + return true; +} + function verify_config(string $php): void { if (empty($php) || !file_exists($php)) { diff --git a/sapi/cli/tests/010-2.phpt b/sapi/cli/tests/010-2.phpt index 88fe1c832a11..ddf7315c8298 100644 --- a/sapi/cli/tests/010-2.phpt +++ b/sapi/cli/tests/010-2.phpt @@ -12,7 +12,7 @@ if (substr(PHP_OS, 0, 3) == 'WIN') { $php = getenv('TEST_PHP_EXECUTABLE_ESCAPED'); -$filename_txt = __DIR__."/010.test.txt"; +$filename_txt = __DIR__."/010-R.test.txt"; $filename_txt_escaped = escapeshellarg($filename_txt); $txt = ' diff --git a/sapi/cli/tests/010.phpt b/sapi/cli/tests/010.phpt index 356b69bebf91..80758f368ab9 100644 --- a/sapi/cli/tests/010.phpt +++ b/sapi/cli/tests/010.phpt @@ -14,7 +14,7 @@ $php = getenv('TEST_PHP_EXECUTABLE_ESCAPED'); $filename = __DIR__."/010.test.php"; $filename_escaped = escapeshellarg($filename); -$filename_txt = __DIR__."/010.test.txt"; +$filename_txt = __DIR__."/010-F.test.txt"; $filename_txt_escaped = escapeshellarg($filename_txt); $code = ' @@ -37,7 +37,7 @@ var_dump(shell_exec("cat $filename_txt_escaped | $php -n -F $filename_escaped")) --CLEAN-- --EXPECT-- string(25) " diff --git a/tests/run-test/automatic_worker_limit.phpt b/tests/run-test/automatic_worker_limit.phpt new file mode 100644 index 000000000000..398e1470c72a --- /dev/null +++ b/tests/run-test/automatic_worker_limit.phpt @@ -0,0 +1,103 @@ +--TEST-- +Automatic worker detection is capped for regular and instrumented runs +--SKIPIF-- + +--ENV-- +TEST_PHP_FORK_SERVER=0 +--FILE-- + + --EXPECT-- + ok + PHPT); +} + +$environment = [ + 'PATH' => $bin . PATH_SEPARATOR . getenv('PATH'), + 'TEST_PHP_EXECUTABLE' => getenv('TEST_PHP_EXECUTABLE'), + 'TEST_PHP_FORK_SERVER' => '0', +]; +foreach (['TEMP', 'TMPDIR'] as $name) { + if (($value = getenv($name)) !== false) { + $environment[$name] = $value; + } +} + +$runTests = static function (array $arguments) use ($environment, $testFiles): array { + $process = proc_open( + [ + getenv('TEST_PHP_EXECUTABLE'), + dirname(__DIR__, 2) . '/run-tests.php', + '-q', + '--no-progress', + ...$arguments, + ...$testFiles, + ], + [ + 0 => ['pipe', 'r'], + 1 => ['pipe', 'w'], + 2 => ['redirect', 1], + ], + $pipes, + null, + $environment, + ); + fclose($pipes[0]); + $output = stream_get_contents($pipes[1]); + fclose($pipes[1]); + + return [proc_close($process), $output]; +}; + +[$exitCode, $output] = $runTests([]); +var_dump($exitCode); +var_dump(str_contains($output, 'Spawning 10 workers...')); +var_dump(str_contains($output, 'Spawning 11 workers...')); + +[$exitCode, $output] = $runTests(['--asan']); +var_dump($exitCode); +var_dump(str_contains($output, 'Spawning 2 workers...')); +var_dump(str_contains($output, 'Spawning 10 workers...')); + +[$exitCode, $output] = $runTests(['--asan', '-j3']); +var_dump($exitCode); +var_dump(str_contains($output, 'Spawning 3 workers...')); + +foreach ($testFiles as $file) { + unlink($file); +} +unlink($nproc); +rmdir($tests); +rmdir($bin); +rmdir($root); +?> +--EXPECT-- +int(0) +bool(true) +bool(false) +int(0) +bool(true) +bool(false) +int(0) +bool(true) From 9fb707788cbb8eeebf8af718b88a68680ce8d944 Mon Sep 17 00:00:00 2001 From: NickSdot Date: Tue, 28 Jul 2026 20:29:30 +0700 Subject: [PATCH 2/8] Reduce parallel test batch size --- run-tests.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run-tests.php b/run-tests.php index 83a107e1325e..543cdbf98873 100755 --- a/run-tests.php +++ b/run-tests.php @@ -1633,10 +1633,10 @@ function run_all_tests_parallel(array $test_files, array $env, ?string $redir_te // - If this is running a small enough number of tests, // reduce the batch size to give batches to more workers. $files = []; - $maxBatchSize = $valgrind ? 1 : ($shuffle ? 4 : 32); + $maxBatchSize = $valgrind ? 1 : 4; $averageFilesPerWorker = max(1, (int) ceil($totalFileCount / count($workerProcs))); $batchSize = min($maxBatchSize, $averageFilesPerWorker); - while (count($files) <= $batchSize && $file = array_pop($test_files)) { + while (count($files) < $batchSize && $file = array_pop($test_files)) { foreach ($fileConflictsWith[$file] as $conflictKey) { if (isset($activeConflicts[$conflictKey])) { $waitingTests[$conflictKey][] = $file; From e1a6312cf6530786e122b449e7661ad8e8f7d0ed Mon Sep 17 00:00:00 2001 From: NickSdot Date: Thu, 30 Jul 2026 22:49:11 +0700 Subject: [PATCH 3/8] review: replaced optional Filter dependency with a regex --- run-tests.php | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/run-tests.php b/run-tests.php index 543cdbf98873..568f8f359b9e 100755 --- a/run-tests.php +++ b/run-tests.php @@ -844,11 +844,13 @@ function get_default_worker_count(): ?int function parse_default_worker_count(string $workerCount): ?int { - $workerCount = filter_var(trim($workerCount), FILTER_VALIDATE_INT, [ - 'options' => ['min_range' => 2], - ]); + $workerCount = trim($workerCount); + if (preg_match('/^[0-9]+$/D', $workerCount) !== 1) { + return null; + } - return $workerCount !== false ? min($workerCount, 10) : null; + $workerCount = (int) $workerCount; + return $workerCount >= 2 ? min($workerCount, 10) : null; } function can_create_parallel_worker_socket(): bool From 472acd1d6540214e701bc5a29fe9bc93970576c7 Mon Sep 17 00:00:00 2001 From: NickSdot Date: Thu, 30 Jul 2026 22:49:11 +0700 Subject: [PATCH 4/8] review: switched automatic worker cleanup to --CLEAN-- --- tests/run-test/automatic_worker_limit.phpt | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/run-test/automatic_worker_limit.phpt b/tests/run-test/automatic_worker_limit.phpt index 398e1470c72a..5bf6df07365e 100644 --- a/tests/run-test/automatic_worker_limit.phpt +++ b/tests/run-test/automatic_worker_limit.phpt @@ -83,14 +83,18 @@ var_dump(str_contains($output, 'Spawning 10 workers...')); [$exitCode, $output] = $runTests(['--asan', '-j3']); var_dump($exitCode); var_dump(str_contains($output, 'Spawning 3 workers...')); - -foreach ($testFiles as $file) { - unlink($file); +?> +--CLEAN-- + --EXPECT-- int(0) From 73fa923ab8615654a06789134c3a713f73e9d4f5 Mon Sep 17 00:00:00 2001 From: NickSdot Date: Fri, 31 Jul 2026 08:04:47 +0700 Subject: [PATCH 5/8] run-tests: used process-specific information files --- .gitignore | 2 +- run-tests.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index b76b5a787caa..f1ed9c288c95 100644 --- a/.gitignore +++ b/.gitignore @@ -253,7 +253,7 @@ php php_test_results_*.txt # Temporary test information generated by `./run-tests.php` -/run-test-info.php +/run-test-info-*.php # Temporary POST data placeholder files generated by `./run-tests.php` phpt.* diff --git a/run-tests.php b/run-tests.php index 568f8f359b9e..f700561b1383 100755 --- a/run-tests.php +++ b/run-tests.php @@ -884,7 +884,7 @@ function write_information(array $user_tests, $phpdbg): void $php_escaped = escapeshellarg($php); // Get info from php - $info_file = __DIR__ . '/run-test-info.php'; + $info_file = __DIR__ . '/run-test-info-' . getmypid() . '.php'; @unlink($info_file); $php_info = ' Date: Fri, 31 Jul 2026 17:15:21 +0700 Subject: [PATCH 6/8] review: moved `run-test-info-*.php` to `sys_get_temp_dir()`` --- .gitignore | 4 ++-- run-tests.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index f1ed9c288c95..769ac62cc5d8 100644 --- a/.gitignore +++ b/.gitignore @@ -252,8 +252,8 @@ php # Test results generated by `./run-tests.php` php_test_results_*.txt -# Temporary test information generated by `./run-tests.php` -/run-test-info-*.php +# Temporary test information generated by `./run-tests.php` (kept for BC; now lives in tmp dir) +/run-test-info.php # Temporary POST data placeholder files generated by `./run-tests.php` phpt.* diff --git a/run-tests.php b/run-tests.php index f700561b1383..9db0a36363ca 100755 --- a/run-tests.php +++ b/run-tests.php @@ -884,7 +884,7 @@ function write_information(array $user_tests, $phpdbg): void $php_escaped = escapeshellarg($php); // Get info from php - $info_file = __DIR__ . '/run-test-info-' . getmypid() . '.php'; + $info_file = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'run-test-info-' . getmypid() . '.php'; @unlink($info_file); $php_info = ' Date: Fri, 31 Jul 2026 17:20:06 +0700 Subject: [PATCH 7/8] review: style nit --- run-tests.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/run-tests.php b/run-tests.php index 9db0a36363ca..0e2105eaa8ad 100755 --- a/run-tests.php +++ b/run-tests.php @@ -645,10 +645,8 @@ function main(): void if (!$workersExplicit && (!$selected_tests || count($test_files) > 1)) { $workers = get_default_worker_count(); - if ( - $workers !== null - && ($valgrind !== null || isset($environment['SKIP_ASAN'])) - ) { + if ($workers !== null + && ($valgrind !== null || isset($environment['SKIP_ASAN']))) { $workers = min($workers, 2); } if ($workers !== null && !can_create_parallel_worker_socket()) { From 9e8de6b2830977686e073603beb2282ebe127a04 Mon Sep 17 00:00:00 2001 From: NickSdot Date: Fri, 31 Jul 2026 17:21:35 +0700 Subject: [PATCH 8/8] review: style nit --- run-tests.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/run-tests.php b/run-tests.php index 0e2105eaa8ad..981f78a1b56e 100755 --- a/run-tests.php +++ b/run-tests.php @@ -829,10 +829,8 @@ function get_default_worker_count(): ?int ]; foreach ($commands as $command) { $workerCount = shell_exec($command); - if ( - is_string($workerCount) - && ($workerCount = parse_default_worker_count($workerCount)) !== null - ) { + if (is_string($workerCount) + && ($workerCount = parse_default_worker_count($workerCount)) !== null) { return $workerCount; } }