diff --git a/.gitignore b/.gitignore index b76b5a787caa..769ac62cc5d8 100644 --- a/.gitignore +++ b/.gitignore @@ -252,7 +252,7 @@ php # Test results generated by `./run-tests.php` php_test_results_*.txt -# Temporary test information generated by `./run-tests.php` +# Temporary test information generated by `./run-tests.php` (kept for BC; now lives in tmp dir) /run-test-info.php # Temporary POST data placeholder files generated by `./run-tests.php` diff --git a/README.md b/README.md index d83203d74c0b..80618f2d6f94 100644 --- a/README.md +++ b/README.md @@ -97,15 +97,15 @@ can be determined using `nproc`. PHP ships with an extensive test suite, the command `make test` is used after successful compilation of the sources to run this test suite. -It is possible to run tests using multiple cores by setting `-jN` in -`TEST_PHP_ARGS` or `TESTS`: +Tests run in parallel by default, using up to 10 detected logical processors. +Set `-jN` in `TEST_PHP_ARGS` or `TESTS` to override the worker count: ```shell make TEST_PHP_ARGS=-j4 test ``` -Shall run `make test` with a maximum of 4 concurrent jobs: Generally the maximum -number of jobs should not exceed the number of cores available. +This runs `make test` with a maximum of 4 concurrent jobs. Alternatively, +use `-j1` to run tests sequentially. Use the `TEST_PHP_ARGS` or `TESTS` variable to test only specific directories: diff --git a/docs/source/miscellaneous/writing-tests.rst b/docs/source/miscellaneous/writing-tests.rst index 8e17674ae481..4339e042e6a4 100644 --- a/docs/source/miscellaneous/writing-tests.rst +++ b/docs/source/miscellaneous/writing-tests.rst @@ -192,6 +192,11 @@ When you are testing your test case it's really important to make sure that you temporary resources (eg files) that you used in the test. There is a special ``--CLEAN--`` section to help you do this — see `here <#clean>`_. +Tests run in parallel by default. Mutable resources such as files, directories, ports, database +objects, and IPC identifiers must therefore be unique to each test. Read-only fixtures may be +shared. If a resource cannot be isolated, declare the narrowest applicable conflict using +``--CONFLICTS--`` or a ``CONFLICTS`` file. + Another good check is to look at what lines of code in the PHP source your test case covers. This is easy to do, there are some instructions on the `PHP Wiki `_. diff --git a/ext/gd/tests/createfromwbmp2.phpt b/ext/gd/tests/createfromwbmp2.phpt index 4608c861323f..7007ff545d74 100644 --- a/ext/gd/tests/createfromwbmp2.phpt +++ b/ext/gd/tests/createfromwbmp2.phpt @@ -8,7 +8,7 @@ gd ?> --FILE-- "); diff --git a/ext/gd/tests/createfromwbmp2_extern.phpt b/ext/gd/tests/createfromwbmp2_extern.phpt index 68895f9a3570..711f2e8ca3ed 100644 --- a/ext/gd/tests/createfromwbmp2_extern.phpt +++ b/ext/gd/tests/createfromwbmp2_extern.phpt @@ -4,7 +4,7 @@ imagecreatefromwbmp with invalid wbmp gd --FILE-- "); @@ -41,4 +41,4 @@ unlink($filename); --EXPECTF-- Warning: imagecreatefromwbmp(): %croduct of memory allocation multiplication would exceed INT_MAX, failing operation gracefully%win %s on line %d -Warning: imagecreatefromwbmp(): "%s_tmp.wbmp" is not a valid WBMP file in %s on line %d +Warning: imagecreatefromwbmp(): "%s_tmp_createfromwbmp2_extern.wbmp" is not a valid WBMP file in %s on line %d diff --git a/ext/zip/tests/oo_addglob_leak.phpt b/ext/zip/tests/oo_addglob_leak.phpt index 9040c5565f84..be7f92dccb90 100644 --- a/ext/zip/tests/oo_addglob_leak.phpt +++ b/ext/zip/tests/oo_addglob_leak.phpt @@ -12,7 +12,7 @@ if(!defined("GLOB_BRACE")) die ('skip requires GLOB_BRACE'); $dirname = __DIR__ . '/'; include $dirname . 'utils.inc'; -$dirname = __DIR__ . '/__tmp_oo_addglob2/'; +$dirname = __DIR__ . '/__tmp_oo_addglob_leak/'; $file = $dirname . 'test.zip'; @mkdir($dirname); @@ -38,7 +38,7 @@ var_dump($zip->addGlob($dirname . 'bar.*', GLOB_BRACE, $options)); --EXPECTF-- array(1) { diff --git a/run-tests.php b/run-tests.php index 998e7e24c337..981f78a1b56e 100755 --- a/run-tests.php +++ b/run-tests.php @@ -34,9 +34,9 @@ function show_usage(): void php run-tests.php [options] [files] [directories] Options: - -j Run up to simultaneous testing processes in parallel for - quicker testing on systems with multiple logical processors. - Note that this is experimental feature. + -j Run up to simultaneous testing processes. By default, + the worker count is detected automatically. Use -j1 to run + tests sequentially. -l Read the testfiles to be executed from . After the test has finished all failed tests are written to the same . @@ -351,6 +351,7 @@ function main(): void $shuffle = false; $bless = false; $workers = null; + $workersExplicit = false; $context_line_count = 3; $num_repeats = 1; $show_progress = true; @@ -412,6 +413,7 @@ function main(): void switch ($switch) { case 'j': + $workersExplicit = true; $workers = substr($argv[$i], 2); if ($workers == 0 || !preg_match('/^\d+$/', $workers)) { error("'$workers' is not a valid number of workers, try e.g. -j16 for 16 workers"); @@ -641,6 +643,17 @@ function main(): void } } + if (!$workersExplicit && (!$selected_tests || count($test_files) > 1)) { + $workers = get_default_worker_count(); + if ($workers !== null + && ($valgrind !== null || isset($environment['SKIP_ASAN']))) { + $workers = min($workers, 2); + } + if ($workers !== null && !can_create_parallel_worker_socket()) { + $workers = null; + } + } + if ($online === null && !isset($environment['SKIP_ONLINE_TESTS'])) { $online = false; } @@ -800,6 +813,53 @@ function main(): void } } +function get_default_worker_count(): ?int +{ + if (IS_WINDOWS) { + $workerCount = getenv('NUMBER_OF_PROCESSORS'); + return is_string($workerCount) ? parse_default_worker_count($workerCount) : null; + } + + $commands = [ + 'nproc 2>/dev/null', + 'getconf _NPROCESSORS_ONLN 2>/dev/null', + 'getconf NPROCESSORS_ONLN 2>/dev/null', + 'sysctl -n hw.logicalcpu 2>/dev/null', + 'sysctl -n hw.ncpu 2>/dev/null', + ]; + foreach ($commands as $command) { + $workerCount = shell_exec($command); + if (is_string($workerCount) + && ($workerCount = parse_default_worker_count($workerCount)) !== null) { + return $workerCount; + } + } + + return null; +} + +function parse_default_worker_count(string $workerCount): ?int +{ + $workerCount = trim($workerCount); + if (preg_match('/^[0-9]+$/D', $workerCount) !== 1) { + return null; + } + + $workerCount = (int) $workerCount; + return $workerCount >= 2 ? min($workerCount, 10) : null; +} + +function can_create_parallel_worker_socket(): bool +{ + $socket = @stream_socket_server('tcp://127.0.0.1:0'); + if ($socket === false) { + return false; + } + + fclose($socket); + return true; +} + function verify_config(string $php): void { if (empty($php) || !file_exists($php)) { @@ -820,7 +880,7 @@ function write_information(array $user_tests, $phpdbg): void $php_escaped = escapeshellarg($php); // Get info from php - $info_file = __DIR__ . '/run-test-info.php'; + $info_file = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'run-test-info-' . getmypid() . '.php'; @unlink($info_file); $php_info = ' --EXPECT-- string(25) " diff --git a/tests/run-test/automatic_worker_limit.phpt b/tests/run-test/automatic_worker_limit.phpt new file mode 100644 index 000000000000..5bf6df07365e --- /dev/null +++ b/tests/run-test/automatic_worker_limit.phpt @@ -0,0 +1,107 @@ +--TEST-- +Automatic worker detection is capped for regular and instrumented runs +--SKIPIF-- + +--ENV-- +TEST_PHP_FORK_SERVER=0 +--FILE-- + + --EXPECT-- + ok + PHPT); +} + +$environment = [ + 'PATH' => $bin . PATH_SEPARATOR . getenv('PATH'), + 'TEST_PHP_EXECUTABLE' => getenv('TEST_PHP_EXECUTABLE'), + 'TEST_PHP_FORK_SERVER' => '0', +]; +foreach (['TEMP', 'TMPDIR'] as $name) { + if (($value = getenv($name)) !== false) { + $environment[$name] = $value; + } +} + +$runTests = static function (array $arguments) use ($environment, $testFiles): array { + $process = proc_open( + [ + getenv('TEST_PHP_EXECUTABLE'), + dirname(__DIR__, 2) . '/run-tests.php', + '-q', + '--no-progress', + ...$arguments, + ...$testFiles, + ], + [ + 0 => ['pipe', 'r'], + 1 => ['pipe', 'w'], + 2 => ['redirect', 1], + ], + $pipes, + null, + $environment, + ); + fclose($pipes[0]); + $output = stream_get_contents($pipes[1]); + fclose($pipes[1]); + + return [proc_close($process), $output]; +}; + +[$exitCode, $output] = $runTests([]); +var_dump($exitCode); +var_dump(str_contains($output, 'Spawning 10 workers...')); +var_dump(str_contains($output, 'Spawning 11 workers...')); + +[$exitCode, $output] = $runTests(['--asan']); +var_dump($exitCode); +var_dump(str_contains($output, 'Spawning 2 workers...')); +var_dump(str_contains($output, 'Spawning 10 workers...')); + +[$exitCode, $output] = $runTests(['--asan', '-j3']); +var_dump($exitCode); +var_dump(str_contains($output, 'Spawning 3 workers...')); +?> +--CLEAN-- + +--EXPECT-- +int(0) +bool(true) +bool(false) +int(0) +bool(true) +bool(false) +int(0) +bool(true)