From 30fc2f7f0d3d16abd10e0a323244230476b97d1c Mon Sep 17 00:00:00 2001 From: Sebastian Bergmann Date: Wed, 27 Aug 2025 11:05:29 +0200 Subject: [PATCH 1/9] Delete existing tests for test reordering --- tests/_files/TestWithDifferentSizes.php | 76 -- .../_files/ClonedDependencyTest.php | 70 -- .../MultiDependencyTest_result_cache.txt | 1 - .../execution-order/_files/StackTest.php | 38 - .../_files/TestWithDifferentDurations.php | 30 - ...ifferentDurations.phpunit.result.cache.txt | 1 - .../_files/order-by-duration.phpunit.xml | 11 - .../execution-order/cache-result.phpt | 34 - .../execution-order/dependencies-clone.phpt | 22 - .../dependencies-isolation.phpt | 54 -- .../depends-as-parameter-with-isolation.phpt | 22 - .../execution-order/depends-as-parameter.phpt | 21 - ...nds-multiple-parameter-with-isolation.phpt | 22 - .../depends-multiple-parameters.phpt | 21 - .../execution-order/depends-on-class.phpt | 58 -- .../order-by-default-invalid-via-cli.phpt | 15 - .../order-by-duration-via-cli.phpt | 50 -- .../order-by-duration-via-phpunit-xml.phpt | 54 -- ...randomized-with-dependency-resolution.phpt | 24 - .../ResultCache/DefaultResultCacheTest.php | 99 --- .../ResultCache/NullTestResultCacheTest.php | 29 - tests/unit/Runner/TestSuiteSorterTest.php | 647 ------------------ 22 files changed, 1399 deletions(-) delete mode 100644 tests/_files/TestWithDifferentSizes.php delete mode 100644 tests/end-to-end/execution-order/_files/ClonedDependencyTest.php delete mode 100644 tests/end-to-end/execution-order/_files/MultiDependencyTest_result_cache.txt delete mode 100644 tests/end-to-end/execution-order/_files/StackTest.php delete mode 100644 tests/end-to-end/execution-order/_files/TestWithDifferentDurations.php delete mode 100644 tests/end-to-end/execution-order/_files/TestWithDifferentDurations.phpunit.result.cache.txt delete mode 100644 tests/end-to-end/execution-order/_files/order-by-duration.phpunit.xml delete mode 100644 tests/end-to-end/execution-order/cache-result.phpt delete mode 100644 tests/end-to-end/execution-order/dependencies-clone.phpt delete mode 100644 tests/end-to-end/execution-order/dependencies-isolation.phpt delete mode 100644 tests/end-to-end/execution-order/depends-as-parameter-with-isolation.phpt delete mode 100644 tests/end-to-end/execution-order/depends-as-parameter.phpt delete mode 100644 tests/end-to-end/execution-order/depends-multiple-parameter-with-isolation.phpt delete mode 100644 tests/end-to-end/execution-order/depends-multiple-parameters.phpt delete mode 100644 tests/end-to-end/execution-order/depends-on-class.phpt delete mode 100644 tests/end-to-end/execution-order/order-by-default-invalid-via-cli.phpt delete mode 100644 tests/end-to-end/execution-order/order-by-duration-via-cli.phpt delete mode 100644 tests/end-to-end/execution-order/order-by-duration-via-phpunit-xml.phpt delete mode 100644 tests/end-to-end/execution-order/test-order-randomized-with-dependency-resolution.phpt delete mode 100644 tests/unit/Runner/ResultCache/DefaultResultCacheTest.php delete mode 100644 tests/unit/Runner/ResultCache/NullTestResultCacheTest.php delete mode 100644 tests/unit/Runner/TestSuiteSorterTest.php diff --git a/tests/_files/TestWithDifferentSizes.php b/tests/_files/TestWithDifferentSizes.php deleted file mode 100644 index 31ff76389ab..00000000000 --- a/tests/_files/TestWithDifferentSizes.php +++ /dev/null @@ -1,76 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\TestFixture; - -use PHPUnit\Framework\TestCase; - -final class TestWithDifferentSizes extends TestCase -{ - public static function provider(): array - { - return [ - [false], - [true], - ]; - } - - public function testWithSizeUnknown(): void - { - $this->assertTrue(true); - } - - /** - * @large - */ - public function testWithSizeLarge(): void - { - $this->assertTrue(true); - } - - /** - * @depends testDataProviderWithSizeMedium - * - * @medium - */ - public function testWithSizeMedium(): void - { - $this->assertTrue(true); - } - - /** - * @depends testWithSizeLarge - * - * @small - */ - public function testWithSizeSmall(): void - { - $this->assertTrue(true); - } - - /** - * @dataProvider provider - * - * @small - */ - public function testDataProviderWithSizeSmall(bool $value): void - { - $this->assertTrue(true); - } - - /** - * @dataProvider provider - * - * @medium - */ - public function testDataProviderWithSizeMedium(bool $value): void - { - $this->assertTrue(true); - } -} diff --git a/tests/end-to-end/execution-order/_files/ClonedDependencyTest.php b/tests/end-to-end/execution-order/_files/ClonedDependencyTest.php deleted file mode 100644 index e557414f091..00000000000 --- a/tests/end-to-end/execution-order/_files/ClonedDependencyTest.php +++ /dev/null @@ -1,70 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\TestFixture; - -use PHPUnit\Framework\TestCase; -use stdClass; - -class ClonedDependencyTest extends TestCase -{ - private static $dependency; - - public static function setUpBeforeClass(): void - { - self::$dependency = new stdClass; - } - - public function testOne() - { - $this->assertTrue(true); - - return self::$dependency; - } - - /** - * @depends testOne - */ - public function testTwo($dependency): void - { - $this->assertSame(self::$dependency, $dependency); - } - - /** - * @depends !clone testOne - */ - public function testThree($dependency): void - { - $this->assertSame(self::$dependency, $dependency); - } - - /** - * @depends clone testOne - */ - public function testFour($dependency): void - { - $this->assertNotSame(self::$dependency, $dependency); - } - - /** - * @depends !shallowClone testOne - */ - public function testFive($dependency): void - { - $this->assertSame(self::$dependency, $dependency); - } - - /** - * @depends shallowClone testOne - */ - public function testSix($dependency): void - { - $this->assertNotSame(self::$dependency, $dependency); - } -} diff --git a/tests/end-to-end/execution-order/_files/MultiDependencyTest_result_cache.txt b/tests/end-to-end/execution-order/_files/MultiDependencyTest_result_cache.txt deleted file mode 100644 index a9735ef50ec..00000000000 --- a/tests/end-to-end/execution-order/_files/MultiDependencyTest_result_cache.txt +++ /dev/null @@ -1 +0,0 @@ -{"version":2,"defects":{"PHPUnit\\TestFixture\\MultiDependencyTest::testFive":1},"times":{"PHPUnit\\TestFixture\\MultiDependencyTest::testOne":0,"PHPUnit\\TestFixture\\MultiDependencyTest::testTwo":0,"PHPUnit\\TestFixture\\MultiDependencyTest::testThree":0,"PHPUnit\\TestFixture\\MultiDependencyTest::testFour":0,"PHPUnit\\TestFixture\\MultiDependencyTest::testFive":0}} diff --git a/tests/end-to-end/execution-order/_files/StackTest.php b/tests/end-to-end/execution-order/_files/StackTest.php deleted file mode 100644 index d11fd67e0b9..00000000000 --- a/tests/end-to-end/execution-order/_files/StackTest.php +++ /dev/null @@ -1,38 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\TestFixture; - -use function array_pop; -use function end; -use PHPUnit\Framework\TestCase; - -class StackTest extends TestCase -{ - public function testPush() - { - $stack = []; - $this->assertCount(0, $stack); - - $stack[] = 'foo'; - $this->assertEquals('foo', end($stack)); - $this->assertCount(1, $stack); - - return $stack; - } - - /** - * @depends testPush - */ - public function testPop(array $stack): void - { - $this->assertEquals('foo', array_pop($stack)); - $this->assertCount(0, $stack); - } -} diff --git a/tests/end-to-end/execution-order/_files/TestWithDifferentDurations.php b/tests/end-to-end/execution-order/_files/TestWithDifferentDurations.php deleted file mode 100644 index b01a39f23db..00000000000 --- a/tests/end-to-end/execution-order/_files/TestWithDifferentDurations.php +++ /dev/null @@ -1,30 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\TestFixture; - -use PHPUnit\Framework\TestCase; - -final class TestWithDifferentDurations extends TestCase -{ - public function testOne(): void - { - $this->assertTrue(true); - } - - public function testTwo(): void - { - $this->assertTrue(true); - } - - public function testThree(): void - { - $this->assertTrue(true); - } -} diff --git a/tests/end-to-end/execution-order/_files/TestWithDifferentDurations.phpunit.result.cache.txt b/tests/end-to-end/execution-order/_files/TestWithDifferentDurations.phpunit.result.cache.txt deleted file mode 100644 index 8fd87ae1f03..00000000000 --- a/tests/end-to-end/execution-order/_files/TestWithDifferentDurations.phpunit.result.cache.txt +++ /dev/null @@ -1 +0,0 @@ -{"version":2,"defects":[],"times":{"PHPUnit\\TestFixture\\TestWithDifferentDurations::testOne":2.006,"PHPUnit\\TestFixture\\TestWithDifferentDurations::testTwo":0,"PHPUnit\\TestFixture\\TestWithDifferentDurations::testThree":3.001,"PHPUnit\\TestFixture\\ExampleTest::testOne":2.006,"PHPUnit\\TestFixture\\ExampleTest::testTwo":3.001,"PHPUnit\\TestFixture\\ExampleTest::testThree":0}} diff --git a/tests/end-to-end/execution-order/_files/order-by-duration.phpunit.xml b/tests/end-to-end/execution-order/_files/order-by-duration.phpunit.xml deleted file mode 100644 index 2ef82a66223..00000000000 --- a/tests/end-to-end/execution-order/_files/order-by-duration.phpunit.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - ./TestWithDifferentDurations.php - - - diff --git a/tests/end-to-end/execution-order/cache-result.phpt b/tests/end-to-end/execution-order/cache-result.phpt deleted file mode 100644 index 32d66e71f90..00000000000 --- a/tests/end-to-end/execution-order/cache-result.phpt +++ /dev/null @@ -1,34 +0,0 @@ ---TEST-- -phpunit --order-by=no-depends,reverse --cache-result --cache-result-file ./tests/_files/MultiDependencyTest.php ---FILE-- -run($_SERVER['argv']); - -print file_get_contents($cacheDirectory . DIRECTORY_SEPARATOR . 'test-results'); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -.SS.. 5 / 5 (100%) - -Time: %s, Memory: %s - -OK, but some tests were skipped! -Tests: 5, Assertions: 3, Skipped: 2. -{"version":2,"defects":{"PHPUnit\\TestFixture\\MultiDependencyTest::testFour":1,"PHPUnit\\TestFixture\\MultiDependencyTest::testThree":1},"times":{"PHPUnit\\TestFixture\\MultiDependencyTest::testFive":%f,"PHPUnit\\TestFixture\\MultiDependencyTest::testFour":%f,"PHPUnit\\TestFixture\\MultiDependencyTest::testThree":%f,"PHPUnit\\TestFixture\\MultiDependencyTest::testTwo":%f,"PHPUnit\\TestFixture\\MultiDependencyTest::testOne":%f}} ---CLEAN-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -...... 6 / 6 (100%) - -Time: %s, Memory: %s - -OK (6 tests, 6 assertions) - diff --git a/tests/end-to-end/execution-order/dependencies-isolation.phpt b/tests/end-to-end/execution-order/dependencies-isolation.phpt deleted file mode 100644 index 4b924f59e57..00000000000 --- a/tests/end-to-end/execution-order/dependencies-isolation.phpt +++ /dev/null @@ -1,54 +0,0 @@ ---TEST-- -phpunit --process-isolation --display-skipped ../../_files/dependencies ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -...FSSSEE 9 / 9 (100%) - -Time: %s, Memory: %s - -There were 2 errors: - -1) PHPUnit\TestFixture\DependencyFailureTest::testHandlesDependsAnnotationForNonexistentTests -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::doesNotExist" which does not exist - -2) PHPUnit\TestFixture\DependencyFailureTest::testHandlesDependsAnnotationWithNoMethodSpecified -This test has an invalid dependency - --- - -There was 1 failure: - -1) PHPUnit\TestFixture\DependencyFailureTest::testOne -Failed asserting that false is true. - -%s:%i - --- - -There were 3 skipped tests: - -1) PHPUnit\TestFixture\DependencyFailureTest::testTwo -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testOne" to pass - -2) PHPUnit\TestFixture\DependencyFailureTest::testThree -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testTwo" to pass - -3) PHPUnit\TestFixture\DependencyFailureTest::testFour -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testOne" to pass - -ERRORS! -Tests: 9, Assertions: 4, Errors: 2, Failures: 1, Skipped: 3. diff --git a/tests/end-to-end/execution-order/depends-as-parameter-with-isolation.phpt b/tests/end-to-end/execution-order/depends-as-parameter-with-isolation.phpt deleted file mode 100644 index 3e1a5ce946f..00000000000 --- a/tests/end-to-end/execution-order/depends-as-parameter-with-isolation.phpt +++ /dev/null @@ -1,22 +0,0 @@ ---TEST-- -phpunit --process-isolation _files/StackTest.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -.. 2 / 2 (100%) - -Time: %s, Memory: %s - -OK (2 tests, 5 assertions) diff --git a/tests/end-to-end/execution-order/depends-as-parameter.phpt b/tests/end-to-end/execution-order/depends-as-parameter.phpt deleted file mode 100644 index d7461f6eb16..00000000000 --- a/tests/end-to-end/execution-order/depends-as-parameter.phpt +++ /dev/null @@ -1,21 +0,0 @@ ---TEST-- -phpunit _files/StackTest.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -.. 2 / 2 (100%) - -Time: %s, Memory: %s - -OK (2 tests, 5 assertions) diff --git a/tests/end-to-end/execution-order/depends-multiple-parameter-with-isolation.phpt b/tests/end-to-end/execution-order/depends-multiple-parameter-with-isolation.phpt deleted file mode 100644 index 9f19922e0aa..00000000000 --- a/tests/end-to-end/execution-order/depends-multiple-parameter-with-isolation.phpt +++ /dev/null @@ -1,22 +0,0 @@ ---TEST-- -phpunit --process-isolation _files/MultiDependencyTest.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -..... 5 / 5 (100%) - -Time: %s, Memory: %s - -OK (5 tests, 6 assertions) diff --git a/tests/end-to-end/execution-order/depends-multiple-parameters.phpt b/tests/end-to-end/execution-order/depends-multiple-parameters.phpt deleted file mode 100644 index 02c6884b487..00000000000 --- a/tests/end-to-end/execution-order/depends-multiple-parameters.phpt +++ /dev/null @@ -1,21 +0,0 @@ ---TEST-- -phpunit _files/MultiDependencyTest.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -..... 5 / 5 (100%) - -Time: %s, Memory: %s - -OK (5 tests, 6 assertions) diff --git a/tests/end-to-end/execution-order/depends-on-class.phpt b/tests/end-to-end/execution-order/depends-on-class.phpt deleted file mode 100644 index ee179155d5a..00000000000 --- a/tests/end-to-end/execution-order/depends-on-class.phpt +++ /dev/null @@ -1,58 +0,0 @@ ---TEST-- -phpunit -c ../../_files/configuration.depends-on-class.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s -Configuration: %s%etests%e_files%econfiguration.depends-on-class.xml - -....SFSSSEE 11 / 11 (100%) - -Time: %s, Memory: %s - -There were 2 errors: - -1) PHPUnit\TestFixture\DependencyFailureTest::testHandlesDependsAnnotationForNonexistentTests -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::doesNotExist" which does not exist - -2) PHPUnit\TestFixture\DependencyFailureTest::testHandlesDependsAnnotationWithNoMethodSpecified -This test has an invalid dependency - --- - -There was 1 failure: - -1) PHPUnit\TestFixture\DependencyFailureTest::testOne -Failed asserting that false is true. - -%s%etests%e_files%edependencies%eDependencyFailureTest.php:18 - --- - -There were 4 skipped tests: - -1) PHPUnit\TestFixture\DependencyOnClassTest::testThatDependsOnAFailingClass -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::class" to pass - -2) PHPUnit\TestFixture\DependencyFailureTest::testTwo -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testOne" to pass - -3) PHPUnit\TestFixture\DependencyFailureTest::testThree -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testTwo" to pass - -4) PHPUnit\TestFixture\DependencyFailureTest::testFour -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testOne" to pass - -ERRORS! -Tests: 11, Assertions: 5, Errors: 2, Failures: 1, Skipped: 4. diff --git a/tests/end-to-end/execution-order/order-by-default-invalid-via-cli.phpt b/tests/end-to-end/execution-order/order-by-default-invalid-via-cli.phpt deleted file mode 100644 index ad7c5efc69c..00000000000 --- a/tests/end-to-end/execution-order/order-by-default-invalid-via-cli.phpt +++ /dev/null @@ -1,15 +0,0 @@ ---TEST-- -phpunit --order-by=default,foobar ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -unrecognized --order-by option: foobar diff --git a/tests/end-to-end/execution-order/order-by-duration-via-cli.phpt b/tests/end-to-end/execution-order/order-by-duration-via-cli.phpt deleted file mode 100644 index 9c9c73aa24a..00000000000 --- a/tests/end-to-end/execution-order/order-by-duration-via-cli.phpt +++ /dev/null @@ -1,50 +0,0 @@ ---TEST-- -phpunit --order-by=duration ./tests/end-to-end/execution-order/_files/TestWithDifferentDurations.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit Started (PHPUnit %s using PHP %s) -Test Runner Configured -Test Suite Loaded (3 tests) -Event Facade Sealed -Test Runner Started -Test Suite Sorted -Test Runner Execution Started (3 tests) -Test Suite Started (PHPUnit\TestFixture\TestWithDifferentDurations, 3 tests) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Suite Finished (PHPUnit\TestFixture\TestWithDifferentDurations, 3 tests) -Test Runner Execution Finished -Test Runner Finished -PHPUnit Finished (Shell Exit Code: 0) ---CLEAN-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit Started (PHPUnit %s using PHP %s) -Test Runner Configured -Test Suite Loaded (3 tests) -Event Facade Sealed -Test Runner Started -Test Suite Sorted -Test Runner Execution Started (3 tests) -Test Suite Started (%sorder-by-duration.phpunit.xml, 3 tests) -Test Suite Started (order-by-duration, 3 tests) -Test Suite Started (PHPUnit\TestFixture\TestWithDifferentDurations, 3 tests) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Suite Finished (PHPUnit\TestFixture\TestWithDifferentDurations, 3 tests) -Test Suite Finished (order-by-duration, 3 tests) -Test Suite Finished (%sorder-by-duration.phpunit.xml, 3 tests) -Test Runner Execution Finished -Test Runner Finished -PHPUnit Finished (Shell Exit Code: 0) ---CLEAN-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s -Random Seed: %d - -..... 5 / 5 (100%) - -Time: %s, Memory: %s - -OK (5 tests, 6 assertions) diff --git a/tests/unit/Runner/ResultCache/DefaultResultCacheTest.php b/tests/unit/Runner/ResultCache/DefaultResultCacheTest.php deleted file mode 100644 index c2c4224b1d6..00000000000 --- a/tests/unit/Runner/ResultCache/DefaultResultCacheTest.php +++ /dev/null @@ -1,99 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\Runner\ResultCache; - -use function sys_get_temp_dir; -use function tempnam; -use function uniqid; -use function unlink; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\Small; -use PHPUnit\Framework\TestCase; -use PHPUnit\Framework\TestStatus\TestStatus; -use PHPUnit\TestFixture\MultiDependencyTest; - -#[CoversClass(DefaultResultCache::class)] -#[Small] -final class DefaultResultCacheTest extends TestCase -{ - public function testGetTimeForNonExistentTestNameReturnsFloatZero(): void - { - $this->assertSame(0.0, (new DefaultResultCache)->time('doesNotExist')); - } - - public function testReadsCacheFromProvidedFilename(): void - { - $cacheFile = TEST_FILES_PATH . '../end-to-end/execution-order/_files/MultiDependencyTest_result_cache.txt'; - $cache = new DefaultResultCache($cacheFile); - $cache->load(); - - $this->assertTrue($cache->status(MultiDependencyTest::class . '::testOne')->isUnknown()); - $this->assertTrue($cache->status(MultiDependencyTest::class . '::testFive')->isSkipped()); - } - - public function testDoesClearCacheBeforeLoad(): void - { - $cacheFile = TEST_FILES_PATH . '../end-to-end/execution-order/_files/MultiDependencyTest_result_cache.txt'; - $cache = new DefaultResultCache($cacheFile); - $cache->setStatus('someTest', TestStatus::failure()); - - $this->assertTrue($cache->status(MultiDependencyTest::class . '::testFive')->isUnknown()); - - $cache->load(); - - $this->assertTrue($cache->status(MultiDependencyTest::class . '::someTest')->isUnknown()); - $this->assertTrue($cache->status(MultiDependencyTest::class . '::testFive')->isSkipped()); - } - - public function testCanPersistCacheToFile(): void - { - $cacheFile = tempnam(sys_get_temp_dir(), 'phpunit_'); - $cache = new DefaultResultCache($cacheFile); - $testName = 'test' . uniqid('', true); - - $cache->setStatus($testName, TestStatus::skipped()); - $cache->persist(); - - $cache = new DefaultResultCache($cacheFile); - $cache->load(); - - $this->assertTrue($cache->status($testName)->isSkipped()); - - unlink($cacheFile); - } - - public function testCanBeMerged(): void - { - $cacheSourceOne = new DefaultResultCache; - $cacheSourceOne->setStatus('status.a', TestStatus::skipped()); - $cacheSourceOne->setStatus('status.b', TestStatus::incomplete()); - $cacheSourceOne->setTime('time.a', 1); - $cacheSourceOne->setTime('time.b', 2); - $cacheSourceTwo = new DefaultResultCache; - $cacheSourceTwo->setStatus('status.c', TestStatus::failure()); - $cacheSourceTwo->setTime('time.c', 4); - - $sum = new DefaultResultCache; - $sum->mergeWith($cacheSourceOne); - - $this->assertSame(TestStatus::skipped()->asString(), $sum->status('status.a')->asString()); - $this->assertSame(TestStatus::incomplete()->asString(), $sum->status('status.b')->asString()); - $this->assertNotSame(TestStatus::failure()->asString(), $sum->status('status.c')->asString()); - - $this->assertSame(1.0, $sum->time('time.a')); - $this->assertSame(2.0, $sum->time('time.b')); - $this->assertNotSame(4.0, $sum->time('time.c')); - - $sum->mergeWith($cacheSourceTwo); - - $this->assertSame(TestStatus::failure()->asString(), $sum->status('status.c')->asString()); - $this->assertSame(4.0, $sum->time('time.c')); - } -} diff --git a/tests/unit/Runner/ResultCache/NullTestResultCacheTest.php b/tests/unit/Runner/ResultCache/NullTestResultCacheTest.php deleted file mode 100644 index c421013190f..00000000000 --- a/tests/unit/Runner/ResultCache/NullTestResultCacheTest.php +++ /dev/null @@ -1,29 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\Runner\ResultCache; - -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\Small; -use PHPUnit\Framework\TestCase; - -#[CoversClass(NullResultCache::class)] -#[Small] -final class NullTestResultCacheTest extends TestCase -{ - public function testHasWorkingStubs(): void - { - $cache = new NullResultCache; - $cache->load(); - $cache->persist(); - - $this->assertTrue($cache->status('testName')->isUnknown()); - $this->assertSame(0.0, $cache->time('testName')); - } -} diff --git a/tests/unit/Runner/TestSuiteSorterTest.php b/tests/unit/Runner/TestSuiteSorterTest.php deleted file mode 100644 index acf68db50ce..00000000000 --- a/tests/unit/Runner/TestSuiteSorterTest.php +++ /dev/null @@ -1,647 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\Runner; - -use function mt_srand; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\DataProvider; -use PHPUnit\Framework\Attributes\Small; -use PHPUnit\Framework\TestCase; -use PHPUnit\Framework\TestStatus\TestStatus; -use PHPUnit\Framework\TestSuite; -use PHPUnit\Runner\ResultCache\DefaultResultCache; -use PHPUnit\TestFixture\MultiDependencyTest; -use PHPUnit\TestFixture\TestWithDifferentSizes; -use ReflectionClass; - -#[CoversClass(TestSuiteSorter::class)] -#[Small] -final class TestSuiteSorterTest extends TestCase -{ - private const IGNORE_DEPENDENCIES = false; - private const RESOLVE_DEPENDENCIES = true; - private const MULTI_DEPENDENCY_TEST_EXECUTION_ORDER = [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ]; - - public static function orderDurationWithoutCacheProvider(): array - { - return [ - 'dependency-ignore' => [ - self::IGNORE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - 'dependency-resolve' => [ - self::RESOLVE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - ]; - } - - public static function orderDurationWithCacheProvider(): array - { - return [ - 'duration-same-dependency-ignore' => [ - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => 1, - 'testTwo' => 1, - 'testThree' => 1, - 'testFour' => 1, - 'testFive' => 1, - ], - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - 'duration-same-dependency-resolve' => [ - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => 1, - 'testTwo' => 1, - 'testThree' => 1, - 'testFour' => 1, - 'testFive' => 1, - ], - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - 'duration-different-dependency-ignore' => [ - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => 5, - 'testTwo' => 3, - 'testThree' => 4, - 'testFour' => 1, - 'testFive' => 2, - ], - [ - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testOne', - ], - ], - 'duration-different-dependency-resolve' => [ - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => 5, - 'testTwo' => 3, - 'testThree' => 4, - 'testFour' => 1, - 'testFive' => 2, - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - ]; - } - - /** - * A data provider for basic execution reordering options based on MultiDependencyTest. - * - * This class has the following relevant properties: - * - * - it has five tests testOne, testTwo, testThree, testFour, testFive - * - testThree depends on testOne and testTwo - * - testFour depends on MultiDependencyTest::testThree - * - testFive has no dependencies - */ - public static function commonSorterOptionsProvider(): array - { - return [ - 'default' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // Activating dependency resolution should have no effect under normal circumstances - 'resolve default' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::RESOLVE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // Reversing without checks should give a simple reverse order - 'reverse' => [ - TestSuiteSorter::ORDER_REVERSED, - self::IGNORE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - ], - ], - - // Reversing with resolution still allows testFive to move to front, testTwo before testOne - 'resolve reverse' => [ - TestSuiteSorter::ORDER_REVERSED, - self::RESOLVE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - ]; - } - - /** - * A data provider for testing defects execution reordering options based on MultiDependencyTest. - * - * This class has the following relevant properties: - * - * - it has five tests testOne, testTwo, testThree, testFour, testFive - * - testThree depends on testOne and testTwo - * - testFour depends on MultiDependencyTest::testThree - * - testFive has no dependencies - */ - public static function defectsSorterOptionsProvider(): array - { - return [ - // The most simple situation should work as normal - 'default, no defects' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::success(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::success(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // Running with an empty cache should not spook the TestSuiteSorter - 'default, empty result cache' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - // empty result cache - ], - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // testFive is independent and can be moved to the front - 'default, testFive skipped' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::success(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::skipped(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - - // Defects in testFive and testTwo, but the faster testFive should be run first - 'default, testTwo testFive skipped' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::skipped(), 'time' => 1], - 'testThree' => ['state' => TestStatus::success(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::skipped(), 'time' => 0], - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - - // Skipping testThree will move it to the front when ignoring dependencies - 'default, testThree skipped' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::skipped(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::success(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // Skipping testThree will move it to the front but behind its dependencies - 'default resolve, testThree skipped' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::skipped(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::success(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // Skipping testThree will move it to the front and keep the others reversed - 'reverse, testThree skipped' => [ - TestSuiteSorter::ORDER_REVERSED, - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::skipped(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::success(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - ], - ], - - // Demonstrate a limit of the dependency resolver: after sorting defects to the front, - // the resolver will mark testFive done before testThree because of dependencies - 'default resolve, testThree skipped, testFive fast' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::skipped(), 'time' => 0], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::skipped(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - - // Torture test - // - incomplete TestResultCache - // - skipped testThree: will move it to the front as far as possible - // - testOne and testTwo are required before testThree, but can be reversed locally - // - testFive is independent will remain reversed up front - 'reverse resolve, testThree skipped' => [ - TestSuiteSorter::ORDER_REVERSED, - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::skipped(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - - // Make sure the dependency resolver is not confused by failing tests. - // Scenario: Four has a @depends on Three and fails. Result: Three is still run first - // testFive also fails but can be moved around freely and will be up front. - 'depends first, then defects' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::success(), 'time' => 1], - 'testFour' => ['state' => TestStatus::failure(), 'time' => 1], - 'testFive' => ['state' => TestStatus::failure(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - ]; - } - - public static function suiteSorterOptionPermutationsProvider(): array - { - $orderValues = [TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, TestSuiteSorter::ORDER_RANDOMIZED]; - $resolveValues = [false, true]; - $orderDefectsValues = [TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_DEFECTS_FIRST]; - - $data = []; - - foreach ($orderValues as $order) { - foreach ($resolveValues as $resolve) { - foreach ($orderDefectsValues as $orderDefects) { - $data[] = [$order, $resolve, $orderDefects]; - } - } - } - - return $data; - } - - public function testThrowsExceptionWhenUsingInvalidOrderOption(): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - $sorter = new TestSuiteSorter; - - $this->expectException(InvalidOrderException::class); - - $sorter->reorderTestsInSuite($suite, -1, false, TestSuiteSorter::ORDER_DEFAULT); - } - - public function testThrowsExceptionWhenUsingInvalidOrderDefectsOption(): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - $sorter = new TestSuiteSorter; - - $this->expectException(InvalidOrderException::class); - - $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_DEFAULT, false, -1); - } - - #[DataProvider('suiteSorterOptionPermutationsProvider')] - public function testShouldNotAffectEmptyTestSuite(int $order, bool $resolveDependencies, int $orderDefects): void - { - $sorter = new TestSuiteSorter; - $suite = TestSuite::empty('test suite name'); - - $sorter->reorderTestsInSuite($suite, $order, $resolveDependencies, $orderDefects); - - $this->assertEmpty($suite->tests()); - $this->assertEmpty($sorter->getOriginalExecutionOrder()); - $this->assertEmpty($sorter->getExecutionOrder()); - } - - #[DataProvider('commonSorterOptionsProvider')] - public function testBasicExecutionOrderOptions(int $order, bool $resolveDependencies, array $expectedOrder): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - $sorter = new TestSuiteSorter; - - $sorter->reorderTestsInSuite($suite, $order, $resolveDependencies, TestSuiteSorter::ORDER_DEFAULT); - - $this->assertSame(self::MULTI_DEPENDENCY_TEST_EXECUTION_ORDER, $sorter->getOriginalExecutionOrder()); - $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); - } - - public function testCanSetRandomizationWithASeed(): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - $sorter = new TestSuiteSorter; - - mt_srand(54321); - $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_RANDOMIZED, false, TestSuiteSorter::ORDER_DEFAULT); - - $expectedOrder = [ - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testOne', - ]; - - $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); - } - - public function testCanSetRandomizationWithDefectsFirst(): void - { - $cache = new DefaultResultCache; - - $runState = [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::error(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::error(), 'time' => 1], - ]; - - foreach ($runState as $testName => $data) { - $cache->setStatus(MultiDependencyTest::class . '::' . $testName, $data['state']); - $cache->setTime(MultiDependencyTest::class . '::' . $testName, $data['time']); - } - - $sorter = new TestSuiteSorter($cache); - - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - - mt_srand(54321); - $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_RANDOMIZED, false, TestSuiteSorter::ORDER_DEFECTS_FIRST); - - $expectedOrder = [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testOne', - ]; - - $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); - } - - public function testCanSetRandomizationWithASeedAndResolveDependencies(): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - $sorter = new TestSuiteSorter; - - mt_srand(54321); - $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_RANDOMIZED, true, TestSuiteSorter::ORDER_DEFAULT); - - $expectedOrder = [ - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ]; - - $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); - } - - #[DataProvider('orderDurationWithoutCacheProvider')] - public function testOrderDurationWithoutCache(bool $resolveDependencies, array $expected): void - { - $suite = TestSuite::empty('test suite name'); - - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - - $sorter = new TestSuiteSorter; - - $sorter->reorderTestsInSuite( - $suite, - TestSuiteSorter::ORDER_DURATION, - $resolveDependencies, - TestSuiteSorter::ORDER_DEFAULT, - ); - - $this->assertSame($expected, $sorter->getExecutionOrder()); - } - - #[DataProvider('orderDurationWithCacheProvider')] - public function testOrderDurationWithCache(bool $resolveDependencies, array $testTimes, array $expected): void - { - $suite = TestSuite::empty('test suite name'); - - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - - $cache = new DefaultResultCache; - - foreach ($testTimes as $testName => $time) { - $cache->setTime(MultiDependencyTest::class . '::' . $testName, $time); - } - - $sorter = new TestSuiteSorter($cache); - - $sorter->reorderTestsInSuite( - $suite, - TestSuiteSorter::ORDER_DURATION, - $resolveDependencies, - TestSuiteSorter::ORDER_DEFAULT, - ); - - $this->assertSame($expected, $sorter->getExecutionOrder()); - } - - #[DataProvider('defectsSorterOptionsProvider')] - public function testSuiteSorterDefectsOptions(int $order, bool $resolveDependencies, array $runState, array $expected): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - - $cache = new DefaultResultCache; - - foreach ($runState as $testName => $data) { - $cache->setStatus(MultiDependencyTest::class . '::' . $testName, $data['state']); - $cache->setTime(MultiDependencyTest::class . '::' . $testName, $data['time']); - } - - $sorter = new TestSuiteSorter($cache); - $sorter->reorderTestsInSuite($suite, $order, $resolveDependencies, TestSuiteSorter::ORDER_DEFECTS_FIRST); - - $this->assertSame($expected, $sorter->getExecutionOrder()); - } - - public function testOrderBySize(): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(TestWithDifferentSizes::class)); - $sorter = new TestSuiteSorter; - - $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_SIZE, true, TestSuiteSorter::ORDER_DEFAULT); - - $expectedOrder = [ - TestWithDifferentSizes::class . '::testDataProviderWithSizeSmall with data set #0', - TestWithDifferentSizes::class . '::testDataProviderWithSizeSmall with data set #1', - TestWithDifferentSizes::class . '::testDataProviderWithSizeMedium with data set #0', - TestWithDifferentSizes::class . '::testDataProviderWithSizeMedium with data set #1', - TestWithDifferentSizes::class . '::testWithSizeMedium', - TestWithDifferentSizes::class . '::testWithSizeLarge', - TestWithDifferentSizes::class . '::testWithSizeSmall', - TestWithDifferentSizes::class . '::testWithSizeUnknown', - ]; - - $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); - } -} From a4e6ee2c1828981c05e174ff6808d3b82ca576ec Mon Sep 17 00:00:00 2001 From: Sebastian Bergmann Date: Wed, 27 Aug 2025 11:06:22 +0200 Subject: [PATCH 2/9] Delete code that was only used by tests in PHPUnit's own test suite (but still run twice for the entire test suite for each test suite that is run) --- src/Runner/TestSuiteSorter.php | 49 ++-------------------------------- 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/src/Runner/TestSuiteSorter.php b/src/Runner/TestSuiteSorter.php index b5c0c19d695..077893108bb 100644 --- a/src/Runner/TestSuiteSorter.php +++ b/src/Runner/TestSuiteSorter.php @@ -76,16 +76,6 @@ final class TestSuiteSorter private array $defectSortOrder = []; private readonly ResultCache $cache; - /** - * @psalm-var array A list of normalized names of tests before reordering - */ - private array $originalExecutionOrder = []; - - /** - * @psalm-var array A list of normalized names of tests affected by reordering - */ - private array $executionOrder = []; - public function __construct(?ResultCache $cache = null) { $this->cache = $cache ?? new NullResultCache; @@ -94,7 +84,7 @@ public function __construct(?ResultCache $cache = null) /** * @throws Exception */ - public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies, int $orderDefects, bool $isRootTestSuite = true): void + public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies, int $orderDefects): void { $allowedOrders = [ self::ORDER_DEFAULT, @@ -117,13 +107,9 @@ public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDepend throw new InvalidOrderException; } - if ($isRootTestSuite) { - $this->originalExecutionOrder = $this->calculateTestExecutionOrder($suite); - } - if ($suite instanceof TestSuite) { foreach ($suite as $_suite) { - $this->reorderTestsInSuite($_suite, $order, $resolveDependencies, $orderDefects, false); + $this->reorderTestsInSuite($_suite, $order, $resolveDependencies, $orderDefects); } if ($orderDefects === self::ORDER_DEFECTS_FIRST) { @@ -132,20 +118,6 @@ public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDepend $this->sort($suite, $order, $resolveDependencies, $orderDefects); } - - if ($isRootTestSuite) { - $this->executionOrder = $this->calculateTestExecutionOrder($suite); - } - } - - public function getOriginalExecutionOrder(): array - { - return $this->originalExecutionOrder; - } - - public function getExecutionOrder(): array - { - return $this->executionOrder; } private function sort(TestSuite $suite, int $order, bool $resolveDependencies, int $orderDefects): void @@ -324,21 +296,4 @@ private function resolveDependencies(array $tests): array return array_merge($newTestOrder, $tests); } - - private function calculateTestExecutionOrder(Test $suite): array - { - $tests = []; - - if ($suite instanceof TestSuite) { - foreach ($suite->tests() as $test) { - if (!$test instanceof TestSuite && $test instanceof Reorderable) { - $tests[] = $test->sortId(); - } else { - $tests = array_merge($tests, $this->calculateTestExecutionOrder($test)); - } - } - } - - return $tests; - } } From bf4ef406b12284074a185f9c83dfae2fcef34358 Mon Sep 17 00:00:00 2001 From: Sebastian Bergmann Date: Wed, 27 Aug 2025 13:27:43 +0200 Subject: [PATCH 3/9] Initial work on new end-to-end tests for test suite sorting --- .../test-classes-with-defects/BarTest.php | 30 +++++++ .../test-classes-with-defects/BazTest.php | 30 +++++++ .../test-classes-with-defects/test-results | 1 + .../EndToEndTest.php | 16 ++++ .../IntegrationTest.php | 16 ++++ .../UnitTest.php | 16 ++++ .../fixture/test-classes/BarTest.php | 14 ++++ .../fixture/test-classes/FooTest.php | 19 +++++ .../test-methods-with-defects/FooTest.php | 30 +++++++ .../test-methods-with-defects/test-results | 1 + .../FooTest.php | 21 +++++ .../SizesTest.php | 33 ++++++++ ...-by-default-test-classes-with-defects.phpt | 73 ++++++++++++++++ ...ult-test-classes-with-different-sizes.phpt | 48 +++++++++++ .../order-by-default-test-classes.phpt | 46 ++++++++++ ...-by-default-test-methods-with-defects.phpt | 50 +++++++++++ ...efault-test-methods-with-dependencies.phpt | 37 ++++++++ ...ult-test-methods-with-different-sizes.phpt | 44 ++++++++++ .../order-by-default-test-methods.phpt | 37 ++++++++ ...fects-but-result-cache-does-not-exist.phpt | 80 ++++++++++++++++++ ...-by-defects-test-classes-with-defects.phpt | 84 +++++++++++++++++++ ...fects-but-result-cache-does-not-exist.phpt | 57 +++++++++++++ ...-by-defects-test-methods-with-defects.phpt | 61 ++++++++++++++ ...epends-test-methods-with-dependencies.phpt | 33 ++++++++ .../order-by-reverse-test-classes.phpt | 46 ++++++++++ .../order-by-reverse-test-methods.phpt | 37 ++++++++ ...ize-test-classes-with-different-sizes.phpt | 50 +++++++++++ ...ize-test-methods-with-different-sizes.phpt | 44 ++++++++++ 28 files changed, 1054 insertions(+) create mode 100644 tests/end-to-end/execution-order/fixture/test-classes-with-defects/BarTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-classes-with-defects/BazTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-classes-with-defects/test-results create mode 100644 tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/EndToEndTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/IntegrationTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/UnitTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-classes/BarTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-classes/FooTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-methods-with-defects/FooTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-methods-with-defects/test-results create mode 100644 tests/end-to-end/execution-order/fixture/test-methods-with-dependencies/FooTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-methods-with-different-sizes/SizesTest.php create mode 100644 tests/end-to-end/execution-order/order-by-default-test-classes-with-defects.phpt create mode 100644 tests/end-to-end/execution-order/order-by-default-test-classes-with-different-sizes.phpt create mode 100644 tests/end-to-end/execution-order/order-by-default-test-classes.phpt create mode 100644 tests/end-to-end/execution-order/order-by-default-test-methods-with-defects.phpt create mode 100644 tests/end-to-end/execution-order/order-by-default-test-methods-with-dependencies.phpt create mode 100644 tests/end-to-end/execution-order/order-by-default-test-methods-with-different-sizes.phpt create mode 100644 tests/end-to-end/execution-order/order-by-default-test-methods.phpt create mode 100644 tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects-but-result-cache-does-not-exist.phpt create mode 100644 tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects.phpt create mode 100644 tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects-but-result-cache-does-not-exist.phpt create mode 100644 tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects.phpt create mode 100644 tests/end-to-end/execution-order/order-by-no-depends-test-methods-with-dependencies.phpt create mode 100644 tests/end-to-end/execution-order/order-by-reverse-test-classes.phpt create mode 100644 tests/end-to-end/execution-order/order-by-reverse-test-methods.phpt create mode 100644 tests/end-to-end/execution-order/order-by-size-test-classes-with-different-sizes.phpt create mode 100644 tests/end-to-end/execution-order/order-by-size-test-methods-with-different-sizes.phpt diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BarTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BarTest.php new file mode 100644 index 00000000000..8cbf188409e --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BarTest.php @@ -0,0 +1,30 @@ +assertTrue(true); + } + + public function testTwo(): void + { + $this->assertTrue(false); + } + + public function testThree(): void + { + throw new RuntimeException('message'); + } + + public function testFour(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BazTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BazTest.php new file mode 100644 index 00000000000..6b8091087af --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BazTest.php @@ -0,0 +1,30 @@ +assertTrue(true); + } + + public function testTwo(): void + { + $this->assertTrue(false); + } + + public function testThree(): void + { + throw new RuntimeException('message'); + } + + public function testFour(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-defects/test-results b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/test-results new file mode 100644 index 00000000000..0939ed3671f --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/test-results @@ -0,0 +1 @@ +{"version":2,"defects":{"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testTwo":7,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testThree":8,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testTwo":7,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testThree":8},"times":{"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testOne":0.002,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testTwo":0.003,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testThree":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testFour":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testOne":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testTwo":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testThree":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testFour":0}} \ No newline at end of file diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/EndToEndTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/EndToEndTest.php new file mode 100644 index 00000000000..a5669e394e1 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/EndToEndTest.php @@ -0,0 +1,16 @@ +assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/IntegrationTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/IntegrationTest.php new file mode 100644 index 00000000000..6d083e0f927 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/IntegrationTest.php @@ -0,0 +1,16 @@ +assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/UnitTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/UnitTest.php new file mode 100644 index 00000000000..b88d8539804 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/UnitTest.php @@ -0,0 +1,16 @@ +assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes/BarTest.php b/tests/end-to-end/execution-order/fixture/test-classes/BarTest.php new file mode 100644 index 00000000000..aaf92e1e99e --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes/BarTest.php @@ -0,0 +1,14 @@ +assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes/FooTest.php b/tests/end-to-end/execution-order/fixture/test-classes/FooTest.php new file mode 100644 index 00000000000..9ad8aa5076d --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes/FooTest.php @@ -0,0 +1,19 @@ +assertTrue(true); + } + + public function testTwo(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-defects/FooTest.php b/tests/end-to-end/execution-order/fixture/test-methods-with-defects/FooTest.php new file mode 100644 index 00000000000..b224c010ac1 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-defects/FooTest.php @@ -0,0 +1,30 @@ +assertTrue(true); + } + + public function testTwo(): void + { + $this->assertTrue(false); + } + + public function testThree(): void + { + throw new RuntimeException('message'); + } + + public function testFour(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-defects/test-results b/tests/end-to-end/execution-order/fixture/test-methods-with-defects/test-results new file mode 100644 index 00000000000..eb8ac71fc22 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-defects/test-results @@ -0,0 +1 @@ +{"version":2,"defects":{"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testTwo":7,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testThree":8},"times":{"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testOne":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testTwo":0.001,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testThree":0.004,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testFour":0}} \ No newline at end of file diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-dependencies/FooTest.php b/tests/end-to-end/execution-order/fixture/test-methods-with-dependencies/FooTest.php new file mode 100644 index 00000000000..c2b74b4b560 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-dependencies/FooTest.php @@ -0,0 +1,21 @@ +assertTrue(true); + } + + public function testTwo(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-different-sizes/SizesTest.php b/tests/end-to-end/execution-order/fixture/test-methods-with-different-sizes/SizesTest.php new file mode 100644 index 00000000000..401a401a791 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-different-sizes/SizesTest.php @@ -0,0 +1,33 @@ +assertTrue(true); + } + + /** + * @large + */ + public function testTwo(): void + { + $this->assertTrue(true); + } + + /** + * @small + */ + public function testThree(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/order-by-default-test-classes-with-defects.phpt b/tests/end-to-end/execution-order/order-by-default-test-classes-with-defects.phpt new file mode 100644 index 00000000000..f06b9456a2f --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-classes-with-defects.phpt @@ -0,0 +1,73 @@ +--TEST-- +Default order: Test classes with defects +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (8 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (8 tests) +Test Suite Started (CLI Arguments, 8 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Suite Finished (CLI Arguments, 8 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-default-test-classes-with-different-sizes.phpt b/tests/end-to-end/execution-order/order-by-default-test-classes-with-different-sizes.phpt new file mode 100644 index 00000000000..70798229c1c --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-classes-with-different-sizes.phpt @@ -0,0 +1,48 @@ +--TEST-- +Default order: Suite with test classes that have different sizes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest, 1 test) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest, 1 test) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest, 1 test) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-default-test-classes.phpt b/tests/end-to-end/execution-order/order-by-default-test-classes.phpt new file mode 100644 index 00000000000..71f606586ed --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-classes.phpt @@ -0,0 +1,46 @@ +--TEST-- +Default order: Suite with test classes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\BarTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\BarTest, 1 test) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-default-test-methods-with-defects.phpt b/tests/end-to-end/execution-order/order-by-default-test-methods-with-defects.phpt new file mode 100644 index 00000000000..c58504764e7 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-methods-with-defects.phpt @@ -0,0 +1,50 @@ +--TEST-- +Default order: Test methods with defects +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (4 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (4 tests) +Test Suite Started (CLI Arguments, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Suite Finished (CLI Arguments, 4 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-default-test-methods-with-dependencies.phpt b/tests/end-to-end/execution-order/order-by-default-test-methods-with-dependencies.phpt new file mode 100644 index 00000000000..83653d503ac --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-methods-with-dependencies.phpt @@ -0,0 +1,37 @@ +--TEST-- +Default order: Test methods with dependencies +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (2 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (2 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest, 2 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest, 2 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-default-test-methods-with-different-sizes.phpt b/tests/end-to-end/execution-order/order-by-default-test-methods-with-different-sizes.phpt new file mode 100644 index 00000000000..1a599ea04fb --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-methods-with-different-sizes.phpt @@ -0,0 +1,44 @@ +--TEST-- +Default order: Test class with test methods that have different sizes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest, 3 tests) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-default-test-methods.phpt b/tests/end-to-end/execution-order/order-by-default-test-methods.phpt new file mode 100644 index 00000000000..1a5989eb0c2 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-methods.phpt @@ -0,0 +1,37 @@ +--TEST-- +Default order: Test methods +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (2 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (2 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects-but-result-cache-does-not-exist.phpt b/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects-but-result-cache-does-not-exist.phpt new file mode 100644 index 00000000000..7ef75b33218 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects-but-result-cache-does-not-exist.phpt @@ -0,0 +1,80 @@ +--TEST-- +Order by defects (without result cache): Test classes with defects +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (8 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (8 tests) +Test Suite Started (CLI Arguments, 8 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Suite Finished (CLI Arguments, 8 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects.phpt b/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects.phpt new file mode 100644 index 00000000000..134895a6f71 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects.phpt @@ -0,0 +1,84 @@ +--TEST-- +Order by defects (with result cache): Test classes with defects +--FILE-- +run($_SERVER['argv']); + +unlink($testResultsFile); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (8 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (8 tests) +Test Suite Started (CLI Arguments, 8 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Suite Finished (CLI Arguments, 8 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects-but-result-cache-does-not-exist.phpt b/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects-but-result-cache-does-not-exist.phpt new file mode 100644 index 00000000000..75290020c23 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects-but-result-cache-does-not-exist.phpt @@ -0,0 +1,57 @@ +--TEST-- +Order by defects (without result cache): Test methods with defects +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (4 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (4 tests) +Test Suite Started (CLI Arguments, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Suite Finished (CLI Arguments, 4 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects.phpt b/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects.phpt new file mode 100644 index 00000000000..5d2326dcf76 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects.phpt @@ -0,0 +1,61 @@ +--TEST-- +Order by defects (with result cache): Test methods with defects +--FILE-- +run($_SERVER['argv']); + +unlink($testResultsFile); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (4 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (4 tests) +Test Suite Started (CLI Arguments, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Suite Finished (CLI Arguments, 4 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-no-depends-test-methods-with-dependencies.phpt b/tests/end-to-end/execution-order/order-by-no-depends-test-methods-with-dependencies.phpt new file mode 100644 index 00000000000..dba45b6b9a4 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-no-depends-test-methods-with-dependencies.phpt @@ -0,0 +1,33 @@ +--TEST-- +Default order: Test methods with dependencies +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (2 tests) +Event Facade Sealed +Test Runner Started +Test Runner Execution Started (2 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest, 2 tests) +Test Skipped (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testOne) +This test depends on "PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo" to pass +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest, 2 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-reverse-test-classes.phpt b/tests/end-to-end/execution-order/order-by-reverse-test-classes.phpt new file mode 100644 index 00000000000..08c8a856202 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-reverse-test-classes.phpt @@ -0,0 +1,46 @@ +--TEST-- +Reverse order: Suite with test classes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\BarTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\BarTest, 1 test) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-reverse-test-methods.phpt b/tests/end-to-end/execution-order/order-by-reverse-test-methods.phpt new file mode 100644 index 00000000000..9d976d17fa6 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-reverse-test-methods.phpt @@ -0,0 +1,37 @@ +--TEST-- +Reverse order: Test methods +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (2 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (2 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-size-test-classes-with-different-sizes.phpt b/tests/end-to-end/execution-order/order-by-size-test-classes-with-different-sizes.phpt new file mode 100644 index 00000000000..36c08dca4ce --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-size-test-classes-with-different-sizes.phpt @@ -0,0 +1,50 @@ +--TEST-- +Order by test size: Suite with test classes that have different sizes +--XFAIL-- +TestSuiteSorter::cmpSize() does not handle TestSuite objects for TestCase classes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest, 1 test) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest, 1 test) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest, 1 test) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-size-test-methods-with-different-sizes.phpt b/tests/end-to-end/execution-order/order-by-size-test-methods-with-different-sizes.phpt new file mode 100644 index 00000000000..bcfce3ee358 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-size-test-methods-with-different-sizes.phpt @@ -0,0 +1,44 @@ +--TEST-- +Order by test size: Test class with test methods that have different sizes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest, 3 tests) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) From 2c4c927f717f58d8847fed78a4e26dd14eb0769f Mon Sep 17 00:00:00 2001 From: Sebastian Bergmann Date: Wed, 27 Aug 2025 13:43:26 +0200 Subject: [PATCH 4/9] Fix CS/WS issues --- .../fixture/test-classes-with-defects/BarTest.php | 8 ++++++++ .../fixture/test-classes-with-defects/BazTest.php | 8 ++++++++ .../test-classes-with-different-sizes/EndToEndTest.php | 8 ++++++++ .../test-classes-with-different-sizes/IntegrationTest.php | 8 ++++++++ .../test-classes-with-different-sizes/UnitTest.php | 8 ++++++++ .../execution-order/fixture/test-classes/BarTest.php | 8 ++++++++ .../execution-order/fixture/test-classes/FooTest.php | 8 ++++++++ .../fixture/test-methods-with-defects/FooTest.php | 8 ++++++++ .../fixture/test-methods-with-dependencies/FooTest.php | 8 ++++++++ .../test-methods-with-different-sizes/SizesTest.php | 8 ++++++++ 10 files changed, 80 insertions(+) diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BarTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BarTest.php index 8cbf188409e..d78e1c5d2aa 100644 --- a/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BarTest.php +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BarTest.php @@ -1,4 +1,12 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ namespace PHPUnit\TestFixture\ExecutionOrder\Defects; use PHPUnit\Framework\Attributes\CoversNothing; diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BazTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BazTest.php index 6b8091087af..4325f660cfb 100644 --- a/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BazTest.php +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BazTest.php @@ -1,4 +1,12 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ namespace PHPUnit\TestFixture\ExecutionOrder\Defects; use PHPUnit\Framework\Attributes\CoversNothing; diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/EndToEndTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/EndToEndTest.php index a5669e394e1..e44d7da4196 100644 --- a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/EndToEndTest.php +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/EndToEndTest.php @@ -1,4 +1,12 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ namespace PHPUnit\TestFixture\ExecutionOrder\DifferentSizes; use PHPUnit\Framework\Attributes\CoversNothing; diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/IntegrationTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/IntegrationTest.php index 6d083e0f927..912cceb656c 100644 --- a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/IntegrationTest.php +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/IntegrationTest.php @@ -1,4 +1,12 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ namespace PHPUnit\TestFixture\ExecutionOrder\DifferentSizes; use PHPUnit\Framework\Attributes\CoversNothing; diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/UnitTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/UnitTest.php index b88d8539804..04130056b04 100644 --- a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/UnitTest.php +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/UnitTest.php @@ -1,4 +1,12 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ namespace PHPUnit\TestFixture\ExecutionOrder\DifferentSizes; use PHPUnit\Framework\Attributes\CoversNothing; diff --git a/tests/end-to-end/execution-order/fixture/test-classes/BarTest.php b/tests/end-to-end/execution-order/fixture/test-classes/BarTest.php index aaf92e1e99e..d0294712f5d 100644 --- a/tests/end-to-end/execution-order/fixture/test-classes/BarTest.php +++ b/tests/end-to-end/execution-order/fixture/test-classes/BarTest.php @@ -1,4 +1,12 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ namespace PHPUnit\TestFixture\ExecutionOrder; use PHPUnit\Framework\Attributes\CoversNothing; diff --git a/tests/end-to-end/execution-order/fixture/test-classes/FooTest.php b/tests/end-to-end/execution-order/fixture/test-classes/FooTest.php index 9ad8aa5076d..e4180f5461f 100644 --- a/tests/end-to-end/execution-order/fixture/test-classes/FooTest.php +++ b/tests/end-to-end/execution-order/fixture/test-classes/FooTest.php @@ -1,4 +1,12 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ namespace PHPUnit\TestFixture\ExecutionOrder; use PHPUnit\Framework\Attributes\CoversNothing; diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-defects/FooTest.php b/tests/end-to-end/execution-order/fixture/test-methods-with-defects/FooTest.php index b224c010ac1..7617de2e0bb 100644 --- a/tests/end-to-end/execution-order/fixture/test-methods-with-defects/FooTest.php +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-defects/FooTest.php @@ -1,4 +1,12 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ namespace PHPUnit\TestFixture\ExecutionOrder\Defects; use PHPUnit\Framework\Attributes\CoversNothing; diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-dependencies/FooTest.php b/tests/end-to-end/execution-order/fixture/test-methods-with-dependencies/FooTest.php index c2b74b4b560..d0111c51d45 100644 --- a/tests/end-to-end/execution-order/fixture/test-methods-with-dependencies/FooTest.php +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-dependencies/FooTest.php @@ -1,4 +1,12 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ namespace PHPUnit\TestFixture\ExecutionOrder\Dependencies; use PHPUnit\Framework\Attributes\CoversNothing; diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-different-sizes/SizesTest.php b/tests/end-to-end/execution-order/fixture/test-methods-with-different-sizes/SizesTest.php index 401a401a791..640689781d7 100644 --- a/tests/end-to-end/execution-order/fixture/test-methods-with-different-sizes/SizesTest.php +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-different-sizes/SizesTest.php @@ -1,4 +1,12 @@ + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ namespace PHPUnit\TestFixture\ExecutionOrder\DifferentSizes; use PHPUnit\Framework\Attributes\CoversNothing; From 04a86f102f955988eadf27c01a470f1a0521aa83 Mon Sep 17 00:00:00 2001 From: Sebastian Bergmann Date: Wed, 24 Sep 2025 15:40:32 +0200 Subject: [PATCH 5/9] Add tests for ordering tests by duration --- .../test-classes-with-duration/BarTest.php | 38 ++++++++++ .../test-classes-with-duration/FooTest.php | 38 ++++++++++ .../test-classes-with-duration/test-results | 1 + .../test-methods-with-duration/FooTest.php | 38 ++++++++++ .../test-methods-with-duration/test-results | 1 + ...asses-but-result-cache-does-not-exist.phpt | 68 ++++++++++++++++++ .../order-by-duration-test-classes.phpt | 72 +++++++++++++++++++ ...thods-but-result-cache-does-not-exist.phpt | 51 +++++++++++++ .../order-by-duration-test-methods.phpt | 55 ++++++++++++++ 9 files changed, 362 insertions(+) create mode 100644 tests/end-to-end/execution-order/fixture/test-classes-with-duration/BarTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-classes-with-duration/FooTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-classes-with-duration/test-results create mode 100644 tests/end-to-end/execution-order/fixture/test-methods-with-duration/FooTest.php create mode 100644 tests/end-to-end/execution-order/fixture/test-methods-with-duration/test-results create mode 100644 tests/end-to-end/execution-order/order-by-duration-test-classes-but-result-cache-does-not-exist.phpt create mode 100644 tests/end-to-end/execution-order/order-by-duration-test-classes.phpt create mode 100644 tests/end-to-end/execution-order/order-by-duration-test-methods-but-result-cache-does-not-exist.phpt create mode 100644 tests/end-to-end/execution-order/order-by-duration-test-methods.phpt diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-duration/BarTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-duration/BarTest.php new file mode 100644 index 00000000000..7639d2ba51d --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-duration/BarTest.php @@ -0,0 +1,38 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\Duration; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\TestCase; + +#[CoversNothing] +final class BarTest extends TestCase +{ + public function testOne(): void + { + // sleep(6); + + $this->assertTrue(true); + } + + public function testTwo(): void + { + // sleep(1); + + $this->assertTrue(true); + } + + public function testThree(): void + { + // sleep(5); + + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-duration/FooTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-duration/FooTest.php new file mode 100644 index 00000000000..52a6244dc1d --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-duration/FooTest.php @@ -0,0 +1,38 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\Duration; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\TestCase; + +#[CoversNothing] +final class FooTest extends TestCase +{ + public function testOne(): void + { + // sleep(3); + + $this->assertTrue(true); + } + + public function testTwo(): void + { + // sleep(4); + + $this->assertTrue(true); + } + + public function testThree(): void + { + // sleep(2); + + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-duration/test-results b/tests/end-to-end/execution-order/fixture/test-classes-with-duration/test-results new file mode 100644 index 00000000000..088148bd071 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-duration/test-results @@ -0,0 +1 @@ +{"version":2,"defects":[],"times":{"PHPUnit\\TestFixture\\ExecutionOrder\\Duration\\BarTest::testOne":6.013,"PHPUnit\\TestFixture\\ExecutionOrder\\Duration\\BarTest::testTwo":1.001,"PHPUnit\\TestFixture\\ExecutionOrder\\Duration\\BarTest::testThree":5.001,"PHPUnit\\TestFixture\\ExecutionOrder\\Duration\\FooTest::testOne":3.001,"PHPUnit\\TestFixture\\ExecutionOrder\\Duration\\FooTest::testTwo":4.002,"PHPUnit\\TestFixture\\ExecutionOrder\\Duration\\FooTest::testThree":2.001}} \ No newline at end of file diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-duration/FooTest.php b/tests/end-to-end/execution-order/fixture/test-methods-with-duration/FooTest.php new file mode 100644 index 00000000000..74235f240fe --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-duration/FooTest.php @@ -0,0 +1,38 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\Duration; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\TestCase; + +#[CoversNothing] +final class FooTest extends TestCase +{ + public function testOne(): void + { + // sleep(3); + + $this->assertTrue(true); + } + + public function testTwo(): void + { + // sleep(1); + + $this->assertTrue(true); + } + + public function testThree(): void + { + // sleep(2); + + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-duration/test-results b/tests/end-to-end/execution-order/fixture/test-methods-with-duration/test-results new file mode 100644 index 00000000000..81a447ed668 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-duration/test-results @@ -0,0 +1 @@ +{"version":2,"defects":[],"times":{"PHPUnit\\TestFixture\\ExecutionOrder\\Duration\\FooTest::testOne":3.013,"PHPUnit\\TestFixture\\ExecutionOrder\\Duration\\FooTest::testTwo":1.001,"PHPUnit\\TestFixture\\ExecutionOrder\\Duration\\FooTest::testThree":2.001}} \ No newline at end of file diff --git a/tests/end-to-end/execution-order/order-by-duration-test-classes-but-result-cache-does-not-exist.phpt b/tests/end-to-end/execution-order/order-by-duration-test-classes-but-result-cache-does-not-exist.phpt new file mode 100644 index 00000000000..7c433de7a61 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-duration-test-classes-but-result-cache-does-not-exist.phpt @@ -0,0 +1,68 @@ +--TEST-- +Order by duration (without result cache) +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (6 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (6 tests) +Test Suite Started (CLI Arguments, 6 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Suite Finished (CLI Arguments, 6 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-duration-test-classes.phpt b/tests/end-to-end/execution-order/order-by-duration-test-classes.phpt new file mode 100644 index 00000000000..6d6005c03a5 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-duration-test-classes.phpt @@ -0,0 +1,72 @@ +--TEST-- +Order by duration (with result cache) +--FILE-- +run($_SERVER['argv']); + +unlink($testResultsFile); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (6 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (6 tests) +Test Suite Started (CLI Arguments, 6 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Suite Finished (CLI Arguments, 6 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-duration-test-methods-but-result-cache-does-not-exist.phpt b/tests/end-to-end/execution-order/order-by-duration-test-methods-but-result-cache-does-not-exist.phpt new file mode 100644 index 00000000000..939af0c6fd1 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-duration-test-methods-but-result-cache-does-not-exist.phpt @@ -0,0 +1,51 @@ +--TEST-- +Order by duration (without result cache) +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-duration-test-methods.phpt b/tests/end-to-end/execution-order/order-by-duration-test-methods.phpt new file mode 100644 index 00000000000..ad3ae1b3ddf --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-duration-test-methods.phpt @@ -0,0 +1,55 @@ +--TEST-- +Order by duration (with result cache) +--FILE-- +run($_SERVER['argv']); + +unlink($testResultsFile); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) From 2a70efd8747ef660d9350404c5d67404c1772970 Mon Sep 17 00:00:00 2001 From: Sebastian Bergmann Date: Wed, 24 Sep 2025 15:52:47 +0200 Subject: [PATCH 6/9] We should be able to assume that every Test is also a Reorderable --- src/Runner/TestSuiteSorter.php | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Runner/TestSuiteSorter.php b/src/Runner/TestSuiteSorter.php index 077893108bb..76043b43ba3 100644 --- a/src/Runner/TestSuiteSorter.php +++ b/src/Runner/TestSuiteSorter.php @@ -13,6 +13,7 @@ use function array_merge; use function array_reverse; use function array_splice; +use function assert; use function count; use function in_array; use function max; @@ -152,9 +153,7 @@ private function addSuiteToDefectSortOrder(TestSuite $suite): void $max = 0; foreach ($suite->tests() as $test) { - if (!$test instanceof Reorderable) { - continue; - } + assert($test instanceof Reorderable); if (!isset($this->defectSortOrder[$test->sortId()])) { $this->defectSortOrder[$test->sortId()] = $this->cache->status($test->sortId())->asInt(); @@ -216,9 +215,8 @@ private function sortBySize(array $tests): array */ private function cmpDefectPriorityAndTime(Test $a, Test $b): int { - if (!($a instanceof Reorderable && $b instanceof Reorderable)) { - return 0; - } + assert($a instanceof Reorderable); + assert($b instanceof Reorderable); $priorityA = $this->defectSortOrder[$a->sortId()] ?? 0; $priorityB = $this->defectSortOrder[$b->sortId()] ?? 0; @@ -241,9 +239,8 @@ private function cmpDefectPriorityAndTime(Test $a, Test $b): int */ private function cmpDuration(Test $a, Test $b): int { - if (!($a instanceof Reorderable && $b instanceof Reorderable)) { - return 0; - } + assert($a instanceof Reorderable); + assert($b instanceof Reorderable); return $this->cache->time($a->sortId()) <=> $this->cache->time($b->sortId()); } From 75833985ba00871d7fc959245cde0ee6407f637c Mon Sep 17 00:00:00 2001 From: Sebastian Bergmann Date: Wed, 24 Sep 2025 16:07:47 +0200 Subject: [PATCH 7/9] Ignore unhappy paths that will become obsolete after refactoring to enumeration (for $order parameter) and boolean (for $orderDefects parameter) --- src/Runner/TestSuiteSorter.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Runner/TestSuiteSorter.php b/src/Runner/TestSuiteSorter.php index 76043b43ba3..b332c4312fa 100644 --- a/src/Runner/TestSuiteSorter.php +++ b/src/Runner/TestSuiteSorter.php @@ -96,7 +96,9 @@ public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDepend ]; if (!in_array($order, $allowedOrders, true)) { + // @codeCoverageIgnoreStart throw new InvalidOrderException; + // @codeCoverageIgnoreEnd } $allowedOrderDefects = [ @@ -105,7 +107,9 @@ public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDepend ]; if (!in_array($orderDefects, $allowedOrderDefects, true)) { + // @codeCoverageIgnoreStart throw new InvalidOrderException; + // @codeCoverageIgnoreEnd } if ($suite instanceof TestSuite) { From baf4e5cf19f6d5807a0072159e7d0d0985ff7607 Mon Sep 17 00:00:00 2001 From: Sebastian Bergmann Date: Wed, 24 Sep 2025 16:15:18 +0200 Subject: [PATCH 8/9] Add test for empty test suite --- .../execution-order/empty-test-suite.phpt | 24 +++++++++++++++++++ .../fixture/empty-test-suite/FooTest.php | 16 +++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 tests/end-to-end/execution-order/empty-test-suite.phpt create mode 100644 tests/end-to-end/execution-order/fixture/empty-test-suite/FooTest.php diff --git a/tests/end-to-end/execution-order/empty-test-suite.phpt b/tests/end-to-end/execution-order/empty-test-suite.phpt new file mode 100644 index 00000000000..76b75c61e5d --- /dev/null +++ b/tests/end-to-end/execution-order/empty-test-suite.phpt @@ -0,0 +1,24 @@ +--TEST-- +Empty test suite +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Runner Triggered Warning (No tests found in class "PHPUnit\TestFixture\ExecutionOrder\EmptyTestSuite\FooTest".) +Test Suite Loaded (0 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (0 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 1) diff --git a/tests/end-to-end/execution-order/fixture/empty-test-suite/FooTest.php b/tests/end-to-end/execution-order/fixture/empty-test-suite/FooTest.php new file mode 100644 index 00000000000..b322b22fc4f --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/empty-test-suite/FooTest.php @@ -0,0 +1,16 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\EmptyTestSuite; + +use PHPUnit\Framework\TestCase; + +final class FooTest extends TestCase +{ +} From 65f669f2aecffd9dc7c5b96bd162d1703fa3e51c Mon Sep 17 00:00:00 2001 From: Sebastian Bergmann Date: Wed, 24 Sep 2025 16:29:16 +0200 Subject: [PATCH 9/9] Add tests for randomized execution order --- .../order-by-random-test-classes.phpt | 63 +++++++++++++++++++ .../order-by-random-test-methods.phpt | 44 +++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 tests/end-to-end/execution-order/order-by-random-test-classes.phpt create mode 100644 tests/end-to-end/execution-order/order-by-random-test-methods.phpt diff --git a/tests/end-to-end/execution-order/order-by-random-test-classes.phpt b/tests/end-to-end/execution-order/order-by-random-test-classes.phpt new file mode 100644 index 00000000000..b971c731fe7 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-random-test-classes.phpt @@ -0,0 +1,63 @@ +--TEST-- +Random order: Suite with test classes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (6 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (6 tests) +Test Suite Started (CLI Arguments, 6 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\BarTest, 3 tests) +Test Suite Finished (CLI Arguments, 6 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-random-test-methods.phpt b/tests/end-to-end/execution-order/order-by-random-test-methods.phpt new file mode 100644 index 00000000000..5918240c4b9 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-random-test-methods.phpt @@ -0,0 +1,44 @@ +--TEST-- +Random order: Suite with test classes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Duration\FooTest, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0)