From 930525d9e6989c497d7b3d72e75791447155ce98 Mon Sep 17 00:00:00 2001 From: Lance Willett Date: Sun, 16 Aug 2026 14:30:48 -0700 Subject: [PATCH 1/3] Build/Test Tools: Publish scheduled PHPUnit timing metrics --- .github/workflows/phpunit-tests.yml | 1 + .../workflows/reusable-phpunit-tests-v3.yml | 25 +++++ .../class-wp-phpunit-timing-metrics.php | 88 +++++++++++++++++ tests/phpunit/prepare-timing-results.php | 25 +++++ .../tests/includes/junitTimingMetrics.php | 97 +++++++++++++++++++ 5 files changed, 236 insertions(+) create mode 100644 tests/phpunit/includes/class-wp-phpunit-timing-metrics.php create mode 100644 tests/phpunit/prepare-timing-results.php create mode 100644 tests/phpunit/tests/includes/junitTimingMetrics.php diff --git a/.github/workflows/phpunit-tests.yml b/.github/workflows/phpunit-tests.yml index f0f1282f73a93..45e1439c9b604 100644 --- a/.github/workflows/phpunit-tests.yml +++ b/.github/workflows/phpunit-tests.yml @@ -81,6 +81,7 @@ jobs: permissions: contents: read secrets: + CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} WPT_REPORT_API_KEY: ${{ secrets.WPT_REPORT_API_KEY }} if: | diff --git a/.github/workflows/reusable-phpunit-tests-v3.yml b/.github/workflows/reusable-phpunit-tests-v3.yml index 4f7addcd4e60e..a1f544714492f 100644 --- a/.github/workflows/reusable-phpunit-tests-v3.yml +++ b/.github/workflows/reusable-phpunit-tests-v3.yml @@ -83,6 +83,9 @@ on: type: string default: '' secrets: + CODEVITALS_PROJECT_TOKEN: + description: 'The authorization token for publishing results to CodeVitals.' + required: false CODECOV_TOKEN: description: 'The Codecov token required for uploading reports.' required: false @@ -123,6 +126,7 @@ jobs: # - Logs debug information about what's installed within the WordPress Docker containers. # - Install WordPress within the Docker container. # - Run the PHPUnit tests. + # - Publish PHPUnit timing metrics to CodeVitals. # - Upload the code coverage report to Codecov.io. # - Ensures version-controlled files are not modified or deleted. # - Checks out the WordPress Test reporter repository. @@ -269,6 +273,27 @@ jobs: TEST_GROUPS: ${{ inputs.phpunit-test-groups }} MULTISITE_FLAG: ${{ inputs.multisite && 'multisite' || 'single' }} + - name: Publish PHPUnit timing metrics + if: ${{ github.event_name == 'schedule' && inputs.php == '8.5' && inputs.report }} + env: + CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} + run: | + if [ -z "$CODEVITALS_PROJECT_TOKEN" ]; then + echo "PHPUnit timing metrics could not be published. 'CODEVITALS_PROJECT_TOKEN' is not set" + exit 1 + fi + COMMITTED_AT="$(git show -s "$GITHUB_SHA" --format='%cI')" + php tests/phpunit/prepare-timing-results.php \ + tests/phpunit/build/logs/junit.xml \ + trunk \ + "$GITHUB_SHA" \ + "$COMMITTED_AT" \ + | curl --fail-with-body --silent --show-error \ + --request POST \ + --header 'Content-Type: application/json' \ + --data-binary @- \ + "https://codevitals.run/api/log?token=${CODEVITALS_PROJECT_TOKEN}" + - name: Run AJAX tests if: ${{ ! inputs.phpunit-test-groups && ! inputs.coverage-report }} continue-on-error: ${{ inputs.allow-errors }} diff --git a/tests/phpunit/includes/class-wp-phpunit-timing-metrics.php b/tests/phpunit/includes/class-wp-phpunit-timing-metrics.php new file mode 100644 index 0000000000000..1276154a32854 --- /dev/null +++ b/tests/phpunit/includes/class-wp-phpunit-timing-metrics.php @@ -0,0 +1,88 @@ + Timing metrics keyed for CodeVitals. + * @throws RuntimeException If the file cannot be read or contains invalid timing data. + */ + public static function from_file( $file ) { + if ( ! is_readable( $file ) ) { + throw new RuntimeException( 'The JUnit timing report could not be read.' ); + } + + $reader = new XMLReader(); + if ( ! $reader->open( $file, null, LIBXML_NONET | LIBXML_COMPACT ) ) { + throw new RuntimeException( 'The JUnit timing report could not be opened.' ); + } + + $suite_time = null; + $test_times = array(); + + while ( $reader->read() ) { + if ( XMLReader::ELEMENT !== $reader->nodeType ) { + continue; + } + + if ( null === $suite_time && 'testsuite' === $reader->name ) { + $time = $reader->getAttribute( 'time' ); + if ( is_numeric( $time ) ) { + $suite_time = (float) $time; + } + continue; + } + + if ( 'testcase' !== $reader->name ) { + continue; + } + + $time = $reader->getAttribute( 'time' ); + if ( ! is_numeric( $time ) ) { + $reader->close(); + throw new RuntimeException( 'A JUnit testcase is missing numeric timing data.' ); + } + + $test_times[] = (float) $time; + } + + $reader->close(); + + if ( ! $test_times ) { + throw new RuntimeException( 'The JUnit timing report contains no testcases.' ); + } + + if ( null === $suite_time ) { + $suite_time = array_sum( $test_times ); + } + + sort( $test_times, SORT_NUMERIC ); + + return array( + 'phpunit-suite-time' => round( $suite_time, 6 ), + 'phpunit-p95-test-time' => round( self::percentile( $test_times, 0.95 ) * 1000, 3 ), + 'phpunit-p99-test-time' => round( self::percentile( $test_times, 0.99 ) * 1000, 3 ), + 'phpunit-max-test-time' => round( max( $test_times ) * 1000, 3 ), + 'phpunit-tests-over-500ms' => count( array_filter( $test_times, static fn ( $time ) => $time > 0.5 ) ), + 'phpunit-tests-over-1s' => count( array_filter( $test_times, static fn ( $time ) => $time > 1 ) ), + ); + } + + /** + * Calculates a nearest-rank percentile from a sorted list. + * + * @param float[] $values Sorted values. + * @param float $percentile Percentile between zero and one. + * @return float Percentile value. + */ + private static function percentile( $values, $percentile ) { + $index = (int) ceil( $percentile * count( $values ) ) - 1; + + return $values[ max( 0, $index ) ]; + } +} diff --git a/tests/phpunit/prepare-timing-results.php b/tests/phpunit/prepare-timing-results.php new file mode 100644 index 0000000000000..912f90632347e --- /dev/null +++ b/tests/phpunit/prepare-timing-results.php @@ -0,0 +1,25 @@ +#!/usr/bin/env php + \n" ); + exit( 1 ); +} + +try { + $timestamp = new DateTimeImmutable( $argv[4] ); + $payload = array( + 'branch' => $argv[2], + 'hash' => $argv[3], + 'baseHash' => $argv[3], + 'timestamp' => $timestamp->format( DATE_ATOM ), + 'metrics' => WP_PHPUnit_Timing_Metrics::from_file( $argv[1] ), + ); + + echo json_encode( $payload, JSON_THROW_ON_ERROR ) . "\n"; +} catch ( Throwable $error ) { + fwrite( STDERR, $error->getMessage() . "\n" ); + exit( 1 ); +} diff --git a/tests/phpunit/tests/includes/junitTimingMetrics.php b/tests/phpunit/tests/includes/junitTimingMetrics.php new file mode 100644 index 0000000000000..1dc91350ea64d --- /dev/null +++ b/tests/phpunit/tests/includes/junitTimingMetrics.php @@ -0,0 +1,97 @@ +temporary_files as $file ) { + unlink( $file ); + } + + parent::tear_down(); + } + + public function test_extracts_aggregate_timing_metrics() { + $times = array_map( + static fn ( $millisecond ) => $millisecond / 1000, + range( 1, 100 ) + ); + $file = $this->create_junit_file( $times, 5.05 ); + + $this->assertSame( + array( + 'phpunit-suite-time' => 5.05, + 'phpunit-p95-test-time' => 95.0, + 'phpunit-p99-test-time' => 99.0, + 'phpunit-max-test-time' => 100.0, + 'phpunit-tests-over-500ms' => 0, + 'phpunit-tests-over-1s' => 0, + ), + WP_PHPUnit_Timing_Metrics::from_file( $file ) + ); + } + + public function test_counts_only_tests_above_slow_test_thresholds() { + $file = $this->create_junit_file( array( 0.5, 0.500001, 1.0, 1.000001 ), 3.000002 ); + + $metrics = WP_PHPUnit_Timing_Metrics::from_file( $file ); + + $this->assertSame( 3, $metrics['phpunit-tests-over-500ms'] ); + $this->assertSame( 1, $metrics['phpunit-tests-over-1s'] ); + } + + public function test_uses_testcase_time_when_suite_time_is_missing() { + $file = $this->create_junit_file( array( 0.1, 0.2, 0.3 ) ); + + $metrics = WP_PHPUnit_Timing_Metrics::from_file( $file ); + + $this->assertSame( 0.6, $metrics['phpunit-suite-time'] ); + } + + public function test_rejects_report_without_testcases() { + $file = $this->create_junit_file( array(), 0.0 ); + + $this->expectException( RuntimeException::class ); + $this->expectExceptionMessage( 'The JUnit timing report contains no testcases.' ); + + WP_PHPUnit_Timing_Metrics::from_file( $file ); + } + + /** + * Creates a JUnit XML file for a test. + * + * @param float[] $times Testcase times in seconds. + * @param float|int $suite_time Optional testsuite time in seconds. + * @return string Path to the temporary file. + */ + private function create_junit_file( $times, $suite_time = null ) { + $file = tempnam( sys_get_temp_dir(), 'junit-timing-' ); + $this->temporary_files[] = $file; + $suite_time_attribute = null === $suite_time ? '' : sprintf( ' time="%s"', $suite_time ); + $testcases = ''; + + foreach ( $times as $index => $time ) { + $testcases .= sprintf( '', $index, $time ); + } + + file_put_contents( + $file, + sprintf( '%3$s', count( $times ), $suite_time_attribute, $testcases ) + ); + + return $file; + } +} From 7c4f94b110f9e9085dd619c72191efe1b536a955 Mon Sep 17 00:00:00 2001 From: Lance Willett Date: Sun, 16 Aug 2026 14:34:46 -0700 Subject: [PATCH 2/3] Build/Test Tools: Verify CodeVitals accepts timing metrics --- .../workflows/reusable-phpunit-tests-v3.yml | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/reusable-phpunit-tests-v3.yml b/.github/workflows/reusable-phpunit-tests-v3.yml index a1f544714492f..d371e5ab268c2 100644 --- a/.github/workflows/reusable-phpunit-tests-v3.yml +++ b/.github/workflows/reusable-phpunit-tests-v3.yml @@ -283,16 +283,23 @@ jobs: exit 1 fi COMMITTED_AT="$(git show -s "$GITHUB_SHA" --format='%cI')" - php tests/phpunit/prepare-timing-results.php \ - tests/phpunit/build/logs/junit.xml \ - trunk \ - "$GITHUB_SHA" \ - "$COMMITTED_AT" \ - | curl --fail-with-body --silent --show-error \ - --request POST \ - --header 'Content-Type: application/json' \ - --data-binary @- \ - "https://codevitals.run/api/log?token=${CODEVITALS_PROJECT_TOKEN}" + RESPONSE="$( + php tests/phpunit/prepare-timing-results.php \ + tests/phpunit/build/logs/junit.xml \ + trunk \ + "$GITHUB_SHA" \ + "$COMMITTED_AT" \ + | curl --fail-with-body --silent --show-error \ + --request POST \ + --header 'Content-Type: application/json' \ + --data-binary @- \ + "https://codevitals.run/api/log?token=${CODEVITALS_PROJECT_TOKEN}" + )" + if ! jq --exit-status '.status == "ok" and .count == 6' <<< "$RESPONSE" > /dev/null; then + echo 'CodeVitals did not accept all six PHPUnit timing metrics.' + exit 1 + fi + echo 'Published six PHPUnit timing metrics to CodeVitals.' - name: Run AJAX tests if: ${{ ! inputs.phpunit-test-groups && ! inputs.coverage-report }} From bfd963734e36ce76100679536b3271f807aaf957 Mon Sep 17 00:00:00 2001 From: Lance Willett Date: Sun, 16 Aug 2026 17:48:09 -0700 Subject: [PATCH 3/3] Build/Test Tools: Address PHPUnit metrics review --- .github/workflows/reusable-phpunit-tests-v3.yml | 6 +++++- tests/phpunit/prepare-timing-results.php | 7 +++++++ tests/phpunit/tests/includes/junitTimingMetrics.php | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-phpunit-tests-v3.yml b/.github/workflows/reusable-phpunit-tests-v3.yml index d371e5ab268c2..4ee981320d15f 100644 --- a/.github/workflows/reusable-phpunit-tests-v3.yml +++ b/.github/workflows/reusable-phpunit-tests-v3.yml @@ -274,7 +274,11 @@ jobs: MULTISITE_FLAG: ${{ inputs.multisite && 'multisite' || 'single' }} - name: Publish PHPUnit timing metrics - if: ${{ github.event_name == 'schedule' && inputs.php == '8.5' && inputs.report }} + if: | + github.event_name == 'push' && + github.ref == 'refs/heads/trunk' && + inputs.php == '8.5' && + inputs.report env: CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} run: | diff --git a/tests/phpunit/prepare-timing-results.php b/tests/phpunit/prepare-timing-results.php index 912f90632347e..d379c6ff7e685 100644 --- a/tests/phpunit/prepare-timing-results.php +++ b/tests/phpunit/prepare-timing-results.php @@ -1,6 +1,13 @@ #!/usr/bin/env php