diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5878a1c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.git +.github +.cache +.env* +vendor +output +wp-test-runner diff --git a/.github/workflows/container-tests-report.yml b/.github/workflows/container-tests-report.yml new file mode 100644 index 0000000..8c439d9 --- /dev/null +++ b/.github/workflows/container-tests-report.yml @@ -0,0 +1,141 @@ +name: Container Tests Report + +on: + # Runs after the Container Tests workflow completes, in the context of the + # base repository. This grants the permission to comment on pull requests + # opened from forks, which the `pull_request` event does not. + # + # The artifacts of the completed run were produced by pull request code and + # must be treated as untrusted. They are only read as text: no code from the + # pull request is checked out or executed here. + workflow_run: + workflows: + - Container Tests + types: + - completed + +permissions: {} + +jobs: + # Posts the summaries of the Container Tests jobs as a pull request comment. + # + # Performs the following steps: + # - Downloads the results artifacts of the completed run. + # - Finds the open pull request for the run's head commit. + # - Creates the comment, or updates the one posted for an earlier run. + comment: + name: Comment on the pull request + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + pull-requests: write + if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion != 'cancelled' }} + + steps: + - name: Download the results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + # A run that failed before uploading anything is still reported. + continue-on-error: true + with: + pattern: results-* + path: results + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Post the summary + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + retries: 2 + script: | + const fs = require( 'fs' ); + const path = require( 'path' ); + + const run = context.payload.workflow_run; + const marker = ''; + // GitHub rejects comments longer than 65536 characters. + const maxLength = 60000; + + // Look the pull request up from the run itself rather than from the + // artifacts, so pull request code cannot redirect the comment. + const pulls = await github.paginate( github.rest.pulls.list, { + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + } ); + // The head repository is null when the fork has been deleted. + const headRepo = run.head_repository ? run.head_repository.full_name : ''; + const pull = pulls.find( + ( pr ) => + pr.head.sha === run.head_sha && + pr.head.repo && + pr.head.repo.full_name === headRepo + ); + if ( ! pull ) { + core.info( `No open pull request found for ${ headRepo || 'a deleted repository' }@${ run.head_sha }.` ); + return; + } + + const summaries = []; + if ( fs.existsSync( 'results' ) ) { + for ( const dir of fs.readdirSync( 'results' ).sort() ) { + const file = path.join( 'results', dir, 'summary.md' ); + // Only read regular files from the untrusted artifacts. + if ( fs.existsSync( file ) && fs.lstatSync( file ).isFile() ) { + summaries.push( fs.readFileSync( file, 'utf8' ).trim() ); + } + } + } + + const icons = { success: '✅', failure: '❌', timed_out: '⏱️' }; + const icon = icons[ run.conclusion ] || 'ℹ️'; + let body = [ + marker, + `# ${ icon } Container Tests: ${ run.conclusion }`, + '', + `Results for ${ run.head_sha.substring( 0, 7 ) } from [workflow run #${ run.run_number }](${ run.html_url }). Logs, \`junit.xml\`, and \`env.json\` are attached to the run as artifacts.`, + '', + summaries.length + ? summaries.join( '\n\n---\n\n' ) + : '_No results were uploaded. Check the workflow run for errors in building the image or starting the containers._', + ].join( '\n' ); + + // Keep mentions in the untrusted summaries from notifying anyone. + body = body.replace( /@(?=[A-Za-z0-9])/g, '@​' ); + + if ( body.length > maxLength ) { + body = body.substring( 0, maxLength ) + `\n\n_The summary was truncated. See the [workflow run](${ run.html_url }) for the full results._`; + } + + const comments = await github.paginate( github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pull.number, + per_page: 100, + } ); + const existing = comments.find( + ( comment ) => + comment.user && + comment.user.login === 'github-actions[bot]' && + comment.body && + comment.body.startsWith( marker ) + ); + + if ( existing ) { + await github.rest.issues.updateComment( { + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + } ); + core.info( `Updated comment ${ existing.html_url }` ); + } else { + const { data } = await github.rest.issues.createComment( { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pull.number, + body, + } ); + core.info( `Created comment ${ data.html_url }` ); + } diff --git a/.github/workflows/container-tests.yml b/.github/workflows/container-tests.yml new file mode 100644 index 0000000..0001d41 --- /dev/null +++ b/.github/workflows/container-tests.yml @@ -0,0 +1,89 @@ +name: Container Tests + +on: + push: + branches: + - master + pull_request: + branches: + - master + paths: + - '**.php' + - 'compose.yaml' + - 'docker/**' + - '.dockerignore' + - '.github/workflows/container-tests.yml' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +permissions: {} + +jobs: + # Builds the runner image and runs prepare.php, test.php, report.php, and + # cleanup.php against a database container, mimicking a hosting environment. + # + # Each step's output, junit.xml, env.json, and a Markdown summary are + # uploaded as an artifact, and the summary is shown on the workflow run. + # + # The job fails when a runner step fails, not when WordPress tests fail. + test: + name: PHP ${{ matrix.php }} / ${{ matrix.db }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: read + + strategy: + fail-fast: false + matrix: + include: + - php: '8.3' + db: 'mysql:8.4' + - php: '8.1' + db: 'mariadb:10.6' + + env: + PHP_VERSION: ${{ matrix.php }} + WPT_DB_IMAGE: ${{ matrix.db }} + # Failing WordPress tests are shown in the summary, but only problems with + # the runner itself, or a test run without junit.xml results, fail the job. + WPT_IGNORE_TEST_FAILURES: 1 + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + persist-credentials: false + + - name: Build the runner image + run: WPT_UID="$(id -u)" docker compose build runner + + - name: Run the test runner + run: | + mkdir -p output + docker compose up --abort-on-container-exit --exit-code-from runner + + - name: Add the summary to the job + if: ${{ always() }} + run: | + if [ -f output/summary.md ]; then + cat output/summary.md >> "$GITHUB_STEP_SUMMARY" + else + echo "The runner did not produce a summary." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload the results + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: results-php${{ matrix.php }}-${{ strategy.job-index }} + path: output/ + if-no-files-found: warn + + - name: Remove the containers + if: ${{ always() }} + run: docker compose down --volumes diff --git a/.gitignore b/.gitignore index 023e69a..6fdfc56 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ wp-test-runner/ *.orig *.patch *.diff + +# Results of containerized runs. +output/ diff --git a/README.md b/README.md index 6e8b015..11a4f48 100644 --- a/README.md +++ b/README.md @@ -633,6 +633,38 @@ journalctl -u testrunner.timer journalctl -n 120 -u testrunner.service ``` +### Docker + +The runner can also be run in containers, with a disposable database, using [Docker Compose](https://docs.docker.com/compose/). The image in [`docker/Dockerfile`](docker/Dockerfile) installs PHP with the extensions the test suite uses, Composer, Node.js, Git, rsync, and an SSH client, so nothing needs to be installed on the host besides Docker. + +```bash +mkdir -p output +WPT_UID=$(id -u) docker compose up --build --abort-on-container-exit --exit-code-from runner +docker compose down --volumes +``` + +This runs `prepare.php`, `test.php`, `report.php`, and `cleanup.php` in order. The test and report steps are skipped when preparing the environment fails, and the cleanup step always runs. The command exits with the exit code of the first step that failed. + +When the run finishes, the `output/` directory contains: + +- `prepare.log`, `test.log`, `report.log`, and `cleanup.log`: the output of each step. +- `junit.xml` and `env.json`: the test results and environment details. +- `summary.md`: a Markdown summary of the steps, the test results, and any failures. + +The PHP and database versions are chosen with environment variables. Any variable from [`.env.default`](.env.default) that is listed in [`compose.yaml`](compose.yaml) is passed through to the runner, so setting `WPT_REPORT_API_KEY` reports the results to WordPress.org. + +```bash +PHP_VERSION=8.1 WPT_DB_IMAGE=mariadb:10.6 docker compose up --build --abort-on-container-exit --exit-code-from runner +``` + +`PHP_VERSION` is any tag of the official [`php`](https://hub.docker.com/_/php) image. The PHP 7.x images are built on Debian releases that no longer receive updates, so the image may fail to build for them. + +Set `WPT_IGNORE_TEST_FAILURES=1` to exit with 0 when WordPress tests fail but every runner step worked. A test run that produces no `junit.xml` results still fails. + +To open a shell in the runner image instead, use `docker compose run --rm runner bash`. + +The [Container Tests](.github/workflows/container-tests.yml) workflow runs the same setup on every pull request. It shows the summary on the workflow run and uploads the `output/` directory as an artifact. The [Container Tests Report](.github/workflows/container-tests-report.yml) workflow then posts the summaries as a comment on the pull request, and updates that comment on later runs. + ## Contributing If you have questions about the process or run into test failures along the way, please [open an issue in the project repository](https://github.com/WordPress/phpunit-test-runner/issues) and we’ll help diagnose/get the documentation updated. Alternatively, you can also pop into the `#hosting` channel on [WordPress.org Slack](https://make.wordpress.org/chat/) for help. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..1b32f76 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,53 @@ +# Runs the test runner and a disposable database in containers. +# +# mkdir -p output +# docker compose up --build --abort-on-container-exit --exit-code-from runner +# docker compose down --volumes +# +# Results, logs, and a summary are written to ./output. +# +# The PHP and database versions are set with environment variables, for example: +# +# PHP_VERSION=8.1 WPT_DB_IMAGE=mariadb:10.6 docker compose up --build ... +# +# Any WPT_* variable from .env.default can be passed through the same way. + +services: + db: + image: ${WPT_DB_IMAGE:-mysql:8.0} + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: wordpress_test + MYSQL_USER: wp_test + MYSQL_PASSWORD: wp_test + # The database is destroyed after every run, so keep it in memory. + tmpfs: + - /var/lib/mysql + + runner: + build: + context: . + dockerfile: docker/Dockerfile + args: + PHP_VERSION: ${PHP_VERSION:-8.3} + NODE_VERSION: ${NODE_VERSION:-24} + WPT_UID: ${WPT_UID:-1000} + depends_on: + - db + environment: + WPT_DB_NAME: wordpress_test + WPT_DB_USER: wp_test + WPT_DB_PASSWORD: wp_test + WPT_DB_HOST: db + WPT_DB_LABEL: ${WPT_DB_IMAGE:-mysql:8.0} + WPT_TABLE_PREFIX: ${WPT_TABLE_PREFIX:-wptests_} + WPT_REPORT_API_KEY: ${WPT_REPORT_API_KEY:-} + WPT_REPORT_URL: ${WPT_REPORT_URL:-} + WPT_FLAVOR: ${WPT_FLAVOR:-0} + WPT_EXTRATESTS: ${WPT_EXTRATESTS:-0} + WPT_PHPUNIT_CMD: ${WPT_PHPUNIT_CMD:-} + WPT_DEBUG: ${WPT_DEBUG:-} + WPT_CERTIFICATE_VALIDATION: ${WPT_CERTIFICATE_VALIDATION:-1} + WPT_IGNORE_TEST_FAILURES: ${WPT_IGNORE_TEST_FAILURES:-} + volumes: + - ./output:/output diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..aef791a --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,79 @@ +# WordPress PHPUnit Test Runner container image. +# +# Bundles everything prepare.php, test.php, report.php, and cleanup.php need: +# PHP with the extensions WordPress tests exercise, Composer, Node.js/npm, +# Git, rsync, wget, unzip, and an SSH client for remote test environments. +# +# Build arguments: +# - PHP_VERSION: PHP version of the official `php` image to build on. +# - NODE_VERSION: Major Node.js version used to build wordpress-develop. +# - WPT_UID: UID of the non-root user that runs the tests. Match it to +# the host user when bind mounting the output directory. + +ARG PHP_VERSION=8.3 +ARG NODE_VERSION=24 + +FROM node:${NODE_VERSION}-bookworm-slim AS node + +FROM php:${PHP_VERSION}-cli + +ARG WPT_UID=1000 + +# System utilities used by the runner scripts. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + git \ + openssh-client \ + rsync \ + unzip \ + wget \ + && rm -rf /var/lib/apt/lists/* + +# PHP extensions used by the WordPress PHPUnit test suite. +# Imagick is built from its GitHub release because older PECL clients cannot +# read the package metadata pecl.php.net currently serves. +COPY --from=mlocati/php-extension-installer:2 /usr/bin/install-php-extensions /usr/local/bin/ +RUN install-php-extensions \ + bcmath \ + exif \ + gd \ + Imagick/imagick@3.8.1 \ + intl \ + mysqli \ + pdo_mysql \ + soap \ + sockets \ + zip \ + && { \ + echo 'memory_limit=1G'; \ + echo 'error_reporting=E_ALL'; \ + echo 'display_errors=On'; \ + } > /usr/local/etc/php/conf.d/wpt-runner.ini + +# Composer. +COPY --from=composer:2 /usr/bin/composer /usr/local/bin/composer + +# Node.js and npm. +COPY --from=node /usr/local/bin/node /usr/local/bin/node +COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules +RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \ + && ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx \ + && node --version \ + && npm --version + +# The test suite must not run as root, see https://core.trac.wordpress.org/ticket/44233. +RUN useradd --create-home --uid "${WPT_UID}" --shell /bin/bash wptests \ + && mkdir -p /output \ + && chown wptests:wptests /output + +WORKDIR /opt/phpunit-test-runner +COPY --chown=wptests:wptests . . + +USER wptests + +ENV WPT_PREPARE_DIR=/tmp/wp-test-runner \ + WPT_TEST_DIR=/tmp/wp-test-runner \ + WPT_OUTPUT_DIR=/output + +ENTRYPOINT ["/opt/phpunit-test-runner/docker/entrypoint.sh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..ce7217c --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# +# Runs the full test runner pipeline inside the container: +# +# prepare.php -> test.php -> report.php -> cleanup.php +# +# The output of every step is written to "$WPT_OUTPUT_DIR/.log", the +# junit.xml and env.json results are copied to "$WPT_OUTPUT_DIR", and a +# Markdown summary of the run is written to "$WPT_OUTPUT_DIR/summary.md". +# +# cleanup.php always runs. The test and report steps are skipped when the +# environment could not be prepared. +# +# The exit code is the exit code of the first step that failed, or 0. Set +# WPT_IGNORE_TEST_FAILURES=1 to only fail on test failures when no junit.xml +# results were produced, so failing WordPress tests do not fail the run. +# +# Pass a different command (for example `bash`) to run it instead. + +set -u + +if [ "$#" -gt 0 ]; then + exec "$@" +fi + +RUNNER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUTPUT_DIR="${WPT_OUTPUT_DIR:-/output}" +STATUS_FILE="${OUTPUT_DIR}/steps.tsv" + +mkdir -p "${OUTPUT_DIR}" +rm -f "${OUTPUT_DIR}"/*.log "${OUTPUT_DIR}/junit.xml" "${OUTPUT_DIR}/env.json" "${OUTPUT_DIR}/summary.md" +: > "${STATUS_FILE}" + +cd "${RUNNER_DIR}" || exit 1 + +exit_code=0 + +# Records the outcome of a step in the status file. +# +# $1 Step name. $2 Status (passed, failed, skipped). $3 Exit code. $4 Seconds. +record() { + printf '%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "$4" >> "${STATUS_FILE}" +} + +# Runs one of the runner scripts, saving its output and outcome. +# +# $1 Step name, matching the .php script. +# $2 Pass "optional" to keep a failure from setting the exit code. +run_step() { + local step="$1" + local optional="${2:-}" + local start=$SECONDS + local code + + echo "==> ${step}.php" + php "${step}.php" 2>&1 | tee "${OUTPUT_DIR}/${step}.log" + code=${PIPESTATUS[0]} + + if [ "${code}" -eq 0 ]; then + record "${step}" passed 0 $(( SECONDS - start )) + else + echo "Error: ${step}.php exited with code ${code}." >&2 + record "${step}" failed "${code}" $(( SECONDS - start )) + if [ "${exit_code}" -eq 0 ] && [ "${optional}" != optional ]; then + exit_code=${code} + fi + fi + + return "${code}" +} + +# Waits for the database server to accept connections with the configured credentials. +wait_for_database() { + local attempts="${WPT_DB_WAIT_ATTEMPTS:-60}" + local i + + echo "Waiting for the database at ${WPT_DB_HOST:-localhost} ..." + for (( i = 1; i <= attempts; i++ )); do + if php -r ' + mysqli_report( MYSQLI_REPORT_OFF ); + $host = getenv( "WPT_DB_HOST" ) ?: "localhost"; + $port = null; + if ( preg_match( "/^([^:]+):([0-9]+)$/", $host, $m ) ) { + $host = $m[1]; + $port = (int) $m[2]; + } + $db = @mysqli_connect( $host, getenv( "WPT_DB_USER" ), getenv( "WPT_DB_PASSWORD" ), getenv( "WPT_DB_NAME" ), $port ); + exit( $db ? 0 : 1 ); + '; then + echo "Database is ready." + return 0 + fi + sleep 2 + done + + echo "Error: the database did not become available after ${attempts} attempts." >&2 + return 1 +} + +# Copies junit.xml and env.json out of the prepare directory before cleanup removes it. +collect_results() { + local prepare_dir="${WPT_PREPARE_DIR:-/tmp/wp-test-runner}" + local logs_dir="${prepare_dir}/tests/phpunit/build/logs" + + if [ -f "${prepare_dir}/junit.xml" ]; then + cp "${prepare_dir}/junit.xml" "${OUTPUT_DIR}/junit.xml" + elif [ -f "${logs_dir}/junit.xml" ]; then + cp "${logs_dir}/junit.xml" "${OUTPUT_DIR}/junit.xml" + fi + + if [ -f "${logs_dir}/env.json" ]; then + cp "${logs_dir}/env.json" "${OUTPUT_DIR}/env.json" + fi +} + +if [ -z "${WPT_SSH_CONNECT:-}" ] && ! wait_for_database; then + record database failed 1 0 + record prepare skipped - 0 + record test skipped - 0 + record report skipped - 0 + exit_code=1 +elif run_step prepare; then + if [ -n "${WPT_IGNORE_TEST_FAILURES:-}" ] && [ "${WPT_IGNORE_TEST_FAILURES}" != 0 ]; then + run_step test optional + else + run_step test + fi + run_step report +else + record test skipped - 0 + record report skipped - 0 +fi + +collect_results + +if [ "${exit_code}" -eq 0 ] && grep -q $'^test\tfailed' "${STATUS_FILE}" && [ ! -s "${OUTPUT_DIR}/junit.xml" ]; then + echo "Error: the tests failed without producing junit.xml results." >&2 + exit_code=1 +fi + +run_step cleanup + +php "${RUNNER_DIR}/docker/summarize.php" "${OUTPUT_DIR}" > "${OUTPUT_DIR}/summary.md" +cat "${OUTPUT_DIR}/summary.md" + +exit "${exit_code}" diff --git a/docker/summarize.php b/docker/summarize.php new file mode 100644 index 0000000..6a70080 --- /dev/null +++ b/docker/summarize.php @@ -0,0 +1,223 @@ +.log: the output of each step. + * - junit.xml: the PHPUnit results, when the tests ran. + * + * The summary is printed to STDOUT so it can be used as a GitHub Actions job + * summary or as the body of a pull request comment. + * + * Usage: php docker/summarize.php + * + * @link https://github.com/wordpress/phpunit-test-runner/ Original source repository + * + * @package WordPress + */ + +// Maximum number of failed tests to list individually. +const WPT_SUMMARY_MAX_FAILURES = 50; + +// Number of trailing log lines shown for a failed step. +const WPT_SUMMARY_LOG_LINES = 40; + +$output_dir = isset( $argv[1] ) ? rtrim( $argv[1], '/' ) : '/output'; + +/** + * Reads the step outcomes recorded by the entrypoint. + * + * @param string $file Path to steps.tsv. + * + * @return array[] List of steps, each with name, status, code, and seconds. + */ +function wpt_summary_read_steps( $file ) { + $steps = array(); + if ( ! is_readable( $file ) ) { + return $steps; + } + + foreach ( file( $file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES ) as $line ) { + $parts = explode( "\t", $line ); + if ( 4 !== count( $parts ) ) { + continue; + } + $steps[] = array( + 'name' => $parts[0], + 'status' => $parts[1], + 'code' => $parts[2], + 'seconds' => (int) $parts[3], + ); + } + + return $steps; +} + +/** + * Collects the totals and failed tests from a junit.xml file. + * + * @param string $file Path to junit.xml. + * + * @return array|null Totals and failures, or null when the file is unusable. + */ +function wpt_summary_read_junit( $file ) { + if ( ! is_readable( $file ) || 0 === filesize( $file ) ) { + return null; + } + + libxml_use_internal_errors( true ); + $xml = simplexml_load_file( $file ); + if ( false === $xml ) { + return null; + } + + $suite = isset( $xml->testsuite ) ? $xml->testsuite[0] : $xml; + + $result = array( + 'tests' => (int) $suite['tests'], + 'failures' => (int) $suite['failures'], + 'errors' => (int) $suite['errors'], + 'skipped' => (int) $suite['skipped'], + 'time' => (float) $suite['time'], + 'failed' => array(), + ); + + foreach ( $xml->xpath( '//testcase[failure or error]' ) as $testcase ) { + $problem = isset( $testcase->failure ) ? $testcase->failure : $testcase->error; + $message = trim( (string) $problem['message'] ); + if ( '' === $message ) { + // PHPUnit puts the test name on the first line, followed by the assertion message. + $lines = array_values( array_filter( array_map( 'trim', explode( "\n", (string) $problem ) ) ) ); + $message = isset( $lines[1] ) ? $lines[1] : ( isset( $lines[0] ) ? $lines[0] : '' ); + } + + $result['failed'][] = array( + 'name' => (string) $testcase['class'] . '::' . (string) $testcase['name'], + 'message' => $message, + ); + } + + return $result; +} + +/** + * Returns the last lines of a file. + * + * @param string $file Path to the file. + * @param int $count Number of lines. + * + * @return string The trailing lines, or an empty string. + */ +function wpt_summary_tail( $file, $count ) { + if ( ! is_readable( $file ) ) { + return ''; + } + + $lines = file( $file, FILE_IGNORE_NEW_LINES ); + + return implode( "\n", array_slice( $lines, -$count ) ); +} + +/** + * Escapes a value for use inside a Markdown table cell. + * + * @param string $value The value. + * + * @return string The escaped value. + */ +function wpt_summary_cell( $value ) { + $value = preg_replace( '/\s+/', ' ', (string) $value ); + + return str_replace( '|', '\|', $value ); +} + +$steps = wpt_summary_read_steps( $output_dir . '/steps.tsv' ); +$junit = wpt_summary_read_junit( $output_dir . '/junit.xml' ); +$failed = array_filter( + $steps, + function ( $step ) { + return 'failed' === $step['status']; + } +); +$icons = array( + 'passed' => '✅', + 'failed' => '❌', + 'skipped' => '⏭️', +); + +$php_version = PHP_MAJOR_VERSION . '.' . PHP_MINOR_VERSION . '.' . PHP_RELEASE_VERSION; +$db_label = getenv( 'WPT_DB_LABEL' ); + +if ( empty( $failed ) ) { + $heading = '✅ passed'; +} elseif ( null !== $junit && array( 'test' ) === array_values( array_column( $failed, 'name' ) ) ) { + // Only the tests failed, and they produced results to look at. + $heading = '⚠️ runner passed, some tests failed'; +} else { + $heading = '❌ runner failed'; +} + +echo '## PHPUnit Test Runner: ' . $heading . "\n\n"; +echo '- PHP: `' . $php_version . "`\n"; +if ( $db_label ) { + echo '- Database: `' . $db_label . "`\n"; +} +echo "\n"; + +echo "| Step | Status | Exit code | Duration |\n"; +echo "| --- | --- | --- | --- |\n"; +foreach ( $steps as $step ) { + $icon = isset( $icons[ $step['status'] ] ) ? $icons[ $step['status'] ] : ''; + printf( + "| %s | %s %s | %s | %ds |\n", + wpt_summary_cell( $step['name'] ), + $icon, + wpt_summary_cell( $step['status'] ), + wpt_summary_cell( $step['code'] ), + $step['seconds'] + ); +} +echo "\n"; + +if ( null !== $junit ) { + echo "### Test results\n\n"; + echo "| Tests | Failures | Errors | Skipped | Time |\n"; + echo "| --- | --- | --- | --- | --- |\n"; + printf( + "| %d | %d | %d | %d | %ds |\n\n", + $junit['tests'], + $junit['failures'], + $junit['errors'], + $junit['skipped'], + $junit['time'] + ); + + if ( ! empty( $junit['failed'] ) ) { + echo '
Failed tests (' . count( $junit['failed'] ) . ")\n\n"; + echo "| Test | Message |\n"; + echo "| --- | --- |\n"; + foreach ( array_slice( $junit['failed'], 0, WPT_SUMMARY_MAX_FAILURES ) as $test ) { + echo '| `' . wpt_summary_cell( $test['name'] ) . '` | ' . wpt_summary_cell( $test['message'] ) . " |\n"; + } + if ( count( $junit['failed'] ) > WPT_SUMMARY_MAX_FAILURES ) { + echo "\n_Showing the first " . WPT_SUMMARY_MAX_FAILURES . ' failures. See junit.xml for the full list._' . "\n"; + } + echo "\n
\n\n"; + } +} else { + echo "_No junit.xml results were produced._\n\n"; +} + +foreach ( $failed as $step ) { + $tail = wpt_summary_tail( $output_dir . '/' . $step['name'] . '.log', WPT_SUMMARY_LOG_LINES ); + if ( '' === $tail ) { + continue; + } + + echo '
Last ' . WPT_SUMMARY_LOG_LINES . ' lines of ' . $step['name'] . ".log\n\n"; + echo "```\n" . str_replace( '```', "'''", $tail ) . "\n```\n\n"; + echo "
\n\n"; +}