Add Docker setup and container CI workflow for the test runner

.dockerignore

@@ -0,0 +1,7 @@
+.git
+.github
+.cache
+.env*
+vendor
+output
+wp-test-runner

.github/workflows/container-tests-report.yml

@@ -0,0 +1,141 @@
+name: Container Tests Report
+
+on:
+  # Runs after the Container Tests workflow completes, in the context of the
+  # base repository. This grants the permission to comment on pull requests
+  # opened from forks, which the `pull_request` event does not.
+  #
+  # The artifacts of the completed run were produced by pull request code and
+  # must be treated as untrusted. They are only read as text: no code from the
+  # pull request is checked out or executed here.
+  workflow_run:
+    workflows:
+      - Container Tests
+    types:
+      - completed
+
+permissions: {}
+
+jobs:
+  # Posts the summaries of the Container Tests jobs as a pull request comment.
+  #
+  # Performs the following steps:
+  # - Downloads the results artifacts of the completed run.
+  # - Finds the open pull request for the run's head commit.
+  # - Creates the comment, or updates the one posted for an earlier run.
+  comment:
+    name: Comment on the pull request
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      pull-requests: write
+    if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion != 'cancelled' }}
+
+    steps:
+      - name: Download the results
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        # A run that failed before uploading anything is still reported.
+        continue-on-error: true
+        with:
+          pattern: results-*
+          path: results
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+
+      - name: Post the summary
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          retries: 2
+          script: |
+            const fs = require( 'fs' );
+            const path = require( 'path' );
+
+            const run = context.payload.workflow_run;
+            const marker = '<!-- wpt-container-tests-report -->';
+            // GitHub rejects comments longer than 65536 characters.
+            const maxLength = 60000;
+
+            // Look the pull request up from the run itself rather than from the
+            // artifacts, so pull request code cannot redirect the comment.
+            const pulls = await github.paginate( github.rest.pulls.list, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: 'open',
+              per_page: 100,
+            } );
+            // The head repository is null when the fork has been deleted.
+            const headRepo = run.head_repository ? run.head_repository.full_name : '';
+            const pull = pulls.find(
+              ( pr ) =>
+                pr.head.sha === run.head_sha &&
+                pr.head.repo &&
+                pr.head.repo.full_name === headRepo
+            );
+            if ( ! pull ) {
+              core.info( `No open pull request found for ${ headRepo || 'a deleted repository' }@${ run.head_sha }.` );
+              return;
+            }
+
+            const summaries = [];
+            if ( fs.existsSync( 'results' ) ) {
+              for ( const dir of fs.readdirSync( 'results' ).sort() ) {
+                const file = path.join( 'results', dir, 'summary.md' );
+                // Only read regular files from the untrusted artifacts.
+                if ( fs.existsSync( file ) && fs.lstatSync( file ).isFile() ) {
+                  summaries.push( fs.readFileSync( file, 'utf8' ).trim() );
+                }
+              }
+            }
+
+            const icons = { success: '✅', failure: '❌', timed_out: '⏱️' };
+            const icon = icons[ run.conclusion ] || 'ℹ️';
+            let body = [
+              marker,
+              `# ${ icon } Container Tests: ${ run.conclusion }`,
+              '',
+              `Results for ${ run.head_sha.substring( 0, 7 ) } from [workflow run #${ run.run_number }](${ run.html_url }). Logs, \`junit.xml\`, and \`env.json\` are attached to the run as artifacts.`,
+              '',
+              summaries.length
+                ? summaries.join( '\n\n---\n\n' )
+                : '_No results were uploaded. Check the workflow run for errors in building the image or starting the containers._',
+            ].join( '\n' );
+
+            // Keep mentions in the untrusted summaries from notifying anyone.
+            body = body.replace( /@(?=[A-Za-z0-9])/g, '@​' );
+
+            if ( body.length > maxLength ) {
+              body = body.substring( 0, maxLength ) + `\n\n_The summary was truncated. See the [workflow run](${ run.html_url }) for the full results._`;
+            }
+
+            const comments = await github.paginate( github.rest.issues.listComments, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: pull.number,
+              per_page: 100,
+            } );
+            const existing = comments.find(
+              ( comment ) =>
+                comment.user &&
+                comment.user.login === 'github-actions[bot]' &&
+                comment.body &&
+                comment.body.startsWith( marker )
+            );
+
+            if ( existing ) {
+              await github.rest.issues.updateComment( {
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              } );
+              core.info( `Updated comment ${ existing.html_url }` );
+            } else {
+              const { data } = await github.rest.issues.createComment( {
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: pull.number,
+                body,
+              } );
+              core.info( `Created comment ${ data.html_url }` );
+            }

.github/workflows/container-tests.yml

@@ -0,0 +1,89 @@
+name: Container Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+    paths:
+      - '**.php'
+      - 'compose.yaml'
+      - 'docker/**'
+      - '.dockerignore'
+      - '.github/workflows/container-tests.yml'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  # Builds the runner image and runs prepare.php, test.php, report.php, and
+  # cleanup.php against a database container, mimicking a hosting environment.
+  #
+  # Each step's output, junit.xml, env.json, and a Markdown summary are
+  # uploaded as an artifact, and the summary is shown on the workflow run.
+  #
+  # The job fails when a runner step fails, not when WordPress tests fail.
+  test:
+    name: PHP ${{ matrix.php }} / ${{ matrix.db }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    permissions:
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - php: '8.3'
+            db: 'mysql:8.4'
+          - php: '8.1'
+            db: 'mariadb:10.6'
+
+    env:
+      PHP_VERSION: ${{ matrix.php }}
+      WPT_DB_IMAGE: ${{ matrix.db }}
+      # Failing WordPress tests are shown in the summary, but only problems with
+      # the runner itself, or a test run without junit.xml results, fail the job.
+      WPT_IGNORE_TEST_FAILURES: 1
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          show-progress: ${{ runner.debug == '1' && 'true' || 'false' }}
+          persist-credentials: false
+
+      - name: Build the runner image
+        run: WPT_UID="$(id -u)" docker compose build runner
+
+      - name: Run the test runner
+        run: |
+          mkdir -p output
+          docker compose up --abort-on-container-exit --exit-code-from runner
+
+      - name: Add the summary to the job
+        if: ${{ always() }}
+        run: |
+          if [ -f output/summary.md ]; then
+            cat output/summary.md >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "The runner did not produce a summary." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload the results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: results-php${{ matrix.php }}-${{ strategy.job-index }}
+          path: output/
+          if-no-files-found: warn
+
+      - name: Remove the containers
+        if: ${{ always() }}
+        run: docker compose down --volumes

.gitignore

@@ -14,3 +14,6 @@ wp-test-runner/
 *.orig
 *.patch
 *.diff
+
+# Results of containerized runs.
+output/

README.md

@@ -633,6 +633,38 @@ journalctl -u testrunner.timer
 journalctl -n 120 -u testrunner.service
 ```
 
+### Docker
+
+The runner can also be run in containers, with a disposable database, using [Docker Compose](https://docs.docker.com/compose/). The image in [`docker/Dockerfile`](docker/Dockerfile) installs PHP with the extensions the test suite uses, Composer, Node.js, Git, rsync, and an SSH client, so nothing needs to be installed on the host besides Docker.
+
+```bash
+mkdir -p output
+WPT_UID=$(id -u) docker compose up --build --abort-on-container-exit --exit-code-from runner
+docker compose down --volumes
+```
+
+This runs `prepare.php`, `test.php`, `report.php`, and `cleanup.php` in order. The test and report steps are skipped when preparing the environment fails, and the cleanup step always runs. The command exits with the exit code of the first step that failed.
+
+When the run finishes, the `output/` directory contains:
+
+- `prepare.log`, `test.log`, `report.log`, and `cleanup.log`: the output of each step.
+- `junit.xml` and `env.json`: the test results and environment details.
+- `summary.md`: a Markdown summary of the steps, the test results, and any failures.
+
+The PHP and database versions are chosen with environment variables. Any variable from [`.env.default`](.env.default) that is listed in [`compose.yaml`](compose.yaml) is passed through to the runner, so setting `WPT_REPORT_API_KEY` reports the results to WordPress.org.
+
+```bash
+PHP_VERSION=8.1 WPT_DB_IMAGE=mariadb:10.6 docker compose up --build --abort-on-container-exit --exit-code-from runner
+```
+
+`PHP_VERSION` is any tag of the official [`php`](https://hub.docker.com/_/php) image. The PHP 7.x images are built on Debian releases that no longer receive updates, so the image may fail to build for them.
+
+Set `WPT_IGNORE_TEST_FAILURES=1` to exit with 0 when WordPress tests fail but every runner step worked. A test run that produces no `junit.xml` results still fails.
+
+To open a shell in the runner image instead, use `docker compose run --rm runner bash`.
+
+The [Container Tests](.github/workflows/container-tests.yml) workflow runs the same setup on every pull request. It shows the summary on the workflow run and uploads the `output/` directory as an artifact. The [Container Tests Report](.github/workflows/container-tests-report.yml) workflow then posts the summaries as a comment on the pull request, and updates that comment on later runs.
+
 ## Contributing
 
 If you have questions about the process or run into test failures along the way, please [open an issue in the project repository](https://github.com/WordPress/phpunit-test-runner/issues) and we’ll help diagnose/get the documentation updated. Alternatively, you can also pop into the `#hosting` channel on [WordPress.org Slack](https://make.wordpress.org/chat/) for help.

compose.yaml

@@ -0,0 +1,53 @@
+# Runs the test runner and a disposable database in containers.
+#
+#   mkdir -p output
+#   docker compose up --build --abort-on-container-exit --exit-code-from runner
+#   docker compose down --volumes
+#
+# Results, logs, and a summary are written to ./output.
+#
+# The PHP and database versions are set with environment variables, for example:
+#
+#   PHP_VERSION=8.1 WPT_DB_IMAGE=mariadb:10.6 docker compose up --build ...
+#
+# Any WPT_* variable from .env.default can be passed through the same way.
+
+services:
+  db:
+    image: ${WPT_DB_IMAGE:-mysql:8.0}
+    environment:
+      MYSQL_ROOT_PASSWORD: root
+      MYSQL_DATABASE: wordpress_test
+      MYSQL_USER: wp_test
+      MYSQL_PASSWORD: wp_test
+    # The database is destroyed after every run, so keep it in memory.
+    tmpfs:
+      - /var/lib/mysql
+
+  runner:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+      args:
+        PHP_VERSION: ${PHP_VERSION:-8.3}
+        NODE_VERSION: ${NODE_VERSION:-24}
+        WPT_UID: ${WPT_UID:-1000}
+    depends_on:
+      - db
+    environment:
+      WPT_DB_NAME: wordpress_test
+      WPT_DB_USER: wp_test
+      WPT_DB_PASSWORD: wp_test
+      WPT_DB_HOST: db
+      WPT_DB_LABEL: ${WPT_DB_IMAGE:-mysql:8.0}
+      WPT_TABLE_PREFIX: ${WPT_TABLE_PREFIX:-wptests_}
+      WPT_REPORT_API_KEY: ${WPT_REPORT_API_KEY:-}
+      WPT_REPORT_URL: ${WPT_REPORT_URL:-}
+      WPT_FLAVOR: ${WPT_FLAVOR:-0}
+      WPT_EXTRATESTS: ${WPT_EXTRATESTS:-0}
+      WPT_PHPUNIT_CMD: ${WPT_PHPUNIT_CMD:-}
+      WPT_DEBUG: ${WPT_DEBUG:-}
+      WPT_CERTIFICATE_VALIDATION: ${WPT_CERTIFICATE_VALIDATION:-1}
+      WPT_IGNORE_TEST_FAILURES: ${WPT_IGNORE_TEST_FAILURES:-}
+    volumes:
+      - ./output:/output