From 3806b1c8ab5219af1b97bbe08df6dfbcaea11f28 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 18:32:43 +0400 Subject: [PATCH 1/4] Add corpus-diff CI: regenerate a pinned WordPress corpus and diff it --- .github/workflows/corpus-diff.yml | 99 +++++++++++++++++++++++++++++++ tools/export-corpus.php | 48 +++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 .github/workflows/corpus-diff.yml create mode 100644 tools/export-corpus.php diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml new file mode 100644 index 0000000..cacf74e --- /dev/null +++ b/.github/workflows/corpus-diff.yml @@ -0,0 +1,99 @@ +# Parses a pinned corpus of WordPress core source with the parser at the PR's +# merge base and at its head, normalizes both JSON outputs with prep-diff.php, +# and diffs them. Anything in the diff is a behavior change this PR makes: +# every hunk must be either intended (and explained in the PR) or a regression. +# +# Policy decisions, deliberate: +# - The corpus is pinned to one WordPress tag so diffs are reproducible. +# - The head checkout's tools/export-corpus.php and prep-diff.php drive both +# sides, so tooling changes never masquerade as parser changes. When +# prep-diff.php itself changes, its effect on normalization shows up in the +# diff and is reviewed like any other change. +# - Non-blocking: the job succeeds even when the diff is non-empty. The diff +# is published as an artifact and summarized. Make it blocking only after +# the signal has proven trustworthy. +# - PHP 7.4, the supported floor, keeps parity with the oldest runtime. + +name: Corpus Diff + +on: + pull_request: + +jobs: + corpus-diff: + name: WordPress corpus regeneration diff + runs-on: ubuntu-latest + + env: + WP_CORPUS_TAG: "6.8" + LC_ALL: C + + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set up PHP + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: "7.4" + coverage: none + + - name: Cache corpus + id: cache-corpus + uses: actions/cache@v4 + with: + path: corpus + key: wp-corpus-${{ env.WP_CORPUS_TAG }} + + - name: Download corpus + if: steps.cache-corpus.outputs.cache-hit != 'true' + run: | + curl -sSfL -o wordpress.zip "https://github.com/WordPress/WordPress/archive/refs/tags/${WP_CORPUS_TAG}.zip" + unzip -q wordpress.zip "WordPress-${WP_CORPUS_TAG}/wp-includes/*" + mkdir -p corpus + mv "WordPress-${WP_CORPUS_TAG}/wp-includes" corpus/wp-includes + rm -rf wordpress.zip "WordPress-${WP_CORPUS_TAG}" + + - name: Check out merge base + run: git worktree add base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)" + + - name: Install Composer dependencies (head) + run: composer install --no-interaction --no-security-blocking + + - name: Install Composer dependencies (base) + run: composer --working-dir=base install --no-interaction --no-security-blocking + + - name: Export corpus (base) + run: php -d memory_limit=4G tools/export-corpus.php base corpus/wp-includes > base.json + + - name: Export corpus (head) + run: php -d memory_limit=4G tools/export-corpus.php . corpus/wp-includes > head.json + + - name: Normalize and diff + run: | + php -d memory_limit=4G prep-diff.php < base.json > base.norm.json + php -d memory_limit=4G prep-diff.php < head.json > head.norm.json + # diff exits 1 on differences (expected) and 2 on trouble (fail). + diff -u --label base --label head base.norm.json head.norm.json > corpus.diff || [ $? -eq 1 ] + if [ -s corpus.diff ]; then + hunks=$(grep -c '^@@' corpus.diff) + lines=$(wc -l < corpus.diff) + { + echo "### Corpus diff: ${hunks} hunks, ${lines} lines" + echo + echo "The parser's output over wp-includes@${WP_CORPUS_TAG} changed." + echo "Review the \`corpus.diff\` artifact: every hunk must be intended and explained in the PR." + } >> "$GITHUB_STEP_SUMMARY" + else + echo "### Corpus diff: 0 hunks (no behavior change)" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload diff + if: always() + uses: actions/upload-artifact@v4 + with: + name: corpus.diff + path: corpus.diff + if-no-files-found: ignore diff --git a/tools/export-corpus.php b/tools/export-corpus.php new file mode 100644 index 0000000..8bf062e --- /dev/null +++ b/tools/export-corpus.php @@ -0,0 +1,48 @@ + > corpus.json + */ + +// Keep stdout pure JSON: vendored code emits deprecation notices on newer +// PHP, and display_errors otherwise interleaves them with the output. +ini_set( 'display_errors', 'stderr' ); + +if ( $argc < 3 ) { + fwrite( STDERR, 'Usage: php tools/export-corpus.php ' . PHP_EOL ); + exit( 1 ); +} + +$parser_root = rtrim( $argv[1], '/' ); +$corpus_dir = rtrim( $argv[2], '/' ); + +if ( ! is_file( $parser_root . '/vendor/autoload.php' ) ) { + fwrite( STDERR, 'No Composer autoloader in ' . $parser_root . '; run composer install there first.' . PHP_EOL ); + exit( 1 ); +} + +if ( ! is_dir( $corpus_dir ) ) { + fwrite( STDERR, 'Corpus directory not found: ' . $corpus_dir . PHP_EOL ); + exit( 1 ); +} + +require $parser_root . '/vendor/autoload.php'; + +$files = \WP_Parser\get_wp_files( $corpus_dir ); + +if ( ! is_array( $files ) ) { + fwrite( STDERR, 'Could not list the PHP files in ' . $corpus_dir . '.' . PHP_EOL ); + exit( 1 ); +} + +echo json_encode( \WP_Parser\parse_files( $files, $corpus_dir ), JSON_PRETTY_PRINT ), PHP_EOL; From 09fa3552c4d2a68c9a507da61c6b1390bbf9fb79 Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 18:45:54 +0400 Subject: [PATCH 2/4] Guard the corpus-diff job against a silently empty corpus --- .github/workflows/corpus-diff.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index cacf74e..c7245f0 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -56,6 +56,15 @@ jobs: mv "WordPress-${WP_CORPUS_TAG}/wp-includes" corpus/wp-includes rm -rf wordpress.zip "WordPress-${WP_CORPUS_TAG}" + # An empty or miscached corpus would make both exports emit `[]` and the + # diff trivially empty — a green job that checked nothing. wp-includes + # has well over 500 PHP files on any supported tag. + - name: Verify corpus + run: | + count=$(find corpus/wp-includes -name '*.php' | wc -l) + echo "Corpus PHP files: ${count}" + [ "$count" -ge 500 ] + - name: Check out merge base run: git worktree add base "$(git merge-base "origin/${{ github.base_ref }}" HEAD)" @@ -65,11 +74,19 @@ jobs: - name: Install Composer dependencies (base) run: composer --working-dir=base install --no-interaction --no-security-blocking + # The size floor guards the same failure mode as Verify corpus: a real + # wp-includes export is tens of megabytes of JSON. - name: Export corpus (base) - run: php -d memory_limit=4G tools/export-corpus.php base corpus/wp-includes > base.json + run: | + php -d memory_limit=4G tools/export-corpus.php base corpus/wp-includes > base.json + ls -l base.json + [ "$(wc -c < base.json)" -ge 1000000 ] - name: Export corpus (head) - run: php -d memory_limit=4G tools/export-corpus.php . corpus/wp-includes > head.json + run: | + php -d memory_limit=4G tools/export-corpus.php . corpus/wp-includes > head.json + ls -l head.json + [ "$(wc -c < head.json)" -ge 1000000 ] - name: Normalize and diff run: | From 66ca1eb6a6e9876d92c3b3aabc5dfb6db84f55dc Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 21:45:41 +0400 Subject: [PATCH 3/4] Pin the corpus to WordPress 7.0.4 7.0.4 is the latest WordPress release. Pinning the point release rather than the 7.0 major tag tracks the patches to wp-includes, so the corpus matches what is shipped. WordPress 7.0 raised core's minimum PHP to 7.4, which is exactly this job's PHP floor. Verified against wp-includes@7.0.4: 1039 PHP files, all of them present in the export, no parse errors, ~49 MB of JSON. Both of the job's guards (>= 500 files, >= 1 MB of JSON) still hold. --- .github/workflows/corpus-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/corpus-diff.yml b/.github/workflows/corpus-diff.yml index c7245f0..0217269 100644 --- a/.github/workflows/corpus-diff.yml +++ b/.github/workflows/corpus-diff.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest env: - WP_CORPUS_TAG: "6.8" + WP_CORPUS_TAG: "7.0.4" LC_ALL: C steps: From 1c23bb71a6a052169a0c2aaa551970a1dc8b451d Mon Sep 17 00:00:00 2001 From: Jon Surrell Date: Tue, 18 Aug 2026 21:46:29 +0400 Subject: [PATCH 4/4] Document the corpus diff procedure Records what the corpus-diff check is, how to run it by hand with the repo's own tools/export-corpus.php and prep-diff.php, and the policy the workflow already states: the head checkout's tooling drives both sides. --- README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/README.md b/README.md index ab35ee4..0057f8b 100644 --- a/README.md +++ b/README.md @@ -50,3 +50,37 @@ In your site's directory: ```bash wp parser create /path/to/source/code --user= ``` + +## Corpus diff + +Unit tests do not cover every shape of real-world documentation, so changes to the parser are also checked against a corpus of WordPress core source. The same corpus is parsed with the parser at two refs — the merge base of a pull request and its head — both JSON outputs are normalized with `prep-diff.php`, and the two are diffed. Everything in that diff is a behavior change the pull request makes: every hunk must be either intended and explained, or it is a regression. + +`.github/workflows/corpus-diff.yml` runs this on every pull request. The corpus is `wp-includes` from a pinned WordPress tag (`WP_CORPUS_TAG` in the workflow), so diffs are reproducible. The job is non-blocking: it uploads the diff as a `corpus.diff` artifact and reports the hunk count in the job summary. The head checkout's `tools/export-corpus.php` and `prep-diff.php` drive both sides, so tooling changes never masquerade as parser changes; when `prep-diff.php` itself changes, its effect on normalization shows up in the diff and is reviewed like any other change. + +To run it locally, get the pinned corpus: + +```bash +curl -sSfL -o wordpress.zip https://github.com/WordPress/WordPress/archive/refs/tags/7.0.4.zip +unzip -q wordpress.zip 'WordPress-7.0.4/wp-includes/*' +``` + +Check out the other side of the comparison and install its dependencies. The exporter runs under plain PHP — it does not load WordPress — but it needs a Composer autoloader in each parser root: + +```bash +git worktree add base "$(git merge-base origin/master HEAD)" +composer --working-dir=base install +composer install +``` + +Export both sides over the same corpus, normalize, and diff. `export-corpus.php` takes the parser root and the corpus directory, and writes JSON to stdout: + +```bash +export LC_ALL=C +php -d memory_limit=4G tools/export-corpus.php base WordPress-7.0.4/wp-includes > base.json +php -d memory_limit=4G tools/export-corpus.php . WordPress-7.0.4/wp-includes > head.json +php -d memory_limit=4G prep-diff.php < base.json > base.norm.json +php -d memory_limit=4G prep-diff.php < head.json > head.norm.json +diff -u base.norm.json head.norm.json > corpus.diff +``` + +An empty `corpus.diff` means the change has no effect on parser output.