diff --git a/.gitignore b/.gitignore index 0c14928..1873c2c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,14 @@ results/ regression.diffs regression.out +# Generated sql/ directory for test/build +# Created by make test-build. See README.asc for details. +test/build/sql/ + +# Auto-generated schedule file for test/install +# Created by make when test/install/*.sql files exist. +test/install/schedule + # Misc tmp/ .DS_Store diff --git a/pgxntool/.claude/settings.json b/pgxntool/.claude/settings.json deleted file mode 100644 index e7bf5a9..0000000 --- a/pgxntool/.claude/settings.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(cat:*)", - "Bash(make test:*)", - "Bash(tee:*)", - "Bash(echo:*)", - "Bash(git show:*)", - "Bash(git log:*)", - "Bash(ls:*)", - "Bash(find:*)", - "Bash(git checkout:*)", - "Bash(head:*)" - ], - "additionalDirectories": [ - "../pgxntool-test/" - ] - } -} diff --git a/pgxntool/.gitattributes b/pgxntool/.gitattributes index a94d824..8dc1599 100644 --- a/pgxntool/.gitattributes +++ b/pgxntool/.gitattributes @@ -1,5 +1,6 @@ .gitattributes export-ignore .claude/ export-ignore +.github/ export-ignore *.md export-ignore .DS_Store export-ignore *.asc export-ignore diff --git a/pgxntool/CLAUDE.md b/pgxntool/CLAUDE.md index d2ea214..75c9d16 100644 --- a/pgxntool/CLAUDE.md +++ b/pgxntool/CLAUDE.md @@ -2,6 +2,34 @@ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +## CI Monitoring After Every Push + +**REQUIRED**: After every `git push`, immediately start a background task to +monitor the CI run for that push. If you pushed to both pgxntool and +pgxntool-test, start a background task for each repo — do not monitor them +sequentially. + +The CI monitor lives in the pgxntool-test checkout: run +`bash ../pgxntool-test/.claude/skills/ci/scripts/monitor-ci.sh` (the `/ci` +skill). It monitors both repos and derives the owner from the current repo. +Pass the exact push SHA when available — `gh run list --branch` has a race +condition: if two pushes land close together on the same branch, `--branch` +may pick up the wrong run. `--commit SHA` targets the exact push and avoids it. + +## Scope of This File + +**CLAUDE.md is for people USING pgxntool** — extension developers who have embedded +pgxntool into their project via `git subtree`. It documents the build system, available +commands, and how pgxntool works. + +**If you are making changes to pgxntool itself**, stop — you are in the wrong place. +See `.claude/` in this directory for developer guidelines. More importantly, pgxntool +development must be done from the **pgxntool-test** repository, not from here. See the +`Development Workflow` section below. + +Any agent working in an extension project should always defer to that project's own +CLAUDE.md and instructions over anything stated here. + ## Git Commit Guidelines **IMPORTANT**: When creating commit messages, do not attribute commits to yourself (Claude). Commit messages should reflect the work being done without AI attribution in the message body. The standard Co-Authored-By trailer is acceptable. @@ -181,9 +209,10 @@ When tests fail, examine the diff output carefully. The actual test output in `t - Validates repo is clean before tagging ### Subtree Sync Support -- `make pgxntool-sync` pulls latest release -- Multiple sync targets: release, stable, local variants -- Uses `git subtree pull --squash` +- `make pgxntool-sync` pulls the latest release (the `release` tag) from the canonical repo +- `pgxntool/pgxntool-sync.sh [ []]` does the work and can be run without make +- `make pgxntool-sync-` pulls from the `pgxntool-sync-` variable (` `) +- Uses `git subtree pull --squash`, then `update-setup-files.sh` for a 3-way merge of copied files - Requires clean repo (no uncommitted changes) ### pg_tle Support diff --git a/pgxntool/HISTORY.asc b/pgxntool/HISTORY.asc index 2b7d5e7..b0894da 100644 --- a/pgxntool/HISTORY.asc +++ b/pgxntool/HISTORY.asc @@ -1,3 +1,75 @@ +2.1.0 +----- +== Fix setup.sh / pgxntool-sync.sh / update-setup-files.sh inside a git worktree +These scripts guarded the project root with `[ -d .git ]`. In a linked worktree +`.git` is a file, not a directory, so the check failed: setup.sh would wrongly +re-run `git init`, and the sync scripts aborted with "Not in a git repository." +They now detect the repo with `git rev-parse --git-dir`, which works in both a +normal clone and a worktree. + +== Fix `verify-results` checking stale results in `make results` +`make results` ran `verify-results` before `make test`, so it checked stale +`regression.diffs` from a prior run. Reordered so `verify-results` always +checks the fresh results. + +== Fix `PGXNTOOL_ENABLE_TEST_BUILD`/`PGXNTOOL_ENABLE_TEST_INSTALL` ignoring command-line values +Without `override`, the `pgxntool_validate_yesno` normalization was silently +skipped when these variables were set on the command line. + +2.0.3 +----- +== Fix pgxntool-sync remote, and make it runnable without make +`make pgxntool-sync` pointed at the old SSH URL, which fails without +GitHub SSH keys; it now uses `https://github.com/Postgres-Extensions/pgxntool.git` +over HTTPS (still the `release` tag). The subtree pull + `update-setup-files.sh` +logic moved into `pgxntool/pgxntool-sync.sh`, which the make targets now wrap, so +you can also sync without `make` by running it directly. + +2.0.2 +----- +== Fix parse_control_file corrupting values with trailing comments +Control file values like `default_version = '1.0.0' # comment` were parsed +incorrectly — the trailing quote was left in the value due to comment removal +happening after quote stripping. Fixed by removing comments first. + +2.0.1 +----- +== Improve bash compatibility (specifically for Mac OS) +Mac OS uses the (very old) bash 3.2 as the default shell; fix a few compatibility bugs. + +2.0.0 +----- +== Remove .source file support +PostgreSQL removed `.source` file processing from `pg_regress` in PG15. +The `input/*.source` → `sql/*.sql` and `output/*.source` → +`expected/*.out` conversion mechanism no longer functions. All related +variables (`TEST__SOURCE__*`), the `make_results.sh` helper script, and +the special-case logic in `make results` have been removed. Extensions +that used `.source` files should convert them to regular `test/sql/*.sql` +and `test/expected/*.out` files using relative paths. + +== Add test-build for pre-test validation +When CREATE EXTENSION fails due to a SQL syntax error, PostgreSQL reports only a cryptic error with limited context. test-build runs your extension SQL directly through pg_regress first, so syntax errors show the exact file, line, and position — cutting debugging time significantly. Place SQL files in `test/build/` to enable; auto-detects based on file presence. + +== Add test/install for one-time test setup +Extensions that install dependencies or run expensive setup in every test file pay that cost once per test. test/install runs setup SQL once before the entire test suite, and all regular tests share the resulting database state. This can dramatically speed up test suites that install extensions or load fixtures. Place SQL files in `test/install/` to enable; auto-detects based on file presence. + +== Add verify-results safeguard for make results +`make results` now refuses to run when tests are failing (detected via `regression.diffs`). Prevents accidentally blessing incorrect output as the new expected results. Enabled by default; disable with `PGXNTOOL_ENABLE_VERIFY_RESULTS=no`. + +== Fix bash 3.2 compatibility and shebang portability +`${#ARRAY[@]:-0}` is a syntax error in bash 3.2; replaced with `${#ARRAY[@]}`. +Shell scripts now use `#!/usr/bin/env bash`. + +1.1.2 +----- +== Fix double --dbname bug that defeated unique test database names +The unique database naming introduced in 1.1.0 was ineffective because +base.mk added --dbname=$(REGRESS_DBNAME) to REGRESS_OPTS while PGXS +also appends --dbname=$(CONTRIB_TESTDB). The second --dbname caused +pg_regress to create a contrib_regression database that collided across +projects. Fixed by overriding CONTRIB_TESTDB after include $(PGXS) instead. + 1.1.1 ----- == Fix pg_tle exception handler and empty upgrade files @@ -72,7 +144,7 @@ VERSION is defined by PGXS itself, so trying to use it causes problems. Old code didn't deal with the lack of a . that can appear in a 10+ version. 0.1.10 ------- +----- ### Remove invalid `git subtree pull` options 0.1.9 diff --git a/pgxntool/README.asc b/pgxntool/README.asc index bf3559d..db6162f 100644 --- a/pgxntool/README.asc +++ b/pgxntool/README.asc @@ -25,7 +25,9 @@ TODO: Create a nice script that will init a new project for you. == Development -If you want to contribute to pgxntool development, work from the https://github.com/decibel/pgxntool-test[pgxntool-test] repository, not from this repository. That repository contains the test infrastructure and development tools needed to validate changes to pgxntool. This repository contains only the framework files that get embedded into extension projects via `git subtree`. +If you want to contribute to pgxntool development, work from the https://github.com/Postgres-Extensions/pgxntool-test[pgxntool-test] repository, not from this repository. That repository contains the test infrastructure and development tools needed to validate changes to pgxntool. This repository contains only the framework files that get embedded into extension projects via `git subtree`. + +Changes are normally paired across both repos: a `pgxntool` change should come with a matching branch (same name, on the same account) and PR in `pgxntool-test`, and CI enforces this pairing. See the https://github.com/Postgres-Extensions/pgxntool-test#ci-and-contributing[CI and Contributing] section in pgxntool-test for the full workflow. == Usage Typically, you can just create a simple Makefile that does nothing but include base.mk: @@ -47,6 +49,120 @@ Runs unit tests via the PGXS `installcheck` target. Unlike a simple `make instal NOTE: While you can still run `make installcheck` or any other valid PGXS make target directly, it's recommended to use `make test` when using pgxntool. The `test` target ensures clean builds, proper test isolation, and correct dependency installation. +=== test-build +Validates that extension SQL files are syntactically correct before running the full test suite. This feature runs SQL files from `test/build/` through `pg_regress`, providing better error messages than `CREATE EXTENSION` failures when there are syntax errors in your extension code. + +**How it works:** + +1. Place SQL files in `test/build/*.sql` +2. Place expected output in `test/build/expected/*.out` +3. These files run through `pg_regress` before `make test` runs the main test suite +4. If any build test fails, the test run stops immediately with clear error messages + +**Directory structure:** + +---- +test/build/ +├── *.sql # SQL test files (checked in) +├── expected/ # Expected output files (checked in) +│ └── *.out +└── sql/ # GENERATED - do not edit or check in + └── *.sql # Synced from *.sql above +---- + +The `sql/` subdirectory is generated automatically by `make test-build`. It is listed in `.gitignore` and removed by `make clean`. Do not place files directly in `test/build/sql/`. + +**Configuration:** + +The feature auto-detects based on whether `test/build/*.sql` files exist: + +- Files present → feature enabled automatically +- No files → feature disabled (no impact on existing projects) + +You can override auto-detection by setting `PGXNTOOL_ENABLE_TEST_BUILD`: +---- +# In your Makefile +PGXNTOOL_ENABLE_TEST_BUILD = yes # or no +---- + +**Example: Validate extension SQL compiles** + +Create `test/build/build.sql` to run your extension's SQL directly: + +---- +\set ECHO none +-- Sets ON_ERROR_STOP, VERBOSITY verbose, and ON_ERROR_ROLLBACK +\i test/pgxntool/psql.sql +-- Suppress column headers and row counts for cleaner expected output +\t + +BEGIN; +SET client_min_messages = WARNING; + +-- Install dependencies your extension requires +CREATE EXTENSION IF NOT EXISTS pgtap CASCADE; + +-- Clean slate +DROP EXTENSION IF EXISTS myext; +DROP SCHEMA IF EXISTS myext; +CREATE SCHEMA myext; + +-- Run the actual extension SQL (not CREATE EXTENSION) +-- psql.sql above ensures errors abort immediately with clear messages +\i sql/myext.sql + +-- If we get here, the build succeeded; ON_ERROR_STOP would have aborted on any error above +\echo # BUILD TEST SUCCEEDED +ROLLBACK; +---- + +This approach catches SQL syntax errors *before* running `CREATE EXTENSION`, giving clearer error messages with line numbers. The `ROLLBACK` ensures nothing persists—this is purely validation. + +**Why use `\i` instead of `CREATE EXTENSION`?** + +When `CREATE EXTENSION` fails, PostgreSQL shows only "syntax error" with limited context. Running the SQL directly via `\i` shows the exact line and position of errors, making debugging much faster. + +=== test/install +Runs setup files before the main test suite within the same `pg_regress` invocation. This allows expensive one-time operations (like extension installation) to set up state that persists into the regular test files. + +**How it works:** + +1. Place SQL files in `test/install/*.sql` +2. Place expected output alongside as `test/install/*.out` +3. A schedule file is auto-generated that lists install files with `../install/` relative paths +4. `pg_regress` processes the install schedule first, then runs regular test files — all in one invocation, so database state persists + +**Directory structure:** + +---- +test/install/ +├── *.sql # SQL setup files (checked in) +├── *.out # Expected output (checked in, alongside .sql) +├── .gitignore # Ignores pg_regress artifacts (*.out.diff) +└── schedule # GENERATED - auto-created by make +---- + +The `schedule` file is generated automatically and listed in `.gitignore`. Do not edit it. + +**Configuration:** + +The feature auto-detects based on whether `test/install/*.sql` files exist: + +- Files present → feature enabled automatically +- No files → feature disabled (no impact on existing projects) + +You can override auto-detection by setting `PGXNTOOL_ENABLE_TEST_INSTALL`: +---- +# In your Makefile +PGXNTOOL_ENABLE_TEST_INSTALL = yes # or no +---- + +**Why this is useful:** + +Without `test/install`, each test file typically needs to run `CREATE EXTENSION` in its setup, which adds overhead and doesn't allow validating the installation step separately. With `test/install`, setup runs once before all tests, and any state it creates (tables, extensions, etc.) is available to every subsequent test file. + +**Key detail:** Install files and regular tests run in a single `pg_regress` invocation. This means the database is NOT dropped between install and test phases — state created by install files persists into the main test suite. + === testdeps This rule allows you to ensure certain actions have taken place before running tests. By default it has a single prerequisite, `pgtap`, which will attempt to install http://pgtap.org[pgtap] from PGXN. This depneds on having the pgxn client installed. @@ -78,6 +194,24 @@ IMPORTANT: *`make results` requires manual verification first*. The correct work Never run `make results` without first verifying the test changes are correct. The `results` target copies files from `test/results/` to `test/expected/`, so running it blindly will make incorrect output become the new expected behavior. +==== verify-results safeguard +By default, `make results` will refuse to run if `test/results/regression.diffs` exists (indicating failing tests). This prevents accidentally updating expected files with incorrect output. + +If tests are failing, you'll see: +---- +ERROR: Tests are failing. Cannot run 'make results'. +Fix test failures first, then run 'make results'. +---- + +To disable this safeguard (not recommended): +---- +# In your Makefile +PGXNTOOL_ENABLE_VERIFY_RESULTS = no + +# Or on the command line +make PGXNTOOL_ENABLE_VERIFY_RESULTS=no results +---- + === tag `make tag` will create a git branch for the current version of your extension, as determined by the META.json file. The reason to do this is so you can always refer to the exact code that went into a released version. @@ -91,11 +225,23 @@ WARNING: You will be very unhappy if you forget to update the .control file for NOTE: Part of the `clean` recipe is cleaning up these .zip files. If you accidentally clean before uploading, just run `make dist-only`. === pgxntool-sync -This rule will pull down the latest released version of PGXNtool via `git subtree pull`. +This rule will pull down the latest released version of PGXNtool via `git subtree pull` and then reconcile the files `setup.sh` copied into your project (`.gitignore`, `test/deps.sql`) with a 3-way merge. NOTE: Your repository must be clean (no modified files) in order to run this. Running this command will produce a git commit of the merge. -TIP: There is also a `pgxntool-sync-%` rule if you need to do more advanced things. +TIP: The actual work is done by `pgxntool/pgxntool-sync.sh`, so you can run it directly (`pgxntool/pgxntool-sync.sh`) if you'd rather not go through `make`. It optionally takes `` and `` arguments to pull from somewhere other than the default. + +TIP: There is also a `pgxntool-sync-%` rule if you need to do more advanced things. `make pgxntool-sync-` pulls from the ` ` defined by the `pgxntool-sync-` make variable. + +=== distclean +`make distclean` removes generated configuration files (`META.json`, `meta.mk`, `control.mk`) that survive a normal `make clean`. + +NOTE: PGXS doesn't provide any special support for `distclean` — its built-in `distclean` target simply depends on `clean`. PGXNtool uses its own `PGXNTOOL_distclean` variable to track files that should only be removed by `distclean`, not `clean`. + +If your extension generates additional files that should be removed by `distclean` but not `clean`, you can add them: +---- +PGXNTOOL_distclean += my_generated_config.mk +---- === pgtle Generates pg_tle (Trusted Language Extensions) registration SQL files for deploying extensions in managed environments like AWS RDS/Aurora. See <<_pg_tle_Support>> for complete documentation. @@ -223,7 +369,7 @@ Location of `asciidoc` or equivalent executable. If not set PGXNtool will search for first `asciidoctor`, then `asciidoc`. ASCIIDOC_EXTS:: File extensions to consider as Asciidoc. -Defined as `+= adoc asciidoc`. +Defined as `+= adoc asciidoc asc`. ASCIIDOC_FILES:: Asciidoc input files. PGXNtool searches each `$(DOC_DIRS)` directory, looking for files with any `$(ASCIIDOC_EXTS)` extension. diff --git a/pgxntool/README.html b/pgxntool/README.html index 41200aa..431358f 100644 --- a/pgxntool/README.html +++ b/pgxntool/README.html @@ -451,14 +451,17 @@

PGXNtool

  • 5. Version-Specific SQL Files @@ -539,7 +542,10 @@

    2. Development

    -

    If you want to contribute to pgxntool development, work from the pgxntool-test repository, not from this repository. That repository contains the test infrastructure and development tools needed to validate changes to pgxntool. This repository contains only the framework files that get embedded into extension projects via git subtree.

    +

    If you want to contribute to pgxntool development, work from the pgxntool-test repository, not from this repository. That repository contains the test infrastructure and development tools needed to validate changes to pgxntool. This repository contains only the framework files that get embedded into extension projects via git subtree.

    +
    +
    +

    Changes are normally paired across both repos: a pgxntool change should come with a matching branch (same name, on the same account) and PR in pgxntool-test, and CI enforces this pairing. See the CI and Contributing section in pgxntool-test for the full workflow.

    @@ -599,7 +605,190 @@

    -

    4.3. testdeps

    +

    4.3. test-build

    +
    +

    Validates that extension SQL files are syntactically correct before running the full test suite. This feature runs SQL files from test/build/ through pg_regress, providing better error messages than CREATE EXTENSION failures when there are syntax errors in your extension code.

    +
    +
    +

    How it works:

    +
    +
    +
      +
    1. +

      Place SQL files in test/build/*.sql

      +
      2. +
    2. +

      Place expected output in test/build/expected/*.out

      +
      3. +
    3. +

      These files run through pg_regress before make test runs the main test suite

      +
      4. +
    4. +

      If any build test fails, the test run stops immediately with clear error messages

      +
      5. +
    +
    +
    +

    Directory structure:

    +
    +
    +
    +
    test/build/
+├── *.sql              # SQL test files (checked in)
+├── expected/          # Expected output files (checked in)
+│   └── *.out
+└── sql/               # GENERATED - do not edit or check in
+    └── *.sql          # Synced from *.sql above
    +
    +
    +
    +

    The sql/ subdirectory is generated automatically by make test-build. It is listed in .gitignore and removed by make clean. Do not place files directly in test/build/sql/.

    +
    +
    +

    Configuration:

    +
    +
    +

    The feature auto-detects based on whether test/build/*.sql files exist:

    +
    +
    +
      +
    • +

      Files present → feature enabled automatically

      +
      • +
    • +

      No files → feature disabled (no impact on existing projects)

      +
      • +
    +
    +
    +

    You can override auto-detection by setting PGXNTOOL_ENABLE_TEST_BUILD:

    +
    +
    +
    +
    # In your Makefile
+PGXNTOOL_ENABLE_TEST_BUILD = yes  # or no
    +
    +
    +
    +

    Example: Validate extension SQL compiles

    +
    +
    +

    Create test/build/build.sql to run your extension’s SQL directly:

    +
    +
    +
    +
    \set ECHO none
+-- Sets ON_ERROR_STOP, VERBOSITY verbose, and ON_ERROR_ROLLBACK
+\i test/pgxntool/psql.sql
+-- Suppress column headers and row counts for cleaner expected output
+\t
+
+BEGIN;
+SET client_min_messages = WARNING;
+
+-- Install dependencies your extension requires
+CREATE EXTENSION IF NOT EXISTS pgtap CASCADE;
+
+-- Clean slate
+DROP EXTENSION IF EXISTS myext;
+DROP SCHEMA IF EXISTS myext;
+CREATE SCHEMA myext;
+
+-- Run the actual extension SQL (not CREATE EXTENSION)
+-- psql.sql above ensures errors abort immediately with clear messages
+\i sql/myext.sql
+
+-- If we get here, the build succeeded; ON_ERROR_STOP would have aborted on any error above
+\echo # BUILD TEST SUCCEEDED
+ROLLBACK;
    +
    +
    +
    +

    This approach catches SQL syntax errors before running CREATE EXTENSION, giving clearer error messages with line numbers. The ROLLBACK ensures nothing persists—this is purely validation.

    +
    +
    +

    Why use \i instead of CREATE EXTENSION?

    +
    +
    +

    When CREATE EXTENSION fails, PostgreSQL shows only "syntax error" with limited context. Running the SQL directly via \i shows the exact line and position of errors, making debugging much faster.

    +
    +
    +
    +

    4.4. test/install

    +
    +

    Runs setup files before the main test suite within the same pg_regress invocation. This allows expensive one-time operations (like extension installation) to set up state that persists into the regular test files.

    +
    +
    +

    How it works:

    +
    +
    +
      +
    1. +

      Place SQL files in test/install/*.sql

      +
      2. +
    2. +

      Place expected output alongside as test/install/*.out

      +
      3. +
    3. +

      A schedule file is auto-generated that lists install files with ../install/ relative paths

      +
      4. +
    4. +

      pg_regress processes the install schedule first, then runs regular test files — all in one invocation, so database state persists

      +
      5. +
    +
    +
    +

    Directory structure:

    +
    +
    +
    +
    test/install/
+├── *.sql              # SQL setup files (checked in)
+├── *.out              # Expected output (checked in, alongside .sql)
+├── .gitignore         # Ignores pg_regress artifacts (*.out.diff)
+└── schedule           # GENERATED - auto-created by make
    +
    +
    +
    +

    The schedule file is generated automatically and listed in .gitignore. Do not edit it.

    +
    +
    +

    Configuration:

    +
    +
    +

    The feature auto-detects based on whether test/install/*.sql files exist:

    +
    +
    +
      +
    • +

      Files present → feature enabled automatically

      +
      • +
    • +

      No files → feature disabled (no impact on existing projects)

      +
      • +
    +
    +
    +

    You can override auto-detection by setting PGXNTOOL_ENABLE_TEST_INSTALL:

    +
    +
    +
    +
    # In your Makefile
+PGXNTOOL_ENABLE_TEST_INSTALL = yes  # or no
    +
    +
    +
    +

    Why this is useful:

    +
    +
    +

    Without test/install, each test file typically needs to run CREATE EXTENSION in its setup, which adds overhead and doesn’t allow validating the installation step separately. With test/install, setup runs once before all tests, and any state it creates (tables, extensions, etc.) is available to every subsequent test file.

    +
    +
    +

    Key detail: Install files and regular tests run in a single pg_regress invocation. This means the database is NOT dropped between install and test phases — state created by install files persists into the main test suite.

    +
    +
    +
    +

    4.5. testdeps

    This rule allows you to ensure certain actions have taken place before running tests. By default it has a single prerequisite, pgtap, which will attempt to install pgtap from PGXN. This depneds on having the pgxn client installed.

    @@ -635,7 +824,7 @@

    -

    4.4. results

    +

    4.6. results

    Because make test ultimately runs installcheck, it’s using the Postgres test suite. Unfortunately, that suite is based on running diff between a raw output file and expected results. I STRONGLY recommend you use pgTap instead! With pgTap, it’s MUCH easier to determine whether a test is passing or not - tests explicitly pass or fail rather than requiring you to examine diff output. The extra effort of learning pgTap will quickly pay for itself. This example might help get you started.

    @@ -670,9 +859,36 @@

    Never run make results without first verifying the test changes are correct. The results target copies files from test/results/ to test/expected/, so running it blindly will make incorrect output become the new expected behavior.

    +
    +

    4.6.1. verify-results safeguard

    +
    +

    By default, make results will refuse to run if test/results/regression.diffs exists (indicating failing tests). This prevents accidentally updating expected files with incorrect output.

    +
    +
    +

    If tests are failing, you’ll see:

    +
    +
    +
    +
    ERROR: Tests are failing. Cannot run 'make results'.
+Fix test failures first, then run 'make results'.
    +
    +
    +
    +

    To disable this safeguard (not recommended):

    +
    +
    +
    +
    # In your Makefile
+PGXNTOOL_ENABLE_VERIFY_RESULTS = no
+
+# Or on the command line
+make PGXNTOOL_ENABLE_VERIFY_RESULTS=no results
    +
    +
    +
    -

    4.5. tag

    +

    4.7. tag

    make tag will create a git branch for the current version of your extension, as determined by the META.json file. The reason to do this is so you can always refer to the exact code that went into a released version.

    @@ -693,7 +909,7 @@

    4.

    -

    4.6. dist

    +

    4.8. dist

    make dist will create a .zip file for your current version that you can upload to PGXN. The file is named after the PGXN name and version (the top-level "name" and "version" attributes in META.json). The .zip file is placed in the parent directory so as not to clutter up your git repo.

    @@ -711,9 +927,9 @@

    -

    4.7. pgxntool-sync

    +

    4.9. pgxntool-sync

    -

    This rule will pull down the latest released version of PGXNtool via git subtree pull.

    +

    This rule will pull down the latest released version of PGXNtool via git subtree pull and then reconcile the files setup.sh copied into your project (.gitignore, test/deps.sql) with a 3-way merge.

    @@ -734,14 +950,52 @@

    Tip

    + +
    -There is also a pgxntool-sync-% rule if you need to do more advanced things. +The actual work is done by pgxntool/pgxntool-sync.sh, so you can run it directly (pgxntool/pgxntool-sync.sh) if you’d rather not go through make. It optionally takes <repo> and <ref> arguments to pull from somewhere other than the default. +
    +
    +
    + + + + + +
    +
    Tip
    +    		 +There is also a pgxntool-sync-% rule if you need to do more advanced things. make pgxntool-sync-<name> pulls from the <repo> <ref> defined by the pgxntool-sync-<name> make variable. +
    +
    +
    +
    +

    4.10. distclean

    +
    +

    make distclean removes generated configuration files (META.json, meta.mk, control.mk) that survive a normal make clean.

    +
    +
    + + + +
    +
    Note
    +    		 +PGXS doesn’t provide any special support for distclean — its built-in distclean target simply depends on clean. PGXNtool uses its own PGXNTOOL_distclean variable to track files that should only be removed by distclean, not clean.
    +
    +

    If your extension generates additional files that should be removed by distclean but not clean, you can add them:

    +
    +
    +
    +
    PGXNTOOL_distclean += my_generated_config.mk
    +
    +
    -

    4.8. pgtle

    +

    4.11. pgtle

    Generates pg_tle (Trusted Language Extensions) registration SQL files for deploying extensions in managed environments like AWS RDS/Aurora. See [_pg_tle_Support] for complete documentation.

    @@ -750,7 +1004,7 @@

    -

    4.9. check-pgtle

    +

    4.12. check-pgtle

    Checks if pg_tle is installed and reports the version. This target: - Reports the version from pg_extension if CREATE EXTENSION pg_tle has been run in the database @@ -766,7 +1020,7 @@

    -

    4.10. run-pgtle

    +

    4.13. run-pgtle

    Registers all extensions with pg_tle by executing the generated pg_tle registration SQL files in a PostgreSQL database. This target: - Requires pg_tle extension to be installed (checked via check-pgtle) @@ -974,7 +1228,7 @@

    <
    ASCIIDOC_EXTS

    File extensions to consider as Asciidoc. -Defined as += adoc asciidoc.

    +Defined as += adoc asciidoc asc.

    ASCIIDOC_FILES
    @@ -1259,7 +1513,7 @@

    diff --git a/pgxntool/_.gitignore b/pgxntool/_.gitignore index 0c14928..1873c2c 100644 --- a/pgxntool/_.gitignore +++ b/pgxntool/_.gitignore @@ -24,6 +24,14 @@ results/ regression.diffs regression.out +# Generated sql/ directory for test/build +# Created by make test-build. See README.asc for details. +test/build/sql/ + +# Auto-generated schedule file for test/install +# Created by make when test/install/*.sql files exist. +test/install/schedule + # Misc tmp/ .DS_Store diff --git a/pgxntool/base.mk b/pgxntool/base.mk index 9b621df..a7ab93c 100644 --- a/pgxntool/base.mk +++ b/pgxntool/base.mk @@ -44,39 +44,155 @@ DOCS += $(foreach dir,$(DOC_DIRS),$(wildcard $(dir)/*)) # Find all asciidoc targets ASCIIDOC ?= $(shell which asciidoctor 2>/dev/null || which asciidoc 2>/dev/null) -ASCIIDOC_EXTS += adoc asciidoc +ASCIIDOC_EXTS += adoc asciidoc asc ASCIIDOC_FILES += $(foreach dir,$(DOC_DIRS),$(foreach ext,$(ASCIIDOC_EXTS),$(wildcard $(dir)/*.$(ext)))) PG_CONFIG ?= pg_config TESTDIR ?= test TESTOUT ?= $(TESTDIR) -# .source files are OPTIONAL - see "pg_regress workflow" comment below for details -TEST__SOURCE__INPUT_FILES += $(wildcard $(TESTDIR)/input/*.source) -TEST__SOURCE__OUTPUT_FILES += $(wildcard $(TESTDIR)/output/*.source) -TEST__SOURCE__INPUT_AS_OUTPUT = $(subst input,output,$(TEST__SOURCE__INPUT_FILES)) TEST_SQL_FILES += $(wildcard $(TESTDIR)/sql/*.sql) TEST_RESULT_FILES = $(patsubst $(TESTDIR)/sql/%.sql,$(TESTDIR)/expected/%.out,$(TEST_SQL_FILES)) -TEST_FILES = $(TEST__SOURCE__INPUT_FILES) $(TEST_SQL_FILES) -# Ephemeral files generated from source files (should be cleaned) -# input/*.source → sql/*.sql (converted by pg_regress) -TEST__SOURCE__SQL_FILES = $(patsubst $(TESTDIR)/input/%.source,$(TESTDIR)/sql/%.sql,$(TEST__SOURCE__INPUT_FILES)) -# output/*.source → expected/*.out (converted by pg_regress) -TEST__SOURCE__EXPECTED_FILES = $(patsubst $(TESTDIR)/output/%.source,$(TESTDIR)/expected/%.out,$(TEST__SOURCE__OUTPUT_FILES)) -REGRESS = $(sort $(notdir $(subst .source,,$(TEST_FILES:.sql=)))) # Sort is to get unique list +TEST_FILES = $(TEST_SQL_FILES) +REGRESS = $(sort $(notdir $(TEST_FILES:.sql=))) REGRESS_OPTS = --inputdir=$(TESTDIR) --outputdir=$(TESTOUT) # See additional setup below +# +# OPTIONAL TEST FEATURES +# +# These sections configure optional test features. Each feature can be enabled/disabled +# via a makefile variable. If not explicitly set, features auto-detect based on +# directory existence or default behavior. The actual feature implementation is +# located later in this file (see test-build target, schedule file generation, etc.). +# + +# Helper function: normalize a yes/no variable to lowercase and validate. +# Usage: $(call pgxntool_validate_yesno,VALUE,VARIABLE_NAME) +# Returns the lowercase value ("yes" or "no"), or errors if invalid. +pgxntool_validate_yesno = $(strip \ + $(if $(filter yes no,$(shell echo "$(1)" | tr '[:upper:]' '[:lower:]')),\ + $(shell echo "$(1)" | tr '[:upper:]' '[:lower:]'),\ + $(error $(2) must be "yes" or "no", got "$(1)"))) + +# ------------------------------------------------------------------------------ +# test-build: Sanity check extension files before running full test suite +# ------------------------------------------------------------------------------ +# Purpose: Validates that extension SQL files are syntactically correct by running +# files from test/build/ through pg_regress. This provides better error +# messages than CREATE EXTENSION failures. +# +# Variable: PGXNTOOL_ENABLE_TEST_BUILD +# - Can be set manually in Makefile or command line +# - Allowed values: "yes" or "no" (case-insensitive) +# - If not set: Auto-detects based on existence of test/build/*.sql files +# - Set to "yes" explicitly to get an error if test/build/ has no SQL files +# (useful to catch accidental deletion of test/build/ contents) +# - Set to "no" explicitly to disable even when test/build/ has SQL files +# +# Implementation: See test-build target definition (search for "test-build:" in this file) +# +TEST_BUILD_SQL_FILES = $(wildcard $(TESTDIR)/build/*.sql) +TEST_BUILD_FILES = $(TEST_BUILD_SQL_FILES) +ifdef PGXNTOOL_ENABLE_TEST_BUILD + # override needed so command-line values (make VAR=YES) are normalized, not silently ignored. + # := needed for immediate evaluation of the function call (avoids infinite recursion with =). + override PGXNTOOL_ENABLE_TEST_BUILD := $(call pgxntool_validate_yesno,$(PGXNTOOL_ENABLE_TEST_BUILD),PGXNTOOL_ENABLE_TEST_BUILD) +else + # Auto-detect: enable if test/build/ directory has SQL files + ifneq ($(strip $(TEST_BUILD_FILES)),) + PGXNTOOL_ENABLE_TEST_BUILD = yes + else + PGXNTOOL_ENABLE_TEST_BUILD = no + endif +endif + +# ------------------------------------------------------------------------------ +# test/install: Run setup files before all tests in the same pg_regress session +# ------------------------------------------------------------------------------ +# Purpose: Runs files from test/install/ before all test/sql/ files within a +# SINGLE pg_regress invocation via schedule files. This ensures that +# state created by install files (tables, extensions, etc.) persists +# into the main test suite. +# +# Variable: PGXNTOOL_ENABLE_TEST_INSTALL +# - Can be set manually in Makefile or command line +# - Allowed values: "yes" or "no" (case-insensitive) +# - If not set: Auto-detects based on existence of test/install/*.sql files +# - Set to "yes" explicitly to get an error if test/install/ has no SQL files +# (useful to catch accidental deletion of test/install/ contents) +# - Set to "no" explicitly to disable even when test/install/ has SQL files +# +# Directory layout (follows ~/code/extensions/archive/ pattern): +# test/install/*.sql - Install SQL files +# test/install/*.out - Expected output (lives alongside .sql files) +# test/install/schedule - Auto-generated schedule file +# test/sql/schedule - Auto-generated schedule file for regular tests +# +# The schedule files use relative paths (../install/testname) so pg_regress +# resolves install files from their original location without copying. +# +# NOTE: The variable normalization pattern below (ifdef/NORM/error/override) is +# identical to test-build and verify-results. Refactoring options: +# 1. A $(call normalize_bool_var,VAR,DEFAULT) Make function +# 2. A small include fragment (e.g. pgxntool/mk/bool-var.mk) +# Either approach would eliminate the ~10-line block repeated for each feature. +TEST_INSTALL_SQL_FILES = $(wildcard $(TESTDIR)/install/*.sql) +ifdef PGXNTOOL_ENABLE_TEST_INSTALL + # override needed so command-line values (make VAR=YES) are normalized, not silently ignored. + # := needed for immediate evaluation of the function call (avoids infinite recursion with =). + override PGXNTOOL_ENABLE_TEST_INSTALL := $(call pgxntool_validate_yesno,$(PGXNTOOL_ENABLE_TEST_INSTALL),PGXNTOOL_ENABLE_TEST_INSTALL) +else + # Auto-detect: enable if test/install/ directory has SQL files + ifneq ($(strip $(TEST_INSTALL_SQL_FILES)),) + PGXNTOOL_ENABLE_TEST_INSTALL = yes + else + PGXNTOOL_ENABLE_TEST_INSTALL = no + endif +endif + +# ------------------------------------------------------------------------------ +# verify-results: Safeguard for make results +# ------------------------------------------------------------------------------ +# Purpose: Prevents accidentally running 'make results' when tests are failing. +# +# Variable: PGXNTOOL_ENABLE_VERIFY_RESULTS +# - Can be set manually in Makefile or command line +# - Allowed values: "yes" or "no" (case-insensitive) +# - Setting to empty on the command line (e.g. PGXNTOOL_ENABLE_VERIFY_RESULTS=) also disables the feature +# - If not set: Defaults to "yes" (enabled by default for all pgxntool projects) +# - Usage: Controls whether verify-results target exists and blocks make results +# +# Variable: PGXNTOOL_VERIFY_RESULTS_MODE +# - Controls how verify-results detects test failures +# - "pgtap" (default): scans test/results/*.out for "not ok" lines and plan +# mismatches (TAP failures). Also checks regression.diffs as a fallback. +# Use this mode when your test suite uses pgTap. +# - "diffs": checks only for regression.diffs existence (classic pg_regress behavior) +# Use this mode when your tests use plain SQL expected-output comparison only. +# +# Implementation: See verify-results target definition and results target modification +# (search for "verify-results" and "results:" in this file) +# +ifdef PGXNTOOL_ENABLE_VERIFY_RESULTS + override PGXNTOOL_ENABLE_VERIFY_RESULTS := $(call pgxntool_validate_yesno,$(PGXNTOOL_ENABLE_VERIFY_RESULTS),PGXNTOOL_ENABLE_VERIFY_RESULTS) +else + # Default to yes (enabled by default for all pgxntool projects) + PGXNTOOL_ENABLE_VERIFY_RESULTS = yes +endif + +# Default mode: pgtap (scans results/*.out for TAP failures) +PGXNTOOL_VERIFY_RESULTS_MODE ?= pgtap + # Generate unique database name for tests to prevent conflicts across projects # Uses project name + first 5 chars of md5 hash of current directory # This prevents multiple test runs in different directories from clobbering each other REGRESS_DBHASH := $(shell echo $(CURDIR) | (md5 2>/dev/null || md5sum) | cut -c1-5) REGRESS_DBNAME := $(or $(PGXN),regression)_$(REGRESS_DBHASH) -REGRESS_OPTS += --dbname=$(REGRESS_DBNAME) MODULES = $(patsubst %.c,%,$(wildcard src/*.c)) ifeq ($(strip $(MODULES)),) MODULES =# Set to NUL so PGXS doesn't puke endif -EXTRA_CLEAN = $(wildcard ../$(PGXN)-*.zip) $(TEST__SOURCE__SQL_FILES) $(TEST__SOURCE__EXPECTED_FILES) pg_tle/ +EXTRA_CLEAN = $(wildcard ../$(PGXN)-*.zip) pg_tle/ # Get Postgres version, as well as major (9.4, etc) version. # NOTE! In at least some versions, PGXS defines VERSION, so we intentionally don't use that variable @@ -93,8 +209,37 @@ ifeq ($(GE91),yes) all: $(EXTENSION_VERSION_FILES) endif -ifeq ($($call test, $(MAJORVER), -lt 13), yes) - REGRESS_OPTS += --load-language=plpgsql +ifeq ($(call test, $(MAJORVER), -lt, 130), yes) +REGRESS_OPTS += --load-language=plpgsql +endif + +# +# test/install: Schedule-based approach +# +# When enabled, generates a schedule file listing install files, and adds it +# to REGRESS_OPTS. pg_regress processes --schedule tests before command-line +# test names, so install files run first in the SAME pg_regress invocation. +# This ensures state created by install files persists into the main test suite. +# +# The schedule uses relative paths (../install/testname) so pg_regress finds +# install files in their original location without copying. +# +ifeq ($(PGXNTOOL_ENABLE_TEST_INSTALL),yes) +PGXNTOOL_INSTALL_SCHEDULE = $(TESTDIR)/install/schedule +EXTRA_CLEAN += $(PGXNTOOL_INSTALL_SCHEDULE) + +# Add install schedule; REGRESS stays as-is (regular tests run after schedule) +REGRESS_OPTS += --schedule=$(PGXNTOOL_INSTALL_SCHEDULE) + +# Always regenerate schedule file to catch added/removed files +.PHONY: $(PGXNTOOL_INSTALL_SCHEDULE) +$(PGXNTOOL_INSTALL_SCHEDULE): + @echo "# Auto-generated - DO NOT EDIT" > $@ + @for f in $(notdir $(basename $(TEST_INSTALL_SQL_FILES))); do \ + echo "test: ../install/$$f" >> $@; \ + done + +installcheck: $(PGXNTOOL_INSTALL_SCHEDULE) endif PGXS := $(shell $(PG_CONFIG) --pgxs) @@ -104,7 +249,7 @@ DATA += $(wildcard *.control) # Don't have installcheck bomb on error .IGNORE: installcheck -installcheck: $(TEST_RESULT_FILES) $(TEST_SQL_FILES) $(TEST__SOURCE__INPUT_FILES) | $(TESTDIR)/sql/ $(TESTDIR)/expected/ $(TESTOUT)/results/ +installcheck: $(TEST_RESULT_FILES) $(TEST_SQL_FILES) | $(TESTDIR)/sql/ $(TESTDIR)/expected/ $(TESTOUT)/results/ # # TEST SUPPORT @@ -118,31 +263,55 @@ installcheck: $(TEST_RESULT_FILES) $(TEST_SQL_FILES) $(TEST__SOURCE__INPUT_FILES # watch-make if you're generating intermediate files. If tests end up needing # clean it's an indication of a missing dependency anyway. .PHONY: test -test: testdeps install installcheck +# Build test dependencies list based on enabled features +TEST_DEPS = testdeps +ifeq ($(PGXNTOOL_ENABLE_TEST_BUILD),yes) +TEST_DEPS += test-build +endif +TEST_DEPS += install installcheck +test: $(TEST_DEPS) @if [ -r $(TESTOUT)/regression.diffs ]; then cat $(TESTOUT)/regression.diffs; fi -# make results: runs `make test` and copy all result files to expected -# DO NOT RUN THIS UNLESS YOU'RE CERTAIN ALL YOUR TESTS ARE PASSING! # -# pg_regress workflow: -# 1. Converts input/*.source → sql/*.sql (with token substitution) -# 2. Converts output/*.source → expected/*.out (with token substitution) -# 3. Runs tests, saving actual output in results/ -# 4. Compares results/ with expected/ +# verify-results: Safeguard for make results # -# NOTE: Both input/*.source and output/*.source are COMPLETELY OPTIONAL and are -# very rarely needed. pg_regress does NOT create the input/ or output/ directories -# - these are optional INPUT directories that users create if they need them. -# Most extensions will never need these directories. +# Checks if tests are passing before allowing make results to proceed +ifeq ($(PGXNTOOL_ENABLE_VERIFY_RESULTS),yes) +.PHONY: verify-results +ifeq ($(PGXNTOOL_VERIFY_RESULTS_MODE),pgtap) +verify-results: + @$(PGXNTOOL_DIR)/verify-results-pgtap.sh $(TESTOUT) +else +verify-results: + @if [ -r $(TESTOUT)/regression.diffs ]; then \ + echo "ERROR: Tests are failing. Cannot run 'make results'."; \ + echo "Fix test failures first, then run 'make results'."; \ + echo ""; \ + echo "See $(TESTOUT)/regression.diffs for details:"; \ + cat $(TESTOUT)/regression.diffs; \ + exit 1; \ + fi +endif +endif + +# make results: runs `make test` and copies all result files to expected. +# DO NOT RUN THIS UNLESS YOU'RE CERTAIN ALL YOUR TESTS ARE PASSING! # -# CRITICAL: Do NOT copy files that have corresponding output/*.source files, because -# those are the source of truth and will be regenerated by pg_regress from the .source files. -# Only copy files from results/ that don't have output/*.source counterparts. +# Dependency chain (verify-results: test) guarantees test completes before verify-results +# checks regression.diffs, even under make -j. Listing both as independent prerequisites +# of results would allow them to run concurrently, letting verify-results see stale state. .PHONY: results +ifeq ($(PGXNTOOL_ENABLE_VERIFY_RESULTS),yes) +verify-results: test +results: verify-results +else results: test - @# Copy .out files from results/ to expected/, excluding those with output/*.source counterparts - @# .out files with output/*.source counterparts are generated from .source files and should NOT be overwritten - @$(PGXNTOOL_DIR)/make_results.sh $(TESTDIR) $(TESTOUT) +endif + @mkdir -p $(TESTDIR)/expected + @for f in $(TESTOUT)/results/*.out; do \ + [ -f "$$f" ] || continue; \ + cp "$$f" $(TESTDIR)/expected/$$(basename "$$f"); \ + done # testdeps is a generic dependency target that you can add targets to .PHONY: testdeps @@ -196,9 +365,34 @@ run-pgtle: pgtle # These targets ensure all the relevant directories exist $(TESTDIR)/sql $(TESTDIR)/expected/ $(TESTOUT)/results/: @mkdir -p $@ +# pg_regress aborts with "could not open file" if an expected output file is +# missing, so create empty placeholders for any test that lacks one. $(TEST_RESULT_FILES): | $(TESTDIR)/expected/ + @# Create empty expected file so pg_regress doesn't abort with "file not found". + @# pg_regress requires an expected/*.out file to exist for each test; without it + @# it stops immediately rather than running the test and showing the diff. @touch $@ +# +# test-build: Sanity check extension files in test/build/ +# +# The sql/ subdirectory is generated - files are synced from test/build/*.sql. +# This directory should be in .gitignore and is cleaned by make clean. +# +ifeq ($(PGXNTOOL_ENABLE_TEST_BUILD),yes) +TEST_BUILD_SQL_DIR = $(TESTDIR)/build/sql +TEST_BUILD_REGRESS = $(sort $(notdir $(basename $(TEST_BUILD_SQL_FILES)))) +.PHONY: test-build +test-build: install + @$(PGXNTOOL_DIR)/run-test-build.sh $(TESTDIR) + $(MAKE) -C . REGRESS="$(TEST_BUILD_REGRESS)" REGRESS_OPTS="--inputdir=$(TESTDIR)/build --outputdir=$(TESTDIR)/build" installcheck + @if [ -r $(TESTDIR)/build/regression.diffs ]; then \ + echo "test-build failed - see $(TESTDIR)/build/regression.diffs"; \ + cat $(TESTDIR)/build/regression.diffs; \ + exit 1; \ + fi +endif + # # DOC SUPPORT @@ -227,8 +421,9 @@ dist: html # But don't add it as an install or test dependency unless we do have asciidoc ifneq (,$(strip $(ASCIIDOC))) -# Need to do this so install & co will pick up ALL targets. Unfortunately this can result in some duplication. -DOCS += $(ASCIIDOC_HTML) +# Add HTML to DOCS for install, deduplicating against any HTML already picked +# up by the wildcard (e.g. pre-built HTML committed to the repo). +DOCS := $(sort $(filter-out $(ASCIIDOC_HTML),$(DOCS)) $(ASCIIDOC_HTML)) # Also need to add html as a dep to all (which will get picked up by install & installcheck all: html @@ -296,28 +491,33 @@ print-% : ; $(info $* is $(flavor $*) variable set to "$($*)") @true # # subtree sync support # -# This is setup to allow any number of pull targets by defining special -# variables. pgxntool-sync-release is an example of this. +# All the real work (git subtree pull + update-setup-files.sh) lives in +# pgxntool/pgxntool-sync.sh so it can be run directly, without make. These +# targets are thin wrappers around that script. +# +# `make pgxntool-sync` pulls the latest released version from the canonical +# repository (the script's built-in default). # -# After the subtree pull, we run update-setup-files.sh to handle files that -# were initially copied by setup.sh (like .gitignore). This script does a -# 3-way merge if both you and pgxntool changed the file. -.PHONY: pgxntool-sync-% +# `make pgxntool-sync-` pulls from the " " defined by the +# pgxntool-sync- variable, allowing any number of custom pull sources. +.PHONY: pgxntool-sync pgxntool-sync-% +pgxntool-sync: + @pgxntool/pgxntool-sync.sh pgxntool-sync-%: - @old_commit=$$(git log -1 --format=%H -- pgxntool/) && \ - git subtree pull -P pgxntool --squash -m "Pull pgxntool from $($@)" $($@) && \ - pgxntool/update-setup-files.sh "$$old_commit" -pgxntool-sync: pgxntool-sync-release + @pgxntool/pgxntool-sync.sh $($@) # DANGER! Use these with caution. They may add extra crap to your history and # could make resolving merges difficult! -pgxntool-sync-release := git@github.com:decibel/pgxntool.git release -pgxntool-sync-stable := git@github.com:decibel/pgxntool.git stable -pgxntool-sync-master := git@github.com:decibel/pgxntool.git master -pgxntool-sync-local := ../pgxntool release # Not the same as PGXNTOOL_DIR! -pgxntool-sync-local-stable := ../pgxntool stable # Not the same as PGXNTOOL_DIR! -pgxntool-sync-local-master := ../pgxntool master # Not the same as PGXNTOOL_DIR! - +# `pgxntool-sync` (no suffix) already pulls the canonical release; these are the +# alternatives. `-master` pulls the bleeding edge; `-local*` pull from a sibling +# ../pgxntool checkout (not the same as PGXNTOOL_DIR!). +pgxntool-sync-master := https://github.com/Postgres-Extensions/pgxntool.git master +pgxntool-sync-local := ../pgxntool release +pgxntool-sync-local-master := ../pgxntool master + +# PGXS doesn't provide any special support for distclean (it just depends on +# clean), so we roll our own. Files that should only be removed by distclean +# (not clean) are added to PGXNTOOL_distclean near their build rules above. distclean: rm -f $(PGXNTOOL_distclean) @@ -328,6 +528,22 @@ DOCS =# Set to NUL so PGXS doesn't puke endif include $(PGXS) + +# Override CONTRIB_TESTDB (set unconditionally by PGXS) with our unique database +# name. This must be after include $(PGXS) because PGXS uses = (not ?=). +# PGXS appends --dbname=$(CONTRIB_TESTDB) to REGRESS_OPTS, so overriding +# CONTRIB_TESTDB is the correct way to control the database name — adding our +# own --dbname would result in two --dbname flags passed to pg_regress. +CONTRIB_TESTDB = $(REGRESS_DBNAME) + +# Clean generated sql/ directory for test-build +ifeq ($(PGXNTOOL_ENABLE_TEST_BUILD),yes) +.PHONY: clean-test-build +clean-test-build: + rm -rf $(TEST_BUILD_SQL_DIR) +clean: clean-test-build +endif + # # pgtap # diff --git a/pgxntool/build_meta.sh b/pgxntool/build_meta.sh index e0fd6b2..4df75bb 100755 --- a/pgxntool/build_meta.sh +++ b/pgxntool/build_meta.sh @@ -1,4 +1,23 @@ -#!/bin/bash +#!/usr/bin/env bash + +# Build META.json from META.in.json template +# +# WHY META.in.json EXISTS: +# META.in.json serves as a template that: +# 1. Shows all possible PGXN metadata fields (both required and optional) with comments +# 2. Can have empty placeholder fields like "key": "" or "key": [ "", "" ] +# 3. Users edit this to fill in their extension's metadata +# +# WHY WE GENERATE META.json: +# The reason we generate META.json from a template is to eliminate empty fields that +# are optional; PGXN.org gets upset about them. In the future it's possible we'll do +# more here (for example, if we added more info to the template we could use it to +# generate control files). +# +# WHY WE COMMIT META.json: +# PGXN.org requires META.json to be present in submitted distributions. We choose +# to commit it to git instead of manually adding it to distributions for simplicity +# (and since it generally only changes once for each new version). # Build META.json from META.in.json template # diff --git a/pgxntool/lib.sh b/pgxntool/lib.sh index 41de512..3cbeb0e 100644 --- a/pgxntool/lib.sh +++ b/pgxntool/lib.sh @@ -1,4 +1,3 @@ -#!/bin/bash # lib.sh - Common utility functions for pgxntool scripts # # This file is meant to be sourced by other scripts, not executed directly. @@ -36,15 +35,46 @@ die() { exit $exit_code } +# Returns true if an array isn't empty. +# +# array_not_empty "${#errors[@]}" +# +# BUT WHY ON EARTH DO THIS?? +# +# This wraps a one-liner intentionally. The function forces any reader +# (human or AI agent) to navigate here and read this comment before +# "simplifying" the call site. Without it, the natural next step is to +# inline the expression — and the natural inline form breaks bash 3.2. +# +# On bash 3.2 (Mac OS default), when using `set -u`, expanding "${arr[@]}" +# on an empty array triggers "unbound variable" even when the array was +# explicitly initialized with arr=(). +# +# The comment inside the function body exists to catch any agent or human who +# navigates to the function without reading this comment first. +array_not_empty() { + # DO NOT EDIT THIS FUNCTION! DO NOT REMOVE THIS COMMENT! (see main function comment) + [ "${1:-0}" -gt 0 ] +} + # Debug function # Usage: debug LEVEL "message" # Outputs message to stderr if DEBUG >= LEVEL -# Debug levels use multiples of 10 (10, 20, 30, 40, etc.) to allow for easy expansion +# +# LEVEL encodes how noisy/esoteric a message is -- roughly, how far you'd crank +# DEBUG before you'd actually want to see it. Higher = noisier, more rarely +# useful. This is signal-to-noise, NOT code nesting depth: a top-level line can +# warrant a high level if it's esoteric, and loop-body detail is usually high +# precisely because it's noisy. +# +# The tiers below are anchors, not strict multiples -- pick any value in range +# to fine-tune between existing calls without renumbering: # - 10: Critical errors, important warnings # - 20: Warnings, significant state changes # - 30: General debugging, function entry/exit, array operations # - 40: Verbose details, loop iterations -# - 50+: Maximum verbosity +# - 50+: Maximum verbosity (per-iteration innards) +# # Enable with: DEBUG=30 scriptname.sh debug() { local level=$1 @@ -55,3 +85,26 @@ debug() { echo "DEBUG[$level]: $message" >&2 fi } + +# Remove pgxntool's own dev-only directories from a consuming project. +# +# `git subtree` copies the ENTIRE pgxntool tree into the consumer, including +# dev-only dirs like .github/ (pgxntool's CI) and .claude/. Those are +# export-ignored from `make dist` and don't belong in a project that merely +# embeds pgxntool. (GitHub only runs workflows at the repo root, so a consumer's +# pgxntool/.github never executes anyway — but it's still clutter.) git subtree +# doesn't honor export-ignore, so we prune them here after a sync. +# +# Must be run from the project root (the dir containing pgxntool/). Safe to call +# repeatedly; a no-op once the dirs are gone. +prune_pgxntool_dev_dirs() { + local d + for d in .github .claude; do + [ -e "pgxntool/$d" ] || continue + echo " pgxntool/$d: pruning (pgxntool dev-only, not for embedding projects)" + # Stage the removal if tracked; rm -rf guarantees it's gone even if not. + # || : keeps this best-effort under `set -e` (rm -rf is the real cleanup). + git rm -rq --ignore-unmatch "pgxntool/$d" >/dev/null 2>&1 || : + rm -rf "pgxntool/$d" + done +} diff --git a/pgxntool/pgtle.sh b/pgxntool/pgtle.sh index abbcfd4..124ca84 100755 --- a/pgxntool/pgtle.sh +++ b/pgxntool/pgtle.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # pgtle.sh - Generate pg_tle registration SQL for PostgreSQL extensions # @@ -459,16 +459,26 @@ parse_control_file() { local key="${BASH_REMATCH[1]}" local value="${BASH_REMATCH[2]}" + # Trim trailing comments and whitespace FIRST, then strip quotes. + # Order matters: stripping quotes before removing comments leaves a rogue + # trailing quote for values like 'version' # note. See issue #25. + value="${value%%#*}" # Remove trailing comments + # Trim all trailing whitespace (spaces and tabs). The prior %% pattern with + # a literal space only removed one character; multiple spaces or a tab before + # the comment (e.g., 'value' # note or 'value'$'\t'# note) would leave + # stray whitespace that breaks the quote-strip below. + if [[ "$value" =~ ^(.*[^[:space:]])[[:space:]]*$ ]]; then + value="${BASH_REMATCH[1]}" + else + value="" + fi + # Strip quotes if present (both single and double) value="${value#\'}" value="${value%\'}" value="${value#\"}" value="${value%\"}" - # Trim trailing whitespace/comments - value="${value%%#*}" # Remove trailing comments - value="${value%% }" # Trim trailing spaces - # Store in global variables case "$key" in default_version) DEFAULT_VERSION="$value" ;; @@ -518,14 +528,14 @@ discover_sql_files() { local default_version_file="sql/${EXTENSION}--${DEFAULT_VERSION}.sql" local base_file="sql/${EXTENSION}.sql" if [ -f "$base_file" ] && ([ ! -f "$default_version_file" ] || [ ! -s "$default_version_file" ]); then - debug 30 "discover_sql_files: Creating default_version file from base file" + debug 40 "discover_sql_files: Creating default_version file from base file" cp "$base_file" "$default_version_file" fi # Find versioned files: sql/{ext}--{version}.sql # Use find to get proper null-delimited output, then filter out upgrade scripts VERSION_FILES=() # Reset array - debug 30 "discover_sql_files: Reset VERSION_FILES array" + debug 40 "discover_sql_files: Reset VERSION_FILES array" while IFS= read -r -d '' file; do local basename=$(basename "$file" .sql) local dash_count=$(echo "$basename" | grep -o -- "--" | wc -l | tr -d '[:space:]') @@ -543,7 +553,7 @@ discover_sql_files() { # Find upgrade scripts: sql/{ext}--{ver1}--{ver2}.sql # These have TWO occurrences of "--" in the filename UPGRADE_FILES=() # Reset array - debug 30 "discover_sql_files: Reset UPGRADE_FILES array" + debug 40 "discover_sql_files: Reset UPGRADE_FILES array" while IFS= read -r -d '' file; do # Empty upgrade files are allowed (no-op upgrades) local basename=$(basename "$file" .sql) @@ -564,8 +574,8 @@ discover_sql_files() { echo " - $f" >&2 done - debug 30 "discover_sql_files: Checking UPGRADE_FILES array, count=${#UPGRADE_FILES[@]:-0}" - if [ ${#UPGRADE_FILES[@]:-0} -gt 0 ]; then + debug 40 "discover_sql_files: Checking UPGRADE_FILES array, count=${#UPGRADE_FILES[@]}" + if array_not_empty "${#UPGRADE_FILES[@]}"; then echo " Found ${#UPGRADE_FILES[@]} upgrade script(s):" >&2 debug 30 "discover_sql_files: Iterating over ${#UPGRADE_FILES[@]} upgrade files" for f in "${UPGRADE_FILES[@]}"; do @@ -633,8 +643,8 @@ build_requires_array() { generate_header() { local pgtle_version="$1" local output_file="$2" - local version_count=${#VERSION_FILES[@]:-0} - local upgrade_count=${#UPGRADE_FILES[@]:-0} + local version_count=${#VERSION_FILES[@]} + local upgrade_count=${#UPGRADE_FILES[@]} # Determine version compatibility message local compat_msg @@ -759,9 +769,9 @@ generate_pgtle_sql() { # Ensure arrays are initialized (defensive programming) # Arrays should already be initialized at top level, but ensure they exist - debug 30 "generate_pgtle_sql: Checking array initialization" - debug 30 "generate_pgtle_sql: VERSION_FILES is ${VERSION_FILES+set}, count=${#VERSION_FILES[@]:-0}" - debug 30 "generate_pgtle_sql: UPGRADE_FILES is ${UPGRADE_FILES+set}, count=${#UPGRADE_FILES[@]:-0}" + debug 40 "generate_pgtle_sql: Checking array initialization" + debug 30 "generate_pgtle_sql: VERSION_FILES is ${VERSION_FILES+set}, count=${#VERSION_FILES[@]}" + debug 30 "generate_pgtle_sql: UPGRADE_FILES is ${UPGRADE_FILES+set}, count=${#UPGRADE_FILES[@]}" if [ -z "${VERSION_FILES+set}" ]; then echo "WARNING: VERSION_FILES not set, initializing" >&2 @@ -807,7 +817,7 @@ EOF fi # Install base version (first version file) - if [ ${#VERSION_FILES[@]} -gt 0 ]; then + if array_not_empty "${#VERSION_FILES[@]}"; then generate_install_extension "${VERSION_FILES[0]}" "$capability" fi @@ -819,14 +829,15 @@ EOF fi # Install all upgrade paths - local upgrade_count=${#UPGRADE_FILES[@]:-0} - debug 30 "generate_pgtle_sql: upgrade_count=$upgrade_count" + local upgrade_count=${#UPGRADE_FILES[@]} + debug 40 "generate_pgtle_sql: upgrade_count=$upgrade_count" if [ "$upgrade_count" -gt 0 ]; then debug 30 "generate_pgtle_sql: Processing $upgrade_count upgrade path(s)" - local i - for ((i=0; i (3-way merge files setup.sh copied out, +# like .gitignore and test/deps.sql) +# +# It is invoked by the `make pgxntool-sync` targets, but can also be run directly +# so you never need make to update pgxntool. +# +# Usage: pgxntool-sync.sh [ []] +# +# repo Git URL (or path) to pull pgxntool from. Defaults to the canonical +# pgxntool repository. +# ref Branch, tag, or commit to pull. Defaults to the `release` tag, which +# always points at the latest released version. +# +# Run from the root of your project (the directory containing pgxntool/). + +set -o errexit -o errtrace -o pipefail +trap 'echo "Error on line ${LINENO}"' ERR + +PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")" +source "$PGXNTOOL_DIR/lib.sh" + +# Canonical source for pgxntool. `release` is a floating tag that the release +# process force-moves to each new version, so it always names the latest +# released version -- exactly what a plain sync should track. +DEFAULT_REPO="https://github.com/Postgres-Extensions/pgxntool.git" +DEFAULT_REF="release" + +repo=${1:-$DEFAULT_REPO} +ref=${2:-$DEFAULT_REF} + +# We must run from the project root: git subtree pull operates on the pgxntool/ +# prefix and update-setup-files.sh resolves paths relative to the current dir. +[[ -d "pgxntool" ]] || die 1 "pgxntool directory not found. Run from your project root." +# Use rev-parse, not [ -d .git ]: in a worktree .git is a file, not a directory. +git rev-parse --git-dir >/dev/null 2>&1 || die 1 "Not in a git repository. Run from your project root." + +# The old commit is the pgxntool subtree HEAD before the pull; update-setup-files.sh +# needs it as the merge base for files that were copied out of pgxntool. +old_commit=$(git log -1 --format=%H -- pgxntool/) + +git subtree pull -P pgxntool --squash -m "Pull pgxntool from $repo $ref" "$repo" "$ref" + +"$PGXNTOOL_DIR/update-setup-files.sh" "$old_commit" diff --git a/pgxntool/run-test-build.sh b/pgxntool/run-test-build.sh new file mode 100755 index 0000000..86a61eb --- /dev/null +++ b/pgxntool/run-test-build.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# pgxntool/run-test-build.sh - Prepare test/build/ for the test-build target +# +# Sets up the generated sql/ directory and ensures expected/*.out files exist +# so pg_regress can run without aborting on missing files. +# +# Usage: run-test-build.sh TESTDIR +# +# Called by the test-build target in base.mk before running installcheck. + +set -e + +TESTDIR="${1:?Usage: run-test-build.sh TESTDIR}" +BUILD_DIR="$TESTDIR/build" +SQL_DIR="$BUILD_DIR/sql" +EXPECTED_DIR="$BUILD_DIR/expected" + +mkdir -p "$SQL_DIR" +mkdir -p "$EXPECTED_DIR" + +# Verify .sql files exist. This script is only called when test-build is +# enabled, so missing files indicate a misconfiguration. +files=("$BUILD_DIR"/*.sql) +if [ ! -f "${files[0]}" ]; then + echo "ERROR: no .sql files found in $BUILD_DIR/" >&2 + exit 1 +fi + +# Sync .sql files to sql/ directory for pg_regress. +# --checksum: compare by content, not size+mtime. rsync's default "quick check" +# assumes equal size+mtime means files are identical, which isn't safe here — +# builds can produce identical-sized files with different content. Checksum +# comparison also avoids unnecessary writes that could trigger antivirus. +# --times: preserve source mtimes on destination files so make's dependency +# tracking works correctly. +# --delete: remove files from sql/ that no longer exist in build/. +# --include/--exclude: select only *.sql from the directory source +# (--delete requires a directory transfer, not individual file arguments). +rsync -r --checksum --times --delete --include='*.sql' --exclude='*' "$BUILD_DIR/" "$SQL_DIR/" + +# Create empty expected/*.out files for .sql tests (if not already present). +# pg_regress requires an expected file to exist for each test; without it +# pg_regress stops immediately rather than running the test and showing the diff. +for file in "$BUILD_DIR"/*.sql; do + out="$EXPECTED_DIR/$(basename "$file" .sql).out" + [ -f "$out" ] || touch "$out" +done diff --git a/pgxntool/setup.sh b/pgxntool/setup.sh index 0a67f0e..54b3633 100755 --- a/pgxntool/setup.sh +++ b/pgxntool/setup.sh @@ -7,7 +7,9 @@ trap 'echo "Error on line ${LINENO}"' ERR PGXNTOOL_DIR="$(dirname "${BASH_SOURCE[0]}")" source "$PGXNTOOL_DIR/lib.sh" -[ -d .git ] || git init +# Use rev-parse, not [ -d .git ]: in a worktree .git is a file, not a directory, +# so the old check would wrongly re-run `git init` inside a valid working tree. +git rev-parse --git-dir >/dev/null 2>&1 || git init if ! git diff --cached --exit-code; then echo "Git repository is not clean; please commit and try again." >&2 diff --git a/pgxntool/update-setup-files.sh b/pgxntool/update-setup-files.sh index 94ae75d..f814605 100755 --- a/pgxntool/update-setup-files.sh +++ b/pgxntool/update-setup-files.sh @@ -146,9 +146,10 @@ process_symlink() { old_commit=$1 -# Verify we're in a git repo with pgxntool subtree +# Verify we're in a git repo with pgxntool subtree. +# Use rev-parse, not [ -d .git ]: in a worktree .git is a file, not a directory. [[ -d "pgxntool" ]] || die 1 "pgxntool directory not found. Run from project root." -[[ -d ".git" ]] || die 1 "Not in a git repository." +git rev-parse --git-dir >/dev/null 2>&1 || die 1 "Not in a git repository." # Verify the old commit is valid if ! git cat-file -e "${old_commit}^{commit}" 2>/dev/null; then @@ -172,5 +173,9 @@ for entry in "${SETUP_SYMLINKS[@]}"; do process_symlink "$dest" "$target" done +# Prune pgxntool's own dev-only dirs (.github/, .claude/) that the subtree pull +# re-introduces but that don't belong in a consuming project (see lib.sh). +prune_pgxntool_dev_dirs + echo echo "Done. Review changes with 'git diff' and commit when ready." diff --git a/pgxntool/verify-results-pgtap.sh b/pgxntool/verify-results-pgtap.sh new file mode 100755 index 0000000..dd8d7e9 --- /dev/null +++ b/pgxntool/verify-results-pgtap.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# pgxntool/verify-results-pgtap.sh - Check pgtap results before 'make results' +# +# Scans pgtap output files for failures and plan mismatches, then checks +# regression.diffs as a fallback. Exits non-zero if any problems are found. +# +# Usage: verify-results-pgtap.sh TESTOUT +# +# Called by the verify-results target in base.mk (pgtap mode). + +set -e + +TESTOUT="${1:?Usage: verify-results-pgtap.sh TESTOUT}" + +# Check for pgtap failures in result files (excluding TODO items) +failed=0 +for f in "$TESTOUT"/results/*.out; do + [ -f "$f" ] || continue + if grep -q '^not ok' "$f"; then + notok=$(grep '^not ok' "$f" | grep -v '# TODO' || true) + if [ -n "$notok" ]; then + echo "ERROR: pgtap failure detected in $f" + echo "$notok" + failed=1 + fi + fi + if grep -q 'Looks like you planned' "$f"; then + echo "ERROR: pgtap plan mismatch in $f" + grep 'Looks like you planned' "$f" + failed=1 + fi +done +if [ $failed -ne 0 ]; then + echo + echo "pgtap failures detected. Cannot run 'make results'." + exit 1 +fi + +# Also check regression.diffs (output mismatch even if pgtap all passed) +if [ -r "$TESTOUT/regression.diffs" ]; then + echo "ERROR: Tests are failing. Cannot run 'make results'." + echo "Fix test failures first, then run 'make results'." + echo + echo "See $TESTOUT/regression.diffs for details:" + cat "$TESTOUT/regression.diffs" + exit 1 +fi