From fe553a0da82bc6e4df0e358fba829a9d8227bd2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Thu, 8 Oct 2026 03:12:43 +0300 Subject: [PATCH 01/11] perf(nix): split debug symbols out of postgres-env Add postgres-env-15-debug and postgres-env-17-debug with the debug output only. Drop the source and debug paths from postgres-env-15 and postgres-env-17. Add realise-postgres-debug to fetch the matching debug env from the binary cache. Keep source and debug in postgres-env-orioledb-17 for its testinfra tests and coredump diagnostics. Refs: MPG-1336 Co-Authored-By: Claude Sonnet 5.5 --- nix/packages/postgres-env.nix | 39 ++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/nix/packages/postgres-env.nix b/nix/packages/postgres-env.nix index 0ae009fff7..8645800947 100644 --- a/nix/packages/postgres-env.nix +++ b/nix/packages/postgres-env.nix @@ -9,6 +9,29 @@ let # Make a bundle of packages, as a single derivation, to be installed into the # postgres user's nix profile, during image provisioning or instance update. + debugPaths = + version: + lib.optionals pkgs.stdenv.isLinux [ self'.packages."postgresql_${version}_debug" ] + # orioledb.so ships as a separate extension package (nix/ext/orioledb.nix), not + # part of the postgresql derivation itself, so its debug output isn't covered by + # postgresql_${version}_debug above and has to be pulled in explicitly. + ++ lib.optionals (pkgs.stdenv.isLinux && version == "orioledb-17") [ + self'.legacyPackages."psql_${version}".exts.orioledb.debug + ]; + + makePostgresEnvDebug = + version: + pkgs.symlinkJoin { + name = "postgres-env-${version}-debug"; + paths = debugPaths version; + }; + + realisePostgresDebug = + version: + pkgs.writeShellScriptBin "realise-postgres-debug" '' + exec nix-store --realise ${builtins.unsafeDiscardStringContext (makePostgresEnvDebug version).outPath} + ''; + makePostgresEnv = version: pkgs.symlinkJoin { @@ -17,16 +40,14 @@ self'.packages."psql_${version}/bin" self'.packages.pg_prove self'.packages.supabase-groonga - self'.packages."postgresql_${version}_src" ] - ++ lib.optionals pkgs.stdenv.isLinux [ self'.packages."postgresql_${version}_debug" ] ++ lib.optionals (pkgs.stdenv.isLinux && version != "15") [ self'.packages.gatekeeper ] - # orioledb.so ships as a separate extension package (nix/ext/orioledb.nix), not - # part of the postgresql derivation itself, so its debug output isn't covered by - # postgresql_${version}_debug above and has to be pulled in explicitly. - ++ lib.optionals (pkgs.stdenv.isLinux && version == "orioledb-17") [ - self'.legacyPackages."psql_${version}".exts.orioledb.debug - ]; + ++ ( + if version == "orioledb-17" then + [ self'.packages."postgresql_${version}_src" ] ++ debugPaths version + else + lib.optionals pkgs.stdenv.isLinux [ (realisePostgresDebug version) ] + ); }; in { @@ -34,6 +55,8 @@ postgres-env-15 = makePostgresEnv "15"; postgres-env-17 = makePostgresEnv "17"; postgres-env-orioledb-17 = makePostgresEnv "orioledb-17"; + postgres-env-15-debug = makePostgresEnvDebug "15"; + postgres-env-17-debug = makePostgresEnvDebug "17"; }; }; } From 45f9f61629fe701c7efb47c2d49127b2b3eedbd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Thu, 8 Oct 2026 07:48:01 +0300 Subject: [PATCH 02/11] perf(nix): strip pgrx extensions and ship their debug info separately Strip pgrx .so files fully and build them with separateDebugInfo. Add the per-version debug outputs of wrappers, pg_graphql, and pg_jsonschema to the debug env. realise-postgres-debug installs the debug env into the postgres-debug profile. Refs: MPG-1336 Co-Authored-By: Claude Sonnet 5.5 --- nix/cargo-pgrx/buildPgrxExtension.nix | 2 ++ nix/ext/pg_graphql/default.nix | 1 + nix/ext/pg_jsonschema/default.nix | 1 + nix/ext/wrappers/default.nix | 1 + nix/packages/postgres-env.nix | 22 ++++++++++++++++++---- 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/nix/cargo-pgrx/buildPgrxExtension.nix b/nix/cargo-pgrx/buildPgrxExtension.nix index a316015a50..f7fd83bc48 100644 --- a/nix/cargo-pgrx/buildPgrxExtension.nix +++ b/nix/cargo-pgrx/buildPgrxExtension.nix @@ -149,6 +149,8 @@ let # we forgot parentheses finalArgs = argsForBuildRustPackage // { buildInputs = (args.buildInputs or [ ]); + separateDebugInfo = true; + stripAllList = args.stripAllList or [ "lib" ]; nativeBuildInputs = (args.nativeBuildInputs or [ ]) diff --git a/nix/ext/pg_graphql/default.nix b/nix/ext/pg_graphql/default.nix index 39d627400e..f4e132b746 100644 --- a/nix/ext/pg_graphql/default.nix +++ b/nix/ext/pg_graphql/default.nix @@ -184,6 +184,7 @@ in --prefix EXT_WRAPPER : "$out" --prefix EXT_NAME : "${pname}" ''; passthru = { + debug = map (p: p.debug) packages; perVersion = lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; diff --git a/nix/ext/pg_jsonschema/default.nix b/nix/ext/pg_jsonschema/default.nix index cd71e85407..995d76ca89 100644 --- a/nix/ext/pg_jsonschema/default.nix +++ b/nix/ext/pg_jsonschema/default.nix @@ -184,6 +184,7 @@ in ''; passthru = { + debug = map (p: p.debug) packages; perVersion = lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 92eddb1cbe..c83ca90f1f 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -371,6 +371,7 @@ in } ''; passthru = { + debug = map (p: p.debug) packages; perVersion = packagesAttrSet; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; diff --git a/nix/packages/postgres-env.nix b/nix/packages/postgres-env.nix index 8645800947..16ac813fd7 100644 --- a/nix/packages/postgres-env.nix +++ b/nix/packages/postgres-env.nix @@ -19,18 +19,32 @@ self'.legacyPackages."psql_${version}".exts.orioledb.debug ]; + extDebug = + version: + let + exts = self'.legacyPackages."psql_${version}".exts; + in + lib.concatMap (e: e.passthru.debug) [ + exts.wrappers + exts.pg_graphql + exts.pg_jsonschema + ]; + makePostgresEnvDebug = version: pkgs.symlinkJoin { name = "postgres-env-${version}-debug"; - paths = debugPaths version; + paths = debugPaths version ++ lib.optionals pkgs.stdenv.isLinux (extDebug version); }; realisePostgresDebug = version: - pkgs.writeShellScriptBin "realise-postgres-debug" '' - exec nix-store --realise ${builtins.unsafeDiscardStringContext (makePostgresEnvDebug version).outPath} - ''; + pkgs.writeShellApplication { + name = "realise-postgres-debug"; + text = '' + nix-env --profile /nix/var/nix/profiles/postgres-debug --set "$(nix-store --realise ${builtins.unsafeDiscardStringContext (makePostgresEnvDebug version).outPath})" + ''; + }; makePostgresEnv = version: From 7381a395e96301d1ecae6a6c40015a2304346caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Sat, 10 Oct 2026 00:15:24 +0300 Subject: [PATCH 03/11] perf(nix): ship orioledb debug separately and add debug profile scripts Add postgres-env-orioledb-17-debug with the debug output, orioledb.debug, and the source tree. Every env now carries realise-postgres-debug and cleanup-postgres-debug. The testinfra tests and the coredump processor read the postgres-debug profile, and the tests realise it first. Refs: MPG-1336 Co-Authored-By: Claude Sonnet 5.5 --- .../coredump/process-orioledb-coredumps.sh | 2 +- nix/docs/README.md | 1 + nix/docs/debug-symbols.md | 39 ++++++++++++++ nix/packages/postgres-env.nix | 30 ++++++----- testinfra/test_ami_nix.py | 51 +++++++++++++++---- 5 files changed, 100 insertions(+), 23 deletions(-) create mode 100644 nix/docs/debug-symbols.md diff --git a/ansible/files/coredump/process-orioledb-coredumps.sh b/ansible/files/coredump/process-orioledb-coredumps.sh index 2ce913f619..c8133c4b61 100644 --- a/ansible/files/coredump/process-orioledb-coredumps.sh +++ b/ansible/files/coredump/process-orioledb-coredumps.sh @@ -25,7 +25,7 @@ EXTRACTION_TIMEOUT=120 MAX_AGE_DAYS=7 MAX_TOTAL_BYTES=$((300 * 1024 * 1024)) # independent of systemd-coredump's own MaxUse -GDB_DEBUG_DIR=/var/lib/postgresql/.nix-profile/lib/debug +GDB_DEBUG_DIR=/nix/var/nix/profiles/postgres-debug/lib/debug GDB_CMDS_FILE=/usr/local/sbin/orioledb-coredump-cmds.gdb PGDATA_CURRENT_LOGFILES=/var/lib/postgresql/data/current_logfiles diff --git a/nix/docs/README.md b/nix/docs/README.md index 650555850e..c033f1efbb 100644 --- a/nix/docs/README.md +++ b/nix/docs/README.md @@ -16,6 +16,7 @@ learn how to play with `postgres` in the [build guide](./build-postgres.md). - **[Development Workflow](./development-workflow.md)** - Complete development and testing workflow - **[Build PostgreSQL](./build-postgres.md)** - Building PostgreSQL from source - **[Receipt Files](./receipt-files.md)** - Understanding build receipts +- **[Debug Symbols](./debug-symbols.md)** - Download debug symbols on an instance - **[Start Client/Server](./start-client-server.md)** - Running PostgreSQL client and server - **[Docker](./docker.md)** - Docker integration and usage - **[Docker Image Size Analyzer](./image-size-analyzer-usage.md)** - Tool to analyze the Docker image sizes diff --git a/nix/docs/debug-symbols.md b/nix/docs/debug-symbols.md new file mode 100644 index 0000000000..a5c5a98d87 --- /dev/null +++ b/nix/docs/debug-symbols.md @@ -0,0 +1,39 @@ +# Debug symbols + +`postgres-env-*` does not carry debug symbols. Download them on the instance when you need them. + +## Download + +```bash +sudo /var/lib/postgresql/.nix-profile/bin/realise-postgres-debug +``` + +This fetches `postgres-env--debug` from the binary cache and installs it into the profile `/nix/var/nix/profiles/postgres-debug`. The debug env always matches the installed binaries, because it is built from the same derivations. + +## Use with gdb + +```bash +gdb -ex 'set debug-file-directory /nix/var/nix/profiles/postgres-debug/lib/debug' \ + -ex 'file /var/lib/postgresql/.nix-profile/bin/postgres' \ + -ex 'core-file ' +``` + +gdb finds the `.debug` files by build-id under `lib/debug/.build-id`. + +## Source + +The 15 and 17 debug envs have symbols only. The orioledb debug env also has the PostgreSQL source tree. Point gdb at it with `set substitute-path /nix/var/nix/profiles/postgres-debug`. + +## Remove + +```bash +sudo /var/lib/postgresql/.nix-profile/bin/cleanup-postgres-debug +``` + +This removes the profile and all its generations. It does not collect garbage. Run `nix-collect-garbage` to free the space. + +## Build + +```bash +nix build .#postgres-env-17-debug +``` diff --git a/nix/packages/postgres-env.nix b/nix/packages/postgres-env.nix index 16ac813fd7..fefa881ac2 100644 --- a/nix/packages/postgres-env.nix +++ b/nix/packages/postgres-env.nix @@ -7,16 +7,12 @@ ... }: let - # Make a bundle of packages, as a single derivation, to be installed into the - # postgres user's nix profile, during image provisioning or instance update. debugPaths = version: lib.optionals pkgs.stdenv.isLinux [ self'.packages."postgresql_${version}_debug" ] - # orioledb.so ships as a separate extension package (nix/ext/orioledb.nix), not - # part of the postgresql derivation itself, so its debug output isn't covered by - # postgresql_${version}_debug above and has to be pulled in explicitly. ++ lib.optionals (pkgs.stdenv.isLinux && version == "orioledb-17") [ self'.legacyPackages."psql_${version}".exts.orioledb.debug + self'.packages."postgresql_${version}_src" ]; extDebug = @@ -37,15 +33,26 @@ paths = debugPaths version ++ lib.optionals pkgs.stdenv.isLinux (extDebug version); }; + debugProfile = "/nix/var/nix/profiles/postgres-debug"; + realisePostgresDebug = version: pkgs.writeShellApplication { name = "realise-postgres-debug"; text = '' - nix-env --profile /nix/var/nix/profiles/postgres-debug --set "$(nix-store --realise ${builtins.unsafeDiscardStringContext (makePostgresEnvDebug version).outPath})" + nix-env --profile ${debugProfile} --set "$(nix-store --realise ${builtins.unsafeDiscardStringContext (makePostgresEnvDebug version).outPath})" ''; }; + cleanupPostgresDebug = pkgs.writeShellApplication { + name = "cleanup-postgres-debug"; + text = '' + rm -f ${debugProfile} ${debugProfile}-*-link + ''; + }; + + # Make a bundle of packages, as a single derivation, to be installed into the + # postgres user's nix profile, during image provisioning or instance update. makePostgresEnv = version: pkgs.symlinkJoin { @@ -56,12 +63,10 @@ self'.packages.supabase-groonga ] ++ lib.optionals (pkgs.stdenv.isLinux && version != "15") [ self'.packages.gatekeeper ] - ++ ( - if version == "orioledb-17" then - [ self'.packages."postgresql_${version}_src" ] ++ debugPaths version - else - lib.optionals pkgs.stdenv.isLinux [ (realisePostgresDebug version) ] - ); + ++ lib.optionals pkgs.stdenv.isLinux [ + (realisePostgresDebug version) + cleanupPostgresDebug + ]; }; in { @@ -71,6 +76,7 @@ postgres-env-orioledb-17 = makePostgresEnv "orioledb-17"; postgres-env-15-debug = makePostgresEnvDebug "15"; postgres-env-17-debug = makePostgresEnvDebug "17"; + postgres-env-orioledb-17-debug = makePostgresEnvDebug "orioledb-17"; }; }; } diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index e46c67aa2e..e97492b355 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -1701,13 +1701,23 @@ def _build_id_of(host, path): return (match.group(1) if match else None), output +POSTGRES_DEBUG_PROFILE = "/nix/var/nix/profiles/postgres-debug" + + +def _realise_postgres_debug(host): + result = run_ssh_command( + host["ssh"], + "sudo /var/lib/postgresql/.nix-profile/bin/realise-postgres-debug", + ) + assert result["succeeded"], f"realise-postgres-debug failed: {result['stderr']}" + + def _debug_file_exists_for_build_id(host, build_id): - """Check whether the postgres nix-profile's debug output has a + """Check whether the postgres-debug profile has a .build-id/xx/yyyy...debug file matching the given build-id.""" + _realise_postgres_debug(host) prefix, rest = build_id[:2], build_id[2:] - debug_path = ( - f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug" - ) + debug_path = f"{POSTGRES_DEBUG_PROFILE}/lib/debug/.build-id/{prefix}/{rest}.debug" result = run_ssh_command(host["ssh"], f"test -f {debug_path} && echo present") return result["succeeded"] and "present" in result["stdout"] @@ -1723,7 +1733,7 @@ def test_postgres_binary_build_id_matches_shipped_debug_symbols(host): f"{readelf_output}" ) assert _debug_file_exists_for_build_id(host, build_id), ( - f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/" + f"No debug file found under {POSTGRES_DEBUG_PROFILE}/lib/debug/" f".build-id/ matching postgres build-id {build_id} - the shipped " f"_debug package may be out of sync with the shipped binary" ) @@ -1743,7 +1753,7 @@ def test_orioledb_library_build_id_matches_shipped_debug_symbols(host): f"{readelf_output}" ) assert _debug_file_exists_for_build_id(host, build_id), ( - f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/" + f"No debug file found under {POSTGRES_DEBUG_PROFILE}/lib/debug/" f".build-id/ matching orioledb.so build-id {build_id}" ) @@ -1761,8 +1771,9 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host): f"{readelf_output}" ) prefix, rest = build_id[:2], build_id[2:] + _realise_postgres_debug(host) debug_file = ( - f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug" + f"{POSTGRES_DEBUG_PROFILE}/lib/debug/.build-id/{prefix}/{rest}.debug" ) # DW_AT_comp_dir is recorded per compilation unit, not once globally - @@ -1803,8 +1814,8 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host): # "Permission denied" on the bare filename before it ever tries the # substitute-path'd absolute path. "cd /var/lib/postgresql && sudo -u postgres gdb --batch -quiet " - "-ex 'set debug-file-directory /var/lib/postgresql/.nix-profile/lib/debug' " - f"-ex 'set substitute-path {comp_dir} /var/lib/postgresql/.nix-profile' " + f"-ex 'set debug-file-directory {POSTGRES_DEBUG_PROFILE}/lib/debug' " + f"-ex 'set substitute-path {comp_dir} {POSTGRES_DEBUG_PROFILE}' " f"-ex 'file {postgres_binary}' " "-ex 'list main' " "2>&1", @@ -1816,7 +1827,7 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host): assert not re.search(r"^\d+\tin /", result["stdout"], re.MULTILINE), ( f"GDB fell back to the 'in ' placeholder, meaning it could not " f"actually read the source file even with substitute-path set from " - f"{comp_dir} to /var/lib/postgresql/.nix-profile:\n{result['stdout']}" + f"{comp_dir} to {POSTGRES_DEBUG_PROFILE}:\n{result['stdout']}" ) numbered_lines = re.findall(r"^\d+\t.+$", result["stdout"], re.MULTILINE) assert len(numbered_lines) >= 3, ( @@ -1825,6 +1836,24 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host): ) +def test_cleanup_postgres_debug_removes_profile_and_generations(host): + _skip_if_not_orioledb(host) + _realise_postgres_debug(host) + result = run_ssh_command( + host["ssh"], + "sudo /var/lib/postgresql/.nix-profile/bin/cleanup-postgres-debug", + ) + assert result["succeeded"], f"cleanup-postgres-debug failed: {result['stderr']}" + leftover = run_ssh_command( + host["ssh"], + f"ls -d {POSTGRES_DEBUG_PROFILE} {POSTGRES_DEBUG_PROFILE}-*-link 2>/dev/null", + ) + assert leftover["stdout"].strip() == "", ( + f"Expected the postgres-debug profile and its generations to be gone, " + f"but found:\n{leftover['stdout']}" + ) + + def test_coredump_processor_produces_diagnostic_bundle_and_deletes_raw_core(host): """Verify that after a real Postgres backend crash, the orioledb-coredump.path-triggered processor turns the raw core into a @@ -1837,6 +1866,8 @@ def test_coredump_processor_produces_diagnostic_bundle_and_deletes_raw_core(host "coredump processor not installed on this AMI (not an OrioleDB build)" ) + _realise_postgres_debug(host) + before_bundles = set( run_ssh_command( host["ssh"], From 181ccc2c0ba60ce5dc666af0040752c4fd4a3afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Sat, 10 Oct 2026 00:18:42 +0300 Subject: [PATCH 04/11] refactor(nix): realise the debug env without a profile realise-postgres-debug now only realises the debug env and prints its store path. Drop the postgres-debug profile and cleanup-postgres-debug. The tests and the coredump processor use the printed path, and garbage collection removes the debug env. Refs: MPG-1336 Co-Authored-By: Claude Sonnet 5.5 --- .../coredump/process-orioledb-coredumps.sh | 2 +- nix/docs/debug-symbols.md | 17 +++---- nix/packages/postgres-env.nix | 16 +------ testinfra/test_ami_nix.py | 44 +++++-------------- 4 files changed, 20 insertions(+), 59 deletions(-) diff --git a/ansible/files/coredump/process-orioledb-coredumps.sh b/ansible/files/coredump/process-orioledb-coredumps.sh index c8133c4b61..5f40ceae14 100644 --- a/ansible/files/coredump/process-orioledb-coredumps.sh +++ b/ansible/files/coredump/process-orioledb-coredumps.sh @@ -25,7 +25,7 @@ EXTRACTION_TIMEOUT=120 MAX_AGE_DAYS=7 MAX_TOTAL_BYTES=$((300 * 1024 * 1024)) # independent of systemd-coredump's own MaxUse -GDB_DEBUG_DIR=/nix/var/nix/profiles/postgres-debug/lib/debug +GDB_DEBUG_DIR="$(/var/lib/postgresql/.nix-profile/bin/realise-postgres-debug || true)/lib/debug" GDB_CMDS_FILE=/usr/local/sbin/orioledb-coredump-cmds.gdb PGDATA_CURRENT_LOGFILES=/var/lib/postgresql/data/current_logfiles diff --git a/nix/docs/debug-symbols.md b/nix/docs/debug-symbols.md index a5c5a98d87..0e45bb1787 100644 --- a/nix/docs/debug-symbols.md +++ b/nix/docs/debug-symbols.md @@ -5,15 +5,16 @@ ## Download ```bash -sudo /var/lib/postgresql/.nix-profile/bin/realise-postgres-debug +/var/lib/postgresql/.nix-profile/bin/realise-postgres-debug ``` -This fetches `postgres-env--debug` from the binary cache and installs it into the profile `/nix/var/nix/profiles/postgres-debug`. The debug env always matches the installed binaries, because it is built from the same derivations. +This fetches `postgres-env--debug` from the binary cache and prints its store path. The debug env always matches the installed binaries, because it is built from the same derivations. It is not pinned, so garbage collection removes it. ## Use with gdb ```bash -gdb -ex 'set debug-file-directory /nix/var/nix/profiles/postgres-debug/lib/debug' \ +DEBUG=$(/var/lib/postgresql/.nix-profile/bin/realise-postgres-debug) +gdb -ex "set debug-file-directory $DEBUG/lib/debug" \ -ex 'file /var/lib/postgresql/.nix-profile/bin/postgres' \ -ex 'core-file ' ``` @@ -22,15 +23,7 @@ gdb finds the `.debug` files by build-id under `lib/debug/.build-id`. ## Source -The 15 and 17 debug envs have symbols only. The orioledb debug env also has the PostgreSQL source tree. Point gdb at it with `set substitute-path /nix/var/nix/profiles/postgres-debug`. - -## Remove - -```bash -sudo /var/lib/postgresql/.nix-profile/bin/cleanup-postgres-debug -``` - -This removes the profile and all its generations. It does not collect garbage. Run `nix-collect-garbage` to free the space. +The 15 and 17 debug envs have symbols only. The orioledb debug env also has the PostgreSQL source tree. Point gdb at it with `set substitute-path $DEBUG`. ## Build diff --git a/nix/packages/postgres-env.nix b/nix/packages/postgres-env.nix index fefa881ac2..fc9f7db639 100644 --- a/nix/packages/postgres-env.nix +++ b/nix/packages/postgres-env.nix @@ -33,24 +33,15 @@ paths = debugPaths version ++ lib.optionals pkgs.stdenv.isLinux (extDebug version); }; - debugProfile = "/nix/var/nix/profiles/postgres-debug"; - realisePostgresDebug = version: pkgs.writeShellApplication { name = "realise-postgres-debug"; text = '' - nix-env --profile ${debugProfile} --set "$(nix-store --realise ${builtins.unsafeDiscardStringContext (makePostgresEnvDebug version).outPath})" + nix-store --realise ${builtins.unsafeDiscardStringContext (makePostgresEnvDebug version).outPath} ''; }; - cleanupPostgresDebug = pkgs.writeShellApplication { - name = "cleanup-postgres-debug"; - text = '' - rm -f ${debugProfile} ${debugProfile}-*-link - ''; - }; - # Make a bundle of packages, as a single derivation, to be installed into the # postgres user's nix profile, during image provisioning or instance update. makePostgresEnv = @@ -63,10 +54,7 @@ self'.packages.supabase-groonga ] ++ lib.optionals (pkgs.stdenv.isLinux && version != "15") [ self'.packages.gatekeeper ] - ++ lib.optionals pkgs.stdenv.isLinux [ - (realisePostgresDebug version) - cleanupPostgresDebug - ]; + ++ lib.optionals pkgs.stdenv.isLinux [ (realisePostgresDebug version) ]; }; in { diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index e97492b355..7ef8d3a78d 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -1701,23 +1701,21 @@ def _build_id_of(host, path): return (match.group(1) if match else None), output -POSTGRES_DEBUG_PROFILE = "/nix/var/nix/profiles/postgres-debug" - - def _realise_postgres_debug(host): result = run_ssh_command( host["ssh"], - "sudo /var/lib/postgresql/.nix-profile/bin/realise-postgres-debug", + "/var/lib/postgresql/.nix-profile/bin/realise-postgres-debug", ) assert result["succeeded"], f"realise-postgres-debug failed: {result['stderr']}" + return result["stdout"].strip() def _debug_file_exists_for_build_id(host, build_id): - """Check whether the postgres-debug profile has a + """Check whether the postgres debug env has a .build-id/xx/yyyy...debug file matching the given build-id.""" - _realise_postgres_debug(host) + debug_env = _realise_postgres_debug(host) prefix, rest = build_id[:2], build_id[2:] - debug_path = f"{POSTGRES_DEBUG_PROFILE}/lib/debug/.build-id/{prefix}/{rest}.debug" + debug_path = f"{debug_env}/lib/debug/.build-id/{prefix}/{rest}.debug" result = run_ssh_command(host["ssh"], f"test -f {debug_path} && echo present") return result["succeeded"] and "present" in result["stdout"] @@ -1733,7 +1731,7 @@ def test_postgres_binary_build_id_matches_shipped_debug_symbols(host): f"{readelf_output}" ) assert _debug_file_exists_for_build_id(host, build_id), ( - f"No debug file found under {POSTGRES_DEBUG_PROFILE}/lib/debug/" + f"No debug file found in the postgres debug env under lib/debug/" f".build-id/ matching postgres build-id {build_id} - the shipped " f"_debug package may be out of sync with the shipped binary" ) @@ -1753,7 +1751,7 @@ def test_orioledb_library_build_id_matches_shipped_debug_symbols(host): f"{readelf_output}" ) assert _debug_file_exists_for_build_id(host, build_id), ( - f"No debug file found under {POSTGRES_DEBUG_PROFILE}/lib/debug/" + f"No debug file found in the postgres debug env under lib/debug/" f".build-id/ matching orioledb.so build-id {build_id}" ) @@ -1771,9 +1769,9 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host): f"{readelf_output}" ) prefix, rest = build_id[:2], build_id[2:] - _realise_postgres_debug(host) + debug_env = _realise_postgres_debug(host) debug_file = ( - f"{POSTGRES_DEBUG_PROFILE}/lib/debug/.build-id/{prefix}/{rest}.debug" + f"{debug_env}/lib/debug/.build-id/{prefix}/{rest}.debug" ) # DW_AT_comp_dir is recorded per compilation unit, not once globally - @@ -1814,8 +1812,8 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host): # "Permission denied" on the bare filename before it ever tries the # substitute-path'd absolute path. "cd /var/lib/postgresql && sudo -u postgres gdb --batch -quiet " - f"-ex 'set debug-file-directory {POSTGRES_DEBUG_PROFILE}/lib/debug' " - f"-ex 'set substitute-path {comp_dir} {POSTGRES_DEBUG_PROFILE}' " + f"-ex 'set debug-file-directory {debug_env}/lib/debug' " + f"-ex 'set substitute-path {comp_dir} {debug_env}' " f"-ex 'file {postgres_binary}' " "-ex 'list main' " "2>&1", @@ -1827,7 +1825,7 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host): assert not re.search(r"^\d+\tin /", result["stdout"], re.MULTILINE), ( f"GDB fell back to the 'in ' placeholder, meaning it could not " f"actually read the source file even with substitute-path set from " - f"{comp_dir} to {POSTGRES_DEBUG_PROFILE}:\n{result['stdout']}" + f"{comp_dir} to {debug_env}:\n{result['stdout']}" ) numbered_lines = re.findall(r"^\d+\t.+$", result["stdout"], re.MULTILINE) assert len(numbered_lines) >= 3, ( @@ -1836,24 +1834,6 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host): ) -def test_cleanup_postgres_debug_removes_profile_and_generations(host): - _skip_if_not_orioledb(host) - _realise_postgres_debug(host) - result = run_ssh_command( - host["ssh"], - "sudo /var/lib/postgresql/.nix-profile/bin/cleanup-postgres-debug", - ) - assert result["succeeded"], f"cleanup-postgres-debug failed: {result['stderr']}" - leftover = run_ssh_command( - host["ssh"], - f"ls -d {POSTGRES_DEBUG_PROFILE} {POSTGRES_DEBUG_PROFILE}-*-link 2>/dev/null", - ) - assert leftover["stdout"].strip() == "", ( - f"Expected the postgres-debug profile and its generations to be gone, " - f"but found:\n{leftover['stdout']}" - ) - - def test_coredump_processor_produces_diagnostic_bundle_and_deletes_raw_core(host): """Verify that after a real Postgres backend crash, the orioledb-coredump.path-triggered processor turns the raw core into a From e5a5e3f6a08d7fa532d160364ad1240bb96a77af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Sat, 10 Oct 2026 03:23:36 +0300 Subject: [PATCH 05/11] refactor(nix): leave extension debug outputs out of the debug env The extensions get their own debug env. Drop the per-version debug outputs of wrappers, pg_graphql, and pg_jsonschema from the postgres debug env. Refs: MPG-1336 Co-Authored-By: Claude Sonnet 5.5 --- nix/ext/pg_graphql/default.nix | 1 - nix/ext/pg_jsonschema/default.nix | 1 - nix/ext/wrappers/default.nix | 1 - nix/packages/postgres-env.nix | 13 +------------ 4 files changed, 1 insertion(+), 15 deletions(-) diff --git a/nix/ext/pg_graphql/default.nix b/nix/ext/pg_graphql/default.nix index f4e132b746..39d627400e 100644 --- a/nix/ext/pg_graphql/default.nix +++ b/nix/ext/pg_graphql/default.nix @@ -184,7 +184,6 @@ in --prefix EXT_WRAPPER : "$out" --prefix EXT_NAME : "${pname}" ''; passthru = { - debug = map (p: p.debug) packages; perVersion = lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; diff --git a/nix/ext/pg_jsonschema/default.nix b/nix/ext/pg_jsonschema/default.nix index 995d76ca89..cd71e85407 100644 --- a/nix/ext/pg_jsonschema/default.nix +++ b/nix/ext/pg_jsonschema/default.nix @@ -184,7 +184,6 @@ in ''; passthru = { - debug = map (p: p.debug) packages; perVersion = lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) versionsToUse; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index c83ca90f1f..92eddb1cbe 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -371,7 +371,6 @@ in } ''; passthru = { - debug = map (p: p.debug) packages; perVersion = packagesAttrSet; versions = versionsBuilt; numberOfVersions = numberOfVersionsBuilt; diff --git a/nix/packages/postgres-env.nix b/nix/packages/postgres-env.nix index fc9f7db639..4ee798c164 100644 --- a/nix/packages/postgres-env.nix +++ b/nix/packages/postgres-env.nix @@ -15,22 +15,11 @@ self'.packages."postgresql_${version}_src" ]; - extDebug = - version: - let - exts = self'.legacyPackages."psql_${version}".exts; - in - lib.concatMap (e: e.passthru.debug) [ - exts.wrappers - exts.pg_graphql - exts.pg_jsonschema - ]; - makePostgresEnvDebug = version: pkgs.symlinkJoin { name = "postgres-env-${version}-debug"; - paths = debugPaths version ++ lib.optionals pkgs.stdenv.isLinux (extDebug version); + paths = debugPaths version; }; realisePostgresDebug = From 4e104a8531f55c81680484307ed4e65ce19172e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Sat, 10 Oct 2026 04:14:58 +0300 Subject: [PATCH 06/11] style: format test_ami_nix Refs: MPG-1336 Co-Authored-By: Claude Sonnet 5.5 --- testinfra/test_ami_nix.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index 7ef8d3a78d..846b6bf3f8 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -1770,9 +1770,7 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host): ) prefix, rest = build_id[:2], build_id[2:] debug_env = _realise_postgres_debug(host) - debug_file = ( - f"{debug_env}/lib/debug/.build-id/{prefix}/{rest}.debug" - ) + debug_file = f"{debug_env}/lib/debug/.build-id/{prefix}/{rest}.debug" # DW_AT_comp_dir is recorded per compilation unit, not once globally - # PostgreSQL's recursive-Makefile build compiles each .c file from its From 3c1366aac58f5d2dce865da9e09a8966e6922a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Sat, 10 Oct 2026 04:15:12 +0300 Subject: [PATCH 07/11] fix(nix): make the debug envs substitutable symlinkJoin is never substituted, so CI did not build or push the debug envs and realise-postgres-debug could not fetch them. Allow substitutes so CI builds and caches them. Refs: MPG-1336 Co-Authored-By: Claude Sonnet 5.5 --- nix/packages/postgres-env.nix | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nix/packages/postgres-env.nix b/nix/packages/postgres-env.nix index 4ee798c164..10480007bc 100644 --- a/nix/packages/postgres-env.nix +++ b/nix/packages/postgres-env.nix @@ -17,10 +17,14 @@ makePostgresEnvDebug = version: - pkgs.symlinkJoin { + (pkgs.symlinkJoin { name = "postgres-env-${version}-debug"; paths = debugPaths version; - }; + }).overrideAttrs + { + allowSubstitutes = true; + preferLocalBuild = false; + }; realisePostgresDebug = version: From 5fd1b9cf45521f21e5cc45c6171b0618a6216d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Sat, 10 Oct 2026 12:03:20 +0300 Subject: [PATCH 08/11] perf(nix): build the debug env with the runtime env and drop the reference debugEnvPath builds the debug env and stores its path in a file without references. realise-postgres-debug reads it. Add the orioledb debug env to the checks so CI builds and caches it. Refs: MPG-1336 Co-Authored-By: Claude Sonnet 5.5 --- nix/checks.nix | 1 + nix/packages/postgres-env.nix | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/nix/checks.nix b/nix/checks.nix index 1a4b21a212..3f2c7614b1 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -923,6 +923,7 @@ postgresql_15_src postgresql_orioledb-17_debug postgresql_orioledb-17_src + postgres-env-orioledb-17-debug postgresql_17_debug postgresql_17_src ; diff --git a/nix/packages/postgres-env.nix b/nix/packages/postgres-env.nix index 10480007bc..70e08aa9c6 100644 --- a/nix/packages/postgres-env.nix +++ b/nix/packages/postgres-env.nix @@ -26,12 +26,24 @@ preferLocalBuild = false; }; + debugEnvPath = + version: + pkgs.runCommand "postgres-debug-env-path" + { + debugEnv = makePostgresEnvDebug version; + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + } + '' + echo -n "$debugEnv" > $out + ''; + realisePostgresDebug = version: pkgs.writeShellApplication { name = "realise-postgres-debug"; text = '' - nix-store --realise ${builtins.unsafeDiscardStringContext (makePostgresEnvDebug version).outPath} + nix-store --realise "$(cat ${debugEnvPath version})" ''; }; From bdaf85a0e98daf8f7539bc52ff4e6d0760bf4b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Tue, 13 Oct 2026 14:13:53 +0300 Subject: [PATCH 09/11] perf(nix): ship orioledb debug env in postgres-env-orioledb-17 Refs: MPG-1336 Co-Authored-By: Claude Sonnet 5.5 --- nix/packages/postgres-env.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nix/packages/postgres-env.nix b/nix/packages/postgres-env.nix index 70e08aa9c6..e5e8552819 100644 --- a/nix/packages/postgres-env.nix +++ b/nix/packages/postgres-env.nix @@ -59,7 +59,10 @@ self'.packages.supabase-groonga ] ++ lib.optionals (pkgs.stdenv.isLinux && version != "15") [ self'.packages.gatekeeper ] - ++ lib.optionals pkgs.stdenv.isLinux [ (realisePostgresDebug version) ]; + ++ lib.optionals pkgs.stdenv.isLinux [ (realisePostgresDebug version) ] + ++ lib.optionals (pkgs.stdenv.isLinux && version == "orioledb-17") [ + (makePostgresEnvDebug version) + ]; }; in { From 8efd24a4eae609b5806ff2fc3fbb6e3ad0810458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Tue, 13 Oct 2026 18:53:34 +0300 Subject: [PATCH 10/11] revert(ansible): read debug symbols from the profile in the coredump processor Refs: MPG-1336 Co-Authored-By: Claude Sonnet 5.5 --- ansible/files/coredump/process-orioledb-coredumps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/coredump/process-orioledb-coredumps.sh b/ansible/files/coredump/process-orioledb-coredumps.sh index 5f40ceae14..2ce913f619 100644 --- a/ansible/files/coredump/process-orioledb-coredumps.sh +++ b/ansible/files/coredump/process-orioledb-coredumps.sh @@ -25,7 +25,7 @@ EXTRACTION_TIMEOUT=120 MAX_AGE_DAYS=7 MAX_TOTAL_BYTES=$((300 * 1024 * 1024)) # independent of systemd-coredump's own MaxUse -GDB_DEBUG_DIR="$(/var/lib/postgresql/.nix-profile/bin/realise-postgres-debug || true)/lib/debug" +GDB_DEBUG_DIR=/var/lib/postgresql/.nix-profile/lib/debug GDB_CMDS_FILE=/usr/local/sbin/orioledb-coredump-cmds.gdb PGDATA_CURRENT_LOGFILES=/var/lib/postgresql/data/current_logfiles From 61f0b989f1f16c1f11f0da9e04a8c5fb121abb0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Boros?= Date: Mon, 19 Oct 2026 14:36:02 +0300 Subject: [PATCH 11/11] perf(nix): ship pgrx extension debug outputs in the 15 and 17 debug envs Refs: MPG-1336 Co-Authored-By: Claude Sonnet 5.5 --- nix/packages/postgres-env.nix | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/nix/packages/postgres-env.nix b/nix/packages/postgres-env.nix index e5e8552819..cc9d84612e 100644 --- a/nix/packages/postgres-env.nix +++ b/nix/packages/postgres-env.nix @@ -7,9 +7,25 @@ ... }: let + pgrxDebugPaths = + version: + lib.concatMap + ( + name: + lib.mapAttrsToList (_: drv: drv.debug) ( + (self'.legacyPackages."psql_${version}".exts.${name} or { }).perVersion or { } + ) + ) + [ + "wrappers" + "pg_graphql" + "pg_jsonschema" + ]; + debugPaths = version: lib.optionals pkgs.stdenv.isLinux [ self'.packages."postgresql_${version}_debug" ] + ++ lib.optionals (pkgs.stdenv.isLinux && version != "orioledb-17") (pgrxDebugPaths version) ++ lib.optionals (pkgs.stdenv.isLinux && version == "orioledb-17") [ self'.legacyPackages."psql_${version}".exts.orioledb.debug self'.packages."postgresql_${version}_src"