perf(nix): split debug symbols out of postgres-env

nix/cargo-pgrx/buildPgrxExtension.nix

@@ -149,6 +149,8 @@ let
   # we forgot parentheses
   finalArgs = argsForBuildRustPackage // {
     buildInputs = (args.buildInputs or [ ]);
+    separateDebugInfo = true;
+    stripAllList = args.stripAllList or [ "lib" ];
 
     nativeBuildInputs =
       (args.nativeBuildInputs or [ ])

nix/checks.nix

@@ -954,6 +954,7 @@
             postgresql_15_src
             postgresql_orioledb-17_debug
             postgresql_orioledb-17_src
+            postgres-env-orioledb-17-debug
             postgresql_17_debug
             postgresql_17_src
             ;

nix/docs/README.md

@@ -14,6 +14,7 @@ learn how to play with `postgres` in the [build guide](./build-postgres.md).
 - **[Development Workflow](./development-workflow.md)** - Complete development and testing workflow
 - **[Build PostgreSQL](./build-postgres.md)** - Building PostgreSQL from source
 - **[Receipt Files](./receipt-files.md)** - Understanding build receipts
+- **[Debug Symbols](./debug-symbols.md)** - Download debug symbols on an instance
 - **[Start Client/Server](./start-client-server.md)** - Running PostgreSQL client and server
 - **[Docker](./docker.md)** - Docker integration and usage
 - **[Multigres image](https://github.com/supabase/postgres/blob/develop/docs/multigres-image.md)** - Building the multigres Docker image

nix/docs/debug-symbols.md

@@ -0,0 +1,32 @@
+# Debug symbols
+
+`postgres-env-*` does not carry debug symbols. Download them on the instance when you need them.
+
+## Download
+
+```bash
+/var/lib/postgresql/.nix-profile/bin/realise-postgres-debug
+```
+
+This fetches `postgres-env-<version>-debug` from the binary cache and prints its store path. The debug env always matches the installed binaries, because it is built from the same derivations. It is not pinned, so garbage collection removes it.
+
+## Use with gdb
+
+```bash
+DEBUG=$(/var/lib/postgresql/.nix-profile/bin/realise-postgres-debug)
+gdb -ex "set debug-file-directory $DEBUG/lib/debug" \
+    -ex 'file /var/lib/postgresql/.nix-profile/bin/postgres' \
+    -ex 'core-file <core>'
+```
+
+gdb finds the `.debug` files by build-id under `lib/debug/.build-id`.
+
+## Source
+
+The 15 and 17 debug envs have symbols only. The orioledb debug env also has the PostgreSQL source tree. Point gdb at it with `set substitute-path <comp_dir> $DEBUG`.
+
+## Build
+
+```bash
+nix build .#postgres-env-17-debug
+```

nix/packages/postgres-env.nix

@@ -7,6 +7,62 @@
       ...
     }:
     let
+      pgrxDebugPaths =
+        version:
+        lib.concatMap
+          (
+            name:
+            lib.mapAttrsToList (_: drv: drv.debug) (
+              (self'.legacyPackages."psql_${version}".exts.${name} or { }).perVersion or { }
+            )
+          )
+          [
+            "wrappers"
+            "pg_graphql"
+            "pg_jsonschema"
+          ];
+
+      debugPaths =
+        version:
+        lib.optionals pkgs.stdenv.isLinux [ self'.packages."postgresql_${version}_debug" ]
+        ++ lib.optionals (pkgs.stdenv.isLinux && version != "orioledb-17") (pgrxDebugPaths version)
+        ++ lib.optionals (pkgs.stdenv.isLinux && version == "orioledb-17") [
+          self'.legacyPackages."psql_${version}".exts.orioledb.debug
+          self'.packages."postgresql_${version}_src"
+        ];
+
+      makePostgresEnvDebug =
+        version:
+        (pkgs.symlinkJoin {
+          name = "postgres-env-${version}-debug";
+          paths = debugPaths version;
+        }).overrideAttrs
+          {
+            allowSubstitutes = true;
+            preferLocalBuild = false;
+          };
+
+      debugEnvPath =
+        version:
+        pkgs.runCommand "postgres-debug-env-path"
+          {
+            debugEnv = makePostgresEnvDebug version;
+            __structuredAttrs = true;
+            unsafeDiscardReferences.out = true;
+          }
+          ''
+            echo -n "$debugEnv" > $out
+          '';
+
+      realisePostgresDebug =
+        version:
+        pkgs.writeShellApplication {
+          name = "realise-postgres-debug";
+          text = ''
+            nix-store --realise "$(cat ${debugEnvPath version})"
+          '';
+        };
+
       # Make a bundle of packages, as a single derivation, to be installed into the
       # postgres user's nix profile, during image provisioning or instance update.
       makePostgresEnv =
@@ -17,15 +73,11 @@
             self'.packages."psql_${version}/bin"
             self'.packages.pg_prove
             self'.packages.supabase-groonga
-            self'.packages."postgresql_${version}_src"
           ]
-          ++ lib.optionals pkgs.stdenv.isLinux [ self'.packages."postgresql_${version}_debug" ]
           ++ lib.optionals (pkgs.stdenv.isLinux && version != "15") [ self'.packages.gatekeeper ]
-          # orioledb.so ships as a separate extension package (nix/ext/orioledb.nix), not
-          # part of the postgresql derivation itself, so its debug output isn't covered by
-          # postgresql_${version}_debug above and has to be pulled in explicitly.
+          ++ lib.optionals pkgs.stdenv.isLinux [ (realisePostgresDebug version) ]
           ++ lib.optionals (pkgs.stdenv.isLinux && version == "orioledb-17") [
-            self'.legacyPackages."psql_${version}".exts.orioledb.debug
+            (makePostgresEnvDebug version)
           ];
         };
 
@@ -50,6 +102,9 @@
         postgres-env-15 = makePostgresEnv "15";
         postgres-env-17 = makePostgresEnv "17";
         postgres-env-orioledb-17 = makePostgresEnv "orioledb-17";
+        postgres-env-15-debug = makePostgresEnvDebug "15";
+        postgres-env-17-debug = makePostgresEnvDebug "17";
+        postgres-env-orioledb-17-debug = makePostgresEnvDebug "orioledb-17";
         nix-store-paths-15 = makeNixStorePaths "15";
         nix-store-paths-17 = makeNixStorePaths "17";
         nix-store-paths-orioledb-17 = makeNixStorePaths "orioledb-17";

testinfra/test_ami_nix.py

@@ -1784,13 +1784,21 @@ def _build_id_of(host, path):
     return (match.group(1) if match else None), output
 
 
+def _realise_postgres_debug(host):
+    result = run_ssh_command(
+        host["ssh"],
+        "/var/lib/postgresql/.nix-profile/bin/realise-postgres-debug",
+    )
+    assert result["succeeded"], f"realise-postgres-debug failed: {result['stderr']}"
+    return result["stdout"].strip()
+
+
 def _debug_file_exists_for_build_id(host, build_id):
-    """Check whether the postgres nix-profile's debug output has a
+    """Check whether the postgres debug env has a
     .build-id/xx/yyyy...debug file matching the given build-id."""
+    debug_env = _realise_postgres_debug(host)
     prefix, rest = build_id[:2], build_id[2:]
-    debug_path = (
-        f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug"
-    )
+    debug_path = f"{debug_env}/lib/debug/.build-id/{prefix}/{rest}.debug"
     result = run_ssh_command(host["ssh"], f"test -f {debug_path} && echo present")
     return result["succeeded"] and "present" in result["stdout"]
 
@@ -1806,7 +1814,7 @@ def test_postgres_binary_build_id_matches_shipped_debug_symbols(host):
         f"{readelf_output}"
     )
     assert _debug_file_exists_for_build_id(host, build_id), (
-        f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/"
+        f"No debug file found in the postgres debug env under lib/debug/"
         f".build-id/ matching postgres build-id {build_id} - the shipped "
         f"_debug package may be out of sync with the shipped binary"
     )
@@ -1826,7 +1834,7 @@ def test_orioledb_library_build_id_matches_shipped_debug_symbols(host):
         f"{readelf_output}"
     )
     assert _debug_file_exists_for_build_id(host, build_id), (
-        f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/"
+        f"No debug file found in the postgres debug env under lib/debug/"
         f".build-id/ matching orioledb.so build-id {build_id}"
     )
 
@@ -1844,9 +1852,8 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
         f"{readelf_output}"
     )
     prefix, rest = build_id[:2], build_id[2:]
-    debug_file = (
-        f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug"
-    )
+    debug_env = _realise_postgres_debug(host)
+    debug_file = f"{debug_env}/lib/debug/.build-id/{prefix}/{rest}.debug"
 
     # DW_AT_comp_dir is recorded per compilation unit, not once globally -
     # PostgreSQL's recursive-Makefile build compiles each .c file from its
@@ -1886,8 +1893,8 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
         # "Permission denied" on the bare filename before it ever tries the
         # substitute-path'd absolute path.
         "cd /var/lib/postgresql && sudo -u postgres gdb --batch -quiet "
-        "-ex 'set debug-file-directory /var/lib/postgresql/.nix-profile/lib/debug' "
-        f"-ex 'set substitute-path {comp_dir} /var/lib/postgresql/.nix-profile' "
+        f"-ex 'set debug-file-directory {debug_env}/lib/debug' "
+        f"-ex 'set substitute-path {comp_dir} {debug_env}' "
         f"-ex 'file {postgres_binary}' "
         "-ex 'list main' "
         "2>&1",
@@ -1899,7 +1906,7 @@ def test_gdb_resolves_postgres_source_via_shipped_src_package(host):
     assert not re.search(r"^\d+\tin /", result["stdout"], re.MULTILINE), (
         f"GDB fell back to the 'in <path>' placeholder, meaning it could not "
         f"actually read the source file even with substitute-path set from "
-        f"{comp_dir} to /var/lib/postgresql/.nix-profile:\n{result['stdout']}"
+        f"{comp_dir} to {debug_env}:\n{result['stdout']}"
     )
     numbered_lines = re.findall(r"^\d+\t.+$", result["stdout"], re.MULTILINE)
     assert len(numbered_lines) >= 3, (
@@ -1920,6 +1927,8 @@ def test_coredump_processor_produces_diagnostic_bundle_and_deletes_raw_core(host
             "coredump processor not installed on this AMI (not an OrioleDB build)"
         )
 
+    _realise_postgres_debug(host)
+
     before_bundles = set(
         run_ssh_command(
             host["ssh"],