feat(nix): add site-nix-gc to the site profile

nix/docs/site-profile.md

@@ -1,6 +1,6 @@
 # Site profile
 
-The `site-env-15`, `site-env-17`, and `site-env-orioledb-17` package sets are deployed to `/nix/var/nix/profiles/site`. They hold supautils, and `activate`.
+The `site-env-15`, `site-env-17`, and `site-env-orioledb-17` package sets are deployed to `/nix/var/nix/profiles/site`. They hold supautils, `activate`, and `site-nix-gc`.
 
 ## Publish
 
@@ -19,6 +19,8 @@ nix-env --profile /nix/var/nix/profiles/site --set <store path>
 
 New sessions load supautils from the new profile. Open sessions keep the copy they loaded until they reconnect. Postgres does not restart. A restart applies the change to every session at once. The same holds after a rollback.
 
+`site-nix-gc` keeps the current generation and one older one for the site, postgres, and default profiles. It then runs `nix-store --gc` to completion. With systemd it runs in a throttled transient unit. Without systemd it runs under `nice` and `ionice`.
+
 ## Roll back
 
 ```sh
@@ -28,7 +30,7 @@ nix-env --profile /nix/var/nix/profiles/site --rollback
 
 Rollback switches to the previous generation. `activate` then runs from that generation and makes the machine match it. There is no separate rollback script.
 
-To go further back than the previous generation, install an older version by path.
+`site-nix-gc` keeps the current generation and one older one. To go further back, install an older version by path.
 
 ## Install any version by path
 

nix/packages/site-env.nix

@@ -9,39 +9,78 @@
       ...
     }:
     let
+      # Scripts run on the host bash to keep Nix's bash and glibc out of the closure.
+      writeHostBash =
+        name: text:
+        pkgs.writeTextFile {
+          inherit name;
+          executable = true;
+          destination = "/bin/${name}";
+          text = ''
+            #!/usr/bin/env bash
+            set -euo pipefail
+            ${text}
+          '';
+          checkPhase = ''
+            ${pkgs.shellcheck}/bin/shellcheck "$target"
+          '';
+        };
+
+      # Prunes old generations, then runs nix-store --gc. Throttled by a transient systemd unit, or by nice and ionice without systemd.
+      site-nix-gc = writeHostBash "site-nix-gc" ''
+        prune() { nix-env --profile "$1" --delete-generations +2; }
+        prune /nix/var/nix/profiles/site
+        prune /nix/var/nix/profiles/default
+        if [[ -L /var/lib/postgresql/.nix-profile ]]; then
+          prune "$(readlink /var/lib/postgresql/.nix-profile)"
+        fi
+
+        export NIX_REMOTE=local
+        gc=("$(command -v nix-store)" --gc --quiet)
+
+        if [[ -d /run/systemd/system ]]; then
+          if systemctl is-active --quiet site-nix-gc; then
+            echo "Garbage collection already running."
+            exit 0
+          fi
+          systemd-run \
+            --unit=site-nix-gc --collect --no-block --setenv=NIX_REMOTE=local \
+            -p Nice=19 -p CPUSchedulingPolicy=idle -p CPUQuota=20% \
+            -p IOSchedulingClass=idle -p IOWeight=1 -p IOWriteBandwidthMax="/nix 20M" \
+            -p MemoryHigh=10% -p MemoryMax=15% -p OOMScoreAdjust=1000 \
+            -p RuntimeMaxSec=2h \
+            "''${gc[@]}" && exit 0
+          echo "systemd-run failed. Running in the foreground."
+        fi
+
+        throttle=(nice -n 19)
+        command -v ionice >/dev/null && throttle+=(ionice -c 3)
+        exec "''${throttle[@]}" "''${gc[@]}"
+      '';
+
       makeSiteEnv =
         version: extraPaths:
         let
           supautils = self'.legacyPackages."psql_${version}".exts.supautils;
           # Site profile activation script. MUST STAY IDEMPOTENT!
-          activate = pkgs.writeTextFile {
-            name = "activate";
-            executable = true;
-            destination = "/bin/activate";
-            # Runs on the host bash to keep Nix's bash and glibc out of the closure.
-            text = ''
-              #!/usr/bin/env bash
-              set -euo pipefail
-              echo "Activating site profile."
-              echo "Configured in /etc/postgresql/postgresql.conf (runtime could differ):"
-              for guc in dynamic_library_path session_preload_libraries; do
-                grep -h "^$guc" /etc/postgresql/postgresql.conf 2>/dev/null || echo "$guc is not set"
-              done
-              echo "Site profile activated."
-            '';
-            checkPhase = ''
-              ${pkgs.shellcheck}/bin/shellcheck "$target"
-            '';
-          };
+          activate = writeHostBash "activate" ''
+            echo "Activating site profile."
+            echo "Configured in /etc/postgresql/postgresql.conf (runtime could differ):"
+            for guc in dynamic_library_path session_preload_libraries; do
+              grep -h "^$guc" /etc/postgresql/postgresql.conf 2>/dev/null || echo "$guc is not set"
+            done
+            echo "Site profile activated."
+          '';
         in
         pkgs.buildEnv {
           name = "site-env-${version}";
           paths = [
             supautils
             activate
+            site-nix-gc
           ]
           ++ extraPaths;
-          passthru = { inherit activate; };
+          passthru = { inherit activate site-nix-gc; };
         };
 
       siteEnvs = {

nix/tests/site.nix

@@ -45,6 +45,17 @@ pkgs.testers.runNixOSTest {
     machine.succeed("su postgres -c '${psql_17}/bin/psql -h /tmp -d postgres -c \"select 1\"'")
     machine.succeed("su postgres -c '${psql_17}/bin/pg_ctl -D /tmp/pgdata stop'")
 
+    # site-nix-gc runs to the end and reports what it freed
+    machine.succeed("nix-store --gc")
+    machine.succeed("echo garbage > /tmp/garbage && nix-store --add /tmp/garbage")
+    machine.succeed("nix-env --profile /nix/var/nix/profiles/default --set $(dirname $(dirname $(readlink -f $(command -v nix-env))))")
     machine.succeed("${site-env-17}/bin/activate")
+    machine.succeed("${site-env-17}/bin/site-nix-gc")
+    machine.wait_until_fails("systemctl is-active site-nix-gc", timeout=1800)
+    print(machine.succeed("journalctl -u site-nix-gc --no-pager | grep freed"))
+
+    # without systemd the gc runs in the foreground
+    machine.succeed("mount -t tmpfs none /run/systemd")
+    machine.succeed("${site-env-17}/bin/site-nix-gc 2>&1 | grep freed")
   '';
 }