diff --git a/ansible/files/postgresql_config/postgresql.service b/ansible/files/postgresql_config/postgresql.service index 07aac3bcae..d311154f60 100644 --- a/ansible/files/postgresql_config/postgresql.service +++ b/ansible/files/postgresql_config/postgresql.service @@ -1,6 +1,7 @@ [Unit] Description=PostgreSQL database server Documentation=man:postgres(1) +Wants=postgresql-collation-refresh.service Requires=database-optimizations.service After=database-optimizations.service diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index c94fdb4cdc..3dab4cba04 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -246,6 +246,13 @@ loop_control: loop_var: 'systemd_svc_item' +- name: Link the collation refresh unit from the site profile + when: stage2 + ansible.builtin.file: + src: /nix/var/nix/profiles/site/lib/systemd/system/postgresql-collation-refresh.service + dest: /etc/systemd/system/postgresql-collation-refresh.service + state: link + - name: initialize pg required state ansible.builtin.file: group: 'postgres' diff --git a/nix/ext/tests/collation-refresh.nix b/nix/ext/tests/collation-refresh.nix new file mode 100644 index 0000000000..483339af50 --- /dev/null +++ b/nix/ext/tests/collation-refresh.nix @@ -0,0 +1,100 @@ +{ self, pkgs }: +let + system = pkgs.pkgsLinux.stdenv.hostPlatform.system; + testLib = import ./lib.nix { inherit self pkgs; }; + refresh = self.packages.${system}.postgres-collation-refresh; + psql = self.packages.${system}."psql_17/bin"; +in +pkgs.testers.runNixOSTest { + name = "collation-refresh"; + nodes.server = + { ... }: + { + imports = [ + (testLib.makeSupabaseTestConfig { + majorVersion = "17"; + }) + ]; + + i18n.supportedLocales = [ + "C.UTF-8/UTF-8" + "en_US.UTF-8/UTF-8" + "de_DE.UTF-8/UTF-8" + ]; + + systemd.tmpfiles.rules = [ + "L+ /var/lib/postgresql/.nix-profile - - - - ${psql}" + "L+ /nix/var/nix/profiles/site - - - - ${refresh}" + ]; + systemd.packages = [ refresh ]; + systemd.services.postgresql.wants = [ "postgresql-collation-refresh.service" ]; + }; + testScript = '' + PSQL = "psql -h localhost -U supabase_admin -d postgres -At -v ON_ERROR_STOP=1" + UNIT = "postgresql-collation-refresh.service" + + def sql(query): + return server.succeed(PSQL + " -c \"" + query + "\"").strip() + + def recorded(db): + return sql("select coalesce(datcollversion, '-') from pg_database where datname = '" + db + "'") + + def set_recorded(db, version): + sql("update pg_database set datcollversion = '" + version + "' where datname = '" + db + "'") + + start_all() + server.wait_for_unit("supabase-db-init.service") + server.wait_for_unit(UNIT) + + DBS = { + "g": "libc locale = 'en_US.UTF-8'", + "out": "libc locale = 'en_US.UTF-8'", + "de": "libc locale = 'de_DE.UTF-8'", + "c": "libc locale = 'C'", + "i": "icu locale = 'en_US.UTF-8' icu_locale = 'en-US'", + } + for db, options in DBS.items(): + sql("create database " + db + " locale_provider = " + options + " template template0") + actual = recorded("g") + fresh = {"g": actual, "out": actual, "de": actual, "i": recorded("i")} + + def reset(stale): + for db, version in {**fresh, **stale}.items(): + set_recorded(db, version) + + def check(title, stale, refreshed, fails): + with subtest(title): + reset(stale) + status, _ = server.execute("systemctl restart " + UNIT) + assert (status != 0) == fails, f"unit status {status}, expected failure: {fails}" + for db, version in stale.items(): + expected = actual if db in refreshed else version + assert recorded(db) == expected, f"{db}: expected {expected}, got {recorded(db)}" + assert recorded("c") == "-" + + check("Nothing stale", {}, [], False) + check("A stale glibc en_US database inside the bounds is refreshed", {"g": "2.31"}, ["g"], False) + check("ICU and non-English glibc databases are left alone", {"de": "2.31", "i": "1.0"}, [], False) + check("A database above 2.44 fails the unit, the others are refreshed", {"out": "2.99", "g": "2.31"}, ["g"], True) + check("A database below 2.31 fails the unit", {"out": "2.28"}, [], True) + + with subtest("A database that fails to refresh does not stop the others"): + sql("create role refresher login") + sql("create database a_other locale_provider = libc locale = 'en_US.UTF-8' template template0") + sql("create database b_owned owner refresher locale_provider = libc locale = 'en_US.UTF-8' template template0") + reset({"a_other": "2.31", "b_owned": "2.31"}) + status, _ = server.execute("psql -h localhost -U refresher -d postgres -X -q -f /nix/var/nix/profiles/site/share/postgres-collation-refresh.sql") + assert status != 0 + assert recorded("a_other") == "2.31" + assert recorded("b_owned") == actual + + with subtest("A postgres restart reruns the unit and leaves postgres active when it fails"): + reset({"out": "2.99", "g": "2.31", "a_other": actual}) + server.succeed("systemctl reset-failed " + UNIT) + server.succeed("systemctl restart postgresql") + server.wait_until_succeeds("systemctl is-failed " + UNIT) + server.succeed("systemctl is-active postgresql.service") + assert recorded("g") == actual + assert recorded("out") == "2.99" + ''; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index 69dc0ed957..0c16dde9e8 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -61,6 +61,7 @@ }; docs = pkgs.callPackage ./docs.nix { }; pgbouncer = pkgs.callPackage ../pgbouncer.nix { }; + postgres-collation-refresh = pkgs.callPackage ./postgres-collation-refresh { }; github-matrix = pkgs.callPackage ./github-matrix { nix-eval-jobs = inputs'.nix-eval-jobs.packages.default; }; diff --git a/nix/packages/postgres-collation-refresh/default.nix b/nix/packages/postgres-collation-refresh/default.nix new file mode 100644 index 0000000000..2ac32bb17a --- /dev/null +++ b/nix/packages/postgres-collation-refresh/default.nix @@ -0,0 +1,22 @@ +{ + lib, + stdenv, + runCommand, + replaceVars, +}: +let + minGlibc = "2.31"; + maxGlibc = "2.44"; +in +assert lib.assertMsg + ( + !stdenv.hostPlatform.isGnu + || lib.versionAtLeast maxGlibc (lib.versions.majorMinor stdenv.cc.libc.version) + ) + "glibc ${stdenv.cc.libc.version} is above the reviewed maximum ${maxGlibc}. Review English collation changes, then raise maxGlibc in nix/packages/postgres-collation-refresh/default.nix."; +runCommand "postgres-collation-refresh" { } '' + install -Dm0644 ${ + replaceVars ./postgres-collation-refresh.sql { inherit minGlibc maxGlibc; } + } $out/share/postgres-collation-refresh.sql + install -Dm0644 ${./postgresql-collation-refresh.service} $out/lib/systemd/system/postgresql-collation-refresh.service +'' diff --git a/nix/packages/postgres-collation-refresh/postgres-collation-refresh.sql b/nix/packages/postgres-collation-refresh/postgres-collation-refresh.sql new file mode 100644 index 0000000000..0442eea325 --- /dev/null +++ b/nix/packages/postgres-collation-refresh/postgres-collation-refresh.sql @@ -0,0 +1,28 @@ +select pg_is_in_recovery() as replica \gset +\if :replica +\quit +\endif + +create temp view stale as +select datname, + string_to_array(datcollversion, '.')::int[] between string_to_array('@minGlibc@', '.')::int[] and string_to_array('@maxGlibc@', '.')::int[] + and string_to_array(pg_database_collation_actual_version(oid), '.')::int[] between string_to_array('@minGlibc@', '.')::int[] and string_to_array('@maxGlibc@', '.')::int[] as in_bounds +from pg_database +where datlocprovider = 'c' + and datcollate in ('en_US.UTF-8', 'en_US.utf8') + and datcollversion is not null + and datcollversion <> pg_database_collation_actual_version(oid); + +\set ON_ERROR_STOP off +select format('alter database %I refresh collation version', datname) from stale where in_bounds order by datname +\gexec + +\set ON_ERROR_STOP on +do $$ +declare names text; +begin + select string_agg(format('%I', datname), ', ' order by datname) into names from stale; + if names is not null then + raise exception 'collation_refresh: still stale: %', names; + end if; +end $$; diff --git a/nix/packages/postgres-collation-refresh/postgresql-collation-refresh.service b/nix/packages/postgres-collation-refresh/postgresql-collation-refresh.service new file mode 100644 index 0000000000..fc4d226168 --- /dev/null +++ b/nix/packages/postgres-collation-refresh/postgresql-collation-refresh.service @@ -0,0 +1,11 @@ +[Unit] +Description=Refresh default collation version +After=postgresql.service +PartOf=postgresql.service + +[Service] +Type=oneshot +RemainAfterExit=yes +User=postgres +TimeoutStartSec=120 +ExecStart=/var/lib/postgresql/.nix-profile/bin/psql -X -q -h localhost -U supabase_admin -d postgres -f /nix/var/nix/profiles/site/share/postgres-collation-refresh.sql diff --git a/nix/packages/site-env.nix b/nix/packages/site-env.nix index 82bc1692e3..1ecf40519a 100644 --- a/nix/packages/site-env.nix +++ b/nix/packages/site-env.nix @@ -39,6 +39,7 @@ paths = [ supautils activate + self'.packages.postgres-collation-refresh ] ++ extraPaths; passthru = { inherit activate; }; diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index 656a660fee..89b1ddb1a3 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -1616,6 +1616,16 @@ def test_postgres_prestart_does_not_reset_core_limit(host): ) +def test_collation_refresh_unit_is_linked_and_not_failed(host): + """The collation refresh unit links from the site profile and is active.""" + unit = "postgresql-collation-refresh.service" + link = run_ssh_command(host["ssh"], f"readlink /etc/systemd/system/{unit}") + assert link["stdout"].startswith("/nix/var/nix/profiles/site/"), link + state = run_ssh_command(host["ssh"], f"systemctl is-active {unit}") + journal = run_ssh_command(host["ssh"], f"journalctl -u {unit} --no-pager -n 40") + assert state["stdout"].strip() == "active", journal["stdout"] + + def _crash_a_backend_and_wait_for_recovery(host): """Grab a real Postgres backend pid, SIGSEGV it, and wait for PostgreSQL's normal crash-recovery to bring the instance back on its own. Returns the