diff --git a/database/migrations/20261013-01-logged-tables.sql b/database/migrations/20261013-01-logged-tables.sql new file mode 100644 index 00000000..d69e5dc2 --- /dev/null +++ b/database/migrations/20261013-01-logged-tables.sql @@ -0,0 +1,156 @@ +-------------------------------------------------------------------------------------------------------------------------- +-- 20261013-01-logged-tables.sql +-- +-- Ben, 2026-10-13: every table is LOGGED. On 2026-10-02 rapid-db's +-- postgres was stopped uncleanly; crash recovery reset every UNLOGGED +-- relation to empty, so the sources, merges and astroobjects child tables +-- came back with no rows while their result sets still read complete, and +-- re-running load was a silent no-op. The rows are not cheap to regenerate, +-- so the WAL cost of logging them is accepted. +-- +-- Two parts. First, every UNLOGGED table in `public` becomes LOGGED: on a +-- fresh database that is `sources` and `xsources` (the baseline's two +-- `SET UNLOGGED` lines); on an older one it also catches every child table +-- the functions below already made. Their indexes follow their tables. +-- Second, the three child-table functions are replaced without their +-- `SET UNLOGGED` line, bodies otherwise byte-identical to their latest +-- definitions (create_sources_child_table from 20260923-09, the two object +-- functions from 20260924-04). CREATE OR REPLACE keeps owner, grants and +-- SECURITY DEFINER; the ALTER/REVOKE lines restate them as those files do. +-- Applied migrations are never edited (README), hence a new file. +-------------------------------------------------------------------------------------------------------------------------- + +DO $$ +DECLARE + r record; +BEGIN + FOR r IN SELECT c.relname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relkind = 'r' AND c.relpersistence = 'u' + ORDER BY c.relname + LOOP + EXECUTE format('ALTER TABLE public.%I SET LOGGED', r.relname); + END LOOP; +END +$$; + +CREATE OR REPLACE FUNCTION create_sources_child_table(obs_date_ text, sca_ integer) + RETURNS boolean + LANGUAGE plpgsql SECURITY DEFINER + SET search_path = public, pg_temp AS $$ +DECLARE + t text := sources_child_table_name(obs_date_, sca_); +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('rapid.sources-child-table.' || t)); + IF to_regclass('public.' || t) IS NOT NULL THEN + RETURN false; + END IF; + + EXECUTE format('CREATE TABLE %I (LIKE sources INCLUDING DEFAULTS INCLUDING CONSTRAINTS)', t); + EXECUTE format('ALTER TABLE %I OWNER TO rapidporole', t); + EXECUTE format('ALTER TABLE %I INHERIT sources', t); + + EXECUTE format('CREATE INDEX %I ON %I (pid)', t || '_pid_idx', t); + EXECUTE format('CREATE INDEX %I ON %I (expid)', t || '_expid_idx', t); + EXECUTE format('CREATE INDEX %I ON %I (sca)', t || '_sca_idx', t); + EXECUTE format('CREATE INDEX %I ON %I (field)', t || '_field_idx', t); + EXECUTE format('CREATE INDEX %I ON %I (flags)', t || '_flags_idx', t); + EXECUTE format('CREATE INDEX %I ON %I (mjdobs)', t || '_mjdobs_idx', t); + EXECUTE format('CREATE INDEX %I ON %I (sid)', t || '_sid_idx', t); + EXECUTE format('CREATE INDEX %I ON %I (q3c_ang2ipix(ra, dec))', t || '_radec_idx', t); + EXECUTE format('CREATE INDEX %I ON %I (result_set)', t || '_result_set_idx', t); + EXECUTE format('CREATE INDEX %I ON %I (run)', t || '_run_idx', t); + + EXECUTE format('REVOKE ALL ON TABLE %I FROM rapidreadrole', t); + EXECUTE format('GRANT SELECT ON TABLE %I TO GROUP rapidreadrole', t); + EXECUTE format('REVOKE ALL ON TABLE %I FROM rapidadminrole', t); + EXECUTE format('GRANT ALL ON TABLE %I TO GROUP rapidadminrole', t); + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'rapid_rebuild_pipeline') THEN + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE %I TO rapid_rebuild_pipeline', t); + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'rapid_read') THEN + EXECUTE format('GRANT SELECT ON TABLE %I TO rapid_read', t); + END IF; + RETURN true; +END +$$; + +CREATE OR REPLACE FUNCTION create_field_object_tables(field_ integer) + RETURNS boolean + LANGUAGE plpgsql SECURITY DEFINER + SET search_path = public, pg_temp AS $$ +DECLARE + a text := object_field_table_name('astroobjects', field_); + m text := object_field_table_name('merges', field_); + made boolean := false; +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('rapid.object-field-tables.' || field_)); + + IF to_regclass('public.' || a) IS NULL THEN + EXECUTE format('CREATE TABLE %I (LIKE astroobjects INCLUDING DEFAULTS INCLUDING CONSTRAINTS)', a); + EXECUTE format('ALTER TABLE %I OWNER TO rapidporole', a); + + EXECUTE format('CREATE INDEX %I ON %I (aid)', a || '_aid_idx', a); + EXECUTE format('CREATE INDEX %I ON %I (q3c_ang2ipix(ra0, dec0))', a || '_radec_idx', a); + + EXECUTE format('REVOKE ALL ON TABLE %I FROM rapidreadrole', a); + EXECUTE format('GRANT SELECT ON TABLE %I TO GROUP rapidreadrole', a); + EXECUTE format('REVOKE ALL ON TABLE %I FROM rapidadminrole', a); + EXECUTE format('GRANT ALL ON TABLE %I TO GROUP rapidadminrole', a); + made := true; + END IF; + PERFORM attach_object_run_model('astroobjects', field_); + + IF to_regclass('public.' || m) IS NULL THEN + EXECUTE format('CREATE TABLE %I (LIKE merges INCLUDING DEFAULTS INCLUDING CONSTRAINTS)', m); + EXECUTE format('ALTER TABLE %I OWNER TO rapidporole', m); + + EXECUTE format('CREATE INDEX %I ON %I USING btree (aid)', m || '_aid_idx', m); + EXECUTE format('CREATE INDEX %I ON %I USING btree (sid)', m || '_sid_idx', m); + + EXECUTE format('REVOKE ALL ON TABLE %I FROM rapidreadrole', m); + EXECUTE format('GRANT SELECT ON TABLE %I TO GROUP rapidreadrole', m); + EXECUTE format('REVOKE ALL ON TABLE %I FROM rapidadminrole', m); + EXECUTE format('GRANT ALL ON TABLE %I TO GROUP rapidadminrole', m); + made := true; + END IF; + PERFORM attach_object_run_model('merges', field_); + + RETURN made; +END +$$; + +CREATE OR REPLACE FUNCTION create_astroobjectsmeta_child_table(field_ integer) + RETURNS boolean + LANGUAGE plpgsql SECURITY DEFINER + SET search_path = public, pg_temp AS $$ +DECLARE + t text := object_field_table_name('astroobjectsmeta', field_); + made boolean := false; +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('rapid.astroobjectsmeta-table.' || t)); + + IF to_regclass('public.' || t) IS NULL THEN + EXECUTE format('CREATE TABLE %I (LIKE astroobjectsmeta INCLUDING DEFAULTS INCLUDING CONSTRAINTS) WITH (fillfactor = 70)', t); + EXECUTE format('ALTER TABLE %I OWNER TO rapidporole', t); + + EXECUTE format('CREATE INDEX %I ON %I (nsources)', t || '_nsources_idx', t); + EXECUTE format('CREATE INDEX %I ON %I (q3c_ang2ipix(meanra, meandec))', t || '_meanradec_idx', t); + + EXECUTE format('REVOKE ALL ON TABLE %I FROM rapidreadrole', t); + EXECUTE format('GRANT SELECT ON TABLE %I TO GROUP rapidreadrole', t); + EXECUTE format('REVOKE ALL ON TABLE %I FROM rapidadminrole', t); + EXECUTE format('GRANT ALL ON TABLE %I TO GROUP rapidadminrole', t); + made := true; + END IF; + PERFORM attach_object_run_model('astroobjectsmeta', field_); + + RETURN made; +END +$$; + +ALTER FUNCTION create_sources_child_table(text, integer) OWNER TO rapidporole; +ALTER FUNCTION create_field_object_tables(integer) OWNER TO rapidporole; +ALTER FUNCTION create_astroobjectsmeta_child_table(integer) OWNER TO rapidporole; +REVOKE ALL ON FUNCTION create_sources_child_table(text, integer) FROM PUBLIC; +REVOKE ALL ON FUNCTION create_field_object_tables(integer) FROM PUBLIC; +REVOKE ALL ON FUNCTION create_astroobjectsmeta_child_table(integer) FROM PUBLIC; diff --git a/tests/db/test_objects_child_tables.py b/tests/db/test_objects_child_tables.py index 5464ec55..23b98ad0 100644 --- a/tests/db/test_objects_child_tables.py +++ b/tests/db/test_objects_child_tables.py @@ -87,7 +87,7 @@ def test_create_makes_devs_field_tables_once(conn): for table in (a, m): cur.execute("SELECT relpersistence, pg_get_userbyid(relowner) FROM pg_class " "WHERE relname = %s", (table,)) - assert cur.fetchone() == ("u", "rapidporole") + assert cur.fetchone() == ("p", "rapidporole") assert _indexes(cur, a) == {f"{a}_aid_idx", f"{a}_radec_idx", f"{a}_result_set_idx", f"{a}_run_idx", f"{a}_set_aid_key"} @@ -112,7 +112,7 @@ def test_create_makes_devs_astroobjectsmeta_table_once(conn): assert objects.ensure_astroobjectsmeta_table(cur, FIELD) is False cur.execute("SELECT relpersistence, pg_get_userbyid(relowner), reloptions FROM pg_class " "WHERE relname = %s", (t,)) - assert cur.fetchone() == ("u", "rapidporole", ["fillfactor=70"]) + assert cur.fetchone() == ("p", "rapidporole", ["fillfactor=70"]) assert _indexes(cur, t) == {f"{t}_nsources_idx", f"{t}_meanradec_idx", f"{t}_result_set_idx", f"{t}_run_idx", f"{t}_set_aid_key"} constraints = _constraints(cur, t) diff --git a/tests/db/test_recovery.py b/tests/db/test_recovery.py index c44d0e36..1c2de874 100644 --- a/tests/db/test_recovery.py +++ b/tests/db/test_recovery.py @@ -132,7 +132,7 @@ def _seed_run(held, **overrides): kwargs = dict(kind="production", selected_stages=CHAIN, max_attempts=3, settings_overlay_ref="s3://settings/overlay.toml", input_selection_ref="s3://selection/set.json", lane="batch", - resource_profile="rebuild", database_target="rapid_rebuild", + resource_profile="rebuild", database_target="rapid", check_policy_ref="rebuild-trial@1", purpose="step 6 demo P6") kwargs.update(overrides) return _make_run(held, **kwargs) @@ -246,7 +246,7 @@ def test_only_failed_copies_the_non_complete_units_and_the_stage_suffix(held, cl "production", "brusholme", f"re-run of failed units of {seed}: step 6 demo P6", ["difference", "register", "load"], "abc123", "sha256:deadbeef", None, "s3://settings/overlay.toml", "s3://selection/set.json", "batch", "rebuild", - "rapid_rebuild", 3, "rebuild-trial@1", seed, False) + "rapid", 3, "rebuild-trial@1", seed, False) units = _all(held, """ SELECT u.stage, u.unit_kind, u.unit_id, u.state, s.unit_id, s.run diff --git a/tests/db/test_sources_child_table.py b/tests/db/test_sources_child_table.py index 355d1caa..3f9c8e7b 100644 --- a/tests/db/test_sources_child_table.py +++ b/tests/db/test_sources_child_table.py @@ -63,7 +63,7 @@ def test_create_makes_devs_child_table_once(conn): "SELECT c.relpersistence, pg_get_userbyid(c.relowner), p.relname " "FROM pg_class c JOIN pg_inherits i ON i.inhrelid = c.oid " "JOIN pg_class p ON p.oid = i.inhparent WHERE c.relname = %s", (table,)) - assert cur.fetchone() == ("u", "rapidporole", "sources") + assert cur.fetchone() == ("p", "rapidporole", "sources") cur.execute("SELECT indexname FROM pg_indexes WHERE tablename = %s", (table,)) indexes = {r[0] for r in cur.fetchall()} diff --git a/tests/db/test_statistics.py b/tests/db/test_statistics.py index 1de9b787..770ac25e 100644 --- a/tests/db/test_statistics.py +++ b/tests/db/test_statistics.py @@ -156,7 +156,7 @@ def test_statistics_writes_one_row_per_object_and_one_complete_set(conn, tmp_pat (entry.instance,)) assert [r[0] for r in cur.fetchall()] == [association] cur.execute("SELECT relpersistence FROM pg_class WHERE relname = %s", (TABLE,)) - assert cur.fetchone() == ("u",) + assert cur.fetchone() == ("p",) # A second attempt on the same membership reuses the set: no second row per object # (the first attempt succeeded). diff --git a/tests/unit/fakereleasedb.py b/tests/unit/fakereleasedb.py index f2ec978c..d1476826 100644 --- a/tests/unit/fakereleasedb.py +++ b/tests/unit/fakereleasedb.py @@ -24,7 +24,7 @@ def __init__(self, dbname: str): class FakeReleaseDB: - def __init__(self, *, dbname: str = "rapid_rebuild", log_path: Path | None = None, + def __init__(self, *, dbname: str = "rapid", log_path: Path | None = None, releases_table: bool = True): self.info = _Info(dbname) self.log_path = log_path diff --git a/tests/unit/test_release_cut.py b/tests/unit/test_release_cut.py index e0f94bb0..77d8ceda 100644 --- a/tests/unit/test_release_cut.py +++ b/tests/unit/test_release_cut.py @@ -74,7 +74,7 @@ def test_full_cut_runs_in_order_and_records_every_state(setup): assert row["schema_version"] == "20260924-08-releases.sql" assert (row["image_digest"], row["image_ref"]) == (DIGEST, IMAGE_REF) assert row["completed_at"] is not None - assert row["notes"]["database_target"] == "rapid_rebuild" + assert row["notes"]["database_target"] == "rapid" assert row["notes"]["skipped"] == [] assert row["notes"]["pins_rows"] == 2 assert db.deployments("rebuild-v0.1") == { @@ -225,7 +225,7 @@ def test_resume_refuses_a_different_database_target(setup): with pytest.raises(HookFailed): _cut(db, repo, hooks) (hooks / "fail-pins").unlink() - db.info.dbname = "rapid" + db.info.dbname = "rapid_other" with pytest.raises(ReleaseRefused, match="database target"): _cut(db, repo, hooks, resume="rebuild-v0.1")