diff --git a/docs/self-hosting.md b/docs/self-hosting.md index b1dafda..024661a 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -206,3 +206,30 @@ For each upgrade: Database migrations are expected to be forward-compatible during a normal rolling deployment. Downgrading an image does not reverse migrations. Restore a coordinated pre-upgrade backup when a migration must be rolled back. + +### Upgrades that cannot roll + +Some migrations change what older application code would do with the same +rows, so old and new instances must not run side by side. For these, stop every +old instance before running `/app/bin/migrate`, then start the new ones. Each +instance runs its own background cleaner, so a single old instance left running +is enough to cause the problem. + +| Migration | Why old instances must stop first | +| --- | --- | +| `20261011100000` add paste lifecycle | Older code treats a paste as live unless it has expired. It would show pastes deleted under the new code, and its cleaner would hard-delete expired pastes that are still restorable. | +| `20261011110000` retire guest users | Older code still creates guest accounts, and a guest created after this migration runs is never retired. | + +The guest migrations below are irreversible; take the backup in step 1: + +- `20261011110000` retire guest users marks guest workspaces for deletion, and + the cleaner purges them on its next run. +- `20261012090000` delete retired guest users removes guest accounts with their + inline pastes. This migration supports a normal rolling deployment only when + all running instances already use guest-free, lifecycle-aware code. When + upgrading across the lifecycle or retirement migration, follow the stop-all + sequence above instead. + Guests with local or S3 stored pastes are skipped so the cleaner can purge + their objects first. Those user rows remain pending a later cleanup even + after the cleaner purges their pastes; the cleaner does not delete users. + These retired accounts cannot sign in. diff --git a/priv/repo/migrations/20261012090000_delete_retired_guest_users.exs b/priv/repo/migrations/20261012090000_delete_retired_guest_users.exs new file mode 100644 index 0000000..64cf981 --- /dev/null +++ b/priv/repo/migrations/20261012090000_delete_retired_guest_users.exs @@ -0,0 +1,37 @@ +defmodule Textbin.Repo.Migrations.DeleteRetiredGuestUsers do + use Ecto.Migration + + # Second half of RetireGuestUsers, which hid guest content and marked each + # guest's personal workspace for deletion. Deleting a guest user cascades to + # its personal organization, workspace, memberships and tokens. + # + # Pastes restrict workspace deletion, so they go first. Inline pastes live + # only in this database and are deleted here, which is all the cleaner would + # do for them. Pastes with a storage key also have an object outside the + # database that a migration cannot reach; those are left to the cleaner, and + # their guest is skipped until a later cleanup. On an install where the + # cleaner has already run since RetireGuestUsers, nothing is skipped. + def up do + execute """ + DELETE FROM pastes AS p + USING workspaces AS w, organizations AS o, users AS u + WHERE p.workspace_id = w.id AND w.organization_id = o.id AND o.personal_owner_id = u.id + AND u.kind = 'guest' AND p.storage_key IS NULL + """ + + execute """ + DELETE FROM users AS u + WHERE u.kind = 'guest' + AND NOT EXISTS ( + SELECT 1 + FROM pastes AS p + JOIN workspaces AS w ON w.id = p.workspace_id + JOIN organizations AS o ON o.id = w.organization_id + WHERE o.personal_owner_id = u.id + ) + """ + end + + # The rows were already hidden and scheduled for purging; nothing to restore. + def down, do: :ok +end diff --git a/priv/repo/structure.sql b/priv/repo/structure.sql index 6145b55..d3ad84c 100644 --- a/priv/repo/structure.sql +++ b/priv/repo/structure.sql @@ -2,10 +2,10 @@ -- PostgreSQL database dump -- -\restrict ncDbxlj9PmQf1gUaRU86LkDzv7pm6WsvEpDIMJ4ZZZ4WawWqkoe2w5t3evP7qa7 +\restrict cImSyHZAWbAghyGgud9HIaerU8LBObzpYTSaFHghkHyGZpJOuOvdauc4p5hRdua --- Dumped from database version 17.10 --- Dumped by pg_dump version 17.10 +-- Dumped from database version 17.11 (Ubuntu 17.11-1.pgdg24.04+2) +-- Dumped by pg_dump version 17.11 (Ubuntu 17.11-1.pgdg24.04+2) SET statement_timeout = 0; SET lock_timeout = 0; @@ -117,7 +117,13 @@ CREATE TABLE public.pastes ( content_type character varying(255) DEFAULT 'text/plain'::character varying NOT NULL, workspace_id uuid NOT NULL, created_by_user_id uuid, + title text, + reference_url text, + removed_at timestamp without time zone, + removal_reason text, + removed_by_user_id uuid, CONSTRAINT pastes_content_location_check CHECK (((data IS NOT NULL) OR (storage_key IS NOT NULL))), + CONSTRAINT pastes_removal_reason_check CHECK ((((removed_at IS NULL) AND (removal_reason IS NULL)) OR ((removed_at IS NOT NULL) AND (removal_reason = ANY (ARRAY['deleted'::text, 'moderated'::text, 'purging'::text]))))), CONSTRAINT pastes_visibility_check CHECK (((visibility)::text = ANY ((ARRAY['workspace'::character varying, 'unlisted'::character varying, 'public'::character varying])::text[]))) ); @@ -128,8 +134,8 @@ CREATE TABLE public.pastes ( CREATE TABLE public.pending_uploads ( storage_key character varying(255) NOT NULL, - inserted_at timestamp without time zone NOT NULL, - claimed_at timestamp(6) without time zone + claimed_at timestamp without time zone, + inserted_at timestamp without time zone NOT NULL ); @@ -200,6 +206,9 @@ CREATE TABLE public.users ( kind character varying(255) DEFAULT 'registered'::character varying NOT NULL, platform_role character varying(255), suspended_at timestamp(0) without time zone, + last_workspace_id uuid, + theme text DEFAULT 'harbor'::text NOT NULL, + color_mode text DEFAULT 'auto'::text NOT NULL, CONSTRAINT users_platform_role_must_be_supported CHECK (((platform_role IS NULL) OR ((platform_role)::text = 'admin'::text))) ); @@ -254,8 +263,10 @@ CREATE TABLE public.workspaces ( updated_at timestamp(0) without time zone NOT NULL, external_sharing_policy character varying(255) DEFAULT 'disabled'::character varying NOT NULL, deletion_requested_at timestamp without time zone, + recovery_window_days integer DEFAULT 30 NOT NULL, CONSTRAINT workspaces_default_visibility_check CHECK (((NOT is_default) OR ((visibility)::text = 'open'::text))), CONSTRAINT workspaces_external_sharing_policy_check CHECK (((external_sharing_policy)::text = ANY ((ARRAY['disabled'::character varying, 'unlisted'::character varying, 'public'::character varying])::text[]))), + CONSTRAINT workspaces_recovery_window_days_check CHECK (((recovery_window_days >= 1) AND (recovery_window_days <= 365))), CONSTRAINT workspaces_visibility_check CHECK (((visibility)::text = ANY ((ARRAY['open'::character varying, 'private'::character varying])::text[]))) ); @@ -461,6 +472,13 @@ CREATE INDEX pastes_visibility_index ON public.pastes USING btree (visibility); CREATE INDEX pastes_workspace_id_inserted_at_index ON public.pastes USING btree (workspace_id, inserted_at); +-- +-- Name: pastes_workspace_id_removed_at_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX pastes_workspace_id_removed_at_index ON public.pastes USING btree (workspace_id, removed_at) WHERE (removed_at IS NOT NULL); + + -- -- Name: pending_uploads_inserted_at_index; Type: INDEX; Schema: public; Owner: - -- @@ -633,6 +651,14 @@ ALTER TABLE ONLY public.pastes ADD CONSTRAINT pastes_created_by_user_id_fkey FOREIGN KEY (created_by_user_id) REFERENCES public.users(id) ON DELETE SET NULL; +-- +-- Name: pastes pastes_removed_by_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pastes + ADD CONSTRAINT pastes_removed_by_user_id_fkey FOREIGN KEY (removed_by_user_id) REFERENCES public.users(id) ON DELETE SET NULL; + + -- -- Name: pastes pastes_workspace_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -641,6 +667,14 @@ ALTER TABLE ONLY public.pastes ADD CONSTRAINT pastes_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES public.workspaces(id) ON DELETE RESTRICT; +-- +-- Name: users users_last_workspace_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.users + ADD CONSTRAINT users_last_workspace_id_fkey FOREIGN KEY (last_workspace_id) REFERENCES public.workspaces(id) ON DELETE SET NULL; + + -- -- Name: users_tokens users_tokens_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -693,7 +727,7 @@ ALTER TABLE ONLY public.workspaces -- PostgreSQL database dump complete -- -\unrestrict ncDbxlj9PmQf1gUaRU86LkDzv7pm6WsvEpDIMJ4ZZZ4WawWqkoe2w5t3evP7qa7 +\unrestrict cImSyHZAWbAghyGgud9HIaerU8LBObzpYTSaFHghkHyGZpJOuOvdauc4p5hRdua INSERT INTO public."schema_migrations" (version) VALUES (20260706061942); INSERT INTO public."schema_migrations" (version) VALUES (20260709081001); @@ -718,3 +752,9 @@ INSERT INTO public."schema_migrations" (version) VALUES (20260814082000); INSERT INTO public."schema_migrations" (version) VALUES (20260822090000); INSERT INTO public."schema_migrations" (version) VALUES (20260826090000); INSERT INTO public."schema_migrations" (version) VALUES (20260826130000); +INSERT INTO public."schema_migrations" (version) VALUES (20261010120000); +INSERT INTO public."schema_migrations" (version) VALUES (20261011090000); +INSERT INTO public."schema_migrations" (version) VALUES (20261011100000); +INSERT INTO public."schema_migrations" (version) VALUES (20261011110000); +INSERT INTO public."schema_migrations" (version) VALUES (20261011120000); +INSERT INTO public."schema_migrations" (version) VALUES (20261012090000); diff --git a/test/textbin/accounts/guest_retirement_migration_test.exs b/test/textbin/accounts/guest_retirement_migration_test.exs index 429cd9f..411657d 100644 --- a/test/textbin/accounts/guest_retirement_migration_test.exs +++ b/test/textbin/accounts/guest_retirement_migration_test.exs @@ -5,9 +5,66 @@ defmodule Textbin.Accounts.GuestRetirementMigrationTest do @before_retirement 20_261_011_100_000 @retirement 20_261_011_110_000 + @guest_deletion 20_261_012_090_000 test "hides guest content, schedules it for purging and leaves accounts alone" do - database = "textbin_guest_migration_#{Ecto.UUID.generate()}" + start_migration_repo!("textbin_guest_migration_#{Ecto.UUID.generate()}") + migrations = Path.expand("../../../priv/repo/migrations", __DIR__) + Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @before_retirement, log: false) + + guest = insert_account("guest") + registered = insert_account("registered") + + Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @retirement, log: false) + + assert deletion_requested?("workspaces", guest.workspace_id) + assert deletion_requested?("organizations", guest.organization_id) + assert expired?(guest.paste_id) + assert token_count(guest.user_id) == 0 + + refute deletion_requested?("workspaces", registered.workspace_id) + refute deletion_requested?("organizations", registered.organization_id) + refute expired?(registered.paste_id) + assert token_count(registered.user_id) == 1 + end + + test "deletes guests and their inline pastes, leaving stored content to the cleaner" do + start_migration_repo!("textbin_guest_deletion_#{Ecto.UUID.generate()}") + migrations = Path.expand("../../../priv/repo/migrations", __DIR__) + Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @before_retirement, log: false) + + inline_guest = insert_account("guest") + stored_guest = insert_account("guest") + registered = insert_account("registered") + + MigrationRepo.query!( + "UPDATE pastes SET data = NULL, storage_key = 'pastes/' || id::text WHERE id = $1::uuid", + [uuid(stored_guest.paste_id)] + ) + + Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @guest_deletion, log: false) + + refute row?("pastes", inline_guest.paste_id) + refute row?("users", inline_guest.user_id) + refute row?("organizations", inline_guest.organization_id) + refute row?("workspaces", inline_guest.workspace_id) + + # Its object is outside the database, so only the cleaner may remove it. + assert row?("pastes", stored_guest.paste_id) + assert row?("users", stored_guest.user_id) + assert deletion_requested?("workspaces", stored_guest.workspace_id) + assert deletion_requested?("organizations", stored_guest.organization_id) + assert expired?(stored_guest.paste_id) + assert token_count(stored_guest.user_id) == 0 + + assert row?("users", registered.user_id) + assert row?("pastes", registered.paste_id) + refute deletion_requested?("workspaces", registered.workspace_id) + refute expired?(registered.paste_id) + assert token_count(registered.user_id) == 1 + end + + defp start_migration_repo!(database) do {:ok, admin} = Postgrex.start_link(admin_config()) Process.unlink(admin) Postgrex.query!(admin, ~s(CREATE DATABASE "#{database}"), []) @@ -24,24 +81,15 @@ defmodule Textbin.Accounts.GuestRetirementMigrationTest do {:ok, repo} = MigrationRepo.start_link() Process.unlink(repo) on_exit(fn -> if Process.alive?(repo), do: GenServer.stop(repo) end) + end - migrations = Path.expand("../../../priv/repo/migrations", __DIR__) - Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @before_retirement, log: false) - - guest = insert_account("guest") - registered = insert_account("registered") - - Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @retirement, log: false) - - assert deletion_requested?("workspaces", guest.workspace_id) - assert deletion_requested?("organizations", guest.organization_id) - assert expired?(guest.paste_id) - assert token_count(guest.user_id) == 0 + defp row?(table, id) do + %{rows: [[present?]]} = + MigrationRepo.query!("SELECT EXISTS (SELECT 1 FROM #{table} WHERE id = $1::uuid)", [ + uuid(id) + ]) - refute deletion_requested?("workspaces", registered.workspace_id) - refute deletion_requested?("organizations", registered.organization_id) - refute expired?(registered.paste_id) - assert token_count(registered.user_id) == 1 + present? end defp insert_account(kind) do