chore: delete retired guests, refresh the schema snapshot, document non-rolling upgrades

docs/self-hosting.md

@@ -206,3 +206,30 @@ For each upgrade:
 Database migrations are expected to be forward-compatible during a normal
 rolling deployment. Downgrading an image does not reverse migrations. Restore a
 coordinated pre-upgrade backup when a migration must be rolled back.
+
+### Upgrades that cannot roll
+
+Some migrations change what older application code would do with the same
+rows, so old and new instances must not run side by side. For these, stop every
+old instance before running `/app/bin/migrate`, then start the new ones. Each
+instance runs its own background cleaner, so a single old instance left running
+is enough to cause the problem.
+
+| Migration | Why old instances must stop first |
+| --- | --- |
+| `20261011100000` add paste lifecycle | Older code treats a paste as live unless it has expired. It would show pastes deleted under the new code, and its cleaner would hard-delete expired pastes that are still restorable. |
+| `20261011110000` retire guest users | Older code still creates guest accounts, and a guest created after this migration runs is never retired. |
+
+The guest migrations below are irreversible; take the backup in step 1:
+
+- `20261011110000` retire guest users marks guest workspaces for deletion, and
+  the cleaner purges them on its next run.
+- `20261012090000` delete retired guest users removes guest accounts with their
+  inline pastes. This migration supports a normal rolling deployment only when
+  all running instances already use guest-free, lifecycle-aware code. When
+  upgrading across the lifecycle or retirement migration, follow the stop-all
+  sequence above instead.
+  Guests with local or S3 stored pastes are skipped so the cleaner can purge
+  their objects first. Those user rows remain pending a later cleanup even
+  after the cleaner purges their pastes; the cleaner does not delete users.
+  These retired accounts cannot sign in.

priv/repo/migrations/20261012090000_delete_retired_guest_users.exs

@@ -0,0 +1,37 @@
+defmodule Textbin.Repo.Migrations.DeleteRetiredGuestUsers do
+  use Ecto.Migration
+
+  # Second half of RetireGuestUsers, which hid guest content and marked each
+  # guest's personal workspace for deletion. Deleting a guest user cascades to
+  # its personal organization, workspace, memberships and tokens.
+  #
+  # Pastes restrict workspace deletion, so they go first. Inline pastes live
+  # only in this database and are deleted here, which is all the cleaner would
+  # do for them. Pastes with a storage key also have an object outside the
+  # database that a migration cannot reach; those are left to the cleaner, and
+  # their guest is skipped until a later cleanup. On an install where the
+  # cleaner has already run since RetireGuestUsers, nothing is skipped.
+  def up do
+    execute """
+    DELETE FROM pastes AS p
+    USING workspaces AS w, organizations AS o, users AS u
+    WHERE p.workspace_id = w.id AND w.organization_id = o.id AND o.personal_owner_id = u.id
+      AND u.kind = 'guest' AND p.storage_key IS NULL
+    """
+
+    execute """
+    DELETE FROM users AS u
+    WHERE u.kind = 'guest'
+      AND NOT EXISTS (
+        SELECT 1
+        FROM pastes AS p
+        JOIN workspaces AS w ON w.id = p.workspace_id
+        JOIN organizations AS o ON o.id = w.organization_id
+        WHERE o.personal_owner_id = u.id
+      )
+    """
+  end
+
+  # The rows were already hidden and scheduled for purging; nothing to restore.
+  def down, do: :ok
+end

priv/repo/structure.sql

@@ -2,10 +2,10 @@
 -- PostgreSQL database dump
 --
 
-\restrict ncDbxlj9PmQf1gUaRU86LkDzv7pm6WsvEpDIMJ4ZZZ4WawWqkoe2w5t3evP7qa7
+\restrict cImSyHZAWbAghyGgud9HIaerU8LBObzpYTSaFHghkHyGZpJOuOvdauc4p5hRdua
 
--- Dumped from database version 17.10
--- Dumped by pg_dump version 17.10
+-- Dumped from database version 17.11 (Ubuntu 17.11-1.pgdg24.04+2)
+-- Dumped by pg_dump version 17.11 (Ubuntu 17.11-1.pgdg24.04+2)
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
@@ -117,7 +117,13 @@ CREATE TABLE public.pastes (
     content_type character varying(255) DEFAULT 'text/plain'::character varying NOT NULL,
     workspace_id uuid NOT NULL,
     created_by_user_id uuid,
+    title text,
+    reference_url text,
+    removed_at timestamp without time zone,
+    removal_reason text,
+    removed_by_user_id uuid,
     CONSTRAINT pastes_content_location_check CHECK (((data IS NOT NULL) OR (storage_key IS NOT NULL))),
+    CONSTRAINT pastes_removal_reason_check CHECK ((((removed_at IS NULL) AND (removal_reason IS NULL)) OR ((removed_at IS NOT NULL) AND (removal_reason = ANY (ARRAY['deleted'::text, 'moderated'::text, 'purging'::text]))))),
     CONSTRAINT pastes_visibility_check CHECK (((visibility)::text = ANY ((ARRAY['workspace'::character varying, 'unlisted'::character varying, 'public'::character varying])::text[])))
 );
 
@@ -128,8 +134,8 @@ CREATE TABLE public.pastes (
 
 CREATE TABLE public.pending_uploads (
     storage_key character varying(255) NOT NULL,
-    inserted_at timestamp without time zone NOT NULL,
-    claimed_at timestamp(6) without time zone
+    claimed_at timestamp without time zone,
+    inserted_at timestamp without time zone NOT NULL
 );
 
 
@@ -200,6 +206,9 @@ CREATE TABLE public.users (
     kind character varying(255) DEFAULT 'registered'::character varying NOT NULL,
     platform_role character varying(255),
     suspended_at timestamp(0) without time zone,
+    last_workspace_id uuid,
+    theme text DEFAULT 'harbor'::text NOT NULL,
+    color_mode text DEFAULT 'auto'::text NOT NULL,
     CONSTRAINT users_platform_role_must_be_supported CHECK (((platform_role IS NULL) OR ((platform_role)::text = 'admin'::text)))
 );
 
@@ -254,8 +263,10 @@ CREATE TABLE public.workspaces (
     updated_at timestamp(0) without time zone NOT NULL,
     external_sharing_policy character varying(255) DEFAULT 'disabled'::character varying NOT NULL,
     deletion_requested_at timestamp without time zone,
+    recovery_window_days integer DEFAULT 30 NOT NULL,
     CONSTRAINT workspaces_default_visibility_check CHECK (((NOT is_default) OR ((visibility)::text = 'open'::text))),
     CONSTRAINT workspaces_external_sharing_policy_check CHECK (((external_sharing_policy)::text = ANY ((ARRAY['disabled'::character varying, 'unlisted'::character varying, 'public'::character varying])::text[]))),
+    CONSTRAINT workspaces_recovery_window_days_check CHECK (((recovery_window_days >= 1) AND (recovery_window_days <= 365))),
     CONSTRAINT workspaces_visibility_check CHECK (((visibility)::text = ANY ((ARRAY['open'::character varying, 'private'::character varying])::text[])))
 );
 
@@ -461,6 +472,13 @@ CREATE INDEX pastes_visibility_index ON public.pastes USING btree (visibility);
 CREATE INDEX pastes_workspace_id_inserted_at_index ON public.pastes USING btree (workspace_id, inserted_at);
 
 
+--
+-- Name: pastes_workspace_id_removed_at_index; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX pastes_workspace_id_removed_at_index ON public.pastes USING btree (workspace_id, removed_at) WHERE (removed_at IS NOT NULL);
+
+
 --
 -- Name: pending_uploads_inserted_at_index; Type: INDEX; Schema: public; Owner: -
 --
@@ -633,6 +651,14 @@ ALTER TABLE ONLY public.pastes
     ADD CONSTRAINT pastes_created_by_user_id_fkey FOREIGN KEY (created_by_user_id) REFERENCES public.users(id) ON DELETE SET NULL;
 
 
+--
+-- Name: pastes pastes_removed_by_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.pastes
+    ADD CONSTRAINT pastes_removed_by_user_id_fkey FOREIGN KEY (removed_by_user_id) REFERENCES public.users(id) ON DELETE SET NULL;
+
+
 --
 -- Name: pastes pastes_workspace_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -641,6 +667,14 @@ ALTER TABLE ONLY public.pastes
     ADD CONSTRAINT pastes_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES public.workspaces(id) ON DELETE RESTRICT;
 
 
+--
+-- Name: users users_last_workspace_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.users
+    ADD CONSTRAINT users_last_workspace_id_fkey FOREIGN KEY (last_workspace_id) REFERENCES public.workspaces(id) ON DELETE SET NULL;
+
+
 --
 -- Name: users_tokens users_tokens_user_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: -
 --
@@ -693,7 +727,7 @@ ALTER TABLE ONLY public.workspaces
 -- PostgreSQL database dump complete
 --
 
-\unrestrict ncDbxlj9PmQf1gUaRU86LkDzv7pm6WsvEpDIMJ4ZZZ4WawWqkoe2w5t3evP7qa7
+\unrestrict cImSyHZAWbAghyGgud9HIaerU8LBObzpYTSaFHghkHyGZpJOuOvdauc4p5hRdua
 
 INSERT INTO public."schema_migrations" (version) VALUES (20260706061942);
 INSERT INTO public."schema_migrations" (version) VALUES (20260709081001);
@@ -718,3 +752,9 @@ INSERT INTO public."schema_migrations" (version) VALUES (20260814082000);
 INSERT INTO public."schema_migrations" (version) VALUES (20260822090000);
 INSERT INTO public."schema_migrations" (version) VALUES (20260826090000);
 INSERT INTO public."schema_migrations" (version) VALUES (20260826130000);
+INSERT INTO public."schema_migrations" (version) VALUES (20261010120000);
+INSERT INTO public."schema_migrations" (version) VALUES (20261011090000);
+INSERT INTO public."schema_migrations" (version) VALUES (20261011100000);
+INSERT INTO public."schema_migrations" (version) VALUES (20261011110000);
+INSERT INTO public."schema_migrations" (version) VALUES (20261011120000);
+INSERT INTO public."schema_migrations" (version) VALUES (20261012090000);

test/textbin/accounts/guest_retirement_migration_test.exs

@@ -5,9 +5,66 @@ defmodule Textbin.Accounts.GuestRetirementMigrationTest do
 
   @before_retirement 20_261_011_100_000
   @retirement 20_261_011_110_000
+  @guest_deletion 20_261_012_090_000
 
   test "hides guest content, schedules it for purging and leaves accounts alone" do
-    database = "textbin_guest_migration_#{Ecto.UUID.generate()}"
+    start_migration_repo!("textbin_guest_migration_#{Ecto.UUID.generate()}")
+    migrations = Path.expand("../../../priv/repo/migrations", __DIR__)
+    Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @before_retirement, log: false)
+
+    guest = insert_account("guest")
+    registered = insert_account("registered")
+
+    Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @retirement, log: false)
+
+    assert deletion_requested?("workspaces", guest.workspace_id)
+    assert deletion_requested?("organizations", guest.organization_id)
+    assert expired?(guest.paste_id)
+    assert token_count(guest.user_id) == 0
+
+    refute deletion_requested?("workspaces", registered.workspace_id)
+    refute deletion_requested?("organizations", registered.organization_id)
+    refute expired?(registered.paste_id)
+    assert token_count(registered.user_id) == 1
+  end
+
+  test "deletes guests and their inline pastes, leaving stored content to the cleaner" do
+    start_migration_repo!("textbin_guest_deletion_#{Ecto.UUID.generate()}")
+    migrations = Path.expand("../../../priv/repo/migrations", __DIR__)
+    Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @before_retirement, log: false)
+
+    inline_guest = insert_account("guest")
+    stored_guest = insert_account("guest")
+    registered = insert_account("registered")
+
+    MigrationRepo.query!(
+      "UPDATE pastes SET data = NULL, storage_key = 'pastes/' || id::text WHERE id = $1::uuid",
+      [uuid(stored_guest.paste_id)]
+    )
+
+    Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @guest_deletion, log: false)
+
+    refute row?("pastes", inline_guest.paste_id)
+    refute row?("users", inline_guest.user_id)
+    refute row?("organizations", inline_guest.organization_id)
+    refute row?("workspaces", inline_guest.workspace_id)
+
+    # Its object is outside the database, so only the cleaner may remove it.
+    assert row?("pastes", stored_guest.paste_id)
+    assert row?("users", stored_guest.user_id)
+    assert deletion_requested?("workspaces", stored_guest.workspace_id)
+    assert deletion_requested?("organizations", stored_guest.organization_id)
+    assert expired?(stored_guest.paste_id)
+    assert token_count(stored_guest.user_id) == 0
+
+    assert row?("users", registered.user_id)
+    assert row?("pastes", registered.paste_id)
+    refute deletion_requested?("workspaces", registered.workspace_id)
+    refute expired?(registered.paste_id)
+    assert token_count(registered.user_id) == 1
+  end
+
+  defp start_migration_repo!(database) do
     {:ok, admin} = Postgrex.start_link(admin_config())
     Process.unlink(admin)
     Postgrex.query!(admin, ~s(CREATE DATABASE "#{database}"), [])
@@ -24,24 +81,15 @@ defmodule Textbin.Accounts.GuestRetirementMigrationTest do
     {:ok, repo} = MigrationRepo.start_link()
     Process.unlink(repo)
     on_exit(fn -> if Process.alive?(repo), do: GenServer.stop(repo) end)
+  end
 
-    migrations = Path.expand("../../../priv/repo/migrations", __DIR__)
-    Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @before_retirement, log: false)
-
-    guest = insert_account("guest")
-    registered = insert_account("registered")
-
-    Ecto.Migrator.run(MigrationRepo, migrations, :up, to: @retirement, log: false)
-
-    assert deletion_requested?("workspaces", guest.workspace_id)
-    assert deletion_requested?("organizations", guest.organization_id)
-    assert expired?(guest.paste_id)
-    assert token_count(guest.user_id) == 0
+  defp row?(table, id) do
+    %{rows: [[present?]]} =
+      MigrationRepo.query!("SELECT EXISTS (SELECT 1 FROM #{table} WHERE id = $1::uuid)", [
+        uuid(id)
+      ])
 
-    refute deletion_requested?("workspaces", registered.workspace_id)
-    refute deletion_requested?("organizations", registered.organization_id)
-    refute expired?(registered.paste_id)
-    assert token_count(registered.user_id) == 1
+    present?
   end
 
   defp insert_account(kind) do