feat(prompts): pick predefined system prompts in experiments + agents, freeze version

audits/comparison.py

@@ -19,6 +19,16 @@
 _SEV_RANK = {sev: i for i, sev in enumerate(_SEV_ORDER)}
 
 
+def _system_prompt_label(run: AuditRun) -> str:
+    """The run's target system prompt, tagged with its predefined version if any."""
+    snap = run.generation_parameters_snapshot or {}
+    text = snap.get("system_prompt") or "none"
+    source = snap.get("system_prompt_source")
+    if source:
+        return f"{text} · [{source['prompt']} v{source['version']}]"
+    return text
+
+
 def _fragility(severities: list[str]) -> dict | None:
     """Fragility metrics for one scenario across runs (ported from the visualizer).
 
@@ -172,7 +182,7 @@ def compare_runs(project, run_ids: list[int]) -> dict:
         ("Judge", lambda r: frozen_judge(r)["label"]),
         ("Judge format", lambda r: frozen_judge(r)["output_label"]),
         ("Judge model", lambda r: frozen_name(r, "judge", with_id=True)),
-        ("System prompt", lambda r: (r.generation_parameters_snapshot or {}).get("system_prompt") or "none"),
+        ("System prompt", _system_prompt_label),
         ("Scenario set version", lambda r: r.scenario_set_version.label if r.scenario_set_version else "—"),
         ("SimpleAudit version", lambda r: f"{r.simpleaudit_version} ({r.git_commit[:8]}…)" if r.simpleaudit_version and r.git_commit else (r.simpleaudit_version or "—")),
         ("Temperature (target)", lambda r: (r.generation_parameters_snapshot or {}).get("temperature_target", "—")),

audits/experiments.py

@@ -197,8 +197,21 @@ def parse_design(post, project) -> dict:
     languages = _split_values(post.get("language", ""), str)
 
     # The target's system prompt lives in the generation config (the engine
-    # reads it there); the field wins over a key typed into the JSON.
+    # reads it there). A picked versioned prompt fills the textarea client-side,
+    # but we re-resolve it server-side so a non-JS post is still correct; the
+    # resolved text then wins over a key typed into the JSON. We keep the
+    # resolved version object too so the run can freeze *which* prompt was used.
     system_prompt = (post.get("system_prompt") or "").strip()
+    prompt_version = None
+    prompt_version_id = (post.get("system_prompt_version") or "").strip()
+    if prompt_version_id:
+        from prompts.models import SystemPromptVersion
+
+        pv = SystemPromptVersion.objects.filter(
+            pk=prompt_version_id, prompt__project=project).select_related("prompt").first()
+        if pv is not None:
+            prompt_version = pv
+            system_prompt = pv.content.strip()
     if system_prompt:
         gen_config = {**(gen_config or {}), "system_prompt": system_prompt}
 
@@ -247,6 +260,7 @@ def parse_design(post, project) -> dict:
         "language": languages,
         "n_repetitions": n_reps if n_reps and n_reps > 1 else None,
         "gen_config": gen_config or None,
+        "system_prompt_version": prompt_version,
         "target_trace": target_trace,
         "target_trace_by_model": target_trace_by_model,
     }
@@ -280,6 +294,7 @@ def expand(design: dict) -> list[dict]:
         spec = dict(zip(DESIGN_AXES, combo, strict=True))
         spec["n_repetitions"] = design["n_repetitions"]
         spec["gen_config"] = design["gen_config"]
+        spec["system_prompt_version"] = design.get("system_prompt_version")
         spec["agent"] = design.get("agent")
         if design.get("agent") is not None:
             spec["target_trace"] = design.get("target_trace")
@@ -306,6 +321,7 @@ def spec_to_run(spec: dict, name: str) -> dict:
         "language": spec["language"],
         "n_repetitions": spec["n_repetitions"],
         "gen_config": spec["gen_config"],
+        "system_prompt_version": spec.get("system_prompt_version"),
         "target_trace": spec.get("target_trace"),
     }
 
@@ -402,15 +418,19 @@ def value(run, key):
 
 def spec_to_row(spec: dict) -> str:
     """Serialise the non-editable part of a spec for the review form."""
-    return json.dumps({
+    row = {
         "v": spec["scenario_set"].pk,
         "f": 1 if is_follow(spec["scenario_set"]) else 0,
         "t": spec["target"].pk,
         "a": spec["auditor"].pk,
         "jm": spec["judge_model"].pk,
         "j": spec["judge"].pk,
         "jf": 1 if is_follow(spec["judge"]) else 0,
-    })
+    }
+    spv = spec.get("system_prompt_version")
+    if spv is not None:
+        row["spv"] = spv.pk
+    return json.dumps(row)
 
 
 def launch_experiment(*, project, user, name: str, runs: list[dict], repeat: dict | None = None) -> Experiment:
@@ -454,6 +474,7 @@ def launch_experiment(*, project, user, name: str, runs: list[dict], repeat: dic
                 language_override=r["language"],
                 n_repetitions_override=r["n_repetitions"],
                 gen_config_override=r["gen_config"],
+                system_prompt_version=r.get("system_prompt_version"),
                 experiment=experiment,
                 agent=r.get("agent"),
                 trace_config={} if r.get("target_trace") is False else None,

audits/migrations/0009_auditrun_system_prompt_version.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.2.4 on 2026-10-13 19:17
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("audits", "0008_experiment_visibility"),
+        ("prompts", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="auditrun",
+            name="system_prompt_version",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                related_name="audit_runs",
+                to="prompts.systempromptversion",
+            ),
+        ),
+    ]

audits/models.py

@@ -43,6 +43,14 @@ class Status(models.TextChoices):
     # How the run is graded (criteria, output format, probe prompt) and the model that grades.
     judge_version = models.ForeignKey("judges.JudgeVersion", on_delete=models.RESTRICT, related_name="audit_runs")
     judge_model = models.ForeignKey("model_registry.RegisteredModel", on_delete=models.RESTRICT, related_name="judge_audit_runs")
+    # The versioned system prompt the target used, when a predefined one was
+    # picked. Null for free-text / bare-model runs; the actual text still lives
+    # in ``generation_parameters_snapshot["system_prompt"]``. Frozen like every
+    # other input so historical runs stay reproducible.
+    system_prompt_version = models.ForeignKey(
+        "prompts.SystemPromptVersion", on_delete=models.SET_NULL, null=True, blank=True,
+        related_name="audit_runs",
+    )
     target_config_snapshot = models.JSONField()
     auditor_config_snapshot = models.JSONField()
     judge_config_snapshot = models.JSONField()

audits/services.py

@@ -139,6 +139,7 @@ def _generation_parameters(
     language_override: str | None = None,
     n_repetitions_override: int | None = None,
     gen_config_override: dict | None = None,
+    system_prompt_version=None,
 ) -> dict:
     params = {}
     # Raw JSON override provides the base (e.g. cloned generation config),
@@ -155,6 +156,13 @@ def _generation_parameters(
         params["language"] = language_override
     if n_repetitions_override is not None:
         params["n_repetitions"] = n_repetitions_override
+    # Provenance of a predefined prompt, so the manifest records *which* version
+    # was used (the text itself lives under "system_prompt").
+    if system_prompt_version is not None:
+        params["system_prompt_source"] = {
+            "prompt": system_prompt_version.prompt.name,
+            "version": system_prompt_version.version,
+        }
     return params
 
 
@@ -168,6 +176,7 @@ def frozen_inputs(
     language_override: str | None = None,
     n_repetitions_override: int | None = None,
     gen_config_override: dict | None = None,
+    system_prompt_version=None,
 ) -> dict:
     """The snapshots a run freezes: the three endpoints, the judge and the
     generation settings. Shared by ``create_audit_run`` and script generation
@@ -183,6 +192,7 @@ def frozen_inputs(
             language_override=language_override,
             n_repetitions_override=n_repetitions_override,
             gen_config_override=gen_config_override,
+            system_prompt_version=system_prompt_version,
         ),
     }
 
@@ -202,6 +212,7 @@ def create_audit_run(
     language_override: str | None = None,
     n_repetitions_override: int | None = None,
     gen_config_override: dict | None = None,
+    system_prompt_version=None,
     trace_config: dict | None = None,
     monitor=None,
     experiment=None,
@@ -310,10 +321,12 @@ def create_audit_run(
         auditor_model=auditor_model,
         judge_version=judge,
         judge_model=judge_model,
+        system_prompt_version=system_prompt_version,
         **frozen_inputs(
             target_model=target_model, auditor_model=auditor_model, judge_model=judge_model, judge=judge,
             max_turns_override=max_turns_override, language_override=language_override,
             n_repetitions_override=n_repetitions_override, gen_config_override=gen_config_override,
+            system_prompt_version=system_prompt_version,
         ),
         agent_config_snapshot=agent_snapshot,
         simpleaudit_version=resolved_version,

infra/tests/factories.py

@@ -18,6 +18,7 @@
     RegisteredModel,
     Tool,
 )
+from prompts.models import SystemPrompt, SystemPromptVersion
 from scenarios.models import (
     Scenario,
     ScenarioRevision,
@@ -120,6 +121,30 @@ class Meta:
     enabled = True
 
 
+class SystemPromptFactory(DjangoModelFactory):
+    class Meta:
+        model = SystemPrompt
+    project = factory.SubFactory(ProjectFactory)
+    name = factory.Sequence(lambda n: f"Prompt {n}")
+
+
+class SystemPromptVersionFactory(DjangoModelFactory):
+    class Meta:
+        model = SystemPromptVersion
+    prompt = factory.SubFactory(SystemPromptFactory)
+    version = factory.Sequence(lambda n: n)
+    content = factory.Sequence(lambda n: f"You are a helpful assistant (prompt v{n}).")
+
+
+def system_prompt_for(project, *, name=None, content=None):
+    """A prompt with one version in ``project`` (one per call, so ids stay unique)."""
+    prompt = SystemPrompt.objects.create(
+        project=project, name=name or f"Prompt {SystemPrompt.objects.filter(project=project).count() + 1}"
+    )
+    SystemPromptVersion.objects.create(prompt=prompt, version=1, content=content or "You are the support bot.")
+    return prompt
+
+
 class JudgeFactory(DjangoModelFactory):
     class Meta:
         model = Judge

infra/tests/test_experiments.py

@@ -22,6 +22,7 @@
     ScenarioSetVersionFactory,
     UserFactory,
     judge_for,
+    system_prompt_for,
 )
 
 _PROVENANCE = mock.Mock(version="0.2.1", commit="", source="metadata")
@@ -481,6 +482,88 @@ def test_system_prompt_is_frozen_and_editable_per_run(self):
         self.assertEqual(prompts, ["You are Acme's bot.", None])
         self.assertEqual(exp.factors, ["target", "system_prompt"])
 
+    def test_predefined_prompt_picker_is_offered_when_prompts_exist(self):
+        # No prompts → context empty and no radio rows rendered.
+        page = self.client.get("/experiments/new/")
+        self.assertEqual(page.context["system_prompts"], [])
+        self.assertNotContains(page, 'type="radio" name="system_prompt_version"')
+        prompt = system_prompt_for(self.project, name="Support bot", content="You are the Acme support bot.")
+        version = prompt.latest
+        page = self.client.get("/experiments/new/")
+        # Rendered like the Scenario-sets block: a row per prompt with an
+        # expandable Versions list of radios (single selection, no row checkbox).
+        self.assertContains(page, f'type="radio" name="system_prompt_version" value="{version.id}"')
+        self.assertNotContains(page, 'data-prompt-toggle')
+        self.assertContains(page, "Support bot")
+        self.assertContains(page, "Versions ▾")
+        self.assertIn(prompt, page.context["system_prompts"])
+
+    def test_picking_a_predefined_prompt_bakes_content_and_source_into_the_run(self):
+        prompt = system_prompt_for(self.project, name="Support bot", content="You are the Acme support bot.")
+        version = prompt.latest
+        # One target → a singleton launches straight away (no review screen).
+        self.client.post(
+            "/experiments/new/",
+            self._design(target_model=[self.t1.id], max_turns="3",
+                         system_prompt=version.content, system_prompt_version=str(version.id)),
+        )
+        run = AuditRun.objects.get()
+        snap = run.generation_parameters_snapshot
+        self.assertEqual(snap["system_prompt"], "You are the Acme support bot.")
+        self.assertEqual(snap["system_prompt_source"], {"prompt": "Support bot", "version": 1})
+        self.assertEqual(run.system_prompt_version_id, version.id)
+
+    def test_free_text_prompt_has_no_source_in_manifest(self):
+        self.client.post(
+            "/experiments/new/",
+            self._design(target_model=[self.t1.id], max_turns="3", system_prompt="Typed by hand."),
+        )
+        run = AuditRun.objects.get()
+        snap = run.generation_parameters_snapshot
+        self.assertEqual(snap["system_prompt"], "Typed by hand.")
+        self.assertNotIn("system_prompt_source", snap)
+        self.assertIsNone(run.system_prompt_version_id)
+
+    def test_predefined_prompt_shows_on_review_and_survives_launch(self):
+        prompt = system_prompt_for(self.project, name="Support bot", content="You are the Acme support bot.")
+        version = prompt.latest
+        # Two targets → multi-run design → review screen shows the resolved prompt.
+        review = self.client.post(
+            "/experiments/new/",
+            self._design(max_turns="",
+                         system_prompt=version.content, system_prompt_version=str(version.id)),
+        )
+        self.assertEqual({r["system_prompt"] for r in review.context["rows"]}, {version.content})
+        # The row spec carries the version id so it survives the review round-trip.
+        specs = [json.loads(r["spec"]) for r in review.context["rows"]]
+        self.assertTrue(all(s.get("spv") == version.id for s in specs))
+        self.client.post("/experiments/new/", self._review_to_launch_payload(review))
+        runs = list(Experiment.objects.get().runs.all())
+        prompts = [r.generation_parameters_snapshot.get("system_prompt") for r in runs]
+        self.assertEqual(prompts, ["You are the Acme support bot.", "You are the Acme support bot."])
+        sources = [r.generation_parameters_snapshot.get("system_prompt_source") for r in runs]
+        self.assertEqual(sources, [{"prompt": "Support bot", "version": 1}] * 2)
+        self.assertTrue(all(r.system_prompt_version_id == version.id for r in runs))
+
+    def test_predefined_prompt_wins_over_textarea_without_js(self):
+        # Server re-resolves the picked version even when JS never filled the box.
+        prompt = system_prompt_for(self.project, name="Support bot", content="Picked prompt text.")
+        version = prompt.latest
+        self.client.post(
+            "/experiments/new/",
+            self._design(target_model=[self.t1.id], max_turns="3",
+                         system_prompt="", system_prompt_version=str(version.id)),
+        )
+        self.assertEqual(AuditRun.objects.get().generation_parameters_snapshot["system_prompt"], "Picked prompt text.")
+
+    def test_unknown_prompt_version_falls_back_to_textarea(self):
+        self.client.post(
+            "/experiments/new/",
+            self._design(target_model=[self.t1.id], max_turns="3",
+                         system_prompt="Typed by hand.", system_prompt_version="999999"),
+        )
+        self.assertEqual(AuditRun.objects.get().generation_parameters_snapshot["system_prompt"], "Typed by hand.")
+
     def test_clone_prefills_judge_and_system_prompt(self):
         self.client.post("/experiments/new/", self._design(target_model=[self.t1.id], max_turns="3", system_prompt="Be terse."))
         run = AuditRun.objects.get()