From b14ec41e7f6fdeac9ec0296976651eae1741b982 Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Tue, 13 Oct 2026 21:31:29 +0200 Subject: [PATCH 1/3] feat(prompts): pick predefined system prompts in experiments + agents, freeze version - New Experiment: "Predefined prompt" is now the same searchable list as Models / Scenario sets (search box + radio rows); picking one fills the system-prompt box, typing your own clears the selection. - Agent editor: same list picker with click-to-fill (previously a plain select with no sync). - Frozen manifest: runs pin the exact SystemPromptVersion used; the version appears in generation_parameters_snapshot.system_prompt_source and on the run detail + comparison views for reproducibility. - Deletion protection: a system prompt cannot be deleted while an audit run pins it or an agent uses its text (mirrors scenario-set in-use guard). Fixes delete_prompt which previously never deleted. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- audits/comparison.py | 12 ++- audits/experiments.py | 27 ++++++- .../0009_auditrun_system_prompt_version.py | 25 ++++++ audits/models.py | 8 ++ audits/services.py | 13 +++ infra/tests/factories.py | 25 ++++++ infra/tests/test_experiments.py | 80 +++++++++++++++++++ infra/tests/test_prompts.py | 60 +++++++++++++- infra/ui.py | 26 +++++- model_registry/tests/test_agent_domain.py | 21 +++++ prompts/services.py | 30 ++++--- prompts/views.py | 10 ++- templates/agents/agent_detail.html | 50 ++++++++---- templates/partials/run_detail_body.html | 5 +- templates/partials/run_settings.html | 37 +++++++-- templates/partials/run_settings_js.html | 19 ++++- 16 files changed, 405 insertions(+), 43 deletions(-) create mode 100644 audits/migrations/0009_auditrun_system_prompt_version.py diff --git a/audits/comparison.py b/audits/comparison.py index 55c92bdc..d3a21c82 100644 --- a/audits/comparison.py +++ b/audits/comparison.py @@ -19,6 +19,16 @@ _SEV_RANK = {sev: i for i, sev in enumerate(_SEV_ORDER)} +def _system_prompt_label(run: AuditRun) -> str: + """The run's target system prompt, tagged with its predefined version if any.""" + snap = run.generation_parameters_snapshot or {} + text = snap.get("system_prompt") or "none" + source = snap.get("system_prompt_source") + if source: + return f"{text} · [{source['prompt']} v{source['version']}]" + return text + + def _fragility(severities: list[str]) -> dict | None: """Fragility metrics for one scenario across runs (ported from the visualizer). @@ -172,7 +182,7 @@ def compare_runs(project, run_ids: list[int]) -> dict: ("Judge", lambda r: frozen_judge(r)["label"]), ("Judge format", lambda r: frozen_judge(r)["output_label"]), ("Judge model", lambda r: frozen_name(r, "judge", with_id=True)), - ("System prompt", lambda r: (r.generation_parameters_snapshot or {}).get("system_prompt") or "none"), + ("System prompt", _system_prompt_label), ("Scenario set version", lambda r: r.scenario_set_version.label if r.scenario_set_version else "—"), ("SimpleAudit version", lambda r: f"{r.simpleaudit_version} ({r.git_commit[:8]}…)" if r.simpleaudit_version and r.git_commit else (r.simpleaudit_version or "—")), ("Temperature (target)", lambda r: (r.generation_parameters_snapshot or {}).get("temperature_target", "—")), diff --git a/audits/experiments.py b/audits/experiments.py index bdb6a18e..21fbce01 100644 --- a/audits/experiments.py +++ b/audits/experiments.py @@ -197,8 +197,21 @@ def parse_design(post, project) -> dict: languages = _split_values(post.get("language", ""), str) # The target's system prompt lives in the generation config (the engine - # reads it there); the field wins over a key typed into the JSON. + # reads it there). A picked versioned prompt fills the textarea client-side, + # but we re-resolve it server-side so a non-JS post is still correct; the + # resolved text then wins over a key typed into the JSON. We keep the + # resolved version object too so the run can freeze *which* prompt was used. system_prompt = (post.get("system_prompt") or "").strip() + prompt_version = None + prompt_version_id = (post.get("system_prompt_version") or "").strip() + if prompt_version_id: + from prompts.models import SystemPromptVersion + + pv = SystemPromptVersion.objects.filter( + pk=prompt_version_id, prompt__project=project).select_related("prompt").first() + if pv is not None: + prompt_version = pv + system_prompt = pv.content.strip() if system_prompt: gen_config = {**(gen_config or {}), "system_prompt": system_prompt} @@ -247,6 +260,7 @@ def parse_design(post, project) -> dict: "language": languages, "n_repetitions": n_reps if n_reps and n_reps > 1 else None, "gen_config": gen_config or None, + "system_prompt_version": prompt_version, "target_trace": target_trace, "target_trace_by_model": target_trace_by_model, } @@ -280,6 +294,7 @@ def expand(design: dict) -> list[dict]: spec = dict(zip(DESIGN_AXES, combo, strict=True)) spec["n_repetitions"] = design["n_repetitions"] spec["gen_config"] = design["gen_config"] + spec["system_prompt_version"] = design.get("system_prompt_version") spec["agent"] = design.get("agent") if design.get("agent") is not None: spec["target_trace"] = design.get("target_trace") @@ -306,6 +321,7 @@ def spec_to_run(spec: dict, name: str) -> dict: "language": spec["language"], "n_repetitions": spec["n_repetitions"], "gen_config": spec["gen_config"], + "system_prompt_version": spec.get("system_prompt_version"), "target_trace": spec.get("target_trace"), } @@ -402,7 +418,7 @@ def value(run, key): def spec_to_row(spec: dict) -> str: """Serialise the non-editable part of a spec for the review form.""" - return json.dumps({ + row = { "v": spec["scenario_set"].pk, "f": 1 if is_follow(spec["scenario_set"]) else 0, "t": spec["target"].pk, @@ -410,7 +426,11 @@ def spec_to_row(spec: dict) -> str: "jm": spec["judge_model"].pk, "j": spec["judge"].pk, "jf": 1 if is_follow(spec["judge"]) else 0, - }) + } + spv = spec.get("system_prompt_version") + if spv is not None: + row["spv"] = spv.pk + return json.dumps(row) def launch_experiment(*, project, user, name: str, runs: list[dict], repeat: dict | None = None) -> Experiment: @@ -454,6 +474,7 @@ def launch_experiment(*, project, user, name: str, runs: list[dict], repeat: dic language_override=r["language"], n_repetitions_override=r["n_repetitions"], gen_config_override=r["gen_config"], + system_prompt_version=r.get("system_prompt_version"), experiment=experiment, agent=r.get("agent"), trace_config={} if r.get("target_trace") is False else None, diff --git a/audits/migrations/0009_auditrun_system_prompt_version.py b/audits/migrations/0009_auditrun_system_prompt_version.py new file mode 100644 index 00000000..08956e36 --- /dev/null +++ b/audits/migrations/0009_auditrun_system_prompt_version.py @@ -0,0 +1,25 @@ +# Generated by Django 5.2.4 on 2026-10-13 19:17 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("audits", "0008_experiment_visibility"), + ("prompts", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="auditrun", + name="system_prompt_version", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="audit_runs", + to="prompts.systempromptversion", + ), + ), + ] diff --git a/audits/models.py b/audits/models.py index 8d1eeae0..af6d4172 100644 --- a/audits/models.py +++ b/audits/models.py @@ -43,6 +43,14 @@ class Status(models.TextChoices): # How the run is graded (criteria, output format, probe prompt) and the model that grades. judge_version = models.ForeignKey("judges.JudgeVersion", on_delete=models.RESTRICT, related_name="audit_runs") judge_model = models.ForeignKey("model_registry.RegisteredModel", on_delete=models.RESTRICT, related_name="judge_audit_runs") + # The versioned system prompt the target used, when a predefined one was + # picked. Null for free-text / bare-model runs; the actual text still lives + # in ``generation_parameters_snapshot["system_prompt"]``. Frozen like every + # other input so historical runs stay reproducible. + system_prompt_version = models.ForeignKey( + "prompts.SystemPromptVersion", on_delete=models.SET_NULL, null=True, blank=True, + related_name="audit_runs", + ) target_config_snapshot = models.JSONField() auditor_config_snapshot = models.JSONField() judge_config_snapshot = models.JSONField() diff --git a/audits/services.py b/audits/services.py index 9bfaa505..d0490ba0 100644 --- a/audits/services.py +++ b/audits/services.py @@ -139,6 +139,7 @@ def _generation_parameters( language_override: str | None = None, n_repetitions_override: int | None = None, gen_config_override: dict | None = None, + system_prompt_version=None, ) -> dict: params = {} # Raw JSON override provides the base (e.g. cloned generation config), @@ -155,6 +156,13 @@ def _generation_parameters( params["language"] = language_override if n_repetitions_override is not None: params["n_repetitions"] = n_repetitions_override + # Provenance of a predefined prompt, so the manifest records *which* version + # was used (the text itself lives under "system_prompt"). + if system_prompt_version is not None: + params["system_prompt_source"] = { + "prompt": system_prompt_version.prompt.name, + "version": system_prompt_version.version, + } return params @@ -168,6 +176,7 @@ def frozen_inputs( language_override: str | None = None, n_repetitions_override: int | None = None, gen_config_override: dict | None = None, + system_prompt_version=None, ) -> dict: """The snapshots a run freezes: the three endpoints, the judge and the generation settings. Shared by ``create_audit_run`` and script generation @@ -183,6 +192,7 @@ def frozen_inputs( language_override=language_override, n_repetitions_override=n_repetitions_override, gen_config_override=gen_config_override, + system_prompt_version=system_prompt_version, ), } @@ -202,6 +212,7 @@ def create_audit_run( language_override: str | None = None, n_repetitions_override: int | None = None, gen_config_override: dict | None = None, + system_prompt_version=None, trace_config: dict | None = None, monitor=None, experiment=None, @@ -310,10 +321,12 @@ def create_audit_run( auditor_model=auditor_model, judge_version=judge, judge_model=judge_model, + system_prompt_version=system_prompt_version, **frozen_inputs( target_model=target_model, auditor_model=auditor_model, judge_model=judge_model, judge=judge, max_turns_override=max_turns_override, language_override=language_override, n_repetitions_override=n_repetitions_override, gen_config_override=gen_config_override, + system_prompt_version=system_prompt_version, ), agent_config_snapshot=agent_snapshot, simpleaudit_version=resolved_version, diff --git a/infra/tests/factories.py b/infra/tests/factories.py index f48e5361..2b9c40d7 100644 --- a/infra/tests/factories.py +++ b/infra/tests/factories.py @@ -18,6 +18,7 @@ RegisteredModel, Tool, ) +from prompts.models import SystemPrompt, SystemPromptVersion from scenarios.models import ( Scenario, ScenarioRevision, @@ -120,6 +121,30 @@ class Meta: enabled = True +class SystemPromptFactory(DjangoModelFactory): + class Meta: + model = SystemPrompt + project = factory.SubFactory(ProjectFactory) + name = factory.Sequence(lambda n: f"Prompt {n}") + + +class SystemPromptVersionFactory(DjangoModelFactory): + class Meta: + model = SystemPromptVersion + prompt = factory.SubFactory(SystemPromptFactory) + version = factory.Sequence(lambda n: n) + content = factory.Sequence(lambda n: f"You are a helpful assistant (prompt v{n}).") + + +def system_prompt_for(project, *, name=None, content=None): + """A prompt with one version in ``project`` (one per call, so ids stay unique).""" + prompt = SystemPrompt.objects.create( + project=project, name=name or f"Prompt {SystemPrompt.objects.filter(project=project).count() + 1}" + ) + SystemPromptVersion.objects.create(prompt=prompt, version=1, content=content or "You are the support bot.") + return prompt + + class JudgeFactory(DjangoModelFactory): class Meta: model = Judge diff --git a/infra/tests/test_experiments.py b/infra/tests/test_experiments.py index 7ac9db42..5486b42f 100644 --- a/infra/tests/test_experiments.py +++ b/infra/tests/test_experiments.py @@ -22,6 +22,7 @@ ScenarioSetVersionFactory, UserFactory, judge_for, + system_prompt_for, ) _PROVENANCE = mock.Mock(version="0.2.1", commit="", source="metadata") @@ -481,6 +482,85 @@ def test_system_prompt_is_frozen_and_editable_per_run(self): self.assertEqual(prompts, ["You are Acme's bot.", None]) self.assertEqual(exp.factors, ["target", "system_prompt"]) + def test_predefined_prompt_picker_is_offered_when_prompts_exist(self): + # No prompts → context empty and no radio rows rendered. + page = self.client.get("/experiments/new/") + self.assertEqual(page.context["system_prompts"], []) + self.assertNotContains(page, 'type="radio" name="system_prompt_version"') + prompt = system_prompt_for(self.project, name="Support bot", content="You are the Acme support bot.") + version = prompt.latest + page = self.client.get("/experiments/new/") + # Rendered as the same searchable list as Models / Scenario sets: radio rows. + self.assertContains(page, f'type="radio" name="system_prompt_version" value="{version.id}"') + self.assertContains(page, "Support bot v1") + self.assertIn(prompt, page.context["system_prompts"]) + + def test_picking_a_predefined_prompt_bakes_content_and_source_into_the_run(self): + prompt = system_prompt_for(self.project, name="Support bot", content="You are the Acme support bot.") + version = prompt.latest + # One target → a singleton launches straight away (no review screen). + self.client.post( + "/experiments/new/", + self._design(target_model=[self.t1.id], max_turns="3", + system_prompt=version.content, system_prompt_version=str(version.id)), + ) + run = AuditRun.objects.get() + snap = run.generation_parameters_snapshot + self.assertEqual(snap["system_prompt"], "You are the Acme support bot.") + self.assertEqual(snap["system_prompt_source"], {"prompt": "Support bot", "version": 1}) + self.assertEqual(run.system_prompt_version_id, version.id) + + def test_free_text_prompt_has_no_source_in_manifest(self): + self.client.post( + "/experiments/new/", + self._design(target_model=[self.t1.id], max_turns="3", system_prompt="Typed by hand."), + ) + run = AuditRun.objects.get() + snap = run.generation_parameters_snapshot + self.assertEqual(snap["system_prompt"], "Typed by hand.") + self.assertNotIn("system_prompt_source", snap) + self.assertIsNone(run.system_prompt_version_id) + + def test_predefined_prompt_shows_on_review_and_survives_launch(self): + prompt = system_prompt_for(self.project, name="Support bot", content="You are the Acme support bot.") + version = prompt.latest + # Two targets → multi-run design → review screen shows the resolved prompt. + review = self.client.post( + "/experiments/new/", + self._design(max_turns="", + system_prompt=version.content, system_prompt_version=str(version.id)), + ) + self.assertEqual({r["system_prompt"] for r in review.context["rows"]}, {version.content}) + # The row spec carries the version id so it survives the review round-trip. + specs = [json.loads(r["spec"]) for r in review.context["rows"]] + self.assertTrue(all(s.get("spv") == version.id for s in specs)) + self.client.post("/experiments/new/", self._review_to_launch_payload(review)) + runs = list(Experiment.objects.get().runs.all()) + prompts = [r.generation_parameters_snapshot.get("system_prompt") for r in runs] + self.assertEqual(prompts, ["You are the Acme support bot.", "You are the Acme support bot."]) + sources = [r.generation_parameters_snapshot.get("system_prompt_source") for r in runs] + self.assertEqual(sources, [{"prompt": "Support bot", "version": 1}] * 2) + self.assertTrue(all(r.system_prompt_version_id == version.id for r in runs)) + + def test_predefined_prompt_wins_over_textarea_without_js(self): + # Server re-resolves the picked version even when JS never filled the box. + prompt = system_prompt_for(self.project, name="Support bot", content="Picked prompt text.") + version = prompt.latest + self.client.post( + "/experiments/new/", + self._design(target_model=[self.t1.id], max_turns="3", + system_prompt="", system_prompt_version=str(version.id)), + ) + self.assertEqual(AuditRun.objects.get().generation_parameters_snapshot["system_prompt"], "Picked prompt text.") + + def test_unknown_prompt_version_falls_back_to_textarea(self): + self.client.post( + "/experiments/new/", + self._design(target_model=[self.t1.id], max_turns="3", + system_prompt="Typed by hand.", system_prompt_version="999999"), + ) + self.assertEqual(AuditRun.objects.get().generation_parameters_snapshot["system_prompt"], "Typed by hand.") + def test_clone_prefills_judge_and_system_prompt(self): self.client.post("/experiments/new/", self._design(target_model=[self.t1.id], max_turns="3", system_prompt="Be terse.")) run = AuditRun.objects.get() diff --git a/infra/tests/test_prompts.py b/infra/tests/test_prompts.py index 52b24c33..9d46807e 100644 --- a/infra/tests/test_prompts.py +++ b/infra/tests/test_prompts.py @@ -5,9 +5,24 @@ """ from django.test import Client, TestCase, tag -from infra.tests.factories import MembershipFactory, ProjectFactory, UserFactory +from infra.tests.factories import ( + AgentFactory, + MembershipFactory, + ModelConnectionFactory, + ProjectFactory, + RegisteredModelFactory, + ScenarioSetFactory, + ScenarioSetVersionFactory, + UserFactory, +) from prompts.models import SystemPrompt -from prompts.services import create_prompt, delete_version, save_version, unique_name +from prompts.services import ( + create_prompt, + delete_prompt, + delete_version, + save_version, + unique_name, +) @tag("slow") @@ -70,6 +85,47 @@ def test_delete_version_removes_intermediate(self): numbers = sorted(p.versions.values_list("version", flat=True)) self.assertEqual(numbers, [1, 3]) # gap kept; numbers not renumbered + def _run_using(self, version): + """A minimal AuditRun that pins ``version`` (no engine submission).""" + from audits.models import AuditRun + from judges.models import Judge, JudgeVersion + + conn = ModelConnectionFactory(project=self.project) + target = RegisteredModelFactory(connection=conn, display_name="T") + judge_model = RegisteredModelFactory(connection=conn, display_name="J") + sset = ScenarioSetFactory(project=self.project, name="S") + sver = ScenarioSetVersionFactory(scenario_set=sset, version=1) + judge, _ = Judge.objects.get_or_create(project=self.project, name="Test judge") + jv = JudgeVersion.objects.create(judge=judge, version=1, base="safety") + return AuditRun.objects.create( + project=self.project, name="R", scenario_set_version=sver, target_model=target, + auditor_model=target, judge_version=jv, judge_model=judge_model, + simpleaudit_version="0.2.1", git_commit="", system_prompt_version=version, + target_config_snapshot={}, auditor_config_snapshot={}, judge_config_snapshot={}, + generation_parameters_snapshot={"system_prompt": version.content}, + ) + + def test_delete_prompt_blocked_when_run_pins_it(self): + p = create_prompt(project=self.project, name="Pinned", content="used text") + self._run_using(p.latest) + with self.assertRaises(ValueError, msg="pinned"): + delete_prompt(p) + self.assertTrue(SystemPrompt.objects.filter(pk=p.pk).exists()) + + def test_delete_prompt_blocked_when_agent_uses_its_text(self): + p = create_prompt(project=self.project, name="Agent used", content="agent text") + conn = ModelConnectionFactory(project=self.project) + model = RegisteredModelFactory(connection=conn, display_name="M") + AgentFactory(project=self.project, base_model=model, system_prompt="agent text") + with self.assertRaises(ValueError, msg="agent"): + delete_prompt(p) + self.assertTrue(SystemPrompt.objects.filter(pk=p.pk).exists()) + + def test_delete_prompt_allowed_when_unused(self): + p = create_prompt(project=self.project, name="Unused", content="free text") + delete_prompt(p) + self.assertFalse(SystemPrompt.objects.filter(pk=p.pk).exists()) + class PromptPageTests(_Base): def test_list_page(self): diff --git a/infra/ui.py b/infra/ui.py index dfeb6d3b..5c24d523 100644 --- a/infra/ui.py +++ b/infra/ui.py @@ -1340,10 +1340,18 @@ def get_context_data(self, **kw): for model in agent_models: model.agent = agent_by_external_id[model.model_id] model.has_otlp = model.connection_id in otlp_connection_ids + # Versioned system prompts for the "pick a predefined prompt" dropdown. + from prompts.models import SystemPrompt + kw.update( sets=sets, connections=connections, agent_models=agent_models, + system_prompts=list( + SystemPrompt.objects.filter(project=p) + .prefetch_related("versions") + .order_by("name") + ), model_roles=[ ("target", "Target", "The model under test.", sel["target"]), ("auditor", "Auditor", "Plays the user and probes the target.", sel["auditor"]), @@ -1439,6 +1447,7 @@ def _launch_single(self, request, spec, repeat): language_override=run_spec["language"], n_repetitions_override=run_spec["n_repetitions"], gen_config_override=run_spec["gen_config"], + system_prompt_version=run_spec.get("system_prompt_version"), agent=run_spec.get("agent"), trace_config={} if run_spec.get("target_trace") is False else None, experiment=experiment, @@ -1475,7 +1484,8 @@ def _row_context(i, spec_objs, *, name, max_turns, language, n_reps, system_prom duplicate=False): from audits import experiments as ex - spec = {"scenario_set": spec_objs["version"], **{r: spec_objs[r] for r in ("target", "auditor", "judge_model", "judge")}} + spec = {"scenario_set": spec_objs["version"], **{r: spec_objs[r] for r in ("target", "auditor", "judge_model", "judge")}, + "system_prompt_version": spec_objs.get("system_prompt_version")} return { "i": i, "spec": ex.spec_to_row(spec), @@ -1539,7 +1549,8 @@ def _review(self, request, design, specs, repeat=None): self._row_context( i, {"version": spec["scenario_set"], "target": spec["target"], "auditor": spec["auditor"], - "judge_model": spec["judge_model"], "judge": spec["judge"]}, + "judge_model": spec["judge_model"], "judge": spec["judge"], + "system_prompt_version": spec.get("system_prompt_version")}, name=ex.spec_label(spec, factors), max_turns=spec["max_turns"], language=spec["language"], @@ -1591,6 +1602,11 @@ def _rows_from_post(self, post, project): judges = JudgeVersion.objects.select_related("judge").filter( judge__project=project ).in_bulk({spec["j"] for spec in specs.values()}) + from prompts.models import SystemPromptVersion + + prompt_versions = SystemPromptVersion.objects.filter( + prompt__project=project + ).in_bulk({spec.get("spv") for spec in specs.values() if spec.get("spv")}) def lookup(found, pk, model): if pk not in found: @@ -1655,6 +1671,7 @@ def lookup(found, pk, model): gen_config = {**(gen_config or {}), "system_prompt": system_prompt} elif gen_config: gen_config.pop("system_prompt", None) # cleared on review + prompt_version = prompt_versions.get(spec.get("spv")) if spec.get("spv") else None runs.append({ "name": name, **objs, @@ -1662,6 +1679,7 @@ def lookup(found, pk, model): "language": lang or None, "n_repetitions": n_reps if n_reps and n_reps > 1 else None, "gen_config": gen_config or None, + "system_prompt_version": prompt_version, }) return rows, runs, errors @@ -3036,7 +3054,9 @@ def get_context_data(self, **kw): ctx["frozen_models"] = [frozen_model(run, role) for role in ROLES] ctx["judge"] = frozen_judge(run) - ctx["system_prompt"] = (run.generation_parameters_snapshot or {}).get("system_prompt", "") + snap = run.generation_parameters_snapshot or {} + ctx["system_prompt"] = snap.get("system_prompt", "") + ctx["system_prompt_source"] = snap.get("system_prompt_source") ctx["frozen_agent"] = frozen_agent(run) ctx["public_header"] = True return ctx diff --git a/model_registry/tests/test_agent_domain.py b/model_registry/tests/test_agent_domain.py index 8856fbca..525e3522 100644 --- a/model_registry/tests/test_agent_domain.py +++ b/model_registry/tests/test_agent_domain.py @@ -14,6 +14,7 @@ RegisteredModelFactory, ToolFactory, UserFactory, + system_prompt_for, ) from model_registry.models import ( Agent, @@ -384,6 +385,26 @@ def test_agent_detail_page(self): self.assertEqual(resp.status_code, 200) self.assertContains(resp, agent.name) + def test_agent_prompt_picker_renders_as_list_and_resolves_version(self): + model = RegisteredModelFactory(project=self.project) + prompt = system_prompt_for(self.project, name="Support bot", content="You are the Acme support bot.") + version = prompt.latest + # The picker is the same searchable list (radio rows), not a None: def delete_prompt(prompt: SystemPrompt) -> None: - """Delete a prompt and all its versions. Raises ProtectedError if agents reference it.""" + """Delete a prompt and all its versions. + + Blocked while anything references it, so history stays reproducible: + - an audit run froze one of its versions (``system_prompt_version`` FK), or + - an agent's instructions match one of its version texts. + + Raises ``ValueError`` with the reason when it can't be deleted. + """ + from audits.models import AuditRun from model_registry.models import Agent - if Agent.objects.filter(project=prompt.project).exists(): - # Check if any agent's current system_prompt matches this prompt's latest content - latest = prompt.latest - if latest and Agent.objects.filter(project=prompt.project, system_prompt=latest.content).exists(): - raise ValueError( - f"Agents still use \u201c{prompt.name}\u201d's latest prompt text. " - "Detach it from those agents first." - ) + # A run pins the exact version it used; deleting the prompt would break that. + if AuditRun.objects.filter(system_prompt_version__prompt=prompt).exists(): + raise ValueError( + f"\u201c{prompt.name}\u201d is pinned by one or more audit runs, so it is kept for reproducibility." + ) + # Agents store raw text; block when their instructions match any version here. + contents = list(prompt.versions.values_list("content", flat=True)) + if contents and Agent.objects.filter(project=prompt.project, system_prompt__in=contents).exists(): + raise ValueError( + f"An agent still uses \u201c{prompt.name}\u201d's text. Detach it from that agent first." + ) + prompt.delete() diff --git a/prompts/views.py b/prompts/views.py index 6a87f3a8..8bfcdd00 100644 --- a/prompts/views.py +++ b/prompts/views.py @@ -9,7 +9,13 @@ from infra.ui import ProjectMixin, _require_write_access from prompts.models import SystemPrompt, SystemPromptVersion -from prompts.services import create_prompt, delete_version, save_version, unique_name +from prompts.services import ( + create_prompt, + delete_prompt, + delete_version, + save_version, + unique_name, +) def _diff_entry(version: SystemPromptVersion) -> dict: @@ -67,7 +73,7 @@ def post(self, request): prompt = SystemPrompt.objects.filter(pk=request.POST.get("prompt_id") or 0, project=p).first() if prompt is not None: try: - prompt.delete() + delete_prompt(prompt) messages.success(request, f"Deleted system prompt \u201c{prompt.name}\u201d.") except ValueError as e: messages.error(request, str(e)) diff --git a/templates/agents/agent_detail.html b/templates/agents/agent_detail.html index 6156adb3..62c5e2b0 100644 --- a/templates/agents/agent_detail.html +++ b/templates/agents/agent_detail.html @@ -98,21 +98,27 @@

{% if agent %}Edit Agent{% else
Instructions {% if system_prompts %} -
- - +
+ {% for sp in system_prompts %}{% if sp.versions.all %} +

{{ sp.name }}

+ {% for v in sp.versions.all %} + + {% endfor %} {% endif %}{% endfor %} - -

Pick a versioned prompt, or type your own below to override it. Manage prompts →

-
+ +
+

Pick a versioned prompt to fill the box below, or type your own to override it. Manage prompts →

+
{% endif %}
@@ -435,6 +441,22 @@

Expose { r.checked = false; }); + }); + })(); + // --- Sync button: POST to /agents/sync/ then reload --- document.querySelectorAll('[data-sync-btn]').forEach(btn => { btn.addEventListener('click', async () => { diff --git a/templates/partials/run_detail_body.html b/templates/partials/run_detail_body.html index d3cb5a30..8391ceb2 100644 --- a/templates/partials/run_detail_body.html +++ b/templates/partials/run_detail_body.html @@ -866,7 +866,10 @@

Frozen Manifest

{% endif %}
System Prompt - {% if system_prompt %}{{ system_prompt|truncatechars:60 }}{% else %}none{% endif %} + + {% if system_prompt_source %}{{ system_prompt_source.prompt }} v{{ system_prompt_source.version }}{% endif %} + {% if system_prompt %}{{ system_prompt|truncatechars:60 }}{% else %}none{% endif %} +
{% if system_prompt|length > 60 %}
Full system prompt diff --git a/templates/partials/run_settings.html b/templates/partials/run_settings.html index 966523c7..a706ee5a 100644 --- a/templates/partials/run_settings.html +++ b/templates/partials/run_settings.html @@ -27,12 +27,37 @@

Run settings

-
- - -

Sent to the target model as its system prompt, like your product would. Leave empty to test the bare model. Edit per run on review to compare prompts.

+
+ {% if system_prompts %} +
+ Predefined prompt (pick one to fill the box) + +
+ {% for sp in system_prompts %}{% if sp.versions.all %} +

{{ sp.name }}

+ {% for v in sp.versions.all %} + + {% endfor %} + {% endif %}{% endfor %} + +
+

Manage prompts →

+
+ {% endif %} +
+ + +

Sent to the target model as its system prompt, like your product would. Leave empty to test the bare model. Edit per run on review to compare prompts.

+
diff --git a/templates/partials/run_settings_js.html b/templates/partials/run_settings_js.html index fab8e78b..a8bbe47b 100644 --- a/templates/partials/run_settings_js.html +++ b/templates/partials/run_settings_js.html @@ -1,8 +1,23 @@ {% comment %} -Behaviour for run_settings.html: generation-config normalisation and -submit-time JSON validation. +Behaviour for run_settings.html: predefined-prompt picker sync, generation-config +normalisation and submit-time JSON validation. {% endcomment %}