diff --git a/audits/comparison.py b/audits/comparison.py index 55c92bd..d3a21c8 100644 --- a/audits/comparison.py +++ b/audits/comparison.py @@ -19,6 +19,16 @@ _SEV_RANK = {sev: i for i, sev in enumerate(_SEV_ORDER)} +def _system_prompt_label(run: AuditRun) -> str: + """The run's target system prompt, tagged with its predefined version if any.""" + snap = run.generation_parameters_snapshot or {} + text = snap.get("system_prompt") or "none" + source = snap.get("system_prompt_source") + if source: + return f"{text} · [{source['prompt']} v{source['version']}]" + return text + + def _fragility(severities: list[str]) -> dict | None: """Fragility metrics for one scenario across runs (ported from the visualizer). @@ -172,7 +182,7 @@ def compare_runs(project, run_ids: list[int]) -> dict: ("Judge", lambda r: frozen_judge(r)["label"]), ("Judge format", lambda r: frozen_judge(r)["output_label"]), ("Judge model", lambda r: frozen_name(r, "judge", with_id=True)), - ("System prompt", lambda r: (r.generation_parameters_snapshot or {}).get("system_prompt") or "none"), + ("System prompt", _system_prompt_label), ("Scenario set version", lambda r: r.scenario_set_version.label if r.scenario_set_version else "—"), ("SimpleAudit version", lambda r: f"{r.simpleaudit_version} ({r.git_commit[:8]}…)" if r.simpleaudit_version and r.git_commit else (r.simpleaudit_version or "—")), ("Temperature (target)", lambda r: (r.generation_parameters_snapshot or {}).get("temperature_target", "—")), diff --git a/audits/experiments.py b/audits/experiments.py index bdb6a18..21fbce0 100644 --- a/audits/experiments.py +++ b/audits/experiments.py @@ -197,8 +197,21 @@ def parse_design(post, project) -> dict: languages = _split_values(post.get("language", ""), str) # The target's system prompt lives in the generation config (the engine - # reads it there); the field wins over a key typed into the JSON. + # reads it there). A picked versioned prompt fills the textarea client-side, + # but we re-resolve it server-side so a non-JS post is still correct; the + # resolved text then wins over a key typed into the JSON. We keep the + # resolved version object too so the run can freeze *which* prompt was used. system_prompt = (post.get("system_prompt") or "").strip() + prompt_version = None + prompt_version_id = (post.get("system_prompt_version") or "").strip() + if prompt_version_id: + from prompts.models import SystemPromptVersion + + pv = SystemPromptVersion.objects.filter( + pk=prompt_version_id, prompt__project=project).select_related("prompt").first() + if pv is not None: + prompt_version = pv + system_prompt = pv.content.strip() if system_prompt: gen_config = {**(gen_config or {}), "system_prompt": system_prompt} @@ -247,6 +260,7 @@ def parse_design(post, project) -> dict: "language": languages, "n_repetitions": n_reps if n_reps and n_reps > 1 else None, "gen_config": gen_config or None, + "system_prompt_version": prompt_version, "target_trace": target_trace, "target_trace_by_model": target_trace_by_model, } @@ -280,6 +294,7 @@ def expand(design: dict) -> list[dict]: spec = dict(zip(DESIGN_AXES, combo, strict=True)) spec["n_repetitions"] = design["n_repetitions"] spec["gen_config"] = design["gen_config"] + spec["system_prompt_version"] = design.get("system_prompt_version") spec["agent"] = design.get("agent") if design.get("agent") is not None: spec["target_trace"] = design.get("target_trace") @@ -306,6 +321,7 @@ def spec_to_run(spec: dict, name: str) -> dict: "language": spec["language"], "n_repetitions": spec["n_repetitions"], "gen_config": spec["gen_config"], + "system_prompt_version": spec.get("system_prompt_version"), "target_trace": spec.get("target_trace"), } @@ -402,7 +418,7 @@ def value(run, key): def spec_to_row(spec: dict) -> str: """Serialise the non-editable part of a spec for the review form.""" - return json.dumps({ + row = { "v": spec["scenario_set"].pk, "f": 1 if is_follow(spec["scenario_set"]) else 0, "t": spec["target"].pk, @@ -410,7 +426,11 @@ def spec_to_row(spec: dict) -> str: "jm": spec["judge_model"].pk, "j": spec["judge"].pk, "jf": 1 if is_follow(spec["judge"]) else 0, - }) + } + spv = spec.get("system_prompt_version") + if spv is not None: + row["spv"] = spv.pk + return json.dumps(row) def launch_experiment(*, project, user, name: str, runs: list[dict], repeat: dict | None = None) -> Experiment: @@ -454,6 +474,7 @@ def launch_experiment(*, project, user, name: str, runs: list[dict], repeat: dic language_override=r["language"], n_repetitions_override=r["n_repetitions"], gen_config_override=r["gen_config"], + system_prompt_version=r.get("system_prompt_version"), experiment=experiment, agent=r.get("agent"), trace_config={} if r.get("target_trace") is False else None, diff --git a/audits/migrations/0009_auditrun_system_prompt_version.py b/audits/migrations/0009_auditrun_system_prompt_version.py new file mode 100644 index 0000000..08956e3 --- /dev/null +++ b/audits/migrations/0009_auditrun_system_prompt_version.py @@ -0,0 +1,25 @@ +# Generated by Django 5.2.4 on 2026-10-13 19:17 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("audits", "0008_experiment_visibility"), + ("prompts", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="auditrun", + name="system_prompt_version", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="audit_runs", + to="prompts.systempromptversion", + ), + ), + ] diff --git a/audits/models.py b/audits/models.py index 8d1eeae..af6d417 100644 --- a/audits/models.py +++ b/audits/models.py @@ -43,6 +43,14 @@ class Status(models.TextChoices): # How the run is graded (criteria, output format, probe prompt) and the model that grades. judge_version = models.ForeignKey("judges.JudgeVersion", on_delete=models.RESTRICT, related_name="audit_runs") judge_model = models.ForeignKey("model_registry.RegisteredModel", on_delete=models.RESTRICT, related_name="judge_audit_runs") + # The versioned system prompt the target used, when a predefined one was + # picked. Null for free-text / bare-model runs; the actual text still lives + # in ``generation_parameters_snapshot["system_prompt"]``. Frozen like every + # other input so historical runs stay reproducible. + system_prompt_version = models.ForeignKey( + "prompts.SystemPromptVersion", on_delete=models.SET_NULL, null=True, blank=True, + related_name="audit_runs", + ) target_config_snapshot = models.JSONField() auditor_config_snapshot = models.JSONField() judge_config_snapshot = models.JSONField() diff --git a/audits/services.py b/audits/services.py index 9bfaa50..d0490ba 100644 --- a/audits/services.py +++ b/audits/services.py @@ -139,6 +139,7 @@ def _generation_parameters( language_override: str | None = None, n_repetitions_override: int | None = None, gen_config_override: dict | None = None, + system_prompt_version=None, ) -> dict: params = {} # Raw JSON override provides the base (e.g. cloned generation config), @@ -155,6 +156,13 @@ def _generation_parameters( params["language"] = language_override if n_repetitions_override is not None: params["n_repetitions"] = n_repetitions_override + # Provenance of a predefined prompt, so the manifest records *which* version + # was used (the text itself lives under "system_prompt"). + if system_prompt_version is not None: + params["system_prompt_source"] = { + "prompt": system_prompt_version.prompt.name, + "version": system_prompt_version.version, + } return params @@ -168,6 +176,7 @@ def frozen_inputs( language_override: str | None = None, n_repetitions_override: int | None = None, gen_config_override: dict | None = None, + system_prompt_version=None, ) -> dict: """The snapshots a run freezes: the three endpoints, the judge and the generation settings. Shared by ``create_audit_run`` and script generation @@ -183,6 +192,7 @@ def frozen_inputs( language_override=language_override, n_repetitions_override=n_repetitions_override, gen_config_override=gen_config_override, + system_prompt_version=system_prompt_version, ), } @@ -202,6 +212,7 @@ def create_audit_run( language_override: str | None = None, n_repetitions_override: int | None = None, gen_config_override: dict | None = None, + system_prompt_version=None, trace_config: dict | None = None, monitor=None, experiment=None, @@ -310,10 +321,12 @@ def create_audit_run( auditor_model=auditor_model, judge_version=judge, judge_model=judge_model, + system_prompt_version=system_prompt_version, **frozen_inputs( target_model=target_model, auditor_model=auditor_model, judge_model=judge_model, judge=judge, max_turns_override=max_turns_override, language_override=language_override, n_repetitions_override=n_repetitions_override, gen_config_override=gen_config_override, + system_prompt_version=system_prompt_version, ), agent_config_snapshot=agent_snapshot, simpleaudit_version=resolved_version, diff --git a/infra/tests/factories.py b/infra/tests/factories.py index f48e536..2b9c40d 100644 --- a/infra/tests/factories.py +++ b/infra/tests/factories.py @@ -18,6 +18,7 @@ RegisteredModel, Tool, ) +from prompts.models import SystemPrompt, SystemPromptVersion from scenarios.models import ( Scenario, ScenarioRevision, @@ -120,6 +121,30 @@ class Meta: enabled = True +class SystemPromptFactory(DjangoModelFactory): + class Meta: + model = SystemPrompt + project = factory.SubFactory(ProjectFactory) + name = factory.Sequence(lambda n: f"Prompt {n}") + + +class SystemPromptVersionFactory(DjangoModelFactory): + class Meta: + model = SystemPromptVersion + prompt = factory.SubFactory(SystemPromptFactory) + version = factory.Sequence(lambda n: n) + content = factory.Sequence(lambda n: f"You are a helpful assistant (prompt v{n}).") + + +def system_prompt_for(project, *, name=None, content=None): + """A prompt with one version in ``project`` (one per call, so ids stay unique).""" + prompt = SystemPrompt.objects.create( + project=project, name=name or f"Prompt {SystemPrompt.objects.filter(project=project).count() + 1}" + ) + SystemPromptVersion.objects.create(prompt=prompt, version=1, content=content or "You are the support bot.") + return prompt + + class JudgeFactory(DjangoModelFactory): class Meta: model = Judge diff --git a/infra/tests/test_experiments.py b/infra/tests/test_experiments.py index 7ac9db4..c4d6442 100644 --- a/infra/tests/test_experiments.py +++ b/infra/tests/test_experiments.py @@ -22,6 +22,7 @@ ScenarioSetVersionFactory, UserFactory, judge_for, + system_prompt_for, ) _PROVENANCE = mock.Mock(version="0.2.1", commit="", source="metadata") @@ -481,6 +482,88 @@ def test_system_prompt_is_frozen_and_editable_per_run(self): self.assertEqual(prompts, ["You are Acme's bot.", None]) self.assertEqual(exp.factors, ["target", "system_prompt"]) + def test_predefined_prompt_picker_is_offered_when_prompts_exist(self): + # No prompts → context empty and no radio rows rendered. + page = self.client.get("/experiments/new/") + self.assertEqual(page.context["system_prompts"], []) + self.assertNotContains(page, 'type="radio" name="system_prompt_version"') + prompt = system_prompt_for(self.project, name="Support bot", content="You are the Acme support bot.") + version = prompt.latest + page = self.client.get("/experiments/new/") + # Rendered like the Scenario-sets block: a row per prompt with an + # expandable Versions list of radios (single selection, no row checkbox). + self.assertContains(page, f'type="radio" name="system_prompt_version" value="{version.id}"') + self.assertNotContains(page, 'data-prompt-toggle') + self.assertContains(page, "Support bot") + self.assertContains(page, "Versions ▾") + self.assertIn(prompt, page.context["system_prompts"]) + + def test_picking_a_predefined_prompt_bakes_content_and_source_into_the_run(self): + prompt = system_prompt_for(self.project, name="Support bot", content="You are the Acme support bot.") + version = prompt.latest + # One target → a singleton launches straight away (no review screen). + self.client.post( + "/experiments/new/", + self._design(target_model=[self.t1.id], max_turns="3", + system_prompt=version.content, system_prompt_version=str(version.id)), + ) + run = AuditRun.objects.get() + snap = run.generation_parameters_snapshot + self.assertEqual(snap["system_prompt"], "You are the Acme support bot.") + self.assertEqual(snap["system_prompt_source"], {"prompt": "Support bot", "version": 1}) + self.assertEqual(run.system_prompt_version_id, version.id) + + def test_free_text_prompt_has_no_source_in_manifest(self): + self.client.post( + "/experiments/new/", + self._design(target_model=[self.t1.id], max_turns="3", system_prompt="Typed by hand."), + ) + run = AuditRun.objects.get() + snap = run.generation_parameters_snapshot + self.assertEqual(snap["system_prompt"], "Typed by hand.") + self.assertNotIn("system_prompt_source", snap) + self.assertIsNone(run.system_prompt_version_id) + + def test_predefined_prompt_shows_on_review_and_survives_launch(self): + prompt = system_prompt_for(self.project, name="Support bot", content="You are the Acme support bot.") + version = prompt.latest + # Two targets → multi-run design → review screen shows the resolved prompt. + review = self.client.post( + "/experiments/new/", + self._design(max_turns="", + system_prompt=version.content, system_prompt_version=str(version.id)), + ) + self.assertEqual({r["system_prompt"] for r in review.context["rows"]}, {version.content}) + # The row spec carries the version id so it survives the review round-trip. + specs = [json.loads(r["spec"]) for r in review.context["rows"]] + self.assertTrue(all(s.get("spv") == version.id for s in specs)) + self.client.post("/experiments/new/", self._review_to_launch_payload(review)) + runs = list(Experiment.objects.get().runs.all()) + prompts = [r.generation_parameters_snapshot.get("system_prompt") for r in runs] + self.assertEqual(prompts, ["You are the Acme support bot.", "You are the Acme support bot."]) + sources = [r.generation_parameters_snapshot.get("system_prompt_source") for r in runs] + self.assertEqual(sources, [{"prompt": "Support bot", "version": 1}] * 2) + self.assertTrue(all(r.system_prompt_version_id == version.id for r in runs)) + + def test_predefined_prompt_wins_over_textarea_without_js(self): + # Server re-resolves the picked version even when JS never filled the box. + prompt = system_prompt_for(self.project, name="Support bot", content="Picked prompt text.") + version = prompt.latest + self.client.post( + "/experiments/new/", + self._design(target_model=[self.t1.id], max_turns="3", + system_prompt="", system_prompt_version=str(version.id)), + ) + self.assertEqual(AuditRun.objects.get().generation_parameters_snapshot["system_prompt"], "Picked prompt text.") + + def test_unknown_prompt_version_falls_back_to_textarea(self): + self.client.post( + "/experiments/new/", + self._design(target_model=[self.t1.id], max_turns="3", + system_prompt="Typed by hand.", system_prompt_version="999999"), + ) + self.assertEqual(AuditRun.objects.get().generation_parameters_snapshot["system_prompt"], "Typed by hand.") + def test_clone_prefills_judge_and_system_prompt(self): self.client.post("/experiments/new/", self._design(target_model=[self.t1.id], max_turns="3", system_prompt="Be terse.")) run = AuditRun.objects.get() diff --git a/infra/tests/test_prompts.py b/infra/tests/test_prompts.py index 52b24c3..9d46807 100644 --- a/infra/tests/test_prompts.py +++ b/infra/tests/test_prompts.py @@ -5,9 +5,24 @@ """ from django.test import Client, TestCase, tag -from infra.tests.factories import MembershipFactory, ProjectFactory, UserFactory +from infra.tests.factories import ( + AgentFactory, + MembershipFactory, + ModelConnectionFactory, + ProjectFactory, + RegisteredModelFactory, + ScenarioSetFactory, + ScenarioSetVersionFactory, + UserFactory, +) from prompts.models import SystemPrompt -from prompts.services import create_prompt, delete_version, save_version, unique_name +from prompts.services import ( + create_prompt, + delete_prompt, + delete_version, + save_version, + unique_name, +) @tag("slow") @@ -70,6 +85,47 @@ def test_delete_version_removes_intermediate(self): numbers = sorted(p.versions.values_list("version", flat=True)) self.assertEqual(numbers, [1, 3]) # gap kept; numbers not renumbered + def _run_using(self, version): + """A minimal AuditRun that pins ``version`` (no engine submission).""" + from audits.models import AuditRun + from judges.models import Judge, JudgeVersion + + conn = ModelConnectionFactory(project=self.project) + target = RegisteredModelFactory(connection=conn, display_name="T") + judge_model = RegisteredModelFactory(connection=conn, display_name="J") + sset = ScenarioSetFactory(project=self.project, name="S") + sver = ScenarioSetVersionFactory(scenario_set=sset, version=1) + judge, _ = Judge.objects.get_or_create(project=self.project, name="Test judge") + jv = JudgeVersion.objects.create(judge=judge, version=1, base="safety") + return AuditRun.objects.create( + project=self.project, name="R", scenario_set_version=sver, target_model=target, + auditor_model=target, judge_version=jv, judge_model=judge_model, + simpleaudit_version="0.2.1", git_commit="", system_prompt_version=version, + target_config_snapshot={}, auditor_config_snapshot={}, judge_config_snapshot={}, + generation_parameters_snapshot={"system_prompt": version.content}, + ) + + def test_delete_prompt_blocked_when_run_pins_it(self): + p = create_prompt(project=self.project, name="Pinned", content="used text") + self._run_using(p.latest) + with self.assertRaises(ValueError, msg="pinned"): + delete_prompt(p) + self.assertTrue(SystemPrompt.objects.filter(pk=p.pk).exists()) + + def test_delete_prompt_blocked_when_agent_uses_its_text(self): + p = create_prompt(project=self.project, name="Agent used", content="agent text") + conn = ModelConnectionFactory(project=self.project) + model = RegisteredModelFactory(connection=conn, display_name="M") + AgentFactory(project=self.project, base_model=model, system_prompt="agent text") + with self.assertRaises(ValueError, msg="agent"): + delete_prompt(p) + self.assertTrue(SystemPrompt.objects.filter(pk=p.pk).exists()) + + def test_delete_prompt_allowed_when_unused(self): + p = create_prompt(project=self.project, name="Unused", content="free text") + delete_prompt(p) + self.assertFalse(SystemPrompt.objects.filter(pk=p.pk).exists()) + class PromptPageTests(_Base): def test_list_page(self): diff --git a/infra/ui.py b/infra/ui.py index dfeb6d3..89e2248 100644 --- a/infra/ui.py +++ b/infra/ui.py @@ -1166,6 +1166,7 @@ def _clone_from_run(source: AuditRun) -> dict: "auditor_model_id": source.auditor_model_id, "judge_model_id": source.judge_model_id, "judge_version_id": source.judge_version_id, + "system_prompt_version_id": source.system_prompt_version_id, "agent_id": source.agent_id, "max_turns": params.get("max_turns", ""), "language": params.get("language", ""), @@ -1192,6 +1193,7 @@ def _design_selection(post=None, clone=None) -> dict: "judge": post.getlist("judge"), "agent_id": agent_id, "agent_trace": "1" in post.getlist(f"agent_trace_{agent_id}") if agent_id else True, + "system_prompt_version": (post.get("system_prompt_version") or "").strip(), } if clone: # Clone pins the cloned run's exact version. @@ -1205,8 +1207,9 @@ def _design_selection(post=None, clone=None) -> dict: "judge": [str(clone["judge_version_id"])], "agent_id": agent_id, "agent_trace": True, + "system_prompt_version": str(clone.get("system_prompt_version_id") or ""), } - return {"sets": [], "versions": [], "target": [], "auditor": [], "judge_model": [], "judge": [], "agent_id": "", "agent_trace": True} + return {"sets": [], "versions": [], "target": [], "auditor": [], "judge_model": [], "judge": [], "agent_id": "", "agent_trace": True, "system_prompt_version": ""} def ex_repeat(repeat: dict) -> str: @@ -1340,10 +1343,31 @@ def get_context_data(self, **kw): for model in agent_models: model.agent = agent_by_external_id[model.model_id] model.has_otlp = model.connection_id in otlp_connection_ids + # Versioned system prompts for the picker: each row lists its versions + # (newest first) and which are ticked — same shape as the scenario-set + # rows so the template + JS can mirror that picker exactly. + from prompts.models import SystemPrompt, SystemPromptVersion + + system_prompts = list( + SystemPrompt.objects.filter(project=p) + .prefetch_related( + Prefetch("versions", queryset=SystemPromptVersion.objects.order_by("-version")) + ) + .order_by("name") + ) + spv_selected = sel["system_prompt_version"] + for sp in system_prompts: + sp.version_list = list(sp.versions.all()) + latest_id = str(sp.version_list[0].id) if sp.version_list else None + sp.ticked = {spv_selected} if spv_selected in {str(v.id) for v in sp.version_list} else set() + # Open the version list when a non-latest version is selected. + sp.show_versions = bool(sp.ticked - {latest_id}) + kw.update( sets=sets, connections=connections, agent_models=agent_models, + system_prompts=system_prompts, model_roles=[ ("target", "Target", "The model under test.", sel["target"]), ("auditor", "Auditor", "Plays the user and probes the target.", sel["auditor"]), @@ -1439,6 +1463,7 @@ def _launch_single(self, request, spec, repeat): language_override=run_spec["language"], n_repetitions_override=run_spec["n_repetitions"], gen_config_override=run_spec["gen_config"], + system_prompt_version=run_spec.get("system_prompt_version"), agent=run_spec.get("agent"), trace_config={} if run_spec.get("target_trace") is False else None, experiment=experiment, @@ -1475,7 +1500,8 @@ def _row_context(i, spec_objs, *, name, max_turns, language, n_reps, system_prom duplicate=False): from audits import experiments as ex - spec = {"scenario_set": spec_objs["version"], **{r: spec_objs[r] for r in ("target", "auditor", "judge_model", "judge")}} + spec = {"scenario_set": spec_objs["version"], **{r: spec_objs[r] for r in ("target", "auditor", "judge_model", "judge")}, + "system_prompt_version": spec_objs.get("system_prompt_version")} return { "i": i, "spec": ex.spec_to_row(spec), @@ -1539,7 +1565,8 @@ def _review(self, request, design, specs, repeat=None): self._row_context( i, {"version": spec["scenario_set"], "target": spec["target"], "auditor": spec["auditor"], - "judge_model": spec["judge_model"], "judge": spec["judge"]}, + "judge_model": spec["judge_model"], "judge": spec["judge"], + "system_prompt_version": spec.get("system_prompt_version")}, name=ex.spec_label(spec, factors), max_turns=spec["max_turns"], language=spec["language"], @@ -1591,6 +1618,11 @@ def _rows_from_post(self, post, project): judges = JudgeVersion.objects.select_related("judge").filter( judge__project=project ).in_bulk({spec["j"] for spec in specs.values()}) + from prompts.models import SystemPromptVersion + + prompt_versions = SystemPromptVersion.objects.filter( + prompt__project=project + ).in_bulk({spec.get("spv") for spec in specs.values() if spec.get("spv")}) def lookup(found, pk, model): if pk not in found: @@ -1655,6 +1687,7 @@ def lookup(found, pk, model): gen_config = {**(gen_config or {}), "system_prompt": system_prompt} elif gen_config: gen_config.pop("system_prompt", None) # cleared on review + prompt_version = prompt_versions.get(spec.get("spv")) if spec.get("spv") else None runs.append({ "name": name, **objs, @@ -1662,6 +1695,7 @@ def lookup(found, pk, model): "language": lang or None, "n_repetitions": n_reps if n_reps and n_reps > 1 else None, "gen_config": gen_config or None, + "system_prompt_version": prompt_version, }) return rows, runs, errors @@ -3036,7 +3070,9 @@ def get_context_data(self, **kw): ctx["frozen_models"] = [frozen_model(run, role) for role in ROLES] ctx["judge"] = frozen_judge(run) - ctx["system_prompt"] = (run.generation_parameters_snapshot or {}).get("system_prompt", "") + snap = run.generation_parameters_snapshot or {} + ctx["system_prompt"] = snap.get("system_prompt", "") + ctx["system_prompt_source"] = snap.get("system_prompt_source") ctx["frozen_agent"] = frozen_agent(run) ctx["public_header"] = True return ctx @@ -3838,13 +3874,30 @@ def get_context_data(self, **kw): # Gate the "+ New" deep links: without the chat proxy there is no # embedded workspace to create in. ctx["chat_enabled"] = chat_enabled() - # Versioned system prompts for the agent's instructions picker. - from prompts.models import SystemPrompt - ctx["system_prompts"] = list( + # Versioned system prompts for the agent's instructions picker — same + # row shape as the experiment picker (versions newest first, ticked). + # Pre-tick the version whose content matches the agent's current text so + # a re-opened agent shows which predefined prompt it uses. + from django.db.models import Prefetch + + from prompts.models import SystemPrompt, SystemPromptVersion + + agent_prompt_text = (agent.system_prompt or "").strip() if agent else "" + system_prompts = list( SystemPrompt.objects.filter(project=project) - .prefetch_related("versions") + .prefetch_related( + Prefetch("versions", queryset=SystemPromptVersion.objects.order_by("-version")) + ) .order_by("name") ) + for sp in system_prompts: + sp.version_list = list(sp.versions.all()) + latest_id = str(sp.version_list[0].id) if sp.version_list else None + match = next((v for v in sp.version_list if (v.content or "").strip() == agent_prompt_text), None) + matched_id = str(match.id) if match else None + sp.ticked = {matched_id} if matched_id else set() + sp.show_versions = bool(sp.ticked - {latest_id}) + ctx["system_prompts"] = system_prompts # Live Open WebUI model entry (None when unsynced or OWUI is down). if agent is not None and chat_enabled(): from model_registry.services import agent_live_openwebui diff --git a/model_registry/tests/test_agent_domain.py b/model_registry/tests/test_agent_domain.py index 8856fbc..525e352 100644 --- a/model_registry/tests/test_agent_domain.py +++ b/model_registry/tests/test_agent_domain.py @@ -14,6 +14,7 @@ RegisteredModelFactory, ToolFactory, UserFactory, + system_prompt_for, ) from model_registry.models import ( Agent, @@ -384,6 +385,26 @@ def test_agent_detail_page(self): self.assertEqual(resp.status_code, 200) self.assertContains(resp, agent.name) + def test_agent_prompt_picker_renders_as_list_and_resolves_version(self): + model = RegisteredModelFactory(project=self.project) + prompt = system_prompt_for(self.project, name="Support bot", content="You are the Acme support bot.") + version = prompt.latest + # The picker is the same searchable list (radio rows), not a None: def delete_prompt(prompt: SystemPrompt) -> None: - """Delete a prompt and all its versions. Raises ProtectedError if agents reference it.""" + """Delete a prompt and all its versions. + + Blocked while anything references it, so history stays reproducible: + - an audit run froze one of its versions (``system_prompt_version`` FK), or + - an agent's instructions match one of its version texts. + + Raises ``ValueError`` with the reason when it can't be deleted. + """ + from audits.models import AuditRun from model_registry.models import Agent - if Agent.objects.filter(project=prompt.project).exists(): - # Check if any agent's current system_prompt matches this prompt's latest content - latest = prompt.latest - if latest and Agent.objects.filter(project=prompt.project, system_prompt=latest.content).exists(): - raise ValueError( - f"Agents still use \u201c{prompt.name}\u201d's latest prompt text. " - "Detach it from those agents first." - ) + # A run pins the exact version it used; deleting the prompt would break that. + if AuditRun.objects.filter(system_prompt_version__prompt=prompt).exists(): + raise ValueError( + f"\u201c{prompt.name}\u201d is pinned by one or more audit runs, so it is kept for reproducibility." + ) + # Agents store raw text; block when their instructions match any version here. + contents = list(prompt.versions.values_list("content", flat=True)) + if contents and Agent.objects.filter(project=prompt.project, system_prompt__in=contents).exists(): + raise ValueError( + f"An agent still uses \u201c{prompt.name}\u201d's text. Detach it from that agent first." + ) + prompt.delete() diff --git a/prompts/views.py b/prompts/views.py index 6a87f3a..8bfcdd0 100644 --- a/prompts/views.py +++ b/prompts/views.py @@ -9,7 +9,13 @@ from infra.ui import ProjectMixin, _require_write_access from prompts.models import SystemPrompt, SystemPromptVersion -from prompts.services import create_prompt, delete_version, save_version, unique_name +from prompts.services import ( + create_prompt, + delete_prompt, + delete_version, + save_version, + unique_name, +) def _diff_entry(version: SystemPromptVersion) -> dict: @@ -67,7 +73,7 @@ def post(self, request): prompt = SystemPrompt.objects.filter(pk=request.POST.get("prompt_id") or 0, project=p).first() if prompt is not None: try: - prompt.delete() + delete_prompt(prompt) messages.success(request, f"Deleted system prompt \u201c{prompt.name}\u201d.") except ValueError as e: messages.error(request, str(e)) diff --git a/pyproject.toml b/pyproject.toml index ce4d5f2..b10d930 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "simpleaudit-studio" -version = "0.5.15" +version = "0.5.16" description = "Production AI model audit platform — one-liner demo mode via uvx" readme = "README.md" requires-python = ">=3.11" diff --git a/templates/agents/agent_detail.html b/templates/agents/agent_detail.html index 6156adb..0e3ecdb 100644 --- a/templates/agents/agent_detail.html +++ b/templates/agents/agent_detail.html @@ -98,21 +98,47 @@

{% if agent %}Edit Agent{% else
Instructions {% if system_prompts %} -
- - + ＋ New +
+
+ {% for sp in system_prompts %}{% if sp.version_list %} +
+
+ + +
+
+ {% for v in sp.version_list %} +
+ + View ↗ +
+ {% endfor %} +
+
{% endif %}{% endfor %} - -

Pick a versioned prompt, or type your own below to override it. Manage prompts →

-
+ + +
{% endif %}
@@ -435,6 +461,44 @@

Expose [...fieldset.querySelectorAll('input[name="system_prompt_version"]')]; + function syncAll() { + const checked = allRadios().find(r => r.checked); + fieldset.querySelectorAll('[data-prompt-row]').forEach(row => { + const rs = [...row.querySelectorAll('input[name="system_prompt_version"]')]; + const rowChecked = rs.find(r => r.checked); + const summary = row.querySelector('[data-prompt-summary]'); + if (!summary) return; + if (rowChecked) summary.textContent = `v${rowChecked.dataset.version}` + (rowChecked.hasAttribute('data-latest') ? ' · latest' : ' · older version'); + else { const latest = rs.find(r => 'latest' in r.dataset); summary.textContent = latest ? `v${latest.dataset.version} · latest` : ''; } + }); + if (checked && ta.value !== checked.dataset.content) ta.value = checked.dataset.content || ''; + } + allRadios().forEach(r => r.addEventListener('change', syncAll)); + fieldset.querySelectorAll('[data-prompt-row]').forEach(row => { + const btn = row.querySelector('[data-version-toggle]'); + if (btn) btn.addEventListener('click', () => { + const list = row.querySelector('[data-version-list]'); + const open = list.classList.toggle('hidden') === false; + btn.setAttribute('aria-expanded', String(open)); + btn.textContent = open ? 'Versions ▴' : 'Versions ▾'; + }); + }); + ta.addEventListener('input', () => { + allRadios().forEach(r => { r.checked = false; }); + fieldset.querySelectorAll('[data-prompt-summary]').forEach(s => s.textContent = 'custom'); + }); + syncAll(); + })(); + // --- Sync button: POST to /agents/sync/ then reload --- document.querySelectorAll('[data-sync-btn]').forEach(btn => { btn.addEventListener('click', async () => { diff --git a/templates/partials/run_detail_body.html b/templates/partials/run_detail_body.html index d3cb5a3..8391ceb 100644 --- a/templates/partials/run_detail_body.html +++ b/templates/partials/run_detail_body.html @@ -866,7 +866,10 @@

Frozen Manifest

{% endif %}
System Prompt - {% if system_prompt %}{{ system_prompt|truncatechars:60 }}{% else %}none{% endif %} + + {% if system_prompt_source %}{{ system_prompt_source.prompt }} v{{ system_prompt_source.version }}{% endif %} + {% if system_prompt %}{{ system_prompt|truncatechars:60 }}{% else %}none{% endif %} +
{% if system_prompt|length > 60 %}
Full system prompt diff --git a/templates/partials/run_settings.html b/templates/partials/run_settings.html index 966523c..ffabb64 100644 --- a/templates/partials/run_settings.html +++ b/templates/partials/run_settings.html @@ -27,12 +27,57 @@

Run settings

-
- - -

Sent to the target model as its system prompt, like your product would. Leave empty to test the bare model. Edit per run on review to compare prompts.

+
+ {% if system_prompts %} +
+ System Prompt +

Pick one prompt to fill the box below; open Versions to choose an older one. Only one can be selected.

+
+ + ＋ New +
+
+ {% for sp in system_prompts %}{% if sp.version_list %} +
+
+ + +
+
+ {% for v in sp.version_list %} +
+ + View ↗ +
+ {% endfor %} +
+
+ {% endif %}{% endfor %} + +
+
+ {% endif %} +
+ + +

Sent to the target model as its system prompt, like your product would. Leave empty to test the bare model. Edit per run on review to compare prompts.

+
diff --git a/templates/partials/run_settings_js.html b/templates/partials/run_settings_js.html index fab8e78..b83f1d0 100644 --- a/templates/partials/run_settings_js.html +++ b/templates/partials/run_settings_js.html @@ -1,8 +1,58 @@ {% comment %} -Behaviour for run_settings.html: generation-config normalisation and -submit-time JSON validation. +Behaviour for run_settings.html: predefined-prompt picker sync, generation-config +normalisation and submit-time JSON validation. {% endcomment %}