From 140e03587c5155c0807ebe0f6ba2d4b57eb035f1 Mon Sep 17 00:00:00 2001 From: fei <204683769+feiiiiii5@users.noreply.github.com> Date: Sun, 1 Nov 2026 16:12:16 +0800 Subject: [PATCH] [FIX] Keep execution-distinct seeds when deduplicating Standalone seeds and singleton groups were deduplicated by the behavioral key alone, so seeds that differed only by seed type, data type, or response_json_schema were silently skipped even though they execute differently. Standalone identity now includes those fields, and singleton/already-stored groups use the full member identity that multi-seed groups already use. Exact duplicates stay skipped; multi-seed group behavior is unchanged. Fixes #3738 --- pyrit/memory/memory_interface.py | 67 +++++++++++-- .../test_interface_seed_dedup_identity.py | 93 +++++++++++++++++++ 2 files changed, 150 insertions(+), 10 deletions(-) create mode 100644 tests/unit/memory/memory_interface/test_interface_seed_dedup_identity.py diff --git a/pyrit/memory/memory_interface.py b/pyrit/memory/memory_interface.py index e230165ae0..2b20c27d05 100644 --- a/pyrit/memory/memory_interface.py +++ b/pyrit/memory/memory_interface.py @@ -5010,11 +5010,13 @@ async def add_seeds_to_memory_async(self, *, seeds: Sequence[Seed[Any]], added_b """ Insert a list of seeds into the memory storage. - Standalone seeds with the same value hash, dataset, and behavioral inputs already - present in storage are skipped. Multi-seed groups are compared as complete groups, - including roles, sequence, metadata, and behavioral inputs, so shared content cannot strip - members from a different conversation. Duplicates *within* ``seeds`` are all inserted, - because the check looks at what storage held when the call started. + Standalone seeds with the same value hash, dataset, seed type, data type, response schema, + and behavioral inputs already present in storage are skipped. Singleton groups, and groups + whose complete membership is already stored, are compared by full member identity, including + roles, sequence, metadata, and behavioral inputs. Multi-seed groups are compared as complete + groups, so shared content cannot strip members from a different conversation. Duplicates + *within* ``seeds`` are all inserted, because the check looks at what storage held when the + call started. When a stored seed is copied to a new group, its caller-visible ID is updated after successful insertion. @@ -5040,6 +5042,21 @@ def _insert_prepared_seeds(self, *, seeds: Sequence[Seed[Any]]) -> None: for dataset, seed in existing_seeds if seed.prompt_group_id is not None } + # Seeds whose group is not retained as a new group (singleton groups, and groups whose + # complete membership is already stored) fall through to the standalone check below. Compare + # them by full member identity so schema- or metadata-distinct groups are not collapsed. + stored_member_pairs = { + (dataset, self._seed_group_member_key(seed)) for dataset, seed in existing_seeds if dataset + } + stored_member_keys = {self._seed_group_member_key(seed) for dataset, seed in existing_seeds if not dataset} + # Standalone identity additionally tracks the seed type and data type, which change what a + # seed executes as even when its value, dataset, and behavioral inputs match. + stored_standalone_pairs = { + (seed.value_sha256, dataset, self._seed_standalone_key(seed)) for dataset, seed in existing_seeds if dataset + } + stored_standalone_hashes = { + (seed.value_sha256, self._seed_standalone_key(seed)) for dataset, seed in existing_seeds if not dataset + } retained_seed_ids = [seed.id for seed in seeds if seed.prompt_group_id in retained_group_ids] stored_groups = ( { @@ -5057,7 +5074,6 @@ def _insert_prepared_seeds(self, *, seeds: Sequence[Seed[Any]]) -> None: for prompt in seeds: if not prompt.value_sha256: continue - behavior_key = self._seed_behavior_key(prompt) # A seed without a dataset name matches the hash in any dataset, mirroring the # filter that is applied when dataset_name is not supplied. if prompt.prompt_group_id in retained_group_ids: @@ -5075,11 +5091,22 @@ def _insert_prepared_seeds(self, *, seeds: Sequence[Seed[Any]]) -> None: copied_seed_ids.append((prompt, entry.id)) entries.append(entry) continue - if prompt.dataset_name: - if (prompt.value_sha256, prompt.dataset_name, behavior_key) in existing_pairs: + if prompt.prompt_group_id is not None: + # A singleton group (or a group already stored as a complete group) has no retained + # group ID; identify it by its full member identity instead of the standalone key. + member_key = self._seed_group_member_key(prompt) + if prompt.dataset_name: + if (prompt.dataset_name, member_key) in stored_member_pairs: + continue + elif member_key in stored_member_keys: + continue + else: + standalone_key = self._seed_standalone_key(prompt) + if prompt.dataset_name: + if (prompt.value_sha256, prompt.dataset_name, standalone_key) in stored_standalone_pairs: + continue + elif (prompt.value_sha256, standalone_key) in stored_standalone_hashes: continue - elif (prompt.value_sha256, behavior_key) in existing_hashes: - continue entries.append(SeedEntry(entry=prompt)) self._insert_entries(entries=entries) @@ -5120,6 +5147,26 @@ def _seed_group_member_key(seed: Seed[Any]) -> str: separators=(",", ":"), ) + @staticmethod + def _seed_standalone_key(seed: Seed[Any]) -> str: + """ + Identify a standalone seed by its type fields and behavioral inputs. + + Returns: + str: Canonical standalone identity. + """ + entry = SeedEntry(entry=seed) + return json.dumps( + { + "seed_type": entry.seed_type, + "data_type": entry.data_type, + "response_json_schema": getattr(seed, "response_json_schema", None), + "behavioral_parameters": SeedIdentifier.get_behavioral_parameters(seed), + }, + sort_keys=True, + separators=(",", ":"), + ) + def _get_new_seed_group_ids( self, *, diff --git a/tests/unit/memory/memory_interface/test_interface_seed_dedup_identity.py b/tests/unit/memory/memory_interface/test_interface_seed_dedup_identity.py new file mode 100644 index 0000000000..aec6ed5621 --- /dev/null +++ b/tests/unit/memory/memory_interface/test_interface_seed_dedup_identity.py @@ -0,0 +1,93 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Regression tests for seed deduplication identity (issue #3738).""" + +import json +from typing import Any + +from pyrit.memory import MemoryInterface +from pyrit.models import Seed, SeedGroup, SeedObjective, SeedPrompt + +DATASET = "dedup-identity" + +SCHEMA_ALPHA = {"type": "object", "properties": {"alpha": {"type": "string"}}, "required": ["alpha"]} +SCHEMA_BETA = {"type": "object", "properties": {"beta": {"type": "string"}}, "required": ["beta"]} + + +async def _add(memory: MemoryInterface, *seeds: Seed[Any]) -> None: + await memory.add_seeds_to_memory_async(seeds=list(seeds), added_by="tester") + + +async def test_add_seeds_keeps_same_value_with_distinct_seed_types(sqlite_instance: MemoryInterface) -> None: + """A prompt and an objective with the same value are execution-distinct, so both must stay.""" + await _add(sqlite_instance, SeedPrompt(value="shared value", dataset_name=DATASET, data_type="text")) + await _add(sqlite_instance, SeedObjective(value="shared value", dataset_name=DATASET)) + + stored = await sqlite_instance.get_seeds_async(dataset_name=DATASET) + assert len(stored) == 2 + + objectives = await sqlite_instance.get_seeds_async(dataset_name=DATASET, seed_type="objective") + assert len(objectives) == 1 + + +async def test_add_standalone_seeds_keep_distinct_data_types(sqlite_instance: MemoryInterface) -> None: + """The data type is execution-relevant, so same-value seeds with different data types must stay.""" + value = "https://example.com/shared-value" + await _add(sqlite_instance, SeedPrompt(value=value, dataset_name=DATASET, data_type="text")) + await _add(sqlite_instance, SeedPrompt(value=value, dataset_name=DATASET, data_type="url")) + + stored = await sqlite_instance.get_seeds_async(dataset_name=DATASET) + assert len(stored) == 2 + assert sorted(seed.data_type for seed in stored) == ["text", "url"] + + +async def test_add_singleton_groups_keep_distinct_response_json_schemas(sqlite_instance: MemoryInterface) -> None: + """Singleton groups that differ only by response schema are distinct and both must stay.""" + alpha_group = SeedGroup( + seeds=[ + SeedPrompt(value="schema value", dataset_name=DATASET, data_type="text", response_json_schema=SCHEMA_ALPHA) + ] + ) + beta_group = SeedGroup( + seeds=[ + SeedPrompt(value="schema value", dataset_name=DATASET, data_type="text", response_json_schema=SCHEMA_BETA) + ] + ) + + await _add(sqlite_instance, *alpha_group.seeds) + await _add(sqlite_instance, *beta_group.seeds) + + stored = await sqlite_instance.get_seeds_async(dataset_name=DATASET) + assert len(stored) == 2 + stored_schemas = sorted( + json.dumps(seed.response_json_schema, sort_keys=True) for seed in stored if isinstance(seed, SeedPrompt) + ) + assert stored_schemas == sorted([json.dumps(SCHEMA_ALPHA, sort_keys=True), json.dumps(SCHEMA_BETA, sort_keys=True)]) + + +async def test_add_singleton_groups_skip_exact_duplicate_content(sqlite_instance: MemoryInterface) -> None: + """Exact duplicate singleton group content is still skipped.""" + first = SeedGroup( + seeds=[ + SeedPrompt(value="repeat value", dataset_name=DATASET, data_type="text", response_json_schema=SCHEMA_ALPHA) + ] + ) + second = SeedGroup( + seeds=[ + SeedPrompt(value="repeat value", dataset_name=DATASET, data_type="text", response_json_schema=SCHEMA_ALPHA) + ] + ) + + await _add(sqlite_instance, *first.seeds) + await _add(sqlite_instance, *second.seeds) + + assert len(await sqlite_instance.get_seeds_async(dataset_name=DATASET)) == 1 + + +async def test_add_standalone_seeds_skip_exact_duplicates(sqlite_instance: MemoryInterface) -> None: + """Exact duplicate standalone seeds are still skipped.""" + await _add(sqlite_instance, SeedPrompt(value="plain value", dataset_name=DATASET, data_type="text")) + await _add(sqlite_instance, SeedPrompt(value="plain value", dataset_name=DATASET, data_type="text")) + + assert len(await sqlite_instance.get_seeds_async(dataset_name=DATASET)) == 1