From e29cf117da954a5db96163f5ce5a2cfa6ea231d3 Mon Sep 17 00:00:00 2001 From: Fazeel Usmani Date: Wed, 28 Oct 2026 16:33:13 +0530 Subject: [PATCH 1/4] FEAT: Browse Scenario result parents and children in the GUI (#3602) Add a read-only "Result hierarchy" section to the Scenario run page, built on the hierarchy endpoints from #3167. Roots load in pages with the root cursor. Expanding a result loads its saved children in pages with that node's own children cursor, so the two never mix. Children are shown at their absolute position under the parent, and missing, duplicate or cyclic references stay visible in place as unresolved. Run and group totals are shown exactly as the backend reports them and are never counted from loaded rows. Rootless results are reported, not turned into roots. A stale cursor (HTTP 400) shows a reload action instead of a generic error. Details reuse the existing attempt dialog, and fall back to the clicked node when the progress feed has not delivered that result yet. Tests: unit tests for child positions and root merging, component tests for paging, stale cursors, reloads, totals and empty/truncated/rootless states, and a Playwright suite against a seeded backend (playwright.scenario-hierarchy.config.ts), added to the CI matrix. --- .github/workflows/frontend_tests.yml | 6 +- .../fixtures/scenario_hierarchy_backend.py | 145 +++++++ frontend/e2e/scenario-hierarchy.spec.ts | 83 ++++ frontend/package.json | 1 + frontend/playwright.config.ts | 2 +- .../playwright.scenario-hierarchy.config.ts | 22 ++ .../Scenarios/ScenarioFlow.test.tsx | 1 + .../ScenarioResultHierarchy.styles.ts | 83 ++++ .../ScenarioResultHierarchy.test.tsx | 358 ++++++++++++++++++ .../Scenarios/ScenarioResultHierarchy.tsx | 353 +++++++++++++++++ .../Scenarios/ScenarioRunPage.test.tsx | 47 +++ .../components/Scenarios/ScenarioRunPage.tsx | 34 +- .../Scenarios/scenarioHierarchy.test.ts | 101 +++++ .../components/Scenarios/scenarioHierarchy.ts | 100 +++++ .../src/hooks/useScenarioResultHierarchy.ts | 325 ++++++++++++++++ frontend/src/services/api.test.ts | 46 +++ frontend/src/services/api.ts | 29 ++ frontend/src/types/index.ts | 100 +++++ pyrit/backend/README.md | 3 +- 19 files changed, 1832 insertions(+), 7 deletions(-) create mode 100644 frontend/e2e/fixtures/scenario_hierarchy_backend.py create mode 100644 frontend/e2e/scenario-hierarchy.spec.ts create mode 100644 frontend/playwright.scenario-hierarchy.config.ts create mode 100644 frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts create mode 100644 frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx create mode 100644 frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx create mode 100644 frontend/src/components/Scenarios/scenarioHierarchy.test.ts create mode 100644 frontend/src/components/Scenarios/scenarioHierarchy.ts create mode 100644 frontend/src/hooks/useScenarioResultHierarchy.ts diff --git a/.github/workflows/frontend_tests.yml b/.github/workflows/frontend_tests.yml index 00821be99f..18106deef6 100644 --- a/.github/workflows/frontend_tests.yml +++ b/.github/workflows/frontend_tests.yml @@ -86,6 +86,10 @@ jobs: config: playwright.paired-outcomes.config.ts shard: 1 total: 1 + - project: scenario-hierarchy + config: playwright.scenario-hierarchy.config.ts + shard: 1 + total: 1 outputs: expected-reports: ${{ strategy.job-total }} permissions: @@ -131,7 +135,7 @@ jobs: run: npx playwright install --with-deps chromium - name: Build the isolated fixture frontend - if: matrix.project == 'analytics' || matrix.project == 'paired-outcomes' + if: matrix.project == 'analytics' || matrix.project == 'paired-outcomes' || matrix.project == 'scenario-hierarchy' run: npm run build - name: Run E2E shard (no credentials needed) diff --git a/frontend/e2e/fixtures/scenario_hierarchy_backend.py b/frontend/e2e/fixtures/scenario_hierarchy_backend.py new file mode 100644 index 0000000000..ee7b4d17b0 --- /dev/null +++ b/frontend/e2e/fixtures/scenario_hierarchy_backend.py @@ -0,0 +1,145 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Disposable saved Scenario run with compound results for hierarchy browser acceptance tests.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING +from unittest.mock import patch +from uuid import UUID + +from frontend.e2e.fixtures import analytics_backend + +if TYPE_CHECKING: + from pyrit.memory import SQLiteMemory + +RUN_ID = UUID(int=0x3602) +CHILD_COUNT = 520 +MISSING_POSITION = 1 +DUPLICATE_POSITION = 3 + + +async def _seed_async(memory: SQLiteMemory) -> None: + from pyrit.common.utils import to_sha256 + from pyrit.models import ( + SCENARIO_RUN_PLAN_METADATA_KEY, + AtomicAttackIdentifier, + AttackIdentifier, + AttackOutcome, + AttackResult, + AttackResultRole, + ScenarioIdentifier, + ScenarioResult, + ScenarioRunPlan, + ScenarioRunPlanAtomicGroup, + ScenarioRunPlanGroupKind, + ScenarioRunPlanSeedGroup, + ScenarioRunState, + TargetIdentifier, + ) + + memory.disable_embedding() + timestamp = datetime(2026, 10, 28, tzinfo=UTC) + target = TargetIdentifier(class_name="MockPromptTarget", class_module="hierarchy_browser_fixture") + identifier = AtomicAttackIdentifier.build( + attack_identifier=AttackIdentifier( + class_name="SequentialAttack", + class_module="pyrit.executor.attack", + objective_target=target, + ) + ) + plan = ScenarioRunPlan( + atomic_groups=[ + ScenarioRunPlanAtomicGroup( + id=name, + atomic_attack_name=name, + display_group=display, + technique_eval_hash=f"{name}-eval", + kind=ScenarioRunPlanGroupKind.ATTACK, + seed_group_ids=["seed"], + ) + for name, display in [("sequential", "Sequential"), ("single", "Single turn")] + ], + seed_groups=[ScenarioRunPlanSeedGroup(id="seed", objective="seed", objective_sha256=to_sha256("seed"))], + ) + + def result( + index: int, + group: str, + *, + objective: str, + outcome: AttackOutcome = AttackOutcome.FAILURE, + child_ids: list[str] | None = None, + attempt_index: int | None = None, + ) -> AttackResult: + return AttackResult( + attack_result_id=str(UUID(int=0x360200000 + index)), + conversation_id=str(UUID(int=0x360300000 + index)), + objective=objective, + outcome=outcome, + timestamp=timestamp + timedelta(seconds=index), + atomic_attack_identifier=identifier, + attribution_parent_id=str(RUN_ID), + attribution_data={ + "seed_group_id": "seed", + "parent_eval_hash": f"{group}-eval", + "parent_collection": group, + # Like SequentialAttack: children share the parent's attribution plus their position. + **({"attempt_index": attempt_index} if attempt_index is not None else {}), + "result_role": ( + AttackResultRole.ORCHESTRATION.value + if child_ids is not None + else AttackResultRole.TARGET_FACING.value + ), + }, + metadata={"child_attack_result_ids": child_ids} if child_ids is not None else {}, + ) + + grandchildren = [ + result(1000 + i, "sequential", objective=f"Nested step {i + 1}", attempt_index=i + 1) for i in range(2) + ] + children = [ + result( + 10 + i, + "sequential", + objective=f"Step {i + 1}", + outcome=AttackOutcome.SUCCESS if i == 4 else AttackOutcome.FAILURE, + child_ids=[g.attack_result_id for g in grandchildren] if i == 0 else None, + attempt_index=i + 1, + ) + for i in range(CHILD_COUNT) + ] + child_ids = [child.attack_result_id for child in children] + child_ids[MISSING_POSITION] = str(UUID(int=0x3602DEAD)) + child_ids[DUPLICATE_POSITION] = child_ids[0] + parent = result(1, "sequential", objective="Sequential parent", child_ids=child_ids) + single = result(2, "single", objective="Single-turn attempt", outcome=AttackOutcome.SUCCESS) + + rows = {"sequential": [parent, *children, *grandchildren], "single": [single]} + await memory.add_attack_results_to_memory_async(attack_results=[row for group in rows.values() for row in group]) + await memory.add_scenario_results_to_memory_async( + scenario_results=[ + ScenarioResult( + id=RUN_ID, + scenario_identifier=ScenarioIdentifier( + class_name="HierarchyBrowserScenario", + class_module="hierarchy_browser_fixture", + version=1, + objective_target=target, + ), + scenario_description="Offline compound-result hierarchy acceptance fixture.", + attack_results=rows, + metadata={SCENARIO_RUN_PLAN_METADATA_KEY: plan.model_dump(mode="json")}, + scenario_run_state=ScenarioRunState.COMPLETED, + creation_time=timestamp, + completion_time=timestamp + timedelta(minutes=1), + ) + ] + ) + + +if __name__ == "__main__": + with patch.object(analytics_backend, "_seed_async", _seed_async): + analytics_backend.main() diff --git a/frontend/e2e/scenario-hierarchy.spec.ts b/frontend/e2e/scenario-hierarchy.spec.ts new file mode 100644 index 0000000000..1c25025980 --- /dev/null +++ b/frontend/e2e/scenario-hierarchy.spec.ts @@ -0,0 +1,83 @@ +import { expect, test, type Page } from '@playwright/test' +import { scenarioRunRoutePath } from '../src/utils/routeParams' + +// Seeded by e2e/fixtures/scenario_hierarchy_backend.py +const run = scenarioRunRoutePath('00000000-0000-0000-0000-000000003602') + +async function openHierarchy(page: Page) { + await page.goto(run) + const section = page.getByRole('region', { name: 'Result hierarchy', exact: true }) + await expect(section.getByText('Sequential parent', { exact: true })).toBeVisible() + return section +} + +test('browses a compound result through the real API, page by page, with stable totals', async ({ page }, testInfo) => { + const errors: string[] = [] + page.on('pageerror', (error) => errors.push(error.message)) + const section = await openHierarchy(page) + const totals = section.getByLabel('Run totals') + const before = await totals.textContent() + + const firstPage = page.waitForResponse((response) => /\/hierarchy\/nodes\/[^/?]+\?limit=200$/.test(response.url())) + await section.getByRole('button', { name: 'Expand children of sequential: Sequential parent' }).click() + expect((await firstPage).status()).toBe(200) + + const children = section.getByRole('list', { name: 'Children of sequential: Sequential parent' }) + await expect(children.getByText('Step 1', { exact: true })).toBeVisible() + await expect(children.locator('li', { hasText: '#2' }).first()).toContainText('Unresolved reference') + await expect(children.locator('li', { hasText: '#4' }).first()).toContainText('Unresolved reference') + await expect(section.getByText('Showing 200 of 520 saved children')).toBeVisible() + + for (const shown of [400, 520]) { + const next = page.waitForResponse((response) => response.url().includes('children_cursor=')) + await section.getByRole('button', { name: 'Load more children' }).click() + expect((await next).status()).toBe(200) + await expect(section.getByText(`Showing ${shown} of 520 saved children`)).toBeVisible() + } + await expect(section.getByRole('button', { name: 'Load more children' })).toHaveCount(0) + await expect(children.getByText('Step 520', { exact: true })).toBeVisible() + + await children.getByRole('button', { name: 'Expand children of #1 sequential: Step 1' }).click() + await expect(children.getByText('Nested step 2', { exact: true })).toBeVisible() + expect(await totals.textContent()).toBe(before) + + await section.getByRole('button', { name: 'Open details for single: Single-turn attempt' }).click() + await expect(page.getByRole('dialog', { name: 'single' })).toBeVisible() + await page.getByRole('button', { name: 'Close' }).click() + await expect(page.getByRole('dialog')).toHaveCount(0) + + await page.getByRole('heading', { name: 'Result hierarchy', exact: true }).evaluate((heading) => + heading.scrollIntoView({ block: 'start' }), + ) + await page.screenshot({ path: testInfo.outputPath('scenario-hierarchy-desktop.png') }) + expect(errors).toEqual([]) +}) + +test.describe('on a narrow touch screen', () => { + // Touch targets grow only for coarse pointers, as in touch-targets.spec.ts. + test.use({ hasTouch: true }) + + test('keeps the hierarchy usable at a narrow viewport and from the keyboard', async ({ page }, testInfo) => { + await page.setViewportSize({ width: 390, height: 844 }) + const section = await openHierarchy(page) + + const expand = section.getByRole('button', { name: 'Expand children of sequential: Sequential parent' }) + await expand.focus() + await page.keyboard.press('Enter') + // The same button is renamed once expanded, so look it up by its new name. + await expect(section.getByRole('button', { name: 'Collapse children of sequential: Sequential parent' })).toHaveAttribute( + 'aria-expanded', + 'true', + ) + await expect(section.getByText('Showing 200 of 520 saved children')).toBeVisible() + + const loadMore = section.getByRole('button', { name: 'Load more children' }) + const size = await loadMore.boundingBox() + expect(size?.height).toBeGreaterThanOrEqual(44) + const overflow = await section.evaluate((element) => element.scrollWidth > element.clientWidth) + expect(overflow).toBe(false) + + await section.scrollIntoViewIfNeeded() + await page.screenshot({ path: testInfo.outputPath('scenario-hierarchy-mobile.png') }) + }) +}) diff --git a/frontend/package.json b/frontend/package.json index d830f0fce1..00d2dfc4b0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -17,6 +17,7 @@ "test:coverage": "jest --coverage", "test:e2e": "playwright test", "test:e2e:paired-outcomes": "npm run build && playwright test --config playwright.paired-outcomes.config.ts", + "test:e2e:scenario-hierarchy": "npm run build && playwright test --config playwright.scenario-hierarchy.config.ts", "test:e2e:analytics": "npm run build && playwright test --config playwright.analytics.config.ts", "test:e2e:seeded": "playwright test --project seeded", "test:e2e:live": "E2E_LIVE_MODE=true playwright test --project live", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index db313d78eb..0c01eee691 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -26,7 +26,7 @@ const E2E_BACKEND_URL = `http://127.0.0.1:${E2E_BACKEND_PORT}`; export default defineConfig({ testDir: "./e2e", - testIgnore: ["**/analytics.spec.ts", "**/paired-outcomes.spec.ts"], + testIgnore: ["**/analytics.spec.ts", "**/paired-outcomes.spec.ts", "**/scenario-hierarchy.spec.ts"], fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, diff --git a/frontend/playwright.scenario-hierarchy.config.ts b/frontend/playwright.scenario-hierarchy.config.ts new file mode 100644 index 0000000000..ffe4eda1d2 --- /dev/null +++ b/frontend/playwright.scenario-hierarchy.config.ts @@ -0,0 +1,22 @@ +import process from 'node:process' +import { defineConfig } from '@playwright/test' +import analytics from './playwright.analytics.config' + +const port = Number(process.env.PYRIT_E2E_HIERARCHY_PORT ?? '18498') +if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error('Invalid scenario hierarchy test port') +const url = `http://127.0.0.1:${port}` + +export default defineConfig({ + ...analytics, + testMatch: 'scenario-hierarchy.spec.ts', + use: { + ...analytics.use, + baseURL: url, + storageState: { cookies: [], origins: [{ origin: url, localStorage: [{ name: 'pyrit-tour-completed', value: 'true' }] }] }, + }, + projects: [{ name: 'scenario-hierarchy', use: { viewport: { width: 1440, height: 1050 } } }], + webServer: { + command: `cd .. && uv run --no-sync python -m frontend.e2e.fixtures.scenario_hierarchy_backend --port ${port}`, + env: { PYTHONIOENCODING: 'utf-8' }, url: `${url}/api/health`, reuseExistingServer: false, timeout: 120_000, + }, +}) diff --git a/frontend/src/components/Scenarios/ScenarioFlow.test.tsx b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx index 6a2d728c3c..f52b71b0f9 100644 --- a/frontend/src/components/Scenarios/ScenarioFlow.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx @@ -24,6 +24,7 @@ jest.mock('@/services/api', () => ({ scenariosApi: { cancelRun: jest.fn(), estimateRun: jest.fn(), + getRunHierarchy: jest.fn(() => new Promise(() => {})), getScenario: jest.fn(), listCatalog: jest.fn(), startRun: jest.fn(), diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts b/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts new file mode 100644 index 0000000000..c02e83a85b --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts @@ -0,0 +1,83 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { NARROW_VIEWPORT_QUERY, mobileTouchTarget } from '@/styles/touchTargets' + +export const useScenarioResultHierarchyStyles = makeStyles({ + root: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + minWidth: 0, + }, + totals: { + display: 'flex', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalL, + }, + hint: { + color: tokens.colorNeutralForeground3, + }, + group: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXS, + paddingTop: tokens.spacingVerticalS, + borderTop: `${tokens.strokeWidthThin} solid ${tokens.colorNeutralStroke2}`, + }, + tree: { + listStyleType: 'none', + margin: 0, + paddingLeft: 0, + }, + subtree: { + listStyleType: 'none', + margin: 0, + paddingLeft: tokens.spacingHorizontalXXL, + borderLeft: `${tokens.strokeWidthThin} solid ${tokens.colorNeutralStroke2}`, + [NARROW_VIEWPORT_QUERY]: { + paddingLeft: tokens.spacingHorizontalM, + }, + }, + row: { + display: 'flex', + alignItems: 'center', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalS, + paddingTop: tokens.spacingVerticalXXS, + paddingBottom: tokens.spacingVerticalXXS, + minWidth: 0, + }, + expandButton: { + ...mobileTouchTarget, + }, + expandSpacer: { + width: '2rem', + flexShrink: 0, + ...mobileTouchTarget, + }, + position: { + color: tokens.colorNeutralForeground3, + fontVariantNumeric: 'tabular-nums', + }, + objective: { + minWidth: 0, + flexBasis: '16rem', + flexGrow: 1, + overflowWrap: 'anywhere', + }, + unresolved: { + color: tokens.colorNeutralForeground3, + fontStyle: 'italic', + overflowWrap: 'anywhere', + }, + footer: { + display: 'flex', + alignItems: 'center', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalS, + paddingTop: tokens.spacingVerticalXS, + }, + touchTarget: { + ...mobileTouchTarget, + }, +}) diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx b/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx new file mode 100644 index 0000000000..edfe0498c5 --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx @@ -0,0 +1,358 @@ +import { render, screen, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' + +import { scenariosApi } from '@/services/api' +import type { ScenarioHierarchyNode, ScenarioHierarchyProjection, ScenarioProgressCounts } from '@/types' + +import ScenarioResultHierarchy from './ScenarioResultHierarchy' + +jest.mock('@/services/api', () => ({ + scenariosApi: { getRunHierarchy: jest.fn(), getRunHierarchyNode: jest.fn() }, +})) + +const getRoots = scenariosApi.getRunHierarchy as jest.Mock +const getNode = scenariosApi.getRunHierarchyNode as jest.Mock + +const TOTALS: ScenarioProgressCounts = { + completed: 7, + planned: 9, + succeeded: 2, + success_percentage: 29, + errors: 1, + retries: 4, +} + +function node(id: string, overrides: Partial = {}): ScenarioHierarchyNode { + return { + attack_result_id: id, + conversation_id: `conv-${id}`, + result_role: 'target_facing', + atomic_attack_name: `attack ${id}`, + atomic_group_id: 'g1', + seed_group_id: 's1', + objective: `objective ${id}`, + outcome: 'failure', + execution_time_ms: 1000, + executed_turns: 1, + timestamp: '2026-10-28T00:00:00Z', + total_retries: 0, + retries: [], + retries_truncated: false, + child_attack_result_ids: [], + child_references_truncated: false, + children: [], + children_truncated: false, + child_offset: 0, + unresolved_child_attack_result_ids: [], + unresolved_child_references_truncated: false, + conversations: [], + conversations_truncated: false, + ...overrides, + } +} + +function projection(roots: ScenarioHierarchyNode[], overrides: Partial = {}) { + return { + scenario_result_id: 'run', + scenario_name: 'scenario', + scenario_version: 1, + status: 'COMPLETED', + groups: [ + { + atomic_group_id: 'g1', + atomic_attack_name: 'sequential', + display_group: 'Sequential', + roots, + counts: TOTALS, + }, + ], + overall_counts: TOTALS, + diagnostics: { + failed_attacks: [], + failed_attacks_count: 0, + failed_attacks_truncated: false, + attack_retries: [], + attack_retries_count: 0, + attack_retries_truncated: false, + total_retries: 4, + rootless_result_ids: [], + rootless_results_count: 0, + rootless_results_truncated: false, + }, + truncated: false, + has_more: false, + next_cursor: null, + ...overrides, + } satisfies ScenarioHierarchyProjection +} + +function cursorError(status: number) { + return { isAxiosError: true, response: { status, data: { detail: 'Cursor does not match' } }, message: 'x' } +} + +/** An orchestration parent whose children the preview deferred. */ +const PARENT = node('parent', { + result_role: 'orchestration', + child_attack_result_ids: ['c1', 'missing'], + children_truncated: true, + children_count: 3, +}) + +function mount(onOpenResult = jest.fn()) { + render( + + + , + ) + return onOpenResult +} + +const totalsText = () => screen.getByLabelText('Run totals').textContent + +beforeEach(() => { + getRoots.mockReset() + getNode.mockReset() +}) + +test('loads children by position, keeps unresolved references visible, and pages with the node cursor', async () => { + const user = userEvent.setup() + getRoots.mockResolvedValue(projection([PARENT])) + getNode + .mockResolvedValueOnce( + node('parent', { + result_role: 'orchestration', + child_attack_result_ids: ['c1', 'missing'], + children: [node('c1', { child_index: 0, parent_attack_result_id: 'parent' })], + children_count: 3, + next_children_cursor: 'children-2', + }), + ) + .mockResolvedValueOnce( + node('parent', { + result_role: 'orchestration', + child_offset: 2, + child_attack_result_ids: ['c3'], + children: [node('c3', { child_index: 2, parent_attack_result_id: 'parent' })], + children_count: 3, + }), + ) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + const children = await screen.findByRole('list', { name: 'Children of attack parent: objective parent' }) + + expect(getNode).toHaveBeenLastCalledWith('run', 'parent', { limit: 200 }, expect.any(AbortSignal)) + expect(within(children).getByText('objective c1')).toBeInTheDocument() + expect(within(children).getByText('Unresolved reference: missing')).toBeInTheDocument() + expect(within(children).getByText('#2')).toBeInTheDocument() + expect(screen.getByText('Showing 2 of 3 saved children')).toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Load more children' })) + + expect(await within(children).findByText('objective c3')).toBeInTheDocument() + expect(getNode).toHaveBeenLastCalledWith( + 'run', + 'parent', + { limit: 200, children_cursor: 'children-2' }, + expect.any(AbortSignal), + ) + expect(within(children).getByText('#3')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Load more children' })).not.toBeInTheDocument() + expect(getRoots).toHaveBeenCalledTimes(1) +}) + +test('run totals come from the backend and do not change as roots and children load', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([PARENT], { has_more: true, next_cursor: 'roots-2' })) + .mockResolvedValueOnce(projection([node('second-root')])) + getNode.mockResolvedValue( + node('parent', { + result_role: 'orchestration', + child_attack_result_ids: ['c1'], + children: [node('c1', { child_index: 0 })], + children_count: 1, + }), + ) + mount() + + await screen.findByLabelText('Run totals') + const before = totalsText() + expect(before).toContain('Completed: 7 of 9') + + await user.click(screen.getByRole('button', { name: 'Load more results' })) + await screen.findByText('objective second-root') + expect(getRoots).toHaveBeenLastCalledWith('run', { max_nodes: 1000, cursor: 'roots-2' }, expect.any(AbortSignal)) + + await user.click(screen.getByRole('button', { name: 'Expand children of attack parent: objective parent' })) + await screen.findByText('objective c1') + + expect(totalsText()).toBe(before) +}) + +test('expanding a node whose preview is complete makes no request', async () => { + const user = userEvent.setup() + getRoots.mockResolvedValue( + projection([ + node('parent', { + result_role: 'orchestration', + child_attack_result_ids: ['c1'], + children: [node('c1', { child_index: 0 })], + children_count: 1, + }), + ]), + ) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + + expect(screen.getByText('objective c1')).toBeInTheDocument() + expect(getNode).not.toHaveBeenCalled() +}) + +test('a stale child cursor asks to reload the children and restarts paging from the first page', async () => { + const user = userEvent.setup() + getRoots.mockResolvedValue(projection([PARENT])) + getNode + .mockResolvedValueOnce( + node('parent', { + child_attack_result_ids: ['c1'], + children: [node('c1', { child_index: 0 })], + children_count: 3, + next_children_cursor: 'children-2', + }), + ) + .mockRejectedValueOnce(cursorError(400)) + .mockResolvedValueOnce( + node('parent', { + child_attack_result_ids: ['c9'], + children: [node('c9', { child_index: 0 })], + children_count: 1, + }), + ) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + await user.click(await screen.findByRole('button', { name: 'Load more children' })) + + expect(await screen.findByText("This result's saved children changed since they were loaded.")).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Load more children' })).not.toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Reload children' })) + + expect(await screen.findByText('objective c9')).toBeInTheDocument() + expect(screen.queryByText('objective c1')).not.toBeInTheDocument() + expect(getNode).toHaveBeenLastCalledWith('run', 'parent', { limit: 200 }, expect.any(AbortSignal)) +}) + +test('a stale root cursor asks to reload the hierarchy', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([node('r1')], { has_more: true, next_cursor: 'roots-2' })) + .mockRejectedValueOnce(cursorError(400)) + .mockResolvedValueOnce(projection([node('fresh')])) + mount() + + await user.click(await screen.findByRole('button', { name: 'Load more results' })) + expect(await screen.findByText("This run's saved results changed since they were loaded.")).toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Reload hierarchy' })) + + expect(await screen.findByText('objective fresh')).toBeInTheDocument() + expect(getRoots).toHaveBeenLastCalledWith('run', { max_nodes: 1000 }, expect.any(AbortSignal)) +}) + +test('a failed first load can be retried', async () => { + const user = userEvent.setup() + getRoots.mockRejectedValueOnce(cursorError(500)).mockResolvedValueOnce(projection([node('r1')])) + mount() + + await user.click(await screen.findByRole('button', { name: 'Retry' })) + + expect(await screen.findByText('objective r1')).toBeInTheDocument() +}) + +test('shows empty, truncated and rootless states without inventing roots', async () => { + getRoots.mockResolvedValue({ + ...projection([], { truncated: true }), + groups: [], + diagnostics: { ...projection([]).diagnostics, rootless_result_ids: ['x', 'y'], rootless_results_count: 2 }, + }) + mount() + + expect(await screen.findByText(/2 saved results have no root in this run/)).toHaveTextContent('Their IDs: x, y.') + expect(screen.getByText(/Some results are not in this preview/)).toBeInTheDocument() + expect(screen.queryByText('No saved results are available for this run yet.')).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: /Expand/ })).not.toBeInTheDocument() +}) + +test('opens details for a node with the clicked button as the focus target', async () => { + const user = userEvent.setup() + getRoots.mockResolvedValue(projection([node('r1')])) + const onOpenResult = mount() + + const button = await screen.findByRole('button', { name: 'Open details for attack r1: objective r1' }) + await user.click(button) + + expect(onOpenResult).toHaveBeenCalledWith(expect.objectContaining({ attack_result_id: 'r1' }), button) +}) + +test('an empty run says so', async () => { + getRoots.mockResolvedValue({ ...projection([]), groups: [] }) + mount() + + expect(await screen.findByText('No saved results are available for this run yet.')).toBeInTheDocument() +}) + +test('shows the totals the backend reports, not totals counted from loaded rows', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([node('r1')], { has_more: true, next_cursor: 'roots-2' })) + .mockResolvedValueOnce(projection([node('r2')], { overall_counts: { ...TOTALS, completed: 8 } })) + mount() + + await user.click(await screen.findByRole('button', { name: 'Load more results' })) + await screen.findByText('objective r2') + + expect(totalsText()).toContain('Completed: 8 of 9') +}) + +test('reloading after expanding starts from a collapsed tree with no stale children', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([PARENT], { has_more: true, next_cursor: 'roots-2' })) + .mockRejectedValueOnce(cursorError(400)) + .mockResolvedValueOnce(projection([PARENT])) + getNode.mockResolvedValue( + node('parent', { child_attack_result_ids: ['c1'], children: [node('c1', { child_index: 0 })], children_count: 1 }), + ) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + await screen.findByText('objective c1') + await user.click(screen.getByRole('button', { name: 'Load more results' })) + await user.click(await screen.findByRole('button', { name: 'Reload hierarchy' })) + + const expand = await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' }) + expect(expand).toHaveAttribute('aria-expanded', 'false') + expect(screen.queryByText('objective c1')).not.toBeInTheDocument() + + await user.click(expand) + expect(await screen.findByText('objective c1')).toBeInTheDocument() + expect(getNode).toHaveBeenCalledTimes(2) +}) + +test('notes that a run still in progress is a snapshot and can be reloaded', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([node('r1')], { status: 'IN_PROGRESS' })) + .mockResolvedValueOnce(projection([node('r1'), node('r2')], { status: 'COMPLETED' })) + mount() + + expect(await screen.findByText(/This run is still in progress/)).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Reload hierarchy' })) + + expect(await screen.findByText('objective r2')).toBeInTheDocument() + expect(screen.queryByText(/This run is still in progress/)).not.toBeInTheDocument() +}) diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx b/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx new file mode 100644 index 0000000000..810803c483 --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx @@ -0,0 +1,353 @@ +import { + Badge, + Button, + MessageBar, + MessageBarActions, + MessageBarBody, + Text, +} from '@fluentui/react-components' +import { ChevronDownRegular, ChevronRightRegular } from '@fluentui/react-icons' + +import OutcomeBadge from '@/components/OutcomeBadge' +import { formatAttackSuccess, formatDuration } from '@/components/AttackResults/attackAttemptFormatting' +import { + useScenarioResultHierarchy, + type HierarchyChildrenState, + type UseScenarioResultHierarchyResult, +} from '@/hooks/useScenarioResultHierarchy' +import type { ScenarioHierarchyNode, ScenarioProgressCounts, ScenarioRunState } from '@/types' + +import { childSlots, hasChildReferences, resultRoleLabel } from './scenarioHierarchy' +import { useScenarioResultHierarchyStyles } from './ScenarioResultHierarchy.styles' + +interface ScenarioResultHierarchyProps { + readonly scenarioResultId: string + readonly onOpenResult: (node: ScenarioHierarchyNode, trigger: HTMLElement) => void +} + +/** Read-only browser for a run's saved result lineage: orchestration parents and their ordered children. */ +export default function ScenarioResultHierarchy({ scenarioResultId, onOpenResult }: ScenarioResultHierarchyProps) { + const styles = useScenarioResultHierarchyStyles() + const hierarchy = useScenarioResultHierarchy(scenarioResultId) + const { roots, expanded, toggleNode: toggle } = hierarchy + + // Plain status text, not a Spinner: a progressbar role here would be read as run progress. + if (roots.status === 'loading') { + return ( + + Loading result hierarchy... + + ) + } + + if (roots.status === 'error') { + return ( + + Could not load the result hierarchy: {roots.error} + + + + + ) + } + + const rootless = roots.diagnostics?.rootless_results_count ?? 0 + const rootlessSample = roots.diagnostics?.rootless_result_ids ?? [] + const runActive = roots.runStatus !== null && ACTIVE_RUN_STATES.has(roots.runStatus) + + return ( +
+ {roots.overallCounts && } + + Totals cover the whole run as reported by the server; they are not counted from the rows shown here. + + {runActive && ( +
+ + This run is still in progress, so this hierarchy shows the results saved when it loaded. + + +
+ )} + + {roots.groups.length === 0 ? ( + rootless === 0 && No saved results are available for this run yet. + ) : ( + roots.groups.map((group) => ( +
+ {group.display_group} + + {group.atomic_attack_name} · {formatCounts(group.counts)} + +
    + {group.roots.map((node, i) => { + const key = `${group.atomic_group_id}:${i}:${node.attack_result_id}` + return ( + + ) + })} +
+
+ )) + )} + + {roots.truncated && ( + + Some results are not in this preview, such as deeper levels or results it could not place. Expand a + result to load its saved children. + + )} + {rootless > 0 && ( + + {rootless} saved {rootless === 1 ? 'result has' : 'results have'} no root in this run (for example, a + cycle of references) and {rootless === 1 ? 'is' : 'are'} not shown in the tree. + {rootlessSample.length > 0 && + ` ${roots.diagnostics?.rootless_results_truncated ? 'Some of their IDs' : 'Their IDs'}: ${rootlessSample.join(', ')}.`} + + )} + + {roots.staleCursor ? ( + + This run's saved results changed since they were loaded. + + + + + ) : roots.error ? ( + + Could not load more results: {roots.error} + + + + + ) : null} + {roots.nextCursor && !roots.staleCursor && !roots.error && ( +
+ +
+ )} +
+ ) +} + +interface HierarchyNodeItemProps { + readonly nodeKey: string + readonly node: ScenarioHierarchyNode + /** Absolute position under the parent, or null for a root. */ + readonly position: number | null + readonly expanded: ReadonlySet + readonly hierarchy: UseScenarioResultHierarchyResult + readonly onToggle: (key: string, node: ScenarioHierarchyNode) => void + readonly onOpenResult: (node: ScenarioHierarchyNode, trigger: HTMLElement) => void +} + +function HierarchyNodeItem({ + nodeKey, + node, + position, + expanded, + hierarchy, + onToggle, + onOpenResult, +}: HierarchyNodeItemProps) { + const styles = useScenarioResultHierarchyStyles() + const isExpanded = expanded.has(nodeKey) + const expandable = hasChildReferences(node) + const panelId = `hierarchy-${nodeKey.replace(/[^a-zA-Z0-9_-]/g, '-')}` + const label = nodeLabel(node, position) + + return ( +
  • +
    + {expandable ? ( + +
    + {expandable && isExpanded && ( + + )} +
    • + ) +} + +interface ChildListProps { + readonly panelId: string + readonly parentKey: string + readonly parent: ScenarioHierarchyNode + readonly state: HierarchyChildrenState | undefined + readonly expanded: ReadonlySet + readonly hierarchy: UseScenarioResultHierarchyResult + readonly onToggle: (key: string, node: ScenarioHierarchyNode) => void + readonly onOpenResult: (node: ScenarioHierarchyNode, trigger: HTMLElement) => void +} + +function ChildList({ + panelId, + parentKey, + parent, + state, + expanded, + hierarchy, + onToggle, + onOpenResult, +}: ChildListProps) { + const styles = useScenarioResultHierarchyStyles() + const pages = state?.pages ?? [] + const slots = pages.flatMap(childSlots) + const loaded = slots.length + const total = state?.childrenCount ?? null + + return ( +
    +
      + {slots.map((slot) => + slot.kind === 'child' ? ( + + ) : ( +
    • +
      + + #{slot.position + 1} + Unresolved reference: {slot.attackResultId} +
      +
      • + ), + )} +
    +
    + {state?.loading && ( + + Loading children... + + )} + {!state?.loading && pages.length > 0 && ( + + {total !== null ? `Showing ${loaded} of ${total} saved children` : `Showing ${loaded} saved children`} + + )} + {state?.staleCursor ? ( + + This result's saved children changed since they were loaded. + + + + + ) : state?.error ? ( + + Could not load children: {state.error} + + + + + ) : null} + {state?.nextCursor && !state.loading && !state.staleCursor && !state.error && ( + + )} +
    +
    + ) +} + +function RunTotals({ counts }: { readonly counts: ScenarioProgressCounts }) { + const styles = useScenarioResultHierarchyStyles() + return ( +
    + Completed: {counts.planned !== null ? `${counts.completed} of ${counts.planned}` : counts.completed} + Succeeded: {counts.succeeded} + Errors: {counts.errors} + Retries: {counts.retries} +
    + ) +} + +const ACTIVE_RUN_STATES: ReadonlySet = new Set(['CREATED', 'QUEUED', 'IN_PROGRESS']) +const LABEL_OBJECTIVE_LENGTH = 60 + +/** Accessible name for a result; sibling results usually share an attack name, so add position and objective. */ +function nodeLabel(node: ScenarioHierarchyNode, position: number | null): string { + const objective = + node.objective.length > LABEL_OBJECTIVE_LENGTH + ? `${node.objective.slice(0, LABEL_OBJECTIVE_LENGTH)}...` + : node.objective + const prefix = position === null ? '' : `#${position + 1} ` + return `${prefix}${node.atomic_attack_name}: ${objective}` +} + +function formatCounts(counts: ScenarioProgressCounts): string { + const completed = counts.planned !== null ? `${counts.completed} of ${counts.planned}` : `${counts.completed}` + return `${completed} completed, ${counts.succeeded} succeeded, ${counts.errors} errors, ${counts.retries} retries` +} diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx index 52fec6b6d7..b3c2947f48 100644 --- a/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx @@ -43,6 +43,9 @@ jest.mock('@/services/api', () => ({ cancelRun: jest.fn(), resumeRun: jest.fn(), getRunProgress: jest.fn(), + // The hierarchy section has its own tests; keep it loading so these tests cover the rest of the page. + getRunHierarchy: jest.fn(() => new Promise(() => {})), + getRunHierarchyNode: jest.fn(), }, })) @@ -687,6 +690,7 @@ describe('ScenarioRunPage', () => { 'Overall progress', 'Paired outcomes', 'Atomic attack groups', + 'Result hierarchy', 'Objective Scorer', 'Techniques', 'Objectives', @@ -1219,6 +1223,49 @@ describe('ScenarioRunPage', () => { } }) + it('opens details for a hierarchy result that the progress feed has not delivered yet', async () => { + const user = userEvent.setup() + const hierarchyOnly = { + attack_result_id: 'hierarchy-only', + conversation_id: 'conversation-h', + result_role: 'orchestration', + atomic_attack_name: 'sequential-parent', + atomic_group_id: 'atomic-1', + seed_group_id: 'seed-unplanned', + objective: 'Objective carried by the hierarchy node', + outcome: 'failure', + execution_time_ms: 1200, + executed_turns: 0, + timestamp: '2026-01-01T00:00:00Z', + total_retries: 0, + retries: [], + retries_truncated: false, + child_attack_result_ids: [], + child_references_truncated: false, + children: [], + children_truncated: false, + child_offset: 0, + unresolved_child_attack_result_ids: [], + unresolved_child_references_truncated: false, + conversations: [], + conversations_truncated: false, + } + const counts = { completed: 1, planned: 1, succeeded: 0, success_percentage: 0, errors: 0, retries: 0 } + ;(scenariosApi.getRunHierarchy as jest.Mock).mockResolvedValueOnce({ + groups: [{ atomic_group_id: 'atomic-1', atomic_attack_name: 'a', display_group: 'Group', roots: [hierarchyOnly], counts }], + overall_counts: counts, + diagnostics: { rootless_results_count: 0 }, + truncated: false, + has_more: false, + }) + renderPage() + + await user.click(await screen.findByRole('button', { name: 'Open details for sequential-parent: Objective carried by the hierarchy node' })) + + const dialog = await screen.findByRole('dialog', { name: 'sequential-parent' }) + expect(within(dialog).getByText('Objective carried by the hierarchy node')).toBeInTheDocument() + }) + it('shows descriptive technique details without result metrics', async () => { const user = userEvent.setup() renderPage() diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.tsx index 0c96338a58..9e4f270609 100644 --- a/frontend/src/components/Scenarios/ScenarioRunPage.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunPage.tsx @@ -51,6 +51,7 @@ import { useScenarioQueue } from '@/hooks/useScenarioQueue' import { scenariosApi } from '@/services/api' import { toApiError } from '@/services/errors' import type { + ScenarioHierarchyNode, ScenarioProgressCounts, ScenarioProgressHeader, ScenarioProgressResult, @@ -75,6 +76,7 @@ import { ObjectiveDetailsDialog, TechniqueDetailsDialog } from './ScenarioRunDia import { useScenarioRunPageStyles } from './ScenarioRunPage.styles' import ScenarioQueue from './ScenarioQueue' import PairedOutcomes from './PairedOutcomes' +import ScenarioResultHierarchy from './ScenarioResultHierarchy' const CLOCK_REFRESH_INTERVAL_MS = 1_000 const MAX_VISIBLE_ATTEMPTS_PER_GROUP = 100 @@ -129,6 +131,7 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu const [atomicGroupsExpandedChoice, setAtomicGroupsExpandedChoice] = useState(null) const [expandedGroupIds, setExpandedGroupIds] = useState>(new Set()) const detailsTriggerRef = useRef(null) + const [hierarchySelection, setHierarchySelection] = useState(null) const navigationState = location.state as { fromScenarioHistory?: boolean scenarioHistorySearch?: string @@ -163,9 +166,12 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu () => new Map(state.summary?.atomic_groups.map((group) => [group.id, group]) ?? []), [state.summary], ) - const selectedAttempt = useMemo( - () => state.results.find((attempt) => attempt.attack_result_id === attackResultId) ?? null, - [attackResultId, state.results], + // A hierarchy node can be opened before the paged progress feed has delivered it. + const selectedAttempt = useMemo( + () => + state.results.find((attempt) => attempt.attack_result_id === attackResultId) ?? + (hierarchySelection?.attack_result_id === attackResultId ? hierarchySelection : null), + [attackResultId, state.results, hierarchySelection], ) const selectedAttemptGroup = selectedAttempt ? plannedAtomicGroups.get(selectedAttempt.atomic_group_id) @@ -199,6 +205,12 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu navigate(scenarioRunAttackRoutePath(scenarioResultId, attempt.attack_result_id), { state: location.state }) } + const openResultDetails = (node: ScenarioHierarchyNode, trigger: HTMLElement): void => { + detailsTriggerRef.current = trigger + setHierarchySelection(node) + navigate(scenarioRunAttackRoutePath(scenarioResultId, node.attack_result_id), { state: location.state }) + } + const toggleDisplayGroup = (groupId: string): void => { setExpandedGroupIds((current) => { const updated = new Set(current) @@ -638,6 +650,17 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu +
    + + Result hierarchy + + +
    +
    Objective Scorer @@ -790,7 +813,10 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu = {}): ScenarioHierarchyNode { + return { + attack_result_id: 'ar', + conversation_id: '', + result_role: 'target_facing', + atomic_attack_name: 'attack', + atomic_group_id: 'g1', + seed_group_id: 's1', + objective: 'objective', + outcome: 'success', + execution_time_ms: 0, + executed_turns: 0, + timestamp: '2026-10-28T00:00:00Z', + total_retries: 0, + retries: [], + retries_truncated: false, + child_attack_result_ids: [], + child_references_truncated: false, + children: [], + children_truncated: false, + child_offset: 0, + unresolved_child_attack_result_ids: [], + unresolved_child_references_truncated: false, + conversations: [], + conversations_truncated: false, + ...overrides, + } +} + +const counts = (completed: number) => ({ + completed, + planned: 10, + succeeded: 0, + success_percentage: null, + errors: 0, + retries: 0, +}) + +function group(id: string, roots: string[], completed: number): ScenarioHierarchyGroup { + return { + atomic_group_id: id, + atomic_attack_name: id, + display_group: id, + roots: roots.map((rootId) => node({ attack_result_id: rootId, atomic_group_id: id })), + counts: counts(completed), + } +} + +describe('childSlots', () => { + it('keeps every saved position, including unresolved and duplicate references', () => { + // a resolves at 200; 'missing' never resolves; the second 'a' is a duplicate and stays unresolved + const slots = childSlots({ + offset: 200, + referenceIds: ['a', 'missing', 'a'], + children: [node({ attack_result_id: 'a', child_index: 200 })], + }) + + expect(slots.map((slot) => [slot.position, slot.kind])).toEqual([ + [200, 'child'], + [201, 'unresolved'], + [202, 'unresolved'], + ]) + expect(slots[2]).toMatchObject({ kind: 'unresolved', attackResultId: 'a' }) + }) + + it('lays out a page where no reference resolves', () => { + const slots = childSlots({ offset: 0, referenceIds: ['x', 'y'], children: [] }) + + expect(slots.every((slot) => slot.kind === 'unresolved')).toBe(true) + expect(slots.map((slot) => slot.position)).toEqual([0, 1]) + }) +}) + +describe('mergeRootPage', () => { + it('appends roots to known groups and keeps the latest exact counts', () => { + const first = mergeRootPage([], { groups: [group('g1', ['r1'], 3)] } as ScenarioHierarchyProjection) + const merged = mergeRootPage(first, { + groups: [group('g1', ['r2'], 3), group('g2', ['r3'], 1)], + } as ScenarioHierarchyProjection) + + expect(merged.map((g) => [g.atomic_group_id, g.roots.map((r) => r.attack_result_id)])).toEqual([ + ['g1', ['r1', 'r2']], + ['g2', ['r3']], + ]) + expect(merged[0].counts.completed).toBe(3) + expect(first[0].roots).toHaveLength(1) + }) +}) + +describe('previewHasAllChildren', () => { + it('needs a request when the preview deferred or paged the children', () => { + expect(previewHasAllChildren(node({ child_attack_result_ids: ['a'], children_count: 1 }))).toBe(true) + expect(previewHasAllChildren(node({ child_attack_result_ids: ['a'], children_truncated: true }))).toBe(false) + expect(previewHasAllChildren(node({ child_attack_result_ids: ['a'], next_children_cursor: 'c' }))).toBe(false) + expect(previewHasAllChildren(node({ child_attack_result_ids: ['a'], children_count: 3 }))).toBe(false) + }) +}) diff --git a/frontend/src/components/Scenarios/scenarioHierarchy.ts b/frontend/src/components/Scenarios/scenarioHierarchy.ts new file mode 100644 index 0000000000..79202943f6 --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioHierarchy.ts @@ -0,0 +1,100 @@ +import type { + AttackResultRole, + ScenarioHierarchyGroup, + ScenarioHierarchyNode, + ScenarioHierarchyProjection, +} from '@/types' + +/** One saved child-reference position under a parent: a loaded child, or a reference that did not resolve. */ +export type HierarchyChildSlot = + | { readonly position: number; readonly kind: 'child'; readonly node: ScenarioHierarchyNode } + | { readonly position: number; readonly kind: 'unresolved'; readonly attackResultId: string } + +/** A loaded page of a parent's saved child references, in saved order. */ +export interface HierarchyChildPage { + readonly offset: number + readonly referenceIds: readonly string[] + readonly children: readonly ScenarioHierarchyNode[] +} + +/** Builds a child page from a node returned by either hierarchy endpoint. */ +export function childPageFromNode(node: ScenarioHierarchyNode): HierarchyChildPage { + return { + offset: node.child_offset, + referenceIds: node.child_attack_result_ids, + children: node.children, + } +} + +/** + * Lays out every saved reference position on a page. + * + * Resolved children are matched by their absolute `child_index`, not by id, because a duplicate + * reference to the same result is unresolved and still occupies its own position. + */ +export function childSlots(page: HierarchyChildPage): HierarchyChildSlot[] { + const childrenByPosition = new Map() + for (const child of page.children) { + if (child.child_index !== null && child.child_index !== undefined) { + childrenByPosition.set(child.child_index, child) + } + } + return page.referenceIds.map((attackResultId, i) => { + const position = page.offset + i + const node = childrenByPosition.get(position) + return node ? { position, kind: 'child', node } : { position, kind: 'unresolved', attackResultId } + }) +} + +/** + * Whether the tree preview already holds every saved child reference of a node, so expanding it + * needs no request. + */ +export function previewHasAllChildren(node: ScenarioHierarchyNode): boolean { + return ( + !node.children_truncated && + !node.child_references_truncated && + !node.next_children_cursor && + node.child_offset === 0 && + (node.children_count === null || + node.children_count === undefined || + node.children_count === node.child_attack_result_ids.length) + ) +} + +/** Whether a node has saved child references at all. */ +export function hasChildReferences(node: ScenarioHierarchyNode): boolean { + return node.child_attack_result_ids.length > 0 || (node.children_count ?? 0) > 0 +} + +/** + * Appends a page of roots to the groups already loaded. Groups keep their first-seen order; + * counts come from the latest page because they are exact run totals, not page totals. + */ +export function mergeRootPage( + groups: readonly ScenarioHierarchyGroup[], + page: ScenarioHierarchyProjection, +): ScenarioHierarchyGroup[] { + const merged = groups.map((group) => ({ ...group, roots: [...group.roots] })) + const indexById = new Map(merged.map((group, i) => [group.atomic_group_id, i])) + for (const group of page.groups) { + const i = indexById.get(group.atomic_group_id) + if (i === undefined) { + indexById.set(group.atomic_group_id, merged.length) + merged.push({ ...group, roots: [...group.roots] }) + } else { + merged[i] = { ...merged[i], counts: group.counts, roots: [...merged[i].roots, ...group.roots] } + } + } + return merged +} + +const ROLE_LABELS: Record = { + orchestration: 'Orchestration', + target_facing: 'Target-facing', + unknown: 'Unknown role', +} + +export function resultRoleLabel(role: AttackResultRole): string { + return ROLE_LABELS[role] ?? ROLE_LABELS.unknown +} diff --git a/frontend/src/hooks/useScenarioResultHierarchy.ts b/frontend/src/hooks/useScenarioResultHierarchy.ts new file mode 100644 index 0000000000..37c5ecaeec --- /dev/null +++ b/frontend/src/hooks/useScenarioResultHierarchy.ts @@ -0,0 +1,325 @@ +import { useCallback, useEffect, useRef, useState } from 'react' + +import { + childPageFromNode, + mergeRootPage, + previewHasAllChildren, + type HierarchyChildPage, +} from '@/components/Scenarios/scenarioHierarchy' +import { scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { + ScenarioHierarchyDiagnostics, + ScenarioHierarchyGroup, + ScenarioHierarchyNode, + ScenarioProgressCounts, + ScenarioRunState, +} from '@/types' + +export const HIERARCHY_MAX_NODES = 1000 +export const HIERARCHY_CHILD_PAGE_SIZE = 200 + +export interface HierarchyRootState { + readonly status: 'loading' | 'ready' | 'error' + /** Run state when the hierarchy was loaded; a run that is still going keeps adding results. */ + readonly runStatus: ScenarioRunState | null + readonly groups: ScenarioHierarchyGroup[] + readonly overallCounts: ScenarioProgressCounts | null + readonly diagnostics: ScenarioHierarchyDiagnostics | null + /** The preview left out deferred descendants, deep levels or results it could not place. */ + readonly truncated: boolean + readonly nextCursor: string | null + readonly loadingMore: boolean + readonly error: string | null + /** The root cursor no longer matches the saved run; paging has to restart. */ + readonly staleCursor: boolean +} + +export interface HierarchyChildrenState { + readonly pages: HierarchyChildPage[] + readonly childrenCount: number | null + readonly nextCursor: string | null + readonly loading: boolean + readonly error: string | null + /** The child cursor no longer matches the parent's saved references; paging has to restart. */ + readonly staleCursor: boolean +} + +const INITIAL_ROOTS: HierarchyRootState = { + status: 'loading', + runStatus: null, + groups: [], + overallCounts: null, + diagnostics: null, + truncated: false, + nextCursor: null, + loadingMore: false, + error: null, + staleCursor: false, +} + +const EMPTY_CHILDREN: HierarchyChildrenState = { + pages: [], + childrenCount: null, + nextCursor: null, + loading: false, + error: null, + staleCursor: false, +} + +export interface UseScenarioResultHierarchyResult { + readonly roots: HierarchyRootState + readonly children: ReadonlyMap + readonly expanded: ReadonlySet + /** Expands or collapses a node; expanding shows its children, loading them if needed. */ + readonly toggleNode: (key: string, node: ScenarioHierarchyNode) => void + readonly reloadRoots: () => void + readonly loadMoreRoots: () => void + /** Shows a node's children, from the preview when it is complete and otherwise from the API. */ + readonly openChildren: (key: string, node: ScenarioHierarchyNode) => void + readonly loadMoreChildren: (key: string, node: ScenarioHierarchyNode) => void + readonly restartChildren: (key: string, node: ScenarioHierarchyNode) => void +} + +/** State is not reset when `scenarioResultId` changes; mount a new instance per run (for example with a `key`). */ +export function useScenarioResultHierarchy(scenarioResultId: string): UseScenarioResultHierarchyResult { + const [roots, setRoots] = useState(INITIAL_ROOTS) + const [children, setChildren] = useState>(new Map()) + const [expanded, setExpanded] = useState>(new Set()) + const [rootEpoch, setRootEpoch] = useState(0) + const controllers = useRef(new Set()) + /** The child request in flight for each node, so a restart can drop a page that is still loading. */ + const childControllers = useRef(new Map()) + + const track = useCallback((): AbortController => { + const controller = new AbortController() + controllers.current.add(controller) + return controller + }, []) + + const release = useCallback((controller: AbortController) => { + controllers.current.delete(controller) + }, []) + + useEffect(() => { + const active = controllers.current + return () => { + for (const controller of active) { + controller.abort() + } + active.clear() + } + }, []) + + useEffect(() => { + const controller = track() + scenariosApi + .getRunHierarchy(scenarioResultId, { max_nodes: HIERARCHY_MAX_NODES }, controller.signal) + .then((page) => { + setRoots({ + status: 'ready', + runStatus: page.status, + groups: mergeRootPage([], page), + overallCounts: page.overall_counts, + diagnostics: page.diagnostics, + truncated: page.truncated, + nextCursor: page.has_more ? page.next_cursor ?? null : null, + loadingMore: false, + error: null, + staleCursor: false, + }) + }) + .catch((err: unknown) => { + if (controller.signal.aborted) { + return + } + setRoots({ ...INITIAL_ROOTS, status: 'error', error: toApiError(err).detail }) + }) + .finally(() => release(controller)) + return () => controller.abort() + }, [scenarioResultId, rootEpoch, track, release]) + + const reloadRoots = useCallback(() => { + // Late responses from before the reload must not land in the fresh tree. + for (const controller of controllers.current) { + controller.abort() + } + controllers.current.clear() + childControllers.current.clear() + setRoots(INITIAL_ROOTS) + setChildren(new Map()) + setExpanded(new Set()) + setRootEpoch((epoch) => epoch + 1) + }, []) + + const loadMoreRoots = useCallback(() => { + const cursor = roots.nextCursor + if (!cursor || roots.loadingMore) { + return + } + const controller = track() + setRoots((state) => ({ ...state, loadingMore: true, error: null, staleCursor: false })) + scenariosApi + .getRunHierarchy(scenarioResultId, { max_nodes: HIERARCHY_MAX_NODES, cursor }, controller.signal) + .then((page) => { + setRoots((state) => ({ + ...state, + runStatus: page.status, + groups: mergeRootPage(state.groups, page), + overallCounts: page.overall_counts, + diagnostics: page.diagnostics, + truncated: state.truncated || page.truncated, + nextCursor: page.has_more ? page.next_cursor ?? null : null, + loadingMore: false, + })) + }) + .catch((err: unknown) => { + if (controller.signal.aborted) { + return + } + const error = toApiError(err) + setRoots((state) => ({ + ...state, + loadingMore: false, + error: error.detail, + staleCursor: error.status === 400, + })) + }) + .finally(() => release(controller)) + }, [roots.nextCursor, roots.loadingMore, scenarioResultId, track, release]) + + const fetchChildren = useCallback( + (key: string, node: ScenarioHierarchyNode, cursor: string | null) => { + childControllers.current.get(key)?.abort() + const controller = track() + childControllers.current.set(key, controller) + setChildren((map) => { + const next = new Map(map) + const current = cursor ? map.get(key) ?? EMPTY_CHILDREN : EMPTY_CHILDREN + next.set(key, { ...current, loading: true, error: null, staleCursor: false }) + return next + }) + scenariosApi + .getRunHierarchyNode( + scenarioResultId, + node.attack_result_id, + cursor + ? { limit: HIERARCHY_CHILD_PAGE_SIZE, children_cursor: cursor } + : { limit: HIERARCHY_CHILD_PAGE_SIZE }, + controller.signal, + ) + .then((page) => { + setChildren((map) => { + const next = new Map(map) + const current = map.get(key) ?? EMPTY_CHILDREN + next.set(key, { + pages: [...current.pages, childPageFromNode(page)], + childrenCount: page.children_count ?? null, + nextCursor: page.next_children_cursor ?? null, + loading: false, + error: null, + staleCursor: false, + }) + return next + }) + }) + .catch((err: unknown) => { + if (controller.signal.aborted) { + return + } + const error = toApiError(err) + setChildren((map) => { + const next = new Map(map) + const current = map.get(key) ?? EMPTY_CHILDREN + next.set(key, { + ...current, + loading: false, + error: error.detail, + staleCursor: cursor !== null && error.status === 400, + }) + return next + }) + }) + .finally(() => { + release(controller) + if (childControllers.current.get(key) === controller) { + childControllers.current.delete(key) + } + }) + }, + [scenarioResultId, track, release], + ) + + const openChildren = useCallback( + (key: string, node: ScenarioHierarchyNode) => { + const existing = children.get(key) + if (existing && (existing.pages.length > 0 || existing.loading)) { + return + } + if (previewHasAllChildren(node)) { + setChildren((map) => { + const next = new Map(map) + next.set(key, { + ...EMPTY_CHILDREN, + pages: [childPageFromNode(node)], + childrenCount: node.children_count ?? node.child_attack_result_ids.length, + }) + return next + }) + return + } + fetchChildren(key, node, null) + }, + [children, fetchChildren], + ) + + const loadMoreChildren = useCallback( + (key: string, node: ScenarioHierarchyNode) => { + const state = children.get(key) + if (!state?.nextCursor || state.loading) { + return + } + fetchChildren(key, node, state.nextCursor) + }, + [children, fetchChildren], + ) + + const restartChildren = useCallback( + (key: string, node: ScenarioHierarchyNode) => { + // Descendants loaded under the old child pages may no longer match; forget them too. + const prefix = `${key}/` + setChildren((map) => new Map([...map].filter(([childKey]) => !childKey.startsWith(prefix)))) + setExpanded((keys) => new Set([...keys].filter((childKey) => !childKey.startsWith(prefix)))) + fetchChildren(key, node, null) + }, + [fetchChildren], + ) + + const toggleNode = useCallback( + (key: string, node: ScenarioHierarchyNode) => { + if (expanded.has(key)) { + setExpanded((keys) => { + const next = new Set(keys) + next.delete(key) + return next + }) + return + } + setExpanded((keys) => new Set(keys).add(key)) + openChildren(key, node) + }, + [expanded, openChildren], + ) + + return { + roots, + children, + expanded, + toggleNode, + reloadRoots, + loadMoreRoots, + openChildren, + loadMoreChildren, + restartChildren, + } +} diff --git a/frontend/src/services/api.test.ts b/frontend/src/services/api.test.ts index b7d5a0e384..447b20d963 100644 --- a/frontend/src/services/api.test.ts +++ b/frontend/src/services/api.test.ts @@ -1121,6 +1121,52 @@ describe("api service", () => { }); }); + it("loads a hierarchy root page with its root cursor", async () => { + const projection = { + scenario_result_id: "sr-1", + groups: [], + has_more: true, + next_cursor: "roots-2", + }; + (apiClient.get as jest.Mock).mockResolvedValueOnce({ data: projection }); + const controller = new AbortController(); + + const result = await scenariosApi.getRunHierarchy( + "sr/1", + { max_nodes: 200, cursor: "roots-1" }, + controller.signal, + ); + + expect(apiClient.get).toHaveBeenCalledWith("/scenarios/runs/sr%2F1/hierarchy", { + params: { max_nodes: 200, cursor: "roots-1" }, + signal: controller.signal, + }); + expect(result).toBe(projection); + }); + + it("loads a node's children with the node's own children cursor", async () => { + const node = { + attack_result_id: "ar/1", + child_offset: 200, + next_children_cursor: "children-3", + }; + (apiClient.get as jest.Mock).mockResolvedValueOnce({ data: node }); + const controller = new AbortController(); + + const result = await scenariosApi.getRunHierarchyNode( + "sr/1", + "ar/1", + { limit: 200, children_cursor: "children-2" }, + controller.signal, + ); + + expect(apiClient.get).toHaveBeenCalledWith( + "/scenarios/runs/sr%2F1/hierarchy/nodes/ar%2F1", + { params: { limit: 200, children_cursor: "children-2" }, signal: controller.signal }, + ); + expect(result).toBe(node); + }); + it("cancels a scenario run by id", async () => { const mockResponse = { data: { diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 0d2ff8f984..217e19b88d 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -66,6 +66,8 @@ import type { ScenarioRunSummary, ScenarioRunListResponse, ScenarioRunProgress, + ScenarioHierarchyNode, + ScenarioHierarchyProjection, PairedOutcomeStatistics, ScenarioQueueSnapshot, ScenarioRunState, @@ -714,6 +716,33 @@ export const scenariosApi = { return response.data }, + /** Loads one page of hierarchy roots. `cursor` continues roots only, never a node's children. */ + getRunHierarchy: async ( + scenarioResultId: string, + params?: { max_nodes?: number; cursor?: string }, + signal?: AbortSignal, + ): Promise => { + const response = await apiClient.get( + `/scenarios/runs/${encodeURIComponent(scenarioResultId)}/hierarchy`, + { params, signal }, + ) + return response.data + }, + + /** Loads one page of a node's saved children, continued with that node's `next_children_cursor`. */ + getRunHierarchyNode: async ( + scenarioResultId: string, + attackResultId: string, + params?: { limit?: number; children_cursor?: string }, + signal?: AbortSignal, + ): Promise => { + const response = await apiClient.get( + `/scenarios/runs/${encodeURIComponent(scenarioResultId)}/hierarchy/nodes/${encodeURIComponent(attackResultId)}`, + { params, signal }, + ) + return response.data + }, + getPairedOutcomes: async ( scenarioResultId: string, params: { baseline_group_id: string; attack_group_id: string }, diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 24c5f13cf6..9e308a2cf0 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -2098,6 +2098,106 @@ export interface ScenarioAtomicGroupProgress extends ScenarioProgressCounts { technique_details?: ScenarioAttackTechniqueDetails | null } +/** What a saved attack result represents; not proof that the objective target was called. */ +export type AttackResultRole = 'target_facing' | 'orchestration' | 'unknown' + +export interface ScenarioOrchestrationParentSummary { + attack_result_id: string + result_role: AttackResultRole + outcome: AttackOutcome + outcome_reason?: string | null + attempt_index?: number | null + execution_time_ms: number + executed_turns: number + completion_policy?: string | null + child_attack_result_ids: string[] + child_references_truncated: boolean + error_type?: string | null + error_message?: string | null + timestamp: string + total_retries: number + retries: RetryEvent[] + retries_truncated: boolean +} + +export interface ScenarioHierarchyNode { + attack_result_id: string + conversation_id: string + parent_attack_result_id?: string | null + /** Absolute position under the immediate parent. */ + child_index?: number | null + result_role: AttackResultRole + attempt_index?: number | null + atomic_attack_name: string + atomic_group_id: string + seed_group_id: string + objective: string + outcome: AttackOutcome + outcome_reason?: string | null + execution_time_ms: number + executed_turns: number + timestamp: string + score?: ScenarioProgressScore | null + error_type?: string | null + error_message?: string | null + total_retries: number + retries: RetryEvent[] + retries_truncated: boolean + child_attack_result_ids: string[] + child_references_truncated: boolean + children: ScenarioHierarchyNode[] + children_truncated: boolean + /** Full saved child-reference count; null when not known. */ + children_count?: number | null + /** Position of the first child-reference on this page. */ + child_offset: number + /** Continuation for this node's children only; never for roots. */ + next_children_cursor?: string | null + unresolved_child_attack_result_ids: string[] + unresolved_child_references_truncated: boolean + conversations: ConversationSummary[] + conversations_count?: number | null + conversations_truncated: boolean + orchestration?: ScenarioOrchestrationParentSummary | null +} + +export interface ScenarioHierarchyGroup { + atomic_group_id: string + atomic_attack_name: string + display_group: string + technique?: ScenarioAttackTechniqueDetails | null + roots: ScenarioHierarchyNode[] + counts: ScenarioProgressCounts +} + +export interface ScenarioHierarchyDiagnostics { + failed_attacks: AttackErrorSummary[] + failed_attacks_count: number + failed_attacks_truncated: boolean + attack_retries: AttackRetrySummary[] + attack_retries_count: number + attack_retries_truncated: boolean + total_retries: number + rootless_result_ids: string[] + rootless_results_count: number + rootless_results_truncated: boolean +} + +export interface ScenarioHierarchyProjection { + scenario_result_id: string + scenario_name: string + scenario_version: number + status: ScenarioRunState + groups: ScenarioHierarchyGroup[] + /** Exact run totals; independent of which roots or children are loaded. */ + overall_counts: ScenarioProgressCounts + diagnostics: ScenarioHierarchyDiagnostics + /** Root continuation only. */ + has_more: boolean + next_cursor?: string | null + truncated: boolean +} + export interface ScenarioObjectiveScorerMetrics { accuracy: number accuracy_standard_error?: number | null diff --git a/pyrit/backend/README.md b/pyrit/backend/README.md index 85bb9c9765..f7a65fed4d 100644 --- a/pyrit/backend/README.md +++ b/pyrit/backend/README.md @@ -75,7 +75,8 @@ and transcript reads, including hierarchy node expansion. ### Scenario result hierarchy These read-only APIs provide the backend foundation for compound-attack browsing. -The hierarchy GUI is separate work; the current scanner still shows flat executions. +The Scenario run page's "Result hierarchy" section browses them read-only; saved +conversation evidence for each node is separate follow-up work. `GET /api/scenarios/runs/{scenario_result_id}/hierarchy?max_nodes=1000` returns a bounded tree preview grouped by atomic attack group. Roots are saved results with From cae01f280aeca42c1f0d25492a606e78d1408195 Mon Sep 17 00:00:00 2001 From: Fazeel Usmani Date: Thu, 29 Oct 2026 13:53:34 +0530 Subject: [PATCH 2/4] FIX: Address hierarchy review on cycles, resumed runs, objectives and layout (#3602) - Cycles across fetched pages: each row now knows the results above it, and a fetched child that is one of them stays in its position as an unresolved reference instead of an expandable row. A single node response can't see the path, so the check lives in the tree. - Resumed runs: the page passes the current run status in. A run that is active again, or whose status changed since the hierarchy loaded, shows the snapshot note and Reload hierarchy. - Details objective: the hierarchy node's objective is used whenever its result ID matches the selected attempt, so a progress row for a seed missing from the plan no longer hides it. - Layout: each group is a table with Result / objective, Result type, Attack outcome, Duration, Retries and Actions. Nesting indents only the first column, so outcomes and numbers stay aligned. Outcomes read Success, Failure, Error or Undetermined, and a note explains that rows are saved attack results and numbered children are a compound attack's component results, not messages or retries. Tests: paged P -> A -> B -> A cycle, failed-to-resumed run (component and through the page's Resume action), objective with the progress row already loaded, and column alignment. Playwright suite updated for the table. --- frontend/e2e/scenario-hierarchy.spec.ts | 17 +- .../AttackResults/attackAttemptFormatting.ts | 5 + .../ScenarioResultHierarchy.styles.ts | 48 +-- .../ScenarioResultHierarchy.test.tsx | 120 +++++- .../Scenarios/ScenarioResultHierarchy.tsx | 366 ++++++++++-------- .../Scenarios/ScenarioRunPage.test.tsx | 118 ++++-- .../components/Scenarios/ScenarioRunPage.tsx | 6 +- 7 files changed, 453 insertions(+), 227 deletions(-) diff --git a/frontend/e2e/scenario-hierarchy.spec.ts b/frontend/e2e/scenario-hierarchy.spec.ts index 1c25025980..2f96db0bd3 100644 --- a/frontend/e2e/scenario-hierarchy.spec.ts +++ b/frontend/e2e/scenario-hierarchy.spec.ts @@ -22,10 +22,13 @@ test('browses a compound result through the real API, page by page, with stable await section.getByRole('button', { name: 'Expand children of sequential: Sequential parent' }).click() expect((await firstPage).status()).toBe(200) - const children = section.getByRole('list', { name: 'Children of sequential: Sequential parent' }) - await expect(children.getByText('Step 1', { exact: true })).toBeVisible() - await expect(children.locator('li', { hasText: '#2' }).first()).toContainText('Unresolved reference') - await expect(children.locator('li', { hasText: '#4' }).first()).toContainText('Unresolved reference') + const table = section.getByRole('table', { name: 'Sequential saved results' }) + const row = (text: string) => table.getByRole('row').filter({ hasText: text }).first() + await expect(table.getByText('Step 1', { exact: true })).toBeVisible() + await expect(row('#2')).toContainText('Unresolved reference') + await expect(row('#4')).toContainText('Unresolved reference') + // Children keep the parent's columns: outcome text sits under "Attack outcome", not nested in the row. + await expect(row('Step 5').getByRole('cell').nth(2)).toHaveText('Success') await expect(section.getByText('Showing 200 of 520 saved children')).toBeVisible() for (const shown of [400, 520]) { @@ -35,10 +38,10 @@ test('browses a compound result through the real API, page by page, with stable await expect(section.getByText(`Showing ${shown} of 520 saved children`)).toBeVisible() } await expect(section.getByRole('button', { name: 'Load more children' })).toHaveCount(0) - await expect(children.getByText('Step 520', { exact: true })).toBeVisible() + await expect(table.getByText('Step 520', { exact: true })).toBeVisible() - await children.getByRole('button', { name: 'Expand children of #1 sequential: Step 1' }).click() - await expect(children.getByText('Nested step 2', { exact: true })).toBeVisible() + await table.getByRole('button', { name: 'Expand children of #1 sequential: Step 1' }).click() + await expect(table.getByText('Nested step 2', { exact: true })).toBeVisible() expect(await totals.textContent()).toBe(before) await section.getByRole('button', { name: 'Open details for single: Single-turn attempt' }).click() diff --git a/frontend/src/components/AttackResults/attackAttemptFormatting.ts b/frontend/src/components/AttackResults/attackAttemptFormatting.ts index 2287a26138..21fabc443c 100644 --- a/frontend/src/components/AttackResults/attackAttemptFormatting.ts +++ b/frontend/src/components/AttackResults/attackAttemptFormatting.ts @@ -19,6 +19,11 @@ export function formatAttackSuccess(outcome: ScenarioProgressResult['outcome']): return outcome.replace(/^\w/, (letter) => letter.toUpperCase()) } +/** The attack's outcome as a word, for places headed "Attack outcome" rather than "Attack success". */ +export function formatAttackOutcome(outcome: ScenarioProgressResult['outcome']): string { + return outcome.replace(/^\w/, (letter) => letter.toUpperCase()) +} + export function formatScore(score: ScenarioProgressResult['score']): string { if (!score || score.status === 'undetermined') { return 'Undetermined' diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts b/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts index c02e83a85b..25af9c77d4 100644 --- a/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts @@ -1,6 +1,6 @@ import { makeStyles, tokens } from '@fluentui/react-components' -import { NARROW_VIEWPORT_QUERY, mobileTouchTarget } from '@/styles/touchTargets' +import { mobileTouchTarget } from '@/styles/touchTargets' export const useScenarioResultHierarchyStyles = makeStyles({ root: { @@ -23,31 +23,37 @@ export const useScenarioResultHierarchyStyles = makeStyles({ gap: tokens.spacingVerticalXS, paddingTop: tokens.spacingVerticalS, borderTop: `${tokens.strokeWidthThin} solid ${tokens.colorNeutralStroke2}`, + minWidth: 0, + }, + // Wide tables scroll inside their own box instead of widening the page on narrow screens. + tableScroll: { + width: '100%', + overflowX: 'auto', + border: `1px solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusLarge, + backgroundColor: tokens.colorNeutralBackground1, + }, + table: { + minWidth: '52rem', + tableLayout: 'auto', }, - tree: { - listStyleType: 'none', - margin: 0, - paddingLeft: 0, - }, - subtree: { - listStyleType: 'none', - margin: 0, - paddingLeft: tokens.spacingHorizontalXXL, - borderLeft: `${tokens.strokeWidthThin} solid ${tokens.colorNeutralStroke2}`, - [NARROW_VIEWPORT_QUERY]: { - paddingLeft: tokens.spacingHorizontalM, - }, - }, - row: { + resultCell: { display: 'flex', alignItems: 'center', - flexWrap: 'wrap', gap: tokens.spacingHorizontalS, - paddingTop: tokens.spacingVerticalXXS, - paddingBottom: tokens.spacingVerticalXXS, - minWidth: 0, + minWidth: '18rem', + }, + outcomeCell: { + display: 'flex', + flexDirection: 'column', + alignItems: 'flex-start', + gap: tokens.spacingVerticalXXS, + }, + nowrap: { + whiteSpace: 'nowrap', }, expandButton: { + flexShrink: 0, ...mobileTouchTarget, }, expandSpacer: { @@ -61,8 +67,6 @@ export const useScenarioResultHierarchyStyles = makeStyles({ }, objective: { minWidth: 0, - flexBasis: '16rem', - flexGrow: 1, overflowWrap: 'anywhere', }, unresolved: { diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx b/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx index edfe0498c5..9f89f196ab 100644 --- a/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx @@ -3,7 +3,12 @@ import userEvent from '@testing-library/user-event' import { FluentProvider, webLightTheme } from '@fluentui/react-components' import { scenariosApi } from '@/services/api' -import type { ScenarioHierarchyNode, ScenarioHierarchyProjection, ScenarioProgressCounts } from '@/types' +import type { + ScenarioHierarchyNode, + ScenarioHierarchyProjection, + ScenarioProgressCounts, + ScenarioRunState, +} from '@/types' import ScenarioResultHierarchy from './ScenarioResultHierarchy' @@ -99,15 +104,20 @@ const PARENT = node('parent', { children_count: 3, }) -function mount(onOpenResult = jest.fn()) { - render( +function mount(onOpenResult = jest.fn(), runStatus: ScenarioRunState | null = null) { + const view = (status: ScenarioRunState | null) => ( - - , + + ) - return onOpenResult + const { rerender } = render(view(runStatus)) + return { onOpenResult, setRunStatus: (status: ScenarioRunState) => rerender(view(status)) } } +const resultsTable = () => screen.getByRole('table', { name: 'Sequential saved results' }) +/** The row whose first cell shows this text. */ +const rowWith = (text: string) => within(resultsTable()).getByText(text).closest('tr') as HTMLElement + const totalsText = () => screen.getByLabelText('Run totals').textContent beforeEach(() => { @@ -140,24 +150,23 @@ test('loads children by position, keeps unresolved references visible, and pages mount() await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) - const children = await screen.findByRole('list', { name: 'Children of attack parent: objective parent' }) + await screen.findByText('objective c1') expect(getNode).toHaveBeenLastCalledWith('run', 'parent', { limit: 200 }, expect.any(AbortSignal)) - expect(within(children).getByText('objective c1')).toBeInTheDocument() - expect(within(children).getByText('Unresolved reference: missing')).toBeInTheDocument() - expect(within(children).getByText('#2')).toBeInTheDocument() + expect(within(rowWith('objective c1')).getByText('#1')).toBeInTheDocument() + expect(within(rowWith('Unresolved reference: missing')).getByText('#2')).toBeInTheDocument() expect(screen.getByText('Showing 2 of 3 saved children')).toBeInTheDocument() await user.click(screen.getByRole('button', { name: 'Load more children' })) - expect(await within(children).findByText('objective c3')).toBeInTheDocument() + expect(await screen.findByText('objective c3')).toBeInTheDocument() expect(getNode).toHaveBeenLastCalledWith( 'run', 'parent', { limit: 200, children_cursor: 'children-2' }, expect.any(AbortSignal), ) - expect(within(children).getByText('#3')).toBeInTheDocument() + expect(within(rowWith('objective c3')).getByText('#3')).toBeInTheDocument() expect(screen.queryByRole('button', { name: 'Load more children' })).not.toBeInTheDocument() expect(getRoots).toHaveBeenCalledTimes(1) }) @@ -290,7 +299,7 @@ test('shows empty, truncated and rootless states without inventing roots', async test('opens details for a node with the clicked button as the focus target', async () => { const user = userEvent.setup() getRoots.mockResolvedValue(projection([node('r1')])) - const onOpenResult = mount() + const { onOpenResult } = mount() const button = await screen.findByRole('button', { name: 'Open details for attack r1: objective r1' }) await user.click(button) @@ -356,3 +365,88 @@ test('notes that a run still in progress is a snapshot and can be reloaded', asy expect(await screen.findByText('objective r2')).toBeInTheDocument() expect(screen.queryByText(/This run is still in progress/)).not.toBeInTheDocument() }) + +test('a reference back to an ancestor stays unresolved even when the cycle arrives across fetched pages', async () => { + const user = userEvent.setup() + // Saved links P -> A -> B -> A; each node's children come from its own request. + const deferred = (id: string, childId: string, extra: Partial = {}) => + node(id, { + result_role: 'orchestration', + child_attack_result_ids: [childId], + children_truncated: true, + children_count: 1, + ...extra, + }) + const page = (id: string, child: ScenarioHierarchyNode) => + node(id, { result_role: 'orchestration', child_attack_result_ids: [child.attack_result_id], children: [child], children_count: 1 }) + getRoots.mockResolvedValue(projection([deferred('P', 'A')])) + getNode + .mockResolvedValueOnce(page('P', deferred('A', 'B', { child_index: 0 }))) + .mockResolvedValueOnce(page('A', deferred('B', 'A', { child_index: 0 }))) + .mockResolvedValueOnce(page('B', deferred('A', 'B', { child_index: 0 }))) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack P: objective P' })) + await user.click(await screen.findByRole('button', { name: 'Expand children of #1 attack A: objective A' })) + await user.click(await screen.findByRole('button', { name: 'Expand children of #1 attack B: objective B' })) + + const cycle = await screen.findByText(/Unresolved reference: A \(it points back to a result above it\)/) + expect(within(cycle.closest('tr') as HTMLElement).getByText('#1')).toBeInTheDocument() + // Only the first A can be expanded; the repeat has no expand control. + expect(screen.getAllByRole('button', { name: /children of #1 attack A:/ })).toHaveLength(1) + expect(getNode).toHaveBeenCalledTimes(3) +}) + +test('a failed run that is resumed offers to reload the hierarchy it loaded before', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([node('r1')], { status: 'FAILED' })) + .mockResolvedValueOnce(projection([node('r1'), node('r2')], { status: 'IN_PROGRESS' })) + const { setRunStatus } = mount(jest.fn(), 'FAILED') + + await screen.findByText('objective r1') + expect(screen.queryByRole('button', { name: 'Reload hierarchy' })).not.toBeInTheDocument() + + setRunStatus('QUEUED') + expect(screen.getByText(/This run is still in progress/)).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Reload hierarchy' })) + expect(await screen.findByText('objective r2')).toBeInTheDocument() + + // Finishing after the reload is a change too; the snapshot is no longer current. + setRunStatus('COMPLETED') + expect(screen.getByText("This run's status changed since this hierarchy loaded.")).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Reload hierarchy' })).toBeInTheDocument() +}) + +test('shows results in aligned columns with the attack outcome spelled out', async () => { + const user = userEvent.setup() + getRoots.mockResolvedValue( + projection([ + node('parent', { + result_role: 'orchestration', + outcome: 'success', + child_attack_result_ids: ['c1'], + children: [node('c1', { child_index: 0, outcome: 'error', error_message: 'Target timed out', total_retries: 2 })], + children_count: 1, + }), + ]), + ) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + await screen.findByText('objective c1') + + expect(within(resultsTable()).getAllByRole('columnheader').map((header) => header.textContent)).toEqual([ + 'Result / objective', + 'Result type', + 'Attack outcome', + 'Duration', + 'Retries', + 'Actions', + ]) + // A child row has the same cells in the same columns as its parent; only the first column is indented. + const cells = (text: string) => within(rowWith(text)).getAllByRole('cell').map((cell) => cell.textContent) + expect(cells('objective parent')).toEqual(['objective parent', 'Orchestration', 'Success', '1s', '0', 'Details']) + expect(cells('objective c1')).toEqual(['#1objective c1', 'Target-facing', 'ErrorTarget timed out', '1s', '2', 'Details']) + expect(screen.getByText(/Each row is one saved attack result/)).toHaveTextContent('not chat messages or retries') +}) diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx b/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx index 810803c483..00158badff 100644 --- a/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx @@ -1,15 +1,23 @@ +import type { CSSProperties } from 'react' + import { Badge, Button, MessageBar, MessageBarActions, MessageBarBody, + Table, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, Text, } from '@fluentui/react-components' import { ChevronDownRegular, ChevronRightRegular } from '@fluentui/react-icons' import OutcomeBadge from '@/components/OutcomeBadge' -import { formatAttackSuccess, formatDuration } from '@/components/AttackResults/attackAttemptFormatting' +import { formatAttackOutcome, formatDuration } from '@/components/AttackResults/attackAttemptFormatting' import { useScenarioResultHierarchy, type HierarchyChildrenState, @@ -22,14 +30,20 @@ import { useScenarioResultHierarchyStyles } from './ScenarioResultHierarchy.styl interface ScenarioResultHierarchyProps { readonly scenarioResultId: string + /** The run's current state, which can change (for example on resume) after the hierarchy loaded. */ + readonly runStatus?: ScenarioRunState | null readonly onOpenResult: (node: ScenarioHierarchyNode, trigger: HTMLElement) => void } /** Read-only browser for a run's saved result lineage: orchestration parents and their ordered children. */ -export default function ScenarioResultHierarchy({ scenarioResultId, onOpenResult }: ScenarioResultHierarchyProps) { +export default function ScenarioResultHierarchy({ + scenarioResultId, + runStatus = null, + onOpenResult, +}: ScenarioResultHierarchyProps) { const styles = useScenarioResultHierarchyStyles() const hierarchy = useScenarioResultHierarchy(scenarioResultId) - const { roots, expanded, toggleNode: toggle } = hierarchy + const { roots } = hierarchy // Plain status text, not a Spinner: a progressbar role here would be read as run progress. if (roots.status === 'loading') { @@ -53,7 +67,9 @@ export default function ScenarioResultHierarchy({ scenarioResultId, onOpenResult const rootless = roots.diagnostics?.rootless_results_count ?? 0 const rootlessSample = roots.diagnostics?.rootless_result_ids ?? [] - const runActive = roots.runStatus !== null && ACTIVE_RUN_STATES.has(roots.runStatus) + const currentStatus = runStatus ?? roots.runStatus + const runActive = currentStatus !== null && ACTIVE_RUN_STATES.has(currentStatus) + const statusChanged = runStatus !== null && roots.runStatus !== null && runStatus !== roots.runStatus return (
    @@ -61,10 +77,12 @@ export default function ScenarioResultHierarchy({ scenarioResultId, onOpenResult Totals cover the whole run as reported by the server; they are not counted from the rows shown here. - {runActive && ( + {(runActive || statusChanged) && (
    - This run is still in progress, so this hierarchy shows the results saved when it loaded. + {runActive + ? 'This run is still in progress, so this hierarchy shows the results saved when it loaded.' + : "This run's status changed since this hierarchy loaded."}
    + + + + {resultRoleLabel(node.result_role)} + + + +
    + + {node.outcome === 'error' && (node.error_message || node.error_type) && ( + + {node.error_message ?? node.error_type} + + )} +
    +     + {formatDuration(node.execution_time_ms)} + {node.total_retries} + -
    + className={styles.touchTarget} + aria-label={`Open details for ${label}`} + onClick={(event) => onOpenResult(node, event.currentTarget)} + > + Details + + + {expandable && isExpanded && ( - )} - + ) } -interface ChildListProps { - readonly panelId: string +interface ChildRowsProps { readonly parentKey: string readonly parent: ScenarioHierarchyNode + readonly depth: number + /** The parent and everything above it. */ + readonly ancestors: ReadonlySet readonly state: HierarchyChildrenState | undefined - readonly expanded: ReadonlySet readonly hierarchy: UseScenarioResultHierarchyResult - readonly onToggle: (key: string, node: ScenarioHierarchyNode) => void readonly onOpenResult: (node: ScenarioHierarchyNode, trigger: HTMLElement) => void } -function ChildList({ - panelId, - parentKey, - parent, - state, - expanded, - hierarchy, - onToggle, - onOpenResult, -}: ChildListProps) { +function ChildRows({ parentKey, parent, depth, ancestors, state, hierarchy, onOpenResult }: ChildRowsProps) { const styles = useScenarioResultHierarchyStyles() const pages = state?.pages ?? [] const slots = pages.flatMap(childSlots) @@ -250,75 +293,88 @@ function ChildList({ const total = state?.childrenCount ?? null return ( -
    -
      - {slots.map((slot) => - slot.kind === 'child' ? ( - + {slots.map((slot) => { + // A fetched page can't know the path to it, so a reference back up the path is caught here. + const cycle = slot.kind === 'child' && ancestors.has(slot.node.attack_result_id) + if (slot.kind === 'child' && !cycle) { + return ( + - ) : ( -
    • -
      + ) + } + const id = slot.kind === 'child' ? slot.node.attack_result_id : slot.attackResultId + return ( + + +
      #{slot.position + 1} - Unresolved reference: {slot.attackResultId} + + Unresolved reference: {id} + {cycle && ' (it points back to a result above it)'} +
      -
      • - ), - )} -
    -
    - {state?.loading && ( - - Loading children... - - )} - {!state?.loading && pages.length > 0 && ( - - {total !== null ? `Showing ${loaded} of ${total} saved children` : `Showing ${loaded} saved children`} - - )} - {state?.staleCursor ? ( - - This result's saved children changed since they were loaded. - - - - - ) : state?.error ? ( - - Could not load children: {state.error} - - + + + ) : state?.error ? ( + + Could not load children: {state.error} + + + + + ) : null} + {state?.nextCursor && !state.loading && !state.staleCursor && !state.error && ( + - - - ) : null} - {state?.nextCursor && !state.loading && !state.staleCursor && !state.error && ( - - )} -
    -
    + )} + + + + ) } @@ -335,7 +391,15 @@ function RunTotals({ counts }: { readonly counts: ScenarioProgressCounts }) { } const ACTIVE_RUN_STATES: ReadonlySet = new Set(['CREATED', 'QUEUED', 'IN_PROGRESS']) +const NO_ANCESTORS: ReadonlySet = new Set() +const COLUMN_COUNT = 6 const LABEL_OBJECTIVE_LENGTH = 60 +const INDENT_STEP = '1.5rem' + +/** Nesting depth, shown only in the first column so the other columns stay aligned. */ +function indent(depth: number): CSSProperties { + return { paddingInlineStart: `calc(${depth} * ${INDENT_STEP})` } +} /** Accessible name for a result; sibling results usually share an attack name, so add position and objective. */ function nodeLabel(node: ScenarioHierarchyNode, position: number | null): string { diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx index b3c2947f48..374d84c0ad 100644 --- a/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx @@ -14,6 +14,7 @@ import { useScenarioQueue } from '@/hooks/useScenarioQueue' import { scenariosApi } from '@/services/api' import type { ScenarioComponentIdentity, + ScenarioHierarchyNode, ScenarioProgressHeader, ScenarioProgressSummary, ScenarioProgressResult, @@ -154,6 +155,47 @@ const ATTEMPT: ScenarioProgressResult = { }, } +function hierarchyNode(overrides: Partial = {}): ScenarioHierarchyNode { + return { + attack_result_id: 'hierarchy-only', + conversation_id: 'conversation-h', + result_role: 'orchestration', + atomic_attack_name: 'sequential-parent', + atomic_group_id: 'atomic-1', + seed_group_id: 'seed-unplanned', + objective: 'Objective carried by the hierarchy node', + outcome: 'failure', + execution_time_ms: 1200, + executed_turns: 0, + timestamp: '2026-01-01T00:00:00Z', + total_retries: 0, + retries: [], + retries_truncated: false, + child_attack_result_ids: [], + child_references_truncated: false, + children: [], + children_truncated: false, + child_offset: 0, + unresolved_child_attack_result_ids: [], + unresolved_child_references_truncated: false, + conversations: [], + conversations_truncated: false, + ...overrides, + } +} + +function mockHierarchy(roots: ScenarioHierarchyNode[], status: ScenarioRunState = 'COMPLETED'): void { + const counts = { completed: 1, planned: 1, succeeded: 0, success_percentage: 0, errors: 0, retries: 0 } + ;(scenariosApi.getRunHierarchy as jest.Mock).mockResolvedValueOnce({ + status, + groups: [{ atomic_group_id: 'atomic-1', atomic_attack_name: 'a', display_group: 'Group', roots, counts }], + overall_counts: counts, + diagnostics: { rootless_results_count: 0 }, + truncated: false, + has_more: false, + }) +} + const SUMMARY: ScenarioProgressSummary = { overall: { completed: 1, @@ -1225,39 +1267,7 @@ describe('ScenarioRunPage', () => { it('opens details for a hierarchy result that the progress feed has not delivered yet', async () => { const user = userEvent.setup() - const hierarchyOnly = { - attack_result_id: 'hierarchy-only', - conversation_id: 'conversation-h', - result_role: 'orchestration', - atomic_attack_name: 'sequential-parent', - atomic_group_id: 'atomic-1', - seed_group_id: 'seed-unplanned', - objective: 'Objective carried by the hierarchy node', - outcome: 'failure', - execution_time_ms: 1200, - executed_turns: 0, - timestamp: '2026-01-01T00:00:00Z', - total_retries: 0, - retries: [], - retries_truncated: false, - child_attack_result_ids: [], - child_references_truncated: false, - children: [], - children_truncated: false, - child_offset: 0, - unresolved_child_attack_result_ids: [], - unresolved_child_references_truncated: false, - conversations: [], - conversations_truncated: false, - } - const counts = { completed: 1, planned: 1, succeeded: 0, success_percentage: 0, errors: 0, retries: 0 } - ;(scenariosApi.getRunHierarchy as jest.Mock).mockResolvedValueOnce({ - groups: [{ atomic_group_id: 'atomic-1', atomic_attack_name: 'a', display_group: 'Group', roots: [hierarchyOnly], counts }], - overall_counts: counts, - diagnostics: { rootless_results_count: 0 }, - truncated: false, - has_more: false, - }) + mockHierarchy([hierarchyNode()]) renderPage() await user.click(await screen.findByRole('button', { name: 'Open details for sequential-parent: Objective carried by the hierarchy node' })) @@ -1266,6 +1276,48 @@ describe('ScenarioRunPage', () => { expect(within(dialog).getByText('Objective carried by the hierarchy node')).toBeInTheDocument() }) + it('keeps the hierarchy objective when the progress row for an unplanned seed is already loaded', async () => { + const user = userEvent.setup() + // The progress row wins as the selected attempt, but it carries no objective and its seed is not in the plan. + const progressRow = { ...ATTEMPT, attack_result_id: 'unplanned-result', seed_group_id: 'seed-unplanned' } + mockHookState(makeState({ results: [ATTEMPT, progressRow] })) + mockHierarchy([ + hierarchyNode({ + attack_result_id: 'unplanned-result', + atomic_attack_name: 'attack-technique', + objective: 'Objective stored on the saved result', + }), + ]) + renderPage() + + await user.click(await screen.findByRole('button', { name: 'Open details for attack-technique: Objective stored on the saved result' })) + + const dialog = await screen.findByRole('dialog', { name: 'attack-technique' }) + expect(within(dialog).getByText('Objective stored on the saved result')).toBeInTheDocument() + expect(within(dialog).queryByText(/Objective text unavailable/)).not.toBeInTheDocument() + }) + + it('offers to reload the result hierarchy after a failed run is resumed', async () => { + const user = userEvent.setup() + const failedState = makeState({ run: { ...makeState().run, status: 'FAILED' } }) + mockHookState(failedState) + mockResumeRun.mockResolvedValueOnce({ ...failedState.run, status: 'QUEUED' }) + mockApplyRunSummary.mockImplementationOnce(() => { + mockHookState({ ...failedState, run: { ...failedState.run, status: 'QUEUED' } }) + }) + mockHierarchy([hierarchyNode()], 'FAILED') + renderPage() + + const hierarchy = screen.getByRole('region', { name: 'Result hierarchy' }) + await within(hierarchy).findByText('Objective carried by the hierarchy node') + expect(within(hierarchy).queryByRole('button', { name: 'Reload hierarchy' })).not.toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Resume run' })) + + expect(await within(hierarchy).findByRole('button', { name: 'Reload hierarchy' })).toBeInTheDocument() + expect(within(hierarchy).getByText(/This run is still in progress/)).toBeInTheDocument() + }) + it('shows descriptive technique details without result metrics', async () => { const user = userEvent.setup() renderPage() diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.tsx index 9e4f270609..80b7981c67 100644 --- a/frontend/src/components/Scenarios/ScenarioRunPage.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunPage.tsx @@ -657,6 +657,7 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu
    @@ -815,7 +816,10 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu attempt={selectedAttempt} objective={ seedObjectives.get(selectedAttempt.seed_group_id) ?? - (selectedAttempt === hierarchySelection ? hierarchySelection.objective : null) + // The progress row for this result carries no objective; the hierarchy node does. + (hierarchySelection?.attack_result_id === selectedAttempt.attack_result_id + ? hierarchySelection.objective + : null) } atomicGroup={selectedAttemptGroup} techniqueDetails={progressAtomicGroups.get(selectedAttempt.atomic_group_id)?.technique_details} From 455764a38ee1b513e9a18e32092b7131e1aef2a6 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sat, 31 Oct 2026 17:19:29 -0700 Subject: [PATCH 3/4] FEAT: consolidate Scenario results and saved Chat navigation Extend the existing atomic attack-group table with saved parent/child rows, separate Details, and direct conversation links that retain Scenario and history context. Preserve backend summary counts and cancel stale descendant requests on parent reload. Seed real offline transcripts and cover navigation, paging, collapse, and cached-request regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../fixtures/scenario_hierarchy_backend.py | 50 ++- frontend/e2e/scenario-hierarchy.spec.ts | 89 +++- frontend/src/App.test.tsx | 23 +- frontend/src/App.tsx | 8 +- .../AttackResults/AttackAttemptDetails.tsx | 14 +- .../src/components/Chat/ChatWindow.test.tsx | 16 +- frontend/src/components/Chat/ChatWindow.tsx | 1 + .../Scenarios/AttackExecutionTable.styles.ts | 58 +++ .../Scenarios/AttackExecutionTable.tsx | 377 ++++++++++++++--- .../ScenarioResultHierarchy.styles.ts | 78 +--- .../ScenarioResultHierarchy.test.tsx | 171 +++++++- .../Scenarios/ScenarioResultHierarchy.tsx | 382 ++---------------- .../Scenarios/ScenarioRunPage.styles.ts | 4 - .../Scenarios/ScenarioRunPage.test.tsx | 239 +++++++++-- .../components/Scenarios/ScenarioRunPage.tsx | 179 ++++---- .../Scenarios/scenarioHierarchy.test.ts | 43 +- .../components/Scenarios/scenarioHierarchy.ts | 21 + .../src/hooks/useScenarioResultHierarchy.ts | 43 +- frontend/src/types/index.ts | 13 + pyrit/backend/README.md | 17 +- 20 files changed, 1143 insertions(+), 683 deletions(-) create mode 100644 frontend/src/components/Scenarios/AttackExecutionTable.styles.ts diff --git a/frontend/e2e/fixtures/scenario_hierarchy_backend.py b/frontend/e2e/fixtures/scenario_hierarchy_backend.py index ee7b4d17b0..bddabe6dee 100644 --- a/frontend/e2e/fixtures/scenario_hierarchy_backend.py +++ b/frontend/e2e/fixtures/scenario_hierarchy_backend.py @@ -30,6 +30,8 @@ async def _seed_async(memory: SQLiteMemory) -> None: AttackOutcome, AttackResult, AttackResultRole, + Conversation, + MessagePiece, ScenarioIdentifier, ScenarioResult, ScenarioRunPlan, @@ -65,10 +67,10 @@ async def _seed_async(memory: SQLiteMemory) -> None: seed_groups=[ScenarioRunPlanSeedGroup(id="seed", objective="seed", objective_sha256=to_sha256("seed"))], ) - def result( + def _result( + *, index: int, group: str, - *, objective: str, outcome: AttackOutcome = AttackOutcome.FAILURE, child_ids: list[str] | None = None, @@ -76,7 +78,7 @@ def result( ) -> AttackResult: return AttackResult( attack_result_id=str(UUID(int=0x360200000 + index)), - conversation_id=str(UUID(int=0x360300000 + index)), + conversation_id="" if child_ids is not None else str(UUID(int=0x360300000 + index)), objective=objective, outcome=outcome, timestamp=timestamp + timedelta(seconds=index), @@ -98,12 +100,13 @@ def result( ) grandchildren = [ - result(1000 + i, "sequential", objective=f"Nested step {i + 1}", attempt_index=i + 1) for i in range(2) + _result(index=1000 + i, group="sequential", objective=f"Nested step {i + 1}", attempt_index=i + 1) + for i in range(2) ] children = [ - result( - 10 + i, - "sequential", + _result( + index=10 + i, + group="sequential", objective=f"Step {i + 1}", outcome=AttackOutcome.SUCCESS if i == 4 else AttackOutcome.FAILURE, child_ids=[g.attack_result_id for g in grandchildren] if i == 0 else None, @@ -114,11 +117,40 @@ def result( child_ids = [child.attack_result_id for child in children] child_ids[MISSING_POSITION] = str(UUID(int=0x3602DEAD)) child_ids[DUPLICATE_POSITION] = child_ids[0] - parent = result(1, "sequential", objective="Sequential parent", child_ids=child_ids) - single = result(2, "single", objective="Single-turn attempt", outcome=AttackOutcome.SUCCESS) + parent = _result(index=1, group="sequential", objective="Sequential parent", child_ids=child_ids) + single = _result(index=2, group="single", objective="Single-turn attempt", outcome=AttackOutcome.SUCCESS) rows = {"sequential": [parent, *children, *grandchildren], "single": [single]} await memory.add_attack_results_to_memory_async(attack_results=[row for group in rows.values() for row in group]) + pieces: list[MessagePiece] = [] + for group in rows.values(): + for row in group: + if not row.conversation_id: + continue + await memory.add_conversation_to_memory_async( + conversation=Conversation( + conversation_id=row.conversation_id, + target_identifier=target, + attack_result_id=row.attack_result_id, + ) + ) + pieces.extend( + [ + MessagePiece( + role="user", + conversation_id=row.conversation_id, + sequence=0, + original_value=f"Synthetic request for {row.objective}. No external model was called.", + ), + MessagePiece( + role="assistant", + conversation_id=row.conversation_id, + sequence=1, + original_value=f"Stored example response for {row.objective}. This is offline sample data.", + ), + ] + ) + await memory.add_message_pieces_to_memory_async(message_pieces=pieces) await memory.add_scenario_results_to_memory_async( scenario_results=[ ScenarioResult( diff --git a/frontend/e2e/scenario-hierarchy.spec.ts b/frontend/e2e/scenario-hierarchy.spec.ts index 2f96db0bd3..23b0c39e03 100644 --- a/frontend/e2e/scenario-hierarchy.spec.ts +++ b/frontend/e2e/scenario-hierarchy.spec.ts @@ -1,78 +1,123 @@ import { expect, test, type Page } from '@playwright/test' import { scenarioRunRoutePath } from '../src/utils/routeParams' -// Seeded by e2e/fixtures/scenario_hierarchy_backend.py -const run = scenarioRunRoutePath('00000000-0000-0000-0000-000000003602') +// Seeded by e2e/fixtures/scenario_hierarchy_backend.py. +const scenarioResultId = '00000000-0000-0000-0000-000000003602' +const run = scenarioRunRoutePath(scenarioResultId) async function openHierarchy(page: Page) { await page.goto(run) - const section = page.getByRole('region', { name: 'Result hierarchy', exact: true }) + const section = page.getByRole('region', { name: 'Atomic attack groups', exact: true }) + await section.getByRole('button', { name: 'Expand attacks in Sequential' }).click() await expect(section.getByText('Sequential parent', { exact: true })).toBeVisible() + await expect(page.getByRole('region', { name: 'Result hierarchy', exact: true })).toHaveCount(0) return section } -test('browses a compound result through the real API, page by page, with stable totals', async ({ page }, testInfo) => { +test('browses compound results in the existing attack-group table with stable totals', async ({ page }, testInfo) => { const errors: string[] = [] page.on('pageerror', (error) => errors.push(error.message)) const section = await openHierarchy(page) - const totals = section.getByLabel('Run totals') + const totals = section.getByText('2 groups, 524 executions', { exact: true }) const before = await totals.textContent() const firstPage = page.waitForResponse((response) => /\/hierarchy\/nodes\/[^/?]+\?limit=200$/.test(response.url())) await section.getByRole('button', { name: 'Expand children of sequential: Sequential parent' }).click() expect((await firstPage).status()).toBe(200) - const table = section.getByRole('table', { name: 'Sequential saved results' }) + const table = section.getByRole('table', { name: 'Sequential attack results' }) const row = (text: string) => table.getByRole('row').filter({ hasText: text }).first() await expect(table.getByText('Step 1', { exact: true })).toBeVisible() await expect(row('#2')).toContainText('Unresolved reference') await expect(row('#4')).toContainText('Unresolved reference') - // Children keep the parent's columns: outcome text sits under "Attack outcome", not nested in the row. await expect(row('Step 5').getByRole('cell').nth(2)).toHaveText('Success') - await expect(section.getByText('Showing 200 of 520 saved children')).toBeVisible() + await expect(section.getByText('Showing 200 of 520 saved child positions')).toBeVisible() for (const shown of [400, 520]) { const next = page.waitForResponse((response) => response.url().includes('children_cursor=')) await section.getByRole('button', { name: 'Load more children' }).click() expect((await next).status()).toBe(200) - await expect(section.getByText(`Showing ${shown} of 520 saved children`)).toBeVisible() + await expect(section.getByText(`Showing ${shown} of 520 saved child positions`)).toBeVisible() } await expect(section.getByRole('button', { name: 'Load more children' })).toHaveCount(0) await expect(table.getByText('Step 520', { exact: true })).toBeVisible() await table.getByRole('button', { name: 'Expand children of #1 sequential: Step 1' }).click() await expect(table.getByText('Nested step 2', { exact: true })).toBeVisible() + await table.getByRole('button', { name: 'Collapse children of sequential: Sequential parent' }).click() + await expect(table.getByText('Nested step 2', { exact: true })).toHaveCount(0) + await table.getByRole('button', { name: 'Expand children of sequential: Sequential parent' }).click() + await expect(table.getByText('Nested step 2', { exact: true })).toBeVisible() + await expect(section.getByText('Showing 520 of 520 saved child positions')).toBeVisible() + + await section.getByRole('button', { name: 'Collapse attacks in Sequential' }).click() + await section.getByRole('button', { name: 'Expand attacks in Sequential' }).click() + await expect(section.getByText('Showing 520 of 520 saved child positions')).toBeVisible() + await section.getByRole('button', { name: 'Collapse atomic attack groups' }).click() + await section.getByRole('button', { name: 'Expand atomic attack groups' }).click() + await expect(section.getByText('Nested step 2', { exact: true })).toBeVisible() expect(await totals.textContent()).toBe(before) - await section.getByRole('button', { name: 'Open details for single: Single-turn attempt' }).click() - await expect(page.getByRole('dialog', { name: 'single' })).toBeVisible() - await page.getByRole('button', { name: 'Close' }).click() + await section.getByRole('button', { name: 'Open details for #5 sequential: Step 5' }).click() + const dialog = page.getByRole('dialog', { name: 'sequential' }) + await expect(dialog.getByText('Step 5', { exact: true })).toBeVisible() + await dialog.getByRole('button', { name: 'Close' }).click() await expect(page.getByRole('dialog')).toHaveCount(0) + await expect(section.getByRole('button', { name: 'Open details for #5 sequential: Step 5' })).toBeFocused() - await page.getByRole('heading', { name: 'Result hierarchy', exact: true }).evaluate((heading) => + await page.getByRole('heading', { name: 'Atomic attack groups', exact: true }).evaluate((heading) => heading.scrollIntoView({ block: 'start' }), ) - await page.screenshot({ path: testInfo.outputPath('scenario-hierarchy-desktop.png') }) + await page.screenshot({ path: testInfo.outputPath('scenario-hierarchy-desktop.png'), animations: 'disabled' }) expect(errors).toEqual([]) }) +test('opens an actual saved child chat and retains the existing scenario breadcrumb after a reload', async ({ page }, testInfo) => { + const section = await openHierarchy(page) + await section.getByRole('button', { name: 'Expand children of sequential: Sequential parent' }).click() + const row = section.getByRole('row', { name: 'View conversation for #5 sequential: Step 5' }) + const chatLink = row.getByRole('link', { name: 'Step 5', exact: true }) + await expect(chatLink).toHaveAttribute('href', new RegExp(`\\?scenarioResultId=${scenarioResultId}$`)) + await chatLink.click() + + await expect(page).toHaveURL(new RegExp(`/attacks/[^/]+/conversations/[^/?]+\\?scenarioResultId=${scenarioResultId}$`)) + await expect(page.getByText('Synthetic request for Step 5. No external model was called.', { exact: true })).toBeVisible() + await expect(page.getByText('Stored example response for Step 5. This is offline sample data.', { exact: true })).toBeVisible() + const breadcrumb = page.getByRole('link', { name: `Return to scenario run ${scenarioResultId}` }) + await expect(breadcrumb).toHaveAttribute('href', run) + + await page.reload() + await expect(page.getByText('Stored example response for Step 5. This is offline sample data.', { exact: true })).toBeVisible() + await expect(breadcrumb).toBeVisible() + await page.screenshot({ path: testInfo.outputPath('scenario-saved-chat-breadcrumb.png'), animations: 'disabled' }) + + await breadcrumb.click() + await expect(page).toHaveURL(run) + const groups = page.getByRole('region', { name: 'Atomic attack groups', exact: true }) + await expect(groups.getByText('2 groups, 524 executions', { exact: true })).toBeVisible() + await groups.getByRole('button', { name: 'Expand attacks in Single turn' }).click() + const single = groups.getByRole('row', { name: 'View conversation for single: Single-turn attempt' }) + await single.focus() + await page.keyboard.press('Enter') + await expect(page.getByText('Stored example response for Single-turn attempt. This is offline sample data.', { exact: true })).toBeVisible() + await page.getByRole('link', { name: `Return to scenario run ${scenarioResultId}` }).click() + await expect(page).toHaveURL(run) +}) + test.describe('on a narrow touch screen', () => { - // Touch targets grow only for coarse pointers, as in touch-targets.spec.ts. test.use({ hasTouch: true }) - test('keeps the hierarchy usable at a narrow viewport and from the keyboard', async ({ page }, testInfo) => { + test('keeps grouped hierarchy rows usable on touch and from the keyboard', async ({ page }, testInfo) => { await page.setViewportSize({ width: 390, height: 844 }) const section = await openHierarchy(page) const expand = section.getByRole('button', { name: 'Expand children of sequential: Sequential parent' }) await expand.focus() await page.keyboard.press('Enter') - // The same button is renamed once expanded, so look it up by its new name. await expect(section.getByRole('button', { name: 'Collapse children of sequential: Sequential parent' })).toHaveAttribute( - 'aria-expanded', - 'true', + 'aria-expanded', 'true', ) - await expect(section.getByText('Showing 200 of 520 saved children')).toBeVisible() + await expect(section.getByText('Showing 200 of 520 saved child positions')).toBeVisible() const loadMore = section.getByRole('button', { name: 'Load more children' }) const size = await loadMore.boundingBox() @@ -80,7 +125,7 @@ test.describe('on a narrow touch screen', () => { const overflow = await section.evaluate((element) => element.scrollWidth > element.clientWidth) expect(overflow).toBe(false) - await section.scrollIntoViewIfNeeded() - await page.screenshot({ path: testInfo.outputPath('scenario-hierarchy-mobile.png') }) + await section.getByText('Sequential', { exact: true }).evaluate((label) => label.scrollIntoView({ block: 'start' })) + await page.screenshot({ path: testInfo.outputPath('scenario-hierarchy-mobile.png'), animations: 'disabled' }) }) }) diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index e47a0a95d4..ce3d2a5af3 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -6,7 +6,7 @@ import { Suspense, useEffect, useLayoutEffect, useRef, useState } from "react"; import { act, render, screen, fireEvent, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import { MemoryRouter, useLocation, useNavigate } from "react-router"; +import { MemoryRouter, parsePath, useLocation, useNavigate } from "react-router"; import App from "./App"; import { attacksApi, operationsApi, targetsApi } from "./services/api"; @@ -300,7 +300,9 @@ jest.mock("./components/Chat/ChatWindow", () => { {labels.operator ?? ""} {JSON.stringify(labels)} {scenarioResultId ?? "none"} - {`${location.pathname}${location.search}`} + + {`${location.pathname}${location.search}`} + @@ -663,9 +665,11 @@ function RouterProbe() { describe("App", () => { // App reads the active view from the URL, so every render needs a router. // initialPath lets a test deep-link straight to a view. - function renderApp(initialPath = "/") { + function renderApp(initialPath: string = "/", navigationState?: Record) { return render( - + ); @@ -1685,7 +1689,8 @@ describe("App", () => { labels: {}, related_conversation_ids: ["conv-456"], }); - renderApp(`/attacks/ar-1?scenarioResultId=${scenarioResultId}`); + const returnContext = { suiteExecutionId: "suite-1", suiteName: "Saved suite" }; + renderApp(`/attacks/ar-1?scenarioResultId=${scenarioResultId}`, returnContext); await waitFor(() => expect(screen.getByTestId("conversation-id")).toHaveTextContent("conv-main") ); @@ -1695,10 +1700,13 @@ describe("App", () => { `/attacks/ar-1/conversations/conv-456?scenarioResultId=${scenarioResultId}` ); expect(screen.getByTestId("scenario-result-id")).toHaveTextContent(scenarioResultId); + expect(screen.getByTestId("route-location")) + .toHaveAttribute("data-navigation-state", JSON.stringify(returnContext)); fireEvent.click(screen.getByTestId("new-attack")); expect(screen.getByTestId("route-location")).toHaveTextContent("/chat"); expect(screen.getByTestId("scenario-result-id")).toHaveTextContent("none"); + expect(screen.getByTestId("route-location")).toHaveAttribute("data-navigation-state", "null"); }); it("uses the conversation from a deep link when it belongs to the attack", async () => { @@ -1766,7 +1774,8 @@ describe("App", () => { labels: {}, related_conversation_ids: [], }); - renderApp(`/attacks/ar-1/conversations/bogus?scenarioResultId=${scenarioResultId}`); + const returnContext = { fromScenarioHistory: true, scenarioHistorySearch: "?status=COMPLETED" }; + renderApp(`/attacks/ar-1/conversations/bogus?scenarioResultId=${scenarioResultId}`, returnContext); await waitFor(() => expect(screen.getByTestId("route-location")).toHaveTextContent( @@ -1774,6 +1783,8 @@ describe("App", () => { ) ); expect(screen.getByTestId("scenario-result-id")).toHaveTextContent(scenarioResultId); + expect(screen.getByTestId("route-location")) + .toHaveAttribute("data-navigation-state", JSON.stringify(returnContext)); }); it("hydrates history filters from the URL query string", () => { diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index e69bb2ab62..59e8390b20 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -583,10 +583,10 @@ function AppContent({ operatorAlias }: { operatorAlias: string | null }) { routeConversationId === readyAttack.mainConversationId || readyAttack.relatedConversationIds.includes(routeConversationId) if (!isKnown && !findingEvidenceId) { - navigate(attackRoutePath(readyAttack.id, scenarioResultId), { replace: true }) + navigate(attackRoutePath(readyAttack.id, scenarioResultId), { replace: true, state: location.state }) } } - }, [readyAttack, routeConversationId, navigate, scenarioResultId, findingEvidenceId]) + }, [readyAttack, routeConversationId, navigate, scenarioResultId, findingEvidenceId, location.state]) const handleNavigate = useCallback((view: ViewName) => { // Re-attach the last filter query so returning to history restores filters. @@ -718,8 +718,8 @@ function AppContent({ operatorAlias }: { operatorAlias: string | null }) { const handleSelectConversation = useCallback((convId: string) => { if (!routeAttackId) return - navigate(attackConversationRoutePath(routeAttackId, convId, scenarioResultId)) - }, [routeAttackId, navigate, scenarioResultId]) + navigate(attackConversationRoutePath(routeAttackId, convId, scenarioResultId), { state: location.state }) + }, [routeAttackId, navigate, scenarioResultId, location.state]) const handleOpenAttack = useCallback((openAttackResultId: string) => { navigate(attackRoutePath(openAttackResultId)) diff --git a/frontend/src/components/AttackResults/AttackAttemptDetails.tsx b/frontend/src/components/AttackResults/AttackAttemptDetails.tsx index ebbcff8147..1ef30e06d4 100644 --- a/frontend/src/components/AttackResults/AttackAttemptDetails.tsx +++ b/frontend/src/components/AttackResults/AttackAttemptDetails.tsx @@ -46,7 +46,8 @@ interface AttackAttemptDetailsProps { /** Scorer that decided attack success. */ readonly objectiveScorer?: ScenarioObjectiveScorer | null /** Route to the attempt's conversation. Supplied by the host so this stays route-agnostic. */ - readonly conversationPath: string + readonly conversationPath: string | null + readonly conversationState?: unknown } export default function AttackAttemptDetails({ @@ -56,6 +57,7 @@ export default function AttackAttemptDetails({ techniqueDetails, objectiveScorer, conversationPath, + conversationState, }: AttackAttemptDetailsProps) { const styles = useAttackAttemptDetailsStyles() @@ -131,9 +133,13 @@ export default function AttackAttemptDetails({ - - View conversation - + {conversationPath ? ( + + View conversation + + ) : ( + No conversation was saved for this result. + )} {errorMessage && ( diff --git a/frontend/src/components/Chat/ChatWindow.test.tsx b/frontend/src/components/Chat/ChatWindow.test.tsx index 23dd85e56d..3d8f1cf11b 100644 --- a/frontend/src/components/Chat/ChatWindow.test.tsx +++ b/frontend/src/components/Chat/ChatWindow.test.tsx @@ -2618,10 +2618,20 @@ describe("ChatWindow Integration", () => { it("returns to the originating scenario run from the breadcrumb", async () => { const user = userEvent.setup(); const scenarioResultId = "123e4567-e89b-12d3-a456-426614174000"; + const navigationState = { + fromScenarioHistory: true, + scenarioHistorySearch: "?status=COMPLETED", + suiteExecutionId: "saved-suite-execution", + suiteName: "Saved suite", + }; + function OriginatingScenario() { + const location = useLocation(); + return

    Originating scenario run

    ; + } render( - + { /> Originating scenario run} + element={} /> @@ -2645,6 +2655,8 @@ describe("ChatWindow Integration", () => { level: 1, name: "Originating scenario run", })).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "Originating scenario run" })) + .toHaveAttribute("data-navigation-state", JSON.stringify(navigationState)); }); it("should apply and persist the Markdown choice across remounts", async () => { diff --git a/frontend/src/components/Chat/ChatWindow.tsx b/frontend/src/components/Chat/ChatWindow.tsx index 24727b822f..60ae6bc0e1 100644 --- a/frontend/src/components/Chat/ChatWindow.tsx +++ b/frontend/src/components/Chat/ChatWindow.tsx @@ -2304,6 +2304,7 @@ export default function ChatWindow({ Scenario run {scenarioResultId.slice(0, 8)} diff --git a/frontend/src/components/Scenarios/AttackExecutionTable.styles.ts b/frontend/src/components/Scenarios/AttackExecutionTable.styles.ts new file mode 100644 index 0000000000..3a0ed1fadc --- /dev/null +++ b/frontend/src/components/Scenarios/AttackExecutionTable.styles.ts @@ -0,0 +1,58 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTarget } from '@/styles/touchTargets' + +export const useAttackExecutionTableStyles = makeStyles({ + tableScroll: { + width: '100%', + overflowX: 'auto', + border: `${tokens.strokeWidthThin} solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusLarge, + backgroundColor: tokens.colorNeutralBackground1, + }, + table: { minWidth: '64rem', tableLayout: 'auto' }, + headerRow: { backgroundColor: tokens.colorNeutralBackground3 }, + resultRow: { + cursor: 'pointer', + ':hover': { backgroundColor: tokens.colorNeutralBackground1Hover }, + ':focus-visible': { + outline: `${tokens.strokeWidthThick} solid ${tokens.colorStrokeFocus2}`, + outlineOffset: `-${tokens.strokeWidthThick}`, + }, + }, + resultCell: { + display: 'flex', + alignItems: 'center', + gap: tokens.spacingHorizontalS, + minWidth: '18rem', + }, + identity: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXXS, + minWidth: 0, + overflowWrap: 'anywhere', + }, + objectiveLink: { + color: tokens.colorBrandForegroundLink, + textDecorationLine: 'none', + ':hover': { textDecorationLine: 'underline' }, + }, + expandSpacer: { width: '2rem', flexShrink: 0, ...mobileTouchTarget }, + position: { color: tokens.colorNeutralForeground3, fontVariantNumeric: 'tabular-nums' }, + hint: { color: tokens.colorNeutralForeground3 }, + unresolved: { + color: tokens.colorNeutralForeground3, + fontStyle: 'italic', + overflowWrap: 'anywhere', + }, + nowrap: { whiteSpace: 'nowrap', fontVariantNumeric: 'tabular-nums' }, + footer: { + display: 'flex', + alignItems: 'center', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalS, + paddingTop: tokens.spacingVerticalXS, + }, + touchTarget: { flexShrink: 0, ...mobileTouchTarget }, +}) diff --git a/frontend/src/components/Scenarios/AttackExecutionTable.tsx b/frontend/src/components/Scenarios/AttackExecutionTable.tsx index ab6287db19..9bf4915ddd 100644 --- a/frontend/src/components/Scenarios/AttackExecutionTable.tsx +++ b/frontend/src/components/Scenarios/AttackExecutionTable.tsx @@ -1,95 +1,360 @@ -import { Table, TableBody, TableCell, TableHeader, TableHeaderCell, TableRow, Text } from '@fluentui/react-components' +import type { CSSProperties } from 'react' + +import { + Badge, + Button, + MessageBar, + MessageBarActions, + MessageBarBody, + Table, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, + Text, +} from '@fluentui/react-components' +import { ChevronDownRegular, ChevronRightRegular } from '@fluentui/react-icons' +import { Link, useLocation, useNavigate } from 'react-router' -import OutcomeBadge from '@/components/OutcomeBadge' import { - formatAttackSuccess, + formatAttackOutcome, formatDuration, formatTimestamp, objectivePreview, } from '@/components/AttackResults/attackAttemptFormatting' -import type { ScenarioProgressResult } from '@/types' +import OutcomeBadge from '@/components/OutcomeBadge' +import type { HierarchyChildrenState, UseScenarioResultHierarchyResult } from '@/hooks/useScenarioResultHierarchy' +import type { ScenarioHierarchyNode, ScenarioProgressResult } from '@/types' +import { attackConversationRoutePath } from '@/utils/routeParams' -import { useScenarioRunPageStyles } from './ScenarioRunPage.styles' +import { useAttackExecutionTableStyles } from './AttackExecutionTable.styles' +import { childSlots, hasChildReferences, resultRoleLabel } from './scenarioHierarchy' interface AttackExecutionTableProps { + readonly scenarioResultId: string + readonly label: string readonly attempts: ScenarioProgressResult[] - readonly seedObjectives: Map - readonly onOpenDetails: (attempt: ScenarioProgressResult, trigger: HTMLElement) => void - /** Total attempts in this group, when more exist than are rendered. */ readonly totalAttempts: number + readonly roots: ScenarioHierarchyNode[] + readonly hierarchy: UseScenarioResultHierarchyResult + readonly seedObjectives: Map + readonly onOpenDetails: (result: ScenarioProgressResult | ScenarioHierarchyNode, trigger: HTMLElement) => void } export default function AttackExecutionTable({ + scenarioResultId, + label, attempts, + totalAttempts, + roots, + hierarchy, seedObjectives, onOpenDetails, - totalAttempts, }: AttackExecutionTableProps) { - const styles = useScenarioRunPageStyles() + const styles = useAttackExecutionTableStyles() + const savedRelationships = hierarchy.roots.status === 'ready' + const results: readonly (ScenarioProgressResult | ScenarioHierarchyNode)[] = savedRelationships ? roots : attempts - if (attempts.length === 0) { + if (results.length === 0) { return ( -
    - No execution details are available for this display group. -
    + + {savedRelationships + ? 'No top-level results are loaded for this group. Expand a parent or load more results.' + : 'No execution details are available for this display group.'} + ) } return (
    - {totalAttempts > attempts.length && ( - + {!savedRelationships && totalAttempts > attempts.length && ( + Showing the latest {attempts.length} of {totalAttempts} executions in this group. )} - +
    - - Attack - Attack Success - Objective - Execution + + Result / objective + Result type + Attack outcome + Duration Retries / error Timestamp + Actions - {attempts.map((attempt) => ( - onOpenDetails(attempt, event.currentTarget)} - onKeyDown={(event) => { - if (event.key === 'Enter' || event.key === ' ') { - event.preventDefault() - onOpenDetails(attempt, event.currentTarget) - } - }} - > - - {attempt.atomic_attack_name} - - - - - - - {objectivePreview(seedObjectives.get(attempt.seed_group_id) ?? null, attempt.seed_group_id)} - - - {formatDuration(attempt.execution_time_ms)} - - {attempt.outcome === 'error' - ? attempt.error_message ?? attempt.error_type ?? 'Error' - : `${attempt.total_retries} retries`} - - {formatTimestamp(attempt.timestamp)} - - ))} + {results.map((result) => { + const key = `${result.atomic_group_id}:${result.attack_result_id}` + return ( + + ) + })}
    ) } + +interface ResultRowsProps { + readonly nodeKey: string + readonly result: ScenarioProgressResult | ScenarioHierarchyNode + readonly objective: string + readonly position: number | null + readonly depth: number + readonly ancestors: ReadonlySet + readonly hierarchy: UseScenarioResultHierarchyResult + readonly scenarioResultId: string + readonly onOpenDetails: AttackExecutionTableProps['onOpenDetails'] +} + +function ResultRows({ + nodeKey, result, objective, position, depth, ancestors, hierarchy, scenarioResultId, onOpenDetails, +}: ResultRowsProps) { + const styles = useAttackExecutionTableStyles() + const navigate = useNavigate() + const location = useLocation() + const node = 'objective' in result ? result : null + const isExpanded = hierarchy.expanded.has(nodeKey) + const expandable = node !== null && hasChildReferences(node) + const preview = objectivePreview(objective || null, result.seed_group_id) + const label = resultLabel(result, preview, position) + const role = node?.result_role ?? 'unknown' + const conversationPath = result.conversation_id + ? attackConversationRoutePath(result.attack_result_id, result.conversation_id, scenarioResultId) + : null + + const openResult = (trigger: HTMLElement): void => { + if (conversationPath) { + navigate(conversationPath, { state: location.state }) + } else if (expandable) { + hierarchy.toggleNode(nodeKey, node) + } else { + onOpenDetails(result, trigger) + } + } + + return ( + <> + { + if (event.target instanceof Element && event.target.closest('a, button')) return + openResult(event.currentTarget) + }} + onKeyDown={(event) => { + if (event.target !== event.currentTarget) return + if (event.key === 'Enter' || event.key === ' ') { + event.preventDefault() + openResult(event.currentTarget) + } + }} + > + +
    + {expandable ? ( +
    +     + + + {resultRoleLabel(role)} + + + + + + {formatDuration(result.execution_time_ms)} + + {result.total_retries} + {result.outcome === 'error' && ( + + {result.error_message ?? result.error_type ?? 'Error'} + + )} + + {formatTimestamp(result.timestamp)} + + + +     + {node && expandable && isExpanded && ( + + )} + + ) +} + +interface ChildRowsProps { + readonly parentKey: string + readonly parent: ScenarioHierarchyNode + readonly depth: number + readonly ancestors: ReadonlySet + readonly state: HierarchyChildrenState | undefined + readonly hierarchy: UseScenarioResultHierarchyResult + readonly scenarioResultId: string + readonly onOpenDetails: AttackExecutionTableProps['onOpenDetails'] +} + +function ChildRows({ + parentKey, parent, depth, ancestors, state, hierarchy, scenarioResultId, onOpenDetails, +}: ChildRowsProps) { + const styles = useAttackExecutionTableStyles() + const pages = state?.pages ?? [] + const slots = pages.flatMap(childSlots) + return ( + <> + {slots.map((slot) => { + const cycle = slot.kind === 'child' && ancestors.has(slot.node.attack_result_id) + if (slot.kind === 'child' && !cycle) { + return ( + + ) + } + const id = slot.kind === 'child' ? slot.node.attack_result_id : slot.attackResultId + return ( + + +
    + + #{slot.position + 1} + + Unresolved reference: {id} + {cycle && ' (it points back to a result above it)'} + +
    +     +     + ) + })} + + +
    + {state?.loading && Loading children...} + {!state?.loading && pages.length > 0 && ( + + {state?.childrenCount != null + ? `Showing ${slots.length} of ${state.childrenCount} saved child positions` + : `Showing ${slots.length} saved child positions`} + + )} + {state?.staleCursor ? ( + + This result's saved children changed since they were loaded. + + + + + ) : state?.error ? ( + + Could not load children: {state.error} + + + + + ) : null} + {state?.nextCursor && !state.loading && !state.staleCursor && !state.error && ( + + )} +
    +     +     + + ) +} + +const NO_ANCESTORS: ReadonlySet = new Set() +const COLUMN_COUNT = 7 +const LABEL_OBJECTIVE_LENGTH = 60 +const INDENT_STEP = '1.5rem' + +function indent(depth: number): CSSProperties { + return { paddingInlineStart: `calc(${depth} * ${INDENT_STEP})` } +} + +function resultLabel(result: ScenarioProgressResult, objective: string, position: number | null): string { + const text = objective.length > LABEL_OBJECTIVE_LENGTH + ? `${objective.slice(0, LABEL_OBJECTIVE_LENGTH)}...` + : objective + return `${position === null ? '' : `#${position + 1} `}${result.atomic_attack_name}: ${text}` +} diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts b/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts index 25af9c77d4..3e59dedbc3 100644 --- a/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts @@ -3,85 +3,13 @@ import { makeStyles, tokens } from '@fluentui/react-components' import { mobileTouchTarget } from '@/styles/touchTargets' export const useScenarioResultHierarchyStyles = makeStyles({ - root: { - display: 'flex', - flexDirection: 'column', - gap: tokens.spacingVerticalM, - minWidth: 0, - }, - totals: { - display: 'flex', - flexWrap: 'wrap', - gap: tokens.spacingHorizontalL, - }, - hint: { - color: tokens.colorNeutralForeground3, - }, - group: { - display: 'flex', - flexDirection: 'column', - gap: tokens.spacingVerticalXS, - paddingTop: tokens.spacingVerticalS, - borderTop: `${tokens.strokeWidthThin} solid ${tokens.colorNeutralStroke2}`, - minWidth: 0, - }, - // Wide tables scroll inside their own box instead of widening the page on narrow screens. - tableScroll: { - width: '100%', - overflowX: 'auto', - border: `1px solid ${tokens.colorNeutralStroke2}`, - borderRadius: tokens.borderRadiusLarge, - backgroundColor: tokens.colorNeutralBackground1, - }, - table: { - minWidth: '52rem', - tableLayout: 'auto', - }, - resultCell: { - display: 'flex', - alignItems: 'center', - gap: tokens.spacingHorizontalS, - minWidth: '18rem', - }, - outcomeCell: { - display: 'flex', - flexDirection: 'column', - alignItems: 'flex-start', - gap: tokens.spacingVerticalXXS, - }, - nowrap: { - whiteSpace: 'nowrap', - }, - expandButton: { - flexShrink: 0, - ...mobileTouchTarget, - }, - expandSpacer: { - width: '2rem', - flexShrink: 0, - ...mobileTouchTarget, - }, - position: { - color: tokens.colorNeutralForeground3, - fontVariantNumeric: 'tabular-nums', - }, - objective: { - minWidth: 0, - overflowWrap: 'anywhere', - }, - unresolved: { - color: tokens.colorNeutralForeground3, - fontStyle: 'italic', - overflowWrap: 'anywhere', - }, + root: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalM, minWidth: 0 }, + hint: { color: tokens.colorNeutralForeground3 }, footer: { display: 'flex', alignItems: 'center', flexWrap: 'wrap', gap: tokens.spacingHorizontalS, - paddingTop: tokens.spacingVerticalXS, - }, - touchTarget: { - ...mobileTouchTarget, }, + touchTarget: { ...mobileTouchTarget }, }) diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx b/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx index 9f89f196ab..a56f76642a 100644 --- a/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx @@ -1,7 +1,9 @@ -import { render, screen, within } from '@testing-library/react' +import { act, render, screen, within } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { FluentProvider, webLightTheme } from '@fluentui/react-components' +import { MemoryRouter } from 'react-router' +import { useScenarioResultHierarchy } from '@/hooks/useScenarioResultHierarchy' import { scenariosApi } from '@/services/api' import type { ScenarioHierarchyNode, @@ -11,6 +13,7 @@ import type { } from '@/types' import ScenarioResultHierarchy from './ScenarioResultHierarchy' +import AttackExecutionTable from './AttackExecutionTable' jest.mock('@/services/api', () => ({ scenariosApi: { getRunHierarchy: jest.fn(), getRunHierarchyNode: jest.fn() }, @@ -98,20 +101,63 @@ function cursorError(status: number) { /** An orchestration parent whose children the preview deferred. */ const PARENT = node('parent', { + conversation_id: '', result_role: 'orchestration', child_attack_result_ids: ['c1', 'missing'], children_truncated: true, children_count: 3, }) -function mount(onOpenResult = jest.fn(), runStatus: ScenarioRunState | null = null) { - const view = (status: ScenarioRunState | null) => ( +function HierarchyHarness({ + onOpenResult, runStatus, scenarioResultId = 'run', +}: { + readonly onOpenResult: jest.Mock + readonly runStatus: ScenarioRunState | null + readonly scenarioResultId?: string +}) { + const hierarchy = useScenarioResultHierarchy(scenarioResultId) + const counts = hierarchy.roots.overallCounts + return ( + <> + {counts &&
    Completed: {counts.completed} of {counts.planned}
    } + + {hierarchy.roots.groups.map((group) => ( + + ))} + + + ) +} + +function mount(onOpenResult: jest.Mock = jest.fn(), runStatus: ScenarioRunState | null = null) { + const view = (status: ScenarioRunState | null, scenarioResultId = 'run') => ( - + + + ) const { rerender } = render(view(runStatus)) - return { onOpenResult, setRunStatus: (status: ScenarioRunState) => rerender(view(status)) } + return { + onOpenResult, + setRunStatus: (status: ScenarioRunState) => rerender(view(status)), + setRunId: (scenarioResultId: string) => rerender(view(runStatus, scenarioResultId)), + } } const resultsTable = () => screen.getByRole('table', { name: 'Sequential saved results' }) @@ -125,6 +171,104 @@ beforeEach(() => { getNode.mockReset() }) +test('resets expanded nodes and cached children when the run changes', async () => { + const user = userEvent.setup() + getRoots.mockResolvedValueOnce(projection([PARENT])).mockResolvedValueOnce(projection([{ + ...PARENT, objective: 'parent in the new run', + }])) + getNode + .mockResolvedValueOnce(node('parent', { + child_attack_result_ids: ['old-child'], + children_count: 1, + children: [node('old-child', { child_index: 0 })], + })) + .mockResolvedValueOnce(node('parent', { + child_attack_result_ids: ['new-child'], + children_count: 1, + children: [node('new-child', { child_index: 0 })], + })) + const view = mount() + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + expect(await screen.findByText('objective old-child')).toBeVisible() + + view.setRunId('another-run') + const expand = await screen.findByRole('button', { name: 'Expand children of attack parent: parent in the new run' }) + expect(expand).toHaveAttribute('aria-expanded', 'false') + expect(screen.queryByText('objective old-child')).not.toBeInTheDocument() + await user.click(expand) + expect(await screen.findByText('objective new-child')).toBeVisible() + expect(getNode).toHaveBeenLastCalledWith('another-run', 'parent', { limit: 200 }, expect.any(AbortSignal)) +}) + +test('ignores a late root response from a previously selected run', async () => { + let resolveOld: ((page: ScenarioHierarchyProjection) => void) | undefined + getRoots + .mockImplementationOnce(() => new Promise((resolve) => { resolveOld = resolve })) + .mockResolvedValueOnce(projection([node('current-run')])) + const view = mount() + view.setRunId('another-run') + expect(await screen.findByText('objective current-run')).toBeVisible() + if (resolveOld === undefined) throw new Error('The old run request was not started') + const respond = resolveOld + + await act(async () => { respond(projection([node('old-run')])) }) + expect(screen.queryByText('objective old-run')).not.toBeInTheDocument() + expect(screen.getByText('objective current-run')).toBeVisible() +}) + +test('does not reuse a late descendant page after restarting its parent', async () => { + const user = userEvent.setup() + const child = node('child', { + child_index: 0, + child_attack_result_ids: ['old-grandchild'], + children_count: 1, + children_truncated: true, + }) + const updatedChild = { ...child, child_attack_result_ids: ['fresh-grandchild'] } + let finishOldChild: ((page: ScenarioHierarchyNode) => void) | undefined + getRoots.mockResolvedValue(projection([PARENT])) + getNode + .mockResolvedValueOnce(node('parent', { + child_attack_result_ids: ['child'], + children: [child], + children_count: 2, + next_children_cursor: 'parent-next', + })) + .mockImplementationOnce(() => new Promise((resolve) => { finishOldChild = resolve })) + .mockRejectedValueOnce(cursorError(400)) + .mockResolvedValueOnce(node('parent', { + child_attack_result_ids: ['child'], + children: [updatedChild], + children_count: 1, + })) + .mockResolvedValueOnce(node('child', { + child_attack_result_ids: ['fresh-grandchild'], + children: [node('fresh-grandchild', { child_index: 0 })], + children_count: 1, + })) + mount() + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + await user.click(await screen.findByRole('button', { name: 'Expand children of #1 attack child: objective child' })) + await user.click(screen.getByRole('button', { name: 'Load more children' })) + await user.click(await screen.findByRole('button', { name: 'Reload children' })) + expect(getNode.mock.calls[1][3]).toHaveProperty('aborted', true) + const expandChild = await screen.findByRole('button', { name: 'Expand children of #1 attack child: objective child' }) + if (finishOldChild === undefined) throw new Error('The descendant request was not started') + const respond = finishOldChild + await act(async () => { + respond(node('child', { + child_attack_result_ids: ['old-grandchild'], + children: [node('old-grandchild', { child_index: 0 })], + children_count: 1, + })) + }) + + await user.click(expandChild) + expect(getNode).toHaveBeenCalledTimes(5) + expect(await screen.findByText('objective fresh-grandchild')).toBeVisible() + expect(screen.queryByText('objective old-grandchild')).not.toBeInTheDocument() +}) + test('loads children by position, keeps unresolved references visible, and pages with the node cursor', async () => { const user = userEvent.setup() getRoots.mockResolvedValue(projection([PARENT])) @@ -155,7 +299,7 @@ test('loads children by position, keeps unresolved references visible, and pages expect(getNode).toHaveBeenLastCalledWith('run', 'parent', { limit: 200 }, expect.any(AbortSignal)) expect(within(rowWith('objective c1')).getByText('#1')).toBeInTheDocument() expect(within(rowWith('Unresolved reference: missing')).getByText('#2')).toBeInTheDocument() - expect(screen.getByText('Showing 2 of 3 saved children')).toBeInTheDocument() + expect(screen.getByText('Showing 2 of 3 saved child positions')).toBeInTheDocument() await user.click(screen.getByRole('button', { name: 'Load more children' })) @@ -414,7 +558,7 @@ test('a failed run that is resumed offers to reload the hierarchy it loaded befo // Finishing after the reload is a change too; the snapshot is no longer current. setRunStatus('COMPLETED') - expect(screen.getByText("This run's status changed since this hierarchy loaded.")).toBeInTheDocument() + expect(screen.getByText("This run's status changed since these results loaded.")).toBeInTheDocument() expect(screen.getByRole('button', { name: 'Reload hierarchy' })).toBeInTheDocument() }) @@ -441,12 +585,17 @@ test('shows results in aligned columns with the attack outcome spelled out', asy 'Result type', 'Attack outcome', 'Duration', - 'Retries', + 'Retries / error', + 'Timestamp', 'Actions', ]) // A child row has the same cells in the same columns as its parent; only the first column is indented. const cells = (text: string) => within(rowWith(text)).getAllByRole('cell').map((cell) => cell.textContent) - expect(cells('objective parent')).toEqual(['objective parent', 'Orchestration', 'Success', '1s', '0', 'Details']) - expect(cells('objective c1')).toEqual(['#1objective c1', 'Target-facing', 'ErrorTarget timed out', '1s', '2', 'Details']) - expect(screen.getByText(/Each row is one saved attack result/)).toHaveTextContent('not chat messages or retries') + expect(cells('objective parent')).toEqual([ + 'objective parentattack parent', 'Orchestration', 'Success', '1s', '0', expect.any(String), 'Details', + ]) + expect(cells('objective c1')).toEqual([ + '#1objective c1attack c1', 'Target-facing', 'Error', '1s', '2Target timed out', expect.any(String), 'Details', + ]) + expect(screen.getByText(/^Result rows are saved attacks\./)).toHaveTextContent('not chat messages or retries') }) diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx b/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx index 00158badff..f41648208a 100644 --- a/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx @@ -1,70 +1,26 @@ -import type { CSSProperties } from 'react' +import type { ReactNode } from 'react' -import { - Badge, - Button, - MessageBar, - MessageBarActions, - MessageBarBody, - Table, - TableBody, - TableCell, - TableHeader, - TableHeaderCell, - TableRow, - Text, -} from '@fluentui/react-components' -import { ChevronDownRegular, ChevronRightRegular } from '@fluentui/react-icons' +import { Button, MessageBar, MessageBarActions, MessageBarBody, Text } from '@fluentui/react-components' -import OutcomeBadge from '@/components/OutcomeBadge' -import { formatAttackOutcome, formatDuration } from '@/components/AttackResults/attackAttemptFormatting' -import { - useScenarioResultHierarchy, - type HierarchyChildrenState, - type UseScenarioResultHierarchyResult, -} from '@/hooks/useScenarioResultHierarchy' -import type { ScenarioHierarchyNode, ScenarioProgressCounts, ScenarioRunState } from '@/types' +import type { UseScenarioResultHierarchyResult } from '@/hooks/useScenarioResultHierarchy' +import type { ScenarioRunState } from '@/types' -import { childSlots, hasChildReferences, resultRoleLabel } from './scenarioHierarchy' import { useScenarioResultHierarchyStyles } from './ScenarioResultHierarchy.styles' interface ScenarioResultHierarchyProps { - readonly scenarioResultId: string - /** The run's current state, which can change (for example on resume) after the hierarchy loaded. */ + readonly hierarchy: UseScenarioResultHierarchyResult readonly runStatus?: ScenarioRunState | null - readonly onOpenResult: (node: ScenarioHierarchyNode, trigger: HTMLElement) => void + readonly children: ReactNode } -/** Read-only browser for a run's saved result lineage: orchestration parents and their ordered children. */ +/** Loading and continuation controls around the existing atomic attack-group tables. */ export default function ScenarioResultHierarchy({ - scenarioResultId, + hierarchy, runStatus = null, - onOpenResult, + children, }: ScenarioResultHierarchyProps) { const styles = useScenarioResultHierarchyStyles() - const hierarchy = useScenarioResultHierarchy(scenarioResultId) const { roots } = hierarchy - - // Plain status text, not a Spinner: a progressbar role here would be read as run progress. - if (roots.status === 'loading') { - return ( - - Loading result hierarchy... - - ) - } - - if (roots.status === 'error') { - return ( - - Could not load the result hierarchy: {roots.error} - - - - - ) - } - const rootless = roots.diagnostics?.rootless_results_count ?? 0 const rootlessSample = roots.diagnostics?.rootless_result_ids ?? [] const currentStatus = runStatus ?? roots.runStatus @@ -73,74 +29,43 @@ export default function ScenarioResultHierarchy({ return (
    - {roots.overallCounts && } - - Totals cover the whole run as reported by the server; they are not counted from the rows shown here. - - {(runActive || statusChanged) && ( + {roots.status === 'loading' ? ( + + Loading result relationships. Available executions are shown as flat rows while they load. + + ) : roots.status === 'error' ? ( + + + Could not load the result relationships: {roots.error}. Available executions are shown as flat rows. + + + + + + ) : ( + + Result rows are saved attacks. Orchestration rows are compound parents: expand them to see the attacks + they ran in saved order, not chat messages or retries. Unresolved rows are references, not extra results. + Click a result with a saved chat to open it; Details shows metadata. Group metrics come from the run + summary, not a count of these rows. + + )} + {roots.status === 'ready' && (runActive || statusChanged) && (
    {runActive - ? 'This run is still in progress, so this hierarchy shows the results saved when it loaded.' - : "This run's status changed since this hierarchy loaded."} + ? 'This run is still in progress, so these rows show the results saved when they loaded.' + : "This run's status changed since these results loaded."}
    )} - - {roots.groups.length === 0 ? ( - rootless === 0 && No saved results are available for this run yet. - ) : ( - <> - - Each row is one saved attack result. An orchestration row summarizes a compound attack; expand it to - see its saved component results, numbered in order. These are separate attack results, not chat - messages or retries. - - {roots.groups.map((group) => ( -
    - {group.display_group} - - {group.atomic_attack_name} · {formatCounts(group.counts)} - -
    - - - - Result / objective - Result type - Attack outcome - Duration - Retries - Actions - - - - {group.roots.map((node, i) => { - const key = `${group.atomic_group_id}:${i}:${node.attack_result_id}` - return ( - - ) - })} - -
    -
    -
    - ))} - + {children} + {roots.status === 'ready' && roots.groups.length === 0 && rootless === 0 && ( + No saved results are available for this run yet. )} - {roots.truncated && ( Some results are not in this preview, such as deeper levels or results it could not place. Expand a @@ -155,7 +80,6 @@ export default function ScenarioResultHierarchy({ ` ${roots.diagnostics?.rootless_results_truncated ? 'Some of their IDs' : 'Their IDs'}: ${rootlessSample.join(', ')}.`} )} - {roots.staleCursor ? ( This run's saved results changed since they were loaded. @@ -163,7 +87,7 @@ export default function ScenarioResultHierarchy({ - ) : roots.error ? ( + ) : roots.status === 'ready' && roots.error ? ( Could not load more results: {roots.error} @@ -182,236 +106,4 @@ export default function ScenarioResultHierarchy({ ) } -interface HierarchyNodeRowsProps { - readonly nodeKey: string - readonly node: ScenarioHierarchyNode - /** Absolute position under the parent, or null for a root. */ - readonly position: number | null - readonly depth: number - /** Results on the path above this one; a child that is one of them closes a cycle. */ - readonly ancestors: ReadonlySet - readonly hierarchy: UseScenarioResultHierarchyResult - readonly onOpenResult: (node: ScenarioHierarchyNode, trigger: HTMLElement) => void -} - -/** A result's row, followed by its child rows when expanded. Only the first column is indented. */ -function HierarchyNodeRows({ - nodeKey, - node, - position, - depth, - ancestors, - hierarchy, - onOpenResult, -}: HierarchyNodeRowsProps) { - const styles = useScenarioResultHierarchyStyles() - const isExpanded = hierarchy.expanded.has(nodeKey) - const expandable = hasChildReferences(node) - const label = nodeLabel(node, position) - - return ( - <> - - -
    - {expandable ? ( -
    -     - - - {resultRoleLabel(node.result_role)} - - - -
    - - {node.outcome === 'error' && (node.error_message || node.error_type) && ( - - {node.error_message ?? node.error_type} - - )} -
    -     - {formatDuration(node.execution_time_ms)} - {node.total_retries} - - - -     - {expandable && isExpanded && ( - - )} - - ) -} - -interface ChildRowsProps { - readonly parentKey: string - readonly parent: ScenarioHierarchyNode - readonly depth: number - /** The parent and everything above it. */ - readonly ancestors: ReadonlySet - readonly state: HierarchyChildrenState | undefined - readonly hierarchy: UseScenarioResultHierarchyResult - readonly onOpenResult: (node: ScenarioHierarchyNode, trigger: HTMLElement) => void -} - -function ChildRows({ parentKey, parent, depth, ancestors, state, hierarchy, onOpenResult }: ChildRowsProps) { - const styles = useScenarioResultHierarchyStyles() - const pages = state?.pages ?? [] - const slots = pages.flatMap(childSlots) - const loaded = slots.length - const total = state?.childrenCount ?? null - - return ( - <> - {slots.map((slot) => { - // A fetched page can't know the path to it, so a reference back up the path is caught here. - const cycle = slot.kind === 'child' && ancestors.has(slot.node.attack_result_id) - if (slot.kind === 'child' && !cycle) { - return ( - - ) - } - const id = slot.kind === 'child' ? slot.node.attack_result_id : slot.attackResultId - return ( - - -
    - - #{slot.position + 1} - - Unresolved reference: {id} - {cycle && ' (it points back to a result above it)'} - -
    -     -     - ) - })} - - -
    - {state?.loading && ( - - Loading children... - - )} - {!state?.loading && pages.length > 0 && ( - - {total !== null ? `Showing ${loaded} of ${total} saved children` : `Showing ${loaded} saved children`} - - )} - {state?.staleCursor ? ( - - This result's saved children changed since they were loaded. - - - - - ) : state?.error ? ( - - Could not load children: {state.error} - - - - - ) : null} - {state?.nextCursor && !state.loading && !state.staleCursor && !state.error && ( - - )} -
    -     -     - - ) -} - -function RunTotals({ counts }: { readonly counts: ScenarioProgressCounts }) { - const styles = useScenarioResultHierarchyStyles() - return ( -
    - Completed: {counts.planned !== null ? `${counts.completed} of ${counts.planned}` : counts.completed} - Succeeded: {counts.succeeded} - Errors: {counts.errors} - Retries: {counts.retries} -
    - ) -} - const ACTIVE_RUN_STATES: ReadonlySet = new Set(['CREATED', 'QUEUED', 'IN_PROGRESS']) -const NO_ANCESTORS: ReadonlySet = new Set() -const COLUMN_COUNT = 6 -const LABEL_OBJECTIVE_LENGTH = 60 -const INDENT_STEP = '1.5rem' - -/** Nesting depth, shown only in the first column so the other columns stay aligned. */ -function indent(depth: number): CSSProperties { - return { paddingInlineStart: `calc(${depth} * ${INDENT_STEP})` } -} - -/** Accessible name for a result; sibling results usually share an attack name, so add position and objective. */ -function nodeLabel(node: ScenarioHierarchyNode, position: number | null): string { - const objective = - node.objective.length > LABEL_OBJECTIVE_LENGTH - ? `${node.objective.slice(0, LABEL_OBJECTIVE_LENGTH)}...` - : node.objective - const prefix = position === null ? '' : `#${position + 1} ` - return `${prefix}${node.atomic_attack_name}: ${objective}` -} - -function formatCounts(counts: ScenarioProgressCounts): string { - const completed = counts.planned !== null ? `${counts.completed} of ${counts.planned}` : `${counts.completed}` - return `${completed} completed, ${counts.succeeded} succeeded, ${counts.errors} errors, ${counts.retries} retries` -} diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.styles.ts b/frontend/src/components/Scenarios/ScenarioRunPage.styles.ts index 807a4d40b1..274737ca14 100644 --- a/frontend/src/components/Scenarios/ScenarioRunPage.styles.ts +++ b/frontend/src/components/Scenarios/ScenarioRunPage.styles.ts @@ -229,10 +229,6 @@ export const useScenarioRunPageStyles = makeStyles({ whiteSpace: 'nowrap', textAlign: 'right', }, - attemptsTable: { - minWidth: '64rem', - tableLayout: 'auto', - }, displayGroupList: { display: 'flex', flexDirection: 'column', diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx index 374d84c0ad..0c510a9c05 100644 --- a/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx @@ -188,7 +188,13 @@ function mockHierarchy(roots: ScenarioHierarchyNode[], status: ScenarioRunState const counts = { completed: 1, planned: 1, succeeded: 0, success_percentage: 0, errors: 0, retries: 0 } ;(scenariosApi.getRunHierarchy as jest.Mock).mockResolvedValueOnce({ status, - groups: [{ atomic_group_id: 'atomic-1', atomic_attack_name: 'a', display_group: 'Group', roots, counts }], + groups: [{ + atomic_group_id: roots[0]?.atomic_group_id ?? 'atomic-1', + atomic_attack_name: 'a', + display_group: 'Group', + roots, + counts, + }], overall_counts: counts, diagnostics: { rootless_results_count: 0 }, truncated: false, @@ -358,7 +364,11 @@ function AttackRouteProbe() { const location = useLocation() const navigate = useNavigate() return ( -
    +
    ) @@ -410,6 +420,8 @@ function renderPage( describe('ScenarioRunPage', () => { beforeEach(() => { jest.clearAllMocks() + ;(scenariosApi.getRunHierarchy as jest.Mock).mockReset().mockImplementation(() => new Promise(() => {})) + ;(scenariosApi.getRunHierarchyNode as jest.Mock).mockReset() mockUseScenarioQueue.mockReturnValue({ snapshot: { revision: 0, snapshot_at: '2026-01-01T00:00:00Z', active: null, queued: [] }, loading: false, @@ -521,7 +533,7 @@ describe('ScenarioRunPage', () => { const section = screen.getByRole('region', { name: 'Atomic attack groups' }) expect(within(section).getByRole('heading', { name: 'Atomic attack groups', level: 2 })).toBeVisible() - expect(within(section).getByText('1,716 groups, 10,296 executions')).toBeVisible() + expect(within(section).getByText('1,716 groups, 10,296 loaded executions')).toBeVisible() expect(within(section).getByRole('button', { name: 'Expand atomic attack groups', expanded: false, @@ -537,7 +549,7 @@ describe('ScenarioRunPage', () => { const user = userEvent.setup() renderPage() const section = screen.getByRole('region', { name: 'Atomic attack groups' }) - expect(within(section).getByText('1 group, 1 execution')).toBeVisible() + expect(within(section).getByText('1 group, 1 loaded execution')).toBeVisible() screen.getByRole('button', { name: 'Cancel run' }).focus() await user.tab() @@ -547,7 +559,7 @@ describe('ScenarioRunPage', () => { await user.keyboard('{Enter}') expect(screen.getByRole('button', { name: 'Collapse atomic attack groups', expanded: true })).toHaveFocus() await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) - expect(screen.getByRole('table', { name: 'Attack executions' })).toBeVisible() + expect(screen.getByRole('table', { name: /attack results$/ })).toBeVisible() const collapse = screen.getByRole('button', { name: 'Collapse atomic attack groups' }) collapse.focus() @@ -555,11 +567,11 @@ describe('ScenarioRunPage', () => { expect(screen.getByRole('button', { name: 'Expand atomic attack groups', expanded: false })).toHaveFocus() expect(within(section).queryAllByRole('article', { hidden: true })).toHaveLength(0) expect(within(section).queryByRole('table', { hidden: true })).not.toBeInTheDocument() - expect(within(section).getByText('1 group, 1 execution')).toBeVisible() + expect(within(section).getByText('1 group, 1 loaded execution')).toBeVisible() await user.keyboard('{Enter}') expect(screen.getByRole('button', { name: 'Collapse attacks in Technique One', expanded: true })).toBeVisible() - expect(screen.getByRole('table', { name: 'Attack executions' })).toBeVisible() + expect(screen.getByRole('table', { name: /attack results$/ })).toBeVisible() }) it('should preserve expanded and collapsed choices across progress updates and completion', async () => { @@ -569,7 +581,7 @@ describe('ScenarioRunPage', () => { mockHookState(makeGroupedState(22)) rerender() - expect(screen.getByText('22 groups, 22 executions')).toBeVisible() + expect(screen.getByText('22 groups, 22 loaded executions')).toBeVisible() expect(screen.getByRole('button', { name: 'Expand atomic attack groups', expanded: false })).toBeVisible() expect(screen.queryByText('Display group 1')).not.toBeInTheDocument() @@ -583,18 +595,18 @@ describe('ScenarioRunPage', () => { rerender() const section = screen.getByRole('region', { name: 'Atomic attack groups' }) - expect(within(section).getByText('3 groups, 6 executions')).toBeVisible() + expect(within(section).getByText('3 groups, 6 loaded executions')).toBeVisible() expect(screen.getByRole('button', { name: 'Collapse atomic attack groups', expanded: true })).toBeVisible() expect(screen.getByRole('button', { name: 'Collapse attacks in Display group 0', expanded: true })).toBeVisible() expect(screen.getByRole('button', { name: 'Expand attacks in Display group 2' })).toBeVisible() expect(within(section).getAllByText('2/2')).toHaveLength(3) - expect(within(screen.getByRole('table', { name: 'Attack executions' })).getAllByRole('row')).toHaveLength(3) + expect(within(screen.getByRole('table', { name: /attack results$/ })).getAllByRole('row')).toHaveLength(3) await user.click(screen.getByRole('button', { name: 'Collapse atomic attack groups' })) mockHookState(makeGroupedState(4)) rerender() expect(screen.getByRole('button', { name: 'Expand atomic attack groups', expanded: false })).toBeVisible() - expect(within(section).getByText('4 groups, 4 executions')).toBeVisible() + expect(within(section).getByText('4 groups, 4 loaded executions')).toBeVisible() expect(within(section).queryAllByRole('article', { hidden: true })).toHaveLength(0) }) @@ -625,7 +637,7 @@ describe('ScenarioRunPage', () => { expect(mockUseScenarioRunProgress).toHaveBeenLastCalledWith(OTHER_SCENARIO_RESULT_ID) expect(screen.getByRole('button', { name: /atomic attack groups$/, expanded: secondGroupCount <= 20 })).toBeVisible() - expect(screen.queryByRole('table', { name: 'Attack executions' })).not.toBeInTheDocument() + expect(screen.queryByRole('table', { name: /attack results$/ })).not.toBeInTheDocument() if (secondGroupCount > 20) { await user.click(screen.getByRole('button', { name: 'Expand atomic attack groups' })) } @@ -643,7 +655,7 @@ describe('ScenarioRunPage', () => { await user.click(screen.getByRole('button', { name: 'Expand atomic attack groups' })) } expect(screen.getByRole('button', { name: 'Expand attacks in Display group 0', expanded: false })).toBeVisible() - expect(screen.queryByRole('table', { name: 'Attack executions' })).not.toBeInTheDocument() + expect(screen.queryByRole('table', { name: /attack results$/ })).not.toBeInTheDocument() }, ) @@ -664,7 +676,7 @@ describe('ScenarioRunPage', () => { expect(within(lastGroup).getByText('Errors')).toBeVisible() expect(within(lastGroup).getByText('Retries')).toBeVisible() await user.click(groupToggles[100]) - await user.click(screen.getByRole('row', { name: 'View details for attack-100' })) + await user.click(screen.getByRole('button', { name: /^Open details for attack-100:/ })) const dialog = await screen.findByRole('dialog', { name: 'attack-100' }) expect(within(dialog).getByText(PLAN.seed_groups[0].objective)).toBeVisible() @@ -679,7 +691,7 @@ describe('ScenarioRunPage', () => { const user = userEvent.setup() mockHookState(makeGroupedState(0)) const { rerender } = renderPage() - expect(screen.getByText('0 groups, 0 executions')).toBeVisible() + expect(screen.getByText('0 groups, 0 loaded executions')).toBeVisible() expect(screen.getByText('No atomic attack groups have been persisted yet.')).toBeVisible() await user.click(screen.getByRole('button', { name: 'Collapse atomic attack groups' })) expect(screen.queryByText('No atomic attack groups have been persisted yet.')).not.toBeInTheDocument() @@ -732,7 +744,6 @@ describe('ScenarioRunPage', () => { 'Overall progress', 'Paired outcomes', 'Atomic attack groups', - 'Result hierarchy', 'Objective Scorer', 'Techniques', 'Objectives', @@ -818,7 +829,7 @@ describe('ScenarioRunPage', () => { expect(screen.getAllByText('1/total unavailable').length).toBeGreaterThan(0) expect(screen.queryByText('1/1')).not.toBeInTheDocument() await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) - expect(screen.getByRole('row', { name: 'View details for attack-technique' })).toBeInTheDocument() + expect(screen.getByRole('row', { name: /^View conversation for attack-technique:/ })).toBeInTheDocument() }) it('shows a stale warning and retries from the explicit action', async () => { @@ -1239,15 +1250,15 @@ describe('ScenarioRunPage', () => { ) }) - it('restores focus to the execution row after closing details', async () => { + it('restores focus to the separate Details button after closing details', async () => { jest.useFakeTimers() try { const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) renderPage() await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) - const detailsRow = screen.getByRole('row', { name: 'View details for attack-technique' }) + const detailsButton = screen.getByRole('button', { name: /^Open details for attack-technique:/ }) - await user.click(detailsRow) + await user.click(detailsButton) await waitFor(() => expect(screen.getByTestId('scanner-route')).toHaveAttribute( 'data-location', `/scanner-history/${SCENARIO_RESULT_ID}/attack-result-1`, @@ -1257,8 +1268,8 @@ describe('ScenarioRunPage', () => { await waitFor(() => expect(dialog).toContainElement(document.activeElement)) await user.click(close) - expect(await screen.findByRole('row', { name: 'View details for attack-technique' })).toBe(detailsRow) - await waitFor(() => expect(detailsRow).toHaveFocus()) + expect(await screen.findByRole('button', { name: /^Open details for attack-technique:/ })).toBe(detailsButton) + await waitFor(() => expect(detailsButton).toHaveFocus()) } finally { act(() => { jest.runOnlyPendingTimers() }) jest.useRealTimers() @@ -1270,6 +1281,7 @@ describe('ScenarioRunPage', () => { mockHierarchy([hierarchyNode()]) renderPage() + await user.click(await screen.findByRole('button', { name: 'Expand attacks in Group' })) await user.click(await screen.findByRole('button', { name: 'Open details for sequential-parent: Objective carried by the hierarchy node' })) const dialog = await screen.findByRole('dialog', { name: 'sequential-parent' }) @@ -1290,6 +1302,7 @@ describe('ScenarioRunPage', () => { ]) renderPage() + await user.click(await screen.findByRole('button', { name: 'Expand attacks in Group' })) await user.click(await screen.findByRole('button', { name: 'Open details for attack-technique: Objective stored on the saved result' })) const dialog = await screen.findByRole('dialog', { name: 'attack-technique' }) @@ -1297,6 +1310,146 @@ describe('ScenarioRunPage', () => { expect(within(dialog).queryByText(/Objective text unavailable/)).not.toBeInTheDocument() }) + it('keeps the saved result objective for a planned seed in the shared Details dialog', async () => { + const user = userEvent.setup() + mockHierarchy([hierarchyNode({ + attack_result_id: ATTEMPT.attack_result_id, + atomic_attack_name: ATTEMPT.atomic_attack_name, + atomic_group_id: ATTEMPT.atomic_group_id, + seed_group_id: ATTEMPT.seed_group_id, + conversation_id: ATTEMPT.conversation_id, + objective: 'The objective actually saved on this result', + })]) + renderPage() + await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) + await user.click(await screen.findByRole('button', { + name: 'Open details for attack-technique: The objective actually saved on this result', + })) + + const dialog = await screen.findByRole('dialog', { name: 'attack-technique' }) + expect(within(dialog).getByText('The objective actually saved on this result')).toBeVisible() + expect(within(dialog).queryByText(PLAN.seed_groups[0].objective)).not.toBeInTheDocument() + }) + + it('consolidates parent and child rows into the existing group without changing its live totals', async () => { + const user = userEvent.setup() + const counts = { completed: 7, planned: 12, succeeded: 3, success_percentage: 43, errors: 2, retries: 5 } + mockHookState(makeState({ + summary: { + ...SUMMARY, + overall: counts, + techniques: SUMMARY.techniques.map((group) => ({ ...group, ...counts })), + atomic_groups: SUMMARY.atomic_groups.map((group) => ({ ...group, ...counts })), + }, + })) + const child = hierarchyNode({ + attack_result_id: 'saved-child', + atomic_group_id: ATTEMPT.atomic_group_id, + objective: 'Saved child objective', + result_role: 'target_facing', + child_index: 0, + }) + mockHierarchy([hierarchyNode({ + atomic_group_id: ATTEMPT.atomic_group_id, + conversation_id: '', + child_attack_result_ids: [child.attack_result_id], + children_count: 1, + children: [child], + })]) + renderPage() + const section = screen.getByRole('region', { name: 'Atomic attack groups' }) + await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) + const table = await screen.findByRole('table', { name: 'Technique One attack results' }) + const parent = await within(table).findByRole('row', { name: /^View child results for sequential-parent:/ }) + expect(screen.queryByRole('region', { name: 'Result hierarchy' })).not.toBeInTheDocument() + expect(within(section).getAllByRole('article')).toHaveLength(1) + expect(within(section).getAllByRole('table')).toHaveLength(1) + expect(within(section).getByText('7/12')).toBeVisible() + + await user.click(parent) + expect(within(table).getByText('Saved child objective')).toBeVisible() + expect(within(section).getByText('7/12')).toBeVisible() + expect(screen.getByTestId('scanner-route')).toHaveAttribute( + 'data-location', `/scanner-history/${SCENARIO_RESULT_ID}`, + ) + + await user.click(screen.getByRole('button', { name: 'Collapse attacks in Technique One' })) + await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) + expect(screen.getByText('Saved child objective')).toBeVisible() + await user.click(screen.getByRole('button', { name: 'Collapse atomic attack groups' })) + await user.click(screen.getByRole('button', { name: 'Expand atomic attack groups' })) + expect(screen.getByText('Saved child objective')).toBeVisible() + expect(scenariosApi.getRunHierarchy).toHaveBeenCalledTimes(1) + }) + + it('uses backend producer counts for executions rather than the loaded progress rows', () => { + mockHookState(makeState({ + summary: { + ...SUMMARY, + overall: { + ...SUMMARY.overall, + producer_counts: { + target_facing: { attempts: 1000, errors: 0, retries: 0 }, + orchestration: { attempts: 10, errors: 0, retries: 0 }, + unknown: { attempts: 3, errors: 0, retries: 0 }, + }, + }, + }, + })) + renderPage() + const section = screen.getByRole('region', { name: 'Atomic attack groups' }) + expect(within(section).getByText('1 group, 1,013 executions')).toBeVisible() + }) + + it('does not offer a nonexistent chat for a conversationless compound result', async () => { + const user = userEvent.setup() + mockHierarchy([hierarchyNode({ + atomic_group_id: ATTEMPT.atomic_group_id, + conversation_id: '', + })]) + renderPage() + await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) + await user.click(await screen.findByRole('button', { + name: 'Open details for sequential-parent: Objective carried by the hierarchy node', + })) + + const dialog = await screen.findByRole('dialog', { name: 'sequential-parent' }) + expect(within(dialog).queryByRole('link', { name: 'View conversation' })).not.toBeInTheDocument() + expect(within(dialog).getByText('No conversation was saved for this result.')).toBeVisible() + }) + + it('keeps the existing execution rows usable when relationship loading fails', async () => { + const user = userEvent.setup() + ;(scenariosApi.getRunHierarchy as jest.Mock).mockRejectedValueOnce(new Error('Relationships unavailable')) + renderPage() + await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) + expect(await screen.findByText(/Could not load the result relationships:/)).toBeVisible() + const table = screen.getByRole('table', { name: 'Technique One attack results' }) + await user.click(within(table).getByRole('link')) + + expect(screen.getByTestId('attack-route')).toHaveAttribute( + 'data-location', + `/attacks/attack-result-1/conversations/conversation-1?scenarioResultId=${SCENARIO_RESULT_ID}`, + ) + }) + + it('preserves the suite return context through primary saved-chat navigation', async () => { + const user = userEvent.setup() + const navigationState = { + suiteExecutionId: 'suite-execution-1', + suiteName: 'Saved suite', + fromScenarioHistory: true, + scenarioHistorySearch: '?status=COMPLETED', + } + renderPage(`/scanner-history/${SCENARIO_RESULT_ID}`, navigationState) + await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) + await user.click(screen.getByRole('row', { name: /^View conversation for attack-technique:/ })) + + expect(screen.getByTestId('attack-route')).toHaveAttribute( + 'data-navigation-state', JSON.stringify(navigationState), + ) + }) + it('offers to reload the result hierarchy after a failed run is resumed', async () => { const user = userEvent.setup() const failedState = makeState({ run: { ...makeState().run, status: 'FAILED' } }) @@ -1308,7 +1461,8 @@ describe('ScenarioRunPage', () => { mockHierarchy([hierarchyNode()], 'FAILED') renderPage() - const hierarchy = screen.getByRole('region', { name: 'Result hierarchy' }) + const hierarchy = screen.getByRole('region', { name: 'Atomic attack groups' }) + await user.click(await screen.findByRole('button', { name: 'Expand attacks in Group' })) await within(hierarchy).findByText('Objective carried by the hierarchy node') expect(within(hierarchy).queryByRole('button', { name: 'Reload hierarchy' })).not.toBeInTheDocument() @@ -1353,22 +1507,25 @@ describe('ScenarioRunPage', () => { expect(within(dialog).queryByText('Errors')).not.toBeInTheDocument() }) - it('navigates to attempt details from the whole execution row', async () => { + it('opens the saved chat from the whole execution row without a Details detour', async () => { const user = userEvent.setup() renderPage() await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) expect(screen.queryByText('attack-result-1')).not.toBeInTheDocument() - const executionsTable = screen.getByRole('table', { name: 'Attack executions' }) - expect(within(executionsTable).getByRole('columnheader', { name: 'Attack' })).toBeInTheDocument() + const executionsTable = screen.getByRole('table', { name: /attack results$/ }) + expect(within(executionsTable).getByRole('columnheader', { name: 'Result / objective' })).toBeInTheDocument() const firstBodyRow = within(executionsTable).getAllByRole('row')[1] - expect(within(firstBodyRow).queryByRole('link')).not.toBeInTheDocument() + expect(within(firstBodyRow).getByRole('link')).toHaveAttribute( + 'href', + `/attacks/attack-result-1/conversations/conversation-1?scenarioResultId=${SCENARIO_RESULT_ID}`, + ) await user.click(firstBodyRow) - await waitFor(() => expect(screen.getByTestId('scanner-route')).toHaveAttribute( + await waitFor(() => expect(screen.getByTestId('attack-route')).toHaveAttribute( 'data-location', - `/scanner-history/${SCENARIO_RESULT_ID}/attack-result-1`, + `/attacks/attack-result-1/conversations/conversation-1?scenarioResultId=${SCENARIO_RESULT_ID}`, )) - expect(await screen.findByRole('dialog', { name: 'attack-technique' })).toBeVisible() + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() }) it('preserves scanner history context through attempt detail navigation', async () => { @@ -1383,9 +1540,9 @@ describe('ScenarioRunPage', () => { }, ) await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) - const executionsTable = screen.getByRole('table', { name: 'Attack executions' }) + const executionsTable = screen.getByRole('table', { name: /attack results$/ }) - await user.click(within(executionsTable).getAllByRole('row')[1]) + await user.click(within(executionsTable).getByRole('button', { name: /^Open details for attack-technique:/ })) const dialog = await screen.findByRole('dialog', { name: 'attack-technique' }) await user.click(within(dialog).getByRole('button', { name: 'Close' })) @@ -1477,7 +1634,7 @@ describe('ScenarioRunPage', () => { expect(screen.getByRole('button', { name: 'Collapse attacks in Technique One' })) .toHaveAttribute('aria-expanded', 'true') - const executionsTable = screen.getByRole('table', { name: 'Attack executions' }) + const executionsTable = screen.getByRole('table', { name: /attack results$/ }) expect(within(executionsTable).getAllByRole('row')).toHaveLength(3) expect(within(executionsTable).getByText('attack-technique')).toBeInTheDocument() expect(within(executionsTable).getByText('attack-technique-two')).toBeInTheDocument() @@ -1497,23 +1654,23 @@ describe('ScenarioRunPage', () => { expect(screen.getByText('Showing the latest 100 of 105 executions in this group.')).toBeInTheDocument() expect(screen.queryByText('attack-result-0')).not.toBeInTheDocument() - expect(within(screen.getByRole('table', { name: 'Attack executions' })) + expect(within(screen.getByRole('table', { name: /attack results$/ })) .getAllByRole('row')).toHaveLength(101) }) - it('navigates to attempt details from the keyboard-accessible execution row', async () => { + it.each(['{Enter}', ' '])('opens the saved chat from the execution row with %s', async (key: string) => { const user = userEvent.setup() renderPage() await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) - const executionRow = screen.getByRole('row', { name: 'View details for attack-technique' }) + const executionRow = screen.getByRole('row', { name: /^View conversation for attack-technique:/ }) executionRow.focus() - await user.keyboard('{Enter}') - await waitFor(() => expect(screen.getByTestId('scanner-route')).toHaveAttribute( + await user.keyboard(key) + await waitFor(() => expect(screen.getByTestId('attack-route')).toHaveAttribute( 'data-location', - `/scanner-history/${SCENARIO_RESULT_ID}/attack-result-1`, + `/attacks/attack-result-1/conversations/conversation-1?scenarioResultId=${SCENARIO_RESULT_ID}`, )) - expect(await screen.findByRole('dialog', { name: 'attack-technique' })).toBeVisible() + expect(screen.queryByRole('dialog')).not.toBeInTheDocument() }) it('opens attempt details from a direct link and returns to the scanner on close', async () => { @@ -1549,7 +1706,7 @@ describe('ScenarioRunPage', () => { })) renderPage() await user.click(screen.getByRole('button', { name: 'Expand attacks in Technique One' })) - await user.click(screen.getByRole('row', { name: 'View details for attack-technique' })) + await user.click(screen.getByRole('button', { name: /^Open details for attack-technique:/ })) const dialog = await screen.findByRole('dialog', { name: 'attack-technique' }) expect(within(dialog).getByText('No score rationale was persisted.')).toBeInTheDocument() diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.tsx index 80b7981c67..acf4365341 100644 --- a/frontend/src/components/Scenarios/ScenarioRunPage.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunPage.tsx @@ -46,6 +46,7 @@ import { } from '@/components/AttackResults/attackAttemptFormatting' import { suiteExecutionRoutePath } from '@/components/ScenarioSuites/suiteRoutes' import { useScenarioRunProgress } from '@/hooks/useScenarioRunProgress' +import { useScenarioResultHierarchy } from '@/hooks/useScenarioResultHierarchy' import { useScenarioRunResume } from '@/hooks/useScenarioRunResume' import { useScenarioQueue } from '@/hooks/useScenarioQueue' import { scenariosApi } from '@/services/api' @@ -77,6 +78,7 @@ import { useScenarioRunPageStyles } from './ScenarioRunPage.styles' import ScenarioQueue from './ScenarioQueue' import PairedOutcomes from './PairedOutcomes' import ScenarioResultHierarchy from './ScenarioResultHierarchy' +import { hierarchyDisplayGroups } from './scenarioHierarchy' const CLOCK_REFRESH_INTERVAL_MS = 1_000 const MAX_VISIBLE_ATTEMPTS_PER_GROUP = 100 @@ -115,6 +117,7 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu const location = useLocation() const navigate = useNavigate() const { state, retry, applyRunSummary } = useScenarioRunProgress(scenarioResultId) + const hierarchy = useScenarioResultHierarchy(scenarioResultId, state.run !== null && state.summary !== null) const queue = useScenarioQueue() const [cancelDialogOpen, setCancelDialogOpen] = useState(false) const [cancelling, setCancelling] = useState(false) @@ -131,6 +134,7 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu const [atomicGroupsExpandedChoice, setAtomicGroupsExpandedChoice] = useState(null) const [expandedGroupIds, setExpandedGroupIds] = useState>(new Set()) const detailsTriggerRef = useRef(null) + const atomicGroupsToggleRef = useRef(null) const [hierarchySelection, setHierarchySelection] = useState(null) const navigationState = location.state as { fromScenarioHistory?: boolean @@ -194,23 +198,22 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu const closeAttemptDetails = (): void => { navigate(scenarioRunRoutePath(scenarioResultId), { replace: true, state: location.state }) - requestAnimationFrame(() => detailsTriggerRef.current?.focus()) + requestAnimationFrame(() => { + const trigger = detailsTriggerRef.current + const focusTarget = trigger?.isConnected ? trigger : atomicGroupsToggleRef.current + focusTarget?.focus() + }) } const openAttemptDetails = ( - attempt: ScenarioProgressResult, + attempt: ScenarioProgressResult | ScenarioHierarchyNode, trigger: HTMLElement, ): void => { detailsTriggerRef.current = trigger + setHierarchySelection('objective' in attempt ? attempt : null) navigate(scenarioRunAttackRoutePath(scenarioResultId, attempt.attack_result_id), { state: location.state }) } - const openResultDetails = (node: ScenarioHierarchyNode, trigger: HTMLElement): void => { - detailsTriggerRef.current = trigger - setHierarchySelection(node) - navigate(scenarioRunAttackRoutePath(scenarioResultId, node.attack_result_id), { state: location.state }) - } - const toggleDisplayGroup = (groupId: string): void => { setExpandedGroupIds((current) => { const updated = new Set(current) @@ -311,7 +314,11 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu techniques, seed_groups: seedGroups, } = state.summary - const displayGroups = summarizedDisplayGroups ?? techniques + const displayGroups = hierarchyDisplayGroups(summarizedDisplayGroups ?? techniques, hierarchy.roots.groups) + const producerCounts = state.summary?.overall.producer_counts + const executionCount = producerCounts + ? producerCounts.target_facing.attempts + producerCounts.orchestration.attempts + producerCounts.unknown.attempts + : state.results.length const atomicGroupsExpanded = atomicGroupsExpandedChoice ?? displayGroups.length <= AUTO_EXPAND_GROUP_LIMIT const unattributedAttempts = state.summary.unattributed_attempts ?? 0 const queued = run.status === 'QUEUED' @@ -578,90 +585,89 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu aria-controls="atomic-groups-panel" aria-label={`${atomicGroupsExpanded ? 'Collapse' : 'Expand'} atomic attack groups`} data-testid="toggle-atomic-groups-btn" + ref={atomicGroupsToggleRef} onClick={() => setAtomicGroupsExpandedChoice(!atomicGroupsExpanded)} > {atomicGroupsExpanded ? 'Collapse' : 'Expand'}
    - {`${displayGroups.length.toLocaleString()} group${displayGroups.length === 1 ? '' : 's'}, ${state.results.length.toLocaleString()} execution${state.results.length === 1 ? '' : 's'}`} + {`${displayGroups.length.toLocaleString()} group${displayGroups.length === 1 ? '' : 's'}, ${executionCount.toLocaleString()} ${producerCounts ? '' : 'loaded '}execution${executionCount === 1 ? '' : 's'}`}
    -
    - - Result hierarchy - - -
    -
    Objective Scorer @@ -815,20 +821,21 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu )} diff --git a/frontend/src/components/Scenarios/scenarioHierarchy.test.ts b/frontend/src/components/Scenarios/scenarioHierarchy.test.ts index 98f83faacf..3bf0d084ba 100644 --- a/frontend/src/components/Scenarios/scenarioHierarchy.test.ts +++ b/frontend/src/components/Scenarios/scenarioHierarchy.test.ts @@ -1,6 +1,11 @@ -import type { ScenarioHierarchyGroup, ScenarioHierarchyNode, ScenarioHierarchyProjection } from '@/types' +import type { + ScenarioDisplayGroupProgress, + ScenarioHierarchyGroup, + ScenarioHierarchyNode, + ScenarioHierarchyProjection, +} from '@/types' -import { childSlots, mergeRootPage, previewHasAllChildren } from './scenarioHierarchy' +import { childSlots, hierarchyDisplayGroups, mergeRootPage, previewHasAllChildren } from './scenarioHierarchy' function node(overrides: Partial = {}): ScenarioHierarchyNode { return { @@ -98,4 +103,38 @@ describe('previewHasAllChildren', () => { expect(previewHasAllChildren(node({ child_attack_result_ids: ['a'], next_children_cursor: 'c' }))).toBe(false) expect(previewHasAllChildren(node({ child_attack_result_ids: ['a'], children_count: 3 }))).toBe(false) }) + + describe('hierarchyDisplayGroups', () => { + it('keeps merged live groups and their full counts instead of counting visible saved roots', () => { + const progress: ScenarioDisplayGroupProgress = { + ...counts(7), + id: 'display-group', + display_group: 'Merged group', + atomic_attack_names: ['g1', 'g2'], + atomic_group_ids: ['g1', 'g2'], + } + const groups = hierarchyDisplayGroups( + [progress], + [group('g1', ['r1'], 2), group('g2', ['r2'], 3)], + ) + expect(groups).toEqual([progress]) + expect(groups[0]).toBe(progress) + expect(groups[0].completed).toBe(7) + expect(groups[0].atomic_group_ids).toEqual(['g1', 'g2']) + }) + + it('surfaces an unplanned saved group with its exact backend counts even with no loaded roots', () => { + const saved = group('unplanned', [], 4) + const groups = hierarchyDisplayGroups([], [saved]) + + expect(groups).toEqual([{ + ...counts(4), + id: 'saved:unplanned', + display_group: 'unplanned', + atomic_attack_names: ['unplanned'], + atomic_group_ids: ['unplanned'], + }]) + expect(saved.roots).toEqual([]) + }) + }) }) diff --git a/frontend/src/components/Scenarios/scenarioHierarchy.ts b/frontend/src/components/Scenarios/scenarioHierarchy.ts index 79202943f6..5c319fa26f 100644 --- a/frontend/src/components/Scenarios/scenarioHierarchy.ts +++ b/frontend/src/components/Scenarios/scenarioHierarchy.ts @@ -1,5 +1,6 @@ import type { AttackResultRole, + ScenarioDisplayGroupProgress, ScenarioHierarchyGroup, ScenarioHierarchyNode, ScenarioHierarchyProjection, @@ -89,6 +90,26 @@ export function mergeRootPage( return merged } +/** Keeps live display-group totals, adding saved groups absent from the progress summary. */ +export function hierarchyDisplayGroups( + displayGroups: readonly ScenarioDisplayGroupProgress[], + hierarchyGroups: readonly ScenarioHierarchyGroup[], +): ScenarioDisplayGroupProgress[] { + const knownIds = new Set(displayGroups.flatMap((group) => group.atomic_group_ids)) + return [ + ...displayGroups, + ...hierarchyGroups + .filter((group) => !knownIds.has(group.atomic_group_id)) + .map((group) => ({ + ...group.counts, + id: `saved:${group.atomic_group_id}`, + display_group: group.display_group, + atomic_attack_names: [group.atomic_attack_name], + atomic_group_ids: [group.atomic_group_id], + })), + ] +} + const ROLE_LABELS: Record = { orchestration: 'Orchestration', target_facing: 'Target-facing', diff --git a/frontend/src/hooks/useScenarioResultHierarchy.ts b/frontend/src/hooks/useScenarioResultHierarchy.ts index 37c5ecaeec..ceb9c9de80 100644 --- a/frontend/src/hooks/useScenarioResultHierarchy.ts +++ b/frontend/src/hooks/useScenarioResultHierarchy.ts @@ -81,8 +81,11 @@ export interface UseScenarioResultHierarchyResult { readonly restartChildren: (key: string, node: ScenarioHierarchyNode) => void } -/** State is not reset when `scenarioResultId` changes; mount a new instance per run (for example with a `key`). */ -export function useScenarioResultHierarchy(scenarioResultId: string): UseScenarioResultHierarchyResult { +/** Mount a keyed owner per run so saved relationship caches cannot carry over to another run. */ +export function useScenarioResultHierarchy( + scenarioResultId: string, + enabled: boolean = true, +): UseScenarioResultHierarchyResult { const [roots, setRoots] = useState(INITIAL_ROOTS) const [children, setChildren] = useState>(new Map()) const [expanded, setExpanded] = useState>(new Set()) @@ -101,6 +104,15 @@ export function useScenarioResultHierarchy(scenarioResultId: string): UseScenari controllers.current.delete(controller) }, []) + const resetHierarchy = useCallback((): void => { + for (const controller of controllers.current) controller.abort() + controllers.current.clear() + childControllers.current.clear() + setRoots(INITIAL_ROOTS) + setChildren(new Map()) + setExpanded(new Set()) + }, []) + useEffect(() => { const active = controllers.current return () => { @@ -112,10 +124,12 @@ export function useScenarioResultHierarchy(scenarioResultId: string): UseScenari }, []) useEffect(() => { + if (!enabled) return const controller = track() scenariosApi .getRunHierarchy(scenarioResultId, { max_nodes: HIERARCHY_MAX_NODES }, controller.signal) .then((page) => { + if (controller.signal.aborted) return setRoots({ status: 'ready', runStatus: page.status, @@ -137,20 +151,12 @@ export function useScenarioResultHierarchy(scenarioResultId: string): UseScenari }) .finally(() => release(controller)) return () => controller.abort() - }, [scenarioResultId, rootEpoch, track, release]) + }, [scenarioResultId, enabled, rootEpoch, track, release]) const reloadRoots = useCallback(() => { - // Late responses from before the reload must not land in the fresh tree. - for (const controller of controllers.current) { - controller.abort() - } - controllers.current.clear() - childControllers.current.clear() - setRoots(INITIAL_ROOTS) - setChildren(new Map()) - setExpanded(new Set()) + resetHierarchy() setRootEpoch((epoch) => epoch + 1) - }, []) + }, [resetHierarchy]) const loadMoreRoots = useCallback(() => { const cursor = roots.nextCursor @@ -162,6 +168,7 @@ export function useScenarioResultHierarchy(scenarioResultId: string): UseScenari scenariosApi .getRunHierarchy(scenarioResultId, { max_nodes: HIERARCHY_MAX_NODES, cursor }, controller.signal) .then((page) => { + if (controller.signal.aborted) return setRoots((state) => ({ ...state, runStatus: page.status, @@ -209,6 +216,7 @@ export function useScenarioResultHierarchy(scenarioResultId: string): UseScenari controller.signal, ) .then((page) => { + if (controller.signal.aborted) return setChildren((map) => { const next = new Map(map) const current = map.get(key) ?? EMPTY_CHILDREN @@ -288,11 +296,18 @@ export function useScenarioResultHierarchy(scenarioResultId: string): UseScenari (key: string, node: ScenarioHierarchyNode) => { // Descendants loaded under the old child pages may no longer match; forget them too. const prefix = `${key}/` + for (const [childKey, controller] of childControllers.current) { + if (childKey.startsWith(prefix)) { + controller.abort() + childControllers.current.delete(childKey) + release(controller) + } + } setChildren((map) => new Map([...map].filter(([childKey]) => !childKey.startsWith(prefix)))) setExpanded((keys) => new Set([...keys].filter((childKey) => !childKey.startsWith(prefix)))) fetchChildren(key, node, null) }, - [fetchChildren], + [fetchChildren, release], ) const toggleNode = useCallback( diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 399834e866..75b7a73f72 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -2060,6 +2060,18 @@ export interface ScenarioRunPlan { seed_groups: ScenarioRunPlanSeedGroup[] } +export interface ScenarioProducerCategoryCounts { + attempts: number + errors: number + retries: number +} + +export interface ScenarioProducerCounts { + target_facing: ScenarioProducerCategoryCounts + orchestration: ScenarioProducerCategoryCounts + unknown: ScenarioProducerCategoryCounts +} + export interface ScenarioProgressCounts { completed: number planned: number | null @@ -2067,6 +2079,7 @@ export interface ScenarioProgressCounts { success_percentage: number | null errors: number retries: number + producer_counts?: ScenarioProducerCounts | null } export interface ScenarioTechniqueProgress extends ScenarioProgressCounts { diff --git a/pyrit/backend/README.md b/pyrit/backend/README.md index f7a65fed4d..c914f20503 100644 --- a/pyrit/backend/README.md +++ b/pyrit/backend/README.md @@ -75,8 +75,21 @@ and transcript reads, including hierarchy node expansion. ### Scenario result hierarchy These read-only APIs provide the backend foundation for compound-attack browsing. -The Scenario run page's "Result hierarchy" section browses them read-only; saved -conversation evidence for each node is separate follow-up work. +The Scenario run page integrates them into its existing "Atomic attack groups" +tables, rather than displaying a second results section. Group summaries retain +their server-derived totals while result rows expand into ordered saved children. +Completion and success count logical execution units. The execution count uses +the backend's raw producer attempt counts; neither is inferred from rendered +rows. Missing or duplicate child references remain visible as saved positions, +not additional results. +Available flat execution rows remain usable while relationships load or if the +hierarchy request fails, with an explicit loading/error notice. + +Clicking a result with a main conversation opens the existing chat view and +preserves scenario-run provenance for its return breadcrumb. Conversationless +compound results expand their children instead. A separate Details action reuses +the existing metadata dialog. Hierarchy requests do not load transcript bodies; +new bounded conversation-evidence rendering remains separate follow-up work. `GET /api/scenarios/runs/{scenario_result_id}/hierarchy?max_nodes=1000` returns a bounded tree preview grouped by atomic attack group. Roots are saved results with From aa135783d8c349322bd4bd9521b45ba80b659705 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sat, 31 Oct 2026 23:00:49 -0700 Subject: [PATCH 4/4] TEST: mock saved hierarchy in shared-label integration checks Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- frontend/src/App.labels.test.tsx | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/frontend/src/App.labels.test.tsx b/frontend/src/App.labels.test.tsx index a4c9be7eb4..6b6b4eb949 100644 --- a/frontend/src/App.labels.test.tsx +++ b/frontend/src/App.labels.test.tsx @@ -66,6 +66,7 @@ jest.mock('@/services/api', () => ({ scenariosApi: { listCatalog: jest.fn(), getScenario: jest.fn(), + getRunHierarchy: jest.fn(), estimateRun: jest.fn(), startRun: jest.fn(), cancelRun: jest.fn(), @@ -183,6 +184,17 @@ describe('Shared new run labels', () => { attack_result_id: 'old_attack', main_conversation_id: 'old_conversation', conversations: [], }) jest.mocked(scenariosApi.getScenario).mockResolvedValue(SCENARIO) + jest.mocked(scenariosApi.getRunHierarchy).mockResolvedValue({ + scenario_result_id: 'saved_run', scenario_name: 'SavedScenario', scenario_version: 1, + status: 'COMPLETED', groups: [], + overall_counts: { completed: 0, planned: 0, succeeded: 0, success_percentage: 0, errors: 0, retries: 0 }, + diagnostics: { + failed_attacks: [], failed_attacks_count: 0, failed_attacks_truncated: false, + attack_retries: [], attack_retries_count: 0, attack_retries_truncated: false, + total_retries: 0, rootless_result_ids: [], rootless_results_count: 0, rootless_results_truncated: false, + }, + has_more: false, truncated: false, + }) jest.mocked(scenariosApi.listCatalog).mockResolvedValue({ items: [SCENARIO], pagination: { limit: 200, has_more: false }, }) @@ -356,6 +368,7 @@ describe('Shared new run labels', () => { expect(saved).toHaveTextContent('original_team') expect(saved).not.toHaveTextContent('future_op') expect(within(saved).queryByRole('button', { name: /^Edit / })).not.toBeInTheDocument() + expect(scenariosApi.getRunHierarchy).toHaveBeenCalledTimes(1) expect(scenariosApi.startRun).not.toHaveBeenCalled() expect(scenariosApi.cancelRun).not.toHaveBeenCalled() })