diff --git a/packages/playwright/src/isomorphic/testServerConnection.ts b/packages/playwright/src/isomorphic/testServerConnection.ts index 17159a4edfb2f..748c0b3a3677e 100644 --- a/packages/playwright/src/isomorphic/testServerConnection.ts +++ b/packages/playwright/src/isomorphic/testServerConnection.ts @@ -219,6 +219,10 @@ export class TestServerConnection implements TestServerInterface, TestServerInte return await this._sendMessage('clearCache', params); } + async updateSnapshot(params: Parameters[0]): ReturnType { + return await this._sendMessage('updateSnapshot', params); + } + async listFiles(params: Parameters[0]): ReturnType { return await this._sendMessage('listFiles', params); } diff --git a/packages/playwright/src/isomorphic/testServerInterface.ts b/packages/playwright/src/isomorphic/testServerInterface.ts index 2450cbdc67ea8..4a5888cbd5f1c 100644 --- a/packages/playwright/src/isomorphic/testServerInterface.ts +++ b/packages/playwright/src/isomorphic/testServerInterface.ts @@ -22,6 +22,19 @@ import type * as reporterTypes from '../../types/testReporter'; export type ReportEntry = JsonEvent; +export type UpdateSnapshotParams = { + testId: string; + resultId: string; + actual: { + name: string; + contentType: string; + }; + expected: { + name: string; + contentType: string; + }; +}; + export interface TestServerInterface { initialize(params: { serializer?: string, @@ -57,6 +70,8 @@ export interface TestServerInterface { clearCache(params: {}): Promise; + updateSnapshot(params: UpdateSnapshotParams): Promise; + listFiles(params: { projects?: string[]; }): Promise<{ diff --git a/packages/playwright/src/runner/testServer.ts b/packages/playwright/src/runner/testServer.ts index 5a31cd6f98bde..64da145b46226 100644 --- a/packages/playwright/src/runner/testServer.ts +++ b/packages/playwright/src/runner/testServer.ts @@ -14,13 +14,15 @@ * limitations under the License. */ -import util from 'util'; import debug from 'debug'; +import fs from 'fs'; import open from 'open'; +import util from 'util'; import { server as coreServer } from 'playwright-core/lib/coreBundle'; import { ManualPromise } from '@isomorphic/manualPromise'; import { isUnderTest } from '@utils/debug'; +import { isPathInside } from '@utils/fileUtils'; import { HttpServer } from '@utils/httpServer'; import { gracefullyProcessExitDoNotHang } from '@utils/processLauncher'; @@ -35,7 +37,7 @@ import { wrapReporterAsV2 } from '../reporters/reporterV2'; import type { Transport } from '@utils/httpServer'; import type * as reporterTypes from '../../types/testReporter'; import type { ConfigLocation } from '../common'; -import type { ReportEntry, TestServerInterface, TestServerInterfaceEventEmitters } from '../isomorphic/testServerInterface'; +import type { ReportEntry, TestServerInterface, TestServerInterfaceEventEmitters, UpdateSnapshotParams } from '../isomorphic/testServerInterface'; import type { ReporterV2 } from '../reporters/reporterV2'; type TraceViewerRedirectOptions = coreServer.TraceViewerRedirectOptions; @@ -49,6 +51,27 @@ const originalStderrWrite = process.stderr.write; const originalStdinIsTTY = process.stdin.isTTY; +function allowedFileRoots(configLocation: ConfigLocation, testRunner?: TestRunner): string[] { + const roots = new Set([process.cwd(), configLocation.configDir]); + const config = testRunner?.lastLoadedConfig(); + if (config) { + for (const project of config.projects) { + roots.add(project.project.outputDir); + roots.add(project.project.snapshotDir); + roots.add(project.project.testDir); + } + } + return [...roots]; +} + +function testResultKey(testId: string, resultId: string): string { + return JSON.stringify([testId, resultId]); +} + +function attachmentKey(attachment: UpdateSnapshotParams['actual']): string { + return JSON.stringify([attachment.name, attachment.contentType]); +} + class TestServer { private _configLocation: ConfigLocation; private _configCLIOverrides: ipc.ConfigCLIOverrides; @@ -70,15 +93,7 @@ class TestServer { } private _allowedFileRoots(): string[] { - const roots = new Set([process.cwd(), this._configLocation.configDir]); - const config = this._dispatcher?._testRunner.lastLoadedConfig(); - if (config) { - for (const project of config.projects) { - roots.add(project.project.outputDir); - roots.add(project.project.testDir); - } - } - return [...roots]; + return allowedFileRoots(this._configLocation, this._dispatcher?._testRunner); } async stop() { @@ -112,13 +127,17 @@ export type RunTestsParams = { export class TestServerDispatcher implements TestServerInterface { readonly transport: Transport; + private _configLocation: ConfigLocation; private _serializer: string | undefined; private _closeOnDisconnect = false; + private _testResultIdForTestId = new Map(); + private _pathForAttachmentForTestResult = new Map>(); _testRunner: TestRunner; private _globalSetupReport: ReportEntry[] | undefined; readonly _dispatchEvent: TestServerInterfaceEventEmitters['dispatchEvent']; constructor(configLocation: ConfigLocation, configCLIOverrides: ipc.ConfigCLIOverrides) { + this._configLocation = configLocation; this._testRunner = new TestRunner(configLocation, configCLIOverrides); this.transport = { onconnect: () => {}, @@ -134,8 +153,41 @@ export class TestServerDispatcher implements TestServerInterface { this._testRunner.on(TestRunnerEvent.TestPaused, params => this._dispatchEvent('testPaused', { errors: params.errors })); } - private async _wireReporter(messageSink: (message: any) => void) { - return await createReporterForTestServer(this._serializer, messageSink); + private async _wireReporter(messageSink: (message: ReportEntry) => void) { + return await createReporterForTestServer(this._serializer, message => { + this._trackAttachmentPaths(message); + messageSink(message); + }); + } + + private _trackAttachmentPaths(message: ReportEntry) { + switch (message.method) { + case 'onTestBegin': { + const { testId, result } = message.params; + const previousResultId = this._testResultIdForTestId.get(testId); + if (previousResultId) + this._pathForAttachmentForTestResult.delete(testResultKey(testId, previousResultId)); + this._testResultIdForTestId.set(testId, result.id); + return; + } + + case 'onAttach': { + if (this._testResultIdForTestId.get(message.params.testId) !== message.params.resultId) + throw new Error(`Unknown test result: ${message.params.resultId}`); + const resultKey = testResultKey(message.params.testId, message.params.resultId); + let paths = this._pathForAttachmentForTestResult.get(resultKey); + if (!paths) { + paths = new Map(); + this._pathForAttachmentForTestResult.set(resultKey, paths); + } + for (const attachment of message.params.attachments) { + // Attachments without paths can also make a name ambiguous. + const key = attachmentKey(attachment); + paths.set(key, paths.has(key) ? null : attachment.path); + } + return; + } + } } private async _collectingReporter(): Promise<{ reporter: ReporterV2, report: ReportEntry[] }> { @@ -195,6 +247,26 @@ export class TestServerDispatcher implements TestServerInterface { await this._testRunner.clearCache(); } + async updateSnapshot(params: Parameters[0]): ReturnType { + const config = this._testRunner.lastLoadedConfig(); + if (!config) + throw new Error('Cannot update snapshot before loading the configuration'); + const actualName = params.actual.name.match(/^(.*)-actual\.png$/)?.[1]; + const expectedName = params.expected.name.match(/^(.*)-expected\.png$/)?.[1]; + if (!actualName || actualName !== expectedName) + throw new Error('Actual and expected attachments do not form a snapshot pair'); + const pathForAttachment = this._pathForAttachmentForTestResult.get(testResultKey(params.testId, params.resultId)); + const actualPath = pathForAttachment?.get(attachmentKey(params.actual)); + const expectedPath = pathForAttachment?.get(attachmentKey(params.expected)); + if (!actualPath || !expectedPath) + throw new Error('Snapshot attachments are not registered or have duplicates for this test result'); + if (!config.projects.some(project => isPathInside(project.project.outputDir, actualPath))) + throw new Error('Actual snapshot path is outside of the configured output directories'); + if (!allowedFileRoots(this._configLocation, this._testRunner).some(root => isPathInside(root, expectedPath))) + throw new Error('Expected snapshot path is outside of the allowed file roots'); + await fs.promises.copyFile(actualPath, expectedPath); + } + async listFiles(params: Parameters[0]): ReturnType { const { reporter, report } = await this._collectingReporter(); const { status } = await this._testRunner.listFiles(reporter, params.projects); @@ -348,7 +420,7 @@ function chunkToPayload(type: 'stdout' | 'stderr', chunk: Buffer | string): Stdi return { type, text: chunk }; } -async function createReporterForTestServer(file: string | undefined, messageSink: (message: any) => void): Promise { +async function createReporterForTestServer(file: string | undefined, messageSink: (message: ReportEntry) => void): Promise { const reporterConstructor = file ? await loadReporter(null, file) : UIModeReporter; return wrapReporterAsV2(new reporterConstructor({ _send: messageSink, diff --git a/packages/trace-viewer/src/ui/attachmentsTab.css b/packages/trace-viewer/src/ui/attachmentsTab.css index 7d487bb3f246c..b032fb01e75f0 100644 --- a/packages/trace-viewer/src/ui/attachmentsTab.css +++ b/packages/trace-viewer/src/ui/attachmentsTab.css @@ -36,6 +36,20 @@ margin-top: 10px; } +.attachments-image-diff-header { + position: relative; +} + +.attachments-update-snapshot { + background-color: var(--vscode-editor-inactiveSelectionBackground); + font-weight: normal; + padding: 4px 12px; + position: absolute; + right: 5px; + text-transform: none; + top: 5px; +} + .attachment-item { margin: 4px 8px; } diff --git a/packages/trace-viewer/src/ui/attachmentsTab.tsx b/packages/trace-viewer/src/ui/attachmentsTab.tsx index e69a9db980af9..f9716cd5cf59e 100644 --- a/packages/trace-viewer/src/ui/attachmentsTab.tsx +++ b/packages/trace-viewer/src/ui/attachmentsTab.tsx @@ -22,11 +22,16 @@ import { CodeMirrorWrapper, lineHeight } from '@web/components/codeMirrorWrapper import { isTextualMimeType } from '@isomorphic/mimeType'; import { Expandable } from '@web/components/expandable'; import { linkifyText } from '@web/renderUtils'; +import { ToolbarButton } from '@web/components/toolbarButton'; import { clsx, useFlash } from '@web/uiUtils'; import { useTraceModel } from './traceModelContext'; import type { Attachment, TraceModel } from '@isomorphic/trace/traceModel'; +type SnapshotAttachment = Pick; + +export type UpdateSnapshot = (params: { actual: SnapshotAttachment, expected: SnapshotAttachment }) => Promise; + type ExpandableAttachmentProps = { attachment: Attachment; reveal: any; @@ -90,9 +95,47 @@ const ExpandableAttachment: React.FunctionComponent = ; }; +function UpdateSnapshotButton({ actual, expected, onUpdateSnapshot }: { + actual: SnapshotAttachment, + expected: SnapshotAttachment, + onUpdateSnapshot: UpdateSnapshot, +}) { + const [saving, setSaving] = React.useState(false); + const [saved, triggerSavedFlash] = useFlash(); + const [error, setError] = React.useState(); + + const updateSnapshot = React.useCallback(async () => { + setSaving(true); + setError(undefined); + try { + await onUpdateSnapshot({ + actual: { name: actual.name, contentType: actual.contentType }, + expected: { name: expected.name, contentType: expected.contentType }, + }); + triggerSavedFlash(); + } catch (error) { + setError(error instanceof Error ? error.message : String(error)); + } finally { + setSaving(false); + } + }, [actual, expected, onUpdateSnapshot, triggerSavedFlash]); + + const label = error ? 'Retry save' : 'Save actual as expected'; + + return {label}; +} + export const AttachmentsTab: React.FunctionComponent<{ revealedAttachmentCallId?: { callId: string }, -}> = ({ revealedAttachmentCallId }) => { + onUpdateSnapshot?: UpdateSnapshot, +}> = ({ revealedAttachmentCallId, onUpdateSnapshot }) => { const model = useTraceModel(); const { diffMap, screenshots, attachments } = React.useMemo(() => { const attachments = new Set(model?.visibleAttachments ?? []); @@ -122,16 +165,25 @@ export const AttachmentsTab: React.FunctionComponent<{ return ; return
- {[...diffMap.values()].map(({ expected, actual, diff }) => { - return <> - {expected && actual &&
Image diff
} - {expected && actual && { + if (!expected || !actual) + return null; + return +
+ Image diff + {onUpdateSnapshot && } +
+ } - ; + }} /> +
; })} {screenshots.size ?
Screenshots
: undefined} {[...screenshots.values()].map((a, i) => { diff --git a/packages/trace-viewer/src/ui/uiModeTraceView.tsx b/packages/trace-viewer/src/ui/uiModeTraceView.tsx index b167ac74adc11..a264be04b91cb 100644 --- a/packages/trace-viewer/src/ui/uiModeTraceView.tsx +++ b/packages/trace-viewer/src/ui/uiModeTraceView.tsx @@ -15,6 +15,7 @@ */ import { artifactsFolderName } from '@testIsomorphic/folders'; +import { TeleTestResult } from '@testIsomorphic/teleReceiver'; import type { TreeItem } from '@testIsomorphic/testTree'; import '@web/common.css'; import '@web/third_party/vscode/codicon.css'; @@ -23,17 +24,20 @@ import React from 'react'; import type { ContextEntry } from '@isomorphic/trace/entries'; import type { SourceLocation } from '@isomorphic/trace/traceModel'; import { TraceModel } from '@isomorphic/trace/traceModel'; +import type { UpdateSnapshotParams } from '@testIsomorphic/testServerInterface'; +import type { UpdateSnapshot } from './attachmentsTab'; import { Workbench } from './workbench'; export const TraceView: React.FC<{ item: { treeItem?: TreeItem, testFile?: SourceLocation, testCase?: reporterTypes.TestCase }, rootDir?: string, onOpenExternally?: (location: SourceLocation) => void, + onUpdateSnapshot?: (params: UpdateSnapshotParams) => Promise, revealSource?: boolean, pathSeparator: string, onModelChange?: (model: TraceModel | undefined) => void, -}> = ({ item, rootDir, onOpenExternally, revealSource, pathSeparator, onModelChange }) => { - const [model, setModel] = React.useState<{ model: TraceModel, isLive: boolean } | undefined>(undefined); +}> = ({ item, rootDir, onOpenExternally, onUpdateSnapshot, revealSource, pathSeparator, onModelChange }) => { + const [model, setModel] = React.useState<{ model: TraceModel, isLive: boolean, result?: reporterTypes.TestResult } | undefined>(undefined); const [counter, setCounter] = React.useState(0); const pollTimer = React.useRef(null); @@ -55,7 +59,7 @@ export const TraceView: React.FC<{ // Test finished. const attachment = result && result.duration >= 0 && result.attachments.find(a => a.name === 'trace'); if (attachment && attachment.path) { - loadSingleTraceFile(attachment.path, result.startTime.getTime()).then(model => setModel({ model, isLive: false })); + loadSingleTraceFile(attachment.path, result.startTime.getTime()).then(model => setModel({ model, isLive: false, result })); return; } @@ -93,6 +97,14 @@ export const TraceView: React.FC<{ onModelChange?.(model?.model); }, [model, onModelChange]); + const testCase = item.testCase; + const testResult = testCase?.results[0]; + const updateSnapshot: UpdateSnapshot | undefined = testCase && testResult instanceof TeleTestResult && model?.result === testResult && onUpdateSnapshot ? params => onUpdateSnapshot({ + ...params, + testId: testCase.id, + resultId: testResult._id, + }) : undefined; + return ; }; diff --git a/packages/trace-viewer/src/ui/uiModeView.tsx b/packages/trace-viewer/src/ui/uiModeView.tsx index 3bf380dd2778b..2506e3e116850 100644 --- a/packages/trace-viewer/src/ui/uiModeView.tsx +++ b/packages/trace-viewer/src/ui/uiModeView.tsx @@ -484,6 +484,7 @@ export const UIModeView: React.FC<{}> = ({ rootDir={testModel?.config?.rootDir} revealSource={revealSource} onOpenExternally={location => testServerConnection?.openNoReply({ location: { file: location.file, line: location.line, column: location.column } })} + onUpdateSnapshot={testServerConnection ? params => testServerConnection.updateSnapshot(params) : undefined} />
} diff --git a/packages/trace-viewer/src/ui/workbench.tsx b/packages/trace-viewer/src/ui/workbench.tsx index ab31fc558b638..838616aea0059 100644 --- a/packages/trace-viewer/src/ui/workbench.tsx +++ b/packages/trace-viewer/src/ui/workbench.tsx @@ -32,6 +32,7 @@ import { Timeline } from './timeline'; import { usePlayback, PlaybackScrubber } from './playbackControl'; import { MetadataView } from './metadataView'; import { AttachmentsTab } from './attachmentsTab'; +import type { UpdateSnapshot } from './attachmentsTab'; import { AnnotationsTab } from './annotationsTab'; import type { Boundaries } from './geometry'; import { InspectorTab } from './inspectorTab'; @@ -61,6 +62,7 @@ export type WorkbenchProps = { defaultAnnotations?: TestAnnotation[]; inert?: boolean; onOpenExternally?: (location: SourceLocation) => void; + onUpdateSnapshot?: UpdateSnapshot; revealSource?: boolean; testRunMetadata?: MetadataWithCommitInfo; }; @@ -73,7 +75,7 @@ export const Workbench: React.FunctionComponent = props => { }; const PartitionedWorkbench: React.FunctionComponent = props => { - const { partition, model, showSourcesFirst, rootDir, fallbackLocation, isLive, hideTimeline, status, inert, onOpenExternally, revealSource, testRunMetadata } = props; + const { partition, model, showSourcesFirst, rootDir, fallbackLocation, isLive, hideTimeline, status, inert, onOpenExternally, onUpdateSnapshot, revealSource, testRunMetadata } = props; // Default annotations come from the test model before the test runs, shown for the empty workbench / trace. const annotations = model?.annotations ?? props.defaultAnnotations; @@ -282,7 +284,10 @@ const PartitionedWorkbench: React.FunctionComponent + render: () => }; const tabs: TabbedPaneTabModel[] = [ diff --git a/tests/playwright-test/test-server.spec.ts b/tests/playwright-test/test-server.spec.ts index 5cc9544da8967..0075bac086a1a 100644 --- a/tests/playwright-test/test-server.spec.ts +++ b/tests/playwright-test/test-server.spec.ts @@ -17,7 +17,9 @@ // @ts-nocheck import { test as baseTest, expect } from './ui-mode-fixtures'; +import { createImage } from './playwright-test-fixtures'; import { TestServerConnection } from '../../packages/playwright/lib/isomorphic'; +import fs from 'fs'; import ws from 'ws'; import type { TestChildProcess } from '../config/commonFixtures'; @@ -172,6 +174,80 @@ test('stdio interception', async ({ startTestServer, writeFiles }) => { ])); }); +for (const side of ['actual', 'expected']) { + for (const duplicate of ['file', 'body before', 'body after']) { + test(`should reject saving a snapshot with duplicate ${side} attachments (${duplicate})`, async ({ startTestServer, writeFiles, deleteFile }, testInfo) => { + const expected = createImage(10, 10, 255, 0, 0); + const actual = createImage(10, 10, 0, 255, 0); + await writeFiles({ + 'playwright.config.ts': `export default { snapshotPathTemplate: '{arg}{ext}' };`, + 'foo.png': expected, + 'bar.png': expected, + 'actual.png': actual, + 'duplicate.png': createImage(10, 10, 0, 0, 255), + 'duplicate.txt': '', + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + import fs from 'fs'; + import path from 'path'; + + test('snapshots', async ({}, testInfo) => { + const hasDuplicate = fs.existsSync(path.join(__dirname, 'duplicate.txt')); + const duplicatePath = path.join(__dirname, 'duplicate.png'); + if (hasDuplicate && '${duplicate}' === 'body before') + await testInfo.attach('foo-${side}.png', { body: fs.readFileSync(duplicatePath), contentType: 'image/png' }); + const actual = fs.readFileSync(path.join(__dirname, 'actual.png')); + expect.soft(actual).toMatchSnapshot('foo.png'); + expect.soft(actual).toMatchSnapshot('bar.png'); + if (hasDuplicate && '${duplicate}' === 'body after') + await testInfo.attach('foo-${side}.png', { body: fs.readFileSync(duplicatePath), contentType: 'image/png' }); + if (hasDuplicate && '${duplicate}' === 'file') { + await testInfo.attach('foo-${side}.png', { path: duplicatePath }); + await testInfo.attach('foo-${side}.png', { path: duplicatePath }); + } + await testInfo.attach('bar-${side}.png', { body: 'a different content type', contentType: 'text/plain' }); + }); + `, + }); + + const connection = await startTestServer(); + await connection.initialize({}); + expect(await connection.runTests({ locations: [], updateSnapshots: 'none' })).toEqual({ status: 'failed' }); + const { testId, result } = connection.events.find(([event, message]) => event === 'report' && message.method === 'onTestBegin')[1].params; + const params = { + testId, + resultId: result.id, + actual: { name: 'foo-actual.png', contentType: 'image/png' }, + expected: { name: 'foo-expected.png', contentType: 'image/png' }, + }; + const attachmentPaths = connection.events + .filter(([event, message]) => event === 'report' && message.method === 'onAttach') + .flatMap(([, message]) => message.params.attachments) + .filter(attachment => attachment.path) + .map(attachment => attachment.path); + const contents = attachmentPaths.map(file => fs.readFileSync(file)); + await expect(connection.updateSnapshot(params)).rejects.toThrow('Snapshot attachments are not registered or have duplicates for this test result'); + expect(attachmentPaths.map(file => fs.readFileSync(file))).toEqual(contents); + + await connection.updateSnapshot({ + ...params, + actual: { name: 'bar-actual.png', contentType: 'image/png' }, + expected: { name: 'bar-expected.png', contentType: 'image/png' }, + }); + expect(fs.readFileSync(testInfo.outputPath('bar.png'))).toEqual(actual); + expect(fs.readFileSync(testInfo.outputPath('foo.png'))).toEqual(expected); + + await deleteFile('duplicate.txt'); + expect(await connection.runTests({ locations: [], updateSnapshots: 'none' })).toEqual({ status: 'failed' }); + await expect(connection.updateSnapshot(params)).rejects.toThrow('Snapshot attachments are not registered or have duplicates for this test result'); + expect(fs.readFileSync(testInfo.outputPath('foo.png'))).toEqual(expected); + const nextResult = connection.events.filter(([event, message]) => event === 'report' && message.method === 'onTestBegin').at(-1)[1].params.result; + await connection.updateSnapshot({ ...params, resultId: nextResult.id }); + expect(fs.readFileSync(testInfo.outputPath('foo.png'))).toEqual(actual); + }); + } +} + test('find related test files errors', async ({ startTestServer, writeFiles }) => { await writeFiles({ 'a.spec.ts': ` diff --git a/tests/playwright-test/ui-mode-trace.spec.ts b/tests/playwright-test/ui-mode-trace.spec.ts index 14aa61d3e93a2..b624f1e021b93 100644 --- a/tests/playwright-test/ui-mode-trace.spec.ts +++ b/tests/playwright-test/ui-mode-trace.spec.ts @@ -20,6 +20,7 @@ import * as path from 'path'; import { createImage } from './playwright-test-fixtures'; import { test, expect, retries } from './ui-mode-fixtures'; +import { iso } from '../../packages/playwright-core/lib/coreBundle'; test.describe.configure({ mode: 'parallel', retries }); @@ -242,18 +243,24 @@ test('should show snapshots for steps', { }); test('should show image diff', async ({ runUITest }) => { - const { page } = await runUITest({ + const firstExpected = createImage(100, 100, 255, 0, 0); + const secondExpected = createImage(100, 100, 0, 255, 0); + const { page, testProcess } = await runUITest({ 'playwright.config.js': ` module.exports = { - snapshotPathTemplate: '{arg}{ext}' + snapshotPathTemplate: 'snapshots/{testFilePath}/{arg}{ext}' }; `, - 'snapshot.png': createImage(100, 100, 255, 0, 0), + 'snapshots/a.test.ts/first.png': firstExpected, + 'snapshots/a.test.ts/second.png': secondExpected, 'a.test.ts': ` import { test, expect } from '@playwright/test'; test('vrt test', async ({ page }) => { await page.setViewportSize({ width: 100, height: 100 }); - await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + await page.setContent(''); + await expect.soft(page).toHaveScreenshot('first.png', { timeout: 2000 }); + await page.setContent(''); + await expect.soft(page).toHaveScreenshot('second.png', { timeout: 2000 }); }); `, }); @@ -262,10 +269,90 @@ test('should show image diff', async ({ runUITest }) => { await expect(page.getByTestId('workbench-run-status')).toContainText('Failed'); await page.getByText(/Attachments/).click(); - await expect(page.getByText('Diff', { exact: true })).toBeVisible(); - await expect(page.getByText('Actual', { exact: true })).toBeVisible(); - await expect(page.getByText('Expected', { exact: true })).toBeVisible(); - await expect(page.getByTestId('test-result-image-mismatch').locator('img')).toBeVisible(); + await expect(page.getByText('Diff', { exact: true })).toHaveCount(2); + await expect(page.getByText('Actual', { exact: true })).toHaveCount(2); + await expect(page.getByText('Expected', { exact: true })).toHaveCount(2); + await expect(page.getByTestId('test-result-image-mismatch')).toHaveCount(2); + + const secondActual = await page.getByRole('link', { name: 'second-actual.png' }).evaluate(async link => { + const response = await fetch((link as HTMLAnchorElement).href); + return [...new Uint8Array(await response.arrayBuffer())]; + }); + expect(Buffer.from(secondActual)).not.toEqual(secondExpected); + + const updateSnapshots = page.locator('.attachments-update-snapshot'); + await expect(updateSnapshots).toHaveCount(2); + await expect(updateSnapshots).toHaveText(['Save actual as expected', 'Save actual as expected']); + await updateSnapshots.nth(1).click(); + await expect(updateSnapshots).toHaveText(['Save actual as expected', 'Save actual as expected']); + await expect(updateSnapshots.nth(1).locator('.codicon-check')).toBeVisible(); + await expect(updateSnapshots.nth(1)).toBeEnabled(); + await expect(updateSnapshots.nth(1).locator('.codicon-check')).toBeHidden(); + + const snapshotDir = path.join(testProcess.params.cwd!, 'snapshots', 'a.test.ts'); + expect(fs.readFileSync(path.join(snapshotDir, 'first.png'))).toEqual(firstExpected); + expect(fs.readFileSync(path.join(snapshotDir, 'second.png'))).toEqual(Buffer.from(secondActual)); +}); + +test('should only save snapshots for the loaded test result', async ({ runUITest }, testInfo) => { + const expected = createImage(10, 10, 255, 0, 0); + const firstActual = createImage(10, 10, 0, 255, 0); + const secondActual = createImage(10, 10, 0, 0, 255); + const { page } = await runUITest({ + 'playwright.config.ts': ` + export default { + outputDir: 'output', + snapshotPathTemplate: 'snapshots/{testName}/{arg}{ext}', + }; + `, + 'snapshots/first/shared.png': expected, + 'snapshots/second/shared.png': expected, + 'first.png': firstActual, + 'second.png': secondActual, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + import fs from 'fs'; + import path from 'path'; + for (const name of ['first', 'second']) { + test(name, () => { + expect(fs.readFileSync(path.join(__dirname, name + '.png'))).toMatchSnapshot('shared.png'); + }); + } + `, + }); + + const traceRequested = new iso.ManualPromise(); + const releaseTrace = new iso.ManualPromise(); + await page.context().route('**/file?*', async route => { + if (new URL(route.request().url()).searchParams.get('path') === testInfo.outputPath('output', 'a-second', 'trace.zip')) { + traceRequested.resolve(); + await releaseTrace; + } + await route.continue(); + }); + + try { + await page.getByTitle('Run all').click(); + await expect(page.getByTestId('status-line')).toContainText('2 failed'); + await page.getByTestId('test-tree').getByText('first', { exact: true }).click(); + await page.getByRole('tab', { name: 'Attachments' }).click(); + const save = page.getByRole('button', { name: 'Save actual as expected' }); + await expect(save).toBeVisible(); + + await page.getByTestId('test-tree').getByText('second', { exact: true }).click(); + await traceRequested; + await expect(save).toHaveCount(0); + expect(fs.readFileSync(testInfo.outputPath('snapshots', 'first', 'shared.png'))).toEqual(expected); + expect(fs.readFileSync(testInfo.outputPath('snapshots', 'second', 'shared.png'))).toEqual(expected); + + releaseTrace.resolve(); + await save.click(); + await expect(save.locator('.codicon-check')).toBeVisible(); + expect(fs.readFileSync(testInfo.outputPath('snapshots', 'first', 'shared.png'))).toEqual(expected); + expect(fs.readFileSync(testInfo.outputPath('snapshots', 'second', 'shared.png'))).toEqual(secondActual); + } finally { + releaseTrace.resolve(); + } }); test('should show screenshot', async ({ runUITest }) => {