diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1464179d7..82d93398b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -199,7 +199,17 @@ steps: command: | buildkite-agent artifact download dist.tar.gz . tar -xzf dist.tar.gz - make test-ios-app-e2e + # Keep the result bundle even when tests fail: its screenshots and + # UI hierarchy are the only record of what the editor showed. + TEST_STATUS=0 + make test-ios-app-e2e || TEST_STATUS=$$? + if [ -d build/ios-e2e.xcresult ]; then + echo "--- :package: Zipping test results" + (cd build && zip -rq ios-e2e.xcresult.zip ios-e2e.xcresult) + fi + exit "$$TEST_STATUS" + artifact_paths: + - 'build/ios-e2e.xcresult.zip' plugins: *plugins - label: ':android: Test Android E2E' diff --git a/Makefile b/Makefile index 385570eec..cf51ad4b8 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ .DEFAULT_GOAL := help SIMULATOR_DESTINATION := OS=latest,name=iPhone 17 +IOS_E2E_RESULT_BUNDLE := build/ios-e2e.xcresult .PHONY: help help: ## Display this help menu @@ -370,12 +371,15 @@ test-ios-app-e2e: ## Run iOS demo app E2E tests against the production build @echo "--- :open_file_folder: Copying build into iOS bundle" @$(MAKE) copy-ios-dist @echo "--- :ios: Running iOS E2E Tests (production build)" +# xcodebuild refuses to overwrite an existing result bundle. + @rm -rf '$(IOS_E2E_RESULT_BUNDLE)' @set -o pipefail && \ xcodebuild test \ -project ./ios/Demo-iOS/Gutenberg.xcodeproj \ -scheme GutenbergUITests \ -sdk iphonesimulator \ -destination '${SIMULATOR_DESTINATION}' \ + -resultBundlePath '$(IOS_E2E_RESULT_BUNDLE)' \ | xcbeautify .PHONY: test-ios-app-e2e-dev diff --git a/ios/Demo-iOS/GutenbergUITests/EditorUITestHelpers.swift b/ios/Demo-iOS/GutenbergUITests/EditorUITestHelpers.swift index 84031afd5..d3beb41bf 100644 --- a/ios/Demo-iOS/GutenbergUITests/EditorUITestHelpers.swift +++ b/ios/Demo-iOS/GutenbergUITests/EditorUITestHelpers.swift @@ -23,10 +23,33 @@ enum EditorUITestHelpers { // Wait for the WebView to appear in the full-screen editor. let webView = app.webViews.firstMatch - XCTAssertTrue(webView.waitForExistence(timeout: 30), "Expected a WKWebView to appear after editor loads") + if !webView.waitForExistence(timeout: 30) { + attachScreenState(app: app) + // The crash notice hides the WebView, so name it rather than + // reporting a WebView that never appeared. + let message = app.buttons["Reload Editor"].exists + ? "Editor crashed while loading; the crash notice replaced the WKWebView" + : "Expected a WKWebView to appear after editor loads" + XCTFail(message) + } return webView } + /// Attaches a screenshot and the accessibility hierarchy to the result + /// bundle, so a CI failure records what the app showed. + private static func attachScreenState(app: XCUIApplication) { + XCTContext.runActivity(named: "Screen state") { activity in + let screenshot = XCTAttachment(screenshot: app.screenshot()) + screenshot.lifetime = .keepAlways + activity.add(screenshot) + + let hierarchy = XCTAttachment(string: app.debugDescription) + hierarchy.name = "Accessibility hierarchy" + hierarchy.lifetime = .keepAlways + activity.add(hierarchy) + } + } + /// Types text into the title field and returns the field element. @discardableResult static func typeInTitle(_ text: String, webView: XCUIElement) -> XCUIElement {