ci: capture iOS E2E failure diagnostics

.buildkite/pipeline.yml

@@ -199,7 +199,17 @@ steps:
       command: |
           buildkite-agent artifact download dist.tar.gz .
           tar -xzf dist.tar.gz
-          make test-ios-app-e2e
+          # Keep the result bundle even when tests fail: its screenshots and
+          # UI hierarchy are the only record of what the editor showed.
+          TEST_STATUS=0
+          make test-ios-app-e2e || TEST_STATUS=$$?
+          if [ -d build/ios-e2e.xcresult ]; then
+              echo "--- :package: Zipping test results"
+              (cd build && zip -rq ios-e2e.xcresult.zip ios-e2e.xcresult)
+          fi
+          exit "$$TEST_STATUS"
+      artifact_paths:
+          - 'build/ios-e2e.xcresult.zip'
       plugins: *plugins
 
     - label: ':android: Test Android E2E'

Makefile

@@ -26,6 +26,7 @@
 .DEFAULT_GOAL := help
 
 SIMULATOR_DESTINATION := OS=latest,name=iPhone 17
+IOS_E2E_RESULT_BUNDLE := build/ios-e2e.xcresult
 
 .PHONY: help
 help: ## Display this help menu
@@ -370,12 +371,15 @@ test-ios-app-e2e: ## Run iOS demo app E2E tests against the production build
 	@echo "--- :open_file_folder: Copying build into iOS bundle"
 	@$(MAKE) copy-ios-dist
 	@echo "--- :ios: Running iOS E2E Tests (production build)"
+# xcodebuild refuses to overwrite an existing result bundle.
+	@rm -rf '$(IOS_E2E_RESULT_BUNDLE)'
 	@set -o pipefail && \
 		xcodebuild test \
 		-project ./ios/Demo-iOS/Gutenberg.xcodeproj \
 		-scheme GutenbergUITests \
 		-sdk iphonesimulator \
 		-destination '${SIMULATOR_DESTINATION}' \
+		-resultBundlePath '$(IOS_E2E_RESULT_BUNDLE)' \
 		| xcbeautify
 
 .PHONY: test-ios-app-e2e-dev

ios/Demo-iOS/GutenbergUITests/EditorUITestHelpers.swift

@@ -23,10 +23,33 @@ enum EditorUITestHelpers {
 
         // Wait for the WebView to appear in the full-screen editor.
         let webView = app.webViews.firstMatch
-        XCTAssertTrue(webView.waitForExistence(timeout: 30), "Expected a WKWebView to appear after editor loads")
+        if !webView.waitForExistence(timeout: 30) {
+            attachScreenState(app: app)
+            // The crash notice hides the WebView, so name it rather than
+            // reporting a WebView that never appeared.
+            let message = app.buttons["Reload Editor"].exists
+                ? "Editor crashed while loading; the crash notice replaced the WKWebView"
+                : "Expected a WKWebView to appear after editor loads"
+            XCTFail(message)
+        }
         return webView
     }
 
+    /// Attaches a screenshot and the accessibility hierarchy to the result
+    /// bundle, so a CI failure records what the app showed.
+    private static func attachScreenState(app: XCUIApplication) {
+        XCTContext.runActivity(named: "Screen state") { activity in
+            let screenshot = XCTAttachment(screenshot: app.screenshot())
+            screenshot.lifetime = .keepAlways
+            activity.add(screenshot)
+
+            let hierarchy = XCTAttachment(string: app.debugDescription)
+            hierarchy.name = "Accessibility hierarchy"
+            hierarchy.lifetime = .keepAlways
+            activity.add(hierarchy)
+        }
+    }
+
     /// Types text into the title field and returns the field element.
     @discardableResult
     static func typeInTitle(_ text: String, webView: XCUIElement) -> XCUIElement {