From 1d8ba2d6c843833ba212632959bddd00b8ee3e33 Mon Sep 17 00:00:00 2001
From: JakeSCahill <jake@redpanda.com>
Date: Tue, 4 Aug 2026 11:52:14 +0100
Subject: [PATCH 1/3] Add temporary scratch workflow for iterating on the
 investigate-failure prompt

Deliberately breaks a real selector to produce a realistic failure, then runs
the same Claude-investigation logic being developed in PR #651. Delete once a
good prompt is found.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .../_temp-test-investigate-prompt.yml         | 104 ++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100644 .github/workflows/_temp-test-investigate-prompt.yml

diff --git a/.github/workflows/_temp-test-investigate-prompt.yml b/.github/workflows/_temp-test-investigate-prompt.yml
new file mode 100644
index 000000000..1188d2c53
--- /dev/null
+++ b/.github/workflows/_temp-test-investigate-prompt.yml
@@ -0,0 +1,104 @@
+name: TEMP - Investigate-prompt iteration (delete after use)
+
+# Scratch workflow for iterating on the "Investigate failure with Claude"
+# prompt used in doc-detective-screenshots.yml. Deliberately breaks a real
+# selector to produce a genuine, realistic failure, then runs the exact
+# investigate-prompt logic against it so we can judge diagnosis/fix quality
+# without waiting on a real failure or merging the real workflow changes.
+# Delete this file once a good prompt is found — it is not meant to be
+# long-lived.
+
+on:
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+
+jobs:
+  test-investigate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: true
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Create .env file for preprod
+        run: |
+          echo "CLOUD_BASE_URL=https://preprod--redpanda-cloud.netlify.app" >> tests/doc-detective/.env
+          echo "CLOUD_LOGIN_EMAIL=${{ secrets.PREPROD_EMAIL }}" >> tests/doc-detective/.env
+          echo "CLOUD_LOGIN_PASSWORD=${{ secrets.PREPROD_PASSWORD }}" >> tests/doc-detective/.env
+          echo "CLOUD_CLUSTER_ID=d9f1nf0d6v8fd1orsa00" >> tests/doc-detective/.env
+
+      # Deliberately breaks one real step so the test fails for a plausible,
+      # realistic reason (a UI label changing) — never committed, only exists
+      # in this ephemeral checkout.
+      - name: Deliberately break a selector to create a realistic failure
+        run: |
+          sed -i 's/"elementText":"Add input"/"elementText":"Add Input Source"/' modules/develop/pages/connect/connect-quickstart.adoc
+          grep -n "Add Input Source" modules/develop/pages/connect/connect-quickstart.adoc
+
+      - name: Run inline tests (expected to fail)
+        id: run_tests
+        continue-on-error: true
+        run: node tests/doc-detective/run-inline.js
+
+      - name: Investigate failure with Claude
+        if: steps.run_tests.outcome == 'failure'
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GH_TOKEN: ${{ github.token }}
+          ENVIRONMENT: preprod
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          REPO: ${{ github.repository }}
+        run: |
+          cat > /tmp/investigate-prompt.md <<PROMPT
+A Doc Detective test failed in this repo's Cloud console screenshot-testing framework, for the **${ENVIRONMENT}** environment. Run: ${RUN_URL}
+
+## Context to read first
+
+- tests/doc-detective/README.md — how this whole system works, including known gotchas (a real AsciiDoc passthrough-macro bug, a non-greedy inline-comment parser, a \$HOME/\$END token collision) that have caused real failures before. Check these before assuming a selector problem.
+- tests/doc-detective/last-inline-report.json — this run's result, already redacted of credentials. Find the FAIL/SKIPPED step(s) and read \`resultDescription\`.
+- modules/develop/pages/connect/connect-quickstart.adoc — the page under test (search for \`// (test\` / \`// (step\`).
+
+## Step 1: Reproduce
+
+Run \`node tests/doc-detective/run-inline.js\` yourself (the ${ENVIRONMENT} .env is already in tests/doc-detective/.env from an earlier step in this job). Confirm you see the same failure. If it passes now, run it two more times before concluding anything — a single pass after one failure is not enough to call it transient.
+
+## Step 2: Diagnose — transient or real?
+
+Decide which of these it is, and say so explicitly in whatever you produce next:
+
+- **Transient**: flaky timing, a screenshot comparison that landed just over the variation threshold by chance, a one-off network blip, or (${ENVIRONMENT} == prod only) the canvas feature flag genuinely not shipped there yet (expected, documented in the workflow's own comments).
+- **Real, fixable here**: a selector, button label, or UI text genuinely changed, and the .adoc file's steps/screenshot need updating to match. This is a normal doc/test maintenance fix.
+- **Real, not fixable here**: an actual product regression, a broken environment, or anything where the right fix is outside this repo (product code, infra). Do not attempt to patch product behavior or paper over a real bug by loosening the test.
+
+## Step 3: Act on your diagnosis — exactly one of these
+
+**A. Real and fixable** — make the minimal change to the .adoc file (selector, expected text, step wording only where the documented procedure itself changed) and/or let the screenshot step regenerate the screenshot. Then:
+1. Re-run \`node tests/doc-detective/run-inline.js\` again and confirm it now reports PASS with zero skipped. Do not open a PR on the strength of a guess — only after you've watched it pass.
+2. If a screenshot changed and is over 100KB, run \`node tests/doc-detective/fix-oversized-image.js <path>\` per the README's screenshot standards, and view the result before committing.
+3. Create a branch, commit exactly the files that needed to change (the .adoc file and/or the regenerated screenshot(s) — nothing unrelated), push it, and open a PR with \`gh pr create\`. The PR body must state: what failed, why (your diagnosis), what you changed, and that you personally re-ran the test and saw it pass. Include the before/after screenshot inline in the PR body if one changed.
+4. Label the PR \`doc-detective,${ENVIRONMENT},prompt-test\`.
+5. Never widen \`maxVariation\`, delete a step, or otherwise weaken what the test actually verifies just to make it pass — that defeats the point of it existing.
+
+**B. Transient, or real-but-not-fixable-here** — do not open a PR. Instead, print a clear paragraph explaining your diagnosis (there is no real issue to close in this test run — just state what you would have said in the issue-close comment).
+
+Produce exactly one outcome: a PR, or a printed diagnosis. Not both, not neither.
+PROMPT
+
+          claude -p "$(cat /tmp/investigate-prompt.md)" \
+            --permission-mode bypassPermissions \
+            --effort high \
+            --output-format json \
+            --max-budget-usd 3

From 3b1d4958ef4950a541a2673791baad5584115db7 Mon Sep 17 00:00:00 2001
From: JakeSCahill <jake@redpanda.com>
Date: Tue, 4 Aug 2026 11:59:33 +0100
Subject: [PATCH 2/3] Switch temp workflow trigger to pull_request so it runs
 on PR #662 without merging

---
 .github/workflows/_temp-test-investigate-prompt.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/_temp-test-investigate-prompt.yml b/.github/workflows/_temp-test-investigate-prompt.yml
index 1188d2c53..6ec2c341d 100644
--- a/.github/workflows/_temp-test-investigate-prompt.yml
+++ b/.github/workflows/_temp-test-investigate-prompt.yml
@@ -9,7 +9,9 @@ name: TEMP - Investigate-prompt iteration (delete after use)
 # long-lived.
 
 on:
-  workflow_dispatch: {}
+  pull_request:
+    paths:
+      - '.github/workflows/_temp-test-investigate-prompt.yml'
 
 permissions:
   contents: write

From 6c64bcaf4504daaf33ed485ec1d4651221d5d1b5 Mon Sep 17 00:00:00 2001
From: JakeSCahill <jake@redpanda.com>
Date: Tue, 4 Aug 2026 12:03:59 +0100
Subject: [PATCH 3/3] Add workflow_dispatch back now that the workflow is
 registered

---
 .github/workflows/_temp-test-investigate-prompt.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/_temp-test-investigate-prompt.yml b/.github/workflows/_temp-test-investigate-prompt.yml
index 6ec2c341d..2b4a12994 100644
--- a/.github/workflows/_temp-test-investigate-prompt.yml
+++ b/.github/workflows/_temp-test-investigate-prompt.yml
@@ -9,6 +9,7 @@ name: TEMP - Investigate-prompt iteration (delete after use)
 # long-lived.
 
 on:
+  workflow_dispatch: {}
   pull_request:
     paths:
       - '.github/workflows/_temp-test-investigate-prompt.yml'