diff --git a/.github/workflows/_temp-test-investigate-prompt.yml b/.github/workflows/_temp-test-investigate-prompt.yml new file mode 100644 index 000000000..2b4a12994 --- /dev/null +++ b/.github/workflows/_temp-test-investigate-prompt.yml @@ -0,0 +1,107 @@ +name: TEMP - Investigate-prompt iteration (delete after use) + +# Scratch workflow for iterating on the "Investigate failure with Claude" +# prompt used in doc-detective-screenshots.yml. Deliberately breaks a real +# selector to produce a genuine, realistic failure, then runs the exact +# investigate-prompt logic against it so we can judge diagnosis/fix quality +# without waiting on a real failure or merging the real workflow changes. +# Delete this file once a good prompt is found — it is not meant to be +# long-lived. + +on: + workflow_dispatch: {} + pull_request: + paths: + - '.github/workflows/_temp-test-investigate-prompt.yml' + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + test-investigate: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: true + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: Install dependencies + run: npm ci + + - name: Create .env file for preprod + run: | + echo "CLOUD_BASE_URL=https://preprod--redpanda-cloud.netlify.app" >> tests/doc-detective/.env + echo "CLOUD_LOGIN_EMAIL=${{ secrets.PREPROD_EMAIL }}" >> tests/doc-detective/.env + echo "CLOUD_LOGIN_PASSWORD=${{ secrets.PREPROD_PASSWORD }}" >> tests/doc-detective/.env + echo "CLOUD_CLUSTER_ID=d9f1nf0d6v8fd1orsa00" >> tests/doc-detective/.env + + # Deliberately breaks one real step so the test fails for a plausible, + # realistic reason (a UI label changing) — never committed, only exists + # in this ephemeral checkout. + - name: Deliberately break a selector to create a realistic failure + run: | + sed -i 's/"elementText":"Add input"/"elementText":"Add Input Source"/' modules/develop/pages/connect/connect-quickstart.adoc + grep -n "Add Input Source" modules/develop/pages/connect/connect-quickstart.adoc + + - name: Run inline tests (expected to fail) + id: run_tests + continue-on-error: true + run: node tests/doc-detective/run-inline.js + + - name: Investigate failure with Claude + if: steps.run_tests.outcome == 'failure' + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GH_TOKEN: ${{ github.token }} + ENVIRONMENT: preprod + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + REPO: ${{ github.repository }} + run: | + cat > /tmp/investigate-prompt.md <\` per the README's screenshot standards, and view the result before committing. +3. Create a branch, commit exactly the files that needed to change (the .adoc file and/or the regenerated screenshot(s) — nothing unrelated), push it, and open a PR with \`gh pr create\`. The PR body must state: what failed, why (your diagnosis), what you changed, and that you personally re-ran the test and saw it pass. Include the before/after screenshot inline in the PR body if one changed. +4. Label the PR \`doc-detective,${ENVIRONMENT},prompt-test\`. +5. Never widen \`maxVariation\`, delete a step, or otherwise weaken what the test actually verifies just to make it pass — that defeats the point of it existing. + +**B. Transient, or real-but-not-fixable-here** — do not open a PR. Instead, print a clear paragraph explaining your diagnosis (there is no real issue to close in this test run — just state what you would have said in the issue-close comment). + +Produce exactly one outcome: a PR, or a printed diagnosis. Not both, not neither. +PROMPT + + claude -p "$(cat /tmp/investigate-prompt.md)" \ + --permission-mode bypassPermissions \ + --effort high \ + --output-format json \ + --max-budget-usd 3