Onboard Build Failure Analysis

.github/aw/actions-lock.json

@@ -35,15 +35,22 @@
       "version": "v0.76.1",
       "sha": "46d564922b082d0db93244972e8005ea6904ee5f"
     },
-    "github/gh-aw-actions/setup@v0.88.2": {
+    "github/gh-aw-actions/setup@v0.89.21": {
       "repo": "github/gh-aw-actions/setup",
-      "version": "v0.88.2",
-      "sha": "9271a1804551c0dc4fb0085a97979950aa2f8489"
+      "version": "v0.89.21",
+      "sha": "924af5fdc64061cfbf66fb584c8b07e2ac230c60"
     },
     "github/gh-aw/actions/setup@v0.67.2": {
       "repo": "github/gh-aw/actions/setup",
       "version": "v0.67.2",
       "sha": "5fb582bad4dfee2841251e97f9d22caa7eb3c82d"
     }
+  },
+  "containers": {
+    "mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-binlog-mcp-amd64": {
+      "image": "mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-binlog-mcp-amd64",
+      "digest": "sha256:ca760fd18bf8e6168d9b304d65c5d2340e4135fc9080c6e2f2eb4800368e3df4",
+      "pinned_image": "mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-binlog-mcp-amd64@sha256:ca760fd18bf8e6168d9b304d65c5d2340e4135fc9080c6e2f2eb4800368e3df4"
+    }
   }
 }

.github/workflows/build-failure-analysis-command.md

@@ -0,0 +1,246 @@
+---
+name: "Build Failure Analysis (command)"
+run-name: "Build failure analysis command ${{ github.event.comment.id }}"
+description: >-
+  Reruns build-failure analysis when a maintainer comments
+  `/analyze-build-failure` on a pull request. It analyzes only that PR's newest
+  completed, failed `fsharp-ci` build and attempts a checked branch fix or a
+  contributor-reviewable patch.
+
+on:
+  slash_command:
+    name: analyze-build-failure
+    events: [pull_request_comment]
+  roles: [admin, maintainer, write]
+  reaction: "eyes"
+  needs: [fetch-binlog]
+
+if: needs.fetch-binlog.outputs.binlog-found == 'true'
+
+permissions:
+  contents: read
+  pull-requests: read
+
+# Queue command analyses for one PR. A successful safe-output publication is
+# deduplicated by the exact comment id below; a new comment requests a new run.
+concurrency:
+  group: ${{ (github.event_name == 'issue_comment' && !startsWith(github.event.comment.body, '/analyze-build-failure') && format('build-failure-analysis-cmd-run-{0}', github.run_id)) || format('build-failure-analysis-cmd-{0}', github.event.issue.number || github.event.pull_request.number || fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number || github.run_id) }}
+  cancel-in-progress: false
+  queue: max
+
+timeout-minutes: 30
+
+imports:
+  - shared/build-failure-analysis-shared.md
+
+engine: copilot
+
+jobs:
+  fetch-binlog:
+    name: Fetch binlogs (Azure Pipelines)
+    if: >-
+      github.event.repository.fork == false &&
+      github.event.issue.pull_request &&
+      contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) &&
+      startsWith(github.event.comment.body, '/analyze-build-failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      actions: read
+      contents: read
+      pull-requests: read
+    outputs:
+      binlog-found: ${{ steps.upload.outcome == 'success' && steps.upload.outputs.artifact-id != '' }}
+      pr-number: ${{ steps.fetch.outputs.pr-number }}
+      pr-head-sha: ${{ steps.fetch.outputs.pr-head-sha }}
+      pr-merge-sha: ${{ steps.fetch.outputs.pr-merge-sha }}
+      pr-head-ref: ${{ steps.fetch.outputs.pr-head-ref }}
+      pr-head-repo: ${{ steps.fetch.outputs.pr-head-repo }}
+      ado-build-id: ${{ steps.fetch.outputs.ado-build-id }}
+      ado-build-url: ${{ steps.fetch.outputs.ado-build-url }}
+    steps:
+      # author_association cannot distinguish an organization member with read
+      # access from a repository maintainer. Resolve the actual base permission
+      # before downloading any artifacts and fail closed on API errors.
+      - name: Verify the command and commenter permission
+        id: permission
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          COMMENTER: ${{ github.event.comment.user.login }}
+          COMMENT_BODY: ${{ github.event.comment.body }}
+          COMMAND_NAME: "analyze-build-failure"
+        run: |
+          set +e
+          case "${COMMENT_BODY}" in
+            "/${COMMAND_NAME}" | "/${COMMAND_NAME} "* | "/${COMMAND_NAME}"$'\n'*) ;;
+            *)
+              echo "Comment does not match the slash-command activation predicate."
+              echo "authorized=false" >> "$GITHUB_OUTPUT"
+              exit 0
+              ;;
+          esac
+          if [[ ! "${COMMENTER}" =~ ^[A-Za-z0-9-]+$ ]]; then
+            echo "::warning::Commenter login is missing or malformed."
+            echo "authorized=false" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          response=$(gh api "repos/${GITHUB_REPOSITORY}/collaborators/${COMMENTER}/permission" 2>/dev/null)
+          permission=$(printf '%s' "${response}" | jq -r '.permission // empty' 2>/dev/null)
+          case "${permission}" in
+            admin|write) authorized=true ;;
+            *) authorized=false ;;
+          esac
+          if [ "${authorized}" = "true" ]; then
+            echo "'${COMMENTER}' has '${permission}' access; proceeding."
+          else
+            echo "::warning::'${COMMENTER}' does not have write access (resolved permission '${permission:-none}')."
+          fi
+          echo "authorized=${authorized}" >> "$GITHUB_OUTPUT"
+
+      - name: Check for completed command publication
+        id: command
+        if: steps.permission.outputs.authorized == 'true'
+        uses: actions/github-script@v9.0.0
+        env:
+          WORKFLOW_FILE: build-failure-analysis-command.lock.yml
+        with:
+          script: |
+            const comment = context.payload.comment;
+            const title = `Build failure analysis command ${comment.id}`;
+            for (let page = 1; ; page++) {
+              const { data } = await github.rest.actions.listWorkflowRuns({
+                ...context.repo,
+                workflow_id: process.env.WORKFLOW_FILE,
+                event: "issue_comment",
+                status: "completed",
+                created: `>=${comment.created_at}`,
+                per_page: 100,
+                page,
+              });
+              if (data.total_count > 1000) {
+                throw new Error("Command history exceeds the GitHub search limit; post a new command.");
+              }
+              for (const run of data.workflow_runs) {
+                if (run.display_title !== title) {
+                  continue;
+                }
+                const jobs = await github.paginate(
+                  github.rest.actions.listJobsForWorkflowRunAttempt,
+                  {
+                    ...context.repo,
+                    run_id: run.id,
+                    attempt_number: run.run_attempt,
+                    per_page: 100,
+                  });
+                const agentCompleted = jobs.some(job =>
+                  job.name === "agent" &&
+                  job.conclusion === "success" &&
+                  job.steps?.some(step =>
+                    step.name === "Execute GitHub Copilot CLI" &&
+                    step.conclusion === "success"));
+                const outputsCompleted = jobs.some(job =>
+                  job.name === "safe_outputs" &&
+                  job.conclusion === "success" &&
+                  job.steps?.some(step =>
+                    step.name === "Process Safe Outputs" &&
+                    step.conclusion === "success"));
+                if (agentCompleted && outputsCompleted) {
+                  core.notice("This command completed publication; post a new command to rerun.");
+                  core.setOutput("completed", "true");
+                  return;
+                }
+              }
+              if (data.workflow_runs.length < 100 || page * 100 >= data.total_count) {
+                break;
+              }
+            }
+            core.setOutput("completed", "false");
+
+      - name: Check out analysis scripts
+        if: steps.command.outputs.completed == 'false'
+        uses: actions/checkout@v7.0.1
+        with:
+          ref: refs/heads/${{ github.event.repository.default_branch }}
+          sparse-checkout: |
+            .github/agents
+            .github/workflows/scripts
+          persist-credentials: false
+
+      - name: Download binlogs from the PR's latest failed Azure Pipelines build
+        id: fetch
+        if: steps.command.outputs.completed == 'false'
+        shell: bash
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_AW_REPO: ${{ github.repository }}
+          RESOLVE_MODE: latest
+          PR_NUMBER: ${{ github.event.issue.number }}
+          BINLOG_DIR: /tmp/binlogs
+          SCRIPT_DIR: ${{ github.workspace }}/.github/workflows/scripts
+        run: cd "${RUNNER_TEMP:-/tmp}" && timeout 600 dotnet fsi --exec "${SCRIPT_DIR}/fetch-build-binlogs.fsx"
+
+      - name: Report an incomplete fetch
+        if: steps.fetch.outcome == 'failure'
+        run: echo "::warning::Binlog fetch did not complete; skipping analysis for this command."
+
+      - name: Stage the trusted analyst playbook
+        if: steps.fetch.outputs.binlog-found == 'true'
+        run: cp .github/agents/build-failure-analyst.agent.md /tmp/binlogs/
+
+      - name: Upload analysis artifact
+        id: upload
+        if: steps.fetch.outputs.binlog-found == 'true'
+        uses: actions/upload-artifact@v7.0.1
+        with:
+          name: build-failure-analysis-data
+          path: /tmp/binlogs
+          if-no-files-found: error
+          retention-days: 1
+
+safe-outputs:
+  needs: [fetch-binlog]
+  max-patch-files: 25
+  messages:
+    footer: "> **Automated content by GitHub Copilot.** Generated by the [{workflow_name}]({agentic_workflow_url}) workflow.{ai_credits_suffix} | [History]({history_link}) | [Request](${{ github.event.comment.html_url }})"
+  data:
+    type: object
+    properties:
+      workflow_artifact:
+        type: string
+        enum: [build-failure-analysis]
+      artifact_kind:
+        type: string
+        enum: [analysis]
+      fix_status:
+        type: string
+        enum: [validated, failed, blocked, not-needed]
+      fix_delivery:
+        type: string
+        enum: [push, patch, none]
+    required: [workflow_artifact, artifact_kind, fix_status, fix_delivery]
+    additionalProperties: false
+  report-failure-as-issue: false
+  add-comment:
+    max: 1
+    target: ${{ needs.fetch-binlog.outputs.pr-number }}
+    hide-older-comments: true
+  push-to-pull-request-branch:
+    max: 1
+    target: ${{ needs.fetch-binlog.outputs.pr-number }}
+    if-no-changes: error
+    fallback-as-pull-request: false
+    protected-files: blocked
+    max-patch-size: 64
+  create-pull-request-review-comment:
+    max: 3
+    target: ${{ needs.fetch-binlog.outputs.pr-number }}
+    side: RIGHT
+    commit-id: ${{ needs.fetch-binlog.outputs.pr-head-sha }}
+  noop:
+    max: 1
+    report-as-issue: false
+---
+
+<!-- Body provided by shared/build-failure-analysis-shared.md. -->

.github/workflows/build-failure-analysis-test-failure.yml

@@ -0,0 +1,111 @@
+name: Build failure workflow test notifications
+
+on:
+  workflow_run:
+    workflows: [Build failure workflow tests]
+    types: [completed]
+
+permissions: {}
+
+concurrency:
+  group: build-failure-analysis-test-notifications
+  cancel-in-progress: false
+
+jobs:
+  notify:
+    if: >-
+      github.repository == 'dotnet/fsharp' &&
+      contains(fromJSON('["failure","timed_out","startup_failure"]'), github.event.workflow_run.conclusion)
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+      pull-requests: read
+    steps:
+      # workflow_run uses the default-branch script, never failed PR code/artifacts.
+      - name: Notify the PR author or repository owners
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          script: |
+            const run = context.payload.workflow_run;
+            if (run.path !== ".github/workflows/build-failure-analysis-tests.yml" ||
+                !Number.isSafeInteger(run.id) || run.id < 1 ||
+                !Number.isSafeInteger(run.run_attempt) || run.run_attempt < 1 ||
+                !/^[a-f0-9]{40}$/.test(run.head_sha)) {
+              throw new Error("Unexpected regression workflow or run identity.");
+            }
+            const {data: current} = await github.rest.actions.getWorkflowRun({
+              ...context.repo, run_id: run.id,
+            });
+            if (current.run_attempt !== run.run_attempt || current.status !== "completed" ||
+                current.conclusion !== run.conclusion) {
+              core.notice("Skipping a regression attempt superseded by a rerun.");
+              return;
+            }
+            const marker = "<!-- build-failure-analysis-test-failure -->";
+            const owner = "@dotnet/fsharp-team-msft";
+            const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${run.id}`;
+            const body = `${marker}\n### Build-failure workflow regression\n\n` +
+              `[Regression run](${url}) failed (\`${run.conclusion}\`) at \`${run.head_sha}\`.\n\n` +
+              `${owner} owns triage under CODEOWNERS. Inspect the failed step, reproduce with ` +
+              "`python3 -m unittest discover -s .github/workflows/tests -v` and the pinned " +
+              "`gh aw compile build-failure-analysis build-failure-analysis-command --strict --validate`. " +
+              "Repair or revert the breaking change; do not disable the check or dismiss a reproducible failure as unrelated.";
+            const isReport = item => item.user?.login === "github-actions[bot]" &&
+              item.user?.type === "Bot" && item.body?.includes(marker);
+            if (run.event === "pull_request") {
+              const associated = run.pull_requests.length ? run.pull_requests :
+                await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, {
+                  ...context.repo, commit_sha: run.head_sha, per_page: 100,
+                });
+              if (associated.length > 10) {
+                throw new Error("Too many PRs associated with the failed regression run.");
+              }
+              for (const candidate of associated) {
+                const {data: pr} = await github.rest.pulls.get({
+                  ...context.repo, pull_number: candidate.number,
+                });
+                if (pr.state !== "open" || pr.head.sha !== run.head_sha ||
+                    pr.base.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`) {
+                  core.info(`Skipping stale or unrelated PR #${pr.number}.`);
+                  continue;
+                }
+                if (!/^[a-z0-9-]+$/i.test(pr.user.login)) {
+                  throw new Error("Invalid PR author identity.");
+                }
+                const comments = await github.paginate(github.rest.issues.listComments, {
+                  ...context.repo, issue_number: pr.number, per_page: 100,
+                });
+                const report = comments.find(isReport);
+                const prBody = `${body}\n\n@${pr.user.login}: as PR author, fix this before merge; ` +
+                  "the owners can help with shared tooling or infrastructure failures.";
+                if (report) {
+                  if (report.body !== prBody) {
+                    await github.rest.issues.updateComment({...context.repo, comment_id: report.id, body: prBody});
+                  }
+                } else {
+                  await github.rest.issues.createComment({...context.repo, issue_number: pr.number, body: prBody});
+                }
+              }
+              if (associated.length) {
+                return;
+              }
+              core.warning("GitHub supplied no associated PR; escalating the run to repository owners.");
+            } else if (!["push", "schedule", "workflow_dispatch"].includes(run.event) ||
+                       run.head_branch !== context.payload.repository.default_branch) {
+              core.notice("Ignoring a regression run outside the default branch.");
+              return;
+            }
+            const issues = await github.paginate(github.rest.issues.listForRepo, {
+              ...context.repo, state: "open", creator: "github-actions[bot]", per_page: 100,
+            });
+            const report = issues.find(item => !item.pull_request && isReport(item));
+            if (report) {
+              await github.rest.issues.update({...context.repo, issue_number: report.number, body});
+            } else {
+              await github.rest.issues.create({
+                ...context.repo, title: "Build-failure workflow regressions need owner triage", body,
+              });
+            }