diff --git a/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py b/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py index 88df0f5279..9abddce3bb 100755 --- a/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py +++ b/.agents/skills/analyze-ci-failure/scripts/analyze_ci_failure.py @@ -209,7 +209,7 @@ def create_plan(job_name, log_path, errors): else "🔍 **Classification**: **Code / Configuration Issue**" ) fix_advice = ( - f"Retry the failed job (`buildkite-retry-job`). {flake_reason}" + f"Retry the failed job (`retry-ci-job`). {flake_reason}" if is_flake else "Resolve the root cause in the relevant source / build files." ) diff --git a/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py b/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py deleted file mode 100755 index 168b1dacc4..0000000000 --- a/.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py +++ /dev/null @@ -1,615 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import json -import re -import subprocess -import sys -import time - -DEFAULT_GH_REPO = "bazel-contrib/rules_python" -GH_ACTIONS_URL_RE = re.compile( - r"github\.com/([^/]+/[^/]+)/actions/runs/(\d+)(?:/job/(\d+))?" -) - - -def check_cli(cmd_name, install_url): - try: - subprocess.run( - [cmd_name, "--version"], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - except Exception: - print( - f"❌ Error: '{cmd_name}' CLI is not installed or not in PATH.", - file=sys.stderr, - ) - print(f"Please install it from {install_url}", file=sys.stderr) - sys.exit(1) - - -def get_pr_checks(pr_number): - check_cli("gh", "https://cli.github.com/") - cmd = ["gh", "pr", "checks", str(pr_number), "--json", "name,link,state,bucket"] - res = subprocess.run(cmd, capture_output=True, text=True) - out = res.stdout - if "[" not in out: - print( - f"❌ Error fetching PR checks for PR #{pr_number}: {res.stderr}", - file=sys.stderr, - ) - sys.exit(1) - json_str = out[out.find("[") : out.rfind("]") + 1] - return json.loads(json_str) - - -def parse_github_actions_url(target): - m = GH_ACTIONS_URL_RE.search(target) - if m: - return m.group(1), m.group(2), m.group(3) - return None - - -def get_github_run_data(repo, run_id): - cmd = [ - "gh", - "run", - "view", - str(run_id), - "--repo", - repo, - "--json", - "status,conclusion,jobs", - ] - res = subprocess.run(cmd, capture_output=True, text=True, check=True) - out = res.stdout - json_str = out[out.find("{") : out.rfind("}") + 1] if "{" in out else "{}" - return json.loads(json_str) - - -def wait_for_github_run_completion( - repo, run_id, job_id=None, poll_interval=15, max_attempts=60 -): - for attempt in range(max_attempts): - try: - data = get_github_run_data(repo, str(run_id)) - except Exception as e: - print( - f"⚠ī¸ Warning: Could not inspect GitHub Actions run {run_id}: {e}", - file=sys.stderr, - ) - return None - - status = (data.get("status") or "").lower() - jobs = data.get("jobs", []) - - if job_id is not None: - active_job_ids = {str(j.get("databaseId")) for j in jobs} - if str(job_id) not in active_job_ids: - # The job ID from an earlier attempt is no longer in the active jobs - # list, meaning a newer run_attempt is already queued/running/done. - if status != "completed" or data.get("conclusion") == "success": - print( - f"ℹī¸ GitHub Actions job {job_id} has already been superseded " - f"by a newer attempt in run {run_id} (status: {status})." - ) - return data - - if status == "completed": - return data - - print( - f"âŗ GitHub Actions run {run_id} is currently '{status}' " - f"(attempt {attempt + 1}/{max_attempts}); waiting {poll_interval}s " - f"for run to complete before rerunning..." - ) - time.sleep(poll_interval) - - print( - f"⚠ī¸ Timed out waiting for GitHub Actions run {run_id} to complete.", - file=sys.stderr, - ) - return None - - -def retry_github_actions(repo, run_id, job_id=None, job_pattern=None, rebuild=False): - check_cli("gh", "https://cli.github.com/") - data = wait_for_github_run_completion(repo, run_id, job_id=job_id) - jobs = data.get("jobs", []) if isinstance(data, dict) else [] - - if rebuild: - print(f"🚀 Re-running entire GitHub Actions run {run_id} ({repo})...") - res = subprocess.run(["gh", "run", "rerun", str(run_id), "--repo", repo]) - return res.returncode - - if job_pattern: - matched = [] - for j in jobs: - name = j.get("name", "") - jid = j.get("databaseId") - conclusion = j.get("conclusion") or j.get("status") or "" - if jid and ( - re.search(job_pattern, name, re.IGNORECASE) - or job_pattern.lower() in name.lower() - ): - matched.append((str(jid), name, conclusion)) - - if not matched: - print( - f"⚠ī¸ No GitHub Actions jobs found matching '{job_pattern}' in run {run_id}.", - file=sys.stderr, - ) - return 1 - - rc = 0 - for jid, name, conclusion in matched: - print( - f"🚀 Retrying GitHub Actions job '{name}' " - f"(ID: {jid}, conclusion: {conclusion})..." - ) - res = subprocess.run(["gh", "run", "rerun", "--job", jid, "--repo", repo]) - if res.returncode != 0: - print( - f"⚠ī¸ Single-job rerun failed for {jid}; falling back to " - f"'gh run rerun {run_id} --failed'..." - ) - res = subprocess.run( - ["gh", "run", "rerun", str(run_id), "--failed", "--repo", repo] - ) - return res.returncode - return rc - - if job_id: - active_jobs_by_id = {str(j.get("databaseId")): j for j in jobs} - if data is not None and str(job_id) not in active_jobs_by_id: - status = (data.get("status") or "").lower() - conclusion = (data.get("conclusion") or "").lower() - if status != "completed" or conclusion == "success": - return 0 - - target_job = active_jobs_by_id.get(str(job_id)) - if target_job and (target_job.get("conclusion") or "").lower() == "success": - print( - f"ℹī¸ GitHub Actions job '{target_job.get('name')}' " - f"(ID: {job_id}) already succeeded." - ) - return 0 - - print(f"🚀 Retrying GitHub Actions job {job_id} in run {run_id} ({repo})...") - res = subprocess.run( - ["gh", "run", "rerun", "--job", str(job_id), "--repo", repo] - ) - if res.returncode == 0: - return 0 - print( - f"⚠ī¸ Single-job rerun for {job_id} exited with {res.returncode}; " - f"falling back to rerunning failed jobs in run {run_id}..." - ) - - if data is not None and (data.get("conclusion") or "").lower() == "success": - print(f"ℹī¸ GitHub Actions run {run_id} already succeeded.") - return 0 - - print(f"🚀 Retrying failed jobs in GitHub Actions run {run_id} ({repo})...") - res = subprocess.run( - ["gh", "run", "rerun", str(run_id), "--failed", "--repo", repo] - ) - return res.returncode - - -def retry_github_job_by_id(job_id, repo=DEFAULT_GH_REPO): - check_cli("gh", "https://cli.github.com/") - cmd = ["gh", "api", f"repos/{repo}/actions/jobs/{job_id}"] - try: - res = subprocess.run(cmd, capture_output=True, text=True, check=True) - out = res.stdout - json_str = out[out.find("{") : out.rfind("}") + 1] if "{" in out else "{}" - job_data = json.loads(json_str) - run_id = job_data.get("run_id") - if run_id: - return retry_github_actions(repo, str(run_id), job_id=str(job_id)) - except Exception as e: - print( - f"⚠ī¸ Could not resolve run_id for GitHub job {job_id} via API: {e}", - file=sys.stderr, - ) - print(f"🚀 Retrying GitHub Actions job ID: {job_id}...") - res = subprocess.run(["gh", "run", "rerun", "--job", str(job_id), "--repo", repo]) - return res.returncode - - -def normalize_build_target(target): - # Transforms https://buildkite.com/bazel/rules-python-python/builds/15707 - # into (org/pipeline, 15707) - m = re.search(r"buildkite\.com/([^/]+/[^/]+)/builds/(\d+)", target) - if m: - return m.group(1), m.group(2) - parts = target.split("/") - if len(parts) == 3: - return f"{parts[0]}/{parts[1]}", parts[2] - elif len(parts) == 2: - return parts[0], parts[1] - return "bazel/rules-python-python", target - - -def retry_job_by_uuid(job_id): - check_cli("bk", "https://github.com/buildkite/cli") - print(f"🚀 Retrying individual Buildkite job UUID: {job_id}") - return subprocess.run(["bk", "job", "retry", job_id]) - - -def unblock_manual_jobs(jobs): - unblocked = [] - for j in jobs: - jstate = j.get("state", "") - is_manual_block = j.get("type") == "manual" or j.get("unblockable") is True - jid = j.get("id") - if jid and jstate in ("blocked", "blocked_failed") and is_manual_block: - jname = j.get("name") or j.get("label") or jid - print(f"🔓 Unblocking manual approval job '{jname}' (ID: {jid})...") - subprocess.run(["bk", "job", "unblock", jid, "--no-input"]) - unblocked.append(j) - return unblocked - - -def retry_jobs_by_name(pipeline, build_number, job_pattern, exit_on_missing=True): - check_cli("bk", "https://github.com/buildkite/cli") - cmd = ["bk", "build", "view", str(build_number), "-p", pipeline, "--json"] - try: - res = subprocess.run(cmd, capture_output=True, text=True, check=True) - data = json.loads(res.stdout) - jobs = data.get("jobs", []) - matched = [] - for j in jobs: - name = j.get("name", "") or j.get("label", "") - state = j.get("state", "") - jid = j.get("id") - if jid and ( - re.search(job_pattern, name, re.IGNORECASE) - or job_pattern.lower() in name.lower() - ): - matched.append((jid, name, state)) - - if not matched: - if exit_on_missing: - print( - f"⚠ī¸ No jobs found matching pattern '{job_pattern}' in build #{build_number}", - file=sys.stderr, - ) - sys.exit(1) - return False - - for jid, name, state in matched: - print(f"🚀 Retrying job '{name}' (ID: {jid}, status: {state})...") - subprocess.run(["bk", "job", "retry", jid]) - return True - except Exception as e: - print(f"❌ Error fetching build jobs: {e}", file=sys.stderr) - if exit_on_missing: - sys.exit(1) - return False - - -def retry_buildkite_build( - pipeline, - build_number, - job_name=None, - unblock=False, - rebuild=False, - rebuild_if_no_failed_jobs=True, -): - check_cli("bk", "https://github.com/buildkite/cli") - if job_name: - return retry_jobs_by_name(pipeline, build_number, job_name) - elif unblock: - cmd = [ - "bk", - "build", - "view", - str(build_number), - "-p", - pipeline, - "--json", - ] - res = subprocess.run(cmd, capture_output=True, text=True, check=True) - data = json.loads(res.stdout) - unblocked = unblock_manual_jobs(data.get("jobs", [])) - if not unblocked: - print( - f"ℹī¸ No blocked manual approval jobs found in build #{build_number}.", - file=sys.stderr, - ) - return True - elif rebuild: - print(f"🚀 Rebuilding entire build #{build_number} for pipeline: {pipeline}") - res = subprocess.run( - ["bk", "build", "rebuild", build_number, "-p", pipeline, "--yes"] - ) - return res.returncode == 0 - else: - cmd = [ - "bk", - "build", - "view", - str(build_number), - "-p", - pipeline, - "--json", - ] - try: - res = subprocess.run(cmd, capture_output=True, text=True, check=True) - data = json.loads(res.stdout) - jobs = data.get("jobs", []) - unblocked_jobs = unblock_manual_jobs(jobs) - failed_jobs = [ - j - for j in jobs - if j.get("state") in ("failed", "broken", "timed_out") - and j.get("id") - and not j.get("retried_in_job_uuid") - and not j.get("retried_at") - ] - if failed_jobs: - for j in failed_jobs: - jid = j["id"] - jname = j.get("name") or j.get("label") or jid - print(f"🚀 Retrying failed job '{jname}' (ID: {jid})...") - subprocess.run(["bk", "job", "retry", jid]) - return True - elif unblocked_jobs: - return True - elif rebuild_if_no_failed_jobs: - print( - f"🚀 Rebuilding entire build #{build_number} for pipeline: {pipeline}" - ) - res = subprocess.run( - [ - "bk", - "build", - "rebuild", - build_number, - "-p", - pipeline, - "--yes", - ] - ) - return res.returncode == 0 - return False - except Exception: - if rebuild_if_no_failed_jobs: - print( - f"🚀 Rebuilding entire build #{build_number} for pipeline: {pipeline}" - ) - res = subprocess.run( - [ - "bk", - "build", - "rebuild", - build_number, - "-p", - pipeline, - "--yes", - ] - ) - return res.returncode == 0 - return False - - -def handle_pr_target(pr_number, job_name=None, unblock=False, rebuild=False): - print(f"🔍 Inspecting PR #{pr_number} checks via gh...") - checks = get_pr_checks(pr_number) - - bk_checks = [] - gh_checks = [] - for c in checks: - link = c.get("link", "") - if "buildkite.com" in link: - bk_checks.append(c) - elif parse_github_actions_url(link): - gh_checks.append(c) - - if unblock: - if not bk_checks: - print(f"ℹī¸ No Buildkite checks found for PR #{pr_number}.", file=sys.stderr) - return 0 - for c in bk_checks: - pipeline, build_number = normalize_build_target(c["link"].split("#")[0]) - retry_buildkite_build(pipeline, build_number, unblock=True) - return 0 - - if job_name: - matched_any = False - matched_gh_runs = {} - for c in gh_checks: - cname = c.get("name", "") - parsed = parse_github_actions_url(c.get("link", "")) - if parsed and ( - re.search(job_name, cname, re.IGNORECASE) - or job_name.lower() in cname.lower() - ): - repo, run_id, job_id = parsed - matched_gh_runs.setdefault((repo, run_id), []).append(job_id) - - for (repo, run_id), job_ids in matched_gh_runs.items(): - matched_any = True - if len(job_ids) == 1 and job_ids[0]: - retry_github_actions(repo, run_id, job_id=job_ids[0]) - else: - retry_github_actions(repo, run_id, job_pattern=job_name) - - for c in bk_checks: - pipeline, build_number = normalize_build_target(c["link"].split("#")[0]) - if retry_jobs_by_name( - pipeline, build_number, job_name, exit_on_missing=False - ): - matched_any = True - - if not matched_any: - print( - f"⚠ī¸ No GitHub Actions or Buildkite jobs found matching " - f"'{job_name}' for PR #{pr_number}.", - file=sys.stderr, - ) - return 1 - return 0 - - # Retry failed GitHub Actions checks (or rebuild all failed runs if --rebuild) - failed_gh_runs = {} - for c in gh_checks: - state = (c.get("state") or "").upper() - bucket = (c.get("bucket") or "").lower() - is_failed = ( - state in ("FAILURE", "FAIL", "CANCELLED", "TIMED_OUT", "STARTUP_FAILURE") - or bucket == "fail" - ) - if is_failed or rebuild: - parsed = parse_github_actions_url(c.get("link", "")) - if parsed: - repo, run_id, job_id = parsed - if is_failed: - failed_gh_runs.setdefault((repo, run_id), []).append(job_id) - - gh_retried = False - for (repo, run_id), job_ids in failed_gh_runs.items(): - gh_retried = True - if rebuild: - retry_github_actions(repo, run_id, rebuild=True) - elif len(job_ids) == 1 and job_ids[0]: - retry_github_actions(repo, run_id, job_id=job_ids[0]) - else: - retry_github_actions(repo, run_id) - - # Check Buildkite builds - for c in bk_checks: - pipeline, build_number = normalize_build_target(c["link"].split("#")[0]) - state = (c.get("state") or "").upper() - bucket = (c.get("bucket") or "").lower() - bk_failed = ( - state in ("FAILURE", "FAIL", "CANCELLED", "TIMED_OUT") or bucket == "fail" - ) - retry_buildkite_build( - pipeline, - build_number, - rebuild=rebuild, - rebuild_if_no_failed_jobs=(bk_failed and not gh_retried), - ) - - if not bk_checks and not gh_retried: - print( - f"ℹī¸ No failing GitHub Actions or Buildkite checks found for PR #{pr_number}." - ) - return 0 - - -def main(): - parser = argparse.ArgumentParser( - description=( - "Retry failed Buildkite or GitHub Actions jobs individually, " - "unblock Buildkite approval steps, or rebuild entire builds." - ) - ) - parser.add_argument( - "args", - nargs="*", - help=( - "Target build (PR#, Buildkite URL, GitHub Actions run/job URL, " - "or org/pipeline/build)" - ), - ) - parser.add_argument( - "--jobs", - "--job-name", - dest="job_name", - help="Specific job name or regex pattern to retry individually", - ) - parser.add_argument( - "--job-id", - dest="job_id", - help=( - "Specific Buildkite job UUID or GitHub Actions job ID to retry individually" - ), - ) - parser.add_argument( - "--rebuild", - action="store_true", - help="Rebuild the entire build afresh", - ) - parser.add_argument( - "--unblock", - action="store_true", - help=( - "Unblock manual approval steps (e.g. .bazelci/presubmit.yml " - "changes) via 'bk job unblock'" - ), - ) - args = parser.parse_args() - - if args.job_id: - gh_parsed = parse_github_actions_url(args.job_id) - if gh_parsed: - repo, run_id, job_id = gh_parsed - sys.exit( - retry_github_actions( - repo, - run_id, - job_id=job_id, - job_pattern=args.job_name, - rebuild=args.rebuild, - ) - ) - if args.job_id.isdigit(): - sys.exit(retry_github_job_by_id(args.job_id)) - res = retry_job_by_uuid(args.job_id) - sys.exit(res.returncode) - - if len(args.args) == 3: - target = f"{args.args[0]}/{args.args[1]}/{args.args[2]}" - elif len(args.args) == 1: - target = args.args[0] - else: - print( - "❌ Error: Invalid arguments. Provide a single target " - "(PR#, Buildkite URL, GitHub Actions URL, or org/pipeline/build).", - file=sys.stderr, - ) - sys.exit(1) - - gh_parsed = parse_github_actions_url(target) - if gh_parsed: - repo, run_id, job_id = gh_parsed - sys.exit( - retry_github_actions( - repo, - run_id, - job_id=job_id, - job_pattern=args.job_name, - rebuild=args.rebuild, - ) - ) - - if target.isdigit() and len(target) < 10: - sys.exit( - handle_pr_target( - target, - job_name=args.job_name, - unblock=args.unblock, - rebuild=args.rebuild, - ) - ) - - pipeline, build_number = normalize_build_target(target) - ok = retry_buildkite_build( - pipeline, - build_number, - job_name=args.job_name, - unblock=args.unblock, - rebuild=args.rebuild, - rebuild_if_no_failed_jobs=True, - ) - sys.exit(0 if ok else 1) - - -if __name__ == "__main__": - main() diff --git a/.agents/skills/merge-pr/SKILL.md b/.agents/skills/merge-pr/SKILL.md index 99f8176442..96304d1244 100644 --- a/.agents/skills/merge-pr/SKILL.md +++ b/.agents/skills/merge-pr/SKILL.md @@ -22,17 +22,17 @@ this PR", or monitor its merge): 3. **Leverage Existing CI Skills**: - Have the subagent use the **`monitor-ci-results`** skill to watch for CI check failures and generate analysis reports. - - Have the subagent use the **`buildkite-retry-job`** skill - (`retry_buildkite_jobs.py `) to automatically retry any + - Have the subagent use the **`retry-ci-job`** skill + (`retry_ci_jobs.py --pr `) to automatically retry any transient network flakes (e.g., HTTP 504 gateway timeouts, downloader errors, `setup-bazel` `Unable to find Bazelisk version` errors) and unblock manual approval steps. - **Blocked Builds (`.bazelci/presubmit.yml` Approval)**: When `.bazelci/presubmit.yml` is modified, Buildkite blocks the build until manually approved. Approve and unblock it immediately via - `./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py - --unblock` (or `bk job unblock --no-input`) so CI does - not sit blocked indefinitely. + `./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py + --pr --unblock` (or `bk job unblock --no-input`) so CI + does not sit blocked indefinitely. - **Soft-Failing Jobs**: Experimental Buildkite jobs (e.g. `*rolling*` Bazel) are non-blocking soft failures; do not treat them as merge blockers. - When the PR is queued, actively discover the merge queue branch via diff --git a/.agents/skills/monitor-ci-results/SKILL.md b/.agents/skills/monitor-ci-results/SKILL.md index 29cba5d2c3..c92bdda027 100644 --- a/.agents/skills/monitor-ci-results/SKILL.md +++ b/.agents/skills/monitor-ci-results/SKILL.md @@ -38,7 +38,7 @@ or when monitoring CI after PR updates: `bk job unblock --no-input` so CI does not sit blocked. If automatic unblocking fails, it notifies the conversation via `agentapi send-message`; unblock Buildkite jobs using the - `buildkite-retry-job` skill (`retry_buildkite_jobs.py + `retry-ci-job` skill (`retry_ci_jobs.py --pr --unblock`). 3. **Failure Reporting**: When any GitHub check or Buildkite job completes with errors, `monitor_remote_ci.py` dispatches a high-priority notification @@ -55,6 +55,6 @@ or when monitoring CI after PR updates: autonomously** launch a separate subagent using `invoke_subagent` with the role "CI Job Retrier" and the prompt template in `.agents/skills/monitor-ci-results/retry-job-prompt.md` to run the - `buildkite-retry-job` skill for ``. Do not execute the script + `retry-ci-job` skill for ``. Do not execute the script directly; use the subagent and skill orchestration instead. Continue monitoring without pausing to ask the user. diff --git a/.agents/skills/monitor-ci-results/retry-job-prompt.md b/.agents/skills/monitor-ci-results/retry-job-prompt.md index 9d948b24a5..f7c338250a 100644 --- a/.agents/skills/monitor-ci-results/retry-job-prompt.md +++ b/.agents/skills/monitor-ci-results/retry-job-prompt.md @@ -1,5 +1,5 @@ -Use the `buildkite-retry-job` skill to retry the failed Buildkite or GitHub -Actions job(s) for PR {pr_number}. -Follow the instructions in `.agents/skills/buildkite-retry-job/SKILL.md` to -retry the failing jobs using the skill. +Use the `retry-ci-job` skill to retry the failed Buildkite or GitHub Actions +job(s) for PR {pr_number}. +Follow the instructions in `.agents/skills/retry-ci-job/SKILL.md` to retry the +failing jobs using the skill. Once the retry request completes, report back to the parent conversation. diff --git a/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py b/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py index 669d117922..e8d3a02809 100755 --- a/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py +++ b/.agents/skills/monitor-ci-results/scripts/monitor_remote_ci.py @@ -260,7 +260,9 @@ def main(): f"⚠ī¸ Remote CI Buildkite Job '{jname}' is blocked!\n\n" f"Build ID: {build_id} | Job ID: {jid}\n" f"Log URL: {log_url}\n\n" - f"Action required: Unblock the job using the buildkite-retry-job skill (`retry_buildkite_jobs.py {args.pr} --unblock`)." + "Action required: Unblock the job using the " + f"retry-ci-job skill (`retry_ci_jobs.py --pr {args.pr} " + "--unblock`)." ) subprocess.run( [ diff --git a/.agents/skills/buildkite-retry-job/SKILL.md b/.agents/skills/retry-ci-job/SKILL.md similarity index 53% rename from .agents/skills/buildkite-retry-job/SKILL.md rename to .agents/skills/retry-ci-job/SKILL.md index 7acbc62148..6737ebdedb 100644 --- a/.agents/skills/buildkite-retry-job/SKILL.md +++ b/.agents/skills/retry-ci-job/SKILL.md @@ -1,12 +1,12 @@ --- -name: buildkite-retry-job +name: retry-ci-job description: Retry failed Buildkite or GitHub Actions jobs, unblock manual approval steps, or rebuild full builds --- -Use `scripts/retry_buildkite_jobs.py` to retry individual Buildkite or GitHub -Actions jobs, unblock manual approval steps (such as `.bazelci/presubmit.yml` -changes), or rebuild full builds. +Use `scripts/retry_ci_jobs.py` to retry individual Buildkite or GitHub Actions +jobs, unblock manual approval steps (such as `.bazelci/presubmit.yml` changes), +or rebuild full builds. When a GitHub Actions job fails early (for example, `Setup Bazel` failing with `Error: Unable to find Bazelisk version ...` due to a transient network error) @@ -17,42 +17,46 @@ automatically waits for an in-progress workflow run to finish before triggering ### Retrying & Unblocking Modes -1. **Default Retry & Unblock (by PR, Buildkite URL, or GitHub Actions URL)**: +1. **Default Retry & Unblock (by PR, URL, or Buildkite Build)**: Unblocks any blocked Buildkite manual approval steps (`bk job unblock --no-input`), retries specific failing Buildkite jobs (`bk job retry `), and reruns failing GitHub Actions jobs/runs (`gh run rerun`): ```bash - ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ - - ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ - "https://github.com///actions/runs//job/" + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py \ + --pr + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py \ + --url "https://github.com///actions/runs//job/" + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py \ + --buildkite-build "//" ``` 2. **Unblock Manual Approval Steps (e.g. `.bazelci/presubmit.yml` changes)**: Approves and unblocks blocked manual approval steps without retrying or rebuilding other jobs: ```bash - ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ - --unblock + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py \ + --pr --unblock ``` 3. **Retry Specific Jobs by Name or ID**: Retries matching Buildkite or GitHub Actions jobs by name/regex pattern, Buildkite job UUID, or numeric GitHub Actions job ID: ```bash - ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ - --jobs "compile_pip_requirements" - ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ - --jobs "Build linux" - ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ - --job-id + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py \ + --pr --jobs "compile_pip_requirements" + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py \ + --pr --jobs "Build linux" + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py \ + --buildkite-job-id + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py \ + --github-job-id ``` 4. **Rebuild Full Build**: Re-runs the entire pipeline/workflow build from scratch via `bk build rebuild` or `gh run rerun`: ```bash - ./.agents/skills/buildkite-retry-job/scripts/retry_buildkite_jobs.py \ - --rebuild + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py \ + --pr --rebuild ``` diff --git a/.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py b/.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py new file mode 100755 index 0000000000..adabc2c851 --- /dev/null +++ b/.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py @@ -0,0 +1,957 @@ +#!/usr/bin/env python3 +"""Retry failed Buildkite or GitHub Actions CI jobs, unblock manual steps, or rebuild builds. + +Target input flags (mutually exclusive, one required): + - `--pr `: Pull request number (e.g. `--pr 4320`) + - `--url `: Buildkite build URL (`https://buildkite.com///builds/`) + or GitHub Actions run/job URL + (`https://github.com///actions/runs/[/job/]`) + - `--buildkite-build `: Buildkite build identifier + - `--buildkite-job-id `: Individual Buildkite job UUID + - `--github-job-id `: Individual GitHub Actions job ID + +Usage: + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py --pr 4320 + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py --pr 4320 --unblock + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py --pr 4320 --jobs "Build linux" + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py --github-job-id 1234567890 + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py --buildkite-job-id 0190a1b2-c3d4 + ./.agents/skills/retry-ci-job/scripts/retry_ci_jobs.py --pr 4320 --rebuild +""" + +from __future__ import annotations + +import argparse +import enum +import json +import re +import subprocess +import sys +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +# ============================================================================== +# 1. Global variables / constants +# ============================================================================== + +_DEFAULT_GH_REPO = "bazel-contrib/rules_python" +_DEFAULT_BK_PIPELINE = "bazel/rules-python-python" + +_GH_ACTIONS_URL_RE = re.compile( + r"github\.com/([^/]+/[^/]+)/actions/runs/(\d+)(?:/job/(\d+))?" +) +_BK_BUILD_URL_RE = re.compile(r"buildkite\.com/([^/]+/[^/]+)/builds/(\d+)") + +# GitHub rejects `gh run rerun` while sibling matrix jobs in the workflow run are +# still executing; poll up to 15 minutes (60 * 15s) for the parent run to finish. +_GH_POLL_INTERVAL_SECONDS = 15 +_GH_MAX_POLL_ATTEMPTS = 60 + +_FAILED_CHECK_STATES = frozenset( + [ + # keep-sorted start + "CANCELLED", + "FAIL", + "FAILURE", + "STARTUP_FAILURE", + "TIMED_OUT", + # keep-sorted end + ] +) + +_FAILED_BK_JOB_STATES = frozenset( + [ + # keep-sorted start + "broken", + "failed", + "timed_out", + # keep-sorted end + ] +) + +_BLOCKED_BK_JOB_STATES = frozenset( + [ + # keep-sorted start + "blocked", + "blocked_failed", + # keep-sorted end + ] +) + + +# ============================================================================== +# 2. Flags / CLI arguments +# ============================================================================== + + +def _parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Retry failed Buildkite or GitHub Actions jobs individually, " + "unblock Buildkite approval steps, or rebuild entire builds." + ) + ) + target_group = parser.add_mutually_exclusive_group(required=True) + target_group.add_argument( + "--pr", + type=int, + default=0, + help="Pull request number to inspect and retry", + ) + target_group.add_argument( + "--url", + default="", + help="Buildkite build URL or GitHub Actions workflow run/job URL", + ) + target_group.add_argument( + "--buildkite-build", + default="", + dest="buildkite_build", + help="Buildkite build identifier (// or )", + ) + target_group.add_argument( + "--buildkite-job-id", + default="", + dest="buildkite_job_id", + help="Specific Buildkite job UUID to retry individually", + ) + target_group.add_argument( + "--github-job-id", + default="", + dest="github_job_id", + help="Specific GitHub Actions job ID to retry individually", + ) + + mode_group = parser.add_mutually_exclusive_group() + mode_group.add_argument( + "--jobs", + default="", + dest="job_name", + help="Specific job name or regex pattern to retry individually", + ) + mode_group.add_argument( + "--rebuild", + action="store_true", + help="Rebuild the entire build afresh", + ) + mode_group.add_argument( + "--unblock", + action="store_true", + help=( + "Unblock manual approval steps (e.g. .bazelci/presubmit.yml " + "changes) via 'bk job unblock'" + ), + ) + return parser.parse_args(list(argv)) + + +# ============================================================================== +# 3. Functions +# ============================================================================== + + +def _check_cli(cmd_name: str, install_url: str) -> None: + try: + subprocess.run( + [cmd_name, "--version"], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except (OSError, subprocess.SubprocessError) as e: + raise CiRetryError( + f"❌ Error: '{cmd_name}' CLI is not installed or not in PATH.\n" + f"Please install it from {install_url}" + ) from e + + +def _extract_json_object(raw_output: str) -> dict[str, object]: + if "{" not in raw_output: + return {} + json_str = raw_output[raw_output.find("{") : raw_output.rfind("}") + 1] + parsed = json.loads(json_str) + if isinstance(parsed, dict): + return parsed + return {} + + +def _extract_json_array(raw_output: str) -> list[dict[str, object]]: + if "[" not in raw_output: + return [] + json_str = raw_output[raw_output.find("[") : raw_output.rfind("]") + 1] + parsed = json.loads(json_str) + if isinstance(parsed, list): + return parsed + return [] + + +def _get_pr_checks(pr_number: int) -> list[dict[str, object]]: + _check_cli("gh", "https://cli.github.com/") + cmd = [ + "gh", + "pr", + "checks", + str(pr_number), + "--json", + "name,link,state,bucket", + ] + res = subprocess.run(cmd, capture_output=True, text=True) + if "[" not in res.stdout: + raise CiRetryError( + f"❌ Error fetching PR checks for PR #{pr_number}: {res.stderr.strip()}" + ) + return _extract_json_array(res.stdout) + + +def _matches_job_pattern(job_name: str, pattern: str) -> bool: + if pattern.lower() in job_name.lower(): + return True + try: + return bool(re.search(pattern, job_name, re.IGNORECASE)) + except re.error: + return False + + +def _is_check_failed(check: Mapping[str, object]) -> bool: + state = str(check.get("state") or "").upper() + bucket = str(check.get("bucket") or "").lower() + return state in _FAILED_CHECK_STATES or bucket == "fail" + + +def _parse_github_actions_url(target: str) -> GitHubRunTarget | None: + match = _GH_ACTIONS_URL_RE.search(target) + if not match: + return None + return GitHubRunTarget( + repo=match.group(1), + run_id=match.group(2), + job_id=match.group(3) or "", + ) + + +def _parse_buildkite_url(url: str) -> BuildkiteBuildTarget | None: + clean_url = url.split("#")[0] + match = _BK_BUILD_URL_RE.search(clean_url) + if not match: + return None + return BuildkiteBuildTarget( + pipeline=match.group(1), + build_number=match.group(2), + ) + + +def _parse_buildkite_build_spec(spec: str) -> BuildkiteBuildTarget: + clean_spec = spec.split("#")[0] + parts = clean_spec.split("/") + if len(parts) == 3: + return BuildkiteBuildTarget( + pipeline=f"{parts[0]}/{parts[1]}", + build_number=parts[2], + ) + if len(parts) == 2: + return BuildkiteBuildTarget( + pipeline=parts[0], + build_number=parts[1], + ) + if len(parts) == 1 and clean_spec: + return BuildkiteBuildTarget( + pipeline=_DEFAULT_BK_PIPELINE, + build_number=clean_spec, + ) + raise CiRetryError( + f"❌ Error: Invalid Buildkite build identifier '{spec}'. " + "Expected '//' or ''." + ) + + +def _resolve_mode(args: argparse.Namespace) -> OperationMode: + if args.unblock: + return OperationMode.UNBLOCK + if args.rebuild: + return OperationMode.REBUILD + if args.job_name: + return OperationMode.MATCH_JOBS + return OperationMode.RETRY_FAILED + + +def _resolve_target( + args: argparse.Namespace, +) -> ( + PrTarget + | BuildkiteBuildTarget + | BuildkiteJobTarget + | GitHubRunTarget + | GitHubJobTarget +): + if args.pr: + return PrTarget(pr_number=args.pr) + if args.buildkite_job_id: + return BuildkiteJobTarget(job_uuid=args.buildkite_job_id) + if args.github_job_id: + return GitHubJobTarget(repo=_DEFAULT_GH_REPO, job_id=args.github_job_id) + if args.buildkite_build: + return _parse_buildkite_build_spec(args.buildkite_build) + if args.url: + gh_run_target = _parse_github_actions_url(args.url) + if gh_run_target: + return gh_run_target + bk_build_target = _parse_buildkite_url(args.url) + if bk_build_target: + return bk_build_target + raise CiRetryError( + f"❌ Error: Unsupported URL '{args.url}'. " + "Expected a Buildkite build URL or GitHub Actions run/job URL." + ) + raise CiRetryError("❌ Error: No target specified.") + + +# ============================================================================== +# 4. Classes +# ============================================================================== + + +class CiRetryError(Exception): + """Raised when CLI prerequisites or target arguments are invalid.""" + + +class OperationMode(enum.Enum): + """Execution mode for retrying, unblocking, or rebuilding CI targets.""" + + RETRY_FAILED = "retry_failed" + UNBLOCK = "unblock" + REBUILD = "rebuild" + MATCH_JOBS = "match_jobs" + + +@dataclass(frozen=True) +class PrTarget: + """Pull request number target.""" + + pr_number: int + + +@dataclass(frozen=True) +class BuildkiteBuildTarget: + """Buildkite pipeline and build number target.""" + + pipeline: str + build_number: str + + +@dataclass(frozen=True) +class BuildkiteJobTarget: + """Individual Buildkite job UUID target.""" + + job_uuid: str + + +@dataclass(frozen=True) +class GitHubRunTarget: + """GitHub Actions workflow run (and optional job ID) target.""" + + repo: str + run_id: str + job_id: str = "" + + +@dataclass(frozen=True) +class GitHubJobTarget: + """Standalone numeric GitHub Actions job ID target.""" + + repo: str + job_id: str + + +class BuildkiteProvider: + """Executes Buildkite job retries, manual step unblocks, and build rebuilds.""" + + def _ensure_cli(self) -> None: + _check_cli("bk", "https://github.com/buildkite/cli") + + def _view_build(self, target: BuildkiteBuildTarget) -> list[dict[str, object]]: + self._ensure_cli() + cmd = [ + "bk", + "build", + "view", + target.build_number, + "-p", + target.pipeline, + "--json", + ] + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + data = json.loads(res.stdout) + jobs = data.get("jobs", []) if isinstance(data, dict) else [] + if isinstance(jobs, list): + return jobs + return [] + + def retry_job_by_uuid(self, target: BuildkiteJobTarget) -> bool: + self._ensure_cli() + print(f"🚀 Retrying individual Buildkite job UUID: {target.job_uuid}") + res = subprocess.run(["bk", "job", "retry", target.job_uuid]) + return res.returncode == 0 + + def rebuild_build(self, target: BuildkiteBuildTarget) -> bool: + self._ensure_cli() + print( + f"🚀 Rebuilding entire build #{target.build_number} " + f"for pipeline: {target.pipeline}" + ) + res = subprocess.run( + [ + "bk", + "build", + "rebuild", + target.build_number, + "-p", + target.pipeline, + "--yes", + ] + ) + return res.returncode == 0 + + def unblock_manual_jobs( + self, jobs: Sequence[Mapping[str, object]] + ) -> tuple[list[Mapping[str, object]], bool]: + unblocked: list[Mapping[str, object]] = [] + all_ok = True + for job in jobs: + jstate = str(job.get("state") or "") + is_manual_block = ( + job.get("type") == "manual" or job.get("unblockable") is True + ) + jid = str(job.get("id") or "") + if jid and jstate in _BLOCKED_BK_JOB_STATES and is_manual_block: + jname = str(job.get("name") or job.get("label") or jid) + print(f"🔓 Unblocking manual approval job '{jname}' (ID: {jid})...") + res = subprocess.run(["bk", "job", "unblock", jid, "--no-input"]) + if res.returncode != 0: + all_ok = False + unblocked.append(job) + return unblocked, all_ok + + def retry_jobs_by_name( + self, + target: BuildkiteBuildTarget, + job_pattern: str, + fail_on_missing: bool = True, + ) -> tuple[bool, bool]: + """Retries Buildkite jobs matching `job_pattern`. + + Returns: + Tuple of `(matched_any, all_succeeded)`. + """ + try: + jobs = self._view_build(target) + except (subprocess.SubprocessError, json.JSONDecodeError) as e: + print(f"❌ Error fetching build jobs: {e}", file=sys.stderr) + return False, False + + matched: list[tuple[str, str, str]] = [] + for job in jobs: + name = str(job.get("name") or job.get("label") or "") + state = str(job.get("state") or "") + jid = str(job.get("id") or "") + if jid and _matches_job_pattern(name, job_pattern): + matched.append((jid, name, state)) + + if not matched: + if fail_on_missing: + print( + f"⚠ī¸ No jobs found matching pattern '{job_pattern}' " + f"in build #{target.build_number}", + file=sys.stderr, + ) + return False, not fail_on_missing + + all_ok = True + for jid, name, state in matched: + print(f"🚀 Retrying job '{name}' (ID: {jid}, status: {state})...") + res = subprocess.run(["bk", "job", "retry", jid]) + if res.returncode != 0: + all_ok = False + return True, all_ok + + def handle_build( + self, + target: BuildkiteBuildTarget, + mode: OperationMode, + job_pattern: str = "", + rebuild_if_no_failed_jobs: bool = True, + ) -> bool: + if mode == OperationMode.MATCH_JOBS: + matched, all_ok = self.retry_jobs_by_name( + target, job_pattern, fail_on_missing=True + ) + return matched and all_ok + + if mode == OperationMode.UNBLOCK: + jobs = self._view_build(target) + unblocked, all_ok = self.unblock_manual_jobs(jobs) + if not unblocked: + print( + f"ℹī¸ No blocked manual approval jobs found in build " + f"#{target.build_number}.", + file=sys.stderr, + ) + return all_ok + + if mode == OperationMode.REBUILD: + return self.rebuild_build(target) + + if mode == OperationMode.RETRY_FAILED: + try: + jobs = self._view_build(target) + except (subprocess.SubprocessError, json.JSONDecodeError): + if rebuild_if_no_failed_jobs: + return self.rebuild_build(target) + return False + + unblocked_jobs, unblock_ok = self.unblock_manual_jobs(jobs) + failed_jobs: list[dict[str, object]] = [] + for job in jobs: + state = str(job.get("state") or "") + if ( + state in _FAILED_BK_JOB_STATES + and job.get("id") + and not job.get("retried_in_job_uuid") + and not job.get("retried_at") + ): + failed_jobs.append(job) + + if failed_jobs: + all_ok = unblock_ok + for job in failed_jobs: + jid = str(job["id"]) + jname = str(job.get("name") or job.get("label") or jid) + print(f"🚀 Retrying failed job '{jname}' (ID: {jid})...") + res = subprocess.run(["bk", "job", "retry", jid]) + if res.returncode != 0: + all_ok = False + return all_ok + + if unblocked_jobs: + return unblock_ok + + if rebuild_if_no_failed_jobs: + return unblock_ok and self.rebuild_build(target) + return unblock_ok + + raise CiRetryError(f"Unsupported operation mode: {mode}") + + +class GitHubActionsProvider: + """Executes GitHub Actions workflow and job reruns via the `gh` CLI.""" + + def _ensure_cli(self) -> None: + _check_cli("gh", "https://cli.github.com/") + + def _get_run_data(self, repo: str, run_id: str) -> dict[str, object]: + cmd = [ + "gh", + "run", + "view", + run_id, + "--repo", + repo, + "--json", + "status,conclusion,jobs", + ] + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + return _extract_json_object(res.stdout) + + def _wait_for_run_completion( + self, + repo: str, + run_id: str, + job_id: str = "", + ) -> dict[str, object]: + for attempt in range(_GH_MAX_POLL_ATTEMPTS): + try: + data = self._get_run_data(repo, run_id) + except (subprocess.SubprocessError, json.JSONDecodeError) as e: + print( + f"⚠ī¸ Warning: Could not inspect GitHub Actions run {run_id}: {e}", + file=sys.stderr, + ) + return {} + + status = str(data.get("status") or "").lower() + raw_jobs = data.get("jobs", []) + jobs = raw_jobs if isinstance(raw_jobs, list) else [] + + if job_id: + active_job_ids = {str(j.get("databaseId")) for j in jobs} + if job_id not in active_job_ids: + # The job ID from an earlier attempt is no longer in the active + # jobs list, meaning a newer run_attempt is queued/running/done. + if ( + status != "completed" + or str(data.get("conclusion") or "").lower() == "success" + ): + print( + f"ℹī¸ GitHub Actions job {job_id} has already been " + f"superseded by a newer attempt in run {run_id} " + f"(status: {status})." + ) + return data + + if status == "completed": + return data + + print( + f"âŗ GitHub Actions run {run_id} is currently '{status}' " + f"(attempt {attempt + 1}/{_GH_MAX_POLL_ATTEMPTS}); waiting " + f"{_GH_POLL_INTERVAL_SECONDS}s for run to complete before " + f"rerunning..." + ) + time.sleep(_GH_POLL_INTERVAL_SECONDS) + + print( + f"⚠ī¸ Timed out waiting for GitHub Actions run {run_id} to complete.", + file=sys.stderr, + ) + return {} + + def _rerun_failed_in_run(self, repo: str, run_id: str) -> bool: + res = subprocess.run(["gh", "run", "rerun", run_id, "--failed", "--repo", repo]) + return res.returncode == 0 + + def _rerun_job_with_fallback( + self, repo: str, run_id: str, job_id: str + ) -> tuple[bool, bool]: + """Reruns a single job, falling back to `--failed` on the parent run. + + Returns: + Tuple of `(succeeded, used_run_fallback)`. + """ + res = subprocess.run(["gh", "run", "rerun", "--job", job_id, "--repo", repo]) + if res.returncode == 0: + return True, False + + print( + f"⚠ī¸ Single-job rerun failed for {job_id} (exit {res.returncode}); " + f"falling back to 'gh run rerun {run_id} --failed'..." + ) + return self._rerun_failed_in_run(repo, run_id), True + + def handle_run( + self, + target: GitHubRunTarget, + mode: OperationMode, + job_pattern: str = "", + ) -> bool: + self._ensure_cli() + if mode == OperationMode.UNBLOCK: + return True + + data = self._wait_for_run_completion( + target.repo, target.run_id, job_id=target.job_id + ) + raw_jobs = data.get("jobs", []) + jobs: list[dict[str, object]] = raw_jobs if isinstance(raw_jobs, list) else [] + + if mode == OperationMode.REBUILD: + print( + f"🚀 Re-running entire GitHub Actions run {target.run_id} " + f"({target.repo})..." + ) + res = subprocess.run( + ["gh", "run", "rerun", target.run_id, "--repo", target.repo] + ) + return res.returncode == 0 + + if mode == OperationMode.MATCH_JOBS: + matched: list[tuple[str, str, str]] = [] + for job in jobs: + name = str(job.get("name") or "") + jid = str(job.get("databaseId") or "") + conclusion = str(job.get("conclusion") or job.get("status") or "") + if jid and _matches_job_pattern(name, job_pattern): + matched.append((jid, name, conclusion)) + + if not matched: + print( + f"⚠ī¸ No GitHub Actions jobs found matching '{job_pattern}' " + f"in run {target.run_id}.", + file=sys.stderr, + ) + return False + + for jid, name, conclusion in matched: + print( + f"🚀 Retrying GitHub Actions job '{name}' " + f"(ID: {jid}, conclusion: {conclusion})..." + ) + ok, used_fallback = self._rerun_job_with_fallback( + target.repo, target.run_id, jid + ) + if used_fallback or not ok: + return ok + return True + + if mode == OperationMode.RETRY_FAILED: + if target.job_id: + active_jobs_by_id = {str(j.get("databaseId")): j for j in jobs} + if data and target.job_id not in active_jobs_by_id: + status = str(data.get("status") or "").lower() + conclusion = str(data.get("conclusion") or "").lower() + if status != "completed" or conclusion == "success": + return True + + target_job = active_jobs_by_id.get(target.job_id) + if ( + target_job + and str(target_job.get("conclusion") or "").lower() == "success" + ): + print( + f"ℹī¸ GitHub Actions job '{target_job.get('name')}' " + f"(ID: {target.job_id}) already succeeded." + ) + return True + + print( + f"🚀 Retrying GitHub Actions job {target.job_id} in run " + f"{target.run_id} ({target.repo})..." + ) + ok, _ = self._rerun_job_with_fallback( + target.repo, target.run_id, target.job_id + ) + return ok + + if data and str(data.get("conclusion") or "").lower() == "success": + print(f"ℹī¸ GitHub Actions run {target.run_id} already succeeded.") + return True + + print( + f"🚀 Retrying failed jobs in GitHub Actions run {target.run_id} " + f"({target.repo})..." + ) + return self._rerun_failed_in_run(target.repo, target.run_id) + + raise CiRetryError(f"Unsupported operation mode: {mode}") + + def retry_job_by_id( + self, + target: GitHubJobTarget, + mode: OperationMode = OperationMode.RETRY_FAILED, + job_pattern: str = "", + ) -> bool: + self._ensure_cli() + cmd = ["gh", "api", f"repos/{target.repo}/actions/jobs/{target.job_id}"] + run_id = "" + try: + res = subprocess.run(cmd, capture_output=True, text=True, check=True) + job_data = _extract_json_object(res.stdout) + run_id = str(job_data.get("run_id") or "") + except (subprocess.SubprocessError, json.JSONDecodeError) as e: + print( + f"⚠ī¸ Could not resolve run_id for GitHub job {target.job_id} " + f"via API: {e}", + file=sys.stderr, + ) + + if run_id: + return self.handle_run( + GitHubRunTarget( + repo=target.repo, + run_id=run_id, + job_id=target.job_id, + ), + mode=mode, + job_pattern=job_pattern, + ) + + print(f"🚀 Retrying GitHub Actions job ID: {target.job_id}...") + res = subprocess.run( + ["gh", "run", "rerun", "--job", target.job_id, "--repo", target.repo] + ) + return res.returncode == 0 + + +class PrOrchestrator: + """Inspects PR checks and routes retries across Buildkite and GitHub Actions.""" + + def __init__( + self, + buildkite: BuildkiteProvider, + github_actions: GitHubActionsProvider, + ) -> None: + self._bk = buildkite + self._gh = github_actions + + def _dispatch_github_runs( + self, + grouped_runs: Mapping[tuple[str, str], Sequence[str]], + mode: OperationMode, + job_pattern: str = "", + ) -> bool: + all_ok = True + for (repo, run_id), job_ids in grouped_runs.items(): + single_job_id = job_ids[0] if len(job_ids) == 1 and job_ids[0] else "" + run_mode = mode + if mode == OperationMode.MATCH_JOBS and single_job_id: + run_mode = OperationMode.RETRY_FAILED + ok = self._gh.handle_run( + GitHubRunTarget( + repo=repo, + run_id=run_id, + job_id=single_job_id if run_mode != OperationMode.REBUILD else "", + ), + mode=run_mode, + job_pattern=job_pattern if run_mode == OperationMode.MATCH_JOBS else "", + ) + if not ok: + all_ok = False + return all_ok + + def handle_pr( + self, + target: PrTarget, + mode: OperationMode, + job_pattern: str = "", + ) -> bool: + print(f"🔍 Inspecting PR #{target.pr_number} checks via gh...") + checks = _get_pr_checks(target.pr_number) + + bk_checks: list[tuple[BuildkiteBuildTarget, bool]] = [] + gh_checks: list[tuple[str, GitHubRunTarget, bool]] = [] + for check in checks: + link = str(check.get("link") or "") + name = str(check.get("name") or "") + is_failed = _is_check_failed(check) + bk_target = _parse_buildkite_url(link) + if bk_target: + bk_checks.append((bk_target, is_failed)) + else: + gh_target = _parse_github_actions_url(link) + if gh_target: + gh_checks.append((name, gh_target, is_failed)) + + if mode == OperationMode.UNBLOCK: + if not bk_checks: + print( + f"ℹī¸ No Buildkite checks found for PR #{target.pr_number}.", + file=sys.stderr, + ) + return True + all_ok = True + for bk_target, _ in bk_checks: + if not self._bk.handle_build(bk_target, OperationMode.UNBLOCK): + all_ok = False + return all_ok + + if mode == OperationMode.MATCH_JOBS: + matched_any = False + all_ok = True + matched_gh_runs: dict[tuple[str, str], list[str]] = {} + for cname, gh_target, _ in gh_checks: + if _matches_job_pattern(cname, job_pattern): + matched_gh_runs.setdefault( + (gh_target.repo, gh_target.run_id), [] + ).append(gh_target.job_id) + + if matched_gh_runs: + matched_any = True + if not self._dispatch_github_runs( + matched_gh_runs, + mode=OperationMode.MATCH_JOBS, + job_pattern=job_pattern, + ): + all_ok = False + + for bk_target, _ in bk_checks: + bk_matched, bk_ok = self._bk.retry_jobs_by_name( + bk_target, job_pattern, fail_on_missing=False + ) + if bk_matched: + matched_any = True + if not bk_ok: + all_ok = False + + if not matched_any: + print( + f"⚠ī¸ No GitHub Actions or Buildkite jobs found matching " + f"'{job_pattern}' for PR #{target.pr_number}.", + file=sys.stderr, + ) + return False + return all_ok + + if mode in (OperationMode.RETRY_FAILED, OperationMode.REBUILD): + target_gh_runs: dict[tuple[str, str], list[str]] = {} + for _, gh_target, is_failed in gh_checks: + if is_failed or mode == OperationMode.REBUILD: + target_gh_runs.setdefault( + (gh_target.repo, gh_target.run_id), [] + ).append(gh_target.job_id) + + all_ok = True + gh_retried = bool(target_gh_runs) + if target_gh_runs: + if not self._dispatch_github_runs(target_gh_runs, mode=mode): + all_ok = False + + for bk_target, bk_failed in bk_checks: + if not self._bk.handle_build( + bk_target, + mode=mode, + rebuild_if_no_failed_jobs=(bk_failed and not gh_retried), + ): + all_ok = False + + if not bk_checks and not gh_retried: + print( + f"ℹī¸ No failing GitHub Actions or Buildkite checks found " + f"for PR #{target.pr_number}." + ) + return all_ok + + raise CiRetryError(f"Unsupported operation mode: {mode}") + + +# ============================================================================== +# 5. Main +# ============================================================================== + + +def main(argv: Sequence[str]) -> int: + args = _parse_args(argv) + mode = _resolve_mode(args) + + try: + target = _resolve_target(args) + bk_provider = BuildkiteProvider() + gh_provider = GitHubActionsProvider() + + if isinstance(target, BuildkiteJobTarget): + ok = bk_provider.retry_job_by_uuid(target) + elif isinstance(target, GitHubJobTarget): + ok = gh_provider.retry_job_by_id( + target, mode=mode, job_pattern=args.job_name + ) + elif isinstance(target, GitHubRunTarget): + ok = gh_provider.handle_run(target, mode=mode, job_pattern=args.job_name) + elif isinstance(target, PrTarget): + orchestrator = PrOrchestrator(bk_provider, gh_provider) + ok = orchestrator.handle_pr(target, mode=mode, job_pattern=args.job_name) + elif isinstance(target, BuildkiteBuildTarget): + ok = bk_provider.handle_build( + target, + mode=mode, + job_pattern=args.job_name, + rebuild_if_no_failed_jobs=True, + ) + else: + raise CiRetryError(f"Unsupported target type: {target}") + except CiRetryError as e: + print(str(e), file=sys.stderr) + return 1 + + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))