diff --git a/src/cfengine_cli/changelog.py b/src/cfengine_cli/changelog.py index 781827ec..aa7a0679 100644 --- a/src/cfengine_cli/changelog.py +++ b/src/cfengine_cli/changelog.py @@ -20,6 +20,8 @@ import re import subprocess import logging +import sys +from cfengine_cli.utils import get_upstream_remote # --------------------------------------------------------------------------- # Regex @@ -42,21 +44,42 @@ def fetch_git_output(path, args): ).stdout.splitlines(keepends=True) +REPO_OWNER = "cfengine" + + +def get_versions(prev, branch, repo, repo_path, git_args, repo_owner=REPO_OWNER): + """Assumes cfengine dev generate-changelog is run inside cfengine repos only""" + if git_args: + return git_args + else: + name = os.path.basename(repo) + remote = get_upstream_remote(name, repo_owner, repo_path) + if remote is None: + logging.error( + "No remote in '{}' points to 'github.com/{}/{}'".format( + repo, repo_owner, name + ) + ) + exit(1) + return [f"{prev}..{remote}/{branch}"] + + # --------------------------------------------------------------------------- # Fetch and merge dependency upgrades / reverts # --------------------------------------------------------------------------- -def collect_version_updates(repos, git_args): +def collect_version_updates(repos, prev, branch, git_args): dep_history: dict[str, list[tuple[str, str]]] = {} for repo in repos: + repo_path = os.path.join(os.getcwd(), repo) for raw in fetch_git_output( - os.path.join(os.getcwd(), repo), + repo_path, [ "log", "--no-merges", "--reverse", "--pretty=format:%s", ] - + git_args, + + get_versions(prev, branch, repo, repo_path, git_args), ): subject = raw.decode().strip() @@ -212,21 +235,22 @@ def add_entry(sha, msg): add_entry(sha, title) -def parse_git_log(repos, git_args): +def parse_git_log(repos, prev, branch, git_args): """Walk git history across repos and return (entry_list, missed_tickets).""" entries = {} # sha -> [msg, ...] linked_shas = {} # sha -> [linked_sha, ...] sha_to_tracker = {} # sha -> set of ticket strings for repo in repos: + repo_path = os.path.join(os.getcwd(), repo) for raw_sha in fetch_git_output( - os.path.join(os.getcwd(), repo), + repo_path, [ "rev-list", "--no-merges", "--reverse", ] - + git_args, + + get_versions(prev, branch, repo, repo_path, git_args), ): parse_sha(raw_sha, entries, sha_to_tracker, linked_shas, repo) @@ -295,6 +319,58 @@ def get_current_branch(): ).strip() +def remote_is_ahead(repo_path, remote, branch): + out = subprocess.check_output( + ["git", "-C", repo_path, "ls-remote", remote, f"refs/heads/{branch}"], + text=True, + ).split() + if not out: + return False # branch doesn't exist on the remote + try: + local = subprocess.check_output( + ["git", "-C", repo_path, "rev-parse", f"{remote}/{branch}"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + except subprocess.CalledProcessError: + return True # never fetched + return out[0] != local + + +def _fetch_prompt(): + # Prompt on stderr so it doesn't end up in the changelog with -o + answer = None + while answer not in ("", "y", "n", "yes", "no"): + print("Fetch? (Y/n): ", end="", file=sys.stderr, flush=True) + answer = input().strip().lower() + return answer in ("", "y", "yes") + + +def fetch_upstream(repos, branch, assume_yes=False, repo_owner=REPO_OWNER): + """Fetch the upstream remote in each repo where it's behind GitHub, + asking first unless assume_yes is set.""" + outdated = {} # repo_path -> remote + for repo in repos: + repo_path = os.path.join(os.getcwd(), repo) + remote = get_upstream_remote(os.path.basename(repo), repo_owner, repo_path) + if remote is None: + continue # reported by get_versions + if remote_is_ahead(repo_path, remote, branch): + outdated[repo_path] = remote + + if not outdated: + return + for repo_path, remote in outdated.items(): + print( + f"'{remote}/{branch}' is out of date in {os.path.basename(repo_path)}", + file=sys.stderr, + ) + if not assume_yes and not _fetch_prompt(): + return + for repo_path, remote in outdated.items(): + subprocess.run(["git", "-C", repo_path, "fetch", remote], check=True) + + # --------------------------------------------------------------------------- # Versionstuff # --------------------------------------------------------------------------- @@ -362,13 +438,15 @@ def generate_changelog_impl(args): f"next version to be {expected}, but cfversion says {actual}" ) - if args.git_args: - versions = args.git_args - else: - # Assumes tag exists and origin follows the same naming scheme for this release - versions = [f"{prev}..origin/{branch}"] + # packaging changes only included in enterprise + is_enterprise = os.path.basename(os.path.abspath(os.curdir)) == "enterprise" + + if not args.git_args and not args.no_fetch: + fetch_upstream( + repos + (REPO_MAP["packaging"] if is_enterprise else []), branch, args.yes + ) - entry_list, missed_tickets = parse_git_log(repos, versions) + entry_list, missed_tickets = parse_git_log(repos, prev, branch, args.git_args) entry_list.sort() output = "" @@ -386,12 +464,14 @@ def generate_changelog_impl(args): else f"## {args.git_args[0].split('..')[-1]}\n" ) - if ( - os.path.basename(os.path.abspath(os.curdir)) == "enterprise" - ): # packaging changes only included in enterprise - pkg_changes, missed_pkg_tickets = parse_git_log(REPO_MAP["packaging"], versions) + if is_enterprise: + pkg_changes, missed_pkg_tickets = parse_git_log( + REPO_MAP["packaging"], prev, branch, args.git_args + ) missed_tickets.update(missed_pkg_tickets) - pkg_changes += collect_version_updates(REPO_MAP["packaging"], versions) + pkg_changes += collect_version_updates( + REPO_MAP["packaging"], prev, branch, args.git_args + ) pkg_changes.sort() if pkg_changes: lines.append("\n**Packaging changes:**") diff --git a/src/cfengine_cli/deptool.py b/src/cfengine_cli/deptool.py index 8e83db67..74648933 100644 --- a/src/cfengine_cli/deptool.py +++ b/src/cfengine_cli/deptool.py @@ -8,6 +8,8 @@ import subprocess import sys +from cfengine_cli.utils import get_upstream_remote + ACTIVE_BRANCHES = ["3.24.x", "3.27.x", "master"] HUMAN_NAME = { @@ -213,20 +215,8 @@ def is_git_branch(self, ref): return False return True - def upstream_remote(self): - result = self.run_command("remote", "-v", capture_output=True) - upstream = "github\\.com[:/]{}/{}(\\.git)?/?$".format( - re.escape(self.repo_owner), re.escape(self.repo_name) - ) - for line in result.stdout.splitlines(): - parts = line.split() - # GitHub owner and repository names are case-insensitive - if len(parts) >= 2 and re.search(upstream, parts[1], re.IGNORECASE): - return parts[0] - return None - def upstream_branch(self, ref): - remote = self.upstream_remote() + remote = get_upstream_remote(self.repo_name, self.repo_owner, self.repo_path) if remote is None: log.warning( "No remote points to github.com/{}/{}, using local refs".format( diff --git a/src/cfengine_cli/main.py b/src/cfengine_cli/main.py index b794cbfa..d56719c9 100644 --- a/src/cfengine_cli/main.py +++ b/src/cfengine_cli/main.py @@ -189,6 +189,19 @@ def _get_arg_parser(): dest="show_version", help="Print the version from .CFVERSION and exit", ) + fetch = parser.add_mutually_exclusive_group() + fetch.add_argument( + "-y", + "--yes", + action="store_true", + help="Fetch out-of-date upstream branches without asking", + ) + fetch.add_argument( + "--no-fetch", + action="store_true", + dest="no_fetch", + help="Don't check for or fetch upstream changes, use local remote-tracking branches as they are", + ) parser.add_argument( "git_args", nargs="*", diff --git a/src/cfengine_cli/utils.py b/src/cfengine_cli/utils.py index 8d0fa9e5..bfb613b8 100644 --- a/src/cfengine_cli/utils.py +++ b/src/cfengine_cli/utils.py @@ -1,3 +1,7 @@ +import subprocess +import re + + class UserError(Exception): """Exception raised when a user has most likely made a mistake""" @@ -7,3 +11,18 @@ def __init__(self, message): def __str__(self): return f"Error: {self.message}" + + +def get_upstream_remote(repo_name, repo_owner, repo_path): + upstream = r"github\.com[:/]{}/{}(\.git)?/?$".format( + re.escape(repo_owner), re.escape(repo_name) + ) + remotes = subprocess.check_output( + ["git", "-C", repo_path, "remote", "-v"], text=True + ) + for line in remotes.splitlines(): + parts = line.split() + # GitHub owner and repository names are case-insensitive + if len(parts) >= 2 and re.search(upstream, parts[1], re.IGNORECASE): + return parts[0] + return None diff --git a/tests/unit/test_changelog.py b/tests/unit/test_changelog.py index 9b37ab26..88e21eaa 100644 --- a/tests/unit/test_changelog.py +++ b/tests/unit/test_changelog.py @@ -18,7 +18,9 @@ class TestCollectVersionUpdates(unittest.TestCase): def _run(self, subjects): lines = _lines(*subjects) with patch.object(cg, "fetch_git_output", return_value=lines): - return cg.collect_version_updates(["../somerepo"], ["3.27.0..3.27.1"]) + return cg.collect_version_updates( + ["../somerepo"], "3.27.0", "3.27.x", ["3.27.0..3.27.1"] + ) def test_single_update(self): result = self._run(["Updated dependency 'openssl' from version 1.1.1 to 3.0.0"])