ENT-14728: Changed hardcoded remote for generate-changelog

src/cfengine_cli/changelog.py

@@ -20,6 +20,8 @@
 import re
 import subprocess
 import logging
+import sys
+from cfengine_cli.utils import get_upstream_remote
 
 # ---------------------------------------------------------------------------
 # Regex
@@ -42,21 +44,42 @@ def fetch_git_output(path, args):
     ).stdout.splitlines(keepends=True)
 
 
+REPO_OWNER = "cfengine"
+
+
+def get_versions(prev, branch, repo, repo_path, git_args, repo_owner=REPO_OWNER):
+    """Assumes cfengine dev generate-changelog is run inside cfengine repos only"""
+    if git_args:
+        return git_args
+    else:
+        name = os.path.basename(repo)
+        remote = get_upstream_remote(name, repo_owner, repo_path)
+        if remote is None:
+            logging.error(
+                "No remote in '{}' points to 'github.com/{}/{}'".format(
+                    repo, repo_owner, name
+                )
+            )
+            exit(1)
+        return [f"{prev}..{remote}/{branch}"]
+
+
 # ---------------------------------------------------------------------------
 # Fetch and merge dependency upgrades / reverts
 # ---------------------------------------------------------------------------
-def collect_version_updates(repos, git_args):
+def collect_version_updates(repos, prev, branch, git_args):
     dep_history: dict[str, list[tuple[str, str]]] = {}
     for repo in repos:
+        repo_path = os.path.join(os.getcwd(), repo)
         for raw in fetch_git_output(
-            os.path.join(os.getcwd(), repo),
+            repo_path,
             [
                 "log",
                 "--no-merges",
                 "--reverse",
                 "--pretty=format:%s",
             ]
-            + git_args,
+            + get_versions(prev, branch, repo, repo_path, git_args),
         ):
             subject = raw.decode().strip()
 
@@ -212,21 +235,22 @@ def add_entry(sha, msg):
         add_entry(sha, title)
 
 
-def parse_git_log(repos, git_args):
+def parse_git_log(repos, prev, branch, git_args):
     """Walk git history across repos and return (entry_list, missed_tickets)."""
     entries = {}  # sha -> [msg, ...]
     linked_shas = {}  # sha -> [linked_sha, ...]
     sha_to_tracker = {}  # sha -> set of ticket strings
 
     for repo in repos:
+        repo_path = os.path.join(os.getcwd(), repo)
         for raw_sha in fetch_git_output(
-            os.path.join(os.getcwd(), repo),
+            repo_path,
             [
                 "rev-list",
                 "--no-merges",
                 "--reverse",
             ]
-            + git_args,
+            + get_versions(prev, branch, repo, repo_path, git_args),
         ):
             parse_sha(raw_sha, entries, sha_to_tracker, linked_shas, repo)
 
@@ -295,6 +319,58 @@ def get_current_branch():
     ).strip()
 
 
+def remote_is_ahead(repo_path, remote, branch):
+    out = subprocess.check_output(
+        ["git", "-C", repo_path, "ls-remote", remote, f"refs/heads/{branch}"],
+        text=True,
+    ).split()
+    if not out:
+        return False  # branch doesn't exist on the remote
+    try:
+        local = subprocess.check_output(
+            ["git", "-C", repo_path, "rev-parse", f"{remote}/{branch}"],
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except subprocess.CalledProcessError:
+        return True  # never fetched
+    return out[0] != local
+
+
+def _fetch_prompt():
+    # Prompt on stderr so it doesn't end up in the changelog with -o
+    answer = None
+    while answer not in ("", "y", "n", "yes", "no"):
+        print("Fetch? (Y/n): ", end="", file=sys.stderr, flush=True)
+        answer = input().strip().lower()
+    return answer in ("", "y", "yes")
+
+
+def fetch_upstream(repos, branch, assume_yes=False, repo_owner=REPO_OWNER):
+    """Fetch the upstream remote in each repo where it's behind GitHub,
+    asking first unless assume_yes is set."""
+    outdated = {}  # repo_path -> remote
+    for repo in repos:
+        repo_path = os.path.join(os.getcwd(), repo)
+        remote = get_upstream_remote(os.path.basename(repo), repo_owner, repo_path)
+        if remote is None:
+            continue  # reported by get_versions
+        if remote_is_ahead(repo_path, remote, branch):
+            outdated[repo_path] = remote
+
+    if not outdated:
+        return
+    for repo_path, remote in outdated.items():
+        print(
+            f"'{remote}/{branch}' is out of date in {os.path.basename(repo_path)}",
+            file=sys.stderr,
+        )
+    if not assume_yes and not _fetch_prompt():
+        return
+    for repo_path, remote in outdated.items():
+        subprocess.run(["git", "-C", repo_path, "fetch", remote], check=True)
+
+
 # ---------------------------------------------------------------------------
 # Versionstuff
 # ---------------------------------------------------------------------------
@@ -362,13 +438,15 @@ def generate_changelog_impl(args):
         f"next version to be {expected}, but cfversion says {actual}"
     )
 
-    if args.git_args:
-        versions = args.git_args
-    else:
-        # Assumes tag exists and origin follows the same naming scheme for this release
-        versions = [f"{prev}..origin/{branch}"]
+    # packaging changes only included in enterprise
+    is_enterprise = os.path.basename(os.path.abspath(os.curdir)) == "enterprise"
+
+    if not args.git_args and not args.no_fetch:
+        fetch_upstream(
+            repos + (REPO_MAP["packaging"] if is_enterprise else []), branch, args.yes
+        )
 
-    entry_list, missed_tickets = parse_git_log(repos, versions)
+    entry_list, missed_tickets = parse_git_log(repos, prev, branch, args.git_args)
     entry_list.sort()
 
     output = ""
@@ -386,12 +464,14 @@ def generate_changelog_impl(args):
             else f"## {args.git_args[0].split('..')[-1]}\n"
         )
 
-    if (
-        os.path.basename(os.path.abspath(os.curdir)) == "enterprise"
-    ):  # packaging changes only included in enterprise
-        pkg_changes, missed_pkg_tickets = parse_git_log(REPO_MAP["packaging"], versions)
+    if is_enterprise:
+        pkg_changes, missed_pkg_tickets = parse_git_log(
+            REPO_MAP["packaging"], prev, branch, args.git_args
+        )
         missed_tickets.update(missed_pkg_tickets)
-        pkg_changes += collect_version_updates(REPO_MAP["packaging"], versions)
+        pkg_changes += collect_version_updates(
+            REPO_MAP["packaging"], prev, branch, args.git_args
+        )
         pkg_changes.sort()
         if pkg_changes:
             lines.append("\n**Packaging changes:**")

src/cfengine_cli/deptool.py

@@ -8,6 +8,8 @@
 import subprocess
 import sys
 
+from cfengine_cli.utils import get_upstream_remote
+
 ACTIVE_BRANCHES = ["3.24.x", "3.27.x", "master"]
 
 HUMAN_NAME = {
@@ -213,20 +215,8 @@ def is_git_branch(self, ref):
             return False
         return True
 
-    def upstream_remote(self):
-        result = self.run_command("remote", "-v", capture_output=True)
-        upstream = "github\\.com[:/]{}/{}(\\.git)?/?$".format(
-            re.escape(self.repo_owner), re.escape(self.repo_name)
-        )
-        for line in result.stdout.splitlines():
-            parts = line.split()
-            # GitHub owner and repository names are case-insensitive
-            if len(parts) >= 2 and re.search(upstream, parts[1], re.IGNORECASE):
-                return parts[0]
-        return None
-
     def upstream_branch(self, ref):
-        remote = self.upstream_remote()
+        remote = get_upstream_remote(self.repo_name, self.repo_owner, self.repo_path)
         if remote is None:
             log.warning(
                 "No remote points to github.com/{}/{}, using local refs".format(

src/cfengine_cli/main.py

@@ -189,6 +189,19 @@ def _get_arg_parser():
         dest="show_version",
         help="Print the version from .CFVERSION and exit",
     )
+    fetch = parser.add_mutually_exclusive_group()
+    fetch.add_argument(
+        "-y",
+        "--yes",
+        action="store_true",
+        help="Fetch out-of-date upstream branches without asking",
+    )
+    fetch.add_argument(
+        "--no-fetch",
+        action="store_true",
+        dest="no_fetch",
+        help="Don't check for or fetch upstream changes, use local remote-tracking branches as they are",
+    )
     parser.add_argument(
         "git_args",
         nargs="*",

src/cfengine_cli/utils.py

@@ -1,3 +1,7 @@
+import subprocess
+import re
+
+
 class UserError(Exception):
     """Exception raised when a user has most likely made a mistake"""
 
@@ -7,3 +11,18 @@ def __init__(self, message):
 
     def __str__(self):
         return f"Error: {self.message}"
+
+
+def get_upstream_remote(repo_name, repo_owner, repo_path):
+    upstream = r"github\.com[:/]{}/{}(\.git)?/?$".format(
+        re.escape(repo_owner), re.escape(repo_name)
+    )
+    remotes = subprocess.check_output(
+        ["git", "-C", repo_path, "remote", "-v"], text=True
+    )
+    for line in remotes.splitlines():
+        parts = line.split()
+        # GitHub owner and repository names are case-insensitive
+        if len(parts) >= 2 and re.search(upstream, parts[1], re.IGNORECASE):
+            return parts[0]
+    return None

tests/unit/test_changelog.py

@@ -18,7 +18,9 @@ class TestCollectVersionUpdates(unittest.TestCase):
     def _run(self, subjects):
         lines = _lines(*subjects)
         with patch.object(cg, "fetch_git_output", return_value=lines):
-            return cg.collect_version_updates(["../somerepo"], ["3.27.0..3.27.1"])
+            return cg.collect_version_updates(
+                ["../somerepo"], "3.27.0", "3.27.x", ["3.27.0..3.27.1"]
+            )
 
     def test_single_update(self):
         result = self._run(["Updated dependency 'openssl' from version 1.1.1 to 3.0.0"])