Add packaging.python.org translation tracking to the dashboard

completion.py

@@ -22,7 +22,7 @@ def branches_from_peps() -> list[str]:
 
 def get_completion(
     clones_dir: str, repo: str
-) -> tuple[float, float, str, float, float]:
+) -> tuple[float, float, str, float, float, int, int]:
     clone_path = Path(clones_dir, 'translations', repo)
     for branch in branches_from_peps() + ['master', 'main']:
         try:
@@ -49,11 +49,13 @@ def get_completion(
         api_url='',
     )
     completion = project.completion
+    total_words = project.words
     core_excludes = ['**/*', '!bugs.po', '!tutorial/*', '!library/functions.po']
     project.filter(
         filters=Filters(False, True, 0, 100, False, False), exclude=core_excludes
     )
     core_completion = project.completion
+    core_total_words = project.words
 
     if completion:
         # Fetch commit from before 30 days ago and checkout
@@ -87,4 +89,4 @@ def get_completion(
     change = completion - month_ago_completion
     core_change = core_completion - month_ago_core_completion
 
-    return core_completion, completion, branch, core_change, change
+    return core_completion, completion, branch, core_change, change, total_words, core_total_words

generate.py

@@ -17,6 +17,11 @@
 import translated_names
 import contribute
 from completion import branches_from_peps, get_completion
+from packaging_completion import (
+    get_packaging_progress,
+    PackagingProjectData,
+    LOCALE_CODE_NORMALISATION,
+)
 from repositories import Language, get_languages_and_repos
 
 generation_time = datetime.now(timezone.utc)
@@ -67,19 +72,20 @@ def get_project_data(
 ) -> LanguageProjectData:
     built = language.code in languages_built
     if repo:
-        core_complation, completion, branch, core_change, change = get_completion(
+        core_completion, completion, branch, core_change, change, total_words, core_total_words = get_completion(
             clones_dir, repo
         )
     else:
-        core_complation = completion = 0.0
+        core_completion = completion = 0.0
         core_change = change = 0.0
+        total_words = core_total_words = 0
         branch = ''
 
     return LanguageProjectData(
         language,
         repo,
         branch,
-        core_complation,
+        core_completion,
         completion,
         core_change,
         change,
@@ -88,6 +94,8 @@ def get_project_data(
         or translated_names.babel_autonym(language.code)
         or '',
         contribution_link=contribute.get_contrib_link(language.code, repo),
+        total_words=total_words,
+        core_total_words=core_total_words,
     )
 
 
@@ -103,6 +111,76 @@ class LanguageProjectData:
     built: bool
     translated_name: str
     contribution_link: str | None
+    total_words: int = 0
+    core_total_words: int = 0
+
+
+@dataclass(frozen=True)
+class CombinedLanguageCard:
+    """One card per language combining CPython-docs and packaging.python.org data."""
+
+    language: Language
+    translated_name: str
+    cpython: LanguageProjectData | None
+    packaging: PackagingProjectData | None
+
+    @property
+    def completion_score(self) -> float:
+        """Volume-weighted completion score: translated words across docs and packaging."""
+        docs_words = self.cpython.total_words if self.cpython else 0
+        pkg_words = self.packaging.total_words if self.packaging else 0
+        total = docs_words + pkg_words
+        if total == 0:
+            return 0.0
+        docs_translated = (self.cpython.completion * docs_words / 100) if self.cpython else 0.0
+        pkg_translated = (self.packaging.completion * pkg_words / 100) if self.packaging else 0.0
+        return 100 * (docs_translated + pkg_translated) / total
+
+    @property
+    def recent_changes_words(self) -> float:
+        """Approximate English words translated in the last 30 days across all resources."""
+        docs_change = (self.cpython.change * self.cpython.total_words / 100) if self.cpython else 0.0
+        pkg_change = (self.packaging.change * self.packaging.total_words / 100) if self.packaging else 0.0
+        return docs_change + pkg_change
+
+
+def _card_sort_key(c: CombinedLanguageCard) -> float:
+    return c.completion_score
+
+
+def merge_progress(
+    completion_progress: list[LanguageProjectData],
+    packaging_progress: list[PackagingProjectData],
+) -> list[CombinedLanguageCard]:
+    """Add packaging progress to cards for CPython documentation languages."""
+    cards: dict[str, dict] = {}
+    for proj in completion_progress:
+        code = proj.language.code
+        cards[code] = {
+            'language': proj.language,
+            'translated_name': proj.translated_name,
+            'cpython': proj,
+            'packaging': None,
+        }
+    for proj in packaging_progress:  # type: ignore[assignment]
+        # Normalise packaging language codes so aliases (e.g. hi-in → hi)
+        # are merged onto the same card as the CPython entry.
+        code = LOCALE_CODE_NORMALISATION.get(proj.language.code, proj.language.code)
+        if code in cards:
+            cards[code]['packaging'] = proj
+    return sorted(
+        [
+            CombinedLanguageCard(
+                language=entry['language'],
+                translated_name=entry['translated_name'],
+                cpython=entry['cpython'],
+                packaging=entry['packaging'],
+            )
+            for entry in cards.values()
+        ],
+        key=_card_sort_key,
+        reverse=True,
+    )
 
 
 if __name__ == '__main__':
@@ -111,10 +189,12 @@ class LanguageProjectData:
     Path('build').mkdir(parents=True, exist_ok=True)
 
     completion_progress = list(get_completion_progress())
+    packaging_progress = get_packaging_progress(Path('clones'))
+    combined_progress = merge_progress(completion_progress, packaging_progress)
 
     env = Environment(loader=FileSystemLoader('templates'))
     index = env.get_template('index.html.jinja').render(
-        completion_progress=completion_progress,
+        combined_progress=combined_progress,
         generation_time=generation_time,
         duration=(datetime.now(timezone.utc) - generation_time).seconds,
     )
@@ -126,3 +206,7 @@ class LanguageProjectData:
     Path('build/index.json').write_text(
         json.dumps([asdict(project) for project in completion_progress], indent=2)
     )
+
+    Path('build/packaging.json').write_text(
+        json.dumps([asdict(project) for project in packaging_progress], indent=2)
+    )

packaging_completion.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+
+import git
+import urllib3
+from potodo.po_file import PoFileStats
+
+from repositories import Language
+
+RTD_TRANSLATIONS_URL = (
+    'https://app.readthedocs.org/api/v3/projects/'
+    'python-packaging-user-guide/translations/'
+)
+PACKAGING_REPO_URL = 'https://github.com/pypa/packaging.python.org.git'
+PACKAGING_REPO_BRANCH = 'translation/source'
+CHANGE_PERIOD = '30 days ago'
+
+# Some locale directory names use script subtags instead of region codes.
+# These explicit overrides take priority over the generic conversion.
+RTD_CODE_TO_LOCALE_OVERRIDES: dict[str, str] = {'zh-cn': 'zh_Hans', 'zh-tw': 'zh_Hant'}
+LOCALE_TO_RTD_CODE_OVERRIDES: dict[str, str] = {
+    v: k for k, v in RTD_CODE_TO_LOCALE_OVERRIDES.items()
+}
+
+# Normalise locale/RTD codes that are aliases for the same language so they
+# map to the canonical language code used elsewhere (e.g. in CPython devguide).
+LOCALE_CODE_NORMALISATION: dict[str, str] = {
+    # Hindi: packaging.python.org uses hi_IN / hi-in, CPython uses hi
+    'hi_IN': 'hi',
+    'hi-in': 'hi',
+}
+
+
+@dataclass(frozen=True)
+class PackagingProjectData:
+    language: Language
+    completion: float
+    change: float
+    built: bool
+    translated_name: str
+    total_words: int = 0
+
+
+def _rtd_code_to_locale(code: str) -> str:
+    """Convert RTD language code (e.g. 'pt-br') to locale dir format ('pt_BR')."""
+    if code in RTD_CODE_TO_LOCALE_OVERRIDES:
+        return RTD_CODE_TO_LOCALE_OVERRIDES[code]
+    parts = code.split('-')
+    if len(parts) == 2:
+        return f'{parts[0]}_{parts[1].upper()}'
+    return code
+
+
+def get_built_languages() -> dict[str, Language]:
+    """Return a dict mapping locale directory name to Language for built languages."""
+    built: dict[str, Language] = {}
+    url: str | None = RTD_TRANSLATIONS_URL
+    while url:
+        resp = urllib3.request('GET', url)
+        if resp.status != 200:
+            logging.error('ReadTheDocs API returned status %d for %s', resp.status, url)
+            break
+        data = json.loads(resp.data)
+        for result in data['results']:
+            rtd_code = result['language']['code']
+            language_name = result['language']['name']
+            locale = _rtd_code_to_locale(rtd_code)
+            built[locale] = Language(code=rtd_code, name=language_name)
+        url = data.get('next')
+    return built
+
+
+def _po_completion(po_path: Path) -> tuple[float, int]:
+    """Return (completion%, total_words) for a .po file."""
+    if not po_path.exists():
+        return 0.0, 0
+    try:
+        stats = PoFileStats(po_path)
+        words = stats.words
+        if words == 0:
+            return 0.0, 0
+        return 100 * stats.translated_words / words, words
+    except Exception:
+        logging.exception('Failed to parse %s', po_path)
+        return 0.0, 0
+
+
+def _get_locale_dirs(repo_path: Path) -> list[str]:
+    locales_dir = repo_path / 'locales'
+    if not locales_dir.exists():
+        return []
+    return [d.name for d in locales_dir.iterdir() if d.is_dir()]
+
+
+def get_packaging_progress(clones_dir: Path) -> list[PackagingProjectData]:
+    import translated_names
+
+    repo_path = clones_dir / 'packaging.python.org'
+    if not repo_path.exists():
+        clone_repo = git.Repo.clone_from(
+            PACKAGING_REPO_URL, repo_path, branch=PACKAGING_REPO_BRANCH
+        )
+    else:
+        clone_repo = git.Repo(repo_path)
+        clone_repo.git.fetch()
+        clone_repo.git.switch(PACKAGING_REPO_BRANCH)
+        clone_repo.git.pull()
+
+    built_languages = get_built_languages()
+
+    locales = _get_locale_dirs(repo_path)
+    po_paths = {
+        locale: repo_path / 'locales' / locale / 'LC_MESSAGES' / 'messages.po'
+        for locale in locales
+    }
+
+    # Calculate current completions for all locales
+    current_completions: dict[str, tuple[float, int]] = {
+        locale: _po_completion(po_paths[locale]) for locale in locales
+    }
+
+    # Find the 30-days-ago commit once and gather historical completions in a
+    # single checkout round-trip (avoids N checkouts, one per locale).
+    month_ago_completions: dict[str, float] = {}
+    if any(pct for pct, _ in current_completions.values()):
+        try:
+            old_commit = next(
+                clone_repo.iter_commits('HEAD', max_count=1, before=CHANGE_PERIOD)
+            )
+        except StopIteration:
+            pass
+        else:
+            clone_repo.git.checkout(old_commit.hexsha)
+            for locale in locales:
+                month_ago_completions[locale] = _po_completion(po_paths[locale])[0]
+            clone_repo.git.checkout(PACKAGING_REPO_BRANCH)
+
+    results = []
+    for locale in locales:
+        completion, total_words = current_completions[locale]
+        change = completion - month_ago_completions.get(locale, 0.0)
+
+        # Determine language code and name.
+        # Normalise known aliases (e.g. hi_IN → hi) before lookup.
+        normalised_locale = LOCALE_CODE_NORMALISATION.get(locale, locale)
+        if normalised_locale in built_languages:
+            language = built_languages[normalised_locale]
+        elif locale in built_languages:
+            language = built_languages[locale]
+        else:
+            # Convert locale dir to RTD-style code, respecting explicit overrides.
+            if normalised_locale in LOCALE_TO_RTD_CODE_OVERRIDES:
+                rtd_code = LOCALE_TO_RTD_CODE_OVERRIDES[normalised_locale]
+            elif locale in LOCALE_TO_RTD_CODE_OVERRIDES:
+                rtd_code = LOCALE_TO_RTD_CODE_OVERRIDES[locale]
+            else:
+                parts = normalised_locale.split('_')
+                if len(parts) == 2:
+                    rtd_code = f'{parts[0]}-{parts[1].lower()}'
+                else:
+                    rtd_code = normalised_locale.lower()
+            # Use babel for name; fall back to the code string itself.
+            lang_name = translated_names.babel_autonym(rtd_code) or rtd_code
+            language = Language(code=rtd_code, name=lang_name)
+
+        translated_name = translated_names.babel_autonym(language.code) or ''
+
+        results.append(
+            PackagingProjectData(
+                language=language,
+                completion=completion,
+                change=change,
+                built=normalised_locale in built_languages or locale in built_languages,
+                translated_name=translated_name,
+                total_words=total_words,
+            )
+        )
+
+    return results