diff --git a/completion.py b/completion.py index 6a661329d..490700dc4 100644 --- a/completion.py +++ b/completion.py @@ -22,7 +22,7 @@ def branches_from_peps() -> list[str]: def get_completion( clones_dir: str, repo: str -) -> tuple[float, float, str, float, float]: +) -> tuple[float, float, str, float, float, int, int]: clone_path = Path(clones_dir, 'translations', repo) for branch in branches_from_peps() + ['master', 'main']: try: @@ -49,11 +49,13 @@ def get_completion( api_url='', ) completion = project.completion + total_words = project.words core_excludes = ['**/*', '!bugs.po', '!tutorial/*', '!library/functions.po'] project.filter( filters=Filters(False, True, 0, 100, False, False), exclude=core_excludes ) core_completion = project.completion + core_total_words = project.words if completion: # Fetch commit from before 30 days ago and checkout @@ -87,4 +89,4 @@ def get_completion( change = completion - month_ago_completion core_change = core_completion - month_ago_core_completion - return core_completion, completion, branch, core_change, change + return core_completion, completion, branch, core_change, change, total_words, core_total_words diff --git a/generate.py b/generate.py index a6df9c6f3..e3a61abd7 100644 --- a/generate.py +++ b/generate.py @@ -17,6 +17,11 @@ import translated_names import contribute from completion import branches_from_peps, get_completion +from packaging_completion import ( + get_packaging_progress, + PackagingProjectData, + LOCALE_CODE_NORMALISATION, +) from repositories import Language, get_languages_and_repos generation_time = datetime.now(timezone.utc) @@ -67,19 +72,20 @@ def get_project_data( ) -> LanguageProjectData: built = language.code in languages_built if repo: - core_complation, completion, branch, core_change, change = get_completion( + core_completion, completion, branch, core_change, change, total_words, core_total_words = get_completion( clones_dir, repo ) else: - core_complation = completion = 0.0 + core_completion = completion = 0.0 core_change = change = 0.0 + total_words = core_total_words = 0 branch = '' return LanguageProjectData( language, repo, branch, - core_complation, + core_completion, completion, core_change, change, @@ -88,6 +94,8 @@ def get_project_data( or translated_names.babel_autonym(language.code) or '', contribution_link=contribute.get_contrib_link(language.code, repo), + total_words=total_words, + core_total_words=core_total_words, ) @@ -103,6 +111,76 @@ class LanguageProjectData: built: bool translated_name: str contribution_link: str | None + total_words: int = 0 + core_total_words: int = 0 + + +@dataclass(frozen=True) +class CombinedLanguageCard: + """One card per language combining CPython-docs and packaging.python.org data.""" + + language: Language + translated_name: str + cpython: LanguageProjectData | None + packaging: PackagingProjectData | None + + @property + def completion_score(self) -> float: + """Volume-weighted completion score: translated words across docs and packaging.""" + docs_words = self.cpython.total_words if self.cpython else 0 + pkg_words = self.packaging.total_words if self.packaging else 0 + total = docs_words + pkg_words + if total == 0: + return 0.0 + docs_translated = (self.cpython.completion * docs_words / 100) if self.cpython else 0.0 + pkg_translated = (self.packaging.completion * pkg_words / 100) if self.packaging else 0.0 + return 100 * (docs_translated + pkg_translated) / total + + @property + def recent_changes_words(self) -> float: + """Approximate English words translated in the last 30 days across all resources.""" + docs_change = (self.cpython.change * self.cpython.total_words / 100) if self.cpython else 0.0 + pkg_change = (self.packaging.change * self.packaging.total_words / 100) if self.packaging else 0.0 + return docs_change + pkg_change + + +def _card_sort_key(c: CombinedLanguageCard) -> float: + return c.completion_score + + +def merge_progress( + completion_progress: list[LanguageProjectData], + packaging_progress: list[PackagingProjectData], +) -> list[CombinedLanguageCard]: + """Add packaging progress to cards for CPython documentation languages.""" + cards: dict[str, dict] = {} + for proj in completion_progress: + code = proj.language.code + cards[code] = { + 'language': proj.language, + 'translated_name': proj.translated_name, + 'cpython': proj, + 'packaging': None, + } + for proj in packaging_progress: # type: ignore[assignment] + # Normalise packaging language codes so aliases (e.g. hi-in → hi) + # are merged onto the same card as the CPython entry. + code = LOCALE_CODE_NORMALISATION.get(proj.language.code, proj.language.code) + if code in cards: + cards[code]['packaging'] = proj + return sorted( + [ + CombinedLanguageCard( + language=entry['language'], + translated_name=entry['translated_name'], + cpython=entry['cpython'], + packaging=entry['packaging'], + ) + for entry in cards.values() + ], + key=_card_sort_key, + reverse=True, + ) if __name__ == '__main__': @@ -111,10 +189,12 @@ class LanguageProjectData: Path('build').mkdir(parents=True, exist_ok=True) completion_progress = list(get_completion_progress()) + packaging_progress = get_packaging_progress(Path('clones')) + combined_progress = merge_progress(completion_progress, packaging_progress) env = Environment(loader=FileSystemLoader('templates')) index = env.get_template('index.html.jinja').render( - completion_progress=completion_progress, + combined_progress=combined_progress, generation_time=generation_time, duration=(datetime.now(timezone.utc) - generation_time).seconds, ) @@ -126,3 +206,7 @@ class LanguageProjectData: Path('build/index.json').write_text( json.dumps([asdict(project) for project in completion_progress], indent=2) ) + + Path('build/packaging.json').write_text( + json.dumps([asdict(project) for project in packaging_progress], indent=2) + ) diff --git a/packaging_completion.py b/packaging_completion.py new file mode 100644 index 000000000..d78b81a28 --- /dev/null +++ b/packaging_completion.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass +from pathlib import Path + +import git +import urllib3 +from potodo.po_file import PoFileStats + +from repositories import Language + +RTD_TRANSLATIONS_URL = ( + 'https://app.readthedocs.org/api/v3/projects/' + 'python-packaging-user-guide/translations/' +) +PACKAGING_REPO_URL = 'https://github.com/pypa/packaging.python.org.git' +PACKAGING_REPO_BRANCH = 'translation/source' +CHANGE_PERIOD = '30 days ago' + +# Some locale directory names use script subtags instead of region codes. +# These explicit overrides take priority over the generic conversion. +RTD_CODE_TO_LOCALE_OVERRIDES: dict[str, str] = {'zh-cn': 'zh_Hans', 'zh-tw': 'zh_Hant'} +LOCALE_TO_RTD_CODE_OVERRIDES: dict[str, str] = { + v: k for k, v in RTD_CODE_TO_LOCALE_OVERRIDES.items() +} + +# Normalise locale/RTD codes that are aliases for the same language so they +# map to the canonical language code used elsewhere (e.g. in CPython devguide). +LOCALE_CODE_NORMALISATION: dict[str, str] = { + # Hindi: packaging.python.org uses hi_IN / hi-in, CPython uses hi + 'hi_IN': 'hi', + 'hi-in': 'hi', +} + + +@dataclass(frozen=True) +class PackagingProjectData: + language: Language + completion: float + change: float + built: bool + translated_name: str + total_words: int = 0 + + +def _rtd_code_to_locale(code: str) -> str: + """Convert RTD language code (e.g. 'pt-br') to locale dir format ('pt_BR').""" + if code in RTD_CODE_TO_LOCALE_OVERRIDES: + return RTD_CODE_TO_LOCALE_OVERRIDES[code] + parts = code.split('-') + if len(parts) == 2: + return f'{parts[0]}_{parts[1].upper()}' + return code + + +def get_built_languages() -> dict[str, Language]: + """Return a dict mapping locale directory name to Language for built languages.""" + built: dict[str, Language] = {} + url: str | None = RTD_TRANSLATIONS_URL + while url: + resp = urllib3.request('GET', url) + if resp.status != 200: + logging.error('ReadTheDocs API returned status %d for %s', resp.status, url) + break + data = json.loads(resp.data) + for result in data['results']: + rtd_code = result['language']['code'] + language_name = result['language']['name'] + locale = _rtd_code_to_locale(rtd_code) + built[locale] = Language(code=rtd_code, name=language_name) + url = data.get('next') + return built + + +def _po_completion(po_path: Path) -> tuple[float, int]: + """Return (completion%, total_words) for a .po file.""" + if not po_path.exists(): + return 0.0, 0 + try: + stats = PoFileStats(po_path) + words = stats.words + if words == 0: + return 0.0, 0 + return 100 * stats.translated_words / words, words + except Exception: + logging.exception('Failed to parse %s', po_path) + return 0.0, 0 + + +def _get_locale_dirs(repo_path: Path) -> list[str]: + locales_dir = repo_path / 'locales' + if not locales_dir.exists(): + return [] + return [d.name for d in locales_dir.iterdir() if d.is_dir()] + + +def get_packaging_progress(clones_dir: Path) -> list[PackagingProjectData]: + import translated_names + + repo_path = clones_dir / 'packaging.python.org' + if not repo_path.exists(): + clone_repo = git.Repo.clone_from( + PACKAGING_REPO_URL, repo_path, branch=PACKAGING_REPO_BRANCH + ) + else: + clone_repo = git.Repo(repo_path) + clone_repo.git.fetch() + clone_repo.git.switch(PACKAGING_REPO_BRANCH) + clone_repo.git.pull() + + built_languages = get_built_languages() + + locales = _get_locale_dirs(repo_path) + po_paths = { + locale: repo_path / 'locales' / locale / 'LC_MESSAGES' / 'messages.po' + for locale in locales + } + + # Calculate current completions for all locales + current_completions: dict[str, tuple[float, int]] = { + locale: _po_completion(po_paths[locale]) for locale in locales + } + + # Find the 30-days-ago commit once and gather historical completions in a + # single checkout round-trip (avoids N checkouts, one per locale). + month_ago_completions: dict[str, float] = {} + if any(pct for pct, _ in current_completions.values()): + try: + old_commit = next( + clone_repo.iter_commits('HEAD', max_count=1, before=CHANGE_PERIOD) + ) + except StopIteration: + pass + else: + clone_repo.git.checkout(old_commit.hexsha) + for locale in locales: + month_ago_completions[locale] = _po_completion(po_paths[locale])[0] + clone_repo.git.checkout(PACKAGING_REPO_BRANCH) + + results = [] + for locale in locales: + completion, total_words = current_completions[locale] + change = completion - month_ago_completions.get(locale, 0.0) + + # Determine language code and name. + # Normalise known aliases (e.g. hi_IN → hi) before lookup. + normalised_locale = LOCALE_CODE_NORMALISATION.get(locale, locale) + if normalised_locale in built_languages: + language = built_languages[normalised_locale] + elif locale in built_languages: + language = built_languages[locale] + else: + # Convert locale dir to RTD-style code, respecting explicit overrides. + if normalised_locale in LOCALE_TO_RTD_CODE_OVERRIDES: + rtd_code = LOCALE_TO_RTD_CODE_OVERRIDES[normalised_locale] + elif locale in LOCALE_TO_RTD_CODE_OVERRIDES: + rtd_code = LOCALE_TO_RTD_CODE_OVERRIDES[locale] + else: + parts = normalised_locale.split('_') + if len(parts) == 2: + rtd_code = f'{parts[0]}-{parts[1].lower()}' + else: + rtd_code = normalised_locale.lower() + # Use babel for name; fall back to the code string itself. + lang_name = translated_names.babel_autonym(rtd_code) or rtd_code + language = Language(code=rtd_code, name=lang_name) + + translated_name = translated_names.babel_autonym(language.code) or '' + + results.append( + PackagingProjectData( + language=language, + completion=completion, + change=change, + built=normalised_locale in built_languages or locale in built_languages, + translated_name=translated_name, + total_words=total_words, + ) + ) + + return results diff --git a/templates/index.html.jinja b/templates/index.html.jinja index 9b5897455..2cae2d339 100644 --- a/templates/index.html.jinja +++ b/templates/index.html.jinja @@ -1,39 +1,92 @@ {% extends "base.html.jinja" %} {% block main %} -