From e31a79f52174318893ced31fed82ebe9f19ad0ae Mon Sep 17 00:00:00 2001 From: yiaany Date: Sat, 29 Aug 2026 21:06:01 +0500 Subject: [PATCH 1/2] docs: add full-site version snapshots --- .gitattributes | 2 + .github/workflows/docs.yaml | 37 ++- .github/workflows/docs_pr.yaml | 37 ++- dev/release/README.md | 23 ++ docs/README.md | 141 +++++++--- docs/scripts/assemble_site.py | 147 +++++++++++ docs/scripts/release_conf.py | 60 +++++ docs/scripts/snapshot_site.py | 282 ++++++++++++++++++++ docs/scripts/validate_site.py | 420 ++++++++++++++++++++++++++++++ docs/scripts/versioned_docs.py | 116 +++++++++ docs/source/_static/versions.json | 14 + docs/source/conf.py | 11 +- docs/tests/test_versioned_docs.py | 330 +++++++++++++++++++++++ 13 files changed, 1575 insertions(+), 45 deletions(-) create mode 100644 docs/scripts/assemble_site.py create mode 100644 docs/scripts/release_conf.py create mode 100644 docs/scripts/snapshot_site.py create mode 100644 docs/scripts/validate_site.py create mode 100644 docs/scripts/versioned_docs.py create mode 100644 docs/source/_static/versions.json create mode 100644 docs/tests/test_versioned_docs.py diff --git a/.gitattributes b/.gitattributes index 84b47a6fc56e1..75ac548699177 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,6 @@ .github/ export-ignore +docs/*.sh text eol=lf +docs/scripts/*.sh text eol=lf datafusion/core/tests/data/newlines_in_values.csv text eol=lf datafusion/proto/src/generated/prost.rs linguist-generated datafusion/proto/src/generated/pbjson.rs linguist-generated diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index c52ed5681c64f..217c6f01b5601 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -20,12 +20,19 @@ on: branches: - main paths: + - .gitattributes - .asf.yaml - .github/workflows/docs.yaml - docs/** + - pyproject.toml + - uv.lock name: Deploy DataFusion site +concurrency: + group: docs-deployment + cancel-in-progress: false + jobs: build-docs: permissions: @@ -34,16 +41,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout docs sources - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Checkout asf-site branch - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: asf-site path: asf-site - name: Setup uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install dependencies run: uv sync --package datafusion-docs @@ -53,7 +60,7 @@ jobs: sudo apt-get update sudo apt-get install -y graphviz - name: Install cargo-depgraph - uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 + uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 with: tool: cargo-depgraph@1.6 @@ -63,7 +70,21 @@ jobs: cd docs uv run --package datafusion-docs ./build.sh - - name: Copy & push the generated HTML + - name: Assemble deployment site + run: | + uv run --package datafusion-docs python docs/scripts/assemble_site.py \ + --current-site docs/build/html \ + --published-site asf-site \ + --output-site deployment-site + cp .asf.yaml deployment-site/ + touch deployment-site/.nojekyll + + - name: Validate deployment site + run: >- + uv run --package datafusion-docs python docs/scripts/validate_site.py + --site-root deployment-site --require-snapshots + + - name: Copy and push the generated HTML run: | set -x cd asf-site/ @@ -71,15 +92,13 @@ jobs: -a \ --delete \ --exclude '/.git/' \ - ../docs/build/html/ \ + ../deployment-site/ \ ./ - cp ../.asf.yaml . - touch .nojekyll git status --porcelain if [ "$(git status --porcelain)" != "" ]; then git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git add --all git commit -m 'Publish built docs triggered by ${{ github.sha }}' - git push || git push --force + git push fi diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml index 2cbb64274a526..7035111b70fb0 100644 --- a/.github/workflows/docs_pr.yaml +++ b/.github/workflows/docs_pr.yaml @@ -25,13 +25,23 @@ concurrency: on: push: paths: + - ".gitattributes" - "docs/**" + - ".github/workflows/docs.yaml" + - ".github/workflows/docs_pr.yaml" + - "pyproject.toml" + - "uv.lock" branches: - main - branch-* pull_request: paths: + - ".gitattributes" - "docs/**" + - ".github/workflows/docs.yaml" + - ".github/workflows/docs_pr.yaml" + - "pyproject.toml" + - "uv.lock" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: @@ -45,12 +55,12 @@ jobs: name: Test doc build runs-on: ubuntu-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: submodules: true fetch-depth: 1 - name: Setup uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - name: Install doc dependencies run: uv sync --package datafusion-docs - name: Install Graphviz @@ -59,7 +69,7 @@ jobs: sudo apt-get update sudo apt-get install -y graphviz - name: Install cargo-depgraph - uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 + uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5 with: tool: cargo-depgraph@1.6 - name: Build docs html and check for warnings @@ -67,3 +77,24 @@ jobs: set -x cd docs uv run --package datafusion-docs ./build.sh # fails on errors + - name: Fetch exact release tag + run: git fetch --depth=1 origin tag 55.0.0 + - name: Test versioned documentation tooling + run: uv run --package datafusion-docs python -m unittest discover -s docs/tests + - name: Validate current site + run: >- + uv run --package datafusion-docs python docs/scripts/validate_site.py + --site-root docs/build/html + - name: Build exact release snapshot + run: | + uv run --package datafusion-docs python docs/scripts/snapshot_site.py \ + 55.0.0 55.0.0 /tmp/datafusion-release-site + - name: Validate release deployment + run: | + uv run --package datafusion-docs python docs/scripts/assemble_site.py \ + --current-site docs/build/html \ + --published-site /tmp/datafusion-release-site \ + --output-site docs/build/validation-site + uv run --package datafusion-docs python docs/scripts/validate_site.py \ + --site-root docs/build/validation-site \ + --require-snapshots diff --git a/dev/release/README.md b/dev/release/README.md index d49c1c9125235..bc2d6beaad6b5 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -301,6 +301,29 @@ git tag 50.3.0 git push apache 50.3.0 ``` +#### Publish the versioned documentation + +For the initial `55.0.0` proof of concept, make the final exact tag available +locally and build its complete documentation site into a temporary publication +package: + +```shell +uv run --package datafusion-docs python docs/scripts/snapshot_site.py 55.0.0 55.0.0 /tmp/datafusion-release-site +``` + +The command verifies the tag's peeled commit and creates the one immutable +publication unit `versions/55.0.0/`. Manually copy that directory to the same +location on the `asf-site` branch, then review, commit, and push the branch. The +command never commits or pushes and refuses to replace an existing release. + +Do not simply publish the snapshot before merging the feature PR: the old docs +deployment can erase it with `rsync --delete`. Keep the feature PR in draft until +maintainers choose either a preliminary retention-only merge before publication, +or a coordinated window that confirms no old docs deployment is running or +queued, publishes the snapshot, merges the retention and picker change, and +verifies `asf-site/versions/55.0.0/` after the new deployment. Automating future +release publication and backfilling older releases remain follow-up work. + ### 10. Publish on Crates.io Only approved releases of the tarball should be published to diff --git a/docs/README.md b/docs/README.md index 48fdcefdeae1a..53a704a61543f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,63 +19,142 @@ # DataFusion Documentation -This folder contains the source content of the [User Guide](./source/user-guide) -and [Contributor Guide](./source/contributor-guide). These are both published to -https://datafusion.apache.org/ as part of the release process. +This folder contains the sources for https://datafusion.apache.org/. The root +site is built continuously from `main`. Complete, immutable release sites are +published under `/versions//`. ## Dependencies -Install build dependencies and build the documentation using +From the repository root, install the documentation dependencies using [uv](https://docs.astral.sh/uv/): ```sh -uv sync -uv run bash build.sh +uv sync --package datafusion-docs ``` The docs build regenerates the workspace dependency graph via `docs/scripts/generate_dependency_graph.sh`, so ensure `cargo`, `cargo-depgraph` (`cargo install cargo-depgraph --version ^1.6 --locked`), and Graphviz `dot` (`brew install graphviz` or `sudo apt-get install -y graphviz`) are available. +`.gitattributes` keeps documentation shell scripts LF-terminated so the same +scripts run from Linux and WSL checkouts. -## Build & Preview +## Build and Preview -Run the provided script to build the HTML pages. +Build the current complete site from the repository root: ```bash -# If using venv, ensure you have activated it -./build.sh +uv run --package datafusion-docs docs/build.sh ``` -The HTML will be generated into a `build` directory. Open `build/html/index.html` -in your preferred browser, e.g. +The HTML is generated in `docs/build/html`. Serve it over HTTP because browsers +do not load the version manifest from `file:` URLs: -Preview the site on Linux by running this command. +```bash +python3 -m http.server --directory docs/build/html 8000 +``` + +Then open http://localhost:8000/. + +The public and assembled layouts are: + +```text +https://datafusion.apache.org/ +|-- user-guide/ +|-- library-user-guide/ +|-- contributor-guide/ +|-- _static/versions.json +`-- versions/55.0.0/ + |-- user-guide/ + |-- library-user-guide/ + |-- contributor-guide/ + |-- download.html + |-- search.html + |-- _sources/ + |-- _static/ + `-- sitemap.xml +``` + +## Release Snapshots + +`docs/source/_static/versions.json` is both the PyData version-picker manifest +and the release catalog. It contains `Development` at the site root and records +each release's semantic version, exact tag, and exact 40-character commit. + +The first snapshot is the lightweight tag `55.0.0`, which peels to +`d5552342012888b7d1a3ab88d92e3d292fc0cde0`. Create its publication package +from the repository root with the tag available locally: ```bash -# On macOS -open build/html/index.html -# On Linux with Firefox -firefox build/html/index.html +uv run --package datafusion-docs python docs/scripts/snapshot_site.py \ + 55.0.0 55.0.0 /tmp/datafusion-release-site ``` -## Making Changes +The command creates only +`/tmp/datafusion-release-site/versions/55.0.0/`. It refuses to overwrite that +directory, rejects output overlapping the repository, builds in an isolated +detached worktree, and does not commit or push. + +The snapshot uses the tag's complete documentation tree, templates, static +files, helper extension, build script dependency graph, and locked documentation +dependencies. `release_conf.py` supplies only the publication prefix, canonical +base URL, exactly pinned `sphinx-sitemap` extension, version picker, exact GitHub +tag, corrected repository name, and a release-local redirect. After Sphinx +builds, `snapshot_site.py` rewrites only published links that would otherwise +escape to current DataFusion docs or mutable DataFusion `main` and `latest` +targets, including links in the agent-facing `llms.txt`. This includes one narrow +fix for the tagged broken `/contributor-guide/gsoc_application_guidelines.html` +link, which is redirected to the tagged +`contributor-guide/gsoc/gsoc_application_guidelines_2025.html` page. Tagged +source files and generated `_sources` files are never modified. In particular, +the tagged statement that 55.0.0 has not been released yet remains unchanged. + +Validate a complete assembled site with: + +```bash +uv run --package datafusion-docs python docs/scripts/assemble_site.py \ + --current-site docs/build/html \ + --published-site /tmp/datafusion-release-site \ + --output-site /tmp/datafusion-site +uv run --package datafusion-docs python docs/scripts/validate_site.py \ + --site-root /tmp/datafusion-site --require-snapshots +``` + +The assembler starts with a fresh current build, copies the existing complete +`asf-site/versions/` archive unchanged, and creates a root sitemap index. It does +not use picker entries as a deletion list, so removing an old release from the +picker cannot erase its archive. + +## Publication Bootstrap + +The old deployment uses unrestricted `rsync --delete`, so publishing a snapshot +before snapshot retention reaches `main` is not race-free. Keep the feature PR +in draft until maintainers agree on one of these bootstrap procedures: -To make changes to the docs, simply make a Pull Request with your -proposed changes as normal. When the PR is merged the docs will be -automatically updated. +1. Merge a preliminary retention-only workflow change, wait for it to deploy, + publish `versions/55.0.0/` manually to `asf-site`, then merge the picker and + validation changes. +2. Use a coordinated window: confirm no old documentation deployment is running + or queued, publish the snapshot, merge this change immediately, wait for the + new serialized deployment, and verify `asf-site/versions/55.0.0/` afterward. -## Release Process +For manual publication, copy the single generated `versions/55.0.0/` directory +to the same location in an `asf-site` worktree. Review, commit, and push that +branch manually. Never replace an existing release directory. The snapshot +tooling never commits, pushes, or publishes. -This documentation is hosted at https://datafusion.apache.org/ +## Site Deployment -When the PR is merged to the `main` branch of the DataFusion -repository, a [github workflow](https://github.com/apache/datafusion/blob/main/.github/workflows/docs.yaml) which: +When a documentation change reaches `main`, the deployment workflow: -1. Builds the html content -2. Pushes the html content to the [`asf-site`](https://github.com/apache/datafusion/tree/asf-site) branch in this repository. +1. Builds the current complete site and replaces all current root files. +2. Copies the entire pre-existing `asf-site/versions/` archive without using the + picker as a retention registry. +3. Builds `sitemap.xml` as an index over `sitemap-main.xml` and every retained + complete release sitemap. +4. Deliberately installs `.asf.yaml` and `.nojekyll`, then uses `rsync --delete` + while excluding `.git` to remove stale current output. +5. Serializes deployments and pushes a normal, non-force commit. -The Apache Software Foundation provides https://datafusion.apache.org/, -which serves content based on the configuration in -[.asf.yaml](https://github.com/apache/datafusion/blob/main/.asf.yaml), -which specifies the target as https://datafusion.apache.org/. +The Apache Software Foundation serves the branch according to +[`.asf.yaml`](https://github.com/apache/datafusion/blob/main/.asf.yaml). diff --git a/docs/scripts/assemble_site.py b/docs/scripts/assemble_site.py new file mode 100644 index 0000000000000..1723397f7471b --- /dev/null +++ b/docs/scripts/assemble_site.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Replace the current site while retaining immutable full-site snapshots.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import shutil +import sys +import tempfile +import xml.etree.ElementTree as ET + +from versioned_docs import SITE_URL, VERSION_PATTERN, is_link, reject_links + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--current-site", type=Path, required=True) + parser.add_argument("--published-site", type=Path, required=True) + parser.add_argument("--output-site", type=Path, required=True) + return parser.parse_args() + + +def validate_paths(current_site: Path, published_site: Path, output_site: Path) -> None: + if not current_site.is_dir(): + raise RuntimeError(f"current site does not exist: {current_site}") + if not published_site.is_dir(): + raise RuntimeError(f"published site does not exist: {published_site}") + if not (current_site / "index.html").is_file(): + raise RuntimeError(f"current site has no index.html: {current_site}") + if is_link(output_site): + raise RuntimeError(f"output site must not be a link: {output_site}") + for left, right in ( + (current_site, output_site), + (published_site, output_site), + ): + left = left.resolve() + right = right.resolve() + if left == right or left.is_relative_to(right) or right.is_relative_to(left): + raise RuntimeError( + "output site and input sites must not contain one another" + ) + + +def archived_versions(published_site: Path) -> list[str]: + versions_root = published_site / "versions" + if not versions_root.exists(): + return [] + if is_link(versions_root) or not versions_root.is_dir(): + raise RuntimeError(f"published versions must be a directory: {versions_root}") + versions: list[str] = [] + for child in versions_root.iterdir(): + if is_link(child) or not child.is_dir(): + raise RuntimeError(f"invalid published version directory: {child}") + if not VERSION_PATTERN.fullmatch(child.name): + raise RuntimeError(f"unsafe published version directory name: {child.name}") + if not (child / "index.html").is_file(): + raise RuntimeError(f"published version has no index.html: {child}") + if not (child / "sitemap.xml").is_file(): + raise RuntimeError(f"published version has no sitemap.xml: {child}") + reject_links(child) + versions.append(child.name) + return sorted(versions, key=lambda value: tuple(map(int, value.split(".")))) + + +def write_sitemap_index(site: Path, versions: list[str]) -> None: + current_sitemap = site / "sitemap.xml" + if not current_sitemap.is_file(): + raise RuntimeError(f"current site sitemap is missing: {current_sitemap}") + current_sitemap.rename(site / "sitemap-main.xml") + locations = [f"{SITE_URL}/sitemap-main.xml"] + [ + f"{SITE_URL}/versions/{version}/sitemap.xml" for version in versions + ] + root = ET.Element( + "sitemapindex", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" + ) + for location in locations: + sitemap = ET.SubElement(root, "sitemap") + ET.SubElement(sitemap, "loc").text = location + ET.indent(root, space=" ") + ET.ElementTree(root).write(current_sitemap, encoding="utf-8", xml_declaration=True) + + +def assemble(current_site: Path, published_site: Path, output_site: Path) -> None: + validate_paths(current_site, published_site, output_site) + current_site = current_site.resolve() + published_site = published_site.resolve() + output_site = output_site.resolve() + versions = archived_versions(published_site) + output_site.parent.mkdir(parents=True, exist_ok=True) + temporary_root = Path( + tempfile.mkdtemp(prefix=f".{output_site.name}-", dir=output_site.parent) + ) + staged_site = temporary_root / "site" + previous_site = temporary_root / "previous-site" + recovery_failed = False + try: + shutil.copytree(current_site, staged_site) + if versions: + shutil.copytree(published_site / "versions", staged_site / "versions") + write_sitemap_index(staged_site, versions) + if output_site.exists(): + if not output_site.is_dir(): + raise RuntimeError(f"output site is not a directory: {output_site}") + output_site.rename(previous_site) + try: + staged_site.rename(output_site) + except BaseException as error: + if previous_site.exists(): + try: + previous_site.rename(output_site) + except OSError as recovery_error: + recovery_failed = True + raise RuntimeError( + f"could not restore {output_site}; previous output remains at " + f"{previous_site}: {recovery_error}" + ) from error + raise + finally: + if not recovery_failed: + shutil.rmtree(temporary_root, ignore_errors=True) + + +if __name__ == "__main__": + try: + args = parse_args() + assemble(args.current_site, args.published_site, args.output_site) + except (OSError, RuntimeError) as error: + print(f"error: {error}", file=sys.stderr) + raise SystemExit(1) from error diff --git a/docs/scripts/release_conf.py b/docs/scripts/release_conf.py new file mode 100644 index 0000000000000..355a3c3264e2d --- /dev/null +++ b/docs/scripts/release_conf.py @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Minimal Sphinx configuration overlay for a full release site.""" + +from __future__ import annotations + +import os +from pathlib import Path +import sys + +source_dir = Path(os.environ["DATAFUSION_RELEASE_SOURCE"]) +docs_dir = source_dir.parent +sys.path.insert(0, str(docs_dir)) +tagged_conf = source_dir / "conf.py" +exec(compile(tagged_conf.read_bytes(), str(tagged_conf), "exec")) + +version = release = os.environ["DATAFUSION_RELEASE_VERSION"] +base_url = f"https://datafusion.apache.org/versions/{version}/" +html_baseurl = base_url +sitemap_url_scheme = "{link}" +extensions = [*extensions, "sphinx_sitemap"] +templates_path = [str(source_dir / path) for path in templates_path] +html_static_path = [str(source_dir / path) for path in html_static_path] +html_extra_path = [str(source_dir / path) for path in html_extra_path] +html_logo = str(source_dir / html_logo) +html_favicon = str(source_dir / html_favicon) + +html_context = dict(html_context) +html_context.update( + github_user="apache", + github_repo="datafusion", + github_version=os.environ["DATAFUSION_RELEASE_TAG"], +) +html_theme_options = dict(html_theme_options) +html_theme_options.update( + check_switcher=False, + navbar_end=["version-switcher", "theme-switcher"], + switcher={ + "json_url": "https://datafusion.apache.org/_static/versions.json", + "version_match": version, + }, +) + +redirects = dict(redirects) +redirects["library-user-guide/upgrading"] = "upgrading/index.html" diff --git a/docs/scripts/snapshot_site.py b/docs/scripts/snapshot_site.py new file mode 100644 index 0000000000000..6ec43684ae025 --- /dev/null +++ b/docs/scripts/snapshot_site.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Build one complete documentation snapshot from an exact release tag.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile + +from versioned_docs import DOCS_DIR, VERSION_PATTERN, is_link, release_entry + +REPOSITORY = DOCS_DIR.parent +INTERNAL_DOC_LINK = re.compile( + rb"https://datafusion\.apache\.org/" + rb"(?=(?:user-guide|library-user-guide|contributor-guide)/)" +) +GITHUB_MAIN_LINK = re.compile( + rb"https://github\.com/apache/(?:arrow-)?datafusion/(blob|tree)/main" + rb"(?=/|[\"'#<)\s]|$)" +) +DOCS_RS_LATEST_LINK = re.compile( + rb"https://docs\.rs/(datafusion(?:-[a-z0-9-]+)?)/latest(?=/|[\"'#<])" +) +TAGGED_GSOC_GUIDELINES_LINK = re.compile( + rb"https://datafusion\.apache\.org/contributor-guide/" + rb"gsoc_application_guidelines\.html" +) + + +def git(*arguments: str, capture: bool = False) -> str: + result = subprocess.run( + ["git", *arguments], + cwd=REPOSITORY, + check=True, + text=True, + stdout=subprocess.PIPE if capture else None, + ) + return result.stdout.strip() if capture else "" + + +def verify_tag(tag: str, expected_commit: str) -> None: + tag_ref = f"refs/tags/{tag}" + try: + object_type = git("cat-file", "-t", tag_ref, capture=True) + commit = git("rev-parse", f"{tag_ref}^{{commit}}", capture=True) + peeled = git("rev-parse", f"{tag_ref}^{{}}", capture=True) + except subprocess.CalledProcessError as error: + raise RuntimeError( + f"tag {tag} is missing; fetch it with: git fetch origin tag {tag}" + ) from error + if object_type not in {"commit", "tag"}: + raise RuntimeError(f"tag {tag} has unexpected object type {object_type}") + if commit != expected_commit or peeled != expected_commit: + raise RuntimeError(f"tag {tag} peels to {commit}, expected {expected_commit}") + + +def validate_package_root(package_root: Path) -> Path: + package_root = package_root.resolve() + repository = REPOSITORY.resolve() + if package_root.is_relative_to(repository) or repository.is_relative_to( + package_root + ): + raise RuntimeError( + "snapshot output must be outside and must not contain the repository" + ) + return package_root + + +def adapt_generated_output(site: Path, version: str, tag: str) -> None: + """Pin published links without changing tagged sources or ``_sources``.""" + prefix = f"https://datafusion.apache.org/versions/{version}/".encode() + tag_bytes = tag.encode() + paths = [*site.rglob("*.html"), site / "llms.txt"] + for path in paths: + if not path.is_file(): + continue + content = path.read_bytes() + if path.suffix == ".html": + content = TAGGED_GSOC_GUIDELINES_LINK.sub( + prefix + + b"contributor-guide/gsoc/gsoc_application_guidelines_2025.html", + content, + ) + content = INTERNAL_DOC_LINK.sub(prefix, content) + content = GITHUB_MAIN_LINK.sub( + rb"https://github.com/apache/datafusion/\1/" + tag_bytes, + content, + ) + content = DOCS_RS_LATEST_LINK.sub( + lambda match: b"https://docs.rs/" + + match.group(1) + + b"/" + + version.encode(), + content, + ) + path.write_bytes(content) + + +def generate_dependency_graph(docs_dir: Path) -> None: + script = docs_dir / "scripts" / "generate_dependency_graph.sh" + if os.name != "nt": + subprocess.run(["bash", str(script)], cwd=docs_dir, check=True) + return + + for command in ("cargo", "dot"): + if shutil.which(command) is None: + raise RuntimeError( + f"{command} is required to build the tagged dependency graph" + ) + output = docs_dir / "source" / "_static" / "data" / "deps.svg" + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("wb") as destination: + cargo = subprocess.Popen( + [ + "cargo", + "depgraph", + "--workspace-only", + "--all-deps", + "--dedup-transitive-deps", + "--exclude", + "gen,gen-common", + ], + cwd=docs_dir.parent, + stdout=subprocess.PIPE, + ) + assert cargo.stdout is not None + dot = subprocess.run( + [ + "dot", + "-Grankdir=TB", + "-Gconcentrate=true", + "-Goverlap=false", + "-Tsvg", + ], + stdin=cargo.stdout, + stdout=destination, + check=False, + ) + cargo.stdout.close() + cargo_status = cargo.wait() + if cargo_status != 0: + raise subprocess.CalledProcessError(cargo_status, cargo.args) + if dot.returncode != 0: + raise subprocess.CalledProcessError(dot.returncode, dot.args) + + +def publish_snapshot(staged_site: Path, package_root: Path, version: str) -> Path: + versions_root = package_root / "versions" + if is_link(versions_root): + raise RuntimeError(f"versions output must not be a link: {versions_root}") + versions_root.mkdir(parents=True, exist_ok=True) + destination = versions_root / version + if destination.exists() or is_link(destination): + raise RuntimeError(f"refusing to overwrite {destination}") + temporary_root = Path(tempfile.mkdtemp(prefix=f".{version}-", dir=versions_root)) + local_stage = temporary_root / "site" + try: + shutil.copytree(staged_site, local_stage) + if destination.exists() or is_link(destination): + raise RuntimeError(f"refusing to overwrite {destination}") + local_stage.rename(destination) + finally: + shutil.rmtree(temporary_root, ignore_errors=True) + return destination + + +def build_snapshot(version: str, tag: str, package_root: Path) -> Path: + if not VERSION_PATTERN.fullmatch(version) or tag != version: + raise RuntimeError("version and tag must be the same exact X.Y.Z release") + release = release_entry(version, tag) + expected_commit = str(release["commit"]) + verify_tag(tag, expected_commit) + package_root = validate_package_root(package_root) + destination = package_root / "versions" / version + if destination.exists() or is_link(destination): + raise RuntimeError(f"refusing to overwrite {destination}") + + temporary_parent = Path(tempfile.mkdtemp(prefix="datafusion-release-site-")) + worktree = temporary_parent / "worktree" + built_site = temporary_parent / "html" + worktree_added = False + try: + git("worktree", "add", "--detach", str(worktree), expected_commit) + worktree_added = True + actual_commit = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=worktree, + check=True, + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + if actual_commit != expected_commit: + raise RuntimeError( + f"temporary worktree is {actual_commit}, expected {expected_commit}" + ) + + docs_dir = worktree / "docs" + generate_dependency_graph(docs_dir) + environment = os.environ.copy() + environment.update( + DATAFUSION_RELEASE_SOURCE=str(docs_dir / "source"), + DATAFUSION_RELEASE_VERSION=version, + DATAFUSION_RELEASE_TAG=tag, + ) + config_dir = temporary_parent / "config" + config_dir.mkdir() + shutil.copy2(DOCS_DIR / "scripts" / "release_conf.py", config_dir / "conf.py") + subprocess.run( + [ + "uv", + "run", + "--project", + str(worktree), + "--package", + "datafusion-docs", + "--with", + "sphinx-sitemap==2.9.0", + "python", + "-m", + "sphinx", + "-W", + "-b", + "html", + "-c", + str(config_dir), + str(docs_dir / "source"), + str(built_site), + ], + cwd=docs_dir, + env=environment, + check=True, + ) + adapt_generated_output(built_site, version, tag) + return publish_snapshot(built_site, package_root, version) + finally: + if worktree_added: + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree)], + cwd=REPOSITORY, + check=False, + ) + shutil.rmtree(temporary_parent, ignore_errors=True) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("version") + parser.add_argument("tag") + parser.add_argument("package_root", type=Path) + args = parser.parse_args() + destination = build_snapshot(args.version, args.tag, args.package_root) + print(f"snapshot created: {destination}") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError) as error: + print(f"error: {error}", file=sys.stderr) + raise SystemExit(1) from error diff --git a/docs/scripts/validate_site.py b/docs/scripts/validate_site.py new file mode 100644 index 0000000000000..7756b30d169ff --- /dev/null +++ b/docs/scripts/validate_site.py @@ -0,0 +1,420 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Validate current and immutable full-site documentation output.""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from functools import partial +from html.parser import HTMLParser +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import re +import sys +import threading +from typing import Iterator +import xml.etree.ElementTree as ET +from urllib.error import HTTPError, URLError +from urllib.parse import unquote, urljoin, urlsplit +from urllib.request import Request, urlopen + +from versioned_docs import SITE_URL, load_versions + +PICKER_URL = re.compile( + r"DOCUMENTATION_OPTIONS\.theme_switcher_json_url\s*=\s*['\"]([^'\"]+)" +) +PICKER_VERSION = re.compile( + r"DOCUMENTATION_OPTIONS\.theme_switcher_version_match\s*=\s*['\"]([^'\"]+)" +) +MUTABLE_RELEASE_LINK = re.compile( + r"https://(?:" + r"github\.com/apache/(?:arrow-)?datafusion/(?:blob|tree)/main(?:/|[\"'#<)\s]|$)|" + r"docs\.rs/datafusion(?:-[a-z0-9-]+)?/latest(?:/|[\"'#<])" + r")" +) +CSS_URL = re.compile(r"url\(\s*(['\"]?)(.*?)\1\s*\)") +META_REFRESH_URL = re.compile(r"(?:^|;)\s*url\s*=\s*(['\"]?)(.*?)\1\s*$", re.I) + + +class ValidationError(RuntimeError): + pass + + +class HTMLReferences(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.references: list[str] = [] + self.canonicals: list[str] = [] + self.ids: set[str] = set() + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + values = {name.lower(): value for name, value in attrs if value is not None} + for attribute in ("href", "src", "action", "poster", "data"): + if attribute in values: + self.references.append(values[attribute]) + for value in values.get("srcset", "").split(","): + if value.strip(): + self.references.append(value.strip().split()[0]) + if tag.lower() == "meta" and values.get("http-equiv", "").lower() == "refresh": + refresh = META_REFRESH_URL.search(values.get("content", "")) + if refresh: + self.references.append(refresh.group(2)) + for identifier in (values.get("id"), values.get("name")): + if identifier: + self.ids.add(identifier) + if tag.lower() == "link" and "canonical" in values.get("rel", "").split(): + self.canonicals.append(values["href"]) + + +def read_html(path: Path) -> tuple[str, HTMLReferences]: + text = path.read_text(encoding="utf-8") + parsed = HTMLReferences() + parsed.feed(text) + return text, parsed + + +def site_target(site_root: Path, page: Path, url: str) -> Path | None: + parsed = urlsplit(url) + if parsed.scheme not in ("", "http", "https"): + return None + if parsed.netloc and parsed.netloc != urlsplit(SITE_URL).netloc: + return None + if parsed.netloc and parsed.path.startswith( + ("/blog/", "/python/", "/java/", "/comet/", "/ballista/") + ): + return None + if not parsed.path: + return page + target = ( + site_root / unquote(parsed.path.lstrip("/")) + if parsed.netloc or parsed.path.startswith("/") + else page.parent / unquote(parsed.path) + ).resolve() + try: + target.relative_to(site_root.resolve()) + except ValueError as error: + raise ValidationError( + f"local target escapes site root: {page}: {url}" + ) from error + if parsed.path.endswith("/"): + target /= "index.html" + return target + + +def switch_candidate(page: str, source_version: str, target_url: str) -> str: + path = urlsplit(page).path.lstrip("/") + prefix = "" if source_version == "main" else f"versions/{source_version}/" + if not path.startswith(prefix): + raise ValidationError(f"page is outside its version prefix: {page}") + relative = path[len(prefix) :] + candidate = urljoin(target_url, relative) + if f"/versions/{source_version}/versions/" in candidate: + raise ValidationError(f"version path is duplicated: {candidate}") + return candidate + + +def validate_page( + site_root: Path, relative: str, version: str, errors: list[str] +) -> None: + path = site_root / relative + if not path.is_file(): + errors.append(f"representative page is missing: {path}") + return + text, parsed = read_html(path) + canonical = SITE_URL + "/" + relative.replace("\\", "/") + if parsed.canonicals != [canonical]: + errors.append(f"incorrect canonical in {path}: expected {canonical}") + picker_url = PICKER_URL.search(text) + if picker_url is None or picker_url.group(1) != f"{SITE_URL}/_static/versions.json": + errors.append(f"incorrect picker manifest URL in {path}") + picker_version = PICKER_VERSION.search(text) + if picker_version is None or picker_version.group(1) != version: + errors.append(f"incorrect picker version in {path}") + + +def validate_local_links(site_root: Path, root: Path, errors: list[str]) -> None: + parsed_pages: dict[Path, HTMLReferences] = {} + for page in root.rglob("*.html"): + if "_static" in page.relative_to(root).parts: + continue + _, parsed = read_html(page) + parsed_pages[page.resolve()] = parsed + expected_canonical = f"{SITE_URL}/{page.relative_to(site_root).as_posix()}" + if parsed.canonicals and parsed.canonicals != [expected_canonical]: + errors.append( + f"incorrect canonical in {page}: expected {expected_canonical}" + ) + for url in parsed.references: + parsed_url = urlsplit(url) + if parsed_url.netloc == urlsplit(SITE_URL).netloc and root == site_root: + # Current-source absolute links are checked by the repository's + # Markdown link checker, not by snapshot hosting validation. + continue + try: + target = site_target(site_root, page, url) + except ValidationError as error: + errors.append(str(error)) + continue + if target is not None and not target.exists(): + errors.append(f"missing local target in {page}: {url}") + continue + fragment = urlsplit(url).fragment + if target is not None and fragment and target.suffix == ".html": + target_parsed = parsed_pages.get(target.resolve()) + if target_parsed is None: + _, target_parsed = read_html(target) + parsed_pages[target.resolve()] = target_parsed + if unquote(fragment) not in target_parsed.ids: + errors.append(f"missing local fragment in {page}: {url}") + for stylesheet in root.rglob("*.css"): + text = stylesheet.read_text(encoding="utf-8") + for _, url in CSS_URL.findall(text): + target = site_target(site_root, stylesheet, url) + if target is not None and not target.exists(): + errors.append(f"missing CSS target in {stylesheet}: {url}") + + +def validate_sitemaps(site_root: Path, versions: list[str], errors: list[str]) -> None: + namespace = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} + if versions: + try: + tree = ET.parse(site_root / "sitemap.xml") + except (OSError, ET.ParseError) as error: + errors.append(f"invalid sitemap index: {error}") + return + locations = [item.text for item in tree.findall("sm:sitemap/sm:loc", namespace)] + expected = [f"{SITE_URL}/sitemap-main.xml"] + [ + f"{SITE_URL}/versions/{version}/sitemap.xml" for version in versions + ] + if locations != expected: + errors.append("sitemap index has incorrect entries") + for version in versions: + sitemap = site_root / "versions" / version / "sitemap.xml" + try: + tree = ET.parse(sitemap) + except (OSError, ET.ParseError) as error: + errors.append(f"invalid release sitemap {sitemap}: {error}") + continue + prefix = f"{SITE_URL}/versions/{version}/" + for location in tree.findall("sm:url/sm:loc", namespace): + if location.text is None or not location.text.startswith(prefix): + errors.append(f"release sitemap URL escapes {prefix}: {location.text}") + + +class QuietHandler(SimpleHTTPRequestHandler): + def log_message(self, format: str, *args: object) -> None: + pass + + +@contextmanager +def local_server(site_root: Path) -> Iterator[str]: + server = ThreadingHTTPServer( + ("127.0.0.1", 0), partial(QuietHandler, directory=str(site_root)) + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + server.server_close() + thread.join() + + +def validate_http(site_root: Path, paths: list[str], errors: list[str]) -> None: + with local_server(site_root) as server: + for relative in paths: + for method in ("GET", "HEAD"): + try: + with urlopen( + Request(f"{server}/{relative}", method=method), timeout=10 + ) as response: + if response.status != 200: + errors.append( + f"HTTP {method} {relative}: {response.status}" + ) + except (HTTPError, URLError, TimeoutError) as error: + errors.append(f"HTTP {method} {relative} failed: {error}") + + +def validate_site( + site_root: Path, *, require_snapshots: bool = False, http_checks: bool = True +) -> None: + site_root = site_root.resolve() + if not site_root.is_dir(): + raise ValidationError(f"site root does not exist: {site_root}") + entries = load_versions() + errors: list[str] = [] + manifest = site_root / "_static" / "versions.json" + if ( + not manifest.is_file() + or json.loads(manifest.read_text(encoding="utf-8")) != entries + ): + errors.append(f"site has a missing or stale versions manifest: {manifest}") + + releases = [str(entry["version"]) for entry in entries[1:]] + versions_root = site_root / "versions" + present = [] + if versions_root.is_dir(): + present = sorted( + ( + child.name + for child in versions_root.iterdir() + if child.is_dir() + and re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", child.name) + ), + key=lambda value: tuple(map(int, value.split("."))), + ) + if require_snapshots and not set(releases).issubset(present): + errors.append("required release snapshots are missing") + + representatives = ( + "index.html", + "user-guide/introduction.html", + "library-user-guide/index.html", + "contributor-guide/index.html", + "download.html", + "search.html", + "_static/theme_overrides.css", + "_sources/index.rst.txt", + "llms.txt", + ) + redirects = ( + "library-user-guide/adding-udfs.html", + "library-user-guide/upgrading.html", + "user-guide/runtime_configs.html", + ) + http_paths = ["_static/versions.json"] + for version in ["main", *present]: + prefix = "" if version == "main" else f"versions/{version}/" + root = site_root if version == "main" else site_root / "versions" / version + if not (root / "searchindex.js").is_file(): + errors.append(f"search index is missing: {root}") + elif (root / "searchindex.js").stat().st_size == 0: + errors.append(f"search index is empty: {root}") + if not (root / "objects.inv").is_file(): + errors.append(f"Sphinx inventory is missing: {root}") + elif (root / "objects.inv").stat().st_size == 0: + errors.append(f"Sphinx inventory is empty: {root}") + for representative in representatives: + relative = prefix + representative + if representative.endswith((".html",)): + validate_page(site_root, relative, version, errors) + elif not (site_root / relative).is_file(): + errors.append(f"representative file is missing: {relative}") + elif (site_root / relative).stat().st_size == 0: + errors.append(f"representative file is empty: {relative}") + http_paths.append(relative) + for redirect in redirects: + relative = prefix + redirect + if not (site_root / relative).is_file(): + errors.append(f"redirect is missing: {relative}") + http_paths.append(relative) + validate_local_links(site_root, root, errors) + + page = f"{SITE_URL}/{prefix}user-guide/sql/select.html" + for entry in entries: + candidate = switch_candidate(page, version, str(entry["url"])) + if "/versions/55.0.0/versions/" in candidate: + errors.append(f"picker duplicates a versions path: {candidate}") + + if version != "main": + release_files = [*root.rglob("*.html"), root / "llms.txt"] + for page_path in release_files: + if page_path.suffix == ".html": + text, parsed = read_html(page_path) + else: + text = page_path.read_text(encoding="utf-8") + parsed = HTMLReferences() + mutable = MUTABLE_RELEASE_LINK.search(text) + if mutable: + errors.append( + f"mutable release link in {page_path}: {mutable.group(0)}" + ) + for url in parsed.references: + parsed_url = urlsplit(url) + if ( + parsed_url.netloc == urlsplit(SITE_URL).netloc + and parsed_url.path.startswith( + ( + "/user-guide/", + "/library-user-guide/", + "/contributor-guide/", + ) + ) + and (site_root / parsed_url.path.lstrip("/")).exists() + ): + errors.append( + f"release documentation link escapes in {page_path}: {url}" + ) + + current_only = site_root / "library-user-guide" / "upgrading" / "56.0.0.html" + release_current_only = ( + site_root + / "versions" + / "55.0.0" + / "library-user-guide" + / "upgrading" + / "56.0.0.html" + ) + if not current_only.is_file(): + errors.append(f"current-only page is missing: {current_only}") + if release_current_only.exists(): + errors.append(f"release contains current-only page: {release_current_only}") + if "55.0.0" in present: + tagged_source = ( + site_root + / "versions" + / "55.0.0" + / "_sources" + / "library-user-guide" + / "upgrading" + / "55.0.0.md.txt" + ) + if ( + not tagged_source.is_file() + or "DataFusion `55.0.0` has not been released yet." + not in tagged_source.read_text(encoding="utf-8") + ): + errors.append("exact tagged 55.0.0 release-note prose was not preserved") + + validate_sitemaps(site_root, present, errors) + if http_checks and not errors: + validate_http(site_root, http_paths, errors) + if errors: + raise ValidationError("site validation failed:\n- " + "\n- ".join(errors)) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--site-root", type=Path, required=True) + parser.add_argument("--require-snapshots", action="store_true") + args = parser.parse_args() + validate_site(args.site_root, require_snapshots=args.require_snapshots) + print("site validation passed") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, ValidationError) as error: + print(f"error: {error}", file=sys.stderr) + raise SystemExit(1) from error diff --git a/docs/scripts/versioned_docs.py b/docs/scripts/versioned_docs.py new file mode 100644 index 0000000000000..d52b6e2be564c --- /dev/null +++ b/docs/scripts/versioned_docs.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared catalog handling for full-site documentation snapshots.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import re +import stat + +DOCS_DIR = Path(__file__).resolve().parent.parent +MANIFEST = DOCS_DIR / "source" / "_static" / "versions.json" +SITE_URL = "https://datafusion.apache.org" +VERSION_PATTERN = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+") +EXPECTED_55_COMMIT = "d5552342012888b7d1a3ab88d92e3d292fc0cde0" + + +def is_link(path: Path) -> bool: + """Return whether a path is a symlink or Windows reparse point.""" + try: + status = path.lstat() + except FileNotFoundError: + return False + reparse_point = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + file_attributes = getattr(status, "st_file_attributes", 0) + return stat.S_ISLNK(status.st_mode) or bool(file_attributes & reparse_point) + + +def reject_links(root: Path) -> None: + """Reject links before copying an immutable archive tree.""" + if is_link(root): + raise RuntimeError(f"archived path must not be a link: {root}") + for directory, directories, files in os.walk(root, followlinks=False): + for name in [*directories, *files]: + path = Path(directory) / name + if is_link(path): + raise RuntimeError(f"archived path must not be a link: {path}") + + +def load_versions(path: Path = MANIFEST) -> list[dict[str, object]]: + try: + entries = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"cannot read versions manifest {path}: {error}") from error + if not isinstance(entries, list) or not entries: + raise ValueError("versions manifest must contain a non-empty list") + + versions: set[str] = set() + urls: set[str] = set() + preferred = 0 + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise ValueError("each versions manifest entry must be an object") + version = entry.get("version") + url = entry.get("url") + if not isinstance(version, str) or version in versions: + raise ValueError(f"invalid or duplicate version: {version!r}") + if not isinstance(url, str) or url in urls: + raise ValueError(f"invalid or duplicate version URL: {url!r}") + versions.add(version) + urls.add(url) + preferred += entry.get("preferred") is True + + if index == 0: + if entry != { + "name": "Development", + "url": f"{SITE_URL}/", + "version": "main", + }: + raise ValueError( + "the first entry must be Development/main at site root" + ) + continue + if not VERSION_PATTERN.fullmatch(version): + raise ValueError(f"invalid semantic release version: {version!r}") + expected_url = f"{SITE_URL}/versions/{version}/" + if entry.get("name") != version or url != expected_url: + raise ValueError(f"release {version} has an invalid name or URL") + if entry.get("tag") != version or not re.fullmatch( + r"[0-9a-f]{40}", str(entry.get("commit", "")) + ): + raise ValueError(f"release {version} must have an exact tag and commit") + if preferred > 1: + raise ValueError("at most one versions manifest entry may be preferred") + + release_55 = next( + (entry for entry in entries if entry["version"] == "55.0.0"), None + ) + if release_55 is None or release_55.get("commit") != EXPECTED_55_COMMIT: + raise ValueError("55.0.0 must resolve to its expected exact commit") + return entries + + +def release_entry(version: str, tag: str) -> dict[str, object]: + for entry in load_versions(): + if entry["version"] == version and entry.get("tag") == tag: + return entry + raise RuntimeError("version and tag must exactly match versions.json") diff --git a/docs/source/_static/versions.json b/docs/source/_static/versions.json new file mode 100644 index 0000000000000..d78cddec398c1 --- /dev/null +++ b/docs/source/_static/versions.json @@ -0,0 +1,14 @@ +[ + { + "name": "Development", + "url": "https://datafusion.apache.org/", + "version": "main" + }, + { + "commit": "d5552342012888b7d1a3ab88d92e3d292fc0cde0", + "name": "55.0.0", + "tag": "55.0.0", + "url": "https://datafusion.apache.org/versions/55.0.0/", + "version": "55.0.0" + } +] diff --git a/docs/source/conf.py b/docs/source/conf.py index 68ba628bba1f2..a4f7460dfd0a4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -37,6 +37,7 @@ project = "Apache DataFusion" copyright = "2019-2025, Apache Software Foundation" author = "Apache Software Foundation" +version = release = "main" # -- General configuration --------------------------------------------------- @@ -95,12 +96,18 @@ }, "use_edit_page_button": True, "navbar_center": [], - "navbar_end": ["theme-switcher"], + "navbar_end": ["version-switcher", "theme-switcher"], + # The persistent production URL is intentionally not fetched at build time. + "check_switcher": False, + "switcher": { + "json_url": "https://datafusion.apache.org/_static/versions.json", + "version_match": version, + }, } html_context = { "github_user": "apache", - "github_repo": "arrow-datafusion", + "github_repo": "datafusion", "github_version": "main", "doc_path": "docs/source", } diff --git a/docs/tests/test_versioned_docs.py b/docs/tests/test_versioned_docs.py new file mode 100644 index 0000000000000..87abe8137e64e --- /dev/null +++ b/docs/tests/test_versioned_docs.py @@ -0,0 +1,330 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +DOCS_DIR = Path(__file__).resolve().parents[1] +SCRIPTS_DIR = DOCS_DIR / "scripts" +sys.path.insert(0, str(SCRIPTS_DIR)) + + +def load_script(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPTS_DIR / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +versioned_docs = load_script("versioned_docs") +snapshot_site = load_script("snapshot_site") +assemble_site = load_script("assemble_site") +validate_site = load_script("validate_site") + + +def write(path: Path, content: str = "file") -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +class VersionedDocsTest(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + + def tearDown(self) -> None: + self.temporary.cleanup() + + def test_manifest_has_one_site_wide_exact_release(self) -> None: + entries = versioned_docs.load_versions() + self.assertEqual( + entries, + [ + { + "name": "Development", + "url": f"{versioned_docs.SITE_URL}/", + "version": "main", + }, + { + "commit": versioned_docs.EXPECTED_55_COMMIT, + "name": "55.0.0", + "tag": "55.0.0", + "url": f"{versioned_docs.SITE_URL}/versions/55.0.0/", + "version": "55.0.0", + }, + ], + ) + + def test_manifest_is_appendable_and_rejects_duplicates(self) -> None: + manifest = self.root / "versions.json" + entries = versioned_docs.load_versions() + entries.append( + { + "commit": "a" * 40, + "name": "56.0.0", + "tag": "56.0.0", + "url": f"{versioned_docs.SITE_URL}/versions/56.0.0/", + "version": "56.0.0", + } + ) + write(manifest, json.dumps(entries)) + self.assertEqual(len(versioned_docs.load_versions(manifest)), 3) + entries[-1]["version"] = "55.0.0" + write(manifest, json.dumps(entries)) + with self.assertRaisesRegex(ValueError, "duplicate version"): + versioned_docs.load_versions(manifest) + + def test_exact_lightweight_tag_and_peeled_commit(self) -> None: + snapshot_site.verify_tag("55.0.0", versioned_docs.EXPECTED_55_COMMIT) + self.assertEqual( + snapshot_site.git("cat-file", "-t", "refs/tags/55.0.0", capture=True), + "commit", + ) + + def test_exact_tag_keeps_unreleased_55_prose(self) -> None: + prose = snapshot_site.git( + "show", + "55.0.0:docs/source/library-user-guide/upgrading/55.0.0.md", + capture=True, + ) + self.assertIn("DataFusion `55.0.0` has not been released yet.", prose) + + def test_missing_tag_error_is_actionable(self) -> None: + repository = self.root / "repository" + subprocess.run( + ["git", "init", str(repository)], check=True, capture_output=True + ) + with mock.patch.object(snapshot_site, "REPOSITORY", repository): + with self.assertRaisesRegex(RuntimeError, "git fetch origin tag missing"): + snapshot_site.verify_tag("missing", "0" * 40) + + def test_generated_html_adaptations_leave_sources_unchanged(self) -> None: + site = self.root / "site" + original = ( + "https://datafusion.apache.org/contributor-guide/testing.html\n" + "https://datafusion.apache.org/contributor-guide/gsoc_application_guidelines.html\n" + "https://github.com/apache/datafusion/blob/main/docs/source/index.rst\n" + "https://docs.rs/datafusion/latest/datafusion/\n" + ) + write(site / "index.html", original) + write(site / "_sources" / "index.rst.txt", original) + write( + site / "llms.txt", + original + "https://github.com/apache/datafusion/tree/main\n", + ) + snapshot_site.adapt_generated_output(site, "55.0.0", "55.0.0") + output = (site / "index.html").read_text(encoding="utf-8") + self.assertIn("/versions/55.0.0/contributor-guide/", output) + self.assertIn( + "/versions/55.0.0/contributor-guide/gsoc/gsoc_application_guidelines_2025.html", + output, + ) + self.assertIn("/blob/55.0.0/docs/source/", output) + self.assertIn("docs.rs/datafusion/55.0.0/datafusion/", output) + self.assertEqual( + (site / "_sources" / "index.rst.txt").read_text(encoding="utf-8"), + original, + ) + llms = (site / "llms.txt").read_text(encoding="utf-8") + self.assertIn("/versions/55.0.0/contributor-guide/", llms) + self.assertIn("/tree/55.0.0", llms) + self.assertNotIn("/tree/main", llms) + + def test_publication_is_one_directory_and_refuses_overwrite(self) -> None: + staged = self.root / "staged" + package = self.root / "package with spaces" + write(staged / "index.html") + destination = snapshot_site.publish_snapshot(staged, package, "55.0.0") + self.assertEqual(destination, package / "versions" / "55.0.0") + self.assertTrue((destination / "index.html").is_file()) + write(staged / "index.html") + with self.assertRaisesRegex(RuntimeError, "refusing to overwrite"): + snapshot_site.publish_snapshot(staged, package, "55.0.0") + + def test_publication_failure_leaves_no_release_directory(self) -> None: + staged = self.root / "staged" + package = self.root / "package" + write(staged / "index.html") + with mock.patch.object( + snapshot_site.shutil, + "copytree", + side_effect=OSError("simulated copy failure"), + ): + with self.assertRaisesRegex(OSError, "simulated copy failure"): + snapshot_site.publish_snapshot(staged, package, "55.0.0") + self.assertFalse((package / "versions" / "55.0.0").exists()) + self.assertEqual(list((package / "versions").iterdir()), []) + + def test_snapshot_rejects_unsafe_output_and_version(self) -> None: + with self.assertRaisesRegex(RuntimeError, "must not contain the repository"): + snapshot_site.validate_package_root(snapshot_site.REPOSITORY.parent) + with self.assertRaisesRegex(RuntimeError, "must be outside"): + snapshot_site.validate_package_root(snapshot_site.REPOSITORY / "docs/build") + with self.assertRaisesRegex(RuntimeError, "exact X.Y.Z"): + snapshot_site.build_snapshot("../55", "../55", self.root / "output") + + def test_assembly_replaces_current_and_retains_complete_archive(self) -> None: + current = self.root / "current" + published = self.root / "published" + output = self.root / "output" + write(current / "index.html", "new") + write(current / "sitemap.xml", "") + write(published / "obsolete.html", "old current") + for version in ("54.0.0", "55.0.0"): + write(published / "versions" / version / "index.html", version) + write(published / "versions" / version / "sitemap.xml", "") + assemble_site.assemble(current, published, output) + self.assertEqual((output / "index.html").read_text(encoding="utf-8"), "new") + self.assertFalse((output / "obsolete.html").exists()) + self.assertTrue((output / "versions" / "54.0.0" / "index.html").is_file()) + self.assertTrue((output / "versions" / "55.0.0" / "index.html").is_file()) + sitemap = (output / "sitemap.xml").read_text(encoding="utf-8") + self.assertIn("sitemap-main.xml", sitemap) + self.assertIn("versions/54.0.0/sitemap.xml", sitemap) + errors: list[str] = [] + validate_site.validate_sitemaps(output, ["54.0.0", "55.0.0"], errors) + self.assertEqual(errors, []) + + def test_removed_picker_entry_does_not_delete_archive(self) -> None: + current = self.root / "current" + published = self.root / "published" + output = self.root / "output" + write(current / "index.html") + write(current / "sitemap.xml", "") + write(published / "versions" / "54.0.0" / "index.html") + write(published / "versions" / "54.0.0" / "sitemap.xml") + assemble_site.assemble(current, published, output) + self.assertTrue((output / "versions" / "54.0.0" / "index.html").is_file()) + + def test_assembly_rejects_unsafe_archive_name_and_path_overlap(self) -> None: + published = self.root / "published" + write(published / "versions" / "latest" / "index.html") + write(published / "versions" / "latest" / "sitemap.xml") + with self.assertRaisesRegex(RuntimeError, "unsafe published version"): + assemble_site.archived_versions(published) + current = self.root / "container" / "current" + write(current / "index.html") + with self.assertRaisesRegex(RuntimeError, "must not contain one another"): + assemble_site.validate_paths(current, published, self.root / "container") + + def test_assembly_rejects_links_in_archive_and_output(self) -> None: + current = self.root / "current" + published = self.root / "published" + external = self.root / "external" + output_link = self.root / "output-link" + write(current / "index.html") + write(published / "versions" / "55.0.0" / "index.html") + write(published / "versions" / "55.0.0" / "sitemap.xml") + write(external / "secret.txt") + archive_link = published / "versions" / "55.0.0" / "external" + try: + archive_link.symlink_to(external, target_is_directory=True) + output_link.symlink_to(external, target_is_directory=True) + except OSError as error: + self.skipTest(f"directory symlinks are unavailable: {error}") + with self.assertRaisesRegex(RuntimeError, "archived path must not be a link"): + assemble_site.archived_versions(published) + archive_link.unlink() + with self.assertRaisesRegex(RuntimeError, "output site must not be a link"): + assemble_site.assemble(current, published, output_link) + self.assertTrue((external / "secret.txt").is_file()) + + def test_assembly_restores_previous_output_on_final_rename_failure(self) -> None: + current = self.root / "current" + published = self.root / "published" + output = self.root / "output" + write(current / "index.html") + write(current / "sitemap.xml", "") + published.mkdir() + write(output / "sentinel.txt", "previous") + real_rename = Path.rename + + def fail_staged(source: Path, destination: Path) -> Path: + if source.name == "site" and destination == output: + raise OSError("simulated failure") + return real_rename(source, destination) + + with mock.patch.object(Path, "rename", fail_staged): + with self.assertRaisesRegex(OSError, "simulated failure"): + assemble_site.assemble(current, published, output) + self.assertEqual( + (output / "sentinel.txt").read_text(encoding="utf-8"), "previous" + ) + + def test_picker_equivalent_paths_and_fallback(self) -> None: + release = f"{versioned_docs.SITE_URL}/versions/55.0.0/" + candidate = validate_site.switch_candidate( + f"{versioned_docs.SITE_URL}/user-guide/sql/select.html", "main", release + ) + self.assertEqual(candidate, release + "user-guide/sql/select.html") + reverse = validate_site.switch_candidate( + candidate, "55.0.0", f"{versioned_docs.SITE_URL}/" + ) + self.assertEqual( + reverse, f"{versioned_docs.SITE_URL}/user-guide/sql/select.html" + ) + current_only = validate_site.switch_candidate( + f"{versioned_docs.SITE_URL}/library-user-guide/upgrading/56.0.0.html", + "main", + release, + ) + self.assertEqual( + current_only, release + "library-user-guide/upgrading/56.0.0.html" + ) + self.assertNotIn("/versions/55.0.0/versions/", current_only) + + @unittest.skipUnless(shutil.which("rsync"), "rsync is not installed") + def test_rsync_keeps_git_and_deletes_stale_current_files(self) -> None: + source = self.root / "source" + destination = self.root / "destination" + write(source / "index.html", "new") + write(source / "versions" / "55.0.0" / "index.html") + write(source / ".asf.yaml") + write(source / ".nojekyll") + write(destination / "stale.html") + write(destination / ".git" / "HEAD", "git") + subprocess.run( + [ + "rsync", + "-a", + "--delete", + "--exclude", + "/.git/", + f"{source}/", + f"{destination}/", + ], + check=True, + ) + self.assertFalse((destination / "stale.html").exists()) + self.assertTrue((destination / "versions" / "55.0.0" / "index.html").is_file()) + self.assertEqual( + (destination / ".git" / "HEAD").read_text(encoding="utf-8"), "git" + ) + + +if __name__ == "__main__": + unittest.main() From d7ac573854c1987944d1fc51138c4402913412e7 Mon Sep 17 00:00:00 2001 From: yiaany Date: Sun, 30 Aug 2026 02:11:34 +0500 Subject: [PATCH 2/2] docs: harden versioned docs publication --- docs/scripts/assemble_site.py | 27 ++++++------------------ docs/scripts/snapshot_site.py | 4 ++++ docs/tests/test_versioned_docs.py | 35 +++++++++++++++++++++++-------- 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/docs/scripts/assemble_site.py b/docs/scripts/assemble_site.py index 1723397f7471b..0536e49607d26 100644 --- a/docs/scripts/assemble_site.py +++ b/docs/scripts/assemble_site.py @@ -57,6 +57,8 @@ def validate_paths(current_site: Path, published_site: Path, output_site: Path) raise RuntimeError( "output site and input sites must not contain one another" ) + if output_site.exists(): + raise RuntimeError(f"output site already exists: {output_site}") def archived_versions(published_site: Path) -> list[str]: @@ -109,33 +111,16 @@ def assemble(current_site: Path, published_site: Path, output_site: Path) -> Non tempfile.mkdtemp(prefix=f".{output_site.name}-", dir=output_site.parent) ) staged_site = temporary_root / "site" - previous_site = temporary_root / "previous-site" - recovery_failed = False try: shutil.copytree(current_site, staged_site) if versions: shutil.copytree(published_site / "versions", staged_site / "versions") write_sitemap_index(staged_site, versions) - if output_site.exists(): - if not output_site.is_dir(): - raise RuntimeError(f"output site is not a directory: {output_site}") - output_site.rename(previous_site) - try: - staged_site.rename(output_site) - except BaseException as error: - if previous_site.exists(): - try: - previous_site.rename(output_site) - except OSError as recovery_error: - recovery_failed = True - raise RuntimeError( - f"could not restore {output_site}; previous output remains at " - f"{previous_site}: {recovery_error}" - ) from error - raise + if output_site.exists() or is_link(output_site): + raise RuntimeError(f"output site already exists: {output_site}") + staged_site.rename(output_site) finally: - if not recovery_failed: - shutil.rmtree(temporary_root, ignore_errors=True) + shutil.rmtree(temporary_root, ignore_errors=True) if __name__ == "__main__": diff --git a/docs/scripts/snapshot_site.py b/docs/scripts/snapshot_site.py index 6ec43684ae025..c9673cc8e87b3 100644 --- a/docs/scripts/snapshot_site.py +++ b/docs/scripts/snapshot_site.py @@ -90,6 +90,8 @@ def validate_package_root(package_root: Path) -> Path: def adapt_generated_output(site: Path, version: str, tag: str) -> None: """Pin published links without changing tagged sources or ``_sources``.""" + shutil.rmtree(site / ".doctrees", ignore_errors=True) + (site / ".buildinfo").unlink(missing_ok=True) prefix = f"https://datafusion.apache.org/versions/{version}/".encode() tag_bytes = tag.encode() paths = [*site.rglob("*.html"), site / "llms.txt"] @@ -243,6 +245,8 @@ def build_snapshot(version: str, tag: str, package_root: Path) -> Path: "-W", "-b", "html", + "-d", + str(temporary_parent / "doctrees"), "-c", str(config_dir), str(docs_dir / "source"), diff --git a/docs/tests/test_versioned_docs.py b/docs/tests/test_versioned_docs.py index 87abe8137e64e..f4193bd1427d2 100644 --- a/docs/tests/test_versioned_docs.py +++ b/docs/tests/test_versioned_docs.py @@ -132,6 +132,8 @@ def test_generated_html_adaptations_leave_sources_unchanged(self) -> None: ) write(site / "index.html", original) write(site / "_sources" / "index.rst.txt", original) + write(site / ".buildinfo", "environment-specific metadata") + write(site / ".doctrees" / "environment.pickle", "temporary paths") write( site / "llms.txt", original + "https://github.com/apache/datafusion/tree/main\n", @@ -153,6 +155,8 @@ def test_generated_html_adaptations_leave_sources_unchanged(self) -> None: self.assertIn("/versions/55.0.0/contributor-guide/", llms) self.assertIn("/tree/55.0.0", llms) self.assertNotIn("/tree/main", llms) + self.assertFalse((site / ".buildinfo").exists()) + self.assertFalse((site / ".doctrees").exists()) def test_publication_is_one_directory_and_refuses_overwrite(self) -> None: staged = self.root / "staged" @@ -253,7 +257,7 @@ def test_assembly_rejects_links_in_archive_and_output(self) -> None: assemble_site.assemble(current, published, output_link) self.assertTrue((external / "secret.txt").is_file()) - def test_assembly_restores_previous_output_on_final_rename_failure(self) -> None: + def test_assembly_refuses_to_replace_existing_output(self) -> None: current = self.root / "current" published = self.root / "published" output = self.root / "output" @@ -261,18 +265,31 @@ def test_assembly_restores_previous_output_on_final_rename_failure(self) -> None write(current / "sitemap.xml", "") published.mkdir() write(output / "sentinel.txt", "previous") - real_rename = Path.rename + with self.assertRaisesRegex(RuntimeError, "output site already exists"): + assemble_site.assemble(current, published, output) + self.assertEqual( + (output / "sentinel.txt").read_text(encoding="utf-8"), "previous" + ) - def fail_staged(source: Path, destination: Path) -> Path: - if source.name == "site" and destination == output: - raise OSError("simulated failure") - return real_rename(source, destination) + def test_assembly_refuses_output_created_while_staging(self) -> None: + current = self.root / "current" + published = self.root / "published" + output = self.root / "output" + write(current / "index.html") + write(current / "sitemap.xml", "") + published.mkdir() + real_copytree = shutil.copytree - with mock.patch.object(Path, "rename", fail_staged): - with self.assertRaisesRegex(OSError, "simulated failure"): + def create_output(source: Path, destination: Path) -> Path: + copied = real_copytree(source, destination) + write(output / "sentinel.txt", "concurrent") + return copied + + with mock.patch.object(assemble_site.shutil, "copytree", create_output): + with self.assertRaisesRegex(RuntimeError, "output site already exists"): assemble_site.assemble(current, published, output) self.assertEqual( - (output / "sentinel.txt").read_text(encoding="utf-8"), "previous" + (output / "sentinel.txt").read_text(encoding="utf-8"), "concurrent" ) def test_picker_equivalent_paths_and_fallback(self) -> None: