docs: add full-site version snapshots

.gitattributes

@@ -1,4 +1,6 @@
 .github/ export-ignore
+docs/*.sh text eol=lf
+docs/scripts/*.sh text eol=lf
 datafusion/core/tests/data/newlines_in_values.csv text eol=lf
 datafusion/proto/src/generated/prost.rs linguist-generated
 datafusion/proto/src/generated/pbjson.rs linguist-generated

.github/workflows/docs.yaml

@@ -20,12 +20,19 @@ on:
     branches:
       - main
     paths:
+      - .gitattributes
       - .asf.yaml
       - .github/workflows/docs.yaml
       - docs/**
+      - pyproject.toml
+      - uv.lock
 
 name: Deploy DataFusion site
 
+concurrency:
+  group: docs-deployment
+  cancel-in-progress: false
+
 jobs:
   build-docs:
     permissions:
@@ -34,16 +41,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout docs sources
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - name: Checkout asf-site branch
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           ref: asf-site
           path: asf-site
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d  # v10.0.1
+        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
 
       - name: Install dependencies
         run: uv sync --package datafusion-docs
@@ -53,7 +60,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y graphviz
       - name: Install cargo-depgraph
-        uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518  # v2.86.5
+        uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5
         with:
           tool: cargo-depgraph@1.6
 
@@ -63,23 +70,35 @@ jobs:
           cd docs
           uv run --package datafusion-docs ./build.sh
 
-      - name: Copy & push the generated HTML
+      - name: Assemble deployment site
+        run: |
+          uv run --package datafusion-docs python docs/scripts/assemble_site.py \
+            --current-site docs/build/html \
+            --published-site asf-site \
+            --output-site deployment-site
+          cp .asf.yaml deployment-site/
+          touch deployment-site/.nojekyll
+
+      - name: Validate deployment site
+        run: >-
+          uv run --package datafusion-docs python docs/scripts/validate_site.py
+          --site-root deployment-site --require-snapshots
+
+      - name: Copy and push the generated HTML
         run: |
           set -x
           cd asf-site/
           rsync \
             -a \
             --delete \
             --exclude '/.git/' \
-            ../docs/build/html/ \
+            ../deployment-site/ \
             ./
-          cp ../.asf.yaml .
-          touch .nojekyll
           git status --porcelain
           if [ "$(git status --porcelain)" != "" ]; then
             git config user.name "github-actions[bot]"
             git config user.email "github-actions[bot]@users.noreply.github.com"
             git add --all
             git commit -m 'Publish built docs triggered by ${{ github.sha }}'
-            git push || git push --force
+            git push
           fi

.github/workflows/docs_pr.yaml

@@ -25,13 +25,23 @@ concurrency:
 on:
   push:
     paths:
+      - ".gitattributes"
       - "docs/**"
+      - ".github/workflows/docs.yaml"
+      - ".github/workflows/docs_pr.yaml"
+      - "pyproject.toml"
+      - "uv.lock"
     branches:
       - main
       - branch-*
   pull_request:
     paths:
+      - ".gitattributes"
       - "docs/**"
+      - ".github/workflows/docs.yaml"
+      - ".github/workflows/docs_pr.yaml"
+      - "pyproject.toml"
+      - "uv.lock"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:
@@ -45,12 +55,12 @@ jobs:
     name: Test doc build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           submodules: true
           fetch-depth: 1
       - name: Setup uv
-        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d  # v10.0.1
+        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
       - name: Install doc dependencies
         run: uv sync --package datafusion-docs
       - name: Install Graphviz
@@ -59,11 +69,32 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y graphviz
       - name: Install cargo-depgraph
-        uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518  # v2.86.5
+        uses: taiki-e/install-action@ba47c86ac325773530516bb756137ac718732518 # v2.86.5
         with:
           tool: cargo-depgraph@1.6
       - name: Build docs html and check for warnings
         run: |
           set -x
           cd docs
           uv run --package datafusion-docs ./build.sh # fails on errors
+      - name: Fetch exact release tag
+        run: git fetch --depth=1 origin tag 55.0.0
+      - name: Test versioned documentation tooling
+        run: uv run --package datafusion-docs python -m unittest discover -s docs/tests
+      - name: Validate current site
+        run: >-
+          uv run --package datafusion-docs python docs/scripts/validate_site.py
+          --site-root docs/build/html
+      - name: Build exact release snapshot
+        run: |
+          uv run --package datafusion-docs python docs/scripts/snapshot_site.py \
+            55.0.0 55.0.0 /tmp/datafusion-release-site
+      - name: Validate release deployment
+        run: |
+          uv run --package datafusion-docs python docs/scripts/assemble_site.py \
+            --current-site docs/build/html \
+            --published-site /tmp/datafusion-release-site \
+            --output-site docs/build/validation-site
+          uv run --package datafusion-docs python docs/scripts/validate_site.py \
+            --site-root docs/build/validation-site \
+            --require-snapshots

dev/release/README.md

@@ -301,6 +301,29 @@ git tag 50.3.0
 git push apache 50.3.0
 ```
 
+#### Publish the versioned documentation
+
+For the initial `55.0.0` proof of concept, make the final exact tag available
+locally and build its complete documentation site into a temporary publication
+package:
+
+```shell
+uv run --package datafusion-docs python docs/scripts/snapshot_site.py 55.0.0 55.0.0 /tmp/datafusion-release-site
+```
+
+The command verifies the tag's peeled commit and creates the one immutable
+publication unit `versions/55.0.0/`. Manually copy that directory to the same
+location on the `asf-site` branch, then review, commit, and push the branch. The
+command never commits or pushes and refuses to replace an existing release.
+
+Do not simply publish the snapshot before merging the feature PR: the old docs
+deployment can erase it with `rsync --delete`. Keep the feature PR in draft until
+maintainers choose either a preliminary retention-only merge before publication,
+or a coordinated window that confirms no old docs deployment is running or
+queued, publishes the snapshot, merges the retention and picker change, and
+verifies `asf-site/versions/55.0.0/` after the new deployment. Automating future
+release publication and backfilling older releases remain follow-up work.
+
 ### 10. Publish on Crates.io
 
 Only approved releases of the tarball should be published to

docs/README.md

@@ -19,63 +19,142 @@
 
 # DataFusion Documentation
 
-This folder contains the source content of the [User Guide](./source/user-guide)
-and [Contributor Guide](./source/contributor-guide). These are both published to
-https://datafusion.apache.org/ as part of the release process.
+This folder contains the sources for https://datafusion.apache.org/. The root
+site is built continuously from `main`. Complete, immutable release sites are
+published under `/versions/<version>/`.
 
 ## Dependencies
 
-Install build dependencies and build the documentation using
+From the repository root, install the documentation dependencies using
 [uv](https://docs.astral.sh/uv/):
 
 ```sh
-uv sync
-uv run bash build.sh
+uv sync --package datafusion-docs
 ```
 
 The docs build regenerates the workspace dependency graph via
 `docs/scripts/generate_dependency_graph.sh`, so ensure `cargo`, `cargo-depgraph`
 (`cargo install cargo-depgraph --version ^1.6 --locked`), and Graphviz `dot`
 (`brew install graphviz` or `sudo apt-get install -y graphviz`) are available.
+`.gitattributes` keeps documentation shell scripts LF-terminated so the same
+scripts run from Linux and WSL checkouts.
 
-## Build & Preview
+## Build and Preview
 
-Run the provided script to build the HTML pages.
+Build the current complete site from the repository root:
 
 ```bash
-# If using venv, ensure you have activated it
-./build.sh
+uv run --package datafusion-docs docs/build.sh
 ```
 
-The HTML will be generated into a `build` directory. Open `build/html/index.html`
-in your preferred browser, e.g.
+The HTML is generated in `docs/build/html`. Serve it over HTTP because browsers
+do not load the version manifest from `file:` URLs:
 
-Preview the site on Linux by running this command.
+```bash
+python3 -m http.server --directory docs/build/html 8000
+```
+
+Then open http://localhost:8000/.
+
+The public and assembled layouts are:
+
+```text
+https://datafusion.apache.org/
+|-- user-guide/
+|-- library-user-guide/
+|-- contributor-guide/
+|-- _static/versions.json
+`-- versions/55.0.0/
+    |-- user-guide/
+    |-- library-user-guide/
+    |-- contributor-guide/
+    |-- download.html
+    |-- search.html
+    |-- _sources/
+    |-- _static/
+    `-- sitemap.xml
+```
+
+## Release Snapshots
+
+`docs/source/_static/versions.json` is both the PyData version-picker manifest
+and the release catalog. It contains `Development` at the site root and records
+each release's semantic version, exact tag, and exact 40-character commit.
+
+The first snapshot is the lightweight tag `55.0.0`, which peels to
+`d5552342012888b7d1a3ab88d92e3d292fc0cde0`. Create its publication package
+from the repository root with the tag available locally:
 
 ```bash
-# On macOS
-open build/html/index.html
-# On Linux with Firefox
-firefox build/html/index.html
+uv run --package datafusion-docs python docs/scripts/snapshot_site.py \
+  55.0.0 55.0.0 /tmp/datafusion-release-site
 ```
 
-## Making Changes
+The command creates only
+`/tmp/datafusion-release-site/versions/55.0.0/`. It refuses to overwrite that
+directory, rejects output overlapping the repository, builds in an isolated
+detached worktree, and does not commit or push.
+
+The snapshot uses the tag's complete documentation tree, templates, static
+files, helper extension, build script dependency graph, and locked documentation
+dependencies. `release_conf.py` supplies only the publication prefix, canonical
+base URL, exactly pinned `sphinx-sitemap` extension, version picker, exact GitHub
+tag, corrected repository name, and a release-local redirect. After Sphinx
+builds, `snapshot_site.py` rewrites only published links that would otherwise
+escape to current DataFusion docs or mutable DataFusion `main` and `latest`
+targets, including links in the agent-facing `llms.txt`. This includes one narrow
+fix for the tagged broken `/contributor-guide/gsoc_application_guidelines.html`
+link, which is redirected to the tagged
+`contributor-guide/gsoc/gsoc_application_guidelines_2025.html` page. Tagged
+source files and generated `_sources` files are never modified. In particular,
+the tagged statement that 55.0.0 has not been released yet remains unchanged.
+
+Validate a complete assembled site with:
+
+```bash
+uv run --package datafusion-docs python docs/scripts/assemble_site.py \
+  --current-site docs/build/html \
+  --published-site /tmp/datafusion-release-site \
+  --output-site /tmp/datafusion-site
+uv run --package datafusion-docs python docs/scripts/validate_site.py \
+  --site-root /tmp/datafusion-site --require-snapshots
+```
+
+The assembler starts with a fresh current build, copies the existing complete
+`asf-site/versions/` archive unchanged, and creates a root sitemap index. It does
+not use picker entries as a deletion list, so removing an old release from the
+picker cannot erase its archive.
+
+## Publication Bootstrap
+
+The old deployment uses unrestricted `rsync --delete`, so publishing a snapshot
+before snapshot retention reaches `main` is not race-free. Keep the feature PR
+in draft until maintainers agree on one of these bootstrap procedures:
 
-To make changes to the docs, simply make a Pull Request with your
-proposed changes as normal. When the PR is merged the docs will be
-automatically updated.
+1. Merge a preliminary retention-only workflow change, wait for it to deploy,
+   publish `versions/55.0.0/` manually to `asf-site`, then merge the picker and
+   validation changes.
+2. Use a coordinated window: confirm no old documentation deployment is running
+   or queued, publish the snapshot, merge this change immediately, wait for the
+   new serialized deployment, and verify `asf-site/versions/55.0.0/` afterward.
 
-## Release Process
+For manual publication, copy the single generated `versions/55.0.0/` directory
+to the same location in an `asf-site` worktree. Review, commit, and push that
+branch manually. Never replace an existing release directory. The snapshot
+tooling never commits, pushes, or publishes.
 
-This documentation is hosted at https://datafusion.apache.org/
+## Site Deployment
 
-When the PR is merged to the `main` branch of the DataFusion
-repository, a [github workflow](https://github.com/apache/datafusion/blob/main/.github/workflows/docs.yaml) which:
+When a documentation change reaches `main`, the deployment workflow:
 
-1. Builds the html content
-2. Pushes the html content to the [`asf-site`](https://github.com/apache/datafusion/tree/asf-site) branch in this repository.
+1. Builds the current complete site and replaces all current root files.
+2. Copies the entire pre-existing `asf-site/versions/` archive without using the
+   picker as a retention registry.
+3. Builds `sitemap.xml` as an index over `sitemap-main.xml` and every retained
+   complete release sitemap.
+4. Deliberately installs `.asf.yaml` and `.nojekyll`, then uses `rsync --delete`
+   while excluding `.git` to remove stale current output.
+5. Serializes deployments and pushes a normal, non-force commit.
 
-The Apache Software Foundation provides https://datafusion.apache.org/,
-which serves content based on the configuration in
-[.asf.yaml](https://github.com/apache/datafusion/blob/main/.asf.yaml),
-which specifies the target as https://datafusion.apache.org/.
+The Apache Software Foundation serves the branch according to
+[`.asf.yaml`](https://github.com/apache/datafusion/blob/main/.asf.yaml).