docs: Add more past versions to the doc site

.github/workflows/docs_versions.yaml

@@ -18,7 +18,9 @@
 # Publishes the documentation for released versions to
 # https://datafusion.apache.org/versions/<version>/
 #
-# The list of versions is in docs/source/_static/versions.json
+# The list of versions is in
+# 1. docs/source/_static/versions.json (in version picker)
+# 2. docs/source/_static/old_versions.json (older versions)
 #
 # The current development docs are published from `main` by docs.yaml.
 
@@ -31,6 +33,7 @@ on:
       - docs/scripts/build_versions.sh
       - docs/scripts/release/conf.py
       - docs/source/_static/versions.json
+      - docs/source/_static/old_versions.json
       - docs/pyproject.toml
       - uv.lock
 
@@ -69,7 +72,8 @@ jobs:
         run: |
           set -x
           cd asf-site/
-          # versions/ mirrors the releases in versions.json; the current docs
+          # versions/ mirrors the releases in versions.json and
+          # old_versions.json; the current docs
           # at the site root are published by docs.yaml
           mkdir -p versions
           # remove any old versions and replace with newly built ones

dev/release/README.md

@@ -331,7 +331,15 @@ git push apache 55.2.0
 
 Open a PR against `main` that adds the release to
 `docs/source/_static/versions.json` immediately after the `Development (main)`
-entry mark it with `"preferred": true`.
+entry and marks it with `"preferred": true` (remove `"preferred"` from the
+previous release).
+
+The version picker only lists the most recent release of each major version.
+If the new release is a minor or patch release (e.g. `55.2.0`), move the
+previous release of the same major version (e.g. `55.1.0`) from `versions.json`
+to `docs/source/_static/old_versions.json`. Releases in `old_versions.json`
+are still built and published under `/versions/<version>/`, so existing links
+keep working, but they are not shown in the version picker.
 
 Once the PR is merged, the
 [Deploy DataFusion versioned docs](../../.github/workflows/docs_versions.yaml)

docs/README.md

@@ -90,6 +90,9 @@ which serves content based on the configuration in
 which specifies the target as https://datafusion.apache.org/.
 
 The documentation for previously released versions is served under
-`/versions/<version>/`. See the
+`/versions/<version>/`. The releases in the version picker are listed in
+`source/_static/versions.json`, and older releases that are still published,
+but not shown in the picker, are listed in `source/_static/old_versions.json`.
+See the
 [release instructions](../dev/release/README.md#publish-the-versioned-documentation)
 for how it is published.

docs/scripts/build_versions.sh

@@ -17,7 +17,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Build the documentation for each release listed in docs/source/_static/versions.json
+# Build the documentation for releases in
+# 1. docs/source/_static/versions.json (shown in the version picker)
+# 2. docs/source/_static/old_versions.json (older releases to keep links working)
 #
 # The release is checked out into a temporary worktree, writing the result
 # to <output>/<version>/.
@@ -29,6 +31,7 @@ DOCS_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
 REPO_DIR="$(cd "${DOCS_DIR}/.." && pwd)"
 OVERLAY_DIR="${SCRIPT_DIR}/release"
 MANIFEST="${DOCS_DIR}/source/_static/versions.json"
+OLD_MANIFEST="${DOCS_DIR}/source/_static/old_versions.json"
 
 # Where to fetch release tags from (forks usually do not contain release tags)
 UPSTREAM_URL="${DATAFUSION_UPSTREAM_URL:-https://github.com/apache/datafusion.git}"
@@ -44,7 +47,8 @@ Build the documentation for released DataFusion versions.
 Usage: $(basename "$0") [options] [VERSION ...]
 
 Builds the documentation for each VERSION from its git tag. With no VERSION,
-builds every release listed in ${MANIFEST#"${REPO_DIR}/"}.
+builds every release listed in ${MANIFEST#"${REPO_DIR}/"}
+or ${OLD_MANIFEST#"${REPO_DIR}/"}.
 
 Options:
   -o, --output DIR   Write the documentation to DIR/<version>/
@@ -99,16 +103,21 @@ for tool in git uv python3; do
   fi
 done
 
-# Releases listed in the version picker manifest, newest first.
+# Releases listed in the version picker manifest followed by the older
+# releases that are published but not in the picker, without duplicates.
 list_versions() {
-  python3 - "${MANIFEST}" <<'PY'
+  python3 - "${MANIFEST}" "${OLD_MANIFEST}" <<'PY'
 import json
 import sys
 
-with open(sys.argv[1]) as f:
-    for entry in json.load(f):
-        if entry["version"] != "main":
-            print(entry["version"])
+seen = set()
+for manifest in sys.argv[1:]:
+    with open(manifest) as f:
+        for entry in json.load(f):
+            version = entry["version"]
+            if version != "main" and version not in seen:
+                seen.add(version)
+                print(version)
 PY
 }
 

docs/scripts/release/conf.py

@@ -34,11 +34,14 @@
 exec(compile((source / "conf.py").read_bytes(), str(source / "conf.py"), "exec"))
 
 # Resolve paths relative to the tag, not this configuration overlay.
+# Releases before 54.0.0 do not set html_extra_path, and releases before
+# 53.0.0 do not set html_favicon.
 templates_path = [str(source / path) for path in templates_path]
 html_static_path = [str(source / path) for path in html_static_path]
-html_extra_path = [str(source / path) for path in html_extra_path]
+html_extra_path = [str(source / path) for path in globals().get("html_extra_path", [])]
 html_logo = str(source / html_logo)
-html_favicon = str(source / html_favicon)
+if globals().get("html_favicon"):
+    html_favicon = str(source / html_favicon)
 
 version = release = os.environ["DATAFUSION_DOCS_VERSION"]
 html_baseurl = f"https://datafusion.apache.org/versions/{version}/"
@@ -56,4 +59,25 @@
     },
 }
 # The old tag's absolute redirect would leave the release documentation.
-redirects = {**redirects, "library-user-guide/upgrading": "upgrading/index.html"}
+# Releases before 53.0.0 have no redirects (the upgrade guide was a single page).
+if "library-user-guide/upgrading" in globals().get("redirects", {}):
+    redirects = {**redirects, "library-user-guide/upgrading": "upgrading/index.html"}
+
+
+# Releases before 51.0.0 have a sidebar template that calls
+# ``generate_nav_html``, which pydata-sphinx-theme renamed to
+# ``generate_toctree_html``.
+def _add_generate_nav_html(app, pagename, templatename, context, doctree):
+    def generate_nav_html(kind, startdepth=1, **kwargs):
+        return context["generate_toctree_html"](kind, startdepth=startdepth, **kwargs)
+
+    context.setdefault("generate_nav_html", generate_nav_html)
+
+
+_release_setup = globals().get("setup")
+
+
+def setup(app):
+    if _release_setup is not None:
+        _release_setup(app)
+    app.connect("html-page-context", _add_generate_nav_html)

docs/source/_static/old_versions.json

@@ -0,0 +1,12 @@
+[
+  {
+    "name": "55.1.0",
+    "url": "/versions/55.1.0/",
+    "version": "55.1.0"
+  },
+  {
+    "name": "55.0.0",
+    "url": "/versions/55.0.0/",
+    "version": "55.0.0"
+  }
+]

docs/source/_static/versions.json

@@ -11,18 +11,28 @@
     "preferred": true
   },
   {
-    "name": "55.1.0",
-    "url": "/versions/55.1.0/",
-    "version": "55.1.0"
+    "name": "54.1.0",
+    "url": "/versions/54.1.0/",
+    "version": "54.1.0"
   },
   {
-    "name": "55.0.0",
-    "url": "/versions/55.0.0/",
-    "version": "55.0.0"
+    "name": "53.0.0",
+    "url": "/versions/53.0.0/",
+    "version": "53.0.0"
   },
   {
-    "name": "54.1.0",
-    "url": "/versions/54.1.0/",
-    "version": "54.1.0"
+    "name": "52.0.0",
+    "url": "/versions/52.0.0/",
+    "version": "52.0.0"
+  },
+  {
+    "name": "51.0.0",
+    "url": "/versions/51.0.0/",
+    "version": "51.0.0"
+  },
+  {
+    "name": "50.0.0",
+    "url": "/versions/50.0.0/",
+    "version": "50.0.0"
   }
 ]