docs: use sphinx cross-ref roles in docstrings for IDE rendering

JetBrains and other IDEs do not understand mkdocstrings autoref
syntax (`[name][path]`) and display it as literal text in docstring
hovers. Switch docstring cross-references to sphinx-style roles
(:func:, :class:, :meth:, :attr:, :mod:, :exc:) which IDEs render
natively as clickable links.

A new griffe extension (`docs/griffe_extensions.py`) rewrites the
sphinx roles back into mkdocstrings autorefs before mkdocstrings
parses each docstring, so the docs site continues to produce
working cross-reference links.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: timsaucer

docs/griffe_extensions.py

@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Griffe extensions for datafusion-python docs.
+
+`SphinxRefsToAutorefs` rewrites sphinx-style cross-reference roles
+(``:func:`~path`, :class:`~path``, etc.) inside docstrings into
+mkdocstrings autoref syntax (``[`tail`][path]``) so that the same
+docstring renders as a clickable cross-reference both in JetBrains-style
+IDEs (which understand sphinx roles) and on the published docs site
+(which understands mkdocstrings autorefs).
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from griffe import Extension, Object
+
+_ROLE_RE = re.compile(
+    r":(?:py:)?(?P<role>func|class|meth|attr|mod|obj|exc|const|data)"
+    r":`(?P<tilde>~?)(?P<target>[\w.]+)`"
+)
+
+
+def _rewrite(text: str) -> str:
+    def repl(match: re.Match[str]) -> str:
+        target = match.group("target")
+        tail = target.rsplit(".", 1)[-1]
+        return f"[`{tail}`][{target}]"
+
+    return _ROLE_RE.sub(repl, text)
+
+
+class SphinxRefsToAutorefs(Extension):
+    """Convert sphinx-style cross-references into mkdocstrings autorefs."""
+
+    def on_object(self, *, obj: Object, **_: Any) -> None:
+        docstring = obj.docstring
+        if docstring is None:
+            return
+        new = _rewrite(docstring.value)
+        if new != docstring.value:
+            docstring.value = new

mkdocs.yml

@@ -67,6 +67,8 @@ plugins:
             - https://arrow.apache.org/docs/objects.inv
             - https://docs.pola.rs/api/python/stable/objects.inv
           options:
+            extensions:
+              - docs/griffe_extensions.py:SphinxRefsToAutorefs
             docstring_style: google
             docstring_options:
               warn_unknown_params: false

python/datafusion/__init__.py

@@ -27,10 +27,10 @@
 - **SessionContext** -- entry point for loading data, running SQL, and creating
   DataFrames.
 - **DataFrame** -- lazy query builder. Every method returns a new DataFrame;
-  call [`collect`][datafusion.dataframe.DataFrame.collect] or a ``to_*``
+  call :meth:`~datafusion.dataframe.DataFrame.collect` or a ``to_*``
   method to execute.
 - **Expr** -- expression tree node for column references, literals, and function
-  calls. Build with [`col`][datafusion.col.col] and [`lit`][datafusion.lit].
+  calls. Build with :func:`~datafusion.col.col` and :func:`~datafusion.lit`.
 - **functions** -- 290+ built-in scalar, aggregate, and window functions.
 
 Examples:

python/datafusion/catalog.py

@@ -220,7 +220,7 @@ def __repr__(self) -> str:
     @staticmethod
     @deprecated("Use Table() constructor instead.")
     def from_dataset(dataset: pa.dataset.Dataset) -> Table:
-        """Turn a `dataset` ``Dataset`` into a [`Table`][datafusion.catalog.Table]."""
+        """Turn a `dataset` ``Dataset`` into a :class:`~datafusion.catalog.Table`."""
         return Table(dataset)
 
     @property