ci: check for fields potentially missing from structs in pyo3-ffi definitions

.github/workflows/build.yml

@@ -81,14 +81,6 @@ jobs:
         run: |
           echo "CARGO_BUILD_TARGET=i686-pc-windows-msvc" >> $GITHUB_ENV
 
-      # windows on arm image contains x86-64 libclang
-      - name: Install LLVM and Clang
-        if: inputs.os == 'windows-11-arm'
-        uses: KyleMayes/install-llvm-action@v2
-        with:
-          # to match windows-2022 images
-          version: "18"
-
       - name: Install zoneinfo backport for Python 3.8
         if: contains(fromJSON('["3.8"]'), inputs.python-version)
         run: python -m pip install backports.zoneinfo

newsfragments/5917.fixed.md

@@ -1 +1 @@
-Fixed missing FFI definition for `PyTypeObject.tp_versions_used` on Python 3.14 and newer.
+Fixed missing FFI definition for `PyTypeObject.tp_versions_used` on Python 3.13 and newer.

noxfile.py

@@ -263,7 +263,6 @@ def _clippy_additional_workspaces(session: nox.Session) -> bool:
     target = os.environ.get("CARGO_BUILD_TARGET")
     if target is None or _get_rust_default_target() == target:
         try:
-            _build_docs_for_ffi_check(session)
             _run_cargo(session, "clippy", _FFI_CHECK, "--workspace", "--all-targets")
         except Exception:
             success = False
@@ -1097,7 +1096,6 @@ def load_pkg_versions():
 
 @nox.session(name="ffi-check")
 def ffi_check(session: nox.Session):
-    _build_docs_for_ffi_check(session)
     _run_cargo(session, "run", _FFI_CHECK)
     _check_raw_dylib_macro(session)
 
@@ -1445,13 +1443,6 @@ def test_introspection(session: nox.Session):
     )
 
 
-def _build_docs_for_ffi_check(session: nox.Session) -> None:
-    # pyo3-ffi-check needs to scrape docs of pyo3-ffi
-    env = os.environ.copy()
-    env["PYO3_PYTHON"] = sys.executable
-    _run_cargo(session, "doc", _FFI_CHECK, "-p", "pyo3-ffi", "--no-deps", env=env)
-
-
 @lru_cache()
 def _get_rust_info() -> Tuple[str, ...]:
     output = _get_output("rustc", "-vV")

pyo3-ffi-check/Cargo.toml

@@ -5,14 +5,14 @@ edition = "2021"
 publish = false
 
 [dependencies]
-pyo3-ffi-check-macro = { path = "./macro" }
-pyo3-ffi = { path = "../pyo3-ffi" }
+pyo3-ffi-check-definitions = { path = "definitions" }
+pyo3-ffi-check-macro = { path = "macro" }
 
 [build-dependencies]
-bindgen = "0.69.4"
 pyo3-build-config = { path = "../pyo3-build-config" }
 
 [workspace]
 members = [
+    "definitions",
     "macro"
 ]

pyo3-ffi-check/build.rs

@@ -1,72 +1,38 @@
-use std::env;
-use std::path::PathBuf;
-
-#[derive(Debug)]
-struct ParseCallbacks;
-
-impl bindgen::callbacks::ParseCallbacks for ParseCallbacks {
-    // these are anonymous fields and structs in CPython that we needed to
-    // invent names for. Bindgen seems to generate stable names, so we remap the
-    // automatically generated names to the names we invented in the FFI
-    fn item_name(&self, _original_item_name: &str) -> Option<String> {
-        if _original_item_name == "_object__bindgen_ty_1__bindgen_ty_1" {
-            Some("PyObjectObFlagsAndRefcnt".into())
-        } else if _original_item_name == "_object__bindgen_ty_1" {
-            Some("PyObjectObRefcnt".into())
-        } else {
-            None
-        }
-    }
-}
-
 fn main() {
+    let out_dir = std::env::var("OUT_DIR").unwrap();
+    let target = std::env::var("TARGET").unwrap();
+
+    let doc_dir = std::path::Path::new(&out_dir)
+        .join(&target)
+        .join("doc")
+        .join("pyo3_ffi_check_definitions");
+
+    // write docs into the build script output directory, they will be read
+    // by the proc macro to resolve what definitions exist
+    let status = std::process::Command::new("cargo")
+        .args(["doc", "-p", "pyo3-ffi-check-definitions", "--no-deps"])
+        .env("CARGO_TARGET_DIR", out_dir)
+        // forward target to the doc buid to ensure `--target` is honored
+        .env("CARGO_BUILD_TARGET", target)
+        .status()
+        .expect("failed to build definitions");
+
+    // macro will use this env var to locate the docs
+    println!(
+        "cargo:rustc-env=PYO3_FFI_CHECK_DOC_DIR={}",
+        doc_dir.to_str().unwrap()
+    );
+
+    status
+        .success()
+        .then_some(())
+        .expect("failed to build definitions, see above for details");
+
+    // rerun if any of the definitions change, to ensure the docs are up to date
+    println!("cargo:rerun-if-changed=definitions");
+    println!("cargo:rerun-if-changed=../pyo3-ffi");
+
+    // Because `pyo3-ffi` is a dependency, libpython is linked, this ensures `main.rs` can run.
+    // Slightly needless (no symbols from libpython are actually called), but simple to do.
     pyo3_build_config::add_libpython_rpath_link_args();
-
-    let config = pyo3_build_config::get();
-
-    let python_include_dir = config
-        .run_python_script(
-            "import sysconfig; print(sysconfig.get_config_var('INCLUDEPY'), end='');",
-        )
-        .expect("failed to get lib dir");
-    let gil_disabled_on_windows = config
-        .run_python_script(
-            "import sysconfig; import platform; print(sysconfig.get_config_var('Py_GIL_DISABLED') == 1 and platform.system() == 'Windows');",
-        )
-        .expect("failed to get Py_GIL_DISABLED").trim_end() == "True";
-
-    let clang_args = if gil_disabled_on_windows {
-        vec![
-            format!("-I{python_include_dir}"),
-            "-DPy_GIL_DISABLED".to_string(),
-        ]
-    } else {
-        vec![format!("-I{python_include_dir}")]
-    };
-
-    println!("cargo:rerun-if-changed=wrapper.h");
-
-    let bindings = bindgen::Builder::default()
-        .header("wrapper.h")
-        .clang_args(clang_args)
-        .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))
-        .parse_callbacks(Box::new(ParseCallbacks))
-        // blocklist some values which apparently have conflicting definitions on unix
-        .blocklist_item("FP_NORMAL")
-        .blocklist_item("FP_SUBNORMAL")
-        .blocklist_item("FP_NAN")
-        .blocklist_item("FP_INFINITE")
-        .blocklist_item("FP_INT_UPWARD")
-        .blocklist_item("FP_INT_DOWNWARD")
-        .blocklist_item("FP_INT_TOWARDZERO")
-        .blocklist_item("FP_INT_TONEARESTFROMZERO")
-        .blocklist_item("FP_INT_TONEAREST")
-        .blocklist_item("FP_ZERO")
-        .generate()
-        .expect("Unable to generate bindings");
-
-    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
-    bindings
-        .write_to_file(out_path.join("bindings.rs"))
-        .expect("Couldn't write bindings!");
 }

pyo3-ffi-check/definitions/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "pyo3-ffi-check-definitions"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+pyo3-ffi = { path = "../../pyo3-ffi" }
+
+[build-dependencies]
+bindgen = "0.72"
+pyo3-build-config = { path = "../../pyo3-build-config" }

pyo3-ffi-check/definitions/build.rs

@@ -0,0 +1,74 @@
+use std::env;
+use std::path::PathBuf;
+
+use bindgen::callbacks::ItemInfo;
+
+#[derive(Debug)]
+struct ParseCallbacks;
+
+impl bindgen::callbacks::ParseCallbacks for ParseCallbacks {
+    // these are anonymous fields and structs in CPython that we needed to
+    // invent names for. Bindgen seems to generate stable names, so we remap the
+    // automatically generated names to the names we invented in the FFI
+    fn item_name(&self, item_info: ItemInfo<'_>) -> Option<String> {
+        match item_info.name {
+            "_object__bindgen_ty_1__bindgen_ty_1" => Some("PyObjectObFlagsAndRefcnt".into()),
+            "_object__bindgen_ty_1" => Some("PyObjectObRefcnt".into()),
+            _ => None,
+        }
+    }
+}
+
+fn main() {
+    let config = pyo3_build_config::get();
+
+    let python_include_dir = config
+        .run_python_script(
+            "import sysconfig; print(sysconfig.get_config_var('INCLUDEPY'), end='');",
+        )
+        .expect("failed to get lib dir");
+    let gil_disabled_on_windows = config
+        .run_python_script(
+            "import sysconfig; import platform; print(sysconfig.get_config_var('Py_GIL_DISABLED') == 1 and platform.system() == 'Windows');",
+        )
+        .expect("failed to get Py_GIL_DISABLED").trim_end() == "True";
+
+    let clang_args = if gil_disabled_on_windows {
+        vec![
+            format!("-I{python_include_dir}"),
+            "-DPy_GIL_DISABLED".to_string(),
+        ]
+    } else {
+        vec![format!("-I{python_include_dir}")]
+    };
+
+    println!("cargo:rerun-if-changed=wrapper.h");
+
+    let bindings = bindgen::Builder::default()
+        .header("wrapper.h")
+        .clang_args(clang_args)
+        .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))
+        .parse_callbacks(Box::new(ParseCallbacks))
+        // blocklist some values which apparently have conflicting definitions on unix
+        .blocklist_item("FP_NORMAL")
+        .blocklist_item("FP_SUBNORMAL")
+        .blocklist_item("FP_NAN")
+        .blocklist_item("FP_INFINITE")
+        .blocklist_item("FP_INT_UPWARD")
+        .blocklist_item("FP_INT_DOWNWARD")
+        .blocklist_item("FP_INT_TOWARDZERO")
+        .blocklist_item("FP_INT_TONEARESTFROMZERO")
+        .blocklist_item("FP_INT_TONEAREST")
+        .blocklist_item("FP_ZERO")
+        // ARM neon intrinsics cause issue on GitHub actions windows CI, also not relevant to
+        // what we're trying to check anyway.
+        .blocklist_file(r".*(\\|/)arm(64)?_neon\.h")
+        .blocklist_file(r".*(\\|/)arm_vector_types\.h")
+        .generate()
+        .expect("Unable to generate bindings");
+
+    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    bindings
+        .write_to_file(out_path.join("bindings.rs"))
+        .expect("Couldn't write bindings!");
+}

pyo3-ffi-check/definitions/src/lib.rs

@@ -0,0 +1,18 @@
+#[allow(
+    non_snake_case,
+    non_camel_case_types,
+    non_upper_case_globals,
+    dead_code,
+    improper_ctypes,
+    clippy::all,
+    // clippy fails with lots of errors if this is not set specifically
+    clippy::used_underscore_binding
+)]
+pub mod bindgen {
+    include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
+}
+
+pub mod pyo3_ffi {
+    #[doc(inline)]
+    pub use pyo3_ffi::*;
+}