check static addresses match in pyo3-ffi-check

newsfragments/6421.fixed.md

@@ -0,0 +1 @@
+Fix many unresolved data symbols when linking for PyPy due to incorrect link names in `pyo3-ffi`.

newsfragments/6421.removed.md

@@ -0,0 +1 @@
+Remove FFI definitions `PyExc_RecursionErrorInst` and `Py_UseClassExceptionsFlag` (not present in supported Python versions).

pyo3-ffi-check/README.md

@@ -2,6 +2,6 @@
 
 This is a simple program which compares ffi definitions from `pyo3-ffi` against those produced by `bindgen`.
 
-If any differ in size, these are printed to stdout and a the process will exit nonzero.
+It checks type layouts, function signatures, and the addresses of functions and statics. Any differences are printed to stdout and the process exits nonzero.
 
 The main purpose of this program is to be run as part of PyO3's continuous integration pipeline to catch possible errors in PyO3's ffi definitions.

pyo3-ffi-check/macro/src/lib.rs

@@ -19,6 +19,11 @@ const PY_3_12: PythonVersion = PythonVersion {
     minor: 12,
 };
 
+const PY_3_11: PythonVersion = PythonVersion {
+    major: 3,
+    minor: 11,
+};
+
 /// Macro which expands to multiple macro calls, one per pyo3-ffi struct.
 #[proc_macro]
 pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
@@ -70,15 +75,20 @@ pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStrea
 static DOC_DIR: LazyLock<PathBuf> =
     LazyLock::new(|| PathBuf::from(env::var_os("PYO3_FFI_CHECK_DOC_DIR").unwrap()));
 
-static BINDGEN_FUNCTION_NAMES: LazyLock<HashSet<String>> = LazyLock::new(|| {
-    // parse all the function names from the bindgen index file
+static BINDGEN_FUNCTION_NAMES: LazyLock<HashSet<String>> =
+    LazyLock::new(|| get_bindgen_names("fn"));
+
+static BINDGEN_STATIC_NAMES: LazyLock<HashSet<String>> =
+    LazyLock::new(|| get_bindgen_names("static"));
+
+fn get_bindgen_names(kind: &str) -> HashSet<String> {
+    // Parse names from the bindgen index file.
     let index_file = DOC_DIR.join("bindgen/index.html");
 
-    // the functions are in `a` elements with class "fn", and the full path is in the
-    // `title` attribute
+    // The full path is in the `title` attribute of each item's link.
     let html = fs::read_to_string(index_file).unwrap();
     let html = scraper::Html::parse_document(&html);
-    let selector = scraper::Selector::parse("a.fn").unwrap();
+    let selector = scraper::Selector::parse(&format!("a.{kind}")).unwrap();
 
     html.select(&selector)
         .map(|el| {
@@ -91,7 +101,81 @@ static BINDGEN_FUNCTION_NAMES: LazyLock<HashSet<String>> = LazyLock::new(|| {
                 .to_string()
         })
         .collect()
-});
+}
+
+fn get_bindgen_name(name: &str, names: &HashSet<String>) -> String {
+    if pyo3_build_config::get().implementation() == PythonImplementation::PyPy
+        && (name.starts_with("Py") || name.starts_with("_Py"))
+    {
+        let prefixed_name = name.replacen("Py", "PyPy", 1);
+        if names.contains(&prefixed_name) {
+            return prefixed_name;
+        }
+    }
+    name.to_owned()
+}
+
+/// Macro which expands to multiple macro calls, one per pyo3-ffi static.
+#[proc_macro]
+pub fn for_all_statics(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+    let macro_name = match get_macro_name_from_input("for_all_statics", input) {
+        Ok(name) => name,
+        Err(err) => return err.into(),
+    };
+
+    let statics_glob = format!("{}/pyo3_ffi/static.*.html", DOC_DIR.display());
+    let mut output = TokenStream::new();
+
+    for entry in glob::glob(&statics_glob).expect("Failed to read glob pattern") {
+        let entry = entry.unwrap();
+        let file_name = entry.file_name().unwrap().to_string_lossy().into_owned();
+        let static_name = file_name
+            .strip_prefix("static.")
+            .unwrap()
+            .strip_suffix(".html")
+            .unwrap();
+
+        if static_name == "PyStructSequence_UnnamedField"
+            && pyo3_build_config::get().target_abi().version() < PY_3_11
+        {
+            // Not marked PyAPI_DATA (and thus not exported reliably) before Python 3.11.
+            // https://github.com/python/cpython/issues/88386
+            continue;
+        }
+
+        let is_pypy = pyo3_build_config::get().implementation() == PythonImplementation::PyPy;
+        if is_pypy && static_name == "PySuper_Type" {
+            // PyPy declares this in its headers but does not export it.
+            continue;
+        }
+
+        // PyPy uses a macro to define these aliases as the same static; CPython has three
+        // separate statics
+        let bindgen_name = get_bindgen_name(
+            if is_pypy
+                && matches!(
+                    static_name,
+                    "PyExc_EnvironmentError" | "PyExc_IOError" | "PyExc_WindowsError"
+                )
+            {
+                "PyExc_OSError"
+            } else {
+                static_name
+            },
+            &BINDGEN_STATIC_NAMES,
+        );
+        if is_pypy && !BINDGEN_STATIC_NAMES.contains(&bindgen_name) {
+            // As with functions, PyPy may not yet offer all of the declared symbols.
+            continue;
+        }
+
+        let static_ident = Ident::new(static_name, Span::call_site());
+        let bindgen_ident = Ident::new(&bindgen_name, Span::call_site());
+        output.extend(quote!(#macro_name!(#static_ident, #bindgen_ident);));
+    }
+
+    output.into()
+}
 
 /// Macro which expands to multiple macro calls, one per field in a pyo3-ffi
 /// struct.
@@ -557,16 +641,8 @@ pub fn for_all_functions(_input: proc_macro::TokenStream) -> proc_macro::TokenSt
             continue;
         }
 
-        let mut bindgen_name = function_name.to_owned();
+        let bindgen_name = get_bindgen_name(function_name, &BINDGEN_FUNCTION_NAMES);
         if pyo3_build_config::get().implementation() == PythonImplementation::PyPy {
-            // For PyPy, some functions are prefixed with "PyPy", we check whether the
-            // bindgen name contains the prefixed name and use that if it does.
-            if function_name.starts_with("Py") || function_name.starts_with("_Py") {
-                let prefixed_name = function_name.replacen("Py", "PyPy", 1);
-                if BINDGEN_FUNCTION_NAMES.contains(&prefixed_name) {
-                    bindgen_name = prefixed_name;
-                }
-            }
             // If the function doesn't exist in PyPy, for now we don't care:
             // - For PyO3 inline functions it's probably fine to include anyway
             // - For extern symbols - PyPy may add them in a future release

pyo3-ffi-check/src/main.rs

@@ -1,4 +1,7 @@
-use std::{ffi::CStr, process::exit};
+use std::{
+    ffi::{c_void, CStr},
+    process::exit,
+};
 
 use pyo3_ffi_check_definitions::{bindgen as bindings, pyo3_ffi};
 
@@ -201,6 +204,26 @@ fn main() {
 
     pyo3_ffi_check_macro::for_all_functions!(check_function);
 
+    macro_rules! check_static {
+        ($name:ident, $bindgen_name:ident) => {{
+            #[allow(deprecated)]
+            let pyo3_ffi_ptr = (&raw const pyo3_ffi::$name).cast::<c_void>();
+            let bindgen_ptr = (&raw const bindings::$bindgen_name).cast::<c_void>();
+
+            if pyo3_ffi_ptr != bindgen_ptr {
+                failed = true;
+                println!(
+                    "error: static address of {} differs between pyo3_ffi ({:p}) and bindgen ({:p})",
+                    stringify!($name),
+                    pyo3_ffi_ptr,
+                    bindgen_ptr
+                );
+            }
+        }};
+    }
+
+    pyo3_ffi_check_macro::for_all_statics!(check_static);
+
     if failed {
         exit(1);
     } else {

pyo3-ffi/src/abstract_.rs

@@ -55,7 +55,7 @@ extern_libpython! {
     ) -> *mut PyObject;
 
     #[cfg(all(PyPy, not(Py_3_13)))] // called internally in PyUnicodeDecodeError_Create on PyPy
-    #[cfg_attr(PyPy, link_name = "_PyPyObject_CallFunction_SizeT")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "_PyPyObject_CallFunction_SizeT")]
     pub(crate) fn _PyObject_CallFunction_SizeT(
         callable_object: *mut PyObject,
         format: *const c_char,

pyo3-ffi/src/boolobject.rs

@@ -14,10 +14,10 @@ extern_libpython! {
     pub fn PyBool_Check(op: *mut PyObject) -> c_int;
 
     #[cfg(all(not(GraalPy), not(all(Py_3_13, Py_LIMITED_API))))]
-    #[cfg_attr(PyPy, link_name = "_PyPy_FalseStruct")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "_PyPy_FalseStruct")]
     static mut _Py_FalseStruct: PyLongObject;
     #[cfg(all(not(GraalPy), not(all(Py_3_13, Py_LIMITED_API))))]
-    #[cfg_attr(PyPy, link_name = "_PyPy_TrueStruct")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "_PyPy_TrueStruct")]
     static mut _Py_TrueStruct: PyLongObject;
 
     #[cfg(GraalPy)]

pyo3-ffi/src/bytearrayobject.rs

@@ -4,7 +4,7 @@ use core::ffi::{c_char, c_int};
 
 #[cfg(not(RustPython))]
 extern_libpython! {
-    #[cfg_attr(PyPy, link_name = "PyPyByteArray_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyByteArray_Type")]
     pub static mut PyByteArray_Type: PyTypeObject;
 
     pub static mut PyByteArrayIter_Type: PyTypeObject;

pyo3-ffi/src/bytesobject.rs

@@ -4,7 +4,7 @@ use core::ffi::{c_char, c_int};
 
 #[cfg(not(RustPython))]
 extern_libpython! {
-    #[cfg_attr(PyPy, link_name = "PyPyBytes_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyBytes_Type")]
     pub static mut PyBytes_Type: PyTypeObject;
     pub static mut PyBytesIter_Type: PyTypeObject;
 }

pyo3-ffi/src/complexobject.rs

@@ -3,7 +3,7 @@ use core::ffi::{c_double, c_int};
 
 #[cfg(not(RustPython))]
 extern_libpython! {
-    #[cfg_attr(PyPy, link_name = "PyPyComplex_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyComplex_Type")]
     pub static mut PyComplex_Type: PyTypeObject;
 }
 

pyo3-ffi/src/cpython/cellobject.rs

@@ -12,6 +12,7 @@ extern_libpython! {
     pub fn PyCell_New(o: *mut PyObject) -> *mut PyObject;
     pub fn PyCell_Get(o: *mut PyObject) -> *mut PyObject;
     pub fn PyCell_Set(o: *mut PyObject, val: *mut PyObject) -> c_int;
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyCell_Type")]
     pub static mut PyCell_Type: PyTypeObject;
 }
 

pyo3-ffi/src/cpython/funcobject.rs

@@ -59,7 +59,7 @@ pub struct PyFunctionObject {
 }
 
 extern_libpython! {
-    #[cfg_attr(PyPy, link_name = "PyPyFunction_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyFunction_Type")]
     pub static mut PyFunction_Type: crate::PyTypeObject;
 }
 
@@ -143,7 +143,9 @@ pub unsafe fn PyFunction_GET_ANNOTATIONS(func: *mut PyObject) -> *mut PyObject {
 }
 
 extern_libpython! {
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyClassMethod_Type")]
     pub static mut PyClassMethod_Type: crate::PyTypeObject;
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyStaticMethod_Type")]
     pub static mut PyStaticMethod_Type: crate::PyTypeObject;
 
     #[cfg_attr(PyPy, link_name = "PyPyClassMethod_New")]

pyo3-ffi/src/cpython/pydebug.rs

@@ -9,6 +9,7 @@ extern_libpython! {
     #[cfg_attr(PyPy, link_name = "PyPy_VerboseFlag")]
     pub static mut Py_VerboseFlag: c_int;
     #[deprecated(note = "Python 3.12")]
+    #[cfg_attr(PyPy, link_name = "PyPy_QuietFlag")]
     pub static mut Py_QuietFlag: c_int;
     #[deprecated(note = "Python 3.12")]
     #[cfg_attr(PyPy, link_name = "PyPy_InteractiveFlag")]
@@ -26,9 +27,6 @@ extern_libpython! {
     #[cfg_attr(PyPy, link_name = "PyPy_BytesWarningFlag")]
     pub static mut Py_BytesWarningFlag: c_int;
     #[deprecated(note = "Python 3.12")]
-    #[cfg_attr(PyPy, link_name = "PyPy_UseClassExceptionsFlag")]
-    pub static mut Py_UseClassExceptionsFlag: c_int;
-    #[deprecated(note = "Python 3.12")]
     #[cfg_attr(PyPy, link_name = "PyPy_FrozenFlag")]
     pub static mut Py_FrozenFlag: c_int;
     #[deprecated(note = "Python 3.12")]
@@ -41,16 +39,19 @@ extern_libpython! {
     #[cfg_attr(PyPy, link_name = "PyPy_NoUserSiteDirectory")]
     pub static mut Py_NoUserSiteDirectory: c_int;
     #[deprecated(note = "Python 3.12")]
+    #[cfg_attr(PyPy, link_name = "PyPy_UnbufferedStdioFlag")]
     pub static mut Py_UnbufferedStdioFlag: c_int;
     #[cfg_attr(PyPy, link_name = "PyPy_HashRandomizationFlag")]
     pub static mut Py_HashRandomizationFlag: c_int;
     #[deprecated(note = "Python 3.12")]
+    #[cfg_attr(PyPy, link_name = "PyPy_IsolatedFlag")]
     pub static mut Py_IsolatedFlag: c_int;
     #[cfg(windows)]
     #[deprecated(note = "Python 3.12")]
     pub static mut Py_LegacyWindowsFSEncodingFlag: c_int;
     #[cfg(windows)]
     #[deprecated(note = "Python 3.12")]
+    #[cfg_attr(PyPy, link_name = "PyPy_LegacyWindowsStdioFlag")]
     pub static mut Py_LegacyWindowsStdioFlag: c_int;
 }
 

pyo3-ffi/src/cpython/pyframe.rs

@@ -20,6 +20,7 @@ pub const PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR: c_int = 4;
 pub const PyUnstable_EXECUTABLE_KINDS: c_int = 5;
 
 extern_libpython! {
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyFrame_Type")]
     pub static mut PyFrame_Type: PyTypeObject;
 
     #[cfg(Py_3_13)]

pyo3-ffi/src/cpython/pystate.rs

@@ -66,6 +66,7 @@ extern_libpython! {
     pub fn PyThreadState_GetUnchecked() -> *mut PyThreadState;
 
     #[cfg(not(Py_3_13))]
+    #[cfg_attr(PyPy, link_name = "_PyPyThreadState_UncheckedGet")]
     pub(crate) fn _PyThreadState_UncheckedGet() -> *mut PyThreadState;
 
     #[cfg(Py_3_11)]

pyo3-ffi/src/descrobject.rs

@@ -37,19 +37,19 @@ impl Default for PyGetSetDef {
 
 #[cfg(not(RustPython))]
 extern_libpython! {
-    #[cfg_attr(PyPy, link_name = "PyPyClassMethodDescr_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyClassMethodDescr_Type")]
     pub static mut PyClassMethodDescr_Type: PyTypeObject;
-    #[cfg_attr(PyPy, link_name = "PyPyGetSetDescr_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyGetSetDescr_Type")]
     pub static mut PyGetSetDescr_Type: PyTypeObject;
-    #[cfg_attr(PyPy, link_name = "PyPyMemberDescr_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyMemberDescr_Type")]
     pub static mut PyMemberDescr_Type: PyTypeObject;
-    #[cfg_attr(PyPy, link_name = "PyPyMethodDescr_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyMethodDescr_Type")]
     pub static mut PyMethodDescr_Type: PyTypeObject;
-    #[cfg_attr(PyPy, link_name = "PyPyWrapperDescr_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyWrapperDescr_Type")]
     pub static mut PyWrapperDescr_Type: PyTypeObject;
-    #[cfg_attr(PyPy, link_name = "PyPyDictProxy_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyDictProxy_Type")]
     pub static mut PyDictProxy_Type: PyTypeObject;
-    #[cfg_attr(PyPy, link_name = "PyPyProperty_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyProperty_Type")]
     pub static mut PyProperty_Type: PyTypeObject;
 }
 

pyo3-ffi/src/dictobject.rs

@@ -4,7 +4,7 @@ use core::ffi::{c_char, c_int};
 
 #[cfg(not(RustPython))]
 extern_libpython! {
-    #[cfg_attr(PyPy, link_name = "PyPyDict_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyDict_Type")]
     pub static mut PyDict_Type: PyTypeObject;
 }
 
@@ -97,7 +97,9 @@ extern_libpython! {
 
 #[cfg(not(RustPython))]
 extern_libpython! {
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyDictKeys_Type")]
     pub static mut PyDictKeys_Type: PyTypeObject;
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyDictValues_Type")]
     pub static mut PyDictValues_Type: PyTypeObject;
     pub static mut PyDictItems_Type: PyTypeObject;
 }

pyo3-ffi/src/enumobject.rs

@@ -2,5 +2,6 @@ use crate::object::PyTypeObject;
 
 extern_libpython! {
     pub static mut PyEnum_Type: PyTypeObject;
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyReversed_Type")]
     pub static mut PyReversed_Type: PyTypeObject;
 }

pyo3-ffi/src/floatobject.rs

@@ -7,7 +7,7 @@ opaque_struct!(pub PyFloatObject);
 
 extern_libpython! {
     #[cfg(not(RustPython))]
-    #[cfg_attr(PyPy, link_name = "PyPyFloat_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyFloat_Type")]
     pub static mut PyFloat_Type: PyTypeObject;
 
     #[cfg(RustPython)]

pyo3-ffi/src/genericaliasobject.rs

@@ -7,5 +7,6 @@ extern_libpython! {
     pub fn Py_GenericAlias(origin: *mut PyObject, args: *mut PyObject) -> *mut PyObject;
 
     #[cfg(not(RustPython))]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPy_GenericAliasType")]
     pub static mut Py_GenericAliasType: PyTypeObject;
 }

pyo3-ffi/src/listobject.rs

@@ -4,7 +4,7 @@ use core::ffi::c_int;
 
 #[cfg(not(RustPython))]
 extern_libpython! {
-    #[cfg_attr(PyPy, link_name = "PyPyList_Type")]
+    #[cfg_attr(all(PyPy, not(Py_3_12)), link_name = "PyPyList_Type")]
     pub static mut PyList_Type: PyTypeObject;
     pub static mut PyListIter_Type: PyTypeObject;
     pub static mut PyListRevIter_Type: PyTypeObject;