[mypyc] Move API table definitions to .c files

mypyc/build.py

@@ -53,11 +53,22 @@ class ModDesc(NamedTuple):
 
 
 LIBRT_MODULES = [
-    ModDesc("librt.internal", ["internal/librt_internal.c"], [], ["internal"]),
-    ModDesc("librt.strings", ["strings/librt_strings.c"], [], ["strings"]),
+    ModDesc(
+        "librt.internal",
+        ["internal/librt_internal_static.c", "internal/librt_internal.c"],
+        [],
+        ["internal"],
+    ),
+    ModDesc(
+        "librt.strings",
+        ["strings/librt_strings_static.c", "strings/librt_strings.c"],
+        [],
+        ["strings"],
+    ),
     ModDesc(
         "librt.base64",
         [
+            "base64/librt_base64_static.c",
             "base64/librt_base64.c",
             "base64/lib.c",
             "base64/codec_choose.c",
@@ -107,6 +118,7 @@ class ModDesc(NamedTuple):
     ModDesc(
         "librt.vecs",
         [
+            "vecs/librt_vecs_static.c",
             "vecs/librt_vecs.c",
             "vecs/vec_i64.c",
             "vecs/vec_i32.c",
@@ -120,7 +132,9 @@ class ModDesc(NamedTuple):
         ["vecs/librt_vecs.h", "vecs/vec_template.c"],
         ["vecs"],
     ),
-    ModDesc("librt.time", ["time/librt_time.c"], ["time/librt_time.h"], []),
+    ModDesc(
+        "librt.time", ["time/librt_time_static.c", "time/librt_time.c"], ["time/librt_time.h"], []
+    ),
 ]
 
 try:
@@ -357,6 +371,7 @@ def build_using_shared_lib(
     deps: list[str],
     build_dir: str,
     extra_compile_args: list[str],
+    extra_include_dirs: list[str],
 ) -> list[Extension]:
     """Produce the list of extension modules when a shared library is needed.
 
@@ -373,7 +388,7 @@ def build_using_shared_lib(
         get_extension()(
             shared_lib_name(group_name),
             sources=cfiles,
-            include_dirs=[include_dir(), build_dir],
+            include_dirs=[include_dir(), build_dir] + extra_include_dirs,
             depends=deps,
             extra_compile_args=extra_compile_args,
         )
@@ -399,7 +414,10 @@ def build_using_shared_lib(
 
 
 def build_single_module(
-    sources: list[BuildSource], cfiles: list[str], extra_compile_args: list[str]
+    sources: list[BuildSource],
+    cfiles: list[str],
+    extra_compile_args: list[str],
+    extra_include_dirs: list[str],
 ) -> list[Extension]:
     """Produce the list of extension modules for a standalone extension.
 
@@ -409,7 +427,7 @@ def build_single_module(
         get_extension()(
             sources[0].module,
             sources=cfiles,
-            include_dirs=[include_dir()],
+            include_dirs=[include_dir()] + extra_include_dirs,
             extra_compile_args=extra_compile_args,
         )
     ]
@@ -513,7 +531,9 @@ def mypyc_build(
     *,
     separate: bool | list[tuple[list[str], str | None]] = False,
     only_compile_paths: Iterable[str] | None = None,
-    skip_cgen_input: tuple[list[list[tuple[str, str]]], list[str]] | None = None,
+    skip_cgen_input: (
+        tuple[list[list[tuple[str, str]]], list[tuple[str, bool, list[str]]]] | None
+    ) = None,
     always_use_shared_lib: bool = False,
 ) -> tuple[emitmodule.Groups, list[tuple[list[str], list[str]]], list[SourceDep]]:
     """Do the front and middle end of mypyc building, producing and writing out C source."""
@@ -547,7 +567,10 @@ def mypyc_build(
         write_file(os.path.join(compiler_options.target_dir, "ops.txt"), ops_text)
     else:
         group_cfiles = skip_cgen_input[0]
-        source_deps = [SourceDep(d) for d in skip_cgen_input[1]]
+        source_deps = [
+            SourceDep(path, has_header=hdr, include_dirs=dirs)
+            for (path, hdr, dirs) in skip_cgen_input[1]
+        ]
 
     # Write out the generated C and collect the files for each group
     # Should this be here??
@@ -664,7 +687,9 @@ def mypycify(
     strip_asserts: bool = False,
     multi_file: bool = False,
     separate: bool | list[tuple[list[str], str | None]] = False,
-    skip_cgen_input: tuple[list[list[tuple[str, str]]], list[str]] | None = None,
+    skip_cgen_input: (
+        tuple[list[list[tuple[str, str]]], list[tuple[str, bool, list[str]]]] | None
+    ) = None,
     target_dir: str | None = None,
     include_runtime_files: bool | None = None,
     strict_dunder_typing: bool = False,
@@ -781,12 +806,15 @@ def mypycify(
     # runtime library in. Otherwise it just gets #included to save on
     # compiler invocations.
     shared_cfilenames = []
+    include_dirs = set()
     if not compiler_options.include_runtime_files:
         # Collect all files to copy: runtime files + conditional source files
         files_to_copy = list(RUNTIME_C_FILES)
         for source_dep in source_deps:
             files_to_copy.append(source_dep.path)
-            files_to_copy.append(source_dep.get_header())
+            if header := source_dep.get_header():
+                files_to_copy.append(header)
+            include_dirs.update(source_dep.include_dirs)
 
         # Copy all files
         for name in files_to_copy:
@@ -797,6 +825,7 @@ def mypycify(
                 shared_cfilenames.append(rt_file)
 
     extensions = []
+    extra_include_dirs = [os.path.join(include_dir(), dir) for dir in include_dirs]
     for (group_sources, lib_name), (cfilenames, deps) in zip(groups, group_cfilenames):
         if lib_name:
             extensions.extend(
@@ -807,11 +836,14 @@ def mypycify(
                     deps,
                     build_dir,
                     cflags,
+                    extra_include_dirs,
                 )
             )
         else:
             extensions.extend(
-                build_single_module(group_sources, cfilenames + shared_cfilenames, cflags)
+                build_single_module(
+                    group_sources, cfilenames + shared_cfilenames, cflags, extra_include_dirs
+                )
             )
 
     if install_librt:

mypyc/codegen/emitmodule.py

@@ -443,6 +443,8 @@ def collect_source_dependencies(modules: dict[str, ModuleIR]) -> set[SourceDep]:
         for dep in module.dependencies:
             if isinstance(dep, SourceDep):
                 source_deps.add(dep)
+            else:
+                source_deps.add(dep.static_data_dep())
     return source_deps
 
 
@@ -585,6 +587,8 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]:
             source_deps = collect_source_dependencies(self.modules)
             for source_dep in sorted(source_deps, key=lambda d: d.path):
                 base_emitter.emit_line(f'#include "{source_dep.path}"')
+            if self.compiler_options.depends_on_librt_internal:
+                base_emitter.emit_line("#include <internal/librt_internal_static.c>")
         base_emitter.emit_line(f'#include "__native{self.short_group_suffix}.h"')
         base_emitter.emit_line(f'#include "__native_internal{self.short_group_suffix}.h"')
         emitter = base_emitter
@@ -634,26 +638,29 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]:
         ext_declarations.emit_line(f"#define MYPYC_NATIVE{self.group_suffix}_H")
         ext_declarations.emit_line("#include <Python.h>")
         ext_declarations.emit_line("#include <CPy.h>")
+
+        declarations = Emitter(self.context)
+        declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H")
+        declarations.emit_line(f"#define MYPYC_LIBRT_INTERNAL{self.group_suffix}_H")
+        declarations.emit_line("#include <Python.h>")
+        declarations.emit_line("#include <CPy.h>")
+
         if self.compiler_options.depends_on_librt_internal:
-            ext_declarations.emit_line("#include <internal/librt_internal.h>")
+            declarations.emit_line("#include <internal/librt_internal.h>")
         if any(LIBRT_BASE64 in mod.dependencies for mod in self.modules.values()):
-            ext_declarations.emit_line("#include <base64/librt_base64.h>")
+            declarations.emit_line("#include <base64/librt_base64.h>")
         if any(LIBRT_STRINGS in mod.dependencies for mod in self.modules.values()):
-            ext_declarations.emit_line("#include <strings/librt_strings.h>")
+            declarations.emit_line("#include <strings/librt_strings.h>")
         if any(LIBRT_TIME in mod.dependencies for mod in self.modules.values()):
-            ext_declarations.emit_line("#include <time/librt_time.h>")
+            declarations.emit_line("#include <time/librt_time.h>")
         if any(LIBRT_VECS in mod.dependencies for mod in self.modules.values()):
-            ext_declarations.emit_line("#include <vecs/librt_vecs.h>")
+            declarations.emit_line("#include <vecs/librt_vecs.h>")
         # Include headers for conditional source files
         source_deps = collect_source_dependencies(self.modules)
         for source_dep in sorted(source_deps, key=lambda d: d.path):
-            ext_declarations.emit_line(f'#include "{source_dep.get_header()}"')
+            if header := source_dep.get_header():
+                declarations.emit_line(f'#include "{header}"')
 
-        declarations = Emitter(self.context)
-        declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H")
-        declarations.emit_line(f"#define MYPYC_LIBRT_INTERNAL{self.group_suffix}_H")
-        declarations.emit_line("#include <Python.h>")
-        declarations.emit_line("#include <CPy.h>")
         declarations.emit_line(f'#include "__native{self.short_group_suffix}.h"')
         declarations.emit_line()
         declarations.emit_line("int CPyGlobalsInit(void);")

mypyc/ir/deps.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import Final
 
 
@@ -17,17 +19,29 @@ def __eq__(self, other: object) -> bool:
     def __hash__(self) -> int:
         return hash(("Capsule", self.name))
 
+    def static_data_dep(self) -> SourceDep:
+        module = self.name.split(".")[-1]
+        return SourceDep(
+            f"{module}/librt_{module}_static.c", has_header=False, include_dirs=[module]
+        )
+
 
 class SourceDep:
     """Defines a C source file that a primitive may require.
 
-    Each source file must also have a corresponding .h file (replace .c with .h)
+    Each source file may also have a corresponding .h file (replace .c with .h)
     that gets implicitly #included if the source is used.
+    include_dirs are passed to the C compiler when the file is compiled as a
+    shared library separate from the C extension.
     """
 
-    def __init__(self, path: str) -> None:
+    def __init__(
+        self, path: str, *, has_header: bool = True, include_dirs: list[str] | None = None
+    ) -> None:
         # Relative path from mypyc/lib-rt, e.g. 'bytes_extra_ops.c'
         self.path: Final = path
+        self.has_header: Final = has_header
+        self.include_dirs: Final = include_dirs or []
 
     def __repr__(self) -> str:
         return f"SourceDep(path={self.path!r})"
@@ -38,9 +52,9 @@ def __eq__(self, other: object) -> bool:
     def __hash__(self) -> int:
         return hash(("SourceDep", self.path))
 
-    def get_header(self) -> str:
+    def get_header(self) -> str | None:
         """Get the header file path by replacing .c with .h"""
-        return self.path.replace(".c", ".h")
+        return self.path.replace(".c", ".h") if self.has_header else None
 
 
 Dependency = Capsule | SourceDep

mypyc/lib-rt/base64/librt_base64.h

@@ -7,7 +7,7 @@
 #define LIBRT_BASE64_API_VERSION 2
 #define LIBRT_BASE64_API_LEN 4
 
-static void *LibRTBase64_API[LIBRT_BASE64_API_LEN];
+extern void *LibRTBase64_API[LIBRT_BASE64_API_LEN];
 
 #define LibRTBase64_ABIVersion (*(int (*)(void)) LibRTBase64_API[0])
 #define LibRTBase64_APIVersion (*(int (*)(void)) LibRTBase64_API[1])

mypyc/lib-rt/base64/librt_base64_static.c

@@ -0,0 +1,3 @@
+#include "librt_base64.h"
+
+void *LibRTBase64_API[LIBRT_BASE64_API_LEN] = {0};

mypyc/lib-rt/internal/librt_internal.h

@@ -2,6 +2,7 @@
 #define LIBRT_INTERNAL_H
 
 #include <Python.h>
+#include <stdbool.h>
 
 // ABI version -- only an exact match is compatible. This will only be changed in
 // very exceptional cases (likely never) due to strict backward compatibility
@@ -44,7 +45,7 @@ static int NativeInternal_API_Version(void);
 
 #else
 
-static void *NativeInternal_API[LIBRT_INTERNAL_API_LEN];
+extern void *NativeInternal_API[LIBRT_INTERNAL_API_LEN];
 
 #define ReadBuffer_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[0])
 #define WriteBuffer_internal (*(PyObject* (*)(void)) NativeInternal_API[1])

mypyc/lib-rt/internal/librt_internal_static.c

@@ -0,0 +1,3 @@
+#include "librt_internal.h"
+
+void *NativeInternal_API[LIBRT_INTERNAL_API_LEN] = {0};

mypyc/lib-rt/strings/librt_strings.h

@@ -30,7 +30,7 @@ import_librt_strings(void)
 // LIBRT_STRINGS_API_VERSION.
 #define LIBRT_STRINGS_API_LEN 14
 
-static void *LibRTStrings_API[LIBRT_STRINGS_API_LEN];
+extern void *LibRTStrings_API[LIBRT_STRINGS_API_LEN];
 
 typedef struct {
     PyObject_HEAD

mypyc/lib-rt/strings/librt_strings_static.c

@@ -0,0 +1,7 @@
+#ifdef MYPYC_EXPERIMENTAL
+
+#include "librt_strings.h"
+
+void *LibRTStrings_API[LIBRT_STRINGS_API_LEN] = {0};
+
+#endif // MYPYC_EXPERIMENTAL

mypyc/lib-rt/time/librt_time.h

@@ -18,7 +18,7 @@ import_librt_time(void)
 #define LIBRT_TIME_API_VERSION 1
 #define LIBRT_TIME_API_LEN 3
 
-static void *LibRTTime_API[LIBRT_TIME_API_LEN];
+extern void *LibRTTime_API[LIBRT_TIME_API_LEN];
 
 #define LibRTTime_ABIVersion (*(int (*)(void)) LibRTTime_API[0])
 #define LibRTTime_APIVersion (*(int (*)(void)) LibRTTime_API[1])

mypyc/lib-rt/time/librt_time_static.c

@@ -0,0 +1,7 @@
+#ifdef MYPYC_EXPERIMENTAL
+
+#include "librt_time.h"
+
+void *LibRTTime_API[LIBRT_TIME_API_LEN] = {0};
+
+#endif // MYPYC_EXPERIMENTAL

mypyc/lib-rt/vecs/librt_vecs.h

@@ -847,15 +847,15 @@ PyObject *Vec_GenericPopWrapper(Py_ssize_t *len, PyObject **items, PyObject *arg
 PyObject *Vec_GenericPop(Py_ssize_t *len, PyObject **items, Py_ssize_t index);
 
 // Global API pointers initialized by import_librt_vecs()
-static VecCapsule *VecApi;
-static VecI64API VecI64Api;
-static VecI32API VecI32Api;
-static VecI16API VecI16Api;
-static VecU8API VecU8Api;
-static VecFloatAPI VecFloatApi;
-static VecBoolAPI VecBoolApi;
-static VecTAPI VecTApi;
-static VecNestedAPI VecNestedApi;
+extern VecCapsule *VecApi;
+extern VecI64API VecI64Api;
+extern VecI32API VecI32Api;
+extern VecI16API VecI16Api;
+extern VecU8API VecU8Api;
+extern VecFloatAPI VecFloatApi;
+extern VecBoolAPI VecBoolApi;
+extern VecTAPI VecTApi;
+extern VecNestedAPI VecNestedApi;
 
 static int
 import_librt_vecs(void)

mypyc/lib-rt/vecs/librt_vecs_static.c

@@ -0,0 +1,15 @@
+#ifdef MYPYC_EXPERIMENTAL
+
+#include "librt_vecs.h"
+
+VecCapsule *VecApi = NULL;
+VecI64API VecI64Api = {0};
+VecI32API VecI32Api = {0};
+VecI16API VecI16Api = {0};
+VecU8API VecU8Api = {0};
+VecFloatAPI VecFloatApi = {0};
+VecBoolAPI VecBoolApi = {0};
+VecTAPI VecTApi = {0};
+VecNestedAPI VecNestedApi = {0};
+
+#endif // MYPYC_EXPERIMENTAL