12 min instead of 49 for generation

bindgen/__init__.py

@@ -5,10 +5,10 @@
 from sys import platform
 from typing import List, Any
 from math import ceil
+from bisect import bisect_left
 
 import logzero
 import toml as toml
-import pandas as pd
 
 from pyparsing import (
     Word,
@@ -59,21 +59,28 @@ def read_settings(p):
     return settings, module_mapping, module_settings
 
 
-def read_symbols(p):
-    """Read provided symbols file and return a dataframe
+class Symbols:
+    """Mangled names of the library symbols, looked up by prefix or suffix
 
     This information is used later for flagging undefined symbols
     """
 
-    if int(pd.__version__.split(".")[0]) >= 2:
-        sym = pd.read_csv(
-            p, header=None, names=["name"], sep="\\s+", on_bad_lines="skip"
-        ).dropna()
-    else:
-        sym = pd.read_csv(
-            p, header=None, names=["name"], delim_whitespace=True, error_bad_lines=False
-        ).dropna()
-    return sym
+    def __init__(self, p):
+        with open(p) as f:
+            names = sorted({line.split()[0] for line in f if line.strip()})
+        self.forward = names
+        self.reverse = sorted(n[::-1] for n in names)
+
+    @staticmethod
+    def _has_prefix(names, s):
+        i = bisect_left(names, s)
+        return i < len(names) and names[i].startswith(s)
+
+    def has_prefix(self, s):
+        return self._has_prefix(self.forward, s)
+
+    def has_suffix(self, s):
+        return self._has_prefix(self.reverse, s[::-1])
 
 
 def remove_undefined_mangled(m, sym):
@@ -90,62 +97,54 @@ def remove_undefined_mangled(m, sym):
         c.methods = [
             el
             for el in c.methods
-            if sym.name.str.endswith(el.mangled_name).any()
+            if sym.has_suffix(el.mangled_name)
             or el.inline
             or el.pure_virtual
             or el.virtual
         ]
         c.methods_byref = [
             el
             for el in c.methods_byref
-            if sym.name.str.endswith(el.mangled_name).any()
+            if sym.has_suffix(el.mangled_name)
             or el.inline
             or el.pure_virtual
             or el.virtual
         ]
         c.methods_return_byref = [
             el
             for el in c.methods_return_byref
-            if sym.name.str.endswith(el.mangled_name).any()
+            if sym.has_suffix(el.mangled_name)
             or el.inline
             or el.pure_virtual
             or el.virtual
         ]
         c.static_methods = [
             el
             for el in c.static_methods
-            if sym.name.str.endswith(el.mangled_name).any() or el.inline
+            if sym.has_suffix(el.mangled_name) or el.inline
         ]
         c.static_methods_byref = [
             el
             for el in c.static_methods_byref
-            if sym.name.str.endswith(el.mangled_name).any() or el.inline
+            if sym.has_suffix(el.mangled_name) or el.inline
         ]
         c.constructors = [
             el
             for el in sorted(c.constructors, key=lambda el: el.full_name)
-            if sym.name.str.endswith(el.mangled_name).any()
+            if sym.has_suffix(el.mangled_name)
             or (el.inline and not el.deleted)
             or el.pure_virtual
             or el.virtual
             or el.default
         ]
 
     # exclude functions
-    m.functions = [
-        f
-        for f in m.functions
-        if sym.name.str.startswith(f.mangled_name).any() or f.inline
-    ]
+    m.functions = [f for f in m.functions if sym.has_prefix(f.mangled_name) or f.inline]
 
     # exclude functions per header
     for h in m.headers:
         h.functions_unfiltered = h.functions
-        h.functions = [
-            f
-            for f in h.functions
-            if sym.name.str.startswith(f.mangled_name).any() or f.inline
-        ]
+        h.functions = [f for f in h.functions if sym.has_prefix(f.mangled_name) or f.inline]
 
 
 def is_byref_arg(arg, byref_types):
@@ -478,9 +477,7 @@ def transform_modules(
     platform=None,
 ) -> tuple[list[ModuleInfo], dict[str, ClassInfo], dict[str, Path], dict[str, Path], dict[str, Path], list[CollectionTypedef]]:
 
-    sym = read_symbols(
-        settings[platform if platform else current_platform()]["symbols"]
-    )
+    sym = Symbols(settings[platform if platform else current_platform()]["symbols"])
 
     # collect collections *before* filtering
     collections = collect_collections(modules, settings)
@@ -865,7 +862,7 @@ def proper_delete_operator(cls):
 
         # split collection registration into multiple TUs due to OOM
         N_coll = len(sorted_collections)
-        coll_chunk_size = 100
+        coll_chunk_size = 25
         N_chunks = ceil(N_coll / coll_chunk_size)
 
         for i in range(N_chunks):

bindgen/header.py

@@ -17,7 +17,6 @@
 
 from .type_parser import parse_type
 from .translation_unit import parse_tu
-from .utils import current_platform
 
 EXCLUDE_NS: List[str] = []
 
@@ -1063,11 +1062,11 @@ def parse(self, path, input_folder, settings, module_name, target_platform):
         tr_unit = parse_tu(
             path,
             input_folder,
-            prefix=settings[current_platform()]["prefix"],
-            platform_includes=settings[current_platform()]["includes"],
+            prefix=settings[target_platform]["prefix"],
+            platform_includes=settings[target_platform]["includes"],
             parsing_header=settings["parsing_header"],
             tu_parsing_header=tu_parsing_header,
-            platform_parsing_header=settings[current_platform()]["parsing_header"],
+            platform_parsing_header=settings[target_platform]["parsing_header"],
             target_platform=target_platform,
         )
 

bindgen/translation_unit.py

@@ -1,10 +1,39 @@
 import logzero
 import pybind11
+import os
+import tempfile
 
 from clang.cindex import TranslationUnit as TU
 
 from .utils import get_index, get_includes
 
+_pch = {}
+_pch_dirs = []
+
+
+def preamble_pch(ix, args, text):
+    """Precompiled header of the preamble shared by every translation unit, built once per process"""
+
+    key = (tuple(args), text)
+    if key not in _pch:
+        _pch_dirs.append(tempfile.TemporaryDirectory())
+        path = os.path.join(_pch_dirs[-1].name, "preamble.hxx")
+        with open(path, "w") as f:
+            f.write(text)
+        tu = ix.parse(
+            path,
+            [a for a in args if a not in ("-x", "c++")] + ["-x", "c++-header"],
+            options=TU.PARSE_INCOMPLETE,
+        )
+        if tu.diagnostics:
+            logzero.logger.warning(path)
+        for d in tu.diagnostics:
+            logzero.logger.warning(d)
+        tu.save(path + ".pch")
+        _pch[key] = path + ".pch"
+
+    return _pch[key]
+
 
 def parse_tu(
     path,
@@ -26,13 +55,16 @@ def parse_tu(
 ):
     """Run a translation unit thorugh clang"""
 
+    args = list(args)
     args.append(f"-I{pybind11.get_include()}")
     args.append(f"-I{input_folder}")
 
     if target_platform == "Windows":
         args.append("--target=x86_64-pc-windows-msvc")
         args.append("-fms-compatibility")
         args.append("-fms-extensions")
+    elif target_platform == "OSX":
+        args.append("--target=x86_64-apple-darwin")
 
     if prefix:
         args.append(f"--sysroot={prefix}")
@@ -52,11 +84,12 @@ def parse_tu(
     if src[0] == "\ufeff":
         src = src[1:]
 
+    pch = preamble_pch(ix, args, f"{parsing_header}\n{platform_parsing_header}\n")
     dummy_code = f"{parsing_header}\n{platform_parsing_header}\n{
             tu_parsing_header}\n{src}"
     tr_unit = ix.parse(
         "dummy.cxx",
-        args,
+        args + ["-include-pch", pch],
         unsaved_files=[("dummy.cxx", dummy_code)],
         options=TU.PARSE_INCOMPLETE,
     )

env.yml

@@ -19,7 +19,6 @@ dependencies:
     - click
     - jinja2
     - logzero
-    - pandas
     - path.py
     - pyparsing
     - schema

setup.py

@@ -12,7 +12,6 @@
         "path",
         "clang",
         "toml",
-        "pandas",
         "joblib",
         "tqdm",
         "jinja2",