diff --git a/bindgen/__init__.py b/bindgen/__init__.py index eb8da3b..77614e5 100644 --- a/bindgen/__init__.py +++ b/bindgen/__init__.py @@ -5,10 +5,10 @@ from sys import platform from typing import List, Any from math import ceil +from bisect import bisect_left import logzero import toml as toml -import pandas as pd from pyparsing import ( Word, @@ -59,21 +59,28 @@ def read_settings(p): return settings, module_mapping, module_settings -def read_symbols(p): - """Read provided symbols file and return a dataframe +class Symbols: + """Mangled names of the library symbols, looked up by prefix or suffix This information is used later for flagging undefined symbols """ - if int(pd.__version__.split(".")[0]) >= 2: - sym = pd.read_csv( - p, header=None, names=["name"], sep="\\s+", on_bad_lines="skip" - ).dropna() - else: - sym = pd.read_csv( - p, header=None, names=["name"], delim_whitespace=True, error_bad_lines=False - ).dropna() - return sym + def __init__(self, p): + with open(p) as f: + names = sorted({line.split()[0] for line in f if line.strip()}) + self.forward = names + self.reverse = sorted(n[::-1] for n in names) + + @staticmethod + def _has_prefix(names, s): + i = bisect_left(names, s) + return i < len(names) and names[i].startswith(s) + + def has_prefix(self, s): + return self._has_prefix(self.forward, s) + + def has_suffix(self, s): + return self._has_prefix(self.reverse, s[::-1]) def remove_undefined_mangled(m, sym): @@ -90,7 +97,7 @@ def remove_undefined_mangled(m, sym): c.methods = [ el for el in c.methods - if sym.name.str.endswith(el.mangled_name).any() + if sym.has_suffix(el.mangled_name) or el.inline or el.pure_virtual or el.virtual @@ -98,7 +105,7 @@ def remove_undefined_mangled(m, sym): c.methods_byref = [ el for el in c.methods_byref - if sym.name.str.endswith(el.mangled_name).any() + if sym.has_suffix(el.mangled_name) or el.inline or el.pure_virtual or el.virtual @@ -106,7 +113,7 @@ def remove_undefined_mangled(m, sym): c.methods_return_byref = [ el for el in c.methods_return_byref - if sym.name.str.endswith(el.mangled_name).any() + if sym.has_suffix(el.mangled_name) or el.inline or el.pure_virtual or el.virtual @@ -114,17 +121,17 @@ def remove_undefined_mangled(m, sym): c.static_methods = [ el for el in c.static_methods - if sym.name.str.endswith(el.mangled_name).any() or el.inline + if sym.has_suffix(el.mangled_name) or el.inline ] c.static_methods_byref = [ el for el in c.static_methods_byref - if sym.name.str.endswith(el.mangled_name).any() or el.inline + if sym.has_suffix(el.mangled_name) or el.inline ] c.constructors = [ el for el in sorted(c.constructors, key=lambda el: el.full_name) - if sym.name.str.endswith(el.mangled_name).any() + if sym.has_suffix(el.mangled_name) or (el.inline and not el.deleted) or el.pure_virtual or el.virtual @@ -132,20 +139,12 @@ def remove_undefined_mangled(m, sym): ] # exclude functions - m.functions = [ - f - for f in m.functions - if sym.name.str.startswith(f.mangled_name).any() or f.inline - ] + m.functions = [f for f in m.functions if sym.has_prefix(f.mangled_name) or f.inline] # exclude functions per header for h in m.headers: h.functions_unfiltered = h.functions - h.functions = [ - f - for f in h.functions - if sym.name.str.startswith(f.mangled_name).any() or f.inline - ] + h.functions = [f for f in h.functions if sym.has_prefix(f.mangled_name) or f.inline] def is_byref_arg(arg, byref_types): @@ -478,9 +477,7 @@ def transform_modules( platform=None, ) -> tuple[list[ModuleInfo], dict[str, ClassInfo], dict[str, Path], dict[str, Path], dict[str, Path], list[CollectionTypedef]]: - sym = read_symbols( - settings[platform if platform else current_platform()]["symbols"] - ) + sym = Symbols(settings[platform if platform else current_platform()]["symbols"]) # collect collections *before* filtering collections = collect_collections(modules, settings) @@ -865,7 +862,7 @@ def proper_delete_operator(cls): # split collection registration into multiple TUs due to OOM N_coll = len(sorted_collections) - coll_chunk_size = 100 + coll_chunk_size = 25 N_chunks = ceil(N_coll / coll_chunk_size) for i in range(N_chunks): diff --git a/bindgen/header.py b/bindgen/header.py index 49b2e6e..be14d20 100644 --- a/bindgen/header.py +++ b/bindgen/header.py @@ -17,7 +17,6 @@ from .type_parser import parse_type from .translation_unit import parse_tu -from .utils import current_platform EXCLUDE_NS: List[str] = [] @@ -1063,11 +1062,11 @@ def parse(self, path, input_folder, settings, module_name, target_platform): tr_unit = parse_tu( path, input_folder, - prefix=settings[current_platform()]["prefix"], - platform_includes=settings[current_platform()]["includes"], + prefix=settings[target_platform]["prefix"], + platform_includes=settings[target_platform]["includes"], parsing_header=settings["parsing_header"], tu_parsing_header=tu_parsing_header, - platform_parsing_header=settings[current_platform()]["parsing_header"], + platform_parsing_header=settings[target_platform]["parsing_header"], target_platform=target_platform, ) diff --git a/bindgen/translation_unit.py b/bindgen/translation_unit.py index aa60a1b..258ed3a 100755 --- a/bindgen/translation_unit.py +++ b/bindgen/translation_unit.py @@ -1,10 +1,39 @@ import logzero import pybind11 +import os +import tempfile from clang.cindex import TranslationUnit as TU from .utils import get_index, get_includes +_pch = {} +_pch_dirs = [] + + +def preamble_pch(ix, args, text): + """Precompiled header of the preamble shared by every translation unit, built once per process""" + + key = (tuple(args), text) + if key not in _pch: + _pch_dirs.append(tempfile.TemporaryDirectory()) + path = os.path.join(_pch_dirs[-1].name, "preamble.hxx") + with open(path, "w") as f: + f.write(text) + tu = ix.parse( + path, + [a for a in args if a not in ("-x", "c++")] + ["-x", "c++-header"], + options=TU.PARSE_INCOMPLETE, + ) + if tu.diagnostics: + logzero.logger.warning(path) + for d in tu.diagnostics: + logzero.logger.warning(d) + tu.save(path + ".pch") + _pch[key] = path + ".pch" + + return _pch[key] + def parse_tu( path, @@ -26,6 +55,7 @@ def parse_tu( ): """Run a translation unit thorugh clang""" + args = list(args) args.append(f"-I{pybind11.get_include()}") args.append(f"-I{input_folder}") @@ -33,6 +63,8 @@ def parse_tu( args.append("--target=x86_64-pc-windows-msvc") args.append("-fms-compatibility") args.append("-fms-extensions") + elif target_platform == "OSX": + args.append("--target=x86_64-apple-darwin") if prefix: args.append(f"--sysroot={prefix}") @@ -52,11 +84,12 @@ def parse_tu( if src[0] == "\ufeff": src = src[1:] + pch = preamble_pch(ix, args, f"{parsing_header}\n{platform_parsing_header}\n") dummy_code = f"{parsing_header}\n{platform_parsing_header}\n{ tu_parsing_header}\n{src}" tr_unit = ix.parse( "dummy.cxx", - args, + args + ["-include-pch", pch], unsaved_files=[("dummy.cxx", dummy_code)], options=TU.PARSE_INCOMPLETE, ) diff --git a/env.yml b/env.yml index 015916e..05d5a44 100644 --- a/env.yml +++ b/env.yml @@ -19,7 +19,6 @@ dependencies: - click - jinja2 - logzero - - pandas - path.py - pyparsing - schema diff --git a/setup.py b/setup.py index db3bdb2..3fc4cd4 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,6 @@ "path", "clang", "toml", - "pandas", "joblib", "tqdm", "jinja2",