From d109f0576bb8d06eeb053da04011907bfec39f49 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Fri, 23 Oct 2026 13:40:10 +0200 Subject: [PATCH 1/2] [cppinterop] Update vendored CppInterOp to ROOT-cppjit-2026-10-21 --- interpreter/CppInterOp/.readthedocs.yaml | 10 +- interpreter/CppInterOp/CMakeLists.txt | 21 +- interpreter/CppInterOp/README.md | 2 +- interpreter/CppInterOp/cppinterop-version.tag | 2 +- interpreter/CppInterOp/docs/conf.py | 4 +- .../CppInterOp/etc/clang23-valgrind.supp | 467 ++++++++++++++ .../CppInterOp/lib/CppInterOp/CMakeLists.txt | 26 +- .../CppInterOp/lib/CppInterOp/Compatibility.h | 76 ++- .../lib/CppInterOp/CompatibilityGLIBC.h | 4 - .../CppInterOp/lib/CppInterOp/CppInterOp.cpp | 429 ++++++++----- .../CppInterOp/lib/CppInterOp/CppInterOp.td | 56 +- .../lib/CppInterOp/CppInterOpAPI.td | 1 + .../DynamicLibraryManagerSymbol.cpp | 1 + .../CppInterOp/lib/CppInterOp/exports.ld | 8 +- .../CppInterOp/APINotes/TestHeader.apinotes | 25 +- .../CppInterOp/APINotes/TestHeader.h | 15 + .../unittests/CppInterOp/CAPITest.cpp | 32 + .../unittests/CppInterOp/CAPITestC.c | 1 + .../unittests/CppInterOp/CMakeLists.txt | 9 + .../unittests/CppInterOp/DiagnosticTest.cpp | 34 +- .../CppInterOp/DynamicLibraryManagerTest.cpp | 8 - .../CppInterOp/FunctionReflectionTest.cpp | 584 +++++++++++++----- .../unittests/CppInterOp/InterpreterTest.cpp | 6 +- .../CppInterOp/ScopeReflectionTest.cpp | 143 ++++- .../CppInterOp/TypeReflectionTest.cpp | 21 +- .../CppInterOp/unittests/CppInterOp/Utils.cpp | 14 + .../CppInterOp/unittests/CppInterOp/Utils.h | 22 + .../CppInterOp/VTableOverlayCrossTUBench.cpp | 7 +- .../CppInterOp/VTableOverlayTest.cpp | 43 +- .../CppInterOp/VariableReflectionTest.cpp | 61 +- 30 files changed, 1692 insertions(+), 440 deletions(-) create mode 100644 interpreter/CppInterOp/etc/clang23-valgrind.supp diff --git a/interpreter/CppInterOp/.readthedocs.yaml b/interpreter/CppInterOp/.readthedocs.yaml index 8d6637fb7dade..a3163c4f41831 100644 --- a/interpreter/CppInterOp/.readthedocs.yaml +++ b/interpreter/CppInterOp/.readthedocs.yaml @@ -5,13 +5,13 @@ sphinx: builder: html build: - os: "ubuntu-24.04" + os: "ubuntu-26.04" tools: python: "3.11" apt_packages: - - clang-20 + - clang-21 - cmake - doxygen - - libclang-20-dev - - llvm-20-dev - - llvm-20-tools + - libclang-21-dev + - llvm-21-dev + - llvm-21-tools diff --git a/interpreter/CppInterOp/CMakeLists.txt b/interpreter/CppInterOp/CMakeLists.txt index 8bbea79346bb9..db7e86757ee78 100644 --- a/interpreter/CppInterOp/CMakeLists.txt +++ b/interpreter/CppInterOp/CMakeLists.txt @@ -27,7 +27,8 @@ option(CPPINTEROP_SHARED_LIBRARY_VERSIONING set(CPPINTEROP_BAKED_INTERPRETER_ARGS "" CACHE STRING "Space-separated arguments passed at interpreter creation (no quoting: arguments containing spaces are not supported)") if(EMSCRIPTEN) - set(CPPINTEROP_EXTRA_WASM_FLAGS "-fwasm-exceptions" CACHE STRING "Extra flags for wasm") + set(CPPINTEROP_LIBRARY_WASM_FLAGS "-Oz -flto" CACHE STRING "Extra flags for build of CppInterOp library") + set(CPPINTEROP_EXTRA_WASM_FLAGS "-fwasm-exceptions" CACHE STRING "Extra flags for wasm (all targets)") endif() if (CPPINTEROP_USE_CLING AND CPPINTEROP_USE_REPL) @@ -104,15 +105,15 @@ if( CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR ) include(GNUInstallDirs) ## Define supported version of clang and llvm - set(CLANG_MIN_SUPPORTED 20.0) - set(CLANG_MAX_SUPPORTED "22.1.x") - set(CLANG_VERSION_UPPER_BOUND 23.0.0) - set(LLD_MIN_SUPPORTED 20.0) - set(LLD_MAX_SUPPORTED "22.1.x") - set(LLD_VERSION_UPPER_BOUND 23.0.0) - set(LLVM_MIN_SUPPORTED 20.0) - set(LLVM_MAX_SUPPORTED "22.1.x") - set(LLVM_VERSION_UPPER_BOUND 23.0.0) + set(CLANG_MIN_SUPPORTED 21.0) + set(CLANG_MAX_SUPPORTED "23.1.x") + set(CLANG_VERSION_UPPER_BOUND 24.0.0) + set(LLD_MIN_SUPPORTED 21.0) + set(LLD_MAX_SUPPORTED "23.1.x") + set(LLD_VERSION_UPPER_BOUND 24.0.0) + set(LLVM_MIN_SUPPORTED 21.0) + set(LLVM_MAX_SUPPORTED "23.1.x") + set(LLVM_VERSION_UPPER_BOUND 24.0.0) ## Set Cmake packages search order diff --git a/interpreter/CppInterOp/README.md b/interpreter/CppInterOp/README.md index 70f23da6cce95..306c709e7ed22 100644 --- a/interpreter/CppInterOp/README.md +++ b/interpreter/CppInterOp/README.md @@ -621,7 +621,7 @@ against an upstream LLVM 22 release (via `compiler-research/ci-workflows`'s may still lag the CMake build. The Bazel build consumes a local LLVM/Clang tree rather than building one (LLVM -20–22 are supported, matching the CMake version range). Point it at one with the +21–23 are supported, matching the CMake version range). Point it at one with the `LLVM_DIR` environment variable -- either an LLVM *build* tree or a release *install* tree; the build adapts to whichever (static-archive or shared `libLLVM.so`) layout the tree ships: diff --git a/interpreter/CppInterOp/cppinterop-version.tag b/interpreter/CppInterOp/cppinterop-version.tag index ee26a7af2c381..33412e6eb6090 100644 --- a/interpreter/CppInterOp/cppinterop-version.tag +++ b/interpreter/CppInterOp/cppinterop-version.tag @@ -1 +1 @@ -d23b04a52cf7f7aab7dc16d1c38b628085469c0a +acfde2400bedb9a243106c7ec868d3b2e29a8d03 diff --git a/interpreter/CppInterOp/docs/conf.py b/interpreter/CppInterOp/docs/conf.py index dcf03a7b8d497..2669bd29b3044 100644 --- a/interpreter/CppInterOp/docs/conf.py +++ b/interpreter/CppInterOp/docs/conf.py @@ -50,8 +50,8 @@ html_extra_path = [CPPINTEROP_ROOT + '/build/docs/'] import subprocess -command = 'mkdir {0}/build; cd {0}/build; cmake ../ -DClang_DIR=/usr/lib/llvm-20/build/lib/cmake/clang\ - -DLLVM_DIR=/usr/lib/llvm-20/build/lib/cmake/llvm -DCPPINTEROP_ENABLE_DOXYGEN=ON\ +command = 'mkdir {0}/build; cd {0}/build; cmake ../ -DClang_DIR=/usr/lib/llvm-21/build/lib/cmake/clang\ + -DLLVM_DIR=/usr/lib/llvm-21/build/lib/cmake/llvm -DCPPINTEROP_ENABLE_DOXYGEN=ON\ -DCPPINTEROP_INCLUDE_DOCS=ON'.format(CPPINTEROP_ROOT) subprocess.call(command, shell=True) # Generate .inc files so doxygen can find the API declarations in CppInterOp.h. diff --git a/interpreter/CppInterOp/etc/clang23-valgrind.supp b/interpreter/CppInterOp/etc/clang23-valgrind.supp new file mode 100644 index 0000000000000..75cd0158ee6a0 --- /dev/null +++ b/interpreter/CppInterOp/etc/clang23-valgrind.supp @@ -0,0 +1,467 @@ +# Upstream LLVM write(buf) uninit-bytes via the EPC transport, still +# present in release/23.x. No llvm/llvm-project issue filed yet; if you +# touch this file, please file one and add a reference here. +{ + Clang 23 Out of Process: uninitialised bytes in FDSimpleRemoteEPCTransport::sendMessage + Memcheck:Param + write(buf) + fun:__libc_write + fun:write + fun:_ZN4llvm3orc26FDSimpleRemoteEPCTransport11sendMessageENS0_21SimpleRemoteEPCOpcodeEmNS0_12ExecutorAddrENS_8ArrayRefIcEE + ... +} + +# llvm/llvm-project#194147: clang-23 still reads uninitialised bytes in many +# AST/Sema/CodeGen paths (regression vs clang-20). Was x86_64-only at +# first; aarch64 caught up after a release/22.x advance and now trips +# its own set (TreeTransform, SemaCUDA, AsmPrinter, ValueEnumerator). +# The blocks below started as a copy of clang22-valgrind.supp. +# Drop the whole #194147 family of blocks once the upstream issue lands. + +{ + llvm/llvm-project#194147: InitListExpr::isTransparent + Memcheck:Cond + fun:_ZNK5clang12InitListExpr13isTransparentEv + ... +} +{ + llvm/llvm-project#194147: InitListExpr::getInit + Memcheck:Cond + fun:_ZN5clang12InitListExpr7getInitEj + ... +} +{ + llvm/llvm-project#194147: InitListExpr::getInit const + Memcheck:Cond + fun:_ZNK5clang12InitListExpr7getInitEj + ... +} +{ + llvm/llvm-project#194147: InitListExpr::setInit + Memcheck:Cond + fun:_ZN5clang12InitListExpr7setInitEjPNS_4ExprE + ... +} +{ + llvm/llvm-project#194147: InitListExpr::resizeInits + Memcheck:Cond + fun:_ZN5clang12InitListExpr11resizeInitsERKNS_10ASTContextEj + ... +} +{ + llvm/llvm-project#194147: InitListExpr::resizeInits via memset + Memcheck:Cond + fun:memset + fun:_ZN5clang12InitListExpr11resizeInitsERKNS_10ASTContextEj + ... +} + +{ + llvm/llvm-project#194147: Sema::CheckForIntOverflow Cond + Memcheck:Cond + fun:_ZN5clang4Sema19CheckForIntOverflowEPKNS_4ExprE + ... +} +{ + llvm/llvm-project#194147: Sema::CheckForIntOverflow Value8 + Memcheck:Value8 + fun:_ZN5clang4Sema19CheckForIntOverflowEPKNS_4ExprE + ... +} +{ + llvm/llvm-project#194147: SequenceChecker::SequenceExpressionsInOrder + Memcheck:Cond + fun:*SequenceChecker*SequenceExpressionsInOrder* + ... +} + +{ + llvm/llvm-project#194147: Stmt::children via AnalyzeImplicitConversions + Memcheck:Cond + fun:_ZN5clang4Stmt8childrenEv + fun:*AnalyzeImplicitConversions* + ... +} +{ + llvm/llvm-project#194147: AnalyzeImplicitConversions leaf + Memcheck:Cond + fun:*AnalyzeImplicitConversions* + ... +} +{ + llvm/llvm-project#194147: CFGBuilder::Visit Cond + Memcheck:Cond + fun:*CFGBuilder*Visit* + ... +} +{ + llvm/llvm-project#194147: CFGBuilder::Visit Value8 + Memcheck:Value8 + fun:*CFGBuilder*Visit* + ... +} + +{ + llvm/llvm-project#194147: RecordExprEvaluator::VisitCXXParenListOrInitListExpr + Memcheck:Cond + fun:*RecordExprEvaluator*VisitCXXParenListOrInitListExpr* + ... +} +{ + llvm/llvm-project#194147: AggExprEmitter init-list visits + Memcheck:Cond + fun:*AggExprEmitter*Visit*InitListExpr* + ... +} + +{ + llvm/llvm-project#194147: InitListChecker::FillInEmptyInitForField + Memcheck:Cond + fun:*InitListChecker*FillInEmptyInitForField* + ... +} +{ + llvm/llvm-project#194147: InitListChecker::FillInEmptyInitializations + Memcheck:Cond + fun:*InitListChecker*FillInEmptyInitializations* + ... +} + +# llvm/llvm-project#194147 (continued): symbols newly tripped after +# release/22.x advanced and the llvm-release recipe republished. The +# FunctionProtoType ctor is the dominant top frame on x86_64; the +# CodeGen + EHScope entries cover the lower-frequency paths. +# +# Earlier iterations of this file carried ~17 more blocks targeting +# symbols extracted from "Uninitialised value was created by" +# allocation-origin frames; valgrind's matcher only considers the +# read-site top frame, so every one of those blocks was dead. +# Removed. (See the matching comment in clang21-valgrind.supp for +# why the over-broad `*FunctionProtoType*` is the only pattern that +# matches the constructor in valgrind's wildcard matcher.) +{ + llvm/llvm-project#194147: FunctionProtoType (any method, see comment) + Memcheck:Cond + fun:*FunctionProtoType* + ... +} +{ + llvm/llvm-project#194147: CodeGenFunction::EmitCall + Memcheck:Cond + fun:*CodeGenFunction*EmitCall* + ... +} +{ + llvm/llvm-project#194147: EHScopeStack::requiresLandingPad + Memcheck:Cond + fun:*EHScopeStack*requiresLandingPad* + ... +} +{ + # Top frame for every unsuppressed error in the llvm22-vg + # x86_64 row after FunctionProtoType started being silenced -- + # 21->173 errors all surfaced here, across 3->5 contexts + # depending on which CppInterOpTests' OutOfProcessJIT subtests + # ran. Origin frame is llvm::Module::setDataLayout (stack + # allocation) but valgrind matches read-site, so suppress + # by the read-site symbol. + llvm/llvm-project#194147: CodeGenModule::SetLLVMFunctionAttributesForDefinition + Memcheck:Cond + fun:*CodeGenModule*SetLLVMFunctionAttributesForDefinition* + ... +} + +# llvm/llvm-project#194147 (continued): symbols newly tripped on +# the arm64 cell after recipe republish. The aarch64 cell was +# previously documented as clean for #194147 (per the existing +# top-of-file note); that's now stale. Different code paths than +# x86_64 trips, mainly TreeTransform / SemaCUDA / AsmPrinter / +# bitcode writer. +{ + llvm/llvm-project#194147: TreeTransform::TransformIfStmt + Memcheck:Cond + fun:*TreeTransform*TransformIfStmt* + ... +} +{ + llvm/llvm-project#194147: SemaCUDA::inferTargetForImplicitSpecialMember + Memcheck:Cond + fun:*SemaCUDA*inferTargetForImplicitSpecialMember* + ... +} +{ + llvm/llvm-project#194147: AsmPrinter::emitPatchableFunctionEntries + Memcheck:Cond + fun:*AsmPrinter*emitPatchableFunctionEntries* + ... +} +{ + llvm/llvm-project#194147: ValueEnumerator::getAttributeGroupID Cond + Memcheck:Cond + fun:*ValueEnumerator*getAttributeGroupID* + ... +} +{ + # Same symbol, different error class: BitcodeWriter reads an + # 8-byte uninit field (AttributeSet pointer pair) when the + # ModuleBitcodeWriter::write() path serializes the module. Cond + # alone doesn't catch Memcheck:Value8 — value-load errors are + # tracked separately by valgrind. + llvm/llvm-project#194147: ValueEnumerator::getAttributeGroupID Value8 + Memcheck:Value8 + fun:*ValueEnumerator*getAttributeGroupID* + ... +} +{ + llvm/llvm-project#194147: TargetLowering::LowerCallTo + Memcheck:Cond + fun:*TargetLowering*LowerCallTo* + ... +} + +# llvm/llvm-project#194147 (continued): new in release/23.x on x86_64. +# The uninitialised bytes come from the heap allocation of X86TargetMachine +# and are read during SelectionDAG load combining. Fires in every test that +# JITs code; the forked children of the JitPthreadAtfork*/JitAtQuickExit* +# tests then exit through --error-exitcode=1 and the tests fail. +{ + llvm/llvm-project#194147: APInt::setLowBits via computeKnownBits + Memcheck:Cond + fun:_ZN4llvm5APInt10setLowBitsEj + fun:*computeKnownBits* + ... +} + +# llvm/llvm-project#194147 (continued): new in release/23.x on aarch64. +# +# In-process JIT: MachineFunction::getPreferredAlignment reads an +# uninitialised field. It fires in every test that JITs code, and the +# forked children of the JitPthreadAtfork*/JitAtQuickExit* tests then exit +# through --error-exitcode=1. +{ + llvm/llvm-project#194147: MachineFunction::getPreferredAlignment + Memcheck:Cond + fun:*MachineFunction*getPreferredAlignment* + ... +} + +# Out-of-process JIT: ORC runs with concurrent compilation, so +# BasicIRLayerMaterializationUnit::materialize calls cloneToNewContext, +# which writes the module to bitcode and reads it back. On aarch64 the +# serialized bytes are already uninitialised (origins: the heap slots of +# BitstreamWriter::EnterSubblock's Block vector and of CloneModule's +# Users), so every reader of the reloaded module trips valgrind: the +# bitcode reader itself, then the whole backend that compiles the module, +# then the destruction of its LLVMContext. The read sites run to hundreds +# of symbols, so these blocks key on a frame anywhere in the recorded +# stack (leading `...`) instead of on the top frame. + +# aarch64, ORC bitcode round trip (serialize/deserialize). +{ + llvm/llvm-project#194147 aarch64: orc::serializeModule Cond + Memcheck:Cond + ... + fun:*orc*serializeModule* +} +{ + llvm/llvm-project#194147 aarch64: orc::serializeModule Value8 + Memcheck:Value8 + ... + fun:*orc*serializeModule* +} +{ + llvm/llvm-project#194147 aarch64: orc::deserializeModule Cond + Memcheck:Cond + ... + fun:*orc*deserializeModule* +} +{ + llvm/llvm-project#194147 aarch64: orc::deserializeModule Value8 + Memcheck:Value8 + ... + fun:*orc*deserializeModule* +} +{ + llvm/llvm-project#194147 aarch64: BitcodeReader Cond + Memcheck:Cond + ... + fun:*BitcodeReader* +} +{ + llvm/llvm-project#194147 aarch64: BitcodeReader Value8 + Memcheck:Value8 + ... + fun:*BitcodeReader* +} +{ + llvm/llvm-project#194147 aarch64: BitstreamCursor Cond + Memcheck:Cond + ... + fun:*BitstreamCursor* +} +{ + llvm/llvm-project#194147 aarch64: BitstreamCursor Value8 + Memcheck:Value8 + ... + fun:*BitstreamCursor* +} +{ + llvm/llvm-project#194147 aarch64: MetadataLoader Cond + Memcheck:Cond + ... + fun:*MetadataLoader* +} +{ + llvm/llvm-project#194147 aarch64: MetadataLoader Value8 + Memcheck:Value8 + ... + fun:*MetadataLoader* +} + +# aarch64, codegen of the reloaded module. +{ + llvm/llvm-project#194147 aarch64: IRTranslator Cond + Memcheck:Cond + ... + fun:*IRTranslator* +} +{ + llvm/llvm-project#194147 aarch64: IRTranslator Value8 + Memcheck:Value8 + ... + fun:*IRTranslator* +} +{ + llvm/llvm-project#194147 aarch64: GISelCSE Cond + Memcheck:Cond + ... + fun:*GISelCSE* +} +{ + llvm/llvm-project#194147 aarch64: GISelCSE Value8 + Memcheck:Value8 + ... + fun:*GISelCSE* +} +{ + llvm/llvm-project#194147 aarch64: GISelInstProfileBuilder Cond + Memcheck:Cond + ... + fun:*GISelInstProfileBuilder* +} +{ + llvm/llvm-project#194147 aarch64: GISelInstProfileBuilder Value8 + Memcheck:Value8 + ... + fun:*GISelInstProfileBuilder* +} +{ + llvm/llvm-project#194147 aarch64: MachineCSE Cond + Memcheck:Cond + ... + fun:*MachineCSE* +} +{ + llvm/llvm-project#194147 aarch64: MachineCSE Value8 + Memcheck:Value8 + ... + fun:*MachineCSE* +} +{ + llvm/llvm-project#194147 aarch64: SelectionDAGISel Cond + Memcheck:Cond + ... + fun:*SelectionDAGISel* +} +{ + llvm/llvm-project#194147 aarch64: SelectionDAGISel Value8 + Memcheck:Value8 + ... + fun:*SelectionDAGISel* +} +{ + llvm/llvm-project#194147 aarch64: ScheduleDAGSDNodes Cond + Memcheck:Cond + ... + fun:*ScheduleDAGSDNodes* +} +{ + llvm/llvm-project#194147 aarch64: ScheduleDAGSDNodes Value8 + Memcheck:Value8 + ... + fun:*ScheduleDAGSDNodes* +} +{ + llvm/llvm-project#194147 aarch64: BranchProbabilityInfo Cond + Memcheck:Cond + ... + fun:*BranchProbabilityInfo* +} +{ + llvm/llvm-project#194147 aarch64: BranchProbabilityInfo Value8 + Memcheck:Value8 + ... + fun:*BranchProbabilityInfo* +} +{ + llvm/llvm-project#194147 aarch64: AllocaInst::getAllocationSize Cond + Memcheck:Cond + ... + fun:*AllocaInst*getAllocationSize* +} +{ + llvm/llvm-project#194147 aarch64: AllocaInst::getAllocationSize Value8 + Memcheck:Value8 + ... + fun:*AllocaInst*getAllocationSize* +} +{ + llvm/llvm-project#194147 aarch64: AsmPrinter Cond + Memcheck:Cond + ... + fun:*AsmPrinter* +} +{ + llvm/llvm-project#194147 aarch64: AsmPrinter Value8 + Memcheck:Value8 + ... + fun:*AsmPrinter* +} +{ + llvm/llvm-project#194147 aarch64: AArch64 Cond + Memcheck:Cond + ... + fun:*AArch64* +} +{ + llvm/llvm-project#194147 aarch64: AArch64 Value8 + Memcheck:Value8 + ... + fun:*AArch64* +} + +# aarch64, teardown of the reloaded module's context. +{ + llvm/llvm-project#194147 aarch64: LLVMContextImpl Cond + Memcheck:Cond + ... + fun:*LLVMContextImpl* +} +{ + llvm/llvm-project#194147 aarch64: LLVMContextImpl Value8 + Memcheck:Value8 + ... + fun:*LLVMContextImpl* +} +{ + llvm/llvm-project#194147 aarch64: removeDeadConstantUsers Cond + Memcheck:Cond + ... + fun:*removeDeadConstantUsers* +} +{ + llvm/llvm-project#194147 aarch64: removeDeadConstantUsers Value8 + Memcheck:Value8 + ... + fun:*removeDeadConstantUsers* +} diff --git a/interpreter/CppInterOp/lib/CppInterOp/CMakeLists.txt b/interpreter/CppInterOp/lib/CppInterOp/CMakeLists.txt index db8f81ebed03a..cc185f6fbae3d 100644 --- a/interpreter/CppInterOp/lib/CppInterOp/CMakeLists.txt +++ b/interpreter/CppInterOp/lib/CppInterOp/CMakeLists.txt @@ -73,11 +73,6 @@ if (LLVM_LINK_LLVM_DYLIB) # Do NOT expand to every LLVM library: some (e.g. LLVMInterpreter) require # optional dependencies (libffi) that may not be installed. set(llvm_component_libs LLVMFrontendDriver) - foreach(component_lib ${llvm_component_libs}) - if(NOT TARGET ${component_lib}) - list(REMOVE_ITEM llvm_component_libs ${component_lib}) - endif() - endforeach() while(NOT "${new_libs}" STREQUAL "") foreach(lib ${new_libs}) if(TARGET ${lib}) @@ -132,6 +127,10 @@ if (LLVM_LINK_LLVM_DYLIB) Option TargetParser ) + # Since LLVM 23 clangCodeGen lowers call ABIs through llvm/lib/ABI. + if(LLVM_VERSION_MAJOR GREATER 22) + list(APPEND LLVM_LINK_COMPONENTS ABI) + endif() endif(LLVM_LINK_LLVM_DYLIB) # Generate .inc files from CppInterOp.td using cppinterop-tblgen. @@ -269,23 +268,23 @@ if(EMSCRIPTEN) string(REPLACE "\n" " " SYMBOLS_LIST "${SYMBOLS_LIST}") target_compile_options(clangCppInterOp - PRIVATE "SHELL: -Oz" - PRIVATE "SHELL: -flto" + PRIVATE "SHELL: ${CPPINTEROP_LIBRARY_WASM_FLAGS}" PRIVATE "SHELL: ${CPPINTEROP_EXTRA_WASM_FLAGS}" ) target_link_options(clangCppInterOp PRIVATE "SHELL: -s WASM_BIGINT" PRIVATE "SHELL: -s SIDE_MODULE=1" PRIVATE "SHELL: ${SYMBOLS_LIST}" - PRIVATE "SHELL: -Oz" - PRIVATE "SHELL: -flto" + PRIVATE "SHELL: ${CPPINTEROP_LIBRARY_WASM_FLAGS}" PRIVATE "SHELL: ${CPPINTEROP_EXTRA_WASM_FLAGS}" ) else() target_compile_options(clangCppInterOp + PRIVATE "SHELL: ${CPPINTEROP_LIBRARY_WASM_FLAGS}" PRIVATE "SHELL: ${CPPINTEROP_EXTRA_WASM_FLAGS}" ) target_link_options(clangCppInterOp + PRIVATE "SHELL: ${CPPINTEROP_LIBRARY_WASM_FLAGS}" PRIVATE "SHELL: -s WASM_BIGINT" PRIVATE "SHELL: ${CPPINTEROP_EXTRA_WASM_FLAGS}" ) @@ -331,6 +330,15 @@ if(NOT _clang_resource_dir AND NOT CODE_COVERAGE set(_clang_resource_dir "${_detected_resource_dir}") endif() endif() +# Some distributions split the clang driver from the libraries and headers +# (e.g. a Nix clang package with lib/clang//include but no bin/clang). +# Fall back to the resource directory's standard location under the prefix. +set(_prefix_resource_dir + "${CLANG_INSTALL_PREFIX}/lib${LLVM_LIBDIR_SUFFIX}/clang/${LLVM_VERSION_MAJOR}") +if(NOT _clang_resource_dir AND NOT CODE_COVERAGE AND CLANG_INSTALL_PREFIX + AND EXISTS "${_prefix_resource_dir}/include") + set(_clang_resource_dir "${_prefix_resource_dir}") +endif() message(STATUS "Clang resource directory: ${_clang_resource_dir}") string(REPLACE ";" "\;" _VER CPPINTEROP_VERSION) diff --git a/interpreter/CppInterOp/lib/CppInterOp/Compatibility.h b/interpreter/CppInterOp/lib/CppInterOp/Compatibility.h index c5b96fa09088a..b49fd0faf90bb 100644 --- a/interpreter/CppInterOp/lib/CppInterOp/Compatibility.h +++ b/interpreter/CppInterOp/lib/CppInterOp/Compatibility.h @@ -8,13 +8,10 @@ #include "clang/AST/Decl.h" #include "clang/AST/DeclTemplate.h" #include "clang/AST/GlobalDecl.h" +#include "clang/AST/PrettyPrinter.h" #include "clang/Basic/DiagnosticIDs.h" #include "clang/Basic/DiagnosticOptions.h" -#if CLANG_VERSION_MAJOR < 21 -#include "clang/Basic/Cuda.h" -#else #include "clang/Basic/OffloadArch.h" -#endif #include "clang/Basic/SourceLocation.h" #include "clang/Basic/Specifiers.h" #include "clang/Basic/Version.h" @@ -31,6 +28,7 @@ #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringSet.h" #include "llvm/IR/Constants.h" @@ -49,6 +47,8 @@ #include #include #include +#include +#include #if CLANG_VERSION_MAJOR < 22 #define clang_driver_options clang::driver::options @@ -80,23 +80,6 @@ static inline char* GetEnv(const char* Var_Name) { #endif } -#if CLANG_VERSION_MAJOR < 21 -#define Print_Canonical_Types PrintCanonicalTypes -#else -#define Print_Canonical_Types PrintAsCanonical -#endif - -#if CLANG_VERSION_MAJOR < 21 -#define clang_LookupResult_Found clang::LookupResult::Found -#define clang_LookupResult_Not_Found clang::LookupResult::NotFound -#define clang_LookupResult_Found_Overloaded clang::LookupResult::FoundOverloaded -#else -#define clang_LookupResult_Found clang::LookupResultKind::Found -#define clang_LookupResult_Not_Found clang::LookupResultKind::NotFound -#define clang_LookupResult_Found_Overloaded \ - clang::LookupResultKind::FoundOverloaded -#endif - #define STRINGIFY(s) STRINGIFY_X(s) #define STRINGIFY_X(...) #__VA_ARGS__ @@ -275,14 +258,8 @@ inline bool detectCudaInstallPath(const std::vector& args, new clang::DiagnosticIDs()); // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) auto* DiagsBuffer = new clang::TextDiagnosticBuffer; -#if CLANG_VERSION_MAJOR < 21 - llvm::IntrusiveRefCntPtr DiagOpts( - new clang::DiagnosticOptions()); - clang::DiagnosticsEngine Diags(DiagID, DiagOpts, DiagsBuffer); -#else clang::DiagnosticOptions DiagOpts; clang::DiagnosticsEngine Diags(DiagID, DiagOpts, DiagsBuffer); -#endif clang::driver::Driver D("clang", TT, Diags); D.setCheckInputsExist(false); @@ -896,6 +873,18 @@ inline clang::QualType GetTypeFromDecl(const clang::TypeDecl* TD) { return TD->getASTContext().getTypeDeclType(TD); } +// Print anonymous tags as a bare "(anonymous struct)" without the trailing +// source location. Clang 23 replaced the AnonymousTagLocations bit with the +// AnonymousTagMode enum stored in AnonymousTagNameStyle. +inline void SuppressAnonymousTagLocations(clang::PrintingPolicy& Policy) { +#if CLANG_VERSION_MAJOR < 23 + Policy.AnonymousTagLocations = false; +#else + Policy.AnonymousTagNameStyle = + llvm::to_underlying(clang::PrintingPolicy::AnonymousTagMode::Plain); +#endif +} + #ifdef CPPINTEROP_USE_CLING using Value = cling::Value; #else @@ -914,21 +903,24 @@ template inline T convertTo(clang::Value V) { #endif // CPPINTEROP_USE_CLING // Refcount-shared payload wrapping a `compat::Value` for Cpp::Box's -// K_PtrOrObj slot. Boxing-via-copy (not move): clang::Value's move ctor -// releases its own storage on construction -- fixed upstream by -// llvm/llvm-project#200888. The copy ctor correctly retains. -// FIXME(llvm 23): static_assert below fails the build once the minimum -// LLVM crosses 23, prompting the move-semantics cleanup. -static_assert(LLVM_VERSION_MAJOR < 23, - "clang::Value::Value(Value&&) was fixed upstream in " - "llvm/llvm-project#200888; switch ValueRefCount to move " - "semantics and drop this workaround."); +// K_PtrOrObj slot. On Clang < 23 the payload must be boxed by copy, not +// move: clang::Value's move ctor released its own storage on construction +// until Clang 23. The copy ctor retains correctly on every supported +// version. +// cling::Value keeps the copy path -- cling tracks a pre-23 Clang. +#if CLANG_VERSION_MAJOR >= 23 && !defined(CPPINTEROP_USE_CLING) +#define CPPINTEROP_VALUE_BOX_MOVE 1 +#endif namespace detail { struct ValueRefCount { std::atomic rc; Value v; +#ifdef CPPINTEROP_VALUE_BOX_MOVE + explicit ValueRefCount(Value&& V) noexcept : rc(1), v(std::move(V)) {} +#else explicit ValueRefCount(const Value& V) noexcept : rc(1), v(V) {} +#endif static void retain(void* p) noexcept { static_cast(p)->rc.fetch_add(1, std::memory_order_relaxed); @@ -943,10 +935,14 @@ struct ValueRefCount { } // namespace detail /// Wrap a compat::Value into a refcount-shared K_PtrOrObj Cpp::Box. -/// `qt` is the opaque QualType (clang::QualType::getAsOpaquePtr()). -inline Cpp::Box MakeValueBox(const Value& V, void* qt) noexcept { - return Cpp::Box::AdoptObject(new detail::ValueRefCount(V), - &detail::ValueRefCount::Ops, qt); +/// `qt` is the opaque clang::QualType +inline Cpp::Box MakeValueBox(Value& V, void* qt) noexcept { +#ifdef CPPINTEROP_VALUE_BOX_MOVE + auto* Payload = new detail::ValueRefCount(std::move(V)); +#else + auto* Payload = new detail::ValueRefCount(V); +#endif + return Cpp::Box::AdoptObject(Payload, &detail::ValueRefCount::Ops, qt); } inline void InstantiateClassTemplateSpecialization( diff --git a/interpreter/CppInterOp/lib/CppInterOp/CompatibilityGLIBC.h b/interpreter/CppInterOp/lib/CppInterOp/CompatibilityGLIBC.h index f5cffc08a55af..c268af59b3519 100644 --- a/interpreter/CppInterOp/lib/CppInterOp/CompatibilityGLIBC.h +++ b/interpreter/CppInterOp/lib/CppInterOp/CompatibilityGLIBC.h @@ -204,11 +204,7 @@ inline llvm::Error addGlibcNonsharedShims(llvm::orc::LLJIT& J, void* Owner) { auto M = std::make_unique("", *TSCtx); M->setDataLayout(J.getDataLayout()); -#if LLVM_VERSION_MAJOR < 21 - M->setTargetTriple(J.getTargetTriple().str()); -#else M->setTargetTriple(J.getTargetTriple()); -#endif llvm::Type* IntTy = llvm::Type::getInt32Ty(*TSCtx); llvm::PointerType* PtrTy = llvm::PointerType::getUnqual(*TSCtx); diff --git a/interpreter/CppInterOp/lib/CppInterOp/CppInterOp.cpp b/interpreter/CppInterOp/lib/CppInterOp/CppInterOp.cpp index 30119c4282838..7c9736e73e064 100644 --- a/interpreter/CppInterOp/lib/CppInterOp/CppInterOp.cpp +++ b/interpreter/CppInterOp/lib/CppInterOp/CppInterOp.cpp @@ -20,13 +20,13 @@ #include "Sins.h" // for access to private members #include "Tracing.h" -// MSan workaround for clang-repl <= 22: __clang_Interpreter_SetValueNoAlloc +// MSan workaround for clang-repl <= 23: __clang_Interpreter_SetValueNoAlloc // receives JIT-emitted values through varargs, and MSan cannot track shadow // across that JIT/native boundary -- the returned Value carries correct bits // but an uninit shadow, which trips downstream reads (convertTo, ~Value). -// Fixed upstream in llvm/llvm-project#196894; unpoison locally for older LLVM. +// Waiting for llvm/llvm-project#196894; unpoison locally until it lands. #if defined(__has_feature) -#if __has_feature(memory_sanitizer) && LLVM_VERSION_MAJOR <= 22 +#if __has_feature(memory_sanitizer) && LLVM_VERSION_MAJOR <= 23 #include #define CPPINTEROP_MSAN_UNPOISON_VALUE(v) __msan_unpoison(&(v), sizeof(v)) #else @@ -86,6 +86,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Config/llvm-config.h" #include "llvm/Demangle/Demangle.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h" @@ -191,15 +192,15 @@ static bool SkipShutDown = false; /// Constructed as a function-local static AFTER sInterpreters, so its dtor /// fires FIRST (reverse-of-construction); llvm_shutdown then drains the /// ManagedStatic registry, including sInterpreters, deterministically. -/// The llvm_shutdown call itself is gated on LLVM 23+, where -/// Platform::lookupResolvedInitSymbols (llvm/llvm-project#196874) makes +/// The llvm_shutdown call itself is gated on LLVM 24+. Waiting for +/// Platform::lookupResolvedInitSymbols (llvm/llvm-project#196874) which makes /// ~Interpreter's JIT deinit skip lazy materialization. On older LLVM /// the same chain SEGFAULTs in cleanUp against destroyed function-local /// statics, so the dtor is a no-op and sInterpreters leaks instead. struct InterpreterShutdown { ~InterpreterShutdown() { if (!SkipShutDown) { -#if LLVM_VERSION_MAJOR > 22 +#if LLVM_VERSION_MAJOR > 23 llvm::llvm_shutdown(); #endif } @@ -724,22 +725,42 @@ static SourceLocation GetValidSLoc(Sema& semaRef) { namespace { class clangSilent { public: - explicit clangSilent(clang::DiagnosticsEngine& diag) - : fDiagEngine(&diag), - fOldDiagValue(fDiagEngine->getSuppressAllDiagnostics()) { - fDiagEngine->setSuppressAllDiagnostics(true); + clangSilent(clang::DiagnosticsEngine& diag) +#if LLVM_VERSION_MAJOR < 23 + : fDiagEngine(diag), + fOldDiagValue(fDiagEngine.getSuppressAllDiagnostics()) { +#else + : fDiagEngine(diag), fOldClient(diag.getClient()), + fOldOwnedClient(diag.takeClient()) { +#endif +#if LLVM_VERSION_MAJOR < 23 + fDiagEngine.setSuppressAllDiagnostics(true); +#else + fDiagEngine.setClient(&fIgnoringClient, /*ShouldOwnClient=*/false); +#endif } - ~clangSilent() { fDiagEngine->setSuppressAllDiagnostics(fOldDiagValue); } - - clangSilent(const clangSilent&) = delete; - clangSilent& operator=(const clangSilent&) = delete; - clangSilent(clangSilent&&) = delete; - clangSilent& operator=(clangSilent&&) = delete; + ~clangSilent() { +#if LLVM_VERSION_MAJOR < 23 + fDiagEngine.setSuppressAllDiagnostics(fOldDiagValue); +#else + if (fOldOwnedClient) + fDiagEngine.setClient(fOldOwnedClient.release(), + /*ShouldOwnClient=*/true); + else + fDiagEngine.setClient(fOldClient, /*ShouldOwnClient=*/false); +#endif + } -private: - clang::DiagnosticsEngine* fDiagEngine; +protected: + clang::DiagnosticsEngine& fDiagEngine; +#if LLVM_VERSION_MAJOR < 23 bool fOldDiagValue; +#else + clang::DiagnosticConsumer* fOldClient; + std::unique_ptr fOldOwnedClient; + clang::IgnoringDiagConsumer fIgnoringClient; +#endif }; } // namespace @@ -812,6 +833,21 @@ DeclRef GetOrForceDefinition(DeclRef DRef) { return INTEROP_RETURN(nullptr); } +// The definition of D if D is a class, forced into the AST if only a +// declaration is loaded (e.g. an autoload-annotated forward declaration +// injected by a runtime ROOT dictionary, which hasDefinition() does not +// complete). const_cast: completing the definition mutates the AST, but the +// definition is logically a read-only property of the class. Taking a mutable +// DeclRef in the public API would make unqualified calls in downstream +// wrappers ambiguous via ADL. +static CXXRecordDecl* GetCXXRecordDefinition(const Decl* D) { + if (!isa_and_nonnull(D)) + return nullptr; + return dyn_cast_or_null(unwrap( + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + GetOrForceDefinition(DeclRef(const_cast(D))))); +} + size_t SizeOf(ConstDeclRef DRef) { INTEROP_TRACE(DRef); assert(DRef); @@ -866,14 +902,8 @@ bool IsTypedefed(ConstDeclRef DRef) { bool IsAbstract(DeclRef DRef) { INTEROP_TRACE(DRef); - const auto* D = unwrap(DRef); - if (llvm::isa_and_nonnull(D)) { - const auto* Def = llvm::dyn_cast_or_null( - unwrap(GetOrForceDefinition(DRef))); - return INTEROP_RETURN(Def && Def->isAbstract()); - } - - return INTEROP_RETURN(false); + const auto* Def = GetCXXRecordDefinition(unwrap(DRef)); + return INTEROP_RETURN(Def && Def->isAbstract()); } bool IsEnumScope(ConstDeclRef DRef) { @@ -1070,7 +1100,7 @@ static std::string GetCompleteNameImpl(ConstDeclRef DRef, bool qualified) { Policy.Suppress_Elab = true; } else { Policy.SuppressScope = true; - Policy.AnonymousTagLocations = false; + compat::SuppressAnonymousTagLocations(Policy); Policy.SuppressTemplateArgsInCXXConstructors = false; Policy.SuppressDefaultTemplateArgs = false; Policy.AlwaysIncludeTypeForTemplateArgument = true; @@ -1230,21 +1260,22 @@ static const clang::Decl* GetUnderlyingScopeImpl(const clang::Decl* D) { // return that target; otherwise return D unchanged. The using-shadow // is the only carrier of the "effective access" introduced into the // derived class, so callers that need access info should consult D -// before unwrapping. -static clang::Decl* UnwrapUsingShadowToFunction(clang::Decl* D) { - if (auto* USD = dyn_cast_or_null(D)) +// before unwrapping. DeclT is Decl or const Decl. +template static DeclT* UnwrapUsingShadowToFunction(DeclT* D) { + if (const auto* USD = dyn_cast_or_null(D)) if (auto* Target = USD->getTargetDecl()) - if (isa(Target) || isa(Target)) + if (isa(Target)) return Target; return D; } -static const clang::Decl* UnwrapUsingShadowToFunction(const clang::Decl* D) { - if (const auto* USD = dyn_cast_or_null(D)) - if (const auto* Target = USD->getTargetDecl()) - if (isa(Target) || isa(Target)) - return Target; - return D; +// Resolves the FunctionDecl a decl names, looking through a using-shadow and +// a template pattern. Null when the decl does not name a function. +static const clang::FunctionDecl* UnwrapToFunctionDecl(const clang::Decl* D) { + D = UnwrapUsingShadowToFunction(D); + if (const auto* FTD = dyn_cast_or_null(D)) + D = FTD->getTemplatedDecl(); + return dyn_cast_or_null(D); } DeclRef GetUnderlyingScope(ConstDeclRef DRef) { @@ -1441,8 +1472,7 @@ DeclRef GetNamed(const std::string& name, ConstDeclRef parent /*= nullptr*/) { // pick the semantics they want, instead of GetNamed approximating both. if (!ND && Within) { if (auto* RD = llvm::dyn_cast(Within)) { - auto* Def = llvm::dyn_cast_or_null( - unwrap(GetOrForceDefinition(DeclRef(RD)))); + auto* Def = GetCXXRecordDefinition(RD); if (!Def) return INTEROP_RETURN(nullptr); auto& S = getSema(); @@ -1476,6 +1506,23 @@ DeclRef GetNamed(const std::string& name, ConstDeclRef parent /*= nullptr*/) { return INTEROP_RETURN(nullptr); } +bool IsVisibleName(const std::string& name, ConstDeclRef parent /*= nullptr*/) { + INTEROP_TRACE(name, parent); + clang::DeclContext* Within = nullptr; + if (parent) { + auto* D = unwrap(GetUnderlyingScope(parent)); + Within = llvm::dyn_cast(D); + } + compat::SynthesizingCodeRAII RAII(&getInterp()); + if (Within) + Within->getPrimaryContext()->buildLookup(); + + // LookupUnqualified returns the (D*)-1 sentinel for overload sets and + // ambiguous results, which still means the name exists. + clang::DeclarationName DName = &getSema().Context.Idents.get(name); + return INTEROP_RETURN(LookupUnqualified(getSema(), DName, Within) != nullptr); +} + DeclRef GetParentScope(ConstDeclRef DRef) { INTEROP_TRACE(DRef); // const_cast: the returned DeclRef is a mutable DRef, so the caller may @@ -1515,42 +1562,26 @@ DeclRef GetParentScope(ConstDeclRef DRef) { size_t GetNumBases(ConstDeclRef DRef) { INTEROP_TRACE(DRef); - // const_cast: completing the definition mutates the AST, but the base list - // is logically a read-only property of the class. Taking a mutable DeclRef - // would make unqualified calls in downstream wrappers ambiguous via ADL. - auto* D = const_cast(unwrap(DRef)); + const auto* D = unwrap(DRef); // hasDefinition() completes the redecl chain (dataPtr), which may // deserialize it; so does getNumBases() below. compat::SynthesizingCodeRAII RAII(&getInterp()); - if (auto* CTSD = llvm::dyn_cast_or_null(D)) + if (const auto* CTSD = + llvm::dyn_cast_or_null(D)) if (!CTSD->hasDefinition()) - compat::InstantiateClassTemplateSpecialization(getInterp(), CTSD); - if (llvm::isa_and_nonnull(D)) { - // A class known only as an autoload-annotated forward declaration - // (e.g., injected by a runtime ROOT dictionary) is not completed by - // hasDefinition(); resolve its definition through Sema first, or the - // base list is silently empty. - if (const auto* Def = llvm::dyn_cast_or_null( - unwrap(GetOrForceDefinition(D)))) - return INTEROP_RETURN(Def->getNumBases()); - } - - return INTEROP_RETURN(0); + // const_cast: see GetCXXRecordDefinition. + compat::InstantiateClassTemplateSpecialization( + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast) + getInterp(), const_cast(CTSD)); + const auto* Def = GetCXXRecordDefinition(D); + return INTEROP_RETURN(Def ? Def->getNumBases() : 0); } DeclRef GetBaseClass(ConstDeclRef DRef, size_t ibase) { INTEROP_TRACE(DRef, ibase); - // const_cast: see GetNumBases. - auto* D = const_cast(unwrap(DRef)); - if (!llvm::isa_and_nonnull(D)) - return INTEROP_RETURN(nullptr); - compat::SynthesizingCodeRAII RAII(&getInterp()); - // See GetNumBases: autoload-annotated forward declarations need Sema to - // resolve their definition before bases are visible. - const auto* CXXRD = llvm::dyn_cast_or_null( - unwrap(GetOrForceDefinition(D))); + const auto* CXXRD = GetCXXRecordDefinition(unwrap(DRef)); if (!CXXRD || CXXRD->getNumBases() <= ibase) return INTEROP_RETURN(nullptr); @@ -1624,9 +1655,9 @@ bool IsSubclass(ConstDeclRef derived, ConstDeclRef base) { // This is an internal helper function for the CppInterOp library (as evident // by the 'static' declaration), while the similar GetBaseClassOffset() // function below is exposed to library users. -static unsigned ComputeBaseOffset(const ASTContext& Context, - const CXXRecordDecl* DerivedRD, - const CXXBasePath& Path) { +static int64_t ComputeBaseOffset(ASTContext& Context, + const CXXRecordDecl* DerivedRD, + const CXXBasePath& Path, const void* address) { CharUnits NonVirtualOffset = CharUnits::Zero(); unsigned NonVirtualStart = 0; @@ -1661,15 +1692,55 @@ static unsigned ComputeBaseOffset(const ASTContext& Context, // CharUnits. // return BaseOffset(DerivedRD, VirtuaBose, aBlnVirtualOffset); if (VirtualBase) { - const ASTRecordLayout& Layout = Context.getASTRecordLayout(DerivedRD); - CharUnits VirtualOffset = Layout.getVBaseClassOffset(VirtualBase); + CharUnits VirtualOffset; + + if (address) { + const char* Object = static_cast(address); + if (Context.getTargetInfo().getCXXABI().isMicrosoft()) { + const ASTRecordLayout& Layout = Context.getASTRecordLayout(DerivedRD); + CharUnits VBPtrOffset = Layout.getVBPtrOffset(); + auto& VTContext = + *cast(Context.getVTableContext()); + unsigned VBTableIndex = + VTContext.getVBTableIndex(DerivedRD, VirtualBase); + const char* VBTable = nullptr; + std::memcpy(static_cast(&VBTable), + Object + VBPtrOffset.getQuantity(), sizeof(VBTable)); + int32_t Offset = 0; + std::memcpy(&Offset, VBTable + (VBTableIndex * sizeof(Offset)), + sizeof(Offset)); + + // Microsoft vbtable entries are relative to the vbptr, not the object. + VirtualOffset = VBPtrOffset + CharUnits::fromQuantity(Offset); + } else { + auto& VTContext = + *cast(Context.getVTableContext()); + CharUnits VBaseOffsetOffset = + VTContext.getVirtualBaseOffsetOffset(DerivedRD, VirtualBase); + const char* VTable = nullptr; + std::memcpy(static_cast(&VTable), Object, sizeof(VTable)); + + // Itanium virtual base offsets are relative to this subobject. The + // slot is relative to the vtable address point and can follow vcall + // offsets, so obtain it from the vtable context. + ptrdiff_t Offset = 0; + std::memcpy(&Offset, VTable + VBaseOffsetOffset.getQuantity(), + sizeof(Offset)); + VirtualOffset = CharUnits::fromQuantity(Offset); + } + } else { + const ASTRecordLayout& Layout = Context.getASTRecordLayout(DerivedRD); + VirtualOffset = Layout.getVBaseClassOffset(VirtualBase); + } + return (NonVirtualOffset + VirtualOffset).getQuantity(); } return NonVirtualOffset.getQuantity(); } -int64_t GetBaseClassOffset(ConstDeclRef derived, ConstDeclRef base) { - INTEROP_TRACE(derived, base); +int64_t GetBaseClassOffset(ConstDeclRef derived, ConstDeclRef base, + const void* address) { + INTEROP_TRACE(derived, base, address); if (base == derived) return INTEROP_RETURN(0); @@ -1702,8 +1773,8 @@ int64_t GetBaseClassOffset(ConstDeclRef derived, ConstDeclRef base) { DCXXRD->isDerivedFrom(BCXXRD, Paths); // FIXME: We might want to cache these requests as they seem expensive. - return INTEROP_RETURN( - ComputeBaseOffset(getSema().getASTContext(), DCXXRD, Paths.front())); + return INTEROP_RETURN(ComputeBaseOffset(getSema().getASTContext(), DCXXRD, + Paths.front(), address)); } template @@ -1942,10 +2013,7 @@ OwnershipBehaviour GetOwnershipBehaviour(ConstFuncRef Fn) { INTEROP_TRACE(Fn); if (!Fn) return INTEROP_RETURN(OwnershipBehaviour::Unknown); - const auto* D = UnwrapUsingShadowToFunction(unwrap(Fn)); - if (const auto* FTD = dyn_cast(D)) - D = FTD->getTemplatedDecl(); - const auto* FD = dyn_cast(D); + const auto* FD = UnwrapToFunctionDecl(unwrap(Fn)); if (!FD) return INTEROP_RETURN(OwnershipBehaviour::Unknown); @@ -1970,10 +2038,7 @@ uint64_t GetDeallocationIndexes(ConstFuncRef Fn) { INTEROP_TRACE(Fn); if (!Fn) return INTEROP_RETURN(uint64_t{0}); - const auto* D = UnwrapUsingShadowToFunction(unwrap(Fn)); - if (const auto* FTD = dyn_cast(D)) - D = FTD->getTemplatedDecl(); - const auto* FD = dyn_cast(D); + const auto* FD = UnwrapToFunctionDecl(unwrap(Fn)); if (!FD) return INTEROP_RETURN(uint64_t{0}); uint64_t result = 0; @@ -1993,10 +2058,7 @@ int GetAllocationSizeParamIndex(ConstFuncRef Fn) { INTEROP_TRACE(Fn); if (!Fn) return INTEROP_RETURN(-1); - const auto* D = UnwrapUsingShadowToFunction(unwrap(Fn)); - if (const auto* FTD = dyn_cast(D)) - D = FTD->getTemplatedDecl(); - const auto* FD = dyn_cast(D); + const auto* FD = UnwrapToFunctionDecl(unwrap(Fn)); if (!FD) return INTEROP_RETURN(-1); for (const auto* attr : FD->specific_attrs()) { @@ -2011,10 +2073,7 @@ AllocType IsAllocator(ConstFuncRef Fn) { INTEROP_TRACE(Fn); if (!Fn) return INTEROP_RETURN(AllocType::Unknown); - const auto* D = UnwrapUsingShadowToFunction(unwrap(Fn)); - if (const auto* FTD = dyn_cast(D)) - D = FTD->getTemplatedDecl(); - if (const auto* FD = dyn_cast(D)) { + if (const auto* FD = UnwrapToFunctionDecl(unwrap(Fn))) { if (FD->getBuiltinID() == Builtin::ID::BImalloc) return INTEROP_RETURN(AllocType::Malloc); if (const auto* FDA = FD->getAttr()) { @@ -2059,21 +2118,108 @@ AllocType IsAllocator(ConstFuncRef Fn) { return INTEROP_RETURN(AllocType::Unknown); } -bool IsDeallocator(ConstFuncRef Fn) { - INTEROP_TRACE(Fn); - if (!Fn) - INTEROP_RETURN(false); - const auto* D = unwrap(Fn); - if (const auto* FD = dyn_cast(D)) { - if (FD->getBuiltinID() == Builtin::ID::BIfree) - return INTEROP_RETURN(true); - if (const auto* FDA = FD->getAttr()) { - if (FDA->getOwnKind() == OwnershipAttr::Takes) - return INTEROP_RETURN(true); +// We follow Clang's convention for ownership attributes, parameters are 1 +// indexed. Therefore if a function has an implicit this, you should use +// cppDeallocDelete_2 to labeling first actual parameter, also it is not allowed +// to label implicit `this`. +// Function appends a DeallocType to the vector taken, for every argument in +// the function including implicit this. If an argument does not have an +// deallocation ownership, Opaque value is added. +bool IsDeallocator(ConstFuncRef Fn, std::vector& VPP) { + INTEROP_TRACE(Fn, INTEROP_OUT(VPP)); + const auto* FD = UnwrapToFunctionDecl(unwrap(Fn)); + if (!FD) + return INTEROP_RETURN(false); + const auto* MD = dyn_cast(FD); + const bool hasImplicitThis = MD && MD->isImplicitObjectMemberFunction(); + const unsigned numParam = FD->getNumParams(); + std::vector newVPP(numParam + hasImplicitThis, + DeallocType::Opaque); + if (FD->getBuiltinID() == Builtin::ID::BIfree) { + VPP.push_back(DeallocType::Free); + return INTEROP_RETURN(true); + } + for (const auto* FDA : FD->specific_attrs()) { + // If it does not return, either holds or takes + if (FDA->getOwnKind() == OwnershipAttr::Returns) + continue; + // If GCC's ownership attrs are used, it is assumed to be free + for (const ParamIdx index : FDA->args()) + // Self can not be attributed with ownership_attr + newVPP[index.getLLVMIndex()] = DeallocType::Free; + } + // Traverse annotate_attrs + for (const auto* FDA : FD->specific_attrs()) { + llvm::StringRef attrName = FDA->getAnnotation(); + DeallocType kind; + if (attrName == "cppDeallocFree") + kind = DeallocType::Free; + else if (attrName == "cppDeallocDelete") + kind = DeallocType::Delete; + else if (attrName == "cppDeallocDeleteArr") + kind = DeallocType::DeleteArr; + else if (attrName == "cppDeallocNone") + kind = DeallocType::None; + // If annotate_attr is any other thing continue + else + continue; + // If attribute has no argument, it is assumed to attribute + // for first index (after implicit this) + if (FDA->args_size() == 0) { + if (numParam >= 1) + newVPP[hasImplicitThis] = kind; + continue; + } + // Traverse all args of annotate_attr + for (const auto* E : FDA->args()) { + // Since we support FTDs, we must do that check + if (E->isValueDependent()) + continue; + Expr::EvalResult evalRes; + // Not int + if (!E->EvaluateAsInt(evalRes, FD->getASTContext())) + continue; + const int64_t index = evalRes.Val.getInt().getSExtValue(); + if (index < 1 || (uint64_t)index > newVPP.size() || + (index == 1 && hasImplicitThis)) + continue; + newVPP[index - 1] = kind; } } - - return INTEROP_RETURN(false); + // Traverse APINotes attrs + for (const auto* FDA : FD->specific_attrs()) { + llvm::StringRef attrName = FDA->getAttribute(); + attrName.consume_front("returns_"); + llvm::SmallVector tokens; + attrName.split(tokens, ' ', -1, false); + for (llvm::StringRef token : tokens) { + DeallocType kind; + if (token.consume_front("cppDeallocDeleteArr")) + kind = DeallocType::DeleteArr; + else if (token.consume_front("cppDeallocDelete")) + kind = DeallocType::Delete; + else if (token.consume_front("cppDeallocFree")) + kind = DeallocType::Free; + else if (token.consume_front("cppDeallocNone")) + kind = DeallocType::None; + else + continue; + // If attribute is cppDealloc`DeallocType` without index, + // it is assumed to be attribute for first index (after implicit this) + if (token.empty()) { + if (numParam >= 1) + newVPP[hasImplicitThis] = kind; + continue; + } + unsigned index; + if (!token.consume_front("_") || token.getAsInteger(10, index) || + index < 1 || index > newVPP.size() || (hasImplicitThis && index == 1)) + continue; + newVPP[index - 1] = kind; + } + } + VPP.insert(VPP.end(), newVPP.begin(), newVPP.end()); + return INTEROP_RETURN(true); } bool IsFunctionProtoType(ConstTypeRef TyRef) { @@ -2665,6 +2811,13 @@ bool IsTemplateParmType(ConstTypeRef TyRef) { return INTEROP_RETURN(QT->isTemplateTypeParmType()); } +bool IsPackExpansionType(ConstTypeRef TyRef) { + INTEROP_TRACE(TyRef); + clang::QualType QT = clang::QualType::getFromOpaquePtr(TyRef.data); + return INTEROP_RETURN( + llvm::isa_and_nonnull(QT.getTypePtrOrNull())); +} + std::string GetFunctionSignature(ConstFuncRef func) { INTEROP_TRACE(func); if (!func) @@ -2815,16 +2968,16 @@ bool GetClassTemplatedMethods(const std::string& name, ConstDeclRef parent, DC->getPrimaryContext()->buildLookup(); CppInternal::utils::Lookup::Named(&S, R, DC); - if (R.getResultKind() == clang_LookupResult_Not_Found && funcs.empty()) + if (R.getResultKind() == clang::LookupResultKind::NotFound && funcs.empty()) return INTEROP_RETURN(false); // Distinct match, single Decl - else if (R.getResultKind() == clang_LookupResult_Found) { + else if (R.getResultKind() == clang::LookupResultKind::Found) { if (IsTemplatedFunction(R.getFoundDecl())) funcs.push_back(R.getFoundDecl()); } // Loop over overload set - else if (R.getResultKind() == clang_LookupResult_Found_Overloaded) { + else if (R.getResultKind() == clang::LookupResultKind::FoundOverloaded) { for (auto* Found : R) { if (IsTemplatedFunction(Found)) funcs.push_back(Found); @@ -3069,6 +3222,24 @@ bool IsExplicit(ConstFuncRef method) { return INTEROP_RETURN(false); } +bool IsOperator(ConstFuncRef func) { + INTEROP_TRACE(func); + const auto* D = UnwrapUsingShadowToFunction(unwrap(func)); + if (const auto* FTD = llvm::dyn_cast_or_null(D)) + D = FTD->getTemplatedDecl(); + if (const auto* FD = llvm::dyn_cast_or_null(D)) + return INTEROP_RETURN(FD->isOverloadedOperator()); + return INTEROP_RETURN(false); +} + +bool IsConversionOperator(ConstFuncRef func) { + INTEROP_TRACE(func); + const auto* D = UnwrapUsingShadowToFunction(unwrap(func)); + if (const auto* FTD = llvm::dyn_cast_or_null(D)) + D = FTD->getTemplatedDecl(); + return INTEROP_RETURN(llvm::isa_and_nonnull(D)); +} + void* GetFunctionAddress(const char* mangled_name) { INTEROP_TRACE(mangled_name); auto& I = getInterp(); @@ -4339,7 +4510,7 @@ static void GetDeclName(const clang::Decl* D, ASTContext& Context, PrintingPolicy Policy(Context.getPrintingPolicy()); Policy.SuppressTagKeyword = true; Policy.SuppressUnwrittenScope = true; - Policy.Print_Canonical_Types = true; + Policy.PrintAsCanonical = true; if (const auto* TD = dyn_cast(D)) { // This is a class, struct, or union member. QualType QT; @@ -4463,7 +4634,7 @@ bool IsCopyConstructorDeleted(QualType QT) { return false; } - RD = unwrap(GetOrForceDefinition(DeclRef(RD))); + RD = GetCXXRecordDefinition(RD); if (!RD) return false; @@ -5858,14 +6029,10 @@ InterpRef CreateInterpreter(const std::vector& Args /*={}*/, std::vector AllArgs; #ifdef CPPINTEROP_BAKED_INTERPRETER_ARGS // Storage for the split arguments; must outlive the pointers in AllArgs. - std::vector BakedStorage; - StringRef Baked(CPPINTEROP_BAKED_INTERPRETER_ARGS); - while (!Baked.empty()) { - StringRef Arg; - std::tie(Arg, Baked) = Baked.split(' '); - if (!Arg.empty()) - BakedStorage.push_back(Arg.str()); - } + SmallVector BakedParts; + StringRef(CPPINTEROP_BAKED_INTERPRETER_ARGS) + .split(BakedParts, ' ', /*MaxSplit=*/-1, /*KeepEmpty=*/false); + std::vector BakedStorage(BakedParts.begin(), BakedParts.end()); for (const std::string& Arg : BakedStorage) AllArgs.push_back(Arg.c_str()); #endif @@ -6030,24 +6197,10 @@ InterpRef CreateInterpreter(const std::vector& Args /*={}*/, #if !defined(CPPINTEROP_USE_CLING) && !defined(EMSCRIPTEN) DefineAbsoluteSymbol(*I, "__ci_newtag", reinterpret_cast(&__ci_newtag)); -// llvm >= 21 has this defined as a C symbol that does not require mangling -#if CLANG_VERSION_MAJOR >= 21 + // A C symbol since LLVM 21, so it needs no mangling. DefineAbsoluteSymbol( *I, "__clang_Interpreter_SetValueWithAlloc", reinterpret_cast(&__clang_Interpreter_SetValueWithAlloc)); -#else - // obtain mangled name - auto* D = - unwrap(Cpp::GetNamed("__clang_Interpreter_SetValueWithAlloc")); - if (auto* FD = llvm::dyn_cast_or_null(D)) { - auto GD = GlobalDecl(FD); - std::string mangledName; - compat::maybeMangleDeclName(GD, mangledName); - DefineAbsoluteSymbol( - *I, mangledName.c_str(), - reinterpret_cast(&__clang_Interpreter_SetValueWithAlloc)); - } -#endif DefineAbsoluteSymbol( *I, "__clang_Interpreter_SetValueNoAlloc", @@ -6154,17 +6307,11 @@ static int Declare(compat::Interpreter& I, const char* code, bool silent) { // distinguish "parsed cleanly" from "parsed with errors". clang::DiagnosticsEngine& Diag = I.getSema().getDiagnostics(); clang::DiagnosticErrorTrap Trap(Diag); - if (silent) { - clangSilent diagSuppr(Diag); - auto result = I.declare(code); - if (Trap.hasErrorOccurred()) - return 1; - return result; - } + std::optional diagSuppr; + if (silent) + diagSuppr.emplace(Diag); auto result = I.declare(code); - if (Trap.hasErrorOccurred()) - return 1; - return result; + return Trap.hasErrorOccurred() ? 1 : result; } int Declare(const char* code, bool silent) { diff --git a/interpreter/CppInterOp/lib/CppInterOp/CppInterOp.td b/interpreter/CppInterOp/lib/CppInterOp/CppInterOp.td index 6aba48d089136..f7044decabd41 100644 --- a/interpreter/CppInterOp/lib/CppInterOp/CppInterOp.td +++ b/interpreter/CppInterOp/lib/CppInterOp/CppInterOp.td @@ -117,6 +117,18 @@ a specific named entity (functions, enums, etcetera).}]; ]; } +def IsVisibleName : CppInterOpAPI { + let Doc = [{Checks whether unqualified name lookup of \c name from within +\c parent (the global scope if null) finds any declaration, including +overload sets and declarations made visible by using-directives.}]; + + let ReturnType = "bool"; + let Args = [ + Arg<"const std::string&", "name">, + Arg<"ConstDeclRef", "parent", "nullptr"> + ]; +} + def GetGlobalScope : CppInterOpAPI { let Doc = "Gets the global scope of the whole C++ instance."; @@ -771,13 +783,17 @@ def GetOperator : CppInterOpAPI { } def GetBaseClassOffset : CppInterOpAPI { - let Doc = [{Each base has its own offset in a Derived Class. This offset can be -used to get to the Base Class fields.}]; + let Doc = [{Returns the byte offset from derived to base. Without an address, +the offset comes from the record layout of a complete derived object. If address +points to a live derived object or subobject and the path to base contains a +virtual base, the offset is read from its vtable/vbtable. This also works when +the complete object's dynamic type is unknown to the interpreter.}]; let ReturnType = "int64_t"; let Args = [ Arg<"ConstDeclRef", "derived">, - Arg<"ConstDeclRef", "base"> + Arg<"ConstDeclRef", "base">, + Arg<"const void*", "address", "nullptr"> ]; } @@ -1365,10 +1381,18 @@ def IsAllocator : CppInterOpAPI { } def IsDeallocator : CppInterOpAPI { - let Doc = "Checks if the provided function is a deallocator function"; + let Doc = [{Records the memory deallocation behaviour of each parameter of + the given function, as declared by its attributes. Appends one entry per + parameter to VPP, preceded by one entry for the object parameter if the + function has an implicit this; whatever VPP already holds is left + untouched. A parameter labelled cppDeallocNone is reported as None, while + one carrying no deallocation attribute is reported as Opaque. Labelling + the object parameter is not supported and is ignored. Returns false if + func is not a function, in which case VPP is not modified.}]; let ReturnType = "bool"; let Args = [ - Arg<"ConstFuncRef", "func"> + Arg<"ConstFuncRef", "func">, + OutArg<"std::vector&", "VPP"> ]; } @@ -1503,6 +1527,14 @@ def IsTemplateParmType : CppInterOpAPI { ]; } +def IsPackExpansionType : CppInterOpAPI { + let Doc = "Checks if the given type is a pack expansion type (e.g. `Ts...`)."; + let ReturnType = "bool"; + let Args = [ + Arg<"ConstTypeRef", "TyRef"> + ]; +} + def IsFunction : CppInterOpAPI { let Doc = "Checks if the scope is a function."; let ReturnType = "bool"; @@ -1521,6 +1553,20 @@ def IsMethod : CppInterOpAPI { let Args = [Arg<"ConstFuncRef", "method">]; } +def IsOperator : CppInterOpAPI { + let Doc = [{Checks if the provided function (or function template) is an +overloaded operator.}]; + let ReturnType = "bool"; + let Args = [Arg<"ConstFuncRef", "func">]; +} + +def IsConversionOperator : CppInterOpAPI { + let Doc = [{Checks if the provided function (or function template) is a +conversion operator, e.g. `operator int()`.}]; + let ReturnType = "bool"; + let Args = [Arg<"ConstFuncRef", "func">]; +} + def IsPODType : CppInterOpAPI { let Doc = "Checks if the provided parameter is a Plain Old Data Type (POD)."; let ReturnType = "bool"; diff --git a/interpreter/CppInterOp/lib/CppInterOp/CppInterOpAPI.td b/interpreter/CppInterOp/lib/CppInterOp/CppInterOpAPI.td index 95a4e78a367f1..7ab5700630ab2 100644 --- a/interpreter/CppInterOp/lib/CppInterOp/CppInterOpAPI.td +++ b/interpreter/CppInterOp/lib/CppInterOp/CppInterOpAPI.td @@ -89,6 +89,7 @@ def CMap_bool : CTypeMap<"bool", "bool">; def CMap_int : CTypeMap<"int", "int">; def CMap_void : CTypeMap<"void", "void">; def CMap_voidp : CTypeMap<"void*", "void*">; +def CMap_constvoidp: CTypeMap<"const void*", "const void*">; def CMap_constcharp: CTypeMap<"const char*", "const char*">; def CMap_size_t : CTypeMap<"size_t", "size_t">; def CMap_unsigned : CTypeMap<"unsigned", "unsigned">; diff --git a/interpreter/CppInterOp/lib/CppInterOp/DynamicLibraryManagerSymbol.cpp b/interpreter/CppInterOp/lib/CppInterOp/DynamicLibraryManagerSymbol.cpp index abc77bbe6e341..57f74b7b8bd9a 100644 --- a/interpreter/CppInterOp/lib/CppInterOp/DynamicLibraryManagerSymbol.cpp +++ b/interpreter/CppInterOp/lib/CppInterOp/DynamicLibraryManagerSymbol.cpp @@ -32,6 +32,7 @@ #include "llvm/Support/WithColor.h" #include +#include #include #include #include diff --git a/interpreter/CppInterOp/lib/CppInterOp/exports.ld b/interpreter/CppInterOp/lib/CppInterOp/exports.ld index e7241a77e19ca..3675e694e605b 100644 --- a/interpreter/CppInterOp/lib/CppInterOp/exports.ld +++ b/interpreter/CppInterOp/lib/CppInterOp/exports.ld @@ -83,11 +83,5 @@ -Wl,--export=_ZN5clang17ExternalASTSource2IDE -Wl,--export=_ZN5clang17ExternalASTSource30FindExternalVisibleDeclsByNameEPKNS_11DeclContextENS_15DeclarationNameES3_ -Wl,--export=_ZN5clang17ExternalASTSource30GetExternalCXXCtorInitializersEy --Wl,--export=_ZN5clang18ExternalSemaSource14ReadMethodPoolENS_8SelectorE --Wl,--export=_ZN5clang18ExternalSemaSource19ReadKnownNamespacesERN4llvm15SmallVectorImplIPNS_13NamespaceDeclEEE --Wl,--export=_ZN5clang18ExternalSemaSource20ReadUndefinedButUsedERN4llvm9MapVectorIPNS_9NamedDeclENS_14SourceLocationENS1_8DenseMapIS4_jNS1_12DenseMapInfoIS4_vEENS1_6detail12DenseMapPairIS4_jEEEENS1_11SmallVectorINSt3__24pairIS4_S5_EELj0EEEEE --Wl,--export=_ZN5clang18ExternalSemaSource23updateOutOfDateSelectorENS_8SelectorE --Wl,--export=_ZN5clang18ExternalSemaSource2IDE --Wl,--export=_ZN5clang18ExternalSemaSource32ReadMismatchingDeleteExpressionsERN4llvm9MapVectorIPNS_9FieldDeclENS1_11SmallVectorINSt3__24pairINS_14SourceLocationEbEELj4EEENS1_8DenseMapIS4_jNS1_12DenseMapInfoIS4_vEENS1_6detail12DenseMapPairIS4_jEEEENS5_INS7_IS4_SA_EELj0EEEEE --Wl,--export=_ZN5clang18ExternalSemaSourceD2Ev +-Wl,--export=_ZN5clang17ExternalASTSourceD2Ev -Wl,--export=_ZNK5clang17ExternalASTSource20getMemoryBufferSizesERNS0_17MemoryBufferSizesE diff --git a/interpreter/CppInterOp/unittests/CppInterOp/APINotes/TestHeader.apinotes b/interpreter/CppInterOp/unittests/CppInterOp/APINotes/TestHeader.apinotes index 16bc930ba0544..899a3b492672b 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/APINotes/TestHeader.apinotes +++ b/interpreter/CppInterOp/unittests/CppInterOp/APINotes/TestHeader.apinotes @@ -17,4 +17,27 @@ Functions: - Name: testNone SwiftReturnOwnership: cppAllocNone - Name: testWeirdAttr - SwiftReturnOwnership: someWeirdAttr \ No newline at end of file + SwiftReturnOwnership: someWeirdAttr + - Name: testDeallocFree + SwiftReturnOwnership: cppDeallocFree_1 + - Name: testDeallocDelete + SwiftReturnOwnership: cppDeallocDelete_1 + - Name: testDeallocDeleteArr + SwiftReturnOwnership: cppDeallocDeleteArr_1 + - Name: testDeallocNone + SwiftReturnOwnership: cppDeallocNone_1 + - Name: testDeallocMulti + SwiftReturnOwnership: "cppDeallocDelete_1 cppDeallocFree_2" + - Name: testDeallocWeirdAttr + SwiftReturnOwnership: someWeirdDeallocAttr +Tags: + - Name: KlassNotes + Methods: + - Name: releaseArg + SwiftReturnOwnership: cppDeallocFree_2 + - Name: releaseSelf + SwiftReturnOwnership: cppDeallocDelete_1 + - Name: releaseBoth + SwiftReturnOwnership: "cppDeallocDelete_1 cppDeallocFree_2" + - Name: releaseBare + SwiftReturnOwnership: cppDeallocDelete diff --git a/interpreter/CppInterOp/unittests/CppInterOp/APINotes/TestHeader.h b/interpreter/CppInterOp/unittests/CppInterOp/APINotes/TestHeader.h index 7a10e22dfd802..a32acff97501b 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/APINotes/TestHeader.h +++ b/interpreter/CppInterOp/unittests/CppInterOp/APINotes/TestHeader.h @@ -11,4 +11,19 @@ void* testOperatorNew(); void* testOperatorNewArr(); void* testNone(); void* testWeirdAttr(); + +void testDeallocFree(void* ptr); +void testDeallocDelete(void* ptr); +void testDeallocDeleteArr(void* ptr); +void testDeallocNone(void* ptr); +void testDeallocMulti(void* a, void* b); +void testDeallocWeirdAttr(void* ptr); + +class KlassNotes { +public: + void releaseArg(void* ptr); + void releaseSelf(); + void releaseBoth(void* ptr); + void releaseBare(void* ptr); +}; #endif diff --git a/interpreter/CppInterOp/unittests/CppInterOp/CAPITest.cpp b/interpreter/CppInterOp/unittests/CppInterOp/CAPITest.cpp index 97bc7e5229eda..88cea1b49ecc8 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/CAPITest.cpp +++ b/interpreter/CppInterOp/unittests/CppInterOp/CAPITest.cpp @@ -7,6 +7,8 @@ #include "gtest/gtest.h" +#include +#include #include #include @@ -18,6 +20,36 @@ TYPED_TEST(CppInterOpTest, CAPI_DeclareAndProcess) { EXPECT_EQ(0, cppinterop_Process("capi_var++;")); } +TYPED_TEST(CppInterOpTest, CAPI_GetBaseClassOffsetWithAddress) { + TestFixture::CreateInterpreter(); +#define Stringify(s) Stringifyx(s) +#define Stringifyx(...) #__VA_ARGS__ +#define CODE \ + struct CAPIBase { \ + int value; \ + }; \ + struct CAPIDerived : virtual CAPIBase { \ + int value; \ + }; + + CODE; + ASSERT_EQ(cppinterop_Declare(Stringify(CODE), false), 0); +#undef Stringifyx +#undef Stringify +#undef CODE + + struct Hidden : CAPIDerived { + std::array padding; + }; + Hidden hidden; + CAPIDerived* p = &hidden; + auto derived = cppinterop_GetScope("CAPIDerived", nullptr); + auto base = cppinterop_GetScope("CAPIBase", nullptr); + int64_t expected = (char*)(CAPIBase*)p - (char*)p; + ASSERT_NE(cppinterop_GetBaseClassOffset(derived, base, nullptr), expected); + EXPECT_EQ(cppinterop_GetBaseClassOffset(derived, base, p), expected); +} + TYPED_TEST(CppInterOpTest, CAPI_LoadLibrary) { TestFixture::CreateInterpreter(); // The hand-written two-argument wrapper still exists and forwards. diff --git a/interpreter/CppInterOp/unittests/CppInterOp/CAPITestC.c b/interpreter/CppInterOp/unittests/CppInterOp/CAPITestC.c index 02d4f474a638a..76d60cae57079 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/CAPITestC.c +++ b/interpreter/CppInterOp/unittests/CppInterOp/CAPITestC.c @@ -49,5 +49,6 @@ int capi_c_test_main(void) { (void)&cppinterop_InstantiateTemplate; (void)&cppinterop_HasTypeQualifier; (void)&cppinterop_GetLanguage; + (void)&cppinterop_GetBaseClassOffset; return 0; } diff --git a/interpreter/CppInterOp/unittests/CppInterOp/CMakeLists.txt b/interpreter/CppInterOp/unittests/CppInterOp/CMakeLists.txt index d6ebe6e43498c..78a4b8d153992 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/CMakeLists.txt +++ b/interpreter/CppInterOp/unittests/CppInterOp/CMakeLists.txt @@ -89,6 +89,15 @@ target_compile_definitions(CppInterOpTests PRIVATE "CPPINTEROP_SRC_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}/../..\"" ) +# cling reads its runtime modulemaps from the directory that holds +# cling/Interpreter/RuntimeUniverse.h. A test that runs with -fmodules +# passes this prefix as an include path. +if (CPPINTEROP_USE_CLING) + target_compile_definitions(CppInterOpTests PRIVATE + "CPPINTEROP_CLING_INCLUDE_DIR=\"${CLING_INSTALL_PREFIX}/include\"" + ) +endif() + set_source_files_properties(InterpreterTest.cpp PROPERTIES COMPILE_DEFINITIONS "LLVM_BINARY_DIR=\"${LLVM_BINARY_DIR}\"" ) diff --git a/interpreter/CppInterOp/unittests/CppInterOp/DiagnosticTest.cpp b/interpreter/CppInterOp/unittests/CppInterOp/DiagnosticTest.cpp index 8015454369ee5..a25f7226f25aa 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/DiagnosticTest.cpp +++ b/interpreter/CppInterOp/unittests/CppInterOp/DiagnosticTest.cpp @@ -117,8 +117,8 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, Consumer_CapturesParseError) { ASSERT_EQ(Cpp::GetPendingDiagnosticCount(), 0U); // Trigger a parse error. silent=false lets the diagnostic reach the - // consumer; silent=true sets SuppressAllDiagnostics, which short- - // circuits before the consumer chain. + // consumer; silent=true swaps the engine's client for a discarding one, + // so the consumer chain never sees it. EXPECT_NE(0, Cpp::Declare("int err = ;", /*silent=*/false)); ASSERT_GT(Cpp::GetPendingDiagnosticCount(), 0U); @@ -145,13 +145,35 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, Consumer_OkDeclareLeavesBufferEmpty) { Cpp::ClearPendingDiagnostics(); } -// silent=true sets SuppressAllDiagnostics on the engine, which -// short-circuits before any consumer in the chain runs. Pin this -// contract: a failed declare under silent=true reports failure via -// the return code but the consumer captures nothing. +// silent=true unplugs the whole consumer chain for the duration of the +// declare, so nothing reaches the capturing consumer. Pin this contract: +// a failed declare under silent=true reports failure via the return code +// but the consumer captures nothing. TYPED_TEST(CPPINTEROP_TEST_MODE, Consumer_SilentSuppressesCapture) { TestFixture::CreateInterpreter(); Cpp::ClearPendingDiagnostics(); EXPECT_NE(0, Cpp::Declare("int err = ;", /*silent=*/true)); EXPECT_EQ(Cpp::GetPendingDiagnosticCount(), 0U); } + +TYPED_TEST(CPPINTEROP_TEST_MODE, Consumer_SilentDeclareSurvivesFailedCodegen) { + TestFixture::CreateInterpreter(); + + EXPECT_EQ(0, Cpp::Declare(R"( + namespace SilentCodegen { + struct Event { int num; union Shared { double a; int b; } shrd; }; + })")); + + // compiling bad code + EXPECT_NE(0, Cpp::Declare(R"( + namespace SilentCodegen { + void init(Shared** self, const double& a = 0, const int& b = 0) { + *self = new Shared{a, b}; + } + })", + /*silent=*/true)); + + // The interpreter is still usable afterwards. + EXPECT_EQ(0, Cpp::Declare("int after_failed_probe = 42;")); + EXPECT_TRUE(Cpp::GetNamed("after_failed_probe")); +} diff --git a/interpreter/CppInterOp/unittests/CppInterOp/DynamicLibraryManagerTest.cpp b/interpreter/CppInterOp/unittests/CppInterOp/DynamicLibraryManagerTest.cpp index 51b55c3860f4b..193073c9563be 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/DynamicLibraryManagerTest.cpp +++ b/interpreter/CppInterOp/unittests/CppInterOp/DynamicLibraryManagerTest.cpp @@ -67,10 +67,6 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, DynamicLibraryManager_Sanity) { GTEST_SKIP() << "Test fails for Emscipten builds"; #endif -#if CLANG_VERSION_MAJOR == 20 && defined(CPPINTEROP_USE_CLING) && \ - defined(_WIN32) - GTEST_SKIP() << "Test fails with Cling on Windows"; -#endif if (TypeParam::isOutOfProcess) GTEST_SKIP() << "Test fails for OOP JIT builds"; @@ -517,11 +513,7 @@ constexpr const char* kProcessGuard = static std::unique_ptr MakeElfModule(llvm::LLVMContext& Ctx) { auto M = std::make_unique("bindProcessWeakGlobals", Ctx); llvm::Triple HostTriple(llvm::sys::getProcessTriple()); -#if CLANG_VERSION_MAJOR < 21 - M->setTargetTriple(HostTriple.str()); -#else M->setTargetTriple(HostTriple); -#endif return M; } diff --git a/interpreter/CppInterOp/unittests/CppInterOp/FunctionReflectionTest.cpp b/interpreter/CppInterOp/unittests/CppInterOp/FunctionReflectionTest.cpp index 80fdec03f0f12..8470c8b5f73a1 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/FunctionReflectionTest.cpp +++ b/interpreter/CppInterOp/unittests/CppInterOp/FunctionReflectionTest.cpp @@ -194,25 +194,13 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, GetAllTopLevelDecls(code, Decls); - std::vector derived_methods; - Cpp::GetClassMethods(Decls[1], derived_methods); - - // Find the using-promoted foo (the two-argument overload). - bool found_using_promoted = false; - Cpp::FuncRef using_promoted; - for (auto m : derived_methods) { - if (Cpp::GetName(Cpp::DeclRef{m.data}) != "foo") - continue; - if (Cpp::GetFunctionNumArgs(m) == 2) { - found_using_promoted = true; - using_promoted = m; - EXPECT_TRUE(Cpp::IsPublicMethod(m)); - EXPECT_FALSE(Cpp::IsProtectedMethod(m)); - EXPECT_FALSE(Cpp::IsConstructor(m)); - } - } - EXPECT_TRUE(found_using_promoted) + // The using-promoted foo (the two-argument overload). + Cpp::FuncRef using_promoted = FindMethod(Decls[1], "foo", 2); + ASSERT_TRUE(using_promoted) << "using-promoted base method missing from GetClassMethods"; + EXPECT_TRUE(Cpp::IsPublicMethod(using_promoted)); + EXPECT_FALSE(Cpp::IsProtectedMethod(using_promoted)); + EXPECT_FALSE(Cpp::IsConstructor(using_promoted)); // Resolving the address of the using-promoted overload must transparently // unwrap the using-shadow to its target before emitting code. This is the @@ -220,15 +208,8 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, // (the reflection-only APIs above all go through the const overload), and the // resolved address must match the one obtained directly from the base method. #ifndef _WIN32 // GetFunctionAddress is disabled on Windows; see its own test. - if (!TypeParam::isOutOfProcess && found_using_promoted) { - std::vector base_methods; - Cpp::GetClassMethods(Decls[0], base_methods); - Cpp::FuncRef base_foo; - for (auto m : base_methods) { - if (Cpp::GetName(Cpp::DeclRef{m.data}) == "foo" && - Cpp::GetFunctionNumArgs(m) == 2) - base_foo = m; - } + if (!TypeParam::isOutOfProcess) { + Cpp::FuncRef base_foo = FindMethod(Decls[0], "foo", 2); ASSERT_TRUE(base_foo); void* shadow_addr = Cpp::GetFunctionAddress(using_promoted); @@ -237,19 +218,10 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, } #endif - std::vector hidden_methods; - Cpp::GetClassMethods(Decls[2], hidden_methods); - - bool found_hidden = false; - for (auto m : hidden_methods) { - if (Cpp::GetName(Cpp::DeclRef{m.data}) == "foo" && - Cpp::GetFunctionNumArgs(m) == 2) { - found_hidden = true; - EXPECT_FALSE(Cpp::IsPublicMethod(m)); - EXPECT_TRUE(Cpp::IsProtectedMethod(m)); - } - } - EXPECT_TRUE(found_hidden); + Cpp::FuncRef hidden = FindMethod(Decls[2], "foo", 2); + ASSERT_TRUE(hidden); + EXPECT_FALSE(Cpp::IsPublicMethod(hidden)); + EXPECT_TRUE(Cpp::IsProtectedMethod(hidden)); } // Companion to the access test above, covering the *call* path: a method @@ -287,16 +259,8 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, GetAllTopLevelDecls(code, Decls, /*filter_implicitGenerated=*/false, /*interpreter_args=*/{"-include", "new"}); - std::vector derived_methods; - Cpp::GetClassMethods(Decls[1], derived_methods); - - // Locate the using-promoted foo (the two-argument overload from the base). - Cpp::FuncRef using_promoted; - for (auto m : derived_methods) { - if (Cpp::GetName(Cpp::DeclRef{m.data}) == "foo" && - Cpp::GetFunctionNumArgs(m) == 2) - using_promoted = m; - } + // The using-promoted foo (the two-argument overload from the base). + Cpp::FuncRef using_promoted = FindMethod(Decls[1], "foo", 2); ASSERT_TRUE(using_promoted); // Compiling this wrapper exercises the relaxAccessControl path: the generated @@ -403,16 +367,8 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, GetAllTopLevelDecls(code, Decls, /*filter_implicitGenerated=*/false, /*interpreter_args=*/{"-include", "new"}); - std::vector derived_methods; - Cpp::GetClassMethods(Decls[2], derived_methods); - - // Locate the using-imported set_value (the one-argument overload). - Cpp::FuncRef imported; - for (auto m : derived_methods) { - if (Cpp::GetName(Cpp::DeclRef{m.data}) == "set_value" && - Cpp::GetFunctionNumArgs(m) == 1) - imported = m; - } + // The using-imported set_value (the one-argument overload). + Cpp::FuncRef imported = FindMethod(Decls[2], "set_value", 1); ASSERT_TRUE(imported); // The declaring scope of the imported method is SecondBase, not Derived. @@ -440,13 +396,7 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, std::array args = {(void*)&v}; SetValue.Invoke(nullptr, {args.data(), /*args_size=*/1}, second_base); - std::vector second_base_methods; - Cpp::GetClassMethods(Decls[1], second_base_methods); - Cpp::FuncRef get_value; - for (auto m : second_base_methods) { - if (Cpp::GetName(Cpp::DeclRef{m.data}) == "get_value") - get_value = m; - } + Cpp::FuncRef get_value = FindMethod(Decls[1], "get_value"); ASSERT_TRUE(get_value); int result = 0; @@ -455,13 +405,7 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, // The FirstBase subobject must be untouched (no write through an // unadjusted pointer). - std::vector first_base_methods; - Cpp::GetClassMethods(Decls[0], first_base_methods); - Cpp::FuncRef get_a; - for (auto m : first_base_methods) { - if (Cpp::GetName(Cpp::DeclRef{m.data}) == "get_a") - get_a = m; - } + Cpp::FuncRef get_a = FindMethod(Decls[0], "get_a"); ASSERT_TRUE(get_a); long long a = 0; @@ -1006,18 +950,32 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_IsAllocator) { EXPECT_EQ(Cpp::IsAllocator(Cpp::ConstFuncRef{Cpp::GetNamed("Klass").data}), Cpp::AllocType::Unknown); + const bool has_modulemaps = TestUtils::HasRuntimeModulemaps(); + Cpp::DeleteInterpreter(); #ifdef EMSCRIPTEN GTEST_SKIP() << "Test fails for Emscipten builds"; #endif std::string include_flag; // APINotes check -#ifndef CPPINTEROP_USE_CLING +#if defined(CPPINTEROP_USE_CLING) && !defined(__linux__) + // We support modules with cling on Linux only + GTEST_SKIP() << "Test runs only on Linux in cling"; +#endif + if (!has_modulemaps) + GTEST_SKIP() << "cling was built without its runtime modulemaps"; include_flag = "-I" + std::string(CPPINTEROP_SRC_DIR) + "/unittests/CppInterOp/APINotes"; std::vector interpreter_args = { "-fmodules", "-fimplicit-module-maps", "-fapinotes-modules", include_flag.c_str()}; +#ifdef CPPINTEROP_USE_CLING + // cling looks up module.modulemap next to cling/Interpreter/ + // RuntimeUniverse.h. This include path makes that header reachable. + std::string cling_include_flag = + "-I" + std::string(CPPINTEROP_CLING_INCLUDE_DIR); + interpreter_args.push_back(cling_include_flag.c_str()); +#endif TestFixture::CreateInterpreter(interpreter_args); code = R"( #include "TestHeader.h" @@ -1035,7 +993,6 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_IsAllocator) { TESTIA(testWeirdAttr, Cpp::AllocType::Unknown); Cpp::DeleteInterpreter(); -#endif #undef TESTIA include_flag = "-I" + std::string(CPPINTEROP_SRC_DIR) + "/unittests/CppInterOp/APINotes"; @@ -1071,28 +1028,295 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_IsDeallocator) { std::vector Decls; std::string code = R"( class Klass{ - int val; + public: + int val = 0; + + // `this` is the only slot of a method without parameters. + __attribute__((annotate("cppDeallocNone"))) + int* getValAdress(){ + return &val; + } + + // `this` occupies source index 1 and labelling it is ignored. + __attribute__((annotate("cppDeallocDelete", 1))) + void releaseSelf(){ + delete this; + } + + // The label on `this` is dropped, `p` is source index 2. + __attribute__((annotate("cppDeallocDelete", 1), + annotate("cppDeallocFree", 2))) + void releaseBoth(void* p); + + // Clang rejects index 1 for the implicit object parameter, so `p` is + // source index 3. + void __attribute__((ownership_takes(malloc, 3))) + methodTakes(int n, void* p); + + // Labelling `this` is ignored on the swift_attr channel as well. + __attribute__((swift_attr("cppDeallocDelete_1"))) + void swiftSelf(void* p); + + // Static methods have no implicit object parameter, so they are counted + // like free functions and index 0 is never valid. + static void __attribute__((ownership_takes(malloc, 2))) + staticTakes(int n, void* p); + + static void __attribute__((annotate("cppDeallocFree", 1))) + staticFree(void* p); + + static void __attribute__((annotate("cppDeallocDelete", 0))) + staticZeroIndex(void* p); + + static void __attribute__((swift_attr("cppDeallocDeleteArr_1"))) + staticSwift(void* p); }; - __attribute__((ownership_takes(malloc, 1))) - void Deallocator(Klass* arg){ - delete arg; + + void Plain(); + + void __attribute__((ownership_takes(malloc, 1))) + Deallocator(void* p); + + void __attribute__((ownership_takes(malloc, 1, 3))) + TakesMulti(void* a, int n, void* b); + + void __attribute__((ownership_holds(malloc, 2))) + HoldsFunc(int n, void* p); + + void* __attribute__((ownership_returns(malloc, 1))) + ReturnsFunc(int size); + + void __attribute__((annotate("cppDeallocFree", 1))) + FreeFunc(void* p); + + void __attribute__((annotate("cppDeallocDelete", 1))) + DeleteFunc(Klass* p); + + void __attribute__((annotate("cppDeallocDeleteArr", 1))) + DeleteArrFunc(Klass* p); + + void __attribute__((annotate("cppDeallocNone", 1))) + NoneDeallocFunc(void* p); + + void __attribute__((annotate("cppDeallocFree", 1, 3))) + MultiFunc(void* a, int n, void* b); + + // Without an index the first parameter is labelled. + void __attribute__((annotate("cppDeallocFree"))) + BareFunc(void* p, int n); + + void __attribute__((annotate("unrelatedAttr", 1))) + UnrelatedFunc(void* p); + + void __attribute__((annotate("cppDeallocFree", 99))) + OutOfRangeFunc(void* p); + + void __attribute__((annotate("cppDeallocFree", 0))) + ZeroIndexFunc(void* p); + + void __attribute__((annotate("cppDeallocFree", -1))) + NegativeFunc(void* p); + + void __attribute__((annotate("cppDeallocFree", "self"))) + NotIntFunc(void* p); + + void __attribute__((annotate("cppDeallocFree", 1), + annotate("cppDeallocDelete", 1))) + LastWinsFunc(void* p); + + void __attribute__((swift_attr("cppDeallocFree_1"))) + SwiftFreeFunc(void* p); + + void __attribute__(( + swift_attr("returns_cppDeallocDelete_1 cppDeallocFree_2"))) + SwiftMultiFunc(Klass* a, void* b); + + void __attribute__((swift_attr("cppDeallocDeleteArr_1"))) + SwiftDeleteArrFunc(Klass* p); + + void __attribute__((swift_attr("cppDeallocNone_1"))) + SwiftNoneFunc(void* p); + + void __attribute__((swift_attr("cppDeallocFree_xyz"))) + SwiftBadIndexFunc(void* p); + + void __attribute__((swift_attr("cppDeallocFree_0"))) + SwiftZeroIndexFunc(void* p); + + void __attribute__((swift_attr("cppDeallocFree_99"))) + SwiftOutOfRangeFunc(void* p); + + void __attribute__((swift_attr("unrelatedSwiftAttr"))) + SwiftUnrelatedFunc(void* p); + + template + void __attribute__((annotate("cppDeallocDelete", 1))) + TemplatedFunc(T* p){ + delete p; } - void foo(); + + template + void __attribute__((annotate("cppDeallocFree", N))) + DependentFunc(void* p){} )"; - GetAllTopLevelDecls(code, Decls, true); - EXPECT_TRUE(Cpp::IsDeallocator(Decls[1])); - EXPECT_FALSE(Cpp::IsDeallocator(Decls[2])); + GetAllTopLevelDecls(code, Decls, true, + {"-std=c++17", "-include", "stdlib.h"}); + + using DT = Cpp::DeallocType; + // A slot carries the value of the source index one above it: for a function + // with an implicit object parameter slot 0 is `this` and the first real + // parameter is slot 1, otherwise the first parameter is slot 0. + // IsDeallocator appends its slots, so the vector starts out empty. A + // parameter without a deallocation attribute is reported as Opaque, which is + // what tells it apart from one labelled cppDeallocNone. +#define TESTIDD(DECL, ...) \ + { \ + std::vector
expected{__VA_ARGS__}; \ + std::vector
result; \ + EXPECT_TRUE(Cpp::IsDeallocator(Cpp::ConstFuncRef{DECL}, result)); \ + EXPECT_EQ(result, expected); \ + } +#define TESTID(N, ...) TESTIDD(Cpp::GetNamed(#N).data, __VA_ARGS__) +#define TESTIDM(SCOPE, N, ...) \ + TESTIDD(Cpp::GetNamed(#N, Cpp::GetNamed(#SCOPE)).data, __VA_ARGS__) + + // The builtin deallocator. + TESTID(free, DT::Free); + + // GCC's ownership attributes, whose indexes need no translation. + TESTID(Deallocator, DT::Free); + TESTID(TakesMulti, DT::Free, DT::Opaque, DT::Free); + TESTID(HoldsFunc, DT::Opaque, DT::Free); + TESTID(ReturnsFunc, DT::Opaque); + + // CppInterOp's own annotate spelling. + TESTID(FreeFunc, DT::Free); + TESTID(DeleteFunc, DT::Delete); + TESTID(DeleteArrFunc, DT::DeleteArr); + // A slot labelled cppDeallocNone is None, not Opaque. + TESTID(NoneDeallocFunc, DT::None); + TESTID(MultiFunc, DT::Free, DT::Opaque, DT::Free); + TESTID(BareFunc, DT::Free, DT::Opaque); + + // Malformed or unrelated annotations label nothing. + TESTID(UnrelatedFunc, DT::Opaque); + TESTID(OutOfRangeFunc, DT::Opaque); + TESTID(ZeroIndexFunc, DT::Opaque); + TESTID(NegativeFunc, DT::Opaque); + TESTID(NotIntFunc, DT::Opaque); + // On a repeated index the last matching attribute wins. + TESTID(LastWinsFunc, DT::Delete); + + // swift_attr spelling, the channel API Notes feeds. + TESTID(SwiftFreeFunc, DT::Free); + TESTID(SwiftMultiFunc, DT::Delete, DT::Free); + TESTID(SwiftDeleteArrFunc, DT::DeleteArr); + TESTID(SwiftNoneFunc, DT::None); + TESTID(SwiftBadIndexFunc, DT::Opaque); + TESTID(SwiftZeroIndexFunc, DT::Opaque); + TESTID(SwiftOutOfRangeFunc, DT::Opaque); + TESTID(SwiftUnrelatedFunc, DT::Opaque); + + // Methods: slot 0 is `this` and cannot be labelled. + TESTIDM(Klass, getValAdress, DT::Opaque); + TESTIDM(Klass, releaseSelf, DT::Opaque); + TESTIDM(Klass, releaseBoth, DT::Opaque, DT::Free); + TESTIDM(Klass, methodTakes, DT::Opaque, DT::Opaque, DT::Free); + TESTIDM(Klass, swiftSelf, DT::Opaque, DT::Opaque); + + // Static methods are counted like free functions. + TESTIDM(Klass, staticTakes, DT::Opaque, DT::Free); + TESTIDM(Klass, staticFree, DT::Free); + TESTIDM(Klass, staticZeroIndex, DT::Opaque); + TESTIDM(Klass, staticSwift, DT::DeleteArr); + + // Templates are unwrapped to their pattern; a value-dependent index cannot + // be folded, therefore it is ignored. GetNamed does not resolve function + // template names, so the two trailing top level decls of the fixture are + // used. + Decl* templatedFunc = Decls[Decls.size() - 2]; + Decl* dependentFunc = Decls[Decls.size() - 1]; + TESTIDD(templatedFunc, DT::Delete); + TESTIDD(dependentFunc, DT::Opaque); + + ASTContext& C = Interp->getCI()->getASTContext(); + std::vector charArg = {C.CharTy.getAsOpaquePtr()}; + TESTIDD(Cpp::InstantiateTemplate(templatedFunc, charArg).data, DT::Delete); + + // After instantiation the index is no longer value-dependent, so it folds. + std::vector oneArg = {{C.IntTy.getAsOpaquePtr(), "1"}}; + TESTIDD(Cpp::InstantiateTemplate(dependentFunc, oneArg).data, DT::Free); + + // A function without parameters contributes no slot at all. + { + std::vector
result; + EXPECT_TRUE(Cpp::IsDeallocator( + Cpp::ConstFuncRef{Cpp::GetNamed("Plain").data}, result)); + EXPECT_TRUE(result.empty()); + } + + // The slots are appended, so what the vector already holds is left alone. + // Unknown is used as the marker because Opaque is what the function itself + // writes into an unlabelled slot. + { + std::vector
result(2, DT::Unknown); + EXPECT_TRUE(Cpp::IsDeallocator( + Cpp::ConstFuncRef{Cpp::GetNamed("MultiFunc").data}, result)); + EXPECT_EQ(result, (std::vector
{DT::Unknown, DT::Unknown, DT::Free, + DT::Opaque, DT::Free})); + + // Appending again grows the vector by the next function's slot count. + EXPECT_TRUE(Cpp::IsDeallocator( + Cpp::ConstFuncRef{Cpp::GetNamed("FreeFunc").data}, result)); + EXPECT_EQ(result, (std::vector
{DT::Unknown, DT::Unknown, DT::Free, + DT::Opaque, DT::Free, DT::Free})); + } + // Casting coverage: a non-function declaration cannot be analyzed, and the + // vector is left untouched. + { + std::vector
result{DT::Unknown}; + EXPECT_FALSE(Cpp::IsDeallocator( + Cpp::ConstFuncRef{Cpp::GetNamed("Klass").data}, result)); + EXPECT_EQ(result, (std::vector
{DT::Unknown})); + } + + Cpp::DeleteInterpreter(); +#ifdef EMSCRIPTEN + GTEST_SKIP() << "Test fails for Emscipten builds"; +#endif + // APINotes check +#ifndef CPPINTEROP_USE_CLING + std::string include_flag = + "-I" + std::string(CPPINTEROP_SRC_DIR) + "/unittests/CppInterOp/APINotes"; + std::vector interpreter_args = { + "-fmodules", "-fimplicit-module-maps", "-fapinotes-modules", + include_flag.c_str()}; + TestFixture::CreateInterpreter(interpreter_args); code = R"( - #include - void test(){ - //Do Nothing - } + #include "TestHeader.h" )"; - TestFixture::CreateInterpreter(); Interp->process(code); - auto freeDecl = Cpp::GetNamed("free"); - EXPECT_TRUE(Cpp::IsDeallocator(Cpp::ConstFuncRef{freeDecl.data})); + + TESTID(testDeallocFree, DT::Free); + TESTID(testDeallocDelete, DT::Delete); + TESTID(testDeallocDeleteArr, DT::DeleteArr); + TESTID(testDeallocNone, DT::None); + TESTID(testDeallocMulti, DT::Delete, DT::Free); + TESTID(testDeallocWeirdAttr, DT::Opaque); + + // Methods carried through API Notes; `this` stays unlabellable there too. + TESTIDM(KlassNotes, releaseArg, DT::Opaque, DT::Free); + TESTIDM(KlassNotes, releaseSelf, DT::Opaque); + TESTIDM(KlassNotes, releaseBoth, DT::Opaque, DT::Free); + // Without an index `this` is skipped and the first parameter is labelled. + TESTIDM(KlassNotes, releaseBare, DT::Opaque, DT::Delete); + + Cpp::DeleteInterpreter(); +#endif +#undef TESTIDM +#undef TESTID +#undef TESTIDD } TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_GetFunctionNumArgs) { @@ -3169,9 +3393,6 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_IsVirtualMethod) { } TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_JitCallAdvanced) { -#if CLANG_VERSION_MAJOR == 20 && defined(CPPINTEROP_USE_CLING) && defined(_WIN32) - GTEST_SKIP() << "Test fails with Cling on Windows"; -#endif #ifdef EMSCRIPTEN #if CLANG_VERSION_MAJOR > 21 GTEST_SKIP() << "Test fails for Emscipten builds using LLVM 22"; @@ -4200,7 +4421,12 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_Construct) { // The wrappers behind Construct and by-value returns placement-new into a // caller-provided buffer. A class-scope operator new with no placement form // hides the global `operator new(size_t, void*)`, so those wrappers only -// compile if they spell it `::new (buf) C(...)`. +// compile if they spell it `::new (buf) C(...)`. That is also the standard +// library's construct-at contract ([specialized.construct]): construction into +// a caller-provided buffer never routes through a class-scope *placement* +// operator new (which unqualified `new (buf)` would pick), while plain heap +// construction still goes through the user's class-scope allocator. The +// counters in WithPlacementNew make the choice observable. TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_ConstructClassScopeNew) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; @@ -4212,7 +4438,6 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_ConstructClassScopeNew) { #endif if (TypeParam::isOutOfProcess) GTEST_SKIP() << "Test fails for OOP JIT builds"; - std::vector interpreter_args = {"-include", "new"}; std::vector Decls; std::string code = R"( @@ -4228,9 +4453,28 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_ConstructClassScopeNew) { static void operator delete[](void* p) { ::operator delete[](p); } }; WithClassNew MakeWithClassNew() { return WithClassNew(); } + + int heap_news = 0; // calls to the class-scope operator new(size_t) + int placement_news = 0; // calls to the class-scope placement form + class WithPlacementNew { + public: + int x; + WithPlacementNew() : x(7) {} + WithPlacementNew Clone() { return WithPlacementNew(); } + static void* operator new(__SIZE_TYPE__ sz) { + ++heap_news; + return ::operator new(sz); + } + static void* operator new(__SIZE_TYPE__, void* where) { + ++placement_news; + return where; + } + static void operator delete(void* p) { ::operator delete(p); } + static void operator delete(void*, void*) {} + }; )"; - GetAllTopLevelDecls(code, Decls, false, interpreter_args); + GetAllTopLevelDecls(code, Decls, false, {"-include", "new"}); Cpp::DeclRef scope = Cpp::GetNamed("WithClassNew"); ASSERT_TRUE(scope); @@ -4270,59 +4514,13 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_ConstructClassScopeNew) { int result = 0; // WithClassNew's layout is a single int JC.Invoke(&result); EXPECT_EQ(result, 42); -} - -// Pins down which operator new the wrappers pick when the class-scope forms -// are all accessible, including a class-scope *placement* operator new. The -// contract is the standard library's construct-at contract -// ([specialized.construct]): construction into a caller-provided buffer is -// spelled `::new (buf) C(...)`, so it constructs at `buf` directly and never -// routes through a class-scope placement operator new (which unqualified -// `new (buf)` would pick). Plain heap construction still goes through the -// user's class-scope allocator. The counters make the choice observable. -TYPED_TEST(CPPINTEROP_TEST_MODE, - FunctionReflection_ConstructClassScopePlacementNew) { -#ifdef _WIN32 - GTEST_SKIP() << "Disabled on Windows. Needs fixing."; -#endif -#ifdef EMSCRIPTEN -#if CLANG_VERSION_MAJOR > 21 - GTEST_SKIP() << "Test fails for Emscipten builds using LLVM 22"; -#endif -#endif - if (TypeParam::isOutOfProcess) - GTEST_SKIP() << "Test fails for OOP JIT builds"; - std::vector interpreter_args = {"-include", "new"}; - std::vector Decls; - std::string code = R"( - int heap_news = 0; // calls to the class-scope operator new(size_t) - int placement_news = 0; // calls to the class-scope placement form - class WithPlacementNew { - public: - int x; - WithPlacementNew() : x(7) {} - WithPlacementNew Clone() { return WithPlacementNew(); } - static void* operator new(__SIZE_TYPE__ sz) { - ++heap_news; - return ::operator new(sz); - } - static void* operator new(__SIZE_TYPE__, void* where) { - ++placement_news; - return where; - } - static void operator delete(void* p) { ::operator delete(p); } - static void operator delete(void*, void*) {} - }; - )"; - - GetAllTopLevelDecls(code, Decls, false, interpreter_args); - Cpp::DeclRef scope = Cpp::GetNamed("WithPlacementNew"); + scope = Cpp::GetNamed("WithPlacementNew"); ASSERT_TRUE(scope); // Heap construction (the wrapper's non-arena `new C(...)`) must keep // honoring the user's class-scope allocator. - Cpp::ObjectRef object = Cpp::Construct(scope); + object = Cpp::Construct(scope); ASSERT_TRUE(object); EXPECT_EQ(*static_cast(object.data), 7); EXPECT_EQ(Cpp::Evaluate("heap_news").unbox(), 1); @@ -4330,7 +4528,7 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, // Construction into a caller-provided buffer bypasses the class-scope // placement operator new, like std::construct_at does. - void* where = Cpp::Allocate(scope).data; + where = Cpp::Allocate(scope).data; ASSERT_TRUE(where); EXPECT_TRUE(where == Cpp::Construct(scope, where).data); EXPECT_EQ(*static_cast(where), 7); @@ -4341,10 +4539,10 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, // A JitCall to a method of the class with a by-value result: the wrapper // stores the result into the caller's buffer with `::new (ret)`, so the // class-scope placement operator new stays out of the call path here too. - Cpp::JitCall JC = Cpp::MakeFunctionCallable( + JC = Cpp::MakeFunctionCallable( Cpp::FuncRef{Cpp::GetNamed("Clone", scope).data}); ASSERT_TRUE(JC.getKind() == Cpp::JitCall::kGenericCall); - int result = 0; // WithPlacementNew's layout is a single int + result = 0; // WithPlacementNew's layout is a single int JC.Invoke(&result, {}, object.data); EXPECT_EQ(result, 7); EXPECT_EQ(Cpp::Evaluate("heap_news").unbox(), 1); @@ -4657,10 +4855,6 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_UndoTest) { #ifdef _WIN32 GTEST_SKIP() << "Disabled on Windows. Needs fixing."; #endif -#if CLANG_VERSION_MAJOR == 20 && defined(CPPINTEROP_USE_CLING) && \ - defined(__APPLE__) - GTEST_SKIP() << "Disabled on osx for cling based on llvm 20. Needs fixing."; -#endif #if defined(CPPINTEROP_USE_CLING) GTEST_SKIP() << "cling unload walks a module already freed by ORC " "clone-on-emit; skip until the cling-side fix lands."; @@ -4858,6 +5052,92 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_IsExplicitDeductionGuide) { "Wrapper"); } +TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_IsOperator) { + std::vector Decls, BaseDecls, DerivedDecls; + std::string code = R"( + struct OpBase { + OpBase& operator+=(const OpBase&) { return *this; } + operator int() const { return 0; } + void method() {} + }; + struct OpDerived : OpBase { + using OpBase::operator+=; + }; + bool operator==(const OpBase&, const OpBase&) { return true; } + template bool operator<(const T&, const OpBase&) { + return true; + } + template void free_template(T) {} + void free_function() {} + )"; + + GetAllTopLevelDecls(code, Decls); + ASSERT_EQ(Decls.size(), 6U); + GetAllSubDecls(Decls[0], BaseDecls, /*filter_implicitGenerated=*/true); + ASSERT_EQ(BaseDecls.size(), 3U); + GetAllSubDecls(Decls[1], DerivedDecls); + + EXPECT_TRUE(Cpp::IsOperator(BaseDecls[0])); // operator+= + EXPECT_FALSE(Cpp::IsOperator(BaseDecls[1])); // operator int + EXPECT_FALSE(Cpp::IsOperator(BaseDecls[2])); // method + + // using OpBase::operator+=; + Decl* Shadow = nullptr; + for (auto* D : DerivedDecls) + if (llvm::isa(D)) + Shadow = D; + ASSERT_TRUE(Shadow); + EXPECT_TRUE(Cpp::IsOperator(Shadow)); + + EXPECT_FALSE(Cpp::IsOperator(Decls[0])); // not a function + EXPECT_TRUE(Cpp::IsOperator(Decls[2])); // operator== + EXPECT_TRUE(Cpp::IsOperator(Decls[3])); // template operator< + EXPECT_FALSE(Cpp::IsOperator(Decls[4])); // free_template + EXPECT_FALSE(Cpp::IsOperator(Decls[5])); // free_function + EXPECT_FALSE(Cpp::IsOperator(nullptr)); +} + +TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_IsConversionOperator) { + std::vector Decls, BaseDecls, DerivedDecls; + std::string code = R"( + struct ConvBase { + operator int() const { return 0; } + explicit operator bool() const { return true; } + template operator T*() const { return nullptr; } + bool operator!() const { return false; } + void method() {} + }; + struct ConvDerived : ConvBase { + using ConvBase::operator int; + }; + void free_function() {} + )"; + + GetAllTopLevelDecls(code, Decls); + ASSERT_EQ(Decls.size(), 3U); + GetAllSubDecls(Decls[0], BaseDecls, /*filter_implicitGenerated=*/true); + ASSERT_EQ(BaseDecls.size(), 5U); + GetAllSubDecls(Decls[1], DerivedDecls); + + EXPECT_TRUE(Cpp::IsConversionOperator(BaseDecls[0])); // operator int + EXPECT_TRUE(Cpp::IsConversionOperator(BaseDecls[1])); // operator bool + EXPECT_TRUE(Cpp::IsConversionOperator(BaseDecls[2])); // operator T* + EXPECT_FALSE(Cpp::IsConversionOperator(BaseDecls[3])); // operator! + EXPECT_FALSE(Cpp::IsConversionOperator(BaseDecls[4])); // method + + // using ConvBase::operator int; + Decl* Shadow = nullptr; + for (auto* D : DerivedDecls) + if (llvm::isa(D)) + Shadow = D; + ASSERT_TRUE(Shadow); + EXPECT_TRUE(Cpp::IsConversionOperator(Shadow)); + + EXPECT_FALSE(Cpp::IsConversionOperator(Decls[0])); // not a function + EXPECT_FALSE(Cpp::IsConversionOperator(Decls[2])); // free_function + EXPECT_FALSE(Cpp::IsConversionOperator(nullptr)); +} + // C++23 "deducing this" (explicit object parameters, P0847R7): the object // parameter binds to the receiver, not the argument list. TYPED_TEST(CPPINTEROP_TEST_MODE, FunctionReflection_DeducingThisIntrospection) { diff --git a/interpreter/CppInterOp/unittests/CppInterOp/InterpreterTest.cpp b/interpreter/CppInterOp/unittests/CppInterOp/InterpreterTest.cpp index 8b756025732db..eb08cbf816efd 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/InterpreterTest.cpp +++ b/interpreter/CppInterOp/unittests/CppInterOp/InterpreterTest.cpp @@ -347,7 +347,7 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, Interpreter_StaticDtorsRunOnDelete) { // ShutdownDoesNotMaterializeAgainstDestroyedGlobals at the CppInterOp // wrapper level: two interpreters each register a static with a // non-trivial dtor, then std::exit drives the C++ static-dtor phase. -// On LLVM 23+ our InterpreterShutdown calls llvm_shutdown which fires +// On LLVM 24+ our InterpreterShutdown calls llvm_shutdown which fires // ~InterpreterInfo -> ~Interpreter -> JIT deinit; the deinit lookup // skips lazy materialization (Platform::lookupResolvedInitSymbols, // llvm/llvm-project#196874) and the process exits cleanly. @@ -365,8 +365,8 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, #endif if (TypeParam::isOutOfProcess) GTEST_SKIP() << "Test fails for OOP JIT builds"; -#if LLVM_VERSION_MAJOR <= 22 - GTEST_SKIP() << "Requires LLVM 23+ (lookupResolvedInitSymbols)"; +#if LLVM_VERSION_MAJOR <= 23 + GTEST_SKIP() << "Requires LLVM 24+ (lookupResolvedInitSymbols)"; #else EXPECT_EXIT( { diff --git a/interpreter/CppInterOp/unittests/CppInterOp/ScopeReflectionTest.cpp b/interpreter/CppInterOp/unittests/CppInterOp/ScopeReflectionTest.cpp index e4ba31e13f61e..749b6eba053c3 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/ScopeReflectionTest.cpp +++ b/interpreter/CppInterOp/unittests/CppInterOp/ScopeReflectionTest.cpp @@ -8,12 +8,12 @@ #include "clang/AST/ASTDumper.h" #include "clang/AST/Decl.h" #include "clang/AST/DeclBase.h" +#include "clang/AST/ExternalASTSource.h" #include "clang/AST/GlobalDecl.h" #include "clang/AST/Type.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/Version.h" #include "clang/Frontend/CompilerInstance.h" -#include "clang/Sema/ExternalSemaSource.h" #include "clang/Sema/Sema.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" @@ -21,6 +21,8 @@ #include "gtest/gtest.h" #include +#include +#include #include #include #include @@ -1035,6 +1037,46 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, ScopeReflection_GetNamedWithUsing) { << "anonymous-namespace member should be visible at TU scope"; } +TYPED_TEST(CPPINTEROP_TEST_MODE, ScopeReflection_IsVisibleName) { + std::string code = R"( + namespace N { + int x; + void f(int); + void f(double); + } + namespace P { using namespace N; } + namespace Q { int y; } + using namespace P; + int z; + )"; + + TestFixture::CreateInterpreter(); + Interp->declare(code); + + Cpp::DeclRef ns_N = Cpp::GetNamed("N"); + Cpp::DeclRef ns_P = Cpp::GetNamed("P"); + Cpp::DeclRef ns_Q = Cpp::GetNamed("Q"); + ASSERT_TRUE(ns_N); + ASSERT_TRUE(ns_P); + ASSERT_TRUE(ns_Q); + + // Direct members, including overload sets. + EXPECT_TRUE(Cpp::IsVisibleName("x", ns_N)); + EXPECT_TRUE(Cpp::IsVisibleName("f", ns_N)); + // Through a using-directive, also from the global scope, where the + // directive is transitive (global -> P -> N). + EXPECT_TRUE(Cpp::IsVisibleName("x", ns_P)); + EXPECT_TRUE(Cpp::IsVisibleName("x")); + EXPECT_TRUE(Cpp::IsVisibleName("f", Cpp::GetGlobalScope())); + // Enclosing scopes are searched too. + EXPECT_TRUE(Cpp::IsVisibleName("z", ns_Q)); + // Names not reachable by unqualified lookup. + EXPECT_FALSE(Cpp::IsVisibleName("y")); + EXPECT_FALSE(Cpp::IsVisibleName("y", ns_N)); + EXPECT_FALSE(Cpp::IsVisibleName("does_not_exist")); + EXPECT_FALSE(Cpp::IsVisibleName("does_not_exist", ns_P)); +} + TYPED_TEST(CPPINTEROP_TEST_MODE, ScopeReflection_GetParentScope) { std::string code = R"(namespace N1 { namespace N2 { @@ -1216,7 +1258,7 @@ namespace { // https://github.com/cms-analysis/HiggsAnalysis-CombinedLimit/issues/1289, // where cppjit built the Python proxy from the incomplete declaration and // silently dropped the entire base hierarchy. -class LazyDefinitionSource : public clang::ExternalSemaSource { +class LazyDefinitionSource : public clang::ExternalASTSource { public: explicit LazyDefinitionSource(std::map Defs) : Pending(std::move(Defs)) {} @@ -1433,6 +1475,103 @@ CODE; (char*)(A*)g.get() - (char*)g.get()); } +TYPED_TEST(CPPINTEROP_TEST_MODE, ScopeReflection_GetBaseClassOffsetWithAddress) { + std::vector Decls; + // Layout fixtures own no resources and do not need custom copy/move members. + // NOLINTBEGIN(cppcoreguidelines-special-member-functions) +#define Stringify(s) Stringifyx(s) +#define Stringifyx(...) #__VA_ARGS__ +#define CODE \ + struct VB { \ + int m_vb; \ + }; \ + struct IC { \ + int m_ic; \ + }; \ + struct Pad { \ + std::array padding; \ + }; \ + struct M : Pad, IC { \ + int m_m; \ + }; \ + struct Lead { \ + virtual ~Lead() = default; \ + std::array padding; \ + }; \ + struct VD2 : Lead, virtual VB { \ + int m_vd2; \ + }; \ + struct ID3 : Lead, virtual M { \ + int m_id3; \ + }; \ + struct NearlyEmpty { \ + virtual ~NearlyEmpty() = default; \ + virtual int id() const { return 42; } \ + }; \ + struct WithVCalls : virtual NearlyEmpty { \ + int value; \ + int id() const override { return 7; } \ + }; + + CODE; + // NOLINTEND(cppcoreguidelines-special-member-functions) + GetAllTopLevelDecls(Stringify(CODE), Decls, false, {"-include", "array"}); +#undef Stringifyx +#undef Stringify +#undef CODE + + // These complete types exist only in the host, not in the interpreter. + // Other also exists solely for its layout and owns no resources. + struct Other { // NOLINT(cppcoreguidelines-special-member-functions) + virtual ~Other() = default; + std::array padding; + }; + struct Hidden : Other, VD2 { + long value; + }; + struct Hidden3 : Other, ID3 { + long value; + }; + struct First : virtual NearlyEmpty {}; + struct HiddenCalls : First, WithVCalls { + long value; + [[nodiscard]] int id() const override { return WithVCalls::id(); } + }; + + Hidden hidden; + VD2* p = &hidden; + int64_t expected = (char*)(VB*)p - (char*)p; + ASSERT_NE(Cpp::GetBaseClassOffset(Decls[5], Decls[0]), expected); + EXPECT_EQ(Cpp::GetBaseClassOffset(Decls[5], Decls[0], p), expected); + + // The last part of ID3 -> virtual M -> IC has a nonzero non-virtual offset. + Hidden3 hidden3; + ID3* p3 = &hidden3; + int64_t expected_ic = (char*)(IC*)p3 - (char*)p3; + ASSERT_NE(Cpp::GetBaseClassOffset(Decls[6], Decls[1]), expected_ic); + EXPECT_EQ(Cpp::GetBaseClassOffset(Decls[6], Decls[1], p3), expected_ic); + + // Preserve the static behavior for ordinary bases and the same class. + EXPECT_EQ(Cpp::GetBaseClassOffset(Decls[5], Decls[4], p), + (char*)(Lead*)p - (char*)p); + EXPECT_EQ(Cpp::GetBaseClassOffset(Decls[5], Decls[5], p), 0); + + // A complete VD2 has a nonzero vbptr offset on Microsoft targets. + VD2 complete; + EXPECT_EQ(Cpp::GetBaseClassOffset(Decls[5], Decls[0], &complete), + Cpp::GetBaseClassOffset(Decls[5], Decls[0])); + EXPECT_EQ(Cpp::GetBaseClassOffset(Decls[5], Decls[0], &complete), + (char*)(VB*)&complete - (char*)&complete); + + // A nearly-empty virtual base introduces vcall slots on Itanium targets. + // It can precede the WithVCalls subobject, requiring a signed offset. + HiddenCalls hidden_calls; + WithVCalls* pc = &hidden_calls; + EXPECT_EQ(pc->id(), 7); + EXPECT_EQ(Cpp::GetBaseClassOffset(Decls[8], Decls[7], pc), + (char*)(NearlyEmpty*)pc - (char*)pc); +} + TYPED_TEST(CPPINTEROP_TEST_MODE, ScopeReflection_GetAllCppNames) { std::vector Decls; std::string code = R"( diff --git a/interpreter/CppInterOp/unittests/CppInterOp/TypeReflectionTest.cpp b/interpreter/CppInterOp/unittests/CppInterOp/TypeReflectionTest.cpp index 3fa88696c7d71..9a642bed830d0 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/TypeReflectionTest.cpp +++ b/interpreter/CppInterOp/unittests/CppInterOp/TypeReflectionTest.cpp @@ -592,11 +592,24 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, TypeReflection_IsTemplateParmType) { EXPECT_FALSE(Cpp::IsTemplateParmType(Cpp::GetFunctionArgType(Decls[0], 2))); } -TYPED_TEST(CPPINTEROP_TEST_MODE, TypeReflection_IsSmartPtrType) { -#if CLANG_VERSION_MAJOR == 20 && defined(CPPINTEROP_USE_CLING) && defined(_WIN32) - GTEST_SKIP() << "Test fails with Cling on Windows"; -#endif +TYPED_TEST(CPPINTEROP_TEST_MODE, TypeReflection_IsPackExpansionType) { + std::vector Decls; + + std::string code = R"( + template void f(T t, Ts... ts) {} + template void g(const Ts&... ts) {} + )"; + GetAllTopLevelDecls(code, Decls); + + EXPECT_FALSE(Cpp::IsPackExpansionType(Cpp::GetFunctionArgType(Decls[0], 0))); + EXPECT_TRUE(Cpp::IsPackExpansionType(Cpp::GetFunctionArgType(Decls[0], 1))); + EXPECT_TRUE(Cpp::IsPackExpansionType(Cpp::GetFunctionArgType(Decls[1], 0))); + EXPECT_FALSE(Cpp::IsPackExpansionType(Cpp::GetType("int"))); + EXPECT_FALSE(Cpp::IsPackExpansionType(nullptr)); +} + +TYPED_TEST(CPPINTEROP_TEST_MODE, TypeReflection_IsSmartPtrType) { std::vector interpreter_args = {"-include", "new"}; TestFixture::CreateInterpreter(interpreter_args); diff --git a/interpreter/CppInterOp/unittests/CppInterOp/Utils.cpp b/interpreter/CppInterOp/unittests/CppInterOp/Utils.cpp index a8c7d247f9a90..2192dd4a69d03 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/Utils.cpp +++ b/interpreter/CppInterOp/unittests/CppInterOp/Utils.cpp @@ -8,6 +8,7 @@ #include "clang/Sema/Lookup.h" #include "clang/Sema/Sema.h" +#include "llvm/Support/FileSystem.h" #include "llvm/TargetParser/Triple.h" #include @@ -28,6 +29,19 @@ std::vector GetInterpreterArgs( } return args; } + +bool HasRuntimeModulemaps() { +#ifdef CPPINTEROP_USE_CLING + std::vector include_paths; + Cpp::GetIncludePaths(include_paths, /*withSystem=*/true); + for (const std::string& dir : include_paths) + if (llvm::sys::fs::exists(dir + "/cling/Interpreter/RuntimeUniverse.h")) + return llvm::sys::fs::exists(dir + "/cling/std.modulemap"); + return false; +#else + return true; +#endif +} } void TestUtils::GetAllTopLevelDecls( diff --git a/interpreter/CppInterOp/unittests/CppInterOp/Utils.h b/interpreter/CppInterOp/unittests/CppInterOp/Utils.h index 72579d310a9ad..63781d26fda8a 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/Utils.h +++ b/interpreter/CppInterOp/unittests/CppInterOp/Utils.h @@ -5,6 +5,7 @@ #include "../../lib/CppInterOp/Unwrap.h" #include "CppInterOp/CppInterOp.h" +#include "CppInterOp/CppInterOpTypes.h" #define CPPINTEROP_TEST_MODE CppInterOpTest #include @@ -58,6 +59,27 @@ void GetAllTopLevelDecls(const std::string& code, const std::vector& interpreter_args = {}); void GetAllSubDecls(clang::Decl* D, std::vector& SubDecls, bool filter_implicitGenerated = false); + +// Whether the current interpreter can be restarted with -fmodules. cling +// mounts its runtime modulemaps (std.modulemap, ...) from the directory that +// holds cling/Interpreter/RuntimeUniverse.h; a build without them (e.g. ROOT +// with runtime_cxxmodules=OFF) aborts on -fmodules. Always true for +// clang-repl. Needs a running interpreter. +bool HasRuntimeModulemaps(); + +// The first method of `scope` called `name` (taking `nargs` arguments, if +// given), or a null FuncRef. Inline, so that test binaries that do not link +// Utils.cpp can use it. +inline Cpp::FuncRef FindMethod(Cpp::DeclRef scope, const std::string& name, + int nargs = -1) { + std::vector methods; + Cpp::GetClassMethods(scope, methods); + for (auto m : methods) + if (Cpp::GetName(Cpp::DeclRef{m.data}) == name && + (nargs < 0 || Cpp::GetFunctionNumArgs(m) == size_t(nargs))) + return m; + return nullptr; +} } // end namespace TestUtils bool IsTargetX86(); diff --git a/interpreter/CppInterOp/unittests/CppInterOp/VTableOverlayCrossTUBench.cpp b/interpreter/CppInterOp/unittests/CppInterOp/VTableOverlayCrossTUBench.cpp index 3d839dc6ede65..5ca2134e1c87e 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/VTableOverlayCrossTUBench.cpp +++ b/interpreter/CppInterOp/unittests/CppInterOp/VTableOverlayCrossTUBench.cpp @@ -47,12 +47,7 @@ Cpp::DeclRef ReflectOverlayBase() { } Cpp::FuncRef Frob(Cpp::DeclRef scope) { - std::vector methods; - Cpp::GetClassMethods(scope, methods); - for (auto m : methods) - if (Cpp::GetName(Cpp::DeclRef{m.data}) == "frob") - return m; - return nullptr; + return TestUtils::FindMethod(scope, "frob"); } Cpp::UniqueVTableOverlay OverlayFrob(void* inst, Cpp::DeclRef base) { diff --git a/interpreter/CppInterOp/unittests/CppInterOp/VTableOverlayTest.cpp b/interpreter/CppInterOp/unittests/CppInterOp/VTableOverlayTest.cpp index 848ffdab915a6..2d8c2db643a67 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/VTableOverlayTest.cpp +++ b/interpreter/CppInterOp/unittests/CppInterOp/VTableOverlayTest.cpp @@ -86,14 +86,7 @@ Cpp::DeclRef DeclareBase() { return Cpp::GetNamed("OverlayB"); } -Cpp::FuncRef Method(Cpp::DeclRef scope, const char* name) { - std::vector methods; - Cpp::GetClassMethods(scope, methods); - for (auto m : methods) - if (Cpp::GetName(Cpp::DeclRef{m.data}) == name) - return m; - return nullptr; -} +using TestUtils::FindMethod; // Dispatch through the installed vtable slot. Calling beta() directly in // this TU would let the compiler devirtualize and bypass the overlay; @@ -119,7 +112,7 @@ TEST(VTableOverlay, ReplacesSlotPreservingOthers) { void* inst = Cpp::Construct(B).data; ASSERT_NE(inst, nullptr); auto ov = Cpp::MakeUniqueVTableOverlay( - inst, B, {{Method(B, "beta"), MethodAddr(&Repl::negate)}}); + inst, B, {{FindMethod(B, "beta"), MethodAddr(&Repl::negate)}}); ASSERT_TRUE(ov); EXPECT_EQ(call_slot(inst, kBeta, 5), -5); // overlaid EXPECT_EQ(call_slot(inst, kAlpha, 5), 15); // preserved: alpha -> x+10 @@ -141,7 +134,7 @@ TEST(VTableOverlay, PreservesPrefixAndUnrelatedSlots) { #endif void* alpha_slot = aot[kAlpha]; auto ov = Cpp::MakeUniqueVTableOverlay( - inst, B, {{Method(B, "beta"), MethodAddr(&Repl::negate)}}); + inst, B, {{FindMethod(B, "beta"), MethodAddr(&Repl::negate)}}); ASSERT_TRUE(ov); void** now = *reinterpret_cast(inst); EXPECT_EQ(now[-1], prefix_m1); @@ -160,7 +153,7 @@ TEST(VTableOverlay, RestoresOnDestroy) { void* aot = *reinterpret_cast(inst); { auto ov = Cpp::MakeUniqueVTableOverlay( - inst, B, {{Method(B, "beta"), MethodAddr(&Repl::negate)}}); + inst, B, {{FindMethod(B, "beta"), MethodAddr(&Repl::negate)}}); ASSERT_TRUE(ov); EXPECT_NE(*reinterpret_cast(inst), aot); } @@ -175,8 +168,8 @@ TEST(VTableOverlay, ReplacesMultipleSlots) { ASSERT_NE(inst, nullptr); auto ov = Cpp::MakeUniqueVTableOverlay( inst, B, - {{Method(B, "alpha"), MethodAddr(&Repl::negate)}, - {Method(B, "beta"), MethodAddr(&Repl::twice)}}); + {{FindMethod(B, "alpha"), MethodAddr(&Repl::negate)}, + {FindMethod(B, "beta"), MethodAddr(&Repl::twice)}}); ASSERT_TRUE(ov); EXPECT_EQ(call_slot(inst, kAlpha, 5), -5); EXPECT_EQ(call_slot(inst, kBeta, 5), 10); @@ -188,7 +181,7 @@ TEST(VTableOverlay, RejectsInvalidInput) { auto B = DeclareBase(); void* inst = Cpp::Construct(B).data; ASSERT_NE(inst, nullptr); - Cpp::ConstFuncRef beta = Method(B, "beta"); + Cpp::ConstFuncRef beta = FindMethod(B, "beta"); Cpp::ConstFuncRef none = nullptr; void* fn = MethodAddr(&Repl::negate); @@ -210,7 +203,7 @@ TEST(VTableOverlay, OverlayIsPerInstance) { void* b_vptr_before = *reinterpret_cast(b); auto ov = Cpp::MakeUniqueVTableOverlay( - a, B, {{Method(B, "beta"), MethodAddr(&Repl::negate)}}); + a, B, {{FindMethod(B, "beta"), MethodAddr(&Repl::negate)}}); ASSERT_TRUE(ov); EXPECT_NE(*reinterpret_cast(a), b_vptr_before); // a swapped @@ -240,7 +233,7 @@ TEST(VTableOverlay, ThunkReadsThisAndDataMember) { ASSERT_NE(inst, nullptr); auto ov = Cpp::MakeUniqueVTableOverlay( - inst, T, {{Method(T, "frob"), MethodAddr(&Repl::read_value)}}); + inst, T, {{FindMethod(T, "frob"), MethodAddr(&Repl::read_value)}}); ASSERT_TRUE(ov); // First user virtual lives at kAlpha (Itanium D1/D0 prefix; MSVC single @@ -270,7 +263,7 @@ TEST(VTableOverlay, DerivedClassWithOverride) { EXPECT_EQ(call_slot(inst, kAlpha, 5), 7); auto ov = Cpp::MakeUniqueVTableOverlay( - inst, D, {{Method(D, "frob"), MethodAddr(&Repl::negate)}}); + inst, D, {{FindMethod(D, "frob"), MethodAddr(&Repl::negate)}}); ASSERT_TRUE(ov); EXPECT_EQ(call_slot(inst, kAlpha, 5), -5); ov.reset(); @@ -292,7 +285,7 @@ TEST(VTableOverlay, MultiLevelInheritance) { EXPECT_EQ(call_slot(inst, kAlpha, 5), 8); // MlC::frob: x+3 auto ov = Cpp::MakeUniqueVTableOverlay( - inst, C, {{Method(C, "frob"), MethodAddr(&Repl::twice)}}); + inst, C, {{FindMethod(C, "frob"), MethodAddr(&Repl::twice)}}); ASSERT_TRUE(ov); EXPECT_EQ(call_slot(inst, kAlpha, 5), 10); // overlay: x*2 ov.reset(); @@ -318,7 +311,7 @@ TEST(VTableOverlay, RejectsMultipleInheritance) { ASSERT_NE(inst, nullptr); auto ov = Cpp::MakeUniqueVTableOverlay( - inst, C, {{Method(C, "af"), MethodAddr(&Repl::negate)}}); + inst, C, {{FindMethod(C, "af"), MethodAddr(&Repl::negate)}}); EXPECT_FALSE(ov); // refuses the layout Cpp::Destruct(inst, C); @@ -338,7 +331,7 @@ TEST(VTableOverlay, RejectsNonPolymorphicBase) { ASSERT_NE(PH, nullptr); void* inst = Cpp::Construct(NP).data; ASSERT_NE(inst, nullptr); - Cpp::ConstFuncRef dummy = Method(PH, "dummy"); + Cpp::ConstFuncRef dummy = FindMethod(PH, "dummy"); void* fn = MethodAddr(&Repl::negate); EXPECT_EQ(Cpp::MakeVTableOverlay(inst, NP, &dummy, &fn, 1), nullptr); Cpp::Destruct(inst, NP); @@ -372,7 +365,7 @@ TEST(VTableOverlay, RejectsOutOfRangeMethodSlot) { ASSERT_NE(Large, nullptr); void* inst = Cpp::Construct(Small).data; ASSERT_NE(inst, nullptr); - Cpp::ConstFuncRef v10 = Method(Large, "v10"); + Cpp::ConstFuncRef v10 = FindMethod(Large, "v10"); void* fn = MethodAddr(&Repl::negate); EXPECT_EQ(Cpp::MakeVTableOverlay(inst, Small, &v10, &fn, 1), nullptr); Cpp::Destruct(inst, Small); @@ -411,11 +404,11 @@ TEST(VTableOverlay, OverlayThroughHierarchyAccessesDataMembers) { auto ov_a = Cpp::MakeUniqueVTableOverlay( &a, A_scope, - {{Method(A_scope, "method"), MethodAddr(&Repl::foo)}}); + {{FindMethod(A_scope, "method"), MethodAddr(&Repl::foo)}}); ASSERT_TRUE(ov_a); auto ov_b = Cpp::MakeUniqueVTableOverlay( &b, B_scope, - {{Method(B_scope, "method"), MethodAddr(&Repl::bar)}}); + {{FindMethod(B_scope, "method"), MethodAddr(&Repl::bar)}}); ASSERT_TRUE(ov_b); EXPECT_EQ(call_slot_no_arg(&a, kAlpha), 15); // foo: A::m_x(5) + 10 @@ -447,7 +440,7 @@ TEST(VTableOverlay, RejectsVirtualInheritance) { ASSERT_NE(inst, nullptr); auto ov = Cpp::MakeUniqueVTableOverlay( - inst, D, {{Method(D, "frob"), MethodAddr(&Repl::negate)}}); + inst, D, {{FindMethod(D, "frob"), MethodAddr(&Repl::negate)}}); EXPECT_FALSE(ov); // refuses the layout Cpp::Destruct(inst, D); @@ -592,7 +585,7 @@ TEST(VTableOverlay, SetsExtraPrefixSlots) { void* inst = Cpp::Construct(B).data; ASSERT_NE(inst, nullptr); auto ov = Cpp::MakeUniqueVTableOverlay( - inst, B, {{Method(B, "beta"), MethodAddr(&Repl::negate)}}, + inst, B, {{FindMethod(B, "beta"), MethodAddr(&Repl::negate)}}, /*n_extra_prefix_slots=*/2); ASSERT_TRUE(ov); void*& slot0 = Cpp::VTableOverlayExtraSlot(inst, 0); diff --git a/interpreter/CppInterOp/unittests/CppInterOp/VariableReflectionTest.cpp b/interpreter/CppInterOp/unittests/CppInterOp/VariableReflectionTest.cpp index 74e5a4fd1e295..ed11ec1dfcee0 100644 --- a/interpreter/CppInterOp/unittests/CppInterOp/VariableReflectionTest.cpp +++ b/interpreter/CppInterOp/unittests/CppInterOp/VariableReflectionTest.cpp @@ -17,8 +17,10 @@ #include #include +#include #include #include +#include using namespace TestUtils; using namespace llvm; @@ -122,10 +124,6 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, VariableReflection_GetDatamembers) { CODE TYPED_TEST(CPPINTEROP_TEST_MODE, VariableReflection_DatamembersWithAnonymousStructOrUnion) { -#if CLANG_VERSION_MAJOR == 20 && defined(CPPINTEROP_USE_CLING) && defined(_WIN32) - GTEST_SKIP() << "Test fails with Cling on Windows"; -#endif - std::vector Decls; #define Stringify(s) Stringifyx(s) #define Stringifyx(...) #__VA_ARGS__ @@ -724,10 +722,6 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, VariableReflection_VariableOffsetsWithInheritan #ifdef __EMSCRIPTEN__ GTEST_SKIP() << "This test crashes for Emscripten builds of CppInterOp"; #endif -#if CLANG_VERSION_MAJOR == 20 && defined(CPPINTEROP_USE_CLING) && defined(_WIN32) - GTEST_SKIP() << "Test fails with Cling on Windows"; -#endif - std::vector interpreter_args = {"-include", "new"}; TestFixture::CreateInterpreter(interpreter_args); @@ -1005,6 +999,57 @@ TYPED_TEST(CPPINTEROP_TEST_MODE, EXPECT_EQ(Cpp::GetEnumConstantValue(datamembers2[5]), 2); } +TYPED_TEST(CPPINTEROP_TEST_MODE, + VariableReflection_EnumConstantsOfClassTemplateFromAST) { + // Bindings query every enumerator of a class when they build its proxy, so + // these queries must be answered from the AST. Evaluating e.g. + // "EnumHolder::kA;" in the interpreter instead costs a parse and JIT + // round trip per enumerator (root-project/root#10724). + std::vector Decls; + std::string code = R"( + template struct EnumHolder { + enum EBits { kA = 1 << 16, kB }; + enum { kAnon = -7 }; + virtual ~EnumHolder(); // never defined, so the vtable can't be emitted + }; + )"; + GetAllTopLevelDecls(code, Decls); + + ASTContext& C = Interp->getCI()->getASTContext(); + std::vector args = {C.CharTy.getAsOpaquePtr()}; + Cpp::DeclRef Holder = Cpp::InstantiateTemplate(Decls[0], args); + ASSERT_TRUE(Holder); + + // Any code handed to the interpreter adds a top-level decl (cling's wrapper + // function, clang-repl's TopLevelStmtDecl). clang-repl starts a new + // TranslationUnitDecl for each input, so count over all of them. + auto CountTopLevelDecls = [&C]() { + size_t N = 0; + for (const auto* TU : C.getTranslationUnitDecl()->redecls()) + for (auto I = TU->noload_decls_begin(), E = TU->noload_decls_end(); + I != E; ++I) + ++N; + return N; + }; + const size_t NumTopLevelDecls = CountTopLevelDecls(); + + std::vector datamembers; + Cpp::GetEnumConstantDatamembers(Holder, datamembers); + const std::vector> Expected = { + {"kA", 1 << 16}, {"kB", (1 << 16) + 1}, {"kAnon", -7}}; + ASSERT_EQ(datamembers.size(), Expected.size()); + for (size_t i = 0; i < datamembers.size(); ++i) { + EXPECT_TRUE(Cpp::IsEnumConstant(datamembers[i])); + EXPECT_EQ(Cpp::GetName(datamembers[i]), Expected[i].first); + EXPECT_EQ(Cpp::GetEnumConstantValue(datamembers[i]), Expected[i].second); + EXPECT_TRUE(Cpp::GetEnumConstantType(datamembers[i])); + // Enumerators have no storage, hence no offset. + EXPECT_EQ(Cpp::GetVariableOffset(datamembers[i], Holder), 0); + } + + EXPECT_EQ(CountTopLevelDecls(), NumTopLevelDecls); +} + TYPED_TEST(CPPINTEROP_TEST_MODE, VariableReflection_Is_Get_Pointer) { TestFixture::CreateInterpreter(); std::vector Decls; From d64b7ebe3b9e385892694152a1711c4037733ed5 Mon Sep 17 00:00:00 2001 From: Aaron Jomy Date: Fri, 23 Oct 2026 13:40:11 +0200 Subject: [PATCH 2/2] [cppjit] Update vendored cppjit to ROOT-2026-10-21 --- bindings/pyroot/cppjit/CMakeLists.txt | 2 +- bindings/pyroot/cppjit/cppjit-version.tag | 2 +- .../pyroot/cppjit/python/cppjit/__init__.py | 17 +- bindings/pyroot/cppjit/src/cpyrt/API.h | 8 - bindings/pyroot/cppjit/src/cpyrt/CPPMethod.h | 3 +- bindings/pyroot/cppjit/src/cpyrt/CPPScope.cxx | 124 +++++--- .../pyroot/cppjit/src/cpyrt/Compatibility.h | 56 ++++ .../pyroot/cppjit/src/cpyrt/Converters.cxx | 48 +-- bindings/pyroot/cppjit/src/cpyrt/Converters.h | 2 - .../pyroot/cppjit/src/cpyrt/Executors.cxx | 52 +--- bindings/pyroot/cppjit/src/cpyrt/Executors.h | 2 - .../pyroot/cppjit/src/cpyrt/PyStrings.cxx | 290 +----------------- bindings/pyroot/cppjit/src/cpyrt/PyStrings.h | 158 +++++----- .../pyroot/cppjit/src/cpyrt/Pythonize.cxx | 23 +- .../pyroot/cppjit/src/cpyrt/TemplateProxy.cxx | 20 +- bindings/pyroot/cppjit/src/cpyrt/Utility.cxx | 151 ++++++++- bindings/pyroot/cppjit/src/cpyrt/Utility.h | 8 + .../cppjit/src/interop/cppjit_interop.h | 14 +- .../cppjit/src/interop/interop_wrapper.cxx | 133 +++++++- bindings/pyroot/cppjit/test/CMakeLists.txt | 6 +- .../pyroot/cppjit/test/cpp/pythonizables.cxx | 28 ++ .../pyroot/cppjit/test/cpp/pythonizables.h | 33 ++ .../pyroot/cppjit/test/test_advancedcpp.py | 68 ++++ bindings/pyroot/cppjit/test/test_datatypes.py | 225 ++++++++++++++ .../pyroot/cppjit/test/test_pythonization.py | 3 + .../pyroot/cppjit/test/test_regression.py | 113 +++++++ 26 files changed, 1060 insertions(+), 529 deletions(-) diff --git a/bindings/pyroot/cppjit/CMakeLists.txt b/bindings/pyroot/cppjit/CMakeLists.txt index 79828f6e307c8..ac5fe0b704c8c 100644 --- a/bindings/pyroot/cppjit/CMakeLists.txt +++ b/bindings/pyroot/cppjit/CMakeLists.txt @@ -11,7 +11,7 @@ include(GNUInstallDirs) # Perhaps this should permanently be OFF and users can build their own CppInterOp if they want to run the tests? option(CPPJIT_ENABLE_CPPINTEROP_TESTS "enable CppInterOp tests" OFF) set(CPPINTEROP_GIT_REPOSITORY "https://github.com/compiler-research/CppInterOp.git" CACHE STRING "") -set(CPPINTEROP_GIT_TAG "c8126249dd18363000b89333b34313ca2a83fb82" CACHE STRING "") +set(CPPINTEROP_GIT_TAG "f4adaa6b50542dd564089fe584d407bf15351c97" CACHE STRING "") set(CPPINTEROP_SOURCE_DIR "" CACHE PATH "Override default CppInterOp built by ExternalProject_Add, with a path to local CppInterOp source") diff --git a/bindings/pyroot/cppjit/cppjit-version.tag b/bindings/pyroot/cppjit/cppjit-version.tag index 402a8ea0b7ba9..baa6611f1a84d 100644 --- a/bindings/pyroot/cppjit/cppjit-version.tag +++ b/bindings/pyroot/cppjit/cppjit-version.tag @@ -1 +1 @@ -0f6cc367f2f7f169c6a09ffaa97169230e52b299 +3c66389821de2084c69cc9fdfcd26e32b47e16f5 diff --git a/bindings/pyroot/cppjit/python/cppjit/__init__.py b/bindings/pyroot/cppjit/python/cppjit/__init__.py index 3d0282af458fa..8b8f3a8da0427 100644 --- a/bindings/pyroot/cppjit/python/cppjit/__init__.py +++ b/bindings/pyroot/cppjit/python/cppjit/__init__.py @@ -241,8 +241,10 @@ def _interpreter_lock(): def cppdef(src, verbose=True): """Declare C++ source to Cling.""" with _stderr_capture() as err, _interpreter_lock(): - errcode = gbl.Cpp.Declare(src, not verbose) - if not errcode == 0 or err.err: + ok = gbl.gInterpreter.Declare(src) + if not verbose: + err.err = "" + if not ok or err.err: if "warning" in err.err.lower() and "error" not in err.err.lower(): warnings.warn(err.err, SyntaxWarning) return True @@ -326,8 +328,8 @@ def load_library(name): def include(header): """Load (and JIT) header file
into Cling.""" with _stderr_capture() as err, _interpreter_lock(): - errcode = gbl.Cpp.Declare('#include "%s"' % header, False) - if not errcode == 0: + ok = gbl.gInterpreter.Declare('#include "%s"' % header) + if not ok: raise ImportError('Failed to load header file "%s"%s' % (header, err.err)) return True @@ -335,14 +337,13 @@ def include(header): def c_include(header): """Load (and JIT) header file
into Cling.""" with _stderr_capture() as err, _interpreter_lock(): - errcode = gbl.Cpp.Declare( + ok = gbl.gInterpreter.Declare( """extern "C" { #include "%s" }""" - % header, - False, + % header ) - if not errcode == 0: + if not ok: raise ImportError('Failed to load header file "%s"%s' % (header, err.err)) return True diff --git a/bindings/pyroot/cppjit/src/cpyrt/API.h b/bindings/pyroot/cppjit/src/cpyrt/API.h index 627132a2e265b..083b5d1fa5c63 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/API.h +++ b/bindings/pyroot/cppjit/src/cpyrt/API.h @@ -145,10 +145,6 @@ typedef Converter* (*ConverterFactory_t)(cdims_t); CPYRT_EXTERN bool RegisterConverter(const std::string& name, ConverterFactory_t); -// register a custom converter that is a reference to an existing converter -CPYRT_EXTERN bool RegisterConverterAlias(const std::string& name, - const std::string& target); - // remove a custom converter CPYRT_EXTERN bool UnregisterConverter(const std::string& name); @@ -176,10 +172,6 @@ CPYRT_EXTERN void DestroyConverter(Converter* p); typedef Executor* (*ExecutorFactory_t)(cdims_t); CPYRT_EXTERN bool RegisterExecutor(const std::string& name, ExecutorFactory_t); -// register a custom executor that is a reference to an existing converter -CPYRT_EXTERN bool RegisterExecutorAlias(const std::string& name, - const std::string& target); - // remove a custom executor CPYRT_EXTERN bool UnregisterExecutor(const std::string& name); diff --git a/bindings/pyroot/cppjit/src/cpyrt/CPPMethod.h b/bindings/pyroot/cppjit/src/cpyrt/CPPMethod.h index 41e9546b4c182..e65a3e12a87a5 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/CPPMethod.h +++ b/bindings/pyroot/cppjit/src/cpyrt/CPPMethod.h @@ -80,6 +80,8 @@ class CPPMethod : public PyCallable { PyObject* Call(CPPInstance*& self, cpyrt_PyArgs_t args, size_t nargsf, PyObject* kwds, CallContext* ctxt = nullptr) override; + interop::TCppMethod_t GetMethod() const { return fMethod; } + protected: virtual bool ProcessArgs(PyCallArgs& args); @@ -89,7 +91,6 @@ class CPPMethod : public PyCallable { CallContext* ctxt = nullptr); PyObject* Execute(void* self, ptrdiff_t offset, CallContext* ctxt = nullptr); - interop::TCppMethod_t GetMethod() { return fMethod; } interop::TCppScope_t GetScope() { return fScope; } Executor* GetExecutor() { return fExecutor; } std::string GetSignatureString(bool show_formalargs = true); diff --git a/bindings/pyroot/cppjit/src/cpyrt/CPPScope.cxx b/bindings/pyroot/cppjit/src/cpyrt/CPPScope.cxx index 9f695789b6683..1533fc44b538f 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/CPPScope.cxx +++ b/bindings/pyroot/cppjit/src/cpyrt/CPPScope.cxx @@ -18,6 +18,7 @@ using namespace cppjit; // Standard #include // for for_each +#include #include #include #include @@ -207,16 +208,11 @@ static PyObject* pt_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) { // creation of the python-side class; extend the size if this is a smart ptr // // A namespace can never be a smart pointer, so skip the check for namespaces. - // This matters beyond performance: GetSmartPtrInfo() resolves the scope name - // through a slow interpreter lookup that triggers autoloading of the library - // providing the scope, which we don't want to do unnecessarily. interop::TCppScope_t raw; interop::TCppMethod_t deref; if (CPPScope_CheckExact(subtype) && !interop::IsNamespace(((CPPScope*)subtype)->fCppType)) { - if (interop::GetSmartPtrInfo( - interop::GetScopedFinalName(((CPPScope*)subtype)->fCppType), &raw, - &deref)) + if (interop::GetSmartPtrInfo(((CPPScope*)subtype)->fCppType, &raw, &deref)) subtype->tp_basicsize = sizeof(CPPSmartClass); } @@ -324,6 +320,74 @@ static PyObject* pt_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) { return (PyObject*)result; } +//---------------------------------------------------------------------------- +static bool IsPlainIdentifier(const std::string& name) { + if (name.empty() || std::isdigit((unsigned char)name[0])) + return false; + for (char c : name) { + if (!std::isalnum((unsigned char)c) && c != '_') + return false; + } + return true; +} + +// Returns weak references to the proxies of the namespaces that the namespace +// \c pyclass pulls in with "using namespace", refreshing the list as needed. +static const std::vector& GetUsingScopes(PyObject* pyclass) { + CPPScope* klass = (CPPScope*)pyclass; + const std::vector& uv = + interop::GetUsingNamespaces(klass->fCppType); + if (!klass->fImp.fUsing || uv.size() != klass->fImp.fUsing->size()) { + if (klass->fImp.fUsing) { + for (auto pyref : *klass->fImp.fUsing) + Py_DECREF(pyref); + klass->fImp.fUsing->clear(); + } else + klass->fImp.fUsing = new std::vector; + + // reload and reset weak refs + for (auto uid : uv) { + std::string uname = interop::GetScopedFinalName(uid); + PyObject* pyuscope = CreateScopeProxy(uname); + if (pyuscope) { + klass->fImp.fUsing->push_back(PyWeakref_NewRef(pyuscope, nullptr)); + // the namespace may not otherwise be held, so tie the lifetimes + PyObject* llname = + PyUnicode_FromString(("__lifeline_" + uname).c_str()); + PyType_Type.tp_setattro(pyclass, llname, pyuscope); + Py_DECREF(llname); + Py_DECREF(pyuscope); + } else { + PyErr_Clear(); + } + } + } + return *klass->fImp.fUsing; +} + +// Checks whether \c pyname is set as a Python attribute on any namespace that +// \c pyclass pulls in with "using namespace", directly or transitively. +static bool HasPythonAttrInUsing(PyObject* pyclass, PyObject* pyname, + std::set& visited) { + for (auto pyref : GetUsingScopes(pyclass)) { + PyObject* pyuscope = cpyrt_GetWeakRef(pyref); + if (!pyuscope) + continue; + bool found = false; + if (visited.insert(pyuscope).second) { + found = _PyType_Lookup((PyTypeObject*)pyuscope, pyname) || + _PyType_Lookup(Py_TYPE(pyuscope), pyname) || + (CPPScope_Check(pyuscope) && + (((CPPScope*)pyuscope)->fFlags & CPPScope::kIsNamespace) && + HasPythonAttrInUsing(pyuscope, pyname, visited)); + } + Py_DECREF(pyuscope); + if (found) + return true; + } + return false; +} + //---------------------------------------------------------------------------- static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) { // normal type-based lookup @@ -411,8 +475,9 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) { } } + interop::TCppScope_t lookup_result = nullptr; if (!attr) { - interop::TCppScope_t lookup_result = interop::GetNamed(name, scope); + lookup_result = interop::GetNamed(name, scope); if (interop::IsVariable(lookup_result) || interop::IsEnumConstant(lookup_result)) { attr = (PyObject*)CPPDataMember_New(scope, lookup_result); @@ -450,7 +515,7 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) { // enums types requested as type (rather than the constants) if (!attr) { interop::TCppScope_t enumerator = - interop::GetUnderlyingScope(interop::GetNamed(name, scope)); + interop::GetUnderlyingScope(lookup_result); if (interop::IsEnumScope(enumerator)) { // enum types (incl. named and class enums) attr = (PyObject*)CPPEnum_New(name, enumerator); @@ -481,38 +546,25 @@ static PyObject* meta_getattro(PyObject* pyclass, PyObject* pyname) { } if (!attr && (klass->fFlags & CPPScope::kIsNamespace)) { - // refresh using list as necessary - const std::vector& uv = - interop::GetUsingNamespaces(klass->fCppType); - if (!klass->fImp.fUsing || uv.size() != klass->fImp.fUsing->size()) { - if (klass->fImp.fUsing) { - for (auto pyref : *klass->fImp.fUsing) - Py_DECREF(pyref); - klass->fImp.fUsing->clear(); - } else - klass->fImp.fUsing = new std::vector; - - // reload and reset weak refs - for (auto uid : uv) { - std::string uname = interop::GetScopedFinalName(uid); - PyObject* pyuscope = CreateScopeProxy(uname); - if (pyuscope) { - klass->fImp.fUsing->push_back(PyWeakref_NewRef(pyuscope, nullptr)); - // the namespace may not otherwise be held, so tie the lifetimes - PyObject* llname = - PyUnicode_FromString(("__lifeline_" + uname).c_str()); - PyType_Type.tp_setattro(pyclass, llname, pyuscope); - Py_DECREF(llname); - Py_DECREF(pyuscope); - } else { - PyErr_Clear(); - } - } + const std::vector& using_scopes = GetUsingScopes(pyclass); + + // Searching the using namespaces repeats the full attribute lookup in each + // of them, which is expensive for names that don't exist. Anything C++ it + // can find is also found by a single unqualified lookup from within this + // namespace, which follows the using-directives. If that finds nothing, + // only attributes set on the Python proxies of the using namespaces remain. + bool search_using = !using_scopes.empty(); + if (search_using && IsPlainIdentifier(name) && !interop::IsBuiltin(name) && + !interop::IsVisibleName(name, klass->fCppType)) { + std::set visited{pyclass}; + search_using = HasPythonAttrInUsing(pyclass, pyname, visited); } // try all outstanding using namespaces in turn to find the attribute (will // cache locally later; TODO: doing so may cause pathological cases) - for (auto pyref : *klass->fImp.fUsing) { + for (auto pyref : using_scopes) { + if (!search_using) + break; PyObject* pyuscope = cpyrt_GetWeakRef(pyref); if (pyuscope) { attr = PyObject_GetAttr(pyuscope, pyname); diff --git a/bindings/pyroot/cppjit/src/cpyrt/Compatibility.h b/bindings/pyroot/cppjit/src/cpyrt/Compatibility.h index e9771d7e6b76d..90f5ad624c119 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/Compatibility.h +++ b/bindings/pyroot/cppjit/src/cpyrt/Compatibility.h @@ -3,6 +3,7 @@ #include "Python.h" +#include "PyStrings.h" #include "cppjit_interop.h" #include @@ -43,6 +44,61 @@ inline std::string ObjToString(interop::TCppScope_t /*klass*/, return ""; } +//- cppyy legacy API ---------------------------------------------------------- +// Backwards compatibility with the API that former cppyy users have in their +// code. The macro is deliberately not a build option: it only exists so that +// the legacy code paths are easy to find, and to remove once there is a +// transition plan. Keep all such shims in this section. +#define CPPYY_LEGACY_API 1 + +// Name under which pyclass defines its explicit pythonization hook, looked up +// in its own __dict__ only (no fall-through to base classes). Prefers the +// __cppjit_* spelling over the old __cppyy_* one. Returns a borrowed reference, +// or nullptr with no error set if there is no such hook. +inline PyObject* ExplicitPythonizeHookName(PyObject* pyclass) { + // go through __dict__ instead of tp_getattr, see HasAttrDirect in + // Pythonize.cxx for the rationale + PyObject* name = nullptr; + PyObject* dct = PyObject_GetAttr(pyclass, PyStrings::gDict); + if (dct) { + PyObject* candidates[] = { + PyStrings::gExPythonize, +#if CPPYY_LEGACY_API + PyStrings::gCppyyExPythonize, +#endif + }; + for (PyObject* candidate : candidates) { + PyObject* attr = PyObject_GetItem(dct, candidate); + if (attr) { + Py_DECREF(attr); + name = candidate; + break; + } + PyErr_Clear(); + } + Py_DECREF(dct); + } + PyErr_Clear(); + return name; +} + +// Normal pythonization hook of pyclass, through regular attribute lookup (so +// it falls through to base classes). Prefers the __cppjit_* spelling over the +// old __cppyy_* one. Returns a new reference, or nullptr with no error set if +// there is no such hook. +inline PyObject* GetPythonizeHook(PyObject* pyclass) { + PyObject* func = PyObject_GetAttr(pyclass, PyStrings::gPythonize); +#if CPPYY_LEGACY_API + if (!func) { + PyErr_Clear(); + func = PyObject_GetAttr(pyclass, PyStrings::gCppyyPythonize); + } +#endif + if (!func) + PyErr_Clear(); + return func; +} + } // namespace cppjit::cpyrt::compat #endif // CPYRT_COMPATIBILITY_H diff --git a/bindings/pyroot/cppjit/src/cpyrt/Converters.cxx b/bindings/pyroot/cppjit/src/cpyrt/Converters.cxx index c21d00ef182e9..f6e15fb9d60ff 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/Converters.cxx +++ b/bindings/pyroot/cppjit/src/cpyrt/Converters.cxx @@ -26,7 +26,6 @@ using namespace cppjit; #include #include #include // for wstring_convert -#include #include #include // for ptrdiff_t #include @@ -457,7 +456,7 @@ static inline bool CArraySetArg(PyObject* pyobject, cpyrt::Parameter& para, if (cpyrt::LowLevelView_Check(pyobject)) { auto llview = ((cpyrt::LowLevelView*)pyobject); if (llview->fBufInfo.itemsize != size || - !strchr(llview->fBufInfo.format, tc)) { + !cpyrt::Utility::BufferFormatCompatible(tc, llview->fBufInfo.format)) { PyErr_Format(PyExc_TypeError, "could not convert argument to buffer or nullptr"); return false; @@ -1787,7 +1786,7 @@ bool cpyrt::StdSpanConverter::SetArg(PyObject* pyobject, Parameter& para, memset(&fBufinfo, 0, sizeof(Py_buffer)); if (PyObject_GetBuffer(pyobject, &fBufinfo, PyBUF_FORMAT) == 0) { - if (!strchr(fBufinfo.format, typecode)) { + if (!Utility::BufferFormatCompatible(typecode, fBufinfo.format)) { PyErr_Format(PyExc_TypeError, "buffer has incompatible type: expected '%c' for C++ type " "'%s', but got format '%s'", @@ -1878,7 +1877,8 @@ bool ToArrayFromBuffer(PyObject* owner, void* address, PyObject* ctxt, convOk = true; \ } else if (LowLevelView_Check(pyobject) && \ ((LowLevelView*)pyobject)->fBufInfo.ndim == 2 && \ - strchr(((LowLevelView*)pyobject)->fBufInfo.format, code)) { \ + Utility::BufferFormatCompatible( \ + code, ((LowLevelView*)pyobject)->fBufInfo.format)) { \ para.fValue.fVoidp = ((LowLevelView*)pyobject)->get_buf(); \ para.fTypeCode = 'p'; \ convOk = true; \ @@ -1905,7 +1905,8 @@ bool ToArrayFromBuffer(PyObject* owner, void* address, PyObject* ctxt, convOk = true; \ } \ } else if (LowLevelView_Check(pyobject) && \ - strchr(((LowLevelView*)pyobject)->fBufInfo.format, code)) { \ + Utility::BufferFormatCompatible( \ + code, ((LowLevelView*)pyobject)->fBufInfo.format)) { \ para.fValue.fVoidp = ((LowLevelView*)pyobject)->get_buf(); \ para.fTypeCode = 'p'; \ convOk = true; \ @@ -3493,8 +3494,11 @@ cppjit::cpyrt::CreateConverter(const std::string& fullType, cdims_t dims) { return (h->second)(dims); } - // resolve typedefs etc. - const std::string& resolvedType = interop::ResolveName(fullType); + // resolve typedefs etc. (the type itself is kept for function pointers) + interop::TCppType_t type = + interop::GetType(fullType, /*enable_slow_lookup=*/true); + const std::string resolvedType = + type ? interop::GetTypeAsString(interop::ResolveType(type)) : fullType; // a full, qualified matching converter is preferred if (resolvedType != fullType) { @@ -3639,7 +3643,7 @@ cppjit::cpyrt::CreateConverter(const std::string& fullType, cdims_t dims) { Converter* result = nullptr; if (interop::TCppScope_t klass = interop::GetFullScope(realType)) { interop::TCppScope_t raw; - if (interop::GetSmartPtrInfo(realType, &raw, nullptr)) { + if (interop::GetSmartPtrInfo(klass, &raw, nullptr)) { if (cpd == "") { result = new SmartPtrConverter(klass, raw, control); } else if (cpd == "&") { @@ -3658,12 +3662,9 @@ cppjit::cpyrt::CreateConverter(const std::string& fullType, cdims_t dims) { // -- Cling WORKAROUND result = selectInstanceCnv(klass, cpd, dims, isConst, control); } - } else { - interop::TCppType_t typ = interop::GetType(fullType); - if (interop::IsFunctionType(typ)) { - // this is a function pointer or reference - result = new FunctionPointerConverter(typ); - } + } else if (interop::IsFunctionType(type)) { + // this is a function pointer or reference + result = new FunctionPointerConverter(type); } const std::string failure_msg("Failed to convert type: " + fullType + "; resolved: " + resolvedType + @@ -3902,7 +3903,7 @@ cppjit::cpyrt::CreateConverter(interop::TCppType_t type, cdims_t dims) { (klass || (klass = interop::GetFullScope(realTypeStr)))) { // std::byte is a special enum class used to access raw memory interop::TCppScope_t raw; - if (interop::GetSmartPtrInfo(realTypeStr, &raw, nullptr)) { + if (interop::GetSmartPtrInfo(klass, &raw, nullptr)) { if (cpd == "") { result = new SmartPtrConverter(klass, raw, control); } else if (cpd == "&") { @@ -3985,23 +3986,6 @@ bool cpyrt::RegisterConverter(const std::string& name, cf_t fac) { return true; } -//---------------------------------------------------------------------------- -CPYRT_EXPORT -bool cpyrt::RegisterConverterAlias(const std::string& name, - const std::string& target) { - // register a custom converter that is a reference to an existing converter - auto f = gConvFactories.find(name); - if (f != gConvFactories.end()) - return false; - - auto t = gConvFactories.find(target); - if (t == gConvFactories.end()) - return false; - - gConvFactories[name] = t->second; - return true; -} - //---------------------------------------------------------------------------- CPYRT_EXPORT bool cpyrt::UnregisterConverter(const std::string& name) { diff --git a/bindings/pyroot/cppjit/src/cpyrt/Converters.h b/bindings/pyroot/cppjit/src/cpyrt/Converters.h index bc2f65d61e20d..d208464825606 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/Converters.h +++ b/bindings/pyroot/cppjit/src/cpyrt/Converters.h @@ -40,8 +40,6 @@ CPYRT_EXPORT Converter* CreateConverter(interop::TCppType_t type, CPYRT_EXPORT void DestroyConverter(Converter* p); typedef Converter* (*cf_t)(cdims_t d); CPYRT_EXPORT bool RegisterConverter(const std::string& name, cf_t fac); -CPYRT_EXPORT bool RegisterConverterAlias(const std::string& name, - const std::string& target); CPYRT_EXPORT bool UnregisterConverter(const std::string& name); // converters for special cases (only here b/c of external use of diff --git a/bindings/pyroot/cppjit/src/cpyrt/Executors.cxx b/bindings/pyroot/cppjit/src/cpyrt/Executors.cxx index 6fc105e417f9f..d68dacb677dbf 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/Executors.cxx +++ b/bindings/pyroot/cppjit/src/cpyrt/Executors.cxx @@ -877,8 +877,11 @@ cpyrt::Executor* cpyrt::CreateExecutor(const std::string& fullType, if (h != gExecFactories.end()) return (h->second)(dims); - // resolve typedefs etc. - const std::string resolvedType = interop::ResolveName(fullType); + // resolve typedefs etc. (the type itself is kept for function pointers) + interop::TCppType_t type = + interop::GetType(fullType, /*enable_slow_lookup=*/true); + const std::string resolvedType = + type ? interop::GetTypeAsString(interop::ResolveType(type)) : fullType; // a full, qualified matching executor is preferred if (resolvedType != fullType) { @@ -949,16 +952,10 @@ cpyrt::Executor* cpyrt::CreateExecutor(const std::string& fullType, result = new InstancePtrRefExecutor(klass); } else result = new InstancePtrExecutor(klass); - } else if (resolvedType.find("(*)") != std::string::npos || - (resolvedType.find("::*)") != std::string::npos)) { + } else if (std::string fnRet, fnSig; + interop::GetFnTypeSigAsString(type, fnRet, fnSig)) { // this is a function pointer - // TODO: find better way of finding the type - auto pos1 = resolvedType.find('('); - auto pos2 = resolvedType.find("*)"); - auto pos3 = resolvedType.rfind(')'); - result = new FunctionPointerExecutor( - resolvedType.substr(0, pos1), - resolvedType.substr(pos2 + 2, pos3 - pos2 - 1)); + result = new FunctionPointerExecutor(fnRet, fnSig); } else { // unknown: void* may work ("user knows best"), void will fail on use of // return value @@ -1091,16 +1088,12 @@ cpyrt::Executor* cpyrt::CreateExecutor(interop::TCppType_t type, cdims_t dims) { result = new InstancePtrRefExecutor(klass); } else result = new InstancePtrExecutor(klass); - } else if (realTypeStr.find("(*)") != std::string::npos || - (realTypeStr.find("::*)") != std::string::npos)) { - // this is a function pointer - // TODO: find better way of finding the type - auto pos1 = realTypeStr.find('('); - auto pos2 = realTypeStr.find("*)"); - auto pos3 = realTypeStr.rfind(')'); - result = new FunctionPointerExecutor( - realTypeStr.substr(0, pos1), - realTypeStr.substr(pos2 + 2, pos3 - pos2 - 1)); + } else if (std::string fnRet, fnSig; + interop::GetFnTypeSigAsString(resolvedType, fnRet, fnSig)) { + // this is a function pointer (checked on the resolved type: the real type + // of a member function pointer is a bare function type, which must not + // pass here, as it would return 16 bytes through an 8-byte return slot) + result = new FunctionPointerExecutor(fnRet, fnSig); } else { // unknown: void* may work ("user knows best"), void will fail on use of // return value @@ -1134,23 +1127,6 @@ bool cpyrt::RegisterExecutor(const std::string& name, ef_t fac) { return true; } -//---------------------------------------------------------------------------- -CPYRT_EXPORT -bool cpyrt::RegisterExecutorAlias(const std::string& name, - const std::string& target) { - // register a custom executor that is a reference to an existing converter - auto f = gExecFactories.find(name); - if (f != gExecFactories.end()) - return false; - - auto t = gExecFactories.find(target); - if (t == gExecFactories.end()) - return false; - - gExecFactories[name] = t->second; - return true; -} - //---------------------------------------------------------------------------- CPYRT_EXPORT bool cpyrt::UnregisterExecutor(const std::string& name) { diff --git a/bindings/pyroot/cppjit/src/cpyrt/Executors.h b/bindings/pyroot/cppjit/src/cpyrt/Executors.h index c8b0702b5fee7..959afd1775c86 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/Executors.h +++ b/bindings/pyroot/cppjit/src/cpyrt/Executors.h @@ -39,8 +39,6 @@ CPYRT_EXPORT Executor* CreateExecutor(interop::TCppType_t type, cdims_t = 0); CPYRT_EXPORT void DestroyExecutor(Executor* p); typedef Executor* (*ef_t)(cdims_t); CPYRT_EXPORT bool RegisterExecutor(const std::string& name, ef_t fac); -CPYRT_EXPORT bool RegisterExecutorAlias(const std::string& name, - const std::string& target); CPYRT_EXPORT bool UnregisterExecutor(const std::string& name); // helper for the actual call diff --git a/bindings/pyroot/cppjit/src/cpyrt/PyStrings.cxx b/bindings/pyroot/cppjit/src/cpyrt/PyStrings.cxx index a0e957f3787f9..9745acaec32d0 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/PyStrings.cxx +++ b/bindings/pyroot/cppjit/src/cpyrt/PyStrings.cxx @@ -5,164 +5,20 @@ using namespace cppjit; #include "PyStrings.h" //- data _____________________________________________________________________ -PyObject* cpyrt::PyStrings::gAssign = nullptr; -PyObject* cpyrt::PyStrings::gBases = nullptr; -PyObject* cpyrt::PyStrings::gBase = nullptr; -PyObject* cpyrt::PyStrings::gContains = nullptr; -PyObject* cpyrt::PyStrings::gCopy = nullptr; -PyObject* cpyrt::PyStrings::gCppBool = nullptr; -PyObject* cpyrt::PyStrings::gCppName = nullptr; -PyObject* cpyrt::PyStrings::gAnnotations = nullptr; -PyObject* cpyrt::PyStrings::gCastCpp = nullptr; -PyObject* cpyrt::PyStrings::gCType = nullptr; -PyObject* cpyrt::PyStrings::gDeref = nullptr; -PyObject* cpyrt::PyStrings::gPreInc = nullptr; -PyObject* cpyrt::PyStrings::gPostInc = nullptr; -PyObject* cpyrt::PyStrings::gDict = nullptr; -PyObject* cpyrt::PyStrings::gEmptyString = nullptr; -PyObject* cpyrt::PyStrings::gEq = nullptr; -PyObject* cpyrt::PyStrings::gFollow = nullptr; -PyObject* cpyrt::PyStrings::gHasValue = nullptr; -PyObject* cpyrt::PyStrings::gGetItem = nullptr; -PyObject* cpyrt::PyStrings::gGetNoCheck = nullptr; -PyObject* cpyrt::PyStrings::gSetItem = nullptr; -PyObject* cpyrt::PyStrings::gInit = nullptr; -PyObject* cpyrt::PyStrings::gIter = nullptr; -PyObject* cpyrt::PyStrings::gLen = nullptr; -PyObject* cpyrt::PyStrings::gLifeLine = nullptr; -PyObject* cpyrt::PyStrings::gModule = nullptr; -PyObject* cpyrt::PyStrings::gMRO = nullptr; -PyObject* cpyrt::PyStrings::gName = nullptr; -PyObject* cpyrt::PyStrings::gNe = nullptr; -PyObject* cpyrt::PyStrings::gRepr = nullptr; -PyObject* cpyrt::PyStrings::gCppRepr = nullptr; -PyObject* cpyrt::PyStrings::gStr = nullptr; -PyObject* cpyrt::PyStrings::gCppStr = nullptr; -PyObject* cpyrt::PyStrings::gTypeCode = nullptr; -PyObject* cpyrt::PyStrings::gCTypesType = nullptr; - -PyObject* cpyrt::PyStrings::gUnderlying = nullptr; -PyObject* cpyrt::PyStrings::gRealInit = nullptr; - -PyObject* cpyrt::PyStrings::gAdd = nullptr; -PyObject* cpyrt::PyStrings::gSub = nullptr; -PyObject* cpyrt::PyStrings::gMul = nullptr; -PyObject* cpyrt::PyStrings::gDiv = nullptr; - -PyObject* cpyrt::PyStrings::gLShift = nullptr; -PyObject* cpyrt::PyStrings::gLShiftC = nullptr; - -PyObject* cpyrt::PyStrings::gAt = nullptr; -PyObject* cpyrt::PyStrings::gBegin = nullptr; -PyObject* cpyrt::PyStrings::gEnd = nullptr; -PyObject* cpyrt::PyStrings::gFirst = nullptr; -PyObject* cpyrt::PyStrings::gSecond = nullptr; -PyObject* cpyrt::PyStrings::gSize = nullptr; -PyObject* cpyrt::PyStrings::gTemplate = nullptr; -PyObject* cpyrt::PyStrings::gVectorAt = nullptr; -PyObject* cpyrt::PyStrings::gInsert = nullptr; -PyObject* cpyrt::PyStrings::gValueType = nullptr; -PyObject* cpyrt::PyStrings::gValueTypePtr = nullptr; -PyObject* cpyrt::PyStrings::gValueSize = nullptr; - -PyObject* cpyrt::PyStrings::gCppReal = nullptr; -PyObject* cpyrt::PyStrings::gCppImag = nullptr; - -PyObject* cpyrt::PyStrings::gThisModule = nullptr; - -PyObject* cpyrt::PyStrings::gDispInit = nullptr; -PyObject* cpyrt::PyStrings::gDispGet = nullptr; - -PyObject* cpyrt::PyStrings::gExPythonize = nullptr; -PyObject* cpyrt::PyStrings::gPythonize = nullptr; - -PyObject* cpyrt::PyStrings::gArray = nullptr; -PyObject* cpyrt::PyStrings::gDType = nullptr; -PyObject* cpyrt::PyStrings::gFromBuffer = nullptr; +#define CPYRT_DEFINE_PYSTRING(var, str) \ + PyObject* cpyrt::PyStrings::var = nullptr; +CPYRT_PYSTRINGS(CPYRT_DEFINE_PYSTRING) +#undef CPYRT_DEFINE_PYSTRING //----------------------------------------------------------------------------- -#define CPPJIT_INITIALIZE_STRING(var, str) \ - if (!(PyStrings::var = PyUnicode_InternFromString((char*)#str))) \ - return false - bool cpyrt::CreatePyStrings() { // Build cache of commonly used python strings (the cache is python intern, so // all strings are shared python-wide, not just in cppjit). - CPPJIT_INITIALIZE_STRING(gAssign, __assign__); - CPPJIT_INITIALIZE_STRING(gBases, __bases__); - CPPJIT_INITIALIZE_STRING(gBase, __base__); - CPPJIT_INITIALIZE_STRING(gContains, contains); - CPPJIT_INITIALIZE_STRING(gCopy, copy); - CPPJIT_INITIALIZE_STRING(gCppBool, __cpp_bool__); - CPPJIT_INITIALIZE_STRING(gCppName, __cpp_name__); - CPPJIT_INITIALIZE_STRING(gAnnotations, __annotations__); - CPPJIT_INITIALIZE_STRING(gCastCpp, __cast_cpp__); - CPPJIT_INITIALIZE_STRING(gCType, __ctype__); - CPPJIT_INITIALIZE_STRING(gDeref, __deref__); - CPPJIT_INITIALIZE_STRING(gPreInc, __preinc__); - CPPJIT_INITIALIZE_STRING(gPostInc, __postinc__); - CPPJIT_INITIALIZE_STRING(gDict, __dict__); - if (!(PyStrings::gEmptyString = PyUnicode_FromString((char*)""))) +#define CPYRT_CREATE_PYSTRING(var, str) \ + if (!(PyStrings::var = PyUnicode_InternFromString(str))) \ return false; - CPPJIT_INITIALIZE_STRING(gEq, __eq__); - CPPJIT_INITIALIZE_STRING(gFollow, __follow__); - CPPJIT_INITIALIZE_STRING(gHasValue, has_value); - CPPJIT_INITIALIZE_STRING(gGetItem, __getitem__); - CPPJIT_INITIALIZE_STRING(gGetNoCheck, _getitem__unchecked); - CPPJIT_INITIALIZE_STRING(gSetItem, __setitem__); - CPPJIT_INITIALIZE_STRING(gInit, __init__); - CPPJIT_INITIALIZE_STRING(gIter, __iter__); - CPPJIT_INITIALIZE_STRING(gLen, __len__); - CPPJIT_INITIALIZE_STRING(gLifeLine, __lifeline); - CPPJIT_INITIALIZE_STRING(gModule, __module__); - CPPJIT_INITIALIZE_STRING(gMRO, __mro__); - CPPJIT_INITIALIZE_STRING(gName, __name__); - CPPJIT_INITIALIZE_STRING(gNe, __ne__); - CPPJIT_INITIALIZE_STRING(gRepr, __repr__); - CPPJIT_INITIALIZE_STRING(gCppRepr, __cpp_repr); - CPPJIT_INITIALIZE_STRING(gStr, __str__); - CPPJIT_INITIALIZE_STRING(gCppStr, __cpp_str); - CPPJIT_INITIALIZE_STRING(gTypeCode, typecode); - CPPJIT_INITIALIZE_STRING(gCTypesType, _type_); - - CPPJIT_INITIALIZE_STRING(gUnderlying, __underlying); - CPPJIT_INITIALIZE_STRING(gRealInit, __real_init); - - CPPJIT_INITIALIZE_STRING(gAdd, __add__); - CPPJIT_INITIALIZE_STRING(gSub, __sub__); - CPPJIT_INITIALIZE_STRING(gMul, __mul__); - CPPJIT_INITIALIZE_STRING(gDiv, __truediv__); - - CPPJIT_INITIALIZE_STRING(gLShift, __lshift__); - CPPJIT_INITIALIZE_STRING(gLShiftC, __lshiftc__); - - CPPJIT_INITIALIZE_STRING(gAt, at); - CPPJIT_INITIALIZE_STRING(gBegin, begin); - CPPJIT_INITIALIZE_STRING(gEnd, end); - CPPJIT_INITIALIZE_STRING(gFirst, first); - CPPJIT_INITIALIZE_STRING(gSecond, second); - CPPJIT_INITIALIZE_STRING(gSize, size); - CPPJIT_INITIALIZE_STRING(gTemplate, Template); - CPPJIT_INITIALIZE_STRING(gVectorAt, _vector__at); - CPPJIT_INITIALIZE_STRING(gInsert, insert); - CPPJIT_INITIALIZE_STRING(gValueType, value_type); - CPPJIT_INITIALIZE_STRING(gValueTypePtr, _value_type); - CPPJIT_INITIALIZE_STRING(gValueSize, value_size); - - CPPJIT_INITIALIZE_STRING(gCppReal, __cpp_real); - CPPJIT_INITIALIZE_STRING(gCppImag, __cpp_imag); - - CPPJIT_INITIALIZE_STRING(gThisModule, cppjit); - - CPPJIT_INITIALIZE_STRING(gDispInit, _init_dispatchptr); - CPPJIT_INITIALIZE_STRING(gDispGet, _get_dispatch); - - CPPJIT_INITIALIZE_STRING(gExPythonize, __cppjit_explicit_pythonize__); - CPPJIT_INITIALIZE_STRING(gPythonize, __cppjit_pythonize__); - - CPPJIT_INITIALIZE_STRING(gArray, __array__); - CPPJIT_INITIALIZE_STRING(gDType, dtype); - CPPJIT_INITIALIZE_STRING(gFromBuffer, frombuffer); + CPYRT_PYSTRINGS(CPYRT_CREATE_PYSTRING) +#undef CPYRT_CREATE_PYSTRING return true; } @@ -170,133 +26,9 @@ bool cpyrt::CreatePyStrings() { //----------------------------------------------------------------------------- PyObject* cpyrt::DestroyPyStrings() { // Remove all cached python strings. - Py_DECREF(PyStrings::gBases); - PyStrings::gBases = nullptr; - Py_DECREF(PyStrings::gBase); - PyStrings::gBase = nullptr; - Py_DECREF(PyStrings::gContains); - PyStrings::gContains = nullptr; - Py_DECREF(PyStrings::gCopy); - PyStrings::gCopy = nullptr; - Py_DECREF(PyStrings::gCppBool); - PyStrings::gCppBool = nullptr; - Py_DECREF(PyStrings::gCppName); - PyStrings::gCppName = nullptr; - Py_DECREF(PyStrings::gAnnotations); - PyStrings::gAnnotations = nullptr; - Py_DECREF(PyStrings::gCType); - PyStrings::gCType = nullptr; - Py_DECREF(PyStrings::gDeref); - PyStrings::gDeref = nullptr; - Py_DECREF(PyStrings::gPreInc); - PyStrings::gPreInc = nullptr; - Py_DECREF(PyStrings::gPostInc); - PyStrings::gPostInc = nullptr; - Py_DECREF(PyStrings::gDict); - PyStrings::gDict = nullptr; - Py_DECREF(PyStrings::gEmptyString); - PyStrings::gEmptyString = nullptr; - Py_DECREF(PyStrings::gEq); - PyStrings::gEq = nullptr; - Py_DECREF(PyStrings::gFollow); - PyStrings::gFollow = nullptr; - Py_DECREF(PyStrings::gHasValue); - PyStrings::gHasValue = nullptr; - Py_DECREF(PyStrings::gGetItem); - PyStrings::gGetItem = nullptr; - Py_DECREF(PyStrings::gGetNoCheck); - PyStrings::gGetNoCheck = nullptr; - Py_DECREF(PyStrings::gSetItem); - PyStrings::gSetItem = nullptr; - Py_DECREF(PyStrings::gInit); - PyStrings::gInit = nullptr; - Py_DECREF(PyStrings::gIter); - PyStrings::gIter = nullptr; - Py_DECREF(PyStrings::gLen); - PyStrings::gLen = nullptr; - Py_DECREF(PyStrings::gLifeLine); - PyStrings::gLifeLine = nullptr; - Py_DECREF(PyStrings::gModule); - PyStrings::gModule = nullptr; - Py_DECREF(PyStrings::gMRO); - PyStrings::gMRO = nullptr; - Py_DECREF(PyStrings::gName); - PyStrings::gName = nullptr; - Py_DECREF(PyStrings::gNe); - PyStrings::gNe = nullptr; - Py_DECREF(PyStrings::gTypeCode); - PyStrings::gTypeCode = nullptr; - Py_DECREF(PyStrings::gCTypesType); - PyStrings::gCTypesType = nullptr; - - Py_DECREF(PyStrings::gUnderlying); - PyStrings::gUnderlying = nullptr; - Py_DECREF(PyStrings::gRealInit); - PyStrings::gRealInit = nullptr; - - Py_DECREF(PyStrings::gAdd); - PyStrings::gAdd = nullptr; - Py_DECREF(PyStrings::gSub); - PyStrings::gSub = nullptr; - Py_DECREF(PyStrings::gMul); - PyStrings::gMul = nullptr; - Py_DECREF(PyStrings::gDiv); - PyStrings::gDiv = nullptr; - - Py_DECREF(PyStrings::gLShift); - PyStrings::gLShift = nullptr; - Py_DECREF(PyStrings::gLShiftC); - PyStrings::gLShiftC = nullptr; - - Py_DECREF(PyStrings::gAt); - PyStrings::gAt = nullptr; - Py_DECREF(PyStrings::gBegin); - PyStrings::gBegin = nullptr; - Py_DECREF(PyStrings::gEnd); - PyStrings::gEnd = nullptr; - Py_DECREF(PyStrings::gFirst); - PyStrings::gFirst = nullptr; - Py_DECREF(PyStrings::gSecond); - PyStrings::gSecond = nullptr; - Py_DECREF(PyStrings::gSize); - PyStrings::gSize = nullptr; - Py_DECREF(PyStrings::gTemplate); - PyStrings::gTemplate = nullptr; - Py_DECREF(PyStrings::gVectorAt); - PyStrings::gVectorAt = nullptr; - Py_DECREF(PyStrings::gInsert); - PyStrings::gInsert = nullptr; - Py_DECREF(PyStrings::gValueType); - PyStrings::gValueType = nullptr; - Py_DECREF(PyStrings::gValueTypePtr); - PyStrings::gValueTypePtr = nullptr; - Py_DECREF(PyStrings::gValueSize); - PyStrings::gValueSize = nullptr; - - Py_DECREF(PyStrings::gCppReal); - PyStrings::gCppReal = nullptr; - Py_DECREF(PyStrings::gCppImag); - PyStrings::gCppImag = nullptr; - - Py_DECREF(PyStrings::gThisModule); - PyStrings::gThisModule = nullptr; - - Py_DECREF(PyStrings::gDispInit); - PyStrings::gDispInit = nullptr; - Py_DECREF(PyStrings::gDispGet); - PyStrings::gDispGet = nullptr; - - Py_DECREF(PyStrings::gExPythonize); - PyStrings::gExPythonize = nullptr; - Py_DECREF(PyStrings::gPythonize); - PyStrings::gPythonize = nullptr; - - Py_DECREF(PyStrings::gArray); - PyStrings::gArray = nullptr; - Py_DECREF(PyStrings::gDType); - PyStrings::gDType = nullptr; - Py_DECREF(PyStrings::gFromBuffer); - PyStrings::gFromBuffer = nullptr; +#define CPYRT_DESTROY_PYSTRING(var, str) Py_CLEAR(PyStrings::var); + CPYRT_PYSTRINGS(CPYRT_DESTROY_PYSTRING) +#undef CPYRT_DESTROY_PYSTRING Py_RETURN_NONE; } diff --git a/bindings/pyroot/cppjit/src/cpyrt/PyStrings.h b/bindings/pyroot/cppjit/src/cpyrt/PyStrings.h index 64724f8b2420c..640704a6696af 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/PyStrings.h +++ b/bindings/pyroot/cppjit/src/cpyrt/PyStrings.h @@ -5,82 +5,92 @@ namespace cppjit::cpyrt { // python strings kept for performance reasons -namespace PyStrings { - -extern PyObject* gAssign; -extern PyObject* gBases; -extern PyObject* gBase; -extern PyObject* gContains; -extern PyObject* gCopy; -extern PyObject* gCppBool; -extern PyObject* gCppName; -extern PyObject* gAnnotations; -extern PyObject* gCastCpp; -extern PyObject* gCType; -extern PyObject* gDeref; -extern PyObject* gPreInc; -extern PyObject* gPostInc; -extern PyObject* gDict; -extern PyObject* gEmptyString; -extern PyObject* gEq; -extern PyObject* gFollow; -extern PyObject* gHasValue; -extern PyObject* gGetItem; -extern PyObject* gGetNoCheck; -extern PyObject* gSetItem; -extern PyObject* gInit; -extern PyObject* gIter; -extern PyObject* gLen; -extern PyObject* gLifeLine; -extern PyObject* gModule; -extern PyObject* gMRO; -extern PyObject* gName; -extern PyObject* gNe; -extern PyObject* gRepr; -extern PyObject* gCppRepr; -extern PyObject* gStr; -extern PyObject* gCppStr; -extern PyObject* gTypeCode; -extern PyObject* gCTypesType; - -extern PyObject* gUnderlying; -extern PyObject* gRealInit; - -extern PyObject* gAdd; -extern PyObject* gSub; -extern PyObject* gMul; -extern PyObject* gDiv; - -extern PyObject* gLShift; -extern PyObject* gLShiftC; +// X(variable, string): each entry declares PyStrings::variable, holding the +// interned python string for "string" +#define CPYRT_PYSTRINGS(X) \ + X(gAssign, "__assign__") \ + X(gBases, "__bases__") \ + X(gBase, "__base__") \ + X(gContains, "contains") \ + X(gCopy, "copy") \ + X(gCppBool, "__cpp_bool__") \ + X(gCppName, "__cpp_name__") \ + X(gAnnotations, "__annotations__") \ + X(gCastCpp, "__cast_cpp__") \ + X(gCType, "__ctype__") \ + X(gDeref, "__deref__") \ + X(gPreInc, "__preinc__") \ + X(gPostInc, "__postinc__") \ + X(gDict, "__dict__") \ + X(gEmptyString, "") \ + X(gEq, "__eq__") \ + X(gFollow, "__follow__") \ + X(gHasValue, "has_value") \ + X(gGetItem, "__getitem__") \ + X(gGetNoCheck, "_getitem__unchecked") \ + X(gSetItem, "__setitem__") \ + X(gInit, "__init__") \ + X(gIter, "__iter__") \ + X(gLen, "__len__") \ + X(gLifeLine, "__lifeline") \ + X(gModule, "__module__") \ + X(gMRO, "__mro__") \ + X(gName, "__name__") \ + X(gNe, "__ne__") \ + X(gRepr, "__repr__") \ + X(gCppRepr, "__cpp_repr") \ + X(gStr, "__str__") \ + X(gCppStr, "__cpp_str") \ + X(gTypeCode, "typecode") \ + X(gCTypesType, "_type_") \ + \ + X(gUnderlying, "__underlying") \ + X(gRealInit, "__real_init") \ + \ + X(gAdd, "__add__") \ + X(gSub, "__sub__") \ + X(gMul, "__mul__") \ + X(gDiv, "__truediv__") \ + \ + X(gLShift, "__lshift__") \ + X(gLShiftC, "__lshiftc__") \ + \ + X(gAt, "at") \ + X(gBegin, "begin") \ + X(gEnd, "end") \ + X(gFirst, "first") \ + X(gSecond, "second") \ + X(gSize, "size") \ + X(gTemplate, "Template") \ + X(gVectorAt, "_vector__at") \ + X(gInsert, "insert") \ + X(gValueType, "value_type") \ + X(gValueTypePtr, "_value_type") \ + X(gValueSize, "value_size") \ + \ + X(gCppReal, "__cpp_real") \ + X(gCppImag, "__cpp_imag") \ + \ + X(gThisModule, "cppjit") \ + \ + X(gDispInit, "_init_dispatchptr") \ + X(gDispGet, "_get_dispatch") \ + \ + X(gExPythonize, "__cppjit_explicit_pythonize__") \ + X(gPythonize, "__cppjit_pythonize__") \ + /* old cppyy names, see the legacy API section in Compatibility.h */ \ + X(gCppyyExPythonize, "__cppyy_explicit_pythonize__") \ + X(gCppyyPythonize, "__cppyy_pythonize__") \ + \ + X(gArray, "__array__") \ + X(gDType, "dtype") \ + X(gFromBuffer, "frombuffer") -extern PyObject* gAt; -extern PyObject* gBegin; -extern PyObject* gEnd; -extern PyObject* gFirst; -extern PyObject* gSecond; -extern PyObject* gSize; -extern PyObject* gTemplate; -extern PyObject* gVectorAt; -extern PyObject* gInsert; -extern PyObject* gValueType; -extern PyObject* gValueTypePtr; -extern PyObject* gValueSize; - -extern PyObject* gCppReal; -extern PyObject* gCppImag; - -extern PyObject* gThisModule; - -extern PyObject* gDispInit; -extern PyObject* gDispGet; - -extern PyObject* gExPythonize; -extern PyObject* gPythonize; +namespace PyStrings { -extern PyObject* gArray; -extern PyObject* gDType; -extern PyObject* gFromBuffer; +#define CPYRT_DECLARE_PYSTRING(var, str) extern PyObject* var; +CPYRT_PYSTRINGS(CPYRT_DECLARE_PYSTRING) +#undef CPYRT_DECLARE_PYSTRING } // namespace PyStrings diff --git a/bindings/pyroot/cppjit/src/cpyrt/Pythonize.cxx b/bindings/pyroot/cppjit/src/cpyrt/Pythonize.cxx index 5781345db3fdd..39aa101b2f493 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/Pythonize.cxx +++ b/bindings/pyroot/cppjit/src/cpyrt/Pythonize.cxx @@ -5,6 +5,7 @@ using namespace cppjit; #include "CPPFunction.h" #include "CPPInstance.h" #include "CPPOverload.h" +#include "Compatibility.h" #include "Converters.h" #include "CustomPyTypes.h" #include "LowLevelViews.h" @@ -686,10 +687,10 @@ static PyObject* vector_iter(PyObject* v) { if (!vi) return nullptr; - Py_INCREF(v); vi->ii_container = v; // tell the iterator code to set a life line if this container is a temporary + // (checked before taking the iterator's own reference) vi->vi_flags = vectoriterobject::kDefault; #if PY_VERSION_HEX >= 0x030e0000 if (PyUnstable_Object_IsUniqueReferencedTemporary(v) || @@ -2337,21 +2338,19 @@ bool cpyrt::Pythonize(PyObject* pyclass, interop::TCppScope_t scope) { // preferred if present // - normal pythonization: only called if explicit isn't present, falls // through to base classes + // The lookups live in Compatibility.h because they also accept the old + // cppyy names of these hooks. bool bUserOk = true; PyObject* res = nullptr; PyObject* pyname = PyUnicode_FromString(name.c_str()); - if (HasAttrDirect(pyclass, PyStrings::gExPythonize)) { - res = PyObject_CallMethodObjArgs(pyclass, PyStrings::gExPythonize, pyclass, - pyname, nullptr); + if (PyObject* exPythonize = compat::ExplicitPythonizeHookName(pyclass)) { + res = PyObject_CallMethodObjArgs(pyclass, exPythonize, pyclass, pyname, + nullptr); + bUserOk = (bool)res; + } else if (PyObject* func = compat::GetPythonizeHook(pyclass)) { + res = PyObject_CallFunctionObjArgs(func, pyclass, pyname, nullptr); + Py_DECREF(func); bUserOk = (bool)res; - } else { - PyObject* func = PyObject_GetAttr(pyclass, PyStrings::gPythonize); - if (func) { - res = PyObject_CallFunctionObjArgs(func, pyclass, pyname, nullptr); - Py_DECREF(func); - bUserOk = (bool)res; - } else - PyErr_Clear(); } if (!bUserOk) { Py_DECREF(pyname); diff --git a/bindings/pyroot/cppjit/src/cpyrt/TemplateProxy.cxx b/bindings/pyroot/cppjit/src/cpyrt/TemplateProxy.cxx index 4fc68fbde55c3..e3b6dddaa16ff 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/TemplateProxy.cxx +++ b/bindings/pyroot/cppjit/src/cpyrt/TemplateProxy.cxx @@ -341,9 +341,23 @@ PyObject* TemplateProxy::Instantiate(const std::string& fname, // Case 3/4: pre-existing method that was either not found b/c the full // templated name was constructed in this call or it failed as overload else if (bIsCppOL) { - // TODO: see above, since the call hasn't happened yet, this overload may - // already exist and fail again. - ((CPPOverload*)pyol)->AdoptMethod(meth); // takes ownership + // The lookup is cached before the call, so when the call then fails + // (e.g. an implicit conversion attempt that is not allowed to nest), the + // next call instantiates the same method again. Adopting it again would + // grow the overload set with duplicates and force the whole set to be + // ranked anew on every call. + bool known = false; + for (PyCallable* pc : ((CPPOverload*)pyol)->fMethodInfo->fMethods) { + auto* existing = dynamic_cast(pc); + if (existing && existing->GetMethod() == cppmeth) { + known = true; + break; + } + } + if (known) + delete meth; + else + ((CPPOverload*)pyol)->AdoptMethod(meth); // takes ownership } // Case 5: must be a template proxy, meaning that current template name is diff --git a/bindings/pyroot/cppjit/src/cpyrt/Utility.cxx b/bindings/pyroot/cppjit/src/cpyrt/Utility.cxx index 9538a57b46260..ab5bf08016a22 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/Utility.cxx +++ b/bindings/pyroot/cppjit/src/cpyrt/Utility.cxx @@ -1200,6 +1200,143 @@ std::unordered_map const& cpyrt::Utility::TypecodeMap() { return typecodeMap; } +namespace { + +// classification of PEP 3118 format characters +enum class EBufferKind { + kSInt, + kUInt, + kFloat, + kComplex, + kChar, + kWChar, + kBool, + kString, + kVoidPtr, + kUnknown +}; + +struct BufferElem { + EBufferKind kind; + std::size_t size; + char code; // the underlying format character +}; + +BufferElem ClassifyFormatChar(char c) { + switch (c) { + case 'b': + return {EBufferKind::kSInt, sizeof(char), c}; + case 'h': + return {EBufferKind::kSInt, sizeof(short), c}; + case 'i': + return {EBufferKind::kSInt, sizeof(int), c}; + case 'l': + return {EBufferKind::kSInt, sizeof(long), c}; + case 'q': + return {EBufferKind::kSInt, sizeof(long long), c}; + case 'n': + return {EBufferKind::kSInt, sizeof(Py_ssize_t), c}; + case 'B': + return {EBufferKind::kUInt, sizeof(unsigned char), c}; + case 'H': + return {EBufferKind::kUInt, sizeof(unsigned short), c}; + case 'I': + return {EBufferKind::kUInt, sizeof(unsigned int), c}; + case 'L': + return {EBufferKind::kUInt, sizeof(unsigned long), c}; + case 'Q': + return {EBufferKind::kUInt, sizeof(unsigned long long), c}; + case 'N': + return {EBufferKind::kUInt, sizeof(std::size_t), c}; + case 'e': + return {EBufferKind::kFloat, 2, c}; // IEEE 754 half + case 'f': + return {EBufferKind::kFloat, sizeof(float), c}; + case 'd': + return {EBufferKind::kFloat, sizeof(double), c}; + case 'g': + return {EBufferKind::kFloat, sizeof(long double), c}; + case '?': + return {EBufferKind::kBool, sizeof(bool), c}; + case 'c': + return {EBufferKind::kChar, sizeof(char), c}; + case 'u': + return {EBufferKind::kWChar, sizeof(wchar_t), c}; + case 'w': + return {EBufferKind::kWChar, sizeof(char32_t), c}; // UCS-4 + case 's': + case 'p': + return {EBufferKind::kString, 0, c}; + case 'P': + return {EBufferKind::kVoidPtr, sizeof(void*), c}; + default: + return {EBufferKind::kUnknown, 0, c}; + } +} + +// reduce a PEP 3118 format string to its element type, skipping an optional +// byte-order/alignment prefix ("!^", *fmt)) + ++fmt; + while ('0' <= *fmt && *fmt <= '9') + ++fmt; + bool isComplex = *fmt == 'Z'; + if (isComplex) + ++fmt; + if (!*fmt || fmt[1] != '\0') + return false; + + BufferElem elem = ClassifyFormatChar(*fmt); + if (isComplex) { + if (elem.kind != EBufferKind::kFloat) + return false; + elem = {EBufferKind::kComplex, 2 * elem.size, *fmt}; + } + elemOut = elem; + return true; +} + +} // namespace + +//---------------------------------------------------------------------------- +bool cpyrt::Utility::BufferFormatCompatible(char tc, const char* fmt) { + // the requested type code: a PEP 3118 character, or a cppjit-internal + // code ('z' = std::complex, 'Z' = std::complex) + BufferElem want; + if (tc == 'z') + want = {EBufferKind::kComplex, 2 * sizeof(float), tc}; + else if (tc == 'Z') + want = {EBufferKind::kComplex, 2 * sizeof(double), tc}; + else + want = ClassifyFormatChar(tc); + if (want.kind == EBufferKind::kUnknown) + return false; + + BufferElem got; + if (!ParseBufferFormat(fmt, got) || got.kind == EBufferKind::kUnknown) + return false; + + // the struct/array modules express bool-ish values as 'signed char' ('b') + if (want.kind == EBufferKind::kBool && got.code == 'b') + return true; + + // 's' and 'p' are both strings but not interchangeable + if (want.kind == EBufferKind::kString) + return got.code == want.code; + + return want.kind == got.kind && want.size == got.size; +} + //---------------------------------------------------------------------------- Py_ssize_t cpyrt::Utility::GetBuffer(PyObject* pyobject, char tc, int size, void*& buf, bool check) { @@ -1230,19 +1367,7 @@ Py_ssize_t cpyrt::Utility::GetBuffer(PyObject* pyobject, char tc, int size, Py_buffer bufinfo; memset(&bufinfo, 0, sizeof(Py_buffer)); if (PyObject_GetBuffer(pyobject, &bufinfo, PyBUF_FORMAT) == 0) { - if (tc == '*' || - strchr(bufinfo.format, tc) - // if `long int` and `int` are the same size (on Windows and 32bit - // Linux, for example), `ctypes` isn't too picky about the type - // format, so make sure both integer types pass the type check - || (sizeof(long int) == sizeof(int) && - ((tc == 'I' && strchr(bufinfo.format, 'L')) || - (tc == 'i' && strchr(bufinfo.format, 'l')))) - // complex float is 'Zf' in bufinfo.format, but 'z' in single char - || (tc == 'z' && strstr(bufinfo.format, "Zf")) - // allow 'signed char' ('b') from array to pass through '?' (bool as - // from struct) - || (tc == '?' && strchr(bufinfo.format, 'b'))) { + if (tc == '*' || BufferFormatCompatible(tc, bufinfo.format)) { buf = bufinfo.buf; if (check && bufinfo.itemsize != size) { diff --git a/bindings/pyroot/cppjit/src/cpyrt/Utility.h b/bindings/pyroot/cppjit/src/cpyrt/Utility.h index acd6936ce7880..28186c31f12f0 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/Utility.h +++ b/bindings/pyroot/cppjit/src/cpyrt/Utility.h @@ -73,6 +73,14 @@ bool InitProxy(PyObject* module, PyTypeObject* pytype, const char* name); std::unordered_map const& TypecodeMap(); +// check whether a PEP 3118 buffer format string describes an element type +// compatible with the requested type code: same kind (signed/unsigned +// integer, floating point, complex, ...) and same native size, so e.g. +// 'long' and 'long long' buffers are interchangeable where they coincide. +// The type code may be a PEP 3118 character or a cppjit-internal code: +// 'z' (std::complex) or 'Z' (std::complex). +bool BufferFormatCompatible(char tc, const char* fmt); + // retrieve the memory buffer from pyobject, return buflength, tc (optional) is // python array.array type code, size is type size, buf will point to buffer, // and if check is true, some heuristics will be applied to check buffer diff --git a/bindings/pyroot/cppjit/src/interop/cppjit_interop.h b/bindings/pyroot/cppjit/src/interop/cppjit_interop.h index db2ccf0b51dc4..574bbdca60607 100644 --- a/bindings/pyroot/cppjit/src/interop/cppjit_interop.h +++ b/bindings/pyroot/cppjit/src/interop/cppjit_interop.h @@ -109,6 +109,9 @@ RPY_EXPORTED TCppScope_t GetNamed(const std::string& scope_name, TCppScope_t parent_scope = TCppScope_t{}); RPY_EXPORTED +bool IsVisibleName(const std::string& name, + TCppScope_t parent_scope = TCppScope_t{}); +RPY_EXPORTED TCppScope_t GetParentScope(TCppScope_t scope); RPY_EXPORTED TCppScope_t GetScopeFromType(TCppType_t type); @@ -248,7 +251,7 @@ bool IsSubclass(TCppScope_t derived, TCppScope_t base); RPY_EXPORTED bool IsSmartPtr(TCppScope_t klass); RPY_EXPORTED -bool GetSmartPtrInfo(const std::string&, TCppScope_t* raw, TCppMethod_t* deref); +bool GetSmartPtrInfo(TCppScope_t klass, TCppScope_t* raw, TCppMethod_t* deref); // calculate offsets between declared and actual type, up-cast: direction > 0; // down-cast: direction < 0 RPY_EXPORTED @@ -297,6 +300,15 @@ RPY_EXPORTED TCppType_t GetFnTypeFromStdFn(TCppType_t fn_type); RPY_EXPORTED void GetFnTypeSig(TCppType_t fn_type, std::vector& arg_types); +/// Spells out a function type, or a pointer or reference to one, as the +/// return type and the parenthesized argument list, e.g. "int" and +/// "(double, const char *)"; the argument list is formatted as by +/// GetMethodSignature(). Member function pointers are not function pointers +/// and are rejected. +/// \returns false if \p fn_type is not a function (pointer) type. +RPY_EXPORTED +bool GetFnTypeSigAsString(TCppType_t fn_type, std::string& ret_type, + std::string& signature); RPY_EXPORTED bool IsSameType(TCppType_t typ1, TCppType_t typ2); RPY_EXPORTED diff --git a/bindings/pyroot/cppjit/src/interop/interop_wrapper.cxx b/bindings/pyroot/cppjit/src/interop/interop_wrapper.cxx index ef48f950e50d2..e4674b8e76a1c 100644 --- a/bindings/pyroot/cppjit/src/interop/interop_wrapper.cxx +++ b/bindings/pyroot/cppjit/src/interop/interop_wrapper.cxx @@ -574,6 +574,36 @@ bool interop::AppendTypesSlow(const std::string& name, std::lock_guard Lock(InterOpMutex); + // Every lookup below declares code through the interpreter, which parses, + // commits a transaction and leaves a new declaration in the AST, so each one + // costs more than the last. The same names come back constantly (overload + // ranking resolves every argument type of every candidate on each call that + // is not memoized), so remember what a name resolved to. A name that fails + // may still resolve once more declarations are available, so only successes + // are cached. + using SlowTypesKey = std::pair; + static std::map> s_cache; + const SlowTypesKey cache_key{name, parent.data}; + // the integral values are strings whose ownership passes to the caller + // (which frees them), so the cache keeps and hands out copies of its own + auto copy_of = [](const Cpp::TemplateArgInfo& info) { + return Cpp::TemplateArgInfo(info.m_Type, info.m_IntegralValue + ? strdup(info.m_IntegralValue) + : nullptr); + }; + if (auto cached = s_cache.find(cache_key); cached != s_cache.end()) { + for (const Cpp::TemplateArgInfo& info : cached->second) + types.push_back(copy_of(info)); + return false; + } + const size_t n_types_before = types.size(); + auto remember = [&] { + std::vector resolved; + for (size_t i = n_types_before; i < types.size(); ++i) + resolved.push_back(copy_of(types[i])); + s_cache.emplace(cache_key, std::move(resolved)); + }; + // We might have an entire expression such as int, double. static unsigned long long struct_count = 0; std::string code = @@ -628,7 +658,10 @@ bool interop::AppendTypesSlow(const std::string& name, continue; // recovered-but-broken decl; try next candidate or split path size_t oldSize = types.size(); Cpp::GetClassTemplateInstantiationArgs(instance_class, types); - return oldSize == types.size(); + if (oldSize == types.size()) + return true; + remember(); + return false; } } @@ -679,6 +712,7 @@ bool interop::AppendTypesSlow(const std::string& name, strdup(std::to_string(Cpp::GetEnumConstantValue(scope)).c_str()); types.emplace_back(type.data, integral_value); } + remember(); return false; } @@ -764,12 +798,30 @@ interop::TCppScope_t interop::GetUnderlyingScope(TCppScope_t scope) { interop::TCppScope_t interop::GetScope(const std::string& name, TCppScope_t parent_scope) { std::unique_lock Lock(InterOpMutex); + // Resolving a templated name below declares a new variable in the + // interpreter (see AppendTypesSlow), so remember the result to not declare + // one, and grow the memory, on every lookup of the same name. + static std::map, TCppScope_t> template_scopes; + const bool is_template = name.find('<') != std::string::npos; + const auto key = std::make_pair(parent_scope.data, name); + if (is_template) + if (auto it = template_scopes.find(key); it != template_scopes.end()) + return it->second; + // CppInterOp directly looks at the AST which is not enough. // We require lazy module loading that ROOT relies on, so we do it here // first. Use TClass::GetClass to trigger auto-loading of dictionaries and // modules. + // This also covers nested names: declarations come from the C++ modules, + // so e.g. ROOT::RDF can be used before its libraries are loaded, and the + // JIT then finds their symbols only by scanning every library on the + // search path (about half a second). if (!parent_scope || parent_scope == Cpp::GetGlobalScope()) TClass::GetClass(name.c_str(), true /* load */, true /* silent */); + else if (Cpp::IsNamespace(parent_scope) || Cpp::IsClass(parent_scope)) + TClass::GetClass( + (Cpp::GetQualifiedCompleteName(parent_scope) + "::" + name).c_str(), + true /* load */, true /* silent */); if (interop::TCppScope_t scope = Cpp::GetScope(name, parent_scope)) return scope; @@ -792,7 +844,7 @@ interop::TCppScope_t interop::GetScope(const std::string& name, } // FIXME: avoid string parsing here - if (name.find('<') != std::string::npos) { + if (is_template) { // Templated type; may need instantiation. Resolve the whole type // expression (e.g. "std::array") and read back its scope. // Splitting off the argument list and resolving it directly cannot @@ -815,8 +867,10 @@ interop::TCppScope_t interop::GetScope(const std::string& name, // it, so the specialization may still be declared-but-undefined. // Force its definition: callers expect a complete scope, e.g. to // walk its base classes. - if (scope) + if (scope) { Cpp::IsComplete(scope); + template_scopes[key] = scope; + } return scope; } } @@ -853,6 +907,11 @@ interop::TCppScope_t interop::GetNamed(const std::string& name, return nullptr; } +bool interop::IsVisibleName(const std::string& name, TCppScope_t parent_scope) { + std::lock_guard Lock(InterOpMutex); + return Cpp::IsVisibleName(name, parent_scope); +} + interop::TCppScope_t interop::GetParentScope(TCppScope_t scope) { std::lock_guard Lock(InterOpMutex); return Cpp::GetParentScope(scope); @@ -1338,6 +1397,12 @@ interop::TCppIndex_t interop::GetNumBases(TCppScope_t klass) { // Autoloading may have registered only a forward declaration of the class; // complete it, or the proxy is built without its bases. Cpp::GetOrForceDefinition(klass); + // Cpp::GetNumBases() instantiates a class template specialization with + // diagnostics, printing an error for one that can't be defined, e.g. a + // specialization of a template that is only declared. IsComplete() + // instantiates it silently. + if (!Cpp::IsComplete(klass)) + return 0; return Cpp::GetNumBases(klass); } @@ -1411,23 +1476,21 @@ bool interop::IsSmartPtr(TCppScope_t klass) { return false; } -bool interop::GetSmartPtrInfo(const std::string& tname, TCppScope_t* raw, +bool interop::GetSmartPtrInfo(TCppScope_t klass, TCppScope_t* raw, TCppMethod_t* deref) { - // TODO: We can directly accept scope instead of name - const std::string& rn = ResolveName(tname); - if (gSmartPtrTypes.find(rn.substr(0, rn.find("<"))) == gSmartPtrTypes.end()) + TCppScope_t scope = interop::GetUnderlyingScope(klass); + if (!scope || !interop::IsSmartPtr(scope)) return false; if (!raw && !deref) return true; - TCppScope_t scope = interop::GetScope(rn); - if (!scope) - return false; - std::vector ops; { std::lock_guard Lock(InterOpMutex); + // the operator can only be found once the specialization is instantiated + if (!Cpp::IsComplete(scope)) + return false; Cpp::GetOperator(scope, Cpp::Operator::OP_Arrow, ops, /*kind=*/Cpp::OperatorArity::kBoth); if (ops.size() != 1) @@ -1453,17 +1516,24 @@ bool interop::GetSmartPtrInfo(const std::string& tname, TCppScope_t* raw, // type offsets -------------------------------------------------------------- ptrdiff_t interop::GetBaseOffset(TCppScope_t derived, TCppScope_t base, - TCppObject_t /*address*/, int direction, + TCppObject_t address, int direction, bool rerror) { std::lock_guard Lock(InterOpMutex); + // Up-casts may start from a subobject whose complete type is unknown. For + // down-casts, callers obtain the complete type with GetActualClass, so use + // its static layout; address points to the base rather than the derived type. + const void* object = direction > 0 ? address.data : nullptr; + // Queried for every downcast object handed to Python; computing it walks - // the inheritance paths, so memoize per class pair. The object address is - // not used, so the offset is fixed per pair once both records are complete. + // the inheritance paths, so memoize per class pair. Only the static layout + // (no object address) is fixed per pair once both records are complete; an + // offset read from the object's vtable is not cached. static std::map, intptr_t> s_base_offset_cache; const auto cacheKey = std::make_pair(derived.data, base.data); - auto cached = s_base_offset_cache.find(cacheKey); + auto cached = + object ? s_base_offset_cache.end() : s_base_offset_cache.find(cacheKey); intptr_t offset = 0; if (cached != s_base_offset_cache.end()) { offset = cached->second; @@ -1475,12 +1545,13 @@ ptrdiff_t interop::GetBaseOffset(TCppScope_t derived, TCppScope_t base, if (!Cpp::IsComplete(derived) || !Cpp::IsComplete(base)) return rerror ? (ptrdiff_t)-1 : 0; - offset = Cpp::GetBaseClassOffset(derived, base); + offset = Cpp::GetBaseClassOffset(derived, base, object); if (offset == -1) // Cling error, treat silently return rerror ? (ptrdiff_t)offset : 0; - s_base_offset_cache.emplace(cacheKey, offset); + if (!object) + s_base_offset_cache.emplace(cacheKey, offset); } return (ptrdiff_t)(direction < 0 ? -offset : offset); @@ -1666,13 +1737,41 @@ interop::TCppType_t interop::GetFnTypeFromStdFn(TCppType_t fn_type) { void interop::GetFnTypeSig(TCppType_t fn_type, std::vector& arg_types) { + if (!fn_type) + return; fn_type = Cpp::IsReferenceType(fn_type) ? Cpp::GetNonReferenceType(fn_type) : fn_type; fn_type = Cpp::IsPointerType(fn_type) ? Cpp::GetPointeeType(fn_type) : fn_type; + // Cpp::GetFnTypeSignature() requires a function prototype: anything else + // (e.g. a member function pointer, or a plain object type) yields nothing + if (!Cpp::IsFunctionProtoType(fn_type)) + return; Cpp::GetFnTypeSignature(fn_type, arg_types); } +bool interop::GetFnTypeSigAsString(TCppType_t fn_type, std::string& ret_type, + std::string& signature) { + // the first entry is the return type, followed by the argument types + std::vector types; + GetFnTypeSig(fn_type, types); + if (types.empty()) + return false; + + ret_type = GetTypeAsString(types[0]); + + std::ostringstream sig; + sig << "("; + for (std::size_t iarg = 1; iarg < types.size(); ++iarg) { + if (iarg != 1) + sig << ", "; + sig << GetTypeAsString(types[iarg]); + } + sig << ")"; + signature = sig.str(); + return true; +} + bool interop::IsSameType(TCppType_t typ1, TCppType_t typ2) { return Cpp::IsSameType(typ1, typ2); } diff --git a/bindings/pyroot/cppjit/test/CMakeLists.txt b/bindings/pyroot/cppjit/test/CMakeLists.txt index 482c2abb89102..5932f461d68f7 100644 --- a/bindings/pyroot/cppjit/test/CMakeLists.txt +++ b/bindings/pyroot/cppjit/test/CMakeLists.txt @@ -14,7 +14,11 @@ cmake_path(CONVERT "${pythonpaths}" TO_NATIVE_PATH_LIST pythonpaths_native) set(common_env PYTHONPATH=${pythonpaths_native} CPPJIT_TEST_SKIP_MAKE=1) if(NOT MSVC) - list(APPEND common_env ${ld_library_path}=${CMAKE_CURRENT_BINARY_DIR}) + # Keep the library path of the environment, which may be needed to find, + # for example, the C++ standard library of the compiler that built ROOT. + set(ld_library_paths ${CMAKE_CURRENT_BINARY_DIR} $ENV{${ld_library_path}}) + cmake_path(CONVERT "${ld_library_paths}" TO_NATIVE_PATH_LIST ld_library_paths_native) + list(APPEND common_env ${ld_library_path}=${ld_library_paths_native}) endif() # Some dictionary sources include Python.h (e.g. pythonizables.cxx), which diff --git a/bindings/pyroot/cppjit/test/cpp/pythonizables.cxx b/bindings/pyroot/cppjit/test/cpp/pythonizables.cxx index c6c7eade2f476..77e89487509f2 100644 --- a/bindings/pyroot/cppjit/test/cpp/pythonizables.cxx +++ b/bindings/pyroot/cppjit/test/cpp/pythonizables.cxx @@ -110,3 +110,31 @@ std::string pyzables::WithCallback2::klass_name{"not set"}; int pyzables::WithCallback3::get_int() { return 2 * m_int; } void pyzables::WithCallback3::set_int(int i) { m_int = 2 * i; } + +pyzables::WithCppyyCallback1::WithCppyyCallback1(int i) : m_int(i) {} + +int pyzables::WithCppyyCallback1::get_int() { return m_int; } +void pyzables::WithCppyyCallback1::set_int(int i) { m_int = i; } + +void pyzables::WithCppyyCallback1::__cppyy_explicit_pythonize__( + PyObject* klass, const std::string& name) { + replace_method_name(klass, "get_int", "GetInt"); + replace_method_name(klass, "set_int", "SetInt"); + klass_name = name; +} + +std::string pyzables::WithCppyyCallback1::klass_name{"not set"}; + +pyzables::WithCppyyCallback2::WithCppyyCallback2(int i) : m_int(i) {} + +int pyzables::WithCppyyCallback2::get_int() { return m_int; } +void pyzables::WithCppyyCallback2::set_int(int i) { m_int = i; } + +void pyzables::WithCppyyCallback2::__cppyy_pythonize__( + PyObject* klass, const std::string& name) { + replace_method_name(klass, "get_int", "GetInt"); + replace_method_name(klass, "set_int", "SetInt"); + klass_name = name; +} + +std::string pyzables::WithCppyyCallback2::klass_name{"not set"}; diff --git a/bindings/pyroot/cppjit/test/cpp/pythonizables.h b/bindings/pyroot/cppjit/test/cpp/pythonizables.h index 3ec1c6f50d800..8eba94fefbef9 100644 --- a/bindings/pyroot/cppjit/test/cpp/pythonizables.h +++ b/bindings/pyroot/cppjit/test/cpp/pythonizables.h @@ -161,4 +161,37 @@ class WithCallback3 : public WithCallback2 { void set_int(int i); }; +// same as WithCallback1 and WithCallback2, but with the old cppyy names +class WithCppyyCallback1 { +public: + WithCppyyCallback1(int i); + +public: + int get_int(); + void set_int(int i); + +private: + int m_int; + +public: + static void __cppyy_explicit_pythonize__(PyObject* klass, const std::string&); + static std::string klass_name; +}; + +class WithCppyyCallback2 { +public: + WithCppyyCallback2(int i); + +public: + int get_int(); + void set_int(int i); + +private: + int m_int; + +public: + static void __cppyy_pythonize__(PyObject* klass, const std::string&); + static std::string klass_name; +}; + } // namespace pyzables diff --git a/bindings/pyroot/cppjit/test/test_advancedcpp.py b/bindings/pyroot/cppjit/test/test_advancedcpp.py index b59ddc59e1af4..cab0d20536bb1 100644 --- a/bindings/pyroot/cppjit/test/test_advancedcpp.py +++ b/bindings/pyroot/cppjit/test/test_advancedcpp.py @@ -1034,6 +1034,74 @@ def __cast_cpp__(self): for norm in [ns.norm_cr, ns.norm_m, ns.norm_v]: assert round(norm(p3) - pynorm, 8) == 0 + def test30_virtual_base_of_hidden_type(self): + """Offset to a virtual base taken from the actual object""" + + import cppjit + + cppjit.cppdef("""\ + namespace vbase_hidden { + struct IB { virtual ~IB() {} virtual long id() const { return 42; } }; + struct ID1 : virtual IB {}; + struct Lead { virtual ~Lead() {} long padding[2] = {}; }; + struct ID2 : Lead, virtual IB {}; + struct IC { virtual ~IC() {} long z = 7; }; + struct M : Lead, IC {}; + struct ID3 : virtual M {}; + long call(IB* p) { return p->id(); } + long call_c(IC* p) { return p->z; } + } + namespace { + struct Other { virtual ~Other() {} long x[100] = {}; }; + struct Hidden : Other, vbase_hidden::ID1, vbase_hidden::ID2 {}; + struct Hidden3 : Other, vbase_hidden::ID1, vbase_hidden::ID3 { long y = 3; }; + } + namespace vbase_hidden { + void* make2() { return static_cast(new Hidden); } + void* make3() { return static_cast(new Hidden3); } + }""") + + ns = cppjit.gbl.vbase_hidden + + d2 = cppjit.bind_object(ns.make2(), ns.ID2) + assert ns.call(d2) == 42 + assert d2.id() == 42 + + d3 = cppjit.bind_object(ns.make3(), ns.ID3) + assert ns.call_c(d3) == 7 + assert d3.z == 7 + + def test31_virtual_base_downcast(self): + """GetActualClass supplies the complete type for virtual-base downcasts""" + + import cppjit + + cppjit.cppdef("""\ + namespace vbase_downcast { + struct IB { virtual ~IB() {} virtual long id() const { return 42; } }; + struct ID1 : virtual IB {}; + struct ID2 : virtual IB {}; + struct IC { virtual ~IC() {} long z = 7; }; + struct Pad { virtual ~Pad() {} long padding[2] = {}; }; + struct M : Pad, IC {}; + struct ID3 : virtual M {}; + struct Visible : ID1, ID2 { long value = 19; }; + struct Visible3 : ID1, ID3 { long value = 23; }; + IB* make() { return new Visible; } + IC* make3() { return new Visible3; } + }""") + + ns = cppjit.gbl.vbase_downcast + obj = ns.make() + assert isinstance(obj, ns.Visible) + assert obj.value == 19 + assert obj.id() == 42 + + obj3 = ns.make3() + assert isinstance(obj3, ns.Visible3) + assert obj3.value == 23 + assert obj3.z == 7 + class TestQUALIFIEDLOOKUP: def test01_qualified_name_in_namespace(self): diff --git a/bindings/pyroot/cppjit/test/test_datatypes.py b/bindings/pyroot/cppjit/test/test_datatypes.py index 5e508d32dcee6..2c41d66ec78ed 100644 --- a/bindings/pyroot/cppjit/test/test_datatypes.py +++ b/bindings/pyroot/cppjit/test/test_datatypes.py @@ -1,3 +1,4 @@ +import subprocess import sys import py @@ -7,6 +8,20 @@ currpath = py.path.local(__file__).dirpath() test_dct = "datatypes_cxx" +IS_MACOS_27_PLUS = False + +try: + vers = ( + subprocess.check_output(["xcrun", "--sdk", "macosx", "--show-sdk-version"]) + .decode() + .strip() + ) + if int(vers.split(".")[0]) >= 27: + IS_MACOS_27_PLUS = True +except FileNotFoundError: + # If xcrun is not on the system, we assume it's not macOS 27+ + pass + def setup_module(mod): setup_make("datatypes") @@ -1009,6 +1024,10 @@ def test15_object_returns(self): assert c.get_pod_ptrref().m_int == 666 assert c.get_pod_ptrref().m_double == 3.14 + @mark.xfail( + condition=IS_MACOS_27_PLUS, + reason="Mutating proxy pointee via T** corrupts memory regulator, resulting in crashes on newer macOS", + ) def test16_object_arguments(self): """Test setting and returning of a POD through arguments""" @@ -2777,6 +2796,212 @@ def test55_qt_cache_alias_collision(self): ns.take_int8(101) raises(TypeError, ns.take_int8, "e") + def test56_function_pointer_return_multiword_types(self): + """A returned function pointer whose signature has multi-word types""" + + import cppjit + + cppjit.cppdef(r"""\ + namespace ns56 { + typedef unsigned long (*func_t)(const char *, unsigned int); + unsigned long add_len(const char *s, unsigned int n) { return std::string(s).size() + n; } + func_t get() { return add_len; } + }""") + + # the std::function wrapping the returned pointer is compiled from + # its signature, which must keep "const char" and "unsigned int" + assert cppjit.gbl.ns56.get()("abc", 4) == 7 + + def test57_long_long_long_array_interchange(self): + """Arrays of 64b integers must convert to both 'long' and 'long long' + pointer parameters, as buffer producers do not distinguish them and the + mapping of int64_t differs across platforms.""" + + import array + + import cppjit + + try: + import numpy as np + except ImportError: + skip("numpy is not installed") + + # 'long' and 'long long' must be same size + if array.array("l", []).itemsize != 8: + skip("test assumes 64b long") + + cppjit.cppdef("""\ + namespace ArrayInterchange { + long long sum_ll(const long long* a, int n) { + long long s = 0; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + long long sum_l(const long* a, int n) { return sum_ll((const long long*)a, n); } + unsigned long long sum_ull(const unsigned long long* a, int n) { + unsigned long long s = 0; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + unsigned long long sum_ul(const unsigned long* a, int n) { return sum_ull((const unsigned long long*)a, n); } + long long sum_i64(const int64_t* a, int n) { return sum_ll((const long long*)a, n); } + const long* data_l() { static long l[3] = {0, 1, 2}; return l; } + const long long* data_ll() { static long long l[3] = {0, 1, 2}; return l; } + }""") + + ns = cppjit.gbl.ArrayInterchange + n = 3 + isum = sum(range(n)) + + # numpy int64 is canonicalized to 'l' or 'q' by the platform + ia = np.arange(n, dtype=np.int64) + ua = np.arange(n, dtype=np.uint64) + for func in [ns.sum_ll, ns.sum_l, ns.sum_i64]: + assert func(ia, n) == isum + for func in [ns.sum_ull, ns.sum_ul]: + assert func(ua, n) == isum + + # same through the standard 'array' module, which can express both formats + al, aq = array.array("l", range(n)), array.array("q", range(n)) + assert ns.sum_ll(al, n) == isum and ns.sum_ll(aq, n) == isum + assert ns.sum_l(al, n) == isum and ns.sum_l(aq, n) == isum + + # same for low-level views returned to Python with the other type + assert ns.sum_ll(ns.data_l(), n) == isum + assert ns.sum_l(ns.data_ll(), n) == isum + + # same through std::span, if available (needs C++20 and ) + if ( + cppjit.evaluate("""#if __cplusplus >= 202002L && __has_include() + 1 + #else + 0 + #endif""") + == 1 + ): + cppjit.cppdef("""\ + #include + namespace ArrayInterchange { + long long sum_span_ll(std::span a) { + long long s = 0; + for (auto v : a) s += v; + return s; + } + long long sum_span_l(std::span a) { + return sum_span_ll(std::span{reinterpret_cast(a.data()), a.size()}); + } + }""") + assert ns.sum_span_ll(ia) == isum and ns.sum_span_l(ia) == isum + assert ns.sum_span_ll(al) == isum and ns.sum_span_ll(aq) == isum + + def test58_buffer_format_parsing(self): + """Buffer-format matching parses PEP 3118 properly: byte-order + prefixes are skipped, the complex marker 'Z' is understood, and + elements match on kind (signed/unsigned/float/complex) and native + size rather than on the exact format character.""" + + import ctypes + + import cppjit + + try: + import numpy as np + except ImportError: + skip("numpy is not installed") + + cppjit.cppdef("""\ + #include + namespace BufferFormats { + double sum_d(const double* a, int n) { + double s = 0.; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + long sum_l(const long* a, int n) { + long s = 0; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + int sum_i(const int* a, int n) { + int s = 0; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + int count_true(const bool* a, int n) { + int s = 0; + for (int i = 0; i < n; ++i) if (a[i]) ++s; + return s; + } + std::complex sum_cf(const std::complex* a, int n) { + std::complex s; + for (int i = 0; i < n; ++i) s += a[i]; + return s; + } + const std::complex* data_cf() { + static std::complex c[2] = {{1.f, 1.f}, {1.f, -1.f}}; + return c; + } + }""") + + ns = cppjit.gbl.BufferFormats + + # ctypes exports explicitly-endian formats + assert ns.sum_d((ctypes.c_double * 3)(0.5, 1.5, 1.0), 3) == 3.0 + if ctypes.sizeof(ctypes.c_long) == 8: + assert ns.sum_l((ctypes.c_long * 3)(1, 2, 3), 3) == 6 + # ctypes exports c_longlong as ' + if ( + cppjit.evaluate("""#if __cplusplus >= 202002L && __has_include() + 1 + #else + 0 + #endif""") + == 1 + ): + cppjit.cppdef("""\ + #include + namespace BufferFormats { + double sum_span_d(std::span a) { + double s = 0.; + for (auto v : a) s += v; + return s; + } + }""") + assert ns.sum_span_d(np.arange(3, dtype=np.float64)) == 3.0 + raises( + TypeError, + ns.sum_span_d, + np.array([1 + 2j, 3 + 4j], dtype=np.complex128), + ) + class TestANONENUM: def test01_anonymous_enum_repeated_access(self): diff --git a/bindings/pyroot/cppjit/test/test_pythonization.py b/bindings/pyroot/cppjit/test/test_pythonization.py index 191f062a8863f..1d4c73cc906ab 100644 --- a/bindings/pyroot/cppjit/test/test_pythonization.py +++ b/bindings/pyroot/cppjit/test/test_pythonization.py @@ -275,6 +275,9 @@ def test09_cpp_side_pythonization(self): for kls in [ cppjit.gbl.pyzables.WithCallback1, cppjit.gbl.pyzables.WithCallback2, + # backwards compatibility with the old cppyy names + cppjit.gbl.pyzables.WithCppyyCallback1, + cppjit.gbl.pyzables.WithCppyyCallback2, ]: w = kls(42) assert hasattr(w, "GetInt") diff --git a/bindings/pyroot/cppjit/test/test_regression.py b/bindings/pyroot/cppjit/test/test_regression.py index cbd5f566667d4..a55de47d0ca7e 100644 --- a/bindings/pyroot/cppjit/test/test_regression.py +++ b/bindings/pyroot/cppjit/test/test_regression.py @@ -1733,3 +1733,116 @@ def test56_enum_arg_typedefed_underlying_type(self): assert ns.pick(ns.S.A) == 1 assert ns.pick(ns.S.B) == 5 + + def test57_function_pointer_types_from_ast(self): + """Function pointer return and argument types are taken from the AST""" + + import cppjit + + cppjit.cppdef(r"""\ + namespace FnPtrTypes { + unsigned int one() { return 1u; } + typedef unsigned int (*uf_t)(); + uf_t get_one() { return one; } + unsigned int call(uf_t f) { return f() + 1; } + + double scale(double x, const char* s) { return x * (s[0] - '0'); } + typedef double (*sf_t)(double, const char*); + sf_t get_scale() { return scale; } + double apply(sf_t f, double x, const char* s) { return f(x, s); } + + int noop(int i) noexcept { return i; } + typedef int (*nf_t)(int) noexcept; + nf_t get_noop() { return noop; } + + struct M { int f(double) { return 42; } }; + typedef int (M::*mf_t)(double); + mf_t get_mf() { return &M::f; } + }""") + + ns = cppjit.gbl.FnPtrTypes + + # multi-word return type (was mangled to "unsignedint" by string parsing) + assert ns.get_one()() == 1 + assert ns.call(ns.one) == 2 + assert ns.call(lambda: 5) == 6 + + # multi-word and pointer argument types + assert ns.get_scale()(2.0, "3") == 6.0 + assert ns.apply(ns.scale, 2.0, "4") == 8.0 + assert ns.apply(lambda x, s: x + 1, 2.0, "4") == 3.0 + + # noexcept function pointers + assert ns.get_noop()(7) == 7 + + # member function pointers are not function pointers; they are not + # supported, but must not crash either + ns.get_mf() + + def test58_repeated_enum_constant_template_argument(self): + """A template argument spelled as an enum constant resolves repeatedly""" + + import cppjit + + cppjit.cppdef("""\ + namespace RepeatedEnumArg { + enum E { A = 1, B = 2 }; + template struct T1 { static constexpr int v = (int)e; }; + template struct T2 { static constexpr int v = (int)e; }; + }""") + + ns = cppjit.gbl.RepeatedEnumArg + + # the resolved argument is memoized by name; its integral value is a + # string the first instantiation frees, so the second needs its own + assert ns.T1["RepeatedEnumArg::E::A"].v == 1 + assert ns.T2["RepeatedEnumArg::E::A"].v == 1 + assert ns.T2["RepeatedEnumArg::E::B"].v == 2 + + def test59_repeated_implicit_tuple_conversion(self): + """Repeated implicit tuple-to-pair conversion does not grow the overload set""" + + import cppjit + + cppjit.cppdef("""\ + #include + namespace RepeatedImplicitTuple { + int first(std::pair&& p) { return p.first; } + }""") + + ns = cppjit.gbl.RepeatedImplicitTuple + pair = cppjit.gbl.std.pair["int", "int"] + + def n_ctor_overloads(): + # the instantiated converting constructor is cached on the class + # under the constructor's own name + ol = pair.__dict__.get("pair") + return len(ol.__doc__.split("\n")) if ol is not None else 0 + + assert ns.first((1, 2)) == 1 + n = n_ctor_overloads() + assert n > 0 + + for i in range(20): + assert ns.first((i, 2)) == i + assert n_ctor_overloads() == n + + def test60_repeated_template_scope_lookup(self): + """Binding to a templated class by name does not grow the memory""" + + from pytest import importorskip + + psutil = importorskip("psutil") + + import cppjit + + v = cppjit.gbl.std.vector[int]() + addr = cppjit.addressof(v) + process = psutil.Process() + cppjit.bind_object(addr, "std::vector") + before = process.memory_info().rss + for _ in range(2000): + assert type(cppjit.bind_object(addr, "std::vector")) is type(v) + # resolving the name used to declare code in the interpreter on every + # call, growing the memory by several kB each time + assert process.memory_info().rss - before < 4 * 1024 * 1024