Refactor: use upstream udf-runner-cpp base for Python VM

.github/workflows/check_bazel_tests.yml

@@ -20,6 +20,7 @@ jobs:
       - name: Install UDF Client Dependencies
         run: |
           sudo bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE"
+
       - name: Build
         run: |
           source "$UDF_CLIENT_ENV_FILE"
@@ -32,7 +33,7 @@ jobs:
      fail-fast: false
      matrix:
         include:
-          - test: "//exaudflib/test/..."
+          - test: "@exaudfclient_base//exaudflib/test/..."
             name: "exaudflib"
     steps:
       - uses: actions/checkout@v6
@@ -52,7 +53,7 @@ jobs:
       - name: Run tests
         run: |
           bazel test ${{ matrix.additional_args }} ${{ matrix.test }}
-        working-directory: ./exaudfclient/base
+        working-directory: ./exaudfclient
         env:
           OPENSSL_LIBRARY_PREFIX: "/usr/lib/x86_64-linux-gnu"
           OPENSSL_INCLUDE_PREFIX: "/usr/include/openssl"

exaudfclient/BUILD

@@ -1,147 +1,102 @@
+package(default_visibility = ["//visibility:public"])
 
 load("//:variables.bzl", "VM_ENABLED_DEFINES")
-load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
+load("@rules_cc//cc:defs.bzl", "cc_binary")
 load("@rules_shell//shell:sh_library.bzl", "sh_library")
 
-
-VM_ENABLED_DEPS=select({
-        "@exaudfclient_base//:benchmark": ["@exaudfclient_base//benchmark_container:benchmark_container"],
-        "//conditions:default": []
-    }) + select({
-        "@exaudfclient_base//:bash": ["@exaudfclient_base//streaming_container:streamingcontainer"],
-        "//conditions:default": []
-    })
-
-VM_PYTHON3_DEPS=select({
-        "@exaudfclient_base//:python": ["@exaudfclient_base//python/python3:pythoncontainer"],
-        "//conditions:default": []
-    })
-
-
-genrule(
-    name = "exaudflib_output_path_header",
-    srcs = ["@exaudfclient_base//:libexaudflib_complete.so"],
-    outs = ["exaudf_lib_output_path.h"],
-    cmd = '''
-      echo "#define CUSTOM_LIBEXAUDFLIB_PATH \\"/exaudf/external/$(rlocationpath @exaudfclient_base//:libexaudflib_complete.so)\\"" > $@
-    ''',
-)
-
-## Here it may seem like exaudflib is linked twice; one at "deps" and another at "data"
-## "deps" is basically used by Bazel for compile and link dependencies.
-## "data" basically places the .so file into the runtime-runfiles-location
-## so that it can be loaded using apis like load_library / load_symbol later
-## therefore, it is not linked twice
-cc_library(
-    name = "exa_udfclient_binary_common_lib",
-    srcs = [
-        "exa_udfclient_binary_common/exa_lib_loader.cc",
-        "exa_udfclient_binary_common/exa_set_env.cc",
-        "exa_udfclient_binary_common/exa_udf_base.cc",
-        "exa_udfclient_binary_common/exa_udf_client.cc",
-        "exa_udfclient_binary_common/exa_vm_factory.cc",
-        "//:exaudflib_output_path_header",
-    ],
-    hdrs = [
-        "exa_udfclient_binary_common/exa_lib_loader.h",
-        "exa_udfclient_binary_common/exa_set_env.h",
-        "exa_udfclient_binary_common/exa_udf_base.h",
-        "exa_udfclient_binary_common/exa_udf_client.h",
-        "exa_udfclient_binary_common/exa_vm_factory.h",
-    ],
-    deps = [
-        "@exaudfclient_base//exaudflib:header",
-        "@exaudfclient_base//utils:utils",
-        "@exaudfclient_base//exaudflib:exaudflib-deps",
-    ] + VM_ENABLED_DEPS + VM_PYTHON3_DEPS,
-    defines = VM_ENABLED_DEFINES,
-    data = ["@exaudfclient_base//:libexaudflib_complete.so"],
+config_setting(
+    name = "python",
+    define_values = {"python": "true"},
 )
 
 cc_binary(
     name = "exaudfclient_bin",
     srcs = [
         "exa_udfclient.cc",
+        "exa_udf_client.cc",
+        "exa_udf_client.h",
+        "exa_vm_factory.cc",
+        "exa_vm_factory.h",
         "@exaudfclient_base//:load_dynamic",
     ],
     linkopts = ["-ldl"],
-    deps = [":exa_udfclient_binary_common_lib"],
-    data = ["@exaudfclient_base//:libexaudflib_complete.so"]
+    deps = [
+        "//base/python/python3:pythoncontainer",
+        "@exaudfclient_base//exa_udfclient_binary_common:exa_udfclient_binary_common_lib",
+    ],
+    defines = VM_ENABLED_DEFINES,
+    data = ["@exaudfclient_base//:libexaudflib_complete.so"],
 )
 
-## The purpose of the static binaries is to verify if the linker namespace test work correctly.
-## It is crucial that all linker symbols of exaudflib and all it's dependency are loaded into a new linker namespace,
-## so that the user can load it's own versions of those dependencies without any conflict.
-## With this binary we simulate an error in our build system, that is a direct dependency to protobuf/zmq,
-## which then must be detected with the linker namespace tests:
-## test/linker_namespace_sanity/linker_namespace_sanity.py checks the wrong configuration
-## Besides this the test under test/python3/all/linker_namespace.py checks the normal build, which expects
-## not to find any occurrence of the dependencies (protobuf/zmq) in the primary linker namespace.
-##
-## We need to explicitly declare the dependency of protobuf/zmq here, as the exaudflib is a static lib (//exaudflib:exaudflib)
-## and hence does not contain dependency information. We cannot declare the shared lib (:exaudflib_complete.so)
-## as dependency as it is a binary for bazel.
-
 cc_binary(
     name = "exaudfclient_static_bin",
     srcs = [
         "exa_udfclient.cc",
+        "exa_udf_client.cc",
+        "exa_udf_client.h",
+        "exa_vm_factory.cc",
+        "exa_vm_factory.h",
         "@exaudfclient_base//:load_dynamic",
     ],
     linkopts = ["-ldl"],
     deps = [
-        ":exa_udfclient_binary_common_lib",
-        "@zmq//:zmq",
+        "//base/python/python3:pythoncontainer",
+        "@exaudfclient_base//exa_udfclient_binary_common:exa_udfclient_binary_common_lib",
         "@protobuf//:protobuf",
+        "@zmq//:zmq",
     ],
+    defines = VM_ENABLED_DEFINES,
     data = ["@exaudfclient_base//:libexaudflib_complete.so"],
 )
 
-# Workaround for the hardcoded paths in exaudfclient for libexaudflib_complete.so and python_ext_dataframe.cc
-# - libexaudflib_complete.so and python_ext_dataframe.cc get dynamically loaded, therefore the exaudfclient needs to know their paths
-# - Most flexible way to provides these paths would environment variables
-# - The exasol database can't provide these paths, because they depend on the container
-# - A workaround to provide these paths would be wrapper bash script which set these environment variables
-# - For performance reason, we can not wrap the binary into a shell script, as such this is only for local debugging and testing
-#   and we hardcode the paths for the production container
-
 sh_library(
     name = "wrapper_generator_bin",
-    srcs=select({
+    srcs = select({
         "@exaudfclient_base//:valgrind_wrapper": ["@exaudfclient_base//:create_binary_wrapper_valgrind.sh"],
         "@exaudfclient_base//:valgrind_massif_wrapper": ["@exaudfclient_base//:create_binary_wrapper_valgrind_massif.sh"],
         "@exaudfclient_base//:stdout_to_bucketfs": ["@exaudfclient_base//:create_binary_wrapper_stdout_to_bucketfs.sh"],
-        "//conditions:default": ["//base:create_binary_wrapper.sh"]
-    })
+        "//conditions:default": ["@exaudfclient_base//:create_binary_wrapper.sh"],
+    }),
 )
 
-SLOW_WRAPPER_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_bin)" "$(location exaudfclient)" "$(location @exaudfclient_base//:exaudfclient.template.sh)" """
-FAST_BINARY="""cp "$(location exaudfclient_bin)" "$(location exaudfclient)" """
-CREATE_BINARY_SCRIPT=select({
-        "@exaudfclient_base//:fast_binary": FAST_BINARY,
-        "@exaudfclient_base//:slow_wrapper": SLOW_WRAPPER_BINARY,
-        "//conditions:default": FAST_BINARY
-    })
+SLOW_WRAPPER_BINARY = """$(location :wrapper_generator_bin) "$(location :exaudfclient_bin)" "$@" "$(location :exaudfclient.template.sh)" """
+FAST_BINARY = """cp "$(location :exaudfclient_bin)" "$@" """
+CREATE_BINARY_SCRIPT = select({
+    "@exaudfclient_base//:fast_binary": FAST_BINARY,
+    "@exaudfclient_base//:slow_wrapper": SLOW_WRAPPER_BINARY,
+    "//conditions:default": FAST_BINARY,
+})
+
 genrule(
     name = "exaudfclient",
     cmd = CREATE_BINARY_SCRIPT,
     outs = ["exaudfclient"],
-    srcs = [":exaudfclient_bin", "@exaudfclient_base//:libexaudflib_complete.so", "@exaudfclient_base//:exaudfclient.template.sh", "//:wrapper_generator_bin"],
-    output_to_bindir = True
+    srcs = [
+        ":exaudfclient_bin",
+        ":exaudfclient.template.sh",
+        ":wrapper_generator_bin",
+        "@exaudfclient_base//:libexaudflib_complete.so",
+    ],
+    output_to_bindir = True,
 )
 
-SLOW_WRAPPER_STATIC_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" "$(location @exaudfclient_base//:exaudfclient.template.sh)" """
-FAST_BINARY_STATIC="""cp "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" """
-CREATE_STATIC_BINARY_SCRIPT=select({
-        "@exaudfclient_base//:fast_binary": FAST_BINARY_STATIC,
-        "@exaudfclient_base//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY,
-        "//conditions:default": FAST_BINARY_STATIC
-    })
+SLOW_WRAPPER_STATIC_BINARY = """$(location :wrapper_generator_bin) "$(location :exaudfclient_static_bin)" "$@" "$(location :exaudfclient.template.sh)" """
+FAST_BINARY_STATIC = """cp "$(location :exaudfclient_static_bin)" "$@" """
+CREATE_STATIC_BINARY_SCRIPT = select({
+    "@exaudfclient_base//:fast_binary": FAST_BINARY_STATIC,
+    "@exaudfclient_base//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY,
+    "//conditions:default": FAST_BINARY_STATIC,
+})
 
 genrule(
     name = "exaudfclient_static",
     cmd = CREATE_STATIC_BINARY_SCRIPT,
     outs = ["exaudfclient_static"],
-    srcs = [":exaudfclient_static_bin", "@exaudfclient_base//:libexaudflib_complete.so", "@exaudfclient_base//:exaudfclient.template.sh", "//:wrapper_generator_bin"],
-    output_to_bindir = True
+    srcs = [
+        ":exaudfclient_static_bin",
+        ":exaudfclient.template.sh",
+        ":wrapper_generator_bin",
+        "@exaudfclient_base//:libexaudflib_complete.so",
+    ],
+    output_to_bindir = True,
 )

exaudfclient/MODULE.bazel

@@ -1,16 +1,21 @@
 module(name="exaudfclient", version = "1.0")
 
 bazel_dep(name = "exaudfclient_base", version = "1.0.0")
-#For now, the Bazel Modul "exaudfclient_base" is stored in subdirectory "base".
-#Later we might fetch it via Http or Git (after splitting up script-languages repository)
-local_path_override(module_name="exaudfclient_base", path="base")
+git_override(
+    module_name = "exaudfclient_base",
+    remote = "https://github.com/exasol/udf-runner-cpp.git",
+    commit = "b3a8c204a2555b373c66110d5e8301a374b32a6a",
+    strip_prefix = "udf-runner-cpp/v1/base",
+    patches = ["//:patches/udf_runner_cpp_build.patch"],
+    patch_args = ["-p0"],
+)
 bazel_dep(name = "rules_cc", version = "0.2.22")
 bazel_dep(name = "rules_shell", version = "0.8.0")
 
-numpy_local_repository = use_repo_rule("@exaudfclient_base//:python_repository.bzl", "numpy_local_repository")
+numpy_local_repository = use_repo_rule("//base:python_repository.bzl", "numpy_local_repository")
 numpy_local_repository(name = "numpy")
 
-python_local_repository = use_repo_rule("@exaudfclient_base//:python_repository.bzl", "python_local_repository")
+python_local_repository = use_repo_rule("//base:python_repository.bzl", "python_local_repository")
 python_local_repository(name = "python3", python_version="python3")
 
 zmq_local_repository = use_repo_rule("@exaudfclient_base//:zmq_repository.bzl", "zmq_local_repository")

exaudfclient/README.md

@@ -25,7 +25,7 @@ For the language support:
 
 ## Start a build
 
-The exaudfclient is a multi-language project. Therefore, we are using [Bazel](https://docs.bazel.build/versions/master/bazel-overview.html) as build system, because it provides build support for many languages and allows to mix these languages. Because the exaudfclient has language bindings to Python 2/3 and R, we need specify where Bazel can find the corresponding library and header files. This is done by environment variables.
+The exaudfclient is a Python project with a C++ runner. It uses [Bazel](https://docs.bazel.build/versions/master/bazel-overview.html) to build the local Python VM against the shared runner base from `exasol/udf-runner-cpp`. Set the Python library and header locations through environment variables.
 
 For executing the build locally, you can use the script 
 
@@ -43,9 +43,7 @@ Both build script can receive parameters, such as Bazel commandline parameter, B
 
 With Bazel defines you can specify which language support is actually compiled into you exaudfclient executbale. The currently supported defines are
 
-    --define r=true
     --define python=true
-    --define benchmark=true # This language is only for test and development purpose and benchmarks the performance of the C++ Implementation
 
 
 The main targets is //:exaudfclient.
@@ -58,15 +56,13 @@ Bazel allows to query the dependencies of a target. Furthermore, it can export t
 
 # How is the exaudfclient structured?
 
-The exaudfclient consists mainly out of three parts the main function in [exa_udfclient.cc](exa_udfclient.cc), the libexaudf and the language implementations. The first part the main function actually only loads the two other parts. However, in this case it is important how it loads the two other parts, because we need the libexaudf in a different linker namespace than the language implementation to prevent library conflicts. The libexaudf uses  [ZeroMQ](http://zeromq.org/) and  [Protobuf](https://developers.google.com/protocol-buffers/) to communicate with the Exsol Database, but UDFs could be use the same libraries in a different version which would lead to library conflict. The following figure shows the dependencies between the components.
-
-![exaudfclient dependencies](docs/exaudfclient.png)
+The local code contains the main function in [exa_udfclient.cc](exa_udfclient.cc) and the Python VM. The shared libexaudf transport, loader, ZeroMQ, and Protobuf dependencies come from the pinned `exasol/udf-runner-cpp` Bazel module and load in a separate linker namespace.
 
 The usage of multiple linker namespace requires some precautions in the build process and in the implementation. 
 
 ## Precautions in the build process
 
-In the build process you need to be cautious which libraries you link together and that no link leaks symbols from a library in one namespace to a library in the other namespace. Furthermore, you have to build a shared library with all dependency linked to it as output target. In our case, we have two main output targets: //:exaudfclient and //:libexaudflib.so. Both get loaded into different linker namespaces. The language containers live in the same namespace as //:exaudfclient. This namespace must not know anything about protobuf and zeromq, because a language container may load protobuf or zeromq in a different version. Protobuf and zeromq are only known in the namespace of //:libexaudflib.so. The target //:libexaudflib.so depends on //base/exaudflib:exaudflib which contains the logic of the exaudflib. You must not depend on //base/exaudflib:exaudflib in //:exaudfclient or a language container, because this would leak zeromq and protobuf. If you need other dependencies of //base/exaudflib:exaudflib that do not depend on protobuf or zeromq, such as //base/exaudflib:script_data_transfer_objects or //base/exaudflib:script_data_transfer_objects_wrapper, use their target directly, the collection of libraries //base/exaudflib:exaudflib-deps, or the collection of headers //base/exaudflib:header.
+Do not link the Python VM directly against the shared transport implementation: use only the `@exaudfclient_base//exaudflib:header` and `@exaudfclient_base//exaudflib:exaudflib-deps` targets. This keeps ZeroMQ and Protobuf out of the Python VM linker namespace.
 
 ## Precautions in the implementations