diff --git a/.github/workflows/check_bazel_tests.yml b/.github/workflows/check_bazel_tests.yml index 14e0131..680698b 100644 --- a/.github/workflows/check_bazel_tests.yml +++ b/.github/workflows/check_bazel_tests.yml @@ -20,45 +20,9 @@ jobs: - name: Install UDF Client Dependencies run: | sudo bash scripts/installUdfClientDeps.sh "$UDF_CLIENT_ENV_FILE" + - name: Build run: | source "$UDF_CLIENT_ENV_FILE" bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config fast-binary --verbose_failures - working-directory: ./exaudfclient/ - - tests: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - include: - - test: "//exaudflib/test/..." - name: "exaudflib" - steps: - - uses: actions/checkout@v6 - - name: Install ZMQ - run: | - sudo apt-get update - sudo apt-get install -y libzmq3-dev valgrind - - name: Install bazel - run: | - BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" - BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE" - curl -L --output /tmp/"$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL" - sudo apt install -y "/tmp/$BAZEL_PACKAGE_FILE" - rm "/tmp/$BAZEL_PACKAGE_FILE" - env: - BAZEL_PACKAGE_VERSION: "8.3.1" - - name: Run tests - run: | - bazel test ${{ matrix.additional_args }} ${{ matrix.test }} - working-directory: ./exaudfclient/base - env: - OPENSSL_LIBRARY_PREFIX: "/usr/lib/x86_64-linux-gnu" - OPENSSL_INCLUDE_PREFIX: "/usr/include/openssl" - - - uses: actions/upload-artifact@v4 - if: failure() - with: - name: "${{ matrix.name }}" - path: /home/runner/.cache/bazel/_bazel_runner/*/execroot/_main/bazel-out/k8-dbg/testlogs/**/test.log + working-directory: ./exaudfclient/ \ No newline at end of file diff --git a/exaudfclient/BUILD b/exaudfclient/BUILD index 08cfedf..dd91a04 100644 --- a/exaudfclient/BUILD +++ b/exaudfclient/BUILD @@ -1,147 +1,102 @@ +package(default_visibility = ["//visibility:public"]) load("//:variables.bzl", "VM_ENABLED_DEFINES") -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") +load("@rules_cc//cc:defs.bzl", "cc_binary") load("@rules_shell//shell:sh_library.bzl", "sh_library") - -VM_ENABLED_DEPS=select({ - "@exaudfclient_base//:benchmark": ["@exaudfclient_base//benchmark_container:benchmark_container"], - "//conditions:default": [] - }) + select({ - "@exaudfclient_base//:bash": ["@exaudfclient_base//streaming_container:streamingcontainer"], - "//conditions:default": [] - }) - -VM_PYTHON3_DEPS=select({ - "@exaudfclient_base//:python": ["@exaudfclient_base//python/python3:pythoncontainer"], - "//conditions:default": [] - }) - - -genrule( - name = "exaudflib_output_path_header", - srcs = ["@exaudfclient_base//:libexaudflib_complete.so"], - outs = ["exaudf_lib_output_path.h"], - cmd = ''' - echo "#define CUSTOM_LIBEXAUDFLIB_PATH \\"/exaudf/external/$(rlocationpath @exaudfclient_base//:libexaudflib_complete.so)\\"" > $@ - ''', -) - -## Here it may seem like exaudflib is linked twice; one at "deps" and another at "data" -## "deps" is basically used by Bazel for compile and link dependencies. -## "data" basically places the .so file into the runtime-runfiles-location -## so that it can be loaded using apis like load_library / load_symbol later -## therefore, it is not linked twice -cc_library( - name = "exa_udfclient_binary_common_lib", - srcs = [ - "exa_udfclient_binary_common/exa_lib_loader.cc", - "exa_udfclient_binary_common/exa_set_env.cc", - "exa_udfclient_binary_common/exa_udf_base.cc", - "exa_udfclient_binary_common/exa_udf_client.cc", - "exa_udfclient_binary_common/exa_vm_factory.cc", - "//:exaudflib_output_path_header", - ], - hdrs = [ - "exa_udfclient_binary_common/exa_lib_loader.h", - "exa_udfclient_binary_common/exa_set_env.h", - "exa_udfclient_binary_common/exa_udf_base.h", - "exa_udfclient_binary_common/exa_udf_client.h", - "exa_udfclient_binary_common/exa_vm_factory.h", - ], - deps = [ - "@exaudfclient_base//exaudflib:header", - "@exaudfclient_base//utils:utils", - "@exaudfclient_base//exaudflib:exaudflib-deps", - ] + VM_ENABLED_DEPS + VM_PYTHON3_DEPS, - defines = VM_ENABLED_DEFINES, - data = ["@exaudfclient_base//:libexaudflib_complete.so"], +config_setting( + name = "python", + define_values = {"python": "true"}, ) cc_binary( name = "exaudfclient_bin", srcs = [ "exa_udfclient.cc", + "exa_udf_client.cc", + "exa_udf_client.h", + "exa_vm_factory.cc", + "exa_vm_factory.h", "@exaudfclient_base//:load_dynamic", ], linkopts = ["-ldl"], - deps = [":exa_udfclient_binary_common_lib"], - data = ["@exaudfclient_base//:libexaudflib_complete.so"] + deps = [ + "//base/python/python3:pythoncontainer", + "@exaudfclient_base//exa_udfclient_binary_common:exa_udfclient_binary_common_lib", + ], + defines = VM_ENABLED_DEFINES, + data = ["@exaudfclient_base//:libexaudflib_complete.so"], ) -## The purpose of the static binaries is to verify if the linker namespace test work correctly. -## It is crucial that all linker symbols of exaudflib and all it's dependency are loaded into a new linker namespace, -## so that the user can load it's own versions of those dependencies without any conflict. -## With this binary we simulate an error in our build system, that is a direct dependency to protobuf/zmq, -## which then must be detected with the linker namespace tests: -## test/linker_namespace_sanity/linker_namespace_sanity.py checks the wrong configuration -## Besides this the test under test/python3/all/linker_namespace.py checks the normal build, which expects -## not to find any occurrence of the dependencies (protobuf/zmq) in the primary linker namespace. -## -## We need to explicitly declare the dependency of protobuf/zmq here, as the exaudflib is a static lib (//exaudflib:exaudflib) -## and hence does not contain dependency information. We cannot declare the shared lib (:exaudflib_complete.so) -## as dependency as it is a binary for bazel. - cc_binary( name = "exaudfclient_static_bin", srcs = [ "exa_udfclient.cc", + "exa_udf_client.cc", + "exa_udf_client.h", + "exa_vm_factory.cc", + "exa_vm_factory.h", "@exaudfclient_base//:load_dynamic", ], linkopts = ["-ldl"], deps = [ - ":exa_udfclient_binary_common_lib", - "@zmq//:zmq", + "//base/python/python3:pythoncontainer", + "@exaudfclient_base//exa_udfclient_binary_common:exa_udfclient_binary_common_lib", "@protobuf//:protobuf", + "@zmq//:zmq", ], + defines = VM_ENABLED_DEFINES, data = ["@exaudfclient_base//:libexaudflib_complete.so"], ) -# Workaround for the hardcoded paths in exaudfclient for libexaudflib_complete.so and python_ext_dataframe.cc -# - libexaudflib_complete.so and python_ext_dataframe.cc get dynamically loaded, therefore the exaudfclient needs to know their paths -# - Most flexible way to provides these paths would environment variables -# - The exasol database can't provide these paths, because they depend on the container -# - A workaround to provide these paths would be wrapper bash script which set these environment variables -# - For performance reason, we can not wrap the binary into a shell script, as such this is only for local debugging and testing -# and we hardcode the paths for the production container - sh_library( name = "wrapper_generator_bin", - srcs=select({ + srcs = select({ "@exaudfclient_base//:valgrind_wrapper": ["@exaudfclient_base//:create_binary_wrapper_valgrind.sh"], "@exaudfclient_base//:valgrind_massif_wrapper": ["@exaudfclient_base//:create_binary_wrapper_valgrind_massif.sh"], "@exaudfclient_base//:stdout_to_bucketfs": ["@exaudfclient_base//:create_binary_wrapper_stdout_to_bucketfs.sh"], - "//conditions:default": ["//base:create_binary_wrapper.sh"] - }) + "//conditions:default": ["@exaudfclient_base//:create_binary_wrapper.sh"], + }), ) -SLOW_WRAPPER_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_bin)" "$(location exaudfclient)" "$(location @exaudfclient_base//:exaudfclient.template.sh)" """ -FAST_BINARY="""cp "$(location exaudfclient_bin)" "$(location exaudfclient)" """ -CREATE_BINARY_SCRIPT=select({ - "@exaudfclient_base//:fast_binary": FAST_BINARY, - "@exaudfclient_base//:slow_wrapper": SLOW_WRAPPER_BINARY, - "//conditions:default": FAST_BINARY - }) +SLOW_WRAPPER_BINARY = """$(location :wrapper_generator_bin) "$(location :exaudfclient_bin)" "$@" "$(location :exaudfclient.template.sh)" """ +FAST_BINARY = """cp "$(location :exaudfclient_bin)" "$@" """ +CREATE_BINARY_SCRIPT = select({ + "@exaudfclient_base//:fast_binary": FAST_BINARY, + "@exaudfclient_base//:slow_wrapper": SLOW_WRAPPER_BINARY, + "//conditions:default": FAST_BINARY, +}) + genrule( name = "exaudfclient", cmd = CREATE_BINARY_SCRIPT, outs = ["exaudfclient"], - srcs = [":exaudfclient_bin", "@exaudfclient_base//:libexaudflib_complete.so", "@exaudfclient_base//:exaudfclient.template.sh", "//:wrapper_generator_bin"], - output_to_bindir = True + srcs = [ + ":exaudfclient_bin", + ":exaudfclient.template.sh", + ":wrapper_generator_bin", + "@exaudfclient_base//:libexaudflib_complete.so", + ], + output_to_bindir = True, ) -SLOW_WRAPPER_STATIC_BINARY="""$(location //:wrapper_generator_bin) "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" "$(location @exaudfclient_base//:exaudfclient.template.sh)" """ -FAST_BINARY_STATIC="""cp "$(location exaudfclient_static_bin)" "$(location exaudfclient_static)" """ -CREATE_STATIC_BINARY_SCRIPT=select({ - "@exaudfclient_base//:fast_binary": FAST_BINARY_STATIC, - "@exaudfclient_base//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY, - "//conditions:default": FAST_BINARY_STATIC - }) +SLOW_WRAPPER_STATIC_BINARY = """$(location :wrapper_generator_bin) "$(location :exaudfclient_static_bin)" "$@" "$(location :exaudfclient.template.sh)" """ +FAST_BINARY_STATIC = """cp "$(location :exaudfclient_static_bin)" "$@" """ +CREATE_STATIC_BINARY_SCRIPT = select({ + "@exaudfclient_base//:fast_binary": FAST_BINARY_STATIC, + "@exaudfclient_base//:slow_wrapper": SLOW_WRAPPER_STATIC_BINARY, + "//conditions:default": FAST_BINARY_STATIC, +}) genrule( name = "exaudfclient_static", cmd = CREATE_STATIC_BINARY_SCRIPT, outs = ["exaudfclient_static"], - srcs = [":exaudfclient_static_bin", "@exaudfclient_base//:libexaudflib_complete.so", "@exaudfclient_base//:exaudfclient.template.sh", "//:wrapper_generator_bin"], - output_to_bindir = True + srcs = [ + ":exaudfclient_static_bin", + ":exaudfclient.template.sh", + ":wrapper_generator_bin", + "@exaudfclient_base//:libexaudflib_complete.so", + ], + output_to_bindir = True, ) diff --git a/exaudfclient/MODULE.bazel b/exaudfclient/MODULE.bazel index b445d66..84eadd6 100644 --- a/exaudfclient/MODULE.bazel +++ b/exaudfclient/MODULE.bazel @@ -1,16 +1,19 @@ module(name="exaudfclient", version = "1.0") bazel_dep(name = "exaudfclient_base", version = "1.0.0") -#For now, the Bazel Modul "exaudfclient_base" is stored in subdirectory "base". -#Later we might fetch it via Http or Git (after splitting up script-languages repository) -local_path_override(module_name="exaudfclient_base", path="base") +git_override( + module_name = "exaudfclient_base", + remote = "https://github.com/exasol/udf-runner-cpp.git", + commit = "fe4ca609677c556fafb99b2be0501203843d850f", + strip_prefix = "udf-runner-cpp/v1/base", +) bazel_dep(name = "rules_cc", version = "0.2.22") bazel_dep(name = "rules_shell", version = "0.8.0") -numpy_local_repository = use_repo_rule("@exaudfclient_base//:python_repository.bzl", "numpy_local_repository") +numpy_local_repository = use_repo_rule("//base:python_repository.bzl", "numpy_local_repository") numpy_local_repository(name = "numpy") -python_local_repository = use_repo_rule("@exaudfclient_base//:python_repository.bzl", "python_local_repository") +python_local_repository = use_repo_rule("//base:python_repository.bzl", "python_local_repository") python_local_repository(name = "python3", python_version="python3") zmq_local_repository = use_repo_rule("@exaudfclient_base//:zmq_repository.bzl", "zmq_local_repository") diff --git a/exaudfclient/README.md b/exaudfclient/README.md index b95fb58..986c4a7 100644 --- a/exaudfclient/README.md +++ b/exaudfclient/README.md @@ -25,7 +25,7 @@ For the language support: ## Start a build -The exaudfclient is a multi-language project. Therefore, we are using [Bazel](https://docs.bazel.build/versions/master/bazel-overview.html) as build system, because it provides build support for many languages and allows to mix these languages. Because the exaudfclient has language bindings to Python 2/3 and R, we need specify where Bazel can find the corresponding library and header files. This is done by environment variables. +The exaudfclient is a Python project with a C++ runner. It uses [Bazel](https://docs.bazel.build/versions/master/bazel-overview.html) to build the local Python VM against the shared runner base from `exasol/udf-runner-cpp`. Set the Python library and header locations through environment variables. For executing the build locally, you can use the script @@ -43,9 +43,7 @@ Both build script can receive parameters, such as Bazel commandline parameter, B With Bazel defines you can specify which language support is actually compiled into you exaudfclient executbale. The currently supported defines are - --define r=true --define python=true - --define benchmark=true # This language is only for test and development purpose and benchmarks the performance of the C++ Implementation The main targets is //:exaudfclient. @@ -58,15 +56,13 @@ Bazel allows to query the dependencies of a target. Furthermore, it can export t # How is the exaudfclient structured? -The exaudfclient consists mainly out of three parts the main function in [exa_udfclient.cc](exa_udfclient.cc), the libexaudf and the language implementations. The first part the main function actually only loads the two other parts. However, in this case it is important how it loads the two other parts, because we need the libexaudf in a different linker namespace than the language implementation to prevent library conflicts. The libexaudf uses [ZeroMQ](http://zeromq.org/) and [Protobuf](https://developers.google.com/protocol-buffers/) to communicate with the Exsol Database, but UDFs could be use the same libraries in a different version which would lead to library conflict. The following figure shows the dependencies between the components. - -![exaudfclient dependencies](docs/exaudfclient.png) +The local code contains the main function in [exa_udfclient.cc](exa_udfclient.cc) and the Python VM. The shared libexaudf transport, loader, ZeroMQ, and Protobuf dependencies come from the pinned `exasol/udf-runner-cpp` Bazel module and load in a separate linker namespace. The usage of multiple linker namespace requires some precautions in the build process and in the implementation. ## Precautions in the build process -In the build process you need to be cautious which libraries you link together and that no link leaks symbols from a library in one namespace to a library in the other namespace. Furthermore, you have to build a shared library with all dependency linked to it as output target. In our case, we have two main output targets: //:exaudfclient and //:libexaudflib.so. Both get loaded into different linker namespaces. The language containers live in the same namespace as //:exaudfclient. This namespace must not know anything about protobuf and zeromq, because a language container may load protobuf or zeromq in a different version. Protobuf and zeromq are only known in the namespace of //:libexaudflib.so. The target //:libexaudflib.so depends on //base/exaudflib:exaudflib which contains the logic of the exaudflib. You must not depend on //base/exaudflib:exaudflib in //:exaudfclient or a language container, because this would leak zeromq and protobuf. If you need other dependencies of //base/exaudflib:exaudflib that do not depend on protobuf or zeromq, such as //base/exaudflib:script_data_transfer_objects or //base/exaudflib:script_data_transfer_objects_wrapper, use their target directly, the collection of libraries //base/exaudflib:exaudflib-deps, or the collection of headers //base/exaudflib:header. +Do not link the Python VM directly against the shared transport implementation: use only the `@exaudfclient_base//exaudflib:header` and `@exaudfclient_base//exaudflib:exaudflib-deps` targets. This keeps ZeroMQ and Protobuf out of the Python VM linker namespace. ## Precautions in the implementations diff --git a/exaudfclient/base/.bazelrc b/exaudfclient/base/.bazelrc deleted file mode 100644 index c6d0048..0000000 --- a/exaudfclient/base/.bazelrc +++ /dev/null @@ -1,18 +0,0 @@ -build --lockfile_mode=off --copt='-std=c++17' --force_pic --action_env=PROTOBUF_BIN --action_env=PROTOBUF_LIBRARY_PREFIX --action_env=PROTOBUF_INCLUDE_PREFIX -build:benchmark --define benchmark=true -build:python --define python=true --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION -build:debug-build --sandbox_debug --config=verbose -build:no-symlinks --symlink_prefix=/ -build:asan --strip=never -build:asan --copt -fsanitize=address -build:asan --copt -DADDRESS_SANITIZER -build:asan --copt -O1 -build:asan --copt -g -build:asan --copt -fno-omit-frame-pointer -build:asan --linkopt -fsanitize=address -build:asan -c dbg -build:valgrind --copt -O1 -build:valgrind -c dbg -build:valgrind --copt -g -build:valgrind --strip=never -build:valgrind --copt -DVALGRIND_ACTIVE diff --git a/exaudfclient/base/BUILD b/exaudfclient/base/BUILD index 5f857bf..fbceeaf 100644 --- a/exaudfclient/base/BUILD +++ b/exaudfclient/base/BUILD @@ -1,90 +1,3 @@ - package(default_visibility = ["//visibility:public"]) -exports_files(["build_integrated.py", - "create_binary_wrapper.sh", "create_binary_wrapper_valgrind.sh", - "create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh", - "replace_swig_import_helper.py"]) - -load("//:variables.bzl", "VM_ENABLED_DEFINES") -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") - -config_setting( - name = "benchmark", - define_values = {"benchmark": "true"}, -) - -config_setting( - name = "python", - define_values = {"python": "true"}, -) - -config_setting( - name = "bash", - define_values = {"bash": "true"}, -) - -config_setting( - name = "fast_binary", - define_values = { - "binary_type": "fast_binary", - }, -) - -config_setting( - name = "slow_wrapper", - define_values = { - "binary_type": "slow_wrapper", - }, -) - -config_setting( - name = "valgrind_wrapper", - define_values = { - "wrapper_type": "valgrind_wrapper", - }, -) - -config_setting( - name = "valgrind_massif_wrapper", - define_values = { - "wrapper_type": "valgrind_massif_wrapper", - }, -) - -config_setting( - name = "stdout_to_bucketfs", - define_values = { - "wrapper_type": "stdout_to_bucketfs", - }, -) - -config_setting( - name = "plugin_client", - define_values = {"plugin_client": "true"}, -) - -cc_library( - name = "load_dynamic", - srcs = [ - "load_dynamic.cc" - ], - deps = ["//exaudflib:header", "//utils:utils", "//exaudflib:exaudflib-deps"], - defines = VM_ENABLED_DEFINES, -) - -# Using cc_binary to build a shared library as output target is a workaround, because -# Bazel's cc_library are only intermediate stages and can't be used as output target. -# It is necessary to include //exaudflib:exaudflib into deps and srcs, because -# otherwise dlmopen won't find the symbols of exaudflib and its dependencies. The target //exaudflib:exaudflib-deps -# won't work either, because it only contains the libraries as dependencies. -# Bazel builts a static library (.a) and a dynamic library (.so) out of exaudflib:exaudflib. -# In the libexaudflib_complete.so-2.params, we saw that Bazel uses both libraries to built libexaudflib_complete.so. -# Experiments have shown that we get missing symbols during loading libexaudflib_complete.so, if we only built the static library. -cc_binary( - name = "libexaudflib_complete.so", - linkshared = 1, - srcs = ["//exaudflib:exaudflib"], - deps = ["//exaudflib:exaudflib"], - defines = VM_ENABLED_DEFINES, -) +exports_files(["python_repository.bzl"]) diff --git a/exaudfclient/base/MODULE.bazel b/exaudfclient/base/MODULE.bazel deleted file mode 100644 index 8cb18b7..0000000 --- a/exaudfclient/base/MODULE.bazel +++ /dev/null @@ -1,27 +0,0 @@ -module(name="exaudfclient_base", version = "1.0") - -bazel_dep(name = "bazel_skylib", version = "1.7.1") -bazel_dep(name = "googletest", version = "1.15.0") -bazel_dep(name = "rules_cc", version = "0.2.22") - -# Since Bazel >8 the Python hermetic build rules must not run as root -# However, since the UDF client is built as root in a docker container -# we must disable this behavior. The only way is to configure the Python hermetic toolchain -bazel_dep(name = "rules_python", version = "1.7.0") - -python = use_extension("@rules_python//python/extensions:python.bzl", "python") -python.defaults(python_version = "3.11") -python.toolchain(python_version = "3.11", ignore_root_user_error = True,) - - -numpy_local_repository = use_repo_rule("@exaudfclient_base//:python_repository.bzl", "numpy_local_repository") -numpy_local_repository(name = "numpy") - -python_local_repository = use_repo_rule("@exaudfclient_base//:python_repository.bzl", "python_local_repository") -python_local_repository(name = "python3", python_version="python3") - -zmq_local_repository = use_repo_rule("@exaudfclient_base//:zmq_repository.bzl", "zmq_local_repository") -zmq_local_repository(name = "zmq") - -protobuf_local_repository = use_repo_rule("@exaudfclient_base//:protobuf_repository.bzl", "protobuf_local_repository") -protobuf_local_repository(name = "protobuf") diff --git a/exaudfclient/base/benchmark_container/BUILD b/exaudfclient/base/benchmark_container/BUILD deleted file mode 100644 index c591e6a..0000000 --- a/exaudfclient/base/benchmark_container/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load("@rules_cc//cc:defs.bzl", "cc_library") - -cc_library( - name = "benchmark_container", - srcs = ["benchmark_container.cc","benchmark_container.h"], - hdrs = ["benchmark_container.h"], - deps = ["//exaudflib:exaudflib-deps","//exaudflib:header", "//utils:utils"] -) diff --git a/exaudfclient/base/benchmark_container/benchmark_container.cc b/exaudfclient/base/benchmark_container/benchmark_container.cc deleted file mode 100644 index 92de436..0000000 --- a/exaudfclient/base/benchmark_container/benchmark_container.cc +++ /dev/null @@ -1,135 +0,0 @@ -#ifndef ENABLE_BENCHMARK_VM -#define ENABLE_BENCHMARK_VM -#endif - -#include "utils/debug_message.h" -#include "benchmark_container.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace SWIGVMContainers; -using namespace std; - -BenchmarkVM::BenchmarkVM(bool checkOnly) - : meta(), inp(), outp(&inp) - -{ - if (meta.inputType() != MULTIPLE - || meta.outputType() != MULTIPLE) - { - throw SWIGVM::exception("BENCHMARK language container only support SET-EMITS UDFs"); - } -} - -void BenchmarkVM::shutdown() -{ -} - -bool BenchmarkVM::run() -{ - DBG_FUNC_BEGIN(cerr); - cerr << "Input Columns:" << endl; - for (unsigned int col = 0; colempty()){ - count++; - } - } - break; - default: - break; - } - } - hasNext = inp.next(); - } while (hasNext); - cerr << "End fetch" << endl; - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - exception_msg = err.what(); - cerr << "Exception BenchmarkVM::run " << exception_msg << endl; - } - - try { - cerr << "Begin emit" << endl; - string str1 = string("test1"); - outp.setString(0,str1.c_str(),strlen(str1.c_str())); - outp.next(); - string str2 = string("test2"); - outp.setString(0,str2.c_str(),strlen(str2.c_str())); - outp.next(); - outp.flush(); - cerr << "End emit" << endl; - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - exception_msg = err.what(); - cerr << "Exception BenchmarkVM::run " << exception_msg << endl; - } - - if (exception_msg.size() > 0) { - throw SWIGVM::exception(exception_msg.c_str()); - } - - } catch (std::exception& ex) { - lock_guard lock(exception_msg_mtx); - exception_msg = ex.what(); - cerr << "Exception BenchmarkVM::run " << exception_msg << endl; - } - DBG_FUNC_END(cerr); - return true; // done -} - -const char* BenchmarkVM::singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args) -{ - throw SWIGVM::exception("singleCall not supported for the STREAMING language container"); -} diff --git a/exaudfclient/base/benchmark_container/benchmark_container.h b/exaudfclient/base/benchmark_container/benchmark_container.h deleted file mode 100644 index e107f4b..0000000 --- a/exaudfclient/base/benchmark_container/benchmark_container.h +++ /dev/null @@ -1,27 +0,0 @@ -#include "exaudflib/vm/swig_vm.h" -#include "exaudflib/swig/swig_meta_data.h" -#include "exaudflib/swig/swig_result_handler.h" -#include "exaudflib/swig/swig_table_iterator.h" - -using namespace SWIGVMContainers; -using namespace std; - - -class BenchmarkVM: public SWIGVM { - public: - struct exception: SWIGVM::exception { - exception(const char *reason): SWIGVM::exception(reason) { } - virtual ~exception() throw() { } - }; - BenchmarkVM(bool checkOnly); - virtual ~BenchmarkVM() {}; - virtual void shutdown(); - virtual bool run(); - virtual const char* singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args); - virtual bool useZmqSocketLocks() {return true;} - private: - SWIGMetadata meta; - SWIGTableIterator inp; - SWIGResultHandler outp; - void importScripts(); -}; \ No newline at end of file diff --git a/exaudfclient/base/build_integrated.py b/exaudfclient/base/build_integrated.py deleted file mode 100644 index c43f0bc..0000000 --- a/exaudfclient/base/build_integrated.py +++ /dev/null @@ -1,35 +0,0 @@ -import sys - - -def build_integrated(target, source): - assert(len(target) == 1 and len(source) > 0) - builddir = '.' - RE='/usr' - - output = [] - for fname in source: - fname_short = str(fname) - if fname_short.startswith(builddir): - fname_short = fname_short[len(builddir):] - fvar = 'integrated_' + fname_short.lower().replace('.', '_').replace('/', '_') - flines = [] - with open(str(fname)) as fname_file: - for line in fname_file: - line = line.replace(", PACKAGE='exascript_r'", '') - line = line.replace('\\', '\\\\') - line = line.replace(r'"', r'\"') - line = line.replace('\r', r'\r') - line = line.replace('\n', r'\n') - line = line.replace('RUNTIME_PATH', RE) - flines.append(line) - output.append('static const char *' + fvar + ' = "' + ''.join(flines) + '";\n') - with open(str(target[0]), 'w') as fd: - fd.write(''.join(output)) - return 0 - - -if __name__ == "__main__": - if len(sys.argv) < 3: - print('Usage: build_integrated.py target source_1 source_2 ...') - sys.exit(1) - build_integrated(sys.argv[1:2], sys.argv[2:]) diff --git a/exaudfclient/base/create_binary_wrapper.sh b/exaudfclient/base/create_binary_wrapper.sh deleted file mode 100755 index c00a66e..0000000 --- a/exaudfclient/base/create_binary_wrapper.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -o errexit -set -o pipefail -CLIENT_BINARY=$1 -CLIENT_WRAPPER=$2 -WRAPPER_TEMPLATE=$3 -touch "$CLIENT_WRAPPER" - -{ -cat "$WRAPPER_TEMPLATE" -echo -echo ./"$(basename "$CLIENT_BINARY")" '$*' -} >> "$CLIENT_WRAPPER" \ No newline at end of file diff --git a/exaudfclient/base/create_binary_wrapper_stdout_to_bucketfs.sh b/exaudfclient/base/create_binary_wrapper_stdout_to_bucketfs.sh deleted file mode 100755 index 4db4327..0000000 --- a/exaudfclient/base/create_binary_wrapper_stdout_to_bucketfs.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2016 - -set -o errexit -set -o pipefail -CLIENT_BINARY=$1 -CLIENT_WRAPPER=$2 -WRAPPER_TEMPLATE=$3 -touch "$CLIENT_WRAPPER" - -{ -cat "$WRAPPER_TEMPLATE" -echo -echo 'NAME="$1"' -#Ignore shellcheck rule, change is too risky. -#shellcheck disable=SC2001 -echo 'OUTPUT_FILE="/tmp/$(echo "$NAME" | sed s#[/:]##g)"' -echo "./$(basename "$CLIENT_BINARY")" '"$@"' '&> "$OUTPUT_FILE"' -echo 'UPLOAD="curl --fail -X PUT -T $OUTPUT_FILE http://w:write@localhost:6583/default$OUTPUT_FILE"' -echo 'echo "$UPLOAD"' -echo '"$UPLOAD"' - -} >> "$CLIENT_WRAPPER" \ No newline at end of file diff --git a/exaudfclient/base/create_binary_wrapper_valgrind.sh b/exaudfclient/base/create_binary_wrapper_valgrind.sh deleted file mode 100755 index 5fb10a1..0000000 --- a/exaudfclient/base/create_binary_wrapper_valgrind.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -o errexit -set -o pipefail -CLIENT_BINARY=$1 -CLIENT_WRAPPER=$2 -WRAPPER_TEMPLATE=$3 -touch "$CLIENT_WRAPPER" - -{ -cat "$WRAPPER_TEMPLATE" -echo -echo valgrind "./$(basename "$CLIENT_BINARY")" '$*' -} >> "$CLIENT_WRAPPER" \ No newline at end of file diff --git a/exaudfclient/base/create_binary_wrapper_valgrind_massif.sh b/exaudfclient/base/create_binary_wrapper_valgrind_massif.sh deleted file mode 100755 index 3446605..0000000 --- a/exaudfclient/base/create_binary_wrapper_valgrind_massif.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2016 - -set -o errexit -set -o pipefail -CLIENT_BINARY=$1 -CLIENT_WRAPPER=$2 -WRAPPER_TEMPLATE=$3 -touch "$CLIENT_WRAPPER" -{ -cat "$WRAPPER_TEMPLATE" -echo -echo 'NAME="$1"' -#Ignore shellcheck rule, change is too risky. -#shellcheck disable=SC2001 -echo 'MASSIF_FILE="/tmp/$(echo "$NAME" | sed s#[/:]##g)"' -echo valgrind --tool=massif --massif-out-file='"$MASSIF_FILE"' "./$(basename "$CLIENT_BINARY")" '"$@"' -echo 'UPLOAD="curl --fail -X PUT -T $MASSIF_FILE http://w:write@localhost:6583/default$MASSIF_FILE"' -echo 'echo "$UPLOAD"' -echo '"$UPLOAD"' -} >> "$CLIENT_WRAPPER" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/BUILD b/exaudfclient/base/exaudflib/BUILD deleted file mode 100644 index 9e6274d..0000000 --- a/exaudfclient/base/exaudflib/BUILD +++ /dev/null @@ -1,99 +0,0 @@ -package(default_visibility = ["//visibility:public"]) -exports_files(["exascript.i"]) - -load("//:variables.bzl", "VM_ENABLED_DEFINES") -load("@rules_cc//cc:defs.bzl", "cc_library") - -cc_library( - name = "script_data_transfer_objects", - srcs = ["swig/script_data_transfer_objects.cc","swig/script_data_transfer_objects.h"], - hdrs = ["swig/script_data_transfer_objects.h"], - copts= select({ - "//:benchmark": ["-fno-lto"], - "//conditions:default": [] - }), - # We deactivate link time optimization for benchmark, - # because otherwise we get linker/compiler error while linking/compiling exaudfclient - # Benchmark causes this, because it doesn't refer it its implementation. -) - -cc_library( - name = "script_data_transfer_objects_wrapper", - srcs = ["swig/script_data_transfer_objects_wrapper.cc","swig/script_data_transfer_objects_wrapper.h"], - deps = [":script_data_transfer_objects"], - hdrs = ["swig/script_data_transfer_objects_wrapper.h"], - copts= select({ - "//:benchmark": ["-fno-lto"], - "//conditions:default": [] - }), - # We deactivate link time optimization for benchmark, - # because otherwise we get linker/compiler error while linking/compiling exaudfclient - # Benchmark causes this, because it doesn't refer it its implementation. -) - -# A collection target for the case that language container want to include all public headers of exaudflib -cc_library( - name = "header", - hdrs = [ - "udf_plugin_interface.h", "load_dynamic.h", "vm/swig_vm.h", - "swig/swig_common.h", "swig/swig_meta_data.h", "swig/swig_result_handler.h", "swig/swig_table_iterator.h" - ], -) - -# A collection target for the case that language container want to depent on all public libraries of exaudflib -# which won't leak protobuf through linker namespaces. Caution: Can't be used for the target libexaudflib-complete.so, -# because this target declares the libraries only as deps. -cc_library( #TODO might be not working as expected - name = "exaudflib-deps", - deps = [":script_data_transfer_objects",":script_data_transfer_objects_wrapper"], -) - -genrule( - name = "zmqcontainer_proto", - cmd = """ - TMPDIR=`mktemp -d` - cp $< "$$TMPDIR" - (cd "$$TMPDIR" - $$PROTOBUF_BIN -I. zmqcontainer.proto --cpp_out=.) - cp "$$TMPDIR"/zmqcontainer.pb.cc $(location zmqcontainer.pb.cc) - cp "$$TMPDIR"/zmqcontainer.pb.h $(location zmqcontainer.pb.h)""", - outs = ["zmqcontainer.pb.cc","zmqcontainer.pb.h"], - srcs = ["zmqcontainer.proto"] -) - -cc_library( - name = "zmqcontainer", - hdrs = [":zmqcontainer_proto"], - srcs = [":zmqcontainer_proto"], - deps = ["@protobuf//:protobuf"], - copts = ["-DNDEBUG=1"] #We use the protobuf shared library compiled with NDEBUG=1 (the conda package is the release version). We need to set NDEBUG=1, otherwise some functions are not visisble for the dependent libraries (compiled inline for NDEBUG=1, compiled as global function otherwise). - -) - -# Do not depend on this library in the langauge container or exaudfclient. -# The only target which must depend on it is //:libexaudflib-complete.so. -# If you depend on this target you will load protobuf into your linker namespace -# which might cause problems with tensorflow. If you need -# "//exaudflib:script_data_transfer_objects" or -# "//exaudflib:script_data_transfer_objects_wrapper" in the language container -# depend directly on them or depend on exaudflib-deps -cc_library( - name = "exaudflib", - srcs = ["vm/swig_vm.cc", - "swig/swig_common.cc", "swig/swig_meta_data.cc", "swig/swig_result_handler.cc", "swig/swig_table_iterator.cc", - "impl/exaudflib_main.cc", - "impl/msg_conversion.cc", "impl/msg_conversion.h", - "impl/check.cc", "impl/check.h", - "impl/socket_info.cc", "impl/socket_info.h", - "impl/socket_low_level.cc", "impl/socket_low_level.h", - "impl/global.cc", "impl/global.h", - "impl/socket_high_level.cc", "impl/socket_high_level.h", - "impl/swig/swig_factory.cc", - "impl/swig/swig_table_iterator.h", "impl/swig/swig_table_iterator.cc", - "impl/swig/swig_meta_data.h", "impl/swig/swig_meta_data.cc", - "impl/swig/swig_result_handler.h", "impl/swig/swig_result_handler.cc", - "impl/swig/swig_general_iterator.h", "impl/swig/swig_general_iterator.cc"], - defines = VM_ENABLED_DEFINES, - linkstatic = False, # Needs to be false, because otherwise we might get missing symbols when loading exaudflib_complete.so - deps = [":exaudflib-deps", ":zmqcontainer", "@zmq//:zmq", ":header", "//utils:utils"], -) diff --git a/exaudfclient/base/exaudflib/exascript.i b/exaudfclient/base/exaudflib/exascript.i deleted file mode 100644 index bc6abf9..0000000 --- a/exaudfclient/base/exaudflib/exascript.i +++ /dev/null @@ -1,136 +0,0 @@ -%begin %{ -#define SWIG_PYTHON_STRICT_BYTE_CHAR -%} - -%{ -#include "exaudflib/swig/swig_result_handler.h" -#include "exaudflib/swig/swig_meta_data.h" - -using namespace SWIGVMContainers; -#include "exaudflib/swig/script_data_transfer_objects_wrapper.h" -%} - -%ignore ExecutionGraph::ConnectionInformationWrapper::ConnectionInformationWrapper; -%include "exaudflib/script_data_transfer_objects_wrapper.h" -%newobject SWIGMetadata::connectionInformation(char*); -%newobject ConnectionInformationWrapper::copyKind(); -%newobject ConnectionInformationWrapper::copyAddress(); -%newobject ConnectionInformationWrapper::copyUser(); -%newobject ConnectionInformationWrapper::copyPassword(); - -%newobject ImportSpecificationWrapper::copySubselectColumnName(); -%newobject ImportSpecificationWrapper::copySubselectColumnType(); -%newobject ImportSpecificationWrapper::copyConnectionName(); -%newobject ImportSpecificationWrapper::copyKey(); -%newobject ImportSpecificationWrapper::copyValue(); - -%newobject ExportSpecificationWrapper::copyConnectionName(); -%newobject ExportSpecificationWrapper::copyKey(); -%newobject ExportSpecificationWrapper::copyValue(); -%newobject ExportSpecificationWrapper::copyCreatedBy(); -%newobject ExportSpecificationWrapper::copySourceColumnName(); - -%rename(Metadata) SWIGMetadata; -%rename(TableIterator) SWIGTableIterator; -%rename(ResultHandler) SWIGResultHandler; - -enum SWIGVM_datatype_e { - UNSUPPORTED = 0, - DOUBLE = 1, - INT32 = 2, - INT64 = 3, - NUMERIC = 4, - TIMESTAMP = 5, - DATE = 6, - STRING = 7, - BOOLEAN = 8 }; - -enum SWIGVM_itertype_e { - EXACTLY_ONCE = 1, - MULTIPLE = 2 -}; - -class SWIGMetadata { - public: - SWIGMetadata(); - - inline const char* databaseName(); - inline const char* databaseVersion(); - inline const char* scriptName(); - inline const char* scriptSchema(); - inline const char* currentUser(); - inline const char* scopeUser(); - inline const char* currentSchema(); - inline const char* scriptCode(); - inline const char* moduleContent(const char* name); - inline const ExecutionGraph::ConnectionInformationWrapper* connectionInformation(const char* connection_name); - inline const unsigned long long sessionID(); - inline const const char *sessionID_S(); - inline const unsigned long statementID(); - inline const unsigned int nodeCount(); - inline const unsigned int nodeID(); - inline const unsigned long long vmID(); - inline const const char *vmID_S(); - inline const unsigned long long memoryLimit(); - - inline unsigned int inputColumnCount(); - inline const char *inputColumnName(unsigned int col); - inline const SWIGVM_datatype_e inputColumnType(unsigned int col); - inline const char *inputColumnTypeName(unsigned int col); - inline const unsigned int inputColumnSize(unsigned int col); - inline const unsigned int inputColumnPrecision(unsigned int col); - inline const unsigned int inputColumnScale(unsigned int col); - inline const SWIGVM_itertype_e inputType(); - - inline unsigned int outputColumnCount(); - inline const char *outputColumnName(unsigned int col); - inline const SWIGVM_datatype_e outputColumnType(unsigned int col); - inline const char *outputColumnTypeName(unsigned int col); - inline const unsigned int outputColumnSize(unsigned int col); - inline const unsigned int outputColumnPrecision(unsigned int col); - inline const unsigned int outputColumnScale(unsigned int col); - inline const SWIGVM_itertype_e outputType(); - - inline const char* checkException(); - inline const char* pluginLanguageName(); -}; - -class SWIGTableIterator { - public: - SWIGTableIterator(); - inline const char* checkException(); - inline void reinitialize(); - inline bool next(); - inline bool eot(); - inline void reset(); - inline unsigned long restBufferSize(); - inline unsigned long rowsInGroup(); - inline unsigned long rowsCompleted(); - inline double getDouble(unsigned int col); - inline const char *getString(unsigned int col); - inline int getInt32(unsigned int col); - inline long long getInt64(unsigned int col); - inline const char *getNumeric(unsigned int col); - inline const char *getDate(unsigned int col); - inline const char *getTimestamp(unsigned int col); - inline bool getBoolean(unsigned int col); - inline bool wasNull(); -}; - -class SWIGResultHandler { - public: - SWIGResultHandler(SWIGTableIterator* it); - inline const char* checkException(); - inline void reinitialize(); - inline bool next(); - inline void flush(); - inline void setDouble(unsigned int col, double v); - inline void setString(unsigned int col, char *v, size_t l); - inline void setInt32(unsigned int col, int v); - inline void setInt64(unsigned int col, long long v); - inline void setNumeric(unsigned int col, char *v); - inline void setDate(unsigned int col, char *v); - inline void setTimestamp(unsigned int col, char *v); - inline void setBoolean(unsigned int col, bool v); - inline void setNull(unsigned int col); -}; diff --git a/exaudfclient/base/exaudflib/impl/check.cc b/exaudfclient/base/exaudflib/impl/check.cc deleted file mode 100644 index 14029d7..0000000 --- a/exaudfclient/base/exaudflib/impl/check.cc +++ /dev/null @@ -1,79 +0,0 @@ -#include "exaudflib/impl/check.h" -#include "exaudflib/impl/socket_info.h" -#include -#include -#include -#include - - -#include "utils/debug_message.h" - -namespace exaudflib { - namespace check { - static std::atomic_bool keep_checking(true); - static pthread_t check_thread; - static int first_ppid=-1; - static bool remote_client; - } -} - -void exaudflib::check::set_remote_client(bool value) { - remote_client = value; -} - -bool exaudflib::check::get_remote_client() { - return remote_client; -} - -void exaudflib::check::external_process_check() { - if (remote_client) return; - if (::access(socket_info::get_socket_file_name(), F_OK) != 0) { - ::sleep(1); // give me a chance to die with my parent process - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1000","exaudfclient aborting ... cannot access socket file " << socket_info::get_socket_url()+6 << "."); - DBG_STREAM_MSG(std::cerr,"### SWIGVM aborting with name '" << socket_info::get_socket_url() << "' (" << ::getppid() << ',' << ::getpid() << ')'); - ::abort(); - } -} - - -static void check_parent_pid(){ - int new_ppid=::getppid(); - if(exaudflib::check::first_ppid==-1){ // Initialize first_ppid - exaudflib::check::first_ppid=new_ppid; - } - // Check if ppid has changed, if client is in own namespace, - // the ppid will be forever 0 and never change. - // If the client runs as udfplugin the ppid will point to the exasql process - // and will change if it gets killed. Then client gets an orphaned process and - // will be adopted by another process - if(exaudflib::check::first_ppid!=new_ppid){ - ::sleep(1); // give me a chance to die with my parent process - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1001","exaudfclient aborting " << exaudflib::socket_info::get_socket_url() << " ... current parent pid " << new_ppid << " different to first parent pid " << exaudflib::check::first_ppid << "." ); - DBG_STREAM_MSG(std::cerr,"### SWIGVM aborting with name '" << exaudflib::socket_info::get_socket_url() << "' (" << ::getppid() << ',' << ::getpid() << ')'); - ::unlink(exaudflib::socket_info::get_socket_file_name()); - ::abort(); - } -} - -static void *check_thread_routine(void* data) -{ - while(exaudflib::check::keep_checking) { - exaudflib::check::external_process_check(); - check_parent_pid(); - ::usleep(100000); - } - return NULL; -} - -void exaudflib::check::start_check_thread() { - if (!remote_client) - pthread_create(&check_thread, NULL, check_thread_routine, NULL); -} - -void exaudflib::check::stop_check_thread() { - keep_checking = false; -} - -void exaudflib::check::cancel_check_thread() { - ::pthread_cancel(check_thread); -} diff --git a/exaudfclient/base/exaudflib/impl/check.h b/exaudfclient/base/exaudflib/impl/check.h deleted file mode 100644 index c856d4a..0000000 --- a/exaudfclient/base/exaudflib/impl/check.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef EXAUDFCLIENT_EXAUDFLIB_CHECK_H -#define EXAUDFCLIENT_EXAUDFLIB_CHECK_H - - -namespace exaudflib { - namespace check { - void external_process_check(); - void start_check_thread(); - void stop_check_thread(); - void cancel_check_thread(); - void set_remote_client(bool value); - bool get_remote_client(); - } -} - - -#endif //EXAUDFCLIENT_EXAUDFLIB_CHECK_H diff --git a/exaudfclient/base/exaudflib/impl/exaudflib_main.cc b/exaudfclient/base/exaudflib/impl/exaudflib_main.cc deleted file mode 100644 index f595f84..0000000 --- a/exaudfclient/base/exaudflib/impl/exaudflib_main.cc +++ /dev/null @@ -1,297 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "utils/debug_message.h" - -// swig lib -#include - - -#include "exaudflib/impl/check.h" -#include "exaudflib/impl/socket_info.h" -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/impl/msg_conversion.h" -#include "exaudflib/impl/global.h" -#include "exaudflib/impl/socket_high_level.h" - -#include "exaudflib/vm/swig_vm.h" - - -#ifndef NDEBUG -#define SWIGVM_LOG_CLIENT -#endif -//#define SWIGVM_LOG_CLIENT -//#define LOG_COMMUNICATION - - -void print_args(int argc,char**argv){ - for (int i = 0; iexception_msg = ""; - vm->shutdown(); // Calls cleanup - if (vm->exception_msg.size()>0) { - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1110","### Caught error in vm->shutdown '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << "): " << vm->exception_msg); - msg ="F-UDF-CL-LIB-1111: Caught exception\n\n"+msg+"\n\n and caught another exception during cleanup\n\n"+vm->exception_msg; - } - } - delete_vm(vm); - } catch (SWIGVMContainers::SWIGVM::exception &err) { - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1112","### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << "): " << err.what()); - }catch(std::exception& err){ - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1113","### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << "): " << err.what()); - }catch(...){ - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1114","### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << "): "); - } - try{ - exaudflib::socket_high_level::send_close(socket, msg); - ::sleep(1); // give me a chance to die with my parent process - } catch (SWIGVMContainers::SWIGVM::exception &err) { - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1115","### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << "): " << err.what()); - }catch(std::exception& err){ - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1116","### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << "): " << err.what()); - }catch(...){ - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1117","### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << ")"); - } - - try{ - stop_all(socket); - } catch (SWIGVMContainers::SWIGVM::exception &err) { - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1118","### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << "): " << err.what()); - }catch(std::exception& err){ - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1119","### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << "): " << err.what()); - }catch(...){ - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1120","### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << "):"); - } - return 1; -} - -extern "C" { - -int exaudfclient_main(std::functionvmMaker,int argc,char**argv) -{ - assert(exaudflib::global.SWIGVM_params_ref != nullptr); - -#ifdef UDF_PLUGIN_CLIENT - std::stringstream socket_name_ss; -#endif - std::string socket_name = argv[1]; - exaudflib::socket_info::set_socket_file_name(argv[1]); - exaudflib::socket_info::set_socket_url(argv[1]); - - exaudflib::check::set_remote_client(false); - - zmq::context_t context(1); - - DBG_COND_FUNC_CALL(std::cerr, print_args(argc,argv)); - - if (socket_name.length() < 5 ) { - PRINT_ERROR_MESSAGE(std::cerr,"F-UDF-CL-LIB-1122", "socket name '" << socket_name << "' is invalid." ); - abort(); - } - - if (socket_name.compare(0, 4, "tcp:") == 0) { - exaudflib::check::set_remote_client(true); - } - - if (socket_name.compare(0, 4, "ipc:") == 0) - { -#ifdef UDF_PLUGIN_CLIENT -/* - DO NOT REMOVE, required for Exasol 6.2 -*/ - //In case of protegrity the /tmp folder can't be used as it runs within the database (and not in an isolated container). - //There, the actual socket file path is defined in a environment variable. We need to replace it here. - if (socket_name.compare(0, 11, "ipc:///tmp/") == 0) { - socket_name_ss << "ipc://" << getenv("NSEXEC_TMP_PATH") << '/' << &(socket_name.c_str()[11]); - socket_name = socket_name_ss.str(); - exaudflib::socket_info::set_socket_file_name(::strdup(socket_name.c_str())); - } -#endif - exaudflib::socket_info::set_socket_file_name(&(exaudflib::socket_info::get_socket_file_name()[6])); - } - - DBG_STREAM_MSG(std::cerr,"### SWIGVM starting " << argv[0] << " with name '" << socket_name << - " (" << ::getppid() << ',' << ::getpid() << "): '" << argv[1] << '\''); - - exaudflib::check::start_check_thread(); - - - int linger_timeout = 0; - int recv_sock_timeout = 1000; - int send_sock_timeout = 1000; - - if (exaudflib::check::get_remote_client()) { - recv_sock_timeout = 10000; - send_sock_timeout = 5000; - } - -reinit: - - DBGMSG(std::cerr,"Reinit"); - zmq::socket_t socket(context, ZMQ_REQ); - - socket.setsockopt(ZMQ_LINGER, &linger_timeout, sizeof(linger_timeout)); - socket.setsockopt(ZMQ_RCVTIMEO, &recv_sock_timeout, sizeof(recv_sock_timeout)); - socket.setsockopt(ZMQ_SNDTIMEO, &send_sock_timeout, sizeof(send_sock_timeout)); - - if (exaudflib::check::get_remote_client()) socket.bind(socket_name.c_str()); - else socket.connect(socket_name.c_str()); - - exaudflib::global.sock = &socket; - exaudflib::global.SWIGVM_params_ref->exch = &exaudflib::global.exchandler; - - SWIGVMContainers::SWIGVM* vm=nullptr; - - if (!exaudflib::socket_high_level::send_init(socket, socket_name)) { - if (!exaudflib::check::get_remote_client() && exaudflib::global.exchandler.exthrowed) { - return handle_error(socket, socket_name, vm, "F-UDF-CL-LIB-1123: " + - exaudflib::global.exchandler.exmsg, false); - }else{ - goto reinit; - } - } - - exaudflib::global.initSwigParams(); - - bool shutdown_vm_in_case_of_error = false; - try { - vm = vmMaker(); - if (vm == nullptr) { - return handle_error(socket, socket_name, vm, "F-UDF-CL-LIB-1124: Unknown or unsupported VM type", false); - } - if (vm->exception_msg.size()>0) { - return handle_error(socket, socket_name, vm, "F-UDF-CL-LIB-1125: "+vm->exception_msg, false); - } - shutdown_vm_in_case_of_error = true; - exaudflib::socket_low_level::init(vm->useZmqSocketLocks()); - if (exaudflib::global.singleCallMode) { - ExecutionGraph::EmptyDTO noArg; // used as dummy arg - for (;;) { - // in single call mode, after MT_RUN from the client, - // EXASolution responds with a CALL message that specifies - // the single call function to be made - if (!exaudflib::socket_high_level::send_run(socket)) { - break; - } - - assert(exaudflib::global.singleCallFunction != SWIGVMContainers::single_call_function_id_e::SC_FN_NIL); - try { - const char* result = nullptr; - switch (exaudflib::global.singleCallFunction) - { - case SWIGVMContainers::single_call_function_id_e::SC_FN_NIL: - break; - case SWIGVMContainers::single_call_function_id_e::SC_FN_DEFAULT_OUTPUT_COLUMNS: - result = vm->singleCall(exaudflib::global.singleCallFunction,noArg); - break; - case SWIGVMContainers::single_call_function_id_e::SC_FN_GENERATE_SQL_FOR_IMPORT_SPEC: - assert(!exaudflib::global.singleCall_ImportSpecificationArg.isEmpty()); - result = vm->singleCall(exaudflib::global.singleCallFunction,exaudflib::global.singleCall_ImportSpecificationArg); - exaudflib::global.singleCall_ImportSpecificationArg = ExecutionGraph::ImportSpecification(); // delete the last argument - break; - case SWIGVMContainers::single_call_function_id_e::SC_FN_GENERATE_SQL_FOR_EXPORT_SPEC: - assert(!exaudflib::global.singleCall_ExportSpecificationArg.isEmpty()); - result = vm->singleCall(exaudflib::global.singleCallFunction,exaudflib::global.singleCall_ExportSpecificationArg); - exaudflib::global.singleCall_ExportSpecificationArg = ExecutionGraph::ExportSpecification(); // delete the last argument - break; - case SWIGVMContainers::single_call_function_id_e::SC_FN_VIRTUAL_SCHEMA_ADAPTER_CALL: - assert(!exaudflib::global.singleCall_StringArg.isEmpty()); - result = vm->singleCall(exaudflib::global.singleCallFunction,exaudflib::global.singleCall_StringArg); - break; - } - if (vm->exception_msg.size()>0) { - return handle_error(socket, socket_name, vm, "F-UDF-CL-LIB-1126: "+vm->exception_msg,true); - } - - if (vm->calledUndefinedSingleCall.size()>0) { - exaudflib::socket_high_level::send_undefined_call(socket, vm->calledUndefinedSingleCall); - } else { - exaudflib::socket_high_level::send_return(socket,result); - } - - if (!exaudflib::socket_high_level::send_done(socket)) { - break; - } - } catch(...) {} - } - } else { - for(;;) { - if (!exaudflib::socket_high_level::send_run(socket)) - break; - exaudflib::global.SWIGVM_params_ref->inp_force_finish = false; - while(!vm->run_()) - { - if (vm->exception_msg.size()>0) { - return handle_error(socket, socket_name, vm, "F-UDF-CL-LIB-1127: "+vm->exception_msg,true); - } - } - if (!exaudflib::socket_high_level::send_done(socket)) - break; - } - } - - if (vm != nullptr) - { - vm->shutdown(); - if (vm->exception_msg.size()>0) { - return handle_error(socket, socket_name, vm, "F-UDF-CL-LIB-1128: "+vm->exception_msg,false); - } - } - exaudflib::socket_high_level::send_finished(socket); - } catch (SWIGVMContainers::SWIGVM::exception &err) { - DBG_STREAM_MSG(std::cerr,"### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << "): " << err.what()); - return handle_error(socket, socket_name, vm, "F-UDF-CL-LIB-1129: "+std::string(err.what()),shutdown_vm_in_case_of_error); - } catch (std::exception &err) { - DBG_STREAM_MSG(std::cerr,"### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << "): " << err.what()); - return handle_error(socket, socket_name, vm, "F-UDF-CL-LIB-1130: "+std::string(err.what()),shutdown_vm_in_case_of_error); - } catch (...) { - DBG_STREAM_MSG(std::cerr,"### SWIGVM crashing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << ')'); - return handle_error(socket, socket_name, vm, "F-UDF-CL-LIB-1131: Internal/Unknown error",shutdown_vm_in_case_of_error); - } - - DBG_STREAM_MSG(std::cerr,"### SWIGVM finishing with name '" << socket_name << " (" << ::getppid() << ',' << ::getpid() << ')'); - - delete_vm(vm); - stop_all(socket); - return 0; -} - - -} // extern "C" diff --git a/exaudfclient/base/exaudflib/impl/global.cc b/exaudfclient/base/exaudflib/impl/global.cc deleted file mode 100644 index d5f570b..0000000 --- a/exaudfclient/base/exaudflib/impl/global.cc +++ /dev/null @@ -1,73 +0,0 @@ -#include "exaudflib/impl/global.h" -#include - -namespace exaudflib { -static std::string g_database_name; -static std::string g_database_version; -static std::string g_script_name; -static std::string g_script_schema; -static std::string g_current_user; -static std::string g_scope_user; -static std::string g_current_schema; -static std::string g_source_code; -static unsigned long long g_session_id; -static unsigned long g_statement_id; -static unsigned int g_node_count; -static unsigned int g_node_id; -static unsigned long long g_vm_id; - -Global global; -} - -exaudflib::Global::Global() -: SWIGVM_params_ref(nullptr), sock(nullptr) {} - -void exaudflib::Global::initSwigParams() { - /** - * We need to store the strings here in static variables as the pointers are shared between the linker namespaces. - * It made trouble allocating the strings on the heap in the past: it might occur that the destructor is - * called twice. As a workaround we keep it as static variables which are deallocated only at the final termination - * of the program: It still might occur that the destructor is called more than once, but at least everything else has - * already finished. - */ - SWIGVM_params_ref->dbname = (char*) g_database_name.c_str(); - SWIGVM_params_ref->dbversion = (char*) g_database_version.c_str(); - SWIGVM_params_ref->script_name = (char*) g_script_name.c_str(); - SWIGVM_params_ref->script_schema = (char*) g_script_schema.c_str(); - SWIGVM_params_ref->current_user = (char*) g_current_user.c_str(); - SWIGVM_params_ref->current_schema = (char*) g_current_schema.c_str(); - SWIGVM_params_ref->scope_user = (char*) g_scope_user.c_str(); - SWIGVM_params_ref->script_code = (char*) g_source_code.c_str(); - SWIGVM_params_ref->session_id = g_session_id; - SWIGVM_params_ref->statement_id = g_statement_id; - SWIGVM_params_ref->node_count = g_node_count; - SWIGVM_params_ref->node_id = g_node_id; - SWIGVM_params_ref->vm_id = g_vm_id; - SWIGVM_params_ref->singleCallMode = singleCallMode; -} - -void exaudflib::Global::writeScriptParams(const exascript_info &rep) { - g_database_name = rep.database_name(); - g_database_version = rep.database_version(); - g_script_name = rep.script_name(); - g_script_schema = rep.script_schema(); - g_current_user = rep.current_user(); - g_scope_user = rep.scope_user(); - if (g_scope_user.size()==0) { // for backward compatibility when testing with EXASOL 6.0.8 installations at OTTO Brain - g_scope_user=g_current_user; - } - g_current_schema = rep.current_schema(); - g_source_code = rep.source_code(); - g_session_id = rep.session_id(); - g_statement_id = rep.statement_id(); - g_node_count = rep.node_count(); - g_node_id = rep.node_id(); - g_vm_id = rep.vm_id(); -} - -extern "C" { - void set_SWIGVM_params(SWIGVMContainers::SWIGVM_params_t* p) { - exaudflib::global.SWIGVM_params_ref = p; - } -} - diff --git a/exaudfclient/base/exaudflib/impl/global.h b/exaudfclient/base/exaudflib/impl/global.h deleted file mode 100644 index 61e0b8c..0000000 --- a/exaudfclient/base/exaudflib/impl/global.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef EXAUDFCLIENT_EXUDFLIB_GLOBAL_H -#define EXAUDFCLIENT_EXUDFLIB_GLOBAL_H - -#include "exaudflib/swig/script_data_transfer_objects.h" -#include "exaudflib/zmqcontainer.pb.h" -#include "exaudflib/swig/swig_common.h" -#include - -namespace exaudflib { - struct Global { - Global(); - void initSwigParams(); - void writeScriptParams(const exascript_info &rep); - SWIGVMContainers::SWIGVM_params_t * SWIGVM_params_ref; - zmq::socket_t *sock; - ExecutionGraph::StringDTO singleCall_StringArg; - bool singleCallMode; - SWIGVMContainers::single_call_function_id_e singleCallFunction; - ExecutionGraph::ImportSpecification singleCall_ImportSpecificationArg; - ExecutionGraph::ExportSpecification singleCall_ExportSpecificationArg; - SWIGVMContainers::SWIGVMExceptionHandler exchandler; - }; - - extern Global global; -} - - -#endif //EXAUDFCLIENT_EXUDFLIB_GLOBAL_H diff --git a/exaudfclient/base/exaudflib/impl/msg_conversion.cc b/exaudfclient/base/exaudflib/impl/msg_conversion.cc deleted file mode 100644 index c09ee02..0000000 --- a/exaudfclient/base/exaudflib/impl/msg_conversion.cc +++ /dev/null @@ -1,71 +0,0 @@ -#include "exaudflib/impl/msg_conversion.h" - -std::string exaudflib::msg_conversion::convert_message_type_to_string(int message_type){ - switch (message_type) - { - case 0: - return "MT_UNKNOWN"; - case 1: - return "MT_CLIENT"; - case 2: - return "MT_INFO"; - case 3: - return "MT_META"; - case 4: - return "MT_CLOSE"; - case 5: - return "MT_IMPORT"; - case 6: - return "MT_NEXT"; - case 7: - return "MT_RESET"; - case 8: - return "MT_EMIT"; - case 9: - return "MT_RUN"; - case 10: - return "MT_DONE"; - case 11: - return "MT_CLEANUP"; - case 12: - return "MT_FINISHED"; - case 13: - return "MT_PING_PONG"; - case 14: - return "MT_TRY_AGAIN"; - case 15: - return "MT_CALL"; - case 16: - return "MT_RETURN"; - case 17: - return "MT_UNDEFINED_CALL"; - default: - return "unknown: " + message_type; - } -} - -std::string exaudflib::msg_conversion::convert_type_to_string(int type){ - switch (type) - { - case 0: - return "PB_UNSUPPORTED"; - case 1: - return "PB_DOUBLE"; - case 2: - return "PB_INT32"; - case 3: - return "PB_INT64"; - case 4: - return "PB_NUMERIC"; - case 5: - return "PB_TIMESTAMP"; - case 6: - return "PB_DATE"; - case 7: - return "PB_STRING"; - case 8: - return "PB_BOOLEAN"; - default: - return "unknown: " + type; - } -} diff --git a/exaudfclient/base/exaudflib/impl/msg_conversion.h b/exaudfclient/base/exaudflib/impl/msg_conversion.h deleted file mode 100644 index fbeec4d..0000000 --- a/exaudfclient/base/exaudflib/impl/msg_conversion.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef EXAUDFCLIENT_EXAUDFLIB_MSG_CONVERSION_H -#define EXAUDFCLIENT_EXAUDFLIB_MSG_CONVERSION_H - -#include - -namespace exaudflib { - namespace msg_conversion { - std::string convert_message_type_to_string(int message_type); - std::string convert_type_to_string(int type); - } -} - - -#endif //EXAUDFCLIENT_EXAUDFLIB_MSG_CONVERSION_H diff --git a/exaudfclient/base/exaudflib/impl/socket_high_level.cc b/exaudfclient/base/exaudflib/impl/socket_high_level.cc deleted file mode 100644 index 13048b6..0000000 --- a/exaudfclient/base/exaudflib/impl/socket_high_level.cc +++ /dev/null @@ -1,528 +0,0 @@ -#include "exaudflib/impl/socket_high_level.h" - -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/zmqcontainer.pb.h" -#include "exaudflib/impl/msg_conversion.h" -#include "exaudflib/impl/global.h" -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/vm/swig_vm.h" -#include -#include "utils/debug_message.h" - -namespace exaudflib { - namespace socket_high_level { - //static exascript_vmtype vm_type; - static exascript_request request; - static exascript_response response; - static std::string output_buffer; - } -} - - -bool exaudflib::socket_high_level::send_init(zmq::socket_t &socket, const std::string client_name) -{ - exaudflib::socket_high_level::request.Clear(); - exaudflib::socket_high_level::request.set_type(MT_CLIENT); - exaudflib::socket_high_level::request.set_connection_id(0); - exascript_client *req = exaudflib::socket_high_level::request.mutable_client(); - req->set_client_name(client_name); - if (!exaudflib::socket_high_level::request.SerializeToString(&exaudflib::socket_high_level::output_buffer)) { - exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1002: Communication error: failed to serialize data"); - return false; - } - zmq::message_t zmsg((void*)exaudflib::socket_high_level::output_buffer.c_str(), - exaudflib::socket_high_level::output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(socket, zmsg); - - zmq::message_t zmsgrecv; - exaudflib::socket_high_level::response.Clear(); - if (!exaudflib::socket_low_level::socket_recv(socket, zmsgrecv, true)) - return false; - if (!exaudflib::socket_high_level::response.ParseFromArray(zmsgrecv.data(), zmsgrecv.size())) { - exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1003: Failed to parse data"); - return false; - } - - exaudflib::global.SWIGVM_params_ref->connection_id = exaudflib::socket_high_level::response.connection_id(); - DBG_STREAM_MSG(std::cerr,"### SWIGVM connected with id " << - std::hex << exaudflib::global.SWIGVM_params_ref->connection_id); - if (exaudflib::socket_high_level::response.type() == MT_CLOSE) { - if (exaudflib::socket_high_level::response.close().has_exception_message()) - exaudflib::global.exchandler.setException( - exaudflib::socket_high_level::response.close().exception_message().c_str()); - else exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1004: Connection closed by server"); - return false; - } - if (exaudflib::socket_high_level::response.type() != MT_INFO) { - exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1005: Wrong message type, should be MT_INFO got "+ - exaudflib::msg_conversion::convert_message_type_to_string(exaudflib::socket_high_level::response.type())); - return false; - } - const exascript_info &rep = exaudflib::socket_high_level::response.info(); - exaudflib::global.writeScriptParams(rep); - //vm_type = rep.vm_type(); - - - exaudflib::global.SWIGVM_params_ref->maximal_memory_limit = rep.maximal_memory_limit(); - struct rlimit d; - d.rlim_cur = d.rlim_max = rep.maximal_memory_limit(); - if (setrlimit(RLIMIT_RSS, &d) != 0) -#ifdef SWIGVM_LOG_CLIENT - std::cerr << "W-UDF-CL-LIB-1006: Failed to set memory limit" << std::endl; -#else - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1007: Failed to set memory limit"); -#endif - d.rlim_cur = d.rlim_max = 0; // 0 for no core dumps, RLIM_INFINITY to enable coredumps of any size - if (setrlimit(RLIMIT_CORE, &d) != 0) -#ifdef SWIGVM_LOG_CLIENT - std::cerr << "W-UDF-CL-LIB-1008: Failed to set core dump size limit" << std::endl; -#else - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1009: Failed to set core dump size limit"); -#endif - /* d.rlim_cur = d.rlim_max = 65536; */ - getrlimit(RLIMIT_NOFILE,&d); - if (d.rlim_max < 32768) - { - //#ifdef SWIGVM_LOG_CLIENT - std::cerr << "W-UDF-CL-LIB-1010: Reducing RLIMIT_NOFILE below 32768" << std::endl; - //#endif - } - d.rlim_cur = d.rlim_max = std::min(32768,(int)d.rlim_max); - if (setrlimit(RLIMIT_NOFILE, &d) != 0) -#ifdef SWIGVM_LOG_CLIENT - std::cerr << "W-UDF-CL-LIB-1011: Failed to set nofile limit" << std::endl; -#else - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1012: Failed to set nofile limit"); -#endif - d.rlim_cur = d.rlim_max = 32768; - if (setrlimit(RLIMIT_NPROC, &d) != 0) - { - std::cerr << "W-UDF-CL-LIB-1013: Failed to set nproc limit to 32k trying 8k ..." << std::endl; - d.rlim_cur = d.rlim_max = 8192; - if (setrlimit(RLIMIT_NPROC, &d) != 0) -#ifdef SWIGVM_LOG_CLIENT - std::cerr << "W-UDF-CL-LIB-1014: Failed to set nproc limit" << std::endl; -#else - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1015: Failed to set nproc limit"); -#endif - } - - { /* send meta request */ - exaudflib::socket_high_level::request.Clear(); - exaudflib::socket_high_level::request.set_type(MT_META); - exaudflib::socket_high_level::request.set_connection_id(exaudflib::global.SWIGVM_params_ref->connection_id); - if (!exaudflib::socket_high_level::request.SerializeToString(&exaudflib::socket_high_level::output_buffer)) { - exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1016: Communication error: failed to serialize data"); - return false; - } - zmq::message_t zmsg((void*)exaudflib::socket_high_level::output_buffer.c_str(), - exaudflib::socket_high_level::output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(socket, zmsg); - } /* receive meta response */ - { zmq::message_t zmsg; - exaudflib::socket_low_level::socket_recv(socket, zmsg); - exaudflib::socket_high_level::response.Clear(); - if (!exaudflib::socket_high_level::response.ParseFromArray(zmsg.data(), zmsg.size())) { - exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1017: Communication error: failed to parse data"); - return false; - } - if (exaudflib::socket_high_level::response.type() == MT_CLOSE) { - if (exaudflib::socket_high_level::response.close().has_exception_message()) - exaudflib::global.exchandler.setException( - exaudflib::socket_high_level::response.close().exception_message().c_str()); - else exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1018: Connection closed by server"); - return false; - } - if (exaudflib::socket_high_level::response.type() != MT_META) { - exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1019: Wrong message type, should be META, got "+ - exaudflib::msg_conversion::convert_message_type_to_string(exaudflib::socket_high_level::response.type())); - return false; - } - const exascript_metadata &rep = exaudflib::socket_high_level::response.meta(); - exaudflib::global.singleCallMode = rep.single_call_mode(); - exaudflib::global.SWIGVM_params_ref->inp_iter_type = (SWIGVMContainers::SWIGVM_itertype_e)(rep.input_iter_type()); - exaudflib::global.SWIGVM_params_ref->out_iter_type = (SWIGVMContainers::SWIGVM_itertype_e)(rep.output_iter_type()); - for (int col = 0; col < rep.input_columns_size(); ++col) { - const exascript_metadata_column_definition &coldef = rep.input_columns(col); - exaudflib::global.SWIGVM_params_ref->inp_names->push_back(coldef.name()); - exaudflib::global.SWIGVM_params_ref->inp_types->push_back(SWIGVMContainers::SWIGVM_columntype_t()); - SWIGVMContainers::SWIGVM_columntype_t &coltype = exaudflib::global.SWIGVM_params_ref->inp_types->back(); - coltype.len = 0; coltype.prec = 0; coltype.scale = 0; - coltype.type_name = coldef.type_name(); - switch (coldef.type()) { - case PB_UNSUPPORTED: - exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1020: Unsupported input column type found"); - return false; - case PB_DOUBLE: - coltype.type = SWIGVMContainers::DOUBLE; - break; - case PB_INT32: - coltype.type = SWIGVMContainers::INT32; - coltype.prec = coldef.precision(); - coltype.scale = coldef.scale(); - break; - case PB_INT64: - coltype.type = SWIGVMContainers::INT64; - coltype.prec = coldef.precision(); - coltype.scale = coldef.scale(); - break; - case PB_NUMERIC: - coltype.type = SWIGVMContainers::NUMERIC; - coltype.prec = coldef.precision(); - coltype.scale = coldef.scale(); - break; - case PB_TIMESTAMP: - coltype.type = SWIGVMContainers::TIMESTAMP; - break; - case PB_DATE: - coltype.type = SWIGVMContainers::DATE; - break; - case PB_STRING: - coltype.type = SWIGVMContainers::STRING; - coltype.len = coldef.size(); - break; - case PB_BOOLEAN: - coltype.type = SWIGVMContainers::BOOLEAN; - break; - default: - exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1021: Unknown input column type found, got "+coldef.type()); - return false; - } - } - for (int col = 0; col < rep.output_columns_size(); ++col) { - const exascript_metadata_column_definition &coldef = rep.output_columns(col); - exaudflib::global.SWIGVM_params_ref->out_names->push_back(coldef.name()); - exaudflib::global.SWIGVM_params_ref->out_types->push_back(SWIGVMContainers::SWIGVM_columntype_t()); - SWIGVMContainers::SWIGVM_columntype_t &coltype = exaudflib::global.SWIGVM_params_ref->out_types->back(); - coltype.len = 0; coltype.prec = 0; coltype.scale = 0; - coltype.type_name = coldef.type_name(); - switch (coldef.type()) { - case PB_UNSUPPORTED: - exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1022: Unsupported output column type found"); - return false; - case PB_DOUBLE: - coltype.type = SWIGVMContainers::DOUBLE; - break; - case PB_INT32: - coltype.type = SWIGVMContainers::INT32; - coltype.prec = coldef.precision(); - coltype.scale = coldef.scale(); - break; - case PB_INT64: - coltype.type = SWIGVMContainers::INT64; - coltype.prec = coldef.precision(); - coltype.scale = coldef.scale(); - break; - case PB_NUMERIC: - coltype.type = SWIGVMContainers::NUMERIC; - coltype.prec = coldef.precision(); - coltype.scale = coldef.scale(); - break; - case PB_TIMESTAMP: - coltype.type = SWIGVMContainers::TIMESTAMP; - break; - case PB_DATE: - coltype.type = SWIGVMContainers::DATE; - break; - case PB_STRING: - coltype.type = SWIGVMContainers::STRING; - coltype.len = coldef.size(); - break; - case PB_BOOLEAN: - coltype.type = SWIGVMContainers::BOOLEAN; - break; - default: - exaudflib::global.exchandler.setException("F-UDF-CL-LIB-1023: Unknown output column type found, got "+coldef.type()); - return false; - } - } - } - return true; -} - -void exaudflib::socket_high_level::send_close(zmq::socket_t &socket, const std::string &exmsg) -{ - exaudflib::socket_high_level::request.Clear(); - exaudflib::socket_high_level::request.set_type(MT_CLOSE); - exaudflib::socket_high_level::request.set_connection_id(exaudflib::global.SWIGVM_params_ref->connection_id); - exascript_close *req = - exaudflib::socket_high_level::request.mutable_close(); - if (exmsg != "") req->set_exception_message(exmsg); - exaudflib::socket_high_level::request.SerializeToString(&exaudflib::socket_high_level::output_buffer); - zmq::message_t zmsg((void*)exaudflib::socket_high_level::output_buffer.c_str(), - exaudflib::socket_high_level::output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(socket, zmsg); - - { /* receive finished response, so we know that the DB knows that we are going to close and - all potential exceptions have been received on DB side */ - zmq::message_t zmsg; - exaudflib::socket_low_level::socket_recv(socket, zmsg); - exaudflib::socket_high_level::response.Clear(); - if(!exaudflib::socket_high_level::response.ParseFromArray(zmsg.data(), zmsg.size())) { - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1024: Communication error: failed to parse data"); - } - else if (exaudflib::socket_high_level::response.type() != MT_FINISHED) { - throw - SWIGVMContainers::SWIGVM::exception( - "F-UDF-CL-LIB-1025: Wrong response type, should be MT_FINISHED, got "+ - exaudflib::msg_conversion::convert_message_type_to_string( - exaudflib::socket_high_level::response.type())); - } - } -} - -bool exaudflib::socket_high_level::send_run(zmq::socket_t &socket) -{ - { - /* send done request */ - exaudflib::socket_high_level::request.Clear(); - exaudflib::socket_high_level::request.set_type(MT_RUN); - exaudflib::socket_high_level::request.set_connection_id(exaudflib::global.SWIGVM_params_ref->connection_id); - if (!exaudflib::socket_high_level::request.SerializeToString(&exaudflib::socket_high_level::output_buffer)) - { - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1026: Communication error: failed to serialize data"); - } - zmq::message_t zmsg((void*)exaudflib::socket_high_level::output_buffer.c_str(), - exaudflib::socket_high_level::output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(socket, zmsg); - } { /* receive done response */ - zmq::message_t zmsg; - exaudflib::socket_low_level::socket_recv(socket, zmsg); - exaudflib::socket_high_level::response.Clear(); - if (!exaudflib::socket_high_level::response.ParseFromArray(zmsg.data(), zmsg.size())) - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1027: Communication error: failed to parse data"); - if (exaudflib::socket_high_level::response.type() == MT_CLOSE) { - if (exaudflib::socket_high_level::response.close().has_exception_message()) - throw SWIGVMContainers::SWIGVM::exception( - exaudflib::socket_high_level::response.close().exception_message().c_str()); - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1028: Wrong response type, got empty MT_CLOSE"); - } else if (exaudflib::socket_high_level::response.type() == MT_CLEANUP) { - return false; - } else if (exaudflib::global.singleCallMode && exaudflib::socket_high_level::response.type() == MT_CALL) { - assert(exaudflib::global.singleCallMode); - exascript_single_call_rep sc = exaudflib::socket_high_level::response.call(); - exaudflib::global.singleCallFunction = static_cast(sc.fn()); - - switch (exaudflib::global.singleCallFunction) - { - case SWIGVMContainers::single_call_function_id_e::SC_FN_NIL: - case SWIGVMContainers::single_call_function_id_e::SC_FN_DEFAULT_OUTPUT_COLUMNS: - break; - case SWIGVMContainers::single_call_function_id_e::SC_FN_GENERATE_SQL_FOR_IMPORT_SPEC: - { - - if (!sc.has_import_specification()) - { - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1029: internal error SC_FN_GENERATE_SQL_FOR_IMPORT_SPEC without import specification"); - } - const import_specification_rep& is_proto = sc.import_specification(); - exaudflib::global.singleCall_ImportSpecificationArg = ExecutionGraph::ImportSpecification(is_proto.is_subselect()); - if (is_proto.has_connection_information()) - { - const connection_information_rep& ci_proto = is_proto.connection_information(); - ExecutionGraph::ConnectionInformation connection_info(ci_proto.kind(), ci_proto.address(), ci_proto.user(), ci_proto.password()); - exaudflib::global.singleCall_ImportSpecificationArg.setConnectionInformation(connection_info); - } - if (is_proto.has_connection_name()) - { - exaudflib::global.singleCall_ImportSpecificationArg.setConnectionName(is_proto.connection_name()); - } - for (int i=0; iset_result(result); - exaudflib::socket_high_level::request.set_allocated_call_result(rr); - exaudflib::socket_high_level::request.set_connection_id(exaudflib::global.SWIGVM_params_ref->connection_id); - if (!exaudflib::socket_high_level::request.SerializeToString(&exaudflib::socket_high_level::output_buffer)) - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1033: Communication error: failed to serialize data"); - zmq::message_t zmsg((void*)exaudflib::socket_high_level::output_buffer.c_str(), - exaudflib::socket_high_level::output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(socket, zmsg); - } { /* receive return response */ - zmq::message_t zmsg; - exaudflib::socket_low_level::socket_recv(socket, zmsg); - exaudflib::socket_high_level::response.Clear(); - if (!exaudflib::socket_high_level::response.ParseFromArray(zmsg.data(), zmsg.size())) - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1034: Communication error: failed to parse data"); - if (exaudflib::socket_high_level::response.type() == MT_CLOSE) { - if (exaudflib::socket_high_level::response.close().has_exception_message()) - throw SWIGVMContainers::SWIGVM::exception( - exaudflib::socket_high_level::response.close().exception_message().c_str()); - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1035: Wrong response type, got empty close response"); - } else if (exaudflib::socket_high_level::response.type() == MT_CLEANUP) { - return false; - } else if (exaudflib::socket_high_level::response.type() != MT_RETURN) { - throw SWIGVMContainers::SWIGVM::exception( - "F-UDF-CL-LIB-1036: Wrong response type, should be MT_RETURN, got "+ - exaudflib::msg_conversion::convert_message_type_to_string( - exaudflib::socket_high_level::response.type())); - } - } - return true; -} - -void exaudflib::socket_high_level::send_undefined_call(zmq::socket_t &socket, const std::string& fn) -{ - { /* send return request */ - exaudflib::socket_high_level::request.Clear(); - exaudflib::socket_high_level::request.set_type(MT_UNDEFINED_CALL); - ::exascript_undefined_call_req* uc = new ::exascript_undefined_call_req(); - uc->set_remote_fn(fn); - exaudflib::socket_high_level::request.set_allocated_undefined_call(uc); - exaudflib::socket_high_level::request.set_connection_id(exaudflib::global.SWIGVM_params_ref->connection_id); - if (!exaudflib::socket_high_level::request.SerializeToString(&exaudflib::socket_high_level::output_buffer)) - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1037: Communication error: failed to serialize data"); - zmq::message_t zmsg((void*)exaudflib::socket_high_level::output_buffer.c_str(), - exaudflib::socket_high_level::output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(socket, zmsg); - } { /* receive return response */ - zmq::message_t zmsg; - exaudflib::socket_low_level::socket_recv(socket, zmsg); - exaudflib::socket_high_level::response.Clear(); - if (!exaudflib::socket_high_level::response.ParseFromArray(zmsg.data(), zmsg.size())) - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1038: Communication error: failed to parse data"); - if (exaudflib::socket_high_level::response.type() != MT_UNDEFINED_CALL) { - throw SWIGVMContainers::SWIGVM::exception( - "F-UDF-CL-LIB-1039: Wrong response type, should be MT_UNDEFINED_CALL, got "+ - exaudflib::msg_conversion::convert_message_type_to_string( - exaudflib::socket_high_level::response.type())); - } - } -} - -bool exaudflib::socket_high_level::send_done(zmq::socket_t &socket) -{ - { /* send done request */ - exaudflib::socket_high_level::request.Clear(); - exaudflib::socket_high_level::request.set_type(MT_DONE); - exaudflib::socket_high_level::request.set_connection_id(exaudflib::global.SWIGVM_params_ref->connection_id); - if (!exaudflib::socket_high_level::request.SerializeToString(&exaudflib::socket_high_level::output_buffer)) - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1040: Communication error: failed to serialize data"); - zmq::message_t zmsg((void*)exaudflib::socket_high_level::output_buffer.c_str(), - exaudflib::socket_high_level::output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(socket, zmsg); - } - { /* receive done response */ - zmq::message_t zmsg; - exaudflib::socket_low_level::socket_recv(socket, zmsg); - exaudflib::socket_high_level::response.Clear(); - if (!exaudflib::socket_high_level::response.ParseFromArray(zmsg.data(), zmsg.size())) - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1041: Communication error: failed to parse data"); - if (exaudflib::socket_high_level::response.type() == MT_CLOSE) { - if (exaudflib::socket_high_level::response.close().has_exception_message()) - throw SWIGVMContainers::SWIGVM::exception( - exaudflib::socket_high_level::response.close().exception_message().c_str()); - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1042: Wrong response type, got empty close response"); - } else if (exaudflib::socket_high_level::response.type() == MT_CLEANUP) { - return false; - } else if (exaudflib::socket_high_level::response.type() != MT_DONE) - throw SWIGVMContainers::SWIGVM::exception( - "F-UDF-CL-LIB-1043: Wrong response type, should be MT_DONE, got "+ - exaudflib::msg_conversion::convert_message_type_to_string( - exaudflib::socket_high_level::response.type())); - } - return true; -} - -void exaudflib::socket_high_level::send_finished(zmq::socket_t &socket) -{ - { /* send done request */ - exaudflib::socket_high_level::request.Clear(); - exaudflib::socket_high_level::request.set_type(MT_FINISHED); - exaudflib::socket_high_level::request.set_connection_id(exaudflib::global.SWIGVM_params_ref->connection_id); - if (!exaudflib::socket_high_level::request.SerializeToString(&exaudflib::socket_high_level::output_buffer)) - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1044: Communication error: failed to serialize data"); - zmq::message_t zmsg((void*)exaudflib::socket_high_level::output_buffer.c_str(), - exaudflib::socket_high_level::output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(socket, zmsg); - } { /* receive done response */ - zmq::message_t zmsg; - exaudflib::socket_low_level::socket_recv(socket, zmsg); - exaudflib::socket_high_level::response.Clear(); - if(!exaudflib::socket_high_level::response.ParseFromArray(zmsg.data(), zmsg.size())) - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1045: Communication error: failed to parse data"); - if (exaudflib::socket_high_level::response.type() == MT_CLOSE) { - if (exaudflib::socket_high_level::response.close().has_exception_message()) - throw SWIGVMContainers::SWIGVM::exception(exaudflib::socket_high_level::response.close().exception_message().c_str()); - throw SWIGVMContainers::SWIGVM::exception("F-UDF-CL-LIB-1046: Wrong response type, got empty MT_CLOSE"); - } else if (exaudflib::socket_high_level::response.type() != MT_FINISHED) - throw SWIGVMContainers::SWIGVM::exception( - "F-UDF-CL-LIB-1047: Wrong response type, should be MT_FINISHED, got"+ - exaudflib::msg_conversion::convert_message_type_to_string( - exaudflib::socket_high_level::response.type())); - } -} diff --git a/exaudfclient/base/exaudflib/impl/socket_high_level.h b/exaudfclient/base/exaudflib/impl/socket_high_level.h deleted file mode 100644 index 5180cc1..0000000 --- a/exaudfclient/base/exaudflib/impl/socket_high_level.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef EXAUDFLIB_SOCKET_HIGH_LEVEL_H -#define EXAUDFLIB_SOCKET_HIGH_LEVEL_H - -#include -#include - -namespace exaudflib { - namespace socket_high_level { - - bool send_init(zmq::socket_t &socket, const std::string client_name); - void send_close(zmq::socket_t &socket, const std::string &exmsg); - bool send_run(zmq::socket_t &socket); - void send_undefined_call(zmq::socket_t &socket, const std::string& fn); - bool send_done(zmq::socket_t &socket); - void send_finished(zmq::socket_t &socket); - bool send_return(zmq::socket_t &socket, const char* result); - - } //socket_high_level -} //exaudflib - - -#endif //EXAUDFLIB_SOCKET_HIGH_LEVEL_H diff --git a/exaudfclient/base/exaudflib/impl/socket_info.cc b/exaudfclient/base/exaudflib/impl/socket_info.cc deleted file mode 100644 index 9df03c2..0000000 --- a/exaudfclient/base/exaudflib/impl/socket_info.cc +++ /dev/null @@ -1,26 +0,0 @@ -#include "exaudflib/impl/socket_info.h" - - -namespace exaudflib { - namespace socket_info { - static const char *socket_file_name; - static const char *socket_name_url; - } -} - - -void exaudflib::socket_info::set_socket_url(const char* the_socket_url) { - socket_name_url = the_socket_url; -} - -const char* exaudflib::socket_info::get_socket_url() { - return socket_name_url; -} - -void exaudflib::socket_info::set_socket_file_name(const char* the_socket_file_name) { - socket_file_name = the_socket_file_name; -} - -const char* exaudflib::socket_info::get_socket_file_name() { - return socket_file_name; -} diff --git a/exaudfclient/base/exaudflib/impl/socket_info.h b/exaudfclient/base/exaudflib/impl/socket_info.h deleted file mode 100644 index 15de4c1..0000000 --- a/exaudfclient/base/exaudflib/impl/socket_info.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef EXAUDFCLIENT_SOCKET_INFO_H -#define EXAUDFCLIENT_SOCKET_INFO_H - -namespace exaudflib { - namespace socket_info { - void set_socket_url(const char*); - void set_socket_file_name(const char*); - const char* get_socket_file_name(); - const char* get_socket_url(); - } -} - - -#endif //EXAUDFCLIENT_SOCKET_INFO_H diff --git a/exaudfclient/base/exaudflib/impl/socket_low_level.cc b/exaudfclient/base/exaudflib/impl/socket_low_level.cc deleted file mode 100644 index 9a324d0..0000000 --- a/exaudfclient/base/exaudflib/impl/socket_low_level.cc +++ /dev/null @@ -1,101 +0,0 @@ -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/impl/check.h" -#include "utils/debug_message.h" -#include -#include -#include - -namespace exaudflib { - namespace socket_low_level { - static std::mutex zmq_socket_mutex; - static bool use_zmq_socket_locks = false; - } -} - -void exaudflib::socket_low_level::init(bool _use_zmq_socket_locks) { - use_zmq_socket_locks = _use_zmq_socket_locks; -} - -void exaudflib::socket_low_level::socket_send(zmq::socket_t &socket, zmq::message_t &zmsg) { - DBG_FUNC_BEGIN(std::cerr); -#ifdef LOG_COMMUNICATION - stringstream sb; - uint32_t len = zmsg.size(); - sb << "/tmp/zmqcomm_log_" << ::getpid() << "_send.data"; - int fd = ::open(sb.str().c_str(), O_CREAT | O_APPEND | O_WRONLY, 00644); - if (fd >= 0) { - if (::write(fd, &len, sizeof(uint32_t)) == -1 ) {perror("Log communication");} - if (::write(fd, zmsg.data(), len) == -1) {perror("Log communication");} - ::close(fd); - } -#endif - for (;;) { - try { - if (use_zmq_socket_locks) { - zmq_socket_mutex.lock(); - } - if (socket.send(zmsg) == true) { - if (use_zmq_socket_locks) { - zmq_socket_mutex.unlock(); - } - return; - } - exaudflib::check::external_process_check(); - } catch (std::exception &err) { - exaudflib::check::external_process_check(); - } catch (...) { - exaudflib::check::external_process_check(); - } - if (use_zmq_socket_locks) { - zmq_socket_mutex.unlock(); - } - ::usleep(100000); - } - if (use_zmq_socket_locks) { - zmq_socket_mutex.unlock(); - } -} - -bool exaudflib::socket_low_level::socket_recv(zmq::socket_t &socket, zmq::message_t &zmsg, bool return_on_error) -{ - DBG_FUNC_BEGIN(std::cerr); - for (;;) { - try { - if (use_zmq_socket_locks) { - zmq_socket_mutex.lock(); - } - if (socket.recv(&zmsg) == true) { -#ifdef LOG_COMMUNICATION - stringstream sb; - uint32_t len = zmsg.size(); - sb << "/tmp/zmqcomm_log_" << ::getpid() << "_recv.data"; - int fd = ::open(sb.str().c_str(), O_CREAT | O_APPEND | O_WRONLY, 00644); - if (fd >= 0) { - if (::write(fd, &len, sizeof(uint32_t)) == -1) {perror("Log communication");} - if (::write(fd, zmsg.data(), len) == -1) {perror("Log communication");} - ::close(fd); - } -#endif - if (use_zmq_socket_locks) { - zmq_socket_mutex.unlock(); - } - return true; - } - exaudflib::check::external_process_check(); - } catch (std::exception &err) { - exaudflib::check::external_process_check(); - - } catch (...) { - exaudflib::check::external_process_check(); - } - if (use_zmq_socket_locks) { - zmq_socket_mutex.unlock(); - } - if (return_on_error) return false; - ::usleep(100000); - } - if (use_zmq_socket_locks) { - zmq_socket_mutex.unlock(); - } - return false; -} \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/impl/socket_low_level.h b/exaudfclient/base/exaudflib/impl/socket_low_level.h deleted file mode 100644 index 1e4bd45..0000000 --- a/exaudfclient/base/exaudflib/impl/socket_low_level.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef EXAUDFCLIENT_EXAUDFLIB_SOCKET_LOW_LEVEL_H -#define EXAUDFCLIENT_EXAUDFLIB_SOCKET_LOW_LEVEL_H - -#include - -namespace exaudflib { - namespace socket_low_level { - void init(bool use_zmq_socket_locks); - void socket_send(zmq::socket_t &socket, zmq::message_t &zmsg); - bool socket_recv(zmq::socket_t &socket, zmq::message_t &zmsg, bool return_on_error=false); - } -} - - -#endif //EXAUDFCLIENT_EXAUDFLIB_SOCKET_LOW_LEVEL_H diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_factory.cc b/exaudfclient/base/exaudflib/impl/swig/swig_factory.cc deleted file mode 100644 index 9de8401..0000000 --- a/exaudfclient/base/exaudflib/impl/swig/swig_factory.cc +++ /dev/null @@ -1,19 +0,0 @@ -#include "exaudflib/impl/swig/swig_meta_data.h" -#include "exaudflib/impl/swig/swig_table_iterator.h" -#include "exaudflib/impl/swig/swig_result_handler.h" - -extern "C" { - -SWIGVMContainers::SWIGMetadataIf* create_SWIGMetaData() { - return new SWIGVMContainers::SWIGMetadata_Impl(); -} - -SWIGVMContainers::AbstractSWIGTableIterator* create_SWIGTableIterator() { - return new SWIGVMContainers::SWIGTableIterator_Impl(); -} - -SWIGVMContainers::SWIGRAbstractResultHandler* create_SWIGResultHandler(SWIGVMContainers::SWIGTableIterator* table_iterator) { - return new SWIGVMContainers::SWIGResultHandler_Impl(table_iterator); -} - -} //extern "C" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.cc b/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.cc deleted file mode 100644 index ec1ffcf..0000000 --- a/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.cc +++ /dev/null @@ -1,3 +0,0 @@ -//Check that swig_generial_iterator.h is self-contained -#include "exaudflib/impl/swig/swig_general_iterator.h" - diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.h b/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.h deleted file mode 100644 index 781274c..0000000 --- a/exaudfclient/base/exaudflib/impl/swig/swig_general_iterator.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef EXAUDFCLIENT_SWIGGENERALITERATOR_H -#define EXAUDFCLIENT_SWIGGENERALITERATOR_H - -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/impl/global.h" - -namespace SWIGVMContainers { - -class SWIGGeneralIterator { -protected: - SWIGVMExceptionHandler *m_exch; -public: - // SWIGGeneralIterator(SWIGVMExceptionHandler *exch): m_exch(exch) { } - SWIGGeneralIterator() - : m_exch(&exaudflib::global.exchandler) - {} - virtual ~SWIGGeneralIterator() { } - inline const char* checkException() { - if (m_exch->exthrowed) { - m_exch->exthrowed = false; - return m_exch->exmsg.c_str(); - } else return NULL; - } -}; - -} //namespace SWIGVMContainers - -#endif //EXAUDFCLIENT_SWIGGENERALITERATOR_H diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.cc b/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.cc deleted file mode 100644 index 0a100b7..0000000 --- a/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.cc +++ /dev/null @@ -1,3 +0,0 @@ -//Check that swig_meta_data.h is self-contained -#include "exaudflib/impl/swig/swig_meta_data.h" - diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.h b/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.h deleted file mode 100644 index bb0f821..0000000 --- a/exaudfclient/base/exaudflib/impl/swig/swig_meta_data.h +++ /dev/null @@ -1,240 +0,0 @@ -#ifndef EXAUDFCLIENT_SWIGMETADATA_H -#define EXAUDFCLIENT_SWIGMETADATA_H - -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/swig/swig_meta_data.h" -#include "exaudflib/zmqcontainer.pb.h" -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/impl/msg_conversion.h" -#include "exaudflib/impl/global.h" -#include -#include -#include - -namespace SWIGVMContainers { - -class SWIGMetadata_Impl : public SWIGMetadataIf { -public: - SWIGMetadata_Impl(): - m_connection_id(exaudflib::global.SWIGVM_params_ref->connection_id), - m_socket(*(exaudflib::global.sock)), - m_exch(&exaudflib::global.exchandler), - m_db_name(exaudflib::global.SWIGVM_params_ref->dbname), - m_db_version(exaudflib::global.SWIGVM_params_ref->dbversion), - m_script_name(exaudflib::global.SWIGVM_params_ref->script_name), - m_script_schema(exaudflib::global.SWIGVM_params_ref->script_schema), - m_current_user(exaudflib::global.SWIGVM_params_ref->current_user), - m_current_schema(exaudflib::global.SWIGVM_params_ref->current_schema), - m_scope_user(exaudflib::global.SWIGVM_params_ref->scope_user), - m_script_code(exaudflib::global.SWIGVM_params_ref->script_code), - m_session_id(exaudflib::global.SWIGVM_params_ref->session_id), - m_statement_id(exaudflib::global.SWIGVM_params_ref->statement_id), - m_node_count(exaudflib::global.SWIGVM_params_ref->node_count), - m_node_id(exaudflib::global.SWIGVM_params_ref->node_id), - m_vm_id(exaudflib::global.SWIGVM_params_ref->vm_id), - m_input_names(*(exaudflib::global.SWIGVM_params_ref->inp_names)), - m_input_types(*(exaudflib::global.SWIGVM_params_ref->inp_types)), - m_input_iter_type(exaudflib::global.SWIGVM_params_ref->inp_iter_type), - m_output_names(*(exaudflib::global.SWIGVM_params_ref->out_names)), - m_output_types(*(exaudflib::global.SWIGVM_params_ref->out_types)), - m_output_iter_type(exaudflib::global.SWIGVM_params_ref->out_iter_type), - m_memory_limit(exaudflib::global.SWIGVM_params_ref->maximal_memory_limit), - m_vm_type(exaudflib::global.SWIGVM_params_ref->vm_type), - m_is_emitted(*(exaudflib::global.SWIGVM_params_ref->is_emitted)), - m_pluginLanguageName(exaudflib::global.SWIGVM_params_ref->pluginName), - m_pluginURI(exaudflib::global.SWIGVM_params_ref->pluginURI), - m_outputAddress(exaudflib::global.SWIGVM_params_ref->outputAddress) - { - { std::stringstream sb; sb << m_session_id; m_session_id_s = sb.str(); } - { std::stringstream sb; sb << m_vm_id; m_vm_id_s = sb.str(); } - } - - virtual ~SWIGMetadata_Impl() { } - inline const char* databaseName() { return m_db_name.c_str(); } - inline const char* databaseVersion() { return m_db_version.c_str(); } - inline const char* scriptName() { return m_script_name.c_str(); } - inline const char* scriptSchema() { return m_script_schema.c_str(); } - inline const char* currentUser() { return m_current_user.c_str(); } - inline const char* scopeUser() { return m_scope_user.c_str(); } - inline const char* currentSchema() {return m_current_schema.c_str();} - inline const char* scriptCode() { return m_script_code.c_str(); } - inline const unsigned long long sessionID() { return m_session_id; } - inline const char *sessionID_S() { return m_session_id_s.c_str(); } - inline const unsigned long statementID() { return m_statement_id; } - inline const unsigned int nodeCount() { return m_node_count; } - inline const unsigned int nodeID() { return m_node_id; } - inline const unsigned long long vmID() { return m_vm_id; } - inline const unsigned long long memoryLimit() { return m_memory_limit; } - inline const VMTYPE vmType() { return m_vm_type; } - inline const char *vmID_S() { return m_vm_id_s.c_str(); } - inline const ExecutionGraph::ConnectionInformationWrapper* connectionInformation(const char* connection_name) - { - exascript_request request; - request.set_type(MT_IMPORT); - request.set_connection_id(m_connection_id); - exascript_import_req *req = request.mutable_import(); - req->set_script_name(connection_name); - req->set_kind(PB_IMPORT_CONNECTION_INFORMATION); - if (!request.SerializeToString(&m_output_buffer)) { - m_exch->setException("F-UDF-CL-LIB-1048: Communication error: failed to serialize data"); - return new ExecutionGraph::ConnectionInformationWrapper(ExecutionGraph::ConnectionInformation()); - } - zmq::message_t zmsg_req((void*)m_output_buffer.c_str(), m_output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(m_socket, zmsg_req); - zmq::message_t zmsg_rep; - exaudflib::socket_low_level::socket_recv(m_socket, zmsg_rep); - exascript_response response; - if (!response.ParseFromArray(zmsg_rep.data(), zmsg_rep.size())) { - m_exch->setException("F-UDF-CL-LIB-1049: Communication error: failed to parse data"); - return new ExecutionGraph::ConnectionInformationWrapper(ExecutionGraph::ConnectionInformation()); - } - if (response.type() != MT_IMPORT) { - m_exch->setException("F-UDF-CL-LIB-1050: Internal error: wrong message type, got "+ - exaudflib::msg_conversion::convert_message_type_to_string(response.type())); - return new ExecutionGraph::ConnectionInformationWrapper(ExecutionGraph::ConnectionInformation()); - } - const exascript_import_rep &rep = response.import(); - if (rep.has_exception_message()) { - m_exch->setException(rep.exception_message().c_str()); - return new ExecutionGraph::ConnectionInformationWrapper(ExecutionGraph::ConnectionInformation()); - } - if (!rep.has_connection_information()) { - m_exch->setException("F-UDF-CL-LIB-1051: Internal error: No connection information returned"); - return new ExecutionGraph::ConnectionInformationWrapper(ExecutionGraph::ConnectionInformation()); - } - connection_information_rep ci = rep.connection_information(); - return new ExecutionGraph::ConnectionInformationWrapper( - ExecutionGraph::ConnectionInformation(ci.kind(), ci.address(), ci.user(), ci.password())); - } - inline const char* moduleContent(const char* name) { - exascript_request request; - request.set_type(MT_IMPORT); - request.set_connection_id(m_connection_id); - exascript_import_req *req = request.mutable_import(); - req->set_script_name(name); - req->set_kind(PB_IMPORT_SCRIPT_CODE); - if (!request.SerializeToString(&m_output_buffer)) { - m_exch->setException("F-UDF-CL-LIB-1052: Communication error: failed to serialize data"); - return NULL; - } - zmq::message_t zmsg_req((void*)m_output_buffer.c_str(), m_output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(m_socket, zmsg_req); - zmq::message_t zmsg_rep; - exaudflib::socket_low_level::socket_recv(m_socket, zmsg_rep); - exascript_response response; - if (!response.ParseFromArray(zmsg_rep.data(), zmsg_rep.size())) { - m_exch->setException("F-UDF-CL-LIB-1053: Communication error: failed to parse data"); - return NULL; - } - if (response.type() != MT_IMPORT) { - m_exch->setException("F-UDF-CL-LIB-1054: Internal error: wrong message type, should MT_IMPORT, got "+ - exaudflib::msg_conversion::convert_message_type_to_string(response.type())); - return NULL; - } - const exascript_import_rep &rep = response.import(); - if (rep.has_exception_message()) { - m_exch->setException(rep.exception_message().c_str()); - return NULL; - } - if (!rep.has_source_code()) { - m_exch->setException("F-UDF-CL-LIB-1055: Internal error: No source code returned"); - return NULL; - } - m_temp_code = rep.source_code(); - return m_temp_code.c_str(); - } - inline const unsigned int inputColumnCount() { return m_input_names.size(); } - inline const char *inputColumnName(unsigned int col) - { return col >= m_input_names.size() ? NULL : m_input_names[col].c_str(); } - inline const SWIGVM_datatype_e inputColumnType(unsigned int col) - { return col >= m_input_types.size() ? UNSUPPORTED : m_input_types[col].type; } - inline const char *inputColumnTypeName(unsigned int col) - { return col >= m_input_types.size() ? NULL : m_input_types[col].type_name.c_str(); } - inline const unsigned int inputColumnSize(unsigned int col) - { return col >= m_input_types.size() ? 0 : m_input_types[col].len; } - inline const unsigned int inputColumnPrecision(unsigned int col) - { return col >= m_input_types.size() ? 0 : m_input_types[col].prec; } - inline const unsigned int inputColumnScale(unsigned int col) - { return col >= m_input_types.size() ? 0 : m_input_types[col].scale; } - inline const SWIGVM_itertype_e inputType() { return m_input_iter_type; } - inline const unsigned int outputColumnCount() { return m_output_names.size(); } - inline const char *outputColumnName(unsigned int col) { - if (m_output_iter_type == EXACTLY_ONCE && col == 0) - return "RETURN"; - return col >= m_output_names.size() ? NULL : m_output_names[col].c_str(); - } - inline const SWIGVM_datatype_e outputColumnType(unsigned int col) - { return col >= m_output_types.size() ? UNSUPPORTED : m_output_types[col].type; } - inline const char *outputColumnTypeName(unsigned int col) - { return col >= m_output_types.size() ? NULL : m_output_types[col].type_name.c_str(); } - inline const unsigned int outputColumnSize(unsigned int col) - { return col >= m_output_types.size() ? 0 : m_output_types[col].len; } - inline const unsigned int outputColumnPrecision(unsigned int col) - { return col >= m_output_types.size() ? 0 : m_output_types[col].prec; } - inline const unsigned int outputColumnScale(unsigned int col) - { return col >= m_output_types.size() ? 0 : m_output_types[col].scale; } - inline const SWIGVM_itertype_e outputType() { return m_output_iter_type; } - inline const bool isEmittedColumn(unsigned int col){ - if (col >= m_is_emitted.size()) - { - abort(); - } - return m_is_emitted[col]; - } - inline const char* checkException() { - if (m_exch->exthrowed) { - m_exch->exthrowed = false; - return m_exch->exmsg.c_str(); - } else return NULL; - } - inline const char* pluginLanguageName() - { - return m_pluginLanguageName.c_str(); - } - inline const char* pluginURI() - { - return m_pluginURI.c_str(); - } - inline const char* outputAddress() - { - return m_outputAddress.c_str(); - } -private: - const uint64_t m_connection_id; - zmq::socket_t &m_socket; - std::string m_output_buffer; - SWIGVMExceptionHandler *m_exch; - const std::string m_db_name; - const std::string m_db_version; - const std::string m_script_name; - const std::string m_script_schema; - const std::string m_current_user; - const std::string m_current_schema; - const std::string m_scope_user; - const std::string m_script_code; - const unsigned long m_session_id; - const unsigned long m_statement_id; - const unsigned int m_node_count; - const unsigned int m_node_id; - const unsigned long m_vm_id; - std::string m_temp_code; - const std::vector &m_input_names; - const std::vector &m_input_types; - const SWIGVM_itertype_e m_input_iter_type; - const std::vector &m_output_names; - const std::vector &m_output_types; - const SWIGVM_itertype_e m_output_iter_type; - const unsigned long long m_memory_limit; - const std::string m_meta_info; - const VMTYPE m_vm_type; - std::string m_session_id_s; - std::string m_vm_id_s; - const std::vector &m_is_emitted; - std::string m_pluginLanguageName; - std::string m_pluginURI; - std::string m_outputAddress; -}; - -} //namespace SWIGVMContainers - -#endif //EXAUDFCLIENT_SWIGMETADATA_H diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.cc b/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.cc deleted file mode 100644 index 987b5e3..0000000 --- a/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.cc +++ /dev/null @@ -1,3 +0,0 @@ -//Check that swig_result_handler.h is self-contained -#include "exaudflib/impl/swig/swig_result_handler.h" - diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.h b/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.h deleted file mode 100644 index cb143db..0000000 --- a/exaudfclient/base/exaudflib/impl/swig/swig_result_handler.h +++ /dev/null @@ -1,282 +0,0 @@ -#ifndef EXAUDFCLIENT_SWIGRESULTHANDLER_H -#define EXAUDFCLIENT_SWIGRESULTHANDLER_H - -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/swig/swig_result_handler.h" -#include "exaudflib/impl/swig/swig_general_iterator.h" -#include "exaudflib/impl/global.h" -#include "exaudflib/impl/msg_conversion.h" -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/zmqcontainer.pb.h" - -namespace SWIGVMContainers { - -class SWIGResultHandler_Impl : public SWIGRAbstractResultHandler, SWIGGeneralIterator { -private: - SWIGTableIterator* m_table_iterator; - const uint64_t m_connection_id; - zmq::socket_t &m_socket; - std::string m_output_buffer; - uint64_t m_message_size; - struct rowdata_t { - std::map null_data; - std::map bool_data; - std::map double_data; - std::map int32_data; - std::map int64_data; - std::map string_data; - }; - rowdata_t m_rowdata; - exascript_request m_emit_request; - uint64_t m_rows_emited; - const std::vector &m_types; -public: - const char* checkException() {return SWIGGeneralIterator::checkException();} - SWIGResultHandler_Impl(SWIGTableIterator* table_iterator) - : m_table_iterator(table_iterator), - m_connection_id(exaudflib::global.SWIGVM_params_ref->connection_id), - m_socket(*(exaudflib::global.sock)), - m_message_size(0), - m_rows_emited(1), - m_types(*(exaudflib::global.SWIGVM_params_ref->out_types)) - {} - - ~SWIGResultHandler_Impl() {} - inline void reinitialize() { - m_rows_emited = 0; - m_message_size = 0; - m_emit_request.Clear(); - m_rowdata = rowdata_t(); - } - inline unsigned long rowsEmited() { - return m_rows_emited; - } - inline void flush() { - exascript_emit_data_req *req = m_emit_request.mutable_emit(); - exascript_table_data *table = req->mutable_table(); - if (table->has_rows() && table->rows() > 0) { - { m_emit_request.set_type(MT_EMIT); - m_emit_request.set_connection_id(m_connection_id); - if (!m_emit_request.SerializeToString(&m_output_buffer)) { - m_exch->setException("F-UDF-CL-LIB-1082: Communication error: failed to serialize data"); - return; - } - zmq::message_t zmsg((void*)m_output_buffer.c_str(), m_output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(m_socket, zmsg); - m_emit_request.Clear(); - m_message_size = 0; - } - { zmq::message_t zmsg; - exaudflib::socket_low_level::socket_recv(m_socket, zmsg); - exascript_response response; - if (!response.ParseFromArray(zmsg.data(), zmsg.size())) { - m_exch->setException("F-UDF-CL-LIB-1083: Communication error: failed to parse data"); - return; - } - if (response.connection_id() != m_connection_id) { - std::stringstream sb; - sb << "F-UDF-CL-LIB-1084: Received wrong connection id " << response.connection_id() - << ", should be " << m_connection_id; - m_exch->setException(sb.str().c_str()); - return; - } - if (response.type() == MT_CLOSE) { - if (!response.close().has_exception_message()) - m_exch->setException("F-UDF-CL-LIB-1085: Unknown error occured"); - else - m_exch->setException(response.close().exception_message().c_str()); - return; - } - if (response.type() != MT_EMIT) { - m_exch->setException("F-UDF-CL-LIB-1086: Wrong response type, got "+ - exaudflib::msg_conversion::convert_message_type_to_string(response.type())); - return; - } - } - } - } - inline bool next() { - ++m_rows_emited; - exascript_emit_data_req *req = m_emit_request.mutable_emit(); - exascript_table_data *table = req->mutable_table(); - for (unsigned int col = 0; col < m_types.size(); ++col) { - bool null_data = m_rowdata.null_data[col]; - table->add_data_nulls(null_data); - if (null_data) continue; - switch (m_types[col].type) { - case UNSUPPORTED: - m_exch->setException("F-UDF-CL-LIB-1087: Unsupported data type found"); - return false; - case DOUBLE: - if (m_rowdata.double_data.find(col) == m_rowdata.double_data.end()) { - m_exch->setException("F-UDF-CL-LIB-1088: Not enough double columns emited"); - return false; - } - m_message_size += sizeof(double); - table->add_data_double(m_rowdata.double_data[col]); - break; - case INT32: - if (m_rowdata.int32_data.find(col) == m_rowdata.int32_data.end()) { - m_exch->setException("F-UDF-CL-LIB-1089: Not enough int32 columns emited"); - return false; - } - m_message_size += sizeof(int32_t); - table->add_data_int32(m_rowdata.int32_data[col]); - break; - case INT64: - if (m_rowdata.int64_data.find(col) == m_rowdata.int64_data.end()) { - m_exch->setException("F-UDF-CL-LIB-1090: Not enough int64 columns emited"); - return false; - } - m_message_size += sizeof(int64_t); - table->add_data_int64(m_rowdata.int64_data[col]); - break; - case NUMERIC: - case TIMESTAMP: - case DATE: - case STRING: - if (m_rowdata.string_data.find(col) == m_rowdata.string_data.end()) { - m_exch->setException("F-UDF-CL-LIB-1091: Not enough string columns emited"); - return false; - } - m_message_size += sizeof(int32_t) + m_rowdata.string_data[col].length(); - *table->add_data_string() = m_rowdata.string_data[col]; - break; - case BOOLEAN: - if (m_rowdata.bool_data.find(col) == m_rowdata.bool_data.end()) { - m_exch->setException("F-UDF-CL-LIB-1092: Not enough boolean columns emited"); - return false; - } - m_message_size += 1; - table->add_data_bool(m_rowdata.bool_data[col]); - break; - default: - m_exch->setException("F-UDF-CL-LIB-1093: Unknown data type found, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - return false; - } - } - table->add_row_number(m_table_iterator->get_current_row()); - m_rowdata = rowdata_t(); - if (!table->has_rows()) table->set_rows(1); - else table->set_rows(table->rows() + 1); - table->set_rows_in_group(0); - if (m_message_size >= SWIG_MAX_VAR_DATASIZE) { - if (exaudflib::global.SWIGVM_params_ref->inp_iter_type == EXACTLY_ONCE && exaudflib::global.SWIGVM_params_ref->out_iter_type == EXACTLY_ONCE) - exaudflib::global.SWIGVM_params_ref->inp_force_finish = true; - else this->flush(); - } - return true; - } - inline void setDouble(unsigned int col, const double v) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1094: Output column does "+std::to_string(col)+" not exist"); - return; - } - if (m_types[col].type != DOUBLE) { - m_exch->setException("E-UDF-CL-LIB-1095: Wrong output column type, expected DOUBLE, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - return; - } - m_rowdata.null_data[col] = false; - m_rowdata.double_data[col] = v; - } - inline void setString(unsigned int col, const char *v, size_t l) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1096: Output column does "+std::to_string(col)+" not exist"); - return; - } - if (m_types[col].type != STRING) { - m_exch->setException("E-UDF-CL-LIB-1097: Wrong output column type, expected STRING, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - return; - } - m_rowdata.null_data[col] = false; - m_rowdata.string_data[col] = v; - } - inline void setInt32(unsigned int col, const int32_t v) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1098: Output column does "+std::to_string(col)+" not exist"); - return; - } - if (m_types[col].type != INT32) { - m_exch->setException("E-UDF-CL-LIB-1099: Wrong output column type, expected INT32, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - return; - } - m_rowdata.null_data[col] = false; - m_rowdata.int32_data[col] = v; - } - inline void setInt64(unsigned int col, const int64_t v) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1100: Output column does "+std::to_string(col)+" not exist"); - return; - } - if (m_types[col].type != INT64) { - m_exch->setException("E-UDF-CL-LIB-1101: Wrong output column type, expected INT64, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - return; - } - m_rowdata.null_data[col] = false; - m_rowdata.int64_data[col] = v; - } - inline void setNumeric(unsigned int col, const char *v) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1102: Output column does "+std::to_string(col)+" not exist"); - return; - } - if (m_types[col].type != NUMERIC) { - m_exch->setException("E-UDF-CL-LIB-1103: Wrong output column type, expected NUMERIC, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - return; - } - m_rowdata.null_data[col] = false; - m_rowdata.string_data[col] = v; - } - inline void setTimestamp(unsigned int col, const char *v) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1104: Output column does "+std::to_string(col)+" not exist"); - return; - } - if (m_types[col].type != TIMESTAMP) { - m_exch->setException("E-UDF-CL-LIB-1105: Wrong output column type, expected TIMESTAMP, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - return; - } - m_rowdata.null_data[col] = false; - m_rowdata.string_data[col] = v; - } - inline void setDate(unsigned int col, const char *v) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1106: Output column does "+std::to_string(col)+" not exist"); - return; - } - if (m_types[col].type != DATE) { - m_exch->setException("E-UDF-CL-LIB-1107: Wrong output column type, expected DATE, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - return; - } - m_rowdata.null_data[col] = false; - m_rowdata.string_data[col] = v; - } - inline void setBoolean(unsigned int col, const bool v) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1108: Output column does "+std::to_string(col)+" not exist"); - return; - } - if (m_types[col].type != BOOLEAN) { - m_exch->setException("E-UDF-CL-LIB-1109: Wrong output column type, expected BOOLEAN, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - return; - } - m_rowdata.null_data[col] = false; - m_rowdata.bool_data[col] = v; - } - inline void setNull(unsigned int col) { - m_rowdata.null_data[col] = true; - } -}; - -} //namespace SWIGVMContainers - -#endif //EXAUDFCLIENT_SWIGRESULTHANDLER_H diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.cc b/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.cc deleted file mode 100644 index 853e79e..0000000 --- a/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.cc +++ /dev/null @@ -1,3 +0,0 @@ -//Check that swig_table_iterator.h is self-contained -#include "exaudflib/impl/swig/swig_table_iterator.h" - diff --git a/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.h b/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.h deleted file mode 100644 index bf5f014..0000000 --- a/exaudfclient/base/exaudflib/impl/swig/swig_table_iterator.h +++ /dev/null @@ -1,346 +0,0 @@ -#ifndef EXAUDFCLIENT_SWIGTABLEITERATOR_H -#define EXAUDFCLIENT_SWIGTABLEITERATOR_H - -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/swig/swig_table_iterator.h" -#include "exaudflib/impl/swig/swig_general_iterator.h" -#include "exaudflib/impl/global.h" -#include "exaudflib/impl/socket_low_level.h" -#include "exaudflib/impl/msg_conversion.h" - -namespace SWIGVMContainers { - -class SWIGTableIterator_Impl : public AbstractSWIGTableIterator, SWIGGeneralIterator { -private: - const uint64_t m_connection_id; - zmq::socket_t &m_socket; - std::string m_output_buffer; - exascript_request m_request; - exascript_response m_next_response; - - uint64_t m_rows_received; - struct values_per_row_t { - uint64_t strings, bools, int32s, int64s, doubles; - values_per_row_t(): strings(0), bools(0), int32s(0), int64s(0), doubles(0) {} - void reset() { strings = bools = int32s = int64s = doubles = 0; } - } m_values_per_row; - uint64_t m_column_count; - std::vector m_col_offsets; - uint64_t m_rows_in_group; - uint64_t m_current_row; - uint64_t m_rows_completed; - uint64_t m_rows_group_completed; - bool m_was_null; - const std::vector &m_types; - void increment_col_offsets(bool reset = false) { - m_current_row = m_next_response.next().table().row_number(m_rows_completed); - uint64_t current_column = 0; - ssize_t null_index = 0; - if (reset) m_values_per_row.reset(); - null_index = m_rows_completed * m_column_count; - if (m_next_response.next().table().data_nulls_size() <= (null_index + (ssize_t)m_column_count - 1)) { - std::stringstream sb; - sb << "F-UDF-CL-LIB-1056: Internal error: not enough nulls in packet: wanted index " << (null_index + m_column_count - 1) - << " but have " << m_next_response.next().table().data_nulls_size() - << " elements"; - m_exch->setException(sb.str().c_str()); - return; - } - for (std::vector::const_iterator - it = m_types.begin(); it != m_types.end(); ++it, ++current_column, ++null_index) - { - if (m_next_response.next().table().data_nulls(null_index)) - continue; - switch (it->type) { - case UNSUPPORTED: m_exch->setException("F-UDF-CL-LIB-1057: Unsupported data type found"); return; - case DOUBLE: m_col_offsets[current_column] = m_values_per_row.doubles++; break; - case INT32: m_col_offsets[current_column] = m_values_per_row.int32s++; break; - case INT64: m_col_offsets[current_column] = m_values_per_row.int64s++; break; - case NUMERIC: - case TIMESTAMP: - case DATE: - case STRING: m_col_offsets[current_column] = m_values_per_row.strings++; break; - case BOOLEAN: m_col_offsets[current_column] = m_values_per_row.bools++; break; - default: m_exch->setException("F-UDF-CL-LIB-1058: Unknown data type found, got "+it->type); return; - } - } - } - void receive_next_data(bool reset) { - m_rows_received = 0; - m_rows_completed = 0; - { - m_request.Clear(); - m_request.set_connection_id(m_connection_id); - if (reset) m_request.set_type(MT_RESET); - else m_request.set_type(MT_NEXT); - if(!m_request.SerializeToString(&m_output_buffer)) { - m_exch->setException("F-UDF-CL-LIB-1059: Communication error: failed to serialize data"); - return; - } - zmq::message_t zmsg((void*)m_output_buffer.c_str(), m_output_buffer.length(), NULL, NULL); - exaudflib::socket_low_level::socket_send(m_socket, zmsg); - } { - zmq::message_t zmsg; - exaudflib::socket_low_level::socket_recv(m_socket, zmsg); - m_next_response.Clear(); - if (!m_next_response.ParseFromArray(zmsg.data(), zmsg.size())) { - m_exch->setException("F-UDF-CL-LIB-1060: Communication error: failed to parse data"); - return; - } - if (m_next_response.connection_id() != m_connection_id) { - std::stringstream sb; - sb << "F-UDF-CL-LIB-1061: Communication error: wrong connection id, expected " - << m_connection_id << " got " << m_next_response.connection_id(); - m_exch->setException(sb.str()); - return; - } - if (m_next_response.type() == MT_DONE) { - return; - } - if (m_next_response.type() == MT_CLOSE) { - const exascript_close &rep = m_next_response.close(); - if (!rep.has_exception_message()) { - if (m_rows_completed == 0) { - return; - } else m_exch->setException("F-UDF-CL-LIB-1062: Unknown error occured"); - } else { - m_exch->setException(rep.exception_message().c_str()); - } - return; - } - if ((reset && (m_next_response.type() != MT_RESET)) || - (!reset && (m_next_response.type() != MT_NEXT))) - { - m_exch->setException("F-UDF-CL-LIB-1063: Communication error: wrong message type, got "+ - exaudflib::msg_conversion::convert_message_type_to_string(m_next_response.type())); - return; - } - m_rows_received = m_next_response.next().table().rows(); - m_rows_in_group = m_next_response.next().table().rows_in_group(); - } - increment_col_offsets(true); - } - inline ssize_t check_index(ssize_t index, ssize_t available, const char *tp, const char *otype, const char *ts) { - if (available > index) return index; - std::stringstream sb; - sb << "F-UDF-CL-LIB-1064: Internal error: not enough " << tp << otype << ts << " in packet: wanted index " - << index << " but have " << available << " elements (on " - << m_rows_received << '/' << m_rows_completed << " of received/completed rows"; - m_exch->setException(sb.str().c_str()); - m_was_null = true; - return -1; - } - inline ssize_t check_value(unsigned int col, ssize_t available, const char *otype) { - m_was_null = false; - ssize_t index = check_index(m_column_count * m_rows_completed + col, - m_next_response.next().table().data_nulls_size(), - "nulls for ", otype, ""); - if (m_was_null) return -1; - m_was_null = m_next_response.next().table().data_nulls(index); - if (m_was_null) return -1; - index = check_index(m_col_offsets[col], available, "", otype, "s"); - if (m_was_null) return -1; - return index; - } -public: - const char* checkException() {return SWIGGeneralIterator::checkException();} - - SWIGTableIterator_Impl(): - m_connection_id(exaudflib::global.SWIGVM_params_ref->connection_id), - m_socket(*(exaudflib::global.sock)), - m_column_count(exaudflib::global.SWIGVM_params_ref->inp_types->size()), - m_col_offsets(exaudflib::global.SWIGVM_params_ref->inp_types->size()), - m_current_row((uint64_t)-1), - m_rows_completed(0), - m_rows_group_completed(1), - m_was_null(false), - m_types(*(exaudflib::global.SWIGVM_params_ref->inp_types)) - { - receive_next_data(false); - } - ~SWIGTableIterator_Impl() {} - uint64_t get_current_row() - { - return m_current_row; - } - inline void reinitialize() { - m_rows_completed = 0; - m_rows_group_completed = 1; - m_values_per_row.reset(); - receive_next_data(false); - } - inline bool next() { - if (m_rows_received == 0) { - m_exch->setException("E-UDF-CL-LIB-1065: Iteration finished"); - return false; - } - ++m_rows_completed; - ++m_rows_group_completed; - if (exaudflib::global.SWIGVM_params_ref->inp_force_finish) - return false; - if (m_rows_completed >= m_rows_received) { - receive_next_data(false); - if (m_rows_received == 0) - return false; - else return true; - } - increment_col_offsets(); - return true; - } - inline bool eot() { - return m_rows_received == 0; - } - inline void reset() { - m_rows_group_completed = 1; - exaudflib::global.SWIGVM_params_ref->inp_force_finish = false; - receive_next_data(true); - } - inline unsigned long restBufferSize() { - if (m_rows_completed < m_rows_received) - return m_rows_received - m_rows_completed; - return 0; - } - inline unsigned long rowsCompleted() { - return m_rows_group_completed; - } - inline unsigned long rowsInGroup() { - return m_rows_in_group; - } - inline double getDouble(unsigned int col) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1066: Input column "+std::to_string(col)+" does not exist"); - m_was_null = true; - return 0.0; - } - if (m_types[col].type != DOUBLE) { - m_exch->setException("E-UDF-CL-LIB-1067: Wrong input column type, expected DOUBLE, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - m_was_null = true; - return 0.0; - } - ssize_t index = check_value(col, m_next_response.next().table().data_double_size(), "double"); - if (m_was_null) { - return 0.0; - } - return m_next_response.next().table().data_double(index); - } - inline const char *getString(unsigned int col, size_t *length = NULL) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1068: Input column "+std::to_string(col)+" does not exist"); - m_was_null = true; - return ""; - } - if (m_types[col].type != STRING) { - m_exch->setException("E-UDF-CL-LIB-1069: Wrong input column type, expected STRING, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - m_was_null = true; - return ""; - } - ssize_t index = check_value(col, m_next_response.next().table().data_string_size(), "string"); - if (m_was_null) return ""; - const std::string &s(m_next_response.next().table().data_string(index)); - if (length != NULL) *length = s.length(); - return s.c_str(); - } - inline int32_t getInt32(unsigned int col) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1070: Input column "+std::to_string(col)+" does not exist"); - m_was_null = true; - return 0; - } - if (m_types[col].type != INT32) { - m_exch->setException("E-UDF-CL-LIB-1071: Wrong input column type, expected INT32, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - m_was_null = true; - return 0; - } - ssize_t index = check_value(col, m_next_response.next().table().data_int32_size(), "int32"); - if (m_was_null) return 0; - return m_next_response.next().table().data_int32(index); - } - inline int64_t getInt64(unsigned int col) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1072: Input column "+std::to_string(col)+" does not exist"); - m_was_null = true; - return 0LL; - } - if (m_types[col].type != INT64) { - m_exch->setException("E-UDF-CL-LIB-1073: Wrong input column type, expected INT64, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - m_was_null = true; - return 0LL; - } - ssize_t index = check_value(col, m_next_response.next().table().data_int64_size(), "int64"); - if (m_was_null) return 0LL; - return m_next_response.next().table().data_int64(index); - } - inline const char *getNumeric(unsigned int col) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1074: Input column "+std::to_string(col)+" does not exist"); - m_was_null = true; - return ""; - } - if (m_types[col].type != NUMERIC) { - m_exch->setException("E-UDF-CL-LIB-1075: Wrong input column type, expected NUMERIC, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - m_was_null = true; - return ""; - } - ssize_t index = check_value(col, m_next_response.next().table().data_string_size(), "string"); - if (m_was_null) return "0"; - return m_next_response.next().table().data_string(index).c_str(); - } - inline const char *getTimestamp(unsigned int col) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1076: Input column "+std::to_string(col)+" does not exist"); - m_was_null = true; - return ""; - } - if (m_types[col].type != TIMESTAMP) { - m_exch->setException("E-UDF-CL-LIB-1077: Wrong input column type, expected TIMESTAMP, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - m_was_null = true; - return ""; - } - ssize_t index = check_value(col, m_next_response.next().table().data_string_size(), "string"); - if (m_was_null) return "1970-01-01 00:00:00.00 0000"; - return m_next_response.next().table().data_string(index).c_str(); - } - inline const char *getDate(unsigned int col) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1078: Input column "+std::to_string(col)+" does not exist"); - m_was_null = true; - return ""; - } - if (m_types[col].type != DATE) { - m_exch->setException("E-UDF-CL-LIB-1079: Wrong input column type, expected DATE, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - m_was_null = true; - return ""; - } - ssize_t index = check_value(col, m_next_response.next().table().data_string_size(), "string"); - if (m_was_null) return "1970-01-01"; - return m_next_response.next().table().data_string(index).c_str(); - } - inline bool getBoolean(unsigned int col) { - if (col >= m_types.size()) { - m_exch->setException("E-UDF-CL-LIB-1080: Input column "+std::to_string(col)+" does not exist"); - m_was_null = true; - return ""; - } - if (m_types[col].type != BOOLEAN) { - m_exch->setException("E-UDF-CL-LIB-1081: Wrong input column type, expected BOOLEAN, got "+ - exaudflib::msg_conversion::convert_type_to_string(m_types[col].type)); - m_was_null = true; - return ""; - } - ssize_t index = check_value(col, m_next_response.next().table().data_bool_size(), "bool"); - if (m_was_null) return false; - return m_next_response.next().table().data_bool(index); - } - inline bool wasNull() { return m_was_null; } -}; - -} //namespace SWIGVMContainers - -#endif //EXAUDFCLIENT_SWIGTABLEITERATOR_H diff --git a/exaudfclient/base/exaudflib/load_dynamic.h b/exaudfclient/base/exaudflib/load_dynamic.h deleted file mode 100644 index e3dd47c..0000000 --- a/exaudfclient/base/exaudflib/load_dynamic.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef EXAUDFLIB_LOAD_DYNAMIC -#define EXAUDFLIB_LOAD_DYNAMIC - -void set_exaudflib_handle(void* handle); -void* load_dynamic(const char* name); - -#endif //EXAUDFLIB_LOAD_DYNAMIC \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/swig/script_data_transfer_objects.cc b/exaudfclient/base/exaudflib/swig/script_data_transfer_objects.cc deleted file mode 100644 index 78a890f..0000000 --- a/exaudfclient/base/exaudflib/swig/script_data_transfer_objects.cc +++ /dev/null @@ -1,465 +0,0 @@ -//#ifdef BUILDINSWIGDIR -//#include "scriptDTO.h" -//#else -#include "script_data_transfer_objects.h" -//#endif -#include -#include - -namespace ExecutionGraph -{ - -////////////////////////////// -EmptyDTO::EmptyDTO() -{} - -void EmptyDTO::accept(ScriptDTOSerializer& serializer) const -{ - return; -} - -bool EmptyDTO::isEmpty() const -{ - return true; -} - -////////////////////////////// - - -void StringDTO::accept(ScriptDTOSerializer& serializer) const -{ - serializer.visit(*this); -} - -StringDTO::StringDTO() - : arg("") -{} - -StringDTO::StringDTO(const std::string& arg_) - : arg(arg_) -{} - - -const std::string StringDTO::getArg() const -{ - return arg; -} - -bool StringDTO::isEmpty() const -{ - return false; -} - -////////////////////////////// - - - -void ConnectionInformation::accept(ScriptDTOSerializer& serializer) const -{ - serializer.visit(*this); -} - -ConnectionInformation::ConnectionInformation(const std::string& kind_, const std::string& address_, const std::string& user_, const std::string& password_) - : kind(kind_), - address(address_), - user(user_), - password(password_) -{ -} - -ConnectionInformation::ConnectionInformation(const std::string& address_, const std::string& user_, const std::string& password_) - : kind("password"), - address(address_), - user(user_), - password(password_) -{} - - -ConnectionInformation::ConnectionInformation() - : kind(""), address(""), user(""), password("") -{} - -ConnectionInformation::ConnectionInformation(const ConnectionInformation& other) - : kind(other.getKind()), - address(other.getAddress()), - user(other.getUser()), - password(other.getPassword()) -{} - -const std::string ConnectionInformation::getKind() const -{ - return kind; -} - -const std::string ConnectionInformation::getAddress() const -{ - return address; -} - -const std::string ConnectionInformation::getUser() const -{ - return user; -} - -const std::string ConnectionInformation::getPassword() const -{ - return password; -} - -bool ConnectionInformation::hasData() const -{ - return kind.size() == 0; -} - -bool ConnectionInformation::isEmpty() const -{ - return false; -} - -////////////////////////////// - - -//ImportSpecification::ImportSpecificationError::ImportSpecificationError(const std::string& msg_) -// :msg(msg_) -//{} - -//ImportSpecification::ImportSpecificationError::~ImportSpecificationError() throw() {} - - - -//const char* ImportSpecification::ImportSpecificationError::what() const throw() -//{ -// return msg.c_str(); -//} - - - -ImportSpecification::ImportSpecification() - : isEmptySpec(true) -{} - -ImportSpecification::ImportSpecification(bool isSubselect__) - : isEmptySpec(false), - isSubselect_(isSubselect__), - subselect_column_names(), - subselect_column_types(), - connection_name(""), - connection_information(), - parameters() -{ -} - - -void ImportSpecification::accept(ScriptDTOSerializer& serializer) const -{ - serializer.visit(*this); -} - -bool ImportSpecification::isEmpty() const -{ - return isEmptySpec; -} - - -bool ImportSpecification::isSubselect() const -{ - return isSubselect_; -} -bool ImportSpecification::hasSubselectColumnNames() const -{ - return subselect_column_names.size()>0; -} -bool ImportSpecification::hasSubselectColumnTypes() const -{ - return subselect_column_types.size()>0; -} -bool ImportSpecification::hasSubselectColumnSpecification() const -{ - return hasSubselectColumnNames() || hasSubselectColumnTypes(); -} -bool ImportSpecification::hasConnectionName() const -{ - return connection_name.size()>0; -} -bool ImportSpecification::hasConnectionInformation() const -{ - return connection_information.hasData() == false; -} -bool ImportSpecification::hasParameters() const -{ - return parameters.size()>0; -} -bool ImportSpecification::hasConsistentColumns() const -{ - return (isSubselect() && subselect_column_names.size() == subselect_column_types.size()) || (!isSubselect() && subselect_column_types.size() == 0); -} - -bool ImportSpecification::isCompleteImportSubselectSpecification() const -{ - return hasConsistentColumns() && hasSubselectColumnNames() && hasSubselectColumnTypes(); -} - -bool ImportSpecification::isCompleteImportIntoTargetTableSpecification() const -{ - return hasConsistentColumns(); -} - - - -const std::vector& ImportSpecification::getSubselectColumnNames() const -{ - if (!isSubselect()) - { - throw Error("import specification error: cannot get column names of non-subselect import specification"); - } - return subselect_column_names; -} - -const std::vector& ImportSpecification::getSubselectColumnTypes() const -{ - if (!isSubselect()) - { - throw Error("import specification error: cannot get column types of non-subselect import specification"); - } - - return subselect_column_types; -} - - -void ImportSpecification::appendSubselectColumnName(const std::string& columnName) -{ - if (!isSubselect()) - { - throw Error("import specification error: cannot add column name to non-subselect import specification"); - } - subselect_column_names.push_back(columnName); -} - -void ImportSpecification::appendSubselectColumnType(const std::string& columnType) -{ - if (!isSubselect()) - { - throw Error("import specification error: cannot add column type to non-subselect import specification"); - } - subselect_column_types.push_back(columnType); -} - -void ImportSpecification::setConnectionName(const std::string& connectionName_) -{ - if (hasConnectionName()) - { - throw Error("import specification error: connection name is set more than once"); - } - if (hasConnectionInformation()) - { - throw Error("import specification error: cannot set connection name, because there is already connection information set"); - } - connection_name = connectionName_; -} - -void ImportSpecification::setConnectionInformation(const ConnectionInformation& connectionInformation_) -{ - if (hasConnectionName()) - { - throw Error("import specification error: cannot set connection information, because there is already a connection name set"); - } - if (hasConnectionInformation()) - { - throw Error("import specification error: cannot set connection information more than once"); - } - connection_information = connectionInformation_; -} - - - -void ImportSpecification::addParameter(const std::string& key, const std::string& value) -{ - if (parameters.find(key) != parameters.end()) - { - std::stringstream sb; - sb << "import specification error: parameter with name '" << key << "', is set more than once"; - throw Error(sb.str()); - } - parameters[key] = value; -} - - -const std::string ImportSpecification::getConnectionName() const -{ - if (!hasConnectionName()) - { - throw Error("import specification error: cannot get connection name because it is not set"); - } - return connection_name; -} - -const ConnectionInformation ImportSpecification::getConnectionInformation() const -{ - if (!hasConnectionInformation()) - { - throw Error("import specification error: cannot get connection information because it is not set"); - } - return connection_information; -} - -const std::map& ImportSpecification::getParameters() const -{ - return parameters; -} - -////////////////////////////// - -ExportSpecification::ExportSpecification() - : isEmptySpec(true), - connection_name(""), - connection_information(), - parameters(), - truncate(false), - replace(false), - created_by(""), - source_column_names() -{ -} - -void ExportSpecification::accept(ScriptDTOSerializer& serializer) const -{ - serializer.visit(*this); -} - -bool ExportSpecification::isEmpty() const -{ - return isEmptySpec; -} - -bool ExportSpecification::hasConnectionName() const -{ - return !connection_name.empty(); -} - -bool ExportSpecification::hasConnectionInformation() const -{ - return !connection_information.hasData(); -} - -bool ExportSpecification::hasParameters() const -{ - return !parameters.empty(); -} - -bool ExportSpecification::hasTruncate() const -{ - return truncate; -} - -bool ExportSpecification::hasReplace() const -{ - return replace; -} - -bool ExportSpecification::hasCreatedBy() const -{ - return !created_by.empty(); -} - -void ExportSpecification::setConnectionName(const std::string& connectionName_) -{ - if (hasConnectionName()) - { - throw Error("export specification error: connection name is set more than once"); - } - if (hasConnectionInformation()) - { - throw Error("export specification error: cannot set connection name, because there is already connection information set"); - } - isEmptySpec = false; - connection_name = connectionName_; -} - -void ExportSpecification::setConnectionInformation(const ConnectionInformation& connectionInformation_) -{ - if (hasConnectionName()) - { - throw Error("export specification error: cannot set connection information, because there is already a connection name set"); - } - if (hasConnectionInformation()) - { - throw Error("export specification error: cannot set connection information more than once"); - } - isEmptySpec = false; - connection_information = connectionInformation_; -} - -void ExportSpecification::addParameter(const std::string& key, const std::string& value) -{ - if (parameters.find(key) != parameters.end()) - { - std::stringstream sb; - sb << "export specification error: parameter with name '" << key << "', is set more than once"; - throw Error(sb.str()); - } - isEmptySpec = false; - parameters[key] = value; -} - -void ExportSpecification::setTruncate(const bool truncate_) -{ - isEmptySpec = false; - truncate = truncate_; -} - -void ExportSpecification::setReplace(const bool replace_) -{ - isEmptySpec = false; - replace = replace_; -} - -void ExportSpecification::setCreatedBy(const std::string& createdBy_) -{ - if (!created_by.empty()) - { - throw Error("export specification error: cannot set created by more than once"); - } - isEmptySpec = false; - created_by = createdBy_; -} - -const std::string ExportSpecification::getConnectionName() const -{ - if (!hasConnectionName()) - { - throw Error("export specification error: cannot get connection name because it is not set"); - } - return connection_name; -} - -const ConnectionInformation ExportSpecification::getConnectionInformation() const -{ - if (!hasConnectionInformation()) - { - throw Error("export specification error: cannot get connection information because it is not set"); - } - return connection_information; -} - -const std::map& ExportSpecification::getParameters() const -{ - return parameters; -} - -const std::string ExportSpecification::getCreatedBy() const -{ - return created_by; -} - -void ExportSpecification::addSourceColumnName(const std::string& sourceColumnName_) -{ - isEmptySpec = false; - source_column_names.push_back(sourceColumnName_); -} - -const std::vector& ExportSpecification::getSourceColumnNames() const -{ - return source_column_names; -} - -} diff --git a/exaudfclient/base/exaudflib/swig/script_data_transfer_objects.h b/exaudfclient/base/exaudflib/swig/script_data_transfer_objects.h deleted file mode 100644 index 438acf0..0000000 --- a/exaudfclient/base/exaudflib/swig/script_data_transfer_objects.h +++ /dev/null @@ -1,242 +0,0 @@ -#ifndef SCRIPT_DATA_TRANSFER_OBJECTS_H -#define SCRIPT_DATA_TRANSFER_OBJECTS_H - -#include -#include -#include -#include -#include -#include - -namespace ExecutionGraph -{ - -class DTOError : public std::exception -{ -public: - explicit DTOError(const std::string& msg_) - : msg(msg_) - {} - virtual ~DTOError() throw() {} - virtual const char* what() const throw() - { - return msg.c_str(); - } - -protected: - std::string msg; -}; - - -class ScriptDTOSerializer -{ -public: - virtual void visit(const class ScriptDTO&) {std::abort();} - virtual void visit(const class ImportSpecification&) = 0; - virtual void visit(const class ExportSpecification&) = 0; - virtual void visit(const class ConnectionInformation&) = 0; - virtual void visit(const class StringDTO&) {std::abort();} // will be refactored. Only supported for Proto Serializer -}; - - -//! -//! -//! -//! - -class ScriptDTO -{ -public: - virtual void accept(ScriptDTOSerializer& serializer) const = 0; - virtual bool isEmpty() const = 0; -}; - -//! -//! -//! -//! - -class EmptyDTO : public ScriptDTO -{ -public: - EmptyDTO(); - - virtual void accept(ScriptDTOSerializer& serializer) const; - virtual bool isEmpty() const; -}; - - -//! -//! -//! -//! - -class ConnectionInformation : public ScriptDTO -{ -public: - class Error : public DTOError - { - public: - explicit Error(const std::string& msg) - : DTOError(msg) - {} - }; - - ConnectionInformation(); - - ConnectionInformation(const ConnectionInformation& other); - - ConnectionInformation(const std::string& kind, - const std::string& address, - const std::string& user, - const std::string& password); - - ConnectionInformation(const std::string& address, - const std::string& user, - const std::string& password); - - virtual void accept(ScriptDTOSerializer& serializer) const; - virtual bool isEmpty() const; - - const std::string getKind() const; - const std::string getAddress() const; - const std::string getUser() const; - const std::string getPassword() const; - - bool hasData() const; - -protected: - std::string kind; - std::string address; - std::string user; - std::string password; - -}; - - -class StringDTO : public ScriptDTO -{ -public: - StringDTO(); - StringDTO(const std::string& arg); - - virtual void accept(ScriptDTOSerializer& serializer) const; - virtual bool isEmpty() const; - - const std::string getArg() const; - -protected: - std::string arg; -}; - - -//! -//! -//! -//! -class ImportSpecification : public ScriptDTO -{ -private: - bool isEmptySpec; -public: - class Error : public DTOError - { - public: - explicit Error(const std::string& msg) - : DTOError(msg) - {} - }; - - virtual void accept(ScriptDTOSerializer& serializer) const; - virtual bool isEmpty() const; - - explicit ImportSpecification(); - explicit ImportSpecification(bool isSubselect__); - - void appendSubselectColumnName(const std::string& columnName); - void appendSubselectColumnType(const std::string& columnType); - void setConnectionName(const std::string& connectionName_); - void setConnectionInformation(const ConnectionInformation& connectionInformation_); - void addParameter(const std::string& key, const std::string& value); - - bool isSubselect() const; - bool hasSubselectColumnNames() const; - bool hasSubselectColumnTypes() const; - bool hasSubselectColumnSpecification() const; - bool hasConnectionName() const; - bool hasConnectionInformation() const; - bool hasParameters() const; - bool hasConsistentColumns() const; - bool isCompleteImportSubselectSpecification() const; - bool isCompleteImportIntoTargetTableSpecification() const; - - const std::vector& getSubselectColumnNames() const; - const std::vector& getSubselectColumnTypes() const; - const std::string getConnectionName() const; - const ConnectionInformation getConnectionInformation() const; - const std::map& getParameters() const; - -protected: - bool isSubselect_; - std::vector subselect_column_names; - std::vector subselect_column_types; - std::string connection_name; - ConnectionInformation connection_information; - std::map parameters; -}; - - -class ExportSpecification : public ScriptDTO -{ -public: - class Error : public DTOError - { - public: - explicit Error(const std::string& msg) - : DTOError(msg) - {} - }; - - virtual void accept(ScriptDTOSerializer& serializer) const; - virtual bool isEmpty() const; - - explicit ExportSpecification(); - - void setConnectionName(const std::string& connectionName_); - void setConnectionInformation(const ConnectionInformation& connectionInformation_); - void addParameter(const std::string& key, const std::string& value); - void setTruncate(const bool truncate_); - void setReplace(const bool replace_); - void setCreatedBy(const std::string& createdBy_); - void addSourceColumnName(const std::string& sourceColumnName_); - - bool hasConnectionName() const; - bool hasConnectionInformation() const; - bool hasParameters() const; - bool hasTruncate() const; - bool hasReplace() const; - bool hasCreatedBy() const; - - const std::string getConnectionName() const; - const ConnectionInformation getConnectionInformation() const; - const std::map& getParameters() const; - const std::string getCreatedBy() const; - const std::vector& getSourceColumnNames() const; - -private: - bool isEmptySpec; - -protected: - std::string connection_name; - ConnectionInformation connection_information; - std::map parameters; - bool truncate; - bool replace; - std::string created_by; - std::vector source_column_names; -}; - - -} - -#endif // SCRIPT_DATA_TRANSFER_OBJECTS_H diff --git a/exaudfclient/base/exaudflib/swig/script_data_transfer_objects_wrapper.cc b/exaudfclient/base/exaudflib/swig/script_data_transfer_objects_wrapper.cc deleted file mode 100644 index ff80750..0000000 --- a/exaudfclient/base/exaudflib/swig/script_data_transfer_objects_wrapper.cc +++ /dev/null @@ -1,103 +0,0 @@ -//#ifdef BUILDINSWIGDIR -//#include "scriptDTOWrapper.h" -//#else -#include "script_data_transfer_objects_wrapper.h" -//#endif - -#include - -namespace ExecutionGraph -{ - -ConnectionInformationWrapper::ConnectionInformationWrapper(const ConnectionInformation& connectionInformation_) - : connectionInformation(connectionInformation_) -{} - -char* ConnectionInformationWrapper::copyKind() const { - return strdup(connectionInformation.getKind().c_str()); -} -char* ConnectionInformationWrapper::copyAddress() const {return strdup(connectionInformation.getAddress().c_str());} -char* ConnectionInformationWrapper::copyUser() const {return strdup(connectionInformation.getUser().c_str());} -char* ConnectionInformationWrapper::copyPassword() const {return strdup(connectionInformation.getPassword().c_str());} - - -ImportSpecificationWrapper::ImportSpecificationWrapper(ImportSpecification* importSpecification_) - : importSpecification(importSpecification_) -{ - if (importSpecification != NULL) - { - // extract all the keys and values in some random but fixed order - for (std::map::const_iterator i = importSpecification->getParameters().begin(); - i != importSpecification->getParameters().end(); - ++i) - { - keys.push_back(i->first); - values.push_back(i->second); - } - } -} -bool ImportSpecificationWrapper::isSubselect() const {return importSpecification->isSubselect();} -size_t ImportSpecificationWrapper::numSubselectColumns() const {return importSpecification->getSubselectColumnNames().size();} -char* ImportSpecificationWrapper::copySubselectColumnName(size_t i) const {return ::strdup(importSpecification->getSubselectColumnNames().at(i).c_str());} -char* ImportSpecificationWrapper::copySubselectColumnType(size_t i) const {return ::strdup(importSpecification->getSubselectColumnTypes().at(i).c_str());} -bool ImportSpecificationWrapper::hasConnectionName() const {return importSpecification->hasConnectionName();} -bool ImportSpecificationWrapper::hasConnectionInformation() const {return importSpecification->hasConnectionInformation();} -char* ImportSpecificationWrapper::copyConnectionName() const {return ::strdup(importSpecification->getConnectionName().c_str());} -const ConnectionInformationWrapper ImportSpecificationWrapper::getConnectionInformation() const { - return ConnectionInformationWrapper(importSpecification->getConnectionInformation()); -} - -size_t ImportSpecificationWrapper::getNumberOfParameters() const {return keys.size();} -char* ImportSpecificationWrapper::copyKey(size_t pos) const {return ::strdup(keys.at(pos).c_str());} -char* ImportSpecificationWrapper::copyValue(size_t pos) const {return ::strdup(values.at(pos).c_str());} - - -ExportSpecificationWrapper::ExportSpecificationWrapper(ExportSpecification* exportSpecification_) - : exportSpecification(exportSpecification_) -{ - if (exportSpecification != NULL) - { - // extract all the keys and values in some random but fixed order - for (std::map::const_iterator i = exportSpecification->getParameters().begin(); - i != exportSpecification->getParameters().end(); - ++i) - { - keys.push_back(i->first); - values.push_back(i->second); - } - - for (std::vector::const_iterator i = exportSpecification->getSourceColumnNames().begin(); - i != exportSpecification->getSourceColumnNames().end(); i++) - { - columnNames.push_back(*i); - } - } -} -bool ExportSpecificationWrapper::hasConnectionName() const {return exportSpecification->hasConnectionName();} -bool ExportSpecificationWrapper::hasConnectionInformation() const {return exportSpecification->hasConnectionInformation();} -char* ExportSpecificationWrapper::copyConnectionName() const {return ::strdup(exportSpecification->getConnectionName().c_str());} -const ConnectionInformationWrapper ExportSpecificationWrapper::getConnectionInformation() const { - return ConnectionInformationWrapper(exportSpecification->getConnectionInformation()); -} -size_t ExportSpecificationWrapper::getNumberOfParameters() const {return keys.size();} -char* ExportSpecificationWrapper::copyKey(size_t pos) const {return ::strdup(keys.at(pos).c_str());} -char* ExportSpecificationWrapper::copyValue(size_t pos) const {return ::strdup(values.at(pos).c_str());} -bool ExportSpecificationWrapper::hasTruncate() const {return exportSpecification->hasTruncate();} -bool ExportSpecificationWrapper::hasReplace() const {return exportSpecification->hasReplace();} -bool ExportSpecificationWrapper::hasCreatedBy() const {return exportSpecification->hasCreatedBy();} -char* ExportSpecificationWrapper::copyCreatedBy() const { - if (exportSpecification->getCreatedBy().empty()) - return 0; - else - return ::strdup(exportSpecification->getCreatedBy().c_str()); -} - -size_t ExportSpecificationWrapper::numSourceColumns() const {return columnNames.size();} -char* ExportSpecificationWrapper::copySourceColumnName(size_t pos) const { - if (columnNames.at(pos).empty()) - return 0; - else - return ::strdup(columnNames.at(pos).c_str()); -} - -} diff --git a/exaudfclient/base/exaudflib/swig/script_data_transfer_objects_wrapper.h b/exaudfclient/base/exaudflib/swig/script_data_transfer_objects_wrapper.h deleted file mode 100644 index 7846d72..0000000 --- a/exaudfclient/base/exaudflib/swig/script_data_transfer_objects_wrapper.h +++ /dev/null @@ -1,74 +0,0 @@ -#ifndef SCRIPT_DATA_TRANSFER_OBJECT_C_WRAPPER_H -#define SCRIPT_DATA_TRANSFER_OBJECT_C_WRAPPER_H - -#include "script_data_transfer_objects.h" - - -namespace ExecutionGraph -{ - -class ConnectionInformationWrapper -{ -public: - explicit ConnectionInformationWrapper(const ConnectionInformation& connectionInformation_); - char* copyKind() const; - char* copyAddress() const; - char* copyUser() const; - char* copyPassword() const; - -protected: - const ConnectionInformation connectionInformation; -}; - - -class ImportSpecificationWrapper -{ -public: - explicit ImportSpecificationWrapper(ExecutionGraph::ImportSpecification* importSpecification_); - bool isSubselect() const; - size_t numSubselectColumns() const; - char* copySubselectColumnName(size_t i) const; - char* copySubselectColumnType(size_t i) const; - bool hasConnectionName() const; - bool hasConnectionInformation() const; - char* copyConnectionName() const; - const ConnectionInformationWrapper getConnectionInformation() const; - size_t getNumberOfParameters() const; - char* copyKey(size_t pos) const; - char* copyValue(size_t pos) const; -protected: - ImportSpecification* importSpecification; - std::vector keys; - std::vector values; -}; - - -class ExportSpecificationWrapper -{ -public: - explicit ExportSpecificationWrapper(ExecutionGraph::ExportSpecification* exportSpecification_); - bool hasConnectionName() const; - bool hasConnectionInformation() const; - char* copyConnectionName() const; - const ConnectionInformationWrapper getConnectionInformation() const; - size_t getNumberOfParameters() const; - char* copyKey(size_t pos) const; - char* copyValue(size_t pos) const; - bool hasTruncate() const; - bool hasReplace() const; - bool hasCreatedBy() const; - char* copyCreatedBy() const; - size_t numSourceColumns() const; - char* copySourceColumnName(size_t i) const; -protected: - ExportSpecification* exportSpecification; - std::vector keys; - std::vector values; - std::vector columnNames; -}; - - - -} - -#endif // SCRIPT_DATA_TRANSFER_OBJECT_C_WRAPPER_H diff --git a/exaudfclient/base/exaudflib/swig/swig_common.cc b/exaudfclient/base/exaudflib/swig/swig_common.cc deleted file mode 100644 index ac2f69f..0000000 --- a/exaudfclient/base/exaudflib/swig/swig_common.cc +++ /dev/null @@ -1,2 +0,0 @@ -//Check that swig_common.h is self-contained -#include "exaudflib/swig/swig_common.h" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/swig/swig_common.h b/exaudfclient/base/exaudflib/swig/swig_common.h deleted file mode 100644 index b61b0fc..0000000 --- a/exaudfclient/base/exaudflib/swig/swig_common.h +++ /dev/null @@ -1,186 +0,0 @@ -#ifndef SWIG_COMMON_H -#define SWIG_COMMON_H - -#include -#include -#include -#include - -namespace SWIGVMContainers { - -enum VMTYPE { - VM_UNSUPPORTED = 0, - VM_PYTHON = 1, - VM_SCHEME = 2, - VM_JAVASCRIPT = 3, - VM_R = 4, - VM_EXTERNAL = 5, - VM_PLUGIN_LANGUAGE = 7, - VM_BENCHMARK = 8 -}; - - -struct SWIGVMExceptionHandler -{ - SWIGVMExceptionHandler() { exthrowed = false; } - ~SWIGVMExceptionHandler() { } - void setException(const char* msg) { - exmsg = msg; exthrowed = true; - } - void setException(const std::string& msg) { - exmsg = msg; exthrowed = true; - } - std::string exmsg; - bool exthrowed; -}; - - -enum SWIGVM_datatype_e { - UNSUPPORTED = 0, - DOUBLE = 1, - INT32 = 2, - INT64 = 3, - NUMERIC = 4, - TIMESTAMP = 5, - DATE = 6, - STRING = 7, - BOOLEAN = 8, - INTERVALYM = 9, - INTERVALDS = 10, - GEOMETRY = 11 -}; - - -enum SWIGVM_itertype_e { - EXACTLY_ONCE = 1, - MULTIPLE = 2 -}; - -enum single_call_function_id_e { - SC_FN_NIL = 0, - SC_FN_DEFAULT_OUTPUT_COLUMNS = 1, - SC_FN_VIRTUAL_SCHEMA_ADAPTER_CALL = 2, - SC_FN_GENERATE_SQL_FOR_IMPORT_SPEC = 3, - SC_FN_GENERATE_SQL_FOR_EXPORT_SPEC = 4, -}; - -struct SWIGVM_columntype_t { - SWIGVM_datatype_e type; - std::string type_name; - unsigned int len; - unsigned int prec; - unsigned int scale; - SWIGVM_columntype_t(const SWIGVM_datatype_e t, const char *n, const unsigned int l, const unsigned int p, const unsigned int s): - type(t), type_name(n), len(l), prec(p), scale(s) { } - SWIGVM_columntype_t(const SWIGVM_datatype_e t, const char *n, const unsigned int l): type(t), type_name(n), len(l), prec(0), scale(0) { } - SWIGVM_columntype_t(const SWIGVM_datatype_e t, const char *n): type(t), type_name(n), len(0), prec(0), scale(0) { } - SWIGVM_columntype_t(): type(UNSUPPORTED), type_name("UNSUPPORTED"), len(0), prec(0), scale(0) { } -}; - - -struct SWIGVM_params_t { - uint64_t connection_id; - SWIGVMExceptionHandler *exch; - char *dbname; - char *dbversion; - char *script_name; - char *script_schema; - char *current_user; - char *current_schema; - char *scope_user; - char *script_code; - unsigned long long session_id; - unsigned long statement_id; - unsigned int node_count; - unsigned int node_id; - unsigned long long vm_id; - VMTYPE vm_type; - unsigned long long maximal_memory_limit; - std::vector *inp_names; - std::vector *inp_types; - SWIGVM_itertype_e inp_iter_type; - bool inp_force_finish; - std::vector *out_names; - std::vector *out_types; - SWIGVM_itertype_e out_iter_type; - bool m_allocate_params; - std::vector *is_emitted; - bool singleCallMode; - std::string pluginName; - std::string pluginURI; - std::string outputAddress; - SWIGVM_params_t(): - connection_id(0), exch(NULL), dbname(NULL), dbversion(NULL), script_name(NULL), script_schema(NULL), current_user(NULL), - current_schema(NULL), scope_user(NULL), script_code(NULL), - session_id(0), statement_id(0), node_count(0), node_id(0), vm_id(0), - vm_type(VM_UNSUPPORTED), maximal_memory_limit(0), - inp_names(NULL), inp_types(NULL), inp_iter_type(MULTIPLE), inp_force_finish(false), - out_names(NULL), out_types(NULL), out_iter_type(MULTIPLE), - m_allocate_params(false), - is_emitted(NULL), singleCallMode(false), pluginName(""), pluginURI(""), outputAddress("") - { } - SWIGVM_params_t(const bool allocate_params): - connection_id(0), exch(NULL), dbname(NULL), dbversion(NULL), script_name(NULL), script_schema(NULL), current_user(NULL), - current_schema(NULL), scope_user(NULL), script_code(NULL), - session_id(0), statement_id(0), node_count(0), node_id(0), vm_id(0), - vm_type(VM_UNSUPPORTED), maximal_memory_limit(0), - inp_names(allocate_params ? new std::vector() : NULL), - inp_types(allocate_params ? new std::vector() : NULL), - inp_iter_type(MULTIPLE), - inp_force_finish(false), - out_names(allocate_params ? new std::vector() : NULL), - out_types(allocate_params ? new std::vector() : NULL), - out_iter_type(MULTIPLE), - m_allocate_params(allocate_params), - is_emitted(allocate_params ? new std::vector() : NULL), - singleCallMode(false), pluginName(""), pluginURI(""), outputAddress("") - { } - SWIGVM_params_t(const SWIGVM_params_t &p): - connection_id(p.connection_id), - dbname(p.dbname), - dbversion(p.dbversion), - script_name(p.script_name), - script_schema(p.script_schema), - current_user(p.current_user), - current_schema(p.current_schema), - scope_user(p.scope_user), - script_code(p.script_code), - session_id(p.session_id), - statement_id(p.statement_id), - node_count(p.node_count), - node_id(p.node_id), - vm_id(p.vm_id), - vm_type(p.vm_type), - maximal_memory_limit(p.maximal_memory_limit), - inp_names(p.inp_names), - inp_types(p.inp_types), - inp_iter_type(p.inp_iter_type), - inp_force_finish(p.inp_force_finish), - out_names(p.out_names), - out_types(p.out_types), - out_iter_type(p.out_iter_type), - m_allocate_params(false), - is_emitted(p.is_emitted), - pluginName(p.pluginName), - pluginURI(p.pluginURI), - outputAddress(p.outputAddress) - { - if (p.m_allocate_params) - abort(); - } - ~SWIGVM_params_t() { - if (m_allocate_params) { - if (inp_names != NULL) { delete inp_names; inp_names = NULL; } - if (inp_types != NULL) { delete inp_types; inp_types = NULL; } - if (out_names != NULL) { delete out_names; out_names = NULL; } - if (out_types != NULL) { delete out_types; out_types = NULL; } - if (is_emitted != NULL) { delete is_emitted; is_emitted = NULL; } - } - } -}; - -extern __thread SWIGVM_params_t *SWIGVM_params; - -} //namespace SWIGVMContainers - -#endif //SWIG_COMMON_H diff --git a/exaudfclient/base/exaudflib/swig/swig_meta_data.cc b/exaudfclient/base/exaudflib/swig/swig_meta_data.cc deleted file mode 100644 index 1b1c6c7..0000000 --- a/exaudfclient/base/exaudflib/swig/swig_meta_data.cc +++ /dev/null @@ -1,2 +0,0 @@ -//Check that swig_meta_data.h is self-contained -#include "exaudflib/swig/swig_meta_data.h" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/swig/swig_meta_data.h b/exaudfclient/base/exaudflib/swig/swig_meta_data.h deleted file mode 100644 index 4296208..0000000 --- a/exaudfclient/base/exaudflib/swig/swig_meta_data.h +++ /dev/null @@ -1,123 +0,0 @@ -#ifndef SWIG_META_DATA_H -#define SWIG_META_DATA_H - -#include "exaudflib/load_dynamic.h" -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/swig/script_data_transfer_objects_wrapper.h" -#include "exaudflib/udf_plugin_interface.h" - -namespace SWIGVMContainers { - -struct SWIGMetadataIf { - - virtual ~SWIGMetadataIf() {}; - virtual const char* databaseName() = 0; - virtual const char* databaseVersion() = 0; - virtual const char* scriptName() = 0; - virtual const char* scriptSchema() = 0; - virtual const char* currentUser() = 0; - virtual const char* scopeUser() = 0; - virtual const char* currentSchema() = 0; - virtual const char* scriptCode() = 0; - virtual const unsigned long long sessionID() = 0; - virtual const char *sessionID_S() = 0; - virtual const unsigned long statementID() = 0; - virtual const unsigned int nodeCount() = 0; - virtual const unsigned int nodeID() = 0; - virtual const unsigned long long vmID() = 0; - virtual const unsigned long long memoryLimit() = 0; - virtual const VMTYPE vmType() = 0; - virtual const char *vmID_S() = 0; - virtual const ExecutionGraph::ConnectionInformationWrapper* connectionInformation(const char* connection_name) = 0; - virtual const char* moduleContent(const char* name) = 0; - virtual const unsigned int inputColumnCount() = 0; - virtual const char *inputColumnName(unsigned int col) = 0; - virtual const SWIGVM_datatype_e inputColumnType(unsigned int col) = 0; - virtual const char *inputColumnTypeName(unsigned int col) = 0; - virtual const unsigned int inputColumnSize(unsigned int col) = 0; - virtual const unsigned int inputColumnPrecision(unsigned int col) = 0; - virtual const unsigned int inputColumnScale(unsigned int col) = 0; - virtual const SWIGVM_itertype_e inputType() = 0; - virtual const unsigned int outputColumnCount() = 0; - virtual const char *outputColumnName(unsigned int col) = 0; - virtual const SWIGVM_datatype_e outputColumnType(unsigned int col) = 0; - virtual const char *outputColumnTypeName(unsigned int col) = 0; - virtual const unsigned int outputColumnSize(unsigned int col) = 0; - virtual const unsigned int outputColumnPrecision(unsigned int col) = 0; - virtual const unsigned int outputColumnScale(unsigned int col) = 0; - virtual const SWIGVM_itertype_e outputType() = 0; - virtual const bool isEmittedColumn(unsigned int col) = 0; - virtual const char* checkException() = 0; - virtual const char* pluginLanguageName() = 0; - virtual const char* pluginURI() = 0; - virtual const char* outputAddress() = 0; -}; - - - -class SWIGMetadata : public SWIGMetadataIf { - SWIGMetadataIf* impl=nullptr; - typedef SWIGVMContainers::SWIGMetadataIf* (*CREATE_METADATA_FUN)(); - public: - SWIGMetadata() - { -#if !defined(UDF_PLUGIN_CLIENT) && !defined(USE_STATIC_SWIG) - CREATE_METADATA_FUN create = (CREATE_METADATA_FUN)load_dynamic("create_SWIGMetaData"); - impl = create(); -#else - impl = create_SWIGMetaData(); -#endif - } - - virtual ~SWIGMetadata() { - if (impl!=nullptr) { - delete impl; - } - } - virtual const char* databaseName() { return impl->databaseName(); } - virtual const char* databaseVersion() { return impl->databaseVersion(); } - virtual const char* scriptName() { return impl->scriptName(); } - virtual const char* scriptSchema() { return impl->scriptSchema(); } - virtual const char* currentUser() { return impl->currentUser(); } - virtual const char* scopeUser() { return impl->scopeUser(); } - virtual const char* currentSchema() {return impl->currentSchema();} - virtual const char* scriptCode() { return impl->scriptCode(); } - virtual const unsigned long long sessionID() { return impl->sessionID(); } - virtual const char *sessionID_S() { return impl->sessionID_S(); } - virtual const unsigned long statementID() { return impl->statementID(); } - virtual const unsigned int nodeCount() { return impl->nodeCount(); } - virtual const unsigned int nodeID() { return impl->nodeID(); } - virtual const unsigned long long vmID() { return impl->vmID(); } - virtual const unsigned long long memoryLimit() { return impl->memoryLimit(); } - virtual const VMTYPE vmType() { return impl->vmType(); } - virtual const char *vmID_S() { return impl->vmID_S(); } - virtual const ExecutionGraph::ConnectionInformationWrapper* connectionInformation(const char* connection_name){ - return impl->connectionInformation(connection_name); - } - virtual const char* moduleContent(const char* name) {return impl->moduleContent(name);} - virtual const unsigned int inputColumnCount() { return impl->inputColumnCount(); } - virtual const char *inputColumnName(unsigned int col) { return impl->inputColumnName(col);} - virtual const SWIGVM_datatype_e inputColumnType(unsigned int col) { return impl->inputColumnType(col);} - virtual const char *inputColumnTypeName(unsigned int col) { return impl->inputColumnTypeName(col); } - virtual const unsigned int inputColumnSize(unsigned int col) {return impl->inputColumnSize(col);} - virtual const unsigned int inputColumnPrecision(unsigned int col) { return impl->inputColumnPrecision(col); } - virtual const unsigned int inputColumnScale(unsigned int col) { return impl->inputColumnScale(col);} - virtual const SWIGVM_itertype_e inputType() { return impl->inputType();} - virtual const unsigned int outputColumnCount() { return impl->outputColumnCount(); } - virtual const char *outputColumnName(unsigned int col) { return impl->outputColumnName(col); } - virtual const SWIGVM_datatype_e outputColumnType(unsigned int col) { return impl->outputColumnType(col); } - virtual const char *outputColumnTypeName(unsigned int col) { return impl->outputColumnTypeName(col); } - virtual const unsigned int outputColumnSize(unsigned int col) { return impl->outputColumnSize(col); } - virtual const unsigned int outputColumnPrecision(unsigned int col) {return impl->outputColumnPrecision(col);} - virtual const unsigned int outputColumnScale(unsigned int col) {return impl->outputColumnScale(col);} - virtual const SWIGVM_itertype_e outputType() { return impl->outputType(); } - virtual const bool isEmittedColumn(unsigned int col){ return impl->isEmittedColumn(col);} - virtual const char* checkException() {return impl->checkException();} - virtual const char* pluginLanguageName() {return impl->pluginLanguageName();} - virtual const char* pluginURI() {return impl->pluginURI();} - virtual const char* outputAddress() {return impl->outputAddress();} -}; - -} // namespace SWIGVMContainers - -#endif //SWIG_META_DATA_H \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/swig/swig_result_handler.cc b/exaudfclient/base/exaudflib/swig/swig_result_handler.cc deleted file mode 100644 index 72365f4..0000000 --- a/exaudfclient/base/exaudflib/swig/swig_result_handler.cc +++ /dev/null @@ -1,2 +0,0 @@ -//Check that swig_result_handler.h is self-contained -#include "exaudflib/swig/swig_result_handler.h" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/swig/swig_result_handler.h b/exaudfclient/base/exaudflib/swig/swig_result_handler.h deleted file mode 100644 index 5357b71..0000000 --- a/exaudfclient/base/exaudflib/swig/swig_result_handler.h +++ /dev/null @@ -1,72 +0,0 @@ -#ifndef SWIG_RESULT_HANDLER_H -#define SWIG_RESULT_HANDLER_H - -#include -#include -#include -#include "exaudflib/load_dynamic.h" -#include "exaudflib/swig/swig_table_iterator.h" -#include "exaudflib/udf_plugin_interface.h" - -#define SWIG_MAX_VAR_DATASIZE 4000000 - -namespace SWIGVMContainers { - -class SWIGRAbstractResultHandler { -public: - virtual ~SWIGRAbstractResultHandler() {}; - virtual void reinitialize()=0; - virtual unsigned long rowsEmited()=0; - virtual void flush()=0; - virtual bool next()=0; - virtual void setDouble(unsigned int col, const double v)=0; - virtual void setString(unsigned int col, const char *v, size_t l)=0; - virtual void setInt32(unsigned int col, const int32_t v)=0; - virtual void setInt64(unsigned int col, const int64_t v)=0; - virtual void setNumeric(unsigned int col, const char *v)=0; - virtual void setTimestamp(unsigned int col, const char *v)=0; - virtual void setDate(unsigned int col, const char *v)=0; - virtual void setBoolean(unsigned int col, const bool v)=0; - virtual void setNull(unsigned int col)=0; - virtual const char* checkException()=0; -}; - - -class SWIGResultHandler { //: public SWIGRAbstractResultHandler { - SWIGRAbstractResultHandler* impl=nullptr; - typedef SWIGVMContainers::SWIGRAbstractResultHandler* (*CREATE_RESULTHANDLER_FUN)(SWIGVMContainers::SWIGTableIterator*); -public: - SWIGResultHandler(SWIGTableIterator* table_iterator) - { -#if !defined(UDF_PLUGIN_CLIENT) && !defined(USE_STATIC_SWIG) - CREATE_RESULTHANDLER_FUN creator = (CREATE_RESULTHANDLER_FUN)load_dynamic("create_SWIGResultHandler"); - impl = creator(table_iterator); -#else - impl = create_SWIGResultHandler(table_iterator); -#endif - } - - virtual ~SWIGResultHandler() { - if (impl!=nullptr) { - delete impl; - } - } - virtual void reinitialize() {impl->reinitialize(); } - virtual unsigned long rowsEmited() {return impl->rowsEmited();} - virtual void flush() {impl->flush();} - virtual bool next() {return impl->next();} - virtual void setDouble(unsigned int col, const double v) {impl->setDouble(col, v);} - virtual void setString(unsigned int col, const char *v, size_t l) {impl->setString(col,v,l);} - virtual void setInt32(unsigned int col, const int32_t v) {impl->setInt32(col,v);} - virtual void setInt64(unsigned int col, const int64_t v) {impl->setInt64(col,v);} - virtual void setNumeric(unsigned int col, const char *v) {impl->setNumeric(col,v);} - virtual void setTimestamp(unsigned int col, const char *v) {impl->setTimestamp(col,v);} - virtual void setDate(unsigned int col, const char *v) {impl->setDate(col,v);} - virtual void setBoolean(unsigned int col, const bool v) {impl->setBoolean(col,v);} - virtual void setNull(unsigned int col) {impl->setNull(col);} - const char* checkException() {return impl->checkException();} -}; - -} //namespace SWIGVMContainers - -#endif //SWIG_RESULT_HANDLER_H \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/swig/swig_table_iterator.cc b/exaudfclient/base/exaudflib/swig/swig_table_iterator.cc deleted file mode 100644 index 2e30676..0000000 --- a/exaudfclient/base/exaudflib/swig/swig_table_iterator.cc +++ /dev/null @@ -1,2 +0,0 @@ -//Check that swig_table_iterator.h is self-contained -#include "exaudflib/swig/swig_table_iterator.h" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/swig/swig_table_iterator.h b/exaudfclient/base/exaudflib/swig/swig_table_iterator.h deleted file mode 100644 index 662c08c..0000000 --- a/exaudfclient/base/exaudflib/swig/swig_table_iterator.h +++ /dev/null @@ -1,79 +0,0 @@ -#ifndef SWIG_TABLE_ITERATOR_H -#define SWIG_TABLE_ITERATOR_H - -#include -#include -#include -#include "exaudflib/load_dynamic.h" -#include "exaudflib/udf_plugin_interface.h" - -namespace SWIGVMContainers { - -class AbstractSWIGTableIterator { -public: - virtual ~AbstractSWIGTableIterator() {} - virtual void reinitialize()=0; - virtual bool next()=0; - virtual bool eot()=0; - virtual void reset()=0; - virtual unsigned long restBufferSize()=0; - virtual unsigned long rowsCompleted()=0; - virtual unsigned long rowsInGroup()=0; - virtual double getDouble(unsigned int col)=0; - virtual const char *getString(unsigned int col, size_t *length = NULL)=0; - virtual int32_t getInt32(unsigned int col)=0; - virtual int64_t getInt64(unsigned int col)=0; - virtual const char *getNumeric(unsigned int col)=0; - virtual const char *getTimestamp(unsigned int col)=0; - virtual const char *getDate(unsigned int col)=0; - virtual bool getBoolean(unsigned int col)=0; - virtual bool wasNull()=0; - virtual uint64_t get_current_row()=0; - virtual const char* checkException()=0; -}; - - - -class SWIGTableIterator { //: public AbstractSWIGTableIterator { - typedef SWIGVMContainers::AbstractSWIGTableIterator* (*CREATE_TABLEITERATOR_FUN)(); - - AbstractSWIGTableIterator* impl=nullptr; -public: - SWIGTableIterator() - { -#if !defined(UDF_PLUGIN_CLIENT) && !defined(USE_STATIC_SWIG) - CREATE_TABLEITERATOR_FUN creator = (CREATE_TABLEITERATOR_FUN)load_dynamic("create_SWIGTableIterator"); - impl = creator(); -#else - impl = create_SWIGTableIterator(); -#endif - } - - virtual ~SWIGTableIterator() { - if (impl!=nullptr) { - delete impl; - } - } - virtual void reinitialize() { impl->reinitialize();} - virtual bool next() { return impl->next(); } - virtual bool eot() { return impl->eot(); } - virtual void reset() { return impl->reset(); } - virtual unsigned long restBufferSize() { return impl->restBufferSize();} - virtual unsigned long rowsCompleted() { return impl->rowsCompleted();} - virtual unsigned long rowsInGroup() { return impl->rowsInGroup();} - virtual double getDouble(unsigned int col) { return impl->getDouble(col);} - virtual const char *getString(unsigned int col, size_t *length = NULL) {return impl->getString(col, length);} - virtual int32_t getInt32(unsigned int col) {return impl->getInt32(col);} - virtual int64_t getInt64(unsigned int col) {return impl->getInt64(col);} - virtual const char *getNumeric(unsigned int col) {return impl->getNumeric(col);} - virtual const char *getTimestamp(unsigned int col) {return impl->getTimestamp(col);} - virtual const char *getDate(unsigned int col) {return impl->getDate(col);} - virtual bool getBoolean(unsigned int col) {return impl->getBoolean(col);} - virtual bool wasNull() { return impl->wasNull();} - virtual uint64_t get_current_row() {return impl->get_current_row();} - const char* checkException() {return impl->checkException();} -}; - -} //namespace SWIGVMContainers - -#endif //SWIG_TABLE_ITERATOR_H \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/test/BUILD b/exaudfclient/base/exaudflib/test/BUILD deleted file mode 100644 index 1026d25..0000000 --- a/exaudfclient/base/exaudflib/test/BUILD +++ /dev/null @@ -1,8 +0,0 @@ -cc_test( - name = "exaudflib-test", - srcs = ["script_data_transfer_objects_test.cpp"], - deps = [ - "//exaudflib:script_data_transfer_objects", - "@googletest//:gtest_main", - ], -) \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/test/script_data_transfer_objects_test.cpp b/exaudfclient/base/exaudflib/test/script_data_transfer_objects_test.cpp deleted file mode 100644 index 6e146c3..0000000 --- a/exaudfclient/base/exaudflib/test/script_data_transfer_objects_test.cpp +++ /dev/null @@ -1,260 +0,0 @@ -#include "exaudflib/swig/script_data_transfer_objects.h" -#include - -using namespace ExecutionGraph; - - -class ImportSpecificationTest : public ::testing::Test { -protected: - virtual void SetUp() { - - } - virtual void TearDown() { - - } -}; - -TEST_F(ImportSpecificationTest, is_subselect) -{ - ImportSpecification is1(true); - ImportSpecification is2(false); - EXPECT_TRUE(is1.isSubselect()); - EXPECT_FALSE(is2.isSubselect()); -} - -TEST_F(ImportSpecificationTest, column_names_and_types_only_for_subselects) -{ - // append - ImportSpecification is1(false); - try { - is1.appendSubselectColumnName("col1"); - ADD_FAILURE(); - } catch (ImportSpecification::Error& e) - { - EXPECT_STREQ(e.what(), "import specification error: cannot add column name to non-subselect import specification"); - } - ImportSpecification is2(true); - try { - is2.appendSubselectColumnName("col1"); - } catch (ImportSpecification::Error& e) - { - ADD_FAILURE(); - } - ImportSpecification is3(false); - try { - is3.appendSubselectColumnType("type1"); - ADD_FAILURE(); - } catch (ImportSpecification::Error& e) - { - EXPECT_STREQ(e.what(), "import specification error: cannot add column type to non-subselect import specification"); - } - ImportSpecification is4(true); - try { - is4.appendSubselectColumnType("type1"); - } catch (ImportSpecification::Error& e) - { - ADD_FAILURE(); - } - // get - ImportSpecification is5(false); - try { - is5.getSubselectColumnNames(); - ADD_FAILURE(); - } catch (ImportSpecification::Error& e) - { - EXPECT_STREQ(e.what(), "import specification error: cannot get column names of non-subselect import specification"); - } - ImportSpecification is6(true); - try { - is6.getSubselectColumnNames(); - } catch (ImportSpecification::Error& e) - { - ADD_FAILURE(); - } - ImportSpecification is7(false); - try { - is7.getSubselectColumnTypes(); - ADD_FAILURE(); - } catch (ImportSpecification::Error& e) - { - EXPECT_STREQ(e.what(), "import specification error: cannot get column types of non-subselect import specification"); - } - ImportSpecification is8(true); - try { - is8.getSubselectColumnTypes(); - } catch (ImportSpecification::Error& e) - { - ADD_FAILURE(); - } -} - -TEST_F(ImportSpecificationTest, column_names_and_types) -{ - ImportSpecification is1(true); - EXPECT_FALSE(is1.hasSubselectColumnSpecification()); - is1.appendSubselectColumnName("col1"); - EXPECT_TRUE(is1.hasSubselectColumnSpecification()); - EXPECT_FALSE(is1.hasConsistentColumns()); - - is1.appendSubselectColumnType("type1"); - EXPECT_TRUE(is1.hasSubselectColumnSpecification()); - EXPECT_TRUE(is1.hasConsistentColumns()); - - is1.appendSubselectColumnName("col2"); - EXPECT_TRUE(is1.hasSubselectColumnSpecification()); - EXPECT_FALSE(is1.hasConsistentColumns()); - - is1.appendSubselectColumnType("type2"); - EXPECT_TRUE(is1.hasSubselectColumnSpecification()); - EXPECT_TRUE(is1.hasConsistentColumns()); - - is1.appendSubselectColumnName("col3"); - EXPECT_TRUE(is1.hasSubselectColumnSpecification()); - EXPECT_FALSE(is1.hasConsistentColumns()); - - is1.appendSubselectColumnType("type3"); - - EXPECT_TRUE(is1.hasSubselectColumnSpecification()); - EXPECT_TRUE(is1.hasConsistentColumns()); - EXPECT_EQ(is1.getSubselectColumnNames().size(), 3); - EXPECT_EQ(is1.getSubselectColumnTypes().size(), 3); - EXPECT_EQ(is1.getSubselectColumnNames()[0], "col1"); - EXPECT_EQ(is1.getSubselectColumnNames()[1], "col2"); - EXPECT_EQ(is1.getSubselectColumnNames()[2], "col3"); - EXPECT_EQ(is1.getSubselectColumnTypes()[0], "type1"); - EXPECT_EQ(is1.getSubselectColumnTypes()[1], "type2"); - EXPECT_EQ(is1.getSubselectColumnTypes()[2], "type3"); -} - -TEST_F(ImportSpecificationTest, connection_name_or_details) -{ - ImportSpecification is1(false); - is1.setConnectionName("some_connection"); - try { - is1.setConnectionInformation(ConnectionInformation("some_address","some_user","some_password")); - ADD_FAILURE(); - } catch(ImportSpecification::Error& e) { - EXPECT_STREQ(e.what(), "import specification error: cannot set connection information, because there is already a connection name set"); - } - - - // - - ImportSpecification is2(false); - is2.setConnectionInformation(ConnectionInformation("some_address","some_user","some_password")); - try { - is2.setConnectionName("some_connection"); - ADD_FAILURE(); - } catch(ImportSpecification::Error& e) { - EXPECT_STREQ(e.what(), "import specification error: cannot set connection name, because there is already connection information set"); - } -} - -TEST_F(ImportSpecificationTest, connection_dont_set_twice) -{ - ImportSpecification is1(false); - is1.setConnectionName("some_connection"); - try { - is1.setConnectionName("some_other_connection"); - ADD_FAILURE(); - } catch(ImportSpecification::Error& e) { - EXPECT_STREQ(e.what(), "import specification error: connection name is set more than once"); - } - ImportSpecification is2(false); - is2.setConnectionInformation(ConnectionInformation("some_address","some_user","some_password")); - try { - is2.setConnectionInformation(ConnectionInformation()); - ADD_FAILURE(); - } catch(ImportSpecification::Error& e) { - EXPECT_STREQ(e.what(), "import specification error: cannot set connection information more than once"); - } -} - - -TEST_F(ImportSpecificationTest, connection_info) -{ - ImportSpecification is1(false); - is1.setConnectionName("some_connection"); - EXPECT_EQ(is1.getConnectionName(), "some_connection"); - - ImportSpecification is2(false); - is2.setConnectionInformation(ConnectionInformation("some_address","some_user","some_password")); - EXPECT_EQ(is2.getConnectionInformation().getAddress(), "some_address"); - EXPECT_EQ(is2.getConnectionInformation().getUser(), "some_user"); - EXPECT_EQ(is2.getConnectionInformation().getPassword(), "some_password"); - EXPECT_EQ(is2.getConnectionInformation().getKind(), "password"); -} - - -TEST_F(ImportSpecificationTest, parameters) -{ - ImportSpecification is1(false); - EXPECT_FALSE(is1.hasParameters()); - is1.addParameter("key1","value1"); - EXPECT_TRUE(is1.hasParameters()); - std::map::const_iterator ps1 = is1.getParameters().begin(); - EXPECT_EQ(ps1->first, "key1"); - EXPECT_EQ(ps1->second, "value1"); - - is1.addParameter("key2","value2"); - EXPECT_TRUE(is1.hasParameters()); - std::map::const_iterator ps2 = is1.getParameters().begin(); - ++ps2; - EXPECT_EQ(ps2->first, "key2"); - EXPECT_EQ(ps2->second, "value2"); -} - -// -// -// -// -class ConnectionInformationTest : public ::testing::Test { -protected: - virtual void SetUp() { - - } - virtual void TearDown() { - - } -}; - -TEST_F(ConnectionInformationTest, address_user_password_construction) -{ - ConnectionInformation ci("some_address","some_user","some_password"); - EXPECT_EQ(ci.getKind(), "password"); - EXPECT_EQ(ci.getAddress(), "some_address"); - EXPECT_EQ(ci.getUser(), "some_user"); - EXPECT_EQ(ci.getPassword(), "some_password"); - EXPECT_FALSE(ci.hasData()); -} - -TEST_F(ConnectionInformationTest, kind_address_user_password_construction) -{ - ConnectionInformation ci("some_kind","some_address","some_user","some_password"); - EXPECT_EQ(ci.getKind(), "some_kind"); - EXPECT_EQ(ci.getAddress(), "some_address"); - EXPECT_EQ(ci.getUser(), "some_user"); - EXPECT_EQ(ci.getPassword(), "some_password"); - EXPECT_FALSE(ci.hasData()); -} - -TEST_F(ConnectionInformationTest, empty_construction) -{ - ConnectionInformation ci; - EXPECT_TRUE(ci.getKind().empty()); - EXPECT_TRUE(ci.getAddress().empty()); - EXPECT_TRUE(ci.getUser().empty()); - EXPECT_TRUE(ci.getPassword().empty()); - EXPECT_TRUE(ci.hasData()); -} - -TEST_F(ConnectionInformationTest, copy) -{ - ConnectionInformation ca("a","b","c","d"); - ConnectionInformation cb(ca); - EXPECT_EQ(ca.getKind(), cb.getKind()); - EXPECT_EQ(ca.getAddress(), cb.getAddress()); - EXPECT_EQ(ca.getUser(), cb.getUser()); - EXPECT_EQ(ca.getPassword(), cb.getPassword()); - EXPECT_EQ(ca.hasData(), cb.hasData()); -} diff --git a/exaudfclient/base/exaudflib/udf_plugin_interface.h b/exaudfclient/base/exaudflib/udf_plugin_interface.h deleted file mode 100644 index 8ff2b8b..0000000 --- a/exaudfclient/base/exaudflib/udf_plugin_interface.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef UDF_PLUGIN_INTERFACE_H -#define UDF_PLUGIN_INTERFACE_H - -#if defined(UDF_PLUGIN_CLIENT) || defined(USE_STATIC_SWIG) -namespace SWIGVMContainers { -class SWIGMetadataIf; -class AbstractSWIGTableIterator; -class SWIGRAbstractResultHandler; -class SWIGTableIterator; -} - -extern "C" { -SWIGVMContainers::SWIGMetadataIf* create_SWIGMetaData(); -SWIGVMContainers::AbstractSWIGTableIterator* create_SWIGTableIterator(); -SWIGVMContainers::SWIGRAbstractResultHandler* create_SWIGResultHandler(SWIGVMContainers::SWIGTableIterator* table_iterator); -} -#endif // UDF_PLUGIN_CLIENT - -#endif // UDF_PLUGIN_INTERFACE_H diff --git a/exaudfclient/base/exaudflib/vm/swig_vm.cc b/exaudfclient/base/exaudflib/vm/swig_vm.cc deleted file mode 100644 index 3e2d98f..0000000 --- a/exaudfclient/base/exaudflib/vm/swig_vm.cc +++ /dev/null @@ -1,2 +0,0 @@ -//Check that swig_vm.h is self-contained -#include "exaudflib/vm/swig_vm.h" \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/vm/swig_vm.h b/exaudfclient/base/exaudflib/vm/swig_vm.h deleted file mode 100644 index 3f88c7a..0000000 --- a/exaudfclient/base/exaudflib/vm/swig_vm.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef SWIGVM_H -#define SWIGVM_H - -#include -#include -#include -#include -#include "exaudflib/swig/swig_common.h" -#include "exaudflib/swig/script_data_transfer_objects_wrapper.h" - -namespace SWIGVMContainers { - -class SWIGVM { - public: - struct exception : public std::exception { - exception(const char *reason): m_reason(reason) { } - exception(const std::string& reason): m_reason(reason) { } - virtual ~exception() throw() { } - const char* what() const throw() { return m_reason.c_str(); } - private: - std::string m_reason; - }; - SWIGVM() { } - virtual ~SWIGVM() { } - virtual void shutdown() {}; - virtual void destroy() {}; - virtual bool run() = 0; - bool run_() { - try { - return run(); - } catch (SWIGVM::exception& ex) { - std::cerr << "SWGVM run_: caught: " << ex.what(); - std::lock_guard lck(exception_msg_mtx); - exception_msg = ex.what(); - return true; /* we are done */ - } - } - virtual bool useZmqSocketLocks() {return false;} - virtual const char* singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args) = 0; - std::string exception_msg; - std::mutex exception_msg_mtx; - std::string calledUndefinedSingleCall; -}; - -} //namespace SWIGVMContainers - -#endif //SWIGVM_H \ No newline at end of file diff --git a/exaudfclient/base/exaudflib/zmqcontainer.proto b/exaudfclient/base/exaudflib/zmqcontainer.proto deleted file mode 100644 index 5cbca1d..0000000 --- a/exaudfclient/base/exaudflib/zmqcontainer.proto +++ /dev/null @@ -1,262 +0,0 @@ -syntax = "proto2"; - -option optimize_for = LITE_RUNTIME; -// --- data definition --------------------------------------------------------- - -enum column_type { - PB_UNSUPPORTED = 0; // represents following SQL types: - PB_DOUBLE = 1; // FLOAT - PB_INT32 = 2; // DECIMAL(4, 0) - PB_INT64 = 3; // DECIMAL(8, 0) - PB_NUMERIC = 4; // all other numeric types - PB_TIMESTAMP = 5; // TIMESTAMP - PB_DATE = 6; // DATE - PB_STRING = 7; // CHAR or VARCHAR - PB_BOOLEAN = 8; // BOOL -} - -// type of iteration: -// scalar, returns -> PB_EXACTLY_ONCE -// set, emits -> PB_MULTIPLE -enum iter_type { PB_EXACTLY_ONCE = 1; PB_MULTIPLE = 2; }; - -message exascript_metadata { - message column_definition { - // Name of column as UTF-8 String - required string name = 1; - // Type of column (see "enum column_type") - optional column_type type = 2; - // Type name how it is shown in EXASolution - required string type_name = 3; - // Size of Type, for CHAR and VARCHAR types - optional uint32 size = 4; - // Precision, for DECIMAL types - optional uint32 precision = 5; - // Scale, for DECIMAL types - optional uint32 scale = 6; - } - required iter_type input_iter_type = 1; - required iter_type output_iter_type = 2; - - repeated column_definition input_columns = 3; - repeated column_definition output_columns = 4; - - // If set, do not enter the run() state but instead wait for single - // synchronous function invokations - required bool single_call_mode = 5; -} - -message exascript_table_data { - // Rows count in current message - required uint64 rows = 1; - // Rows count in current group in EXASolution - // Can be 0 if no group defined - required uint64 rows_in_group = 8; - - // Storage for following types: NUMERIC, TIMESTAMP, DATE and STRING - repeated string data_string = 2; - - repeated bool data_nulls = 3 [packed = true]; - repeated bool data_bool = 4 [packed = true]; - repeated int32 data_int32 = 5 [packed = true]; - repeated int64 data_int64 = 6 [packed = true]; - repeated double data_double = 7 [packed = true]; - // Local row numbers - repeated uint64 row_number = 9 [packed = true]; -} - -// --- initialization process -------------------------------------------------- - -message exascript_client { - // URL of the client in form: tcp://10.10.1.1:2000 - required string client_name = 1; - // UTF-8 Text, which is received from redirector and send to VM - // This string will not be interpreted by EXASolution - optional string meta_info = 2; -} - -/* -enum exascript_vmtype { - PB_VM_UNSUPPORTED = 0; - PB_VM_PYTHON = 1; - PB_VM_SCHEME = 2; - PB_VM_JAVASCRIPT = 3; - PB_VM_R = 4; - PB_VM_EXTERNAL = 5; - PM_VM_PLUGIN_LANGUAGE = 7; -} -*/ - -message exascript_info { - required string database_name = 1; - required string database_version = 2; - required string script_name = 3; - required string source_code = 4; - // Session ID: long unique number of current session in EXASolution - required uint64 session_id = 5; - // Number of SQL Statement in current Session - required uint32 statement_id = 6; - // Number of nodes in EXASolution - required uint32 node_count = 7; - // Current node number, starting at 0 - required uint32 node_id = 8; - // Long unique number of VM - required uint64 vm_id = 9; - // Type of the VM - //required exascript_vmtype vm_type = 10; - // A VM should not exceed this memory limit - // For external UDF's it doesn't matter - required uint64 maximal_memory_limit = 11; - // meta_info contains the string from meta_info in exascript_client message - optional string meta_info = 12; - required string script_schema = 13; - optional string current_user = 14; - optional string current_schema = 15; - optional string scope_user = 16; -} -message exascript_ping { - // the string from exascript_client message - required string meta_info = 1; -} - -// --- connection finalization ------------------------------------------------- - -message exascript_close { - optional string exception_message = 1; -} - -// --- script and other imports -------------------------------------------------------- - -enum import_type { - PB_IMPORT_SCRIPT_CODE = 1; - PB_IMPORT_CONNECTION_INFORMATION = 2; -} - -message exascript_import_req { - required string script_name = 1; // todo?: rename to "object_name" - optional import_type kind = 2 [default=PB_IMPORT_SCRIPT_CODE]; -} - -message connection_information_rep { - required string kind = 1; - required string address = 2; - required string user = 3; - required string password = 4; -} - -message exascript_import_rep { - optional string source_code = 1; - optional string exception_message = 2; - optional connection_information_rep connection_information = 3; -} - -// --- sending data from exasolution to process -------------------------------- - -message exascript_next_data_rep { - required exascript_table_data table = 2; -} - -// --- sending data from process to exasolution -------------------------------- - -message exascript_emit_data_req { - required exascript_table_data table = 2; -} - -// --- sending a single result from process to exasolution --------------------- -message exascript_return_req { - required string result = 1; -} - -message exascript_undefined_call_req { - required string remote_fn = 1; -} - -enum single_call_function_id { - SC_FN_NIL = 0; - SC_FN_DEFAULT_OUTPUT_COLUMNS = 1; - SC_FN_VIRTUAL_SCHEMA_ADAPTER_CALL = 2; - SC_FN_GENERATE_SQL_FOR_IMPORT_SPEC = 3; - SC_FN_GENERATE_SQL_FOR_EXPORT_SPEC = 4; -} - - -message key_value_pair { - required string key = 1; - required string value = 2; -} - -message import_specification_rep { - required bool is_subselect = 1; - optional connection_information_rep connection_information = 2; - optional string connection_name = 3; - repeated exascript_metadata.column_definition subselect_column_specification = 4; - repeated key_value_pair parameters = 5; -} - -message export_specification_rep { - required bool has_truncate = 1; - required bool has_replace = 2; - optional string created_by = 3; - repeated string source_column_names = 4; - optional connection_information_rep connection_information = 5; - optional string connection_name = 6; - repeated key_value_pair parameters = 7; -} - -message exascript_single_call_rep { - required single_call_function_id fn = 1; - optional import_specification_rep import_specification = 2; - optional string json_arg = 3; - optional export_specification_rep export_specification = 4; -} - -// --- general request and response messages ----------------------------------- - -enum message_type { - MT_UNKNOWN = 0; - MT_CLIENT = 1; - MT_INFO = 2; - MT_META = 3; - MT_CLOSE = 4; - MT_IMPORT = 5; - MT_NEXT = 6; - MT_RESET = 7; - MT_EMIT = 8; - MT_RUN = 9; - MT_DONE = 10; - MT_CLEANUP = 11; - MT_FINISHED = 12; - MT_PING_PONG = 13; - MT_TRY_AGAIN = 14; - MT_CALL = 15; // this message request the synchronous invocation of a function in the remote container - MT_RETURN = 16; // this message is used to communicate the result of a synchronous function invocation from the container to EXASolution - MT_UNDEFINED_CALL = 17; // this message tells EXASolution that the requested function in not available in the container -} - -; - -message exascript_request { - required message_type type = 1; - required uint64 connection_id = 2; - optional exascript_client client = 3; /* E-UDF service -> EXASolution */ - optional exascript_info info = 4; /* EXASolution -> Redirector */ - optional exascript_close close = 5; - optional exascript_import_req import = 6; - optional exascript_emit_data_req emit = 7; - optional exascript_ping ping = 8; - optional exascript_return_req call_result = 9; - optional exascript_undefined_call_req undefined_call = 10; -} - -message exascript_response { - required message_type type = 1; - required uint64 connection_id = 2; - optional exascript_client client = 3; /* Redirector -> EXASolution */ - optional exascript_info info = 4; /* EXASolution -> E-UDF service */ - optional exascript_metadata meta = 5; - optional exascript_close close = 6; - optional exascript_import_rep import = 7; - optional exascript_next_data_rep next = 8; - optional exascript_ping ping = 9; - optional exascript_single_call_rep call = 10; -} diff --git a/exaudfclient/base/load_dynamic.cc b/exaudfclient/base/load_dynamic.cc deleted file mode 100644 index f8d9ee4..0000000 --- a/exaudfclient/base/load_dynamic.cc +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef UDF_PLUGIN_CLIENT -#include //This is required for dynamic linking in new linker namespace, not required for plugins -#endif - -#include -#include "exaudflib/vm/swig_vm.h" - -static void* exaudflib_handle = nullptr; - -#ifndef UDF_PLUGIN_CLIENT - -void set_exaudflib_handle(void* handle) { - if(handle != nullptr) { - exaudflib_handle = handle; - } -} - -void* load_dynamic(const char* name) { - if(exaudflib_handle == nullptr) { - throw SWIGVMContainers::SWIGVM::exception("Error: exaudflib_handle is not set."); - } - void* res = dlsym(exaudflib_handle, name); - char* error; - if ((error = dlerror()) != nullptr) - { - std::stringstream sb; - sb << "Error when trying to load function '" << name << "': " << error; - throw SWIGVMContainers::SWIGVM::exception(sb.str().c_str()); - } - return res; -} -#endif - diff --git a/exaudfclient/base/protobuf_repository.bzl b/exaudfclient/base/protobuf_repository.bzl deleted file mode 100644 index eb98c85..0000000 --- a/exaudfclient/base/protobuf_repository.bzl +++ /dev/null @@ -1,32 +0,0 @@ -def _protobuf_local_repository_impl(repository_ctx): - if 'PROTOBUF_LIBRARY_PREFIX' in repository_ctx.os.environ: - library_prefix = repository_ctx.os.environ['PROTOBUF_LIBRARY_PREFIX'] - else: - fail("Environment Variable PROTOBUF_LIBRARY_PREFIX not found") - print("protobuf library prefix in environment specified; %s"%library_prefix) - - if 'PROTOBUF_INCLUDE_PREFIX' in repository_ctx.os.environ: - include_prefix = repository_ctx.os.environ['PROTOBUF_INCLUDE_PREFIX'] - else: - fail("Environment Variable PROTOBUF_INCLUDE_PREFIX not found") - print("protobuf include prefix in environment specified; %s"%include_prefix) - build_file_content = """ -load("@rules_cc//cc:defs.bzl", "cc_library") - -cc_library( - name = "protobuf", - srcs = glob(["protobuf/lib/**/libprotobuf*.so"]), - hdrs = glob(["protobuf/include/**"]), - includes = ["protobuf/include/"], - visibility = ["//visibility:public"] -)""" - print(build_file_content) - - repository_ctx.symlink(library_prefix, "./protobuf/lib") - repository_ctx.symlink(include_prefix, "./protobuf/include") - repository_ctx.file("BUILD", build_file_content) - -protobuf_local_repository = repository_rule( - implementation=_protobuf_local_repository_impl, - local = True, - environ = ["PROTOBUF_LIBRARY_PREFIX","PROTOBUF_BIN","PROTOBUF_INCLUDE_PREFIX"]) diff --git a/exaudfclient/base/python/exascript_python_wrap.py b/exaudfclient/base/python/exascript_python_wrap.py index d79fd1b..a49bbab 100644 --- a/exaudfclient/base/python/exascript_python_wrap.py +++ b/exaudfclient/base/python/exascript_python_wrap.py @@ -12,8 +12,9 @@ #print("sys.path append",path_to_pyexadataframe) sys.path.append(path_to_pyexadataframe) else: - #Path "external/exaudfclient_base+/" comes from external module "exaudfclient_base" - path_to_pyexadataframe="/exaudf/external/exaudfclient_base+/python/python3" + # The Python VM is built by this module, while the shared transport lives in + # the external exaudfclient_base module. + path_to_pyexadataframe="/exaudf/base/python/python3" #print("sys.path append",path_to_pyexadataframe) sys.path.append(path_to_pyexadataframe) @@ -357,4 +358,3 @@ def __pythonvm_wrapped_singleCall(fn,arg=None): "F-UDF-CL-SL-PYTHON-1118", "Exception during singleCall %s"%fn.__name__, sys.exc_info()) - diff --git a/exaudfclient/base/python/python3/BUILD b/exaudfclient/base/python/python3/BUILD index cb62d1a..d41d5eb 100644 --- a/exaudfclient/base/python/python3/BUILD +++ b/exaudfclient/base/python/python3/BUILD @@ -7,17 +7,17 @@ genrule( cmd = """ INCLUDES=`$$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION-config --includes` mkdir -p build_exascript_python_tmp_cc/exaudflib - cp "$(location //exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //exaudflib:exascript.i)" build_exascript_python_tmp_cc/exaudflib + cp "$(location @exaudfclient_base//exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location @exaudfclient_base//exaudflib:exascript.i)" build_exascript_python_tmp_cc/exaudflib cd build_exascript_python_tmp_cc swig -v $$INCLUDES -O -DEXTERNAL_PROCESS -Wall -c++ -python -py3 -addextern -module exascript_python -o "../$(location exascript_python.cc)" exaudflib/exascript.i #Run replace_swig_import_helper.py because the generated exascript_python.py is not compatible with Python 3.12 #TODO: Update Swig to a new version which produces code compatible with Python 3.12 cd .. - $$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION "$(location //:replace_swig_import_helper.py)" "$(location exascript_python.py)" "$(location exascript_python.py)" + $$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION "$(location @exaudfclient_base//:replace_swig_import_helper.py)" "$(location exascript_python.py)" "$(location exascript_python.py)" """, outs = ["exascript_python.cc", "exascript_python.py"], - srcs = ["//exaudflib:exascript.i","//exaudflib:swig/script_data_transfer_objects_wrapper.h"], - tools = ["//:replace_swig_import_helper.py"], + srcs = ["@exaudfclient_base//exaudflib:exascript.i","@exaudfclient_base//exaudflib:swig/script_data_transfer_objects_wrapper.h"], + tools = ["@exaudfclient_base//:replace_swig_import_helper.py"], ) genrule( @@ -31,39 +31,39 @@ genrule( genrule( name = "extend_exascript_python_preset_py", - cmd = 'bash $(location //python:extend_exascript_python_preset_py.sh) "$(location //python:exascript_python_preset_core.py)" "$(location exascript_python_preset.py)" "$$PYTHON3_PREFIX" "$$PYTHON3_VERSION" ""', + cmd = 'bash $(location //base/python:extend_exascript_python_preset_py.sh) "$(location //base/python:exascript_python_preset_core.py)" "$(location exascript_python_preset.py)" "$$PYTHON3_PREFIX" "$$PYTHON3_VERSION" ""', outs = ["exascript_python_preset.py"], - srcs = ["//python:exascript_python_preset_core.py"], - tools = ["//python:extend_exascript_python_preset_py.sh"] + srcs = ["//base/python:exascript_python_preset_core.py"], + tools = ["//base/python:extend_exascript_python_preset_py.sh"] ) genrule( name = "exascript_python_int", cmd = """ cp $(SRCS) . - $$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION $(location //:build_integrated.py) "$(location exascript_python_int.h)" "exascript_python.py" "exascript_python_wrap.py" "exascript_python_preset.py" + $$PYTHON3_PREFIX/bin/$$PYTHON3_VERSION $(location @exaudfclient_base//:build_integrated.py) "$(location exascript_python_int.h)" "exascript_python.py" "exascript_python_wrap.py" "exascript_python_preset.py" """, outs = ["exascript_python_int.h"], - srcs = [":exascript_python_tmp_cc", "//python:exascript_python_wrap.py", ":extend_exascript_python_preset_py"], - tools = ["//:build_integrated.py"] + srcs = [":exascript_python_tmp_cc", "//base/python:exascript_python_wrap.py", ":extend_exascript_python_preset_py"], + tools = ["@exaudfclient_base//:build_integrated.py"] ) cc_library( name = "exascript_python", srcs = [":exascript_python_tmp_cc", ":exascript_python_tmp_h"], hdrs = [":exascript_python_tmp_h"], - deps = ["@python3//:python3","//exaudflib:exaudflib-deps","//exaudflib:header"], + deps = ["@python3//:python3","@exaudfclient_base//exaudflib:exaudflib-deps","@exaudfclient_base//exaudflib:header"], alwayslink=True, ) cc_library( name = "pythoncontainer", - srcs = ["//python:pythoncontainer.cc", ":dummy.h"], + srcs = ["//base/python:pythoncontainer.cc", ":dummy.h"], data = [":extend_exascript_python_preset_py"], hdrs = [":exascript_python_int", ":exascript_python_tmp_h"], include_prefix = ".", - deps = ["@python3//:python3",":exascript_python","//exaudflib:header", - "//utils:utils", "//python:pythoncontainer_header"], + deps = ["@python3//:python3",":exascript_python","@exaudfclient_base//exaudflib:header", + "@exaudfclient_base//utils:utils", "//base/python:pythoncontainer_header"], alwayslink=True, ) @@ -80,5 +80,5 @@ cc_binary( linkshared = 1, srcs = [":python_ext_dataframe.cc"], deps = ["@python3//:python3","@numpy//:numpy", - "//exaudflib:header", "//utils:utils"], + "@exaudfclient_base//exaudflib:header", "@exaudfclient_base//utils:utils"], ) diff --git a/exaudfclient/base/replace_swig_import_helper.py b/exaudfclient/base/replace_swig_import_helper.py deleted file mode 100644 index 3be049e..0000000 --- a/exaudfclient/base/replace_swig_import_helper.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -This script replaces the function `swig_import_helper` -in the automatically generated `exascript_python.py` with a simple import statement. -This is needed for Python 3.12, as the automatically generated code uses the library `imp` -which was removed in this Python version. -#TODO: Update Swig to a new version which produces code compatible with Python 3.12 -""" -import sys -from pathlib import Path - -old_function = """ - def swig_import_helper(): - from os.path import dirname - import imp - fp = None - try: - fp, pathname, description = imp.find_module('_exascript_python', [dirname(__file__)]) - except ImportError: - import _exascript_python - return _exascript_python - if fp is not None: - try: - _mod = imp.load_module('_exascript_python', fp, pathname, description) - finally: - fp.close() - return _mod - _exascript_python = swig_import_helper() - del swig_import_helper -""" - -new_function = """ - import _exascript_python -""" - -def replace_swig_import_helper(target, source): - source_code = Path(source).read_text() - - assert old_function in source_code - - new_source_code = source_code.replace(old_function, new_function) - - with open(target, 'w') as f: - f.write(new_source_code) - -if __name__ == "__main__": - if len(sys.argv) != 3: - print('Usage: replace_swig_import_helper.py target source') - sys.exit(1) - print(f"replacing function swig_import_helper() using source={sys.argv[2]} and target={sys.argv[1]}") - replace_swig_import_helper(sys.argv[1], sys.argv[2]) diff --git a/exaudfclient/base/streaming_container/BUILD b/exaudfclient/base/streaming_container/BUILD deleted file mode 100644 index 2107368..0000000 --- a/exaudfclient/base/streaming_container/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load("@rules_cc//cc:defs.bzl", "cc_library") - -cc_library( - name = "streamingcontainer", - srcs = ["streamingcontainer.cc","streamingcontainer.h"], - hdrs = ["streamingcontainer.h"], - deps = ["//exaudflib:exaudflib-deps","//exaudflib:header"] -) \ No newline at end of file diff --git a/exaudfclient/base/streaming_container/streamingcontainer.cc b/exaudfclient/base/streaming_container/streamingcontainer.cc deleted file mode 100644 index 57e4f9d..0000000 --- a/exaudfclient/base/streaming_container/streamingcontainer.cc +++ /dev/null @@ -1,344 +0,0 @@ -#include "streamingcontainer.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace SWIGVMContainers; -using namespace std; - - -bool mexec(const std::string& cmd_, std::string& result) { - char buffer[128]; - std::stringstream cmd; - cmd << "ulimit -v 500000; "; - cmd << cmd_ << " 2>&1"; - - FILE* pipe = popen(cmd.str().c_str(), "r"); - if (!pipe) { - result = "Cannot start command `" + cmd.str() + "`"; - return false; - } - while (!feof(pipe)) { - if (fgets(buffer, 128, pipe) != NULL) { - result += buffer; - } - } - int s = pclose(pipe); - if (s == -1) - { - return false; - } - if (WEXITSTATUS(s)) - { - return false; - } - return true; -} - - -StreamingVM::StreamingVM(bool checkOnly) - : meta(), inp(), outp(&inp), - csvPrinters{ -{UNSUPPORTED,[](ostream&, unsigned int) -> void { - throw SWIGVM::exception("Trying to serialize UNSUPPORTED datatype to CSV"); -}}, -{DOUBLE,[&](ostream&str, unsigned int col) {str << inp.getDouble(col);}}, -{INT32,[&](ostream&str, unsigned int col) {str << inp.getInt32(col);}}, -{INT64,[&](ostream&str, unsigned int col) {str << inp.getInt64(col);}}, -{NUMERIC,[&](ostream&str, unsigned int col) {str << inp.getNumeric(col);}}, -{TIMESTAMP,[&](ostream&str, unsigned int col) {str << inp.getTimestamp(col);}}, -{DATE,[&](ostream&str, unsigned int col) {str << inp.getDate(col);}}, -{STRING,[&](ostream&str, unsigned int col) { - //string o = inp.getString(col); - //std::cerr << "Sending Input: [" << o << "] for column " << col << std::endl; - str << inp.getString(col); -}}, -{BOOLEAN,[&](ostream&str, unsigned int col) {str << (inp.getBoolean(col)?"true":"false");}}, -{INTERVALYM,[&](ostream&, unsigned int) { - throw SWIGVM::exception("Trying to serialize INTERVALYM datatype to CSV"); -}}, -{INTERVALDS,[&](ostream&, unsigned int) { - throw SWIGVM::exception("Trying to serialize INTERVALDS datatype to CSV"); -}}, -{GEOMETRY,[&](ostream&, unsigned int) { - throw SWIGVM::exception("Trying to serialize GEOMETRY datatype to CSV"); -}} -}, - - -csvReaders{ - -// {UNSUPPORTED,[](ostream&, unsigned int) -> void { -// throw SWIGVM::exception("Trying to serialize UNSUPPORTED datatype to CSV"); -// }}, -// {DOUBLE,[&](ostream&str, unsigned int col) {str << inp.getDouble(col);}}, -// {INT32,[&](ostream&str, unsigned int col) {str << inp.getInt32(col);}}, -// {INT64,[&](ostream&str, unsigned int col) {str << inp.getInt64(col);}}, -// {NUMERIC,[&](ostream&str, unsigned int col) {str << inp.getNumeric(col);}}, -// {TIMESTAMP,[&](ostream&str, unsigned int col) {str << inp.getTimestamp(col);}}, -// {DATE,[&](ostream&str, unsigned int col) {str << inp.getDate(col);}}, - {STRING,[&](string&str, unsigned int col) { - std::cerr << "******* SETTING RESULT: [" << str << "] for column " << col << std::endl; - outp.setString(col,str.c_str(),-1);}} -// , -// {BOOLEAN,[&](ostream&str, unsigned int col) {str << (inp.getBoolean(col)?"true":"false");}}, -// {INTERVALYM,[&](ostream&, unsigned int) { -// throw SWIGVM::exception("Trying to serialize INTERVALYM datatype to CSV"); -// }}, -// {INTERVALDS,[&](ostream&, unsigned int) { -// throw SWIGVM::exception("Trying to serialize INTERVALDS datatype to CSV"); -// }}, -// {GEOMETRY,[&](ostream&, unsigned int) { -// throw SWIGVM::exception("Trying to serialize GEOMETRY datatype to CSV"); -// }} - -} - -{ - - - if (meta.inputType() != MULTIPLE - || meta.outputType() != MULTIPLE) - { - throw SWIGVM::exception("STREAMING language container only support SET-EMITS UDFs"); - } - - - ofstream f("/tmp/program"); - f << meta.scriptCode(); - f.close(); - - string res; - if (!mexec("chmod +x /tmp/program",res)) { - std::stringstream msg; - msg << "Error when making script code executable: " << res; - throw SWIGVM::exception(msg.str().c_str()); - } - - readBuffer.reserve(1024 * 1024 * 2); -} - -void StreamingVM::shutdown() -{ - -} - -void StreamingVM::inputToCSV(ostream&str) -{ - if (meta.inputColumnCount()>0) { - csvPrinters[meta.inputColumnType(0)](str,0); - for (unsigned int col = 1; col 0) { - outp.next(); // make room for the data unless it is the first emitted row - } - csvReaders[meta.outputColumnType(0)](readBuffer,0); - for (unsigned int col = 1; col outFilebuf(pproc_in[PIPE_WRITE_END], std::ios::out); - ostream os(&outFilebuf); - - __gnu_cxx::stdio_filebuf inFilebuf(pproc_out[PIPE_READ_END], std::ios::in); - istream is(&inFilebuf); - - __gnu_cxx::stdio_filebuf errorFilebuf(pproc_error[PIPE_READ_END], std::ios::in); - istream es(&errorFilebuf); - - thread t1([&]() - { - try { - do { - inputToCSV(os); - } while (inp.next()); - os.flush(); - close(pproc_in[PIPE_WRITE_END]); - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - exception_msg = err.what(); - } - }); - - thread t2([&]() - { - try { - while (!is.eof()) { - if (!CSVToOutput(is)) { - break; - } - } - outp.flush(); - } catch (std::exception& err) { - lock_guard lock(exception_msg_mtx); - exception_msg = err.what(); - } - }); - - t1.join(); - t2.join(); - - - close(pproc_out[PIPE_READ_END]); - - if (exception_msg.size() == 0) { - readBuffer.clear(); // error messages from the subprocess? - while (readBuffer.size()<1000) { - int current = es.get(); - if (current == string::traits_type::eof()) { - break; - } - readBuffer += current; - } - - if (!es.eof() && readBuffer.size()>0) { - readBuffer += "..."; - } - } - close(pproc_error[PIPE_READ_END]); - if (waitpid(pproc_pid,NULL,0) < 0) { - perror("waitpid"); - exit(EXIT_FAILURE); - } - - if (exception_msg.size() > 0) { - throw SWIGVM::exception(exception_msg.c_str()); - } - - if (readBuffer.size()>0) { - string error_msg = "Error output during script execution:\n"+readBuffer; - throw SWIGVM::exception(error_msg.c_str()); - } - - - } catch (std::exception& ex) { - lock_guard lock(exception_msg_mtx); - exception_msg = ex.what(); - } - return true; // done -} - -const char* StreamingVM::singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args) -{ - throw SWIGVM::exception("singleCall not supported for the STREAMING language container"); -} \ No newline at end of file diff --git a/exaudfclient/base/streaming_container/streamingcontainer.h b/exaudfclient/base/streaming_container/streamingcontainer.h deleted file mode 100644 index b45faac..0000000 --- a/exaudfclient/base/streaming_container/streamingcontainer.h +++ /dev/null @@ -1,30 +0,0 @@ - -using namespace SWIGVMContainers; -using namespace std; - - -class StreamingVM: public SWIGVM { - public: - struct exception: SWIGVM::exception { - exception(const char *reason): SWIGVM::exception(reason) { } - virtual ~exception() throw() { } - }; - StreamingVM(bool checkOnly); - virtual ~StreamingVM() {}; - virtual void shutdown(); - virtual bool run(); - virtual const char* singleCall(single_call_function_id_e fn, const ExecutionGraph::ScriptDTO& args); - virtual bool useZmqSocketLocks() {return true;} - private: - SWIGMetadata meta; - SWIGTableIterator inp; - SWIGResultHandler outp; - void importScripts(); - map > csvPrinters; - map > csvReaders; - void inputToCSV(ostream&str); - bool CSVToOutput(istream&str); - string readBuffer; - // returns true if a an entry could be read - bool readCSVValue(istream&str, unsigned int column); -}; diff --git a/exaudfclient/base/swig_factory/BUILD b/exaudfclient/base/swig_factory/BUILD deleted file mode 100644 index 19e5f85..0000000 --- a/exaudfclient/base/swig_factory/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -load("@rules_cc//cc:defs.bzl", "cc_library") - -cc_library( - name = "swig_factory_if", - hdrs = [":swig_factory.h"], -) - - -cc_library( - name = "swig_factory", - hdrs = [":swig_factory.h", ":swig_factory_impl.h"], - srcs = [":swig_factory_impl.cc"], - deps = ["//exaudflib:header"], -) diff --git a/exaudfclient/base/swig_factory/swig_factory.h b/exaudfclient/base/swig_factory/swig_factory.h deleted file mode 100644 index 48f93c1..0000000 --- a/exaudfclient/base/swig_factory/swig_factory.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef SWIG_FACTORY_H -#define SWIG_FACTORY_H 1 - -namespace SWIGVMContainers { - -struct SWIGMetadataIf; - -struct SwigFactory { - virtual ~SwigFactory() {}; - - virtual SWIGMetadataIf* makeSwigMetadata() = 0; - -}; - - -} // namespace SWIGVMContainer - -#endif //SWIG_FACTORY_H \ No newline at end of file diff --git a/exaudfclient/base/swig_factory/swig_factory_impl.cc b/exaudfclient/base/swig_factory/swig_factory_impl.cc deleted file mode 100644 index b4b3207..0000000 --- a/exaudfclient/base/swig_factory/swig_factory_impl.cc +++ /dev/null @@ -1,19 +0,0 @@ -#include "swig_factory/swig_factory_impl.h" -#include "exaudflib/load_dynamic.h" -#include "exaudflib/swig/swig_common.h" - -namespace SWIGVMContainers { - -SWIGMetadataIf* SwigFactoryImpl::makeSwigMetadata() { - SWIGMetadataIf* impl=nullptr; - typedef SWIGVMContainers::SWIGMetadataIf* (*CREATE_METADATA_FUN)(); -#ifndef UDF_PLUGIN_CLIENT - CREATE_METADATA_FUN create = (CREATE_METADATA_FUN)load_dynamic("create_SWIGMetaData"); - impl = create(); -#else - impl = create_SWIGMetaData(); -#endif - return impl; -} - -} //namespace SWIGVMContainers diff --git a/exaudfclient/base/swig_factory/swig_factory_impl.h b/exaudfclient/base/swig_factory/swig_factory_impl.h deleted file mode 100644 index 5bfbb00..0000000 --- a/exaudfclient/base/swig_factory/swig_factory_impl.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef SWIG_FACTORY_IMPL_H -#define SWIG_FACTORY_IMPL_H 1 - -#include "swig_factory/swig_factory.h" - -namespace SWIGVMContainers { - -struct SwigFactoryImpl : public SwigFactory { - - virtual SWIGMetadataIf* makeSwigMetadata() override; - -}; - -} // namespace SWIGVMContainers - - -#endif //namespace SWIG_FACTORY_IMPL_H \ No newline at end of file diff --git a/exaudfclient/base/utils/BUILD b/exaudfclient/base/utils/BUILD deleted file mode 100644 index ddf4e03..0000000 --- a/exaudfclient/base/utils/BUILD +++ /dev/null @@ -1,12 +0,0 @@ - -package(default_visibility = ["//visibility:public"]) - -load("@rules_cc//cc:defs.bzl", "cc_library") - -cc_library( - name = "utils", - hdrs = [ - "debug_message.h", "exceptions.h" - ], -) - diff --git a/exaudfclient/base/utils/debug_message.h b/exaudfclient/base/utils/debug_message.h deleted file mode 100644 index e74d892..0000000 --- a/exaudfclient/base/utils/debug_message.h +++ /dev/null @@ -1,71 +0,0 @@ -#ifndef DEBUG_MESSAGE_H -#define DEBUG_MESSAGE_H - -#define PRINT_EXCEPTION( os, error_code, ex ) \ - (os) << (error_code) << ":" << " Caught Exception: " << (ex.what()) << std::endl - -#define PRINT_ERROR_MESSAGE( os, error_code, error_message ) \ - (os) << (error_code) << ":" << error_message << std::endl - -#ifndef NDEBUG - -#define DBG_EXCEPTION( os, ex ) \ - (os) << "DBG: " << __FILE__ << "(" << __LINE__ << ") in " << __func__ << " " \ - << "EXCEPTION: " << #ex << " = [" << (ex.what()) << "]" << std::endl - -#define DBGVAR( os, var ) \ - (os) << "DBG: " << __FILE__ << "(" << __LINE__ << ") in " << __func__ << " " \ - << "VAR: " << #var << " = [" << (var) << "]" << std::endl - -#define DBGMSG( os, msg ) \ - (os) << "DBG: " << __FILE__ << "(" << __LINE__ << ") in " << __func__ << " " \ - << (msg) << std::endl - -#define DBG_STREAM_MSG( os, msg ) \ - (os) << "DBG: " << __FILE__ << "(" << __LINE__ << ") in " << __func__ << " " \ - << msg << std::endl - - -#define DBG_FUNC_BEGIN( os ) \ - (os) << "DBG: " << __FILE__ << "(" << __LINE__ << ") BEGIN FUNC: " \ - << __func__ << std::endl - -#define DBG_FUNC_END( os ) \ - (os) << "DBG: " << __FILE__ << "(" << __LINE__ << ") END FUNC: " \ - << __func__ << std::endl - -#define DBG_FUNC_CALL( os, call ) \ - (os) << "DBG: " << __FILE__ << "(" << __LINE__ << ") " \ - << "CALL BEGIN: " << #call << std::endl; \ - call; \ - (os) << "DBG: " << __FILE__ << "(" << __LINE__ << ") " \ - << "CALL END: " << #call << std::endl - -#define DBG_COND_FUNC_CALL( os, call ) \ - (os) << "DBG: " << __FILE__ << "(" << __LINE__ << ") " \ - << "CALL BEGIN: " << #call << std::endl; \ - call; \ - (os) << "DBG: " << __FILE__ << "(" << __LINE__ << ") " \ - << "CALL END: " << #call << std::endl - -#else - -#define DBG_EXCEPTION( os, ex ) - -#define DBGVAR( os, var ) - -#define DBGMSG( os, msg ) - -#define DBG_STREAM_MSG( os, msg ) - -#define DBG_FUNC_BEGIN( os ) - -#define DBG_FUNC_END( os ) - -#define DBG_FUNC_CALL( os, call ) call - -#define DBG_COND_FUNC_CALL( os, call ) - -#endif - -#endif diff --git a/exaudfclient/base/utils/exceptions.h b/exaudfclient/base/utils/exceptions.h deleted file mode 100644 index ec37a87..0000000 --- a/exaudfclient/base/utils/exceptions.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef UTILS_EXCEPTION_H -#define UTILS_EXCEPTION_H - -#include -#include - -namespace Utils { - -template -inline void rethrow(const TException& sourceException, const std::string_view& prefix) { - throw TException(std::string(prefix) + " " + sourceException.what()); -} - -} //namespace Utils - -#endif //UTILS_EXCEPTION_H \ No newline at end of file diff --git a/exaudfclient/base/variables.bzl b/exaudfclient/base/variables.bzl deleted file mode 100644 index a7ebd5d..0000000 --- a/exaudfclient/base/variables.bzl +++ /dev/null @@ -1,17 +0,0 @@ -BENCHMARK_VM_ENABLED_DEFINE=select({ - "//:benchmark": ["ENABLE_BENCHMARK_VM"], - "//conditions:default": [] - }) -STREAMING_VM_ENABLED_DEFINE=select({ - "//:bash": ["ENABLE_STREAMING_VM"], - "//conditions:default": [] - }) -PYTHON_VM_ENABLED_DEFINE=select({ - "//:python": ["ENABLE_PYTHON_VM"], - "//conditions:default": [] - }) -PLUGIN_CLIENT_ENABLED_DEFINE=select({ - "//:plugin_client": ["UDF_PLUGIN_CLIENT"], - "//conditions:default": [] - }) -VM_ENABLED_DEFINES=BENCHMARK_VM_ENABLED_DEFINE+PYTHON_VM_ENABLED_DEFINE+STREAMING_VM_ENABLED_DEFINE+PLUGIN_CLIENT_ENABLED_DEFINE diff --git a/exaudfclient/base/zmq_repository.bzl b/exaudfclient/base/zmq_repository.bzl deleted file mode 100644 index dbab068..0000000 --- a/exaudfclient/base/zmq_repository.bzl +++ /dev/null @@ -1,32 +0,0 @@ -def _zmq_local_repository_impl(repository_ctx): - if 'ZMQ_LIBRARY_PREFIX' in repository_ctx.os.environ: - library_prefix = repository_ctx.os.environ['ZMQ_LIBRARY_PREFIX'] - else: - fail("Environment Variable ZMQ_LIBRARY_PREFIX not found") - print("zmq library prefix in environment specified; %s"%library_prefix) - - if 'ZMQ_INCLUDE_PREFIX' in repository_ctx.os.environ: - include_prefix = repository_ctx.os.environ['ZMQ_INCLUDE_PREFIX'] - else: - fail("Environment Variable ZMQ_INCLUDE_PREFIX not found") - print("zmq include prefix in environment specified; %s"%include_prefix) - build_file_content = """ -load("@rules_cc//cc:defs.bzl", "cc_library") - -cc_library( - name = "zmq", - srcs = glob(["zmq/lib/**/libzmq.so"]), - hdrs = glob(["zmq/include/zmq*"]), - includes = ["zmq/include/"], - visibility = ["//visibility:public"] -)""" - print(build_file_content) - - repository_ctx.symlink(library_prefix, "./zmq/lib") - repository_ctx.symlink(include_prefix, "./zmq/include") - repository_ctx.file("BUILD", build_file_content) - -zmq_local_repository = repository_rule( - implementation=_zmq_local_repository_impl, - local = True, - environ = ["ZMQ_LIBRARY_PREFIX","ZMQ_INCLUDE_PREFIX"]) diff --git a/exaudfclient/docs/diagrams/CTPGParserHandler.drawio.png b/exaudfclient/docs/diagrams/CTPGParserHandler.drawio.png deleted file mode 100644 index 1b4836e..0000000 Binary files a/exaudfclient/docs/diagrams/CTPGParserHandler.drawio.png and /dev/null differ diff --git a/exaudfclient/docs/diagrams/LegacyParserHandler.drawio.png b/exaudfclient/docs/diagrams/LegacyParserHandler.drawio.png deleted file mode 100644 index 0907f6d..0000000 Binary files a/exaudfclient/docs/diagrams/LegacyParserHandler.drawio.png and /dev/null differ diff --git a/exaudfclient/docs/diagrams/OveralScriptOptionalsBuildingBlocks.drawio.png b/exaudfclient/docs/diagrams/OveralScriptOptionalsBuildingBlocks.drawio.png deleted file mode 100644 index b834931..0000000 Binary files a/exaudfclient/docs/diagrams/OveralScriptOptionalsBuildingBlocks.drawio.png and /dev/null differ diff --git a/exaudfclient/docs/diagrams/ScriptOptionsExtractorInterface.drawio.png b/exaudfclient/docs/diagrams/ScriptOptionsExtractorInterface.drawio.png deleted file mode 100644 index 6583634..0000000 Binary files a/exaudfclient/docs/diagrams/ScriptOptionsExtractorInterface.drawio.png and /dev/null differ diff --git a/exaudfclient/docs/diagrams/ScriptOptionsParserHandlerSequence.drawio.png b/exaudfclient/docs/diagrams/ScriptOptionsParserHandlerSequence.drawio.png deleted file mode 100644 index dbaac3f..0000000 Binary files a/exaudfclient/docs/diagrams/ScriptOptionsParserHandlerSequence.drawio.png and /dev/null differ diff --git a/exaudfclient/docs/diagrams/V2ImportScriptFlow.drawio.png b/exaudfclient/docs/diagrams/V2ImportScriptFlow.drawio.png deleted file mode 100644 index 9ddd77e..0000000 Binary files a/exaudfclient/docs/diagrams/V2ImportScriptFlow.drawio.png and /dev/null differ diff --git a/exaudfclient/docs/exaudfclient.drawio b/exaudfclient/docs/exaudfclient.drawio deleted file mode 100644 index d686c21..0000000 --- a/exaudfclient/docs/exaudfclient.drawio +++ /dev/null @@ -1 +0,0 @@ -7V1Zc9s4Ev41rtp9kAoHLz3GRzLeimdSk0zG2RcVLUEyxxSppSgf8+uXkEiJAJo3KDGZ5CUWCFFko/Gh++tG44JerV4/RO768S6cM/+CoPnrBb2+IMk/myT/8Za3fYtj033DMvLm+yZ8bPjs/c3SRpS2br052wgd4zD0Y28tNs7CIGCzWGhzoyh8EbstQl/81bW7ZErD55nrq61/evP4MX0LEx3bf2He8jH7ZYzSKys365w2bB7defiSa6I3F/QqCsN4/9fq9Yr5XHiZXPbfe19w9fBgEQviOl8g0Yfbm+jX2+f//nU5DUd/mDeb5xF29rd5dv1t+sbp08ZvmQiicBvMGb8LvqCXL49ezD6v3Rm/+pIMetL2GK/89PIiDOJ0FLHBP3u+fxX6YbS7F52bzJnz9k0chU8sd8UhD9Sykivqe6Wv+syimL3mmtL3/MDCFYujt6RLenWEaSr0VOuwY+0/vxzHEBtpn8fc+Fm2kepOqjfLw82Pok3+SKXbRNJGE0mjZpLGeuRGqSg1pAiNGLYqNGyj7kJbWtefwnX0n6f1l+nT1LycTr7SEa2WWYWU3M16DwsL75VL9jwKaZqCYA0LkixWJXto7CLZl9/R+uab7c/dZxe/4V/+wBN7RADJWn6c6pUgYut/2zC7MNrsNO5d0oE669fjxeSvJf+fvbrb+cL3HrLbJU+3v+P+ejed5xL3EnB+53vLIGmLw7UuzZcAw1ABw7EAwDisBNp1Xzsyi4q/cGZsNoMU/8ExDRNpkqslytW0VLnaDiDXwxe1a76KKY/Jz/rseWe5bMLZE+P6v9gGs9gLg003jdUgQ8MUZehQVYYEQYtZXyK0VNU0Lz+6wfLCvP56d8ENkf3kf4iyeX8bxCxacCmdW5yybWASY2ybqnlAAInqmOygRLOltlCkijxXa3+gWKoI2DHHpKaAe1NZjFUBy+JjwfwddxmSTzPf3Wy8mSgxUbyFktqE22jGagx27EZLFlfPMzYXfBRV8DmpmiVaGzHfjb1n0bOBJJ3+wqfQ25kCGQ4hcVyNiXSL/Yun38p7IvKNJEAzZCtnLxjlRruBP7x2B9unhlnZTBfYqxff8ytjhK308zd+bWyaJP18/ZrrfP2WfQiSd7nPf9h/z0JW1nD84u5T9k3ZkHWZswDXc2vmsIdF9v1PLPISAbIo9zRyWy3V/nAz+bS4uYq3X+/IH/cf738zbv8eYVNVbbL9/SG+nH99en/97frT6mn02/rLaDIo1R6hseGg4z/sEEFDE2dgjOjk+M9qqfmSOWQQCe8KND9RRfct123NO2yK38cRf8ZOvdqix5K7U1Tandil/ZM/9s/bdtKC/jNVJ+124yXrI0F+6M6n87fAXXmzf/1bncqJjn10HxLzTvQM03VvxoKd5isL4sqbz/k9LiOWeD3uw+5+fHqk0k9ubl4my7Pki5vZ5/QJiDqbylFJXmgP5Fb6CBd5/gikP9AYO9juNjOyLuFisWH9wLDKiGw3TLW320FxAr05IMY1QXhMkZkH4kSUKNG8ciQGcbUWhpZBY6V1QAcFoQYWQcHCmjDy4DK2xsgOigsOkFltPrTmTDU5ORYVGSdipXAgKAOgDQcOUD/jZIHInMFkGMWP4TIMXP/m2CrN7WOfjyH3VnbS+4vF8VsqPncbh6JsD1MbXwj2lVkxp2tNX/g9AROo0/ytPTG7DY+q1Z9fvOVVopuuF7Bos2P9ZDeUd7ljsXvtxu5075NCXX5nm60f/+IGyYIalfT7wpe42wRE3Tgs6lcy187p9TqGxNIQgKWBXF5Hg8sLEohAwOFAKSTth5FVBJq8cSxKTfQpgjBgkgOSNtW3qaBhEgcyj4qaaHhLjG8YmR2bGyKI46V9jZCtjBBI6wxuPLiMtTBFCIvLlIUQRBWddFAmXXGwBgTWQD8A+PjDWO6KD82u4ZO30xaUrHkE8Rvc399P+yIGzxI3s6TAGSWqGUPxKUEVIGpnEXPjnvyWg6vSiDyqMm5qh4IuHYzpwcE9OLRqmL9wrCutoEygkhlUw09yYNU5jbsj80FZTOYQVJiMbfGedf0fk0j+jxx00MQRjSyR47Ll/BGpv8TqN+xupUiviyOCJydRJmeeGxo/qqDKw3zJN+ZsAeJ6BbV08lAgkdy67BaVcRXJh9aHhzXSWhoCYUuP7QigFIkcPCJmOYwCFLgEkQg5LkIQRCJk2qi2x1hmceShslS9B8L42FIYh8pkdmvE68z41HtgYgq0eGdMAiMiVDUYvGDmb+eQxXB6proAgYo4Z1HBSxChEMJ4aMvGwjiMcDd91EpVw+8EZDx0Q7kW0bssxihgWwWyHYltNCECnmrltPPQBc8CB4SuuuB4LoibyPE63BLiJlKIzZHdV00QN5kYIianvG+voTkCpao1SybEJpRMmN3FyxqKIn6HVENP/mpV+uGJIXcCzqJyzNERD0zWZmNAIAv6c2qA94MfPrj8e1/dyONvu1Et9OsCy306jR8Tx3zOqZE/bz/s0tKmidDdlbruNs0hLsTBJinqUryMAmQkhiI0h8ZOKerQAKiL3ObFW05n4WoVBpDzxBmv+Y7xQp/jaDuLgfHJ9bkJthqE30O0TOEhExw992CoNOQ+tXrme2yHZ7K+r1wvGKJwDSlbxHLwWKV4oRxgHSQeKFv9yYCy3GBm7nClwOjaJFZAnP1qSuHv2t57/AWbcXeHNO42hmPhuLdKbizzegfizJo2GSPLPBB6jmj3WW1THSW7zyD9uLbUKKXbKvsb6BR2oopphdz5uTzhvMqXY4cWswxJikaGZKSB29iUIfTDlwFvnrClpGWbWMDq09v2CXge9L76QIjfSySp/TpB1WWiTOEGskzIxoyJWma5yZynYfazMFgK8WCVPpfc3zZPsDBoz8cvmhBIMIEqwgLtdRtKAfup3PqV25HA3TEE5e5HWbUHvGqhN9+i1gbAiRYEr5un/FPJuyh5B1sN3oyjaKoXeLGqrae1tY/pIi2jTsWzsszUxo7IjA8q6AS+kpqQu9zzoRDdyYlxn125vj+9v79Xr/OR5ylin3YM6JEzP/Z4ibgtPou8dXzsdG7rXc4iOeXWZ3BQ9AcCazFHGf7Xyn+w62eQtbdx6trv5qDR37B1oX9PxI4l+69OedqVZXbrbzm4fxNKzXvubRKNHSps6KqiYI+zyEATwQBDzZOI2m8M6+ZW/Jxz3YLoRPaB9e5vhTMkVFaIhy8HRI3qNNfs0mU/sdaoVIuKdFPF/q01ldve/RD6yhu6JocXuohjZFEBpSh1eiH6WpcJ6OQ6/gSyTkBmTiQgQ5Py55L6W9YJ+BQgGHu3C2W/53vVtrw+I0EfveCJb5tBv7ortll3riMkT6jmu1sqtvi3yEGQI3IXsqfTW8UceFzUBenmWNzt1CNSI87d94hgoB7iiYdE9T4/smWylCiiH9yWwfJdQg2ySaQxOd0OTnhEytN0xonKDn9o4KTE5iltEyKabEgla046ONnclBLa/jkjIm+fJPa5R6RG4lXr0hmahEYlK48iFWMIIqrUdKSrgX6/SpdwJneX0TqcsopyphOlhiI2uNBnX7qm2i9cZP+UuW9QWY1PN/fhxEB1PJTRyHnFqZRr7Jc+urw7gm+MnMwlrnCBpQ2GlaFS2SRdMAs2Sef25AEpCZ1HPz+f07l/iHkuw7O1q10GHt95rUmlHHDbWpNU3sGjy9VOEyjlEt+FrrbU36SnyLNRQ4pe8JioWjwgmrGyUN4BSHRkYBI7Kz46QKIRJIqbwWiT4qUCkVgrOUrZzqdp+3Px9pSzAZJk3phGS0AypXXZmNQDJF3q099+fV4rUdxjOsZ2zXBbI90rCas1VchSL2XYGkklEpnK/kddjVQcnp72plryjjexgkhPJUFqFGtsqe9obNlibDnRf6emwqOx4ziC0iM8Kdd6rQHmTpVHbXCynWsiyNEUSz6OpnUdCjk18OwZfUCQcZewpWZzhWtt+y8AO0tRrmI2xRHjuCAtYKq6ocMNhZMitVNQLQujdRLr5LCLJTNbiSpXMDih46wmWLCqYSoUXxosFa5hOLCEHASoU9cX2QIPhmrm8WKBbMUCaJO2/jJX3cSJFPU2gQNEoMRPHeWLS0820HZ+mwY5JXOOSEsfhs+yIaqkcO5YFu07rNWKILeZ+rl8YVJV8Hl15z4BxWgHsIN9hJEoZtsgUJnSvvaww/qoxoLzkUe4HMAPA7gjOVPFUQvhnTSyRbX71TI7ptgTlOJ3SPQpMGlx6IFEg9dKNO83FawEUQbicyi7iFrTQXIqWE06qKnzLfNXTgppvea0YjVyCZ/w8SPsQKKVW5CQjUVnKCPjBkw2qykUHWGtSwLqiVjq2nVNin30c+ESlo9Wa1uwjlQF4DThknyWVLbfpe7ZU9QQUlr7wTEgbeWHwLGSKV+KY6Yt4diQ9lLCNHB/R/41jlzsTTV0jI3sv8r3NuqsyVl9Vp+hItr3UHdYsb3aBj4MmbWxJXe451BcpoMFJM13Di4l87AMXDCiQ0ITeOBUNNnz8FNe6XAaR26wWbBoGj78xWbAULZOjCfn4pvlsjCH7LI8ywQ53Dpy4eEx6NXhJoJbPZmIm6xQRYJZUaTuXrR0v+Wv1cs5q12HXksgHDuNVoizBQDlA6flopy1nXFZy3s6esNwJHYb6d1nBW8whcqB/CCF6EsQohDS0Nig1uCXGuDYW40wZ4jsoXRsxt5CbQRzGsEH2OxeshgPBnvaJqrK2GPWTFTVpmgqV+cFm9gNYm8IFTr78XVpuTnKE9iQ6OoOHzB6tYvygKGlPlm98mSnZfD+4YjSuPawXEKm4vh4uX8WAil002WBNOwvpeL3ZF1pzzMsXqcpneSmHk7ckYqaD9oW6rr52wOcVrZcvKEtM372hbrXEmTFAN9r/UktAA9sdRqgIn53+K7kbZuk9Lnk/oah9+jH0gOmupxBZENnEO2OQFm4szjkg1pwqJB6RnnDn8bQT++L40/58TV36dnCUOHGiqfS9gzC8cTnfBDhKGXh+KfSo55as66lvkeDnF1HtpPscaa2eRoVwh6MbS3JevDcUdPKA/aicVHJeIT73JLQLEcJjR0k8Kajw7Ge7QJmTRecUtCp3DCxH7qhrDhW64M65RVHTiZvbfokH6OQz/dj92RKPt6Fc04w3Pwf \ No newline at end of file diff --git a/exaudfclient/docs/exaudfclient.png b/exaudfclient/docs/exaudfclient.png deleted file mode 100644 index 701fa73..0000000 Binary files a/exaudfclient/docs/exaudfclient.png and /dev/null differ diff --git a/exaudfclient/exa_udf_client.cc b/exaudfclient/exa_udf_client.cc new file mode 100644 index 0000000..8359c04 --- /dev/null +++ b/exaudfclient/exa_udf_client.cc @@ -0,0 +1,25 @@ +#include +#include + +#include "exa_udf_client.h" +#include "exa_vm_factory.h" + +void ExaUdfClient::parse_arguments(int argc, char** argv) { + m_languageArg = argv[2]; +} + +bool ExaUdfClient::validate_arguments(int argc, char** argv) { + if (argc != 3 || strcmp(argv[2], "lang=python") != 0) { + usage(argv[0]); + return false; + } + return true; +} + +void ExaUdfClient::usage(const std::string& programName) { + std::cerr << "Usage: " << programName << " lang=python" << std::endl; +} + +std::function ExaUdfClient::create_vm() { + return ::create_vm(m_languageArg); +} diff --git a/exaudfclient/exa_udfclient_binary_common/exa_udf_client.h b/exaudfclient/exa_udf_client.h similarity index 88% rename from exaudfclient/exa_udfclient_binary_common/exa_udf_client.h rename to exaudfclient/exa_udf_client.h index 062c3f5..8fc8881 100644 --- a/exaudfclient/exa_udfclient_binary_common/exa_udf_client.h +++ b/exaudfclient/exa_udf_client.h @@ -1,7 +1,8 @@ #pragma once + #include -#include "exa_udf_base.h" +#include "exa_udfclient_binary_common/exa_udf_base.h" #include "exaudflib/vm/swig_vm.h" class ExaUdfClient : public ExaUdfClientBase { @@ -11,7 +12,7 @@ class ExaUdfClient : public ExaUdfClientBase { void usage(const std::string& programName) override; bool validate_arguments(int argc, char** argv) override; std::function create_vm() override; - + protected: std::string m_languageArg; }; diff --git a/exaudfclient/exa_udfclient.cc b/exaudfclient/exa_udfclient.cc index 37aae3a..4a1935c 100644 --- a/exaudfclient/exa_udfclient.cc +++ b/exaudfclient/exa_udfclient.cc @@ -1,6 +1,6 @@ -#include "exa_udfclient_binary_common/exa_udf_client.h" +#include "exa_udf_client.h" int main(int argc, char **argv) { ExaUdfClient client; return client.startClientBase(argc, argv); -} \ No newline at end of file +} diff --git a/exaudfclient/exa_udfclient_binary_common/exa_lib_loader.cc b/exaudfclient/exa_udfclient_binary_common/exa_lib_loader.cc deleted file mode 100644 index de05b03..0000000 --- a/exaudfclient/exa_udfclient_binary_common/exa_lib_loader.cc +++ /dev/null @@ -1,36 +0,0 @@ -#include -#include -#include -#include - -#include "exa_lib_loader.h" -#include "utils/debug_message.h" - -namespace exa_lib { - void* load_library(const std::string& stdLibPath) { - if(stdLibPath.empty()) { - return nullptr; - } - - void* handle = dlmopen(LM_ID_NEWLM, stdLibPath.c_str(), RTLD_NOW); - if(!handle) { - std::cerr << "dlmopen error: " << dlerror() << "; while loading " << stdLibPath << std::endl; - return nullptr; - } - return handle; - } - - void* load_symbol(void *handle, const std::string& symbol_name) { - void *p_res = nullptr; - char *error = nullptr; - if(handle) { - p_res = dlsym(handle, symbol_name.c_str()); - - if((error = dlerror()) != nullptr) { - std::cerr << "Error when trying to load symbol '" << symbol_name << "': " << error << std::endl; - return nullptr; - } - } - return p_res; - } -} diff --git a/exaudfclient/exa_udfclient_binary_common/exa_lib_loader.h b/exaudfclient/exa_udfclient_binary_common/exa_lib_loader.h deleted file mode 100644 index 4bc0c88..0000000 --- a/exaudfclient/exa_udfclient_binary_common/exa_lib_loader.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once -#include - -namespace exa_lib { - void* load_library(const std::string& stdLibPath); - void* load_symbol(void *handle, const std::string& symbol_name); -} \ No newline at end of file diff --git a/exaudfclient/exa_udfclient_binary_common/exa_set_env.cc b/exaudfclient/exa_udfclient_binary_common/exa_set_env.cc deleted file mode 100644 index 5718d1b..0000000 --- a/exaudfclient/exa_udfclient_binary_common/exa_set_env.cc +++ /dev/null @@ -1,22 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include "exaudflib/vm/swig_vm.h" -#include "exa_set_env.h" - -namespace exa_env { - void setup_environment() { - if (::setenv("HOME", "/tmp", 1) == -1) { - std::cerr << "Failed setting HOME env var: " << std::strerror(errno) << std::endl; - throw SWIGVMContainers::SWIGVM::exception("Failed to set HOME directory"); - } - if (::setlocale(LC_ALL, "en_US.utf8") == nullptr) { - std::cerr << "Failed setting locale: " << std::strerror(errno) << std::endl; - throw SWIGVMContainers::SWIGVM::exception("Failed to set locale"); - } - } -} \ No newline at end of file diff --git a/exaudfclient/exa_udfclient_binary_common/exa_set_env.h b/exaudfclient/exa_udfclient_binary_common/exa_set_env.h deleted file mode 100644 index 9f6c822..0000000 --- a/exaudfclient/exa_udfclient_binary_common/exa_set_env.h +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once - -namespace exa_env { - void setup_environment(); -} diff --git a/exaudfclient/exa_udfclient_binary_common/exa_udf_base.cc b/exaudfclient/exa_udfclient_binary_common/exa_udf_base.cc deleted file mode 100644 index 56710fb..0000000 --- a/exaudfclient/exa_udfclient_binary_common/exa_udf_base.cc +++ /dev/null @@ -1,49 +0,0 @@ -#include -#include -#include -#include - -#include "exaudf_lib_output_path.h" -#include "exaudflib/vm/swig_vm.h" -#include "exa_udf_base.h" -#include "exa_lib_loader.h" -#include "exa_set_env.h" -#include "exaudflib/load_dynamic.h" -#include "utils/debug_message.h" - -namespace SWIGVMContainers { -__thread SWIGVM_params_t * SWIGVM_params = nullptr; -} - -int ExaUdfClientBase::startClientBase(int argc, char** argv) { - if (!validate_arguments(argc, argv)) { - usage(argv[0]); - exit(EXIT_FAILURE); - } - parse_arguments(argc, argv); - - std::string libexaudflibPath; -#ifdef CUSTOM_LIBEXAUDFLIB_PATH - libexaudflibPath = std::string(CUSTOM_LIBEXAUDFLIB_PATH); -#else - libexaudflibPath = std::string(::getenv("LIBEXAUDFLIB_PATH")); -#endif - DBGMSG(std::cerr, "Load libexaudflib"); - DBGVAR(std::cerr, libexaudflibPath); - void* exaudflib_handle = exa_lib::load_library(libexaudflibPath); - if (!exaudflib_handle) { - std::cerr << "Failed to load library: " << libexaudflibPath << std::endl; - exit(EXIT_FAILURE); - } - set_exaudflib_handle(exaudflib_handle); - - MAIN_FUN exaudfclient_main = (MAIN_FUN)exa_lib::load_symbol(exaudflib_handle, "exaudfclient_main"); - SET_SWIGVM_PARAMS set_SWIGVM_params = (SET_SWIGVM_PARAMS)exa_lib::load_symbol(exaudflib_handle, "set_SWIGVM_params"); - - exa_env::setup_environment(); - std::function vmMaker = create_vm(); - - SWIGVMContainers::SWIGVM_params = new SWIGVMContainers::SWIGVM_params_t(true); - set_SWIGVM_params(SWIGVMContainers::SWIGVM_params); - return exaudfclient_main(vmMaker, argc, argv); -} \ No newline at end of file diff --git a/exaudfclient/exa_udfclient_binary_common/exa_udf_base.h b/exaudfclient/exa_udfclient_binary_common/exa_udf_base.h deleted file mode 100644 index 59eb30b..0000000 --- a/exaudfclient/exa_udfclient_binary_common/exa_udf_base.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once -#include -#include - -#include "exaudflib/vm/swig_vm.h" - -typedef bool (*SET_SWIGVM_PARAMS)(SWIGVMContainers::SWIGVM_params_t*); -typedef int (*MAIN_FUN)(std::function vmMaker, int, char**); - -class ExaUdfClientBase { -public: - virtual ~ExaUdfClientBase() = default; - virtual void usage(const std::string& programName) = 0; - virtual std::function create_vm() = 0; - virtual bool validate_arguments(int argc, char** argv) = 0; - virtual void parse_arguments(int argc, char** argv) = 0; - virtual int startClientBase(int argc, char** argv); -}; \ No newline at end of file diff --git a/exaudfclient/exa_udfclient_binary_common/exa_udf_client.cc b/exaudfclient/exa_udfclient_binary_common/exa_udf_client.cc deleted file mode 100644 index 3b5ea09..0000000 --- a/exaudfclient/exa_udfclient_binary_common/exa_udf_client.cc +++ /dev/null @@ -1,37 +0,0 @@ -#include -#include - -#include "exa_udf_client.h" -#include "exa_vm_factory.h" - -void ExaUdfClient::parse_arguments(int argc, char** argv) { - // Now assumption is that all cmd line aguments are already validated. - m_languageArg = argv[2]; - -} - -bool ExaUdfClient::validate_arguments(int argc, char** argv) { - if (argc != 3) { - usage(argv[0]); - return false; - } - - if (!((strcmp(argv[2], "lang=python") == 0) - || (strcmp(argv[2], "lang=streaming") == 0) - || (strcmp(argv[2], "lang=benchmark") == 0))) { - usage(argv[0]); - return false; - } - - return true; -} - -void ExaUdfClient::usage(const std::string& programName) { - std::cerr << "Usage: " << programName - << " lang=python|lang=streaming|lang=benchmark" - << std::endl; -} - -std::function ExaUdfClient::create_vm() { - return ::create_vm(m_languageArg, false); -} diff --git a/exaudfclient/exa_udfclient_binary_common/exa_vm_factory.cc b/exaudfclient/exa_udfclient_binary_common/exa_vm_factory.cc deleted file mode 100644 index b6a3770..0000000 --- a/exaudfclient/exa_udfclient_binary_common/exa_vm_factory.cc +++ /dev/null @@ -1,50 +0,0 @@ -#include -#include -#include "exa_vm_factory.h" - -#ifdef ENABLE_PYTHON_VM -#include "python/pythoncontainer.h" -#endif //ENABLE_PYTHON_VM - -#ifdef ENABLE_STREAMING_VM -#include "streaming_container/streamingcontainer.h" -#endif - -#ifdef ENABLE_BENCHMARK_VM -#include "benchmark_container/benchmark_container.h" -#endif - -std::function create_vm(const std::string& argv_lang, bool use_ctpg_options_parser) { - if(argv_lang.compare("lang=python") == 0) { - #ifdef ENABLE_PYTHON_VM - char *path_var = getenv("PATH"); - if(path_var != NULL) { - std::string path_var_str = std::string(path_var); - path_var_str.insert(0, "/opt/conda/bin:"); - if (::setenv("PATH", path_var_str.c_str(), 1) == -1) { - std::cerr << "Unable to prefix PATH env variable with /opt/conda/bin"; - } - } - return []() { return new SWIGVMContainers::PythonVM(false); }; - #else - throw SWIGVMContainers::SWIGVM::exception("this exaudfclient has been compilied without Python support"); - #endif - } - else if(argv_lang.compare("lang=streaming") == 0) { - #ifdef ENABLE_STREAMING_VM - return []() { return new SWIGVMContainers::StreamingVM(false); }; - #else - throw SWIGVMContainers::SWIGVM::exception("this exaudfclient has been compilied without Streaming support"); - #endif - } - else if(argv_lang.compare("lang=benchmark") == 0) { - #ifdef ENABLE_BENCHMARK_VM - return []() { return new SWIGVMContainers::BenchmarkVM(false); }; - #else - throw SWIGVMContainers::SWIGVM::exception("this exaudfclient has been compilied without Benchmark support"); - #endif - } - else { - throw SWIGVMContainers::SWIGVM::exception("unsupported language specified in argv"); - } -} diff --git a/exaudfclient/exa_vm_factory.cc b/exaudfclient/exa_vm_factory.cc new file mode 100644 index 0000000..1133971 --- /dev/null +++ b/exaudfclient/exa_vm_factory.cc @@ -0,0 +1,28 @@ +#include +#include +#include + +#include "exa_vm_factory.h" + +#ifdef ENABLE_PYTHON_VM +#include "base/python/pythoncontainer.h" +#endif + +std::function create_vm(const std::string& argv_lang) { + if (argv_lang != "lang=python") { + throw SWIGVMContainers::SWIGVM::exception("unsupported language specified in argv"); + } +#ifdef ENABLE_PYTHON_VM + char *path_var = getenv("PATH"); + if (path_var != nullptr) { + std::string path_var_str = std::string(path_var); + path_var_str.insert(0, "/opt/conda/bin:"); + if (setenv("PATH", path_var_str.c_str(), 1) == -1) { + std::cerr << "Unable to prefix PATH env variable with /opt/conda/bin"; + } + } + return []() { return new SWIGVMContainers::PythonVM(false); }; +#else + throw SWIGVMContainers::SWIGVM::exception("this exaudfclient has been compiled without Python support"); +#endif +} diff --git a/exaudfclient/exa_udfclient_binary_common/exa_vm_factory.h b/exaudfclient/exa_vm_factory.h similarity index 64% rename from exaudfclient/exa_udfclient_binary_common/exa_vm_factory.h rename to exaudfclient/exa_vm_factory.h index 7d9e306..617bd81 100644 --- a/exaudfclient/exa_udfclient_binary_common/exa_vm_factory.h +++ b/exaudfclient/exa_vm_factory.h @@ -1,4 +1,8 @@ #pragma once + +#include +#include + #include "exaudflib/vm/swig_vm.h" -std::function create_vm(const std::string& argv_lang, bool use_ctpg_options_parser); +std::function create_vm(const std::string& argv_lang); diff --git a/exaudfclient/base/exaudfclient.template.sh b/exaudfclient/exaudfclient.template.sh similarity index 82% rename from exaudfclient/base/exaudfclient.template.sh rename to exaudfclient/exaudfclient.template.sh index 68d5af9..7790862 100644 --- a/exaudfclient/base/exaudfclient.template.sh +++ b/exaudfclient/exaudfclient.template.sh @@ -3,7 +3,7 @@ SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" echo "Changing to script directory $SCRIPT_DIR" cd "$SCRIPT_DIR" || return 1 -export LIBPYEXADATAFRAME_DIR="$SCRIPT_DIR/external/exaudfclient_base+/python/python3" +export LIBPYEXADATAFRAME_DIR="$SCRIPT_DIR/base/python/python3" if [[ -d /opt/conda ]] then diff --git a/exaudfclient/run_local.sh b/exaudfclient/run_local.sh index 68368ec..f7a5565 100644 --- a/exaudfclient/run_local.sh +++ b/exaudfclient/run_local.sh @@ -4,4 +4,4 @@ #shellcheck disable=SC1091 source .env export VERBOSE_BUILD="--subcommands --verbose_failures" -bash run.sh --define streaming=true --define python=true --define benchmark=true "$@" +bash run.sh --define python=true "$@" diff --git a/exaudfclient/variables.bzl b/exaudfclient/variables.bzl index 0d5fc1b..62e8c42 100644 --- a/exaudfclient/variables.bzl +++ b/exaudfclient/variables.bzl @@ -1,13 +1,5 @@ -BENCHMARK_VM_ENABLED_DEFINE=select({ - "@exaudfclient_base//:benchmark": ["ENABLE_BENCHMARK_VM"], - "//conditions:default": [] - }) -STREAMING_VM_ENABLED_DEFINE=select({ - "@exaudfclient_base//:bash": ["ENABLE_STREAMING_VM"], - "//conditions:default": [] - }) PYTHON_VM_ENABLED_DEFINE=select({ - "@exaudfclient_base//:python": ["ENABLE_PYTHON_VM"], + "//:python": ["ENABLE_PYTHON_VM"], "//conditions:default": [] }) -VM_ENABLED_DEFINES=BENCHMARK_VM_ENABLED_DEFINE+PYTHON_VM_ENABLED_DEFINE+STREAMING_VM_ENABLED_DEFINE +VM_ENABLED_DEFINES=PYTHON_VM_ENABLED_DEFINE diff --git a/flavors/template-Exasol-8-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-8-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile index 555207c..a812477 100644 --- a/flavors/template-Exasol-8-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-8-python-3.10-cuda-conda/flavor_base/base_test_build_run/Dockerfile @@ -1,5 +1,7 @@ FROM {{base_test_deps}} +ARG UDF_RUNNER_CPP_COMMIT=b3a8c204a2555b373c66110d5e8301a374b32a6a + RUN mkdir /conf /buckets RUN exaslpm export-variables --out-file /env @@ -20,5 +22,5 @@ RUN ./test_udfclient.sh /exaudf/exaudfclient_static WORKDIR / RUN mkdir /exasol_emulator COPY emulator/ /exasol_emulator -COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator +RUN curl -fsSL "https://raw.githubusercontent.com/exasol/udf-runner-cpp/${UDF_RUNNER_CPP_COMMIT}/udf-runner-cpp/v1/base/exaudflib/zmqcontainer.proto" -o /exasol_emulator/zmqcontainer.proto RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=. diff --git a/flavors/template-Exasol-8-python-3.10-cuda-conda/flavor_base/packages.yml b/flavors/template-Exasol-8-python-3.10-cuda-conda/flavor_base/packages.yml index e948dec..d3aa9e6 100644 --- a/flavors/template-Exasol-8-python-3.10-cuda-conda/flavor_base/packages.yml +++ b/flavors/template-Exasol-8-python-3.10-cuda-conda/flavor_base/packages.yml @@ -1,26 +1,39 @@ build_steps: - name: build_deps phases: + - name: install_apt_packages + apt: + packages: + - name: git + version: '*' + validation_cfg: + version_mandatory: true - name: install_conda_packages conda: channels: - conda-forge packages: - name: protobuf - version: =7.35.1 + version: =* - name: pcre - version: '=8.45' - - name: bazel - version: =9.2.0 + version: =* - name: make - version: '=4.3' + version: =* - name: chrpath - version: '=0.16' + version: =* - name: gxx - version: =13.4.0 + version: =* + - name: binutils + version: =* - name: gcc - version: =13.4.0 + version: =* binary: Mamba + - name: install_bazel + tools: + bazel: + version: '8.3.1' + validation_cfg: + version_mandatory: true variables: PROTOBUF_BIN: '$MAMBA_ROOT_PREFIX/bin/protoc' LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/' @@ -37,17 +50,17 @@ build_steps: apt: packages: - name: gdb - version: 12.1-0ubuntu* + version: '*' - name: valgrind - version: 1:3.18.1-1ubuntu* + version: '*' - name: gdbserver - version: 12.1-0ubuntu* + version: '*' - name: binutils - version: 2.38-4ubuntu* + version: '*' - name: patchelf - version: 0.14.3-1 + version: '*' - name: strace - version: 5.16-0ubuntu* + version: '*' validation_cfg: version_mandatory: true - name: security_scan @@ -68,4 +81,4 @@ build_steps: - name: trivy validation_cfg: version_mandatory: false -version: 1.0.0 +version: '*' diff --git a/flavors/template-Exasol-8-python-3.10-cuda-conda/packages.yml b/flavors/template-Exasol-8-python-3.10-cuda-conda/packages.yml index ec542ce..b7cb82f 100644 --- a/flavors/template-Exasol-8-python-3.10-cuda-conda/packages.yml +++ b/flavors/template-Exasol-8-python-3.10-cuda-conda/packages.yml @@ -5,17 +5,17 @@ build_steps: apt: packages: - name: coreutils - version: 8.32-4.1ubuntu* + version: '*' - name: locales - version: 2.35-0ubuntu* + version: '*' - name: curl - version: 7.81.0-1ubuntu* + version: '*' - name: ca-certificates - version: 20260601~22.04.1 + version: '*' - name: bzip2 - version: 1.0.8-5build1 + version: '*' - name: libxml2 - version: 2.9.13+dfsg-1ubuntu* + version: '*' - name: install_micromamba tools: micromamba: @@ -33,13 +33,13 @@ build_steps: - conda-forge packages: - name: libprotobuf - version: =7.35.1 + version: =* - name: zeromq - version: =4.3.5 + version: =* - name: nss - version: '=3.118' + version: =* - name: cppzmq - version: =4.11.0 + version: =* binary: Micromamba variables: PROTOBUF_LIBRARY_PREFIX: $MAMBA_ROOT_PREFIX/lib @@ -61,7 +61,7 @@ build_steps: - nvidia packages: - name: python - version: =3.10.20 + version: =3.10.* - name: numpy version: =2.2.6 - name: pandas @@ -69,11 +69,11 @@ build_steps: - name: pyarrow version: =25.0.0 - name: cuda-toolkit - version: =13.3.1 + version: =* - name: libarrow - version: =25.0.0=*_cuda + version: =*=*_cuda - name: mamba - version: =2.8.1 + version: =* binary: Micromamba variables: PYTHON3_PREFIX: $MAMBA_ROOT_PREFIX @@ -90,7 +90,7 @@ build_steps: apt: packages: - name: tzdata - version: 2026* + version: '*' - name: install_conda_packages conda: channels: diff --git a/flavors/template-Exasol-8-python-3.12-cuda-conda/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-8-python-3.12-cuda-conda/flavor_base/base_test_build_run/Dockerfile index 26623b7..fcfb71f 100644 --- a/flavors/template-Exasol-8-python-3.12-cuda-conda/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-8-python-3.12-cuda-conda/flavor_base/base_test_build_run/Dockerfile @@ -1,5 +1,7 @@ FROM {{base_test_deps}} +ARG UDF_RUNNER_CPP_COMMIT=b3a8c204a2555b373c66110d5e8301a374b32a6a + RUN mkdir /conf /buckets RUN exaslpm export-variables --out-file /env @@ -22,5 +24,5 @@ RUN ./test_udfclient.sh /exaudf/exaudfclient_static WORKDIR / RUN mkdir /exasol_emulator COPY emulator/ /exasol_emulator -COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator +RUN curl -fsSL "https://raw.githubusercontent.com/exasol/udf-runner-cpp/${UDF_RUNNER_CPP_COMMIT}/udf-runner-cpp/v1/base/exaudflib/zmqcontainer.proto" -o /exasol_emulator/zmqcontainer.proto RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=. diff --git a/flavors/template-Exasol-8-python-3.12-cuda-conda/flavor_base/packages.yml b/flavors/template-Exasol-8-python-3.12-cuda-conda/flavor_base/packages.yml index c835ece..d3aa9e6 100644 --- a/flavors/template-Exasol-8-python-3.12-cuda-conda/flavor_base/packages.yml +++ b/flavors/template-Exasol-8-python-3.12-cuda-conda/flavor_base/packages.yml @@ -1,26 +1,39 @@ build_steps: - name: build_deps phases: + - name: install_apt_packages + apt: + packages: + - name: git + version: '*' + validation_cfg: + version_mandatory: true - name: install_conda_packages conda: channels: - conda-forge packages: - name: protobuf - version: =7.35.1 + version: =* - name: pcre - version: '=8.45' - - name: bazel - version: =9.2.0 + version: =* - name: make - version: =4.4.1 + version: =* - name: chrpath - version: '=0.16' + version: =* - name: gxx - version: =13.4.0 + version: =* + - name: binutils + version: =* - name: gcc - version: =13.4.0 + version: =* binary: Mamba + - name: install_bazel + tools: + bazel: + version: '8.3.1' + validation_cfg: + version_mandatory: true variables: PROTOBUF_BIN: '$MAMBA_ROOT_PREFIX/bin/protoc' LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/' @@ -37,17 +50,17 @@ build_steps: apt: packages: - name: gdb - version: 15.0.50.20240403-0ubuntu* + version: '*' - name: valgrind - version: 1:3.22.0-0ubuntu* + version: '*' - name: gdbserver - version: 15.0.50.20240403-0ubuntu* + version: '*' - name: binutils - version: 2.42-4ubuntu* + version: '*' - name: patchelf - version: 0.18.0-1.1build1 + version: '*' - name: strace - version: 6.8-0ubuntu* + version: '*' validation_cfg: version_mandatory: true - name: security_scan @@ -68,4 +81,4 @@ build_steps: - name: trivy validation_cfg: version_mandatory: false -version: 1.0.0 +version: '*' diff --git a/flavors/template-Exasol-8-python-3.12-cuda-conda/packages.yml b/flavors/template-Exasol-8-python-3.12-cuda-conda/packages.yml index 6487624..d3c91d0 100644 --- a/flavors/template-Exasol-8-python-3.12-cuda-conda/packages.yml +++ b/flavors/template-Exasol-8-python-3.12-cuda-conda/packages.yml @@ -5,19 +5,19 @@ build_steps: apt: packages: - name: coreutils - version: 9.4-3ubuntu* + version: '*' - name: adduser - version: 3.137ubuntu* + version: '*' - name: locales - version: 2.39-0ubuntu* + version: '*' - name: curl - version: 8.5.0-2ubuntu* + version: '*' - name: ca-certificates - version: '20260601~24.04.1' + version: '*' - name: bzip2 - version: 1.0.8-5.1build0.1 + version: '*' - name: libxml2 - version: 2.9.14+dfsg-1.3ubuntu* + version: '*' - name: install_micromamba tools: micromamba: @@ -35,13 +35,13 @@ build_steps: - conda-forge packages: - name: libprotobuf - version: =7.35.1 + version: =* - name: zeromq - version: =4.3.5 + version: =* - name: nss - version: '=3.118' + version: =* - name: cppzmq - version: =4.11.0 + version: =* binary: Micromamba variables: PROTOBUF_LIBRARY_PREFIX: $MAMBA_ROOT_PREFIX/lib @@ -63,19 +63,19 @@ build_steps: - nvidia packages: - name: python - version: =3.12.13 + version: =3.12.* - name: numpy - version: =2.5.1 + version: =2.2.6 - name: pandas version: =2.3.3 - name: pyarrow version: =25.0.0 - name: libarrow - version: =25.0.0=*_cuda + version: =*=*_cuda - name: mamba - version: =2.8.1 + version: =* - name: cuda-toolkit - version: =13.3.1 + version: =* binary: Micromamba variables: PYTHON3_PREFIX: $MAMBA_ROOT_PREFIX @@ -92,7 +92,7 @@ build_steps: apt: packages: - name: tzdata - version: 2026* + version: '*' - name: install_conda_packages conda: channels: diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile index 0680c99..8c2cc00 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/base_test_build_run/Dockerfile @@ -1,5 +1,7 @@ FROM {{base_test_deps}} +ARG UDF_RUNNER_CPP_COMMIT=b3a8c204a2555b373c66110d5e8301a374b32a6a + RUN mkdir /conf /buckets RUN exaslpm export-variables --out-file /env @@ -21,5 +23,5 @@ RUN ./test_udfclient.sh /exaudf/exaudfclient_static WORKDIR / RUN mkdir /exasol_emulator COPY emulator/ /exasol_emulator -COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator +RUN curl -fsSL "https://raw.githubusercontent.com/exasol/udf-runner-cpp/${UDF_RUNNER_CPP_COMMIT}/udf-runner-cpp/v1/base/exaudflib/zmqcontainer.proto" -o /exasol_emulator/zmqcontainer.proto RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=. diff --git a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/packages.yml b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/packages.yml index 1789e61..f806aa8 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/flavor_base/packages.yml +++ b/flavors/template-Exasol-all-python-3.10-conda/flavor_base/packages.yml @@ -1,26 +1,39 @@ build_steps: - name: build_deps phases: + - name: install_apt_packages + apt: + packages: + - name: git + version: '*' + validation_cfg: + version_mandatory: true - name: install_conda_packages conda: channels: - conda-forge packages: - name: protobuf - version: =7.35.1 + version: =* - name: pcre - version: '=8.45' - - name: bazel - version: =9.2.0 + version: =* - name: gxx - version: =12.4.0 + version: =* - name: gcc - version: =12.4.0 + version: =* - name: make - version: '=4.3' + version: =* + - name: binutils + version: =* - name: chrpath - version: '=0.16' + version: =* binary: Mamba + - name: install_bazel + tools: + bazel: + version: '8.3.1' + validation_cfg: + version_mandatory: true variables: PROTOBUF_BIN: /opt/conda/bin/protoc LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/' @@ -37,17 +50,17 @@ build_steps: apt: packages: - name: gdb - version: 12.1-0ubuntu* + version: '*' - name: valgrind - version: 1:3.18.1-1ubuntu* + version: '*' - name: gdbserver - version: 12.1-0ubuntu* + version: '*' - name: binutils - version: 2.38-4ubuntu* + version: '*' - name: patchelf - version: 0.14.3-1 + version: '*' - name: strace - version: 5.16-0ubuntu* + version: '*' validation_cfg: version_mandatory: true - name: security_scan @@ -68,4 +81,4 @@ build_steps: - name: trivy validation_cfg: version_mandatory: false -version: 1.0.0 +version: '*' diff --git a/flavors/template-Exasol-all-python-3.10-conda/packages.yml b/flavors/template-Exasol-all-python-3.10-conda/packages.yml index 8910fab..87e3f20 100644 --- a/flavors/template-Exasol-all-python-3.10-conda/packages.yml +++ b/flavors/template-Exasol-all-python-3.10-conda/packages.yml @@ -5,17 +5,17 @@ build_steps: apt: packages: - name: coreutils - version: 8.32-4.1ubuntu* + version: '*' - name: locales - version: 2.35-0ubuntu* + version: '*' - name: curl - version: 7.81.0-1ubuntu* + version: '*' - name: ca-certificates - version: 20260601~22.04.1 + version: '*' - name: bzip2 - version: 1.0.8-5build1 + version: '*' - name: libxml2 - version: 2.9.13+dfsg-1ubuntu* + version: '*' - name: install_micromamba tools: micromamba: @@ -33,13 +33,13 @@ build_steps: - conda-forge packages: - name: libprotobuf - version: =7.35.1 + version: =* - name: zeromq - version: =4.3.5 + version: =* - name: nss - version: '=3.118' + version: =* - name: cppzmq - version: =4.11.0 + version: =* binary: Micromamba variables: PROTOBUF_LIBRARY_PREFIX: $MAMBA_ROOT_PREFIX/lib @@ -56,7 +56,7 @@ build_steps: - conda-forge packages: - name: python - version: =3.10.20 + version: =3.10.* - name: numpy version: =2.2.6 - name: pandas @@ -64,7 +64,7 @@ build_steps: - name: pyarrow version: =25.0.0 - name: mamba - version: =2.8.1 + version: =* binary: Micromamba variables: PYTHON3_PREFIX: $MAMBA_ROOT_PREFIX @@ -81,7 +81,7 @@ build_steps: apt: packages: - name: tzdata - version: 2026* + version: '*' - name: install_conda_packages conda: channels: diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile index 5637382..d155d3e 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/base_test_build_run/Dockerfile @@ -1,5 +1,7 @@ FROM {{language_deps}} +ARG UDF_RUNNER_CPP_COMMIT=b3a8c204a2555b373c66110d5e8301a374b32a6a + RUN mkdir /conf /buckets COPY --from={{base_test_deps}} /usr /usr @@ -38,5 +40,5 @@ RUN ./test_udfclient.sh /exaudf/exaudfclient_static WORKDIR / RUN mkdir /exasol_emulator COPY emulator/ /exasol_emulator -COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator +RUN curl -fsSL "https://raw.githubusercontent.com/exasol/udf-runner-cpp/${UDF_RUNNER_CPP_COMMIT}/udf-runner-cpp/v1/base/exaudflib/zmqcontainer.proto" -o /exasol_emulator/zmqcontainer.proto RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=. diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/packages.yml b/flavors/template-Exasol-all-python-3.10/flavor_base/packages.yml index 2219ef1..7d61142 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/packages.yml +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/packages.yml @@ -5,17 +5,17 @@ build_steps: apt: packages: - name: gdb - version: 12.1-0ubuntu* + version: '*' - name: valgrind - version: 1:3.18.1-1ubuntu* + version: '*' - name: gdbserver - version: 12.1-0ubuntu* + version: '*' - name: binutils - version: 2.38-4ubuntu* + version: '*' - name: patchelf - version: 0.14.3-1 + version: '*' - name: strace - version: 5.16-0ubuntu* + version: '*' validation_cfg: version_mandatory: true - name: build_deps @@ -23,26 +23,28 @@ build_steps: - name: install_apt_packages apt: packages: + - name: git + version: '*' - name: coreutils - version: 8.32-4.1ubuntu* + version: '*' - name: locales - version: 2.35-0ubuntu* + version: '*' - name: tar - version: 1.34+dfsg-1ubuntu* + version: '*' - name: curl - version: 7.81.0-1ubuntu* + version: '*' - name: openjdk-11-jdk-headless - version: 11.0.32+9-1ubuntu* + version: '*' - name: ca-certificates - version: 20260601~22.04.1 + version: '*' - name: build-essential - version: 12.9ubuntu* + version: '*' - name: libpcre3-dev - version: 2:8.39-13ubuntu* + version: '*' - name: protobuf-compiler - version: 3.12.4-1ubuntu* + version: '*' - name: chrpath - version: 0.16-2 + version: '*' variables: PROTOBUF_BIN: /usr/bin/protoc - name: install_bazel @@ -69,4 +71,4 @@ build_steps: - name: trivy validation_cfg: version_mandatory: false -version: 1.0.0 +version: '*' diff --git a/flavors/template-Exasol-all-python-3.10/packages.yml b/flavors/template-Exasol-all-python-3.10/packages.yml index 93ca8ee..9ce82ea 100644 --- a/flavors/template-Exasol-all-python-3.10/packages.yml +++ b/flavors/template-Exasol-all-python-3.10/packages.yml @@ -5,17 +5,17 @@ build_steps: apt: packages: - name: coreutils - version: 8.32-4.1ubuntu* + version: '*' - name: locales - version: 2.35-0ubuntu* + version: '*' - name: libnss-db - version: 2.2.3pre1-6ubuntu* + version: '*' - name: libzmq3-dev - version: 4.3.4-2 + version: '*' - name: libprotobuf-dev - version: 3.12.4-1ubuntu* + version: '*' - name: libssl-dev - version: 3.0.2-0ubuntu* + version: '*' variables: PROTOBUF_LIBRARY_PREFIX: /usr/lib/ PROTOBUF_INCLUDE_PREFIX: /usr/include/ @@ -29,13 +29,13 @@ build_steps: apt: packages: - name: ca-certificates - version: 20260601~22.04.1 + version: '*' - name: python3.10-dev - version: 3.10.12* + version: '*' - name: python3-distutils - version: 3.10.8* + version: '*' - name: curl - version: 7.81.0-1ubuntu* + version: '*' variables: PYTHON3_PREFIX: /usr PYTHON3_VERSION: python3.10 @@ -68,15 +68,15 @@ build_steps: apt: packages: - name: unzip - version: 6.0-26ubuntu* + version: '*' - name: git - version: 1:2.34.1-1ubuntu* + version: '*' - name: libcurl4-openssl-dev - version: 7.81.0-1ubuntu* + version: '*' - name: build-essential - version: 12.9ubuntu* + version: '*' - name: tzdata - version: 2026* + version: '*' validation_cfg: version_mandatory: true - name: flavor_customization diff --git a/flavors/template-Exasol-all-python-3.12-conda/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12-conda/flavor_base/base_test_build_run/Dockerfile index 555207c..a812477 100644 --- a/flavors/template-Exasol-all-python-3.12-conda/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12-conda/flavor_base/base_test_build_run/Dockerfile @@ -1,5 +1,7 @@ FROM {{base_test_deps}} +ARG UDF_RUNNER_CPP_COMMIT=b3a8c204a2555b373c66110d5e8301a374b32a6a + RUN mkdir /conf /buckets RUN exaslpm export-variables --out-file /env @@ -20,5 +22,5 @@ RUN ./test_udfclient.sh /exaudf/exaudfclient_static WORKDIR / RUN mkdir /exasol_emulator COPY emulator/ /exasol_emulator -COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator +RUN curl -fsSL "https://raw.githubusercontent.com/exasol/udf-runner-cpp/${UDF_RUNNER_CPP_COMMIT}/udf-runner-cpp/v1/base/exaudflib/zmqcontainer.proto" -o /exasol_emulator/zmqcontainer.proto RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=. diff --git a/flavors/template-Exasol-all-python-3.12-conda/flavor_base/packages.yml b/flavors/template-Exasol-all-python-3.12-conda/flavor_base/packages.yml index 38982d5..fed4dd7 100644 --- a/flavors/template-Exasol-all-python-3.12-conda/flavor_base/packages.yml +++ b/flavors/template-Exasol-all-python-3.12-conda/flavor_base/packages.yml @@ -1,26 +1,39 @@ build_steps: - name: build_deps phases: + - name: install_apt_packages + apt: + packages: + - name: git + version: '*' + validation_cfg: + version_mandatory: true - name: install_conda_packages conda: channels: - conda-forge packages: - name: protobuf - version: =7.35.1 + version: =* - name: pcre - version: '=8.45' - - name: bazel - version: =9.2.0 + version: =* - name: make - version: =4.4.1 + version: =* - name: chrpath - version: '=0.16' + version: =* - name: gxx - version: =15.1.0 + version: =* + - name: binutils + version: =* - name: gcc - version: =15.1.0 + version: =* binary: Mamba + - name: install_bazel + tools: + bazel: + version: '8.3.1' + validation_cfg: + version_mandatory: true variables: PROTOBUF_BIN: '$MAMBA_ROOT_PREFIX/bin/protoc' LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/' @@ -37,17 +50,17 @@ build_steps: apt: packages: - name: gdb - version: 15.0.50.20240403-0ubuntu* + version: '*' - name: valgrind - version: 1:3.22.0-0ubuntu* + version: '*' - name: gdbserver - version: 15.0.50.20240403-0ubuntu* + version: '*' - name: binutils - version: 2.42-4ubuntu* + version: '*' - name: patchelf - version: 0.18.0-1.1build1 + version: '*' - name: strace - version: 6.8-0ubuntu* + version: '*' validation_cfg: version_mandatory: true - name: security_scan @@ -68,4 +81,4 @@ build_steps: - name: trivy validation_cfg: version_mandatory: false -version: 1.0.0 +version: '*' diff --git a/flavors/template-Exasol-all-python-3.12-conda/packages.yml b/flavors/template-Exasol-all-python-3.12-conda/packages.yml index 8d62059..5ab0da2 100644 --- a/flavors/template-Exasol-all-python-3.12-conda/packages.yml +++ b/flavors/template-Exasol-all-python-3.12-conda/packages.yml @@ -5,19 +5,19 @@ build_steps: apt: packages: - name: coreutils - version: 9.4-3ubuntu* + version: '*' - name: adduser - version: 3.137ubuntu* + version: '*' - name: locales - version: 2.39-0ubuntu* + version: '*' - name: curl - version: 8.5.0-2ubuntu* + version: '*' - name: ca-certificates - version: '20260601~24.04.1' + version: '*' - name: bzip2 - version: 1.0.8-5.1build0.1 + version: '*' - name: libxml2 - version: 2.9.14+dfsg-1.3ubuntu* + version: '*' - name: install_micromamba tools: micromamba: @@ -35,13 +35,13 @@ build_steps: - conda-forge packages: - name: libprotobuf - version: =7.35.1 + version: =* - name: zeromq - version: =4.3.5 + version: =* - name: nss - version: '=3.118' + version: =* - name: cppzmq - version: =4.11.0 + version: =* binary: Micromamba variables: PROTOBUF_LIBRARY_PREFIX: $MAMBA_ROOT_PREFIX/lib @@ -58,15 +58,15 @@ build_steps: - conda-forge packages: - name: python - version: =3.12.13 + version: =3.12.* - name: numpy - version: =2.5.1 + version: =2.2.6 - name: pandas version: =2.3.3 - name: pyarrow version: =25.0.0 - name: mamba - version: =2.8.1 + version: =* binary: Micromamba variables: PYTHON3_PREFIX: $MAMBA_ROOT_PREFIX @@ -83,7 +83,7 @@ build_steps: apt: packages: - name: tzdata - version: 2026* + version: '*' - name: install_conda_packages conda: channels: diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile index 5637382..d155d3e 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile @@ -1,5 +1,7 @@ FROM {{language_deps}} +ARG UDF_RUNNER_CPP_COMMIT=b3a8c204a2555b373c66110d5e8301a374b32a6a + RUN mkdir /conf /buckets COPY --from={{base_test_deps}} /usr /usr @@ -38,5 +40,5 @@ RUN ./test_udfclient.sh /exaudf/exaudfclient_static WORKDIR / RUN mkdir /exasol_emulator COPY emulator/ /exasol_emulator -COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator +RUN curl -fsSL "https://raw.githubusercontent.com/exasol/udf-runner-cpp/${UDF_RUNNER_CPP_COMMIT}/udf-runner-cpp/v1/base/exaudflib/zmqcontainer.proto" -o /exasol_emulator/zmqcontainer.proto RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=. diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/packages.yml b/flavors/template-Exasol-all-python-3.12/flavor_base/packages.yml index 91eeafd..020cfbf 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/packages.yml +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/packages.yml @@ -4,26 +4,28 @@ build_steps: - name: install_apt_packages apt: packages: + - name: git + version: '*' - name: coreutils - version: 9.4-3ubuntu* + version: '*' - name: tar - version: 1.35+dfsg-3ubuntu* + version: '*' - name: curl - version: 8.5.0-2ubuntu* + version: '*' - name: openjdk-17-jdk-headless - version: 17.0.20+8-1~24.04 + version: '*' - name: ca-certificates - version: '20260601~24.04.1' + version: '*' - name: build-essential - version: 12.10ubuntu* + version: '*' - name: libpcre3-dev - version: 2:8.39-15build1 + version: '*' - name: protobuf-compiler - version: 3.21.12-8.2ubuntu* + version: '*' - name: chrpath - version: 0.16-2build1 + version: '*' - name: locales - version: 2.39-0ubuntu* + version: '*' variables: PROTOBUF_BIN: /usr/bin/protoc - name: install_bazel @@ -38,15 +40,15 @@ build_steps: apt: packages: - name: gdb - version: 15.0.50.20240403-0ubuntu* + version: '*' - name: valgrind - version: 1:3.22.0-0ubuntu* + version: '*' - name: binutils - version: 2.42-4ubuntu* + version: '*' - name: patchelf - version: 0.18.0-1.1build1 + version: '*' - name: strace - version: 6.8-0ubuntu* + version: '*' validation_cfg: version_mandatory: true - name: security_scan @@ -67,4 +69,4 @@ build_steps: - name: trivy validation_cfg: version_mandatory: false -version: 1.0.0 +version: '*' diff --git a/flavors/template-Exasol-all-python-3.12/packages.yml b/flavors/template-Exasol-all-python-3.12/packages.yml index 289fb3e..a30128e 100644 --- a/flavors/template-Exasol-all-python-3.12/packages.yml +++ b/flavors/template-Exasol-all-python-3.12/packages.yml @@ -5,11 +5,11 @@ build_steps: apt: packages: - name: ca-certificates - version: '20260601~24.04.1' + version: '*' - name: python3.12-dev - version: 3.12.3-1ubuntu* + version: '*' - name: curl - version: 8.5.0-2ubuntu* + version: '*' variables: PYTHON3_PREFIX: /usr PYTHON3_VERSION: python3.12 @@ -42,21 +42,21 @@ build_steps: apt: packages: - name: coreutils - version: 9.4-3ubuntu* + version: '*' - name: adduser - version: 3.137ubuntu* + version: '*' - name: locales - version: 2.39-0ubuntu* + version: '*' - name: libnss-db - version: 2.2.3pre1-9build3 + version: '*' - name: libzmq3-dev - version: 4.3.5-1build2 + version: '*' - name: cppzmq-dev - version: 4.10.0-1build1 + version: '*' - name: libprotobuf-dev - version: 3.21.12-8.2ubuntu* + version: '*' - name: libssl-dev - version: 3.0.13-0ubuntu* + version: '*' variables: PROTOBUF_LIBRARY_PREFIX: /usr/lib/ PROTOBUF_INCLUDE_PREFIX: /usr/include/ @@ -70,13 +70,13 @@ build_steps: apt: packages: - name: unzip - version: 6.0-28ubuntu* + version: '*' - name: git - version: 1:2.43.0-1ubuntu* + version: '*' - name: libcurl4-openssl-dev - version: 8.5.0-2ubuntu* + version: '*' - name: build-essential - version: 12.10ubuntu* + version: '*' validation_cfg: version_mandatory: true - name: flavor_customization diff --git a/flavors/test-Exasol-8-cuda-ml/flavor_base/base_test_build_run/Dockerfile b/flavors/test-Exasol-8-cuda-ml/flavor_base/base_test_build_run/Dockerfile index 555207c..a812477 100644 --- a/flavors/test-Exasol-8-cuda-ml/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/test-Exasol-8-cuda-ml/flavor_base/base_test_build_run/Dockerfile @@ -1,5 +1,7 @@ FROM {{base_test_deps}} +ARG UDF_RUNNER_CPP_COMMIT=b3a8c204a2555b373c66110d5e8301a374b32a6a + RUN mkdir /conf /buckets RUN exaslpm export-variables --out-file /env @@ -20,5 +22,5 @@ RUN ./test_udfclient.sh /exaudf/exaudfclient_static WORKDIR / RUN mkdir /exasol_emulator COPY emulator/ /exasol_emulator -COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator +RUN curl -fsSL "https://raw.githubusercontent.com/exasol/udf-runner-cpp/${UDF_RUNNER_CPP_COMMIT}/udf-runner-cpp/v1/base/exaudflib/zmqcontainer.proto" -o /exasol_emulator/zmqcontainer.proto RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=. diff --git a/flavors/test-Exasol-8-cuda-ml/flavor_base/packages.yml b/flavors/test-Exasol-8-cuda-ml/flavor_base/packages.yml index c835ece..d3aa9e6 100644 --- a/flavors/test-Exasol-8-cuda-ml/flavor_base/packages.yml +++ b/flavors/test-Exasol-8-cuda-ml/flavor_base/packages.yml @@ -1,26 +1,39 @@ build_steps: - name: build_deps phases: + - name: install_apt_packages + apt: + packages: + - name: git + version: '*' + validation_cfg: + version_mandatory: true - name: install_conda_packages conda: channels: - conda-forge packages: - name: protobuf - version: =7.35.1 + version: =* - name: pcre - version: '=8.45' - - name: bazel - version: =9.2.0 + version: =* - name: make - version: =4.4.1 + version: =* - name: chrpath - version: '=0.16' + version: =* - name: gxx - version: =13.4.0 + version: =* + - name: binutils + version: =* - name: gcc - version: =13.4.0 + version: =* binary: Mamba + - name: install_bazel + tools: + bazel: + version: '8.3.1' + validation_cfg: + version_mandatory: true variables: PROTOBUF_BIN: '$MAMBA_ROOT_PREFIX/bin/protoc' LD_LIBRARY_PATH: '$LD_LIBRARY_PATH:$MAMBA_ROOT_PREFIX/lib/' @@ -37,17 +50,17 @@ build_steps: apt: packages: - name: gdb - version: 15.0.50.20240403-0ubuntu* + version: '*' - name: valgrind - version: 1:3.22.0-0ubuntu* + version: '*' - name: gdbserver - version: 15.0.50.20240403-0ubuntu* + version: '*' - name: binutils - version: 2.42-4ubuntu* + version: '*' - name: patchelf - version: 0.18.0-1.1build1 + version: '*' - name: strace - version: 6.8-0ubuntu* + version: '*' validation_cfg: version_mandatory: true - name: security_scan @@ -68,4 +81,4 @@ build_steps: - name: trivy validation_cfg: version_mandatory: false -version: 1.0.0 +version: '*' diff --git a/flavors/test-Exasol-8-cuda-ml/packages.yml b/flavors/test-Exasol-8-cuda-ml/packages.yml index 4bba218..f9c8fe1 100644 --- a/flavors/test-Exasol-8-cuda-ml/packages.yml +++ b/flavors/test-Exasol-8-cuda-ml/packages.yml @@ -5,19 +5,19 @@ build_steps: apt: packages: - name: coreutils - version: 9.4-3ubuntu* + version: '*' - name: adduser - version: 3.137ubuntu* + version: '*' - name: locales - version: 2.39-0ubuntu* + version: '*' - name: curl - version: 8.5.0-2ubuntu* + version: '*' - name: ca-certificates - version: '20260601~24.04.1' + version: '*' - name: bzip2 - version: 1.0.8-5.1build0.1 + version: '*' - name: libxml2 - version: 2.9.14+dfsg-1.3ubuntu* + version: '*' - name: install_micromamba tools: micromamba: @@ -35,13 +35,13 @@ build_steps: - conda-forge packages: - name: libprotobuf - version: =7.35.1 + version: =* - name: zeromq - version: =4.3.5 + version: =* - name: nss - version: '=3.118' + version: =* - name: cppzmq - version: =4.11.0 + version: =* binary: Micromamba variables: PROTOBUF_LIBRARY_PREFIX: $MAMBA_ROOT_PREFIX/lib @@ -63,17 +63,17 @@ build_steps: - nvidia packages: - name: python - version: =3.12.13 + version: =3.12.* - name: numpy - version: =2.4.6 + version: =2.2.6 - name: pandas version: =2.3.3 - name: pyarrow version: =25.0.0 - name: libarrow - version: =25.0.0=*_cuda + version: =*=*_cuda - name: mamba - version: =2.8.1 + version: =* binary: Micromamba variables: PYTHON3_PREFIX: $MAMBA_ROOT_PREFIX @@ -90,7 +90,7 @@ build_steps: apt: packages: - name: tzdata - version: 2026* + version: '*' - name: install_conda_packages conda: channels: @@ -98,20 +98,20 @@ build_steps: - nvidia packages: - name: cudnn - version: =9.25.0.15 + version: =* - name: cuda-toolkit - version: =13.3.1 + version: =* - name: cuda-nvcc - version: =13.3.73 + version: =* - name: cuda-nvrtc - version: =13.3.33 + version: =* - name: numba - version: =0.66.0 + version: =* - name: numba-cuda - version: =0.30.4 + version: =* - name: pytorch - version: =2.13.0=cuda130_generic_py312_* + version: =*=cuda130_generic_py312_* binary: Mamba validation_cfg: version_mandatory: true -version: 1.0.0 +version: =*