Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 3 additions & 49 deletions .github/workflows/check_bazel_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
- name: Build
run: |
source "$UDF_CLIENT_ENV_FILE"
bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config java --config fast-binary --verbose_failures
bazel build --lockfile_mode=off --config no-tty -c dbg --config python --config fast-binary --verbose_failures
working-directory: ./exaudfclient/

tests:
Expand All @@ -32,60 +32,14 @@ jobs:
fail-fast: false
matrix:
include:
- test: "//javacontainer/test:ExaStackTraceCleanerTest"
name: "ExaStackTraceCleanerTest"
java_version: "openjdk-11-jdk"
- test: "//javacontainer/test:ExaStackTraceCleanerTest"
name: "ExaStackTraceCleanerTest"
java_version: "openjdk-17-jdk"
- test: "//javacontainer/test:javacontainer-test-extractor-legacy"
name: "javacontainer-test-extractor-legacy"
java_version: "openjdk-11-jdk"
- test: "//javacontainer/test:javacontainer-test-extractor-v2"
name: "javacontainer-test-extractor-v2"
java_version: "openjdk-11-jdk"
- test: "//javacontainer/script_options/..."
name: "javacontainer-script_options"
java_version: ""
- test: "//exaudflib/test/..."
name: "exaudflib"
java_version: ""
- test: "//script_options_parser/ctpg/..."
name: "script_options_parser_ctpg"
java_version: ""
- test: "//script_options_parser/legacy/..."
name: "script_options_parser_legacy"
java_version: ""
- test: "--run_under='valgrind --leak-check=yes' --config=valgrind //javacontainer/test:javacontainer-test-extractor-legacy"
name: "javacontainer-test-extractor-legacy-with-valgrind"
java_version: "openjdk-11-jdk"
- test: "--run_under='valgrind --leak-check=yes' --config=valgrind //javacontainer/test:javacontainer-test-extractor-v2"
name: "javacontainer-test-extractor-v2-with-valgrind"
java_version: "openjdk-11-jdk"
- test: "--run_under='valgrind --leak-check=yes' --config=valgrind //script_options_parser/ctpg/..."
name: "script_options_parser_ctpg_with_valgrind"
java_version: ""
- test: "--run_under='valgrind --leak-check=yes' --config=valgrind //script_options_parser/legacy/..."
name: "script_options_parser_legacy_with_valgrind"
java_version: ""
- test: "--config=asan //javacontainer/test:javacontainer-test-extractor-legacy"
name: "javacontainer-test-extractor-legacy-with-asan"
java_version: "openjdk-11-jdk"
- test: "--config=asan //javacontainer/test:javacontainer-test-extractor-v2"
name: "javacontainer-test-extractor-v2-with-asan"
java_version: "openjdk-11-jdk"
- test: "--config=asan //script_options_parser/ctpg/..."
name: "script_options_parser_ctpg_with_asan"
java_version: ""
- test: "--config=asan //script_options_parser/legacy/..."
name: "script_options_parser_legacy_with_asan"
java_version: ""
steps:
- uses: actions/checkout@v6
- name: Install JDK and ZMQ
- name: Install ZMQ
run: |
sudo apt-get update
sudo apt-get install -y ${{ matrix.java_version }} libzmq3-dev valgrind
sudo apt-get install -y libzmq3-dev valgrind
- name: Install bazel
run: |
BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb"
Expand Down
13 changes: 0 additions & 13 deletions emulator/MY_TEST_JAVA

This file was deleted.

11 changes: 2 additions & 9 deletions emulator/exasolution.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,8 +125,7 @@ def parse_number(num):
input_file = output_file_fd = output_file = script_lines = script_input_defs = script_output_defs = None

if not single_call_mode:
if script_name.endswith('.java'): comment = '//'
else: comment = '#'
comment = '#'
for line in script_lines:
if line.startswith(comment + 'input_column:'):
script_input_defs.append(line[len(comment + 'input_column:'):])
Expand Down Expand Up @@ -301,12 +300,6 @@ def handle_client():
#response.info.vm_type = PB_VM_EXTERNAL
response.info.maximal_memory_limit = memory_limit
response.info.meta_info = meta_info
#if redirector_name != None: response.info.vm_type = PB_VM_EXTERNAL
#else:
# if script_name.endswith('.py'): response.info.vm_type = PB_VM_PYTHON
# elif script_name.endswith('.R'): response.info.vm_type = PB_VM_R
# elif script_name.endswith('.java'): response.info.vm_type = PB_VM_JAVA
# else: raise RuntimeError("Unsupported VM type")
send_message(response.SerializeToString())
initialized = True

Expand Down Expand Up @@ -355,7 +348,7 @@ def handle_import():
if kind == PB_IMPORT_SCRIPT_CODE:
if script_name.endswith('.py'): fname = fname + '.py'
elif script_name.endswith('.R'): fname = fname + '.R'
else: fname = fname + '.java'
else: raise RuntimeError("Unsupported script language")
try: getattr(response, 'import').source_code = open(fname).read()
except Exception, err:
response.exception_message = str(err)
Expand Down
1 change: 0 additions & 1 deletion exaudfclient/.bazelrc
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
build --lockfile_mode=off --copt='-std=c++17' --force_pic --action_env=PROTOBUF_BIN --action_env=PROTOBUF_LIBRARY_PREFIX --action_env=PROTOBUF_INCLUDE_PREFIX
# TODO add environment variables for R libraries
build:benchmark --define benchmark=true
build:java --define java=true --action_env=JAVA_PREFIX
build:python --define python=true --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION
build:fast-binary --define binary_type=fast_binary //:exaudfclient
build:slow-wrapper --define binary_type=slow_wrapper //:exaudfclient
Expand Down
3 changes: 1 addition & 2 deletions exaudfclient/.env.template
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
export JAVA_PREFIX=
export PYTHON3_PREFIX=
export PYTHON3_VERSION=
export PROTOBUF_BIN=
Expand All @@ -8,4 +7,4 @@ export OPENSSL_LIBRARY_PREFIX=
export OPENSSL_INCLUDE_PREFIX=
export VERBOSE_BUILD= #"--subcommands --verbose_failures"
export ZMQ_LIBRARY_PREFIX=
export ZMQ_INCLUDE_PREFIX=
export ZMQ_INCLUDE_PREFIX=
3 changes: 0 additions & 3 deletions exaudfclient/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,6 @@ load("@rules_shell//shell:sh_library.bzl", "sh_library")
VM_ENABLED_DEPS=select({
"@exaudfclient_base//:benchmark": ["@exaudfclient_base//benchmark_container:benchmark_container"],
"//conditions:default": []
}) + select({
"@exaudfclient_base//:java": ["@exaudfclient_base//javacontainer:javacontainer"],
"//conditions:default": []
}) + select({
"@exaudfclient_base//:bash": ["@exaudfclient_base//streaming_container:streamingcontainer"],
"//conditions:default": []
Expand Down
16 changes: 0 additions & 16 deletions exaudfclient/MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ bazel_dep(name = "exaudfclient_base", version = "1.0.0")
#Later we might fetch it via Http or Git (after splitting up script-languages repository)
local_path_override(module_name="exaudfclient_base", path="base")
bazel_dep(name = "rules_cc", version = "0.2.22")
bazel_dep(name = "rules_jvm_external", version = "6.7")
bazel_dep(name = "rules_shell", version = "0.8.0")

numpy_local_repository = use_repo_rule("@exaudfclient_base//:python_repository.bzl", "numpy_local_repository")
Expand All @@ -19,18 +18,3 @@ zmq_local_repository(name = "zmq")

protobuf_local_repository = use_repo_rule("@exaudfclient_base//:protobuf_repository.bzl", "protobuf_local_repository")
protobuf_local_repository(name = "protobuf")

java_local_repository = use_repo_rule("@exaudfclient_base//:java_repository.bzl", "java_local_repository")
java_local_repository(name = "java")


maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven")
maven.install(
artifacts = [
"com.exasol:udf-api-java:1.0.5",
],
repositories = [
"https://repo1.maven.org/maven2",
],
)
use_repo(maven, "maven")
10 changes: 3 additions & 7 deletions exaudfclient/README.md
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
# What is the exaudfclient?

The exaudfclient connects to the database via [ZeroMQ](http://zeromq.org/) and fetches the tuples which then get processed by the user-defined functions (UDFs). Currently, the exaudfclient supports UDFs in the language Python, Java and R. Further languages can be integrated via language binding between C/C++ and the desired langauge. Python 3, Java and R use [SWIG](http://www.swig.org/) for the language binding.
The exaudfclient connects to the database via [ZeroMQ](http://zeromq.org/) and fetches the tuples which then get processed by the user-defined functions (UDFs). Currently, the exaudfclient supports UDFs in Python. Further languages can be integrated via language binding between C/C++ and the desired language. Python 3 uses [SWIG](http://www.swig.org/) for the language binding.

# How to build the exaudfclient?

## Prerequisites

For the build system:

- Open JDK 11
- bazel-7.2.1 for more details see [Bazel documentation](https://docs.bazel.build/versions/master/install.html)

The exaudfclient was tested with the following versions of its dependencies:
Expand All @@ -23,12 +22,10 @@ For the language support:

- Python 3.10 for pythoncontainer
- for Python 3 the build requires [Numpy](https://www.numpy.org/) and [Pandas](https://pandas.pydata.org/) in addition for the Pandas Dataframe Support
- OpenJDK 11 for javacontainer
- R 4.4 for the rcontainer

## Start a build

The exaudfclient is a multi-language project. Therefore, we are using [Bazel](https://docs.bazel.build/versions/master/bazel-overview.html) as build system, because it provides build support for many languages and allows to mix these languages. Because the exaudfclient has language bindings to Python 2/3, Java and R, we need specify where Bazel can find the correponding library- and header-files. This is done by Environment Variables.
The exaudfclient is a multi-language project. Therefore, we are using [Bazel](https://docs.bazel.build/versions/master/bazel-overview.html) as build system, because it provides build support for many languages and allows to mix these languages. Because the exaudfclient has language bindings to Python 2/3 and R, we need specify where Bazel can find the corresponding library and header files. This is done by environment variables.

For executing the build locally, you can use the script

Expand All @@ -46,7 +43,6 @@ Both build script can receive parameters, such as Bazel commandline parameter, B

With Bazel defines you can specify which language support is actually compiled into you exaudfclient executbale. The currently supported defines are

--define java=true
--define r=true
--define python=true
--define benchmark=true # This language is only for test and development purpose and benchmarks the performance of the C++ Implementation
Expand All @@ -70,7 +66,7 @@ The usage of multiple linker namespace requires some precautions in the build pr

## Precautions in the build process

In the build process you need to be cautious which libraries you link together and that no link leaks symbols from a library in one namespace to a library in the other namespace. Furthermore, you have to build a shared library with all dependency linked to it as output target. In our case, we have to main output targets //:exaudfclient and //:libexaudflib.so. Both get loaded into different linker namespaces. The language container live in the same namespace as //:exaudfclient. This namespace must not know anyhing about protobuf and zeromq, because it is possible that a language container may load protobuf or zeromq in a different version. Protobuf and zeromq are only known in the namespace of //:libexaudflib.so. The target //:libexaudflib.so depends on //base/exaudflib:exaudflib which contains the logic of the exaudflib. You must not depend on //base/exaudflib:exaudflib in //:exaudfclient or the langauge container, because this would leak zeromq and protobuf. If you need to depend on the other dependency of //base/exaudflib:exaudflib which not depend on protobuf or zeromq them self, such as //base/exaudflib:script_data_transfer_objects, //base/exaudflib:script_data_transfer_objects_wrapper, //base/script_options_parser:scriptoptionlines, use either their target as self, the collection of libraries //base/exaudflib:exaudflib-deps or the collection of headers //base/exaudflib:header.
In the build process you need to be cautious which libraries you link together and that no link leaks symbols from a library in one namespace to a library in the other namespace. Furthermore, you have to build a shared library with all dependency linked to it as output target. In our case, we have two main output targets: //:exaudfclient and //:libexaudflib.so. Both get loaded into different linker namespaces. The language containers live in the same namespace as //:exaudfclient. This namespace must not know anything about protobuf and zeromq, because a language container may load protobuf or zeromq in a different version. Protobuf and zeromq are only known in the namespace of //:libexaudflib.so. The target //:libexaudflib.so depends on //base/exaudflib:exaudflib which contains the logic of the exaudflib. You must not depend on //base/exaudflib:exaudflib in //:exaudfclient or a language container, because this would leak zeromq and protobuf. If you need other dependencies of //base/exaudflib:exaudflib that do not depend on protobuf or zeromq, such as //base/exaudflib:script_data_transfer_objects or //base/exaudflib:script_data_transfer_objects_wrapper, use their target directly, the collection of libraries //base/exaudflib:exaudflib-deps, or the collection of headers //base/exaudflib:header.

## Precautions in the implementations

Expand Down
1 change: 0 additions & 1 deletion exaudfclient/base/.bazelrc
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
build --lockfile_mode=off --copt='-std=c++17' --force_pic --action_env=PROTOBUF_BIN --action_env=PROTOBUF_LIBRARY_PREFIX --action_env=PROTOBUF_INCLUDE_PREFIX
build:benchmark --define benchmark=true
build:java --define java=true --action_env=JAVA_PREFIX
build:python --define python=true --action_env=NUMPY_PREFIX --action_env=PYTHON3_SYSPATH --action_env=PYTHON3_PREFIX --action_env=PYTHON3_VERSION
build:debug-build --sandbox_debug --config=verbose
build:no-symlinks --symlink_prefix=/
Expand Down
7 changes: 1 addition & 6 deletions exaudfclient/base/BUILD
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

package(default_visibility = ["//visibility:public"])
exports_files(["filter_swig_code.py", "build_integrated.py",
exports_files(["build_integrated.py",
"create_binary_wrapper.sh", "create_binary_wrapper_valgrind.sh",
"create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh",
"replace_swig_import_helper.py"])
Expand All @@ -18,11 +18,6 @@ config_setting(
define_values = {"python": "true"},
)

config_setting(
name = "java",
define_values = {"java": "true"},
)

config_setting(
name = "bash",
define_values = {"bash": "true"},
Expand Down
16 changes: 0 additions & 16 deletions exaudfclient/base/MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@ module(name="exaudfclient_base", version = "1.0")
bazel_dep(name = "bazel_skylib", version = "1.7.1")
bazel_dep(name = "googletest", version = "1.15.0")
bazel_dep(name = "rules_cc", version = "0.2.22")
bazel_dep(name = "rules_jvm_external", version = "6.2")

# The following is necessary to build the UDF client for Java
# Since Bazel >8 the Python hermetic build rules must not run as root
# However, since the UDF client is built as root in a docker container
# we must disable this behavior. The only way is to configure the Python hermetic toolchain
Expand All @@ -27,17 +25,3 @@ zmq_local_repository(name = "zmq")

protobuf_local_repository = use_repo_rule("@exaudfclient_base//:protobuf_repository.bzl", "protobuf_local_repository")
protobuf_local_repository(name = "protobuf")

java_local_repository = use_repo_rule("@exaudfclient_base//:java_repository.bzl", "java_local_repository")
java_local_repository(name = "java")

maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven")
maven.install(
artifacts = [
"com.exasol:udf-api-java:1.0.5",
],
repositories = [
"https://repo1.maven.org/maven2",
],
)
use_repo(maven, "maven")
4 changes: 2 additions & 2 deletions exaudfclient/base/benchmark_container/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@ cc_library(
name = "benchmark_container",
srcs = ["benchmark_container.cc","benchmark_container.h"],
hdrs = ["benchmark_container.h"],
deps = ["//exaudflib:exaudflib-deps","//exaudflib:header", "//utils:utils"]+["//script_options_parser:scriptoptionlines"]
)
deps = ["//exaudflib:exaudflib-deps","//exaudflib:header", "//utils:utils"]
)
1 change: 0 additions & 1 deletion exaudfclient/base/exaudflib/swig/swig_common.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ enum VMTYPE {
VM_JAVASCRIPT = 3,
VM_R = 4,
VM_EXTERNAL = 5,
VM_JAVA = 6,
VM_PLUGIN_LANGUAGE = 7,
VM_BENCHMARK = 8
};
Expand Down
1 change: 0 additions & 1 deletion exaudfclient/base/exaudflib/zmqcontainer.proto
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@ enum exascript_vmtype {
PB_VM_JAVASCRIPT = 3;
PB_VM_R = 4;
PB_VM_EXTERNAL = 5;
PB_VM_JAVA = 6;
PM_VM_PLUGIN_LANGUAGE = 7;
}
*/
Expand Down
Loading