#3: Created a test flavor

flavors/test-udf-runner-python-v1/FLAVOR_DESCRIPTION.md

@@ -0,0 +1,8 @@
+# Details for test-udf-runner-python-v1
+
+This flavor provides a lightweight CPU-only test build for the Python UDF
+runner.
+
+It is derived from `template-Exasol-all-python-3.10` and uses the template's
+build, test, release, and security-scan stages. The flavor-specific CI
+configuration runs the core Python tests and language-definition setup checks.

flavors/test-udf-runner-python-v1/ci.json

@@ -0,0 +1,18 @@
+{
+  "build_runners": ["int-linux-x64-4core-ubuntu24.04-1", "int-linux-arm-4core-ubuntu24.04-1"],
+  "test_config": {
+    "default_test_runners": ["int-linux-x64-2core-ubuntu24.04-1"],
+    "test_sets": [
+      {
+        "name": "python",
+        "files": [],
+        "folders": [
+          "python3/all",
+          "standard-flavor/setup"
+        ],
+        "goal": "release",
+        "generic_language_tests": []
+      }
+    ]
+  }
+}

flavors/test-udf-runner-python-v1/flavor_base/base_test_build_run/Dockerfile

@@ -0,0 +1,35 @@
+FROM {{language_deps}}
+
+RUN mkdir /conf /buckets
+
+COPY --from={{base_test_deps}} /usr /usr
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{base_test_deps}} /lib /lib
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{base_test_deps}} /bin /bin
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{base_test_deps}} /opt /opt
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{base_test_deps}} /etc /etc
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{base_test_deps}} /build_info /build_info
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+RUN exaslpm export-variables --out-file /env
+RUN ldconfig
+
+RUN mkdir /exaudfclient /exaudf
+COPY /exaudfclient/ /exaudfclient/
+WORKDIR /exaudfclient/
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries"]
+RUN cp -r -L bazel-bin/* /exaudf
+
+WORKDIR /exaudfclient/base
+RUN ./test_udfclient.sh /exaudf/exaudfclient
+RUN ./test_udfclient.sh /exaudf/exaudfclient_static
+
+WORKDIR /
+RUN mkdir /exasol_emulator
+COPY emulator/ /exasol_emulator
+COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator
+RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=.

flavors/test-udf-runner-python-v1/flavor_base/base_test_deps/Dockerfile

@@ -0,0 +1,6 @@
+FROM {{build_deps}}
+
+RUN mkdir -p /build_info/packages
+
+COPY base_test_deps_packages.yml /build_info/packages/base_test_deps_packages.yml
+RUN exaslpm install --package-file /build_info/packages/base_test_deps_packages.yml --build-step base_test_deps

flavors/test-udf-runner-python-v1/flavor_base/build_deps/Dockerfile

@@ -0,0 +1,19 @@
+FROM exasol/script-language-container:exaslpm-1.3.1-ubuntu-22.04
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc
+
+RUN mkdir -p /build_info/packages
+
+COPY build_deps_packages.yml /build_info/packages/build_deps_packages.yml
+RUN exaslpm install --package-file /build_info/packages/build_deps_packages.yml --build-step build_deps
+
+RUN curl -fsSL -o swig-2.0.4.tar.gz \
+        https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \
+    tar zxf swig-2.0.4.tar.gz && \
+    (cd swig-2.0.4 && ./configure --build=$(arch)-unknown-linux-gnu --prefix=/usr && make -j"$(nproc)" && make install) && \
+    rm -rf swig-2.0.4 swig-2.0.4.tar.gz
+
+RUN locale-gen en_US.UTF-8 && \
+    update-locale LC_ALL=en_US.UTF-8 && \
+    ldconfig

flavors/test-udf-runner-python-v1/flavor_base/build_run/Dockerfile

@@ -0,0 +1,31 @@
+FROM {{language_deps}}
+
+COPY --from={{build_deps}} /usr /usr
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{build_deps}} /lib /lib
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{build_deps}} /bin /bin
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{build_deps}} /opt /opt
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{build_deps}} /etc /etc
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{build_deps}} /build_info /build_info
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+RUN ldconfig
+RUN mkdir /exaudfclient /exaudf
+COPY exaudfclient/ /exaudfclient/
+RUN exaslpm export-variables --out-file /env
+
+WORKDIR /exaudfclient/
+RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary"]
+RUN cp -r -L bazel-bin/* /exaudf
+RUN rm -r /root/.cache/bazel
+
+WORKDIR /exaudfclient/base
+RUN ./test_udfclient.sh /exaudf/exaudfclient
+
+WORKDIR /exaudf
+RUN rm -r /exaudfclient
+RUN mkdir /conf /buckets

flavors/test-udf-runner-python-v1/flavor_base/build_steps.py

@@ -0,0 +1,139 @@
+from typing import Dict
+
+from exasol.slc.internal.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask
+
+
+class AnalyzeUDFClientDeps(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "udfclient_deps"
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeLanguageDeps(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "language_deps"
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {"scripts": "ext/scripts"}
+
+    def requires_tasks(self):
+        return {"udfclient_deps": AnalyzeUDFClientDeps}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeBuildDeps(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "build_deps"
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeBuildRun(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "build_run"
+
+    def requires_tasks(self):
+        return {"build_deps": AnalyzeBuildDeps, "language_deps": AnalyzeLanguageDeps}
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {"exaudfclient": "exaudfclient"}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeBaseTestDeps(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "base_test_deps"
+
+    def requires_tasks(self):
+        return {"build_deps": AnalyzeBuildDeps}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeBaseTestBuildRun(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "base_test_build_run"
+
+    def requires_tasks(self):
+        return {"base_test_deps": AnalyzeBaseTestDeps, "language_deps": AnalyzeLanguageDeps}
+
+    def get_additional_build_directories_mapping(self) -> Dict[str, str]:
+        return {"exaudfclient": "exaudfclient", "emulator": "emulator"}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeFlavorBaseDeps(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "flavor_base_deps"
+
+    def requires_tasks(self):
+        return {"language_deps": AnalyzeLanguageDeps}
+
+    def get_additional_build_directories_mapping(self):
+        return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeFlavorCustomization(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "flavor_customization"
+
+    def requires_tasks(self):
+        return {"flavor_base_deps": AnalyzeFlavorBaseDeps}
+
+
+class AnalyzeFlavorTestBuildRun(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "flavor_test_build_run"
+
+    def requires_tasks(self):
+        return {"flavor_customization": AnalyzeFlavorCustomization,
+                "base_test_build_run": AnalyzeBaseTestBuildRun}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class AnalyzeRelease(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "release"
+
+    def requires_tasks(self):
+        return {"flavor_customization": AnalyzeFlavorCustomization,
+                "build_run": AnalyzeBuildRun,
+                "flavor_base_deps": AnalyzeFlavorBaseDeps}
+
+    def get_language_definition(self) -> str:
+        return "language_definitions.json"
+
+    def get_path_in_flavor(self):
+        return "flavor_base"
+
+
+class SecurityScan(DockerFlavorAnalyzeImageTask):
+    def get_build_step(self) -> str:
+        return "security_scan"
+
+    def requires_tasks(self):
+        return {"release": AnalyzeRelease}
+
+    def get_path_in_flavor(self):
+        return "flavor_base"

flavors/test-udf-runner-python-v1/flavor_base/flavor_base_deps/Dockerfile

@@ -0,0 +1,6 @@
+FROM {{ language_deps }}
+
+RUN mkdir -p /build_info/packages
+
+COPY flavor_base_deps_packages.yml /build_info/packages/flavor_base_deps_packages.yml
+RUN exaslpm install --package-file /build_info/packages/flavor_base_deps_packages.yml --build-step flavor_base_deps

flavors/test-udf-runner-python-v1/flavor_base/flavor_test_build_run/Dockerfile

@@ -0,0 +1,18 @@
+FROM {{base_test_build_run}}
+
+RUN mkdir -p /conf /buckets
+
+COPY --from={{flavor_customization}} /usr /usr
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{flavor_customization}} /opt /opt
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{flavor_customization}} /lib /lib
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{flavor_customization}} /bin /bin
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{flavor_customization}} /etc /etc
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+COPY --from={{flavor_customization}} /build_info /build_info
+RUN true # workaround for https://github.com/moby/moby/issues/37965
+
+RUN ldconfig

flavors/test-udf-runner-python-v1/flavor_base/language_definition

@@ -0,0 +1 @@
+PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient

flavors/test-udf-runner-python-v1/flavor_base/language_definitions.json

@@ -0,0 +1,14 @@
+{
+  "schema_version": 2,
+  "language_definitions": [
+    {
+      "protocol": "localzmq+protobuf",
+      "aliases": ["PYTHON3"],
+      "parameters": [{"key": "lang", "value": "python"}],
+      "udf_client_path": {
+        "executable": "/exaudf/exaudfclient"
+      },
+      "deprecation": null
+    }
+  ]
+}

flavors/test-udf-runner-python-v1/flavor_base/language_deps/Dockerfile

@@ -0,0 +1,6 @@
+FROM {{udfclient_deps}}
+
+RUN mkdir -p /build_info/packages/language_deps
+
+COPY language_deps_packages.yml /build_info/packages/language_deps_packages.yml
+RUN exaslpm install --package-file /build_info/packages/language_deps_packages.yml --build-step language_deps

flavors/test-udf-runner-python-v1/flavor_base/packages.yml

@@ -0,0 +1,71 @@
+build_steps:
+- name: base_test_deps
+  phases:
+  - name: install_apt_packages
+    apt:
+      packages:
+      - name: gdb
+        version: 12.1-0ubuntu*
+      - name: valgrind
+        version: 1:3.18.1-1ubuntu*
+      - name: gdbserver
+        version: 12.1-0ubuntu*
+      - name: binutils
+        version: 2.38-4ubuntu*
+      - name: patchelf
+        version: 0.14.3-1
+      - name: strace
+        version: 5.16-0ubuntu*
+  validation_cfg:
+    version_mandatory: true
+- name: build_deps
+  phases:
+  - name: install_apt_packages
+    apt:
+      packages:
+      - name: coreutils
+        version: 8.32-4.1ubuntu*
+      - name: locales
+        version: 2.35-0ubuntu*
+      - name: tar
+        version: 1.34+dfsg-1ubuntu*
+      - name: curl
+        version: 7.81.0-1ubuntu*
+      - name: openjdk-11-jdk-headless
+        version: 11.0.31+11-1ubuntu*
+      - name: build-essential
+        version: 12.9ubuntu*
+      - name: libpcre3-dev
+        version: 2:8.39-13ubuntu*
+      - name: protobuf-compiler
+        version: 3.12.4-1ubuntu*
+      - name: chrpath
+        version: 0.16-2
+    variables:
+      PROTOBUF_BIN: /usr/bin/protoc
+      JAVA_HOME: /usr/lib/jvm/java-1.11.0-openjdk-{% if platform == 'x86_64' %}amd64{% else %}arm64{% endif %}
+  - name: install_bazel
+    tools:
+      bazel:
+        version: '8.3.1'
+  validation_cfg:
+    version_mandatory: true
+- name: security_scan
+  phases:
+  - name: install_deps
+    apt:
+      packages:
+        - name: gnupg
+  - name: install_trivy
+    apt:
+      repos:
+        trivy:
+          key_url: https://aquasecurity.github.io/trivy-repo/deb/public.key
+          entry: deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb
+            generic main
+          out_file: trivy.list
+      packages:
+        - name: trivy
+  validation_cfg:
+    version_mandatory: false
+version: 1.0.0