diff --git a/tests/spark/README.md b/tests/spark/README.md new file mode 100644 index 00000000..852ef280 --- /dev/null +++ b/tests/spark/README.md @@ -0,0 +1,12 @@ +This application was used to produce the `delta-table` for the delta kuttl test. + +It cannot be placed in the delta test folder because beku tries to recurse into it and fails. + +To run it, you need a S3 (minio) available. Update the variables below as needed: + + S3_ENDPOINT=https://localhost:9000 \ + S3_ACCESS_KEY=minioAccessKey \ + S3_SECRET_KEY=minioSecretKey \ + S3_SKIP_TLS_VALIDATION=true \ + DELTA_TABLE_PATH=s3a://trino/delta-table \ + ./gradlew run diff --git a/tests/spark/build.gradle b/tests/spark/build.gradle new file mode 100644 index 00000000..6014d3ae --- /dev/null +++ b/tests/spark/build.gradle @@ -0,0 +1,39 @@ +plugins { + id 'java' + id 'application' +} + +group = 'org.example' +version = '1.0-SNAPSHOT' + +application { + mainClass = 'org.example.Main' + applicationDefaultJvmArgs = [ + '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED' + ] +} + +repositories { + mavenCentral() +} + + +dependencies { + + // Source: https://mvnrepository.com/artifact/org.apache.spark/spark-sql + implementation 'org.apache.spark:spark-sql_2.12:3.5.5' + // Source: https://mvnrepository.com/artifact/io.delta/delta-spark + implementation 'io.delta:delta-spark_2.12:3.3.2' + // Source: https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws + implementation 'org.apache.hadoop:hadoop-aws:3.3.4' + // Source: https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-bundle + implementation 'com.amazonaws:aws-java-sdk-bundle:1.12.262' + + testImplementation platform('org.junit:junit-bom:5.10.0') + testImplementation 'org.junit.jupiter:junit-jupiter' + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' +} + +test { + useJUnitPlatform() +} diff --git a/tests/spark/gradle/wrapper/gradle-wrapper.jar b/tests/spark/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..d997cfc6 Binary files /dev/null and b/tests/spark/gradle/wrapper/gradle-wrapper.jar differ diff --git a/tests/spark/gradle/wrapper/gradle-wrapper.properties b/tests/spark/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..c61a118f --- /dev/null +++ b/tests/spark/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.1-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/tests/spark/gradlew b/tests/spark/gradlew new file mode 100755 index 00000000..739907df --- /dev/null +++ b/tests/spark/gradlew @@ -0,0 +1,248 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/2d6327017519d23b96af35865dc997fcb544fb40/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/tests/spark/gradlew.bat b/tests/spark/gradlew.bat new file mode 100644 index 00000000..c4bdd3ab --- /dev/null +++ b/tests/spark/gradlew.bat @@ -0,0 +1,93 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/tests/spark/src/main/java/org/example/Main.java b/tests/spark/src/main/java/org/example/Main.java new file mode 100644 index 00000000..3fa1d0ca --- /dev/null +++ b/tests/spark/src/main/java/org/example/Main.java @@ -0,0 +1,46 @@ +package org.example; + +import org.apache.spark.sql.SparkSession; + +import static org.apache.spark.sql.functions.*; + +public class Main { + public static void main(String[] args) { + var s3Endpoint = System.getenv().getOrDefault("S3_ENDPOINT", "https://localhost:9000"); + var s3AccessKey = System.getenv().getOrDefault("S3_ACCESS_KEY", "minioAccessKey"); + var s3SecretKey = System.getenv().getOrDefault("S3_SECRET_KEY", "minioSecretKey"); + var deltaTablePath = System.getenv().getOrDefault("DELTA_TABLE_PATH", "s3a://trino/delta-table"); + var skipTlsValidation = Boolean.parseBoolean(System.getenv().getOrDefault("S3_SKIP_TLS_VALIDATION", "true")); + var sslEnabled = s3Endpoint.startsWith("https://"); + + if (skipTlsValidation) { + System.setProperty("com.amazonaws.sdk.disableCertChecking", "true"); + } + + var spark = SparkSession.builder() + .appName("Quickstart") + .master("local[*]") + .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") + .config("spark.hadoop.fs.s3a.access.key", s3AccessKey) + .config("spark.hadoop.fs.s3a.secret.key", s3SecretKey) + .config("spark.hadoop.fs.s3a.path.style.access", "true") + .config("spark.hadoop.fs.s3a.endpoint", s3Endpoint) + .config("spark.hadoop.fs.s3a.connection.ssl.enabled", String.valueOf(sslEnabled)) + .config("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") + .getOrCreate(); + + var data = spark.range(0, 50_000) + .withColumn("year", lit("2026")) + .withColumn("id_mandant", lit("0157")) + .withColumn("text", concat(lit("text"), try_divide(col("id"), lit(5000)).cast("int"))) + .select(col("year"), col("id_mandant"), col("text")); + + data.repartition(col("year"), col("id_mandant")) + .write() + .format("delta") + .partitionBy("year", "id_mandant") + .mode("append") + .save(deltaTablePath); + } +} diff --git a/tests/templates/kuttl/delta/00-assert.yaml.j2 b/tests/templates/kuttl/delta/00-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/delta/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/delta/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/delta/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/delta/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/delta/00-limit-range.yaml b/tests/templates/kuttl/delta/00-limit-range.yaml new file mode 100644 index 00000000..8fd02210 --- /dev/null +++ b/tests/templates/kuttl/delta/00-limit-range.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: limit-request-ratio +spec: + limits: + - type: "Container" + maxLimitRequestRatio: + cpu: 5 + memory: 1 diff --git a/tests/templates/kuttl/delta/00-patch-ns.yaml.j2 b/tests/templates/kuttl/delta/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/delta/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/delta/00-rbac.yaml.j2 b/tests/templates/kuttl/delta/00-rbac.yaml.j2 new file mode 100644 index 00000000..9cbf0351 --- /dev/null +++ b/tests/templates/kuttl/delta/00-rbac.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: integration-tests-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: use-integration-tests-scc +subjects: + - kind: ServiceAccount + name: integration-tests-sa +roleRef: + kind: Role + name: use-integration-tests-scc + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/delta/00-secrets.yaml.j2 b/tests/templates/kuttl/delta/00-secrets.yaml.j2 new file mode 100644 index 00000000..72fe2f49 --- /dev/null +++ b/tests/templates/kuttl/delta/00-secrets.yaml.j2 @@ -0,0 +1,77 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-credentials + labels: + secrets.stackable.tech/class: s3-credentials-class +timeout: 240 +stringData: + accessKey: minioAccessKey + secretKey: minioSecretKey + # The following two entries are used by the Bitnami chart for MinIO to + # set up credentials for accessing buckets managed by the MinIO tenant. + root-user: minioAccessKey + root-password: minioSecretKey +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: s3-credentials-class +spec: + backend: + k8sSearch: + searchNamespace: + pod: {} +--- +apiVersion: v1 +kind: Secret +metadata: + name: centralized-minio-users +type: Opaque +stringData: + username1: | + username=trino + password=trinotrino + disabled=false + policies=readwrite,consoleAdmin,diagnostics + setPolicies=false + +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: SecretClass +metadata: + name: minio-tls-certificates +spec: + backend: + k8sSearch: + searchNamespace: + pod: {} +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-tls-certificates + labels: + secrets.stackable.tech/class: minio-tls-certificates +# Have a look at the folder certs on how to create this +data: + ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQyVENDQXNHZ0F3SUJBZ0lVTmpxdUdZV3R5SjVhNnd5MjNIejJHUmNNbHdNd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2V6RUxNQWtHQTFVRUJoTUNSRVV4R3pBWkJnTlZCQWdNRWxOamFHeGxjM2RwWnkxSWIyeHpkR1ZwYmpFTwpNQXdHQTFVRUJ3d0ZWMlZrWld3eEtEQW1CZ05WQkFvTUgxTjBZV05yWVdKc1pTQlRhV2R1YVc1bklFRjFkR2h2CmNtbDBlU0JKYm1NeEZUQVRCZ05WQkFNTURITjBZV05yWVdKc1pTNWtaVEFnRncweU16QTJNVFl4TWpVeE1ESmEKR0E4eU1USXpNRFV5TXpFeU5URXdNbG93ZXpFTE1Ba0dBMVVFQmhNQ1JFVXhHekFaQmdOVkJBZ01FbE5qYUd4bApjM2RwWnkxSWIyeHpkR1ZwYmpFT01Bd0dBMVVFQnd3RlYyVmtaV3d4S0RBbUJnTlZCQW9NSDFOMFlXTnJZV0pzClpTQlRhV2R1YVc1bklFRjFkR2h2Y21sMGVTQkpibU14RlRBVEJnTlZCQU1NREhOMFlXTnJZV0pzWlM1a1pUQ0MKQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFOblYvdmJ5M1JvNTdhMnF2UVJubjBqZQplS01VMitGMCtsWk5DQXZpR1VENWJtOGprOTFvUFpuazBiaFFxZXlFcm1EUzRXVDB6ZXZFUklCSkpEamZMMEQ4CjQ2QmU3UGlNS2UwZEdqb3FJM3o1Y09JZWpjOGFMUEhTSWxnTjZsVDNmSXJ1UzE2Y29RZ0c0dWFLaUhGNStlV0YKRFJVTGR1NmRzWXV6NmRLanFSaVVPaEh3RHd0VUprRHdQditFSXRxbzBIK01MRkxMWU0wK2xFSWFlN2RONUNRNQpTbzVXaEwyY3l2NVZKN2xqL0VBS0NWaUlFZ0NtekRSRGNSZ1NTald5SDRibjZ5WDIwMjZmUEl5V0pGeUVkTC82CmpBT0pBRERSMEd5aE5PWHJFZXFob2NTTW5JYlFWcXdBVDBrTWh1WFN2d3Zscm5MeVRwRzVqWm00bFVNMzRrTUMKQXdFQUFhTlRNRkV3SFFZRFZSME9CQllFRkVJM1JNTWl5aUJqeVExUlM4bmxPUkpWZDFwQk1COEdBMVVkSXdRWQpNQmFBRkVJM1JNTWl5aUJqeVExUlM4bmxPUkpWZDFwQk1BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0RRWUpLb1pJCmh2Y05BUUVMQlFBRGdnRUJBSHRLUlhkRmR0VWh0VWpvZG1ZUWNlZEFEaEhaT2hCcEtpbnpvdTRicmRrNEhmaEYKTHIvV0ZsY1JlbWxWNm1Cc0xweU11SytUZDhaVUVRNkpFUkx5NmxTL2M2cE9HeG5CNGFDbEU4YXQrQytUakpBTwpWbTNXU0k2VlIxY0ZYR2VaamxkVlE2eGtRc2tNSnpPN2RmNmlNVFB0VjVSa01lSlh0TDZYYW1FaTU0ckJvZ05ICk5yYStFSkJRQmwvWmU5ME5qZVlidjIwdVFwWmFhWkZhYVNtVm9OSERwQndsYTBvdXkrTWpPYkMzU3BnT3ExSUMKUGwzTnV3TkxWOFZiT3I1SHJoUUFvS21nU05iM1A4dmFUVnV4L1gwWWZqeS9TN045a1BCYUs5bUZqNzR6d1Y5dwpxU1ExNEtsNWpPM1YzaHJHV1laRWpET2diWnJyRVgxS1hFdXN0K1E9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR5RENDQXJDZ0F3SUJBZ0lVQ0kyUE5OcnR6cDZRbDdHa3VhRnhtRGE2VUJvd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2V6RUxNQWtHQTFVRUJoTUNSRVV4R3pBWkJnTlZCQWdNRWxOamFHeGxjM2RwWnkxSWIyeHpkR1ZwYmpFTwpNQXdHQTFVRUJ3d0ZWMlZrWld3eEtEQW1CZ05WQkFvTUgxTjBZV05yWVdKc1pTQlRhV2R1YVc1bklFRjFkR2h2CmNtbDBlU0JKYm1NeEZUQVRCZ05WQkFNTURITjBZV05yWVdKc1pTNWtaVEFnRncweU16QTJNVFl4TWpVeE1ESmEKR0E4eU1USXpNRFV5TXpFeU5URXdNbG93WGpFTE1Ba0dBMVVFQmhNQ1JFVXhHekFaQmdOVkJBZ01FbE5qYUd4bApjM2RwWnkxSWIyeHpkR1ZwYmpFT01Bd0dBMVVFQnd3RlYyVmtaV3d4RWpBUUJnTlZCQW9NQ1ZOMFlXTnJZV0pzClpURU9NQXdHQTFVRUF3d0ZiV2x1YVc4d2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUIKQVFDanluVnorWEhCOE9DWTRwc0VFWW1qb2JwZHpUbG93d2NTUU4rWURQQ2tCZW9yMFRiODdFZ0x6SksrSllidQpwb1hCbE5JSlBRYW93SkVvL1N6U2s4ZnUyWFNNeXZBWlk0RldHeEp5Mnl4SXh2UC9pYk9HT1l1aVBHWEsyNHQ2ClpjR1RVVmhhdWlaR1Nna1dyZWpXV2g3TWpGUytjMXZhWVpxQitRMXpQczVQRk1sYzhsNVYvK2I4WjdqTUppODQKbU9mSVB4amt2SXlKcjVVa2VGM1VmTHFKUzV5NExGNHR5NEZ0MmlBZDdiYmZIYW5mdlltdjZVb0RWdE1YdFdvMQpvUVBmdjNzaFdybVJMenc2ZXVJQXRiWGM1Q2pCeUlha0NiaURuQVU4cktnK0IxSjRtdlFnckx3bzNxUHJ5Smd4ClNkaWRtWjJtRVI3RXorYzVCMG0vTGlJaEFnTUJBQUdqWHpCZE1Cc0dBMVVkRVFRVU1CS0NCVzFwYm1sdmdnbHMKYjJOaGJHaHZjM1F3SFFZRFZSME9CQllFRkpRMGdENWtFdFFyK3REcERTWjdrd1o4SDVoR01COEdBMVVkSXdRWQpNQmFBRkVJM1JNTWl5aUJqeVExUlM4bmxPUkpWZDFwQk1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQmNkaGQrClI0Sm9HdnFMQms1OWRxSVVlY2N0dUZzcmRQeHNCaU9GaFlOZ1pxZWRMTTBVTDVEenlmQUhmVk8wTGZTRURkZFgKUkpMOXlMNytrTVUwVDc2Y3ZkQzlYVkFJRTZIVXdUbzlHWXNQcXN1eVpvVmpOcEVESkN3WTNDdm9ubEpWZTRkcQovZ0FiSk1ZQitUU21ZNXlEUHovSkZZL1haellhUGI3T2RlR3VqYlZUNUl4cDk3QXBTOFlJaXY3M0Mwd1ViYzZSCmgwcmNmUmJ5a1NRVWg5dmdWZFhSU1I4RFQzV0NmZHFOek5CWVh2OW1xZlc1ejRzYkdqK2wzd1VsL0kzRi9tSXcKZnlPNEN0aTRha2lHVkhsZmZFeTB3a3pWYUJ4aGNYajJJM0JVVGhCNFpxamxzc2llVmFGa3d2WG1teVJUMG9FVwo1SCtOUEhjcXVTMXpQc2NsCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2QUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktZd2dnU2lBZ0VBQW9JQkFRQ2p5blZ6K1hIQjhPQ1kKNHBzRUVZbWpvYnBkelRsb3d3Y1NRTitZRFBDa0Jlb3IwVGI4N0VnTHpKSytKWWJ1cG9YQmxOSUpQUWFvd0pFbwovU3pTazhmdTJYU015dkFaWTRGV0d4SnkyeXhJeHZQL2liT0dPWXVpUEdYSzI0dDZaY0dUVVZoYXVpWkdTZ2tXCnJlaldXaDdNakZTK2MxdmFZWnFCK1ExelBzNVBGTWxjOGw1Vi8rYjhaN2pNSmk4NG1PZklQeGprdkl5SnI1VWsKZUYzVWZMcUpTNXk0TEY0dHk0RnQyaUFkN2JiZkhhbmZ2WW12NlVvRFZ0TVh0V28xb1FQZnYzc2hXcm1STHp3NgpldUlBdGJYYzVDakJ5SWFrQ2JpRG5BVThyS2crQjFKNG12UWdyTHdvM3FQcnlKZ3hTZGlkbVoybUVSN0V6K2M1CkIwbS9MaUloQWdNQkFBRUNnZ0VBQWQzdDVzdUNFMjdXY0llc3NxZ3NoSFAwZHRzKyswVzF6K3h6WC8xTnhPRFkKWVhWNkJmbi9mRHJ4dFQ4aVFaZ2VVQzJORTFQaHZveXJXdWMvMm9xYXJjdEd1OUFZV29HNjJLdG9VMnpTSFdZLwpJN3VERTFXV2xOdlJZVFdOYW5DOGV4eGpRRzE4d0RKWjFpdFhTeEl0NWJEM3lrL3dUUlh0dCt1SnpyVjVqb2N1CmNoeERMd293aXUxQWo2ZFJDWk5CejlUSnh5TnI1ME5ZVzJVWEJhVC84N1hyRkZkSndNVFZUMEI3SE9uRzdSQlYKUWxLdzhtcVZiYU5lbmhjdk1qUjI5c3hUekhSK2p4SU8zQndPNk9Hai9PRmhGQllVN1RMWGVsZDFxb2UwdmIyRwpiOGhQcEd1cHRyNUF0OWx3MXc1d1EzSWdpdXRQTkg1cXlEeUNwRWw2RVFLQmdRRGNkYnNsT2ZLSmo3TzJMQXlZCkZ0a1RwaWxFMFYzajBxbVE5M0lqclY0K0RSbUxNRUIyOTk0MDdCVVlRUWoxL0RJYlFjb1oyRUVjVUI1cGRlSHMKN0RNRUQ2WExIYjJKVTEyK2E3c1d5Q05kS2VjZStUNy9JYmxJOFR0MzQwVWxIUTZ6U01TRGNqdmZjRkhWZ3YwcwpDYWpoRng3TmtMRVhUWnI4ZlQzWUloajR2UUtCZ1FDK01nWjFVbW9KdzlJQVFqMnVJVTVDeTl4aldlWURUQU8vCllhWEl6d2xnZTQzOE1jYmI0Y04yU2FOU0dEZ1Y3bnU1a3FpaWhwalBZV0lpaU9CcDlrVFJIWE9kUFc0N3N5ZUkKdDNrd3JwMnpWbFVnbGNNWlo2bW1WM1FWYUFOWmdqVTRSU3Y0ZS9WeFVMamJaYWZqUHRaUnNqWkdwSzBZVTFvdApWajhJZVE3Zk5RS0JnQ1ArWk11ekpsSW5VQ1FTRlF4UHpxbFNtN0pNckpPaHRXV2h3TlRxWFZTc050dHV5VmVqCktIaGpneDR1b0JQcFZSVDJMTlVEWmI0RnByRjVPYVhBK3FOVEdyS0s3SU1iUlZidHArSVVVeEhHNGFGQStIUVgKUVhVVFRhNUpRT1RLVmJnWHpWM1lyTVhTUk1valZNcDMyVWJHeTVTc1p2MXpBamJ2QzhYWjYxSFJBb0dBZEJjUQp2aGU1eFpBUzVEbUtjSGkvemlHa3ViZXJuNk9NUGdxYUtJSEdsVytVOExScFR0ajBkNFRtL1Rydk1PUEovVEU1CllVcUtoenBIcmhDaCtjdHBvY0k2U1dXdm5SenpLbzNpbVFaY0Y1VEFqUTBjY3F0RmI5UzlkRHR5bi9YTUNqYWUKYWlNdll5VUVVRll5TFpDelBGWnNycDNoVVpHKzN5RmZoQXB3TzJrQ2dZQkh3WWFQSWRXNld3NytCMmhpbjBvdwpqYTNjZXN2QTRqYU1Qd1NMVDhPTnRVMUdCU01md2N6TWJuUEhMclJ2Qjg3bjlnUGFSMndRR1VtckZFTzNMUFgvCmtSY09HcFlCSHBEWEVqRGhLa1dkUnVMT0ZnNEhMWmRWOEFOWmxRMFZTY0U4dTNkRERVTzg5cEdEbjA4cVRBcmwKeDlreHN1ZEVWcmtlclpiNVV4RlZxUT09Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K +--- +apiVersion: s3.stackable.tech/v1alpha1 +kind: S3Connection +metadata: + name: minio +spec: + host: minio + port: 9000 + accessStyle: Path + credentials: + secretClass: s3-credentials-class + tls: + verification: + server: + caCert: + secretClass: minio-tls-certificates diff --git a/tests/templates/kuttl/delta/02-install-minio.yaml.j2 b/tests/templates/kuttl/delta/02-install-minio.yaml.j2 new file mode 100644 index 00000000..0be8b7a3 --- /dev/null +++ b/tests/templates/kuttl/delta/02-install-minio.yaml.j2 @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install minio + --namespace $NAMESPACE + --version 12.6.4 + -f helm-bitnami-minio-values.yaml + --repo https://charts.bitnami.com/bitnami minio + timeout: 240 diff --git a/tests/templates/kuttl/delta/04-prepare-bucket.yaml.j2 b/tests/templates/kuttl/delta/04-prepare-bucket.yaml.j2 new file mode 100644 index 00000000..ff712680 --- /dev/null +++ b/tests/templates/kuttl/delta/04-prepare-bucket.yaml.j2 @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # give minio enough time to start + - command: sleep 5 + - script: | + POD=$(kubectl -n $NAMESPACE get pod -l app.kubernetes.io/instance=minio -o name | head -n1 | sed 's#pod/##') + kubectl -n $NAMESPACE cp delta-table "$POD:/tmp/delta-table" + kubectl -n $NAMESPACE exec "$POD" -- mc rm -r --force local/trino/delta-table || true + kubectl -n $NAMESPACE exec "$POD" -- mc mirror --overwrite /tmp/delta-table local/trino/delta-table diff --git a/tests/templates/kuttl/delta/07-assert.yaml b/tests/templates/kuttl/delta/07-assert.yaml new file mode 100644 index 00000000..1ac12423 --- /dev/null +++ b/tests/templates/kuttl/delta/07-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/delta/07-install-postgres.yaml b/tests/templates/kuttl/delta/07-install-postgres.yaml new file mode 100644 index 00000000..93c93a16 --- /dev/null +++ b/tests/templates/kuttl/delta/07-install-postgres.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 300 +commands: + - script: >- + helm upgrade postgresql + --install + --version=12.5.6 + --namespace $NAMESPACE + -f helm-bitnami-postgresql-values.yaml + --repo https://charts.bitnami.com/bitnami postgresql diff --git a/tests/templates/kuttl/delta/08-assert.yaml b/tests/templates/kuttl/delta/08-assert.yaml new file mode 100644 index 00000000..50c27fd9 --- /dev/null +++ b/tests/templates/kuttl/delta/08-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 900 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hive-metastore-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/delta/08-install-hive.yaml.j2 b/tests/templates/kuttl/delta/08-install-hive.yaml.j2 new file mode 100644 index 00000000..9eff5093 --- /dev/null +++ b/tests/templates/kuttl/delta/08-install-hive.yaml.j2 @@ -0,0 +1,35 @@ +--- +apiVersion: hive.stackable.tech/v1alpha1 +kind: HiveCluster +metadata: + name: hive +spec: + image: + productVersion: "{{ test_scenario['values']['hive-latest'] }}" + pullPolicy: IfNotPresent + clusterConfig: + database: + connString: jdbc:postgresql://postgresql:5432/hive + credentialsSecret: postgres-credentials + dbType: postgres + s3: + reference: minio +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + metastore: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-credentials +type: Opaque +stringData: + username: hive + password: hive diff --git a/tests/templates/kuttl/delta/10-assert.yaml b/tests/templates/kuttl/delta/10-assert.yaml new file mode 100644 index 00000000..a7d76fff --- /dev/null +++ b/tests/templates/kuttl/delta/10-assert.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 720 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino --timeout 301s +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: trino-coordinator-default +spec: + template: + spec: + terminationGracePeriodSeconds: 900 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: trino-worker-default +spec: + template: + spec: + terminationGracePeriodSeconds: 75 # 5s gracefulShutdownTimeout + 2 x 30s grace period + 10s safety buffer +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: trino-coordinator +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: trino-worker +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 diff --git a/tests/templates/kuttl/delta/10-install-trino.yaml.j2 b/tests/templates/kuttl/delta/10-install-trino.yaml.j2 new file mode 100644 index 00000000..48a4ef42 --- /dev/null +++ b/tests/templates/kuttl/delta/10-install-trino.yaml.j2 @@ -0,0 +1,99 @@ +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + # The name of the catalog as it will appear in Trino + name: delta + labels: + trino: trino +spec: + connector: + # Specify deltaLake here when defining a Delta Lake catalog + deltaLake: + metastore: + configMap: hive + s3: + reference: minio + configOverrides: + # + # Needed for writing (deleting) data on S3 + # + delta.enable-non-concurrent-writes: 'true' + # + # Needed for the register_table() call in the Python test script. + # + delta.register-table-procedure.enabled: 'true' + # + # 2026-03-27: This is needed to avoid this bug: https://github.com/trinodb/trino/issues/28885 + # Should be removed as soon as that issue is solved or we have a patched Trino version. + # + parquet.use-column-index: 'false' +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: trino-users-auth +spec: + provider: + static: + userCredentialsSecret: + name: trino-users +--- +apiVersion: v1 +kind: Secret +metadata: + name: trino-users +type: kubernetes.io/opaque +stringData: + admin: admin + alice: alice + bob: bob +--- +apiVersion: v1 +kind: Secret +metadata: + name: my-postgresql-credentials-secret +stringData: + user: hive + password: hive +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCluster +metadata: + name: trino +spec: + image: +{% if test_scenario['values']['trino-delta'].find(",") > 0 %} + custom: "{{ test_scenario['values']['trino-delta'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['trino-delta'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['trino-delta'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + catalogLabelSelector: + matchLabels: + trino: trino + authentication: + - authenticationClass: trino-users-auth +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + coordinators: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + envOverrides: + COMMON_VAR: role-value # overridden by role group below + ROLE_VAR: role-value # only defined here at role level + roleGroups: + default: + replicas: 1 + workers: + config: + gracefulShutdownTimeout: 5s # Let the test run faster + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/delta/20-assert.yaml b/tests/templates/kuttl/delta/20-assert.yaml new file mode 100644 index 00000000..f926c3d0 --- /dev/null +++ b/tests/templates/kuttl/delta/20-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 120 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: delta-check +status: + succeeded: 1 diff --git a/tests/templates/kuttl/delta/20-run-delta-queries.yaml b/tests/templates/kuttl/delta/20-run-delta-queries.yaml new file mode 100644 index 00000000..95cd8a4b --- /dev/null +++ b/tests/templates/kuttl/delta/20-run-delta-queries.yaml @@ -0,0 +1,116 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: delta-check-script +data: + check-delta.py: | + #!/usr/bin/env python + # + # This script is used to run a series of SQL statements against the Trino cluster to verify that the Delta Lake connector is working correctly. + # + # It assumes that data is already present in the S3 location and that the Trino cluster is configured to access it. The script will: + # 1. Register the Delta table using the system.register_table procedure. + # 2. Run a SELECT query to verify that the expected data is present. + # 3. Run a DELETE query to remove some data. + # 4. Run another SELECT query to verify that the data was deleted. + # + import argparse + import sys + + import trino + + if not sys.warnoptions: + import warnings + + warnings.simplefilter("ignore") + + def get_connection(username, password, coordinator): + conn = trino.dbapi.connect( + host=coordinator, + port=8443, + user=username, + http_scheme="https", + auth=trino.auth.BasicAuthentication(username, password), + session_properties={"query_max_execution_time": "60s"}, + ) + conn._http_session.verify = False + return conn + + + def run_query(connection, query): + print(f"[DEBUG] Executing query {query}") + cursor = connection.cursor() + cursor.execute(query) + return cursor.fetchall() + + + if __name__ == "__main__": + all_args = argparse.ArgumentParser() + all_args.add_argument( + "-c", + "--coordinator", + required=True, + help="Trino Coordinator Host to connect to", + ) + args = vars(all_args.parse_args()) + + coordinator = args["coordinator"] + + print("[INFO] Running Delta SQL commands") + connection = get_connection("admin", "admin", coordinator) + + statement = "DROP TABLE IF EXISTS delta.default.delta_table" + run_query(connection, statement) + + # Must use the register_table procedure to create the table because data is already present in the S3 location + # and Trino's CREATE TABLE AS doesn't support that. + statement = "CALL delta.system.register_table(schema_name => 'default', table_name => 'delta_table', table_location => 's3a://trino/delta-table')" + run_query(connection, statement) + + statement = "select count(*) from delta.default.delta_table where text='text5'" + result = run_query(connection, statement) + assert result[0][0] == 5000, ( + f"Expected 5000 rows, got {result}\nQuery: {statement}" + ) + + statement = "delete from delta.default.delta_table where text = 'text5'" + run_query(connection, statement) + + statement = "select count(*) from delta.default.delta_table where text='text5'" + result = run_query(connection, statement) + assert result[0][0] == 0, ( + f"Expected 0 rows, got {result}\nQuery: {statement}" + ) + + print("[SUCCESS] Delta checks succeeded.") +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: delta-check +spec: + backoffLimit: 0 + template: + spec: + serviceAccountName: integration-tests-sa + restartPolicy: Never + containers: + - name: trino-test-helper + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev + command: ["python", "/tmp/check-delta.py", "-c", "trino-coordinator"] + resources: + requests: + cpu: "250m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "64Mi" + volumeMounts: + - name: delta-check-script + mountPath: /tmp/check-delta.py + subPath: check-delta.py + volumes: + - name: delta-check-script + configMap: + name: delta-check-script diff --git a/tests/templates/kuttl/delta/certs/ca.crt b/tests/templates/kuttl/delta/certs/ca.crt new file mode 100644 index 00000000..4d91d2cc --- /dev/null +++ b/tests/templates/kuttl/delta/certs/ca.crt @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID2TCCAsGgAwIBAgIUNjquGYWtyJ5a6wy23Hz2GRcMlwMwDQYJKoZIhvcNAQEL +BQAwezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxlc3dpZy1Ib2xzdGVpbjEO +MAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJsZSBTaWduaW5nIEF1dGhv +cml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTAgFw0yMzA2MTYxMjUxMDJa +GA8yMTIzMDUyMzEyNTEwMlowezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxl +c3dpZy1Ib2xzdGVpbjEOMAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJs +ZSBTaWduaW5nIEF1dGhvcml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANnV/vby3Ro57a2qvQRnn0je +eKMU2+F0+lZNCAviGUD5bm8jk91oPZnk0bhQqeyErmDS4WT0zevERIBJJDjfL0D8 +46Be7PiMKe0dGjoqI3z5cOIejc8aLPHSIlgN6lT3fIruS16coQgG4uaKiHF5+eWF +DRULdu6dsYuz6dKjqRiUOhHwDwtUJkDwPv+EItqo0H+MLFLLYM0+lEIae7dN5CQ5 +So5WhL2cyv5VJ7lj/EAKCViIEgCmzDRDcRgSSjWyH4bn6yX2026fPIyWJFyEdL/6 +jAOJADDR0GyhNOXrEeqhocSMnIbQVqwAT0kMhuXSvwvlrnLyTpG5jZm4lUM34kMC +AwEAAaNTMFEwHQYDVR0OBBYEFEI3RMMiyiBjyQ1RS8nlORJVd1pBMB8GA1UdIwQY +MBaAFEI3RMMiyiBjyQ1RS8nlORJVd1pBMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBAHtKRXdFdtUhtUjodmYQcedADhHZOhBpKinzou4brdk4HfhF +Lr/WFlcRemlV6mBsLpyMuK+Td8ZUEQ6JERLy6lS/c6pOGxnB4aClE8at+C+TjJAO +Vm3WSI6VR1cFXGeZjldVQ6xkQskMJzO7df6iMTPtV5RkMeJXtL6XamEi54rBogNH +Nra+EJBQBl/Ze90NjeYbv20uQpZaaZFaaSmVoNHDpBwla0ouy+MjObC3SpgOq1IC +Pl3NuwNLV8VbOr5HrhQAoKmgSNb3P8vaTVux/X0Yfjy/S7N9kPBaK9mFj74zwV9w +qSQ14Kl5jO3V3hrGWYZEjDOgbZrrEX1KXEust+Q= +-----END CERTIFICATE----- diff --git a/tests/templates/kuttl/delta/certs/client.crt.pem b/tests/templates/kuttl/delta/certs/client.crt.pem new file mode 100644 index 00000000..1f35befd --- /dev/null +++ b/tests/templates/kuttl/delta/certs/client.crt.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDyDCCArCgAwIBAgIUCI2PNNrtzp6Ql7GkuaFxmDa6UBowDQYJKoZIhvcNAQEL +BQAwezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxlc3dpZy1Ib2xzdGVpbjEO +MAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJsZSBTaWduaW5nIEF1dGhv +cml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTAgFw0yMzA2MTYxMjUxMDJa +GA8yMTIzMDUyMzEyNTEwMlowXjELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxl +c3dpZy1Ib2xzdGVpbjEOMAwGA1UEBwwFV2VkZWwxEjAQBgNVBAoMCVN0YWNrYWJs +ZTEOMAwGA1UEAwwFbWluaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQCjynVz+XHB8OCY4psEEYmjobpdzTlowwcSQN+YDPCkBeor0Tb87EgLzJK+JYbu +poXBlNIJPQaowJEo/SzSk8fu2XSMyvAZY4FWGxJy2yxIxvP/ibOGOYuiPGXK24t6 +ZcGTUVhauiZGSgkWrejWWh7MjFS+c1vaYZqB+Q1zPs5PFMlc8l5V/+b8Z7jMJi84 +mOfIPxjkvIyJr5UkeF3UfLqJS5y4LF4ty4Ft2iAd7bbfHanfvYmv6UoDVtMXtWo1 +oQPfv3shWrmRLzw6euIAtbXc5CjByIakCbiDnAU8rKg+B1J4mvQgrLwo3qPryJgx +SdidmZ2mER7Ez+c5B0m/LiIhAgMBAAGjXzBdMBsGA1UdEQQUMBKCBW1pbmlvggls +b2NhbGhvc3QwHQYDVR0OBBYEFJQ0gD5kEtQr+tDpDSZ7kwZ8H5hGMB8GA1UdIwQY +MBaAFEI3RMMiyiBjyQ1RS8nlORJVd1pBMA0GCSqGSIb3DQEBCwUAA4IBAQBcdhd+ +R4JoGvqLBk59dqIUecctuFsrdPxsBiOFhYNgZqedLM0UL5DzyfAHfVO0LfSEDddX +RJL9yL7+kMU0T76cvdC9XVAIE6HUwTo9GYsPqsuyZoVjNpEDJCwY3CvonlJVe4dq +/gAbJMYB+TSmY5yDPz/JFY/XZzYaPb7OdeGujbVT5Ixp97ApS8YIiv73C0wUbc6R +h0rcfRbykSQUh9vgVdXRSR8DT3WCfdqNzNBYXv9mqfW5z4sbGj+l3wUl/I3F/mIw +fyO4Cti4akiGVHlffEy0wkzVaBxhcXj2I3BUThB4ZqjlssieVaFkwvXmmyRT0oEW +5H+NPHcquS1zPscl +-----END CERTIFICATE----- diff --git a/tests/templates/kuttl/delta/certs/client.csr.pem b/tests/templates/kuttl/delta/certs/client.csr.pem new file mode 100644 index 00000000..f735c317 --- /dev/null +++ b/tests/templates/kuttl/delta/certs/client.csr.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC0TCCAbkCAQAwXjELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxlc3dpZy1I +b2xzdGVpbjEOMAwGA1UEBwwFV2VkZWwxEjAQBgNVBAoMCVN0YWNrYWJsZTEOMAwG +A1UEAwwFbWluaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCjynVz ++XHB8OCY4psEEYmjobpdzTlowwcSQN+YDPCkBeor0Tb87EgLzJK+JYbupoXBlNIJ +PQaowJEo/SzSk8fu2XSMyvAZY4FWGxJy2yxIxvP/ibOGOYuiPGXK24t6ZcGTUVha +uiZGSgkWrejWWh7MjFS+c1vaYZqB+Q1zPs5PFMlc8l5V/+b8Z7jMJi84mOfIPxjk +vIyJr5UkeF3UfLqJS5y4LF4ty4Ft2iAd7bbfHanfvYmv6UoDVtMXtWo1oQPfv3sh +WrmRLzw6euIAtbXc5CjByIakCbiDnAU8rKg+B1J4mvQgrLwo3qPryJgxSdidmZ2m +ER7Ez+c5B0m/LiIhAgMBAAGgLjAsBgkqhkiG9w0BCQ4xHzAdMBsGA1UdEQQUMBKC +BW1pbmlvgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQADggEBAIXs2PGYTo8N6IZc +eVVa82AxtJPSaIeRqrTuAiKloQPZxhpEaTSAR8wxWpzyIeilMgp8UuMo2M0euxGM +gxhzOyNXFekDSoLbuvKHidhvJ+rVEVHag3VdKA22P4/OYV8HwP6yXsNWNXK6Sp6J +pKwRE3PpSN4vPbEmbxLndM9SOVghV9RCVdLMPFXg+pfTNPm2H3cYGg4yU+Cdl5Dj +voUOQzRMuvflScf+gzjBIx7xVvwo/d9szsyqPfMyTlK40kU+KGl5Mz+C7Icyljnj +8F92l4NbDrXpWuyNjoUwEH8Kdb4ioPACHgStTY0Js8vdVS7wWj6ylXPBRUd9Yxxg +BWC7YHc= +-----END CERTIFICATE REQUEST----- diff --git a/tests/templates/kuttl/delta/certs/client.key.pem b/tests/templates/kuttl/delta/certs/client.key.pem new file mode 100644 index 00000000..d8820420 --- /dev/null +++ b/tests/templates/kuttl/delta/certs/client.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCjynVz+XHB8OCY +4psEEYmjobpdzTlowwcSQN+YDPCkBeor0Tb87EgLzJK+JYbupoXBlNIJPQaowJEo +/SzSk8fu2XSMyvAZY4FWGxJy2yxIxvP/ibOGOYuiPGXK24t6ZcGTUVhauiZGSgkW +rejWWh7MjFS+c1vaYZqB+Q1zPs5PFMlc8l5V/+b8Z7jMJi84mOfIPxjkvIyJr5Uk +eF3UfLqJS5y4LF4ty4Ft2iAd7bbfHanfvYmv6UoDVtMXtWo1oQPfv3shWrmRLzw6 +euIAtbXc5CjByIakCbiDnAU8rKg+B1J4mvQgrLwo3qPryJgxSdidmZ2mER7Ez+c5 +B0m/LiIhAgMBAAECggEAAd3t5suCE27WcIessqgshHP0dts++0W1z+xzX/1NxODY +YXV6Bfn/fDrxtT8iQZgeUC2NE1PhvoyrWuc/2oqarctGu9AYWoG62KtoU2zSHWY/ +I7uDE1WWlNvRYTWNanC8exxjQG18wDJZ1itXSxIt5bD3yk/wTRXtt+uJzrV5jocu +chxDLwowiu1Aj6dRCZNBz9TJxyNr50NYW2UXBaT/87XrFFdJwMTVT0B7HOnG7RBV +QlKw8mqVbaNenhcvMjR29sxTzHR+jxIO3BwO6OGj/OFhFBYU7TLXeld1qoe0vb2G +b8hPpGuptr5At9lw1w5wQ3IgiutPNH5qyDyCpEl6EQKBgQDcdbslOfKJj7O2LAyY +FtkTpilE0V3j0qmQ93IjrV4+DRmLMEB299407BUYQQj1/DIbQcoZ2EEcUB5pdeHs +7DMED6XLHb2JU12+a7sWyCNdKece+T7/IblI8Tt340UlHQ6zSMSDcjvfcFHVgv0s +CajhFx7NkLEXTZr8fT3YIhj4vQKBgQC+MgZ1UmoJw9IAQj2uIU5Cy9xjWeYDTAO/ +YaXIzwlge438Mcbb4cN2SaNSGDgV7nu5kqiihpjPYWIiiOBp9kTRHXOdPW47syeI +t3kwrp2zVlUglcMZZ6mmV3QVaANZgjU4RSv4e/VxULjbZafjPtZRsjZGpK0YU1ot +Vj8IeQ7fNQKBgCP+ZMuzJlInUCQSFQxPzqlSm7JMrJOhtWWhwNTqXVSsNttuyVej +KHhjgx4uoBPpVRT2LNUDZb4FprF5OaXA+qNTGrKK7IMbRVbtp+IUUxHG4aFA+HQX +QXUTTa5JQOTKVbgXzV3YrMXSRMojVMp32UbGy5SsZv1zAjbvC8XZ61HRAoGAdBcQ +vhe5xZAS5DmKcHi/ziGkubern6OMPgqaKIHGlW+U8LRpTtj0d4Tm/TrvMOPJ/TE5 +YUqKhzpHrhCh+ctpocI6SWWvnRzzKo3imQZcF5TAjQ0ccqtFb9S9dDtyn/XMCjae +aiMvYyUEUFYyLZCzPFZsrp3hUZG+3yFfhApwO2kCgYBHwYaPIdW6Ww7+B2hin0ow +ja3cesvA4jaMPwSLT8ONtU1GBSMfwczMbnPHLrRvB87n9gPaR2wQGUmrFEO3LPX/ +kRcOGpYBHpDXEjDhKkWdRuLOFg4HLZdV8ANZlQ0VScE8u3dDDUO89pGDn08qTArl +x9kxsudEVrkerZb5UxFVqQ== +-----END PRIVATE KEY----- diff --git a/tests/templates/kuttl/delta/certs/generate.sh b/tests/templates/kuttl/delta/certs/generate.sh new file mode 100755 index 00000000..bbf0cd22 --- /dev/null +++ b/tests/templates/kuttl/delta/certs/generate.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +echo "Creating client cert" +FQDN="minio" + +echo "Creating Root Certificate Authority" +openssl genrsa \ + -out root-ca.key.pem \ + 2048 + +echo "Self-signing the Root Certificate Authority" +openssl req \ + -x509 \ + -new \ + -nodes \ + -key root-ca.key.pem \ + -days 36500 \ + -out root-ca.crt.pem \ + -subj "/C=DE/ST=Schleswig-Holstein/L=Wedel/O=Stackable Signing Authority Inc/CN=stackable.de" + +openssl genrsa \ + -out client.key.pem \ + 2048 + +echo "Creating the CSR" +openssl req -new \ + -key client.key.pem \ + -out client.csr.pem \ + -subj "/C=DE/ST=Schleswig-Holstein/L=Wedel/O=Stackable/CN=${FQDN}" \ + -addext "subjectAltName = DNS:${FQDN}, DNS:localhost" + +echo "Signing the client cert with the root ca" +openssl x509 \ + -req -in client.csr.pem \ + -CA root-ca.crt.pem \ + -CAkey root-ca.key.pem \ + -CAcreateserial \ + -out client.crt.pem \ + -days 36500 \ + -copy_extensions copy + +echo "Copying the files to match the api of the secret-operator" +cp root-ca.crt.pem ca.crt +cp client.key.pem tls.key +cp client.crt.pem tls.crt + +echo "To create a k8s secret run" +echo "kubectl create secret generic foo --from-file=ca.crt --from-file=tls.crt --from-file=tls.key" diff --git a/tests/templates/kuttl/delta/certs/root-ca.crt.pem b/tests/templates/kuttl/delta/certs/root-ca.crt.pem new file mode 100644 index 00000000..4d91d2cc --- /dev/null +++ b/tests/templates/kuttl/delta/certs/root-ca.crt.pem @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIID2TCCAsGgAwIBAgIUNjquGYWtyJ5a6wy23Hz2GRcMlwMwDQYJKoZIhvcNAQEL +BQAwezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxlc3dpZy1Ib2xzdGVpbjEO +MAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJsZSBTaWduaW5nIEF1dGhv +cml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTAgFw0yMzA2MTYxMjUxMDJa +GA8yMTIzMDUyMzEyNTEwMlowezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxl +c3dpZy1Ib2xzdGVpbjEOMAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJs +ZSBTaWduaW5nIEF1dGhvcml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANnV/vby3Ro57a2qvQRnn0je +eKMU2+F0+lZNCAviGUD5bm8jk91oPZnk0bhQqeyErmDS4WT0zevERIBJJDjfL0D8 +46Be7PiMKe0dGjoqI3z5cOIejc8aLPHSIlgN6lT3fIruS16coQgG4uaKiHF5+eWF +DRULdu6dsYuz6dKjqRiUOhHwDwtUJkDwPv+EItqo0H+MLFLLYM0+lEIae7dN5CQ5 +So5WhL2cyv5VJ7lj/EAKCViIEgCmzDRDcRgSSjWyH4bn6yX2026fPIyWJFyEdL/6 +jAOJADDR0GyhNOXrEeqhocSMnIbQVqwAT0kMhuXSvwvlrnLyTpG5jZm4lUM34kMC +AwEAAaNTMFEwHQYDVR0OBBYEFEI3RMMiyiBjyQ1RS8nlORJVd1pBMB8GA1UdIwQY +MBaAFEI3RMMiyiBjyQ1RS8nlORJVd1pBMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggEBAHtKRXdFdtUhtUjodmYQcedADhHZOhBpKinzou4brdk4HfhF +Lr/WFlcRemlV6mBsLpyMuK+Td8ZUEQ6JERLy6lS/c6pOGxnB4aClE8at+C+TjJAO +Vm3WSI6VR1cFXGeZjldVQ6xkQskMJzO7df6iMTPtV5RkMeJXtL6XamEi54rBogNH +Nra+EJBQBl/Ze90NjeYbv20uQpZaaZFaaSmVoNHDpBwla0ouy+MjObC3SpgOq1IC +Pl3NuwNLV8VbOr5HrhQAoKmgSNb3P8vaTVux/X0Yfjy/S7N9kPBaK9mFj74zwV9w +qSQ14Kl5jO3V3hrGWYZEjDOgbZrrEX1KXEust+Q= +-----END CERTIFICATE----- diff --git a/tests/templates/kuttl/delta/certs/root-ca.crt.srl b/tests/templates/kuttl/delta/certs/root-ca.crt.srl new file mode 100644 index 00000000..ca297b16 --- /dev/null +++ b/tests/templates/kuttl/delta/certs/root-ca.crt.srl @@ -0,0 +1 @@ +7046F738B5BC7F7DC43E1E7E2EF5B23832C7A59A diff --git a/tests/templates/kuttl/delta/certs/root-ca.key.pem b/tests/templates/kuttl/delta/certs/root-ca.key.pem new file mode 100644 index 00000000..bd7a07ae --- /dev/null +++ b/tests/templates/kuttl/delta/certs/root-ca.key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDZ1f728t0aOe2t +qr0EZ59I3nijFNvhdPpWTQgL4hlA+W5vI5PdaD2Z5NG4UKnshK5g0uFk9M3rxESA +SSQ43y9A/OOgXuz4jCntHRo6KiN8+XDiHo3PGizx0iJYDepU93yK7ktenKEIBuLm +iohxefnlhQ0VC3bunbGLs+nSo6kYlDoR8A8LVCZA8D7/hCLaqNB/jCxSy2DNPpRC +Gnu3TeQkOUqOVoS9nMr+VSe5Y/xACglYiBIApsw0Q3EYEko1sh+G5+sl9tNunzyM +liRchHS/+owDiQAw0dBsoTTl6xHqoaHEjJyG0FasAE9JDIbl0r8L5a5y8k6RuY2Z +uJVDN+JDAgMBAAECggEABVrQUKu5qapg4FMBIHmXncfyOTgLC6i/ep2cJAajzkgT +YeIDAX9NfFn2mcxJ0QmV68VjSwMFiNUjRfAGVuuNktBknA2ZT6bKZQzBF0rv4mOT +VcugesXO8wbSV03IQ9xtkFC5Q5MgFj1tGHOxVPDFptG1d533h3gS5DdA+S+SuYrn +n8JUqjenVzYgC5CFprDXEy/ZOC/Is/oq/GujC3e6VJINueCOOrVkNKhMtktq8qkg +UtkjZQYP/d0nzR8bYGXN818MBZOg+RyA0asgkDe+n6Lr6gNzaqhECDdITDejq/0h +D8ldKD4v8CYTRAY1AteIAF0jUg2YuWZwgQ8IzL0viQKBgQD2R0AB0RHhxHJsnWoQ +EhWPyD3fo3w8Q5dIxugpP2LcloyOH6Ew5xlwnmPu0wL31D8CHad+vZnUVEqMIZqk +rZy9r0EZc2JmC1Sgpv7NkpY3johBrONIAKsStTpNpr/3EB0rAeQ9KzS0TafeYXKt +buz8Fx3kcBPosLCs9+r/+TDzhQKBgQDib1K0oQV55pFTn9OkoHPTitLsOOwpLxjC +ui8R73PhhvCK3UNftsq3U/Gj6L3wi5ATeE0SC1xCu2ZvU8K7EBRflvoGMmWfAh+O +XMfE21yLyrSgUVhsDeC2VILyxIB0sgcT7uze3TjD2Dm0vg0NQrCNa8euLAn5WIHS +QE6jd8t1JwKBgAlLDP5EjmEvkYXJttveYtPnIXaT67c2cbn8T3xm+OsL/0fJp8J5 +pfsa7vhvG/iQGMSSq+RbcTeS6rE4/2Xhaz25JEK6mOby3IGna4wEUQjNpxSbWoQ4 +CjyNfCK7/Rhskj0yOBOa0sVO/NumX7ZtriGhGa6qEAZCzJfqTwLTu2YlAoGAaEmt +ZdPjmck/law+5cugjQWbL4DoA+/VD5qAo1oNnQlxMAPITAT8SIM4/6zqDie5K750 +gKMK0xFMlGmXfmBhgcUfUktT0nA/6GmC+H+vmBK8LjpI5ztdC5zQ0s79+sEj0WJx +ZhOtWUX1DfGaQUk912SUivttfJHu+M71aQR7iHECgYEAqIPW6opxX5p3bSrfGDY0 +vqcxpTLjBAUtCig0UoI+01PiuEudCB9ed2AWkk+h0KzvtKP2VW8r4tYJFeDU2Jt3 +s3mcO3Ix1cwHdb2CjzBm4dReyBsIUIzJRPl9spz0cRYhdQkREIHJUoCskwrqCsS0 +O9W/M6BZHjhM/7eA8StNHEU= +-----END PRIVATE KEY----- diff --git a/tests/templates/kuttl/delta/certs/tls.crt b/tests/templates/kuttl/delta/certs/tls.crt new file mode 100644 index 00000000..1f35befd --- /dev/null +++ b/tests/templates/kuttl/delta/certs/tls.crt @@ -0,0 +1,23 @@ +-----BEGIN CERTIFICATE----- +MIIDyDCCArCgAwIBAgIUCI2PNNrtzp6Ql7GkuaFxmDa6UBowDQYJKoZIhvcNAQEL +BQAwezELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxlc3dpZy1Ib2xzdGVpbjEO +MAwGA1UEBwwFV2VkZWwxKDAmBgNVBAoMH1N0YWNrYWJsZSBTaWduaW5nIEF1dGhv +cml0eSBJbmMxFTATBgNVBAMMDHN0YWNrYWJsZS5kZTAgFw0yMzA2MTYxMjUxMDJa +GA8yMTIzMDUyMzEyNTEwMlowXjELMAkGA1UEBhMCREUxGzAZBgNVBAgMElNjaGxl +c3dpZy1Ib2xzdGVpbjEOMAwGA1UEBwwFV2VkZWwxEjAQBgNVBAoMCVN0YWNrYWJs +ZTEOMAwGA1UEAwwFbWluaW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQCjynVz+XHB8OCY4psEEYmjobpdzTlowwcSQN+YDPCkBeor0Tb87EgLzJK+JYbu +poXBlNIJPQaowJEo/SzSk8fu2XSMyvAZY4FWGxJy2yxIxvP/ibOGOYuiPGXK24t6 +ZcGTUVhauiZGSgkWrejWWh7MjFS+c1vaYZqB+Q1zPs5PFMlc8l5V/+b8Z7jMJi84 +mOfIPxjkvIyJr5UkeF3UfLqJS5y4LF4ty4Ft2iAd7bbfHanfvYmv6UoDVtMXtWo1 +oQPfv3shWrmRLzw6euIAtbXc5CjByIakCbiDnAU8rKg+B1J4mvQgrLwo3qPryJgx +SdidmZ2mER7Ez+c5B0m/LiIhAgMBAAGjXzBdMBsGA1UdEQQUMBKCBW1pbmlvggls +b2NhbGhvc3QwHQYDVR0OBBYEFJQ0gD5kEtQr+tDpDSZ7kwZ8H5hGMB8GA1UdIwQY +MBaAFEI3RMMiyiBjyQ1RS8nlORJVd1pBMA0GCSqGSIb3DQEBCwUAA4IBAQBcdhd+ +R4JoGvqLBk59dqIUecctuFsrdPxsBiOFhYNgZqedLM0UL5DzyfAHfVO0LfSEDddX +RJL9yL7+kMU0T76cvdC9XVAIE6HUwTo9GYsPqsuyZoVjNpEDJCwY3CvonlJVe4dq +/gAbJMYB+TSmY5yDPz/JFY/XZzYaPb7OdeGujbVT5Ixp97ApS8YIiv73C0wUbc6R +h0rcfRbykSQUh9vgVdXRSR8DT3WCfdqNzNBYXv9mqfW5z4sbGj+l3wUl/I3F/mIw +fyO4Cti4akiGVHlffEy0wkzVaBxhcXj2I3BUThB4ZqjlssieVaFkwvXmmyRT0oEW +5H+NPHcquS1zPscl +-----END CERTIFICATE----- diff --git a/tests/templates/kuttl/delta/certs/tls.key b/tests/templates/kuttl/delta/certs/tls.key new file mode 100644 index 00000000..d8820420 --- /dev/null +++ b/tests/templates/kuttl/delta/certs/tls.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCjynVz+XHB8OCY +4psEEYmjobpdzTlowwcSQN+YDPCkBeor0Tb87EgLzJK+JYbupoXBlNIJPQaowJEo +/SzSk8fu2XSMyvAZY4FWGxJy2yxIxvP/ibOGOYuiPGXK24t6ZcGTUVhauiZGSgkW +rejWWh7MjFS+c1vaYZqB+Q1zPs5PFMlc8l5V/+b8Z7jMJi84mOfIPxjkvIyJr5Uk +eF3UfLqJS5y4LF4ty4Ft2iAd7bbfHanfvYmv6UoDVtMXtWo1oQPfv3shWrmRLzw6 +euIAtbXc5CjByIakCbiDnAU8rKg+B1J4mvQgrLwo3qPryJgxSdidmZ2mER7Ez+c5 +B0m/LiIhAgMBAAECggEAAd3t5suCE27WcIessqgshHP0dts++0W1z+xzX/1NxODY +YXV6Bfn/fDrxtT8iQZgeUC2NE1PhvoyrWuc/2oqarctGu9AYWoG62KtoU2zSHWY/ +I7uDE1WWlNvRYTWNanC8exxjQG18wDJZ1itXSxIt5bD3yk/wTRXtt+uJzrV5jocu +chxDLwowiu1Aj6dRCZNBz9TJxyNr50NYW2UXBaT/87XrFFdJwMTVT0B7HOnG7RBV +QlKw8mqVbaNenhcvMjR29sxTzHR+jxIO3BwO6OGj/OFhFBYU7TLXeld1qoe0vb2G +b8hPpGuptr5At9lw1w5wQ3IgiutPNH5qyDyCpEl6EQKBgQDcdbslOfKJj7O2LAyY +FtkTpilE0V3j0qmQ93IjrV4+DRmLMEB299407BUYQQj1/DIbQcoZ2EEcUB5pdeHs +7DMED6XLHb2JU12+a7sWyCNdKece+T7/IblI8Tt340UlHQ6zSMSDcjvfcFHVgv0s +CajhFx7NkLEXTZr8fT3YIhj4vQKBgQC+MgZ1UmoJw9IAQj2uIU5Cy9xjWeYDTAO/ +YaXIzwlge438Mcbb4cN2SaNSGDgV7nu5kqiihpjPYWIiiOBp9kTRHXOdPW47syeI +t3kwrp2zVlUglcMZZ6mmV3QVaANZgjU4RSv4e/VxULjbZafjPtZRsjZGpK0YU1ot +Vj8IeQ7fNQKBgCP+ZMuzJlInUCQSFQxPzqlSm7JMrJOhtWWhwNTqXVSsNttuyVej +KHhjgx4uoBPpVRT2LNUDZb4FprF5OaXA+qNTGrKK7IMbRVbtp+IUUxHG4aFA+HQX +QXUTTa5JQOTKVbgXzV3YrMXSRMojVMp32UbGy5SsZv1zAjbvC8XZ61HRAoGAdBcQ +vhe5xZAS5DmKcHi/ziGkubern6OMPgqaKIHGlW+U8LRpTtj0d4Tm/TrvMOPJ/TE5 +YUqKhzpHrhCh+ctpocI6SWWvnRzzKo3imQZcF5TAjQ0ccqtFb9S9dDtyn/XMCjae +aiMvYyUEUFYyLZCzPFZsrp3hUZG+3yFfhApwO2kCgYBHwYaPIdW6Ww7+B2hin0ow +ja3cesvA4jaMPwSLT8ONtU1GBSMfwczMbnPHLrRvB87n9gPaR2wQGUmrFEO3LPX/ +kRcOGpYBHpDXEjDhKkWdRuLOFg4HLZdV8ANZlQ0VScE8u3dDDUO89pGDn08qTArl +x9kxsudEVrkerZb5UxFVqQ== +-----END PRIVATE KEY----- diff --git a/tests/templates/kuttl/delta/delta-table/_delta_log/00000000000000000000.crc b/tests/templates/kuttl/delta/delta-table/_delta_log/00000000000000000000.crc new file mode 100644 index 00000000..167b50cd --- /dev/null +++ b/tests/templates/kuttl/delta/delta-table/_delta_log/00000000000000000000.crc @@ -0,0 +1 @@ +{"txnId":"5bec127e-ea8a-4eef-ba09-3e549a66fc0c","tableSizeBytes":702,"numFiles":1,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"0e582e3d-6324-4194-bab0-4b3039c761bc","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"year\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"id_mandant\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"text\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["year","id_mandant"],"configuration":{},"createdTime":1774618197458},"protocol":{"minReaderVersion":1,"minWriterVersion":2},"allFiles":[{"path":"year=2026/id_mandant=0157/part-00000-45386c31-ae84-4fab-9d03-a074a8bae1a2.c000.snappy.parquet","partitionValues":{"year":"2026","id_mandant":"0157"},"size":702,"modificationTime":1774618199000,"dataChange":false,"stats":"{\"numRecords\":50000,\"minValues\":{\"text\":\"text0\"},\"maxValues\":{\"text\":\"text9\"},\"nullCount\":{\"text\":0}}"}]} diff --git a/tests/templates/kuttl/delta/delta-table/_delta_log/00000000000000000000.json b/tests/templates/kuttl/delta/delta-table/_delta_log/00000000000000000000.json new file mode 100644 index 00000000..a7285bf3 --- /dev/null +++ b/tests/templates/kuttl/delta/delta-table/_delta_log/00000000000000000000.json @@ -0,0 +1,4 @@ +{"commitInfo":{"timestamp":1774618199441,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"year\",\"id_mandant\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"50000","numOutputBytes":"702"},"engineInfo":"Apache-Spark/3.5.5 Delta-Lake/3.3.2","txnId":"5bec127e-ea8a-4eef-ba09-3e549a66fc0c"}} +{"metaData":{"id":"0e582e3d-6324-4194-bab0-4b3039c761bc","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"year\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"id_mandant\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"text\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["year","id_mandant"],"configuration":{},"createdTime":1774618197458}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"add":{"path":"year=2026/id_mandant=0157/part-00000-45386c31-ae84-4fab-9d03-a074a8bae1a2.c000.snappy.parquet","partitionValues":{"year":"2026","id_mandant":"0157"},"size":702,"modificationTime":1774618199000,"dataChange":true,"stats":"{\"numRecords\":50000,\"minValues\":{\"text\":\"text0\"},\"maxValues\":{\"text\":\"text9\"},\"nullCount\":{\"text\":0}}"}} diff --git a/tests/templates/kuttl/delta/delta-table/year=2026/id_mandant=0157/part-00000-45386c31-ae84-4fab-9d03-a074a8bae1a2.c000.snappy.parquet b/tests/templates/kuttl/delta/delta-table/year=2026/id_mandant=0157/part-00000-45386c31-ae84-4fab-9d03-a074a8bae1a2.c000.snappy.parquet new file mode 100644 index 00000000..ded3f22c Binary files /dev/null and b/tests/templates/kuttl/delta/delta-table/year=2026/id_mandant=0157/part-00000-45386c31-ae84-4fab-9d03-a074a8bae1a2.c000.snappy.parquet differ diff --git a/tests/templates/kuttl/delta/helm-bitnami-minio-values.yaml.j2 b/tests/templates/kuttl/delta/helm-bitnami-minio-values.yaml.j2 new file mode 100644 index 00000000..3436e86b --- /dev/null +++ b/tests/templates/kuttl/delta/helm-bitnami-minio-values.yaml.j2 @@ -0,0 +1,76 @@ +--- +global: + security: + allowInsecureImages: true # needed starting with Chart version 14.9.0 if modifying images + +image: + repository: bitnamilegacy/minio +clientImage: + repository: bitnamilegacy/minio-client +defaultInitContainers: + volumePermissions: # volumePermissions moved under defaultInitContainers starting with Chart version 17.0.0 + enabled: false + image: + repository: bitnamilegacy/os-shell +console: + image: + repository: bitnamilegacy/minio-object-browser + +mode: standalone +disableWebUI: false +extraEnvVars: + - name: MINIO_LOG_LEVEL + value: DEBUG + +#defaultBuckets: hive + +provisioning: + enabled: true + buckets: + - name: trino + usersExistingSecrets: + - centralized-minio-users + resources: + requests: + memory: 1Gi + cpu: "512m" + limits: + memory: "1Gi" + cpu: "1" + podSecurityContext: + enabled: false + containerSecurityContext: + enabled: false + +# volumePermissions can be removed starting with Chart version 17.0.0, moved under defaultInitContainers +volumePermissions: + enabled: false + image: + repository: bitnamilegacy/os-shell + +podSecurityContext: + enabled: false + +containerSecurityContext: + enabled: false + +persistence: + enabled: false + +resources: + requests: + memory: 1Gi + cpu: "512m" + limits: + memory: "1Gi" + cpu: "1" + +auth: + existingSecret: minio-credentials + +service: + type: NodePort + +tls: + enabled: true + existingSecret: minio-tls-certificates diff --git a/tests/templates/kuttl/delta/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/delta/helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..40dfcd33 --- /dev/null +++ b/tests/templates/kuttl/delta/helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,42 @@ +--- +global: + security: + allowInsecureImages: true # needed starting with Chart version 16.3.0 if modifying images + +image: + repository: bitnamilegacy/postgresql + +volumePermissions: + enabled: false + image: + repository: bitnamilegacy/os-shell + securityContext: + runAsUser: auto + +metrics: + image: + repository: bitnamilegacy/postgres-exporter + +primary: + extendedConfiguration: | + password_encryption=md5 + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "512Mi" + cpu: "512m" + limits: + memory: "512Mi" + cpu: "1" + +auth: + username: hive + password: hive + database: hive diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 9a8db602..f717d3dd 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -27,7 +27,10 @@ dimensions: values: - "479" # To use a custom image, add a comma and the full name after the product version - # - 477,oci.stackable.tech/sdp/trino:477-stackable0.0.0-dev + # - 470,oci.stackable.tech/sdp/trino:470-stackable0.0.0-dev + - name: trino-delta + values: + - "479" - name: hive values: - 3.1.3 @@ -124,6 +127,13 @@ tests: dimensions: - trino - openshift + - name: delta + # NOTE: this test only works because we set `parquet.use-column-index=false` in the delta catalog + # to work around this bug: https://github.com/trinodb/trino/issues/28885 + dimensions: + - trino-delta + - hive-latest + - openshift suites: - name: nightly # Run with the latest product versions and tls true and false to cover different tls code paths.