HBASE-30087: Create a nightly Jenkins test for the Read-Replica feature

.dockerignore

@@ -0,0 +1,10 @@
+# Trim Docker build contexts that use the repository root (e.g. dev-support image).
+
+.git
+**/.git
+**/target
+**/__pycache__
+**/.venv
+**/node_modules
+dev-support/read-replica/hbase
+dev-support/read-replica/tmp-read-replica-data

dev-support/README.md

@@ -60,6 +60,8 @@ configurations in this directory:
 
 - `docker/` -- Dockerfile for CI build environment
 - `hbase_docker/`, `hbase_docker.sh` -- Docker-based local test cluster
+- `read-replica/` -- Jenkins pipeline for Read-Replica feature integration
+  tests using Docker and Python
 - `adhoc_run_tests/` -- Scripts for running test suites outside CI
 - `integration-test/` -- Integration test support
 

dev-support/docker/Dockerfile

@@ -58,6 +58,10 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \
     libxml2-dev='2.9.13+dfsg-*' \
     libxml2-utils='2.9.13+dfsg-*' \
     zip='3.0-*' \
+# read-replica nightly tests use this Dockerfile and need these docker and python dependencies
+    docker.io='20.10.*' \
+    docker-compose-v2='2.*' \
+    python3-venv='3.10.6-*' \
     && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* \

dev-support/read-replica/.dockerignore

@@ -0,0 +1,5 @@
+# Outer build context for read-replica/Dockerfile (compose, tests, output).
+tmp-read-replica-data
+output
+.venv
+__pycache__

dev-support/read-replica/.env

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# The name of the HBase Docker image
+HBASE_IMAGE=${HBASE_IMAGE:-default/hbase-docker:read-replica-jenkins}
+# The name of the HBase docker container
+HBASE_CONTAINER_NAME=hbase-docker
+# The directory within the docker container that contains the config files
+HBASE_CONF_DIR=/opt/hbase/conf
+# The directory containing the 'data-store/<hbase.rootdir>' directory.
+# This directory is mounted with the HBase Docker container.
+HBASE_DATA_STORE_ROOT=./tmp-read-replica-data
+# Local path to the active cluster's HBase config file.
+# This file is part of a mounted volume, so modifying it
+# locally also modifies it within the container (and vise-versa).
+ACTIVE_CLUSTER_CONF_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster1/conf
+# Local path to docker volume where HBase sends active cluster logs
+ACTIVE_CLUSTER_LOGS_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster1/logs
+# Same as above, except for the replica cluster
+REPLICA_CLUSTER_CONF_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster2/conf
+# Local path to docker volume where HBase sends replica cluster logs
+REPLICA_CLUSTER_LOGS_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster2/logs
+# The path to the docker-compose file
+DOCKER_COMPOSE_FILE=${HBASE_ROOT}/dev-support/read-replica/docker-compose.yml
+# The location of the utils directory within the docker container
+CONTAINER_UTILS_DIR=/opt/utils
+# The log level for the Python scripts
+LOG_LEVEL=INFO

dev-support/read-replica/Dockerfile

@@ -0,0 +1,152 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Stage 0: Cache Maven dependencies
+FROM registry.access.redhat.com/ubi8/openjdk-17:1.23 AS cache-stage
+
+# Switch to user root (UID 0)
+USER 0
+
+# Install necessary packages for building Maven dependencies
+# hadolint ignore=DL3041
+RUN microdnf update -y && microdnf install -y \
+        maven \
+        git \
+        hostname \
+        diffutils \
+    && microdnf clean all
+
+# Copy the entire source code to cache dependencies
+COPY ./hbase /opt/hbase-src
+
+WORKDIR /opt/hbase-src
+
+# Download and cache all dependencies
+RUN mvn clean install -DskipTests -Dskip.license.check=true
+
+# Switch to default user (UID 1001) at end of stage
+USER 1001
+
+# Stage 1: Build the HBase source code
+FROM registry.access.redhat.com/ubi8/openjdk-17:1.23 AS build-stage
+
+# Switch to user root (UID 0)
+USER 0
+
+# Install necessary build packages
+# hadolint ignore=DL3041
+RUN microdnf update -y && microdnf install -y \
+        maven \
+        git \
+        hostname \
+        diffutils \
+    && microdnf clean all
+
+# Copy the cached Maven dependencies
+COPY --from=cache-stage /root/.m2 /root/.m2
+
+# Copy the HBase source code
+COPY ./hbase /opt/hbase-src
+
+WORKDIR /opt/hbase-src
+
+# Build HBase source code using cached dependencies and enable parallel build
+RUN mvn clean package -DskipTests -Dskip.license.check=true assembly:single -T 1C
+
+# Switch to default user (UID 1001) at end of stage
+USER 1001
+
+# Stage 2: Create the final Docker image
+FROM registry.access.redhat.com/ubi8/openjdk-17:1.23
+
+# Switch to user root (UID 0)
+USER 0
+
+# Set environment variables
+ENV HBASE_HOME=/opt/hbase
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk \
+    HBASE_USER=hbase \
+    HBASE_CONF_DIR=${HBASE_HOME}/conf \
+    HBASE_LIB_DIR=${HBASE_HOME}/lib \
+    HBASE_LOGS_DIR=${HBASE_HOME}/logs \
+    DATA_DIR=/data-store
+
+# Install necessary runtime packages
+# hadolint ignore=DL3041
+RUN microdnf update -y && microdnf install -y \
+        unzip \
+        gzip \
+        wget \
+        hostname \
+        maven \
+        git \
+        diffutils \
+        vim \
+        openssh-clients \
+        python3 \
+        procps \
+    && microdnf clean all
+
+# Copy the built HBase binaries from the build-stage
+COPY --from=build-stage /opt/hbase-src/hbase-assembly/target/hbase-*-bin.tar.gz /opt/hbase-dist/
+
+# Extract the full HBase binary distribution (not the client-only assembly)
+RUN archive="$(find /opt/hbase-dist -maxdepth 1 -type f -name 'hbase-*-bin.tar.gz' ! -name 'hbase-*-client-bin.tar.gz' -print -quit)" \
+    && mkdir -p /opt/hbase \
+    && tar -xzf "${archive}" -C /opt/hbase --strip-components=1 \
+    && rm -rf /opt/hbase-dist
+
+COPY --from=build-stage /root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-common/*/hadoop-mapreduce-client-common-*.jar \
+                        ${HBASE_HOME}/lib/
+
+# Apply custom HBase build steps
+RUN sed -i "s,^. export JAVA_HOME.*,export JAVA_HOME=$JAVA_HOME," ${HBASE_CONF_DIR}/hbase-env.sh \
+    && sed -E -i 's/(.*)hbase\-daemons\.sh(.*zookeeper)/\1hbase-daemon.sh\2/g' ${HBASE_HOME}/bin/start-hbase.sh \
+    && printf "JAVA_HOME=%s\nexport JAVA_HOME\nexport PATH=%s/jre/bin:%s\n" "$JAVA_HOME" "$JAVA_HOME" "$PATH" \
+            > /etc/profile.d/defaults.sh \
+    && ln -sf ${HBASE_HOME}/bin/* /usr/bin
+
+# Create HBase user and home directory
+RUN useradd -u 1000 -m ${HBASE_USER} \
+    && mkdir -p /home/${HBASE_USER} \
+    && chown -R ${HBASE_USER}:${HBASE_USER} /opt /home/${HBASE_USER} \
+    && mkdir "$DATA_DIR" && chown -R ${HBASE_USER}:${HBASE_USER} "$DATA_DIR"
+
+# Set permissions for jboss home directory
+RUN chown -R ${HBASE_USER}:${HBASE_USER} /home/jboss
+
+# Copy configuration files - Removed in order to mount config files individually #
+
+# Expose required ports for HBase and related services
+EXPOSE 8000 8080 8085 9090 9095 2181 16000 16010 16020 16030
+
+# Set up the utils directory as a volume
+VOLUME ["/opt/utils"]
+
+# Switch to the HBase user (UID 1000)
+USER 1000
+
+# Create necessary directories for HBase to run
+RUN mkdir -p "$DATA_DIR"/hbase "$DATA_DIR"/run "$DATA_DIR"/logs && \
+    chmod -R 777 "$DATA_DIR"
+
+# Add the 'ls -l' alias
+RUN echo 'alias ll="ls -l"' >> /home/hbase/.bashrc && \
+    echo 'alias ll="ls -l"' >> /home/jboss/.bashrc
+
+# Start HBase and keep it running
+ENTRYPOINT ["/bin/bash", "-c", "${HBASE_HOME}/bin/start-hbase.sh && tail -f ${HBASE_LOGS_DIR}/*.log"]