From e0071c71e62c72e3d3e484280d3eb26a9b9ff766 Mon Sep 17 00:00:00 2001 From: Kevin Geiszler Date: Thu, 17 Sep 2026 12:39:59 -0400 Subject: [PATCH] HBASE-30087: Create a nightly Jenkins test for the Read-Replica feature (#8517) - Changes were partially generated by Claude Opus 4.6 --- .dockerignore | 10 + dev-support/README.md | 2 + dev-support/docker/Dockerfile | 4 + dev-support/read-replica/.dockerignore | 5 + dev-support/read-replica/.env | 42 ++ dev-support/read-replica/Dockerfile | 152 ++++ dev-support/read-replica/Jenkinsfile | 211 ++++++ dev-support/read-replica/README.md | 350 ++++++++++ dev-support/read-replica/build-images.sh | 77 ++ .../read-replica/cluster1/conf/hbase-site.xml | 73 ++ .../cluster1/conf/log4j2.properties | 137 ++++ .../read-replica/cluster1/conf/zoo.cfg | 20 + .../read-replica/cluster2/conf/hbase-site.xml | 77 ++ .../cluster2/conf/log4j2.properties | 137 ++++ .../read-replica/cluster2/conf/zoo.cfg | 20 + dev-support/read-replica/docker-compose.yml | 44 ++ .../hbase_nightly_read_replica_test.sh | 233 +++++++ dev-support/read-replica/python/__init__.py | 16 + .../python/proto/proto_compiler.py | 50 ++ .../read-replica/python/scripts/__init__.py | 16 + .../python/scripts/render_console_report.py | 309 +++++++++ .../python/scripts/verify_hbase_start.py | 58 ++ .../read-replica/python/src/__init__.py | 20 + .../python/src/environment_loader.py | 28 + .../python/src/hbase_docker_client.py | 655 ++++++++++++++++++ .../read-replica/python/src/logger_config.py | 63 ++ dev-support/read-replica/python/src/utils.py | 269 +++++++ .../read-replica/python/test/__init__.py | 16 + .../read-replica/python/test/conftest.py | 111 +++ .../test_bulkloaded_data_and_region_splits.py | 236 +++++++ ...st_cannot_promote_second_active_cluster.py | 116 ++++ .../python/test/test_create_drop_behavior.py | 108 +++ .../test/test_dual_active_cluster_startup.py | 176 +++++ .../test/test_put_get_delete_behavior.py | 145 ++++ .../test/test_read_only_flag_flipping.py | 121 ++++ .../python/test/test_read_replica_feature.py | 48 ++ dev-support/read-replica/requirements.txt | 35 + .../run_read_replica_integration_tests.sh | 360 ++++++++++ dev-support/read-replica/utils/bulkload.sh | 75 ++ .../read-replica/utils/tsv_generator.py | 50 ++ 40 files changed, 4675 insertions(+) create mode 100644 .dockerignore create mode 100644 dev-support/read-replica/.dockerignore create mode 100644 dev-support/read-replica/.env create mode 100755 dev-support/read-replica/Dockerfile create mode 100644 dev-support/read-replica/Jenkinsfile create mode 100644 dev-support/read-replica/README.md create mode 100755 dev-support/read-replica/build-images.sh create mode 100755 dev-support/read-replica/cluster1/conf/hbase-site.xml create mode 100644 dev-support/read-replica/cluster1/conf/log4j2.properties create mode 100755 dev-support/read-replica/cluster1/conf/zoo.cfg create mode 100755 dev-support/read-replica/cluster2/conf/hbase-site.xml create mode 100644 dev-support/read-replica/cluster2/conf/log4j2.properties create mode 100755 dev-support/read-replica/cluster2/conf/zoo.cfg create mode 100644 dev-support/read-replica/docker-compose.yml create mode 100755 dev-support/read-replica/hbase_nightly_read_replica_test.sh create mode 100644 dev-support/read-replica/python/__init__.py create mode 100644 dev-support/read-replica/python/proto/proto_compiler.py create mode 100644 dev-support/read-replica/python/scripts/__init__.py create mode 100755 dev-support/read-replica/python/scripts/render_console_report.py create mode 100644 dev-support/read-replica/python/scripts/verify_hbase_start.py create mode 100644 dev-support/read-replica/python/src/__init__.py create mode 100644 dev-support/read-replica/python/src/environment_loader.py create mode 100644 dev-support/read-replica/python/src/hbase_docker_client.py create mode 100644 dev-support/read-replica/python/src/logger_config.py create mode 100644 dev-support/read-replica/python/src/utils.py create mode 100644 dev-support/read-replica/python/test/__init__.py create mode 100644 dev-support/read-replica/python/test/conftest.py create mode 100644 dev-support/read-replica/python/test/test_bulkloaded_data_and_region_splits.py create mode 100644 dev-support/read-replica/python/test/test_cannot_promote_second_active_cluster.py create mode 100644 dev-support/read-replica/python/test/test_create_drop_behavior.py create mode 100644 dev-support/read-replica/python/test/test_dual_active_cluster_startup.py create mode 100644 dev-support/read-replica/python/test/test_put_get_delete_behavior.py create mode 100644 dev-support/read-replica/python/test/test_read_only_flag_flipping.py create mode 100644 dev-support/read-replica/python/test/test_read_replica_feature.py create mode 100644 dev-support/read-replica/requirements.txt create mode 100755 dev-support/read-replica/run_read_replica_integration_tests.sh create mode 100755 dev-support/read-replica/utils/bulkload.sh create mode 100644 dev-support/read-replica/utils/tsv_generator.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000000..0a69fe3550c0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +# Trim Docker build contexts that use the repository root (e.g. dev-support image). + +.git +**/.git +**/target +**/__pycache__ +**/.venv +**/node_modules +dev-support/read-replica/hbase +dev-support/read-replica/tmp-read-replica-data diff --git a/dev-support/README.md b/dev-support/README.md index 9dc3f3143616..cbdd8923c85f 100644 --- a/dev-support/README.md +++ b/dev-support/README.md @@ -60,6 +60,8 @@ configurations in this directory: - `docker/` -- Dockerfile for CI build environment - `hbase_docker/`, `hbase_docker.sh` -- Docker-based local test cluster +- `read-replica/` -- Jenkins pipeline for Read-Replica feature integration + tests using Docker and Python - `adhoc_run_tests/` -- Scripts for running test suites outside CI - `integration-test/` -- Integration test support diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index cc587b829ccd..717c4c067050 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -58,6 +58,10 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \ libxml2-dev='2.9.13+dfsg-*' \ libxml2-utils='2.9.13+dfsg-*' \ zip='3.0-*' \ +# read-replica nightly tests use this Dockerfile and need these docker and python dependencies + docker.io='20.10.*' \ + docker-compose-v2='2.*' \ + python3-venv='3.10.6-*' \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ diff --git a/dev-support/read-replica/.dockerignore b/dev-support/read-replica/.dockerignore new file mode 100644 index 000000000000..95cf6626f6e8 --- /dev/null +++ b/dev-support/read-replica/.dockerignore @@ -0,0 +1,5 @@ +# Outer build context for read-replica/Dockerfile (compose, tests, output). +tmp-read-replica-data +output +.venv +__pycache__ diff --git a/dev-support/read-replica/.env b/dev-support/read-replica/.env new file mode 100644 index 000000000000..835a6c58e194 --- /dev/null +++ b/dev-support/read-replica/.env @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The name of the HBase Docker image +HBASE_IMAGE=${HBASE_IMAGE:-default/hbase-docker:read-replica-jenkins} +# The name of the HBase docker container +HBASE_CONTAINER_NAME=hbase-docker +# The directory within the docker container that contains the config files +HBASE_CONF_DIR=/opt/hbase/conf +# The directory containing the 'data-store/' directory. +# This directory is mounted with the HBase Docker container. +HBASE_DATA_STORE_ROOT=./tmp-read-replica-data +# Local path to the active cluster's HBase config file. +# This file is part of a mounted volume, so modifying it +# locally also modifies it within the container (and vise-versa). +ACTIVE_CLUSTER_CONF_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster1/conf +# Local path to docker volume where HBase sends active cluster logs +ACTIVE_CLUSTER_LOGS_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster1/logs +# Same as above, except for the replica cluster +REPLICA_CLUSTER_CONF_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster2/conf +# Local path to docker volume where HBase sends replica cluster logs +REPLICA_CLUSTER_LOGS_DIR=${HBASE_ROOT}/dev-support/read-replica/cluster2/logs +# The path to the docker-compose file +DOCKER_COMPOSE_FILE=${HBASE_ROOT}/dev-support/read-replica/docker-compose.yml +# The location of the utils directory within the docker container +CONTAINER_UTILS_DIR=/opt/utils +# The log level for the Python scripts +LOG_LEVEL=INFO diff --git a/dev-support/read-replica/Dockerfile b/dev-support/read-replica/Dockerfile new file mode 100755 index 000000000000..08999c51dee0 --- /dev/null +++ b/dev-support/read-replica/Dockerfile @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Stage 0: Cache Maven dependencies +FROM registry.access.redhat.com/ubi8/openjdk-17:1.23 AS cache-stage + +# Switch to user root (UID 0) +USER 0 + +# Install necessary packages for building Maven dependencies +# hadolint ignore=DL3041 +RUN microdnf update -y && microdnf install -y \ + maven \ + git \ + hostname \ + diffutils \ + && microdnf clean all + +# Copy the entire source code to cache dependencies +COPY ./hbase /opt/hbase-src + +WORKDIR /opt/hbase-src + +# Download and cache all dependencies +RUN mvn clean install -DskipTests -Dskip.license.check=true + +# Switch to default user (UID 1001) at end of stage +USER 1001 + +# Stage 1: Build the HBase source code +FROM registry.access.redhat.com/ubi8/openjdk-17:1.23 AS build-stage + +# Switch to user root (UID 0) +USER 0 + +# Install necessary build packages +# hadolint ignore=DL3041 +RUN microdnf update -y && microdnf install -y \ + maven \ + git \ + hostname \ + diffutils \ + && microdnf clean all + +# Copy the cached Maven dependencies +COPY --from=cache-stage /root/.m2 /root/.m2 + +# Copy the HBase source code +COPY ./hbase /opt/hbase-src + +WORKDIR /opt/hbase-src + +# Build HBase source code using cached dependencies and enable parallel build +RUN mvn clean package -DskipTests -Dskip.license.check=true assembly:single -T 1C + +# Switch to default user (UID 1001) at end of stage +USER 1001 + +# Stage 2: Create the final Docker image +FROM registry.access.redhat.com/ubi8/openjdk-17:1.23 + +# Switch to user root (UID 0) +USER 0 + +# Set environment variables +ENV HBASE_HOME=/opt/hbase +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk \ + HBASE_USER=hbase \ + HBASE_CONF_DIR=${HBASE_HOME}/conf \ + HBASE_LIB_DIR=${HBASE_HOME}/lib \ + HBASE_LOGS_DIR=${HBASE_HOME}/logs \ + DATA_DIR=/data-store + +# Install necessary runtime packages +# hadolint ignore=DL3041 +RUN microdnf update -y && microdnf install -y \ + unzip \ + gzip \ + wget \ + hostname \ + maven \ + git \ + diffutils \ + vim \ + openssh-clients \ + python3 \ + procps \ + && microdnf clean all + +# Copy the built HBase binaries from the build-stage +COPY --from=build-stage /opt/hbase-src/hbase-assembly/target/hbase-*-bin.tar.gz /opt/hbase-dist/ + +# Extract the full HBase binary distribution (not the client-only assembly) +RUN archive="$(find /opt/hbase-dist -maxdepth 1 -type f -name 'hbase-*-bin.tar.gz' ! -name 'hbase-*-client-bin.tar.gz' -print -quit)" \ + && mkdir -p /opt/hbase \ + && tar -xzf "${archive}" -C /opt/hbase --strip-components=1 \ + && rm -rf /opt/hbase-dist + +COPY --from=build-stage /root/.m2/repository/org/apache/hadoop/hadoop-mapreduce-client-common/*/hadoop-mapreduce-client-common-*.jar \ + ${HBASE_HOME}/lib/ + +# Apply custom HBase build steps +RUN sed -i "s,^. export JAVA_HOME.*,export JAVA_HOME=$JAVA_HOME," ${HBASE_CONF_DIR}/hbase-env.sh \ + && sed -E -i 's/(.*)hbase\-daemons\.sh(.*zookeeper)/\1hbase-daemon.sh\2/g' ${HBASE_HOME}/bin/start-hbase.sh \ + && printf "JAVA_HOME=%s\nexport JAVA_HOME\nexport PATH=%s/jre/bin:%s\n" "$JAVA_HOME" "$JAVA_HOME" "$PATH" \ + > /etc/profile.d/defaults.sh \ + && ln -sf ${HBASE_HOME}/bin/* /usr/bin + +# Create HBase user and home directory +RUN useradd -u 1000 -m ${HBASE_USER} \ + && mkdir -p /home/${HBASE_USER} \ + && chown -R ${HBASE_USER}:${HBASE_USER} /opt /home/${HBASE_USER} \ + && mkdir "$DATA_DIR" && chown -R ${HBASE_USER}:${HBASE_USER} "$DATA_DIR" + +# Set permissions for jboss home directory +RUN chown -R ${HBASE_USER}:${HBASE_USER} /home/jboss + +# Copy configuration files - Removed in order to mount config files individually # + +# Expose required ports for HBase and related services +EXPOSE 8000 8080 8085 9090 9095 2181 16000 16010 16020 16030 + +# Set up the utils directory as a volume +VOLUME ["/opt/utils"] + +# Switch to the HBase user (UID 1000) +USER 1000 + +# Create necessary directories for HBase to run +RUN mkdir -p "$DATA_DIR"/hbase "$DATA_DIR"/run "$DATA_DIR"/logs && \ + chmod -R 777 "$DATA_DIR" + +# Add the 'ls -l' alias +RUN echo 'alias ll="ls -l"' >> /home/hbase/.bashrc && \ + echo 'alias ll="ls -l"' >> /home/jboss/.bashrc + +# Start HBase and keep it running +ENTRYPOINT ["/bin/bash", "-c", "${HBASE_HOME}/bin/start-hbase.sh && tail -f ${HBASE_LOGS_DIR}/*.log"] diff --git a/dev-support/read-replica/Jenkinsfile b/dev-support/read-replica/Jenkinsfile new file mode 100644 index 000000000000..477123327c47 --- /dev/null +++ b/dev-support/read-replica/Jenkinsfile @@ -0,0 +1,211 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +pipeline { + agent { + node { + label 'hbase' + } + } + triggers { + pollSCM(getCronParams(env.BRANCH_NAME)) + } + options { + buildDiscarder(logRotator(numToKeepStr: '20')) + timeout (time: 2, unit: 'HOURS') + timestamps() + skipDefaultCheckout() + disableConcurrentBuilds() + } + environment { + OUTPUT_DIR_RELATIVE_READ_REPLICA = 'output-read-replica' + } + parameters { + booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.') + } + stages { + stage ('scm-checkout') { + steps { + dir('component') { + checkout scm + } + } + } + stage ('init health results') { + steps { + stash name: 'read-replica-result', allowEmpty: true, includes: "${OUTPUT_DIR_RELATIVE_READ_REPLICA}/doesn't-match" + } + } + stage ('hbase read-replica feature checks') { + when { + anyOf { + branch 'master' + branch 'branch-3' + } + } + environment { + BASEDIR = "${env.WORKSPACE}/component" + OUTPUT_DIR_RELATIVE = "${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}" + OUTPUT_DIR = "${env.WORKSPACE}/${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}" + } + steps { + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}" && mkdir -p "${OUTPUT_DIR}" + echo '(x) {color:red}-1 read-replica checks{color}' >"${OUTPUT_DIR}/commentfile" + echo "-- Something went wrong running this stage, please [check relevant console output|${BUILD_URL}/console]." >> "${OUTPUT_DIR}/commentfile" + ''' + dir('component') { + checkout scm + } + sh '''#!/usr/bin/env bash + set -e + rm -rf "${OUTPUT_DIR}/machine" && mkdir -p "${OUTPUT_DIR}/machine" + "${BASEDIR}/dev-support/gather_machine_environment.sh" "${OUTPUT_DIR_RELATIVE}/machine" + echo "got the following saved stats in '${OUTPUT_DIR_RELATIVE}/machine'" + ls -lh "${OUTPUT_DIR_RELATIVE}/machine" + ''' + script { + def ret = sh( + returnStatus: true, + script: '''#!/usr/bin/env bash + set -e + declare -i status=0 + if "${BASEDIR}/dev-support/read-replica/hbase_nightly_read_replica_test.sh" ; then + echo '(/) {color:green}+1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile" + else + echo '(x) {color:red}-1 read-replica checks{color}' > "${OUTPUT_DIR}/commentfile" + status=1 + fi + echo "-- For more information [see read-replica test report|${BUILD_URL}Read_20Replica_20Console_20Report/]" >> "${OUTPUT_DIR}/commentfile" + exit "${status}" + ''' + ) + if (ret != 0) { + // mark the build as UNSTABLE instead of FAILURE to allow post processing of test reports (HBASE-26339) + currentBuild.result = 'UNSTABLE' + } + } + } + post { + always { + junit testResults: "${env.OUTPUT_DIR_RELATIVE}/read-replica-nightly-test-results.xml", allowEmptyResults: true + + publishHTML target: [ + allowMissing : true, + keepAll : true, + alwaysLinkToLastBuild: true, + reportDir : "${env.OUTPUT_DIR_RELATIVE}", + reportFiles : 'read-replica-console-report.html', + reportName : 'Read Replica Console Report' + ] + + stash name: 'read-replica-result', includes: "${OUTPUT_DIR_RELATIVE}/commentfile" + archiveArtifacts artifacts: "${env.OUTPUT_DIR_RELATIVE}/**/*", allowEmptyArchive: true + } + } + } + } + post { + always { + script { + try { + sh "printenv" + sh ''' + echo "Clean up result directories" + rm -rf ${OUTPUT_DIR_RELATIVE_READ_REPLICA} + ''' + unstash 'read-replica-result' + + def results = ["${env.OUTPUT_DIR_RELATIVE_READ_REPLICA}/commentfile"] + echo env.BRANCH_NAME + echo env.BUILD_URL + echo currentBuild.result + echo currentBuild.durationString + def comment = "Results for branch ${env.BRANCH_NAME}\n" + comment += "\t[build ${currentBuild.displayName} on builds.a.o|${env.BUILD_URL}]: " + if (currentBuild.result == null || currentBuild.result == "SUCCESS") { + comment += "(/) *{color:green}+1 overall{color}*\n" + } else { + comment += "(x) *{color:red}-1 overall{color}*\n" + } + comment += "----\ndetails (if available):\n\n" + echo "" + echo "[DEBUG] trying to aggregate step-wise results" + comment += results.collect { fileExists(file: it) ? readFile(file: it) : "" }.join("\n\n") + echo "[INFO] Comment:" + echo comment + echo "" + echo "[DEBUG] checking to see if feature branch" + def jiras = getJirasToComment(env.BRANCH_NAME, []) + if (jiras.isEmpty()) { + echo "[DEBUG] non-feature branch, checking change messages for jira keys." + echo "[INFO] There are ${currentBuild.changeSets.size()} change sets." + jiras = getJirasToCommentFromChangesets(currentBuild) + } + jiras.each { currentIssue -> + jiraComment issueKey: currentIssue, body: comment + } + } catch (Exception exception) { + echo "Got exception: ${exception}" + echo " ${exception.getStackTrace()}" + } + } + } + } +} +import org.jenkinsci.plugins.workflow.support.steps.build.RunWrapper +@NonCPS +List getJirasToCommentFromChangesets(RunWrapper thisBuild) { + def seenJiras = [] + thisBuild.changeSets.each { cs -> + cs.getItems().each { change -> + CharSequence msg = change.msg + echo "change: ${change}" + echo " ${msg}" + echo " ${change.commitId}" + echo " ${change.author}" + echo "" + seenJiras = getJirasToComment(msg, seenJiras) + } + } + return seenJiras +} +@NonCPS +List getJirasToComment(CharSequence source, List seen) { + source.eachMatch("HBASE-[0-9]+") { currentIssue -> + echo "[DEBUG] found jira key: ${currentIssue}" + if (currentIssue in seen) { + echo "[DEBUG] already commented on ${currentIssue}." + } else { + echo "[INFO] commenting on ${currentIssue}." + seen << currentIssue + } + } + return seen +} +@NonCPS +String getCronParams(String branchName) { + if (branchName == 'master') { + return 'H H 1-31/3 * *' + } else if (branchName == 'branch-3') { + return 'H H 1-31/3 * *' + } else if (branchName == 'branch-2') { + return 'H H 2-31/3 * *' + } else { + return 'H H 3-31/3 * *' + } +} diff --git a/dev-support/read-replica/README.md b/dev-support/read-replica/README.md new file mode 100644 index 000000000000..a1c2088d7121 --- /dev/null +++ b/dev-support/read-replica/README.md @@ -0,0 +1,350 @@ + + +# Read-Replica Integration Tests + +Integration test framework for HBase's Read-Replica feature, driven by Jenkins and Docker. + +## Why Docker? + +HBase's `MiniHBaseCluster` cannot support multi-cluster Read-Replica testing because the +static `META_TABLE_NAME` variable is shared across JVMs (see HBASE-29691). This framework +sidesteps that limitation by running two fully isolated HBase clusters in Docker containers +that share a filesystem-based `hbase.rootdir`. + +## Architecture + +``` +┌──────────────────────────────────────────────────────────────────────┐ +│ Host (Jenkins Agent or Local) │ +│ │ +│ ┌────────────────────────────────────────────────────────────────┐ │ +│ │ Test Environment Container (dev-support image) │ │ +│ │ Built from: dev-support/docker/Dockerfile │ │ +│ │ Contains: JDK, Maven, Python, Docker CLI │ │ +│ │ Entry: run_read_replica_integration_tests.sh │ │ +│ │ │ │ +│ │ ┌──────────────────┐ ┌──────────────────┐ │ │ +│ │ │ hbase-docker │ │ hbase-docker-2 │ │ │ +│ │ │ (Active Cluster)│ │ (Replica Cluster)│ │ │ +│ │ │ │ │ │ │ │ +│ │ │ ZooKeeper │ │ ZooKeeper │ │ │ +│ │ │ HMaster │ │ HMaster │ │ │ +│ │ │ RegionServer │ │ RegionServer │ │ │ +│ │ │ │ │ │ │ │ +│ │ │ read-only=false │ │ read-only=true │ │ │ +│ │ │ suffix=(empty) │ │ suffix=replica1 │ │ │ +│ │ └────────┬─────────┘ └────────┬─────────┘ │ │ +│ │ │ │ │ │ +│ │ └────────────┬───────────────┘ │ │ +│ │ │ │ │ +│ │ ┌─────────▼───────────┐ │ │ +│ │ │ Shared Data Store │ │ │ +│ │ │ (hbase.rootdir) │ │ │ +│ │ │ │ │ │ +│ │ │ /data-store/hbase │ │ │ +│ │ └─────────────────────┘ │ │ +│ │ │ │ +│ │ ┌──────────────────────────────────────────────────────┐ │ │ +│ │ │ Python Test Scripts │ │ │ +│ │ │ (communicate via `docker exec` + HBase Shell) │ │ │ +│ │ └──────────────────────────────────────────────────────┘ │ │ +│ └────────────────────────────────────────────────────────────────┘ │ +│ │ +│ Docker socket bind-mounted into the test environment container │ +│ (Docker-outside-of-Docker / DooD) │ +└──────────────────────────────────────────────────────────────────────┘ +``` + +The test environment container is built from `dev-support/docker/Dockerfile` and provides +JDK, Maven, Python, and the Docker CLI. It uses Docker-outside-of-Docker (DooD) by +bind-mounting the host's Docker socket, so the HBase cluster containers run as siblings on +the host's Docker daemon. + +Both HBase clusters mount the same `data-store/hbase` directory as their `hbase.rootdir`. +The active cluster writes data and the replica cluster reads it after explicit `refresh_meta` +/ `refresh_hfiles` calls. Each cluster has its own ZooKeeper instance and distinct +`hbase.meta.table.suffix` to avoid meta table collisions. + +## Directory Structure + +``` +read-replica/ +├── .env # Environment variables (image name, ports, paths) +├── Dockerfile # Multi-stage build for the hbase-docker image +├── build-images.sh # Builds the Docker image from the local HBase checkout +├── docker-compose.yml # Defines the two container services +├── requirements.txt # Python dependencies +├── hbase_nightly_read_replica_test.sh # Outer driver script (runs on host) +├── run_read_replica_integration_tests.sh # Inner test runner (runs inside container) +├── cluster1/ # Active cluster configuration +│ └── conf/ +│ ├── hbase-site.xml # hbase.global.readonly.enabled=false +│ ├── log4j2.properties +│ └── zoo.cfg +├── cluster2/ # Replica cluster configuration +│ └── conf/ +│ ├── hbase-site.xml # hbase.global.readonly.enabled=true, suffix=replica1 +│ ├── log4j2.properties +│ └── zoo.cfg +├── python/ +│ ├── proto/ +│ │ ├── ActiveClusterSuffix.proto # Automatically copied to this location by +│ │ │ # run_read_replica_integration_tests.sh +│ │ └── proto_compiler.py # Compiles .proto files into python/proto/generated/ +│ ├── scripts/ +│ │ └── verify_hbase_start.py # Standalone startup verification (not part of pytest suite) +│ ├── src/ +│ │ ├── hbase_docker_client.py # Core client — talks to containers via docker exec +│ │ ├── environment_loader.py # Reads env vars with validation +│ │ ├── logger_config.py # Logging setup +│ │ └── utils.py # Shared test utilities +│ └── test/ +│ ├── test_read_replica_feature.py # Pytest entry point (TestReadReplica class) +│ ├── test_dual_active_cluster_startup.py +│ ├── test_create_drop_behavior.py +│ ├── test_put_get_delete_behavior.py +│ ├── test_read_only_flag_flipping.py +│ ├── test_cannot_promote_second_active_cluster.py +│ └── test_bulkloaded_data_and_region_splits.py +└── utils/ + ├── bulkload.sh # In-container script for ImportTsv + completebulkload + └── tsv_generator.py # Generates random TSV data for bulkloading +``` + +## Scripts + +The test framework uses two scripts in a layered architecture: + +### `hbase_nightly_read_replica_test.sh` (Outer Driver) + +Runs on the host (Jenkins agent or local machine). It builds a test environment Docker image +from `dev-support/docker/Dockerfile`, starts a container, and either runs the inner test +script automatically or drops into an interactive shell for development. + +| Step | Description | +|------|-------------| +| 1 | Build the test environment Docker image (`hbase-dev-support:`) | +| 2 | Start a container from that image, mounting the HBase repo, Docker socket, and `.m2` cache | +| 3 | Run `run_read_replica_integration_tests.sh` inside the container (or start an interactive shell in dev mode) | + +**Options:** + +| Flag | Description | +|------|-------------| +| `-d` / `--dev` | Start a detached dev container for interactive exploration instead of running the test suite. See [Dev Mode](#dev-mode). | +| `-k ` | Pytest `-k` filter expression forwarded to the inner test script for test selection. See [Running Specific Tests](#running-specific-tests). | +| `-m` / `--m2 ` | Parent directory of the `.m2` Maven cache to bind-mount into the container. Defaults to `$HOME`. The directory `/.m2` will be created if it does not exist. | +| `-j` / `--java-version ` | JVM version forwarded to the inner test script, which uses it to set `JAVA_HOME` inside the container. | + +### `run_read_replica_integration_tests.sh` (Inner Test Runner) + +Runs inside the test environment container. Normally invoked by the outer driver, but can +also be run directly when using dev mode. + +| Step | Description | +|------|-------------| +| 1 | Rsync a trimmed HBase tree into `read-replica/hbase/` for the Docker build context (excludes `.git`, `target/`, nested `read-replica/hbase`, etc.) | +| 2 | Copy and compile the `ActiveClusterSuffix.proto` protobuf definition | +| 3 | Create a Python virtual environment and install dependencies | +| 4 | Build the `hbase-read-replica` Docker image for the active and replica clusters | +| 5 | Run the pytest integration suite | + +**Options:** + +| Flag | Description | +|------|-------------| +| `-i` / `--keep-image` | Do not remove the `hbase-read-replica` Docker image on exit. Useful for re-running tests without rebuilding. | +| `-c` / `--keep-containers` | Do not run `docker compose down` on exit. Lets you inspect container state after tests. | +| `-k ` | Pytest `-k` filter expression for test selection. | +| `-j` / `--java-version ` | JVM version to use (default: 17). Sets `JAVA_HOME` to `/usr/lib/jvm/java-`. | + +## CI: Jenkins Nightly Pipeline + +**Files:** +- `dev-support/read-replica/Jenkinsfile` — pipeline definition (`hbase read-replica feature checks`) +- `dev-support/read-replica/hbase_nightly_read_replica_test.sh` — outer driver script +- `dev-support/read-replica/run_read_replica_integration_tests.sh` — inner test runner + +### When It Runs + +The read-replica tests run as their own standalone nightly pipeline on the `master` and `branch-3` +branches, separate from the main HBase nightly build. + +### On Failure + +If any test fails, the build is marked `UNSTABLE` (not `FAILURE`) so that later result +publishing stages are not skipped. HBase logs from both containers are archived as Jenkins +build artifacts for debugging. + +## Test Suite + +Tests live in `python/test/` and are run via pytest through the `TestReadReplica` class in +`test_read_replica_feature.py`. Each test method delegates to a corresponding module's +`run_test()` function. The class is decorated with `@pytest.mark.flaky(reruns=2, reruns_delay=2)`, +so any failing test is automatically retried up to 2 times before being marked as failed. + +| Test | What it verifies | +|------|-----------------| +| `test_dual_active_cluster_startup` | Exactly one of two active clusters fails to start with an error about another active cluster already existing | +| `test_create_drop_behavior` | `create` and `drop` are rejected on the replica; `refresh_meta` propagates DDL changes | +| `test_put_get_delete_behavior` | `put`, `delete`, and `flush` are rejected on the replica; data propagates after `refresh_hfiles` | +| `test_read_only_flag_flipping` | Clusters can swap roles (15 iterations); `active.cluster.suffix.id` protobuf file stays consistent | +| `test_cannot_promote_second_active_cluster` | Disabling read-only on the replica raises `ReadOnlyTransitionException` while an active cluster exists | +| `test_bulkloaded_data_and_region_splits` | `ImportTsv` + `completebulkload` works on active, is rejected on replica; region splits propagate | + +### Test Results + +Pytest produces output files in the `output/` directory: + +- **Console report** (`read-replica-console-report.html`) — a Yetus-style HTML report with + per-stage vote, runtime, and status. Generated by `python/scripts/render_console_report.py`. + Published via `publishHTML` in the Jenkinsfile. +- **JUnit XML** (`read-replica-nightly-test-results.xml`) — parsed by Jenkins to display + individual test results in the build UI + +## Key Components + +### HBaseDockerClient + +`python/src/hbase_docker_client.py` — The core interface to HBase containers. It: + +- Executes commands inside containers via `docker exec ... bash -c` +- Wraps HBase Shell commands with retry logic and timeout handling +- Provides assertion helpers (`assert_read_only_error_occurs`, `assert_table_row_count`, etc.) +- Manages cluster lifecycle (start/stop containers, wait for readiness) +- Manipulates `hbase-site.xml` to toggle `hbase.global.readonly.enabled` at runtime + +### .env File + +Defines environment variables consumed by Docker Compose, the build script, and Python tests: + +- `HBASE_IMAGE` — Docker image tag +- `HBASE_DATA_STORE_ROOT` — Host path for the shared data store +- `DOCKER_COMPOSE_FILE` — Absolute path to `docker-compose.yml` + +### Protobuf Verification + +The `ActiveClusterSuffix.proto` message defines the format of the `active.cluster.suffix.id` +file written to the shared data store. Tests compile this proto and deserialize the file to +verify that the recorded active cluster matches the expected configuration after role swaps. +The `ActiveClusterSuffix.proto` file is copied directly from the HBase repo during runtime +of `run_read_replica_integration_tests.sh`. + +## Running Locally + +The same script used by Jenkins works locally. The only prerequisite is Docker — the test +environment container provides JDK, Maven, Python, and everything else needed to build and +test. + +### Running the Full Suite + +From the repo root: + +```bash +bash dev-support/read-replica/hbase_nightly_read_replica_test.sh +``` + +When run locally (no `BUILD_NUMBER` environment variable), the test environment image is +preserved so subsequent runs skip the image build. In Jenkins, the image is cleaned up +automatically. + +### Running Specific Tests + +Use `-k` to pass a pytest `-k` filter expression that selects tests by name +(see [pytest docs](https://docs.pytest.org/en/stable/example/markers.html#using-k-expr-to-select-tests-based-on-their-name) for details): + +```bash +# Run two specific tests +bash dev-support/read-replica/hbase_nightly_read_replica_test.sh \ + -k "test_create_drop_behavior or test_put_get_delete_behavior" + +# Run a single test +bash dev-support/read-replica/hbase_nightly_read_replica_test.sh \ + -k "test_put_get_delete_behavior" +``` + +### Dev Mode + +Dev mode (`-d` / `--dev`) builds and starts the test environment container but does **not** +run the test suite. This gives you an interactive environment where you can run the inner +test script as many times as you need — useful for reproducing bugs and iterating on test +development. + +**1. Start the dev container:** + +```bash +bash dev-support/read-replica/hbase_nightly_read_replica_test.sh -d +``` + +The script prints the container ID and instructions for entering it. + +**2. Enter the container:** + +```bash +docker exec -it bash +``` + +**3. Run the tests inside the container:** + +```bash +# Run the full suite +bash run_read_replica_integration_tests.sh + +# Run specific tests +bash run_read_replica_integration_tests.sh \ + -k "test_create_drop_behavior or test_put_get_delete_behavior" +``` + +**4. Speed up re-runs with `--keep-image`:** + +By default, the inner script removes the `hbase-read-replica` cluster Docker image on exit. +Pass `--keep-image` to preserve it, so subsequent runs skip the Maven build and image +creation — saving significant time during development: + +```bash +bash run_read_replica_integration_tests.sh --keep-image + +# Combine with -k to iterate on a specific test +bash run_read_replica_integration_tests.sh --keep-image \ + -k "test_create_drop_behavior" +``` + +**5. Stop and remove the dev container when done:** + +```bash +docker stop && docker rm +``` + +### Mounted Volumes + +The data-store directory (`tmp-read-replica-data/`) is mounted into the containers. Between +`docker compose down` and `docker compose up`, consider removing this directory to start with +a clean state: + +```bash +rm -rf tmp-read-replica-data +``` + +**Prerequisites:** Docker. + +## Related + +- **JIRA:** [HBASE-30087](https://issues.apache.org/jira/browse/HBASE-30087) +- **Read-Replica feature PR:** [#8364](https://github.com/apache/hbase/pull/8364) +- **Background (MiniHBaseCluster limitation):** [HBASE-29691](https://issues.apache.org/jira/browse/HBASE-29691) diff --git a/dev-support/read-replica/build-images.sh b/dev-support/read-replica/build-images.sh new file mode 100755 index 000000000000..ca9e95a7e833 --- /dev/null +++ b/dev-support/read-replica/build-images.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Load environment variables from .env file +set -a +. ./.env +set +a + +# Check if required environment variables are set +if [ -z "$HBASE_IMAGE" ]; then + echo "Error: HBASE_IMAGE is not set in .env file." + exit 1 +fi + +# Trimmed HBase tree under read-replica/hbase (see run_read_replica_integration_tests.sh). +HBASE_SOURCE_DIR="${HBASE_SOURCE_DIR:-./hbase}" + +if [ ! -f "${HBASE_SOURCE_DIR}/pom.xml" ]; then + echo "Error: HBase source (pom.xml) not found at ${HBASE_SOURCE_DIR}." + exit 1 +fi + +# Run Maven clean to remove previous build artifacts +echo "Running 'mvn clean' in $HBASE_SOURCE_DIR..." +MVN_CLEAN_START=${SECONDS} +cd "$HBASE_SOURCE_DIR" || exit 1 +mvn clean +MVN_CLEAN_EXIT=$? +MVN_CLEAN_SEC=$((SECONDS - MVN_CLEAN_START)) + +if [ ${MVN_CLEAN_EXIT} -ne 0 ]; then + echo "Error: 'mvn clean' failed." + exit 1 +else + echo "'mvn clean' completed successfully (${MVN_CLEAN_SEC}s)." +fi + +# Return to the original directory +cd - || exit 1 + +# Build HBase Docker image (Maven package runs inside the Dockerfile) +echo "Building HBase Docker image: ${HBASE_IMAGE}" +DOCKER_BUILD_START=${SECONDS} +DOCKER_BUILDKIT=1 docker build -t "${HBASE_IMAGE}" ./ +DOCKER_BUILD_EXIT=$? +DOCKER_BUILD_SEC=$((SECONDS - DOCKER_BUILD_START)) + +if [ -n "${READ_REPLICA_TIMING_FILE}" ]; then + mkdir -p "$(dirname "${READ_REPLICA_TIMING_FILE}")" + cat > "${READ_REPLICA_TIMING_FILE}" < + + + + + hbase.zookeeper.quorum + hbase-docker + + + hbase.zookeeper.property.dataDir + /data-store/zk + + + hbase.rootdir + file:///data-store/hbase + + + hbase.wal.dir + file:///data-store/wal + + + hbase.store.file-tracker.impl + FILE + + + + hbase.master.info.bindAddress + hbase-docker + + + hbase.regionserver.info.bindAddress + hbase-docker + + + hbase.unsafe.stream.capability.enforce + false + + + hbase.cluster.distributed + true + + + hbase.rest.port + 8000 + + + + hbase.global.readonly.enabled + false + + + hbase.master.hbck.chore.interval + 0 + + diff --git a/dev-support/read-replica/cluster1/conf/log4j2.properties b/dev-support/read-replica/cluster1/conf/log4j2.properties new file mode 100644 index 000000000000..91957fdc8a31 --- /dev/null +++ b/dev-support/read-replica/cluster1/conf/log4j2.properties @@ -0,0 +1,137 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# Console appender +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t%notEmpty{ %X}] %c{2}: %.1000m%n + +# Daily Rolling File Appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.DRFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%d{yyyy-MM-dd} +appender.DRFA.createOnDemand = true +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.policies.size.type = SizeBasedTriggeringPolicy +appender.DRFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Rolling File Appender +appender.RFA.type = RollingFile +appender.RFA.name = RFA +appender.RFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.RFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%i +appender.RFA.createOnDemand = true +appender.RFA.layout.type = PatternLayout +appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFA.policies.type = Policies +appender.RFA.policies.size.type = SizeBasedTriggeringPolicy +appender.RFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.RFA.strategy.type = DefaultRolloverStrategy +appender.RFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Security Audit Appender +appender.RFAS.type = RollingFile +appender.RFAS.name = RFAS +appender.RFAS.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit} +appender.RFAS.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit}.%i +appender.RFAS.createOnDemand = true +appender.RFAS.layout.type = PatternLayout +appender.RFAS.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFAS.policies.type = Policies +appender.RFAS.policies.size.type = SizeBasedTriggeringPolicy +appender.RFAS.policies.size.size = ${sys:hbase.security.log.maxfilesize:-256MB} +appender.RFAS.strategy.type = DefaultRolloverStrategy +appender.RFAS.strategy.max = ${sys:hbase.security.log.maxbackupindex:-20} + +# Http Access Log RFA, uncomment this if you want an http access.log +# appender.AccessRFA.type = RollingFile +# appender.AccessRFA.name = AccessRFA +# appender.AccessRFA.fileName = /var/log/hbase/access.log +# appender.AccessRFA.filePattern = /var/log/hbase/access.log.%i +# appender.AccessRFA.createOnDemand = true +# appender.AccessRFA.layout.type = PatternLayout +# appender.AccessRFA.layout.pattern = %m%n +# appender.AccessRFA.policies.type = Policies +# appender.AccessRFA.policies.size.type = SizeBasedTriggeringPolicy +# appender.AccessRFA.policies.size.size = 200MB +# appender.AccessRFA.strategy.type = DefaultRolloverStrategy +# appender.AccessRFA.strategy.max = 10 + +# Null Appender +appender.NullAppender.type = Null +appender.NullAppender.name = NullAppender + +rootLogger = ${sys:hbase.root.logger:-INFO,console} + +logger.SecurityLogger.name = SecurityLogger +logger.SecurityLogger = ${sys:hbase.security.logger:-INFO,console} +logger.SecurityLogger.additivity = false + +# Custom Logging levels +# logger.zookeeper.name = org.apache.zookeeper +# logger.zookeeper.level = ERROR + +# logger.FSNamesystem.name = org.apache.hadoop.fs.FSNamesystem +# logger.FSNamesystem.level = DEBUG + +logger.hbase.name = org.apache.hadoop.hbase +logger.hbase.level = DEBUG + +# logger.META.name = org.apache.hadoop.hbase.META +# logger.META.level = DEBUG + +# Make these two classes below DEBUG to see more zk debug. +# logger.ZKUtil.name = org.apache.hadoop.hbase.zookeeper.ZKUtil +# logger.ZKUtil.level = DEBUG + +# logger.ZKWatcher.name = org.apache.hadoop.hbase.zookeeper.ZKWatcher +# logger.ZKWatcher.level = DEBUG + +# logger.dfs.name = org.apache.hadoop.dfs +# logger.dfs.level = DEBUG + +# Prevent metrics subsystem start/stop messages (HBASE-17722) +logger.MetricsConfig.name = org.apache.hadoop.metrics2.impl.MetricsConfig +logger.MetricsConfig.level = WARN + +logger.MetricsSinkAdapte.name = org.apache.hadoop.metrics2.impl.MetricsSinkAdapter +logger.MetricsSinkAdapte.level = WARN + +logger.MetricsSystemImpl.name = org.apache.hadoop.metrics2.impl.MetricsSystemImpl +logger.MetricsSystemImpl.level = WARN + +# Disable request log by default, you can enable this by changing the appender +logger.http.name = http.requests +logger.http.additivity = false +logger.http = INFO,NullAppender +# Replace the above with this configuration if you want an http access.log +# logger.http = INFO,AccessRFA diff --git a/dev-support/read-replica/cluster1/conf/zoo.cfg b/dev-support/read-replica/cluster1/conf/zoo.cfg new file mode 100755 index 000000000000..8e281c60a6a6 --- /dev/null +++ b/dev-support/read-replica/cluster1/conf/zoo.cfg @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +clientPort=2181 +clientPortAddress=hbase-docker +server.1=hbase-docker:2888:3888 diff --git a/dev-support/read-replica/cluster2/conf/hbase-site.xml b/dev-support/read-replica/cluster2/conf/hbase-site.xml new file mode 100755 index 000000000000..efadf178a22c --- /dev/null +++ b/dev-support/read-replica/cluster2/conf/hbase-site.xml @@ -0,0 +1,77 @@ + + + + + + hbase.zookeeper.quorum + hbase-docker-2 + + + hbase.zookeeper.property.dataDir + /data-store/zk + + + hbase.rootdir + file:///data-store/hbase + + + hbase.wal.dir + file:///data-store/wal + + + hbase.store.file-tracker.impl + FILE + + + + hbase.master.info.bindAddress + hbase-docker-2 + + + hbase.regionserver.info.bindAddress + hbase-docker-2 + + + hbase.unsafe.stream.capability.enforce + false + + + hbase.cluster.distributed + true + + + hbase.rest.port + 8000 + + + + hbase.meta.table.suffix + replica1 + + + hbase.global.readonly.enabled + true + + + hbase.master.hbck.chore.interval + 0 + + diff --git a/dev-support/read-replica/cluster2/conf/log4j2.properties b/dev-support/read-replica/cluster2/conf/log4j2.properties new file mode 100644 index 000000000000..91957fdc8a31 --- /dev/null +++ b/dev-support/read-replica/cluster2/conf/log4j2.properties @@ -0,0 +1,137 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +status = warn +dest = err +name = PropertiesConfig + +# Console appender +appender.console.type = Console +appender.console.target = SYSTEM_ERR +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{ISO8601} %-5p [%t%notEmpty{ %X}] %c{2}: %.1000m%n + +# Daily Rolling File Appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.DRFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%d{yyyy-MM-dd} +appender.DRFA.createOnDemand = true +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.policies.size.type = SizeBasedTriggeringPolicy +appender.DRFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Rolling File Appender +appender.RFA.type = RollingFile +appender.RFA.name = RFA +appender.RFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log} +appender.RFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%i +appender.RFA.createOnDemand = true +appender.RFA.layout.type = PatternLayout +appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFA.policies.type = Policies +appender.RFA.policies.size.type = SizeBasedTriggeringPolicy +appender.RFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB} +appender.RFA.strategy.type = DefaultRolloverStrategy +appender.RFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20} + +# Security Audit Appender +appender.RFAS.type = RollingFile +appender.RFAS.name = RFAS +appender.RFAS.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit} +appender.RFAS.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit}.%i +appender.RFAS.createOnDemand = true +appender.RFAS.layout.type = PatternLayout +appender.RFAS.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n +appender.RFAS.policies.type = Policies +appender.RFAS.policies.size.type = SizeBasedTriggeringPolicy +appender.RFAS.policies.size.size = ${sys:hbase.security.log.maxfilesize:-256MB} +appender.RFAS.strategy.type = DefaultRolloverStrategy +appender.RFAS.strategy.max = ${sys:hbase.security.log.maxbackupindex:-20} + +# Http Access Log RFA, uncomment this if you want an http access.log +# appender.AccessRFA.type = RollingFile +# appender.AccessRFA.name = AccessRFA +# appender.AccessRFA.fileName = /var/log/hbase/access.log +# appender.AccessRFA.filePattern = /var/log/hbase/access.log.%i +# appender.AccessRFA.createOnDemand = true +# appender.AccessRFA.layout.type = PatternLayout +# appender.AccessRFA.layout.pattern = %m%n +# appender.AccessRFA.policies.type = Policies +# appender.AccessRFA.policies.size.type = SizeBasedTriggeringPolicy +# appender.AccessRFA.policies.size.size = 200MB +# appender.AccessRFA.strategy.type = DefaultRolloverStrategy +# appender.AccessRFA.strategy.max = 10 + +# Null Appender +appender.NullAppender.type = Null +appender.NullAppender.name = NullAppender + +rootLogger = ${sys:hbase.root.logger:-INFO,console} + +logger.SecurityLogger.name = SecurityLogger +logger.SecurityLogger = ${sys:hbase.security.logger:-INFO,console} +logger.SecurityLogger.additivity = false + +# Custom Logging levels +# logger.zookeeper.name = org.apache.zookeeper +# logger.zookeeper.level = ERROR + +# logger.FSNamesystem.name = org.apache.hadoop.fs.FSNamesystem +# logger.FSNamesystem.level = DEBUG + +logger.hbase.name = org.apache.hadoop.hbase +logger.hbase.level = DEBUG + +# logger.META.name = org.apache.hadoop.hbase.META +# logger.META.level = DEBUG + +# Make these two classes below DEBUG to see more zk debug. +# logger.ZKUtil.name = org.apache.hadoop.hbase.zookeeper.ZKUtil +# logger.ZKUtil.level = DEBUG + +# logger.ZKWatcher.name = org.apache.hadoop.hbase.zookeeper.ZKWatcher +# logger.ZKWatcher.level = DEBUG + +# logger.dfs.name = org.apache.hadoop.dfs +# logger.dfs.level = DEBUG + +# Prevent metrics subsystem start/stop messages (HBASE-17722) +logger.MetricsConfig.name = org.apache.hadoop.metrics2.impl.MetricsConfig +logger.MetricsConfig.level = WARN + +logger.MetricsSinkAdapte.name = org.apache.hadoop.metrics2.impl.MetricsSinkAdapter +logger.MetricsSinkAdapte.level = WARN + +logger.MetricsSystemImpl.name = org.apache.hadoop.metrics2.impl.MetricsSystemImpl +logger.MetricsSystemImpl.level = WARN + +# Disable request log by default, you can enable this by changing the appender +logger.http.name = http.requests +logger.http.additivity = false +logger.http = INFO,NullAppender +# Replace the above with this configuration if you want an http access.log +# logger.http = INFO,AccessRFA diff --git a/dev-support/read-replica/cluster2/conf/zoo.cfg b/dev-support/read-replica/cluster2/conf/zoo.cfg new file mode 100755 index 000000000000..49243987a4cb --- /dev/null +++ b/dev-support/read-replica/cluster2/conf/zoo.cfg @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +clientPort=2181 +clientPortAddress=hbase-docker-2 +server.1=hbase-docker-2:2888:3888 diff --git a/dev-support/read-replica/docker-compose.yml b/dev-support/read-replica/docker-compose.yml new file mode 100644 index 000000000000..b3953cfe52a9 --- /dev/null +++ b/dev-support/read-replica/docker-compose.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# version: '3.8' + +# Variables are created in .env file +services: + hbase: + image: ${HBASE_IMAGE} + container_name: ${HBASE_CONTAINER_NAME} + hostname: ${HBASE_CONTAINER_NAME} + volumes: + - ./utils:${CONTAINER_UTILS_DIR} + - ${HBASE_DATA_STORE_ROOT}/data-store/hbase:/data-store/hbase + - ${ACTIVE_CLUSTER_CONF_DIR}/hbase-site.xml:${HBASE_CONF_DIR}/hbase-site.xml + - ${ACTIVE_CLUSTER_CONF_DIR}/zoo.cfg:${HBASE_CONF_DIR}/zoo.cfg + - ${ACTIVE_CLUSTER_CONF_DIR}/log4j2.properties:${HBASE_CONF_DIR}/log4j2.properties + - ${ACTIVE_CLUSTER_LOGS_DIR}:/opt/hbase/logs + + hbase2: + image: ${HBASE_IMAGE} + container_name: ${HBASE_CONTAINER_NAME}-2 + hostname: ${HBASE_CONTAINER_NAME}-2 + volumes: + - ./utils:${CONTAINER_UTILS_DIR} + - ${HBASE_DATA_STORE_ROOT}/data-store/hbase:/data-store/hbase + - ${REPLICA_CLUSTER_CONF_DIR}/hbase-site.xml:${HBASE_CONF_DIR}/hbase-site.xml + - ${REPLICA_CLUSTER_CONF_DIR}/zoo.cfg:${HBASE_CONF_DIR}/zoo.cfg + - ${REPLICA_CLUSTER_CONF_DIR}/log4j2.properties:${HBASE_CONF_DIR}/log4j2.properties + - ${REPLICA_CLUSTER_LOGS_DIR}:/opt/hbase/logs diff --git a/dev-support/read-replica/hbase_nightly_read_replica_test.sh b/dev-support/read-replica/hbase_nightly_read_replica_test.sh new file mode 100755 index 000000000000..36b52e8f9c25 --- /dev/null +++ b/dev-support/read-replica/hbase_nightly_read_replica_test.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# hbase_nightly_read_replica_test.sh +# +# Outer driver script for the HBase read-replica integration test suite. +# Builds a dev-support Docker container image (from +# hbase/dev-support/docker/Dockerfile) using the HBase source tree as +# build context, then either: +# +# - Launches run_read_replica_integration_tests.sh inside that +# container to execute the full test suite (default), or +# - Starts a long-lived dev container for interactive exploration +# (-d|--dev mode). +# +# The container uses Docker-outside-of-Docker (DooD) by bind-mounting +# the host's Docker socket, and mounts .m2 for Maven cache reuse +# (override the default $HOME location with -m|--m2). +# +# For usage information, run: ./hbase_nightly_read_replica_test.sh --help +set -e + +format_duration_hms() { + local total_sec=$1 + printf '%d:%02d:%02d' $((total_sec / 3600)) $(((total_sec % 3600) / 60)) $((total_sec % 60)) +} + +REPLICA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export HBASE_ROOT="$(cd "${REPLICA_DIR}/../.." && pwd)" + +DEV_IMAGE_NAME="hbase-dev-support:${BUILD_NUMBER:-local}" + +PYTEST_K_VALUE="" +JAVA_VERSION="" +M2_DIR="${HOME}" +EFFECTIVE_OUTPUT_DIR="${OUTPUT_DIR:-${REPLICA_DIR}/output}" +DEV_MODE=false + +print_usage() { + SCRIPT=$(basename "${BASH_SOURCE}") + + cat << __EOF + +hbase_nightly_read_replica_test.sh + +Outer driver script for the HBase read-replica integration test suite. Builds a +dev-support Docker container image and either runs the inner test script +(run_read_replica_integration_tests.sh) inside it, or starts a long-lived dev +container for interactive exploration. + +Usage: ${SCRIPT} [options] + + -h | --help Show this help message and exit. + -d | --dev Start a detached dev container for interactive + exploration instead of running the test suite. + Prints the container ID and instructions for + entering and stopping it. + -j | --java-version JVM version forwarded to the inner test script + (run_read_replica_integration_tests.sh), which + uses it to set JAVA_HOME inside the container. + -k Pytest -k filter expression forwarded to the + inner test script for test selection. (See + Pytest documentation on -k for more info.) + -m | --m2 Parent directory of the .m2 Maven cache to + bind-mount into the container. Defaults to + \$HOME. The directory /.m2 will be + created if it does not exist. + +__EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + print_usage + exit 0 + ;; + -d|--dev) + DEV_MODE=true + shift + ;; + -j|--java-version) + if [[ -n "$2" && "$2" != -* ]]; then + JAVA_VERSION="$2" + shift 2 + else + echo "Error: Argument for $1 is missing" >&2 + print_usage >&2 + exit 1 + fi + ;; + -k) + if [[ -n "$2" && "$2" != -* ]]; then + PYTEST_K_VALUE="$2" + shift 2 + else + echo "Error: Argument for $1 is missing" >&2 + print_usage >&2 + exit 1 + fi + ;; + -m|--m2) + if [[ -n "$2" && "$2" != -* ]]; then + M2_DIR="$2" + shift 2 + else + echo "Error: Argument for $1 is missing" >&2 + print_usage >&2 + exit 1 + fi + ;; + *) + echo "Unknown option: $1" >&2 + print_usage >&2 + exit 1 + ;; + esac +done + +echo "=== HBase Read-Replica Integration Test Driver ===" +echo "HBase Root: ${HBASE_ROOT}" +echo "Replica Dir: ${REPLICA_DIR}" +echo "Dev Container Image: ${DEV_IMAGE_NAME}" +echo "M2 Dir: ${M2_DIR}/.m2" +echo "Output Dir: ${EFFECTIVE_OUTPUT_DIR}" + +cleanup_host() { + local exit_code=$? + set +e + if [ -z "${BUILD_NUMBER}" ]; then + echo "Local execution complete. Preserving local container image ${DEV_IMAGE_NAME}." + else + echo "Jenkins execution complete. Cleaning up image ${DEV_IMAGE_NAME}..." + docker rmi --force "${DEV_IMAGE_NAME}" 2>/dev/null || true + fi + exit "${exit_code}" +} +trap cleanup_host EXIT + +# Build the dev-support container image using HBASE_ROOT as the build context +echo "Building dev-support Docker image..." +DEV_SUPPORT_IMAGE_BUILD_START=${SECONDS} +docker build --platform linux/amd64 \ + -t "${DEV_IMAGE_NAME}" \ + -f "${HBASE_ROOT}/dev-support/docker/Dockerfile" \ + "${HBASE_ROOT}" +DEV_SUPPORT_IMAGE_BUILD_SEC=$((SECONDS - DEV_SUPPORT_IMAGE_BUILD_START)) +echo "Dev-support Docker image built (${DEV_SUPPORT_IMAGE_BUILD_SEC}s, $(format_duration_hms "${DEV_SUPPORT_IMAGE_BUILD_SEC}"))." + +# Ensure host .m2 directory exists for caching +mkdir -p "${M2_DIR}/.m2" +mkdir -p "${EFFECTIVE_OUTPUT_DIR}" + +# Construct the base URL for per-test log directory links in the console +# report. On Jenkins, links resolve to archived artifact directories; +# locally the variable stays empty and the report falls back to relative links. +if [[ -n "${BUILD_URL}" && -n "${OUTPUT_DIR_RELATIVE}" ]]; then + LOGS_URL="${BUILD_URL}artifact/${OUTPUT_DIR_RELATIVE}" +else + LOGS_URL="" +fi + +if [ "${DEV_MODE}" = "true" ]; then + # Start a detached dev container for interactive exploration + echo "Starting dev container in background..." + CONTAINER_ID=$(docker run -d \ + --platform linux/amd64 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "${HBASE_ROOT}:${HBASE_ROOT}" \ + -v "${EFFECTIVE_OUTPUT_DIR}:${EFFECTIVE_OUTPUT_DIR}" \ + -v "${M2_DIR}/.m2:/root/.m2" \ + -e OUTPUT_DIR="${EFFECTIVE_OUTPUT_DIR}" \ + -e BUILD_NUMBER="${BUILD_NUMBER:-local}" \ + -e LOGS_URL="${LOGS_URL}" \ + -w "${REPLICA_DIR}" \ + "${DEV_IMAGE_NAME}" \ + sleep infinity) + + echo "" + echo "=== Dev container is ready ===" + echo "Container ID: ${CONTAINER_ID}" + echo "Image: ${DEV_IMAGE_NAME}" + echo "" + echo "Enter the container:" + echo "" + echo "docker exec -it ${CONTAINER_ID} bash" + echo "" + echo "Stop and remove the container when done:" + echo "" + echo "docker stop ${CONTAINER_ID} && docker rm ${CONTAINER_ID}" + echo "" +else + JAVA_VERSION_ARGS=() + if [[ -n "${JAVA_VERSION}" ]]; then + JAVA_VERSION_ARGS=(-j "${JAVA_VERSION}") + fi + + PYTEST_K_ARGS=() + if [[ -n "${PYTEST_K_VALUE}" ]]; then + PYTEST_K_ARGS=(-k "${PYTEST_K_VALUE}") + fi + + # Run the inner test script inside the dev-support container via DooD + echo "Launching dev container and starting test suite..." + docker run --rm \ + --platform linux/amd64 \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "${HBASE_ROOT}:${HBASE_ROOT}" \ + -v "${EFFECTIVE_OUTPUT_DIR}:${EFFECTIVE_OUTPUT_DIR}" \ + -v "${M2_DIR}/.m2:/root/.m2" \ + -e OUTPUT_DIR="${EFFECTIVE_OUTPUT_DIR}" \ + -e BUILD_NUMBER="${BUILD_NUMBER:-local}" \ + -e DEV_SUPPORT_IMAGE_BUILD_SEC="${DEV_SUPPORT_IMAGE_BUILD_SEC}" \ + -e LOGS_URL="${LOGS_URL}" \ + -w "${REPLICA_DIR}" \ + "${DEV_IMAGE_NAME}" \ + ./run_read_replica_integration_tests.sh "${JAVA_VERSION_ARGS[@]}" "${PYTEST_K_ARGS[@]}" +fi diff --git a/dev-support/read-replica/python/__init__.py b/dev-support/read-replica/python/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/dev-support/read-replica/python/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/dev-support/read-replica/python/proto/proto_compiler.py b/dev-support/read-replica/python/proto/proto_compiler.py new file mode 100644 index 000000000000..b3f9962c8493 --- /dev/null +++ b/dev-support/read-replica/python/proto/proto_compiler.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Compiles all *.proto files in the 'python/proto' directory. The compiled output is sent to 'python/proto/generated'. +This script removes any existing 'generated' directory and creates a new one every time. +""" +import os +import shutil + +from grpc_tools import protoc +from python.src.logger_config import get_logger + +logger = get_logger(__name__) + + +if __name__ == '__main__': + proto_dir = os.path.dirname(__file__) + generated_dir = os.path.join(proto_dir, 'generated') + + if os.path.exists(generated_dir): + shutil.rmtree(generated_dir) + os.mkdir(generated_dir) + + proto_files = [file for file in os.listdir(proto_dir) if file.endswith('.proto')] + for file in proto_files: + logger.info(f"Compiling {file} and sending output to {generated_dir}") + protoc.main(( + '', + f'-I{proto_dir}', + f'--python_out={generated_dir}/.', + f'--pyi_out={generated_dir}/.', + os.path.join(proto_dir, file), + )) diff --git a/dev-support/read-replica/python/scripts/__init__.py b/dev-support/read-replica/python/scripts/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/dev-support/read-replica/python/scripts/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/dev-support/read-replica/python/scripts/render_console_report.py b/dev-support/read-replica/python/scripts/render_console_report.py new file mode 100755 index 000000000000..ef30b6c9b645 --- /dev/null +++ b/dev-support/read-replica/python/scripts/render_console_report.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Generate a Yetus-style console report for the read-replica integration tests. + +Reads infrastructure timing from a KEY=VALUE env file and per-test results from +JUnit XML, then produces an HTML fragment matching the format used by the +upstream HBase Nightly Build (Apache Yetus). +""" + +from __future__ import annotations + +import argparse +import sys +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class TestResult: + name: str + time_sec: float + passed: bool + rerun_count: int = 0 + + +def parse_timing_env(path: Path) -> dict[str, int]: + timing: dict[str, int] = {} + if not path.exists(): + return timing + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + key, _, value = line.partition("=") + try: + timing[key.strip()] = int(value.strip()) + except ValueError: + pass + return timing + + +def parse_junit_xml(path: Path) -> list[TestResult]: + tree = ET.parse(path) + root = tree.getroot() + + testcases: list[tuple[str, float, bool]] = [] + for tc in root.iter("testcase"): + name = tc.get("name", "unknown") + try: + time_sec = float(tc.get("time", "0")) + except ValueError: + time_sec = 0.0 + has_failure = tc.find("failure") is not None or tc.find("error") is not None + testcases.append((name, time_sec, not has_failure)) + + seen_order: list[str] = [] + entries: dict[str, list[tuple[float, bool]]] = {} + for name, time_sec, passed in testcases: + if name not in entries: + seen_order.append(name) + entries[name] = [] + entries[name].append((time_sec, passed)) + + results: list[TestResult] = [] + for name in seen_order: + runs = entries[name] + total_time = sum(t for t, _ in runs) + final_passed = runs[-1][1] + rerun_count = len(runs) - 1 + results.append(TestResult(name=name, time_sec=total_time, + passed=final_passed, rerun_count=rerun_count)) + return results + + +def format_runtime(total_sec: int | float) -> str: + total_sec = int(round(total_sec)) + minutes = total_sec // 60 + seconds = total_sec % 60 + return f"{minutes:>3d}m {seconds:>2d}s" + + +def _log_link(test_name: str, color: str, logs_url: str, + output_dir: Path) -> str: + test_log_dir = output_dir / test_name + if not test_log_dir.is_dir(): + return "" + if logs_url: + href = f"{logs_url.rstrip('/')}/{test_name}/" + else: + href = f"{test_name}/" + return f'/{test_name}/' + + +def _row(vote: str, color: str, subsystem: str, runtime: str, + log: str, comment: str) -> str: + return ( + "\n" + f'\t\t{vote}\n' + f'\t\t {subsystem} \n' + f'\t\t{runtime}\n' + f"{log}\n" + f' {comment} \n' + "\n" + ) + + +def _infra_row(subsystem: str, seconds: int, comment: str) -> str: + return _row("0", "blue", subsystem, format_runtime(seconds), "", comment) + + +def _detail_row(color: str, log: str, comment: str) -> str: + return ( + "\n" + "\t\t\n" + "\t\t\n" + "\t\t\n" + f"{log}\n" + f' {comment} \n' + "\n" + ) + + +def _pytest_rows(test_results: list[TestResult], timing: dict[str, int], + logs_url: str, output_dir: Path) -> list[str]: + rows: list[str] = [] + + passed_count = sum(1 for t in test_results if t.passed) + failed_count = sum(1 for t in test_results if not t.passed) + total_reruns = sum(t.rerun_count for t in test_results) + + # At least one test ran and there were no failures + all_passed = bool(test_results) and failed_count == 0 + + if all_passed: + vote, color = "+1", "green" + else: + vote, color = "-1", "red" + + pytest_sec = timing.get("PYTEST_SEC", 0) + if pytest_sec == 0: + pytest_sec = int(round(sum(t.time_sec for t in test_results))) + + comment = f"{passed_count} passed, {failed_count} failed, {total_reruns} rerun(s)" + rows.append(_row(vote, color, "pytest", format_runtime(pytest_sec), "", comment)) + + for t in test_results: + if not t.passed: + log = _log_link(t.name, "red", logs_url, output_dir) + rows.append(_detail_row("red", log, f"failed {t.name}")) + + if all_passed: + for t in test_results: + if t.rerun_count > 0: + log = _log_link(t.name, "yellow", logs_url, output_dir) + rows.append(_detail_row("yellow", log, f"rerun {t.name}")) + + return rows + + +def _total_row(total_sec: int) -> str: + return ( + "\n" + '\t\t\n' + '\t\t \n' + f'\t\t{format_runtime(total_sec)}\n' + "\n" + '\n' + "\n" + ) + + +_INFRA_STAGES = [ + ("Docker test env", "DEV_SUPPORT_IMAGE_BUILD_SEC", + "Build time for Docker test environment image"), + ("rsync", "RSYNC_SEC", + "Copy hbase repo for Docker build context with read-replica image"), + ("mvn clean", "MVN_CLEAN_SEC", + "Maven clean to remove previous build artifacts for read-replica image"), + ("read-replica image", "DOCKER_BUILD_SEC", + "Docker image used for containers running an HBase read-replica setup"), +] + + +def build_console_report(timing: dict[str, int], + test_results: list[TestResult], + logs_url: str = "", + output_dir: Path | None = None) -> str: + all_passed = all(t.passed for t in test_results) + + if all_passed: + overall_color, overall_vote = "green", "+1" + else: + overall_color, overall_vote = "red", "-1" + + # Table 1: Overall header + header_table = ( + "\n" + f'\n" + "
' + f"{overall_vote} overall
\n" + ) + + # Title + title_table = ( + "\n" + "\n" + "
Read-Replica Nightly Test Console Report
\n" + ) + + # Table 2: Stage results + rows: list[str] = [] + + for subsystem, timing_key, comment in _INFRA_STAGES: + seconds = timing.get(timing_key, 0) + rows.append(_infra_row(subsystem, seconds, comment)) + + effective_output_dir = output_dir if output_dir is not None else Path() + rows.extend(_pytest_rows(test_results, timing, logs_url, effective_output_dir)) + + total_sec = timing.get("TOTAL_SEC", 0) + rows.append(_total_row(total_sec)) + + stages_table = ( + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + + "".join(rows) + + "
VoteSubsystemRuntimeLogComment
\n" + ) + + return ( + header_table + + "

\n" + + title_table + + "

\n" + + stages_table + + "

\n" + + "

This message was automatically generated.

\n" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Generate a Yetus-style console report for read-replica tests." + ) + parser.add_argument( + "--timing", required=True, type=Path, + help="Path to timing env file (KEY=VALUE pairs)", + ) + parser.add_argument( + "--junit", required=True, type=Path, + help="Path to JUnit XML results file", + ) + parser.add_argument( + "--output", required=True, type=Path, + help="Path to write the HTML console report", + ) + parser.add_argument( + "--logs-url", type=str, default="", + help="Base URL for per-test log directories. " + "On Jenkins: ${BUILD_URL}artifact/${OUTPUT_DIR_RELATIVE}. " + "When empty (default), test rows use relative links.", + ) + args = parser.parse_args(argv) + + if not args.junit.exists(): + print(f"JUnit XML not found: {args.junit}", file=sys.stderr) + return 1 + + timing = parse_timing_env(args.timing) + test_results = parse_junit_xml(args.junit) + report_html = build_console_report( + timing, test_results, + logs_url=args.logs_url, + output_dir=args.output.parent, + ) + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(report_html, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/dev-support/read-replica/python/scripts/verify_hbase_start.py b/dev-support/read-replica/python/scripts/verify_hbase_start.py new file mode 100644 index 000000000000..4b96f0a0a3db --- /dev/null +++ b/dev-support/read-replica/python/scripts/verify_hbase_start.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Verifies the hbase-docker containers have started properly. For each cluster, the script first +curls the HBase UI until it receives a 200 response and then gets the server status to verify +there are no dead clusters +""" +import argparse + +from python.src import get_env +from python.src.hbase_docker_client import HBaseDockerClient +from python.src.logger_config import get_logger +from python.src.utils import (load_env_and_set_up_clients, log_script_start, log_script_end, + add_common_skip_container_stop_or_restart_arg) + +logger = get_logger(__name__) + + +def main(): + start_time = log_script_start(__file__, logger) + + parser = argparse.ArgumentParser() + parser = add_common_skip_container_stop_or_restart_arg(parser) + args = parser.parse_args() + + active_cluster, replica_cluster = load_env_and_set_up_clients(cluster1_name="Active Cluster", + cluster2_name="Read-Replica Cluster") + data_store_root = get_env("HBASE_DATA_STORE_ROOT") + docker_compose_file = get_env("DOCKER_COMPOSE_FILE") + + if not args.skip_container_start_or_restart: + HBaseDockerClient.start_or_restart_containers(docker_compose_file=docker_compose_file, + data_store_root=f'{data_store_root}') + + HBaseDockerClient.wait_for_clusters_to_start([active_cluster, replica_cluster]) + + log_script_end(__file__, logger, start_time) + + +if __name__ == "__main__": + main() diff --git a/dev-support/read-replica/python/src/__init__.py b/dev-support/read-replica/python/src/__init__.py new file mode 100644 index 000000000000..380ba83067c2 --- /dev/null +++ b/dev-support/read-replica/python/src/__init__.py @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .hbase_docker_client import HBaseDockerClient +from .environment_loader import get_env +from .logger_config import get_logger diff --git a/dev-support/read-replica/python/src/environment_loader.py b/dev-support/read-replica/python/src/environment_loader.py new file mode 100644 index 000000000000..458f917821c1 --- /dev/null +++ b/dev-support/read-replica/python/src/environment_loader.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + + +def get_env(key, default=None): + """Retrieve environment variables, ensuring they are loaded from the GitHub Actions runner.""" + val = os.environ.get(key, default) + if val is None: + raise RuntimeError(f"Error: Environment variable {key} is not set.") + return val diff --git a/dev-support/read-replica/python/src/hbase_docker_client.py b/dev-support/read-replica/python/src/hbase_docker_client.py new file mode 100644 index 000000000000..6982470150e6 --- /dev/null +++ b/dev-support/read-replica/python/src/hbase_docker_client.py @@ -0,0 +1,655 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import ast +import logging +import re +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError + +import docker +import subprocess +import time +import xml.etree.ElementTree as ET + +from .logger_config import get_logger + +logger = get_logger(__name__) + + +class DockerExecCommandError(Exception): + pass + + +class HBaseShellCommandError(DockerExecCommandError): + pass + + +class DockerExecCommandTimeoutError(DockerExecCommandError): + pass + + +class HBaseInitializationError(Exception): + pass + + +class HBaseDockerClient: + def __init__(self, container_name: str, local_conf: str, hbase_ui_port: int = 16010, + cluster_name: str = "HBase Cluster", max_retries: int = 12, sleep_time: int = 5) -> None: + self._container_name = container_name + self._local_conf = local_conf + self._hbase_ui_port = hbase_ui_port + self._cluster_name = cluster_name + self._max_retries = max_retries + self._sleep_time = sleep_time + self._docker_client = docker.from_env() + + @property + def name(self) -> str: + return self._cluster_name + + def run_docker_exec_command(self, bash_cmd: str, timeout: int | None = None) -> str: + """ + Uses the Docker SDK to exec a Bash command in the object's Docker container. + Equivalent to: docker exec bash -c + """ + cmd = ["bash", "-c", bash_cmd] + cmd_str = f"docker exec {self._container_name} bash -c {bash_cmd}" + logger.debug(f"Running command on {self._cluster_name}: {cmd_str}") + + try: + container = self._docker_client.containers.get(self._container_name) + + if timeout is not None: + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit(container.exec_run, cmd, demux=True) + try: + result = future.result(timeout=timeout) + except FuturesTimeoutError: + raise DockerExecCommandTimeoutError( + f"Command timed out after {timeout}s on {self._cluster_name} " + f"({self._container_name}): {bash_cmd}\n" + f"The command used to run this was: {cmd_str}\n" + ) + finally: + pool.shutdown(wait=False, cancel_futures=True) + else: + result = container.exec_run(cmd, demux=True) + except docker.errors.DockerException as e: + raise DockerExecCommandError( + f"The following command failed on {self._cluster_name} ({self._container_name}): {bash_cmd}\n" + f"The command used to run this was: {cmd_str}\n" + f"Docker error: {e}\n" + ) + + exit_code, (stdout, stderr) = result + stdout_str = (stdout or b'').decode('utf-8') + if exit_code != 0: + raise DockerExecCommandError( + f"The following command failed on {self._cluster_name} ({self._container_name}): {bash_cmd}\n" + f"The command used to run this was: {cmd_str}\n" + f"The command's STDERR was:\n{(stderr or b'').decode('utf-8')}\n" + f"The command's STDOUT was:\n{stdout_str}\n" + ) + return stdout_str + + def run_hbase_shell_command(self, hbase_cmd: str, timeout: int | None = None) -> str: + """ + Uses 'docker exec' to run the provided HBase shell command in the object's Docker container. + The command looks like: docker exec bash -c hbase shell -n <<< "" + """ + hbase_shell_cmd = f'''hbase shell -n <<< "{hbase_cmd}"''' + try: + return self.run_docker_exec_command(hbase_shell_cmd, timeout=timeout) + except DockerExecCommandTimeoutError: + # DockerExecCommandTimeoutError is a subclass of DockerExecCommandError, so we need to make sure + # it's specifically caught and re-raised. Otherwise, it's swallowed when catching DockerExecCommandError + raise + except DockerExecCommandError as e: + raise HBaseShellCommandError(e) + + def _get_pid_from_jps(self, process_name: str) -> int | None: + """Runs jps inside the container and returns the PID of the named process, or None.""" + try: + output = self.run_docker_exec_command("jps") + for line in output.strip().splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1] == process_name: + return int(parts[0]) + except DockerExecCommandError: + pass + return None + + def wait_for_hbase_ui(self) -> bool: + """ + Checks for a 200 OK on the HBase Master UI inside the container using curl. + """ + check_cmd = f"curl -s -f --max-time 3 http://{self._container_name}:{self._hbase_ui_port} > /dev/null" + + logger.info(f"Waiting for HBase UI inside container: {self._cluster_name} ({self._container_name})") + last_error = None + for attempt in range(1, self._max_retries + 1): + try: + self.run_docker_exec_command(check_cmd, timeout=self._sleep_time) + logger.info(f"SUCCESS: {self._cluster_name} HBase UI is up.") + return True + except DockerExecCommandError as e: + last_error = e + + logger.info(f"Waiting {self._sleep_time} seconds before checking HBase UI inside container again") + time.sleep(self._sleep_time) + + raise HBaseInitializationError( + f"\nTIMEOUT: {self._cluster_name} UI failed to respond inside container after " + f"{self._max_retries} attempts.\nLast error: {last_error}" + ) + + def wait_for_master_initialization(self) -> bool: + """Waits for the current HMaster process to log 'Master has completed initialization'.""" + logger.info(f"Waiting for Master initialization: {self._cluster_name} ({self._container_name})") + for attempt in range(1, self._max_retries + 1): + pid = self._get_pid_from_jps("HMaster") + if pid is not None: + awk_cmd = ( + f"awk '/env:JVM_PID={pid}/{{seen=1; found=0}} " + f"seen && /Master has completed initialization/{{found=1}} " + f"END{{exit !found}}' /opt/hbase/logs/hbase-*-master-*.log" + ) + try: + self.run_docker_exec_command(awk_cmd) + logger.info(f"SUCCESS: {self._cluster_name} Master has completed initialization.") + return True + except DockerExecCommandError: + pass + logging.info(f"Waiting {self._sleep_time} seconds before checking Master initialization again") + time.sleep(self._sleep_time) + + raise HBaseInitializationError( + f"\nTIMEOUT: {self._cluster_name} Master failed to initialize after " + f"{self._max_retries} attempts." + ) + + def wait_for_region_server_initialization(self) -> bool: + """Waits for the current HRegionServer process to log 'Serving as' message.""" + logger.info(f"Waiting for RegionServer initialization: {self._cluster_name} ({self._container_name})") + for attempt in range(1, self._max_retries + 1): + pid = self._get_pid_from_jps("HRegionServer") + if pid is not None: + awk_cmd = ( + f"awk '/env:JVM_PID={pid}/{{seen=1; found=0}} " + f"seen && /Serving as {self._container_name},/{{found=1}} " + f"END{{exit !found}}' /opt/hbase/logs/hbase-*-regionserver-*.log" + ) + try: + self.run_docker_exec_command(awk_cmd) + logger.info(f"SUCCESS: {self._cluster_name} RegionServer is serving.") + return True + except DockerExecCommandError: + pass + logging.info(f"Waiting {self._sleep_time} seconds before checking RegionServer initialization again") + time.sleep(self._sleep_time) + + raise HBaseInitializationError( + f"\nTIMEOUT: {self._cluster_name} RegionServer failed to initialize after " + f"{self._max_retries} attempts." + ) + + def check_server_status(self, desired_status: dict | None = None) -> bool: + """Runs 'status' inside the HBase shell and validates the output.""" + if desired_status is None: + desired_status = {'masters': '1', 'region_servers': '1', 'dead_servers': '0'} + logger.info(f"Validating Cluster Status: {self._cluster_name} ({self._container_name})") + for attempt in range(1, self._max_retries + 1): + try: + output = self.get_hbase_status() + + # The cluster's status should have 1 active master, 1 region server, + # and no dead servers + validations = { + "Active Master": f"{desired_status['masters']} active master" in output, + "Region Server": f"{desired_status['region_servers']} servers" in output, + "No Dead Servers": f"{desired_status['dead_servers']} dead" in output + } + + if all(validations.values()): + for check, status in validations.items(): + logger.info(f" [PASS] {check}") + logger.info(f"SUCCESS: {self._cluster_name} is fully operational.") + return True + else: + logger.warning(f"{self._cluster_name} is responding, but not all " + f"components are ready...") + logger.info(f"HBase 'status' command output:\n{output}") + + except HBaseShellCommandError: + pass + + logging.info(f"Waiting {self._sleep_time} seconds before getting status on {self.name} again") + time.sleep(self._sleep_time) + + raise HBaseInitializationError( + f"\nTIMEOUT: {self._cluster_name} shell check failed after {self._max_retries} attempts." + ) + + def get_hbase_status(self) -> str: + logger.debug(f"Getting status of {self.name}") + return self.run_hbase_shell_command("status") + + def wait_for_cluster_to_start(self) -> None: + """curls the cluster's HBase UI to make sure it is up and then makes sure all desired servers are up""" + self.wait_for_hbase_ui() + self.wait_for_master_initialization() + self.wait_for_region_server_initialization() + self.check_server_status() + + def create_table(self, table_name: str, column_family: str) -> bool: + logger.info(f"Creating table '{table_name}' on {self._cluster_name}") + create_cmd = f"create '{table_name}', '{column_family}'" + output = self.run_hbase_shell_command(create_cmd) + + if f"Created table {table_name}" not in output: + logger.error(f"Could not create table '{table_name}' on {self._cluster_name}") + return False + return True + + def disable_table(self, table_name: str) -> None: + logger.debug(f"Disabling table '{table_name}' on {self.name}") + self.run_hbase_shell_command(f"disable '{table_name}'") + + def drop_table(self, table_name: str) -> None: + logger.info(f"Dropping table '{table_name}' on {self.name}") + self.run_hbase_shell_command(f"drop '{table_name}'") + + def list_tables(self) -> list: + """Gets the list of HBase tables and returns it as a Python list""" + logger.debug(f"Getting the list of tables in HBase on {self.name}") + pattern = r'\[(.*?)\]' + output = self.run_hbase_shell_command("list") + output = output.replace('\n', ' ') + match = re.search(pattern, output) + return ast.literal_eval(match.group(0)) + + def list_regions(self, table_name: str) -> str: + """Gets list of regions and their info for the provided table""" + logger.info(f"Getting list of regions for table '{table_name}'") + return self.run_hbase_shell_command(f"list_regions '{table_name}'") + + def put(self, table_name: str, row: str, column: str, data: str, spec_map: str | None = None) -> None: + """ + Performs an HBase put command. + :param table_name: the table we are inserting data into + :param row: row of the table we are inserting data into + :param column: column of the table we are inserting data into + :param data: the actual data we are inserting (as a string) + :param spec_map: additional attributes input as a string + (e.g. "{ATTRIBUTES=>{'my-key'=>'my-value'}}") + """ + logger.info(f"Adding data to table '{table_name}' on {self.name}") + put_cmd = f"put '{table_name}', '{row}', '{column}', '{data}'" + if spec_map: + put_cmd += f", {spec_map}" + self.run_hbase_shell_command(put_cmd) + + def get(self, table_name: str, row: str, column: str | None = None, spec_map: str | None = None) -> str: + logger.info(f"Getting data from table '{table_name}' on {self.name}") + get_cmd = f"get '{table_name}', '{row}'" + if column: + get_cmd += f", '{column}'" + if spec_map: + get_cmd += f", {spec_map}" + output = self.run_hbase_shell_command(get_cmd) + logger.debug(f"Got data:\n{output}") + return output + + def delete(self, table_name: str, row: str, column: str, timestamp: int | None = None, + spec_map: str | None = None) -> None: + logger.info(f"Deleting data from table '{table_name}' on {self.name}") + delete_cmd = f"delete '{table_name}', '{row}', '{column}'" + if timestamp: + delete_cmd += f", {timestamp}" + if spec_map: + delete_cmd += f", {spec_map}" + self.run_hbase_shell_command(delete_cmd) + + def scan(self, table_name: str, spec_map: str | None = None) -> str: + log_msg = f"Scanning table '{table_name}' on {self.name}" + scan_cmd = f"scan '{table_name}'" + if spec_map: + scan_cmd += f", {spec_map}" + log_msg += f" with spec_map {spec_map}" + logging.info(log_msg) + return self.run_hbase_shell_command(scan_cmd) + + def count(self, table_name: str, spec: str | None = None) -> str: + logger.info(f"Counting rows for table '{table_name}' on {self.name}") + count_cmd = f"count '{table_name}'" + if spec: + count_cmd += f"{spec}" + return self.run_hbase_shell_command(count_cmd) + + def flush(self, table_name: str, timeout: int | None = None) -> None: + logger.debug(f"Flushing table '{table_name}' on {self.name}") + self.run_hbase_shell_command(f"flush '{table_name}'", timeout=timeout) + + def split(self, thing_to_split: str, split_key: str | None = None) -> None: + log_msg = f"Splitting '{thing_to_split}'" + split_cmd = f"split '{thing_to_split}'" + + if split_key: + log_msg += f" on key '{split_key}'" + split_cmd += f", '{split_key}'" + + log_msg += f" on {self.name}" + + logger.info(log_msg) + self.run_hbase_shell_command(split_cmd) + + def flush_and_split(self, thing_to_split: str, split_key: str | None = None) -> None: + """ + Flushes the table and triggers an asynchronous region split. Split the entire table or pass a region to split an + individual region. With the second parameter, you can specify an explicit split key for the region. + + thing_to_split - TABLENAME, REGIONNAME, or ENCODED_REGIONNAME + split_key - where to have the region split + """ + self.flush(thing_to_split) + self.split(thing_to_split, split_key) + + def major_compact(self, table_or_region: str, column_family: str | None = None, mob: str | None = None) -> None: + log_msg = f"Running major_compact on '{table_or_region}'" + command = f"major_compact '{table_or_region}'" + + if column_family: + log_msg += f" for column family '{column_family}'" + command += f", '{column_family}'" + + if mob and column_family: + log_msg += " with MOB" + command += f", 'MOB'" + elif mob and not column_family: + log_msg += " with MOB" + command += ", nil, 'MOB'" + + self.run_hbase_shell_command(command) + + def major_compact_and_wait(self, table_or_region: str, column_family: str | None = None, mob: str | None = None, + timeout: int = 30, sleep_time: int = 1) -> bool: + """Triggers major compaction on a table and blocks until it completes.""" + logger.info(f"Triggering major compaction on '{table_or_region}' on {self.name}...") + self.major_compact(table_or_region, column_family, mob) + + start_time = time.time() + while time.time() - start_time < timeout: + output = self.run_hbase_shell_command(f"compaction_state '{table_or_region}'") + + # When all regions finish compacting, compaction_state returns NONE + if "NONE" in output: + logger.info(f"SUCCESS: Major compaction completed for '{table_or_region}'.") + return True + + logger.debug(f"Compaction still in progress for '{table_or_region}'... waiting {sleep_time}s") + time.sleep(sleep_time) + + raise RuntimeError( + f"TIMEOUT: Major compaction on table '{table_or_region}' failed to complete within {timeout} seconds." + ) + + def catalogjanitor_run(self) -> None: + """Forces the CatalogJanitor to immediately clean up split parent regions in hbase:meta.""" + logger.info(f"Running catalogjanitor_run on {self.name}") + self.run_hbase_shell_command("catalogjanitor_run") + + def refresh_meta(self) -> None: + logger.info(f"Refreshing meta on {self.name}") + self.run_hbase_shell_command("refresh_meta") + + def refresh_hfiles(self) -> None: + logger.info(f"Refreshing HFiles on {self.name}") + self.run_hbase_shell_command("refresh_hfiles") + + def refresh_meta_and_hfiles(self) -> None: + """Consecutively runs refresh_meta and refresh_hfiles in the HBase shell""" + self.refresh_meta() + self.refresh_hfiles() + + def enable_read_only_mode(self, run_update_all_config: bool = True) -> None: + """ + Sets hbase.global.readonly.enabled to 'true' in the local hbase-site.xml file and runs update_all_config + to dynamically update the configuration. This method assumes the hbase-site.xml file is a mounted volume + in the docker-compose file, which allows the config file within the docker container to be updated as well. + """ + self._set_read_only_mode(new_read_only_flag=True, run_update_all_config=run_update_all_config) + + def disable_read_only_mode(self, run_update_all_config: bool = True) -> None: + """ + Sets hbase.global.readonly.enabled to 'false' in the local hbase-site.xml file and runs update_all_config + to dynamically update the configuration. This method assumes the hbase-site.xml file is a mounted volume + in the docker-compose file, which allows the config file within the docker container to be updated as well. + """ + self._set_read_only_mode(new_read_only_flag=False, run_update_all_config=run_update_all_config) + + def _set_read_only_mode(self, new_read_only_flag: bool, run_update_all_config: bool = True) -> None: + action = "Enabling" if new_read_only_flag else "Disabling" + conjunction_adverb = "and then" if run_update_all_config else "but not" + logger.info(f"{action} read-only mode in conf for {self.name} " + f"{conjunction_adverb} running update_all_config after") + + new_read_only_flag = str(new_read_only_flag).lower() + self.set_hbase_conf_property_value('hbase.global.readonly.enabled', new_read_only_flag) + actual = self.get_hbase_conf_property_value('hbase.global.readonly.enabled') + assert actual == new_read_only_flag, ( + f"Expected hbase.global.readonly.enabled={new_read_only_flag} on {self.name}, but got '{actual}'" + ) + if run_update_all_config: + self.update_all_config() + + def update_all_config(self) -> None: + logger.debug(f"Running update_all_config on {self.name} to dynamically update the configuration") + self.run_hbase_shell_command("update_all_config") + + def get_hbase_conf_property_value(self, conf_prop: str) -> str | None: + tree = ET.parse(self._local_conf) + root = tree.getroot() + for prop in root.findall('property'): + name_elem = prop.find('name') + if name_elem is not None and name_elem.text == conf_prop: + return prop.find('value').text + + def set_hbase_conf_property_value(self, conf_prop: str, value: str) -> None: + """Sets hbase.global.readonly.enabled to a new value in a local hbase-site.xml file""" + tree = ET.parse(self._local_conf) + root = tree.getroot() + for prop in root.findall('property'): + name_elem = prop.find('name') + if name_elem is not None and name_elem.text == conf_prop: + value_elem = prop.find('value') + if value_elem is not None: + value_elem.text = str(value) + break + tree.write(self._local_conf, encoding='utf-8', xml_declaration=True) + # The conf file is a Docker volume - wait for the updated version to sync + time.sleep(1) + + def assert_read_only_error_occurs(self, cmd_type: str, table_name: str, column: str, + row: str | None = None, data: str | None = None) -> None: + """ + Runs a command on read-only cluster and expects an error to occur as a result. + """ + logger.info(f"Verifying we cannot perform a '{cmd_type}' on {self.name} " + f"since it is in read-only mode") + try: + # This should throw an exception + match cmd_type.lower(): + case 'create': + self.create_table(table_name, column) + case 'drop': + self.drop_table(table_name) + case 'put': + self.put(table_name, row, column, data) + case 'delete': + self.delete(table_name, row, column) + case _: + raise RuntimeError(f"Unexpected command type: {cmd_type}") + + # If we get here, then the command succeeded on the read-replica cluster, which should + # not have happened. + raise RuntimeError(f"Expected {cmd_type} attempt on {self.name} " + f"to result in an error") + except HBaseShellCommandError as e: + # Verify the command we ran on the read-replica cluster produced the expected exception + expected_error = ("org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: " + "Operation not allowed in Read-Only Mode") + assert expected_error in str(e), (f"Expected exception to contain the following: " + f"{expected_error}\n" + f"The actual exception was:\n{e}") + logger.info(f"{cmd_type.capitalize()} attempt on {self.name} failed as expected") + + def assert_table_does_not_exist(self, table_name: str) -> None: + logger.info(f"Verifying '{table_name}' is not in the list of tables on {self.name}") + assert table_name not in self.list_tables(), \ + f"Expected table '{table_name}' to not exist on {self.name}" + + def assert_table_exists(self, table_name: str) -> None: + logger.info(f"Verifying '{table_name}' is in the list of tables on {self.name}") + assert table_name in self.list_tables(), \ + f"Expected table '{table_name}' to exist on {self.name}" + + def assert_table_row_count(self, table_name: str, expected_row_count: int) -> None: + logger.info(f"Verifying table '{table_name}' on {self.name} has {expected_row_count} row(s)") + output = self.count(table_name) + match = re.search(r'^(\d+) row\(s\)$', output, re.MULTILINE) + actual_row_count = int(match.group(1)) if match else None + assert actual_row_count == expected_row_count, \ + (f"Expected table '{table_name}' on {self.name} to have {expected_row_count} row(s). " + f"Instead got {actual_row_count}") + + def assert_get_output(self, table: str, row: str, cf: str, expected_data: str) -> str: + output = self.get(table, row, cf) + assert f"value={expected_data}" in output, \ + f"Expected get command to retrieve a row with value={expected_data}. Output instead was:\n{output}" + return output + + def assert_region_count_for_table(self, table_name: str, expected_region_count: int) -> None: + logger.info(f"Verifying table '{table_name}' has {expected_region_count} region(s)") + output = self.list_regions(table_name) + match = re.search(r'^ (\d+) rows$', output, re.MULTILINE) + actual_region_count = int(match.group(1)) if match else None + assert actual_region_count == expected_region_count, \ + (f"Expected table '{table_name}' on {self.name} to have {expected_region_count} region(s). " + f"Instead got {actual_region_count}") + + @staticmethod + def __run_subprocess_command(command: list | str, error_msg: str, + shell: bool = False) -> subprocess.CompletedProcess: + if shell: + cmd_msg = command + else: + cmd_msg = f"{' '.join(command)}" + logger.info(f"Running: {cmd_msg}") + result = subprocess.run(command, capture_output=True, text=True, shell=shell) + if result.returncode != 0: + raise RuntimeError( + f"Command failed: {cmd_msg}\n" + f"{error_msg} (exit {result.returncode}):\n" + f"STDOUT: {result.stdout}\nSTDERR: {result.stderr}" + ) + return result + + @staticmethod + def wait_for_clusters_to_start(clusters: list) -> None: + for cluster in clusters: + cluster.wait_for_cluster_to_start() + logger.info("=" * 40) + logger.info("ALL CLUSTERS VERIFIED AND READY") + logger.info("=" * 40) + + @staticmethod + def are_containers_running(docker_compose_file: str | None = None) -> bool: + logger.info("Checking if docker containers are running") + command = ["docker", "compose"] + if docker_compose_file: + command += ["-f", docker_compose_file] + command += ["ps", "--status", "running", "-q"] + result = HBaseDockerClient.__run_subprocess_command(command, "Failed to get docker container status") + return bool(result.stdout.strip()) + + @staticmethod + def set_up_data_store_dir(data_store_root: str) -> None: + command = ["mkdir", "-p", f"{data_store_root}/data-store/hbase", f"{data_store_root}/data-store/run", + f"{data_store_root}/data-store/logs", f"{data_store_root}/data-store/zk"] + HBaseDockerClient.__run_subprocess_command(command, + error_msg=f"Failed to create {data_store_root} and its sub-dirs") + command = ["chmod", "-R", "777", f"{data_store_root}"] + HBaseDockerClient.__run_subprocess_command(command, + error_msg=f"Failed to give {data_store_root} " + f"and its sub-dirs full permissions") + + @staticmethod + def remove_data_store_dir(data_store_root: str, sudo: bool = False) -> None: + command = ["rm", "-rf", data_store_root] + if sudo: + command = ["sudo"] + command + logger.info(f"Deleting HBase data root dir at: {data_store_root}") + HBaseDockerClient.__run_subprocess_command(command, + f"Could not delete data store root: {data_store_root}") + + @staticmethod + def start_or_restart_containers(docker_compose_file: str | None = None, data_store_root: str | None = None) -> None: + if data_store_root: + HBaseDockerClient.set_up_data_store_dir(data_store_root) + + if HBaseDockerClient.are_containers_running(docker_compose_file): + logger.info("Restarting docker containers") + command = ["docker", "compose"] + if docker_compose_file: + command += ["-f", docker_compose_file] + command += ["restart"] + action = "restart" + else: + logger.info("Starting docker containers") + command = ["docker", "compose"] + if docker_compose_file: + command += ["-f", docker_compose_file] + command += ["up", "-d"] + action = "start" + + HBaseDockerClient.__run_subprocess_command(command, f"docker compose {action} failed") + logger.info(f"docker compose {action} completed successfully") + + @staticmethod + def start_service(service_name: str, docker_compose_file: str | None = None) -> None: + logger.info(f"Starting docker compose service: {service_name}") + command = ["docker", "compose"] + if docker_compose_file: + command += ["-f", docker_compose_file] + command += ["up", "-d", service_name] + HBaseDockerClient.__run_subprocess_command(command, f"Failed to start service '{service_name}'") + + @staticmethod + def stop_containers(docker_compose_file: str | None = None, data_store_root: str | None = None, + sudo: bool = False) -> None: + command = ["docker", "compose"] + if docker_compose_file: + command += ["-f", docker_compose_file] + command += ["down"] + logger.info("Stopping docker containers") + HBaseDockerClient.__run_subprocess_command(command, "stop_containers failed") + logger.info("Successfully stopped docker containers") + + if data_store_root: + HBaseDockerClient.remove_data_store_dir(data_store_root, sudo) diff --git a/dev-support/read-replica/python/src/logger_config.py b/dev-support/read-replica/python/src/logger_config.py new file mode 100644 index 000000000000..e914ebe40929 --- /dev/null +++ b/dev-support/read-replica/python/src/logger_config.py @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging +import sys + +from dotenv import load_dotenv +from .environment_loader import get_env + +LOG_FORMAT = '%(asctime)s %(levelname)-5s %(module)s.%(funcName)s(%(lineno)d): %(message)s' + +# Load settings from .env file +load_dotenv() + + +def configure_logging(level=get_env('LOG_LEVEL')): + """ + Centralized logging configuration for HBase testing scripts. + """ + # Convert string level ('DEBUG', 'INFO') to integer level (10, 20) + if isinstance(level, str): + numeric_level = logging.getLevelName(level.upper()) + else: + numeric_level = level + + logging.basicConfig( + format=LOG_FORMAT, + level=numeric_level, + handlers=[ + logging.StreamHandler(sys.stdout) + ] + ) + + # Suppress verbose HTTP connection logs from Python Docker SDK and urllib3 + # when the root logger is set to DEBUG or more verbose. + if numeric_level <= logging.DEBUG: + logging.getLogger("urllib3").setLevel(logging.INFO) + logging.getLogger("docker").setLevel(logging.INFO) + + +def get_logger(name): + """ + Helper to get a logger. This can be used to ensure the config + is applied whenever a logger is requested. + """ + # If the root logger has no handlers, configure it now + if not logging.getLogger().hasHandlers(): + configure_logging() + return logging.getLogger(name) diff --git a/dev-support/read-replica/python/src/utils.py b/dev-support/read-replica/python/src/utils.py new file mode 100644 index 000000000000..a7f410a8b602 --- /dev/null +++ b/dev-support/read-replica/python/src/utils.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import os +import time + +from dotenv import load_dotenv + +import python.proto.generated.ActiveClusterSuffix_pb2 as acs + +from python.src.environment_loader import get_env +from python.src.hbase_docker_client import HBaseDockerClient +from python.src.logger_config import get_logger + +logger = get_logger(__name__) + + +def log_script_start(file: str, script_logger=None): + (script_logger or logger).info(f"========== START {os.path.basename(file)} ==========") + return time.time() + + +def log_script_end(file: str, script_logger=None, start_time=None): + elapsed = "" + if start_time is not None: + total_seconds = int(time.time() - start_time) + hours, remainder = divmod(total_seconds, 3600) + minutes, seconds = divmod(remainder, 60) + elapsed = f" ({hours}h {minutes}m {seconds}s)" + (script_logger or logger).info(f"========== END {os.path.basename(file)}{elapsed} ==========") + + +def add_common_drop_existing_tables_arg(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + parser.add_argument('-d', '--drop-existing-tables', action='store_true', + help='Drop all existing tables on the active cluster and ' + 'sync the replica cluster by the refreshing meta') + return parser + + +def add_common_new_containers_arg(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + parser.add_argument('-n', '--new-containers', action='store_true', + help='Starts or restarts a clean slate of Docker containers with no pre-existing HBase data') + return parser + + +def load_env_and_set_up_clients(cluster1_name: str = "Cluster 1", + cluster2_name: str = "Cluster 2") -> tuple[HBaseDockerClient, HBaseDockerClient]: + load_dotenv() + container_name = get_env("HBASE_CONTAINER_NAME") + + active_cluster = HBaseDockerClient(container_name=container_name, + local_conf=f"{get_env('ACTIVE_CLUSTER_CONF_DIR')}/hbase-site.xml", + cluster_name=cluster1_name) + replica_cluster = HBaseDockerClient(container_name=f'{container_name}-2', + local_conf=f"{get_env('REPLICA_CLUSTER_CONF_DIR')}/hbase-site.xml", + cluster_name=cluster2_name) + return active_cluster, replica_cluster + + +def run_put_and_get(cluster: HBaseDockerClient, table: str, row: str, cf: str, data: str): + cluster.put(table, row, cf, data) + cluster.assert_table_row_count(table, expected_row_count=1) + return cluster.assert_get_output(table, row, cf, expected_data=data) + + +def assert_crud_operations_work_on_active_cluster(cluster: HBaseDockerClient): + table = 'crud-test-table1' + cf = 'cf' + row = 'r1' + data = '1' + + # Create + cluster.create_table(table, cf) + cluster.assert_table_exists(table) + + # Retrieve + run_put_and_get(cluster, table, row, cf, data) + + # "Update" (there are no true updates in HBase) + data = '2' + run_put_and_get(cluster, table, row, cf, data) + + # Delete + # This row has two versions. This only deletes the first version + cluster.delete(table, row, column=f"{cf}:") + cluster.assert_table_row_count(table, expected_row_count=1) + cluster.assert_get_output(table, row, cf, expected_data='1') + + # Delete the final version + cluster.delete(table, row, column=f"{cf}:") + cluster.assert_table_row_count(table, expected_row_count=0) + + # Drop table + cluster.disable_table(table) + cluster.drop_table(table) + cluster.assert_table_does_not_exist(table) + + +def assert_correct_active_cluster_suffix(cluster: HBaseDockerClient, data_store_root: str): + logger.info(f"Verifying active cluster suffix file matches 'hbase.meta.table.suffix' " + f"in conf file for {cluster.name}") + active_cluster_file = f'{data_store_root}/data-store/hbase/active.cluster.suffix.id' + active_cluster_suffix = acs.ActiveClusterSuffix() + + # The active cluster suffix file may not get created right away + retries = 0 + while not os.path.exists(active_cluster_file): + if retries >= 5: + raise RuntimeError(f"Timed out waiting for active cluster file to exist: {active_cluster_file}") + logger.info(f"Waiting for active cluster file to exist: {active_cluster_file}") + time.sleep(1) + retries += 1 + + # Parse the active cluster suffix protobuf message file + with open(active_cluster_file, 'rb') as f: + data = f.read() + header = b'PBUF' + if data.startswith(header): + active_cluster_suffix.ParseFromString(data[len(header):]) + else: + active_cluster_suffix.ParseFromString(data) + actual_suffix = active_cluster_suffix.suffix + + # Assume the meta table suffix is blank if hbase.meta.table.suffix does not exist in HBase conf + expected_suffix = cluster.get_hbase_conf_property_value('hbase.meta.table.suffix') + if expected_suffix is None: + expected_suffix = '' + + # Verify the active cluster suffix file has the expected meta table suffix + assert actual_suffix == expected_suffix, (f"Expected {cluster.name} to have meta table suffix '{expected_suffix}', " + f"but got '{actual_suffix}' instead") + + +def reset_cluster_setup(active_cluster: HBaseDockerClient, replica_cluster: HBaseDockerClient, + new_containers: bool, docker_compose_file: str, data_store_root: str, sudo: bool = False): + """ + Resets the Read-Replica cluster setup where one cluster is the active cluster (read-write mode) and the other + cluster is the replica cluster (read-only mode). + """ + if new_containers: + HBaseDockerClient.stop_containers(docker_compose_file=docker_compose_file, data_store_root=data_store_root, + sudo=sudo) + + # If the containers are still running, then we need to run update_all_config in the HBase shell to update + # read-only mode on each cluster. Otherwise, we can just modify the conf files and the containers will be restarted + # in the desired read-only mode. + if new_containers: + run_update_all_config = False + else: + run_update_all_config = True + + # First, make sure both clusters are read-only to prevent an error due to trying to have two active clusters + active_cluster.enable_read_only_mode(run_update_all_config=run_update_all_config) + replica_cluster.enable_read_only_mode(run_update_all_config=run_update_all_config) + + # Now activate read-write mode on our active cluster + active_cluster.disable_read_only_mode(run_update_all_config=run_update_all_config) + + if new_containers: + HBaseDockerClient.start_or_restart_containers(docker_compose_file=docker_compose_file, + data_store_root=f'{data_store_root}') + HBaseDockerClient.wait_for_clusters_to_start([active_cluster, replica_cluster]) + + +def reset_docker_container_environment(new_containers: bool = False, + cluster1_name: str = "Cluster 1", + cluster2_name: str = "Cluster 2") -> tuple[HBaseDockerClient, HBaseDockerClient]: + if new_containers: + logger.info("Docker containers will be started/restarted with a fresh data store directory") + + cluster1, cluster2 = load_env_and_set_up_clients(cluster1_name, cluster2_name) + data_store_root = get_env("HBASE_DATA_STORE_ROOT") + docker_compose_file = get_env("DOCKER_COMPOSE_FILE") + + reset_cluster_setup(active_cluster=cluster1, replica_cluster=cluster2, + new_containers=new_containers, docker_compose_file=docker_compose_file, + data_store_root=data_store_root) + + return cluster1, cluster2 + + +def clean_up_tables(active_cluster: HBaseDockerClient, replica_cluster: HBaseDockerClient) -> None: + """ + Drops all tables on the active cluster and then runs 'refresh_meta' on the + read-replica cluster to remove those tables + """ + tables = active_cluster.list_tables() + if tables: + logger.info(f"Removing all existing tables on {active_cluster.name}: {tables}") + for table in tables: + active_cluster.disable_table(table) + active_cluster.drop_table(table) + logger.info(f"Running 'refresh_meta' and 'refresh_hfiles' on {replica_cluster.name} to sync it with " + f"{active_cluster.name}") + replica_cluster.refresh_meta() + replica_cluster.refresh_hfiles() + + +def swap_cluster_roles(new_active_cluster, new_replica_cluster, run_update_all_config=True): + logger.info(f"Making {new_active_cluster.name} the active cluster and " + f"{new_replica_cluster.name} the replica cluster") + new_replica_cluster.enable_read_only_mode(run_update_all_config=run_update_all_config) + new_active_cluster.disable_read_only_mode(run_update_all_config=run_update_all_config) + + +def create_table_on_active_cluster(active_cluster: HBaseDockerClient, column_family: str): + """Create a new table on the active cluster and assert it exists""" + tables = active_cluster.list_tables() + new_table = f't{len(tables)+1}' + active_cluster.create_table(new_table, column_family) + active_cluster.assert_table_exists(new_table) + return new_table + + +def add_data_to_each_table_on_active_cluster(active_cluster: HBaseDockerClient, tables: list, column_family: str): + """Add data to each table in the active cluster""" + for i, table in enumerate(tables[::-1], 1): + active_cluster.put(table, f'r{i}', column_family, i) + active_cluster.flush(table) + + +def refresh_replica_and_verify_tables(replica_cluster: HBaseDockerClient, new_table: str, tables: list): + """ + Refresh meta and HFiles on the replica cluster, and verify the new table + exists and each table has the correct number of rows + """ + replica_cluster.refresh_meta_and_hfiles() + replica_cluster.assert_table_exists(new_table) + for i, table in enumerate(tables[::-1], 1): + replica_cluster.assert_table_row_count(table, i) + + +def create_table_and_test_active_and_replica_clusters(active_cluster: HBaseDockerClient, + replica_cluster: HBaseDockerClient, + column_family: str): + """ + Creates a new table and iteratively adds data to each existing table, including the new one. + Also verifies expected behavior for the replica cluster, such as verifying the new table is not + on the replica before refreshing meta, and then verify new table and data existence after + refreshing meta and HFiles. + """ + new_table = create_table_on_active_cluster(active_cluster, column_family) + + # The new table should not exist on the replica cluster before refreshing meta + replica_cluster.assert_table_does_not_exist(new_table) + + tables = active_cluster.list_tables() + # HBase sorts table list by string: ['t1', 't10', 't2, ..., 't9'] + # We want the list sorted by creation time, so we're sorting on the integer: ['t1', 't2, ..., 't9', 't10'] + tables.sort(key=lambda x: int(x[1:])) + add_data_to_each_table_on_active_cluster(active_cluster, tables, column_family) + refresh_replica_and_verify_tables(replica_cluster, new_table, tables) diff --git a/dev-support/read-replica/python/test/__init__.py b/dev-support/read-replica/python/test/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/dev-support/read-replica/python/test/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/dev-support/read-replica/python/test/conftest.py b/dev-support/read-replica/python/test/conftest.py new file mode 100644 index 000000000000..6696211ccf4b --- /dev/null +++ b/dev-support/read-replica/python/test/conftest.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# conftest.py is a special pytest file. Fixtures and hooks defined here are +# automatically available to all tests in this directory and its subdirectories. + +import logging +import os +import shutil + +import pytest + +from python.src.logger_config import LOG_FORMAT + +DEFAULT_OUTPUT_DIR = os.path.join(os.path.dirname(__file__), '..', '..', 'output') + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + setattr(item, f"rep_{report.when}", report) + + +def _chmod_recursive(path, mode=0o777): + for dirpath, dirnames, filenames in os.walk(path): + os.chmod(dirpath, mode) + for filename in filenames: + os.chmod(os.path.join(dirpath, filename), mode) + + +def _clear_directory(path): + if not os.path.isdir(path): + return + for entry in os.listdir(path): + entry_path = os.path.join(path, entry) + if os.path.isdir(entry_path): + shutil.rmtree(entry_path) + else: + os.remove(entry_path) + + +@pytest.fixture(autouse=True) +def per_test_log_file(request): + output_dir = os.environ.get('OUTPUT_DIR', DEFAULT_OUTPUT_DIR) + test_name = request.node.name + execution_count = getattr(request.node, 'execution_count', 1) + + test_dir = os.path.join(output_dir, test_name) + run_dir = os.path.join(test_dir, f"run{execution_count}") + os.makedirs(run_dir, mode=0o777, exist_ok=True) + _chmod_recursive(test_dir) + + log_path = os.path.join(run_dir, f"{test_name}.run{execution_count}.log") + + active_logs_dir = os.environ.get('ACTIVE_CLUSTER_LOGS_DIR') + replica_logs_dir = os.environ.get('REPLICA_CLUSTER_LOGS_DIR') + + for logs_dir in (active_logs_dir, replica_logs_dir): + if logs_dir: + os.makedirs(logs_dir, mode=0o777, exist_ok=True) + os.chmod(logs_dir, 0o777) + _clear_directory(logs_dir) + + handler = logging.FileHandler(log_path, mode='w') + handler.setFormatter(logging.Formatter(LOG_FORMAT)) + handler.setLevel(logging.DEBUG) + + root_logger = logging.getLogger() + root_logger.addHandler(handler) + + yield + + rep_call = getattr(request.node, 'rep_call', None) + if rep_call is not None and rep_call.failed and rep_call.longreprtext: + logging.getLogger().error( + "TEST FAILED - pytest traceback:\n%s", rep_call.longreprtext + ) + + root_logger.removeHandler(handler) + handler.close() + os.chmod(log_path, 0o777) + + if active_logs_dir and os.path.isdir(active_logs_dir): + dest = os.path.join(run_dir, f"hbase-cluster1-run{execution_count}-logs") + shutil.copytree(active_logs_dir, dest, + dirs_exist_ok=True, + ignore=shutil.ignore_patterns('*.out', '*.out.*')) + _chmod_recursive(dest) + if replica_logs_dir and os.path.isdir(replica_logs_dir): + dest = os.path.join(run_dir, f"hbase-cluster2-run{execution_count}-logs") + shutil.copytree(replica_logs_dir, dest, + dirs_exist_ok=True, + ignore=shutil.ignore_patterns('*.out', '*.out.*')) + _chmod_recursive(dest) diff --git a/dev-support/read-replica/python/test/test_bulkloaded_data_and_region_splits.py b/dev-support/read-replica/python/test/test_bulkloaded_data_and_region_splits.py new file mode 100644 index 000000000000..ee48e47adf80 --- /dev/null +++ b/dev-support/read-replica/python/test/test_bulkloaded_data_and_region_splits.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +This script tests bulk-loading data with Read-Replica HBase clusters. +""" +import argparse +import time + +from python.src import get_logger, HBaseDockerClient +from python.src.environment_loader import get_env +from python.src.hbase_docker_client import DockerExecCommandError +from python.src.utils import (add_common_new_containers_arg, swap_cluster_roles, + log_script_start, log_script_end, reset_docker_container_environment) + +logger = get_logger(__name__) + + +class Bulkloader: + def __init__(self, bulkload_script: str): + self.bulkload_script = bulkload_script + + def bulkload_data(self, active_cluster: HBaseDockerClient, table_name: str, column_family: str = 'cf', + num_rows: int = 500, initial_row_num: int = 0): + logger.info(f"Running {self.bulkload_script} to bulkload {num_rows} rows into table '{table_name}' " + f"on {active_cluster.name}, starting with row {initial_row_num}") + active_cluster.run_docker_exec_command( + f"{self.bulkload_script} {table_name} {column_family} -n {num_rows} -i {initial_row_num}") + + +def assert_cannot_bulkload_data_onto_replica(bulkloader: Bulkloader, replica_cluster: HBaseDockerClient): + logger.info(f"Verifying data cannot be loaded onto {replica_cluster.name} because read-only mode is enabled") + try: + bulkloader.bulkload_data(replica_cluster, table_name='replica-blt1', column_family='cf') + raise RuntimeError(f"Expected bulkloading data onto replica cluster {replica_cluster.name} " + f"to result in an error") + except DockerExecCommandError as e: + expected_error_msg = ("org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: " + "Operation not allowed in Read-Only Mode") + assert expected_error_msg in str(e), (f"Expected exception to contain the following error message after " + f"attempting to bulkload data on to a replica cluster:\n" + f"{expected_error_msg}\n" + f"The actual exception was:\n{e}") + logger.info(f"Bulkload onto replica cluster {replica_cluster.name} failed as expected") + + +def assert_cannot_split_regions_on_replica(replica_cluster: HBaseDockerClient, table: str): + logger.info(f"Verifying regions cannot be split on {replica_cluster.name} because read-only mode is enabled") + try: + replica_cluster.split(table) + raise RuntimeError(f"Expected region split on replica cluster {replica_cluster.name} to result in an error") + except DockerExecCommandError as e: + expected_error_msg = ("org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: " + "Operation not allowed in Read-Only Mode") + assert expected_error_msg in str(e), (f"Expected exception to contain the following error message after " + f"attempting to split a region on a replica cluster:\n" + f"{expected_error_msg}\n" + f"The actual exception was:\n{e}") + logger.info(f"Region splitting on replica cluster {replica_cluster.name} failed as expected") + + +def run_test(new_containers: bool = False): + start_time = log_script_start(__file__, logger) + + cluster1, cluster2 = reset_docker_container_environment(new_containers=new_containers) + + table1 = 'blt1' + table2 = 'blt2' + table3 = 'blt3' + tables = [table1, table2, table3] + container_utils_dir = get_env("CONTAINER_UTILS_DIR") + + bulkloader = Bulkloader(bulkload_script=f"{container_utils_dir}/bulkload.sh") + + logger.info(f"The active cluster is {cluster1.name} and the replica cluster is {cluster2.name}") + + assert_cannot_bulkload_data_onto_replica(bulkloader, replica_cluster=cluster2) + + # Bulkload data to active cluster and verify the data is there + logger.info(f"Bulkloading data to '{table1}' on the active cluster and verifying the data is there") + bulkloader.bulkload_data(active_cluster=cluster1, table_name=table1) + cluster1.assert_table_exists(table1) + cluster1.assert_table_row_count(table1, expected_row_count=500) + + # Replica cluster should not see bulkloaded data until meta and HFiles have been refreshed + logger.info(f"The replica cluster {cluster2.name} should not see bulkloaded data until " + f"meta and HFiles have been refreshed") + cluster2.assert_table_does_not_exist(table1) + cluster2.refresh_meta_and_hfiles() + cluster2.assert_table_exists(table1) + cluster2.assert_table_row_count(table1, expected_row_count=500) + + # Cluster 1 is now a replica and Cluster 2 is now the active cluster + swap_cluster_roles(new_active_cluster=cluster2, new_replica_cluster=cluster1) + + # Bulkload more data into the existing table on Cluster 2 + logger.info(f"Bulkloading more data into the existing table on {cluster2.name}") + bulkloader.bulkload_data(active_cluster=cluster2, table_name=table1, num_rows=300, initial_row_num=500) + cluster2.assert_table_row_count(table1, expected_row_count=800) + + # Cluster 1 should not see the newly bulkloaded data until its meta and HFiles have been refreshed + logger.info(f"The replica cluster {cluster1.name} should not see bulkloaded data until " + f"meta and HFiles have been refreshed") + cluster1.assert_table_row_count(table1, expected_row_count=500) + cluster1.refresh_meta_and_hfiles() + cluster1.assert_table_row_count(table1, expected_row_count=800) + + # Bulkload data into a new table on Cluster 2 + logger.info(f"Bulkloading data into new table '{table2}' on active cluster {cluster2.name}") + bulkloader.bulkload_data(active_cluster=cluster2, table_name=table2, num_rows=600) + cluster2.assert_table_exists(table2) + cluster2.assert_table_row_count(table2, expected_row_count=600) + + # Cluster 1 should not see this new table until after refreshing meta and HFiles + logger.info(f"The replica cluster {cluster1.name} should not see '{table2}' until after refreshing meta and HFiles") + cluster1.assert_table_does_not_exist(table2) + cluster1.refresh_meta_and_hfiles() + cluster1.assert_table_exists(table2) + cluster1.assert_table_row_count(table2, expected_row_count=600) + cluster1.assert_table_row_count(table1, expected_row_count=800) + + # Cluster 1 is back to being the active cluster and Cluster 2 is once again the replica cluster + swap_cluster_roles(new_active_cluster=cluster1, new_replica_cluster=cluster2) + + # Bulkload data onto both existing tables, and a new third table + logger.info(f"Bulkloading data onto '{table1}' and '{table2}', as well as a new table '{table3}'") + bulkloader.bulkload_data(active_cluster=cluster1, table_name=table1, num_rows=400, initial_row_num=800) + bulkloader.bulkload_data(active_cluster=cluster1, table_name=table2, num_rows=600, initial_row_num=600) + bulkloader.bulkload_data(active_cluster=cluster1, table_name=table3, num_rows=1200) + for table in tables: + cluster1.assert_table_row_count(table, expected_row_count=1200) + + # Cluster 2 should see the old row counts for the existing tables. It won't see the new table + # or the updated row counts until after its meta and HFiles have been refreshed. + logger.info(f"The replica cluster {cluster2.name} should not see '{table3}' or updated values for " + f"'{table1}' and '{table2}' until after refreshing meta and HFiles") + cluster2.assert_table_row_count(table1, expected_row_count=800) + cluster2.assert_table_row_count(table2, expected_row_count=600) + cluster2.assert_table_does_not_exist(table3) + cluster2.refresh_meta_and_hfiles() + for table in tables: + cluster2.assert_table_row_count(table, expected_row_count=1200) + + # Cluster 2 is now the active cluster and Cluster 1 is the replica cluster + swap_cluster_roles(new_active_cluster=cluster2, new_replica_cluster=cluster1) + + # Split regions on two tables on the active cluster + for table in [table1, table2]: + logger.info(f"Splitting table '{table}' on {cluster2.name}") + cluster2.flush_and_split(table) + cluster2.major_compact_and_wait(table) + cluster2.catalogjanitor_run() + time.sleep(5) + cluster2.assert_region_count_for_table(table, expected_region_count=2) + + # Bulkload more rows into each table on Cluster 2 and into a new table + table4 = 'blt4' + logger.info(f"Bulkloading data onto '{table1}', '{table2}', and '{table3}', as well as a new table '{table4}'") + for table in tables: + bulkloader.bulkload_data(active_cluster=cluster2, table_name=table, num_rows=1200, initial_row_num=1200) + bulkloader.bulkload_data(active_cluster=cluster2, table_name=table4, num_rows=2400) + tables.append(table4) + for table in tables: + cluster2.assert_table_row_count(table, expected_row_count=2400) + + # The replica cluster should see the old row counts for the existing tables. It won't see the new table + # or the updated row counts until after its meta and HFiles have been refreshed. + logger.info(f"The replica cluster {cluster1.name} should not see '{table4}' or row counts/region splits for " + f"'{table1}', '{table2}', and '{table3}' until after refreshing meta and HFiles") + for table in tables[:-1]: + cluster1.assert_table_row_count(table, expected_row_count=1200) + cluster1.assert_table_does_not_exist(table4) + + # The replica cluster should not see any region splits until after refreshing meta and HFiles + for table in tables[:-1]: + cluster1.assert_region_count_for_table(table, expected_region_count=1) + + # The replica cluster will now see updated row counts and region splits + cluster1.refresh_meta_and_hfiles() + logger.info(f"Replica cluster {cluster1.name} should now see updated row counts and region splits") + for table, num_regions in zip(tables, [2, 2, 1, 1]): + cluster1.assert_table_row_count(table, expected_row_count=2400) + cluster1.assert_region_count_for_table(table, expected_region_count=num_regions) + + # Make Cluster 1 the active cluster and Cluster 2 the replica cluster + swap_cluster_roles(new_active_cluster=cluster1, new_replica_cluster=cluster2) + + assert_cannot_split_regions_on_replica(replica_cluster=cluster2, table=table3) + + # Split regions on the active cluster. The replica cluster won't see the updated region count until meta and HFiles + # have been refreshed + for table, num_regions in zip(tables, [4, 4, 2, 2]): + cluster1.flush_and_split(table) + cluster1.major_compact(table) + cluster1.catalogjanitor_run() + time.sleep(5) + cluster1.assert_region_count_for_table(table, num_regions) + + # Replica cluster still has old region count + cluster2.assert_region_count_for_table(table, num_regions/2) + + # Update the replica cluster and verify new region count + cluster2.refresh_meta_and_hfiles() + cluster2.assert_region_count_for_table(table, num_regions) + + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser = add_common_new_containers_arg(parser) + parsed_args = parser.parse_args(args) + + run_test( + new_containers=parsed_args.new_containers + ) + + +if __name__ == '__main__': + main() diff --git a/dev-support/read-replica/python/test/test_cannot_promote_second_active_cluster.py b/dev-support/read-replica/python/test/test_cannot_promote_second_active_cluster.py new file mode 100644 index 000000000000..1450ba575353 --- /dev/null +++ b/dev-support/read-replica/python/test/test_cannot_promote_second_active_cluster.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Verifies a cluster cannot be promoted to an active cluster when another active cluster already exists. + +The test starts with two Read-Replica HBase clusters, where one cluster is the active cluster and the other cluster is +the replica cluster. The test tries to promote the replica cluster to a second active cluster and expects an error to +occur. It then verifies this "second active cluster" is still in read-only mode and that data can still be added to the +actual active cluster. + +This test script verifies the fix for: + +HBASE-30220: A replica cluster can have read-only mode disabled even when another active cluster already exists +https://issues.apache.org/jira/browse/HBASE-30220 + +Before implementing the fix for HBASE-30220, a cluster could be promoted to from a replica cluster to an active cluster +even when another active cluster already existed. +""" +import argparse +from time import sleep + +from python.src.environment_loader import get_env +from python.src.hbase_docker_client import HBaseDockerClient, DockerExecCommandError +from python.src.logger_config import get_logger +from python.src.utils import (assert_crud_operations_work_on_active_cluster, assert_correct_active_cluster_suffix, + add_common_new_containers_arg, clean_up_tables, + create_table_and_test_active_and_replica_clusters, + log_script_start, log_script_end, reset_docker_container_environment) + +logger = get_logger(__name__) + +COLUMN_FAMILY = "cf" +EXPECTED_ERROR_MSG = ("ReadOnlyTransitionException: Cannot disable read-only mode because another active cluster " + "already exists on this storage location. The read-only coprocessors have not been removed.") + + +def assert_error_when_trying_to_have_second_active_cluster(replica_cluster: HBaseDockerClient, expected_error: str): + try: + replica_cluster.disable_read_only_mode() + raise RuntimeError(f"Expected an DockerExecCommandError with the following error message:\n\n" + f"{expected_error}") + except DockerExecCommandError as e: + assert expected_error in str(e), (f"Expected DockerExecCommandError to contain the following message:\n\n" + f"{str(expected_error)}\n\n" + f"Got the following message instead:\n\n{str(e)}") + logger.info(f"Successfully prevented {replica_cluster.name} from becoming a second active cluster") + + +def run_test_iteration(active_cluster: HBaseDockerClient, replica_cluster: HBaseDockerClient, data_root: str): + create_table_and_test_active_and_replica_clusters(active_cluster, replica_cluster, column_family='cf') + assert_error_when_trying_to_have_second_active_cluster(replica_cluster, EXPECTED_ERROR_MSG) + + # Cluster should still be in read-only mode after failed transition from read-only to read-write mode + replica_cluster.assert_read_only_error_occurs('create', 'test_table', COLUMN_FAMILY) + + assert_crud_operations_work_on_active_cluster(active_cluster) + + # Demote active cluster to replica and promote original replica to be the new active cluster + active_cluster.enable_read_only_mode() + replica_cluster.disable_read_only_mode() + active_cluster = replica_cluster + + # Wait for active cluster file to be updated and verify its contents + sleep(3) + assert_correct_active_cluster_suffix(active_cluster, data_root) + + +def run_test(new_containers: bool = False): + start_time = log_script_start(__file__, logger) + + cluster1, cluster2 = reset_docker_container_environment(new_containers=new_containers) + data_store_root = get_env("HBASE_DATA_STORE_ROOT") + + assert_correct_active_cluster_suffix(cluster1, data_store_root) + clean_up_tables(active_cluster=cluster1, replica_cluster=cluster2) + + test_iterations = 5 + for i in range(1, test_iterations+1): + logger.info(f"---------- Iteration {i} ----------") + if i % 2 == 1: + run_test_iteration(active_cluster=cluster1, replica_cluster=cluster2, data_root=data_store_root) + else: + run_test_iteration(active_cluster=cluster2, replica_cluster=cluster1, data_root=data_store_root) + logger.info(f"Finished iteration {i} of {test_iterations}") + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser = add_common_new_containers_arg(parser) + parsed_args = parser.parse_args(args) + + run_test( + new_containers=parsed_args.new_containers + ) + + +if __name__ == '__main__': + main() diff --git a/dev-support/read-replica/python/test/test_create_drop_behavior.py b/dev-support/read-replica/python/test/test_create_drop_behavior.py new file mode 100644 index 000000000000..e1b03f2cf7ba --- /dev/null +++ b/dev-support/read-replica/python/test/test_create_drop_behavior.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Tests table creation behavior for read-replica clusters. It creates a table on the active +cluster, and then runs refresh_meta on the replica cluster and verifies the table's existence. +It does a similar process when dropping the table on the active cluster. It also verifies +tables cannot be created/dropped on the replica cluster. +""" +import argparse + +from python.src.logger_config import get_logger +from python.src.utils import (add_common_drop_existing_tables_arg, add_common_new_containers_arg, clean_up_tables, + log_script_start, log_script_end, reset_docker_container_environment) + +logger = get_logger(__name__) + + +def test_table_creation_behavior(active_cluster, replica_cluster, table_name, column_family): + # We should not be able to create a new table on the read-replica cluster + replica_cluster.assert_read_only_error_occurs('create', table_name, column_family) + + active_cluster.create_table(table_name, column_family) + + # Read-Replica cluster should not see the newly created table yet + logger.info(f"Verifying {active_cluster.name} now has table '{table_name}', " + f"while {replica_cluster.name} cluster does not") + active_cluster.assert_table_exists(table_name) + replica_cluster.assert_table_does_not_exist(table_name) + + # Read-Replica cluster should now see the newly created table + replica_cluster.refresh_meta() + logger.info(f"Verifying {replica_cluster.name} has table '{table_name}' after refreshing meta") + replica_cluster.assert_table_exists(table_name) + active_cluster.assert_table_exists(table_name) + + # Cannot drop the table on the Read-Replica cluster. A WriteAttemptedOnReadOnlyClusterException should occur + replica_cluster.disable_table(table_name) + replica_cluster.assert_read_only_error_occurs('drop', table_name, column_family) + # The table should still exist on the read-replica cluster since drops are not allowed + replica_cluster.assert_table_exists(table_name) + + # Drop the table on the active cluster + active_cluster.disable_table(table_name) + active_cluster.drop_table(table_name) + + # The read-replica cluster should still have the table that was dropped on the active + # cluster since 'refresh_meta' has not been run yet. + logger.info(f"Verifying {replica_cluster.name} still has table '{table_name}'") + active_cluster.assert_table_does_not_exist(table_name) + replica_cluster.assert_table_exists(table_name) + + # The read-replica cluster no longer has the dropped table after running 'refresh_meta'. + logger.info(f"Verifying {replica_cluster.name} no longer has table '{table_name}' after " + f"refreshing meta") + replica_cluster.refresh_meta() + replica_cluster.assert_table_does_not_exist(table_name) + + +def run_test(drop_existing_tables: bool = False, new_containers: bool = False): + start_time = log_script_start(__file__, logger) + + active_cluster, replica_cluster = reset_docker_container_environment( + new_containers=new_containers, + cluster1_name="Active Cluster", + cluster2_name="Replica Cluster" + ) + + table_name = "t1" + column_family = "cf" + if drop_existing_tables: + clean_up_tables(active_cluster, replica_cluster) + + test_table_creation_behavior(active_cluster, replica_cluster, table_name, column_family) + + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser = add_common_drop_existing_tables_arg(parser) + parser = add_common_new_containers_arg(parser) + parsed_args = parser.parse_args(args) + + run_test( + drop_existing_tables=parsed_args.drop_existing_tables, + new_containers=parsed_args.new_containers + ) + + +if __name__ == "__main__": + main() diff --git a/dev-support/read-replica/python/test/test_dual_active_cluster_startup.py b/dev-support/read-replica/python/test/test_dual_active_cluster_startup.py new file mode 100644 index 000000000000..1ed5ef8b61e9 --- /dev/null +++ b/dev-support/read-replica/python/test/test_dual_active_cluster_startup.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Verifies that two clusters cannot both start with read-only mode disabled (both as active clusters) +on the same shared data store. One cluster must fail to start, with the HMaster process not +running, and an error logged to the master log. + +Usage: python3 ./python/scripts/test_dual_active_cluster_startup.py +""" +import argparse +import os +import time + +from python.src.environment_loader import get_env +from python.src.hbase_docker_client import HBaseDockerClient, DockerExecCommandError, HBaseInitializationError +from python.src.logger_config import get_logger +from python.src.utils import load_env_and_set_up_clients, log_script_start, log_script_end + +logger = get_logger(__name__) + +STARTUP_WAIT_SECONDS = 60 +EXPECTED_ERROR_MSG = "Another cluster is running in active (read-write) mode on this storage location" +HMASTER_STARTUP_MARKER = "STARTING service HMaster" +CLUSTER1_SERVICE_NAME = "hbase" +CLUSTER2_SERVICE_NAME = "hbase2" + + +def is_process_running(cluster: HBaseDockerClient, process_name: str) -> bool: + output = cluster.run_docker_exec_command("jps") + return process_name in output + + +def check_cluster_processes(cluster: HBaseDockerClient) -> bool: + hmaster_running = is_process_running(cluster, "HMaster") + logger.info(f" {cluster.name}: HMaster={'running' if hmaster_running else 'down'}") + return hmaster_running + + +def assert_error_in_master_log(cluster: HBaseDockerClient): + logger.info(f"Checking {cluster.name} master log for expected error message") + awk_cmd = ( + f"awk '/{HMASTER_STARTUP_MARKER}/{{found=0}} " + f"index($0, \"{EXPECTED_ERROR_MSG}\"){{found=1}} " + f"END{{exit !found}}' /opt/hbase/logs/hbase-*-master-*.log" + ) + try: + cluster.run_docker_exec_command(awk_cmd) + except DockerExecCommandError: + log_tail = cluster.run_docker_exec_command( + "tail -100 /opt/hbase/logs/hbase-*-master-*.log || true" + ) + raise AssertionError( + f"Expected {cluster.name}'s master log to contain " + f"(after the most recent '{HMASTER_STARTUP_MARKER}'):\n" + f" '{EXPECTED_ERROR_MSG}'\n" + f"but it was not found.\nLog tail:\n{log_tail}" + ) + logger.info(f" [PASS] Found expected error message in {cluster.name}'s master log") + + +def wait_for_active_cluster_file(data_store_root: str, timeout_seconds: int = 30) -> None: + file_path = f"{data_store_root}/data-store/hbase/active.cluster.suffix.id" + logger.info(f"Waiting for active cluster suffix id file: {file_path}") + start = time.time() + while not os.path.exists(file_path): + if time.time() - start > timeout_seconds: + raise HBaseInitializationError(f"Timed out after {timeout_seconds}s waiting for: {file_path}") + time.sleep(1) + logger.info(f"Active cluster suffix id file detected: {file_path}") + + +def run_test(clean_up_containers: bool = False): + start_time = log_script_start(__file__, logger) + + cluster1, cluster2 = load_env_and_set_up_clients() + data_store_root = get_env("HBASE_DATA_STORE_ROOT") + docker_compose_file = get_env("DOCKER_COMPOSE_FILE") + + test_iterations = 4 + for i in range(1, test_iterations+1): + logger.info(f"---------- Iteration {i} ----------") + + HBaseDockerClient.stop_containers(docker_compose_file=docker_compose_file, data_store_root=data_store_root) + + # Make both clusters an active cluster (read-only disabled) + cluster1.disable_read_only_mode(run_update_all_config=False) + cluster2.disable_read_only_mode(run_update_all_config=False) + + HBaseDockerClient.set_up_data_store_dir(data_store_root) + + # Alternate which cluster starts first + if i % 2 == 1: + first_cluster, second_cluster = cluster1, cluster2 + first_service, second_service = CLUSTER1_SERVICE_NAME, CLUSTER2_SERVICE_NAME + else: + first_cluster, second_cluster = cluster2, cluster1 + first_service, second_service = CLUSTER2_SERVICE_NAME, CLUSTER1_SERVICE_NAME + + # Start the first cluster and wait for it to claim the active role + logger.info(f"Starting {first_cluster.name} first (service: {first_service})") + HBaseDockerClient.start_service(first_service, docker_compose_file=docker_compose_file) + wait_for_active_cluster_file(data_store_root) + + # Start the second cluster - it should detect the existing active cluster and fail + logger.info(f"Starting {second_cluster.name} second (service: {second_service})") + HBaseDockerClient.start_service(second_service, docker_compose_file=docker_compose_file) + + logger.info(f"Waiting {STARTUP_WAIT_SECONDS}s for {second_cluster.name} to attempt startup...") + time.sleep(STARTUP_WAIT_SECONDS) + + logger.info("Checking HBase processes on both clusters") + first_running = check_cluster_processes(first_cluster) + second_running = check_cluster_processes(second_cluster) + + if not first_running: + raise HBaseInitializationError( + f"Expected {first_cluster.name} (started first) to be running, but HMaster is down" + ) + + assert not second_running, ( + f"Expected {second_cluster.name} (started second) to have failed, " + f"but HMaster is still running" + ) + + logger.info(f"[PASS] {first_cluster.name} is running as the active cluster") + logger.info(f"[PASS] {second_cluster.name} failed to start (HMaster is down)") + + assert_error_in_master_log(second_cluster) + + logger.info(f"Finished iteration {i} of {test_iterations}") + + logger.info("=" * 70) + logger.info("TEST PASSED: All dual active cluster startups were correctly rejected") + logger.info("=" * 70) + + if clean_up_containers: + logger.info("Stopping Docker containers and reverting test environment to having " + "one active cluster and one replica cluster") + HBaseDockerClient.stop_containers(docker_compose_file=docker_compose_file, data_store_root=data_store_root) + cluster1.disable_read_only_mode(run_update_all_config=False) + cluster2.enable_read_only_mode(run_update_all_config=False) + + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--clean-up-containers', action='store_true', + help='Stop Docker containers and revert cluster configurations to one ' + 'active cluster and one replica cluster after the test finishes') + parsed_args = parser.parse_args(args) + + run_test( + clean_up_containers=parsed_args.clean_up_containers + ) + + +if __name__ == '__main__': + main() diff --git a/dev-support/read-replica/python/test/test_put_get_delete_behavior.py b/dev-support/read-replica/python/test/test_put_get_delete_behavior.py new file mode 100644 index 000000000000..676b197de767 --- /dev/null +++ b/dev-support/read-replica/python/test/test_put_get_delete_behavior.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Verifies data can be added to/deleted from the active cluster and the read-replica cluster +does not see this data until refresh_hfiles has been run. Also verifies put and delete +operations on the read-replica cluster result in an error. +""" +import argparse + +from python.src.hbase_docker_client import HBaseDockerClient, DockerExecCommandError, DockerExecCommandTimeoutError +from python.src.logger_config import get_logger +from python.src.utils import (add_common_drop_existing_tables_arg, add_common_new_containers_arg, clean_up_tables, + log_script_start, log_script_end, reset_docker_container_environment) + +logger = get_logger(__name__) + + +def assert_cannot_flush_table_on_replica(replica_cluster: HBaseDockerClient, table: str, timeout: int = 60): + logger.info(f"Verifying table '{table}' cannot be flushed on {replica_cluster.name} " + f"because read-only mode is enabled") + try: + replica_cluster.flush(table, timeout=timeout) + raise RuntimeError(f"Expected flush on replica cluster {replica_cluster.name} to result in an error") + except DockerExecCommandTimeoutError: + raise RuntimeError( + f"TIMEOUT: flush on replica cluster '{replica_cluster.name}' did not complete within " + f"{timeout} seconds. This may indicate HBASE-30301 has not been fixed on this cluster." + ) + except DockerExecCommandError as e: + expected_error_msg = ("org.apache.hadoop.hbase.WriteAttemptedOnReadOnlyClusterException: " + "Operation not allowed in Read-Only Mode") + assert expected_error_msg in str(e), (f"Expected exception to contain the following error message after " + f"attempting a flush on replica cluster {replica_cluster.name}:\n" + f"{expected_error_msg}\n" + f"The actual exception was:\n{e}") + logger.info(f"Flush for table '{table}' on replica cluster {replica_cluster.name} failed as expected") + + +def test_put_delete_behavior(active_cluster, replica_cluster, table_name, column): + # Add data to the table on the active cluster + logger.info(f"Adding data to '{table_name}' on {active_cluster.name} and verifying it exists") + active_cluster.put(table_name, "row1", column, "value1") + active_cluster.assert_table_row_count(table_name, 1) + active_cluster.assert_get_output(table_name, "row1", column, "value1") + + # Verify the read-replica cluster does not see this new data + logger.info(f"Verifying '{table_name}' on {replica_cluster.name} still has 0 rows") + replica_cluster.assert_table_row_count(table_name, 0) + + # Flush the table's data on the active cluster + logger.info(f"Flushing '{table_name}' on {active_cluster.name} and refreshing meta and " + f"HFiles on {replica_cluster.name}") + active_cluster.flush(table_name) + + # Refresh meta and HFiles, and verify the read-replica cluster now sees the data + logger.info(f"Refreshing meta and HFiles on {replica_cluster.name}") + replica_cluster.refresh_meta_and_hfiles() + logger.info(f"Verifying '{table_name}' on {replica_cluster.name} has data after refreshing HFiles") + replica_cluster.assert_table_row_count(table_name, 1) + replica_cluster.assert_get_output(table_name, "row1", column, "value1") + + # Verify replica clusters cannot flush tables + assert_cannot_flush_table_on_replica(replica_cluster, table_name) + + # Verify data cannot be added to the table on the read-replica cluster + logger.info(f"Verifying data cannot be added to '{table_name}' on {replica_cluster.name}") + replica_cluster.assert_read_only_error_occurs('put', table_name, column, 'row2', 'value2') + + # Verify data cannot be deleted from the table on the read-replica cluster + logger.info(f"Verifying data cannot be deleted from '{table_name}' on {replica_cluster.name}") + replica_cluster.assert_read_only_error_occurs('delete', table_name, column, 'row2') + + # Delete data from the active cluster + logger.info(f"Deleting row from '{table_name}' on {active_cluster.name} " + f"and verifying it is gone") + active_cluster.delete(table_name, "row1", column) + active_cluster.flush(table_name) + active_cluster.assert_table_row_count(table_name, 0) + + # Verify deleted data still exists on the read-replica cluster + logger.info(f"Verifying deleted row still exists on {replica_cluster.name}") + replica_cluster.assert_table_row_count(table_name, 1) + replica_cluster.assert_get_output(table_name, "row1", column, "value1") + + # Verify the read-replica cluster no longer has the data after refreshing HFiles + replica_cluster.refresh_hfiles() + replica_cluster.assert_table_row_count(table_name, 0) + + +def run_test(drop_existing_tables: bool = False, new_containers: bool = False): + start_time = log_script_start(__file__, logger) + + active_cluster, replica_cluster = reset_docker_container_environment( + new_containers=new_containers, + cluster1_name="Active Cluster", + cluster2_name="Replica Cluster" + ) + + table_name = "t1" + column_family = "cf" + column = f"{column_family}:c1" + + if drop_existing_tables: + clean_up_tables(active_cluster, replica_cluster) + + # Create a table on the active cluster and have it appear on the read-replica cluster + active_cluster.create_table(table_name, column_family) + replica_cluster.refresh_meta() + + test_put_delete_behavior(active_cluster, replica_cluster, table_name, column) + + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser = add_common_drop_existing_tables_arg(parser) + parser = add_common_new_containers_arg(parser) + parsed_args = parser.parse_args(args) + + run_test( + drop_existing_tables=parsed_args.drop_existing_tables, + new_containers=parsed_args.new_containers + ) + + +if __name__ == '__main__': + main() diff --git a/dev-support/read-replica/python/test/test_read_only_flag_flipping.py b/dev-support/read-replica/python/test/test_read_only_flag_flipping.py new file mode 100644 index 000000000000..87f20ffc43dc --- /dev/null +++ b/dev-support/read-replica/python/test/test_read_only_flag_flipping.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +This test starts with two Read-Replica HBase clusters, where one cluster is the active cluster and the other cluster is +the replica cluster. The test creates a table on the active cluster, adds data to the cluster, and verifies this data +is consistent on the replica cluster after refreshing the meta HFiles. It also verifies write operations cannot be +performed on the replica cluster. Then, the two clusters swap roles, where the active cluster becomes a replica and the +former replica becomes the new active cluster. The previous steps then repeat in an iterative fashion. + +This test script verifies behavior for multiple bug fixes: + +1. HBASE-30090: Table on replica cluster not refreshing after flipping read-only flag twice + https://issues.apache.org/jira/browse/HBASE-30090 + + Before implementing this fix, an existing table on a read-replica cluster was not getting updated after making that + cluster the active cluster and then making it read-only again. + +2. HBASE-30180: Can still add data to read-only region after flipping read-only flag multiple times + https://issues.apache.org/jira/browse/HBASE-30180 + + Before implementing this fix, this cluster setup and series of steps would eventually get to a scenario where data + could be added to a table on cluster with read-only mode disabled. +""" +import argparse + +from python.src.environment_loader import get_env +from python.src.hbase_docker_client import HBaseDockerClient +from python.src.logger_config import get_logger +from python.src.utils import (assert_correct_active_cluster_suffix, add_common_new_containers_arg, + clean_up_tables, create_table_and_test_active_and_replica_clusters, + log_script_start, log_script_end, reset_docker_container_environment) + +COLUMN_FAMILY = "cf" +logger = get_logger(__name__) + + +def flip_read_only_flag(new_active_cluster: HBaseDockerClient, + new_replica_cluster: HBaseDockerClient): + # Make cluster read-only and verify it cannot create a table or put data + new_replica_cluster.enable_read_only_mode() + new_replica_cluster.assert_read_only_error_occurs('create', 'testTable', COLUMN_FAMILY) + new_replica_cluster.assert_read_only_error_occurs( + 'put', 't1', COLUMN_FAMILY, row='r2', data='2') + + # Make cluster active + new_active_cluster.disable_read_only_mode() + + +def create_table_and_test_clusters_then_flip_read_only_flag(cluster1, cluster2, data_store_root): + create_table_and_test_active_and_replica_clusters(active_cluster=cluster1, replica_cluster=cluster2, + column_family='cf') + flip_read_only_flag(new_active_cluster=cluster2, new_replica_cluster=cluster1) + assert_correct_active_cluster_suffix(cluster2, data_store_root) + + +def run_test(new_containers: bool = False): + start_time = log_script_start(__file__, logger) + + cluster1, cluster2 = reset_docker_container_environment(new_containers=new_containers) + data_store_root = get_env("HBASE_DATA_STORE_ROOT") + + test_iterations = 1 + read_only_flag_flips_per_iteration = 15 + for i in range(1, test_iterations + 1): + logger.info(f"---------- Iteration {i} ----------") + if i > 1: + logger.info(f"Ensuring clusters are in proper modes. " + f"Making both clusters a replica, and then making {cluster1.name} the active cluster") + cluster1.enable_read_only_mode() + cluster2.enable_read_only_mode() + cluster1.disable_read_only_mode() + + # Create table on active cluster + clean_up_tables(cluster1, cluster2) + + # One iteration flips the read-only flag on each cluster and then flips it back. + flip_num = 1 + while flip_num <= read_only_flag_flips_per_iteration: + logger.info(f"*** Testing read-only flag flip number {flip_num} ***") + if flip_num % 2 == 1: + # Cluster 1 is active and Cluster 2 is replica + create_table_and_test_clusters_then_flip_read_only_flag(cluster1, cluster2, data_store_root) + else: + # Cluster 2 is active and Cluster 1 is replica + create_table_and_test_clusters_then_flip_read_only_flag(cluster2, cluster1, data_store_root) + logger.info(f"Finished read-only flag flip {flip_num} of {read_only_flag_flips_per_iteration}") + flip_num += 1 + logger.info(f"Finished iteration {i} of {test_iterations}") + + log_script_end(__file__, logger, start_time) + + +def main(args=None): + parser = argparse.ArgumentParser() + parser = add_common_new_containers_arg(parser) + parsed_args = parser.parse_args(args) + + run_test( + new_containers=parsed_args.new_containers + ) + + +if __name__ == '__main__': + main() diff --git a/dev-support/read-replica/python/test/test_read_replica_feature.py b/dev-support/read-replica/python/test/test_read_replica_feature.py new file mode 100644 index 000000000000..e07494414549 --- /dev/null +++ b/dev-support/read-replica/python/test/test_read_replica_feature.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +import python.test.test_dual_active_cluster_startup as test_dual_active_cluster_startup +import python.test.test_create_drop_behavior as test_create_drop_behavior +import python.test.test_put_get_delete_behavior as test_put_get_delete_behavior +import python.test.test_read_only_flag_flipping as test_read_only_flag_flipping +import python.test.test_cannot_promote_second_active_cluster as test_cannot_promote_second_active_cluster +import python.test.test_bulkloaded_data_and_region_splits as test_bulkloaded_data_and_region_splits + + +@pytest.mark.flaky(reruns=2, reruns_delay=2) +class TestReadReplica: + def test_dual_active_cluster_startup(self): + test_dual_active_cluster_startup.run_test(clean_up_containers=True) + + def test_create_drop_behavior(self): + test_create_drop_behavior.run_test(drop_existing_tables=True, new_containers=True) + + def test_put_get_delete_behavior(self): + test_put_get_delete_behavior.run_test(drop_existing_tables=True, new_containers=True) + + def test_read_only_flag_flipping(self): + test_read_only_flag_flipping.run_test(new_containers=True) + + def test_cannot_promote_second_active_cluster(self): + test_cannot_promote_second_active_cluster.run_test(new_containers=True) + + def test_bulkloaded_data_and_region_splits(self): + test_bulkloaded_data_and_region_splits.run_test(new_containers=True) diff --git a/dev-support/read-replica/requirements.txt b/dev-support/read-replica/requirements.txt new file mode 100644 index 000000000000..8d5df3c0e474 --- /dev/null +++ b/dev-support/read-replica/requirements.txt @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +certifi==2026.2.25 +charset-normalizer==3.4.7 +docker==7.2.0 +grpcio==1.80.0 +grpcio-tools==1.80.0 +idna==3.11 +iniconfig==2.3.0 +packaging==26.3 +pluggy==1.6.0 +protobuf==6.33.6 +Pygments==2.21.0 +pytest==9.1.1 +pytest-rerunfailures==16.6.1 +python-dotenv==1.2.2 +requests==2.33.1 +setuptools==82.0.1 +typing_extensions==4.15.0 +urllib3==2.6.3 diff --git a/dev-support/read-replica/run_read_replica_integration_tests.sh b/dev-support/read-replica/run_read_replica_integration_tests.sh new file mode 100755 index 000000000000..4c1f290f2df7 --- /dev/null +++ b/dev-support/read-replica/run_read_replica_integration_tests.sh @@ -0,0 +1,360 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# run_read_replica_integration_tests.sh +# +# Inner test runner executed INSIDE the Docker container built from +# hbase/dev-support/docker/Dockerfile. In a typical test run, this +# script is invoked by hbase_nightly_read_replica_test.sh. It can be +# run on its own as well as long as it is done within the container +# mentioned above. +# +# What it does: +# 1. Rsyncs a trimmed HBase tree into ${REPLICA_DIR}/hbase for Docker build context +# 2. Copies and compiles the protobuf definitions needed by the Python tests +# 3. Creates a Python virtual environment and installs dependencies +# 4. Builds a Docker image for the active and replica clusters in a read-replica +# setup +# 5. Runs the pytest read-replica integration suite using these clusters +# +# For usage information, run: ./run_read_replica_integration_tests.sh --help +set -e + +format_duration_hms() { + local total_sec=$1 + printf '%d:%02d:%02d' $((total_sec / 3600)) $(((total_sec % 3600) / 60)) $((total_sec % 60)) +} + +print_timing_summary() { + if [ "${TIMING_SUMMARY_PRINTED}" = true ]; then + return + fi + TIMING_SUMMARY_PRINTED=true + + local total_sec=$(( SECONDS - OVERALL_START_SEC + ${DEV_SUPPORT_IMAGE_BUILD_SEC:-0} )) + local dev_image_sec="${DEV_SUPPORT_IMAGE_BUILD_SEC}" + local rsync_sec="${RSYNC_SEC:--}" + local mvn_clean_sec="${MVN_CLEAN_SEC:--}" + local docker_build_sec="${DOCKER_BUILD_SEC:--}" + local build_images_sec="${BUILD_IMAGES_SEC:--}" + local pytest_sec="${PYTEST_SEC:--}" + + echo "" + echo "=== Read-replica run timing summary ===" + if [ -n "${dev_image_sec}" ]; then + printf " 1. Test-env image build (host): %6ss (%s)\n" \ + "${dev_image_sec}" "$(format_duration_hms "${dev_image_sec}")" + else + echo " 1. Test-env image build (host): N/A (dev mode)" + fi + if [ "${rsync_sec}" != "-" ]; then + printf " 2. Rsync source staging (read-replica/hbase): %6ss (%s)\n" \ + "${rsync_sec}" "$(format_duration_hms "${rsync_sec}")" + else + echo " 2. Rsync source staging: (not run)" + fi + if [ "${mvn_clean_sec}" != "-" ]; then + printf " 3. Maven clean (pre-Docker): %6ss (%s)\n" \ + "${mvn_clean_sec}" "$(format_duration_hms "${mvn_clean_sec}")" + else + echo " 3. Maven clean (pre-Docker): (not run)" + fi + if [ "${docker_build_sec}" != "-" ]; then + printf " 4. Docker image build (incl. Maven in image): %6ss (%s)\n" \ + "${docker_build_sec}" "$(format_duration_hms "${docker_build_sec}")" + else + echo " 4. Docker image build: (not run)" + fi + if [ "${build_images_sec}" != "-" ]; then + printf " build-images.sh total (3+4): %6ss (%s)\n" \ + "${build_images_sec}" "$(format_duration_hms "${build_images_sec}")" + fi + if [ "${pytest_sec}" != "-" ]; then + printf " 5. Pytest integration suite: %6ss (%s)\n" \ + "${pytest_sec}" "$(format_duration_hms "${pytest_sec}")" + else + echo " 5. Pytest integration suite: (not run)" + fi + printf " 6. Total wall time: %6ss (%s)\n" \ + "${total_sec}" "$(format_duration_hms "${total_sec}")" + echo "========================================" +} + +REPLICA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUTPUT_DIR="${OUTPUT_DIR:-${REPLICA_DIR}/output}" +export HBASE_ROOT="$(cd "${REPLICA_DIR}/../.." && pwd)" + +export HBASE_IMAGE="hbase-read-replica:${BUILD_NUMBER:-local}" + +OVERALL_START_SEC=${SECONDS} +TIMING_SUMMARY_PRINTED=false +CLEANUP_RAN=false +DEV_SUPPORT_IMAGE_BUILD_SEC="${DEV_SUPPORT_IMAGE_BUILD_SEC:-}" +RSYNC_SEC="" +MVN_CLEAN_SEC="" +DOCKER_BUILD_SEC="" +BUILD_IMAGES_SEC="" +PYTEST_SEC="" + +JAVA_VERSION=17 +KEEP_IMAGE=false +KEEP_CONTAINERS=false +PYTEST_K_VALUE="" + +print_usage() { + SCRIPT=$(basename "${BASH_SOURCE}") + + cat << __EOF + +run_read_replica_integration_tests.sh + +Inner test runner executed inside the Docker container built from +hbase/dev-support/docker/Dockerfile. Copies the HBase source, builds Docker +images for the active and read-replica clusters, and runs the pytest +integration suite. + +This script is normally invoked by hbase_nightly_read_replica_test.sh, but +it can also be run on its own within the container mentioned above. + +Usage: ${SCRIPT} [options] + + -h | --help Show this help message and exit. + -j | --java-version JVM version to use (default: 17). Sets JAVA_HOME + to /usr/lib/jvm/java-. + -i | --keep-image Do not remove the Docker image on exit. + -c | --keep-containers Do not run 'docker compose down' on exit. + -k Pytest -k filter expression for test selection. + (See Pytest documentation on -k for more info.) + +__EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + print_usage + exit 0 + ;; + -j|--java-version) + if [[ -n "$2" && "$2" != -* ]]; then + JAVA_VERSION="$2" + shift 2 + else + echo "Error: Argument for $1 is missing" >&2 + print_usage >&2 + exit 1 + fi + ;; + -i|--keep-image) + KEEP_IMAGE=true + shift + ;; + -c|--keep-containers) + KEEP_CONTAINERS=true + shift + ;; + -k) + if [[ -n "$2" && "$2" != -* ]]; then + PYTEST_K_VALUE="$2" + shift 2 + else + echo "Error: Argument for $1 is missing" >&2 + print_usage >&2 + exit 1 + fi + ;; + *) + echo "Unknown option: $1" >&2 + print_usage >&2 + exit 1 + ;; + esac +done + +# Set JAVA_HOME and update PATH for the selected JVM (e.g. 8, 11, 17, 21) +# MAVEN_HOME is set in hbase/dev-support/docker/Dockerfile +export JAVA_HOME="/usr/lib/jvm/java-${JAVA_VERSION}" +export PATH="${JAVA_HOME}/bin:${MAVEN_HOME}/bin:${PATH}" + +echo "Using JAVA_HOME=${JAVA_HOME}" + +echo "=== Inside Dev Container Environment ===" +echo "Replica dir: ${REPLICA_DIR}" +echo "Output dir: ${OUTPUT_DIR}" +echo "HBase root: ${HBASE_ROOT}" + +echo "Changing to replica dir: ${REPLICA_DIR}" +cd "${REPLICA_DIR}" + +echo "Sourcing environment file: $(pwd)/.env" +set -a +source .env +set +a + +echo "HBASE_IMAGE=${HBASE_IMAGE}" +echo "ACTIVE_CLUSTER_CONF_DIR=${ACTIVE_CLUSTER_CONF_DIR}" +echo "ACTIVE_CLUSTER_LOGS_DIR=${ACTIVE_CLUSTER_LOGS_DIR}" +echo "REPLICA_CLUSTER_CONF_DIR=${REPLICA_CLUSTER_CONF_DIR}" +echo "REPLICA_CLUSTER_LOGS_DIR=${REPLICA_CLUSTER_LOGS_DIR}" +echo "DOCKER_COMPOSE_FILE=${DOCKER_COMPOSE_FILE}" +echo "HBASE_DATA_STORE_ROOT=${HBASE_DATA_STORE_ROOT}" +echo "realpath of HBASE_DATA_STORE_ROOT=$(realpath ${HBASE_DATA_STORE_ROOT})" + +# Docker COPY does not follow symlinks; stage a trimmed tree for the build context. +# Excludes build artifacts and the nested read-replica staging tree. +echo "Syncing trimmed HBase tree into ${REPLICA_DIR}/hbase for Docker build context..." +mkdir -p "${REPLICA_DIR}/hbase" +RSYNC_START=${SECONDS} +rsync -a --delete \ + --exclude .git \ + --exclude target \ + --exclude dev-support/read-replica/hbase \ + --exclude dev-support/read-replica/tmp-read-replica-data \ + --exclude dev-support/read-replica/output \ + --exclude node_modules \ + --exclude .venv \ + "${HBASE_ROOT}/" "${REPLICA_DIR}/hbase/" +RSYNC_SEC=$((SECONDS - RSYNC_START)) +echo "Rsync completed (${RSYNC_SEC}s, $(format_duration_hms "${RSYNC_SEC}"))." +export HBASE_SOURCE_DIR="${REPLICA_DIR}/hbase" + +cleanup() { + local exit_code=${1:-$?} + set +e + if [ "${CLEANUP_RAN}" = true ]; then + exit "${exit_code}" + fi + CLEANUP_RAN=true + print_timing_summary + if [ ${exit_code} -ne 0 ]; then + echo "=== FAILURE ===" + echo "An error occurred during this stage of the run." + fi + if [ "${KEEP_CONTAINERS}" = "false" ]; then + echo "=== Cleanup: Stopping Docker containers ===" + docker compose -f "${DOCKER_COMPOSE_FILE}" down 2>/dev/null || true + else + echo "=== Cleanup: Keeping Docker containers (--keep-containers) ===" + fi + if [ "${KEEP_IMAGE}" = "false" ]; then + echo "=== Cleanup: Removing Docker image: ${HBASE_IMAGE} ===" + docker rmi --force "${HBASE_IMAGE}" 2>/dev/null || true + else + echo "=== Cleanup: Keeping Docker image: ${HBASE_IMAGE} (--keep-image) ===" + fi + echo "=== Cleanup: Deleting copied HBase directory: ${REPLICA_DIR}/hbase ===" + rm -rf "${REPLICA_DIR}/hbase" + echo "=== Cleanup: Deleting artifacts that would be owned by root on the CI host ===" + rm -rf "${REPLICA_DIR}/.venv" + rm -rf "${REPLICA_DIR}/.pytest_cache" + rm -rf "${REPLICA_DIR}/python/test/.pytest_cache" + rm -rf "${REPLICA_DIR}/python/__pycache__" + rm -rf "${REPLICA_DIR}/python/test/__pycache__" + rm -rf "${REPLICA_DIR}/python/src/__pycache__" + rm -rf "${REPLICA_DIR}/python/proto/generated" + rm -rf "${REPLICA_DIR}/python/proto/ActiveClusterSuffix.proto" + rm -rf "${REPLICA_DIR}/tmp-read-replica-data" + rm -rf "${REPLICA_DIR}/cluster1/logs" + rm -rf "${REPLICA_DIR}/cluster2/logs" + exit "${exit_code}" +} +trap cleanup EXIT + +# Copy latest proto file from source +echo "Copying latest version of ActiveClusterSuffix.proto to $(pwd)/python/proto/" +cp "${HBASE_ROOT}/hbase-protocol-shaded/src/main/protobuf/server/ActiveClusterSuffix.proto" \ + python/proto/ + +export PYTHONPATH="$(pwd)" +echo "Set PYTHONPATH=${PYTHONPATH}" + +# Create Python environment +echo "Creating Python environment: .venv" +python3 -m venv .venv +source .venv/bin/activate + +# Install Python dependencies +echo "Installing Python libraries" +pip install --upgrade pip +pip install -r requirements.txt + +# Compile protobuf +echo "Compiling Protobuf" +python3 python/proto/proto_compiler.py + +# Build Docker images +echo "Building hbase-docker image" +export READ_REPLICA_TIMING_FILE="${OUTPUT_DIR}/read-replica-timing.env" +rm -f "${READ_REPLICA_TIMING_FILE}" +BUILD_IMAGES_START=${SECONDS} +./build-images.sh +BUILD_IMAGES_WALL_SEC=$((SECONDS - BUILD_IMAGES_START)) +if [ -f "${READ_REPLICA_TIMING_FILE}" ]; then + # shellcheck disable=SC1090 + source "${READ_REPLICA_TIMING_FILE}" +fi +echo "build-images.sh wall time: ${BUILD_IMAGES_WALL_SEC}s ($(format_duration_hms "${BUILD_IMAGES_WALL_SEC}"))" + +# Run read-replica integration test suite +PYTEST_K_ARGS=() +if [[ -n "${PYTEST_K_VALUE}" ]]; then + PYTEST_K_ARGS=(-k "${PYTEST_K_VALUE}") +fi + +echo "Starting read-replica integration test suite via Pytest..." +PYTEST_START=${SECONDS} +PYTEST_EXIT_CODE=0 +pytest -o log_cli=true --log-cli-level=INFO \ + --log-cli-format='%(asctime)s %(levelname)-5s %(module)s.%(funcName)s(%(lineno)d): %(message)s' \ + --log-cli-date-format='%Y-%m-%d %H:%M:%S' \ + --junitxml="${OUTPUT_DIR}/read-replica-nightly-test-results.xml" \ + python/test/test_read_replica_feature.py \ + "${PYTEST_K_ARGS[@]}" || PYTEST_EXIT_CODE=$? +PYTEST_SEC=$((SECONDS - PYTEST_START)) +echo "Pytest wall time: ${PYTEST_SEC}s ($(format_duration_hms "${PYTEST_SEC}"))" + +# Write comprehensive timing file for the console report generator. +TOTAL_SEC=$(( SECONDS - OVERALL_START_SEC + ${DEV_SUPPORT_IMAGE_BUILD_SEC:-0} )) +cat > "${OUTPUT_DIR}/read-replica-all-timing.env" < [-n|--num-rows NUM_ROWS] [-i|--initial-row-value INITIAL_ROW_VALUE]" + exit 1 +} + +if [ "$#" -lt 2 ]; then + usage +fi + +TABLE_NAME=$1 +COLUMN_FAMILY=$2 +shift 2 + +NUM_ROWS="" +INITIAL_ROW_VALUE="" + +while [ "$#" -gt 0 ]; do + case "$1" in + -n|--num-rows) + NUM_ROWS="$2" + shift 2 + ;; + -i|--initial-row-value) + INITIAL_ROW_VALUE="$2" + shift 2 + ;; + *) + usage + ;; + esac +done + +TSV_GENERATOR_ARGS="" +if [ -n "$NUM_ROWS" ]; then + TSV_GENERATOR_ARGS="$TSV_GENERATOR_ARGS -n $NUM_ROWS" +fi +if [ -n "$INITIAL_ROW_VALUE" ]; then + TSV_GENERATOR_ARGS="$TSV_GENERATOR_ARGS -i $INITIAL_ROW_VALUE" +fi + +# Clean up any existing bulkload directories +rm -rf /tmp/bulkload + +# Re-create the necessary directory structure +mkdir -p /tmp/bulkload/tsvdata + +# Generate TSV data and save to the specified directory +python3 /opt/utils/tsv_generator.py /tmp/bulkload/tsvdata $TSV_GENERATOR_ARGS + +# Import TSV data to create HFiles for bulk loading +hbase org.apache.hadoop.hbase.mapreduce.ImportTsv \ + -Dimporttsv.columns=HBASE_ROW_KEY,$COLUMN_FAMILY:col0,$COLUMN_FAMILY:col1,$COLUMN_FAMILY:col2,$COLUMN_FAMILY:col3,$COLUMN_FAMILY:col4,$COLUMN_FAMILY:col5,$COLUMN_FAMILY:col6,$COLUMN_FAMILY:col7,$COLUMN_FAMILY:col8,$COLUMN_FAMILY:col9 \ + -Dimporttsv.bulk.output=/tmp/bulkload/HFiles \ + $TABLE_NAME /tmp/bulkload/tsvdata/output.tsv + +# Bulk load the generated HFiles into the HBase table +hbase completebulkload /tmp/bulkload/HFiles $TABLE_NAME diff --git a/dev-support/read-replica/utils/tsv_generator.py b/dev-support/read-replica/utils/tsv_generator.py new file mode 100644 index 000000000000..cc858e851670 --- /dev/null +++ b/dev-support/read-replica/utils/tsv_generator.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import os +import random + +NUM_COLUMNS = 10 + + +def generate_data(row_key): + columns = [str(random.randint(1, 100)) for _ in range(NUM_COLUMNS)] + return f"{row_key}\t" + "\t".join(columns) + "\n" + + +def main(output_dir, num_rows, initial_row_value): + tsv_file = os.path.join(output_dir, "output.tsv") + + rows_written = 0 + with open(tsv_file, "w") as f: + for i in range(initial_row_value, num_rows+initial_row_value): + row_key = f"row{i}" + f.write(generate_data(row_key)) + rows_written += 1 + + print(f"TSV file generated at {tsv_file} with {rows_written} rows and {NUM_COLUMNS} columns.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Generate a TSV file with random data for HBase bulk loading.") + parser.add_argument("output_directory", help="Directory to write the output TSV file") + parser.add_argument("-n", "--num-rows", type=int, default=500, help="Number of rows to generate (default: 500)") + parser.add_argument("-i", "--initial-row-value", type=int, default=0, help="Starting row number (default: 0)") + args = parser.parse_args() + + main(args.output_directory, args.num_rows, args.initial_row_value)