fix(test): make local VM execution portable

.github/workflows/environment-compatibility-tests.yml

@@ -0,0 +1,71 @@
+name: Environment Compatibility Tests
+
+on:
+  pull_request:
+    paths:
+      - rustfs-table-test.sh
+      - rustfs-replication-test.sh
+      - rustfs-security-test.sh
+      - rustfs-upgrade-test.sh
+      - rustfs_performance_test.sh
+      - scripts/check_execution_environment.py
+      - scripts/lib/**
+      - test_check_execution_environment.py
+      - test_rustfs_table.py
+      - test_rustfs_performance_testing.py
+      - test_rustfs_replication.py
+      - .github/workflows/environment-compatibility-tests.yml
+  push:
+    branches: [main]
+    paths:
+      - rustfs-table-test.sh
+      - rustfs-replication-test.sh
+      - rustfs-security-test.sh
+      - rustfs-upgrade-test.sh
+      - rustfs_performance_test.sh
+      - scripts/check_execution_environment.py
+      - scripts/lib/**
+      - test_check_execution_environment.py
+      - test_rustfs_table.py
+      - test_rustfs_performance_testing.py
+      - test_rustfs_replication.py
+      - .github/workflows/environment-compatibility-tests.yml
+
+permissions:
+  contents: read
+
+jobs:
+  environment-compatibility:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a
+        with:
+          persist-credentials: false
+      - name: Validate shell syntax
+        run: |
+          for script in \
+            rustfs-table-test.sh \
+            rustfs-replication-test.sh \
+            rustfs-security-test.sh \
+            rustfs-upgrade-test.sh \
+            rustfs_performance_test.sh
+          do
+            bash -n "${script}"
+          done
+          test -x rustfs-security-test.sh
+          test -x rustfs-upgrade-test.sh
+      - name: Test portable execution helpers
+        run: >-
+          python3 -B -m unittest -v
+          test_check_execution_environment.py
+          test_rustfs_table.py
+          test_rustfs_performance_testing.py
+          test_rustfs_replication.py
+      - name: Test missing HOME
+        run: env -u HOME bash test_rustfs_pool_expand.sh
+      - name: Smoke-test environment preflight
+        run: >-
+          python3 scripts/check_execution_environment.py
+          --command bash
+          --minimum-bash-major 4

README.md

@@ -1,6 +1,31 @@
 # auto-testing
 Auto testing scripts
 
+## Portable execution environments
+
+The suite defaults and CI topology remain unchanged. Local fleets can run the
+read-only compatibility preflight before any mutation:
+
+```bash
+python3 scripts/check_execution_environment.py \
+  --command aws --command curl --command jq --command openssl --command ssh \
+  --minimum-bash-major 4 \
+  --check-aws-list-objects \
+  --data-root /data/rustfs --drive-count 4 \
+  --minimum-free-gib 40 --require-distinct-devices \
+  --require-port 18080
+```
+
+The command prints JSON and exits `2` when a requirement is blocked. It does
+not create files, install packages, or change test behavior. A local VM that
+maps separate data disks to `/data/rustfs1` through `/data/rustfs4` can remove
+the root-filesystem disk-check bypass used by ad hoc single-disk deployments.
+
+The tier webhook receiver is configurable with `RUSTFS_RECEIVER_PORT`; choose a
+port that does not conflict with the local package server or another service.
+All top-level suite entrypoints are directly executable; `bash <script>` also
+works when a local checkout or transport drops executable modes.
+
 ## Release validation migration
 
 `validation-cases.json` snapshots all 158 canonical cases from

rustfs-replication-test.sh

@@ -29,6 +29,9 @@
 #
 set -Eeuo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+JQ_LIB="${SCRIPT_DIR}/scripts/lib"
+
 # ==================== Configuration ====================
 
 if [ -n "${RUSTFS_NODES:-}" ]; then
@@ -678,7 +681,7 @@ rm_bucket_force() {
         --delete "${payload}" >/dev/null 2>&1 || true
     done < <(aws --endpoint-url "${url}" --region "${REGION}" s3api list-object-versions \
         --bucket "${b}" --output json 2>/dev/null \
-        | jq -c '[.Versions[]?, .DeleteMarkers[]?] | map({Key, VersionId}) | select(length > 0) | _nwise(1000) | {Objects: ., Quiet: true}')
+        | jq -c -L "${JQ_LIB}" 'include "jq_nwise"; [.Versions[]?, .DeleteMarkers[]?] | map({Key, VersionId}) | select(length > 0) | nwise(1000)[] | {Objects: ., Quiet: true}')
     aws --endpoint-url "${url}" --region "${REGION}" s3api delete-bucket --bucket "${b}" >/dev/null 2>&1 || true
     bucket_exists "${ep}" "${b}" || return 0
     sleep 2

rustfs-table-test.sh

@@ -31,7 +31,7 @@ COMPAT_PATH="${RUSTFS_TABLE_COMPAT_PATH:-/_iceberg}"
 TEST_BUCKET_PREFIX="${RUSTFS_TABLE_TEST_BUCKET:-tblsuite}"
 LOG_FILE="${RUSTFS_TABLE_LOG_FILE:-}"
 SMOKE_SCRIPT="${RUSTFS_TABLE_SMOKE_SCRIPT:-}"   # optional: path to product pyiceberg_smoke.py
-SUDO=(); [ "$(id -u)" -ne 0 ] && SUDO=(sudo -n)
+LOCAL_SUDO=(); [ "$(id -u)" -ne 0 ] && LOCAL_SUDO=(sudo -n)
 DRY_RUN=0
 SKIP_SMOKE=0
 PASS=0; FAIL=0; UNSUPPORTED=0
@@ -146,31 +146,56 @@ rest_body() { sed -n '2,$p' <<<"${REST_OUT}"; }
 
 # ==================== Environment (install + parallel start, shared-fleet style) ====================
 remote() { ssh -o BatchMode=yes -o ConnectTimeout=10 -p "${SSH_PORT}" "${SSH_USER}@${1}" "$2"; }
+remote_root() {
+  ssh -o BatchMode=yes -o ConnectTimeout=10 -p "${SSH_PORT}" "${SSH_USER}@${1}" \
+    "if [ \"\$(id -u)\" -eq 0 ]; then bash -s; else sudo -n bash -s; fi"
+}
+
+rustfs_table_config() {
+  cat <<CFG
+RUSTFS_ACCESS_KEY=${ACCESS_KEY}
+RUSTFS_SECRET_KEY=${SECRET_KEY}
+RUSTFS_VOLUMES="http://rustfs-node{1...4}:9000/data/rustfs{1...4}/mnmd"
+RUSTFS_ADDRESS=":9000"
+RUSTFS_CONSOLE_ADDRESS=":9001"
+RUSTFS_CONSOLE_ENABLE=true
+RUSTFS_OBS_LOGGER_LEVEL=error
+RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
+CFG
+}
 
 setup_environment() {
   local v
   for v in "${NODES[@]}"; do
-    remote "${v}" "curl -fsSL --retry 3 -o /tmp/rustfs-table.deb '${PACKAGE_URL}' && ${SUDO[*]} dpkg -i --force-confold /tmp/rustfs-table.deb >/dev/null 2>&1 && ${SUDO[*]} mkdir -p /data/rustfs{1..4}/mnmd && ${SUDO[*]} chown -R rustfs:rustfs /data/rustfs{1..4}/mnmd" || die "install failed on ${v}"
+    remote_root "${v}" <<EOF || die "install failed on ${v}"
+set -e
+curl -fsSL --retry 3 -o /tmp/rustfs-table.deb '${PACKAGE_URL}'
+dpkg -i --force-confold /tmp/rustfs-table.deb >/dev/null
+mkdir -p /data/rustfs{1..4}/mnmd
+chown -R rustfs:rustfs /data/rustfs{1..4}/mnmd
+EOF
   done
   write_node_config() {
-    remote "$1" "${SUDO[*]} tee /etc/default/rustfs >/dev/null <<'CFG'
-RUSTFS_ACCESS_KEY=${ACCESS_KEY}
-RUSTFS_SECRET_KEY=${SECRET_KEY}
-RUSTFS_VOLUMES=\"http://rustfs-node{1...4}:9000/data/rustfs{1...4}/mnmd\"
-RUSTFS_ADDRESS=\":9000\"
-RUSTFS_CONSOLE_ADDRESS=\":9001\"
-RUSTFS_CONSOLE_ENABLE=true
-RUSTFS_OBS_LOGGER_LEVEL=error
-RUSTFS_OBS_LOG_DIRECTORY=\"/var/log/rustfs/\"
-CFG"
+    remote_root "$1" <<EOF
+set -e
+cat > /etc/default/rustfs <<'CFG'
+$(rustfs_table_config)
+CFG
+EOF
   }
   local cfg_pids=()
   for v in "${NODES[@]}"; do write_node_config "${v}" & cfg_pids+=("$!"); done
   # wait on explicit job pids only: a bare `wait` also waits for the log
   # process substitution (bash >= 5.1) and would hang forever on tee
   wait "${cfg_pids[@]}"
   local start_pids=()
-  for v in "${NODES[@]}"; do remote "${v}" "${SUDO[*]} systemctl reset-failed rustfs 2>/dev/null; ${SUDO[*]} systemctl start rustfs" & start_pids+=("$!"); done
+  for v in "${NODES[@]}"; do
+    remote_root "${v}" <<'EOF' &
+systemctl reset-failed rustfs 2>/dev/null || true
+systemctl start rustfs
+EOF
+    start_pids+=("$!")
+  done
   wait "${start_pids[@]}"
   local deadline=$(( $(date +%s) + 180 )) i=0 ok
   while [ "$(date +%s)" -lt "${deadline}" ]; do
@@ -193,7 +218,14 @@ CFG"
 teardown_environment() {
   local v
   local clean_pids=()
-  for v in "${NODES[@]}"; do remote "${v}" "${SUDO[*]} systemctl stop rustfs 2>/dev/null; ${SUDO[*]} dpkg -P rustfs >/dev/null 2>&1; ${SUDO[*]} rm -rf /data/rustfs{1..4}/mnmd /etc/default/rustfs" & clean_pids+=("$!"); done
+  for v in "${NODES[@]}"; do
+    remote_root "${v}" <<'EOF' &
+systemctl stop rustfs 2>/dev/null || true
+dpkg -P rustfs >/dev/null 2>&1 || true
+rm -rf /data/rustfs{1..4}/mnmd /etc/default/rustfs
+EOF
+    clean_pids+=("$!")
+  done
   wait "${clean_pids[@]}"
   log "environment torn down"
 }
@@ -218,7 +250,7 @@ test_tbl101() {
   local venv="/tmp/tbl-venv-${RUN_ID}"
   if ! python3 -m venv "${venv}" >/tmp/tbl-venv-err.log 2>&1; then
     echo "tbl101: venv create failed, installing python3-venv: $(head -3 /tmp/tbl-venv-err.log)"
-    ${SUDO[*]} apt-get install -y python3-venv >/dev/null 2>&1 || true
+    "${LOCAL_SUDO[@]}" apt-get install -y python3-venv >/dev/null 2>&1 || true
     python3 -m venv "${venv}" >>/tmp/tbl-venv-err.log 2>&1 \
       || { echo "tbl101: venv create failed twice: $(tail -3 /tmp/tbl-venv-err.log)"; return 2; }
   fi
@@ -369,27 +401,29 @@ test_tbl107() {
 }
 
 # ==================== Main ====================
-TEST_TMP="$(mktemp -d /tmp/rustfs-table-test.XXXXXX)"
-trap 'cleanup_all; rm -rf "${TEST_TMP}"' EXIT
-if [ -n "${LOG_FILE}" ]; then
-  mkdir -p "$(dirname "${LOG_FILE}")"
-  exec > >(tee -a "${LOG_FILE}") 2>&1
-fi
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+  TEST_TMP="$(mktemp -d /tmp/rustfs-table-test.XXXXXX)"
+  trap 'cleanup_all; rm -rf "${TEST_TMP}"' EXIT
+  if [ -n "${LOG_FILE}" ]; then
+    mkdir -p "$(dirname "${LOG_FILE}")"
+    exec > >(tee -a "${LOG_FILE}") 2>&1
+  fi
 
-log "S3 Tables (Iceberg REST Catalog) suite"
-log "endpoint=${API_ENDPOINT} rest=${REST_PATH} bucket-prefix=${TEST_BUCKET_PREFIX}"
-[ "${DRY_RUN}" -eq 0 ] && setup_environment
+  log "S3 Tables (Iceberg REST Catalog) suite"
+  log "endpoint=${API_ENDPOINT} rest=${REST_PATH} bucket-prefix=${TEST_BUCKET_PREFIX}"
+  [ "${DRY_RUN}" -eq 0 ] && setup_environment
 
-run_test TBL-101 "PyIcebergEndToEnd"            test_tbl101
-run_test TBL-102 "EnableAndGetTableBucket"      test_tbl102
-run_test TBL-103 "NamespaceAndTableCrud"        test_tbl103
-run_test TBL-104 "ReservedPrefixProtection"     test_tbl104
-run_test TBL-106 "CompatPrefixAndConfig"        test_tbl106
-run_test TBL-107 "TableBucketListIsolation"     test_tbl107
-run_test TBL-105 "CommitCasConflict"            test_tbl105
+  run_test TBL-101 "PyIcebergEndToEnd"            test_tbl101
+  run_test TBL-102 "EnableAndGetTableBucket"      test_tbl102
+  run_test TBL-103 "NamespaceAndTableCrud"        test_tbl103
+  run_test TBL-104 "ReservedPrefixProtection"     test_tbl104
+  run_test TBL-106 "CompatPrefixAndConfig"        test_tbl106
+  run_test TBL-107 "TableBucketListIsolation"     test_tbl107
+  run_test TBL-105 "CommitCasConflict"            test_tbl105
 
-cleanup_all
-log "table suite: ${PASS} passed, ${FAIL} failed, ${UNSUPPORTED} unsupported"
-[ "${FAIL}" -eq 0 ] || printf 'failed: %s\n' "${FAILED_TESTS[*]}"
-[ "${DRY_RUN}" -eq 0 ] || exit 0
-[ "${FAIL}" -eq 0 ]
+  cleanup_all
+  log "table suite: ${PASS} passed, ${FAIL} failed, ${UNSUPPORTED} unsupported"
+  [ "${FAIL}" -eq 0 ] || printf 'failed: %s\n' "${FAILED_TESTS[*]}"
+  [ "${DRY_RUN}" -eq 0 ] || exit 0
+  [ "${FAIL}" -eq 0 ]
+fi

rustfs-upgrade-test.sh

@@ -42,6 +42,9 @@
 #   RUSTFS_NODES / RUSTFS_SSH_USER          shared test VMs (default vm000..002 / azureuser)
 set -Eeuo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+JQ_LIB="${SCRIPT_DIR}/scripts/lib"
+
 TEST_TMP="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-upgrade.XXXXXX")"
 trap 'rm -rf "${TEST_TMP}"' EXIT
 
@@ -862,8 +865,8 @@ seed_warp_bucket() {
       aws_run s3api delete-objects --bucket "${bucket}" --delete "${payload}"
       assert_aws_ok || die "seed: trim ${bucket} (delete batch failed)"
     done < <(printf '%s' "${keys_json}" \
-             | jq -c --argjson k "${keep}" \
-               '.[0:(length - $k)] | _nwise(1000) | {Objects: [.[] | {Key: .}], Quiet: true}')
+             | jq -c --argjson k "${keep}" -L "${JQ_LIB}" \
+               'include "jq_nwise"; .[0:(length - $k)] | nwise(1000)[] | {Objects: [.[] | {Key: .}], Quiet: true}')
     aws_run s3api list-objects-v2 --bucket "${bucket}" --query 'length(Contents)'
     assert_aws_ok || die "seed: post-trim list ${bucket}"
     remaining="$(printf '%s' "${AWS_OUT}" | tr -d '[:space:]')"

rustfs_performance_test.sh

@@ -76,19 +76,21 @@ RUSTFS_CONSOLE_ENABLE=true
 RUSTFS_OBS_LOGGER_LEVEL=error
 RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
 
-# Benchmark runner (step 5/6): prefer the default Obsidian location, fall back
-# to a rustfs-performance-testing.sh next to this script (e.g. in the repo or
-# on a jumpbox).
-_DEFAULT_BENCH="${HOME}/Documents/Obsidian Vault/rustfs-performance-testing.sh"
+# Benchmark runner (step 5/6): prefer the default Obsidian location when HOME
+# is available, then fall back to a runner next to this script.
 _SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-if [ -x "${_DEFAULT_BENCH}" ]; then
+_DEFAULT_BENCH=""
+if [ -n "${HOME:-}" ]; then
+  _DEFAULT_BENCH="${HOME}/Documents/Obsidian Vault/rustfs-performance-testing.sh"
+fi
+if [ -n "${_DEFAULT_BENCH}" ] && [ -x "${_DEFAULT_BENCH}" ]; then
   _BENCH_RESOLVED="${_DEFAULT_BENCH}"
 elif [ -x "${_SCRIPT_DIR}/rustfs-performance-testing.sh" ]; then
   _BENCH_RESOLVED="${_SCRIPT_DIR}/rustfs-performance-testing.sh"
 elif [ -x "${_SCRIPT_DIR}/rustfs_performance_testing.sh" ]; then
   _BENCH_RESOLVED="${_SCRIPT_DIR}/rustfs_performance_testing.sh"
 else
-  _BENCH_RESOLVED="${_DEFAULT_BENCH}"
+  _BENCH_RESOLVED="${_DEFAULT_BENCH:-${_SCRIPT_DIR}/rustfs_performance_testing.sh}"
 fi
 BENCH_SCRIPT="${RUSTFS_BENCH_SCRIPT:-${_BENCH_RESOLVED}}"
 RESULT_DIR="${RUSTFS_RESULT_DIR:-$(pwd)/warp-bench-results-$(date +%Y%m%d-%H%M%S)}"