From 0d393bc964a6a856924dcbf582cdb597940d7f79 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Tue, 18 Aug 2026 12:42:41 +0000 Subject: [PATCH 01/14] feat(storage): add run_benchmark_tests.sh for automated time-based GCS read microbenchmarks --- .../cloudbuild/run_benchmark_tests.sh | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100755 packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh new file mode 100755 index 000000000000..7735c9018aed --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# ============================================================================== +# Automated Google Cloud Storage Read Microbenchmark Runner +# Intended for GitHub CI/CD & GCE High-Bandwidth Tier-1 VMs (C4/N2/C3 series) +# Location: packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +# ============================================================================== + +set -eo pipefail + +# Configurable defaults +PROCESSES="${PROCESSES:-48}" +COROS="${COROS:-1}" +FILE_SIZE_MIB="${FILE_SIZE_MIB:-10240}" # 10 GiB files by default +CHUNK_SIZE_KIB="${CHUNK_SIZE_KIB:-102400}" # ~100 MiB read chunks by default +BUCKET_TYPE="${BUCKET_TYPE:-zonal}" # "zonal" uses BidiReadObject gRPC DirectPath +TARGET_BUCKET="${DEFAULT_RAPID_ZONAL_BUCKET:-shradhakatyal-read-bench-zb}" +OUT_JSON="${OUT_JSON:-/tmp/bench_result.json}" +UPLOAD_GCS_PREFIX="${UPLOAD_GCS_PREFIX:-}" + +echo "========================================================================" +echo " GCS Read Microbenchmark Runner (gRPC BidiReadObject / REST)" +echo " Processes: ${PROCESSES}" +echo " Coroutines/proc: ${COROS}" +echo " File Size: ${FILE_SIZE_MIB} MiB" +echo " Chunk Size: ${CHUNK_SIZE_KIB} KiB" +echo " Bucket Type: ${BUCKET_TYPE} (zonal = BidiReadObject gRPC DirectPath)" +echo " Target Bucket: gs://${TARGET_BUCKET}" +echo "========================================================================" + +# Ensure HOME is exported for gRPC / ALTS Application Default Credentials +export HOME="${HOME:-/root}" +export DEFAULT_RAPID_ZONAL_BUCKET="${TARGET_BUCKET}" +export DEFAULT_STANDARD_BUCKET="${TARGET_BUCKET}" + +# Determine repository root +REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +cd "${REPO_ROOT}/packages/google-cloud-storage" 2>/dev/null || cd "$(pwd)" + +echo "--- 1. Checking Python dependencies ---" +if ! python3 -c "import pytest, psutil, yaml" 2>/dev/null; then + echo "Installing test dependencies..." + pip install --upgrade pip + pip install -e . + pip install pytest pytest-benchmark psutil pyyaml google-cloud-testutils google-cloud-kms +fi + +CONFIG_PATH="tests/perf/microbenchmarks/time_based/reads/config.yaml" +if [ ! -f "${CONFIG_PATH}" ]; then + echo "ERROR: Could not find ${CONFIG_PATH}. Please run from google-cloud-storage root." + exit 1 +fi + +echo "--- 2. Updating ${CONFIG_PATH} parameters ---" +python3 -c " +import yaml +path = '${CONFIG_PATH}' +with open(path) as f: + d = yaml.safe_load(f) +d['common']['file_sizes_mib'] = [${FILE_SIZE_MIB}] +d['common']['chunk_sizes_kib'] = [${CHUNK_SIZE_KIB}] +d['common']['bucket_types'] = ['${BUCKET_TYPE}'] +for w in d['workload']: + w['processes'] = [${PROCESSES}] + w['coros'] = [${COROS}] +with open(path, 'w') as f: + yaml.dump(d, f) +" + +# Patch config.py so 1-to-1 process-to-file indexing prevents 404 on multi-coroutine runs +sed -i 's/num_files = num_processes \* num_coros/num_files = num_processes/g' tests/perf/microbenchmarks/time_based/reads/config.py || true +sed -i 's/num_files = num_processes \* num_coros/num_files = num_processes/g' tests/perf/microbenchmarks/reads/config.py || true + +echo "--- 3. Pre-seeding & verifying ${PROCESSES} test objects (${FILE_SIZE_MIB} MiB each) in gs://${TARGET_BUCKET} ---" +python3 -c " +import multiprocessing, os, time +from google.cloud import storage + +bucket_name = '${TARGET_BUCKET}' +client = storage.Client() +bucket = client.bucket(bucket_name) + +local_file = '/tmp/benchmark_test_payload' +expected_size = ${FILE_SIZE_MIB} * 1024 * 1024 + +def ensure_object(idx): + obj_name = f'fio-go_storage_fio.0.{idx}' + blob = bucket.get_blob(obj_name) + if not blob or blob.size != expected_size: + if not os.path.exists(local_file): + print(f'Generating {expected_size} bytes payload locally...') + os.system(f'dd if=/dev/urandom of={local_file} bs=1M count=${FILE_SIZE_MIB} status=none') + t0 = time.time() + print(f'Uploading {obj_name} ({FILE_SIZE_MIB} MiB)...') + blob_new = bucket.blob(obj_name) + blob_new.upload_from_filename(local_file) + print(f'Uploaded {obj_name} in {time.time()-t0:.1f}s') + +print(f'Verifying {${PROCESSES}} objects in bucket {bucket_name}...') +with multiprocessing.Pool(min(16, ${PROCESSES})) as pool: + pool.map(ensure_object, range(${PROCESSES})) +" + +echo "--- 4. Executing pytest benchmark suite ---" +pytest --benchmark-json="${OUT_JSON}" \ + -vv -s \ + --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' \ + tests/perf/microbenchmarks/time_based/reads/test_reads.py || true + +if [ -s "${OUT_JSON}" ]; then + echo "========================================================================" + echo " BENCHMARK STATS SUMMARY" + echo "========================================================================" + grep -E '"name":|"avg_throughput_mib_s":|"net_throughput_mb_s":|"cpu_max_global":' "${OUT_JSON}" -B 1 -A 2 || true + + if [ -n "${UPLOAD_GCS_PREFIX}" ]; then + GCS_DEST="${UPLOAD_GCS_PREFIX}/test_result_$(hostname)_$(date +%s).json" + echo "Uploading JSON report to ${GCS_DEST}..." + gcloud storage cp "${OUT_JSON}" "${GCS_DEST}" + fi +fi + +echo "--- Benchmark Run Complete ---" From e01584f0b59a407e8119a21056035b2365005bf2 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Tue, 18 Aug 2026 12:54:45 +0000 Subject: [PATCH 02/14] ci(storage): add benchmarks-cloudbuild.yaml trigger config for high-bandwidth Tier-1 VM runner --- .../cloudbuild/benchmarks-cloudbuild.yaml | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml new file mode 100644 index 000000000000..134e698ac0b5 --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -0,0 +1,116 @@ +substitutions: + _ZONE: "us-central1-b" + _MACHINE_TYPE: "c4-standard-192" + _SHORT_BUILD_ID: ${BUILD_ID:0:8} + _VM_NAME: "read-bench-${_SHORT_BUILD_ID}" + _ULIMIT: "65536" + _PROCESSES: "48" + _COROS: "1" + _FILE_SIZE_MIB: "10240" + _CHUNK_SIZE_KIB: "102400" + _ZONAL_BUCKET: "shradhakatyal-read-bench-zb" + _ZONAL_VM_SERVICE_ACCOUNT: "" + +steps: + # Step 0: Generate a persistent SSH key for this build run. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "generate-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + mkdir -p /workspace/.ssh + ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb + cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub + gcloud compute os-login ssh-keys add \ + --key-file=/workspace/.ssh/google_compute_engine.pub \ + --ttl=1h + waitFor: ["-"] + + # Step 1: Package google-cloud-storage directory for direct transfer to VM + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "package-code" + entrypoint: "bash" + args: + - "-c" + - | + tar --exclude='.nox' --exclude='venv_*' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' -czf /workspace/google-cloud-storage.tar.gz -C /workspace/packages google-cloud-storage + waitFor: ["-"] + + # Step 2: Create a high-bandwidth GCE Tier-1 VM to run the read microbenchmarks. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "create-vm" + entrypoint: "gcloud" + args: + - "compute" + - "instances" + - "create" + - "${_VM_NAME}" + - "--project=${PROJECT_ID}" + - "--zone=${_ZONE}" + - "--machine-type=${_MACHINE_TYPE}" + - "--image-family=debian-12" + - "--image-project=debian-cloud" + - "--network-interface=nic-type=GVNIC" + - "--network-performance-configs=total-egress-bandwidth-tier=TIER_1" + - "--service-account=${_ZONAL_VM_SERVICE_ACCOUNT}" + - "--scopes=https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/cloudkms" + - "--metadata=enable-oslogin=TRUE" + waitFor: ["-"] + + # Step 3: Run the read microbenchmark suite inside the VM and cleanup cleanly. + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "run-tests-and-delete-vm" + entrypoint: "bash" + args: + - "-c" + - | + set -e + # Wait for the VM to be fully initialized and SSH to be ready + for i in {1..12}; do + if gcloud compute ssh "${_VM_NAME}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then + break + fi + echo "Waiting for VM to become available... (attempt $i/12)" + sleep 15 + done + + # Copy runner script and tarball to the VM + gcloud compute scp packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh /workspace/google-cloud-storage.tar.gz "${_VM_NAME}":~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + + # Execute run_benchmark_tests.sh on the VM via SSH + set +e + gcloud compute ssh "${_VM_NAME}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine \ + --command="ulimit -n ${_ULIMIT}; tar -xzf google-cloud-storage.tar.gz && cp run_benchmark_tests.sh google-cloud-storage/ && cd google-cloud-storage && PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} TARGET_BUCKET=${_ZONAL_BUCKET} bash run_benchmark_tests.sh" + EXIT_CODE=$? + set -e + + echo "--- Deleting GCE VM ---" + gcloud compute instances delete "${_VM_NAME}" --zone=${_ZONE} --quiet + + exit $$EXIT_CODE + waitFor: + - "create-vm" + - "generate-ssh-key" + - "package-code" + + # Step 4: Cleanup temporary OS Login SSH key + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "cleanup-ssh-key" + entrypoint: "bash" + args: + - "-c" + - | + echo "--- Removing SSH key from OS Login profile ---" + gcloud compute os-login ssh-keys remove \ + --key-file=/workspace/gcb_ssh_key.pub || true + waitFor: + - "run-tests-and-delete-vm" + +timeout: "3600s" # 60 minutes + +options: + logging: CLOUD_LOGGING_ONLY + dynamicSubstitutions: true + pool: + name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool" From 9f50945ca56a2166d812f7e07a6b4a6b0c72e63f Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Tue, 18 Aug 2026 12:59:21 +0000 Subject: [PATCH 03/14] fix(storage): resolve dd payload race condition and f-string variable name in run_benchmark_tests.sh --- .../cloudbuild/run_benchmark_tests.sh | 51 ++++++++++++++----- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh index 7735c9018aed..2ea461e2c105 100755 --- a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -76,28 +76,51 @@ import multiprocessing, os, time from google.cloud import storage bucket_name = '${TARGET_BUCKET}' -client = storage.Client() -bucket = client.bucket(bucket_name) - +file_size_mib = int('${FILE_SIZE_MIB}') +num_processes = int('${PROCESSES}') +expected_size = file_size_mib * 1024 * 1024 local_file = '/tmp/benchmark_test_payload' -expected_size = ${FILE_SIZE_MIB} * 1024 * 1024 -def ensure_object(idx): +def check_object(idx): + client = storage.Client() + bucket = client.bucket(bucket_name) obj_name = f'fio-go_storage_fio.0.{idx}' - blob = bucket.get_blob(obj_name) - if not blob or blob.size != expected_size: - if not os.path.exists(local_file): - print(f'Generating {expected_size} bytes payload locally...') - os.system(f'dd if=/dev/urandom of={local_file} bs=1M count=${FILE_SIZE_MIB} status=none') + try: + blob = bucket.get_blob(obj_name) + if not blob or blob.size != expected_size: + return idx + except Exception as e: + print(f'Error checking {obj_name}: {e}') + return idx + return None + +def upload_object(idx): + client = storage.Client() + bucket = client.bucket(bucket_name) + obj_name = f'fio-go_storage_fio.0.{idx}' + try: t0 = time.time() - print(f'Uploading {obj_name} ({FILE_SIZE_MIB} MiB)...') + print(f'Uploading {obj_name} ({file_size_mib} MiB)...') blob_new = bucket.blob(obj_name) blob_new.upload_from_filename(local_file) print(f'Uploaded {obj_name} in {time.time()-t0:.1f}s') + except Exception as e: + print(f'Error uploading {obj_name}: {e}') + +if __name__ == '__main__': + print(f'Verifying {num_processes} objects in bucket {bucket_name}...') + with multiprocessing.Pool(min(16, num_processes)) as pool: + results = pool.map(check_object, range(num_processes)) + + missing_indices = [r for r in results if r is not None] + if missing_indices: + print(f'Found {len(missing_indices)} missing/incomplete objects.') + if not os.path.exists(local_file): + print(f'Generating {expected_size} bytes payload locally...') + os.system(f'dd if=/dev/urandom of={local_file} bs=1M count={file_size_mib} status=none') -print(f'Verifying {${PROCESSES}} objects in bucket {bucket_name}...') -with multiprocessing.Pool(min(16, ${PROCESSES})) as pool: - pool.map(ensure_object, range(${PROCESSES})) + with multiprocessing.Pool(min(16, len(missing_indices))) as pool: + pool.map(upload_object, missing_indices) " echo "--- 4. Executing pytest benchmark suite ---" From 048ce17c2165dfdbfc34a4a443b7ad7dd98d66cf Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Tue, 18 Aug 2026 13:01:42 +0000 Subject: [PATCH 04/14] style(storage): add defensive isinstance type validation when modifying config.yaml --- .../cloudbuild/run_benchmark_tests.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh index 2ea461e2c105..fc1677d9f6fa 100755 --- a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -56,12 +56,18 @@ import yaml path = '${CONFIG_PATH}' with open(path) as f: d = yaml.safe_load(f) -d['common']['file_sizes_mib'] = [${FILE_SIZE_MIB}] -d['common']['chunk_sizes_kib'] = [${CHUNK_SIZE_KIB}] -d['common']['bucket_types'] = ['${BUCKET_TYPE}'] -for w in d['workload']: - w['processes'] = [${PROCESSES}] - w['coros'] = [${COROS}] +if isinstance(d, dict): + common = d.get('common') + if isinstance(common, dict): + common['file_sizes_mib'] = [${FILE_SIZE_MIB}] + common['chunk_sizes_kib'] = [${CHUNK_SIZE_KIB}] + common['bucket_types'] = ['${BUCKET_TYPE}'] + workloads = d.get('workload') + if isinstance(workloads, list): + for w in workloads: + if isinstance(w, dict): + w['processes'] = [${PROCESSES}] + w['coros'] = [${COROS}] with open(path, 'w') as f: yaml.dump(d, f) " From 7ca06c63f5135a5ba4a4a52d052ba1ed2d4fff27 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Wed, 19 Aug 2026 13:15:19 +0000 Subject: [PATCH 05/14] refactor(cloudbuild): target standing VM in us-west4-a and zonal bucket --- .../cloudbuild/benchmarks-cloudbuild.yaml | 59 ++++--------------- .../cloudbuild/run_benchmark_tests.sh | 2 +- 2 files changed, 13 insertions(+), 48 deletions(-) diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index 134e698ac0b5..44c387fa7f4d 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -1,15 +1,12 @@ substitutions: - _ZONE: "us-central1-b" - _MACHINE_TYPE: "c4-standard-192" - _SHORT_BUILD_ID: ${BUILD_ID:0:8} - _VM_NAME: "read-bench-${_SHORT_BUILD_ID}" + _ZONE: "us-west4-a" + _VM_NAME: "shradhakatyal-benchmarks-us-west4-a" _ULIMIT: "65536" _PROCESSES: "48" _COROS: "1" _FILE_SIZE_MIB: "10240" _CHUNK_SIZE_KIB: "102400" - _ZONAL_BUCKET: "shradhakatyal-read-bench-zb" - _ZONAL_VM_SERVICE_ACCOUNT: "" + _ZONAL_BUCKET: "shradhakatyal-read-bench-zb-us-west4-a" steps: # Step 0: Generate a persistent SSH key for this build run. @@ -37,64 +34,34 @@ steps: tar --exclude='.nox' --exclude='venv_*' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' -czf /workspace/google-cloud-storage.tar.gz -C /workspace/packages google-cloud-storage waitFor: ["-"] - # Step 2: Create a high-bandwidth GCE Tier-1 VM to run the read microbenchmarks. + # Step 2: Run the read microbenchmark suite inside the standing GCE VM via SSH. - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "create-vm" - entrypoint: "gcloud" - args: - - "compute" - - "instances" - - "create" - - "${_VM_NAME}" - - "--project=${PROJECT_ID}" - - "--zone=${_ZONE}" - - "--machine-type=${_MACHINE_TYPE}" - - "--image-family=debian-12" - - "--image-project=debian-cloud" - - "--network-interface=nic-type=GVNIC" - - "--network-performance-configs=total-egress-bandwidth-tier=TIER_1" - - "--service-account=${_ZONAL_VM_SERVICE_ACCOUNT}" - - "--scopes=https://www.googleapis.com/auth/devstorage.full_control,https://www.googleapis.com/auth/cloudkms" - - "--metadata=enable-oslogin=TRUE" - waitFor: ["-"] - - # Step 3: Run the read microbenchmark suite inside the VM and cleanup cleanly. - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "run-tests-and-delete-vm" + id: "run-tests-on-vm" entrypoint: "bash" args: - "-c" - | set -e - # Wait for the VM to be fully initialized and SSH to be ready - for i in {1..12}; do + # Verify SSH connectivity to existing standing VM + for i in {1..6}; do if gcloud compute ssh "${_VM_NAME}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then break fi - echo "Waiting for VM to become available... (attempt $i/12)" - sleep 15 + echo "Waiting for VM connectivity... (attempt $i/6)" + sleep 10 done - # Copy runner script and tarball to the VM + # Copy runner script and tarball to the standing VM gcloud compute scp packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh /workspace/google-cloud-storage.tar.gz "${_VM_NAME}":~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine # Execute run_benchmark_tests.sh on the VM via SSH - set +e gcloud compute ssh "${_VM_NAME}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine \ --command="ulimit -n ${_ULIMIT}; tar -xzf google-cloud-storage.tar.gz && cp run_benchmark_tests.sh google-cloud-storage/ && cd google-cloud-storage && PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} TARGET_BUCKET=${_ZONAL_BUCKET} bash run_benchmark_tests.sh" - EXIT_CODE=$? - set -e - - echo "--- Deleting GCE VM ---" - gcloud compute instances delete "${_VM_NAME}" --zone=${_ZONE} --quiet - - exit $$EXIT_CODE waitFor: - - "create-vm" - "generate-ssh-key" - "package-code" - # Step 4: Cleanup temporary OS Login SSH key + # Step 3: Cleanup temporary OS Login SSH key - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "cleanup-ssh-key" entrypoint: "bash" @@ -105,12 +72,10 @@ steps: gcloud compute os-login ssh-keys remove \ --key-file=/workspace/gcb_ssh_key.pub || true waitFor: - - "run-tests-and-delete-vm" + - "run-tests-on-vm" timeout: "3600s" # 60 minutes options: logging: CLOUD_LOGGING_ONLY dynamicSubstitutions: true - pool: - name: "projects/${PROJECT_ID}/locations/us-central1/workerPools/cloud-build-worker-pool" diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh index fc1677d9f6fa..bcee8cd16dca 100755 --- a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -13,7 +13,7 @@ COROS="${COROS:-1}" FILE_SIZE_MIB="${FILE_SIZE_MIB:-10240}" # 10 GiB files by default CHUNK_SIZE_KIB="${CHUNK_SIZE_KIB:-102400}" # ~100 MiB read chunks by default BUCKET_TYPE="${BUCKET_TYPE:-zonal}" # "zonal" uses BidiReadObject gRPC DirectPath -TARGET_BUCKET="${DEFAULT_RAPID_ZONAL_BUCKET:-shradhakatyal-read-bench-zb}" +TARGET_BUCKET="${DEFAULT_RAPID_ZONAL_BUCKET:-shradhakatyal-read-bench-zb-us-west4-a}" OUT_JSON="${OUT_JSON:-/tmp/bench_result.json}" UPLOAD_GCS_PREFIX="${UPLOAD_GCS_PREFIX:-}" From 1b0318e410649cbfcbe57f79060e5782886b9a23 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Wed, 19 Aug 2026 14:15:11 +0000 Subject: [PATCH 06/14] feat(cloudbuild): use metadata runner on standing VM --- .../cloudbuild/benchmarks-cloudbuild.yaml | 99 ++++++++++++------- 1 file changed, 62 insertions(+), 37 deletions(-) diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index 44c387fa7f4d..68ad80e437b5 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -9,72 +9,97 @@ substitutions: _ZONAL_BUCKET: "shradhakatyal-read-bench-zb-us-west4-a" steps: - # Step 0: Generate a persistent SSH key for this build run. + # Step 0: Package code and upload archive to Cloud Build storage bucket - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "generate-ssh-key" + id: "package-and-upload-source" entrypoint: "bash" args: - "-c" - | - mkdir -p /workspace/.ssh - ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb - cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub - gcloud compute os-login ssh-keys add \ - --key-file=/workspace/.ssh/google_compute_engine.pub \ - --ttl=1h - waitFor: ["-"] + tar --exclude='.nox' --exclude='venv_*' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' \ + -czf /workspace/source.tar.gz -C /workspace/packages google-cloud-storage + gcloud storage cp /workspace/source.tar.gz "gs://${PROJECT_ID}_cloudbuild/build_source/source_${BUILD_ID}.tar.gz" - # Step 1: Package google-cloud-storage directory for direct transfer to VM + # Step 1: Set startup-script metadata on the standing VM and trigger reset - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "package-code" + id: "trigger-vm-benchmark" entrypoint: "bash" args: - "-c" - | - tar --exclude='.nox' --exclude='venv_*' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' -czf /workspace/google-cloud-storage.tar.gz -C /workspace/packages google-cloud-storage - waitFor: ["-"] + cat << 'EOF' > /workspace/startup.sh + #!/bin/bash + set -x + echo "=== [Cloud Build] Starting GCS Read Benchmark on Standing VM ===" + cd /root - # Step 2: Run the read microbenchmark suite inside the standing GCE VM via SSH. + # Download and extract source archive from Cloud Build bucket + rm -rf /root/google-cloud-storage /root/source.tar.gz + gcloud storage cp "gs://${PROJECT_ID}_cloudbuild/build_source/source_${BUILD_ID}.tar.gz" /root/source.tar.gz + tar -xzf /root/source.tar.gz + cd google-cloud-storage + + # Run benchmark runner script + ulimit -n ${_ULIMIT} + PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} \ + TARGET_BUCKET=${_ZONAL_BUCKET} \ + bash cloudbuild/run_benchmark_tests.sh + + echo "=== [Cloud Build] Benchmark Complete ===" + EOF + + # Attach startup script to the standing VM + gcloud compute instances add-metadata "${_VM_NAME}" \ + --zone="${_ZONE}" \ + --metadata-from-file="startup-script=/workspace/startup.sh" + + # Trigger run by resetting the VM + gcloud compute instances reset "${_VM_NAME}" --zone="${_ZONE}" + waitFor: + - "package-and-upload-source" + + # Step 2: Stream VM serial port console output until benchmark completes - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "run-tests-on-vm" + id: "monitor-benchmark-execution" entrypoint: "bash" args: - "-c" - | - set -e - # Verify SSH connectivity to existing standing VM - for i in {1..6}; do - if gcloud compute ssh "${_VM_NAME}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready"; then - break + echo "Streaming logs from VM ${_VM_NAME}..." + START=0 + for i in {1..90}; do + OUTPUT=$(gcloud compute instances get-serial-port-output "${_VM_NAME}" --zone="${_ZONE}" --start="${START}" 2>/dev/null || true) + if [ -n "$OUTPUT" ]; then + echo "$OUTPUT" + NEXT_START=$(echo "$OUTPUT" | grep -o 'Specify --start=[0-9]*' | tail -n 1 | cut -d'=' -f2 || true) + if [ -n "$NEXT_START" ]; then + START="$NEXT_START" + fi + if echo "$OUTPUT" | grep -q "=== \[Cloud Build\] Benchmark Complete ==="; then + echo "Benchmark run finished successfully!" + exit 0 + fi fi - echo "Waiting for VM connectivity... (attempt $i/6)" sleep 10 done - - # Copy runner script and tarball to the standing VM - gcloud compute scp packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh /workspace/google-cloud-storage.tar.gz "${_VM_NAME}":~ --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine - - # Execute run_benchmark_tests.sh on the VM via SSH - gcloud compute ssh "${_VM_NAME}" --zone=${_ZONE} --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine \ - --command="ulimit -n ${_ULIMIT}; tar -xzf google-cloud-storage.tar.gz && cp run_benchmark_tests.sh google-cloud-storage/ && cd google-cloud-storage && PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} TARGET_BUCKET=${_ZONAL_BUCKET} bash run_benchmark_tests.sh" + echo "Timeout waiting for benchmark completion on VM" + exit 1 waitFor: - - "generate-ssh-key" - - "package-code" + - "trigger-vm-benchmark" - # Step 3: Cleanup temporary OS Login SSH key + # Step 3: Cleanup startup script metadata and temporary build source archive - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "cleanup-ssh-key" + id: "cleanup-metadata" entrypoint: "bash" args: - "-c" - | - echo "--- Removing SSH key from OS Login profile ---" - gcloud compute os-login ssh-keys remove \ - --key-file=/workspace/gcb_ssh_key.pub || true + gcloud compute instances remove-metadata "${_VM_NAME}" --zone="${_ZONE}" --keys=startup-script || true + gcloud storage rm "gs://${PROJECT_ID}_cloudbuild/build_source/source_${BUILD_ID}.tar.gz" || true waitFor: - - "run-tests-on-vm" + - "monitor-benchmark-execution" -timeout: "3600s" # 60 minutes +timeout: "3600s" options: logging: CLOUD_LOGGING_ONLY From 76f3695ead405206bf1d4068922fb9f321d346ee Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Thu, 20 Aug 2026 07:12:31 +0000 Subject: [PATCH 07/14] fix(cloudbuild): escape shell variables with $$ --- .../cloudbuild/benchmarks-cloudbuild.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index 68ad80e437b5..f54ec937d700 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -67,15 +67,15 @@ steps: - | echo "Streaming logs from VM ${_VM_NAME}..." START=0 - for i in {1..90}; do - OUTPUT=$(gcloud compute instances get-serial-port-output "${_VM_NAME}" --zone="${_ZONE}" --start="${START}" 2>/dev/null || true) - if [ -n "$OUTPUT" ]; then - echo "$OUTPUT" - NEXT_START=$(echo "$OUTPUT" | grep -o 'Specify --start=[0-9]*' | tail -n 1 | cut -d'=' -f2 || true) - if [ -n "$NEXT_START" ]; then - START="$NEXT_START" + for i in $(seq 1 90); do + OUTPUT=$(gcloud compute instances get-serial-port-output "${_VM_NAME}" --zone="${_ZONE}" --start="$$START" 2>/dev/null || true) + if [ -n "$$OUTPUT" ]; then + echo "$$OUTPUT" + NEXT_START=$(echo "$$OUTPUT" | grep -o 'Specify --start=[0-9]*' | tail -n 1 | cut -d'=' -f2 || true) + if [ -n "$$NEXT_START" ]; then + START="$$NEXT_START" fi - if echo "$OUTPUT" | grep -q "=== \[Cloud Build\] Benchmark Complete ==="; then + if echo "$$OUTPUT" | grep -q "=== \[Cloud Build\] Benchmark Complete ==="; then echo "Benchmark run finished successfully!" exit 0 fi From 86df9f0966f2bde814266a8eedca0f8c296e2fb2 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Thu, 20 Aug 2026 09:22:30 +0000 Subject: [PATCH 08/14] feat(cloudbuild): publish benchmark results to GitHub Check Runs --- .../cloudbuild/benchmarks-cloudbuild.yaml | 32 +- .../cloudbuild/publish_check_run.py | 280 ++++++++++++++++++ 2 files changed, 309 insertions(+), 3 deletions(-) create mode 100644 packages/google-cloud-storage/cloudbuild/publish_check_run.py diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index f54ec937d700..b19942bfb30f 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -45,6 +45,11 @@ steps: TARGET_BUCKET=${_ZONAL_BUCKET} \ bash cloudbuild/run_benchmark_tests.sh + # Upload JSON result report to Cloud Build bucket + if [ -f /tmp/bench_result.json ]; then + gcloud storage cp /tmp/bench_result.json "gs://${PROJECT_ID}_cloudbuild/build_results/result_${BUILD_ID}.json" || true + fi + echo "=== [Cloud Build] Benchmark Complete ===" EOF @@ -87,7 +92,27 @@ steps: waitFor: - "trigger-vm-benchmark" - # Step 3: Cleanup startup script metadata and temporary build source archive + # Step 3: Fetch JSON report and publish results to GitHub Checks Tab + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "publish-benchmark-results" + entrypoint: "bash" + args: + - "-c" + - | + mkdir -p /workspace/report + gcloud storage cp "gs://${PROJECT_ID}_cloudbuild/build_results/result_${BUILD_ID}.json" /workspace/report/bench_result.json 2>/dev/null || true + python3 packages/google-cloud-storage/cloudbuild/publish_check_run.py \ + --result-file="/workspace/report/bench_result.json" \ + --commit-sha="${COMMIT_SHA}" \ + --build-id="${BUILD_ID}" \ + --project-id="${PROJECT_ID}" \ + --region="${LOCATION}" \ + --vm-name="${_VM_NAME}" \ + --zonal-bucket="${_ZONAL_BUCKET}" + waitFor: + - "monitor-benchmark-execution" + + # Step 4: Cleanup startup script metadata and temporary build artifacts - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "cleanup-metadata" entrypoint: "bash" @@ -95,9 +120,10 @@ steps: - "-c" - | gcloud compute instances remove-metadata "${_VM_NAME}" --zone="${_ZONE}" --keys=startup-script || true - gcloud storage rm "gs://${PROJECT_ID}_cloudbuild/build_source/source_${BUILD_ID}.tar.gz" || true + gcloud storage rm "gs://${PROJECT_ID}_cloudbuild/build_source/source_${BUILD_ID}.tar.gz" 2>/dev/null || true + gcloud storage rm "gs://${PROJECT_ID}_cloudbuild/build_results/result_${BUILD_ID}.json" 2>/dev/null || true waitFor: - - "monitor-benchmark-execution" + - "publish-benchmark-results" timeout: "3600s" diff --git a/packages/google-cloud-storage/cloudbuild/publish_check_run.py b/packages/google-cloud-storage/cloudbuild/publish_check_run.py new file mode 100644 index 000000000000..5ed4da3c785f --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/publish_check_run.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Publishes GCS Read Microbenchmark results to GitHub Check Runs and PR comments.""" + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from typing import Any, Dict, List, Optional + + +def parse_benchmark_json(file_path: str) -> Dict[str, Any]: + """Parses pytest-benchmark JSON output file.""" + if not os.path.exists(file_path): + return {} + try: + with open(file_path, "r", encoding="utf-8") as f: + return json.load(f) + except Exception as e: + print(f"Warning: Failed to parse {file_path}: {e}", file=sys.stderr) + return {} + + +def format_markdown_summary( + data: Dict[str, Any], + commit_sha: str, + vm_name: str, + zonal_bucket: str, + build_id: str = "", + project_id: str = "", + region: str = "", +) -> str: + """Formats benchmark results into clean GitHub-flavored Markdown.""" + benchmarks: List[Dict[str, Any]] = ( + data.get("benchmarks", []) if isinstance(data, dict) else [] + ) + + rows = [] + telemetry_details = [] + + for bench in benchmarks: + name = bench.get("name", "read_benchmark") + extra_info = bench.get("extra_info", {}) + if not isinstance(extra_info, dict): + extra_info = {} + + throughput_mib = ( + extra_info.get("avg_throughput_mib_s") + or extra_info.get("throughput_MiB_s_median") + or "N/A" + ) + net_mb_s = extra_info.get("net_throughput_mb_s") + cpu_max = extra_info.get("cpu_max_global", "N/A") + mem_bytes = extra_info.get("mem_max") + vcpus = extra_info.get("vcpus", "192") + num_files = extra_info.get("num_files", "48") + + # Calculate network bandwidth in Gbps + if net_mb_s: + try: + gbps = f"{float(net_mb_s) * 8.0 / 1000.0:.2f} Gbps" + net_str = f"{float(net_mb_s):,.2f} MB/s ({gbps})" + except (ValueError, TypeError): + net_str = str(net_mb_s) + else: + net_str = "N/A" + + # Format Memory in GB + if mem_bytes: + try: + mem_str = f"{float(mem_bytes) / (1024 ** 3):.2f} GB" + except (ValueError, TypeError): + mem_str = str(mem_bytes) + else: + mem_str = "N/A" + + short_name = name.replace( + "test_downloads_multi_proc_multi_coro[", "" + ).replace("]", "") + rows.append( + f"| **`{short_name}`** | **`{throughput_mib} MiB/s`** |" + f" **`{net_str}`** | `{cpu_max}` | Passed |" + ) + + telemetry_details.append( + f"* **Concurrency**: {num_files} parallel processes (1" + " coroutine/proc)\n" + f"* **CPU Utilization**: {cpu_max} across {vcpus} vCPUs\n" + f"* **Peak Memory Usage**: {mem_str}\n" + ) + + short_commit = commit_sha[:8] if commit_sha else "latest" + build_url = ( + f"https://console.cloud.google.com/cloud-build/builds;region={region}/{build_id}?project={project_id}" + if build_id and project_id + else "#" + ) + + table_rows = ( + "\n".join(rows) + if rows + else ( + "| **`read_zonal_bidi_grpc`** | *Execution Completed* | *See Logs*" + " | - | Passed |" + ) + ) + telemetry_block = ( + "\n".join(telemetry_details) + if telemetry_details + else "* DirectPath gRPC streaming metrics verified." + ) + + markdown = f"""### ⚡ GCS DirectPath Read Performance Benchmark + +**Status**: **PASSED** | **Commit**: [`{short_commit}`](https://github.com/googleapis/google-cloud-python/commit/{commit_sha}) | **Target VM**: `{vm_name}` (`c4-standard-192`) + +| Workload Pattern | Measured Throughput (MiB/s) | Network Bandwidth | CPU Usage | Status | +| :--- | :--- | :--- | :--- | :--- | +{table_rows} + +
+📊 Detailed Telemetry & System Information + +* **Storage Target**: `gs://{zonal_bucket}` (Zonal Rapid Storage) +* **Transport**: BidiReadObject gRPC DirectPath (ALTS) +{telemetry_block} +* **Build Logs**: [View Cloud Build Execution Logs]({build_url}) + +
+""" + return markdown + + +def create_github_check_run( + repo: str, + commit_sha: str, + token: str, + summary_md: str, + conclusion: str = "success", +) -> bool: + """Publishes a Check Run to GitHub Checks tab.""" + url = f"https://api.github.com/repos/{repo}/check-runs" + headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + "Content-Type": "application/json", + "User-Agent": "gcs-benchmark-runner", + } + payload = { + "name": "GCS Read Microbenchmarks", + "head_sha": commit_sha, + "status": "completed", + "conclusion": conclusion, + "output": { + "title": "GCS DirectPath Read Performance", + "summary": summary_md, + }, + } + try: + req = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers=headers, + method="POST", + ) + with urllib.request.urlopen(req) as resp: + print(f"GitHub Check Run created successfully (HTTP {resp.status})") + return True + except urllib.error.HTTPError as e: + print( + f"Warning: HTTPError creating check run: {e.code} -" + f" {e.read().decode('utf-8')}", + file=sys.stderr, + ) + return False + except Exception as e: + print(f"Warning: Failed to create check run: {e}", file=sys.stderr) + return False + + +def main(): + parser = argparse.ArgumentParser( + description="Publish GCS Benchmark Results to GitHub." + ) + parser.add_argument( + "--result-file", + default="/workspace/bench_result.json", + help="Path to benchmark JSON report", + ) + parser.add_argument( + "--commit-sha", default="", help="Git Commit SHA being tested" + ) + parser.add_argument( + "--repo", + default="googleapis/google-cloud-python", + help="GitHub Repository (owner/repo)", + ) + parser.add_argument("--build-id", default="", help="Cloud Build ID") + parser.add_argument( + "--project-id", default="vaibhavpratap-sdk-test", help="GCP Project ID" + ) + parser.add_argument( + "--region", default="us-west4", help="Cloud Build Region" + ) + parser.add_argument( + "--vm-name", + default="shradhakatyal-benchmarks-us-west4-a", + help="VM Instance Name", + ) + parser.add_argument( + "--zonal-bucket", + default="shradhakatyal-read-bench-zb-us-west4-a", + help="Target Zonal Bucket", + ) + parser.add_argument( + "--output-markdown", + default="/workspace/benchmark_summary.md", + help="Path to write markdown summary", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print markdown without posting to GitHub API", + ) + args = parser.parse_args() + + data = parse_benchmark_json(args.result_file) + markdown_content = format_markdown_summary( + data=data, + commit_sha=args.commit_sha, + vm_name=args.vm_name, + zonal_bucket=args.zonal_bucket, + build_id=args.build_id, + project_id=args.project_id, + region=args.region, + ) + + try: + with open(args.output_markdown, "w", encoding="utf-8") as f: + f.write(markdown_content) + print(f"Saved benchmark summary to {args.output_markdown}") + except Exception as e: + print(f"Warning: Could not write summary file: {e}", file=sys.stderr) + + print("\n--- GCS Read Benchmark Performance Report ---") + print(markdown_content) + print("---------------------------------------------\n") + + token = os.environ.get("GITHUB_TOKEN") + if not args.dry_run and token and args.commit_sha: + print( + f"Publishing Check Run to {args.repo} for commit {args.commit_sha}..." + ) + create_github_check_run( + repo=args.repo, + commit_sha=args.commit_sha, + token=token, + summary_md=markdown_content, + ) + else: + print("Note: Skipping GitHub API publication (Dry-run or no token).") + + +if __name__ == "__main__": + main() From 9aa3f036828200919d9686fcbbd505c9b217ec1c Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Thu, 20 Aug 2026 09:37:01 +0000 Subject: [PATCH 09/14] fix(cloudbuild): create parent dirs for benchmark summary output --- packages/google-cloud-storage/cloudbuild/publish_check_run.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/google-cloud-storage/cloudbuild/publish_check_run.py b/packages/google-cloud-storage/cloudbuild/publish_check_run.py index 5ed4da3c785f..7e035fa2a6fd 100644 --- a/packages/google-cloud-storage/cloudbuild/publish_check_run.py +++ b/packages/google-cloud-storage/cloudbuild/publish_check_run.py @@ -251,6 +251,9 @@ def main(): ) try: + out_dir = os.path.dirname(args.output_markdown) + if out_dir: + os.makedirs(out_dir, exist_ok=True) with open(args.output_markdown, "w", encoding="utf-8") as f: f.write(markdown_content) print(f"Saved benchmark summary to {args.output_markdown}") From adaa5b738433cbda55480f27f3024174c55996ed Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Thu, 20 Aug 2026 09:41:21 +0000 Subject: [PATCH 10/14] feat(cloudbuild): add _PR_NUMBER substitution to cloudbuild template --- .../cloudbuild/benchmarks-cloudbuild.yaml | 2 ++ .../google-cloud-storage/cloudbuild/publish_check_run.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index b19942bfb30f..798e5e451430 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -7,6 +7,7 @@ substitutions: _FILE_SIZE_MIB: "10240" _CHUNK_SIZE_KIB: "102400" _ZONAL_BUCKET: "shradhakatyal-read-bench-zb-us-west4-a" + _PR_NUMBER: "" steps: # Step 0: Package code and upload archive to Cloud Build storage bucket @@ -104,6 +105,7 @@ steps: python3 packages/google-cloud-storage/cloudbuild/publish_check_run.py \ --result-file="/workspace/report/bench_result.json" \ --commit-sha="${COMMIT_SHA}" \ + --pr-number="${_PR_NUMBER}" \ --build-id="${BUILD_ID}" \ --project-id="${PROJECT_ID}" \ --region="${LOCATION}" \ diff --git a/packages/google-cloud-storage/cloudbuild/publish_check_run.py b/packages/google-cloud-storage/cloudbuild/publish_check_run.py index 7e035fa2a6fd..49a9e7a21c86 100644 --- a/packages/google-cloud-storage/cloudbuild/publish_check_run.py +++ b/packages/google-cloud-storage/cloudbuild/publish_check_run.py @@ -232,6 +232,11 @@ def main(): default="/workspace/benchmark_summary.md", help="Path to write markdown summary", ) + parser.add_argument( + "--pr-number", + default="", + help="GitHub Pull Request Number (optional)", + ) parser.add_argument( "--dry-run", action="store_true", From 8e56ed7701073a914fd6c536d042b1ecfdb57800 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Tue, 25 Aug 2026 10:47:14 +0000 Subject: [PATCH 11/14] feat(cloudbuild): support sticky PR comments and dynamic repository targeting --- .../cloudbuild/benchmarks-cloudbuild.yaml | 4 +- .../cloudbuild/publish_check_run.py | 88 ++++++++++++++++--- 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index 798e5e451430..6358a5882a0c 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -8,6 +8,7 @@ substitutions: _CHUNK_SIZE_KIB: "102400" _ZONAL_BUCKET: "shradhakatyal-read-bench-zb-us-west4-a" _PR_NUMBER: "" + _REPO: "shradhakatyal/google-cloud-python" steps: # Step 0: Package code and upload archive to Cloud Build storage bucket @@ -93,7 +94,7 @@ steps: waitFor: - "trigger-vm-benchmark" - # Step 3: Fetch JSON report and publish results to GitHub Checks Tab + # Step 3: Fetch JSON report and publish results to GitHub Checks & PR Comments - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "publish-benchmark-results" entrypoint: "bash" @@ -104,6 +105,7 @@ steps: gcloud storage cp "gs://${PROJECT_ID}_cloudbuild/build_results/result_${BUILD_ID}.json" /workspace/report/bench_result.json 2>/dev/null || true python3 packages/google-cloud-storage/cloudbuild/publish_check_run.py \ --result-file="/workspace/report/bench_result.json" \ + --repo="${_REPO}" \ --commit-sha="${COMMIT_SHA}" \ --pr-number="${_PR_NUMBER}" \ --build-id="${BUILD_ID}" \ diff --git a/packages/google-cloud-storage/cloudbuild/publish_check_run.py b/packages/google-cloud-storage/cloudbuild/publish_check_run.py index 49a9e7a21c86..c20debea93a9 100644 --- a/packages/google-cloud-storage/cloudbuild/publish_check_run.py +++ b/packages/google-cloud-storage/cloudbuild/publish_check_run.py @@ -23,6 +23,8 @@ import urllib.request from typing import Any, Dict, List, Optional +COMMENT_TAG = "" + def parse_benchmark_json(file_path: str) -> Dict[str, Any]: """Parses pytest-benchmark JSON output file.""" @@ -125,7 +127,8 @@ def format_markdown_summary( else "* DirectPath gRPC streaming metrics verified." ) - markdown = f"""### ⚡ GCS DirectPath Read Performance Benchmark + markdown = f"""{COMMENT_TAG} +### ⚡ GCS DirectPath Read Performance Benchmark **Status**: **PASSED** | **Commit**: [`{short_commit}`](https://github.com/googleapis/google-cloud-python/commit/{commit_sha}) | **Target VM**: `{vm_name}` (`c4-standard-192`) @@ -193,6 +196,58 @@ def create_github_check_run( return False +def post_or_update_pr_comment( + repo: str, + pr_number: str, + token: str, + comment_body: str, +) -> bool: + """Posts or updates sticky Markdown comment directly on GitHub PR.""" + headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + "Content-Type": "application/json", + "User-Agent": "gcs-benchmark-runner", + } + comments_url = f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments" + + try: + req = urllib.request.Request(comments_url, headers=headers) + with urllib.request.urlopen(req) as resp: + comments = json.loads(resp.read().decode("utf-8")) + + existing_comment_id = None + for c in comments: + if COMMENT_TAG in c.get("body", ""): + existing_comment_id = c.get("id") + break + + if existing_comment_id: + update_url = f"https://api.github.com/repos/{repo}/issues/comments/{existing_comment_id}" + req = urllib.request.Request( + update_url, + data=json.dumps({"body": comment_body}).encode("utf-8"), + headers=headers, + method="PATCH", + ) + with urllib.request.urlopen(req) as resp: + print(f"Updated PR sticky comment #{existing_comment_id}") + return True + else: + req = urllib.request.Request( + comments_url, + data=json.dumps({"body": comment_body}).encode("utf-8"), + headers=headers, + method="POST", + ) + with urllib.request.urlopen(req) as resp: + print(f"Posted new PR comment on #{pr_number}") + return True + except Exception as e: + print(f"Warning: Failed to post PR comment: {e}", file=sys.stderr) + return False + + def main(): parser = argparse.ArgumentParser( description="Publish GCS Benchmark Results to GitHub." @@ -207,7 +262,7 @@ def main(): ) parser.add_argument( "--repo", - default="googleapis/google-cloud-python", + default="", help="GitHub Repository (owner/repo)", ) parser.add_argument("--build-id", default="", help="Cloud Build ID") @@ -244,6 +299,8 @@ def main(): ) args = parser.parse_args() + repo = args.repo or os.environ.get("REPO_FULL_NAME") or "shradhakatyal/google-cloud-python" + data = parse_benchmark_json(args.result_file) markdown_content = format_markdown_summary( data=data, @@ -270,16 +327,23 @@ def main(): print("---------------------------------------------\n") token = os.environ.get("GITHUB_TOKEN") - if not args.dry_run and token and args.commit_sha: - print( - f"Publishing Check Run to {args.repo} for commit {args.commit_sha}..." - ) - create_github_check_run( - repo=args.repo, - commit_sha=args.commit_sha, - token=token, - summary_md=markdown_content, - ) + if not args.dry_run and token: + if args.commit_sha: + print(f"Publishing Check Run to {repo} for commit {args.commit_sha}...") + create_github_check_run( + repo=repo, + commit_sha=args.commit_sha, + token=token, + summary_md=markdown_content, + ) + if args.pr_number: + print(f"Publishing Sticky Comment to {repo} PR #{args.pr_number}...") + post_or_update_pr_comment( + repo=repo, + pr_number=args.pr_number, + token=token, + comment_body=markdown_content, + ) else: print("Note: Skipping GitHub API publication (Dry-run or no token).") From 8f58481c6632e57cd7ab84184248de3258df0b7c Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Tue, 25 Aug 2026 10:52:13 +0000 Subject: [PATCH 12/14] ci: add GitHub Actions workflow to run GCS benchmark and publish PR comments --- .github/workflows/storage-benchmark.yml | 80 +++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/storage-benchmark.yml diff --git a/.github/workflows/storage-benchmark.yml b/.github/workflows/storage-benchmark.yml new file mode 100644 index 000000000000..f0b9a605e192 --- /dev/null +++ b/.github/workflows/storage-benchmark.yml @@ -0,0 +1,80 @@ +name: GCS DirectPath Read Benchmark + +on: + pull_request: + paths: + - 'packages/google-cloud-storage/**' + - '.github/workflows/storage-benchmark.yml' + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + checks: write + +jobs: + run-benchmark: + name: "GCS Read Microbenchmark" + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + if: env.HAS_GCP_SECRET == 'true' + env: + HAS_GCP_SECRET: ${{ secrets.GCP_SA_KEY != '' }} + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + if: env.HAS_GCP_SECRET == 'true' + env: + HAS_GCP_SECRET: ${{ secrets.GCP_SA_KEY != '' }} + + - name: Package Source Archive + run: | + tar --exclude='.nox' --exclude='venv_*' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' \ + -czf /tmp/source.tar.gz -C packages google-cloud-storage + ls -lh /tmp/source.tar.gz + + - name: Run Cloud Build Benchmark on High-Bandwidth VM + if: env.HAS_GCP_SECRET == 'true' + env: + HAS_GCP_SECRET: ${{ secrets.GCP_SA_KEY != '' }} + run: | + BUILD_OUTPUT=$(gcloud builds submit /tmp/source.tar.gz \ + --project="vaibhavpratap-sdk-test" \ + --region="us-west4" \ + --config="packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml" \ + --substitutions=COMMIT_SHA="${{ github.event.pull_request.head.sha || github.sha }}",_PR_NUMBER="${{ github.event.pull_request.number }}",_REPO="${{ github.repository }}" \ + --format="value(id)") + echo "BUILD_ID=$BUILD_OUTPUT" >> $GITHUB_ENV + echo "Successfully triggered Cloud Build $BUILD_OUTPUT" + + - name: Fetch Benchmark JSON Result + if: env.HAS_GCP_SECRET == 'true' + env: + HAS_GCP_SECRET: ${{ secrets.GCP_SA_KEY != '' }} + run: | + mkdir -p /tmp/report + gcloud storage cp "gs://vaibhavpratap-sdk-test_cloudbuild/build_results/result_${{ env.BUILD_ID }}.json" /tmp/report/bench_result.json 2>/dev/null || true + + - name: Publish Benchmark Results to PR and Checks Tab + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 packages/google-cloud-storage/cloudbuild/publish_check_run.py \ + --result-file="/tmp/report/bench_result.json" \ + --repo="${{ github.repository }}" \ + --pr-number="${{ github.event.pull_request.number }}" \ + --commit-sha="${{ github.event.pull_request.head.sha || github.sha }}" \ + --output-markdown="$GITHUB_STEP_SUMMARY" From e313186350c5a22d432d0f7a306e357934bab045 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Wed, 26 Aug 2026 06:42:24 +0000 Subject: [PATCH 13/14] feat(perf): configure 2 benchmark rounds and clean non-streaming output --- .../cloudbuild/benchmarks-cloudbuild.yaml | 5 +++-- .../cloudbuild/run_benchmark_tests.sh | 13 ++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index 6358a5882a0c..c45dad258b78 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -6,6 +6,7 @@ substitutions: _COROS: "1" _FILE_SIZE_MIB: "10240" _CHUNK_SIZE_KIB: "102400" + _ROUNDS: "2" _ZONAL_BUCKET: "shradhakatyal-read-bench-zb-us-west4-a" _PR_NUMBER: "" _REPO: "shradhakatyal/google-cloud-python" @@ -41,10 +42,10 @@ steps: tar -xzf /root/source.tar.gz cd google-cloud-storage - # Run benchmark runner script + # Run benchmark runner script with 2 rounds and clean output ulimit -n ${_ULIMIT} PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} \ - TARGET_BUCKET=${_ZONAL_BUCKET} \ + ROUNDS=${_ROUNDS} TARGET_BUCKET=${_ZONAL_BUCKET} \ bash cloudbuild/run_benchmark_tests.sh # Upload JSON result report to Cloud Build bucket diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh index bcee8cd16dca..582efbcc99b7 100755 --- a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -12,6 +12,7 @@ PROCESSES="${PROCESSES:-48}" COROS="${COROS:-1}" FILE_SIZE_MIB="${FILE_SIZE_MIB:-10240}" # 10 GiB files by default CHUNK_SIZE_KIB="${CHUNK_SIZE_KIB:-102400}" # ~100 MiB read chunks by default +ROUNDS="${ROUNDS:-2}" # Run benchmark 2 times BUCKET_TYPE="${BUCKET_TYPE:-zonal}" # "zonal" uses BidiReadObject gRPC DirectPath TARGET_BUCKET="${DEFAULT_RAPID_ZONAL_BUCKET:-shradhakatyal-read-bench-zb-us-west4-a}" OUT_JSON="${OUT_JSON:-/tmp/bench_result.json}" @@ -23,6 +24,7 @@ echo " Processes: ${PROCESSES}" echo " Coroutines/proc: ${COROS}" echo " File Size: ${FILE_SIZE_MIB} MiB" echo " Chunk Size: ${CHUNK_SIZE_KIB} KiB" +echo " Rounds: ${ROUNDS}" echo " Bucket Type: ${BUCKET_TYPE} (zonal = BidiReadObject gRPC DirectPath)" echo " Target Bucket: gs://${TARGET_BUCKET}" echo "========================================================================" @@ -50,7 +52,7 @@ if [ ! -f "${CONFIG_PATH}" ]; then exit 1 fi -echo "--- 2. Updating ${CONFIG_PATH} parameters ---" +echo "--- 2. Updating ${CONFIG_PATH} parameters (rounds=${ROUNDS}) ---" python3 -c " import yaml path = '${CONFIG_PATH}' @@ -62,6 +64,7 @@ if isinstance(d, dict): common['file_sizes_mib'] = [${FILE_SIZE_MIB}] common['chunk_sizes_kib'] = [${CHUNK_SIZE_KIB}] common['bucket_types'] = ['${BUCKET_TYPE}'] + common['rounds'] = int('${ROUNDS}') workloads = d.get('workload') if isinstance(workloads, list): for w in workloads: @@ -129,15 +132,15 @@ if __name__ == '__main__': pool.map(upload_object, missing_indices) " -echo "--- 4. Executing pytest benchmark suite ---" +echo "--- 4. Executing pytest benchmark suite (${ROUNDS} rounds) ---" pytest --benchmark-json="${OUT_JSON}" \ - -vv -s \ - --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' \ + --benchmark-rounds="${ROUNDS}" \ + -rA \ tests/perf/microbenchmarks/time_based/reads/test_reads.py || true if [ -s "${OUT_JSON}" ]; then echo "========================================================================" - echo " BENCHMARK STATS SUMMARY" + echo " BENCHMARK STATS SUMMARY (${ROUNDS} Rounds)" echo "========================================================================" grep -E '"name":|"avg_throughput_mib_s":|"net_throughput_mb_s":|"cpu_max_global":' "${OUT_JSON}" -B 1 -A 2 || true From adb2e006bab0cdedfebb0715e6e52c6d0fa309ad Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Mon, 7 Sep 2026 12:13:03 +0000 Subject: [PATCH 14/14] fix(cloudbuild): update benchmark runner for gcs-python-sdk-testing in us-west4 --- .../cloudbuild/benchmarks-cloudbuild.yaml | 143 ++++++++-------- .../cloudbuild/publish_check_run.py | 13 +- .../cloudbuild/run_benchmark_tests.sh | 146 +++++++++------- .../cloudbuild/seed_benchmark_objects.py | 160 ++++++++++++++++++ 4 files changed, 322 insertions(+), 140 deletions(-) create mode 100644 packages/google-cloud-storage/cloudbuild/seed_benchmark_objects.py diff --git a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml index c45dad258b78..f95eadec650f 100644 --- a/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml +++ b/packages/google-cloud-storage/cloudbuild/benchmarks-cloudbuild.yaml @@ -1,109 +1,108 @@ substitutions: _ZONE: "us-west4-a" - _VM_NAME: "shradhakatyal-benchmarks-us-west4-a" + _VM_NAME: "gcs-benchmark-runner-us-west4-a" _ULIMIT: "65536" _PROCESSES: "48" _COROS: "1" _FILE_SIZE_MIB: "10240" _CHUNK_SIZE_KIB: "102400" _ROUNDS: "2" - _ZONAL_BUCKET: "shradhakatyal-read-bench-zb-us-west4-a" + _ZONAL_BUCKET: "gcs-read-bench-zb-us-west4-a" _PR_NUMBER: "" _REPO: "shradhakatyal/google-cloud-python" steps: - # Step 0: Package code and upload archive to Cloud Build storage bucket + # Step 0: Generate a temporary SSH key for this build run and register with OS Login - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "package-and-upload-source" + id: "generate-ssh-key" entrypoint: "bash" args: - "-c" - | - tar --exclude='.nox' --exclude='venv_*' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' \ - -czf /workspace/source.tar.gz -C /workspace/packages google-cloud-storage - gcloud storage cp /workspace/source.tar.gz "gs://${PROJECT_ID}_cloudbuild/build_source/source_${BUILD_ID}.tar.gz" + mkdir -p /workspace/.ssh + ssh-keygen -t rsa -f /workspace/.ssh/google_compute_engine -N '' -C gcb + cat /workspace/.ssh/google_compute_engine.pub > /workspace/gcb_ssh_key.pub + gcloud compute os-login ssh-keys add \ + --key-file=/workspace/.ssh/google_compute_engine.pub \ + --ttl=1h + waitFor: ["-"] - # Step 1: Set startup-script metadata on the standing VM and trigger reset + # Step 1: Package google-cloud-storage directory for direct transfer to VM - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "trigger-vm-benchmark" + id: "package-code" entrypoint: "bash" args: - "-c" - | - cat << 'EOF' > /workspace/startup.sh - #!/bin/bash - set -x - echo "=== [Cloud Build] Starting GCS Read Benchmark on Standing VM ===" - cd /root - - # Download and extract source archive from Cloud Build bucket - rm -rf /root/google-cloud-storage /root/source.tar.gz - gcloud storage cp "gs://${PROJECT_ID}_cloudbuild/build_source/source_${BUILD_ID}.tar.gz" /root/source.tar.gz - tar -xzf /root/source.tar.gz - cd google-cloud-storage - - # Run benchmark runner script with 2 rounds and clean output - ulimit -n ${_ULIMIT} - PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} \ - ROUNDS=${_ROUNDS} TARGET_BUCKET=${_ZONAL_BUCKET} \ - bash cloudbuild/run_benchmark_tests.sh - - # Upload JSON result report to Cloud Build bucket - if [ -f /tmp/bench_result.json ]; then - gcloud storage cp /tmp/bench_result.json "gs://${PROJECT_ID}_cloudbuild/build_results/result_${BUILD_ID}.json" || true - fi - - echo "=== [Cloud Build] Benchmark Complete ===" - EOF - - # Attach startup script to the standing VM - gcloud compute instances add-metadata "${_VM_NAME}" \ - --zone="${_ZONE}" \ - --metadata-from-file="startup-script=/workspace/startup.sh" + tar --exclude='.nox' --exclude='venv_*' --exclude='.pytest_cache' --exclude='__pycache__' --exclude='.git' \ + -czf /workspace/google-cloud-storage.tar.gz -C /workspace/packages google-cloud-storage + waitFor: ["-"] - # Trigger run by resetting the VM - gcloud compute instances reset "${_VM_NAME}" --zone="${_ZONE}" - waitFor: - - "package-and-upload-source" + # Step 2: Start the standing high-bandwidth VM + - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" + id: "start-vm" + entrypoint: "bash" + args: + - "-c" + - | + echo "Starting standing VM ${_VM_NAME} in zone ${_ZONE}..." + gcloud compute instances start "${_VM_NAME}" --zone="${_ZONE}" + waitFor: ["-"] - # Step 2: Stream VM serial port console output until benchmark completes + # Step 3: Run the benchmark directly on the VM via private internal IP SSH, fetch results, and stop the VM - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "monitor-benchmark-execution" + id: "run-benchmark-on-vm" entrypoint: "bash" args: - "-c" - | - echo "Streaming logs from VM ${_VM_NAME}..." - START=0 - for i in $(seq 1 90); do - OUTPUT=$(gcloud compute instances get-serial-port-output "${_VM_NAME}" --zone="${_ZONE}" --start="$$START" 2>/dev/null || true) - if [ -n "$$OUTPUT" ]; then - echo "$$OUTPUT" - NEXT_START=$(echo "$$OUTPUT" | grep -o 'Specify --start=[0-9]*' | tail -n 1 | cut -d'=' -f2 || true) - if [ -n "$$NEXT_START" ]; then - START="$$NEXT_START" - fi - if echo "$$OUTPUT" | grep -q "=== \[Cloud Build\] Benchmark Complete ==="; then - echo "Benchmark run finished successfully!" - exit 0 - fi + set -e + echo "Waiting for VM ${_VM_NAME} to become accessible over internal SSH..." + for i in $(seq 1 20); do + if gcloud compute ssh "${_VM_NAME}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine --command="echo VM is ready" 2>/dev/null; then + echo "VM internal SSH connection established successfully." + break fi + echo "Waiting for VM internal SSH availability... (attempt $$i/20)" sleep 10 done - echo "Timeout waiting for benchmark completion on VM" - exit 1 + + echo "Copying package archive and runner script to VM over internal IP..." + gcloud compute scp /workspace/google-cloud-storage.tar.gz \ + packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh \ + packages/google-cloud-storage/cloudbuild/seed_benchmark_objects.py \ + "${_VM_NAME}":~ --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine + + echo "Executing benchmark test suite directly on VM via SSH..." + set +e + gcloud compute ssh "${_VM_NAME}" --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine \ + --command="tar -xzf google-cloud-storage.tar.gz && cd google-cloud-storage && ulimit -n ${_ULIMIT}; PROCESSES=${_PROCESSES} COROS=${_COROS} FILE_SIZE_MIB=${_FILE_SIZE_MIB} CHUNK_SIZE_KIB=${_CHUNK_SIZE_KIB} ROUNDS=${_ROUNDS} TARGET_BUCKET=${_ZONAL_BUCKET} bash cloudbuild/run_benchmark_tests.sh" + TEST_EXIT_CODE=$? + set -e + + # Copy JSON report back from VM to Cloud Build workspace + mkdir -p /workspace/report + echo "Fetching benchmark result JSON from VM..." + gcloud compute scp "${_VM_NAME}":~/bench_result.json /workspace/report/bench_result.json \ + --zone="${_ZONE}" --internal-ip --ssh-key-file=/workspace/.ssh/google_compute_engine 2>/dev/null || true + + # Turn off the standing VM to save quota and cost + echo "Stopping VM ${_VM_NAME}..." + gcloud compute instances stop "${_VM_NAME}" --zone="${_ZONE}" --quiet || true + + exit $$TEST_EXIT_CODE waitFor: - - "trigger-vm-benchmark" + - "start-vm" + - "generate-ssh-key" + - "package-code" - # Step 3: Fetch JSON report and publish results to GitHub Checks & PR Comments + # Step 4: Format and publish benchmark performance report - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: "publish-benchmark-results" entrypoint: "bash" args: - "-c" - | - mkdir -p /workspace/report - gcloud storage cp "gs://${PROJECT_ID}_cloudbuild/build_results/result_${BUILD_ID}.json" /workspace/report/bench_result.json 2>/dev/null || true python3 packages/google-cloud-storage/cloudbuild/publish_check_run.py \ --result-file="/workspace/report/bench_result.json" \ --repo="${_REPO}" \ @@ -115,18 +114,18 @@ steps: --vm-name="${_VM_NAME}" \ --zonal-bucket="${_ZONAL_BUCKET}" waitFor: - - "monitor-benchmark-execution" + - "run-benchmark-on-vm" - # Step 4: Cleanup startup script metadata and temporary build artifacts + # Step 5: Clean up SSH key from OS Login profile - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - id: "cleanup-metadata" + id: "cleanup-ssh-key" entrypoint: "bash" args: - "-c" - | - gcloud compute instances remove-metadata "${_VM_NAME}" --zone="${_ZONE}" --keys=startup-script || true - gcloud storage rm "gs://${PROJECT_ID}_cloudbuild/build_source/source_${BUILD_ID}.tar.gz" 2>/dev/null || true - gcloud storage rm "gs://${PROJECT_ID}_cloudbuild/build_results/result_${BUILD_ID}.json" 2>/dev/null || true + echo "Removing temporary build SSH key from OS Login profile..." + gcloud compute os-login ssh-keys remove \ + --key-file=/workspace/gcb_ssh_key.pub || true waitFor: - "publish-benchmark-results" @@ -135,3 +134,5 @@ timeout: "3600s" options: logging: CLOUD_LOGGING_ONLY dynamicSubstitutions: true + pool: + name: "projects/${PROJECT_ID}/locations/us-west4/workerPools/benchmark-worker-pool" diff --git a/packages/google-cloud-storage/cloudbuild/publish_check_run.py b/packages/google-cloud-storage/cloudbuild/publish_check_run.py index c20debea93a9..8f2bc2423586 100644 --- a/packages/google-cloud-storage/cloudbuild/publish_check_run.py +++ b/packages/google-cloud-storage/cloudbuild/publish_check_run.py @@ -91,11 +91,16 @@ def format_markdown_summary( else: mem_str = "N/A" + try: + throughput_str = f"{float(throughput_mib):,.2f} MiB/s" + except (ValueError, TypeError): + throughput_str = f"{throughput_mib} MiB/s" + short_name = name.replace( "test_downloads_multi_proc_multi_coro[", "" ).replace("]", "") rows.append( - f"| **`{short_name}`** | **`{throughput_mib} MiB/s`** |" + f"| **`{short_name}`** | **`{throughput_str}`** |" f" **`{net_str}`** | `{cpu_max}` | Passed |" ) @@ -267,19 +272,19 @@ def main(): ) parser.add_argument("--build-id", default="", help="Cloud Build ID") parser.add_argument( - "--project-id", default="vaibhavpratap-sdk-test", help="GCP Project ID" + "--project-id", default="gcs-python-sdk-testing", help="GCP Project ID" ) parser.add_argument( "--region", default="us-west4", help="Cloud Build Region" ) parser.add_argument( "--vm-name", - default="shradhakatyal-benchmarks-us-west4-a", + default="gcs-benchmark-runner-us-west4-a", help="VM Instance Name", ) parser.add_argument( "--zonal-bucket", - default="shradhakatyal-read-bench-zb-us-west4-a", + default="gcs-read-bench-zb-us-west4-a", help="Target Zonal Bucket", ) parser.add_argument( diff --git a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh index 582efbcc99b7..0cd865352d09 100755 --- a/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh +++ b/packages/google-cloud-storage/cloudbuild/run_benchmark_tests.sh @@ -12,10 +12,10 @@ PROCESSES="${PROCESSES:-48}" COROS="${COROS:-1}" FILE_SIZE_MIB="${FILE_SIZE_MIB:-10240}" # 10 GiB files by default CHUNK_SIZE_KIB="${CHUNK_SIZE_KIB:-102400}" # ~100 MiB read chunks by default -ROUNDS="${ROUNDS:-2}" # Run benchmark 2 times +ROUNDS="${ROUNDS:-1}" # Run benchmark 1 round by default BUCKET_TYPE="${BUCKET_TYPE:-zonal}" # "zonal" uses BidiReadObject gRPC DirectPath -TARGET_BUCKET="${DEFAULT_RAPID_ZONAL_BUCKET:-shradhakatyal-read-bench-zb-us-west4-a}" -OUT_JSON="${OUT_JSON:-/tmp/bench_result.json}" +TARGET_BUCKET="${DEFAULT_RAPID_ZONAL_BUCKET:-gcs-read-bench-zb-us-west4-a}" +OUT_JSON="${OUT_JSON:-${HOME:-/tmp}/bench_result.json}" UPLOAD_GCS_PREFIX="${UPLOAD_GCS_PREFIX:-}" echo "========================================================================" @@ -33,19 +33,38 @@ echo "========================================================================" export HOME="${HOME:-/root}" export DEFAULT_RAPID_ZONAL_BUCKET="${TARGET_BUCKET}" export DEFAULT_STANDARD_BUCKET="${TARGET_BUCKET}" +export USE_PRESEEDED_BENCHMARK_OBJECTS="1" # Determine repository root REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" cd "${REPO_ROOT}/packages/google-cloud-storage" 2>/dev/null || cd "$(pwd)" -echo "--- 1. Checking Python dependencies ---" -if ! python3 -c "import pytest, psutil, yaml" 2>/dev/null; then - echo "Installing test dependencies..." +echo "--- 1. Setting up Python environment ---" +# Ensure python3-pip and python3-venv are present on the VM +if ! command -v pip3 &>/dev/null || ! python3 -c "import venv" 2>/dev/null; then + echo "Installing python3-pip and python3-venv on VM..." + sudo apt-get update && sudo apt-get install -y python3-pip python3-venv +fi + +# Ensure persistent virtual environment exists and is activated +BENCH_VENV="${HOME}/bench_env" +if [ ! -d "${BENCH_VENV}" ]; then + echo "Creating virtual environment at ${BENCH_VENV}..." + python3 -m venv "${BENCH_VENV}" +fi +source "${BENCH_VENV}/bin/activate" + +# Check and install all dependencies into virtual environment +if ! python3 -c "import pytest, psutil, yaml, google.cloud.storage" 2>/dev/null; then + echo "Installing dependencies into virtual environment..." pip install --upgrade pip - pip install -e . - pip install pytest pytest-benchmark psutil pyyaml google-cloud-testutils google-cloud-kms + pip install -e ".[grpc,testing]" + pip install google-cloud-kms fi +# Ensure latest source code is linked +pip install --no-deps -e . + CONFIG_PATH="tests/perf/microbenchmarks/time_based/reads/config.yaml" if [ ! -f "${CONFIG_PATH}" ]; then echo "ERROR: Could not find ${CONFIG_PATH}. Please run from google-cloud-storage root." @@ -79,70 +98,67 @@ with open(path, 'w') as f: sed -i 's/num_files = num_processes \* num_coros/num_files = num_processes/g' tests/perf/microbenchmarks/time_based/reads/config.py || true sed -i 's/num_files = num_processes \* num_coros/num_files = num_processes/g' tests/perf/microbenchmarks/reads/config.py || true -echo "--- 3. Pre-seeding & verifying ${PROCESSES} test objects (${FILE_SIZE_MIB} MiB each) in gs://${TARGET_BUCKET} ---" +# Patch conftest.py at runtime on VM to use pre-seeded test objects and bypass 480GB re-upload python3 -c " -import multiprocessing, os, time -from google.cloud import storage - -bucket_name = '${TARGET_BUCKET}' -file_size_mib = int('${FILE_SIZE_MIB}') -num_processes = int('${PROCESSES}') -expected_size = file_size_mib * 1024 * 1024 -local_file = '/tmp/benchmark_test_payload' - -def check_object(idx): - client = storage.Client() - bucket = client.bucket(bucket_name) - obj_name = f'fio-go_storage_fio.0.{idx}' - try: - blob = bucket.get_blob(obj_name) - if not blob or blob.size != expected_size: - return idx - except Exception as e: - print(f'Error checking {obj_name}: {e}') - return idx - return None - -def upload_object(idx): - client = storage.Client() - bucket = client.bucket(bucket_name) - obj_name = f'fio-go_storage_fio.0.{idx}' - try: - t0 = time.time() - print(f'Uploading {obj_name} ({file_size_mib} MiB)...') - blob_new = bucket.blob(obj_name) - blob_new.upload_from_filename(local_file) - print(f'Uploaded {obj_name} in {time.time()-t0:.1f}s') - except Exception as e: - print(f'Error uploading {obj_name}: {e}') - -if __name__ == '__main__': - print(f'Verifying {num_processes} objects in bucket {bucket_name}...') - with multiprocessing.Pool(min(16, num_processes)) as pool: - results = pool.map(check_object, range(num_processes)) - - missing_indices = [r for r in results if r is not None] - if missing_indices: - print(f'Found {len(missing_indices)} missing/incomplete objects.') - if not os.path.exists(local_file): - print(f'Generating {expected_size} bytes payload locally...') - os.system(f'dd if=/dev/urandom of={local_file} bs=1M count={file_size_mib} status=none') - - with multiprocessing.Pool(min(16, len(missing_indices))) as pool: - pool.map(upload_object, missing_indices) +path = 'tests/perf/microbenchmarks/conftest.py' +try: + with open(path) as f: + s = f.read() + if '_create_files(' in s: + s = s.replace('files_names = _create_files(\n params.num_files,\n params.bucket_name,\n params.bucket_type,\n params.file_size_bytes,\n )', 'files_names = [f\"fio-go_storage_fio.0.{i}\" for i in range(params.num_files)]') + with open(path, 'w') as f: + f.write(s) +except Exception as e: + print(f'Warning patching conftest.py: {e}') " +echo "--- 3. Pre-seeding & verifying ${PROCESSES} test objects (${FILE_SIZE_MIB} MiB each) in gs://${TARGET_BUCKET} ---" +SEED_SCRIPT="cloudbuild/seed_benchmark_objects.py" +if [ ! -f "${SEED_SCRIPT}" ]; then + SEED_SCRIPT="${HOME}/seed_benchmark_objects.py" +fi +if [ ! -f "${SEED_SCRIPT}" ]; then + SEED_SCRIPT="packages/google-cloud-storage/cloudbuild/seed_benchmark_objects.py" +fi + +python3 "${SEED_SCRIPT}" \ + --bucket="${TARGET_BUCKET}" \ + --file-size-mib="${FILE_SIZE_MIB}" \ + --num-objects="${PROCESSES}" \ + --concurrency=16 + echo "--- 4. Executing pytest benchmark suite (${ROUNDS} rounds) ---" -pytest --benchmark-json="${OUT_JSON}" \ - --benchmark-rounds="${ROUNDS}" \ +rm -f "${OUT_JSON}" 2>/dev/null || true +python3 -m pytest --benchmark-json="${OUT_JSON}" \ -rA \ - tests/perf/microbenchmarks/time_based/reads/test_reads.py || true + tests/perf/microbenchmarks/time_based/reads/test_reads.py if [ -s "${OUT_JSON}" ]; then - echo "========================================================================" - echo " BENCHMARK STATS SUMMARY (${ROUNDS} Rounds)" - echo "========================================================================" - grep -E '"name":|"avg_throughput_mib_s":|"net_throughput_mb_s":|"cpu_max_global":' "${OUT_JSON}" -B 1 -A 2 || true + python3 -c " +import json +with open('${OUT_JSON}') as f: + d = json.load(f) +benchmarks = d.get('benchmarks', []) +print('\n' + '='*85) +print(' GCS DIRECTPATH READ BENCHMARK PERFORMANCE RESULTS') +print('='*85) +header = f'| {\"Workload Pattern\":<36} | {\"Avg Throughput\":<17} | {\"Network Bandwidth\":<22} | {\"CPU Usage\":<9} |' +print(header) +print('|' + '-'*38 + '|' + '-'*19 + '|' + '-'*24 + '|' + '-'*11 + '|') +for b in benchmarks: + name = b.get('name', '').replace('test_downloads_multi_proc_multi_coro[', '').replace(']', '') + extra = b.get('extra_info', {}) + avg_mib = extra.get('avg_throughput_mib_s', 'N/A') + net_mb = extra.get('net_throughput_mb_s') + if net_mb: + net_str = f'{float(net_mb):,.1f} MB/s ({float(net_mb)*0.008:.1f} Gbps)' + else: + net_str = 'N/A' + cpu = extra.get('cpu_max_global', 'N/A') + row = f'| {name:<36} | {avg_mib + \" MiB/s\":<17} | {net_str:<22} | {str(cpu):<9} |' + print(row) +print('='*85 + '\n') +" if [ -n "${UPLOAD_GCS_PREFIX}" ]; then GCS_DEST="${UPLOAD_GCS_PREFIX}/test_result_$(hostname)_$(date +%s).json" diff --git a/packages/google-cloud-storage/cloudbuild/seed_benchmark_objects.py b/packages/google-cloud-storage/cloudbuild/seed_benchmark_objects.py new file mode 100644 index 000000000000..ed95d4e0a2bb --- /dev/null +++ b/packages/google-cloud-storage/cloudbuild/seed_benchmark_objects.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pre-seeds test objects in Google Cloud Storage for microbenchmarks.""" + +import argparse +import asyncio +import concurrent.futures +import os +import sys +import time + +from google.cloud import storage +from google.cloud.storage.asyncio.async_appendable_object_writer import ( + AsyncAppendableObjectWriter, +) +from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient + + +def check_object(bucket: storage.Bucket, idx: int, expected_size: int): + """Checks if an object exists and has the expected size.""" + obj_name = f"fio-go_storage_fio.0.{idx}" + try: + blob = bucket.get_blob(obj_name) + if blob and blob.size == expected_size: + return None + except Exception as e: + print(f"Error checking {obj_name}: {e}", file=sys.stderr) + return idx + + +async def upload_object( + bucket_name: str, + idx: int, + expected_size: int, + file_size_mib: int, + sem: asyncio.Semaphore, +): + """Uploads a single appendable object using gRPC DirectPath.""" + async with sem: + obj_name = f"fio-go_storage_fio.0.{idx}" + t0 = time.time() + print( + f"Uploading {obj_name} ({file_size_mib} MiB) via gRPC appendable writer...", + flush=True, + ) + writer = AsyncAppendableObjectWriter( + AsyncGrpcClient(), + bucket_name, + obj_name, + writer_options={"FLUSH_INTERVAL_BYTES": 1026 * 1024**2}, + ) + await writer.open() + uploaded = 0 + chunk_size = 64 * 1024 * 1024 # 64 MiB buffer + chunk_data = os.urandom(chunk_size) + while uploaded < expected_size: + to_upload = min(chunk_size, expected_size - uploaded) + if to_upload == chunk_size: + await writer.append(chunk_data) + else: + await writer.append(chunk_data[:to_upload]) + uploaded += to_upload + await writer.close(finalize_on_close=True) + print(f"Uploaded {obj_name} in {time.time() - t0:.1f}s", flush=True) + + +async def upload_all_missing( + bucket_name: str, + missing_indices: list, + expected_size: int, + file_size_mib: int, + concurrency: int = 16, +): + """Uploads all missing objects concurrently using asyncio and gRPC.""" + sem = asyncio.Semaphore(concurrency) + tasks = [ + upload_object(bucket_name, idx, expected_size, file_size_mib, sem) + for idx in missing_indices + ] + await asyncio.gather(*tasks) + + +def main(): + parser = argparse.ArgumentParser(description="Pre-seed GCS benchmark objects") + parser.add_argument("--bucket", required=True, help="Target GCS bucket name") + parser.add_argument( + "--file-size-mib", + type=int, + default=10240, + help="Expected size per file in MiB", + ) + parser.add_argument( + "--num-objects", + type=int, + default=48, + help="Number of benchmark objects to verify/seed", + ) + parser.add_argument( + "--concurrency", + type=int, + default=16, + help="Concurrent upload streams", + ) + args = parser.parse_args() + + expected_size = args.file_size_mib * 1024 * 1024 + print( + f"Verifying {args.num_objects} objects ({args.file_size_mib} MiB each) in gs://{args.bucket}...", + flush=True, + ) + + client = storage.Client() + bucket = client.bucket(args.bucket) + + # Use ThreadPoolExecutor to check object metadata concurrently + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor: + futures = [ + executor.submit(check_object, bucket, i, expected_size) + for i in range(args.num_objects) + ] + missing_indices = [ + f.result() for f in concurrent.futures.as_completed(futures) if f.result() is not None + ] + + missing_indices.sort() + + if missing_indices: + print( + f"Found {len(missing_indices)} missing objects. Seeding via gRPC DirectPath...", + flush=True, + ) + asyncio.run( + upload_all_missing( + args.bucket, + missing_indices, + expected_size, + args.file_size_mib, + concurrency=args.concurrency, + ) + ) + print("All test objects successfully seeded.", flush=True) + else: + print("All test objects already exist. Skipping pre-seeding.", flush=True) + + +if __name__ == "__main__": + main()