|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +# Runs test/smoke-test.sh and test/executor-smoke-test.sh on a new GCP VM booted from $IMAGE, |
| 4 | +# then deletes the VM |
| 5 | + |
| 6 | +cd "$(dirname "${BASH_SOURCE[0]}")/.." |
| 7 | +set -euo pipefail |
| 8 | + |
| 9 | +project="sourcegraph-ci" |
| 10 | +zone="us-central1-f" |
| 11 | + |
| 12 | +# GCP creates a disk, then a VM, both with the same name. Naming them after the job |
| 13 | +# lets the cleanup find them if the job is cancelled before the VM exists |
| 14 | +instance_name_prefix="docker-test-job-" |
| 15 | +instance_name="${instance_name_prefix}${BUILDKITE_JOB_ID}" |
| 16 | + |
| 17 | +# The builder account key, and an SSH key for this job only |
| 18 | +temporary_directory="$(mktemp -d)" |
| 19 | + |
| 20 | +# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses, |
| 21 | +# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM |
| 22 | +request_deletion() { |
| 23 | + { |
| 24 | + gcloud compute instances list --project="$project" --filter="$1" --format="value(selfLink)" && |
| 25 | + gcloud compute disks list --project="$project" --filter="($1) AND -users:*" --format="value(selfLink)" |
| 26 | + } | tee /dev/stderr | |
| 27 | + xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \ |
| 28 | + --request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)" |
| 29 | +} |
| 30 | + |
| 31 | +cleanup() { |
| 32 | + request_deletion "name=${instance_name}" || |
| 33 | + echo "Could not delete this job's VM. GCP deletes it an hour after it started" |
| 34 | + rm -rf "$temporary_directory" |
| 35 | +} |
| 36 | +trap cleanup EXIT |
| 37 | + |
| 38 | +# The VM's guest agent authorizes this key for the buildkite user. |
| 39 | +# The keepalive ends the connection to a VM which stopped answering, after a minute |
| 40 | +ssh-keygen -q -t ed25519 -N "" -C buildkite -f "$temporary_directory/ssh-key" |
| 41 | +ssh_options=(-i "$temporary_directory/ssh-key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR |
| 42 | + -o ConnectTimeout=5 -o ServerAliveInterval=15 -o ServerAliveCountMax=4) |
| 43 | + |
| 44 | +echo --- ":lock: builder account key" |
| 45 | +gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project="$project" >"$temporary_directory/builder-key.json" |
| 46 | +DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project="$project")" |
| 47 | +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$temporary_directory/builder-key.json" |
| 48 | + |
| 49 | +echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup" |
| 50 | +# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job. |
| 51 | +# GCP deletes the VMs itself after an hour, but not a disk whose VM was never created |
| 52 | +request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks" |
| 53 | + |
| 54 | +echo "--- :gcloud: starting VM $instance_name" |
| 55 | +# `beta`, because --max-run-duration is only there in the agents' gcloud 417. It is in every track from gcloud 483 |
| 56 | +ip_address="$(gcloud beta compute instances create "$instance_name" \ |
| 57 | + --project="$project" --zone="$zone" \ |
| 58 | + --image="$IMAGE" --image-project="$project" \ |
| 59 | + --machine-type=custom-16-20480 \ |
| 60 | + --boot-disk-size=20GB --boot-disk-type=pd-standard \ |
| 61 | + --max-run-duration=1h --instance-termination-action=DELETE \ |
| 62 | + --network=default --no-address \ |
| 63 | + --service-account="e2e-builder@${project}.iam.gserviceaccount.com" --scopes=compute-rw,storage-ro \ |
| 64 | + --labels=team=dev-infra,type=smoke-test \ |
| 65 | + --metadata="ssh-keys=buildkite:$(cat "$temporary_directory/ssh-key.pub")" \ |
| 66 | + --format="value(networkInterfaces[0].networkIP)")" |
| 67 | +remote="buildkite@${ip_address}" |
| 68 | + |
| 69 | +echo --- ":ssh: waiting for the VM to accept SSH" |
| 70 | +deadline=$((SECONDS + 300)) |
| 71 | +until ssh "${ssh_options[@]}" "$remote" true; do |
| 72 | + if ((SECONDS > deadline)); then |
| 73 | + echo "The VM did not accept SSH within 5 minutes" |
| 74 | + exit 1 |
| 75 | + fi |
| 76 | + sleep 2 |
| 77 | +done |
| 78 | + |
| 79 | +copy_repository() { |
| 80 | + tar --create --gzip --exclude=.git --exclude=node_modules . | |
| 81 | + ssh "${ssh_options[@]}" "$remote" "sudo mkdir -p /deploy-sourcegraph-docker && sudo tar --extract --gzip --directory=/deploy-sourcegraph-docker" |
| 82 | +} |
| 83 | + |
| 84 | +echo --- ":package: copying the repository to the VM" |
| 85 | +# A VM can accept SSH, then stop answering for a while. ssh exits with 255 when the connection fails, |
| 86 | +# so retry the copy on the same VM. The tests have not started yet, so nothing runs twice |
| 87 | +exit_code=0 |
| 88 | +copy_repository || exit_code=$? |
| 89 | +for retry in {1..10}; do |
| 90 | + if [ "$exit_code" != 255 ]; then |
| 91 | + break |
| 92 | + fi |
| 93 | + echo "The VM does not answer SSH, retry $retry of 10 in 15 seconds" |
| 94 | + sleep 15 |
| 95 | + exit_code=0 |
| 96 | + copy_repository || exit_code=$? |
| 97 | +done |
| 98 | + |
| 99 | +if [ "$exit_code" = 0 ]; then |
| 100 | + echo --- ":docker: running the smoke tests" |
| 101 | + # Runs as root on the VM. The Docker Hub password comes over stdin, so it does not show in the VM's process list |
| 102 | + remote_script=' |
| 103 | +set -euo pipefail |
| 104 | +read -r DOCKER_PASSWORD |
| 105 | +export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD |
| 106 | +/deploy-sourcegraph-docker/test/smoke-test.sh |
| 107 | +# The Ubuntu 20.04 VM image has no jq. Skip /etc/apt/sources.list.d: that image lacks the key for its Chrome repository, so a full update fails |
| 108 | +command -v jq >/dev/null || { apt-get update --quiet --option Dir::Etc::sourceparts=- && apt-get install --quiet --yes jq; } |
| 109 | +/deploy-sourcegraph-docker/test/executor-smoke-test.sh |
| 110 | +' |
| 111 | + # shellcheck disable=SC2029 # remote_script is quoted here, to run on the VM |
| 112 | + ssh "${ssh_options[@]}" "$remote" "sudo bash -c $(printf %q "$remote_script")" <<<"$DOCKER_PASSWORD" || exit_code=$? |
| 113 | + |
| 114 | + if [ "$exit_code" != 0 ]; then |
| 115 | + # Copy back the container logs the smoke test wrote, for the step's artifacts |
| 116 | + ssh "${ssh_options[@]}" "$remote" "cd /deploy-sourcegraph-docker && sudo tar --create --gzip -- *.log" | tar --extract --gzip || true |
| 117 | + fi |
| 118 | +fi |
| 119 | + |
| 120 | +# Not a plain `exit`, after which shellcheck reports that cleanup is never invoked (SC2329) |
| 121 | +[ "$exit_code" = 0 ] || exit "$exit_code" |
0 commit comments