Skip to content

Commit ebbd5b6

Browse files
ci: Run the smoke test VM with gcloud instead of Vagrant
smoke-test-vm.sh creates the VM with gcloud, copies the repository to it over SSH, runs test/smoke-test.sh, and asks GCP to delete the VM without waiting. GCP deletes each VM an hour after it starts, and each job deletes the VMs and disks older jobs left behind. The boot disk is 20 GB. If the VM stops answering SSH while the repository is copied to it, the script retries the copy on the same VM. Amp-Thread-ID: https://ampcode.com/threads/T-01a1c7a5-6b65-723b-980c-b85e1e9008b5 Co-authored-by: Amp <amp@ampcode.com>
1 parent 3290bd2 commit ebbd5b6

6 files changed

Lines changed: 121 additions & 130 deletions

File tree

‎.buildkite/pipeline.yaml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
$schema: "http://json.schemastore.org/buildkite"
22
env:
3-
# GCP VM image for the Vagrant smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images`
3+
# GCP VM image for the smoke test. Build it in sourcegraph/infrastructure by running `make e2e` in `gcp/images`
44
IMAGE: "e2e-db8a40720-2024-08-26t08-44-10z"
5-
VAGRANT_RUN_ENV: "CI"
65
steps:
76
- label: ":docker:"
87
command: .buildkite/validate-docker-compose.sh
@@ -16,9 +15,11 @@ steps:
1615
# so it finds no changes in a commit which is already on main, unless it is the newest
1716
- &smoke_test
1817
label: ":rice: docker-compose-test"
19-
command: .buildkite/vagrant-run.sh docker-test
18+
command: .buildkite/smoke-test-vm.sh
2019
artifact_paths: ./*.log
2120
agents: { queue: "vagrant" }
21+
# smoke-test-vm.sh deletes VMs and disks older than an hour, so no job may run that long
22+
timeout_in_minutes: 30
2223
if: build.branch == "main"
2324

2425
# Builds on other branches skip the smoke test if they only change docs

‎.buildkite/smoke-test-vm.sh‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#!/usr/bin/env bash
2+
3+
# Runs test/smoke-test.sh on a new GCP VM booted from $IMAGE, then deletes the VM
4+
5+
cd "$(dirname "${BASH_SOURCE[0]}")/.."
6+
set -euo pipefail
7+
8+
project="sourcegraph-ci"
9+
zone="us-central1-f"
10+
11+
# GCP creates a disk, then a VM, both with the same name. Naming them after the job
12+
# lets the cleanup find them if the job is cancelled before the VM exists
13+
instance_name_prefix="docker-test-job-"
14+
instance_name="${instance_name_prefix}${BUILDKITE_JOB_ID}"
15+
16+
# The builder account key, and an SSH key for this job only
17+
temporary_directory="$(mktemp -d)"
18+
19+
# Ask GCP to delete the VMs matching a filter, and the matching disks which no VM uses,
20+
# without waiting the 1-2 minutes a deletion takes. GCP deletes a VM's disk with the VM
21+
request_deletion() {
22+
{
23+
gcloud compute instances list --project="$project" --filter="$1" --format="value(selfLink)" &&
24+
gcloud compute disks list --project="$project" --filter="($1) AND -users:*" --format="value(selfLink)"
25+
} | tee /dev/stderr |
26+
xargs --no-run-if-empty --max-args=1 curl --fail --silent --show-error --output /dev/null \
27+
--request DELETE --header "Authorization: Bearer $(gcloud auth print-access-token)"
28+
}
29+
30+
cleanup() {
31+
request_deletion "name=${instance_name}" ||
32+
echo "Could not delete this job's VM. GCP deletes it an hour after it started"
33+
rm -rf "$temporary_directory"
34+
}
35+
trap cleanup EXIT
36+
37+
# The VM's guest agent authorizes this key for the buildkite user.
38+
# The keepalive ends the connection to a VM which stopped answering, after a minute
39+
ssh-keygen -q -t ed25519 -N "" -C buildkite -f "$temporary_directory/ssh-key"
40+
ssh_options=(-i "$temporary_directory/ssh-key" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR
41+
-o ConnectTimeout=5 -o ServerAliveInterval=15 -o ServerAliveCountMax=4)
42+
43+
echo --- ":lock: builder account key"
44+
gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project="$project" >"$temporary_directory/builder-key.json"
45+
DOCKER_PASSWORD="$(gcloud secrets versions access latest --secret=DOCKER_PASSWORD_BUILDKITE_AGENT --quiet --project="$project")"
46+
export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$temporary_directory/builder-key.json"
47+
48+
echo --- ":wastebasket: deleting VMs and disks left behind by jobs which died before their cleanup"
49+
# The smoke test step times out after 30 minutes, so a VM or disk older than an hour has no job.
50+
# GCP deletes the VMs itself after an hour, but not a disk whose VM was never created
51+
request_deletion "name~^${instance_name_prefix} AND creationTimestamp<-PT1H" || echo "Could not delete stale VMs and disks"
52+
53+
echo "--- :gcloud: starting VM $instance_name"
54+
# `beta`, because --max-run-duration is only there in the agents' gcloud 417. It is in every track from gcloud 483
55+
ip_address="$(gcloud beta compute instances create "$instance_name" \
56+
--project="$project" --zone="$zone" \
57+
--image="$IMAGE" --image-project="$project" \
58+
--machine-type=custom-16-20480 \
59+
--boot-disk-size=20GB --boot-disk-type=pd-standard \
60+
--max-run-duration=1h --instance-termination-action=DELETE \
61+
--network=default --no-address \
62+
--service-account="e2e-builder@${project}.iam.gserviceaccount.com" --scopes=compute-rw,storage-ro \
63+
--labels=team=dev-infra,type=smoke-test \
64+
--metadata="ssh-keys=buildkite:$(cat "$temporary_directory/ssh-key.pub")" \
65+
--format="value(networkInterfaces[0].networkIP)")"
66+
remote="buildkite@${ip_address}"
67+
68+
echo --- ":ssh: waiting for the VM to accept SSH"
69+
deadline=$((SECONDS + 300))
70+
until ssh "${ssh_options[@]}" "$remote" true; do
71+
if ((SECONDS > deadline)); then
72+
echo "The VM did not accept SSH within 5 minutes"
73+
exit 1
74+
fi
75+
sleep 2
76+
done
77+
78+
copy_repository() {
79+
tar --create --gzip --exclude=.git --exclude=node_modules . |
80+
ssh "${ssh_options[@]}" "$remote" "sudo mkdir -p /deploy-sourcegraph-docker && sudo tar --extract --gzip --directory=/deploy-sourcegraph-docker"
81+
}
82+
83+
echo --- ":package: copying the repository to the VM"
84+
# A VM can accept SSH, then stop answering for a while. ssh exits with 255 when the connection fails,
85+
# so retry the copy on the same VM. The tests have not started yet, so nothing runs twice
86+
exit_code=0
87+
copy_repository || exit_code=$?
88+
for retry in {1..10}; do
89+
if [ "$exit_code" != 255 ]; then
90+
break
91+
fi
92+
echo "The VM does not answer SSH, retry $retry of 10 in 15 seconds"
93+
sleep 15
94+
exit_code=0
95+
copy_repository || exit_code=$?
96+
done
97+
98+
if [ "$exit_code" = 0 ]; then
99+
echo --- ":docker: running the smoke test"
100+
# Runs as root on the VM. The Docker Hub password comes over stdin, so it does not show in the VM's process list
101+
remote_script='
102+
set -euo pipefail
103+
read -r DOCKER_PASSWORD
104+
export DOCKER_USERNAME=sourcegraph DOCKER_PASSWORD
105+
/deploy-sourcegraph-docker/test/smoke-test.sh
106+
'
107+
# shellcheck disable=SC2029 # remote_script is quoted here, to run on the VM
108+
ssh "${ssh_options[@]}" "$remote" "sudo bash -c $(printf %q "$remote_script")" <<<"$DOCKER_PASSWORD" || exit_code=$?
109+
110+
if [ "$exit_code" != 0 ]; then
111+
# Copy back the container logs the smoke test wrote, for the step's artifacts
112+
ssh "${ssh_options[@]}" "$remote" "cd /deploy-sourcegraph-docker && sudo tar --create --gzip -- *.log" | tar --extract --gzip || true
113+
fi
114+
fi
115+
116+
# Not a plain `exit`, after which shellcheck reports that cleanup is never invoked (SC2329)
117+
[ "$exit_code" = 0 ] || exit "$exit_code"

‎.buildkite/vagrant-run.sh‎

Lines changed: 0 additions & 38 deletions
This file was deleted.

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
.env
22
.envrc
3-
.vagrant
43
*.log
54
node_modules/

‎test/Vagrantfile‎

Lines changed: 0 additions & 74 deletions
This file was deleted.

‎test/servers.yaml‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

0 commit comments

Comments
 (0)