From 3fcb01305c173e40b3ea5779ca174af4fa2f6dc2 Mon Sep 17 00:00:00 2001 From: aram price Date: Fri, 19 Jun 2026 17:02:11 -0700 Subject: [PATCH] CI: remove light-aws-builder specs, use binary from image. These specs moved to that pipeline, and the docker image now contains the `light-aws-builder` binary. Also contains: - shellcheck cleanup - removes aws-light-stemcell-builder src --- ci/pipelines/publisher.yml | 158 ++---------------- ci/tasks/light-aws/build.sh | 94 +++++------ ci/tasks/light-aws/build.yml | 1 - ci/tasks/light-aws/cleanup-ami.sh | 121 +++++++------- ci/tasks/light-aws/cleanup-ami.yml | 19 +-- ci/tasks/light-aws/run-upload-test.sh | 2 +- ci/tasks/light-aws/run-upload-test.yml | 3 +- ci/tasks/light-aws/test-drivers.sh | 63 ------- ci/tasks/light-aws/test-drivers.yml | 23 --- ci/tasks/light-aws/test-integration.sh | 48 ------ ci/tasks/light-aws/test-integration.yml | 18 -- ci/tasks/light-aws/test-unit.sh | 18 -- ci/tasks/light-aws/test-unit.yml | 8 - ci/tasks/light-aws/us-gov-merge-builds.yml | 1 - ci/tasks/light-google/create-public-image.sh | 18 +- ci/tasks/light-google/deploy-skeletal.sh | 4 +- .../make-raw-from-heavy-stemcell.sh | 29 ++-- 17 files changed, 161 insertions(+), 467 deletions(-) mode change 100755 => 100644 ci/tasks/light-aws/cleanup-ami.sh delete mode 100755 ci/tasks/light-aws/test-drivers.sh delete mode 100644 ci/tasks/light-aws/test-drivers.yml delete mode 100755 ci/tasks/light-aws/test-integration.sh delete mode 100644 ci/tasks/light-aws/test-integration.yml delete mode 100755 ci/tasks/light-aws/test-unit.sh delete mode 100644 ci/tasks/light-aws/test-unit.yml diff --git a/ci/pipelines/publisher.yml b/ci/pipelines/publisher.yml index 83c1dca19f..a1736370fc 100644 --- a/ci/pipelines/publisher.yml +++ b/ci/pipelines/publisher.yml @@ -3,44 +3,29 @@ #@yaml/text-templated-strings --- -#@ def build_light_aws_stemcell_new(stemcell_os, stemcell_version, builder_src, input_stemcell, output_stemcell, prefix, region, bucket_prefix, tag, ami_destinations, efi, ami_excluded_destinations): +#@ def build_light_aws_stemcell_new(stemcell_os, stemcell_version, prefix, efi, ami_destinations, ami_excluded_destinations): do: - in_parallel: - - get: (@= input_stemcell @) + - get: (@= prefix @)-input-stemcell params: include_files: - bosh-stemcell-*-aws-xen-hvm-(@= stemcell_os @)*.tgz resource: candidate-(@= stemcell_os @)-stemcell-(@= stemcell_version @) trigger: true version: every - #@ if tag != "": - tags: - - (@= tag @) - #@ end - - get: (@= builder_src @) - passed: - - test-aws-unit - - test-aws-integration - - test-aws-drivers - resource: light-aws-builder-src - #@ if tag != "": - tags: - - (@= tag @) - #@ end - - task: build-(@= region @)-stemcell + - task: build-(@= prefix @)-stemcell file: bosh-stemcells-ci/ci/tasks/light-aws/build.yml - image: light-stemcell-builder-registry-image + image: aws-light-stemcell-builder-registry-image input_mapping: - builder-src: (@= builder_src @) - input-stemcell: (@= input_stemcell @) + input-stemcell: (@= prefix @)-input-stemcell output_mapping: - light-stemcell: (@= output_stemcell @) + light-stemcell: (@= prefix @)-light-stemcell params: AWS_PAGER: ami_access_key: ((aws_publish_(@= prefix @)_access_key)) ami_secret_key: ((aws_publish_(@= prefix @)_secret_key)) - ami_region: ((aws_publish_(@= region @)_region)) - ami_bucket_name: ((aws_publish_(@= bucket_prefix @)_bucket)) + ami_region: ((aws_publish_(@= prefix @)_region)) + ami_bucket_name: ((aws_publish_(@= prefix @)_bucket)) ami_description: Light Stemcell Builder Prod AMI #@ if ami_destinations != "": ami_destinations: (@= ami_destinations @) @@ -55,10 +40,6 @@ do: ami_visibility: public S3_API_ENDPOINT: storage.googleapis.com efi: (@= str(efi).lower() @) - #@ if tag != "": - tags: - - (@= tag @) - #@ end #@ end #@yaml/text-templated-strings @@ -94,17 +75,9 @@ params: #@yaml/text-templated-strings --- -anchors: - ci_bot: - email: &ci_bot_email bots@cloudfoundry.org - name: &ci_bot_name CI Bot - groups: - name: all jobs: - - test-aws-unit - - test-aws-drivers - - test-aws-integration - build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @) - build-light-google-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @) - publish-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @) @@ -120,7 +93,6 @@ groups: - publish-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @) - name: cleanup-aws-light-stemcells jobs: - - cleanup-light-aws-builder-test-amis - cleanup-published-aws-light-stemcells-older-than-three-years - cleanup-unpublished-(@= data.values.stemcell_details.os_name @)-aws-light-stemcells @@ -145,104 +117,18 @@ jobs: - #@ cleanup_unpublished_light_stemcells("us-gov", "us-gov") #!- #@ cleanup_unpublished_light_stemcells("cn", "cn_north") -- name: test-aws-unit - plan: - - get: bosh-stemcells-ci - - get: bosh-integration-registry-image - - get: builder-src - resource: light-aws-builder-src - trigger: true - - file: bosh-stemcells-ci/ci/tasks/light-aws/test-unit.yml - image: bosh-integration-registry-image - task: test - serial: true - -- name: test-aws-drivers - plan: - - get: bosh-stemcells-ci - - get: bosh-integration-registry-image - - get: builder-src - resource: light-aws-builder-src - trigger: true - - file: bosh-stemcells-ci/ci/tasks/light-aws/test-drivers.yml - image: bosh-integration-registry-image - params: - AWS_PAGER: - aws_account_id: ((aws_test_account_id)) - access_key: ((aws_test_access_key)) - secret_key: ((aws_test_secret_key)) - bucket_name: ((aws_test_bucket_name)) - copy_region: ((aws_test_copy_region)) - region: ((aws_test_region)) - ami_fixture_id: ((aws_test_ami_fixture_id)) - private_ami_fixture_id: ((aws_test_private_ami_fixture_id)) - existing_snapshot_id: ((aws_test_snapshot_fixture_id)) - existing_volume_id: ((aws_test_volume_fixture_id)) - #! kms key id should be the one created in the region of "copy_region" - kms_key_id: ((aws_test_kms_key_id)) - kms_multi_region_key: ((aws_test_kms_multi_region_key_id)) - kms_multi_region_key_replication_test: ((aws_test_kms_multi_region_replication_test_key_id)) - uploaded_machine_image_url: https://stemcell-test-publish.s3.eu-central-1.amazonaws.com/fixtures/root.img - task: test - attempts: 3 - serial: true - -- name: test-aws-integration - plan: - - get: bosh-stemcells-ci - - get: bosh-integration-registry-image - - get: builder-src - resource: light-aws-builder-src - trigger: true - - file: bosh-stemcells-ci/ci/tasks/light-aws/test-integration.yml - image: bosh-integration-registry-image - params: - AWS_PAGER: - access_key: ((aws_test_access_key)) - secret_key: ((aws_test_secret_key)) - bucket_name: ((aws_test_bucket_name)) - #! cn_access_key: ((test__cn_access_key)) - #! cn_bucket_name: ((test__cn_bucket_name)) - #! cn_region: ((test__cn_region)) - #! cn_secret_key: ((test__cn_secret_key)) - copy_region: ((aws_test_copy_region)) - region: ((aws_test_region)) - - task: test - serial: true - -- name: cleanup-light-aws-builder-test-amis - plan: - - get: every-week-on-monday - trigger: true - - get: bosh-stemcells-ci - - get: bosh-integration-registry-image - - task: cleanup-aws-test-amis - file: bosh-stemcells-ci/ci/tasks/light-aws/cleanup-ami.yml - image: bosh-integration-registry-image - params: - AWS_PAGER: - ami_access_key: ((aws_test_access_key)) - ami_secret_key: ((aws_test_secret_key)) - ami_region: ((aws_test_region)) - ami_older_than_days: 1 - ami_keep_latest: 0 - snapshot_id: ((aws_test_snapshot_fixture_id)) - - name: build-light-aws-(@= data.values.stemcell_details.os_name @)-(@= str(data.values.stemcell_details.major_version) @) plan: - get: bosh-stemcells-ci - - get: light-stemcell-builder-registry-image + - get: aws-light-stemcell-builder-registry-image - get: bosh-integration-registry-image - in_parallel: - do: - - #@ build_light_aws_stemcell_new(data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us-gov-builder-src", "us-gov-input-stemcell", "us-gov-light-stemcell", "us-gov", "us-gov", "us-gov", "", "", data.values.stemcell_details.use_efi, "") - - #@ build_light_aws_stemcell_new(data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us-builder-src", "us-input-stemcell", "us-light-stemcell", "us", "us", "us", "", "", data.values.stemcell_details.use_efi, '["me-central-1"]') - - file: bosh-stemcells-ci/ci/tasks/light-aws/us-gov-merge-builds.yml - image: light-stemcell-builder-registry-image - input_mapping: - builder-src: us-builder-src - task: merge-builds + - #@ build_light_aws_stemcell_new(data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us-gov", data.values.stemcell_details.use_efi, "", "") + - #@ build_light_aws_stemcell_new(data.values.stemcell_details.os_name, str(data.values.stemcell_details.major_version), "us", data.values.stemcell_details.use_efi, "", '["me-central-1"]') + - task: merge-builds + file: bosh-stemcells-ci/ci/tasks/light-aws/us-gov-merge-builds.yml + image: aws-light-stemcell-builder-registry-image - do: - in_parallel: - get: bosh-cpi-src @@ -285,7 +171,6 @@ jobs: file: bosh-stemcells-ci/ci/tasks/light-aws/run-upload-test.yml image: bosh-integration-registry-image input_mapping: - builder-src: us-builder-src stemcell: light-stemcell ensure: do: @@ -315,8 +200,8 @@ jobs: files: - light-stemcell/*.tgz options: - author_email: *ci_bot_email - author_name: *ci_bot_name + author_email: &ci_bot_email bots@cloudfoundry.org + author_name: &ci_bot_name CI Bot message: 'candidate (light aws): (@= data.values.stemcell_details.os_name @)/(@= str(data.values.stemcell_details.major_version) @).x' version: us-input-stemcell/.resource/version serial: true @@ -334,12 +219,11 @@ jobs: - get: bosh-stemcells-ci - get: bosh-cpi-release resource: bosh-google-cpi-release - - get: light-stemcell-builder-registry-image - get: gce-cpi-release-registry-image - get: bosh-integration-registry-image - task: make-raw-from-heavy-stemcell file: bosh-stemcells-ci/ci/tasks/light-google/make-raw-from-heavy-stemcell.yml - image: light-stemcell-builder-registry-image + image: gce-cpi-release-registry-image params: BUCKET_NAME: bosh-gce-raw-stemcells-new STEMCELL_BUCKET_PATH: bosh-gce-light-stemcells @@ -623,12 +507,6 @@ resources: - ci uri: https://github.com/cloudfoundry/bosh-linux-stemcell-builder.git -- name: light-aws-builder-src - type: git - source: - branch: master - uri: https://github.com/cloudfoundry/bosh-aws-light-stemcell-builder - - name: bosh-aws-cpi-release type: git source: @@ -854,7 +732,7 @@ resources: username: ((github_read_write_packages.username)) password: ((github_read_write_packages.password)) -- name: light-stemcell-builder-registry-image +- name: aws-light-stemcell-builder-registry-image type: registry-image source: repository: bosh/light-stemcell-builder diff --git a/ci/tasks/light-aws/build.sh b/ci/tasks/light-aws/build.sh index aff06ce410..7383e3e06f 100755 --- a/ci/tasks/light-aws/build.sh +++ b/ci/tasks/light-aws/build.sh @@ -14,20 +14,20 @@ ami_kms_key_id=${ami_kms_key_id:-} ami_server_side_encryption=${ami_server_side_encryption:-} ami_excluded_destinations=${ami_excluded_destinations:-} -: ${bosh_io_bucket_name:?} -: ${ami_description:?} -: ${ami_virtualization_type:?} -: ${ami_visibility:?} -: ${ami_region:?} -: ${ami_access_key:?} -: ${ami_secret_key:?} -: ${ami_bucket_name:?} -: ${ami_encrypted:?} -: ${efi:?} - -export AWS_ACCESS_KEY_ID=$ami_access_key -export AWS_SECRET_ACCESS_KEY=$ami_secret_key -export AWS_DEFAULT_REGION=$ami_region +: "${bosh_io_bucket_name:?}" +: "${ami_description:?}" +: "${ami_virtualization_type:?}" +: "${ami_visibility:?}" +: "${ami_region:?}" +: "${ami_access_key:?}" +: "${ami_secret_key:?}" +: "${ami_bucket_name:?}" +: "${ami_encrypted:?}" +: "${efi:?}" + +export AWS_ACCESS_KEY_ID=${ami_access_key} +export AWS_SECRET_ACCESS_KEY=${ami_secret_key} +export AWS_DEFAULT_REGION=${ami_region} saved_ami_destinations="$( aws ec2 describe-regions \ --query "Regions[?RegionName != '${ami_region}'][].RegionName" \ @@ -38,7 +38,7 @@ if [[ -n "${ami_excluded_destinations}" ]]; then | jq --argjson exclude "$ami_excluded_destinations" '. - $exclude' -c )" fi -: ${ami_destinations:=$saved_ami_destinations} +: "${ami_destinations:=$saved_ami_destinations}" stemcell_path=$(ls "${REPO_PARENT}"/input-stemcell/*.tgz) version=$(cat "${REPO_PARENT}/input-stemcell/.resource/version") @@ -55,14 +55,11 @@ if [ "${ami_virtualization_type}" = "hvm" ]; then fi bosh_io_light_stemcell_url="https://$S3_API_ENDPOINT/$bosh_io_bucket_name/$version/$light_stemcell_name" -set +e -wget --spider "$bosh_io_light_stemcell_url" -if [[ "$?" == "0" ]]; then +if wget --spider "$bosh_io_light_stemcell_url"; then echo "AWS light stemcell '$light_stemcell_name' already exists!" echo "You can download here: $bosh_io_light_stemcell_url" exit 1 fi -set -e echo "Building light stemcell..." echo " Starting region: ${ami_region}" @@ -70,40 +67,40 @@ echo " Copy regions: ${ami_destinations}" export CONFIG_PATH="${REPO_PARENT}/config.json" -cat > $CONFIG_PATH << EOF +cat > "${CONFIG_PATH}" << EOF { "ami_configuration": { - "description": "$ami_description", - "virtualization_type": "$ami_virtualization_type", - "encrypted": $ami_encrypted, - "kms_key_id": "$ami_kms_key_id", - "visibility": "$ami_visibility", + "description": "${ami_description}", + "virtualization_type": "${ami_virtualization_type}", + "encrypted": ${ami_encrypted}, + "kms_key_id": "${ami_kms_key_id}", + "visibility": "${ami_visibility}", "efi": ${efi} }, "ami_regions": [ { - "name": "$ami_region", + "name": "${ami_region}", "credentials": { - "access_key": "$ami_access_key", - "secret_key": "$ami_secret_key" + "access_key": "${ami_access_key}", + "secret_key": "${ami_secret_key}" }, - "bucket_name": "$ami_bucket_name", - "server_side_encryption": "$ami_server_side_encryption", - "destinations": $ami_destinations + "bucket_name": "${ami_bucket_name}", + "server_side_encryption": "${ami_server_side_encryption}", + "destinations": ${ami_destinations} } ] } EOF extracted_stemcell_dir="${REPO_PARENT}/extracted-stemcell" -mkdir -p ${extracted_stemcell_dir} -tar -C ${extracted_stemcell_dir} -xf ${stemcell_path} -tar -xf ${extracted_stemcell_dir}/image +mkdir -p "${extracted_stemcell_dir}" +tar -C "${extracted_stemcell_dir}" -xf "${stemcell_path}" +tar -xf "${extracted_stemcell_dir}"/image # image format can be raw or stream optimized vmdk stemcell_image="$(echo "${REPO_PARENT}"/root.*)" stemcell_manifest=${extracted_stemcell_dir}/stemcell.MF -manifest_contents="$(cat ${stemcell_manifest})" +manifest_contents="$(cat "${stemcell_manifest}")" disk_regex="disk: ([0-9]+)" format_regex="disk_format: ([a-z]+)" @@ -120,23 +117,22 @@ disk_size_gb=$(mb_to_gb "${BASH_REMATCH[1]}") [[ "${manifest_contents}" =~ ${format_regex} ]] disk_format="${BASH_REMATCH[1]}" -pushd "${REPO_PARENT}/builder-src" > /dev/null - # Make sure we've closed the manifest file before writing to it - go run main.go \ - -c $CONFIG_PATH \ - --image ${stemcell_image} \ - --format ${disk_format} \ - --volume-size ${disk_size_gb} \ - --manifest ${stemcell_manifest} \ - | tee tmp-manifest +# Make sure we've closed the manifest file before writing to it +# see https://github.com/cloudfoundry/bosh-aws-light-stemcell-builder/blob/master/ci/docker/Dockerfile#L30 +light-stemcell-builder \ + -c "${CONFIG_PATH}" \ + --image "${stemcell_image}" \ + --format "${disk_format}" \ + --volume-size "${disk_size_gb}" \ + --manifest "${stemcell_manifest}" \ + | tee tmp-manifest - mv tmp-manifest ${stemcell_manifest} +mv tmp-manifest "${stemcell_manifest}" -popd - -pushd ${extracted_stemcell_dir} - > image +pushd "${extracted_stemcell_dir}" + : > image # the bosh cli sees the stemcell as invalid if tar contents have leading ./ + # shellcheck disable=SC2035 tar -czf "${REPO_PARENT}/light-stemcell/${light_stemcell_name}" * popd diff --git a/ci/tasks/light-aws/build.yml b/ci/tasks/light-aws/build.yml index 5d553e7a9b..19e321c918 100644 --- a/ci/tasks/light-aws/build.yml +++ b/ci/tasks/light-aws/build.yml @@ -2,7 +2,6 @@ platform: linux inputs: -- name: builder-src - name: bosh-stemcells-ci - name: input-stemcell diff --git a/ci/tasks/light-aws/cleanup-ami.sh b/ci/tasks/light-aws/cleanup-ami.sh old mode 100755 new mode 100644 index c1a24a3d9e..7d566079d6 --- a/ci/tasks/light-aws/cleanup-ami.sh +++ b/ci/tasks/light-aws/cleanup-ami.sh @@ -6,16 +6,14 @@ REPO_PARENT="$( cd "${REPO_ROOT}/.." && pwd )" if [[ -n "${DEBUG:-}" ]]; then set -x - export BOSH_LOG_LEVEL=debug - export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}" fi -: ${ami_older_than_days:?} -: ${ami_keep_latest:?} +: "${ami_older_than_days:?}" +: "${ami_keep_latest:?}" -export AWS_ACCESS_KEY_ID=${ami_access_key} -export AWS_SECRET_ACCESS_KEY=${ami_secret_key} -export AWS_DEFAULT_REGION=${ami_region} +export AWS_ACCESS_KEY_ID=${ami_access_key?'must be set'} +export AWS_SECRET_ACCESS_KEY=${ami_secret_key?'must be set'} +export AWS_DEFAULT_REGION=${ami_region?'must be set'} if [ -n "${ami_role_arn:-}" ]; then export AWS_ROLE_ARN=${ami_role_arn} @@ -29,66 +27,67 @@ if [ -n "${ami_role_arn:-}" ]; then export AWS_PROFILE=resource_account fi -__PASTDUE=$(date --date="$ami_older_than_days days ago" +"%Y-%m-%d") - +past_due=$(date --date="${ami_older_than_days} days ago" +"%Y-%m-%d") +# shellcheck disable=SC2016 +past_due_query='sort_by(Images,&CreationDate)[?CreationDate<`'"${past_due}"'`].{ImageId: ImageId, date:CreationDate, SnapshotId: BlockDeviceMappings[0].Ebs.SnapshotId,Version: Tags[?Key==`name`]|[0].Value}' ami_destinations="$(aws ec2 describe-regions --output text --query "Regions[?RegionName][].RegionName")" -for region in $ami_destinations; do - ami_list="[]" +for region in ${ami_destinations}; do + ami_list="[]" - if [ "${remove_public_images:-}" == "true" ]; then - results=$(aws ec2 describe-images \ - --owners self \ - --output json \ - --region ${region} \ - --filters "Name=name,Values=BOSH*" "Name=is-public,Values=true" \ - --query 'sort_by(Images,&CreationDate)[?CreationDate<`'"$__PASTDUE"'`].{ImageId: ImageId, date:CreationDate, SnapshotId: BlockDeviceMappings[0].Ebs.SnapshotId,Version: Tags[?Key==`name`]|[0].Value}') - ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}")) - fi + if [ "${remove_public_images:-}" == "true" ]; then + results=$(aws ec2 describe-images \ + --owners self \ + --output json \ + --region "${region}" \ + --filters "Name=name,Values=BOSH*" "Name=is-public,Values=true" \ + --query "${past_due_query}") + ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}")) + fi - if [ -n "${os_name:-}" ]; then - # 'ami_ids' array should be orderered by creation date - results=$(aws ec2 describe-images \ - --owners self \ - --output json \ - --region ${region} \ - --filters "Name=name,Values=BOSH*" "Name=tag:published,Values=false" "Name=tag:distro,Values=${os_name}" \ - --query 'sort_by(Images,&CreationDate)[?CreationDate<`'"$__PASTDUE"'`].{ImageId: ImageId, date:CreationDate, SnapshotId: BlockDeviceMappings[0].Ebs.SnapshotId,Version: Tags[?Key==`name`]|[0].Value}' | jq 'reverse | del(.[range(env.ami_keep_latest|tonumber)])') - ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}")) - fi + if [ -n "${os_name:-}" ]; then + # 'ami_ids' array should be ordered by creation date + results=$(aws ec2 describe-images \ + --owners self \ + --output json \ + --region "${region}" \ + --filters "Name=name,Values=BOSH*" "Name=tag:published,Values=false" "Name=tag:distro,Values=${os_name}" \ + --query "${past_due_query}" | jq 'reverse | del(.[range(env.ami_keep_latest|tonumber)])') + ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}")) + fi - if [ -n "${snapshot_id:-}" ]; then - results=$(aws ec2 describe-images \ - --owners self \ - --output json \ - --region ${region} \ - --filters "Name=block-device-mapping.snapshot-id,Values=${snapshot_id}" \ - --query 'sort_by(Images,&CreationDate)[?CreationDate<`'"$__PASTDUE"'`].{ImageId: ImageId, date:CreationDate, SnapshotId: BlockDeviceMappings[0].Ebs.SnapshotId,Version: Tags[?Key==`name`]|[0].Value}' | jq 'reverse | del(.[range(env.ami_keep_latest|tonumber)])') - ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}")) - fi + if [ -n "${snapshot_id:-}" ]; then + results=$(aws ec2 describe-images \ + --owners self \ + --output json \ + --region "${region}" \ + --filters "Name=block-device-mapping.snapshot-id,Values=${snapshot_id}" \ + --query "${past_due_query}" | jq 'reverse | del(.[range(env.ami_keep_latest|tonumber)])') + ami_list=$(jq -s '.[0] + .[1]' <(echo "${ami_list}") <(echo "${results}")) + fi - # 'ami_list' is a json array of objects, each object is an ami and its snapshot - for row in $(echo "${ami_list}" | jq -r '.[] | @base64'); do - _jq() { - echo ${row} | base64 --decode | jq -r ${1} - } - echo " - =============================================== - Cleaning up Ami and its snashots in $region - Ami id: $(_jq '.ImageId') - Version: $(_jq '.Version') - Creation data: $(_jq '.date') - Snapshot id: $(_jq '.SnapshotId') - " + # 'ami_list' is a json array of objects, each object is an ami and its snapshot + for row in $(echo "${ami_list}" | jq -r '.[] | @base64'); do + _jq() { + echo "${row}" | base64 --decode | jq -r "${1}" + } + echo " + =============================================== + Cleaning up Ami and its snashots in ${region} + Ami id: $(_jq '.ImageId') + Version: $(_jq '.Version') + Creation data: $(_jq '.date') + Snapshot id: $(_jq '.SnapshotId') + " - aws ec2 deregister-image \ - --image-id $(_jq '.ImageId') \ - --region $region + aws ec2 deregister-image \ + --image-id "$(_jq '.ImageId')" \ + --region "${region}" - if [ "${snapshot_id:-}" != "$(_jq '.SnapshotId')" ]; then - aws ec2 delete-snapshot \ - --snapshot-id $(_jq '.SnapshotId') \ - --region $region - fi - done + if [ "${snapshot_id:-}" != "$(_jq '.SnapshotId')" ]; then + aws ec2 delete-snapshot \ + --snapshot-id "$(_jq '.SnapshotId')" \ + --region "${region}" + fi + done done diff --git a/ci/tasks/light-aws/cleanup-ami.yml b/ci/tasks/light-aws/cleanup-ami.yml index feae8d0c4a..e30236167c 100644 --- a/ci/tasks/light-aws/cleanup-ami.yml +++ b/ci/tasks/light-aws/cleanup-ami.yml @@ -1,19 +1,18 @@ --- platform: linux - inputs: - name: bosh-stemcells-ci run: path: bosh-stemcells-ci/ci/tasks/light-aws/cleanup-ami.sh params: - ami_region: "eu-central-1" # AWS default region - ami_access_key: "" - ami_secret_key: "" - ami_role_arn: "" - ami_older_than_days: "60" # Number of days AMI to keep excluding those currently being running - ami_keep_latest: "5" # Number of previous AMI to keep excluding those currently being running - os_name: "" # e.g ubuntu-jammy - snapshot_id: "" # Snapshot id to delete - remove_public_images: "false" + ami_region: "eu-central-1" # AWS default region + ami_access_key: "" + ami_secret_key: "" + ami_role_arn: "" + ami_older_than_days: "60" # Number of days AMI to keep excluding those currently being running + ami_keep_latest: "5" # Number of previous AMI to keep excluding those currently being running + os_name: "" # e.g ubuntu-jammy + snapshot_id: "" # Snapshot id to delete + remove_public_images: "false" diff --git a/ci/tasks/light-aws/run-upload-test.sh b/ci/tasks/light-aws/run-upload-test.sh index d9c11e1b32..7826e164ca 100755 --- a/ci/tasks/light-aws/run-upload-test.sh +++ b/ci/tasks/light-aws/run-upload-test.sh @@ -13,6 +13,6 @@ fi source "${REPO_PARENT}/director-state/director.env" pushd "${REPO_PARENT}/stemcell" - time bosh -n upload-stemcell *.tgz + time bosh -n upload-stemcell ./*.tgz popd diff --git a/ci/tasks/light-aws/run-upload-test.yml b/ci/tasks/light-aws/run-upload-test.yml index 2f56f6ec86..978fb5bd88 100644 --- a/ci/tasks/light-aws/run-upload-test.yml +++ b/ci/tasks/light-aws/run-upload-test.yml @@ -8,10 +8,9 @@ inputs: - name: environment - name: stemcell - name: director-state - - name: builder-src run: path: bosh-stemcells-ci/ci/tasks/light-aws/run-upload-test.sh params: - BOSH_DEBUG_LEVEL: info + BOSH_DEBUG_LEVEL: info diff --git a/ci/tasks/light-aws/test-drivers.sh b/ci/tasks/light-aws/test-drivers.sh deleted file mode 100755 index 685f092fb3..0000000000 --- a/ci/tasks/light-aws/test-drivers.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -set -eu -o pipefail - -REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )" -REPO_PARENT="$( cd "${REPO_ROOT}/.." && pwd )" - -if [[ -n "${DEBUG:-}" ]]; then - set -x - export BOSH_LOG_LEVEL=debug - export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}" -fi - -tmp_dir="$(mktemp -d /tmp/stemcell_builder.XXXXXXX)" -trap '{ rm -rf ${tmp_dir}; }' EXIT - -: ${aws_account_id:?must be set} -: ${access_key:?must be set} -: ${secret_key:?must be set} -: ${bucket_name:?must be set} -: ${region:?must be set} -: ${copy_region:?must be set} -: ${ami_fixture_id:?must be set} -: ${private_ami_fixture_id:?must be set} -: ${existing_volume_id:?must be set} -: ${existing_snapshot_id:?must be set} -: ${uploaded_machine_image_url:?must be set} -: ${kms_key_id:?must be set} -: ${kms_multi_region_key:?must be set} -: ${kms_multi_region_key_replication_test:?must be set} - -: ${uploaded_machine_image_format:=RAW} - -# US Regions -export AWS_ACCOUNT=$aws_account_id -export AWS_ACCESS_KEY_ID=$access_key -export AWS_SECRET_ACCESS_KEY=$secret_key -export AWS_BUCKET_NAME=$bucket_name -export AWS_REGION=$region -export AWS_DESTINATION_REGION=${copy_region} -export AWS_KMS_KEY_ID=${kms_key_id} -export MULTI_REGION_KEY=${kms_multi_region_key} -export MULTI_REGION_KEY_REPLICATION_TEST=${kms_multi_region_key_replication_test} - -# Fixtures -export S3_MACHINE_IMAGE_URL=${uploaded_machine_image_url} -export S3_MACHINE_IMAGE_FORMAT=${uploaded_machine_image_format} -export EBS_VOLUME_ID=${existing_volume_id} -export EBS_SNAPSHOT_ID=${existing_snapshot_id} -export AMI_FIXTURE_ID=${ami_fixture_id} -export PRIVATE_AMI_FIXTURE_ID=${private_ami_fixture_id} - -echo "Downloading machine image" -export MACHINE_IMAGE_PATH=${tmp_dir}/image.iso -export MACHINE_IMAGE_FORMAT="RAW" -wget -O ${MACHINE_IMAGE_PATH} http://tinycorelinux.net/7.x/x86_64/archive/7.1/TinyCorePure64-7.1.iso - -echo "Running driver tests" - -pushd "${REPO_PARENT}/builder-src" > /dev/null - # Run all driver specs in parallel to reduce test time - spec_count="$(grep "It(" -r driver | wc -l)" - go run github.com/onsi/ginkgo/v2/ginkgo -nodes ${spec_count} -r driver -popd diff --git a/ci/tasks/light-aws/test-drivers.yml b/ci/tasks/light-aws/test-drivers.yml deleted file mode 100644 index d6ca5d227d..0000000000 --- a/ci/tasks/light-aws/test-drivers.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -platform: linux - -inputs: -- name: builder-src -- name: bosh-stemcells-ci -run: - path: bosh-stemcells-ci/ci/tasks/light-aws/test-drivers.sh -params: - aws_account_id: "" - access_key: "" - secret_key: "" - bucket_name: "" - region: "" - copy_region: "" - ami_fixture_id: "" - private_ami_fixture_id: "" - kms_key_id: "" - kms_multi_region_key: "" - kms_multi_region_key_replication_test: "" - existing_volume_id: "" - existing_snapshot_id: "" - uploaded_machine_image_url: "" diff --git a/ci/tasks/light-aws/test-integration.sh b/ci/tasks/light-aws/test-integration.sh deleted file mode 100755 index 47224938b3..0000000000 --- a/ci/tasks/light-aws/test-integration.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -set -eu -o pipefail - -REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )" -REPO_PARENT="$( cd "${REPO_ROOT}/.." && pwd )" - -if [[ -n "${DEBUG:-}" ]]; then - set -x - export BOSH_LOG_LEVEL=debug - export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}" -fi - -tmp_dir="$(mktemp -d /tmp/stemcell_builder.XXXXXXX)" -trap '{ rm -rf ${tmp_dir}; }' EXIT - -: ${access_key:?must be set} -: ${secret_key:?must be set} -: ${bucket_name:?must be set} -: ${region:?must be set} -: ${copy_region:?must be set} -# : ${cn_access_key:?must be set} -# : ${cn_secret_key:?must be set} -# : ${cn_bucket_name:?must be set} -# : ${cn_region:?must be set} - -# US Regions -export AWS_ACCESS_KEY_ID=$access_key -export AWS_SECRET_ACCESS_KEY=$secret_key -export AWS_BUCKET_NAME=$bucket_name -export AWS_REGION=$region -export AWS_DESTINATION_REGION=${copy_region} - -# # China Region -# export AWS_CN_ACCESS_KEY_ID=$cn_access_key -# export AWS_CN_SECRET_ACCESS_KEY=$cn_secret_key -# export AWS_CN_BUCKET_NAME=$cn_bucket_name -# export AWS_CN_REGION=$cn_region - -echo "Downloading machine image" -export MACHINE_IMAGE_PATH=${tmp_dir}/image.iso -export MACHINE_IMAGE_FORMAT="RAW" -wget -O ${MACHINE_IMAGE_PATH} http://tinycorelinux.net/7.x/x86_64/archive/7.1/TinyCorePure64-7.1.iso - -echo "Running integration tests" - -pushd "${REPO_PARENT}/builder-src" > /dev/null - go run github.com/onsi/ginkgo/v2/ginkgo -v -r integration -popd diff --git a/ci/tasks/light-aws/test-integration.yml b/ci/tasks/light-aws/test-integration.yml deleted file mode 100644 index f92f66431b..0000000000 --- a/ci/tasks/light-aws/test-integration.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -platform: linux - -inputs: -- name: builder-src -- name: bosh-stemcells-ci -run: - path: bosh-stemcells-ci/ci/tasks/light-aws/test-integration.sh -params: - access_key: "" - secret_key: "" - bucket_name: "" - region: "" - copy_region: "" - # cn_access_key: "" - # cn_secret_key: "" - # cn_bucket_name: "" - # cn_region: "" diff --git a/ci/tasks/light-aws/test-unit.sh b/ci/tasks/light-aws/test-unit.sh deleted file mode 100755 index f8bbd52a90..0000000000 --- a/ci/tasks/light-aws/test-unit.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -eu -o pipefail - -REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )" -REPO_PARENT="$( cd "${REPO_ROOT}/.." && pwd )" - -if [[ -n "${DEBUG:-}" ]]; then - set -x - export BOSH_LOG_LEVEL=debug - export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}" -fi - -echo "Running unit tests" - -pushd "${REPO_PARENT}/builder-src" > /dev/null - go run github.com/onsi/ginkgo/v2/ginkgo -p -r --skip-package "driver,integration" - go run github.com/onsi/ginkgo/v2/ginkgo -p -r driverset # driverset is skipped by previous command -popd diff --git a/ci/tasks/light-aws/test-unit.yml b/ci/tasks/light-aws/test-unit.yml deleted file mode 100644 index 72017042b4..0000000000 --- a/ci/tasks/light-aws/test-unit.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -platform: linux - -inputs: -- name: builder-src -- name: bosh-stemcells-ci -run: - path: bosh-stemcells-ci/ci/tasks/light-aws/test-unit.sh diff --git a/ci/tasks/light-aws/us-gov-merge-builds.yml b/ci/tasks/light-aws/us-gov-merge-builds.yml index 68b41ba348..4dfcd106e1 100644 --- a/ci/tasks/light-aws/us-gov-merge-builds.yml +++ b/ci/tasks/light-aws/us-gov-merge-builds.yml @@ -3,7 +3,6 @@ platform: linux inputs: -- name: builder-src - name: bosh-stemcells-ci - name: us-light-stemcell - name: cn-north-light-stemcell diff --git a/ci/tasks/light-google/create-public-image.sh b/ci/tasks/light-google/create-public-image.sh index 81dc90058c..453900d305 100755 --- a/ci/tasks/light-google/create-public-image.sh +++ b/ci/tasks/light-google/create-public-image.sh @@ -10,8 +10,8 @@ if [[ -n "${DEBUG:-}" ]]; then export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}" fi -: ${PROJECT_NAME:?} -: ${GCP_SERVICE_ACCOUNT_KEY:?} +: "${PROJECT_NAME:?}" +: "${GCP_SERVICE_ACCOUNT_KEY:?}" echo "Creating light stemcell..." @@ -25,10 +25,12 @@ raw_stemcell_filename="$(basename "${raw_stemcell}")" raw_stemcell_uri="$(cat "${REPO_PARENT}/base-oss-google-ubuntu-stemcell/url")" -image_name=$(echo "$raw_stemcell_filename" | sed -e 's/[^0-9a-zA-Z]/-/g' -e 's/-tar-gz$//' -e 's/-go-agent-raw//' -e 's/^bosh-//') +image_name=$(echo "$raw_stemcell_filename" \ + | sed -e 's/[^0-9a-zA-Z]/-/g' -e 's/-tar-gz$//' -e 's/-go-agent-raw//' -e 's/^bosh-//') # authenticate with service account -echo ${GCP_SERVICE_ACCOUNT_KEY} | gcloud auth activate-service-account --key-file - --project ${PROJECT_NAME} +echo "${GCP_SERVICE_ACCOUNT_KEY}" \ + | gcloud auth activate-service-account --key-file - --project "${PROJECT_NAME}" guest_os_features=() if [[ "${EFI:-false}" == "true" ]]; then @@ -45,14 +47,14 @@ if (( ${#guest_os_features[@]} > 0 )); then fi # create image +# shellcheck disable=SC2086 gcloud compute images create "${image_name}" \ --project="${PROJECT_NAME}" \ --source-uri="${raw_stemcell_uri}" \ ${guest_os_features_flag} \ --storage-location=eu - -gcloud compute images add-iam-policy-binding ${image_name} \ +gcloud compute images add-iam-policy-binding "${image_name}" \ --member='allAuthenticatedUsers' \ --role='roles/compute.imageUser' @@ -61,14 +63,14 @@ pushd "${REPO_PARENT}/working_dir" # create final light stemcell tar xvf "${original_stemcell}" - > image + : > image packaged_image_stemcell_sha1=$(sha1sum image | awk '{print $1}') cp stemcell.MF /tmp/stemcell.MF.tmp bosh int \ -o "${REPO_ROOT}/ci/tasks/light-google/assets/public-image-stemcell-ops.yml" \ - -v "packaged_image_stemcell_sha1=$packaged_image_stemcell_sha1" \ + -v "packaged_image_stemcell_sha1=${packaged_image_stemcell_sha1}" \ -v 'stemcell_formats=["google-light"]' \ -v "image_url=https://www.googleapis.com/compute/v1/projects/${PROJECT_NAME}/global/images/${image_name}" \ /tmp/stemcell.MF.tmp > stemcell.MF diff --git a/ci/tasks/light-google/deploy-skeletal.sh b/ci/tasks/light-google/deploy-skeletal.sh index d0bee28e16..a5e57c4df5 100755 --- a/ci/tasks/light-google/deploy-skeletal.sh +++ b/ci/tasks/light-google/deploy-skeletal.sh @@ -11,8 +11,8 @@ if [[ -n "${DEBUG:-}" ]]; then fi # env -: ${SSH_PRIVATE_KEY:?} -: ${GCE_CREDENTIALS_JSON:?} +: "${SSH_PRIVATE_KEY:?}" +: "${GCE_CREDENTIALS_JSON:?}" mkdir -p "${REPO_PARENT}/deployment-state/assets/" diff --git a/ci/tasks/light-google/make-raw-from-heavy-stemcell.sh b/ci/tasks/light-google/make-raw-from-heavy-stemcell.sh index ca39e402bb..0603fc587c 100755 --- a/ci/tasks/light-google/make-raw-from-heavy-stemcell.sh +++ b/ci/tasks/light-google/make-raw-from-heavy-stemcell.sh @@ -10,18 +10,21 @@ if [[ -n "${DEBUG:-}" ]]; then export BOSH_LOG_PATH="${BOSH_LOG_PATH:-${REPO_PARENT}/bosh-debug.log}" fi -: ${BUCKET_NAME:?} -: ${STEMCELL_BUCKET_PATH:?} # used to check if current stemcell already exists +: "${BUCKET_NAME:?}" +: "${STEMCELL_BUCKET_PATH:?}" # used to check if current stemcell already exists stemcell_url() { - resource="/${STEMCELL_BUCKET_PATH}/${light_stemcell_name}" + local name + name=${1} + + resource="/${STEMCELL_BUCKET_PATH}/${name}" if [ ! -z "$AWS_ACCESS_KEY_ID" ]; then expires=$(date +%s) expires=$((expires + 30)) string_to_sign="HEAD\n\n\n${expires}\n${resource}" - signature=$(echo -en "$string_to_sign" | openssl sha1 -hmac ${AWS_SECRET_ACCESS_KEY} -binary | base64) + signature=$(echo -en "$string_to_sign" | openssl sha1 -hmac "${AWS_SECRET_ACCESS_KEY}" -binary | base64) signature=$(python -c "import urllib; print urllib.quote_plus('${signature}')") echo -n "https://${S3_API_ENDPOINT}${resource}?AWSAccessKeyId=${AWS_ACCESS_KEY_ID}&Expires=${expires}&Signature=${signature}" else @@ -32,22 +35,20 @@ stemcell_url() { echo "Creating light stemcell..." salt=$(date +%s) -original_stemcell="$(echo ${REPO_PARENT}/stemcell/*.tgz)" +original_stemcell="$(echo "${REPO_PARENT}"/stemcell/*.tgz)" original_stemcell_name="$(basename "${original_stemcell}")" -raw_stemcell_name="$(basename "${original_stemcell}" .tgz)-raw-$salt.tar.gz" -light_stemcell_name="light-${original_stemcell_name}" +raw_stemcell_name="$(basename "${original_stemcell}" .tgz)-raw-${salt}.tar.gz" echo "Using raw stemcell name: ${raw_stemcell_name}" -light_stemcell_url="$(stemcell_url)" -set +e -wget --spider "$light_stemcell_url" -if [[ "$?" == "0" ]]; then - echo "Google light stemcell '$light_stemcell_name' already exists!" - echo "You can download here: $light_stemcell_url" +light_stemcell_name="light-${original_stemcell_name}" +light_stemcell_url="$(stemcell_url "${light_stemcell_name}")" + +if wget --spider "${light_stemcell_url}"; then + echo "Google light stemcell '${light_stemcell_name}' already exists!" + echo "You can download here: ${light_stemcell_url}" exit 1 fi -set -e mkdir "${REPO_PARENT}/working_dir" pushd "${REPO_PARENT}/working_dir"