From ec3123ca1439195e42db27167b9232d4e6795c9b Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 16 Sep 2026 13:51:32 +0530 Subject: [PATCH 1/7] fix: resolve recurring Terraform state lock in CI - Replace matrix.architecture (undefined in deploy-receiver-mock) with github.run_id in receiver-mock BUCKET_KEY, eliminating the shared receivermock..nodejs state path that caused concurrent-run lock collisions - Add terraform init to destroy-receiver-mock and remove the cache dependency, so destroy is reliable even when deploy fails (cache is never saved on apply failure) - Change Cleanup AWS environment from if: success() to if: always() so Lambda infra is destroyed on test failure, not just on success --- .github/workflows/tests.yml | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cd44774..0a9c1b5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,7 +27,7 @@ jobs: receivermockendpoint: ${{ steps.get-receiver-mock-endpoint.outputs.endpoint }} env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: receivermock.${{ matrix.architecture}}.${{ inputs.LANGUAGE }}/terraform_state.tfstate + BUCKET_KEY: receivermock.${{ github.run_id }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate LANGUAGE: ${{ inputs.LANGUAGE }} TERRAFORM_DIRECTORY: utils/receiver-mock/deploy steps: @@ -40,11 +40,6 @@ jobs: role-to-assume: ${{ env.AWS_OPENID_ROLE_ARN }} aws-region: ${{ matrix.aws_region }} mask-aws-account-id: false - - name: Cache terraform working directory - uses: actions/cache@v3 - with: - path: ${{ env.TERRAFORM_DIRECTORY }} - key: ${{ inputs.LANGUAGE }}-${{ github.run_id }} - uses: hashicorp/setup-terraform@v3 with: terraform_wrapper: false @@ -61,7 +56,7 @@ jobs: run: terraform apply -auto-approve working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: lambda-layers-${{ inputs.LANGUAGE }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} - name: Wait for task container to be ready run: sleep 30 - name: Get receiver-mock endpoint @@ -69,7 +64,7 @@ jobs: run: echo "endpoint=$(terraform output -raw loadbalancer_ip)" >> $GITHUB_OUTPUT working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: lambda-layers-${{ inputs.LANGUAGE }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} run-tests: name: Run tests runs-on: ubuntu-22.04 @@ -83,7 +78,7 @@ jobs: aws_region: [ eu-central-1 ] env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: ${{ matrix.architecture}}.${{ inputs.LANGUAGE }}/terraform_state.tfstate + BUCKET_KEY: ${{ matrix.architecture}}.${{ inputs.LANGUAGE }}.${{ github.run_id }}/terraform_state.tfstate LANGUAGE: ${{ inputs.LANGUAGE }} TERRAFORM_DIRECTORY: ${{ inputs.LANGUAGE }}/tests/deploy steps: @@ -170,7 +165,7 @@ jobs: cd tests/lambdalayer go test -v -run TestSpans${{ env.LANGUAGE }} - name: Cleanup AWS environment - if: success() + if: always() run: terraform destroy -auto-approve working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: @@ -190,6 +185,8 @@ jobs: matrix: aws_region: [ eu-central-1 ] env: + BUCKET_NAME: lambda-tests-terraform-state-bucket + BUCKET_KEY: receivermock.${{ github.run_id }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate TERRAFORM_DIRECTORY: utils/receiver-mock/deploy steps: - uses: actions/checkout@v4 @@ -204,13 +201,17 @@ jobs: - uses: hashicorp/setup-terraform@v3 with: terraform_wrapper: false - - name: Cache terraform working directory - uses: actions/cache@v3 - with: - path: ${{ env.TERRAFORM_DIRECTORY }} - key: ${{ inputs.LANGUAGE }}-${{ github.run_id }} + - name: Initialize terraform + run: | + terraform init \ + --backend-config "bucket=${{ env.BUCKET_NAME }}" \ + --backend-config "key=${{ env.BUCKET_KEY }}" \ + --backend-config "region=${{ matrix.aws_region }}" \ + --backend-config "encrypt=true" \ + --backend-config "dynamodb_table=SumoOtelLambdaTfStateLock" + working-directory: ${{ env.TERRAFORM_DIRECTORY }} - name: Destroy terraform run: terraform destroy -auto-approve working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: lambda-layers-${{ inputs.LANGUAGE }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} From 74fff18f882792e92a0c375f4f49a2383a63aa7b Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 16 Sep 2026 14:30:00 +0530 Subject: [PATCH 2/7] fix: add fail-fast false, run_attempt isolation, and artifact name sanitization - Add fail-fast: false to run-tests matrix so arm64 cleanup always runs even when amd64 fails, preventing Lambda infra leaks on partial failures - Append github.run_attempt to all state keys and resource names so re-runs get a clean slate instead of colliding with a partially destroyed prior attempt - Replace github.head_ref with github.run_id in pr-build artifact names; branch names with forward slashes (e.g. fix/foo) are invalid artifact names and cause the build to fail immediately --- .github/workflows/pr-build-java.yml | 4 ++-- .github/workflows/pr-build-nodejs.yml | 4 ++-- .github/workflows/pr-build-python.yml | 4 ++-- .github/workflows/tests.yml | 13 +++++++------ 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr-build-java.yml b/.github/workflows/pr-build-java.yml index 5e16396..9aa24e3 100644 --- a/.github/workflows/pr-build-java.yml +++ b/.github/workflows/pr-build-java.yml @@ -17,14 +17,14 @@ jobs: BUILD_COMMAND: make build-java LANGUAGE: java ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-java-wrapper - ARTIFACT_NAME: ${{ github.head_ref }} + ARTIFACT_NAME: ${{ github.run_id }} create-dev-lambda-layer: needs: build-artifacts uses: ./.github/workflows/publish-dev-layer.yml with: ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-java-wrapper - ARTIFACT_NAME: ${{ github.head_ref }} + ARTIFACT_NAME: ${{ github.run_id }} LANGUAGE: java run-tests: diff --git a/.github/workflows/pr-build-nodejs.yml b/.github/workflows/pr-build-nodejs.yml index 5d39a4d..3b164f9 100644 --- a/.github/workflows/pr-build-nodejs.yml +++ b/.github/workflows/pr-build-nodejs.yml @@ -17,14 +17,14 @@ jobs: BUILD_COMMAND: make build-nodejs LANGUAGE: nodejs ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-nodejs - ARTIFACT_NAME: ${{ github.head_ref }} + ARTIFACT_NAME: ${{ github.run_id }} create-dev-lambda-layer: needs: build-artifacts uses: ./.github/workflows/publish-dev-layer.yml with: ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-nodejs - ARTIFACT_NAME: ${{ github.head_ref }} + ARTIFACT_NAME: ${{ github.run_id }} LANGUAGE: nodejs run-tests: diff --git a/.github/workflows/pr-build-python.yml b/.github/workflows/pr-build-python.yml index cc939e3..65507ca 100644 --- a/.github/workflows/pr-build-python.yml +++ b/.github/workflows/pr-build-python.yml @@ -17,14 +17,14 @@ jobs: BUILD_COMMAND: make build-python LANGUAGE: python ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-python - ARTIFACT_NAME: ${{ github.head_ref }} + ARTIFACT_NAME: ${{ github.run_id }} create-dev-lambda-layer: needs: build-artifacts uses: ./.github/workflows/publish-dev-layer.yml with: ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-python - ARTIFACT_NAME: ${{ github.head_ref }} + ARTIFACT_NAME: ${{ github.run_id }} LANGUAGE: python run-tests: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0a9c1b5..1af486a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,7 +27,7 @@ jobs: receivermockendpoint: ${{ steps.get-receiver-mock-endpoint.outputs.endpoint }} env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: receivermock.${{ github.run_id }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate + BUCKET_KEY: receivermock.${{ github.run_id }}-${{ github.run_attempt }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate LANGUAGE: ${{ inputs.LANGUAGE }} TERRAFORM_DIRECTORY: utils/receiver-mock/deploy steps: @@ -56,7 +56,7 @@ jobs: run: terraform apply -auto-approve working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} - name: Wait for task container to be ready run: sleep 30 - name: Get receiver-mock endpoint @@ -64,7 +64,7 @@ jobs: run: echo "endpoint=$(terraform output -raw loadbalancer_ip)" >> $GITHUB_OUTPUT working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} run-tests: name: Run tests runs-on: ubuntu-22.04 @@ -73,12 +73,13 @@ jobs: id-token: write contents: read strategy: + fail-fast: false matrix: architecture: [ amd64, arm64 ] aws_region: [ eu-central-1 ] env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: ${{ matrix.architecture}}.${{ inputs.LANGUAGE }}.${{ github.run_id }}/terraform_state.tfstate + BUCKET_KEY: ${{ matrix.architecture}}.${{ inputs.LANGUAGE }}.${{ github.run_id }}-${{ github.run_attempt }}/terraform_state.tfstate LANGUAGE: ${{ inputs.LANGUAGE }} TERRAFORM_DIRECTORY: ${{ inputs.LANGUAGE }}/tests/deploy steps: @@ -186,7 +187,7 @@ jobs: aws_region: [ eu-central-1 ] env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: receivermock.${{ github.run_id }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate + BUCKET_KEY: receivermock.${{ github.run_id }}-${{ github.run_attempt }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate TERRAFORM_DIRECTORY: utils/receiver-mock/deploy steps: - uses: actions/checkout@v4 @@ -214,4 +215,4 @@ jobs: run: terraform destroy -auto-approve working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} From 2e2f659987971f1580ed5d97018095410e8f2584 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 16 Sep 2026 15:15:59 +0530 Subject: [PATCH 3/7] fix: use run_id only for state keys, run_id+run_attempt for artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State keys and resource names use run_id only — Terraform's apply is idempotent so re-runs reconcile against existing state rather than creating orphaned resources from a new empty state. Artifact names use run_id+run_attempt — upload-artifact@v4 keeps names immutable within a run so re-runs need a fresh name to avoid the already-exists error. --- .github/workflows/pr-build-java.yml | 4 ++-- .github/workflows/pr-build-nodejs.yml | 4 ++-- .github/workflows/pr-build-python.yml | 4 ++-- .github/workflows/tests.yml | 12 ++++++------ 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr-build-java.yml b/.github/workflows/pr-build-java.yml index 9aa24e3..93ef230 100644 --- a/.github/workflows/pr-build-java.yml +++ b/.github/workflows/pr-build-java.yml @@ -17,14 +17,14 @@ jobs: BUILD_COMMAND: make build-java LANGUAGE: java ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-java-wrapper - ARTIFACT_NAME: ${{ github.run_id }} + ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} create-dev-lambda-layer: needs: build-artifacts uses: ./.github/workflows/publish-dev-layer.yml with: ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-java-wrapper - ARTIFACT_NAME: ${{ github.run_id }} + ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} LANGUAGE: java run-tests: diff --git a/.github/workflows/pr-build-nodejs.yml b/.github/workflows/pr-build-nodejs.yml index 3b164f9..ad4a9d9 100644 --- a/.github/workflows/pr-build-nodejs.yml +++ b/.github/workflows/pr-build-nodejs.yml @@ -17,14 +17,14 @@ jobs: BUILD_COMMAND: make build-nodejs LANGUAGE: nodejs ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-nodejs - ARTIFACT_NAME: ${{ github.run_id }} + ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} create-dev-lambda-layer: needs: build-artifacts uses: ./.github/workflows/publish-dev-layer.yml with: ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-nodejs - ARTIFACT_NAME: ${{ github.run_id }} + ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} LANGUAGE: nodejs run-tests: diff --git a/.github/workflows/pr-build-python.yml b/.github/workflows/pr-build-python.yml index 65507ca..7817dda 100644 --- a/.github/workflows/pr-build-python.yml +++ b/.github/workflows/pr-build-python.yml @@ -17,14 +17,14 @@ jobs: BUILD_COMMAND: make build-python LANGUAGE: python ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-python - ARTIFACT_NAME: ${{ github.run_id }} + ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} create-dev-lambda-layer: needs: build-artifacts uses: ./.github/workflows/publish-dev-layer.yml with: ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-python - ARTIFACT_NAME: ${{ github.run_id }} + ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} LANGUAGE: python run-tests: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1af486a..ab502c4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,7 +27,7 @@ jobs: receivermockendpoint: ${{ steps.get-receiver-mock-endpoint.outputs.endpoint }} env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: receivermock.${{ github.run_id }}-${{ github.run_attempt }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate + BUCKET_KEY: receivermock.${{ github.run_id }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate LANGUAGE: ${{ inputs.LANGUAGE }} TERRAFORM_DIRECTORY: utils/receiver-mock/deploy steps: @@ -56,7 +56,7 @@ jobs: run: terraform apply -auto-approve working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} - name: Wait for task container to be ready run: sleep 30 - name: Get receiver-mock endpoint @@ -64,7 +64,7 @@ jobs: run: echo "endpoint=$(terraform output -raw loadbalancer_ip)" >> $GITHUB_OUTPUT working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} run-tests: name: Run tests runs-on: ubuntu-22.04 @@ -79,7 +79,7 @@ jobs: aws_region: [ eu-central-1 ] env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: ${{ matrix.architecture}}.${{ inputs.LANGUAGE }}.${{ github.run_id }}-${{ github.run_attempt }}/terraform_state.tfstate + BUCKET_KEY: ${{ matrix.architecture}}.${{ inputs.LANGUAGE }}.${{ github.run_id }}/terraform_state.tfstate LANGUAGE: ${{ inputs.LANGUAGE }} TERRAFORM_DIRECTORY: ${{ inputs.LANGUAGE }}/tests/deploy steps: @@ -187,7 +187,7 @@ jobs: aws_region: [ eu-central-1 ] env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: receivermock.${{ github.run_id }}-${{ github.run_attempt }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate + BUCKET_KEY: receivermock.${{ github.run_id }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate TERRAFORM_DIRECTORY: utils/receiver-mock/deploy steps: - uses: actions/checkout@v4 @@ -215,4 +215,4 @@ jobs: run: terraform destroy -auto-approve working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} From d9a77ef9ac8d48a8ddb87a0d84efbc5d195389e4 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 16 Sep 2026 15:36:11 +0530 Subject: [PATCH 4/7] fix: add run_attempt to all run-scoped resource names and state keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ensures re-runs get fully isolated state files, AWS resource names, S3 buckets, and Lambda layer names. Without run_attempt, a re-run targeting the same run_id hits the previous attempt's DynamoDB lock, existing S3 bucket, and stale layer — failing or producing incorrect results. --- .github/workflows/publish-dev-layer.yml | 4 ++-- .github/workflows/tests.yml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-dev-layer.yml b/.github/workflows/publish-dev-layer.yml index 46f2249..1fa43c7 100644 --- a/.github/workflows/publish-dev-layer.yml +++ b/.github/workflows/publish-dev-layer.yml @@ -33,11 +33,11 @@ jobs: aws_region: [ eu-central-1 ] env: LANGUAGE: ${{ inputs.LANGUAGE }} - LAYER_NAME: sumologic-otel-lambda-dev-${{ inputs.LANGUAGE }}-${{ matrix.architecture }}-${{ github.run_id }} + LAYER_NAME: sumologic-otel-lambda-dev-${{ inputs.LANGUAGE }}-${{ matrix.architecture }}-${{ github.run_id }}-${{ github.run_attempt }} ARCHITECTURE: ${{ matrix.architecture }} ARTIFACT_ARCHIVE_BASE_NAME: ${{ inputs.ARTIFACT_ARCHIVE_BASE_NAME }} ARTIFACT_NAME: ${{ inputs.ARTIFACT_NAME }} - BUCKET_NAME: sumologic-otel-lambda-dev-${{ github.run_id }}-${{ matrix.architecture }} + BUCKET_NAME: sumologic-otel-lambda-dev-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.architecture }} BUCKET_KEY: layer-${{ matrix.architecture }}-${{ matrix.aws_region }}-.zip DIRECTORY: ${{ inputs.LANGUAGE }} REGION: ${{ matrix.aws_region }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ab502c4..1af486a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,7 +27,7 @@ jobs: receivermockendpoint: ${{ steps.get-receiver-mock-endpoint.outputs.endpoint }} env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: receivermock.${{ github.run_id }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate + BUCKET_KEY: receivermock.${{ github.run_id }}-${{ github.run_attempt }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate LANGUAGE: ${{ inputs.LANGUAGE }} TERRAFORM_DIRECTORY: utils/receiver-mock/deploy steps: @@ -56,7 +56,7 @@ jobs: run: terraform apply -auto-approve working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} - name: Wait for task container to be ready run: sleep 30 - name: Get receiver-mock endpoint @@ -64,7 +64,7 @@ jobs: run: echo "endpoint=$(terraform output -raw loadbalancer_ip)" >> $GITHUB_OUTPUT working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} run-tests: name: Run tests runs-on: ubuntu-22.04 @@ -79,7 +79,7 @@ jobs: aws_region: [ eu-central-1 ] env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: ${{ matrix.architecture}}.${{ inputs.LANGUAGE }}.${{ github.run_id }}/terraform_state.tfstate + BUCKET_KEY: ${{ matrix.architecture}}.${{ inputs.LANGUAGE }}.${{ github.run_id }}-${{ github.run_attempt }}/terraform_state.tfstate LANGUAGE: ${{ inputs.LANGUAGE }} TERRAFORM_DIRECTORY: ${{ inputs.LANGUAGE }}/tests/deploy steps: @@ -187,7 +187,7 @@ jobs: aws_region: [ eu-central-1 ] env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: receivermock.${{ github.run_id }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate + BUCKET_KEY: receivermock.${{ github.run_id }}-${{ github.run_attempt }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate TERRAFORM_DIRECTORY: utils/receiver-mock/deploy steps: - uses: actions/checkout@v4 @@ -215,4 +215,4 @@ jobs: run: terraform destroy -auto-approve working-directory: ${{ env.TERRAFORM_DIRECTORY }} env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }} + TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} From 76c141e3d6fb0f924dafcdca4550aa58b8c5bb67 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 17 Sep 2026 14:08:12 +0530 Subject: [PATCH 5/7] fix(ci): use S3-native state locking and stable run-scoped names Terraform 1.16 warns that the "dynamodb_table" backend parameter is deprecated. "use_lockfile" moves locking into S3 and removes the DynamoDB stuck-lock failure mode instead of routing around it. Drop run_attempt from the Terraform state keys. A re-run must reuse the state of the previous attempt. If it does not, it starts from an empty state and permanently orphans every resource the failed attempt created. Hash the run id into the receiver-mock app_name. The embedded run id left the load balancer name at 31 of its 32 character limit, so a 12-digit run id or a two-digit attempt number would break the apply. The hash is a fixed 8 characters, which holds the name at 26. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/tests.yml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1af486a..a619823 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,11 +27,16 @@ jobs: receivermockendpoint: ${{ steps.get-receiver-mock-endpoint.outputs.endpoint }} env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: receivermock.${{ github.run_id }}-${{ github.run_attempt }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate + BUCKET_KEY: receivermock.${{ github.run_id }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate LANGUAGE: ${{ inputs.LANGUAGE }} TERRAFORM_DIRECTORY: utils/receiver-mock/deploy steps: - uses: actions/checkout@v4 + - name: Set receiver-mock resource name + run: | + # Load balancer and target group names cap at 32 characters, so hash the run id instead of embedding it. + run_hash="$(printf '%s' '${{ github.run_id }}' | sha256sum | cut -c1-8)" + echo "TF_VAR_app_name=ll-${{ inputs.LANGUAGE }}-${run_hash}" >> "$GITHUB_ENV" - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4.0.1 env: @@ -50,21 +55,17 @@ jobs: --backend-config "key=${{ env.BUCKET_KEY }}" \ --backend-config "region=${{ matrix.aws_region }}" \ --backend-config "encrypt=true" \ - --backend-config "dynamodb_table=SumoOtelLambdaTfStateLock" + --backend-config "use_lockfile=true" working-directory: ${{ env.TERRAFORM_DIRECTORY }} - name: Apply terraform run: terraform apply -auto-approve working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} - name: Wait for task container to be ready run: sleep 30 - name: Get receiver-mock endpoint id: get-receiver-mock-endpoint run: echo "endpoint=$(terraform output -raw loadbalancer_ip)" >> $GITHUB_OUTPUT working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} run-tests: name: Run tests runs-on: ubuntu-22.04 @@ -79,7 +80,7 @@ jobs: aws_region: [ eu-central-1 ] env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: ${{ matrix.architecture}}.${{ inputs.LANGUAGE }}.${{ github.run_id }}-${{ github.run_attempt }}/terraform_state.tfstate + BUCKET_KEY: ${{ matrix.architecture}}.${{ inputs.LANGUAGE }}.${{ github.run_id }}/terraform_state.tfstate LANGUAGE: ${{ inputs.LANGUAGE }} TERRAFORM_DIRECTORY: ${{ inputs.LANGUAGE }}/tests/deploy steps: @@ -134,7 +135,7 @@ jobs: --backend-config "key=${{ env.BUCKET_KEY }}" \ --backend-config "region=${{ matrix.aws_region }}" \ --backend-config "encrypt=true" \ - --backend-config "dynamodb_table=SumoOtelLambdaTfStateLock" + --backend-config "use_lockfile=true" working-directory: ${{ env.TERRAFORM_DIRECTORY }} - name: Deploy lambda sample function run: terraform apply -auto-approve @@ -187,10 +188,15 @@ jobs: aws_region: [ eu-central-1 ] env: BUCKET_NAME: lambda-tests-terraform-state-bucket - BUCKET_KEY: receivermock.${{ github.run_id }}-${{ github.run_attempt }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate + BUCKET_KEY: receivermock.${{ github.run_id }}.${{ inputs.LANGUAGE }}/terraform_state.tfstate TERRAFORM_DIRECTORY: utils/receiver-mock/deploy steps: - uses: actions/checkout@v4 + - name: Set receiver-mock resource name + run: | + # Must match the name computed in deploy-receiver-mock. + run_hash="$(printf '%s' '${{ github.run_id }}' | sha256sum | cut -c1-8)" + echo "TF_VAR_app_name=ll-${{ inputs.LANGUAGE }}-${run_hash}" >> "$GITHUB_ENV" - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4.0.1 env: @@ -209,10 +215,8 @@ jobs: --backend-config "key=${{ env.BUCKET_KEY }}" \ --backend-config "region=${{ matrix.aws_region }}" \ --backend-config "encrypt=true" \ - --backend-config "dynamodb_table=SumoOtelLambdaTfStateLock" + --backend-config "use_lockfile=true" working-directory: ${{ env.TERRAFORM_DIRECTORY }} - name: Destroy terraform run: terraform destroy -auto-approve working-directory: ${{ env.TERRAFORM_DIRECTORY }} - env: - TF_VAR_app_name: ll-${{ inputs.LANGUAGE }}-${{ github.run_id }}-${{ github.run_attempt }} From d85b065297756a52d54ae7ab688e64f69b7cf6d1 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 17 Sep 2026 16:11:03 +0530 Subject: [PATCH 6/7] fix(ci): make artifact name attempt-invariant build-artifacts uploads the layer artifact and publish-dev-layer downloads it. A partial re-run can re-execute one of the two jobs and carry the other over, so an attempt number in the name makes the download look for an artifact that the upload job never created. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/pr-build-java.yml | 4 ++-- .github/workflows/pr-build-nodejs.yml | 4 ++-- .github/workflows/pr-build-python.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-build-java.yml b/.github/workflows/pr-build-java.yml index 93ef230..9aa24e3 100644 --- a/.github/workflows/pr-build-java.yml +++ b/.github/workflows/pr-build-java.yml @@ -17,14 +17,14 @@ jobs: BUILD_COMMAND: make build-java LANGUAGE: java ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-java-wrapper - ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} + ARTIFACT_NAME: ${{ github.run_id }} create-dev-lambda-layer: needs: build-artifacts uses: ./.github/workflows/publish-dev-layer.yml with: ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-java-wrapper - ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} + ARTIFACT_NAME: ${{ github.run_id }} LANGUAGE: java run-tests: diff --git a/.github/workflows/pr-build-nodejs.yml b/.github/workflows/pr-build-nodejs.yml index ad4a9d9..3b164f9 100644 --- a/.github/workflows/pr-build-nodejs.yml +++ b/.github/workflows/pr-build-nodejs.yml @@ -17,14 +17,14 @@ jobs: BUILD_COMMAND: make build-nodejs LANGUAGE: nodejs ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-nodejs - ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} + ARTIFACT_NAME: ${{ github.run_id }} create-dev-lambda-layer: needs: build-artifacts uses: ./.github/workflows/publish-dev-layer.yml with: ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-nodejs - ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} + ARTIFACT_NAME: ${{ github.run_id }} LANGUAGE: nodejs run-tests: diff --git a/.github/workflows/pr-build-python.yml b/.github/workflows/pr-build-python.yml index 7817dda..65507ca 100644 --- a/.github/workflows/pr-build-python.yml +++ b/.github/workflows/pr-build-python.yml @@ -17,14 +17,14 @@ jobs: BUILD_COMMAND: make build-python LANGUAGE: python ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-python - ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} + ARTIFACT_NAME: ${{ github.run_id }} create-dev-lambda-layer: needs: build-artifacts uses: ./.github/workflows/publish-dev-layer.yml with: ARTIFACT_ARCHIVE_BASE_NAME: opentelemetry-python - ARTIFACT_NAME: ${{ github.run_id }}-${{ github.run_attempt }} + ARTIFACT_NAME: ${{ github.run_id }} LANGUAGE: python run-tests: From 8b9dad4198d0a29e62765635785f0cd254cf242c Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Thu, 17 Sep 2026 16:23:25 +0530 Subject: [PATCH 7/7] fix(ci): overwrite artifacts instead of failing on a repeat upload The artifact name no longer holds run_attempt, so a full re-run uploads under the name that the first attempt used. upload-artifact fails on a name that exists unless overwrite is set. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/build-artifacts.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 7809c59..f68bbb4 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -90,6 +90,7 @@ jobs: name: ${{ env.ARTIFACT_NAME }}-${{ env.LANGUAGE }}-${{ matrix.architecture }}-artifacts path: ${{ env.LANGUAGE }}/${{ env.ARTIFACT_ARCHIVE_BASE_NAME }}-${{ matrix.architecture }}.zip if-no-files-found: error + overwrite: true - name: Rename sample-app java artifact if: env.LANGUAGE == 'java' && success() run: | @@ -102,6 +103,7 @@ jobs: name: sample-app-${{ env.LANGUAGE }}-${{ matrix.architecture }}-artifacts path: opentelemetry-lambda/${{ env.LANGUAGE }}/sample-apps/aws-sdk/build/libs/java-sample-app.jar if-no-files-found: error + overwrite: true - name: Upload artifacts - nodejs if: env.LANGUAGE == 'nodejs' && success() uses: actions/upload-artifact@v4 @@ -109,6 +111,7 @@ jobs: name: ${{ env.ARTIFACT_NAME }}-${{ env.LANGUAGE }}-${{ matrix.architecture }}-artifacts path: ${{ env.LANGUAGE }}/${{ env.ARTIFACT_ARCHIVE_BASE_NAME }}-${{ matrix.architecture }}.zip if-no-files-found: error + overwrite: true - name: Rename sample-app nodejs artifact if: env.LANGUAGE == 'nodejs' && success() run: | @@ -121,6 +124,7 @@ jobs: name: sample-app-${{ env.LANGUAGE }}-${{ matrix.architecture }}-artifacts path: opentelemetry-lambda/${{ env.LANGUAGE }}/sample-apps/aws-sdk/build/nodejs-sample-app.zip if-no-files-found: error + overwrite: true - name: Upload artifacts - python layer if: env.LANGUAGE == 'python' && success() uses: actions/upload-artifact@v4 @@ -128,6 +132,7 @@ jobs: name: ${{ env.ARTIFACT_NAME }}-${{ env.LANGUAGE }}-${{ matrix.architecture }}-artifacts path: ${{ env.LANGUAGE }}/${{ env.ARTIFACT_ARCHIVE_BASE_NAME }}-${{ matrix.architecture }}.zip if-no-files-found: error + overwrite: true - name: Rename sample-app python artifact if: env.LANGUAGE == 'python' && success() run: | @@ -140,3 +145,4 @@ jobs: name: sample-app-${{ env.LANGUAGE }}-${{ matrix.architecture }}-artifacts path: opentelemetry-lambda/${{ env.LANGUAGE }}/sample-apps/build/python-sample-app.zip if-no-files-found: error + overwrite: true