From 3dcc945abe1a0c52bb8306889388a502d50ed9df Mon Sep 17 00:00:00 2001
From: Petr Heinz <petr@betterstack.com>
Date: Fri, 23 Oct 2026 18:35:47 +0200
Subject: [PATCH] T-21950 [CI] Let a dispatched E2E run hold the source for a
 removal in Better Stack

A workflow_dispatch input, hold_for_removal, skips the read-back and keeps
the connected source until someone removes it in Better Stack, then waits
until the four log groups the first sync subscribed lose their
BetterStackLogs filter, before the usual cleanup deletes the stack.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/e2e.yml | 40 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 7b33e5a..642cde5 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -32,6 +32,10 @@ on:
       template_url:
         description: 'Template to deploy instead of the checkout (any URL)'
         type: string
+      hold_for_removal:
+        description: 'Skip the read-back, keep the connected source until someone removes it in Better Stack, then expect its log groups to lose the subscription filter'
+        type: boolean
+        default: false
 
 permissions:
   id-token: write
@@ -239,6 +243,7 @@ jobs:
           done
 
       - name: "[Test] Produce logs, a custom metric, a two-segment trace and a CloudTrail management event"
+        if: ${{ !inputs.hold_for_removal }}
         run: |
           echo "SINCE=dt > toDateTime('$(date -u -d '-10 minutes' '+%F %T')')" >> "$GITHUB_ENV"
           for group in "$LOG_GROUP" "$STATIC_LOG_GROUP"; do
@@ -266,6 +271,7 @@ jobs:
           echo "20 log events into each of $LOG_GROUP and $STATIC_LOG_GROUP, 3 datapoints of E2eTest/AwsStack RunMarker, trace $TRACE_ID, PutRetentionPolicy on $LOG_GROUP"
 
       - name: "[Test] Read everything back through the SQL API connection"
+        if: ${{ !inputs.hold_for_removal }}
         run: |
           set -o pipefail
           mkdir -p out
@@ -403,6 +409,40 @@ jobs:
             fi
           } > out/summary.md
 
+      # For checking a source removal in Better Stack by hand (workflow_dispatch with hold_for_removal): the connected
+      # source stays until someone removes it, and the removal should take the subscription filter off every log group
+      # the first sync subscribed, while the stack and its integration role still exist.
+      - name: "[Hold] Wait until the source is removed, then until its log groups lose the subscription filter"
+        if: ${{ inputs.hold_for_removal }}
+        run: |
+          echo "source $SOURCE_ID is connected and its log groups are subscribed; remove it in Better Stack within 45 min"
+          DEADLINE=$((SECONDS + 2700))
+          until [ "$(curl -sS -o /dev/null -w '%{http_code}' "$API/api/v1/sources/$SOURCE_ID" -H "Authorization: Bearer $TEST_TEAM_TELEMETRY_API")" = 404 ]; do
+            [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::source $SOURCE_ID was not removed within 45 min"; exit 1; }
+            sleep 15
+          done
+          REMOVED_AT=$(date -u +%T)
+          echo "source $SOURCE_ID is gone at $REMOVED_AT UTC"
+          # The removal archives the source at once and destroys it in a background job, which enqueues the teardown.
+          DEADLINE=$((SECONDS + 900))
+          while :; do
+            SUBSCRIBED=""
+            for group in "$LOG_GROUP" "$STATIC_LOG_GROUP" aws/spans /aws/cloudtrail/better-stack; do
+              N=$(aws logs describe-subscription-filters --log-group-name "$group" --query "length(subscriptionFilters[?filterName=='BetterStackLogs'])" --output text) || N="?"
+              [ "$N" = 0 ] || SUBSCRIBED="$SUBSCRIBED $group"
+            done
+            [ -z "$SUBSCRIBED" ] && break
+            [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::15 min after the removal still subscribed:$SUBSCRIBED"; exit 1; }
+            echo "still subscribed:$SUBSCRIBED"; sleep 15
+          done
+          mkdir -p out
+          {
+            echo "### AWS CloudFormation stack E2E \`$RUN_ID\` (hold for removal)"
+            echo
+            echo "- source $SOURCE_ID on \`$DATA_REGION\`, stack \`$STACK_NAME\`, removed in Better Stack at $REMOVED_AT UTC"
+            echo "- by $(date -u +%T) UTC no BetterStackLogs filter left on \`$LOG_GROUP\`, \`$STATIC_LOG_GROUP\`, \`aws/spans\` and \`/aws/cloudtrail/better-stack\`"
+          } | tee out/summary.md
+
       - name: "[Cleanup] Delete this run's resources in Better Stack and AWS"
         if: ${{ always() }}
         run: bash e2e/cleanup.sh