From 06d768d34d5b914bb72b9355ad6385cb8ca9f571 Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:52:29 -0700 Subject: [PATCH 1/8] chore: run integration test workflows using CodeBuild --- .github/workflows/encryption_integration_tests.yml | 4 ++-- .github/workflows/gdb_integration_tests.yml | 2 +- .github/workflows/integration_tests.yml | 6 +++--- .github/workflows/mysql_performance.yml | 2 +- .github/workflows/pg_performance.yml | 2 +- .github/workflows/run-bg-integration-tests.yml | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/encryption_integration_tests.yml b/.github/workflows/encryption_integration_tests.yml index a9b76bf..223b9eb 100644 --- a/.github/workflows/encryption_integration_tests.yml +++ b/.github/workflows/encryption_integration_tests.yml @@ -18,7 +18,7 @@ concurrency: jobs: kms-key: name: Find or create the test KMS key - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} outputs: key_id: ${{ steps.kms.outputs.key_id }} @@ -61,7 +61,7 @@ jobs: encryption-integration-tests: name: Run KMS Encryption Integration Tests needs: kms-key - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} strategy: fail-fast: false matrix: diff --git a/.github/workflows/gdb_integration_tests.yml b/.github/workflows/gdb_integration_tests.yml index a661103..3252a8d 100644 --- a/.github/workflows/gdb_integration_tests.yml +++ b/.github/workflows/gdb_integration_tests.yml @@ -17,7 +17,7 @@ concurrency: jobs: gdb-integration-tests: name: GDB Integration Tests (${{ matrix.environment }}) - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} timeout-minutes: 120 strategy: fail-fast: false diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 37da31d..43fac36 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -17,7 +17,7 @@ concurrency: jobs: default-integration-tests: name: Run Integration Tests (Ruby 3.3, RDS default) - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} strategy: fail-fast: false matrix: @@ -68,7 +68,7 @@ jobs: latest-integration-tests: name: Run Integration Tests (Ruby 4.0, RDS latest) - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} strategy: fail-fast: false matrix: @@ -119,7 +119,7 @@ jobs: multi-az-integration-tests: name: Run Multi-AZ Cluster Integration Tests - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} timeout-minutes: 120 strategy: fail-fast: false diff --git a/.github/workflows/mysql_performance.yml b/.github/workflows/mysql_performance.yml index e5085fa..c8a11e9 100644 --- a/.github/workflows/mysql_performance.yml +++ b/.github/workflows/mysql_performance.yml @@ -14,7 +14,7 @@ concurrency: jobs: aurora-mysql-performance-tests: name: 'Run Aurora MySQL performance tests' - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: 'Clone repository' diff --git a/.github/workflows/pg_performance.yml b/.github/workflows/pg_performance.yml index a373515..5d68b6d 100644 --- a/.github/workflows/pg_performance.yml +++ b/.github/workflows/pg_performance.yml @@ -14,7 +14,7 @@ concurrency: jobs: aurora-pg-performance-tests: name: 'Run Aurora PostgreSQL performance tests' - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: 'Clone repository' diff --git a/.github/workflows/run-bg-integration-tests.yml b/.github/workflows/run-bg-integration-tests.yml index 243a15a..b233673 100644 --- a/.github/workflows/run-bg-integration-tests.yml +++ b/.github/workflows/run-bg-integration-tests.yml @@ -17,7 +17,7 @@ concurrency: jobs: bg-integration-tests: name: BG Integration Tests (${{ matrix.testTask }}) - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} timeout-minutes: 120 strategy: fail-fast: false From c2513bd6b00071e96e62bf643ab6e058b4e7892b Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:55:42 -0700 Subject: [PATCH 2/8] chore: run lint and unit tests using CodeBuild --- .github/workflows/main.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1978201..bebdcbb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,7 @@ permissions: jobs: markdown-link-check: - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: tcort/github-action-markdown-link-check@e047c5b37f24ab722bbef1a27b6fab7f96bc4068 # v1 @@ -29,7 +29,7 @@ jobs: max-depth: 0 lint: - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Set up Ruby @@ -53,7 +53,7 @@ jobs: run: ruby .github/scripts/check_pg_sql_qualified.rb test: - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} strategy: fail-fast: false matrix: @@ -87,7 +87,7 @@ jobs: secret-scan: name: Gitleaks secret scan - runs-on: ubuntu-latest + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: Checkout (full history) uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -104,7 +104,12 @@ jobs: -o gitleaks.tar.gz echo "${GITLEAKS_SHA256} gitleaks.tar.gz" | sha256sum -c - tar -xzf gitleaks.tar.gz gitleaks - sudo install gitleaks /usr/local/bin/gitleaks + # CodeBuild runners run as root and may not ship sudo. + if [ "$(id -u)" -eq 0 ]; then + install gitleaks /usr/local/bin/gitleaks + else + sudo install gitleaks /usr/local/bin/gitleaks + fi rm -f gitleaks.tar.gz gitleaks gitleaks version - name: Run Gitleaks (full git history) From ebab45275165534af7e1182d0ddebf7be079f0f8 Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:37:47 -0700 Subject: [PATCH 3/8] chore: set up docker hub --- .../workflows/encryption_integration_tests.yml | 6 ++++++ .github/workflows/gdb_integration_tests.yml | 6 ++++++ .github/workflows/integration_tests.yml | 18 ++++++++++++++++++ .github/workflows/main.yml | 5 +++++ .github/workflows/mysql_performance.yml | 6 ++++++ .github/workflows/pg_performance.yml | 6 ++++++ .github/workflows/run-bg-integration-tests.yml | 4 ++++ 7 files changed, 51 insertions(+) diff --git a/.github/workflows/encryption_integration_tests.yml b/.github/workflows/encryption_integration_tests.yml index 223b9eb..fcc5920 100644 --- a/.github/workflows/encryption_integration_tests.yml +++ b/.github/workflows/encryption_integration_tests.yml @@ -95,6 +95,12 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: 'Log in to Docker Hub' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: 'Run KMS Encryption Integration Tests' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-${{ matrix.environment }}-encryption --info env: diff --git a/.github/workflows/gdb_integration_tests.yml b/.github/workflows/gdb_integration_tests.yml index 3252a8d..73dad06 100644 --- a/.github/workflows/gdb_integration_tests.yml +++ b/.github/workflows/gdb_integration_tests.yml @@ -51,6 +51,12 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: 'Log in to Docker Hub' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: 'Run GDB Integration Tests' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-${{ matrix.environment }} --info env: diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 43fac36..a69fcee 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -50,6 +50,12 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: 'Log in to Docker Hub' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: 'Run Integration Tests (Ruby 3.3, RDS default)' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-3.3-${{ matrix.environment }} --info env: @@ -101,6 +107,12 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: 'Log in to Docker Hub' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: 'Run Integration Tests (Ruby 4.0, RDS latest)' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-4.0-${{ matrix.environment }} --info env: @@ -153,6 +165,12 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: 'Log in to Docker Hub' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: 'Run Multi-AZ Integration Tests' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-${{ matrix.environment }}-multi-az --info env: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bebdcbb..4dbf630 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,6 +21,11 @@ jobs: runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: 'Log in to Docker Hub' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - uses: tcort/github-action-markdown-link-check@e047c5b37f24ab722bbef1a27b6fab7f96bc4068 # v1 with: use-quiet-mode: 'yes' diff --git a/.github/workflows/mysql_performance.yml b/.github/workflows/mysql_performance.yml index c8a11e9..f43279a 100644 --- a/.github/workflows/mysql_performance.yml +++ b/.github/workflows/mysql_performance.yml @@ -43,6 +43,12 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: 'Log in to Docker Hub' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: 'Run MySQL performance tests' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-3.3-mysql-performance --info env: diff --git a/.github/workflows/pg_performance.yml b/.github/workflows/pg_performance.yml index 5d68b6d..fcd3bd2 100644 --- a/.github/workflows/pg_performance.yml +++ b/.github/workflows/pg_performance.yml @@ -43,6 +43,12 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - name: 'Log in to Docker Hub' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: 'Run PostgreSQL performance tests' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-3.3-pg-performance --info env: diff --git a/.github/workflows/run-bg-integration-tests.yml b/.github/workflows/run-bg-integration-tests.yml index b233673..f7dcb19 100644 --- a/.github/workflows/run-bg-integration-tests.yml +++ b/.github/workflows/run-bg-integration-tests.yml @@ -49,6 +49,10 @@ jobs: role-session-name: ruby_bg_integration_tests role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Run BG Integration Tests run: ./gradlew --no-parallel --no-daemon :integration-testing:${{ matrix.testTask }} --info From cb9cc843a0397096b8de9ad7361a48c2ba367617 Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:05:43 -0700 Subject: [PATCH 4/8] chore: fetch images from ECR --- .../encryption_integration_tests.yml | 7 +- .github/workflows/gdb_integration_tests.yml | 7 +- .github/workflows/integration_tests.yml | 23 +-- .github/workflows/main.yml | 3 +- .github/workflows/mysql_performance.yml | 7 +- .github/workflows/pg_performance.yml | 7 +- .../workflows/run-bg-integration-tests.yml | 5 +- spec/integration/host/build.gradle.kts | 6 + .../integration/host/TestEnvironment.java | 14 +- .../integration/util/ContainerHelper.java | 24 +-- .../integration/util/ContainerImages.java | 138 ++++++++++++++++++ 11 files changed, 165 insertions(+), 76 deletions(-) create mode 100644 spec/integration/host/src/test/java/integration/util/ContainerImages.java diff --git a/.github/workflows/encryption_integration_tests.yml b/.github/workflows/encryption_integration_tests.yml index fcc5920..1c8d26c 100644 --- a/.github/workflows/encryption_integration_tests.yml +++ b/.github/workflows/encryption_integration_tests.yml @@ -95,17 +95,12 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: 'Log in to Docker Hub' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: 'Run KMS Encryption Integration Tests' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-${{ matrix.environment }}-encryption --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} KMS_KEY_ID: ${{ needs.kms-key.outputs.key_id }} + TEST_CONTAINER_REGISTRY: ecr - name: 'Archive encryption results' if: always() diff --git a/.github/workflows/gdb_integration_tests.yml b/.github/workflows/gdb_integration_tests.yml index 73dad06..b4cf72a 100644 --- a/.github/workflows/gdb_integration_tests.yml +++ b/.github/workflows/gdb_integration_tests.yml @@ -51,17 +51,12 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: 'Log in to Docker Hub' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: 'Run GDB Integration Tests' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-${{ matrix.environment }} --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} GDB_SECONDARY_REGION: ${{ secrets.GDB_SECONDARY_REGION }} + TEST_CONTAINER_REGISTRY: ecr - name: 'Archive GDB results' if: always() diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index a69fcee..b576d8b 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -50,18 +50,15 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: 'Log in to Docker Hub' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: 'Run Integration Tests (Ruby 3.3, RDS default)' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-3.3-${{ matrix.environment }} --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} MYSQL_VERSION: default PG_VERSION: default + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr - name: 'Archive results (Ruby 3.3, RDS default)' if: always() @@ -107,18 +104,13 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: 'Log in to Docker Hub' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: 'Run Integration Tests (Ruby 4.0, RDS latest)' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-4.0-${{ matrix.environment }} --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} MYSQL_VERSION: latest PG_VERSION: latest + TEST_CONTAINER_REGISTRY: ecr - name: 'Archive results (Ruby 4.0, RDS latest)' if: always() @@ -165,16 +157,11 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: 'Log in to Docker Hub' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: 'Run Multi-AZ Integration Tests' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-${{ matrix.environment }}-multi-az --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + TEST_CONTAINER_REGISTRY: ecr - name: 'Archive Multi-AZ results' if: always() diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4dbf630..ca82416 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,7 +18,8 @@ permissions: jobs: markdown-link-check: - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + # Intentionally left on a GitHub-hosted runner while the rest of this workflow runs on CodeBuild. + runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: 'Log in to Docker Hub' diff --git a/.github/workflows/mysql_performance.yml b/.github/workflows/mysql_performance.yml index f43279a..523244e 100644 --- a/.github/workflows/mysql_performance.yml +++ b/.github/workflows/mysql_performance.yml @@ -43,16 +43,11 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: 'Log in to Docker Hub' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: 'Run MySQL performance tests' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-3.3-mysql-performance --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + TEST_CONTAINER_REGISTRY: ecr - name: 'Archive performance results' if: always() diff --git a/.github/workflows/pg_performance.yml b/.github/workflows/pg_performance.yml index fcd3bd2..ca01920 100644 --- a/.github/workflows/pg_performance.yml +++ b/.github/workflows/pg_performance.yml @@ -43,16 +43,11 @@ jobs: role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - name: 'Log in to Docker Hub' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: 'Run PostgreSQL performance tests' run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-3.3-pg-performance --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + TEST_CONTAINER_REGISTRY: ecr - name: 'Archive performance results' if: always() diff --git a/.github/workflows/run-bg-integration-tests.yml b/.github/workflows/run-bg-integration-tests.yml index f7dcb19..175c81e 100644 --- a/.github/workflows/run-bg-integration-tests.yml +++ b/.github/workflows/run-bg-integration-tests.yml @@ -49,15 +49,12 @@ jobs: role-session-name: ruby_bg_integration_tests role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Run BG Integration Tests run: ./gradlew --no-parallel --no-daemon :integration-testing:${{ matrix.testTask }} --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + TEST_CONTAINER_REGISTRY: ecr - name: Archive results if: always() diff --git a/spec/integration/host/build.gradle.kts b/spec/integration/host/build.gradle.kts index fceaa43..b092c24 100644 --- a/spec/integration/host/build.gradle.kts +++ b/spec/integration/host/build.gradle.kts @@ -99,6 +99,12 @@ tasks.withType { systemProperty("java.util.logging.config.file", "${project.layout.buildDirectory.get()}/resources/test/logging-test.properties") + // Forwarded explicitly: the test JVM does not inherit the launcher's system properties, so + // -Dtest-container-registry would otherwise be dropped before ContainerImages reads it. + System.getProperty("test-container-registry")?.let { + systemProperty("test-container-registry", it) + } + if (System.getProperty("os.name", "").lowercase().contains("windows")) { environment("DOCKER_HOST", System.getenv("DOCKER_HOST") ?: "tcp://localhost:2377") environment("TESTCONTAINERS_RYUK_DISABLED", "true") diff --git a/spec/integration/host/src/test/java/integration/host/TestEnvironment.java b/spec/integration/host/src/test/java/integration/host/TestEnvironment.java index a9e6457..5ec99f3 100644 --- a/spec/integration/host/src/test/java/integration/host/TestEnvironment.java +++ b/spec/integration/host/src/test/java/integration/host/TestEnvironment.java @@ -35,6 +35,7 @@ import integration.host.TestEnvironmentProvider.EnvPreCreateInfo; import integration.util.AuroraTestUtility; import integration.util.ContainerHelper; +import integration.util.ContainerImages; import java.io.IOException; import java.net.URISyntaxException; import java.net.UnknownHostException; @@ -1602,7 +1603,7 @@ private static void createTestContainer(TestEnvironment env) { env.testContainer = containerHelper.createTestContainer( "aws/rds-test-container", - getContainerBaseImageName(env.info.getRequest())); + ContainerImages.rubyBase(env.info.getRequest().getTargetRubyVersion())); TestEnvironmentConfiguration config = new TestEnvironmentConfiguration(); @@ -1695,17 +1696,6 @@ private static void createTelemetryOtlpContainer(TestEnvironment env) { env.telemetryOtlpContainer.start(); } - private static String getContainerBaseImageName(TestEnvironmentRequest request) { - switch (request.getTargetRubyVersion()) { - case RUBY_3_3: - return "ruby:3.3"; - case RUBY_4_0: - return "ruby:4.0"; - default: - throw new NotImplementedException(request.getTargetRubyVersion().toString()); - } - } - private static void waitForConnectivity(String host, int port) { final int maxAttempts = 10; final int delaySeconds = 15; diff --git a/spec/integration/host/src/test/java/integration/util/ContainerHelper.java b/spec/integration/host/src/test/java/integration/util/ContainerHelper.java index 42cef1b..e51c71a 100644 --- a/spec/integration/host/src/test/java/integration/util/ContainerHelper.java +++ b/spec/integration/host/src/test/java/integration/util/ContainerHelper.java @@ -60,19 +60,9 @@ @SuppressWarnings("unchecked") public class ContainerHelper { - private static final String MYSQL_CONTAINER_IMAGE_NAME = "mysql:8.0.31"; - private static final String POSTGRES_CONTAINER_IMAGE_NAME = "postgres:latest"; - private static final String VALKEY_CONTAINER_IMAGE_NAME = "valkey/valkey:8.1"; - // Note: this image version may need to be occasionally updated to keep it up-to-date and prevent toxiproxy issues. - private static final DockerImageName TOXIPROXY_IMAGE = - DockerImageName.parse("ghcr.io/shopify/toxiproxy:2.11.0"); - private static final int PROXY_CONTROL_PORT = 8474; private static final int PROXY_PORT = 8666; - private static final String XRAY_TELEMETRY_IMAGE_NAME = "amazon/aws-xray-daemon"; - private static final String OTLP_TELEMETRY_IMAGE_NAME = "amazon/aws-otel-collector"; - private static final String RETRIEVE_TOPOLOGY_SQL_POSTGRES = "SELECT SERVER_ID, SESSION_ID FROM pg_catalog.aurora_replica_status() " + "ORDER BY CASE WHEN SESSION_ID OPERATOR(pg_catalog.=) 'MASTER_SESSION_ID' THEN 0 ELSE 1 END"; @@ -207,7 +197,7 @@ public GenericContainer createTelemetryXrayContainer( new ImageFromDockerfile("xray-daemon", true) .withDockerfileFromBuilder( builder -> builder - .from(XRAY_TELEMETRY_IMAGE_NAME) + .from(ContainerImages.xrayDaemon()) .entryPoint("/xray", "-t", "0.0.0.0:2000", "-b", "0.0.0.0:2000", @@ -226,7 +216,7 @@ public GenericContainer createTelemetryOtlpContainer( Network network, String networkAlias) { - return new FixedExposedPortContainer<>(DockerImageName.parse(OTLP_TELEMETRY_IMAGE_NAME)) + return new FixedExposedPortContainer<>(ContainerImages.otelCollector()) .withExposedPort(2000) .withExposedPort(1777) .withExposedPort(4317) @@ -432,7 +422,7 @@ protected boolean isRunning(InspectContainerResponse containerInfo) { public MySQLContainer createMysqlContainer( Network network, String networkAlias, String testDbName, String username, String password) { - return new MySQLContainer<>(MYSQL_CONTAINER_IMAGE_NAME) + return new MySQLContainer<>(ContainerImages.mysql()) .withNetwork(network) .withNetworkAliases(networkAlias) .withDatabaseName(testDbName) @@ -458,7 +448,7 @@ public MySQLContainer createMysqlContainer( public PostgreSQLContainer createPostgresContainer( Network network, String networkAlias, String testDbName, String username, String password) { - return new PostgreSQLContainer<>(POSTGRES_CONTAINER_IMAGE_NAME) + return new PostgreSQLContainer<>(ContainerImages.postgres()) .withNetwork(network) .withNetworkAliases(networkAlias) .withDatabaseName(testDbName) @@ -473,7 +463,7 @@ public ToxiproxyContainer createAndStartProxyContainer( String hostname, int port) throws IOException { final ToxiproxyContainer container = - new ToxiproxyContainer(TOXIPROXY_IMAGE) + new ToxiproxyContainer(ContainerImages.toxiproxy()) .withNetwork(network) .withNetworkAliases(networkAlias, networkUrl); container.start(); @@ -490,7 +480,7 @@ public GenericContainer createValkeyContainer( boolean authEnabled, boolean tlsEnabled) { - GenericContainer container = new GenericContainer<>(VALKEY_CONTAINER_IMAGE_NAME) + GenericContainer container = new GenericContainer<>(ContainerImages.valkey()) .withNetwork(network) .withNetworkAliases(networkAlias); @@ -562,7 +552,7 @@ public void createProxy(final ToxiproxyClient client, String hostname, int port) public ToxiproxyContainer createProxyContainer( final Network network, TestInstanceInfo instance, String proxyDomainNameSuffix) { - return new ToxiproxyContainer(TOXIPROXY_IMAGE) + return new ToxiproxyContainer(ContainerImages.toxiproxy()) .withNetwork(network) .withNetworkAliases( "proxy-instance-" + instance.getInstanceId(), diff --git a/spec/integration/host/src/test/java/integration/util/ContainerImages.java b/spec/integration/host/src/test/java/integration/util/ContainerImages.java new file mode 100644 index 0000000..25ca47b --- /dev/null +++ b/spec/integration/host/src/test/java/integration/util/ContainerImages.java @@ -0,0 +1,138 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration.util; + +import integration.TargetRubyVersion; +import java.util.Locale; +import java.util.logging.Logger; +import org.testcontainers.shaded.org.apache.commons.lang3.NotImplementedException; +import org.testcontainers.utility.DockerImageName; + +/** + * Single source for the container images the integration tests pull, so a run can take them either + * from Docker Hub or from Amazon ECR Public. + * + *

Docker Hub rate limits anonymous pulls per source IP. A runner that shares its IP with others, + * or any run that has no Docker Hub account to authenticate with, hits that limit and fails while + * pulling. Such a run selects the ECR Public mirrors instead, which need no credentials: + * + * 

+ *   ./gradlew :integration-testing:test-ruby-3.3-pg -Dtest-container-registry=ecr
+ *   TEST_CONTAINER_REGISTRY=ecr ./gradlew :integration-testing:test-ruby-3.3-pg
+ *
+ * + *

The system property wins over the environment variable; the default is {@code dockerhub}, which + * keeps every image exactly where it was before this setting existed. + * + *

Toxiproxy is unaffected by this setting: it is served from GitHub's registry, which does not + * apply Docker Hub's limits, and has no ECR Public mirror. + */ +public final class ContainerImages { + + private static final Logger LOGGER = Logger.getLogger(ContainerImages.class.getName()); + + private static final String DOCKER_HUB = "dockerhub"; + private static final String ECR = "ecr"; + + /** ECR Public mirror of the Docker Official Images, tag-for-tag identical to Docker Hub. */ + private static final String ECR_DOCKER_LIBRARY = "public.ecr.aws/docker/library/"; + + private static final boolean USE_ECR = resolveUseEcr(); + + private ContainerImages() { + } + + /** Base image of the container the integration tests themselves run in. */ + public static String rubyBase(TargetRubyVersion rubyVersion) { + switch (rubyVersion) { + case RUBY_3_3: + return dockerOfficialImage("ruby", "3.3"); + case RUBY_4_0: + return dockerOfficialImage("ruby", "4.0"); + default: + throw new NotImplementedException(rubyVersion.toString()); + } + } + + public static DockerImageName mysql() { + return dockerOfficial("mysql", "8.0.31"); + } + + public static DockerImageName postgres() { + return dockerOfficial("postgres", "latest"); + } + + public static DockerImageName valkey() { + return DockerImageName.parse( + pick("valkey/valkey:8.1", "public.ecr.aws/valkey/valkey:8.1")); + } + + /** Used as a Dockerfile {@code FROM}, so this returns the raw coordinate. */ + public static String xrayDaemon() { + return pick("amazon/aws-xray-daemon", "public.ecr.aws/xray/aws-xray-daemon"); + } + + public static DockerImageName otelCollector() { + return DockerImageName.parse(pick( + "amazon/aws-otel-collector", + "public.ecr.aws/aws-observability/aws-otel-collector")); + } + + /** + * Note: this image version may need to be occasionally updated to keep it up-to-date and prevent + * toxiproxy issues. + */ + public static DockerImageName toxiproxy() { + return DockerImageName.parse("ghcr.io/shopify/toxiproxy:2.11.0"); + } + + /** + * Resolves a Docker Official Image and marks it as a substitute for its canonical Docker Hub name, + * which the Testcontainers database container classes assert against. + */ + private static DockerImageName dockerOfficial(String repository, String tag) { + return DockerImageName.parse(dockerOfficialImage(repository, tag)) + .asCompatibleSubstituteFor(repository); + } + + private static String dockerOfficialImage(String repository, String tag) { + return (USE_ECR ? ECR_DOCKER_LIBRARY : "") + repository + ":" + tag; + } + + private static String pick(String dockerHubImage, String ecrImage) { + return USE_ECR ? ecrImage : dockerHubImage; + } + + private static boolean resolveUseEcr() { + String registry = System.getProperty("test-container-registry"); + if (StringUtils.isNullOrEmpty(registry)) { + registry = System.getenv("TEST_CONTAINER_REGISTRY"); + } + if (StringUtils.isNullOrEmpty(registry)) { + registry = DOCKER_HUB; + } + final String resolved = registry.trim().toLowerCase(Locale.ROOT); + + if (!ECR.equals(resolved) && !DOCKER_HUB.equals(resolved)) { + throw new IllegalArgumentException(String.format( + "Unsupported container registry '%s'. Expected '%s' or '%s'.", resolved, DOCKER_HUB, ECR)); + } + + LOGGER.finest(() -> "Pulling integration test container images from: " + resolved); + return ECR.equals(resolved); + } +} From 4f8ee3d6fc9cc748e45b7858e175dfd3954aea99 Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:04:12 -0700 Subject: [PATCH 5/8] chore: address review comments # Conflicts: # .github/workflows/encryption_integration_tests.yml # .github/workflows/gdb_integration_tests.yml # .github/workflows/integration_tests.yml # .github/workflows/mysql_performance.yml # .github/workflows/pg_performance.yml # .github/workflows/run-bg-integration-tests.yml --- .../encryption_integration_tests.yml | 6 +++++- .github/workflows/gdb_integration_tests.yml | 6 +++++- .github/workflows/integration_tests.yml | 10 +++++----- .github/workflows/main.yml | 19 ++++--------------- .github/workflows/mysql_performance.yml | 6 +++++- .github/workflows/pg_performance.yml | 6 +++++- .../workflows/run-bg-integration-tests.yml | 6 +++++- spec/integration/host/build.gradle.kts | 11 +++++++++++ .../integration/util/ContainerImages.java | 2 +- 9 files changed, 46 insertions(+), 26 deletions(-) diff --git a/.github/workflows/encryption_integration_tests.yml b/.github/workflows/encryption_integration_tests.yml index 1c8d26c..d13b38a 100644 --- a/.github/workflows/encryption_integration_tests.yml +++ b/.github/workflows/encryption_integration_tests.yml @@ -10,6 +10,11 @@ permissions: id-token: write contents: read +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + # These workflows create real RDS clusters and a real KMS key. concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -100,7 +105,6 @@ jobs: env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} KMS_KEY_ID: ${{ needs.kms-key.outputs.key_id }} - TEST_CONTAINER_REGISTRY: ecr - name: 'Archive encryption results' if: always() diff --git a/.github/workflows/gdb_integration_tests.yml b/.github/workflows/gdb_integration_tests.yml index b4cf72a..d7473b6 100644 --- a/.github/workflows/gdb_integration_tests.yml +++ b/.github/workflows/gdb_integration_tests.yml @@ -10,6 +10,11 @@ permissions: id-token: write contents: read +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -56,7 +61,6 @@ jobs: env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} GDB_SECONDARY_REGION: ${{ secrets.GDB_SECONDARY_REGION }} - TEST_CONTAINER_REGISTRY: ecr - name: 'Archive GDB results' if: always() diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index b576d8b..5682a61 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -10,6 +10,11 @@ permissions: id-token: write contents: read +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -56,9 +61,6 @@ jobs: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} MYSQL_VERSION: default PG_VERSION: default - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. - TEST_CONTAINER_REGISTRY: ecr - name: 'Archive results (Ruby 3.3, RDS default)' if: always() @@ -110,7 +112,6 @@ jobs: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} MYSQL_VERSION: latest PG_VERSION: latest - TEST_CONTAINER_REGISTRY: ecr - name: 'Archive results (Ruby 4.0, RDS latest)' if: always() @@ -161,7 +162,6 @@ jobs: run: ./gradlew --no-parallel --no-daemon :integration-testing:test-${{ matrix.environment }}-multi-az --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - TEST_CONTAINER_REGISTRY: ecr - name: 'Archive Multi-AZ results' if: always() diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ca82416..1978201 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,15 +18,9 @@ permissions: jobs: markdown-link-check: - # Intentionally left on a GitHub-hosted runner while the rest of this workflow runs on CodeBuild. runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - - name: 'Log in to Docker Hub' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - uses: tcort/github-action-markdown-link-check@e047c5b37f24ab722bbef1a27b6fab7f96bc4068 # v1 with: use-quiet-mode: 'yes' @@ -35,7 +29,7 @@ jobs: max-depth: 0 lint: - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Set up Ruby @@ -59,7 +53,7 @@ jobs: run: ruby .github/scripts/check_pg_sql_qualified.rb test: - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -93,7 +87,7 @@ jobs: secret-scan: name: Gitleaks secret scan - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest steps: - name: Checkout (full history) uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -110,12 +104,7 @@ jobs: -o gitleaks.tar.gz echo "${GITLEAKS_SHA256} gitleaks.tar.gz" | sha256sum -c - tar -xzf gitleaks.tar.gz gitleaks - # CodeBuild runners run as root and may not ship sudo. - if [ "$(id -u)" -eq 0 ]; then - install gitleaks /usr/local/bin/gitleaks - else - sudo install gitleaks /usr/local/bin/gitleaks - fi + sudo install gitleaks /usr/local/bin/gitleaks rm -f gitleaks.tar.gz gitleaks gitleaks version - name: Run Gitleaks (full git history) diff --git a/.github/workflows/mysql_performance.yml b/.github/workflows/mysql_performance.yml index 523244e..3f5f5bc 100644 --- a/.github/workflows/mysql_performance.yml +++ b/.github/workflows/mysql_performance.yml @@ -7,6 +7,11 @@ permissions: id-token: write contents: read +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -47,7 +52,6 @@ jobs: run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-3.3-mysql-performance --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - TEST_CONTAINER_REGISTRY: ecr - name: 'Archive performance results' if: always() diff --git a/.github/workflows/pg_performance.yml b/.github/workflows/pg_performance.yml index ca01920..6c7f9e4 100644 --- a/.github/workflows/pg_performance.yml +++ b/.github/workflows/pg_performance.yml @@ -7,6 +7,11 @@ permissions: id-token: write contents: read +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -47,7 +52,6 @@ jobs: run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-3.3-pg-performance --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - TEST_CONTAINER_REGISTRY: ecr - name: 'Archive performance results' if: always() diff --git a/.github/workflows/run-bg-integration-tests.yml b/.github/workflows/run-bg-integration-tests.yml index 175c81e..3ad4442 100644 --- a/.github/workflows/run-bg-integration-tests.yml +++ b/.github/workflows/run-bg-integration-tests.yml @@ -10,6 +10,11 @@ permissions: id-token: write contents: read +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -54,7 +59,6 @@ jobs: run: ./gradlew --no-parallel --no-daemon :integration-testing:${{ matrix.testTask }} --info env: RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} - TEST_CONTAINER_REGISTRY: ecr - name: Archive results if: always() diff --git a/spec/integration/host/build.gradle.kts b/spec/integration/host/build.gradle.kts index b092c24..2c7518e 100644 --- a/spec/integration/host/build.gradle.kts +++ b/spec/integration/host/build.gradle.kts @@ -82,6 +82,10 @@ tasks.test { filter.excludeTestsMatching("integration.*") } +val usingEcrImages = + (System.getProperty("test-container-registry") ?: System.getenv("TEST_CONTAINER_REGISTRY")) + ?.trim()?.lowercase() == "ecr" + tasks.withType { useJUnitPlatform() outputs.upToDateWhen { false } @@ -108,6 +112,13 @@ tasks.withType { if (System.getProperty("os.name", "").lowercase().contains("windows")) { environment("DOCKER_HOST", System.getenv("DOCKER_HOST") ?: "tcp://localhost:2377") environment("TESTCONTAINERS_RYUK_DISABLED", "true") + } else if (usingEcrImages && System.getenv("TESTCONTAINERS_RYUK_DISABLED") == null) { + // Selecting ECR moves every image this suite names off Docker Hub, but Testcontainers also + // starts its own Ryuk cleanup container, and testcontainers/ryuk has no ECR Public mirror. + // Left enabled, a run that chose ECR to escape Docker Hub's anonymous pull limit would + // still make an anonymous Docker Hub pull and could still fail on that limit. The runners + // that select ECR are ephemeral, so there are no leftover containers for Ryuk to clean up. + environment("TESTCONTAINERS_RYUK_DISABLED", "true") } } diff --git a/spec/integration/host/src/test/java/integration/util/ContainerImages.java b/spec/integration/host/src/test/java/integration/util/ContainerImages.java index 25ca47b..4f0bd49 100644 --- a/spec/integration/host/src/test/java/integration/util/ContainerImages.java +++ b/spec/integration/host/src/test/java/integration/util/ContainerImages.java @@ -73,7 +73,7 @@ public static DockerImageName mysql() { } public static DockerImageName postgres() { - return dockerOfficial("postgres", "latest"); + return dockerOfficial("postgres", "17.11"); } public static DockerImageName valkey() { From 5e1f9e01e77e8ede633a0a8c05c857a4fb11a8ae Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Wed, 14 Oct 2026 19:14:52 -0700 Subject: [PATCH 6/8] fix: performance test naming error --- spec/integration/container/performance_spec.rb | 6 +++--- .../test/java/integration/host/TestEnvironment.java | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/spec/integration/container/performance_spec.rb b/spec/integration/container/performance_spec.rb index 9d952dc..df10809 100644 --- a/spec/integration/container/performance_spec.rb +++ b/spec/integration/container/performance_spec.rb @@ -86,9 +86,9 @@ def native_config def wrapper_props(timeout_sec) { - AwsRubyDatabaseDriverWrapper::PropertyDefinition::PLUGINS.name => 'failover', - AwsRubyDatabaseDriverWrapper::PropertyDefinition::FAILOVER_TIMEOUT_SEC.name => 300, - AwsRubyDatabaseDriverWrapper::PropertyDefinition::CLUSTER_INSTANCE_HOST_PATTERN.name => + AwsAdvancedRubyDriverWrapper::PropertyDefinition::PLUGINS.name => 'failover', + AwsAdvancedRubyDriverWrapper::PropertyDefinition::FAILOVER_TIMEOUT_SEC.name => 300, + AwsAdvancedRubyDriverWrapper::PropertyDefinition::CLUSTER_INSTANCE_HOST_PATTERN.name => "?.#{proxy_info.instance_endpoint_suffix}:#{proxy_info.instance_endpoint_port}", connect_timeout: PERF_RECONNECT_TIMEOUT_SEC, **driver_timeout_props(timeout_sec) diff --git a/spec/integration/host/src/test/java/integration/host/TestEnvironment.java b/spec/integration/host/src/test/java/integration/host/TestEnvironment.java index 5ec99f3..c6a405a 100644 --- a/spec/integration/host/src/test/java/integration/host/TestEnvironment.java +++ b/spec/integration/host/src/test/java/integration/host/TestEnvironment.java @@ -1264,8 +1264,9 @@ private static void deAuthorizeIP(TestEnvironment env) { } if (!env.reuseDb) { - env.auroraUtil.ec2DeauthorizesIP(env.runnerIP); - LOGGER.finest(String.format("Test runner IP de-authorized. Usage count: %d", + // TODO: TEMPORARY + LOGGER.finest(String.format( + "Test runner IP de-authorization skipped (temporarily disabled). Usage count: %d", ipAddressUsageRefCount.get())); } else { LOGGER.finest("The IP address usage count hit 0, but the REUSE_RDS_DB was set to true, so IP " @@ -2002,8 +2003,11 @@ public void close() throws Exception { try { AuroraTestUtility secondaryUtil = new AuroraTestUtility(secondaryRegionCleanup, null); String runnerIP = secondaryUtil.getPublicIPAddress(); - secondaryUtil.ec2DeauthorizesIP(runnerIP); - LOGGER.finer("De-authorized runner IP in secondary region " + secondaryRegionCleanup); + // TODO: TEMPORARY - see deAuthorizeIP. Not even refcount-guarded, so the first GDB leg + // to finish revoked the secondary-region ingress the other leg was still using. + // secondaryUtil.ec2DeauthorizesIP(runnerIP); + LOGGER.finer("De-authorization skipped (temporarily disabled) in secondary region " + + secondaryRegionCleanup); } catch (Exception ex) { LOGGER.warning("Failed to de-authorize IP in secondary region: " + ex.getMessage()); } From b099c087b157f45be5a771d17ffff01c6dc705ab Mon Sep 17 00:00:00 2001 From: Sophia Chu <112967780+sophia-bq@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:17:55 -0700 Subject: [PATCH 7/8] refactor: dual gh and codebuild approach --- ...encryption_integration_tests-codebuild.yml | 128 +++++++++++ .../encryption_integration_tests.yml | 24 +- .../gdb_integration_tests-codebuild.yml | 83 +++++++ .github/workflows/gdb_integration_tests.yml | 22 +- .../workflows/integration_tests-codebuild.yml | 184 +++++++++++++++ .github/workflows/integration_tests.yml | 26 ++- .github/workflows/main-codebuild.yml | 214 ++++++++++++++++++ .github/workflows/main.yml | 81 ++++++- .../workflows/mysql_performance-codebuild.yml | 75 ++++++ .github/workflows/mysql_performance.yml | 15 +- .../workflows/pg_performance-codebuild.yml | 75 ++++++ .github/workflows/pg_performance.yml | 15 +- .../remove-old-artifacts-codebuild.yml | 32 +++ .github/workflows/remove-old-artifacts.yml | 13 +- .../run-bg-integration-tests-codebuild.yml | 80 +++++++ .../workflows/run-bg-integration-tests.yml | 22 +- .../integration/util/AuroraTestUtility.java | 116 ++++------ 17 files changed, 1094 insertions(+), 111 deletions(-) create mode 100644 .github/workflows/encryption_integration_tests-codebuild.yml create mode 100644 .github/workflows/gdb_integration_tests-codebuild.yml create mode 100644 .github/workflows/integration_tests-codebuild.yml create mode 100644 .github/workflows/main-codebuild.yml create mode 100644 .github/workflows/mysql_performance-codebuild.yml create mode 100644 .github/workflows/pg_performance-codebuild.yml create mode 100644 .github/workflows/remove-old-artifacts-codebuild.yml create mode 100644 .github/workflows/run-bg-integration-tests-codebuild.yml diff --git a/.github/workflows/encryption_integration_tests-codebuild.yml b/.github/workflows/encryption_integration_tests-codebuild.yml new file mode 100644 index 0000000..ea78479 --- /dev/null +++ b/.github/workflows/encryption_integration_tests-codebuild.yml @@ -0,0 +1,128 @@ +# CodeBuild variant of encryption_integration_tests.yml. +# +# Runs the same jobs on the self-hosted CodeBuild runners instead of GitHub-hosted ones. +# Triggers: the same as encryption_integration_tests.yml (manual dispatch and push to main). The +# push trigger is commented out in encryption_integration_tests.yml, so each push to main runs only +# this copy. Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild +# RDS workflows, so only one of them claims DB clusters, security-group rules, and the KMS key at a +# time. + +name: Encryption Integration Tests (CodeBuild) + +on: + workflow_dispatch: + push: + branches: + - main + +permissions: + id-token: write + contents: read + +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + +# These workflows create real RDS clusters and a real KMS key. +concurrency: + # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests-codebuild + queue: max + +jobs: + kms-key: + name: Find or create the test KMS key + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + outputs: + key_id: ${{ steps.kms.outputs.key_id }} + + steps: + - name: 'Configure AWS Credentials' + id: creds + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: ruby_encryption_kms_key + role-duration-seconds: 900 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Find or create the test KMS key' + id: kms + run: | + ALIAS="alias/ruby-encryption-key" + KEY_ID=$(aws kms list-aliases \ + --query "Aliases[?AliasName=='${ALIAS}'].TargetKeyId" --output text 2>/dev/null || echo "") + + if [ -n "$KEY_ID" ]; then + STATE=$(aws kms describe-key --key-id "$KEY_ID" \ + --query 'KeyMetadata.KeyState' --output text 2>/dev/null || echo "") + if [ "$STATE" != "Enabled" ]; then KEY_ID=""; fi + fi + + if [ -z "$KEY_ID" ]; then + KEY_ID=$(aws kms create-key \ + --description "aws_advanced_ruby_driver_wrapper encryption test key (cached)" \ + --key-usage ENCRYPT_DECRYPT --key-spec SYMMETRIC_DEFAULT \ + --tags TagKey=Project,TagValue=aws_advanced_ruby_driver_wrapper TagKey=Purpose,TagValue=testing \ + --query 'KeyMetadata.KeyId' --output text) + aws kms create-alias --alias-name "$ALIAS" --target-key-id "$KEY_ID" + aws kms enable-key-rotation --key-id "$KEY_ID" + fi + + echo "key_id=$KEY_ID" >> "$GITHUB_OUTPUT" + echo "Using KMS key: $KEY_ID" + + encryption-integration-tests: + name: Run KMS Encryption Integration Tests + needs: kms-key + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + strategy: + fail-fast: false + matrix: + environment: [ 'mysql', 'pg' ] + + steps: + - name: 'Clone repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: 'Validate Gradle wrapper' + uses: gradle/actions/wrapper-validation@3f5f9adaf7d9fecd50b5935e54106014257a94e6 # v6.4.0 + + - name: 'Set up Ruby 3.3' + uses: ruby/setup-ruby@651ff0d62c5c79bed6d53ec518387aae20e72c1a # v1 + with: + ruby-version: '3.3' + bundler-cache: true + + - name: 'Set up JDK 8' + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6 + with: + distribution: 'corretto' + java-version: 8 + + - name: 'Configure AWS Credentials' + id: creds + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: ruby_encryption_integration_tests + role-duration-seconds: 21600 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Run KMS Encryption Integration Tests' + run: ./gradlew --no-parallel --no-daemon :integration-testing:test-${{ matrix.environment }}-encryption --info + env: + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + KMS_KEY_ID: ${{ needs.kms-key.outputs.key_id }} + + - name: 'Archive encryption results' + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: rspec-encryption-report-${{ matrix.environment }} + path: ./spec/integration/host/build/reports/tests/test/ + if-no-files-found: ignore + retention-days: 5 diff --git a/.github/workflows/encryption_integration_tests.yml b/.github/workflows/encryption_integration_tests.yml index d13b38a..c5671e1 100644 --- a/.github/workflows/encryption_integration_tests.yml +++ b/.github/workflows/encryption_integration_tests.yml @@ -1,10 +1,17 @@ +# GitHub-hosted backup of the CodeBuild workflow encryption_integration_tests-codebuild.yml. +# +# Runs the same jobs on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the +# CodeBuild variant owns the push trigger (commented out below). Shares the rds-integration-tests +# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). + name: Encryption Integration Tests on: workflow_dispatch: - push: - branches: - - main + # The push trigger moved to the CodeBuild variant encryption_integration_tests-codebuild.yml. + # push: + # branches: + # - main permissions: id-token: write @@ -17,13 +24,16 @@ env: # These workflows create real RDS clusters and a real KMS key. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests + queue: max jobs: kms-key: name: Find or create the test KMS key - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest outputs: key_id: ${{ steps.kms.outputs.key_id }} @@ -66,7 +76,7 @@ jobs: encryption-integration-tests: name: Run KMS Encryption Integration Tests needs: kms-key - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/gdb_integration_tests-codebuild.yml b/.github/workflows/gdb_integration_tests-codebuild.yml new file mode 100644 index 0000000..3a5c3fa --- /dev/null +++ b/.github/workflows/gdb_integration_tests-codebuild.yml @@ -0,0 +1,83 @@ +# CodeBuild variant of gdb_integration_tests.yml. +# +# Runs the same jobs on the self-hosted CodeBuild runners instead of GitHub-hosted ones. +# Triggers: the same as gdb_integration_tests.yml (manual dispatch and a weekly schedule). The +# schedule is commented out in gdb_integration_tests.yml, so only this copy runs on schedule. +# Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild RDS +# workflows, so only one of them claims DB clusters and security-group rules at a time. + +name: Global Database Integration Tests (CodeBuild) + +on: + workflow_dispatch: + schedule: + # Every Monday at 9:00 AM UTC + - cron: '0 9 * * 1' + +permissions: + id-token: write + contents: read + +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + +concurrency: + # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests-codebuild + queue: max + +jobs: + gdb-integration-tests: + name: GDB Integration Tests (${{ matrix.environment }}) + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + environment: [ 'gdb-mysql', 'gdb-pg' ] + + steps: + - name: 'Clone repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: 'Validate Gradle wrapper' + uses: gradle/actions/wrapper-validation@3f5f9adaf7d9fecd50b5935e54106014257a94e6 # v6.4.0 + + - name: 'Set up Ruby 4.0' + uses: ruby/setup-ruby@651ff0d62c5c79bed6d53ec518387aae20e72c1a # v1 + with: + ruby-version: '4.0' + bundler-cache: true + + - name: 'Set up JDK 8' + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6 + with: + distribution: 'corretto' + java-version: 8 + + - name: 'Configure AWS Credentials' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: ruby_gdb_integration_tests + role-duration-seconds: 21600 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Run GDB Integration Tests' + run: ./gradlew --no-parallel --no-daemon :integration-testing:test-${{ matrix.environment }} --info + env: + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + GDB_SECONDARY_REGION: ${{ secrets.GDB_SECONDARY_REGION }} + + - name: 'Archive GDB results' + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: rspec-gdb-integration-report-${{ matrix.environment }} + path: ./spec/integration/host/build/reports/tests/test/ + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/gdb_integration_tests.yml b/.github/workflows/gdb_integration_tests.yml index d7473b6..470539a 100644 --- a/.github/workflows/gdb_integration_tests.yml +++ b/.github/workflows/gdb_integration_tests.yml @@ -1,10 +1,17 @@ +# GitHub-hosted backup of the CodeBuild workflow gdb_integration_tests-codebuild.yml. +# +# Runs the same jobs on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the +# CodeBuild variant owns the schedule (commented out below). Shares the rds-integration-tests +# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). + name: Global Database Integration Tests on: workflow_dispatch: - schedule: - # Every Monday at 9:00 AM UTC - - cron: '0 9 * * 1' + # The schedule moved to the CodeBuild variant gdb_integration_tests-codebuild.yml. + # schedule: + # # Every Monday at 9:00 AM UTC + # - cron: '0 9 * * 1' permissions: id-token: write @@ -16,13 +23,16 @@ env: TEST_CONTAINER_REGISTRY: ecr concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests + queue: max jobs: gdb-integration-tests: name: GDB Integration Tests (${{ matrix.environment }}) - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest timeout-minutes: 120 strategy: fail-fast: false diff --git a/.github/workflows/integration_tests-codebuild.yml b/.github/workflows/integration_tests-codebuild.yml new file mode 100644 index 0000000..0d55e67 --- /dev/null +++ b/.github/workflows/integration_tests-codebuild.yml @@ -0,0 +1,184 @@ +# CodeBuild variant of integration_tests.yml. +# +# Runs the same jobs on the self-hosted CodeBuild runners instead of GitHub-hosted ones. +# Triggers: the same as integration_tests.yml (manual dispatch and push to main). The push trigger +# is commented out in integration_tests.yml, so each push to main runs only this copy. +# Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild RDS +# workflows, so only one of them claims DB clusters and security-group rules at a time. + +name: Integration Tests (CodeBuild) + +on: + workflow_dispatch: + push: + branches: + - main + +permissions: + id-token: write + contents: read + +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + +concurrency: + # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests-codebuild + queue: max + +jobs: + default-integration-tests: + name: Run Integration Tests (Ruby 3.3, RDS default) + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + strategy: + fail-fast: false + matrix: + environment: [ 'mysql', 'pg' ] + + steps: + - name: 'Clone repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: 'Set up Ruby 3.3' + uses: ruby/setup-ruby@651ff0d62c5c79bed6d53ec518387aae20e72c1a # v1 + with: + ruby-version: '3.3' + bundler-cache: true + + - name: 'Set up JDK 8' + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6 + with: + distribution: 'corretto' + java-version: 8 + + - name: 'Set up Gradle' + uses: gradle/actions/setup-gradle@3f5f9adaf7d9fecd50b5935e54106014257a94e6 # v6.4.0 + + - name: 'Configure AWS Credentials' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: ruby_integration_tests + role-duration-seconds: 21600 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Run Integration Tests (Ruby 3.3, RDS default)' + run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-3.3-${{ matrix.environment }} --info + env: + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + MYSQL_VERSION: default + PG_VERSION: default + + - name: 'Archive results (Ruby 3.3, RDS default)' + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: rspec-integration-report-3.3-${{ matrix.environment }}-default + path: ./spec/integration/host/build/reports/tests/test/ + if-no-files-found: ignore + retention-days: 5 + + latest-integration-tests: + name: Run Integration Tests (Ruby 4.0, RDS latest) + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + strategy: + fail-fast: false + matrix: + environment: [ 'mysql', 'pg' ] + + steps: + - name: 'Clone repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: 'Set up Ruby 4.0' + uses: ruby/setup-ruby@651ff0d62c5c79bed6d53ec518387aae20e72c1a # v1 + with: + ruby-version: '4.0' + bundler-cache: true + + - name: 'Set up JDK 8' + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6 + with: + distribution: 'corretto' + java-version: 8 + + - name: 'Set up Gradle' + uses: gradle/actions/setup-gradle@3f5f9adaf7d9fecd50b5935e54106014257a94e6 # v6.4.0 + + - name: 'Configure AWS Credentials' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: ruby_integration_tests + role-duration-seconds: 21600 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Run Integration Tests (Ruby 4.0, RDS latest)' + run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-4.0-${{ matrix.environment }} --info + env: + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + MYSQL_VERSION: latest + PG_VERSION: latest + + - name: 'Archive results (Ruby 4.0, RDS latest)' + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: rspec-integration-report-4.0-${{ matrix.environment }}-latest + path: ./spec/integration/host/build/reports/tests/test/ + if-no-files-found: ignore + retention-days: 5 + + multi-az-integration-tests: + name: Run Multi-AZ Cluster Integration Tests + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + environment: [ 'mysql', 'pg' ] + + steps: + - name: 'Clone repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: 'Set up Ruby 3.3' + uses: ruby/setup-ruby@651ff0d62c5c79bed6d53ec518387aae20e72c1a # v1 + with: + ruby-version: '3.3' + bundler-cache: true + + - name: 'Set up JDK 8' + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6 + with: + distribution: 'corretto' + java-version: 8 + + - name: 'Set up Gradle' + uses: gradle/actions/setup-gradle@3f5f9adaf7d9fecd50b5935e54106014257a94e6 # v6.4.0 + + - name: 'Configure AWS Credentials' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: ruby_multi_az_integration_tests + role-duration-seconds: 21600 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Run Multi-AZ Integration Tests' + run: ./gradlew --no-parallel --no-daemon :integration-testing:test-${{ matrix.environment }}-multi-az --info + env: + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Archive Multi-AZ results' + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: rspec-integration-report-multi-az-${{ matrix.environment }} + path: ./spec/integration/host/build/reports/tests/test/ + if-no-files-found: ignore + retention-days: 5 diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 5682a61..d751be2 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1,10 +1,17 @@ +# GitHub-hosted backup of the CodeBuild workflow integration_tests-codebuild.yml. +# +# Runs the same jobs on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the +# CodeBuild variant owns the push trigger (commented out below). Shares the rds-integration-tests +# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). + name: Integration Tests on: workflow_dispatch: - push: - branches: - - main + # The push trigger moved to the CodeBuild variant integration_tests-codebuild.yml. + # push: + # branches: + # - main permissions: id-token: write @@ -16,13 +23,16 @@ env: TEST_CONTAINER_REGISTRY: ecr concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests + queue: max jobs: default-integration-tests: name: Run Integration Tests (Ruby 3.3, RDS default) - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -73,7 +83,7 @@ jobs: latest-integration-tests: name: Run Integration Tests (Ruby 4.0, RDS latest) - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -124,7 +134,7 @@ jobs: multi-az-integration-tests: name: Run Multi-AZ Cluster Integration Tests - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest timeout-minutes: 120 strategy: fail-fast: false diff --git a/.github/workflows/main-codebuild.yml b/.github/workflows/main-codebuild.yml new file mode 100644 index 0000000..a7c49a9 --- /dev/null +++ b/.github/workflows/main-codebuild.yml @@ -0,0 +1,214 @@ +# CodeBuild variant of main.yml. +# +# Runs the same jobs on the self-hosted CodeBuild runners instead of GitHub-hosted ones. +# Triggers: the same as main.yml (manual dispatch, push to main, pull_request). The push and +# pull_request triggers are commented out in main.yml, so each event runs only this copy. +# pull_request runs a fork PR's code on these self-hosted runners. In place: the repository requires +# maintainer approval before workflows run for all external contributors, and each job is a +# separate CodeBuild build. As on GitHub-hosted runners, a fork PR's run gets no secrets and no +# id-token: write. + +name: Check and Unit Tests (CodeBuild) + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + branches: + - '*' + +concurrency: + # Runs in this group execute one at a time; with `queue: max`, later runs wait in FIFO order + # (up to 100 pending) instead of being cancelled. + group: ${{ github.workflow }}-${{ github.ref }} + queue: max + +permissions: + contents: read + +jobs: + check-changes: + name: 'Check for changes' + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + outputs: + only_docs: ${{ steps.check.outputs.ONLY_DOCS }} + docs_changed: ${{ steps.check.outputs.DOCS_CHANGED }} + steps: + - name: 'Clone repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 50 + - name: 'Get changed files' + id: changed-files-specific + uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 + with: + files_yaml: | + doc: + - '**/*.md' + - '**/docs/**' + - '**/*.jpg' + - '**/*.png' + - name: 'Determine change type' + id: check + run: | + if [[ "${{ steps.changed-files-specific.outputs.doc_only_changed }}" == "true" || "${{ steps.changed-files-specific.outputs.doc_only_modified }}" == "true" ]]; then + echo "ONLY_DOCS=true" >> $GITHUB_OUTPUT + else + echo "ONLY_DOCS=false" >> $GITHUB_OUTPUT + fi + if [[ "${{ steps.changed-files-specific.outputs.doc_any_changed }}" == "true" ]]; then + echo "DOCS_CHANGED=true" >> $GITHUB_OUTPUT + else + echo "DOCS_CHANGED=false" >> $GITHUB_OUTPUT + fi + + markdown-link-check: + name: 'Markdown link check' + needs: check-changes + if: ${{ needs.check-changes.outputs.docs_changed == 'true' }} + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + # main.yml uses tcort/github-action-markdown-link-check here. That is a Dockerfile action, so + # the runner has to `docker build` it, and on CodeBuild the build goes to the project's remote + # Docker server, which this project cannot reach. These steps run what that action + # (@e047c5b37f24ab722bbef1a27b6fab7f96bc4068, inputs use-quiet-mode: yes, folder-path: '.', + # max-depth: 0, config-file: mlc_config.json) runs: markdown-link-check over every *.md file + # in the repository, failing if the output reports an error. + - name: 'Set up Node.js' + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 24 + - name: 'Check Markdown links' + run: | + set -o pipefail + npm i -g markdown-link-check@3.15.0 + CONFIG_FILE='mlc_config.json' + MLC_ARGS=() + if [ -f "$CONFIG_FILE" ]; then + MLC_ARGS+=('--config' "$CONFIG_FILE") + else + echo "Cannot find $CONFIG_FILE, checking without a configuration file." + fi + MLC_ARGS+=('-q') + MLC_LOG="$RUNNER_TEMP/markdown-link-check.txt" + find . -name '*.md' -not -path './node_modules/*' -exec markdown-link-check '{}' "${MLC_ARGS[@]}" ';' 2>&1 | tee "$MLC_LOG" + if grep -q 'ERROR:' "$MLC_LOG"; then + exit 1 + fi + echo 'All links are good!' + + lint: + name: 'Static analysis' + needs: check-changes + if: ${{ needs.check-changes.outputs.only_docs == 'false' }} + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Set up Ruby + uses: ruby/setup-ruby@651ff0d62c5c79bed6d53ec518387aae20e72c1a # v1 + with: + ruby-version: '3.3' + bundler-cache: true + - name: Run RuboCop + run: bundle exec rubocop + + pg-sql-qualification: + name: PostgreSQL SQL qualification check + needs: check-changes + if: ${{ needs.check-changes.outputs.only_docs == 'false' }} + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Set up Ruby + uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1 + with: + ruby-version: '3.3' + - name: Check internal PostgreSQL SQL is schema-qualified + run: ruby .github/scripts/check_pg_sql_qualified.rb + + test: + name: 'Unit tests' + needs: check-changes + if: ${{ needs.check-changes.outputs.only_docs == 'false' }} + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + strategy: + fail-fast: false + matrix: + include: + - ruby-version: '3.3' + gemfile: gemfiles/activerecord_7.2.gemfile + - ruby-version: '3.4' + gemfile: gemfiles/activerecord_8.0.gemfile + - ruby-version: '4.0' + gemfile: gemfiles/activerecord_8.1.gemfile + env: + BUNDLE_GEMFILE: ${{ matrix.gemfile }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Set up Ruby ${{ matrix.ruby-version }} + uses: ruby/setup-ruby@651ff0d62c5c79bed6d53ec518387aae20e72c1a # v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true + - name: Run unit tests + run: bundle exec rspec spec/unit + - name: Upload coverage reports + if: matrix.ruby-version == '3.3' + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7 + with: + files: ./coverage/coverage.xml + flags: unittests + name: ruby-wrapper-unit-coverage + token: ${{ secrets.CODECOV_TOKEN }} + continue-on-error: true + + secret-scan: + name: Gitleaks secret scan + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - name: Checkout (full history) + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + - name: Install Gitleaks + env: + GITLEAKS_VERSION: 8.30.1 + GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb + run: | + set -euo pipefail + curl -sSfL \ + "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" \ + -o gitleaks.tar.gz + echo "${GITLEAKS_SHA256} gitleaks.tar.gz" | sha256sum -c - + tar -xzf gitleaks.tar.gz gitleaks + # CodeBuild runners run as root and may not ship sudo. + if [ "$(id -u)" -eq 0 ]; then + install gitleaks /usr/local/bin/gitleaks + else + sudo install gitleaks /usr/local/bin/gitleaks + fi + rm -f gitleaks.tar.gz gitleaks + gitleaks version + - name: Run Gitleaks (full git history) + run: gitleaks detect --source . --redact --verbose + + ci-gate: + name: 'CI Gate (CodeBuild)' + needs: [check-changes, markdown-link-check, lint, pg-sql-qualification, test, secret-scan] + if: ${{ always() }} + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - name: 'Verify required jobs did not fail' + run: | + # Fail only if a needed job actually failed or was cancelled. + # Skipped jobs (e.g. tests on doc-only PRs) are treated as passing, + # so this gate is the single required status check for branch protection. + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" \ + || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "A required job failed or was cancelled." + exit 1 + fi + echo "All required jobs passed or were legitimately skipped." diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1978201..9b50226 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,22 +2,63 @@ name: Check and Unit Tests on: workflow_dispatch: - push: - branches: - - main - pull_request: - branches: - - '*' + # The push and pull_request triggers moved to the CodeBuild variant main-codebuild.yml. + # push: + # branches: + # - main + # pull_request: + # branches: + # - '*' concurrency: + # Runs in this group execute one at a time; with `queue: max`, later runs wait in FIFO order + # (up to 100 pending) instead of being cancelled. group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + queue: max permissions: contents: read jobs: + check-changes: + name: 'Check for changes' + runs-on: ubuntu-latest + outputs: + only_docs: ${{ steps.check.outputs.ONLY_DOCS }} + docs_changed: ${{ steps.check.outputs.DOCS_CHANGED }} + steps: + - name: 'Clone repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 50 + - name: 'Get changed files' + id: changed-files-specific + uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 + with: + files_yaml: | + doc: + - '**/*.md' + - '**/docs/**' + - '**/*.jpg' + - '**/*.png' + - name: 'Determine change type' + id: check + run: | + if [[ "${{ steps.changed-files-specific.outputs.doc_only_changed }}" == "true" || "${{ steps.changed-files-specific.outputs.doc_only_modified }}" == "true" ]]; then + echo "ONLY_DOCS=true" >> $GITHUB_OUTPUT + else + echo "ONLY_DOCS=false" >> $GITHUB_OUTPUT + fi + if [[ "${{ steps.changed-files-specific.outputs.doc_any_changed }}" == "true" ]]; then + echo "DOCS_CHANGED=true" >> $GITHUB_OUTPUT + else + echo "DOCS_CHANGED=false" >> $GITHUB_OUTPUT + fi + markdown-link-check: + name: 'Markdown link check' + needs: check-changes + if: ${{ needs.check-changes.outputs.docs_changed == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -29,6 +70,9 @@ jobs: max-depth: 0 lint: + name: 'Static analysis' + needs: check-changes + if: ${{ needs.check-changes.outputs.only_docs == 'false' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -42,6 +86,8 @@ jobs: pg-sql-qualification: name: PostgreSQL SQL qualification check + needs: check-changes + if: ${{ needs.check-changes.outputs.only_docs == 'false' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -53,6 +99,9 @@ jobs: run: ruby .github/scripts/check_pg_sql_qualified.rb test: + name: 'Unit tests' + needs: check-changes + if: ${{ needs.check-changes.outputs.only_docs == 'false' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -109,3 +158,21 @@ jobs: gitleaks version - name: Run Gitleaks (full git history) run: gitleaks detect --source . --redact --verbose + + ci-gate: + name: 'CI Gate' + needs: [check-changes, markdown-link-check, lint, pg-sql-qualification, test, secret-scan] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: 'Verify required jobs did not fail' + run: | + # Fail only if a needed job actually failed or was cancelled. + # Skipped jobs (e.g. tests on doc-only PRs) are treated as passing, + # so this gate is the single required status check for branch protection. + if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" \ + || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then + echo "A required job failed or was cancelled." + exit 1 + fi + echo "All required jobs passed or were legitimately skipped." diff --git a/.github/workflows/mysql_performance-codebuild.yml b/.github/workflows/mysql_performance-codebuild.yml new file mode 100644 index 0000000..4457f2f --- /dev/null +++ b/.github/workflows/mysql_performance-codebuild.yml @@ -0,0 +1,75 @@ +# CodeBuild variant of mysql_performance.yml. +# +# Runs the same job on the self-hosted CodeBuild runners instead of GitHub-hosted ones. +# Trigger: manual dispatch only, the same as mysql_performance.yml. +# Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild RDS +# workflows, so only one of them claims DB clusters and security-group rules at a time. + +name: Run Aurora MySQL Performance Tests (CodeBuild) + +on: + workflow_dispatch: + +permissions: + id-token: write + contents: read + +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + +concurrency: + # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests-codebuild + queue: max + +jobs: + aurora-mysql-performance-tests: + name: 'Run Aurora MySQL performance tests' + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + + steps: + - name: 'Clone repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: 'Validate Gradle wrapper' + uses: gradle/actions/wrapper-validation@3f5f9adaf7d9fecd50b5935e54106014257a94e6 # v6.4.0 + + - name: 'Set up Ruby 3.3' + uses: ruby/setup-ruby@651ff0d62c5c79bed6d53ec518387aae20e72c1a # v1 + with: + ruby-version: '3.3' + bundler-cache: true + + - name: 'Set up JDK 8' + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6 + with: + distribution: 'corretto' + java-version: 8 + + - name: 'Configure AWS Credentials' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: ruby_perf_test_mysql + role-duration-seconds: 21600 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Run MySQL performance tests' + run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-3.3-mysql-performance --info + env: + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Archive performance results' + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: 'mysql-performance-results' + path: | + ./spec/integration/host/build/reports/tests/test/ + ./spec/integration/container/reports/ + if-no-files-found: ignore + retention-days: 5 diff --git a/.github/workflows/mysql_performance.yml b/.github/workflows/mysql_performance.yml index 3f5f5bc..3ca3d22 100644 --- a/.github/workflows/mysql_performance.yml +++ b/.github/workflows/mysql_performance.yml @@ -1,3 +1,9 @@ +# GitHub-hosted backup of the CodeBuild workflow mysql_performance-codebuild.yml. +# +# Runs the same job on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the +# CodeBuild variant is the one wired to any automated trigger. Shares the rds-integration-tests +# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). + name: Run Aurora MySQL Performance Tests on: @@ -13,13 +19,16 @@ env: TEST_CONTAINER_REGISTRY: ecr concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests + queue: max jobs: aurora-mysql-performance-tests: name: 'Run Aurora MySQL performance tests' - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest steps: - name: 'Clone repository' diff --git a/.github/workflows/pg_performance-codebuild.yml b/.github/workflows/pg_performance-codebuild.yml new file mode 100644 index 0000000..b338672 --- /dev/null +++ b/.github/workflows/pg_performance-codebuild.yml @@ -0,0 +1,75 @@ +# CodeBuild variant of pg_performance.yml. +# +# Runs the same job on the self-hosted CodeBuild runners instead of GitHub-hosted ones. +# Trigger: manual dispatch only, the same as pg_performance.yml. +# Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild RDS +# workflows, so only one of them claims DB clusters and security-group rules at a time. + +name: Run Aurora PostgreSQL Performance Tests (CodeBuild) + +on: + workflow_dispatch: + +permissions: + id-token: write + contents: read + +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + +concurrency: + # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests-codebuild + queue: max + +jobs: + aurora-pg-performance-tests: + name: 'Run Aurora PostgreSQL performance tests' + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + + steps: + - name: 'Clone repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: 'Validate Gradle wrapper' + uses: gradle/actions/wrapper-validation@3f5f9adaf7d9fecd50b5935e54106014257a94e6 # v6.4.0 + + - name: 'Set up Ruby 3.3' + uses: ruby/setup-ruby@651ff0d62c5c79bed6d53ec518387aae20e72c1a # v1 + with: + ruby-version: '3.3' + bundler-cache: true + + - name: 'Set up JDK 8' + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6 + with: + distribution: 'corretto' + java-version: 8 + + - name: 'Configure AWS Credentials' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: ruby_perf_test_pg + role-duration-seconds: 21600 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Run PostgreSQL performance tests' + run: ./gradlew --no-parallel --no-daemon :integration-testing:test-ruby-3.3-pg-performance --info + env: + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: 'Archive performance results' + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: 'pg-performance-results' + path: | + ./spec/integration/host/build/reports/tests/test/ + ./spec/integration/container/reports/ + if-no-files-found: ignore + retention-days: 5 diff --git a/.github/workflows/pg_performance.yml b/.github/workflows/pg_performance.yml index 6c7f9e4..24fe79d 100644 --- a/.github/workflows/pg_performance.yml +++ b/.github/workflows/pg_performance.yml @@ -1,3 +1,9 @@ +# GitHub-hosted backup of the CodeBuild workflow pg_performance-codebuild.yml. +# +# Runs the same job on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the +# CodeBuild variant is the one wired to any automated trigger. Shares the rds-integration-tests +# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). + name: Run Aurora PostgreSQL Performance Tests on: @@ -13,13 +19,16 @@ env: TEST_CONTAINER_REGISTRY: ecr concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests + queue: max jobs: aurora-pg-performance-tests: name: 'Run Aurora PostgreSQL performance tests' - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest steps: - name: 'Clone repository' diff --git a/.github/workflows/remove-old-artifacts-codebuild.yml b/.github/workflows/remove-old-artifacts-codebuild.yml new file mode 100644 index 0000000..0adbabe --- /dev/null +++ b/.github/workflows/remove-old-artifacts-codebuild.yml @@ -0,0 +1,32 @@ +# CodeBuild variant of remove-old-artifacts.yml. +# +# Runs the same job on the self-hosted CodeBuild runners instead of GitHub-hosted ones. +# Triggers: the schedule of remove-old-artifacts.yml plus manual dispatch. The schedule is commented +# out in remove-old-artifacts.yml, so only this copy runs on schedule. + +name: Remove Old Artifacts (CodeBuild) + +on: + workflow_dispatch: + schedule: + # Every day at 1am + - cron: '0 1 * * *' + +permissions: + actions: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + remove-old-artifacts: + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + timeout-minutes: 10 + + steps: + - name: Remove Old Artifacts + uses: c-hive/gha-remove-artifacts@62c2fbea931baa7dd4a6b73ea5a799984a818f61 # v1 + with: + age: '1 week' + skip-tags: true diff --git a/.github/workflows/remove-old-artifacts.yml b/.github/workflows/remove-old-artifacts.yml index ca3138d..b3e7a92 100644 --- a/.github/workflows/remove-old-artifacts.yml +++ b/.github/workflows/remove-old-artifacts.yml @@ -1,9 +1,16 @@ +# GitHub-hosted backup of the CodeBuild workflow remove-old-artifacts-codebuild.yml. +# +# Runs the same job on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the +# CodeBuild variant owns the schedule (commented out below). + name: Remove Old Artifacts on: - schedule: - # Every day at 1am - - cron: '0 1 * * *' + workflow_dispatch: + # The schedule moved to the CodeBuild variant remove-old-artifacts-codebuild.yml. + # schedule: + # # Every day at 1am + # - cron: '0 1 * * *' permissions: actions: write diff --git a/.github/workflows/run-bg-integration-tests-codebuild.yml b/.github/workflows/run-bg-integration-tests-codebuild.yml new file mode 100644 index 0000000..cc93e93 --- /dev/null +++ b/.github/workflows/run-bg-integration-tests-codebuild.yml @@ -0,0 +1,80 @@ +# CodeBuild variant of run-bg-integration-tests.yml. +# +# Runs the same jobs on the self-hosted CodeBuild runners instead of GitHub-hosted ones. +# Triggers: the same as run-bg-integration-tests.yml (manual dispatch and a weekly schedule). The +# schedule is commented out in run-bg-integration-tests.yml, so only this copy runs on schedule. +# Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild RDS +# workflows, so only one of them claims DB clusters and security-group rules at a time. + +name: Run Blue/Green Integration Tests (CodeBuild) + +on: + workflow_dispatch: + schedule: + # Every Sunday at 9:00 AM UTC + - cron: '0 9 * * 0' + +permissions: + id-token: write + contents: read + +env: + # Pull test container images from ECR Public rather than Docker Hub. These runners share an + # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + TEST_CONTAINER_REGISTRY: ecr + +concurrency: + # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests-codebuild + queue: max + +jobs: + bg-integration-tests: + name: BG Integration Tests (${{ matrix.testTask }}) + runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + testTask: + - test-bgd-mysql-aurora + - test-bgd-pg-aurora + - test-bgd-mysql-instance + - test-bgd-pg-instance + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: 'Validate Gradle wrapper' + uses: gradle/actions/wrapper-validation@3f5f9adaf7d9fecd50b5935e54106014257a94e6 # v6.4.0 + + - uses: ruby/setup-ruby@651ff0d62c5c79bed6d53ec518387aae20e72c1a # v1 + with: + ruby-version: '3.3' + bundler-cache: true + + - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6 + with: + distribution: 'corretto' + java-version: '8' + + - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: ruby_bg_integration_tests + role-duration-seconds: 21600 + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: Run BG Integration Tests + run: ./gradlew --no-parallel --no-daemon :integration-testing:${{ matrix.testTask }} --info + env: + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: Archive results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: bg-report-${{ matrix.testTask }} + path: ./spec/integration/host/build/reports/tests/test/ + retention-days: 5 diff --git a/.github/workflows/run-bg-integration-tests.yml b/.github/workflows/run-bg-integration-tests.yml index 3ad4442..1b605f3 100644 --- a/.github/workflows/run-bg-integration-tests.yml +++ b/.github/workflows/run-bg-integration-tests.yml @@ -1,10 +1,17 @@ +# GitHub-hosted backup of the CodeBuild workflow run-bg-integration-tests-codebuild.yml. +# +# Runs the same jobs on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the +# CodeBuild variant owns the schedule (commented out below). Shares the rds-integration-tests +# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). + name: Run Blue/Green Integration Tests on: workflow_dispatch: - schedule: - # Every Sunday at 9:00 AM UTC - - cron: '0 9 * * 0' + # The schedule moved to the CodeBuild variant run-bg-integration-tests-codebuild.yml. + # schedule: + # # Every Sunday at 9:00 AM UTC + # - cron: '0 9 * * 0' permissions: id-token: write @@ -16,13 +23,16 @@ env: TEST_CONTAINER_REGISTRY: ecr concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and + # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs + # wait in FIFO order (up to 100 pending) instead of being cancelled. + group: rds-integration-tests + queue: max jobs: bg-integration-tests: name: BG Integration Tests (${{ matrix.testTask }}) - runs-on: codebuild-rubyWrapper-${{ github.run_id }}-${{ github.run_attempt }} + runs-on: ubuntu-latest timeout-minutes: 120 strategy: fail-fast: false diff --git a/spec/integration/host/src/test/java/integration/util/AuroraTestUtility.java b/spec/integration/host/src/test/java/integration/util/AuroraTestUtility.java index 2219e4b..a0d4fe0 100644 --- a/spec/integration/host/src/test/java/integration/util/AuroraTestUtility.java +++ b/spec/integration/host/src/test/java/integration/util/AuroraTestUtility.java @@ -67,6 +67,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -80,6 +81,7 @@ import software.amazon.awssdk.core.waiters.WaiterResponse; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.ec2.Ec2Client; +import software.amazon.awssdk.services.ec2.model.AuthorizeSecurityGroupIngressRequest; import software.amazon.awssdk.services.ec2.model.DescribeSecurityGroupRulesRequest; import software.amazon.awssdk.services.ec2.model.DescribeSecurityGroupRulesResponse; import software.amazon.awssdk.services.ec2.model.DescribeSecurityGroupsRequest; @@ -155,6 +157,9 @@ public class AuroraTestUtility { private static final Logger LOGGER = Logger.getLogger(AuroraTestUtility.class.getName()); private static final String DUPLICATE_IP_ERROR_CODE = "InvalidPermission.Duplicate"; private static final String DEFAULT_SECURITY_GROUP = "default"; + // The only ports the test harness reaches over the network: TCP 3306 (MySQL / MariaDB) and + // TCP 5432 (PostgreSQL). Security-group rules are scoped to these instead of all ports. + private static final int[] TEST_DATABASE_PORTS = {3306, 5432}; private static final String DEFAULT_STORAGE_TYPE = "gp3"; private static final int DEFAULT_IOPS = 64000; private static final int DEFAULT_ALLOCATED_STORAGE = 400; @@ -698,71 +703,43 @@ public void ec2AuthorizeIP(String ipAddress) { if (StringUtils.isNullOrEmpty(ipAddress)) { return; } - - if (ipExists(ipAddress)) { - return; - } - - try { - IpRange ipRange = IpRange.builder() - .cidrIp(ipAddress + "/32") - .description("Test run at " + Instant.now()) - .build(); - IpPermission ipPermission = IpPermission.builder() - .ipRanges(ipRange) - .ipProtocol("-1") // All protocols - .fromPort(0) // For all ports - .toPort(65535) - .build(); - ec2Client.authorizeSecurityGroupIngress( - (builder) -> builder.groupName(DEFAULT_SECURITY_GROUP).ipPermissions(ipPermission)); - } catch (Ec2Exception exception) { - if (!DUPLICATE_IP_ERROR_CODE.equalsIgnoreCase(exception.awsErrorDetails().errorCode())) { - throw exception; - } - } + authorizeTestDatabasePorts(ipAddress, builder -> builder.groupName(DEFAULT_SECURITY_GROUP)); } public void ec2AuthorizeIPByGroupId(String securityGroupId, String ipAddress) { if (StringUtils.isNullOrEmpty(ipAddress) || StringUtils.isNullOrEmpty(securityGroupId)) { return; } + authorizeTestDatabasePorts(ipAddress, builder -> builder.groupId(securityGroupId)); + } - try { - IpRange ipRange = IpRange.builder() - .cidrIp(ipAddress + "/32") - .description("Test run at " + Instant.now()) - .build(); - IpPermission ipPermission = IpPermission.builder() - .ipRanges(ipRange) - .ipProtocol("-1") - .fromPort(0) - .toPort(65535) - .build(); - ec2Client.authorizeSecurityGroupIngress( - (builder) -> builder.groupId(securityGroupId).ipPermissions(ipPermission)); - } catch (Ec2Exception exception) { - if (!DUPLICATE_IP_ERROR_CODE.equalsIgnoreCase(exception.awsErrorDetails().errorCode())) { - throw exception; - } - } - } - - private boolean ipExists(String ipAddress) { - final String cidr = ipAddress + "/32"; - final DescribeSecurityGroupsResponse response = - ec2Client.describeSecurityGroups( - (builder) -> builder.groupNames(DEFAULT_SECURITY_GROUP)); - - if (response == null || response.securityGroups().isEmpty()) { - return false; + // Opens only the two ports the test harness reaches over the network: TCP 3306 (MySQL / MariaDB) + // and TCP 5432 (PostgreSQL). + private void authorizeTestDatabasePorts( + String ipAddress, Consumer groupSelector) { + final String description = "Test run at " + Instant.now(); + for (int port : TEST_DATABASE_PORTS) { + try { + IpRange ipRange = IpRange.builder() + .cidrIp(ipAddress + "/32") + .description(description) + .build(); + IpPermission ipPermission = IpPermission.builder() + .ipRanges(ipRange) + .ipProtocol("tcp") + .fromPort(port) + .toPort(port) + .build(); + ec2Client.authorizeSecurityGroupIngress((builder) -> { + groupSelector.accept(builder); + builder.ipPermissions(ipPermission); + }); + } catch (Ec2Exception exception) { + if (!DUPLICATE_IP_ERROR_CODE.equalsIgnoreCase(exception.awsErrorDetails().errorCode())) { + throw exception; + } + } } - - return response.securityGroups().stream() - .flatMap(sg -> sg.ipPermissions().stream()) - .filter(perm -> "-1".equals(perm.ipProtocol())) - .flatMap(perm -> perm.ipRanges().stream()) - .anyMatch(range -> cidr.equals(range.cidrIp())); } /** @@ -774,17 +751,20 @@ public void ec2DeauthorizesIP(String ipAddress) { if (StringUtils.isNullOrEmpty(ipAddress)) { return; } - try { - ec2Client.revokeSecurityGroupIngress( - (builder) -> - builder - .groupName(DEFAULT_SECURITY_GROUP) - .cidrIp(ipAddress + "/32") - .ipProtocol("-1") // All protocols - .fromPort(0) // For all ports - .toPort(65535)); - } catch (Ec2Exception exception) { - // Ignore + // Mirror ec2AuthorizeIP: revoke only the TCP 3306 and 5432 rules it created. + for (int port : TEST_DATABASE_PORTS) { + try { + ec2Client.revokeSecurityGroupIngress( + (builder) -> + builder + .groupName(DEFAULT_SECURITY_GROUP) + .cidrIp(ipAddress + "/32") + .ipProtocol("tcp") + .fromPort(port) + .toPort(port)); + } catch (Ec2Exception exception) { + // Ignore + } } } From cb865e94880f331b5262064848a0ec882607bf7f Mon Sep 17 00:00:00 2001 From: Sophia Chu <112967780+sophia-bq@users.noreply.github.com> Date: Wed, 21 Oct 2026 12:55:26 -0700 Subject: [PATCH 8/8] chore: serialize workflows --- ...encryption_integration_tests-codebuild.yml | 20 +++++++++---------- .../encryption_integration_tests.yml | 19 +++++++++--------- .../gdb_integration_tests-codebuild.yml | 19 +++++++++--------- .github/workflows/gdb_integration_tests.yml | 19 +++++++++--------- .../workflows/integration_tests-codebuild.yml | 19 +++++++++--------- .github/workflows/integration_tests.yml | 19 +++++++++--------- .github/workflows/main-codebuild.yml | 6 +++--- .github/workflows/main.yml | 6 +++--- .../workflows/mysql_performance-codebuild.yml | 19 +++++++++--------- .github/workflows/mysql_performance.yml | 19 +++++++++--------- .../workflows/pg_performance-codebuild.yml | 19 +++++++++--------- .github/workflows/pg_performance.yml | 19 +++++++++--------- .../run-bg-integration-tests-codebuild.yml | 19 +++++++++--------- .../workflows/run-bg-integration-tests.yml | 19 +++++++++--------- 14 files changed, 126 insertions(+), 115 deletions(-) diff --git a/.github/workflows/encryption_integration_tests-codebuild.yml b/.github/workflows/encryption_integration_tests-codebuild.yml index ea78479..e67a076 100644 --- a/.github/workflows/encryption_integration_tests-codebuild.yml +++ b/.github/workflows/encryption_integration_tests-codebuild.yml @@ -3,9 +3,8 @@ # Runs the same jobs on the self-hosted CodeBuild runners instead of GitHub-hosted ones. # Triggers: the same as encryption_integration_tests.yml (manual dispatch and push to main). The # push trigger is commented out in encryption_integration_tests.yml, so each push to main runs only -# this copy. Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild -# RDS workflows, so only one of them claims DB clusters, security-group rules, and the KMS key at a -# time. +# this copy. Uses a per-ref concurrency group with cancel-in-progress, so a newer run on the same +# ref supersedes the one in progress. name: Encryption Integration Tests (CodeBuild) @@ -20,17 +19,18 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. CodeBuild builds egress + # through the account's shared NAT IP and carry no Docker Hub credentials, so anonymous Docker Hub + # pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr # These workflows create real RDS clusters and a real KMS key. concurrency: - # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests-codebuild - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: kms-key: diff --git a/.github/workflows/encryption_integration_tests.yml b/.github/workflows/encryption_integration_tests.yml index c5671e1..bb55a18 100644 --- a/.github/workflows/encryption_integration_tests.yml +++ b/.github/workflows/encryption_integration_tests.yml @@ -1,8 +1,8 @@ # GitHub-hosted backup of the CodeBuild workflow encryption_integration_tests-codebuild.yml. # # Runs the same jobs on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the -# CodeBuild variant owns the push trigger (commented out below). Shares the rds-integration-tests -# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). +# CodeBuild variant owns the push trigger (commented out below). Uses a per-ref +# concurrency group with cancel-in-progress, so a newer run on the same ref supersedes the one in progress. name: Encryption Integration Tests @@ -18,17 +18,18 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. GitHub-hosted runners pull + # from a pool of egress IPs shared across all GitHub customers, with no Docker Hub credentials, so + # anonymous Docker Hub pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr # These workflows create real RDS clusters and a real KMS key. concurrency: - # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: kms-key: diff --git a/.github/workflows/gdb_integration_tests-codebuild.yml b/.github/workflows/gdb_integration_tests-codebuild.yml index 3a5c3fa..69f34dd 100644 --- a/.github/workflows/gdb_integration_tests-codebuild.yml +++ b/.github/workflows/gdb_integration_tests-codebuild.yml @@ -3,8 +3,8 @@ # Runs the same jobs on the self-hosted CodeBuild runners instead of GitHub-hosted ones. # Triggers: the same as gdb_integration_tests.yml (manual dispatch and a weekly schedule). The # schedule is commented out in gdb_integration_tests.yml, so only this copy runs on schedule. -# Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild RDS -# workflows, so only one of them claims DB clusters and security-group rules at a time. +# Uses a per-ref concurrency group with cancel-in-progress, so a newer run on the same ref +# supersedes the one in progress. name: Global Database Integration Tests (CodeBuild) @@ -19,16 +19,17 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. CodeBuild builds egress + # through the account's shared NAT IP and carry no Docker Hub credentials, so anonymous Docker Hub + # pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr concurrency: - # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests-codebuild - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: gdb-integration-tests: diff --git a/.github/workflows/gdb_integration_tests.yml b/.github/workflows/gdb_integration_tests.yml index 470539a..684521a 100644 --- a/.github/workflows/gdb_integration_tests.yml +++ b/.github/workflows/gdb_integration_tests.yml @@ -1,8 +1,8 @@ # GitHub-hosted backup of the CodeBuild workflow gdb_integration_tests-codebuild.yml. # # Runs the same jobs on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the -# CodeBuild variant owns the schedule (commented out below). Shares the rds-integration-tests -# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). +# CodeBuild variant owns the schedule (commented out below). Uses a per-ref +# concurrency group with cancel-in-progress, so a newer run on the same ref supersedes the one in progress. name: Global Database Integration Tests @@ -18,16 +18,17 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. GitHub-hosted runners pull + # from a pool of egress IPs shared across all GitHub customers, with no Docker Hub credentials, so + # anonymous Docker Hub pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr concurrency: - # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: gdb-integration-tests: diff --git a/.github/workflows/integration_tests-codebuild.yml b/.github/workflows/integration_tests-codebuild.yml index 0d55e67..8cf72e9 100644 --- a/.github/workflows/integration_tests-codebuild.yml +++ b/.github/workflows/integration_tests-codebuild.yml @@ -3,8 +3,8 @@ # Runs the same jobs on the self-hosted CodeBuild runners instead of GitHub-hosted ones. # Triggers: the same as integration_tests.yml (manual dispatch and push to main). The push trigger # is commented out in integration_tests.yml, so each push to main runs only this copy. -# Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild RDS -# workflows, so only one of them claims DB clusters and security-group rules at a time. +# Uses a per-ref concurrency group with cancel-in-progress, so a newer run on the same ref +# supersedes the one in progress. name: Integration Tests (CodeBuild) @@ -19,16 +19,17 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. CodeBuild builds egress + # through the account's shared NAT IP and carry no Docker Hub credentials, so anonymous Docker Hub + # pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr concurrency: - # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests-codebuild - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: default-integration-tests: diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d751be2..feac1cb 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1,8 +1,8 @@ # GitHub-hosted backup of the CodeBuild workflow integration_tests-codebuild.yml. # # Runs the same jobs on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the -# CodeBuild variant owns the push trigger (commented out below). Shares the rds-integration-tests -# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). +# CodeBuild variant owns the push trigger (commented out below). Uses a per-ref +# concurrency group with cancel-in-progress, so a newer run on the same ref supersedes the one in progress. name: Integration Tests @@ -18,16 +18,17 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. GitHub-hosted runners pull + # from a pool of egress IPs shared across all GitHub customers, with no Docker Hub credentials, so + # anonymous Docker Hub pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr concurrency: - # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: default-integration-tests: diff --git a/.github/workflows/main-codebuild.yml b/.github/workflows/main-codebuild.yml index a7c49a9..c706221 100644 --- a/.github/workflows/main-codebuild.yml +++ b/.github/workflows/main-codebuild.yml @@ -20,10 +20,10 @@ on: - '*' concurrency: - # Runs in this group execute one at a time; with `queue: max`, later runs wait in FIFO order - # (up to 100 pending) instead of being cancelled. + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push or PR update supersedes a stale commit instead of queueing behind it. group: ${{ github.workflow }}-${{ github.ref }} - queue: max + cancel-in-progress: true permissions: contents: read diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b50226..ac9901c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,10 +11,10 @@ on: # - '*' concurrency: - # Runs in this group execute one at a time; with `queue: max`, later runs wait in FIFO order - # (up to 100 pending) instead of being cancelled. + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push or PR update supersedes a stale commit instead of queueing behind it. group: ${{ github.workflow }}-${{ github.ref }} - queue: max + cancel-in-progress: true permissions: contents: read diff --git a/.github/workflows/mysql_performance-codebuild.yml b/.github/workflows/mysql_performance-codebuild.yml index 4457f2f..ead3a4b 100644 --- a/.github/workflows/mysql_performance-codebuild.yml +++ b/.github/workflows/mysql_performance-codebuild.yml @@ -2,8 +2,8 @@ # # Runs the same job on the self-hosted CodeBuild runners instead of GitHub-hosted ones. # Trigger: manual dispatch only, the same as mysql_performance.yml. -# Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild RDS -# workflows, so only one of them claims DB clusters and security-group rules at a time. +# Uses a per-ref concurrency group with cancel-in-progress, so a newer run on the same ref +# supersedes the one in progress. name: Run Aurora MySQL Performance Tests (CodeBuild) @@ -15,16 +15,17 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. CodeBuild builds egress + # through the account's shared NAT IP and carry no Docker Hub credentials, so anonymous Docker Hub + # pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr concurrency: - # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests-codebuild - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: aurora-mysql-performance-tests: diff --git a/.github/workflows/mysql_performance.yml b/.github/workflows/mysql_performance.yml index 3ca3d22..749a057 100644 --- a/.github/workflows/mysql_performance.yml +++ b/.github/workflows/mysql_performance.yml @@ -1,8 +1,8 @@ # GitHub-hosted backup of the CodeBuild workflow mysql_performance-codebuild.yml. # # Runs the same job on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the -# CodeBuild variant is the one wired to any automated trigger. Shares the rds-integration-tests -# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). +# CodeBuild variant is the one wired to any automated trigger. Uses a per-ref +# concurrency group with cancel-in-progress, so a newer run on the same ref supersedes the one in progress. name: Run Aurora MySQL Performance Tests @@ -14,16 +14,17 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. GitHub-hosted runners pull + # from a pool of egress IPs shared across all GitHub customers, with no Docker Hub credentials, so + # anonymous Docker Hub pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr concurrency: - # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: aurora-mysql-performance-tests: diff --git a/.github/workflows/pg_performance-codebuild.yml b/.github/workflows/pg_performance-codebuild.yml index b338672..568e5e1 100644 --- a/.github/workflows/pg_performance-codebuild.yml +++ b/.github/workflows/pg_performance-codebuild.yml @@ -2,8 +2,8 @@ # # Runs the same job on the self-hosted CodeBuild runners instead of GitHub-hosted ones. # Trigger: manual dispatch only, the same as pg_performance.yml. -# Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild RDS -# workflows, so only one of them claims DB clusters and security-group rules at a time. +# Uses a per-ref concurrency group with cancel-in-progress, so a newer run on the same ref +# supersedes the one in progress. name: Run Aurora PostgreSQL Performance Tests (CodeBuild) @@ -15,16 +15,17 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. CodeBuild builds egress + # through the account's shared NAT IP and carry no Docker Hub credentials, so anonymous Docker Hub + # pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr concurrency: - # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests-codebuild - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: aurora-pg-performance-tests: diff --git a/.github/workflows/pg_performance.yml b/.github/workflows/pg_performance.yml index 24fe79d..b349c96 100644 --- a/.github/workflows/pg_performance.yml +++ b/.github/workflows/pg_performance.yml @@ -1,8 +1,8 @@ # GitHub-hosted backup of the CodeBuild workflow pg_performance-codebuild.yml. # # Runs the same job on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the -# CodeBuild variant is the one wired to any automated trigger. Shares the rds-integration-tests -# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). +# CodeBuild variant is the one wired to any automated trigger. Uses a per-ref +# concurrency group with cancel-in-progress, so a newer run on the same ref supersedes the one in progress. name: Run Aurora PostgreSQL Performance Tests @@ -14,16 +14,17 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. GitHub-hosted runners pull + # from a pool of egress IPs shared across all GitHub customers, with no Docker Hub credentials, so + # anonymous Docker Hub pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr concurrency: - # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: aurora-pg-performance-tests: diff --git a/.github/workflows/run-bg-integration-tests-codebuild.yml b/.github/workflows/run-bg-integration-tests-codebuild.yml index cc93e93..e19c60b 100644 --- a/.github/workflows/run-bg-integration-tests-codebuild.yml +++ b/.github/workflows/run-bg-integration-tests-codebuild.yml @@ -3,8 +3,8 @@ # Runs the same jobs on the self-hosted CodeBuild runners instead of GitHub-hosted ones. # Triggers: the same as run-bg-integration-tests.yml (manual dispatch and a weekly schedule). The # schedule is commented out in run-bg-integration-tests.yml, so only this copy runs on schedule. -# Shares the rds-integration-tests-codebuild concurrency group with the other CodeBuild RDS -# workflows, so only one of them claims DB clusters and security-group rules at a time. +# Uses a per-ref concurrency group with cancel-in-progress, so a newer run on the same ref +# supersedes the one in progress. name: Run Blue/Green Integration Tests (CodeBuild) @@ -19,16 +19,17 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. CodeBuild builds egress + # through the account's shared NAT IP and carry no Docker Hub credentials, so anonymous Docker Hub + # pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr concurrency: - # Shared by every CodeBuild RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests-codebuild - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: bg-integration-tests: diff --git a/.github/workflows/run-bg-integration-tests.yml b/.github/workflows/run-bg-integration-tests.yml index 1b605f3..2135d06 100644 --- a/.github/workflows/run-bg-integration-tests.yml +++ b/.github/workflows/run-bg-integration-tests.yml @@ -1,8 +1,8 @@ # GitHub-hosted backup of the CodeBuild workflow run-bg-integration-tests-codebuild.yml. # # Runs the same jobs on GitHub-hosted ubuntu runners. Kept as a manual-dispatch-only fallback; the -# CodeBuild variant owns the schedule (commented out below). Shares the rds-integration-tests -# concurrency group with the other GitHub-hosted RDS workflows (separate from the CodeBuild group). +# CodeBuild variant owns the schedule (commented out below). Uses a per-ref +# concurrency group with cancel-in-progress, so a newer run on the same ref supersedes the one in progress. name: Run Blue/Green Integration Tests @@ -18,16 +18,17 @@ permissions: contents: read env: - # Pull test container images from ECR Public rather than Docker Hub. These runners share an - # outbound IP and have no Docker Hub credentials, so anonymous pulls get rate limited. + # Pull test container images from ECR Public rather than Docker Hub. GitHub-hosted runners pull + # from a pool of egress IPs shared across all GitHub customers, with no Docker Hub credentials, so + # anonymous Docker Hub pulls get rate limited. TEST_CONTAINER_REGISTRY: ecr concurrency: - # Shared by every GitHub-hosted RDS workflow, so only one of them at a time claims DB clusters and - # security-group rules. Runs in this group execute one at a time; with `queue: max`, later runs - # wait in FIFO order (up to 100 pending) instead of being cancelled. - group: rds-integration-tests - queue: max + # One run per workflow per ref: a newer run on the same ref cancels the one in progress, so a new + # push supersedes a stale commit. The "Clean up RDS resources" step runs on cancellation, so a + # superseded run tears down the DB clusters it created instead of leaking them. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: bg-integration-tests: