From c3b0dcd2f6c4d609f6cfba4467a35235ee15575f Mon Sep 17 00:00:00 2001 From: David Ho Date: Mon, 11 May 2026 12:49:58 -0700 Subject: [PATCH 1/4] Define shared list of modules to skip --- .github/workflows/new-module-verification.yml | 24 ++----- buildspecs/resources/modules-to-skip-docs.txt | 4 ++ buildspecs/resources/modules-to-skip.txt | 19 ++++++ scripts/setup-new-module | 66 ++----------------- 4 files changed, 32 insertions(+), 81 deletions(-) create mode 100644 buildspecs/resources/modules-to-skip-docs.txt create mode 100644 buildspecs/resources/modules-to-skip.txt diff --git a/.github/workflows/new-module-verification.yml b/.github/workflows/new-module-verification.yml index 47e3f7d9f076..2178a35af28e 100644 --- a/.github/workflows/new-module-verification.yml +++ b/.github/workflows/new-module-verification.yml @@ -89,28 +89,12 @@ jobs: echo "Verifying test module requirements..." - # 1. Check if excluded from maven deploy command - if ! grep -q "$MODULE_NAME" buildspecs/release-to-maven.yml 2>/dev/null; then - echo "::error::Module $MODULE_NAME is not excluded from maven deploy command in buildspecs/release-to-maven.yml" + # 1. Check if excluded from maven publishing and javadoc generation + if ! grep -qx "$MODULE_NAME" buildspecs/resources/modules-to-skip.txt 2>/dev/null; then + echo "::error::Module $MODULE_NAME is not excluded from maven publishing and javadoc generation in buildspecs/resources/modules-to-skip.txt" HAS_ERRORS=1 else - echo "✅ Module is excluded from maven deploy command" - fi - - # 2. Check if excluded from maven deploy command - if ! grep -q "$MODULE_NAME" buildspecs/release-to-maven-central.yml 2>/dev/null; then - echo "::error::Module $MODULE_NAME is not excluded from maven deploy command in buildspecs/release-to-maven-central.yml" - HAS_ERRORS=1 - else - echo "✅ Module is excluded from maven deploy command" - fi - - # 3. Check if excluded from javadoc generation - if ! grep -q "$MODULE_NAME" buildspecs/release-javadoc.yml 2>/dev/null; then - echo "::error::Module $MODULE_NAME is not excluded from javadoc generation in buildspecs/release-javadoc.yml" - HAS_ERRORS=1 - else - echo "✅ Module is excluded from javadoc generation" + echo "✅ Module is excluded from maven publishing and javadoc generation" fi # 4. Check if Brazil import is skipped diff --git a/buildspecs/resources/modules-to-skip-docs.txt b/buildspecs/resources/modules-to-skip-docs.txt new file mode 100644 index 000000000000..2631631ccd6a --- /dev/null +++ b/buildspecs/resources/modules-to-skip-docs.txt @@ -0,0 +1,4 @@ +http-client-tests +ruleset-testing-core +test-utils +v2-migration diff --git a/buildspecs/resources/modules-to-skip.txt b/buildspecs/resources/modules-to-skip.txt new file mode 100644 index 000000000000..4e92f1791ba0 --- /dev/null +++ b/buildspecs/resources/modules-to-skip.txt @@ -0,0 +1,19 @@ +architecture-tests +auth-tests +bundle-shading-tests +codegen-generated-classes-test +crt-unavailable-tests +http-client-benchmarks +module-path-tests +old-client-version-compatibility-test +protocol-tests +protocol-tests-core +region-testing +s3-benchmarks +s3-tests +sdk-benchmarks +sdk-native-image-test +sdk-standard-benchmarks +stability-tests +tests-coverage-reporting +v2-migration-tests diff --git a/scripts/setup-new-module b/scripts/setup-new-module index fff112a75ce9..0711de637e39 100755 --- a/scripts/setup-new-module +++ b/scripts/setup-new-module @@ -274,67 +274,11 @@ def update_brazil_json(root_dir, module_name, is_test): with open(brazil_json, 'w') as f: f.write(modified_content) def update_buildspecs(root_dir, module_name): - """Update buildspec files for test modules.""" - release_maven = os.path.join(root_dir, 'buildspecs/release-to-maven.yml') - release_javadoc = os.path.join(root_dir, 'buildspecs/release-javadoc.yml') - - # Update release-to-maven.yml - if os.path.isfile(release_maven): - with open(release_maven, 'r') as f: - content = f.read() - - # Look for MODULES_TO_SKIP variable - modules_pattern = r'MODULES_TO_SKIP="([^"]*)"' - match = re.search(modules_pattern, content) - - if match: - current_modules = match.group(1) - new_modules = f"{current_modules},{module_name}" if current_modules else module_name - - # Update the file - modified_content = re.sub( - modules_pattern, - f'MODULES_TO_SKIP="{new_modules}"', - content - ) - - with open(release_maven, 'w') as f: - f.write(modified_content) - - print(f"Updated MODULES_TO_SKIP in {release_maven} to include {module_name}") - else: - print(f"MODULES_TO_SKIP variable not found in {release_maven}. Please manually update.") - else: - print(f"Warning: {release_maven} does not exist. Skipping.") - - # Update release-javadoc.yml - if os.path.isfile(release_javadoc): - with open(release_javadoc, 'r') as f: - content = f.read() - - # Look for MODULES_TO_SKIP variable - modules_pattern = r'MODULES_TO_SKIP="([^"]*)"' - match = re.search(modules_pattern, content) - - if match: - current_modules = match.group(1) - new_modules = f"{current_modules},{module_name}" if current_modules else module_name - - # Update the file - modified_content = re.sub( - modules_pattern, - f'MODULES_TO_SKIP="{new_modules}"', - content - ) - - with open(release_javadoc, 'w') as f: - f.write(modified_content) - - print(f"Updated MODULES_TO_SKIP in {release_javadoc} to include {module_name}") - else: - print(f"MODULES_TO_SKIP variable not found in {release_javadoc}. Please manually update.") - else: - print(f"Warning: {release_javadoc} does not exist. Skipping.") + """Update module skip lists for test modules.""" + path = os.path.join(root_dir, 'buildspecs/resources/modules-to-skip.txt') + with open(path, 'a') as f: + f.write(f"{module_name}\n") + print(f"Added {module_name} to {path}") def main(): """Main function to set up a new module.""" args = parse_arguments() From 4b2dedb89fdb21580c13c49ee35729f181305e94 Mon Sep 17 00:00:00 2001 From: David Ho Date: Wed, 13 May 2026 10:38:05 -0700 Subject: [PATCH 2/4] Use allowlist to generate modules to skip --- .github/workflows/new-module-verification.yml | 9 ++++----- buildspecs/resources/generate-modules-to-skip.sh | 13 +++++++++++++ .../resources/test-modules-publish-allowlist.txt | 5 +++++ scripts/setup-new-module | 14 +++++++++----- 4 files changed, 31 insertions(+), 10 deletions(-) create mode 100755 buildspecs/resources/generate-modules-to-skip.sh create mode 100644 buildspecs/resources/test-modules-publish-allowlist.txt diff --git a/.github/workflows/new-module-verification.yml b/.github/workflows/new-module-verification.yml index 2178a35af28e..81cbb5e90f0b 100644 --- a/.github/workflows/new-module-verification.yml +++ b/.github/workflows/new-module-verification.yml @@ -89,12 +89,11 @@ jobs: echo "Verifying test module requirements..." - # 1. Check if excluded from maven publishing and javadoc generation - if ! grep -qx "$MODULE_NAME" buildspecs/resources/modules-to-skip.txt 2>/dev/null; then - echo "::error::Module $MODULE_NAME is not excluded from maven publishing and javadoc generation in buildspecs/resources/modules-to-skip.txt" - HAS_ERRORS=1 + # 1. Check if excluded from Maven publishing + if grep -qx "$MODULE_NAME" buildspecs/resources/test-modules-publish-allowlist.txt 2>/dev/null; then + echo "Module is in test-modules-publish-allowlist.txt - will be published to Maven" else - echo "✅ Module is excluded from maven publishing and javadoc generation" + echo "Module is not in test-modules-publish-allowlist.txt - will be excluded from Maven publishing" fi # 4. Check if Brazil import is skipped diff --git a/buildspecs/resources/generate-modules-to-skip.sh b/buildspecs/resources/generate-modules-to-skip.sh new file mode 100755 index 000000000000..1e9f142107e7 --- /dev/null +++ b/buildspecs/resources/generate-modules-to-skip.sh @@ -0,0 +1,13 @@ +#!/bin/bash +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +REPO_ROOT=$(cd "$SCRIPT_DIR/../.." && pwd) +ALLOWLIST="$SCRIPT_DIR/test-modules-publish-allowlist.txt" + +if [[ "$1" == "--docs" ]]; then + echo "$(ls "$REPO_ROOT/test/" | tr '\n' ',' | sed 's/,$//'),v2-migration" +elif [[ "$1" == "--maven" ]]; then + ls "$REPO_ROOT/test/" | grep -vxFf "$ALLOWLIST" | tr '\n' ',' | sed 's/,$//' +else + echo "Usage: $0 --maven | --docs" >&2 + exit 1 +fi diff --git a/buildspecs/resources/test-modules-publish-allowlist.txt b/buildspecs/resources/test-modules-publish-allowlist.txt new file mode 100644 index 000000000000..cc8e26de1875 --- /dev/null +++ b/buildspecs/resources/test-modules-publish-allowlist.txt @@ -0,0 +1,5 @@ +bundle-logging-bridge-binding-test +http-client-tests +ruleset-testing-core +service-test-utils +test-utils diff --git a/scripts/setup-new-module b/scripts/setup-new-module index 0711de637e39..f09921892aa7 100755 --- a/scripts/setup-new-module +++ b/scripts/setup-new-module @@ -274,11 +274,15 @@ def update_brazil_json(root_dir, module_name, is_test): with open(brazil_json, 'w') as f: f.write(modified_content) def update_buildspecs(root_dir, module_name): - """Update module skip lists for test modules.""" - path = os.path.join(root_dir, 'buildspecs/resources/modules-to-skip.txt') - with open(path, 'a') as f: - f.write(f"{module_name}\n") - print(f"Added {module_name} to {path}") + """Update module publish allowlist for test modules.""" + answer = input(f"Should {module_name} be published to Maven Central? (y/N): ").strip().lower() + if answer == 'y': + path = os.path.join(root_dir, 'buildspecs/resources/test-modules-publish-allowlist.txt') + with open(path, 'a') as f: + f.write(f"{module_name}\n") + print(f"Added {module_name} to {path}") + else: + print(f"{module_name} will be excluded from Maven publishing") def main(): """Main function to set up a new module.""" args = parse_arguments() From 4d778a38ec4042dca89d64ed15fe1343e3b308b5 Mon Sep 17 00:00:00 2001 From: David Ho Date: Wed, 13 May 2026 10:48:20 -0700 Subject: [PATCH 3/4] Use allowlist to generate modules to skip --- buildspecs/resources/modules-to-skip-docs.txt | 4 ---- buildspecs/resources/modules-to-skip.txt | 19 ------------------- 2 files changed, 23 deletions(-) delete mode 100644 buildspecs/resources/modules-to-skip-docs.txt delete mode 100644 buildspecs/resources/modules-to-skip.txt diff --git a/buildspecs/resources/modules-to-skip-docs.txt b/buildspecs/resources/modules-to-skip-docs.txt deleted file mode 100644 index 2631631ccd6a..000000000000 --- a/buildspecs/resources/modules-to-skip-docs.txt +++ /dev/null @@ -1,4 +0,0 @@ -http-client-tests -ruleset-testing-core -test-utils -v2-migration diff --git a/buildspecs/resources/modules-to-skip.txt b/buildspecs/resources/modules-to-skip.txt deleted file mode 100644 index 4e92f1791ba0..000000000000 --- a/buildspecs/resources/modules-to-skip.txt +++ /dev/null @@ -1,19 +0,0 @@ -architecture-tests -auth-tests -bundle-shading-tests -codegen-generated-classes-test -crt-unavailable-tests -http-client-benchmarks -module-path-tests -old-client-version-compatibility-test -protocol-tests -protocol-tests-core -region-testing -s3-benchmarks -s3-tests -sdk-benchmarks -sdk-native-image-test -sdk-standard-benchmarks -stability-tests -tests-coverage-reporting -v2-migration-tests From 950e073827b3aece128a855dcad029aaf9fd808c Mon Sep 17 00:00:00 2001 From: David Ho Date: Thu, 14 May 2026 10:17:28 -0700 Subject: [PATCH 4/4] Address comments --- buildspecs/resources/generate-modules-to-skip.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/buildspecs/resources/generate-modules-to-skip.sh b/buildspecs/resources/generate-modules-to-skip.sh index 1e9f142107e7..36ac5247b715 100755 --- a/buildspecs/resources/generate-modules-to-skip.sh +++ b/buildspecs/resources/generate-modules-to-skip.sh @@ -1,13 +1,21 @@ #!/bin/bash +# Generates a comma-separated list of modules to skip during publishing. +# +# Usage: +# generate-modules-to-skip.sh --release Modules to skip for Maven Central release publishing. +# Skips all test/ modules except those in test-modules-publish-allowlist.txt. +# generate-modules-to-skip.sh --docs Modules to skip for javadoc generation. +# Skips all test/ modules + v2-migration. +set -euo pipefail SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) REPO_ROOT=$(cd "$SCRIPT_DIR/../.." && pwd) ALLOWLIST="$SCRIPT_DIR/test-modules-publish-allowlist.txt" if [[ "$1" == "--docs" ]]; then echo "$(ls "$REPO_ROOT/test/" | tr '\n' ',' | sed 's/,$//'),v2-migration" -elif [[ "$1" == "--maven" ]]; then +elif [[ "$1" == "--release" ]]; then ls "$REPO_ROOT/test/" | grep -vxFf "$ALLOWLIST" | tr '\n' ',' | sed 's/,$//' else - echo "Usage: $0 --maven | --docs" >&2 + echo "Usage: $0 --release | --docs" >&2 exit 1 fi