From 77276ee50712b058b88faaa2cfbcf324e6750e24 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Wed, 26 Aug 2026 18:20:57 -0400 Subject: [PATCH 1/2] ci: avoid duplicate parametric system tests --- .gitlab/extract_tracer_release_scenarios.py | 56 +++++++++ .gitlab/generate-package.php | 12 +- .../test_extract_tracer_release_scenarios.py | 114 ++++++++++++++++++ 3 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 .gitlab/extract_tracer_release_scenarios.py create mode 100644 .gitlab/test_extract_tracer_release_scenarios.py diff --git a/.gitlab/extract_tracer_release_scenarios.py b/.gitlab/extract_tracer_release_scenarios.py new file mode 100644 index 0000000000..b54005a094 --- /dev/null +++ b/.gitlab/extract_tracer_release_scenarios.py @@ -0,0 +1,56 @@ +import json +import sys + + +def extract_scenarios(data: object) -> list[str]: + if not isinstance(data, dict): + raise ValueError("workflow parameters must be an object") + + endtoend_defs = data.get("endtoend_defs") + if not isinstance(endtoend_defs, dict): + raise ValueError("endtoend_defs must be an object") + + parallel_jobs = endtoend_defs.get("parallel_jobs") + if not isinstance(parallel_jobs, list): + raise ValueError("endtoend_defs.parallel_jobs must be a list") + + scenarios = set() + for job in parallel_jobs: + if not isinstance(job, dict): + raise ValueError("each parallel job must be an object") + + job_scenarios = job.get("scenarios") + if not isinstance(job_scenarios, list): + raise ValueError("each parallel job scenarios must be a list") + + for scenario in job_scenarios: + if not isinstance(scenario, str): + raise ValueError("each scenario must be a string") + if not scenario or any(character.isspace() for character in scenario): + raise ValueError( + "scenario names must be nonempty and contain no whitespace" + ) + scenarios.add(scenario) + + if not scenarios: + raise ValueError("scenario selection must not be empty") + + return sorted(scenarios) + + +def main() -> int: + try: + scenarios = extract_scenarios(json.load(sys.stdin)) + except json.JSONDecodeError as error: + print(f"invalid JSON: {error.msg}", file=sys.stderr) + return 1 + except ValueError as error: + print(error, file=sys.stderr) + return 1 + + print(" ".join(scenarios)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index a952d859af..19ab12846e 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -168,6 +168,16 @@ function appsec_image_from_tag_mapping(string $tag): string REQUIREMENTS_BLOCK_JSON_PATH: "loader/packaging/block_tests.json" REQUIREMENTS_ALLOW_JSON_PATH: "loader/packaging/allow_tests.json" +tracer_release_scenario_extractor_test: + stage: prepare + image: registry.ddbuild.io/images/mirror/python:3.12-slim-bullseye + tags: [ "arch:amd64" ] + needs: [] + variables: + GIT_SUBMODULE_STRATEGY: none + script: + - python3 .gitlab/test_extract_tracer_release_scenarios.py + # dd-trace-php release packaging "prepare code": @@ -1400,7 +1410,7 @@ function appsec_image_from_tag_mapping(string $tag): string script: - DD_API_KEY=$(cat /tmp/.dd-api-key 2>/dev/null) || { echo "Failed to fetch DD_API_KEY"; exit 1; } - export DD_API_KEY - - SCENARIOS=$(PYTHONPATH=. venv/bin/python utils/scripts/compute-workflow-parameters.py php -g tracer_release -f json | python3 -c "import sys,json;d=json.load(sys.stdin);s=set();[s.update(v['scenarios']) for v in d.values() if isinstance(v,dict) and 'scenarios' in v];print(' '.join(sorted(s)))") + - SCENARIOS=$(PYTHONPATH=. venv/bin/python utils/scripts/compute-workflow-parameters.py php -g tracer_release --excluded-scenarios PARAMETRIC -f json | python3 "$CI_PROJECT_DIR/.gitlab/extract_tracer_release_scenarios.py") - FAILED=""; for S in $SCENARIOS; do echo "=== Running $S ==="; ./run.sh $S || FAILED="$FAILED $S"; done; if [ -n "$FAILED" ]; then echo "Failed scenarios:$FAILED"; exit 1; fi diff --git a/.gitlab/test_extract_tracer_release_scenarios.py b/.gitlab/test_extract_tracer_release_scenarios.py new file mode 100644 index 0000000000..abe9f5978f --- /dev/null +++ b/.gitlab/test_extract_tracer_release_scenarios.py @@ -0,0 +1,114 @@ +import json +from pathlib import Path +import subprocess +import sys +import unittest + + +HELPER = Path(__file__).with_name("extract_tracer_release_scenarios.py") + + +class ExtractTracerReleaseScenariosTest(unittest.TestCase): + def run_helper(self, fixture, *, optimize=False): + command = [sys.executable] + if optimize: + command.append("-O") + command.append(str(HELPER)) + payload = fixture if isinstance(fixture, str) else json.dumps(fixture) + return subprocess.run(command, input=payload, text=True, capture_output=True) + + def test_extracts_sorted_unique_scenarios_from_canonical_jobs(self): + fixture = { + "endtoend_defs": { + "parallel_jobs": [ + {"scenarios": ["INTEGRATIONS", "DEFAULT"]}, + {"scenarios": ["DEFAULT"]}, + ] + } + } + + result = self.run_helper(fixture) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "DEFAULT INTEGRATIONS\n") + + def test_rejects_missing_fields(self): + fixtures = [ + ({}, "endtoend_defs"), + ({"endtoend_defs": {}}, "parallel_jobs"), + ({"endtoend_defs": {"parallel_jobs": [{}]}}, "scenarios"), + ] + + for fixture, expected_error in fixtures: + with self.subTest(fixture=fixture): + result = self.run_helper(fixture) + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected_error, result.stderr) + + def test_rejects_wrong_types(self): + fixtures = [ + ([], "workflow parameters"), + ({"endtoend_defs": []}, "endtoend_defs"), + ({"endtoend_defs": {"parallel_jobs": {}}}, "parallel_jobs"), + ({"endtoend_defs": {"parallel_jobs": [None]}}, "parallel job"), + ({"endtoend_defs": {"parallel_jobs": [{"scenarios": {}}]}}, "scenarios"), + ( + {"endtoend_defs": {"parallel_jobs": [{"scenarios": [1]}]}}, + "scenario must be a string", + ), + ( + {"endtoend_defs": {"parallel_jobs": [{"scenarios": "DEFAULT"}]}}, + "scenarios", + ), + ] + + for fixture, expected_error in fixtures: + with self.subTest(fixture=fixture): + result = self.run_helper(fixture) + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected_error, result.stderr) + + def test_rejects_empty_and_whitespace_scenarios(self): + fixtures = [ + ({"endtoend_defs": {"parallel_jobs": []}}, "scenario selection"), + ( + {"endtoend_defs": {"parallel_jobs": [{"scenarios": []}]}}, + "scenario selection", + ), + ( + {"endtoend_defs": {"parallel_jobs": [{"scenarios": [""]}]}}, + "scenario name", + ), + ( + {"endtoend_defs": {"parallel_jobs": [{"scenarios": [" "]}]}}, + "scenario name", + ), + ( + {"endtoend_defs": {"parallel_jobs": [{"scenarios": ["BAD NAME"]}]}}, + "scenario name", + ), + ] + + for fixture, expected_error in fixtures: + with self.subTest(fixture=fixture): + result = self.run_helper(fixture) + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected_error, result.stderr) + + def test_rejects_malformed_json(self): + result = self.run_helper("{") + + self.assertNotEqual(result.returncode, 0) + self.assertIn("invalid JSON", result.stderr) + + def test_validation_survives_optimized_python(self): + fixture = {"endtoend_defs": {"parallel_jobs": [{"scenarios": [" "]}]}} + + result = self.run_helper(fixture, optimize=True) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("scenario name", result.stderr) + + +if __name__ == "__main__": + unittest.main() From d397123d7f062efdba6d1c0f3eb62712508dbdf3 Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Thu, 27 Aug 2026 10:32:54 -0400 Subject: [PATCH 2/2] ci: inline tracer-release scenario extraction --- .gitlab/extract_tracer_release_scenarios.py | 56 --------- .gitlab/generate-package.php | 30 +++-- .../test_extract_tracer_release_scenarios.py | 114 ------------------ 3 files changed, 19 insertions(+), 181 deletions(-) delete mode 100644 .gitlab/extract_tracer_release_scenarios.py delete mode 100644 .gitlab/test_extract_tracer_release_scenarios.py diff --git a/.gitlab/extract_tracer_release_scenarios.py b/.gitlab/extract_tracer_release_scenarios.py deleted file mode 100644 index b54005a094..0000000000 --- a/.gitlab/extract_tracer_release_scenarios.py +++ /dev/null @@ -1,56 +0,0 @@ -import json -import sys - - -def extract_scenarios(data: object) -> list[str]: - if not isinstance(data, dict): - raise ValueError("workflow parameters must be an object") - - endtoend_defs = data.get("endtoend_defs") - if not isinstance(endtoend_defs, dict): - raise ValueError("endtoend_defs must be an object") - - parallel_jobs = endtoend_defs.get("parallel_jobs") - if not isinstance(parallel_jobs, list): - raise ValueError("endtoend_defs.parallel_jobs must be a list") - - scenarios = set() - for job in parallel_jobs: - if not isinstance(job, dict): - raise ValueError("each parallel job must be an object") - - job_scenarios = job.get("scenarios") - if not isinstance(job_scenarios, list): - raise ValueError("each parallel job scenarios must be a list") - - for scenario in job_scenarios: - if not isinstance(scenario, str): - raise ValueError("each scenario must be a string") - if not scenario or any(character.isspace() for character in scenario): - raise ValueError( - "scenario names must be nonempty and contain no whitespace" - ) - scenarios.add(scenario) - - if not scenarios: - raise ValueError("scenario selection must not be empty") - - return sorted(scenarios) - - -def main() -> int: - try: - scenarios = extract_scenarios(json.load(sys.stdin)) - except json.JSONDecodeError as error: - print(f"invalid JSON: {error.msg}", file=sys.stderr) - return 1 - except ValueError as error: - print(error, file=sys.stderr) - return 1 - - print(" ".join(scenarios)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 19ab12846e..605237d8cb 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -168,16 +168,6 @@ function appsec_image_from_tag_mapping(string $tag): string REQUIREMENTS_BLOCK_JSON_PATH: "loader/packaging/block_tests.json" REQUIREMENTS_ALLOW_JSON_PATH: "loader/packaging/allow_tests.json" -tracer_release_scenario_extractor_test: - stage: prepare - image: registry.ddbuild.io/images/mirror/python:3.12-slim-bullseye - tags: [ "arch:amd64" ] - needs: [] - variables: - GIT_SUBMODULE_STRATEGY: none - script: - - python3 .gitlab/test_extract_tracer_release_scenarios.py - # dd-trace-php release packaging "prepare code": @@ -1410,7 +1400,25 @@ function appsec_image_from_tag_mapping(string $tag): string script: - DD_API_KEY=$(cat /tmp/.dd-api-key 2>/dev/null) || { echo "Failed to fetch DD_API_KEY"; exit 1; } - export DD_API_KEY - - SCENARIOS=$(PYTHONPATH=. venv/bin/python utils/scripts/compute-workflow-parameters.py php -g tracer_release --excluded-scenarios PARAMETRIC -f json | python3 "$CI_PROJECT_DIR/.gitlab/extract_tracer_release_scenarios.py") + - | + set -o pipefail + SCENARIOS=$( + PYTHONPATH=. venv/bin/python utils/scripts/compute-workflow-parameters.py php -g tracer_release --excluded-scenarios PARAMETRIC -f json | + python3 -c ' + import json, sys + + data = json.load(sys.stdin) + scenarios = { + scenario + for job in data["endtoend_defs"]["parallel_jobs"] + for scenario in job["scenarios"] + } + invalid = any(not isinstance(scenario, str) or not scenario or any(character.isspace() for character in scenario) for scenario in scenarios) + if not scenarios or invalid or "PARAMETRIC" in scenarios: + raise SystemExit(1) + print(" ".join(sorted(scenarios))) + ' + ) || { echo "Failed to compute tracer-release scenarios"; exit 1; } - FAILED=""; for S in $SCENARIOS; do echo "=== Running $S ==="; ./run.sh $S || FAILED="$FAILED $S"; done; if [ -n "$FAILED" ]; then echo "Failed scenarios:$FAILED"; exit 1; fi diff --git a/.gitlab/test_extract_tracer_release_scenarios.py b/.gitlab/test_extract_tracer_release_scenarios.py deleted file mode 100644 index abe9f5978f..0000000000 --- a/.gitlab/test_extract_tracer_release_scenarios.py +++ /dev/null @@ -1,114 +0,0 @@ -import json -from pathlib import Path -import subprocess -import sys -import unittest - - -HELPER = Path(__file__).with_name("extract_tracer_release_scenarios.py") - - -class ExtractTracerReleaseScenariosTest(unittest.TestCase): - def run_helper(self, fixture, *, optimize=False): - command = [sys.executable] - if optimize: - command.append("-O") - command.append(str(HELPER)) - payload = fixture if isinstance(fixture, str) else json.dumps(fixture) - return subprocess.run(command, input=payload, text=True, capture_output=True) - - def test_extracts_sorted_unique_scenarios_from_canonical_jobs(self): - fixture = { - "endtoend_defs": { - "parallel_jobs": [ - {"scenarios": ["INTEGRATIONS", "DEFAULT"]}, - {"scenarios": ["DEFAULT"]}, - ] - } - } - - result = self.run_helper(fixture) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout, "DEFAULT INTEGRATIONS\n") - - def test_rejects_missing_fields(self): - fixtures = [ - ({}, "endtoend_defs"), - ({"endtoend_defs": {}}, "parallel_jobs"), - ({"endtoend_defs": {"parallel_jobs": [{}]}}, "scenarios"), - ] - - for fixture, expected_error in fixtures: - with self.subTest(fixture=fixture): - result = self.run_helper(fixture) - self.assertNotEqual(result.returncode, 0) - self.assertIn(expected_error, result.stderr) - - def test_rejects_wrong_types(self): - fixtures = [ - ([], "workflow parameters"), - ({"endtoend_defs": []}, "endtoend_defs"), - ({"endtoend_defs": {"parallel_jobs": {}}}, "parallel_jobs"), - ({"endtoend_defs": {"parallel_jobs": [None]}}, "parallel job"), - ({"endtoend_defs": {"parallel_jobs": [{"scenarios": {}}]}}, "scenarios"), - ( - {"endtoend_defs": {"parallel_jobs": [{"scenarios": [1]}]}}, - "scenario must be a string", - ), - ( - {"endtoend_defs": {"parallel_jobs": [{"scenarios": "DEFAULT"}]}}, - "scenarios", - ), - ] - - for fixture, expected_error in fixtures: - with self.subTest(fixture=fixture): - result = self.run_helper(fixture) - self.assertNotEqual(result.returncode, 0) - self.assertIn(expected_error, result.stderr) - - def test_rejects_empty_and_whitespace_scenarios(self): - fixtures = [ - ({"endtoend_defs": {"parallel_jobs": []}}, "scenario selection"), - ( - {"endtoend_defs": {"parallel_jobs": [{"scenarios": []}]}}, - "scenario selection", - ), - ( - {"endtoend_defs": {"parallel_jobs": [{"scenarios": [""]}]}}, - "scenario name", - ), - ( - {"endtoend_defs": {"parallel_jobs": [{"scenarios": [" "]}]}}, - "scenario name", - ), - ( - {"endtoend_defs": {"parallel_jobs": [{"scenarios": ["BAD NAME"]}]}}, - "scenario name", - ), - ] - - for fixture, expected_error in fixtures: - with self.subTest(fixture=fixture): - result = self.run_helper(fixture) - self.assertNotEqual(result.returncode, 0) - self.assertIn(expected_error, result.stderr) - - def test_rejects_malformed_json(self): - result = self.run_helper("{") - - self.assertNotEqual(result.returncode, 0) - self.assertIn("invalid JSON", result.stderr) - - def test_validation_survives_optimized_python(self): - fixture = {"endtoend_defs": {"parallel_jobs": [{"scenarios": [" "]}]}} - - result = self.run_helper(fixture, optimize=True) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("scenario name", result.stderr) - - -if __name__ == "__main__": - unittest.main()