From 4c34d9cbc73bbf2d72cf90f47243e16e7e5c910e Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 17 Sep 2026 10:14:54 +1200 Subject: [PATCH 1/2] [ML] Add test suite for PR pipeline generator Exercises every label/comment code path in pipeline.json.py that calls os.environ, so that a missing import (or similar name error) is caught by the dev-tools pytest CI step before it reaches a real PR build. The test was prompted by a missing 'import os' on the 9.4 branch that caused any backport PR carrying ci:run-qa-tests to fail in ~25s before any build step ran. Co-Authored-By: Claude Sonnet 4.6 --- dev-tools/unittest/test_pr_pipeline.py | 136 +++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 dev-tools/unittest/test_pr_pipeline.py diff --git a/dev-tools/unittest/test_pr_pipeline.py b/dev-tools/unittest/test_pr_pipeline.py new file mode 100644 index 000000000..87ea9842d --- /dev/null +++ b/dev-tools/unittest/test_pr_pipeline.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +"""Tests for .buildkite/pipeline.json.py (the PR build pipeline generator). + +Exercises every label/comment code path that uses os.environ so that a missing +'import os' (or similar name-error) is caught before it silently breaks CI on +backport PRs that carry labels like ci:run-qa-tests. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_PIPELINE = _REPO_ROOT / ".buildkite" / "pipeline.json.py" + + +def _run(extra_env: dict[str, str] | None = None, remove_keys: list[str] | None = None) -> dict: + """Run pipeline.json.py and return the parsed JSON output.""" + env = os.environ.copy() + # Strip any real CI env vars that would alter the code path under test. + for key in (remove_keys or []): + env.pop(key, None) + for key in ["GITHUB_PR_TRIGGER_COMMENT", "GITHUB_PR_LABELS", + "GITHUB_PR_COMMENT_VAR_ACTION", "GITHUB_PR_COMMENT_VAR_ARCH", + "GITHUB_PR_COMMENT_VAR_PLATFORM", "QAF_TESTS_TO_RUN"]: + env.pop(key, None) + if extra_env: + env.update(extra_env) + out = subprocess.check_output( + [sys.executable, str(_PIPELINE)], + cwd=str(_REPO_ROOT), + env=env, + text=True, + ) + return json.loads(out) + + +# --------------------------------------------------------------------------- +# Label-triggered paths +# --------------------------------------------------------------------------- + +def test_no_labels_builds_all_platforms() -> None: + pipeline = _run({"GITHUB_PR_LABELS": ""}) + steps = pipeline["steps"] + labels = [s.get("label", "") for s in steps] + assert any("Linux" in l for l in labels) + assert any("MacOS" in l for l in labels) + assert any("Windows" in l for l in labels) + + +def test_qa_label_produces_valid_json() -> None: + """ci:run-qa-tests triggers os.environ.get() — a missing 'import os' crashes here.""" + pipeline = _run({"GITHUB_PR_LABELS": "ci:run-qa-tests"}) + assert isinstance(pipeline, dict) + assert "steps" in pipeline + env = pipeline.get("env", {}) + assert "QAF_TESTS_TO_RUN" in env + assert "ml_cpp_pr" in env["QAF_TESTS_TO_RUN"] + + +def test_pytorch_label_produces_valid_json() -> None: + """ci:run-pytorch-tests is the other label that enters the os.environ code path.""" + pipeline = _run({"GITHUB_PR_LABELS": "ci:run-pytorch-tests"}) + assert isinstance(pipeline, dict) + assert "steps" in pipeline + env = pipeline.get("env", {}) + assert "QAF_TESTS_TO_RUN" in env + assert "pytorch_tests" in env["QAF_TESTS_TO_RUN"] + + +def test_qa_and_pytorch_labels_deduped() -> None: + pipeline = _run({"GITHUB_PR_LABELS": "ci:run-qa-tests,ci:run-pytorch-tests"}) + env = pipeline.get("env", {}) + suites = env.get("QAF_TESTS_TO_RUN", "").split(",") + assert suites.count("ml_cpp_pr") == 1 + assert suites.count("pytorch_tests") == 1 + + +def test_qa_label_with_qaf_override_env() -> None: + """QAF_TESTS_TO_RUN env override is split and forwarded, not appended whole.""" + pipeline = _run({ + "GITHUB_PR_LABELS": "ci:run-qa-tests", + "QAF_TESTS_TO_RUN": "suite_a,suite_b", + }) + env = pipeline.get("env", {}) + suites = env.get("QAF_TESTS_TO_RUN", "").split(",") + assert "suite_a" in suites + assert "suite_b" in suites + assert "ml_cpp_pr" not in suites + + +# --------------------------------------------------------------------------- +# Comment-triggered paths +# --------------------------------------------------------------------------- + +def test_comment_run_qa_tests_produces_valid_json() -> None: + pipeline = _run({ + "GITHUB_PR_TRIGGER_COMMENT": "buildkite run_qa_tests", + "GITHUB_PR_COMMENT_VAR_ACTION": "run_qa_tests", + }) + assert isinstance(pipeline, dict) + assert "steps" in pipeline + + +def test_comment_run_pytorch_tests_produces_valid_json() -> None: + pipeline = _run({ + "GITHUB_PR_TRIGGER_COMMENT": "buildkite run_pytorch_tests", + "GITHUB_PR_COMMENT_VAR_ACTION": "run_pytorch_tests", + }) + assert isinstance(pipeline, dict) + assert "steps" in pipeline + + +# --------------------------------------------------------------------------- +# Output is always valid JSON with required top-level keys +# --------------------------------------------------------------------------- + +def test_output_always_has_env_and_steps() -> None: + for labels in ["", "ci:run-qa-tests", "ci:run-pytorch-tests", + "ci:build-linux", "ci:build-macos", "ci:build-windows"]: + pipeline = _run({"GITHUB_PR_LABELS": labels}) + assert "steps" in pipeline, f"missing 'steps' for labels={labels!r}" + assert "env" in pipeline, f"missing 'env' for labels={labels!r}" From ad3dab2cfeaa05ba6c333057313dc4a0efcfd263 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 17 Sep 2026 12:21:00 +1200 Subject: [PATCH 2/2] [ML] Enable verbose pytest output for dev-tools test suite Co-Authored-By: Claude Sonnet 4.6 --- dev-tools/pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-tools/pytest.ini b/dev-tools/pytest.ini index 254dbbf2b..5606977c0 100644 --- a/dev-tools/pytest.ini +++ b/dev-tools/pytest.ini @@ -1,6 +1,7 @@ [pytest] testpaths = unittest pythonpath = . +addopts = -v markers = integration: requires git fetch from origin; opt-in via VERSION_BUMP_GIT_INTEGRATION=1 (see test file)