diff --git a/airflow-core/src/airflow/provider.yaml.schema.json b/airflow-core/src/airflow/provider.yaml.schema.json index 5714b8db658c5..2f020b2733050 100644 --- a/airflow-core/src/airflow/provider.yaml.schema.json +++ b/airflow-core/src/airflow/provider.yaml.schema.json @@ -58,6 +58,13 @@ "type": "string" } }, + "excluded-platforms": { + "description": "List of platforms (e.g. linux/arm64) excluded for that provider. Used to skip providers whose native dependencies are unavailable on a given architecture.", + "type": "array", + "items": { + "type": "string" + } + }, "integrations": { "description": "List of integrations supported by the provider.", "type": "array", diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 9176fed4183d0..3514c558476c7 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1319,6 +1319,53 @@ def core_test_types_list_as_strings_in_json(self) -> str | None: list_of_list_of_types = _split_list(current_test_types, NUMBER_OF_CORE_SLICES) return json.dumps(_get_test_list_as_json(list_of_list_of_types)) + @cached_property + def _platform_excluded_providers(self) -> set[str]: + """Provider ids that opt out of the current ``self.platform`` via provider.yaml. + + Mirrors the ``excluded-python-versions`` mechanism but keyed by Docker platform + string (e.g. ``linux/arm64``) so providers whose native dependencies are unavailable + on a given architecture can be removed from the test matrix at planning time. + """ + excluded: set[str] = set() + for provider_id, provider_info in get_provider_dependencies().items(): + if self.platform in provider_info.get("excluded-platforms", []): + excluded.add(provider_id) + return excluded + + def _filter_platform_excluded_test_types(self, current_test_types: set[str]) -> None: + """Rewrite ``Providers[...]`` entries in-place to honor ``excluded-platforms``. + + Handles three shapes produced upstream: + + * ``Providers[foo]`` — dropped entirely when ``foo`` is excluded. + * ``Providers[a,foo,b]`` — rewritten to ``Providers[a,b]``; dropped if empty. + * ``Providers[-amazon,celery,google,standard]`` (negative, "all except") — + excluded providers are appended to the negation so they remain skipped. + """ + excluded = self._platform_excluded_providers + if not excluded: + return + for original in tuple(current_test_types): + if not original.startswith("Providers[") or not original.endswith("]"): + continue + inner = original[len("Providers[") : -1] + if inner.startswith("-"): + negated = [p for p in inner[1:].split(",") if p] + additions = sorted(p for p in excluded if p not in negated) + if not additions: + continue + current_test_types.remove(original) + current_test_types.add(f"Providers[-{','.join(sorted(negated + additions))}]") + continue + providers_in = [p for p in inner.split(",") if p] + kept = [p for p in providers_in if p not in excluded] + if kept == providers_in: + continue + current_test_types.remove(original) + if kept: + current_test_types.add(f"Providers[{','.join(kept)}]") + @cached_property def providers_test_types_list_as_strings_in_json(self) -> str: if not self.run_unit_tests: @@ -1335,6 +1382,7 @@ def providers_test_types_list_as_strings_in_json(self) -> str: test_types_to_remove.add(test_type) current_test_types = current_test_types - test_types_to_remove self._extract_long_provider_tests(current_test_types) + self._filter_platform_excluded_test_types(current_test_types) return json.dumps(_get_test_list_as_json([sorted(current_test_types)])) def _get_individual_providers_list(self): @@ -1344,6 +1392,7 @@ def _get_individual_providers_list(self): current_test_types.update( {f"Providers[{provider}]" for provider in get_available_distributions(include_not_ready=True)} ) + self._filter_platform_excluded_test_types(current_test_types) return current_test_types @cached_property diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 3ab0298735891..79ce05ecd3580 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -3178,6 +3178,78 @@ def test_testable_providers_integrations_excludes_arm_disabled_on_arm(): assert "ydb" not in result +def test_individual_providers_excludes_platform_excluded_on_arm(): + """ibm.mq declares `excluded-platforms: [linux/arm64]`, so it must be absent from + the ARM individual-providers matrix (used by the Low-dep ARM canary job) and + present on AMD.""" + arm_checks = SelectiveChecks( + files=("airflow-core/tests/test_example.py",), + commit_ref=NEUTRAL_COMMIT, + github_event=GithubEvents.SCHEDULE, + github_context_dict={"ref_name": "main"}, + default_branch="main", + pr_labels=("full tests needed",), + ) + with patch.object( + SelectiveChecks, "runner_type", new_callable=lambda: property(lambda self: '["ubuntu-22.04-arm"]') + ): + assert arm_checks.platform == "linux/arm64" + arm_output = arm_checks.individual_providers_test_types_list_as_strings_in_json + assert arm_output is not None + assert "Providers[ibm.mq]" not in arm_output + + amd_checks = SelectiveChecks( + files=("airflow-core/tests/test_example.py",), + commit_ref=NEUTRAL_COMMIT, + github_event=GithubEvents.SCHEDULE, + github_context_dict={"ref_name": "main"}, + default_branch="main", + pr_labels=("full tests needed",), + ) + with patch.object( + SelectiveChecks, "runner_type", new_callable=lambda: property(lambda self: PUBLIC_AMD_RUNNERS) + ): + assert amd_checks.platform == "linux/amd64" + amd_output = amd_checks.individual_providers_test_types_list_as_strings_in_json + assert amd_output is not None + assert "Providers[ibm.mq]" in amd_output + + +def test_filter_platform_excluded_test_types_handles_all_shapes(): + """Direct unit check of the in-place filter for the three Providers[...] shapes.""" + checks = SelectiveChecks( + files=(), + commit_ref=NEUTRAL_COMMIT, + github_event=GithubEvents.SCHEDULE, + github_context_dict={"ref_name": "main"}, + default_branch="main", + ) + with patch.object( + SelectiveChecks, + "_platform_excluded_providers", + new_callable=lambda: property(lambda self: {"ibm.mq"}), + ): + # Bare match drops entry. + ts = {"Providers[ibm.mq]"} + checks._filter_platform_excluded_test_types(ts) + assert ts == set() + + # Combined positive form drops just the excluded id. + ts = {"Providers[amazon,ibm.mq,google]"} + checks._filter_platform_excluded_test_types(ts) + assert ts == {"Providers[amazon,google]"} + + # Negative form gets the excluded id appended. + ts = {"Providers[-amazon,celery,google,standard]"} + checks._filter_platform_excluded_test_types(ts) + assert ts == {"Providers[-amazon,celery,google,ibm.mq,standard]"} + + # Non-Providers entries are untouched. + ts = {"Core", "Always"} + checks._filter_platform_excluded_test_types(ts) + assert ts == {"Core", "Always"} + + @patch("airflow_breeze.utils.selective_checks.run_command") def test_provider_dependency_bump_check_no_changes(mock_run_command): """Test that provider dependency bump check passes when no pyproject.toml files are changed.""" diff --git a/providers/ibm/mq/provider.yaml b/providers/ibm/mq/provider.yaml index d1c56376f2f75..341980678aa60 100644 --- a/providers/ibm/mq/provider.yaml +++ b/providers/ibm/mq/provider.yaml @@ -24,6 +24,14 @@ lifecycle: incubation source-date-epoch: 1758787200 description: | `IBM MQ `__ +# IBM MQ ships its native C client library only for Linux x86_64, Windows x64 and Java +# (https://www.ibm.com/support/pages/downloading-ibm-mq-94 — Redist clients). The +# `ibmmq` Python bindings link against that C client at build time, so `uv sync +# --resolution lowest-direct --all-extras` fails to build the sdist on aarch64. +# Exclude the provider from the linux/arm64 test matrix until IBM publishes an aarch64 +# redistributable. +excluded-platforms: + - linux/arm64 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have diff --git a/scripts/ci/prek/update_providers_dependencies.py b/scripts/ci/prek/update_providers_dependencies.py index b114e55a04726..41b0c4fe79f4d 100755 --- a/scripts/ci/prek/update_providers_dependencies.py +++ b/scripts/ci/prek/update_providers_dependencies.py @@ -218,6 +218,8 @@ def check_if_different_provider_used(file_path: Path) -> None: ) excluded_versions = ALL_PROVIDERS[key].get("excluded-python-versions") unique_sorted_dependencies[key]["excluded-python-versions"] = excluded_versions or [] + excluded_platforms = ALL_PROVIDERS[key].get("excluded-platforms") + unique_sorted_dependencies[key]["excluded-platforms"] = excluded_platforms or [] unique_sorted_dependencies[key]["state"] = STATES[key] if errors: console.print()