diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index aed5d0e1ec91d..5859b545e47cc 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -34,7 +34,7 @@ from datetime import datetime from enum import Enum from functools import partial -from multiprocessing import Pool +from multiprocessing import get_context from pathlib import Path from subprocess import DEVNULL from typing import IO, TYPE_CHECKING, Any, Literal, NamedTuple @@ -3611,7 +3611,10 @@ def generate_providers_metadata( ) console_print("\n[info]Checking provider.yaml versions[1:] against PyPI for stale entries...[/]\n") - with Pool() as pypi_pool: + # "spawn" (not the platform-default fork): the parent has already used GitPython and + # opened network sockets before reaching here, and forking that state into workers + # deadlocks. See get_all_constraint_files_and_airflow_releases for the same reasoning. + with get_context("spawn").Pool() as pypi_pool: pruned_per_provider = pypi_pool.map(prune_unreleased_versions_from_provider_yaml, package_ids) total_pruned = 0 for pid, pruned in zip(package_ids, pruned_per_provider): @@ -3636,7 +3639,7 @@ def generate_providers_metadata( airflow_release_dates=airflow_release_dates, current_metadata=current_metadata, ) - with Pool() as pool: + with get_context("spawn").Pool() as pool: results = pool.map( partial_generate_providers_metadata, package_ids, diff --git a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py index a985a65b55de5..d67e0ec9bfc1b 100644 --- a/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py +++ b/dev/breeze/src/airflow_breeze/utils/provider_dependencies.py @@ -26,7 +26,7 @@ import urllib.request from collections.abc import Generator from functools import cache, partial -from multiprocessing import Pool +from multiprocessing import get_context from pathlib import Path from threading import Lock from typing import NamedTuple @@ -276,7 +276,12 @@ def get_all_constraint_files_and_airflow_releases( airflow_release_dates_path.write_text(json.dumps(airflow_release_dates, indent=2)) console_print(f"[info]Airflow release dates saved in: {airflow_release_dates_path}[/]") with ci_group("Downloading constraints for all Airflow versions for all historical Python versions"): - with Pool() as pool: + # Use the "spawn" start method rather than the platform default: GitPython + # (used in the workers via get_tag_date) opens persistent `git cat-file --batch` + # subprocesses and is not fork-safe, and the parent already holds open network + # sockets from the version/constraints downloads. Forking that state into workers + # deadlocks; spawn gives each worker a clean interpreter. + with get_context("spawn").Pool() as pool: # We use partial to pass the common parameters to the function get_constraints_for_python_version_partial = partial( get_constraints_for_python_version,