diff --git a/docs/pypi/download-workspace.md b/docs/pypi/download-workspace.md index 4912f71103..10e86d252e 100644 --- a/docs/pypi/download-workspace.md +++ b/docs/pypi/download-workspace.md @@ -5,14 +5,16 @@ This documentation page covers how to download PyPI dependencies in the legacy `WORKSPACE` setup. -To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function and -call it to create the central external repo and individual wheel external repos. +To add pip dependencies to your `WORKSPACE`, load the {obj}`pip_parse` or +{obj}`uv_lock` function and call it to create the central external repo and +individual wheel external repos. ```starlark load("@rules_python//python:pip.bzl", "pip_parse") # Create a central repo that knows about the dependencies needed from -# requirements_lock.txt. +# requirements_lock.txt (or use uv_lock(name = "my_deps", uv_lock = "//path/to:uv.lock") +# for a uv.lock file). pip_parse( name = "my_deps", requirements_lock = "//path/to:requirements_lock.txt", diff --git a/docs/pypi/lock.md b/docs/pypi/lock.md index 727795c1ef..86ee6d4133 100644 --- a/docs/pypi/lock.md +++ b/docs/pypi/lock.md @@ -4,8 +4,7 @@ # Lock :::{note} -`rules_python` supports `requirements.txt` and `uv.lock` (bzlmod only) -formats. +`rules_python` supports `requirements.txt` and `uv.lock` formats. #{gh-issue}`2787` tracks `pylock.toml` support. ::: @@ -231,7 +230,7 @@ lock( `diff_test` from `bazel_skylib`. ::: -## uv.lock (bzlmod only) +## uv.lock The {obj}`lock` rule also supports generating cross-platform `uv.lock` files using `uv lock` when {arg}`lock.out` ends with `.lock` (such as `uv.lock`). @@ -251,7 +250,8 @@ lock( ``` Run `bazel run //:uv_lock.update` to generate or update `uv.lock`, then pass it -to {obj}`pip.parse` in `MODULE.bazel` via {attr}`pip.parse.uv_lock`: +to {obj}`pip.parse` in `MODULE.bazel` via {attr}`pip.parse.uv_lock` (or to +{obj}`uv_lock` in `WORKSPACE`): ```starlark pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") diff --git a/news/uv-lock-workspace.added.md b/news/uv-lock-workspace.added.md new file mode 100644 index 0000000000..ceb9594e6c --- /dev/null +++ b/news/uv-lock-workspace.added.md @@ -0,0 +1,2 @@ +(pypi) Added the {obj}`uv_lock` `WORKSPACE` macro in `//python:pip.bzl` to +install PyPI dependencies from a resolved `uv.lock` file. diff --git a/python/BUILD.bazel b/python/BUILD.bazel index e0e32c5d72..a473df123f 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -229,6 +229,7 @@ bzl_library( "//python/private/pypi:package_annotation", "//python/private/pypi:pip_compile", "//python/private/pypi:pip_repository", + "//python/private/pypi:uv_lock", "//python/private/pypi:whl_library_alias", "//python/private/whl_filegroup", ], diff --git a/python/pip.bzl b/python/pip.bzl index 44ee69d65b..6d260ce24f 100644 --- a/python/pip.bzl +++ b/python/pip.bzl @@ -14,9 +14,9 @@ """Rules for pip integration. This contains a set of rules that are used to support inclusion of third-party -dependencies via fully locked `requirements.txt` files. Some of the exported -symbols should not be used and they are either undocumented here or marked as -for internal use only. +dependencies via fully locked `requirements.txt` or `uv.lock` files. Some of the +exported symbols should not be used and they are either undocumented here or +marked as for internal use only. If you are using a bazel version 7 or above with `bzlmod`, you should only care about the {bzl:obj}`compile_pip_requirements` macro exposed in this file. The @@ -28,12 +28,14 @@ load("//python/private/pypi:multi_pip_parse.bzl", _multi_pip_parse = "multi_pip_ load("//python/private/pypi:package_annotation.bzl", _package_annotation = "package_annotation") load("//python/private/pypi:pip_compile.bzl", "pip_compile") load("//python/private/pypi:pip_repository.bzl", "pip_repository") +load("//python/private/pypi:uv_lock.bzl", _uv_lock = "uv_lock") load("//python/private/pypi:whl_library_alias.bzl", _whl_library_alias = "whl_library_alias") load("//python/private/whl_filegroup:whl_filegroup.bzl", _whl_filegroup = "whl_filegroup") compile_pip_requirements = pip_compile package_annotation = _package_annotation pip_parse = pip_repository +uv_lock = _uv_lock whl_filegroup = _whl_filegroup # Extra utilities visible to rules_python users. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 149074dd3f..2594a42f9c 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -245,7 +245,6 @@ bzl_library( ":select_whl", "//python/private:normalize_name", "//python/private:repo_utils", - "//python/uv/private:uv_lock_to_requirements", ], ) @@ -323,6 +322,7 @@ bzl_library( name = "pip_repository", srcs = ["pip_repository.bzl"], deps = [ + ":hash", ":parse_requirements", ":pep508_env", ":pip_repository_attrs", @@ -435,6 +435,37 @@ bzl_library( ], ) +bzl_library( + name = "uv_lock", + srcs = ["uv_lock.bzl"], + deps = [ + ":uv_lock_repo", + "//python:versions", + "@bazel_skylib//lib:types", + ], +) + +bzl_library( + name = "uv_lock_repo", + srcs = ["uv_lock_repo.bzl"], + deps = [ + ":attrs", + ":parse_requirements", + ":pep508_env", + ":pip_repository", + ":pip_repository_attrs", + ":pypi_repo_utils", + ":render_pkg_aliases", + ":whl_repo_name", + "//python/private:full_version", + "//python/private:normalize_name", + "//python/private:repo_utils", + "//python/private:text_util", + "@bazel_skylib//lib:sets", + "@toml.bzl//:toml", + ], +) + bzl_library( name = "venv_entry_point", srcs = ["venv_entry_point.bzl"], diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 2ac8b3724f..4e4193aa3c 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -28,7 +28,6 @@ behavior. load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "repo_utils") -load("//python/uv/private:uv_lock_to_requirements.bzl", "uv_lock_extras_map") # buildifier: disable=bzl-visibility load(":argparse.bzl", "argparse") load(":hash.bzl", "hash") load(":index_sources.bzl", "index_sources") @@ -161,14 +160,16 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p Returns: {type}`list[struct]` The same format as {func}`parse_requirements`. """ - extras_map = uv_lock_extras_map(uv_lock) + extras_map = _uv_lock_extras_map(uv_lock) uv_packages = {} if not platforms: fail("BUG: platforms must be configured") - for pkg in uv_lock["package"]: + reachable = _uv_lock_reachable_platforms(uv_lock, all_platforms, platforms) + + for idx, pkg in enumerate(uv_lock["package"]): name = pkg["name"] version = pkg["version"] norm_name = normalize_name(name) @@ -189,16 +190,7 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p pkg_extras = sorted(extras_map.get(name, [])) extra_str = "[{}]".format(",".join(pkg_extras)) if pkg_extras else "" - markers = pkg.get("resolution-markers", []) - if markers: - marker_expr = " or ".join(markers) - pkg_platforms = [ - p - for p in all_platforms - if evaluate(marker_expr, env = platforms[p].env) - ] - else: - pkg_platforms = list(all_platforms) + pkg_platforms = reachable[idx] # Prepare candidates candidates = [] @@ -299,16 +291,18 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p ret = [] for norm_name, info in sorted(uv_packages.items()): versions = sorted(info["versions"].keys()) + resolved_platforms = {} + for src in info["resolved_srcs"]: + for p in src.target_platforms: + resolved_platforms[p] = None item = struct( name = norm_name, - # Only expose packages that resolved to at least one source. uv - # workspace/root members (e.g. `source = { virtual = "." }` or - # editable installs) resolve to no wheel/sdist, so exposing them - # would add a dangling entry to the hub's `all_requirements` / - # `all_whl_requirements` and create an alias to a subpackage that - # doesn't exist. This mirrors the requirements path, which also - # gates `is_exposed`. - is_exposed = bool(info["resolved_srcs"]), + # Only expose packages that resolved to a source on every target + # platform, matching `_parse_requirements_from_req_files`. Workspace + # root members and platform-conditional packages are omitted from + # `all_requirements` / `all_whl_requirements` so building those + # target lists succeeds on every platform. + is_exposed = len(resolved_platforms) == len(all_platforms), is_multiple_versions = len(versions) > 1, index_url = info["index_url"], srcs = info["resolved_srcs"], @@ -318,6 +312,104 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p logger.debug(lambda: "Parsed {} packages from uv.lock".format(len(ret))) return ret +def _uv_lock_reachable_platforms(uv_lock, all_platforms, platforms): + """Compute reachable target platforms for each package index in `uv.lock`. + + A package in `uv.lock` may have no `resolution-markers` of its own when only + a single version is locked, yet only be reachable via marker-guarded + dependency edges (for example, `keyring -> secretstorage` with + `marker = "sys_platform == 'linux'"` -> `cryptography` -> `cffi`). + Evaluating edge markers per platform from root packages ensures transitive + dependencies are restricted to platforms where they are actually reachable. + + Args: + uv_lock: {type}`dict` Decoded `uv.lock` contents. + all_platforms: {type}`list[str]` Target platform names. + platforms: {type}`dict[str, struct]` Platform name to platform info. + + Returns: + {type}`list[list[str]]` Parallel to `uv_lock["package"]`, listing the + subset of `all_platforms` on which each package entry is reachable. + """ + packages = uv_lock.get("package", []) + num_pkgs = len(packages) + name_to_indices = {} + pkg_edges = [] + has_incoming = {} + + for idx, pkg in enumerate(packages): + norm_name = normalize_name(pkg["name"]) + name_to_indices.setdefault(norm_name, []).append(idx) + + edges = [] + for dep in pkg.get("dependencies", []): + edges.append(dep) + for dep_list in pkg.get("optional-dependencies", {}).values(): + for dep in dep_list: + edges.append(dep) + for dep_list in pkg.get("dev-dependencies", {}).values(): + for dep in dep_list: + edges.append(dep) + pkg_edges.append(edges) + + for dep in edges: + dep_norm = normalize_name(dep["name"]) + if dep_norm != norm_name: + has_incoming[dep_norm] = True + + roots = [ + idx + for idx, pkg in enumerate(packages) + if normalize_name(pkg["name"]) not in has_incoming + ] + if not roots: + roots = list(range(num_pkgs)) + + reachable = [[] for _ in range(num_pkgs)] + for p in all_platforms: + platform = platforms.get(p) + if not platform: + continue + env = platform.env + + active = [] + for pkg in packages: + markers = pkg.get("resolution-markers", []) + if markers: + active.append(evaluate(" or ".join(markers), env = env)) + else: + active.append(True) + + visited = {} + queue = [] + for root_idx in roots: + if active[root_idx] and root_idx not in visited: + visited[root_idx] = True + queue.append(root_idx) + + for step in range(num_pkgs): + if step >= len(queue): + break + curr = queue[step] + for dep in pkg_edges[curr]: + dep_marker = dep.get("marker", "") + if dep_marker and not evaluate(dep_marker, env = env): + continue + dep_norm = normalize_name(dep["name"]) + dep_ver = dep.get("version", "") + for target_idx in name_to_indices.get(dep_norm, []): + if not active[target_idx] or target_idx in visited: + continue + if dep_ver and packages[target_idx]["version"] != dep_ver: + continue + visited[target_idx] = True + queue.append(target_idx) + + for idx in visited: + reachable[idx].append(p) + + return reachable + def _uv_lock_git_requirement(git_source): """Turn a uv.lock git source into a pip direct reference. @@ -376,6 +468,43 @@ def _parse_uv_lock_hash(hash_str): algo, _, hex_digest = hash_str.partition(":") return hash.digest(algo, hex_digest) +def _uv_lock_extras_map(uv_lock): + """Compute extras for each package from uv.lock data.""" + extras_map = {} + for pkg in uv_lock.get("package", []): + pkg_name = pkg.get("name", "") + + for extra in pkg.get("provides-extras", pkg.get("extras", [])): + _add_extras(extras_map, pkg_name, [extra]) + + opt_deps = pkg.get("optional-dependencies", {}) + if opt_deps: + _add_extras(extras_map, pkg_name, sorted(opt_deps.keys())) + + for dep in pkg.get("dependencies", []): + dep_name = dep.get("name", "") + dep_extras_raw = dep.get("extra", []) + dep_extras = [dep_extras_raw] if type(dep_extras_raw) == "string" else dep_extras_raw + if dep_extras and dep_name != pkg_name: + _add_extras(extras_map, dep_name, dep_extras) + + metadata = pkg.get("metadata", {}) + for rd in metadata.get("requires-dist", []): + rd_name = rd.get("name", "") + rd_extras_raw = rd.get("extras", []) + rd_extras = [rd_extras_raw] if type(rd_extras_raw) == "string" else rd_extras_raw + if rd_extras and rd_name != pkg_name: + _add_extras(extras_map, rd_name, rd_extras) + + return extras_map + +def _add_extras(extras_map, pkg_name, extras): + existing = extras_map.get(pkg_name, []) + for extra in extras: + if extra not in existing: + existing.append(extra) + extras_map[pkg_name] = sorted(existing) + def _parse_requirements_from_req_files( ctx, *, diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..1cc2deb482 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -46,6 +46,8 @@ def _repr_dict(*, value_repr = repr, **kwargs): return {k: value_repr(v) for k, v in kwargs.items() if v} def _repr_config_setting(alias): + if type(alias) == type(""): + return repr(alias) if alias.target_platforms: return render.call( "whl_config_setting", diff --git a/python/private/pypi/uv_lock.bzl b/python/private/pypi/uv_lock.bzl new file mode 100644 index 0000000000..8c163348fc --- /dev/null +++ b/python/private/pypi/uv_lock.bzl @@ -0,0 +1,75 @@ +"""A WORKSPACE macro for installing PyPI dependencies from a uv.lock file.""" + +load("@bazel_skylib//lib:types.bzl", "types") +load("//python:versions.bzl", "MINOR_MAPPING") +load(":uv_lock_repo.bzl", "uv_lock_repo") + +def uv_lock( + name, + uv_lock, + default_version = None, + python_versions = None, + python_interpreter_target = None, + minor_mapping = None, + **kwargs): + """Accepts a resolved `uv.lock` file and installs the dependencies listed within. + + Supports both single-version and multi-version Python setups in `WORKSPACE` + using a unified multi-platform hub repository backed by per-wheel `whl_library` + repositories. + + :::{versionadded} VERSION_NEXT_FEATURE + ::: + + Args: + name: {type}`str` The name of the generated PyPI hub repository. + uv_lock: {type}`Label` A resolved `uv.lock` file. + default_version: {type}`str | None` Optional default Python version when + configuring multiple Python versions. + python_versions: {type}`list[str] | None` Optional list of Python versions + when `python_interpreter_target` is a dict. Defaults to the keys of + `python_interpreter_target`. + python_interpreter_target: {type}`Label | str | dict[str, Label] | None` + Either a single target label for the host Python interpreter, or a + dict mapping Python `X.Y` versions to host interpreter targets. + minor_mapping: {type}`dict[str, str] | None` Optional mapping from `X.Y` + to `X.Y.Z` Python versions for multi-version hub aliases. Defaults + to `MINOR_MAPPING` from `//python:versions.bzl`. + **kwargs: Additional repository rule arguments forwarded to the + underlying `uv_lock_repo` repository rule. + """ + _uv_lock_repo = kwargs.pop("_uv_lock_repo", uv_lock_repo) + + if not types.is_dict(python_interpreter_target): + if python_interpreter_target != None: + kwargs["python_interpreter_target"] = python_interpreter_target + _uv_lock_repo( + name = name, + uv_lock = uv_lock, + **kwargs + ) + return + + python_versions = python_versions or sorted(python_interpreter_target.keys()) + minor_mapping = minor_mapping or MINOR_MAPPING + for python_version in python_versions: + if python_version not in python_interpreter_target: + fail( + "Missing python_interpreter_target for Python version %s in '%s'" % ( + python_version, + name, + ), + ) + + _uv_lock_repo( + name = name, + uv_lock = uv_lock, + default_version = default_version or "", + python_versions = python_versions, + python_interpreter_targets = { + k: str(v) + for k, v in python_interpreter_target.items() + }, + minor_mapping = minor_mapping, + **kwargs + ) diff --git a/python/private/pypi/uv_lock_repo.bzl b/python/private/pypi/uv_lock_repo.bzl new file mode 100644 index 0000000000..77214941e7 --- /dev/null +++ b/python/private/pypi/uv_lock_repo.bzl @@ -0,0 +1,539 @@ +"""A WORKSPACE repository rule for installing PyPI dependencies from uv.lock.""" + +load("@bazel_skylib//lib:sets.bzl", "sets") +load("@toml.bzl", "toml") +load("//python/private:full_version.bzl", "full_version") +load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") +load("//python/private:text_util.bzl", "render") +load(":attrs.bzl", COMMON_ATTRS = "ATTRS") +load(":parse_requirements.bzl", "parse_requirements") +load(":pep508_env.bzl", "env") +load(":pip_repository.bzl", "use_isolated") +load(":pip_repository_attrs.bzl", "ATTRS") +load(":pypi_repo_utils.bzl", "pypi_repo_utils") +load(":render_pkg_aliases.bzl", "render_pkg_aliases") +load(":whl_repo_name.bzl", "pypi_repo_name", "whl_repo_name") + +_DEFAULT_PLATFORMS = { + "linux_aarch64": struct( + arch_name = "aarch64", + constraints = ["@platforms//os:linux", "@platforms//cpu:aarch64"], + os_name = "linux", + whl_platform_tags = ["linux_aarch64", "manylinux_*_aarch64"], + ), + "linux_x86_64": struct( + arch_name = "x86_64", + constraints = ["@platforms//os:linux", "@platforms//cpu:x86_64"], + os_name = "linux", + whl_platform_tags = ["linux_x86_64", "manylinux_*_x86_64"], + ), + "osx_aarch64": struct( + arch_name = "aarch64", + constraints = ["@platforms//os:osx", "@platforms//cpu:aarch64"], + os_name = "osx", + whl_platform_tags = ["macosx_*_universal2", "macosx_*_arm64"], + ), + "osx_x86_64": struct( + arch_name = "x86_64", + constraints = ["@platforms//os:osx", "@platforms//cpu:x86_64"], + os_name = "osx", + whl_platform_tags = ["macosx_*_universal2", "macosx_*_x86_64"], + ), + "windows_x86_64": struct( + arch_name = "x86_64", + constraints = ["@platforms//os:windows", "@platforms//cpu:x86_64"], + os_name = "windows", + whl_platform_tags = ["win_amd64"], + ), +} + +_BUILD_FILE_TEMPLATE = """\ +package(default_visibility = ["//visibility:public"]) + +# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it +exports_files({exports}) +""" + +_REQUIREMENTS_BZL_TEMPLATE = """\ +\"\"\"Starlark representation of locked requirements. + +@generated by rules_python uv_lock. +\"\"\" + +load("@rules_python//python:pip.bzl", "pip_utils") +load("@rules_python//python/pip_install:pip_repository.bzl", "whl_library") + +all_requirements = {all_requirements} + +all_whl_requirements_by_package = {all_whl_requirements_by_package} + +all_whl_requirements = all_whl_requirements_by_package.values() + +all_data_requirements = {all_data_requirements} + +_packages = {packages} + +_config = {config} + +def _clean_name(name): + return pip_utils.normalize_name(name) + +def requirement(name): + return "@{hub}//{{}}:pkg".format(_clean_name(name)) + +def whl_requirement(name): + return "@{hub}//{{}}:whl".format(_clean_name(name)) + +def data_requirement(name): + return "@{hub}//{{}}:data".format(_clean_name(name)) + +def dist_info_requirement(name): + return "@{hub}//{{}}:dist_info".format(_clean_name(name)) + +def entry_point(pkg, script = None): + if not script: + script = pkg + return "@{hub}//{{}}:rules_python_wheel_entry_point_{{}}".format( + _clean_name(pkg), + script, + ) + +def install_deps(**whl_library_kwargs): + whl_config = dict(_config) + whl_config.update(whl_library_kwargs) + for name, kwargs in _packages.items(): + whl_library( + name = name, + **(whl_config | kwargs) + ) +""" + +def _get_python_interpreter_attr(rctx): + if rctx.attr.python_interpreter: + return rctx.attr.python_interpreter + if "win" in rctx.os.name: + return "python.exe" + return "python3" + +def _parse_major_minor(version_str): + parts = version_str.split(".") + major = parts[0] if len(parts) > 0 else "3" + minor = parts[1] if len(parts) > 1 else "11" + return major, minor + +def _platforms_for_python_version(python_version): + major, minor = _parse_major_minor(python_version) + abi_tags = ["none", "abi3", "cp{}{}".format(major, minor)] + return { + plat_name: struct( + env = env( + os = meta.os_name, + arch = meta.arch_name, + python_version = python_version, + ), + whl_abi_tags = abi_tags, + whl_platform_tags = ["any"] + meta.whl_platform_tags, + ) + for plat_name, meta in _DEFAULT_PLATFORMS.items() + } + +def _render_uv_lock_config_build(is_multi_version, version_entries, rules_python_repo): + lines = [ + 'package(default_visibility = ["//:__subpackages__"])', + ] + for plat_name, meta in sorted(_DEFAULT_PLATFORMS.items()): + lines.extend([ + "", + "config_setting(", + ' name = "is_{}",'.format(plat_name), + " constraint_values = {},".format(repr(meta.constraints)), + ")", + ]) + if is_multi_version: + for entry in version_entries: + py_nodot = entry.py_nodot + for plat_name, meta in sorted(_DEFAULT_PLATFORMS.items()): + lines.extend([ + "", + "config_setting(", + ' name = "is_cp{}_{}",'.format(py_nodot, plat_name), + " constraint_values = {},".format(repr(meta.constraints)), + " flag_values = {", + ' "@{rules_python}//python/config_settings:python_version_major_minor": "{py_key}",'.format( + rules_python = rules_python_repo, + py_key = entry.key, + ), + " },", + ")", + ]) + lines.append("") + return "\n".join(lines) + +def _build_version_entries(rctx): + if rctx.attr.python_interpreter_targets: + versions = rctx.attr.python_versions or sorted(rctx.attr.python_interpreter_targets.keys()) + entries = [] + for py_ver in versions: + if py_ver not in rctx.attr.python_interpreter_targets: + fail("Missing python_interpreter_target for Python version {} in '{}'".format( + py_ver, + rctx.attr.name, + )) + full_ver = full_version( + version = py_ver, + minor_mapping = rctx.attr.minor_mapping, + ) + major, minor = _parse_major_minor(full_ver) + entries.append(struct( + key = py_ver, + py_nodot = "{}{}".format(major, minor), + full_version = full_ver, + interpreter_target = rctx.attr.python_interpreter_targets[py_ver], + )) + return entries, True + + py_exec = pypi_repo_utils.resolve_python_interpreter( + rctx, + python_interpreter = rctx.attr.python_interpreter, + python_interpreter_target = rctx.attr.python_interpreter_target, + ) + res = repo_utils.execute_checked( + rctx, + op = "GetPythonVersion", + arguments = [py_exec, "-c", "import sys; print(sys.version.split()[0])"], + ) + detected_version = res.stdout.strip().splitlines()[-1] + major, minor = _parse_major_minor(detected_version) + interp_target = str(rctx.attr.python_interpreter_target) if rctx.attr.python_interpreter_target else "" + return [ + struct( + key = "{}.{}".format(major, minor), + py_nodot = "{}{}".format(major, minor), + full_version = detected_version, + interpreter_target = interp_target, + ), + ], False + +def build_uv_lock_hub_packages( + parsed_by_version, + *, + hub_name, + is_multi_version = False, + default_version = "", + annotation_labels = None, + requirement_cycles = None): + """Build per-wheel `whl_library` kwargs and hub `pkg_aliases` maps. + + Args: + parsed_by_version: {type}`list[tuple[struct, list[struct]]]` Pairs of + `(version_entry, parsed_whls)` where each `version_entry` has fields + `key`, `py_nodot`, `full_version`, and `interpreter_target`. + hub_name: {type}`str` Name of the generated PyPI hub repository. + is_multi_version: {type}`bool` Whether multiple Python versions are configured. + default_version: {type}`str` Optional default Python version (`X.Y`). + annotation_labels: {type}`dict[str, str] | None` Optional map from normalized + package name to hub annotation file label. + requirement_cycles: {type}`dict[str, list[str]] | None` Optional map from + normalized cycle group name to normalized member package names. + + Returns: + `struct(packages = dict, aliases = dict, bzl_packages = list[str])`. + """ + annotation_labels = annotation_labels or {} + requirement_cycles = requirement_cycles or {} + group_by_whl = { + whl_name: group_name + for group_name, group_whls in requirement_cycles.items() + for whl_name in group_whls + } + packages = {} + pkg_alias_maps = {} + exposed_pkgs = {} + all_target_platforms = sorted(_DEFAULT_PLATFORMS.keys()) + + for entry, parsed_whls in parsed_by_version: + for whl in parsed_whls: + if not whl.srcs: + continue + if whl.is_exposed: + exposed_pkgs[whl.name] = None + alias_map = pkg_alias_maps.setdefault(whl.name, {}) + for src in whl.srcs: + if src.filename: + repo_suffix = whl_repo_name(src.filename, src.digest) + else: + repo_suffix = pypi_repo_name(whl.name) + if is_multi_version: + spoke_name = "{}_{}_{}".format(hub_name, entry.py_nodot, repo_suffix) + else: + spoke_name = "{}_{}".format(hub_name, repo_suffix) + + if spoke_name not in packages: + kwargs = {} + if src.filename and src.filename.endswith(".whl") and src.url: + kwargs["filename"] = src.filename + kwargs["requirement"] = src.requirement_line + kwargs["urls"] = [src.url] + if src.digest: + algo, _, hex_digest = src.digest.partition(":") + if algo == "sha256" and hex_digest: + kwargs["sha256"] = hex_digest + else: + req_line = src.requirement_line + if src.digest: + req_line = "{} --hash={}".format(req_line, src.digest) + kwargs["requirement"] = req_line + + if entry.interpreter_target: + kwargs["python_interpreter_target"] = entry.interpreter_target + ann_label = annotation_labels.get(whl.name) + if ann_label: + kwargs["annotation"] = ann_label + group_name = group_by_whl.get(whl.name) + if group_name: + kwargs["group_name"] = group_name + kwargs["group_deps"] = requirement_cycles[group_name] + packages[spoke_name] = kwargs + + for plat in src.target_platforms: + if is_multi_version: + cfg_label = "//_config:is_cp{}_{}".format(entry.py_nodot, plat) + alias_map.setdefault(cfg_label, spoke_name) + if default_version and entry.key == default_version: + alias_map.setdefault("//_config:is_{}".format(plat), spoke_name) + else: + alias_map.setdefault("//_config:is_{}".format(plat), spoke_name) + + aliases = {} + for pkg_name, alias_map in sorted(pkg_alias_maps.items()): + if not alias_map: + continue + unique_repos = {repo: None for repo in alias_map.values()} + if not is_multi_version and len(unique_repos) == 1 and len(alias_map) == len(all_target_platforms): + aliases[pkg_name] = unique_repos.keys()[0] + else: + aliases[pkg_name] = alias_map + + return struct( + packages = packages, + aliases = aliases, + bzl_packages = sorted(exposed_pkgs.keys()), + ) + +def render_uv_lock_repository( + rctx, + *, + uv_lock, + toml_decode): + """Render a multi-platform WORKSPACE PyPI hub repository from `uv.lock`. + + Args: + rctx: {type}`repository_ctx` Repository rule context. + uv_lock: {type}`Label` Label of the `uv.lock` file to parse. + toml_decode: {type}`callable` Function decoding TOML text into a dict. + """ + logger = repo_utils.logger(rctx) + hub_name = rctx.attr.name + rules_python_repo = rctx.attr._rules_python_workspace.repo_name + version_entries, is_multi_version = _build_version_entries(rctx) + default_ver = rctx.attr.default_version + + # Normalize and write any per-package annotations into the hub repository. + annotation_labels = {} + exported_files = ["requirements.bzl"] + extra_hub_aliases = { + normalize_name(k): list(v) + for k, v in rctx.attr.extra_hub_aliases.items() + } + for pkg, ann_json in rctx.attr.annotations.items(): + norm_pkg = normalize_name(pkg) + ann_filename = "{}.annotation.json".format(norm_pkg) + decoded = json.decode(ann_json) + rctx.file(ann_filename, json.encode_indent(decoded)) + exported_files.append(ann_filename) + annotation_labels[norm_pkg] = "@{}//:{}".format(hub_name, ann_filename) + for exe_target in decoded.get("copy_executables", {}).keys(): + existing = extra_hub_aliases.setdefault(norm_pkg, []) + if exe_target not in existing: + existing.append(exe_target) + + parsed_by_version = [] + for entry in version_entries: + parsed_whls = parse_requirements( + rctx, + extra_pip_args = rctx.attr.extra_pip_args, + platforms = _platforms_for_python_version(entry.full_version), + uv_lock = uv_lock, + toml_decode = toml_decode, + logger = logger, + ) + parsed_by_version.append((entry, parsed_whls)) + + # Normalize and validate requirement cycles before building package kwargs. + raw_cycles = { + name: sorted(sets.to_list(sets.make(deps))) + for name, deps in rctx.attr.experimental_requirement_cycles.items() + } + for i in range(len(raw_cycles)): + left_group = raw_cycles.keys()[i] + left_deps = raw_cycles.values()[i] + for j in range(len(raw_cycles) - (i + 1)): + right_deps = raw_cycles.values()[1 + i + j] + right_group = raw_cycles.keys()[1 + i + j] + for d in left_deps: + if d in right_deps: + fail("Error: Requirement %s cannot be repeated between cycles %s and %s; please merge the cycles." % (d, left_group, right_group)) + + present_pkgs = { + whl.name: None + for _, parsed_whls in parsed_by_version + for whl in parsed_whls + if whl.srcs + } + requirement_cycles = { + normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in present_pkgs]) + for name, group in raw_cycles.items() + } + + built = build_uv_lock_hub_packages( + parsed_by_version, + hub_name = hub_name, + is_multi_version = is_multi_version, + default_version = default_ver, + annotation_labels = annotation_labels, + requirement_cycles = requirement_cycles, + ) + packages = built.packages + aliases = built.aliases + bzl_packages = built.bzl_packages + all_pkg_names = sorted(aliases.keys()) + + rctx.file( + "BUILD.bazel", + _BUILD_FILE_TEMPLATE.format(exports = repr(sorted(exported_files))), + ) + rctx.file( + "_config/BUILD.bazel", + _render_uv_lock_config_build( + is_multi_version = is_multi_version, + version_entries = version_entries, + rules_python_repo = rules_python_repo, + ), + ) + rctx.template( + "config.bzl", + rctx.attr._config_template, + substitutions = { + "%%PACKAGES%%": render.dict({p: None for p in all_pkg_names}), + }, + ) + + alias_files = render_pkg_aliases( + aliases = aliases, + requirement_cycles = requirement_cycles, + extra_hub_aliases = extra_hub_aliases, + ) + for path, contents in alias_files.items(): + rctx.file(path, contents) + + config = { + "dep_template": "@{}//{{name}}:{{target}}".format(hub_name), + "download_only": rctx.attr.download_only, + "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs, + "environment": rctx.attr.environment, + "envsubst": rctx.attr.envsubst, + "extra_pip_args": rctx.attr.extra_pip_args, + "isolated": use_isolated(rctx, rctx.attr), + "pip_data_exclude": rctx.attr.pip_data_exclude, + "python_interpreter": _get_python_interpreter_attr(rctx), + "quiet": rctx.attr.quiet, + "repo": hub_name, + "timeout": rctx.attr.timeout, + } + + macro_tmpl = "@{}//{{}}:{{}}".format(hub_name) + rctx.file( + "requirements.bzl", + _REQUIREMENTS_BZL_TEMPLATE.format( + all_data_requirements = render.list([ + macro_tmpl.format(p, "data") + for p in bzl_packages + ]), + all_requirements = render.list([ + macro_tmpl.format(p, "pkg") + for p in bzl_packages + ]), + all_whl_requirements_by_package = render.dict({ + p: macro_tmpl.format(p, "whl") + for p in bzl_packages + }), + config = render.dict(config), + hub = hub_name, + packages = render.dict(packages), + ), + ) + +def _uv_lock_repo_impl(rctx): + render_uv_lock_repository( + rctx, + uv_lock = rctx.attr.uv_lock, + toml_decode = toml.decode, + ) + +uv_lock_repo = repository_rule( + attrs = dict( + annotations = attr.string_dict( + doc = """ +Optional annotations to apply to packages. Keys should be package names, with +capitalization matching the input lockfile, and values should be generated +using the `package_annotation` macro. +""", + ), + default_version = attr.string( + doc = """ +Optional default Python version (`X.Y`) when `python_interpreter_targets` is set. +""", + ), + experimental_requirement_cycles = ATTRS["experimental_requirement_cycles"], + extra_hub_aliases = ATTRS["extra_hub_aliases"], + minor_mapping = attr.string_dict( + doc = """ +Mapping from `X.Y` to `X.Y.Z` Python versions for multi-version hub aliases. +""", + ), + python_interpreter_targets = attr.string_dict( + doc = """ +Optional mapping from Python `X.Y` versions to host interpreter target label strings. +""", + ), + python_versions = attr.string_list( + doc = """ +Optional ordered list of Python `X.Y` versions when configuring multiple versions. +""", + ), + use_hub_alias_dependencies = ATTRS["use_hub_alias_dependencies"], + uv_lock = attr.label( + allow_single_file = True, + mandatory = True, + doc = """ +A resolved `uv.lock` file containing the transitive set of dependencies. +""", + ), + _config_template = attr.label( + default = ":config.bzl.tmpl", + ), + _rules_python_workspace = attr.label( + default = Label("//:WORKSPACE"), + ), + **COMMON_ATTRS + ), + doc = """ +Accepts a resolved `uv.lock` file and installs the dependencies listed within. +""", + implementation = _uv_lock_repo_impl, + environ = [ + "RULES_PYTHON_PIP_ISOLATED", + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/uv/private/BUILD.bazel b/python/uv/private/BUILD.bazel index e4e652ae78..1eb5c46a52 100644 --- a/python/uv/private/BUILD.bazel +++ b/python/uv/private/BUILD.bazel @@ -146,16 +146,6 @@ bzl_library( deps = ["//python/private:text_util"], ) -# keep -bzl_library( - name = "uv_lock_to_requirements", - srcs = ["uv_lock_to_requirements.bzl"], - visibility = [ - "//python/private:__subpackages__", - "//python/uv:__subpackages__", - ], -) - bzl_library( name = "uv_toolchain_config", srcs = ["uv_toolchain_config.bzl"], diff --git a/python/uv/private/uv_lock_to_requirements.bzl b/python/uv/private/uv_lock_to_requirements.bzl deleted file mode 100644 index 73d912bbfd..0000000000 --- a/python/uv/private/uv_lock_to_requirements.bzl +++ /dev/null @@ -1,139 +0,0 @@ -"""Convert a parsed uv.lock to requirements.txt format.""" - -def uv_lock_extras_map(uv_lock): - """Compute extras for each package from uv.lock data. - - Args: - uv_lock: a decoded JSON struct from a uv.lock file. - - Returns: - A dict of {package_name: [extra1, extra2, ...]} for packages with extras. - """ - extras_map = {} - for pkg in uv_lock.get("package", []): - pkg_name = pkg.get("name", "") - - for extra in pkg.get("provides-extras", pkg.get("extras", [])): - _add_extras(extras_map, pkg_name, [extra]) - - opt_deps = pkg.get("optional-dependencies", {}) - if opt_deps: - _add_extras(extras_map, pkg_name, _sorted(opt_deps.keys())) - - deps = pkg.get("dependencies", []) - for dep in deps: - dep_name = dep.get("name", "") - dep_extras_raw = dep.get("extra", []) - dep_extras = [dep_extras_raw] if type(dep_extras_raw) == "string" else dep_extras_raw - if dep_extras and dep_name != pkg_name: - _add_extras(extras_map, dep_name, dep_extras) - - metadata = pkg.get("metadata", {}) - for rd in metadata.get("requires-dist", []): - rd_name = rd.get("name", "") - rd_extras_raw = rd.get("extras", []) - rd_extras = [rd_extras_raw] if type(rd_extras_raw) == "string" else rd_extras_raw - if rd_extras and rd_name != pkg_name: - _add_extras(extras_map, rd_name, rd_extras) - - return extras_map - -def uv_lock_to_requirements(uv_lock): - """Convert a parsed uv.lock JSON struct to a requirements.txt formatted string. - - Args: - uv_lock: a decoded JSON struct from a uv.lock file. - - Returns: - A requirements.txt formatted string. - """ - packages = uv_lock.get("package", []) - extras_map = uv_lock_extras_map(uv_lock) - - dependents = {} - for pkg in packages: - pkg_name = pkg.get("name", "") - deps = pkg.get("dependencies", []) - for dep in deps: - dep_name = dep.get("name", "") - if dep_name != pkg_name: - _add_dependent(dependents, dep_name, pkg_name) - opt_deps = pkg.get("optional-dependencies", {}) - for _extra, deps in opt_deps.items(): - for dep in deps: - dep_name = dep.get("name", "") - if dep_name != pkg_name: - _add_dependent(dependents, dep_name, pkg_name) - - lines = [] - for pkg in packages: - source = pkg.get("source", {}) - if not source.get("registry"): - continue - - pkg_name = pkg.get("name", "") - version = pkg.get("version", "") - - markers = pkg.get("resolution-markers", []) - hashes = _collect_hashes(pkg) - - pkg_extras = extras_map.get(pkg_name, []) - if pkg_extras: - req = "{}[{}]=={}".format(pkg_name, ",".join(pkg_extras), version) - else: - req = "{}=={}".format(pkg_name, version) - if markers: - req += " ; " + " or ".join(markers) - - if hashes: - req += " \\" - lines.append(req) - - _emit_hashes(lines, hashes) - - dep_vias = _sorted(dependents.get(pkg_name, [])) - if dep_vias: - if len(dep_vias) == 1: - lines.append(" # via " + dep_vias[0]) - else: - lines.append(" # via") - for via in dep_vias: - lines.append(" # " + via) - - lines.append("") - - return "\n".join(lines) - -def _add_dependent(dependents, dep_name, dependent_name): - if dep_name not in dependents: - dependents[dep_name] = [] - if dependent_name not in dependents[dep_name]: - dependents[dep_name].append(dependent_name) - -def _add_extras(extras_map, pkg_name, extras): - existing = extras_map.get(pkg_name, []) - for extra in extras: - if extra not in existing: - existing.append(extra) - extras_map[pkg_name] = _sorted(existing) - -def _collect_hashes(pkg): - hashes = [] - for wheel in pkg.get("wheels", []): - whash = wheel.get("hash", "") - if whash.startswith("sha256:"): - hashes.append(whash[len("sha256:"):]) - sdist = pkg.get("sdist") - if sdist: - shash = sdist.get("hash", "") - if shash.startswith("sha256:"): - hashes.append(shash[len("sha256:"):]) - return _sorted(hashes) - -def _sorted(items): - return sorted(items) - -def _emit_hashes(lines, hashes): - for i, h in enumerate(hashes): - suffix = " \\" if i < len(hashes) - 1 else "" - lines.append(" --hash=sha256:{}{}".format(h, suffix)) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 733abe42d4..0520e99a3c 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -100,6 +100,7 @@ foo==0.0.1 --hash=sha512:deadbeef foo[extra]==0.0.2 --hash=sha256:deadbeef bar==0.0.1 --hash=sha256:deadb00f """, + "uv_lock_edge_marker_reachability": """{"package":[{"dependencies":[{"name":"keyring"}],"name":"root-pkg","source":{"virtual":"."},"version":"0.0.0"},{"dependencies":[{"marker":"sys_platform == 'linux'","name":"secretstorage"}],"name":"keyring","source":{"registry":"https://pypi.org/simple"},"version":"25.7.0","wheels":[{"hash":"sha256:aaa","url":"https://files.pythonhosted.org/packages/keyring-25.7.0-py3-none-any.whl"}]},{"dependencies":[{"name":"cryptography"}],"name":"secretstorage","source":{"registry":"https://pypi.org/simple"},"version":"3.3.3","wheels":[{"hash":"sha256:bbb","url":"https://files.pythonhosted.org/packages/secretstorage-3.3.3-py3-none-any.whl"}]},{"name":"cryptography","sdist":{"hash":"sha256:ddd","url":"https://files.pythonhosted.org/packages/cryptography-50.0.2.tar.gz"},"source":{"registry":"https://pypi.org/simple"},"version":"50.0.2","wheels":[{"hash":"sha256:ccc","url":"https://files.pythonhosted.org/packages/cryptography-50.0.2-cp39-abi3-manylinux_2_17_x86_64.whl"}]}]}""", "uv_lock_empty": """{"package":[]}""", "uv_lock_foo": """{"package":[{"dependencies":[{"extra":"extra","name":"bar"}],"name":"foo","source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/foo-0.0.1-py3-none-any.whl"}]}]}""", "uv_lock_foo_bar": """{"package":[{"name":"bar","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"sdist":{"hash":"sha256:deadb00f","url":"https://files.pythonhosted.org/packages/bar-0.0.1.tar.gz"}},{"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/foo-0.0.1-py3-none-any.whl"}]}]}""", @@ -1745,6 +1746,94 @@ def _test_uv_lock_requires_dist_extras(env): _tests.append(_test_uv_lock_requires_dist_extras) +def _test_uv_lock_edge_marker_reachability(env): + """Test that transitive deps behind platform-specific edge markers are restricted to reachable platforms.""" + got = parse_requirements( + uv_lock = "uv_lock_edge_marker_reachability", + platforms = { + "linux_x86_64": struct( + env = pep508_env(python_version = "3.11.0", os = "linux", arch = "x86_64"), + whl_abi_tags = ["abi3", "none"], + whl_platform_tags = ["manylinux_2_17_x86_64", "any"], + ), + "osx_aarch64": struct( + env = pep508_env(python_version = "3.11.0", os = "osx", arch = "aarch64"), + whl_abi_tags = ["abi3", "none"], + whl_platform_tags = ["macosx_11_0_arm64", "any"], + ), + "windows_x86_64": struct( + env = pep508_env(python_version = "3.11.0", os = "windows", arch = "x86_64"), + whl_abi_tags = ["abi3", "none"], + whl_platform_tags = ["win_amd64", "any"], + ), + }, + ) + env.expect.that_collection(got).contains_exactly([ + struct( + name = "cryptography", + index_url = "https://pypi.org/simple/cryptography", + is_exposed = False, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "cryptography", + extra_pip_args = [], + requirement_line = "cryptography==50.0.2", + target_platforms = ["linux_x86_64"], + filename = "cryptography-50.0.2-cp39-abi3-manylinux_2_17_x86_64.whl", + digest = "sha256:ccc", + url = "https://files.pythonhosted.org/packages/cryptography-50.0.2-cp39-abi3-manylinux_2_17_x86_64.whl", + yanked = None, + ), + ], + ), + struct( + name = "keyring", + index_url = "https://pypi.org/simple/keyring", + is_exposed = True, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "keyring", + extra_pip_args = [], + requirement_line = "keyring==25.7.0", + target_platforms = ["linux_x86_64", "osx_aarch64", "windows_x86_64"], + filename = "keyring-25.7.0-py3-none-any.whl", + digest = "sha256:aaa", + url = "https://files.pythonhosted.org/packages/keyring-25.7.0-py3-none-any.whl", + yanked = None, + ), + ], + ), + struct( + name = "root_pkg", + index_url = "", + is_exposed = False, + is_multiple_versions = False, + srcs = [], + ), + struct( + name = "secretstorage", + index_url = "https://pypi.org/simple/secretstorage", + is_exposed = False, + is_multiple_versions = False, + srcs = [ + struct( + distribution = "secretstorage", + extra_pip_args = [], + requirement_line = "secretstorage==3.3.3", + target_platforms = ["linux_x86_64"], + filename = "secretstorage-3.3.3-py3-none-any.whl", + digest = "sha256:bbb", + url = "https://files.pythonhosted.org/packages/secretstorage-3.3.3-py3-none-any.whl", + yanked = None, + ), + ], + ), + ]) + +_tests.append(_test_uv_lock_edge_marker_reachability) + def parse_requirements_test_suite(name): """Create the test suite. diff --git a/tests/pypi/uv_lock/BUILD.bazel b/tests/pypi/uv_lock/BUILD.bazel new file mode 100644 index 0000000000..c41a97f9c9 --- /dev/null +++ b/tests/pypi/uv_lock/BUILD.bazel @@ -0,0 +1,3 @@ +load(":uv_lock_tests.bzl", "uv_lock_test_suite") + +uv_lock_test_suite(name = "uv_lock_tests") diff --git a/tests/pypi/uv_lock/uv_lock_tests.bzl b/tests/pypi/uv_lock/uv_lock_tests.bzl new file mode 100644 index 0000000000..2a6ee30b99 --- /dev/null +++ b/tests/pypi/uv_lock/uv_lock_tests.bzl @@ -0,0 +1,261 @@ +"""Unit tests for the WORKSPACE uv_lock macro and hub package builder.""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python:pip.bzl", "uv_lock") +load("//python/private/pypi:uv_lock_repo.bzl", "build_uv_lock_hub_packages") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") + +_tests = [] + +def _test_single_version(env): + mock_repo = mocks.repo() + + uv_lock( + name = "pypi", + python_interpreter_target = "@python3_14_host//:python", + uv_lock = "//:uv.lock", + extra_pip_args = ["--retries=5"], + _uv_lock_repo = mock_repo.rule, + ) + + env.expect.that_collection(mock_repo.calls).contains_exactly([ + { + "extra_pip_args": ["--retries=5"], + "name": "pypi", + "python_interpreter_target": "@python3_14_host//:python", + "uv_lock": "//:uv.lock", + }, + ]) + +_tests.append(_test_single_version) + +def _test_multi_version(env): + mock_repo = mocks.repo() + + uv_lock( + name = "pypi", + default_version = "3.10", + minor_mapping = { + "3.10": "3.10.20", + "3.11": "3.11.15", + }, + python_interpreter_target = { + "3.10": "@python_3_10_host//:python", + "3.11": "@python_3_11_host//:python", + }, + uv_lock = "//requirements:uv.lock", + _uv_lock_repo = mock_repo.rule, + ) + + env.expect.that_collection(mock_repo.calls).contains_exactly([ + { + "default_version": "3.10", + "minor_mapping": { + "3.10": "3.10.20", + "3.11": "3.11.15", + }, + "name": "pypi", + "python_interpreter_targets": { + "3.10": "@python_3_10_host//:python", + "3.11": "@python_3_11_host//:python", + }, + "python_versions": ["3.10", "3.11"], + "uv_lock": "//requirements:uv.lock", + }, + ]) + +_tests.append(_test_multi_version) + +def _test_build_uv_lock_hub_packages_single_version(env): + """Verify per-wheel `whl_library` kwargs and hub alias maps in single-version mode.""" + certifi_hex = "035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae" + linux_hex = "1111111111111111111111111111111111111111111111111111111111111111" + osx_hex = "2222222222222222222222222222222222222222222222222222222222222222" + sdist_hex = "feedcafe0123456789abcdef0123456789abcdef0123456789abcdef01234567" + + version_entry = struct( + key = "3.11", + py_nodot = "311", + full_version = "3.11.14", + interpreter_target = "@python3_11_host//:python", + ) + parsed_whls = [ + struct( + name = "certifi", + is_exposed = True, + srcs = [ + struct( + distribution = "certifi", + extra_pip_args = [], + requirement_line = "certifi==2026.10.19", + target_platforms = [ + "linux_aarch64", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", + ], + filename = "certifi-2026.10.19-py3-none-any.whl", + digest = "sha256:{}".format(certifi_hex), + url = "https://files.pythonhosted.org/packages/certifi-2026.10.19-py3-none-any.whl", + yanked = None, + ), + ], + ), + struct( + name = "charset_normalizer", + is_exposed = False, + srcs = [ + struct( + distribution = "charset-normalizer", + extra_pip_args = [], + requirement_line = "charset-normalizer==3.5.2", + target_platforms = ["linux_x86_64"], + filename = "charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.whl", + digest = "sha256:{}".format(linux_hex), + url = "https://files.pythonhosted.org/packages/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.whl", + yanked = None, + ), + struct( + distribution = "charset-normalizer", + extra_pip_args = [], + requirement_line = "charset-normalizer==3.5.2", + target_platforms = ["osx_aarch64", "osx_x86_64"], + filename = "charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", + digest = "sha256:{}".format(osx_hex), + url = "https://files.pythonhosted.org/packages/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", + yanked = None, + ), + ], + ), + struct( + name = "sdist_only", + is_exposed = True, + srcs = [ + struct( + distribution = "sdist-only", + extra_pip_args = [], + requirement_line = "sdist-only==1.0.0", + target_platforms = ["linux_x86_64"], + filename = "sdist_only-1.0.0.tar.gz", + digest = "sha256:{}".format(sdist_hex), + url = "https://files.pythonhosted.org/packages/sdist_only-1.0.0.tar.gz", + yanked = None, + ), + ], + ), + struct( + name = "git_pkg", + is_exposed = True, + srcs = [ + struct( + distribution = "git-pkg", + extra_pip_args = [], + requirement_line = "git-pkg @ git+https://github.com/org/repo.git@deadbeef", + target_platforms = ["linux_x86_64"], + filename = "", + digest = "", + url = "", + yanked = None, + ), + ], + ), + struct( + name = "virtual_root", + is_exposed = False, + srcs = [], + ), + ] + + built = build_uv_lock_hub_packages( + [(version_entry, parsed_whls)], + hub_name = "pypi", + is_multi_version = False, + ) + + env.expect.that_collection(built.bzl_packages).contains_exactly([ + "certifi", + "git_pkg", + "sdist_only", + ]) + env.expect.that_str(built.aliases["certifi"]).equals("pypi_certifi_py3_none_any_035c1e85") + env.expect.that_dict(built.aliases["charset_normalizer"]).contains_exactly({ + "//_config:is_linux_x86_64": "pypi_charset_normalizer_cp37_abi3_manylinux_2_5_x86_64_11111111", + "//_config:is_osx_aarch64": "pypi_charset_normalizer_cp37_abi3_macosx_10_9_universal2_22222222", + "//_config:is_osx_x86_64": "pypi_charset_normalizer_cp37_abi3_macosx_10_9_universal2_22222222", + }) + env.expect.that_dict(built.packages["pypi_certifi_py3_none_any_035c1e85"]).contains_exactly({ + "filename": "certifi-2026.10.19-py3-none-any.whl", + "python_interpreter_target": "@python3_11_host//:python", + "requirement": "certifi==2026.10.19", + "sha256": certifi_hex, + "urls": ["https://files.pythonhosted.org/packages/certifi-2026.10.19-py3-none-any.whl"], + }) + env.expect.that_dict(built.packages["pypi_sdist_only_sdist_feedcafe"]).contains_exactly({ + "python_interpreter_target": "@python3_11_host//:python", + "requirement": "sdist-only==1.0.0 --hash=sha256:{}".format(sdist_hex), + }) + env.expect.that_dict(built.packages["pypi_git_pkg"]).contains_exactly({ + "python_interpreter_target": "@python3_11_host//:python", + "requirement": "git-pkg @ git+https://github.com/org/repo.git@deadbeef", + }) + +_tests.append(_test_build_uv_lock_hub_packages_single_version) + +def _test_build_uv_lock_hub_packages_multi_version(env): + """Verify multi-version mode generates per-version spokes and default fallback aliases.""" + whl_hex = "035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae" + py310 = struct( + key = "3.10", + py_nodot = "310", + full_version = "3.10.20", + interpreter_target = "@python_3_10_host//:python", + ) + py311 = struct( + key = "3.11", + py_nodot = "311", + full_version = "3.11.15", + interpreter_target = "@python_3_11_host//:python", + ) + pkg = struct( + name = "requests", + is_exposed = True, + srcs = [ + struct( + distribution = "requests", + extra_pip_args = [], + requirement_line = "requests==2.32.3", + target_platforms = ["linux_x86_64"], + filename = "requests-2.32.3-py3-none-any.whl", + digest = "sha256:{}".format(whl_hex), + url = "https://files.pythonhosted.org/packages/requests-2.32.3-py3-none-any.whl", + yanked = None, + ), + ], + ) + + built = build_uv_lock_hub_packages( + [ + (py310, [pkg]), + (py311, [pkg]), + ], + hub_name = "pypi", + is_multi_version = True, + default_version = "3.10", + ) + + env.expect.that_dict(built.aliases["requests"]).contains_exactly({ + "//_config:is_cp310_linux_x86_64": "pypi_310_requests_py3_none_any_035c1e85", + "//_config:is_cp311_linux_x86_64": "pypi_311_requests_py3_none_any_035c1e85", + "//_config:is_linux_x86_64": "pypi_310_requests_py3_none_any_035c1e85", + }) + +_tests.append(_test_build_uv_lock_hub_packages_multi_version) + +def uv_lock_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/uv/lock/BUILD.bazel b/tests/uv/lock/BUILD.bazel index 156b9dbe6f..acab393a69 100644 --- a/tests/uv/lock/BUILD.bazel +++ b/tests/uv/lock/BUILD.bazel @@ -1,6 +1,5 @@ load(":cache_tests.bzl", "cache_test_suite") load(":lock_tests.bzl", "lock_test_suite") -load(":uv_lock_to_requirements_tests.bzl", "uv_lock_to_requirements_test_suite") exports_files( glob(["testdata/*"]), @@ -11,8 +10,4 @@ lock_test_suite( name = "lock_tests", ) -uv_lock_to_requirements_test_suite( - name = "uv_lock_to_requirements_tests", -) - cache_test_suite(name = "cache_tests") diff --git a/tests/uv/lock/uv_lock_to_requirements_tests.bzl b/tests/uv/lock/uv_lock_to_requirements_tests.bzl deleted file mode 100644 index 1b947877c9..0000000000 --- a/tests/uv/lock/uv_lock_to_requirements_tests.bzl +++ /dev/null @@ -1,289 +0,0 @@ -"""Tests for uv_lock_to_requirements.""" - -load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/uv/private:uv_lock_to_requirements.bzl", "uv_lock_to_requirements") # buildifier: disable=bzl-visibility - -_tests = [] - -def _test_empty(env): - got = uv_lock_to_requirements(json.decode("""{"package":[]}""")) - env.expect.that_str(got).equals("") - -_tests.append(_test_empty) - -def _test_simple_package(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/foo.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -foo==0.0.1 \\ - --hash=sha256:deadbeef -""") - -_tests.append(_test_simple_package) - -def _test_package_with_markers(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"resolution-markers":["python_full_version < '3.10'"],"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/foo.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -foo==0.0.1 ; python_full_version < '3.10' \\ - --hash=sha256:deadbeef -""") - -_tests.append(_test_package_with_markers) - -def _test_package_with_multiple_markers(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"resolution-markers":["python_full_version == '3.10.*'","python_full_version >= '3.14'"],"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/foo.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -foo==0.0.1 ; python_full_version == '3.10.*' or python_full_version >= '3.14' \\ - --hash=sha256:deadbeef -""") - -_tests.append(_test_package_with_multiple_markers) - -def _test_package_with_deps(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"bar","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:baadbeef","url":"https://example.org/bar.whl"}]}, - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"bar"}],"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/foo.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -bar==0.0.1 \\ - --hash=sha256:baadbeef - # via foo - -foo==0.0.1 \\ - --hash=sha256:deadbeef -""") - -_tests.append(_test_package_with_deps) - -def _test_package_with_optional_deps(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"bar","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:baadbeef","url":"https://example.org/bar.whl"}]}, - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"optional-dependencies":{"extra1":[{"name":"bar"}]},"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/foo.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -bar==0.0.1 \\ - --hash=sha256:baadbeef - # via foo - -foo[extra1]==0.0.1 \\ - --hash=sha256:deadbeef -""") - -_tests.append(_test_package_with_optional_deps) - -def _test_self_edge_excluded(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"pydantic","version":"2.0.0","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"pydantic","extra":["email"]}],"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/pydantic.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -pydantic==2.0.0 \\ - --hash=sha256:deadbeef -""") - -_tests.append(_test_self_edge_excluded) - -def _test_multiple_dependents(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"common","version":"1.0.0","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:aaaa","url":"https://example.org/common.whl"}]}, - {"name":"pkg_a","version":"0.1.0","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"common"}],"wheels":[{"hash":"sha256:bbbb","url":"https://example.org/a.whl"}]}, - {"name":"pkg_b","version":"0.2.0","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"common"}],"wheels":[{"hash":"sha256:cccc","url":"https://example.org/b.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -common==1.0.0 \\ - --hash=sha256:aaaa - # via - # pkg_a - # pkg_b - -pkg_a==0.1.0 \\ - --hash=sha256:bbbb - -pkg_b==0.2.0 \\ - --hash=sha256:cccc -""") - -_tests.append(_test_multiple_dependents) - -def _test_git_source_skipped(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"foo","version":"0.1.0","source":{"git":"https://github.com/org/foo.git"}}, - {"name":"bar","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/bar.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -bar==0.0.1 \\ - --hash=sha256:deadbeef -""") - -_tests.append(_test_git_source_skipped) - -def _test_virtual_source_skipped(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"virtual-pkg","version":"0.0.0","source":{"virtual":true}}, - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/foo.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -foo==0.0.1 \\ - --hash=sha256:deadbeef -""") - -_tests.append(_test_virtual_source_skipped) - -def _test_sdist_hash(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"bar","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"sdist":{"hash":"sha256:deadb00f","url":"https://example.org/bar.tar.gz"}} - ]}""")) - env.expect.that_str(got).equals("""\ -bar==0.0.1 \\ - --hash=sha256:deadb00f -""") - -_tests.append(_test_sdist_hash) - -def _test_wheel_and_sdist_hashes(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"sdist":{"hash":"sha256:feedcafe","url":"https://example.org/foo.tar.gz"},"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/foo.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -foo==0.0.1 \\ - --hash=sha256:deadbeef \\ - --hash=sha256:feedcafe -""") - -_tests.append(_test_wheel_and_sdist_hashes) - -def _test_multiple_versions_same_package(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"resolution-markers":["python_full_version < '3.10'"],"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/foo-0.0.1.whl"}]}, - {"name":"foo","version":"0.0.2","source":{"registry":"https://pypi.org/simple"},"resolution-markers":["python_full_version >= '3.10'"],"wheels":[{"hash":"sha256:deadb11f","url":"https://example.org/foo-0.0.2.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -foo==0.0.1 ; python_full_version < '3.10' \\ - --hash=sha256:deadbeef - -foo==0.0.2 ; python_full_version >= '3.10' \\ - --hash=sha256:deadb11f -""") - -_tests.append(_test_multiple_versions_same_package) - -def _test_dep_with_extras(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/foo.whl"}]}, - {"name":"bar","version":"0.0.2","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"foo","extra":["extra1"]}],"wheels":[{"hash":"sha256:baadbeef","url":"https://example.org/bar.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -foo[extra1]==0.0.1 \\ - --hash=sha256:deadbeef - # via bar - -bar==0.0.2 \\ - --hash=sha256:baadbeef -""") - -_tests.append(_test_dep_with_extras) - -def _test_multiple_hashes_from_wheels(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"wheels":[ - {"hash":"sha256:aaaa","url":"https://example.org/foo-0.0.1-cp39.whl"}, - {"hash":"sha256:bbbb","url":"https://example.org/foo-0.0.1-py3-none-any.whl"} - ]} - ]}""")) - env.expect.that_str(got).equals("""\ -foo==0.0.1 \\ - --hash=sha256:aaaa \\ - --hash=sha256:bbbb -""") - -_tests.append(_test_multiple_hashes_from_wheels) - -def _test_package_no_hashes_no_deps(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"}} - ]}""")) - env.expect.that_str(got).equals("""\ -foo==0.0.1 -""") - -_tests.append(_test_package_no_hashes_no_deps) - -def _test_package_with_multiple_optional_deps(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"bar","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:baadbeef","url":"https://example.org/bar.whl"}]}, - {"name":"baz","version":"0.0.2","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:deadc0de","url":"https://example.org/baz.whl"}]}, - {"name":"foo","version":"0.0.3","source":{"registry":"https://pypi.org/simple"},"optional-dependencies":{"extra1":[{"name":"bar"}],"extra2":[{"name":"baz"}]},"wheels":[{"hash":"sha256:feedcafe","url":"https://example.org/foo.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -bar==0.0.1 \\ - --hash=sha256:baadbeef - # via foo - -baz==0.0.2 \\ - --hash=sha256:deadc0de - # via foo - -foo[extra1,extra2]==0.0.3 \\ - --hash=sha256:feedcafe -""") - -_tests.append(_test_package_with_multiple_optional_deps) - -def _test_dep_with_multiple_extras(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"foo","version":"0.0.1","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:deadbeef","url":"https://example.org/foo.whl"}]}, - {"name":"bar","version":"0.0.2","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"foo","extra":["extra1","extra2"]}],"wheels":[{"hash":"sha256:baadbeef","url":"https://example.org/bar.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -foo[extra1,extra2]==0.0.1 \\ - --hash=sha256:deadbeef - # via bar - -bar==0.0.2 \\ - --hash=sha256:baadbeef -""") - -_tests.append(_test_dep_with_multiple_extras) - -def _test_extras_from_multiple_dependents(env): - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"common","version":"1.0.0","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:aaaa","url":"https://example.org/common.whl"}]}, - {"name":"pkg_a","version":"0.1.0","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"common","extra":["extra1"]}],"wheels":[{"hash":"sha256:bbbb","url":"https://example.org/a.whl"}]}, - {"name":"pkg_b","version":"0.2.0","source":{"registry":"https://pypi.org/simple"},"dependencies":[{"name":"common","extra":["extra2"]}],"wheels":[{"hash":"sha256:cccc","url":"https://example.org/b.whl"}]} - ]}""")) - env.expect.that_str(got).equals("""\ -common[extra1,extra2]==1.0.0 \\ - --hash=sha256:aaaa - # via - # pkg_a - # pkg_b - -pkg_a==0.1.0 \\ - --hash=sha256:bbbb - -pkg_b==0.2.0 \\ - --hash=sha256:cccc -""") - -_tests.append(_test_extras_from_multiple_dependents) - -def _test_requires_dist_extras(env): - """Test that extras from metadata.requires-dist are included.""" - got = uv_lock_to_requirements(json.decode("""{"package":[ - {"name":"pytest-bazel","version":"0.1.6","source":{"registry":"https://pypi.org/simple"},"wheels":[{"hash":"sha256:a29e80e1d67c3db801bdd4d0b6b742f2bfb48cd6841caa33401458e5c4e29c21","url":"https://example.org/pytest_bazel-0.1.6.whl"}]}, - {"name":"root-pkg","version":"0.0.0","source":{"virtual":"."},"dependencies":[{"name":"pytest-bazel"}],"metadata":{"requires-dist":[{"name":"pytest-bazel","extras":["all"]}]}} - ]}""")) - env.expect.that_str(got).equals("""\ -pytest-bazel[all]==0.1.6 \\ - --hash=sha256:a29e80e1d67c3db801bdd4d0b6b742f2bfb48cd6841caa33401458e5c4e29c21 - # via root-pkg -""") - -_tests.append(_test_requires_dist_extras) - -def uv_lock_to_requirements_test_suite(name): - test_suite(name = name, basic_tests = _tests)