diff --git a/.github/workflows/check-ldconfig.yaml b/.github/workflows/check-ldconfig.yaml index 2dede145b..9103b42ab 100644 --- a/.github/workflows/check-ldconfig.yaml +++ b/.github/workflows/check-ldconfig.yaml @@ -20,7 +20,7 @@ jobs: persist-credentials: false - name: Set up QEMU - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 - name: Check ldconfig caches - run: bazel test //python3:check_ldconfig_tests + run: bazel test //python3:check_ldconfig_tests //python:check_ldconfig_tests diff --git a/.github/workflows/update-deb-package-snapshots.yml b/.github/workflows/update-deb-package-snapshots.yml index c1077b60e..2df2ab538 100644 --- a/.github/workflows/update-deb-package-snapshots.yml +++ b/.github/workflows/update-deb-package-snapshots.yml @@ -29,11 +29,11 @@ jobs: - name: Set up QEMU if: env.DISTROLESS_DIFF - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 - name: Update ldconfig if: env.DISTROLESS_DIFF - run: bazel run //python3:update_ldconfig + run: bazel run //python3:update_ldconfig //python:update_ldconfig - name: Run update sboms script if: env.DISTROLESS_DIFF diff --git a/.gitignore b/.gitignore index 3718e7ab6..7be0b3f95 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,8 @@ /output/ # ignore user bazelrc .bazelrc.user -*.log \ No newline at end of file +*.log +.gocache/ +/.gotmp + +/python/Dockerfile diff --git a/BUILD b/BUILD index 9063b4f90..e64bb3778 100644 --- a/BUILD +++ b/BUILD @@ -4,11 +4,14 @@ load("//java:config.bzl", "JAVA_ARCHITECTURES", "JAVA_DISTROS", "JAVA_MAJOR_VERS load("//nodejs:config.bzl", "NODEJS_ARCHITECTURES", "NODEJS_DISTROS", "NODEJS_MAJOR_VERSIONS") load("//private/oci:defs.bzl", "sign_and_push_all") load("//private/tools/lifecycle:defs.bzl", "attach_lifecycle_tags") +load("//python:config.bzl", PBS_PYTHON_ARCHITECTURES = "PYTHON_ARCHITECTURES", PBS_PYTHON_DISTROS = "PYTHON_DISTROS", PBS_PYTHON_MAJOR_VERSIONS = "PYTHON_MAJOR_VERSIONS") load("//python3:config.bzl", "PYTHON_ARCHITECTURES", "PYTHON_DISTROS") load("//static:config.bzl", "STATIC_ARCHITECTURES", "STATIC_DISTROS") package(default_visibility = ["//visibility:public"]) +exports_files(["MODULE.bazel"]) + DEFAULT_DISTRO = "debian13" VARIANTS = [ @@ -196,6 +199,37 @@ NODEJS |= { for (tag_base, debug_mode, user) in VARIANTS } +############### +# PYTHON (PBS) # +############### +PYTHON = { + "{REGISTRY}/{PROJECT_ID}/python" + version + "-" + distro + ":" + tag_base + "-" + arch: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + arch + "_" + distro + for version in PBS_PYTHON_MAJOR_VERSIONS + for distro in PBS_PYTHON_DISTROS + for arch in PBS_PYTHON_ARCHITECTURES[distro][version] + for (tag_base, debug_mode, user) in VARIANTS +} + +PYTHON |= { + "{REGISTRY}/{PROJECT_ID}/python" + version + "-" + distro + ":" + tag_base: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + distro + for version in PBS_PYTHON_MAJOR_VERSIONS + for distro in PBS_PYTHON_DISTROS + for (tag_base, debug_mode, user) in VARIANTS +} + +PYTHON |= { + "{REGISTRY}/{PROJECT_ID}/python" + version + ":" + tag_base + "-" + arch: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + arch + "_" + DEFAULT_DISTRO + for version in PBS_PYTHON_MAJOR_VERSIONS + for arch in PBS_PYTHON_ARCHITECTURES[DEFAULT_DISTRO][version] + for (tag_base, debug_mode, user) in VARIANTS +} + +PYTHON |= { + "{REGISTRY}/{PROJECT_ID}/python" + version + ":" + tag_base: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + DEFAULT_DISTRO + for version in PBS_PYTHON_MAJOR_VERSIONS + for (tag_base, debug_mode, user) in VARIANTS +} + ############### # JAVA_BASE # ############### @@ -270,6 +304,8 @@ ALL |= PYTHON3 ALL |= NODEJS +ALL |= PYTHON + ALL |= JAVA_BASE ALL |= JAVA diff --git a/MODULE.bazel b/MODULE.bazel index 4954f049b..d8da03e8f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -55,5 +55,10 @@ node = use_extension("//private/extensions:node.bzl", "node") node.archive() use_repo(node, "node_versions", "nodejs22_amd64", "nodejs22_arm", "nodejs22_arm64", "nodejs22_ppc64le", "nodejs22_s390x", "nodejs24_amd64", "nodejs24_arm64", "nodejs24_ppc64le", "nodejs24_s390x", "nodejs26_amd64", "nodejs26_arm64", "nodejs26_ppc64le", "nodejs26_s390x") +### PYTHON ### +py = use_extension("//private/extensions:python.bzl", "python") +py.archive() +use_repo(py, "python313_amd64", "python313_arm64", "python313_riscv64", "python313_s390x", "python314_amd64", "python314_arm64", "python314_riscv64", "python314_s390x", "python_versions") + ### DEBIAN ### include("//private/repos/deb:deb.MODULE.bazel") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 7af9dd1a8..30dbdffe9 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -645,6 +645,153 @@ "recordedRepoMappingEntries": [] } }, + "//private/extensions:python.bzl%python": { + "general": { + "bzlTransitiveDigest": "1Ko7/R5xpXOJ/JA3GD9LzrmcNheDq/ENqQ14jDpCLLk=", + "usagesDigest": "c4TzkyPW7cDJnkOuzqF5JKto51Qz2U++tuhW9H+H22w=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "python313_amd64": { + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", + "attributes": { + "sha256": "0651dd7157d3debf769e15a52c1de9de7fbcdc36ba72faf79fde3c44f14d9461", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-x86_64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260901", + "python_version": "3.13", + "architecture": "amd64" + } + }, + "python313_arm64": { + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", + "attributes": { + "sha256": "76ed18125286d7dc96ce24023d1e319dbd55a89a767102411b1ea23846113f69", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-aarch64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260901", + "python_version": "3.13", + "architecture": "arm64" + } + }, + "python313_s390x": { + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", + "attributes": { + "sha256": "b738ddb7271b2591d6e89fcd4ef42b6cbbcac7c489daf807b6b01442ae065a0a", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-s390x-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260901", + "python_version": "3.13", + "architecture": "s390x" + } + }, + "python313_riscv64": { + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", + "attributes": { + "sha256": "584a5a197a3d1ce8d8e45496515802872f1c18c4d20ee54f088081f75e7e8b4b", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-riscv64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260901", + "python_version": "3.13", + "architecture": "riscv64" + } + }, + "python314_amd64": { + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", + "attributes": { + "sha256": "0ab3305457051cd3e7c031857e005f1bda17c218a1990567dacaaac6dd1d14f0", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-x86_64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260901", + "python_version": "3.14", + "architecture": "amd64" + } + }, + "python314_arm64": { + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", + "attributes": { + "sha256": "30f1cc489be654477d895b441e196bb080738bf0456da82080ad4ab66a22d80f", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-aarch64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260901", + "python_version": "3.14", + "architecture": "arm64" + } + }, + "python314_s390x": { + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", + "attributes": { + "sha256": "dc27e917b88db2560fe213244ffa6a656e8197ca0beb4ddeaaee5286041488ad", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-s390x-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260901", + "python_version": "3.14", + "architecture": "s390x" + } + }, + "python314_riscv64": { + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", + "attributes": { + "sha256": "cf71ad2f451ba9af30992391b2a403cddb6965f11ef5255ee2f5d3e504f75add", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-riscv64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260901", + "python_version": "3.14", + "architecture": "riscv64" + } + }, + "python_versions": { + "repoRuleId": "@@//private/extensions:python.bzl%python_versions_repo", + "attributes": { + "versions": { + "3.13_amd64": "3.13.15", + "3.13_arm64": "3.13.15", + "3.13_s390x": "3.13.15", + "3.13_riscv64": "3.13.15", + "3.14_amd64": "3.14.7", + "3.14_arm64": "3.14.7", + "3.14_s390x": "3.14.7", + "3.14_riscv64": "3.14.7" + } + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [ + "python_versions", + "python313_amd64", + "python313_arm64", + "python313_s390x", + "python313_riscv64", + "python314_amd64", + "python314_arm64", + "python314_s390x", + "python314_riscv64" + ], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + }, + "recordedRepoMappingEntries": [] + } + }, "//private/extensions:version.bzl%version": { "general": { "bzlTransitiveDigest": "a4FYxHYUQ822kcAcKD5tzEjksar43fkO3cC2oeyx6zA=", diff --git a/knife b/knife index 51f818c81..8e628b6fa 100755 --- a/knife +++ b/knife @@ -159,6 +159,66 @@ function cmd_lint () { buildifier -mode=$mode $(find . -type f \( -name 'BUILD*' -o -name 'WORKSPACE*' -o -name '*.bzl' \)) } +function cmd_update_python_archives () { + if ! which jq > /dev/null; then + echo "🧱 No jq executable was found" + exit 1 + fi + if ! which curl > /dev/null; then + echo "🧱 No curl executable was found" + exit 1 + fi + + VERBOSE=0 + DRY_RUN=0 + for arg in "$@"; do + case "$arg" in + --verbose) + VERBOSE=1 + ;; + --dry-run) + DRY_RUN=1 + ;; + *) + echo "unknown option: $arg" + exit 1 + ;; + esac + done + export VERBOSE DRY_RUN + + source knife.d/update_python_archives.sh + old_versions=$(get_python_versions) + if ! out=$(generate_python_archives); then + case "$out" in + *MUTATED_PARTIAL*) + echo "update failed mid-mutation; files may be partially updated." >&2 + echo "Restore the affected files from version control before retrying." >&2 + ;; + *) + echo "update failed; no files were mutated (pre-flight validation)." >&2 + ;; + esac + return 1 + fi + if [ "$out" = "NO_CHANGE" ]; then + echo "python archives are up to date" + return 0 + fi + if [ "$DRY_RUN" = 1 ]; then + echo "dry-run: no files written" + return 0 + fi + echo "updated python archives to release $out" + if ! update_test_versions_python "$old_versions"; then + echo "testdata update failed after the archive update." >&2 + echo "Restore the affected files from version control before retrying." >&2 + return 1 + fi + # The extension changed; refresh the lockfile before building. + echo "Refresh the lockfile before building." >&2 +} + function cmd_update_node_archives () { if ! which jq > /dev/null; then echo "🧱 No jq executable was found" @@ -289,8 +349,11 @@ update-node-archives) update-node-keys) cmd_update_node_keys ;; +update-python-archives) + cmd_update_python_archives "${@:2}" + ;; ~~nocmd) # no command provided - echo "provide a command: lock, update-snapshots, github-update-snapshots, update-non-snapshots, test, deb-versions, update-node-archives, update-node-keys" + echo "provide a command: lock, update-snapshots, github-update-snapshots, update-non-snapshots, test, deb-versions, update-node-archives, update-node-keys, update-python-archives" exit 1 ;; *) # unknown command diff --git a/knife.d/BUILD.bazel b/knife.d/BUILD.bazel new file mode 100644 index 000000000..6f14faccf --- /dev/null +++ b/knife.d/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["update_python_archives.sh"]) diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh new file mode 100644 index 000000000..06319439f --- /dev/null +++ b/knife.d/update_python_archives.sh @@ -0,0 +1,492 @@ +set -o pipefail -o errexit -o nounset + +# Copyright 2026 Google Inc. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Functions used by knife to update python-build-standalone archives. +# +# Reads the Python matrix and pinned versions, queries the latest PBS release, +# then rewrites archive and version data when needed. +# +# An update is triggered when the CPython patch moves or the PBS release tag +# changes under an unchanged patch, indicating rebuilt native libraries. + +PYTHON_TRIPLES=( + "amd64=x86_64-unknown-linux-gnu" + "arm64=aarch64-unknown-linux-gnu" + "s390x=s390x-unknown-linux-gnu" + "riscv64=riscv64-unknown-linux-gnu" +) + +# Use a temporary file for portable in-place editing on BSD and GNU sed. +sed_inplace() { # $1 = sed expression, $2 = file + local tmp mode + tmp="${2}.tmp.$$" + sed -e "$1" "$2" > "$tmp" || { rm -f "$tmp"; return 1; } + mode=$(stat -c %a "$2" 2>/dev/null || stat -f %Lp "$2") + chmod "$mode" "$tmp" + mv "$tmp" "$2" +} + +# Print "_ " for each matrix entry. +function get_python_versions() { + sed -n '/python_versions_repo(/,/^ )$/p' private/extensions/python.bzl \ + | grep -oE '"[0-9]+\.[0-9]+_[a-z0-9]+": "[0-9]+\.[0-9]+\.[0-9]+(b[0-9]+|rc[0-9]+)?"' \ + | sed -E 's/"([^"]+)": "([^"]+)"/\1 \2/' +} + +# Print the matrix's minor versions, one per line. +function get_python_minors() { + sed -n 's/^PYTHON_MAJOR_VERSIONS = \[\(.*\)\]$/\1/p' python/config.bzl \ + | grep -oE '"[0-9]+\.[0-9]+"' | tr -d '"' +} + +# Print the architectures for a minor, one per line. +function get_python_archs() { + local minor="$1" + # A missing minor returns no architectures; the caller reuses the previous + # minor's list for a newly detected minor. + grep "\"${minor}\": \[" python/config.bzl \ + | grep -oE '"[a-z0-9]+"' | tr -d '"' || true +} + +function triple_for_arch() { + local entry + for entry in "${PYTHON_TRIPLES[@]}"; do + [[ "$entry" == "$1="* ]] && echo "${entry#*=}" && return 0 + done + return 1 +} + +# Print the current version for a minor and architecture, if any. +function current_version() { + get_python_versions | awk -v key="$1_$2" '$1 == key { print $2 }' +} + +# Print the full pinned version (patch + PBS release tag) for a minor and +# architecture, if any. +# A tag-only bump (same patch, rebuilt native libs) must trigger an update. +function pinned_version() { + local minor="$1" arch="$2" short + short=$(echo "$minor" | tr -d '.') + awk -v name="python${short}_${arch}" ' + $0 ~ ("name = \"" name "\"") { found = 1 } + found && /version = / { + sub(/.*version = "/, ""); sub(/".*/, ""); print; exit + } + ' private/extensions/python.bzl +} + +# Rewrite archives, versions, metadata, and repository visibility. Print the +# new release tag on success or "NO_CHANGE" when current. +function generate_python_archives() { + local latest_release sha256sums + local minors minor arch triple version sha python_short arch_anchor matrix_min published latest_minor + local version_re stable_only + PYTHON_MUTATED=0 + local -a archive_blocks versions_entries metadata_deps changes repos verify_assets + local changed=0 verbose=${VERBOSE:-0} dry_run=${DRY_RUN:-0} current + local module_tmp="" repos_sorted module_line current_module_line module_changed=0 + + # Tests can supply release data through PBS_RELEASE_FILE and + # PBS_SHA256SUMS_FILE; otherwise use the live release files. + if [ -n "${PBS_RELEASE_FILE:-}" ] && [ -n "${PBS_SHA256SUMS_FILE:-}" ]; then + latest_release=$(sed -n 's/.*"tag"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$PBS_RELEASE_FILE" | head -1) + sha256sums=$(cat "$PBS_SHA256SUMS_FILE") + else + # Prefer PBS's machine-readable release file; fall back to the API if it is unavailable. + latest_release=$(curl -sSL https://raw.githubusercontent.com/astral-sh/python-build-standalone/latest-release/latest-release.json 2>/dev/null | jq -r '.tag') + if [ -z "$latest_release" ] || [ "$latest_release" = "null" ]; then + latest_release=$(curl -sSL https://api.github.com/repos/astral-sh/python-build-standalone/releases/latest 2>/dev/null | jq -r '.tag_name') + fi + sha256sums=$(curl -sSL "https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/SHA256SUMS") + fi + [ -n "$latest_release" ] && [ "$latest_release" != "null" ] || { echo "no latest PBS release found" >&2; exit 1; } + [ -n "$sha256sums" ] || { echo "no SHA256SUMS for ${latest_release}" >&2; exit 1; } + + local -a minors + minors=() + local minors_tmp archs + minors_tmp=$(mktemp) + get_python_minors > "$minors_tmp" + while IFS= read -r minor; do minors+=("$minor"); done < "$minors_tmp" + rm -f "$minors_tmp" + [ ${#minors[@]} -gt 0 ] || { echo "no minors parsed from python/config.bzl" >&2; exit 1; } + + # Sort and deduplicate to avoid depending on matrix order. + minors=($(printf '%s\n' "${minors[@]}" | sort -uV)) + arch_anchor=${minors[${#minors[@]} - 1]} + matrix_min=${minors[0]} + + # Find stable PBS minors from amd64 assets. Prereleases are excluded by + # requiring 3.X.Y. The matrix is a contiguous support window; fill published + # minors above its oldest entry. + published=$(echo "$sha256sums" \ + | { grep -oE 'cpython-3\.[0-9]+\.[0-9]+\+[0-9]+-x86_64-unknown-linux-gnu-install_only\.tar\.gz' || true; } \ + | sed -E 's/cpython-(3\.[0-9]+)\.[0-9]+.*/\1/' | sort -uV) + local fill fill_m + fill=() + for fill_m in $published; do + if [ "$fill_m" != "$matrix_min" ] \ + && [ "$(printf '%s\n' "$matrix_min" "$fill_m" | sort -V | tail -1)" = "$fill_m" ] \ + && [[ " ${minors[*]} " != *" $fill_m "* ]]; then + fill+=("$fill_m") + fi + done + if [ ${#fill[@]} -gt 0 ]; then + echo "new stable minor(s) detected: ${fill[*]}" >&2 + for fill_m in "${fill[@]}"; do + minors+=("$fill_m") + changes+=("add minor ${fill_m}") + done + minors=($(printf '%s\n' "${minors[@]}" | sort -uV)) + # A missing matrix entry is a change even if version data already exists. + changed=1 + fi + latest_minor=$arch_anchor + + for minor in "${minors[@]}"; do + python_short=$(echo "$minor" | tr -d '.') + # Reuse the previous minor's architectures for a newly detected minor. + archs=$(get_python_archs "$minor") + if [ -z "$archs" ] && [ ${#fill[@]} -gt 0 ] && [[ " ${fill[*]} " == *" $minor "* ]]; then + archs=$(get_python_archs "$latest_minor") + fi + [ -n "$archs" ] || { echo "no archs for ${minor} in python/config.bzl" >&2; exit 1; } + stable_only=0 + if [ ${#fill[@]} -gt 0 ] && [[ " ${fill[*]} " == *" $minor "* ]]; then + stable_only=1 + fi + while IFS= read -r arch; do + triple=$(triple_for_arch "$arch") || { echo "no triple for ${arch}" >&2; exit 1; } + # Select the latest stable patch or opted-in prerelease. + version_re='[0-9]+(b[0-9]+|rc[0-9]+)?' + [ "$stable_only" = 1 ] && version_re='[0-9]+' + version=$(echo "$sha256sums" \ + | { grep -oE "cpython-${minor}\\.${version_re}\\+${latest_release}-${triple}-install_only\\.tar\\.gz" || true; } \ + | sed -E "s/cpython-(${minor}\\.${version_re})\\+.*/\\1/" | sort -V | tail -1) + [ -n "$version" ] || { echo "required matrix entry ${minor} ${arch} has no install_only PBS asset in ${latest_release}" >&2; exit 1; } + sha=$(echo "$sha256sums" | grep "cpython-${version}+${latest_release}-${triple}-install_only.tar.gz" | cut -d' ' -f1) + [ -n "$sha" ] || { echo "no sha for ${version} ${triple}" >&2; exit 1; } + verify_assets+=("${minor}|${arch}|${version}|${triple}|${sha}") + + current=$(current_version "$minor" "$arch") + pinned=$(pinned_version "$minor" "$arch") + new_full="${version}+${latest_release}" + if [ "$verbose" = 1 ]; then + if [ "$current" = "$version" ] && [ "$pinned" = "$new_full" ]; then + echo " ${minor} ${arch}: ${pinned} (unchanged)" >&2 + else + echo " ${minor} ${arch}: ${pinned:-} -> ${new_full} (update)" >&2 + fi + fi + # Update when the patch or PBS release tag moves. + if [ "$current" != "$version" ] || [ "$pinned" != "$new_full" ]; then + changed=1 + changes+=("update ${minor} ${arch}: ${pinned:-} -> ${new_full}") + fi + + archive_blocks+=(" python_archive( + name = \"python${python_short}_${arch}\", + sha256 = \"${sha}\", + strip_prefix = \"python/\", + urls = [\"https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/cpython-${version}+${latest_release}-${triple}-install_only.tar.gz\"], + version = \"${version}+${latest_release}\", + python_version = \"${minor}\", + architecture = \"${arch}\", + )") + versions_entries+=(" \"${minor}_${arch}\": \"${version}\",") + metadata_deps+=(" \"python${python_short}_${arch}\",") + repos+=("python${python_short}_${arch}") + done <<< "$archs" + done + + # Keep root-module repository visibility aligned with the selected matrix. + repos_sorted=$(printf '%s\n' "${repos[@]}" | sort | sed 's/^/"/; s/$/"/' | paste -sd, - | sed 's/,/, /g') + module_line="use_repo(py, ${repos_sorted}, \"python_versions\")" + current_module_line=$(grep '^use_repo(py,' MODULE.bazel || true) + if [ "$current_module_line" != "$module_line" ]; then + module_changed=1 + changes+=("sync MODULE.bazel repositories") + changed=1 + fi + + # Regenerate the PBS SPDX SBOM from the release component manifest. + local downloads_file sbom_tmp="" downloads_tmp="" + if [ -n "${PBS_DOWNLOADS_FILE:-}" ]; then + downloads_file="$PBS_DOWNLOADS_FILE" + else + downloads_tmp=$(mktemp) + if ! curl -sSL "https://raw.githubusercontent.com/astral-sh/python-build-standalone/${latest_release}/pythonbuild/downloads.py" -o "$downloads_tmp"; then + echo "no PBS downloads manifest for ${latest_release}" >&2 + rm -f "$downloads_tmp" + exit 1 + fi + downloads_file="$downloads_tmp" + fi + [ -s "$downloads_file" ] || { echo "empty PBS downloads manifest" >&2; exit 1; } + sbom_tmp=$(mktemp) + if ! python3 python/gen_pbs_sbom.py "$downloads_file" "$latest_release" "$sbom_tmp"; then + echo "PBS SBOM generation failed" >&2 + rm -f "$sbom_tmp" "$downloads_tmp" + exit 1 + fi + grep -q "$latest_release" "$sbom_tmp" || { echo "PBS SBOM does not mention ${latest_release}" >&2; rm -f "$sbom_tmp" "$downloads_tmp"; exit 1; } + # Verify detectable native-library versions in each selected archive against + # the manifest. The archive is SHA-verified; tests can supply fixture archives. + local tarball tarball_tmp so_dir so_path + local verify_entry verify_minor verify_arch verify_version verify_triple verify_sha + local fname tarball_sha got_sha verify_root + verify_root=$(mktemp -d) + if [ -n "${PBS_TARBALL_FILE:-}" ]; then + verify_assets=("fixture||||") + fi + if [ -z "${PBS_TARBALL_FILE:-}" ]; then + [ ${#verify_assets[@]} -gt 0 ] || { echo "no PBS matrix archives to verify" >&2; rm -rf "$verify_root"; exit 1; } + fi + for verify_entry in "${verify_assets[@]}"; do + IFS='|' read -r verify_minor verify_arch verify_version verify_triple verify_sha <<< "$verify_entry" + tarball_tmp="" + if [ -n "${PBS_TARBALL_FILE:-}" ]; then + tarball="$PBS_TARBALL_FILE" + fname="fixture" + else + fname="cpython-${verify_version}+${latest_release}-${verify_triple}-install_only.tar.gz" + if [ -n "${PBS_TARBALL_DIR:-}" ]; then + tarball="${PBS_TARBALL_DIR}/${fname}" + [ -f "$tarball" ] || { echo "missing PBS test tarball ${fname}" >&2; rm -f "$sbom_tmp" "$downloads_tmp"; rm -rf "$verify_root"; exit 1; } + else + tarball_tmp=$(mktemp) + tarball_sha="$verify_sha" + if [ -z "$tarball_sha" ]; then + echo "no sha for ${fname}" >&2 + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" + rm -rf "$verify_root" + exit 1 + fi + if ! curl -sSL "https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/${fname}" -o "$tarball_tmp"; then + echo "cannot download ${fname}" >&2 + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" + rm -rf "$verify_root" + exit 1 + fi + got_sha=$(python3 -c "import hashlib,sys; print(hashlib.sha256(open(sys.argv[1],'rb').read()).hexdigest())" "$tarball_tmp") + if [ "$got_sha" != "$tarball_sha" ]; then + echo "sha256 mismatch for ${fname}: expected ${tarball_sha}, got ${got_sha}" >&2 + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" + rm -rf "$verify_root" + exit 1 + fi + tarball="$tarball_tmp" + fi + fi + so_dir=$(mktemp -d "$verify_root/so.XXXXXX") + if ! python3 - "$tarball" "$so_dir" <<'PYEOF'; then +import os, shutil, sys, tarfile +tar = tarfile.open(sys.argv[1]) +for member in tar.getmembers(): + basename = member.name.rsplit("/", 1)[-1] + if (member.name.startswith("python/lib/") + and ".." not in member.name.split("/") + and basename.startswith("libpython3.") + and ".so" in basename + and member.isfile()): + source = tar.extractfile(member) + if source is None: + continue + with source, open(os.path.join(sys.argv[2], basename), "wb") as target: + shutil.copyfileobj(source, target) + sys.exit(0) +sys.exit("no libpython3*.so in tarball") +PYEOF + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" + rm -rf "$verify_root" + exit 1 + fi + so_path=$(find "$so_dir" -type f -name 'libpython3*.so*' | head -1) + if [ -z "$so_path" ]; then + echo "no libpython3*.so in ${fname} (${verify_minor} ${verify_arch})" >&2 + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" + rm -rf "$verify_root" + exit 1 + fi + # Keep stdout for the release tag and send diagnostics to stderr. + if ! python3 python/pbs_embedded_versions.py "$so_path" "$downloads_file" >&2; then + echo "PBS embedded native libraries drift detected in ${fname} (${verify_minor} ${verify_arch})" >&2 + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" + rm -rf "$verify_root" + exit 1 + fi + [ -z "$tarball_tmp" ] || rm -f "$tarball_tmp" + done + rm -rf "$verify_root" + [ -n "${PBS_DOWNLOADS_FILE:-}" ] || rm -f "$downloads_tmp" + + if [ "$dry_run" = 1 ]; then + rm -f "$sbom_tmp" + rm -f "$module_tmp" + if [ "$changed" = 0 ]; then + echo "NO_CHANGE" + else + printf 'would %s\n' "${changes[@]}" >&2 + echo "DRY_RUN" + fi + return 0 + fi + + if [ "$changed" = 0 ]; then + rm -f "$sbom_tmp" + rm -f "$module_tmp" + echo "NO_CHANGE" + return 0 + fi + + printf '%s\n' "${changes[@]}" >&2 + + local start end section tmp + start=$(grep -n '# Python from python-build-standalone' private/extensions/python.bzl | head -1 | cut -d: -f1) + end=$(grep -n '^_archive = tag_class' private/extensions/python.bzl | head -1 | cut -d: -f1) + local minors_list + minors_list=$(echo "${minors[*]}" | sed 's/ /, /g') + section=" # Python from python-build-standalone (https://github.com/astral-sh/python-build-standalone) + # Release ${latest_release}. Linux targets only (distroless images). + # Versions ${minors_list}. New stable minors are added by the updater. + # armv7 is excluded: PBS publishes soft-float gnueabi builds + # (interpreter /lib/ld-linux.so.3) which cannot run on the distroless armhf base + # (loader /usr/lib/ld-linux-armhf.so.3); ppc64le is not published by PBS. +$(printf '%s\n\n' "${archive_blocks[@]}") + + python_versions_repo( + name = \"python_versions\", + versions = { +$(printf '%s\n' "${versions_entries[@]}") + }, + ) + + return module_ctx.extension_metadata( + root_module_direct_deps = [ + \"python_versions\", +$(printf '%s\n' "${metadata_deps[@]}") + ], + root_module_direct_dev_deps = [], + ) + +" + tmp=$(mktemp) + head -n $((start - 1)) private/extensions/python.bzl > "$tmp" + printf '%s' "$section" >> "$tmp" + tail -n +"$end" private/extensions/python.bzl >> "$tmp" + # Verify the generated extension before replacing the committed file. + if which buildifier >/dev/null 2>&1; then + if ! buildifier -mode=fix "$tmp"; then + echo "generated extension failed buildifier; aborting without changes" >&2 + rm -f "$tmp" + return 1 + fi + fi + # Apply and verify matrix changes in a temporary file before replacing the file. + local config_tmp="" + if [ ${#fill[@]} -gt 0 ]; then + local arch_list fill_entries fill_m2 + grep -q '^PYTHON_MAJOR_VERSIONS = ' python/config.bzl \ + || { echo "PYTHON_MAJOR_VERSIONS line not found in python/config.bzl" >&2; rm -f "$tmp"; return 1; } + grep -q "\"${latest_minor}\": \[" python/config.bzl \ + || { echo "arch map for ${latest_minor} not found in python/config.bzl" >&2; rm -f "$tmp"; return 1; } + arch_list=$(get_python_archs "$latest_minor" | sed 's/^/"/; s/$/",/' | tr '\n' ' ' | sed 's/ $//') + minors_quoted=$(printf '\"%s\", ' "${minors[@]}" | sed 's/, $//') + fill_entries="" + for fill_m2 in "${fill[@]}"; do + fill_entries="${fill_entries}\n \"${fill_m2}\": [${arch_list}]," + done + config_tmp=$(mktemp) + cp python/config.bzl "$config_tmp" + sed_inplace "s/^PYTHON_MAJOR_VERSIONS = .*/PYTHON_MAJOR_VERSIONS = [${minors_quoted}]/" "$config_tmp" + sed_inplace "s|\"${latest_minor}\": \[[^]]*\],|\"${latest_minor}\": [${arch_list}],${fill_entries}|" "$config_tmp" + for fill_m2 in "${fill[@]}"; do + grep -qE "PYTHON_MAJOR_VERSIONS = .*\"${fill_m2}\"" "$config_tmp" \ + || { echo "config.bzl update for ${fill_m2} did not land (format drift?)" >&2; rm -f "$config_tmp" "$tmp"; return 1; } + grep -q "\"${fill_m2}\": \[" "$config_tmp" \ + || { echo "config.bzl arch map for ${fill_m2} did not land" >&2; rm -f "$config_tmp" "$tmp"; return 1; } + done + fi + + # All generated content is verified; apply it. + if [ "$module_changed" = 1 ]; then + module_tmp=$(mktemp) + cp MODULE.bazel "$module_tmp" + sed_inplace "s/^use_repo(py, .*/${module_line}/" "$module_tmp" + grep -Fxq "$module_line" "$module_tmp" \ + || { echo "MODULE.bazel use_repo update did not land" >&2; rm -f "$module_tmp"; rm -f "$config_tmp" "$tmp"; return 1; } + fi + PYTHON_MUTATED=1 + mv "$tmp" private/extensions/python.bzl || { echo "extension update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + mv "$sbom_tmp" python/pbs-sbom.spdx.json || { echo "SBOM update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + if [ -n "$config_tmp" ]; then + mv "$config_tmp" python/config.bzl || { echo "config.bzl update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + fi + if [ -n "$module_tmp" ]; then + mv "$module_tmp" MODULE.bazel || { echo "MODULE.bazel update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + fi + + # Normalize formatting when buildifier is available. + if which buildifier >/dev/null 2>&1; then + if ! buildifier -mode=fix private/extensions/python.bzl python/config.bzl; then + echo "buildifier failed on the updated files" >&2 + echo "MUTATED_PARTIAL" + return 1 + fi + fi + + echo "$latest_release" +} + +# Generated files are verified before replacement. Failures after the first +# replacement are reported as partial updates. Refresh the lockfile after success. + +# Update version-specific testdata and create files for new minors. $1 is the +# version snapshot taken before generate_python_archives. +function update_test_versions_python() { + local old_snapshot minor file old new expected + old_snapshot=$1 + # Keep the smoke test's expected minors in sync with the matrix. + expected=$(get_python_minors | tr '\n' ' ') + sed_inplace "s/\"\$minors\" = \"[^\"]*\"/\"\$minors\" = \"${expected}\"/" python/update_python_archives_test.sh + for minor in $(get_python_minors); do + file="python/testdata/python3.$(echo "$minor" | cut -d. -f2).yaml" + new=$(current_version "$minor" "amd64") + if [ ! -f "$file" ]; then + [ -n "$new" ] || continue + cat > "$file" <&2; return 1; } + echo "bumped $file to ${new}" + fi + done +} diff --git a/private/extensions/BUILD.bazel b/private/extensions/BUILD.bazel index e69de29bb..648b61304 100644 --- a/private/extensions/BUILD.bazel +++ b/private/extensions/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["python.bzl"]) diff --git a/private/extensions/python.bzl b/private/extensions/python.bzl new file mode 100644 index 000000000..fcf2142d1 --- /dev/null +++ b/private/extensions/python.bzl @@ -0,0 +1,233 @@ +"python" + +BUILD_TMPL = """\ +# GENERATED FILE; DO NOT EDIT. +load("@distroless//private/pkg:package_spdx.bzl", "package_spdx") +load("@distroless//private/util:merge_providers.bzl", "merge_providers") +load("@distroless//private/util:tar.bzl", "tar") + +tar( + name = "data", + extension = "tar.gz", + srcs = glob( + [ + "output/bin/python{pyver}", + "output/lib/python{pyver}/**", + "output/lib/libpython{pyver}*", + ], + exclude = [ + "output/lib/python{pyver}/site-packages/pip/**", + "output/lib/python{pyver}/site-packages/pip-*.dist-info/**", + ], + ), + symlinks = {{ + "/python/bin/python": "python{pyver}", + "/python/bin/python3": "python{pyver}", + }}, + package_dir = "/python", + strip_prefix = "external/{canonical_name}/output" +) + +package_spdx( + name = "spdx", + package_name = "{package_name}", + version = "{version}", + spdx_id = "{spdx_id}", + sha256 = "{sha256}", + urls = [{urls}] +) + +merge_providers( + name = "{name}", + srcs = [":data", ":spdx"], + visibility = ["//visibility:public"], +) +""" + +def _impl(rctx): + rctx.report_progress("Fetching {}".format(rctx.attr.package_name)) + rctx.download_and_extract( + url = rctx.attr.urls, + sha256 = rctx.attr.sha256, + type = rctx.attr.type, + stripPrefix = rctx.attr.strip_prefix, + output = "output", + ) + rctx.file( + "BUILD.bazel", + content = BUILD_TMPL.format( + canonical_name = rctx.attr.name, + name = rctx.attr.name.split("+")[-1], + package_name = rctx.attr.package_name, + version = rctx.attr.version, + spdx_id = rctx.attr.name, + pyver = rctx.attr.python_version, + urls = ",".join(['"%s"' % url for url in rctx.attr.urls]), + sha256 = rctx.attr.sha256, + ), + ) + +python_archive = repository_rule( + implementation = _impl, + attrs = { + "urls": attr.string_list(mandatory = True), + "sha256": attr.string(mandatory = True), + "type": attr.string(default = ".tar.gz"), + "strip_prefix": attr.string(), + "package_name": attr.string(default = "python"), + "version": attr.string(mandatory = True), + # The x.y version used for binary and library paths, such as "3.14". + "python_version": attr.string(mandatory = True), + "architecture": attr.string(mandatory = True), + }, +) + +_PYTHON_VERSIONS_TMPL = """\ +"python versions" + +# AUTO GENERATED. DO NOT EDIT. +PYTHON_VERSIONS = {versions} +""" + +def _python_versions_repo_impl(rctx): + rctx.file("versions.bzl", _PYTHON_VERSIONS_TMPL.format(versions = str(rctx.attr.versions))) + rctx.file("BUILD.bazel", 'exports_files(["versions.bzl"])') + +python_versions_repo = repository_rule( + implementation = _python_versions_repo_impl, + attrs = { + "versions": attr.string_dict(), + }, +) + +def _python_impl(module_ctx): + mod = module_ctx.modules[0] + + if len(module_ctx.modules) > 1: + fail("python.archive should be called only once") + if not mod.is_root: + fail("python.archive should be called from root module only.") + + # Python from python-build-standalone (https://github.com/astral-sh/python-build-standalone) + # Release 20260901. Linux targets only (distroless images). + # Versions 3.13 and 3.14. New stable minors are added by the updater. + # armv7 is excluded: PBS publishes soft-float gnueabi builds + # (interpreter /lib/ld-linux.so.3) which cannot run on the distroless armhf base + # (loader /usr/lib/ld-linux-armhf.so.3); ppc64le is not published by PBS. + python_archive( + name = "python313_amd64", + sha256 = "0651dd7157d3debf769e15a52c1de9de7fbcdc36ba72faf79fde3c44f14d9461", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-x86_64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260901", + python_version = "3.13", + architecture = "amd64", + ) + + python_archive( + name = "python313_arm64", + sha256 = "76ed18125286d7dc96ce24023d1e319dbd55a89a767102411b1ea23846113f69", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-aarch64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260901", + python_version = "3.13", + architecture = "arm64", + ) + + python_archive( + name = "python313_s390x", + sha256 = "b738ddb7271b2591d6e89fcd4ef42b6cbbcac7c489daf807b6b01442ae065a0a", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-s390x-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260901", + python_version = "3.13", + architecture = "s390x", + ) + + python_archive( + name = "python313_riscv64", + sha256 = "584a5a197a3d1ce8d8e45496515802872f1c18c4d20ee54f088081f75e7e8b4b", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-riscv64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260901", + python_version = "3.13", + architecture = "riscv64", + ) + + python_archive( + name = "python314_amd64", + sha256 = "0ab3305457051cd3e7c031857e005f1bda17c218a1990567dacaaac6dd1d14f0", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-x86_64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260901", + python_version = "3.14", + architecture = "amd64", + ) + + python_archive( + name = "python314_arm64", + sha256 = "30f1cc489be654477d895b441e196bb080738bf0456da82080ad4ab66a22d80f", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-aarch64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260901", + python_version = "3.14", + architecture = "arm64", + ) + + python_archive( + name = "python314_s390x", + sha256 = "dc27e917b88db2560fe213244ffa6a656e8197ca0beb4ddeaaee5286041488ad", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-s390x-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260901", + python_version = "3.14", + architecture = "s390x", + ) + + python_archive( + name = "python314_riscv64", + sha256 = "cf71ad2f451ba9af30992391b2a403cddb6965f11ef5255ee2f5d3e504f75add", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-riscv64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260901", + python_version = "3.14", + architecture = "riscv64", + ) + + python_versions_repo( + name = "python_versions", + versions = { + "3.13_amd64": "3.13.15", + "3.13_arm64": "3.13.15", + "3.13_s390x": "3.13.15", + "3.13_riscv64": "3.13.15", + "3.14_amd64": "3.14.7", + "3.14_arm64": "3.14.7", + "3.14_s390x": "3.14.7", + "3.14_riscv64": "3.14.7", + }, + ) + + return module_ctx.extension_metadata( + root_module_direct_deps = [ + "python_versions", + "python313_amd64", + "python313_arm64", + "python313_s390x", + "python313_riscv64", + "python314_amd64", + "python314_arm64", + "python314_s390x", + "python314_riscv64", + ], + root_module_direct_dev_deps = [], + ) + +_archive = tag_class(attrs = {}) + +python = module_extension( + implementation = _python_impl, + tag_classes = { + "archive": _archive, + }, +) diff --git a/private/pkg/test/oci_image/BUILD.bazel b/private/pkg/test/oci_image/BUILD.bazel index 4753f6039..c5e862c8e 100644 --- a/private/pkg/test/oci_image/BUILD.bazel +++ b/private/pkg/test/oci_image/BUILD.bazel @@ -7,6 +7,7 @@ oci_image( name = "image_arm64", architecture = "arm64", os = "linux", + tags = ["debian12"], tars = [ deb.package("arm64", "debian12", "base-files"), ], @@ -21,6 +22,7 @@ oci_image( name = "image_amd64", architecture = "amd64", os = "linux", + tags = ["debian12"], tars = [ deb.package("amd64", "debian12", "netbase"), ], diff --git a/private/util/validate_usr_symlinks.bzl b/private/util/validate_usr_symlinks.bzl index 6faa55055..a2e3cdee8 100644 --- a/private/util/validate_usr_symlinks.bzl +++ b/private/util/validate_usr_symlinks.bzl @@ -11,6 +11,8 @@ load("@aspect_bazel_lib//lib:tar.bzl", "tar_lib") def _validate_usr_symlink_impl(target, ctx): if target.label.name.find("debian12") != -1: return [] + if hasattr(ctx.rule.attr, "tags") and "debian12" in ctx.rule.attr.tags: + return [] if not hasattr(ctx.rule.files, "tars"): return [] diff --git a/python/BUILD b/python/BUILD new file mode 100644 index 000000000..5e2324385 --- /dev/null +++ b/python/BUILD @@ -0,0 +1,77 @@ +load("//private/util:tar.bzl", "tar") +load(":config.bzl", "PYTHON_ARCHITECTURES", "PYTHON_DISTROS", "PYTHON_MAJOR_VERSIONS", "PYTHON_PACKAGES") +load(":ldconfig.bzl", "python_ldconfig") +load(":python.bzl", "python_image", "python_image_index") + +package(default_visibility = ["//visibility:public"]) + +sh_test( + name = "update_python_archives_test", + srcs = ["update_python_archives_test.sh"], + data = [ + ":config.bzl", + "//:MODULE.bazel", + "//knife.d:update_python_archives.sh", + "//private/extensions:python.bzl", + ], +) + +# Offline contract test for a complete PBS release update. +sh_test( + name = "update_python_archives_contract_test", + srcs = ["update_python_archives_contract_test.sh"], + data = [ + "testdata/python3.13.yaml", + "testdata/python3.14.yaml", + ":config.bzl", + ":gen_pbs_sbom.py", + ":pbs_embedded_versions.py", + ":update_python_archives_test.sh", + "//:MODULE.bazel", + "//knife.d:update_python_archives.sh", + "//private/extensions:python.bzl", + ], +) + +[ + python_image( + arch = arch, + distro = distro, + major_version = major_version, + packages = PYTHON_PACKAGES[distro], + ) + for distro in PYTHON_DISTROS + for major_version in PYTHON_MAJOR_VERSIONS + for arch in PYTHON_ARCHITECTURES[distro][major_version] +] + +[ + python_image_index( + architectures = PYTHON_ARCHITECTURES[distro][major_version], + distro = distro, + major_version = major_version, + ) + for distro in PYTHON_DISTROS + for major_version in PYTHON_MAJOR_VERSIONS +] + +# Ship an ld.so.cache per architecture so ctypes.util.find_library() works. +# Generate it by running ldconfig inside the image. +[ + tar( + name = "ldconfig_cache_{}".format(arch), + srcs = ["ldconfig/ld.so.cache.{}".format(arch)], + args = [ + "--format", + "gnutar", + ], + extension = "tar.gz", + mtree = ["etc/ld.so.cache uid=0 gid=0 uname=root gname=root mode=0644 time=0 type=file content=$(location ldconfig/ld.so.cache.{})".format(arch)], + ) + for arch in PYTHON_ARCHITECTURES["debian13"]["3.14"] +] + +python_ldconfig( + architectures = PYTHON_ARCHITECTURES["debian13"]["3.14"], + distro = "debian13", +) diff --git a/python/README.md b/python/README.md new file mode 100644 index 000000000..75736ff24 --- /dev/null +++ b/python/README.md @@ -0,0 +1,82 @@ +# Documentation for `python/` (python-build-standalone) + +## Image Contents + +These images contain a minimal Linux, Python-based runtime built from a +[python-build-standalone](https://github.com/astral-sh/python-build-standalone) +prebuilt tarball instead of the Debian Python package. + +Supported versions: + +- Python 3.13 (`python3.13-debian13`) +- Python 3.14 (`python3.14-debian13`) + +Each image contains everything in the [base image](../base/README.md), +plus a standalone CPython install under `/python`: + +- `/python/bin/python3.X` (with `/python/bin/python` and `/python/bin/python3` symlinks) +- `/python/lib/python3.X/` (standard library) +- `/python/lib/libpython3.X.so.1.0` + +No shell, include files, or `pip` command is included. The bundled `ensurepip` +wheel can bootstrap pip in a builder stage; install dependencies into +`site-packages` there to keep images reproducible. + +python-build-standalone builds are relocatable, dynamically link only to glibc, +and include selected Debian runtime files from the base image. + +## Usage + +The entrypoint of these images is set to `python3.X`, so they expect users to +supply a path to a .py file in the CMD. + +See the Python [Hello World](../examples/python3/) directory for an example. + +### Compatibility + +When using these images in a multi-stage build, use a build image with the same +Python version to avoid ABI-related errors. For example, when targeting +`python3.14-debian13`, use a `python:3.14-slim-trixie` (or python-build-standalone) +build stage. + +When building virtual environments, note that the distroless Python path is +`/python/bin/python3.14`. To ensure your virtual environment's internal links +are correct, your build environment should match this path. If your build image +uses a different path, create a symlink during the build stage, e.g.: + +```Dockerfile +RUN ln -s /usr/local/bin/python3.14 /python/bin/python3.14 +``` + +### ctypes.util.find_library + +`ctypes.util.find_library()` works for system libraries (e.g. `libc`, `libssl`, `libz`): +the image ships `/sbin/ldconfig` (from `libc-bin`) and a pre-generated `ld.so.cache` at +`/etc/ld.so.cache`. The cache is generated per arch by running ldconfig inside the image: + + bazel run //python:update_ldconfig + +and verified by `bazel test //python:check_ldconfig_tests` (also in CI). + +Libraries that python-build-standalone links statically into the interpreter +(sqlite3, ncurses, bz2, lzma, expat, zlib, ...) have no shared object to find — +the corresponding stdlib modules work out of the box. `python -m venv` works and +bootstraps pip from the bundled `ensurepip` wheels. + +## Provenance + +Python comes from a [python-build-standalone](https://github.com/astral-sh/python-build-standalone) +prebuilt tarball (`cpython-3.X.Y+--unknown-linux-gnu-install_only.tar.gz`). +The exact release, versions, and SHA-256 digests are pinned in the extension +definition and updated via `knife update-python-archives`. +A generated SPDX SBOM for the pinned release is committed at `python/pbs-sbom.spdx.json` +(regenerated by the updater; it records PBS manifest provenance, including +native libraries statically embedded in the interpreter). It is not an image +SBOM or a complete inventory of the finished binary. + +When a release changes, the updater checks detectable native-library version +markers in the x86_64 install tarball against the manifest. This supplementary +check does not establish complete coverage of statically embedded code. + +For an image SBOM, use the existing Bazel image SBOM output. PBS does not +currently provide an authoritative SBOM for its statically linked components. diff --git a/python/config.bzl b/python/config.bzl new file mode 100644 index 000000000..769880bf1 --- /dev/null +++ b/python/config.bzl @@ -0,0 +1,18 @@ +"python configurations" + +PYTHON_DISTROS = ["debian13"] + +# Full CPython versions are defined in the extension. +PYTHON_MAJOR_VERSIONS = ["3.13", "3.14"] + +PYTHON_ARCHITECTURES = { + "debian13": { + "3.13": ["amd64", "arm64", "s390x", "riscv64"], + "3.14": ["amd64", "arm64", "s390x", "riscv64"], + }, +} + +# libc-bin provides ldconfig for the generated library cache. +PYTHON_PACKAGES = { + "debian13": ["libc-bin"], +} diff --git a/python/gen_pbs_sbom.py b/python/gen_pbs_sbom.py new file mode 100644 index 000000000..13012eb39 --- /dev/null +++ b/python/gen_pbs_sbom.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Emit an SPDX 2.3 SBOM for a python-build-standalone release. + +Reads the release component manifest and lists runtime components: the CPython +versions, bundled ensurepip wheel, and native libraries statically linked into +the interpreters. Build-time-only tools (autoconf, binutils, musl, llvm, ...) +are excluded. This is PBS release provenance, not a complete image SBOM. + +Usage: gen_pbs_sbom.py +""" +import ast +import datetime +import json +import sys + +LICENSE_REF = {"bzip2-1.0.6": "LicenseRef-bzip2-1.0.6"} +# Bundled PyPI components whose licenses are absent from the manifest. +# These stable, well-known values are used only as fallbacks. +PYPI_LICENSE = {"pip": "MIT"} + + +def declared(name, entry): + ids = entry.get("licenses") or [] + if not ids and name in PYPI_LICENSE: + ids = [PYPI_LICENSE[name]] + out = [LICENSE_REF.get(i) or i for i in ids] + if not ids and entry.get("library_names"): + out = ["LicenseRef-Public-Domain"] # SQLite has no SPDX identifier for public domain. + return " AND ".join(out) if out else "NOASSERTION" + + +def purl(name, version): + if name in PYPI_LICENSE: + return "pkg:pypi/{}@{}".format(name, version) + # C libraries built from source tarballs use generic purls for provenance. + return "pkg:generic/{}@{}".format(name, version) + + +def is_runtime(name, entry): + # The pip wheel supports ensurepip but is not installed as a command; + # setuptools is a PBS build-time tool and is not shipped in install_only. + return name.startswith("cpython") or name == "pip" or bool(entry.get("library_names")) + + +def load_downloads(path): + with open(path, encoding="utf-8") as fh: + tree = ast.parse(fh.read(), filename=path) + for node in tree.body: + if isinstance(node, ast.Assign): + targets = node.targets + elif isinstance(node, ast.AnnAssign): + targets = [node.target] + else: + continue + if any(isinstance(target, ast.Name) and target.id == "DOWNLOADS" for target in targets): + downloads = ast.literal_eval(node.value) + if isinstance(downloads, dict): + return downloads + break + raise ValueError("DOWNLOADS must be a literal dictionary") + + +def main(): + manifest, release, output = sys.argv[1], sys.argv[2], sys.argv[3] + try: + downloads = load_downloads(manifest) + except (OSError, SyntaxError, ValueError) as err: + sys.exit("cannot parse manifest {}: {}".format(manifest, err)) + + doc = { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "python-build-standalone-{}-linux".format(release), + "documentNamespace": "https://github.com/astral-sh/python-build-standalone/releases/tag/{}/spdx.json".format(release), + "creationInfo": { + "created": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "creators": ["Tool: distroless-pbs-sbom"], + }, + "packages": [ + { + "name": "python-build-standalone", + "SPDXID": "SPDXRef-PBS", + "versionInfo": release, + "downloadLocation": "https://github.com/astral-sh/python-build-standalone/releases/tag/{}".format(release), + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "supplier": "Organization: astral-sh", + "filesAnalyzed": False, + } + ], + "relationships": [], + } + for name, entry in sorted(downloads.items()): + if not is_runtime(name, entry): + continue + pid = "SPDXRef-" + name.replace("-", "_").replace(".", "_") + # Prefer actual_version; SQLite also stores SQLITE_VERSION_NUMBER in version. + version = str(entry.get("actual_version") or entry.get("version", "")) + doc["packages"].append( + { + "name": name, + "SPDXID": pid, + "versionInfo": version, + "downloadLocation": entry.get("url", "NOASSERTION"), + "licenseConcluded": "NOASSERTION", + "licenseDeclared": declared(name, entry), + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": purl(name, version), + } + ], + "checksums": [{"algorithm": "SHA256", "checksumValue": entry.get("sha256", "")}], + "filesAnalyzed": False, + } + ) + doc["relationships"].append( + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": pid, + } + ) + with open(output, "w") as fh: + json.dump(doc, fh, indent=1) + fh.write("\n") + + +if __name__ == "__main__": + main() diff --git a/python/ldconfig.bzl b/python/ldconfig.bzl new file mode 100644 index 000000000..773f2541f --- /dev/null +++ b/python/ldconfig.bzl @@ -0,0 +1,71 @@ +"Macros for generating ldconfig cache for python images" + +load("@rules_oci//oci:defs.bzl", "oci_load") + +def python_ldconfig(architectures, distro): + """Generate ldconfig cache targets for Python images. + + Args: + architectures: architectures to support + distro: distribution name, such as debian13 + """ + + for arch in architectures: + oci_load( + name = "load_python314_root_{}_{}".format(arch, distro), + image = ":python314_root_{}_{}".format(arch, distro), + repo_tags = ["bazel/python:python314_root_{}_{}".format(arch, distro)], + ) + + for arch in architectures: + native.genrule( + name = "do_load_{}".format(arch), + outs = ["do_load_{}.done".format(arch)], + cmd = "$(location :load_python314_root_{arch}_{distro}) && touch $@".format( + arch = arch, + distro = distro, + ), + tags = [ + "local", + "no-sandbox", + ], + tools = [":load_python314_root_{}_{}".format(arch, distro)], + ) + + native.sh_binary( + name = "update_ldconfig", + srcs = ["ldconfig/ldconfig.sh"], + args = ["update"] + architectures, + data = ["ldconfig/ldconfig.sh"] + [":do_load_{}".format(arch) for arch in architectures], + tags = ["local"], + ) + + for arch in architectures: + native.sh_test( + name = "check_ldconfig_{}_test".format(arch), + srcs = ["ldconfig/ldconfig.sh"], + args = [ + "check", + arch, + ], + data = [ + "ldconfig/ld.so.cache.{}".format(arch), + ":do_load_{}".format(arch), + ], + tags = [ + arch, + "local", + "manual", + "external", + "no-cache", + ], + ) + + native.test_suite( + name = "check_ldconfig_tests", + tests = [ + ":check_ldconfig_{}_test".format(arch) + for arch in architectures + ], + tags = ["manual"], + ) diff --git a/python/ldconfig/ld.so.cache.amd64 b/python/ldconfig/ld.so.cache.amd64 new file mode 100644 index 000000000..e50ae2e20 Binary files /dev/null and b/python/ldconfig/ld.so.cache.amd64 differ diff --git a/python/ldconfig/ld.so.cache.arm64 b/python/ldconfig/ld.so.cache.arm64 new file mode 100644 index 000000000..161a442d6 Binary files /dev/null and b/python/ldconfig/ld.so.cache.arm64 differ diff --git a/python/ldconfig/ld.so.cache.riscv64 b/python/ldconfig/ld.so.cache.riscv64 new file mode 100644 index 000000000..b21da562c Binary files /dev/null and b/python/ldconfig/ld.so.cache.riscv64 differ diff --git a/python/ldconfig/ld.so.cache.s390x b/python/ldconfig/ld.so.cache.s390x new file mode 100644 index 000000000..73d4ee695 Binary files /dev/null and b/python/ldconfig/ld.so.cache.s390x differ diff --git a/python/ldconfig/ldconfig.sh b/python/ldconfig/ldconfig.sh new file mode 100755 index 000000000..ad78c1214 --- /dev/null +++ b/python/ldconfig/ldconfig.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -eo pipefail + +MODE=$1 +shift + +if [[ "${MODE}" != "update" && "${MODE}" != "check" ]]; then + echo "Usage: $0 ..." >&2 + exit 1 +fi + +ROOT="python/ldconfig" +if [[ -n "${BUILD_WORKSPACE_DIRECTORY}" ]]; then + cd "${BUILD_WORKSPACE_DIRECTORY}" +elif [[ ! -d "${ROOT}" ]]; then + ROOT=$(dirname "$0") +fi + +CID="" +TEMP_DIR="" +cleanup() { + local exit_status=$? + if [[ -n "${CID}" ]]; then docker rm -f "${CID}"; fi + if [[ -n "${TEMP_DIR}" ]]; then rm -rf "${TEMP_DIR}"; fi + exit "${exit_status}" +} +trap cleanup EXIT + +for arch in "$@"; do + if [ "$MODE" = "update" ]; then echo "Updating ldconfig cache for ${arch}"; else echo "Checking ldconfig cache for ${arch}"; fi + + CID=$(docker create --platform "linux/${arch}" --entrypoint /sbin/ldconfig "bazel/python:python314_root_${arch}_debian13") + docker start -a "${CID}" + + if [[ "${MODE}" == "update" ]]; then + docker cp "${CID}:/etc/ld.so.cache" "${ROOT}/ld.so.cache.${arch}" + echo "Updated ${arch} cache." + else + TEMP_DIR=$(mktemp -d) + docker cp "${CID}:/etc/ld.so.cache" "${TEMP_DIR}/ld.so.cache.${arch}" + if ! cmp -s "${TEMP_DIR}/ld.so.cache.${arch}" "${ROOT}/ld.so.cache.${arch}"; then + echo "ERROR: ldconfig cache for ${arch} is out of date." >&2 + echo "Run the ldconfig update command to regenerate it." >&2 + exit 1 + fi + rm -rf "${TEMP_DIR}" + TEMP_DIR="" + echo "ldconfig cache for ${arch} is up to date." + fi + docker rm -f "${CID}" + CID="" +done diff --git a/python/pbs-sbom.spdx.json b/python/pbs-sbom.spdx.json new file mode 100644 index 000000000..fc4902096 --- /dev/null +++ b/python/pbs-sbom.spdx.json @@ -0,0 +1,810 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "python-build-standalone-20260901-linux", + "documentNamespace": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260901/spdx.json", + "creationInfo": { + "created": "2026-09-02T05:57:59Z", + "creators": [ + "Tool: distroless-pbs-sbom" + ] + }, + "packages": [ + { + "name": "python-build-standalone", + "SPDXID": "SPDXRef-PBS", + "versionInfo": "20260901", + "downloadLocation": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260901", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "supplier": "Organization: astral-sh", + "filesAnalyzed": false + }, + { + "name": "bdb", + "SPDXID": "SPDXRef-bdb", + "versionInfo": "6.0.19", + "downloadLocation": "https://ftp.osuosl.org/pub/blfs/conglomeration/db/db-6.0.19.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Sleepycat", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/bdb@6.0.19" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2917c28f60903908c2ca4587ded1363b812c4e830a5326aaa77c9879d13ae18e" + } + ], + "filesAnalyzed": false + }, + { + "name": "bzip2", + "SPDXID": "SPDXRef-bzip2", + "versionInfo": "1.0.8", + "downloadLocation": "https://astral-sh.github.io/mirror/files/bzip2-1.0.8.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "LicenseRef-bzip2-1.0.6", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/bzip2@1.0.8" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.10", + "SPDXID": "SPDXRef-cpython_3_10", + "versionInfo": "3.10.21", + "downloadLocation": "https://www.python.org/ftp/python/3.10.21/Python-3.10.21.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.10@3.10.21" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a0da1e72132e950154eca0f6f47d5db828454700de20e5113667940d81e0db04" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.11", + "SPDXID": "SPDXRef-cpython_3_11", + "versionInfo": "3.11.16", + "downloadLocation": "https://www.python.org/ftp/python/3.11.16/Python-3.11.16.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.11@3.11.16" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "91bcdebfdde239a003ae93738a7fce0f9230fee5c4bc2b86f6e6e8c6f98aabe8" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.12", + "SPDXID": "SPDXRef-cpython_3_12", + "versionInfo": "3.12.14", + "downloadLocation": "https://www.python.org/ftp/python/3.12.14/Python-3.12.14.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.12@3.12.14" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "5c8462af5790baf43a321a1559dbe0db06d1be4300fb85fb53c40060668e548a" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.13", + "SPDXID": "SPDXRef-cpython_3_13", + "versionInfo": "3.13.15", + "downloadLocation": "https://www.python.org/ftp/python/3.13.15/Python-3.13.15.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.13@3.13.15" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "1e66a7945a48390ee4c2a4268a0e4185884059a13c4aab6d148aa208deea4a76" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.14", + "SPDXID": "SPDXRef-cpython_3_14", + "versionInfo": "3.14.7", + "downloadLocation": "https://www.python.org/ftp/python/3.14.7/Python-3.14.7.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.14@3.14.7" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "3b48dac8fb59f62eaa67ac83c1eb12bda1b7a08406dd286e252c11a66be27f81" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.15", + "SPDXID": "SPDXRef-cpython_3_15", + "versionInfo": "3.15.0rc2", + "downloadLocation": "https://www.python.org/ftp/python/3.15.0/Python-3.15.0rc2.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.15@3.15.0rc2" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "8d93af5eaaaea5adfd41bd786a7ba3f03f2ad1ab57c6a65e0b963deab91d5ad7" + } + ], + "filesAnalyzed": false + }, + { + "name": "expat", + "SPDXID": "SPDXRef-expat", + "versionInfo": "2.8.4", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_4/expat-2.8.4.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/expat@2.8.4" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "656ae1cc8da3b4ea513bb4e254f33e6243938084c0ec6239da873376b09985a7" + } + ], + "filesAnalyzed": false + }, + { + "name": "libX11", + "SPDXID": "SPDXRef-libX11", + "versionInfo": "1.8.13", + "downloadLocation": "https://www.x.org/releases/individual/lib/libX11-1.8.13.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT AND X11", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libX11@1.8.13" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "acf0e7cd7541110e6330ecb539441a2d53061f386ec7be6906dfde0de2598470" + } + ], + "filesAnalyzed": false + }, + { + "name": "libXau", + "SPDXID": "SPDXRef-libXau", + "versionInfo": "1.0.12", + "downloadLocation": "https://www.x.org/releases/individual/lib/libXau-1.0.12.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libXau@1.0.12" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2402dd938da4d0a332349ab3d3586606175e19cb32cb9fe013c19f1dc922dcee" + } + ], + "filesAnalyzed": false + }, + { + "name": "libedit", + "SPDXID": "SPDXRef-libedit", + "versionInfo": "20240808-3.1", + "downloadLocation": "https://thrysoee.dk/editline/libedit-20240808-3.1.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-3-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libedit@20240808-3.1" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "5f0573349d77c4a48967191cdd6634dd7aa5f6398c6a57fe037cc02696d6099f" + } + ], + "filesAnalyzed": false + }, + { + "name": "libffi", + "SPDXID": "SPDXRef-libffi", + "versionInfo": "3.4.8", + "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.4.8/libffi-3.4.8.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libffi@3.4.8" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "bc9842a18898bfacb0ed1252c4febcc7e78fa139fd27fdc7a3e30d9d9356119b" + } + ], + "filesAnalyzed": false + }, + { + "name": "libxcb", + "SPDXID": "SPDXRef-libxcb", + "versionInfo": "1.17.0", + "downloadLocation": "https://xcb.freedesktop.org/dist/libxcb-1.17.0.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libxcb@1.17.0" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2c69287424c9e2128cb47ffe92171e10417041ec2963bceafb65cb3fcf8f0b85" + } + ], + "filesAnalyzed": false + }, + { + "name": "mpdecimal", + "SPDXID": "SPDXRef-mpdecimal", + "versionInfo": "4.0.0", + "downloadLocation": "https://astral-sh.github.io/mirror/files/mpdecimal-4.0.0.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-2-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/mpdecimal@4.0.0" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c" + } + ], + "filesAnalyzed": false + }, + { + "name": "ncurses", + "SPDXID": "SPDXRef-ncurses", + "versionInfo": "6.5", + "downloadLocation": "https://astral-sh.github.io/mirror/files/ncurses-6.5.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "X11", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/ncurses@6.5" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6" + } + ], + "filesAnalyzed": false + }, + { + "name": "openssl-1.1", + "SPDXID": "SPDXRef-openssl_1_1", + "versionInfo": "1.1.1w", + "downloadLocation": "https://www.openssl.org/source/openssl-1.1.1w.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "OpenSSL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/openssl-1.1@1.1.1w" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8" + } + ], + "filesAnalyzed": false + }, + { + "name": "openssl-3.5", + "SPDXID": "SPDXRef-openssl_3_5", + "versionInfo": "3.5.8", + "downloadLocation": "https://github.com/openssl/openssl/releases/download/openssl-3.5.8/openssl-3.5.8.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Apache-2.0", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/openssl-3.5@3.5.8" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a8f84a39918ec6415ce765d9b429d313ba97b8143169c172e734b9514464f5b2" + } + ], + "filesAnalyzed": false + }, + { + "name": "pip", + "SPDXID": "SPDXRef-pip", + "versionInfo": "26.2.1", + "downloadLocation": "https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:pypi/pip@26.2.1" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e" + } + ], + "filesAnalyzed": false + }, + { + "name": "sqlite", + "SPDXID": "SPDXRef-sqlite", + "versionInfo": "3.53.1.0", + "downloadLocation": "https://www.sqlite.org/2026/sqlite-autoconf-3530100.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "LicenseRef-Public-Domain", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/sqlite@3.53.1.0" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "83e6b2020a034e9a7ad4a72feea59e1ad52f162e09cbd26735a3ffb98359fc4f" + } + ], + "filesAnalyzed": false + }, + { + "name": "tcl", + "SPDXID": "SPDXRef-tcl", + "versionInfo": "9.0.4", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tcl9.0.4-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tcl@9.0.4" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d0aed49230bc02a65c1e0229e65f34590a4b037ec40d546f32573b467f7551ea" + } + ], + "filesAnalyzed": false + }, + { + "name": "tcl-8612", + "SPDXID": "SPDXRef-tcl_8612", + "versionInfo": "8.6.12", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tcl8.6.12-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tcl-8612@8.6.12" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "26c995dd0f167e48b11961d891ee555f680c175f7173ff8cb829f4ebcde4c1a6" + } + ], + "filesAnalyzed": false + }, + { + "name": "tk", + "SPDXID": "SPDXRef-tk", + "versionInfo": "9.0.4", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tk9.0.4-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tk@9.0.4" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d7a146d2917eb8b5cc95276dbf0e3d03c7464d2b19c1675357857c989301dbb4" + } + ], + "filesAnalyzed": false + }, + { + "name": "tk-8612", + "SPDXID": "SPDXRef-tk_8612", + "versionInfo": "8.6.12", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tk8.6.12-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tk-8612@8.6.12" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "12395c1f3fcb6bed2938689f797ea3cdf41ed5cb6c4766eec8ac949560310630" + } + ], + "filesAnalyzed": false + }, + { + "name": "uuid", + "SPDXID": "SPDXRef-uuid", + "versionInfo": "1.0.3", + "downloadLocation": "https://sourceforge.net/projects/libuuid/files/libuuid-1.0.3.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-3-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/uuid@1.0.3" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "46af3275291091009ad7f1b899de3d0cea0252737550e7919d17237997db5644" + } + ], + "filesAnalyzed": false + }, + { + "name": "xz", + "SPDXID": "SPDXRef-xz", + "versionInfo": "5.8.3", + "downloadLocation": "https://github.com/tukaani-project/xz/releases/download/v5.8.3/xz-5.8.3.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "0BSD", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/xz@5.8.3" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "3d3a1b973af218114f4f889bbaa2f4c037deaae0c8e815eec381c3d546b974a0" + } + ], + "filesAnalyzed": false + }, + { + "name": "zlib", + "SPDXID": "SPDXRef-zlib", + "versionInfo": "1.3.2", + "downloadLocation": "https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Zlib", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/zlib@1.3.2" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "bb329a0a2cd0274d05519d61c667c062e06990d72e125ee2dfa8de64f0119d16" + } + ], + "filesAnalyzed": false + }, + { + "name": "zlib-ng", + "SPDXID": "SPDXRef-zlib_ng", + "versionInfo": "2.2.4", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Zlib", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/zlib-ng@2.2.4" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004" + } + ], + "filesAnalyzed": false + }, + { + "name": "zstd", + "SPDXID": "SPDXRef-zstd", + "versionInfo": "1.5.7", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-3-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/zstd@1.5.7" + } + ], + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e" + } + ], + "filesAnalyzed": false + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-bdb" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-bzip2" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_10" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_11" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_12" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_13" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_14" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_15" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-expat" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libX11" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libXau" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libedit" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libffi" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libxcb" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-mpdecimal" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-ncurses" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-openssl_1_1" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-openssl_3_5" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-pip" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-sqlite" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tcl" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tcl_8612" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tk" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tk_8612" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-uuid" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-xz" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-zlib" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-zlib_ng" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-zstd" + } + ] +} diff --git a/python/pbs_embedded_versions.py b/python/pbs_embedded_versions.py new file mode 100644 index 000000000..73f99f524 --- /dev/null +++ b/python/pbs_embedded_versions.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Verify native libraries embedded in a PBS libpython shared object. + +The PBS manifest lists source components, while selected libraries are statically +linked into libpython. Compare detectable version markers with the manifest. + +Usage: pbs_embedded_versions.py +""" +import ast +import re +import sys + + +# Manifest key to a version-marker pattern with one capture group. +STRONG = [ + ("openssl-3.5", re.compile(rb"OpenSSL (\d+\.\d+\.\d+[a-z]?)\s+\d{1,2} [A-Z][a-z]{2} \d{4}")), + ("openssl-1.1", re.compile(rb"OpenSSL (1\.1\.1[a-z]?)\s+\d{1,2} [A-Z][a-z]{2} \d{4}")), + ("zlib", re.compile(rb"deflate (\d+\.\d+\.\d+) Copyright")), + ("expat", re.compile(rb"expat_(\d+\.\d+\.\d+)")), + ("ncurses", re.compile(rb"ncurses (\d+\.\d+\.\d+(?:\.\d+)?)")), + ("bzip2", re.compile(rb"(\d+\.\d+\.\d+), \d{1,2}-[A-Z][a-z]{2}-\d{4}")), +] +WEAK = ("sqlite", "xz", "zstd", "mpdecimal") + + +def load_manifest(path): + with open(path, encoding="utf-8") as fh: + tree = ast.parse(fh.read(), filename=path) + for node in tree.body: + if isinstance(node, ast.Assign): + targets = node.targets + elif isinstance(node, ast.AnnAssign): + targets = [node.target] + else: + continue + if any(isinstance(target, ast.Name) and target.id == "DOWNLOADS" for target in targets): + downloads = ast.literal_eval(node.value) + if isinstance(downloads, dict): + return downloads + break + raise ValueError("DOWNLOADS must be a literal dictionary") + + +def expected_version(entry, name=None): + value = entry.get("actual_version") or entry.get("version") or "" + if name == "sqlite": + return re.sub(r"\.0$", "", value) + return value + + +def marker_version(blob, regex): + match = regex.search(blob) + return match.group(1).decode() if match else None + + +def weak_version(blob, name, entry): + expected = expected_version(entry, name) + pattern = re.compile(rb"(?I", 42), + lambda: importlib.import_module("csv").reader(["a,b"]).__next__(), + lambda: importlib.import_module("re").match(r"\d+", "123x").group(), + lambda: importlib.import_module("string").ascii_letters.__len__(), + lambda: importlib.import_module("ssl").create_default_context(), + lambda: importlib.import_module("ssl").get_default_verify_paths(), + lambda: importlib.import_module("socket").getaddrinfo("localhost", 80), + lambda: importlib.import_module("urllib.request").urlopen("https://example.com", timeout=10).status, + lambda: importlib.import_module("subprocess").check_output([sys.executable, "-c", "print(1)"]).strip(), + lambda: importlib.import_module("os").getpid(), + lambda: importlib.import_module("math").sqrt(2), + lambda: importlib.import_module("statistics").mean([1, 2, 3, 4]), + lambda: importlib.import_module("fractions").Fraction(1, 3) * 3, + lambda: importlib.import_module("calendar").isleap(2024), + lambda: importlib.import_module("html").escape("&"), + lambda: importlib.import_module("glob").glob("/python/bin/*").__len__(), + lambda: importlib.import_module("shutil").which(os.path.basename(sys.executable)), + lambda: importlib.import_module("tempfile").gettempdir(), + lambda: importlib.import_module("threading").current_thread().name, + lambda: importlib.import_module("queue").Queue().empty(), + lambda: importlib.import_module("concurrent.futures").ThreadPoolExecutor(max_workers=1).submit(lambda: 1).result(), +] + +for i, call in enumerate(calls): + try: + call() + except Exception as e: + failed.append("call %d: %s: %s" % (i, type(e).__name__, e)) + +if failed: + for f in failed: + print("FAIL " + f) + print("FAILED %d" % len(failed)) + sys.exit(1) + +print("calls OK: %d" % len(calls)) +print("ALL GOOD on Python %s (%s)" % (sys.version.split()[0], sys.platform)) diff --git a/python/update_python_archives_contract_test.sh b/python/update_python_archives_contract_test.sh new file mode 100755 index 000000000..1be6cfd94 --- /dev/null +++ b/python/update_python_archives_contract_test.sh @@ -0,0 +1,427 @@ +#!/usr/bin/env bash +# Black-box contract test for the Python updater. +# +# Contract: after a new PBS release, the updater rewrites all files needed to +# build consistent images and converge: +# - archive blocks and version data +# - the matrix and repository visibility for selected minors +# - version-specific testdata +# - a second run returns NO_CHANGE (the updater reaches a fixed point) +# The test runs offline; fixture data is supplied through PBS_RELEASE_FILE, +# PBS_SHA256SUMS_FILE, PBS_DOWNLOADS_FILE, PBS_TARBALL_FILE, or PBS_TARBALL_DIR. +set -euo pipefail + +cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" + +FIX=$(mktemp -d) +trap 'rm -rf "$FIX"' EXIT + +# Copy every file mutated by the updater into the test workspace. +mkdir -p "$FIX/private/extensions" "$FIX/python/testdata" +cp knife.d/update_python_archives.sh "$FIX/" +cp private/extensions/python.bzl "$FIX/private/extensions/" +cp python/config.bzl "$FIX/python/" +cp MODULE.bazel "$FIX/" +cp python/update_python_archives_test.sh "$FIX/python/" +cp python/gen_pbs_sbom.py "$FIX/python/" +cp python/testdata/python3.13.yaml python/testdata/python3.14.yaml "$FIX/python/testdata/" +cp python/pbs_embedded_versions.py "$FIX/python/" + +# Minimal PBS component manifest for SBOM generation. +cat > "$FIX/downloads.py" <<'EOF' +DOWNLOADS = { + "cpython-3.14": { + "url": "https://www.python.org/ftp/python/3.14.7/Python-3.14.7.tar.xz", + "version": "3.14.7", + "licenses": ["Python-2.0"], + }, + "expat": { + "url": "https://example.invalid/expat.tar.gz", + "version": "2.8.3", + "licenses": ["MIT"], + "library_names": ["expat"], + }, + "mpdecimal": { + "url": "https://example.invalid/mpdecimal.tar.gz", + "version": "4.0.0", + "library_names": ["mpdecimal"], + }, + "sqlite": { + "url": "https://example.invalid/sqlite.tar.gz", + "version": "3530100", + "actual_version": "3.53.1.0", + "library_names": ["sqlite3"], + }, + "zlib": { + "url": "https://example.invalid/zlib.tar.gz", + "version": "1.3.2", + "licenses": ["Zlib"], + "library_names": ["z"], + }, + "zstd": { + "url": "https://example.invalid/zstd.tar.gz", + "version": "1.5.7", + "library_names": ["zstd"], + }, +} +EOF + +# Minimal PBS install tarball containing the markers checked by the verifier. +make_tarball() { # $1 = output path; markers must match the fixture manifest. + python3 - "$1" <<'PYEOF' +import sys, tarfile, io +blob = ( + b"OpenSSL 3.5.7 9 Jun 2026\n" + b"deflate 1.3.2 Copyright 1995-2026 Jean-loup Gailly and Mark Adler\n" + b"expat_2.8.3\n" + b"ncurses 6.5.20240427\n" + b"1.0.8, 13-Jul-2019\n" + b"3.53.1\n5.8.3\n" + b"1.5.7\n4.0.0\n" +) +with tarfile.open(sys.argv[1], "w:gz") as tar: + info = tarfile.TarInfo("python/lib/libpython3.13.so.1.0") + info.size = len(blob) + tar.addfile(info, io.BytesIO(blob)) + link = tarfile.TarInfo("python/lib/libpython3.13.so") + link.type = tarfile.SYMTYPE + link.linkname = "libpython3.13.so.1.0" + tar.addfile(link) +PYEOF +} +make_tarball "$FIX/tarball.tar.gz" + +cd "$FIX" +source update_python_archives.sh + +python3 - <<'PY' +import sys +sys.path.insert(0, "python") +from pbs_embedded_versions import expected_version, weak_version +assert expected_version({"actual_version": "3.53.1.0"}, "sqlite") == "3.53.1" +assert expected_version({"version": "4.0.0"}, "mpdecimal") == "4.0.0" +assert weak_version(b"zstd 1.5.7", "zstd", {"version": "1.5.7"}) == "1.5.7" +PY + +# --- fixtures --------------------------------------------------------------- +# Test tarballs use placeholder hashes because PBS_TARBALL_DIR skips downloads. +SHA_A=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +SHA_B=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +SHA_C=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +SHA_D=dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd +TRIPLES=(x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu s390x-unknown-linux-gnu riscv64-unknown-linux-gnu) + +make_sha256sums() { # Arguments: release, 3.13 patch, 3.14 patch, optional 3.15 patches. + local release=$1 p313=$2 p314=$3 p315=${4:-} p315_alt=${5:-} + : > SHA256SUMS + for t in "${TRIPLES[@]}"; do + echo "$SHA_A cpython-${p313}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + echo "$SHA_B cpython-${p314}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + if [ -n "$p315" ]; then + echo "$SHA_C cpython-${p315}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + fi + if [ -n "$p315_alt" ]; then + echo "$SHA_D cpython-${p315_alt}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + fi + done + rm -rf "$FIX/tarballs" + mkdir -p "$FIX/tarballs" + for t in "${TRIPLES[@]}"; do + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-${p313}+${release}-${t}-install_only.tar.gz" + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-${p314}+${release}-${t}-install_only.tar.gz" + if [ -n "$p315" ]; then + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-${p315}+${release}-${t}-install_only.tar.gz" + fi + if [ -n "$p315_alt" ]; then + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-${p315_alt}+${release}-${t}-install_only.tar.gz" + fi + done +} + +make_numeric_fill_sha256sums() { + local release=$1 + : > SHA256SUMS + for t in "${TRIPLES[@]}"; do + echo "$SHA_A cpython-3.9.9+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + echo "$SHA_B cpython-3.10.1+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + echo "$SHA_D cpython-3.10.1rc1+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + done + rm -rf "$FIX/tarballs" + mkdir -p "$FIX/tarballs" + for t in "${TRIPLES[@]}"; do + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-3.9.9+${release}-${t}-install_only.tar.gz" + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-3.10.1+${release}-${t}-install_only.tar.gz" + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-3.10.1rc1+${release}-${t}-install_only.tar.gz" + done +} + +make_prerelease_only_sha256sums() { + local release=$1 + : > SHA256SUMS + for t in "${TRIPLES[@]}"; do + echo "$SHA_C cpython-3.15.0rc1+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + done + rm -rf "$FIX/tarballs" + mkdir -p "$FIX/tarballs" + for t in "${TRIPLES[@]}"; do + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-3.15.0rc1+${release}-${t}-install_only.tar.gz" + done +} + +run_updater() { # Print stdout and fail the test on a nonzero exit. + # Run in bash -c so an updater failure does not terminate this test script. + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" +} + +run_updater_expect_fail() { # A nonzero exit is expected for drift. + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_FILE="$FIX/drift.tar.gz" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" && return 1 || return 0 +} + +run_updater_path_traversal() { + TMPDIR="$FIX/tmp" PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_FILE="$FIX/path-traversal.tar.gz" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" && return 1 || return 0 +} + +run_updater_noop() { + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/missing-downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" +} + +run_updater_bad_manifest() { + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/bad-downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" +} + +run_updater_required_asset_fail() { + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" && return 1 || return 0 +} + +enable_315() { + python3 - <<'PYEOF' +from pathlib import Path + +path = Path("python/config.bzl") +text = path.read_text() +text = text.replace( + 'PYTHON_MAJOR_VERSIONS = ["3.13", "3.14"]', + 'PYTHON_MAJOR_VERSIONS = ["3.13", "3.14", "3.15"]', +) +text = text.replace( + ' "3.14": ["amd64", "arm64", "s390x", "riscv64"],\n', + ' "3.14": ["amd64", "arm64", "s390x", "riscv64"],\n' + ' "3.15": ["amd64", "arm64", "s390x", "riscv64"],\n', +) +path.write_text(text) +PYEOF +} + +enable_315_without_arches() { + python3 - <<'PYEOF' +from pathlib import Path + +path = Path("python/config.bzl") +text = path.read_text().replace( + 'PYTHON_MAJOR_VERSIONS = ["3.13", "3.14"]', + 'PYTHON_MAJOR_VERSIONS = ["3.13", "3.14", "3.15"]', +) +path.write_text(text) +PYEOF +} + +disable_315() { + python3 - <<'PYEOF' +from pathlib import Path + +path = Path("python/config.bzl") +text = path.read_text() +text = text.replace(', "3.15"]', ']') +text = text.replace( + ' "3.15": ["amd64", "arm64", "s390x", "riscv64"],\n', + '', +) +path.write_text(text) +PYEOF +} + +# --- tag-only bump (new release, same patches) ------------------------------- +echo '{"tag": "20990101"}' > release.json +make_sha256sums 20990101 3.13.15 3.14.7 +[ "$(run_updater)" = "20990101" ] || { echo "phase A: expected update to 20990101"; cat "$FIX/updater.err"; exit 1; } +grep -q 'releases/download/20990101/' private/extensions/python.bzl || { echo "phase A: archives not re-pointed at new release"; exit 1; } +! grep -q '20260807' private/extensions/python.bzl || { echo "phase A: old release still referenced"; exit 1; } +grep -q '3.13.15+20990101' private/extensions/python.bzl || { echo "phase A: archive version lacks new release tag"; exit 1; } +grep -q '"3.13_amd64": "3.13.15"' private/extensions/python.bzl || { echo "phase A: versions dict must not change on a tag-only bump"; exit 1; } +grep -q 'Python 3.13.15' python/testdata/python3.13.yaml || { echo "phase A: testdata must not change on a tag-only bump"; exit 1; } +grep -q 'OK.*zstd.*1.5.7' "$FIX/updater.err" || { echo "phase A: zstd marker not checked"; exit 1; } +grep -q 'OK.*mpdecimal.*4.0.0' "$FIX/updater.err" || { echo "phase A: mpdecimal marker not checked"; exit 1; } +if run_updater_noop; then + echo "phase A: no-op must inspect PBS artifacts" + exit 1 +fi +printf 'DOWNLOADS = [' > "$FIX/bad-downloads.py" +if run_updater_bad_manifest; then + echo "phase A: invalid manifest must fail" + exit 1 +fi +grep -q 'PBS SBOM generation failed' "$FIX/updater.err" || { echo "phase A: SBOM failure was not reported"; exit 1; } +! grep -q 'unbound variable' "$FIX/updater.err" || { echo "phase A: SBOM failure cleanup used an unset variable"; exit 1; } +grep -q '20990101' python/pbs-sbom.spdx.json || { echo "phase A: SBOM not regenerated for the new release"; exit 1; } +grep -q '"expat"' python/pbs-sbom.spdx.json || { echo "phase A: SBOM missing bundled component"; exit 1; } + +# --- embedded-library drift --------------------------------------------------- +# A binary version mismatch must fail the updater without changing the workspace. +python3 - "$FIX/drift.tar.gz" <<'PYEOF' +import sys, tarfile, io +blob = ( + b"OpenSSL 3.5.7 9 Jun 2026\n" + b"deflate 1.3.3 Copyright 1995-2026 Jean-loup Gailly and Mark Adler\n" + b"expat_2.8.3\n" + b"ncurses 6.5.20240427\n" + b"1.0.8, 13-Jul-2019\n" + b"3.53.1\n5.8.3\n" +) +with tarfile.open(sys.argv[1], "w:gz") as tar: + info = tarfile.TarInfo("python/lib/libpython3.13.so") + info.size = len(blob) + tar.addfile(info, io.BytesIO(blob)) +PYEOF +echo '{"tag": "20990102"}' > release.json +make_sha256sums 20990102 3.13.15 3.14.7 +run_updater_expect_fail || { echo "phase DRIFT: expected the updater to fail"; cat "$FIX/updater.err"; exit 1; } +grep -qi 'drift' "$FIX/updater.err" || { echo "phase DRIFT: missing drift error message"; cat "$FIX/updater.err"; exit 1; } +! grep -q '20990102' private/extensions/python.bzl || { echo "phase DRIFT: workspace must be untouched after a RED"; exit 1; } + +# --- archive path traversal ---------------------------------------------------- +mkdir -p "$FIX/tmp" "$FIX/escape" +python3 - "$FIX/path-traversal.tar.gz" "$(basename "$FIX")" <<'PYEOF' +import sys, tarfile, io + +member = f"python/lib/../../../../../{sys.argv[2]}/escape/libpython3.13.so" +with tarfile.open(sys.argv[1], "w:gz") as tar: + blob = b"not a Python library" + info = tarfile.TarInfo(member) + info.size = len(blob) + tar.addfile(info, io.BytesIO(blob)) +PYEOF +run_updater_path_traversal || { echo "phase PATH: traversal archive must fail"; cat "$FIX/updater.err"; exit 1; } +[ ! -e "$FIX/escape/libpython3.13.so" ] || { echo "phase PATH: archive escaped the extraction directory"; exit 1; } + +# --- patch bump (new release, new patches) ----------------------------------- +snap_b=$(get_python_versions) +echo '{"tag": "20990102"}' > release.json +make_sha256sums 20990102 3.13.16 3.14.8 +[ "$(run_updater)" = "20990102" ] || { echo "phase B: expected update to 20990102"; cat "$FIX/updater.err"; exit 1; } +grep -q '3.13.16+20990102' private/extensions/python.bzl || { echo "phase B: archive version not bumped"; exit 1; } +grep -q '"3.13_amd64": "3.13.16"' private/extensions/python.bzl || { echo "phase B: versions dict not bumped"; exit 1; } +update_test_versions_python "$snap_b" +grep -q 'Python 3.13.16' python/testdata/python3.13.yaml || { echo "phase B: testdata 3.13 not bumped"; exit 1; } +grep -q 'Python 3.14.8' python/testdata/python3.14.yaml || { echo "phase B: testdata 3.14 not bumped"; exit 1; } +grep -q '20990102' python/pbs-sbom.spdx.json || { echo "phase B: SBOM not regenerated"; exit 1; } + +# --- prerelease remains opt-in ------------------------------------------------- +echo '{"tag": "20990103"}' > release.json +make_sha256sums 20990103 3.13.16 3.14.8 3.15.0b4 +[ "$(run_updater)" = "20990103" ] || { echo "phase PRE: expected stable update"; cat "$FIX/updater.err"; exit 1; } +! grep -q '"3.15"' python/config.bzl || { echo "phase PRE: beta must not extend the matrix"; exit 1; } +! grep -q 'python315_' private/extensions/python.bzl || { echo "phase PRE: beta must not enter the extension"; exit 1; } +make_prerelease_only_sha256sums 20990103 +run_updater_required_asset_fail || { echo "phase PRE: missing stable assets must fail"; cat "$FIX/updater.err"; exit 1; } +grep -q 'required matrix entry 3.13 amd64' "$FIX/updater.err" || { echo "phase PRE: missing stable asset error is unclear"; cat "$FIX/updater.err"; exit 1; } + +# --- opted-in prerelease ------------------------------------------------------- +enable_315_without_arches +echo '{"tag": "20990104"}' > release.json +make_sha256sums 20990104 3.13.16 3.14.8 +run_updater_required_asset_fail || { echo "phase OPT-IN: incomplete matrix must fail"; cat "$FIX/updater.err"; exit 1; } +grep -q 'no archs for 3.15' "$FIX/updater.err" || { echo "phase OPT-IN: incomplete matrix error is unclear"; cat "$FIX/updater.err"; exit 1; } +enable_315 +echo '{"tag": "20990104"}' > release.json +make_sha256sums 20990104 3.13.16 3.14.8 +run_updater_required_asset_fail || { echo "phase OPT-IN: missing required asset must fail"; cat "$FIX/updater.err"; exit 1; } +grep -q 'required matrix entry 3.15 amd64' "$FIX/updater.err" || { echo "phase OPT-IN: missing asset error is unclear"; cat "$FIX/updater.err"; exit 1; } +! grep -q '20990104' private/extensions/python.bzl || { echo "phase OPT-IN: failed update changed the extension"; exit 1; } +! grep -q 'python315_' MODULE.bazel || { echo "phase OPT-IN: failed update changed repository visibility"; exit 1; } +make_sha256sums 20990104 3.13.16 3.14.8 3.15.0b4 +[ "$(run_updater)" = "20990104" ] || { echo "phase OPT-IN: expected beta update"; cat "$FIX/updater.err"; exit 1; } +grep -q '3.15.0b4+20990104' private/extensions/python.bzl || { echo "phase OPT-IN: beta archive version wrong"; exit 1; } +make_sha256sums 20990104 3.13.16 3.14.8 3.15.0rc1 3.15.0rc2 +for t in "${TRIPLES[@]}"; do + echo "$SHA_C cpython-3.15.0+20990104-${t}-install_only.tar.gz" >> SHA256SUMS + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-3.15.0+20990104-${t}-install_only.tar.gz" +done +[ "$(run_updater)" = "20990104" ] || { echo "phase OPT-IN: expected prerelease update"; cat "$FIX/updater.err"; exit 1; } +grep -q 'python315_amd64' MODULE.bazel || { echo "phase OPT-IN: selected repos not visible"; exit 1; } +grep -q '3.15.0rc2+20990104' private/extensions/python.bzl || { echo "phase OPT-IN: latest RC archive version wrong"; exit 1; } + +# --- opt-out remains removed --------------------------------------------------- +disable_315 +make_sha256sums 20990104 3.13.16 3.14.8 +[ "$(run_updater)" = "20990104" ] || { echo "phase OPT-OUT: expected cleanup update"; cat "$FIX/updater.err"; exit 1; } +! grep -q 'python315_' MODULE.bazel || { echo "phase OPT-OUT: removed repos were repaired"; exit 1; } +! grep -q 'python315_' private/extensions/python.bzl || { echo "phase OPT-OUT: removed archives were repaired"; exit 1; } + +# --- new stable minor -------------------------------------------------------- +snap_c=$(get_python_versions) +echo '{"tag": "20990105"}' > release.json +make_sha256sums 20990105 3.13.16 3.14.8 3.15.0 +[ "$(run_updater)" = "20990105" ] || { echo "phase C: expected update to 20990105"; cat "$FIX/updater.err"; exit 1; } +grep -q '"3.15"' python/config.bzl || { echo "phase C: 3.15 missing from PYTHON_MAJOR_VERSIONS"; exit 1; } +grep -q '"3.15": \[' python/config.bzl || { echo "phase C: 3.15 arch map missing from config.bzl"; exit 1; } +grep -q 'python315_amd64' MODULE.bazel || { echo "phase C: use_repo not extended with python315 repos"; exit 1; } +grep -q 'python315_amd64' private/extensions/python.bzl || { echo "phase C: 3.15 archive missing from extension"; exit 1; } +grep -q '3.15.0+20990105' private/extensions/python.bzl || { echo "phase C: 3.15 archive version wrong"; exit 1; } +update_test_versions_python "$snap_c" +[ -f python/testdata/python3.15.yaml ] || { echo "phase C: python3.15.yaml not created"; exit 1; } +grep -q 'Python 3.15.0' python/testdata/python3.15.yaml || { echo "phase C: python3.15.yaml version wrong"; exit 1; } + +# --- convergence and consistency --------------------------------------------- +[ "$(run_updater)" = "NO_CHANGE" ] || { echo "final: expected NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } +grep -q '20990105' python/pbs-sbom.spdx.json || { echo "final: SBOM must stay on the last release"; exit 1; } +for minor in $(get_python_minors); do + for arch in $(get_python_archs "$minor"); do + v=$(current_version "$minor" "$arch") + [ -n "$v" ] || { echo "final: no versions-dict entry for ${minor}_${arch}"; exit 1; } + p=$(pinned_version "$minor" "$arch") + [ -n "$p" ] || { echo "final: no archive pin for ${minor}_${arch}"; exit 1; } + case "$p" in + "$v"+[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]) ;; + *) echo "final: pin for ${minor}_${arch} must be '+', got [$p]"; exit 1 ;; + esac + done +done + +# --- numeric stable minor ordering --------------------------------------------- +python3 - <<'PYEOF' +from pathlib import Path + +Path("python/config.bzl").write_text('''"python configurations" + +PYTHON_DISTROS = ["debian13"] +PYTHON_MAJOR_VERSIONS = ["3.9"] +PYTHON_ARCHITECTURES = { + "debian13": { + "3.9": ["amd64", "arm64", "s390x", "riscv64"], + }, +} +PYTHON_PACKAGES = { + "debian13": ["libc-bin"], +} +''') +PYEOF +echo '{"tag": "20990106"}' > release.json +make_numeric_fill_sha256sums 20990106 +[ "$(run_updater)" = "20990106" ] || { echo "phase NUMERIC: expected update to 20990106"; cat "$FIX/updater.err"; exit 1; } +grep -q '"3.10"' python/config.bzl || { echo "phase NUMERIC: 3.10 was not added after 3.9"; exit 1; } +grep -q '3.10.1+20990106' private/extensions/python.bzl || { echo "phase NUMERIC: 3.10 archive missing"; exit 1; } +! grep -q '3.10.1rc1+20990106' private/extensions/python.bzl || { echo "phase NUMERIC: stable minor selected an RC"; exit 1; } + +echo "update_python_archives contract OK (tag-only bump, patch bump, new minor, convergence)" diff --git a/python/update_python_archives_test.sh b/python/update_python_archives_test.sh new file mode 100755 index 000000000..d640a5459 --- /dev/null +++ b/python/update_python_archives_test.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Hermetic smoke test for the Python updater's parser functions. +# Verifies that the matrix and extension remain consistent without network access +# or fixtures. +set -euo pipefail + +cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" + +source knife.d/update_python_archives.sh + +minors=$(get_python_minors | tr '\n' ' ') +[ "$minors" = "3.13 3.14 " ] || { echo "unexpected minors: [$minors]"; exit 1; } + +for minor in $(get_python_minors); do + for arch in $(get_python_archs "$minor"); do + version=$(current_version "$minor" "$arch") + [ -n "$version" ] || { echo "missing version for ${minor}_${arch}"; exit 1; } + # Archive pins include the PBS release tag because change detection uses it. + # A tag-only bump must not be silently ignored. + pinned=$(pinned_version "$minor" "$arch") + [ -n "$pinned" ] || { echo "missing pinned version for ${minor}_${arch}"; exit 1; } + case "$pinned" in + "$version"+[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]) ;; + *) echo "pinned version for ${minor}_${arch} must be '+', got: [$pinned]"; exit 1 ;; + esac + done +done + +echo "update-python-archives parser OK"